def ingest(self, something, meta=None, overwrite=True): """ Import a given object into the package as a source. The object can be either a URL, a file or folder name, an open file handle or a HTTP returned object from urllib, urllib2 or requests. If ``overwrite`` is ``False``, the source file will be renamed until the name is not taken. """ ingestors = list(Ingestor.analyze(something)) if len(ingestors) != 1: raise ValueError("Can't ingest: %r" % something) ingestor = ingestors[0] try: meta = ingestor.generate_meta(meta) name = None for i in count(1): suffix = '-%s' % i if i > 1 else '' name = '%s%s.%s' % (meta['slug'], suffix, meta['extension']) if overwrite or not self.has(Source, name): break source = Source(self, name) source.meta.update(meta) ingestor.store(source) self.save() return source finally: ingestor.dispose()
def ingest(collection, something, package_id=None, meta={}): for ingestor in Ingestor.analyze(something): try: if package_id is None: package_id = ingestor.hash() package = collection.get(package_id) package.ingest(ingestor, meta=meta) process_package.delay(collection.name, package.id) except Exception, e: log.exception(e) finally:
def ingest(self, something, meta=None): """ Import a given object into the collection. The object can be either a URL, a file or folder name, an open file handle or a HTTP returned object from urllib, urllib2 or requests. Before importing it, a SHA1 hash will be generated and used as the package ID. If a package with the given name already exists, it will be overwritten. If you do not desire SHA1 de-duplication, create a package directly and ingest from there. """ for ingestor in Ingestor.analyze(something): try: package = self.get(ingestor.hash()) package.ingest(ingestor, meta=meta) finally: ingestor.dispose()