def metafolder(self): if not hasattr(self, '_metafolder'): path = self.config.data.get('path', '.') path = os.path.expandvars(path) path = os.path.expanduser(path) path = os.path.abspath(path) log.info("Saving output to: %r", path) self._metafolder = metafolder.open(path) return self._metafolder
def metafolder(self): if not hasattr(self, '_metafolder'): path = self.config.get('path', '.') path = os.path.expandvars(path) path = os.path.expanduser(path) path = os.path.abspath(path) log.info("Saving output to: %r", path) self._metafolder = metafolder.open(path) return self._metafolder
def crawl(self, folder, source=None): mf = metafolder.open(folder) sources = {} for item in mf: try: self.crawl_item(item, sources, source) except Exception as ex: process.exception(process.INDEX, component=self.name, foreign_id=item.identifier, source_location=folder, meta=item.meta, exception=ex)
def setUp(self): self.mf_path = tempfile.mkdtemp() self.mf = metafolder.open(self.mf_path)
def crawl(self, folder): mf = metafolder.open(folder) self.collections = {} for item in mf: self.crawl_item(item)
def crawl(self, folder, source=None): mf = metafolder.open(folder) self.sources = {} for item in mf: self.crawl_item(item, source)