def metafolder(self):
if not hasattr(self, '_metafolder'):
path = self.config.data.get('path', '.')
path = os.path.expandvars(path)
path = os.path.expanduser(path)
path = os.path.abspath(path)
log.info("Saving output to: %r", path)
self._metafolder = metafolder.open(path)
return self._metafolder
def metafolder(self):
if not hasattr(self, '_metafolder'):
path = self.config.get('path', '.')
path = os.path.expandvars(path)
path = os.path.expanduser(path)
path = os.path.abspath(path)
log.info("Saving output to: %r", path)
self._metafolder = metafolder.open(path)
return self._metafolder
def crawl(self, folder, source=None):
mf = metafolder.open(folder)
sources = {}
for item in mf:
try:
self.crawl_item(item, sources, source)
except Exception as ex:
process.exception(process.INDEX, component=self.name,
foreign_id=item.identifier,
source_location=folder, meta=item.meta,
exception=ex)
def crawl(self, folder, source=None):
mf = metafolder.open(folder)
sources = {}
for item in mf:
try:
self.crawl_item(item, sources, source)
except Exception as ex:
process.exception(process.INDEX,
component=self.name,
foreign_id=item.identifier,
source_location=folder,
meta=item.meta,
exception=ex)
def setUp(self):
self.mf_path = tempfile.mkdtemp()
self.mf = metafolder.open(self.mf_path)
def crawl(self, folder):
mf = metafolder.open(folder)
self.collections = {}
for item in mf:
self.crawl_item(item)
def crawl(self, folder, source=None):
mf = metafolder.open(folder)
self.sources = {}
for item in mf:
self.crawl_item(item, source)
def crawl(self, folder):
mf = metafolder.open(folder)
self.collections = {}
for item in mf:
self.crawl_item(item)
