def local_extract(dataset_name, extractor_slug, timeout, retry_failed, skip_existing):
# init storage and loader
ex = get_extractor_cls(extractor_slug)
failed_slug = extractor_slug if retry_failed else None
skip_slug = extractor_slug if skip_existing else None
loader = LocalDatasetLoader(dataset_name,
load_failed=failed_slug,
skip_existing=skip_slug)
storage = LocalResultStorage(dataset_name, ex)
logger.info('started extracting content from %s dataset using %s', dataset_name, ex.NAME)
for doc in loader:
storage.push_result(doc)
if timeout:
time.sleep(timeout)
storage.dump_summary()
logger.info('finished with %s dataset', dataset_name)
