def crawl_dir(api: AlephAPI, path: str, foreign_id: str, config: Dict):
"""Crawl a directory and upload its content to a collection
params
------
path: path of the directory
foreign_id: foreign_id of the collection to use.
language: language hint for the documents
"""
_path = Path(path).resolve()
collection = api.load_collection_by_foreign_id(foreign_id, config)
collection_id = collection.get('id')
_queue: Queue = Queue()
_queue.put((_path, None, 1))
threads = []
for i in range(settings.THREADS):
args = (_queue, api, collection_id, _path)
thread = threading.Thread(target=_upload, args=args)
thread.daemon = True
thread.start()
threads.append(thread)
# block until all tasks are done
_queue.join()
for thread in threads:
thread.join()
def crawl_dir(api: AlephAPI,
path: str,
foreign_id: str,
config: Dict,
index: bool = True):
"""Crawl a directory and upload its content to a collection
params
------
path: path of the directory
foreign_id: foreign_id of the collection to use.
language: language hint for the documents
"""
root = Path(path).resolve()
collection = api.load_collection_by_foreign_id(foreign_id, config)
crawler = CrawlDirectory(api, collection, root, index=index)
threads = []
for i in range(settings.THREADS):
thread = threading.Thread(target=crawler.execute)
thread.daemon = True
thread.start()
threads.append(thread)
# block until all tasks are done
crawler.queue.join()
for thread in threads:
thread.join()
def bulk_load(api: AlephAPI, mapping_file: str):
data = load_config_file(mapping_file)
if not isinstance(data, dict):
raise AlephException('mapping_file has to be a json dictionary')
for foreign_id, config in data.items():
collection = api.load_collection_by_foreign_id(foreign_id, config)
collection_id = collection['id']
log.info(f"Bulk mapping collection ID: {collection_id}")
api.map_collection(collection_id, data)
def crawl_dir(
api: AlephAPI,
path: str,
foreign_id: str,
config: Dict,
index: bool = True,
nojunk: bool = False,
parallel: int = 1,
):
"""Crawl a directory and upload its content to a collection
params
------
path: path of the directory
foreign_id: foreign_id of the collection to use.
language: language hint for the documents
"""
root = Path(path).resolve()
collection = api.load_collection_by_foreign_id(foreign_id, config)
crawler = CrawlDirectory(api, collection, root, index=index, nojunk=nojunk)
consumers = []
# Use one thread to produce using scandir and at least one to consume
# files for upload.
producer = threading.Thread(target=crawler.crawl, daemon=True)
producer.start()
for i in range(max(1, parallel)):
consumer = threading.Thread(target=crawler.consume, daemon=True)
consumer.start()
consumers.append(consumer)
# Block until the producer is done with queueing the tree.
producer.join()
# Block until the file upload queue is drained.
crawler.queue.join()
# Poison the queue to signal end to each consumer.
for consumer in consumers:
crawler.queue.put((None, None))
# Block until all file upload queue consumers are done.
for consumer in consumers:
consumer.join()
def load_entities(json_file, root_path):
api = AlephAPI()
collection = api.load_collection_by_foreign_id('zz_occrp_pdi')
cid = collection.get('id')
api.write_entities(cid, generate_entities(json_file, root_path, api, cid))