def get_context(collection, pipeline):
"""Set some task context variables that configure the ingestors."""
from aleph.logic.aggregator import get_aggregator_name
return {
"languages": collection.languages,
"ftmstore": get_aggregator_name(collection),
"namespace": collection.foreign_id,
"pipeline": pipeline,
}
def ingest_entity(collection, proxy):
"""Send the given FtM entity proxy to the ingest-file service."""
if not proxy.schema.is_a(Document.SCHEMA):
return
log.debug("Ingest entity [%s]: %s", proxy.id, proxy.caption)
queue = get_queue(collection, OP_INGEST)
from aleph.logic.aggregator import get_aggregator_name
context = {
'languages': collection.languages,
'balkhash_name': get_aggregator_name(collection)
}
queue.queue_task(proxy.to_dict(), context)
def ingest_entity(collection, proxy, job_id=None, sync=False):
"""Send the given FtM entity proxy to the ingest-file service."""
log.debug("Ingest entity [%s]: %s", proxy.id, proxy.caption)
stage = get_stage(collection, OP_INGEST, job_id=job_id)
from aleph.logic.aggregator import get_aggregator_name
context = {
'languages': collection.languages,
'balkhash_name': get_aggregator_name(collection),
'next_stage': OP_INDEX,
'sync': sync
}
stage.queue(proxy.to_dict(), context)
def ingest_entity(collection, proxy, job_id=None, index=True):
"""Send the given FtM entity proxy to the ingest-file service."""
from aleph.logic.aggregator import get_aggregator_name
log.debug("Ingest entity [%s]: %s", proxy.id, proxy.caption)
stage = get_stage(collection, OP_INGEST, job_id=job_id)
pipeline = [OP_ANALYZE, OP_INDEX] if index else [OP_ANALYZE]
context = {
'languages': collection.languages,
'ftmstore': get_aggregator_name(collection),
'namespace': collection.foreign_id,
'pipeline': pipeline
}
stage.queue(proxy.to_dict(), context)
def ingest_entity(collection, proxy, job_id=None, index=True):
"""Send the given FtM entity proxy to the ingest-file service."""
from aleph.logic.aggregator import get_aggregator_name
log.debug("Ingest entity [%s]: %s", proxy.id, proxy.caption)
stage = get_stage(collection, OP_INGEST, job_id=job_id)
pipeline = list(settings.INGEST_PIPELINE)
if index:
pipeline.append(OP_INDEX)
context = {
"languages": collection.languages,
"ftmstore": get_aggregator_name(collection),
"namespace": collection.foreign_id,
"pipeline": pipeline,
}
stage.queue(proxy.to_dict(), context)