def dispatch_task(self, task):
collection = get_dataset_collection_id(task.job.dataset.name)
if collection is not None:
collection = Collection.by_id(collection, deleted=True)
handler = get_entry_point("aleph.task_handlers", task.stage.stage)
if handler is None:
raise RuntimeError(f"Invalid task handler: {task.stage}")
log.info(f"Task [{task.job.dataset}]: {task.stage.stage} (started)")
handler(collection, task)
log.info(f"Task [{task.job.dataset}]: {task.stage.stage} (done)")
def dispatch_task(self, task):
collection = get_dataset_collection_id(task.job.dataset.name)
if collection is not None:
collection = Collection.by_id(collection, deleted=True)
handler = get_entry_point("aleph.task_handlers", task.stage.stage)
if handler is None:
log.warning(
f"Task handler not found for task [task.job.dataset]: task.stage.stage",
)
return
log.info(f"Task [{task.job.dataset}]: {task.stage.stage} (started)")
handler(collection, task)
log.info(f"Task [{task.job.dataset}]: {task.stage.stage} (done)")
def status():
"""
---
get:
summary: Get an overview of collections and exports being processed
description: >
List collections being processed currently and pending task counts
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SystemStatusResponse'
tags:
- System
"""
require(request.authz.logged_in)
request.rate_limit = None
status = get_active_dataset_status()
datasets = status.pop("datasets", {})
collections = (get_dataset_collection_id(d) for d in datasets.keys())
collections = (c for c in collections if c is not None)
collections = Collection.all_by_ids(collections, deleted=True).all()
collections = {c.id: c for c in collections}
serializer = CollectionSerializer(reference=True)
results = []
for dataset, status in sorted(datasets.items()):
collection_id = get_dataset_collection_id(dataset)
if not request.authz.can(collection_id, request.authz.READ):
continue
collection = collections.get(collection_id)
if collection is not None:
status["collection"] = serializer.serialize(collection.to_dict())
results.append(status)
return jsonify({"results": results, "total": len(results)})
def handle(self, task):
with app.app_context():
collection = get_dataset_collection_id(task.job.dataset.name)
if collection is not None:
collection = Collection.by_id(collection, deleted=True)
self.dispatch_task(collection, task)