@task(name='humfrey.archive.update_dataset_archives', ignore_result=True) def update_dataset_archives(sender, update_definition, store_graphs, when, **kwargs): for store in store_graphs: graph_names = store_graphs[store] endpoint = Endpoint(store.query_endpoint) if DATASET_NOTATION: notation_clause = """ OPTIONAL {{ ?dataset skos:notation ?notation . FILTER (DATATYPE(?notation) = {0} ) }}""".format(DATASET_NOTATION.n3()) else: notation_clause = "" query = """ SELECT ?dataset ?notation WHERE {{ VALUES ?graph {{ {0} }} ?graph void:inDataset ?dataset . {1} }}""".format(" ".join(g.n3() for g in graph_names), notation_clause) datasets = dict(endpoint.query(query)) logger.debug("Found %d datasets to archive", len(datasets)) for dataset in datasets: logger.debug("Archiving dataset: %s", dataset) notation = datasets[dataset] archiver = DatasetArchiver(store, dataset, notation, when) archiver.archive() update_completed.connect(update_dataset_archives.delay)
import datetime from celery.task import task from humfrey.signals import update_completed from .models import Index from .update import IndexUpdater @task(name='humfrey.elasticsearch.update_indexes_after_dataset_update', ignore_result=True) def update_indexes_after_dataset_update(sender, update_definition, store_graphs, when, **kwargs): for index in Index.objects.filter(update_after=update_definition): update_index(index) update_completed.connect(update_indexes_after_dataset_update.delay) @task(name='humfrey.elasticsearch.update_index', ignore_result=True) def update_index(index): if isinstance(index, basestring): index = Index.objects.get(slug=index) index.status = 'active' index.last_started = datetime.datetime.now() index.save() try: index_updater = IndexUpdater() index_updater.update(index) finally: index.status = 'idle' index.last_completed = datetime.datetime.now() index.update_mapping = False index.save()
from celery.task import task from humfrey.signals import update_completed from humfrey.update.models import UpdateDefinition, UpdateDefinitionAlreadyQueued @task(name="humfrey.update.run_dependents") def run_dependents(sender, update_definition, store_graphs, when, **kwargs): """ Runs dataset updates when updates they depend upon have completed. """ update_definitions = UpdateDefinition.objects.filter(depends_on=update_definition) return for update_definition in update_definitions: try: update_definition.queue() except UpdateDefinitionAlreadyQueued: pass update_completed.connect(run_dependents.delay)
from celery.task import task from humfrey.signals import update_completed from humfrey.update.models import UpdateDefinition, UpdateDefinitionAlreadyQueued @task(name='humfrey.update.run_dependents') def run_dependents(sender, update_definition, store_graphs, when, **kwargs): """ Runs dataset updates when updates they depend upon have completed. """ update_definitions = UpdateDefinition.objects.filter(depends_on=update_definition) return for update_definition in update_definitions: try: update_definition.queue() except UpdateDefinitionAlreadyQueued: pass update_completed.connect(run_dependents.delay)