@task(name='humfrey.archive.update_dataset_archives', ignore_result=True)
def update_dataset_archives(sender, update_definition, store_graphs, when, **kwargs):
for store in store_graphs:
graph_names = store_graphs[store]
endpoint = Endpoint(store.query_endpoint)
if DATASET_NOTATION:
notation_clause = """
OPTIONAL {{ ?dataset skos:notation ?notation .
FILTER (DATATYPE(?notation) = {0} ) }}""".format(DATASET_NOTATION.n3())
else:
notation_clause = ""
query = """
SELECT ?dataset ?notation WHERE {{
VALUES ?graph {{ {0} }}
?graph void:inDataset ?dataset .
{1}
}}""".format(" ".join(g.n3() for g in graph_names),
notation_clause)
datasets = dict(endpoint.query(query))
logger.debug("Found %d datasets to archive", len(datasets))
for dataset in datasets:
logger.debug("Archiving dataset: %s", dataset)
notation = datasets[dataset]
archiver = DatasetArchiver(store, dataset, notation, when)
archiver.archive()
update_completed.connect(update_dataset_archives.delay)
import datetime
from celery.task import task
from humfrey.signals import update_completed
from .models import Index
from .update import IndexUpdater
@task(name='humfrey.elasticsearch.update_indexes_after_dataset_update', ignore_result=True)
def update_indexes_after_dataset_update(sender, update_definition, store_graphs, when, **kwargs):
for index in Index.objects.filter(update_after=update_definition):
update_index(index)
update_completed.connect(update_indexes_after_dataset_update.delay)
@task(name='humfrey.elasticsearch.update_index', ignore_result=True)
def update_index(index):
if isinstance(index, basestring):
index = Index.objects.get(slug=index)
index.status = 'active'
index.last_started = datetime.datetime.now()
index.save()
try:
index_updater = IndexUpdater()
index_updater.update(index)
finally:
index.status = 'idle'
index.last_completed = datetime.datetime.now()
index.update_mapping = False
index.save()
from celery.task import task
from humfrey.signals import update_completed
from humfrey.update.models import UpdateDefinition, UpdateDefinitionAlreadyQueued
@task(name="humfrey.update.run_dependents")
def run_dependents(sender, update_definition, store_graphs, when, **kwargs):
"""
Runs dataset updates when updates they depend upon have completed.
"""
update_definitions = UpdateDefinition.objects.filter(depends_on=update_definition)
return
for update_definition in update_definitions:
try:
update_definition.queue()
except UpdateDefinitionAlreadyQueued:
pass
update_completed.connect(run_dependents.delay)
from celery.task import task
from humfrey.signals import update_completed
from humfrey.update.models import UpdateDefinition, UpdateDefinitionAlreadyQueued
@task(name='humfrey.update.run_dependents')
def run_dependents(sender, update_definition, store_graphs, when, **kwargs):
"""
Runs dataset updates when updates they depend upon have completed.
"""
update_definitions = UpdateDefinition.objects.filter(depends_on=update_definition)
return
for update_definition in update_definitions:
try:
update_definition.queue()
except UpdateDefinitionAlreadyQueued:
pass
update_completed.connect(run_dependents.delay)
