def bulk_reingest(self, suid_qs):
"""Create IngestJobs and `ingest` tasks for all suids in the given queryset
"""
jobs = self.bulk_schedule(suid_qs, superfluous=True, claim=True)
for job in jobs:
ingest.delay(**self._reingest_kwargs(job))
return jobs
def create(self, request, *args, **kwargs):
serializer = self.get_serializer_class()(data=request.data,
context={
'request': request
})
serializer.is_valid(raise_exception=True)
with transaction.atomic():
# Hack for back-compat: Ingest halfway synchronously, then apply changes asynchronously
ingester = Ingester(serializer.validated_data['data']).as_user(
request.user).ingest(apply_changes=False)
ingester.job.reschedule(claim=True)
nd_id = models.NormalizedData.objects.filter(
raw=ingester.raw,
ingest_jobs=ingester.job).order_by('-created_at').values_list(
'id', flat=True).first()
async_result = ingest.delay(job_id=ingester.job.id, urgent=True)
# TODO Use an actual serializer
return Response(
{
'id': IDObfuscator.encode_id(nd_id, models.NormalizedData),
'type': 'NormalizedData',
'attributes': {
'task':
async_result.id,
'ingest_job':
request.build_absolute_uri(
reverse('api:ingestjob-detail',
args=[IDObfuscator.encode(ingester.job)])),
}
},
status=status.HTTP_202_ACCEPTED)
def ingest_async(self, start_task=True, **kwargs):
# "There's pizza in the fridge."
assert 'job_id' not in kwargs
self._setup_ingest(claim_job=start_task)
if start_task:
self.async_task = ingest.delay(job_id=self.job.id, exhaust=False, **kwargs)
return self
def _bulk_schedule_ingest(self, job, datums):
# HACK to allow scheduling ingest tasks without cyclical imports
from share.tasks import ingest
job_kwargs = {
'source_config': job.source_config,
'source_config_version': job.source_config.version,
'transformer_version': job.source_config.transformer.version,
'regulator_version': Regulator.VERSION,
}
created_jobs = IngestJob.objects.bulk_get_or_create(
[IngestJob(raw_id=datum.id, suid_id=datum.suid_id, **job_kwargs) for datum in datums]
)
if not settings.INGEST_ONLY_CANONICAL_DEFAULT or job.source_config.source.canonical:
for job in created_jobs:
ingest.delay(job_id=job.id)
def ingest_async(self, start_task=True, **kwargs):
# "There's pizza in the fridge."
assert 'job_id' not in kwargs
self._setup_ingest(claim_job=start_task)
if start_task:
self.async_task = ingest.delay(job_id=self.job.id,
exhaust=False,
**kwargs)
return self
def create(self, request, *args, **kwargs):
serializer = self.get_serializer_class()(data=request.data, context={'request': request})
serializer.is_valid(raise_exception=True)
with transaction.atomic():
# Hack for back-compat: Ingest halfway synchronously, then apply changes asynchronously
ingester = Ingester(serializer.validated_data['data']).as_user(request.user).ingest(apply_changes=False)
ingester.job.reschedule(claim=True)
nd_id = models.NormalizedData.objects.filter(
raw=ingester.raw,
ingest_jobs=ingester.job
).order_by('-created_at').values_list('id', flat=True).first()
async_result = ingest.delay(job_id=ingester.job.id, urgent=True)
# TODO Use an actual serializer
return Response({
'id': IDObfuscator.encode_id(nd_id, models.NormalizedData),
'type': 'NormalizedData',
'attributes': {
'task': async_result.id,
'ingest_job': request.build_absolute_uri(reverse('api:ingestjob-detail', args=[IDObfuscator.encode(ingester.job)])),
}
}, status=status.HTTP_202_ACCEPTED)
def reingest_async(self, suid):
"""Create an IngestJob for the given suid, and an `ingest` task assigned to it.
"""
job = self.schedule(suid, superfluous=True, claim=True)
ingest.delay(**self._reingest_kwargs(job))
return job
def restart_tasks(self, request, queryset):
queryset.update(status=AbstractBaseJob.STATUS.created)
for job_id in queryset.values_list('id', flat=True):
ingest.delay(job_id=job_id)