def _try_job(self, job, hacky_merger):
for i in range(MAX_RETRIES):
self.stdout.write('Attempt {} of {}:'.format(i + 1, MAX_RETRIES))
try:
IngestJobConsumer().consume(job_id=job.id, exhaust=False)
except MergeRequired as e:
(_, *dupe_sets) = e.args
try:
for dupes in dupe_sets:
hacky_merger.merge(dupes)
except RejectMerge:
self.stdout.write('Skipping job...',
style_func=self.style.WARNING)
return
except CannotMerge as e:
self.stdout.write('Failed to merge:',
style_func=self.style.ERROR)
self.stdout.write('\t{!r}'.format(e))
return
if hacky_merger.dry_run:
return
continue
except Exception as e:
self.stdout.write('Failed in a way we cannot fix:',
style_func=self.style.ERROR)
self.stdout.write('\t{!r}'.format(e))
return
self.stdout.write('Success!', style_func=self.style.SUCCESS)
return
self.stdout.write('Failed to fix after {} tries'.format(MAX_RETRIES),
style_func=self.style.ERROR)
def ingest(self, **kwargs):
# "Here comes the airplane!"
assert 'job_id' not in kwargs
self._setup_ingest(claim_job=True)
IngestJobConsumer().consume(job_id=self.job.id,
exhaust=False,
**kwargs)
return self
def ingest(self, only_canonical=None, **kwargs):
"""Ingest the data of the given IngestJob or the next available IngestJob.
Keyword arguments from JobConsumer.consume
"""
if only_canonical is None:
only_canonical = settings.INGEST_ONLY_CANONICAL_DEFAULT
IngestJobConsumer(task=self,
only_canonical=only_canonical).consume(**kwargs)
def _try_job(self, job, dry_run):
for i in range(self.MAX_RETRIES):
self.stdout.write('Attempt {} of {}:'.format(i + 1, self.MAX_RETRIES))
try:
IngestJobConsumer().consume(job_id=job.id, exhaust=False)
except MergeRequired as e:
(_, model, queries) = e.args
if not self._fix_merge_error(model, queries, dry_run):
return
if dry_run:
return
continue
except Exception as e:
self.stdout.write('Failed in a way we cant fix:', style_func=self.style.ERROR)
self.stdout.write(str(e))
return
self.stdout.write('Success!', style_func=self.style.SUCCESS)
return
self.stdout.write('Failed to fix after {} tries'.format(self.MAX_RETRIES), style_func=self.style.ERROR)
def ingest(self, **kwargs):
"""Ingest the data of the given IngestJob or the next available IngestJob.
Keyword arguments from JobConsumer.consume
"""
IngestJobConsumer(task=self).consume(**kwargs)
def reingest(self, suid):
"""Synchronously reingest the given suid.
"""
job = self.schedule(suid, superfluous=True, claim=True)
IngestJobConsumer().consume(**self._reingest_kwargs(job))
return job