def requeue_downloader_job(last_job: DownloaderJob) -> None:
"""Queues a new downloader job.
The new downloader job will have num_retries one greater than
last_job.num_retries.
"""
num_retries = last_job.num_retries + 1
new_job = DownloaderJob(num_retries=num_retries,
downloader_task=last_job.downloader_task,
accession_code=last_job.accession_code)
new_job.save()
for original_file in last_job.original_files.all():
DownloaderJobOriginalFileAssociation.objects.get_or_create(
downloader_job=new_job, original_file=original_file)
logger.debug(
"Requeuing Downloader Job which had ID %d with a new Downloader Job with ID %d.",
last_job.id, new_job.id)
try:
if send_job(Downloaders[last_job.downloader_task],
job=new_job,
is_dispatch=True):
last_job.retried = True
last_job.success = False
last_job.retried_job = new_job
last_job.save()
else:
# Can't communicate with nomad just now, leave the job for a later loop.
new_job.delete()
except:
logger.error(
"Failed to requeue Downloader Job which had ID %d with a new Downloader Job with ID %d.",
last_job.id, new_job.id)
# Can't communicate with nomad just now, leave the job for a later loop.
new_job.delete()
def requeue_downloader_job(last_job: DownloaderJob) -> (bool, str):
"""Queues a new downloader job.
The new downloader job will have num_retries one greater than
last_job.num_retries.
Returns True and the volume index of the downloader job upon successful dispatching,
False and an empty string otherwise.
"""
num_retries = last_job.num_retries + 1
ram_amount = last_job.ram_amount
# If there's no start time then it's likely that the instance got
# cycled which means we didn't get OOM-killed, so we don't need to
# increase the RAM amount.
if last_job.start_time and last_job.failure_reason is None:
if ram_amount == 1024:
ram_amount = 4096
elif ram_amount == 4096:
ram_amount = 16384
original_file = last_job.original_files.first()
if not original_file:
last_job.no_retry = True
last_job.success = False
last_job.failure_reason = (
"Foreman told to requeue a DownloaderJob without an OriginalFile - why?!"
)
last_job.save()
logger.info(
"Foreman told to requeue a DownloaderJob without an OriginalFile - why?!",
last_job=str(last_job),
)
return False
if not original_file.needs_processing():
last_job.no_retry = True
last_job.success = False
last_job.failure_reason = "Foreman told to redownload job with prior successful processing."
last_job.save()
logger.info(
"Foreman told to redownload job with prior successful processing.",
last_job=str(last_job),
)
return False
first_sample = original_file.samples.first()
# This is a magic string that all the dbGaP studies appear to have
if first_sample and ("in the dbGaP study" in first_sample.title):
last_job.no_retry = True
last_job.success = False
last_job.failure_reason = "Sample is dbGaP access controlled."
last_job.save()
logger.info(
"Avoiding requeuing for DownloaderJob for dbGaP run accession: " +
str(first_sample.accession_code))
return False
new_job = DownloaderJob(
num_retries=num_retries,
downloader_task=last_job.downloader_task,
ram_amount=ram_amount,
accession_code=last_job.accession_code,
was_recreated=last_job.was_recreated,
)
new_job.save()
for original_file in last_job.original_files.all():
DownloaderJobOriginalFileAssociation.objects.get_or_create(
downloader_job=new_job, original_file=original_file)
logger.debug(
"Requeuing Downloader Job which had ID %d with a new Downloader Job with ID %d.",
last_job.id,
new_job.id,
)
try:
if send_job(Downloaders[last_job.downloader_task],
job=new_job,
is_dispatch=True):
last_job.retried = True
last_job.success = False
last_job.retried_job = new_job
last_job.save()
else:
# Can't communicate with Batch just now, leave the job for a later loop.
new_job.delete()
return False
except Exception:
logger.error(
"Failed to requeue DownloaderJob which had ID %d with a new DownloaderJob with ID %d.",
last_job.id,
new_job.id,
)
# Can't communicate with Batch just now, leave the job for a later loop.
new_job.delete()
return False
return True