def test_not_retrying_lost_downloader_jobs(self, mock_nomad,
mock_send_job):
"""Make sure that we don't retry downloader jobs we shouldn't."""
mock_send_job.return_value = True
def mock_init_nomad(host, port=0, timeout=0):
ret_value = MagicMock()
ret_value.job = MagicMock()
ret_value.job.get_job = MagicMock()
ret_value.job.get_job.side_effect = lambda _: {"Status": "pending"}
return ret_value
mock_nomad.side_effect = mock_init_nomad
job = self.create_downloader_job()
job.created_at = timezone.now()
job.save()
# Just run it once, not forever so get the function that is
# decorated with @do_forever
main.retry_lost_downloader_jobs()
self.assertEqual(len(mock_send_job.mock_calls), 0)
jobs = DownloaderJob.objects.order_by('id')
original_job = jobs[0]
self.assertFalse(original_job.retried)
self.assertEqual(original_job.num_retries, 0)
self.assertEqual(original_job.success, None)
# Make sure no additional job was created.
self.assertEqual(jobs.count(), 1)
def test_retrying_lost_downloader_jobs(self, mock_nomad, mock_send_job):
mock_send_job.return_value = True
def mock_init_nomad(host, port=0, timeout=0):
ret_value = MagicMock()
ret_value.job = MagicMock()
ret_value.job.get_job = MagicMock()
ret_value.job.get_job.side_effect = lambda _: {"Status": "dead"}
return ret_value
mock_nomad.side_effect = mock_init_nomad
job = self.create_downloader_job()
job.created_at = timezone.now()
job.save()
# Just run it once, not forever so get the function that is
# decorated with @do_forever
main.retry_lost_downloader_jobs()
self.assertEqual(len(mock_send_job.mock_calls), 1)
jobs = DownloaderJob.objects.order_by('id')
original_job = jobs[0]
self.assertTrue(original_job.retried)
self.assertEqual(original_job.num_retries, 0)
self.assertFalse(original_job.success)
retried_job = jobs[1]
self.assertEqual(retried_job.num_retries, 1)
def test_retrying_lost_downloader_jobs_time(self, mock_send_job):
mock_send_job.return_value = True
job = self.create_downloader_job()
job.created_at = timezone.now() - (main.MIN_LOOP_TIME +
timedelta(minutes=1))
job.save()
# Just run it once, not forever so get the function that is
# decorated with @do_forever
main.retry_lost_downloader_jobs()
self.assertEqual(len(mock_send_job.mock_calls), 1)
jobs = DownloaderJob.objects.order_by('id')
original_job = jobs[0]
self.assertTrue(original_job.retried)
self.assertEqual(original_job.num_retries, 0)
self.assertFalse(original_job.success)
retried_job = jobs[1]
self.assertEqual(retried_job.num_retries, 1)
def test_sra_redownloading(self):
"""Survey, download, then process an experiment we know is SRA."""
# Clear out pre-existing work dirs so there's no conflicts:
self.env = EnvironmentVarGuard()
self.env.set("RUNING_IN_CLOUD", "False")
with self.env:
for work_dir in glob.glob(LOCAL_ROOT_DIR + "/processor_job_*"):
shutil.rmtree(work_dir)
# prevent a call being made to NCBI's API to determine
# organism name/id.
organism = Organism(name="HOMO_SAPIENS",
taxonomy_id=9606,
is_scientific_name=True)
organism.save()
survey_job = surveyor.survey_experiment("SRP040623", "SRA")
self.assertTrue(survey_job.success)
# This experiment has 4 samples that each need a downloader job.
downloader_jobs = DownloaderJob.objects.all()
self.assertEqual(downloader_jobs.count(), 4)
# We want one ProcessorJob to fail because it doesn't have
# the file it was expecting, so we need to wait until one
# DownloaderJob finishes, delete a file that is
# downloaded, and then not delete any more.
logger.info(
"Survey Job finished, waiting for Downloader Jobs to complete."
)
start_time = timezone.now()
file_deleted = False
for downloader_job in downloader_jobs:
# We want to try and delete the file as quickly as
# possible, so pass a short loop time and let the waiting
# loop spin really fast so we lose as little time as
# possible.
downloader_job = wait_for_job(downloader_job, DownloaderJob,
start_time, 0.1)
self.assertTrue(downloader_job.success)
if not file_deleted:
for original_file in OriginalFile.objects.filter(
is_downloaded=True):
if not original_file.is_archive:
original_file.delete_local_file()
file_deleted = True
# And then to make sure that we can handle
# cases where the downloader job is missing:
downloader_job.delete()
break
# There's a chance that the processor job with a missing
# file is aborted before the last downloader job
# completes, therefore just check that there's at least 3
# processor jobs.
processor_jobs = ProcessorJob.objects.all()
self.assertGreater(processor_jobs.count(), 2)
doomed_processor_job = original_file.processor_jobs.all()[0]
logger.info(
"Waiting on processor Nomad job %s to fail because it realized it is missing a file.",
doomed_processor_job.nomad_job_id,
)
start_time = timezone.now()
wait_for_job(doomed_processor_job, ProcessorJob, start_time)
# The processor job that had a missing file will have
# recreated its DownloaderJob, which means there should
# now be 5, but we also deleted on on purpose so there's 4.
downloader_jobs = DownloaderJob.objects.all().order_by("-id")
self.assertEqual(downloader_jobs.count(), 4)
# However DownloaderJobs don't get queued immediately, so
# we have to run a foreman function to make it happen:
retry_lost_downloader_jobs()
# And we can check that the most recently created
# DownloaderJob was successful as well:
recreated_job = downloader_jobs[0]
recreated_job.refresh_from_db()
logger.info("Waiting on downloader Nomad job %s",
recreated_job.nomad_job_id)
recreated_job = wait_for_job(recreated_job, DownloaderJob,
start_time)
self.assertTrue(recreated_job.success)
# Once the Downloader job succeeds, it should create one
# and only one processor job, which the total goes back up to 4:
self.assertEqual(ProcessorJob.objects.all().count(), 4)
# And finally we can make sure that all of the processor
# jobs got started correctly, including the one that got
# recreated. However in order to save time when running
# tests, we don't actually want to run the full salmon
# processor. Therefore we don't have the transcriptome
# index that is needed for this organism so the jobs will
# fail, but that failure happens past the point that we're
# testing.
# So we're gonna check for the correct failure_reason.
logger.info(
"Downloader Jobs finished, waiting for processor Jobs to complete."
)
good_failure_reason = "Missing transcriptome index."
successful_processor_jobs = []
for processor_job in processor_jobs:
# One of the two calls to wait_for_job will fail
# because the job is going to abort when it
# finds that the file it wants to process is missing.
try:
processor_job = wait_for_job(processor_job, ProcessorJob,
start_time)
if not processor_job.success and processor_job.failure_reason.startswith(
good_failure_reason):
successful_processor_jobs.append(processor_job)
except Exception:
pass
self.assertEqual(len(successful_processor_jobs), 4)
def test_transcriptome_redownloading(self, mock_surveyor):
"""Survey, download, then process a transcriptome index. """
mock_surveyor.side_effect = build_surveyor_init_mock(
"TRANSCRIPTOME_INDEX")
# Clear out pre-existing work dirs so there's no conflicts:
self.env = EnvironmentVarGuard()
self.env.set("RUNING_IN_CLOUD", "False")
with self.env:
# I'm not sure why, but sometimes there are already downloader jobs
# in the database from previous tests even though they should be
# removed, so pause a bit
time.sleep(10)
downloader_jobs = DownloaderJob.objects.all()
for job in downloader_jobs:
print(job)
print(job.accession_code)
self.assertEqual(downloader_jobs.count(), 0)
for length in ["LONG", "SHORT"]:
work_dir_glob = (LOCAL_ROOT_DIR + "/Caenorhabditis_elegans/" +
length + "/processor_job_*")
for work_dir in glob.glob(work_dir_glob):
shutil.rmtree(work_dir)
# Prevent a call being made to NCBI's API to determine
# organism name/id.
organism = Organism(name="CAENORHABDITIS_ELEGANS",
taxonomy_id=6239,
is_scientific_name=True)
organism.save()
# Make sure that we can delete the file before the processors begin
# by preventing the downloaders from sending the processors
# automatically. We send the jobs manually later
no_dispatch = EnvironmentVarGuard()
no_dispatch.set("AUTO_DISPATCH_NOMAD_JOBS", "False")
with no_dispatch:
survey_job = surveyor.survey_transcriptome_index(
"Caenorhabditis elegans", "Ensembl")
self.assertTrue(survey_job.success)
downloader_jobs = DownloaderJob.objects.all()
self.assertEqual(downloader_jobs.count(), 1)
logger.info(
"Survey Job finished, waiting for Downloader Job with Nomad ID %s to complete.",
downloader_jobs[0].nomad_job_id,
)
downloader_job = wait_for_job(downloader_jobs[0], DownloaderJob,
timezone.now())
self.assertTrue(downloader_job.success)
og_file_to_delete = OriginalFile.objects.all()[0]
os.remove(og_file_to_delete.absolute_file_path)
processor_jobs = ProcessorJob.objects.all()
for processor_job in processor_jobs:
# FIXME: we run these in serial because of
# https://github.com/AlexsLemonade/refinebio/issues/2321
send_job(
ProcessorPipeline[processor_job.pipeline_applied],
job=processor_job,
is_dispatch=True,
)
try:
wait_for_job(processor_job, ProcessorJob, timezone.now())
except Exception:
pass
# The processor job that had a missing file will have
# recreated its DownloaderJob, which means there should now be two.
downloader_jobs = DownloaderJob.objects.all().order_by("-id")
self.assertEqual(downloader_jobs.count(), 2)
# However DownloaderJobs don't get queued immediately, so
# we have to run a foreman function to make it happen:
retry_lost_downloader_jobs()
# And we can check that the most recently created
# DownloaderJob was successful as well:
recreated_job = downloader_jobs[0]
recreated_job.refresh_from_db()
logger.info("Waiting on downloader Nomad job %s",
recreated_job.nomad_job_id)
recreated_job = wait_for_job(recreated_job, DownloaderJob,
timezone.now())
self.assertTrue(recreated_job.success)
# Once the Downloader job succeeds, it should create two
# processor jobs, one for long and one for short indices.:
processor_jobs = ProcessorJob.objects.all()
self.assertEqual(processor_jobs.count(), 4)
# Wait for the processor jobs to be dispatched
time.sleep(15)
# And finally we can make sure that both of the
# processor jobs were successful, including the one that
# got recreated.
logger.info(
"Downloader Jobs finished, waiting for processor Jobs to complete."
)
successful_processor_jobs = []
for processor_job in processor_jobs:
processor_job.refresh_from_db()
# One of the calls to wait_for_job will fail if the
# job aborts before it we selected all the
# processor jobs.
processor_job = wait_for_job(processor_job, ProcessorJob,
timezone.now())
if processor_job.success:
successful_processor_jobs.append(processor_job)
# While one of the original ProcessorJobs will be aborted
# it is hard to be sure of what will happen
# to the other because of the racing that happens between
# processor jobs getting started and us deleting the files
# they need.
# Therefore, we're just going to verify that one processor
# job completed successfully for each length, since that
# is the main thing we need.
has_long = False
has_short = False
for processor_job in successful_processor_jobs:
if processor_job.pipeline_applied == "TRANSCRIPTOME_INDEX_LONG":
has_long = True
elif processor_job.pipeline_applied == "TRANSCRIPTOME_INDEX_SHORT":
has_short = True
self.assertTrue(has_long)
self.assertTrue(has_short)
def test_geo_celgz_redownloading(self):
"""Survey, download, then process an experiment we know is Affymetrix.
Each of the experiment's samples are in their own .cel.gz
file, which is another way we expect GEO data to come.
This is another test which uses Aspera so it unfortunately
cannot be made to run without relying on NCBI's aspera server.
"""
self.env = EnvironmentVarGuard()
self.env.set("RUNING_IN_CLOUD", "False")
with self.env:
# Clear out pre-existing work dirs so there's no conflicts:
for work_dir in glob.glob(LOCAL_ROOT_DIR + "/processor_job_*"):
shutil.rmtree(work_dir)
# Prevent a call being made to NCBI's API to determine
# organism name/id.
organism = Organism(name="MUS_MUSCULUS",
taxonomy_id=10090,
is_scientific_name=True)
organism.save()
accession_code = "GSE100388"
survey_job = surveyor.survey_experiment(accession_code, "GEO")
SAMPLES_IN_EXPERIMENT = 15
self.assertTrue(survey_job.success)
# This experiment's samples each have their own file so
# they each get their own downloader job.
downloader_jobs = DownloaderJob.objects.all()
self.assertEqual(downloader_jobs.count(), SAMPLES_IN_EXPERIMENT)
logger.info(
"Survey Job finished, waiting for Downloader Jobs to complete."
)
# We're going to spin as fast as we can so we can delete
# the file in between when the downloader jobs finishes and
# the processor job starts.
start_time = timezone.now()
file_deleted = False
while not file_deleted and timezone.now(
) - start_time < MAX_WAIT_TIME:
non_archive_files = OriginalFile.objects.filter(
is_archive=False)
for original_file in non_archive_files:
if original_file.absolute_file_path and os.path.exists(
original_file.absolute_file_path):
os.remove(original_file.absolute_file_path)
file_deleted = True
break
# Wait for each of the DownloaderJobs to finish
for downloader_job in downloader_jobs:
downloader_job = wait_for_job(downloader_job, DownloaderJob,
start_time)
self.assertTrue(downloader_job.success)
try:
doomed_processor_job = original_file.processor_jobs.all()[0]
except Exception:
# The doomed job may be aborted before we can get
# it. This is fine, we just can't look at it.
doomed_processor_job = None
if doomed_processor_job:
logger.info(
"Waiting on processor Nomad job %s to fail because it realized it is missing a file.",
doomed_processor_job.nomad_job_id,
)
start_time = timezone.now()
doomed_processor_job = wait_for_job(doomed_processor_job,
ProcessorJob, start_time)
self.assertTrue(doomed_processor_job.abort)
# The processor job that had a missing file will have
# recreated its DownloaderJob, which means there should
# now be SAMPLES_IN_EXPERIMENT + 1 downloader jobs.
downloader_jobs = DownloaderJob.objects.all().order_by("-id")
self.assertEqual(downloader_jobs.count(),
SAMPLES_IN_EXPERIMENT + 1)
# However DownloaderJobs don't get queued immediately, so
# we have to run a foreman function to make it happen:
retry_lost_downloader_jobs()
# And we can check that the most recently created
# DownloaderJob was successful as well:
recreated_job = downloader_jobs[0]
recreated_job.refresh_from_db()
logger.info("Waiting on downloader Nomad job %s",
recreated_job.nomad_job_id)
recreated_job = wait_for_job(recreated_job, DownloaderJob,
start_time)
self.assertTrue(recreated_job.success)
# And finally we can make sure that all of the processor
# jobs were successful, including the one that got
# recreated. The processor job that recreated that job has
# abort=True
logger.info(
"Downloader Jobs finished, waiting for processor Jobs to complete."
)
processor_jobs = ProcessorJob.objects.all().exclude(
abort=True) # exclude aborted jobs
for processor_job in processor_jobs:
processor_job = wait_for_job(processor_job, ProcessorJob,
start_time)
self.assertTrue(processor_job.success)
self.assertEqual(processor_jobs.count(), SAMPLES_IN_EXPERIMENT)
def test_geo_archive_redownloading(self):
"""Survey, download, then process an experiment we know is NO_OP.
All the data for the experiment are in the same archive, which
is one of ways we expect GEO data to come.
This is another test which uses Aspera so it unfortunately
cannot be made to run without relying on NCBI's aspera server.
"""
# Clear out pre-existing work dirs so there's no conflicts:
self.env = EnvironmentVarGuard()
self.env.set("RUNING_IN_CLOUD", "False")
with self.env:
for work_dir in glob.glob(LOCAL_ROOT_DIR + "/processor_job_*"):
shutil.rmtree(work_dir)
# Prevent a call being made to NCBI's API to determine
# organism name/id.
organism = Organism(name="HOMO_SAPIENS",
taxonomy_id=9606,
is_scientific_name=True)
organism.save()
accession_code = "GSE102571"
survey_job = surveyor.survey_experiment(accession_code, "GEO")
self.assertTrue(survey_job.success)
# This experiment has multiple samples that are contained in the
# same archive, so only one job is needed.
downloader_jobs = DownloaderJob.objects.all()
self.assertEqual(downloader_jobs.count(), 1)
logger.info(
"Survey Job finished, waiting for Downloader Job with Nomad ID %s to complete.",
downloader_jobs[0].nomad_job_id,
)
# We're going to spin as fast as we can so we can delete
# the file in between when the downloader job finishes and
# the processor job starts.
start_time = timezone.now()
file_deleted = False
while not file_deleted and timezone.now(
) - start_time < MAX_WAIT_TIME:
non_archive_files = OriginalFile.objects.filter(
is_archive=False)
for original_file in non_archive_files:
if original_file.absolute_file_path and os.path.exists(
original_file.absolute_file_path):
os.remove(original_file.absolute_file_path)
file_deleted = True
break
downloader_job = wait_for_job(downloader_jobs[0], DownloaderJob,
start_time)
self.assertTrue(downloader_job.success)
try:
doomed_processor_job = original_file.processor_jobs.all()[0]
except Exception:
# The doomed job may be aborted before we can get
# it. This is fine, we just can't look at it.
doomed_processor_job = None
if doomed_processor_job:
logger.info(
"Waiting on processor Nomad job %s to fail because it realized it is missing a file.",
doomed_processor_job.nomad_job_id,
)
start_time = timezone.now()
doomed_processor_job = wait_for_job(doomed_processor_job,
ProcessorJob, start_time)
self.assertTrue(doomed_processor_job.abort)
# The processor job that had a missing file will have
# recreated its DownloaderJob, which means there should now be two.
downloader_jobs = DownloaderJob.objects.all().order_by("-id")
self.assertEqual(downloader_jobs.count(), 2)
# However DownloaderJobs don't get queued immediately, so
# we have to run a foreman function to make it happen:
retry_lost_downloader_jobs()
# And we can check that the most recently created
# DownloaderJob was successful as well:
recreated_job = downloader_jobs[0]
recreated_job.refresh_from_db()
logger.info("Waiting on downloader Nomad job %s",
recreated_job.nomad_job_id)
recreated_job = wait_for_job(recreated_job, DownloaderJob,
start_time)
self.assertTrue(recreated_job.success)
# And finally we can make sure that all of the
# processor jobs were successful, including the one that
# got recreated.
logger.info(
"Downloader Jobs finished, waiting for processor Jobs to complete."
)
processor_jobs = ProcessorJob.objects.all().exclude(
abort=True) # exclude aborted processor jobs
for processor_job in processor_jobs:
processor_job = wait_for_job(processor_job, ProcessorJob,
start_time)
if not processor_job.success:
logger.error(processor_job.failure_reason)
self.assertTrue(processor_job.success)
# Apparently this experiment has a variable number of
# files because GEO processed experiments sometimes do...
# However this is okay because there's at least one file
# per sample, so each sample will get processed at least
# once and it's the best we can do with the state of GEO.
# Anyway, all of that is an explanation for why we count
# how many samples there are rather than just expecting
# how many we know the experiment has.
self.assertEqual(processor_jobs.count(),
Sample.objects.all().count())
def test_array_express_redownloading(self, mock_surveyor):
"""Survey, download, then process an experiment we know is NO_OP."""
mock_surveyor.side_effect = build_surveyor_init_mock("ARRAY_EXPRESS")
# Clear out pre-existing work dirs so there's no conflicts:
self.env = EnvironmentVarGuard()
self.env.set("RUNING_IN_CLOUD", "False")
with self.env:
for work_dir in glob.glob(LOCAL_ROOT_DIR + "/processor_job_*"):
shutil.rmtree(work_dir)
# Prevent a call being made to NCBI's API to determine
# organism name/id.
organism = Organism(name="HOMO_SAPIENS",
taxonomy_id=9606,
is_scientific_name=True)
organism.save()
NUM_SAMPLES_IN_EXPERIMENT = 12
accession_code = "E-GEOD-3303"
survey_job = surveyor.survey_experiment(accession_code,
"ARRAY_EXPRESS")
self.assertTrue(survey_job.success)
# All of this experiment's samples are contained in the
# same archive, so only one job is needed.
downloader_jobs = DownloaderJob.objects.all()
self.assertEqual(downloader_jobs.count(), 1)
logger.info(
"Survey Job finished, waiting for Downloader Jobs to complete."
)
start_time = timezone.now()
# We want to try and delete the file as quickly as
# possible, so pass a short loop time and let the waiting
# loop spin really fast so we lose as little time as
# possible.
downloader_job = wait_for_job(downloader_jobs[0], DownloaderJob,
start_time, 0.1)
self.assertTrue(downloader_job.success)
# Now we're going to delete one of the extracted files but not the other.
deleted_file = OriginalFile.objects.filter(
is_archive=False).first()
self.assertIsNotNone(deleted_file)
deleted_file.delete_local_file()
# The one downloader job should have extracted all the files
# and created as many processor jobs.
processor_jobs = ProcessorJob.objects.all()
self.assertEqual(processor_jobs.count(), NUM_SAMPLES_IN_EXPERIMENT)
doomed_processor_job = deleted_file.processor_jobs.all()[0]
logger.info(
"Waiting on processor Nomad job %s to fail because it realized it is missing a file.",
doomed_processor_job.nomad_job_id,
)
start_time = timezone.now()
doomed_processor_job = wait_for_job(doomed_processor_job,
ProcessorJob, start_time)
self.assertTrue(doomed_processor_job.abort)
# The processor job that had a missing file will have
# recreated its DownloaderJob, which means there should now be two.
downloader_jobs = DownloaderJob.objects.all().order_by("-id")
self.assertEqual(downloader_jobs.count(), 2)
# However DownloaderJobs don't get queued immediately, so
# we have to run a foreman function to make it happen:
retry_lost_downloader_jobs()
# And we can check that the most recently created
# DownloaderJob was successful as well:
recreated_job = downloader_jobs[0]
recreated_job.refresh_from_db()
logger.info("Waiting on downloader Nomad job %s",
recreated_job.nomad_job_id)
recreated_job = wait_for_job(recreated_job, DownloaderJob,
start_time)
self.assertTrue(recreated_job.success)
# Once the Downloader job succeeds, it should create one
# and only one processor job, after which the total goes back up
# to NUM_SAMPLES_IN_EXPERIMENT:
processor_jobs = ProcessorJob.objects.all().exclude(
abort=True) # exclude aborted processor jobs
logger.error(processor_jobs)
self.assertEqual(processor_jobs.count(), NUM_SAMPLES_IN_EXPERIMENT)
# And finally we can make sure that all of the
# processor jobs were successful, including the one that
# got recreated.
logger.info(
"Downloader Jobs finished, waiting for processor Jobs to complete."
)
for processor_job in processor_jobs:
processor_job = wait_for_job(processor_job, ProcessorJob,
start_time)
self.assertTrue(processor_job.success)
def test_transcriptome_redownloading(self):
"""Survey, download, then process a transcriptome index."""
# Clear out pre-existing work dirs so there's no conflicts:
self.env = EnvironmentVarGuard()
self.env.set('RUNING_IN_CLOUD', 'False')
with self.env:
for length in ["LONG", "SHORT"]:
work_dir_glob = LOCAL_ROOT_DIR + "/Caenorhabditis_elegans/" + length + "/processor_job_*"
for work_dir in glob.glob(work_dir_glob):
shutil.rmtree(work_dir)
# Prevent a call being made to NCBI's API to determine
# organism name/id.
organism = Organism(name="HOMO_SAPIENS",
taxonomy_id=9606,
is_scientific_name=True)
organism.save()
survey_job = surveyor.survey_transcriptome_index(
"Caenorhabditis elegans", "Ensembl")
self.assertTrue(survey_job.success)
downloader_jobs = DownloaderJob.objects.all()
self.assertEqual(downloader_jobs.count(), 1)
logger.info(
"Survey Job finished, waiting for Downloader Job with Nomad ID %s to complete.",
downloader_jobs[0].nomad_job_id)
og_file_to_delete = OriginalFile.objects.all()[0]
start_time = timezone.now()
# We're going to spin as fast as we can so we can delete
# the file in between when the downloader job finishes and
# the processor job starts.
while timezone.now() - start_time < MAX_WAIT_TIME:
og_file_to_delete.refresh_from_db()
if og_file_to_delete.absolute_file_path and os.path.exists(
og_file_to_delete.absolute_file_path):
os.remove(og_file_to_delete.absolute_file_path)
break
# We want to try and delete the file as quickly as
# possible, so pass a short loop time and let the waiting
# loop spin really fast so we lose as little time as
# possible.
downloader_job = wait_for_job(downloader_jobs[0], DownloaderJob,
start_time)
self.assertTrue(downloader_job.success)
start_time = timezone.now()
processor_jobs = ProcessorJob.objects.all()
for processor_job in processor_jobs:
# It's hard to guarantee that we'll be able to delete
# the files before the first job starts, but since
# they both don't start at the same time we'll
# definitely get it before the second one. This is
# actually kinda desirable for testing though because
# we should be able to handle it either way.
try:
wait_for_job(processor_job, ProcessorJob, start_time)
except:
pass
# The processor job that had a missing file will have
# recreated its DownloaderJob, which means there should now be two.
downloader_jobs = DownloaderJob.objects.all().order_by('-id')
self.assertEqual(downloader_jobs.count(), 2)
# However DownloaderJobs don't get queued immediately, so
# we have to run a foreman function to make it happen:
retry_lost_downloader_jobs()
# And we can check that the most recently created
# DownloaderJob was successful as well:
recreated_job = downloader_jobs[0]
recreated_job.refresh_from_db()
logger.info("Waiting on downloader Nomad job %s",
recreated_job.nomad_job_id)
recreated_job = wait_for_job(recreated_job, DownloaderJob,
start_time)
self.assertTrue(recreated_job.success)
# Once the Downloader job succeeds, it should create one
# and only one processor job, which the total goes back up to 2:
processor_jobs = ProcessorJob.objects.all()
self.assertEqual(processor_jobs.count(), 3)
# And finally we can make sure that both of the
# processor jobs were successful, including the one that
# got recreated.
logger.info(
"Downloader Jobs finished, waiting for processor Jobs to complete."
)
successful_processor_jobs = []
for processor_job in processor_jobs:
# One of the calls to wait_for_job will fail if the
# job deletes itself before it we selected all the
# processor jobs.
try:
processor_job = wait_for_job(processor_job, ProcessorJob,
start_time)
if processor_job.success:
successful_processor_jobs.append(processor_job)
except:
pass
# While one of the original ProcessorJobs will definitely
# delete itself, it is hard to be sure of what will happen
# to the other because of the racing that happens between
# processor jobs getting started and us deleting the files
# they need.
# Therefore, we're just going to verify that one processor
# job completed successfully for each length, since that
# is the main thing we need.
has_long = False
has_short = False
for processor_job in successful_processor_jobs:
if processor_job.pipeline_applied == "TRANSCRIPTOME_INDEX_LONG":
has_long = True
elif processor_job.pipeline_applied == "TRANSCRIPTOME_INDEX_SHORT":
has_short = True
self.assertTrue(has_long)
self.assertTrue(has_short)
