def test_bad_restart(self):
with self.settings(RUNNING_IN_CLOUD=True):
job = ProcessorJob()
job.start_time = timezone.now()
job.success = True
job.save()
job_context = utils.start_job({"job": job})
job = ProcessorJob()
job.start_time = timezone.now()
job.success = False
job.save()
job_context = utils.start_job({"job": job})
self.assertRaises(utils.start_job({"job": job}))
def test_run_salmon_failure(self):
batch, first_fastq_file, second_fastq_file = init_objects()
processor_job = ProcessorJob.create_job_and_relationships(
batches=[batch])
# Mock out the job_context with everything the function under
# test will expect
input_file_path_1 = first_fastq_file.get_temp_pre_path("dummy")
input_file_path_2 = second_fastq_file.get_temp_pre_path("dummy")
job_context = utils.start_job({
"job": processor_job,
"job_id": processor_job.id,
"job_dir_prefix": "dummy",
"batches": [batch],
"index_directory": "missing",
"input_file_path": input_file_path_1,
"input_file_path_2": input_file_path_2,
"output_directory": "blah"
})
# The function being tested.
job_context = salmon._run_salmon(job_context)
self.assertFalse(job_context["success"])
self.assertNotEqual(processor_job.failure_reason, None)
self.assertFalse(os.path.isfile(batch.files[0].get_temp_pre_path()))
def test_failure(self):
"""Fails because there are no files for the job."""
processor_job = ProcessorJob()
processor_job.save()
job_context = utils.start_job({"job": processor_job})
self.assertFalse(job_context["success"])
def test_zip_index_failure(self):
# Initialize test objects
batch, gtf_file, _ = init_objects()
processor_job = ProcessorJob.create_job_and_relationships(
batches=[batch])
# Mock out the job_context with everything the function under
# test will expect
job_context = utils.start_job({
"job": processor_job,
"job_id": processor_job.id,
"gtf_file": gtf_file,
"output_dir": "missing/index"
})
job_context = transcriptome_index._set_job_prefix(job_context)
# The function being tested.
job_context = transcriptome_index._zip_index(job_context)
self.assertFalse(job_context["success"])
self.assertEqual(
processor_job.failure_reason,
("Exception caught while zipping index directory /home/user"
"/data_store/temp/EnsemblPlants/TRANSCRIPTOME_INDEX/{}"
"/aegilops_tauschii_short.tar.gz").format(
job_context["job_dir_prefix"]))
self.assertFalse(os.path.isfile(batch.files[0].get_temp_pre_path()))
def test_create_index_failure(self, mocked_subprocess):
# Initialize mock and test objects
mocked_subprocess.return_value = CompletedProcess(
[], 1, stdout=None, stderr="Error: something went wrong.")
batch, gtf_file, fasta_file = init_objects()
processor_job = ProcessorJob.create_job_and_relationships(
batches=[batch])
# Mock out the job_context with everything the function under
# test will expect
job_context = utils.start_job({
"job": processor_job,
"job_id": processor_job.id,
"gtf_file": gtf_file,
"gtf_file_path": "dummy",
"fasta_file": fasta_file,
"fasta_file_path": "dummy",
"genes_to_transcripts_path": "dummy"
})
job_context = transcriptome_index._set_job_prefix(job_context)
# The function being tested.
job_context = transcriptome_index._create_index(job_context)
self.assertFalse(job_context["success"])
self.assertEqual(
processor_job.failure_reason,
("Shell call to rsem-prepare-reference failed because: "
"Error: something went wrong."))
self.assertFalse(os.path.isfile(batch.files[0].get_temp_pre_path()))
def test_success(self):
batch, batch2 = init_objects()
processor_job = ProcessorJob.create_job_and_relationships(
batches=[batch, batch2])
job_context = utils.start_job({"job": processor_job})
# start_job preserves the "job" key
self.assertEqual(job_context["job"], processor_job)
# start_job finds the batches and returns them
self.assertEqual(len(job_context["batches"]), 2)
def test_success(self):
processor_job = prepare_job()
job_context = utils.start_job({"job": processor_job})
# start_job preserves the "job" key
self.assertEqual(job_context["job"], processor_job)
job_context['success'] = True
job_context = utils.end_job(job_context)
for sample in job_context['samples']:
self.assertTrue(sample.is_processed)
def test_failure(self):
batch = init_objects()
processor_job = ProcessorJob.create_job_and_relationships(batches=[batch])
job_context = utils.start_job({"job": processor_job,
"job_id": processor_job.id})
job_context = array_express._prepare_files(job_context)
self.assertFalse(job_context["success"])
self.assertEqual(processor_job.failure_reason,
"Exception caught while retrieving raw file CE1234.CEL")
self.assertFalse(os.path.isfile(batch.files[0].get_temp_pre_path()))
def test_success(self):
batch = init_objects()
file = batch.files[0]
processor_job = ProcessorJob.create_job_and_relationships(batches=[batch])
os.makedirs(file.get_raw_dir(), exist_ok=True)
raw_path = file.get_raw_path()
with open(raw_path, "w") as dummy_pcl:
dummy_pcl.write("This is a dummy file for tests to operate upon.")
job_context = utils.start_job({"job": processor_job})
job_context = array_express._prepare_files(job_context)
input_file_path = job_context["input_file_path"]
self.assertIsInstance(input_file_path, str)
self.assertIsInstance(job_context["output_file_path"], str)
self.assertTrue(os.path.isfile(input_file_path))
os.remove(raw_path)
os.remove(input_file_path)
def test_download_index_not_found(self):
batch, _, _ = init_objects()
processor_job = ProcessorJob.create_job_and_relationships(
batches=[batch])
job_context = utils.start_job({
"job": processor_job,
"job_id": processor_job.id,
"kmer_size": "23"
})
job_context = salmon._prepare_files(job_context)
# Function we're testing.
salmon._download_index(job_context)
self.assertFalse(job_context["success"])
self.assertEqual(
processor_job.failure_reason,
"Failed to find an index for organism H**O SAPIENS with kmer_size of 23."
)
def test_zip_and_upload_failure(self):
# Initialize test objects
batch, _, _ = init_objects()
processor_job = ProcessorJob.create_job_and_relationships(
batches=[batch])
# Mock out the job_context with everything the function under
# test will expect
job_context = utils.start_job({
"job": processor_job,
"job_id": processor_job.id,
"job_dir_prefix": "dummy",
"output_directory": "missing/index"
})
# The function being tested.
job_context = salmon._zip_and_upload(job_context)
self.assertFalse(job_context["success"])
self.assertEqual(
processor_job.failure_reason,
"Exception caught while zipping processed directory ERR003000_1.fastq.gz"
)
self.assertFalse(os.path.isfile(batch.files[0].get_temp_pre_path()))
def test_download_index_missing(self, mock_download_processed_file):
batch, _, _ = init_objects()
_insert_salmon_index()
processor_job = ProcessorJob.create_job_and_relationships(
batches=[batch])
job_context = utils.start_job({
"job": processor_job,
"job_id": processor_job.id,
"job_dir_prefix": "dummy",
"kmer_size": "23"
})
job_context = salmon._prepare_files(job_context)
mock_download_processed_file.side_effect = FileNotFoundError()
# The function being testing.
salmon._download_index(job_context)
self.assertFalse(job_context["success"])
self.assertEqual(
processor_job.failure_reason,
"Failed to download and extract index tarball Homo_sapiens_short.gtf.gz"
)
def test_success(self):
"""Tests the successful path of the module under test."""
logger.info("STARTING SALMON SUCCESS TEST!!!!!!!!")
# Set up test environment.
batch, first_file, second_file = init_objects()
_insert_salmon_index()
# Change the batch/files to point to test-specific locations
batch.platform_accession_code = "TEST"
batch.save()
first_file.internal_location = "TEST/SALMON"
first_file.save()
second_file.internal_location = "TEST/SALMON"
second_file.save()
processor_job = ProcessorJob.create_job_and_relationships(
batches=[batch])
processor_job.save()
job_context = utils.start_job({
"job": processor_job,
"job_id": processor_job.id
})
job_context = salmon._set_job_prefix(job_context)
# Ensure temp dir isn't leftover from a previous test.
temp_dir = first_file.get_temp_dir(job_context["job_dir_prefix"])
shutil.rmtree(temp_dir, ignore_errors=True)
# One of the functions being tested:
job_context = salmon._prepare_files(job_context)
input_file_path = job_context["input_file_path"]
self.assertIsInstance(input_file_path, str)
self.assertTrue(os.path.isfile(input_file_path))
input_file_path_2 = job_context["input_file_path_2"]
self.assertIsInstance(input_file_path_2, str)
self.assertTrue(os.path.isfile(input_file_path_2))
output_directory_path = job_context["output_directory"]
self.assertIsInstance(output_directory_path, str)
self.assertTrue(os.path.isdir(output_directory_path))
job_context = salmon._determine_index_length(job_context)
# The 'kmer_size' key has been added to job_context with the
# correct value.
self.assertEqual(job_context["kmer_size"], "23")
# Another function being tested
job_context = salmon._download_index(job_context)
self.assertTrue(job_context["success"])
self.assertTrue("index_directory" in job_context)
self.assertTrue(os.path.isdir(job_context["index_directory"]))
self.assertEqual(9, len(os.listdir(job_context["index_directory"])))
# Another function being tested
job_context = salmon._run_salmon(job_context)
self.assertTrue(job_context["success"])
self.assertGreater(len(os.listdir(output_directory_path)), 1)
# The last function to test
job_context = salmon._zip_and_upload(job_context)
self.assertTrue(job_context["success"])
self.assertTrue(os.path.exists(first_file.get_processed_path()))
# Clean up both input and output files
first_file.remove_temp_directory()
shutil.rmtree(first_file.get_processed_dir())
logger.info("ENDING SALMON SUCCESS TEST!!!!!!!!")
def test_success(self):
"""Tests the successful path of the module under test."""
# Set up test environment.
batch, gtf_file, fasta_file = init_objects()
# Change the batch/files to point to test-specific locations
batch.platform_accession_code = "TEST"
batch.save()
gtf_file.internal_location = "TEST/TRANSCRIPTOME_INDEX"
gtf_file.save()
fasta_file.internal_location = "TEST/TRANSCRIPTOME_INDEX"
fasta_file.save()
processor_job = ProcessorJob.create_job_and_relationships(
batches=[batch])
processor_job.save()
job_context = utils.start_job({
"job": processor_job,
"job_id": processor_job.id
})
job_context = transcriptome_index._set_job_prefix(job_context)
# Ensure temp dir isn't leftover from a previous test.
temp_dir = gtf_file.get_temp_dir(job_context["job_dir_prefix"])
shutil.rmtree(temp_dir, ignore_errors=True)
# One of the functions being tested:
job_context = transcriptome_index._prepare_files(job_context)
gtf_file_path = job_context["gtf_file_path"]
self.assertIsInstance(gtf_file_path, str)
self.assertTrue(os.path.isfile(gtf_file_path))
fasta_file_path = job_context["fasta_file_path"]
self.assertIsInstance(fasta_file_path, str)
self.assertTrue(os.path.isfile(fasta_file_path))
self.assertIsInstance(job_context["gtf_file"], File)
self.assertIsInstance(job_context["fasta_file"], File)
job_context = transcriptome_index._process_gtf(job_context)
self.assertTrue(job_context["success"])
# A new gtf file is created by _process_gtf
self.assertNotEqual(job_context["gtf_file_path"], gtf_file_path)
self.assertTrue(os.path.isfile(job_context["gtf_file_path"]))
self.assertTrue("genes_to_transcripts_path" in job_context)
self.assertTrue(
os.path.isfile(job_context["genes_to_transcripts_path"]))
# Another function being tested
job_context = transcriptome_index._create_index(job_context)
self.assertTrue(job_context["success"])
self.assertTrue("output_dir" in job_context)
self.assertTrue(os.path.isdir(job_context["output_dir"]))
self.assertEqual(9, len(os.listdir(job_context["output_dir"])))
# Another function being tested
job_context = transcriptome_index._zip_index(job_context)
self.assertTrue("files_to_upload" in job_context)
self.assertEqual(job_context["gtf_file"].id, gtf_file.id)
self.assertEqual(job_context["files_to_upload"][0].id, gtf_file.id)
zipped_path = gtf_file.get_temp_post_path(
job_context["job_dir_prefix"])
self.assertTrue(os.path.isfile(zipped_path))
# Clean up both input and output files
utils.end_job(job_context)