def test_Monitor_mark_job_for_resubmission(self):
test_monitor = self.test_monitor
test_monitor.pipeline_steps[
self.testing_step_classname] = TestingStep()
test_sample = Sample(sample_id=1,
sample_name="1",
fastq_file_paths="1",
adapter_sequences="1",
pooled=False)
# Contruct what the submitted job should look like:
test_job = Job(job_id=1,
job_command="",
sample_id=test_sample.sample_id,
step_name=self.testing_step_classname,
scheduler_arguments={},
validation_attributes=None,
output_directory_path="",
system_id=None,
dependency_list=None)
test_monitor.running_list[test_job.job_id] = test_job
test_monitor.mark_job_for_resubmission(test_job.job_id)
# Comparing the reprs is a quick way to test for equality across all of
# the job objects' attributes.
self.assertTrue(repr(test_monitor.resubmission_list[test_job.job_id]) == repr(test_job) and \
test_job.job_id not in test_monitor.running_list)
def test_Monitor_is_processing_complete_running_job_in_running_queue(self):
test_monitor = self.test_monitor
test_monitor.pipeline_steps[
self.testing_step_classname] = TestingStep()
test_sample = Sample(sample_id=1,
sample_name="1",
fastq_file_paths="1",
adapter_sequences="1",
pooled=False)
test_step_object = TestingStep()
test_job_status = "RUNNING"
# Contruct what the completed job should look like:
test_job = Job(
job_id=1,
job_command="",
sample_id=test_sample.sample_id,
step_name=self.testing_step_classname,
scheduler_arguments="None",
validation_attributes=test_step_object.get_validation_attributes(),
output_directory_path="",
system_id=test_job_status,
dependency_list=[])
test_monitor.running_list[test_job.job_id] = test_job
self.assertTrue(test_monitor.is_processing_complete() is False and \
repr(test_monitor.running_list[test_job.job_id]) == repr(test_job) and \
test_job.job_id not in test_monitor.resubmission_list)
def test_Monitor_resubmit_job_scheduler_failed(self):
test_monitor = self.test_monitor
test_monitor.pipeline_steps[
self.testing_step_classname] = TestingStep()
test_sample = Sample(sample_id=1,
sample_name="1",
fastq_file_paths="1",
adapter_sequences="1",
pooled=False)
# Contruct what the submitted job should look like:
test_job = Job(
job_id=1,
job_command="",
sample_id=test_sample.sample_id,
step_name=self.testing_step_classname,
# The TestingScheduler class will mimic a submission error
# if the 'additional_args' parameter of the submtted Job
# object is set to "ERROR".
scheduler_arguments={'additional_args': "ERROR"},
validation_attributes=None,
output_directory_path="",
system_id=None,
dependency_list=None)
test_monitor.resubmission_list[test_job.job_id] = test_job
with self.assertRaisesRegex(
JobMonitorException,
"Job submission failed for .*. See log file "
"for full details."):
test_monitor.resubmit_job(test_job.job_id)
def test_Monitor_resubmit_job_already_in_pending_queue(self):
test_monitor = self.test_monitor
test_monitor.pipeline_steps[
self.testing_step_classname] = TestingStep()
test_sample = Sample(sample_id=1,
sample_name="1",
fastq_file_paths="1",
adapter_sequences="1",
pooled=False)
# Contruct what the submitted job should look like:
test_job = Job(job_id=1,
job_command="",
sample_id=test_sample.sample_id,
step_name=self.testing_step_classname,
scheduler_arguments={},
validation_attributes=None,
output_directory_path="",
system_id=None,
dependency_list=None)
test_monitor.pending_list[test_job.job_id] = test_job
with self.assertRaisesRegex(
JobMonitorException,
"Resubmitted job is already in the list of "
"running or pending jobs."):
test_monitor.resubmit_job(test_job.job_id)
def test_Monitor_resubmit_job_resubmission_limit_reached(self):
test_monitor = self.test_monitor
test_monitor.pipeline_steps[
self.testing_step_classname] = TestingStep()
test_sample = Sample(sample_id=1,
sample_name="1",
fastq_file_paths="1",
adapter_sequences="1",
pooled=False)
# Contruct what the submitted job should look like:
test_job = Job(job_id=1,
job_command="",
sample_id=test_sample.sample_id,
step_name=self.testing_step_classname,
scheduler_arguments={},
validation_attributes=None,
output_directory_path="",
system_id=None,
dependency_list=None)
test_job.resubmission_counter = test_monitor.max_resub_limit
test_monitor.resubmission_list[test_job.job_id] = test_job
with self.assertRaisesRegex(
JobMonitorException,
"exceeded the maximum resubmission limit of"):
test_monitor.resubmit_job(test_job.job_id)
def test_Monitor_submit_new_job_step_not_in_pipline(self):
test_monitor = self.test_monitor
test_monitor.pipeline_steps[
self.testing_step_classname] = TestingStep()
step_not_in_pipeline = "Not" + self.testing_step_classname
test_sample = Sample(sample_id=1,
sample_name="1",
fastq_file_paths="1",
adapter_sequences="1",
pooled=False)
# Contruct what the submitted job should look like:
test_job = Job(job_id=1,
job_command="",
sample_id=test_sample.sample_id,
step_name=step_not_in_pipeline,
scheduler_arguments={},
validation_attributes=None,
output_directory_path="",
system_id=None,
dependency_list=None)
with self.assertRaisesRegex(
JobMonitorException,
"Could not add job .* to the scheduler because "
"its associated pipeline step (.*) is not "
"currently tracked by the job monitor"):
test_monitor.submit_new_job(
job_id=test_job.job_id,
job_command=test_job.job_command,
sample=test_sample,
step_name=test_job.step_name,
scheduler_arguments=test_job.scheduler_arguments,
validation_attributes=test_job.validation_attributes,
output_directory_path=test_job.output_directory,
system_id=test_job.system_id)
def deserialize(file_path):
"""
This method re-rendered that serialized, compressed data found in the gzipped file located via the given
file path, into a fully restored object of the ClusterPacket class.
:param file_path: The locatation of the gzipped file containing the serialized object.
"""
cluster_lines = []
clusters = []
cluster_packet_id = 0
sample = None
with gzip.open(file_path, 'rb') as obj_file:
for line_number, line in enumerate(obj_file):
line = line.rstrip()
if line_number == 0:
cluster_packet_id = int(line[1:].decode())
elif line_number == 1:
sample = Sample.deserialize(line.decode())
else:
if line.decode() == '-':
if cluster_lines:
clusters.append(Cluster.deserialize("\n".join(cluster_lines)))
cluster_lines = []
else:
cluster_lines.append(line.decode())
return ClusterPacket(cluster_packet_id, sample, clusters)
def deserialize(file_path):
molecules = []
with open(file_path, 'rb') as obj_file:
for line_number, line in enumerate(obj_file):
line = line.rstrip()
if line_number == 0:
molecule_packet_id = int(line[1:].decode(encoding="ascii"))
elif line_number == 1:
sample = Sample.deserialize(line.decode(encoding="ascii"))
else:
molecules.append(
Molecule.deserialize(line.decode(encoding="ascii")))
return MoleculePacket(molecule_packet_id, sample, molecules)
def test_Monitor_are_dependencies_satisfied_none_completed(self):
test_monitor = self.test_monitor
test_monitor.pipeline_steps[
self.testing_step_classname] = TestingStep()
test_sample = Sample(sample_id=1,
sample_name="1",
fastq_file_paths="1",
adapter_sequences="1",
pooled=False)
# Construct two dependencies
dep_job_1 = Job(job_id="Dependency_1",
job_command="",
sample_id=test_sample.sample_id,
step_name=self.testing_step_classname,
scheduler_arguments={},
validation_attributes=None,
output_directory_path="",
system_id=None,
dependency_list=None)
dep_job_2 = Job(job_id="Dependency_2",
job_command="",
sample_id=test_sample.sample_id,
step_name=self.testing_step_classname,
scheduler_arguments={},
validation_attributes=None,
output_directory_path="",
system_id=None,
dependency_list=None)
# Contruct what the pending job should look like:
test_job = Job(job_id=1,
job_command="",
sample_id=test_sample.sample_id,
step_name=self.testing_step_classname,
scheduler_arguments={},
validation_attributes=None,
output_directory_path="",
system_id=None,
dependency_list=[dep_job_1.job_id, dep_job_2.job_id])
test_monitor.pending_list[test_job.job_id] = test_job
test_monitor.pending_list[dep_job_1.job_id] = dep_job_1
test_monitor.running_list[dep_job_2.job_id] = dep_job_2
self.assertFalse(
test_monitor.are_dependencies_satisfied(test_job.job_id))
def test_Monitor_is_processing_complete_job_in_resubmission_queue(self):
test_monitor = self.test_monitor
test_monitor.pipeline_steps[
self.testing_step_classname] = TestingStep()
test_sample = Sample(sample_id=1,
sample_name="1",
fastq_file_paths="1",
adapter_sequences="1",
pooled=False)
# Contruct what the pending job should look like:
test_job = Job(job_id=1,
job_command="",
sample_id=test_sample.sample_id,
step_name=self.testing_step_classname,
scheduler_arguments={},
validation_attributes=None,
output_directory_path="",
system_id=None,
dependency_list=None)
test_monitor.resubmission_list[test_job.job_id] = test_job
self.assertFalse(test_monitor.is_processing_complete())