def test_get_workflow_times_start_stop_per_manifest_multi(self):
db_obj = FakeDBObj(self)
we = WorkflowsExtractor()
job_list = {
"job_name": [
"wf_manifest-2_S0", "wf_manifest-2_S1_dS0",
"wf_manifest-2_S2_dS0", "wf_manifest-2_S3_dS2",
"wf_manifest-2_S4_dS3", "wf_manifest-2_S5_dS4-dS1",
"wf_manifest-2_S6_dS0", "sim_job", "wf_manifest-3_S0",
"wf_manifest-3_S1_dS0", "wf_manifest-3_S2_dS0",
"wf_manifest-3_S3_dS1-dS2", "wf_manifestA-4_S0"
],
"id_job": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14],
"time_start": [1, 15, 17, 22, 27, 42, 12, 20, 1, 15, 17, 22, 4],
"time_end": [10, 20, 40, 25, 29, 50, 70, 30, 10, 20, 19, 25, 10],
"time_submit": [1, 1, 1, 1, 1, 1, 1, 20, 2, 2, 2, 2, 3],
"cpus_alloc": [1, 2, 3, 4, 5, 6, 7, 1, 1, 2, 3, 4, 1]
}
we.extract(job_list)
we.do_processing()
manifests = we._get_per_manifest_workflow_times(submit_start=2,
submit_stop=None)
self.assertEqual(manifests["manifest"]["wf_runtime"], [24])
self.assertEqual(manifests["manifestA"]["wf_runtime"], [6])
def test_get_workflow_times_start_stop(self):
db_obj = FakeDBObj(self)
we = WorkflowsExtractor()
job_list = {
"job_name": [
"wf_manifest-2_S0", "wf_manifest-2_S1_dS0",
"wf_manifest-2_S2_dS0", "wf_manifest-2_S3_dS2",
"wf_manifest-2_S4_dS3", "wf_manifest-2_S5_dS4-dS1",
"wf_manifest-2_S6_dS0", "sim_job", "wf_manifest-3_S0",
"wf_manifest-3_S1_dS0", "wf_manifest-3_S2_dS0",
"wf_manifest-3_S3_dS1-dS2"
],
"id_job": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
"time_start": [1, 15, 17, 22, 27, 42, 12, 20, 1, 15, 17, 22],
"time_end": [10, 20, 40, 25, 29, 50, 70, 30, 10, 20, 19, 25],
"time_submit": [1, 1, 1, 1, 1, 1, 1, 20, 2, 2, 2, 2],
"cpus_alloc": [1, 2, 3, 4, 5, 6, 7, 1, 1, 2, 3, 4]
}
we.extract(job_list)
we.do_processing()
(wf_runtime, wf_waittime, wf_turnaround, wf_stretch_factor,
wf_jobs_runtime,
wf_jobs_cores) = we._get_workflow_times(submit_start=2, submit_stop=3)
self.assertEqual(wf_runtime, [24])
def test_calculate_job_results(self):
db_obj = FakeDBObj(self)
we = WorkflowsExtractor()
job_list = {
"job_name": [
"wf_manifest-2_S0", "wf_manifest-2_S1_dS0",
"wf_manifest-2_S2_dS0", "wf_manifest-2_S3_dS2",
"wf_manifest-2_S4_dS3", "wf_manifest-2_S5_dS4-dS1",
"wf_manifest-2_S6_dS0", "sim_job", "wf_manifest-3_S0",
"wf_manifest-3_S1_dS0", "wf_manifest-3_S2_dS0",
"wf_manifest-3_S3_dS1-dS2"
],
"id_job": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
"time_start": [1, 15, 17, 22, 27, 42, 12, 20, 1, 15, 17, 22],
"time_end": [10, 20, 40, 25, 29, 50, 70, 30, 10, 20, 19, 25],
"time_submit": [1, 1, 1, 1, 1, 1, 1, 20, 2, 2, 2, 2],
"cpus_alloc": [1, 2, 3, 4, 5, 6, 7, 1, 1, 2, 3, 4]
}
we.extract(job_list)
we.do_processing()
we.calculate_overall_results(True, db_obj, 1)
self.assertEqual(db_obj._id_count, 12)
self.assertEqual(db_obj._set_fields, [
"wf_runtime_cdf", "wf_runtime_stats", "wf_waittime_cdf",
"wf_waittime_stats", "wf_turnaround_cdf", "wf_turnaround_stats",
"wf_stretch_factor_cdf", "wf_stretch_factor_stats",
"wf_jobs_runtime_cdf", "wf_jobs_runtime_stats",
"wf_jobs_cores_cdf", "wf_jobs_cores_stats"
])
self.assertEqual(db_obj._hist_count, 6)
self.assertEqual(db_obj._stats_count, 6)
def test_extract_process_single(self):
db_obj = FakeDBObj(self)
job_list = {
"job_name": ["wf_manifest-0", "wf_manifest-1", "sim_job"],
"id_job": [0, 1, 2],
"time_submit": [1, 3, 4],
"time_start": [1, 15, 17],
"time_end": [11, 0, 40],
"cpus_alloc": [100, 100, 300]
}
we = WorkflowsExtractor()
we.extract(job_list)
we.do_processing()
self.assertEqual(len(we._workflows), 1)
self.assertEqual(list(we._workflows.keys()), ["manifest-0"])
wt = we.get_workflow("manifest-0")
t0 = wt.get_all_tasks()[0]
#t1 = wt._tasks["S1"]
#t2 = wt._tasks["S2"]
#t3 = wt._tasks["S3"]
self.assertEqual(wt._start_task, t0)
self.assertEqual(wt._critical_path, [t0])
self.assertEqual(wt._critical_path_runtime, 10)
we.calculate_overall_results(True, db_obj, 1)
def test_fill_overall_values(self):
db_obj = FakeDBObj(self)
we = WorkflowsExtractor()
job_list = {
"job_name": [
"wf_manifest-2", "wf_manifest-2_S0", "wf_manifest-2_S1_dS0",
"wf_manifest-2_S2_dS0", "wf_manifest-2_S3_dS2",
"wf_manifest-2_S4_dS3", "wf_manifest-2_S5_dS4-dS1",
"wf_manifest-2_S6_dS0", "sim_job", "wf_manifest-3_S0",
"wf_manifest-3_S1_dS0", "wf_manifest-3_S2_dS0",
"wf_manifest-3_S3_dS1-dS2"
],
"id_job": [2, 0, 1, 33, 3, 4, 5, 6, 7, 8, 9, 10, 11],
"time_start": [2, 2, 15, 17, 22, 27, 42, 12, 20, 2, 15, 17, 22],
"time_end": [2, 10, 20, 40, 25, 29, 50, 70, 30, 10, 20, 19, 25],
"time_submit": [1, 1, 1, 1, 1, 1, 1, 1, 20, 2, 2, 2, 2],
"cpus_alloc": [33, 1, 2, 3, 4, 5, 6, 7, 1, 1, 2, 3, 4]
}
we.extract(job_list)
we.do_processing()
we.fill_overall_values()
self.assertEqual(we._wf_runtime, [68, 23])
self.assertEqual(we._wf_waittime, [1, 0])
self.assertEqual(we._wf_turnaround, [69, 23])
self.assertEqual(len(we._wf_stretch_factor), 2)
self.assertEqual(len(we._wf_jobs_runtime), 11)
self.assertEqual(len(we._wf_jobs_cores), 11)
we.extract(job_list)
we.do_processing()
we.fill_overall_values(append=True)
self.assertEqual(we._wf_runtime, [68, 23, 68, 23])
self.assertEqual(we._wf_waittime, [1, 0, 1, 0])
self.assertEqual(we._wf_turnaround, [69, 23, 69, 23])
self.assertEqual(len(we._wf_stretch_factor), 4)
self.assertEqual(len(we._wf_jobs_runtime), 22)
self.assertEqual(len(we._wf_jobs_cores), 22)
def test_fill_per_manifest_values(self):
db_obj = FakeDBObj(self)
we = WorkflowsExtractor()
job_list = {
"job_name": [
"wf_manifest-2_S0", "wf_manifest-2_S1_dS0",
"wf_manifest-2_S2_dS0", "wf_manifest-2_S3_dS2",
"wf_manifest-2_S4_dS3", "wf_manifest-2_S5_dS4-dS1",
"wf_manifest-2_S6_dS0", "sim_job", "wf_manifest-3_S0",
"wf_manifest-3_S1_dS0", "wf_manifest-3_S2_dS0",
"wf_manifest-3_S3_dS1-dS2", "wf_manifest2-4_S0"
],
"id_job": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
"time_start": [1, 15, 17, 22, 27, 42, 12, 20, 2, 15, 17, 22, 30],
"time_end": [10, 20, 40, 25, 29, 50, 70, 30, 10, 20, 19, 25, 35],
"time_submit": [1, 1, 1, 1, 1, 1, 1, 20, 2, 2, 2, 2, 3],
"cpus_alloc": [1, 2, 3, 4, 5, 6, 7, 1, 1, 2, 3, 4, 33]
}
we.extract(job_list)
we.do_processing()
we.fill_per_manifest_values()
self.assertEqual(sorted(we._detected_manifests),
["manifest", "manifest2"])
self.assertEqual(we._manifests_values["manifest"]["wf_runtime"],
[69, 23])
self.assertEqual(we._manifests_values["manifest"]["wf_waittime"],
[0, 0])
self.assertEqual(we._manifests_values["manifest"]["wf_turnaround"],
[69, 23])
self.assertEqual(
len(we._manifests_values["manifest"]["wf_stretch_factor"]), 2)
self.assertEqual(
len(we._manifests_values["manifest"]["wf_jobs_runtime"]), 11)
self.assertEqual(
len(we._manifests_values["manifest"]["wf_jobs_cores"]), 11)
self.assertEqual(we._manifests_values["manifest2"]["wf_runtime"], [5])
self.assertEqual(we._manifests_values["manifest2"]["wf_waittime"],
[27])
self.assertEqual(we._manifests_values["manifest2"]["wf_turnaround"],
[32])
self.assertEqual(
len(we._manifests_values["manifest2"]["wf_stretch_factor"]), 1)
self.assertEqual(
len(we._manifests_values["manifest2"]["wf_jobs_runtime"]), 1)
self.assertEqual(
len(we._manifests_values["manifest2"]["wf_jobs_cores"]), 1)
we.extract(job_list)
we.do_processing()
we.fill_per_manifest_values(append=True)
self.assertEqual(we._manifests_values["manifest"]["wf_runtime"],
[69, 23, 69, 23])
self.assertEqual(we._manifests_values["manifest"]["wf_waittime"],
[0, 0, 0, 0])
self.assertEqual(we._manifests_values["manifest"]["wf_turnaround"],
[69, 23, 69, 23])
self.assertEqual(
len(we._manifests_values["manifest"]["wf_stretch_factor"]), 4)
self.assertEqual(
len(we._manifests_values["manifest"]["wf_jobs_runtime"]), 22)
self.assertEqual(
len(we._manifests_values["manifest"]["wf_jobs_cores"]), 22)
self.assertEqual(we._manifests_values["manifest2"]["wf_runtime"],
[5, 5])
self.assertEqual(we._manifests_values["manifest2"]["wf_waittime"],
[27, 27])
self.assertEqual(we._manifests_values["manifest2"]["wf_turnaround"],
[32, 32])
self.assertEqual(
len(we._manifests_values["manifest2"]["wf_stretch_factor"]), 2)
self.assertEqual(
len(we._manifests_values["manifest2"]["wf_jobs_runtime"]), 2)
self.assertEqual(
len(we._manifests_values["manifest2"]["wf_jobs_cores"]), 2)