def test_calculate_job_results(self):
db_obj = FakeDBObj(self)
we = WorkflowsExtractor()
job_list = {
"job_name": [
"wf_manifest-2_S0", "wf_manifest-2_S1_dS0",
"wf_manifest-2_S2_dS0", "wf_manifest-2_S3_dS2",
"wf_manifest-2_S4_dS3", "wf_manifest-2_S5_dS4-dS1",
"wf_manifest-2_S6_dS0", "sim_job", "wf_manifest-3_S0",
"wf_manifest-3_S1_dS0", "wf_manifest-3_S2_dS0",
"wf_manifest-3_S3_dS1-dS2"
],
"id_job": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
"time_start": [1, 15, 17, 22, 27, 42, 12, 20, 1, 15, 17, 22],
"time_end": [10, 20, 40, 25, 29, 50, 70, 30, 10, 20, 19, 25],
"time_submit": [1, 1, 1, 1, 1, 1, 1, 20, 2, 2, 2, 2],
"cpus_alloc": [1, 2, 3, 4, 5, 6, 7, 1, 1, 2, 3, 4]
}
we.extract(job_list)
we.do_processing()
we.calculate_overall_results(True, db_obj, 1)
self.assertEqual(db_obj._id_count, 12)
self.assertEqual(db_obj._set_fields, [
"wf_runtime_cdf", "wf_runtime_stats", "wf_waittime_cdf",
"wf_waittime_stats", "wf_turnaround_cdf", "wf_turnaround_stats",
"wf_stretch_factor_cdf", "wf_stretch_factor_stats",
"wf_jobs_runtime_cdf", "wf_jobs_runtime_stats",
"wf_jobs_cores_cdf", "wf_jobs_cores_stats"
])
self.assertEqual(db_obj._hist_count, 6)
self.assertEqual(db_obj._stats_count, 6)
def test_extract_process_single(self):
db_obj = FakeDBObj(self)
job_list = {
"job_name": ["wf_manifest-0", "wf_manifest-1", "sim_job"],
"id_job": [0, 1, 2],
"time_submit": [1, 3, 4],
"time_start": [1, 15, 17],
"time_end": [11, 0, 40],
"cpus_alloc": [100, 100, 300]
}
we = WorkflowsExtractor()
we.extract(job_list)
we.do_processing()
self.assertEqual(len(we._workflows), 1)
self.assertEqual(list(we._workflows.keys()), ["manifest-0"])
wt = we.get_workflow("manifest-0")
t0 = wt.get_all_tasks()[0]
#t1 = wt._tasks["S1"]
#t2 = wt._tasks["S2"]
#t3 = wt._tasks["S3"]
self.assertEqual(wt._start_task, t0)
self.assertEqual(wt._critical_path, [t0])
self.assertEqual(wt._critical_path_runtime, 10)
we.calculate_overall_results(True, db_obj, 1)
def test_load_job_results(self):
db_obj = self._db
hist = Histogram()
stat = NumericStats()
self.addCleanup(self._del_table, "histograms")
self.addCleanup(self._del_table, "numericStats")
hist.create_table(db_obj)
stat.create_table(db_obj)
we = WorkflowsExtractor()
job_list = {
"job_name": [
"wf_manifest-2_S0", "wf_manifest-2_S1_dS0",
"wf_manifest-2_S2_dS0", "wf_manifest-2_S3_dS2",
"wf_manifest-2_S4_dS3", "wf_manifest-2_S5_dS4-dS1",
"wf_manifest-2_S6_dS0", "sim_job", "wf_manifest-3_S0",
"wf_manifest-3_S1_dS0", "wf_manifest-3_S2_dS0",
"wf_manifest-3_S3_dS1-dS2"
],
"id_job": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
"time_start": [1, 15, 17, 22, 27, 42, 12, 20, 1, 15, 17, 22],
"time_end": [10, 20, 40, 25, 29, 50, 70, 30, 10, 20, 19, 25],
"time_submit": [1, 1, 1, 1, 1, 1, 1, 20, 2, 2, 2, 2],
"cpus_alloc": [1, 2, 3, 4, 5, 6, 7, 1, 1, 2, 3, 4]
}
we.extract(job_list)
we.do_processing()
old_results = we.calculate_overall_results(True, db_obj, 1)
new_we = WorkflowsExtractor()
new_results = new_we.load_overall_results(db_obj, 1)
for field in [
"wf_runtime_cdf", "wf_runtime_stats", "wf_waittime_cdf",
"wf_waittime_stats", "wf_turnaround_cdf",
"wf_turnaround_stats", "wf_stretch_factor_cdf",
"wf_stretch_factor_stats", "wf_jobs_runtime_cdf",
"wf_jobs_runtime_stats", "wf_jobs_cores_cdf",
"wf_jobs_cores_stats"
]:
assertEqualResult(self, old_results[field], new_results[field],
field)