def test_calculate_job_results_per_manifest(self):
db_obj = FakeDBObj(self)
we = WorkflowsExtractor()
job_list = {
"job_name": [
"wf_manifest-2_S0", "wf_manifest-2_S1_dS0",
"wf_manifest-2_S2_dS0", "wf_manifest-2_S3_dS2",
"wf_manifest-2_S4_dS3", "wf_manifest-2_S5_dS4-dS1",
"wf_manifest-2_S6_dS0", "sim_job", "wf_manifest-3_S0",
"wf_manifest-3_S1_dS0", "wf_manifest-3_S2_dS0",
"wf_manifest-3_S3_dS1-dS2", "wf_manifest2-4_S0"
],
"id_job": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
"time_start": [1, 15, 17, 22, 27, 42, 12, 20, 1, 15, 17, 22, 30],
"time_end": [10, 20, 40, 25, 29, 50, 70, 30, 10, 20, 19, 25, 35],
"time_submit": [1, 1, 1, 1, 1, 1, 1, 20, 2, 2, 2, 2, 3],
"cpus_alloc": [1, 2, 3, 4, 5, 6, 7, 1, 1, 2, 3, 4, 33]
}
we.extract(job_list)
we.do_processing()
we.calculate_per_manifest_results(True, db_obj, 1)
self.assertEqual(db_obj._id_count, 24)
self.assertEqual(
sorted(db_obj._set_fields),
sorted([
"m_manifest2_wf_runtime_cdf", "m_manifest2_wf_runtime_stats",
"m_manifest2_wf_waittime_cdf", "m_manifest2_wf_waittime_stats",
"m_manifest2_wf_turnaround_cdf",
"m_manifest2_wf_turnaround_stats",
"m_manifest2_wf_stretch_factor_cdf",
"m_manifest2_wf_stretch_factor_stats",
"m_manifest2_wf_jobs_runtime_cdf",
"m_manifest2_wf_jobs_runtime_stats",
"m_manifest2_wf_jobs_cores_cdf",
"m_manifest2_wf_jobs_cores_stats", "m_manifest_wf_runtime_cdf",
"m_manifest_wf_runtime_stats", "m_manifest_wf_waittime_cdf",
"m_manifest_wf_waittime_stats", "m_manifest_wf_turnaround_cdf",
"m_manifest_wf_turnaround_stats",
"m_manifest_wf_stretch_factor_cdf",
"m_manifest_wf_stretch_factor_stats",
"m_manifest_wf_jobs_runtime_cdf",
"m_manifest_wf_jobs_runtime_stats",
"m_manifest_wf_jobs_cores_cdf",
"m_manifest_wf_jobs_cores_stats"
]))
self.assertEqual(db_obj._hist_count, 12)
self.assertEqual(db_obj._stats_count, 12)
def test_load_job_results_per_manifest(self):
db_obj = self._db
hist = Histogram()
stat = NumericStats()
self.addCleanup(self._del_table, "histograms")
self.addCleanup(self._del_table, "numericStats")
hist.create_table(db_obj)
stat.create_table(db_obj)
we = WorkflowsExtractor()
job_list = {
"job_name": [
"wf_manifest-2_S0", "wf_manifest-2_S1_dS0",
"wf_manifest-2_S2_dS0", "wf_manifest-2_S3_dS2",
"wf_manifest-2_S4_dS3", "wf_manifest-2_S5_dS4-dS1",
"wf_manifest-2_S6_dS0", "sim_job", "wf_manifest-3_S0",
"wf_manifest-3_S1_dS0", "wf_manifest-3_S2_dS0",
"wf_manifest-3_S3_dS1-dS2", "wf_manifest2-4_S0"
],
"id_job": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
"time_start": [1, 15, 17, 22, 27, 42, 12, 20, 1, 15, 17, 22, 30],
"time_end": [10, 20, 40, 25, 29, 50, 70, 30, 10, 20, 19, 25, 35],
"time_submit": [1, 1, 1, 1, 1, 1, 1, 20, 2, 2, 2, 2, 3],
"cpus_alloc": [1, 2, 3, 4, 5, 6, 7, 1, 1, 2, 3, 4, 33]
}
we.extract(job_list)
we.do_processing()
old_results = we.calculate_per_manifest_results(True, db_obj, 1)
new_we = WorkflowsExtractor()
new_results = new_we.load_per_manifest_results(db_obj, 1)
self.assertEqual(sorted(list(new_results.keys())),
sorted(["manifest2", "manifest"]))
for manifest in ["manifest2", "manifest"]:
for field in [
"wf_runtime_cdf", "wf_runtime_stats", "wf_waittime_cdf",
"wf_waittime_stats", "wf_turnaround_cdf",
"wf_turnaround_stats", "wf_stretch_factor_cdf",
"wf_stretch_factor_stats", "wf_jobs_runtime_cdf",
"wf_jobs_runtime_stats", "wf_jobs_cores_cdf",
"wf_jobs_cores_stats"
]:
field = "m_" + manifest + "_" + field
assertEqualResult(self, old_results[manifest][field],
new_results[manifest][field], field)