def do_work(self, central_db_obj, sched_db_obj, trace_id=None):
"""
Args:
- central_db_obj: DB object configured to access the analysis database.
- sched_db_obj: DB object configured to access the slurm database of
an experiment worker.
- trace_id: If set to an experiment valid trace_id, it runs only the
experiment identified by trace_id.
"""
there_are_more=True
while there_are_more:
ed = ExperimentDefinition()
if trace_id:
ed.load(central_db_obj, trace_id)
ed.mark_pre_simulating(central_db_obj)
else:
there_are_more = ed.load_fresh(central_db_obj)
if there_are_more:
print(("About to run exp({0}):{1}".format(
ed._trace_id, ed._name)))
er = ExperimentRunner(ed)
if(er.do_full_run(sched_db_obj, central_db_obj)):
print(("Exp({0}) Done".format(
ed._trace_id)))
else:
print(("Exp({0}) Error!".format(
ed._trace_id)))
if trace_id:
break
def rescue_exp(self, central_db_obj, sched_db_obj, trace_id=None):
"""Retrieves the job trace from the database of an experiment worker and
stores it in the central db.
Args:
- central_db_obj: DB object configured to access the analysis database.
- sched_db_obj: DB object configured to access the slurm database of
an experiment worker.
- trace_id: trace_id of the experiment to which the rescued trace
corresponds.
"""
there_are_more=True
while there_are_more:
ed = ExperimentDefinition()
if trace_id:
ed.load(central_db_obj, trace_id)
ed.mark_simulation_done(central_db_obj)
else:
there_are_more = ed.load_next_state("simulation_failed",
"simulation_done")
if there_are_more:
print(("About to run resque({0}):{1}".format(
ed._trace_id, ed._name)))
er = ExperimentRunner(ed)
if(er.check_trace_and_store(sched_db_obj, central_db_obj)):
er.clean_trace_file()
print(("Exp({0}) Done".format(
ed._trace_id)))
else:
print(("Exp({0}) Error!".format(
ed._trace_id)))
if trace_id:
break
def test_store_load(self):
ed_old = ExperimentDefinition(seed="seeeed",
machine="machine",
trace_type="double",
manifest_list=[{
"share": 0.2,
"manifest": "man1.json"
}, {
"share": 0.8,
"manifest": "man2.json"
}],
workflow_policy="period",
workflow_period_s=20,
workflow_share=30.0,
workflow_handling="manifest",
subtraces=[100002, 10003],
preload_time_s=3600 * 24 * 3,
workload_duration_s=3600 * 24 * 8,
work_state="fresher",
analysis_state="1",
overload_target=2.0,
conf_file="my.conf")
ed = ExperimentDefinition()
self.addCleanup(self._del_table, "experiment")
ed.create_table(self._db)
trace_id = ed_old.store(self._db)
ed.load(self._db, trace_id)
self.assertEqual(
ed._experiment_set, "machine-double-m[0.2|man1.json,"
"0.8|man2.json]-period-p20-%30.0-manifest-"
"t[100002,10003]"
"-3d-8d-O2.0-my.conf")
self.assertEqual(
ed._name, "machine-double-m[0.2|man1.json,"
"0.8|man2.json]"
"-period-p20-%30.0-manifest-t[100002,10003]-3d-8d-O2.0"
"-my.conf-s[seeeed]")
self.assertEqual(ed._seed, "seeeed")
self.assertEqual(ed._machine, "machine")
self.assertEqual(ed._trace_type, "double")
self.assertEqual(ed._manifest_list, [
dict(share=0.2, manifest="man1.json"),
dict(share=0.8, manifest="man2.json")
])
self.assertEqual(ed._workflow_policy, "period")
self.assertEqual(ed._workflow_period_s, 20)
self.assertEqual(ed._workflow_share, 30.0)
self.assertEqual(ed._workflow_handling, "manifest")
self.assertEqual(ed._subtraces, [100002, 10003])
self.assertEqual(ed._preload_time_s, 3 * 24 * 3600)
self.assertEqual(ed._workload_duration_s, 8 * 24 * 3600)
self.assertEqual(ed._work_state, "fresher")
self.assertEqual(ed._analysis_state, "1")
self.assertEqual(ed._table_name, "experiment")
self.assertEqual(ed._overload_target, 2.0)
self.assertEqual(ed._conf_file, "my.conf")
def extract_usage(db_obj,
trace_id_rows,
fill_none=True,
factor=1.0,
mean=False):
"""Takes a list of lists of trace_is and produces a list of lists of results
corresponding to them.
Args:
- db_obj: DBManager object connted to a db where the results will be pulled
from.
"""
exp_rows = []
my = ResultTrace()
res_type = "usage"
if mean:
res_type = "usage_mean"
for row in trace_id_rows:
new_row = []
exp_rows.append(new_row)
for trace_id in row:
exp = ExperimentDefinition()
exp.load(db_obj, trace_id)
result = my._get_utilization_result()
if exp.is_analysis_done():
result.load(db_obj, trace_id, res_type)
else:
result._set("utilization", 0)
result._set("waste", 0)
result._set("corrected_utilization", 0)
result.apply_factor(factor)
new_row.append(result)
return exp_rows
def produce_plot_config(db_obj, trace_id_rows_colors):
""" Produces the coloring and hatches matrixes for a matrix style plot.
For that it conencts to a dabase, and depending on the scheduling algorithm
used in the experiment, it chooses a cooresponding coloring and hatches.
Args:
- db_obj: DBManager object connted to a db where the results will be pulled
from.
- trace_id_rows_colors: list of lists of integers as trace_ids of experiments.
returns:
- color_rows: list of list of matplotlib colors corresponding to each
experiment subplot.
- hatches_rows: list of lists of the hatches to be used in each experiment
subplot.
- legend: legend list of the format ("series names", "color"), listing the
scheduling algorithms present in the experiments.
"""
colors_dic = {
"no": "white",
"manifest": "lightgreen",
"single": "lightblue",
"multi": "pink",
"": "white"
}
hatches_dic = {
"no": None,
"manifest": "-",
"single": "\\",
"multi": "/",
"": None
}
detected_handling = {}
color_rows = []
hatches_rows = []
for row in trace_id_rows_colors:
this_color_row = []
color_rows.append(this_color_row)
this_hatches_row = []
hatches_rows.append(this_hatches_row)
for trace_id in row:
exp = ExperimentDefinition()
exp.load(db_obj, trace_id)
handling = exp.get_true_workflow_handling()
detected_handling[handling] = 1
this_color_row.append(get_dic_val(colors_dic, handling))
this_hatches_row.append(get_dic_val(hatches_dic, handling))
legend = [("n/a", "white", "no", None),
("aware", "lightgreen", "manifest", "-"),
("waste", "lightblue", "single", "\\"),
("wait", "pink", "multi", "/")]
new_legend = []
for item in legend:
if item[2] in list(detected_handling.keys()):
new_legend.append(item)
return color_rows, hatches_rows, new_legend
def extract_grouped_results(db_obj, trace_id_rows_colors, edges, result_type):
"""Takes a list of lists of trace_is and produces a list of lists of results
corresponding to them.
Args:
- db_obj: DBManager object connted to a db where the results will be pulled
from.
- trace_id_rows_colors: list of lists of integers as trace_ids of experiments.
- edges: if set to [""], it does no effect, the function extracts results
of the type result_type. If set to a list of items, results will be
pulled for each element as: "g"+str(edge)+_str(result_type)
- result_type: string indentifying which type of result are we pulling. It
correspond to the type of the NumericStats stored in db_obj.
Returns: a dictionary indexed by edges. Each element is a list of lists of
same dimension of trace_id_rows_colors, each element a NumericStats object
corresponding to the result of that component.
"""
exp_rows = {}
for edge in edges:
exp_rows[edge] = extract_results(
db_obj, trace_id_rows_colors,
ResultTrace.get_result_type_edge(edge, result_type))
return exp_rows
exp_rows = {}
for edge in edges:
exp_rows[edge] = []
for row in trace_id_rows_colors:
these_rows = {}
for edge in edges:
these_rows[edge] = []
exp_rows[edge].append(these_rows[edge])
for trace_id in row:
exp = ExperimentDefinition()
exp.load(db_obj, trace_id)
for edge in edges:
result = None
if exp.is_it_ready_to_process():
if edge == "":
key = ResultTrace.get_result_type_edge(
edge, result_type)
else:
key = result_type
key += "_stats"
result = NumericStats()
result.load(db_obj, trace_id, key)
else:
result = NumericStats()
result.calculate([0, 0, 0])
these_rows[edge].append(result)
return exp_rows
def test_mark_simulation_failed(self):
ed = ExperimentDefinition()
self.addCleanup(self._del_table, "experiment")
ed.create_table(self._db)
my_id = ed.store(self._db)
ed.mark_simulation_failed(self._db)
now_time = datetime.datetime.now()
new_ed = ExperimentDefinition()
new_ed.load(self._db, my_id)
self.assertEqual(new_ed._work_state, "simulation_failed")
self.assertLess(now_time - new_ed._simulating_end,
datetime.timedelta(10))
def test_reset_simulating_time(self):
ed = ExperimentDefinition()
self.addCleanup(self._del_table, "experiment")
ed.create_table(self._db)
my_id = ed.store(self._db)
ed.update_simulating_start(self._db)
ed.update_simulating_end(self._db)
new_ed = ExperimentDefinition()
new_ed.load(self._db, my_id)
self.assertNotEqual(new_ed._simulating_end, None)
self.assertNotEqual(new_ed._simulating_start, None)
ed.reset_simulating_time(self._db)
new_ed.load(self._db, my_id)
self.assertEqual(new_ed._simulating_end, None)
self.assertEqual(new_ed._simulating_start, None)
def do_full_analysis(self, db_obj):
result_trace = self.load_trace(db_obj)
first = True
last = False
for trace_id in self._definition._subtraces:
last = trace_id == self._definition._subtraces[-1]
result_trace.load_trace(db_obj, trace_id)
result_trace.do_workflow_pre_processing(append=not first)
one_definition = ExperimentDefinition()
one_definition.load(db_obj, trace_id)
result_trace.fill_job_values(
start=one_definition.get_start_epoch(),
stop=one_definition.get_end_epoch(),
append=not first)
result_trace.fill_workflow_values(
start=one_definition.get_start_epoch(),
stop=one_definition.get_end_epoch(),
append=not first)
result_trace.calculate_job_results_grouped_core_seconds(
one_definition.get_machine().get_core_seconds_edges(),
last,
db_obj,
self._definition._trace_id,
start=one_definition.get_start_epoch(),
stop=one_definition.get_end_epoch(),
append=not first)
first = False
result_trace.calculate_and_store_job_results(
store=True, db_obj=db_obj, trace_id=self._definition._trace_id)
result_trace._wf_extractor.calculate_and_store_overall_results(
store=True, db_obj=db_obj, trace_id=self._definition._trace_id)
result_trace._wf_extractor.calculate_and_store_per_manifest_results(
store=True, db_obj=db_obj, trace_id=self._definition._trace_id)
result_trace.calculate_utilization_median_result(
self._definition._subtraces,
store=True,
db_obj=db_obj,
trace_id=self._definition._trace_id)
result_trace.calculate_utilization_mean_result(
self._definition._subtraces,
store=True,
db_obj=db_obj,
trace_id=self._definition._trace_id)
self._definition.mark_analysis_done(db_obj)
def extract_results(db_obj,
trace_id_rows_colors,
result_type,
factor=None,
fill_none=True,
second_pass=False):
"""Takes a list of lists of trace_is and produces a list of lists of results
corresponding to them.
Args:
- db_obj: DBManager object connted to a db where the results will be pulled
from.
- trace_id_rows_colors: list of lists of integers as trace_ids of experiments.
- result_type: string indentifying which type of result are we pulling. It
correspond to the type of the NumericStats stored in db_obj.
Returns: a list of lists of
same dimension of trace_id_rows_colors, each element a NumericStats object
corresponding to the result of that component.
"""
exp_rows = []
for row in trace_id_rows_colors:
new_row = []
exp_rows.append(new_row)
for trace_id in row:
exp = ExperimentDefinition()
exp.load(db_obj, trace_id)
if exp.is_analysis_done(second_pass=second_pass):
key = result_type + "_stats"
result = NumericStats()
result.load(db_obj, trace_id, key)
if factor:
result.apply_factor(factor)
else:
result = NumericStats()
result.calculate([0, 0, 0])
if fill_none and result._get("median") is None:
result = NumericStats()
result.calculate([0, 0, 0])
new_row.append(result)
return exp_rows
def do_work_single(self, db_obj, trace_id=None):
"""Processes single type experiment results.
Args:
- db_obj: DB object configured to access the analysis database.
- trace_id: If set to "None", it processes all experiments in
"simulation_state". If set to an integer, it will analyze the
experiment identified by trace_id.
"""
there_are_more=True
while there_are_more:
ed = ExperimentDefinition()
if trace_id:
ed.load(db_obj, trace_id)
ed.mark_pre_analyzing(db_obj)
else:
there_are_more = ed.load_pending(db_obj)
if there_are_more:
print(("Analyzing experiment {0}".format(ed._trace_id)))
er = AnalysisRunnerSingle(ed)
er.do_full_analysis(db_obj)
if trace_id:
break
def compare_traces_jobs(db_obj, t1, t2, num_jobs, workflows=False):
if not workflows:
jobs1 = get_jobs(db_obj, t1, num_jobs)
jobs2 = get_jobs(db_obj, t2, num_jobs)
else:
jobs1 = get_workflow_jobs(db_obj, t1, num_jobs)
jobs2 = get_workflow_jobs(db_obj, t2, num_jobs)
exp1 = ExperimentDefinition()
exp1.load(db_obj, t1)
exp2 = ExperimentDefinition()
exp2.load(db_obj, t2)
different = False
for (job1, job2) in zip(jobs1, jobs2):
if job1 != job2:
different = True
break
if not different and exp1._seed != exp2._seed:
print(
"Exps({0},{1}) have the exact same first {3} jobs with different"
" seeds.".format(t1, t2, num_jobs))
return False
return True
def check_trace_ok(db_obj, trace_id):
exp = ExperimentDefinition()
exp.load(db_obj, trace_id)
workflow_names = [x["manifest"] for x in exp._manifest_list]
job_list = get_workflow_jobs(db_obj, trace_id, 10)
if len(job_list) == 0 and exp._workflow_policy != "no":
print("Exp({0}) should have worklfow jobs (0 found)".format(trace_id))
return False
if exp._workflow_handling == "single":
for job in job_list:
name, stage_id, deps = TaskTracker.extract_wf_name(job["job_name"])
if not name.split("-")[0] in workflow_names:
print(
"Exp({0}) uses a workflow that should not be there"
" job({1}): {2} vs {3}"
"".format(trace_id, job["id_job"], job["job_name"],
workflow_names))
return False
if stage_id != "" or deps != []:
print(
"Exp({0}) is a single job workflows, and job({1}) is {2}"
"".format(trace_id, job["id_job"], job["job_name"]))
return False
elif exp._workflow_handling == "manifest":
first_job = job_list[0]
first_name, first_stage_id, first_deps = TaskTracker.extract_wf_name(
first_job["job_name"])
if not first_name.split("-")[0] in workflow_names:
print(
"Exp({0}) uses a workflow that should not be there"
" job({1}): {2} vs {3}"
"".format(trace_id, first_job["id_job"], first_job["job_name"],
workflow_names))
return False
if first_stage_id != "" or first_deps != []:
print(
"Exp({0}) uses manifests, the first job of the workflow"
" ({1} should not have dependencies: {2}"
"".format(trace_id, first_job["id_job"],
first_job["job_name"]))
return False
other_tasks_found = False
for job in job_list[1:]:
name, stage_id, deps = TaskTracker.extract_wf_name(job["job_name"])
if name == first_name and stage_id != "":
other_tasks_found = True
break
if not other_tasks_found:
print(
"Exp({0}) uses manifests, however the tasks for the worklow "
"are not there: {1}".format(trace_id, first_job["job_name"]))
return False
elif exp._workflow_handling == "multi":
for job in job_list:
name, stage_id, deps = TaskTracker.extract_wf_name(job["job_name"])
if not name.split("-")[0] in workflow_names:
print(
"Exp({0}) uses a workflow that should not be there"
" job({1}): {2} vs {3}"
"".format(trace_id, job["id_job"], job["job_name"],
workflow_names))
return False
if stage_id == "":
print(
"Exp({0}) uses dependencies, however the tasks for "
"the worklow are not there: {1}".format(
trace_id, job["job_name"]))
return False
else:
print("Exp({0}) unknown worklfow handling: {1}".format(
trace_id, exp._workflow_handling))
return False
print("Exp({0}) is correct".format(trace_id))
return True
for trace_id in exp_count_multi[key]:
if (check_trace_ok(db_obj, trace_id)):
good.append(trace_id)
else:
bad.append(trace_id)
if good > 1 and not do_deep:
all_suspicious_pairs += check_all_good(db_obj, good, 200)
all_good += good
all_bad += bad
print("""Final result:""")
print("""Good traces({1}): {0}""".format(" ".join([str(x) for x in all_good]),
len(all_good)))
print("""Bad traces({1}): {0}""".format(" ".join([str(x) for x in all_bad]),
len(all_bad)))
print("""Suspicios pairs({1}): {0}""".format(
" ".join([str(x) for x in all_suspicious_pairs]),
len(all_suspicious_pairs)))
theval = raw_input(
"Press Enter to continue and reset those experiments that are not ok: "
"{0}".format(" ".join([str(x) for x in all_bad])))
for bad_trace_id in all_bad:
print("Cleaning experiment trace and results: {0}".format(bad_trace_id))
ed = ExperimentDefinition()
ed.load(db_obj, bad_trace_id)
ed.del_trace(db_obj)
ed.del_results(db_obj)
ed.upate_state(db_obj, "fresh")
if len(sys.argv)>=3:
new_state = sys.argv[2]
if len(sys.argv)==4:
trace_id=int(sys.argv[3])
else:
trace_id=None
central_db_obj = get_central_db()
print("Reseting experiments in state {0}".format(state))
there_are_more = True
while (there_are_more):
ed = ExperimentDefinition()
if trace_id is not None:
ed.load(central_db_obj, trace_id)
if ed._work_state!=state:
print("Error, unexpected state {0} for trace {1}".format(
ed._work_state,
trace_id))
exit()
ed.upate_state(central_db_obj, new_state)
there_are_more=False
else:
there_are_more=ed.load_next_state(central_db_obj, state, new_state)
if ed._trace_id!=None:
print("Reset experiment({0}: {1}): {2} -> {3}".format(ed._trace_id,
ed._name,
state,
new_state))
if new_state == "fresh":
from stats.trace import ResultTrace
matplotlib.use('Agg')
db_obj = get_central_db()
target_dir = "utilization-20160616-udog"
if len(sys.argv) == 3:
target_dir = sys.argv[1]
trace_id = sys.argv[2]
else:
raise ValueError("Missing trace id to analyze")
exp = ExperimentDefinition()
exp.load(db_obj, trace_id)
rt = ResultTrace()
rt.load_trace(db_obj, trace_id)
machine = exp.get_machine()
max_cores = machine.get_total_cores()
max_submit_time = rt._lists_submit["time_submit"][-1]
def adjust_ut_plot(ut_stamps, ut_values):
new_stamps = []
new_values = []
last_value = None
for (st, vl) in zip(ut_stamps, ut_values):
if last_value is not None:
db_obj = get_central_db()
if len(sys.argv) < 2:
raise ValueError("At least one argument must specified with the trace_id"
" to plot.")
trace_id = int(sys.argv[1])
arg_name = None
dest_dir = "./out"
if not (os.path.exists(dest_dir)):
os.makedirs(dest_dir)
if len(sys.argv) == 3:
arg_name = sys.argv[2]
ed = ExperimentDefinition()
ed.load(db_obj, trace_id)
rt = ResultTrace()
rt.load_analysis(db_obj, trace_id)
if arg_name is None:
arg_name = ed._name
for (key, result) in rt.jobs_results.iteritems():
if "_cdf" in key:
bins, edges = result.get_data()
histogram_cdf(edges,
bins,
key,
file_name=arg_name + "-" + key,
x_axis_label=key,
y_axis_label="Norm share",
target_folder=dest_dir,
def do_work_second_pass(self, db_obj, pre_trace_id):
"""Takes three experiments, and repeast the workflow analysis for
each experiment but only taking into account the first n workflows
in each one. n = minimum number of workflows acroos the three traces.
Args:
- db_obj: DB object configured to access the analysis database.
- trace_id: If set to an integer, it will analyze the
experiments identified by trace_id, trace_id+1, trace_id+2.
"""
there_are_more=True
while there_are_more:
ed_manifest = ExperimentDefinition()
ed_single = ExperimentDefinition()
ed_multi = ExperimentDefinition()
if pre_trace_id:
trace_id=int(pre_trace_id)
ed_manifest.load(db_obj, trace_id)
there_are_more=True
else:
there_are_more = ed_manifest.load_next_ready_for_pass(db_obj)
trace_id=int(ed_manifest._trace_id)
if there_are_more:
ed_single.load(db_obj, trace_id+1)
ed_multi.load(db_obj, trace_id+2)
ed_list=[ed_manifest, ed_single, ed_multi]
print(("Reading workflow info for traces: {0}".format(
[ed._trace_id for ed in ed_list])))
if (ed_manifest._workflow_handling!="manifest" or
ed_single._workflow_handling!="single" or
ed_multi._workflow_handling!="multi"):
print(("Incorrect workflow handling for traces"
"({0}, {1}, {2}): ({3}, {4}, {5})",format(
ed_manifest._trace_id,
ed_single._trace_id,
ed_multi._trace_id,
ed_manifest._workflow_handling,
ed_single._workflow_handling,
ed_multi._workflow_handling)
))
print ("Exiting...")
exit()
for ed in ed_list:
ed.mark_pre_second_pass(db_obj)
num_workflows=None
for ed in ed_list:
exp_wfs=self.get_num_workflows(db_obj, ed._trace_id)
if num_workflows is None:
num_workflows = exp_wfs
else:
num_workflows=min(num_workflows, exp_wfs)
print(("Final workflow count: {0}".format(num_workflows)))
for ed in ed_list:
print(("Doing second pass for trace: {0}".format(
ed._trace_id)))
er = AnalysisRunnerSingle(ed)
er.do_workflow_limited_analysis(db_obj, num_workflows)
print(("Second pass completed for {0}".format(
[ed._trace_id for ed in ed_list])))
if pre_trace_id:
break
matplotlib.use('Agg')
db_obj = get_central_db()
base_trace_id_percent, lim = get_args(2459, True)
print("Base Exp", base_trace_id_percent)
print("Using analysis of limited workflows:", lim)
edge_keys= {0: "[0,48] core.h", 48*3600:"(48, 960] core.h",
960*3600:"(960, inf.) core.h"}
trace_id_rows = []
base_exp=170
exp=ExperimentDefinition()
exp.load(db_obj, base_exp)
core_seconds_edges=exp.get_machine().get_core_seconds_edges()
trace_id_rows= gen_trace_ids_exps(base_trace_id_percent,
inverse=False,
group_jump=18, block_count=6,
base_exp_group=None)
time_labels = ["", "10%", "", "", "25%", "",
"", "50%", "", "", "75%", "",
"", "100%", ""]
manifest_label=["floodP", "longW", "wideL",
"cybers", "sipht", "montage"]
class StartTimeCorrector(object):
def __init__(self):
self.manifest_dics = {}
def correct_times(self, db_obj, trace_id):
self._experiment = ExperimentDefinition()
self._experiment.load(db_obj, trace_id)
self._trace = ResultTrace()
print("Loading trace {0}".format(trace_id))
self._trace.load_trace(db_obj, trace_id)
trace_type = self._experiment._workflow_handling
print(
"Calculating corrected start times for trace {0}".format(trace_id))
modified_start_times = self.get_corrected_start_times(trace_type)
print(("Found {0} jobs which start time was 0, but had ended.".format(
len(modified_start_times))))
print("About to update times")
self.apply_new_times(db_obj, modified_start_times)
def apply_new_times(self, db_obj, modified_start_times):
trace_id = self._experiment._trace_id
for id_job in list(modified_start_times.keys()):
time_start = modified_start_times[id_job]
print(("updating trace_id({0}), id_job({1}) with time_start: {2}"
"".format(trace_id, id_job, time_start)))
self.update_time_start(db_obj, trace_id, id_job, time_start)
def update_time_start(self, db_obj, trace_id, id_job, time_start):
"""
query =("update traces set time_start={0} where trace_id={1} and "
" id_job={2}",format(time_start, trace_id, id_job))
"""
db_obj.setFieldOnTable("traces",
"time_start",
str(time_start),
"id_job",
str(id_job),
extra_cond="and trace_id={0}".format(trace_id),
no_commas=True)
def get_corrected_start_times(self, trace_type):
modified_start_times = {}
for (id_job, job_name, time_submit, time_start,
time_end) in zip(self._trace._lists_submit["id_job"],
self._trace._lists_submit["job_name"],
self._trace._lists_submit["time_submit"],
self._trace._lists_submit["time_start"],
self._trace._lists_submit["time_end"]):
if time_start == 0 and time_end != 0 and "wf" == job_name[:2]:
modified_start_times[id_job] = self.get_time_start(
job_name, time_end, trace_type)
return modified_start_times
def get_workflow_info(self, workflow_file):
if not workflow_file in list(self.manifest_dics.keys()):
from orchestration.running import ExperimentRunner
manifest_route = path.join(ExperimentRunner.get_manifest_folder(),
workflow_file)
cores, runtime, tasks = WorkflowGeneratorMultijobs.parse_all_jobs(
manifest_route)
self.manifest_dics[workflow_file] = {
"cores": cores,
"runtime": runtime,
"tasks": tasks
}
return self.manifest_dics[workflow_file]
def get_time_start(self, job_name, time_end, trace_type="manifest"):
name, stage_id, deps = TaskTracker.extract_wf_name(job_name)
workflow_file = "-".join(name.split("-")[:-1])
from orchestration import ExperimentRunner
manifest_info = self.get_workflow_info(workflow_file)
(cores, runtime,
tasks) = (manifest_info["cores"], manifest_info["runtime"],
manifest_info["tasks"])
if stage_id is "":
if trace_type == "multi":
raise SystemError("Found a bounding job ({0}) in a "
"dependencies type trace.".format(job_name))
if trace_type == "manifest":
return time_end
else:
return time_end - runtime
else:
return time_end - int(tasks[stage_id]["runtime_sim"])
@classmethod
def get_traces_with_bad_time_starts(cls, db_obj):
query = """
SELECT traces.trace_id tid, name, count(*) cc
FROM traces, experiment
WHERE traces.trace_id=experiment.trace_id
AND time_start=0 AND time_end!=0
AND job_name!="sim_job"
group by traces.trace_id
"""
result_list = db_obj.doQueryDic(query)
trace_id_list = [res["tid"] for res in result_list]
return trace_id_list