def rescue_exp(self, central_db_obj, sched_db_obj, trace_id=None):
"""Retrieves the job trace from the database of an experiment worker and
stores it in the central db.
Args:
- central_db_obj: DB object configured to access the analysis database.
- sched_db_obj: DB object configured to access the slurm database of
an experiment worker.
- trace_id: trace_id of the experiment to which the rescued trace
corresponds.
"""
there_are_more=True
while there_are_more:
ed = ExperimentDefinition()
if trace_id:
ed.load(central_db_obj, trace_id)
ed.mark_simulation_done(central_db_obj)
else:
there_are_more = ed.load_next_state("simulation_failed",
"simulation_done")
if there_are_more:
print(("About to run resque({0}):{1}".format(
ed._trace_id, ed._name)))
er = ExperimentRunner(ed)
if(er.check_trace_and_store(sched_db_obj, central_db_obj)):
er.clean_trace_file()
print(("Exp({0}) Done".format(
ed._trace_id)))
else:
print(("Exp({0}) Error!".format(
ed._trace_id)))
if trace_id:
break
def test_place_trace_files_remote_and_clean(self):
ExperimentRunner.configure("/tmp/tests/tmp/dest",
"/tmp/tests/tmp/orig",
True,
"locahost",
None,
scheduler_folder="/tmp/tests/tmp/sched",
scheduler_conf_dir="/tmp/tests/tmp/conf",
manifest_folder="manifests")
self.assertEqual(ExperimentRunner._trace_folder, "/tmp/tests/tmp/dest")
self.assertEqual(ExperimentRunner._trace_generation_folder,
"/tmp/tests/tmp/orig")
self.assertEqual(ExperimentRunner._local, True)
ensureDir("/tmp/tests/tmp/dest")
ensureDir("/tmp/tests/tmp/orig")
ensureDir("/tmp/tests/tmp/sched")
ensureDir("/tmp/tests/tmp/conf")
ed = ExperimentDefinition(seed="seeeed",
machine="edison",
trace_type="single",
manifest_list=[{
"share": 1.0,
"manifest": "manifestSim.json"
}],
workflow_policy="period",
workflow_period_s=5,
workflow_handling="single",
preload_time_s=20,
start_date=datetime(2016, 1, 1),
workload_duration_s=41,
overload_target=1.1)
er = ExperimentRunner(ed)
filenames = er._generate_trace_files(ed)
er._place_trace_file(filenames[0])
er._place_users_file(filenames[2])
self.assertTrue(
os.path.exists(
"/tmp/tests/tmp/dest/edison-single-m1.0manifestSim.json"
"-period-p5-0.0-single-t-0d-0d-O1.1"
"-sseeeed.trace"))
self.assertTrue(os.path.exists("/tmp/tests/tmp/conf/users.sim"))
self.assertFalse(
os.path.exists(
"/tmp/tests/tmp/orig/edison-single-m1.0manifestSim.json"
"-period-p5-0.0-single-t-0d-0d-O1.1"
"-sseeeed.trace"))
er.clean_trace_file()
self.assertFalse(
os.path.exists(
"/tmp/tests/tmp/dest/edison-single-m1.0manifestSim.json"
"-period-p5-0.0-single-t-0d-0d-O1.1"
"-sseeeed.trace"))
self.assertFalse(
os.path.exists(
"/tmp/tests/tmp/dest/edison-single-m1.0manifestSim.json"
"-period-p5-0.0-single-t-0d-0d-O1.1"
"-sseeeed.users"))