def rescue_exp(self, central_db_obj, sched_db_obj, trace_id=None):
"""Retrieves the job trace from the database of an experiment worker and
stores it in the central db.
Args:
- central_db_obj: DB object configured to access the analysis database.
- sched_db_obj: DB object configured to access the slurm database of
an experiment worker.
- trace_id: trace_id of the experiment to which the rescued trace
corresponds.
"""
there_are_more=True
while there_are_more:
ed = ExperimentDefinition()
if trace_id:
ed.load(central_db_obj, trace_id)
ed.mark_simulation_done(central_db_obj)
else:
there_are_more = ed.load_next_state("simulation_failed",
"simulation_done")
if there_are_more:
print(("About to run resque({0}):{1}".format(
ed._trace_id, ed._name)))
er = ExperimentRunner(ed)
if(er.check_trace_and_store(sched_db_obj, central_db_obj)):
er.clean_trace_file()
print(("Exp({0}) Done".format(
ed._trace_id)))
else:
print(("Exp({0}) Error!".format(
ed._trace_id)))
if trace_id:
break
def test_is_it_ready_to_process_delta(self):
ed = ExperimentDefinition()
self.addCleanup(self._del_table, "experiment")
ed.create_table(self._db)
t1 = ExperimentDefinition()
id1 = t1.store(self._db)
t2 = ExperimentDefinition()
id2 = t2.store(self._db)
t3 = DeltaExperimentDefinition(subtraces=[id1, id2])
t3.store(self._db)
self.assertFalse(
t3.is_it_ready_to_process(self._db), "The subtraces"
" are still pending, it should not be possible to"
" process it.")
t1.mark_simulation_done(self._db)
self.assertFalse(
t3.is_it_ready_to_process(self._db), "One subtrace"
" is still pending, it should not be possible to"
" process it.")
t2.mark_simulation_done(self._db)
self.assertTrue(
t3.is_it_ready_to_process(self._db), "Subtraces "
"are genreated, t3, should be ready to run.")
def test_is_it_ready_to_process(self):
ed = ExperimentDefinition()
self.addCleanup(self._del_table, "experiment")
ed.create_table(self._db)
t1 = ExperimentDefinition()
id1 = t1.store(self._db)
t2 = ExperimentDefinition()
id2 = t2.store(self._db)
t3 = GroupExperimentDefinition(subtraces=[id1, id2])
t3.store(self._db)
self.assertFalse(
t3.is_it_ready_to_process(self._db), "The subtraces"
" are still pending, it should not be possible to"
" process it.")
t1.mark_simulation_done(self._db)
self.assertFalse(
t3.is_it_ready_to_process(self._db), "One subtrace"
" is still pending, it should not be possible to"
" process it.")
t2.mark_simulation_done(self._db)
self.assertFalse(
t3.is_it_ready_to_process(self._db), "Subtraces "
"have to be analyzed for this the grouped to be "
"ready")
t1.mark_analysis_done(self._db)
t2.mark_analysis_done(self._db)
self.assertTrue(t3.is_it_ready_to_process(self._db), "Subtraces "
"are analyzed. It should be ready")
def test_mark_simulation_done(self):
ed = ExperimentDefinition()
self.addCleanup(self._del_table, "experiment")
ed.create_table(self._db)
my_id = ed.store(self._db)
ed.mark_simulation_done(self._db)
now_time = datetime.datetime.now()
new_ed = ExperimentDefinition()
new_ed.load(self._db, my_id)
self.assertEqual(new_ed._work_state, "simulation_done")
self.assertLess(now_time - new_ed._simulating_end,
datetime.timedelta(10))