def rescue_exp(self, central_db_obj, sched_db_obj, trace_id=None): """Retrieves the job trace from the database of an experiment worker and stores it in the central db. Args: - central_db_obj: DB object configured to access the analysis database. - sched_db_obj: DB object configured to access the slurm database of an experiment worker. - trace_id: trace_id of the experiment to which the rescued trace corresponds. """ there_are_more=True while there_are_more: ed = ExperimentDefinition() if trace_id: ed.load(central_db_obj, trace_id) ed.mark_simulation_done(central_db_obj) else: there_are_more = ed.load_next_state("simulation_failed", "simulation_done") if there_are_more: print(("About to run resque({0}):{1}".format( ed._trace_id, ed._name))) er = ExperimentRunner(ed) if(er.check_trace_and_store(sched_db_obj, central_db_obj)): er.clean_trace_file() print(("Exp({0}) Done".format( ed._trace_id))) else: print(("Exp({0}) Error!".format( ed._trace_id))) if trace_id: break
def test_is_it_ready_to_process_delta(self): ed = ExperimentDefinition() self.addCleanup(self._del_table, "experiment") ed.create_table(self._db) t1 = ExperimentDefinition() id1 = t1.store(self._db) t2 = ExperimentDefinition() id2 = t2.store(self._db) t3 = DeltaExperimentDefinition(subtraces=[id1, id2]) t3.store(self._db) self.assertFalse( t3.is_it_ready_to_process(self._db), "The subtraces" " are still pending, it should not be possible to" " process it.") t1.mark_simulation_done(self._db) self.assertFalse( t3.is_it_ready_to_process(self._db), "One subtrace" " is still pending, it should not be possible to" " process it.") t2.mark_simulation_done(self._db) self.assertTrue( t3.is_it_ready_to_process(self._db), "Subtraces " "are genreated, t3, should be ready to run.")
def test_is_it_ready_to_process(self): ed = ExperimentDefinition() self.addCleanup(self._del_table, "experiment") ed.create_table(self._db) t1 = ExperimentDefinition() id1 = t1.store(self._db) t2 = ExperimentDefinition() id2 = t2.store(self._db) t3 = GroupExperimentDefinition(subtraces=[id1, id2]) t3.store(self._db) self.assertFalse( t3.is_it_ready_to_process(self._db), "The subtraces" " are still pending, it should not be possible to" " process it.") t1.mark_simulation_done(self._db) self.assertFalse( t3.is_it_ready_to_process(self._db), "One subtrace" " is still pending, it should not be possible to" " process it.") t2.mark_simulation_done(self._db) self.assertFalse( t3.is_it_ready_to_process(self._db), "Subtraces " "have to be analyzed for this the grouped to be " "ready") t1.mark_analysis_done(self._db) t2.mark_analysis_done(self._db) self.assertTrue(t3.is_it_ready_to_process(self._db), "Subtraces " "are analyzed. It should be ready")
def test_mark_simulation_done(self): ed = ExperimentDefinition() self.addCleanup(self._del_table, "experiment") ed.create_table(self._db) my_id = ed.store(self._db) ed.mark_simulation_done(self._db) now_time = datetime.datetime.now() new_ed = ExperimentDefinition() new_ed.load(self._db, my_id) self.assertEqual(new_ed._work_state, "simulation_done") self.assertLess(now_time - new_ed._simulating_end, datetime.timedelta(10))