def rescue_exp(self, central_db_obj, sched_db_obj, trace_id=None): """Retrieves the job trace from the database of an experiment worker and stores it in the central db. Args: - central_db_obj: DB object configured to access the analysis database. - sched_db_obj: DB object configured to access the slurm database of an experiment worker. - trace_id: trace_id of the experiment to which the rescued trace corresponds. """ there_are_more=True while there_are_more: ed = ExperimentDefinition() if trace_id: ed.load(central_db_obj, trace_id) ed.mark_simulation_done(central_db_obj) else: there_are_more = ed.load_next_state("simulation_failed", "simulation_done") if there_are_more: print(("About to run resque({0}):{1}".format( ed._trace_id, ed._name))) er = ExperimentRunner(ed) if(er.check_trace_and_store(sched_db_obj, central_db_obj)): er.clean_trace_file() print(("Exp({0}) Done".format( ed._trace_id))) else: print(("Exp({0}) Error!".format( ed._trace_id))) if trace_id: break
def test_place_trace_files_remote_and_clean(self): ExperimentRunner.configure("/tmp/tests/tmp/dest", "/tmp/tests/tmp/orig", True, "locahost", None, scheduler_folder="/tmp/tests/tmp/sched", scheduler_conf_dir="/tmp/tests/tmp/conf", manifest_folder="manifests") self.assertEqual(ExperimentRunner._trace_folder, "/tmp/tests/tmp/dest") self.assertEqual(ExperimentRunner._trace_generation_folder, "/tmp/tests/tmp/orig") self.assertEqual(ExperimentRunner._local, True) ensureDir("/tmp/tests/tmp/dest") ensureDir("/tmp/tests/tmp/orig") ensureDir("/tmp/tests/tmp/sched") ensureDir("/tmp/tests/tmp/conf") ed = ExperimentDefinition(seed="seeeed", machine="edison", trace_type="single", manifest_list=[{ "share": 1.0, "manifest": "manifestSim.json" }], workflow_policy="period", workflow_period_s=5, workflow_handling="single", preload_time_s=20, start_date=datetime(2016, 1, 1), workload_duration_s=41, overload_target=1.1) er = ExperimentRunner(ed) filenames = er._generate_trace_files(ed) er._place_trace_file(filenames[0]) er._place_users_file(filenames[2]) self.assertTrue( os.path.exists( "/tmp/tests/tmp/dest/edison-single-m1.0manifestSim.json" "-period-p5-0.0-single-t-0d-0d-O1.1" "-sseeeed.trace")) self.assertTrue(os.path.exists("/tmp/tests/tmp/conf/users.sim")) self.assertFalse( os.path.exists( "/tmp/tests/tmp/orig/edison-single-m1.0manifestSim.json" "-period-p5-0.0-single-t-0d-0d-O1.1" "-sseeeed.trace")) er.clean_trace_file() self.assertFalse( os.path.exists( "/tmp/tests/tmp/dest/edison-single-m1.0manifestSim.json" "-period-p5-0.0-single-t-0d-0d-O1.1" "-sseeeed.trace")) self.assertFalse( os.path.exists( "/tmp/tests/tmp/dest/edison-single-m1.0manifestSim.json" "-period-p5-0.0-single-t-0d-0d-O1.1" "-sseeeed.users"))