def test_pickle_serializable_experiment_success(self): experiment = PickleOnlySerializableExeperiment() runner = ExperimentRunner() specification = {"test": "test"} runner.run("test", [specification], experiment, specification_runner=MainRunner()) self.assertIn("run.pkl", os.listdir(get_save_file_directory('test', specification))) self.assertIn("specification.json", os.listdir(get_save_file_directory('test', specification)))
def test_checkpoint_handler_rotates_checkpoints_properly(self): experiment = SerializableExperimentFailsAfter4Steps() runner = ExperimentRunner() specification = {"test": "test"} runner.run("test", [specification], experiment, specification_runner=MainRunner()) self.assertEqual(3, len(os.listdir(get_partial_save_directory("test", specification)))) partial_experiment = CheckpointedExperimentHandler().load_most_recent("test", specification) self.assertEqual(partial_experiment.j, 3)
def test_with_runner(self): experiment = SerializableExperiment() runner = ExperimentRunner() specification = {"test": "test"} runner.run("test", [specification], experiment, specification_runner=MainRunner()) self.assertEqual( 1, len(os.listdir(get_save_file_directory('test', specification))))
def testmain(self): # Same specification as before generation_specification = {"seed": [1, 2, 3, 4, 5, 6, 7, 8], "num_calls": [[10, 20, 30]]} specifications = SpecificationGenerator().generate(generation_specification) output_generation_specification = {"seed": [1, 2, 3, 4, 5, 6, 7, 8], "num_calls": [10, 20, 30]} output_specifications = SpecificationGenerator().generate(output_generation_specification) name = "test" # This time we will run them all in parallel runner = ExperimentRunner() runner.run(name, specifications, SimpleExperiment(), specification_runner=MainRunner(), use_dashboard=False, propagate_exceptions=True) for result in experiment_iterator(name): if result["result"] != []: output_specifications.remove(result["specification"]) self.assertEqual([],output_specifications)
f"Expt {name}:\t{len(specifications)/num_seeds} specs to run, over {num_seeds} seeds" ) for spec in specifications: if spec["seed"] == 0: print(spec) runner = ExperimentRunner() map_memory(base_specs["file"], base_specs["state_space_dimensionality"]) DEBUG = False if DEBUG: runner.run(name, specifications, PlanningExperiment(), propagate_exceptions=True, specification_runner=MainRunner(), use_dashboard=False, force_pickle=True, context_type="fork") else: gpus = 4 jobs_per_gpu = 2 resources = list(product(list(range(gpus)), list(range(jobs_per_gpu)))) runner.run(name, specifications, PlanningExperiment(), propagate_exceptions=False, specification_runner=MultiprocessingRunner(), context_type="fork", use_dashboard=True, force_pickle=True)
def test_un_serializable_experiment_failure(self): experiment = UnserializableExperiment() runner = ExperimentRunner() specification = {"test": "test"} runner.run("test", [specification], experiment, specification_runner=MainRunner()) self.assertEqual(0, len(os.listdir(get_save_file_directory('test', specification, runner.diff_namer))))
visualizer.tSNE(prefix_str+'tsne.png') logging.getLogger(self.get_logger_name()).info("Visualization complete.") return {"discriminative score mean": results[0], "predictive score mean": results[2]} def get_hash(self): return self.get_logger_name() # In the generation specification keys that have lists as their values will be cross producted with other list valued keys to create many specifications # in this instance there will be 8 * 3 = 24 specifications generation_specification = { # just trying out random values for testing "total_iterations": [1], "sub_iterations": [2], "data_size": [300], "max_seq_length": [12], "iterations": [10001], "batch_size": [128], # "module_name": ['gru', 'lstm', 'lstmLN'] } # Call the generate method. Will create the cross product. specifications = SpecificationGenerator().generate(generation_specification) print(specifications) expt = TsganExperiment() name = "tsgan_unseen_metrics" #+expt.get_hash() runner = ExperimentRunner() runner.run(name, specifications, expt, specification_runner=MainRunner(), propagate_exceptions=True)
"seed": list(range(5)), "alpha_param": 6, "beta_param":1, "epsilon": 10, "delta": 0.1, "plan_commitment_algorithm": "n_steps", "plan_threshold": [1], "sample_observations": False, "use_expected_improvement":False, "planning_steps": 200 } ##Create shared memory map_memory(generation_specifications["file"], generation_specifications["state_space_dimensionality"]) specifications = SpecificationGenerator().generate(generation_specifications) runner = ExperimentRunner() DEBUG = False if DEBUG: runner.run(name, specifications, PlanningExperiment(), propagate_exceptions=True, specification_runner=MainRunner(), use_dashboard=False, force_pickle=True, context_type="fork") else: runner.run(name, specifications, PlanningExperiment(), propagate_exceptions=False, specification_runner=MultiprocessingRunner(), context_type="fork", use_dashboard=True, force_pickle=True)