def testmain(self): # Same specification as before generation_specification = {"seed": [1, 2, 3, 4, 5, 6, 7, 8], "num_calls": [[10, 20, 30]]} specifications = SpecificationGenerator().generate(generation_specification) output_generation_specification = {"seed": [1, 2, 3, 4, 5, 6, 7, 8], "num_calls": [10, 20, 30]} output_specifications = SpecificationGenerator().generate(output_generation_specification) name = "test" # This time we will run them all in parallel runner = ExperimentRunner() expr = SimpleExperiment() runner.run(name, specifications, expr, specification_runner=MultiprocessingRunner(), use_dashboard=True, propagate_exceptions=True,context_type="spawn") log_base = os.path.join("experiment_runs",name,"logs") for root, dirs, files in os.walk(log_base): for file in files: with open(os.path.join(root,file),"r") as f: lines = f.readlines() self.assertNotEqual([],lines) for result in experiment_iterator(name): if result["result"] != []: output_specifications.remove(result["specification"]) self.assertEqual([],output_specifications)
def testmain(self): # Same specification as before generation_specification = { "seed": [1, 2, 3, 4, 5, 6, 7, 8], "num_calls": [[10, 20, 30]] } specifications = SpecificationGenerator().generate( generation_specification) output_generation_specification = { "seed": [1, 2, 3, 4, 5, 6, 7, 8], "num_calls": [10, 20, 30] } output_specifications = SpecificationGenerator().generate( output_generation_specification) name = "test" # This time we will run them all in parallel runner = ExperimentRunner() runner.run(name, specifications, SimpleExperiment(), specification_runner=MultiprocessingRunner(), use_dashboard=False, propagate_exceptions=True) for result in experiment_iterator(name): if result["result"] != []: output_specifications.remove(result["specification"]) self.assertEqual([], output_specifications)
"beta_param": 1, "epsilon": 10, "delta": 0.1, "sample_observations": False, "use_expected_improvement": False, "planning_steps": [200], "rollout_allocation_method": ["fixed"], "waste_unused_rollouts": [False], } gen_baseline = base_specs.copy() gen_baseline.update({ "plan_commitment_algorithm": "n_steps", "plan_threshold": [1], }) specs_baseline = SpecificationGenerator().generate(gen_baseline) gen_ugapec = base_specs.copy() gen_ugapec.update({ "plan_commitment_algorithm": "ugapec", # 1000 should allow for more points to added to self.current_traj. This means fewer rollouts "plan_threshold": [5, 10], # [1, 5, 10, 100, 1000], }) specs_ugapec = SpecificationGenerator().generate(gen_ugapec) gen_tTest = base_specs.copy() gen_tTest.update({ "plan_commitment_algorithm": "tTest", # p-value "plan_threshold": [0.05, 0.1], # [0.0, 0.25, 0.5], })
raise Exception( "Something bad happened, a moth flew into the computer!") if self.i >= self.num_calls: #Done with the experiment, return the results dictionary like normal return {"number": self.r} else: #This experiment isn't done, return the progress as a tuple to update the dashboard return (self.i, self.num_calls) #Same specification as before generation_specification = { "seed": [1, 2, 3, 4, 5, 6, 7, 8], "num_calls": [1, 2, 3] } specifications = SpecificationGenerator().generate(generation_specification) name = "checkpointed_run" #This time we will run them all in parallel runner = ExperimentRunner() runner.run(name, specifications, SimpleExperiment(), specification_runner=MultiprocessingRunner()) #Some of our experiments may have failed, let's call run again to hopefully solve that runner.run(name, specifications, SimpleExperiment(), specification_runner=MultiprocessingRunner())
for root, _, files in os.walk(get_experiment_save_directory("random_number")): for fname in files: if ".json" in fname: with open(os.path.join(root, fname), "r") as f: results = json.load(f) print(results["specification"]["seed"]) print(results["result"]["number"]) from smallab.specification_generator import SpecificationGenerator # If you want to run a lot of experiments but not manual write out each one, use the specification generator. # Note: This is also JSON serializable, so you could store this in a json file generation_specification = {"seed": [1, 2, 3, 4, 5, 6, 7, 8], "num_calls": [1, 2, 3]} # Call the generate method. Will create the cross product. specifications = SpecificationGenerator().generate(generation_specification) print(specifications) runner.run("random_number_from_generator", specifications, SimpleExperiment(), continue_from_last_run=True) # Read back our results for root, _, files in os.walk(get_save_directory("random_number_from_generator")): for fname in files: if ".pkl" in fname: with open(os.path.join(root, fname), "rb") as f: results = dill.load(f) print(results["specification"]["seed"]) print(results["result"]["number"]) # If you have an experiment you want run on a lot of computers you can use the MultiComputerGenerator # You assign each computer a number from 0..number_of_computers-1 and it gives each computer every number_of_computerth specification
"beta_param": 1, "epsilon": 10, "delta": 0.1, "sample_observations": False, "use_expected_improvement": False, "planning_steps": [num_steps], } gen_baseline = base_specs.copy() gen_baseline.update({ "plan_commitment_algorithm": "n_steps", "plan_threshold": [1], "rollout_allocation_method": ["fixed"], "waste_unused_rollouts": [False], }) specs_baseline = SpecificationGenerator().generate(gen_baseline) gen_our_best = base_specs.copy() gen_our_best.update({ "plan_commitment_algorithm": "tTest", "plan_threshold": [0.05], "rollout_allocation_method": ["beta-ugapeb"], "waste_unused_rollouts": [True], }) specs_our_best = SpecificationGenerator().generate(gen_our_best) specifications = [] specifications += specs_baseline specifications += specs_our_best print(