def setUp(self): launchpad.reset(password=None, require_password=False) self.mc = MissionControl(**db_info) self.mc.reset(hard=True) self.db = launchpad.db self.dims_basic = [(1, 10), (10.0, 20.0), ['blue', 'green', 'red', 'orange']] self.dims_accuracy = [(1.0, 10.0), (10.0, 20.0), (20.0, 30.0)] self.dims_parallel = [(1, 5), (1, 5), (1, 5)] self.c = getattr(self.db, opt_label)
integers between 1 and 5 (inclusive). Returns: (Workflow): A workflow containing one FireWork (two FireTasks) which is automatically set up to run the optimization loop. """ spec = {'_x': x} # ObjectiveFuncTask writes _y field to the spec internally. firework1 = Firework([ObjectiveFuncTask(), OptTask(**db_info)], spec=spec) return Workflow([firework1]) if __name__ == "__main__": # Make a MissionControl object mc = MissionControl(**db_info) # Reset the launchpad and optimization db for this example launchpad.reset(password=None, require_password=False) mc.reset(hard=True) # Configure the optimization db with MissionControl mc.configure(wf_creator=wf_creator, dimensions=x_dim) # Run the optimization loop 10 times. launchpad.add_wf(wf_creator([5, 5, 2])) rapidfire(launchpad, nlaunches=10, sleep_time=0) # Examine results plt = mc.plot() plt.show()
class TestWorkflows(unittest.TestCase): def setUp(self): launchpad.reset(password=None, require_password=False) self.mc = MissionControl(**db_info) self.mc.reset(hard=True) self.db = launchpad.db self.dims_basic = [(1, 10), (10.0, 20.0), ['blue', 'green', 'red', 'orange']] self.dims_accuracy = [(1.0, 10.0), (10.0, 20.0), (20.0, 30.0)] self.dims_parallel = [(1, 5), (1, 5), (1, 5)] self.c = getattr(self.db, opt_label) def test_basic(self): self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, **common_kwargs) launchpad.add_wf(wf_creator_basic([5, 11, 'blue'])) launch_rocket(launchpad) manager = self.c.find_one({'doctype': "manager"}) done = self.c.find_one({'y': {'$exists': 1, '$ne': 'reserved'}}) reserved = {'y': 'reserved'} self.assertEqual(self.c.count_documents({}), 4) self.assertEqual(manager['lock'], None) self.assertEqual(manager['queue'], []) self.assertEqual(done['x'], [5, 11, 'blue']) self.assertEqual(done['index'], 1) self.assertEqual(self.c.count_documents(reserved), 1) def test_alternate_builtin_predictor(self): gp = "GaussianProcessRegressor" self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, predictor=gp) config = self.c.find_one({"doctype": "config"}) self.assertEqual(config["predictor"], gp) launchpad.add_wf(wf_creator_basic([5, 11, 'blue'])) launch_rocket(launchpad) done = self.c.find_one({'y': {'$exists': 1, '$ne': 'reserved'}}) self.assertEqual(done["predictor"], f"{gp} with acquisition: Expected Improvement") def test_custom_predictor(self): self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, predictor=custom_predictor) launchpad.add_wf(wf_creator_basic([5, 11, 'blue'])) launch_rocket(launchpad) manager = self.c.find_one({'doctype': "manager"}) done = self.c.find_one({'y': {'$exists': 1, '$ne': 'reserved'}}) reserved = self.c.find_one({'y': 'reserved'}) self.assertEqual(self.c.count_documents({}), 4) self.assertEqual(manager['lock'], None) self.assertEqual(manager['queue'], []) self.assertEqual(done['x'], [5, 11, 'blue']) self.assertEqual(done['x_new'], [3, 12, 'green']) self.assertEqual(done['index'], 1) self.assertEqual(reserved['x'], [3, 12, 'green']) def test_complex(self): self.mc.configure(wf_creator=wf_creator_complex, dimensions=self.dims_basic, **common_kwargs) launchpad.add_wf(wf_creator_complex([5, 11, 'blue'])) for _ in range(10): launch_rocket(launchpad) self.assertEqual(self.c.count_documents({}), 5) self.assertEqual(self.c.count_documents({'y': 'reserved'}), 1) # should return one doc, for first WF self.assertEqual(self.c.count_documents({'index': 1}), 1) # loop 1 # should return one doc, for second WF self.assertEqual(self.c.count_documents({'index': 2}), 1) # loop 2 def test_duplicates(self): self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, duplicate_check=True, tolerances=[0, 1e-6, None], predictor=custom_predictor, acq=None) launchpad.add_wf(wf_creator_basic([5, 11, 'blue'])) for _ in range(2): launch_rocket(launchpad) self.assertEqual(self.c.count_documents({}), 5) self.assertEqual(self.c.count_documents({'y': 'reserved'}), 1) # should return one doc, for the first WF self.assertEqual(self.c.count_documents({'x': [5, 11, 'blue']}), 1) # should return one doc, for the second WF # no duplicates are in the db self.assertEqual(self.c.count_documents({'x': [3, 12, 'green']}), 1) def test_get_z(self): self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, predictor=custom_predictor, duplicate_check=True, tolerances=[0, 1e-6, None], get_z=get_z, acq=None) launchpad.reset(password=None, require_password=False) launchpad.add_wf(wf_creator_basic([5, 11, 'blue'])) for _ in range(2): launch_rocket(launchpad) loop1 = self.c.find_one({'index': 1}) loop2 = self.c.find_one({'index': 2}) self.assertEqual(self.c.count_documents({}), 5) self.assertEqual(loop1['x'], [5, 11, 'blue']) self.assertEqual(loop1['z'], [25.0, 121.0]) self.assertEqual(loop2['x'], [3, 12.0, 'green']) self.assertEqual(loop2['z'], [9, 144.0]) def test_accuracy(self): best = [None] * 10 avg_random_best = 7.23002918931 # calculated with 1,000,000 calcs for n in range(10): self.mc.reset(hard=True) self.mc.configure(wf_creator=wf_creator_accuracy, dimensions=self.dims_accuracy, maximize=True, **common_kwargs) launchpad.reset(password=None, require_password=False) launchpad.add_wf(wf_creator_accuracy([1, 10.1, 30.1])) for _ in range(20): launch_rocket(launchpad) # We want to maximize the function. The minimum is 0.5, and the # maximum is 10. for doc in self.c.find({'y': { '$exists': 1, '$ne': 'reserved' }}, sort=[('y', pymongo.DESCENDING)], limit=1): best[n] = doc['y'] print(best) self.assertGreater(np.mean(best), avg_random_best) config = self.c.find_one({"doctype": "config"}) self.assertEqual(config["predictor"], common_kwargs["predictor"]) def test_multi(self): self.mc.configure(wf_creator=wf_creator_multiobjective, dimensions=self.dims_basic, predictor_kwargs={'random_state': 1}, **common_kwargs) launchpad.add_wf(wf_creator_multiobjective([5, 11, 'blue'])) launch_rocket(launchpad) manager = self.c.find_one({"doctype": "manager"}) done = self.c.find_one({'y': {'$exists': 1, '$ne': 'reserved'}}) reserved = self.c.find_one({'y': 'reserved'}) self.assertEqual(self.c.count_documents({}), 4) self.assertEqual(manager['lock'], None) self.assertEqual(manager['queue'], []) self.assertEqual(done['x'], [5, 11, 'blue']) self.assertEqual(done['index'], 1) self.assertEqual(len(done['y']), 2) # Loop 2, to make sure optimizations will keep running launch_rocket(launchpad) self.assertEqual(self.c.count_documents({}), 5) self.assertEqual(manager['lock'], None) self.assertEqual(manager['queue'], []) def test_missioncontrol(self): self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, **common_kwargs) launchpad.add_wf(wf_creator_basic([5, 11, 'blue'])) rapidfire(launchpad, nlaunches=20, sleep_time=0) plt = self.mc.plot() savepath = os.path.join(lp_filedir, "missioncontrol_plot.png") plt.savefig(savepath) self.assertTrue(os.path.exists(savepath)) def test_acqfuncs(self): self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, predictor="GaussianProcessRegressor", acq=None) launchpad.add_wf(wf_creator_basic([5, 11, 'blue'])) rapidfire(launchpad, nlaunches=10, sleep_time=0) self.mc.reset(hard=False) self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, predictor="GaussianProcessRegressor", acq="ei") rapidfire(launchpad, nlaunches=2, sleep_time=0) self.mc.reset(hard=False) self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, predictor="GaussianProcessRegressor", acq="pi") rapidfire(launchpad, nlaunches=2, sleep_time=0) self.mc.reset(hard=False) self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, predictor="GaussianProcessRegressor", acq="lcb") rapidfire(launchpad, nlaunches=2, sleep_time=0) def test_parallel(self): n_procs = 10 self.mc.configure(wf_creator=wf_creator_accuracy, dimensions=self.dims_parallel, opt_label='test_parallel', maximize=True, **common_kwargs) for i in range(n_procs): # Assume the worst case, with n_procs forced duplicates launchpad.add_wf(wf_creator_accuracy([1, 5, 3])) launch_multiprocess(launchpad, None, 'INFO', 12, n_procs, 0) try: for _ in range(10): launch_rocket(launchpad) except ExhaustedSpaceError: pass self.assertEqual(125, self.c.count_documents({'y': {'$exists': 1}})) all_x, all_y = self.mc.fetch_matrices(include_reserved=True) self.assertEqual(len(all_x), 125) self.assertEqual(len(all_x), 125) all_x, all_y = self.mc.fetch_matrices(include_reserved=False) self.assertGreaterEqual(len(all_x), 114) self.assertGreaterEqual(len(all_y), 114) def tearDown(self): try: launchpad.reset(password=None, require_password=False) except Exception: warnings.warn("LaunchPad {} could not be reset! There may be " "fireworks from these tests remaining on the " "LaunchPad.".format(launchpad.to_dict())) try: self.db.drop_collection(opt_label) except BaseException: pass if launchpad.host == 'localhost' \ and launchpad.port == 27017 \ and launchpad.name == test_db_name: try: launchpad.connection.drop_database(test_db_name) except BaseException: pass
are constrained to the search space given in x_dim above. Returns: (list): The z vector, to be used for learning. """ fin_len = x[0] fin_angle = x[1] useful_feature1 = fin_len + fin_angle**2 useful_feature2 = fin_angle + fin_len return x + [useful_feature1, useful_feature2] if __name__ == "__main__": # Make a MissionControl object mc = MissionControl(**db_info) # Reset the launchpad and optimization db for this example launchpad.reset(password=None, require_password=False) mc.reset(hard=True) # Configure the optimization db with MissionControl mc.configure(wf_creator=wf_creator, dimensions=x_dim, acq="maximin", predictor="GaussianProcessRegressor", get_z=get_z) # Run 30 workflows + optimization launchpad.add_wf(wf_creator([100, 45.0, "dolphin fin"])) rapidfire(launchpad, nlaunches=30)
are constrained to the search space given in x_dim above. Returns: (list): The z vector, to be used for learning. """ fin_len = x[0] fin_angle = x[1] useful_feature1 = fin_len + fin_angle ** 2 useful_feature2 = fin_angle + fin_len return x + [useful_feature1, useful_feature2] if __name__ == "__main__": # Make a MissionControl object mc = MissionControl(**db_info) # Reset the launchpad and optimization db for this example launchpad.reset(password=None, require_password=False) mc.reset(hard=True) # Configure the optimization db with MissionControl mc.configure(wf_creator=wf_creator, dimensions=x_dim, acq="maximin", predictor="GaussianProcessRegressor", get_z=get_z) # Run 30 workflows + optimization launchpad.add_wf(wf_creator([100, 45.0, "dolphin fin"])) rapidfire(launchpad, nlaunches=30)
class TestWorkflows(unittest.TestCase): def setUp(self): launchpad.reset(password=None, require_password=False) self.mc = MissionControl(**db_info) self.mc.reset(hard=True) self.db = launchpad.db self.dims_basic = [(1, 10), (10.0, 20.0), ['blue', 'green', 'red', 'orange']] self.dims_accuracy = [(1.0, 10.0), (10.0, 20.0), (20.0, 30.0)] self.dims_parallel = [(1, 5), (1, 5), (1, 5)] self.c = getattr(self.db, opt_label) def test_basic(self): self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, **common_kwargs) launchpad.add_wf(wf_creator_basic([5, 11, 'blue'])) launch_rocket(launchpad) manager = self.c.find_one({'doctype': "manager"}) done = self.c.find_one({'y': {'$exists': 1, '$ne': 'reserved'}}) reserved = {'y': 'reserved'} self.assertEqual(self.c.count_documents({}), 4) self.assertEqual(manager['lock'], None) self.assertEqual(manager['queue'], []) self.assertEqual(done['x'], [5, 11, 'blue']) self.assertEqual(done['index'], 1) self.assertEqual(self.c.count_documents(reserved), 1) def test_custom_predictor(self): self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, predictor=custom_predictor) launchpad.add_wf(wf_creator_basic([5, 11, 'blue'])) launch_rocket(launchpad) manager = self.c.find_one({'doctype': "manager"}) done = self.c.find_one({'y': {'$exists': 1, '$ne': 'reserved'}}) reserved = self.c.find_one({'y': 'reserved'}) self.assertEqual(self.c.count_documents({}), 4) self.assertEqual(manager['lock'], None) self.assertEqual(manager['queue'], []) self.assertEqual(done['x'], [5, 11, 'blue']) self.assertEqual(done['x_new'], [3, 12, 'green']) self.assertEqual(done['index'], 1) self.assertEqual(reserved['x'], [3, 12, 'green']) def test_complex(self): self.mc.configure(wf_creator=wf_creator_complex, dimensions=self.dims_basic, **common_kwargs) launchpad.add_wf(wf_creator_complex([5, 11, 'blue'])) for _ in range(10): launch_rocket(launchpad) self.assertEqual(self.c.count_documents({}), 5) self.assertEqual(self.c.count_documents({'y': 'reserved'}), 1) # should return one doc, for first WF self.assertEqual(self.c.count_documents({'index': 1}), 1) # loop 1 # should return one doc, for second WF self.assertEqual(self.c.count_documents({'index': 2}), 1) # loop 2 def test_duplicates(self): self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, duplicate_check=True, tolerances=[0, 1e-6, None], predictor=custom_predictor, acq=None ) launchpad.add_wf(wf_creator_basic([5, 11, 'blue'])) for _ in range(2): launch_rocket(launchpad) self.assertEqual(self.c.count_documents({}), 5) self.assertEqual(self.c.count_documents({'y': 'reserved'}), 1) # should return one doc, for the first WF self.assertEqual(self.c.count_documents({'x': [5, 11, 'blue']}), 1) # should return one doc, for the second WF # no duplicates are in the db self.assertEqual(self.c.count_documents({'x': [3, 12, 'green']}), 1) def test_get_z(self): self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, predictor=custom_predictor, duplicate_check=True, tolerances=[0, 1e-6, None], get_z=get_z, acq=None) launchpad.reset(password=None, require_password=False) launchpad.add_wf(wf_creator_basic([5, 11, 'blue'])) for _ in range(2): launch_rocket(launchpad) loop1 = self.c.find_one({'index': 1}) loop2 = self.c.find_one({'index': 2}) self.assertEqual(self.c.count_documents({}), 5) self.assertEqual(loop1['x'], [5, 11, 'blue']) self.assertEqual(loop1['z'], [25.0, 121.0]) self.assertEqual(loop2['x'], [3, 12.0, 'green']) self.assertEqual(loop2['z'], [9, 144.0]) def test_accuracy(self): best = [None] * 10 avg_random_best = 7.23002918931 # calculated with 1,000,000 calcs for n in range(10): self.mc.reset(hard=True) self.mc.configure(wf_creator=wf_creator_accuracy, dimensions=self.dims_accuracy, maximize=True, **common_kwargs) launchpad.reset(password=None, require_password=False) launchpad.add_wf(wf_creator_accuracy([1, 10.1, 30.1])) for _ in range(20): launch_rocket(launchpad) # We want to maximize the function. The minimum is 0.5, and the # maximum is 10. for doc in self.c.find({'y': {'$exists': 1, '$ne': 'reserved'}}, sort=[('y', pymongo.DESCENDING)], limit=1): best[n] = doc['y'] print(best) self.assertGreater(np.mean(best), avg_random_best) def test_multi(self): self.mc.configure(wf_creator=wf_creator_multiobjective, dimensions=self.dims_basic, predictor_kwargs={'random_state': 1}, **common_kwargs) launchpad.add_wf(wf_creator_multiobjective([5, 11, 'blue'])) launch_rocket(launchpad) manager = self.c.find_one({"doctype": "manager"}) done = self.c.find_one({'y': {'$exists': 1, '$ne': 'reserved'}}) reserved = self.c.find_one({'y': 'reserved'}) self.assertEqual(self.c.count_documents({}), 4) self.assertEqual(manager['lock'], None) self.assertEqual(manager['queue'], []) self.assertEqual(done['x'], [5, 11, 'blue']) self.assertEqual(done['index'], 1) self.assertEqual(len(done['y']), 2) # Loop 2, to make sure optimizations will keep running launch_rocket(launchpad) self.assertEqual(self.c.count_documents({}), 5) self.assertEqual(manager['lock'], None) self.assertEqual(manager['queue'], []) def test_missioncontrol(self): self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, **common_kwargs) launchpad.add_wf(wf_creator_basic([5, 11, 'blue'])) rapidfire(launchpad, nlaunches=20, sleep_time=0) plt = self.mc.plot() savepath = os.path.join(lp_filedir, "missioncontrol_plot.png") plt.savefig(savepath) self.assertTrue(os.path.exists(savepath)) def test_acqfuncs(self): self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, predictor="GaussianProcessRegressor", acq=None) launchpad.add_wf(wf_creator_basic([5, 11, 'blue'])) rapidfire(launchpad, nlaunches=10, sleep_time=0) self.mc.reset(hard=False) self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, predictor="GaussianProcessRegressor", acq="ei") rapidfire(launchpad, nlaunches=2, sleep_time=0) self.mc.reset(hard=False) self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, predictor="GaussianProcessRegressor", acq="pi") rapidfire(launchpad, nlaunches=2, sleep_time=0) self.mc.reset(hard=False) self.mc.configure(wf_creator=wf_creator_basic, dimensions=self.dims_basic, predictor="GaussianProcessRegressor", acq="lcb") rapidfire(launchpad, nlaunches=2, sleep_time=0) def test_parallel(self): n_procs = 10 self.mc.configure(wf_creator=wf_creator_accuracy, dimensions=self.dims_parallel, opt_label='test_parallel', maximize=True, **common_kwargs) for i in range(n_procs): # Assume the worst case, with n_procs forced duplicates launchpad.add_wf(wf_creator_accuracy([1, 5, 3])) launch_multiprocess(launchpad, None, 'INFO', 12, n_procs, 0) try: for _ in range(10): launch_rocket(launchpad) except ExhaustedSpaceError: pass self.assertEqual(125, self.c.count_documents({'y': {'$exists': 1}})) all_x, all_y = self.mc.fetch_matrices(include_reserved=True) self.assertEqual(len(all_x), 125) self.assertEqual(len(all_x), 125) all_x, all_y = self.mc.fetch_matrices(include_reserved=False) self.assertGreaterEqual(len(all_x), 114) self.assertGreaterEqual(len(all_y), 114) def tearDown(self): try: launchpad.reset(password=None, require_password=False) except Exception: warnings.warn("LaunchPad {} could not be reset! There may be " "fireworks from these tests remaining on the " "LaunchPad.".format(launchpad.to_dict())) try: self.db.drop_collection(opt_label) except BaseException: pass if launchpad.host == 'localhost' \ and launchpad.port == 27017 \ and launchpad.name == test_db_name: try: launchpad.connection.drop_database(test_db_name) except BaseException: pass