def setUp(self):
launchpad.reset(password=None, require_password=False)
self.mc = MissionControl(**db_info)
self.mc.reset(hard=True)
self.db = launchpad.db
self.dims_basic = [(1, 10), (10.0, 20.0),
['blue', 'green', 'red', 'orange']]
self.dims_accuracy = [(1.0, 10.0), (10.0, 20.0), (20.0, 30.0)]
self.dims_parallel = [(1, 5), (1, 5), (1, 5)]
self.c = getattr(self.db, opt_label)
def setUp(self):
launchpad.reset(password=None, require_password=False)
self.mc = MissionControl(**db_info)
self.mc.reset(hard=True)
self.db = launchpad.db
self.dims_basic = [(1, 10), (10.0, 20.0),
['blue', 'green', 'red', 'orange']]
self.dims_accuracy = [(1.0, 10.0), (10.0, 20.0), (20.0, 30.0)]
self.dims_parallel = [(1, 5), (1, 5), (1, 5)]
self.c = getattr(self.db, opt_label)
integers between 1 and 5 (inclusive).
Returns:
(Workflow): A workflow containing one FireWork (two FireTasks) which
is automatically set up to run the optimization loop.
"""
spec = {'_x': x}
# ObjectiveFuncTask writes _y field to the spec internally.
firework1 = Firework([ObjectiveFuncTask(), OptTask(**db_info)], spec=spec)
return Workflow([firework1])
if __name__ == "__main__":
# Make a MissionControl object
mc = MissionControl(**db_info)
# Reset the launchpad and optimization db for this example
launchpad.reset(password=None, require_password=False)
mc.reset(hard=True)
# Configure the optimization db with MissionControl
mc.configure(wf_creator=wf_creator, dimensions=x_dim)
# Run the optimization loop 10 times.
launchpad.add_wf(wf_creator([5, 5, 2]))
rapidfire(launchpad, nlaunches=10, sleep_time=0)
# Examine results
plt = mc.plot()
plt.show()
class TestWorkflows(unittest.TestCase):
def setUp(self):
launchpad.reset(password=None, require_password=False)
self.mc = MissionControl(**db_info)
self.mc.reset(hard=True)
self.db = launchpad.db
self.dims_basic = [(1, 10), (10.0, 20.0),
['blue', 'green', 'red', 'orange']]
self.dims_accuracy = [(1.0, 10.0), (10.0, 20.0), (20.0, 30.0)]
self.dims_parallel = [(1, 5), (1, 5), (1, 5)]
self.c = getattr(self.db, opt_label)
def test_basic(self):
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
**common_kwargs)
launchpad.add_wf(wf_creator_basic([5, 11, 'blue']))
launch_rocket(launchpad)
manager = self.c.find_one({'doctype': "manager"})
done = self.c.find_one({'y': {'$exists': 1, '$ne': 'reserved'}})
reserved = {'y': 'reserved'}
self.assertEqual(self.c.count_documents({}), 4)
self.assertEqual(manager['lock'], None)
self.assertEqual(manager['queue'], [])
self.assertEqual(done['x'], [5, 11, 'blue'])
self.assertEqual(done['index'], 1)
self.assertEqual(self.c.count_documents(reserved), 1)
def test_alternate_builtin_predictor(self):
gp = "GaussianProcessRegressor"
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
predictor=gp)
config = self.c.find_one({"doctype": "config"})
self.assertEqual(config["predictor"], gp)
launchpad.add_wf(wf_creator_basic([5, 11, 'blue']))
launch_rocket(launchpad)
done = self.c.find_one({'y': {'$exists': 1, '$ne': 'reserved'}})
self.assertEqual(done["predictor"],
f"{gp} with acquisition: Expected Improvement")
def test_custom_predictor(self):
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
predictor=custom_predictor)
launchpad.add_wf(wf_creator_basic([5, 11, 'blue']))
launch_rocket(launchpad)
manager = self.c.find_one({'doctype': "manager"})
done = self.c.find_one({'y': {'$exists': 1, '$ne': 'reserved'}})
reserved = self.c.find_one({'y': 'reserved'})
self.assertEqual(self.c.count_documents({}), 4)
self.assertEqual(manager['lock'], None)
self.assertEqual(manager['queue'], [])
self.assertEqual(done['x'], [5, 11, 'blue'])
self.assertEqual(done['x_new'], [3, 12, 'green'])
self.assertEqual(done['index'], 1)
self.assertEqual(reserved['x'], [3, 12, 'green'])
def test_complex(self):
self.mc.configure(wf_creator=wf_creator_complex,
dimensions=self.dims_basic,
**common_kwargs)
launchpad.add_wf(wf_creator_complex([5, 11, 'blue']))
for _ in range(10):
launch_rocket(launchpad)
self.assertEqual(self.c.count_documents({}), 5)
self.assertEqual(self.c.count_documents({'y': 'reserved'}), 1)
# should return one doc, for first WF
self.assertEqual(self.c.count_documents({'index': 1}), 1) # loop 1
# should return one doc, for second WF
self.assertEqual(self.c.count_documents({'index': 2}), 1) # loop 2
def test_duplicates(self):
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
duplicate_check=True,
tolerances=[0, 1e-6, None],
predictor=custom_predictor,
acq=None)
launchpad.add_wf(wf_creator_basic([5, 11, 'blue']))
for _ in range(2):
launch_rocket(launchpad)
self.assertEqual(self.c.count_documents({}), 5)
self.assertEqual(self.c.count_documents({'y': 'reserved'}), 1)
# should return one doc, for the first WF
self.assertEqual(self.c.count_documents({'x': [5, 11, 'blue']}), 1)
# should return one doc, for the second WF
# no duplicates are in the db
self.assertEqual(self.c.count_documents({'x': [3, 12, 'green']}), 1)
def test_get_z(self):
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
predictor=custom_predictor,
duplicate_check=True,
tolerances=[0, 1e-6, None],
get_z=get_z,
acq=None)
launchpad.reset(password=None, require_password=False)
launchpad.add_wf(wf_creator_basic([5, 11, 'blue']))
for _ in range(2):
launch_rocket(launchpad)
loop1 = self.c.find_one({'index': 1})
loop2 = self.c.find_one({'index': 2})
self.assertEqual(self.c.count_documents({}), 5)
self.assertEqual(loop1['x'], [5, 11, 'blue'])
self.assertEqual(loop1['z'], [25.0, 121.0])
self.assertEqual(loop2['x'], [3, 12.0, 'green'])
self.assertEqual(loop2['z'], [9, 144.0])
def test_accuracy(self):
best = [None] * 10
avg_random_best = 7.23002918931 # calculated with 1,000,000 calcs
for n in range(10):
self.mc.reset(hard=True)
self.mc.configure(wf_creator=wf_creator_accuracy,
dimensions=self.dims_accuracy,
maximize=True,
**common_kwargs)
launchpad.reset(password=None, require_password=False)
launchpad.add_wf(wf_creator_accuracy([1, 10.1, 30.1]))
for _ in range(20):
launch_rocket(launchpad)
# We want to maximize the function. The minimum is 0.5, and the
# maximum is 10.
for doc in self.c.find({'y': {
'$exists': 1,
'$ne': 'reserved'
}},
sort=[('y', pymongo.DESCENDING)],
limit=1):
best[n] = doc['y']
print(best)
self.assertGreater(np.mean(best), avg_random_best)
config = self.c.find_one({"doctype": "config"})
self.assertEqual(config["predictor"], common_kwargs["predictor"])
def test_multi(self):
self.mc.configure(wf_creator=wf_creator_multiobjective,
dimensions=self.dims_basic,
predictor_kwargs={'random_state': 1},
**common_kwargs)
launchpad.add_wf(wf_creator_multiobjective([5, 11, 'blue']))
launch_rocket(launchpad)
manager = self.c.find_one({"doctype": "manager"})
done = self.c.find_one({'y': {'$exists': 1, '$ne': 'reserved'}})
reserved = self.c.find_one({'y': 'reserved'})
self.assertEqual(self.c.count_documents({}), 4)
self.assertEqual(manager['lock'], None)
self.assertEqual(manager['queue'], [])
self.assertEqual(done['x'], [5, 11, 'blue'])
self.assertEqual(done['index'], 1)
self.assertEqual(len(done['y']), 2)
# Loop 2, to make sure optimizations will keep running
launch_rocket(launchpad)
self.assertEqual(self.c.count_documents({}), 5)
self.assertEqual(manager['lock'], None)
self.assertEqual(manager['queue'], [])
def test_missioncontrol(self):
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
**common_kwargs)
launchpad.add_wf(wf_creator_basic([5, 11, 'blue']))
rapidfire(launchpad, nlaunches=20, sleep_time=0)
plt = self.mc.plot()
savepath = os.path.join(lp_filedir, "missioncontrol_plot.png")
plt.savefig(savepath)
self.assertTrue(os.path.exists(savepath))
def test_acqfuncs(self):
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
predictor="GaussianProcessRegressor",
acq=None)
launchpad.add_wf(wf_creator_basic([5, 11, 'blue']))
rapidfire(launchpad, nlaunches=10, sleep_time=0)
self.mc.reset(hard=False)
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
predictor="GaussianProcessRegressor",
acq="ei")
rapidfire(launchpad, nlaunches=2, sleep_time=0)
self.mc.reset(hard=False)
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
predictor="GaussianProcessRegressor",
acq="pi")
rapidfire(launchpad, nlaunches=2, sleep_time=0)
self.mc.reset(hard=False)
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
predictor="GaussianProcessRegressor",
acq="lcb")
rapidfire(launchpad, nlaunches=2, sleep_time=0)
def test_parallel(self):
n_procs = 10
self.mc.configure(wf_creator=wf_creator_accuracy,
dimensions=self.dims_parallel,
opt_label='test_parallel',
maximize=True,
**common_kwargs)
for i in range(n_procs):
# Assume the worst case, with n_procs forced duplicates
launchpad.add_wf(wf_creator_accuracy([1, 5, 3]))
launch_multiprocess(launchpad, None, 'INFO', 12, n_procs, 0)
try:
for _ in range(10):
launch_rocket(launchpad)
except ExhaustedSpaceError:
pass
self.assertEqual(125, self.c.count_documents({'y': {'$exists': 1}}))
all_x, all_y = self.mc.fetch_matrices(include_reserved=True)
self.assertEqual(len(all_x), 125)
self.assertEqual(len(all_x), 125)
all_x, all_y = self.mc.fetch_matrices(include_reserved=False)
self.assertGreaterEqual(len(all_x), 114)
self.assertGreaterEqual(len(all_y), 114)
def tearDown(self):
try:
launchpad.reset(password=None, require_password=False)
except Exception:
warnings.warn("LaunchPad {} could not be reset! There may be "
"fireworks from these tests remaining on the "
"LaunchPad.".format(launchpad.to_dict()))
try:
self.db.drop_collection(opt_label)
except BaseException:
pass
if launchpad.host == 'localhost' \
and launchpad.port == 27017 \
and launchpad.name == test_db_name:
try:
launchpad.connection.drop_database(test_db_name)
except BaseException:
pass
are constrained to the search space given in x_dim above.
Returns:
(list): The z vector, to be used for learning.
"""
fin_len = x[0]
fin_angle = x[1]
useful_feature1 = fin_len + fin_angle**2
useful_feature2 = fin_angle + fin_len
return x + [useful_feature1, useful_feature2]
if __name__ == "__main__":
# Make a MissionControl object
mc = MissionControl(**db_info)
# Reset the launchpad and optimization db for this example
launchpad.reset(password=None, require_password=False)
mc.reset(hard=True)
# Configure the optimization db with MissionControl
mc.configure(wf_creator=wf_creator,
dimensions=x_dim,
acq="maximin",
predictor="GaussianProcessRegressor",
get_z=get_z)
# Run 30 workflows + optimization
launchpad.add_wf(wf_creator([100, 45.0, "dolphin fin"]))
rapidfire(launchpad, nlaunches=30)
integers between 1 and 5 (inclusive).
Returns:
(Workflow): A workflow containing one FireWork (two FireTasks) which
is automatically set up to run the optimization loop.
"""
spec = {'_x': x}
# ObjectiveFuncTask writes _y field to the spec internally.
firework1 = Firework([ObjectiveFuncTask(), OptTask(**db_info)], spec=spec)
return Workflow([firework1])
if __name__ == "__main__":
# Make a MissionControl object
mc = MissionControl(**db_info)
# Reset the launchpad and optimization db for this example
launchpad.reset(password=None, require_password=False)
mc.reset(hard=True)
# Configure the optimization db with MissionControl
mc.configure(wf_creator=wf_creator, dimensions=x_dim)
# Run the optimization loop 10 times.
launchpad.add_wf(wf_creator([5, 5, 2]))
rapidfire(launchpad, nlaunches=10, sleep_time=0)
# Examine results
plt = mc.plot()
plt.show()
are constrained to the search space given in x_dim above.
Returns:
(list): The z vector, to be used for learning.
"""
fin_len = x[0]
fin_angle = x[1]
useful_feature1 = fin_len + fin_angle ** 2
useful_feature2 = fin_angle + fin_len
return x + [useful_feature1, useful_feature2]
if __name__ == "__main__":
# Make a MissionControl object
mc = MissionControl(**db_info)
# Reset the launchpad and optimization db for this example
launchpad.reset(password=None, require_password=False)
mc.reset(hard=True)
# Configure the optimization db with MissionControl
mc.configure(wf_creator=wf_creator,
dimensions=x_dim,
acq="maximin",
predictor="GaussianProcessRegressor",
get_z=get_z)
# Run 30 workflows + optimization
launchpad.add_wf(wf_creator([100, 45.0, "dolphin fin"]))
rapidfire(launchpad, nlaunches=30)
class TestWorkflows(unittest.TestCase):
def setUp(self):
launchpad.reset(password=None, require_password=False)
self.mc = MissionControl(**db_info)
self.mc.reset(hard=True)
self.db = launchpad.db
self.dims_basic = [(1, 10), (10.0, 20.0),
['blue', 'green', 'red', 'orange']]
self.dims_accuracy = [(1.0, 10.0), (10.0, 20.0), (20.0, 30.0)]
self.dims_parallel = [(1, 5), (1, 5), (1, 5)]
self.c = getattr(self.db, opt_label)
def test_basic(self):
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
**common_kwargs)
launchpad.add_wf(wf_creator_basic([5, 11, 'blue']))
launch_rocket(launchpad)
manager = self.c.find_one({'doctype': "manager"})
done = self.c.find_one({'y': {'$exists': 1, '$ne': 'reserved'}})
reserved = {'y': 'reserved'}
self.assertEqual(self.c.count_documents({}), 4)
self.assertEqual(manager['lock'], None)
self.assertEqual(manager['queue'], [])
self.assertEqual(done['x'], [5, 11, 'blue'])
self.assertEqual(done['index'], 1)
self.assertEqual(self.c.count_documents(reserved), 1)
def test_custom_predictor(self):
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
predictor=custom_predictor)
launchpad.add_wf(wf_creator_basic([5, 11, 'blue']))
launch_rocket(launchpad)
manager = self.c.find_one({'doctype': "manager"})
done = self.c.find_one({'y': {'$exists': 1, '$ne': 'reserved'}})
reserved = self.c.find_one({'y': 'reserved'})
self.assertEqual(self.c.count_documents({}), 4)
self.assertEqual(manager['lock'], None)
self.assertEqual(manager['queue'], [])
self.assertEqual(done['x'], [5, 11, 'blue'])
self.assertEqual(done['x_new'], [3, 12, 'green'])
self.assertEqual(done['index'], 1)
self.assertEqual(reserved['x'], [3, 12, 'green'])
def test_complex(self):
self.mc.configure(wf_creator=wf_creator_complex,
dimensions=self.dims_basic,
**common_kwargs)
launchpad.add_wf(wf_creator_complex([5, 11, 'blue']))
for _ in range(10):
launch_rocket(launchpad)
self.assertEqual(self.c.count_documents({}), 5)
self.assertEqual(self.c.count_documents({'y': 'reserved'}), 1)
# should return one doc, for first WF
self.assertEqual(self.c.count_documents({'index': 1}), 1) # loop 1
# should return one doc, for second WF
self.assertEqual(self.c.count_documents({'index': 2}), 1) # loop 2
def test_duplicates(self):
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
duplicate_check=True,
tolerances=[0, 1e-6, None],
predictor=custom_predictor,
acq=None
)
launchpad.add_wf(wf_creator_basic([5, 11, 'blue']))
for _ in range(2):
launch_rocket(launchpad)
self.assertEqual(self.c.count_documents({}), 5)
self.assertEqual(self.c.count_documents({'y': 'reserved'}), 1)
# should return one doc, for the first WF
self.assertEqual(self.c.count_documents({'x': [5, 11, 'blue']}), 1)
# should return one doc, for the second WF
# no duplicates are in the db
self.assertEqual(self.c.count_documents({'x': [3, 12, 'green']}), 1)
def test_get_z(self):
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
predictor=custom_predictor,
duplicate_check=True,
tolerances=[0, 1e-6, None],
get_z=get_z,
acq=None)
launchpad.reset(password=None, require_password=False)
launchpad.add_wf(wf_creator_basic([5, 11, 'blue']))
for _ in range(2):
launch_rocket(launchpad)
loop1 = self.c.find_one({'index': 1})
loop2 = self.c.find_one({'index': 2})
self.assertEqual(self.c.count_documents({}), 5)
self.assertEqual(loop1['x'], [5, 11, 'blue'])
self.assertEqual(loop1['z'], [25.0, 121.0])
self.assertEqual(loop2['x'], [3, 12.0, 'green'])
self.assertEqual(loop2['z'], [9, 144.0])
def test_accuracy(self):
best = [None] * 10
avg_random_best = 7.23002918931 # calculated with 1,000,000 calcs
for n in range(10):
self.mc.reset(hard=True)
self.mc.configure(wf_creator=wf_creator_accuracy,
dimensions=self.dims_accuracy,
maximize=True,
**common_kwargs)
launchpad.reset(password=None, require_password=False)
launchpad.add_wf(wf_creator_accuracy([1, 10.1, 30.1]))
for _ in range(20):
launch_rocket(launchpad)
# We want to maximize the function. The minimum is 0.5, and the
# maximum is 10.
for doc in self.c.find({'y': {'$exists': 1, '$ne': 'reserved'}},
sort=[('y', pymongo.DESCENDING)],
limit=1):
best[n] = doc['y']
print(best)
self.assertGreater(np.mean(best), avg_random_best)
def test_multi(self):
self.mc.configure(wf_creator=wf_creator_multiobjective,
dimensions=self.dims_basic,
predictor_kwargs={'random_state': 1},
**common_kwargs)
launchpad.add_wf(wf_creator_multiobjective([5, 11, 'blue']))
launch_rocket(launchpad)
manager = self.c.find_one({"doctype": "manager"})
done = self.c.find_one({'y': {'$exists': 1, '$ne': 'reserved'}})
reserved = self.c.find_one({'y': 'reserved'})
self.assertEqual(self.c.count_documents({}), 4)
self.assertEqual(manager['lock'], None)
self.assertEqual(manager['queue'], [])
self.assertEqual(done['x'], [5, 11, 'blue'])
self.assertEqual(done['index'], 1)
self.assertEqual(len(done['y']), 2)
# Loop 2, to make sure optimizations will keep running
launch_rocket(launchpad)
self.assertEqual(self.c.count_documents({}), 5)
self.assertEqual(manager['lock'], None)
self.assertEqual(manager['queue'], [])
def test_missioncontrol(self):
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
**common_kwargs)
launchpad.add_wf(wf_creator_basic([5, 11, 'blue']))
rapidfire(launchpad, nlaunches=20, sleep_time=0)
plt = self.mc.plot()
savepath = os.path.join(lp_filedir, "missioncontrol_plot.png")
plt.savefig(savepath)
self.assertTrue(os.path.exists(savepath))
def test_acqfuncs(self):
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
predictor="GaussianProcessRegressor",
acq=None)
launchpad.add_wf(wf_creator_basic([5, 11, 'blue']))
rapidfire(launchpad, nlaunches=10, sleep_time=0)
self.mc.reset(hard=False)
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
predictor="GaussianProcessRegressor",
acq="ei")
rapidfire(launchpad, nlaunches=2, sleep_time=0)
self.mc.reset(hard=False)
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
predictor="GaussianProcessRegressor",
acq="pi")
rapidfire(launchpad, nlaunches=2, sleep_time=0)
self.mc.reset(hard=False)
self.mc.configure(wf_creator=wf_creator_basic,
dimensions=self.dims_basic,
predictor="GaussianProcessRegressor",
acq="lcb")
rapidfire(launchpad, nlaunches=2, sleep_time=0)
def test_parallel(self):
n_procs = 10
self.mc.configure(wf_creator=wf_creator_accuracy,
dimensions=self.dims_parallel,
opt_label='test_parallel',
maximize=True,
**common_kwargs)
for i in range(n_procs):
# Assume the worst case, with n_procs forced duplicates
launchpad.add_wf(wf_creator_accuracy([1, 5, 3]))
launch_multiprocess(launchpad, None, 'INFO', 12, n_procs, 0)
try:
for _ in range(10):
launch_rocket(launchpad)
except ExhaustedSpaceError:
pass
self.assertEqual(125, self.c.count_documents({'y': {'$exists': 1}}))
all_x, all_y = self.mc.fetch_matrices(include_reserved=True)
self.assertEqual(len(all_x), 125)
self.assertEqual(len(all_x), 125)
all_x, all_y = self.mc.fetch_matrices(include_reserved=False)
self.assertGreaterEqual(len(all_x), 114)
self.assertGreaterEqual(len(all_y), 114)
def tearDown(self):
try:
launchpad.reset(password=None, require_password=False)
except Exception:
warnings.warn("LaunchPad {} could not be reset! There may be "
"fireworks from these tests remaining on the "
"LaunchPad.".format(launchpad.to_dict()))
try:
self.db.drop_collection(opt_label)
except BaseException:
pass
if launchpad.host == 'localhost' \
and launchpad.port == 27017 \
and launchpad.name == test_db_name:
try:
launchpad.connection.drop_database(test_db_name)
except BaseException:
pass