def test_run(self):
target = lambda x: x**2
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
rval = taf.run(config=2)
self.assertFalse(taf._accepts_instance)
self.assertFalse(taf._accepts_seed)
self.assertEqual(rval[0], StatusType.SUCCESS)
self.assertEqual(rval[1], 4)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], dict())
target = lambda x, seed: (x**2, {'key': seed})
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
rval = taf.run(config=2, instance='test')
self.assertFalse(taf._accepts_instance)
self.assertTrue(taf._accepts_seed)
self.assertEqual(rval[0], StatusType.SUCCESS)
self.assertEqual(rval[1], 4)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], {'key': 12345})
target = lambda x, seed, instance: (x**2, {
'key': seed,
'instance': instance
})
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
rval = taf.run(config=2, instance='test')
self.assertTrue(taf._accepts_instance)
self.assertTrue(taf._accepts_seed)
self.assertEqual(rval[0], StatusType.SUCCESS)
self.assertEqual(rval[1], 4)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], {'key': 12345, 'instance': 'test'})
def test_run_wo_pynisher(self):
def target(x):
return x**2
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
use_pynisher=False)
rval = taf.run(config=2)
self.assertFalse(taf._accepts_instance)
self.assertFalse(taf._accepts_seed)
self.assertEqual(rval[0], StatusType.SUCCESS)
self.assertEqual(rval[1], 4)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], dict())
def target(x, seed, instance):
return x**2, {'key': seed, 'instance': instance}
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
use_pynisher=False)
rval = taf.run(config=2, instance='test')
self.assertTrue(taf._accepts_instance)
self.assertTrue(taf._accepts_seed)
self.assertEqual(rval[0], StatusType.SUCCESS)
self.assertEqual(rval[1], 4)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], {'key': 12345, 'instance': 'test'})
def target(x):
return None
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
use_pynisher=False)
rval = taf.run(config=2)
self.assertFalse(taf._accepts_instance)
self.assertFalse(taf._accepts_seed)
self.assertEqual(rval[0], StatusType.CRASHED)
self.assertEqual(rval[1], 2147483647.0)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], dict())
def target(x):
raise Exception(x)
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
use_pynisher=False)
rval = taf.run(config=2)
self.assertFalse(taf._accepts_instance)
self.assertFalse(taf._accepts_seed)
self.assertEqual(rval[0], StatusType.CRASHED)
self.assertEqual(rval[1], 2147483647.0)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], dict())
def test_memout(self):
def fill_memory(*args):
a = np.random.random_sample((10000, 10000)).astype(np.float64)
return np.sum(a)
taf = ExecuteTAFuncDict(ta=fill_memory,
stats=self.stats,
memory_limit=1024)
rval = taf.run(config=None)
platform = os.getenv('TRAVIS_OS_NAME')
if platform is None:
platform = {'linux': 'linux', 'darwin': 'osx'}.get(sys.platform)
print(platform, sys.platform)
if platform == 'linux':
self.assertEqual(rval[0], StatusType.MEMOUT)
self.assertEqual(rval[1], 2147483647.0)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], dict())
elif platform == 'osx':
self.assertEqual(rval[0], StatusType.SUCCESS)
else:
raise ValueError('Test cannot be performed on platform %s' %
sys.platform)
def test_non_serializable(self):
# cost non serializable
def target(x):
return np.int32(x)
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
msg = "Please ensure all objects returned are JSON serializable."
with self.assertRaisesRegex(TypeError, msg):
taf.run(config=2)
# additional info non serializable
def target(x):
return x, {'x': np.int32(x)}
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
msg = "Please ensure all objects returned are JSON serializable."
with self.assertRaisesRegex(TypeError, msg):
taf.run(config=2)
def test_fail_silent(self):
def function(*args):
return
taf = ExecuteTAFuncDict(ta=function, stats=self.stats)
rval = taf.run(config=None, cutoff=1)
self.assertEqual(rval[0], StatusType.CRASHED)
self.assertEqual(rval[1], 2147483647.0)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], dict())
def test_timeout(self):
def run_over_time(*args):
time.sleep(5)
taf = ExecuteTAFuncDict(ta=run_over_time, stats=self.stats)
rval = taf.run(config=None, cutoff=1)
self.assertEqual(rval[0], StatusType.TIMEOUT)
self.assertEqual(rval[1], 2147483647.0)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], dict())
def test_timeout_runtime(self):
def run_over_time(*args):
time.sleep(5)
taf = ExecuteTAFuncDict(ta=run_over_time,
stats=self.stats,
run_obj='runtime',
par_factor=11)
rval = taf.run(config=None, cutoff=1)
self.assertEqual(rval[0], StatusType.TIMEOUT)
self.assertGreaterEqual(rval[1], 11)
self.assertGreaterEqual(rval[2], 1)
self.assertEqual(rval[3], dict())
def test_run_wo_pynisher(self):
target = lambda x: x**2
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
use_pynisher=False)
rval = taf.run(config=2)
self.assertFalse(taf._accepts_instance)
self.assertFalse(taf._accepts_seed)
self.assertEqual(rval[0], StatusType.SUCCESS)
self.assertEqual(rval[1], 4)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], dict())
target = lambda x: None
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
use_pynisher=False)
rval = taf.run(config=2)
self.assertFalse(taf._accepts_instance)
self.assertFalse(taf._accepts_seed)
self.assertEqual(rval[0], StatusType.CRASHED)
self.assertEqual(rval[1], 2147483647.0)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], dict())
cond = InCondition(child=p, parent=previous_param, values=[1])
cs.add_condition(cond)
previous_param = p
# SMAC scenario object
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternative runtime)
"runcount-limit": n_params * 2, # at most 200 function evaluations
"cs": cs, # configuration space
"deterministic": "true"
})
# register function to be optimize
taf = ExecuteTAFuncDict(leading_ones)
# example call of the function
# it returns: Status, Cost, Runtime, Additional Infos
def_value = taf.run(cs.get_default_configuration())[1]
print("Default Value: %.2f" % (def_value))
# Optimize
smac = SMAC(scenario=scenario, rng=np.random.RandomState(42), tae_runner=taf)
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
inc_value = taf.run(incumbent)[1]
print("Optimized Value: %.2f" % (inc_value))
logging.basicConfig(level=logging.DEBUG)
# Get Configuration Space
cs = logReg.get_configuration_space()
# Prepare TA
ta_call = functools.partial(mini_wrapper, bench=logReg)
taf = ExecuteTAFuncDict(ta_call)
# SMAC scenario object
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": 5,
"cs": cs,
"deterministic": "true",
"instances": list([str(i) for i in range(10)]),
"test_instances": ["10"]
})
# Use SMAC to optimize function
smac = SMAC(scenario=scenario, rng=1, tae_runner=taf)
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
# Evaluate final configuration
inc_value = taf.run(incumbent, instance=10)[1]
print("Optimized Value: %.2f" % inc_value)