def test_run(self):
"""Makes sure that we are able to run a configuration and
return the expected values/types"""
# We use the funcdict as a mechanism to test Parallel Runner
runner = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj='quality')
runner = DaskParallelRunner(runner, n_workers=2)
self.assertIsInstance(runner, DaskParallelRunner)
run_info = RunInfo(config=2,
instance='test',
instance_specific="0",
seed=0,
cutoff=None,
capped=False,
budget=0.0)
# submit runs! then get the value
runner.submit_run(run_info)
run_values = runner.get_finished_runs()
# Run will not have finished so fast
self.assertEqual(len(run_values), 0)
runner.wait()
run_values = runner.get_finished_runs()
self.assertEqual(len(run_values), 1)
self.assertIsInstance(run_values, list)
self.assertIsInstance(run_values[0][0], RunInfo)
self.assertIsInstance(run_values[0][1], RunValue)
self.assertEqual(run_values[0][1].cost, 4)
self.assertEqual(run_values[0][1].status, StatusType.SUCCESS)
def test_parallel_runs(self):
"""Make sure because there are 2 workers, the runs are launched
closely in time together"""
# We use the funcdict as a mechanism to test Runner
runner = ExecuteTAFuncDict(ta=target_delayed,
stats=self.stats,
run_obj='quality')
runner = DaskParallelRunner(runner, n_workers=2)
run_info = RunInfo(config=2,
instance='test',
instance_specific="0",
seed=0,
cutoff=None,
capped=False,
budget=0.0)
runner.submit_run(run_info)
run_info = RunInfo(config=3,
instance='test',
instance_specific="0",
seed=0,
cutoff=None,
capped=False,
budget=0.0)
runner.submit_run(run_info)
# At this stage, we submitted 2 jobs, that are running in remote
# workers. We have to wait for each one of them to complete. The
# runner provides a wait() method to do so, yet it can only wait for
# a single job to be completed. It does internally via dask wait(<list of futures>)
# so we take wait for the first job to complete, and take it out
runner.wait()
run_values = runner.get_finished_runs()
# To be on the safe side, we don't check for:
# self.assertEqual(len(run_values), 1)
# In the ideal world, two runs were launched which take the same time
# so waiting for one means, the second one is completed. But some
# overhead might cause it to be delayed.
# Above took the first run results and put it on run_values
# But for this check we need the second run we submitted
# Again, the runs might take slightly different time depending if we have
# heterogeneous workers, so calling wait 2 times is a guarantee that 2
# jobs are finished
runner.wait()
run_values.extend(runner.get_finished_runs())
self.assertEqual(len(run_values), 2)
# To make it is parallel, we just make sure that the start of the second
# run is earlier than the end of the first run
# Results are returned in left to right
self.assertLessEqual(int(run_values[0][1].starttime),
int(run_values[1][1].endtime))
def test_additional_info_crash_msg(self):
"""
We want to make sure we catch errors as additional info,
and in particular when doing multiprocessing runs, we
want to make sure we capture dask exceptions
"""
def target_nonpickable(x, seed, instance):
return x**2, {'key': seed, 'instance': instance}
runner = ExecuteTAFuncDict(ta=target_nonpickable, stats=self.stats, run_obj='quality')
runner = DaskParallelRunner(runner, n_workers=2)
run_info = RunInfo(config=2, instance='test', instance_specific="0",
seed=0, cutoff=None, capped=False, budget=0.0)
runner.submit_run(run_info)
runner.wait()
run_info, result = runner.get_finished_runs()[0]
# Make sure the traceback message is included
self.assertIn('traceback', result.additional_info)
self.assertIn(
# We expect the problem to occur in the run wrapper
# So traceback should show this!
'target_nonpickable',
result.additional_info['traceback'])
# Make sure the error message is included
self.assertIn('error', result.additional_info)
self.assertIn(
'Can\'t pickle local object',
result.additional_info['error'])
def test_serial_runs(self):
# We use the funcdict as a mechanism to test SerialRunner
runner = ExecuteTAFuncDict(ta=target_delayed, stats=self.stats, run_obj='quality')
self.assertIsInstance(runner, SerialRunner)
run_info = RunInfo(config=2, instance='test', instance_specific="0",
seed=0, cutoff=None, capped=False, budget=0.0)
runner.submit_run(run_info)
run_info = RunInfo(config=3, instance='test', instance_specific="0",
seed=0, cutoff=None, capped=False, budget=0.0)
runner.submit_run(run_info)
run_values = runner.get_finished_runs()
self.assertEqual(len(run_values), 2)
# To make sure runs launched serially, we just make sure that the end time of
# a run is later than the other
# Results are returned in left to right
self.assertLessEqual(int(run_values[1][1].endtime), int(run_values[0][1].starttime))
# No wait time in serial runs!
start = time.time()
runner.wait()
# The run takes a second, so 0.5 is sufficient
self.assertLess(time.time() - start, 0.5)
pass
