def test_run(self): """Makes sure that we are able to run a configuration and return the expected values/types""" # We use the funcdict as a mechanism to test SerialRunner runner = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj='quality') self.assertIsInstance(runner, SerialRunner) run_info = RunInfo(config=2, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) # submit runs! then get the value runner.submit_run(run_info) run_values = runner.get_finished_runs() self.assertEqual(len(run_values), 1) self.assertIsInstance(run_values, list) self.assertIsInstance(run_values[0][0], RunInfo) self.assertIsInstance(run_values[0][1], RunValue) self.assertEqual(run_values[0][1].cost, 4) self.assertEqual(run_values[0][1].status, StatusType.SUCCESS)
def test_additional_info_crash_msg(self): """ We want to make sure we catch errors as additional info, and in particular when doing multiprocessing runs, we want to make sure we capture dask exceptions """ def target_nonpickable(x, seed, instance): return x**2, {'key': seed, 'instance': instance} runner = ExecuteTAFuncDict(ta=target_nonpickable, stats=self.stats, run_obj='quality') runner = DaskParallelRunner(runner, n_workers=2) run_info = RunInfo(config=2, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) runner.submit_run(run_info) runner.wait() run_info, result = runner.get_finished_runs()[0] # Make sure the traceback message is included self.assertIn('traceback', result.additional_info) self.assertIn( # We expect the problem to occur in the run wrapper # So traceback should show this! 'target_nonpickable', result.additional_info['traceback']) # Make sure the error message is included self.assertIn('error', result.additional_info) self.assertIn( 'Can\'t pickle local object', result.additional_info['error'])
def test_serial_runs(self): # We use the funcdict as a mechanism to test SerialRunner runner = ExecuteTAFuncDict(ta=target_delayed, stats=self.stats, run_obj='quality') self.assertIsInstance(runner, SerialRunner) run_info = RunInfo(config=2, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) runner.submit_run(run_info) run_info = RunInfo(config=3, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) runner.submit_run(run_info) run_values = runner.get_finished_runs() self.assertEqual(len(run_values), 2) # To make sure runs launched serially, we just make sure that the end time of # a run is later than the other # Results are returned in left to right self.assertLessEqual(int(run_values[1][1].endtime), int(run_values[0][1].starttime)) # No wait time in serial runs! start = time.time() runner.wait() # The run takes a second, so 0.5 is sufficient self.assertLess(time.time() - start, 0.5) pass
def test_parallel_runs(self): """Make sure because there are 2 workers, the runs are launched closely in time together""" # We use the funcdict as a mechanism to test Runner runner = ExecuteTAFuncDict(ta=target_delayed, stats=self.stats, run_obj='quality') runner = DaskParallelRunner(runner, n_workers=2) run_info = RunInfo(config=2, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) runner.submit_run(run_info) run_info = RunInfo(config=3, instance='test', instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0) runner.submit_run(run_info) # At this stage, we submitted 2 jobs, that are running in remote # workers. We have to wait for each one of them to complete. The # runner provides a wait() method to do so, yet it can only wait for # a single job to be completed. It does internally via dask wait(<list of futures>) # so we take wait for the first job to complete, and take it out runner.wait() run_values = runner.get_finished_runs() # To be on the safe side, we don't check for: # self.assertEqual(len(run_values), 1) # In the ideal world, two runs were launched which take the same time # so waiting for one means, the second one is completed. But some # overhead might cause it to be delayed. # Above took the first run results and put it on run_values # But for this check we need the second run we submitted # Again, the runs might take slightly different time depending if we have # heterogeneous workers, so calling wait 2 times is a guarantee that 2 # jobs are finished runner.wait() run_values.extend(runner.get_finished_runs()) self.assertEqual(len(run_values), 2) # To make it is parallel, we just make sure that the start of the second # run is earlier than the end of the first run # Results are returned in left to right self.assertLessEqual(int(run_values[0][1].starttime), int(run_values[1][1].endtime))