def test_crashed_cost_value(self, test_run): ''' test cost on crashed runs ''' # Patch run-function for custom-return scen = Scenario(scenario={ 'cs': ConfigurationSpace(), 'run_obj': 'quality' }, cmd_options=None) stats = Stats(scen) stats.start_timing() stats.ta_runs += 1 # Check quality test_run.return_value = StatusType.CRASHED, np.nan, np.nan, {} eta = ExecuteTARun(ta=lambda *args: None, stats=stats, run_obj='quality', cost_for_crash=100) self.assertEqual(100, eta.start(config={}, instance=1)[1]) # Check runtime eta = ExecuteTARun(ta=lambda *args: None, stats=stats, run_obj='runtime', cost_for_crash=10.7) self.assertEqual(20.0, eta.start(config={}, instance=1, cutoff=20)[1])
def get_tae(obj): """ Create ExecuteTARun-object for testing. """ scen = Scenario(scenario={ 'cs': ConfigurationSpace(), 'run_obj': obj, 'cutoff_time': '10' }, cmd_options=None) stats = Stats(scen) stats.start_timing() # Add first run to not trigger FirstRunCrashedException stats.ta_runs += 1 eta = ExecuteTARun(ta=lambda *args: None, stats=stats, run_obj=obj) return eta
def test_memory_limit_usage(self, run_mock): run_mock.return_value = StatusType.SUCCESS, 12345.0, 1.2345, {} stats = Stats(Scenario({})) stats.start_timing() tae = ExecuteTARun(lambda x: x**2, stats) self.assertRaisesRegex( ValueError, 'Target algorithm executor ' 'ExecuteTARun does not support restricting the memory usage.', tae.start, {'x': 2}, 'a', memory_limit=123) tae._supports_memory_limit = True rval = tae.start({'x': 2}, 'a', memory_limit=10) self.assertEqual(rval, run_mock.return_value)
def test_start_tae_return_abort(self, test_run): ''' testing abort ''' # Patch run-function for custom-return test_run.return_value = StatusType.ABORT, 12345.0, 1.2345, {} scen = Scenario(scenario={ 'cs': ConfigurationSpace(), 'output_dir': '' }, cmd_args=None) stats = Stats(scen) stats.start_timing() eta = ExecuteTARun(ta=lambda *args: None, stats=stats) self.assertRaises(TAEAbortException, eta.start, config={}, instance=1)
def test_start_crash_first_run(self, test_run): ''' testing crash-on-first-run ''' # Patch run-function for custom-return test_run.return_value = StatusType.CRASHED, 12345.0, 1.2345, {} scen = Scenario(scenario={ 'cs': ConfigurationSpace(), 'output_dir': '' }, cmd_args=None) stats = Stats(scen) stats.start_timing() eta = ExecuteTARun(ta=lambda *args: None, stats=stats) self.assertRaises(FirstRunCrashedException, eta.start, config={}, instance=1)
def test_start_exhausted_budget(self): ''' testing exhausted budget ''' # Set time-limit negative in scenario-options to trigger exception scen = Scenario(scenario={ 'wallclock_limit': -1, 'cs': ConfigurationSpace(), 'output_dir': '' }, cmd_args=None) stats = Stats(scen) stats.start_timing() eta = ExecuteTARun( ta=lambda *args: None, # Dummy-function stats=stats) self.assertRaises(BudgetExhaustedException, eta.start, config={}, instance=1)
def runhistory_builder(ta,scenario_dic,rng): tae_runner = ExecuteTARun(ta=ta) scenario = Scenario(scenario_dic) stats = Stats(scenario=scenario) traj_logger = TrajLogger(stats=stats,output_dir="/home/dfki/Desktop/temp") # if tae_runner.stats is None: # new_smac =SMAC(scenario=scenario,tae_runner=tae_runner) # tae_runner.stats = new_smac.stats stats.start_timing() deful_config_builder = DefaultConfiguration(tae_runner,scenario,stats,traj_logger,rng) config_milad =deful_config_builder._select_configuration() config_milad._values = None config_milad._values = {'balancing:strategy': 'none', 'categorical_encoding:__choice__': 'one_hot_encoding', 'classifier:__choice__': 'random_forest', 'imputation:strategy': 'mean', 'preprocessor:__choice__': 'no_preprocessing', 'rescaling:__choice__': 'standardize', 'categorical_encoding:one_hot_encoding:use_minimum_fraction': 'True', 'classifier:random_forest:bootstrap': 'True', 'classifier:random_forest:criterion': 'gini', 'classifier:random_forest:max_depth': 10, 'classifier:random_forest:max_features': 0.5, 'classifier:random_forest:max_leaf_nodes': 'None', 'classifier:random_forest:min_impurity_decrease': 0.0, 'classifier:random_forest:min_samples_leaf': 1, 'classifier:random_forest:min_samples_split': 2, 'classifier:random_forest:min_weight_fraction_leaf': 0.0, 'classifier:random_forest:n_estimators': 100, 'categorical_encoding:one_hot_encoding:minimum_fraction': 0.01} # config_milad._values = {'balancing:strategy': 'none', # 'categorical_encoding:__choice__': 'no_encoding', # 'classifier:__choice__': 'random_forest', # 'imputation:strategy': 'mean', # 'preprocessor:__choice__': 'pca', # 'preprocessor:copy':True, # 'preprocessor:iterated_power':'auto', # 'preprocessor:n_components':'None', # 'preprocessor:random_state':'None', # 'preprocessor:svd_solver':'auto', # 'preprocessor:tol':0.0, # 'preprocessor:whiten':'False', # 'rescaling:__choice__': 'None', # 'classifier:random_forest:bootstrap': 'True', # 'classifier:random_forest:class_weight': 'None', # 'classifier:random_forest:criterion': 'gini', # 'classifier:random_forest:max_depth': 'None', # 'classifier:random_forest:max_features': 'auto', # 'classifier:random_forest:max_leaf_nodes': 'None', # 'classifier:random_forest:min_impurity_decrease': 0.0, # 'classifier:random_forest:min_impurity_split': '1e-07', # 'classifier:random_forest:min_samples_leaf': 1, # 'classifier:random_forest:min_samples_split': 2, # 'classifier:random_forest:min_weight_fraction_leaf': 0.0, # 'classifier:random_forest:n_estimators': 10, # 'classifier:random_forest:n_jobs': 1, # 'classifier:random_forest:oob_score': 'False', # 'classifier:random_forest:random_state': 'None', # 'classifier:random_forest:verbose': 0, # 'classifier:random_forest:warm_start': 'False', # } # config_milad._vector =None status, cost, runtime, additional_info = tae_runner.start(config=config_milad,instance=None) print(status, cost, runtime, additional_info) runhistory = RunHistory(aggregate_func=average_cost) runhistory.add( config=config_milad, cost=cost, time=runtime, status=status, instance_id=None, additional_info=additional_info) return runhistory