def evaluate(mth, run_i, seed):
print(mth, run_i, seed, '===== start =====', flush=True)
def objective_function(config):
y = problem.evaluate_config(config)
return y
from smac.scenario.scenario import Scenario
from smac.facade.smac_facade import SMAC
from test.reproduction.smac_modified import RunHistory_modified # use modified RunHistory to save record
# Scenario object
scenario = Scenario({"run_obj": "quality",
"runcount_limit": max_runs,
"cs": cs,
"cutoff_time": time_limit_per_trial,
"initial_incumbent": "RANDOM",
"deterministic": "true",
})
runhistory = RunHistory_modified(None) # aggregate_func handled by smac_facade.SMAC
smac = SMAC(scenario=scenario, runhistory=runhistory,
tae_runner=objective_function, run_id=seed, # set run_id for smac output_dir
rng=np.random.RandomState(seed))
smac.optimize()
# keys = [k.config_id for k in smac.runhistory.data.keys()]
# perfs = [v.cost for v in smac.runhistory.data.values()]
config_list = smac.runhistory.config_list
perf_list = smac.runhistory.perf_list
time_list = smac.runhistory.time_list
return config_list, perf_list, time_list
def main():
# Initialize scenario, using runcount_limit as budget.
orig_scen_dict = {
'algo': 'python cmdline_wrapper.py',
'paramfile': 'param_config_space.pcs',
'run_obj': 'quality',
'runcount_limit': 25,
'deterministic': True,
'output_dir': 'restore_me'
}
original_scenario = Scenario(orig_scen_dict)
smac = SMAC(scenario=original_scenario)
smac.optimize()
print(
"\n########## BUDGET EXHAUSTED! Restoring optimization: ##########\n")
# Now the output is in the folder 'restore_me'
#
# We could simply modify the scenario-object, stored in
# 'smac.solver.scenario' and start optimization again:
#smac.solver.scenario.ta_run_limit = 50
#smac.optimize()
# Or, to show the whole process of recovering a SMAC-run from the output
# directory, create a new scenario with an extended budget:
new_scenario = Scenario(
orig_scen_dict,
cmd_args={
'runcount_limit': 50, # overwrite these args
'output_dir': 'restored'
})
# We load the runhistory, ...
rh_path = os.path.join(original_scenario.output_dir, "runhistory.json")
runhistory = RunHistory(aggregate_func=None)
runhistory.load_json(rh_path, new_scenario.cs)
# ... stats, ...
stats_path = os.path.join(original_scenario.output_dir, "stats.json")
stats = Stats(new_scenario)
stats.load(stats_path)
# ... and trajectory.
traj_path = os.path.join(original_scenario.output_dir, "traj_aclib2.json")
trajectory = TrajLogger.read_traj_aclib_format(fn=traj_path,
cs=new_scenario.cs)
incumbent = trajectory[-1]["incumbent"]
# Because we changed the output_dir, we might want to copy the old
# trajectory-file (runhistory and stats will be complete)
new_traj_path = os.path.join(new_scenario.output_dir, "traj_aclib2.json")
shutil.copy(traj_path, new_traj_path)
# Now we can initialize SMAC with the recovered objects and restore the
# state where we left off. By providing stats and a restore_incumbent, SMAC
# automatically detects the intention of restoring a state.
smac = SMAC(scenario=new_scenario,
runhistory=runhistory,
stats=stats,
restore_incumbent=incumbent)
smac.optimize()
def _getBasePredictor(self, randomSeed):
f = open(self._baseDataPath, "w")
f.truncate()
f.close()
self._lowTrainData = self._trainData.sample(fraction=self._lowRatio,
seed=randomSeed).cache()
self._midTrainData = self._trainData.sample(fraction=self._midRatio,
seed=randomSeed).cache()
cs = self.getPCS()
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": self._BPDS,
"cs": cs,
"deterministic": "true"
})
# Optimize, using a SMAC-object
smac = SMAC(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=self._baseEval)
smac.optimize()
df = self._spark.read.format("libsvm").load(self._baseDataPath)
rf = RandomForestRegressor()
rfModel = rf.fit(df)
self._lowTrainData.unpersist()
self._midTrainData.unpersist()
return rfModel
def tune_hyperparamerer_smac():
configs, results = list(), list()
config_space = create_hyperspace(regressor_id)
output_dir = "logs/smac3_output_%s" % (datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d_%H:%M:%S_%f'))
scenario_dict = {
'abort_on_first_run_crash': False,
"run_obj": "quality",
"cs": config_space,
"deterministic": "true",
"runcount-limit": trial_num,
'output_dir': output_dir
}
optimizer = SMAC(scenario=Scenario(scenario_dict),
rng=np.random.RandomState(45),
tae_runner=obj_func_example)
optimizer.optimize()
runhistory = optimizer.solver.runhistory
runkeys = list(runhistory.data.keys())
for key in runkeys:
_val = runhistory.data[key][0]
_config = runhistory.ids_config[key[0]]
configs.append(_config.get_dictionary())
results.append(_val)
inc_idx = np.argmin(results)
return configs[inc_idx], (configs, results)
def run(self, scen):
optimizer = SMAC(scenario=scen)
try:
optimizer.optimize()
except (TAEAbortException, FirstRunCrashedException) as err:
self.logger.error(err)
def run_smac(self,
num_iterations,
deterministic=True,
working_directory='/tmp'):
"""
Wrapper around SMAC to return a HpBandSter Result object to integrate better with the other methods
"""
try:
from smac.facade.smac_facade import SMAC
from smac.scenario.scenario import Scenario
except ImportError:
raise ImportError(
'To run SMAC, please install the latest python implementation of SMAC (pip install smac)!'
)
except:
raise
os.makedirs(working_directory, exist_ok=True)
tmp_fn = os.path.join(working_directory, 'smac_%s.json' % self.run_id)
# SMAC puts every call into a subprocess, so the data has to be stored on disk to
# be persistent. Here, we simply pickle the run data to disk after every call and
# load it before the actual call.
with open(tmp_fn, 'wb') as fh:
pickle.dump(self.run_data, fh)
def smac_objective(config, **kwargs):
with open(tmp_fn, 'rb') as fh:
self.run_data = pickle.load(fh)
loss = self.evaluate_and_log(config, budget=self.max_budget)
with open(tmp_fn, 'wb') as fh:
pickle.dump(self.run_data, fh)
return loss, []
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": num_iterations,
"cs": self.configspace,
"deterministic": deterministic,
"initial_incumbent": "RANDOM",
"output_dir": working_directory
})
smac = SMAC(scenario=scenario, tae_runner=smac_objective)
smac.optimize()
with open(tmp_fn, 'rb') as fh:
self.run_data = pickle.load(fh)
os.remove(tmp_fn)
return (self.get_result())
def main_cli(self):
'''
main function of SMAC for CLI interface
'''
cmd_reader = CMDReader()
args_, misc_args = cmd_reader.read_cmd()
logging.basicConfig(level=args_.verbose_level)
root_logger = logging.getLogger()
root_logger.setLevel(args_.verbose_level)
scen = Scenario(args_.scenario_file, misc_args)
rh = None
if args_.warmstart_runhistory:
aggregate_func = average_cost
rh = RunHistory(aggregate_func=aggregate_func)
scen, rh = merge_foreign_data_from_file(
scenario=scen,
runhistory=rh,
in_scenario_fn_list=args_.warmstart_scenario,
in_runhistory_fn_list=args_.warmstart_runhistory,
cs=scen.cs,
aggregate_func=aggregate_func)
initial_configs = None
if args_.warmstart_incumbent:
initial_configs = [scen.cs.get_default_configuration()]
for traj_fn in args_.warmstart_incumbent:
trajectory = TrajLogger.read_traj_aclib_format(fn=traj_fn,
cs=scen.cs)
initial_configs.append(trajectory[-1]["incumbent"])
if args_.modus == "SMAC":
optimizer = SMAC(scenario=scen,
rng=np.random.RandomState(args_.seed),
runhistory=rh,
initial_configurations=initial_configs)
elif args_.modus == "ROAR":
optimizer = ROAR(scenario=scen,
rng=np.random.RandomState(args_.seed),
runhistory=rh,
initial_configurations=initial_configs)
try:
optimizer.optimize()
finally:
# ensure that the runhistory is always dumped in the end
if scen.output_dir is not None:
optimizer.solver.runhistory.save_json(
fn=os.path.join(scen.output_dir, "runhistory.json"))
def main_cli(self):
'''
main function of SMAC for CLI interface
'''
self.logger.info("SMAC call: %s" % (" ".join(sys.argv)))
cmd_reader = CMDReader()
args_, misc_args = cmd_reader.read_cmd()
logging.basicConfig(level=args_.verbose_level)
root_logger = logging.getLogger()
root_logger.setLevel(args_.verbose_level)
scen = Scenario(args_.scenario_file, misc_args, run_id=args_.seed)
rh = None
if args_.warmstart_runhistory:
aggregate_func = average_cost
rh = RunHistory(aggregate_func=aggregate_func)
scen, rh = merge_foreign_data_from_file(
scenario=scen,
runhistory=rh,
in_scenario_fn_list=args_.warmstart_scenario,
in_runhistory_fn_list=args_.warmstart_runhistory,
cs=scen.cs,
aggregate_func=aggregate_func)
initial_configs = None
if args_.warmstart_incumbent:
initial_configs = [scen.cs.get_default_configuration()]
for traj_fn in args_.warmstart_incumbent:
trajectory = TrajLogger.read_traj_aclib_format(fn=traj_fn,
cs=scen.cs)
initial_configs.append(trajectory[-1]["incumbent"])
if args_.modus == "SMAC":
optimizer = SMAC(scenario=scen,
rng=np.random.RandomState(args_.seed),
runhistory=rh,
initial_configurations=initial_configs)
elif args_.modus == "ROAR":
optimizer = ROAR(scenario=scen,
rng=np.random.RandomState(args_.seed),
runhistory=rh,
initial_configurations=initial_configs)
try:
optimizer.optimize()
except (TAEAbortException, FirstRunCrashedException) as err:
self.logger.error(err)
def optimize(scenario, run, forest=False, seed=8, ratio=0.8):
types, bounds = get_types(scenario.cs, scenario.feature_array)
rfr = RandomForestWithInstances(types=types, bounds=bounds, instance_features=scenario.feature_array, seed=seed)
ei = EI(model=rfr)
if forest:
optimizer = ForestSearch(ei, scenario.cs, ratio=ratio)
else:
optimizer = InterleavedLocalAndRandomSearch(ei, scenario.cs)
scenario.output_dir = "%s_%s_%d_%lf" % ("./logs/run_", "forest_" if forest else "random_", seed, time.time())
smac = SMAC(
scenario=scenario,
rng=np.random.RandomState(seed),
model=rfr,
acquisition_function=ei,
acquisition_function_optimizer=optimizer,
tae_runner=run,
)
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
return smac.get_tae_runner().run(incumbent, 1)[1]
def smac():
# Build Configuration Space which defines all parameters and their ranges
configuration_space = ConfigurationSpace()
#Define initial ranges
num_of_hidden_nodes = UniformIntegerHyperparameter("num_of_hidden_nodes",
1,
max_hidden_nodes,
default_value=1)
decay = UniformFloatHyperparameter("decay", 0, 0.1, default_value=0)
configuration_space.add_hyperparameters([num_of_hidden_nodes, decay])
# creating the scenario object
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": hyperparameter_tuning_configs.SMAC_RUNCOUNT_LIMIT,
"cs": configuration_space,
"deterministic": "true",
"abort_on_first_run_crash": "false"
})
# optimize using an SMAC object
smac = SMAC(scenario=scenario,
rng=np.random.RandomState(1),
tae_runner=train_model)
incumbent = smac.optimize()
return incumbent.get_dictionary()
def getBestModel(self,
cutoff_time=None,
wallclock_limit=None,
runcount_limit=None,
tuner_timeout=None):
cs = self.getPCS()
dic = {"run_obj": "quality", "cs": cs, "deterministic": "true"}
if wallclock_limit:
dic["wallclock_limit"] = wallclock_limit
if runcount_limit:
dic["runcount_limit"] = runcount_limit
if tuner_timeout:
dic["tuner_timeout"] = tuner_timeout
scenario = Scenario(dic)
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, \
this might take a few minutes.")
self._trainData.cache()
smac = SMAC(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=self._eval)
cfg = smac.optimize()
cfg = self.getConfigurationDict(cfg)
estimator = self._estimator(**cfg)
model = estimator.fit(self._trainData)
self._trainData.unpersist()
return model
def fit(self, X, y):
"""Execture hyperparameter search.
Args:
X (array-like): Predictor matrix.
y (array-like): Target variable vector.
"""
# NB:
self._X, self._y = self._check_X_y(X, y)
# Location to store metadata from hyperparameter search.
search_metadata_dir = os.path.join(
self.output_dir, f'{self.experiment_id}_{self.random_state}'
)
if not os.path.isdir(search_metadata_dir):
os.makedirs(search_metadata_dir)
# NOTE: See https://automl.github.io/SMAC3/dev/options.html for configs.
scenario = Scenario(
{
'use_ta_time': True,
'wallclock_limit': float(700),
'cs': self.hparam_space,
'output_dir': search_metadata_dir,
'runcount-limit': self.max_evals,
'run_obj': 'quality',
'deterministic': 'true',
'abort_on_first_run_crash': 'true',
}
)
smac = SMAC(
scenario=scenario,
rng=np.random.RandomState(self.random_state),
tae_runner=self.cv_objective_fn
)
self._best_config = smac.optimize()
def smac_gm(self):
clu_cs = ConfigurationSpace()
cov_t = CategoricalHyperparameter(
"covariance_type", ["full", "tied", "diag", "spherical"])
tol = UniformFloatHyperparameter("tol", 1e-6, 0.1)
reg_c = UniformFloatHyperparameter("reg_covar", 1e-10, 0.1)
n_com = UniformIntegerHyperparameter("n_components", 2, 10)
max_iter = UniformIntegerHyperparameter("max_iter", 10, 1000)
clu_cs.add_hyperparameters([cov_t, tol, reg_c, n_com, max_iter])
clu_scenario = Scenario({
"run_obj":
"quality", # we optimize quality (alternatively runtime)
# "runcount-limit": Constants.num_eval, # maximum function evaluations
"cs": clu_cs, # configuration space
"deterministic": "true",
"tuner-timeout": constants.timeout,
"wallclock_limit": constants.timeout,
"cutoff_time": constants.timeout
})
print('Run GM SMAC ' + self.name)
smac = SMAC(scenario=clu_scenario, tae_runner=self.run_gm)
parameters = smac.optimize()
value = smac.get_runhistory().get_cost(parameters)
return value, parameters
def smac_em(self):
clu_cs = ConfigurationSpace()
n_init = UniformIntegerHyperparameter("n_init", 1, 15)
n_com = UniformIntegerHyperparameter("n_components", 2, 10)
reg_c = UniformFloatHyperparameter("min_covar", 1e-6, 0.1)
max_iter = UniformIntegerHyperparameter("n_iters", 10, 1000)
tr = UniformFloatHyperparameter("threshold", 1e-6, 0.1)
clu_cs.add_hyperparameters([n_init, tr, reg_c, n_com, max_iter])
clu_scenario = Scenario({
"run_obj":
"quality", # we optimize quality (alternatively runtime)
# "runcount-limit": Constants.num_eval, # maximum function evaluations
"cs": clu_cs, # configuration space
"deterministic": "true",
"tuner-timeout": constants.em_timeout,
"wallclock_limit": constants.em_timeout,
"cutoff_time": constants.em_timeout,
"runcount-limit": 1
})
print('Run EM SMAC ' + self.name)
smac = SMAC(scenario=clu_scenario, tae_runner=self.run_em)
parameters = smac.optimize()
value = smac.get_runhistory().get_cost(parameters)
return value, parameters
def maxsat(n_eval, n_variables, random_seed):
assert n_variables in [28, 43, 60]
if n_variables == 28:
evaluator = MaxSAT28(random_seed)
elif n_variables == 43:
evaluator = MaxSAT43(random_seed)
elif n_variables == 60:
evaluator = MaxSAT60(random_seed)
name_tag = 'maxsat' + str(n_variables) + '_' + datetime.now().strftime(
"%Y-%m-%d-%H:%M:%S:%f")
cs = ConfigurationSpace()
for i in range(n_variables):
car_var = CategoricalHyperparameter('x' + str(i + 1).zfill(2),
[str(elm) for elm in range(2)],
default_value='0')
cs.add_hyperparameter(car_var)
init_points_numpy = evaluator.suggested_init.long().numpy()
init_points = []
for i in range(init_points_numpy.shape[0]):
init_points.append(
Configuration(
cs, {
'x' + str(j + 1).zfill(2): str(init_points_numpy[i][j])
for j in range(n_variables)
}))
def evaluate(x):
x_tensor = torch.LongTensor(
[int(x['x' + str(j + 1).zfill(2)]) for j in range(n_variables)])
return evaluator.evaluate(x_tensor).item()
print('Began at ' + datetime.now().strftime("%H:%M:%S"))
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": n_eval,
"cs": cs,
"deterministic": "true",
'output_dir': os.path.join(EXP_DIR, name_tag)
})
smac = SMAC(scenario=scenario,
tae_runner=evaluate,
initial_configurations=init_points)
smac.optimize()
evaluations, optimum = evaluations_from_smac(smac)
print('Finished at ' + datetime.now().strftime("%H:%M:%S"))
return optimum
def branin(n_eval):
evaluator = Branin()
name_tag = '_'.join(
['branin', datetime.now().strftime("%Y-%m-%d-%H:%M:%S:%f")])
cs = ConfigurationSpace()
for i in range(len(evaluator.n_vertices)):
car_var = UniformIntegerHyperparameter('x' + str(i + 1).zfill(2),
0,
int(evaluator.n_vertices[i]) -
1,
default_value=25)
cs.add_hyperparameter(car_var)
init_points_numpy = evaluator.suggested_init.long().numpy()
init_points = []
for i in range(init_points_numpy.shape[0]):
init_points.append(
Configuration(
cs, {
'x' + str(j + 1).zfill(2): int(init_points_numpy[i][j])
for j in range(len(evaluator.n_vertices))
}))
def evaluate(x):
x_tensor = torch.LongTensor([
int(x['x' + str(j + 1).zfill(2)])
for j in range(len(evaluator.n_vertices))
])
return evaluator.evaluate(x_tensor).item()
print('Began at ' + datetime.now().strftime("%H:%M:%S"))
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": n_eval,
"cs": cs,
"deterministic": "true",
'output_dir': os.path.join(EXP_DIR, name_tag)
})
smac = SMAC(scenario=scenario,
tae_runner=evaluate,
initial_configurations=init_points)
smac.optimize()
evaluations, optimum = evaluations_from_smac(smac)
print('Finished at ' + datetime.now().strftime("%H:%M:%S"))
return optimum
def contamination(n_eval, lamda, random_seed_pair):
evaluator = Contamination(lamda, random_seed_pair)
name_tag = '_'.join([
'contamination', ('%.2E' % lamda),
datetime.now().strftime("%Y-%m-%d-%H:%M:%S:%f")
])
cs = ConfigurationSpace()
for i in range(CONTAMINATION_N_STAGES):
car_var = CategoricalHyperparameter('x' + str(i + 1).zfill(2),
[str(elm) for elm in range(2)],
default_value='0')
cs.add_hyperparameter(car_var)
init_points_numpy = evaluator.suggested_init.long().numpy()
init_points = []
for i in range(init_points_numpy.shape[0]):
init_points.append(
Configuration(
cs, {
'x' + str(j + 1).zfill(2): str(init_points_numpy[i][j])
for j in range(CONTAMINATION_N_STAGES)
}))
def evaluate(x):
x_tensor = torch.LongTensor([
int(x['x' + str(j + 1).zfill(2)])
for j in range(CONTAMINATION_N_STAGES)
])
return evaluator.evaluate(x_tensor).item()
print('Began at ' + datetime.now().strftime("%H:%M:%S"))
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": n_eval,
"cs": cs,
"deterministic": "true",
'output_dir': os.path.join(EXP_DIR, name_tag)
})
smac = SMAC(scenario=scenario,
tae_runner=evaluate,
initial_configurations=init_points)
smac.optimize()
evaluations, optimum = evaluations_from_smac(smac)
print('Finished at ' + datetime.now().strftime("%H:%M:%S"))
return optimum
def pest_control(n_eval, random_seed):
evaluator = PestControl(random_seed)
name_tag = 'pestcontrol_' + datetime.now().strftime("%Y-%m-%d-%H:%M:%S:%f")
cs = ConfigurationSpace()
for i in range(PESTCONTROL_N_STAGES):
car_var = CategoricalHyperparameter(
'x' + str(i + 1).zfill(2),
[str(elm) for elm in range(PESTCONTROL_N_CHOICE)],
default_value='0')
cs.add_hyperparameter(car_var)
init_points_numpy = sample_init_points([PESTCONTROL_N_CHOICE] *
PESTCONTROL_N_STAGES, 20,
random_seed).long().numpy()
init_points = []
for i in range(init_points_numpy.shape[0]):
init_points.append(
Configuration(
cs, {
'x' + str(j + 1).zfill(2): str(init_points_numpy[i][j])
for j in range(PESTCONTROL_N_STAGES)
}))
def evaluate(x):
x_tensor = torch.LongTensor([
int(x['x' + str(j + 1).zfill(2)])
for j in range(PESTCONTROL_N_STAGES)
])
return evaluator.evaluate(x_tensor).item()
print('Began at ' + datetime.now().strftime("%H:%M:%S"))
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": n_eval,
"cs": cs,
"deterministic": "true",
'output_dir': os.path.join(EXP_DIR, name_tag)
})
smac = SMAC(scenario=scenario,
tae_runner=evaluate,
initial_configurations=init_points)
smac.optimize()
evaluations, optimum = evaluations_from_smac(smac)
print('Finished at ' + datetime.now().strftime("%H:%M:%S"))
return optimum
def getBestModelWithSubSampling(self,
lowRatio=0.05,
midRatio=0.2,
taeLimit=50,
basePredictorsNum=5,
**kw):
self._lowRatio = lowRatio
self._midRatio = midRatio
self._taeLimit = taeLimit
self._basePredictors = self.__getBasePredictors(num=basePredictorsNum)
self._finalLowTrainData = self._trainData.sample(
fraction=self._lowRatio, seed=42)
self._finalPredictor = None
bestEvaluation = 0
bestModel = None
z = np.array([[]])
y = np.array([[]])
cs = self.getPCS()
self._omiga = np.zeros((basePredictorsNum + 1, 1))
if "run_obj" not in kw:
kw["run_obj"] = "quality"
kw["cs"] = cs
kw["deterministic"] = "true"
scenario = Scenario(kw)
for i in range(taeLimit):
# Optimize, using a SMAC-object
smac = SMAC(scenario=scenario,
rng=np.random.RandomState(i + 1),
tae_runner=self.__finalEval)
cfg = smac.optimize()
cfg = {k: cfg[k] for k in cfg if cfg[k]}
cfgList = [cfg[k] for k in sorted(cfg.keys())]
estimator = self._estimator(**cfg)
paramMap = estimator.extractParamMap()
paramGrid = ParamGridBuilder().build()
cv = CrossValidator(estimator=estimator,
evaluator=self._evaluator,
estimatorParamMaps=paramGrid,
numFolds=3)
lowModel = cv.fit(self._finalLowTrainData, params=paramMap)
hModel = cv.fit(self._trainData, params=paramMap)
if hModel.avgMetrics[0] > bestEvaluation:
bestModel = hModel.bestModel
bestEvaluation = hModel.avgMetrics[0]
transformData = self.__getBaseTransformedData(cfgList)
if not z.any():
z = np.array(transformData).reshape(
(-1, basePredictorsNum + 1))
else:
z = np.row_stack((z, transformData))
if not y.any():
y = np.array(hModel.avgMetrics[0] -
lowModel.avgMetrics[0]).reshape((-1, 1))
else:
y = np.row_stack(
(y, hModel.avgMetrics[0] - lowModel.avgMetrics[0]))
self._omiga = pinv(z.T.dot(z)).dot(z.T).dot(y)
return bestModel
def main():
# Prepare the data for training.
(x_train, y_train), (x_test, y_test), num_cls = load_data(dataset_name)
dmtrain = xgb.DMatrix(x_train, label=y_train)
dmvalid = xgb.DMatrix(x_test, label=y_test)
cs = create_configspace()
def objective_func(params):
num_round = int(params['n_estimators'])
parameters = {}
for p in params:
parameters[p] = params[p]
if num_cls > 2:
parameters['num_class'] = num_cls
parameters['objective'] = 'multi:softmax'
parameters['eval_metric'] = 'merror'
elif num_cls == 2:
parameters['objective'] = 'binary:logistic'
parameters['eval_metric'] = 'error'
parameters['tree_method'] = 'hist'
parameters['booster'] = 'gbtree'
parameters['nthread'] = args.nthread
parameters['silent'] = 1
watchlist = [(dmtrain, 'train'), (dmvalid, 'valid')]
model = xgb.train(parameters,
dmtrain,
num_round,
watchlist,
verbose_eval=1)
pred = model.predict(dmvalid)
if num_cls == 2:
pred = [int(i > 0.5) for i in pred]
acc = accuracy_score(dmvalid.get_label(), pred)
spill_result((params.get_dictionary(), params.get_array(), acc))
return 1 - acc
# Create scenario object
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": run_count,
"cs": cs,
"deterministic": "true"
})
smac = SMAC(scenario=scenario,
rng=np.random.RandomState(60),
tae_runner=objective_func)
incumbent = smac.optimize()
incumbent_value = objective_func(incumbent)
print('The best configuration is', incumbent)
print('The best performance, error:', incumbent_value)
# Rename the temporary file.
rename()
def fit(self, X_train, y_train, X_valid, y_valid,
max_epochs:int,
runcount_limit:int=100,
loss_func=categorical_crossentropy,
metrics=['accuracy']):
def obj_func(config, instance=None, seed=None, pc=None):
# continuing training if pc is given
# otherwise, construct new DNN
if pc is None:
K.clear_session()
pc = ParamFCNetClassification(config=config, n_feat=X_train.shape[1],
n_classes=self.n_classes,
max_num_epochs=max_epochs,
metrics=metrics,
verbose=0)
history = pc.train(X_train=X_train, y_train=y_train, X_valid=X_valid,
y_valid=y_valid, n_epochs=1)
final_loss = history.history["val_loss"][-1]
return final_loss, {"model": pc}
taf = SimpleTAFunc(obj_func)
cs = ParamFCNetClassification.get_config_space(max_num_layers=self.max_layers,
loss_functions=[loss_func],
output_activations=['softmax'],
use_l2_regularization=self.use_l2_regularization,
use_dropout=self.use_dropout)
ac_scenario = Scenario({"run_obj": "quality", # we optimize quality
"runcount-limit": max_epochs*runcount_limit,
"cost_for_crash": 10,
"cs": cs,
"deterministic": "true",
"output-dir": ""
})
intensifier = Intensifier(tae_runner=taf, stats=None,
traj_logger=None,
rng=np.random.RandomState(42),
run_limit=100,
max_epochs=max_epochs)
smac = SMAC(scenario=ac_scenario,
tae_runner=taf,
rng=np.random.RandomState(42),
intensifier=intensifier)
smac.solver.runhistory.overwrite_existing_runs = True
incumbent = smac.optimize()
return incumbent
def optimize():
# We load the iris-dataset (a widely used benchmark)
iris = datasets.load_iris()
#logger = logging.getLogger("SVMExample")
logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output
# Build Configuration Space which defines all parameters and their ranges
cs = ConfigurationSpace()
# We define a few possible types of SVM-kernels and add them as "kernel" to our cs
kernel = CategoricalHyperparameter("kernel", ["linear", "rbf", "poly", "sigmoid"], default="poly")
cs.add_hyperparameter(kernel)
# There are some hyperparameters shared by all kernels
C = UniformFloatHyperparameter("C", 0.001, 1000.0, default=1.0)
shrinking = CategoricalHyperparameter("shrinking", ["true", "false"], default="true")
cs.add_hyperparameters([C, shrinking])
# Others are kernel-specific, so we can add conditions to limit the searchspace
degree = UniformIntegerHyperparameter("degree", 1, 5, default=3) # Only used by kernel poly
coef0 = UniformFloatHyperparameter("coef0", 0.0, 10.0, default=0.0) # poly, sigmoid
cs.add_hyperparameters([degree, coef0])
use_degree = InCondition(child=degree, parent=kernel, values=["poly"])
use_coef0 = InCondition(child=coef0, parent=kernel, values=["poly", "sigmoid"])
cs.add_conditions([use_degree, use_coef0])
# This also works for parameters that are a mix of categorical and values from a range of numbers
# For example, gamma can be either "auto" or a fixed float
gamma = CategoricalHyperparameter("gamma", ["auto", "value"], default="auto") # only rbf, poly, sigmoid
gamma_value = UniformFloatHyperparameter("gamma_value", 0.0001, 8, default=1)
cs.add_hyperparameters([gamma, gamma_value])
# We only activate gamma_value if gamma is set to "value"
cs.add_condition(InCondition(child=gamma_value, parent=gamma, values=["value"]))
# And again we can restrict the use of gamma in general to the choice of the kernel
cs.add_condition(InCondition(child=gamma, parent=kernel, values=["rbf", "poly", "sigmoid"]))
# Scenario object
scenario = Scenario("test/test_files/svm_scenario.txt")
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = svm_from_cfg(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC(scenario=scenario, rng=np.random.RandomState(42),
tae_runner=svm_from_cfg)
incumbent = smac.optimize()
inc_value = svm_from_cfg(incumbent)
print("Optimized Value: %.2f" % (inc_value))
def main_loop(problem):
logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output
cs = ConfigurationSpace()
n_neighbors = UniformIntegerHyperparameter("n_neighbors", 2,10, default_value=5)
cs.add_hyperparameter(n_neighbors)
weights = CategoricalHyperparameter("weights", ["uniform","distance"], default_value="uniform")
algorithm = CategoricalHyperparameter("algorithm", ["ball_tree", "kd_tree","brute","auto"], default_value="auto")
cs.add_hyperparameters([weights, algorithm])
leaf_size = UniformIntegerHyperparameter("leaf_size", 1, 100, default_value=50)
cs.add_hyperparameter(leaf_size)
use_leaf_size= InCondition(child=leaf_size, parent=algorithm, values=["ball_tree","kd_tree"])
cs.add_condition(use_leaf_size)
p = UniformIntegerHyperparameter("p", 1,3, default_value=2)
cs.add_hyperparameter(p)
# Scenario object
max_eval=100000
scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit": max_eval, # maximum function evaluations
"cs": cs, # configuration space
"shared_model": True,
"output_dir": "/home/naamah/Documents/CatES/result_All/smac/KNN/run_{}_{}_{}".format(max_eval,
datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S'),
problem)
# "output_dir": "/home/naamah/Documents/CatES/result_All/smac/KNN/{}/run_{}_{}".format(problem,max_eval, datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S_%f')),
# "input_psmac_dirs":"/home/naamah/Documents/CatES/result_All/",
# "deterministic": "true"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = svm_from_cfg(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC(scenario=scenario,tae_runner=svm_from_cfg)
incumbent = smac.optimize()
inc_value = svm_from_cfg(incumbent)
print("Optimized Value: %.2f" % (inc_value))
return (incumbent)
# main_loop()
def optimize(algorithm,
data_location,
max_evals=10,
output_file='results/optimization',
do_plotting=False):
if experiment_type == 'mnist':
data = mnist_data
dictionary = dict_classifiers
loss_function = accuracy_score
classification = True
elif experiment_type == 'galaxy':
data = gz_data
dictionary = dict_regressors
loss_function = lambda x, y: np.sqrt(mean_squared_error(x, y))
elif experiment_type == 'neal':
data = neal_data
dictionary = dict_classifiers
loss_function = accuracy_score
classification = True
#logger = logging.getLogger("SVMExample")
logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output
cs = get_config_space()
# Scenario object
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit": max_evals, # maximum function evaluations
"cs": cs, # configuration space
"deterministic": "true"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = ml_cfg(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))
# Optimize, using a SMAC-object
print(
"Optimizing! Depending on your machine, this might take a few minutes."
)
model = ml_cfg(data)
smac = SMAC(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=model)
incumbent = smac.optimize()
inc_value = svm_from_cfg(incumbent)
print("Optimized Value: %.2f" % (inc_value))
def main(b, seed):
# Runs SMAC on a given benchmark
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": b.get_meta_information()['num_function_evals'],
"cs": b.get_configuration_space(),
"deterministic": "true",
"output_dir": "./{:s}/run-{:d}".format(b.get_meta_information()['name'],
seed)})
smac = SMAC(scenario=scenario, tae_runner=b,
rng=np.random.RandomState(seed))
x_star = smac.optimize()
print("Best value found:\n {:s}".format(str(x_star)))
print("with {:s}".format(str(b.objective_function(x_star))))
def opt_rosenbrock():
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformFloatHyperparameter("x1", -5, 5, default_value=-3))
cs.add_hyperparameter(UniformFloatHyperparameter("x2", -5, 5, default_value=-4))
scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit": 50, # maximum function evaluations
"cs": cs, # configuration space
"deterministic": "true",
"intensification_percentage": 0.000000001
})
smac = SMAC(scenario=scenario, rng=np.random.RandomState(42),
tae_runner=rosenbrock_2d)
incumbent = smac.optimize()
return incumbent
def dont_test_smac_choice(self):
import numpy as np
from sklearn import svm, datasets
from sklearn.model_selection import cross_val_score
# Import ConfigSpace and different types of parameters
from smac.configspace import ConfigurationSpace
# Import SMAC-utilities
from smac.tae.execute_func import ExecuteTAFuncDict
from smac.scenario.scenario import Scenario
from smac.facade.smac_facade import SMAC
tfm = PCA() | Nystroem() | NoOp()
planned_pipeline1 = (
OneHotEncoder(handle_unknown='ignore', sparse=False)
| NoOp()) >> tfm >> (LogisticRegression() | KNeighborsClassifier())
cs: ConfigurationSpace = get_smac_space(planned_pipeline1,
lale_num_grids=5)
# Scenario object
scenario = Scenario({
"run_obj":
"quality", # we optimize quality (alternatively runtime)
"runcount-limit": 1, # maximum function evaluations
"cs": cs, # configuration space
"deterministic": "true"
})
# Optimize, using a SMAC-object
tae = test_iris_fmin_tae(planned_pipeline1, num_folds=2)
print(
"Optimizing! Depending on your machine, this might take a few minutes."
)
smac = SMAC(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=tae)
incumbent = smac.optimize()
inc_value = tae(incumbent)
print("Optimized Value: %.2f" % (inc_value))
def test_smac(self):
import numpy as np
from sklearn import svm, datasets
from sklearn.model_selection import cross_val_score
# Import ConfigSpace and different types of parameters
from smac.configspace import ConfigurationSpace
# Import SMAC-utilities
from smac.tae.execute_func import ExecuteTAFuncDict
from smac.scenario.scenario import Scenario
from smac.facade.smac_facade import SMAC
from lale.search.SMAC import get_smac_space
lr = LogisticRegression()
cs: ConfigurationSpace = get_smac_space(lr)
# Scenario object
scenario = Scenario({
"run_obj":
"quality", # we optimize quality (alternatively runtime)
"runcount-limit": 1, # maximum function evaluations
"cs": cs, # configuration space
"deterministic": "true",
"abort_on_first_run_crash": False
})
# Optimize, using a SMAC-object
tae = test_iris_fmin_tae(lr, num_folds=2)
print(
"Optimizing! Depending on your machine, this might take a few minutes."
)
smac = SMAC(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=tae)
incumbent = smac.optimize()
inc_value = tae(incumbent)
print("Optimized Value: %.2f" % (inc_value))
def main():
cs = ConfigurationSpace()
cell_size = CategoricalHyperparameter("cell_size", [128],
default_value=128) # kick up to 256
n_cell = CategoricalHyperparameter("n_cell", [2], default_value=2)
dropout = CategoricalHyperparameter("dropout", [0.5], default_value=0.5)
activation = CategoricalHyperparameter("activation", ['sigmoid'],
default_value='sigmoid')
optimizer = CategoricalHyperparameter("optimizer", ['adam'],
default_value='adam')
optimizer_lr = CategoricalHyperparameter(
"optimizer_lr", [.001, .003, .006, .01, 0.03, 0.1], default_value=.01)
learning_decay_rate = UniformFloatHyperparameter("learning_decay_rate",
0,
0.9,
default_value=.6)
nn_type = CategoricalHyperparameter("nn_type", ['RNN', 'LSTM', 'GRU'],
default_value='LSTM')
epochs = CategoricalHyperparameter("epochs", [10], default_value=10)
cs.add_hyperparameters([
cell_size, n_cell, dropout, nn_type, activation, optimizer,
optimizer_lr, learning_decay_rate, epochs
])
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": 32,
"cs": cs,
"deterministic": "true"
})
scenario.output_dir_for_this_run = "C:\\NNwork\\HFSF\\SMAC3out"
scenario.output_dir = "C:\\NNwork\\HFSF\\SMAC3out"
smac = SMAC(scenario=scenario,
rng=np.random.RandomState(23),
tae_runner=rnn_from_cfg)
best_model = smac.optimize()
print_incumb(best_model)
np.save("C:\\NNwork\\HFSF\\SMAC3out\\best.cfg", best_model)
def main(dataset_name):
(x_train, y_train), (x_test, y_test), cls_num = load_dataset(dataset_name)
epochs_num, run_count = get_default_setting(dataset_name)
# Scenario object
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": run_count,
"cs": create_configspace(),
"deterministic": "true",
})
def wrapper_for_resnet_tuning(cfg):
val_error, test_error = train(cls_num,
epochs_num,
cfg, (x_train, y_train),
(x_test, y_test),
proportion=0.2,
seed=32)
print('the validation accuracy is ', 1 - val_error)
print('the test accuracy is ', 1 - test_error)
tmp_file = 'data/tmp_%s_result.pkl' % dataset_name
if not os.path.exists(tmp_file):
data = []
else:
with open(tmp_file, 'rb') as f:
data = pickle.load(f)
data.append(
(cfg.get_dictionary(), cfg.get_array(), val_error, test_error))
with open(tmp_file, 'wb') as f:
pickle.dump(data, f)
return val_error
smac = SMAC(scenario=scenario,
rng=np.random.RandomState(60),
tae_runner=wrapper_for_resnet_tuning)
incumbent = smac.optimize()
print('The best configuration is', incumbent)
tmp_file = 'data/tmp_%s_result.pkl' % dataset_name
target_file = 'data/result_%s_%d.pkl' % (dataset_name, run_count)
os.rename(tmp_file, target_file)
elif benchmark == "bohachevsky":
b = hpobench.Bohachevsky()
elif benchmark == "levy":
b = hpobench.Levy()
info = b.get_meta_information()
scenario = Scenario({"run_obj": "quality",
"runcount-limit": n_iters,
"cs": b.get_configuration_space(),
"deterministic": "true",
"initial_incumbent": "RANDOM",
"output_dir": ""})
smac = SMAC(scenario=scenario, tae_runner=b)
smac.optimize()
rh = smac.runhistory
incs = []
inc = None
idx = 1
t = smac.get_trajectory()
for i in range(n_iters):
if idx < len(t) and i == t[idx].ta_runs - 1:
inc = t[idx].incumbent
idx += 1
incs.append(inc)
# Offline Evaluation
regret = []
