def optimize(max_evals, max_ppn):
space = hp.choice('schedule', [
('static', hp.qloguniform('chunk_s', 2, 11, 10),
hp.randint('ppn_s', max_ppn)),
('dynamic', hp.qloguniform('chunk_d', 2, 11, 10),
hp.randint('ppn_d', max_ppn)),
('guided', hp.qloguniform('chunk_g', 2, 11, 10),
hp.randint('ppn_g', max_ppn)),
])
trials = Trials()
best = fmin(function, space=space, algo=tpe.suggest,
max_evals=max_evals, trials=trials)
return best, trials
def hyper_model() -> Trials:
space = {
'look_back': hp.randint('look_back', 1, 49),
'hidden_dim': hp.randint('hidden_dim', 5, 51),
'num_layers': hp.randint('num_layers', 1, 4)
}
trials = Trials()
fmin(objective_model,
space,
tpe.suggest,
max_evals=50,
timeout=10 * 60,
trials=trials)
print("Best result:", trials.argmin)
joblib.dump(trials, f'./log/{DATASET}_{MODEL.__name__}_model_optim.joblib')
return trials
def get_hyperparameter_search_space(dataset_properties=None,
optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
n_neighbors = UniformIntegerHyperparameter(name="n_neighbors",
lower=1,
upper=100,
log=True,
default_value=1)
weights = CategoricalHyperparameter(
name="weights",
choices=["uniform", "distance"],
default_value="uniform")
p = CategoricalHyperparameter(name="p",
choices=[1, 2],
default_value=2)
cs.add_hyperparameters([n_neighbors, weights, p])
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {
'n_neighbors': hp.randint('knn_n_neighbors', 100) + 1,
'weights': hp.choice('knn_weights', ['uniform', 'distance']),
'p': hp.choice('knn_p', [1, 2])
}
init_trial = {'n_neighbors': 1, 'weights': "uniform", 'p': 2}
return space
def branin(n_eval, random_seed):
evaluator = Branin()
space = [
hp.randint('x' + str(i), evaluator.n_vertices[i])
for i in range(len(evaluator.n_vertices))
]
init_points = sample_init_points(evaluator.n_vertices, 1,
random_seed).long().numpy()
init_points = [{
'x' + str(j): init_points[i][j]
for j in range(len(evaluator.n_vertices))
} for i in range(init_points.shape[0])]
def evaluate(x):
return evaluator.evaluate(torch.from_numpy(np.array(x))).item()
trials = base.Trials()
fmin(evaluate,
space,
algo=tpe.suggest,
max_evals=n_eval,
points_to_evaluate=init_points,
trials=trials)
evaluations, optimum = evaluations_from_trials(trials)
return optimum
def __init__(self, train_fname, test_fname, feat_types_fname, work_dir, task, max_evals,\
n_cv_folds, num_trees, num_threads, time_log_fname, train_query_fname=None, test_query_fname=None):
super().__init__(train_fname, test_fname, feat_types_fname, work_dir, task, max_evals,\
n_cv_folds, time_log_fname, train_query_fname=train_query_fname, test_query_fname=test_query_fname)
print("using XGBoost in " + str(xgb.__file__))
if self.task == "ranking":
self.objective = "rank:ndcg"
elif self.task == "binary":
self.objective = "binary:logistic"
elif self.task == "regression":
self.objective = "reg:linear"
else:
raise NotImplementedError("Unknown task " + self.task)
self.num_threads = num_threads
self.num_trees = num_trees
param_dict = {
'eta': hp.loguniform('eta', -7, 0),
'max_depth':0,
'max_leaves' : hp.qloguniform('max_leaves', 1, 7, 1),
'max_bin': 2 ** (hp.randint('max_bin_minus_4', 7) + 4),
'colsample_bytree': hp.uniform('colsample_bytree', 0.5, 1),
'subsample': hp.uniform('subsample', 0.5, 1),
'min_child_weight': hp.loguniform('min_child_weight', -16, 5),
'alpha': hp.choice('alpha', [0, hp.loguniform('alpha_log', -16, 2)]),
'lambda': hp.choice('lambda', [0, hp.loguniform('lambda_log', -16, 2)]),
'gamma': hp.choice('gamma', [0, hp.loguniform('gamma_log', -16, 2)]),
"tree_method":"hist",
"grow_policy":"lossguide"
}
self.param_space = param_dict
self.metric = None
def optimize(max_evals, max_ppn):
space = hp.choice('schedule', [
('static', hp.qloguniform('chunk_s', 2, 11,
10), hp.randint('ppn_s', max_ppn)),
('dynamic', hp.qloguniform('chunk_d', 2, 11,
10), hp.randint('ppn_d', max_ppn)),
('guided', hp.qloguniform('chunk_g', 2, 11,
10), hp.randint('ppn_g', max_ppn)),
])
trials = Trials()
best = fmin(function,
space=space,
algo=tpe.suggest,
max_evals=max_evals,
trials=trials)
return best, trials
def param_hyperopt(train):
"""
:param train:
:return:
"""
label = 'target'
features = train.columns.tolist()
features.remove('card_id')
features.remove('target')
train_data = lgb.Dataset(train[features], train[label], silent=True)
def hyperopt_objective(params):
"""
:param params:
:return:
"""
params = params_append(params)
print(params)
res = lgb.cv(params, train_data, 1000,
nfold=2,
stratified=False,
shuffle=True,
metrics='rmse',
early_stopping_rounds=20,
verbose_eval=False,
show_stdv=False,
seed=2020)
return min(res['rmse-mean'])
params_space = {
'learning_rate': hp.uniform('learning_rate', 1e-2, 5e-1),
'bagging_fraction': hp.uniform('bagging_fraction', 0.5, 1),
'feature_fraction': hp.uniform('feature_fraction', 0.5, 1),
'num_leaves': hp.choice('num_leaves', list(range(10, 300, 10))),
'reg_alpha': hp.randint('reg_alpha', 0, 10),
'reg_lambda': hp.uniform('reg_lambda', 0, 10),
'bagging_freq': hp.randint('bagging_freq', 1, 10),
'min_child_samples': hp.choice('min_child_samples', list(range(1, 30, 5)))
}
params_best = fmin(
hyperopt_objective,
space=params_space,
algo=tpe.suggest,
max_evals=30,
rstate=RandomState(2020))
return params_best
def visitSearchSpaceNumber(self,
space: SearchSpaceNumber,
path: str,
counter=None):
label = self.mk_label(path, counter)
if space.pgo is not None:
return scope.pgo_sample(
space.pgo, hp.quniform(label, 0,
len(space.pgo) - 1, 1))
dist = "uniform"
if space.distribution:
dist = space.distribution
if space.maximum is None:
raise SearchSpaceError(
path,
f"maximum not specified for a number with distribution {dist}")
max = space.getInclusiveMax()
# These distributions need only a maximum
if dist == "integer":
if not space.discrete:
raise SearchSpaceError(
path,
"integer distribution specified for a non discrete numeric type"
)
return hp.randint(label, max)
if space.minimum is None:
raise SearchSpaceError(
path,
f"minimum not specified for a number with distribution {dist}")
min = space.getInclusiveMin()
if dist == "uniform":
if space.discrete:
return scope.int(hp.quniform(label, min, max, 1))
else:
return hp.uniform(label, min, max)
elif dist == "loguniform":
# for log distributions, hyperopt requires that we provide the log of the min/max
if min <= 0:
raise SearchSpaceError(
path,
f"minimum of 0 specified with a {dist} distribution. This is not allowed; please set it (possibly using minimumForOptimizer) to be positive"
)
if min > 0:
min = math.log(min)
if max > 0:
max = math.log(max)
if space.discrete:
return scope.int(hp.qloguniform(label, min, max, 1))
else:
return hp.loguniform(label, min, max)
else:
raise SearchSpaceError(path, f"Unknown distribution type: {dist}")
def full_hyper_space(self):
from hyperopt import hp
hyper_space, hyper_choices = super(EmbeddingLSTM,
self).full_hyper_space()
hyper_space.update(
{"fex_loop_sequences": hp.randint("fex_loop_sequences", 2)})
return hyper_space, hyper_choices
def _optimization_space(self) -> Dict:
return {
'algorithm':
'SVM',
'stop_loss_pips':
hp.randint('stop_loss_pips', 200) + 1,
'take_profit_pips':
hp.randint('take_profit_pips', 200) + 1,
'maximum_days_to_hold_per_trade':
hp.randint('maximum_days_to_hold_per_trade', 10) + 1,
'bot_confidence_treshold':
hp.uniform('bot_confidence_treshold', 0, 1),
'bot_metadata': {
'kernel': hp.choice('kernel',
['linear', 'poly', 'rbf', 'sigmoid'])
}
}
def _optimization_space(self) -> Dict:
return {
'algorithm':
'KNN',
'stop_loss_pips':
10,
'take_profit_pips':
100,
'maximum_days_to_hold_per_trade':
hp.randint('maximum_days_to_hold_per_trade', 10) + 1,
'bot_confidence_treshold':
hp.uniform('bot_confidence_treshold', 0, 1),
'bot_metadata': {
'n_neighbors': hp.randint('n_neighbors', 20) + 1,
'weights': hp.choice('weights', ['uniform', 'distance'])
}
}
def optCodeFold(numEvals):
space = {
'epoch': 3+hp.randint('epoch',6),
'samples': 10000+5000*hp.randint('samps',3),
'lrate': 10**-(7+hp.randint('lrate',3)),
'bsize': 2+(hp.randint('bsize',3)),
}
trials = Trials()
best = fmin(opt,space=space,algo=tpe.suggest,max_evals=numEvals,trials=trials)
print 'best: ',best
return trials
def bayesian_opt(self, iterations):
"""
Function for managing a Bayesian hyperparameter optimization strategy. We define a hyperparameter space to
optimize over and choose an objective function to minimize.
:param iterations: Number of times to run the optimization strategy
:return: hyperopt's choice of best hyperparameters
"""
max_layers = 10
max_triangle_layers = 7
max_width = 20
max_filters = 5
max_filter_size = 10
max_batchsize = 128
if self.model_type == 'dense_rectangle':
space = {
'num_layers': 1 + hp.randint('num_layers', max_layers),
'layer_width': 1 + hp.randint('layer_width', max_width),
'batch_size': 2 + hp.randint('batch_size', max_batchsize)
}
elif self.model_type == 'dense_triangle':
space = {
'num_layers': 1 + hp.randint('num_layers', max_triangle_layers),
'batch_size': 2 + hp.randint('batch_size', max_batchsize)
}
elif self.model_type == 'conv':
space = {
'num_layers': 1 + hp.randint('num_layers', max_layers),
'num_filters': 1 + hp.randint('num_filters', max_filters),
'filter_size': 1 + hp.randint('filter_size', max_filter_size),
'batch_size': 2 + hp.randint('batch_size', max_batchsize)
}
else:
sys.exit('Choose dense_rectangle, dense_triangle or conv for model_type')
# Run the optimization strategy. tpe.suggest automatically chooses an appropriate algorithm for the
# Bayesian optimization scheme. fn is given the function that we want to minimize.
tpe_best = fmin(fn=self.objective_function, space=space, algo=tpe.suggest, max_evals=iterations)
# Generate a plot and csv record of costs and accuracies for all iterations
self.report_tpe_best(tpe_best)
self.generate_report()
return 'Optimized architecture: ' + str(tpe_best)
def get_hyperparameter_search_space(dataset_properties=None,
optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter(name="n_estimators",
lower=50,
upper=500,
default_value=50,
log=False)
learning_rate = UniformFloatHyperparameter(name="learning_rate",
lower=0.01,
upper=2,
default_value=0.1,
log=True)
algorithm = CategoricalHyperparameter(name="algorithm",
choices=["SAMME.R", "SAMME"],
default_value="SAMME.R")
max_depth = UniformIntegerHyperparameter(name="max_depth",
lower=1,
upper=10,
default_value=1,
log=False)
cs.add_hyperparameters(
[n_estimators, learning_rate, algorithm, max_depth])
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {
'n_estimators':
hp.randint('ab_n_estimators', 451) + 50,
'learning_rate':
hp.loguniform('ab_learning_rate', np.log(0.01), np.log(2)),
'algorithm':
hp.choice('ab_algorithm', ["SAMME.R", "SAMME"]),
'max_depth':
hp.randint('ab_max_depth', 10) + 1
}
init_trial = {
'n_estimators': 50,
'learning_rate': 0.1,
'algorithm': "SAMME.R",
'max_depth': 1
}
return space
def _get_space(self, options):
spaces = {}
for param_name, data in options.items():
algo, opt = data
if algo == 'randint':
spaces[param_name] = opt[1] + hp.randint(opt[0], opt[2] - opt[1])
else:
spaces[param_name] = getattr(hp, algo)(*opt)
return spaces
def _get_search_space(param_name, param):
if isinstance(param, p.Instrumentation):
space = {}
space["args"] = {
str(idx_param):
_get_search_space(str(idx_param),
param[0][idx_param]) # type: ignore
for idx_param in range(len(param[0].value))
}
space["kwargs"] = {
param_name: _get_search_space(param_name,
param[1][param_name]) # type: ignore
for param_name in param[1].value.keys()
}
return space
elif isinstance(param, (p.Log, p.Scalar)):
if (param.bounds[0][0] is None) or (param.bounds[1][0] is None):
if isinstance(param, p.Scalar) and not param.integer:
return hp.lognormal(label=param_name, mu=0, sigma=1)
raise ValueError(f"Scalar {param_name} not bounded.")
elif isinstance(param, p.Log):
return hp.loguniform(label=param_name,
low=np.log(param.bounds[0][0]),
high=np.log(param.bounds[1][0]))
elif isinstance(param, p.Scalar):
if param.integer:
return hp.randint(label=param_name,
low=int(param.bounds[0][0]),
high=int(param.bounds[1][0]))
else:
return hp.uniform(label=param_name,
low=param.bounds[0][0],
high=param.bounds[1][0])
elif isinstance(param, p.Choice):
list_types = [
type(param.choices[i]) for i in range(len(param.choices))
if not isinstance(param.choices[i], (p.Instrumentation,
p.Constant))
]
if len(list_types) != len(set(list_types)):
raise NotImplementedError
return hp.choice(
param_name,
[
_get_search_space(param_name + "__" + str(i), param.choices[i])
for i in range(len(param.choices))
],
)
elif isinstance(param, p.Constant):
return param.value
# Hyperopt do not support array
raise NotImplementedError
def many_dists():
a = hp.choice("a", [0, 1, 2])
b = hp.randint("b", 10)
bb = hp.randint("bb", 12, 25)
c = hp.uniform("c", 4, 7)
d = hp.loguniform("d", -2, 0)
e = hp.quniform("e", 0, 10, 3)
f = hp.qloguniform("f", 0, 3, 2)
g = hp.normal("g", 4, 7)
h = hp.lognormal("h", -2, 2)
i = hp.qnormal("i", 0, 10, 2)
j = hp.qlognormal("j", 0, 2, 1)
k = hp.pchoice("k", [(0.1, 0), (0.9, 1)])
z = a + b + bb + c + d + e + f + g + h + i + j + k
return {
"loss": scope.float(scope.log(1e-12 + z**2)),
"status": base.STATUS_OK
}
def complexSpace(searchDict):
# define your own complex space
try:
# combined
searchDict['rf__max_features'] = hp.choice(
'rf__max_features',
[hp.choice('c1', ['sqrt', 'log2']), 1 + hp.randint('c2', 3)])
except:
print("not working")
return searchDict
def _uniform_perm(key, tweak: Tweaks.UniformPermutation):
n = len(tweak.values)
return { # TODO use a namedtuple
'indexes': [
hp.randint(str(key) + '-perm-' + str(i), n - i)
for i in range(tweak.k)
],
'values':
list(tweak.values)
}, _expand_perm
def minimize_lunarlander():
# define a search space
space = {
'alpha': hp.randint('alpha', 1, 128) / 10000,
'beta': hp.randint('beta', 1, 128) / 10000,
'gamma': hp.randint('gamma', 8192, 10000) / 10000,
}
# minimize the objective over the space
def _objective(strategy):
env = gym.make("LunarLander-v2")
return lunar_lander_objective(strategy, env)
best_configuration = fmin(_objective,
space,
algo=tpe.suggest,
max_evals=100)
print(best_configuration)
def _estimate_local_cv(X, y):
"""Estimate of the score using 2x5, i.e. nested cross-validation strategies."""
pipe = Pipeline([('clf', XGBClassifier(objective='binary:logistic'))])
space = {}
space['clf__max_depth'] = 5 + hp.randint('clf__max_depth', 5) # (5, 10)
space['clf__learning_rate'] = hp.uniform('clf__learning_rate', 0.01, 3.0)
space['clf__n_estimators'] = 70 + 5 * hp.randint('clf__n_estimators',
10) # 50:5:100
space['clf__gamma'] = hp.loguniform('clf__gamma', 0.1, 1.0)
space['clf__min_child_weight'] = 1 + hp.randint('clf__min_child_weight',
9) # (2, 10)
space['clf__max_delta_step'] = hp.uniform('clf__max_delta_step', 0.0, 0.1)
space['clf__subsample'] = hp.uniform('clf__subsample', 0.5, 1.0)
space['clf__colsample_bytree'] = hp.uniform('clf__colsample_bytree', 0.5,
1.)
def objective(params):
"""Objective is to minimize log_loss. So, output log_loss scores."""
pipe.set_params(**params)
scores = cross_val_score(pipe,
X,
y,
scoring='neg_log_loss',
cv=2,
n_jobs=-1)
return -1.0 * scores.mean()
# to store details of each iteration
# Note: with MongoTrials() as mentioned in http://bit.ly/2miT1Uc
# custom objective functions don't work because of pickling errors
trials = Trials()
# run hyperparameter search using tpe algorithm
best = fmin(objective,
space,
algo=tpe.suggest,
max_evals=50,
trials=trials,
verbose=3)
# get values of optimal parameters
best_params = space_eval(space, best)
return pipe, best_params, objective(best_params)
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
n_estimators = Constant("n_estimators", 100)
criterion = CategoricalHyperparameter(
"criterion", ["gini", "entropy"], default_value="gini")
max_features = UniformFloatHyperparameter("max_features", 0, 1,
default_value=0.5, q=0.05)
max_depth = UnParametrizedHyperparameter(name="max_depth", value="None")
max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None")
min_samples_split = UniformIntegerHyperparameter(
"min_samples_split", 2, 20, default_value=2)
min_samples_leaf = UniformIntegerHyperparameter(
"min_samples_leaf", 1, 20, default_value=1)
min_weight_fraction_leaf = UnParametrizedHyperparameter(
'min_weight_fraction_leaf', 0.)
min_impurity_decrease = UnParametrizedHyperparameter(
'min_impurity_decrease', 0.)
bootstrap = CategoricalHyperparameter(
"bootstrap", ["True", "False"], default_value="False")
cs.add_hyperparameters([n_estimators, criterion, max_features,
max_depth, max_leaf_nodes, min_samples_split,
min_samples_leaf, min_weight_fraction_leaf,
min_impurity_decrease, bootstrap])
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {'n_estimators': 100,
'criterion': hp.choice('ets_criterion', ['gini', 'entropy']),
'max_features': hp.uniform('ets_max_features', 0, 1),
'max_depth': "None",
'max_leaf_nodes': "None",
'min_samples_leaf': hp.randint('ets_samples_leaf', 20) + 1,
'min_samples_split': hp.randint('ets_samples_split', 19) + 2,
'min_weight_fraction_leaf': 0.,
'min_impurity_decrease': 0.,
'bootstrap': hp.choice('ets_bootstrap', ['True', 'False'])}
return space
def _convert_randint(k: str, v: RandInt) -> Any:
_high = adjust_high(
0, v.high - v.low, v.q, include_high=v.include_high # type:ignore
)
n = int(np.round(_high / v.q))
if not v.log:
return hp.randint(k, 0, n) * v.q + v.low, lambda x: int(np.round(x))
return (
hp.qloguniform(k, np.log(v.q), np.log(_high), q=v.q) + v.low - v.q,
lambda x: int(np.round(x)),
)
def testConvertHyperOpt(self):
from ray.tune.suggest.hyperopt import HyperOptSearch
from hyperopt import hp
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(-15, -10).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = HyperOptSearch.convert_search_space(config)
hyperopt_config = {
"a": hp.choice("a", [2, 3, 4]),
"b": {
"x": hp.randint("x", -15, -10),
"y": 4,
"z": hp.loguniform("z", np.log(1e-4), np.log(1e-2))
}
}
searcher1 = HyperOptSearch(space=converted_config,
random_state_seed=1234,
metric="a",
mode="max")
searcher2 = HyperOptSearch(space=hyperopt_config,
random_state_seed=1234,
metric="a",
mode="max")
config1 = searcher1.suggest("0")
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(-15, -10)))
self.assertEqual(config1["b"]["y"], 4)
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = HyperOptSearch(metric="a", mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
assert trial.config["a"] in [2, 3, 4]
mixed_config = {"a": tune.uniform(5, 6), "b": hp.uniform("b", 8, 9)}
searcher = HyperOptSearch(space=mixed_config, metric="a", mode="max")
config = searcher.suggest("0")
self.assertTrue(5 <= config["a"] <= 6)
self.assertTrue(8 <= config["b"] <= 9)
def init_lgbm_param_grid(seed):
params = dict()
params['num_iterations'] = hp.choice('num_iterations',
[10, 100, 1000, 2000, 3000])
params['early_stopping_round'] = hp.choice('early_stopping_round',
[50, 100])
params['learning_rate'] = hp.loguniform('learning_rate', -5, -3)
params['boosting_type'] = hp.choice('boosting_type', ['gbdt', 'dart'])
params['objective'] = 'binary'
params['metric'] = 'binary_logloss'
params['num_leaves'] = 2 + hp.randint('num_leaves', 21),
params['max_depth'] = 3 + hp.randint('max_depth', 11),
params['num_threads'] = 8
params['feature_fraction'] = hp.uniform('feature_fraction', 0.6, 0.95)
params['bagging_fraction'] = hp.uniform('bagging_fraction', 0.4, 0.95)
params['bagging_freq'] = 1 + hp.randint('bagging_freq', 9),
params['seed'] = seed
params['bagging_seed'] = seed
params['verbose'] = -1
return params
def _conv_hyperopt_param_dist(param_name, param_dist):
if param_dist["valtype"] == "numeric":
if param_dist["dtype"] == "int":
param_dist = hp.randint(param_name, int(param_dist["high"]))
elif param_dist["dtype"] == "float":
param_dist = hp.uniform(param_name, param_dist["low"],
param_dist["high"])
elif param_dist["valtype"] == "category":
param_dist = hp.choice(param_name, param_dist["categories"])
return param_dist
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter(
name="n_estimators", lower=50, upper=500, default_value=50, log=False)
sampling_strategy = CategoricalHyperparameter(
name="sampling_strategy", choices=["majority", "not minority", "not majority", "all"],
default_value="not minority")
replacement = CategoricalHyperparameter(
"replacement", ["True", "False"], default_value="False")
ab_n_estimators = UniformIntegerHyperparameter(
name="ab_n_estimators", lower=50, upper=500, default_value=50, log=False)
ab_learning_rate = UniformFloatHyperparameter(
name="ab_learning_rate", lower=0.01, upper=2, default_value=0.1, log=True)
ab_algorithm = CategoricalHyperparameter(
name="ab_algorithm", choices=["SAMME.R", "SAMME"], default_value="SAMME.R")
ab_max_depth = UniformIntegerHyperparameter(
name="ab_max_depth", lower=1, upper=10, default_value=1, log=False)
cs.add_hyperparameters([n_estimators, sampling_strategy, replacement, ab_n_estimators,
ab_learning_rate, ab_algorithm, ab_max_depth])
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {'n_estimators': hp.randint('easy_ensemble_n_estimators', 451) + 50,
'sampling_strategy': hp.choice('easy_ensemble_sampling_strategy',
["majority", "not minority", "not majority", "all"]),
'replacement': hp.choice('easy_ensemble_replacement', ["True", "False"]),
'ab_n_estimators': hp.randint('ab_n_estimators', 451) + 50,
'ab_learning_rate': hp.loguniform('ab_learning_rate', np.log(0.01), np.log(2)),
'ab_algorithm': hp.choice('ab_algorithm', ["SAMME.R", "SAMME"]),
'ab_max_depth': hp.randint('ab_max_depth', 10) + 1}
init_trial = {'n_estimators': 10,
'sampling_strategy': "not minority",
'replacement': "False",
'ab_n_estimators': 50,
'ab_learning_rate': 0.1,
'ab_algorithm': "SAMME.R",
'ab_max_depth': 1}
return space
def test_hyperopt2skopt_space():
hyperopt_space = {
'int_uniform': scope.int(hp.uniform('l_int_uniform', 1, 7)),
'randint': hp.randint('l_randint', 7),
'uniform': hp.uniform('l_uniform', -3, 3),
'uniform_named': hp.uniform('l_uniform_named', low=1, high=10),
'uniform_part_named': hp.uniform('l_uniform_part_named', 1, high=10),
'unsupported': hp.loguniform('l_unsupported', -1, 5),
'choice': hp.choice('choice', ['a', 'b', 4]),
'random_param': 'just_one_val',
}
space, ind2names = hyperopt2skopt_space(hyperopt_space, sample_size=100)
assert len(space) == len(ind2names)
named_space = {ind2names[i]: space[i] for i in range(len(space))}
int_uniform = named_space['int_uniform']
assert isinstance(int_uniform, Integer)
assert int_uniform.low == 1
assert int_uniform.high == 7
randint = named_space['randint']
assert isinstance(randint, Integer)
assert randint.low == 0
assert randint.high == 7
uniform = named_space['uniform']
assert isinstance(uniform, Real)
assert uniform.low == -3
assert uniform.high == 3
uniform_named = named_space['uniform_named']
assert isinstance(uniform_named, Real)
assert uniform_named.low == 1
assert uniform_named.high == 10
uniform_part_named = named_space['uniform_part_named']
assert isinstance(uniform_part_named, Real)
assert uniform_part_named.low == 1
assert uniform_part_named.high == 10
unsupported = named_space['unsupported']
assert isinstance(unsupported, Categorical)
assert len(unsupported.categories) == 100
assert all([np.exp(-1) <= x <= np.exp(5) for x in unsupported.categories])
choice = named_space['choice']
assert isinstance(choice, Categorical)
assert set(choice.categories) == {'a', 'b', 4}
random_param = named_space['random_param']
assert isinstance(random_param, Categorical)
assert set(random_param.categories) == {'just_one_val'}
def __init__(self, ):
"""
maximal depth of the decision tree and, to a lesser degree, the learning rate.
One interesting observation is that, in contrast to other
ensembletechniques, the number of iterations did not seem to
influenceperformance too much. The minimum value (50) appears
to alreadybe large enough to ensure good performance, and
increasing it doesnot lead to significantly better results.
the maximum depth of the decision tree in Adaboost should typicallybe set to a large value
"""
self.name = 'Adaptive Boosting classifier'
self.short_name = 'AdaBoost'
self.scale = None
self.default_parameters = {
"base_estimator": None,
"n_estimators": 50,
"learning_rate": 1.,
"algorithm": 'SAMME.R',
"random_state": None
}
self.parameters_mandatory_first_check = [{
"base_estimator":
DecisionTree().get_skl_estimator(max_depth=10)
}]
self.parameters_range = {
"learning_rate": [0.01, 2.0], # (log-scale)
"base_estimator":
DecisionTree().get_skl_estimator(max_depth=[
1, 10
]) # !!! как тут быть? nested? (10 optimal maybe need more)
}
self.hpo_results = []
self.search_space = {
'name': 'AdaBoost',
'model': ensemble.AdaBoostClassifier,
'param': {
"learning_rate": hp.uniform('AdaBoost_p1', 0.01, 2.0),
"base_estimator":
{ # если захочу добавить другой базовый алгоритм то hp.choise
"name": 'DecisionTree',
"model": tree.DecisionTreeClassifier,
"max_depth": 1 + hp.randint('AdaBoost_p2', 12)
}
}
}
def minimize_cartpole():
# define a search space
space = {
'alpha': hp.randint('alpha', 1, 128) /
10000, # alpha and beta between 0.0001 and 0.0256
'beta': hp.randint('beta', 1, 128) / 10000,
'gamma': hp.randint('gamma', 8192, 10000) /
10000, # discount factor between .8192 and 1.0
}
# minimize the objective over the space
def _objective(strategy):
env = gym.make("CartPole-v1")
return cart_pole_objective(strategy, env)
best_configuration = fmin(_objective,
space,
algo=tpe.suggest,
max_evals=100)
print(best_configuration)
def test_basic():
space = hp.randint("a", 5)
x = np.zeros(5)
rng = np.random.default_rng(123)
for i in range(0, 1000):
nesto = hyperopt.pyll.stochastic.sample(space, rng=rng)
x[nesto] += 1
print(x)
for i in x:
assert 100 < i < 300
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter(
name="n_estimators", lower=50, upper=500, default_value=50, log=False)
max_features = UniformFloatHyperparameter(
"max_features", 0., 1., default_value=0.5)
bootstrap = CategoricalHyperparameter(
"bootstrap", ["True", "False"], default_value="True")
bootstrap_features = CategoricalHyperparameter(
"bootstrap_features", ["True", "False"], default_value="False")
sampling_strategy = CategoricalHyperparameter(
name="sampling_strategy", choices=["majority", "not minority", "not majority", "all"],
default_value="not minority")
replacement = CategoricalHyperparameter(
"replacement", ["True", "False"], default_value="False")
max_depth = UniformIntegerHyperparameter(
name="max_depth", lower=1, upper=10, default_value=1, log=False)
cs.add_hyperparameters(
[n_estimators, max_features, bootstrap, bootstrap_features, sampling_strategy, replacement,
max_depth])
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {'n_estimators': hp.randint('bal_bagging_n_estimators', 451) + 50,
'max_features': hp.uniform('bal_bagging_max_features', 0, 1),
'bootstrap': hp.choice('bal_bagging_bootstrap', ["True", "False"]),
'bootstrap_features': hp.choice('bal_bagging_bootstrap_features', ["True", "False"]),
'sampling_strategy': hp.choice('bal_bagging_sampling_strategy',
["majority", "not minority", "not majority", "all"]),
'replacement': hp.choice('bal_bagging_replacement', ["True", "False"]),
'max_depth': hp.randint('bal_bagging_max_depth', 10) + 1}
init_trial = {'n_estimators': 10,
'max_features': 0.5,
'bootstrap': "True",
'bootstrap_features': "False",
'sampling_strategy': "not minority",
'replacement': "False",
'max_depth': 1}
return space
def _convert_categorical(pp_var, bindings):
val_type = pp_var.keywords['value_type']
assert is_sequence_of_literals(val_type), \
"val_type for categorical must be sequence of literals"
n_choices = len(val_type.args)
assert is_literal(pp_var.keywords['name'])
if 'p' in pp_var.keywords:
p = pp_var.keywords['p']
randint_stoch = pyll.scope.categorical(bindings[p], upper=n_choices)
name = bindings[pp_var.keywords['name']]
randint = pyll.scope.hyperopt_param(name, randint_stoch)
else:
randint = hp.randint(bindings[pp_var.keywords['name']], n_choices)
return bindings[val_type][randint]
def test_expr_to_config():
z = hp.randint('z', 10)
a = hp.choice('a',
[
hp.uniform('b', -1, 1) + z,
{'c': 1, 'd': hp.choice('d',
[3 + hp.loguniform('c', 0, 1),
1 + hp.loguniform('e', 0, 1)])
}])
expr = as_apply((a, z))
hps = {}
expr_to_config(expr, (True,), hps)
for label, dct in hps.items():
print label
print ' dist: %s(%s)' % (
dct['node'].name,
', '.join(map(str, [ii.eval() for ii in dct['node'].inputs()])))
if len(dct['conditions']) > 1:
print ' conditions (OR):'
for condseq in dct['conditions']:
print ' ', ' AND '.join(map(str, condseq))
elif dct['conditions']:
for condseq in dct['conditions']:
print ' conditions :', ' AND '.join(map(str, condseq))
assert hps['a']['node'].name == 'randint'
assert hps['b']['node'].name == 'uniform'
assert hps['c']['node'].name == 'loguniform'
assert hps['d']['node'].name == 'randint'
assert hps['e']['node'].name == 'loguniform'
assert hps['z']['node'].name == 'randint'
assert set([(True, EQ('a', 0))]) == set([(True, EQ('a', 0))])
assert hps['a']['conditions'] == set([(True,)])
assert hps['b']['conditions'] == set([
(True, EQ('a', 0))]), hps['b']['conditions']
assert hps['c']['conditions'] == set([
(True, EQ('a', 1), EQ('d', 0))])
assert hps['d']['conditions'] == set([
(True, EQ('a', 1))])
assert hps['e']['conditions'] == set([
(True, EQ('a', 1), EQ('d', 1))])
assert hps['z']['conditions'] == set([
(True,),
(True, EQ('a', 0))])
def hyper_parameter_search(max_evals=100):
from hyperopt import fmin, tpe, hp, STATUS_OK, STATUS_FAIL
data = read_dataset('../datasets/internet-traffic-data-5minutes.csv')
space = {
'nneurons': hp.randint('nneurons', 41),
'window': hp.randint('window', 2048),
'season': hp.choice('season', ['no_season', 'half_day', 'full_day']),
'activation_function': hp.choice('func', ['sigmoid', 'tanh', 'relu'])
}
def objective(params):
nneurons = params['nneurons']
if params['season'] == 'full_day':
window = create_window_array(params['window'], season_lag=288)
if params['season'] == 'half_day':
window = create_window_array(params['window'], season_lag=168)
else:
window = create_window_array(params['window'])
if not any(window) or nneurons < 2:
return {'status': STATUS_FAIL}
X_train, y_train, *_ = split_dataset(
data, window, ratio=0.90, standardize=True)
model = compile_model(
nneurons, input_dim=sum(1 for x in window if x), loss_fn='mse',
activation_fn=params['activation_function'])
hist = model.fit(
X_train, y_train, nb_epoch=50, validation_split=0.1,
callbacks=[EarlyStopping(monitor='val_loss', patience=2)],
verbose=0)
return {'loss': hist.history['val_loss'][-1], 'status': STATUS_OK}
return fmin(objective, space=space, algo=tpe.suggest, max_evals=max_evals)
def many_dists():
a = hp.choice('a', [0, 1, 2])
b = hp.randint('b', 10)
c = hp.uniform('c', 4, 7)
d = hp.loguniform('d', -2, 0)
e = hp.quniform('e', 0, 10, 3)
f = hp.qloguniform('f', 0, 3, 2)
g = hp.normal('g', 4, 7)
h = hp.lognormal('h', -2, 2)
i = hp.qnormal('i', 0, 10, 2)
j = hp.qlognormal('j', 0, 2, 1)
k = hp.pchoice('k', [(.1, 0), (.9, 1)])
z = a + b + c + d + e + f + g + h + i + j + k
return {'loss': scope.float(scope.log(1e-12 + z ** 2)),
'status': base.STATUS_OK}
def __MethodsProcess(self):
assert self.datasets is not None
res = self.training_method.Run(self.params, self.datasets, self.result.__class__)
self.result = res
return res
def reduce_flatten(reduce_func, L):
if isinstance(L, Iterable):
return reduce_func(reduce_flatten(reduce_func, element) for element in L)
else:
return L
def fn(hyperparams):
print "Trying with %s..." % (str(hyperparams),)
params_dict = self.hp_params.get_values_from_dict(hyperparams)
params = self.params.__class__.from_dict(params_dict)
ds = copy.deepcopy(self.datasets)
res = self.training_method.Run(params, ds, self.result.__class__)
sum_all_costs = reduce_flatten(sum, res.costs)
return {"loss": sum_all_costs, "results": res, "status" : STATUS_OK}
args = self.hp_params.get_hyper_args()
min_nb_layers = self.hp_params.min_nb_layers
max_nb_layers = self.hp_params.max_nb_layers
space = scope.switch(
hp.randint('d', max_nb_layers - min_nb_layers),
*args
)
trials = Trials()
best = fmin(fn=fn, space=space, algo=tpe.suggest, max_evals=50, trials=trials)
best_trial = min(trials.results, key=lambda res: res["loss"])
self.result = best_trial["results"]
print "Best params : %s, Loss : %.2f" % (str(best), best_trial["loss"],)
args.append(vars)
def from_dict(d):
depth = d["depth"]
corruption = [None] * depth
values = {}
for name in templates.keys():
values[name] = [None] * depth
for name, value in d.items():
for template_name in templates.keys():
if name.startswith(template_name):
layer = int(name[len(template_name):])
values[template_name][layer - 1] = value
return values
space = scope.switch(
hp.randint('d', 3) + 1,
*args
)
def fn(args):
print args
return {"loss" : 1, "status": STATUS_OK, "cost": 150
trials = Trials()
best = fmin(fn=fn, space=space, max_evals=10, algo=tpe.suggest, trials=trials)
print trials.results
print ("subsample,{}".format (subsample), file=f1)
print ("col_sample_bytree,{}".format (col_sample_bytree), file=f1)
print ("min_child_weight,{}".format (min_child_weight), file=f1)
print ("gamma,{}".format (gamma), file=f1)
ro.globalenv['params'] = rParams
rmse = ro.r('fitFunc (fitFormula, trainData, params)')
print (rmse[0])
print ("Result is,{}".format (rmse[0]), file=f1)
f1.close()
return (rmse[0])
space = (
hp.loguniform ('eta', -6, 0),
hp.randint ('max_depth', 6) + 2,
hp.uniform ('subsample', 0.5, 1),
hp.uniform ('col_sample_bytree', 0.5, 1),
hp.uniform ('min_child_weight', 0, 3),
hp.uniform ('gamma', 0, 3)
)
pandas2ri.activate()
ro.r('source ("xgbDirect-hyperopt.R")')
best = fmin (objective,
space = space,
algo = tpe.suggest,
max_evals= 100)
import time
# why objective on the example on the pc takes args (=2)?
def objective(x):
return {'loss': x ** 2 + 300,
'status': STATUS_FAIL if x > 0 else STATUS_OK,
'eval_time': time.time(),
'random_stuff': {'something': 'string?'},
'attachments': {'time_module': pickle.dumps(time.time)}
}
space = hp.choice('a',
[
(1 + hp.randint('c1', 10)),
(hp.uniform('c2', -10, 10))
])
# why does hp.randint('x', 10) always return same number?
# works without space=____
trials = Trials()
best = fmin(objective,
space=space,
# space=hp.quniform('x', -10, 10, .00001),
algo=tpe.suggest,
max_evals=10,
trials=trials)
# why is this red? where is the syntax error?
print best
print hyperopt.space_eval(space, best)
__license__ = "BSD-3"
import numpy as np
from hyperopt import hp
layer1 = { 'depth' : 1,
'n_hid_0': hp.qloguniform('n_hid_0', np.log(2**7), np.log(2**12), q=16),
'W_idist_0': hp.choice('W_idist_0', [0, 1]), # ['uniform', 'normal']
#The scale heuristic
'W_ialgo_0': hp.choice('W_ialgo_0', [{
'W_ialgo_0': 0, # 'old'
'W_imult_0': hp.lognormal('W_imult_0', 0, 1)},
{'W_ialgo_0': 1}]), #'Glorot'
'cd_lr_0': hp.lognormal('cd_lr_0', np.log(.01), 2),
'cd_seed_0': hp.randint('cd_seed_0', 10),
'cd_epochs_0': hp.qloguniform('cd_epochs_0', np.log(1), np.log(3000), q=1),
'sample_v0s_0': hp.choice('sample_v0s_0', [0, 1]), # [False, True]
'lr_anneal_0': hp.qloguniform('lr_anneal_0',
np.log(10), np.log(10000), q=1)}
layer2 = dict(layer1)
layer2.update({'depth' : 2,
'n_hid_1': hp.qloguniform('n_hid_1', np.log(2**7), np.log(2**12), q=16),
'W_idist_1': hp.choice('W_idist_1', [0, 1]), # ['uniform', 'normal']
'W_ialgo_1': hp.choice('W_ialgo_1', [{
'W_ialgo_1': 0, # 'old'
'W_imult_1': hp.lognormal('W_imult_1', 0, 1)},
{'W_ialgo_1': 1 # 'Glorot'
}]),
'cd_lr_1': hp.lognormal('cd_lr_1', np.log(.01), 2),
return log_loss(y_train, bst.predict(dtrain))
evallistcv = [(dcv, "eval"), (dtrain2, "train")]
def objective_cv(param):
num_round = param.pop("num_round")
bst = xgb.train(param, dtrain2, num_round, evallistcv, early_stopping_rounds=20)
return log_loss(y_cv, bst.predict(dcv, ntree_limit=bst.best_iteration))
space = {}
space["objective"] = "multi:softprob"
space["silent"] = 1
space["num_round"] = hp.randint("num_round", 100) + 30
space["eta"] = hp.loguniform("eta", -1.5, 0)
space["gamma"] = hp.loguniform("gamma", -3, 1)
space["max_depth"] = hp.randint("max_depth", 50)
space["min_child_weight"] = hp.loguniform("min_child_weight", -1, 1)
space["max_delta_step"] = hp.loguniform("max_delta_step", -3, 1)
space["subsample"] = hp.loguniform("subsample", -1.5, 0)
space["colsample_by_tree"] = hp.loguniform("colsample_by_tree", -1.5, 0)
space["num_class"] = np.max(y) + 1
if docv:
best = fmin(objective_cv, space, algo=tpe.suggest, max_evals=max_evals)
else:
best = fmin(objective, space, algo=tpe.suggest, max_evals=max_evals)
for key in best.keys():
def test_simple_traits():
s = TestModel.space()
assert set(s.keys()) == {'y', 'z', '__name__', '__module__'}
s = TestModel.space(x = hp.randint('x', 0, 5))
assert set(s.keys()) == {'x', 'y', 'z', '__name__', '__module__'}
reader =config.data_reader_class()
inner = False
if FLAGS.merge == 'inner':
inner = True
data_dict = reader.load_data( config.data_folder, inner )
# Configure the search space
space = hp.choice('algorithm_type', [
{
'config': config,
'seed': FLAGS.seed,
'hof': FLAGS.hof,
'mut_max_step': hp.quniform('mut_max_step', 1, 10, 1),
'cx_prob': hp.uniform('cx_prob', 0, 1),
'mut_prob': hp.uniform('mut_prob', 0, 1),
'mut_x_prob': hp.uniform('mut_x_prob', 0, 1),
'pop_size': 1 + hp.randint('pop_size', 10),
'offspring_size': 1 + hp.randint('offspring_size', 10),
'batch_size': 10 + hp.randint('batch_size', 100)
}
])
best = fmin(objective, space, algo=tpe.suggest, max_evals=3)
print(best)
try:
with open(config.results_folder + config.config_name + '-' + str(FLAGS.seed) + '-sol.txt','w') as f:
f.write(str(best))
f.close()
except IOError:
print('Unable to store the solution')
try:
with open(config.results_folder + config.config_name + '-' + str(FLAGS.seed) + '-log.txt','w') as f:
for test in tests:
def preproc_space(
sup_min_epochs=300,
sup_max_epochs=2000,
max_seconds=60 * 60,
):
"""
Return a hyperopt-compatible pyll expression for a trained neural network.
The trained neural network will have 0, 1, 2, or 3 hidden layers, and may
have an affine first layer that does column normalization or PCA
pre-processing.
Each layer of the network will be pre-trained by some amount of
contrastive divergence before being fine-tuning by SGD.
The training program is built using stub literals `pyll_stubs.train_task`
and `pyll_stubs.valid_task`. When evaluating the pyll program, these
literals must be replaced with skdata Task objects with
`vector_classification` semantics. See `skdata_learning_algo.py` for how
to use the `use_obj_for_literal_in_memo` function to swap live Task
objects in for these stubs.
The search space described by this function corresponds to the DBN model
used in [1] and [2].
"""
train_task_x = scope.getattr(pyll_stubs.train_task, 'x')
nnet0 = scope.NNet([], n_out=scope.getattr(train_task_x, 'shape')[1])
nnet1 = hp.choice('preproc',
[
nnet0, # -- raw data
scope.nnet_add_layers( # -- ZCA of data
nnet0,
scope.zca_layer(
train_task_x,
energy=hp.uniform('pca_energy', .5, 1),
eps=1e-14,
)),
])
param_seed = hp.choice('iseed', [5, 6, 7, 8])
time_limit = scope.time() + max_seconds
nnets = [nnet1]
nnet_i_pt = nnet1
for ii, cd_epochs_max in enumerate([3000, 2000, 1500]):
layer = scope.random_sigmoid_layer(
# -- hack to get different seeds for dif't layers
seed=param_seed + cd_epochs_max,
n_in=scope.getattr(nnet_i_pt, 'n_out'),
n_out=hp.qloguniform('n_hid_%i' % ii,
np.log(2**7),
np.log(2**12),
q=16),
dist=hp.choice('W_idist_%i' % ii, ['uniform', 'normal']),
scale_heuristic=hp.choice(
'W_ialgo_%i' % ii, [
('old', hp.lognormal('W_imult_%i' % ii, 0, 1)),
('Glorot',)]),
squash='logistic',
)
nnet_i_raw = scope.nnet_add_layer(nnet_i_pt, layer)
# -- repeatedly calculating lower-layers wastes some CPU, but keeps
# memory usage much more stable across jobs (good for cluster)
# and the wasted CPU is not so much overall.
nnet_i_pt = scope.nnet_pretrain_top_layer_cd(
nnet_i_raw,
train_task_x,
lr=hp.lognormal('cd_lr_%i' % ii, np.log(.01), 2),
seed=1 + hp.randint('cd_seed_%i' % ii, 10),
n_epochs=hp.qloguniform('cd_epochs_%i' % ii,
np.log(1),
np.log(cd_epochs_max),
q=1),
# -- for whatever reason (?), this was fixed at 100
batchsize=100,
sample_v0s=hp.choice('sample_v0s_%i' % ii, [False, True]),
lr_anneal_start=hp.qloguniform('lr_anneal_%i' % ii,
np.log(10),
np.log(10000),
q=1),
time_limit=time_limit,
)
nnets.append(nnet_i_pt)
# this prior is not what I would do now, but it is what I did then...
nnet_features = hp.pchoice(
'depth',
[(.5, nnets[0]),
(.25, nnets[1]),
(.125, nnets[2]),
(.125, nnets[3])])
sup_nnet = scope.nnet_add_layer(
nnet_features,
scope.zero_softmax_layer(
n_in=scope.getattr(nnet_features, 'n_out'),
n_out=scope.getattr(pyll_stubs.train_task, 'n_classes')))
nnet4, report = scope.nnet_sgd_finetune_classifier(
sup_nnet,
pyll_stubs.train_task,
pyll_stubs.valid_task,
fixed_nnet=nnet1,
max_epochs=sup_max_epochs,
min_epochs=sup_min_epochs,
batch_size=hp.choice('batch_size', [20, 100]),
lr=hp.lognormal('lr', np.log(.01), 3.),
lr_anneal_start=hp.qloguniform(
'lr_anneal_start',
np.log(100),
np.log(10000),
q=1),
l2_penalty=hp.choice('l2_penalty', [
0,
hp.lognormal('l2_penalty_nz', np.log(1.0e-6), 2.)]),
time_limit=time_limit,
)
return nnet4, report
def _random_state(name, random_state):
if random_state is None:
return hp.randint(name, 5)
else:
return random_state
def get_space(Utype=True,
UNBktype=helper_naive_type(),
Ualpha=hp.lognormal('alpha_', 0, 1),
Ufit_prior=hp.choice('bool_', [True, False]),
Ubinarize=hp.choice('binarize_', [.0,
hp.lognormal('threshold_', 0, 1)]),
UC=hp.lognormal('svm_C', 0, 2),
Uwidth=hp.lognormal('svm_rbf_width', 0, 1),
USVMktype=helper_svm(),
Ucriterion=hp.choice('dtree_criterion', ['entropy',
'gini']),
Umax_depth=hp.choice('dtree_max_depth',
[None, 1 + hp.qlognormal('dtree_max_depth_int', 3, 1, 1)]),
Umin_samples_split=1 + hp.qlognormal('dtree_min_samples_split', 2, 1, 1),
Uweights=hp.choice('weighting', ['uniform', 'distance']),
Ualgo=hp.choice('algos', ['auto', 'brute',
'ball_tree', 'kd_tree']),
Uleaf_sz=20+hp.randint('size', 20),
Up=hp.choice('distance', [1, 2]),
Un_neighbors=hp.quniform('num', 3, 19, 1),
Uradius=hp.uniform('rad', 0, 2),
UNktype=helper_neighbors(),
Uout_label=None,
Upreprocess=True,
Unorm=choice(['l1', 'l2']),
Unaxis=1, Uw_mean=choice([True, False]), Uw_std=True,
Usaxis=0, Ufeature_range=(0, 1), Un_components=None,
Uwhiten=hp.choice('whiten_chose', [True, False])):
give_me_bayes = get_bayes(
UNBktype, Ualpha, Ufit_prior, Ubinarize, Upreprocess, Unorm,
Unaxis, Uw_mean, Uw_std, Usaxis, Ufeature_range, Un_components,
Uwhiten)
give_me_svm = get_svm(
UC, USVMktype, Uwidth, Upreprocess, Unorm, Unaxis, Uw_mean,
Uw_std, Usaxis, Ufeature_range, Un_components, Uwhiten)
give_me_dtree = get_dtree(
Ucriterion, Umax_depth, Umin_samples_split, Upreprocess, Unorm,
Unaxis, Uw_mean, Uw_std, Usaxis, Ufeature_range, Un_components,
Uwhiten)
give_me_neighbors = get_neighbors(
UNktype, Uweights, Ualgo, Uleaf_sz, Up, Un_neighbors, Uradius,
Uout_label, Upreprocess, Unorm, Unaxis, Uw_mean, Uw_std,
Usaxis, Ufeature_range, Un_components, Uwhiten)
if Utype == 'naive_bayes':
res_space = give_me_bayes
elif Utype == 'svm':
res_space = give_me_svm
elif Utype == 'dtree':
res_space = give_me_dtree
elif Utype == 'neighbors':
res_space = give_me_neighbors
else:
return hp.choice('quick_fix',
[give_me_bayes,
give_me_svm,
give_me_dtree,
give_me_neighbors])
return hp.choice('quick_fix', [res_space])
["hyperopt-mongo-worker",
"--mongo=localhost:1234/assoc",
"--max-consecutive-failures","4",
"--reserve-timeout", "2.0",
#"--workdir","~/cogsci2014/",
]
print "Worker Call String"
print worker_call_string
workers = []
for i in range(num_mongo_workers):
p = subprocess.Popen(worker_call_string)
workers.append(p)
space = {
'seed':hp.randint('seed', 1000000),
'oja_scale':hp.uniform('oja_scale', 1, 5),
'radius':hp.uniform('radius', .65, .75),
'oja_learning_rate':hp.uniform('oja_learning_rate', 0.01, 0.05),
'ens_intercept':hp.uniform('ens_intercept', 0.08, 0.15),
'cleanup_intercept':hp.uniform('cleanup_intercept', 0.09, 0.16),
'tau_rc':hp.uniform('tau_rc', 0.08, 0.14),
'tau_ref':hp.uniform('tau_ref', 0.003, 0.005),
'ctau_rc':hp.uniform('ctau_rc', 0.25, 0.4),
'ctau_ref':hp.uniform('ctau_ref', 0.002, 0.005),
}
then = time.time()
print "Calling fMin"
best = fmin(objective,
import subprocess
def wrapper(lst):
args = ['april-ann', 'scripts/TRAIN/train_stack_mlp_gbm.lua']
args.extend(map(str,lst))
print(args)
proc = subprocess.Popen(args, stdout=subprocess.PIPE)
proc_out, proc_err = proc.communicate()
# <you might have to do some more elaborate parsing of foo's output here>
line = proc_out.splitlines()[-1]
score = float(line.split()[3])
print(line)
return score
space = [
2**hp.randint('h1', 10),
2**hp.randint('h2', 10),
2**hp.randint('h3', 10),
0.0,
0.0,
0.0,
0.1,
0.2,
0.01,
0.1,
0.02,
]
best_n = fmin(wrapper, space, algo=tpe.suggest, max_evals=100)
print best_n