def init2(self):
Files.mkdir("../model/others")
self.fpath = "../model/others/hopt_xgb.txt"
if config.test:
self.max_evals = 1
else:
self.max_evals = 50
self.space = {
'task': 'regression',
'booster': 'gbtree',
'objective': 'reg:linear',
'eval_metric': "rmse",
'eta': hp.quniform('eta', 0.001, 0.05, 0.001),
'gamma': hp.quniform('gamma', 0.0, 1.0, 0.1),
'min_child_weight': hp.quniform('min_child_weight', 3, 7, 1),
'max_depth': hp.quniform('max_depth', 5, 30, 1),
'subsample': hp.quniform('subsample', 0.7, 1, 0.1),
'colsample_bytree': hp.quniform('colsample_bytree', 0.4, 0.7, 0.1),
"seed": 12345,
'lambda': hp.loguniform("lambda", np.log(0.1), np.log(10)),
'alpha': hp.loguniform("alpha", np.log(0.1), np.log(10)),
'num_rounds': None,
"silent": 1,
}
self.i_folds = [("B", 0)]
self.output_items = []
self.output_items += ["loss", "n", "rsme"]
self.output_items += ["loss{}".format(i) for i, i_fold in enumerate(self.i_folds)]
self.output_items += ["n{}".format(i) for i, i_fold in enumerate(self.i_folds)]
self.output_items += ["rsme{}".format(i) for i, i_fold in enumerate(self.i_folds)]
def init2(self):
Files.mkdir("../model/others")
self.fpath = "../model/others/hopt_keras.txt"
if config.test:
self.max_evals = 1
else:
self.max_evals = 50
self.space = {
"adadelta_eps": hp.loguniform("adadelta_eps", np.log(1e-07), np.log(1e-05)),
"adadelta_lr": hp.loguniform("adadelta_lr", np.log(0.1), np.log(1.0)),
"adadelta_rho_m": hp.loguniform("adadelta_rho_m", np.log(0.01), np.log(0.1)),
"decay": hp.loguniform("decay", np.log(0.0001), np.log(0.1)),
"dropout1": hp.quniform('dropout1', 0.1, 0.5, 0.1),
"dropout2": hp.quniform('dropout2', 0.1, 0.5, 0.1),
"h1": hp.quniform('h1', 50, 500, 10), # 450.0,
"h2": hp.quniform('h2', 20, 250, 5), # 200.0,
"nb_epochs": None, # 22,
}
self.i_folds = [("B", 0)]
self.output_items = []
self.output_items += ["loss", "n", "rsme"]
self.output_items += ["loss{}".format(i) for i, i_fold in enumerate(self.i_folds)]
self.output_items += ["n{}".format(i) for i, i_fold in enumerate(self.i_folds)]
self.output_items += ["rsme{}".format(i) for i, i_fold in enumerate(self.i_folds)]
def set_finetune_space(self, config_file):
''' Given the original deep net architecture, and a set of pretrained weights
and biases, define the configuration space to search for fintuning parameters '''
# we know these fields won't change, so go ahead and set them as
# defaults now
model_params = nt.get_model_params(config_file)
optim_params = nt.get_optim_params(config_file)
default_finetune_model_params = {k: model_params[k] for k in ('num_hids', 'activs', 'd', 'k')}
default_finetune_model_params['loss_terms'] = ['cross_entropy']
default_finetune_optim_params = {k: optim_params[k] for k in ('optim_method', 'optim_type')}
# define the space of hyperparameters we wish to
search_finetune_model_params = {'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]),
'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])}
search_finetune_optim_params = {'learn_rate': hp.uniform('learn_rate', 0, 1),
'rho': hp.uniform('rho', 0, 1),
'num_epochs': hp.qloguniform('num_epochs', log(10), log(5e3), 1),
'batch_size': hp.quniform('batch_size', 128, 1024, 1),
'init_method': hp.choice('init_method', ['gauss', 'fan-io']),
'scale_factor': hp.uniform('scale_factor', 0, 1)}
# combine the default and search parameters into a dictionary to define the
# full space - this is what will be passed into the objective function
finetune_model_params = self.merge_default_search(
default_finetune_model_params, search_finetune_model_params)
finetune_optim_params = self.merge_default_search(
default_finetune_optim_params, search_finetune_optim_params)
finetune_hyperspace = {
'finetune_model_params': finetune_model_params, 'finetune_optim_params': finetune_optim_params}
return finetune_hyperspace
def get_linear_model(model_num):
return {'model_' + model_num: 'LR',
'regularizer_lr_' + model_num: hp.choice('regularizer_lr_' + model_num,[
('l1', hp.loguniform('l1_strength_lr_' + model_num, -5,5)),
('l2', hp.loguniform('l2_strength_lr_' + model_num, -5,5))
]),
'converg_tol_' + model_num: hp.loguniform('converg_tol_' + model_num, -10, -1)
}
def sgd_regression(name,
loss=None, #default - 'hinge'
penalty=None, #default - 'l2'
alpha=None, #default - 0.0001
l1_ratio=None, #default - 0.15, must be within [0, 1]
fit_intercept=None, #default - True
n_iter=None, #default - 5
shuffle=None, #default - False
random_state=None, #default - None
epsilon=None, #default - 0.1
learning_rate=None, #default - 'invscaling'
eta0=None, #default - 0.01
power_t=None, #default - 0.5
warm_start=False,
verbose=0,
):
def _name(msg):
return '%s.%s_%s' % (name, 'sgd', msg)
rval = scope.sklearn_SGDRegressor(
loss=hp.pchoice(
_name('loss'),
[ (0.25, 'squared_loss'),
(0.25, 'huber'),
(0.25, 'epsilon_insensitive'),
(0.25, 'squared_epsilon_insensitive') ] ) if loss is None else loss,
penalty=hp.pchoice(
_name('penalty'),
[ (0.40, 'l2'),
(0.35, 'l1'),
(0.25, 'elasticnet') ] ) if penalty is None else penalty,
alpha=hp.loguniform(
_name('alpha'),
np.log(1e-7),
np.log(1)) if alpha is None else alpha,
l1_ratio=hp.uniform(
_name('l1_ratio'),
0, 1 ) if l1_ratio is None else l1_ratio,
fit_intercept=hp.pchoice(
_name('fit_intercept'),
[ (0.8, True), (0.2, False) ]) if fit_intercept is None else fit_intercept,
epsilon=hp.loguniform(
_name('epsilon'),
np.log(1e-7),
np.log(1)) if epsilon is None else epsilon,
learning_rate='invscaling' if learning_rate is None else learning_rate,
eta0=hp.loguniform(
_name('eta0'),
np.log(1e-5),
np.log(1e-1)) if eta0 is None else eta0,
power_t=hp.uniform(
_name('power_t'),
0, 1) if power_t is None else power_t,
verbose=verbose,
random_state=random_state,
)
return rval
def param_space_reg_skl_svr():
return {
'task': 'reg_skl_svr',
'C': hp.loguniform("C", np.log(0.00000001), np.log(10)),
'gamma': hp.loguniform("gamma", np.log(0.001), np.log(0.1)),
'degree': hp.quniform('degree', 1, 5, 1),
'epsilon': hp.loguniform("epsilon", np.log(0.001), np.log(0.1)),
'kernel': hp.choice('kernel', ['rbf', 'poly', 'sigmoid']),
"max_evals": max_evals,
}
def get_hp_space():
space = (hp.qloguniform('n_hidden', log(100), log(5000), 10),
hp.qloguniform('n_hidden_recurrent', log(100), log(5000), 1),
hp.qloguniform('temporal_order', log(10), log(100), 10),
hp.loguniform('learning_rate_RBM', log(0.0001), log(1)),
hp.loguniform('learning_rate_RNN', log(0.0001), log(1)),
hp.qloguniform('K', log(2), log(20), 1)
# Different learninf rate for RBM and RNN
# hp.choice('activation_func', ['tanh', 'sigmoid']),
# hp.choice('sampling_positive', ['true', 'false'])
# gibbs_sampling_step_test ???
)
return space
def test_expr_to_config():
z = hp.randint('z', 10)
a = hp.choice('a',
[
hp.uniform('b', -1, 1) + z,
{'c': 1, 'd': hp.choice('d',
[3 + hp.loguniform('c', 0, 1),
1 + hp.loguniform('e', 0, 1)])
}])
expr = as_apply((a, z))
hps = {}
expr_to_config(expr, (True,), hps)
for label, dct in hps.items():
print label
print ' dist: %s(%s)' % (
dct['node'].name,
', '.join(map(str, [ii.eval() for ii in dct['node'].inputs()])))
if len(dct['conditions']) > 1:
print ' conditions (OR):'
for condseq in dct['conditions']:
print ' ', ' AND '.join(map(str, condseq))
elif dct['conditions']:
for condseq in dct['conditions']:
print ' conditions :', ' AND '.join(map(str, condseq))
assert hps['a']['node'].name == 'randint'
assert hps['b']['node'].name == 'uniform'
assert hps['c']['node'].name == 'loguniform'
assert hps['d']['node'].name == 'randint'
assert hps['e']['node'].name == 'loguniform'
assert hps['z']['node'].name == 'randint'
assert set([(True, EQ('a', 0))]) == set([(True, EQ('a', 0))])
assert hps['a']['conditions'] == set([(True,)])
assert hps['b']['conditions'] == set([
(True, EQ('a', 0))]), hps['b']['conditions']
assert hps['c']['conditions'] == set([
(True, EQ('a', 1), EQ('d', 0))])
assert hps['d']['conditions'] == set([
(True, EQ('a', 1))])
assert hps['e']['conditions'] == set([
(True, EQ('a', 1), EQ('d', 1))])
assert hps['z']['conditions'] == set([
(True,),
(True, EQ('a', 0))])
def set_multilayer_dropout_space(self):
''' defines a hyperspace for a "modern" neural networks: at least two layers with dropout + reLU '''
# Force at least 2 layers, cuz we're modern
min_layers = 2
max_layers = 3
# sets up the neural network
nnets = [None] * (max_layers - min_layers + 1)
for i, num_layers in enumerate(range(min_layers, max_layers + 1)):
num_hids = [None] * num_layers
for j in range(num_layers):
num_hids[j] = hp.qloguniform(
'num_hid_%i%i' % (i, j), log(100), log(1000), 1)
nnets[i] = num_hids
default_mln_model_params = {
'd': self.d, 'k': self.k, 'loss_terms': ['cross_entropy', 'dropout']}
search_mln_model_params = {
'arch': hp.choice('arch', nnets),
'input_p': hp.uniform('ip', 0, 1),
'hidden_p': hp.uniform('hp', 0, 1),
'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]),
'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])}
default_mln_optim_params = {
'optim_type': 'minibatch', 'optim_method': 'RMSPROP'}
search_mln_optim_params = {
'learn_rate': hp.uniform('learn_rate', 0, 1),
'rho': hp.uniform('rho', 0, 1),
'num_epochs': hp.qloguniform('num_epochs', log(1e2), log(2000), 1),
'batch_size': hp.quniform('batch_size', 128, 1024, 1),
'init_method': hp.choice('init_method', ['gauss', 'fan-io']),
'scale_factor': hp.uniform('scale_factor', 0, 1)}
# merge the default and search spaces
mln_model_params = self.merge_default_search(
default_mln_model_params, search_mln_model_params)
mln_optim_params = self.merge_default_search(
default_mln_optim_params, search_mln_optim_params)
# define the hyperparamater space to search
hyperspace = {'mln_model_params': mln_model_params,
'mln_optim_params': mln_optim_params}
return hyperspace
def get_xgboost_model(model_num):
return {
'model_' + model_num: 'XGBoost',
'eta_' + model_num: hp.loguniform('eta_' + model_num,-5,0),
'gamma_' + model_num: hp.uniform('gamma_' + model_num,0,10),
'max_depth_' + model_num: hp.quniform('max_depth_' + model_num, 1,30,1),
'min_child_weight_' + model_num: hp.uniform('min_child_weight_' + model_num, 0, 10),
'max_delta_step_' + model_num: hp.uniform('max_delta_step_' + model_num, 0, 10),
'num_round_' + model_num: hp.quniform('num_round_' + model_num, 1, 10, 1),
'subsample_' + model_num: 1,
'regularizer_xgb_' + model_num: hp.choice('regularizer_xgb_' + model_num,[
('l1', hp.loguniform('l1_strength_xgb_' + model_num, -5,5)),
('l2', hp.loguniform('l2_strength_xgb_' + model_num, -5,5))
])
}
def _svc_max_iter(name):
return scope.patience_param(
scope.int(
hp.loguniform(
name + '.max_iter',
np.log(1e7),
np.log(1e9))))
def ridge(name,
alpha=None, #default - 1.0
normalize=None, #default - False,
tol=None, #default - 0.001
solver=None, #default - 'auto'
fit_intercept=None, #default - True
):
def _name(msg):
return '%s.%s_%s' % (name, 'sgd', msg)
rval = scope.sklearn_Ridge(
alpha=hp.loguniform(
_name('alpha'),
np.log(1e-3),
np.log(1e3)) if alpha is None else alpha,
normalize=hp.pchoice(
_name('normalize'),
[ (0.8, True), (0.2, False) ]) if normalize is None else normalize,
fit_intercept=hp.pchoice(
_name('fit_intercept'),
[ (0.8, True), (0.2, False) ]) if fit_intercept is None else fit_intercept,
tol=0.001 if tol is None else tol,
solver="auto" if solver is None else solver,
)
return rval
def param_space_clf_skl_lr():
return {
'task': 'clf_skl_lr',
'C': hp.loguniform("C", np.log(0.00000001), np.log(10)),
'random_state': rdmseed,
"max_evals": max_evals,
}
def param_space_reg_skl_lasso():
return {
'task': 'reg_skl_lasso',
'alpha': hp.loguniform("alpha", np.log(0.00000001), np.log(10)),
'random_state': rdmseed,
"max_evals": max_evals,
}
def set_old_space(self):
''' defines an old net from the 80s - simple sigmoid layers, nothing fancy'''
min_layers = 1
max_layers = 3
# sets up the neural network
nnets = [None] * (max_layers - min_layers + 1)
for i, num_layers in enumerate(range(min_layers, max_layers + 1)):
num_hids = [None] * num_layers
for j in range(num_layers):
num_hids[j] = hp.qloguniform(
'num_hid_%i%i' % (i, j), log(10), log(100), 1)
nnets[i] = num_hids
default_mln_model_params = {
'd': self.d, 'k': self.k, 'loss_terms': ['cross_entropy']}
search_mln_model_params = {
'arch': hp.choice('arch', nnets),
'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]),
'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])}
default_mln_optim_params = {
'optim_type': 'minibatch', 'optim_method': 'RMSPROP'}
search_mln_optim_params = {
'learn_rate': hp.uniform('learn_rate', 0, 1),
'rho': hp.uniform('rho', 0, 1),
'num_epochs': hp.qloguniform('num_epochs', log(1e2), log(2000), 1),
'batch_size': hp.quniform('batch_size', 128, 1024, 1),
'init_method': hp.choice('init_method', ['gauss', 'fan-io']),
'scale_factor': hp.uniform('scale_factor', 0, 1)}
# merge the default and search spaces
mln_model_params = self.merge_default_search(
default_mln_model_params, search_mln_model_params)
mln_optim_params = self.merge_default_search(
default_mln_optim_params, search_mln_optim_params)
# define the hyperparamater space to search
hyperspace = {'mln_model_params': mln_model_params,
'mln_optim_params': mln_optim_params}
return hyperspace
def test_loguniform(self):
prefix = "test"
# For whatever reason, this returns a float
param = hp.loguniform('%sp_order_real' % prefix,
mu=np.log(0.1), sigma=np.log(10))
expected = '"testp_order_real": hp.loguniform("testp_order_real", lower=-2.30258509299, upper=2.30258509299),\n'
rval = convert_convnet_searchspace.convert_float(param, 0)
self.assertEqual(rval, expected)
def test_build_space():
from hyperopt import hp
net = scope.Model()
net1 = scope.mlp_layer(net, "mlp1",
size=hp.quniform("mlp1_size", 3, 64, 1),
dropout=hp.choice("mlp1_dropout", [True, False]),
lr_fact=hp.loguniform("mlp_lrfact", np.log(0.01), np.log(10.)),
with_bias=(True, hp.uniform("mlp1_bias", 0., 1.)),
nonlin=hp.choice("mlp1_nonlin", "tanh rectified_linear".split(" ")),
)
net2 = scope.logistic_regression(net1,
n_classes=10,
dropout=hp.choice("linreg_dropout", [False, True]))
return scope.test_classifier(model=net2,
lr=hp.loguniform("lr", np.log(0.0001), np.log(1.00)),
)
def get_linear_model_params(name="linear_common"):
return scope.get_lr_model(
C=hp.loguniform(get_full_name(name, 'C'), -15, 15),
penalty=hp.choice(get_full_name(name, 'penalty'), ('l1', 'l2')),
class_weight=hp.choice(get_full_name(name, 'class_weight'), (defaultdict(lambda: 1.0), 'balanced')),
fit_intercept=hp.choice(get_full_name(name, 'fit_intercept'), (False, True)),
random_state=RANDOM_STATE,
solver='liblinear',
)
def get_hp_space():
space = (hp.qloguniform('n_hidden', log(100), log(5000), 10),
hp.qloguniform('temporal_order', log(1), log(30), 1),
hp.loguniform('learning_rate', log(0.0001), log(1)),
hp.quniform('batch_size', 10, 500, 10)
# hp.choice('activation_func', ['tanh', 'sigmoid']),
# hp.choice('sampling_positive', ['true', 'false'])
# gibbs_sampling_step_test ???
)
return space
def main_hp(args):
hparams = {
"numsteps": args.numsteps,
"measure": ComplexityMeasure(),
"continuous": False,
}
pobjective = partial(objective, hparams=hparams)
print pobjective(params = np.random.uniform(-1.0, 1.0, (2,2)))
def objective_hp(params):
# print "params", params
targ = np.array(params) # .astype(np.float32)
# print targ.dtype
now = time.time()
func = pobjective # args["func"]
ret = func(targ) # cma.fcts.griewank(targ)
took = time.time() - now
print "feval took %f s with ret = %s" % (took, ret["loss"])
return ret
space = [hp.loguniform("m%d" % i, -5, 2.0) for i in range(4)]
trials = Trials()
suggest = tpe.suggest # something
bests = []
initevals = 0
maxevals = 500
lrstate = np.random.RandomState(123)
for i in range(initevals, maxevals):
print "fmin iter %d" % i,
bests.append(fmin(objective_hp,
space,
algo=suggest,
max_evals=i+1,
rstate=lrstate, # 1, 10, 123,
trials=trials,
verbose=1))
lrstate = np.random.RandomState()
best = bests[-1]
for i in range(5):
print("best[%d]" % (-1-i), bests[-1-i])
# for k,v in best:
pkeys = best.keys()
pkeys.sort()
ret = np.zeros((len(pkeys)))
for i,k in enumerate(pkeys):
# print k
ret[i] = best[k]
return ret
def create_space(n_feature):
""" Function to create a search space for the hypers of the a sklearn 0.18 GP kernel.
The order of the hypers is designed to work with obj_function().
Return a search space for hyperopt_fmin()"""
n_sample = X_train.shape[0]
size = [ X[:, i].max() - X[:, i].min() for i in range(n_feature)]
constant_space = [hp.loguniform('constant_value%s'%i, np.log(10)*-2, np.log(10)*2) for i in range(n_feature)]
l_scale_space = [hp.uniform('length_scale%s'%i, (size[i]/n_sample), size[i]) for i in range(n_feature)]
space =[]
#create an alternate search space for hyper-opt
for j in range(n_feature):
space.append(constant_space[j])
space.append(l_scale_space[j])
space.append(hp.loguniform('alpha', -11, 2))
space.append(hp.loguniform('ridge', -2, 2))
return space
def get_model_based_feature_selector_params(
inner_model_params,
name='model_based_feature_selector',
estimator=None,
):
if estimator is None:
estimator = get_feature_selector_estimator_params()
return scope.get_model_based_feature_selection_model(
estimator=estimator,
inner_model=inner_model_params,
feature_selection_threshold_coef=hp.loguniform(get_full_name(name, "threshold"), -5, 5),
)
def test_read_loguniform(self):
# 0 float
# 1 hyperopt_param
# 2 Literal{colnorm_thresh}
# 3 loguniform
# 4 Literal{-20.7232658369}
# 5 Literal{-6.90775527898}
loguniform = hp.loguniform('colnorm_thresh', np.log(1e-9),
np.log(1e-3)).inputs()[0].inputs()[1]
ret = self.pyll_reader.read_loguniform(loguniform, 'colnorm_thresh')
expected = configuration_space.UniformFloatHyperparameter(
'colnorm_thresh', 1e-9, 1e-3, base=np.e)
self.assertEqual(expected, ret)
def _svm_gamma(name, n_features=1):
'''Generator of default gamma values for SVMs.
This setting is based on the following rationales:
1. The gamma hyperparameter is basically an amplifier for the
original dot product or l2 norm.
2. The original dot product or l2 norm shall be normalized by
the number of features first.
'''
# -- making these non-conditional variables
# probably helps the GP algorithm generalize
# assert n_features >= 1
return hp.loguniform(name,
np.log(1. / n_features * 1e-3),
np.log(1. / n_features * 1e3))
def add_pseudo_options():
space['psuedo-documents'] = {
'paragraph':
hp.choice('labels_paragraph', [
{
'use': False
},
{
'use': True,
'weight': hp.loguniform('para_weight', -4, 1)
}
]),
'sentence':
hp.choice('labels_sentence', [
{
'use': False
},
{
'use': True,
'weight': hp.loguniform('sent_weight', -4, 1)
}
]),
}
def many_dists():
a = hp.choice('a', [0, 1, 2])
b = hp.randint('b', 10)
c = hp.uniform('c', 4, 7)
d = hp.loguniform('d', -2, 0)
e = hp.quniform('e', 0, 10, 3)
f = hp.qloguniform('f', 0, 3, 2)
g = hp.normal('g', 4, 7)
h = hp.lognormal('h', -2, 2)
i = hp.qnormal('i', 0, 10, 2)
j = hp.qlognormal('j', 0, 2, 1)
k = hp.pchoice('k', [(.1, 0), (.9, 1)])
z = a + b + c + d + e + f + g + h + i + j + k
return {'loss': scope.float(scope.log(1e-12 + z ** 2)),
'status': base.STATUS_OK}
def _process_vw_argument(self, arg, value, algorithm):
try:
distr_part = self.distr_pattern.findall(value)[0]
except IndexError:
distr_part = ''
range_part = self.range_pattern.findall(value)[0]
is_continuous = '..' in range_part
ocd = self.only_continuous.findall(value)
if not is_continuous and len(ocd)> 0 and ocd[0] != '':
raise ValueError(("Need a range instead of a list of discrete values to define "
"uniform or log-uniform distribution. "
"Please, use [min..max]%s form") % (distr_part))
if is_continuous and arg == '-q':
raise ValueError(("You must directly specify namespaces for quadratic features "
"as a list of values, not as a parametric distribution"))
hp_choice_name = "_".join([algorithm, arg.replace('-', '')])
try_omit_zero = 'O' in distr_part
distr_part = distr_part.replace('O', '')
if is_continuous:
vmin, vmax = [float(i) for i in range_part.split('..')]
if distr_part == 'L':
distrib = hp.loguniform(hp_choice_name, log(vmin), log(vmax))
elif distr_part == '':
distrib = hp.uniform(hp_choice_name, vmin, vmax)
elif distr_part == 'I':
distrib = hp.quniform(hp_choice_name, vmin, vmax, 1)
elif distr_part in {'LI', 'IL'}:
distrib = hp.qloguniform(hp_choice_name, log(vmin), log(vmax), 1)
else:
raise ValueError("Cannot recognize distribution: %s" % (distr_part))
else:
possible_values = range_part.split(',')
if arg == '-q':
possible_values = [v.replace('+', ' -q ') for v in possible_values]
distrib = hp.choice(hp_choice_name, possible_values)
if try_omit_zero:
hp_choice_name_outer = hp_choice_name + '_outer'
distrib = hp.choice(hp_choice_name_outer, ['omit', distrib])
return distrib
def test_switch_multiple_types(self):
prefix = "test"
order = hp.choice('%sp_order' % prefix,
[1, 2, hp.loguniform('%sp_order_real' % prefix,
mu=np.log(1), sigma=np.log(3))])
rval = convert_convnet_searchspace.convert_switch(order, 0)
expected = '"testp_order": hp.choice("testp_order", [\n'\
' {\n' \
' "testp_order": 0,\n' \
' },\n' \
' {\n' \
' "testp_order": 1,\n' \
' },\n' \
' {\n' \
' "testp_order": 2,\n' \
' "testp_order_real": hp.loguniform("testp_order_real", lower=0.0, upper=1.09861228867),\n' \
' },\n' \
']),\n'
self.assertEqual(rval, expected)
def get_frn_params(
inner_model,
feature_importances_getter=None,
name='frn_common'
):
if feature_importances_getter is None:
feature_importances_getter = get_feature_selector_estimator_params()
return scope.get_frn(
inner_model=inner_model,
feature_importances_getter=feature_importances_getter,
lmbda=hp.loguniform(
get_full_name(name, 'lmbda'),
-5, 15,
),
feature_selection_threshold_coef=hp.uniform(
get_full_name(name, 'threshold_coef'),
0.0, 1.0,
),
)
def optimize(trials, X, y, y_ix, reps, max_evals):
space = {
'C': hp.loguniform('C', -3, 2),
'max_iter': hp.quniform('max_iter', 100, 300, 50),
'n_jobs': -1,
'class_weight': 'balanced',
'penalty': 'l1'
}
s = Score(X, y, y_ix, reps)
best = fmin(s.get_score,
space,
algo=tpe.suggest,
trials=trials,
max_evals=max_evals
)
best['max_iter'] = int(best['max_iter'])
best['n_jobs'] = -1
best['class_weight'] = 'balanced'
best['penalty'] = 'l1'
del s
return best
hp.quniform('input_dropout', 0, 0.2, 0.05),
'hidden_layers':
hp.quniform('hidden_layers', 2, 4, 1),
'hidden_units':
hp.quniform('hidden_units', 32, 256, 32),
'hidden_activation':
hp.choice('hidden_activation', ['prelu', 'relu']),
'hidden_dropout':
hp.quniform('hidden_dropout', 0, 0.3, 0.05),
'batch_norm':
hp.choice('batch_norm', ['before_act', 'no']),
'optimizer':
hp.choice('optimizer',
[{
'type': 'adam',
'lr': hp.loguniform('adam_lr', np.log(0.00001), np.log(0.01))
}, {
'type': 'sgd',
'lr': hp.loguniform('sgd_lr', np.log(0.00001), np.log(0.01))
}]),
'batch_size':
hp.quniform('batch_size', 32, 128, 32),
}
class MLP:
def __init__(self, params):
self.params = params
self.scaler = None
self.model = None
def hyper(args):
ds = io.create_dataset(args.input,
output_file=os.path.join(args.outputdir,
'input.zarr'),
transpose=args.transpose,
test_split=args.testsplit,
size_factors=args.normtype)
hyper_params = {
"data": {
"norm_input_log": hp.choice('d_norm_log', (True, False)),
"norm_input_zeromean": hp.choice('d_norm_zeromean', (True, False)),
"norm_input_sf": hp.choice('d_norm_sf', (True, False)),
},
"model": {
"lr":
hp.loguniform("m_lr", np.log(1e-3), np.log(1e-2)),
"ridge":
hp.loguniform("m_ridge", np.log(1e-5), np.log(1e-2)),
"l1_enc_coef":
hp.loguniform("m_l1_enc_coef", np.log(1e-5), np.log(1e-2)),
"hidden_size":
hp.choice("m_hiddensize",
((64, 32, 64), (32, 16, 32), (64, 64), (32, 32),
(16, 16), (16, ), (32, ), (64, ), (128, ))),
"activation":
hp.choice("m_activation",
('relu', 'selu', 'elu', 'PReLU', 'linear', 'LeakyReLU')),
"batchnorm":
hp.choice("m_batchnorm", (True, False)),
"dropout":
hp.uniform("m_do", 0, 0.7),
"input_dropout":
hp.uniform("m_input_do", 0, 0.8),
},
"fit": {
"epochs": 100
}
}
def data_fn(norm_input_log, norm_input_zeromean, norm_input_sf):
if norm_input_sf:
sf_mat = ds.train.size_factors[:]
else:
sf_mat = np.ones((ds.train.matrix.shape[0], 1), dtype=np.float32)
x_train = {
'count':
io.normalize(ds.train.matrix[:],
sf_mat,
logtrans=norm_input_log,
sfnorm=norm_input_sf,
zeromean=norm_input_zeromean),
'size_factors':
sf_mat
}
y_train = ds.train.matrix[:]
return (x_train, y_train),
def model_fn(train_data, lr, hidden_size, activation, batchnorm, dropout,
input_dropout, ridge, l1_enc_coef):
net = AE_types[args.type](train_data[1].shape[1],
hidden_size=hidden_size,
l2_coef=0.0,
l1_coef=0.0,
l2_enc_coef=0.0,
l1_enc_coef=l1_enc_coef,
ridge=ridge,
hidden_dropout=dropout,
input_dropout=input_dropout,
batchnorm=batchnorm,
activation=activation,
init='glorot_uniform',
debug=args.debug)
net.build()
optimizer = opt.__dict__['rmsprop'](lr=lr, clipvalue=5.0)
net.model.compile(loss=net.loss, optimizer=optimizer)
return net.model
output_dir = os.path.join(args.outputdir, 'hyperopt_results')
objective = CompileFN('autoencoder_hyperpar_db',
'myexp1',
data_fn=data_fn,
model_fn=model_fn,
loss_metric='loss',
loss_metric_mode='min',
valid_split=.2,
save_model=None,
save_results=True,
use_tensorboard=True,
save_dir=output_dir)
test_fn(objective, hyper_params, save_model=None)
trials = Trials()
best = fmin(objective,
hyper_params,
trials=trials,
algo=tpe.suggest,
max_evals=1000)
with open(os.path.join(output_dir, 'trials.pickle'), 'wb') as f:
pickle.dump(trials, f)
print(best)
def train_model(self, config, reverse_dictionary, train_data, train_labels,
test_data, test_labels, tool_tr_samples, class_weights):
"""
Train a model and report accuracy
"""
# convert items to integer
l_batch_size = list(map(int, config["batch_size"].split(",")))
l_embedding_size = list(map(int, config["embedding_size"].split(",")))
l_units = list(map(int, config["units"].split(",")))
# convert items to float
l_learning_rate = list(map(float, config["learning_rate"].split(",")))
l_dropout = list(map(float, config["dropout"].split(",")))
l_spatial_dropout = list(
map(float, config["spatial_dropout"].split(",")))
l_recurrent_dropout = list(
map(float, config["recurrent_dropout"].split(",")))
optimize_n_epochs = int(config["optimize_n_epochs"])
# get dimensions
dimensions = len(reverse_dictionary) + 1
best_model_params = dict()
early_stopping = EarlyStopping(monitor='val_loss',
mode='min',
verbose=1,
min_delta=1e-1,
restore_best_weights=True)
# specify the search space for finding the best combination of parameters using Bayesian optimisation
params = {
"embedding_size":
hp.quniform("embedding_size", l_embedding_size[0],
l_embedding_size[1], 1),
"units":
hp.quniform("units", l_units[0], l_units[1], 1),
"batch_size":
hp.quniform("batch_size", l_batch_size[0], l_batch_size[1], 1),
"learning_rate":
hp.loguniform("learning_rate", np.log(l_learning_rate[0]),
np.log(l_learning_rate[1])),
"dropout":
hp.uniform("dropout", l_dropout[0], l_dropout[1]),
"spatial_dropout":
hp.uniform("spatial_dropout", l_spatial_dropout[0],
l_spatial_dropout[1]),
"recurrent_dropout":
hp.uniform("recurrent_dropout", l_recurrent_dropout[0],
l_recurrent_dropout[1])
}
def create_model(params):
model = Sequential()
model.add(
Embedding(dimensions,
int(params["embedding_size"]),
mask_zero=True))
model.add(SpatialDropout1D(params["spatial_dropout"]))
model.add(
GRU(int(params["units"]),
dropout=params["dropout"],
recurrent_dropout=params["recurrent_dropout"],
return_sequences=True,
activation="elu"))
model.add(Dropout(params["dropout"]))
model.add(
GRU(int(params["units"]),
dropout=params["dropout"],
recurrent_dropout=params["recurrent_dropout"],
return_sequences=False,
activation="elu"))
model.add(Dropout(params["dropout"]))
model.add(Dense(2 * dimensions, activation="sigmoid"))
optimizer_rms = RMSprop(lr=params["learning_rate"])
batch_size = int(params["batch_size"])
model.compile(loss=utils.weighted_loss(class_weights),
optimizer=optimizer_rms)
print(model.summary())
model_fit = model.fit_generator(
utils.balanced_sample_generator(train_data, train_labels,
batch_size, tool_tr_samples,
reverse_dictionary),
steps_per_epoch=len(train_data) // batch_size,
epochs=optimize_n_epochs,
callbacks=[early_stopping],
validation_data=(test_data, test_labels),
verbose=2,
shuffle=True)
return {
'loss': model_fit.history["val_loss"][-1],
'status': STATUS_OK,
'model': model
}
# minimize the objective function using the set of parameters above
trials = Trials()
learned_params = fmin(create_model,
params,
trials=trials,
algo=tpe.suggest,
max_evals=int(config["max_evals"]))
best_model = trials.results[np.argmin(
[r['loss'] for r in trials.results])]['model']
# set the best params with respective values
for item in learned_params:
item_val = learned_params[item]
best_model_params[item] = item_val
return best_model_params, best_model
"""
Cart-pole balancing with continuous / Kernelized iFDD
"""
from rlpy.Domains.FiniteTrackCartPole import FiniteCartPoleBalanceOriginal, FiniteCartPoleBalanceModern
from rlpy.Agents import SARSA, Q_LEARNING
from rlpy.Representations import *
from rlpy.Policies import eGreedy
from rlpy.Experiments import Experiment
import numpy as np
from hyperopt import hp
from rlpy.Representations import KernelizediFDD
param_space = {
'kernel_resolution':
hp.loguniform("kernel_resolution", np.log(5), np.log(50)),
'discover_threshold':
hp.loguniform("discover_threshold", np.log(1e-1), np.log(1e3)),
'lambda_':
hp.uniform("lambda_", 0., 1.),
'boyan_N0':
hp.loguniform("boyan_N0", np.log(1e1), np.log(1e5)),
'initial_learn_rate':
hp.loguniform("initial_learn_rate", np.log(5e-2), np.log(1))
}
def make_experiment(exp_id=1,
path="./Results/Temp/{domain}/{agent}/{representation}/",
discover_threshold=.21,
boyan_N0=37.,
lambda_=.9,
test_loss, test_aucroc, xgb_params['eta'], xgb_params['max_depth'],
NB_SAMPLES)
sub.to_csv(submission_path, index=False, float_format='%8.5f')
return {'loss': test_loss, 'status': STATUS_OK}
if USE_HYPEROPT:
log_writer = open('../data/lgb-hyperopt-log.txt', 'w')
space = {
'num_leaves': hp.quniform('num_leaves', 10, 200, 1),
'min_data_in_leaf': hp.quniform('min_data_in_leaf', 10, 1000, 1),
'feature_fraction': hp.uniform('feature_fraction', 0.75, 1.0),
'bagging_fraction': hp.uniform('bagging_fraction', 0.75, 1.0),
'learning_rate': hp.loguniform('learning_rate', -3.0, -1.2),
'min_sum_hessian_in_leaf': hp.loguniform('min_sum_hessian_in_leaf', 0,
2.3),
'max_bin': hp.quniform('max_bin', 64, 512, 1),
'bagging_freq': hp.quniform('bagging_freq', 1, 5, 1),
'lambda_l1': hp.uniform('lambda_l1', 0, 10),
'lambda_l2': hp.uniform('lambda_l2', 0, 10),
}
trials = Trials()
best = hyperopt.fmin(fn=objective,
space=space,
algo=HYPEROPT_ALGO,
max_evals=N_HYPEROPT_PROBES,
trials=trials,
verbose=1)
import hpnnet.nips2011
space = {
'preproc':
hp.choice(
'preproc',
[
{
'preproc': 0
},
{
'preproc':
1, # Column Normalization
'colnorm_thresh':
hp.loguniform('colnorm_thresh', np.log(1e-9), np.log(1e-3)),
},
{
'preproc': 2, # PCA
'pca_energy': hp.uniform('pca_energy', .5, 1),
}
]),
'nhid1':
hp.qloguniform('nhid1', np.log(16), np.log(1024), q=16),
'dist1':
hp.choice('dist1', [0, 1]), # 0 = Uniform, 1 = Normal
'scale_heur1':
hp.choice(
'scale_heur1',
[
{
# yapf: disable
# __plot_scheduler_begin__
# Plot by epoch
ax = None # This plots everything on the same plot
for d in dfs.values():
ax = d.mean_accuracy.plot(ax=ax, legend=False)
# __plot_scheduler_end__
# yapf: enable
# __run_searchalg_begin__
from hyperopt import hp
from ray.tune.suggest.hyperopt import HyperOptSearch
space = {
"lr": hp.loguniform("lr", 1e-10, 0.1),
"momentum": hp.uniform("momentum", 0.1, 0.9),
}
hyperopt_search = HyperOptSearch(space, metric="mean_accuracy", mode="max")
analysis = tune.run(train_mnist, num_samples=10, search_alg=hyperopt_search)
# __run_searchalg_end__
# __run_analysis_begin__
import os
df = analysis.results_df
logdir = analysis.get_best_logdir("mean_accuracy", mode="max")
state_dict = torch.load(os.path.join(logdir, "model.pth"))
def compute(self, label):
return hp.loguniform(label, self.low, self.high, self.q)
from utils.python_utils import quniform_int
steps = [('undersampler', TomekLinks(random_state=random_seed)),
('pca', PCA(n_components=50, random_state=random_seed)),
('xgb',
xgb.XGBClassifier(n_estimators=1000,
silent=True,
nthread=3,
seed=random_seed))]
model = Pipeline(steps=steps)
params_space = {
'pca__n_components':
quniform_int('n_components', 20, 200, 10),
'xgb__max_depth':
quniform_int('max_depth', 10, 30, 1),
'xgb__min_child_weight':
hp.quniform('min_child_weight', 1, 20, 1),
'xgb__subsample':
hp.uniform('subsample', 0.8, 1),
'xgb__n_estimators':
quniform_int('n_estimators', 1000, 10000, 50),
'xgb__learning_rate':
hp.loguniform('learning_rate', np.log(0.0001), np.log(0.5)) - 0.0001,
'xgb__gamma':
hp.loguniform('gamma', np.log(0.0001), np.log(5)) - 0.0001,
'xgb__colsample_bytree':
hp.quniform('colsample_bytree', 0.5, 1, 0.05)
}
if stratified:
for _ in range(num_evals):
best = fmin(hyperopt_stratified_score, search_params, tpe.suggest, len(trials)+1, trials, verbose=verbosity, show_progressbar=verbosity)
pickle.dump(trials, open(trials_file, "wb"))
else:
for _ in range(num_evals):
best = fmin(hyperopt_score, search_params, tpe.suggest, len(trials)+1, trials, verbose=verbosity, show_progressbar=verbosity)
pickle.dump(trials, open(trials_file, "wb"))
return best
_search_params_lgb = {
'class_weight': hp.choice('class_weight', [None, 'balanced']),
'boosting_type': hp.choice('boosting_type',
['gbdt','dart','goss']),
'num_leaves': scope.int(hp.quniform('num_leaves', 20, 200, 1)),
'learning_rate': hp.loguniform('learning_rate', np.log(0.01), np.log(0.2)),
'subsample_for_bin': scope.int(hp.quniform('subsample_for_bin', 20000, 300000, 20000)),
'feature_fraction': hp.uniform('feature_fraction', 0.5, 1),
'bagging_fraction': hp.uniform('bagging_fraction', 0.5, 1),
'min_data_in_leaf': scope.int(hp.qloguniform('min_data_in_leaf', 0, 6, 1)),
'lambda_l1': hp.choice('lambda_l1', [0, hp.loguniform('lambda_l1_positive', -16, 2)]),
'lambda_l2': hp.choice('lambda_l2', [0, hp.loguniform('lambda_l2_positive', -16, 2)]),
'verbose': -1,
'subsample': None,
'reg_alpha': None,
'reg_lambda': None,
'min_sum_hessian_in_leaf': None,
'min_child_samples': None,
'colsample_bytree': None,
#'min_child_samples': hp.quniform('min_child_samples', 20, 500, 5),
'min_child_weight': hp.loguniform('min_child_weight', -16, 5),
def optimize(self):
"""
This is the optimization function that given a space (space here) of
hyperparameters and a scoring function (score here), finds the best hyperparameters.
"""
# XGBoost space
if self.model_type == "xgboost" or self.model_type == "xgboost_classifier":
space = {
'n_estimators': 100 + hp.randint('n_estimators', 1000),
'eta': hp.loguniform('eta', np.log(1e-5), np.log(1.0)),
# A problem with max_depth casted to float instead of int with
# the hp.quniform method.
# TODO: Was too big, trying smaller for now (maybe it caused bad malloc?)
'max_depth': hp.choice('max_depth', np.arange(1, 14,
dtype=int)),
'min_child_weight': hp.quniform('min_child_weight', 1, 6, 1),
'subsample': hp.quniform('subsample', 0.5, 1, 0.05),
'gamma': hp.quniform('gamma', 0.5, 1, 0.05),
'colsample_bytree': hp.quniform('colsample_bytree', 0.5, 1,
0.05),
# TODO: Try GPU later.
'tree_method': 'hist',
'seed': self.seed,
'n_jobs': 8
}
if self.model_type == "xgboost_classifier":
print((self.y == 0).sum() / (self.y == 1).sum())
space["scale_pos_weight"] = (self.y == 0).sum() / (self.y
== 1).sum()
else:
# LigthGBM space
# TODO: Try other boosting types later.
# Smaller n_estimtors.
space = {
'boosting_type':
'goss',
'n_estimators':
100 + hp.randint('n_estimators', 400),
'learning_rate':
hp.loguniform('learning_rate', np.log(1e-8), np.log(1.0)),
'max_depth':
hp.choice('max_depth', np.arange(1, 14, dtype=int)),
'num_leaves':
hp.choice('num_leaves', [15, 31, 63, 127, 200]),
'subsample':
hp.quniform('subsample', 0.5, 1, 0.05),
'colsample_bytree':
hp.quniform('colsample_bytree', 0.5, 1, 0.05),
'min_child_weight':
hp.quniform('min_child_weight', 1, 100, 1),
'reg_alpha':
hp.quniform('reg_alpha', 1, 100, 1),
'min_split_gain':
hp.quniform('min_split_gain', 1, 100, 1),
'min_data_in_leaf':
10 + hp.randint('min_data_in_leaf', 100),
'bagging_seed':
self.seed,
'drop_seed':
self.seed,
'seed':
self.seed
}
# Use the fmin function from Hyperopt to find the best hyperparameters
best = fmin(self.score,
space,
algo=tpe.suggest,
max_evals=self.max_evals,
trials=self.trials)
return best
"""
Intruder Monitoring with incremental tabular representation, Q-Learning
"""
from rlpy.Domains.IntruderMonitoring import IntruderMonitoring
from rlpy.Agents import Q_Learning
from rlpy.Representations import IncrementalTabular
from rlpy.Policies import eGreedy
from rlpy.Experiments import Experiment
import numpy as np
from hyperopt import hp
param_space = {'boyan_N0': hp.loguniform("boyan_N0", np.log(1e1), np.log(1e5)),
'initial_learn_rate': hp.loguniform("initial_learn_rate", np.log(5e-2), np.log(1))}
def make_experiment(
exp_id=1, path="./Results/Temp/{domain}/{agent}/{representation}/",
boyan_N0=120,
initial_learn_rate=.06,
discretization=50):
opt = {}
opt["path"] = path
opt["exp_id"] = exp_id
opt["max_steps"] = 100000
opt["num_policy_checks"] = 10
opt["checks_per_policy"] = 1
domain = IntruderMonitoring()
opt["domain"] = domain
representation = IncrementalTabular(domain)
policy = eGreedy(representation, epsilon=0.1)
CMs.append(confusion_matrix(y[test_idx], y_preds[test_idx]))
CM = np.sum(CMs, axis=0)
FN = CM[1][0]
TP = CM[1][1]
FP = CM[0][1]
print("TP = {}".format(TP))
print("FP = {}".format(FP))
print("FN = {}".format(FN))
f1 = 2. * TP / (2. * TP + FP + FN)
print("F1 : ", f1)
return{'loss': 1-f1, 'status': STATUS_OK}
space = {'C': hp.loguniform('C', -9.0, 3.0),
'cw': hp.uniform('cw', 1, 30)}
trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=100,
trials=trials)
pprint(hyperopt.space_eval(space, best))
best_pars = hyperopt.space_eval(space, best)
# clf = LogisticRegression(C=0.011, class_weight={0: 1, 1: 4.36},
# random_state=1, max_iter=300, n_jobs=1,
model = Net().to(device)
loss = train_one_epoch(model, device, train_dataset, batch_size,
learning_rate, momentum)
return {'loss': loss.item(), 'status': STATUS_OK}
# COMMAND ----------
# MAGIC %md
# MAGIC **Define the search space over hyperparameters**
# COMMAND ----------
search_space = {
'batch_size': hp.uniform('batch_size', 10, 200),
'learning_rate': hp.loguniform('learning_rate', -2.0, 0.),
'momentum': hp.uniform('momentum', 0.1, 0.5),
}
# COMMAND ----------
# MAGIC %md
# MAGIC **Select a search algorithm**
# COMMAND ----------
algo = tpe.suggest # Tree of Parzen Estimators (a "Bayesian" method)
# COMMAND ----------
# MAGIC %md
model.add(Conv1D(filters=16, kernel_size=11, activation='relu'))
model.add(MaxPool1D(strides=4))
model.add(BatchNormalization())
model.add(Conv1D(filters=32, kernel_size=11, activation='relu'))
model.add(MaxPool1D(strides=4))
model.add(BatchNormalization())
model.add(Conv1D(filters=64, kernel_size=11, activation='relu'))
model.add(MaxPool1D(strides=4))
model.add(Flatten())
model.add(Dropout(dropout1))
model.add(Dense(64, activation='relu'))
model.add(Dropout(dropout2))
model.add(Dense(64, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer=Adam(lr, decay=2e-4),
loss='binary_crossentropy',
metrics=['accuracy'])
return model
model = KerasBatchClassifier(build_fn=create_model,
epochs=40,
batch_size=32,
verbose=2)
params_space = {
'lr': hp.loguniform('lr', -10, -4),
'dropout1': hp.quniform('dropout1', 0.25, .75, 0.25),
'dropout2': hp.quniform('dropout2', 0.25, .75, 0.25),
}
'boosting_type':
hp.choice('boosting_type',
[{
'boosting_type': 'gbdt',
'subsample': hp.uniform('gdbt_subsample', 0.5, 1)
}, {
'boosting_type': 'dart',
'subsample': hp.uniform('dart_subsample', 0.5, 1)
}, {
'boosting_type': 'goss',
'subsample': 1.0
}]),
'num_leaves':
hp.quniform('num_leaves', 30, 150, 1),
'learning_rate':
hp.loguniform('learning_rate', np.log(0.01), np.log(0.2)),
'subsample_for_bin':
hp.quniform('subsample_for_bin', 20000, 300000, 20000),
'min_child_samples':
hp.quniform('min_child_samples', 20, 500, 5),
'reg_alpha':
hp.uniform('reg_alpha', 0.0, 1.0),
'reg_lambda':
hp.uniform('reg_lambda', 0.0, 1.0),
'colsample_bytree':
hp.uniform('colsample_by_tree', 0.6, 1.0)
}
x = sample(space)
# Conditional logic to assign top-level keys
subsample = x['boosting_type'].get('subsample', 1.0)
# Return cross-validation loss
# NB: we are minimizing here zo we need to take 1-accuracy
return ({
"loss": history["validation_loss"][which_min],
"parameters": parameters,
"iteration": which_min,
'status': STATUS_OK
})
# Define the search space
space = {
'hidden_size': hp.choice('hidden_units', [32, 64, 128]),
'sent_length': hp.choice("sent_length", [8, 10, 12, 15]),
'use_class_weights': hp.choice("use_class_weights", [True, False]),
'learning_rate': hp.loguniform('learning_rate', np.log(0.001),
np.log(0.03)),
'dropout_prop': hp.uniform("dropout", 0, 0.5)
}
#%% Test space
# Test if works
from hyperopt.pyll.stochastic import sample
parameters = sample(space)
print(parameters)
po = HAN_search(parameters)
#%% Run the optimizer
# Algorithm
log_writer.flush()
if test_loss<cur_best_loss:
cur_best_loss = test_loss
print(colorama.Fore.GREEN + 'NEW BEST LOSS={}'.format(cur_best_loss) + colorama.Fore.RESET)
return{'loss':test_loss, 'status': STATUS_OK }
# --------------------------------------------------------------------------------
space ={
'depth': hp.quniform("depth", 4, 7, 1),
'rsm': hp.uniform ('rsm', 0.75, 1.0),
'learning_rate': hp.loguniform('learning_rate', -3.0, -0.7),
'l2_leaf_reg': hp.uniform('l2_leaf_reg', 1, 10),
}
trials = Trials()
best = hyperopt.fmin(fn=objective,
space=space,
algo=HYPEROPT_ALGO,
max_evals=N_HYPEROPT_PROBES,
trials=trials,
verbose=1)
print('-'*50)
print('The best params:')
print( best )
## gradient boosting regressor 一种跟xgboost类似的梯度自举树
param_space_reg_skl_gbm = {
'task': 'reg_skl_gbm',
'n_estimators': hp.choice('n_estimators', np.arange(skl_min_n_estimators, skl_max_n_estimators, skl_n_estimators_step, dtype=int)),
'learning_rate': hp.quniform("learning_rate", 0.01, 0.5, 0.01),
'max_features': hp.quniform("max_features", 0.05, 1.0, 0.05),
'max_depth': hp.choice('max_depth', np.arange(1, 15, dtype=int)),
'subsample': hp.quniform('subsample', 0.5, 1, 0.1),
'random_state': skl_random_seed,
"max_evals": hyperopt_param["gbm_max_evals"],
}
## support vector regression
param_space_reg_skl_svr = {
'task': 'reg_skl_svr',
'C': hp.loguniform("C", np.log(1), np.log(100)),
'gamma': hp.loguniform("gamma", np.log(0.001), np.log(0.1)),
'degree': hp.choice('degree', np.arange(1, 5, 1, dtype=int)),
'epsilon': hp.loguniform("epsilon", np.log(0.001), np.log(0.1)),
'kernel': hp.choice('kernel', ['rbf', 'poly']),
"max_evals": hyperopt_param["svr_max_evals"],
}
## ridge regression
param_space_reg_skl_ridge = {
'task': 'reg_skl_ridge',
'alpha': hp.loguniform("alpha", np.log(0.01), np.log(20)),
'random_state': skl_random_seed,
"max_evals": hyperopt_param["ridge_max_evals"],
}
'conv1d_weight': 0.5272285958816297,
'FM_FTRL_weight': 0.9188890188824305,
}
param_space_vanila_GRU = {
'denselayer_units': hp.uniform("denselayer_units", 128, 256),
'description_Len': hp.uniform("description_Len", 80, 90),
'name_Len': hp.uniform("name_Len", 8, 12),
'embed_name': hp.uniform("embed_name", 45, 61),
'embed_desc': hp.uniform("embed_desc", 45, 61),
'embed_brand': hp.uniform("embed_brand", 20, 31),
'embed_cat_2': hp.uniform("embed_cat_2", 8, 16),
'embed_cat_3': hp.uniform("embed_cat_3", 30, 41),
'rnn_dim_name': hp.uniform("rnn_dim_name",20, 30),
'rnn_dim_desc': hp.uniform("rnn_dim_desc",20, 30),
'lr': hp.loguniform("lr", np.log(0.001), np.log(0.1)),
'batch_size': hp.uniform("batch_size",512, 2048),
'dense_drop': hp.loguniform("dense_drop", np.log(0.001), np.log(0.01)),
}
# 0.42395 959s
param_space_best_vanila_GRU = {
'denselayer_units': 232,
'description_Len': 85,
'name_Len': 10,
'embed_name': 49,
'embed_desc': 56,
'embed_brand': 29,
'embed_cat_2': 12,
'embed_cat_3': 37,
'rnn_dim_name': 23,
return {
'status': STATUS_OK,
'loss': np.average(min_dev_loss),
'epochs': epochs,
'metrics': {
'accuracys': max_dev_acc,
'accuracy': np.average(max_dev_acc),
'losses': min_dev_loss
}
}
# ================== step3: 调参 =================
space = {
'learning_rate': hp.loguniform('learning_rate', np.log(0.0001),
np.log(0.01)), # 学习率
'batch_size': hp.quniform('batch_size', 32, 512, 8), # mini batch大小
'keep_prob': hp.uniform('keep_prob', 0.3, 1.0), # drop out
'l2reg': hp.loguniform('l2reg', np.log(0.0001), np.log(0.01)), # L2正则化
}
ts = Trials()
from functools import partial
best = fmin(dev_loss,
space,
algo=partial(tpe.suggest, n_startup_jobs=15),
max_evals=50,
trials=ts)
print('TPE best: {}'.format(best))
for trial in ts.trials:
import mlflow
import mlflow.pyfunc
import mlflow.sklearn
from sklearn.model_selection import cross_val_score
from hyperopt import fmin, tpe, hp, Trials, STATUS_OK
from hyperopt.pyll import scope
from IPython.display import Image
import numpy as np
import lightgbm as lgb
from lightgbm import LGBMModel, LGBMClassifier
hyperparameters = {
"max_depth": scope.int(hp.quniform("max_depth", 2, 100, 5)),
"n_estimators": scope.int(hp.quniform("n_estimators", 2, 100, 1)),
"num_leaves": scope.int(hp.quniform("num_leaves", 2, 50, 1)),
"reg_alpha": hp.loguniform('reg_li', -5, 5),
"random_state": 1,
"learning_rate": hp.loguniform("learning_rate", np.log(0.01), np.log(0.5)),
"min_child_weight": hp.uniform('min_child_weight', 0.5, 10),
"boosting": hp.choice("boosting", ["gbdt", "dart", "goss"]),
"objective": "binary"
}
# +
def train_model(parameters):
mlflow.lightgbm.autolog()
with mlflow.start_run(nested=True):
booster = lgb.LGBMClassifier()
booster.set_params(**parameters)
booster.fit(X_train_prepared, y_train)
import hyperopt.hp as hp
import mlflow
import sbi.utils as sbi_utils
import torch
from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, tpe
from snpe.inference.inference_class import TimeSeriesInference
ARTIFACT_PATH = Path("./artifacts/hyperparameter_tuning/cnn_tuning")
SEARCH_SPACE = {
"batch_size": hp.choice("batch_size", [32, 64, 128, 256]),
# log(1e-6)=-13.82, log(1e-2)=-4.6, as a result learning rate is log-uniform distributed between
# 1e-6 and 1e-2 in the setup below
"learning_rate": hp.loguniform("learning_rate", -13.82, -4.6),
"hidden_features": hp.choice("hidden_features", range(20, 80, 5)),
"num_transforms": hp.choice("num_transforms", range(2, 9, 1)),
"num_conv_layers": hp.choice("num_conv_layers", range(1, 7, 1)),
"num_channels": hp.choice("num_channels", range(3, 16, 1)),
"conv_kernel_size": hp.choice("conv_kernel_size", range(3, 20, 2)),
"maxpool_kernel_size": hp.choice("maxpool_kernel_size", range(3, 20, 2)),
"num_dense_layers": hp.choice("num_dense_layers", range(1, 6, 1)),
}
def hyperopt_run(inferrer: TimeSeriesInference, parameters: Dict,
**kwargs) -> Dict:
with mlflow.start_run(nested=True):
# Log parameters
mlflow.log_params(parameters)
import math
import os
import sys
sys.path.append(os.path.dirname(sys.argv[0]) + "/../TensorFlowScripts")
import train_GED
from hyperopt import hp, fmin, rand, tpe, STATUS_OK, Trials
space = {
'lr':
hp.loguniform('lr', math.log(1e-5), math.log(30)),
'wd':
hp.choice(
'wd',
[1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9, 1e-10, 0.0]),
'hsize':
hp.choice('hsize', [2048]),
'nlayers':
hp.choice('nlayers', [3]),
'Lambda':
hp.choice('Lambda', [0.0]),
'gamma':
hp.choice('gamma', [0.999])
}
def objective(params):
train_loss, val_loss = train_GED.main(params)
print "LOSSES: ", train_loss, val_loss
return {
"""
System Administrator with incremental tabular representation, Q-Learning
"""
from rlpy.Domains.SystemAdministrator import SystemAdministrator
from rlpy.Agents import Q_Learning
from rlpy.Representations import IncrementalTabular
from rlpy.Policies import eGreedy
from rlpy.Experiments import Experiment
import numpy as np
from hyperopt import hp
param_space = {
'boyan_N0':
hp.loguniform("boyan_N0", np.log(1e1), np.log(1e5)),
'initial_learn_rate':
hp.loguniform("initial_learn_rate", np.log(5e-2), np.log(1))
}
def make_experiment(exp_id=1,
path="./Results/Temp/{domain}/{agent}/{representation}/",
boyan_N0=120,
initial_learn_rate=.06,
discretization=50):
opt = {}
opt["exp_id"] = exp_id
opt["max_steps"] = 100000
opt["num_policy_checks"] = 10
opt["checks_per_policy"] = 1
domain = SystemAdministrator()
def convert(self, param_settings):
name = param_settings["name"]
domain = param_settings["domain"]
dtype = param_settings["dtype"]
data = param_settings["data"]
assert isinstance(
data, list
), "precondition violation. data of type {} not allowed!".format(
type(data))
assert len(
data
) >= 2, "precondition violation, data must be of length 2, [left_bound, right_bound]"
assert isinstance(
domain, str
), "precondition violation. domain of type {} not allowed!".format(
type(domain))
assert isinstance(
dtype, str
), "precondition violation. dtype of type {} not allowed!".format(
type(dtype))
if domain == "uniform":
if dtype == "float" or dtype == "double":
return hp.uniform(name, data[0], data[1])
elif dtype == "int":
data = list(np.arange(int(data[0]), int(data[1] + 1)))
return hp.choice(name, data)
else:
msg = "cannot convert the type {} in domain {}".format(
dtype, domain)
LOG.error(msg)
raise LookupError(msg)
elif domain == "loguniform":
if dtype == "float" or dtype == "double":
if data[0] == 0:
data[0] += 1e-23
assert data[0] > 0, "precondition Violation, a < 0!"
assert data[0] < data[1], "precondition Violation, a > b!"
assert data[1] > 0, "precondition Violation, b < 0!"
lexp = np.log(data[0])
rexp = np.log(data[1])
assert lexp is not np.nan, "precondition violation, left bound input error, results in nan!"
assert rexp is not np.nan, "precondition violation, right bound input error, results in nan!"
return hp.loguniform(name, lexp, rexp)
else:
msg = "cannot convert the type {} in domain {}".format(
dtype, domain)
LOG.error(msg)
raise LookupError(msg)
elif domain == "normal":
if dtype == "float" or dtype == "double":
mu = (data[1] - data[0]) / 2.0
sigma = mu / 3
return hp.normal(name, data[0] + mu, sigma)
else:
msg = "cannot convert the type {} in domain {}".format(
dtype, domain)
LOG.error(msg)
raise LookupError(msg)
elif domain == "categorical":
if dtype == 'str':
return hp.choice(name, data)
elif dtype == 'bool':
data = []
for elem in data:
if elem == "true" or elem == "True" or elem == 1 or elem == "1":
data.append(True)
elif elem == "false" or elem == "False" or elem == 0 or elem == "0":
data.append(False)
else:
msg = "cannot convert the type {} in domain {}, unknown bool type value".format(
dtype, domain)
LOG.error(msg)
raise LookupError(msg)
return hp.choice(name, data)
else:
msg = "Precondition violation, domain named {} not available!".format(
domain)
LOG.error(msg)
raise IOError(msg)
import keras.backend as K
from hyperopt import hp, tpe, fmin, Trials
import pickle
import traceback
__author__ = "Guillaume Chevalier"
__copyright__ = "Copyright 2017, Guillaume Chevalier"
__license__ = "MIT License"
space = {
# This loguniform scale will multiply the learning rate, so as to make
# it vary exponentially, in a multiplicative fashion rather than in
# a linear fashion, to handle his exponentialy varying nature:
'lr_rate_mult':
hp.loguniform('lr_rate_mult', -0.5, 0.5),
# L2 weight decay:
'l2_weight_reg_mult':
hp.loguniform('l2_weight_reg_mult', -1.3, 1.3),
# Batch size fed for each gradient update
'batch_size':
hp.quniform('batch_size', 100, 450, 5),
# Choice of optimizer:
'optimizer':
hp.choice('optimizer', ['Adam', 'Nadam', 'RMSprop']),
# Coarse labels importance for weights updates:
'coarse_labels_weight':
hp.uniform('coarse_labels_weight', 0.1, 0.7),
# Uniform distribution in finding appropriate dropout values, conv layers
'conv_dropout_drop_proba':
hp.uniform('conv_dropout_proba', 0.0, 0.35),
INPUT_SNRs = range(-10, 31, 5)
k = 0
############################################################
# ############################################################
name = '2d-grid'
n = 20
Y_HIGH = 10
Y_LOW = -5
Gnx, signal_2d, y_true, xs, ys = create2DSignal(k,
n,
Y_HIGH=Y_HIGH,
Y_LOW=Y_LOW)
pspace = (hp.loguniform('gamma_mult', -4, 4), hp.loguniform('rho_mult', -3, 3))
max_evals = 40
d = 10
# ############################################################
############################################################
# name = 'minnesota'
# G = pg.graphs.Minnesota(connect=True)
# Gnx = nx.from_scipy_sparse_matrix(G.A.astype(float), edge_attribute='weight')
# y_true = np.load('datasets/minnesota/beta_0.npy')
# pspace = (
# hp.loguniform('gamma_mult', -3, 5),
# hp.loguniform('rho_mult', -3, 3)
# )
# max_evals = 100
experiment,
exp_prefix=exp_prefix,
seed=seed,
mode=mode,
variant=variant,
exp_id=exp_id,
sync_s3_log=True,
sync_s3_pkl=True,
periodic_sync_interval=600,
)
if run_mode == 'hyperopt':
search_space = {
'algo_params.qf_learning_rate':
hp.loguniform(
'algo_params.qf_learning_rate',
np.log(1e-5),
np.log(1e-2),
),
'algo_params.policy_learning_rate':
hp.loguniform(
'algo_params.policy_learning_rate',
np.log(1e-5),
np.log(1e-2),
),
'algo_params.discount':
hp.uniform(
'algo_params.discount',
0.0,
1.0,
),
'algo_params.target_hard_update_period':
residuals = params["residuals"]
'''
hyperopt_space = {
"batch_size":
hp.choice("batch_size", [16, 32, 64]),
"num_hidden":
hp.qloguniform("num_hidden", log(41), log(100), 1),
"num_fc_layers":
hp.choice("num_fc_layers", [0, 1, 2]),
"num_conv_layers":
hp.quniform("num_conv_layers", 4, 10, 1),
"conv_depth":
hp.quniform("conv_depth", 16, 50, 1),
"conv_depth_mul":
hp.loguniform("conv_depth_mul", log(1), log(2)),
"conv_width":
hp.quniform("conv_width", 1, 10, 1),
"conv_height":
hp.quniform("conv_height", 1, 20, 1),
"pooling_size":
hp.choice("pooling_size", [2, 3]),
"pooling_type":
hp.choice("pooling_type",
["local_max", "local_avg", "global_max", "global_avg"]),
"dropout_coeff":
hp.uniform("dropout_coeff", 0.4, 0.6),
"reg_coeff":
hp.uniform("reg_coeff", -5, -3),
"residuals":
hp.choice("residuals", ["resnet", "densenet", ""]),
