def run_svm_vote(
X_train: pd.DataFrame,
X_test: pd.DataFrame,
y_train: pd.DataFrame,
y_test: pd.DataFrame,
config: Dict[str, Any],
) -> Any:
LOGGER.info("Finding best svm..")
search_space = {
"type": "svm",
"C": hp.lognormal("C", 0, 100.0),
"gamma": hp.lognormal("gamma", 0, 1.0),
"kernel": hp.choice("kernel", ["rbf"]),
}
best_params = hyperopt_search(X_train, y_train, search_space, config)
model = make_pipeline(
get_scaler(config),
SVC(**best_params, class_weight="balanced", probability=True),
)
mean_cross_val_score = cross_validate_model(model, X_train, y_train)
LOGGER.info(f"SVM cross validation score: {mean_cross_val_score}")
if config["test"]:
print(classification_report(model.predict(X_test), y_test))
return model
def main():
client = Client()
print 'n. clients: ', len(client)
digits = load_digits()
X = MinMaxScaler().fit_transform(digits.data)
y = digits.target
pre_processing = hp.choice('preproc_algo', [
scope.PCA(
n_components=1 + hp.qlognormal(
'pca_n_comp', np.log(10), np.log(10), 1),
whiten=hp.choice(
'pca_whiten', [False, True])),
scope.GMM(
n_components=1 + hp.qlognormal(
'gmm_n_comp', np.log(100), np.log(10), 1),
covariance_type=hp.choice(
'gmm_covtype', ['spherical', 'tied', 'diag', 'full'])),
])
classifier = hp.choice('classifier', [
scope.DecisionTreeClassifier(
criterion=hp.choice('dtree_criterion', ['gini', 'entropy']),
max_features=hp.uniform('dtree_max_features', 0, 1),
max_depth=hp.quniform('dtree_max_depth', 1, 25, 1)),
scope.SVC(
C=hp.lognormal('svc_rbf_C', 0, 3),
kernel='rbf',
gamma=hp.lognormal('svc_rbf_gamma', 0, 2),
tol=hp.lognormal('svc_rbf_tol', np.log(1e-3), 1)),
])
sklearn_space = {'pre_processing': pre_processing,
'classifier': classifier}
digits_cv_split_filenames = mmap_utils.persist_cv_splits(
X, y, name='digits_10', n_cv_iter=10)
mmap_utils.warm_mmap_on_cv_splits(client, digits_cv_split_filenames)
trials = hyperselect.IPythonTrials(client)
trials.fmin(
partial(compute_evaluation,
cv_split_filename=digits_cv_split_filenames[0],
),
sklearn_space,
algo=hyperopt.tpe.suggest,
max_evals=30,
verbose=1,
)
trials.wait()
print trials.best_trial
def helper_naive_type():
return hp.choice('naive_subtype', [
{'ktype': 'gaussian'},
{'ktype': 'multinomial', 'alpha': hp.lognormal('alpha_mult', 0, 1),
'fit_prior': hp.choice('bool_mult', [False, True])},
{'ktype': 'bernoulli', 'alpha': hp.lognormal('alpha_ber', 0, 1),
'fit_prior': hp.choice('bool_ber', [False, True]),
'binarize': hp.choice('binarize_or_not',
[
.0,
hp.lognormal('threshold', 0, 1)
])}
])
def test_landing_screen():
# define an objective function
def objective(args):
case, val = args
if case == 'case 1':
return val
else:
return val ** 2
# define a search space
from hyperopt import hp
space = hp.choice('a',
[
('case 1', 1 + hp.lognormal('c1', 0, 1)),
('case 2', hp.uniform('c2', -10, 10))
])
# minimize the objective over the space
import hyperopt
best = hyperopt.fmin(objective, space,
algo=hyperopt.tpe.suggest,
max_evals=100)
print best
# -> {'a': 1, 'c2': 0.01420615366247227}
print hyperopt.space_eval(space, best)
def get_cnn_model(model_num, search_space):
space = cnn_space(search_space)
hparams = {'model_' + model_num: 'CNN',
'word_vectors_' + model_num: ('word2vec', True),
'delta_' + model_num: True,
'flex_' + model_num: (True, .15),
'filters_' + model_num: hp.quniform('filters_' + model_num, *space['filters_'], 1),
'kernel_size_' + model_num: hp.quniform('kernel_size_' + model_num, *space['kernel_size_'], 1),
'kernel_increment_' + model_num: hp.quniform('kernel_increment_' + model_num, *space['kernel_increment_'], 1),
'kernel_num_' + model_num: hp.quniform('kernel_num_' + model_num, *space['kernel_num_'], 1),
'dropout_' + model_num: hp.uniform('dropout_' + model_num, *space['dropout_']),
'batch_size_' + model_num: hp.quniform('batch_size_' + model_num, *space['batch_size_'], 1),
'activation_fn_' + model_num: hp.choice('activation_fn_' + model_num, space['activation_fn_'])}
if space['no_reg']:
hparams['regularizer_cnn_' + model_num] = hp.choice('regularizer_cnn_' + model_num, [
(None, 0.0),
('l2', hp.uniform('l2_strength_cnn_' + model_num, *space['l2_'])),
('l2_clip', hp.uniform('l2_clip_norm_' + model_num, *space['l2_clip_']))
])
else:
hparams['regularizer_cnn_' + model_num] = hp.choice('regularizer_cnn_' + model_num, [
('l2', hp.uniform('l2_strength_cnn_' + model_num, *space['l2_'])),
('l2_clip', hp.uniform('l2_clip_norm_' + model_num, *space['l2_clip_']))
])
if space['search_lr']:
hparams['learning_rate_' + model_num] = hp.lognormal('learning_rate_' + model_num, 0, 1) / 3000
else:
hparams['learning_rate_' + model_num] = .0003
def svr_linear(name,
C=None,
epsilon=None,
shrinking=None,
tol=None,
max_iter=None,
verbose=False,
random_state=None,
cache_size=_svc_default_cache_size):
"""
Return a pyll graph with hyperparamters that will construct
a sklearn.svm.SVR model with a linear kernel.
"""
def _name(msg):
return '%s.%s_%s' % (name, 'linear', msg)
rval = scope.sklearn_SVR(
kernel='linear',
C=_svc_C(name + '.linear') if C is None else C,
epsilon=hp.lognormal(
_name("epsilon"),
np.log(1e-3),
np.log(1e3)),
shrinking=hp_bool(
_name('shrinking')) if shrinking is None else shrinking,
tol=_svc_tol(name) if tol is None else tol,
max_iter=_svc_max_iter(name) if max_iter is None else max_iter,
verbose=verbose,
random_state=_random_state(_name('.rstate'), random_state),
cache_size=cache_size,
)
return rval
def passive_aggressive(name,
loss=None,
C=None,
fit_intercept=False,
n_iter=None,
n_jobs=1,
shuffle=True,
random_state=None,
verbose=False):
def _name(msg):
return '%s.%s_%s' % (name, 'sgd', msg)
rval = scope.sklearn_PassiveAggressiveClassifier(
loss=hp.choice(
_name('loss'),
['hinge', 'squared_hinge']) if loss is None else loss,
C=hp.lognormal(
_name('learning_rate'),
np.log(0.01),
np.log(10),
) if C is None else C,
fit_intercept=fit_intercept,
n_iter=scope.int(
hp.qloguniform(
_name('n_iter'),
np.log(1),
np.log(1000),
q=1,
)) if n_iter is None else n_iter,
n_jobs=n_jobs,
random_state=_random_state(_name('rstate'), random_state),
verbose=verbose
)
return rval
def q1_lognormal():
"""
About the simplest problem you could ask for:
optimize a one-variable quadratic function.
"""
return {'loss': scope.max(-(hp.lognormal('x', 0, 2) - 3) ** 2, -100),
'status': base.STATUS_OK }
def test_space_eval():
space = hp.choice('a',
[
('case 1', 1 + hp.lognormal('c1', 0, 1)),
('case 2', hp.uniform('c2', -10, 10))
])
assert space_eval(space, {'a': 0, 'c1': 1.0}) == ('case 1', 2.0)
assert space_eval(space, {'a': 1, 'c2': 3.5}) == ('case 2', 3.5)
def _get_search_space(param_name, param):
if isinstance(param, p.Instrumentation):
space = {}
space["args"] = {
str(idx_param):
_get_search_space(str(idx_param),
param[0][idx_param]) # type: ignore
for idx_param in range(len(param[0].value))
}
space["kwargs"] = {
param_name: _get_search_space(param_name,
param[1][param_name]) # type: ignore
for param_name in param[1].value.keys()
}
return space
elif isinstance(param, (p.Log, p.Scalar)):
if (param.bounds[0][0] is None) or (param.bounds[1][0] is None):
if isinstance(param, p.Scalar) and not param.integer:
return hp.lognormal(label=param_name, mu=0, sigma=1)
raise ValueError(f"Scalar {param_name} not bounded.")
elif isinstance(param, p.Log):
return hp.loguniform(label=param_name,
low=np.log(param.bounds[0][0]),
high=np.log(param.bounds[1][0]))
elif isinstance(param, p.Scalar):
if param.integer:
return hp.randint(label=param_name,
low=int(param.bounds[0][0]),
high=int(param.bounds[1][0]))
else:
return hp.uniform(label=param_name,
low=param.bounds[0][0],
high=param.bounds[1][0])
elif isinstance(param, p.Choice):
list_types = [
type(param.choices[i]) for i in range(len(param.choices))
if not isinstance(param.choices[i], (p.Instrumentation,
p.Constant))
]
if len(list_types) != len(set(list_types)):
raise NotImplementedError
return hp.choice(
param_name,
[
_get_search_space(param_name + "__" + str(i), param.choices[i])
for i in range(len(param.choices))
],
)
elif isinstance(param, p.Constant):
return param.value
# Hyperopt do not support array
raise NotImplementedError
def test_opt():
from hyperopt import hp
space = hp.choice('a', [('case 1', 1 + hp.lognormal('c1', 0, 1)),
('case 2', hp.uniform('c2', -10, 10))])
from hyperopt import fmin, tpe, space_eval
best = fmin(objective, space, algo=tpe.suggest, max_evals=100)
print(best)
print(space_eval(space, best))
print(objective(space_eval(space, best)))
class GradientBoostingModel(TreeBasedModel):
@staticmethod
def build_estimator(args, train_data=None):
return GradientBoostingRegressor(random_state=RANDOM_STATE,
presort=True,
**args)
loss_alpha = hp.choice('loss_alpha',
[('ls', 0.9), ('lad', 0.9),
('huber', hp.uniform('gbr_alpha', 0.85, 0.95)),
('quantile', 0.5)])
hp_space = {
'n_estimators':
scope.int(
hp.qloguniform('n_estimators', np.log(10.5), np.log(1000.5), 1)),
'learning_rate':
hp.lognormal('learning_rate', np.log(0.01), np.log(10.0)),
'criterion':
hp.choice('criterion', ['mse', 'friedman_mse', 'mae']),
'max_depth':
hp.pchoice('max_depth', [(0.2, 2), (0.5, 3), (0.2, 4), (0.1, 5)]),
'min_samples_leaf':
hp.choice(
'min_samples_leaf_enabled',
[
1, # most common choice.
scope.int(
hp.qloguniform('min_samples_leaf', np.log(1.5),
np.log(50.5), 1))
]),
'subsample':
hp.pchoice(
'subsample_enabled',
[
(0.2, 1.0), # default choice.
(0.8, hp.uniform('subsample', 0.5, 1.0)
) # stochastic grad boosting.
]),
'max_features':
hp.pchoice(
'max_features_str',
[
(0.1, 'sqrt'), # most common choice.
(0.2, 'log2'), # less common choice.
(0.1, None), # all features, less common choice.
(0.6, hp.uniform('max_features_str_frac', 0., 1.))
]),
'loss':
loss_alpha[0],
'alpha':
loss_alpha[1]
}
def test_read_lognormal(self):
# 0 float
# 1 hyperopt_param
# 2 Literal{lr}
# 3 lognormal
# 4 Literal{-4.60517018599}
# 5 Literal{3.0}
lognormal = hp.lognormal('lr', np.log(.01), 3.).inputs()[0].inputs()[1]
ret = self.pyll_reader.read_lognormal(lognormal, "lr")
expected = configuration_space.NormalFloatHyperparameter(
"lr", np.log(0.01), 3.0, base=np.e)
self.assertEqual(expected, ret)
def run_svm(
X_train: pd.DataFrame,
X_test: pd.DataFrame,
y_train: pd.DataFrame,
y_test: pd.DataFrame,
config: Dict[str, Any],
) -> Tuple[Any, Any]:
LOGGER.info("Running SVM..")
if config["find_optimal_model"]:
search_space = {
"type": "svm",
"C": hp.lognormal("C", 0, 10.0),
"gamma": hp.lognormal("gamma", 0, 1.0),
"kernel": hp.choice("kernel", ["rbf"]),
}
best_params = hyperopt_search(X_train, y_train, search_space, config)
model = make_pipeline(SVC(**best_params)).fit(X_train, y_train)
mean_cross_val_score = cross_validate_model(model, X_train, y_train)
LOGGER.info(
f"SVM classifier cross validation score: {mean_cross_val_score}")
else:
model = make_pipeline(SVC(**config["models"]["svm"])).fit(
X_train, y_train)
mean_cross_val_score = cross_validate_model(model, X_train, y_train)
LOGGER.info(f"SVM cross validation score: {mean_cross_val_score}")
if config["test"]:
print(classification_report(model.predict(X_test), y_test))
if config["test"]:
y_pred = model.predict(X_test)
score = accuracy_score(y_pred=y_pred, y_true=y_test)
LOGGER.info(f"SVM has a test accuracy of {score}")
return model, y_pred
return model, None
def log_normal_from_bounds(label, left_bound, right_bound, quantization=None):
log_left_bound = np.log(left_bound)
log_right_bound = np.log(right_bound)
log_mean = (log_left_bound + log_right_bound) / 2.0
log_sigma = (log_right_bound - log_left_bound) / 4.0
mean = np.exp(log_mean)
hp_variable = (
hp.lognormal(label, log_mean, log_sigma)
if quantization is None
else hp.qlognormal(label, log_mean, log_sigma, quantization)
)
dist = stats.lognorm(log_sigma, scale=mean)
return Parameter(label, mean, hp_variable, dist.logpdf, dist.cdf)
def svr_poly(name,
C=None,
epsilon=None,
gamma=None,
coef0=None,
degree=None,
shrinking=None,
tol=None,
max_iter=None,
verbose=False,
random_state=None,
cache_size=_svc_default_cache_size):
"""
Return a pyll graph with hyperparamters that will construct
a sklearn.svm.SVR model with an RBF kernel.
"""
def _name(msg):
return '%s.%s_%s' % (name, 'poly', msg)
# -- (K(x, y) + coef0)^d
coef0nz = hp.choice(_name('coef0nz'), [0, 1])
coef0 = hp.uniform(_name('coef0'), 0.0, 1.0)
poly_coef0 = coef0nz * coef0
rval = scope.sklearn_SVR(
kernel='poly',
C=_svc_C(name + '.poly') if C is None else C,
epsilon=hp.lognormal(
_name("epsilon"),
np.log(1e-3),
np.log(1e3)),
gamma=_svc_gamma(name + '.poly') if gamma is None else gamma,
coef0=poly_coef0 if coef0 is None else coef0,
degree=hp.quniform(
_name('degree'),
low=1.5,
high=8.5,
q=1) if degree is None else degree,
shrinking=hp_bool(
_name('shrinking')) if shrinking is None else shrinking,
tol=_svc_tol(name + '.poly') if tol is None else tol,
max_iter=(_svc_max_iter(name + '.poly')
if max_iter is None else max_iter),
verbose=verbose,
random_state=_random_state(_name('.rstate'), random_state),
cache_size=cache_size,
)
return rval
def many_dists():
a = hp.choice('a', [0, 1, 2])
b = hp.randint('b', 10)
c = hp.uniform('c', 4, 7)
d = hp.loguniform('d', -2, 0)
e = hp.quniform('e', 0, 10, 3)
f = hp.qloguniform('f', 0, 3, 2)
g = hp.normal('g', 4, 7)
h = hp.lognormal('h', -2, 2)
i = hp.qnormal('i', 0, 10, 2)
j = hp.qlognormal('j', 0, 2, 1)
k = hp.pchoice('k', [(.1, 0), (.9, 1)])
z = a + b + c + d + e + f + g + h + i + j + k
return {'loss': scope.float(scope.log(1e-12 + z ** 2)),
'status': base.STATUS_OK}
def colkmeans(name,
n_clusters=None,
init=None,
n_init=None,
max_iter=None,
tol=None,
precompute_distances=True,
verbose=0,
random_state=None,
copy_x=True,
n_jobs=1):
rval = scope.sklearn_ColumnKMeans(
n_clusters=scope.int(
hp.qloguniform(
name + '.n_clusters',
low=np.log(1.51),
high=np.log(19.5),
q=1.0)) if n_clusters is None else n_clusters,
init=hp.choice(
name + '.init',
['k-means++', 'random'],
) if init is None else init,
n_init=hp.choice(
name + '.n_init',
[1, 2, 10, 20],
) if n_init is None else n_init,
max_iter=scope.int(
hp.qlognormal(
name + '.max_iter',
np.log(300),
np.log(10),
q=1,
)) if max_iter is None else max_iter,
tol=hp.lognormal(
name + '.tol',
np.log(0.0001),
np.log(10),
) if tol is None else tol,
precompute_distances=precompute_distances,
verbose=verbose,
random_state=random_state,
copy_x=copy_x,
n_jobs=n_jobs,
)
return rval
def svr_sigmoid(name,
C=None,
epsilon=None,
gamma=None,
coef0=None,
shrinking=None,
tol=None,
max_iter=None,
verbose=False,
random_state=None,
cache_size=_svc_default_cache_size):
"""
Return a pyll graph with hyperparamters that will construct
a sklearn.svm.SVR model with an RBF kernel.
"""
def _name(msg):
return '%s.%s_%s' % (name, 'sigmoid', msg)
# -- tanh(K(x, y) + coef0)
coef0nz = hp.choice(_name('coef0nz'), [0, 1])
coef0 = hp.normal(_name('coef0'), 0.0, 1.0)
sigm_coef0 = coef0nz * coef0
rval = scope.sklearn_SVR(
kernel='sigmoid',
C=_svc_C(name + '.sigmoid') if C is None else C,
epsilon=hp.lognormal(
_name("epsilon"),
np.log(1e-3),
np.log(1e3)),
gamma=_svc_gamma(name + '.sigmoid') if gamma is None else gamma,
coef0=sigm_coef0 if coef0 is None else coef0,
shrinking=hp_bool(
_name('shrinking')) if shrinking is None else shrinking,
tol=_svc_tol(name + '.sigmoid') if tol is None else tol,
max_iter=(_svc_max_iter(name + '.sigmoid')
if max_iter is None else max_iter),
verbose=verbose,
random_state=_random_state(_name('rstate'), random_state),
cache_size=cache_size)
return rval
def svr_rbf(name,
C=None,
epsilon=None,
degree=None,
gamma=None,
shrinking=None,
tol=None,
max_iter=None,
verbose=False,
random_state=None,
cache_size=_svc_default_cache_size):
"""
Return a pyll graph with hyperparamters that will construct
a sklearn.svm.SVR model with an RBF kernel.
"""
def _name(msg):
return '%s.%s_%s' % (name, 'rbf', msg)
rval = scope.sklearn_SVR(
kernel='rbf',
C=_svc_C(name + '.rbf') if C is None else C,
epsilon=hp.lognormal(
_name("epsilon"),
np.log(1e-3),
np.log(1e3)),
degree=hp.quniform(
_name('degree'),
low=1.5,
high=5.5,
q=1) if degree is None else degree,
gamma=_svc_gamma(name) if gamma is None else gamma,
shrinking=hp_bool(
_name('shrinking')) if shrinking is None else shrinking,
tol=_svc_tol(name + '.rbf') if tol is None else tol,
max_iter=(_svc_max_iter(name + '.rbf')
if max_iter is None else max_iter),
verbose=verbose,
cache_size=cache_size,
random_state=_random_state(_name('rstate'), random_state),
)
return rval
def train(self, documents, labels):
"""
Train classifier and perform hyper paramater optimization
"""
if len(documents) != len(labels):
raise Exception("No of documents doesn't match the number of labels")
# Obtain corpus data
logger.info("Shuffling dataset")
documents, labels = shuffle(documents, labels)
logger.info("Obtaining feature matrix for data")
self.input_matrix = self.vectorizer.obtain_feature_matrix(documents)
logger.info("Constructing evaluation function for hyper paramater optimization")
eval = self.contruct_cost_function(
self.input_matrix,
labels
)
# Perform hyper paramater optimization
best_parameters = fmin(
fn=eval,
space=hp.lognormal("c", 0, 1),
algo=tpe.suggest,
max_evals=self.max_evals,
)
best_c = best_parameters["c"]
logger.info("Best value obtained for c={}".format(best_c))
# Train SVM
logger.info("Training SVM".format(best_c))
classifier = svm.SVC(best_c, kernel="linear")
classifier.fit(self.input_matrix, labels)
# Dumping trained SVM
pickle.dump(classifier, open(self.clf_path, "wb"))
def rbm(name,
n_components=None,
learning_rate=None,
batch_size=None,
n_iter=None,
verbose=False,
random_state=None):
def _name(msg):
return '%s.%s_%s' % (name, 'rbm', msg)
rval = scope.sklearn_BernoulliRBM(
n_components=scope.int(
hp.qloguniform(
name + '.n_components',
low=np.log(0.51),
high=np.log(999.5),
q=1.0)) if n_components is None else n_components,
learning_rate=hp.lognormal(
name + '.learning_rate',
np.log(0.01),
np.log(10),
) if learning_rate is None else learning_rate,
batch_size=scope.int(
hp.qloguniform(
name + '.batch_size',
np.log(1),
np.log(100),
q=1,
)) if batch_size is None else batch_size,
n_iter=scope.int(
hp.qloguniform(
name + '.n_iter',
np.log(1),
np.log(1000), # -- max sweeps over the *whole* train set
q=1,
)) if n_iter is None else n_iter,
verbose=verbose,
random_state=_random_state(_name('rstate'), random_state),
)
return rval
except ValueError:
# -- already defined these symbols
pass
# -- (1) DEFINE A SEARCH SPACE
classifier = hp.choice('classifier', [
scope.DecisionTreeClassifier(
criterion=hp.choice(
'dtree_criterion', ['gini', 'entropy']),
max_features=hp.uniform(
'dtree_max_features', 0, 1),
max_depth=hp.quniform(
'dtree_max_depth', 0, 25, 1)),
scope.SVC(
C=hp.lognormal(
'svc_rbf_C', 0, 3),
kernel='rbf',
gamma=hp.lognormal(
'svc_rbf_gamma', 0, 2),
tol=hp.lognormal(
'svc_rbf_tol', np.log(1e-3), 1)),
])
pre_processing = hp.choice('preproc_algo', [
scope.PCA(
n_components=1 + hp.qlognormal(
'pca_n_comp', np.log(10), np.log(10), 1),
whiten=hp.choice(
'pca_whiten', [False, True])),
scope.GMM(
n_components=1 + hp.qlognormal(
# neuron params
'amp': 0.063,
# 'tau_ref': 0.001,
# 'tau_rc': 0.05,
# 'alpha': 0.825,
# 'tau_ref': hp.uniform('tau_ref', 0.001, 0.005),
'tau_ref': 0.002,
'tau_rc': hp.uniform('tau_rc', 0.01, 0.06),
# 'alpha': hp.uniform('alpha', 0.1, 10.0),
'alpha': 1.0,
'sigma': 0.02,
'noise': 10.,
# learning rate params
'epsW_schedule': hp.choice('epsW_schedule', ['linear', 'exp']),
'epsW_base': hp.lognormal('epsW_base', np.log(1e-3), np.log(1e1)),
'epsW_tgtFactor': hp.qloguniform('epsW_tgtFactor', np.log(1), np.log(1e4), 1),
'epsB_schedule': hp.choice('epsB_schedule', ['linear', 'exp']),
'epsB_base': hp.lognormal('epsB_base', np.log(1e-3), np.log(1e1)),
'epsB_tgtFactor': hp.qloguniform('epsB_tgtFactor', np.log(1), np.log(1e4), 1),
'momW': hp.uniform('momW', 0.001, 0.999),
'momB': hp.uniform('momB', 0.001, 0.999),
# initial weight params
'initW1': hp.lognormal('initW1', np.log(1e-4), np.log(1e2)),
'initW2': hp.lognormal('initW2', np.log(1e-2), np.log(1e1)),
'initW3': hp.lognormal('initW3', np.log(1e-2), np.log(1e1)),
'initW4': hp.lognormal('initW4', np.log(1e-2), np.log(1e1)),
'initW5': hp.lognormal('initW5', np.log(1e-2), np.log(1e1)),
# weight costs
Distribution looks like round(normal(mu, sigma)/ q)* q
"""
return hp.normal(name, mu, sigma, q)
def lognormal(name, (mu, sigma)):
"""
Function to create hyperopt lognormal variable
Input
------------------
name - Variable name
(mu, sigma, q) - Tuple of mean, standard deviation and q value.
Distribution looks like exp(normal(mu, sigma))
"""
return hp.lognormal(name, mu, sigma)
def qlognormal(name, (mu, sigma, q)):
"""
Function to create hyperopt qlognormal variable
Input
------------------
name - Variable name
(mu, sigma, q) - Tuple of mean, standard deviation and q value.
Distribution looks like round(exp(normal(mu, sigma))/ q)* q
"""
return hp.qlognormal(name, mu, sigma, q)
errors[arg_key(args)] = error
args_s = "{%s}" % ', '.join("%r: %s" % (k, args[k]) for k in sorted(space))
print("%s: %s" % (error, args_s))
return error
r = np.log(10)
# space = {
# 'laplacian_ksize': pyll.scope.int(1 + hp.quniform('laplacian_ksize', 0, 20, 2)),
# 'data_weight': hp.lognormal('dw', -2 * r, 1 * r),
# 'data_max': hp.lognormal('dm', 2 * r, 1 * r),
# 'disc_max': hp.lognormal('cm', 1 * r, 1 * r),
# }
space = collections.OrderedDict([
('laplacian_ksize', pyll.scope.int(1 + hp.quniform('laplacian_ksize', 0, 20, 2))),
('laplacian_scale', hp.lognormal('laplacian_scale', 0, r)),
('data_weight', hp.lognormal('data_weight', -2*r, r)),
('data_max', hp.lognormal('data_max', 2*r, r)),
('data_exp', hp.lognormal('data_exp', 0, r)),
('disc_max', hp.lognormal('disc_max', r, r)),
('smooth', hp.lognormal('smooth', 0, r)),
# ('post_smooth', hp.lognormal('post_smooth', 0, r)),
])
expr = pyll.as_apply(space)
arg_key = lambda args: tuple(args[key] for key in space)
def get_args(params):
memo = {node: params[node.arg['label'].obj]
for node in pyll.dfs(expr) if node.name == 'hyperopt_param'}
return pyll.rec_eval(expr, memo=memo)
def helper_svm():
return hp.choice('svm_kernel', [
{'ktype': 'linear'},
{'ktype': 'rbf', 'width': hp.lognormal('svm_rbf_width', 0, 1)}
])
from hyperopt import hp
import hpnnet.nips2011
space = {'preproc': hp.choice('preproc', [{
'preproc' : 0}, {
'preproc' : 1, # Column Normalization
'colnorm_thresh' : hp.loguniform('colnorm_thresh', np.log(1e-9), np.log(1e-3)),
}, {
'preproc' : 2, # PCA
'pca_energy' : hp.uniform('pca_energy', .5, 1),
}]),
'nhid1': hp.qloguniform('nhid1', np.log(16), np.log(1024), q=16),
'dist1': hp.choice('dist1', [0, 1]), # 0 = Uniform, 1 = Normal
'scale_heur1': hp.choice('scale_heur1', [{
'scale_heur1' : 0, # Old
'scale_mult1': hp.uniform('scale_mult1', .2, 2)}, {
'scale_heur1': 1}]), # Glorot
'squash' : hp.choice('squash', [0, 1]), # ['tanh', 'logistic']
'iseed': hp.choice('iseed', [0, 1, 2, 3]), # Are 5, 6, 7, 8
'batch_size': hp.choice('batch_size', [0, 1]), # 20 or 100
'lr': hp.lognormal('lr', np.log(.01), 3.),
'lr_anneal_start': hp.qloguniform('lr_anneal_start', np.log(100), np.log(10000), q=1),
'l2_penalty': hp.choice('l2_penalty', [{
'l2_penalty' : 0}, { # Zero
'l2_penalty' : 1, # notzero
'l2_penalty_nz': hp.lognormal('l2_penalty_nz', np.log(1.0e-6), 2.)}])
}
def get_space(Utype=True,
UNBktype=helper_naive_type(),
Ualpha=hp.lognormal('alpha_', 0, 1),
Ufit_prior=hp.choice('bool_', [True, False]),
Ubinarize=hp.choice('binarize_', [.0,
hp.lognormal('threshold_', 0, 1)]),
UC=hp.lognormal('svm_C', 0, 2),
Uwidth=hp.lognormal('svm_rbf_width', 0, 1),
USVMktype=helper_svm(),
Ucriterion=hp.choice('dtree_criterion', ['entropy',
'gini']),
Umax_depth=hp.choice('dtree_max_depth',
[None, 1 + hp.qlognormal('dtree_max_depth_int', 3, 1, 1)]),
Umin_samples_split=1 + hp.qlognormal('dtree_min_samples_split', 2, 1, 1),
Uweights=hp.choice('weighting', ['uniform', 'distance']),
Ualgo=hp.choice('algos', ['auto', 'brute',
'ball_tree', 'kd_tree']),
Uleaf_sz=20+hp.randint('size', 20),
Up=hp.choice('distance', [1, 2]),
Un_neighbors=hp.quniform('num', 3, 19, 1),
Uradius=hp.uniform('rad', 0, 2),
UNktype=helper_neighbors(),
Uout_label=None,
Upreprocess=True,
Unorm=choice(['l1', 'l2']),
Unaxis=1, Uw_mean=choice([True, False]), Uw_std=True,
Usaxis=0, Ufeature_range=(0, 1), Un_components=None,
Uwhiten=hp.choice('whiten_chose', [True, False])):
give_me_bayes = get_bayes(
UNBktype, Ualpha, Ufit_prior, Ubinarize, Upreprocess, Unorm,
Unaxis, Uw_mean, Uw_std, Usaxis, Ufeature_range, Un_components,
Uwhiten)
give_me_svm = get_svm(
UC, USVMktype, Uwidth, Upreprocess, Unorm, Unaxis, Uw_mean,
Uw_std, Usaxis, Ufeature_range, Un_components, Uwhiten)
give_me_dtree = get_dtree(
Ucriterion, Umax_depth, Umin_samples_split, Upreprocess, Unorm,
Unaxis, Uw_mean, Uw_std, Usaxis, Ufeature_range, Un_components,
Uwhiten)
give_me_neighbors = get_neighbors(
UNktype, Uweights, Ualgo, Uleaf_sz, Up, Un_neighbors, Uradius,
Uout_label, Upreprocess, Unorm, Unaxis, Uw_mean, Uw_std,
Usaxis, Ufeature_range, Un_components, Uwhiten)
if Utype == 'naive_bayes':
res_space = give_me_bayes
elif Utype == 'svm':
res_space = give_me_svm
elif Utype == 'dtree':
res_space = give_me_dtree
elif Utype == 'neighbors':
res_space = give_me_neighbors
else:
return hp.choice('quick_fix',
[give_me_bayes,
give_me_svm,
give_me_dtree,
give_me_neighbors])
return hp.choice('quick_fix', [res_space])
import hyperopt
from hyperopt import hp, fmin, tpe, Trials
# define an objective function
def objective(args):
case, val = args
if case == 'case 1':
return val
else:
return val ** 2
if __name__ == '__main__':
trials = Trials()
# define a search space
space = hp.choice('a',
[
('case 1', 1 + hp.lognormal('c1', 0, 1)),
('case 2', hp.uniform('c2', -10, 10))
])
# minimize the objective over the space
best = fmin(objective, space, algo=tpe.suggest, max_evals=250, trials=trials)
print best
# -> {'a': 1, 'c2': 0.01420615366247227}
print hyperopt.space_eval(space, best)
# -> ('case 2', 0.01420615366247227}
def _svc_gamma(name):
# -- making these non-conditional variables
# probably helps the GP algorithm generalize
gammanz = hp.choice(name + '.gammanz', [0, 1])
gamma = hp.lognormal(name + '.gamma', np.log(0.01), 2.5)
return gammanz * gamma
def test_space_eval():
space = hp.choice("a", [("case 1", 1 + hp.lognormal("c1", 0, 1)), ("case 2", hp.uniform("c2", -10, 10))])
assert space_eval(space, {"a": 0, "c1": 1.0}) == ("case 1", 2.0)
assert space_eval(space, {"a": 1, "c2": 3.5}) == ("case 2", 3.5)
def _svc_tol(name):
return scope.inv_patience_param(
hp.lognormal(
name + '.tol',
np.log(1e-3),
2.0))
def _svc_C(name):
return hp.lognormal(name + '.C', np.log(1000.0), 3.0)
def _svc_C(name):
return hp.lognormal(name + '.C', np.log(1e-5), np.log(1e5))
