def normal_from_bounds(label, left_bound, right_bound, quantization=None):
mean = (left_bound + right_bound) / 2.0
sigma = (right_bound - left_bound) / 4.0
hp_variable = (hp.normal(label, mean, sigma) if quantization is None
else hp.qnormal(label, mean, sigma, quantization))
dist = stats.norm(mean, sigma)
return Parameter(label, mean, hp_variable, dist.logpdf, dist.cdf)
def blja_test():
space = {
'n_estimators': hp.qnormal('n_estimators', 1000, 200, 10),
'learning_rate': hp.normal('learning_rate', 0.1, 0.05)
}
for x in range(1000):
print pyll.stochastic.sample(space)
def do_test(runs, flder):
df = load_train()
space = {
'k': hp.qnormal('k', 25, 10, 1),
'f': hp.loguniform('f', log(0.1), log(5))
}
trials = MongoTrials(
'mongo://10.20.0.144:27017/bid_exp_family_and_zero_column/jobs',
exp_key='exp1')
log_file = os.path.join(flder, 'log.txt')
objective = partial(bid_and_zero_optimizer.loss_for_batch,
df=df,
runs=runs,
flder=flder,
log_file=log_file)
best = fmin(objective,
space=space,
algo=tpe.suggest,
trials=trials,
max_evals=10000)
print
print 'curr={}'.format(best)
print 'best={}'.format(get_the_best_loss(trials))
def parse_search_space(self, learner_space):
'''
search space is dictionary
{'n_estimators': ('uniform', 1, 1000, 'discrete')}
'''
search_space = dict()
for k, v in learner_space.iteritems():
if v[2] == 'samples':
v = (v[0], v[1], min(100, self.X.shape[0]/len(self.kf)-1), v[3])
if v[3] == 'discrete':
search_space[k] = hp.quniform(k, v[1], v[2], 1)
elif v[0] == 'uniform':
search_space[k] = hp.uniform(k, v[1], v[2])
elif v[0] == 'loguniform':
search_space[k] = hp.loguniform(k, v[1], v[2])
elif v[0] == 'normal':
search_space[k] = hp.normal(k, v[1], v[2])
elif v[0] == 'lognormal':
search_space[k] = hp.lognormal(k, v[1], v[2])
elif v[0] == 'quniform':
search_space[k] = hp.quniform(k, v[1], v[2], v[3])
elif v[0] == 'qloguniform':
search_space[k] = hp.qloguniform(k, v[1], v[2], v[3])
elif v[0] == 'qnormal':
search_space[k] = hp.qnormal(k, v[1], v[2], v[3])
elif v[0] == 'qlognormal':
search_space[k] = hp.qlognormal(k, v[1], v[2], v[3])
return search_space
def build_dist_func_instance(hp_name, func, args, hp_size=None):
'''
args:
hp_name: the name of the hyperparameter associated with this func
func: name of hyperopt dist func
args: list of float values
processing:
instantiate the named dist func with specified args
return:
instance of hyperopt dist func
'''
if func == "choice":
dist = hp.choice(hp_name, args)
elif func == "randint":
max_value = 65535 if len(args) == 0 else args[0]
# specify "size=None" to workaround hyperopt bug
if hp_size:
# let size default to () (error if we try to set it explictly)
dist = hp.randint(hp_name, max_value)
else:
dist = hp.randint(hp_name, max_value, size=None)
elif func == "uniform":
arg_check(func, args, count=2)
dist = hp.uniform(hp_name, *args)
elif func == "normal":
arg_check(func, args, count=2)
dist = hp.normal(hp_name, *args)
elif func == "loguniform":
arg_check(func, args, count=2)
dist = hp.loguniform(hp_name, *args)
elif func == "lognormal":
arg_check(func, args, count=2)
dist = hp.lognormal(hp_name, *args)
elif func == "quniform":
arg_check(func, args, count=3)
dist = hp.quniform(hp_name, *args)
elif func == "qnormal":
arg_check(func, args, count=3)
dist = hp.qnormal(hp_name, *args)
elif func == "qloguniform":
arg_check(func, args, count=3)
dist = hp.qloguniform(hp_name, *args)
elif func == "qlognormal":
arg_check(func, args, count=3)
dist = hp.qlognormal(hp_name, *args)
return dist
def normal_from_bounds(label, left_bound, right_bound, quantization=None):
mean = (left_bound + right_bound) / 2.0
sigma = (right_bound - left_bound) / 4.0
hp_variable = (
hp.normal(label, mean, sigma) if quantization is None else hp.qnormal(label, mean, sigma, quantization)
)
dist = stats.norm(mean, sigma)
return Parameter(label, mean, hp_variable, dist.logpdf, dist.cdf)
def do_test(runs, fldr):
df = load_train()
space = {
'k': hp.qnormal('k', 25, 10, 1),
'f': hp.loguniform('f', log(0.1), log(5))
}
trials = Trials()
best = fmin(lambda s: loss_for_batch(df,s,trials, runs,fldr), space=space, algo=tpe.suggest, trials=trials, max_evals=10000)
print
print 'curr={}'.format(best)
print 'best={}'.format(get_the_best_loss(trials))
def test_read_qnormal(self):
# 0 float
# 1 hyperopt_param
# 2 Literal{qnormal}
# 3 qnormal
# 4 Literal{0.0}
# 5 Literal{1.0}
# 6 Literal{0.5}
qnormal = hp.qnormal("qnormal", 0.0, 1.0, 0.5).inputs()[0].inputs()[1]
ret = self.pyll_reader.read_qnormal(qnormal, "qnormal")
expected = configuration_space.NormalFloatHyperparameter(
"qnormal", 0.0, 1.0, q=0.5)
self.assertEqual(expected, ret)
def do_test(folds):
df = load_train()
space = {
'n_estimators': hp.qnormal('n_estimators', 1000, 200, 10),
'learning_rate': hp.normal('learning_rate', 0.1, 0.05)
}
trials = Trials()
best = fmin(lambda s: simple_cross_val(folds, s, df, trials),
space=space,
algo=tpe.suggest,
trials=trials,
max_evals=100)
print best
print get_the_best_loss(trials)
def many_dists():
a = hp.choice("a", [0, 1, 2])
b = hp.randint("b", 10)
bb = hp.randint("bb", 12, 25)
c = hp.uniform("c", 4, 7)
d = hp.loguniform("d", -2, 0)
e = hp.quniform("e", 0, 10, 3)
f = hp.qloguniform("f", 0, 3, 2)
g = hp.normal("g", 4, 7)
h = hp.lognormal("h", -2, 2)
i = hp.qnormal("i", 0, 10, 2)
j = hp.qlognormal("j", 0, 2, 1)
k = hp.pchoice("k", [(0.1, 0), (0.9, 1)])
z = a + b + bb + c + d + e + f + g + h + i + j + k
return {"loss": scope.float(scope.log(1e-12 + z ** 2)), "status": base.STATUS_OK}
def many_dists():
a = hp.choice('a', [0, 1, 2])
b = hp.randint('b', 10)
c = hp.uniform('c', 4, 7)
d = hp.loguniform('d', -2, 0)
e = hp.quniform('e', 0, 10, 3)
f = hp.qloguniform('f', 0, 3, 2)
g = hp.normal('g', 4, 7)
h = hp.lognormal('h', -2, 2)
i = hp.qnormal('i', 0, 10, 2)
j = hp.qlognormal('j', 0, 2, 1)
k = hp.pchoice('k', [(.1, 0), (.9, 1)])
z = a + b + c + d + e + f + g + h + i + j + k
return {'loss': scope.float(scope.log(1e-12 + z ** 2)),
'status': base.STATUS_OK}
def do_test(folds):
df = load_train()
space = {
'n_estimators': hp.qnormal('n_estimators', 1000, 200, 10),
'learning_rate': hp.normal('learning_rate', 0.1, 0.05),
'gamma': hp.choice('gamma', [0, 0.1, 0.01, 0.2]),
'max_depth': hp.choice('max_depth', [2, 3, 4, 5]),
'min_child_weight': hp.choice('min_child_weight', [1, 2, 3])
}
trials = Trials()
best = fmin(lambda s: man_id_cross_val(folds, s, df, trials),
space=space,
algo=tpe.suggest,
trials=trials,
max_evals=10000)
print best
print get_the_best_loss(trials)
def do_test(runs):
df = load_train()
space = {
'k': hp.qnormal('k', 10, 7, 1),
'f': hp.loguniform('f', log(0.1), log(5)),
'n': hp.choice('n', [2, 3, 4, 5, 6, 7, 10])
}
trials = MongoTrials('mongo://10.20.0.144:27017/mngr_id_hcc_08_08/jobs',
exp_key='exp1')
objective = partial(mngr_id_hcc_optimizer.loss_for_batch, df=df, runs=runs)
best = fmin(objective,
space=space,
algo=tpe.suggest,
trials=trials,
max_evals=10000)
print
print 'curr={}'.format(best)
print 'best={}'.format(get_the_best_loss(trials))
def do_test(runs):
df = load_train()
space = {
'k': hp.qnormal('k', 25, 10, 1),
'f': hp.loguniform('f', log(0.1), log(5))
}
trials = MongoTrials(
'mongo://10.20.0.144:27017/display_address_to_high_card/jobs',
exp_key='exp1')
objective = partial(display_address_to_high_card_optimizer.loss_for_batch,
df=df,
runs=runs)
best = fmin(objective,
space=space,
algo=tpe.suggest,
trials=trials,
max_evals=10000)
print
print 'curr={}'.format(best)
print 'best={}'.format(get_the_best_loss(trials))
numround = int(space['numround'])
#evallist = [(dtest,'eval'), (dtrain,'train')]
bst = xgb.train(param, dtrain, numround)
ypred = bst.predict(dtest)
auc = roc_auc_score(y2, ypred)
print("SCORE: %s" % auc)
return {'loss': 1 - auc, 'status': STATUS_OK}
space = {
'max_depth': hp.qnormal("x_max_depth", 9, 1.2, 1),
'min_child_weight': 0.5,
'eta': 0.1,
'lambda': hp.qnormal('x_lambda', 6, 2, 1),
'alpha': hp.normal('x_alpha', 6, 2, 1),
'subsample': 1,
'colsample_bytree': 1,
#'scale_pos_weight': hp.normal('x_scale_pos_weight',0.5,0.5),
'numround': hp.quniform('x_numround', 400, 600, 1)
}
# In[ ]:
trials = Trials()
best = fmin(fn=objective,
space=space,
def createHyperoptSpace(self):
name = self.root
if 'anyOf' in self.config or 'oneOf' in self.config:
data = []
if 'anyOf' in self.config:
data = self.config['anyOf']
else:
data = self.config['oneOf']
choices = hp.choice(name, [
Hyperparameter(param,
name + "." + str(index)).createHyperoptSpace()
for index, param in enumerate(data)
])
return choices
elif self.config['type'] == 'object':
space = {}
for key in self.config['properties'].keys():
config = self.config['properties'][key]
space[key] = Hyperparameter(config, name + "." +
key).createHyperoptSpace()
return space
elif self.config['type'] == 'number':
mode = self.config.get('mode', 'uniform')
scaling = self.config.get('scaling', 'linear')
if mode == 'uniform':
min = self.config.get('min', 0)
max = self.config.get('max', 1)
rounding = self.config.get('rounding', None)
if scaling == 'linear':
if rounding is not None:
return hp.quniform(name, min, max, rounding)
else:
return hp.uniform(name, min, max)
elif scaling == 'logarithmic':
if rounding is not None:
return hp.qloguniform(name, math.log(min),
math.log(max), rounding)
else:
return hp.loguniform(name, math.log(min),
math.log(max))
if mode == 'normal':
mean = self.config.get('mean', 0)
stddev = self.config.get('stddev', 1)
rounding = self.config.get('rounding', None)
if scaling == 'linear':
if rounding is not None:
return hp.qnormal(name, mean, stddev, rounding)
else:
return hp.normal(name, mean, stddev)
elif scaling == 'logarithmic':
if rounding is not None:
return hp.qlognormal(name, math.log(mean),
math.log(stddev), rounding)
else:
return hp.lognormal(name, math.log(mean),
math.log(stddev))
def createHyperoptSpace(self, lockedValues=None):
name = self.root
if lockedValues is None:
lockedValues = {}
if 'anyOf' in self.config or 'oneOf' in self.config:
data = []
if 'anyOf' in self.config:
data = self.config['anyOf']
else:
data = self.config['oneOf']
subSpaces = [
Hyperparameter(param, self, name + "." +
str(index)).createHyperoptSpace(lockedValues)
for index, param in enumerate(data)
]
for index, space in enumerate(subSpaces):
space["$index"] = index
choices = hp.choice(self.hyperoptVariableName, subSpaces)
return choices
elif 'enum' in self.config:
if self.name in lockedValues:
return lockedValues[self.name]
choices = hp.choice(self.hyperoptVariableName, self.config['enum'])
return choices
elif 'constant' in self.config:
if self.name in lockedValues:
return lockedValues[self.name]
return self.config['constant']
elif self.config['type'] == 'object':
space = {}
for key in self.config['properties'].keys():
config = self.config['properties'][key]
space[key] = Hyperparameter(
config, self,
name + "." + key).createHyperoptSpace(lockedValues)
return space
elif self.config['type'] == 'number':
if self.name in lockedValues:
return lockedValues[self.name]
mode = self.config.get('mode', 'uniform')
scaling = self.config.get('scaling', 'linear')
if mode == 'uniform':
min = self.config.get('min', 0)
max = self.config.get('max', 1)
rounding = self.config.get('rounding', None)
if scaling == 'linear':
if rounding is not None:
return hp.quniform(self.hyperoptVariableName, min, max,
rounding)
else:
return hp.uniform(self.hyperoptVariableName, min, max)
elif scaling == 'logarithmic':
if rounding is not None:
return hp.qloguniform(self.hyperoptVariableName,
math.log(min), math.log(max),
rounding)
else:
return hp.loguniform(self.hyperoptVariableName,
math.log(min), math.log(max))
if mode == 'randint':
max = self.config.get('max', 1)
return hp.randint(self.hyperoptVariableName, max)
if mode == 'normal':
mean = self.config.get('mean', 0)
stddev = self.config.get('stddev', 1)
rounding = self.config.get('rounding', None)
if scaling == 'linear':
if rounding is not None:
return hp.qnormal(self.hyperoptVariableName, mean,
stddev, rounding)
else:
return hp.normal(self.hyperoptVariableName, mean,
stddev)
elif scaling == 'logarithmic':
if rounding is not None:
return hp.qlognormal(self.hyperoptVariableName,
math.log(mean), math.log(stddev),
rounding)
else:
return hp.lognormal(self.hyperoptVariableName,
math.log(mean), math.log(stddev))
space = {
'noise_std':
hp.normal('noise_std', 0.5, 0.3),
'lr':
hp.loguniform('lr', np.log(0.005), np.log(0.2)),
'denoising_cost': [
hp.uniform('dc1', 0.0, 1.0),
hp.uniform('dc2', 0.0, 1.0),
hp.uniform('dc3', 0.0, 1.0),
hp.uniform('dc4', 0.0, 1.0),
hp.uniform('dc5', 0.0, 1.0),
hp.uniform('dc6', 0.0, 1.0)
],
'batch_size':
hp.qnormal('batch_size', 100, 20, 1)
}
# optimization algorithm
tpe_algorithm = tpe.suggest
# Keep track of results
bayes_trials = Trials()
# File to save first results
out_file = 'results/trials.csv'
of_connection = open(out_file, 'w')
writer = csv.writer(of_connection)
# Write the headers to the file
writer.writerow(
def hyperopt(self):
return hp.qnormal(self.label.name, self.mu, self.sigma, self.q)
def qnormal(label, *args, **kwargs):
return hp.qnormal(label, *args, **kwargs)
