This is just serving as an example, for all intents and
purposes think of the internals of this function, i.e.: the process
which generates its output values, as unknown.
"""
for i in range(9):
x = x + i + y
print(x, i)
return x
# Bounded region of parameter space
pbounds = {'x': (2, 4), 'y': (-3, 3)}
optimizer = BayesianOptimization(
f=black_box_function,
pbounds=pbounds,
random_state=1,
)
# optimizer.probe(
# params={"x": 0.5, "y": 0.7},
# lazy=True,
# )
# optimizer.maximize(
# init_points=2,
# n_iter=3,
# )
optimizer.maximize(init_points=0, n_iter=0)
def optimize():
# Lasso optimization
# lasso_BO = BayesianOptimization(lasso_func, {'alpha': (0.000001, 1000)})
# lasso_BO.explore({'alpha': [.00001, 0.001, 0.01, 0.1, 1, 10, 100]})
# lasso_BO.maximize(n_iter=100)
# print(lasso_BO.res['max'])
# KRR optimization
# krr_BO = BayesianOptimization(krr_func, {'alpha': (0,10000), 'degree': (1,5), 'coef0': (0, 10000)})
# krr_BO.explore({'alpha': [0.001, 0.1, 1], 'degree':[2, 3, 4], 'coef0':[0, .5, 10]})
# krr_BO.maximize(n_iter=100)
# print(krr_BO.res['max'])
# Elastic optimization
# elastic_BO = BayesianOptimization(elastic_func, {'alpha': (0,10000), 'l1_ratio': (0,1)})
# elastic_BO.explore({'alpha': [0.001, 0.1, 1, 10, 100, 1000, 5000], 'l1_ratio': [0, .1, .2, .3, .5, .7, .9]})
# elastic_BO.maximize(n_iter=100)
# print(elastic_BO.res['max'])
# Random forest optimization
# rf_BO = BayesianOptimization(rf_func, {'n_estimators': (1,1000), 'max_depth': (1,500)})
# rf_BO.explore({'n_estimators': [25, 50, 100, 200, 400], 'max_depth':[10, 40, 80, 320, 500]})
# rf_BO.maximize(n_iter=100)
# print(rf_BO.res['max'])
# svr_BO = BayesianOptimization(svr_func, {'C': (0, 10), 'epsilon':(0,10)})
# svr_BO.explore({'C': [.001, 0.01, 0.1, 1, 10, 100, 1000], 'epsilon':[.001, .01, .1, 1, 10, 100, 1000]})
# svr_BO.maximize(n_iter=100)
# print(svr_BO.res['max'])
# XGBoost optimization
# xgb_BO = BayesianOptimization(xgb_func, {'min_child_weight': (1, 20),
# 'colsample_bytree': (0.1, 1),
# 'max_depth': (5, 15),
# 'subsample': (0.5, 1),
# 'gamma': (0, 10),
# 'alpha': (0, 10),
# 'num_rounds': (0, 10000),
# })
# xgb_BO.maximize(init_points=5, n_iter=100)
# print(xgb_BO.res['max'])
# Lgb optimization
lgb_BO = BayesianOptimization(
lgb_func, {
'num_leaves': (1, 20),
'lr': (0.01, .05),
'num_estimators': (5, 15),
'max_bin': (2, 100),
'bagging_fraction': (0, 1),
'bagging_freq': (1, 10),
'feature_fraction': (0, 1),
'min_data_in_leaf': (1, 10),
'min_sum_hessian_in_leaf': (1, 20),
})
lgb_BO.explore({
'num_leaves': [5, 5, 5, 4, 5, 6, 7],
'lr': [.01, .01, .01, .01, .01, .01, .01],
'num_estimators': [200, 300, 500, 700, 800, 900, 1000],
'max_bin': [10, 40, 80, 100, 55, 30, 53],
'bagging_fraction': [.7, .8, .6, .2, .9, .7, .3],
'bagging_freq': [4, 5, 2, 7, 2, 4, 5],
'feature_fraction': [.2, .3, .25, .3, .21, .15, .13],
'min_data_in_leaf': [5, 6, 5, 4, 4, 7, 7],
'min_sum_hessian_in_leaf': [8, 2, 5, 10, 11, 15, 17]
})
lgb_BO.maximize(init_points=5, n_iter=100)
print(lgb_BO.res['max'])
def bayes_parameter_opt_lgb(X,
y,
init_round=15,
opt_round=25,
n_folds=5,
random_seed=6,
n_estimators=10000,
learning_rate=0.1,
output_process=False):
# prepare data
# categorical_feature如果不需要定义可以删掉
train_data = lgb.Dataset(data=X,
label=y,
categorical_feature=categorical_feats,
free_raw_data=False)
# parameters
def lgb_eval(num_leaves, feature_fraction, bagging_fraction, max_depth,
lambda_l1, lambda_l2, min_split_gain, min_child_weight):
params = {
'application': 'binary',
'num_iterations': n_estimators,
'learning_rate': learning_rate,
'early_stopping_round': 150,
'metric': 'auc'
}
params["num_leaves"] = int(round(num_leaves))
params['feature_fraction'] = max(min(feature_fraction, 1), 0)
params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)
params['max_depth'] = int(round(max_depth))
params['lambda_l1'] = max(lambda_l1, 0)
params['lambda_l2'] = max(lambda_l2, 0)
params['min_split_gain'] = min_split_gain
params['min_child_weight'] = min_child_weight
cv_result = lgb.cv(params,
train_data,
nfold=n_folds,
seed=random_seed,
stratified=True,
verbose_eval=200,
metrics=['auc'])
return max(cv_result['auc-mean'])
# range
lgbBO = BayesianOptimization(lgb_eval, {
'num_leaves': (24, 45),
'feature_fraction': (0.1, 0.9),
'bagging_fraction': (0.8, 1),
'max_depth': (5, 8.99),
'lambda_l1': (0, 5),
'lambda_l2': (0, 3),
'min_split_gain': (0.001, 0.1),
'min_child_weight': (5, 50)
},
random_state=0)
# optimize
optimizer.maximize(init_points=init_round, n_iter=opt_round)
# output optimization process
if output_process == True: lgbBO.points_to_csv("bayes_opt_result.csv")
# return best parameters
print("Final result:", optimizer.max)
## CV process
def rfcv(n_estimators, min_samples_split, max_features, min_samples_leaf):
return cv_s(ExtraTreesClassifier(n_estimators=int(n_estimators),
min_samples_split=int(min_samples_split),
max_features=min(max_features, 0.999),
min_samples_leaf=int(min_samples_leaf),
criterion="gini"),
cx,
cy,
"accuracy",
cv=4).mean()
ERTBO = BayesianOptimization(
rfcv, {
'n_estimators': (100, 800),
'min_samples_split': (1, 15),
'max_features': (0.05, 1),
'min_samples_leaf': (1, 10)
})
#0.81220
#{'n_estimators': (748),
#'min_samples_split': (4),
#'max_features': (0.6237),
#'min_samples_leaf':(1)})
ERTBO.maximize(init_points=35, n_iter=365)
etime = float(time.time() - stime)
def tune(runner, kernel_options, device_options, tuning_options):
""" Find the best performing kernel configuration in the parameter space
:params runner: A runner from kernel_tuner.runners
:type runner: kernel_tuner.runner
:param kernel_options: A dictionary with all options for the kernel.
:type kernel_options: kernel_tuner.interface.Options
:param device_options: A dictionary with all options for the device
on which the kernel should be tuned.
:type device_options: kernel_tuner.interface.Options
:param tuning_options: A dictionary with all options regarding the tuning
process.
:type tuning_options: kernel_tuner.interface.Options
:returns: A list of dictionaries for executed kernel configurations and their
execution times. And a dictionary that contains a information
about the hardware/software environment on which the tuning took place.
:rtype: list(dict()), dict()
"""
#Bayesian Optimization strategy seems to need some hyper parameter tuning to
#become better than random sampling for auto-tuning GPU kernels.
#alpha, normalize_y, and n_restarts_optimizer are options to
#https://scikit-learn.org/stable/modules/generated/sklearn.gaussian_process.GaussianProcessRegressor.html
#defaults used by Baysian Optimization are:
# alpha=1e-6, #1e-3 recommended for very noisy or discrete search spaces
# n_restarts_optimizer=5,
# normalize_y=True,
#several exploration friendly settings are: (default is acq="ucb", kappa=2.576)
# acq="poi", xi=1e-1
# acq="ei", xi=1e-1
# acq="ucb", kappa=10
if not bayes_opt_present:
raise ImportError(
"Error: optional dependency Bayesian Optimization not installed")
#defaults as used by Bayesian Optimization Python package
acq = tuning_options.strategy_options.get("method", "poi")
kappa = tuning_options.strategy_options.get("kappa", 2.576)
xi = tuning_options.strategy_options.get("xi", 0.0)
init_points = tuning_options.strategy_options.get("popsize", 5)
n_iter = tuning_options.strategy_options.get("maxiter", 25)
alpha = tuning_options.strategy_options.get("alpha", 1e-6)
tuning_options["scaling"] = True
results = []
#function to pass to the optimizer
def func(**kwargs):
args = [kwargs[key] for key in tuning_options.tune_params.keys()]
return -1.0 * minimize._cost_func(args, kernel_options, tuning_options,
runner, results)
bounds, _, _ = minimize.get_bounds_x0_eps(tuning_options)
pbounds = OrderedDict(zip(tuning_options.tune_params.keys(), bounds))
verbose = 0
if tuning_options.verbose:
verbose = 2
optimizer = BayesianOptimization(f=func,
pbounds=pbounds,
verbose=verbose,
alpha=alpha)
optimizer.maximize(init_points=init_points,
n_iter=n_iter,
acq=acq,
kappa=kappa,
xi=xi)
if tuning_options.verbose:
print(optimizer.max)
return results, runner.dev.get_environment()
# Hyperparameter tuning.
# The function we want to optimise with Bayesian optimisation. Since it will try to maximise rather than minimise the value, we have to return the *negative* RMSE score
def xgb_evaluate(max_depth, gamma, colsample_bytree, subsample):
params = {'eval_metric': 'rmse',
'max_depth': int(max_depth),
'eta': 0.1,
'subsample': subsample,
'gamma': gamma,
'colsample_bytree': colsample_bytree}
cv_result = xgb.cv(params, dtrain, num_boost_round=1000, nfold=5)
return -1.0 * cv_result['test-rmse-mean'].iloc[-1]
# Specifying the bounds of each hyperparameter we want to tune.
xgb_bo = BayesianOptimization(xgb_evaluate, {'max_depth': (3, 7),
'gamma': (0, 1),
'colsample_bytree': (0.3, 0.9),
'subsample': (0.5, 0.9)})
xgb_bo.maximize(init_points=10, n_iter=20, acq='ei')
# Specifiying we want to use the best params which were found during the Bayesian optimisation step.
params = xgb_bo.max['params']
params['max_depth'] = int(params['max_depth'])
model = xgb.train(params, dtrain, num_boost_round=250)
# Adding the senior_jobs column to the test data
jobs = X_test[:,6]
senior_job = []
for j in jobs:
found=False
def decision_tree_regressor(x_train, y_train, x_color_goal, y_color_goal):
'''Optimizing the hyperparameters of the Decision tree regressor'''
df = pd.read_excel('Food dye Spectra.xlsx')
red = df[['e_red']].values
blue = df[['e_blue']].values
green = df[['e_green']].values
wavelength = df[['Wavelength ']].values
def func1(x, y):
estimator = DecisionTreeRegressor(max_depth=np.int(x))
clf = BaggingRegressor(base_estimator=estimator,
n_estimators=np.int(y))
clf = clf.fit(x_train, y_train)
scores = cross_val_score(clf, x_train, y_train, cv=5)
MSE = np.mean(scores)
return 1 / MSE
xmin = 1
xmax = 50
ymin = 1
ymax = 50
pbounds = {'x': (xmin, xmax), 'y': (ymin, ymax)}
optimizer = BayesianOptimization(f=func1, pbounds=pbounds, verbose=3)
optimizer.maximize(init_points=10, n_iter=20)
best_params = optimizer.max["params"]
found_x = best_params['x']
found_y = best_params['y']
max_value = func1(found_x, found_y)
MSE = 1 / func1(found_x, found_y)
###########################################################################################
'''Training the Decision Tree Regressor with the best hyperparameters'''
estimator = DecisionTreeRegressor(max_depth=np.int(found_x))
clf = BaggingRegressor(base_estimator=estimator,
n_estimators=np.int(found_y))
clf = clf.fit(x_train, y_train)
###########################################################################################
'''Using the Decision Tree Regressor to obtain the best concentrations for a desired color'''
def func2(x, y, z):
prediction_array = clf.predict(
np.array([x, y, z]).reshape(-1, 1).transpose())[0]
x_color = prediction_array[0]
y_color = prediction_array[1]
error = (x_color - x_color_goal)**2 + (y_color - y_color_goal)**2
return 1 / error
xmin = 0
xmax = 1
ymin = 0
ymax = 1
zmin = 0
zmax = 1
pbounds = {'x': (xmin, xmax), 'y': (ymin, ymax), 'z': (zmin, zmax)}
optimizer = BayesianOptimization(f=func2, pbounds=pbounds, verbose=3)
optimizer.maximize(init_points=20, n_iter=30)
best_params = optimizer.max["params"]
found_x = best_params['x']
found_y = best_params['y']
found_z = best_params['z']
max_value = func2(found_x, found_y, found_z)
###########################################################################################
a = np.array([found_x, found_y, found_z]).reshape(1, -1)
x_predicted, y_predicted = clf.predict(a)[0]
print('The best concentrations are: ')
print('Red:', found_x)
print('Green:', found_y)
print('Blue:', found_z)
print()
print('Which give predicted color coordinates of:')
print('x predicted', x_predicted)
print('y predicted', y_predicted)
x_actual, y_actual, z_actual = concentration_to_color(
found_x, found_y, found_z, red, green, blue, wavelength)
print('These concentrations give acutal color coordinates of:')
print('x actual', x_actual)
print('y actual', y_actual)
return found_x, found_y, found_z
# work with validation split
print "Creating validation for bayes tuning."
idx0 = np.where(fold_index != 1)
idx1 = np.where(fold_index == 1)
x0 = np.array(xtrain)[idx0, :][0]
x1 = np.array(xtrain)[idx1, :][0]
y0 = np.array(ytrain)[idx0].astype('int32')
y1 = np.array(ytrain)[idx1].astype('int32')
Xn_train = reader.transform(x0)
Xn_valid = reader.transform(x1)
proximal_bayes = BayesianOptimization(proximal_bayes,
{"alpha": (0.001, 0.5),
"L1": (0., 2.),
"L2": (0., 2.),
"alpha_fm": (0.0001, 0.5),
"L1_fm": (0., 2.),
"L2_fm": (0., 2.),
"fm_dim": (int(4), int(10)),
"fm_initDev": (0.05, 0.3),
"epoch": (int(2), int(10))
})
proximal_bayes.maximize(init_points=5, n_iter=20)
print('-' * 53)
print('Final Results')
print('PromixmalFM: %f' % proximal_bayes.res['max']['max_val'])
print(proximal_bayes.res['max']['max_params'])
def main(batch=False):
'''This main function allows quick testing of the batch and non-batch versions
of the simulation.
Keyword Arguments:
batch {bool} -- if True, the simulation will run a batch experiment (default: {False})
'''
np.random.seed(1234)
# Note: all probabilities are in units p(event) per hour
params = {
# Intake Probabilities (Note, 1-sum(these) is probability of no intake)
'pSusceptibleIntake': 0.125,
'pInfectIntake': 0.02,
'pSymptomaticIntake': 0.0,
'pInsusceptibleIntake': 0.05,
# Survival of Illness
'pSurviveInfected': 0.0,
'pSurviveSymptomatic': 0.0,
# Alternate Death Rate
'pDieAlternate': 0.001,
# Discharge and Cleaning
'pDischarge': 0.0,
'pCleaning': 1.0,
# Disease Refractory Period
'refractoryPeriod': 7.0*24.0,
# Death and Symptoms of Illness
'pSymptomatic': 0.0,
'pDie': 0.0,
# Infection Logic
'infection_kernel': [0.05, 0.01],
'infection_kernel_function': 'lambda node, k: k*(1-node[\'occupant\'][\'immunity\'])',
# Immunity Growth (a0*immunity+a1)
# (1.03, 0.001 represents full immunity in 5 days)
#'immunity_growth_factors': [1.03, 0.001],
'immunity_growth_factors': [0.0114, 0.0129, 0.0146, 0.0166, 0.0187, 0.0212, 0.0240,
0.0271, 0.0306, 0.0346, 0.0390, 0.0440, 0.0496, 0.0559,
0.0629, 0.0707, 0.0794, 0.0891, 0.0998, 0.1117, 0.1248,
0.1392, 0.1549, 0.1721, 0.1908, 0.2109, 0.2326, 0.2558,
0.2804, 0.3065, 0.3338, 0.3623, 0.3918, 0.4221, 0.4530,
0.4843, 0.5157, 0.5470, 0.5779, 0.6082, 0.6377, 0.6662,
0.6935, 0.7196, 0.7442, 0.7674, 0.7891, 0.8092, 0.8279,
0.8451, 0.8608, 0.8752, 0.8883, 0.9002, 0.9109, 0.9206,
0.9293, 0.9371, 0.9441, 0.9504, 0.9560, 0.9610, 0.9654,
0.9694, 0.9729, 0.9760, 0.9788, 0.9813, 0.9834, 0.9854,
0.9871, 0.9886],
'immunity_lut': True,
# End Conditions
'max_time': 31*24, # One month
'max_intakes': None,
# Intervention
'intervention': 'TimedRemovalIntervention()' # Different interventions can go here
}
if not batch:
print(params['intervention'])
params['pSusceptibleIntake'] = best_params['pSusceptibleIntake']
params['pInfectIntake'] = best_params['pInfectIntake']
params['pInsusceptibleIntake'] = best_params['pInsusceptibleIntake']
params['pDieAlternate'] = best_params['pDieAlternate']
params['infection_kernel'] = [best_params['infection_kernel_0'], best_params['infection_kernel_1']]
with open('./sim_params.json', 'w+') as out:
json.dump(params, out)
sim = simulation.Simulation(params,
spatial_visualization=True,
aggregate_visualization=True,
return_on_equillibrium=True,)
#print(sim.run())
else:
# Run batch simulation comparing interventions
"""
Grid Search Method with Baysian Optimization
`pSusceptibleIntake`, `pInfectIntake`, `pInsusceptibleIntake`, `pDieAlternate`, and `infection_kernel`
"""
from bayes_opt import BayesianOptimization
from bayes_opt.observer import JSONLogger
from bayes_opt.event import Events
import warnings
log_name = 'APA-XGB_BO-Distemper-03-16-2019-v1'
logger = JSONLogger(path='./'+log_name+'.json')
orig_params = params.copy()
Test = False
Target = {
'Total Intake': 847,
'E2I':68,
'sum_S2D_IS2D':68,
'E2S':432,
'E2IS':347,
'S2I':111
}
def _get_results(_p):
runs = 2
results = simulation.BatchSimulation(_p, runs).run()
results_dataframe = pd.DataFrame.from_records(results)
results_dataframe = results_dataframe.drop(['S', 'IS', 'SY', 'D'], axis=1)
results_dataframe = results_dataframe.rename(index=str,
columns={"E": "Total Intake",
"I": "Total Infected"})
results_dataframe['Infection Rate'] = \
results_dataframe['Total Infected'] / results_dataframe['Total Intake']
means = results_dataframe.mean()
stes = results_dataframe.std() / np.sqrt(len(results_dataframe))
cols = results_dataframe.columns
return means, stes, cols
def _heuristic(
pSusceptibleIntake,
pInfectIntake,
pInsusceptibleIntake,
pDieAlternate,
infection_kernel_0,
infection_kernel_1
):
params = orig_params.copy()
params['pSusceptibleIntake'] = pSusceptibleIntake
params['pInfectIntake'] = pInfectIntake
params['pInsusceptibleIntake'] = pInsusceptibleIntake
params['pDieAlternate'] = pDieAlternate
params['infection_kernel'] = [infection_kernel_0,infection_kernel_1]
m_0, s_0, c_0 = _get_results(params)
if Test:
return m_0
else:
loss = 0
for key, value in Target.items():
# category-wise normalized L2 loss
loss += abs((m_0[key]-value)/value)
loss /= len(Target)
return -1.*loss
"""
Desired ouput
Total Intake = 847
Empty->Infected = 68,
Susceptible->Dead + Insusceptible->Dead = 68,
Empty->Susceptible=432,
Empty->Insusceptible=347,
Susceptible->Infected=111
When
'pSusceptibleIntake': 0.125,
'pInfectIntake': 0.02,
'pSymptomaticIntake': 0.0,
'pInsusceptibleIntake': 0.05,
# Survival of Illness
'pSurviveInfected': 0.0,
'pSurviveSymptomatic': 0.0,
# Alternate Death Rate
'pDieAlternate': 0.001,
# Discharge and Cleaning
'pDischarge': 0.0,
'pCleaning': 1.0,
# Disease Refractory Period
'refractoryPeriod': 7.0*24.0,
# Death and Symptoms of Illness
'pSymptomatic': 0.0,
'pDie': 0.0,
# Infection Logic
'infection_kernel': [0.05, 0.01],
'infection_kernel_function': 'lambda node, k: k*(1-node[\'occupant\'][\'immunity\'])',
We have
{'E': 987, 'S': 0, 'IS': 511, 'I': 369, 'SY': 0, 'D': 46, 'E2I': 98, 'sum_S2D_IS2D': 26, 'E2S': 623, 'E2IS': 266, 'S2I': 271}
We need to
Decrease E2I (↓pInfectIntake)
Increase sum_S2D_IS2D (↑pDieAlternate)
Decrese E2S (↓pSusceptibleIntake)
Increase E2IS (↑pInsusceptibleIntake)
Decrease S2I (↓infection_kernel)
"""
BO_wrapper = BayesianOptimization(
_heuristic,
{
'pSusceptibleIntake':(0.05,0.2),
'pInfectIntake':(0.005,0.03),
'pInsusceptibleIntake':(0.025,0.2),
'pDieAlternate':(0.0025,0.01),
'infection_kernel_0':(0.01,0.1),
'infection_kernel_1':(0.001,0.01)
}
)
BO_wrapper.subscribe(Events.OPTMIZATION_STEP, logger)
print('-'*130)
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
BO_wrapper.maximize(init_points=20, n_iter=50, acq='ei', xi=0.01)
print('-'*130)
print('Final Results')
print('Maximum value: %f' % BO_wrapper.max['target'])
print('Best parameters: ', BO_wrapper.max['params'])
Test = True
m_0 = _heuristic(**BO_wrapper.max['params'])
for key, value in Target.items():
print(key, value, m_0[key])
file.write("\n")
file.close()
K.clear_session()
# bayes opt is a maximization algorithm, to minimize validation_loss, return 1-this
bayes_opt_score = 1.0 - score[1]
return bayes_opt_score
# bayesian optimization
optimizer = BayesianOptimization(
f=train_model,
pbounds={
'lstm': (0, 1.999), # 350 if smaller than 1 else 500
'lstm_number': (2, 3.999),
'lr': (0.001, 0.0001),
'batch_size': (1, 2.999), # 16 if smaller than 2 else 64
# 'kernel_size': (3, 5.999)
},
verbose=2)
optimizer.maximize(init_points=2, n_iter=12)
# training-test-evaluation iterations with best params
if os.path.isdir('results') is False:
os.mkdir('results')
targets = [e['target'] for e in optimizer.res]
bs_fname = 'bs_taxiNYC.json'
with open(os.path.join('results', bs_fname), 'w') as f:
json.dump(optimizer.res, f, indent=2)
best_index = targets.index(max(targets))
parser.add_argument("-l", "--log-dir", type=str, default="./logs.json")
args, unknown = parser.parse_known_args()
c1 = args.c1
c2 = args.c2
w = args.w
k = args.k
p = args.p
n_particles = args.n_particles
epochs = args.epochs
search = args.search
log_dir = args.log_dir
mode = args.mode.lower()
if search:
import os
from bayes_opt import BayesianOptimization
from bayes_opt.observer import JSONLogger
from bayes_opt.event import Events
from bayes_opt.util import load_logs
pbounds = {"c1": (0, 1.0), "c2": (0, 1.0), "w": (0, 1.0)}
optimizer = BayesianOptimization(f=pso, pbounds=pbounds)
if os.path.exists(log_dir):
load_logs(optimizer, logs=[log_dir])
optimizer.subscribe(Events.OPTMIZATION_STEP, JSONLogger(path=log_dir))
optimizer.maximize(init_points=100, n_iter=25)
print(optimizer.max)
else:
pso(c1, c2, w, k, p, n_particles, epochs, mode, verbose=2, visualize=1)
for decay_test in range(-60,-30,10):#[0,10**-6,10**-5,10**-4]:
max = -1e-6
decay_test = 10**(decay_test/10)
val = get_vals(lr = lr_test,decay = decay_test)
file.write(str(lr_test) + "," + str(decay_test)+ "," + str(val) + "\n")
if val > max:
lr = lr_test
decay = decay_test
send_slack_message("Finished Lr and decay optimisation images")
'''
bo = BayesianOptimization(
lambda conv1_size, conv2_size, no_layers: get_vals(
conv1_size, conv2_size, no_layers, opt=opt, lr=lr, decay=decay), {
"conv1_size": (2, 5),
"conv2_size": (2, 5),
"no_layers": (2, 4)
})
bo.explore({"conv1_size": (2, 5), "conv2_size": (2, 5), "no_layers": (2, 4)})
bo.maximize(init_points=2, n_iter=50, kappa=10, acq="ucb") #, acq="ucb"
json.dump(bo.res['max'], open("bayes_rcnn_pt_results.txt", 'w'))
print(bo.res['all'])
np.save("bayes_rcnn_opt_all_values", bo.res['all']['values'])
with open("bayes_opt_images_all_results.txt", 'w') as file:
for i in bo.res['all']['params']:
json.dump(i, file)
file.write("\n")
loss_mat.append(np.reshape(loss_record, [-1, 1]))
error_u_mat.append(np.reshape(error_u_record, [-1, 1]))
return np.mean(error_vec)
def eval_nn_performance(node_per_layer):
num_layer = 10
node_per_layer = int(np.round(node_per_layer))
f = open("./width_cmd.txt", "a")
f.write('python run_BO.py %d %d\n' % (node_per_layer, num_layer))
return -main_fun(num_try=1,
width=node_per_layer,
depth=num_layer,
iter_num=50000,
learning_rate=5.0e-4)
if __name__ == "__main__":
bo = BayesianOptimization(
lambda node_per_layer: eval_nn_performance(node_per_layer),
{'node_per_layer': (2, 100)})
bo.maximize(init_points=2, n_iter=48)
# print(bo.res['max'])
# node_per_layer_star = bo.res['max']['max_params']['node_per_layer']
# num_layer_star = bo.res['max']['max_params']['num_layer']
# print(eval_nn_performance(node_per_layer_star, num_layer_star))
error_u_mat.append(np.reshape(error_u_record, [-1, 1]))
return np.mean(error_vec)
def eval_nn_performance(num_layer):
node_per_layer = 10
num_layer = int(np.round(num_layer))
f = open("./depth_cmd.txt", "a")
f.write('python run_BO.py %d %d\n' % (node_per_layer, num_layer))
f.close()
return -main_fun(num_try=1, width=node_per_layer, depth=num_layer, iter_num=50000, learning_rate=5.0e-4)
if __name__ == "__main__":
bo = BayesianOptimization(lambda num_layer: eval_nn_performance(num_layer),
{'num_layer': (1, 40)})
bo.maximize(init_points=2, n_iter=48)
# print(bo.res['max'])
# node_per_layer_star = bo.res['max']['max_params']['node_per_layer']
# num_layer_star = bo.res['max']['max_params']['num_layer']
# print(eval_nn_performance(node_per_layer_star, num_layer_star))
def predict_score(rounds):
train_df = pd.read_csv('data/train.csv')
test_df = pd.read_csv('data/test.csv')
train_X = train_df.drop('Unnamed: 0',
axis=1).drop('name',
axis=1).drop('score',
axis=1).drop('id_ref',
axis=1)
train_Y = train_df['score']
test_X = test_df.drop('Unnamed: 0',
axis=1).drop('name',
axis=1).drop('score',
axis=1).drop('id_ref',
axis=1)
X_train, X_test, y_train, y_test = train_test_split(train_X, train_Y)
dtrain = xgb.DMatrix(X_train, label=y_train)
del (X_train)
dtest = xgb.DMatrix(X_test)
del (X_test)
def xgb_evaluate(max_depth, gamma, colsample_bytree):
params = {
'nthread': 1,
'eval_metric': 'rmse',
'max_depth': int(max_depth),
'subsample': 0.8,
'eta': 0.1,
'gamma': gamma,
'colsample_bytree': colsample_bytree
#'min_child_weight': min_child_weight,
}
# Used around 1000 boosting rounds in the full model
cv_result = xgb.cv(params, dtrain, num_boost_round=1000, nfold=5)
# Bayesian optimization only knows how to maximize, not minimize, so return the negative RMSE
return -1.0 * cv_result['test-rmse-mean'].iloc[-1]
xgb_bo = BayesianOptimization(
xgb_evaluate,
{
'max_depth': (3, 7),
'gamma': (0, 1),
'colsample_bytree': (0.3, 0.9)
#'min_child_weight': (5, 30),
#'eta': (0.1, 1.0)
})
for n in range(rounds):
try:
temp_df = pd.read_csv('data/params/params{}.csv'.format(n))
params = {}
for k in temp_df['keys'].values:
temp = temp_df.loc[temp_df['keys'] == k]
print(temp['keys'].values[0])
params[temp['keys'].values[0]] = temp['values'].values[0]
params['max_depth'] = int(params['max_depth'])
except:
# Use the expected improvement acquisition function to handle negative numbers
# Optimally needs quite a few more initiation points and number of iterations
xgb_bo.maximize(init_points=3, n_iter=5, acq='ei')
params = xgb_bo.res[-1]['params']
params['max_depth'] = int(params['max_depth'])
df = pd.DataFrame(data={
'keys': list(params.keys()),
'values': list(params.values())
})
df.to_csv('data/params{}.csv'.format(n))
feature_columns = train_X.columns.values
dmatrix = xgb.DMatrix(train_X.values,
train_Y.values,
feature_names=train_X.columns.values)
clf = xgb.train(params, dmatrix)
dmatrixtest = xgb.DMatrix(test_X.values,
feature_names=test_X.columns.values)
res = clf.predict(dmatrixtest)
res = pd.DataFrame(res, columns=['prediction'])
final_res = pd.concat([test_df[['name', 'id_ref']], res], axis=1)
final_res = final_res.sort_values(by=['prediction'], ascending=False)
final_res.to_csv('data/result{}.csv'.format(n))
fscores = pd.DataFrame({
'X': list(clf.get_fscore().keys()),
'Y': list(clf.get_fscore().values())
})
fscores = fscores.sort_values(by=['Y'], ascending=False)
fscores.to_csv('data/fscores{}.csv'.format(n))
print('Finished score prediction')
def fit(self, X_train, y_train):
"""
Function to run xgboost.cv() method first to find the best number of boosting round
and train a model based on that on (X_train, y_train) set and returns it.
"""
# helper function for xgboost eval
def _xgb_eval(
max_depth,
subsample,
colsample_bytree,
min_child_weight,
learning_rate,
gamma,
reg_alpha,
reg_lambda,
):
"""
Helper Function to eval bayesian optimization
"""
params = {
"eval_metric": "auc",
"tree_method": "hist",
"objective": self.objective,
"max_delta_step": 1,
"silent": True,
"nthread": 4,
"scale_pos_weight": 1,
"reg_alpha": reg_alpha,
"reg_lambda": reg_lambda,
"learning_rate": learning_rate,
"max_depth": int(max_depth),
"min_child_weight": min_child_weight,
"gamma": gamma,
"subsample": subsample,
"colsample_bytree": colsample_bytree,
}
cv_result = xgb.cv(
params=params,
dtrain=self.dtrain_,
num_boost_round=self.num_boost_round,
nfold=self.n_splits,
stratified=self.stratified,
metrics=self.metrics,
early_stopping_rounds=self.early_stopping_rounds,
seed=self.random_state,
shuffle=True,
)
# set to return + or - results based on metric for maximization
if self.metrics in ["logloss", "error"]:
return (-1) * cv_result.iloc[-1][2]
else:
return cv_result.iloc[-1][2]
# creating dtrain
self.dtrain_ = self._dtrain(X_train, y_train)
# xgb_bo definition
self.optimizer_ = BayesianOptimization(
f=_xgb_eval,
pbounds=self.pbounds,
random_state=self.random_state,
verbose=self.verbose,
)
# maximizing xgb_bo
self.optimizer_.maximize(init_points=self.init_points,
n_iter=self.n_iter,
acq=self.acq)
# initiate the results
self.optimization_results_ = self.get_optimization_results()
# initiate the best params
self.best_params_ = self.get_best_params()
# initiate the best performance
self.best_performance_ = self.get_best_performance()
return None
# rfc = RandomForestClassifier(i)
# rfc.fit(X_train, y_train)
#
# score = rfc.score(X_test, y_test)
return history.history['val_acc'][-1]
# Bounded region of parameter space
pbounds = {
'n1': (1, 100),
'n2': (1, 100),
'n2': (1, 100),
'dropout1': (0.1, 0.5),
'dropout2': (0.1, 0.5),
'batchsize': (1, 100)
}
optimizer = BayesianOptimization(
f=eval_network,
pbounds=pbounds,
random_state=1,
)
optimizer.maximize(
init_points=15,
n_iter=2,
)
print(optimizer.max)
def bayesian_search(
config_path,
inferencecfg,
pbounds,
edgewisecondition=True,
shuffle=1,
trainingsetindex=0,
modelprefix="",
snapshotindex=-1,
target="rpck_test",
maximize=True,
init_points=20,
n_iter=50,
acq="ei",
log_file=None,
dcorr=5,
leastbpts=3,
printingintermediatevalues=True,
): #
if "rpck" in target:
assert maximize == True
if "rmse" in target:
assert maximize == False
cfg = auxiliaryfunctions.read_config(config_path)
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
cfg["TrainingFraction"][int(trainingsetindex)],
shuffle,
cfg,
modelprefix=modelprefix,
)
),
)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
cfg["TrainingFraction"][int(trainingsetindex)],
cfg["iteration"],
modelprefix=modelprefix,
)
# load params
fns = return_evaluate_network_data(
config_path,
shuffle=shuffle,
trainingsetindex=trainingsetindex,
modelprefix=modelprefix,
)
predictionsfn = fns[snapshotindex]
data, metadata = auxfun_multianimal.LoadFullMultiAnimalData(predictionsfn)
params = set_up_evaluation(data)
columns = ["train_iter", "train_frac", "shuffle"]
columns += [
"_".join((b, a))
for a in ("train", "test")
for b in ("rmse", "hits", "misses", "falsepos", "ndetects", "pck", "rpck")
]
train_iter = trainingsetindex # int(predictionsfn.split('-')[-1].split('.')[0])
train_frac = cfg["TrainingFraction"][
train_iter
] # int(predictionsfn.split('trainset')[1].split('shuffle')[0])
trainIndices = metadata["data"]["trainIndices"]
testIndices = metadata["data"]["testIndices"]
if edgewisecondition:
mf = str(
auxiliaryfunctions.GetModelFolder(
cfg["TrainingFraction"][int(trainingsetindex)],
shuffle,
cfg,
modelprefix=modelprefix,
)
)
modelfolder = os.path.join(cfg["project_path"], mf)
path_inferencebounds_config = (
Path(modelfolder) / "test" / "inferencebounds.yaml"
)
try:
inferenceboundscfg = auxiliaryfunctions.read_plainconfig(
path_inferencebounds_config
)
except FileNotFoundError:
print("Computing distances...")
from deeplabcut.pose_estimation_tensorflow import calculatepafdistancebounds
inferenceboundscfg = calculatepafdistancebounds(
config_path, shuffle, trainingsetindex
)
auxiliaryfunctions.write_plainconfig(
path_inferencebounds_config, inferenceboundscfg
)
partaffinityfield_graph = params["paf_graph"]
upperbound = np.array(
[
float(
inferenceboundscfg[str(edge[0]) + "_" + str(edge[1])]["intra_max"]
)
for edge in partaffinityfield_graph
]
)
lowerbound = np.array(
[
float(
inferenceboundscfg[str(edge[0]) + "_" + str(edge[1])]["intra_min"]
)
for edge in partaffinityfield_graph
]
)
upperbound *= inferencecfg["upperbound_factor"]
lowerbound *= inferencecfg["lowerbound_factor"]
else:
lowerbound = None
upperbound = None
def dlc_hyperparams(**kwargs):
inferencecfg.update(kwargs)
# Ensure type consistency
for k, (bound, _) in pbounds.items():
inferencecfg[k] = type(bound)(inferencecfg[k])
stats = compute_crossval_metrics_preloadeddata(
params,
columns,
inferencecfg,
data,
trainIndices,
testIndices,
train_iter,
train_frac,
shuffle,
lowerbound,
upperbound,
dcorr=dcorr,
leastbpts=leastbpts,
)
# stats = compute_crossval_metrics(config_path, inferencecfg, shuffle,trainingsetindex,
# dcorr=dcorr,leastbpts=leastbpts,modelprefix=modelprefix)
if printingintermediatevalues:
print(
"rpck",
stats["rpck_test"].values[0],
"rpck train:",
stats["rpck_train"].values[0],
)
print(
"rmse",
stats["rmse_test"].values[0],
"miss",
stats["misses_test"].values[0],
"hit",
stats["hits_test"].values[0],
)
# val = stats['rmse_test'].values[0]*(1+stats['misses_test'].values[0]*1./stats['hits_test'].values[0])
val = stats[target].values[0]
if np.isnan(val):
if maximize: # pck case
val = -1e9 # random small number
else: # RMSE, return a large RMSE
val = 1e9
if not maximize:
val = -val
return val
opt = BayesianOptimization(f=dlc_hyperparams, pbounds=pbounds, random_state=42)
if log_file:
load_logs(opt, log_file)
logger = JSONLogger(
path=os.path.join(evaluationfolder, "opti_log" + DLCscorer + ".json")
)
opt.subscribe(Events.OPTIMIZATION_STEP, logger)
opt.maximize(init_points=init_points, n_iter=n_iter, acq=acq)
inferencecfg.update(opt.max["params"])
for k, (bound, _) in pbounds.items():
tmp = type(bound)(inferencecfg[k])
if isinstance(tmp, np.floating):
tmp = np.round(tmp, 2).item()
inferencecfg[k] = tmp
return inferencecfg, opt
return model_ft, best_acc, best_f1
pbounds = {
'lr': (0.05, 0.069),
'momentum': (0.001, 0.4),
'decay': (0.00001, 0.01)
}
bayesian_opt = False
if bayesian_opt:
print('Hyperparameter tuning started: ')
optimizer = BayesianOptimization(
f=objective1,
pbounds=pbounds,
random_state=1,
)
optimizer.maximize(
init_points=1,
n_iter=20,
)
# 0.625 | 16.0 | 0.0001 | 1e-06 | 0.99
# 0.9306 | 8.728 | 0.003729 | 0.05634 | 0.06412
# 0.9167 | 11.95 | 0.0001 | 0.059 | 0.05
# 0.9167 | 5.678 | 0.0001 | 0.059 | 0.05
# linknet: 15.53 | 0.009325 | 0.005589 | 0.3748
# objective1 (0.005589,0.3748,15,0.009325)
max_depth=int(max_depth),
random_state=2016,
n_jobs=6),
X,
y,
scoring=score,
n_jobs=3,
cv=3).mean()
if __name__ == "__main__":
rfcBO = BayesianOptimization(
rfrcv, {
'n_estimators': (10, 500),
'min_samples_split': (2, 25),
'max_features': (0.1, 0.999),
'max_depth': (2, 25)
})
rfcBO.maximize()
print('-' * 53)
print('Final Results')
print('RFC: %f' % rfcBO.res['max']['max_val'])
print('\nFinal Results', file=log_file)
print('RandomForest: %f' % rfcBO.res['max']['max_val'], file=log_file)
log_file.flush()
log_file.close()
#####################################
# -----------------------------------------------------
# Final Results
def bayes_parameter_opt_lgb(X,
y,
init_round=15,
opt_round=25,
n_folds=3,
random_seed=6,
n_estimators=10000,
output_process=False):
# prepare data
train_data = lgb.Dataset(data=X, label=y, free_raw_data=False)
# parameters
def lgb_eval(learning_rate, num_leaves, feature_fraction, bagging_fraction,
max_depth, max_bin, min_data_in_leaf, min_sum_hessian_in_leaf,
subsample):
params = {'application': 'binary', 'metric': 'auc'}
params['learning_rate'] = max(min(learning_rate, 1), 0)
params['num_leaves'] = int(round(num_leaves))
params['feature_fraction'] = max(min(feature_fraction, 1), 0)
params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)
params['max_depth'] = int(round(max_depth))
params['max_bin'] = int(round(max_bin))
params['min_data_in_leaf'] = int(round(min_data_in_leaf))
params['min_sum_hessian_in_leaf'] = min_sum_hessian_in_leaf
params['subsample'] = max(min(subsample, 1), 0)
cv_result = lgb.cv(params,
train_data,
nfold=n_folds,
seed=random_seed,
stratified=True,
verbose_eval=200,
metrics=['auc'])
return max(cv_result['auc-mean'])
lgb0 = BayesianOptimization(lgb_eval, {
'learning_rate': (0.01, 1.0),
'num_leaves': (24, 80),
'feature_fraction': (0.1, 0.9),
'bagging_fraction': (0.8, 1.0),
'max_depth': (5, 30),
'max_bin': (20, 90),
'min_data_in_leaf': (20, 80),
'min_sum_hessian_in_leaf': (0, 100),
'subsample': (0.01, 1.0)
},
random_state=200)
# n_iter：How many steps of bayesian optimization you want to perform. The more steps the more likely to find a good maximum you are.
# init_points：How many steps of random exploration you want to perform. Random exploration can help by diversifying the exploration space.
lgb0.maximize(n_iter=opt_round, init_points=init_round)
model_auc = []
for model in range(len(lgb0.res)):
model_auc.append(lgb0.res[model]['target'])
# return best parameters
return lgb0.res[pd.Series(model_auc).idxmax()]['target'], lgb0.res[
pd.Series(model_auc).idxmax()]['params']
train_pred, test_pred, mean, std, full_score = five_fold_with_baging_with_target_encode(
train,
target,
test,
train_id,
test_id,
metric,
mdl,
mdl_type='xgb',
seed=1024,
stratified=False,
n_splits=1,
n_folds=5,
n_bags=1,
early_stop=50,
verbose=False)
return mean
xgbBO = BayesianOptimization(
xgb_evaluate, {
'max_depth': (3, 7),
'colsample_bytree': (0.4, 1),
'subsample': (0.5, 1),
'gamma': (0, 10),
'min_child_weight': (1, 50),
})
xgbBO.maximize(init_points=init_points, n_iter=num_iter)
if __name__ == '__main__':
num_iter = 15
init_points = 15
"""
float reward_first_step_idle = 0.001f;
float reward_sooner_later_ratio = 0.98f;
float reward_extraBombPowerupPoints = 0.6f;
float reward_extraRangePowerupPoints = 0.4f;
float reward_otherKickPowerupPoints = 0.2f;
float reward_firstKickPowerupPoints = 0.7f;
float reward_move_to_enemy = 100.0f;
float reward_move_to_pickup = 1000.0f;
"""
xgbBO = BayesianOptimization(
xgb_evaluate, {
'reward_first_step_idle': (0.01, 0.0001),
'reward_sooner_later_ratio': (0.90, 0.99),
'reward_extraBombPowerupPoints': (0.4, 1.0),
'reward_extraRangePowerupPoints': (0.2, 1.0),
'reward_otherKickPowerupPoints': (0.01, 1.0),
'reward_firstKickPowerupPoints': (0.2, 0.7),
'reward_move_to_enemy': (80, 130),
'reward_move_to_pickup': (800, 1300),
})
xgbBO.maximize(init_points=init_points, n_iter=num_iter)
X_Test = pd.concat((X_Test, x_test_sp), axis=1)
dtrain = xgb.DMatrix(X, Y)
dtest = xgb.DMatrix(X_Test)
# A parameter grid for XGBoost
params = {
'min_child_weight': (1, 20),
'gamma': (0, 10),
'subsample': (0.5, 1),
'colsample_bytree': (0.1, 1),
'max_depth': (2, 10)
}
# Initialize BO optimizer
xgb_bayesopt = BayesianOptimization(train_xgb, params)
# Maximize R2 score
xgb_bayesopt.maximize(init_points=5, n_iter=25)
# Get the best params
p = xgb_bayesopt.res['max']['max_params']
xgb_params = {
'n_trees': 250,
'eta': 0.01,
'max_depth': int(p['max_depth']),
'subsample': max(min(p['subsample'], 1), 0),
'objective': 'reg:linear',
'base_score': np.mean(Y), # base prediction = mean(target)
'silent': 1,
import subprocess
import os
import numpy as np
import sys
sys.path.append('/Users/malithjayasinghe/BayesianOptimization/')
def black_box_function(x):
print(x)
#return -1*0.01*x*x
return -1.0*np.sin(x/5.0)*x
from bayes_opt import BayesianOptimization
pbounds = {'x': (60, 100)}
optimizer = BayesianOptimization(
f=black_box_function,
pbounds=pbounds,
verbose=2, # verbose = 1 prints only when a maximum is observed, verbose = 0 is silent
random_state=1,
)
optimizer.maximize(
init_points=1,
n_iter=200,
)
print(optimizer.max)
# return function (with unknown internals) we wish to maximize.
return -CreditCost
optimizer = BayesianOptimization(
f=objective,
#pbounds=pbounds,
pbounds={
'SCI_RELU': (1, 1.99),
'SCI_BIAS': (1, 1.99),
'SCI_loss_type': (2, 2.99),
'SCI_optimizer': (8, 8.99),
'SCI_LR': (0.1, 0.4),
'SCI_MM': (0.001, 0.999),
'SCI_REGULARIZATION': (0, 0.8),
'SCI_EPOCHS': (200, 200),
'SCI_BATCH_SIZE': (128, 194),
'SCI_DROPOUT': (0.3, 0.7),
'SCI_L_SECOND': (1, 4),
'SCI_BN_MOMENTUM': (0.1, 0.1),
'SCI_SGD_MOMENTUM': (0, 0.999),
'SCI_BN_EPS': (0, 14.99),
'SCI_BN_STATS': (0, 0.99),
'SCI_LAST_LAYER': (1, 1.99),
'SCI_ACT_LAYER': (2, 2.99)
},
verbose=
2, # verbose = 1 prints only when a maximum is observed, verbose = 0 is silent
random_state=1,
)
#optimizer.maximize(
#n_iter=TRIALS, acq="ucb", kappa=0.1
r_value_ = stats.pearsonr(x_, y_)
r_value_all.append(r_value_[0])
p_value_all.append(r_value_[1])
return mse_generated_test, mae_generated_test, np.mean(r_value_all)
if __name__ == '__main__':
test_test('VAE_Hawkes_sigmoid_MIMIC_train__3_3_重新训练_8_17.txt')
Encode_Decode_Time_BO = BayesianOptimization(
train, {
'hidden_size': (5, 8),
'z_dims': (5, 8),
'learning_rate': (-5, 1),
'l2_regularization': (-5, 1),
'kl_imbalance': (-6, 1),
'reconstruction_imbalance': (-6, 1),
'generated_mse_imbalance': (-6, 1),
'likelihood_imbalance': (-6, 1)
})
Encode_Decode_Time_BO.maximize()
print(Encode_Decode_Time_BO.max)
# mse_all = []
# r_value_all = []
# mae_all = []
# for i in range(50):
# mse, mae, r_value = train(hidden_size=256,
# learning_rate=0.004353202451279688,
# l2_regularization=1e-5,
#presision=cm[1][1]
#presision=(recall_score(Y_validation, valid_y_pred)+precision_score(Y_validation, valid_y_pred))
#presision=precision_score(Y_validation, valid_y_pred)
presision = accuracy_score(Y_validation, valid_y_pred)
#presision=recall_score(Y_validation, valid_y_pred)
return presision
from bayes_opt import BayesianOptimization
xgb_bo = BayesianOptimization(
xgb_evaluate,
{
'min_child_weight': (0.5, 100),
'gamma': (0, 30), #30
'subsample': (0.6, 1),
'colsample_bytree': (0.4, 1),
'max_depth': (1, 30),
'learning_rate': (0.05, 0.3),
'scale_pos_weight': (1, 3),
'num_round': (100, 500),
'reg_lambda': (0, 100),
'reg_alpha': (0, 100)
})
start_time = timer(
None) # timing starts from this point for "start_time" variableper
xgb_bo.maximize(init_points=10, n_iter=5, acq='ei')
timer(start_time) # timing ends here for "start_time" variable
params = xgb_bo.max['params']
num_round = int(params["num_round"])
params["max_depth"] = int(params["max_depth"])
fi=pd.read_excel(tmpfile,index_col=0)
fscore = fi['Expected Gain'].to_dict() #Gain, FScore, wFScore, Average wFScore, Average Gain, Expected Gain
meanscore = np.average(fscore.values())
for k in fscore.keys(): fscore[k]/=meanscore*len(fscore)
featimpmean=featimpmean.fillna(1./featimp.shape[1])
normalization = featimpmean[chosen_feat].sum()/featimpmean.sum()/np.sum(fscore.values())/kftune.get_n_splits()
for k,v in fscore.iteritems():
fscores[k]+=normalization*v
curscore = -mean_absolute_error(yback(y_pred,params),yback(train_y,params))
featimp = featimp.append(pd.Series(fscores,name=round(curscore,4)))
return curscore
while True:
init_points=args.init
n_iter=args.iter
bo = BayesianOptimization(score, p_range)
if p: bo.initialize(p)
else: init_points,n_iter=1,0
if args.trunc: init_points,n_iter=0,0
bo.maximize(init_points=init_points, n_iter=n_iter, acq=args.acq)
featimp_cur=featimp
p_new = {}
for i in xrange(len(bo.Y)):
if bo.Y[i] not in bo.y_init:
p_new[bo.Y[i].round(4)]={bo.keys[j]:bo.X[i,j].round(4) for j in xrange(len(bo.keys))}
if not os.path.isfile(picklefile): break
with open(picklefile,'rb') as infile:
try:
(featimp,p_now,n_rounds_without_improve,n_wait)=pickle.load(infile)
p.update(p_now)
'z_dim': (50, 150), # example
# DISCRIMINATOR_LR and GENERATOR_LR are learning rates for discriminator and generator
'discriminator_lr': (0.0002, 0.0002),
'generator_lr': (0.0002, 0.0002),
# LR_DECAY_EPOCH is the # epochs between each time it decays the learning rate
'lr_decay_epoch': (100, 100),
# NUM_EMBEDDING is the number of caption embeddings per image that they FC-layer down to embedding_dim
'num_embedding': (4, 4),
# EMBEDDING_DIM is self explanatory
'embedding_dim': (128, 128),
# GF_DIM is the generator feature vector length, like the depth of the downscaled image volume (kinda. actually it's this divided by 4)
'gf_dim': (128, 128),
# DF_DIM is similar but for the discriminator (and it's divided by 8)
'df_dim': (64, 64),
# COEFF.KL is the kullbach-liebler coefficient for the conditioning augmentation
'coeff_KL': (2.0, 2.0)
}
optimizer = BayesianOptimization(f=train_test_wrapper.run_model,
pbounds=pbounds,
random_state=1)
# init_points: How many steps of random exploration you want to perform.
# Random exploration can help by diversifying the exploration space.
# n_iter: How many steps of bayesian optimization you want to perform.
# The more steps the more likely to find a good maximum you are.
optimizer.maximize(init_points=10, n_iter=100)
print("best values found over 100 iterations of bayesian optimization:")
print(optimizer.max)
