class CombinedLogger():
def __init__(self, path):
self.json = JSONLogger(path)
self.console = ScreenLogger()
def update(self, event, instance):
self.json.update(event, instance)
self.console.update(event, instance)
def bo_search(self):
# Start timing the code
start_time = time.time()
# Bounded region of parameter space
pbounds = {'alpha': (0.01, 0.99), 'beta': (1, 10)}
optimizer = BayesianOptimization(
f=self.spd2flow,
pbounds=pbounds,
random_state=98,
)
logger = JSONLogger(path=ROOT_dir + "/results/logs_" + self.here_seg +
"_" + str(self.direction) + ".json")
optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
optimizer.maximize(
init_points=8,
n_iter=100,
)
print(optimizer.max)
print(
self.here_seg, "is done. Elapsed time was %g seconds" %
(time.time() - start_time))
return {
'alpha': optimizer.max['params']['alpha'],
'beta': optimizer.max['params']['beta'],
'mse': -optimizer.max['target']
}
def optimize(self, load_from: str = None):
self.idx = 0
Popen('ulimit -n 4096', shell=True)
optimizer = BayesianOptimization(
f=self._black_box_function,
pbounds=self.p_bounds,
random_state=1,
)
logger = JSONLogger(path=self.opt_filepath)
optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
if load_from is not None:
logfile = os.path.join(self.out_path, load_from)
if Path(logfile).is_file():
logging.info('Loading logs from ' + logfile)
load_logs(optimizer, logs=[logfile])
else:
logging.info('Could not find a log file under {}'.format(logfile))
optimizer.maximize(
init_points=self.init_points,
n_iter=self.n_iter,
)
def execute(self):
search_log.info('-' * 63)
search_log.info('Performing Bayesian search for hyperparameters:')
for i in range(len(self.bayes_search.hyperparameters)):
search_log.info(
f' {self.bayes_search.hyperparameters[i]}: {self.bayes_search.hyperparameter_sets[i]}'
)
search_log.info('-' * 63)
# Create bayes-opt bounds from configuration and create an optimization object
bounds = self.bayes_search.get_parameter_bounds(
self.bayes_search.hyperparameters,
self.bayes_search.hyperparameter_sets)
optimizer = BayesianOptimization(f=self.perform_bayes_search,
pbounds=bounds,
random_state=1,
verbose=0)
# Load search observations from log files
if self.args.bayes_load:
log_files_list = self.get_load_log_paths()
search_log.info(
f'Loading Bayesian optimization observations from:')
for log in log_files_list:
search_log.info(f' {str(log)}')
bayes_opt.util.load_logs(optimizer, logs=log_files_list)
# Save search observations to log file
if self.args.bayes_save:
bayes_log_path = self.get_save_log_path()
search_log.info(
f'Saving Bayesian optimization observations to \'{bayes_log_path}\''
)
bayes_logger = JSONLogger(path=str(bayes_log_path))
optimizer.subscribe(Events.OPTIMIZATION_STEP, bayes_logger)
# Perform Bayesian searches
optimizer.maximize(init_points=self.args.bayesian[0],
n_iter=self.args.bayesian[1])
optimizer_max = self.get_optimizer_max(optimizer)
search_log.info('-' * 63)
search_log.info('Bayesian search complete')
search_log.info(f'Best Configuration ({optimizer_max["target"]}):')
best_configuration = self.bayes_search.get_search_config_from_bounds(
optimizer_max["params"])
for key, value in best_configuration.items():
search_log.info(f' {key}: {value}')
search_log.info('-' * 63)
self.save_max_to_file(optimizer_max)
search_log.info('-' * 63)
if self.search_config_path.exists():
self.search_config_path.unlink()
def optimize(self, reset=False):
# Bounded region of parameter space
pbounds = {
'men_kappa_fulltime': (0.3, 0.6),
'men_kappa_parttime':
(0.3, 0.7), #'men_mu_scale': (0.01,0.3), 'men_mu_age': (57,62),
'women_kappa_fulltime': (0.3, 0.6),
'women_kappa_parttime': (0.3, 0.7)
} #, 'women_mu_scale': (0.01,0.3), 'women_mu_age': (57,62)}
optimizer = BayesianOptimization(
f=self.black_box_function,
pbounds=pbounds,
verbose=
2, # verbose = 1 prints only when a maximum is observed, verbose = 0 is silent
random_state=1,
)
LOG_DIR = Path().absolute() / 'bayes_opt_logs'
LOG_DIR.mkdir(exist_ok=True)
filename = 'log_0.json'
# talletus
logfile = str(LOG_DIR / filename)
logger = JSONLogger(path=logfile)
if Path(logfile).exists() and not reset:
load_logs(optimizer, logs=[logfile])
optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
optimizer.maximize(
init_points=2,
n_iter=20,
)
print('The best parameters found {}'.format(optimizer.max))
def solve(self, init_points=15, n_iter=5000):
logger = JSONLogger(path="ABC_Results/user_parameters_p{}.json".format(
self.participant))
self.optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
self.optimizer.maximize(
init_points=init_points,
n_iter=n_iter,
)
print(self.optimizer.max)
def optimize():
# define bounds for the params you want to optimize. Can be multivariate. Check https://github.com/fmfn/BayesianOptimization on how to
bounds = {
'localAreaDensity': (0.01, 0.15),
'permanenceIncrement': (0.01, 0.1),
}
optimizer = BayesianOptimization(
f=target_func,
pbounds=bounds,
random_state=1,
)
# We can start from saved logs
if os.path.isfile('./local_area_density_optimization_logs_base.json'):
print('Loading Logs...')
load_logs(optimizer, logs=["./local_area_density_optimization_logs_base.json"]);
# The new log file to write to
json_logger = JSONLogger(path="./local_area_density_optimization_logs.json")
optimizer.subscribe(Events.OPTIMIZATION_STEP, json_logger)
# Additionally log to console
screen_logger = ScreenLogger()
optimizer.subscribe(Events.OPTIMIZATION_STEP, screen_logger)
# If you want to guide the optimization process
val = 0.02
while val <= 0.04:
optimizer.probe(
params={
'localAreaDensity': val,
'permanenceIncrement': 0.04,
},
lazy=True,
)
val = round(val + 0.001, 3)
optimizer.maximize(
init_points=20,
n_iter=50,
)
print(optimizer.max)
# cleanup temp dir
shutil.rmtree(os.path.join('temp'))
def hyper():
opt = BayesianOptimization(f=train,
pbounds={
'lr': (0.0001, 0.001),
'lm': (0.75, 0.95),
'tpri': (10, 150),
'rpr': (0.25, 0.75),
'dmc': (0.6, 0.9),
'wd': (0.00001, 0.001)
},
verbose=2)
logger = JSONLogger(path="./bo_logs.json")
opt.subscribe(Events.OPTIMIZATION_STEP, logger)
opt.maximize(init_points=3, n_iter=10)
load_logs(opt, logs=["./bo_logs.json"])
print('maximum: ', opt.max)
def optimize(self, path, filename, bound_trafo):
assert (self.logger and (path is not None) and (filename is not None),
'Filename and path are necessary when the logger is active')
pbounds = self.create_iterables()
if bound_trafo:
bounds_transformer = SequentialDomainReductionTransformer()
optimizer = BayesianOptimization(
f=self.objective,
pbounds=pbounds,
verbose=self.verbose,
random_state=self.seed,
bounds_transformer=bounds_transformer,
)
else:
optimizer = BayesianOptimization(
f=self.objective,
pbounds=pbounds,
verbose=self.verbose,
random_state=self.seed,
)
if self.logger:
logger = JSONLogger(path=path + filename + '.txt')
optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
optimizer.maximize(init_points=self.initial, n_iter=self.iterations)
# write bound-trajectories in separate log-file,
# if bounds have been transformed
if bound_trafo:
for i, p in enumerate(pbounds.keys()):
bounds = {p: {}}
bounds[p]['min'] = [b[i][0] for b in bounds_transformer.bounds]
bounds[p]['max'] = [b[i][1] for b in bounds_transformer.bounds]
with open(path + filename + '_bounds.txt', "a") as f:
f.write(json.dumps(bounds))
def optimize_local_area_density():
# optimize localAreaDensity
bounds = {
'localAreaDensity': (0.01, 0.15),
}
optimizer = BayesianOptimization(
f=target_func,
pbounds=bounds,
random_state=1,
)
if os.path.isfile('./local_area_density_optimization_logs_base.json'):
print('Loading Logs...')
load_logs(optimizer,
logs=["./local_area_density_optimization_logs_base.json"])
logger = JSONLogger(path="./local_area_density_optimization_logs.json")
optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
val = 0.02
while val <= 0.04:
print('Adding', val)
optimizer.probe(
params={
'localAreaDensity': val,
},
lazy=True,
)
val = round(val + 0.001, 3)
print('Starting optimization...')
optimizer.maximize(
init_points=20,
n_iter=50,
)
print(optimizer.max)
batch = int(batch)
epochs = int(epochs)
return Bayes_Unet(batch, epochs, lr, weight_decay, lr_multiplier)
# Bounded region of parameter space
pbounds = {'batch': (4, 16), 'epochs': (60, 140), 'lr': (1e-6, 1e-2), 'weight_decay': (1e-8, 1e-3), 'lr_multiplier': (0.1, 1)}
hyperparams = BayesianOptimization(
f=function_to_be_optimized,
pbounds=pbounds,
verbose=2,
random_state=1,
)
logger = JSONLogger(path="/content/drive/MyDrive/UNET/Training models/logs.json")
hyperparams.subscribe(Events.OPTIMIZATION_STEP, logger)
hyperparams.maximize(
init_points=3, # 3 random trials
n_iter=30 # 30 Bayesian steps
)
"""## **Testing + visualization** """
import numpy as np
import matplotlib.pyplot as plt
from IPython.core.debugger import set_trace
import torch
import torch.nn as nn
'cutoff': (0, 0.2),
'instates': (10, 26),
'prestates': (10, 26),
'C': (0, 50)
}
optimizer = BayesianOptimization(
f=config.lpcfg_optimize.opt,
pbounds=pbounds,
verbose=2,
random_state=42,
)
# This is dangerous as it deletes the original file if you have one
# waiting for the next update that fixes this
logger = JSONLogger(path="./logs.json")
optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
# from bayes_opt.util import load_logs
# load_logs(optimizer, logs=["./logs.json"])
optimizer.probe(params={
'cutoff': 0.01,
'instates': 16,
'prestates': 16,
'C': 10
})
optimizer.maximize(
init_points=5,
n_iter=500,
verbose=2,
random_state=1,
lazy_gpr=True,
lag=10000)
else:
opt = BayesianOptimization(f=black_box_function,
pbounds=pbounds,
verbose=2,
random_state=1,
lazy_gpr=False)
if (args.restart):
print('restart')
load_logs(opt, logs=[args.log_gpr])
def_logger = JSONLogger(path=logfile)
opt.subscribe(Events.OPTMIZATION_STEP, def_logger)
#print(args.iter)
opt.maximize(
init_points=args.seed,
n_iter=args.iter,
samples=None,
eps=0.035, # to reach an accuracy of 0.965
solution=1,
acq="ei",
xi=0.01)
end = time.time()
passed = end - start
print('target : ', opt.max['target'], ', lazy_gpr: ', lazy_gpr)
print('Time(sec.) : ' + str(passed) + "\n")
lazy_gpr = True
'ridge_alpha': 0.0006696012127733874
}
]
optimizer = BayesianOptimization(
f=black_box_function,
pbounds=pbounds,
random_state=42,
)
for point in start_points:
optimizer.probe(
params=point,
lazy=True,
)
if LOAD_PREV:
load_logs(optimizer, logs=["./calc_shifts_params.json"]);
logger = JSONLogger(path="./calc_shifts_params.json")
scrlogger = ScreenLogger()
optimizer.subscribe(Events.OPTIMIZATION_STEP, scrlogger)
optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
optimizer.maximize(
init_points=500,
n_iter=500,
)
print(optimizer.max)
'freeze_base_model': False,
'contrastive_learning_rate': lr,
}
PARAMS[
'contrastive_model_path'] = f"/home/eyal/privet_dev/cov3_sagi/contrastive_models/contrastive_{PARAMS['contrastive_epochs']}.h5"
params = AttrDict(PARAMS)
print(params)
accuracy = covid_with_neptune(params)
return accuracy
# Bounded region of parameter space
pbounds = {'contrastive_epochs': (2, 10), 'lr': (0.0000001, 0.0001)}
optimizer = BayesianOptimization(
f=black_box_covid,
pbounds=pbounds,
random_state=1,
)
logger = JSONLogger(path="./logs_opt1.json")
optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
optimizer.maximize(
init_points=10,
n_iter=200,
)
print(f'best params so far: {optimizer.max}')
sum = 0.0
for i in trange(1, MAX0):
for j in trange(1, MAX1):
name = str(i) + '_' + str(j)
image_x = np.load(
os.getcwd() +
"/encodedArray/bit_{0}/{1}.npy".format(STEN_X, name))
image_y = np.load(
os.getcwd() +
"/decodedArray/bit_{0}/{1}.npy".format(STEN_X, name))
sum += mse(
generator_g.predict(
np.asarray([image_x / 255.0], dtype='float32')),
np.asarray([image_y / 255.0], dtype='float32'))
avg = sum / ((MAX0 - 1) * (MAX1 - 1))
return avg
bounds = {
'EPOCHS': EPOCHS_RANGE,
'LAMBDA': LAMBDA_RANGE,
'steps_per_epochs': STEPS_PER_EPOCH_RANGE
}
optimizer = BayesianOptimization(f=Black_Box, pbounds=bounds, random_state=1)
logger = JSONLogger(path="./logs_{0}.json".format(STEN_X))
optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
optimizer.maximize(init_points=2, n_iter=10)
return bayes_opt_score
# bayesian optimization
optimizer = BayesianOptimization(
f=train_model,
pbounds={
'encoder_blocks': (2, 2),
'lr': (0.001, 0.0001),
'batch_size': (1, 3.999), # *16
'kernel_size': (3, 4.999)
},
verbose=2)
bs_fname = 'bs_taxiNYC.json'
logger = JSONLogger(path="./results/" + bs_fname)
optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
optimizer.maximize(init_points=2, n_iter=10)
# New optimizer is loaded with previously seen points
# load_logs(optimizer, logs=["./results/" + bs_fname], reset = False)
# optimizer.maximize(init_points=10, n_iter=10)
# training-test-evaluation iterations with best params
targets = [e['target'] for e in optimizer.res]
# bs_fname = 'bs_taxiBJ.json'
# with open(os.path.join('results', bs_fname), 'w') as f:
# json.dump(optimizer.res, f, indent=2)
best_index = targets.index(max(targets))
'squeeze_scale_exp': (1.5, 1.5), # 2
'small_filter_rate': (0.5, 0.5), # 10
'max_lr_exp': (-4, -2), # 6
'max_momentum': (0.8, 0.99),
'num_epoch': (20, 50)
}
optimizer = BayesianOptimization(
f=OneCycleTrain,
pbounds=pbounds,
random_state=1,
)
try:
load_logs(optimizer, logs=["./one_cycle_baysian_logs.json"])
except:
print('no one_cycle_baysian_logs.json')
logger = JSONLogger(path="./one_cycle_baysian_logs.json")
optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
# run the optimization
optimizer.maximize(
init_points=300, # determine according to the boundary of each parameter
n_iter=8, # also determine by the boundary of each parameter
)
# access history and result
for i, res in enumerate(optimizer.res):
print("Iteration {}: \n\t{}".format(i, res))
print("Final Max:", optimizer.max)
'squeeze_scale_exp': (1.5, 1.5), # 2
'small_filter_rate': (0.5, 0.5), # 10
'lr_exp': (-4.7, -3.7) # 6
}
optimizer = BayesianOptimization(
f=SqueezeNetFunction,
pbounds=pbounds,
random_state=7,
)
try:
load_logs(optimizer, logs=["./baysian_logs.json"])
except:
print('no baysian_logs')
# subsribing the optimizing history
logger = JSONLogger(path="./baysian_logs.json")
optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
# run the optimization
optimizer.maximize(
init_points=300, # determine according to the boundary of each parameter
n_iter=8, # also determine by the boundary of each parameter
)
# access history and result
for i, res in enumerate(optimizer.res):
print("Iteration {}: \n\t{}".format(i, res))
print("Final Max:", optimizer.max)
def main():
# Parse arguments
import argparse
parser = argparse.ArgumentParser(description='Runs a bayesian optimisation for some of the algorithms defined in the PTSP framework')
parser.add_argument('--algorithm',
choices=["QD-MCTS", "S-MCTS", "MS-MCTS", "VanillaGA", "VanillaMCTS"],
help='The algorithm that should be optimized',
default="S-MCTS")
parser.add_argument('--outputDir',
default="./optimizationResults",
help='The output directory for all data generated by the optimization')
parser.add_argument("--ptspPath",
default="./ptsp.jar",
help="The path to the .jar file containing the PTSP framework")
parser.add_argument("--iters",
default="10",
type=int,
help="Number of parameter-points to test by the bayesian optimization")
args = parser.parse_args()
args.outputPath = f"{args.outputDir}/{args.algorithm}"
# Find all previous logs for this optimization
logs = glob.glob(f"{args.outputPath}/optimizationLogs*.json")
csvLogs = glob.glob(f"{args.outputPath}/*.csv")
# Launch the JVM
jpype.startJVM()
jpype.addClassPath(args.ptspPath)
import framework.Optimization as optim
# Move java output into a file
from java.lang import System
from java.io import PrintStream, FileOutputStream
pathlib.Path(args.outputPath).mkdir(parents=True, exist_ok=True)
System.setOut(PrintStream(FileOutputStream(f"{args.outputPath}/cout.txt", True)))
# Algorithm specific data
bounds = {
"QD-MCTS" : {
"lowER": (0.01, 10), # Exploration rate low-level search
"highER": (0.01, 10), # Exploration rate high-level search
"steps": (300, 600), # Number of steps for low-level search
"rd": (10, 30) # rolloutDepth
},
"S-MCTS" : {
"cellSize": (5, 30), # Size of a cell in the subgoal grid (aka distance between subgoals)
"er": (0.01, 10), # Exploration rate high-level search
"steps": (300, 600), # Number of steps for low-level search
"rd": (10, 30) # rolloutDepth
},
"MS-MCTS" : {
"cellSize": (5, 30), # Size of a cell in the subgoal grid (aka distance between subgoals)
"er": (0.01, 10), # Exploration rate high-level search
"steps": (300, 600), # Number of steps for low-level search
"rd": (10, 30) # rolloutDepth
},
"VanillaGA" : {
"gl": (10, 30), # How many base-actions does a genome contain
"ps": (1, 100), # How many genomes in one population
"mr": (0.1, 0.9), # Probability that an action is mutated
},
"VanillaMCTS" : {
"er": (0.01, 10), # Exploration rate
"rd": (10, 30), # RolloutDepth
}
}
funcs = {
"QD-MCTS" : lambda lowER, highER, steps, rd: execSafe(optim.runQD_MCTS, lowER, highER, round(steps), round(rd)),
"S-MCTS" : lambda cellSize, er, steps, rd: execSafe(optim.runSMCTS, cellSize, er, round(steps), round(rd)),
"MS-MCTS" : lambda cellSize, er, steps, rd: execSafe(optim.runMS_MCTS, cellSize, er, round(steps), round(rd)),
"VanillaGA" : lambda gl, ps, mr: execSafe(optim.runVanillaGA, round(gl), round(ps), mr),
"VanillaMCTS" : lambda er, rd: execSafe(optim.runVanillaMCTS, er, round(rd))
}
# Probe points for each algorithm, only one which I've used previously
probes = {
"QD-MCTS" : {"lowER": math.sqrt(2), "highER": 4, "steps": 400, "rd": 25},
"S-MCTS" : {"cellSize": 20, "er": 4, "steps": 400, "rd": 25},
"MS-MCTS" : {"cellSize": 20, "er": 4, "steps": 400, "rd": 25},
"VanillaGA" : {"gl": 20, "ps": 50, "mr": 1. / 20},
"VanillaMCTS" : {"er": math.sqrt(2), "rd": 12}
}
# Initialize optimization
optim.setupRun(len(logs) * 11) # Different seed for each run
optim.RUN_COUNTER = len(csvLogs) # Make sure java logs into a new csv file
optim.NUM_TRIALS = 10
optim.OutputDir = args.outputPath
optim.m_mapNames = glob.glob("./maps/**/*.map", recursive=True)
optimizer = BayesianOptimization(
f=funcs[args.algorithm],
pbounds=bounds[args.algorithm],
random_state=len(logs) * 11, # Change behaviour for each run
)
print(f"Optimizing {args.algorithm} with bounds:")
print(bounds[args.algorithm])
# Probe if necessary
init_points = 0
if len(logs) == 0:
print("Found no previous logs... Probing to improve results:")
print(probes[args.algorithm])
optimizer.probe(params=probes[args.algorithm], lazy=True)
init_points = 5
else: # If we found logs, load them
print(f"Reading previous logs into optimizer...")
load_logs(optimizer, logs=logs);
for log in logs:
print(f"Successfully loaded {log}")
logger = JSONLogger(path=f"{args.outputPath}/optimizationLogs{len(logs) + 1}.json")
optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
# Run optimization
print(f"Starting optimisation for {args.algorithm}...")
optimizer.maximize(init_points=init_points, n_iter=args.iters)
print("Finished optimisation")
print(optimizer.max)
def bayes_optimize(
data,
init_round=15, #TODO
opt_round=25, #TODO
n_folds=5,
random_seed=6,
n_estimators=10000,
learning_rate=0.05,
logging=True,
output_process=False):
print("\n\n\bayes_optimize")
print("Splitting data...")
x_train, y_train, x_val, y_val, test = Data.seperate(
data, '2016-03-27', '2016-04-24')
# prepare data
train_data = lgb.Dataset(x_train[features], y_train)
# parameters
def lgb_eval(num_leaves, feature_fraction, bagging_fraction, max_depth,
lambda_l1, lambda_l2, min_split_gain, min_child_weight):
params = {
'application': 'regression_l1',
'num_iterations': n_estimators,
'learning_rate': learning_rate,
'early_stopping_round': 100,
'metric': 'auc'
}
params["num_leaves"] = int(round(num_leaves))
params['feature_fraction'] = max(min(feature_fraction, 1), 0)
params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)
params['max_depth'] = int(round(max_depth))
params['lambda_l1'] = max(lambda_l1, 0)
params['lambda_l2'] = max(lambda_l2, 0)
params['min_split_gain'] = min_split_gain
params['min_child_weight'] = min_child_weight
cv_result = lgb.cv(params,
train_data,
nfold=n_folds,
seed=random_seed,
stratified=True,
verbose_eval=200,
metrics=['auc'])
return max(cv_result['auc-mean'])
# range
lgbBO = BayesianOptimization(lgb_eval, {
'num_leaves': (24, 4000),
'max_depth': (5, 60),
'lambda_l1': (0, 5),
'lambda_l2': (0, 3),
'feature_fraction': (0.1, 0.9),
'bagging_fraction': (0.8, 1),
'min_split_gain': (0.001, 0.1),
'min_child_weight': (5, 50)
},
random_state=0)
# Logging
if logging:
logger = JSONLogger(path="./lboLog.json")
lgbBO.subscribe(Events.OPTMIZATION_STEP, logger)
# optimize
lgbBO.maximize(init_points=init_round, n_iter=opt_round)
print(lgbBO.max)
# output optimization process
if output_process == True:
lgbBO.points_to_csv("bayes_opt_result.csv")
# return best parameters
return lgbBO.res['max']['max_params']
c = c.split(' ')
c = float(c[1])
g+=1
else:
parameters = [str(x),str(y),str(z),str(a)]
parameters = ' '.join(parameters)
print('Here X = %s'% parameters)
sock.send(parameters.encode())
sock.close()
s.listen(1)
sock, addr =s.accept()
c = sock.recv(1024).decode()
c = c.split(' ')
c =float(c[1])
print('Message from %s:%s'% addr)
print('score:%f'% c)
f.write('Score:'+str(c)+' Parameters:'+parameters+'\n')
return c
optimizer = BayesianOptimization(
f = black_box_function,
pbounds = pbounds,
verbose = 2,
random_state = 1,
)
logger = JSONLogger(path='./data/%s.json'%localtime)
optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
optimizer.maximize(init_points=20,n_iter=80)
print(optimizer.max)
cv_result = xgb.cv(
xgb_params,
dtrain,
num_boost_round=int(boost_rounds),
nfold=nfold,
stratified=True,
seed=2001,
metrics="auc",
)
# Bayesian optimization only knows how to maximize, not minimize.
gc.collect()
return cv_result["test-auc-mean"].iloc[-1]
logger2 = JSONLogger(path="./bayesopt_logs.json")
param_ranges = {
"max_depth": (1, 20),
"gamma": (0.0, 0.05),
"colsample_bytree": (0.60, 0.95),
"subsample": (0.6, 0.98),
"min_child_weight": (0.2, 2.0),
"reg_lambda": (0.10, 100.0),
"reg_alpha": (0.01, 50.0),
# 'scale_pos_weight' can be set based on the class balance in your dataset, (# 0's class / # 1's class)
# but 1.0 will often deliver max AUC and accurate pred probs
#'scale_pos_weight': (0.5, 2.0),
"eta": (0.010, 0.30),
"boost_rounds": (50, 3000),
}
def bayesian_search(
config_path,
inferencecfg,
pbounds,
edgewisecondition=True,
shuffle=1,
trainingsetindex=0,
modelprefix="",
snapshotindex=-1,
target="rpck_test",
maximize=True,
init_points=20,
n_iter=50,
acq="ei",
log_file=None,
dcorr=5,
leastbpts=3,
printingintermediatevalues=True,
): #
if "rpck" in target:
assert maximize == True
if "rmse" in target:
assert maximize == False
cfg = auxiliaryfunctions.read_config(config_path)
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.GetEvaluationFolder(
cfg["TrainingFraction"][int(trainingsetindex)],
shuffle,
cfg,
modelprefix=modelprefix,
)),
)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName(
cfg,
shuffle,
cfg["TrainingFraction"][int(trainingsetindex)],
cfg["iteration"],
modelprefix=modelprefix,
)
# load params
fns = return_evaluate_network_data(
config_path,
shuffle=shuffle,
trainingsetindex=trainingsetindex,
modelprefix=modelprefix,
)
predictionsfn = fns[snapshotindex]
data, metadata = auxfun_multianimal.LoadFullMultiAnimalData(predictionsfn)
params = set_up_evaluation(data)
columns = ["train_iter", "train_frac", "shuffle"]
columns += [
"_".join((b, a)) for a in ("train", "test")
for b in ("rmse", "hits", "misses", "falsepos", "ndetects", "pck",
"rpck")
]
train_iter = trainingsetindex # int(predictionsfn.split('-')[-1].split('.')[0])
train_frac = cfg["TrainingFraction"][
train_iter] # int(predictionsfn.split('trainset')[1].split('shuffle')[0])
trainIndices = metadata["data"]["trainIndices"]
testIndices = metadata["data"]["testIndices"]
if edgewisecondition:
mf = str(
auxiliaryfunctions.GetModelFolder(
cfg["TrainingFraction"][int(trainingsetindex)],
shuffle,
cfg,
modelprefix=modelprefix,
))
modelfolder = os.path.join(cfg["project_path"], mf)
path_inferencebounds_config = (Path(modelfolder) / "test" /
"inferencebounds.yaml")
try:
inferenceboundscfg = auxiliaryfunctions.read_plainconfig(
path_inferencebounds_config)
except FileNotFoundError:
print("Computing distances...")
from deeplabcut.pose_estimation_tensorflow import calculatepafdistancebounds
inferenceboundscfg = calculatepafdistancebounds(
config_path, shuffle, trainingsetindex)
auxiliaryfunctions.write_plainconfig(path_inferencebounds_config,
inferenceboundscfg)
partaffinityfield_graph = params["paf_graph"]
upperbound = np.array([
float(inferenceboundscfg[str(edge[0]) + "_" +
str(edge[1])]["intra_max"])
for edge in partaffinityfield_graph
])
lowerbound = np.array([
float(inferenceboundscfg[str(edge[0]) + "_" +
str(edge[1])]["intra_min"])
for edge in partaffinityfield_graph
])
upperbound *= inferencecfg["upperbound_factor"]
lowerbound *= inferencecfg["lowerbound_factor"]
else:
lowerbound = None
upperbound = None
def dlc_hyperparams(**kwargs):
inferencecfg.update(kwargs)
# Ensure type consistency
for k, (bound, _) in pbounds.items():
inferencecfg[k] = type(bound)(inferencecfg[k])
stats = compute_crossval_metrics_preloadeddata(
params,
columns,
inferencecfg,
data,
trainIndices,
testIndices,
train_iter,
train_frac,
shuffle,
lowerbound,
upperbound,
dcorr=dcorr,
leastbpts=leastbpts,
)
# stats = compute_crossval_metrics(config_path, inferencecfg, shuffle,trainingsetindex,
# dcorr=dcorr,leastbpts=leastbpts,modelprefix=modelprefix)
if printingintermediatevalues:
print(
"rpck",
stats["rpck_test"].values[0],
"rpck train:",
stats["rpck_train"].values[0],
)
print(
"rmse",
stats["rmse_test"].values[0],
"miss",
stats["misses_test"].values[0],
"hit",
stats["hits_test"].values[0],
)
# val = stats['rmse_test'].values[0]*(1+stats['misses_test'].values[0]*1./stats['hits_test'].values[0])
val = stats[target].values[0]
if np.isnan(val):
if maximize: # pck case
val = -1e9 # random small number
else: # RMSE, return a large RMSE
val = 1e9
if not maximize:
val = -val
return val
opt = BayesianOptimization(f=dlc_hyperparams,
pbounds=pbounds,
random_state=42)
if log_file:
load_logs(opt, log_file)
logger = JSONLogger(path=os.path.join(evaluationfolder, "opti_log" +
DLCscorer + ".json"))
opt.subscribe(Events.OPTIMIZATION_STEP, logger)
opt.maximize(init_points=init_points, n_iter=n_iter, acq=acq)
inferencecfg.update(opt.max["params"])
for k, (bound, _) in pbounds.items():
tmp = type(bound)(inferencecfg[k])
if isinstance(tmp, np.floating):
tmp = np.round(tmp, 2).item()
inferencecfg[k] = tmp
return inferencecfg, opt
gs = RegionParaSearch(region=region2search)
gs.region_data_load()
# Start timing the code
start_time = time.time()
# Bounded region of parameter space
pbounds = {
'p': (0.01, 0.99),
'gamma': (0.01, 0.99),
'beta': (0.01, 0.99)
}
optimizer = BayesianOptimization(
f=gs.gs_para,
pbounds=pbounds,
random_state=98,
)
logger = JSONLogger(path=ROOT_dir + "/results/para-search-r1/logs_" +
gs.region + ".json")
optimizer.subscribe(Events.OPTIMIZATION_STEP, logger)
optimizer.maximize(
init_points=8,
n_iter=50,
)
print(optimizer.max)
print(
region2search, "is done. Elapsed time was %g seconds" %
(time.time() - start_time))
def __init__(self, path):
self.json = JSONLogger(path)
self.console = ScreenLogger()
'eta': (0.001, 1),
'max_depth': (4, 15),
'gamma': (0.001, 1.0),
'min_child_weight': (10, 400),
'max_delta_step': (0, 20),
'subsample': (0.1, 1.0),
'colsample_bytree': (0.1, 1.0),
'alpha': (0.001, 0.5)
},
random_state=2019)
# define optimization log
# everytime it probes the function and obtains a new parameter-target combination
# it will fire an `Events.OPTIMIZATION_STEP` event, which our logger will listen to.
bo_logname = BO_LOGNAME
bo_logger = JSONLogger(path=bo_logname)
xgb_bo.subscribe(Events.OPTMIZATION_STEP, bo_logger)
# explore
bo_probe(xgb_bo)
# search
print("start search", flush=True)
bo_init_points = BO_INIT_POINTS
bo_n_iter = BO_N_ITER
# init_points: How many steps of random exploraton you want to perform. at begining
# n_iter: How many steps of bayesian optimization you want to perform.
xgb_bo.maximize(init_points=bo_init_points, n_iter=bo_n_iter, acq='ei')
# done search and train
print("best parameters and target value:")