class CombinedLogger(): def __init__(self, path): self.json = JSONLogger(path) self.console = ScreenLogger() def update(self, event, instance): self.json.update(event, instance) self.console.update(event, instance)
def bo_search(self): # Start timing the code start_time = time.time() # Bounded region of parameter space pbounds = {'alpha': (0.01, 0.99), 'beta': (1, 10)} optimizer = BayesianOptimization( f=self.spd2flow, pbounds=pbounds, random_state=98, ) logger = JSONLogger(path=ROOT_dir + "/results/logs_" + self.here_seg + "_" + str(self.direction) + ".json") optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) optimizer.maximize( init_points=8, n_iter=100, ) print(optimizer.max) print( self.here_seg, "is done. Elapsed time was %g seconds" % (time.time() - start_time)) return { 'alpha': optimizer.max['params']['alpha'], 'beta': optimizer.max['params']['beta'], 'mse': -optimizer.max['target'] }
def optimize(self, load_from: str = None): self.idx = 0 Popen('ulimit -n 4096', shell=True) optimizer = BayesianOptimization( f=self._black_box_function, pbounds=self.p_bounds, random_state=1, ) logger = JSONLogger(path=self.opt_filepath) optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) if load_from is not None: logfile = os.path.join(self.out_path, load_from) if Path(logfile).is_file(): logging.info('Loading logs from ' + logfile) load_logs(optimizer, logs=[logfile]) else: logging.info('Could not find a log file under {}'.format(logfile)) optimizer.maximize( init_points=self.init_points, n_iter=self.n_iter, )
def execute(self): search_log.info('-' * 63) search_log.info('Performing Bayesian search for hyperparameters:') for i in range(len(self.bayes_search.hyperparameters)): search_log.info( f' {self.bayes_search.hyperparameters[i]}: {self.bayes_search.hyperparameter_sets[i]}' ) search_log.info('-' * 63) # Create bayes-opt bounds from configuration and create an optimization object bounds = self.bayes_search.get_parameter_bounds( self.bayes_search.hyperparameters, self.bayes_search.hyperparameter_sets) optimizer = BayesianOptimization(f=self.perform_bayes_search, pbounds=bounds, random_state=1, verbose=0) # Load search observations from log files if self.args.bayes_load: log_files_list = self.get_load_log_paths() search_log.info( f'Loading Bayesian optimization observations from:') for log in log_files_list: search_log.info(f' {str(log)}') bayes_opt.util.load_logs(optimizer, logs=log_files_list) # Save search observations to log file if self.args.bayes_save: bayes_log_path = self.get_save_log_path() search_log.info( f'Saving Bayesian optimization observations to \'{bayes_log_path}\'' ) bayes_logger = JSONLogger(path=str(bayes_log_path)) optimizer.subscribe(Events.OPTIMIZATION_STEP, bayes_logger) # Perform Bayesian searches optimizer.maximize(init_points=self.args.bayesian[0], n_iter=self.args.bayesian[1]) optimizer_max = self.get_optimizer_max(optimizer) search_log.info('-' * 63) search_log.info('Bayesian search complete') search_log.info(f'Best Configuration ({optimizer_max["target"]}):') best_configuration = self.bayes_search.get_search_config_from_bounds( optimizer_max["params"]) for key, value in best_configuration.items(): search_log.info(f' {key}: {value}') search_log.info('-' * 63) self.save_max_to_file(optimizer_max) search_log.info('-' * 63) if self.search_config_path.exists(): self.search_config_path.unlink()
def optimize(self, reset=False): # Bounded region of parameter space pbounds = { 'men_kappa_fulltime': (0.3, 0.6), 'men_kappa_parttime': (0.3, 0.7), #'men_mu_scale': (0.01,0.3), 'men_mu_age': (57,62), 'women_kappa_fulltime': (0.3, 0.6), 'women_kappa_parttime': (0.3, 0.7) } #, 'women_mu_scale': (0.01,0.3), 'women_mu_age': (57,62)} optimizer = BayesianOptimization( f=self.black_box_function, pbounds=pbounds, verbose= 2, # verbose = 1 prints only when a maximum is observed, verbose = 0 is silent random_state=1, ) LOG_DIR = Path().absolute() / 'bayes_opt_logs' LOG_DIR.mkdir(exist_ok=True) filename = 'log_0.json' # talletus logfile = str(LOG_DIR / filename) logger = JSONLogger(path=logfile) if Path(logfile).exists() and not reset: load_logs(optimizer, logs=[logfile]) optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) optimizer.maximize( init_points=2, n_iter=20, ) print('The best parameters found {}'.format(optimizer.max))
def solve(self, init_points=15, n_iter=5000): logger = JSONLogger(path="ABC_Results/user_parameters_p{}.json".format( self.participant)) self.optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) self.optimizer.maximize( init_points=init_points, n_iter=n_iter, ) print(self.optimizer.max)
def optimize(): # define bounds for the params you want to optimize. Can be multivariate. Check https://github.com/fmfn/BayesianOptimization on how to bounds = { 'localAreaDensity': (0.01, 0.15), 'permanenceIncrement': (0.01, 0.1), } optimizer = BayesianOptimization( f=target_func, pbounds=bounds, random_state=1, ) # We can start from saved logs if os.path.isfile('./local_area_density_optimization_logs_base.json'): print('Loading Logs...') load_logs(optimizer, logs=["./local_area_density_optimization_logs_base.json"]); # The new log file to write to json_logger = JSONLogger(path="./local_area_density_optimization_logs.json") optimizer.subscribe(Events.OPTIMIZATION_STEP, json_logger) # Additionally log to console screen_logger = ScreenLogger() optimizer.subscribe(Events.OPTIMIZATION_STEP, screen_logger) # If you want to guide the optimization process val = 0.02 while val <= 0.04: optimizer.probe( params={ 'localAreaDensity': val, 'permanenceIncrement': 0.04, }, lazy=True, ) val = round(val + 0.001, 3) optimizer.maximize( init_points=20, n_iter=50, ) print(optimizer.max) # cleanup temp dir shutil.rmtree(os.path.join('temp'))
def hyper(): opt = BayesianOptimization(f=train, pbounds={ 'lr': (0.0001, 0.001), 'lm': (0.75, 0.95), 'tpri': (10, 150), 'rpr': (0.25, 0.75), 'dmc': (0.6, 0.9), 'wd': (0.00001, 0.001) }, verbose=2) logger = JSONLogger(path="./bo_logs.json") opt.subscribe(Events.OPTIMIZATION_STEP, logger) opt.maximize(init_points=3, n_iter=10) load_logs(opt, logs=["./bo_logs.json"]) print('maximum: ', opt.max)
def optimize(self, path, filename, bound_trafo): assert (self.logger and (path is not None) and (filename is not None), 'Filename and path are necessary when the logger is active') pbounds = self.create_iterables() if bound_trafo: bounds_transformer = SequentialDomainReductionTransformer() optimizer = BayesianOptimization( f=self.objective, pbounds=pbounds, verbose=self.verbose, random_state=self.seed, bounds_transformer=bounds_transformer, ) else: optimizer = BayesianOptimization( f=self.objective, pbounds=pbounds, verbose=self.verbose, random_state=self.seed, ) if self.logger: logger = JSONLogger(path=path + filename + '.txt') optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) optimizer.maximize(init_points=self.initial, n_iter=self.iterations) # write bound-trajectories in separate log-file, # if bounds have been transformed if bound_trafo: for i, p in enumerate(pbounds.keys()): bounds = {p: {}} bounds[p]['min'] = [b[i][0] for b in bounds_transformer.bounds] bounds[p]['max'] = [b[i][1] for b in bounds_transformer.bounds] with open(path + filename + '_bounds.txt', "a") as f: f.write(json.dumps(bounds))
def optimize_local_area_density(): # optimize localAreaDensity bounds = { 'localAreaDensity': (0.01, 0.15), } optimizer = BayesianOptimization( f=target_func, pbounds=bounds, random_state=1, ) if os.path.isfile('./local_area_density_optimization_logs_base.json'): print('Loading Logs...') load_logs(optimizer, logs=["./local_area_density_optimization_logs_base.json"]) logger = JSONLogger(path="./local_area_density_optimization_logs.json") optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) val = 0.02 while val <= 0.04: print('Adding', val) optimizer.probe( params={ 'localAreaDensity': val, }, lazy=True, ) val = round(val + 0.001, 3) print('Starting optimization...') optimizer.maximize( init_points=20, n_iter=50, ) print(optimizer.max)
batch = int(batch) epochs = int(epochs) return Bayes_Unet(batch, epochs, lr, weight_decay, lr_multiplier) # Bounded region of parameter space pbounds = {'batch': (4, 16), 'epochs': (60, 140), 'lr': (1e-6, 1e-2), 'weight_decay': (1e-8, 1e-3), 'lr_multiplier': (0.1, 1)} hyperparams = BayesianOptimization( f=function_to_be_optimized, pbounds=pbounds, verbose=2, random_state=1, ) logger = JSONLogger(path="/content/drive/MyDrive/UNET/Training models/logs.json") hyperparams.subscribe(Events.OPTIMIZATION_STEP, logger) hyperparams.maximize( init_points=3, # 3 random trials n_iter=30 # 30 Bayesian steps ) """## **Testing + visualization** """ import numpy as np import matplotlib.pyplot as plt from IPython.core.debugger import set_trace import torch import torch.nn as nn
'cutoff': (0, 0.2), 'instates': (10, 26), 'prestates': (10, 26), 'C': (0, 50) } optimizer = BayesianOptimization( f=config.lpcfg_optimize.opt, pbounds=pbounds, verbose=2, random_state=42, ) # This is dangerous as it deletes the original file if you have one # waiting for the next update that fixes this logger = JSONLogger(path="./logs.json") optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) # from bayes_opt.util import load_logs # load_logs(optimizer, logs=["./logs.json"]) optimizer.probe(params={ 'cutoff': 0.01, 'instates': 16, 'prestates': 16, 'C': 10 }) optimizer.maximize( init_points=5, n_iter=500,
verbose=2, random_state=1, lazy_gpr=True, lag=10000) else: opt = BayesianOptimization(f=black_box_function, pbounds=pbounds, verbose=2, random_state=1, lazy_gpr=False) if (args.restart): print('restart') load_logs(opt, logs=[args.log_gpr]) def_logger = JSONLogger(path=logfile) opt.subscribe(Events.OPTMIZATION_STEP, def_logger) #print(args.iter) opt.maximize( init_points=args.seed, n_iter=args.iter, samples=None, eps=0.035, # to reach an accuracy of 0.965 solution=1, acq="ei", xi=0.01) end = time.time() passed = end - start print('target : ', opt.max['target'], ', lazy_gpr: ', lazy_gpr) print('Time(sec.) : ' + str(passed) + "\n") lazy_gpr = True
'ridge_alpha': 0.0006696012127733874 } ] optimizer = BayesianOptimization( f=black_box_function, pbounds=pbounds, random_state=42, ) for point in start_points: optimizer.probe( params=point, lazy=True, ) if LOAD_PREV: load_logs(optimizer, logs=["./calc_shifts_params.json"]); logger = JSONLogger(path="./calc_shifts_params.json") scrlogger = ScreenLogger() optimizer.subscribe(Events.OPTIMIZATION_STEP, scrlogger) optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) optimizer.maximize( init_points=500, n_iter=500, ) print(optimizer.max)
'freeze_base_model': False, 'contrastive_learning_rate': lr, } PARAMS[ 'contrastive_model_path'] = f"/home/eyal/privet_dev/cov3_sagi/contrastive_models/contrastive_{PARAMS['contrastive_epochs']}.h5" params = AttrDict(PARAMS) print(params) accuracy = covid_with_neptune(params) return accuracy # Bounded region of parameter space pbounds = {'contrastive_epochs': (2, 10), 'lr': (0.0000001, 0.0001)} optimizer = BayesianOptimization( f=black_box_covid, pbounds=pbounds, random_state=1, ) logger = JSONLogger(path="./logs_opt1.json") optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) optimizer.maximize( init_points=10, n_iter=200, ) print(f'best params so far: {optimizer.max}')
sum = 0.0 for i in trange(1, MAX0): for j in trange(1, MAX1): name = str(i) + '_' + str(j) image_x = np.load( os.getcwd() + "/encodedArray/bit_{0}/{1}.npy".format(STEN_X, name)) image_y = np.load( os.getcwd() + "/decodedArray/bit_{0}/{1}.npy".format(STEN_X, name)) sum += mse( generator_g.predict( np.asarray([image_x / 255.0], dtype='float32')), np.asarray([image_y / 255.0], dtype='float32')) avg = sum / ((MAX0 - 1) * (MAX1 - 1)) return avg bounds = { 'EPOCHS': EPOCHS_RANGE, 'LAMBDA': LAMBDA_RANGE, 'steps_per_epochs': STEPS_PER_EPOCH_RANGE } optimizer = BayesianOptimization(f=Black_Box, pbounds=bounds, random_state=1) logger = JSONLogger(path="./logs_{0}.json".format(STEN_X)) optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) optimizer.maximize(init_points=2, n_iter=10)
return bayes_opt_score # bayesian optimization optimizer = BayesianOptimization( f=train_model, pbounds={ 'encoder_blocks': (2, 2), 'lr': (0.001, 0.0001), 'batch_size': (1, 3.999), # *16 'kernel_size': (3, 4.999) }, verbose=2) bs_fname = 'bs_taxiNYC.json' logger = JSONLogger(path="./results/" + bs_fname) optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) optimizer.maximize(init_points=2, n_iter=10) # New optimizer is loaded with previously seen points # load_logs(optimizer, logs=["./results/" + bs_fname], reset = False) # optimizer.maximize(init_points=10, n_iter=10) # training-test-evaluation iterations with best params targets = [e['target'] for e in optimizer.res] # bs_fname = 'bs_taxiBJ.json' # with open(os.path.join('results', bs_fname), 'w') as f: # json.dump(optimizer.res, f, indent=2) best_index = targets.index(max(targets))
'squeeze_scale_exp': (1.5, 1.5), # 2 'small_filter_rate': (0.5, 0.5), # 10 'max_lr_exp': (-4, -2), # 6 'max_momentum': (0.8, 0.99), 'num_epoch': (20, 50) } optimizer = BayesianOptimization( f=OneCycleTrain, pbounds=pbounds, random_state=1, ) try: load_logs(optimizer, logs=["./one_cycle_baysian_logs.json"]) except: print('no one_cycle_baysian_logs.json') logger = JSONLogger(path="./one_cycle_baysian_logs.json") optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) # run the optimization optimizer.maximize( init_points=300, # determine according to the boundary of each parameter n_iter=8, # also determine by the boundary of each parameter ) # access history and result for i, res in enumerate(optimizer.res): print("Iteration {}: \n\t{}".format(i, res)) print("Final Max:", optimizer.max)
'squeeze_scale_exp': (1.5, 1.5), # 2 'small_filter_rate': (0.5, 0.5), # 10 'lr_exp': (-4.7, -3.7) # 6 } optimizer = BayesianOptimization( f=SqueezeNetFunction, pbounds=pbounds, random_state=7, ) try: load_logs(optimizer, logs=["./baysian_logs.json"]) except: print('no baysian_logs') # subsribing the optimizing history logger = JSONLogger(path="./baysian_logs.json") optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) # run the optimization optimizer.maximize( init_points=300, # determine according to the boundary of each parameter n_iter=8, # also determine by the boundary of each parameter ) # access history and result for i, res in enumerate(optimizer.res): print("Iteration {}: \n\t{}".format(i, res)) print("Final Max:", optimizer.max)
def main(): # Parse arguments import argparse parser = argparse.ArgumentParser(description='Runs a bayesian optimisation for some of the algorithms defined in the PTSP framework') parser.add_argument('--algorithm', choices=["QD-MCTS", "S-MCTS", "MS-MCTS", "VanillaGA", "VanillaMCTS"], help='The algorithm that should be optimized', default="S-MCTS") parser.add_argument('--outputDir', default="./optimizationResults", help='The output directory for all data generated by the optimization') parser.add_argument("--ptspPath", default="./ptsp.jar", help="The path to the .jar file containing the PTSP framework") parser.add_argument("--iters", default="10", type=int, help="Number of parameter-points to test by the bayesian optimization") args = parser.parse_args() args.outputPath = f"{args.outputDir}/{args.algorithm}" # Find all previous logs for this optimization logs = glob.glob(f"{args.outputPath}/optimizationLogs*.json") csvLogs = glob.glob(f"{args.outputPath}/*.csv") # Launch the JVM jpype.startJVM() jpype.addClassPath(args.ptspPath) import framework.Optimization as optim # Move java output into a file from java.lang import System from java.io import PrintStream, FileOutputStream pathlib.Path(args.outputPath).mkdir(parents=True, exist_ok=True) System.setOut(PrintStream(FileOutputStream(f"{args.outputPath}/cout.txt", True))) # Algorithm specific data bounds = { "QD-MCTS" : { "lowER": (0.01, 10), # Exploration rate low-level search "highER": (0.01, 10), # Exploration rate high-level search "steps": (300, 600), # Number of steps for low-level search "rd": (10, 30) # rolloutDepth }, "S-MCTS" : { "cellSize": (5, 30), # Size of a cell in the subgoal grid (aka distance between subgoals) "er": (0.01, 10), # Exploration rate high-level search "steps": (300, 600), # Number of steps for low-level search "rd": (10, 30) # rolloutDepth }, "MS-MCTS" : { "cellSize": (5, 30), # Size of a cell in the subgoal grid (aka distance between subgoals) "er": (0.01, 10), # Exploration rate high-level search "steps": (300, 600), # Number of steps for low-level search "rd": (10, 30) # rolloutDepth }, "VanillaGA" : { "gl": (10, 30), # How many base-actions does a genome contain "ps": (1, 100), # How many genomes in one population "mr": (0.1, 0.9), # Probability that an action is mutated }, "VanillaMCTS" : { "er": (0.01, 10), # Exploration rate "rd": (10, 30), # RolloutDepth } } funcs = { "QD-MCTS" : lambda lowER, highER, steps, rd: execSafe(optim.runQD_MCTS, lowER, highER, round(steps), round(rd)), "S-MCTS" : lambda cellSize, er, steps, rd: execSafe(optim.runSMCTS, cellSize, er, round(steps), round(rd)), "MS-MCTS" : lambda cellSize, er, steps, rd: execSafe(optim.runMS_MCTS, cellSize, er, round(steps), round(rd)), "VanillaGA" : lambda gl, ps, mr: execSafe(optim.runVanillaGA, round(gl), round(ps), mr), "VanillaMCTS" : lambda er, rd: execSafe(optim.runVanillaMCTS, er, round(rd)) } # Probe points for each algorithm, only one which I've used previously probes = { "QD-MCTS" : {"lowER": math.sqrt(2), "highER": 4, "steps": 400, "rd": 25}, "S-MCTS" : {"cellSize": 20, "er": 4, "steps": 400, "rd": 25}, "MS-MCTS" : {"cellSize": 20, "er": 4, "steps": 400, "rd": 25}, "VanillaGA" : {"gl": 20, "ps": 50, "mr": 1. / 20}, "VanillaMCTS" : {"er": math.sqrt(2), "rd": 12} } # Initialize optimization optim.setupRun(len(logs) * 11) # Different seed for each run optim.RUN_COUNTER = len(csvLogs) # Make sure java logs into a new csv file optim.NUM_TRIALS = 10 optim.OutputDir = args.outputPath optim.m_mapNames = glob.glob("./maps/**/*.map", recursive=True) optimizer = BayesianOptimization( f=funcs[args.algorithm], pbounds=bounds[args.algorithm], random_state=len(logs) * 11, # Change behaviour for each run ) print(f"Optimizing {args.algorithm} with bounds:") print(bounds[args.algorithm]) # Probe if necessary init_points = 0 if len(logs) == 0: print("Found no previous logs... Probing to improve results:") print(probes[args.algorithm]) optimizer.probe(params=probes[args.algorithm], lazy=True) init_points = 5 else: # If we found logs, load them print(f"Reading previous logs into optimizer...") load_logs(optimizer, logs=logs); for log in logs: print(f"Successfully loaded {log}") logger = JSONLogger(path=f"{args.outputPath}/optimizationLogs{len(logs) + 1}.json") optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) # Run optimization print(f"Starting optimisation for {args.algorithm}...") optimizer.maximize(init_points=init_points, n_iter=args.iters) print("Finished optimisation") print(optimizer.max)
def bayes_optimize( data, init_round=15, #TODO opt_round=25, #TODO n_folds=5, random_seed=6, n_estimators=10000, learning_rate=0.05, logging=True, output_process=False): print("\n\n\bayes_optimize") print("Splitting data...") x_train, y_train, x_val, y_val, test = Data.seperate( data, '2016-03-27', '2016-04-24') # prepare data train_data = lgb.Dataset(x_train[features], y_train) # parameters def lgb_eval(num_leaves, feature_fraction, bagging_fraction, max_depth, lambda_l1, lambda_l2, min_split_gain, min_child_weight): params = { 'application': 'regression_l1', 'num_iterations': n_estimators, 'learning_rate': learning_rate, 'early_stopping_round': 100, 'metric': 'auc' } params["num_leaves"] = int(round(num_leaves)) params['feature_fraction'] = max(min(feature_fraction, 1), 0) params['bagging_fraction'] = max(min(bagging_fraction, 1), 0) params['max_depth'] = int(round(max_depth)) params['lambda_l1'] = max(lambda_l1, 0) params['lambda_l2'] = max(lambda_l2, 0) params['min_split_gain'] = min_split_gain params['min_child_weight'] = min_child_weight cv_result = lgb.cv(params, train_data, nfold=n_folds, seed=random_seed, stratified=True, verbose_eval=200, metrics=['auc']) return max(cv_result['auc-mean']) # range lgbBO = BayesianOptimization(lgb_eval, { 'num_leaves': (24, 4000), 'max_depth': (5, 60), 'lambda_l1': (0, 5), 'lambda_l2': (0, 3), 'feature_fraction': (0.1, 0.9), 'bagging_fraction': (0.8, 1), 'min_split_gain': (0.001, 0.1), 'min_child_weight': (5, 50) }, random_state=0) # Logging if logging: logger = JSONLogger(path="./lboLog.json") lgbBO.subscribe(Events.OPTMIZATION_STEP, logger) # optimize lgbBO.maximize(init_points=init_round, n_iter=opt_round) print(lgbBO.max) # output optimization process if output_process == True: lgbBO.points_to_csv("bayes_opt_result.csv") # return best parameters return lgbBO.res['max']['max_params']
c = c.split(' ') c = float(c[1]) g+=1 else: parameters = [str(x),str(y),str(z),str(a)] parameters = ' '.join(parameters) print('Here X = %s'% parameters) sock.send(parameters.encode()) sock.close() s.listen(1) sock, addr =s.accept() c = sock.recv(1024).decode() c = c.split(' ') c =float(c[1]) print('Message from %s:%s'% addr) print('score:%f'% c) f.write('Score:'+str(c)+' Parameters:'+parameters+'\n') return c optimizer = BayesianOptimization( f = black_box_function, pbounds = pbounds, verbose = 2, random_state = 1, ) logger = JSONLogger(path='./data/%s.json'%localtime) optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) optimizer.maximize(init_points=20,n_iter=80) print(optimizer.max)
cv_result = xgb.cv( xgb_params, dtrain, num_boost_round=int(boost_rounds), nfold=nfold, stratified=True, seed=2001, metrics="auc", ) # Bayesian optimization only knows how to maximize, not minimize. gc.collect() return cv_result["test-auc-mean"].iloc[-1] logger2 = JSONLogger(path="./bayesopt_logs.json") param_ranges = { "max_depth": (1, 20), "gamma": (0.0, 0.05), "colsample_bytree": (0.60, 0.95), "subsample": (0.6, 0.98), "min_child_weight": (0.2, 2.0), "reg_lambda": (0.10, 100.0), "reg_alpha": (0.01, 50.0), # 'scale_pos_weight' can be set based on the class balance in your dataset, (# 0's class / # 1's class) # but 1.0 will often deliver max AUC and accurate pred probs #'scale_pos_weight': (0.5, 2.0), "eta": (0.010, 0.30), "boost_rounds": (50, 3000), }
def bayesian_search( config_path, inferencecfg, pbounds, edgewisecondition=True, shuffle=1, trainingsetindex=0, modelprefix="", snapshotindex=-1, target="rpck_test", maximize=True, init_points=20, n_iter=50, acq="ei", log_file=None, dcorr=5, leastbpts=3, printingintermediatevalues=True, ): # if "rpck" in target: assert maximize == True if "rmse" in target: assert maximize == False cfg = auxiliaryfunctions.read_config(config_path) evaluationfolder = os.path.join( cfg["project_path"], str( auxiliaryfunctions.GetEvaluationFolder( cfg["TrainingFraction"][int(trainingsetindex)], shuffle, cfg, modelprefix=modelprefix, )), ) DLCscorer, DLCscorerlegacy = auxiliaryfunctions.GetScorerName( cfg, shuffle, cfg["TrainingFraction"][int(trainingsetindex)], cfg["iteration"], modelprefix=modelprefix, ) # load params fns = return_evaluate_network_data( config_path, shuffle=shuffle, trainingsetindex=trainingsetindex, modelprefix=modelprefix, ) predictionsfn = fns[snapshotindex] data, metadata = auxfun_multianimal.LoadFullMultiAnimalData(predictionsfn) params = set_up_evaluation(data) columns = ["train_iter", "train_frac", "shuffle"] columns += [ "_".join((b, a)) for a in ("train", "test") for b in ("rmse", "hits", "misses", "falsepos", "ndetects", "pck", "rpck") ] train_iter = trainingsetindex # int(predictionsfn.split('-')[-1].split('.')[0]) train_frac = cfg["TrainingFraction"][ train_iter] # int(predictionsfn.split('trainset')[1].split('shuffle')[0]) trainIndices = metadata["data"]["trainIndices"] testIndices = metadata["data"]["testIndices"] if edgewisecondition: mf = str( auxiliaryfunctions.GetModelFolder( cfg["TrainingFraction"][int(trainingsetindex)], shuffle, cfg, modelprefix=modelprefix, )) modelfolder = os.path.join(cfg["project_path"], mf) path_inferencebounds_config = (Path(modelfolder) / "test" / "inferencebounds.yaml") try: inferenceboundscfg = auxiliaryfunctions.read_plainconfig( path_inferencebounds_config) except FileNotFoundError: print("Computing distances...") from deeplabcut.pose_estimation_tensorflow import calculatepafdistancebounds inferenceboundscfg = calculatepafdistancebounds( config_path, shuffle, trainingsetindex) auxiliaryfunctions.write_plainconfig(path_inferencebounds_config, inferenceboundscfg) partaffinityfield_graph = params["paf_graph"] upperbound = np.array([ float(inferenceboundscfg[str(edge[0]) + "_" + str(edge[1])]["intra_max"]) for edge in partaffinityfield_graph ]) lowerbound = np.array([ float(inferenceboundscfg[str(edge[0]) + "_" + str(edge[1])]["intra_min"]) for edge in partaffinityfield_graph ]) upperbound *= inferencecfg["upperbound_factor"] lowerbound *= inferencecfg["lowerbound_factor"] else: lowerbound = None upperbound = None def dlc_hyperparams(**kwargs): inferencecfg.update(kwargs) # Ensure type consistency for k, (bound, _) in pbounds.items(): inferencecfg[k] = type(bound)(inferencecfg[k]) stats = compute_crossval_metrics_preloadeddata( params, columns, inferencecfg, data, trainIndices, testIndices, train_iter, train_frac, shuffle, lowerbound, upperbound, dcorr=dcorr, leastbpts=leastbpts, ) # stats = compute_crossval_metrics(config_path, inferencecfg, shuffle,trainingsetindex, # dcorr=dcorr,leastbpts=leastbpts,modelprefix=modelprefix) if printingintermediatevalues: print( "rpck", stats["rpck_test"].values[0], "rpck train:", stats["rpck_train"].values[0], ) print( "rmse", stats["rmse_test"].values[0], "miss", stats["misses_test"].values[0], "hit", stats["hits_test"].values[0], ) # val = stats['rmse_test'].values[0]*(1+stats['misses_test'].values[0]*1./stats['hits_test'].values[0]) val = stats[target].values[0] if np.isnan(val): if maximize: # pck case val = -1e9 # random small number else: # RMSE, return a large RMSE val = 1e9 if not maximize: val = -val return val opt = BayesianOptimization(f=dlc_hyperparams, pbounds=pbounds, random_state=42) if log_file: load_logs(opt, log_file) logger = JSONLogger(path=os.path.join(evaluationfolder, "opti_log" + DLCscorer + ".json")) opt.subscribe(Events.OPTIMIZATION_STEP, logger) opt.maximize(init_points=init_points, n_iter=n_iter, acq=acq) inferencecfg.update(opt.max["params"]) for k, (bound, _) in pbounds.items(): tmp = type(bound)(inferencecfg[k]) if isinstance(tmp, np.floating): tmp = np.round(tmp, 2).item() inferencecfg[k] = tmp return inferencecfg, opt
gs = RegionParaSearch(region=region2search) gs.region_data_load() # Start timing the code start_time = time.time() # Bounded region of parameter space pbounds = { 'p': (0.01, 0.99), 'gamma': (0.01, 0.99), 'beta': (0.01, 0.99) } optimizer = BayesianOptimization( f=gs.gs_para, pbounds=pbounds, random_state=98, ) logger = JSONLogger(path=ROOT_dir + "/results/para-search-r1/logs_" + gs.region + ".json") optimizer.subscribe(Events.OPTIMIZATION_STEP, logger) optimizer.maximize( init_points=8, n_iter=50, ) print(optimizer.max) print( region2search, "is done. Elapsed time was %g seconds" % (time.time() - start_time))
def __init__(self, path): self.json = JSONLogger(path) self.console = ScreenLogger()
'eta': (0.001, 1), 'max_depth': (4, 15), 'gamma': (0.001, 1.0), 'min_child_weight': (10, 400), 'max_delta_step': (0, 20), 'subsample': (0.1, 1.0), 'colsample_bytree': (0.1, 1.0), 'alpha': (0.001, 0.5) }, random_state=2019) # define optimization log # everytime it probes the function and obtains a new parameter-target combination # it will fire an `Events.OPTIMIZATION_STEP` event, which our logger will listen to. bo_logname = BO_LOGNAME bo_logger = JSONLogger(path=bo_logname) xgb_bo.subscribe(Events.OPTMIZATION_STEP, bo_logger) # explore bo_probe(xgb_bo) # search print("start search", flush=True) bo_init_points = BO_INIT_POINTS bo_n_iter = BO_N_ITER # init_points: How many steps of random exploraton you want to perform. at begining # n_iter: How many steps of bayesian optimization you want to perform. xgb_bo.maximize(init_points=bo_init_points, n_iter=bo_n_iter, acq='ei') # done search and train print("best parameters and target value:")