def main(argv=None): #using the chocolate to tune space = create_space() conn = choco.SQLiteConnection(url="sqlite:///db2.db") cv = choco.Repeat(repetitions=3, reduce=np.mean, rep_col="_repetition_id") sampler = choco.Bayes(conn, space, crossvalidation=cv) #train(x_train, y_train, vocab_processor, x_dev, y_dev) token, params = sampler.next() print(type(token)) print(token) x_train, y_train, x_dev, y_dev = preprocess( params["eps"], params["dev_sample_percentage"]) loss = train(x_train, y_train, x_dev, y_dev, params) print(loss) sampler.update(token, loss) results = conn.results_as_dataframe() print(results) results = pd.melt(results, id_vars=["_loss"], value_name='value', var_name="variable") sns.lmplot(x="value", y="_loss", data=results, col="variable", col_wrap=3, sharex=False) plt.show()
def example_run_bayesian(): # initialisation mean = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] variances = [4, 5, 4, 5, 4, 5, 4, 5, 4, 5] x1, y1, x2, y2 = 60, 342, 726, 725 dimension = len(mean) number = 20 space = {} for x in range(dimension): space["{}".format(x)] = choco.uniform(mean[x] - variances[x],mean[x] + variances[x]) # pip install sclite3 # sclite3 TEST.db conn = choco.SQLiteConnection("sqlite:///TEST.db") conn.lock() bay = choco.Bayes(conn, space, clear_db=True) (token, point_next) = bay.next() point = format_next(point_next) all_pos = [] all_score = [] for x in range(number): loss = extract_score(x, x1, y1, x2, y2, point) bay.update(token, loss) (token, point_next) = bay.next() point = format_next(point_next) print("\rProgress : {}%".format(100*x//number), end="") all_pos.append(point) all_score.append(1-loss) np.savetxt("Score_list", all_score) np.savetxt("Point_list", all_pos) return True
def readTune(): # Establish a connection to a SQLite local database conn = choco.SQLiteConnection("sqlite:///hpTuning.db") results = conn.results_as_dataframe() results = pd.melt(results, id_vars=["_loss"], value_name='value', var_name="variable") sns.lmplot(x="value", y="loss", data=results, col="variable", col_wrap=3, sharex=False) plt.show()
def __init__(self, algorithm_name, search_space): self.conn = choco.SQLiteConnection(DB_ADDRESS) self.search_space = search_space self.chocolate_optimizer = None self.create_optimizer(algorithm_name) # created_trials is the list of dicts with all created trials assignments, loss and trial name # _chocolate_id is the ID of the trial, Assignment names are encoded, _loss is the target metric, _trial_name is the Trial name # One row example: # {'_chocolate_id': 0, 'LS1scg==': 0.001, 'LS1udW0tZXBvY2hz': 1, 'LS1udW0tbGF5ZXJz': 2, "_loss": "0.97", "_trial_name": "grid-example-hsdvfdwl"} self.created_trials = [] self.recorded_trials_names = []
def init_choco_sampler(args): conn = choco.SQLiteConnection(args['sqlite_dbase']) space = { "lr": choco.log(low=-5, high=-3, base=10), "lr_decay": choco.uniform(high=1, low=0), "reg_scale": choco.uniform(low=0, high=1), "last_reg_scale": choco.uniform(low=0, high=1), "weight": choco.uniform(low=1, high=50), "weight_decay": choco.uniform(low=0, high=1), "contrast": choco.uniform(low=-100, high=100) } sampler = choco.CMAES(conn, space) return (sampler, conn)
def main(): X, y = load_boston(return_X_y=True) # Connect to sqlite database in current directory conn = choco.SQLiteConnection(url="sqlite:///gbt-boston.db") s = {"learning_rate": choco.uniform(0.001, 0.1), "n_estimators": choco.quantized_uniform(25, 525, 1), "max_depth": choco.quantized_uniform(2, 25, 1), "subsample": choco.uniform(0.7, 1.0)} sampler = choco.QuasiRandom(conn, s, random_state=110, skip=3) token, params = sampler.next() loss = score_gbt(X, y, params) sampler.update(token, loss)
def main(): conn = choco.SQLiteConnection("sqlite:///my_db.db") # results = conn.results_as_dataframe() space = {"x": choco.uniform(-6, 6), "y": choco.uniform(-6, 6)} # Refer to https://chocolate.readthedocs.io/tutorials/algo.html sampler = choco.QuasiRandom(conn, space, clear_db=True) #sampler = choco.MOCMAES(conn, space, mu=0.1, clear_db=True) # Token: {'_chocolate_id': 0} # Params: {'y': 1.4641226269602674, 'x': 2.5223111999723393} token, params = sampler.next() loss = himmelblau(**params) sampler.update(token, loss) print("Token: {}, loss: {}".format(token, loss)) """
def func(data): trn_x, trn_y, tst_x, tst_y, dbid = data conn = choco.SQLiteConnection(url="sqlite:///hpo/hpo_%s.db" % str(dbid)) sampler = choco.Random(conn, space) searcher = choco.Bayes(conn, space) print('START %s' % dbid) for _ in range(nseed): token, params = sampler.next() # print('START % 4d %s' % (i, params['model'])) loss = f1_score_model(trn_x, trn_y, tst_x, tst_y, **params) sampler.update(token, loss) # print('DONE % 4d %s' % (i, params['model'])) for _ in range(nruns): token, params = searcher.next() # print('START % 4d %s' % (i, params['model'])) loss = f1_score_model(trn_x, trn_y, tst_x, tst_y, **params) searcher.update(token, loss)
def run_chocolate(X_train, X_test, Y_train, Y_test, y_scaler_nl,run,space,iterations,clear_db): # CREATE and Connect to sqlite database in current directory conn = choco.SQLiteConnection(url="sqlite:///"+run+".db") #repeat each model run three times and take average #cv = choco.Repeat(repetitions=3, reduce=np.mean, rep_col="_repetition_id") #search strategy - Bayes attempts to "learn" patterns from ALL previous runs sampler = choco.Bayes(conn, space, clear_db=False)#, crossvalidation=cv) #sampler = choco.Grid(conn, space, clear_db=clear_db) #lets run 10 times and see what if we get a better answer for i in range(0,iterations): #examine db and pick next experiment token, params = sampler.next() #run experiment loss = _score(X_train, X_test, Y_train, Y_test, y_scaler_nl, params) #print("finished iteration",str(i),"loss",str(loss)) #add new result to database sampler.update(token, loss)
def best(self, tune_db=None, connection=None): """Get (current) best set of hyper-parameters Parameters ---------- connection : chocolate.SQLiteConnection, optional Existing connection to SQLite database. tune_db : str, optional Path to SQLite database where trial results will be stored. Has no effect when `connection` is provided. At least one of `tune_db` or `connection` must be provided. Returns ------- status : dict ['loss'] (`float`) best loss so far ['params'] (`dict`) corresponding set of hyper-parameters ['n_trials'] (`int`) total number of trials """ if connection is None: # start connection to SQLite database # (this is where trials are stored) connection = chocolate.SQLiteConnection(f'sqlite:///{tune_db}') # get current best set of hyper-parameter (and its loss) trials = connection.results_as_dataframe() best_params = dict(trials.iloc[trials['_loss'].idxmin()]) best_loss = best_params.pop('_loss') best_params = { name: np.asscalar(value) for name, value in best_params.items() } return { 'loss': best_loss, 'params': best_params, 'n_trials': len(trials) }
def getSuggestions(self, search_space, trials, request_number): """ Get the new suggested trials with chocolate algorithm. """ # Example: {"x" : choco.uniform(-6, 6), "y" : choco.uniform(-6, 6)} chocolate_search_space = {} for param in search_space.params: key = BaseChocolateService.encode(param.name) if param.type == INTEGER: chocolate_search_space[key] = choco.quantized_uniform( int(param.min), int(param.max), 1) elif param.type == DOUBLE: chocolate_search_space[key] = choco.quantized_uniform( float(param.min), float(param.max), float(param.step)) elif param.type == CATEGORICAL: chocolate_search_space[key] = choco.choice(param.list) else: chocolate_search_space[key] = choco.choice( [float(e) for e in param.list]) conn = choco.SQLiteConnection("sqlite:///my_db.db") # Refer to https://chocolate.readthedocs.io/tutorials/algo.html if self.algorithm_name == "grid": sampler = choco.Grid(conn, chocolate_search_space, clear_db=True) # hyperopt-random is the default option in katib. elif self.algorithm_name == "chocolate-random": sampler = choco.Random(conn, chocolate_search_space, clear_db=True) elif self.algorithm_name == "chocolate-quasirandom": sampler = choco.QuasiRandom(conn, chocolate_search_space, clear_db=True) elif self.algorithm_name == "chocolate-bayesian-optimization": sampler = choco.Bayes(conn, chocolate_search_space, clear_db=True) # elif self.algorithm_name == "chocolate-CMAES": # sampler = choco.CMAES(conn, chocolate_search_space, clear_db=True) elif self.algorithm_name == "chocolate-MOCMAES": mu = 1 sampler = choco.MOCMAES(conn, chocolate_search_space, mu=mu, clear_db=True) else: raise Exception('"Failed to create the algortihm: {}'.format( self.algorithm_name)) for index, trial in enumerate(trials): loss_for_choco = float(trial.target_metric.value) if search_space.goal == MAX_GOAL: loss_for_choco = -1 * loss_for_choco entry = {"_chocolate_id": index, "_loss": loss_for_choco} for param in search_space.params: param_assignment = None for assignment in trial.assignments: if param.name == assignment.name: param_assignment = assignment.value break if param.type == INTEGER: param_assignment = int(param_assignment) elif param.type == DOUBLE: param_assignment = float(param_assignment) entry.update({ BaseChocolateService.encode(param.name): param_assignment }) logger.info(entry) # Should not use sampler.update(token, loss), because we will create # a new BaseChocolateService instance for every request. Thus we need # to insert all previous trials every time. conn.insert_result(entry) list_of_assignments = [] for i in range(request_number): try: token, chocolate_params = sampler.next() list_of_assignments.append( BaseChocolateService.convert(search_space, chocolate_params)) except StopIteration: logger.info( "Chocolate db is exhausted, increase Search Space or decrease maxTrialCount!" ) return list_of_assignments
def tune_iter(self, tune_db, protocol, subset='development', sampler=None): """Tune pipeline forever Parameters ---------- tune_db : str Path to SQLite database where trial results will be stored. protocol : pyannote.database.Protocol Protocol on which to tune the pipeline. subset : {'train', 'development', 'test'}, optional Subset on which to tune the pipeline. Defaults to 'development'. sampler : chocolate sampler, optional Defaults to chocolate.CMAES Yields ------ status : dict ['latest']['loss'] (`float`) loss obtained by the latest trial ['latest']['params'] (`dict`) corresponding set of hyper-parameters ['latest']['n_trials'] (`int`) total number of trials in thes session ['new_best']['loss'] (`float`) best loss so far ['new_best']['params'] (`dict`) corresponding set of hyper-parameters ['new_best']['n_trials'] (`int`) total number of trials """ # start connection to SQLite database # (this is where trials are stored) connection = chocolate.SQLiteConnection(f'sqlite:///{tune_db}') # get hyper-parameter space space = self.get_tune_space() # instantiate sampler if sampler is None: sampler = chocolate.CMAES sampler = sampler(connection, space) # TODO add option to use another sampler i = 0 best = {'loss': np.inf} while True: i += 1 # get next set of hyper-parameters to try token, params = sampler.next() # instantiate pipeline with this set of parameters # and compute the objective function loss = self.with_params(**params).objective(protocol, subset=subset, learning=True) latest = {'loss': loss, 'params': params, 'n_trials': i} # tell the sampler what was the result sampler.update(token, loss) if loss < best['loss'] or i == 1: # if loss is better than previous known best # check in the database what is the current best best = self.best(connection=connection) yield {'latest': latest, 'new_best': best} else: yield {'latest': latest}
def get_new_suggestions(self, study, trials=[], number=1): """ Get the new suggested trials with Chocolate algorithm. """ # 1. Construct search space # Example: {"x" : choco.uniform(-6, 6), "y" : choco.uniform(-6, 6)} chocolate_search_space = {} # study = Study.objects.get(name=study_name) study_configuration_json = json.loads(study.study_configuration) params = study_configuration_json["params"] for param in params: param_name = param["parameterName"] if param["type"] == "INTEGER": # TODO: Support int type of search space) pass elif param["type"] == "DOUBLE": chocolate_search_space[param_name] = choco.uniform( param["minValue"], param["maxValue"]) elif param["type"] == "DISCRETE" or param["type"] == "CATEGORICAL": feasible_point_list = [ value.strip() for value in param["feasiblePoints"].split(",") ] chocolate_search_space[param_name] = choco.choice( feasible_point_list) conn = choco.SQLiteConnection("sqlite:///my_db.db") # Refer to https://chocolate.readthedocs.io/tutorials/algo.html if self.algorithm_name == "Grid": sampler = choco.Grid(conn, chocolate_search_space, clear_db=True) elif self.algorithm_name == "Random": sampler = choco.Random(conn, chocolate_search_space, clear_db=True) elif self.algorithm_name == "QuasiRandom": sampler = choco.QuasiRandom(conn, chocolate_search_space, clear_db=True) elif self.algorithm_name == "Bayes": sampler = choco.Bayes(conn, chocolate_search_space, clear_db=True) elif self.algorithm_name == "CMAES": sampler = choco.CMAES(conn, chocolate_search_space, clear_db=True) elif self.algorithm_name == "MOCMAES": mu = 1 sampler = choco.MOCMAES(conn, chocolate_search_space, mu=mu, clear_db=True) # 2. Update with completed advisor trials # completed_advisor_trials = Trial.objects.filter( # study_name=study_name, status="Completed") completed_advisor_trials = [ i for i in trials if i.status == "Completed" ] for index, advisor_trial in enumerate(completed_advisor_trials): parameter_values_json = json.loads(advisor_trial.parameter_values) loss = advisor_trial.objective_value if study_configuration_json["goal"] == "MAXIMIZE": loss = -1 * loss entry = {"_chocolate_id": index, "_loss": loss} entry.update(parameter_values_json) # Should not use sampler.update(token, loss) conn.insert_result(entry) # 3. Run algorithm and construct return advisor trials return_trial_list = [] for i in range(number): # Example: {'_chocolate_id': 1} # Example: {u'hidden2': u'32', u'learning_rate': 0.07122424534644338, u'l1_normalization': 0.8402644688674471, u'optimizer': u'adam'} token, chocolate_params = sampler.next() parameter_values_json = {} for param in params: if (param["type"] == "INTEGER" or param["type"] == "DOUBLE" or param["type"] == "CATEGORICAL"): parameter_values_json[ param["parameterName"]] = chocolate_params[ param["parameterName"]] elif param["type"] == "DISCRETE": parameter_values_json[param["parameterName"]] = int( chocolate_params[param["parameterName"]]) new_advisor_trial = Trial.create(study.name, "ChocolateTrial") new_advisor_trial.parameter_values = json.dumps( parameter_values_json) # new_advisor_trial.save() return_trial_list.append(new_advisor_trial) return return_trial_list
self.validation_error = 100000.0 if __name__ == '__main__': if len(sys.argv) == 1: # original params from article model=LatentAttention(frac_train=0.99, n_z=20, batchsize=100, learning_rate=0.001, max_epochs=10, e_h1=16, e_h2=32, d_h1=32, d_h2=16, run_id=-1); model.train() print("loss={}".format(float(model.validation_error))) exit(0) # Params from optimizer search_space = { "n_z": choco.quantized_uniform(5, 100, 1), "learning_rate": choco.log(-20, -8, 2), "max_epochs": choco.quantized_uniform(5, 200, 1), "e_h1": choco.quantized_uniform(16, 256, 1), "e_h2": choco.quantized_uniform(16, 256, 1), "d_h1": choco.quantized_uniform(16, 256, 1), "d_h2": choco.quantized_uniform(16, 256, 1), } connection = choco.SQLiteConnection("sqlite:///no_labels_results.sqlite3") sampler = choco.Bayes(connection, search_space) token, sample = sampler.next() print("Parameters: {} Token: {}".format(sample, token)) run_id = token['_chocolate_id'] model = LatentAttention(0.99, batchsize=150, run_id=run_id, **sample) model.train() sampler.update(token, float(model.validation_error))
'XGBClassifier': XGBClassifier, 'RandomForestClassifier': RandomForestClassifier, 'GaussianNB': GaussianNB, 'KNeighborsClassifier': KNeighborsClassifier, } if __name__ == "__main__": datafn = 'HOUR_00024.csv' dbid = datafn.split('_')[1].split('.')[0] # dbid = datetime.datetime.now().strftime('%m%d%y%H%M%S') # dbid = 1 N_RUNS = 1024 N_PROC = 8 datafn = os.path.join(DATA_PATH, 'hour', datafn) ids_fn = os.path.join(RAW_PATH, 'd_ids_split.pickle') trn_x, trn_y, tst_x, tst_y = load_or_gen_data(datafn, ids_fn) conn = choco.SQLiteConnection(url="sqlite:///hpo/hpo_%s.db" % str(dbid)) # searcher = choco.Random(conn, space) searcher = choco.Bayes(conn, space) f = getProcFunc(conn, searcher) with mp.Pool(processes=N_PROC) as pool: pool.map(f, range(N_RUNS)) df = conn.results_as_dataframe() df.to_csv("hpo/hpo_%s.csv" % str(dbid))
def main(argv): # parse arguments args = parse_all_args() '''''' #Chocolate Code # Define the conditional search space space = { "lr": choco.uniform(low=.001, high=.1) } # Establish a connection to a SQLite local database conn = choco.SQLiteConnection("sqlite:///hpTuning.db") # Construct the optimizer sampler = choco.Bayes(conn, space) # Sample the next point token, params = sampler.next() '''''' train_set = PrototypicalDataset(args.input_path, args.train_path, n_support=args.support, n_query=args.query) dev_set = PrototypicalDataset(args.input_path, args.dev_path, apply_enhancements=False, n_support=args.support, n_query=args.query) # Use the same minibatch size to make each dataset use the same episode size train_loader = torch.utils.data.DataLoader(train_set, shuffle=True, drop_last=False, batch_size=args.mb, num_workers=0, pin_memory=True, collate_fn=protoCollate) dev_loader = torch.utils.data.DataLoader(dev_set, shuffle=True, drop_last=False, batch_size=args.mb, num_workers=0, pin_memory=True, collate_fn=protoCollate) Filter_specs = parse_filter_specs(args.filter_specs) Pre_trained_filters = None if not args.pre_trained is None: Pre_trained_filters = torch.load(args.pre_trained) model = ConvNeuralNet(args.embed_dim, args.f1, train_set.image_shape, Filter_specs=Filter_specs, Pre_trained_filters=Pre_trained_filters) if (args.checkpoint_path): state = torch.load(args.checkpoint_path) model.load_state_dict(state) print("Loaded checkpoint %s" % (args.checkpoint_path)) # torch saves the device the model was on, so we don't need to re-load to CUDA if it was saved from CUDA else: if (torch.cuda.is_available()): model = model.cuda() train_out = AggregatePerformanceRecord("train",args.out_path,dbg=args.print_reports) dev_out = AggregatePerformanceRecord("dev",args.out_path,dbg=args.print_reports) # test_out = PerformanceRecord("test",args.out_path,dbg=args.print_reports) N = len(train_set) # Calculate the loss for the sampled point (minimized) # This would be your training code loss = train(model,train_loader,dev_loader,train_out,dev_out,N,args,**params) # Add the loss to the database sampler.update(token, loss) # # Get test set performance # test_set = PrototypicalDataset(args.input_path, args.test_path, apply_enhancements=False, n_support=args.support, n_query=args.query) # test_loader = torch.utils.data.DataLoader(test_set, shuffle=True, # drop_last=False, batch_size=args.mb, num_workers=0, pin_memory=True, # collate_fn=protoCollate) # # we should make sure this fn works, but we should not run this on the actual test set even once before we are completely done training # # evaluate_test(model, test_loader, test_out, args) readTune()
def hyperparameter_job_(train_methyl_array, val_methyl_array, interest_col, n_bins, custom_loss, torque, search_strategy, total_time, delay_time, gpu, additional_command, additional_options, update, n_epochs, job, survival, optimize_time, random_state, capsule_choice, custom_capsule_file, retrain_top_job, batch_size, output_top_job_params, limited_capsule_names_file, min_capsule_len_low_bound, gsea_superset, tissue, number_sets, use_set, gene_context, select_subtypes, custom_hyperparameters, min_capsules, fit_spw, l1_l2): additional_params=dict(train_methyl_array=train_methyl_array, val_methyl_array=val_methyl_array, interest_col=interest_col, n_bins=n_bins, custom_loss=custom_loss, job=job, batch_size=batch_size, number_sets=number_sets, min_capsules=min_capsules ) if n_epochs: additional_params['n_epochs']=n_epochs if gsea_superset: additional_params['gsea_superset']=gsea_superset if l1_l2: additional_params['l1_l2']=l1_l2 if tissue: additional_params['tissue']=tissue if custom_capsule_file: additional_params['custom_capsule_file']=custom_capsule_file if output_top_job_params: retrain_top_job=True if limited_capsule_names_file: additional_params['limited_capsule_names_file']=limited_capsule_names_file if update and not (retrain_top_job and output_top_job_params): additional_params['capsule_choice']=capsule_choice select_subtypes=list(filter(None,select_subtypes)) if select_subtypes: additional_params['select_subtypes']=select_subtypes if use_set: additional_params['use_set']=use_set if gene_context: additional_params['gene_context']=gene_context if fit_spw: additional_params['fit_spw']=fit_spw else: select_subtypes=list(filter(None,select_subtypes)) if select_subtypes: additional_params['select_subtypes']=' -ss '.join(list(filter(None,select_subtypes))) additional_params['capsule_choice']=' -cc '.join(list(filter(None,capsule_choice))) if use_set: additional_params['use_set']='' if gene_context: additional_params['gene_context']='' if fit_spw: additional_params['fit_spw']='' if not survival: additional_params['gamma2']=1e-2 def score_loss(params): #job=np.random.randint(0,1000000) start_time=time.time() params['hidden_topology']=','.join([str(int(params['el{}s'.format(j)])) for j in range(params['nehl']+1)]) params['decoder_topology']=','.join([str(int(params['dl{}s'.format(j)])) for j in range(params['ndhl']+1)]) del_params=['el{}s'.format(j) for j in range(params['nehl']+1)]+['dl{}s'.format(j) for j in range(params['ndhl']+1)] del_params=set(del_params+[k for k in params if k.startswith('el') or k.startswith('dl')]) # for k in list(params.keys()): # if k.endswith('_size'): # del params[k] # print(params) # print(params['nehl'],params['ndhl']) # print(del_params) for param in del_params: del params[param] del params['nehl'], params['ndhl'] params.update(additional_params) print(params) command='{} methylcaps-model model_capsnet {} || methylcaps-model report_loss -j {}'.format('CUDA_VISIBLE_DEVICES=0' if gpu and not torque else '',' '.join(['--{} {}'.format(k,v) for k,v in params.items() if v or k=='use_set']),params['job'])#,'&' if not torque else '') if output_top_job_params and retrain_top_job: print('Top params command: ') print('{} --predict'.format(command.split('||')[0])) exit() elif output_top_job_params: print('Continuing training of random parameters, please specify retrain_top_job.') if update: val_loss = model_capsnet_(**params) else: val_loss = return_val_loss(command, torque, total_time, delay_time, job, gpu, additional_command, additional_options) end_time=time.time() if optimize_time: return val_loss, start_time-end_time else: return val_loss grid=dict(n_epochs=dict(low=10, high=50, step=10), bin_len=dict(low=500000, high=1000000, step=100000), min_capsule_len=dict(low=min_capsule_len_low_bound, high=500, step=25), primary_caps_out_len=dict(low=10, high=100, step=5), caps_out_len=dict(low=10, high=100, step=5), nehl=dict(low=10,high=300,step=10,n_layers=3), ndhl=dict(low=100,high=300,step=10,n_layers=3), learning_rate=dict(low=-5,high=-1,step=1,base=10), gamma=dict(low=-5,high=-1,step=1,base=10), gamma2=dict(low=-5,high=-1,step=1,base=10), overlap=dict(low=0., high=.5, step=.1), routing_iterations=dict(low=2, high=4, step=1)) if os.path.exists(custom_hyperparameters): from ruamel.yaml import safe_load as load with open(custom_hyperparameters) as f: new_grid = load(f) print(new_grid) for k in new_grid: for k2 in new_grid[k]: grid[k][k2]=new_grid[k][k2] n_layers=dict(encoder=grid['nehl'].pop('n_layers'),decoder=grid['ndhl'].pop('n_layers')) grid=dict(n_epochs=choco.quantized_uniform(**grid['n_epochs']), bin_len=choco.quantized_uniform(**grid['bin_len']), min_capsule_len=choco.quantized_uniform(**grid['min_capsule_len']), primary_caps_out_len=choco.quantized_uniform(**grid['primary_caps_out_len']), caps_out_len=choco.quantized_uniform(**grid['caps_out_len']), nehl={i: {'el{}s'.format(j):choco.quantized_uniform(**grid['nehl']) for j in range(i+1)} for i in range(n_layers['encoder'])}, gamma=choco.quantized_log(**grid['gamma']), ndhl={i: {'dl{}s'.format(j):choco.quantized_uniform(**grid['ndhl']) for j in range(i+1)} for i in range(n_layers['decoder'])}, learning_rate=choco.quantized_log(**grid['learning_rate']), routing_iterations=choco.quantized_uniform(**grid['routing_iterations']), overlap=choco.quantized_uniform(**grid['overlap']), gamma2=choco.quantized_log(**grid['gamma2']) ) # ADD BATCH SIZE if n_epochs: grid.pop('n_epochs') if not survival: grid.pop('gamma2') if 'genomic_binned' not in list(capsule_choice): for k in ['overlap','bin_len']: grid.pop(k) if retrain_top_job: conn=choco.SQLiteConnection('sqlite:///hyperparameter_scan.db') results=conn.results_as_dataframe() results=results[~results['_loss'].isnull()] params=dict(results.iloc[np.argmin(results['_loss'].values)]) for k in ['bin_len','caps_out_len','min_capsule_len','ndhl','nehl','primary_caps_out_len','routing_iterations']: if k in params: params[k]=int(params[k]) del params['_loss'] top_loss=score_loss(params) pickle.dump(top_loss,open('top_loss.pkl','wb')) else: optimization_method = search_strategy#'bayes' optimization_methods=['random','quasi','bayes'] sampler_opts={} if optimization_method in ['random']: sampler_opts['n_bootstrap']=10000 #sampler_opts['random_state']=random_state elif optimization_method in ['quasi']: sampler_opts['seed']=random_state sampler_opts['skip']=3 elif optimization_method in ['bayes']: sampler_opts['n_bootstrap']=35 sampler_opts['utility_function']='ei' sampler_opts['xi']=0.1 #sampler_opts['random_state']=42 #print(optimization_method) optimizer = dict(random=choco.Bayes,quasi=choco.QuasiRandom,bayes=choco.Bayes)[optimization_method] # Random hyp_conn = choco.SQLiteConnection(url="sqlite:///hyperparameter_scan.db") sampler = optimizer(hyp_conn, grid, **sampler_opts) #print(sampler) if 0 and optimization_method in ['bayes']: sampler.random_state=np.random.RandomState(42) token,params=sampler.next() loss=score_loss(params) if (loss if not optimize_time else loss[0])>=0: sampler.update(token, loss)
file_name = "../validate_time/params_data_tpe/" + path + "/logs.txt" data_file_name = "../validate_time/params_data_tpe/" + path + "/data.txt" plot_data_path = "../validate_time/params_data_tpe/" + path + "/plot_data.csv" data_dict_file = "./data_dict/" + path + "/t_tpe.csv" t_main_3(data_manager, file_name, data_file_name, plot_data_path, data_dict_file) print("第" + str(i) + "次实验:" + "加载第" + str(j) + "个数据集***TPE***方法结束") # CMAES 方法 print("第" + str(i) + "次实验:" + "加载第" + str(j) + "个数据集***CMAES***方法开始") n = 1 os.mkdir("../validate_time/params_data_cmaes/" + str(i) + "_cmaes_" + str(j)) path = str(i) + "_cmaes_" + str(j) file_name = "../validate_time/params_data_cmaes/" + path + "/data.csv" url_path = "sqlite:///" + "../validate_time/params_data_cmaes/" + path + "/mnistdb.db" conn = choco.SQLiteConnection(url=url_path) CMAES(data_manager, n, file_name, conn) print("第" + str(i) + "次实验:" + "加载第" + str(j) + "个数据集***CMAES***方法结束") # # agent加引导 手写数字 # os.mkdir("../validate_time/params_data_agent(chen)/digits_bp_" + str(i)) # os.mkdir("./data_dict/digits_bp_" + str(i)) # path = "digits_bp_" + str(i) # plot_time_reward = "../validate_time/params_data_agent(chen)/" + path + "/plot_time_data.csv" # t_main_bp(data_manager_digits, plot_time_reward) # # agent加预测 手写数字 # os.mkdir("../validate_time/params_data_agent(chen)/digits_pre_" + str(i)) # os.mkdir("./data_dict/digits_pre_" + str(i)) # path = "digits_pre_" + str(i) # plot_time_reward = "../validate_time/params_data_agent(chen)/" + path + "/plot_time_data.csv" # t_main_pre(data_manager_digits, plot_time_reward) # 基于贝叶斯优化 手写数字
"mean": choco.choice([0.]), "decision_eps": choco.choice([1.]), "theta" : choco.choice([1.]), "invert_actions" : choco.choice([False]), "lr": choco.choice([1e-3, 5e-4, 1e-4, 5e-5, 1e-5]), }, } # initialize chocolate i = 0 sampler = None while sampler is None: try: # connect database_dir = "sqlite:///roper_experiments" + "_".join(params["tunable_params"].keys()) + str(i) + ".db" conn = choco.SQLiteConnection(database_dir) sampler = choco.QuasiRandom(conn, params["hyperparam_mutations"], seed=2) print("saving results in", database_dir) except: i += 1 pass k = 1 for i in range(calculate_space_size(params)): # Get one hyperparameter configuration from the space token, next_params = sampler.next() # Combine Tunable and Fixed Hyperparameters. hyperparams = {**params, **next_params} print("params", hyperparams) # Create autoencoder model. model = Trainable(hyperparams,token['_chocolate_id'], ...) # TODO: complete parameters