def init2(self): Files.mkdir("../model/others") self.fpath = "../model/others/hopt_xgb.txt" if config.test: self.max_evals = 1 else: self.max_evals = 50 self.space = { 'task': 'regression', 'booster': 'gbtree', 'objective': 'reg:linear', 'eval_metric': "rmse", 'eta': hp.quniform('eta', 0.001, 0.05, 0.001), 'gamma': hp.quniform('gamma', 0.0, 1.0, 0.1), 'min_child_weight': hp.quniform('min_child_weight', 3, 7, 1), 'max_depth': hp.quniform('max_depth', 5, 30, 1), 'subsample': hp.quniform('subsample', 0.7, 1, 0.1), 'colsample_bytree': hp.quniform('colsample_bytree', 0.4, 0.7, 0.1), "seed": 12345, 'lambda': hp.loguniform("lambda", np.log(0.1), np.log(10)), 'alpha': hp.loguniform("alpha", np.log(0.1), np.log(10)), 'num_rounds': None, "silent": 1, } self.i_folds = [("B", 0)] self.output_items = [] self.output_items += ["loss", "n", "rsme"] self.output_items += ["loss{}".format(i) for i, i_fold in enumerate(self.i_folds)] self.output_items += ["n{}".format(i) for i, i_fold in enumerate(self.i_folds)] self.output_items += ["rsme{}".format(i) for i, i_fold in enumerate(self.i_folds)]
def init2(self): Files.mkdir("../model/others") self.fpath = "../model/others/hopt_keras.txt" if config.test: self.max_evals = 1 else: self.max_evals = 50 self.space = { "adadelta_eps": hp.loguniform("adadelta_eps", np.log(1e-07), np.log(1e-05)), "adadelta_lr": hp.loguniform("adadelta_lr", np.log(0.1), np.log(1.0)), "adadelta_rho_m": hp.loguniform("adadelta_rho_m", np.log(0.01), np.log(0.1)), "decay": hp.loguniform("decay", np.log(0.0001), np.log(0.1)), "dropout1": hp.quniform('dropout1', 0.1, 0.5, 0.1), "dropout2": hp.quniform('dropout2', 0.1, 0.5, 0.1), "h1": hp.quniform('h1', 50, 500, 10), # 450.0, "h2": hp.quniform('h2', 20, 250, 5), # 200.0, "nb_epochs": None, # 22, } self.i_folds = [("B", 0)] self.output_items = [] self.output_items += ["loss", "n", "rsme"] self.output_items += ["loss{}".format(i) for i, i_fold in enumerate(self.i_folds)] self.output_items += ["n{}".format(i) for i, i_fold in enumerate(self.i_folds)] self.output_items += ["rsme{}".format(i) for i, i_fold in enumerate(self.i_folds)]
def set_finetune_space(self, config_file): ''' Given the original deep net architecture, and a set of pretrained weights and biases, define the configuration space to search for fintuning parameters ''' # we know these fields won't change, so go ahead and set them as # defaults now model_params = nt.get_model_params(config_file) optim_params = nt.get_optim_params(config_file) default_finetune_model_params = {k: model_params[k] for k in ('num_hids', 'activs', 'd', 'k')} default_finetune_model_params['loss_terms'] = ['cross_entropy'] default_finetune_optim_params = {k: optim_params[k] for k in ('optim_method', 'optim_type')} # define the space of hyperparameters we wish to search_finetune_model_params = {'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]), 'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])} search_finetune_optim_params = {'learn_rate': hp.uniform('learn_rate', 0, 1), 'rho': hp.uniform('rho', 0, 1), 'num_epochs': hp.qloguniform('num_epochs', log(10), log(5e3), 1), 'batch_size': hp.quniform('batch_size', 128, 1024, 1), 'init_method': hp.choice('init_method', ['gauss', 'fan-io']), 'scale_factor': hp.uniform('scale_factor', 0, 1)} # combine the default and search parameters into a dictionary to define the # full space - this is what will be passed into the objective function finetune_model_params = self.merge_default_search( default_finetune_model_params, search_finetune_model_params) finetune_optim_params = self.merge_default_search( default_finetune_optim_params, search_finetune_optim_params) finetune_hyperspace = { 'finetune_model_params': finetune_model_params, 'finetune_optim_params': finetune_optim_params} return finetune_hyperspace
def get_linear_model(model_num): return {'model_' + model_num: 'LR', 'regularizer_lr_' + model_num: hp.choice('regularizer_lr_' + model_num,[ ('l1', hp.loguniform('l1_strength_lr_' + model_num, -5,5)), ('l2', hp.loguniform('l2_strength_lr_' + model_num, -5,5)) ]), 'converg_tol_' + model_num: hp.loguniform('converg_tol_' + model_num, -10, -1) }
def sgd_regression(name, loss=None, #default - 'hinge' penalty=None, #default - 'l2' alpha=None, #default - 0.0001 l1_ratio=None, #default - 0.15, must be within [0, 1] fit_intercept=None, #default - True n_iter=None, #default - 5 shuffle=None, #default - False random_state=None, #default - None epsilon=None, #default - 0.1 learning_rate=None, #default - 'invscaling' eta0=None, #default - 0.01 power_t=None, #default - 0.5 warm_start=False, verbose=0, ): def _name(msg): return '%s.%s_%s' % (name, 'sgd', msg) rval = scope.sklearn_SGDRegressor( loss=hp.pchoice( _name('loss'), [ (0.25, 'squared_loss'), (0.25, 'huber'), (0.25, 'epsilon_insensitive'), (0.25, 'squared_epsilon_insensitive') ] ) if loss is None else loss, penalty=hp.pchoice( _name('penalty'), [ (0.40, 'l2'), (0.35, 'l1'), (0.25, 'elasticnet') ] ) if penalty is None else penalty, alpha=hp.loguniform( _name('alpha'), np.log(1e-7), np.log(1)) if alpha is None else alpha, l1_ratio=hp.uniform( _name('l1_ratio'), 0, 1 ) if l1_ratio is None else l1_ratio, fit_intercept=hp.pchoice( _name('fit_intercept'), [ (0.8, True), (0.2, False) ]) if fit_intercept is None else fit_intercept, epsilon=hp.loguniform( _name('epsilon'), np.log(1e-7), np.log(1)) if epsilon is None else epsilon, learning_rate='invscaling' if learning_rate is None else learning_rate, eta0=hp.loguniform( _name('eta0'), np.log(1e-5), np.log(1e-1)) if eta0 is None else eta0, power_t=hp.uniform( _name('power_t'), 0, 1) if power_t is None else power_t, verbose=verbose, random_state=random_state, ) return rval
def param_space_reg_skl_svr(): return { 'task': 'reg_skl_svr', 'C': hp.loguniform("C", np.log(0.00000001), np.log(10)), 'gamma': hp.loguniform("gamma", np.log(0.001), np.log(0.1)), 'degree': hp.quniform('degree', 1, 5, 1), 'epsilon': hp.loguniform("epsilon", np.log(0.001), np.log(0.1)), 'kernel': hp.choice('kernel', ['rbf', 'poly', 'sigmoid']), "max_evals": max_evals, }
def get_hp_space(): space = (hp.qloguniform('n_hidden', log(100), log(5000), 10), hp.qloguniform('n_hidden_recurrent', log(100), log(5000), 1), hp.qloguniform('temporal_order', log(10), log(100), 10), hp.loguniform('learning_rate_RBM', log(0.0001), log(1)), hp.loguniform('learning_rate_RNN', log(0.0001), log(1)), hp.qloguniform('K', log(2), log(20), 1) # Different learninf rate for RBM and RNN # hp.choice('activation_func', ['tanh', 'sigmoid']), # hp.choice('sampling_positive', ['true', 'false']) # gibbs_sampling_step_test ??? ) return space
def test_expr_to_config(): z = hp.randint('z', 10) a = hp.choice('a', [ hp.uniform('b', -1, 1) + z, {'c': 1, 'd': hp.choice('d', [3 + hp.loguniform('c', 0, 1), 1 + hp.loguniform('e', 0, 1)]) }]) expr = as_apply((a, z)) hps = {} expr_to_config(expr, (True,), hps) for label, dct in hps.items(): print label print ' dist: %s(%s)' % ( dct['node'].name, ', '.join(map(str, [ii.eval() for ii in dct['node'].inputs()]))) if len(dct['conditions']) > 1: print ' conditions (OR):' for condseq in dct['conditions']: print ' ', ' AND '.join(map(str, condseq)) elif dct['conditions']: for condseq in dct['conditions']: print ' conditions :', ' AND '.join(map(str, condseq)) assert hps['a']['node'].name == 'randint' assert hps['b']['node'].name == 'uniform' assert hps['c']['node'].name == 'loguniform' assert hps['d']['node'].name == 'randint' assert hps['e']['node'].name == 'loguniform' assert hps['z']['node'].name == 'randint' assert set([(True, EQ('a', 0))]) == set([(True, EQ('a', 0))]) assert hps['a']['conditions'] == set([(True,)]) assert hps['b']['conditions'] == set([ (True, EQ('a', 0))]), hps['b']['conditions'] assert hps['c']['conditions'] == set([ (True, EQ('a', 1), EQ('d', 0))]) assert hps['d']['conditions'] == set([ (True, EQ('a', 1))]) assert hps['e']['conditions'] == set([ (True, EQ('a', 1), EQ('d', 1))]) assert hps['z']['conditions'] == set([ (True,), (True, EQ('a', 0))])
def set_multilayer_dropout_space(self): ''' defines a hyperspace for a "modern" neural networks: at least two layers with dropout + reLU ''' # Force at least 2 layers, cuz we're modern min_layers = 2 max_layers = 3 # sets up the neural network nnets = [None] * (max_layers - min_layers + 1) for i, num_layers in enumerate(range(min_layers, max_layers + 1)): num_hids = [None] * num_layers for j in range(num_layers): num_hids[j] = hp.qloguniform( 'num_hid_%i%i' % (i, j), log(100), log(1000), 1) nnets[i] = num_hids default_mln_model_params = { 'd': self.d, 'k': self.k, 'loss_terms': ['cross_entropy', 'dropout']} search_mln_model_params = { 'arch': hp.choice('arch', nnets), 'input_p': hp.uniform('ip', 0, 1), 'hidden_p': hp.uniform('hp', 0, 1), 'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]), 'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])} default_mln_optim_params = { 'optim_type': 'minibatch', 'optim_method': 'RMSPROP'} search_mln_optim_params = { 'learn_rate': hp.uniform('learn_rate', 0, 1), 'rho': hp.uniform('rho', 0, 1), 'num_epochs': hp.qloguniform('num_epochs', log(1e2), log(2000), 1), 'batch_size': hp.quniform('batch_size', 128, 1024, 1), 'init_method': hp.choice('init_method', ['gauss', 'fan-io']), 'scale_factor': hp.uniform('scale_factor', 0, 1)} # merge the default and search spaces mln_model_params = self.merge_default_search( default_mln_model_params, search_mln_model_params) mln_optim_params = self.merge_default_search( default_mln_optim_params, search_mln_optim_params) # define the hyperparamater space to search hyperspace = {'mln_model_params': mln_model_params, 'mln_optim_params': mln_optim_params} return hyperspace
def get_xgboost_model(model_num): return { 'model_' + model_num: 'XGBoost', 'eta_' + model_num: hp.loguniform('eta_' + model_num,-5,0), 'gamma_' + model_num: hp.uniform('gamma_' + model_num,0,10), 'max_depth_' + model_num: hp.quniform('max_depth_' + model_num, 1,30,1), 'min_child_weight_' + model_num: hp.uniform('min_child_weight_' + model_num, 0, 10), 'max_delta_step_' + model_num: hp.uniform('max_delta_step_' + model_num, 0, 10), 'num_round_' + model_num: hp.quniform('num_round_' + model_num, 1, 10, 1), 'subsample_' + model_num: 1, 'regularizer_xgb_' + model_num: hp.choice('regularizer_xgb_' + model_num,[ ('l1', hp.loguniform('l1_strength_xgb_' + model_num, -5,5)), ('l2', hp.loguniform('l2_strength_xgb_' + model_num, -5,5)) ]) }
def _svc_max_iter(name): return scope.patience_param( scope.int( hp.loguniform( name + '.max_iter', np.log(1e7), np.log(1e9))))
def ridge(name, alpha=None, #default - 1.0 normalize=None, #default - False, tol=None, #default - 0.001 solver=None, #default - 'auto' fit_intercept=None, #default - True ): def _name(msg): return '%s.%s_%s' % (name, 'sgd', msg) rval = scope.sklearn_Ridge( alpha=hp.loguniform( _name('alpha'), np.log(1e-3), np.log(1e3)) if alpha is None else alpha, normalize=hp.pchoice( _name('normalize'), [ (0.8, True), (0.2, False) ]) if normalize is None else normalize, fit_intercept=hp.pchoice( _name('fit_intercept'), [ (0.8, True), (0.2, False) ]) if fit_intercept is None else fit_intercept, tol=0.001 if tol is None else tol, solver="auto" if solver is None else solver, ) return rval
def param_space_clf_skl_lr(): return { 'task': 'clf_skl_lr', 'C': hp.loguniform("C", np.log(0.00000001), np.log(10)), 'random_state': rdmseed, "max_evals": max_evals, }
def param_space_reg_skl_lasso(): return { 'task': 'reg_skl_lasso', 'alpha': hp.loguniform("alpha", np.log(0.00000001), np.log(10)), 'random_state': rdmseed, "max_evals": max_evals, }
def set_old_space(self): ''' defines an old net from the 80s - simple sigmoid layers, nothing fancy''' min_layers = 1 max_layers = 3 # sets up the neural network nnets = [None] * (max_layers - min_layers + 1) for i, num_layers in enumerate(range(min_layers, max_layers + 1)): num_hids = [None] * num_layers for j in range(num_layers): num_hids[j] = hp.qloguniform( 'num_hid_%i%i' % (i, j), log(10), log(100), 1) nnets[i] = num_hids default_mln_model_params = { 'd': self.d, 'k': self.k, 'loss_terms': ['cross_entropy']} search_mln_model_params = { 'arch': hp.choice('arch', nnets), 'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]), 'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])} default_mln_optim_params = { 'optim_type': 'minibatch', 'optim_method': 'RMSPROP'} search_mln_optim_params = { 'learn_rate': hp.uniform('learn_rate', 0, 1), 'rho': hp.uniform('rho', 0, 1), 'num_epochs': hp.qloguniform('num_epochs', log(1e2), log(2000), 1), 'batch_size': hp.quniform('batch_size', 128, 1024, 1), 'init_method': hp.choice('init_method', ['gauss', 'fan-io']), 'scale_factor': hp.uniform('scale_factor', 0, 1)} # merge the default and search spaces mln_model_params = self.merge_default_search( default_mln_model_params, search_mln_model_params) mln_optim_params = self.merge_default_search( default_mln_optim_params, search_mln_optim_params) # define the hyperparamater space to search hyperspace = {'mln_model_params': mln_model_params, 'mln_optim_params': mln_optim_params} return hyperspace
def test_loguniform(self): prefix = "test" # For whatever reason, this returns a float param = hp.loguniform('%sp_order_real' % prefix, mu=np.log(0.1), sigma=np.log(10)) expected = '"testp_order_real": hp.loguniform("testp_order_real", lower=-2.30258509299, upper=2.30258509299),\n' rval = convert_convnet_searchspace.convert_float(param, 0) self.assertEqual(rval, expected)
def test_build_space(): from hyperopt import hp net = scope.Model() net1 = scope.mlp_layer(net, "mlp1", size=hp.quniform("mlp1_size", 3, 64, 1), dropout=hp.choice("mlp1_dropout", [True, False]), lr_fact=hp.loguniform("mlp_lrfact", np.log(0.01), np.log(10.)), with_bias=(True, hp.uniform("mlp1_bias", 0., 1.)), nonlin=hp.choice("mlp1_nonlin", "tanh rectified_linear".split(" ")), ) net2 = scope.logistic_regression(net1, n_classes=10, dropout=hp.choice("linreg_dropout", [False, True])) return scope.test_classifier(model=net2, lr=hp.loguniform("lr", np.log(0.0001), np.log(1.00)), )
def get_linear_model_params(name="linear_common"): return scope.get_lr_model( C=hp.loguniform(get_full_name(name, 'C'), -15, 15), penalty=hp.choice(get_full_name(name, 'penalty'), ('l1', 'l2')), class_weight=hp.choice(get_full_name(name, 'class_weight'), (defaultdict(lambda: 1.0), 'balanced')), fit_intercept=hp.choice(get_full_name(name, 'fit_intercept'), (False, True)), random_state=RANDOM_STATE, solver='liblinear', )
def get_hp_space(): space = (hp.qloguniform('n_hidden', log(100), log(5000), 10), hp.qloguniform('temporal_order', log(1), log(30), 1), hp.loguniform('learning_rate', log(0.0001), log(1)), hp.quniform('batch_size', 10, 500, 10) # hp.choice('activation_func', ['tanh', 'sigmoid']), # hp.choice('sampling_positive', ['true', 'false']) # gibbs_sampling_step_test ??? ) return space
def main_hp(args): hparams = { "numsteps": args.numsteps, "measure": ComplexityMeasure(), "continuous": False, } pobjective = partial(objective, hparams=hparams) print pobjective(params = np.random.uniform(-1.0, 1.0, (2,2))) def objective_hp(params): # print "params", params targ = np.array(params) # .astype(np.float32) # print targ.dtype now = time.time() func = pobjective # args["func"] ret = func(targ) # cma.fcts.griewank(targ) took = time.time() - now print "feval took %f s with ret = %s" % (took, ret["loss"]) return ret space = [hp.loguniform("m%d" % i, -5, 2.0) for i in range(4)] trials = Trials() suggest = tpe.suggest # something bests = [] initevals = 0 maxevals = 500 lrstate = np.random.RandomState(123) for i in range(initevals, maxevals): print "fmin iter %d" % i, bests.append(fmin(objective_hp, space, algo=suggest, max_evals=i+1, rstate=lrstate, # 1, 10, 123, trials=trials, verbose=1)) lrstate = np.random.RandomState() best = bests[-1] for i in range(5): print("best[%d]" % (-1-i), bests[-1-i]) # for k,v in best: pkeys = best.keys() pkeys.sort() ret = np.zeros((len(pkeys))) for i,k in enumerate(pkeys): # print k ret[i] = best[k] return ret
def create_space(n_feature): """ Function to create a search space for the hypers of the a sklearn 0.18 GP kernel. The order of the hypers is designed to work with obj_function(). Return a search space for hyperopt_fmin()""" n_sample = X_train.shape[0] size = [ X[:, i].max() - X[:, i].min() for i in range(n_feature)] constant_space = [hp.loguniform('constant_value%s'%i, np.log(10)*-2, np.log(10)*2) for i in range(n_feature)] l_scale_space = [hp.uniform('length_scale%s'%i, (size[i]/n_sample), size[i]) for i in range(n_feature)] space =[] #create an alternate search space for hyper-opt for j in range(n_feature): space.append(constant_space[j]) space.append(l_scale_space[j]) space.append(hp.loguniform('alpha', -11, 2)) space.append(hp.loguniform('ridge', -2, 2)) return space
def get_model_based_feature_selector_params( inner_model_params, name='model_based_feature_selector', estimator=None, ): if estimator is None: estimator = get_feature_selector_estimator_params() return scope.get_model_based_feature_selection_model( estimator=estimator, inner_model=inner_model_params, feature_selection_threshold_coef=hp.loguniform(get_full_name(name, "threshold"), -5, 5), )
def test_read_loguniform(self): # 0 float # 1 hyperopt_param # 2 Literal{colnorm_thresh} # 3 loguniform # 4 Literal{-20.7232658369} # 5 Literal{-6.90775527898} loguniform = hp.loguniform('colnorm_thresh', np.log(1e-9), np.log(1e-3)).inputs()[0].inputs()[1] ret = self.pyll_reader.read_loguniform(loguniform, 'colnorm_thresh') expected = configuration_space.UniformFloatHyperparameter( 'colnorm_thresh', 1e-9, 1e-3, base=np.e) self.assertEqual(expected, ret)
def _svm_gamma(name, n_features=1): '''Generator of default gamma values for SVMs. This setting is based on the following rationales: 1. The gamma hyperparameter is basically an amplifier for the original dot product or l2 norm. 2. The original dot product or l2 norm shall be normalized by the number of features first. ''' # -- making these non-conditional variables # probably helps the GP algorithm generalize # assert n_features >= 1 return hp.loguniform(name, np.log(1. / n_features * 1e-3), np.log(1. / n_features * 1e3))
def add_pseudo_options(): space['psuedo-documents'] = { 'paragraph': hp.choice('labels_paragraph', [ { 'use': False }, { 'use': True, 'weight': hp.loguniform('para_weight', -4, 1) } ]), 'sentence': hp.choice('labels_sentence', [ { 'use': False }, { 'use': True, 'weight': hp.loguniform('sent_weight', -4, 1) } ]), }
def many_dists(): a = hp.choice('a', [0, 1, 2]) b = hp.randint('b', 10) c = hp.uniform('c', 4, 7) d = hp.loguniform('d', -2, 0) e = hp.quniform('e', 0, 10, 3) f = hp.qloguniform('f', 0, 3, 2) g = hp.normal('g', 4, 7) h = hp.lognormal('h', -2, 2) i = hp.qnormal('i', 0, 10, 2) j = hp.qlognormal('j', 0, 2, 1) k = hp.pchoice('k', [(.1, 0), (.9, 1)]) z = a + b + c + d + e + f + g + h + i + j + k return {'loss': scope.float(scope.log(1e-12 + z ** 2)), 'status': base.STATUS_OK}
def _process_vw_argument(self, arg, value, algorithm): try: distr_part = self.distr_pattern.findall(value)[0] except IndexError: distr_part = '' range_part = self.range_pattern.findall(value)[0] is_continuous = '..' in range_part ocd = self.only_continuous.findall(value) if not is_continuous and len(ocd)> 0 and ocd[0] != '': raise ValueError(("Need a range instead of a list of discrete values to define " "uniform or log-uniform distribution. " "Please, use [min..max]%s form") % (distr_part)) if is_continuous and arg == '-q': raise ValueError(("You must directly specify namespaces for quadratic features " "as a list of values, not as a parametric distribution")) hp_choice_name = "_".join([algorithm, arg.replace('-', '')]) try_omit_zero = 'O' in distr_part distr_part = distr_part.replace('O', '') if is_continuous: vmin, vmax = [float(i) for i in range_part.split('..')] if distr_part == 'L': distrib = hp.loguniform(hp_choice_name, log(vmin), log(vmax)) elif distr_part == '': distrib = hp.uniform(hp_choice_name, vmin, vmax) elif distr_part == 'I': distrib = hp.quniform(hp_choice_name, vmin, vmax, 1) elif distr_part in {'LI', 'IL'}: distrib = hp.qloguniform(hp_choice_name, log(vmin), log(vmax), 1) else: raise ValueError("Cannot recognize distribution: %s" % (distr_part)) else: possible_values = range_part.split(',') if arg == '-q': possible_values = [v.replace('+', ' -q ') for v in possible_values] distrib = hp.choice(hp_choice_name, possible_values) if try_omit_zero: hp_choice_name_outer = hp_choice_name + '_outer' distrib = hp.choice(hp_choice_name_outer, ['omit', distrib]) return distrib
def test_switch_multiple_types(self): prefix = "test" order = hp.choice('%sp_order' % prefix, [1, 2, hp.loguniform('%sp_order_real' % prefix, mu=np.log(1), sigma=np.log(3))]) rval = convert_convnet_searchspace.convert_switch(order, 0) expected = '"testp_order": hp.choice("testp_order", [\n'\ ' {\n' \ ' "testp_order": 0,\n' \ ' },\n' \ ' {\n' \ ' "testp_order": 1,\n' \ ' },\n' \ ' {\n' \ ' "testp_order": 2,\n' \ ' "testp_order_real": hp.loguniform("testp_order_real", lower=0.0, upper=1.09861228867),\n' \ ' },\n' \ ']),\n' self.assertEqual(rval, expected)
def get_frn_params( inner_model, feature_importances_getter=None, name='frn_common' ): if feature_importances_getter is None: feature_importances_getter = get_feature_selector_estimator_params() return scope.get_frn( inner_model=inner_model, feature_importances_getter=feature_importances_getter, lmbda=hp.loguniform( get_full_name(name, 'lmbda'), -5, 15, ), feature_selection_threshold_coef=hp.uniform( get_full_name(name, 'threshold_coef'), 0.0, 1.0, ), )
def optimize(trials, X, y, y_ix, reps, max_evals): space = { 'C': hp.loguniform('C', -3, 2), 'max_iter': hp.quniform('max_iter', 100, 300, 50), 'n_jobs': -1, 'class_weight': 'balanced', 'penalty': 'l1' } s = Score(X, y, y_ix, reps) best = fmin(s.get_score, space, algo=tpe.suggest, trials=trials, max_evals=max_evals ) best['max_iter'] = int(best['max_iter']) best['n_jobs'] = -1 best['class_weight'] = 'balanced' best['penalty'] = 'l1' del s return best
hp.quniform('input_dropout', 0, 0.2, 0.05), 'hidden_layers': hp.quniform('hidden_layers', 2, 4, 1), 'hidden_units': hp.quniform('hidden_units', 32, 256, 32), 'hidden_activation': hp.choice('hidden_activation', ['prelu', 'relu']), 'hidden_dropout': hp.quniform('hidden_dropout', 0, 0.3, 0.05), 'batch_norm': hp.choice('batch_norm', ['before_act', 'no']), 'optimizer': hp.choice('optimizer', [{ 'type': 'adam', 'lr': hp.loguniform('adam_lr', np.log(0.00001), np.log(0.01)) }, { 'type': 'sgd', 'lr': hp.loguniform('sgd_lr', np.log(0.00001), np.log(0.01)) }]), 'batch_size': hp.quniform('batch_size', 32, 128, 32), } class MLP: def __init__(self, params): self.params = params self.scaler = None self.model = None
def hyper(args): ds = io.create_dataset(args.input, output_file=os.path.join(args.outputdir, 'input.zarr'), transpose=args.transpose, test_split=args.testsplit, size_factors=args.normtype) hyper_params = { "data": { "norm_input_log": hp.choice('d_norm_log', (True, False)), "norm_input_zeromean": hp.choice('d_norm_zeromean', (True, False)), "norm_input_sf": hp.choice('d_norm_sf', (True, False)), }, "model": { "lr": hp.loguniform("m_lr", np.log(1e-3), np.log(1e-2)), "ridge": hp.loguniform("m_ridge", np.log(1e-5), np.log(1e-2)), "l1_enc_coef": hp.loguniform("m_l1_enc_coef", np.log(1e-5), np.log(1e-2)), "hidden_size": hp.choice("m_hiddensize", ((64, 32, 64), (32, 16, 32), (64, 64), (32, 32), (16, 16), (16, ), (32, ), (64, ), (128, ))), "activation": hp.choice("m_activation", ('relu', 'selu', 'elu', 'PReLU', 'linear', 'LeakyReLU')), "batchnorm": hp.choice("m_batchnorm", (True, False)), "dropout": hp.uniform("m_do", 0, 0.7), "input_dropout": hp.uniform("m_input_do", 0, 0.8), }, "fit": { "epochs": 100 } } def data_fn(norm_input_log, norm_input_zeromean, norm_input_sf): if norm_input_sf: sf_mat = ds.train.size_factors[:] else: sf_mat = np.ones((ds.train.matrix.shape[0], 1), dtype=np.float32) x_train = { 'count': io.normalize(ds.train.matrix[:], sf_mat, logtrans=norm_input_log, sfnorm=norm_input_sf, zeromean=norm_input_zeromean), 'size_factors': sf_mat } y_train = ds.train.matrix[:] return (x_train, y_train), def model_fn(train_data, lr, hidden_size, activation, batchnorm, dropout, input_dropout, ridge, l1_enc_coef): net = AE_types[args.type](train_data[1].shape[1], hidden_size=hidden_size, l2_coef=0.0, l1_coef=0.0, l2_enc_coef=0.0, l1_enc_coef=l1_enc_coef, ridge=ridge, hidden_dropout=dropout, input_dropout=input_dropout, batchnorm=batchnorm, activation=activation, init='glorot_uniform', debug=args.debug) net.build() optimizer = opt.__dict__['rmsprop'](lr=lr, clipvalue=5.0) net.model.compile(loss=net.loss, optimizer=optimizer) return net.model output_dir = os.path.join(args.outputdir, 'hyperopt_results') objective = CompileFN('autoencoder_hyperpar_db', 'myexp1', data_fn=data_fn, model_fn=model_fn, loss_metric='loss', loss_metric_mode='min', valid_split=.2, save_model=None, save_results=True, use_tensorboard=True, save_dir=output_dir) test_fn(objective, hyper_params, save_model=None) trials = Trials() best = fmin(objective, hyper_params, trials=trials, algo=tpe.suggest, max_evals=1000) with open(os.path.join(output_dir, 'trials.pickle'), 'wb') as f: pickle.dump(trials, f) print(best)
def train_model(self, config, reverse_dictionary, train_data, train_labels, test_data, test_labels, tool_tr_samples, class_weights): """ Train a model and report accuracy """ # convert items to integer l_batch_size = list(map(int, config["batch_size"].split(","))) l_embedding_size = list(map(int, config["embedding_size"].split(","))) l_units = list(map(int, config["units"].split(","))) # convert items to float l_learning_rate = list(map(float, config["learning_rate"].split(","))) l_dropout = list(map(float, config["dropout"].split(","))) l_spatial_dropout = list( map(float, config["spatial_dropout"].split(","))) l_recurrent_dropout = list( map(float, config["recurrent_dropout"].split(","))) optimize_n_epochs = int(config["optimize_n_epochs"]) # get dimensions dimensions = len(reverse_dictionary) + 1 best_model_params = dict() early_stopping = EarlyStopping(monitor='val_loss', mode='min', verbose=1, min_delta=1e-1, restore_best_weights=True) # specify the search space for finding the best combination of parameters using Bayesian optimisation params = { "embedding_size": hp.quniform("embedding_size", l_embedding_size[0], l_embedding_size[1], 1), "units": hp.quniform("units", l_units[0], l_units[1], 1), "batch_size": hp.quniform("batch_size", l_batch_size[0], l_batch_size[1], 1), "learning_rate": hp.loguniform("learning_rate", np.log(l_learning_rate[0]), np.log(l_learning_rate[1])), "dropout": hp.uniform("dropout", l_dropout[0], l_dropout[1]), "spatial_dropout": hp.uniform("spatial_dropout", l_spatial_dropout[0], l_spatial_dropout[1]), "recurrent_dropout": hp.uniform("recurrent_dropout", l_recurrent_dropout[0], l_recurrent_dropout[1]) } def create_model(params): model = Sequential() model.add( Embedding(dimensions, int(params["embedding_size"]), mask_zero=True)) model.add(SpatialDropout1D(params["spatial_dropout"])) model.add( GRU(int(params["units"]), dropout=params["dropout"], recurrent_dropout=params["recurrent_dropout"], return_sequences=True, activation="elu")) model.add(Dropout(params["dropout"])) model.add( GRU(int(params["units"]), dropout=params["dropout"], recurrent_dropout=params["recurrent_dropout"], return_sequences=False, activation="elu")) model.add(Dropout(params["dropout"])) model.add(Dense(2 * dimensions, activation="sigmoid")) optimizer_rms = RMSprop(lr=params["learning_rate"]) batch_size = int(params["batch_size"]) model.compile(loss=utils.weighted_loss(class_weights), optimizer=optimizer_rms) print(model.summary()) model_fit = model.fit_generator( utils.balanced_sample_generator(train_data, train_labels, batch_size, tool_tr_samples, reverse_dictionary), steps_per_epoch=len(train_data) // batch_size, epochs=optimize_n_epochs, callbacks=[early_stopping], validation_data=(test_data, test_labels), verbose=2, shuffle=True) return { 'loss': model_fit.history["val_loss"][-1], 'status': STATUS_OK, 'model': model } # minimize the objective function using the set of parameters above trials = Trials() learned_params = fmin(create_model, params, trials=trials, algo=tpe.suggest, max_evals=int(config["max_evals"])) best_model = trials.results[np.argmin( [r['loss'] for r in trials.results])]['model'] # set the best params with respective values for item in learned_params: item_val = learned_params[item] best_model_params[item] = item_val return best_model_params, best_model
""" Cart-pole balancing with continuous / Kernelized iFDD """ from rlpy.Domains.FiniteTrackCartPole import FiniteCartPoleBalanceOriginal, FiniteCartPoleBalanceModern from rlpy.Agents import SARSA, Q_LEARNING from rlpy.Representations import * from rlpy.Policies import eGreedy from rlpy.Experiments import Experiment import numpy as np from hyperopt import hp from rlpy.Representations import KernelizediFDD param_space = { 'kernel_resolution': hp.loguniform("kernel_resolution", np.log(5), np.log(50)), 'discover_threshold': hp.loguniform("discover_threshold", np.log(1e-1), np.log(1e3)), 'lambda_': hp.uniform("lambda_", 0., 1.), 'boyan_N0': hp.loguniform("boyan_N0", np.log(1e1), np.log(1e5)), 'initial_learn_rate': hp.loguniform("initial_learn_rate", np.log(5e-2), np.log(1)) } def make_experiment(exp_id=1, path="./Results/Temp/{domain}/{agent}/{representation}/", discover_threshold=.21, boyan_N0=37., lambda_=.9,
test_loss, test_aucroc, xgb_params['eta'], xgb_params['max_depth'], NB_SAMPLES) sub.to_csv(submission_path, index=False, float_format='%8.5f') return {'loss': test_loss, 'status': STATUS_OK} if USE_HYPEROPT: log_writer = open('../data/lgb-hyperopt-log.txt', 'w') space = { 'num_leaves': hp.quniform('num_leaves', 10, 200, 1), 'min_data_in_leaf': hp.quniform('min_data_in_leaf', 10, 1000, 1), 'feature_fraction': hp.uniform('feature_fraction', 0.75, 1.0), 'bagging_fraction': hp.uniform('bagging_fraction', 0.75, 1.0), 'learning_rate': hp.loguniform('learning_rate', -3.0, -1.2), 'min_sum_hessian_in_leaf': hp.loguniform('min_sum_hessian_in_leaf', 0, 2.3), 'max_bin': hp.quniform('max_bin', 64, 512, 1), 'bagging_freq': hp.quniform('bagging_freq', 1, 5, 1), 'lambda_l1': hp.uniform('lambda_l1', 0, 10), 'lambda_l2': hp.uniform('lambda_l2', 0, 10), } trials = Trials() best = hyperopt.fmin(fn=objective, space=space, algo=HYPEROPT_ALGO, max_evals=N_HYPEROPT_PROBES, trials=trials, verbose=1)
import hpnnet.nips2011 space = { 'preproc': hp.choice( 'preproc', [ { 'preproc': 0 }, { 'preproc': 1, # Column Normalization 'colnorm_thresh': hp.loguniform('colnorm_thresh', np.log(1e-9), np.log(1e-3)), }, { 'preproc': 2, # PCA 'pca_energy': hp.uniform('pca_energy', .5, 1), } ]), 'nhid1': hp.qloguniform('nhid1', np.log(16), np.log(1024), q=16), 'dist1': hp.choice('dist1', [0, 1]), # 0 = Uniform, 1 = Normal 'scale_heur1': hp.choice( 'scale_heur1', [ {
# yapf: disable # __plot_scheduler_begin__ # Plot by epoch ax = None # This plots everything on the same plot for d in dfs.values(): ax = d.mean_accuracy.plot(ax=ax, legend=False) # __plot_scheduler_end__ # yapf: enable # __run_searchalg_begin__ from hyperopt import hp from ray.tune.suggest.hyperopt import HyperOptSearch space = { "lr": hp.loguniform("lr", 1e-10, 0.1), "momentum": hp.uniform("momentum", 0.1, 0.9), } hyperopt_search = HyperOptSearch(space, metric="mean_accuracy", mode="max") analysis = tune.run(train_mnist, num_samples=10, search_alg=hyperopt_search) # __run_searchalg_end__ # __run_analysis_begin__ import os df = analysis.results_df logdir = analysis.get_best_logdir("mean_accuracy", mode="max") state_dict = torch.load(os.path.join(logdir, "model.pth"))
def compute(self, label): return hp.loguniform(label, self.low, self.high, self.q)
from utils.python_utils import quniform_int steps = [('undersampler', TomekLinks(random_state=random_seed)), ('pca', PCA(n_components=50, random_state=random_seed)), ('xgb', xgb.XGBClassifier(n_estimators=1000, silent=True, nthread=3, seed=random_seed))] model = Pipeline(steps=steps) params_space = { 'pca__n_components': quniform_int('n_components', 20, 200, 10), 'xgb__max_depth': quniform_int('max_depth', 10, 30, 1), 'xgb__min_child_weight': hp.quniform('min_child_weight', 1, 20, 1), 'xgb__subsample': hp.uniform('subsample', 0.8, 1), 'xgb__n_estimators': quniform_int('n_estimators', 1000, 10000, 50), 'xgb__learning_rate': hp.loguniform('learning_rate', np.log(0.0001), np.log(0.5)) - 0.0001, 'xgb__gamma': hp.loguniform('gamma', np.log(0.0001), np.log(5)) - 0.0001, 'xgb__colsample_bytree': hp.quniform('colsample_bytree', 0.5, 1, 0.05) }
if stratified: for _ in range(num_evals): best = fmin(hyperopt_stratified_score, search_params, tpe.suggest, len(trials)+1, trials, verbose=verbosity, show_progressbar=verbosity) pickle.dump(trials, open(trials_file, "wb")) else: for _ in range(num_evals): best = fmin(hyperopt_score, search_params, tpe.suggest, len(trials)+1, trials, verbose=verbosity, show_progressbar=verbosity) pickle.dump(trials, open(trials_file, "wb")) return best _search_params_lgb = { 'class_weight': hp.choice('class_weight', [None, 'balanced']), 'boosting_type': hp.choice('boosting_type', ['gbdt','dart','goss']), 'num_leaves': scope.int(hp.quniform('num_leaves', 20, 200, 1)), 'learning_rate': hp.loguniform('learning_rate', np.log(0.01), np.log(0.2)), 'subsample_for_bin': scope.int(hp.quniform('subsample_for_bin', 20000, 300000, 20000)), 'feature_fraction': hp.uniform('feature_fraction', 0.5, 1), 'bagging_fraction': hp.uniform('bagging_fraction', 0.5, 1), 'min_data_in_leaf': scope.int(hp.qloguniform('min_data_in_leaf', 0, 6, 1)), 'lambda_l1': hp.choice('lambda_l1', [0, hp.loguniform('lambda_l1_positive', -16, 2)]), 'lambda_l2': hp.choice('lambda_l2', [0, hp.loguniform('lambda_l2_positive', -16, 2)]), 'verbose': -1, 'subsample': None, 'reg_alpha': None, 'reg_lambda': None, 'min_sum_hessian_in_leaf': None, 'min_child_samples': None, 'colsample_bytree': None, #'min_child_samples': hp.quniform('min_child_samples', 20, 500, 5), 'min_child_weight': hp.loguniform('min_child_weight', -16, 5),
def optimize(self): """ This is the optimization function that given a space (space here) of hyperparameters and a scoring function (score here), finds the best hyperparameters. """ # XGBoost space if self.model_type == "xgboost" or self.model_type == "xgboost_classifier": space = { 'n_estimators': 100 + hp.randint('n_estimators', 1000), 'eta': hp.loguniform('eta', np.log(1e-5), np.log(1.0)), # A problem with max_depth casted to float instead of int with # the hp.quniform method. # TODO: Was too big, trying smaller for now (maybe it caused bad malloc?) 'max_depth': hp.choice('max_depth', np.arange(1, 14, dtype=int)), 'min_child_weight': hp.quniform('min_child_weight', 1, 6, 1), 'subsample': hp.quniform('subsample', 0.5, 1, 0.05), 'gamma': hp.quniform('gamma', 0.5, 1, 0.05), 'colsample_bytree': hp.quniform('colsample_bytree', 0.5, 1, 0.05), # TODO: Try GPU later. 'tree_method': 'hist', 'seed': self.seed, 'n_jobs': 8 } if self.model_type == "xgboost_classifier": print((self.y == 0).sum() / (self.y == 1).sum()) space["scale_pos_weight"] = (self.y == 0).sum() / (self.y == 1).sum() else: # LigthGBM space # TODO: Try other boosting types later. # Smaller n_estimtors. space = { 'boosting_type': 'goss', 'n_estimators': 100 + hp.randint('n_estimators', 400), 'learning_rate': hp.loguniform('learning_rate', np.log(1e-8), np.log(1.0)), 'max_depth': hp.choice('max_depth', np.arange(1, 14, dtype=int)), 'num_leaves': hp.choice('num_leaves', [15, 31, 63, 127, 200]), 'subsample': hp.quniform('subsample', 0.5, 1, 0.05), 'colsample_bytree': hp.quniform('colsample_bytree', 0.5, 1, 0.05), 'min_child_weight': hp.quniform('min_child_weight', 1, 100, 1), 'reg_alpha': hp.quniform('reg_alpha', 1, 100, 1), 'min_split_gain': hp.quniform('min_split_gain', 1, 100, 1), 'min_data_in_leaf': 10 + hp.randint('min_data_in_leaf', 100), 'bagging_seed': self.seed, 'drop_seed': self.seed, 'seed': self.seed } # Use the fmin function from Hyperopt to find the best hyperparameters best = fmin(self.score, space, algo=tpe.suggest, max_evals=self.max_evals, trials=self.trials) return best
""" Intruder Monitoring with incremental tabular representation, Q-Learning """ from rlpy.Domains.IntruderMonitoring import IntruderMonitoring from rlpy.Agents import Q_Learning from rlpy.Representations import IncrementalTabular from rlpy.Policies import eGreedy from rlpy.Experiments import Experiment import numpy as np from hyperopt import hp param_space = {'boyan_N0': hp.loguniform("boyan_N0", np.log(1e1), np.log(1e5)), 'initial_learn_rate': hp.loguniform("initial_learn_rate", np.log(5e-2), np.log(1))} def make_experiment( exp_id=1, path="./Results/Temp/{domain}/{agent}/{representation}/", boyan_N0=120, initial_learn_rate=.06, discretization=50): opt = {} opt["path"] = path opt["exp_id"] = exp_id opt["max_steps"] = 100000 opt["num_policy_checks"] = 10 opt["checks_per_policy"] = 1 domain = IntruderMonitoring() opt["domain"] = domain representation = IncrementalTabular(domain) policy = eGreedy(representation, epsilon=0.1)
CMs.append(confusion_matrix(y[test_idx], y_preds[test_idx])) CM = np.sum(CMs, axis=0) FN = CM[1][0] TP = CM[1][1] FP = CM[0][1] print("TP = {}".format(TP)) print("FP = {}".format(FP)) print("FN = {}".format(FN)) f1 = 2. * TP / (2. * TP + FP + FN) print("F1 : ", f1) return{'loss': 1-f1, 'status': STATUS_OK} space = {'C': hp.loguniform('C', -9.0, 3.0), 'cw': hp.uniform('cw', 1, 30)} trials = Trials() best = fmin(fn=objective, space=space, algo=tpe.suggest, max_evals=100, trials=trials) pprint(hyperopt.space_eval(space, best)) best_pars = hyperopt.space_eval(space, best) # clf = LogisticRegression(C=0.011, class_weight={0: 1, 1: 4.36}, # random_state=1, max_iter=300, n_jobs=1,
model = Net().to(device) loss = train_one_epoch(model, device, train_dataset, batch_size, learning_rate, momentum) return {'loss': loss.item(), 'status': STATUS_OK} # COMMAND ---------- # MAGIC %md # MAGIC **Define the search space over hyperparameters** # COMMAND ---------- search_space = { 'batch_size': hp.uniform('batch_size', 10, 200), 'learning_rate': hp.loguniform('learning_rate', -2.0, 0.), 'momentum': hp.uniform('momentum', 0.1, 0.5), } # COMMAND ---------- # MAGIC %md # MAGIC **Select a search algorithm** # COMMAND ---------- algo = tpe.suggest # Tree of Parzen Estimators (a "Bayesian" method) # COMMAND ---------- # MAGIC %md
model.add(Conv1D(filters=16, kernel_size=11, activation='relu')) model.add(MaxPool1D(strides=4)) model.add(BatchNormalization()) model.add(Conv1D(filters=32, kernel_size=11, activation='relu')) model.add(MaxPool1D(strides=4)) model.add(BatchNormalization()) model.add(Conv1D(filters=64, kernel_size=11, activation='relu')) model.add(MaxPool1D(strides=4)) model.add(Flatten()) model.add(Dropout(dropout1)) model.add(Dense(64, activation='relu')) model.add(Dropout(dropout2)) model.add(Dense(64, activation='relu')) model.add(Dense(1, activation='sigmoid')) model.compile(optimizer=Adam(lr, decay=2e-4), loss='binary_crossentropy', metrics=['accuracy']) return model model = KerasBatchClassifier(build_fn=create_model, epochs=40, batch_size=32, verbose=2) params_space = { 'lr': hp.loguniform('lr', -10, -4), 'dropout1': hp.quniform('dropout1', 0.25, .75, 0.25), 'dropout2': hp.quniform('dropout2', 0.25, .75, 0.25), }
'boosting_type': hp.choice('boosting_type', [{ 'boosting_type': 'gbdt', 'subsample': hp.uniform('gdbt_subsample', 0.5, 1) }, { 'boosting_type': 'dart', 'subsample': hp.uniform('dart_subsample', 0.5, 1) }, { 'boosting_type': 'goss', 'subsample': 1.0 }]), 'num_leaves': hp.quniform('num_leaves', 30, 150, 1), 'learning_rate': hp.loguniform('learning_rate', np.log(0.01), np.log(0.2)), 'subsample_for_bin': hp.quniform('subsample_for_bin', 20000, 300000, 20000), 'min_child_samples': hp.quniform('min_child_samples', 20, 500, 5), 'reg_alpha': hp.uniform('reg_alpha', 0.0, 1.0), 'reg_lambda': hp.uniform('reg_lambda', 0.0, 1.0), 'colsample_bytree': hp.uniform('colsample_by_tree', 0.6, 1.0) } x = sample(space) # Conditional logic to assign top-level keys subsample = x['boosting_type'].get('subsample', 1.0)
# Return cross-validation loss # NB: we are minimizing here zo we need to take 1-accuracy return ({ "loss": history["validation_loss"][which_min], "parameters": parameters, "iteration": which_min, 'status': STATUS_OK }) # Define the search space space = { 'hidden_size': hp.choice('hidden_units', [32, 64, 128]), 'sent_length': hp.choice("sent_length", [8, 10, 12, 15]), 'use_class_weights': hp.choice("use_class_weights", [True, False]), 'learning_rate': hp.loguniform('learning_rate', np.log(0.001), np.log(0.03)), 'dropout_prop': hp.uniform("dropout", 0, 0.5) } #%% Test space # Test if works from hyperopt.pyll.stochastic import sample parameters = sample(space) print(parameters) po = HAN_search(parameters) #%% Run the optimizer # Algorithm
log_writer.flush() if test_loss<cur_best_loss: cur_best_loss = test_loss print(colorama.Fore.GREEN + 'NEW BEST LOSS={}'.format(cur_best_loss) + colorama.Fore.RESET) return{'loss':test_loss, 'status': STATUS_OK } # -------------------------------------------------------------------------------- space ={ 'depth': hp.quniform("depth", 4, 7, 1), 'rsm': hp.uniform ('rsm', 0.75, 1.0), 'learning_rate': hp.loguniform('learning_rate', -3.0, -0.7), 'l2_leaf_reg': hp.uniform('l2_leaf_reg', 1, 10), } trials = Trials() best = hyperopt.fmin(fn=objective, space=space, algo=HYPEROPT_ALGO, max_evals=N_HYPEROPT_PROBES, trials=trials, verbose=1) print('-'*50) print('The best params:') print( best )
## gradient boosting regressor 一种跟xgboost类似的梯度自举树 param_space_reg_skl_gbm = { 'task': 'reg_skl_gbm', 'n_estimators': hp.choice('n_estimators', np.arange(skl_min_n_estimators, skl_max_n_estimators, skl_n_estimators_step, dtype=int)), 'learning_rate': hp.quniform("learning_rate", 0.01, 0.5, 0.01), 'max_features': hp.quniform("max_features", 0.05, 1.0, 0.05), 'max_depth': hp.choice('max_depth', np.arange(1, 15, dtype=int)), 'subsample': hp.quniform('subsample', 0.5, 1, 0.1), 'random_state': skl_random_seed, "max_evals": hyperopt_param["gbm_max_evals"], } ## support vector regression param_space_reg_skl_svr = { 'task': 'reg_skl_svr', 'C': hp.loguniform("C", np.log(1), np.log(100)), 'gamma': hp.loguniform("gamma", np.log(0.001), np.log(0.1)), 'degree': hp.choice('degree', np.arange(1, 5, 1, dtype=int)), 'epsilon': hp.loguniform("epsilon", np.log(0.001), np.log(0.1)), 'kernel': hp.choice('kernel', ['rbf', 'poly']), "max_evals": hyperopt_param["svr_max_evals"], } ## ridge regression param_space_reg_skl_ridge = { 'task': 'reg_skl_ridge', 'alpha': hp.loguniform("alpha", np.log(0.01), np.log(20)), 'random_state': skl_random_seed, "max_evals": hyperopt_param["ridge_max_evals"], }
'conv1d_weight': 0.5272285958816297, 'FM_FTRL_weight': 0.9188890188824305, } param_space_vanila_GRU = { 'denselayer_units': hp.uniform("denselayer_units", 128, 256), 'description_Len': hp.uniform("description_Len", 80, 90), 'name_Len': hp.uniform("name_Len", 8, 12), 'embed_name': hp.uniform("embed_name", 45, 61), 'embed_desc': hp.uniform("embed_desc", 45, 61), 'embed_brand': hp.uniform("embed_brand", 20, 31), 'embed_cat_2': hp.uniform("embed_cat_2", 8, 16), 'embed_cat_3': hp.uniform("embed_cat_3", 30, 41), 'rnn_dim_name': hp.uniform("rnn_dim_name",20, 30), 'rnn_dim_desc': hp.uniform("rnn_dim_desc",20, 30), 'lr': hp.loguniform("lr", np.log(0.001), np.log(0.1)), 'batch_size': hp.uniform("batch_size",512, 2048), 'dense_drop': hp.loguniform("dense_drop", np.log(0.001), np.log(0.01)), } # 0.42395 959s param_space_best_vanila_GRU = { 'denselayer_units': 232, 'description_Len': 85, 'name_Len': 10, 'embed_name': 49, 'embed_desc': 56, 'embed_brand': 29, 'embed_cat_2': 12, 'embed_cat_3': 37, 'rnn_dim_name': 23,
return { 'status': STATUS_OK, 'loss': np.average(min_dev_loss), 'epochs': epochs, 'metrics': { 'accuracys': max_dev_acc, 'accuracy': np.average(max_dev_acc), 'losses': min_dev_loss } } # ================== step3: 调参 ================= space = { 'learning_rate': hp.loguniform('learning_rate', np.log(0.0001), np.log(0.01)), # 学习率 'batch_size': hp.quniform('batch_size', 32, 512, 8), # mini batch大小 'keep_prob': hp.uniform('keep_prob', 0.3, 1.0), # drop out 'l2reg': hp.loguniform('l2reg', np.log(0.0001), np.log(0.01)), # L2正则化 } ts = Trials() from functools import partial best = fmin(dev_loss, space, algo=partial(tpe.suggest, n_startup_jobs=15), max_evals=50, trials=ts) print('TPE best: {}'.format(best)) for trial in ts.trials:
import mlflow import mlflow.pyfunc import mlflow.sklearn from sklearn.model_selection import cross_val_score from hyperopt import fmin, tpe, hp, Trials, STATUS_OK from hyperopt.pyll import scope from IPython.display import Image import numpy as np import lightgbm as lgb from lightgbm import LGBMModel, LGBMClassifier hyperparameters = { "max_depth": scope.int(hp.quniform("max_depth", 2, 100, 5)), "n_estimators": scope.int(hp.quniform("n_estimators", 2, 100, 1)), "num_leaves": scope.int(hp.quniform("num_leaves", 2, 50, 1)), "reg_alpha": hp.loguniform('reg_li', -5, 5), "random_state": 1, "learning_rate": hp.loguniform("learning_rate", np.log(0.01), np.log(0.5)), "min_child_weight": hp.uniform('min_child_weight', 0.5, 10), "boosting": hp.choice("boosting", ["gbdt", "dart", "goss"]), "objective": "binary" } # + def train_model(parameters): mlflow.lightgbm.autolog() with mlflow.start_run(nested=True): booster = lgb.LGBMClassifier() booster.set_params(**parameters) booster.fit(X_train_prepared, y_train)
import hyperopt.hp as hp import mlflow import sbi.utils as sbi_utils import torch from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, tpe from snpe.inference.inference_class import TimeSeriesInference ARTIFACT_PATH = Path("./artifacts/hyperparameter_tuning/cnn_tuning") SEARCH_SPACE = { "batch_size": hp.choice("batch_size", [32, 64, 128, 256]), # log(1e-6)=-13.82, log(1e-2)=-4.6, as a result learning rate is log-uniform distributed between # 1e-6 and 1e-2 in the setup below "learning_rate": hp.loguniform("learning_rate", -13.82, -4.6), "hidden_features": hp.choice("hidden_features", range(20, 80, 5)), "num_transforms": hp.choice("num_transforms", range(2, 9, 1)), "num_conv_layers": hp.choice("num_conv_layers", range(1, 7, 1)), "num_channels": hp.choice("num_channels", range(3, 16, 1)), "conv_kernel_size": hp.choice("conv_kernel_size", range(3, 20, 2)), "maxpool_kernel_size": hp.choice("maxpool_kernel_size", range(3, 20, 2)), "num_dense_layers": hp.choice("num_dense_layers", range(1, 6, 1)), } def hyperopt_run(inferrer: TimeSeriesInference, parameters: Dict, **kwargs) -> Dict: with mlflow.start_run(nested=True): # Log parameters mlflow.log_params(parameters)
import math import os import sys sys.path.append(os.path.dirname(sys.argv[0]) + "/../TensorFlowScripts") import train_GED from hyperopt import hp, fmin, rand, tpe, STATUS_OK, Trials space = { 'lr': hp.loguniform('lr', math.log(1e-5), math.log(30)), 'wd': hp.choice( 'wd', [1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9, 1e-10, 0.0]), 'hsize': hp.choice('hsize', [2048]), 'nlayers': hp.choice('nlayers', [3]), 'Lambda': hp.choice('Lambda', [0.0]), 'gamma': hp.choice('gamma', [0.999]) } def objective(params): train_loss, val_loss = train_GED.main(params) print "LOSSES: ", train_loss, val_loss return {
""" System Administrator with incremental tabular representation, Q-Learning """ from rlpy.Domains.SystemAdministrator import SystemAdministrator from rlpy.Agents import Q_Learning from rlpy.Representations import IncrementalTabular from rlpy.Policies import eGreedy from rlpy.Experiments import Experiment import numpy as np from hyperopt import hp param_space = { 'boyan_N0': hp.loguniform("boyan_N0", np.log(1e1), np.log(1e5)), 'initial_learn_rate': hp.loguniform("initial_learn_rate", np.log(5e-2), np.log(1)) } def make_experiment(exp_id=1, path="./Results/Temp/{domain}/{agent}/{representation}/", boyan_N0=120, initial_learn_rate=.06, discretization=50): opt = {} opt["exp_id"] = exp_id opt["max_steps"] = 100000 opt["num_policy_checks"] = 10 opt["checks_per_policy"] = 1 domain = SystemAdministrator()
def convert(self, param_settings): name = param_settings["name"] domain = param_settings["domain"] dtype = param_settings["dtype"] data = param_settings["data"] assert isinstance( data, list ), "precondition violation. data of type {} not allowed!".format( type(data)) assert len( data ) >= 2, "precondition violation, data must be of length 2, [left_bound, right_bound]" assert isinstance( domain, str ), "precondition violation. domain of type {} not allowed!".format( type(domain)) assert isinstance( dtype, str ), "precondition violation. dtype of type {} not allowed!".format( type(dtype)) if domain == "uniform": if dtype == "float" or dtype == "double": return hp.uniform(name, data[0], data[1]) elif dtype == "int": data = list(np.arange(int(data[0]), int(data[1] + 1))) return hp.choice(name, data) else: msg = "cannot convert the type {} in domain {}".format( dtype, domain) LOG.error(msg) raise LookupError(msg) elif domain == "loguniform": if dtype == "float" or dtype == "double": if data[0] == 0: data[0] += 1e-23 assert data[0] > 0, "precondition Violation, a < 0!" assert data[0] < data[1], "precondition Violation, a > b!" assert data[1] > 0, "precondition Violation, b < 0!" lexp = np.log(data[0]) rexp = np.log(data[1]) assert lexp is not np.nan, "precondition violation, left bound input error, results in nan!" assert rexp is not np.nan, "precondition violation, right bound input error, results in nan!" return hp.loguniform(name, lexp, rexp) else: msg = "cannot convert the type {} in domain {}".format( dtype, domain) LOG.error(msg) raise LookupError(msg) elif domain == "normal": if dtype == "float" or dtype == "double": mu = (data[1] - data[0]) / 2.0 sigma = mu / 3 return hp.normal(name, data[0] + mu, sigma) else: msg = "cannot convert the type {} in domain {}".format( dtype, domain) LOG.error(msg) raise LookupError(msg) elif domain == "categorical": if dtype == 'str': return hp.choice(name, data) elif dtype == 'bool': data = [] for elem in data: if elem == "true" or elem == "True" or elem == 1 or elem == "1": data.append(True) elif elem == "false" or elem == "False" or elem == 0 or elem == "0": data.append(False) else: msg = "cannot convert the type {} in domain {}, unknown bool type value".format( dtype, domain) LOG.error(msg) raise LookupError(msg) return hp.choice(name, data) else: msg = "Precondition violation, domain named {} not available!".format( domain) LOG.error(msg) raise IOError(msg)
import keras.backend as K from hyperopt import hp, tpe, fmin, Trials import pickle import traceback __author__ = "Guillaume Chevalier" __copyright__ = "Copyright 2017, Guillaume Chevalier" __license__ = "MIT License" space = { # This loguniform scale will multiply the learning rate, so as to make # it vary exponentially, in a multiplicative fashion rather than in # a linear fashion, to handle his exponentialy varying nature: 'lr_rate_mult': hp.loguniform('lr_rate_mult', -0.5, 0.5), # L2 weight decay: 'l2_weight_reg_mult': hp.loguniform('l2_weight_reg_mult', -1.3, 1.3), # Batch size fed for each gradient update 'batch_size': hp.quniform('batch_size', 100, 450, 5), # Choice of optimizer: 'optimizer': hp.choice('optimizer', ['Adam', 'Nadam', 'RMSprop']), # Coarse labels importance for weights updates: 'coarse_labels_weight': hp.uniform('coarse_labels_weight', 0.1, 0.7), # Uniform distribution in finding appropriate dropout values, conv layers 'conv_dropout_drop_proba': hp.uniform('conv_dropout_proba', 0.0, 0.35),
INPUT_SNRs = range(-10, 31, 5) k = 0 ############################################################ # ############################################################ name = '2d-grid' n = 20 Y_HIGH = 10 Y_LOW = -5 Gnx, signal_2d, y_true, xs, ys = create2DSignal(k, n, Y_HIGH=Y_HIGH, Y_LOW=Y_LOW) pspace = (hp.loguniform('gamma_mult', -4, 4), hp.loguniform('rho_mult', -3, 3)) max_evals = 40 d = 10 # ############################################################ ############################################################ # name = 'minnesota' # G = pg.graphs.Minnesota(connect=True) # Gnx = nx.from_scipy_sparse_matrix(G.A.astype(float), edge_attribute='weight') # y_true = np.load('datasets/minnesota/beta_0.npy') # pspace = ( # hp.loguniform('gamma_mult', -3, 5), # hp.loguniform('rho_mult', -3, 3) # ) # max_evals = 100
experiment, exp_prefix=exp_prefix, seed=seed, mode=mode, variant=variant, exp_id=exp_id, sync_s3_log=True, sync_s3_pkl=True, periodic_sync_interval=600, ) if run_mode == 'hyperopt': search_space = { 'algo_params.qf_learning_rate': hp.loguniform( 'algo_params.qf_learning_rate', np.log(1e-5), np.log(1e-2), ), 'algo_params.policy_learning_rate': hp.loguniform( 'algo_params.policy_learning_rate', np.log(1e-5), np.log(1e-2), ), 'algo_params.discount': hp.uniform( 'algo_params.discount', 0.0, 1.0, ), 'algo_params.target_hard_update_period':
residuals = params["residuals"] ''' hyperopt_space = { "batch_size": hp.choice("batch_size", [16, 32, 64]), "num_hidden": hp.qloguniform("num_hidden", log(41), log(100), 1), "num_fc_layers": hp.choice("num_fc_layers", [0, 1, 2]), "num_conv_layers": hp.quniform("num_conv_layers", 4, 10, 1), "conv_depth": hp.quniform("conv_depth", 16, 50, 1), "conv_depth_mul": hp.loguniform("conv_depth_mul", log(1), log(2)), "conv_width": hp.quniform("conv_width", 1, 10, 1), "conv_height": hp.quniform("conv_height", 1, 20, 1), "pooling_size": hp.choice("pooling_size", [2, 3]), "pooling_type": hp.choice("pooling_type", ["local_max", "local_avg", "global_max", "global_avg"]), "dropout_coeff": hp.uniform("dropout_coeff", 0.4, 0.6), "reg_coeff": hp.uniform("reg_coeff", -5, -3), "residuals": hp.choice("residuals", ["resnet", "densenet", ""]),