def clf_loop(models_to_run, clfs, grid, X_train, X_test, y_train, y_test, training_dates, testing_dates): """Runs the loop using models_to_run, clfs, gridm and the data """ results = [] for n in range(1, 2): for index, clf in enumerate([clfs[x] for x in models_to_run]): print(models_to_run[index]) parameter_values = grid[models_to_run[index]] for p in ParameterGrid(parameter_values): try: clf.set_params(**p) model_fit = clf.fit(X_train, y_train) y_pred_probs = model_fit.predict_proba(X_test)[:, 1] # Store metrics and model info for comparison y_pred_probs_sorted, y_test_sorted = zip( *sorted(zip(y_pred_probs, y_test), reverse=True)) row = [ training_dates, testing_dates, models_to_run[index], clf, p, baseline(y_test), roc_auc_score(y_test, y_pred_probs), accuracy_at_k(y_test_sorted, y_pred_probs_sorted, 5.0), accuracy_at_k(y_test_sorted, y_pred_probs_sorted, 20.0), accuracy_at_k(y_test_sorted, y_pred_probs_sorted, 50.0), f1_at_k(y_test_sorted, y_pred_probs_sorted, 5.0), f1_at_k(y_test_sorted, y_pred_probs_sorted, 20.0), f1_at_k(y_test_sorted, y_pred_probs_sorted, 50.0), precision_at_k(y_test_sorted, y_pred_probs_sorted, 1.0), precision_at_k(y_test_sorted, y_pred_probs_sorted, 5.0), precision_at_k(y_test_sorted, y_pred_probs_sorted, 10.0), precision_at_k(y_test_sorted, y_pred_probs_sorted, 20.0), precision_at_k(y_test_sorted, y_pred_probs_sorted, 50.0), recall_at_k(y_test_sorted, y_pred_probs_sorted, 1.0), recall_at_k(y_test_sorted, y_pred_probs_sorted, 5.0), recall_at_k(y_test_sorted, y_pred_probs_sorted, 10.0), recall_at_k(y_test_sorted, y_pred_probs_sorted, 20.0), recall_at_k(y_test_sorted, y_pred_probs_sorted, 50.0), ] results.append(row) #plot_precision_recall_n(y_test,y_pred_probs,clf) except IndexError as e: print('Error:', e) continue return results
def best_classifier(X,Y,Xvs,Yvs): parameters = {'C':[3,13,67,330,1636,8103]} pg = ParameterGrid(parameters) clas = Parallel(n_jobs=4)(delayed(pfit)(p,X,Y,Xvs,Yvs) for p in pg) clas.sort(reverse=True) (sc,cla) = clas[0] print '-'*20 print 'best is ',cla,sc print '-'*20 return cla,sc
def __get_param_iterable(self, param_grid): if self.ramdonized_search_enable: parameter_iterable = ParameterSampler( param_grid, self.randomized_search_n_iter, random_state=self.ramdonized_search_random_state) else: parameter_iterable = ParameterGrid(param_grid) return parameter_iterable
def fit(self, X, y): if self.verbose >=1: print 'Running GridSearchCV...' #p = Pool(processes=self.n_jobs) p = MyPool(processes=self.n_jobs) #Get parameters for multiprocessing task: #get parameters that belongs only to first estimator. The remaining belongs to the other estimators param_grid_mine, param_grid_others = self.split_params(self.param_grid, self.estimators.steps[0][0]) params_vars = list(ParameterGrid(param_grid_mine)) #vary all parameters for this estimator only if False: #set to TRUE to make debug of inner functions easier. It does not use multi-task in this case print 'vai dar erro, remover isso daki!' lst_scores, lst_params = self.est_var(self.estimators.steps, self.param_grid, X, y,params_vars=params_vars) params_pool =[] #divide the first estimator parameters among n_jobs for i in range(self.n_jobs): params_pool.append(params_vars[i::self.n_jobs]) #for multiple arguments and python 3.3, use pool.starmap() j=len(params_pool) results = p.map(unwrap_self_est_var, zip([self]*j, [self.estimators.steps]*j, [param_grid_others]*j, [X]*j, [y]*j, [None]*j, [None]*j, params_pool)) p.close() #terminate process p.join() lst_scores = [] lst_params = [] for result in results: lst_scores.extend(result[0]) lst_params.extend(result[1]) #get the best score and best parameters best_score = np.asarray(lst_scores).min() idx_best = np.where(lst_scores == best_score) best_params = [lst_params[i] for i in idx_best[0]] self.best_score_ =best_score self.best_params_ = best_params if self.refit: # fit the best estimator using the entire dataset self.set_params(self.best_params_[0]) self.estimators= self.estimators.fit(X, y) if self.verbose>=1: print 'best_params = ' for best_param in best_params: print best_param+'\n' return self
def test_parameter_grid(): """Test basic properties of ParameterGrid.""" params1 = {"foo": [1, 2, 3]} grid1 = ParameterGrid(params1) assert_true(isinstance(grid1, Iterable)) assert_true(isinstance(grid1, Sized)) assert_equal(len(grid1), 3) params2 = {"foo": [4, 2], "bar": ["ham", "spam", "eggs"]} grid2 = ParameterGrid(params2) assert_equal(len(grid2), 6) # loop to assert we can iterate over the grid multiple times for i in xrange(2): # tuple + chain transforms {"a": 1, "b": 2} to ("a", 1, "b", 2) points = set(tuple(chain(*p.items())) for p in grid2) assert_equal( points, set(("foo", x, "bar", y) for x, y in product(params2["foo"], params2["bar"])))
def run(self): with self.input()[0] as i: graphIndex, GR = i.query() with self.input()[1] as i: R = i.query() train_index = np.array(self.train_index) test_index = list(range(len(GR))) test_index = [i for i in test_index if i not in train_index] y_bin = [None] * len(GR) for index, D in R.items(): if index not in graphIndex: continue gI = graphIndex[index] score = D['score'] time = evalTime(D['time_rank']) y_bin[gI] = (score['IUV'], score['ESBMC'], time) y_bin = np.array(y_bin) y_test = y_bin[test_index] y_train = y_bin[train_index] TGR = GR[train_index][:, train_index] EGR = GR[test_index][:, train_index] param_grid = { 'C_tester': self.C_tester.value, 'C_verifier': self.C_verifier.value, 'C_time': self.C_time.value } max_mean = -math.inf max_param = None for params in ParameterGrid(param_grid): tick(self) mean, _ = self._k_fold_cv_gram( TGR, y_train, (params['C_tester'], params['C_verifier'], params['C_time'])) if mean > max_mean: max_mean = mean max_param = params max_param['mean'] = max_mean max_param['h'] = self.h max_param['D'] = self.D max_param['train_matrix'] = TGR.tolist() max_param['train_y'] = y_train.tolist() max_param['test_matrix'] = EGR.tolist() max_param['test_y'] = y_test.tolist() with self.output() as o: o.emit(max_param)
def search_test_params(base_clf, cv_params, X, y, train, test, scoring): parameter_iterable = ParameterGrid(cv_params) grid_scores = Parallel(n_jobs=-1)( delayed(_fit_and_score)(clone(base_clf), X, y, scoring, train, test, 0, parameters, None, return_parameters=True) for parameters in parameter_iterable) # grid_scores = [_fit_and_score(clone(base_clf), X, y, scoring, train, test, 0, parameters, None, return_parameters=True) for parameters in parameter_iterable] grid_scores = sorted(grid_scores, key=lambda x: x[0], reverse=True) scores, _, _, parameters = grid_scores[0] return scores, parameters
def generate_models(self, input_shape, output_dim): loss_type = self.grid.params_grid["loss"][0] for layers in self.create_network_structures(self.grid.params_grid["layers"], self.grid.params_grid["layer_nums"], input_shape): print "Current network: %s" % "->".join(layers) flat_params_grid = self.grid.create_flat_layers_grid(layers, input_shape, output_dim) for optimizer_name in self.grid.params_grid["optimizers"]: flat_grid = flat_params_grid.copy() flat_grid.update(self.grid.create_flat_optimizer_grid(optimizer_name)) for params in ParameterGrid(flat_grid): nn_params = self.grid.fold_params(params) yield self.model_factory.create_model(layers, nn_params, loss_type)
def _grid_search_params_iter(self, train_X, train_y): if callable(self.inner_cv): inner_cv = self.inner_cv(train_X, train_y) else: inner_cv = _check_cv(self.inner_cv, train_X, train_y, classifier=is_classifier(self.estimator)) param_iter = ParameterGrid(self.param_grid) LOG.info("Performing grid search over %d configurations" % len(param_iter)) for fold_id, (train_index, test_index) in enumerate(inner_cv): for parameters in param_iter: yield fold_id + 1, train_index, test_index, parameters
def cv_trials(X, y, folds, model, hyper): N = len(y) cv_scores = [] predictions = { 'pred': np.zeros(N, dtype=np.bool), 'proba': np.zeros(N), 'foldno': np.zeros(N, dtype=np.int32) - 1, } pg = list(ParameterGrid(hyper)) for foldno, (train, val, test) in enumerate(folds): train_X, train_y = X[train], y[train] val_X, val_y = X[val], y[val] test_X, test_y = X[test], y[test] best_params = None best_val_f1 = None for these_params in pg: model.set_params(**these_params) model.fit(train_X, train_y) this_val_f1 = metrics.f1_score(val_y, model.predict(val_X), average="weighted") if not best_params or this_val_f1 > best_val_f1: best_params = these_params best_val_f1 = this_val_f1 if len(pg) > 1: model.set_params(**best_params) model.fit(train_X, train_y) train_f1 = metrics.f1_score(train_y, model.predict(train_X), average="weighted") preds_y = model.predict(test_X) predictions['pred'][test] = preds_y predictions['foldno'][test] = foldno fold_eval = {'f1': metrics.f1_score(test_y, preds_y, average="weighted"), 'p': metrics.precision_score(test_y, preds_y, average="weighted"), 'r': metrics.recall_score(test_y, preds_y, average="weighted"), 'a': metrics.accuracy_score(test_y, preds_y)} print "[%02d] Best hyper [train %.3f -> val %.3f -> test %.3f] %s" % (foldno, train_f1, best_val_f1, fold_eval['f1'], best_params) cv_scores.append(fold_eval) np.set_printoptions(suppress=True) # now we want to compute global evaluations, and consolidate metrics cv_scores = consolidate(cv_scores) preds_y = predictions['pred'] pooled_eval = {'f1': metrics.f1_score(y, preds_y, average="weighted"), 'p': metrics.precision_score(y, preds_y, average="weighted"), 'r': metrics.recall_score(y, preds_y, average="weighted"), 'a': metrics.accuracy_score(y, preds_y)} return pooled_eval, predictions, cv_scores
def _create_batches(self): param_iter = ParameterGrid(self.param_grid) # divide work into batches equal to the communicator's size work_batches = [[] for _ in range(comm_size)] i = 0 for fold_id, (train_index, test_index) in enumerate(self.cv_iter): for parameters in param_iter: work_batches[i % comm_size].append((fold_id + 1, train_index, test_index, parameters)) i += 1 return work_batches
def grid_generator(search_space): param_grid = ParameterGrid(search_space) all_params = [] for p in param_grid: all_params.append(p) for key in search_space.keys(): if (isinstance(search_space[key], dict)): new_params=[] for param in all_params: if (search_space[key][param[key]] is None): new_params.append(param) else: param_grid = ParameterGrid(search_space[key][param[key]]) add_params = [p for p in param_grid] for aparam in add_params: tparam = copy.copy(param) tparam.update(aparam) new_params.append(tparam) all_params = new_params for param in all_params: yield param
def performGridSearchWithKx2CV(clf, X, y, params): grid = list(ParameterGrid(params)) performances = [] for p in grid: new_clf = base.clone(clf) new_clf.set_params(**p) performances.append(np.mean(getKx2CVScores(new_clf, X, y)[0])) print performances[-1] return [grid, performances]
def fit(self, X, y=None): """Run fit with all sets of parameters. Parameters ---------- X : array-like, shape = [n_samples, n_features] Training vector, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] or [n_samples, n_output], optional Target relative to X for classification or regression; None for unsupervised learning. """ return self._fit(X, y, ParameterGrid(self.param_grid))
def clf_loop(models_to_run, clfs, grid, X, y): results_df = pd.DataFrame( columns=('model_type', 'clf', 'parameters', 'time_used', 'auc-roc', 'p_at_5', 'recall_at_5', 'accuracy_at_5', 'f1_at_5', 'p_at_10', 'recall_at_10', 'accuracy_at_10', 'f1_at_10', 'p_at_20', 'recall_at_20', 'accuracy_at_20', 'f1_at_20')) for n in range(1, 2): # create training and valdation sets X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=0) for index, clf in enumerate([clfs[x] for x in models_to_run]): print(models_to_run[index]) parameter_values = grid[models_to_run[index]] for p in ParameterGrid(parameter_values): try: clf.set_params(**p) start_time = time.time() y_pred_probs = clf.fit(X_train, y_train).predict_proba(X_test)[:, 1] end_time = time.time() elapsed_time = end_time - start_time # you can also store the model, feature importances, and prediction scores # we're only storing the metrics for now y_pred_probs_sorted, y_test_sorted = zip( *sorted(zip(y_pred_probs, y_test), reverse=True)) results_df.loc[len(results_df)] = [ models_to_run[index], clf, p, elapsed_time, roc_auc_score(y_test, y_pred_probs), precision_at_k(y_test_sorted, y_pred_probs_sorted, 5.0), recall(y_test_sorted, y_pred_probs_sorted, 5.0), accuracy(y_test_sorted, y_pred_probs_sorted, 5.0), f1(y_test_sorted, y_pred_probs_sorted, 5.0), precision_at_k(y_test_sorted, y_pred_probs_sorted, 10.0), recall(y_test_sorted, y_pred_probs_sorted, 10.0), accuracy(y_test_sorted, y_pred_probs_sorted, 10.0), f1(y_test_sorted, y_pred_probs_sorted, 10.0), precision_at_k(y_test_sorted, y_pred_probs_sorted, 20.0), recall(y_test_sorted, y_pred_probs_sorted, 20.0), accuracy(y_test_sorted, y_pred_probs_sorted, 20.0), f1(y_test_sorted, y_pred_probs_sorted, 20.0) ] if NOTEBOOK == 1: eval_model_precision_recall(y_test, y_pred_probs, clf) #eval_model_roc(y_test,y_pred_probs,clf) except IndexError: print("IndexError") continue return results_df
def initialize_classifiers(self, model_setup): for clf_name, clf in self.valid_classifiers.items(): if clf_name in model_setup: parameter_grid = model_setup[clf_name] for params in ParameterGrid(parameter_grid): clf.set_params(**params) self.classifiers_to_fit.append(clone(clf)) else: continue assert self.classifiers_to_fit != [], "Could not find any valid classifiers"
def trainSklearn(model, grid, train, target, cv, refit=True, n_jobs=5, multi=False): """ Train a sklearn pipeline or model using textual data as input. """ from joblib import Parallel, delayed from sklearn.grid_search import ParameterGrid from numpy import zeros if multi: pred = zeros((train.shape[0], target.unique().shape[0])) from sklearn.metrics import accuracy_score score_func = accuracy_score else: from sklearn.metrics import roc_auc_score score_func = roc_auc_score pred = zeros(train.shape[0]) best_score = 0 for g in ParameterGrid(grid): model.set_params(**g) if len([True for x in list(g.keys()) if x.find('nthread') != -1]) > 0: results = [ fitSklearn(train, target, list(cv), i, model, multi) for i in range(cv.n_folds) ] else: results = Parallel(n_jobs=n_jobs)( delayed(fitSklearn)(train, target, list(cv), i, model, multi) for i in range(cv.n_folds)) if multi: for i in results: pred[i['index'], :] = i['pred'] score = score_func(target, pred.argmax(1)) else: for i in results: pred[i['index']] = i['pred'] score = score_func(target, pred) if score > best_score: best_score = score best_pred = pred.copy() best_grid = g print("Best Score: %0.5f" % best_score) print("Best Grid", best_grid) if refit: model.set_params(**best_grid) model.fit(train, target) return best_pred, model
def create_grid(mp_in=None, npoints=3, provided_keys=None, ga=None): ''' Description, create a grid of evenly spaced samples in model parameters to search over. Inputs: npoints, type: Integer: number of sample points per parameter nparams, type: Integer: number of parameters to use, conflicts, with next argument. nparams, iterates through a list of parameters, and assigns the nparams to use via stupid counting. provided keys: explicitly define state the model parameters that are used to create the grid of samples, by keying into an existing of parameters. This method needs the user of the method to declare a dictionary of model parameters in a path: neuronunit.optimization.model_parameters. Miscallenous, once grid created by this function has been evaluated using neuronunit it can be used for informing a more refined second pass fine grained grid # smaller is a dictionary thats not necessarily as big # as the grid defined in the model_params file. Its not necessarily # a smaller dictionary, if it is smaller it is reduced by reducing sampling # points. ''' if type(mp_in) is type(None): from neuronunit.models.NeuroML2 import model_parameters as modelp mp_in = OrderedDict(modelp.model_params) whole_p_set = {} whole_p_set = OrderedDict(sample_points(copy.copy(mp_in), npoints=npoints)) print(type(provided_keys), 'provided keys') if type(provided_keys) is type(dict): subset = OrderedDict( {k: whole_p_set[k] for k in list(provided_keys.keys())}) elif len(provided_keys) == 1 or type(provided_keys) is type(str('')): subset = OrderedDict({provided_keys: whole_p_set[provided_keys]}) else: subset = OrderedDict({k: whole_p_set[k] for k in provided_keys}) maps = create_a_map(subset) if type(ga) is not type(None): if npoints > 1: for k, v in subset.items(): v[0] = v[0] * 1.0 / 3.0 v[1] = v[1] * 2.0 / 3.0 # The function of maps is to map floating point sample spaces onto a monochromataic matrix indicies. grid = list(ParameterGrid(subset)) return grid, maps
def run_the_models(data, models_to_run, response, features): """ This runs models and produces evaluation output: inputs: data: dataframe with data models_to_run: list of models to run responce: column name of y variable features: list of column names for model features returns: dataframe """ thresholds = [0.01, 0.02, 0.05, 0.10, 0.20, 0.30, 0.50] precision_cols = ["precision_at_{}".format(str(x)) for x in thresholds] recall_cols = ["recall_at_{}".format(str(x)) for x in thresholds] cols = [ 'model', 'parameters', 'train_start', 'train_end', 'test_start', 'test_end', 'f1_score', 'auc' ] + precision_cols + recall_cols model_results = [] splits = temporal_train_test_split(data, 'date_posted', freq='6M') for train, test in splits: X_train, X_test, y_train, y_test = split_data_by_time( data, 'date_posted', train, test, response, features) for m in models_to_run: if m not in MODELS: print(m, 'bad model') break clf = MODELS[m] parameter_grid = ParameterGrid(PARAMS[m]) for p in parameter_grid: try: # initialize list to keep track of results res = [m, p, train[0], train[1], test[0], test[1]] clf.set_params(**p) clf.fit(X_train, y_train) predicted_scores = clf.predict_proba(X_test)[:, 1] predicted_vals = clf.predict(X_test) true_labels = y_test precise = calculate_precision_at_threshold_multi( predicted_scores, true_labels, thresholds) recall = calculate_recall_at_threshold_multi( predicted_scores, true_labels, thresholds) auc = sklearn.metrics.roc_auc_score( true_labels, predicted_vals) f1 = sklearn.metrics.f1_score(true_labels, predicted_vals) # append metrics to list res = res + [auc, f1] + precise + recall model_results.append(res) except Exception as e: print(e, m, p) df = pd.DataFrame(model_results, columns=cols) return df
def clf_loop(models_to_run, clfs, grid, X_train, X_test, y_train, y_test, test_end_year, label): #results_df=pd.DataFrame(columns=('timestamp','model_type','clf','parameters','auc-roc','baseline','p_at_0.5','p_at_1','p_at_5','p_at_10','p_at_10')) index = 0 for index, clf in enumerate([clfs[x] for x in models_to_run]): print 'clf:', clf print models_to_run[index] clf_nm = models_to_run[index] parameter_values = grid[models_to_run[index]] for p in ParameterGrid(parameter_values): try: clf.set_params(**p) print clf y_pred_probs = clf.fit(X_train, y_train).predict_proba(X_test)[:, 1] print('Model fitted') #you can also store the model, feature importances, and prediction scores #we're only string the metrics for now y_pred_probs_sorted, y_test_sorted = zip( *sorted(zip(y_pred_probs, y_test), reverse=True)) print('scoring') results_df = pd.DataFrame( columns=('timestamp', 'model_type', 'clf', 'parameters', 'auc-roc', 'baseline', 'p_at_0.5', 'p_at_1', 'p_at_5', 'p_at_10', 'p_at_10')) results_df.loc[len(results_df)] = [ str(datetime.datetime.now()), models_to_run[index], clf, p, roc_auc_score(y_test, y_pred_probs), precision_at_k(y_test_sorted, y_pred_probs_sorted, 100), precision_at_k(y_test_sorted, y_pred_probs_sorted, 0.5), precision_at_k(y_test_sorted, y_pred_probs_sorted, 1.0), precision_at_k(y_test_sorted, y_pred_probs_sorted, 5.0), precision_at_k(y_test_sorted, y_pred_probs_sorted, 10.0), precision_at_k(y_test_sorted, y_pred_probs_sorted, 20.0) ] filename = "results" + "_" + str( test_end_year) + "_" + label + '.csv' results_df.to_csv(filename, index=True, mode='a') if NOTEBOOK == 1: plot_precision_recall_n(y_test, y_pred_probs, clf, fname=clf_nm + '_' + str(index) + '_' + label + '_' + str(test_end_year) + '.png') index += 1 except IndexError, e: print 'Error:', e continue
def create_misc_confs(): from sklearn.grid_search import ParameterGrid params = { 'break_width': [1.5, 2.0, 3.6, 5.0], 'recognizer': ['probout', 'hmm'], 'combine_hangoff': [.4, .6, .8], 'postprocess': [True, False], 'segmenter': ['experimental', 'stochastic'], 'line_cluster_pos': ['top', 'center'], } grid = ParameterGrid(params) for pr in grid: Config(save_conf=True, **pr)
def parallel_grid_search(lb_view, clf, cv_split_filenames, param_grid): all_tasks = [] all_parameters = list(ParameterGrid(param_grid)) # iterate over parameter combinations for i, params in enumerate(all_parameters): task_for_params = [] # iterate over the K folds for j, cv_split_filename in enumerate(cv_split_filenames): t = lb_view.apply( compute_evaluation, cv_split_filename, clf, params) task_for_params.append(t) all_tasks.append(task_for_params) return all_parameters, all_tasks
def test_iforest(): """Check Isolation Forest for various parameter settings.""" X_train = np.array([[0, 1], [1, 2]]) X_test = np.array([[2, 1], [1, 1]]) grid = ParameterGrid({"n_estimators": [3], "max_samples": [0.5, 1.0, 3], "bootstrap": [True, False]}) with ignore_warnings(): for params in grid: IsolationForest(random_state=rng, **params).fit(X_train).predict(X_test)
def fit(self, X, y): """Actual fitting, performing the search over parameters.""" parameter_iterable = ParameterGrid(self.param_grid) estimator = self.estimator cv = self.cv n_samples = _num_samples(X) X, y = indexable(X, y) if y is not None: if len(y) != n_samples: raise ValueError('Target variable (y) has a different number ' 'of samples (%i) than data (X: %i samples)' % (len(y), n_samples)) cv = check_cv(cv, X, y, classifier=is_classifier(estimator)) if self.verbose > 0: if isinstance(parameter_iterable, Sized): n_candidates = len(parameter_iterable) print("Fitting {0} folds for each of {1} candidates, totalling" " {2} fits".format(len(cv), n_candidates, n_candidates * len(cv))) base_estimator = clone(self.estimator) pre_dispatch = self.pre_dispatch out = Parallel( n_jobs=self.n_jobs, verbose=self.verbose, pre_dispatch=pre_dispatch)(delayed(cv_fit_and_score)( clone(base_estimator), X, y, self.scoring, parameters, cv=cv) for parameters in parameter_iterable) best = sorted(out, key=lambda x: x[0])[-1] self.best_params_ = best[1] self.best_score_ = best[0] if self.refit: # fit the best estimator using the entire dataset # clone first to work around broken estimators best_estimator = clone(base_estimator).set_params(**best[1]) if y is not None: best_estimator.fit(X, y, **self.fit_params) else: best_estimator.fit(X, **self.fit_params) self.best_estimator_ = best_estimator return self
def clf_loop(models_to_run, clfs, grid, X_train, X_test, y_train, y_test): ''' Runs the loop using models_to_run, clfs, gridm and the data ''' results_df = pd.DataFrame(columns=('model_type','clf', 'parameters', 'auc-roc', 'baseline', 'p_at_1', 'p_at_2', 'p_at_5', 'p_at_10', 'p_at_20', 'p_at_30', 'p_at_50', 'r_at_1', 'r_at_2', 'r_at_5', 'r_at_10', 'r_at_20', 'r_at_30', 'r_at_50', 'f_at_1', 'f_at_2', 'f_at_5', 'f_at_10', 'f_at_20', 'f_at_30', 'f_at_50')) for n in range(1, 2): for index,clf in enumerate([clfs[x] for x in models_to_run]): print(models_to_run[index]) parameter_values = grid[models_to_run[index]] for p in ParameterGrid(parameter_values): try: clf.set_params(**p) y_pred_probs = clf.fit(X_train, y_train.values.ravel()).predict_proba(X_test)[:,1] y_pred_probs_sorted, y_test_sorted = zip(*sorted(zip(y_pred_probs, y_test), reverse=True)) y_pred_probs_sorted = np.asarray(y_pred_probs_sorted) results_df.loc[len(results_df)] = [models_to_run[index],clf, p, roc_auc_score(y_test, y_pred_probs), precision_at_k(y_test_sorted,y_pred_probs_sorted,100.0), precision_at_k(y_test_sorted,y_pred_probs_sorted,1.0), precision_at_k(y_test_sorted,y_pred_probs_sorted,2.0), precision_at_k(y_test_sorted,y_pred_probs_sorted,5.0), precision_at_k(y_test_sorted,y_pred_probs_sorted,10.0), precision_at_k(y_test_sorted,y_pred_probs_sorted,20.0), precision_at_k(y_test_sorted,y_pred_probs_sorted,30.0), precision_at_k(y_test_sorted,y_pred_probs_sorted,50.0), recall_at_k(y_test_sorted,y_pred_probs_sorted, 1.0), recall_at_k(y_test_sorted,y_pred_probs_sorted, 2.0), recall_at_k(y_test_sorted,y_pred_probs_sorted, 5.0), recall_at_k(y_test_sorted,y_pred_probs_sorted, 10.0), recall_at_k(y_test_sorted,y_pred_probs_sorted, 20.0), recall_at_k(y_test_sorted,y_pred_probs_sorted, 30.0), recall_at_k(y_test_sorted,y_pred_probs_sorted, 50.0), F_score_at_k(y_test_sorted,y_pred_probs_sorted, 1.0), F_score_at_k(y_test_sorted,y_pred_probs_sorted, 2.0), F_score_at_k(y_test_sorted,y_pred_probs_sorted, 5.0), F_score_at_k(y_test_sorted,y_pred_probs_sorted, 10.0), F_score_at_k(y_test_sorted,y_pred_probs_sorted, 20.0), F_score_at_k(y_test_sorted,y_pred_probs_sorted, 30.0), F_score_at_k(y_test_sorted,y_pred_probs_sorted, 50.0)] plot_precision_recall_n(y_test,y_pred_probs,clf) except IndexError as e: print('Error:',e) continue return results_df
def run_models(): stream = open("generate_predictions.yml", 'r') docs = yaml.load_all(stream) for doc in docs: train_df = pd.read_csv(doc['train_table_name']) test_df = pd.read_csv(doc['test_table_name']) models_to_run = doc['models_to_run'].replace(' ', '').split(',') prediction_var = doc['prediction_var'] feats_to_use = doc['feats_to_use'].replace(' ', '').split(',') X_train = train_df[feats_to_use] X_test = test_df[feats_to_use] y_train = train_df[prediction_var] y_test = test_df[prediction_var] clfs, grid = define_clfs_params() for n in range(1, 2): for index, clf in enumerate([clfs[x] for x in models_to_run]): parameter_values = grid[models_to_run[index]] for p in ParameterGrid(parameter_values): try: filename = models_to_run[index] + '-' + str(p).replace( ' ', '').strip('{}').replace('\'', '').replace( ',', '-').replace( ':', '_') + '-' + '+'.join(feats_to_use) if os.path.isfile("./model_output/" + filename + ".p"): continue print clf clf.set_params(**p) y_pred_probs = clf.fit(X_train, y_train).predict_proba(X_test)[:, 1] try: zipped_imps = sorted(zip(X_train.columns, clf.feature_importances_), key=lambda x: x[1]) top_3_feats = [i[0] for i in zipped_imps[:3]] except AttributeError: top_3_feats = ['NA'] print "---------------" result = pd.DataFrame() result['true_val'] = y_test result['score'] = y_pred_probs pickle.dump([result, top_3_feats], open("./model_output/" + filename + ".p", "wb")) except IndexError, e: print 'Error:', e continue
def clf_loop(models_to_run, clfs, grid, X_train, y_train, X_test, y_test): results_df = pd.DataFrame( columns=('model_type', 'clf', 'parameters', 'time_used', 'auc-roc', 'accuracy', 'p_at_5', 'p_at_10', 'p_at_30', 'p_at_50', 'r_at_5', 'r_at_10', 'r_at_30', 'r_at_50', 'a_at_5', 'a_at_10', 'a_at_30', 'a_at_50', 'f1_at_5', 'f1_at_10', 'f1_at_30', 'f1_at_50', 'feature_importance', 'col_used_for_feat_importance')) for index, clf in enumerate([clfs[x] for x in models_to_run]): print(models_to_run[index]) parameter_values = grid[models_to_run[index]] for p in ParameterGrid(parameter_values): try: clf.set_params(**p) start_time = time.time() clf.fit(X_train, y_train) y_pred = clf.predict(X_test) y_pred_probs = list(clf.predict_proba(X_test)[:, 1]) end_time = time.time() elapsed_time = end_time - start_time y_pred_probs_sorted, y_test_sorted = zip( *sorted(zip(y_pred_probs, y_test), reverse=True)) results_df.loc[len(results_df)] = [ models_to_run[index], clf, p, elapsed_time, roc_auc_score(y_test, y_pred_probs), accuracy_score(y_test, y_pred), precision_at_k(y_test_sorted, y_pred_probs_sorted, 5.0), precision_at_k(y_test_sorted, y_pred_probs_sorted, 10.0), precision_at_k(y_test_sorted, y_pred_probs_sorted, 30.0), precision_at_k(y_test_sorted, y_pred_probs_sorted, 50.0), recall_at_k(y_test_sorted, y_pred_probs_sorted, 5.0), recall_at_k(y_test_sorted, y_pred_probs_sorted, 10.0), recall_at_k(y_test_sorted, y_pred_probs_sorted, 30.0), recall_at_k(y_test_sorted, y_pred_probs_sorted, 50.0), accuracy_at_k(y_test_sorted, y_pred_probs_sorted, 5.0), accuracy_at_k(y_test_sorted, y_pred_probs_sorted, 10.0), accuracy_at_k(y_test_sorted, y_pred_probs_sorted, 30.0), accuracy_at_k(y_test_sorted, y_pred_probs_sorted, 50.0), f1_at_k(y_test_sorted, y_pred_probs_sorted, 5.0), f1_at_k(y_test_sorted, y_pred_probs_sorted, 10.0), f1_at_k(y_test_sorted, y_pred_probs_sorted, 30.0), f1_at_k(y_test_sorted, y_pred_probs_sorted, 50.0), get_feature_importance(clf, models_to_run[index]), list(X_train.columns.values) ] except IndexError: print("IndexError") continue return results_df
def optimize(self, data, search_space, val_data=None): """ Parameters: ----------- data: [X, Y] - arrays This data will be used for crossval training (considering 'train_test_split' parameter) search_space: dict Dict with parameters to optimize. E.g. 'units' : [[1000,1000,500], [2500,1000]] val_data: [X, Y] - arrays Default - None. If specified than optimizer metric will be evaluated on val_data. Also if specified than 'train_test_split' parameter will be ignored. """ train_manager = self._create_train_manager(data, val_data, search_space) param_grid = ParameterGrid(search_space) all_params = [] for p in param_grid: all_params.append(p) for key in search_space.keys(): if (isinstance(search_space[key], dict)): new_params = [] for param in all_params: if (search_space[key][param[key]] is None): new_params.append(param) else: param_grid = ParameterGrid( search_space[key][param[key]]) add_params = [p for p in param_grid] for aparam in add_params: tparam = copy.copy(param) tparam.update(aparam) new_params.append(tparam) all_params = new_params for param in all_params: param_to_pass = self._preprocess_params(param) train_manager.train(**param)
def generateParams(): # Set the parameters by cross-validation paramaters_grid = {'eta': [0.05], 'min_child_weight' : [4], 'colsample_bytree' : [0.8], 'subsample' : [0.90], 'gamma' : [0], 'max_depth' : [12]}; paramaters_search = list(ParameterGrid(paramaters_grid)); parameters_to_try = []; for ps in paramaters_search: params = {'eval_metric' : 'mlogloss', 'objective' : 'multi:softprob', 'num_class' : 9, 'nthread' : 8}; for param in ps.keys(): params[str(param)] = ps[param]; parameters_to_try.append(copy.copy(params)); return parameters_to_try;
def pipeline(self): if self.param_dist: grid_size = len(ParameterGrid(self.param_dist)) if grid_size < self.n_iter_search: classifier = GridSearchCV(self.classifier, self.param_dist) else: classifier = RandomizedSearchCV( self.classifier, param_distributions=self.param_dist, n_iter=self.n_iter_search, random_state=self.random_state) else: classifier = self.classifier return Pipeline(self.transformers + [('classifier', classifier)])