def main(): param_space = { 'C': [1e-4, 1, 1e4], 'gamma': [1e-3, 1, 1e3], 'class_weight': [None, 'balanced'] } model = SVC(kernel='rbf') digits = load_digits() X_train, X_test, y_train, y_test = tts(digits.data, digits.target, test_size=0.3) print("Starting local cluster") cluster = LocalCluster() client = Client(cluster) print(client) print("Start searching") search = GridSearchCV(model, param_space, cv=3) search.fit(X_train, y_train) print("Prepare report") print( classification_report(y_true=y_test, y_pred=search.best_estimator_.predict(X_test)))
def getprobabilities(X, y, p_grid, cv, ac_sens): """getprobabilities(X,y,p_grid, cv, ac_sens) - X and y: Inputs and outputs - p_grid: grid of parameters to search over - cv: Number of cross-validation folds (this is different from the outer number of x-val folds) Gets the probability of picking each of the options provided by p_grid given the data in X and y. The algorithm is as follows: - Find the sensitivity of the SSE for each parameter-values - Find the SSE of each parameter-values - Find the probability of selecting those parameter-values """ kern = GPy.kern.RBF(2.0, lengthscale=25.0, variance=1.0) errorlimit = ac_sens * 4.0 ####find sensitivity of the SSE (for each param combo) #this call gets the sensivities not the scores: #TODO This probably should be done locally as it's quick. clf = GridSearchCV(estimator=DPCloaking(sensitivity=ac_sens, inducing=4, getxvalfoldsensitivities=True, kern=kern, errorlimit=errorlimit), param_grid=p_grid, cv=cv) clf.fit(X, y) nparamcombos = len(clf.cv_results_['mean_test_score']) temp_sens = np.zeros([clf.cv, nparamcombos]) for k in range(clf.cv): temp_sens[k, :] = clf.cv_results_['split%d_test_score' % k] #sensitivity of the sum squared error: print(np.sort(temp_sens, axis=0)) sse_sens = ac_sens**2 + 2 * ac_sens * errorlimit + ac_sens**2 * np.max( np.sum(np.sort(temp_sens, axis=0)[0:clf.cv - 1, :], 0)) ####find the SSE (for each param combo) clf = GridSearchCV(estimator=DPCloaking(sensitivity=ac_sens, inducing=4, getxvalfoldsensitivities=False, kern=kern, errorlimit=errorlimit), param_grid=p_grid, cv=cv) clf.fit(X, y) nparamcombos = len(clf.cv_results_['mean_test_score']) temp_scores = np.zeros([clf.cv, nparamcombos]) for k in range(clf.cv): temp_scores[k, :] = clf.cv_results_['split%d_test_score' % k] scores = np.sum(temp_scores, 0) ####compute the probability of selecting that param combo using the exponential mechanism selection_epsilon = 1 param_probabilities = np.exp(selection_epsilon * scores / (2 * sse_sens)) param_probabilities = param_probabilities / np.sum(param_probabilities) return param_probabilities
def getscores(X, y, p_grid, cv, ac_sens): """ Compute the negative RMSE of each of the fold/param combos """ kern = GPy.kern.RBF(2.0, lengthscale=25.0, variance=1.0) clf = GridSearchCV(estimator=DPCloaking(sensitivity=ac_sens, inducing=4, getxvalfoldsensitivities=False, kern=kern), scoring='neg_mean_squared_error', param_grid=p_grid, cv=cv) clf.fit(X, y) nparamcombos = len(clf.cv_results_['mean_test_score']) scores = np.zeros([clf.cv, nparamcombos]) for k in range(clf.cv): scores[k, :] = clf.cv_results_['split%d_test_score' % k] return scores
def grid_search(self, X, y, parameters, scoring=None, cv=5, refit=True, verbose=False): '''Perform an exhaustive search over hyperparameter combinations. # Arguments X: np.ndarray, features y: np.ndarray, labels parameters: dict, hyperparameter ranges scoring: dict, scoring functions e.g. {'acc': accuracy_score, ...} refit: bool, fit an estimator with the best parameters if True verbose: int, controls the verbosity: the higher, the more messages ''' self.grid_search_parameters = { 'estimator__estimator__' + k: v for k, v in parameters.items() } clf = self.clf if verbose is not True: warnings.filterwarnings("ignore", category=UserWarning) self.clf_grid_search = GridSearchCV(clf, self.grid_search_parameters, cv=cv, scoring=scoring, refit=refit, n_jobs=self.n_jobs) self.clf_grid_search.fit(X, y) print('\n`clf.best_estimator_`:\n', self.clf_grid_search.best_estimator_, '\n', sep='')
def main(): param_space = { "C": [1e-4, 1, 1e4], "gamma": [1e-3, 1, 1e3], "class_weight": [None, "balanced"], "kernel": ["linear", "poly", "rbf", "sigmoid"] } model = SVC() # param_space = {"n_estimators":[100, 200, 300, 400, 500], # "criterion":["gini", "entropy"], # "max_features":["auto", "sqrt", "log2"], # "max_depth":[2, 3, 4, 5, 6, 7, 8]} # model = RandomForestClassifier() digits = load_digits() # classifier = GridSearchCV(model, param_space, n_jobs=-1, cv=5) classifier = GridSearchCV(model, param_space, n_jobs=-1, cv=5) classifier.fit(digits.data, digits.target) print("Grid Scores:") means = classifier.cv_results_["mean_test_score"] standard_deviations = classifier.cv_results_["std_test_score"] for mean, standard_deviation, parameter in zip( means, standard_deviations, classifier.cv_results_["params"]): print("%0.3f (+/-%0.03f) for %r" % (mean, standard_deviation * 2, parameter)) print() print("Best Score: %0.3f" % (classifier.best_score_)) print() print("Best Parameters:") print(classifier.best_params_) print()
def evaluate_classifier(X, y, list_of_queries, set_k_range, k_function, alpha_range, l1_ratio): ''' Run a classifier setup on a set of queries. Loop through each query; train and test the classifier using the hyperparameters input as parameters; populate the metrics dictionary with some metrics of which parameters were selected and how well the classifier did for that query. ''' # A dictionary to hold the performance metrics. metrics_dict = {} # Loop through each query; train and test the classifer; populate the metrics dictionary. for query in list_of_queries: num_samples = query[2]['total'] num_positives = query[2]['positive'] # Subset by gene. y_query = y[query[0]] # Subset by diseases. disease_cols = [col for col in covariates.columns if col.endswith(tuple(query[1]))] has_disease = covariates[disease_cols].max(axis=1) > 0 covariates_query = covariates[has_disease] X_query = X[X.index.isin(covariates_query.index)] y_query = y_query[y_query.index.isin(covariates_query.index)] # Test Train split test_size = 0.2 X_train, X_test, y_train, y_test = train_test_split(X_query, y_query, stratify=y_query, test_size=test_size, random_state=RANDOMSEED) # PCA. scaler = StandardScaler() if query[2]['total']*(1-test_size)*(1-(1/3)) > 350: n_comp = 350 else: n_comp = int(query[2]['total']*(1-test_size) - 1) pca = PCA(n_components = n_comp, random_state = RANDOMSEED) scaler.fit(X_train) X_train_scaled = scaler.transform(X_train) pca.fit(X_train_scaled) X_train = pca.transform(X_train_scaled) X_test_scaled = scaler.transform(X_test) X_test = pca.transform(X_test_scaled) if set_k_range: k_range = set_k_range else: k_range = k_function(num_samples=num_samples, num_positives=num_positives, ) # Parameter Sweep for Hyperparameters param_grid = { 'select__k': k_range, 'classify__loss': ['log'], 'classify__penalty': ['elasticnet'], 'classify__alpha': alpha_range, 'classify__l1_ratio': l1_ratio, } pipeline = Pipeline(steps=[ ('select', SelectKBest(variance_scorer)), ('classify', SGDClassifier(random_state=RANDOMSEED, class_weight='balanced')) ]) cv_pipeline = GridSearchCV(estimator=pipeline, param_grid=param_grid, n_jobs=1, scoring='roc_auc') cv_pipeline.fit(X=X_train, y=y_train) y_pred_train = cv_pipeline.decision_function(X_train) y_pred_test = cv_pipeline.decision_function(X_test) # Get ROC info. def get_threshold_metrics(y_true, y_pred): roc_columns = ['fpr', 'tpr', 'threshold'] roc_items = zip(roc_columns, roc_curve(y_true, y_pred)) roc_df = pd.DataFrame.from_items(roc_items) auroc = roc_auc_score(y_true, y_pred) return {'auroc': auroc, 'roc_df': roc_df} metrics_train = get_threshold_metrics(y_train, y_pred_train) metrics_test = get_threshold_metrics(y_test, y_pred_test) # Populate the metrics dictionary. # Get metrics for the classifier. overfit = metrics_train['auroc'] - metrics_test['auroc'] # Understand how the parameter grid worked... any params at the edge? if cv_pipeline.best_params_['select__k'] == min(param_grid['select__k']): n_comp_status = 'min' elif cv_pipeline.best_params_['select__k'] == max(param_grid['select__k']): n_comp_status = 'max' else: n_comp_status = 'OK' if cv_pipeline.best_params_['classify__alpha'] == min(param_grid['classify__alpha']): alpha_status = 'min' elif cv_pipeline.best_params_['classify__alpha'] == max(param_grid['classify__alpha']): alpha_status = 'max' else: alpha_status = 'OK' metrics = {'num_samples': num_samples, 'num_positive': num_positives, 'balance': num_positives/num_samples, 'train_auroc': metrics_train['auroc'], 'test_auroc': metrics_test['auroc'], 'n_components': cv_pipeline.best_params_['select__k'], 'alpha': cv_pipeline.best_params_['classify__alpha'], 'overfit': overfit, 'n_comp_status': n_comp_status, 'alpha_status': alpha_status } # Add the metrics to the dictonary. metrics_dict[query[0]+str(query[2]['total'])] = metrics # Change the metrics dict into a formatted pandas dataframe. metrics_df = pd.DataFrame(metrics_dict) metrics_df = metrics_df.T metrics_df.sort_values(by='num_positive', ascending=True, inplace=True) metrics_df = metrics_df[['num_samples', 'num_positive', 'balance', 'n_components','n_comp_status', 'alpha', 'alpha_status','train_auroc', 'test_auroc', 'overfit']] return(metrics_df)
stratify=strat) clf_parameters = { 'classify__loss': ['log'], 'classify__penalty': ['elasticnet'], 'classify__alpha': alphas, 'classify__l1_ratio': l1_ratios } estimator = Pipeline(steps=[( 'classify', SGDClassifier(random_state=0, class_weight='balanced', loss='log'))]) cv_pipeline = GridSearchCV(estimator=estimator, param_grid=clf_parameters, n_jobs=-1, cv=folds, scoring='roc_auc') cv_pipeline.fit(X=x_train, y=y_train) cv_results = pd.concat([ pd.DataFrame(cv_pipeline.cv_results_).drop('params', axis=1), pd.DataFrame.from_records(cv_pipeline.cv_results_['params']) ], axis=1) # Cross-validated performance heatmap cv_score_mat = pd.pivot_table(cv_results, values='mean_test_score', index='classify__l1_ratio', columns='classify__alpha')
'expressions': Pipeline([('features', FeatureUnion([('expressions', expression_features) ])), ('classify', classifier)]), 'covariates': Pipeline([('features', FeatureUnion([('covariates', covariate_features)])), ('classify', classifier)]) } # Construct cross-validated grid searches cv_pipelines = dict() for model, pipeline in pipeline_definitions.items(): cv = StratifiedKFold(n_splits=3, random_state=0) grid_search = GridSearchCV( estimator=pipeline, param_grid=param_grids[model], cv=cv, n_jobs=1, scoring='roc_auc', ) cv_pipelines[model] = grid_search # In[13]: # Fit the models for model, pipeline in cv_pipelines.items(): print('Fitting CV for model: {0}'.format(model)) start_time = time.perf_counter() pipeline.fit(X=X_train, y=y_train) end_time = time.perf_counter() elapsed = datetime.timedelta(seconds=end_time - start_time) print('\truntime: {}'.format(elapsed))
def evaluate_classifier(X_train, X_test, y, y_train_allgenes, y_test_allgenes, list_of_genes, set_k_range, k_function, alpha_range, l1_ratio): ''' Run a classifier setup on a set of queries. Loop through each query; train and test the classifier using the hyperparameters input as parameters; populate the metrics dictionary with some metrics of which parameters were selected and how well the classifier did for that query. ''' # A dictionary to hold the performance metrics. metrics_dict = {} # Loop through each query; train and test the classifer; populate the metrics dictionary. for gene in list_of_genes: # Train and test the classifier. y_gene = y[gene] y_train = y_train_allgenes[gene] y_test = y_test_allgenes[gene] num_positives = int(y_gene.value_counts(True)[1] * len(y_gene)) if set_k_range: k_range = set_k_range else: k_range = k_function(num_positives) # Parameter Sweep for Hyperparameters param_grid = { 'select__k': k_range, 'classify__loss': ['log'], 'classify__penalty': ['elasticnet'], 'classify__alpha': alpha_range, 'classify__l1_ratio': l1_ratio, } pipeline = Pipeline(steps=[('select', SelectKBest(variance_scorer)), ('classify', SGDClassifier(random_state=RANDOMSEED, class_weight='balanced'))]) cv_pipeline = GridSearchCV(estimator=pipeline, param_grid=param_grid, n_jobs=1, scoring='roc_auc') cv_pipeline.fit(X=X_train, y=y_train) y_pred_train = cv_pipeline.decision_function(X_train) y_pred_test = cv_pipeline.decision_function(X_test) # Get ROC info. def get_threshold_metrics(y_true, y_pred): roc_columns = ['fpr', 'tpr', 'threshold'] roc_items = zip(roc_columns, roc_curve(y_true, y_pred)) roc_df = pd.DataFrame.from_items(roc_items) auroc = roc_auc_score(y_true, y_pred) return {'auroc': auroc, 'roc_df': roc_df} metrics_train = get_threshold_metrics(y_train, y_pred_train) metrics_test = get_threshold_metrics(y_test, y_pred_test) # Populate the metrics dictionary. # Get metrics for the classifier. overfit = metrics_train['auroc'] - metrics_test['auroc'] # Understand how the parameter grid worked... any params at the edge? if cv_pipeline.best_params_['select__k'] == min( param_grid['select__k']): n_comp_status = 'min' elif cv_pipeline.best_params_['select__k'] == max( param_grid['select__k']): n_comp_status = 'max' else: n_comp_status = 'OK' if cv_pipeline.best_params_['classify__alpha'] == min( param_grid['classify__alpha']): alpha_status = 'min' elif cv_pipeline.best_params_['classify__alpha'] == max( param_grid['classify__alpha']): alpha_status = 'max' else: alpha_status = 'OK' metrics = { 'num_positive': num_positives, 'train_auroc': metrics_train['auroc'], 'test_auroc': metrics_test['auroc'], 'n_components': cv_pipeline.best_params_['select__k'], 'alpha': cv_pipeline.best_params_['classify__alpha'], 'overfit': overfit, 'n_comp_status': n_comp_status, 'alpha_status': alpha_status } # Add the metrics to the dictonary. metrics_dict[gene] = metrics # Change the metrics dict into a formatted pandas dataframe. metrics_df = pd.DataFrame(metrics_dict) metrics_df = metrics_df.T metrics_df.sort_values(by='num_positive', ascending=True, inplace=True) metrics_df = metrics_df[[ 'num_positive', 'n_components', 'n_comp_status', 'alpha', 'alpha_status', 'train_auroc', 'test_auroc', 'overfit' ]] return (metrics_df)
def evaluate_classifier(X_train, X_test, y, y_train_allgenes, y_test_allgenes, list_of_genes, set_k_range, k_function, alpha_range, l1_ratio): ''' Run a classifier setup on a set of queries. Loop through each query; train and test the classifier using the hyperparameters input as parameters; populate the metrics dictionary with some metrics of which parameters were selected and how well the classifier did for that query. ''' # A dictionary to hold the performance metrics. metrics_dict = {} # Loop through each query; train and test the classifer; populate the metrics dictionary. for gene in list_of_genes: # Train and test the classifier. y_gene = y[gene] y_train = y_train_allgenes[gene] y_test = y_test_allgenes[gene] num_positives = int(y_gene.value_counts(True)[1]*len(y_gene)) if set_k_range: k_range = set_k_range else: k_range = k_function(num_positives) # Parameter Sweep for Hyperparameters param_grid = { 'select__k': k_range, 'classify__loss': ['log'], 'classify__penalty': ['elasticnet'], 'classify__alpha': alpha_range, 'classify__l1_ratio': l1_ratio, } pipeline = Pipeline(steps=[ ('select', SelectKBest(variance_scorer)), ('classify', SGDClassifier(random_state=RANDOMSEED, class_weight='balanced')) ]) cv_pipeline = GridSearchCV(estimator=pipeline, param_grid=param_grid, n_jobs=1, scoring='roc_auc') cv_pipeline.fit(X=X_train, y=y_train) y_pred_train = cv_pipeline.decision_function(X_train) y_pred_test = cv_pipeline.decision_function(X_test) # Get ROC info. def get_threshold_metrics(y_true, y_pred): roc_columns = ['fpr', 'tpr', 'threshold'] roc_items = zip(roc_columns, roc_curve(y_true, y_pred)) roc_df = pd.DataFrame.from_items(roc_items) auroc = roc_auc_score(y_true, y_pred) return {'auroc': auroc, 'roc_df': roc_df} metrics_train = get_threshold_metrics(y_train, y_pred_train) metrics_test = get_threshold_metrics(y_test, y_pred_test) # Populate the metrics dictionary. # Get metrics for the classifier. overfit = metrics_train['auroc'] - metrics_test['auroc'] # Understand how the parameter grid worked... any params at the edge? if cv_pipeline.best_params_['select__k'] == min(param_grid['select__k']): n_comp_status = 'min' elif cv_pipeline.best_params_['select__k'] == max(param_grid['select__k']): n_comp_status = 'max' else: n_comp_status = 'OK' if cv_pipeline.best_params_['classify__alpha'] == min(param_grid['classify__alpha']): alpha_status = 'min' elif cv_pipeline.best_params_['classify__alpha'] == max(param_grid['classify__alpha']): alpha_status = 'max' else: alpha_status = 'OK' metrics = {'num_positive': num_positives, 'train_auroc': metrics_train['auroc'], 'test_auroc': metrics_test['auroc'], 'n_components': cv_pipeline.best_params_['select__k'], 'alpha': cv_pipeline.best_params_['classify__alpha'], 'overfit': overfit, 'n_comp_status': n_comp_status, 'alpha_status': alpha_status } # Add the metrics to the dictonary. metrics_dict[gene] = metrics # Change the metrics dict into a formatted pandas dataframe. metrics_df = pd.DataFrame(metrics_dict) metrics_df = metrics_df.T metrics_df.sort_values(by='num_positive', ascending=True, inplace=True) metrics_df = metrics_df[['num_positive', 'n_components','n_comp_status', 'alpha', 'alpha_status','train_auroc', 'test_auroc', 'overfit']] return(metrics_df)
if(model_choice == 'random forests'): print("\n\n") print("\t\t Special Grid Search for random foreset") print("\n\n") c = Client(scheduler_address, set_as_default=True) grid_search = DaskRandomizedSearchCV( model, param_grid, cv = cv_temp, get= c.get ) else: c = Client(scheduler_address, set_as_default=True) grid_search = GridSearchCV( model, param_grid, cv = cv_temp ) grid_search.fit(X, y) time_elapse = time()-t0 runing_time.append([model_choice, cv_temp, time_elapse]) print(" running time: ", time_elapse) num_graph = len(grid_search.dask_graph_) print(" size of graph: ", num_graph) runing_time_df = pd.DataFrame(data = runing_time, columns = ['model', 'cv', 'time']) runing_time_df['n_workers'] = n_workers runing_time_df['n_graph'] = num_graph runing_time_df['sample'] = sample
class Classifier(Base): '''Classifier base class. ''' def __init__(self, clf, scale=False, n_jobs=1): super().__init__() # A classifier is built using a `Pipeline` for convenience of chaining # multiple preprocessing steps before the classifier pipeline = [] # Centre data by scaling to zero mean and unit variance if scale is True: pipeline.append(('standard_scaler', StandardScaler())) # Add the `clf` estimator and build the `Pipeline` pipeline.append(('estimator', clf)) self.clf = Pipeline(pipeline) self.n_jobs = n_jobs def __name__(self): return self.__class__.__name__ def grid_search(self, X, y, parameters, scoring=None, cv=5, refit=True, verbose=False): '''Perform an exhaustive search over hyperparameter combinations. # Arguments X: np.ndarray, features y: np.ndarray, labels parameters: dict, hyperparameter ranges scoring: dict, scoring functions e.g. {'acc': accuracy_score, ...} refit: bool, fit an estimator with the best parameters if True verbose: int, controls the verbosity: the higher, the more messages ''' self.grid_search_parameters = { 'estimator__estimator__' + k: v for k, v in parameters.items() } clf = self.clf if verbose is not True: warnings.filterwarnings("ignore", category=UserWarning) self.clf_grid_search = GridSearchCV(clf, self.grid_search_parameters, cv=cv, scoring=scoring, refit=refit, n_jobs=self.n_jobs) self.clf_grid_search.fit(X, y) print('\n`clf.best_estimator_`:\n', self.clf_grid_search.best_estimator_, '\n', sep='') def fit(self, X, y): '''Fit the estimator. # Arguments X: np.ndarray, features y: np.ndarray, labels ''' # Fit classifier using the best parameters from GridSearchCV try: getattr(self.clf_grid_search, 'best_estimator_') fit_using = "clf_grid_search" except AttributeError: # Fit classifier from __init__ fit_using = "clf" self.clf.fit(X, y) finally: print(f'\nFit using `{fit_using}`') def get_clf(self): '''Get the best estimator. If a grid search has been performed, then the `best_estimator_` is returned, else the estimator used to initialise the object is returned. # Returns clf: sklearn estimator ''' try: return self.clf_grid_search.best_estimator_ except AttributeError: return self.clf def predict(self, X): '''Predict the classes of samples using features. # Arguments X: np.ndarray, features # Returns predictions: np.ndarray, class predictions ''' self.predictions = self.get_clf().predict(X) return self.predictions def predict_proba(self, X): '''Predict the class-membership probabilities of samples. # Arguments X: np.ndarray, features # Returns probabilities: np.ndarray, class probabilities ''' self.probabilities = self.get_clf().predict_proba(X) return self.probabilities def decision_function(self, X): '''Decision function. # Arguments X: np.ndarray, features # Returns decisions: np.ndarray, distances of samples to the decision boundary ''' try: self.decisions = self.get_clf().decision_function(X) return self.decisions except AttributeError as err: raise AttributeError( f'decision_function is not implemented for {self.__name__()}')\ from None def score(self, X, y): '''Mean accuracy score on test data. # Arguments X: np.ndarray, test features y: np.ndarray, test labels ''' return self.get_clf().score(X, y) def accuracy(self, y_true): '''Accuracy score. # Arguments y_true: np.ndarry, true labels # Returns accuracy_score: float ''' self.accuracy_score = accuracy_score(y_true, self.predictions) return self.accuracy_score
clf_parameters = {'classify__C': cs, 'classify__penalty': penalties} estimator = Pipeline(steps=[('classify', LogisticRegression(random_state=123, class_weight='balanced', multi_class='ovr', max_iter=100, solver='saga'))]) # Custom scorer that optimizes f1 score weighted by class proportion weighted_f1_scorer = make_scorer(f1_score, average='weighted') # Cross validation pipeline cv_pipeline = GridSearchCV(estimator=estimator, param_grid=clf_parameters, n_jobs=-1, cv=5, return_train_score=True, scoring=weighted_f1_scorer) # ### Fit Model # # This takes a couple minutes to train. For many model parameters, sklearn will throw convergence warnings. This means that after 100 iterations, an optimal solution is not found. Prevent the warnings from being printed redundantly. # In[8]: get_ipython().run_cell_magic( 'time', '', 'with warnings.catch_warnings():\n warnings.simplefilter("ignore")\n cv_pipeline.fit(X=x_df, y=y_df.ras_status)' ) # In[9]:
'expressions': Pipeline([('features', FeatureUnion([('expressions', expression_features) ])), ('classify', classifier)]), 'covariates': Pipeline([('features', FeatureUnion([('covariates', covariate_features)])), ('classify', classifier)]) } # Construct cross-validated grid searches cv_pipelines = dict() for model, pipeline in pipeline_definitions.items(): cv = StratifiedKFold(n_splits=3, random_state=0) grid_search = GridSearchCV( estimator=pipeline, param_grid=param_grids[model], cv=cv, n_jobs=1, scoring='roc_auc', ) cv_pipelines[model] = grid_search # In[13]: # Fit the models for model, pipeline in cv_pipelines.items(): print('Fitting CV for model: {0}'.format(model)) start_time = time.perf_counter() pipeline.fit(X=X_train, y=y_train) end_time = time.perf_counter() elapsed = datetime.timedelta(seconds=end_time - start_time) print('\truntime: {}'.format(elapsed))
def evaluate_classifier(X, y, list_of_queries, set_k_range, k_function, alpha_range, l1_ratio): ''' Run a classifier setup on a set of queries. Loop through each query; train and test the classifier using the hyperparameters input as parameters; populate the metrics dictionary with some metrics of which parameters were selected and how well the classifier did for that query. ''' # A dictionary to hold the performance metrics. metrics_dict = {} # Loop through each query; train and test the classifer; populate the metrics dictionary. for query in list_of_queries: num_samples = query[2]['total'] num_positives = query[2]['positive'] # Subset by gene. y_query = y[query[0]] # Subset by diseases. disease_cols = [ col for col in covariates.columns if col.endswith(tuple(query[1])) ] has_disease = covariates[disease_cols].max(axis=1) > 0 covariates_query = covariates[has_disease] X_query = X[X.index.isin(covariates_query.index)] y_query = y_query[y_query.index.isin(covariates_query.index)] # Test Train split test_size = 0.2 X_train, X_test, y_train, y_test = train_test_split( X_query, y_query, stratify=y_query, test_size=test_size, random_state=RANDOMSEED) # PCA. scaler = StandardScaler() if query[2]['total'] * (1 - test_size) * (1 - (1 / 3)) > 350: n_comp = 350 else: n_comp = int(query[2]['total'] * (1 - test_size) - 1) pca = PCA(n_components=n_comp, random_state=RANDOMSEED) scaler.fit(X_train) X_train_scaled = scaler.transform(X_train) pca.fit(X_train_scaled) X_train = pca.transform(X_train_scaled) X_test_scaled = scaler.transform(X_test) X_test = pca.transform(X_test_scaled) if set_k_range: k_range = set_k_range else: k_range = k_function( num_samples=num_samples, num_positives=num_positives, ) # Parameter Sweep for Hyperparameters param_grid = { 'select__k': k_range, 'classify__loss': ['log'], 'classify__penalty': ['elasticnet'], 'classify__alpha': alpha_range, 'classify__l1_ratio': l1_ratio, } pipeline = Pipeline(steps=[('select', SelectKBest(variance_scorer)), ('classify', SGDClassifier(random_state=RANDOMSEED, class_weight='balanced'))]) cv_pipeline = GridSearchCV(estimator=pipeline, param_grid=param_grid, n_jobs=1, scoring='roc_auc') cv_pipeline.fit(X=X_train, y=y_train) y_pred_train = cv_pipeline.decision_function(X_train) y_pred_test = cv_pipeline.decision_function(X_test) # Get ROC info. def get_threshold_metrics(y_true, y_pred): roc_columns = ['fpr', 'tpr', 'threshold'] roc_items = zip(roc_columns, roc_curve(y_true, y_pred)) roc_df = pd.DataFrame.from_items(roc_items) auroc = roc_auc_score(y_true, y_pred) return {'auroc': auroc, 'roc_df': roc_df} metrics_train = get_threshold_metrics(y_train, y_pred_train) metrics_test = get_threshold_metrics(y_test, y_pred_test) # Populate the metrics dictionary. # Get metrics for the classifier. overfit = metrics_train['auroc'] - metrics_test['auroc'] # Understand how the parameter grid worked... any params at the edge? if cv_pipeline.best_params_['select__k'] == min( param_grid['select__k']): n_comp_status = 'min' elif cv_pipeline.best_params_['select__k'] == max( param_grid['select__k']): n_comp_status = 'max' else: n_comp_status = 'OK' if cv_pipeline.best_params_['classify__alpha'] == min( param_grid['classify__alpha']): alpha_status = 'min' elif cv_pipeline.best_params_['classify__alpha'] == max( param_grid['classify__alpha']): alpha_status = 'max' else: alpha_status = 'OK' metrics = { 'num_samples': num_samples, 'num_positive': num_positives, 'balance': num_positives / num_samples, 'train_auroc': metrics_train['auroc'], 'test_auroc': metrics_test['auroc'], 'n_components': cv_pipeline.best_params_['select__k'], 'alpha': cv_pipeline.best_params_['classify__alpha'], 'overfit': overfit, 'n_comp_status': n_comp_status, 'alpha_status': alpha_status } # Add the metrics to the dictonary. metrics_dict[query[0] + str(query[2]['total'])] = metrics # Change the metrics dict into a formatted pandas dataframe. metrics_df = pd.DataFrame(metrics_dict) metrics_df = metrics_df.T metrics_df.sort_values(by='num_positive', ascending=True, inplace=True) metrics_df = metrics_df[[ 'num_samples', 'num_positive', 'balance', 'n_components', 'n_comp_status', 'alpha', 'alpha_status', 'train_auroc', 'test_auroc', 'overfit' ]] return (metrics_df)
'classify__loss': ['log'], 'classify__penalty': ['elasticnet'], 'classify__alpha': alphas, 'classify__l1_ratio': l1_ratios } estimator = Pipeline(steps=[('classify', SGDClassifier(random_state=0, class_weight='balanced', loss='log', max_iter=50, tol=1e-3))]) cv_pipeline = GridSearchCV(estimator=estimator, param_grid=clf_parameters, n_jobs=-1, cv=n_folds, scoring='average_precision', return_train_score=True) shuffle_cv_pipeline = GridSearchCV(estimator=estimator, param_grid=clf_parameters, n_jobs=-1, cv=n_folds, scoring='average_precision', return_train_score=True) # In[10]: # Fit Regular Pipeline cv_pipeline.fit(X=x_train_df, y=y_train_df.status.tolist())