def _do_outer_cv(searcher, X, y, outer_cv, scoring, error_score="raise", outfile=None): """Do outer cross-validation for nested CV Parameters ---------- searcher : object SearchCV object X : numpy array Containing features y : numpy array Target values or labels outer_cv : int or CV splitter Control the cv splitting scoring : object Scorer error_score: str, float or numpy float Whether to raise fit error or return an value outfile : str File path to store the restuls """ if error_score == "raise": rval = cross_validate( searcher, X, y, scoring=scoring, cv=outer_cv, n_jobs=N_JOBS, verbose=0, error_score=error_score, ) else: warnings.simplefilter("always", FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: rval = cross_validate( searcher, X, y, scoring=scoring, cv=outer_cv, n_jobs=N_JOBS, verbose=0, error_score=error_score, ) except ValueError: pass for warning in w: print(repr(warning.message)) keys = list(rval.keys()) for k in keys: if k.startswith("test"): rval["mean_" + k] = np.mean(rval[k]) rval["std_" + k] = np.std(rval[k]) if k.endswith("time"): rval.pop(k) rval = pd.DataFrame(rval) rval = rval[sorted(rval.columns)] rval.to_csv(path_or_buf=outfile, sep="\t", header=True, index=False)
def _fit(self, ds, cv_attr=None): """General method to fit data""" self.scoring, _ = _check_multimetric_scoring(self.estimator, scoring=self.scoring) X, y = get_ds_data(ds) y = LabelEncoder().fit_transform(y) indices = self._get_permutation_indices(len(y)) values = [] groups = None if cv_attr != None: groups = LabelEncoder().fit_transform(ds.sa[cv_attr].value) for idx in indices: y_ = y[idx] scores = cross_validate(self.estimator, X, y_, groups, self.scoring, self.cv, self.n_jobs, self.verbose, return_estimator=True, return_splits=True) values.append(scores) return values
def task3(): df = pd.read_csv('people.csv') # This should go in the clean data function but I wanted to show it explicitly df['age'] = pd.to_numeric(df['age'], errors='coerce') df['age'] = df['age'].fillna(round(df['age'].mean())) df['age'] = df['age'].astype(int) df = df.dropna() clean_data(df) encode_data(df) older_females = df[(df['Gender'] == 'Female') & (df['age'] > 30)] # Dataset for females older than 30 X = older_females[['education', 'age', 'job']] y = older_females[['workType']].values.ravel() models = [] models.append(('Decision Tree Classifier', DecisionTreeClassifier())) models.append(('Naive Bayes', GaussianNB())) models.append(('Random Forest Classifier', RandomForestClassifier())) results = {} for name, model in models: cv_results = cross_validate( model, X, y, cv=5, scoring='accuracy', return_train_score=True) # Setting Cross Validation to 5 results[name] = cv_results for models in results: print(models) print('Training:', results[models]['train_score'].mean()) print('Test: ', results[models]['test_score'].mean()) labels = [ 'Decision Tree Classifier', 'Naive Bayes', 'Random Forest Classifier' ] train_means = [results[models]['train_score'].mean() for models in results] test_means = [results[models]['test_score'].mean() for models in results] x = np.arange(len(labels)) width = 0.35 fig, ax = plt.subplots() ax.bar(x - width / 2, train_means, width, label="Training Accuracy") rects2 = ax.bar(x + width / 2, test_means, width, label="Testing Accuracy") ax.set_xlabel('Classifier') ax.set_ylabel('Average Accuracy') ax.set_title('Accuracy Scores') ax.set_xticks(x) ax.set_xticklabels(labels) ax.legend() plt.show() '''
def fit_and_save(estimator, X, y=None, groups=None, scoring=None, cv=None, n_jobs=1, verbose=0, fit_params=None, pre_dispatch='2*n_jobs', return_train_score=True, parameters=dict(), uuid='', url='http://127.0.0.1:8000'): import json, requests, numpy from sklearn.model_selection._validation import cross_validate X, y, groups = indexable(X, y, groups) cv = check_cv(cv, y, classifier=is_classifier(estimator)) scorers, _ = _check_multimetric_scoring(estimator, scoring=scoring) _base_scores = [0. for _ in range(cv.get_n_splits(X, y, groups))] cv_score = {} cv_score.update( {'train_%s' % s: numpy.array(_base_scores) for s in scorers}) cv_score.update( {'test_%s' % s: numpy.array(_base_scores) for s in scorers}) cv_score.update({'fit_time': _base_scores, 'score_time': _base_scores}) try: cv_score = cross_validate(estimator, X, y, groups, scorers, cv, n_jobs, verbose, fit_params, pre_dispatch, return_train_score) error = None except Exception as e: error = '{}: {}'.format(type(e).__name__, str(e)) try: for k, v in cv_score.items(): if type(v) == type(numpy.array([])): cv_score[k] = v.tolist() response = requests.post('{url}/grids/{uuid}/results'.format( url=url, uuid=uuid), data={ 'gridsearch': uuid, 'params': json.dumps(parameters), 'errors': error, 'cv_data': json.dumps(cv_score) }) except requests.exceptions.ConnectionError as e: response = None if response is None: return return response
def main(): # Checks checkUsage() # Parameters k = 10 nbTrees = 100 dataSetPath = sys.argv[1] data = pd.read_json(dataSetPath) # Shuffle Data data = shuffle(data) # Random Forest Model classifierKFold = RandomForestClassifier(n_estimators=nbTrees, random_state=0, verbose=0) X = data.iloc[:, [1, 3, 6, 7, 8, 9, 10, 11, 12]].values y = data.iloc[:, 4].values # Cross validation, K = 10, using stratified folds scoring = { 'precision': make_scorer(precision), 'recall': make_scorer(recall), 'f1': make_scorer(f1), 'auc': make_scorer(auc), 'fpr': make_scorer(fpr), 'tnr': make_scorer(tnr), 'mcc': make_scorer(mcc) } scores = cross_validate(classifierKFold, X, y, cv=k, scoring=scoring, verbose=4) print("\nMetrics with K-fold:", k, ", nbTree:", nbTrees) displayScores(scores['test_precision'], "Precision") displayScores(scores['test_recall'], "Recall") displayScores(scores['test_f1'], "F1") displayScores(scores['test_tnr'], "TNR") displayScores(scores['test_fpr'], "FPR") displayScores(scores['test_auc'], "AUC") displayScores(scores['test_mcc'], "MCC")
def main(inputs, infile_estimator, infile1, infile2, outfile_result, outfile_object=None, outfile_weights=None, groups=None, ref_seq=None, intervals=None, targets=None, fasta_path=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str File path to estimator infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target values outfile_result : str File path to save the results, either cv_results or test result outfile_object : str, optional File path to save searchCV object outfile_weights : str, optional File path to save model weights groups : str File path to dataset containing groups labels ref_seq : str File path to dataset containing genome sequence file intervals : str File path to dataset containing interval file targets : str File path to dataset compressed target bed file fasta_path : str File path to dataset containing fasta file """ warnings.simplefilter('ignore') # store read dataframe object loaded_df = {} with open(inputs, 'r') as param_handler: params = json.load(param_handler) # Override the refit parameter params['search_schemes']['options']['refit'] = True \ if params['save'] != 'nope' else False with open(infile_estimator, 'rb') as estimator_handler: estimator = load_model(estimator_handler) optimizer = params['search_schemes']['selected_search_scheme'] optimizer = getattr(model_selection, optimizer) # handle gridsearchcv options options = params['search_schemes']['options'] if groups: header = 'infer' if ( options['cv_selector']['groups_selector']['header_g']) else None column_option = ( options['cv_selector']['groups_selector'] ['column_selector_options_g']['selected_column_selector_option_g']) if column_option in [ 'by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name' ]: c = (options['cv_selector']['groups_selector'] ['column_selector_options_g']['col_g']) else: c = None df_key = groups + repr(header) groups = pd.read_csv(groups, sep='\t', header=header, parse_dates=True) loaded_df[df_key] = groups groups = read_columns(groups, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) groups = groups.ravel() options['cv_selector']['groups_selector'] = groups splitter, groups = get_cv(options.pop('cv_selector')) options['cv'] = splitter primary_scoring = options['scoring']['primary_scoring'] options['scoring'] = get_scoring(options['scoring']) if options['error_score']: options['error_score'] = 'raise' else: options['error_score'] = np.NaN if options['refit'] and isinstance(options['scoring'], dict): options['refit'] = primary_scoring if 'pre_dispatch' in options and options['pre_dispatch'] == '': options['pre_dispatch'] = None params_builder = params['search_schemes']['search_params_builder'] param_grid = _eval_search_params(params_builder) estimator = clean_params(estimator) # save the SearchCV object without fit if params['save'] == 'save_no_fit': searcher = optimizer(estimator, param_grid, **options) print(searcher) with open(outfile_object, 'wb') as output_handler: pickle.dump(searcher, output_handler, pickle.HIGHEST_PROTOCOL) return 0 # read inputs and loads new attributes, like paths estimator, X, y = _handle_X_y(estimator, params, infile1, infile2, loaded_df=loaded_df, ref_seq=ref_seq, intervals=intervals, targets=targets, fasta_path=fasta_path) # cache iraps_core fits could increase search speed significantly memory = joblib.Memory(location=CACHE_DIR, verbose=0) main_est = get_main_estimator(estimator) if main_est.__class__.__name__ == 'IRAPSClassifier': main_est.set_params(memory=memory) searcher = optimizer(estimator, param_grid, **options) split_mode = params['outer_split'].pop('split_mode') if split_mode == 'nested_cv': # make sure refit is choosen # this could be True for sklearn models, but not the case for # deep learning models if not options['refit'] and \ not all(hasattr(estimator, attr) for attr in ('config', 'model_type')): warnings.warn("Refit is change to `True` for nested validation!") setattr(searcher, 'refit', True) outer_cv, _ = get_cv(params['outer_split']['cv_selector']) # nested CV, outer cv using cross_validate if options['error_score'] == 'raise': rval = cross_validate( searcher, X, y, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=options['verbose'], return_estimator=(params['save'] == 'save_estimator'), error_score=options['error_score'], return_train_score=True) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: rval = cross_validate( searcher, X, y, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=options['verbose'], return_estimator=(params['save'] == 'save_estimator'), error_score=options['error_score'], return_train_score=True) except ValueError: pass for warning in w: print(repr(warning.message)) fitted_searchers = rval.pop('estimator', []) if fitted_searchers: import os pwd = os.getcwd() save_dir = os.path.join(pwd, 'cv_results_in_folds') try: os.mkdir(save_dir) for idx, obj in enumerate(fitted_searchers): target_name = 'cv_results_' + '_' + 'split%d' % idx target_path = os.path.join(pwd, save_dir, target_name) cv_results_ = getattr(obj, 'cv_results_', None) if not cv_results_: print("%s is not available" % target_name) continue cv_results_ = pd.DataFrame(cv_results_) cv_results_ = cv_results_[sorted(cv_results_.columns)] cv_results_.to_csv(target_path, sep='\t', header=True, index=False) except Exception as e: print(e) finally: del os keys = list(rval.keys()) for k in keys: if k.startswith('test'): rval['mean_' + k] = np.mean(rval[k]) rval['std_' + k] = np.std(rval[k]) if k.endswith('time'): rval.pop(k) rval = pd.DataFrame(rval) rval = rval[sorted(rval.columns)] rval.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) return 0 # deprecate train test split mode """searcher = _do_train_test_split_val( searcher, X, y, params, primary_scoring=primary_scoring, error_score=options['error_score'], groups=groups, outfile=outfile_result)""" # no outer split else: searcher.set_params(n_jobs=N_JOBS) if options['error_score'] == 'raise': searcher.fit(X, y, groups=groups) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: searcher.fit(X, y, groups=groups) except ValueError: pass for warning in w: print(repr(warning.message)) cv_results = pd.DataFrame(searcher.cv_results_) cv_results = cv_results[sorted(cv_results.columns)] cv_results.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) memory.clear(warn=False) # output best estimator, and weights if applicable if outfile_object: best_estimator_ = getattr(searcher, 'best_estimator_', None) if not best_estimator_: warnings.warn("GridSearchCV object has no attribute " "'best_estimator_', because either it's " "nested gridsearch or `refit` is False!") return # clean prams best_estimator_ = clean_params(best_estimator_) main_est = get_main_estimator(best_estimator_) if hasattr(main_est, 'model_') \ and hasattr(main_est, 'save_weights'): if outfile_weights: main_est.save_weights(outfile_weights) del main_est.model_ del main_est.fit_params del main_est.model_class_ del main_est.validation_data if getattr(main_est, 'data_generator_', None): del main_est.data_generator_ with open(outfile_object, 'wb') as output_handler: print("Best estimator is saved: %s " % repr(best_estimator_)) pickle.dump(best_estimator_, output_handler, pickle.HIGHEST_PROTOCOL)
def main(inputs, infile_estimator, infile1, infile2, outfile_result, outfile_object=None, outfile_weights=None, groups=None, ref_seq=None, intervals=None, targets=None, fasta_path=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str File path to estimator infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target values outfile_result : str File path to save the results, either cv_results or test result outfile_object : str, optional File path to save searchCV object outfile_weights : str, optional File path to save model weights groups : str File path to dataset containing groups labels ref_seq : str File path to dataset containing genome sequence file intervals : str File path to dataset containing interval file targets : str File path to dataset compressed target bed file fasta_path : str File path to dataset containing fasta file """ warnings.simplefilter('ignore') with open(inputs, 'r') as param_handler: params = json.load(param_handler) params_builder = params['search_schemes']['search_params_builder'] with open(infile_estimator, 'rb') as estimator_handler: estimator = load_model(estimator_handler) estimator_params = estimator.get_params() # store read dataframe object loaded_df = {} input_type = params['input_options']['selected_input'] # tabular input if input_type == 'tabular': header = 'infer' if params['input_options']['header1'] else None column_option = (params['input_options']['column_selector_options_1'] ['selected_column_selector_option']) if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: c = params['input_options']['column_selector_options_1']['col1'] else: c = None df_key = infile1 + repr(header) df = pd.read_csv(infile1, sep='\t', header=header, parse_dates=True) loaded_df[df_key] = df X = read_columns(df, c=c, c_option=column_option).astype(float) # sparse input elif input_type == 'sparse': X = mmread(open(infile1, 'r')) # fasta_file input elif input_type == 'seq_fasta': pyfaidx = get_module('pyfaidx') sequences = pyfaidx.Fasta(fasta_path) n_seqs = len(sequences.keys()) X = np.arange(n_seqs)[:, np.newaxis] for param in estimator_params.keys(): if param.endswith('fasta_path'): estimator.set_params( **{param: fasta_path}) break else: raise ValueError( "The selected estimator doesn't support " "fasta file input! Please consider using " "KerasGBatchClassifier with " "FastaDNABatchGenerator/FastaProteinBatchGenerator " "or having GenomeOneHotEncoder/ProteinOneHotEncoder " "in pipeline!") elif input_type == 'refseq_and_interval': path_params = { 'data_batch_generator__ref_genome_path': ref_seq, 'data_batch_generator__intervals_path': intervals, 'data_batch_generator__target_path': targets } estimator.set_params(**path_params) n_intervals = sum(1 for line in open(intervals)) X = np.arange(n_intervals)[:, np.newaxis] # Get target y header = 'infer' if params['input_options']['header2'] else None column_option = (params['input_options']['column_selector_options_2'] ['selected_column_selector_option2']) if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: c = params['input_options']['column_selector_options_2']['col2'] else: c = None df_key = infile2 + repr(header) if df_key in loaded_df: infile2 = loaded_df[df_key] else: infile2 = pd.read_csv(infile2, sep='\t', header=header, parse_dates=True) loaded_df[df_key] = infile2 y = read_columns( infile2, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) if len(y.shape) == 2 and y.shape[1] == 1: y = y.ravel() if input_type == 'refseq_and_interval': estimator.set_params( data_batch_generator__features=y.ravel().tolist()) y = None # end y optimizer = params['search_schemes']['selected_search_scheme'] optimizer = getattr(model_selection, optimizer) # handle gridsearchcv options options = params['search_schemes']['options'] if groups: header = 'infer' if (options['cv_selector']['groups_selector'] ['header_g']) else None column_option = (options['cv_selector']['groups_selector'] ['column_selector_options_g'] ['selected_column_selector_option_g']) if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: c = (options['cv_selector']['groups_selector'] ['column_selector_options_g']['col_g']) else: c = None df_key = groups + repr(header) if df_key in loaded_df: groups = loaded_df[df_key] groups = read_columns( groups, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) groups = groups.ravel() options['cv_selector']['groups_selector'] = groups splitter, groups = get_cv(options.pop('cv_selector')) options['cv'] = splitter options['n_jobs'] = N_JOBS primary_scoring = options['scoring']['primary_scoring'] options['scoring'] = get_scoring(options['scoring']) if options['error_score']: options['error_score'] = 'raise' else: options['error_score'] = np.NaN if options['refit'] and isinstance(options['scoring'], dict): options['refit'] = primary_scoring if 'pre_dispatch' in options and options['pre_dispatch'] == '': options['pre_dispatch'] = None # del loaded_df del loaded_df # handle memory memory = joblib.Memory(location=CACHE_DIR, verbose=0) # cache iraps_core fits could increase search speed significantly if estimator.__class__.__name__ == 'IRAPSClassifier': estimator.set_params(memory=memory) else: # For iraps buried in pipeline for p, v in estimator_params.items(): if p.endswith('memory'): # for case of `__irapsclassifier__memory` if len(p) > 8 and p[:-8].endswith('irapsclassifier'): # cache iraps_core fits could increase search # speed significantly new_params = {p: memory} estimator.set_params(**new_params) # security reason, we don't want memory being # modified unexpectedly elif v: new_params = {p, None} estimator.set_params(**new_params) # For now, 1 CPU is suggested for iprasclassifier elif p.endswith('n_jobs'): new_params = {p: 1} estimator.set_params(**new_params) # for security reason, types of callbacks are limited elif p.endswith('callbacks'): for cb in v: cb_type = cb['callback_selection']['callback_type'] if cb_type not in ALLOWED_CALLBACKS: raise ValueError( "Prohibited callback type: %s!" % cb_type) param_grid = _eval_search_params(params_builder) searcher = optimizer(estimator, param_grid, **options) # do nested split split_mode = params['outer_split'].pop('split_mode') # nested CV, outer cv using cross_validate if split_mode == 'nested_cv': outer_cv, _ = get_cv(params['outer_split']['cv_selector']) if options['error_score'] == 'raise': rval = cross_validate( searcher, X, y, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=0, error_score=options['error_score']) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: rval = cross_validate( searcher, X, y, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=0, error_score=options['error_score']) except ValueError: pass for warning in w: print(repr(warning.message)) keys = list(rval.keys()) for k in keys: if k.startswith('test'): rval['mean_' + k] = np.mean(rval[k]) rval['std_' + k] = np.std(rval[k]) if k.endswith('time'): rval.pop(k) rval = pd.DataFrame(rval) rval = rval[sorted(rval.columns)] rval.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) else: if split_mode == 'train_test_split': train_test_split = try_get_attr( 'galaxy_ml.model_validations', 'train_test_split') # make sure refit is choosen # this could be True for sklearn models, but not the case for # deep learning models if not options['refit'] and \ not all(hasattr(estimator, attr) for attr in ('config', 'model_type')): warnings.warn("Refit is change to `True` for nested " "validation!") setattr(searcher, 'refit', True) split_options = params['outer_split'] # splits if split_options['shuffle'] == 'stratified': split_options['labels'] = y X, X_test, y, y_test = train_test_split(X, y, **split_options) elif split_options['shuffle'] == 'group': if groups is None: raise ValueError("No group based CV option was " "choosen for group shuffle!") split_options['labels'] = groups if y is None: X, X_test, groups, _ =\ train_test_split(X, groups, **split_options) else: X, X_test, y, y_test, groups, _ =\ train_test_split(X, y, groups, **split_options) else: if split_options['shuffle'] == 'None': split_options['shuffle'] = None X, X_test, y, y_test =\ train_test_split(X, y, **split_options) # end train_test_split # shared by both train_test_split and non-split if options['error_score'] == 'raise': searcher.fit(X, y, groups=groups) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: searcher.fit(X, y, groups=groups) except ValueError: pass for warning in w: print(repr(warning.message)) # no outer split if split_mode == 'no': # save results cv_results = pd.DataFrame(searcher.cv_results_) cv_results = cv_results[sorted(cv_results.columns)] cv_results.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) # train_test_split, output test result using best_estimator_ # or rebuild the trained estimator using weights if applicable. else: scorer_ = searcher.scorer_ if isinstance(scorer_, collections.Mapping): is_multimetric = True else: is_multimetric = False best_estimator_ = getattr(searcher, 'best_estimator_', None) if not best_estimator_: raise ValueError("GridSearchCV object has no " "`best_estimator_` when `refit`=False!") if best_estimator_.__class__.__name__ == 'KerasGBatchClassifier' \ and hasattr(estimator.data_batch_generator, 'target_path'): test_score = best_estimator_.evaluate( X_test, scorer=scorer_, is_multimetric=is_multimetric) else: test_score = _score(best_estimator_, X_test, y_test, scorer_, is_multimetric=is_multimetric) if not is_multimetric: test_score = {primary_scoring: test_score} for key, value in test_score.items(): test_score[key] = [value] result_df = pd.DataFrame(test_score) result_df.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) memory.clear(warn=False) if outfile_object: best_estimator_ = getattr(searcher, 'best_estimator_', None) if not best_estimator_: warnings.warn("GridSearchCV object has no attribute " "'best_estimator_', because either it's " "nested gridsearch or `refit` is False!") return main_est = best_estimator_ if isinstance(best_estimator_, pipeline.Pipeline): main_est = best_estimator_.steps[-1][-1] if hasattr(main_est, 'model_') \ and hasattr(main_est, 'save_weights'): if outfile_weights: main_est.save_weights(outfile_weights) del main_est.model_ del main_est.fit_params del main_est.model_class_ del main_est.validation_data if getattr(main_est, 'data_generator_', None): del main_est.data_generator_ del main_est.data_batch_generator with open(outfile_object, 'wb') as output_handler: pickle.dump(best_estimator_, output_handler, pickle.HIGHEST_PROTOCOL)
#Cross-validation analysis of the model vectoriser = HashingVectorizer(norm=None, stop_words='english', n_features=10000, decode_error='ignore') vectoriser.fit_transform(data.data) X = vectoriser.transform(data.data) Y = data.target mean_error=[]; std_error=[] c_range = [0.001, 0.01, 0.1, 0.5, 0.75, 1.0, 5.0] precision_score = [] precision_var = [] for c in c_range: svm_model = SVC(C=c, kernel='rbf', gamma=1) scores = cross_validate(svm_model, X, Y, cv=4, scoring=('balanced_accuracy', 'precision_micro', 'f1_micro')) precision_score.append(scores['test_precision_micro'].mean()) precision_var.append(scores['test_precision_micro'].var()) print('C = %0.3f' % c) print('Accuracy: [%s]' % ', '.join(map(str, scores['test_balanced_accuracy']))) print('Precision: [%s]' % ', '.join(map(str, scores['test_precision_micro']))) print('F1: [%s]' % ', '.join(map(str, scores['test_f1_micro']))) print('') fig = plt.figure() plt.plot(c_range, precision_score) plt.errorbar(c_range, precision_score, yerr=precision_var) plt.title('Precision Vs C') plt.xlabel('C') plt.xlim((0,2)) plt.ylabel('Precision')
def main( inputs, infile_estimator, infile1, infile2, outfile_result, outfile_object=None, outfile_weights=None, groups=None, ref_seq=None, intervals=None, targets=None, fasta_path=None, ): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str File path to estimator infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target values outfile_result : str File path to save the results, either cv_results or test result outfile_object : str, optional File path to save searchCV object outfile_weights : str, optional File path to save model weights groups : str File path to dataset containing groups labels ref_seq : str File path to dataset containing genome sequence file intervals : str File path to dataset containing interval file targets : str File path to dataset compressed target bed file fasta_path : str File path to dataset containing fasta file """ warnings.simplefilter("ignore") # store read dataframe object loaded_df = {} with open(inputs, "r") as param_handler: params = json.load(param_handler) # Override the refit parameter params["search_schemes"]["options"]["refit"] = ( True if params["save"] != "nope" else False ) with open(infile_estimator, "rb") as estimator_handler: estimator = load_model(estimator_handler) optimizer = params["search_schemes"]["selected_search_scheme"] optimizer = getattr(model_selection, optimizer) # handle gridsearchcv options options = params["search_schemes"]["options"] if groups: header = ( "infer" if (options["cv_selector"]["groups_selector"]["header_g"]) else None ) column_option = options["cv_selector"]["groups_selector"][ "column_selector_options_g" ]["selected_column_selector_option_g"] if column_option in [ "by_index_number", "all_but_by_index_number", "by_header_name", "all_but_by_header_name", ]: c = options["cv_selector"]["groups_selector"]["column_selector_options_g"][ "col_g" ] else: c = None df_key = groups + repr(header) groups = pd.read_csv(groups, sep="\t", header=header, parse_dates=True) loaded_df[df_key] = groups groups = read_columns( groups, c=c, c_option=column_option, sep="\t", header=header, parse_dates=True, ) groups = groups.ravel() options["cv_selector"]["groups_selector"] = groups splitter, groups = get_cv(options.pop("cv_selector")) options["cv"] = splitter primary_scoring = options["scoring"]["primary_scoring"] # get_scoring() expects secondary_scoring to be a comma separated string (not a list) # Check if secondary_scoring is specified secondary_scoring = options["scoring"].get("secondary_scoring", None) if secondary_scoring is not None: # If secondary_scoring is specified, convert the list into comman separated string options["scoring"]["secondary_scoring"] = ",".join( options["scoring"]["secondary_scoring"] ) options["scoring"] = get_scoring(options["scoring"]) if options["error_score"]: options["error_score"] = "raise" else: options["error_score"] = np.nan if options["refit"] and isinstance(options["scoring"], dict): options["refit"] = primary_scoring if "pre_dispatch" in options and options["pre_dispatch"] == "": options["pre_dispatch"] = None params_builder = params["search_schemes"]["search_params_builder"] param_grid = _eval_search_params(params_builder) estimator = clean_params(estimator) # save the SearchCV object without fit if params["save"] == "save_no_fit": searcher = optimizer(estimator, param_grid, **options) print(searcher) with open(outfile_object, "wb") as output_handler: pickle.dump(searcher, output_handler, pickle.HIGHEST_PROTOCOL) return 0 # read inputs and loads new attributes, like paths estimator, X, y = _handle_X_y( estimator, params, infile1, infile2, loaded_df=loaded_df, ref_seq=ref_seq, intervals=intervals, targets=targets, fasta_path=fasta_path, ) # cache iraps_core fits could increase search speed significantly memory = joblib.Memory(location=CACHE_DIR, verbose=0) main_est = get_main_estimator(estimator) if main_est.__class__.__name__ == "IRAPSClassifier": main_est.set_params(memory=memory) searcher = optimizer(estimator, param_grid, **options) split_mode = params["outer_split"].pop("split_mode") if split_mode == "nested_cv": # make sure refit is choosen # this could be True for sklearn models, but not the case for # deep learning models if not options["refit"] and not all( hasattr(estimator, attr) for attr in ("config", "model_type") ): warnings.warn("Refit is change to `True` for nested validation!") setattr(searcher, "refit", True) outer_cv, _ = get_cv(params["outer_split"]["cv_selector"]) # nested CV, outer cv using cross_validate if options["error_score"] == "raise": rval = cross_validate( searcher, X, y, scoring=options["scoring"], cv=outer_cv, n_jobs=N_JOBS, verbose=options["verbose"], return_estimator=(params["save"] == "save_estimator"), error_score=options["error_score"], return_train_score=True, ) else: warnings.simplefilter("always", FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: rval = cross_validate( searcher, X, y, scoring=options["scoring"], cv=outer_cv, n_jobs=N_JOBS, verbose=options["verbose"], return_estimator=(params["save"] == "save_estimator"), error_score=options["error_score"], return_train_score=True, ) except ValueError: pass for warning in w: print(repr(warning.message)) fitted_searchers = rval.pop("estimator", []) if fitted_searchers: import os pwd = os.getcwd() save_dir = os.path.join(pwd, "cv_results_in_folds") try: os.mkdir(save_dir) for idx, obj in enumerate(fitted_searchers): target_name = "cv_results_" + "_" + "split%d" % idx target_path = os.path.join(pwd, save_dir, target_name) cv_results_ = getattr(obj, "cv_results_", None) if not cv_results_: print("%s is not available" % target_name) continue cv_results_ = pd.DataFrame(cv_results_) cv_results_ = cv_results_[sorted(cv_results_.columns)] cv_results_.to_csv(target_path, sep="\t", header=True, index=False) except Exception as e: print(e) finally: del os keys = list(rval.keys()) for k in keys: if k.startswith("test"): rval["mean_" + k] = np.mean(rval[k]) rval["std_" + k] = np.std(rval[k]) if k.endswith("time"): rval.pop(k) rval = pd.DataFrame(rval) rval = rval[sorted(rval.columns)] rval.to_csv(path_or_buf=outfile_result, sep="\t", header=True, index=False) # deprecate train test split mode """searcher = _do_train_test_split_val( searcher, X, y, params, primary_scoring=primary_scoring, error_score=options['error_score'], groups=groups, outfile=outfile_result)""" return 0 # no outer split else: searcher.set_params(n_jobs=N_JOBS) if options["error_score"] == "raise": searcher.fit(X, y, groups=groups) else: warnings.simplefilter("always", FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: searcher.fit(X, y, groups=groups) except ValueError: pass for warning in w: print(repr(warning.message)) cv_results = pd.DataFrame(searcher.cv_results_) cv_results = cv_results[sorted(cv_results.columns)] cv_results.to_csv( path_or_buf=outfile_result, sep="\t", header=True, index=False ) memory.clear(warn=False) # output best estimator, and weights if applicable if outfile_object: best_estimator_ = getattr(searcher, "best_estimator_", None) if not best_estimator_: warnings.warn( "GridSearchCV object has no attribute " "'best_estimator_', because either it's " "nested gridsearch or `refit` is False!" ) return # clean prams best_estimator_ = clean_params(best_estimator_) main_est = get_main_estimator(best_estimator_) if hasattr(main_est, "model_") and hasattr(main_est, "save_weights"): if outfile_weights: main_est.save_weights(outfile_weights) del main_est.model_ del main_est.fit_params del main_est.model_class_ del main_est.validation_data if getattr(main_est, "data_generator_", None): del main_est.data_generator_ with open(outfile_object, "wb") as output_handler: print("Best estimator is saved: %s " % repr(best_estimator_)) pickle.dump(best_estimator_, output_handler, pickle.HIGHEST_PROTOCOL)
def main(inputs, infile_estimator, infile1, infile2, outfile_result, outfile_object=None, groups=None, ref_seq=None, intervals=None, targets=None, fasta_path=None): """ Parameter --------- inputs : str File path to galaxy tool parameter. infile_estimator : str File path to estimator. infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target values outfile_result : str File path to save the results, either cv_results or test result outfile_object : str, optional File path to save searchCV object groups : str File path to dataset containing groups labels ref_seq : str File path to dataset containing genome sequence file intervals : str File path to dataset containing interval file targets : str File path to dataset compressed target bed file fasta_path : str File path to dataset containing fasta file """ warnings.simplefilter('ignore') # store read dataframe object loaded_df = {} with open(inputs, 'r') as param_handler: params = json.load(param_handler) # Override the refit parameter params['search_schemes']['options']['refit'] = True \ if (params['save'] != 'nope' or params['outer_split']['split_mode'] == 'nested_cv') else False estimator = load_model_from_h5(infile_estimator) estimator = clean_params(estimator) if estimator.__class__.__name__ == 'KerasGBatchClassifier': _fit_and_score = try_get_attr('galaxy_ml.model_validations', '_fit_and_score') setattr(_search, '_fit_and_score', _fit_and_score) setattr(_validation, '_fit_and_score', _fit_and_score) optimizer = params['search_schemes']['selected_search_scheme'] if optimizer == 'skopt.BayesSearchCV': optimizer = BayesSearchCV else: optimizer = getattr(model_selection, optimizer) # handle gridsearchcv options options = params['search_schemes']['options'] if groups: header = 'infer' if ( options['cv_selector']['groups_selector']['header_g']) else None column_option = ( options['cv_selector']['groups_selector'] ['column_selector_options_g']['selected_column_selector_option_g']) if column_option in [ 'by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name' ]: c = (options['cv_selector']['groups_selector'] ['column_selector_options_g']['col_g']) else: c = None df_key = groups + repr(header) groups = pd.read_csv(groups, sep='\t', header=header, parse_dates=True) loaded_df[df_key] = groups groups = read_columns(groups, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) groups = groups.ravel() options['cv_selector']['groups_selector'] = groups cv_selector = options.pop('cv_selector') if Version(galaxy_ml_version) < Version('0.8.3'): cv_selector.pop('n_stratification_bins', None) splitter, groups = get_cv(cv_selector) options['cv'] = splitter primary_scoring = options['scoring']['primary_scoring'] options['scoring'] = get_scoring(options['scoring']) # TODO make BayesSearchCV support multiple scoring if optimizer == 'skopt.BayesSearchCV' and \ isinstance(options['scoring'], dict): options['scoring'] = options['scoring'][primary_scoring] warnings.warn("BayesSearchCV doesn't support multiple " "scorings! Primary scoring is used.") if options['error_score']: options['error_score'] = 'raise' else: options['error_score'] = np.NaN if options['refit'] and isinstance(options['scoring'], dict): options['refit'] = primary_scoring if 'pre_dispatch' in options and options['pre_dispatch'] == '': options['pre_dispatch'] = None params_builder = params['search_schemes']['search_params_builder'] param_grid = _eval_search_params(params_builder) # save the SearchCV object without fit if params['save'] == 'save_no_fit': searcher = optimizer(estimator, param_grid, **options) dump_model_to_h5(searcher, outfile_object) return 0 # read inputs and loads new attributes, like paths estimator, X, y = _handle_X_y(estimator, params, infile1, infile2, loaded_df=loaded_df, ref_seq=ref_seq, intervals=intervals, targets=targets, fasta_path=fasta_path) # cache iraps_core fits could increase search speed significantly memory = joblib.Memory(location=CACHE_DIR, verbose=0) estimator = _set_memory(estimator, memory) searcher = optimizer(estimator, param_grid, **options) split_mode = params['outer_split'].pop('split_mode') # Nested CV if split_mode == 'nested_cv': cv_selector = params['outer_split']['cv_selector'] if Version(galaxy_ml_version) < Version('0.8.3'): cv_selector.pop('n_stratification_bins', None) outer_cv, _ = get_cv(cv_selector) # nested CV, outer cv using cross_validate if options['error_score'] == 'raise': rval = cross_validate( searcher, X, y, groups=groups, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=options['verbose'], fit_params={'groups': groups}, return_estimator=(params['save'] == 'save_estimator'), error_score=options['error_score'], return_train_score=True) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: rval = cross_validate( searcher, X, y, groups=groups, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=options['verbose'], fit_params={'groups': groups}, return_estimator=(params['save'] == 'save_estimator'), error_score=options['error_score'], return_train_score=True) except ValueError: pass for warning in w: print(repr(warning.message)) fitted_searchers = rval.pop('estimator', []) if fitted_searchers: import os pwd = os.getcwd() save_dir = os.path.join(pwd, 'cv_results_in_folds') try: os.mkdir(save_dir) for idx, obj in enumerate(fitted_searchers): target_name = 'cv_results_' + '_' + 'split%d' % idx target_path = os.path.join(pwd, save_dir, target_name) cv_results_ = getattr(obj, 'cv_results_', None) if not cv_results_: print("%s is not available" % target_name) continue cv_results_ = pd.DataFrame(cv_results_) cv_results_ = cv_results_[sorted(cv_results_.columns)] cv_results_.to_csv(target_path, sep='\t', header=True, index=False) except Exception as e: print(e) finally: del os keys = list(rval.keys()) for k in keys: if k.startswith('test'): rval['mean_' + k] = np.mean(rval[k]) rval['std_' + k] = np.std(rval[k]) if k.endswith('time'): rval.pop(k) rval = pd.DataFrame(rval) rval = rval[sorted(rval.columns)] rval.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) return 0 # deprecate train test split mode """searcher = _do_train_test_split_val( searcher, X, y, params, primary_scoring=primary_scoring, error_score=options['error_score'], groups=groups, outfile=outfile_result)""" # no outer split else: searcher.set_params(n_jobs=N_JOBS) if options['error_score'] == 'raise': searcher.fit(X, y, groups=groups) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: searcher.fit(X, y, groups=groups) except ValueError: pass for warning in w: print(repr(warning.message)) cv_results = pd.DataFrame(searcher.cv_results_) cv_results = cv_results[sorted(cv_results.columns)] cv_results.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) memory.clear(warn=False) # output best estimator, and weights if applicable if outfile_object: best_estimator_ = getattr(searcher, 'best_estimator_', None) if not best_estimator_: warnings.warn("GridSearchCV object has no attribute " "'best_estimator_', because either it's " "nested gridsearch or `refit` is False!") return print("Saving best estimator: %s " % repr(best_estimator_)) dump_model_to_h5(best_estimator_, outfile_object)
def main(inputs, infile_estimator, infile1, infile2, outfile_result, outfile_object=None, groups=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str File path to estimator infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target values outfile_result : str File path to save the results, either cv_results or test result outfile_object : str, optional File path to save searchCV object groups : str File path to dataset containing groups labels """ warnings.simplefilter('ignore') with open(inputs, 'r') as param_handler: params = json.load(param_handler) if groups: (params['search_schemes']['options']['cv_selector'] ['groups_selector']['infile_g']) = groups params_builder = params['search_schemes']['search_params_builder'] input_type = params['input_options']['selected_input'] if input_type == 'tabular': header = 'infer' if params['input_options']['header1'] else None column_option = (params['input_options']['column_selector_options_1'] ['selected_column_selector_option']) if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: c = params['input_options']['column_selector_options_1']['col1'] else: c = None X = read_columns( infile1, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True).astype(float) else: X = mmread(open(infile1, 'r')) header = 'infer' if params['input_options']['header2'] else None column_option = (params['input_options']['column_selector_options_2'] ['selected_column_selector_option2']) if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: c = params['input_options']['column_selector_options_2']['col2'] else: c = None y = read_columns( infile2, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) if len(y.shape) == 2 and y.shape[1] == 1: y = y.ravel() optimizer = params['search_schemes']['selected_search_scheme'] optimizer = getattr(model_selection, optimizer) # handle gridsearchcv options options = params['search_schemes']['options'] splitter, groups = get_cv(options.pop('cv_selector')) options['cv'] = splitter options['n_jobs'] = N_JOBS primary_scoring = options['scoring']['primary_scoring'] options['scoring'] = get_scoring(options['scoring']) if options['error_score']: options['error_score'] = 'raise' else: options['error_score'] = np.NaN if options['refit'] and isinstance(options['scoring'], dict): options['refit'] = primary_scoring if 'pre_dispatch' in options and options['pre_dispatch'] == '': options['pre_dispatch'] = None with open(infile_estimator, 'rb') as estimator_handler: estimator = load_model(estimator_handler) # handle memory memory = joblib.Memory(location=CACHE_DIR, verbose=0) # cache iraps_core fits could increase search speed significantly if estimator.__class__.__name__ == 'IRAPSClassifier': estimator.set_params(memory=memory) else: # For iraps buried in pipeline for p, v in estimator.get_params().items(): if p.endswith('memory'): # for case of `__irapsclassifier__memory` if len(p) > 8 and p[:-8].endswith('irapsclassifier'): # cache iraps_core fits could increase search # speed significantly new_params = {p: memory} estimator.set_params(**new_params) # security reason, we don't want memory being # modified unexpectedly elif v: new_params = {p, None} estimator.set_params(**new_params) # For now, 1 CPU is suggested for iprasclassifier elif p.endswith('n_jobs'): new_params = {p: 1} estimator.set_params(**new_params) param_grid = _eval_search_params(params_builder) searcher = optimizer(estimator, param_grid, **options) # do nested split split_mode = params['outer_split'].pop('split_mode') # nested CV, outer cv using cross_validate if split_mode == 'nested_cv': outer_cv, _ = get_cv(params['outer_split']['cv_selector']) if options['error_score'] == 'raise': rval = cross_validate( searcher, X, y, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=0, error_score=options['error_score']) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: rval = cross_validate( searcher, X, y, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=0, error_score=options['error_score']) except ValueError: pass for warning in w: print(repr(warning.message)) keys = list(rval.keys()) for k in keys: if k.startswith('test'): rval['mean_' + k] = np.mean(rval[k]) rval['std_' + k] = np.std(rval[k]) if k.endswith('time'): rval.pop(k) rval = pandas.DataFrame(rval) rval = rval[sorted(rval.columns)] rval.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) else: if split_mode == 'train_test_split': train_test_split = try_get_attr( 'model_validations', 'train_test_split') # make sure refit is choosen if not options['refit']: raise ValueError("Refit must be `True` for shuffle splitting!") split_options = params['outer_split'] # splits if split_options['shuffle'] == 'stratified': split_options['labels'] = y X, X_test, y, y_test = train_test_split(X, y, **split_options) elif split_options['shuffle'] == 'group': if not groups: raise ValueError("No group based CV option was " "choosen for group shuffle!") split_options['labels'] = groups X, X_test, y, y_test, groups, _ =\ train_test_split(X, y, **split_options) else: if split_options['shuffle'] == 'None': split_options['shuffle'] = None X, X_test, y, y_test =\ train_test_split(X, y, **split_options) # end train_test_split # shared by both train_test_split and non-split if options['error_score'] == 'raise': searcher.fit(X, y, groups=groups) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: searcher.fit(X, y, groups=groups) except ValueError: pass for warning in w: print(repr(warning.message)) # no outer split if split_mode == 'no': # save results cv_results = pandas.DataFrame(searcher.cv_results_) cv_results = cv_results[sorted(cv_results.columns)] cv_results.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) # train_test_split, output test result using best_estimator_ else: best_estimator_ = searcher.best_estimator_ scorer_ = searcher.scorer_ if isinstance(scorer_, collections.Mapping): is_multimetric = True else: is_multimetric = False test_score = _score(best_estimator_, X_test, y_test, scorer_, is_multimetric=is_multimetric) if not is_multimetric: test_score = {primary_scoring: test_score} for key, value in test_score.items(): test_score[key] = [value] result_df = pandas.DataFrame(test_score) result_df.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) memory.clear(warn=False) if outfile_object: with open(outfile_object, 'wb') as output_handler: pickle.dump(searcher, output_handler, pickle.HIGHEST_PROTOCOL)