def main(inputs, infile_estimator, infile1, infile2, out_object, out_weights=None): """ main Parameters ---------- inputs : str File path to galaxy tool parameter infile_estimator : str File paths of input estimator infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target labels out_object : str File path for output of fitted model or skeleton out_weights : str File path for output of weights """ with open(inputs, 'r') as param_handler: params = json.load(param_handler) # load model with open(infile_estimator, 'rb') as est_handler: estimator = load_model(est_handler) estimator = clean_params(estimator, n_jobs=N_JOBS) X_train, y_train = _get_X_y(params, infile1, infile2) estimator.fit(X_train, y_train) main_est = estimator if isinstance(main_est, Pipeline): main_est = main_est.steps[-1][-1] if hasattr(main_est, 'model_') \ and hasattr(main_est, 'save_weights'): if out_weights: main_est.save_weights(out_weights) del main_est.model_ del main_est.fit_params del main_est.model_class_ del main_est.validation_data if getattr(main_est, 'data_generator_', None): del main_est.data_generator_ with open(out_object, 'wb') as output_handler: pickle.dump(estimator, output_handler, pickle.HIGHEST_PROTOCOL)
def main(inputs, infile_estimator, infile1, infile2, outfile_result, outfile_object=None, outfile_weights=None, outfile_y_true=None, outfile_y_preds=None, groups=None, ref_seq=None, intervals=None, targets=None, fasta_path=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str File path to estimator infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target values outfile_result : str File path to save the results, either cv_results or test result outfile_object : str, optional File path to save searchCV object outfile_weights : str, optional File path to save deep learning model weights outfile_y_true : str, optional File path to target values for prediction outfile_y_preds : str, optional File path to save deep learning model weights groups : str File path to dataset containing groups labels ref_seq : str File path to dataset containing genome sequence file intervals : str File path to dataset containing interval file targets : str File path to dataset compressed target bed file fasta_path : str File path to dataset containing fasta file """ warnings.simplefilter('ignore') with open(inputs, 'r') as param_handler: params = json.load(param_handler) # load estimator with open(infile_estimator, 'rb') as estimator_handler: estimator = load_model(estimator_handler) estimator = clean_params(estimator) # swap hyperparameter swapping = params['experiment_schemes']['hyperparams_swapping'] swap_params = _eval_swap_params(swapping) estimator.set_params(**swap_params) estimator_params = estimator.get_params() # store read dataframe object loaded_df = {} input_type = params['input_options']['selected_input'] # tabular input if input_type == 'tabular': header = 'infer' if params['input_options']['header1'] else None column_option = (params['input_options']['column_selector_options_1'] ['selected_column_selector_option']) if column_option in [ 'by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name' ]: c = params['input_options']['column_selector_options_1']['col1'] else: c = None df_key = infile1 + repr(header) df = pd.read_csv(infile1, sep='\t', header=header, parse_dates=True) loaded_df[df_key] = df X = read_columns(df, c=c, c_option=column_option).astype(float) # sparse input elif input_type == 'sparse': X = mmread(open(infile1, 'r')) # fasta_file input elif input_type == 'seq_fasta': pyfaidx = get_module('pyfaidx') sequences = pyfaidx.Fasta(fasta_path) n_seqs = len(sequences.keys()) X = np.arange(n_seqs)[:, np.newaxis] for param in estimator_params.keys(): if param.endswith('fasta_path'): estimator.set_params(**{param: fasta_path}) break else: raise ValueError( "The selected estimator doesn't support " "fasta file input! Please consider using " "KerasGBatchClassifier with " "FastaDNABatchGenerator/FastaProteinBatchGenerator " "or having GenomeOneHotEncoder/ProteinOneHotEncoder " "in pipeline!") elif input_type == 'refseq_and_interval': path_params = { 'data_batch_generator__ref_genome_path': ref_seq, 'data_batch_generator__intervals_path': intervals, 'data_batch_generator__target_path': targets } estimator.set_params(**path_params) n_intervals = sum(1 for line in open(intervals)) X = np.arange(n_intervals)[:, np.newaxis] # Get target y header = 'infer' if params['input_options']['header2'] else None column_option = (params['input_options']['column_selector_options_2'] ['selected_column_selector_option2']) if column_option in [ 'by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name' ]: c = params['input_options']['column_selector_options_2']['col2'] else: c = None df_key = infile2 + repr(header) if df_key in loaded_df: infile2 = loaded_df[df_key] else: infile2 = pd.read_csv(infile2, sep='\t', header=header, parse_dates=True) loaded_df[df_key] = infile2 y = read_columns(infile2, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) if len(y.shape) == 2 and y.shape[1] == 1: y = y.ravel() if input_type == 'refseq_and_interval': estimator.set_params(data_batch_generator__features=y.ravel().tolist()) y = None # end y # load groups if groups: groups_selector = (params['experiment_schemes']['test_split'] ['split_algos']).pop('groups_selector') header = 'infer' if groups_selector['header_g'] else None column_option = \ (groups_selector['column_selector_options_g'] ['selected_column_selector_option_g']) if column_option in [ 'by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name' ]: c = groups_selector['column_selector_options_g']['col_g'] else: c = None df_key = groups + repr(header) if df_key in loaded_df: groups = loaded_df[df_key] groups = read_columns(groups, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) groups = groups.ravel() # del loaded_df del loaded_df # cache iraps_core fits could increase search speed significantly memory = joblib.Memory(location=CACHE_DIR, verbose=0) main_est = get_main_estimator(estimator) if main_est.__class__.__name__ == 'IRAPSClassifier': main_est.set_params(memory=memory) # handle scorer, convert to scorer dict scoring = params['experiment_schemes']['metrics']['scoring'] scorer = get_scoring(scoring) scorer, _ = _check_multimetric_scoring(estimator, scoring=scorer) # handle test (first) split test_split_options = ( params['experiment_schemes']['test_split']['split_algos']) if test_split_options['shuffle'] == 'group': test_split_options['labels'] = groups if test_split_options['shuffle'] == 'stratified': if y is not None: test_split_options['labels'] = y else: raise ValueError("Stratified shuffle split is not " "applicable on empty target values!") X_train, X_test, y_train, y_test, groups_train, groups_test = \ train_test_split_none(X, y, groups, **test_split_options) exp_scheme = params['experiment_schemes']['selected_exp_scheme'] # handle validation (second) split if exp_scheme == 'train_val_test': val_split_options = ( params['experiment_schemes']['val_split']['split_algos']) if val_split_options['shuffle'] == 'group': val_split_options['labels'] = groups_train if val_split_options['shuffle'] == 'stratified': if y_train is not None: val_split_options['labels'] = y_train else: raise ValueError("Stratified shuffle split is not " "applicable on empty target values!") X_train, X_val, y_train, y_val, groups_train, groups_val = \ train_test_split_none(X_train, y_train, groups_train, **val_split_options) # train and eval if hasattr(estimator, 'config') and hasattr(estimator, 'model_type'): if exp_scheme == 'train_val_test': estimator.fit(X_train, y_train, validation_data=(X_val, y_val)) else: estimator.fit(X_train, y_train, validation_data=(X_test, y_test)) else: estimator.fit(X_train, y_train) if isinstance(estimator, KerasGBatchClassifier): scores = {} steps = estimator.prediction_steps batch_size = estimator.batch_size data_generator = estimator.data_generator_ scores, predictions, y_true = _evaluate_keras_and_sklearn_scores( estimator, data_generator, X_test, y=y_test, sk_scoring=sk_scoring, steps=steps, batch_size=batch_size, return_predictions=bool(outfile_y_true)) else: scores = {} if hasattr(estimator, 'model_') \ and hasattr(estimator.model_, 'metrics_names'): batch_size = estimator.batch_size score_results = estimator.model_.evaluate(X_test, y=y_test, batch_size=batch_size, verbose=0) metrics_names = estimator.model_.metrics_names if not isinstance(metrics_names, list): scores[metrics_names] = score_results else: scores = dict(zip(metrics_names, score_results)) if hasattr(estimator, 'predict_proba'): predictions = estimator.predict_proba(X_test) else: predictions = estimator.predict(X_test) y_true = y_test sk_scores = _score(estimator, X_test, y_test, scorer, is_multimetric=True) scores.update(sk_scores) # handle output if outfile_y_true: try: pd.DataFrame(y_true).to_csv(outfile_y_true, sep='\t', index=False) pd.DataFrame(predictions).astype(np.float32).to_csv( outfile_y_preds, sep='\t', index=False, float_format='%g', chunksize=10000) except Exception as e: print("Error in saving predictions: %s" % e) # handle output for name, score in scores.items(): scores[name] = [score] df = pd.DataFrame(scores) df = df[sorted(df.columns)] df.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) memory.clear(warn=False) if outfile_object: main_est = estimator if isinstance(estimator, Pipeline): main_est = estimator.steps[-1][-1] if hasattr(main_est, 'model_') \ and hasattr(main_est, 'save_weights'): if outfile_weights: main_est.save_weights(outfile_weights) del main_est.model_ del main_est.fit_params del main_est.model_class_ main_est.callbacks = [] if getattr(main_est, 'data_generator_', None): del main_est.data_generator_ with open(outfile_object, 'wb') as output_handler: pickle.dump(estimator, output_handler, pickle.HIGHEST_PROTOCOL)
def main(inputs, infile_estimator=None, infile1=None, infile2=None, outfile_result=None, outfile_object=None, groups=None, ref_seq=None, intervals=None, targets=None, fasta_path=None, model_config=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str, default is None File path to estimator infile1 : str, default is None File path to dataset containing features or true labels. infile2 : str, default is None File path to dataset containing target values or predicted probabilities. outfile_result : str, default is None File path to save the results, either cv_results or test result outfile_object : str, default is None File path to save searchCV object groups : str, default is None File path to dataset containing groups labels ref_seq : str, default is None File path to dataset containing genome sequence file intervals : str, default is None File path to dataset containing interval file targets : str, default is None File path to dataset compressed target bed file fasta_path : str, default is None File path to dataset containing fasta file model_config : str, default is None File path to dataset containing JSON config for neural networks """ warnings.simplefilter('ignore') with open(inputs, 'r') as param_handler: params = json.load(param_handler) title = params['plotting_selection']['title'].strip() plot_type = params['plotting_selection']['plot_type'] if plot_type == 'feature_importances': with open(infile_estimator, 'rb') as estimator_handler: estimator = load_model(estimator_handler) column_option = ( params['plotting_selection']['column_selector_options'] ['selected_column_selector_option']) if column_option in [ 'by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name' ]: c = (params['plotting_selection']['column_selector_options'] ['col1']) else: c = None _, input_df = read_columns(infile1, c=c, c_option=column_option, return_df=True, sep='\t', header='infer', parse_dates=True) feature_names = input_df.columns.values if isinstance(estimator, Pipeline): for st in estimator.steps[:-1]: if isinstance(st[-1], SelectorMixin): mask = st[-1].get_support() feature_names = feature_names[mask] estimator = estimator.steps[-1][-1] if hasattr(estimator, 'coef_'): coefs = estimator.coef_ else: coefs = getattr(estimator, 'feature_importances_', None) if coefs is None: raise RuntimeError('The classifier does not expose ' '"coef_" or "feature_importances_" ' 'attributes') threshold = params['plotting_selection']['threshold'] if threshold is not None: mask = (coefs > threshold) | (coefs < -threshold) coefs = coefs[mask] feature_names = feature_names[mask] # sort indices = np.argsort(coefs)[::-1] trace = go.Bar(x=feature_names[indices], y=coefs[indices]) layout = go.Layout(title=title or "Feature Importances") fig = go.Figure(data=[trace], layout=layout) elif plot_type == 'pr_curve': df1 = pd.read_csv(infile1, sep='\t', header=None) df2 = pd.read_csv(infile2, sep='\t', header=None) precision = {} recall = {} ap = {} pos_label = params['plotting_selection']['pos_label'].strip() \ or None for col in df1.columns: y_true = df1[col].values y_score = df2[col].values precision[col], recall[col], _ = precision_recall_curve( y_true, y_score, pos_label=pos_label) ap[col] = average_precision_score(y_true, y_score, pos_label=pos_label or 1) if len(df1.columns) > 1: precision["micro"], recall["micro"], _ = precision_recall_curve( df1.values.ravel(), df2.values.ravel(), pos_label=pos_label) ap['micro'] = average_precision_score(df1.values, df2.values, average='micro', pos_label=pos_label or 1) data = [] for key in precision.keys(): trace = go.Scatter(x=recall[key], y=precision[key], mode='lines', name='%s (area = %.2f)' % (key, ap[key]) if key == 'micro' else 'column %s (area = %.2f)' % (key, ap[key])) data.append(trace) layout = go.Layout(title=title or "Precision-Recall curve", xaxis=dict(title='Recall'), yaxis=dict(title='Precision')) fig = go.Figure(data=data, layout=layout) elif plot_type == 'roc_curve': df1 = pd.read_csv(infile1, sep='\t', header=None) df2 = pd.read_csv(infile2, sep='\t', header=None) fpr = {} tpr = {} roc_auc = {} pos_label = params['plotting_selection']['pos_label'].strip() \ or None for col in df1.columns: y_true = df1[col].values y_score = df2[col].values fpr[col], tpr[col], _ = roc_curve(y_true, y_score, pos_label=pos_label) roc_auc[col] = auc(fpr[col], tpr[col]) if len(df1.columns) > 1: fpr["micro"], tpr["micro"], _ = roc_curve(df1.values.ravel(), df2.values.ravel(), pos_label=pos_label) roc_auc['micro'] = auc(fpr["micro"], tpr["micro"]) data = [] for key in fpr.keys(): trace = go.Scatter(x=fpr[key], y=tpr[key], mode='lines', name='%s (area = %.2f)' % (key, roc_auc[key]) if key == 'micro' else 'column %s (area = %.2f)' % (key, roc_auc[key])) data.append(trace) trace = go.Scatter(x=[0, 1], y=[0, 1], mode='lines', line=dict(color='black', dash='dash'), showlegend=False) data.append(trace) layout = go.Layout(title=title or "Receiver operating characteristic curve", xaxis=dict(title='False Positive Rate'), yaxis=dict(title='True Positive Rate')) fig = go.Figure(data=data, layout=layout) elif plot_type == 'rfecv_gridscores': input_df = pd.read_csv(infile1, sep='\t', header='infer') scores = input_df.iloc[:, 0] steps = params['plotting_selection']['steps'].strip() steps = safe_eval(steps) data = go.Scatter(x=list(range(len(scores))), y=scores, text=[str(_) for _ in steps] if steps else None, mode='lines') layout = go.Layout(xaxis=dict(title="Number of features selected"), yaxis=dict(title="Cross validation score"), title=title or None) fig = go.Figure(data=[data], layout=layout) elif plot_type == 'learning_curve': input_df = pd.read_csv(infile1, sep='\t', header='infer') plot_std_err = params['plotting_selection']['plot_std_err'] data1 = go.Scatter( x=input_df['train_sizes_abs'], y=input_df['mean_train_scores'], error_y=dict( array=input_df['std_train_scores']) if plot_std_err else None, mode='lines', name="Train Scores", ) data2 = go.Scatter( x=input_df['train_sizes_abs'], y=input_df['mean_test_scores'], error_y=dict( array=input_df['std_test_scores']) if plot_std_err else None, mode='lines', name="Test Scores", ) layout = dict(xaxis=dict(title='No. of samples'), yaxis=dict(title='Performance Score'), title=title or 'Learning Curve') fig = go.Figure(data=[data1, data2], layout=layout) elif plot_type == 'keras_plot_model': with open(model_config, 'r') as f: model_str = f.read() model = model_from_json(model_str) plot_model(model, to_file="output.png") __import__('os').rename('output.png', 'output') return 0 plotly.offline.plot(fig, filename="output.html", auto_open=False) # to be discovered by `from_work_dir` __import__('os').rename('output.html', 'output')
def main(inputs, infile_estimator, infile1, infile2, outfile_result, outfile_object=None, outfile_weights=None, groups=None, ref_seq=None, intervals=None, targets=None, fasta_path=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str File path to estimator infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target values outfile_result : str File path to save the results, either cv_results or test result outfile_object : str, optional File path to save searchCV object outfile_weights : str, optional File path to save deep learning model weights groups : str File path to dataset containing groups labels ref_seq : str File path to dataset containing genome sequence file intervals : str File path to dataset containing interval file targets : str File path to dataset compressed target bed file fasta_path : str File path to dataset containing fasta file """ warnings.simplefilter('ignore') with open(inputs, 'r') as param_handler: params = json.load(param_handler) # load estimator with open(infile_estimator, 'rb') as estimator_handler: estimator = load_model(estimator_handler) # swap hyperparameter swapping = params['experiment_schemes']['hyperparams_swapping'] swap_params = _eval_swap_params(swapping) estimator.set_params(**swap_params) estimator_params = estimator.get_params() # store read dataframe object loaded_df = {} input_type = params['input_options']['selected_input'] # tabular input if input_type == 'tabular': header = 'infer' if params['input_options']['header1'] else None column_option = (params['input_options']['column_selector_options_1'] ['selected_column_selector_option']) if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: c = params['input_options']['column_selector_options_1']['col1'] else: c = None df_key = infile1 + repr(header) df = pd.read_csv(infile1, sep='\t', header=header, parse_dates=True) loaded_df[df_key] = df X = read_columns(df, c=c, c_option=column_option).astype(float) # sparse input elif input_type == 'sparse': X = mmread(open(infile1, 'r')) # fasta_file input elif input_type == 'seq_fasta': pyfaidx = get_module('pyfaidx') sequences = pyfaidx.Fasta(fasta_path) n_seqs = len(sequences.keys()) X = np.arange(n_seqs)[:, np.newaxis] for param in estimator_params.keys(): if param.endswith('fasta_path'): estimator.set_params( **{param: fasta_path}) break else: raise ValueError( "The selected estimator doesn't support " "fasta file input! Please consider using " "KerasGBatchClassifier with " "FastaDNABatchGenerator/FastaProteinBatchGenerator " "or having GenomeOneHotEncoder/ProteinOneHotEncoder " "in pipeline!") elif input_type == 'refseq_and_interval': path_params = { 'data_batch_generator__ref_genome_path': ref_seq, 'data_batch_generator__intervals_path': intervals, 'data_batch_generator__target_path': targets } estimator.set_params(**path_params) n_intervals = sum(1 for line in open(intervals)) X = np.arange(n_intervals)[:, np.newaxis] # Get target y header = 'infer' if params['input_options']['header2'] else None column_option = (params['input_options']['column_selector_options_2'] ['selected_column_selector_option2']) if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: c = params['input_options']['column_selector_options_2']['col2'] else: c = None df_key = infile2 + repr(header) if df_key in loaded_df: infile2 = loaded_df[df_key] else: infile2 = pd.read_csv(infile2, sep='\t', header=header, parse_dates=True) loaded_df[df_key] = infile2 y = read_columns( infile2, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) if len(y.shape) == 2 and y.shape[1] == 1: y = y.ravel() if input_type == 'refseq_and_interval': estimator.set_params( data_batch_generator__features=y.ravel().tolist()) y = None # end y # load groups if groups: groups_selector = (params['experiment_schemes']['test_split'] ['split_algos']).pop('groups_selector') header = 'infer' if groups_selector['header_g'] else None column_option = \ (groups_selector['column_selector_options_g'] ['selected_column_selector_option_g']) if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: c = groups_selector['column_selector_options_g']['col_g'] else: c = None df_key = groups + repr(header) if df_key in loaded_df: groups = loaded_df[df_key] groups = read_columns( groups, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) groups = groups.ravel() # del loaded_df del loaded_df # handle memory memory = joblib.Memory(location=CACHE_DIR, verbose=0) # cache iraps_core fits could increase search speed significantly if estimator.__class__.__name__ == 'IRAPSClassifier': estimator.set_params(memory=memory) else: # For iraps buried in pipeline new_params = {} for p, v in estimator_params.items(): if p.endswith('memory'): # for case of `__irapsclassifier__memory` if len(p) > 8 and p[:-8].endswith('irapsclassifier'): # cache iraps_core fits could increase search # speed significantly new_params[p] = memory # security reason, we don't want memory being # modified unexpectedly elif v: new_params[p] = None # handle n_jobs elif p.endswith('n_jobs'): # For now, 1 CPU is suggested for iprasclassifier if len(p) > 8 and p[:-8].endswith('irapsclassifier'): new_params[p] = 1 else: new_params[p] = N_JOBS # for security reason, types of callback are limited elif p.endswith('callbacks'): for cb in v: cb_type = cb['callback_selection']['callback_type'] if cb_type not in ALLOWED_CALLBACKS: raise ValueError( "Prohibited callback type: %s!" % cb_type) estimator.set_params(**new_params) # handle scorer, convert to scorer dict scoring = params['experiment_schemes']['metrics']['scoring'] scorer = get_scoring(scoring) scorer, _ = _check_multimetric_scoring(estimator, scoring=scorer) # handle test (first) split test_split_options = (params['experiment_schemes'] ['test_split']['split_algos']) if test_split_options['shuffle'] == 'group': test_split_options['labels'] = groups if test_split_options['shuffle'] == 'stratified': if y is not None: test_split_options['labels'] = y else: raise ValueError("Stratified shuffle split is not " "applicable on empty target values!") X_train, X_test, y_train, y_test, groups_train, groups_test = \ train_test_split_none(X, y, groups, **test_split_options) exp_scheme = params['experiment_schemes']['selected_exp_scheme'] # handle validation (second) split if exp_scheme == 'train_val_test': val_split_options = (params['experiment_schemes'] ['val_split']['split_algos']) if val_split_options['shuffle'] == 'group': val_split_options['labels'] = groups_train if val_split_options['shuffle'] == 'stratified': if y_train is not None: val_split_options['labels'] = y_train else: raise ValueError("Stratified shuffle split is not " "applicable on empty target values!") X_train, X_val, y_train, y_val, groups_train, groups_val = \ train_test_split_none(X_train, y_train, groups_train, **val_split_options) # train and eval if hasattr(estimator, 'validation_data'): if exp_scheme == 'train_val_test': estimator.fit(X_train, y_train, validation_data=(X_val, y_val)) else: estimator.fit(X_train, y_train, validation_data=(X_test, y_test)) else: estimator.fit(X_train, y_train) if hasattr(estimator, 'evaluate'): scores = estimator.evaluate(X_test, y_test=y_test, scorer=scorer, is_multimetric=True) else: scores = _score(estimator, X_test, y_test, scorer, is_multimetric=True) # handle output for name, score in scores.items(): scores[name] = [score] df = pd.DataFrame(scores) df = df[sorted(df.columns)] df.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) memory.clear(warn=False) if outfile_object: main_est = estimator if isinstance(estimator, pipeline.Pipeline): main_est = estimator.steps[-1][-1] if hasattr(main_est, 'model_') \ and hasattr(main_est, 'save_weights'): if outfile_weights: main_est.save_weights(outfile_weights) del main_est.model_ del main_est.fit_params del main_est.model_class_ del main_est.validation_data if getattr(main_est, 'data_generator_', None): del main_est.data_generator_ with open(outfile_object, 'wb') as output_handler: pickle.dump(estimator, output_handler, pickle.HIGHEST_PROTOCOL)
def main(inputs, infile_estimator, infile1, infile2, outfile_result, outfile_object=None, outfile_weights=None, groups=None, ref_seq=None, intervals=None, targets=None, fasta_path=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str File path to estimator infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target values outfile_result : str File path to save the results, either cv_results or test result outfile_object : str, optional File path to save searchCV object outfile_weights : str, optional File path to save model weights groups : str File path to dataset containing groups labels ref_seq : str File path to dataset containing genome sequence file intervals : str File path to dataset containing interval file targets : str File path to dataset compressed target bed file fasta_path : str File path to dataset containing fasta file """ warnings.simplefilter('ignore') # store read dataframe object loaded_df = {} with open(inputs, 'r') as param_handler: params = json.load(param_handler) # Override the refit parameter params['search_schemes']['options']['refit'] = True \ if params['save'] != 'nope' else False with open(infile_estimator, 'rb') as estimator_handler: estimator = load_model(estimator_handler) optimizer = params['search_schemes']['selected_search_scheme'] optimizer = getattr(model_selection, optimizer) # handle gridsearchcv options options = params['search_schemes']['options'] if groups: header = 'infer' if ( options['cv_selector']['groups_selector']['header_g']) else None column_option = ( options['cv_selector']['groups_selector'] ['column_selector_options_g']['selected_column_selector_option_g']) if column_option in [ 'by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name' ]: c = (options['cv_selector']['groups_selector'] ['column_selector_options_g']['col_g']) else: c = None df_key = groups + repr(header) groups = pd.read_csv(groups, sep='\t', header=header, parse_dates=True) loaded_df[df_key] = groups groups = read_columns(groups, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) groups = groups.ravel() options['cv_selector']['groups_selector'] = groups splitter, groups = get_cv(options.pop('cv_selector')) options['cv'] = splitter primary_scoring = options['scoring']['primary_scoring'] options['scoring'] = get_scoring(options['scoring']) if options['error_score']: options['error_score'] = 'raise' else: options['error_score'] = np.NaN if options['refit'] and isinstance(options['scoring'], dict): options['refit'] = primary_scoring if 'pre_dispatch' in options and options['pre_dispatch'] == '': options['pre_dispatch'] = None params_builder = params['search_schemes']['search_params_builder'] param_grid = _eval_search_params(params_builder) estimator = clean_params(estimator) # save the SearchCV object without fit if params['save'] == 'save_no_fit': searcher = optimizer(estimator, param_grid, **options) print(searcher) with open(outfile_object, 'wb') as output_handler: pickle.dump(searcher, output_handler, pickle.HIGHEST_PROTOCOL) return 0 # read inputs and loads new attributes, like paths estimator, X, y = _handle_X_y(estimator, params, infile1, infile2, loaded_df=loaded_df, ref_seq=ref_seq, intervals=intervals, targets=targets, fasta_path=fasta_path) # cache iraps_core fits could increase search speed significantly memory = joblib.Memory(location=CACHE_DIR, verbose=0) main_est = get_main_estimator(estimator) if main_est.__class__.__name__ == 'IRAPSClassifier': main_est.set_params(memory=memory) searcher = optimizer(estimator, param_grid, **options) split_mode = params['outer_split'].pop('split_mode') if split_mode == 'nested_cv': # make sure refit is choosen # this could be True for sklearn models, but not the case for # deep learning models if not options['refit'] and \ not all(hasattr(estimator, attr) for attr in ('config', 'model_type')): warnings.warn("Refit is change to `True` for nested validation!") setattr(searcher, 'refit', True) outer_cv, _ = get_cv(params['outer_split']['cv_selector']) # nested CV, outer cv using cross_validate if options['error_score'] == 'raise': rval = cross_validate( searcher, X, y, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=options['verbose'], return_estimator=(params['save'] == 'save_estimator'), error_score=options['error_score'], return_train_score=True) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: rval = cross_validate( searcher, X, y, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=options['verbose'], return_estimator=(params['save'] == 'save_estimator'), error_score=options['error_score'], return_train_score=True) except ValueError: pass for warning in w: print(repr(warning.message)) fitted_searchers = rval.pop('estimator', []) if fitted_searchers: import os pwd = os.getcwd() save_dir = os.path.join(pwd, 'cv_results_in_folds') try: os.mkdir(save_dir) for idx, obj in enumerate(fitted_searchers): target_name = 'cv_results_' + '_' + 'split%d' % idx target_path = os.path.join(pwd, save_dir, target_name) cv_results_ = getattr(obj, 'cv_results_', None) if not cv_results_: print("%s is not available" % target_name) continue cv_results_ = pd.DataFrame(cv_results_) cv_results_ = cv_results_[sorted(cv_results_.columns)] cv_results_.to_csv(target_path, sep='\t', header=True, index=False) except Exception as e: print(e) finally: del os keys = list(rval.keys()) for k in keys: if k.startswith('test'): rval['mean_' + k] = np.mean(rval[k]) rval['std_' + k] = np.std(rval[k]) if k.endswith('time'): rval.pop(k) rval = pd.DataFrame(rval) rval = rval[sorted(rval.columns)] rval.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) return 0 # deprecate train test split mode """searcher = _do_train_test_split_val( searcher, X, y, params, primary_scoring=primary_scoring, error_score=options['error_score'], groups=groups, outfile=outfile_result)""" # no outer split else: searcher.set_params(n_jobs=N_JOBS) if options['error_score'] == 'raise': searcher.fit(X, y, groups=groups) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: searcher.fit(X, y, groups=groups) except ValueError: pass for warning in w: print(repr(warning.message)) cv_results = pd.DataFrame(searcher.cv_results_) cv_results = cv_results[sorted(cv_results.columns)] cv_results.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) memory.clear(warn=False) # output best estimator, and weights if applicable if outfile_object: best_estimator_ = getattr(searcher, 'best_estimator_', None) if not best_estimator_: warnings.warn("GridSearchCV object has no attribute " "'best_estimator_', because either it's " "nested gridsearch or `refit` is False!") return # clean prams best_estimator_ = clean_params(best_estimator_) main_est = get_main_estimator(best_estimator_) if hasattr(main_est, 'model_') \ and hasattr(main_est, 'save_weights'): if outfile_weights: main_est.save_weights(outfile_weights) del main_est.model_ del main_est.fit_params del main_est.model_class_ del main_est.validation_data if getattr(main_est, 'data_generator_', None): del main_est.data_generator_ with open(outfile_object, 'wb') as output_handler: print("Best estimator is saved: %s " % repr(best_estimator_)) pickle.dump(best_estimator_, output_handler, pickle.HIGHEST_PROTOCOL)
def main(inputs_path, output_obj, base_paths=None, meta_path=None, outfile_params=None): """ Parameter --------- inputs_path : str File path for Galaxy parameters output_obj : str File path for ensemble estimator ouput base_paths : str File path or paths concatenated by comma. meta_path : str File path outfile_params : str File path for params output """ with open(inputs_path, 'r') as param_handler: params = json.load(param_handler) estimator_type = params['algo_selection']['estimator_type'] # get base estimators base_estimators = [] for idx, base_file in enumerate(base_paths.split(',')): if base_file and base_file != 'None': with open(base_file, 'rb') as handler: model = load_model(handler) else: estimator_json = ( params['base_est_builder'][idx]['estimator_selector']) model = get_estimator(estimator_json) if estimator_type.startswith('sklearn'): named = model.__class__.__name__.lower() named = 'base_%d_%s' % (idx, named) base_estimators.append((named, model)) else: base_estimators.append(model) # get meta estimator, if applicable if estimator_type.startswith('mlxtend'): if meta_path: with open(meta_path, 'rb') as f: meta_estimator = load_model(f) else: estimator_json = (params['algo_selection']['meta_estimator'] ['estimator_selector']) meta_estimator = get_estimator(estimator_json) options = params['algo_selection']['options'] cv_selector = options.pop('cv_selector', None) if cv_selector: splitter, groups = get_cv(cv_selector) options['cv'] = splitter # set n_jobs options['n_jobs'] = N_JOBS weights = options.pop('weights', None) if weights: weights = ast.literal_eval(weights) if weights: options['weights'] = weights mod_and_name = estimator_type.split('_') mod = sys.modules[mod_and_name[0]] klass = getattr(mod, mod_and_name[1]) if estimator_type.startswith('sklearn'): options['n_jobs'] = N_JOBS ensemble_estimator = klass(base_estimators, **options) elif mod == mlxtend.classifier: ensemble_estimator = klass(classifiers=base_estimators, meta_classifier=meta_estimator, **options) else: ensemble_estimator = klass(regressors=base_estimators, meta_regressor=meta_estimator, **options) print(ensemble_estimator) for base_est in base_estimators: print(base_est) with open(output_obj, 'wb') as out_handler: pickle.dump(ensemble_estimator, out_handler, pickle.HIGHEST_PROTOCOL) if params['get_params'] and outfile_params: results = get_search_params(ensemble_estimator) df = pd.DataFrame(results, columns=['', 'Parameter', 'Value']) df.to_csv(outfile_params, sep='\t', index=False)
def main( inputs, infile_estimator, outfile_predict, infile_weights=None, infile1=None, fasta_path=None, ref_seq=None, vcf_path=None, ): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : strgit File path to trained estimator input outfile_predict : str File path to save the prediction results, tabular infile_weights : str File path to weights input infile1 : str File path to dataset containing features fasta_path : str File path to dataset containing fasta file ref_seq : str File path to dataset containing the reference genome sequence. vcf_path : str File path to dataset containing variants info. """ warnings.filterwarnings("ignore") with open(inputs, "r") as param_handler: params = json.load(param_handler) # load model with open(infile_estimator, "rb") as est_handler: estimator = load_model(est_handler) main_est = estimator if isinstance(estimator, Pipeline): main_est = estimator.steps[-1][-1] if hasattr(main_est, "config") and hasattr(main_est, "load_weights"): if not infile_weights or infile_weights == "None": raise ValueError("The selected model skeleton asks for weights, " "but dataset for weights wan not selected!") main_est.load_weights(infile_weights) # handle data input input_type = params["input_options"]["selected_input"] # tabular input if input_type == "tabular": header = "infer" if params["input_options"]["header1"] else None column_option = params["input_options"]["column_selector_options_1"][ "selected_column_selector_option"] if column_option in [ "by_index_number", "all_but_by_index_number", "by_header_name", "all_but_by_header_name", ]: c = params["input_options"]["column_selector_options_1"]["col1"] else: c = None df = pd.read_csv(infile1, sep="\t", header=header, parse_dates=True) X = read_columns(df, c=c, c_option=column_option).astype(float) if params["method"] == "predict": preds = estimator.predict(X) else: preds = estimator.predict_proba(X) # sparse input elif input_type == "sparse": X = mmread(open(infile1, "r")) if params["method"] == "predict": preds = estimator.predict(X) else: preds = estimator.predict_proba(X) # fasta input elif input_type == "seq_fasta": if not hasattr(estimator, "data_batch_generator"): raise ValueError("To do prediction on sequences in fasta input, " "the estimator must be a `KerasGBatchClassifier`" "equipped with data_batch_generator!") pyfaidx = get_module("pyfaidx") sequences = pyfaidx.Fasta(fasta_path) n_seqs = len(sequences.keys()) X = np.arange(n_seqs)[:, np.newaxis] seq_length = estimator.data_batch_generator.seq_length batch_size = getattr(estimator, "batch_size", 32) steps = (n_seqs + batch_size - 1) // batch_size seq_type = params["input_options"]["seq_type"] klass = try_get_attr("galaxy_ml.preprocessors", seq_type) pred_data_generator = klass(fasta_path, seq_length=seq_length) if params["method"] == "predict": preds = estimator.predict(X, data_generator=pred_data_generator, steps=steps) else: preds = estimator.predict_proba(X, data_generator=pred_data_generator, steps=steps) # vcf input elif input_type == "variant_effect": klass = try_get_attr("galaxy_ml.preprocessors", "GenomicVariantBatchGenerator") options = params["input_options"] options.pop("selected_input") if options["blacklist_regions"] == "none": options["blacklist_regions"] = None pred_data_generator = klass(ref_genome_path=ref_seq, vcf_path=vcf_path, **options) pred_data_generator.set_processing_attrs() variants = pred_data_generator.variants # predict 1600 sample at once then write to file gen_flow = pred_data_generator.flow(batch_size=1600) file_writer = open(outfile_predict, "w") header_row = "\t".join( ["chrom", "pos", "name", "ref", "alt", "strand"]) file_writer.write(header_row) header_done = False steps_done = 0 # TODO: multiple threading try: while steps_done < len(gen_flow): index_array = next(gen_flow.index_generator) batch_X = gen_flow._get_batches_of_transformed_samples( index_array) if params["method"] == "predict": batch_preds = estimator.predict( batch_X, # The presence of `pred_data_generator` below is to # override model carrying data_generator if there # is any. data_generator=pred_data_generator, ) else: batch_preds = estimator.predict_proba( batch_X, # The presence of `pred_data_generator` below is to # override model carrying data_generator if there # is any. data_generator=pred_data_generator, ) if batch_preds.ndim == 1: batch_preds = batch_preds[:, np.newaxis] batch_meta = variants[index_array] batch_out = np.column_stack([batch_meta, batch_preds]) if not header_done: heads = np.arange(batch_preds.shape[-1]).astype(str) heads_str = "\t".join(heads) file_writer.write("\t%s\n" % heads_str) header_done = True for row in batch_out: row_str = "\t".join(row) file_writer.write("%s\n" % row_str) steps_done += 1 finally: file_writer.close() # TODO: make api `pred_data_generator.close()` pred_data_generator.close() return 0 # end input # output if len(preds.shape) == 1: rval = pd.DataFrame(preds, columns=["Predicted"]) else: rval = pd.DataFrame(preds) rval.to_csv(outfile_predict, sep="\t", header=True, index=False)
def main(inputs_path, output_obj, base_paths=None, meta_path=None, outfile_params=None): """ Parameter --------- inputs_path : str File path for Galaxy parameters output_obj : str File path for ensemble estimator ouput base_paths : str File path or paths concatenated by comma. meta_path : str File path outfile_params : str File path for params output """ with open(inputs_path, "r") as param_handler: params = json.load(param_handler) estimator_type = params["algo_selection"]["estimator_type"] # get base estimators base_estimators = [] for idx, base_file in enumerate(base_paths.split(",")): if base_file and base_file != "None": with open(base_file, "rb") as handler: model = load_model(handler) else: estimator_json = params["base_est_builder"][idx][ "estimator_selector"] model = get_estimator(estimator_json) if estimator_type.startswith("sklearn"): named = model.__class__.__name__.lower() named = "base_%d_%s" % (idx, named) base_estimators.append((named, model)) else: base_estimators.append(model) # get meta estimator, if applicable if estimator_type.startswith("mlxtend"): if meta_path: with open(meta_path, "rb") as f: meta_estimator = load_model(f) else: estimator_json = params["algo_selection"]["meta_estimator"][ "estimator_selector"] meta_estimator = get_estimator(estimator_json) options = params["algo_selection"]["options"] cv_selector = options.pop("cv_selector", None) if cv_selector: splitter, _groups = get_cv(cv_selector) options["cv"] = splitter # set n_jobs options["n_jobs"] = N_JOBS weights = options.pop("weights", None) if weights: weights = ast.literal_eval(weights) if weights: options["weights"] = weights mod_and_name = estimator_type.split("_") mod = sys.modules[mod_and_name[0]] klass = getattr(mod, mod_and_name[1]) if estimator_type.startswith("sklearn"): options["n_jobs"] = N_JOBS ensemble_estimator = klass(base_estimators, **options) elif mod == mlxtend.classifier: ensemble_estimator = klass(classifiers=base_estimators, meta_classifier=meta_estimator, **options) else: ensemble_estimator = klass(regressors=base_estimators, meta_regressor=meta_estimator, **options) print(ensemble_estimator) for base_est in base_estimators: print(base_est) with open(output_obj, "wb") as out_handler: pickle.dump(ensemble_estimator, out_handler, pickle.HIGHEST_PROTOCOL) if params["get_params"] and outfile_params: results = get_search_params(ensemble_estimator) df = pd.DataFrame(results, columns=["", "Parameter", "Value"]) df.to_csv(outfile_params, sep="\t", index=False)
def main( inputs, infile_estimator, outfile_eval, infile_weights=None, infile1=None, infile2=None, ): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : strgit File path to trained estimator input outfile_eval : str File path to save the evalulation results, tabular infile_weights : str File path to weights input infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target values """ warnings.filterwarnings("ignore") with open(inputs, "r") as param_handler: params = json.load(param_handler) X_test, y_test = _get_X_y(params, infile1, infile2) # load model with open(infile_estimator, "rb") as est_handler: estimator = load_model(est_handler) main_est = estimator if isinstance(estimator, Pipeline): main_est = estimator.steps[-1][-1] if hasattr(main_est, "config") and hasattr(main_est, "load_weights"): if not infile_weights or infile_weights == "None": raise ValueError("The selected model skeleton asks for weights, " "but no dataset for weights was provided!") main_est.load_weights(infile_weights) # handle scorer, convert to scorer dict # Check if scoring is specified scoring = params["scoring"] if scoring is not None: # get_scoring() expects secondary_scoring to be a comma separated string (not a list) # Check if secondary_scoring is specified secondary_scoring = scoring.get("secondary_scoring", None) if secondary_scoring is not None: # If secondary_scoring is specified, convert the list into comman separated string scoring["secondary_scoring"] = ",".join( scoring["secondary_scoring"]) scorer = get_scoring(scoring) scorer, _ = _check_multimetric_scoring(estimator, scoring=scorer) if hasattr(estimator, "evaluate"): scores = estimator.evaluate(X_test, y_test=y_test, scorer=scorer, is_multimetric=True) else: scores = _score(estimator, X_test, y_test, scorer, is_multimetric=True) # handle output for name, score in scores.items(): scores[name] = [score] df = pd.DataFrame(scores) df = df[sorted(df.columns)] df.to_csv(path_or_buf=outfile_eval, sep="\t", header=True, index=False)
def main(inputs, infile_estimator, infile1, infile2, outfile_result, outfile_object=None, outfile_weights=None, groups=None, ref_seq=None, intervals=None, targets=None, fasta_path=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str File path to estimator infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target values outfile_result : str File path to save the results, either cv_results or test result outfile_object : str, optional File path to save searchCV object outfile_weights : str, optional File path to save model weights groups : str File path to dataset containing groups labels ref_seq : str File path to dataset containing genome sequence file intervals : str File path to dataset containing interval file targets : str File path to dataset compressed target bed file fasta_path : str File path to dataset containing fasta file """ warnings.simplefilter('ignore') with open(inputs, 'r') as param_handler: params = json.load(param_handler) params_builder = params['search_schemes']['search_params_builder'] with open(infile_estimator, 'rb') as estimator_handler: estimator = load_model(estimator_handler) estimator_params = estimator.get_params() # store read dataframe object loaded_df = {} input_type = params['input_options']['selected_input'] # tabular input if input_type == 'tabular': header = 'infer' if params['input_options']['header1'] else None column_option = (params['input_options']['column_selector_options_1'] ['selected_column_selector_option']) if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: c = params['input_options']['column_selector_options_1']['col1'] else: c = None df_key = infile1 + repr(header) df = pd.read_csv(infile1, sep='\t', header=header, parse_dates=True) loaded_df[df_key] = df X = read_columns(df, c=c, c_option=column_option).astype(float) # sparse input elif input_type == 'sparse': X = mmread(open(infile1, 'r')) # fasta_file input elif input_type == 'seq_fasta': pyfaidx = get_module('pyfaidx') sequences = pyfaidx.Fasta(fasta_path) n_seqs = len(sequences.keys()) X = np.arange(n_seqs)[:, np.newaxis] for param in estimator_params.keys(): if param.endswith('fasta_path'): estimator.set_params( **{param: fasta_path}) break else: raise ValueError( "The selected estimator doesn't support " "fasta file input! Please consider using " "KerasGBatchClassifier with " "FastaDNABatchGenerator/FastaProteinBatchGenerator " "or having GenomeOneHotEncoder/ProteinOneHotEncoder " "in pipeline!") elif input_type == 'refseq_and_interval': path_params = { 'data_batch_generator__ref_genome_path': ref_seq, 'data_batch_generator__intervals_path': intervals, 'data_batch_generator__target_path': targets } estimator.set_params(**path_params) n_intervals = sum(1 for line in open(intervals)) X = np.arange(n_intervals)[:, np.newaxis] # Get target y header = 'infer' if params['input_options']['header2'] else None column_option = (params['input_options']['column_selector_options_2'] ['selected_column_selector_option2']) if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: c = params['input_options']['column_selector_options_2']['col2'] else: c = None df_key = infile2 + repr(header) if df_key in loaded_df: infile2 = loaded_df[df_key] else: infile2 = pd.read_csv(infile2, sep='\t', header=header, parse_dates=True) loaded_df[df_key] = infile2 y = read_columns( infile2, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) if len(y.shape) == 2 and y.shape[1] == 1: y = y.ravel() if input_type == 'refseq_and_interval': estimator.set_params( data_batch_generator__features=y.ravel().tolist()) y = None # end y optimizer = params['search_schemes']['selected_search_scheme'] optimizer = getattr(model_selection, optimizer) # handle gridsearchcv options options = params['search_schemes']['options'] if groups: header = 'infer' if (options['cv_selector']['groups_selector'] ['header_g']) else None column_option = (options['cv_selector']['groups_selector'] ['column_selector_options_g'] ['selected_column_selector_option_g']) if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: c = (options['cv_selector']['groups_selector'] ['column_selector_options_g']['col_g']) else: c = None df_key = groups + repr(header) if df_key in loaded_df: groups = loaded_df[df_key] groups = read_columns( groups, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) groups = groups.ravel() options['cv_selector']['groups_selector'] = groups splitter, groups = get_cv(options.pop('cv_selector')) options['cv'] = splitter options['n_jobs'] = N_JOBS primary_scoring = options['scoring']['primary_scoring'] options['scoring'] = get_scoring(options['scoring']) if options['error_score']: options['error_score'] = 'raise' else: options['error_score'] = np.NaN if options['refit'] and isinstance(options['scoring'], dict): options['refit'] = primary_scoring if 'pre_dispatch' in options and options['pre_dispatch'] == '': options['pre_dispatch'] = None # del loaded_df del loaded_df # handle memory memory = joblib.Memory(location=CACHE_DIR, verbose=0) # cache iraps_core fits could increase search speed significantly if estimator.__class__.__name__ == 'IRAPSClassifier': estimator.set_params(memory=memory) else: # For iraps buried in pipeline for p, v in estimator_params.items(): if p.endswith('memory'): # for case of `__irapsclassifier__memory` if len(p) > 8 and p[:-8].endswith('irapsclassifier'): # cache iraps_core fits could increase search # speed significantly new_params = {p: memory} estimator.set_params(**new_params) # security reason, we don't want memory being # modified unexpectedly elif v: new_params = {p, None} estimator.set_params(**new_params) # For now, 1 CPU is suggested for iprasclassifier elif p.endswith('n_jobs'): new_params = {p: 1} estimator.set_params(**new_params) # for security reason, types of callbacks are limited elif p.endswith('callbacks'): for cb in v: cb_type = cb['callback_selection']['callback_type'] if cb_type not in ALLOWED_CALLBACKS: raise ValueError( "Prohibited callback type: %s!" % cb_type) param_grid = _eval_search_params(params_builder) searcher = optimizer(estimator, param_grid, **options) # do nested split split_mode = params['outer_split'].pop('split_mode') # nested CV, outer cv using cross_validate if split_mode == 'nested_cv': outer_cv, _ = get_cv(params['outer_split']['cv_selector']) if options['error_score'] == 'raise': rval = cross_validate( searcher, X, y, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=0, error_score=options['error_score']) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: rval = cross_validate( searcher, X, y, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=0, error_score=options['error_score']) except ValueError: pass for warning in w: print(repr(warning.message)) keys = list(rval.keys()) for k in keys: if k.startswith('test'): rval['mean_' + k] = np.mean(rval[k]) rval['std_' + k] = np.std(rval[k]) if k.endswith('time'): rval.pop(k) rval = pd.DataFrame(rval) rval = rval[sorted(rval.columns)] rval.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) else: if split_mode == 'train_test_split': train_test_split = try_get_attr( 'galaxy_ml.model_validations', 'train_test_split') # make sure refit is choosen # this could be True for sklearn models, but not the case for # deep learning models if not options['refit'] and \ not all(hasattr(estimator, attr) for attr in ('config', 'model_type')): warnings.warn("Refit is change to `True` for nested " "validation!") setattr(searcher, 'refit', True) split_options = params['outer_split'] # splits if split_options['shuffle'] == 'stratified': split_options['labels'] = y X, X_test, y, y_test = train_test_split(X, y, **split_options) elif split_options['shuffle'] == 'group': if groups is None: raise ValueError("No group based CV option was " "choosen for group shuffle!") split_options['labels'] = groups if y is None: X, X_test, groups, _ =\ train_test_split(X, groups, **split_options) else: X, X_test, y, y_test, groups, _ =\ train_test_split(X, y, groups, **split_options) else: if split_options['shuffle'] == 'None': split_options['shuffle'] = None X, X_test, y, y_test =\ train_test_split(X, y, **split_options) # end train_test_split # shared by both train_test_split and non-split if options['error_score'] == 'raise': searcher.fit(X, y, groups=groups) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: searcher.fit(X, y, groups=groups) except ValueError: pass for warning in w: print(repr(warning.message)) # no outer split if split_mode == 'no': # save results cv_results = pd.DataFrame(searcher.cv_results_) cv_results = cv_results[sorted(cv_results.columns)] cv_results.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) # train_test_split, output test result using best_estimator_ # or rebuild the trained estimator using weights if applicable. else: scorer_ = searcher.scorer_ if isinstance(scorer_, collections.Mapping): is_multimetric = True else: is_multimetric = False best_estimator_ = getattr(searcher, 'best_estimator_', None) if not best_estimator_: raise ValueError("GridSearchCV object has no " "`best_estimator_` when `refit`=False!") if best_estimator_.__class__.__name__ == 'KerasGBatchClassifier' \ and hasattr(estimator.data_batch_generator, 'target_path'): test_score = best_estimator_.evaluate( X_test, scorer=scorer_, is_multimetric=is_multimetric) else: test_score = _score(best_estimator_, X_test, y_test, scorer_, is_multimetric=is_multimetric) if not is_multimetric: test_score = {primary_scoring: test_score} for key, value in test_score.items(): test_score[key] = [value] result_df = pd.DataFrame(test_score) result_df.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) memory.clear(warn=False) if outfile_object: best_estimator_ = getattr(searcher, 'best_estimator_', None) if not best_estimator_: warnings.warn("GridSearchCV object has no attribute " "'best_estimator_', because either it's " "nested gridsearch or `refit` is False!") return main_est = best_estimator_ if isinstance(best_estimator_, pipeline.Pipeline): main_est = best_estimator_.steps[-1][-1] if hasattr(main_est, 'model_') \ and hasattr(main_est, 'save_weights'): if outfile_weights: main_est.save_weights(outfile_weights) del main_est.model_ del main_est.fit_params del main_est.model_class_ del main_est.validation_data if getattr(main_est, 'data_generator_', None): del main_est.data_generator_ del main_est.data_batch_generator with open(outfile_object, 'wb') as output_handler: pickle.dump(best_estimator_, output_handler, pickle.HIGHEST_PROTOCOL)
def main(inputs, infile_estimator, outfile_eval, infile_weights=None, infile1=None, infile2=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : strgit File path to trained estimator input outfile_eval : str File path to save the evalulation results, tabular infile_weights : str File path to weights input infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target values """ warnings.filterwarnings('ignore') with open(inputs, 'r') as param_handler: params = json.load(param_handler) X_test, y_test = _get_X_y(params, infile1, infile2) # load model with open(infile_estimator, 'rb') as est_handler: estimator = load_model(est_handler) main_est = estimator if isinstance(estimator, Pipeline): main_est = estimator.steps[-1][-1] if hasattr(main_est, 'config') and hasattr(main_est, 'load_weights'): if not infile_weights or infile_weights == 'None': raise ValueError("The selected model skeleton asks for weights, " "but no dataset for weights was provided!") main_est.load_weights(infile_weights) # handle scorer, convert to scorer dict scoring = params['scoring'] scorer = get_scoring(scoring) scorer, _ = _check_multimetric_scoring(estimator, scoring=scorer) if hasattr(estimator, 'evaluate'): scores = estimator.evaluate(X_test, y_test=y_test, scorer=scorer, is_multimetric=True) else: scores = _score(estimator, X_test, y_test, scorer, is_multimetric=True) # handle output for name, score in scores.items(): scores[name] = [score] df = pd.DataFrame(scores) df = df[sorted(df.columns)] df.to_csv(path_or_buf=outfile_eval, sep='\t', header=True, index=False)
def main( inputs, infile_estimator, infile1, infile2, outfile_result, outfile_object=None, outfile_weights=None, groups=None, ref_seq=None, intervals=None, targets=None, fasta_path=None, ): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str File path to estimator infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target values outfile_result : str File path to save the results, either cv_results or test result outfile_object : str, optional File path to save searchCV object outfile_weights : str, optional File path to save model weights groups : str File path to dataset containing groups labels ref_seq : str File path to dataset containing genome sequence file intervals : str File path to dataset containing interval file targets : str File path to dataset compressed target bed file fasta_path : str File path to dataset containing fasta file """ warnings.simplefilter("ignore") # store read dataframe object loaded_df = {} with open(inputs, "r") as param_handler: params = json.load(param_handler) # Override the refit parameter params["search_schemes"]["options"]["refit"] = ( True if params["save"] != "nope" else False ) with open(infile_estimator, "rb") as estimator_handler: estimator = load_model(estimator_handler) optimizer = params["search_schemes"]["selected_search_scheme"] optimizer = getattr(model_selection, optimizer) # handle gridsearchcv options options = params["search_schemes"]["options"] if groups: header = ( "infer" if (options["cv_selector"]["groups_selector"]["header_g"]) else None ) column_option = options["cv_selector"]["groups_selector"][ "column_selector_options_g" ]["selected_column_selector_option_g"] if column_option in [ "by_index_number", "all_but_by_index_number", "by_header_name", "all_but_by_header_name", ]: c = options["cv_selector"]["groups_selector"]["column_selector_options_g"][ "col_g" ] else: c = None df_key = groups + repr(header) groups = pd.read_csv(groups, sep="\t", header=header, parse_dates=True) loaded_df[df_key] = groups groups = read_columns( groups, c=c, c_option=column_option, sep="\t", header=header, parse_dates=True, ) groups = groups.ravel() options["cv_selector"]["groups_selector"] = groups splitter, groups = get_cv(options.pop("cv_selector")) options["cv"] = splitter primary_scoring = options["scoring"]["primary_scoring"] # get_scoring() expects secondary_scoring to be a comma separated string (not a list) # Check if secondary_scoring is specified secondary_scoring = options["scoring"].get("secondary_scoring", None) if secondary_scoring is not None: # If secondary_scoring is specified, convert the list into comman separated string options["scoring"]["secondary_scoring"] = ",".join( options["scoring"]["secondary_scoring"] ) options["scoring"] = get_scoring(options["scoring"]) if options["error_score"]: options["error_score"] = "raise" else: options["error_score"] = np.nan if options["refit"] and isinstance(options["scoring"], dict): options["refit"] = primary_scoring if "pre_dispatch" in options and options["pre_dispatch"] == "": options["pre_dispatch"] = None params_builder = params["search_schemes"]["search_params_builder"] param_grid = _eval_search_params(params_builder) estimator = clean_params(estimator) # save the SearchCV object without fit if params["save"] == "save_no_fit": searcher = optimizer(estimator, param_grid, **options) print(searcher) with open(outfile_object, "wb") as output_handler: pickle.dump(searcher, output_handler, pickle.HIGHEST_PROTOCOL) return 0 # read inputs and loads new attributes, like paths estimator, X, y = _handle_X_y( estimator, params, infile1, infile2, loaded_df=loaded_df, ref_seq=ref_seq, intervals=intervals, targets=targets, fasta_path=fasta_path, ) # cache iraps_core fits could increase search speed significantly memory = joblib.Memory(location=CACHE_DIR, verbose=0) main_est = get_main_estimator(estimator) if main_est.__class__.__name__ == "IRAPSClassifier": main_est.set_params(memory=memory) searcher = optimizer(estimator, param_grid, **options) split_mode = params["outer_split"].pop("split_mode") if split_mode == "nested_cv": # make sure refit is choosen # this could be True for sklearn models, but not the case for # deep learning models if not options["refit"] and not all( hasattr(estimator, attr) for attr in ("config", "model_type") ): warnings.warn("Refit is change to `True` for nested validation!") setattr(searcher, "refit", True) outer_cv, _ = get_cv(params["outer_split"]["cv_selector"]) # nested CV, outer cv using cross_validate if options["error_score"] == "raise": rval = cross_validate( searcher, X, y, scoring=options["scoring"], cv=outer_cv, n_jobs=N_JOBS, verbose=options["verbose"], return_estimator=(params["save"] == "save_estimator"), error_score=options["error_score"], return_train_score=True, ) else: warnings.simplefilter("always", FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: rval = cross_validate( searcher, X, y, scoring=options["scoring"], cv=outer_cv, n_jobs=N_JOBS, verbose=options["verbose"], return_estimator=(params["save"] == "save_estimator"), error_score=options["error_score"], return_train_score=True, ) except ValueError: pass for warning in w: print(repr(warning.message)) fitted_searchers = rval.pop("estimator", []) if fitted_searchers: import os pwd = os.getcwd() save_dir = os.path.join(pwd, "cv_results_in_folds") try: os.mkdir(save_dir) for idx, obj in enumerate(fitted_searchers): target_name = "cv_results_" + "_" + "split%d" % idx target_path = os.path.join(pwd, save_dir, target_name) cv_results_ = getattr(obj, "cv_results_", None) if not cv_results_: print("%s is not available" % target_name) continue cv_results_ = pd.DataFrame(cv_results_) cv_results_ = cv_results_[sorted(cv_results_.columns)] cv_results_.to_csv(target_path, sep="\t", header=True, index=False) except Exception as e: print(e) finally: del os keys = list(rval.keys()) for k in keys: if k.startswith("test"): rval["mean_" + k] = np.mean(rval[k]) rval["std_" + k] = np.std(rval[k]) if k.endswith("time"): rval.pop(k) rval = pd.DataFrame(rval) rval = rval[sorted(rval.columns)] rval.to_csv(path_or_buf=outfile_result, sep="\t", header=True, index=False) # deprecate train test split mode """searcher = _do_train_test_split_val( searcher, X, y, params, primary_scoring=primary_scoring, error_score=options['error_score'], groups=groups, outfile=outfile_result)""" return 0 # no outer split else: searcher.set_params(n_jobs=N_JOBS) if options["error_score"] == "raise": searcher.fit(X, y, groups=groups) else: warnings.simplefilter("always", FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: searcher.fit(X, y, groups=groups) except ValueError: pass for warning in w: print(repr(warning.message)) cv_results = pd.DataFrame(searcher.cv_results_) cv_results = cv_results[sorted(cv_results.columns)] cv_results.to_csv( path_or_buf=outfile_result, sep="\t", header=True, index=False ) memory.clear(warn=False) # output best estimator, and weights if applicable if outfile_object: best_estimator_ = getattr(searcher, "best_estimator_", None) if not best_estimator_: warnings.warn( "GridSearchCV object has no attribute " "'best_estimator_', because either it's " "nested gridsearch or `refit` is False!" ) return # clean prams best_estimator_ = clean_params(best_estimator_) main_est = get_main_estimator(best_estimator_) if hasattr(main_est, "model_") and hasattr(main_est, "save_weights"): if outfile_weights: main_est.save_weights(outfile_weights) del main_est.model_ del main_est.fit_params del main_est.model_class_ del main_est.validation_data if getattr(main_est, "data_generator_", None): del main_est.data_generator_ with open(outfile_object, "wb") as output_handler: print("Best estimator is saved: %s " % repr(best_estimator_)) pickle.dump(best_estimator_, output_handler, pickle.HIGHEST_PROTOCOL)
def main(inputs, infile_estimator, outfile_predict, infile_weights=None, infile1=None, fasta_path=None, ref_seq=None, vcf_path=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : strgit File path to trained estimator input outfile_predict : str File path to save the prediction results, tabular infile_weights : str File path to weights input infile1 : str File path to dataset containing features fasta_path : str File path to dataset containing fasta file ref_seq : str File path to dataset containing the reference genome sequence. vcf_path : str File path to dataset containing variants info. """ warnings.filterwarnings('ignore') with open(inputs, 'r') as param_handler: params = json.load(param_handler) # load model with open(infile_estimator, 'rb') as est_handler: estimator = load_model(est_handler) main_est = estimator if isinstance(estimator, Pipeline): main_est = estimator.steps[-1][-1] if hasattr(main_est, 'config') and hasattr(main_est, 'load_weights'): if not infile_weights or infile_weights == 'None': raise ValueError("The selected model skeleton asks for weights, " "but dataset for weights wan not selected!") main_est.load_weights(infile_weights) # handle data input input_type = params['input_options']['selected_input'] # tabular input if input_type == 'tabular': header = 'infer' if params['input_options']['header1'] else None column_option = (params['input_options']['column_selector_options_1'] ['selected_column_selector_option']) if column_option in [ 'by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name' ]: c = params['input_options']['column_selector_options_1']['col1'] else: c = None df = pd.read_csv(infile1, sep='\t', header=header, parse_dates=True) X = read_columns(df, c=c, c_option=column_option).astype(float) if params['method'] == 'predict': preds = estimator.predict(X) else: preds = estimator.predict_proba(X) # sparse input elif input_type == 'sparse': X = mmread(open(infile1, 'r')) if params['method'] == 'predict': preds = estimator.predict(X) else: preds = estimator.predict_proba(X) # fasta input elif input_type == 'seq_fasta': if not hasattr(estimator, 'data_batch_generator'): raise ValueError("To do prediction on sequences in fasta input, " "the estimator must be a `KerasGBatchClassifier`" "equipped with data_batch_generator!") pyfaidx = get_module('pyfaidx') sequences = pyfaidx.Fasta(fasta_path) n_seqs = len(sequences.keys()) X = np.arange(n_seqs)[:, np.newaxis] seq_length = estimator.data_batch_generator.seq_length batch_size = getattr(estimator, 'batch_size', 32) steps = (n_seqs + batch_size - 1) // batch_size seq_type = params['input_options']['seq_type'] klass = try_get_attr('galaxy_ml.preprocessors', seq_type) pred_data_generator = klass(fasta_path, seq_length=seq_length) if params['method'] == 'predict': preds = estimator.predict(X, data_generator=pred_data_generator, steps=steps) else: preds = estimator.predict_proba(X, data_generator=pred_data_generator, steps=steps) # vcf input elif input_type == 'variant_effect': klass = try_get_attr('galaxy_ml.preprocessors', 'GenomicVariantBatchGenerator') options = params['input_options'] options.pop('selected_input') if options['blacklist_regions'] == 'none': options['blacklist_regions'] = None pred_data_generator = klass(ref_genome_path=ref_seq, vcf_path=vcf_path, **options) pred_data_generator.fit() preds = estimator.model_.predict_generator( pred_data_generator.flow(batch_size=32), workers=N_JOBS, use_multiprocessing=True) if preds.min() < 0. or preds.max() > 1.: warnings.warn('Network returning invalid probability values. ' 'The last layer might not normalize predictions ' 'into probabilities ' '(like softmax or sigmoid would).') if params['method'] == 'predict_proba' and preds.shape[1] == 1: # first column is probability of class 0 and second is of class 1 preds = np.hstack([1 - preds, preds]) elif params['method'] == 'predict': if preds.shape[-1] > 1: # if the last activation is `softmax`, the sum of all # probibilities will 1, the classification is considered as # multi-class problem, otherwise, we take it as multi-label. act = getattr(estimator.model_.layers[-1], 'activation', None) if act and act.__name__ == 'softmax': classes = preds.argmax(axis=-1) else: preds = (preds > 0.5).astype('int32') else: classes = (preds > 0.5).astype('int32') preds = estimator.classes_[classes] # end input # output if input_type == 'variant_effect': # TODO: save in batchs rval = pd.DataFrame(preds) meta = pd.DataFrame( pred_data_generator.variants, columns=['chrom', 'pos', 'name', 'ref', 'alt', 'strand']) rval = pd.concat([meta, rval], axis=1) elif len(preds.shape) == 1: rval = pd.DataFrame(preds, columns=['Predicted']) else: rval = pd.DataFrame(preds) rval.to_csv(outfile_predict, sep='\t', header=True, index=False)
def main(inputs, infile_estimator, outfile_predict, infile_weights=None, infile1=None, fasta_path=None, ref_seq=None, vcf_path=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str File path to trained estimator input outfile_predict : str File path to save the prediction results, tabular infile_weights : str File path to weights input infile1 : str File path to dataset containing features fasta_path : str File path to dataset containing fasta file ref_seq : str File path to dataset containing the reference genome sequence. vcf_path : str File path to dataset containing variants info. """ warnings.filterwarnings('ignore') with open(inputs, 'r') as param_handler: params = json.load(param_handler) # load model with open(infile_estimator, 'rb') as est_handler: estimator = load_model(est_handler) main_est = estimator if isinstance(estimator, Pipeline): main_est = estimator.steps[-1][-1] if hasattr(main_est, 'config') and hasattr(main_est, 'load_weights'): if not infile_weights or infile_weights == 'None': raise ValueError("The selected model skeleton asks for weights, " "but dataset for weights wan not selected!") main_est.load_weights(infile_weights) # handle data input input_type = params['input_options']['selected_input'] # tabular input if input_type == 'tabular': header = 'infer' if params['input_options']['header1'] else None column_option = (params['input_options']['column_selector_options_1'] ['selected_column_selector_option']) if column_option in [ 'by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name' ]: c = params['input_options']['column_selector_options_1']['col1'] else: c = None df = pd.read_csv(infile1, sep='\t', header=header, parse_dates=True) X = read_columns(df, c=c, c_option=column_option).astype(float) if params['method'] == 'predict': preds = estimator.predict(X) else: preds = estimator.predict_proba(X) # sparse input elif input_type == 'sparse': X = mmread(open(infile1, 'r')) if params['method'] == 'predict': preds = estimator.predict(X) else: preds = estimator.predict_proba(X) # fasta input elif input_type == 'seq_fasta': if not hasattr(estimator, 'data_batch_generator'): raise ValueError("To do prediction on sequences in fasta input, " "the estimator must be a `KerasGBatchClassifier`" "equipped with data_batch_generator!") pyfaidx = get_module('pyfaidx') sequences = pyfaidx.Fasta(fasta_path) n_seqs = len(sequences.keys()) X = np.arange(n_seqs)[:, np.newaxis] seq_length = estimator.data_batch_generator.seq_length batch_size = getattr(estimator, 'batch_size', 32) steps = (n_seqs + batch_size - 1) // batch_size seq_type = params['input_options']['seq_type'] klass = try_get_attr('galaxy_ml.preprocessors', seq_type) pred_data_generator = klass(fasta_path, seq_length=seq_length) if params['method'] == 'predict': preds = estimator.predict(X, data_generator=pred_data_generator, steps=steps) else: preds = estimator.predict_proba(X, data_generator=pred_data_generator, steps=steps) # vcf input elif input_type == 'variant_effect': klass = try_get_attr('galaxy_ml.preprocessors', 'GenomicVariantBatchGenerator') options = params['input_options'] options.pop('selected_input') if options['blacklist_regions'] == 'none': options['blacklist_regions'] = None pred_data_generator = klass(ref_genome_path=ref_seq, vcf_path=vcf_path, **options) pred_data_generator.set_processing_attrs() variants = pred_data_generator.variants # predict 1600 sample at once then write to file gen_flow = pred_data_generator.flow(batch_size=1600) file_writer = open(outfile_predict, 'w') header_row = '\t'.join( ['chrom', 'pos', 'name', 'ref', 'alt', 'strand']) file_writer.write(header_row) header_done = False steps_done = 0 # TODO: multiple threading try: while steps_done < len(gen_flow): index_array = next(gen_flow.index_generator) batch_X = gen_flow._get_batches_of_transformed_samples( index_array) if params['method'] == 'predict': batch_preds = estimator.predict( batch_X, # The presence of `pred_data_generator` below is to # override model carrying data_generator if there # is any. data_generator=pred_data_generator) else: batch_preds = estimator.predict_proba( batch_X, # The presence of `pred_data_generator` below is to # override model carrying data_generator if there # is any. data_generator=pred_data_generator) if batch_preds.ndim == 1: batch_preds = batch_preds[:, np.newaxis] batch_meta = variants[index_array] batch_out = np.column_stack([batch_meta, batch_preds]) if not header_done: heads = np.arange(batch_preds.shape[-1]).astype(str) heads_str = '\t'.join(heads) file_writer.write("\t%s\n" % heads_str) header_done = True for row in batch_out: row_str = '\t'.join(row) file_writer.write("%s\n" % row_str) steps_done += 1 finally: file_writer.close() # TODO: make api `pred_data_generator.close()` pred_data_generator.close() return 0 # end input # output if len(preds.shape) == 1: rval = pd.DataFrame(preds, columns=['Predicted']) else: rval = pd.DataFrame(preds) rval.to_csv(outfile_predict, sep='\t', header=True, index=False)
def main( inputs, infile_estimator, infile1, infile2, outfile_result, outfile_object=None, outfile_weights=None, outfile_y_true=None, outfile_y_preds=None, groups=None, ref_seq=None, intervals=None, targets=None, fasta_path=None, ): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str File path to estimator infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target values outfile_result : str File path to save the results, either cv_results or test result outfile_object : str, optional File path to save searchCV object outfile_weights : str, optional File path to save deep learning model weights outfile_y_true : str, optional File path to target values for prediction outfile_y_preds : str, optional File path to save deep learning model weights groups : str File path to dataset containing groups labels ref_seq : str File path to dataset containing genome sequence file intervals : str File path to dataset containing interval file targets : str File path to dataset compressed target bed file fasta_path : str File path to dataset containing fasta file """ warnings.simplefilter("ignore") with open(inputs, "r") as param_handler: params = json.load(param_handler) # load estimator with open(infile_estimator, "rb") as estimator_handler: estimator = load_model(estimator_handler) estimator = clean_params(estimator) # swap hyperparameter swapping = params["experiment_schemes"]["hyperparams_swapping"] swap_params = _eval_swap_params(swapping) estimator.set_params(**swap_params) estimator_params = estimator.get_params() # store read dataframe object loaded_df = {} input_type = params["input_options"]["selected_input"] # tabular input if input_type == "tabular": header = "infer" if params["input_options"]["header1"] else None column_option = params["input_options"]["column_selector_options_1"][ "selected_column_selector_option" ] if column_option in [ "by_index_number", "all_but_by_index_number", "by_header_name", "all_but_by_header_name", ]: c = params["input_options"]["column_selector_options_1"]["col1"] else: c = None df_key = infile1 + repr(header) df = pd.read_csv(infile1, sep="\t", header=header, parse_dates=True) loaded_df[df_key] = df X = read_columns(df, c=c, c_option=column_option).astype(float) # sparse input elif input_type == "sparse": X = mmread(open(infile1, "r")) # fasta_file input elif input_type == "seq_fasta": pyfaidx = get_module("pyfaidx") sequences = pyfaidx.Fasta(fasta_path) n_seqs = len(sequences.keys()) X = np.arange(n_seqs)[:, np.newaxis] for param in estimator_params.keys(): if param.endswith("fasta_path"): estimator.set_params(**{param: fasta_path}) break else: raise ValueError( "The selected estimator doesn't support " "fasta file input! Please consider using " "KerasGBatchClassifier with " "FastaDNABatchGenerator/FastaProteinBatchGenerator " "or having GenomeOneHotEncoder/ProteinOneHotEncoder " "in pipeline!" ) elif input_type == "refseq_and_interval": path_params = { "data_batch_generator__ref_genome_path": ref_seq, "data_batch_generator__intervals_path": intervals, "data_batch_generator__target_path": targets, } estimator.set_params(**path_params) n_intervals = sum(1 for line in open(intervals)) X = np.arange(n_intervals)[:, np.newaxis] # Get target y header = "infer" if params["input_options"]["header2"] else None column_option = params["input_options"]["column_selector_options_2"][ "selected_column_selector_option2" ] if column_option in [ "by_index_number", "all_but_by_index_number", "by_header_name", "all_but_by_header_name", ]: c = params["input_options"]["column_selector_options_2"]["col2"] else: c = None df_key = infile2 + repr(header) if df_key in loaded_df: infile2 = loaded_df[df_key] else: infile2 = pd.read_csv(infile2, sep="\t", header=header, parse_dates=True) loaded_df[df_key] = infile2 y = read_columns( infile2, c=c, c_option=column_option, sep="\t", header=header, parse_dates=True ) if len(y.shape) == 2 and y.shape[1] == 1: y = y.ravel() if input_type == "refseq_and_interval": estimator.set_params(data_batch_generator__features=y.ravel().tolist()) y = None # end y # load groups if groups: groups_selector = ( params["experiment_schemes"]["test_split"]["split_algos"] ).pop("groups_selector") header = "infer" if groups_selector["header_g"] else None column_option = groups_selector["column_selector_options_g"][ "selected_column_selector_option_g" ] if column_option in [ "by_index_number", "all_but_by_index_number", "by_header_name", "all_but_by_header_name", ]: c = groups_selector["column_selector_options_g"]["col_g"] else: c = None df_key = groups + repr(header) if df_key in loaded_df: groups = loaded_df[df_key] groups = read_columns( groups, c=c, c_option=column_option, sep="\t", header=header, parse_dates=True, ) groups = groups.ravel() # del loaded_df del loaded_df # cache iraps_core fits could increase search speed significantly memory = joblib.Memory(location=CACHE_DIR, verbose=0) main_est = get_main_estimator(estimator) if main_est.__class__.__name__ == "IRAPSClassifier": main_est.set_params(memory=memory) # handle scorer, convert to scorer dict scoring = params["experiment_schemes"]["metrics"]["scoring"] if scoring is not None: # get_scoring() expects secondary_scoring to be a comma separated string (not a list) # Check if secondary_scoring is specified secondary_scoring = scoring.get("secondary_scoring", None) if secondary_scoring is not None: # If secondary_scoring is specified, convert the list into comman separated string scoring["secondary_scoring"] = ",".join(scoring["secondary_scoring"]) scorer = get_scoring(scoring) scorer, _ = _check_multimetric_scoring(estimator, scoring=scorer) # handle test (first) split test_split_options = params["experiment_schemes"]["test_split"]["split_algos"] if test_split_options["shuffle"] == "group": test_split_options["labels"] = groups if test_split_options["shuffle"] == "stratified": if y is not None: test_split_options["labels"] = y else: raise ValueError( "Stratified shuffle split is not " "applicable on empty target values!" ) ( X_train, X_test, y_train, y_test, groups_train, _groups_test, ) = train_test_split_none(X, y, groups, **test_split_options) exp_scheme = params["experiment_schemes"]["selected_exp_scheme"] # handle validation (second) split if exp_scheme == "train_val_test": val_split_options = params["experiment_schemes"]["val_split"]["split_algos"] if val_split_options["shuffle"] == "group": val_split_options["labels"] = groups_train if val_split_options["shuffle"] == "stratified": if y_train is not None: val_split_options["labels"] = y_train else: raise ValueError( "Stratified shuffle split is not " "applicable on empty target values!" ) ( X_train, X_val, y_train, y_val, groups_train, _groups_val, ) = train_test_split_none(X_train, y_train, groups_train, **val_split_options) # train and eval if hasattr(estimator, "validation_data"): if exp_scheme == "train_val_test": estimator.fit(X_train, y_train, validation_data=(X_val, y_val)) else: estimator.fit(X_train, y_train, validation_data=(X_test, y_test)) else: estimator.fit(X_train, y_train) if hasattr(estimator, "evaluate"): steps = estimator.prediction_steps batch_size = estimator.batch_size generator = estimator.data_generator_.flow( X_test, y=y_test, batch_size=batch_size ) predictions, y_true = _predict_generator( estimator.model_, generator, steps=steps ) scores = _evaluate(y_true, predictions, scorer, is_multimetric=True) else: if hasattr(estimator, "predict_proba"): predictions = estimator.predict_proba(X_test) else: predictions = estimator.predict(X_test) y_true = y_test scores = _score(estimator, X_test, y_test, scorer, is_multimetric=True) if outfile_y_true: try: pd.DataFrame(y_true).to_csv(outfile_y_true, sep="\t", index=False) pd.DataFrame(predictions).astype(np.float32).to_csv( outfile_y_preds, sep="\t", index=False, float_format="%g", chunksize=10000, ) except Exception as e: print("Error in saving predictions: %s" % e) # handle output for name, score in scores.items(): scores[name] = [score] df = pd.DataFrame(scores) df = df[sorted(df.columns)] df.to_csv(path_or_buf=outfile_result, sep="\t", header=True, index=False) memory.clear(warn=False) if outfile_object: main_est = estimator if isinstance(estimator, Pipeline): main_est = estimator.steps[-1][-1] if hasattr(main_est, "model_") and hasattr(main_est, "save_weights"): if outfile_weights: main_est.save_weights(outfile_weights) del main_est.model_ del main_est.fit_params del main_est.model_class_ if getattr(main_est, "validation_data", None): del main_est.validation_data if getattr(main_est, "data_generator_", None): del main_est.data_generator_ with open(outfile_object, "wb") as output_handler: pickle.dump(estimator, output_handler, pickle.HIGHEST_PROTOCOL)
def main(inputs, infile_estimator=None, infile1=None, infile2=None, outfile_result=None, outfile_object=None, groups=None, ref_seq=None, intervals=None, targets=None, fasta_path=None, model_config=None, true_labels=None, predicted_labels=None, plot_color=None, title=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str, default is None File path to estimator infile1 : str, default is None File path to dataset containing features or true labels. infile2 : str, default is None File path to dataset containing target values or predicted probabilities. outfile_result : str, default is None File path to save the results, either cv_results or test result outfile_object : str, default is None File path to save searchCV object groups : str, default is None File path to dataset containing groups labels ref_seq : str, default is None File path to dataset containing genome sequence file intervals : str, default is None File path to dataset containing interval file targets : str, default is None File path to dataset compressed target bed file fasta_path : str, default is None File path to dataset containing fasta file model_config : str, default is None File path to dataset containing JSON config for neural networks true_labels : str, default is None File path to dataset containing true labels predicted_labels : str, default is None File path to dataset containing true predicted labels plot_color : str, default is None Color of the confusion matrix heatmap title : str, default is None Title of the confusion matrix heatmap """ warnings.simplefilter('ignore') with open(inputs, 'r') as param_handler: params = json.load(param_handler) title = params['plotting_selection']['title'].strip() plot_type = params['plotting_selection']['plot_type'] plot_format = params['plotting_selection']['plot_format'] if plot_type == 'feature_importances': with open(infile_estimator, 'rb') as estimator_handler: estimator = load_model(estimator_handler) column_option = ( params['plotting_selection']['column_selector_options'] ['selected_column_selector_option']) if column_option in [ 'by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name' ]: c = (params['plotting_selection']['column_selector_options'] ['col1']) else: c = None _, input_df = read_columns(infile1, c=c, c_option=column_option, return_df=True, sep='\t', header='infer', parse_dates=True) feature_names = input_df.columns.values if isinstance(estimator, Pipeline): for st in estimator.steps[:-1]: if isinstance(st[-1], SelectorMixin): mask = st[-1].get_support() feature_names = feature_names[mask] estimator = estimator.steps[-1][-1] if hasattr(estimator, 'coef_'): coefs = estimator.coef_ else: coefs = getattr(estimator, 'feature_importances_', None) if coefs is None: raise RuntimeError('The classifier does not expose ' '"coef_" or "feature_importances_" ' 'attributes') threshold = params['plotting_selection']['threshold'] if threshold is not None: mask = (coefs > threshold) | (coefs < -threshold) coefs = coefs[mask] feature_names = feature_names[mask] # sort indices = np.argsort(coefs)[::-1] trace = go.Bar(x=feature_names[indices], y=coefs[indices]) layout = go.Layout(title=title or "Feature Importances") fig = go.Figure(data=[trace], layout=layout) plotly.offline.plot(fig, filename="output.html", auto_open=False) # to be discovered by `from_work_dir` os.rename('output.html', 'output') return 0 elif plot_type in ('pr_curve', 'roc_curve'): df1 = pd.read_csv(infile1, sep='\t', header='infer') df2 = pd.read_csv(infile2, sep='\t', header='infer').astype(np.float32) minimum = params['plotting_selection']['report_minimum_n_positives'] # filter out columns whose n_positives is beblow the threhold if minimum: mask = df1.sum(axis=0) >= minimum df1 = df1.loc[:, mask] df2 = df2.loc[:, mask] pos_label = params['plotting_selection']['pos_label'].strip() \ or None if plot_type == 'pr_curve': if plot_format == 'plotly_html': visualize_pr_curve_plotly(df1, df2, pos_label, title=title) else: visualize_pr_curve_matplotlib(df1, df2, pos_label, title) else: # 'roc_curve' drop_intermediate = ( params['plotting_selection']['drop_intermediate']) if plot_format == 'plotly_html': visualize_roc_curve_plotly(df1, df2, pos_label, drop_intermediate=drop_intermediate, title=title) else: visualize_roc_curve_matplotlib( df1, df2, pos_label, drop_intermediate=drop_intermediate, title=title) return 0 elif plot_type == 'rfecv_gridscores': input_df = pd.read_csv(infile1, sep='\t', header='infer') scores = input_df.iloc[:, 0] steps = params['plotting_selection']['steps'].strip() steps = safe_eval(steps) data = go.Scatter(x=list(range(len(scores))), y=scores, text=[str(_) for _ in steps] if steps else None, mode='lines') layout = go.Layout( xaxis=dict(title="Number of features selected"), yaxis=dict(title="Cross validation score"), title=dict(text=title or None, x=0.5, y=0.92, xanchor='center', yanchor='top'), font=dict(family="sans-serif", size=11), # control backgroud colors plot_bgcolor='rgba(255,255,255,0)') """ # legend=dict( # x=0.95, # y=0, # traceorder="normal", # font=dict( # family="sans-serif", # size=9, # color="black" # ), # bgcolor="LightSteelBlue", # bordercolor="Black", # borderwidth=2 # ), """ fig = go.Figure(data=[data], layout=layout) plotly.offline.plot(fig, filename="output.html", auto_open=False) # to be discovered by `from_work_dir` os.rename('output.html', 'output') return 0 elif plot_type == 'learning_curve': input_df = pd.read_csv(infile1, sep='\t', header='infer') plot_std_err = params['plotting_selection']['plot_std_err'] data1 = go.Scatter( x=input_df['train_sizes_abs'], y=input_df['mean_train_scores'], error_y=dict( array=input_df['std_train_scores']) if plot_std_err else None, mode='lines', name="Train Scores", ) data2 = go.Scatter( x=input_df['train_sizes_abs'], y=input_df['mean_test_scores'], error_y=dict( array=input_df['std_test_scores']) if plot_std_err else None, mode='lines', name="Test Scores", ) layout = dict( xaxis=dict(title='No. of samples'), yaxis=dict(title='Performance Score'), # modify these configurations to customize image title=dict(text=title or 'Learning Curve', x=0.5, y=0.92, xanchor='center', yanchor='top'), font=dict(family="sans-serif", size=11), # control backgroud colors plot_bgcolor='rgba(255,255,255,0)') """ # legend=dict( # x=0.95, # y=0, # traceorder="normal", # font=dict( # family="sans-serif", # size=9, # color="black" # ), # bgcolor="LightSteelBlue", # bordercolor="Black", # borderwidth=2 # ), """ fig = go.Figure(data=[data1, data2], layout=layout) plotly.offline.plot(fig, filename="output.html", auto_open=False) # to be discovered by `from_work_dir` os.rename('output.html', 'output') return 0 elif plot_type == 'keras_plot_model': with open(model_config, 'r') as f: model_str = f.read() model = model_from_json(model_str) plot_model(model, to_file="output.png") os.rename('output.png', 'output') return 0 elif plot_type == 'classification_confusion_matrix': plot_selection = params["plotting_selection"] input_true = get_dataframe(true_labels, plot_selection, "header_true", "column_selector_options_true") header_predicted = 'infer' if plot_selection[ "header_predicted"] else None input_predicted = pd.read_csv(predicted_labels, sep='\t', parse_dates=True, header=header_predicted) true_classes = input_true.iloc[:, -1].copy() predicted_classes = input_predicted.iloc[:, -1].copy() axis_labels = list(set(true_classes)) c_matrix = confusion_matrix(true_classes, predicted_classes) fig, ax = plt.subplots(figsize=(7, 7)) im = plt.imshow(c_matrix, cmap=plot_color) for i in range(len(c_matrix)): for j in range(len(c_matrix)): ax.text(j, i, c_matrix[i, j], ha="center", va="center", color="k") ax.set_ylabel('True class labels') ax.set_xlabel('Predicted class labels') ax.set_title(title) ax.set_xticks(axis_labels) ax.set_yticks(axis_labels) fig.colorbar(im, ax=ax) fig.tight_layout() plt.savefig("output.png", dpi=125) os.rename('output.png', 'output') return 0
def main(inputs, infile_estimator, infile_images, infile_dataframe, outfile_result, infile_weights=None, outfile_object=None, outfile_weights=None, outfile_y_true=None, outfile_y_preds=None, groups=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str File path to estimator infile_images : str File path to datasets containing images infile_dataframe : str File path to tabular dataset containing image information outfile_result : str File path to save the results, either cv_results or test result infile_weights : str File path to input model weights, used in evaluation and prediction mode. outfile_object : str, optional File path to save searchCV object outfile_weights : str, optional File path to save deep learning model weights outfile_y_true : str, optional File path to target values for prediction outfile_y_preds : str, optional File path to save deep learning model weights groups : str File path to dataset containing groups labels """ warnings.simplefilter('ignore') with open(inputs, 'r') as param_handler: params = json.load(param_handler) # load estimator with open(infile_estimator, 'rb') as estimator_handler: estimator = load_model(estimator_handler) estimator = clean_params(estimator) if not isinstance(estimator, KerasGBatchClassifier): raise ValueError( "Only `galaxy_ml.keras_galaxy_models.KerasGBatchClassifier` " "is supported!") # read DataFrame for images data_frame = pd.read_csv(infile_dataframe, sep='\t', header='infer') kwargs = _handle_image_generator_params(params['input_options'], data_frame) # build data generator image_generator = ImageDataFrameBatchGenerator(dataframe=data_frame, directory=IMAGES_DIR, **kwargs) estimator.set_params(data_batch_generator=image_generator) steps = estimator.prediction_steps batch_size = estimator.batch_size # Get X and y X = np.arange(data_frame.shape[0])[:, np.newaxis] if isinstance(kwargs['y_col'], list): y = None else: y = data_frame[kwargs['y_col']].ravel() # load groups if groups: groups_selector = (params['experiment_schemes']['test_split'] ['split_algos']).pop('groups_selector') header = 'infer' if groups_selector['header_g'] else None column_option = \ (groups_selector['column_selector_options_g'] ['selected_column_selector_option_g']) if column_option in [ 'by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name' ]: c = groups_selector['column_selector_options_g']['col_g'] else: c = None if groups == infile_dataframe: groups = data_frame groups = read_columns(groups, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) groups = groups.ravel() exp_scheme = params['experiment_schemes']['selected_exp_scheme'] # Model Predictions if exp_scheme == 'model_predict': estimator.load_weights(infile_weights) steps = params['experiment_schemes']['pred_steps'] generator = image_generator.flow(X, y=None, batch_size=batch_size) predictions = estimator.model_.predict_generator(generator, steps=steps) try: pd.DataFrame(predictions).astype(np.float32).to_csv( outfile_result, sep='\t', index=False, float_format='%g', chunksize=10000) except Exception as e: print("Error in saving predictions: %s" % e) return 0 # Model Evaluation if exp_scheme == 'model_eval': estimator.load_weights(infile_weights) # compile model estimator.model_.compile(loss=estimator.loss, optimizer=estimator._optimizer, metrics=estimator.metrics) steps = params['experiment_schemes']['eval_steps'] sk_scoring = params['experiment_schemes']['metrics']['scoring'] scores, predictions, y_true = _evaluate_keras_and_sklearn_scores( estimator, image_generator, X, sk_scoring=sk_scoring, steps=steps, batch_size=batch_size, return_predictions=bool(outfile_y_true)) # for other two modes, train/val and train/val/test else: # swap hyperparameter swapping = params['experiment_schemes']['hyperparams_swapping'] swap_params = _eval_swap_params(swapping) estimator.set_params(**swap_params) # handle test (first) split test_split_options = ( params['experiment_schemes']['test_split']['split_algos']) if test_split_options['shuffle'] == 'group': test_split_options['labels'] = groups if test_split_options['shuffle'] == 'stratified': if y is not None: test_split_options['labels'] = y else: raise ValueError("Stratified shuffle split is not " "applicable on empty target values or " "multiple output targets!") X_train, X_test, y_train, y_test, groups_train, groups_test = \ train_test_split_none(X, y, groups, **test_split_options) # handle validation (second) split if exp_scheme == 'train_val_test': val_split_options = ( params['experiment_schemes']['val_split']['split_algos']) if val_split_options['shuffle'] == 'group': val_split_options['labels'] = groups_train if val_split_options['shuffle'] == 'stratified': if y_train is not None: val_split_options['labels'] = y_train else: raise ValueError("Stratified shuffle split is not " "applicable on empty target values!") X_train, X_val, y_train, y_val, groups_train, groups_val = \ train_test_split_none(X_train, y_train, groups_train, **val_split_options) # In image data generator, `y_val` must be None # labels will be retrived in generator. estimator.fit(X_train, y_train, validation_data=(X_val, )) else: estimator.fit(X_train, y_train, validation_data=(X_test, )) data_generator = estimator.data_generator_ sk_scoring = params['experiment_schemes']['metrics']['scoring'] steps = estimator.prediction_steps scores, predictions, y_true = _evaluate_keras_and_sklearn_scores( estimator, data_generator, X_test, sk_scoring=sk_scoring, steps=steps, batch_size=batch_size, return_predictions=bool(outfile_y_true)) # handle output if outfile_y_true: try: pd.DataFrame(y_true).to_csv(outfile_y_true, sep='\t', index=False) pd.DataFrame(predictions).astype(np.float32).to_csv( outfile_y_preds, sep='\t', index=False, float_format='%g', chunksize=10000) except Exception as e: print("Error in saving predictions: %s" % e) # handle output for name, score in scores.items(): scores[name] = [score] df = pd.DataFrame(scores) df = df[sorted(df.columns)] df.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) if outfile_object: main_est = get_main_estimator(estimator) if hasattr(main_est, 'model_') \ and hasattr(main_est, 'save_weights'): if outfile_weights: main_est.save_weights(outfile_weights) del main_est.model_ del main_est.fit_params del main_est.model_class_ main_est.data_batch_generator.dataframe = None main_est.callbacks = [] if getattr(main_est, 'data_generator_', None): del main_est.data_generator_ with open(outfile_object, 'wb') as output_handler: pickle.dump(estimator, output_handler, pickle.HIGHEST_PROTOCOL)