def get_batch_generator(config): """Construct keras online data generator from Galaxy tool parameters Parameters ----------- config : dictionary, galaxy tool parameters loaded by JSON """ generator_type = config.pop('generator_type') klass = try_get_attr('galaxy_ml.preprocessors', generator_type) if generator_type == 'GenomicIntervalBatchGenerator': config['ref_genome_path'] = 'to_be_determined' config['intervals_path'] = 'to_be_determined' config['target_path'] = 'to_be_determined' config['features'] = 'to_be_determined' else: config['fasta_path'] = 'to_be_determined' return klass(**config)
def get_batch_generator(config): """ Construct keras online data generator from Galaxy tool parameters Parameters ----------- config : dictionary, galaxy tool parameters loaded by JSON """ generator_type = config.pop("generator_type") if generator_type == "none": return None klass = try_get_attr("galaxy_ml.preprocessors", generator_type) if generator_type == "GenomicIntervalBatchGenerator": config["ref_genome_path"] = "to_be_determined" config["intervals_path"] = "to_be_determined" config["target_path"] = "to_be_determined" config["features"] = "to_be_determined" else: config["fasta_path"] = "to_be_determined" return klass(**config)
def main(inputs, infile_estimator, outfile_predict, infile_weights=None, infile1=None, fasta_path=None, ref_seq=None, vcf_path=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str File path to trained estimator input outfile_predict : str File path to save the prediction results, tabular infile_weights : str File path to weights input infile1 : str File path to dataset containing features fasta_path : str File path to dataset containing fasta file ref_seq : str File path to dataset containing the reference genome sequence. vcf_path : str File path to dataset containing variants info. """ warnings.filterwarnings('ignore') with open(inputs, 'r') as param_handler: params = json.load(param_handler) # load model with open(infile_estimator, 'rb') as est_handler: estimator = load_model(est_handler) main_est = estimator if isinstance(estimator, Pipeline): main_est = estimator.steps[-1][-1] if hasattr(main_est, 'config') and hasattr(main_est, 'load_weights'): if not infile_weights or infile_weights == 'None': raise ValueError("The selected model skeleton asks for weights, " "but dataset for weights wan not selected!") main_est.load_weights(infile_weights) # handle data input input_type = params['input_options']['selected_input'] # tabular input if input_type == 'tabular': header = 'infer' if params['input_options']['header1'] else None column_option = (params['input_options']['column_selector_options_1'] ['selected_column_selector_option']) if column_option in [ 'by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name' ]: c = params['input_options']['column_selector_options_1']['col1'] else: c = None df = pd.read_csv(infile1, sep='\t', header=header, parse_dates=True) X = read_columns(df, c=c, c_option=column_option).astype(float) if params['method'] == 'predict': preds = estimator.predict(X) else: preds = estimator.predict_proba(X) # sparse input elif input_type == 'sparse': X = mmread(open(infile1, 'r')) if params['method'] == 'predict': preds = estimator.predict(X) else: preds = estimator.predict_proba(X) # fasta input elif input_type == 'seq_fasta': if not hasattr(estimator, 'data_batch_generator'): raise ValueError("To do prediction on sequences in fasta input, " "the estimator must be a `KerasGBatchClassifier`" "equipped with data_batch_generator!") pyfaidx = get_module('pyfaidx') sequences = pyfaidx.Fasta(fasta_path) n_seqs = len(sequences.keys()) X = np.arange(n_seqs)[:, np.newaxis] seq_length = estimator.data_batch_generator.seq_length batch_size = getattr(estimator, 'batch_size', 32) steps = (n_seqs + batch_size - 1) // batch_size seq_type = params['input_options']['seq_type'] klass = try_get_attr('galaxy_ml.preprocessors', seq_type) pred_data_generator = klass(fasta_path, seq_length=seq_length) if params['method'] == 'predict': preds = estimator.predict(X, data_generator=pred_data_generator, steps=steps) else: preds = estimator.predict_proba(X, data_generator=pred_data_generator, steps=steps) # vcf input elif input_type == 'variant_effect': klass = try_get_attr('galaxy_ml.preprocessors', 'GenomicVariantBatchGenerator') options = params['input_options'] options.pop('selected_input') if options['blacklist_regions'] == 'none': options['blacklist_regions'] = None pred_data_generator = klass(ref_genome_path=ref_seq, vcf_path=vcf_path, **options) pred_data_generator.set_processing_attrs() variants = pred_data_generator.variants # predict 1600 sample at once then write to file gen_flow = pred_data_generator.flow(batch_size=1600) file_writer = open(outfile_predict, 'w') header_row = '\t'.join( ['chrom', 'pos', 'name', 'ref', 'alt', 'strand']) file_writer.write(header_row) header_done = False steps_done = 0 # TODO: multiple threading try: while steps_done < len(gen_flow): index_array = next(gen_flow.index_generator) batch_X = gen_flow._get_batches_of_transformed_samples( index_array) if params['method'] == 'predict': batch_preds = estimator.predict( batch_X, # The presence of `pred_data_generator` below is to # override model carrying data_generator if there # is any. data_generator=pred_data_generator) else: batch_preds = estimator.predict_proba( batch_X, # The presence of `pred_data_generator` below is to # override model carrying data_generator if there # is any. data_generator=pred_data_generator) if batch_preds.ndim == 1: batch_preds = batch_preds[:, np.newaxis] batch_meta = variants[index_array] batch_out = np.column_stack([batch_meta, batch_preds]) if not header_done: heads = np.arange(batch_preds.shape[-1]).astype(str) heads_str = '\t'.join(heads) file_writer.write("\t%s\n" % heads_str) header_done = True for row in batch_out: row_str = '\t'.join(row) file_writer.write("%s\n" % row_str) steps_done += 1 finally: file_writer.close() # TODO: make api `pred_data_generator.close()` pred_data_generator.close() return 0 # end input # output if len(preds.shape) == 1: rval = pd.DataFrame(preds, columns=['Predicted']) else: rval = pd.DataFrame(preds) rval.to_csv(outfile_predict, sep='\t', header=True, index=False)
gaussian_process, kernel_approximation, metrics, model_selection, naive_bayes, neighbors, pipeline, preprocessing, svm, linear_model, tree, discriminant_analysis) from sklearn.exceptions import FitFailedWarning from sklearn.metrics.scorer import _check_multimetric_scoring from sklearn.model_selection._validation import _score, cross_validate from sklearn.model_selection import _search, _validation from sklearn.utils import indexable, safe_indexing from galaxy_ml.model_validations import train_test_split from galaxy_ml.utils import (SafeEval, get_scoring, load_model, read_columns, try_get_attr, get_module) _fit_and_score = try_get_attr('galaxy_ml.model_validations', '_fit_and_score') setattr(_search, '_fit_and_score', _fit_and_score) setattr(_validation, '_fit_and_score', _fit_and_score) N_JOBS = int(__import__('os').environ.get('GALAXY_SLOTS', 1)) CACHE_DIR = './cached' NON_SEARCHABLE = ('n_jobs', 'pre_dispatch', 'memory', '_path', 'nthread', 'callbacks') ALLOWED_CALLBACKS = ('EarlyStopping', 'TerminateOnNaN', 'ReduceLROnPlateau', 'CSVLogger', 'None') def _eval_swap_params(params_builder): swap_params = {} for p in params_builder['param_set']:
def _do_train_test_split_val(searcher, X, y, params, error_score='raise', primary_scoring=None, groups=None, outfile=None): """ do train test split, searchCV validates on the train and then use the best_estimator_ to evaluate on the test Returns -------- Fitted SearchCV object """ train_test_split = try_get_attr('galaxy_ml.model_validations', 'train_test_split') split_options = params['outer_split'] # splits if split_options['shuffle'] == 'stratified': split_options['labels'] = y X, X_test, y, y_test = train_test_split(X, y, **split_options) elif split_options['shuffle'] == 'group': if groups is None: raise ValueError("No group based CV option was choosen for " "group shuffle!") split_options['labels'] = groups if y is None: X, X_test, groups, _ =\ train_test_split(X, groups, **split_options) else: X, X_test, y, y_test, groups, _ =\ train_test_split(X, y, groups, **split_options) else: if split_options['shuffle'] == 'None': split_options['shuffle'] = None X, X_test, y, y_test =\ train_test_split(X, y, **split_options) if error_score == 'raise': searcher.fit(X, y, groups=groups) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: searcher.fit(X, y, groups=groups) except ValueError: pass for warning in w: print(repr(warning.message)) scorer_ = searcher.scorer_ if isinstance(scorer_, collections.Mapping): is_multimetric = True else: is_multimetric = False best_estimator_ = getattr(searcher, 'best_estimator_') # TODO Solve deep learning models in pipeline if best_estimator_.__class__.__name__ == 'KerasGBatchClassifier': test_score = best_estimator_.evaluate(X_test, scorer=scorer_, is_multimetric=is_multimetric) else: test_score = _score(best_estimator_, X_test, y_test, scorer_, is_multimetric=is_multimetric) if not is_multimetric: test_score = {primary_scoring: test_score} for key, value in test_score.items(): test_score[key] = [value] result_df = pd.DataFrame(test_score) result_df.to_csv(path_or_buf=outfile, sep='\t', header=True, index=False) return searcher
def main( inputs, infile_estimator, outfile_predict, infile_weights=None, infile1=None, fasta_path=None, ref_seq=None, vcf_path=None, ): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : strgit File path to trained estimator input outfile_predict : str File path to save the prediction results, tabular infile_weights : str File path to weights input infile1 : str File path to dataset containing features fasta_path : str File path to dataset containing fasta file ref_seq : str File path to dataset containing the reference genome sequence. vcf_path : str File path to dataset containing variants info. """ warnings.filterwarnings("ignore") with open(inputs, "r") as param_handler: params = json.load(param_handler) # load model with open(infile_estimator, "rb") as est_handler: estimator = load_model(est_handler) main_est = estimator if isinstance(estimator, Pipeline): main_est = estimator.steps[-1][-1] if hasattr(main_est, "config") and hasattr(main_est, "load_weights"): if not infile_weights or infile_weights == "None": raise ValueError("The selected model skeleton asks for weights, " "but dataset for weights wan not selected!") main_est.load_weights(infile_weights) # handle data input input_type = params["input_options"]["selected_input"] # tabular input if input_type == "tabular": header = "infer" if params["input_options"]["header1"] else None column_option = params["input_options"]["column_selector_options_1"][ "selected_column_selector_option"] if column_option in [ "by_index_number", "all_but_by_index_number", "by_header_name", "all_but_by_header_name", ]: c = params["input_options"]["column_selector_options_1"]["col1"] else: c = None df = pd.read_csv(infile1, sep="\t", header=header, parse_dates=True) X = read_columns(df, c=c, c_option=column_option).astype(float) if params["method"] == "predict": preds = estimator.predict(X) else: preds = estimator.predict_proba(X) # sparse input elif input_type == "sparse": X = mmread(open(infile1, "r")) if params["method"] == "predict": preds = estimator.predict(X) else: preds = estimator.predict_proba(X) # fasta input elif input_type == "seq_fasta": if not hasattr(estimator, "data_batch_generator"): raise ValueError("To do prediction on sequences in fasta input, " "the estimator must be a `KerasGBatchClassifier`" "equipped with data_batch_generator!") pyfaidx = get_module("pyfaidx") sequences = pyfaidx.Fasta(fasta_path) n_seqs = len(sequences.keys()) X = np.arange(n_seqs)[:, np.newaxis] seq_length = estimator.data_batch_generator.seq_length batch_size = getattr(estimator, "batch_size", 32) steps = (n_seqs + batch_size - 1) // batch_size seq_type = params["input_options"]["seq_type"] klass = try_get_attr("galaxy_ml.preprocessors", seq_type) pred_data_generator = klass(fasta_path, seq_length=seq_length) if params["method"] == "predict": preds = estimator.predict(X, data_generator=pred_data_generator, steps=steps) else: preds = estimator.predict_proba(X, data_generator=pred_data_generator, steps=steps) # vcf input elif input_type == "variant_effect": klass = try_get_attr("galaxy_ml.preprocessors", "GenomicVariantBatchGenerator") options = params["input_options"] options.pop("selected_input") if options["blacklist_regions"] == "none": options["blacklist_regions"] = None pred_data_generator = klass(ref_genome_path=ref_seq, vcf_path=vcf_path, **options) pred_data_generator.set_processing_attrs() variants = pred_data_generator.variants # predict 1600 sample at once then write to file gen_flow = pred_data_generator.flow(batch_size=1600) file_writer = open(outfile_predict, "w") header_row = "\t".join( ["chrom", "pos", "name", "ref", "alt", "strand"]) file_writer.write(header_row) header_done = False steps_done = 0 # TODO: multiple threading try: while steps_done < len(gen_flow): index_array = next(gen_flow.index_generator) batch_X = gen_flow._get_batches_of_transformed_samples( index_array) if params["method"] == "predict": batch_preds = estimator.predict( batch_X, # The presence of `pred_data_generator` below is to # override model carrying data_generator if there # is any. data_generator=pred_data_generator, ) else: batch_preds = estimator.predict_proba( batch_X, # The presence of `pred_data_generator` below is to # override model carrying data_generator if there # is any. data_generator=pred_data_generator, ) if batch_preds.ndim == 1: batch_preds = batch_preds[:, np.newaxis] batch_meta = variants[index_array] batch_out = np.column_stack([batch_meta, batch_preds]) if not header_done: heads = np.arange(batch_preds.shape[-1]).astype(str) heads_str = "\t".join(heads) file_writer.write("\t%s\n" % heads_str) header_done = True for row in batch_out: row_str = "\t".join(row) file_writer.write("%s\n" % row_str) steps_done += 1 finally: file_writer.close() # TODO: make api `pred_data_generator.close()` pred_data_generator.close() return 0 # end input # output if len(preds.shape) == 1: rval = pd.DataFrame(preds, columns=["Predicted"]) else: rval = pd.DataFrame(preds) rval.to_csv(outfile_predict, sep="\t", header=True, index=False)
import numpy as np import pandas as pd from galaxy_ml.externals.selene_sdk.utils import compute_score from galaxy_ml.keras_galaxy_models import _predict_generator from galaxy_ml.model_validations import train_test_split from galaxy_ml.utils import (clean_params, get_main_estimator, get_module, get_scoring, load_model, read_columns, SafeEval, try_get_attr) from scipy.io import mmread from sklearn.metrics.scorer import _check_multimetric_scoring from sklearn.model_selection import _search, _validation from sklearn.model_selection._validation import _score from sklearn.pipeline import Pipeline from sklearn.utils import indexable, safe_indexing _fit_and_score = try_get_attr("galaxy_ml.model_validations", "_fit_and_score") setattr(_search, "_fit_and_score", _fit_and_score) setattr(_validation, "_fit_and_score", _fit_and_score) N_JOBS = int(os.environ.get("GALAXY_SLOTS", 1)) CACHE_DIR = os.path.join(os.getcwd(), "cached") del os NON_SEARCHABLE = ("n_jobs", "pre_dispatch", "memory", "_path", "nthread", "callbacks") ALLOWED_CALLBACKS = ( "EarlyStopping", "TerminateOnNaN", "ReduceLROnPlateau", "CSVLogger", "None", )
def main(inputs, infile_estimator, infile1, infile2, outfile_result, outfile_object=None, outfile_weights=None, groups=None, ref_seq=None, intervals=None, targets=None, fasta_path=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str File path to estimator infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target values outfile_result : str File path to save the results, either cv_results or test result outfile_object : str, optional File path to save searchCV object outfile_weights : str, optional File path to save model weights groups : str File path to dataset containing groups labels ref_seq : str File path to dataset containing genome sequence file intervals : str File path to dataset containing interval file targets : str File path to dataset compressed target bed file fasta_path : str File path to dataset containing fasta file """ warnings.simplefilter('ignore') with open(inputs, 'r') as param_handler: params = json.load(param_handler) params_builder = params['search_schemes']['search_params_builder'] with open(infile_estimator, 'rb') as estimator_handler: estimator = load_model(estimator_handler) estimator_params = estimator.get_params() # store read dataframe object loaded_df = {} input_type = params['input_options']['selected_input'] # tabular input if input_type == 'tabular': header = 'infer' if params['input_options']['header1'] else None column_option = (params['input_options']['column_selector_options_1'] ['selected_column_selector_option']) if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: c = params['input_options']['column_selector_options_1']['col1'] else: c = None df_key = infile1 + repr(header) df = pd.read_csv(infile1, sep='\t', header=header, parse_dates=True) loaded_df[df_key] = df X = read_columns(df, c=c, c_option=column_option).astype(float) # sparse input elif input_type == 'sparse': X = mmread(open(infile1, 'r')) # fasta_file input elif input_type == 'seq_fasta': pyfaidx = get_module('pyfaidx') sequences = pyfaidx.Fasta(fasta_path) n_seqs = len(sequences.keys()) X = np.arange(n_seqs)[:, np.newaxis] for param in estimator_params.keys(): if param.endswith('fasta_path'): estimator.set_params( **{param: fasta_path}) break else: raise ValueError( "The selected estimator doesn't support " "fasta file input! Please consider using " "KerasGBatchClassifier with " "FastaDNABatchGenerator/FastaProteinBatchGenerator " "or having GenomeOneHotEncoder/ProteinOneHotEncoder " "in pipeline!") elif input_type == 'refseq_and_interval': path_params = { 'data_batch_generator__ref_genome_path': ref_seq, 'data_batch_generator__intervals_path': intervals, 'data_batch_generator__target_path': targets } estimator.set_params(**path_params) n_intervals = sum(1 for line in open(intervals)) X = np.arange(n_intervals)[:, np.newaxis] # Get target y header = 'infer' if params['input_options']['header2'] else None column_option = (params['input_options']['column_selector_options_2'] ['selected_column_selector_option2']) if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: c = params['input_options']['column_selector_options_2']['col2'] else: c = None df_key = infile2 + repr(header) if df_key in loaded_df: infile2 = loaded_df[df_key] else: infile2 = pd.read_csv(infile2, sep='\t', header=header, parse_dates=True) loaded_df[df_key] = infile2 y = read_columns( infile2, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) if len(y.shape) == 2 and y.shape[1] == 1: y = y.ravel() if input_type == 'refseq_and_interval': estimator.set_params( data_batch_generator__features=y.ravel().tolist()) y = None # end y optimizer = params['search_schemes']['selected_search_scheme'] optimizer = getattr(model_selection, optimizer) # handle gridsearchcv options options = params['search_schemes']['options'] if groups: header = 'infer' if (options['cv_selector']['groups_selector'] ['header_g']) else None column_option = (options['cv_selector']['groups_selector'] ['column_selector_options_g'] ['selected_column_selector_option_g']) if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: c = (options['cv_selector']['groups_selector'] ['column_selector_options_g']['col_g']) else: c = None df_key = groups + repr(header) if df_key in loaded_df: groups = loaded_df[df_key] groups = read_columns( groups, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) groups = groups.ravel() options['cv_selector']['groups_selector'] = groups splitter, groups = get_cv(options.pop('cv_selector')) options['cv'] = splitter options['n_jobs'] = N_JOBS primary_scoring = options['scoring']['primary_scoring'] options['scoring'] = get_scoring(options['scoring']) if options['error_score']: options['error_score'] = 'raise' else: options['error_score'] = np.NaN if options['refit'] and isinstance(options['scoring'], dict): options['refit'] = primary_scoring if 'pre_dispatch' in options and options['pre_dispatch'] == '': options['pre_dispatch'] = None # del loaded_df del loaded_df # handle memory memory = joblib.Memory(location=CACHE_DIR, verbose=0) # cache iraps_core fits could increase search speed significantly if estimator.__class__.__name__ == 'IRAPSClassifier': estimator.set_params(memory=memory) else: # For iraps buried in pipeline for p, v in estimator_params.items(): if p.endswith('memory'): # for case of `__irapsclassifier__memory` if len(p) > 8 and p[:-8].endswith('irapsclassifier'): # cache iraps_core fits could increase search # speed significantly new_params = {p: memory} estimator.set_params(**new_params) # security reason, we don't want memory being # modified unexpectedly elif v: new_params = {p, None} estimator.set_params(**new_params) # For now, 1 CPU is suggested for iprasclassifier elif p.endswith('n_jobs'): new_params = {p: 1} estimator.set_params(**new_params) # for security reason, types of callbacks are limited elif p.endswith('callbacks'): for cb in v: cb_type = cb['callback_selection']['callback_type'] if cb_type not in ALLOWED_CALLBACKS: raise ValueError( "Prohibited callback type: %s!" % cb_type) param_grid = _eval_search_params(params_builder) searcher = optimizer(estimator, param_grid, **options) # do nested split split_mode = params['outer_split'].pop('split_mode') # nested CV, outer cv using cross_validate if split_mode == 'nested_cv': outer_cv, _ = get_cv(params['outer_split']['cv_selector']) if options['error_score'] == 'raise': rval = cross_validate( searcher, X, y, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=0, error_score=options['error_score']) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: rval = cross_validate( searcher, X, y, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=0, error_score=options['error_score']) except ValueError: pass for warning in w: print(repr(warning.message)) keys = list(rval.keys()) for k in keys: if k.startswith('test'): rval['mean_' + k] = np.mean(rval[k]) rval['std_' + k] = np.std(rval[k]) if k.endswith('time'): rval.pop(k) rval = pd.DataFrame(rval) rval = rval[sorted(rval.columns)] rval.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) else: if split_mode == 'train_test_split': train_test_split = try_get_attr( 'galaxy_ml.model_validations', 'train_test_split') # make sure refit is choosen # this could be True for sklearn models, but not the case for # deep learning models if not options['refit'] and \ not all(hasattr(estimator, attr) for attr in ('config', 'model_type')): warnings.warn("Refit is change to `True` for nested " "validation!") setattr(searcher, 'refit', True) split_options = params['outer_split'] # splits if split_options['shuffle'] == 'stratified': split_options['labels'] = y X, X_test, y, y_test = train_test_split(X, y, **split_options) elif split_options['shuffle'] == 'group': if groups is None: raise ValueError("No group based CV option was " "choosen for group shuffle!") split_options['labels'] = groups if y is None: X, X_test, groups, _ =\ train_test_split(X, groups, **split_options) else: X, X_test, y, y_test, groups, _ =\ train_test_split(X, y, groups, **split_options) else: if split_options['shuffle'] == 'None': split_options['shuffle'] = None X, X_test, y, y_test =\ train_test_split(X, y, **split_options) # end train_test_split # shared by both train_test_split and non-split if options['error_score'] == 'raise': searcher.fit(X, y, groups=groups) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: searcher.fit(X, y, groups=groups) except ValueError: pass for warning in w: print(repr(warning.message)) # no outer split if split_mode == 'no': # save results cv_results = pd.DataFrame(searcher.cv_results_) cv_results = cv_results[sorted(cv_results.columns)] cv_results.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) # train_test_split, output test result using best_estimator_ # or rebuild the trained estimator using weights if applicable. else: scorer_ = searcher.scorer_ if isinstance(scorer_, collections.Mapping): is_multimetric = True else: is_multimetric = False best_estimator_ = getattr(searcher, 'best_estimator_', None) if not best_estimator_: raise ValueError("GridSearchCV object has no " "`best_estimator_` when `refit`=False!") if best_estimator_.__class__.__name__ == 'KerasGBatchClassifier' \ and hasattr(estimator.data_batch_generator, 'target_path'): test_score = best_estimator_.evaluate( X_test, scorer=scorer_, is_multimetric=is_multimetric) else: test_score = _score(best_estimator_, X_test, y_test, scorer_, is_multimetric=is_multimetric) if not is_multimetric: test_score = {primary_scoring: test_score} for key, value in test_score.items(): test_score[key] = [value] result_df = pd.DataFrame(test_score) result_df.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) memory.clear(warn=False) if outfile_object: best_estimator_ = getattr(searcher, 'best_estimator_', None) if not best_estimator_: warnings.warn("GridSearchCV object has no attribute " "'best_estimator_', because either it's " "nested gridsearch or `refit` is False!") return main_est = best_estimator_ if isinstance(best_estimator_, pipeline.Pipeline): main_est = best_estimator_.steps[-1][-1] if hasattr(main_est, 'model_') \ and hasattr(main_est, 'save_weights'): if outfile_weights: main_est.save_weights(outfile_weights) del main_est.model_ del main_est.fit_params del main_est.model_class_ del main_est.validation_data if getattr(main_est, 'data_generator_', None): del main_est.data_generator_ del main_est.data_batch_generator with open(outfile_object, 'wb') as output_handler: pickle.dump(best_estimator_, output_handler, pickle.HIGHEST_PROTOCOL)
def build_keras_model(inputs, outfile, model_json, infile_weights=None, batch_mode=False, outfile_params=None): """ for `keras_model_builder` tool Parameters ---------- inputs : dict loaded galaxy tool parameters from `keras_model_builder` tool. outfile : str Path to galaxy dataset containing the keras_galaxy model output. model_json : str Path to dataset containing keras model JSON. infile_weights : str or None If string, path to dataset containing model weights. batch_mode : bool, default=False Whether to build online batch classifier. outfile_params : str, default=None File path to search parameters output. """ with open(model_json, 'r') as f: json_model = json.load(f) config = json_model['config'] options = {} if json_model['class_name'] == 'Sequential': options['model_type'] = 'sequential' klass = Sequential elif json_model['class_name'] == 'Model': options['model_type'] = 'functional' klass = Model else: raise ValueError("Unknow Keras model class: %s" % json_model['class_name']) # load prefitted model if inputs['mode_selection']['mode_type'] == 'prefitted': estimator = klass.from_config(config) estimator.load_weights(infile_weights) # build train model else: cls_name = inputs['mode_selection']['learning_type'] klass = try_get_attr('galaxy_ml.keras_galaxy_models', cls_name) options['loss'] = (inputs['mode_selection']['compile_params']['loss']) options['optimizer'] =\ (inputs['mode_selection']['compile_params'] ['optimizer_selection']['optimizer_type']).lower() options.update((inputs['mode_selection']['compile_params'] ['optimizer_selection']['optimizer_options'])) train_metrics = ( inputs['mode_selection']['compile_params']['metrics']).split(',') if train_metrics[-1] == 'none': train_metrics = train_metrics[:-1] options['metrics'] = train_metrics options.update(inputs['mode_selection']['fit_params']) options['seed'] = inputs['mode_selection']['random_seed'] if batch_mode: generator = get_batch_generator( inputs['mode_selection']['generator_selection']) options['data_batch_generator'] = generator options['prediction_steps'] = \ inputs['mode_selection']['prediction_steps'] options['class_positive_factor'] = \ inputs['mode_selection']['class_positive_factor'] estimator = klass(config, **options) if outfile_params: hyper_params = get_search_params(estimator) # TODO: remove this after making `verbose` tunable for h_param in hyper_params: if h_param[1].endswith('verbose'): h_param[0] = '@' df = pd.DataFrame(hyper_params, columns=['', 'Parameter', 'Value']) df.to_csv(outfile_params, sep='\t', index=False) print(repr(estimator)) # save model by pickle with open(outfile, 'wb') as f: pickle.dump(estimator, f, pickle.HIGHEST_PROTOCOL)
def main(inputs, infile_estimator, infile1, infile2, outfile_result, outfile_object=None, outfile_weights=None, groups=None, ref_seq=None, intervals=None, targets=None, fasta_path=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : str File path to estimator infile1 : str File path to dataset containing features infile2 : str File path to dataset containing target values outfile_result : str File path to save the results, either cv_results or test result outfile_object : str, optional File path to save searchCV object outfile_weights : str, optional File path to save model weights groups : str File path to dataset containing groups labels ref_seq : str File path to dataset containing genome sequence file intervals : str File path to dataset containing interval file targets : str File path to dataset compressed target bed file fasta_path : str File path to dataset containing fasta file """ warnings.simplefilter('ignore') # store read dataframe object loaded_df = {} with open(inputs, 'r') as param_handler: params = json.load(param_handler) # Override the refit parameter params['search_schemes']['options']['refit'] = True \ if (params['save'] != 'nope' or params['outer_split']['split_mode'] == 'nested_cv') else False with open(infile_estimator, 'rb') as estimator_handler: estimator = load_model(estimator_handler) if estimator.__class__.__name__ == 'KerasGBatchClassifier': _fit_and_score = try_get_attr('galaxy_ml.model_validations', '_fit_and_score') setattr(_search, '_fit_and_score', _fit_and_score) setattr(_validation, '_fit_and_score', _fit_and_score) optimizer = params['search_schemes']['selected_search_scheme'] if optimizer == 'skopt.BayesSearchCV': optimizer = BayesSearchCV else: optimizer = getattr(model_selection, optimizer) # handle gridsearchcv options options = params['search_schemes']['options'] if groups: header = 'infer' if ( options['cv_selector']['groups_selector']['header_g']) else None column_option = ( options['cv_selector']['groups_selector'] ['column_selector_options_g']['selected_column_selector_option_g']) if column_option in [ 'by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name' ]: c = (options['cv_selector']['groups_selector'] ['column_selector_options_g']['col_g']) else: c = None df_key = groups + repr(header) groups = pd.read_csv(groups, sep='\t', header=header, parse_dates=True) loaded_df[df_key] = groups groups = read_columns(groups, c=c, c_option=column_option, sep='\t', header=header, parse_dates=True) groups = groups.ravel() options['cv_selector']['groups_selector'] = groups cv_selector = options.pop('cv_selector') if Version(galaxy_ml_version) < Version('0.8.3'): cv_selector.pop('n_stratification_bins', None) splitter, groups = get_cv(cv_selector) options['cv'] = splitter primary_scoring = options['scoring']['primary_scoring'] options['scoring'] = get_scoring(options['scoring']) # TODO make BayesSearchCV support multiple scoring if optimizer == 'skopt.BayesSearchCV' and \ isinstance(options['scoring'], dict): options['scoring'] = options['scoring'][primary_scoring] warnings.warn("BayesSearchCV doesn't support multiple " "scorings! Primary scoring is used.") if options['error_score']: options['error_score'] = 'raise' else: options['error_score'] = np.NaN if options['refit'] and isinstance(options['scoring'], dict): options['refit'] = primary_scoring if 'pre_dispatch' in options and options['pre_dispatch'] == '': options['pre_dispatch'] = None params_builder = params['search_schemes']['search_params_builder'] param_grid = _eval_search_params(params_builder) estimator = clean_params(estimator) # save the SearchCV object without fit if params['save'] == 'save_no_fit': searcher = optimizer(estimator, param_grid, **options) print(searcher) with open(outfile_object, 'wb') as output_handler: pickle.dump(searcher, output_handler, pickle.HIGHEST_PROTOCOL) return 0 # read inputs and loads new attributes, like paths estimator, X, y = _handle_X_y(estimator, params, infile1, infile2, loaded_df=loaded_df, ref_seq=ref_seq, intervals=intervals, targets=targets, fasta_path=fasta_path) # cache iraps_core fits could increase search speed significantly memory = joblib.Memory(location=CACHE_DIR, verbose=0) estimator = _set_memory(estimator, memory) searcher = optimizer(estimator, param_grid, **options) split_mode = params['outer_split'].pop('split_mode') # Nested CV if split_mode == 'nested_cv': cv_selector = params['outer_split']['cv_selector'] if Version(galaxy_ml_version) < Version('0.8.3'): cv_selector.pop('n_stratification_bins', None) outer_cv, _ = get_cv(cv_selector) # nested CV, outer cv using cross_validate if options['error_score'] == 'raise': rval = cross_validate( searcher, X, y, groups=groups, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=options['verbose'], fit_params={'groups': groups}, return_estimator=(params['save'] == 'save_estimator'), error_score=options['error_score'], return_train_score=True) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: rval = cross_validate( searcher, X, y, groups=groups, scoring=options['scoring'], cv=outer_cv, n_jobs=N_JOBS, verbose=options['verbose'], fit_params={'groups': groups}, return_estimator=(params['save'] == 'save_estimator'), error_score=options['error_score'], return_train_score=True) except ValueError: pass for warning in w: print(repr(warning.message)) fitted_searchers = rval.pop('estimator', []) if fitted_searchers: import os pwd = os.getcwd() save_dir = os.path.join(pwd, 'cv_results_in_folds') try: os.mkdir(save_dir) for idx, obj in enumerate(fitted_searchers): target_name = 'cv_results_' + '_' + 'split%d' % idx target_path = os.path.join(pwd, save_dir, target_name) cv_results_ = getattr(obj, 'cv_results_', None) if not cv_results_: print("%s is not available" % target_name) continue cv_results_ = pd.DataFrame(cv_results_) cv_results_ = cv_results_[sorted(cv_results_.columns)] cv_results_.to_csv(target_path, sep='\t', header=True, index=False) except Exception as e: print(e) finally: del os keys = list(rval.keys()) for k in keys: if k.startswith('test'): rval['mean_' + k] = np.mean(rval[k]) rval['std_' + k] = np.std(rval[k]) if k.endswith('time'): rval.pop(k) rval = pd.DataFrame(rval) rval = rval[sorted(rval.columns)] rval.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) return 0 # deprecate train test split mode """searcher = _do_train_test_split_val( searcher, X, y, params, primary_scoring=primary_scoring, error_score=options['error_score'], groups=groups, outfile=outfile_result)""" # no outer split else: searcher.set_params(n_jobs=N_JOBS) if options['error_score'] == 'raise': searcher.fit(X, y, groups=groups) else: warnings.simplefilter('always', FitFailedWarning) with warnings.catch_warnings(record=True) as w: try: searcher.fit(X, y, groups=groups) except ValueError: pass for warning in w: print(repr(warning.message)) cv_results = pd.DataFrame(searcher.cv_results_) cv_results = cv_results[sorted(cv_results.columns)] cv_results.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) memory.clear(warn=False) # output best estimator, and weights if applicable if outfile_object: best_estimator_ = getattr(searcher, 'best_estimator_', None) if not best_estimator_: warnings.warn("GridSearchCV object has no attribute " "'best_estimator_', because either it's " "nested gridsearch or `refit` is False!") return # clean prams best_estimator_ = clean_params(best_estimator_) main_est = get_main_estimator(best_estimator_) if hasattr(main_est, 'model_') \ and hasattr(main_est, 'save_weights'): if outfile_weights: main_est.save_weights(outfile_weights) del main_est.model_ del main_est.fit_params del main_est.model_class_ main_est.callbacks = [] if getattr(main_est, 'data_generator_', None): del main_est.data_generator_ with open(outfile_object, 'wb') as output_handler: print("Best estimator is saved: %s " % repr(best_estimator_)) pickle.dump(best_estimator_, output_handler, pickle.HIGHEST_PROTOCOL)
def build_keras_model( inputs, outfile, model_json, infile_weights=None, batch_mode=False, outfile_params=None, ): """ for `keras_model_builder` tool Parameters ---------- inputs : dict loaded galaxy tool parameters from `keras_model_builder` tool. outfile : str Path to galaxy dataset containing the keras_galaxy model output. model_json : str Path to dataset containing keras model JSON. infile_weights : str or None If string, path to dataset containing model weights. batch_mode : bool, default=False Whether to build online batch classifier. outfile_params : str, default=None File path to search parameters output. """ with open(model_json, "r") as f: json_model = json.load(f) config = json_model["config"] options = {} if json_model["class_name"] == "Sequential": options["model_type"] = "sequential" klass = Sequential elif json_model["class_name"] == "Model": options["model_type"] = "functional" klass = Model else: raise ValueError("Unknow Keras model class: %s" % json_model["class_name"]) # load prefitted model if inputs["mode_selection"]["mode_type"] == "prefitted": estimator = klass.from_config(config) estimator.load_weights(infile_weights) # build train model else: cls_name = inputs["mode_selection"]["learning_type"] klass = try_get_attr("galaxy_ml.keras_galaxy_models", cls_name) options["loss"] = inputs["mode_selection"]["compile_params"]["loss"] options["optimizer"] = ( inputs["mode_selection"]["compile_params"]["optimizer_selection"] ["optimizer_type"]).lower() options.update((inputs["mode_selection"]["compile_params"] ["optimizer_selection"]["optimizer_options"])) train_metrics = inputs["mode_selection"]["compile_params"]["metrics"] if train_metrics[-1] == "none": train_metrics = train_metrics[:-1] options["metrics"] = train_metrics options.update(inputs["mode_selection"]["fit_params"]) options["seed"] = inputs["mode_selection"]["random_seed"] if batch_mode: generator = get_batch_generator( inputs["mode_selection"]["generator_selection"]) options["data_batch_generator"] = generator options["prediction_steps"] = inputs["mode_selection"][ "prediction_steps"] options["class_positive_factor"] = inputs["mode_selection"][ "class_positive_factor"] estimator = klass(config, **options) if outfile_params: hyper_params = get_search_params(estimator) # TODO: remove this after making `verbose` tunable for h_param in hyper_params: if h_param[1].endswith("verbose"): h_param[0] = "@" df = pd.DataFrame(hyper_params, columns=["", "Parameter", "Value"]) df.to_csv(outfile_params, sep="\t", index=False) print(repr(estimator)) # save model by pickle with open(outfile, "wb") as f: pickle.dump(estimator, f, pickle.HIGHEST_PROTOCOL)
def main(inputs, infile_estimator, outfile_predict, infile_weights=None, infile1=None, fasta_path=None, ref_seq=None, vcf_path=None): """ Parameter --------- inputs : str File path to galaxy tool parameter infile_estimator : strgit File path to trained estimator input outfile_predict : str File path to save the prediction results, tabular infile_weights : str File path to weights input infile1 : str File path to dataset containing features fasta_path : str File path to dataset containing fasta file ref_seq : str File path to dataset containing the reference genome sequence. vcf_path : str File path to dataset containing variants info. """ warnings.filterwarnings('ignore') with open(inputs, 'r') as param_handler: params = json.load(param_handler) # load model with open(infile_estimator, 'rb') as est_handler: estimator = load_model(est_handler) main_est = estimator if isinstance(estimator, Pipeline): main_est = estimator.steps[-1][-1] if hasattr(main_est, 'config') and hasattr(main_est, 'load_weights'): if not infile_weights or infile_weights == 'None': raise ValueError("The selected model skeleton asks for weights, " "but dataset for weights wan not selected!") main_est.load_weights(infile_weights) # handle data input input_type = params['input_options']['selected_input'] # tabular input if input_type == 'tabular': header = 'infer' if params['input_options']['header1'] else None column_option = (params['input_options']['column_selector_options_1'] ['selected_column_selector_option']) if column_option in [ 'by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name' ]: c = params['input_options']['column_selector_options_1']['col1'] else: c = None df = pd.read_csv(infile1, sep='\t', header=header, parse_dates=True) X = read_columns(df, c=c, c_option=column_option).astype(float) if params['method'] == 'predict': preds = estimator.predict(X) else: preds = estimator.predict_proba(X) # sparse input elif input_type == 'sparse': X = mmread(open(infile1, 'r')) if params['method'] == 'predict': preds = estimator.predict(X) else: preds = estimator.predict_proba(X) # fasta input elif input_type == 'seq_fasta': if not hasattr(estimator, 'data_batch_generator'): raise ValueError("To do prediction on sequences in fasta input, " "the estimator must be a `KerasGBatchClassifier`" "equipped with data_batch_generator!") pyfaidx = get_module('pyfaidx') sequences = pyfaidx.Fasta(fasta_path) n_seqs = len(sequences.keys()) X = np.arange(n_seqs)[:, np.newaxis] seq_length = estimator.data_batch_generator.seq_length batch_size = getattr(estimator, 'batch_size', 32) steps = (n_seqs + batch_size - 1) // batch_size seq_type = params['input_options']['seq_type'] klass = try_get_attr('galaxy_ml.preprocessors', seq_type) pred_data_generator = klass(fasta_path, seq_length=seq_length) if params['method'] == 'predict': preds = estimator.predict(X, data_generator=pred_data_generator, steps=steps) else: preds = estimator.predict_proba(X, data_generator=pred_data_generator, steps=steps) # vcf input elif input_type == 'variant_effect': klass = try_get_attr('galaxy_ml.preprocessors', 'GenomicVariantBatchGenerator') options = params['input_options'] options.pop('selected_input') if options['blacklist_regions'] == 'none': options['blacklist_regions'] = None pred_data_generator = klass(ref_genome_path=ref_seq, vcf_path=vcf_path, **options) pred_data_generator.fit() preds = estimator.model_.predict_generator( pred_data_generator.flow(batch_size=32), workers=N_JOBS, use_multiprocessing=True) if preds.min() < 0. or preds.max() > 1.: warnings.warn('Network returning invalid probability values. ' 'The last layer might not normalize predictions ' 'into probabilities ' '(like softmax or sigmoid would).') if params['method'] == 'predict_proba' and preds.shape[1] == 1: # first column is probability of class 0 and second is of class 1 preds = np.hstack([1 - preds, preds]) elif params['method'] == 'predict': if preds.shape[-1] > 1: # if the last activation is `softmax`, the sum of all # probibilities will 1, the classification is considered as # multi-class problem, otherwise, we take it as multi-label. act = getattr(estimator.model_.layers[-1], 'activation', None) if act and act.__name__ == 'softmax': classes = preds.argmax(axis=-1) else: preds = (preds > 0.5).astype('int32') else: classes = (preds > 0.5).astype('int32') preds = estimator.classes_[classes] # end input # output if input_type == 'variant_effect': # TODO: save in batchs rval = pd.DataFrame(preds) meta = pd.DataFrame( pred_data_generator.variants, columns=['chrom', 'pos', 'name', 'ref', 'alt', 'strand']) rval = pd.concat([meta, rval], axis=1) elif len(preds.shape) == 1: rval = pd.DataFrame(preds, columns=['Predicted']) else: rval = pd.DataFrame(preds) rval.to_csv(outfile_predict, sep='\t', header=True, index=False)