def get_dataframe(file_path, plot_selection, header_name, column_name):
header = 'infer' if plot_selection[header_name] else None
column_option = plot_selection[column_name][
"selected_column_selector_option"]
if column_option in [
"by_index_number", "all_but_by_index_number", "by_header_name",
"all_but_by_header_name"
]:
col = plot_selection[column_name]["col1"]
else:
col = None
_, input_df = read_columns(file_path,
c=col,
c_option=column_option,
return_df=True,
sep='\t',
header=header,
parse_dates=True)
return input_df
def main(inputs,
infile_estimator=None,
infile1=None,
infile2=None,
outfile_result=None,
outfile_object=None,
groups=None,
ref_seq=None,
intervals=None,
targets=None,
fasta_path=None,
model_config=None):
"""
Parameter
---------
inputs : str
File path to galaxy tool parameter
infile_estimator : str, default is None
File path to estimator
infile1 : str, default is None
File path to dataset containing features or true labels.
infile2 : str, default is None
File path to dataset containing target values or predicted
probabilities.
outfile_result : str, default is None
File path to save the results, either cv_results or test result
outfile_object : str, default is None
File path to save searchCV object
groups : str, default is None
File path to dataset containing groups labels
ref_seq : str, default is None
File path to dataset containing genome sequence file
intervals : str, default is None
File path to dataset containing interval file
targets : str, default is None
File path to dataset compressed target bed file
fasta_path : str, default is None
File path to dataset containing fasta file
model_config : str, default is None
File path to dataset containing JSON config for neural networks
"""
warnings.simplefilter('ignore')
with open(inputs, 'r') as param_handler:
params = json.load(param_handler)
title = params['plotting_selection']['title'].strip()
plot_type = params['plotting_selection']['plot_type']
if plot_type == 'feature_importances':
with open(infile_estimator, 'rb') as estimator_handler:
estimator = load_model(estimator_handler)
column_option = (
params['plotting_selection']['column_selector_options']
['selected_column_selector_option'])
if column_option in [
'by_index_number', 'all_but_by_index_number', 'by_header_name',
'all_but_by_header_name'
]:
c = (params['plotting_selection']['column_selector_options']
['col1'])
else:
c = None
_, input_df = read_columns(infile1,
c=c,
c_option=column_option,
return_df=True,
sep='\t',
header='infer',
parse_dates=True)
feature_names = input_df.columns.values
if isinstance(estimator, Pipeline):
for st in estimator.steps[:-1]:
if isinstance(st[-1], SelectorMixin):
mask = st[-1].get_support()
feature_names = feature_names[mask]
estimator = estimator.steps[-1][-1]
if hasattr(estimator, 'coef_'):
coefs = estimator.coef_
else:
coefs = getattr(estimator, 'feature_importances_', None)
if coefs is None:
raise RuntimeError('The classifier does not expose '
'"coef_" or "feature_importances_" '
'attributes')
threshold = params['plotting_selection']['threshold']
if threshold is not None:
mask = (coefs > threshold) | (coefs < -threshold)
coefs = coefs[mask]
feature_names = feature_names[mask]
# sort
indices = np.argsort(coefs)[::-1]
trace = go.Bar(x=feature_names[indices], y=coefs[indices])
layout = go.Layout(title=title or "Feature Importances")
fig = go.Figure(data=[trace], layout=layout)
elif plot_type == 'pr_curve':
df1 = pd.read_csv(infile1, sep='\t', header=None)
df2 = pd.read_csv(infile2, sep='\t', header=None)
precision = {}
recall = {}
ap = {}
pos_label = params['plotting_selection']['pos_label'].strip() \
or None
for col in df1.columns:
y_true = df1[col].values
y_score = df2[col].values
precision[col], recall[col], _ = precision_recall_curve(
y_true, y_score, pos_label=pos_label)
ap[col] = average_precision_score(y_true,
y_score,
pos_label=pos_label or 1)
if len(df1.columns) > 1:
precision["micro"], recall["micro"], _ = precision_recall_curve(
df1.values.ravel(), df2.values.ravel(), pos_label=pos_label)
ap['micro'] = average_precision_score(df1.values,
df2.values,
average='micro',
pos_label=pos_label or 1)
data = []
for key in precision.keys():
trace = go.Scatter(x=recall[key],
y=precision[key],
mode='lines',
name='%s (area = %.2f)' % (key, ap[key]) if key
== 'micro' else 'column %s (area = %.2f)' %
(key, ap[key]))
data.append(trace)
layout = go.Layout(title=title or "Precision-Recall curve",
xaxis=dict(title='Recall'),
yaxis=dict(title='Precision'))
fig = go.Figure(data=data, layout=layout)
elif plot_type == 'roc_curve':
df1 = pd.read_csv(infile1, sep='\t', header=None)
df2 = pd.read_csv(infile2, sep='\t', header=None)
fpr = {}
tpr = {}
roc_auc = {}
pos_label = params['plotting_selection']['pos_label'].strip() \
or None
for col in df1.columns:
y_true = df1[col].values
y_score = df2[col].values
fpr[col], tpr[col], _ = roc_curve(y_true,
y_score,
pos_label=pos_label)
roc_auc[col] = auc(fpr[col], tpr[col])
if len(df1.columns) > 1:
fpr["micro"], tpr["micro"], _ = roc_curve(df1.values.ravel(),
df2.values.ravel(),
pos_label=pos_label)
roc_auc['micro'] = auc(fpr["micro"], tpr["micro"])
data = []
for key in fpr.keys():
trace = go.Scatter(x=fpr[key],
y=tpr[key],
mode='lines',
name='%s (area = %.2f)' % (key, roc_auc[key]) if
key == 'micro' else 'column %s (area = %.2f)' %
(key, roc_auc[key]))
data.append(trace)
trace = go.Scatter(x=[0, 1],
y=[0, 1],
mode='lines',
line=dict(color='black', dash='dash'),
showlegend=False)
data.append(trace)
layout = go.Layout(title=title
or "Receiver operating characteristic curve",
xaxis=dict(title='False Positive Rate'),
yaxis=dict(title='True Positive Rate'))
fig = go.Figure(data=data, layout=layout)
elif plot_type == 'rfecv_gridscores':
input_df = pd.read_csv(infile1, sep='\t', header='infer')
scores = input_df.iloc[:, 0]
steps = params['plotting_selection']['steps'].strip()
steps = safe_eval(steps)
data = go.Scatter(x=list(range(len(scores))),
y=scores,
text=[str(_) for _ in steps] if steps else None,
mode='lines')
layout = go.Layout(xaxis=dict(title="Number of features selected"),
yaxis=dict(title="Cross validation score"),
title=title or None)
fig = go.Figure(data=[data], layout=layout)
elif plot_type == 'learning_curve':
input_df = pd.read_csv(infile1, sep='\t', header='infer')
plot_std_err = params['plotting_selection']['plot_std_err']
data1 = go.Scatter(
x=input_df['train_sizes_abs'],
y=input_df['mean_train_scores'],
error_y=dict(
array=input_df['std_train_scores']) if plot_std_err else None,
mode='lines',
name="Train Scores",
)
data2 = go.Scatter(
x=input_df['train_sizes_abs'],
y=input_df['mean_test_scores'],
error_y=dict(
array=input_df['std_test_scores']) if plot_std_err else None,
mode='lines',
name="Test Scores",
)
layout = dict(xaxis=dict(title='No. of samples'),
yaxis=dict(title='Performance Score'),
title=title or 'Learning Curve')
fig = go.Figure(data=[data1, data2], layout=layout)
elif plot_type == 'keras_plot_model':
with open(model_config, 'r') as f:
model_str = f.read()
model = model_from_json(model_str)
plot_model(model, to_file="output.png")
__import__('os').rename('output.png', 'output')
return 0
plotly.offline.plot(fig, filename="output.html", auto_open=False)
# to be discovered by `from_work_dir`
__import__('os').rename('output.html', 'output')
def main(inputs, infile_estimator, infile1, infile2,
outfile_result, outfile_object=None,
outfile_weights=None, groups=None,
ref_seq=None, intervals=None, targets=None,
fasta_path=None):
"""
Parameter
---------
inputs : str
File path to galaxy tool parameter
infile_estimator : str
File path to estimator
infile1 : str
File path to dataset containing features
infile2 : str
File path to dataset containing target values
outfile_result : str
File path to save the results, either cv_results or test result
outfile_object : str, optional
File path to save searchCV object
outfile_weights : str, optional
File path to save deep learning model weights
groups : str
File path to dataset containing groups labels
ref_seq : str
File path to dataset containing genome sequence file
intervals : str
File path to dataset containing interval file
targets : str
File path to dataset compressed target bed file
fasta_path : str
File path to dataset containing fasta file
"""
warnings.simplefilter('ignore')
with open(inputs, 'r') as param_handler:
params = json.load(param_handler)
# load estimator
with open(infile_estimator, 'rb') as estimator_handler:
estimator = load_model(estimator_handler)
# swap hyperparameter
swapping = params['experiment_schemes']['hyperparams_swapping']
swap_params = _eval_swap_params(swapping)
estimator.set_params(**swap_params)
estimator_params = estimator.get_params()
# store read dataframe object
loaded_df = {}
input_type = params['input_options']['selected_input']
# tabular input
if input_type == 'tabular':
header = 'infer' if params['input_options']['header1'] else None
column_option = (params['input_options']['column_selector_options_1']
['selected_column_selector_option'])
if column_option in ['by_index_number', 'all_but_by_index_number',
'by_header_name', 'all_but_by_header_name']:
c = params['input_options']['column_selector_options_1']['col1']
else:
c = None
df_key = infile1 + repr(header)
df = pd.read_csv(infile1, sep='\t', header=header,
parse_dates=True)
loaded_df[df_key] = df
X = read_columns(df, c=c, c_option=column_option).astype(float)
# sparse input
elif input_type == 'sparse':
X = mmread(open(infile1, 'r'))
# fasta_file input
elif input_type == 'seq_fasta':
pyfaidx = get_module('pyfaidx')
sequences = pyfaidx.Fasta(fasta_path)
n_seqs = len(sequences.keys())
X = np.arange(n_seqs)[:, np.newaxis]
for param in estimator_params.keys():
if param.endswith('fasta_path'):
estimator.set_params(
**{param: fasta_path})
break
else:
raise ValueError(
"The selected estimator doesn't support "
"fasta file input! Please consider using "
"KerasGBatchClassifier with "
"FastaDNABatchGenerator/FastaProteinBatchGenerator "
"or having GenomeOneHotEncoder/ProteinOneHotEncoder "
"in pipeline!")
elif input_type == 'refseq_and_interval':
path_params = {
'data_batch_generator__ref_genome_path': ref_seq,
'data_batch_generator__intervals_path': intervals,
'data_batch_generator__target_path': targets
}
estimator.set_params(**path_params)
n_intervals = sum(1 for line in open(intervals))
X = np.arange(n_intervals)[:, np.newaxis]
# Get target y
header = 'infer' if params['input_options']['header2'] else None
column_option = (params['input_options']['column_selector_options_2']
['selected_column_selector_option2'])
if column_option in ['by_index_number', 'all_but_by_index_number',
'by_header_name', 'all_but_by_header_name']:
c = params['input_options']['column_selector_options_2']['col2']
else:
c = None
df_key = infile2 + repr(header)
if df_key in loaded_df:
infile2 = loaded_df[df_key]
else:
infile2 = pd.read_csv(infile2, sep='\t',
header=header, parse_dates=True)
loaded_df[df_key] = infile2
y = read_columns(
infile2,
c=c,
c_option=column_option,
sep='\t',
header=header,
parse_dates=True)
if len(y.shape) == 2 and y.shape[1] == 1:
y = y.ravel()
if input_type == 'refseq_and_interval':
estimator.set_params(
data_batch_generator__features=y.ravel().tolist())
y = None
# end y
# load groups
if groups:
groups_selector = (params['experiment_schemes']['test_split']
['split_algos']).pop('groups_selector')
header = 'infer' if groups_selector['header_g'] else None
column_option = \
(groups_selector['column_selector_options_g']
['selected_column_selector_option_g'])
if column_option in ['by_index_number', 'all_but_by_index_number',
'by_header_name', 'all_but_by_header_name']:
c = groups_selector['column_selector_options_g']['col_g']
else:
c = None
df_key = groups + repr(header)
if df_key in loaded_df:
groups = loaded_df[df_key]
groups = read_columns(
groups,
c=c,
c_option=column_option,
sep='\t',
header=header,
parse_dates=True)
groups = groups.ravel()
# del loaded_df
del loaded_df
# handle memory
memory = joblib.Memory(location=CACHE_DIR, verbose=0)
# cache iraps_core fits could increase search speed significantly
if estimator.__class__.__name__ == 'IRAPSClassifier':
estimator.set_params(memory=memory)
else:
# For iraps buried in pipeline
new_params = {}
for p, v in estimator_params.items():
if p.endswith('memory'):
# for case of `__irapsclassifier__memory`
if len(p) > 8 and p[:-8].endswith('irapsclassifier'):
# cache iraps_core fits could increase search
# speed significantly
new_params[p] = memory
# security reason, we don't want memory being
# modified unexpectedly
elif v:
new_params[p] = None
# handle n_jobs
elif p.endswith('n_jobs'):
# For now, 1 CPU is suggested for iprasclassifier
if len(p) > 8 and p[:-8].endswith('irapsclassifier'):
new_params[p] = 1
else:
new_params[p] = N_JOBS
# for security reason, types of callback are limited
elif p.endswith('callbacks'):
for cb in v:
cb_type = cb['callback_selection']['callback_type']
if cb_type not in ALLOWED_CALLBACKS:
raise ValueError(
"Prohibited callback type: %s!" % cb_type)
estimator.set_params(**new_params)
# handle scorer, convert to scorer dict
scoring = params['experiment_schemes']['metrics']['scoring']
scorer = get_scoring(scoring)
scorer, _ = _check_multimetric_scoring(estimator, scoring=scorer)
# handle test (first) split
test_split_options = (params['experiment_schemes']
['test_split']['split_algos'])
if test_split_options['shuffle'] == 'group':
test_split_options['labels'] = groups
if test_split_options['shuffle'] == 'stratified':
if y is not None:
test_split_options['labels'] = y
else:
raise ValueError("Stratified shuffle split is not "
"applicable on empty target values!")
X_train, X_test, y_train, y_test, groups_train, groups_test = \
train_test_split_none(X, y, groups, **test_split_options)
exp_scheme = params['experiment_schemes']['selected_exp_scheme']
# handle validation (second) split
if exp_scheme == 'train_val_test':
val_split_options = (params['experiment_schemes']
['val_split']['split_algos'])
if val_split_options['shuffle'] == 'group':
val_split_options['labels'] = groups_train
if val_split_options['shuffle'] == 'stratified':
if y_train is not None:
val_split_options['labels'] = y_train
else:
raise ValueError("Stratified shuffle split is not "
"applicable on empty target values!")
X_train, X_val, y_train, y_val, groups_train, groups_val = \
train_test_split_none(X_train, y_train, groups_train,
**val_split_options)
# train and eval
if hasattr(estimator, 'validation_data'):
if exp_scheme == 'train_val_test':
estimator.fit(X_train, y_train,
validation_data=(X_val, y_val))
else:
estimator.fit(X_train, y_train,
validation_data=(X_test, y_test))
else:
estimator.fit(X_train, y_train)
if hasattr(estimator, 'evaluate'):
scores = estimator.evaluate(X_test, y_test=y_test,
scorer=scorer,
is_multimetric=True)
else:
scores = _score(estimator, X_test, y_test, scorer,
is_multimetric=True)
# handle output
for name, score in scores.items():
scores[name] = [score]
df = pd.DataFrame(scores)
df = df[sorted(df.columns)]
df.to_csv(path_or_buf=outfile_result, sep='\t',
header=True, index=False)
memory.clear(warn=False)
if outfile_object:
main_est = estimator
if isinstance(estimator, pipeline.Pipeline):
main_est = estimator.steps[-1][-1]
if hasattr(main_est, 'model_') \
and hasattr(main_est, 'save_weights'):
if outfile_weights:
main_est.save_weights(outfile_weights)
del main_est.model_
del main_est.fit_params
del main_est.model_class_
del main_est.validation_data
if getattr(main_est, 'data_generator_', None):
del main_est.data_generator_
with open(outfile_object, 'wb') as output_handler:
pickle.dump(estimator, output_handler,
pickle.HIGHEST_PROTOCOL)
def main(inputs,
infile_estimator,
infile1,
infile2,
outfile_result,
outfile_object=None,
outfile_weights=None,
groups=None,
ref_seq=None,
intervals=None,
targets=None,
fasta_path=None):
"""
Parameter
---------
inputs : str
File path to galaxy tool parameter
infile_estimator : str
File path to estimator
infile1 : str
File path to dataset containing features
infile2 : str
File path to dataset containing target values
outfile_result : str
File path to save the results, either cv_results or test result
outfile_object : str, optional
File path to save searchCV object
outfile_weights : str, optional
File path to save model weights
groups : str
File path to dataset containing groups labels
ref_seq : str
File path to dataset containing genome sequence file
intervals : str
File path to dataset containing interval file
targets : str
File path to dataset compressed target bed file
fasta_path : str
File path to dataset containing fasta file
"""
warnings.simplefilter('ignore')
# store read dataframe object
loaded_df = {}
with open(inputs, 'r') as param_handler:
params = json.load(param_handler)
# Override the refit parameter
params['search_schemes']['options']['refit'] = True \
if params['save'] != 'nope' else False
with open(infile_estimator, 'rb') as estimator_handler:
estimator = load_model(estimator_handler)
optimizer = params['search_schemes']['selected_search_scheme']
optimizer = getattr(model_selection, optimizer)
# handle gridsearchcv options
options = params['search_schemes']['options']
if groups:
header = 'infer' if (
options['cv_selector']['groups_selector']['header_g']) else None
column_option = (
options['cv_selector']['groups_selector']
['column_selector_options_g']['selected_column_selector_option_g'])
if column_option in [
'by_index_number', 'all_but_by_index_number', 'by_header_name',
'all_but_by_header_name'
]:
c = (options['cv_selector']['groups_selector']
['column_selector_options_g']['col_g'])
else:
c = None
df_key = groups + repr(header)
groups = pd.read_csv(groups, sep='\t', header=header, parse_dates=True)
loaded_df[df_key] = groups
groups = read_columns(groups,
c=c,
c_option=column_option,
sep='\t',
header=header,
parse_dates=True)
groups = groups.ravel()
options['cv_selector']['groups_selector'] = groups
splitter, groups = get_cv(options.pop('cv_selector'))
options['cv'] = splitter
primary_scoring = options['scoring']['primary_scoring']
options['scoring'] = get_scoring(options['scoring'])
if options['error_score']:
options['error_score'] = 'raise'
else:
options['error_score'] = np.NaN
if options['refit'] and isinstance(options['scoring'], dict):
options['refit'] = primary_scoring
if 'pre_dispatch' in options and options['pre_dispatch'] == '':
options['pre_dispatch'] = None
params_builder = params['search_schemes']['search_params_builder']
param_grid = _eval_search_params(params_builder)
estimator = clean_params(estimator)
# save the SearchCV object without fit
if params['save'] == 'save_no_fit':
searcher = optimizer(estimator, param_grid, **options)
print(searcher)
with open(outfile_object, 'wb') as output_handler:
pickle.dump(searcher, output_handler, pickle.HIGHEST_PROTOCOL)
return 0
# read inputs and loads new attributes, like paths
estimator, X, y = _handle_X_y(estimator,
params,
infile1,
infile2,
loaded_df=loaded_df,
ref_seq=ref_seq,
intervals=intervals,
targets=targets,
fasta_path=fasta_path)
# cache iraps_core fits could increase search speed significantly
memory = joblib.Memory(location=CACHE_DIR, verbose=0)
main_est = get_main_estimator(estimator)
if main_est.__class__.__name__ == 'IRAPSClassifier':
main_est.set_params(memory=memory)
searcher = optimizer(estimator, param_grid, **options)
split_mode = params['outer_split'].pop('split_mode')
if split_mode == 'nested_cv':
# make sure refit is choosen
# this could be True for sklearn models, but not the case for
# deep learning models
if not options['refit'] and \
not all(hasattr(estimator, attr)
for attr in ('config', 'model_type')):
warnings.warn("Refit is change to `True` for nested validation!")
setattr(searcher, 'refit', True)
outer_cv, _ = get_cv(params['outer_split']['cv_selector'])
# nested CV, outer cv using cross_validate
if options['error_score'] == 'raise':
rval = cross_validate(
searcher,
X,
y,
scoring=options['scoring'],
cv=outer_cv,
n_jobs=N_JOBS,
verbose=options['verbose'],
return_estimator=(params['save'] == 'save_estimator'),
error_score=options['error_score'],
return_train_score=True)
else:
warnings.simplefilter('always', FitFailedWarning)
with warnings.catch_warnings(record=True) as w:
try:
rval = cross_validate(
searcher,
X,
y,
scoring=options['scoring'],
cv=outer_cv,
n_jobs=N_JOBS,
verbose=options['verbose'],
return_estimator=(params['save'] == 'save_estimator'),
error_score=options['error_score'],
return_train_score=True)
except ValueError:
pass
for warning in w:
print(repr(warning.message))
fitted_searchers = rval.pop('estimator', [])
if fitted_searchers:
import os
pwd = os.getcwd()
save_dir = os.path.join(pwd, 'cv_results_in_folds')
try:
os.mkdir(save_dir)
for idx, obj in enumerate(fitted_searchers):
target_name = 'cv_results_' + '_' + 'split%d' % idx
target_path = os.path.join(pwd, save_dir, target_name)
cv_results_ = getattr(obj, 'cv_results_', None)
if not cv_results_:
print("%s is not available" % target_name)
continue
cv_results_ = pd.DataFrame(cv_results_)
cv_results_ = cv_results_[sorted(cv_results_.columns)]
cv_results_.to_csv(target_path,
sep='\t',
header=True,
index=False)
except Exception as e:
print(e)
finally:
del os
keys = list(rval.keys())
for k in keys:
if k.startswith('test'):
rval['mean_' + k] = np.mean(rval[k])
rval['std_' + k] = np.std(rval[k])
if k.endswith('time'):
rval.pop(k)
rval = pd.DataFrame(rval)
rval = rval[sorted(rval.columns)]
rval.to_csv(path_or_buf=outfile_result,
sep='\t',
header=True,
index=False)
return 0
# deprecate train test split mode
"""searcher = _do_train_test_split_val(
searcher, X, y, params,
primary_scoring=primary_scoring,
error_score=options['error_score'],
groups=groups,
outfile=outfile_result)"""
# no outer split
else:
searcher.set_params(n_jobs=N_JOBS)
if options['error_score'] == 'raise':
searcher.fit(X, y, groups=groups)
else:
warnings.simplefilter('always', FitFailedWarning)
with warnings.catch_warnings(record=True) as w:
try:
searcher.fit(X, y, groups=groups)
except ValueError:
pass
for warning in w:
print(repr(warning.message))
cv_results = pd.DataFrame(searcher.cv_results_)
cv_results = cv_results[sorted(cv_results.columns)]
cv_results.to_csv(path_or_buf=outfile_result,
sep='\t',
header=True,
index=False)
memory.clear(warn=False)
# output best estimator, and weights if applicable
if outfile_object:
best_estimator_ = getattr(searcher, 'best_estimator_', None)
if not best_estimator_:
warnings.warn("GridSearchCV object has no attribute "
"'best_estimator_', because either it's "
"nested gridsearch or `refit` is False!")
return
# clean prams
best_estimator_ = clean_params(best_estimator_)
main_est = get_main_estimator(best_estimator_)
if hasattr(main_est, 'model_') \
and hasattr(main_est, 'save_weights'):
if outfile_weights:
main_est.save_weights(outfile_weights)
del main_est.model_
del main_est.fit_params
del main_est.model_class_
del main_est.validation_data
if getattr(main_est, 'data_generator_', None):
del main_est.data_generator_
with open(outfile_object, 'wb') as output_handler:
print("Best estimator is saved: %s " % repr(best_estimator_))
pickle.dump(best_estimator_, output_handler,
pickle.HIGHEST_PROTOCOL)
def _handle_X_y(estimator,
params,
infile1,
infile2,
loaded_df={},
ref_seq=None,
intervals=None,
targets=None,
fasta_path=None):
"""read inputs
Params
-------
estimator : estimator object
params : dict
Galaxy tool parameter inputs
infile1 : str
File path to dataset containing features
infile2 : str
File path to dataset containing target values
loaded_df : dict
Contains loaded DataFrame objects with file path as keys
ref_seq : str
File path to dataset containing genome sequence file
interval : str
File path to dataset containing interval file
targets : str
File path to dataset compressed target bed file
fasta_path : str
File path to dataset containing fasta file
Returns
-------
estimator : estimator object after setting new attributes
X : numpy array
y : numpy array
"""
estimator_params = estimator.get_params()
input_type = params['input_options']['selected_input']
# tabular input
if input_type == 'tabular':
header = 'infer' if params['input_options']['header1'] else None
column_option = (params['input_options']['column_selector_options_1']
['selected_column_selector_option'])
if column_option in [
'by_index_number', 'all_but_by_index_number', 'by_header_name',
'all_but_by_header_name'
]:
c = params['input_options']['column_selector_options_1']['col1']
else:
c = None
df_key = infile1 + repr(header)
if df_key in loaded_df:
infile1 = loaded_df[df_key]
df = pd.read_csv(infile1, sep='\t', header=header, parse_dates=True)
loaded_df[df_key] = df
X = read_columns(df, c=c, c_option=column_option).astype(float)
# sparse input
elif input_type == 'sparse':
X = mmread(open(infile1, 'r'))
# fasta_file input
elif input_type == 'seq_fasta':
pyfaidx = get_module('pyfaidx')
sequences = pyfaidx.Fasta(fasta_path)
n_seqs = len(sequences.keys())
X = np.arange(n_seqs)[:, np.newaxis]
for param in estimator_params.keys():
if param.endswith('fasta_path'):
estimator.set_params(**{param: fasta_path})
break
else:
raise ValueError(
"The selected estimator doesn't support "
"fasta file input! Please consider using "
"KerasGBatchClassifier with "
"FastaDNABatchGenerator/FastaProteinBatchGenerator "
"or having GenomeOneHotEncoder/ProteinOneHotEncoder "
"in pipeline!")
elif input_type == 'refseq_and_interval':
path_params = {
'data_batch_generator__ref_genome_path': ref_seq,
'data_batch_generator__intervals_path': intervals,
'data_batch_generator__target_path': targets
}
estimator.set_params(**path_params)
n_intervals = sum(1 for line in open(intervals))
X = np.arange(n_intervals)[:, np.newaxis]
# Get target y
header = 'infer' if params['input_options']['header2'] else None
column_option = (params['input_options']['column_selector_options_2']
['selected_column_selector_option2'])
if column_option in [
'by_index_number', 'all_but_by_index_number', 'by_header_name',
'all_but_by_header_name'
]:
c = params['input_options']['column_selector_options_2']['col2']
else:
c = None
df_key = infile2 + repr(header)
if df_key in loaded_df:
infile2 = loaded_df[df_key]
else:
infile2 = pd.read_csv(infile2,
sep='\t',
header=header,
parse_dates=True)
loaded_df[df_key] = infile2
y = read_columns(infile2,
c=c,
c_option=column_option,
sep='\t',
header=header,
parse_dates=True)
if len(y.shape) == 2 and y.shape[1] == 1:
y = y.ravel()
if input_type == 'refseq_and_interval':
estimator.set_params(data_batch_generator__features=y.ravel().tolist())
y = None
# end y
return estimator, X, y
def main(
inputs,
infile_estimator,
outfile_predict,
infile_weights=None,
infile1=None,
fasta_path=None,
ref_seq=None,
vcf_path=None,
):
"""
Parameter
---------
inputs : str
File path to galaxy tool parameter
infile_estimator : strgit
File path to trained estimator input
outfile_predict : str
File path to save the prediction results, tabular
infile_weights : str
File path to weights input
infile1 : str
File path to dataset containing features
fasta_path : str
File path to dataset containing fasta file
ref_seq : str
File path to dataset containing the reference genome sequence.
vcf_path : str
File path to dataset containing variants info.
"""
warnings.filterwarnings("ignore")
with open(inputs, "r") as param_handler:
params = json.load(param_handler)
# load model
with open(infile_estimator, "rb") as est_handler:
estimator = load_model(est_handler)
main_est = estimator
if isinstance(estimator, Pipeline):
main_est = estimator.steps[-1][-1]
if hasattr(main_est, "config") and hasattr(main_est, "load_weights"):
if not infile_weights or infile_weights == "None":
raise ValueError("The selected model skeleton asks for weights, "
"but dataset for weights wan not selected!")
main_est.load_weights(infile_weights)
# handle data input
input_type = params["input_options"]["selected_input"]
# tabular input
if input_type == "tabular":
header = "infer" if params["input_options"]["header1"] else None
column_option = params["input_options"]["column_selector_options_1"][
"selected_column_selector_option"]
if column_option in [
"by_index_number",
"all_but_by_index_number",
"by_header_name",
"all_but_by_header_name",
]:
c = params["input_options"]["column_selector_options_1"]["col1"]
else:
c = None
df = pd.read_csv(infile1, sep="\t", header=header, parse_dates=True)
X = read_columns(df, c=c, c_option=column_option).astype(float)
if params["method"] == "predict":
preds = estimator.predict(X)
else:
preds = estimator.predict_proba(X)
# sparse input
elif input_type == "sparse":
X = mmread(open(infile1, "r"))
if params["method"] == "predict":
preds = estimator.predict(X)
else:
preds = estimator.predict_proba(X)
# fasta input
elif input_type == "seq_fasta":
if not hasattr(estimator, "data_batch_generator"):
raise ValueError("To do prediction on sequences in fasta input, "
"the estimator must be a `KerasGBatchClassifier`"
"equipped with data_batch_generator!")
pyfaidx = get_module("pyfaidx")
sequences = pyfaidx.Fasta(fasta_path)
n_seqs = len(sequences.keys())
X = np.arange(n_seqs)[:, np.newaxis]
seq_length = estimator.data_batch_generator.seq_length
batch_size = getattr(estimator, "batch_size", 32)
steps = (n_seqs + batch_size - 1) // batch_size
seq_type = params["input_options"]["seq_type"]
klass = try_get_attr("galaxy_ml.preprocessors", seq_type)
pred_data_generator = klass(fasta_path, seq_length=seq_length)
if params["method"] == "predict":
preds = estimator.predict(X,
data_generator=pred_data_generator,
steps=steps)
else:
preds = estimator.predict_proba(X,
data_generator=pred_data_generator,
steps=steps)
# vcf input
elif input_type == "variant_effect":
klass = try_get_attr("galaxy_ml.preprocessors",
"GenomicVariantBatchGenerator")
options = params["input_options"]
options.pop("selected_input")
if options["blacklist_regions"] == "none":
options["blacklist_regions"] = None
pred_data_generator = klass(ref_genome_path=ref_seq,
vcf_path=vcf_path,
**options)
pred_data_generator.set_processing_attrs()
variants = pred_data_generator.variants
# predict 1600 sample at once then write to file
gen_flow = pred_data_generator.flow(batch_size=1600)
file_writer = open(outfile_predict, "w")
header_row = "\t".join(
["chrom", "pos", "name", "ref", "alt", "strand"])
file_writer.write(header_row)
header_done = False
steps_done = 0
# TODO: multiple threading
try:
while steps_done < len(gen_flow):
index_array = next(gen_flow.index_generator)
batch_X = gen_flow._get_batches_of_transformed_samples(
index_array)
if params["method"] == "predict":
batch_preds = estimator.predict(
batch_X,
# The presence of `pred_data_generator` below is to
# override model carrying data_generator if there
# is any.
data_generator=pred_data_generator,
)
else:
batch_preds = estimator.predict_proba(
batch_X,
# The presence of `pred_data_generator` below is to
# override model carrying data_generator if there
# is any.
data_generator=pred_data_generator,
)
if batch_preds.ndim == 1:
batch_preds = batch_preds[:, np.newaxis]
batch_meta = variants[index_array]
batch_out = np.column_stack([batch_meta, batch_preds])
if not header_done:
heads = np.arange(batch_preds.shape[-1]).astype(str)
heads_str = "\t".join(heads)
file_writer.write("\t%s\n" % heads_str)
header_done = True
for row in batch_out:
row_str = "\t".join(row)
file_writer.write("%s\n" % row_str)
steps_done += 1
finally:
file_writer.close()
# TODO: make api `pred_data_generator.close()`
pred_data_generator.close()
return 0
# end input
# output
if len(preds.shape) == 1:
rval = pd.DataFrame(preds, columns=["Predicted"])
else:
rval = pd.DataFrame(preds)
rval.to_csv(outfile_predict, sep="\t", header=True, index=False)
def main(inputs,
infile_estimator,
infile1,
infile2,
outfile_result,
outfile_object=None,
groups=None,
ref_seq=None,
intervals=None,
targets=None,
fasta_path=None):
"""
Parameter
---------
inputs : str
File path to galaxy tool parameter.
infile_estimator : str
File path to estimator.
infile1 : str
File path to dataset containing features
infile2 : str
File path to dataset containing target values
outfile_result : str
File path to save the results, either cv_results or test result
outfile_object : str, optional
File path to save searchCV object
groups : str
File path to dataset containing groups labels
ref_seq : str
File path to dataset containing genome sequence file
intervals : str
File path to dataset containing interval file
targets : str
File path to dataset compressed target bed file
fasta_path : str
File path to dataset containing fasta file
"""
warnings.simplefilter('ignore')
# store read dataframe object
loaded_df = {}
with open(inputs, 'r') as param_handler:
params = json.load(param_handler)
# Override the refit parameter
params['search_schemes']['options']['refit'] = True \
if (params['save'] != 'nope' or
params['outer_split']['split_mode'] == 'nested_cv') else False
estimator = load_model_from_h5(infile_estimator)
estimator = clean_params(estimator)
if estimator.__class__.__name__ == 'KerasGBatchClassifier':
_fit_and_score = try_get_attr('galaxy_ml.model_validations',
'_fit_and_score')
setattr(_search, '_fit_and_score', _fit_and_score)
setattr(_validation, '_fit_and_score', _fit_and_score)
optimizer = params['search_schemes']['selected_search_scheme']
if optimizer == 'skopt.BayesSearchCV':
optimizer = BayesSearchCV
else:
optimizer = getattr(model_selection, optimizer)
# handle gridsearchcv options
options = params['search_schemes']['options']
if groups:
header = 'infer' if (
options['cv_selector']['groups_selector']['header_g']) else None
column_option = (
options['cv_selector']['groups_selector']
['column_selector_options_g']['selected_column_selector_option_g'])
if column_option in [
'by_index_number', 'all_but_by_index_number', 'by_header_name',
'all_but_by_header_name'
]:
c = (options['cv_selector']['groups_selector']
['column_selector_options_g']['col_g'])
else:
c = None
df_key = groups + repr(header)
groups = pd.read_csv(groups, sep='\t', header=header, parse_dates=True)
loaded_df[df_key] = groups
groups = read_columns(groups,
c=c,
c_option=column_option,
sep='\t',
header=header,
parse_dates=True)
groups = groups.ravel()
options['cv_selector']['groups_selector'] = groups
cv_selector = options.pop('cv_selector')
if Version(galaxy_ml_version) < Version('0.8.3'):
cv_selector.pop('n_stratification_bins', None)
splitter, groups = get_cv(cv_selector)
options['cv'] = splitter
primary_scoring = options['scoring']['primary_scoring']
options['scoring'] = get_scoring(options['scoring'])
# TODO make BayesSearchCV support multiple scoring
if optimizer == 'skopt.BayesSearchCV' and \
isinstance(options['scoring'], dict):
options['scoring'] = options['scoring'][primary_scoring]
warnings.warn("BayesSearchCV doesn't support multiple "
"scorings! Primary scoring is used.")
if options['error_score']:
options['error_score'] = 'raise'
else:
options['error_score'] = np.NaN
if options['refit'] and isinstance(options['scoring'], dict):
options['refit'] = primary_scoring
if 'pre_dispatch' in options and options['pre_dispatch'] == '':
options['pre_dispatch'] = None
params_builder = params['search_schemes']['search_params_builder']
param_grid = _eval_search_params(params_builder)
# save the SearchCV object without fit
if params['save'] == 'save_no_fit':
searcher = optimizer(estimator, param_grid, **options)
dump_model_to_h5(searcher, outfile_object)
return 0
# read inputs and loads new attributes, like paths
estimator, X, y = _handle_X_y(estimator,
params,
infile1,
infile2,
loaded_df=loaded_df,
ref_seq=ref_seq,
intervals=intervals,
targets=targets,
fasta_path=fasta_path)
# cache iraps_core fits could increase search speed significantly
memory = joblib.Memory(location=CACHE_DIR, verbose=0)
estimator = _set_memory(estimator, memory)
searcher = optimizer(estimator, param_grid, **options)
split_mode = params['outer_split'].pop('split_mode')
# Nested CV
if split_mode == 'nested_cv':
cv_selector = params['outer_split']['cv_selector']
if Version(galaxy_ml_version) < Version('0.8.3'):
cv_selector.pop('n_stratification_bins', None)
outer_cv, _ = get_cv(cv_selector)
# nested CV, outer cv using cross_validate
if options['error_score'] == 'raise':
rval = cross_validate(
searcher,
X,
y,
groups=groups,
scoring=options['scoring'],
cv=outer_cv,
n_jobs=N_JOBS,
verbose=options['verbose'],
fit_params={'groups': groups},
return_estimator=(params['save'] == 'save_estimator'),
error_score=options['error_score'],
return_train_score=True)
else:
warnings.simplefilter('always', FitFailedWarning)
with warnings.catch_warnings(record=True) as w:
try:
rval = cross_validate(
searcher,
X,
y,
groups=groups,
scoring=options['scoring'],
cv=outer_cv,
n_jobs=N_JOBS,
verbose=options['verbose'],
fit_params={'groups': groups},
return_estimator=(params['save'] == 'save_estimator'),
error_score=options['error_score'],
return_train_score=True)
except ValueError:
pass
for warning in w:
print(repr(warning.message))
fitted_searchers = rval.pop('estimator', [])
if fitted_searchers:
import os
pwd = os.getcwd()
save_dir = os.path.join(pwd, 'cv_results_in_folds')
try:
os.mkdir(save_dir)
for idx, obj in enumerate(fitted_searchers):
target_name = 'cv_results_' + '_' + 'split%d' % idx
target_path = os.path.join(pwd, save_dir, target_name)
cv_results_ = getattr(obj, 'cv_results_', None)
if not cv_results_:
print("%s is not available" % target_name)
continue
cv_results_ = pd.DataFrame(cv_results_)
cv_results_ = cv_results_[sorted(cv_results_.columns)]
cv_results_.to_csv(target_path,
sep='\t',
header=True,
index=False)
except Exception as e:
print(e)
finally:
del os
keys = list(rval.keys())
for k in keys:
if k.startswith('test'):
rval['mean_' + k] = np.mean(rval[k])
rval['std_' + k] = np.std(rval[k])
if k.endswith('time'):
rval.pop(k)
rval = pd.DataFrame(rval)
rval = rval[sorted(rval.columns)]
rval.to_csv(path_or_buf=outfile_result,
sep='\t',
header=True,
index=False)
return 0
# deprecate train test split mode
"""searcher = _do_train_test_split_val(
searcher, X, y, params,
primary_scoring=primary_scoring,
error_score=options['error_score'],
groups=groups,
outfile=outfile_result)"""
# no outer split
else:
searcher.set_params(n_jobs=N_JOBS)
if options['error_score'] == 'raise':
searcher.fit(X, y, groups=groups)
else:
warnings.simplefilter('always', FitFailedWarning)
with warnings.catch_warnings(record=True) as w:
try:
searcher.fit(X, y, groups=groups)
except ValueError:
pass
for warning in w:
print(repr(warning.message))
cv_results = pd.DataFrame(searcher.cv_results_)
cv_results = cv_results[sorted(cv_results.columns)]
cv_results.to_csv(path_or_buf=outfile_result,
sep='\t',
header=True,
index=False)
memory.clear(warn=False)
# output best estimator, and weights if applicable
if outfile_object:
best_estimator_ = getattr(searcher, 'best_estimator_', None)
if not best_estimator_:
warnings.warn("GridSearchCV object has no attribute "
"'best_estimator_', because either it's "
"nested gridsearch or `refit` is False!")
return
print("Saving best estimator: %s " % repr(best_estimator_))
dump_model_to_h5(best_estimator_, outfile_object)
def _get_X_y(params, infile1, infile2):
"""read from inputs and output X and y
Parameters
----------
params : dict
Tool inputs parameter
infile1 : str
File path to dataset containing features
infile2 : str
File path to dataset containing target values
"""
# store read dataframe object
loaded_df = {}
input_type = params["input_options"]["selected_input"]
# tabular input
if input_type == "tabular":
header = "infer" if params["input_options"]["header1"] else None
column_option = params["input_options"]["column_selector_options_1"][
"selected_column_selector_option"
]
if column_option in [
"by_index_number",
"all_but_by_index_number",
"by_header_name",
"all_but_by_header_name",
]:
c = params["input_options"]["column_selector_options_1"]["col1"]
else:
c = None
df_key = infile1 + repr(header)
df = pd.read_csv(infile1, sep="\t", header=header, parse_dates=True)
loaded_df[df_key] = df
X = read_columns(df, c=c, c_option=column_option).astype(float)
# sparse input
elif input_type == "sparse":
X = mmread(open(infile1, "r"))
# Get target y
header = "infer" if params["input_options"]["header2"] else None
column_option = params["input_options"]["column_selector_options_2"][
"selected_column_selector_option2"
]
if column_option in [
"by_index_number",
"all_but_by_index_number",
"by_header_name",
"all_but_by_header_name",
]:
c = params["input_options"]["column_selector_options_2"]["col2"]
else:
c = None
df_key = infile2 + repr(header)
if df_key in loaded_df:
infile2 = loaded_df[df_key]
else:
infile2 = pd.read_csv(infile2, sep="\t", header=header, parse_dates=True)
loaded_df[df_key] = infile2
y = read_columns(
infile2, c=c, c_option=column_option, sep="\t", header=header, parse_dates=True
)
if len(y.shape) == 2 and y.shape[1] == 1:
y = y.ravel()
return X, y
def _get_single_cv_split(params,
array,
infile_labels=None,
infile_groups=None):
""" output (train, test) subset from a cv splitter
Parameters
----------
params : dict
Galaxy tool inputs
array : pandas DataFrame object
The target dataset to split
infile_labels : str
File path to dataset containing target values
infile_groups : str
File path to dataset containing group values
"""
y = None
groups = None
nth_split = params['mode_selection']['nth_split']
# read groups
if infile_groups:
header = 'infer' if (params['mode_selection']['cv_selector']
['groups_selector']['header_g']) else None
column_option = (
params['mode_selection']['cv_selector']['groups_selector']
['column_selector_options_g']['selected_column_selector_option_g'])
if column_option in [
'by_index_number', 'all_but_by_index_number', 'by_header_name',
'all_but_by_header_name'
]:
c = (params['mode_selection']['cv_selector']['groups_selector']
['column_selector_options_g']['col_g'])
else:
c = None
groups = read_columns(infile_groups,
c=c,
c_option=column_option,
sep='\t',
header=header,
parse_dates=True)
groups = groups.ravel()
params['mode_selection']['cv_selector']['groups_selector'] = groups
# read labels
if infile_labels:
target_input = (
params['mode_selection']['cv_selector'].pop('target_input'))
header = 'infer' if target_input['header1'] else None
col_index = target_input['col'][0] - 1
df = pd.read_csv(infile_labels,
sep='\t',
header=header,
parse_dates=True)
y = df.iloc[:, col_index].values
# construct the cv splitter object
splitter, groups = get_cv(params['mode_selection']['cv_selector'])
total_n_splits = splitter.get_n_splits(array.values, y=y, groups=groups)
if nth_split > total_n_splits:
raise ValueError("Total number of splits is {}, but got `nth_split` "
"= {}".format(total_n_splits, nth_split))
i = 1
for train_index, test_index in splitter.split(array.values,
y=y,
groups=groups):
# suppose nth_split >= 1
if i == nth_split:
break
else:
i += 1
train = array.iloc[train_index, :]
test = array.iloc[test_index, :]
return train, test
def main(
inputs,
infile_estimator,
infile1,
infile2,
outfile_result,
outfile_object=None,
outfile_weights=None,
groups=None,
ref_seq=None,
intervals=None,
targets=None,
fasta_path=None,
):
"""
Parameter
---------
inputs : str
File path to galaxy tool parameter
infile_estimator : str
File path to estimator
infile1 : str
File path to dataset containing features
infile2 : str
File path to dataset containing target values
outfile_result : str
File path to save the results, either cv_results or test result
outfile_object : str, optional
File path to save searchCV object
outfile_weights : str, optional
File path to save model weights
groups : str
File path to dataset containing groups labels
ref_seq : str
File path to dataset containing genome sequence file
intervals : str
File path to dataset containing interval file
targets : str
File path to dataset compressed target bed file
fasta_path : str
File path to dataset containing fasta file
"""
warnings.simplefilter("ignore")
# store read dataframe object
loaded_df = {}
with open(inputs, "r") as param_handler:
params = json.load(param_handler)
# Override the refit parameter
params["search_schemes"]["options"]["refit"] = (
True if params["save"] != "nope" else False
)
with open(infile_estimator, "rb") as estimator_handler:
estimator = load_model(estimator_handler)
optimizer = params["search_schemes"]["selected_search_scheme"]
optimizer = getattr(model_selection, optimizer)
# handle gridsearchcv options
options = params["search_schemes"]["options"]
if groups:
header = (
"infer" if (options["cv_selector"]["groups_selector"]["header_g"]) else None
)
column_option = options["cv_selector"]["groups_selector"][
"column_selector_options_g"
]["selected_column_selector_option_g"]
if column_option in [
"by_index_number",
"all_but_by_index_number",
"by_header_name",
"all_but_by_header_name",
]:
c = options["cv_selector"]["groups_selector"]["column_selector_options_g"][
"col_g"
]
else:
c = None
df_key = groups + repr(header)
groups = pd.read_csv(groups, sep="\t", header=header, parse_dates=True)
loaded_df[df_key] = groups
groups = read_columns(
groups,
c=c,
c_option=column_option,
sep="\t",
header=header,
parse_dates=True,
)
groups = groups.ravel()
options["cv_selector"]["groups_selector"] = groups
splitter, groups = get_cv(options.pop("cv_selector"))
options["cv"] = splitter
primary_scoring = options["scoring"]["primary_scoring"]
# get_scoring() expects secondary_scoring to be a comma separated string (not a list)
# Check if secondary_scoring is specified
secondary_scoring = options["scoring"].get("secondary_scoring", None)
if secondary_scoring is not None:
# If secondary_scoring is specified, convert the list into comman separated string
options["scoring"]["secondary_scoring"] = ",".join(
options["scoring"]["secondary_scoring"]
)
options["scoring"] = get_scoring(options["scoring"])
if options["error_score"]:
options["error_score"] = "raise"
else:
options["error_score"] = np.nan
if options["refit"] and isinstance(options["scoring"], dict):
options["refit"] = primary_scoring
if "pre_dispatch" in options and options["pre_dispatch"] == "":
options["pre_dispatch"] = None
params_builder = params["search_schemes"]["search_params_builder"]
param_grid = _eval_search_params(params_builder)
estimator = clean_params(estimator)
# save the SearchCV object without fit
if params["save"] == "save_no_fit":
searcher = optimizer(estimator, param_grid, **options)
print(searcher)
with open(outfile_object, "wb") as output_handler:
pickle.dump(searcher, output_handler, pickle.HIGHEST_PROTOCOL)
return 0
# read inputs and loads new attributes, like paths
estimator, X, y = _handle_X_y(
estimator,
params,
infile1,
infile2,
loaded_df=loaded_df,
ref_seq=ref_seq,
intervals=intervals,
targets=targets,
fasta_path=fasta_path,
)
# cache iraps_core fits could increase search speed significantly
memory = joblib.Memory(location=CACHE_DIR, verbose=0)
main_est = get_main_estimator(estimator)
if main_est.__class__.__name__ == "IRAPSClassifier":
main_est.set_params(memory=memory)
searcher = optimizer(estimator, param_grid, **options)
split_mode = params["outer_split"].pop("split_mode")
if split_mode == "nested_cv":
# make sure refit is choosen
# this could be True for sklearn models, but not the case for
# deep learning models
if not options["refit"] and not all(
hasattr(estimator, attr) for attr in ("config", "model_type")
):
warnings.warn("Refit is change to `True` for nested validation!")
setattr(searcher, "refit", True)
outer_cv, _ = get_cv(params["outer_split"]["cv_selector"])
# nested CV, outer cv using cross_validate
if options["error_score"] == "raise":
rval = cross_validate(
searcher,
X,
y,
scoring=options["scoring"],
cv=outer_cv,
n_jobs=N_JOBS,
verbose=options["verbose"],
return_estimator=(params["save"] == "save_estimator"),
error_score=options["error_score"],
return_train_score=True,
)
else:
warnings.simplefilter("always", FitFailedWarning)
with warnings.catch_warnings(record=True) as w:
try:
rval = cross_validate(
searcher,
X,
y,
scoring=options["scoring"],
cv=outer_cv,
n_jobs=N_JOBS,
verbose=options["verbose"],
return_estimator=(params["save"] == "save_estimator"),
error_score=options["error_score"],
return_train_score=True,
)
except ValueError:
pass
for warning in w:
print(repr(warning.message))
fitted_searchers = rval.pop("estimator", [])
if fitted_searchers:
import os
pwd = os.getcwd()
save_dir = os.path.join(pwd, "cv_results_in_folds")
try:
os.mkdir(save_dir)
for idx, obj in enumerate(fitted_searchers):
target_name = "cv_results_" + "_" + "split%d" % idx
target_path = os.path.join(pwd, save_dir, target_name)
cv_results_ = getattr(obj, "cv_results_", None)
if not cv_results_:
print("%s is not available" % target_name)
continue
cv_results_ = pd.DataFrame(cv_results_)
cv_results_ = cv_results_[sorted(cv_results_.columns)]
cv_results_.to_csv(target_path, sep="\t", header=True, index=False)
except Exception as e:
print(e)
finally:
del os
keys = list(rval.keys())
for k in keys:
if k.startswith("test"):
rval["mean_" + k] = np.mean(rval[k])
rval["std_" + k] = np.std(rval[k])
if k.endswith("time"):
rval.pop(k)
rval = pd.DataFrame(rval)
rval = rval[sorted(rval.columns)]
rval.to_csv(path_or_buf=outfile_result, sep="\t", header=True, index=False)
# deprecate train test split mode
"""searcher = _do_train_test_split_val(
searcher, X, y, params,
primary_scoring=primary_scoring,
error_score=options['error_score'],
groups=groups,
outfile=outfile_result)"""
return 0
# no outer split
else:
searcher.set_params(n_jobs=N_JOBS)
if options["error_score"] == "raise":
searcher.fit(X, y, groups=groups)
else:
warnings.simplefilter("always", FitFailedWarning)
with warnings.catch_warnings(record=True) as w:
try:
searcher.fit(X, y, groups=groups)
except ValueError:
pass
for warning in w:
print(repr(warning.message))
cv_results = pd.DataFrame(searcher.cv_results_)
cv_results = cv_results[sorted(cv_results.columns)]
cv_results.to_csv(
path_or_buf=outfile_result, sep="\t", header=True, index=False
)
memory.clear(warn=False)
# output best estimator, and weights if applicable
if outfile_object:
best_estimator_ = getattr(searcher, "best_estimator_", None)
if not best_estimator_:
warnings.warn(
"GridSearchCV object has no attribute "
"'best_estimator_', because either it's "
"nested gridsearch or `refit` is False!"
)
return
# clean prams
best_estimator_ = clean_params(best_estimator_)
main_est = get_main_estimator(best_estimator_)
if hasattr(main_est, "model_") and hasattr(main_est, "save_weights"):
if outfile_weights:
main_est.save_weights(outfile_weights)
del main_est.model_
del main_est.fit_params
del main_est.model_class_
del main_est.validation_data
if getattr(main_est, "data_generator_", None):
del main_est.data_generator_
with open(outfile_object, "wb") as output_handler:
print("Best estimator is saved: %s " % repr(best_estimator_))
pickle.dump(best_estimator_, output_handler, pickle.HIGHEST_PROTOCOL)
def main(inputs, infile_estimator, infile1, infile2,
outfile_result, outfile_object=None,
outfile_weights=None, groups=None,
ref_seq=None, intervals=None, targets=None,
fasta_path=None):
"""
Parameter
---------
inputs : str
File path to galaxy tool parameter
infile_estimator : str
File path to estimator
infile1 : str
File path to dataset containing features
infile2 : str
File path to dataset containing target values
outfile_result : str
File path to save the results, either cv_results or test result
outfile_object : str, optional
File path to save searchCV object
outfile_weights : str, optional
File path to save model weights
groups : str
File path to dataset containing groups labels
ref_seq : str
File path to dataset containing genome sequence file
intervals : str
File path to dataset containing interval file
targets : str
File path to dataset compressed target bed file
fasta_path : str
File path to dataset containing fasta file
"""
warnings.simplefilter('ignore')
with open(inputs, 'r') as param_handler:
params = json.load(param_handler)
params_builder = params['search_schemes']['search_params_builder']
with open(infile_estimator, 'rb') as estimator_handler:
estimator = load_model(estimator_handler)
estimator_params = estimator.get_params()
# store read dataframe object
loaded_df = {}
input_type = params['input_options']['selected_input']
# tabular input
if input_type == 'tabular':
header = 'infer' if params['input_options']['header1'] else None
column_option = (params['input_options']['column_selector_options_1']
['selected_column_selector_option'])
if column_option in ['by_index_number', 'all_but_by_index_number',
'by_header_name', 'all_but_by_header_name']:
c = params['input_options']['column_selector_options_1']['col1']
else:
c = None
df_key = infile1 + repr(header)
df = pd.read_csv(infile1, sep='\t', header=header,
parse_dates=True)
loaded_df[df_key] = df
X = read_columns(df, c=c, c_option=column_option).astype(float)
# sparse input
elif input_type == 'sparse':
X = mmread(open(infile1, 'r'))
# fasta_file input
elif input_type == 'seq_fasta':
pyfaidx = get_module('pyfaidx')
sequences = pyfaidx.Fasta(fasta_path)
n_seqs = len(sequences.keys())
X = np.arange(n_seqs)[:, np.newaxis]
for param in estimator_params.keys():
if param.endswith('fasta_path'):
estimator.set_params(
**{param: fasta_path})
break
else:
raise ValueError(
"The selected estimator doesn't support "
"fasta file input! Please consider using "
"KerasGBatchClassifier with "
"FastaDNABatchGenerator/FastaProteinBatchGenerator "
"or having GenomeOneHotEncoder/ProteinOneHotEncoder "
"in pipeline!")
elif input_type == 'refseq_and_interval':
path_params = {
'data_batch_generator__ref_genome_path': ref_seq,
'data_batch_generator__intervals_path': intervals,
'data_batch_generator__target_path': targets
}
estimator.set_params(**path_params)
n_intervals = sum(1 for line in open(intervals))
X = np.arange(n_intervals)[:, np.newaxis]
# Get target y
header = 'infer' if params['input_options']['header2'] else None
column_option = (params['input_options']['column_selector_options_2']
['selected_column_selector_option2'])
if column_option in ['by_index_number', 'all_but_by_index_number',
'by_header_name', 'all_but_by_header_name']:
c = params['input_options']['column_selector_options_2']['col2']
else:
c = None
df_key = infile2 + repr(header)
if df_key in loaded_df:
infile2 = loaded_df[df_key]
else:
infile2 = pd.read_csv(infile2, sep='\t',
header=header, parse_dates=True)
loaded_df[df_key] = infile2
y = read_columns(
infile2,
c=c,
c_option=column_option,
sep='\t',
header=header,
parse_dates=True)
if len(y.shape) == 2 and y.shape[1] == 1:
y = y.ravel()
if input_type == 'refseq_and_interval':
estimator.set_params(
data_batch_generator__features=y.ravel().tolist())
y = None
# end y
optimizer = params['search_schemes']['selected_search_scheme']
optimizer = getattr(model_selection, optimizer)
# handle gridsearchcv options
options = params['search_schemes']['options']
if groups:
header = 'infer' if (options['cv_selector']['groups_selector']
['header_g']) else None
column_option = (options['cv_selector']['groups_selector']
['column_selector_options_g']
['selected_column_selector_option_g'])
if column_option in ['by_index_number', 'all_but_by_index_number',
'by_header_name', 'all_but_by_header_name']:
c = (options['cv_selector']['groups_selector']
['column_selector_options_g']['col_g'])
else:
c = None
df_key = groups + repr(header)
if df_key in loaded_df:
groups = loaded_df[df_key]
groups = read_columns(
groups,
c=c,
c_option=column_option,
sep='\t',
header=header,
parse_dates=True)
groups = groups.ravel()
options['cv_selector']['groups_selector'] = groups
splitter, groups = get_cv(options.pop('cv_selector'))
options['cv'] = splitter
options['n_jobs'] = N_JOBS
primary_scoring = options['scoring']['primary_scoring']
options['scoring'] = get_scoring(options['scoring'])
if options['error_score']:
options['error_score'] = 'raise'
else:
options['error_score'] = np.NaN
if options['refit'] and isinstance(options['scoring'], dict):
options['refit'] = primary_scoring
if 'pre_dispatch' in options and options['pre_dispatch'] == '':
options['pre_dispatch'] = None
# del loaded_df
del loaded_df
# handle memory
memory = joblib.Memory(location=CACHE_DIR, verbose=0)
# cache iraps_core fits could increase search speed significantly
if estimator.__class__.__name__ == 'IRAPSClassifier':
estimator.set_params(memory=memory)
else:
# For iraps buried in pipeline
for p, v in estimator_params.items():
if p.endswith('memory'):
# for case of `__irapsclassifier__memory`
if len(p) > 8 and p[:-8].endswith('irapsclassifier'):
# cache iraps_core fits could increase search
# speed significantly
new_params = {p: memory}
estimator.set_params(**new_params)
# security reason, we don't want memory being
# modified unexpectedly
elif v:
new_params = {p, None}
estimator.set_params(**new_params)
# For now, 1 CPU is suggested for iprasclassifier
elif p.endswith('n_jobs'):
new_params = {p: 1}
estimator.set_params(**new_params)
# for security reason, types of callbacks are limited
elif p.endswith('callbacks'):
for cb in v:
cb_type = cb['callback_selection']['callback_type']
if cb_type not in ALLOWED_CALLBACKS:
raise ValueError(
"Prohibited callback type: %s!" % cb_type)
param_grid = _eval_search_params(params_builder)
searcher = optimizer(estimator, param_grid, **options)
# do nested split
split_mode = params['outer_split'].pop('split_mode')
# nested CV, outer cv using cross_validate
if split_mode == 'nested_cv':
outer_cv, _ = get_cv(params['outer_split']['cv_selector'])
if options['error_score'] == 'raise':
rval = cross_validate(
searcher, X, y, scoring=options['scoring'],
cv=outer_cv, n_jobs=N_JOBS, verbose=0,
error_score=options['error_score'])
else:
warnings.simplefilter('always', FitFailedWarning)
with warnings.catch_warnings(record=True) as w:
try:
rval = cross_validate(
searcher, X, y,
scoring=options['scoring'],
cv=outer_cv, n_jobs=N_JOBS,
verbose=0,
error_score=options['error_score'])
except ValueError:
pass
for warning in w:
print(repr(warning.message))
keys = list(rval.keys())
for k in keys:
if k.startswith('test'):
rval['mean_' + k] = np.mean(rval[k])
rval['std_' + k] = np.std(rval[k])
if k.endswith('time'):
rval.pop(k)
rval = pd.DataFrame(rval)
rval = rval[sorted(rval.columns)]
rval.to_csv(path_or_buf=outfile_result, sep='\t',
header=True, index=False)
else:
if split_mode == 'train_test_split':
train_test_split = try_get_attr(
'galaxy_ml.model_validations', 'train_test_split')
# make sure refit is choosen
# this could be True for sklearn models, but not the case for
# deep learning models
if not options['refit'] and \
not all(hasattr(estimator, attr)
for attr in ('config', 'model_type')):
warnings.warn("Refit is change to `True` for nested "
"validation!")
setattr(searcher, 'refit', True)
split_options = params['outer_split']
# splits
if split_options['shuffle'] == 'stratified':
split_options['labels'] = y
X, X_test, y, y_test = train_test_split(X, y, **split_options)
elif split_options['shuffle'] == 'group':
if groups is None:
raise ValueError("No group based CV option was "
"choosen for group shuffle!")
split_options['labels'] = groups
if y is None:
X, X_test, groups, _ =\
train_test_split(X, groups, **split_options)
else:
X, X_test, y, y_test, groups, _ =\
train_test_split(X, y, groups, **split_options)
else:
if split_options['shuffle'] == 'None':
split_options['shuffle'] = None
X, X_test, y, y_test =\
train_test_split(X, y, **split_options)
# end train_test_split
# shared by both train_test_split and non-split
if options['error_score'] == 'raise':
searcher.fit(X, y, groups=groups)
else:
warnings.simplefilter('always', FitFailedWarning)
with warnings.catch_warnings(record=True) as w:
try:
searcher.fit(X, y, groups=groups)
except ValueError:
pass
for warning in w:
print(repr(warning.message))
# no outer split
if split_mode == 'no':
# save results
cv_results = pd.DataFrame(searcher.cv_results_)
cv_results = cv_results[sorted(cv_results.columns)]
cv_results.to_csv(path_or_buf=outfile_result, sep='\t',
header=True, index=False)
# train_test_split, output test result using best_estimator_
# or rebuild the trained estimator using weights if applicable.
else:
scorer_ = searcher.scorer_
if isinstance(scorer_, collections.Mapping):
is_multimetric = True
else:
is_multimetric = False
best_estimator_ = getattr(searcher, 'best_estimator_', None)
if not best_estimator_:
raise ValueError("GridSearchCV object has no "
"`best_estimator_` when `refit`=False!")
if best_estimator_.__class__.__name__ == 'KerasGBatchClassifier' \
and hasattr(estimator.data_batch_generator, 'target_path'):
test_score = best_estimator_.evaluate(
X_test, scorer=scorer_, is_multimetric=is_multimetric)
else:
test_score = _score(best_estimator_, X_test,
y_test, scorer_,
is_multimetric=is_multimetric)
if not is_multimetric:
test_score = {primary_scoring: test_score}
for key, value in test_score.items():
test_score[key] = [value]
result_df = pd.DataFrame(test_score)
result_df.to_csv(path_or_buf=outfile_result, sep='\t',
header=True, index=False)
memory.clear(warn=False)
if outfile_object:
best_estimator_ = getattr(searcher, 'best_estimator_', None)
if not best_estimator_:
warnings.warn("GridSearchCV object has no attribute "
"'best_estimator_', because either it's "
"nested gridsearch or `refit` is False!")
return
main_est = best_estimator_
if isinstance(best_estimator_, pipeline.Pipeline):
main_est = best_estimator_.steps[-1][-1]
if hasattr(main_est, 'model_') \
and hasattr(main_est, 'save_weights'):
if outfile_weights:
main_est.save_weights(outfile_weights)
del main_est.model_
del main_est.fit_params
del main_est.model_class_
del main_est.validation_data
if getattr(main_est, 'data_generator_', None):
del main_est.data_generator_
del main_est.data_batch_generator
with open(outfile_object, 'wb') as output_handler:
pickle.dump(best_estimator_, output_handler,
pickle.HIGHEST_PROTOCOL)
def main(inputs,
infile_estimator,
outfile_predict,
infile_weights=None,
infile1=None,
fasta_path=None,
ref_seq=None,
vcf_path=None):
"""
Parameter
---------
inputs : str
File path to galaxy tool parameter
infile_estimator : strgit
File path to trained estimator input
outfile_predict : str
File path to save the prediction results, tabular
infile_weights : str
File path to weights input
infile1 : str
File path to dataset containing features
fasta_path : str
File path to dataset containing fasta file
ref_seq : str
File path to dataset containing the reference genome sequence.
vcf_path : str
File path to dataset containing variants info.
"""
warnings.filterwarnings('ignore')
with open(inputs, 'r') as param_handler:
params = json.load(param_handler)
# load model
with open(infile_estimator, 'rb') as est_handler:
estimator = load_model(est_handler)
main_est = estimator
if isinstance(estimator, Pipeline):
main_est = estimator.steps[-1][-1]
if hasattr(main_est, 'config') and hasattr(main_est, 'load_weights'):
if not infile_weights or infile_weights == 'None':
raise ValueError("The selected model skeleton asks for weights, "
"but dataset for weights wan not selected!")
main_est.load_weights(infile_weights)
# handle data input
input_type = params['input_options']['selected_input']
# tabular input
if input_type == 'tabular':
header = 'infer' if params['input_options']['header1'] else None
column_option = (params['input_options']['column_selector_options_1']
['selected_column_selector_option'])
if column_option in [
'by_index_number', 'all_but_by_index_number', 'by_header_name',
'all_but_by_header_name'
]:
c = params['input_options']['column_selector_options_1']['col1']
else:
c = None
df = pd.read_csv(infile1, sep='\t', header=header, parse_dates=True)
X = read_columns(df, c=c, c_option=column_option).astype(float)
if params['method'] == 'predict':
preds = estimator.predict(X)
else:
preds = estimator.predict_proba(X)
# sparse input
elif input_type == 'sparse':
X = mmread(open(infile1, 'r'))
if params['method'] == 'predict':
preds = estimator.predict(X)
else:
preds = estimator.predict_proba(X)
# fasta input
elif input_type == 'seq_fasta':
if not hasattr(estimator, 'data_batch_generator'):
raise ValueError("To do prediction on sequences in fasta input, "
"the estimator must be a `KerasGBatchClassifier`"
"equipped with data_batch_generator!")
pyfaidx = get_module('pyfaidx')
sequences = pyfaidx.Fasta(fasta_path)
n_seqs = len(sequences.keys())
X = np.arange(n_seqs)[:, np.newaxis]
seq_length = estimator.data_batch_generator.seq_length
batch_size = getattr(estimator, 'batch_size', 32)
steps = (n_seqs + batch_size - 1) // batch_size
seq_type = params['input_options']['seq_type']
klass = try_get_attr('galaxy_ml.preprocessors', seq_type)
pred_data_generator = klass(fasta_path, seq_length=seq_length)
if params['method'] == 'predict':
preds = estimator.predict(X,
data_generator=pred_data_generator,
steps=steps)
else:
preds = estimator.predict_proba(X,
data_generator=pred_data_generator,
steps=steps)
# vcf input
elif input_type == 'variant_effect':
klass = try_get_attr('galaxy_ml.preprocessors',
'GenomicVariantBatchGenerator')
options = params['input_options']
options.pop('selected_input')
if options['blacklist_regions'] == 'none':
options['blacklist_regions'] = None
pred_data_generator = klass(ref_genome_path=ref_seq,
vcf_path=vcf_path,
**options)
pred_data_generator.fit()
preds = estimator.model_.predict_generator(
pred_data_generator.flow(batch_size=32),
workers=N_JOBS,
use_multiprocessing=True)
if preds.min() < 0. or preds.max() > 1.:
warnings.warn('Network returning invalid probability values. '
'The last layer might not normalize predictions '
'into probabilities '
'(like softmax or sigmoid would).')
if params['method'] == 'predict_proba' and preds.shape[1] == 1:
# first column is probability of class 0 and second is of class 1
preds = np.hstack([1 - preds, preds])
elif params['method'] == 'predict':
if preds.shape[-1] > 1:
# if the last activation is `softmax`, the sum of all
# probibilities will 1, the classification is considered as
# multi-class problem, otherwise, we take it as multi-label.
act = getattr(estimator.model_.layers[-1], 'activation', None)
if act and act.__name__ == 'softmax':
classes = preds.argmax(axis=-1)
else:
preds = (preds > 0.5).astype('int32')
else:
classes = (preds > 0.5).astype('int32')
preds = estimator.classes_[classes]
# end input
# output
if input_type == 'variant_effect': # TODO: save in batchs
rval = pd.DataFrame(preds)
meta = pd.DataFrame(
pred_data_generator.variants,
columns=['chrom', 'pos', 'name', 'ref', 'alt', 'strand'])
rval = pd.concat([meta, rval], axis=1)
elif len(preds.shape) == 1:
rval = pd.DataFrame(preds, columns=['Predicted'])
else:
rval = pd.DataFrame(preds)
rval.to_csv(outfile_predict, sep='\t', header=True, index=False)
def main():
parser = argparse.ArgumentParser(description='RDKit screen')
parser.add_argument('-i', '--infile', help="Input file")
parser.add_argument('--header',
action='store_true',
help="Include the header row or skip it")
parser.add_argument('-c', '--columns', type=str.lower, default='all', choices=['by_index_number', 'all_but_by_index_number',\
'by_header_name', 'all_but_by_header_name', 'all_columns'],
help="Choose to select all columns, or exclude/include some")
parser.add_argument(
'-ci',
'--column_indices',
type=str.lower,
help="Choose to select all columns, or exclude/include some")
parser.add_argument('-n', '--number', nargs='?', type=int, default=None,\
help="Number of components to keep. If not set, all components are kept")
parser.add_argument('--whiten',
action='store_true',
help="Whiten the components")
parser.add_argument('-t',
'--pca_type',
type=str.lower,
default='classical',
choices=['classical', 'incremental', 'kernel'],
help="Choose which flavour of PCA to use")
parser.add_argument('-s',
'--svd_solver',
type=str.lower,
default='auto',
choices=['auto', 'full', 'arpack', 'randomized'],
help="Choose the type of svd solver.")
parser.add_argument('-b', '--batch_size', nargs='?', type=int, default=None,\
help="The number of samples to use for each batch")
parser.add_argument('-k', '--kernel', type=str.lower, default='linear',\
choices=['linear', 'poly', 'rbf', 'sigmoid', 'cosine', 'precomputed'],
help="Choose the type of kernel.")
parser.add_argument(
'-g',
'--gamma',
nargs='?',
type=float,
default=None,
help=
'Kernel coefficient for rbf, poly and sigmoid kernels. Ignored by other kernels'
)
parser.add_argument(
'-tol',
'--tolerance',
type=float,
default=0.0,
help=
'Convergence tolerance for arpack. If 0, optimal value will be chosen by arpack'
)
parser.add_argument('-mi', '--max_iter', nargs='?', type=int, default=None,\
help="Maximum number of iterations for arpack")
parser.add_argument('-d', '--degree', type=int, default=3,\
help="Degree for poly kernels. Ignored by other kernels")
parser.add_argument('-cf',
'--coef0',
type=float,
default=1.0,
help='Independent term in poly and sigmoid kernels')
parser.add_argument('-e',
'--eigen_solver',
type=str.lower,
default='auto',
choices=['auto', 'dense', 'arpack'],
help="Choose the type of eigen solver.")
parser.add_argument('-o',
'--outfile',
help="Base name for output file (no extension).")
args = parser.parse_args()
usecols = None
cols = []
pca_params = {}
if args.columns == 'by_index_number' or args.columns == 'all_but_by_index_number':
usecols = [int(i) for i in args.column_indices.split(',')]
elif args.columns == 'by_header_name' or args.columns == 'all_but_by_header_name':
usecols = args.column_indices
header = 'infer' if args.header else None
pca_input = read_columns(f=args.infile,
c=usecols,
c_option=args.columns,
sep='\t',
header=header,
parse_dates=True,
encoding=None,
index_col=None)
pca_params.update({'n_components': args.number})
if args.pca_type == 'classical':
pca_params.update({
'svd_solver': args.svd_solver,
'whiten': args.whiten
})
if args.svd_solver == 'arpack':
pca_params.update({'tol': args.tolerance})
pca = PCA()
elif args.pca_type == 'incremental':
pca_params.update({
'batch_size': args.batch_size,
'whiten': args.whiten
})
pca = IncrementalPCA()
elif args.pca_type == 'kernel':
pca_params.update({
'kernel': args.kernel,
'eigen_solver': args.eigen_solver,
'gamma': args.gamma
})
if args.kernel == 'poly':
pca_params.update({'degree': args.degree, 'coef0': args.coef0})
elif args.kernel == 'sigmoid':
pca_params.update({'coef0': args.coef0})
elif args.kernel == 'precomputed':
pca_input = np.dot(pca_input, pca_input.T)
if args.eigen_solver == 'arpack':
pca_params.update({
'tol': args.tolerance,
'max_iter': args.max_iter
})
pca = KernelPCA()
print(pca_params)
pca.set_params(**pca_params)
pca_output = pca.fit_transform(pca_input)
np.savetxt(fname=args.outfile, X=pca_output, fmt='%.4f', delimiter='\t')
def main(inputs,
infile_estimator,
infile_images,
infile_dataframe,
outfile_result,
infile_weights=None,
outfile_object=None,
outfile_weights=None,
outfile_y_true=None,
outfile_y_preds=None,
groups=None):
"""
Parameter
---------
inputs : str
File path to galaxy tool parameter
infile_estimator : str
File path to estimator
infile_images : str
File path to datasets containing images
infile_dataframe : str
File path to tabular dataset containing image information
outfile_result : str
File path to save the results, either cv_results or test result
infile_weights : str
File path to input model weights, used in evaluation and
prediction mode.
outfile_object : str, optional
File path to save searchCV object
outfile_weights : str, optional
File path to save deep learning model weights
outfile_y_true : str, optional
File path to target values for prediction
outfile_y_preds : str, optional
File path to save deep learning model weights
groups : str
File path to dataset containing groups labels
"""
warnings.simplefilter('ignore')
with open(inputs, 'r') as param_handler:
params = json.load(param_handler)
# load estimator
with open(infile_estimator, 'rb') as estimator_handler:
estimator = load_model(estimator_handler)
estimator = clean_params(estimator)
if not isinstance(estimator, KerasGBatchClassifier):
raise ValueError(
"Only `galaxy_ml.keras_galaxy_models.KerasGBatchClassifier` "
"is supported!")
# read DataFrame for images
data_frame = pd.read_csv(infile_dataframe, sep='\t', header='infer')
kwargs = _handle_image_generator_params(params['input_options'],
data_frame)
# build data generator
image_generator = ImageDataFrameBatchGenerator(dataframe=data_frame,
directory=IMAGES_DIR,
**kwargs)
estimator.set_params(data_batch_generator=image_generator)
steps = estimator.prediction_steps
batch_size = estimator.batch_size
# Get X and y
X = np.arange(data_frame.shape[0])[:, np.newaxis]
if isinstance(kwargs['y_col'], list):
y = None
else:
y = data_frame[kwargs['y_col']].ravel()
# load groups
if groups:
groups_selector = (params['experiment_schemes']['test_split']
['split_algos']).pop('groups_selector')
header = 'infer' if groups_selector['header_g'] else None
column_option = \
(groups_selector['column_selector_options_g']
['selected_column_selector_option_g'])
if column_option in [
'by_index_number', 'all_but_by_index_number', 'by_header_name',
'all_but_by_header_name'
]:
c = groups_selector['column_selector_options_g']['col_g']
else:
c = None
if groups == infile_dataframe:
groups = data_frame
groups = read_columns(groups,
c=c,
c_option=column_option,
sep='\t',
header=header,
parse_dates=True)
groups = groups.ravel()
exp_scheme = params['experiment_schemes']['selected_exp_scheme']
# Model Predictions
if exp_scheme == 'model_predict':
estimator.load_weights(infile_weights)
steps = params['experiment_schemes']['pred_steps']
generator = image_generator.flow(X, y=None, batch_size=batch_size)
predictions = estimator.model_.predict_generator(generator,
steps=steps)
try:
pd.DataFrame(predictions).astype(np.float32).to_csv(
outfile_result,
sep='\t',
index=False,
float_format='%g',
chunksize=10000)
except Exception as e:
print("Error in saving predictions: %s" % e)
return 0
# Model Evaluation
if exp_scheme == 'model_eval':
estimator.load_weights(infile_weights)
# compile model
estimator.model_.compile(loss=estimator.loss,
optimizer=estimator._optimizer,
metrics=estimator.metrics)
steps = params['experiment_schemes']['eval_steps']
sk_scoring = params['experiment_schemes']['metrics']['scoring']
scores, predictions, y_true = _evaluate_keras_and_sklearn_scores(
estimator,
image_generator,
X,
sk_scoring=sk_scoring,
steps=steps,
batch_size=batch_size,
return_predictions=bool(outfile_y_true))
# for other two modes, train/val and train/val/test
else:
# swap hyperparameter
swapping = params['experiment_schemes']['hyperparams_swapping']
swap_params = _eval_swap_params(swapping)
estimator.set_params(**swap_params)
# handle test (first) split
test_split_options = (
params['experiment_schemes']['test_split']['split_algos'])
if test_split_options['shuffle'] == 'group':
test_split_options['labels'] = groups
if test_split_options['shuffle'] == 'stratified':
if y is not None:
test_split_options['labels'] = y
else:
raise ValueError("Stratified shuffle split is not "
"applicable on empty target values or "
"multiple output targets!")
X_train, X_test, y_train, y_test, groups_train, groups_test = \
train_test_split_none(X, y, groups, **test_split_options)
# handle validation (second) split
if exp_scheme == 'train_val_test':
val_split_options = (
params['experiment_schemes']['val_split']['split_algos'])
if val_split_options['shuffle'] == 'group':
val_split_options['labels'] = groups_train
if val_split_options['shuffle'] == 'stratified':
if y_train is not None:
val_split_options['labels'] = y_train
else:
raise ValueError("Stratified shuffle split is not "
"applicable on empty target values!")
X_train, X_val, y_train, y_val, groups_train, groups_val = \
train_test_split_none(X_train, y_train, groups_train,
**val_split_options)
# In image data generator, `y_val` must be None
# labels will be retrived in generator.
estimator.fit(X_train, y_train, validation_data=(X_val, ))
else:
estimator.fit(X_train, y_train, validation_data=(X_test, ))
data_generator = estimator.data_generator_
sk_scoring = params['experiment_schemes']['metrics']['scoring']
steps = estimator.prediction_steps
scores, predictions, y_true = _evaluate_keras_and_sklearn_scores(
estimator,
data_generator,
X_test,
sk_scoring=sk_scoring,
steps=steps,
batch_size=batch_size,
return_predictions=bool(outfile_y_true))
# handle output
if outfile_y_true:
try:
pd.DataFrame(y_true).to_csv(outfile_y_true, sep='\t', index=False)
pd.DataFrame(predictions).astype(np.float32).to_csv(
outfile_y_preds,
sep='\t',
index=False,
float_format='%g',
chunksize=10000)
except Exception as e:
print("Error in saving predictions: %s" % e)
# handle output
for name, score in scores.items():
scores[name] = [score]
df = pd.DataFrame(scores)
df = df[sorted(df.columns)]
df.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False)
if outfile_object:
main_est = get_main_estimator(estimator)
if hasattr(main_est, 'model_') \
and hasattr(main_est, 'save_weights'):
if outfile_weights:
main_est.save_weights(outfile_weights)
del main_est.model_
del main_est.fit_params
del main_est.model_class_
main_est.data_batch_generator.dataframe = None
main_est.callbacks = []
if getattr(main_est, 'data_generator_', None):
del main_est.data_generator_
with open(outfile_object, 'wb') as output_handler:
pickle.dump(estimator, output_handler, pickle.HIGHEST_PROTOCOL)
def main(inputs,
infile_estimator,
infile1,
infile2,
outfile_result,
outfile_object=None,
outfile_weights=None,
outfile_y_true=None,
outfile_y_preds=None,
groups=None,
ref_seq=None,
intervals=None,
targets=None,
fasta_path=None):
"""
Parameter
---------
inputs : str
File path to galaxy tool parameter
infile_estimator : str
File path to estimator
infile1 : str
File path to dataset containing features
infile2 : str
File path to dataset containing target values
outfile_result : str
File path to save the results, either cv_results or test result
outfile_object : str, optional
File path to save searchCV object
outfile_weights : str, optional
File path to save deep learning model weights
outfile_y_true : str, optional
File path to target values for prediction
outfile_y_preds : str, optional
File path to save deep learning model weights
groups : str
File path to dataset containing groups labels
ref_seq : str
File path to dataset containing genome sequence file
intervals : str
File path to dataset containing interval file
targets : str
File path to dataset compressed target bed file
fasta_path : str
File path to dataset containing fasta file
"""
warnings.simplefilter('ignore')
with open(inputs, 'r') as param_handler:
params = json.load(param_handler)
# load estimator
with open(infile_estimator, 'rb') as estimator_handler:
estimator = load_model(estimator_handler)
estimator = clean_params(estimator)
# swap hyperparameter
swapping = params['experiment_schemes']['hyperparams_swapping']
swap_params = _eval_swap_params(swapping)
estimator.set_params(**swap_params)
estimator_params = estimator.get_params()
# store read dataframe object
loaded_df = {}
input_type = params['input_options']['selected_input']
# tabular input
if input_type == 'tabular':
header = 'infer' if params['input_options']['header1'] else None
column_option = (params['input_options']['column_selector_options_1']
['selected_column_selector_option'])
if column_option in [
'by_index_number', 'all_but_by_index_number', 'by_header_name',
'all_but_by_header_name'
]:
c = params['input_options']['column_selector_options_1']['col1']
else:
c = None
df_key = infile1 + repr(header)
df = pd.read_csv(infile1, sep='\t', header=header, parse_dates=True)
loaded_df[df_key] = df
X = read_columns(df, c=c, c_option=column_option).astype(float)
# sparse input
elif input_type == 'sparse':
X = mmread(open(infile1, 'r'))
# fasta_file input
elif input_type == 'seq_fasta':
pyfaidx = get_module('pyfaidx')
sequences = pyfaidx.Fasta(fasta_path)
n_seqs = len(sequences.keys())
X = np.arange(n_seqs)[:, np.newaxis]
for param in estimator_params.keys():
if param.endswith('fasta_path'):
estimator.set_params(**{param: fasta_path})
break
else:
raise ValueError(
"The selected estimator doesn't support "
"fasta file input! Please consider using "
"KerasGBatchClassifier with "
"FastaDNABatchGenerator/FastaProteinBatchGenerator "
"or having GenomeOneHotEncoder/ProteinOneHotEncoder "
"in pipeline!")
elif input_type == 'refseq_and_interval':
path_params = {
'data_batch_generator__ref_genome_path': ref_seq,
'data_batch_generator__intervals_path': intervals,
'data_batch_generator__target_path': targets
}
estimator.set_params(**path_params)
n_intervals = sum(1 for line in open(intervals))
X = np.arange(n_intervals)[:, np.newaxis]
# Get target y
header = 'infer' if params['input_options']['header2'] else None
column_option = (params['input_options']['column_selector_options_2']
['selected_column_selector_option2'])
if column_option in [
'by_index_number', 'all_but_by_index_number', 'by_header_name',
'all_but_by_header_name'
]:
c = params['input_options']['column_selector_options_2']['col2']
else:
c = None
df_key = infile2 + repr(header)
if df_key in loaded_df:
infile2 = loaded_df[df_key]
else:
infile2 = pd.read_csv(infile2,
sep='\t',
header=header,
parse_dates=True)
loaded_df[df_key] = infile2
y = read_columns(infile2,
c=c,
c_option=column_option,
sep='\t',
header=header,
parse_dates=True)
if len(y.shape) == 2 and y.shape[1] == 1:
y = y.ravel()
if input_type == 'refseq_and_interval':
estimator.set_params(data_batch_generator__features=y.ravel().tolist())
y = None
# end y
# load groups
if groups:
groups_selector = (params['experiment_schemes']['test_split']
['split_algos']).pop('groups_selector')
header = 'infer' if groups_selector['header_g'] else None
column_option = \
(groups_selector['column_selector_options_g']
['selected_column_selector_option_g'])
if column_option in [
'by_index_number', 'all_but_by_index_number', 'by_header_name',
'all_but_by_header_name'
]:
c = groups_selector['column_selector_options_g']['col_g']
else:
c = None
df_key = groups + repr(header)
if df_key in loaded_df:
groups = loaded_df[df_key]
groups = read_columns(groups,
c=c,
c_option=column_option,
sep='\t',
header=header,
parse_dates=True)
groups = groups.ravel()
# del loaded_df
del loaded_df
# cache iraps_core fits could increase search speed significantly
memory = joblib.Memory(location=CACHE_DIR, verbose=0)
main_est = get_main_estimator(estimator)
if main_est.__class__.__name__ == 'IRAPSClassifier':
main_est.set_params(memory=memory)
# handle scorer, convert to scorer dict
scoring = params['experiment_schemes']['metrics']['scoring']
scorer = get_scoring(scoring)
scorer, _ = _check_multimetric_scoring(estimator, scoring=scorer)
# handle test (first) split
test_split_options = (
params['experiment_schemes']['test_split']['split_algos'])
if test_split_options['shuffle'] == 'group':
test_split_options['labels'] = groups
if test_split_options['shuffle'] == 'stratified':
if y is not None:
test_split_options['labels'] = y
else:
raise ValueError("Stratified shuffle split is not "
"applicable on empty target values!")
X_train, X_test, y_train, y_test, groups_train, groups_test = \
train_test_split_none(X, y, groups, **test_split_options)
exp_scheme = params['experiment_schemes']['selected_exp_scheme']
# handle validation (second) split
if exp_scheme == 'train_val_test':
val_split_options = (
params['experiment_schemes']['val_split']['split_algos'])
if val_split_options['shuffle'] == 'group':
val_split_options['labels'] = groups_train
if val_split_options['shuffle'] == 'stratified':
if y_train is not None:
val_split_options['labels'] = y_train
else:
raise ValueError("Stratified shuffle split is not "
"applicable on empty target values!")
X_train, X_val, y_train, y_val, groups_train, groups_val = \
train_test_split_none(X_train, y_train, groups_train,
**val_split_options)
# train and eval
if hasattr(estimator, 'config') and hasattr(estimator, 'model_type'):
if exp_scheme == 'train_val_test':
estimator.fit(X_train, y_train, validation_data=(X_val, y_val))
else:
estimator.fit(X_train, y_train, validation_data=(X_test, y_test))
else:
estimator.fit(X_train, y_train)
if isinstance(estimator, KerasGBatchClassifier):
scores = {}
steps = estimator.prediction_steps
batch_size = estimator.batch_size
data_generator = estimator.data_generator_
scores, predictions, y_true = _evaluate_keras_and_sklearn_scores(
estimator,
data_generator,
X_test,
y=y_test,
sk_scoring=sk_scoring,
steps=steps,
batch_size=batch_size,
return_predictions=bool(outfile_y_true))
else:
scores = {}
if hasattr(estimator, 'model_') \
and hasattr(estimator.model_, 'metrics_names'):
batch_size = estimator.batch_size
score_results = estimator.model_.evaluate(X_test,
y=y_test,
batch_size=batch_size,
verbose=0)
metrics_names = estimator.model_.metrics_names
if not isinstance(metrics_names, list):
scores[metrics_names] = score_results
else:
scores = dict(zip(metrics_names, score_results))
if hasattr(estimator, 'predict_proba'):
predictions = estimator.predict_proba(X_test)
else:
predictions = estimator.predict(X_test)
y_true = y_test
sk_scores = _score(estimator,
X_test,
y_test,
scorer,
is_multimetric=True)
scores.update(sk_scores)
# handle output
if outfile_y_true:
try:
pd.DataFrame(y_true).to_csv(outfile_y_true, sep='\t', index=False)
pd.DataFrame(predictions).astype(np.float32).to_csv(
outfile_y_preds,
sep='\t',
index=False,
float_format='%g',
chunksize=10000)
except Exception as e:
print("Error in saving predictions: %s" % e)
# handle output
for name, score in scores.items():
scores[name] = [score]
df = pd.DataFrame(scores)
df = df[sorted(df.columns)]
df.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False)
memory.clear(warn=False)
if outfile_object:
main_est = estimator
if isinstance(estimator, Pipeline):
main_est = estimator.steps[-1][-1]
if hasattr(main_est, 'model_') \
and hasattr(main_est, 'save_weights'):
if outfile_weights:
main_est.save_weights(outfile_weights)
del main_est.model_
del main_est.fit_params
del main_est.model_class_
main_est.callbacks = []
if getattr(main_est, 'data_generator_', None):
del main_est.data_generator_
with open(outfile_object, 'wb') as output_handler:
pickle.dump(estimator, output_handler, pickle.HIGHEST_PROTOCOL)
def _get_X_y(params, infile1, infile2):
""" read from inputs and output X and y
Parameters
----------
params : dict
Tool inputs parameter
infile1 : str
File path to dataset containing features
infile2 : str
File path to dataset containing target values
"""
# store read dataframe object
loaded_df = {}
input_type = params['input_options']['selected_input']
# tabular input
if input_type == 'tabular':
header = 'infer' if params['input_options']['header1'] else None
column_option = (params['input_options']['column_selector_options_1']
['selected_column_selector_option'])
if column_option in [
'by_index_number', 'all_but_by_index_number', 'by_header_name',
'all_but_by_header_name'
]:
c = params['input_options']['column_selector_options_1']['col1']
else:
c = None
df_key = infile1 + repr(header)
df = pd.read_csv(infile1, sep='\t', header=header, parse_dates=True)
loaded_df[df_key] = df
X = read_columns(df, c=c, c_option=column_option).astype(float)
# sparse input
elif input_type == 'sparse':
X = mmread(open(infile1, 'r'))
# Get target y
header = 'infer' if params['input_options']['header2'] else None
column_option = (params['input_options']['column_selector_options_2']
['selected_column_selector_option2'])
if column_option in [
'by_index_number', 'all_but_by_index_number', 'by_header_name',
'all_but_by_header_name'
]:
c = params['input_options']['column_selector_options_2']['col2']
else:
c = None
df_key = infile2 + repr(header)
if df_key in loaded_df:
infile2 = loaded_df[df_key]
else:
infile2 = pd.read_csv(infile2,
sep='\t',
header=header,
parse_dates=True)
loaded_df[df_key] = infile2
y = read_columns(infile2,
c=c,
c_option=column_option,
sep='\t',
header=header,
parse_dates=True)
if len(y.shape) == 2 and y.shape[1] == 1:
y = y.ravel()
return X, y
def main(inputs,
infile_estimator,
outfile_predict,
infile_weights=None,
infile1=None,
fasta_path=None,
ref_seq=None,
vcf_path=None):
"""
Parameter
---------
inputs : str
File path to galaxy tool parameter
infile_estimator : str
File path to trained estimator input
outfile_predict : str
File path to save the prediction results, tabular
infile_weights : str
File path to weights input
infile1 : str
File path to dataset containing features
fasta_path : str
File path to dataset containing fasta file
ref_seq : str
File path to dataset containing the reference genome sequence.
vcf_path : str
File path to dataset containing variants info.
"""
warnings.filterwarnings('ignore')
with open(inputs, 'r') as param_handler:
params = json.load(param_handler)
# load model
with open(infile_estimator, 'rb') as est_handler:
estimator = load_model(est_handler)
main_est = estimator
if isinstance(estimator, Pipeline):
main_est = estimator.steps[-1][-1]
if hasattr(main_est, 'config') and hasattr(main_est, 'load_weights'):
if not infile_weights or infile_weights == 'None':
raise ValueError("The selected model skeleton asks for weights, "
"but dataset for weights wan not selected!")
main_est.load_weights(infile_weights)
# handle data input
input_type = params['input_options']['selected_input']
# tabular input
if input_type == 'tabular':
header = 'infer' if params['input_options']['header1'] else None
column_option = (params['input_options']['column_selector_options_1']
['selected_column_selector_option'])
if column_option in [
'by_index_number', 'all_but_by_index_number', 'by_header_name',
'all_but_by_header_name'
]:
c = params['input_options']['column_selector_options_1']['col1']
else:
c = None
df = pd.read_csv(infile1, sep='\t', header=header, parse_dates=True)
X = read_columns(df, c=c, c_option=column_option).astype(float)
if params['method'] == 'predict':
preds = estimator.predict(X)
else:
preds = estimator.predict_proba(X)
# sparse input
elif input_type == 'sparse':
X = mmread(open(infile1, 'r'))
if params['method'] == 'predict':
preds = estimator.predict(X)
else:
preds = estimator.predict_proba(X)
# fasta input
elif input_type == 'seq_fasta':
if not hasattr(estimator, 'data_batch_generator'):
raise ValueError("To do prediction on sequences in fasta input, "
"the estimator must be a `KerasGBatchClassifier`"
"equipped with data_batch_generator!")
pyfaidx = get_module('pyfaidx')
sequences = pyfaidx.Fasta(fasta_path)
n_seqs = len(sequences.keys())
X = np.arange(n_seqs)[:, np.newaxis]
seq_length = estimator.data_batch_generator.seq_length
batch_size = getattr(estimator, 'batch_size', 32)
steps = (n_seqs + batch_size - 1) // batch_size
seq_type = params['input_options']['seq_type']
klass = try_get_attr('galaxy_ml.preprocessors', seq_type)
pred_data_generator = klass(fasta_path, seq_length=seq_length)
if params['method'] == 'predict':
preds = estimator.predict(X,
data_generator=pred_data_generator,
steps=steps)
else:
preds = estimator.predict_proba(X,
data_generator=pred_data_generator,
steps=steps)
# vcf input
elif input_type == 'variant_effect':
klass = try_get_attr('galaxy_ml.preprocessors',
'GenomicVariantBatchGenerator')
options = params['input_options']
options.pop('selected_input')
if options['blacklist_regions'] == 'none':
options['blacklist_regions'] = None
pred_data_generator = klass(ref_genome_path=ref_seq,
vcf_path=vcf_path,
**options)
pred_data_generator.set_processing_attrs()
variants = pred_data_generator.variants
# predict 1600 sample at once then write to file
gen_flow = pred_data_generator.flow(batch_size=1600)
file_writer = open(outfile_predict, 'w')
header_row = '\t'.join(
['chrom', 'pos', 'name', 'ref', 'alt', 'strand'])
file_writer.write(header_row)
header_done = False
steps_done = 0
# TODO: multiple threading
try:
while steps_done < len(gen_flow):
index_array = next(gen_flow.index_generator)
batch_X = gen_flow._get_batches_of_transformed_samples(
index_array)
if params['method'] == 'predict':
batch_preds = estimator.predict(
batch_X,
# The presence of `pred_data_generator` below is to
# override model carrying data_generator if there
# is any.
data_generator=pred_data_generator)
else:
batch_preds = estimator.predict_proba(
batch_X,
# The presence of `pred_data_generator` below is to
# override model carrying data_generator if there
# is any.
data_generator=pred_data_generator)
if batch_preds.ndim == 1:
batch_preds = batch_preds[:, np.newaxis]
batch_meta = variants[index_array]
batch_out = np.column_stack([batch_meta, batch_preds])
if not header_done:
heads = np.arange(batch_preds.shape[-1]).astype(str)
heads_str = '\t'.join(heads)
file_writer.write("\t%s\n" % heads_str)
header_done = True
for row in batch_out:
row_str = '\t'.join(row)
file_writer.write("%s\n" % row_str)
steps_done += 1
finally:
file_writer.close()
# TODO: make api `pred_data_generator.close()`
pred_data_generator.close()
return 0
# end input
# output
if len(preds.shape) == 1:
rval = pd.DataFrame(preds, columns=['Predicted'])
else:
rval = pd.DataFrame(preds)
rval.to_csv(outfile_predict, sep='\t', header=True, index=False)
def main(
inputs,
infile_estimator,
infile1,
infile2,
outfile_result,
outfile_object=None,
outfile_weights=None,
outfile_y_true=None,
outfile_y_preds=None,
groups=None,
ref_seq=None,
intervals=None,
targets=None,
fasta_path=None,
):
"""
Parameter
---------
inputs : str
File path to galaxy tool parameter
infile_estimator : str
File path to estimator
infile1 : str
File path to dataset containing features
infile2 : str
File path to dataset containing target values
outfile_result : str
File path to save the results, either cv_results or test result
outfile_object : str, optional
File path to save searchCV object
outfile_weights : str, optional
File path to save deep learning model weights
outfile_y_true : str, optional
File path to target values for prediction
outfile_y_preds : str, optional
File path to save deep learning model weights
groups : str
File path to dataset containing groups labels
ref_seq : str
File path to dataset containing genome sequence file
intervals : str
File path to dataset containing interval file
targets : str
File path to dataset compressed target bed file
fasta_path : str
File path to dataset containing fasta file
"""
warnings.simplefilter("ignore")
with open(inputs, "r") as param_handler:
params = json.load(param_handler)
# load estimator
with open(infile_estimator, "rb") as estimator_handler:
estimator = load_model(estimator_handler)
estimator = clean_params(estimator)
# swap hyperparameter
swapping = params["experiment_schemes"]["hyperparams_swapping"]
swap_params = _eval_swap_params(swapping)
estimator.set_params(**swap_params)
estimator_params = estimator.get_params()
# store read dataframe object
loaded_df = {}
input_type = params["input_options"]["selected_input"]
# tabular input
if input_type == "tabular":
header = "infer" if params["input_options"]["header1"] else None
column_option = params["input_options"]["column_selector_options_1"][
"selected_column_selector_option"
]
if column_option in [
"by_index_number",
"all_but_by_index_number",
"by_header_name",
"all_but_by_header_name",
]:
c = params["input_options"]["column_selector_options_1"]["col1"]
else:
c = None
df_key = infile1 + repr(header)
df = pd.read_csv(infile1, sep="\t", header=header, parse_dates=True)
loaded_df[df_key] = df
X = read_columns(df, c=c, c_option=column_option).astype(float)
# sparse input
elif input_type == "sparse":
X = mmread(open(infile1, "r"))
# fasta_file input
elif input_type == "seq_fasta":
pyfaidx = get_module("pyfaidx")
sequences = pyfaidx.Fasta(fasta_path)
n_seqs = len(sequences.keys())
X = np.arange(n_seqs)[:, np.newaxis]
for param in estimator_params.keys():
if param.endswith("fasta_path"):
estimator.set_params(**{param: fasta_path})
break
else:
raise ValueError(
"The selected estimator doesn't support "
"fasta file input! Please consider using "
"KerasGBatchClassifier with "
"FastaDNABatchGenerator/FastaProteinBatchGenerator "
"or having GenomeOneHotEncoder/ProteinOneHotEncoder "
"in pipeline!"
)
elif input_type == "refseq_and_interval":
path_params = {
"data_batch_generator__ref_genome_path": ref_seq,
"data_batch_generator__intervals_path": intervals,
"data_batch_generator__target_path": targets,
}
estimator.set_params(**path_params)
n_intervals = sum(1 for line in open(intervals))
X = np.arange(n_intervals)[:, np.newaxis]
# Get target y
header = "infer" if params["input_options"]["header2"] else None
column_option = params["input_options"]["column_selector_options_2"][
"selected_column_selector_option2"
]
if column_option in [
"by_index_number",
"all_but_by_index_number",
"by_header_name",
"all_but_by_header_name",
]:
c = params["input_options"]["column_selector_options_2"]["col2"]
else:
c = None
df_key = infile2 + repr(header)
if df_key in loaded_df:
infile2 = loaded_df[df_key]
else:
infile2 = pd.read_csv(infile2, sep="\t", header=header, parse_dates=True)
loaded_df[df_key] = infile2
y = read_columns(
infile2, c=c, c_option=column_option, sep="\t", header=header, parse_dates=True
)
if len(y.shape) == 2 and y.shape[1] == 1:
y = y.ravel()
if input_type == "refseq_and_interval":
estimator.set_params(data_batch_generator__features=y.ravel().tolist())
y = None
# end y
# load groups
if groups:
groups_selector = (
params["experiment_schemes"]["test_split"]["split_algos"]
).pop("groups_selector")
header = "infer" if groups_selector["header_g"] else None
column_option = groups_selector["column_selector_options_g"][
"selected_column_selector_option_g"
]
if column_option in [
"by_index_number",
"all_but_by_index_number",
"by_header_name",
"all_but_by_header_name",
]:
c = groups_selector["column_selector_options_g"]["col_g"]
else:
c = None
df_key = groups + repr(header)
if df_key in loaded_df:
groups = loaded_df[df_key]
groups = read_columns(
groups,
c=c,
c_option=column_option,
sep="\t",
header=header,
parse_dates=True,
)
groups = groups.ravel()
# del loaded_df
del loaded_df
# cache iraps_core fits could increase search speed significantly
memory = joblib.Memory(location=CACHE_DIR, verbose=0)
main_est = get_main_estimator(estimator)
if main_est.__class__.__name__ == "IRAPSClassifier":
main_est.set_params(memory=memory)
# handle scorer, convert to scorer dict
scoring = params["experiment_schemes"]["metrics"]["scoring"]
if scoring is not None:
# get_scoring() expects secondary_scoring to be a comma separated string (not a list)
# Check if secondary_scoring is specified
secondary_scoring = scoring.get("secondary_scoring", None)
if secondary_scoring is not None:
# If secondary_scoring is specified, convert the list into comman separated string
scoring["secondary_scoring"] = ",".join(scoring["secondary_scoring"])
scorer = get_scoring(scoring)
scorer, _ = _check_multimetric_scoring(estimator, scoring=scorer)
# handle test (first) split
test_split_options = params["experiment_schemes"]["test_split"]["split_algos"]
if test_split_options["shuffle"] == "group":
test_split_options["labels"] = groups
if test_split_options["shuffle"] == "stratified":
if y is not None:
test_split_options["labels"] = y
else:
raise ValueError(
"Stratified shuffle split is not " "applicable on empty target values!"
)
(
X_train,
X_test,
y_train,
y_test,
groups_train,
_groups_test,
) = train_test_split_none(X, y, groups, **test_split_options)
exp_scheme = params["experiment_schemes"]["selected_exp_scheme"]
# handle validation (second) split
if exp_scheme == "train_val_test":
val_split_options = params["experiment_schemes"]["val_split"]["split_algos"]
if val_split_options["shuffle"] == "group":
val_split_options["labels"] = groups_train
if val_split_options["shuffle"] == "stratified":
if y_train is not None:
val_split_options["labels"] = y_train
else:
raise ValueError(
"Stratified shuffle split is not "
"applicable on empty target values!"
)
(
X_train,
X_val,
y_train,
y_val,
groups_train,
_groups_val,
) = train_test_split_none(X_train, y_train, groups_train, **val_split_options)
# train and eval
if hasattr(estimator, "validation_data"):
if exp_scheme == "train_val_test":
estimator.fit(X_train, y_train, validation_data=(X_val, y_val))
else:
estimator.fit(X_train, y_train, validation_data=(X_test, y_test))
else:
estimator.fit(X_train, y_train)
if hasattr(estimator, "evaluate"):
steps = estimator.prediction_steps
batch_size = estimator.batch_size
generator = estimator.data_generator_.flow(
X_test, y=y_test, batch_size=batch_size
)
predictions, y_true = _predict_generator(
estimator.model_, generator, steps=steps
)
scores = _evaluate(y_true, predictions, scorer, is_multimetric=True)
else:
if hasattr(estimator, "predict_proba"):
predictions = estimator.predict_proba(X_test)
else:
predictions = estimator.predict(X_test)
y_true = y_test
scores = _score(estimator, X_test, y_test, scorer, is_multimetric=True)
if outfile_y_true:
try:
pd.DataFrame(y_true).to_csv(outfile_y_true, sep="\t", index=False)
pd.DataFrame(predictions).astype(np.float32).to_csv(
outfile_y_preds,
sep="\t",
index=False,
float_format="%g",
chunksize=10000,
)
except Exception as e:
print("Error in saving predictions: %s" % e)
# handle output
for name, score in scores.items():
scores[name] = [score]
df = pd.DataFrame(scores)
df = df[sorted(df.columns)]
df.to_csv(path_or_buf=outfile_result, sep="\t", header=True, index=False)
memory.clear(warn=False)
if outfile_object:
main_est = estimator
if isinstance(estimator, Pipeline):
main_est = estimator.steps[-1][-1]
if hasattr(main_est, "model_") and hasattr(main_est, "save_weights"):
if outfile_weights:
main_est.save_weights(outfile_weights)
del main_est.model_
del main_est.fit_params
del main_est.model_class_
if getattr(main_est, "validation_data", None):
del main_est.validation_data
if getattr(main_est, "data_generator_", None):
del main_est.data_generator_
with open(outfile_object, "wb") as output_handler:
pickle.dump(estimator, output_handler, pickle.HIGHEST_PROTOCOL)
def main(inputs,
infile_estimator=None,
infile1=None,
infile2=None,
outfile_result=None,
outfile_object=None,
groups=None,
ref_seq=None,
intervals=None,
targets=None,
fasta_path=None,
model_config=None,
true_labels=None,
predicted_labels=None,
plot_color=None,
title=None):
"""
Parameter
---------
inputs : str
File path to galaxy tool parameter
infile_estimator : str, default is None
File path to estimator
infile1 : str, default is None
File path to dataset containing features or true labels.
infile2 : str, default is None
File path to dataset containing target values or predicted
probabilities.
outfile_result : str, default is None
File path to save the results, either cv_results or test result
outfile_object : str, default is None
File path to save searchCV object
groups : str, default is None
File path to dataset containing groups labels
ref_seq : str, default is None
File path to dataset containing genome sequence file
intervals : str, default is None
File path to dataset containing interval file
targets : str, default is None
File path to dataset compressed target bed file
fasta_path : str, default is None
File path to dataset containing fasta file
model_config : str, default is None
File path to dataset containing JSON config for neural networks
true_labels : str, default is None
File path to dataset containing true labels
predicted_labels : str, default is None
File path to dataset containing true predicted labels
plot_color : str, default is None
Color of the confusion matrix heatmap
title : str, default is None
Title of the confusion matrix heatmap
"""
warnings.simplefilter('ignore')
with open(inputs, 'r') as param_handler:
params = json.load(param_handler)
title = params['plotting_selection']['title'].strip()
plot_type = params['plotting_selection']['plot_type']
plot_format = params['plotting_selection']['plot_format']
if plot_type == 'feature_importances':
with open(infile_estimator, 'rb') as estimator_handler:
estimator = load_model(estimator_handler)
column_option = (
params['plotting_selection']['column_selector_options']
['selected_column_selector_option'])
if column_option in [
'by_index_number', 'all_but_by_index_number', 'by_header_name',
'all_but_by_header_name'
]:
c = (params['plotting_selection']['column_selector_options']
['col1'])
else:
c = None
_, input_df = read_columns(infile1,
c=c,
c_option=column_option,
return_df=True,
sep='\t',
header='infer',
parse_dates=True)
feature_names = input_df.columns.values
if isinstance(estimator, Pipeline):
for st in estimator.steps[:-1]:
if isinstance(st[-1], SelectorMixin):
mask = st[-1].get_support()
feature_names = feature_names[mask]
estimator = estimator.steps[-1][-1]
if hasattr(estimator, 'coef_'):
coefs = estimator.coef_
else:
coefs = getattr(estimator, 'feature_importances_', None)
if coefs is None:
raise RuntimeError('The classifier does not expose '
'"coef_" or "feature_importances_" '
'attributes')
threshold = params['plotting_selection']['threshold']
if threshold is not None:
mask = (coefs > threshold) | (coefs < -threshold)
coefs = coefs[mask]
feature_names = feature_names[mask]
# sort
indices = np.argsort(coefs)[::-1]
trace = go.Bar(x=feature_names[indices], y=coefs[indices])
layout = go.Layout(title=title or "Feature Importances")
fig = go.Figure(data=[trace], layout=layout)
plotly.offline.plot(fig, filename="output.html", auto_open=False)
# to be discovered by `from_work_dir`
os.rename('output.html', 'output')
return 0
elif plot_type in ('pr_curve', 'roc_curve'):
df1 = pd.read_csv(infile1, sep='\t', header='infer')
df2 = pd.read_csv(infile2, sep='\t', header='infer').astype(np.float32)
minimum = params['plotting_selection']['report_minimum_n_positives']
# filter out columns whose n_positives is beblow the threhold
if minimum:
mask = df1.sum(axis=0) >= minimum
df1 = df1.loc[:, mask]
df2 = df2.loc[:, mask]
pos_label = params['plotting_selection']['pos_label'].strip() \
or None
if plot_type == 'pr_curve':
if plot_format == 'plotly_html':
visualize_pr_curve_plotly(df1, df2, pos_label, title=title)
else:
visualize_pr_curve_matplotlib(df1, df2, pos_label, title)
else: # 'roc_curve'
drop_intermediate = (
params['plotting_selection']['drop_intermediate'])
if plot_format == 'plotly_html':
visualize_roc_curve_plotly(df1,
df2,
pos_label,
drop_intermediate=drop_intermediate,
title=title)
else:
visualize_roc_curve_matplotlib(
df1,
df2,
pos_label,
drop_intermediate=drop_intermediate,
title=title)
return 0
elif plot_type == 'rfecv_gridscores':
input_df = pd.read_csv(infile1, sep='\t', header='infer')
scores = input_df.iloc[:, 0]
steps = params['plotting_selection']['steps'].strip()
steps = safe_eval(steps)
data = go.Scatter(x=list(range(len(scores))),
y=scores,
text=[str(_) for _ in steps] if steps else None,
mode='lines')
layout = go.Layout(
xaxis=dict(title="Number of features selected"),
yaxis=dict(title="Cross validation score"),
title=dict(text=title or None,
x=0.5,
y=0.92,
xanchor='center',
yanchor='top'),
font=dict(family="sans-serif", size=11),
# control backgroud colors
plot_bgcolor='rgba(255,255,255,0)')
"""
# legend=dict(
# x=0.95,
# y=0,
# traceorder="normal",
# font=dict(
# family="sans-serif",
# size=9,
# color="black"
# ),
# bgcolor="LightSteelBlue",
# bordercolor="Black",
# borderwidth=2
# ),
"""
fig = go.Figure(data=[data], layout=layout)
plotly.offline.plot(fig, filename="output.html", auto_open=False)
# to be discovered by `from_work_dir`
os.rename('output.html', 'output')
return 0
elif plot_type == 'learning_curve':
input_df = pd.read_csv(infile1, sep='\t', header='infer')
plot_std_err = params['plotting_selection']['plot_std_err']
data1 = go.Scatter(
x=input_df['train_sizes_abs'],
y=input_df['mean_train_scores'],
error_y=dict(
array=input_df['std_train_scores']) if plot_std_err else None,
mode='lines',
name="Train Scores",
)
data2 = go.Scatter(
x=input_df['train_sizes_abs'],
y=input_df['mean_test_scores'],
error_y=dict(
array=input_df['std_test_scores']) if plot_std_err else None,
mode='lines',
name="Test Scores",
)
layout = dict(
xaxis=dict(title='No. of samples'),
yaxis=dict(title='Performance Score'),
# modify these configurations to customize image
title=dict(text=title or 'Learning Curve',
x=0.5,
y=0.92,
xanchor='center',
yanchor='top'),
font=dict(family="sans-serif", size=11),
# control backgroud colors
plot_bgcolor='rgba(255,255,255,0)')
"""
# legend=dict(
# x=0.95,
# y=0,
# traceorder="normal",
# font=dict(
# family="sans-serif",
# size=9,
# color="black"
# ),
# bgcolor="LightSteelBlue",
# bordercolor="Black",
# borderwidth=2
# ),
"""
fig = go.Figure(data=[data1, data2], layout=layout)
plotly.offline.plot(fig, filename="output.html", auto_open=False)
# to be discovered by `from_work_dir`
os.rename('output.html', 'output')
return 0
elif plot_type == 'keras_plot_model':
with open(model_config, 'r') as f:
model_str = f.read()
model = model_from_json(model_str)
plot_model(model, to_file="output.png")
os.rename('output.png', 'output')
return 0
elif plot_type == 'classification_confusion_matrix':
plot_selection = params["plotting_selection"]
input_true = get_dataframe(true_labels, plot_selection, "header_true",
"column_selector_options_true")
header_predicted = 'infer' if plot_selection[
"header_predicted"] else None
input_predicted = pd.read_csv(predicted_labels,
sep='\t',
parse_dates=True,
header=header_predicted)
true_classes = input_true.iloc[:, -1].copy()
predicted_classes = input_predicted.iloc[:, -1].copy()
axis_labels = list(set(true_classes))
c_matrix = confusion_matrix(true_classes, predicted_classes)
fig, ax = plt.subplots(figsize=(7, 7))
im = plt.imshow(c_matrix, cmap=plot_color)
for i in range(len(c_matrix)):
for j in range(len(c_matrix)):
ax.text(j,
i,
c_matrix[i, j],
ha="center",
va="center",
color="k")
ax.set_ylabel('True class labels')
ax.set_xlabel('Predicted class labels')
ax.set_title(title)
ax.set_xticks(axis_labels)
ax.set_yticks(axis_labels)
fig.colorbar(im, ax=ax)
fig.tight_layout()
plt.savefig("output.png", dpi=125)
os.rename('output.png', 'output')
return 0
def main():
parser = argparse.ArgumentParser(description="RDKit screen")
parser.add_argument("-i", "--infile", help="Input file")
parser.add_argument("--header",
action="store_true",
help="Include the header row or skip it")
parser.add_argument(
"-c",
"--columns",
type=str.lower,
default="all",
choices=[
"by_index_number",
"all_but_by_index_number",
"by_header_name",
"all_but_by_header_name",
"all_columns",
],
help="Choose to select all columns, or exclude/include some",
)
parser.add_argument(
"-ci",
"--column_indices",
type=str.lower,
help="Choose to select all columns, or exclude/include some",
)
parser.add_argument(
"-n",
"--number",
nargs="?",
type=int,
default=None,
help=
"Number of components to keep. If not set, all components are kept",
)
parser.add_argument("--whiten",
action="store_true",
help="Whiten the components")
parser.add_argument(
"-t",
"--pca_type",
type=str.lower,
default="classical",
choices=["classical", "incremental", "kernel"],
help="Choose which flavour of PCA to use",
)
parser.add_argument(
"-s",
"--svd_solver",
type=str.lower,
default="auto",
choices=["auto", "full", "arpack", "randomized"],
help="Choose the type of svd solver.",
)
parser.add_argument(
"-b",
"--batch_size",
nargs="?",
type=int,
default=None,
help="The number of samples to use for each batch",
)
parser.add_argument(
"-k",
"--kernel",
type=str.lower,
default="linear",
choices=["linear", "poly", "rbf", "sigmoid", "cosine", "precomputed"],
help="Choose the type of kernel.",
)
parser.add_argument(
"-g",
"--gamma",
nargs="?",
type=float,
default=None,
help=
"Kernel coefficient for rbf, poly and sigmoid kernels. Ignored by other kernels",
)
parser.add_argument(
"-tol",
"--tolerance",
type=float,
default=0.0,
help=
"Convergence tolerance for arpack. If 0, optimal value will be chosen by arpack",
)
parser.add_argument(
"-mi",
"--max_iter",
nargs="?",
type=int,
default=None,
help="Maximum number of iterations for arpack",
)
parser.add_argument(
"-d",
"--degree",
type=int,
default=3,
help="Degree for poly kernels. Ignored by other kernels",
)
parser.add_argument(
"-cf",
"--coef0",
type=float,
default=1.0,
help="Independent term in poly and sigmoid kernels",
)
parser.add_argument(
"-e",
"--eigen_solver",
type=str.lower,
default="auto",
choices=["auto", "dense", "arpack"],
help="Choose the type of eigen solver.",
)
parser.add_argument("-o",
"--outfile",
help="Base name for output file (no extension).")
args = parser.parse_args()
usecols = None
pca_params = {}
if args.columns == "by_index_number" or args.columns == "all_but_by_index_number":
usecols = [int(i) for i in args.column_indices.split(",")]
elif args.columns == "by_header_name" or args.columns == "all_but_by_header_name":
usecols = args.column_indices
header = "infer" if args.header else None
pca_input = read_columns(
f=args.infile,
c=usecols,
c_option=args.columns,
sep="\t",
header=header,
parse_dates=True,
encoding=None,
index_col=None,
)
pca_params.update({"n_components": args.number})
if args.pca_type == "classical":
pca_params.update({
"svd_solver": args.svd_solver,
"whiten": args.whiten
})
if args.svd_solver == "arpack":
pca_params.update({"tol": args.tolerance})
pca = PCA()
elif args.pca_type == "incremental":
pca_params.update({
"batch_size": args.batch_size,
"whiten": args.whiten
})
pca = IncrementalPCA()
elif args.pca_type == "kernel":
pca_params.update({
"kernel": args.kernel,
"eigen_solver": args.eigen_solver,
"gamma": args.gamma,
})
if args.kernel == "poly":
pca_params.update({"degree": args.degree, "coef0": args.coef0})
elif args.kernel == "sigmoid":
pca_params.update({"coef0": args.coef0})
elif args.kernel == "precomputed":
pca_input = np.dot(pca_input, pca_input.T)
if args.eigen_solver == "arpack":
pca_params.update({
"tol": args.tolerance,
"max_iter": args.max_iter
})
pca = KernelPCA()
print(pca_params)
pca.set_params(**pca_params)
pca_output = pca.fit_transform(pca_input)
np.savetxt(fname=args.outfile, X=pca_output, fmt="%.4f", delimiter="\t")
