def predict_with_existing(self,
X_train, y_train_regression, y_train_classification,
X_val, y_val_regression, y_val_classification,
X_test, y_test_regression, y_test_classification,
PMTNN_weight_file):
model = self.setup_model_ensemble()
model.load_weights(PMTNN_weight_file)
y_pred_on_train = model.predict(X_train)
y_pred_on_val = model.predict(X_val)
y_pred_on_test = model.predict(X_test)
print
print('train precision: {}'.format(precision_auc_single(y_train_classification, y_pred_on_train)))
print('train roc: {}'.format(roc_auc_single(y_train_classification, y_pred_on_train)))
print('train bedroc: {}'.format(bedroc_auc_single(y_train_classification, y_pred_on_train)))
print
print('validation precision: {}'.format(precision_auc_single(y_val_classification, y_pred_on_val)))
print('validation roc: {}'.format(roc_auc_single(y_val_classification, y_pred_on_val)))
print('validation bedroc: {}'.format(bedroc_auc_single(y_val_classification, y_pred_on_val)))
print
print('test precision: {}'.format(precision_auc_single(y_test_classification, y_pred_on_test)))
print('test roc: {}'.format(roc_auc_single(y_test_classification, y_pred_on_test)))
print('test bedroc: {}'.format(bedroc_auc_single(y_test_classification, y_pred_on_test)))
print
for EF_ratio in self.EF_ratio_list:
n_actives, ef, ef_max = enrichment_factor_single(y_test_classification, y_pred_on_test, EF_ratio)
print('ratio: {}, EF: {},\tactive: {}'.format(EF_ratio, ef, n_actives))
return
def enrichement_factor_fetcher(y_test, y_pred_on_test, EF_ratio_list):
ef_values = []
ef_max_values = []
for EF_ratio in EF_ratio_list:
n_actives, ef, ef_max = enrichment_factor_single(y_test, y_pred_on_test, EF_ratio)
ef_values.append(ef)
ef_max_values.append(ef_max)
return ef_values, ef_max_values
def get_ef_table(file_path, target_name, efr_list, ef_header, title):
"""
:param file_path: Docking results
:param efr_list: EF ratio list
:param ef_header: Table header
:param title: Markdown Table caption
:return: the markdown content
example run: get_ef_table(file_path='../../output/docking_result/lc123-pria-dockdata-qnorm.csv.gz',
target_name='Keck_Pria_AS_Retest',
efr_list=[0.02, 0.01, 0.0015, 0.001],
ef_header=['EF_2', 'EF_1', 'EF_015', 'EF_01'],
title='Enrichment Factor for Docking Methods')
"""
pria_pd = pd.read_csv(file_path)
title = '## {}'.format(title)
header = '| docking method |'
for name in ef_header:
header = '{} {} |'.format(header, name)
splitter = '| --- |'
for _ in efr_list:
splitter = '{} {} |'.format(splitter, '---')
if target_name == 'Keck_Pria_AS_Retest':
ground = '../../output/docking/stage_1/lc123-pria-dockdata-qnorm.csv.gz'
elif target_name == 'Keck_Pria_FP_data':
ground = '../../output/docking/stage_1/lc123-pria-dockdata-qnorm.csv.gz'
elif target_name == 'Keck_RMI_cdd':
ground = '../../output/docking/stage_1/lc123-rmi-dockdata-qnorm.csv.gz'
else:
raise ValueError('Target name {} not found.'.format(target_name))
ground_pd = pd.read_csv(ground)
ground_pd = ground_pd[['Unnamed: 0', target_name]]
ground_pd.columns = ['molid', target_name]
pria_pd = pd.merge(pria_pd, ground_pd, on='molid', how='outer')
content = ''
for docking_method in docking_methods:
# temp_pd = pria_pd[['Unnamed: 0', target_name, docking_method]]
temp_pd = pria_pd[['molid', target_name, docking_method]]
filtered_pd = temp_pd.dropna()
# TODO: may find the difference with panda.series for EF calculation
# true_label_list = filtered_pd[target_name]
# docking_ranked_list = filtered_pd[docking_method]
true_label_list = np.array(filtered_pd[target_name].tolist())
docking_ranked_list = np.array(filtered_pd[docking_method].tolist())
row = '| {} |'.format(docking_method)
for ratio in efr_list:
n_actives, ef, ef_max = enrichment_factor_single(
true_label_list, docking_ranked_list, ratio)
row = '{} {} |'.format(row, ef)
content = '{}{}\n'.format(content, row)
content = '{}\n{}\n{}\n{}'.format(title, header, splitter, content)
return content
def get_EF_score_with_existing_model(self, X_test, y_test, file_path,
EF_ratio):
model = self.setup_model()
model.load_weights(file_path)
y_pred_on_test = model.predict(X_test)
n_actives, ef, ef_max = enrichment_factor_single(
y_test, y_pred_on_test, EF_ratio)
print('test precision: {}'.format(
get_model_precision_auc(y_test, y_pred_on_test)))
print('test auc: {}'.format(get_model_roc_auc(y_test, y_pred_on_test)))
print('EF: {},\tactive: {}'.format(ef, n_actives))
return
def get_EF_score_with_existing_model(self,
X_test, y_test, y_test_classification,
file_path, EF_ratio):
model = self.setup_model_ensemble()
model.load_weights(file_path)
y_pred_on_test = model.predict(X_test)
print('test precision: {}'.format(precision_auc_single(y_test_classification, y_pred_on_test)))
print('test roc: {}'.format(roc_auc_single(y_test_classification, y_pred_on_test)))
print('test bedroc: {}'.format(bedroc_auc_single(y_test_classification, y_pred_on_test)))
print
n_actives, ef, ef_max = enrichment_factor_single(y_test_classification, y_pred_on_test, EF_ratio)
print('EF: {},\tactive: {}'.format(ef, n_actives))
return
def predict_with_existing(self, X_train, y_train, X_val, y_val, X_test,
y_test, PMTNN_weight_file):
model = self.setup_model()
model.load_weights(PMTNN_weight_file)
y_pred_on_train = model.predict(X_train)[:, -1]
y_train = y_train[:, -1]
y_pred_on_val = model.predict(X_val)[:, -1]
y_val = y_val[:, -1]
if X_test is not None:
y_pred_on_test = model.predict(X_test)[:, -1]
y_test = y_test[:, -1]
print y_train.shape, '\t', y_pred_on_test.shape
print
print('train precision: {}'.format(
precision_auc_single(y_train, y_pred_on_train)))
print('train roc: {}'.format(roc_auc_single(y_train, y_pred_on_train)))
print('train bedroc: {}'.format(
bedroc_auc_single(y_train, y_pred_on_train)))
print
print('validation precision: {}'.format(
precision_auc_single(y_val, y_pred_on_val)))
print('validation roc: {}'.format(roc_auc_single(y_val,
y_pred_on_val)))
print('validation bedroc: {}'.format(
bedroc_auc_single(y_val, y_pred_on_val)))
print
if X_test is not None:
print('test precision: {}'.format(
precision_auc_single(y_test, y_pred_on_test)))
print('test roc: {}'.format(roc_auc_single(y_test,
y_pred_on_test)))
print('test bedroc: {}'.format(
bedroc_auc_single(y_test, y_pred_on_test)))
print
if X_test is not None:
for EF_ratio in self.EF_ratio_list:
n_actives, ef, ef_max = enrichment_factor_single(
y_test, y_pred_on_test, EF_ratio)
nef = ef / ef_max
print('ratio: {}, EF: {},\tactive: {}'.format(
EF_ratio, ef, n_actives))
print('ratio: {}, NEF: {}'.format(EF_ratio, nef))
return
def train_and_predict(self,
X_train, y_train_regression, y_train_classification,
X_val, y_val_regression, y_val_classification,
X_test, y_test_regression, y_test_classification,
PMTNN_weight_file):
model = self.setup_model()
sw = get_sample_weight(self, y_train_regression)
print 'Sample Weight\t', sw
model.compile(loss=self.compile_loss, optimizer=self.compile_optimizer)
model.fit(x=X_train, y=y_train_regression,
nb_epoch=self.fit_nb_epoch,
batch_size=self.fit_batch_size,
verbose=self.fit_verbose,
sample_weight=sw,
validation_data=[X_val, y_val_regression],
shuffle=True)
model.save_weights(PMTNN_weight_file)
y_pred_on_train = model.predict(X_train)
y_pred_on_val = model.predict(X_val)
if X_test is not None:
y_pred_on_test = model.predict(X_test)
print
print('train precision: {}'.format(precision_auc_single(y_train_classification, y_pred_on_train)))
print('train roc: {}'.format(roc_auc_single(y_train_classification, y_pred_on_train)))
print('train bedroc: {}'.format(bedroc_auc_single(y_train_classification, y_pred_on_train)))
print
print('validation precision: {}'.format(precision_auc_single(y_val_classification, y_pred_on_val)))
print('validation roc: {}'.format(roc_auc_single(y_val_classification, y_pred_on_val)))
print('validation bedroc: {}'.format(bedroc_auc_single(y_val_classification, y_pred_on_val)))
print
if X_test is not None:
print('test precision: {}'.format(precision_auc_single(y_test_classification, y_pred_on_test)))
print('test roc: {}'.format(roc_auc_single(y_test_classification, y_pred_on_test)))
print('test bedroc: {}'.format(bedroc_auc_single(y_test_classification, y_pred_on_test)))
print
if X_test is not None:
for EF_ratio in self.EF_ratio_list:
n_actives, ef, ef_max = enrichment_factor_single(y_test_classification, y_pred_on_test, EF_ratio)
nef = ef / ef_max
print('ratio: {}, EF: {},\tactive: {}'.format(EF_ratio, ef, n_actives))
print('ratio: {}, NEF: {}'.format(EF_ratio, nef))
return
def get_EF_values_single_task(task, X_test, y_test, model_weight,
EF_ratio_list):
model = task.setup_model()
model.load_weights(model_weight)
y_pred_on_test = model.predict(X_test)
print('test precision: {}'.format(
precision_auc_single(y_test, y_pred_on_test)))
print('test roc: {}'.format(roc_auc_single(y_test, y_pred_on_test)))
print('test bedroc: {}'.format(bedroc_auc_single(y_test, y_pred_on_test)))
print
ef_values = []
ef_max_values = []
for EF_ratio in EF_ratio_list:
n_actives, ef, ef_max = enrichment_factor_single(
y_test, y_pred_on_test, EF_ratio)
ef_values.append(ef)
ef_max_values.append(ef_max)
return ef_values, ef_max_values
def get_rf(self, X_train, y_train, X_val, y_val, X_test, y_test):
max_features = 'log2'
n_estimators = 4000
min_samples_leaf = 1
class_weight = 'balanced'
rnd_state = 1337
np.random.seed(seed=rnd_state)
rf = RandomForestClassifier(n_estimators=n_estimators,
max_features=max_features,
min_samples_leaf=min_samples_leaf,
n_jobs=3,
class_weight=class_weight,
random_state=rnd_state,
oob_score=False,
verbose=1)
rf.fit(X_train, y_train)
y_pred_on_train = reshape_data_into_2_dim(rf.predict(X_train))
y_pred_on_val = reshape_data_into_2_dim(rf.predict(X_val))
y_pred_on_test = reshape_data_into_2_dim(rf.predict(X_test))
print('train precision: {}'.format(precision_auc_single(y_train, y_pred_on_train)))
print('train roc: {}'.format(roc_auc_single(y_train, y_pred_on_train)))
print('train bedroc: {}'.format(bedroc_auc_single(y_train, y_pred_on_train)))
print
print('validation precision: {}'.format(precision_auc_single(y_val, y_pred_on_val)))
print('validation roc: {}'.format(roc_auc_single(y_val, y_pred_on_val)))
print('validation bedroc: {}'.format(bedroc_auc_single(y_val, y_pred_on_val)))
print
print('test precision: {}'.format(precision_auc_single(y_test, y_pred_on_test)))
print('test roc: {}'.format(roc_auc_single(y_test, y_pred_on_test)))
print('test bedroc: {}'.format(bedroc_auc_single(y_test, y_pred_on_test)))
print
for EF_ratio in self.EF_ratio_list:
n_actives, ef, ef_max = enrichment_factor_single(y_test, y_pred_on_test, EF_ratio)
print('ratio: {}, EF: {},\tactive: {}'.format(EF_ratio, ef, n_actives))
return rf
def train_and_predict(self,
X_train, y_train, y_train_classification,
X_val, y_val, y_val_classification,
X_test, y_test, y_test_classification,
mode):
model = Sequential()
conf = self.conf
batch_normalizer_epsilon = conf['batch']['epsilon']
batch_normalizer_mode = conf['batch']['mode']
batch_normalizer_axis = conf['batch']['axis']
batch_normalizer_momentum = conf['batch']['momentum']
batch_normalizer_weights = conf['batch']['weights']
batch_normalizer_beta_init = conf['batch']['beta_init']
batch_normalizer_gamma_init = conf['batch']['gamma_init']
batch_normalizer = BatchNormalization(epsilon=batch_normalizer_epsilon,
mode=batch_normalizer_mode,
axis=batch_normalizer_axis,
momentum=batch_normalizer_momentum,
weights=batch_normalizer_weights,
beta_init=batch_normalizer_beta_init,
gamma_init=batch_normalizer_gamma_init)
if mode == 'classification':
model.add(Dense(2048, input_dim=1024, init='glorot_normal', activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1024, init='glorot_normal', activation='relu'))
model.add(Dropout(0.5))
if self.batch_is_use:
model.add(batch_normalizer)
model.add(Dense(1, init='glorot_normal', activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam')
else:
model.add(Dense(2048, input_dim=1024, init='glorot_normal', activation='sigmoid'))
model.add(Dropout(0.5))
model.add(Dense(1024, init='glorot_normal', activation='sigmoid'))
model.add(Dropout(0.5))
if self.batch_is_use:
model.add(batch_normalizer)
model.add(Dense(1, init='glorot_normal', activation='linear'))
model.compile(loss='mse', optimizer='adam')
model.fit(X_train, y_train,
batch_size=self.fit_batch_size,
nb_epoch=self.fit_nb_epoch,
verbose=self.fit_verbose,
validation_data=(X_val, y_val))
y_pred_on_train = model.predict(X_train)
y_pred_on_val = model.predict(X_val)
y_pred_on_test = model.predict(X_test)
print
print 'this is mode ', mode
print('train precision: {}'.format(precision_auc_single(y_train_classification, y_pred_on_train)))
print('train roc: {}'.format(roc_auc_single(y_train_classification, y_pred_on_train)))
print('train bedroc: {}'.format(bedroc_auc_single(y_train_classification, y_pred_on_train)))
print
print('validation precision: {}'.format(precision_auc_single(y_val_classification, y_pred_on_val)))
print('validation roc: {}'.format(roc_auc_single(y_val_classification, y_pred_on_val)))
print('validation bedroc: {}'.format(bedroc_auc_single(y_val_classification, y_pred_on_val)))
print
print('test precision: {}'.format(precision_auc_single(y_test_classification, y_pred_on_test)))
print('test roc: {}'.format(roc_auc_single(y_test_classification, y_pred_on_test)))
print('test bedroc: {}'.format(bedroc_auc_single(y_test_classification, y_pred_on_test)))
print
for EF_ratio in self.EF_ratio_list:
n_actives, ef, ef_max = enrichment_factor_single(y_test_classification, y_pred_on_test, EF_ratio)
print('ratio: {}, EF: {},\tactive: {}'.format(EF_ratio, ef, n_actives))
return y_pred_on_test
def train_and_predict_ensemble(self,
X_train, y_train_regression, y_train_classification,
X_val, y_val_regression, y_val_classification,
X_test, y_test_regression, y_test_classification,
PMTNN_weight_file):
model = self.setup_model_ensemble()
# TODO: remove
print model.summary()
if self.weight_schema == 'weighted':
loss_weight = {'classification_output_layer': 1., 'regression_output_layer': 100.}
elif self.weight_schema == 'no_weight':
loss_weight = {'classification_output_layer': 1., 'regression_output_layer': 1.}
else:
raise ValueError('Wrong weight schema. Should be no_weight, or weighted.')
model.compile(optimizer=self.compile_optimizer,
loss={'classification_output_layer': 'binary_crossentropy', 'regression_output_layer': 'mse'},
loss_weights=loss_weight)
model.fit({'input_layer': X_train},
{'classification_output_layer': y_train_classification,
'regression_output_layer': y_train_regression},
nb_epoch=self.fit_nb_epoch,
batch_size=self.fit_batch_size,
verbose=self.fit_verbose,
validation_data=({'input_layer': X_val},
{'classification_output_layer': y_val_classification,
'regression_output_layer': y_val_regression}),
shuffle=True)
model.save_weights(PMTNN_weight_file)
y_pred_on_train_ensemble = np.array(model.predict(X_train))
y_pred_on_val_ensmble = np.array(model.predict(X_val))
y_pred_on_test_ensemble = np.array(model.predict(X_test))
print
print 'TreeNet Ensemble'
mode_list = ['TreeNet classification', 'TreeNet regression']
for mode in range(2):
print
print mode_list[mode]
y_pred_on_train = y_pred_on_train_ensemble[mode]
y_pred_on_val = y_pred_on_val_ensmble[mode]
y_pred_on_test = y_pred_on_test_ensemble[mode]
print('train precision: {}'.format(precision_auc_single(y_train_classification, y_pred_on_train)))
print('train roc: {}'.format(roc_auc_single(y_train_classification, y_pred_on_train)))
print('train bedroc: {}'.format(bedroc_auc_single(y_train_classification, y_pred_on_train)))
print
print('validation precision: {}'.format(precision_auc_single(y_val_classification, y_pred_on_val)))
print('validation roc: {}'.format(roc_auc_single(y_val_classification, y_pred_on_val)))
print('validation bedroc: {}'.format(bedroc_auc_single(y_val_classification, y_pred_on_val)))
print
print('test precision: {}'.format(precision_auc_single(y_test_classification, y_pred_on_test)))
print('test roc: {}'.format(roc_auc_single(y_test_classification, y_pred_on_test)))
print('test bedroc: {}'.format(bedroc_auc_single(y_test_classification, y_pred_on_test)))
print
for EF_ratio in self.EF_ratio_list:
n_actives, ef, ef_max = enrichment_factor_single(y_test_classification, y_pred_on_test, EF_ratio)
print('ratio: {}, EF: {},\tactive: {}'.format(EF_ratio, ef, n_actives))
return y_pred_on_test_ensemble
def train_and_predict(self,
X_train,
y_train,
X_val,
y_val,
X_test,
y_test,
PMTNN_weight_file,
score_file,
sample_weight,
eval_indices=[-1],
eval_mean_or_median=np.mean):
def get_model_roc_auc(true_label,
predicted_label,
eval_indices=eval_indices,
eval_mean_or_median=eval_mean_or_median):
return roc_auc_multi(true_label, predicted_label, eval_indices,
eval_mean_or_median)
def get_model_bedroc_auc(true_label,
predicted_label,
eval_indices=eval_indices,
eval_mean_or_median=eval_mean_or_median):
return bedroc_auc_multi(true_label, predicted_label, eval_indices,
eval_mean_or_median)
def get_model_precision_auc(true_label,
predicted_label,
eval_indices=eval_indices,
eval_mean_or_median=eval_mean_or_median):
return precision_auc_multi(true_label, predicted_label,
eval_indices, eval_mean_or_median)
model = self.setup_model()
if self.early_stopping_option == 'auc':
early_stopping = MultiCallBackOnROC(
X_train,
y_train,
X_val,
y_val,
eval_indices,
eval_mean_or_median,
patience=self.early_stopping_patience,
file_path=PMTNN_weight_file)
callbacks = [early_stopping]
elif self.early_stopping_option == 'precision':
early_stopping = MultiCallBackOnPR(
X_train,
y_train,
X_val,
y_val,
eval_indices,
eval_mean_or_median,
patience=self.early_stopping_patience,
file_path=PMTNN_weight_file)
callbacks = [early_stopping]
else:
callbacks = []
if self.weight_schema == 'no_weight':
cw = get_class_weight(self, y_train)
print 'cw ', cw
model.compile(loss=self.compile_loss,
optimizer=self.compile_optimizer)
model.fit(X_train,
y_train,
nb_epoch=self.fit_nb_epoch,
batch_size=self.fit_batch_size,
verbose=self.fit_verbose,
class_weight=cw,
shuffle=True,
callbacks=callbacks)
else:
reference = []
total_num = X_train.shape[0]
for i in range(self.output_layer_dimension):
active = sum(y_train[:, i])
active_and_inactive = sum(sample_weight[:, i])
inactive = active_and_inactive - active
missing = total_num - active_and_inactive
reference.append({'-1': missing, '0': inactive, '1': active})
cw = get_class_weight(self, y_train, reference=reference)
print 'cw ', cw
# TODO: customize loss function
def customized_loss(y_true, y_pred):
import theano.tensor as T
y_pred = T.clip(y_pred, epsilon, 1.0 - epsilon)
sum_ = 0
print y_true.shape
# T.nnet.binary_crossentropy(x_recons, x).mean()
bce = T.nnet.binary_crossentropy(y_pred, y_true).mean(axis=-1)
return bce
# model.compile(loss=self.compile_loss, optimizer=self.compile_optimizer, sample_weight_mode="temporal")
model.compile(loss=self.compile_loss,
optimizer=self.compile_optimizer)
model.fit(X_train,
y_train,
nb_epoch=self.fit_nb_epoch,
batch_size=self.fit_batch_size,
verbose=self.fit_verbose,
class_weight=cw,
shuffle=True,
callbacks=callbacks)
if self.early_stopping_option == 'auc' or self.early_stopping_option == 'precision':
model = early_stopping.get_best_model()
y_pred_on_train = model.predict(X_train)
y_pred_on_val = model.predict(X_val)
if X_test is not None:
y_pred_on_test = model.predict(X_test)
print('train precision: {}'.format(
get_model_precision_auc(y_train, y_pred_on_train)))
print('train roc: {}'.format(
get_model_roc_auc(y_train, y_pred_on_train)))
print('train bedroc: {}'.format(
get_model_bedroc_auc(y_train, y_pred_on_train)))
print
print('validation precision: {}'.format(
get_model_precision_auc(y_val, y_pred_on_val)))
print('validation roc: {}'.format(
get_model_roc_auc(y_val, y_pred_on_val)))
print('validation bedroc: {}'.format(
get_model_bedroc_auc(y_val, y_pred_on_val)))
print
if X_test is not None:
print('test precision: {}'.format(
get_model_precision_auc(y_test, y_pred_on_test)))
print('test roc: {}'.format(
get_model_roc_auc(y_test, y_pred_on_test)))
print('test bedroc: {}'.format(
get_model_bedroc_auc(y_test, y_pred_on_test)))
print
if X_test is not None:
# Just print last target EF into output file.
for EF_ratio in self.EF_ratio_list:
n_actives, ef, ef_max = enrichment_factor_single(
y_test[:, -1], y_pred_on_test[:, -1], EF_ratio)
nef = ef / ef_max
print('ratio: {}, EF: {},\tactive: {}'.format(
EF_ratio, ef, n_actives))
print('ratio: {}, NEF: {}'.format(EF_ratio, nef))
# Store all the target EF into score file.
out = open(score_file, 'w')
print >> out, "EF"
for EF_ratio in self.EF_ratio_list:
for i in range(y_test.shape[1]):
n_actives, ef, ef_max = enrichment_factor_single(
y_test[:, i], y_pred_on_test[:, i], EF_ratio)
nef = ef / ef_max
print >> out, 'ratio:', EF_ratio, 'EF:', ef, 'active:', n_actives
print >> out, 'ratio:', EF_ratio, 'EF:', nef
return
def train_and_predict(self, X_train, y_train, X_val, y_val, X_test, y_test,
PMTNN_weight_file):
model = self.setup_model()
if self.early_stopping_option == 'auc':
early_stopping = KeckCallBackOnROC(
X_train,
y_train,
X_val,
y_val,
patience=self.early_stopping_patience,
file_path=PMTNN_weight_file)
callbacks = [early_stopping]
elif self.early_stopping_option == 'precision':
early_stopping = KeckCallBackOnPrecision(
X_train,
y_train,
X_val,
y_val,
patience=self.early_stopping_patience,
file_path=PMTNN_weight_file)
callbacks = [early_stopping]
else:
callbacks = []
cw = get_class_weight(self, y_train)
print 'cw ', cw
model.compile(loss=self.compile_loss, optimizer=self.compile_optimizer)
model.fit(X_train,
y_train,
nb_epoch=self.fit_nb_epoch,
batch_size=self.fit_batch_size,
verbose=self.fit_verbose,
class_weight=cw,
shuffle=True,
callbacks=callbacks)
if self.early_stopping_option == 'auc' or self.early_stopping_option == 'precision':
model = early_stopping.get_best_model()
y_pred_on_train = model.predict(X_train)
y_pred_on_val = model.predict(X_val)
if X_test is not None:
y_pred_on_test = model.predict(X_test)
print
print('train precision: {}'.format(
precision_auc_single(y_train, y_pred_on_train)))
print('train roc: {}'.format(roc_auc_single(y_train, y_pred_on_train)))
print('train bedroc: {}'.format(
bedroc_auc_single(y_train, y_pred_on_train)))
print
print('validation precision: {}'.format(
precision_auc_single(y_val, y_pred_on_val)))
print('validation roc: {}'.format(roc_auc_single(y_val,
y_pred_on_val)))
print('validation bedroc: {}'.format(
bedroc_auc_single(y_val, y_pred_on_val)))
print
if X_test is not None:
print('test precision: {}'.format(
precision_auc_single(y_test, y_pred_on_test)))
print('test roc: {}'.format(roc_auc_single(y_test,
y_pred_on_test)))
print('test bedroc: {}'.format(
bedroc_auc_single(y_test, y_pred_on_test)))
print
if X_test is not None:
for EF_ratio in self.EF_ratio_list:
n_actives, ef, ef_max = enrichment_factor_single(
y_test, y_pred_on_test, EF_ratio)
nef = ef / ef_max
print('ratio: {}, EF: {},\tactive: {}'.format(
EF_ratio, ef, n_actives))
print('ratio: {}, NEF: {}'.format(EF_ratio, nef))
return