def plot_single_SVM(prediction,
mutation_data,
label_type,
show_plots=False,
show_ROC=False):
if type(prediction) is not pd.core.frame.DataFrame:
if os.path.isfile(prediction):
prediction = pd.read_hdf(prediction)
keys = prediction.keys()
SVMs = list()
label = keys[0]
SVMs = prediction[label]['classifiers']
Y_test = prediction[label]['Y_test']
X_test = prediction[label]['X_test']
Y_train = prediction[label]['X_train']
Y_score = list()
# print(len(X_test[0][0]))
# print(config)
# X_train = data2['19q']['X_train']
# Y_train = data2['19q']['Y_train']
# mutation_data = gp.load_mutation_status(patientinfo, [[label]])
if type(mutation_data) is not dict:
if os.path.isfile(mutation_data):
mutation_data = gp.load_mutation_status(mutation_data,
[[label_type]])
patient_IDs = mutation_data['patient_IDs']
mutation_label = mutation_data['mutation_label']
# mutation_name = mutation_data['mutation_name']
# print(len(SVMs))
N_iterations = float(len(SVMs))
# mutation_label = np.asarray(mutation_label)
sensitivity = list()
specificity = list()
precision = list()
accuracy = list()
auc = list()
# auc_train = list()
f1_score_list = list()
patient_classification_list = dict()
for i in range(0, len(Y_test)):
# print(Y_test[i])
# if Y_test[i].shape[1] > 1:
# # print(Y_test[i])
# y_truth = np.prod(Y_test[i][:, 0:2], axis=1)
# else:
# y_truth_test = Y_test[i]
test_patient_IDs = prediction[label]['patient_ID_test'][i]
if 'LGG-Radiogenomics-046' in test_patient_IDs:
wrong_index = np.where(test_patient_IDs == 'LGG-Radiogenomics-046')
test_patient_IDs = np.delete(test_patient_IDs, wrong_index)
X_temp = X_test[i]
print(X_temp.shape)
X_temp = np.delete(X_test[i], wrong_index, axis=0)
print(X_temp.shape)
# X_test.pop(wrong_index[0])
# print(len(X_test))
else:
X_temp = X_test[i]
test_indices = list()
for i_ID in test_patient_IDs:
test_indices.append(np.where(patient_IDs == i_ID)[0][0])
if i_ID not in patient_classification_list:
patient_classification_list[i_ID] = dict()
patient_classification_list[i_ID]['N_test'] = 0
patient_classification_list[i_ID]['N_correct'] = 0
patient_classification_list[i_ID]['N_wrong'] = 0
patient_classification_list[i_ID]['N_test'] += 1
y_truth = [mutation_label[0][k] for k in test_indices]
# print(y_truth)
# print(y_truth_test)
# print(test_patient_IDs)
y_predict_1 = SVMs[i].predict(X_temp)
# print(y_predict_1).shape
y_prediction = y_predict_1
# y_prediction = np.prod(y_prediction, axis=0)
print "Truth: ", y_truth
print "Prediction: ", y_prediction
for i_truth, i_predict, i_test_ID in zip(y_truth, y_prediction,
test_patient_IDs):
if i_truth == i_predict:
patient_classification_list[i_test_ID]['N_correct'] += 1
else:
patient_classification_list[i_test_ID]['N_wrong'] += 1
# print('bla')
# print(y_truth)
# print(y_prediction)
c_mat = confusion_matrix(y_truth, y_prediction)
TN = c_mat[0, 0]
FN = c_mat[1, 0]
TP = c_mat[1, 1]
FP = c_mat[0, 1]
if FN == 0 and TP == 0:
sensitivity.append(0)
else:
sensitivity.append(float(TP) / (TP + FN))
if FP == 0 and TN == 0:
specificity.append(0)
else:
specificity.append(float(TN) / (FP + TN))
if TP == 0 and FP == 0:
precision.append(0)
else:
precision.append(float(TP) / (TP + FP))
accuracy.append(accuracy_score(y_truth, y_prediction))
y_score = SVMs[i].decision_function(X_temp)
Y_score.append(y_score)
auc.append(roc_auc_score(y_truth, y_score))
f1_score_list.append(
f1_score(y_truth, y_prediction, average='weighted'))
# if show_ROC:
# ROC_target_folder = '/archive/wkessels/output/ROC_temp/'
# if not os.path.exists(ROC_target_folder):
# os.makedirs(ROC_target_folder)
#
# luck = [0, 1]
#
# fpr, tpr, _ = roc_curve(y_truth, y_score)
# plt.figure()
# plt.plot(fpr, tpr, color='blue', label='ROC (AUC = {})'.format(auc[-1]))
# plt.plot(luck, luck, '--', color='red', label='luck')
# plt.xlabel('1-specificity')
# plt.ylabel('sensitivity')
# plt.axis([0, 1, 0, 1])
# plt.legend()
# plt.savefig(ROC_target_folder + 'ROC_cv{}.png'.format(i))
# print('Saved ROC figure in {}!'.format(ROC_target_folder))
# Adjusted according to "Inference for the Generelization error"
accuracy_mean = np.mean(accuracy)
S_uj = 1.0 / max((N_iterations - 1), 1) * np.sum(
(accuracy_mean - accuracy)**2.0)
print Y_test
N_1 = float(len(Y_train[0]))
N_2 = float(len(Y_test[0]))
print(N_1)
print(N_2)
accuracy_var = np.sqrt((1.0 / N_iterations + N_2 / N_1) * S_uj)
print(accuracy_var)
print(np.sqrt(1 / N_iterations * S_uj))
print(st.sem(accuracy))
stats = dict()
stats["Accuracy 95%:"] = str(
compute_CI.compute_confidence(accuracy, N_1, N_2, 0.95))
stats["AUC 95%:"] = str(compute_CI.compute_confidence(auc, N_1, N_2, 0.95))
stats["F1-score 95%:"] = str(
compute_CI.compute_confidence(f1_score_list, N_1, N_2, 0.95))
stats["Precision 95%:"] = str(
compute_CI.compute_confidence(precision, N_1, N_2, 0.95))
stats["Sensitivity 95%: "] = str(
compute_CI.compute_confidence(sensitivity, N_1, N_2, 0.95))
stats["Specificity 95%:"] = str(
compute_CI.compute_confidence(specificity, N_1, N_2, 0.95))
print("Accuracy 95%:" +
str(compute_CI.compute_confidence(accuracy, N_1, N_2, 0.95)))
print("AUC 95%:" + str(compute_CI.compute_confidence(auc, N_1, N_2, 0.95)))
print("F1-score 95%:" +
str(compute_CI.compute_confidence(f1_score_list, N_1, N_2, 0.95)))
print("Precision 95%:" +
str(compute_CI.compute_confidence(precision, N_1, N_2, 0.95)))
print("Sensitivity 95%: " +
str(compute_CI.compute_confidence(sensitivity, N_1, N_2, 0.95)))
print("Specificity 95%:" +
str(compute_CI.compute_confidence(specificity, N_1, N_2, 0.95)))
what_to_print = ['always', 'mostly']
for what in what_to_print:
if what == 'always':
alwaysright = dict()
alwayswrong = dict()
for i_ID in patient_classification_list:
percentage_right = patient_classification_list[i_ID][
'N_correct'] / float(
patient_classification_list[i_ID]['N_test'])
# print(i_ID + ' , ' + str(patient_classification_list[i_ID]['N_test']) + ' : ' + str(percentage_right) + '\n')
if percentage_right == 1.0:
label = mutation_label[0][np.where(i_ID == patient_IDs)]
label = label[0][0]
alwaysright[i_ID] = label
# alwaysright.append(('{} ({})').format(i_ID, label))
print(("Always Right: {}, label {}").format(i_ID, label))
if percentage_right == 0:
label = mutation_label[0][np.where(
i_ID == patient_IDs)].tolist()
label = label[0][0]
alwayswrong[i_ID] = label
# alwayswrong.append(('{} ({})').format(i_ID, label))
print(("Always Wrong: {}, label {}").format(i_ID, label))
stats["Always right"] = alwaysright
stats["Always wrong"] = alwayswrong
elif what == 'mostly':
margin = float(0.2)
min_right = float(1 - margin) #for mostly right
max_right = float(margin) #for mostly wrong
mostlyright = dict()
mostlywrong = dict()
for i_ID in patient_classification_list:
percentage_right = patient_classification_list[i_ID][
'N_correct'] / float(
patient_classification_list[i_ID]['N_test'])
if percentage_right > min_right:
label = mutation_label[0][np.where(i_ID == patient_IDs)]
label = label[0][0]
mostlyright[i_ID] = [
label, "{}%".format(100 * percentage_right)
]
print((
"Mostly Right: {}, label {}, percentage: {}%").format(
i_ID, label, 100 * percentage_right))
if percentage_right < max_right:
label = mutation_label[0][np.where(
i_ID == patient_IDs)].tolist()
label = label[0][0]
mostlywrong[i_ID] = [
label, "{}%".format(100 * percentage_right)
]
print((
"Mostly Wrong: {}, label {}, percentage: {}%").format(
i_ID, label, 100 * percentage_right))
stats["Mostly right"] = mostlyright
stats["Mostly wrong"] = mostlywrong
else:
raise IOError('Unknown argument given...')
if show_plots:
import matplotlib.pyplot as plt
plt.figure()
plt.boxplot(accuracy)
plt.ylim([-0.05, 1.05])
plt.ylabel('Accuracy')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(auc)
plt.ylim([-0.05, 1.05])
plt.ylabel('AUC')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(precision)
plt.ylim([-0.05, 1.05])
plt.ylabel('Precision')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(sensitivity)
plt.ylim([-0.05, 1.05])
plt.ylabel('Sensitivity')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(specificity)
plt.ylim([-0.05, 1.05])
plt.ylabel('Specificity')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
# Save Y_score values
Y_score_dict = dict()
for j in range(len(Y_score)):
Y_score_dict['CV_{}'.format(j)] = Y_score[j]
Y_score = pd.DataFrame(Y_score_dict)
Y_score.to_hdf('/archive/wkessels/output/Lipo_SVM/Y_score.hdf5', 'Y_score')
# write_to_txt('Y_test', Y_test, ROC_data_folder)
# write_to_txt('X_test', X_test, ROC_data_folder)
# write_to_txt('Y_train', Y_train, ROC_data_folder)
# write_to_txt('mutation_data', mutation_data, ROC_data_folder)
# write_to_txt('patient_IDs', patient_IDs, ROC_data_folder)
# write_to_txt('mutation_label',mutation_label, ROC_data_folder)
# write_to_txt('y_truth', y_truth, ROC_data_folder)
# write_to_txt('y_prediction', y_prediction, ROC_data_folder)
# write_to_txt('y_score', y_score, ROC_data_folder)
# write_to_txt('N_1', N_1, ROC_data_folder)
# write_to_txt('N_2', N_2, ROC_data_folder)
# write_to_txt('stats', stats, ROC_data_folder)
return stats
def plot_SVM(prediction,
label_data,
label_type,
show_plots=False,
alpha=0.95,
ensemble=False,
verbose=True,
ensemble_scoring=None,
output='stats'):
'''
Plot the output of a single binary estimator, e.g. a SVM.
Parameters
----------
prediction: pandas dataframe or string, mandatory
output of trainclassifier function, either a pandas dataframe
or a HDF5 file
label_data: string, mandatory
Contains the path referring to a .txt file containing the
patient label(s) and value(s) to be used for learning. See
the Github Wiki for the format.
label_type: string, mandatory
Name of the label to extract from the label data to test the
estimator on.
show_plots: Boolean, default False
Determine whether matplotlib performance plots are made.
alpha: float, default 0.95
Significance of confidence intervals.
ensemble: False, integer or 'Caruana'
Determine whether an ensemble will be created. If so,
either provide an integer to determine how many of the
top performing classifiers should be in the ensemble, or use
the string "Caruana" to use smart ensembling based on
Caruana et al. 2004.
verbose: boolean, default True
Plot intermedate messages.
ensemble_scoring: string, default None
Metric to be used for evaluating the ensemble. If None,
the option set in the prediction object will be used.
output: string, default stats
Determine which results are put out. If stats, the statistics of the
estimator will be returned. If scores, the scores will be returned.
Returns
----------
Depending on the output parameters, the following outputs are returned:
If output == 'stats':
stats: dictionary
Contains the confidence intervals of the performance metrics
and the number of times each patient was classifier correctly
or incorrectly.
If output == 'scores':
y_truths: list
Contains the true label for each object.
y_scores: list
Contains the score (e.g. posterior) for each object.
y_predictions: list
Contains the predicted label for each object.
PIDs: list
Contains the patient ID/name for each object.
'''
# Load the prediction object if it's a hdf5 file
if type(prediction) is not pd.core.frame.DataFrame:
if os.path.isfile(prediction):
prediction = pd.read_hdf(prediction)
# Select the estimator from the pandas dataframe to use
keys = prediction.keys()
SVMs = list()
if label_type is None:
label_type = keys[0]
if type(label_type) is list:
# FIXME: Support for multiple label types not supported yet.
print(
'[PREDICT Warning] Support for multiple label types not supported yet. Taking first label for plot_SVM.'
)
label_type = label_type[0]
# Extract the estimators, features and labels
SVMs = prediction[label_type]['classifiers']
Y_test = prediction[label_type]['Y_test']
X_test = prediction[label_type]['X_test']
X_train = prediction[label_type]['X_train']
Y_train = prediction[label_type]['Y_train']
feature_labels = prediction[label_type]['feature_labels']
# Load the label data
if type(label_data) is not dict:
if os.path.isfile(label_data):
label_data = gp.load_mutation_status(label_data, [[label_type]])
patient_IDs = label_data['patient_IDs']
mutation_label = label_data['mutation_label']
# Create lists for performance measures
sensitivity = list()
specificity = list()
precision = list()
accuracy = list()
auc = list()
f1_score_list = list()
patient_classification_list = dict()
if output in ['scores', 'decision']:
# Keep track of all groundth truths and scores
y_truths = list()
y_scores = list()
y_predictions = list()
PIDs = list()
# Loop over the test sets, which probably correspond with cross validation
# iterations
for i in range(0, len(Y_test)):
print("\n")
print("Cross validation {} / {}.").format(str(i + 1), str(len(Y_test)))
test_patient_IDs = prediction[label_type]['patient_ID_test'][i]
train_patient_IDs = prediction[label_type]['patient_ID_train'][i]
X_test_temp = X_test[i]
X_train_temp = X_train[i]
Y_train_temp = Y_train[i]
Y_test_temp = Y_test[i]
test_indices = list()
# Check which patients are in the test set.
for i_ID in test_patient_IDs:
test_indices.append(np.where(patient_IDs == i_ID)[0][0])
# Initiate counting how many times a patient is classified correctly
if i_ID not in patient_classification_list:
patient_classification_list[i_ID] = dict()
patient_classification_list[i_ID]['N_test'] = 0
patient_classification_list[i_ID]['N_correct'] = 0
patient_classification_list[i_ID]['N_wrong'] = 0
patient_classification_list[i_ID]['N_test'] += 1
# Extract ground truth
y_truth = Y_test_temp
# If requested, first let the SearchCV object create an ensemble
if ensemble:
# NOTE: Added for backwards compatability
if not hasattr(SVMs[i], 'cv_iter'):
cv_iter = list(SVMs[i].cv.split(X_train_temp, Y_train_temp))
SVMs[i].cv_iter = cv_iter
# Create the ensemble
X_train_temp = [(x, feature_labels) for x in X_train_temp]
SVMs[i].create_ensemble(X_train_temp,
Y_train_temp,
method=ensemble,
verbose=verbose,
scoring=ensemble_scoring)
# Create prediction
y_prediction = SVMs[i].predict(X_test_temp)
print("Truth: " + str(y_truth))
print("Prediction: " + str(y_prediction))
# Add if patient was classified correctly or not to counting
for i_truth, i_predict, i_test_ID in zip(y_truth, y_prediction,
test_patient_IDs):
if i_truth == i_predict:
patient_classification_list[i_test_ID]['N_correct'] += 1
else:
patient_classification_list[i_test_ID]['N_wrong'] += 1
y_score = SVMs[i].decision_function(X_test_temp)
print('AUC: ' + str(roc_auc_score(y_truth, y_score)))
if output == 'decision':
# Output the posteriors
y_scores.append(y_score)
y_truths.append(y_truth)
y_predictions.append(y_prediction)
PIDs.append(test_patient_IDs)
elif output == 'scores':
# Output the posteriors
y_scores.append(SVMs[i].predict_proba(X_test_temp)[1])
y_truths.append(y_truth)
y_predictions.append(y_prediction)
PIDs.append(test_patient_IDs)
elif output == 'stats':
# Compute statistics
# Compute confusion matrix and use for sensitivity/specificity
c_mat = confusion_matrix(y_truth, y_prediction)
TN = c_mat[0, 0]
FN = c_mat[1, 0]
TP = c_mat[1, 1]
FP = c_mat[0, 1]
if FN == 0 and TP == 0:
sensitivity.append(0)
else:
sensitivity.append(float(TP) / (TP + FN))
if FP == 0 and TN == 0:
specificity.append(0)
else:
specificity.append(float(TN) / (FP + TN))
if TP == 0 and FP == 0:
precision.append(0)
else:
precision.append(float(TP) / (TP + FP))
# Additionally, compute accuracy, AUC and f1-score
accuracy.append(accuracy_score(y_truth, y_prediction))
auc.append(roc_auc_score(y_truth, y_score))
f1_score_list.append(
f1_score(y_truth, y_prediction, average='weighted'))
if output in ['scores', 'decision']:
# Return the scores and true values of all patients
return y_truths, y_scores, y_predictions, PIDs
elif output == 'stats':
# Compute statistics
# Extract sample size
N_1 = float(len(train_patient_IDs))
N_2 = float(len(test_patient_IDs))
# Compute alpha confidence intervallen
stats = dict()
stats["Accuracy 95%:"] = str(
compute_CI.compute_confidence(accuracy, N_1, N_2, alpha))
stats["AUC 95%:"] = str(
compute_CI.compute_confidence(auc, N_1, N_2, alpha))
stats["F1-score 95%:"] = str(
compute_CI.compute_confidence(f1_score_list, N_1, N_2, alpha))
stats["Precision 95%:"] = str(
compute_CI.compute_confidence(precision, N_1, N_2, alpha))
stats["Sensitivity 95%: "] = str(
compute_CI.compute_confidence(sensitivity, N_1, N_2, alpha))
stats["Specificity 95%:"] = str(
compute_CI.compute_confidence(specificity, N_1, N_2, alpha))
print("Accuracy 95%:" +
str(compute_CI.compute_confidence(accuracy, N_1, N_2, alpha)))
print("AUC 95%:" +
str(compute_CI.compute_confidence(auc, N_1, N_2, alpha)))
print(
"F1-score 95%:" +
str(compute_CI.compute_confidence(f1_score_list, N_1, N_2, alpha)))
print("Precision 95%:" +
str(compute_CI.compute_confidence(precision, N_1, N_2, alpha)))
print("Sensitivity 95%: " +
str(compute_CI.compute_confidence(sensitivity, N_1, N_2, alpha)))
print("Specificity 95%:" +
str(compute_CI.compute_confidence(specificity, N_1, N_2, alpha)))
# Extract statistics on how often patients got classified correctly
alwaysright = dict()
alwayswrong = dict()
percentages = dict()
for i_ID in patient_classification_list:
percentage_right = patient_classification_list[i_ID][
'N_correct'] / float(
patient_classification_list[i_ID]['N_test'])
label = mutation_label[0][np.where(i_ID == patient_IDs)]
label = label[0][0]
percentages[i_ID] = str(label) + ': ' + str(
round(percentage_right, 2) * 100) + '%'
if percentage_right == 1.0:
alwaysright[i_ID] = label
print(("Always Right: {}, label {}").format(i_ID, label))
elif percentage_right == 0:
alwayswrong[i_ID] = label
print(("Always Wrong: {}, label {}").format(i_ID, label))
stats["Always right"] = alwaysright
stats["Always wrong"] = alwayswrong
stats['Percentages'] = percentages
if show_plots:
# Plot some characteristics in boxplots
import matplotlib.pyplot as plt
plt.figure()
plt.boxplot(accuracy)
plt.ylim([-0.05, 1.05])
plt.ylabel('Accuracy')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(auc)
plt.ylim([-0.05, 1.05])
plt.ylabel('AUC')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(precision)
plt.ylim([-0.05, 1.05])
plt.ylabel('Precision')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(sensitivity)
plt.ylim([-0.05, 1.05])
plt.ylabel('Sensitivity')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(specificity)
plt.ylim([-0.05, 1.05])
plt.ylabel('Specificity')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
return stats
def plot_multi_SVM(prediction,
mutation_data,
label_type,
show_plots=False,
key=None,
n_classifiers=[1],
outputfolder=None):
if type(prediction) is not pd.core.frame.DataFrame:
if os.path.isfile(prediction):
prediction = pd.read_hdf(prediction)
keys = prediction.keys()
SVMs = list()
if key is None:
label = keys[0]
else:
label = key
SVMs = prediction[label]['classifiers']
Y_test = prediction[label]['Y_test']
X_test = prediction[label]['X_test']
X_train = prediction[label]['X_train']
Y_train = prediction[label]['Y_train']
test_patient_IDs = prediction[label]['patient_ID_test']
train_patient_IDs = prediction[label]['patient_ID_train']
feature_labels = prediction[label]['feature_labels']
# print(len(X_test[0][0]))
# print(config)
# X_train = data2['19q']['X_train']
# Y_train = data2['19q']['Y_train']
# mutation_data = gp.load_mutation_status(patientinfo, [[label]])
if type(mutation_data) is not dict:
if os.path.isfile(mutation_data):
label_data = gp.load_mutation_status(mutation_data, [[label_type]])
patient_IDs = label_data['patient_IDs']
mutation_label = label_data['mutation_label']
# print(len(SVMs))
N_iterations = float(len(SVMs))
# mutation_label = np.asarray(mutation_label)
for n_class in n_classifiers:
# output_json = os.path.join(outputfolder, ('performance_{}.json').format(str(n_class)))
sensitivity = list()
specificity = list()
precision = list()
accuracy = list()
auc = list()
# auc_train = list()
f1_score_list = list()
patient_classification_list = dict()
trained_classifiers = list()
y_score = list()
y_test = list()
pid_test = list()
y_predict = list()
# csvfile = os.path.join(outputfolder, ('scores_{}.csv').format(str(n_class)))
# towrite = list()
#
# csvfile_plain = os.path.join(outputfolder, ('scores_plain_{}.csv').format(str(n_class)))
# towrite_plain = list()
empty_scores = {k: '' for k in natsort.natsorted(patient_IDs)}
empty_scores = collections.OrderedDict(sorted(empty_scores.items()))
# towrite.append(["Patient"] + empty_scores.keys())
params = dict()
for num, s in enumerate(SVMs):
scores = empty_scores.copy()
print("Processing {} / {}.").format(str(num + 1), str(len(SVMs)))
trained_classifiers.append(s)
# Extract test info
test_patient_IDs_temp = test_patient_IDs[num]
train_patient_IDs_temp = train_patient_IDs[num]
X_train_temp = X_train[num]
Y_train_temp = Y_train[num]
X_test_temp = X_test[num]
Y_test_temp = Y_test[num]
# Extract sample size
N_1 = float(len(train_patient_IDs_temp))
N_2 = float(len(test_patient_IDs_temp))
test_indices = list()
for i_ID in test_patient_IDs_temp:
test_indices.append(np.where(patient_IDs == i_ID)[0][0])
if i_ID not in patient_classification_list:
patient_classification_list[i_ID] = dict()
patient_classification_list[i_ID]['N_test'] = 0
patient_classification_list[i_ID]['N_correct'] = 0
patient_classification_list[i_ID]['N_wrong'] = 0
patient_classification_list[i_ID]['N_test'] += 1
# y_truth = [mutation_label[0][k] for k in test_indices]
# FIXME: order can be switched, need to find a smart fix
# 1 for normal, 0 for KM
# y_truth = [mutation_label[0][k][0] for k in test_indices]
y_truth = Y_test_temp
# Predict using the top N classifiers
results = s.cv_results_['rank_test_score']
indices = range(0, len(results))
sortedindices = [x for _, x in sorted(zip(results, indices))]
sortedindices = sortedindices[0:n_class]
y_prediction = np.zeros([n_class, len(y_truth)])
y_score = np.zeros([n_class, len(y_truth)])
# Get some base objects required
base_estimator = s.estimator
y_train = Y_train_temp
y_train_prediction = np.zeros([n_class, len(y_train)])
scorer = s.scorer_
train = np.asarray(range(0, len(y_train)))
test = train # This is in order to use the full training dataset to train the model
# Remove the NaN features
X_notnan = X_train_temp[:]
for pnum, (pid, x) in enumerate(
zip(train_patient_IDs_temp, X_train_temp)):
for fnum, (f, fid) in enumerate(zip(x, feature_labels)):
if np.isnan(f):
print(
"[PREDICT WARNING] NaN found, patient {}, label {}. Replacing with zero."
).format(pid, fid)
# Note: X is a list of lists, hence we cannot index the element directly
features_notnan = x[:]
features_notnan[fnum] = 0
X_notnan[pnum] = features_notnan
X_train_temp = X_notnan[:]
X_train_temp = [(x, feature_labels) for x in X_train_temp]
X_notnan = X_test_temp[:]
for pnum, (pid,
x) in enumerate(zip(test_patient_IDs_temp,
X_test_temp)):
for fnum, (f, fid) in enumerate(zip(x, feature_labels)):
if np.isnan(f):
print(
"[PREDICT WARNING] NaN found, patient {}, label {}. Replacing with zero."
).format(pid, fid)
# Note: X is a list of lists, hence we cannot index the element directly
features_notnan = x[:]
features_notnan[fnum] = 0
X_notnan[pnum] = features_notnan
X_test_temp = X_notnan[:]
# X_test_temp = [(x, feature_labels) for x in X_test_temp]
# NOTE: need to build this in the SearchCVFastr Object
for i, index in enumerate(sortedindices):
print("Processing number {} of {} classifiers.").format(
str(i + 1), str(n_class))
X_testtemp = X_test_temp[:]
# Get the parameters from the index
parameters_est = s.cv_results_['params'][index]
parameters_all = s.cv_results_['params_all'][index]
print parameters_all
print s.cv_results_['mean_test_score'][index]
# NOTE: kernel parameter can be unicode
kernel = str(parameters_est[u'kernel'])
del parameters_est[u'kernel']
del parameters_all[u'kernel']
parameters_est['kernel'] = kernel
parameters_all['kernel'] = kernel
# Refit a classifier using the settings given
print("Refitting classifier with best settings.")
# Only when using fastr this is an entry
if 'Number' in parameters_est.keys():
del parameters_est['Number']
best_estimator = clone(base_estimator).set_params(
**parameters_est)
# ret, GroupSel, VarSel, SelectModel, feature_labels[0], scaler =\
# fit_and_score(best_estimator, X_train, y_train, scorer,
# train, test, True, parameters_all,
# t.fit_params,
# t.return_train_score,
# True, True, True,
# t.error_score)
ret, GroupSel, VarSel, SelectModel, _, scaler =\
fit_and_score(estimator=best_estimator,
X=X_train_temp,
y=y_train,
scorer=scorer,
train=train, test=test,
verbose=True,
para=parameters_all,
fit_params=s.fit_params,
return_train_score=s.return_train_score,
return_n_test_samples=True,
return_times=True,
return_parameters=True,
error_score=s.error_score)
X = [x[0] for x in X_train_temp]
if GroupSel is not None:
X = GroupSel.transform(X)
X_testtemp = GroupSel.transform(X_testtemp)
if SelectModel is not None:
X = SelectModel.transform(X)
X_testtemp = SelectModel.transform(X_testtemp)
if VarSel is not None:
X = VarSel.transform(X)
X_testtemp = VarSel.transform(X_testtemp)
if scaler is not None:
X = scaler.transform(X)
X_testtemp = scaler.transform(X_testtemp)
try:
if y_train is not None:
best_estimator.fit(X, y_train, **s.fit_params)
else:
best_estimator.fit(X, **s.fit_params)
# Predict the posterios using the fitted classifier for the training set
print("Evaluating performance on training set.")
if hasattr(best_estimator, 'predict_proba'):
probabilities = best_estimator.predict_proba(X)
y_train_prediction[i, :] = probabilities[:, 1]
else:
# Regression has no probabilities
probabilities = best_estimator.predict(X)
y_train_prediction[i, :] = probabilities[:]
# Predict the posterios using the fitted classifier for the test set
print("Evaluating performance on test set.")
if hasattr(best_estimator, 'predict_proba'):
probabilities = best_estimator.predict_proba(
X_testtemp)
y_prediction[i, :] = probabilities[:, 1]
else:
# Regression has no probabilities
probabilities = best_estimator.predict(X_testtemp)
y_prediction[i, :] = probabilities[:]
if type(s.estimator) == sklearn.svm.classes.SVC:
y_score[i, :] = best_estimator.decision_function(
X_testtemp)
else:
y_score[i, :] = best_estimator.decision_function(
X_testtemp)[:, 0]
except ValueError:
# R2 score was set to zero previously
y_train_prediction[i, :] = np.asarray([0.5] * len(X))
y_prediction[i, :] = np.asarray([0.5] * len(X_testtemp))
y_score[i, :] = np.asarray([0.5] * len(X_testtemp))
probabilities = []
# Add number parameter settings
for k in parameters_all.keys():
if k not in params.keys():
params[k] = list()
params[k].append(parameters_all[k])
# Save some memory
del best_estimator, X, X_testtemp, ret, GroupSel, VarSel, SelectModel, scaler, parameters_est, parameters_all, probabilities
# Take mean over posteriors of top n
y_train_prediction_m = np.mean(y_train_prediction, axis=0)
y_prediction_m = np.mean(y_prediction, axis=0)
# NOTE: Not sure if this is best way to compute AUC
y_score = y_prediction_m
if type(s.estimator) == sklearn.svm.classes.SVC:
# Look for optimal F1 performance on training set
thresholds = np.arange(0, 1, 0.01)
f1_scores = list()
y_train_prediction = np.zeros(y_train_prediction_m.shape)
for t in thresholds:
for ip, y in enumerate(y_train_prediction_m):
if y > t:
y_train_prediction[ip] = 1
else:
y_train_prediction[ip] = 0
f1_scores.append(
f1_score(y_train_prediction,
y_train,
average='weighted'))
# Use best threshold to determine test score
best_index = np.argmax(f1_scores)
best_thresh = thresholds[best_index]
best_thresh = 0.5
y_prediction = np.zeros(y_prediction_m.shape)
for ip, y in enumerate(y_prediction_m):
if y > best_thresh:
y_prediction[ip] = 1
else:
y_prediction[ip] = 0
# y_prediction = t.predict(X_temp)
y_prediction = [min(max(y, 0), 1) for y in y_prediction]
else:
y_prediction = y_prediction_m
y_prediction = [min(max(y, 0), 1) for y in y_prediction]
# NOTE: start of old function part
print "Truth: ", y_truth
print "Prediction: ", y_prediction
for i_truth, i_predict, i_test_ID in zip(y_truth, y_prediction,
test_patient_IDs_temp):
if i_truth == i_predict:
patient_classification_list[i_test_ID]['N_correct'] += 1
else:
patient_classification_list[i_test_ID]['N_wrong'] += 1
# print('bla')
# print(y_truth)
# print(y_prediction)
c_mat = confusion_matrix(y_truth, y_prediction)
TN = c_mat[0, 0]
FN = c_mat[1, 0]
TP = c_mat[1, 1]
FP = c_mat[0, 1]
if FN == 0 and TP == 0:
sensitivity.append(0)
else:
sensitivity.append(float(TP) / (TP + FN))
if FP == 0 and TN == 0:
specificity.append(0)
else:
specificity.append(float(TN) / (FP + TN))
if TP == 0 and FP == 0:
precision.append(0)
else:
precision.append(float(TP) / (TP + FP))
accuracy.append(accuracy_score(y_truth, y_prediction))
auc.append(roc_auc_score(y_truth, y_score))
f1_score_list.append(
f1_score(y_truth, y_prediction, average='weighted'))
# Adjusted according to "Inference for the Generelization error"
accuracy_mean = np.mean(accuracy)
S_uj = 1.0 / max((N_iterations - 1), 1) * np.sum(
(accuracy_mean - accuracy)**2.0)
print Y_test
N_1 = float(len(Y_train[0]))
N_2 = float(len(Y_test[0]))
print(N_1)
print(N_2)
accuracy_var = np.sqrt((1.0 / N_iterations + N_2 / N_1) * S_uj)
print(accuracy_var)
print(np.sqrt(1 / N_iterations * S_uj))
print(st.sem(accuracy))
stats = dict()
stats["Accuracy 95%:"] = str(
compute_CI.compute_confidence(accuracy, N_1, N_2, 0.95))
stats["AUC 95%:"] = str(
compute_CI.compute_confidence(auc, N_1, N_2, 0.95))
stats["F1-score 95%:"] = str(
compute_CI.compute_confidence(f1_score_list, N_1, N_2, 0.95))
stats["Precision 95%:"] = str(
compute_CI.compute_confidence(precision, N_1, N_2, 0.95))
stats["Sensitivity 95%: "] = str(
compute_CI.compute_confidence(sensitivity, N_1, N_2, 0.95))
stats["Specificity 95%:"] = str(
compute_CI.compute_confidence(specificity, N_1, N_2, 0.95))
print("Accuracy 95%:" +
str(compute_CI.compute_confidence(accuracy, N_1, N_2, 0.95)))
print("AUC 95%:" +
str(compute_CI.compute_confidence(auc, N_1, N_2, 0.95)))
print(
"F1-score 95%:" +
str(compute_CI.compute_confidence(f1_score_list, N_1, N_2, 0.95)))
print("Precision 95%:" +
str(compute_CI.compute_confidence(precision, N_1, N_2, 0.95)))
print("Sensitivity 95%: " +
str(compute_CI.compute_confidence(sensitivity, N_1, N_2, 0.95)))
print("Specificity 95%:" +
str(compute_CI.compute_confidence(specificity, N_1, N_2, 0.95)))
alwaysright = dict()
alwayswrong = dict()
for i_ID in patient_classification_list:
percentage_right = patient_classification_list[i_ID][
'N_correct'] / float(
patient_classification_list[i_ID]['N_test'])
# print(i_ID + ' , ' + str(patient_classification_list[i_ID]['N_test']) + ' : ' + str(percentage_right) + '\n')
if percentage_right == 1.0:
label = mutation_label[0][np.where(i_ID == patient_IDs)]
label = label[0][0]
alwaysright[i_ID] = label
# alwaysright.append(('{} ({})').format(i_ID, label))
print(("Always Right: {}, label {}").format(i_ID, label))
if percentage_right == 0:
label = mutation_label[0][np.where(
i_ID == patient_IDs)].tolist()
label = label[0][0]
alwayswrong[i_ID] = label
# alwayswrong.append(('{} ({})').format(i_ID, label))
print(("Always Wrong: {}, label {}").format(i_ID, label))
stats["Always right"] = alwaysright
stats["Always wrong"] = alwayswrong
if show_plots:
import matplotlib.pyplot as plt
plt.figure()
plt.boxplot(accuracy)
plt.ylim([-0.05, 1.05])
plt.ylabel('Accuracy')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(auc)
plt.ylim([-0.05, 1.05])
plt.ylabel('AUC')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(precision)
plt.ylim([-0.05, 1.05])
plt.ylabel('Precision')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(sensitivity)
plt.ylim([-0.05, 1.05])
plt.ylabel('Sensitivity')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(specificity)
plt.ylim([-0.05, 1.05])
plt.ylabel('Specificity')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
return stats
def plot_single_SVM(prediction, mutation_data, show_plots=False):
if os.path.isfile(prediction):
prediction = pd.read_hdf(prediction)
keys = prediction.keys()
SVMs = list()
label = keys[0]
SVMs = prediction[label]['svms']
Y_test = prediction[label]['Y_test']
X_test = prediction[label]['X_test']
Y_train = prediction[label]['X_train']
# print(len(X_test[0][0]))
# print(config)
# X_train = data2['19q']['X_train']
# Y_train = data2['19q']['Y_train']
# mutation_data = gp.load_mutation_status(patientinfo, [[label]])
if os.path.isfile(mutation_data):
mutation_data = gp.load_mutation_status(mutation_data, [[label]])
patient_IDs = mutation_data['patient_IDs']
mutation_label = mutation_data['mutation_label']
# mutation_name = mutation_data['mutation_name']
# print(len(SVMs))
N_iterations = float(len(SVMs))
# mutation_label = np.asarray(mutation_label)
sensitivity = list()
specificity = list()
precision = list()
accuracy = list()
auc = list()
# auc_train = list()
f1_score_list = list()
patient_classification_list = dict()
for i in range(0, len(Y_test)):
# print(Y_test[i])
# if Y_test[i].shape[1] > 1:
# # print(Y_test[i])
# y_truth = np.prod(Y_test[i][:, 0:2], axis=1)
# else:
# y_truth_test = Y_test[i]
test_patient_IDs = prediction[label]['patient_ID_test'][i]
if 'LGG-Radiogenomics-046' in test_patient_IDs:
wrong_index = np.where(test_patient_IDs == 'LGG-Radiogenomics-046')
test_patient_IDs = np.delete(test_patient_IDs, wrong_index)
X_temp = X_test[i]
print(X_temp.shape)
X_temp = np.delete(X_test[i], wrong_index, axis=0)
print(X_temp.shape)
# X_test.pop(wrong_index[0])
# print(len(X_test))
else:
X_temp = X_test[i]
test_indices = list()
for i_ID in test_patient_IDs:
test_indices.append(np.where(patient_IDs == i_ID)[0][0])
if i_ID not in patient_classification_list:
patient_classification_list[i_ID] = dict()
patient_classification_list[i_ID]['N_test'] = 0
patient_classification_list[i_ID]['N_correct'] = 0
patient_classification_list[i_ID]['N_wrong'] = 0
patient_classification_list[i_ID]['N_test'] += 1
y_truth = [mutation_label[0][k] for k in test_indices]
# print(y_truth)
# print(y_truth_test)
# print(test_patient_IDs)
y_predict_1 = SVMs[i].predict(X_temp)
# print(y_predict_1).shape
y_prediction = y_predict_1
# y_prediction = np.prod(y_prediction, axis=0)
print "Truth: ", y_truth
print "Prediction: ", y_prediction
for i_truth, i_predict, i_test_ID in zip(y_truth, y_prediction,
test_patient_IDs):
if i_truth == i_predict:
patient_classification_list[i_test_ID]['N_correct'] += 1
else:
patient_classification_list[i_test_ID]['N_wrong'] += 1
# print('bla')
# print(y_truth)
# print(y_prediction)
c_mat = confusion_matrix(y_truth, y_prediction)
TN = c_mat[0, 0]
FN = c_mat[1, 0]
TP = c_mat[1, 1]
FP = c_mat[0, 1]
if FN == 0 and TP == 0:
sensitivity.append(0)
else:
sensitivity.append(float(TP) / (TP + FN))
if FP == 0 and TN == 0:
specificity.append(0)
else:
specificity.append(float(TN) / (FP + TN))
if TP == 0 and FP == 0:
precision.append(0)
else:
precision.append(float(TP) / (TP + FP))
accuracy.append(accuracy_score(y_truth, y_prediction))
auc.append(roc_auc_score(y_truth, y_prediction))
f1_score_list.append(
f1_score(y_truth, y_prediction, average='weighted'))
# Adjusted according to "Inference for the Generelization error"
accuracy_mean = np.mean(accuracy)
S_uj = 1.0 / max((N_iterations - 1), 1) * np.sum(
(accuracy_mean - accuracy)**2.0)
print Y_test
N_1 = float(len(Y_train[0]))
N_2 = float(len(Y_test[0]))
print(N_1)
print(N_2)
accuracy_var = np.sqrt((1.0 / N_iterations + N_2 / N_1) * S_uj)
print(accuracy_var)
print(np.sqrt(1 / N_iterations * S_uj))
print(st.sem(accuracy))
stats = dict()
stats["Accuracy 95%:"] = str(
compute_CI.compute_confidence(accuracy, N_1, N_2, 0.95))
stats["AUC 95%:"] = str(compute_CI.compute_confidence(auc, N_1, N_2, 0.95))
stats["F1-score 95%:"] = str(
compute_CI.compute_confidence(f1_score_list, N_1, N_2, 0.95))
stats["Precision 95%:"] = str(
compute_CI.compute_confidence(precision, N_1, N_2, 0.95))
stats["Sensitivity 95%: "] = str(
compute_CI.compute_confidence(sensitivity, N_1, N_2, 0.95))
stats["Specificity 95%:"] = str(
compute_CI.compute_confidence(specificity, N_1, N_2, 0.95))
print("Accuracy 95%:" +
str(compute_CI.compute_confidence(accuracy, N_1, N_2, 0.95)))
print("AUC 95%:" + str(compute_CI.compute_confidence(auc, N_1, N_2, 0.95)))
print("F1-score 95%:" +
str(compute_CI.compute_confidence(f1_score_list, N_1, N_2, 0.95)))
print("Precision 95%:" +
str(compute_CI.compute_confidence(precision, N_1, N_2, 0.95)))
print("Sensitivity 95%: " +
str(compute_CI.compute_confidence(sensitivity, N_1, N_2, 0.95)))
print("Specificity 95%:" +
str(compute_CI.compute_confidence(specificity, N_1, N_2, 0.95)))
alwaysright = dict()
alwayswrong = dict()
for i_ID in patient_classification_list:
percentage_right = patient_classification_list[i_ID][
'N_correct'] / float(patient_classification_list[i_ID]['N_test'])
# print(i_ID + ' , ' + str(patient_classification_list[i_ID]['N_test']) + ' : ' + str(percentage_right) + '\n')
if percentage_right == 1.0:
label = mutation_label[0][np.where(i_ID == patient_IDs)]
label = label[0][0]
alwaysright[i_ID] = label
# alwaysright.append(('{} ({})').format(i_ID, label))
print(("Always Right: {}, label {}").format(i_ID, label))
if percentage_right == 0:
label = mutation_label[0][np.where(i_ID == patient_IDs)].tolist()
label = label[0][0]
alwayswrong[i_ID] = label
# alwayswrong.append(('{} ({})').format(i_ID, label))
print(("Always Wrong: {}, label {}").format(i_ID, label))
stats["Always right"] = alwaysright
stats["Always wrong"] = alwayswrong
if show_plots:
import matplotlib.pyplot as plt
plt.figure()
plt.boxplot(accuracy)
plt.ylim([-0.05, 1.05])
plt.ylabel('Accuracy')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(auc)
plt.ylim([-0.05, 1.05])
plt.ylabel('AUC')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(precision)
plt.ylim([-0.05, 1.05])
plt.ylabel('Precision')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(sensitivity)
plt.ylim([-0.05, 1.05])
plt.ylabel('Sensitivity')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(specificity)
plt.ylim([-0.05, 1.05])
plt.ylabel('Specificity')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
return stats
def plot_single_SVM(prediction, label_data, label_type, show_plots=False,
key=None, alpha=0.95, ensemble=False):
'''
Plot the output of a single binary estimator, e.g. a SVM.
Parameters
----------
prediction: pandas dataframe or string, mandatory
output of trainclassifier function, either a pandas dataframe
or a HDF5 file
label_data: string, mandatory
Contains the path referring to a .txt file containing the
patient label(s) and value(s) to be used for learning. See
the Github Wiki for the format.
label_type: string, mandatory
Name of the label to extract from the label data to test the
estimator on.
show_plots: Boolean, default False
Determine whether matplotlib performance plots are made.
key: string, default None
As the prediction object can contain multiple estimators,
the key is used to select the desired estimator. If None, the
first key and estimator will be used
alpha: float, default 0.95
Significance of confidence intervals.
'''
# Load the prediction object if it's a hdf5 file
if type(prediction) is not pd.core.frame.DataFrame:
if os.path.isfile(prediction):
prediction = pd.read_hdf(prediction)
# Select the estimator from the pandas dataframe to use
keys = prediction.keys()
SVMs = list()
if key is None:
label = keys[0]
else:
label = key
# Extract the estimators, features and labels
SVMs = prediction[label]['classifiers']
Y_test = prediction[label]['Y_test']
X_test = prediction[label]['X_test']
X_train = prediction[label]['X_train']
Y_train = prediction[label]['Y_train']
feature_labels = prediction[label]['feature_labels']
# Load the label data
if type(label_data) is not dict:
if os.path.isfile(label_data):
label_data = gp.load_mutation_status(label_data, [[label_type]])
patient_IDs = label_data['patient_IDs']
mutation_label = label_data['mutation_label']
# Create lists for performance measures
N_iterations = float(len(SVMs))
sensitivity = list()
specificity = list()
precision = list()
accuracy = list()
auc = list()
f1_score_list = list()
patient_classification_list = dict()
# Loop over the test sets, which probably correspond with cross validation
# iterations
for i in range(0, len(Y_test)):
print("Cross validation {} / {}.").format(str(i + 1), str(len(Y_test)))
test_patient_IDs = prediction[label]['patient_ID_test'][i]
train_patient_IDs = prediction[label]['patient_ID_train'][i]
X_test_temp = X_test[i]
X_train_temp = X_train[i]
Y_train_temp = Y_train[i]
Y_test_temp = Y_test[i]
test_indices = list()
# Check which patients are in the test set.
for i_ID in test_patient_IDs:
test_indices.append(np.where(patient_IDs == i_ID)[0][0])
# Initiate counting how many times a patient is classified correctly
if i_ID not in patient_classification_list:
patient_classification_list[i_ID] = dict()
patient_classification_list[i_ID]['N_test'] = 0
patient_classification_list[i_ID]['N_correct'] = 0
patient_classification_list[i_ID]['N_wrong'] = 0
patient_classification_list[i_ID]['N_test'] += 1
# Extract ground truth
y_truth = Y_test_temp
# If requested, first let the SearchCV object create an ensemble
if ensemble:
# NOTE: Remove these two lines
cv_iter = list(SVMs[i].cv.split(X_train_temp, Y_train_temp))
SVMs[i].cv_iter = cv_iter
X_train_temp = [(x, feature_labels) for x in X_train_temp]
SVMs[i].create_ensemble(X_train_temp, Y_train_temp, verbose=True)
# Create prediction
y_prediction = SVMs[i].predict(X_test_temp)
print "Truth: ", y_truth
print "Prediction: ", y_prediction
# Add if patient was classified correctly or not to counting
for i_truth, i_predict, i_test_ID in zip(y_truth, y_prediction, test_patient_IDs):
if i_truth == i_predict:
patient_classification_list[i_test_ID]['N_correct'] += 1
else:
patient_classification_list[i_test_ID]['N_wrong'] += 1
# Compute confusion matrix and use for sensitivity/specificity
c_mat = confusion_matrix(y_truth, y_prediction)
TN = c_mat[0, 0]
FN = c_mat[1, 0]
TP = c_mat[1, 1]
FP = c_mat[0, 1]
if FN == 0 and TP == 0:
sensitivity.append(0)
else:
sensitivity.append(float(TP)/(TP+FN))
if FP == 0 and TN == 0:
specificity.append(0)
else:
specificity.append(float(TN)/(FP+TN))
if TP == 0 and FP == 0:
precision.append(0)
else:
precision.append(float(TP)/(TP+FP))
# Additionally, compute accuracy, AUC and f1-score
accuracy.append(accuracy_score(y_truth, y_prediction))
y_score = SVMs[i].decision_function(X_test_temp)
auc.append(roc_auc_score(y_truth, y_score))
f1_score_list.append(f1_score(y_truth, y_prediction, average='weighted'))
# Extract sample size
N_1 = float(len(train_patient_IDs))
N_2 = float(len(test_patient_IDs))
# Compute alpha confidence intervallen
stats = dict()
stats["Accuracy 95%:"] = str(compute_CI.compute_confidence(accuracy, N_1, N_2, alpha))
stats["AUC 95%:"] = str(compute_CI.compute_confidence(auc, N_1, N_2, alpha))
stats["F1-score 95%:"] = str(compute_CI.compute_confidence(f1_score_list, N_1, N_2, alpha))
stats["Precision 95%:"] = str(compute_CI.compute_confidence(precision, N_1, N_2, alpha))
stats["Sensitivity 95%: "] = str(compute_CI.compute_confidence(sensitivity, N_1, N_2, alpha))
stats["Specificity 95%:"] = str(compute_CI.compute_confidence(specificity, N_1, N_2, alpha))
print("Accuracy 95%:" + str(compute_CI.compute_confidence(accuracy, N_1, N_2, alpha)))
print("AUC 95%:" + str(compute_CI.compute_confidence(auc, N_1, N_2, alpha)))
print("F1-score 95%:" + str(compute_CI.compute_confidence(f1_score_list, N_1, N_2, alpha)))
print("Precision 95%:" + str(compute_CI.compute_confidence(precision, N_1, N_2, alpha)))
print("Sensitivity 95%: " + str(compute_CI.compute_confidence(sensitivity, N_1, N_2, alpha)))
print("Specificity 95%:" + str(compute_CI.compute_confidence(specificity, N_1, N_2, alpha)))
# Extract statistics on how often patients got classified correctly
alwaysright = dict()
alwayswrong = dict()
percentages = dict()
for i_ID in patient_classification_list:
percentage_right = patient_classification_list[i_ID]['N_correct'] / float(patient_classification_list[i_ID]['N_test'])
label = mutation_label[0][np.where(i_ID == patient_IDs)]
label = label[0][0]
if percentage_right == 1.0:
alwaysright[i_ID] = label
print(("Always Right: {}, label {}").format(i_ID, label))
elif percentage_right == 0:
alwayswrong[i_ID] = label
print(("Always Wrong: {}, label {}").format(i_ID, label))
else:
percentages[i_ID] = str(label) + ': ' + str(round(percentage_right, 2) * 100) + '%'
stats["Always right"] = alwaysright
stats["Always wrong"] = alwayswrong
stats['Percentages'] = percentages
if show_plots:
# Plot some characteristics in boxplots
import matplotlib.pyplot as plt
plt.figure()
plt.boxplot(accuracy)
plt.ylim([-0.05, 1.05])
plt.ylabel('Accuracy')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(auc)
plt.ylim([-0.05, 1.05])
plt.ylabel('AUC')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(precision)
plt.ylim([-0.05, 1.05])
plt.ylabel('Precision')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(sensitivity)
plt.ylim([-0.05, 1.05])
plt.ylabel('Sensitivity')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(specificity)
plt.ylim([-0.05, 1.05])
plt.ylabel('Specificity')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
return stats
def plot_multi_SVM(prediction, label_data, label_type, show_plots=False,
key=None, n_classifiers=[1, 10, 50], alpha=0.95):
'''
Plot the output of a an ensemble of binary estimators, e.g. a SVMs.
Parameters
----------
prediction: pandas dataframe or string, mandatory
output of trainclassifier function, either a pandas dataframe
or a HDF5 file
label_data: string, mandatory
Contains the path referring to a .txt file containing the
patient label(s) and value(s) to be used for learning. See
the Github Wiki for the format.
label_type: string, mandatory
Name of the label to extract from the label data to test the
estimator on.
show_plots: Boolean, default False
Determine whether matplotlib performance plots are made.
key: string, default None
As the prediction object can contain multiple estimators,
the key is used to select the desired estimator. If None, the
first key and estimator will be used
alpha: float, default 0.95
Significance of confidence intervals.
'''
# Load the prediction object if it's a hdf5 file
if type(prediction) is not pd.core.frame.DataFrame:
if os.path.isfile(prediction):
prediction = pd.read_hdf(prediction)
# Select the estimator from the pandas dataframe to use
keys = prediction.keys()
SVMs = list()
if key is None:
label = keys[0]
else:
label = key
# Load the estmators, the test and training features, labels and patient IDs
SVMs = prediction[label]['classifiers']
N_iterations = float(len(SVMs))
Y_test = prediction[label]['Y_test']
X_test = prediction[label]['X_test']
X_train = prediction[label]['X_train']
Y_train = prediction[label]['Y_train']
test_patient_IDs = prediction[label]['patient_ID_test']
train_patient_IDs = prediction[label]['patient_ID_train']
feature_labels = prediction[label]['feature_labels']
# Load the label data
if type(label_data) is not dict:
if os.path.isfile(label_data):
label_data = gp.load_mutation_status(label_data, [[label_type]])
print label_data
patient_IDs = label_data['patient_IDs']
mutation_label = label_data['mutation_label']
# Generate output for each ensemble consisting of n_classifiers
stats = dict()
predictions = dict()
for n_class in n_classifiers:
# Create listst for output performance
sensitivity = list()
specificity = list()
precision = list()
accuracy = list()
auc = list()
f1_score_list = list()
patient_classification_list = dict()
trained_classifiers = list()
y_score = list()
y_test = list()
pid_test = list()
y_predict = list()
# Create empty score entries for each patient
empty_scores = {k: '' for k in natsort.natsorted(patient_IDs)}
empty_scores = collections.OrderedDict(sorted(empty_scores.items()))
predictions[str(n_class)] = dict()
predictions[str(n_class)]['y_score'] = list()
predictions[str(n_class)]['y_test'] = list()
# Loop over the estimators, which probably correspond with cross validation
# iterations
params = dict()
for num, s in enumerate(SVMs):
scores = empty_scores.copy()
print("Processing {} / {}.").format(str(num + 1), str(len(SVMs)))
trained_classifiers.append(s)
# Extract test info
test_patient_IDs_temp = test_patient_IDs[num]
train_patient_IDs_temp = train_patient_IDs[num]
X_train_temp = X_train[num]
Y_train_temp = Y_train[num]
X_test_temp = X_test[num]
Y_test_temp = Y_test[num]
# Stack feature values and labels together again in single object
X_train_temp = [(x, feature_labels) for x in X_train_temp]
# Extract sample size
N_1 = float(len(train_patient_IDs_temp))
N_2 = float(len(test_patient_IDs_temp))
# Check which patients are in the test set.
test_indices = list()
for i_ID in test_patient_IDs_temp:
test_indices.append(np.where(patient_IDs == i_ID)[0][0])
# Initiate counting how many times a patient is classified correctly
if i_ID not in patient_classification_list:
patient_classification_list[i_ID] = dict()
patient_classification_list[i_ID]['N_test'] = 0
patient_classification_list[i_ID]['N_correct'] = 0
patient_classification_list[i_ID]['N_wrong'] = 0
patient_classification_list[i_ID]['N_test'] += 1
# Get ground truth labels
y_truth = Y_test_temp
# Predict labels using the top N classifiers
results = s.cv_results_['rank_test_score']
indices = range(0, len(results))
sortedindices = [x for _, x in sorted(zip(results, indices))]
sortedindices = sortedindices[0:n_class]
y_prediction = np.zeros([n_class, len(y_truth)])
y_score = np.zeros([n_class, len(y_truth)])
# Get some base objects required
y_train = Y_train_temp
y_train_prediction = np.zeros([n_class, len(y_train)])
train = np.asarray(range(0, len(y_train)))
test = train # This is in order to use the full training dataset to train the model
# Loop over the sortedindice selected and refit an estimator for each setting
# NOTE: need to build this in the SearchCVFastr Object
for i, index in enumerate(sortedindices):
print("Processing number {} of {} classifiers.").format(str(i + 1), str(n_class))
X_testtemp = X_test_temp[:]
s_temp = clone(s)
# Get the parameters from the index
parameters_est = s.cv_results_['params'][index]
parameters_all = s.cv_results_['params_all'][index]
# NOTE: kernel parameter can be unicode, which we fix
kernel = str(parameters_est[u'kernel'])
del parameters_est[u'kernel']
del parameters_all[u'kernel']
parameters_est['kernel'] = kernel
parameters_all['kernel'] = kernel
# Refit a classifier using the settings given
print("Refitting classifier with best settings.")
# Only when using fastr this is an entry, but needs to be deleted
if 'Number' in parameters_est.keys():
del parameters_est['Number']
# Refit object with selected settings
s_temp.refit_and_score(X_train_temp, y_train, parameters_all,
parameters_est, train, test)
# Throw away the feature labels
X_train_temp_eval = [x[0] for x in X_train_temp]
# Predict the posterios using the fitted classifier for the training set
print("Evaluating performance on training set.")
if hasattr(s_temp, 'predict_proba'):
probabilities = s_temp.predict_proba(X_train_temp_eval)
y_train_prediction[i, :] = probabilities[:, 1]
else:
# Regression has no probabilities
probabilities = s_temp.predict(X_train_temp_eval)
y_train_prediction[i, :] = probabilities[:]
# Predict the posterios using the fitted classifier for the test set
print("Evaluating performance on test set.")
if hasattr(s_temp, 'predict_proba'):
probabilities = s_temp.predict_proba(X_testtemp)
y_prediction[i, :] = probabilities[:, 1]
else:
# Regression has no probabilities
probabilities = s_temp.predict(X_testtemp)
y_prediction[i, :] = probabilities[:]
# For a VM, we can compute a score per patient with the decision function
if type(s.estimator) == sklearn.svm.classes.SVC:
y_score[i, :] = s_temp.decision_function(X_testtemp)
else:
y_score[i, :] = s_temp.decision_function(X_testtemp)[:, 0]
# Add the parameter settngs of this iteration to the collection object
for k in parameters_all.keys():
if k not in params.keys():
params[k] = list()
params[k].append(parameters_all[k])
# Save some memory
del s_temp, parameters_est, parameters_all, probabilities
# Take mean over posteriors of top n
y_train_prediction_m = np.mean(y_train_prediction, axis=0)
y_prediction_m = np.mean(y_prediction, axis=0)
y_score = y_prediction_m
if type(s.estimator) == sklearn.svm.classes.SVC:
# Look for optimal F1 performance on training set to set threshold
thresholds = np.arange(0, 1, 0.01)
f1_scores = list()
y_train_prediction = np.zeros(y_train_prediction_m.shape)
for t in thresholds:
for ip, y in enumerate(y_train_prediction_m):
if y > t:
y_train_prediction[ip] = 1
else:
y_train_prediction[ip] = 0
f1_scores.append(f1_score(y_train_prediction, y_train, average='weighted'))
# Use best threshold to determine test score
best_index = np.argmax(f1_scores)
best_thresh = thresholds[best_index]
# NOTE: due to overfitting in past, we do not fit a threshold here after all.
best_thresh = 0.5
y_prediction = np.zeros(y_prediction_m.shape)
for ip, y in enumerate(y_prediction_m):
if y > best_thresh:
y_prediction[ip] = 1
else:
y_prediction[ip] = 0
y_prediction = [min(max(y, 0), 1) for y in y_prediction]
else:
y_prediction = y_prediction_m
y_prediction = [min(max(y, 0), 1) for y in y_prediction]
predictions[str(n_class)]['y_score'].append(y_score[:])
predictions[str(n_class)]['y_test'].append(y_truth[:])
print "Truth: ", y_truth
print "Prediction: ", y_prediction
# Add if patient was classified correctly or not to counting
for i_truth, i_predict, i_test_ID in zip(y_truth, y_prediction, test_patient_IDs_temp):
if i_truth == i_predict:
patient_classification_list[i_test_ID]['N_correct'] += 1
else:
patient_classification_list[i_test_ID]['N_wrong'] += 1
# Compute confusion matrix and use for sensitivity/specificity
c_mat = confusion_matrix(y_truth, y_prediction)
TN = c_mat[0, 0]
FN = c_mat[1, 0]
TP = c_mat[1, 1]
FP = c_mat[0, 1]
if FN == 0 and TP == 0:
sensitivity.append(0)
else:
sensitivity.append(float(TP)/(TP+FN))
if FP == 0 and TN == 0:
specificity.append(0)
else:
specificity.append(float(TN)/(FP+TN))
if TP == 0 and FP == 0:
precision.append(0)
else:
precision.append(float(TP)/(TP+FP))
# Additionally, compute accuracy, AUC and f1-score
accuracy.append(accuracy_score(y_truth, y_prediction))
auc.append(roc_auc_score(y_truth, y_score))
f1_score_list.append(f1_score(y_truth, y_prediction, average='weighted'))
# Compute alpha confidence intervallen
stats[str(n_class)] = dict()
stats[str(n_class)]["Accuracy 95%:"] = str(compute_CI.compute_confidence(accuracy, N_1, N_2, alpha))
stats[str(n_class)]["AUC 95%:"] = str(compute_CI.compute_confidence(auc, N_1, N_2, alpha))
stats[str(n_class)]["F1-score 95%:"] = str(compute_CI.compute_confidence(f1_score_list, N_1, N_2, alpha))
stats[str(n_class)]["Precision 95%:"] = str(compute_CI.compute_confidence(precision, N_1, N_2, alpha))
stats[str(n_class)]["Sensitivity 95%: "] = str(compute_CI.compute_confidence(sensitivity, N_1, N_2, alpha))
stats[str(n_class)]["Specificity 95%:"] = str(compute_CI.compute_confidence(specificity, N_1, N_2, alpha))
print("Accuracy 95%:" + str(compute_CI.compute_confidence(accuracy, N_1, N_2, alpha)))
print("AUC 95%:" + str(compute_CI.compute_confidence(auc, N_1, N_2, alpha)))
print("F1-score 95%:" + str(compute_CI.compute_confidence(f1_score_list, N_1, N_2, alpha)))
print("Precision 95%:" + str(compute_CI.compute_confidence(precision, N_1, N_2, alpha)))
print("Sensitivity 95%: " + str(compute_CI.compute_confidence(sensitivity, N_1, N_2, alpha)))
print("Specificity 95%:" + str(compute_CI.compute_confidence(specificity, N_1, N_2, alpha)))
# Extract statistics on how often patients got classified correctly
alwaysright = dict()
alwayswrong = dict()
percentages = dict()
for i_ID in patient_classification_list:
percentage_right = patient_classification_list[i_ID]['N_correct'] / float(patient_classification_list[i_ID]['N_test'])
label = mutation_label[0][np.where(i_ID == patient_IDs)]
label = label[0][0]
if percentage_right == 1.0:
alwaysright[i_ID] = label
print(("Always Right: {}, label {}").format(i_ID, label))
elif percentage_right == 0:
alwayswrong[i_ID] = label
print(("Always Wrong: {}, label {}").format(i_ID, label))
else:
percentages[i_ID] = str(label) + ': ' + str(round(percentage_right, 2) * 100) + '%'
stats[str(n_class)]["Always right"] = alwaysright
stats[str(n_class)]["Always wrong"] = alwayswrong
stats[str(n_class)]['Percentages'] = percentages
if show_plots:
# Plot some characteristics in boxplots
import matplotlib.pyplot as plt
plt.figure()
plt.boxplot(accuracy)
plt.ylim([-0.05, 1.05])
plt.ylabel('Accuracy')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(auc)
plt.ylim([-0.05, 1.05])
plt.ylabel('AUC')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(precision)
plt.ylim([-0.05, 1.05])
plt.ylabel('Precision')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(sensitivity)
plt.ylim([-0.05, 1.05])
plt.ylabel('Sensitivity')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
plt.figure()
plt.boxplot(specificity)
plt.ylim([-0.05, 1.05])
plt.ylabel('Specificity')
plt.tick_params(
axis='x', # changes apply to the x-axis
which='both', # both major and minor ticks are affected
bottom='off', # ticks along the bottom edge are off
top='off', # ticks along the top edge are off
labelbottom='off') # labels along the bottom edge are off
plt.tight_layout()
plt.show()
return stats, predictions
def plot_single_SVR(prediction,
mutation_data,
label_type,
survival=False,
show_plots=False,
alpha=0.95):
if type(prediction) is not pd.core.frame.DataFrame:
if os.path.isfile(prediction):
prediction = pd.read_hdf(prediction)
keys = prediction.keys()
SVRs = list()
label = keys[0]
SVRs = prediction[label]['classifiers']
Y_test = prediction[label]['Y_test']
X_test = prediction[label]['X_test']
Y_train = prediction[label]['X_train']
if survival:
# Also extract time to event and if event occurs from mutation data
labels = [[label_type], ['E'], ['T']]
else:
labels = [[label_type]]
if type(mutation_data) is not dict:
if os.path.isfile(mutation_data):
mutation_data = gp.load_mutation_status(mutation_data, labels)
patient_IDs = mutation_data['patient_IDs']
mutation_label = mutation_data['mutation_label']
# Initialize scoring metrics
r2score = list()
MSE = list()
coefICC = list()
PearsonC = list()
PearsonP = list()
SpearmanC = list()
SpearmanP = list()
if survival:
cindex = list()
coxp = list()
coxcoef = list()
patient_MSE = dict()
for i in range(0, len(Y_test)):
test_patient_IDs = prediction[label]['patient_ID_test'][i]
# FIXME: Put some wrong patient IDs in test files
for num in range(0, len(test_patient_IDs)):
if 'features_' in test_patient_IDs[num]:
test_patient_IDs[num] = test_patient_IDs[num][9::]
if '__tpl.hdf5' in test_patient_IDs[num]:
test_patient_IDs[num] = test_patient_IDs[num][0:-10]
test_patient_IDs = np.asarray(test_patient_IDs)
X_temp = X_test[i]
test_indices = list()
for i_ID in test_patient_IDs:
# FIXME: Error in specific study
if i_ID == '112_recurrence-preop':
i_ID = '112_recurrence_preop'
test_indices.append(np.where(patient_IDs == i_ID)[0][0])
y_truth = [mutation_label[0][k][0] for k in test_indices]
if type(SVRs) == list or type(SVRs) == tuple:
estimator = SVRs[i]
else:
estimator = SVRs
scaler = estimator.best_scaler
try:
y_prediction = estimator.predict(scaler.transform(X_temp))
except ValueError:
y_prediction = estimator.predict(X_temp)
y_truth = np.asarray(y_truth)
# if survival:
# # Normalize the scores
# y_prediction = np.subtract(1.01, np.divide(y_prediction, np.max(y_prediction)))
print "Truth: ", y_truth
print "Prediction: ", y_prediction
# Compute error per patient
for i_truth, i_predict, i_test_ID in zip(y_truth, y_prediction,
test_patient_IDs):
if i_test_ID not in patient_MSE.keys():
patient_MSE[i_test_ID] = list()
patient_MSE[i_test_ID].append((i_truth - i_predict)**2)
# Compute evaluation metrics
r2score.append(r2_score(y_truth, y_prediction))
MSE.append(mean_squared_error(y_truth, y_prediction))
coefICC.append(ICC(np.column_stack((y_prediction, y_truth))))
C = pearsonr(y_prediction, y_truth)
PearsonC.append(C[0])
PearsonP.append(C[1])
C = spearmanr(y_prediction, y_truth)
SpearmanC.append(C.correlation)
SpearmanP.append(C.pvalue)
if survival:
# Extract time to event and event from label data
E_truth = np.asarray(
[mutation_label[1][k][0] for k in test_indices])
T_truth = np.asarray(
[mutation_label[2][k][0] for k in test_indices])
# Concordance index
cindex.append(
1 - ll.utils.concordance_index(T_truth, y_prediction, E_truth))
# Fit Cox model using SVR output, time to event and event
data = {'predict': y_prediction, 'E': E_truth, 'T': T_truth}
data = pd.DataFrame(data=data, index=test_patient_IDs)
cph = ll.CoxPHFitter()
cph.fit(data, duration_col='T', event_col='E')
coxcoef.append(cph.summary['coef']['predict'])
coxp.append(cph.summary['p']['predict'])
# Compute confidence intervals for given metrics
N_1 = float(len(Y_train[0]))
N_2 = float(len(Y_test[0]))
if len(r2score) == 1:
# No confidence intevals, just take the scores
stats = dict()
stats["r2_score:"] = str(r2score[0])
stats["MSE:"] = str(MSE[0])
stats["ICC:"] = str(coefICC[0])
stats["PearsonC:"] = str(PearsonC[0])
stats["SpearmanC: "] = str(SpearmanC[0])
stats["PearsonP:"] = str(PearsonP[0])
stats["SpearmanP: "] = str(SpearmanP[0])
if survival:
stats["Concordance:"] = str(cindex[0])
stats["Cox coef.:"] = str(coxcoef[0])
stats["Cox p:"] = str(coxp[0])
else:
# Compute confidence intervals from cross validations
stats = dict()
stats["r2_score 95%:"] = str(
compute_CI.compute_confidence(r2score, N_1, N_2, alpha))
stats["MSE 95%:"] = str(
compute_CI.compute_confidence(MSE, N_1, N_2, alpha))
stats["ICC 95%:"] = str(
compute_CI.compute_confidence(coefICC, N_1, N_2, alpha))
stats["PearsonC 95%:"] = str(
compute_CI.compute_confidence(PearsonC, N_1, N_2, alpha))
stats["SpearmanC 95%: "] = str(
compute_CI.compute_confidence(SpearmanC, N_1, N_2, alpha))
stats["PearsonP 95%:"] = str(
compute_CI.compute_confidence(PearsonP, N_1, N_2, alpha))
stats["SpearmanP 95%: "] = str(
compute_CI.compute_confidence(SpearmanP, N_1, N_2, alpha))
if survival:
stats["Concordance 95%:"] = str(
compute_CI.compute_confidence(cindex, N_1, N_2, alpha))
stats["Cox coef. 95%:"] = str(
compute_CI.compute_confidence(coxcoef, N_1, N_2, alpha))
stats["Cox p 95%:"] = str(
compute_CI.compute_confidence(coxp, N_1, N_2, alpha))
for k, v in stats.iteritems():
print k, v
# Calculate and sort individual patient MSE
patient_MSE = {k: np.mean(v) for k, v in patient_MSE.iteritems()}
order = np.argsort(patient_MSE.values())
sortedkeys = np.asarray(patient_MSE.keys())[order].tolist()
sortedvalues = np.asarray(patient_MSE.values())[order].tolist()
patient_MSE = [(k, v) for k, v in zip(sortedkeys, sortedvalues)]
for p in patient_MSE:
print p[0], p[1]
stats["Patient_MSE"] = patient_MSE
if show_plots:
# TODO: Plot metrics, see also plot_SVM
pass
return stats