def cross_val(x_train, y_train, x_test, y_test, folds=10):
skf = StratifiedKFold(n_splits=folds)
model = SVC(C=1, kernel='poly', random_state=47, verbose=True)
accuracy = []
mcc = []
precision = []
roc_auc = []
Sensitivity = []
Specificity = []
score = []
auc_score = []
f1 = []
for train_index, test_index in skf.split(x_train, y_train):
X_train, X_test = x_train[train_index], x_train[test_index]
Y_train, Y_test = y_train[train_index], y_train[test_index]
model.fit(X_train, Y_train)
y_predict = model.predict(X_test)
# print(y_predict)
score.append(model.score(X_test, Y_test))
accuracy.append(accuracy_score(Y_test, y_predict))
mcc.append(matthews_corrcoef(Y_test, y_predict))
precision.append(precision_score(Y_test, y_predict))
roc_auc.append(roc_auc_score(Y_test, y_predict))
auc_score.append(auc(Y_test, y_predict))
f1.append(f1_score(Y_test, y_predict))
Sensitivity.append(sensitivity(Y_test, y_predict))
Specificity.append(specificity(Y_test, y_predict))
res = 'Human SPD3\n'
res += "{} folds\n".format(folds)
res += "******************** Cross Validation Score ********************\n"
res += "Accuracy: {}\n".format(np.mean(accuracy))
res += "MCC: {}\n".format(np.mean(mcc))
res += "Precision: {}\n".format(np.mean(precision))
res += "Roc AUC score: {}\n".format(np.mean(roc_auc))
res += "AUC score: {}\n".format(np.mean(auc_score))
res += "F1 score: {}\n".format(np.mean(f1))
res += "Sensitivity: {}\n".format(np.mean(Sensitivity))
res += "Specifity: {}\n".format(np.mean(Specificity))
y_test_predict = model.predict(x_test)
res += "\n******************** Independent Test Score ********************\n"
res += "Accuracy: {}\n".format(accuracy_score(y_test, y_test_predict))
res += "MCC: {}\n".format(matthews_corrcoef(y_test, y_test_predict))
res += "Precision: {}\n".format(precision_score(y_test, y_test_predict))
res += "Roc AUC score: {}\n".format(roc_auc_score(y_test, y_test_predict))
res += "AUC score: {}\n".format(auc(y_test, y_test_predict))
res += "F1 score: {}\n".format(f1_score(y_test, y_test_predict))
res += "Sensitivity: {}\n".format(sensitivity(y_test, y_test_predict))
res += "Specifity: {}\n\n\n".format(specificity(y_test, y_test_predict))
res += '\n\n\n\n'
print(res)
with open('svm-poly.txt', 'a') as fp:
fp.write(res)
def load_model(x_train, y_train, x_test, y_test, folds):
skf = StratifiedKFold(n_splits=folds, random_state=47)
model = pickle.load(open('../data/rotation_forest_human.pkl', 'rb'))
accuracy = []
mcc = []
precision = []
roc_auc = []
Sensitivity = []
Specificity = []
auc_score = []
f1 = []
score = []
for x in list(range(10)):
for train_index, test_index in skf.split(x_train, y_train):
X_train, X_test = x_train[train_index], x_train[test_index]
Y_train, Y_test = y_train[train_index], y_train[test_index]
y_predict = model.predict(X_test)
score.append(model.score(X_test, Y_test))
accuracy.append(accuracy_score(Y_test, y_predict))
mcc.append(matthews_corrcoef(Y_test, y_predict))
precision.append(precision_score(Y_test, y_predict))
roc_auc.append(roc_auc_score(Y_test, y_predict))
auc_score.append(auc(Y_test, y_predict))
f1.append(f1_score(Y_test, y_predict))
Sensitivity.append(sensitivity(Y_test, y_predict))
Specificity.append(specificity(Y_test, y_predict))
res = "{} folds\n".format(folds)
res += "******************** Cross Validation Score ********************\n"
res += "Accuracy: {}\n".format(np.mean(accuracy))
res += "MCC: {}\n".format(np.mean(mcc))
res += "Precision: {}\n".format(np.mean(precision))
res += "Roc AUC score: {}\n".format(np.mean(roc_auc))
res += "AUC score: {}\n".format(np.mean(auc_score))
res += "F1 score: {}\n".format(np.mean(f1))
res += "Sensitivity: {}\n".format(np.mean(Sensitivity))
res += "Specifity: {}\n".format(np.mean(Specificity))
y_test_predict = model.predict(x_test)
res += "\n******************** Independent Test Score ********************\n"
res += "Accuracy: {}\n".format(accuracy_score(y_test, y_test_predict))
res += "MCC: {}\n".format(matthews_corrcoef(y_test, y_test_predict))
res += "Precision: {}\n".format(precision_score(y_test, y_test_predict))
res += "Roc AUC score: {}\n".format(roc_auc_score(y_test, y_test_predict))
res += "AUC score: {}\n".format(auc(y_test, y_test_predict))
res += "F1 score: {}\n".format(f1_score(y_test, y_test_predict))
res += "Sensitivity: {}\n".format(sensitivity(y_test, y_test_predict))
res += "Specifity: {}\n\n\n".format(specificity(y_test, y_test_predict))
print(res)
i_want_roc(model, x_test, y_test)
def kmal_sp(x_test, y_test):
model = pickle.load(open('../data/rotation_forest_human.pkl', 'rb'))
y_test_predict = model.predict(x_test)
with tf.Session() as sess:
sen = sess.run(sensitivity(y_test, y_test_predict))
spe = sess.run(specificity(y_test, y_test_predict))
res = "\n******************** Independent Test Score ********************\n"
res += "Accuracy: {}\n".format(accuracy_score(y_test, y_test_predict))
res += "MCC: {}\n".format(matthews_corrcoef(y_test, y_test_predict))
res += "Precision: {}\n".format(precision_score(y_test, y_test_predict))
res += "Roc AUC score: {}\n".format(roc_auc_score(y_test, y_test_predict))
res += "AUC score: {}\n".format(auc(y_test, y_test_predict))
res += "F1 score: {}\n".format(f1_score(y_test, y_test_predict))
res += "Sensitivity: {}\n".format(sen)
res += "Specifity: {}\n\n\n".format(spe)
print(res)
def cross_val(x_train, y_train):
skf = StratifiedKFold(n_splits=10)
model = RotationForestClassifier(n_estimators=100,
random_state=47,
verbose=4,
n_jobs=-2)
accuracy = []
mcc = []
precision = []
roc_auc = []
Sensitivity = []
Specificity = []
score = []
f1 = []
for x in range(10):
for train_index, test_index in skf.split(x_train, y_train):
X_train, X_test = x_train[train_index], x_train[test_index]
Y_train, Y_test = y_train[train_index], y_train[test_index]
model.fit(X_train, Y_train)
y_predict = model.predict(X_test)
score.append(model.score(X_test, Y_test))
accuracy.append(accuracy_score(Y_test, y_predict))
mcc.append(matthews_corrcoef(Y_test, y_predict))
precision.append(precision_score(Y_test, y_predict))
f1.append(f1_score(Y_test, y_predict))
roc_auc.append(roc_auc_score(Y_test, y_predict))
Sensitivity.append(sensitivity(Y_test, y_predict))
Specificity.append(specificity(Y_test, y_predict))
with open('./data/rotation_forest_knn_human100.pkl', 'wb') as f:
pickle.dump(model, f)
print("****************************************")
print("Accuracy: ", np.mean(accuracy))
print("MCC: ", np.mean(mcc))
print("Precision: ", np.mean(precision))
print("Roc auc score: ", np.mean(roc_auc))
print("F1 score: {}\n".format(np.mean(f1)))
print("Sensitivity: ", np.mean(Sensitivity))
print("Specifity: ", np.mean(Specificity))
def evaluate():
print(len(y_test), len(y_test_predict))
with tf.Session() as sess:
sen = sess.run(sensitivity(y_test, y_test_predict))
spe = sess.run(specificity(y_test, y_test_predict))
res = "\n******************** Independent Test Score ********************\n"
res += "Accuracy: {}\n".format(accuracy_score(y_test, y_test_predict))
res += "MCC: {}\n".format(matthews_corrcoef(y_test, y_test_predict))
res += "Precision: {}\n".format(
precision_score(y_test, y_test_predict, pos_label=1))
res += "Roc AUC score: {}\n".format(roc_auc_score(y_test, y_test_predict))
# res += "AUC score: {}\n".format(auc(y_test, y_test_predict))
res += "F1 score: {}\n".format(f1_score(y_test, y_test_predict))
res += "Sensitivity: {}\n".format(sen)
res += "Specifity: {}\n\n\n".format(spe)
print(res)
print(f"TP: {TP}, TN: {TN}, FP: {FP}, FN: {FN}")
def svm(x_train, y_train, x_test, y_test):
model = RotationForestClassifier(n_estimators=100,
random_state=47,
verbose=4)
# model = SVC(C=1.0, kernel='poly', gamma=1.0, random_state=47, verbose=True)
model.fit(x_train, y_train)
with open('./data/svm-knn.pkl', 'wb') as f:
pickle.dump(model, f)
y_predict = model.predict(x_test)
print("****************************************")
print("Accuracy: ", accuracy_score(y_test, y_predict))
print("MCC: ", matthews_corrcoef(y_test, y_predict))
print("Precision: ", precision_score(y_test, y_predict))
print("ROC auc score: ", roc_auc_score(y_test, y_predict))
print("AUC def: ", auc(y_test, y_predict))
print("F1 score: ", f1_score(y_test, y_predict))
print("Sensitivity: ", sensitivity(y_test, y_predict))
print("Specifity: ", specificity(y_test, y_predict))
def robust_cross_val(x_train, y_train, x_test, y_test, folds):
skf = StratifiedKFold(n_splits=folds, random_state=47)
model = RotationForestClassifier(n_estimators=100,
random_state=47,
verbose=4,
n_jobs=-2)
accuracy = []
mcc = []
precision = []
roc_auc = []
Sensitivity = []
Specificity = []
auc_score = []
f1 = []
score = []
for x in range(10):
for train_index, test_index in skf.split(x_train, y_train):
X_train, X_test = x_train[train_index], x_train[test_index]
Y_train, Y_test = y_train[train_index], y_train[test_index]
model.fit(X_train, Y_train)
y_predict = model.predict(X_test)
score.append(model.score(X_test, Y_test))
accuracy.append(accuracy_score(Y_test, y_predict))
mcc.append(matthews_corrcoef(Y_test, y_predict))
precision.append(precision_score(Y_test, y_predict))
roc_auc.append(roc_auc_score(Y_test, y_predict))
auc_score.append(auc(Y_test, y_predict))
f1.append(f1_score(Y_test, y_predict))
Sensitivity.append(sensitivity(Y_test, y_predict))
Specificity.append(specificity(Y_test, y_predict))
with open('../data/rotation_forest_human.pkl', 'wb') as f:
pickle.dump(model, f)
res = "{} folds\n".format(folds)
res += "******************** Cross Validation Score ********************\n"
res += "Accuracy: {}\n".format(np.mean(accuracy))
res += "MCC: {}\n".format(np.mean(mcc))
res += "Precision: {}\n".format(np.mean(precision))
res += "Roc AUC score: {}\n".format(np.mean(roc_auc))
res += "AUC score: {}\n".format(np.mean(auc_score))
res += "F1 score: {}\n".format(np.mean(f1))
res += "Sensitivity: {}\n".format(np.mean(Sensitivity))
res += "Specifity: {}\n".format(np.mean(Specificity))
y_test_predict = model.predict(x_test)
res += "\n******************** Independent Test Score ********************\n"
res += "Accuracy: {}\n".format(accuracy_score(y_test, y_test_predict))
res += "MCC: {}\n".format(matthews_corrcoef(y_test, y_test_predict))
res += "Precision: {}\n".format(precision_score(y_test, y_test_predict))
res += "Roc AUC score: {}\n".format(roc_auc_score(y_test, y_test_predict))
res += "AUC score: {}\n".format(auc(y_test, y_test_predict))
res += "F1 score: {}\n".format(f1_score(y_test, y_test_predict))
res += "Sensitivity: {}\n".format(sensitivity(y_test, y_test_predict))
res += "Specifity: {}\n\n\n".format(specificity(y_test, y_test_predict))
with open('../data/rotation_forest_human_result.txt', 'a') as f:
f.write(res)