ax1.hist(rec_errors_same, 250, facecolor='blue', alpha=0.4, histtype='stepfilled')
ax2.hist(rec_errors_diff, 250, facecolor='green', alpha=0.4, histtype='stepfilled')
ax3.hist(rec_errors_diff, 250, facecolor='green', alpha=0.4, histtype='stepfilled', normed=True)
ax3.hist(rec_errors_signal, 250, facecolor='red', alpha=0.4, histtype='stepfilled', normed=True)
# Plotting - raw variables
figA, axsA = plt.subplots(6, 6)
nColumn = 0
for axA in axsA.ravel():
axA.hist(X_train[:,nColumn], 250, facecolor='blue', alpha=0.4, histtype='stepfilled', normed=True)
#axA.hist(X_signal[:,nColumn][rec_errors_signal > 0.1], 250, facecolor='red', alpha=0.4, histtype='stepfilled', normed=True)
axA.hist(X_signal[:,nColumn], 250, facecolor='red', alpha=0.4, histtype='stepfilled', normed=True)
nColumn = nColumn+1
# Plotting - performance curves
true_positive,false_positive,precisions,recalls,f1s = makeMetrics(200,rec_errors_signal,rec_errors_diff)
print "Area under ROC = ",areaUnderROC(true_positive,false_positive)
figB, axsB = plt.subplots(1,2)
axB1,axB2 = axsB.ravel()
# ROC
axB1.plot(false_positive, true_positive, label='ROC curve')
axB1.plot([0, 1], [0, 1], 'k--')
axB1.set_xlim([0.0, 1.0])
axB1.set_ylim([0.0, 1.05])
axB1.set_xlabel('False Anomaly Rate')
axB1.set_ylabel('True Anomaly Rate')
# Precision, recall
axB2.plot(recalls, precisions, label='Precision-recall curve')
axB2.plot([0, 1.0], [0.5, 0.5], 'k--')
axB2.set_xlim([0.0, 1.0])
axB2.set_ylim([0.0, 1.05])
histtype='stepfilled')
ax3.hist(densities_test,
1000,
range=[-100.0, 100.0],
facecolor='green',
alpha=0.4,
histtype='stepfilled')
ax3.hist(densities_signal,
1000,
range=[-100.0, 100.0],
facecolor='red',
alpha=0.4,
histtype='stepfilled')
# ROC curve
true_positive, false_positive, precisions, recalls, f1s = makeMetrics(
5000, densities_signal, densities_test, reverse=True)
print "Area under ROC = ", areaUnderROC(true_positive, false_positive)
figB, axsB = plt.subplots(1, 2)
axB1, axB2 = axsB.ravel()
# ROC
axB1.plot(false_positive, true_positive, label='ROC curve')
axB1.plot([0, 1], [0, 1], 'k--')
axB1.set_xlim([0.0, 1.0])
axB1.set_ylim([0.0, 1.05])
axB1.set_xlabel('False Anomaly Rate')
axB1.set_ylabel('True Anomaly Rate')
# Precision, recall
axB2.plot(recalls, precisions, label='Precision-recall curve')
axB2.plot([0, 1.0], [0.5, 0.5], 'k--')
axB2.set_xlim([0.0, 1.0])
axB2.set_ylim([0.0, 1.05])
for fn in os.listdir('trained/'):
print "Processing ",fn
splitname = fn.split("_")
type = splitname[0]
nUnits = splitname[1]
learningRate = splitname[2]
nCycles = (splitname[3]).split('.')[0]
inputFile = 'trained/'+fn
if (type=='ae'):
nn = pickle.load(open(inputFile, 'rb'))
reconstucted_background = nn.predict(X_test_background)
reconstucted_signal = nn.predict(X_test_signal)
errors_background = reconstructionError(X_test_background,reconstucted_background)
errors_signal = reconstructionError(X_test_signal,reconstucted_signal)
true_positive,false_positive,precisions,recalls,f1s = makeMetrics(500,errors_signal,errors_background)
tmpAUC = areaUnderROC(true_positive,false_positive)
outputTextReport.write(type+' '+nUnits+' '+learningRate+' '+nCycles+' '+str(tmpAUC)+'\n')
if tmpAUC > bestAEScore:
bestAEScore = tmpAUC
bestAE = [tmpAUC,type,nUnits,learningRate,nCycles]
if (type=='cl'):
nn = pickle.load(open(inputFile, 'rb'))
predicted = nn.predict(X_test)
probabilities = nn.predict_proba(X_test)
fpr, tpr, thresholds = roc_curve(Y, probabilities[:,1], pos_label=1)
tmpAUC = roc_auc_score(Y, probabilities[:,1])
outputTextReport.write(type+' '+nUnits+' '+learningRate+' '+nCycles+' '+str(tmpAUC)+'\n')
if tmpAUC > bestCLScore:
bestCLScore = tmpAUC
# Density plotting
fig, axs = plt.subplots(3, 1)
ax1, ax2, ax3 = axs.ravel()
for ax in ax1, ax2, ax3:
ax.set_ylabel("Events")
ax.set_xlabel("Density")
ax1.hist(densities_train, 1000, facecolor='blue', alpha=0.4, histtype='stepfilled')
ax2.hist(densities_train, 1000, facecolor='blue', alpha=0.4, histtype='stepfilled')
ax2.hist(densities_test, 1000, facecolor='green', alpha=0.4, histtype='stepfilled')
ax3.hist(densities_test, 1000, range=[-100.0,100.0], facecolor='green', alpha=0.4, histtype='stepfilled')
ax3.hist(densities_signal, 1000, range=[-100.0,100.0], facecolor='red', alpha=0.4, histtype='stepfilled')
# ROC curve
true_positive,false_positive,precisions,recalls,f1s = makeMetrics(5000,densities_signal,densities_test,reverse=True)
print "Area under ROC = ",areaUnderROC(true_positive,false_positive)
figB, axsB = plt.subplots(1,2)
axB1,axB2 = axsB.ravel()
# ROC
axB1.plot(false_positive, true_positive, label='ROC curve')
axB1.plot([0, 1], [0, 1], 'k--')
axB1.set_xlim([0.0, 1.0])
axB1.set_ylim([0.0, 1.05])
axB1.set_xlabel('False Anomaly Rate')
axB1.set_ylabel('True Anomaly Rate')
# Precision, recall
axB2.plot(recalls, precisions, label='Precision-recall curve')
axB2.plot([0, 1.0], [0.5, 0.5], 'k--')
axB2.set_xlim([0.0, 1.0])
axB2.set_ylim([0.0, 1.05])
print "Processing ", fn
splitname = fn.split("_")
type = splitname[0]
nUnits = splitname[1]
learningRate = splitname[2]
nCycles = (splitname[3]).split('.')[0]
inputFile = 'trained/' + fn
if (type == 'ae'):
nn = pickle.load(open(inputFile, 'rb'))
reconstucted_background = nn.predict(X_test_background)
reconstucted_signal = nn.predict(X_test_signal)
errors_background = reconstructionError(X_test_background,
reconstucted_background)
errors_signal = reconstructionError(X_test_signal, reconstucted_signal)
true_positive, false_positive, precisions, recalls, f1s = makeMetrics(
500, errors_signal, errors_background)
tmpAUC = areaUnderROC(true_positive, false_positive)
outputTextReport.write(type + ' ' + nUnits + ' ' + learningRate + ' ' +
nCycles + ' ' + str(tmpAUC) + '\n')
if tmpAUC > bestAEScore:
bestAEScore = tmpAUC
bestAE = [tmpAUC, type, nUnits, learningRate, nCycles]
if (type == 'cl'):
nn = pickle.load(open(inputFile, 'rb'))
predicted = nn.predict(X_test)
probabilities = nn.predict_proba(X_test)
fpr, tpr, thresholds = roc_curve(Y, probabilities[:, 1], pos_label=1)
tmpAUC = roc_auc_score(Y, probabilities[:, 1])
outputTextReport.write(type + ' ' + nUnits + ' ' + learningRate + ' ' +
nCycles + ' ' + str(tmpAUC) + '\n')