def run_eval(probs, y_test, method=1, **kwparams):
if method == 1:
return caldis(probs, y_test)
elif method == 2:
return calplot(probs, y_test, **kwparams)
elif method == 3:
return report(probs, y_test)
elif method == 4:
return roc(probs, y_test, **kwparams)
elif method == 5:
return confusion(probs, y_test)
else:
raise Exception("Invalid method argument given")
def run_eval(probs, y_test, method=1, **kwparams):
if method == 1:
return caldis(probs, y_test)
elif method == 2:
return calplot(probs, y_test, **kwparams)
elif method == 3:
return report(probs, y_test)
elif method == 4:
return roc(probs, y_test, **kwparams)
elif method == 5:
return confusion(probs, y_test)
else:
raise Exception("Invalid method argument given")
def __roc(roc_results, labels, plabel, names, suffix):
print " - computing ROC graph and curves"
curves, graph = roc.roc(names, roc_results, labels, plabel)
print " - generating figures"
fig = pylab.figure()
ax = fig.add_subplot(111)
cu = []
for cx, cy in curves:
cu.append(ax.plot(cx, cy))
ax.set_xlim((0.0, 1.0))
ax.set_ylim((0.0, 1.0))
fig.legend(cu, names, 'lower right')
ax.set_ylabel('True positive rate')
ax.set_xlabel('False positive rate')
title = 'ROC curve per classifier'
title += '\n' + filename
if data_size != 0:
title += ' [Data size = ' + str(data_size) + ']'
ax.set_title(title)
pylab.savefig("images/" + filename + suffix + "_roc" + ".pdf")
pylab.savefig("images/" + filename + suffix + "_roc" + ".eps")
pylab.savefig("images/" + filename + suffix + "_roc" + ".svg")
fig = pylab.figure()
ax = fig.add_subplot(111)
cu = []
for cx, cy in graph:
cu.append(ax.plot([cx], [cy], marker='o'))
ax.set_xlim((0.0, 1.0))
ax.set_ylim((0.0, 1.0))
fig.legend(cu, names, 'lower right')
ax.set_ylabel('True positive rate')
ax.set_xlabel('False positive rate')
title = 'ROC graph per classifier'
title += '\n' + filename
if data_size != 0:
title += ' [Data size = ' + str(data_size) + ']'
ax.set_title(title)
pylab.savefig("images/" + filename + suffix + "_roc_graph" + ".pdf")
pylab.savefig("images/" + filename + suffix + "_roc_graph" + ".eps")
pylab.savefig("images/" + filename + suffix + "_roc_graph" + ".svg")
def plot_roc(P,GT):
from pylab import figure,xticks,yticks,axis,setp,gray,colorbar,savefig,gca,clf,plot,legend,xlabel,ylabel
from roc import roc
AUC=[]
CURVE=[]
for i,c in enumerate(test_classnames):
class_id = classnames.index(c)
tp,fp,auc=roc(None,GT==class_id, P[:,i] ) # larger is better
print "AUC: %s %5.3f" % (c,auc)
AUC.append(auc)
CURVE.append(array([fp,tp]))
order = argsort(AUC)[::-1]
styles=['-','-','-','-','-','-','-','--','--','--']
figure(figsize=(9,5))
for i in order:
c = test_classnames[i]
plot(CURVE[i][0],CURVE[i][1],label='%s (AUC: %3.2f)' % (c,AUC[i]),linewidth=3,linestyle=styles[i])
legend(loc='lower right')
xticks([0.0,0.2,0.4,0.6,0.8,1.0], [r'$0$', r'$0.2$',r'$0.4$',r'$0.6$',r'$0.8$',r'$1.0$'],fontsize=18)
yticks([0.0,0.2,0.4,0.6,0.8,1.0], [r'$0$', r'$0.2$',r'$0.4$',r'$0.6$',r'$0.8$',r'$1.0$'],fontsize=18)
xlabel('false negative rate',fontsize=18)
ylabel('true positive rate',fontsize=18)
savefig('AwA-ROC-DAP.pdf')
from roc import load_iris_data, cross_validate, knn, nb, lr, roc, svm
(XX,yy,y)=load_iris_data()
#classfiers_to_cv=[("kNN",knn),("Naive Bayes",nb), ("Linear Regression",lr)]
classfiers_to_cv=[("Naive Bayes",nb)]
classfiers_to_cv=[('SVM',svm)]
ROC = True
for (c_label, classifer) in classfiers_to_cv :
print
print "---> Current classifier: %s <---" % c_label
best_k=0
best_cv_a=0
for k_f in [2] :
if ROC:
outfile= 'roc.png'
roc(XX, yy, classifer, k_f, outfile)
print 'ROC plot file writen to %s' % outfile
break
if cv_a > best_cv_a :
best_cv_a=cv_a
best_k=k_f
print "fold <<%s>> :: acc <<%s>>" % (k_f, cv_a)
if not ROC:
print "\n %s Highest Accuracy: fold <<%s>> :: <<%s>>\n" % (c_label, best_k, best_cv_a)
def plot_roc(data):
data = roc(data, 5)
plot(data, ['roc'], 'roc')
def Experiment():
X, y = getExamples()
B, Y = genBags(y)
model = cuda(Model())
optmod = optim.Adam(model.parameters(),
lr=0.01,
weight_decay=0.000010,
betas=(0.9, 0.999))
episodes = 1000
aggop = torch.mean # torch.max #
L = []
Pid, Nid = np.where(Y == 1)[0], np.where(Y == -1)[0]
for e in range(episodes):
pidx, nidx = np.random.choice(Pid), np.random.choice(Nid)
Bp, Bn = X[B[pidx]], X[B[nidx]]
Zp, Zn = model(tensor(Bp)), model(tensor(Bn))
zp, zn = aggop(Zp), aggop(Zn)
zz = cuda(
Variable(torch.from_numpy(np.array([0.0
]))).type(torch.FloatTensor))
loss = torch.max(zz, (zn - zp + 1)) #hinge(1.0, zp)+hinge(-1.0, zn)#
Zpn = torch.cat((Zp, Zn))
Zpn = 1 / (1 + torch.exp(-Zpn))
Bpn = np.vstack((Bp, Bn))
K = tensor(np.exp(-0.01 * distance_matrix(Bpn, Bpn)))
s = torch.mean(K * ((Zpn - Zpn.T)**2))
tloss = loss + 1 * s
optmod.zero_grad()
tloss.backward(retain_graph=True)
optmod.step()
L.append([float(tloss), float(loss), float(s)])
#%%
plt.close("all")
plt.figure()
plt.plot(np.log(L))
for param in model.parameters():
param.requires_grad = False
def clf(inputs):
return model(tensor(inputs))
Z = np.array([float(aggop(clf(X[b]))) for b in B])
plt.figure()
plotit(X,
y,
clf=clf,
conts=[
np.min(Z), 0.5 * (np.mean(Z[Y == 1]) + np.mean(Z[Y == -1])),
np.max(Z)
],
colors='scaled')
for b in Pid:
plt.plot(X[B[b], 0], X[B[b], 1], '^')
for b in Nid:
plt.plot(X[B[b], 0], X[B[b], 1], 'v')
from roc import roc
fpr, tpr, auci = roc(list(y[y != 0]), list(clf(X[y != 0])))
plt.figure()
plt.plot(fpr, tpr)
plt.title("Instance level: " + str(auci))
plt.axis([0, 1, 0, 1])
fpr, tpr, aucg = roc(list(Y), list(Z))
plt.figure()
plt.plot(fpr, tpr)
fpr, tpr, auc = roc(list(Y), [aggop(tensor(y[b])) for b in B])
plt.plot(fpr, tpr)
plt.axis([0, 1, 0, 1])
plt.legend(
['group predictions: ' + str(aucg), 'group labels: ' + str(auc)])
return auci, aucg, auc
