def score(self, in_, out, method):
'''
Score the neural network on the provided input
'''
if not self.fit_yet:
raise ModelNotFit('Must train the neural network before scoring.')
else:
if method == 'accuracy':
cat_out = to_categorical(out, num_classes=self.num_classes)
return self.model.evaluate(in_, cat_out, batch_size=128)[1]
elif method == 'fscore':
pred = self.model.predict(in_)
pred = np.array([np.argmax(x) for x in pred])
score = f_score(out, pred)
return score
elif method == 'both':
cat_out = to_categorical(out, num_classes=self.num_classes)
accuracy = self.model.evaluate(in_, cat_out, batch_size=128)[1]
pred = self.model.predict(in_)
pred = np.array([np.argmax(x) for x in pred])
fscore = f_score(out, pred)
return accuracy, fscore
def eval(self, features, gt_scores, cps, gt_picks, n_frame_per_seg, n_frame):
self.SK.eval()
cps = cps.cpu().data.numpy()[0]
gt_picks = gt_picks.cpu().data.numpy()[0]
gt_scores = gt_scores.cpu().data.numpu()[0]
n_frame_per_seg = n_frame_per_seg.cpu().data.numpy()[0]
_, picks = self.SK(features.to("cuda"))
picks = picks.cpu().data.numpy()
pred_scores = np.zeros((gt_picks.shape[0], 1)) # (n, 1)
pred_scores[picks, :] = 1
pred_seg_scores = score_shot(
cps, pred_scores, gt_picks, n_frame_per_seg) # (n_cp, )
gt_seg_scores = score_shot(
cps, gt_scores, gt_picks, n_frame_per_seg) # (n_cp, )
# the length of the summary
length = int(n_frame.cpu().data.numpy()[0] * 0.15)
gt_seg_idx = knapsack(gt_seg_scores, n_frame_per_seg, length)
pred_seg_idx = knapsack(pred_seg_scores, n_frame_per_seg, length)
F = f_score(gt_seg_idx, pred_seg_idx, n_frame_per_seg,)
return F
def get_f_score(tp, fp, fn, tn):
try:
precision = tp / (tp + fp)
recall = tp / (tp + fn)
except ZeroDivisionError:
precision = 0.0
recall = 0.0
return f_score(precision, recall)
def score(self, in_, out, method):
'''
Score the model on the provided input
'''
if not self.fit_yet:
raise ModelNotFit('Must fit the Naive Bayes model before scoring.')
else:
if method == 'accuracy':
return self.model.score(in_, out)
elif method == 'fscore':
pred = self.model.predict(in_)
score = f_score(out, pred)
return score
elif method == 'both':
pred = self.model.predict(in_)
fscore = f_score(out, pred)
accuracy = self.model.score(in_, out)
return accuracy, fscore
def test(e, dataset):
if dataset == TestSet:
discriminator.load_state_dict(torch.load("Pmodel.tar"))
discriminator.eval()
preds = []
labels = []
pred_subtypes = []
golden_subtypes = []
for words, pos1, pos2, loc, loc_mark, subtype, argRole, maskL, maskM, maskR, label, subtype_golden in dataset.batchs_argProb(
):
words, pos1, pos2, loc, loc_mark, subtype, argRole, maskL, maskM, maskR, label = words.to(
device), pos1.to(device), pos2.to(device), loc.to(
device), loc_mark.to(device), subtype.to(device), argRole.to(
device), maskL.to(device), maskM.to(device), maskR.to(
device), label.to(device)
loss, scores, pred = discriminator(words, pos1, pos2, loc, loc_mark,
subtype, argRole, maskL, maskM,
maskR, label)
preds.append(pred[1].cpu().numpy())
labels.append(label.cpu().numpy())
pred_subtypes.append(subtype.cpu().numpy())
golden_subtypes.append(subtype_golden.cpu().numpy())
cnt = 0
cnt1 = 0
FN = 0
FP = 0
preds = np.concatenate(preds, 0)
labels = np.concatenate(labels, 0)
pred_subtypes = np.concatenate(pred_subtypes, 0)
golden_subtypes = np.concatenate(golden_subtypes, 0)
if dataset == TestSet or True:
for i in range(0, preds.shape[0]):
if labels[i] == 0:
cnt1 += 1
if preds[i] != labels[i]:
cnt += 1
if preds[i] == 0 and labels[i] != 0:
FN += 1
if preds[i] != 0 and labels[i] == 0:
FP += 1
print("EVAL %s #Wrong %d #NegToPos %d #PosToNeg %d #All %d #Negs %d" %
("Test", cnt, FP, FN, len(preds), cnt1))
acc, _, f1 = f_score(preds, labels, pred_subtypes, golden_subtypes)
print("acc:{}, f1:{}".format(acc, f1))
global bst
if f1 > bst and dataset == DevelopSet:
torch.save(discriminator.state_dict(), "Pmodel.tar")
bst = f1
print("BST: %f, epoch: %d" % (bst, e))
elif dataset == TestSet:
print("Test acc: %f, F1: %f" % (acc, f1))
return f1
def score(self, in_, out, method):
'''
Using the fit knn model, return the accuracy of the
model on the specified input data. Valid scoring methods
are 'accuracy', 'fscore', and 'both'
'''
if not self.fit_yet:
raise ModelNotFit('Must fit the Knn model before scoring.')
else:
if method == 'accuracy':
return self.model.score(in_, out)
elif method == 'fscore':
pred = self.model.predict(in_)
score = f_score(out, pred)
return score
elif method == 'both':
pred = self.model.predict(in_)
fscore = f_score(out, pred)
accuracy = self.model.score(in_, out)
return accuracy, fscore
def test(e, dataset):
if dataset == TestSet:
discriminator.load_state_dict(torch.load("Pmodel_bert.tar"))
discriminator.eval()
preds=[]
labels=[]
pred_subtypes=[]
golden_subtypes=[]
for words, inMask, maskL, maskM, maskR, label, ett_idx, ett_length, tri_idx, subtype, subtype_pred in dataset.batchs_argProb():
words, inMask, maskL, maskM, maskR, label = words.to(device), inMask.to(device), maskL.to(device), maskM.to(device), maskR.to(device), label.to(device)
loss, scores, pred = discriminator(words, inMask, maskL, maskM, maskR, label)
preds.append(pred[1].cpu().numpy())
labels.append(label.cpu().numpy())
pred_subtypes.extend(subtype_pred)
golden_subtypes.extend(subtype)
cnt=0
cnt1=0
FN=0
FP=0
preds=np.concatenate(preds,0)
labels=np.concatenate(labels,0)
pred_subtypes=np.array(pred_subtypes)
golden_subtypes=np.array(golden_subtypes)
if dataset == TestSet or True:
for i in range(0,preds.shape[0]):
if labels[i]==0:
cnt1+=1
if preds[i]!=labels[i]:
cnt+=1
if preds[i]==0 and labels[i]!=0:
FN+=1
if preds[i]!=0 and labels[i]==0:
FP+=1
print("EVAL %s #Wrong %d #NegToPos %d #PosToNeg %d #All %d #Negs %d"%("Test",cnt,FP,FN,len(preds),cnt1))
acc, _, f1 = f_score(preds, labels, pred_subtypes, golden_subtypes)
print ("acc:{}, f1:{}".format(acc,f1))
global bst, e_bst
if f1 > bst:
bst = f1
e_bst = e
torch.save(discriminator.state_dict(), "Pmodel_bert.tar")
print("BST: %f, epoch: %d"%(bst, e_bst))
return f1
def test(self, ts, batchsz=1, phase='Test', conll_file=None, txts=None):
total_correct = total_sum = fscore = 0
total_gold_count = total_guess_count = total_overlap_count = 0
start_time = time.time()
steps = int(math.floor(len(ts) / float(batchsz)))
# Only if they provide a file and the raw txts, we can write CONLL file
handle = None
if conll_file is not None and txts is not None:
handle = open(conll_file, "w")
for i in range(steps):
ts_i = batch(ts, i, batchsz)
correct, count, overlaps, golds, guesses = self._batch(
ts_i, handle, txts)
total_correct += correct
total_sum += count
total_gold_count += golds
total_guess_count += guesses
total_overlap_count += overlaps
duration = time.time() - start_time
total_acc = total_correct / float(total_sum)
# Only show the fscore if requested
if self.fscore > 0:
fscore = f_score(total_overlap_count, total_gold_count,
total_guess_count, self.fscore)
print('%s (F%d = %.4f) (Acc %d/%d = %.4f) (%.3f sec)' %
(phase, self.fscore, fscore, total_correct, total_sum,
total_acc, duration))
else:
print('%s (Acc %d/%d = %.4f) (%.3f sec)' %
(phase, total_correct, total_sum, total_acc, duration))
if handle is not None:
handle.close()
return total_acc, fscore
x2 = D.get_data('Data1/Class2.txt')
x3 = D.get_data('Data1/Class3.txt')
if not os.path.isdir("plots_multi"):
os.mkdir("plots_multi")
w1 = [200.0, -200.0, 200.0]
w2 = [-200.0, 200.0, 200.0]
w3 = [200.0, 200.0, -200.0]
perceptron(w1, x1, x2, x3, 0.5, 100, 1)
perceptron(w2, x2, x3, x1, 0.5, 100, 2)
perceptron(w3, x3, x1, x2, 0.5, 100, 3)
plt.figure()
decision_boundary(plt, w1, w2, w3)
conf_mat = get_conf(x1, x2, x3, w1, w2, w3, plt)
plt.savefig('plots_multi/decision_boundary.png')
print(conf_mat)
print("Accuracy: ", U.accuracy(conf_mat))
print("Precision for class 1: ", U.precision(conf_mat, 0))
print("Precision for class 2: ", U.precision(conf_mat, 1))
print("Precision for class 3: ", U.precision(conf_mat, 2))
print("Recall for class 1: ", U.recall(conf_mat, 0))
print("Recall for class 2: ", U.recall(conf_mat, 1))
print("Recall for class 3: ", U.recall(conf_mat, 2))
print("F-Score for class 1: ", U.f_score(conf_mat, 0))
print("F-Score for class 2: ", U.f_score(conf_mat, 1))
print("F-Score for class 3: ", U.f_score(conf_mat, 2))
def gibbs(dataset):
discriminator.eval()
get_prob.eval()
# get_prob2.eval()
preds = []
labels = []
subtypes = []
subtypes_pred = []
senLabels = []
ettIdxs = []
ettLengths = []
triIdxs = []
triLengths = []
results = []
words_ori = []
# get the pred0
for words, inMask, maskL, maskM, maskR, label, ettIdx, ettLength, triIdx, triLength, subtype, senLabel, subtype, subtype_pred in dataset.batchs_gibbs(
):
words, inMask, maskL, maskM, maskR, label = words.to(
device), inMask.to(device), maskL.to(device), maskM.to(
device), maskR.to(device), label.to(device)
loss, scores, pred = discriminator(words, inMask, maskL, maskM, maskR,
label)
preds.extend(pred[1].cpu().numpy())
labels.extend(label.cpu().numpy())
words_ori.extend(words)
senLabels.extend(senLabel)
subtypes.extend(subtype)
subtypes_pred.extend(subtype_pred)
ettIdxs.extend(ettIdx)
ettLengths.extend(ettLength)
triIdxs.extend(triIdx)
triLengths.extend(triLength)
for i in range(len(words)):
state = {}
results.append(state)
preds0 = copy.deepcopy(preds)
# Transfer for N+K times
for trans_time in range(int(1 / k_an)):
if trans_time % 5 == 0:
cnt = 0
cnt1 = 0
FN = 0
FP = 0
for i in range(len(preds)):
if labels[i] == 0:
cnt1 += 1
if preds[i] != labels[i]:
cnt += 1
if preds[i] == 0 and labels[i] != 0:
FN += 1
if preds[i] != 0 and labels[i] == 0:
FP += 1
print(
"EVAL %s #Wrong %d #NegToPos %d #PosToNeg %d #All %d #Negs %d"
% ("Test", cnt, FP, FN, len(preds), cnt1))
acc, _, f1 = f_score(preds, labels, subtypes_pred, subtypes)
print("trans_time:{}, acc:{}, f1:{}".format(trans_time, acc, f1))
# generate the argRoles
L = 0
R = 0
sen_idx = 0
Aword_prob = []
Ain_mask_prob = []
AmaskR_prob = []
AmaskL_prob = []
AmaskM_prob = []
for i in range(len(senLabels)):
if senLabels[i] == sen_idx:
R += 1
else:
sen_idx = senLabels[i]
if L != R:
Aword, Ain_mask, AmaskR, AmaskM, AmaskL = gen_argRole(
words_ori[L:R], preds[L:R], ettIdxs[L:R],
ettLengths[L:R], triIdxs[L:R], triLengths[L:R],
subtypes[L:R])
Aword_prob.extend(Aword)
Ain_mask_prob.extend(Ain_mask)
AmaskR_prob.extend(AmaskR)
AmaskM_prob.extend(AmaskM)
AmaskL_prob.extend(AmaskL)
L = R
R += 1
if i == len(senLabels) - 1:
Aword, Ain_mask, AmaskR, AmaskM, AmaskL = gen_argRole(
words_ori[L:R], preds[L:R], ettIdxs[L:R], ettLengths[L:R],
triIdxs[L:R], triLengths[L:R], subtypes[L:R])
Aword_prob.extend(Aword)
Ain_mask_prob.extend(Ain_mask)
AmaskR_prob.extend(AmaskR)
AmaskM_prob.extend(AmaskM)
AmaskL_prob.extend(AmaskL)
# get prob
probs = []
probs_max = []
probs_argmax = []
words_sum = []
Aword_prob = np.array(Aword_prob)
Ain_mask_prob = np.array(Ain_mask_prob)
AmaskR_prob = np.array(AmaskR_prob)
AmaskM_prob = np.array(AmaskM_prob)
AmaskL_prob = np.array(AmaskL_prob)
L = 0
for i in range(0, len(Aword_prob) // BatchSize_arg + 1):
L = i * BatchSize_arg
if L >= len(Aword_prob):
break
R = min((i + 1) * BatchSize_arg, len(Aword_prob))
words, inMask, maskR, maskM, maskL, label = torch.LongTensor(
Aword_prob[L:R]), torch.LongTensor(
Ain_mask_prob[L:R]), torch.FloatTensor(
AmaskR_prob[L:R]), torch.FloatTensor(
AmaskM_prob[L:R]), torch.FloatTensor(
AmaskL_prob[L:R]), torch.LongTensor(
labels[L:R])
words, inMask, maskR, maskM, maskL, label = words.to(
device), inMask.to(device), maskR.to(device), maskM.to(
device), maskL.to(device), label.to(device)
_, prob, _ = get_prob(words, inMask, maskR, maskM, maskL, label)
prob_max, prob_argmax = torch.max(prob, dim=1)
prob_max = prob_max.detach().cpu().numpy().tolist()
prob_argmax = prob_argmax.detach().cpu().numpy().tolist()
prob = prob.detach().cpu().numpy().tolist()
words_sum.extend(words)
probs.extend(prob)
probs_max.extend(prob_max)
probs_argmax.extend(prob_argmax)
# transfer and sum the states
probs_max_an = []
sen_idx = 0
L = 0
R = 0
for i, prob in enumerate(probs):
if senLabels[i] == sen_idx:
R += 1
else:
sen_idx = senLabels[i]
if L != R:
probMax_sum = 0
probs_max_an = []
for idx in range(L, R):
probMax_sum += probs_max[idx]**(
1 / (1 - k_an * trans_time))
for idx in range(L, R):
probs_max_an.append(
probs_max[idx]**(1 / (1 - k_an * trans_time)) /
probMax_sum)
idx_trans = random_index(probs_max_an) + L
preds[idx_trans] = probs_argmax[idx_trans]
L = R
R += 1
# print the results
sen_idx = 0
L = 0
R = 0
for i in range(len(senLabels)):
if senLabels[i] == sen_idx:
R += 1
else:
sen_idx = senLabels[i]
if L != R:
pred1 = np.zeros((R - L), dtype=np.int64)
print('---sentence%d---' % sen_idx)
print('pred0:', preds0[L:R])
print('label:', labels[L:R])
print('result:', preds[L:R])
L = R
R += 1
def gibbs(dataset):
discriminator.eval()
get_prob.eval()
# get_prob2.eval()
preds = []
labels = []
pred_subtypes = []
golden_subtypes = []
senLabels = []
ettIdxs = []
ettLengths = []
results = []
# get the pred0
for words, pos1, pos2, loc, loc_mark, subtype, maskL, maskM, maskR, label, ettIdx, ettLength, senLabel, subtype_golden in dataset.batchs_gibbs(
):
words, pos1, pos2, loc, loc_mark, subtype, maskL, maskM, maskR, label, ettIdx, ettLength, senLabel = words.to(
device), pos1.to(device), pos2.to(device), loc.to(
device), loc_mark.to(device), subtype.to(device), maskL.to(
device), maskM.to(device), maskR.to(device), label.to(
device), ettIdx.to(device), ettLength.to(
device), senLabel.to(device)
loss, scores, pred = discriminator(words, pos1, pos2, loc, loc_mark,
subtype, maskL, maskM, maskR, label)
preds.extend(pred[1].cpu().numpy())
labels.extend(label.cpu().numpy())
pred_subtypes.extend(subtype.cpu().numpy())
golden_subtypes.extend(subtype_golden.cpu().numpy())
senLabels.extend(senLabel.cpu().numpy().tolist())
ettIdxs.extend(ettIdx)
ettLengths.extend(ettLength)
for i in range(len(words)):
state = {}
results.append(state)
preds0 = copy.deepcopy(preds)
# Transfer for N+K times
global argRoles
for trans_time in range(int(1 / k_an) - 1):
if trans_time % 10 == 0:
cnt = 0
cnt1 = 0
FN = 0
FP = 0
for i in range(len(preds)):
if labels[i] == 0:
cnt1 += 1
if preds[i] != labels[i]:
cnt += 1
if preds[i] == 0 and labels[i] != 0:
FN += 1
if preds[i] != 0 and labels[i] == 0:
FP += 1
print(
"EVAL %s #Wrong %d #NegToPos %d #PosToNeg %d #All %d #Negs %d"
% ("Test", cnt, FP, FN, len(preds), cnt1))
acc, _, f1 = f_score(preds, labels, pred_subtypes, golden_subtypes)
print("trans_time:{}, acc:{}, f1:{}".format(trans_time, acc, f1))
# generate the argRoles
L = 0
R = 0
sen_idx = senLabels[0]
argRoles = []
for i in range(len(senLabels)):
if senLabels[i] == sen_idx:
R += 1
else:
sen_idx = senLabels[i]
if L != R:
gen_argRole(preds[L:R], ettIdxs[L:R], ettLengths[L:R])
L = R
R += 1
sen_idx = senLabels[i]
if L < len(preds):
gen_argRole(preds[L:], ettIdxs[L:], ettLengths[L:])
# get prob
probs = []
probs_max = []
probs_argmax = []
words_sum = []
L = 0
for words, pos1, pos2, loc, loc_mark, subtype, maskL, maskM, maskR, label, ettIdx, ettLength, senLabel, subtype_golden in dataset.batchs_gibbs(
):
argRole = torch.LongTensor(argRoles[L:L + len(words)]).to(device)
L += len(words)
words, pos1, pos2, loc, loc_mark, subtype, maskL, maskM, maskR, label, ettIdx, ettLength, senLabel = words.to(
device), pos1.to(device), pos2.to(device), loc.to(
device), loc_mark.to(device), subtype.to(device), maskL.to(
device), maskM.to(device), maskR.to(device), label.to(
device), ettIdx.to(device), ettLength.to(
device), senLabel.to(device)
_, prob, _ = get_prob(words, pos1, pos2, loc, loc_mark, subtype,
argRole, maskL, maskM, maskR, label)
prob_max, prob_argmax = torch.max(prob, dim=1)
prob_max = prob_max.detach().cpu().numpy().tolist()
prob_argmax = prob_argmax.detach().cpu().numpy().tolist()
prob = prob.detach().cpu().numpy().tolist()
words_sum.extend(words)
probs.extend(prob)
probs_max.extend(prob_max)
probs_argmax.extend(prob_argmax)
# transfer and sum the states
probs_max_an = []
sen_idx = 0
L = 0
R = 0
for i, prob in enumerate(probs):
if senLabels[i] == sen_idx:
R += 1
else:
sen_idx = senLabels[i]
if L != R:
probMax_sum = 0
probs_max_an = []
for idx in range(L, R):
probMax_sum += probs_max[idx]**(
1 / (1 - k_an * trans_time))
for idx in range(L, R):
probs_max_an.append(
probs_max[idx]**(1 / (1 - k_an * trans_time)) /
probMax_sum)
idx_trans = random_index(probs_max_an) + L
preds[idx_trans] = probs_argmax[idx_trans]
L = R
R += 1
# print the results
sen_idx = 0
L = 0
R = 0
for i in range(len(senLabels)):
if senLabels[i] == sen_idx:
R += 1
else:
sen_idx = senLabels[i]
if L != R:
pred1 = np.zeros((R - L), dtype=np.int64)
print('---sentence%d---' % sen_idx)
print('pred0:', preds0[L:R])
print('label:', labels[L:R])
print('result:', preds[L:R])
L = R
R += 1
