def score(self, in_, out, method): ''' Score the neural network on the provided input ''' if not self.fit_yet: raise ModelNotFit('Must train the neural network before scoring.') else: if method == 'accuracy': cat_out = to_categorical(out, num_classes=self.num_classes) return self.model.evaluate(in_, cat_out, batch_size=128)[1] elif method == 'fscore': pred = self.model.predict(in_) pred = np.array([np.argmax(x) for x in pred]) score = f_score(out, pred) return score elif method == 'both': cat_out = to_categorical(out, num_classes=self.num_classes) accuracy = self.model.evaluate(in_, cat_out, batch_size=128)[1] pred = self.model.predict(in_) pred = np.array([np.argmax(x) for x in pred]) fscore = f_score(out, pred) return accuracy, fscore
def eval(self, features, gt_scores, cps, gt_picks, n_frame_per_seg, n_frame): self.SK.eval() cps = cps.cpu().data.numpy()[0] gt_picks = gt_picks.cpu().data.numpy()[0] gt_scores = gt_scores.cpu().data.numpu()[0] n_frame_per_seg = n_frame_per_seg.cpu().data.numpy()[0] _, picks = self.SK(features.to("cuda")) picks = picks.cpu().data.numpy() pred_scores = np.zeros((gt_picks.shape[0], 1)) # (n, 1) pred_scores[picks, :] = 1 pred_seg_scores = score_shot( cps, pred_scores, gt_picks, n_frame_per_seg) # (n_cp, ) gt_seg_scores = score_shot( cps, gt_scores, gt_picks, n_frame_per_seg) # (n_cp, ) # the length of the summary length = int(n_frame.cpu().data.numpy()[0] * 0.15) gt_seg_idx = knapsack(gt_seg_scores, n_frame_per_seg, length) pred_seg_idx = knapsack(pred_seg_scores, n_frame_per_seg, length) F = f_score(gt_seg_idx, pred_seg_idx, n_frame_per_seg,) return F
def get_f_score(tp, fp, fn, tn): try: precision = tp / (tp + fp) recall = tp / (tp + fn) except ZeroDivisionError: precision = 0.0 recall = 0.0 return f_score(precision, recall)
def score(self, in_, out, method): ''' Score the model on the provided input ''' if not self.fit_yet: raise ModelNotFit('Must fit the Naive Bayes model before scoring.') else: if method == 'accuracy': return self.model.score(in_, out) elif method == 'fscore': pred = self.model.predict(in_) score = f_score(out, pred) return score elif method == 'both': pred = self.model.predict(in_) fscore = f_score(out, pred) accuracy = self.model.score(in_, out) return accuracy, fscore
def test(e, dataset): if dataset == TestSet: discriminator.load_state_dict(torch.load("Pmodel.tar")) discriminator.eval() preds = [] labels = [] pred_subtypes = [] golden_subtypes = [] for words, pos1, pos2, loc, loc_mark, subtype, argRole, maskL, maskM, maskR, label, subtype_golden in dataset.batchs_argProb( ): words, pos1, pos2, loc, loc_mark, subtype, argRole, maskL, maskM, maskR, label = words.to( device), pos1.to(device), pos2.to(device), loc.to( device), loc_mark.to(device), subtype.to(device), argRole.to( device), maskL.to(device), maskM.to(device), maskR.to( device), label.to(device) loss, scores, pred = discriminator(words, pos1, pos2, loc, loc_mark, subtype, argRole, maskL, maskM, maskR, label) preds.append(pred[1].cpu().numpy()) labels.append(label.cpu().numpy()) pred_subtypes.append(subtype.cpu().numpy()) golden_subtypes.append(subtype_golden.cpu().numpy()) cnt = 0 cnt1 = 0 FN = 0 FP = 0 preds = np.concatenate(preds, 0) labels = np.concatenate(labels, 0) pred_subtypes = np.concatenate(pred_subtypes, 0) golden_subtypes = np.concatenate(golden_subtypes, 0) if dataset == TestSet or True: for i in range(0, preds.shape[0]): if labels[i] == 0: cnt1 += 1 if preds[i] != labels[i]: cnt += 1 if preds[i] == 0 and labels[i] != 0: FN += 1 if preds[i] != 0 and labels[i] == 0: FP += 1 print("EVAL %s #Wrong %d #NegToPos %d #PosToNeg %d #All %d #Negs %d" % ("Test", cnt, FP, FN, len(preds), cnt1)) acc, _, f1 = f_score(preds, labels, pred_subtypes, golden_subtypes) print("acc:{}, f1:{}".format(acc, f1)) global bst if f1 > bst and dataset == DevelopSet: torch.save(discriminator.state_dict(), "Pmodel.tar") bst = f1 print("BST: %f, epoch: %d" % (bst, e)) elif dataset == TestSet: print("Test acc: %f, F1: %f" % (acc, f1)) return f1
def score(self, in_, out, method): ''' Using the fit knn model, return the accuracy of the model on the specified input data. Valid scoring methods are 'accuracy', 'fscore', and 'both' ''' if not self.fit_yet: raise ModelNotFit('Must fit the Knn model before scoring.') else: if method == 'accuracy': return self.model.score(in_, out) elif method == 'fscore': pred = self.model.predict(in_) score = f_score(out, pred) return score elif method == 'both': pred = self.model.predict(in_) fscore = f_score(out, pred) accuracy = self.model.score(in_, out) return accuracy, fscore
def test(e, dataset): if dataset == TestSet: discriminator.load_state_dict(torch.load("Pmodel_bert.tar")) discriminator.eval() preds=[] labels=[] pred_subtypes=[] golden_subtypes=[] for words, inMask, maskL, maskM, maskR, label, ett_idx, ett_length, tri_idx, subtype, subtype_pred in dataset.batchs_argProb(): words, inMask, maskL, maskM, maskR, label = words.to(device), inMask.to(device), maskL.to(device), maskM.to(device), maskR.to(device), label.to(device) loss, scores, pred = discriminator(words, inMask, maskL, maskM, maskR, label) preds.append(pred[1].cpu().numpy()) labels.append(label.cpu().numpy()) pred_subtypes.extend(subtype_pred) golden_subtypes.extend(subtype) cnt=0 cnt1=0 FN=0 FP=0 preds=np.concatenate(preds,0) labels=np.concatenate(labels,0) pred_subtypes=np.array(pred_subtypes) golden_subtypes=np.array(golden_subtypes) if dataset == TestSet or True: for i in range(0,preds.shape[0]): if labels[i]==0: cnt1+=1 if preds[i]!=labels[i]: cnt+=1 if preds[i]==0 and labels[i]!=0: FN+=1 if preds[i]!=0 and labels[i]==0: FP+=1 print("EVAL %s #Wrong %d #NegToPos %d #PosToNeg %d #All %d #Negs %d"%("Test",cnt,FP,FN,len(preds),cnt1)) acc, _, f1 = f_score(preds, labels, pred_subtypes, golden_subtypes) print ("acc:{}, f1:{}".format(acc,f1)) global bst, e_bst if f1 > bst: bst = f1 e_bst = e torch.save(discriminator.state_dict(), "Pmodel_bert.tar") print("BST: %f, epoch: %d"%(bst, e_bst)) return f1
def test(self, ts, batchsz=1, phase='Test', conll_file=None, txts=None): total_correct = total_sum = fscore = 0 total_gold_count = total_guess_count = total_overlap_count = 0 start_time = time.time() steps = int(math.floor(len(ts) / float(batchsz))) # Only if they provide a file and the raw txts, we can write CONLL file handle = None if conll_file is not None and txts is not None: handle = open(conll_file, "w") for i in range(steps): ts_i = batch(ts, i, batchsz) correct, count, overlaps, golds, guesses = self._batch( ts_i, handle, txts) total_correct += correct total_sum += count total_gold_count += golds total_guess_count += guesses total_overlap_count += overlaps duration = time.time() - start_time total_acc = total_correct / float(total_sum) # Only show the fscore if requested if self.fscore > 0: fscore = f_score(total_overlap_count, total_gold_count, total_guess_count, self.fscore) print('%s (F%d = %.4f) (Acc %d/%d = %.4f) (%.3f sec)' % (phase, self.fscore, fscore, total_correct, total_sum, total_acc, duration)) else: print('%s (Acc %d/%d = %.4f) (%.3f sec)' % (phase, total_correct, total_sum, total_acc, duration)) if handle is not None: handle.close() return total_acc, fscore
x2 = D.get_data('Data1/Class2.txt') x3 = D.get_data('Data1/Class3.txt') if not os.path.isdir("plots_multi"): os.mkdir("plots_multi") w1 = [200.0, -200.0, 200.0] w2 = [-200.0, 200.0, 200.0] w3 = [200.0, 200.0, -200.0] perceptron(w1, x1, x2, x3, 0.5, 100, 1) perceptron(w2, x2, x3, x1, 0.5, 100, 2) perceptron(w3, x3, x1, x2, 0.5, 100, 3) plt.figure() decision_boundary(plt, w1, w2, w3) conf_mat = get_conf(x1, x2, x3, w1, w2, w3, plt) plt.savefig('plots_multi/decision_boundary.png') print(conf_mat) print("Accuracy: ", U.accuracy(conf_mat)) print("Precision for class 1: ", U.precision(conf_mat, 0)) print("Precision for class 2: ", U.precision(conf_mat, 1)) print("Precision for class 3: ", U.precision(conf_mat, 2)) print("Recall for class 1: ", U.recall(conf_mat, 0)) print("Recall for class 2: ", U.recall(conf_mat, 1)) print("Recall for class 3: ", U.recall(conf_mat, 2)) print("F-Score for class 1: ", U.f_score(conf_mat, 0)) print("F-Score for class 2: ", U.f_score(conf_mat, 1)) print("F-Score for class 3: ", U.f_score(conf_mat, 2))
def gibbs(dataset): discriminator.eval() get_prob.eval() # get_prob2.eval() preds = [] labels = [] subtypes = [] subtypes_pred = [] senLabels = [] ettIdxs = [] ettLengths = [] triIdxs = [] triLengths = [] results = [] words_ori = [] # get the pred0 for words, inMask, maskL, maskM, maskR, label, ettIdx, ettLength, triIdx, triLength, subtype, senLabel, subtype, subtype_pred in dataset.batchs_gibbs( ): words, inMask, maskL, maskM, maskR, label = words.to( device), inMask.to(device), maskL.to(device), maskM.to( device), maskR.to(device), label.to(device) loss, scores, pred = discriminator(words, inMask, maskL, maskM, maskR, label) preds.extend(pred[1].cpu().numpy()) labels.extend(label.cpu().numpy()) words_ori.extend(words) senLabels.extend(senLabel) subtypes.extend(subtype) subtypes_pred.extend(subtype_pred) ettIdxs.extend(ettIdx) ettLengths.extend(ettLength) triIdxs.extend(triIdx) triLengths.extend(triLength) for i in range(len(words)): state = {} results.append(state) preds0 = copy.deepcopy(preds) # Transfer for N+K times for trans_time in range(int(1 / k_an)): if trans_time % 5 == 0: cnt = 0 cnt1 = 0 FN = 0 FP = 0 for i in range(len(preds)): if labels[i] == 0: cnt1 += 1 if preds[i] != labels[i]: cnt += 1 if preds[i] == 0 and labels[i] != 0: FN += 1 if preds[i] != 0 and labels[i] == 0: FP += 1 print( "EVAL %s #Wrong %d #NegToPos %d #PosToNeg %d #All %d #Negs %d" % ("Test", cnt, FP, FN, len(preds), cnt1)) acc, _, f1 = f_score(preds, labels, subtypes_pred, subtypes) print("trans_time:{}, acc:{}, f1:{}".format(trans_time, acc, f1)) # generate the argRoles L = 0 R = 0 sen_idx = 0 Aword_prob = [] Ain_mask_prob = [] AmaskR_prob = [] AmaskL_prob = [] AmaskM_prob = [] for i in range(len(senLabels)): if senLabels[i] == sen_idx: R += 1 else: sen_idx = senLabels[i] if L != R: Aword, Ain_mask, AmaskR, AmaskM, AmaskL = gen_argRole( words_ori[L:R], preds[L:R], ettIdxs[L:R], ettLengths[L:R], triIdxs[L:R], triLengths[L:R], subtypes[L:R]) Aword_prob.extend(Aword) Ain_mask_prob.extend(Ain_mask) AmaskR_prob.extend(AmaskR) AmaskM_prob.extend(AmaskM) AmaskL_prob.extend(AmaskL) L = R R += 1 if i == len(senLabels) - 1: Aword, Ain_mask, AmaskR, AmaskM, AmaskL = gen_argRole( words_ori[L:R], preds[L:R], ettIdxs[L:R], ettLengths[L:R], triIdxs[L:R], triLengths[L:R], subtypes[L:R]) Aword_prob.extend(Aword) Ain_mask_prob.extend(Ain_mask) AmaskR_prob.extend(AmaskR) AmaskM_prob.extend(AmaskM) AmaskL_prob.extend(AmaskL) # get prob probs = [] probs_max = [] probs_argmax = [] words_sum = [] Aword_prob = np.array(Aword_prob) Ain_mask_prob = np.array(Ain_mask_prob) AmaskR_prob = np.array(AmaskR_prob) AmaskM_prob = np.array(AmaskM_prob) AmaskL_prob = np.array(AmaskL_prob) L = 0 for i in range(0, len(Aword_prob) // BatchSize_arg + 1): L = i * BatchSize_arg if L >= len(Aword_prob): break R = min((i + 1) * BatchSize_arg, len(Aword_prob)) words, inMask, maskR, maskM, maskL, label = torch.LongTensor( Aword_prob[L:R]), torch.LongTensor( Ain_mask_prob[L:R]), torch.FloatTensor( AmaskR_prob[L:R]), torch.FloatTensor( AmaskM_prob[L:R]), torch.FloatTensor( AmaskL_prob[L:R]), torch.LongTensor( labels[L:R]) words, inMask, maskR, maskM, maskL, label = words.to( device), inMask.to(device), maskR.to(device), maskM.to( device), maskL.to(device), label.to(device) _, prob, _ = get_prob(words, inMask, maskR, maskM, maskL, label) prob_max, prob_argmax = torch.max(prob, dim=1) prob_max = prob_max.detach().cpu().numpy().tolist() prob_argmax = prob_argmax.detach().cpu().numpy().tolist() prob = prob.detach().cpu().numpy().tolist() words_sum.extend(words) probs.extend(prob) probs_max.extend(prob_max) probs_argmax.extend(prob_argmax) # transfer and sum the states probs_max_an = [] sen_idx = 0 L = 0 R = 0 for i, prob in enumerate(probs): if senLabels[i] == sen_idx: R += 1 else: sen_idx = senLabels[i] if L != R: probMax_sum = 0 probs_max_an = [] for idx in range(L, R): probMax_sum += probs_max[idx]**( 1 / (1 - k_an * trans_time)) for idx in range(L, R): probs_max_an.append( probs_max[idx]**(1 / (1 - k_an * trans_time)) / probMax_sum) idx_trans = random_index(probs_max_an) + L preds[idx_trans] = probs_argmax[idx_trans] L = R R += 1 # print the results sen_idx = 0 L = 0 R = 0 for i in range(len(senLabels)): if senLabels[i] == sen_idx: R += 1 else: sen_idx = senLabels[i] if L != R: pred1 = np.zeros((R - L), dtype=np.int64) print('---sentence%d---' % sen_idx) print('pred0:', preds0[L:R]) print('label:', labels[L:R]) print('result:', preds[L:R]) L = R R += 1
def gibbs(dataset): discriminator.eval() get_prob.eval() # get_prob2.eval() preds = [] labels = [] pred_subtypes = [] golden_subtypes = [] senLabels = [] ettIdxs = [] ettLengths = [] results = [] # get the pred0 for words, pos1, pos2, loc, loc_mark, subtype, maskL, maskM, maskR, label, ettIdx, ettLength, senLabel, subtype_golden in dataset.batchs_gibbs( ): words, pos1, pos2, loc, loc_mark, subtype, maskL, maskM, maskR, label, ettIdx, ettLength, senLabel = words.to( device), pos1.to(device), pos2.to(device), loc.to( device), loc_mark.to(device), subtype.to(device), maskL.to( device), maskM.to(device), maskR.to(device), label.to( device), ettIdx.to(device), ettLength.to( device), senLabel.to(device) loss, scores, pred = discriminator(words, pos1, pos2, loc, loc_mark, subtype, maskL, maskM, maskR, label) preds.extend(pred[1].cpu().numpy()) labels.extend(label.cpu().numpy()) pred_subtypes.extend(subtype.cpu().numpy()) golden_subtypes.extend(subtype_golden.cpu().numpy()) senLabels.extend(senLabel.cpu().numpy().tolist()) ettIdxs.extend(ettIdx) ettLengths.extend(ettLength) for i in range(len(words)): state = {} results.append(state) preds0 = copy.deepcopy(preds) # Transfer for N+K times global argRoles for trans_time in range(int(1 / k_an) - 1): if trans_time % 10 == 0: cnt = 0 cnt1 = 0 FN = 0 FP = 0 for i in range(len(preds)): if labels[i] == 0: cnt1 += 1 if preds[i] != labels[i]: cnt += 1 if preds[i] == 0 and labels[i] != 0: FN += 1 if preds[i] != 0 and labels[i] == 0: FP += 1 print( "EVAL %s #Wrong %d #NegToPos %d #PosToNeg %d #All %d #Negs %d" % ("Test", cnt, FP, FN, len(preds), cnt1)) acc, _, f1 = f_score(preds, labels, pred_subtypes, golden_subtypes) print("trans_time:{}, acc:{}, f1:{}".format(trans_time, acc, f1)) # generate the argRoles L = 0 R = 0 sen_idx = senLabels[0] argRoles = [] for i in range(len(senLabels)): if senLabels[i] == sen_idx: R += 1 else: sen_idx = senLabels[i] if L != R: gen_argRole(preds[L:R], ettIdxs[L:R], ettLengths[L:R]) L = R R += 1 sen_idx = senLabels[i] if L < len(preds): gen_argRole(preds[L:], ettIdxs[L:], ettLengths[L:]) # get prob probs = [] probs_max = [] probs_argmax = [] words_sum = [] L = 0 for words, pos1, pos2, loc, loc_mark, subtype, maskL, maskM, maskR, label, ettIdx, ettLength, senLabel, subtype_golden in dataset.batchs_gibbs( ): argRole = torch.LongTensor(argRoles[L:L + len(words)]).to(device) L += len(words) words, pos1, pos2, loc, loc_mark, subtype, maskL, maskM, maskR, label, ettIdx, ettLength, senLabel = words.to( device), pos1.to(device), pos2.to(device), loc.to( device), loc_mark.to(device), subtype.to(device), maskL.to( device), maskM.to(device), maskR.to(device), label.to( device), ettIdx.to(device), ettLength.to( device), senLabel.to(device) _, prob, _ = get_prob(words, pos1, pos2, loc, loc_mark, subtype, argRole, maskL, maskM, maskR, label) prob_max, prob_argmax = torch.max(prob, dim=1) prob_max = prob_max.detach().cpu().numpy().tolist() prob_argmax = prob_argmax.detach().cpu().numpy().tolist() prob = prob.detach().cpu().numpy().tolist() words_sum.extend(words) probs.extend(prob) probs_max.extend(prob_max) probs_argmax.extend(prob_argmax) # transfer and sum the states probs_max_an = [] sen_idx = 0 L = 0 R = 0 for i, prob in enumerate(probs): if senLabels[i] == sen_idx: R += 1 else: sen_idx = senLabels[i] if L != R: probMax_sum = 0 probs_max_an = [] for idx in range(L, R): probMax_sum += probs_max[idx]**( 1 / (1 - k_an * trans_time)) for idx in range(L, R): probs_max_an.append( probs_max[idx]**(1 / (1 - k_an * trans_time)) / probMax_sum) idx_trans = random_index(probs_max_an) + L preds[idx_trans] = probs_argmax[idx_trans] L = R R += 1 # print the results sen_idx = 0 L = 0 R = 0 for i in range(len(senLabels)): if senLabels[i] == sen_idx: R += 1 else: sen_idx = senLabels[i] if L != R: pred1 = np.zeros((R - L), dtype=np.int64) print('---sentence%d---' % sen_idx) print('pred0:', preds0[L:R]) print('label:', labels[L:R]) print('result:', preds[L:R]) L = R R += 1