def evaluate_dataset_point(model, data , addbase): pred_trees = [] gold_trees = [] for i, inst in enumerate(data): pred_scores = [] for tree in inst.kbest: if tree.size == inst.gold.size: pred_scores.append(model.predict(tree)) else: print 'error' pred_scores.append(-1000) #data_util.normalize(pred_scores) if addbase: data_util.normalize(inst.scores) scores = [p_s + b_s for p_s, b_s in zip(pred_scores, inst.scores)] else: scores = pred_scores max_id = scores.index(max(scores)) #print max_id,scores[max_id] for line in inst.lines[max_id]: pred_trees.append(line) pred_trees.append('\n') for line in inst.gold_lines: gold_trees.append(line) gold_trees.append('\n') res = eval_tool.evaluate(pred_trees,gold_trees) print 'f1score: %.4f' % (res[0]) return res
def evaluate_dataset_pair(model, data, addbase, ratio=1): pred_trees = [] gold_trees = [] for i, inst in enumerate(data): lens = len(inst.kbest) max = 0 for j in range(1, lens): loss = 0 if inst.kbest[j].size == inst.gold.size: #loss = np.mean(model.train_pairwise(inst.kbest[max],inst.kbest[j],True)) # score_j - score_best if addbase: baseloss = inst.scores[j] - inst.scores[max] #print "loss: %.4f baseloss: %.4f" % (loss,baseloss) loss += baseloss if loss > 0: max = j for line in inst.lines[max]: pred_trees.append(line) pred_trees.append('\n') for line in inst.gold_lines: gold_trees.append(line) gold_trees.append('\n') res = eval_tool.evaluate(pred_trees, gold_trees) print 'ratio: %f f1score: %.4f' % (ratio, res[0]) return res
def evaluate_dataset_point(model, data, addbase, ratio=1): pred_trees = [] gold_trees = [] for i, inst in enumerate(data): pred_scores = [ model.predict(tree) for tree in inst.kbest if tree.size == inst.gold.size ] data_util.normalize(pred_scores) if addbase: data_util.normalize(inst.scores) scores = [ ratio * p_s + (1 - ratio) * b_s for p_s, b_s in zip(pred_scores, inst.scores) ] else: scores = pred_scores max_id = scores.index(max(scores)) #print "pred: %.4f base: %.4f" % (pred_scores[max_id],inst.scores[max_id]) for line in inst.lines[max_id]: pred_trees.append(line) pred_trees.append('\n') for line in inst.gold_lines: gold_trees.append(line) gold_trees.append('\n') res = eval_tool.evaluate(pred_trees, gold_trees) print 'ratio: %f f1score: %.4f' % (ratio, res[0]) return res
def evaluate_baseline(data): pred_trees = [] gold_trees = [] for i, inst in enumerate(data): for line in inst.lines[len(inst.kbest)-1]: pred_trees.append(line) pred_trees.append('\n') for line in inst.gold_lines: gold_trees.append(line) gold_trees.append('\n') print 'baseline: %.4f' % (eval_tool.evaluate(pred_trees,gold_trees)[0])
def evaluate_oracle_worst(data): oracle_trees = [] worst_trees = [] gold_trees = [] pred_trees = [] for i, inst in enumerate(data): max = 0 maxid = 0 min = 1 minid = 0 for line in inst.gold_lines: gold_trees.append(line) gold_trees.append('\n') for line in inst.lines[len(inst.kbest)-1]: pred_trees.append(line) pred_trees.append('\n') i = 0 for list in inst.lines: temp = [] for line in list: temp.append(line) temp.append('\n') res = eval_tool.evaluate(temp, inst.gold_lines)[0] if res > max : max = res maxid = i if res < min : min = res minid = i i += 1 for line in inst.lines[maxid]: oracle_trees.append(line) oracle_trees.append('\n') for line in inst.lines[minid]: worst_trees.append(line) worst_trees.append('\n') print 'f1score: %.4f' % (eval_tool.evaluate(pred_trees, gold_trees)[0]) print 'oracle: %.4f' % (eval_tool.evaluate(oracle_trees, gold_trees)[0]) print 'worst: %.4f' % (eval_tool.evaluate(worst_trees, gold_trees)[0])
def evaluate_baseline_random(data): random.seed(189) pred_trees = [] gold_trees = [] for i, inst in enumerate(data): rand = int(random.random()*len(inst.kbest)) for line in inst.lines[rand]: pred_trees.append(line) pred_trees.append('\n') for line in inst.gold_lines: gold_trees.append(line) gold_trees.append('\n') print 'baseline: %.4f' % (eval_tool.evaluate(pred_trees,gold_trees)[0])
def get_oracle_index(self): max = 0 maxid = 0 i = 0 for list in self.lines: temp = [] for line in list: temp.append(line) temp.append('\n') res = eval_tool.evaluate(temp, self.gold_lines)[0] self.f1score.append(res) if res > max: max = res maxid = i i += 1 return maxid
def evaluate_dataset_pair(model, data): pred_trees = [] gold_trees = [] for i, inst in enumerate(data): lens = len(inst.kbest) max = 0 for j in range(1, lens): loss = 0 if inst.kbest[j].size == inst.gold.size: loss = np.mean(model.train_pairwise(inst.kbest[max],inst.kbest[j],True)) if loss > 0: max = j for line in inst.lines[max]: pred_trees.append(line) pred_trees.append('\n') for line in inst.gold_lines: gold_trees.append(line) gold_trees.append('\n') res = eval_tool.evaluate(pred_trees,gold_trees) print 'f1score: %.4f' % (res[0]) return res
def set_f1(self): for l in self.lines: f1 = eval_tool.evaluate(l, self.gold_lines)[0] self.f1score.append(f1)