def populateExamples():
submenu = gtk.Menu()
for dir in ["examples", "libraries"]:
if misc.get_path(dir, "\0") != "\0":
d = os.listdir(misc.get_path(dir))
else:
continue
q = []
for i in d:
q.append(misc.get_path(os.path.join(dir, i)))
for c in sorted(q):
populateExampleLine(c, submenu)
paths = []
if config.user_library != None and config.user_library != -1:
paths.extend(i.strip() for i in config.user_library.split(';'))
for p in paths:
if os.path.exists(p):
q = []
if os.path.isdir(os.path.join(p, "examples")):
q.append(p)
for c in sorted(q):
populateExampleLine(c, submenu)
ex = gtk.MenuItem(_("E_xamples"), use_underline=True)
ex.set_submenu(submenu)
gui.get_object("filemenu").insert(ex, 2)
def obtain_score_truth_from_dataset(dataset,
method,
use_lemma=True,
qualitative=False):
global word2vec_unk_counter
if method == "word2vec":
word2vec_unk_counter = 0
lemma_predictions = []
scores = []
truth = []
if qualitative:
samples = []
for pr, hy, type_pr, id_pr, type_hy, id_hy,\
reversed_pr, reversed_hy, is_entailment,\
sh_scores in read_in_dataset(dataset):
method_args = [
pr, hy, type_pr, id_pr, type_hy, id_hy, reversed_pr, reversed_hy
]
if use_lemma:
lemma_predictions.append(lemma(*method_args))
if method == "Sherlock+ESR":
score = functools.reduce(lambda a, b: a * b, sh_scores)
else:
score = score_fun[method](*method_args)
scores.append(score)
truth.append(is_entailment)
if qualitative:
path_pr = get_path(id_pr)
path_hy = get_path(id_hy)
samples.append((path_pr, path_hy))
if use_lemma:
lemma_score = max(scores)
scores = [
lemma_score if lemma_pred else s
for s, lemma_pred in zip(scores, lemma_predictions)
]
if method == "word2vec":
logger.info("word2vec UNKs: {} (on {})".format(word2vec_unk_counter,
dataset))
if qualitative:
return scores, truth, samples
else:
return scores, truth
def populateImport():
submenu = gtk.Menu()
dir = "libraries"
if misc.get_path(dir, "\0") != "\0":
d = os.listdir(misc.get_path(dir))
q = []
for i in d: q.append(misc.get_path(os.path.join(dir, i)))
for c in sorted(q): populateImportLine(c, submenu)
paths = []
if config.user_library != None and config.user_library != -1:
paths.extend(i.strip() for i in config.user_library.split(';'))
for p in paths:
if os.path.exists(p):
q = []
if os.path.isdir(os.path.join(p, "examples")):
q.append(p)
for c in sorted(q): populateImportLine(c, submenu)
ex = gtk.MenuItem(_("Import Library"), use_underline=True)
ex.set_submenu(submenu)
gui.get_object("sketchmenu").insert(ex, 2)
def path2dbformat(rel_id, relation_reversed):
path = get_path(rel_id)
words = words_from_id(rel_id)
if path.startswith("nsubjpass"):
words.insert(0, "be")
if path.endswith("nsubjpass") or path.endswith("nsubjpass^-"):
words.append("be")
if relation_reversed:
words.reverse()
return " ".join(words)
def run_baseline(dataset,
method,
examples=False,
use_lemma=True,
threshold=0.5):
global bl_name_to_fun
global word2vec_unk_counter
if method == "word2vec":
word2vec_unk_counter = 0
predictor = None
if method in bl_name_to_fun:
predictor = bl_name_to_fun[method]
elif method in score_fun:
predictor = tunable_predictor(method, threshold)
else:
raise ValueError("Unknown method name: {}".format(method))
pred = []
truth = []
if examples:
correct_positives = []
correct_negatives = []
false_positives = []
false_negatives = []
logger.info("Evaluating {} on {}.".format(method, dataset))
if method in ["PPDB", "Patty", "All Rules"]:
logger.info("This might take a while.")
for pr, hy,\
type_pr, id_pr, type_hy, id_hy,\
reversed_pr, reversed_hy,\
is_entailment, sh_scores in read_in_dataset(dataset):
method_args = [
pr, hy, type_pr, id_pr, type_hy, id_hy, reversed_pr, reversed_hy
]
if use_lemma and lemma(*method_args):
pred.append(True)
else:
try:
pred.append(predictor(*method_args))
except KeyError:
logger.error("Cannot produce results for {}".format(method))
if examples:
return [], [], []
else:
return [], []
truth.append(is_entailment)
if examples:
path_pr = get_path(id_pr)
path_hy = get_path(id_hy)
if pred[-1] == truth[-1]:
if pred[-1]:
correct_positives.append((path_pr, path_hy))
else:
correct_negatives.append((path_pr, path_hy))
else:
if pred[-1]:
false_positives.append((path_pr, path_hy))
else:
false_negatives.append((path_pr, path_hy))
if method == "word2vec":
logger.info("word2vec UNKs: {}".format(word2vec_unk_counter))
if examples:
return truth, pred,\
[correct_positives, correct_negatives,
false_positives, false_negatives]
else:
return truth, pred
