def run_tests_glove(self, _query_lst, cache=True):
print('glove tests')
for question_no, (query) in enumerate(_query_lst):
scores_dict = dict()
entities_lst = query.get_query()
for entity in entities_lst:
scores_dict[entity] = 0.0
for i, e1 in enumerate(entities_lst):
v1 = _index.get_glove_vector(e1)
for j in range(i + 1, len(entities_lst)):
e2 = entities_lst[j]
if cache and (
e1,
e2,
) in self.glove_cached_dict:
sim_score = self.glove_cached_dict[(
e1,
e2,
)]
else:
v2 = _index.get_glove_vector(e2)
sim_score = functions.calculate_cosine_similarity(
v1, v2)
self.glove_cached_dict[(
e1,
e2,
)] = sim_score
self.glove_cached_dict[(
e2,
e1,
)] = sim_score
scores_dict[e1] += sim_score
scores_dict[e2] += sim_score
return
def __run_test_eve__(self, _type, _query_lst, _index, cache=True):
dimensions = _index.get_eve_dimensions()
for question_no, (query) in enumerate(_query_lst):
entities_lst = query.get_query()
for i, e1 in enumerate(entities_lst):
v1 = _index.get_eve_vector(e1)
for j in range(i + 1, len(entities_lst)):
e2 = entities_lst[j]
v2 = _index.get_eve_vector(e2)
if cache and (
e1,
e2,
) in self.cached_eve_dict:
score = self.cached_eve_dict[(
e1,
e2,
)]
else:
score = functions.calculate_cosine_similarity(v1, v2)
self.cached_eve_dict[(
e1,
e2,
)] = score
self.cached_eve_dict[(
e2,
e1,
)] = score
# print(score)
self.explain_cluster_eve(_type, _index, dimensions)
return
def run_tests_glove(self, _query_lst, echo=True, cache=True):
similarity_cached_dict = dict()
_measures = {'p@k': 0, 'ap': 0}
results_lst = list()
print('glove tests')
for question_no, (query) in enumerate(_query_lst):
if echo:
print('Question:' + str(question_no + 1) + '/' + str(len(_query_lst)),
query.get_description(), 'Known Answer:', query.get_all_correct_answer())
scores_dict = dict()
e1 = query.get_query()
v1 = _index.get_glove_vector(e1)
for e2 in query.get_possible_answer_set():
if cache and (e1, e2,) in similarity_cached_dict:
sim_score = similarity_cached_dict[(e1, e2,)]
else:
v2 = _index.get_glove_vector(e2)
sim_score = functions.calculate_cosine_similarity(v1, v2)
similarity_cached_dict[(e1, e2,)] = sim_score
similarity_cached_dict[(e2, e1,)] = sim_score
scores_dict[e1, e2] = sim_score
query.set_discovered_answer(scores_dict, echo=echo)
for k, v in query.get_measures().items():
_measures[k] += v
if echo:
print(' >', query.get_measures())
if echo:
print('Measure: ' + ','.join([k + ': ' + str(v / len(_query_lst)) for k, v in _measures.items()]),
'Total Queries:', len(_query_lst))
print('')
results_lst.append((_measures, len(_query_lst)))
return results_lst
def __run_test_eve__(_query_lst, _index, echo=True, explain_wrong=True, explain_always=False, cache=True,
max_n_features=40, top_n_answers=3):
similarity_cached_dict = dict()
_measures = {'p@k': 0, 'ap': 0}
dimensions = _index.get_eve_dimensions()
for question_no, (query) in enumerate(_query_lst):
if echo:
print('Question:' + str(question_no + 1) + '/' + str(len(_query_lst)),
query.get_description(), 'Known Answer:', query.get_all_correct_answer())
scores_dict = dict()
e1 = query.get_query()
v1 = _index.get_eve_vector(e1)
for e2 in query.get_possible_answer_set():
v2 = _index.get_eve_vector(e2)
if cache and (e1, e2,) in similarity_cached_dict:
score = similarity_cached_dict[(e1, e2,)]
else:
score = functions.calculate_cosine_similarity(v1, v2)
similarity_cached_dict[(e1, e2,)] = score
similarity_cached_dict[(e2, e1,)] = score
scores_dict[e1, e2] = score
query.set_discovered_answer(scores_dict, echo=echo)
for k, v in query.get_measures().items():
_measures[k] += v
if echo:
print(' >', query.get_measures())
if (explain_wrong and not query.get_answered_status()) or explain_always:
RankingTask.explain_result(v1, e1, query, _index, dimensions,
max_n=max_n_features, top_n_answers=top_n_answers)
if echo:
print('Measure: ' + ','.join([k + ': ' + str(v/len(_query_lst)) for k, v in _measures.items()]),
'Total Queries:', len(_query_lst))
print('')
return _measures, len(_query_lst)
def run_tests_fasttext(self, _query_lst, sg=None, echo=True, cache=True):
similarity_cached_dict = dict()
results_lst = list()
print('fasttext tests, sg =', sg)
total_correct_answer = 0
algo_results = list()
for question_no, (query) in enumerate(_query_lst):
if echo:
print(
'Question:' +
str(question_no + 1) + '/' + str(len(_query_lst)),
query.get_description(), 'Known Answer:',
query.get_all_correct_answer())
scores_dict = dict()
entities_lst = query.get_query()
for entity in entities_lst:
scores_dict[entity] = 0.0
for i, e1 in enumerate(entities_lst):
v1 = _index.get_fasttext_vector(e1, sg)
for j in range(i + 1, len(entities_lst)):
e2 = entities_lst[j]
if cache and (
e1,
e2,
) in similarity_cached_dict:
sim_score = similarity_cached_dict[(
e1,
e2,
)]
else:
v2 = _index.get_fasttext_vector(e2, sg)
sim_score = functions.calculate_cosine_similarity(
v1, v2)
similarity_cached_dict[(
e1,
e2,
)] = sim_score
similarity_cached_dict[(
e2,
e1,
)] = sim_score
scores_dict[e1] += sim_score
scores_dict[e2] += sim_score
query.set_discovered_answer(scores_dict, echo=echo)
if echo:
print(' >', query.get_discovered_answer(),
query.get_answered_status())
if query.get_answered_status():
total_correct_answer += 1
algo_results.append(
(query.get_query(), query.get_answered_status()))
if echo:
print('Total Correct Answer', total_correct_answer, 'Out of',
len(_query_lst), total_correct_answer / len(_query_lst))
results_lst.append(
(total_correct_answer, len(_query_lst), algo_results))
return results_lst
def __run_test_eve__(_query_lst,
_index,
echo=True,
final_result_echo=False,
explain_wrong=True,
explain_always=False,
cache=True):
cached_dict = dict()
total_correct_answer = 0
algo_results = list()
for question_no, (query) in enumerate(_query_lst):
if echo:
print(
'Question:' +
str(question_no + 1) + '/' + str(len(_query_lst)),
query.get_description(), 'Known Answer:',
query.get_all_correct_answer())
scores_dict = dict()
entities_lst = query.get_query()
for entity in entities_lst:
scores_dict[entity] = 0.0
for i, e1 in enumerate(entities_lst):
v1 = _index.get_eve_vector(e1)
for j in range(i + 1, len(entities_lst)):
e2 = entities_lst[j]
v2 = _index.get_eve_vector(e2)
if cache and (
e1,
e2,
) in cached_dict:
score = cached_dict[(
e1,
e2,
)]
else:
score = functions.calculate_cosine_similarity(v1, v2)
cached_dict[(
e1,
e2,
)] = score
cached_dict[(
e2,
e1,
)] = score
# print(score)
scores_dict[e1] += score
scores_dict[e2] += score
query.set_discovered_answer(scores_dict, echo=echo)
if echo:
print(' >', query.get_discovered_answer(),
query.get_answered_status())
if query.get_answered_status():
total_correct_answer += 1
if (explain_wrong
and not query.get_answered_status()) or explain_always:
IntruderTask.explain_result(query,
_index,
entities_lst,
max_n=40)
algo_results.append(
(query.get_query(), query.get_answered_status()))
if echo or final_result_echo:
print('Total Correct Answer', total_correct_answer, 'Out of',
len(_query_lst), total_correct_answer / len(_query_lst))
print('')
return total_correct_answer, len(_query_lst), algo_results