def check_engine_quality(self, query_num, list_of_docs):
"""
:param query_num:
:param list_of_docs:
:return: no return. prints metrics of the query. precision, recall, map.
"""
benchmark_path = "data\\benchmark_lbls_train.csv"
df = pd.read_csv(benchmark_path)
df_prec = df[df['query'] == query_num]
df_prec = df_prec[df_prec['tweet'].isin(list_of_docs)]
dict_for_data = df_prec.set_index('tweet')['y_true'].to_dict()
rmv_lst = []
ranking = []
# Add to list for rank
for doc in list_of_docs:
try:
ranking.append(dict_for_data[int(doc)])
except:
rmv_lst.append(doc)
for d in rmv_lst:
list_of_docs.remove(d)
data_df = pd.DataFrame({
'query': query_num,
'tweet': list_of_docs,
'y_true': ranking
})
df_rec = df[df['query'] == query_num]
recall_total = len(df_rec[df_rec['y_true'] == 1.0])
# print("total Relevant doc found with tag 1 :" , len (data_df[data_df['y_true'] == 1.0]))
# print("total NON relevant doc found with tag 0 :" , len (data_df[data_df['y_true'] == 0]))
# print("found total of", len(df_prec), "tagged docs")
# Calculate and print
prec5 = metrics.precision_at_n(data_df, query_num, 5)
prec10 = metrics.precision_at_n(data_df, query_num, 10)
prec50 = metrics.precision_at_n(data_df, query_num, 50)
prec_total = metrics.precision(data_df, True, query_number=query_num)
map_of_query = metrics.map(data_df)
recall_val = metrics.recall_single(data_df, recall_total, query_num)
self.map_list.append(map_of_query)
self.prec5_list.append(prec5)
self.prec10_list.append(prec10)
self.prec50_list.append(prec50)
self.prec_total_list.append(prec_total)
self.recall_list.append(recall_val)
print()
print("precision at 5 of query", query_num, "is :", prec5)
print("precision at 10 of query", query_num, "is :", prec10)
print("precision at 50 of query", query_num, "is :", prec50)
print("precision of query", query_num, "is :", prec_total)
print("recall of query", query_num, "is :", recall_val)
print("map of query", query_num, "is :", map_of_query)
)
queries_results = pd.DataFrame(queries_results,
columns=['query', 'tweet'])
# merge query results with labels benchmark
q_results_labeled = None
if bench_lbls is not None and len(queries_results) > 0:
q_results_labeled = pd.merge(queries_results,
bench_lbls,
on=['query', 'tweet'],
how='inner',
suffixes=('_result',
'_bench'))
# q_results_labeled.rename(columns={'y_true': 'label'})
zero_recall_qs = [q_id for q_id, rel in q2n_relevant.items() \
if metrics.recall_single(q_results_labeled, rel, q_id) == 0]
if len(zero_recall_qs) > 0:
logging.warning(
f"{engine_module}'s recall for the following queries was zero {zero_recall_qs}."
)
if q_results_labeled is not None:
# test that MAP > 0
results_map = metrics.map(q_results_labeled)
logging.debug(
f"{engine_module} results have MAP value of {results_map}."
)
if results_map <= 0 or results_map > 1:
logging.error(
f'{engine_module} results MAP value is out of range (0,1).'
)
)
queries_results = pd.DataFrame(queries_results,
columns=['query', 'tweet'])
# merge query results with labels benchmark
q_results_labeled = None
if bench_lbls is not None and len(queries_results) > 0:
q_results_labeled = pd.merge(queries_results,
bench_lbls,
on=['query', 'tweet'],
how='inner',
suffixes=('_result',
'_bench'))
# q_results_labeled.rename(columns={'y_true': 'label'})
zero_recall_qs = [q_id for q_id, rel in q2n_relevant.items() \
if metrics.recall_single(q_results_labeled, rel, q_id) == 0]
if len(zero_recall_qs) > 0:
logging.warning(
f"{engine_module}'s recall for the following queries was zero {zero_recall_qs}."
)
if q_results_labeled is not None:
# test that MAP > 0
results_map = metrics.map(q_results_labeled)
logging.debug(
f"{engine_module} results have MAP value of {results_map}."
)
if results_map <= 0 or results_map > 1:
logging.error(
f'{engine_module} results MAP value is out of range (0,1).'
)
def calcQueryEvaluation(q_res, num_q):
queries_results = pd.DataFrame(q_res, columns=['query', 'tweet'])
if bench_lbls is not None and len(queries_results) > 0:
q_results_labeled = pd.merge(queries_results,
bench_lbls,
on=['query', 'tweet'],
how='inner',
suffixes=('_result', '_bench'))
# q_results_labeled.rename(columns={'y_true': 'label'})
zero_recall_qs = [q_id for q_id, rel in q2n_relevant.items() \
if metrics.recall_single(q_results_labeled, rel, q_id) == 0]
if len(zero_recall_qs) > 0:
logging.warning(
f"{engine_module}'s recall for the following queries was zero {zero_recall_qs}."
)
if q_results_labeled is not None:
# test that MAP > 0
results_map = metrics.map(q_results_labeled)
logging.debug(
f"{engine_module} results have MAP value of {results_map}."
)
if results_map <= 0 or results_map > 1:
logging.error(
f'{engine_module} results MAP value is out of range (0,1).'
)
prec, p5, p10, p50, recall = \
metrics.precision(q_results_labeled), \
metrics.precision(q_results_labeled.groupby('query').head(5)), \
metrics.precision(q_results_labeled.groupby('query').head(10)), \
metrics.precision(q_results_labeled.groupby('query').head(50)), \
metrics.recall_single(q_results_labeled, q2n_relevant[num_q], num_q)
# logging.debug(f"{engine_module} on query {num_q} results produced average precision of {prec}.")
# logging.debug(f"{engine_module} on query {num_q} results produced average precision@5 of {p5}.")
# logging.debug(f"{engine_module} on query {num_q} results produced average precision@10 of {p10}.")
# logging.debug(f"{engine_module} on query {num_q} results produced average precision@50 of {p50}.")
# logging.debug(f"{engine_module} on query {num_q} results produced average recall of {recall}.")
if prec < 0 or prec > 1:
logging.error(
f"The average precision for {engine_module} is out of range [0,1]."
)
if p5 < 0 or p5 > 1:
logging.error(
f"The average precision@5 for {engine_module} is out of range [0,1]."
)
if p5 < 0 or p5 > 1:
logging.error(
f"The average precision@5 for {engine_module} is out of range [0,1]."
)
if p50 < 0 or p50 > 1:
logging.error(
f"The average precision@50 for {engine_module} is out of range [0,1]."
)
if recall < 0 or recall > 1:
logging.error(
f"The average recall for {engine_module} is out of range [0,1]."
)
precision_list.append(prec)
precisiion_5.append(p5)
precisiom_10.append(p10)
presicion_50.append(p50)
recall_list.append(recall)