def check_engine_quality(self, query_num, list_of_docs): """ :param query_num: :param list_of_docs: :return: no return. prints metrics of the query. precision, recall, map. """ benchmark_path = "data\\benchmark_lbls_train.csv" df = pd.read_csv(benchmark_path) df_prec = df[df['query'] == query_num] df_prec = df_prec[df_prec['tweet'].isin(list_of_docs)] dict_for_data = df_prec.set_index('tweet')['y_true'].to_dict() rmv_lst = [] ranking = [] # Add to list for rank for doc in list_of_docs: try: ranking.append(dict_for_data[int(doc)]) except: rmv_lst.append(doc) for d in rmv_lst: list_of_docs.remove(d) data_df = pd.DataFrame({ 'query': query_num, 'tweet': list_of_docs, 'y_true': ranking }) df_rec = df[df['query'] == query_num] recall_total = len(df_rec[df_rec['y_true'] == 1.0]) # print("total Relevant doc found with tag 1 :" , len (data_df[data_df['y_true'] == 1.0])) # print("total NON relevant doc found with tag 0 :" , len (data_df[data_df['y_true'] == 0])) # print("found total of", len(df_prec), "tagged docs") # Calculate and print prec5 = metrics.precision_at_n(data_df, query_num, 5) prec10 = metrics.precision_at_n(data_df, query_num, 10) prec50 = metrics.precision_at_n(data_df, query_num, 50) prec_total = metrics.precision(data_df, True, query_number=query_num) map_of_query = metrics.map(data_df) recall_val = metrics.recall_single(data_df, recall_total, query_num) self.map_list.append(map_of_query) self.prec5_list.append(prec5) self.prec10_list.append(prec10) self.prec50_list.append(prec50) self.prec_total_list.append(prec_total) self.recall_list.append(recall_val) print() print("precision at 5 of query", query_num, "is :", prec5) print("precision at 10 of query", query_num, "is :", prec10) print("precision at 50 of query", query_num, "is :", prec50) print("precision of query", query_num, "is :", prec_total) print("recall of query", query_num, "is :", recall_val) print("map of query", query_num, "is :", map_of_query)
) queries_results = pd.DataFrame(queries_results, columns=['query', 'tweet']) # merge query results with labels benchmark q_results_labeled = None if bench_lbls is not None and len(queries_results) > 0: q_results_labeled = pd.merge(queries_results, bench_lbls, on=['query', 'tweet'], how='inner', suffixes=('_result', '_bench')) # q_results_labeled.rename(columns={'y_true': 'label'}) zero_recall_qs = [q_id for q_id, rel in q2n_relevant.items() \ if metrics.recall_single(q_results_labeled, rel, q_id) == 0] if len(zero_recall_qs) > 0: logging.warning( f"{engine_module}'s recall for the following queries was zero {zero_recall_qs}." ) if q_results_labeled is not None: # test that MAP > 0 results_map = metrics.map(q_results_labeled) logging.debug( f"{engine_module} results have MAP value of {results_map}." ) if results_map <= 0 or results_map > 1: logging.error( f'{engine_module} results MAP value is out of range (0,1).' )
def calcQueryEvaluation(q_res, num_q): queries_results = pd.DataFrame(q_res, columns=['query', 'tweet']) if bench_lbls is not None and len(queries_results) > 0: q_results_labeled = pd.merge(queries_results, bench_lbls, on=['query', 'tweet'], how='inner', suffixes=('_result', '_bench')) # q_results_labeled.rename(columns={'y_true': 'label'}) zero_recall_qs = [q_id for q_id, rel in q2n_relevant.items() \ if metrics.recall_single(q_results_labeled, rel, q_id) == 0] if len(zero_recall_qs) > 0: logging.warning( f"{engine_module}'s recall for the following queries was zero {zero_recall_qs}." ) if q_results_labeled is not None: # test that MAP > 0 results_map = metrics.map(q_results_labeled) logging.debug( f"{engine_module} results have MAP value of {results_map}." ) if results_map <= 0 or results_map > 1: logging.error( f'{engine_module} results MAP value is out of range (0,1).' ) prec, p5, p10, p50, recall = \ metrics.precision(q_results_labeled), \ metrics.precision(q_results_labeled.groupby('query').head(5)), \ metrics.precision(q_results_labeled.groupby('query').head(10)), \ metrics.precision(q_results_labeled.groupby('query').head(50)), \ metrics.recall_single(q_results_labeled, q2n_relevant[num_q], num_q) # logging.debug(f"{engine_module} on query {num_q} results produced average precision of {prec}.") # logging.debug(f"{engine_module} on query {num_q} results produced average precision@5 of {p5}.") # logging.debug(f"{engine_module} on query {num_q} results produced average precision@10 of {p10}.") # logging.debug(f"{engine_module} on query {num_q} results produced average precision@50 of {p50}.") # logging.debug(f"{engine_module} on query {num_q} results produced average recall of {recall}.") if prec < 0 or prec > 1: logging.error( f"The average precision for {engine_module} is out of range [0,1]." ) if p5 < 0 or p5 > 1: logging.error( f"The average precision@5 for {engine_module} is out of range [0,1]." ) if p5 < 0 or p5 > 1: logging.error( f"The average precision@5 for {engine_module} is out of range [0,1]." ) if p50 < 0 or p50 > 1: logging.error( f"The average precision@50 for {engine_module} is out of range [0,1]." ) if recall < 0 or recall > 1: logging.error( f"The average recall for {engine_module} is out of range [0,1]." ) precision_list.append(prec) precisiion_5.append(p5) precisiom_10.append(p10) presicion_50.append(p50) recall_list.append(recall)