def plotDifferenceWith(targetRun,
otherRuns,
qrels,
measure,
outputFile,
style=PLOT_STYLE):
avg_baseline, baseline_scores = pytrec_eval.evaluate(
targetRun, qrels, measure, True)
bar_chart = pygal.Bar()
bar_chart.style = style
allTopics = list(qrels.getTopicIds())
bar_chart.label_font_size = 8
bar_chart.legend_at_bottom = True
bar_chart.legend_font_size = 10
bar_chart.legend_box_size = 8
bar_chart.x_label_rotation = 90
bar_chart.x_labels = allTopics
bar_chart.x_title = 'Topic Id'
bar_chart.y_title = 'Difference from ' + targetRun.name + ' (' + pytrec_eval.METRICS_NAMES[
measure] + ')'
for otherRun in otherRuns:
_, other_scores = pytrec_eval.evaluate(otherRun, qrels, measure, True)
points = [
(other_scores[topicId] if topicId in other_scores else 0) -
(baseline_scores[topicId] if topicId in baseline_scores else 0)
for topicId in allTopics
]
bar_chart.add(otherRun.name, points)
bar_chart.render_to_file(outputFile)
def plotEvaluation(trecRun, qrels, measure, outputFile=None, showPlot=True):
"""
Plots an histogram with one bar per topic.
Each bar represents the difference between measure computed on the topic
and the average measure among all topics.
If outputFile is a string specifying the name of a file, then the plot
is saved into that file.
If showPlot then the plot is shown to the user (but not necessarily stored
into a file).
"""
plt.clf()
avg, details = pytrec_eval.evaluate(trecRun, qrels, measure, True)
# to be sure that qId order is the same of score order (maybe it's not necessary...)
lstDetails = [(qId, score) for qId, score in details.items()]
lstDetails.sort(key=lambda x: x[0])
qIds = [qId for qId, _ in lstDetails]
scores = [score - avg for _, score in lstDetails]
plt.figure(1)
x = [i for i in range(len(qIds))] # np.arange(len(qIds))
plt.bar(x, scores, width=0.6)
plt.xticks(x, qIds, rotation=90, size=5)
plt.xlim(xmax=len(qIds))
plt.xlabel('Topic Id')
plt.ylabel('Difference of ' + pytrec_eval.METRICS_NAMES[measure] + ' from Average')
if showPlot: plt.show()
if outputFile is not None: plt.savefig(outputFile, bbox_inches=0)
def plotEvaluationAll(trecRuns, qrels, measure, outputFile, style=PLOT_STYLE):
"""
Plots an histogram with one bar per topic.
Each bar represents the difference between measure computed on the topic
and the average measure among all topics.
OutputFile is a string specifying the name of the file the plot
is saved into.
"""
qIds = list(qrels.getTopicIds())
qIds.sort()
bar_chart = pygal.Bar()
# bar_chart.spacing = 50
bar_chart.label_font_size = 8
bar_chart.style = style
bar_chart.x_label_rotation = 90
bar_chart.x_labels = qIds
bar_chart.x_title = 'Topic Id'
bar_chart.legend_at_bottom = True
bar_chart.legend_font_size = 10
bar_chart.legend_box_size = 8
bar_chart.y_title = pytrec_eval.METRICS_NAMES[measure]
for trecRun in trecRuns:
avg, details = pytrec_eval.evaluate(trecRun, qrels, measure, True)
lstDetails = [details[topicId] if topicId in details else 0 for topicId in qIds]
bar_chart.add(trecRun.name, lstDetails)
bar_chart.render_to_file(outputFile)
def plotEvaluation(trecRun, qrels, measure, outputFile, style=PLOT_STYLE):
"""
Plots an histogram with one bar per topic.
Each bar represents the difference between measure computed on the topic
and the average measure among all topics.
OutputFile is a string specifying the name of the file the plot
is saved into.
"""
avg, details = pytrec_eval.evaluate(trecRun, qrels, measure, True)
# to be sure that qId order is the same of score order (maybe it's not necessary...)
bar_chart = pygal.Bar()
bar_chart.style = style
lstDetails = [(qId, score) for qId, score in details.items()]
lstDetails.sort(key=lambda x: x[0])
qIds = [qId for qId, _ in lstDetails]
scores = [score for _, score in lstDetails]
bar_chart.add(trecRun.name, scores)
bar_chart.label_font_size = 8
bar_chart.legend_at_bottom = True
bar_chart.legend_font_size = 10
bar_chart.legend_box_size = 8
bar_chart.x_label_rotation = 90
bar_chart.x_labels = qIds
bar_chart.x_title = 'query ids'
bar_chart.y_title = pytrec_eval.METRICS_NAMES[measure]
bar_chart.render_to_file(outputFile)
def plotEvaluationAll(trecRuns, qrels, measure, outputFile, style=PLOT_STYLE):
"""
Plots an histogram with one bar per topic.
Each bar represents the difference between measure computed on the topic
and the average measure among all topics.
OutputFile is a string specifying the name of the file the plot
is saved into.
"""
qIds = list(qrels.getTopicIds())
qIds.sort()
bar_chart = pygal.Bar()
# bar_chart.spacing = 50
bar_chart.label_font_size = 8
bar_chart.style = style
bar_chart.x_label_rotation = 90
bar_chart.x_labels = qIds
bar_chart.x_title = 'Topic Id'
bar_chart.legend_at_bottom = True
bar_chart.legend_font_size = 10
bar_chart.legend_box_size = 8
bar_chart.y_title = pytrec_eval.METRICS_NAMES[measure]
for trecRun in trecRuns:
avg, details = pytrec_eval.evaluate(trecRun, qrels, measure, True)
lstDetails = [
details[topicId] if topicId in details else 0 for topicId in qIds
]
bar_chart.add(trecRun.name, lstDetails)
bar_chart.render_to_file(outputFile)
def plotEvaluation(trecRun, qrels, measure, outputFile, style=PLOT_STYLE):
"""
Plots an histogram with one bar per topic.
Each bar represents the difference between measure computed on the topic
and the average measure among all topics.
OutputFile is a string specifying the name of the file the plot
is saved into.
"""
avg, details = pytrec_eval.evaluate(trecRun, qrels, measure, True)
# to be sure that qId order is the same of score order (maybe it's not necessary...)
bar_chart = pygal.Bar()
bar_chart.style = style
lstDetails = [(qId, score) for qId, score in details.items()]
lstDetails.sort(key=lambda x: x[0])
qIds = [qId for qId, _ in lstDetails]
scores = [score for _, score in lstDetails]
bar_chart.add(trecRun.name, scores)
bar_chart.label_font_size = 8
bar_chart.legend_at_bottom = True
bar_chart.legend_font_size = 10
bar_chart.legend_box_size = 8
bar_chart.x_label_rotation = 90
bar_chart.x_labels = qIds
bar_chart.x_title = 'query ids'
bar_chart.y_title = pytrec_eval.METRICS_NAMES[measure]
bar_chart.render_to_file(outputFile)
def rankRuns(runs, qrels, measure):
"""Ranks the runs based on measure.
Returns a list of pairs (run, score) ordered by
score descending.
"""
rank = [ (run, pytrec_eval.evaluate(run, qrels, [measure])[0]) for run in runs ]
rank.sort(key= lambda x : x[1], reverse=True)
return rank
def ttest(victim_run, allTheOther_runs, qrels, metric):
"""
Computes ttest between victim_run and all runs contained in allTheOther_runs
using relevance judgements contained in qrels to compute the specified metric.
Returns a dictionary d[otherRunName] = p-value
"""
victimAvg, victimDetails = evaluation.evaluate(victim_run, qrels, metric, True)
# to read the scores always in the same order
keyList = list(victimDetails.keys())
victimScores = [ victimDetails[k] for k in keyList ]
result = {}
for othertrun in allTheOther_runs:
otherAvg, otherDetails = evaluation.evaluate(othertrun, qrels, metric, True)
otherScores = [otherDetails[k] for k in keyList]
_, p = stats.ttest_ind(victimScores, otherScores)
result[othertrun.name] = p
return result
def rankRuns(runs, qrels, measure):
"""Ranks the runs based on measure.
Returns a list of pairs (run, score) ordered by
score descending.
"""
rank = [(run, pytrec_eval.evaluate(run, qrels, [measure])[0])
for run in runs]
rank.sort(key=lambda x: x[1], reverse=True)
return rank
def ttest(victim_run, allTheOther_runs, qrels, metric):
"""
Computes ttest between victim_run and all runs contained in allTheOther_runs
using relevance judgements contained in qrels to compute the specified metric.
Returns a dictionary d[otherRunName] = p-value
"""
victimAvg, victimDetails = evaluation.evaluate(victim_run, qrels, metric,
True)
# to read the scores always in the same order
keyList = list(victimDetails.keys())
victimScores = [victimDetails[k] for k in keyList]
result = {}
for othertrun in allTheOther_runs:
otherAvg, otherDetails = evaluation.evaluate(othertrun, qrels, metric,
True)
otherScores = [otherDetails[k] for k in keyList]
_, p = stats.ttest_ind(victimScores, otherScores)
result[othertrun.name] = p
return result
def plotDifferenceWith(targetRun, otherRuns, qrels, measure, outputFile, style=PLOT_STYLE):
avg_baseline, baseline_scores = pytrec_eval.evaluate(targetRun, qrels, measure, True)
bar_chart = pygal.Bar()
bar_chart.style = style
allTopics = list(qrels.getTopicIds())
bar_chart.label_font_size = 8
bar_chart.legend_at_bottom = True
bar_chart.legend_font_size = 10
bar_chart.legend_box_size = 8
bar_chart.x_label_rotation = 90
bar_chart.x_labels = allTopics
bar_chart.x_title = 'Topic Id'
bar_chart.y_title = 'Difference from ' + targetRun.name + ' (' + pytrec_eval.METRICS_NAMES[measure] + ')'
for otherRun in otherRuns:
_, other_scores = pytrec_eval.evaluate(otherRun, qrels, measure, True)
points = [(other_scores[topicId] if topicId in other_scores else 0) - (baseline_scores[topicId] if topicId in baseline_scores else 0)
for topicId in allTopics]
bar_chart.add(otherRun.name, points)
bar_chart.render_to_file(outputFile)
from pytrec_eval import TrecRun
from pytrec_eval import QRels
from pytrec_eval import evaluate
from pytrec_eval import metrics
from pytrec_eval import precisionAt
run = TrecRun(
"/home/smsarwar/PycharmProjects/deep-siamese-text-similarity/results/results_multitask.txt"
)
qrel = QRels(
'/home/smsarwar/PycharmProjects/deep-siamese-text-similarity/results/qrel.txt'
)
print(qrel)
print(evaluate(run, qrel, metrics.recall))
print(evaluate(run, qrel, metrics.precisionAt(10)))
print(evaluate(run, qrel, metrics.avgPrec))
#print(metrics.recall(run, qrel, detailed=True))
sdm_doc_score_map = get_scores(sdm_docs[str(topic_number)])
cur_topic_subgraph = scores[str(topic_number)]
doc_count = 1
for doc, score in sorted(cur_topic_subgraph.items(), key=lambda item: item[1], reverse = True):
sdm_score = float(sdm_doc_score_map[doc])
try:
centrality_score = normalize(score, cur_topic_subgraph.values())
except:
centrality_score = 0
combined_score = lambda1*sdm_score + lambda2*centrality_score
temp_results.append(str(topic_number) + " Q0 " + doc + " " + str(doc_count) + " " + str(combined_score)+ " STANDARD")
doc_count+=1
with open("temp_file.test", "w") as outfile:
outfile.write("\n".join(temp_results))
run = pytrec_eval.TrecRun('temp_file.test')
qrels = pytrec_eval.QRels('qrels_file.test')
curr_result = pytrec_eval.evaluate(run, qrels, [pytrec_eval.ndcg])[0]
if curr_result > best_score:
if best_results is not None:
best_results.clear()
best_score = curr_result
best_results = list(temp_results)
print("Run completed with lambda1=" + str(lambda1) + ", lambda2=" + str(lambda2) + " and NDCG=" + str(curr_result) + ". Took: " + str(time() - start_load) + " s")
for result in best_results:
output.append(result)
with open("results_file.test", "w") as outfile:
outfile.write("\n".join(output))
print ("Results took %.2f seconds to run." %(time() - start_project))