Example #1
0
def plotEvaluation(trecRun, qrels, measure, outputFile=None, showPlot=True):
    """
    Plots an histogram with one bar per topic.
    Each bar represents the difference between measure computed on the topic
    and the average measure among all topics.
    If outputFile is a string specifying the name of a file, then the plot
    is saved into that file.
    If showPlot then the plot is shown to the user (but not necessarily stored
     into a file).
    """
    plt.clf()
    avg, details = pytrec_eval.evaluate(trecRun, qrels, measure, True)
    # to be sure that qId order is the same of score order (maybe it's not necessary...)
    lstDetails = [(qId, score) for qId, score in details.items()]
    lstDetails.sort(key=lambda x: x[0])
    qIds = [qId for qId, _ in lstDetails]
    scores = [score - avg for _, score in lstDetails]
    plt.figure(1)
    x = [i for i in range(len(qIds))]  # np.arange(len(qIds))
    plt.bar(x, scores, width=0.6)
    plt.xticks(x, qIds, rotation=90, size=5)
    plt.xlim(xmax=len(qIds))
    plt.xlabel('Topic Id')
    plt.ylabel('Difference of ' + pytrec_eval.METRICS_NAMES[measure] + ' from Average')
    if showPlot: plt.show()
    if outputFile is not None: plt.savefig(outputFile, bbox_inches=0)
Example #2
0
def plotEvaluationAll(trecRuns, qrels, measure, outputFile, style=PLOT_STYLE):
    """
    Plots an histogram with one bar per topic.
    Each bar represents the difference between measure computed on the topic
    and the average measure among all topics.
    OutputFile is a string specifying the name of the file the plot
    is saved into.
    """
    qIds = list(qrels.getTopicIds())
    qIds.sort()

    bar_chart = pygal.Bar()
    # bar_chart.spacing = 50
    bar_chart.label_font_size = 8
    bar_chart.style = style
    bar_chart.x_label_rotation = 90
    bar_chart.x_labels = qIds
    bar_chart.x_title = 'Topic Id'
    bar_chart.legend_at_bottom = True
    bar_chart.legend_font_size = 10
    bar_chart.legend_box_size = 8

    bar_chart.y_title = pytrec_eval.METRICS_NAMES[measure]

    for trecRun in trecRuns:
        avg, details = pytrec_eval.evaluate(trecRun, qrels, measure, True)
        lstDetails = [details[topicId] if topicId in details else 0 for topicId in qIds]
        bar_chart.add(trecRun.name, lstDetails)

    bar_chart.render_to_file(outputFile)
Example #3
0
def plotEvaluation(trecRun, qrels, measure, outputFile, style=PLOT_STYLE):
    """
    Plots an histogram with one bar per topic.
    Each bar represents the difference between measure computed on the topic
    and the average measure among all topics.
    OutputFile is a string specifying the name of the file the plot
    is saved into.
    """

    avg, details = pytrec_eval.evaluate(trecRun, qrels, measure, True)
    # to be sure that qId order is the same of score order (maybe it's not necessary...)
    bar_chart = pygal.Bar()
    bar_chart.style = style
    lstDetails = [(qId, score) for qId, score in details.items()]
    lstDetails.sort(key=lambda x: x[0])
    qIds = [qId for qId, _ in lstDetails]
    scores = [score for _, score in lstDetails]
    bar_chart.add(trecRun.name, scores)

    bar_chart.label_font_size = 8
    bar_chart.legend_at_bottom = True
    bar_chart.legend_font_size = 10
    bar_chart.legend_box_size = 8

    bar_chart.x_label_rotation = 90
    bar_chart.x_labels = qIds
    bar_chart.x_title = 'query ids'
    bar_chart.y_title = pytrec_eval.METRICS_NAMES[measure]
    bar_chart.render_to_file(outputFile)
Example #4
0
def rankRuns(runs, qrels, measure):
    """Ranks the runs based on measure.
     Returns a list of pairs (run, score) ordered by
     score descending.
    """
    rank = [ (run, pytrec_eval.evaluate(run, qrels, [measure])[0]) for run in runs ]
    rank.sort(key= lambda x : x[1], reverse=True)
    return rank
def ttest(victim_run, allTheOther_runs, qrels, metric):
    """
    Computes ttest between victim_run and all runs contained in allTheOther_runs
    using relevance judgements contained in qrels to compute the specified metric.
    Returns a dictionary d[otherRunName] = p-value
    """
    victimAvg, victimDetails = evaluation.evaluate(victim_run, qrels, metric, True)
    # to read the scores always in the same order
    keyList = list(victimDetails.keys())
    victimScores = [ victimDetails[k] for k in keyList ]
    result = {}
    for othertrun in allTheOther_runs:
        otherAvg, otherDetails = evaluation.evaluate(othertrun, qrels, metric, True)
        otherScores = [otherDetails[k] for k in keyList]
        _, p = stats.ttest_ind(victimScores, otherScores)
        result[othertrun.name] = p
    return result
Example #6
0
def plotDifferenceWith(targetRun, otherRuns, qrels, measure, outputFile, style=PLOT_STYLE):
    avg_baseline, baseline_scores = pytrec_eval.evaluate(targetRun, qrels, measure, True)

    bar_chart = pygal.Bar()
    bar_chart.style = style
    allTopics = list(qrels.getTopicIds())

    bar_chart.label_font_size = 8
    bar_chart.legend_at_bottom = True
    bar_chart.legend_font_size = 10
    bar_chart.legend_box_size = 8

    bar_chart.x_label_rotation = 90
    bar_chart.x_labels = allTopics
    bar_chart.x_title = 'Topic Id'
    bar_chart.y_title = 'Difference from ' + targetRun.name + ' (' + pytrec_eval.METRICS_NAMES[measure] + ')'

    for otherRun in otherRuns:
        _, other_scores = pytrec_eval.evaluate(otherRun, qrels, measure, True)
        points = [(other_scores[topicId] if topicId in other_scores else 0) - (baseline_scores[topicId] if topicId in baseline_scores else 0)
                  for topicId in allTopics]
        bar_chart.add(otherRun.name, points)
    bar_chart.render_to_file(outputFile)