def plot_recall(hp, files, average_recall, err, show=False): k = hp['k'] add_eval_set_to_string = True param_string = utils.create_param_string(hp, add_eval_set_to_string) metrics, fs, lw, ms = get_models_style() k_vec = range(1, k + 1) fig, ax = plt.subplots(figsize=(8, 5)) max_r = 0 for key in metrics: r = average_recall[metrics[key]['name']].tolist()[0] if max(r) > max_r: max_r = max(r) plt.errorbar(k_vec, r, yerr=err[metrics[key]['name']], linewidth=lw, label=metrics[key]['legend'], fmt=metrics[key]['fmt'], color=metrics[key]['color'], marker=metrics[key]['marker'], markersize=ms) plt.legend(bbox_to_anchor=(-0.04, 1.02, 1, 0.2), loc="lower left", mode="expand", ncol=3, prop={'size': 14}, frameon=False, borderaxespad=0) ax.set_xlabel('Top-k', fontsize=fs['axis']) ax.set_ylabel('Recall', fontsize=fs['axis']) ax.set_xlim(left=1, right=k) ax.set_ylim(bottom=0, top=max_r + 0.1) plt.rc('xtick', labelsize=fs['ticks']) plt.rc('ytick', labelsize=fs['ticks']) if show: plt.show() save_figure_and_close(files, param_string, 'recall.png')
def write_models_to_html(image_metrics, hp, files): image_metrics.sort_values(['ImageID', 'cH'], ascending=[True, False], inplace=True) add_eval_set = True param_string_eval_set = utils.create_param_string(hp, add_eval_set) out_filename = ( '%s/%s/%s.html' % (files['results_dir'], param_string_eval_set, param_string_eval_set)) # Map metric to its display name. metrics, metrics_display = [], dict() metrics_style, fs, line_width, marker_size = get_models_style() for k in metrics_style.keys(): metrics_display[metrics_style[k]['name']] = metrics_style[k]['legend'] metrics.append(metrics_style[k]['name']) metrics_display['y_true'] = 'GT' # Read URLs id_url = utils.image_to_url(files['images_path']) images = list(set(image_metrics.ImageID.values.tolist())) f = open(out_filename, 'w') f.write('<html><body><table border ="2">') for c, im in enumerate(images): url = id_url[im] f.write('<tr><td>' + str(c) + '</td>') f.write('<td><img src ="' + str(url) + '" width=200></td><td>') im_df = image_metrics.loc[(image_metrics.ImageID == im), :] f.write( im_df.to_html(border=4, index=False, justify='center', col_space=60).encode('utf-8', errors='ignore')) f.write('</td><td>') for metric in metrics: metric_name = metrics_display[metric] # Return the label that was ranked first by the metric. l_first = im_df.sort_values(by=metric, ascending=False)['DisplayName'].iat[0] l_true = im_df.sort_values(by='y_true', ascending=False)['DisplayName'].iat[0] f.write('<b>' + metric_name + '</b> : ' + l_first) # Print a red cross if conf. H is correct. if metric == 'cH' and l_first == l_true: f.write('<font color="red"><b> + </b></font>') f.write('<br>') f.write('</td>') f.write('</td></tr>') f.write('</table></body></html>') f.close() print('Finished writing html to [%s]' % utils.blue(out_filename))
def plot_precision(hp, files, average_precision, err, raters_p, show=False): k = hp['k'] metrics, fs, lw, ms = get_models_style() k_vec = range(1, k + 1) fig, ax = plt.subplots(figsize=(8, 5)) max_p = 0 for key in metrics: p = average_precision[metrics[key]['name']].tolist()[0] if max(p) > max_p: max_p = max(p) plt.errorbar(k_vec, p, yerr=err[metrics[key]['name']], linewidth=lw, label=metrics[key]['legend'], fmt=metrics[key]['fmt'], color=metrics[key]['color'], marker=metrics[key]['marker'], markersize=ms) ax.set_xlabel('Top-k', fontsize=fs['axis']) ax.set_ylabel('Precision', fontsize=fs['axis']) ax.set_xlim(left=1, right=k) ax.set_ylim(bottom=0, top=max_p + 0.1) plt.rc('xtick', labelsize=fs['ticks']) plt.rc('ytick', labelsize=fs['ticks']) # Plot raters agreement plt.axhline(y=raters_p, color='grey', linestyle='--', linewidth=1, label='rater agreement') plt.legend(bbox_to_anchor=(-0.04, 1.02, 1, 0.2), loc="lower left", mode="expand", ncol=3, prop={'size': 14}, frameon=False, borderaxespad=0) if show: plt.show() add_eval_set_to_string = True param_string = utils.create_param_string(hp, add_eval_set_to_string) save_figure_and_close(files, param_string, 'precision.png')
def plot_robustness_to_num_trees(hp): trees = [1, 3, 5, 10] print('Computing for trees: %s' % str(trees)) fig, ax = plt.subplots(figsize=(8, 5)) tree_to_p = dict() for num_trees in trees: hp['seed'] = num_trees param_string = utils.create_param_string(hp, True) path = 'Results/' + param_string + '/results.pkl' print('Read results from %s' % path) (precision, sem_p, recall, sem_r) = pickle.load(open(path, 'r')) tree_to_p[num_trees] = precision['cH'][0][0] plt.plot(tree_to_p.keys(), tree_to_p.values(), 'sr', markersize=7) ax.set_xlabel('number of trees', fontsize=18) ax.set_ylabel('$cw\Delta{H}$ precision', fontsize=18) ax.set_xlim(left=0, right=num_trees + 1) fn = 'Results/Robust/tree' + param_string + '.png' plt.savefig(fn, bbox_inches='tight', facecolor='white') print('Write results to %s' % fn)
def plot_correlation(hp, files, df, headers=[ 'Confidence', 'cH', 'cMI', 'cSingleton', 'H', 'px', 'random', 'mi', 'singleton', 'cPX', 'cDKL', 'dkl' ], show=False): add_eval_set_to_string = True param_string = utils.create_param_string(hp, add_eval_set_to_string) metrics, font_size, lw, ms = get_models_style() disp = dict() for n in metrics.values(): disp[n['name']] = n['legend'] cor_cols = df[headers] corr = cor_cols.corr(method='pearson') d = [disp[n] for n in corr.columns] sns.set(font_scale=1) sns.heatmap(corr, xticklabels=d, yticklabels=d, cmap='viridis') if show: plt.show() save_figure_and_close(files, param_string, 'correlation.png')
hp['eval_method'] = args.eval_method hp['do_verify'] = args.do_verify hp['plot_figures'] = args.plot_figures hp['min_rater_count'] = args.min_rater_count hp['y_force'] = args.y_force hp['gt_vocab'] = args.gt_vocab # True - drop the image if its gt not in vocab. hp['k'] = int(args.k) # Loop over seeds. models = dict() for seed in range(hp['max_seed'] + 1): hp['seed'] = seed # Set file path. files = utils.set_file_path(hp) param_string = utils.create_param_string(hp, False) # Load model. if os.path.isfile(files['model_fn']): print('Load model from: %s' % files['model_fn']) models[seed] = pickle.load(open(files['model_fn'], 'rb')) continue # Load labels and counts. (count, pair_count, mis_dict, c22_dict, num_images, label_to_ind, ind_to_label, singles, num_labels) = model.load_count_data(files['counts_fn'], files['annotations'], hp) # Build an MI-based MST. mis_csr = model.dict_to_sparse(mis_dict, label_to_ind)
def plot_robustness_to_vocab_size(hp, atleast, show=False): colors = dict(mcolors.BASE_COLORS, **mcolors.CSS4_COLORS) atleast_to_voc, atleast_to_nImages, atleast_to_precision = \ OrderedDict(), dict(), dict() metrics, fs, lw, ms = get_models_style() hp['seed'] = hp['max_seed'] tf_to_cH, tf_to_cDKL, tf_to_cSingleton, tf_to_cPX, tf_to_cMI, \ tf_to_confidence = dict(), dict(), dict(), dict(), dict(), dict() for t in atleast: hp['atleast'] = t param_string = utils.create_param_string(hp, True) # Read precision path_results = 'Results/robust/vocab/' + param_string + '/results.pkl' print('Read results from %s' % path_results) (precision, sem_p, recall, sem_r) = pickle.load(open(path_results, 'r')) atleast_to_precision[t] = { 'cH': precision['cH'][0][0], 'cDKL': precision['cDKL'][0][0], 'cSingleton': precision['cSingleton'][0][0], 'cPX': precision['cPX'][0][0], 'cMI': precision['cMI'][0][0], 'Confidence': precision['Confidence'][0][0] } tf_to_cH[t] = precision['cH'][0][0] tf_to_cDKL[t] = precision['cDKL'][0][0] tf_to_cSingleton[t] = precision['cSingleton'][0][0] tf_to_cPX[t] = precision['cPX'][0][0] tf_to_cMI[t] = precision['cMI'][0][0] tf_to_confidence[t] = precision['Confidence'][0][0] # Read vocabulary size. path_voc = 'Results/robust/vocab/' + param_string + \ '/avg_label_metrics.pkl' label_metrics = pickle.load(open(path_voc, 'r')) atleast_to_voc[t] = label_metrics.shape[0] fig, ax = plt.subplots(figsize=(8, 5)) voc_size = atleast_to_voc.values() plt.plot(voc_size, tf_to_cH.values(), color=colors['royalblue'], markersize=ms, marker=metrics[0]['marker'], label=metrics[0]['legend']) plt.plot(voc_size, tf_to_cDKL.values(), '-', markersize=ms, color=colors['lightcoral'], marker=metrics[1]['marker'], label=metrics[1]['legend']) plt.plot(voc_size, tf_to_cMI.values(), '-', markersize=ms, marker=metrics[2]['marker'], color=colors['skyblue'], label=metrics[2]['legend']) plt.plot(voc_size, tf_to_cSingleton.values(), '-', markersize=ms, marker=metrics[3]['marker'], color=colors['mediumpurple'], label=metrics[3]['legend']) plt.plot(voc_size, tf_to_cPX.values(), '-', markersize=ms, marker=metrics[4]['marker'], color=colors['palegreen'], label=metrics[4]['legend']) plt.plot(voc_size, tf_to_confidence.values(), '-', markersize=ms, marker=metrics[5]['marker'], color=colors['orange'], label=metrics[5]['legend']) plt.box(on=None) ax.legend(loc='upper right', frameon=False, fontsize=fs['legend'], ncol=3, bbox_to_anchor=(0.5, 0.8, 0.5, 0.5)) ax.set_ylabel('precision@1', fontsize=fs['axis']) ax.set_xlabel('vocabulary size', fontsize=fs['axis']) plt.rc('xtick', labelsize=fs['ticks']) plt.rc('ytick', labelsize=fs['ticks']) if show: plt.show() fn = 'Results/robust/vocab/vocab_' + param_string + '.png' plt.savefig(fn, bbox_inches='tight', facecolor='white') print('Write results to %s' % fn)
def plot_precision_vs_recall(hp, files, average_precision, average_recall, raters_p, show=False, legend_loc='none'): add_eval_set_to_string = True param_string = utils.create_param_string(hp, add_eval_set_to_string) metrics, fs, lw, ms = get_models_style() fig, ax = plt.subplots(figsize=(10, 7)) # fig, ax = plt.subplots() max_pr = 0 for key in metrics.keys()[::-1]: r = average_recall[metrics[key]['name']].tolist()[0] p = average_precision[metrics[key]['name']].tolist()[0] if max(p) > max_pr: max_pr = max(p) plt.plot(r, p, lw=lw, label=metrics[key]['legend'], color=metrics[key]['color'], marker=metrics[key]['marker'], mew=2, ms=ms) # plt.axis('equal') ax.set_xlabel('Recall', fontsize=fs['axis']) ax.set_ylabel('Precision', fontsize=fs['axis']) ax.set_xlim(left=0, right=1) ax.set_ylim(bottom=0, top=1) # max_pr+0.1) plt.rc('xtick', labelsize=fs['ticks']) plt.rc('ytick', labelsize=fs['ticks']) plt.axhline(y=raters_p, color='grey', linestyle='--', linewidth=1, label='rater agreement') plt.text(0.02, 0.685, 'Inter-rater agreement', fontsize=20) # plt.text(0.02, 0.65, 'Inter-rater agreement', fontsize=20) legend_fs = 18 if legend_loc == 'up': plt.legend(bbox_to_anchor=(-0.04, 1.02, 1, 0.2), loc="lower left", mode="expand", ncol=3, prop={'size': legend_fs}, frameon=False, borderaxespad=0) if legend_loc == 'in': plt.legend(loc="upper left", ncol=2, prop={ 'size': legend_fs, 'weight': 'bold' }, frameon=False, borderaxespad=0.9) if legend_loc == 'side': plt.legend(bbox_to_anchor=(1.04, 1), loc="upper left", ncol=1, prop={'size': legend_fs}, frameon=False, borderaxespad=0.9) if show: plt.show() fig.dpi = 600 # 200 save_figure_and_close(files, param_string, 'pr', extension='.png')