def plot_recall(hp, files, average_recall, err, show=False):
k = hp['k']
add_eval_set_to_string = True
param_string = utils.create_param_string(hp, add_eval_set_to_string)
metrics, fs, lw, ms = get_models_style()
k_vec = range(1, k + 1)
fig, ax = plt.subplots(figsize=(8, 5))
max_r = 0
for key in metrics:
r = average_recall[metrics[key]['name']].tolist()[0]
if max(r) > max_r: max_r = max(r)
plt.errorbar(k_vec,
r,
yerr=err[metrics[key]['name']],
linewidth=lw,
label=metrics[key]['legend'],
fmt=metrics[key]['fmt'],
color=metrics[key]['color'],
marker=metrics[key]['marker'],
markersize=ms)
plt.legend(bbox_to_anchor=(-0.04, 1.02, 1, 0.2),
loc="lower left",
mode="expand",
ncol=3,
prop={'size': 14},
frameon=False,
borderaxespad=0)
ax.set_xlabel('Top-k', fontsize=fs['axis'])
ax.set_ylabel('Recall', fontsize=fs['axis'])
ax.set_xlim(left=1, right=k)
ax.set_ylim(bottom=0, top=max_r + 0.1)
plt.rc('xtick', labelsize=fs['ticks'])
plt.rc('ytick', labelsize=fs['ticks'])
if show: plt.show()
save_figure_and_close(files, param_string, 'recall.png')
def write_models_to_html(image_metrics, hp, files):
image_metrics.sort_values(['ImageID', 'cH'],
ascending=[True, False],
inplace=True)
add_eval_set = True
param_string_eval_set = utils.create_param_string(hp, add_eval_set)
out_filename = (
'%s/%s/%s.html' %
(files['results_dir'], param_string_eval_set, param_string_eval_set))
# Map metric to its display name.
metrics, metrics_display = [], dict()
metrics_style, fs, line_width, marker_size = get_models_style()
for k in metrics_style.keys():
metrics_display[metrics_style[k]['name']] = metrics_style[k]['legend']
metrics.append(metrics_style[k]['name'])
metrics_display['y_true'] = 'GT'
# Read URLs
id_url = utils.image_to_url(files['images_path'])
images = list(set(image_metrics.ImageID.values.tolist()))
f = open(out_filename, 'w')
f.write('<html><body><table border ="2">')
for c, im in enumerate(images):
url = id_url[im]
f.write('<tr><td>' + str(c) + '</td>')
f.write('<td><img src ="' + str(url) + '" width=200></td><td>')
im_df = image_metrics.loc[(image_metrics.ImageID == im), :]
f.write(
im_df.to_html(border=4,
index=False,
justify='center',
col_space=60).encode('utf-8', errors='ignore'))
f.write('</td><td>')
for metric in metrics:
metric_name = metrics_display[metric]
# Return the label that was ranked first by the metric.
l_first = im_df.sort_values(by=metric,
ascending=False)['DisplayName'].iat[0]
l_true = im_df.sort_values(by='y_true',
ascending=False)['DisplayName'].iat[0]
f.write('<b>' + metric_name + '</b> : ' + l_first)
# Print a red cross if conf. H is correct.
if metric == 'cH' and l_first == l_true:
f.write('<font color="red"><b> + </b></font>')
f.write('<br>')
f.write('</td>')
f.write('</td></tr>')
f.write('</table></body></html>')
f.close()
print('Finished writing html to [%s]' % utils.blue(out_filename))
def plot_precision(hp, files, average_precision, err, raters_p, show=False):
k = hp['k']
metrics, fs, lw, ms = get_models_style()
k_vec = range(1, k + 1)
fig, ax = plt.subplots(figsize=(8, 5))
max_p = 0
for key in metrics:
p = average_precision[metrics[key]['name']].tolist()[0]
if max(p) > max_p: max_p = max(p)
plt.errorbar(k_vec,
p,
yerr=err[metrics[key]['name']],
linewidth=lw,
label=metrics[key]['legend'],
fmt=metrics[key]['fmt'],
color=metrics[key]['color'],
marker=metrics[key]['marker'],
markersize=ms)
ax.set_xlabel('Top-k', fontsize=fs['axis'])
ax.set_ylabel('Precision', fontsize=fs['axis'])
ax.set_xlim(left=1, right=k)
ax.set_ylim(bottom=0, top=max_p + 0.1)
plt.rc('xtick', labelsize=fs['ticks'])
plt.rc('ytick', labelsize=fs['ticks'])
# Plot raters agreement
plt.axhline(y=raters_p,
color='grey',
linestyle='--',
linewidth=1,
label='rater agreement')
plt.legend(bbox_to_anchor=(-0.04, 1.02, 1, 0.2),
loc="lower left",
mode="expand",
ncol=3,
prop={'size': 14},
frameon=False,
borderaxespad=0)
if show: plt.show()
add_eval_set_to_string = True
param_string = utils.create_param_string(hp, add_eval_set_to_string)
save_figure_and_close(files, param_string, 'precision.png')
def plot_robustness_to_num_trees(hp):
trees = [1, 3, 5, 10]
print('Computing for trees: %s' % str(trees))
fig, ax = plt.subplots(figsize=(8, 5))
tree_to_p = dict()
for num_trees in trees:
hp['seed'] = num_trees
param_string = utils.create_param_string(hp, True)
path = 'Results/' + param_string + '/results.pkl'
print('Read results from %s' % path)
(precision, sem_p, recall, sem_r) = pickle.load(open(path, 'r'))
tree_to_p[num_trees] = precision['cH'][0][0]
plt.plot(tree_to_p.keys(), tree_to_p.values(), 'sr', markersize=7)
ax.set_xlabel('number of trees', fontsize=18)
ax.set_ylabel('$cw\Delta{H}$ precision', fontsize=18)
ax.set_xlim(left=0, right=num_trees + 1)
fn = 'Results/Robust/tree' + param_string + '.png'
plt.savefig(fn, bbox_inches='tight', facecolor='white')
print('Write results to %s' % fn)
def plot_correlation(hp,
files,
df,
headers=[
'Confidence', 'cH', 'cMI', 'cSingleton', 'H', 'px',
'random', 'mi', 'singleton', 'cPX', 'cDKL', 'dkl'
],
show=False):
add_eval_set_to_string = True
param_string = utils.create_param_string(hp, add_eval_set_to_string)
metrics, font_size, lw, ms = get_models_style()
disp = dict()
for n in metrics.values():
disp[n['name']] = n['legend']
cor_cols = df[headers]
corr = cor_cols.corr(method='pearson')
d = [disp[n] for n in corr.columns]
sns.set(font_scale=1)
sns.heatmap(corr, xticklabels=d, yticklabels=d, cmap='viridis')
if show: plt.show()
save_figure_and_close(files, param_string, 'correlation.png')
hp['eval_method'] = args.eval_method
hp['do_verify'] = args.do_verify
hp['plot_figures'] = args.plot_figures
hp['min_rater_count'] = args.min_rater_count
hp['y_force'] = args.y_force
hp['gt_vocab'] = args.gt_vocab # True - drop the image if its gt not in vocab.
hp['k'] = int(args.k)
# Loop over seeds.
models = dict()
for seed in range(hp['max_seed'] + 1):
hp['seed'] = seed
# Set file path.
files = utils.set_file_path(hp)
param_string = utils.create_param_string(hp, False)
# Load model.
if os.path.isfile(files['model_fn']):
print('Load model from: %s' % files['model_fn'])
models[seed] = pickle.load(open(files['model_fn'], 'rb'))
continue
# Load labels and counts.
(count, pair_count, mis_dict, c22_dict, num_images, label_to_ind,
ind_to_label, singles,
num_labels) = model.load_count_data(files['counts_fn'],
files['annotations'], hp)
# Build an MI-based MST.
mis_csr = model.dict_to_sparse(mis_dict, label_to_ind)
def plot_robustness_to_vocab_size(hp, atleast, show=False):
colors = dict(mcolors.BASE_COLORS, **mcolors.CSS4_COLORS)
atleast_to_voc, atleast_to_nImages, atleast_to_precision = \
OrderedDict(), dict(), dict()
metrics, fs, lw, ms = get_models_style()
hp['seed'] = hp['max_seed']
tf_to_cH, tf_to_cDKL, tf_to_cSingleton, tf_to_cPX, tf_to_cMI, \
tf_to_confidence = dict(), dict(), dict(), dict(), dict(), dict()
for t in atleast:
hp['atleast'] = t
param_string = utils.create_param_string(hp, True)
# Read precision
path_results = 'Results/robust/vocab/' + param_string + '/results.pkl'
print('Read results from %s' % path_results)
(precision, sem_p, recall,
sem_r) = pickle.load(open(path_results, 'r'))
atleast_to_precision[t] = {
'cH': precision['cH'][0][0],
'cDKL': precision['cDKL'][0][0],
'cSingleton': precision['cSingleton'][0][0],
'cPX': precision['cPX'][0][0],
'cMI': precision['cMI'][0][0],
'Confidence': precision['Confidence'][0][0]
}
tf_to_cH[t] = precision['cH'][0][0]
tf_to_cDKL[t] = precision['cDKL'][0][0]
tf_to_cSingleton[t] = precision['cSingleton'][0][0]
tf_to_cPX[t] = precision['cPX'][0][0]
tf_to_cMI[t] = precision['cMI'][0][0]
tf_to_confidence[t] = precision['Confidence'][0][0]
# Read vocabulary size.
path_voc = 'Results/robust/vocab/' + param_string + \
'/avg_label_metrics.pkl'
label_metrics = pickle.load(open(path_voc, 'r'))
atleast_to_voc[t] = label_metrics.shape[0]
fig, ax = plt.subplots(figsize=(8, 5))
voc_size = atleast_to_voc.values()
plt.plot(voc_size,
tf_to_cH.values(),
color=colors['royalblue'],
markersize=ms,
marker=metrics[0]['marker'],
label=metrics[0]['legend'])
plt.plot(voc_size,
tf_to_cDKL.values(),
'-',
markersize=ms,
color=colors['lightcoral'],
marker=metrics[1]['marker'],
label=metrics[1]['legend'])
plt.plot(voc_size,
tf_to_cMI.values(),
'-',
markersize=ms,
marker=metrics[2]['marker'],
color=colors['skyblue'],
label=metrics[2]['legend'])
plt.plot(voc_size,
tf_to_cSingleton.values(),
'-',
markersize=ms,
marker=metrics[3]['marker'],
color=colors['mediumpurple'],
label=metrics[3]['legend'])
plt.plot(voc_size,
tf_to_cPX.values(),
'-',
markersize=ms,
marker=metrics[4]['marker'],
color=colors['palegreen'],
label=metrics[4]['legend'])
plt.plot(voc_size,
tf_to_confidence.values(),
'-',
markersize=ms,
marker=metrics[5]['marker'],
color=colors['orange'],
label=metrics[5]['legend'])
plt.box(on=None)
ax.legend(loc='upper right',
frameon=False,
fontsize=fs['legend'],
ncol=3,
bbox_to_anchor=(0.5, 0.8, 0.5, 0.5))
ax.set_ylabel('precision@1', fontsize=fs['axis'])
ax.set_xlabel('vocabulary size', fontsize=fs['axis'])
plt.rc('xtick', labelsize=fs['ticks'])
plt.rc('ytick', labelsize=fs['ticks'])
if show: plt.show()
fn = 'Results/robust/vocab/vocab_' + param_string + '.png'
plt.savefig(fn, bbox_inches='tight', facecolor='white')
print('Write results to %s' % fn)
def plot_precision_vs_recall(hp,
files,
average_precision,
average_recall,
raters_p,
show=False,
legend_loc='none'):
add_eval_set_to_string = True
param_string = utils.create_param_string(hp, add_eval_set_to_string)
metrics, fs, lw, ms = get_models_style()
fig, ax = plt.subplots(figsize=(10, 7))
# fig, ax = plt.subplots()
max_pr = 0
for key in metrics.keys()[::-1]:
r = average_recall[metrics[key]['name']].tolist()[0]
p = average_precision[metrics[key]['name']].tolist()[0]
if max(p) > max_pr: max_pr = max(p)
plt.plot(r,
p,
lw=lw,
label=metrics[key]['legend'],
color=metrics[key]['color'],
marker=metrics[key]['marker'],
mew=2,
ms=ms)
# plt.axis('equal')
ax.set_xlabel('Recall', fontsize=fs['axis'])
ax.set_ylabel('Precision', fontsize=fs['axis'])
ax.set_xlim(left=0, right=1)
ax.set_ylim(bottom=0, top=1) # max_pr+0.1)
plt.rc('xtick', labelsize=fs['ticks'])
plt.rc('ytick', labelsize=fs['ticks'])
plt.axhline(y=raters_p,
color='grey',
linestyle='--',
linewidth=1,
label='rater agreement')
plt.text(0.02, 0.685, 'Inter-rater agreement', fontsize=20)
# plt.text(0.02, 0.65, 'Inter-rater agreement', fontsize=20)
legend_fs = 18
if legend_loc == 'up':
plt.legend(bbox_to_anchor=(-0.04, 1.02, 1, 0.2),
loc="lower left",
mode="expand",
ncol=3,
prop={'size': legend_fs},
frameon=False,
borderaxespad=0)
if legend_loc == 'in':
plt.legend(loc="upper left",
ncol=2,
prop={
'size': legend_fs,
'weight': 'bold'
},
frameon=False,
borderaxespad=0.9)
if legend_loc == 'side':
plt.legend(bbox_to_anchor=(1.04, 1),
loc="upper left",
ncol=1,
prop={'size': legend_fs},
frameon=False,
borderaxespad=0.9)
if show: plt.show()
fig.dpi = 600 # 200
save_figure_and_close(files, param_string, 'pr', extension='.png')
