ids_10 = np.random.choice(ids_10, min(N_EXAMPLE_MNPS,
ids_10.size), replace=False)
except ValueError:
pass
try:
ids_11 = np.random.choice(ids_11, min(N_EXAMPLE_MNPS,
ids_11.size), replace=False)
except ValueError:
pass
tmpH = plt.figure(figsize=(5.5,5.5))
mpl.rc('text', usetex=True)
mpl.rc('font', family='sans-serif')
helpers._plot_column(tmpH, 0, results['start_x'][-BATCH_SIZE:][ids_01],
results['end_x'][-BATCH_SIZE:][ids_01], im_shape=im_shape,
txt=r"0$\Rightarrow$1", clr1=GREEN, clr2=RED,
freqs=[freq_01, freq_00])
helpers._plot_column(tmpH, 0.25, results['start_x'][-BATCH_SIZE:][ids_00],
results['end_x'][-BATCH_SIZE:][ids_00], im_shape=im_shape,
txt=r"0$\Rightarrow$0", clr1=RED)
helpers._plot_column(tmpH, 0.5, results['start_x'][-BATCH_SIZE:][ids_10],
results['end_x'][-BATCH_SIZE:][ids_10], im_shape=im_shape,
txt=r"1$\Rightarrow$0", clr1=GREEN, clr2=RED,
freqs=[freq_10, freq_11])
helpers._plot_column(tmpH, 0.75, results['start_x'][-BATCH_SIZE:][ids_11],
results['end_x'][-BATCH_SIZE:][ids_11], im_shape=im_shape,
txt="1$\Rightarrow$1", clr1=RED)
plt.savefig(FIG_DIR+'final_MNIST.pdf', format='pdf')
def report(model, results, report_pdf, title,
batch_size, im_shape=(10,10)):
""" Report the reults of a causal feature
learning experiment in a pdf. """
print(' Printing batch report...')
N_EXAMPLE_MNPS = 9
# Gather model statistics.
time_vals = model.monitor.channels['valid_y_misclass'].epoch_record
true_time_vals = model.monitor.channels['valid_y_misclass'].\
time_record
valid_misclass = model.monitor.\
channels['valid_y_misclass'].val_record
training_misclass = model.monitor.\
channels['train_y_misclass'].val_record
test_misclass = model.monitor.\
channels['test_y_misclass'].val_record
arch = [layer.get_weights().shape[1] for layer in model.layers]
# Gather data statistics.
try:
end_id = np.where(results['end_y']==-1)[0][0]
except IndexError:
end_id = results['end_y'].size
start_id = max(end_id-batch_size, 0)
y_labels = np.unique(results['start_y']).size
succ_mnp = results['end_y'][start_id:end_id].flatten()==\
results['mnp_targets'][start_id:end_id].flatten()
succ_mnp = float((succ_mnp*(results['mnp_confs'][start_id:end_id].\
flatten() > 1./y_labels)).sum())
mnp_err = 1-succ_mnp/(end_id-start_id)
mnp_dist = np.abs(results['start_x'][start_id:end_id]-\
results['end_x'][start_id:end_id]).mean()
# Print era information.
tmpH = plt.figure()
plt.text(0.1, 0.9, 'Architecture {0}'.format(arch), fontsize=20)
plt.text(0.1, 0.8, title, fontsize=20)
plt.text(0.1, 0.7, 'Causal valid misclass {0:.2g}'.\
format(float(valid_misclass[-1])), fontsize=20)
plt.text(0.1, 0.6, 'Training set size {0}'.\
format(results['dataset_size'][-1]), fontsize=20)
plt.text(0.1, 0.5, 'Manipulation error {0:.2g}'.\
format(mnp_err),
fontsize=20)
plt.text(0.1, 0.4, 'Mnp distance / pixel {0:.2g}'.\
format(mnp_dist), fontsize=20)
plt.text(0.1, 0.3, 'Test misclass {0:.2g}'.\
format(float(test_misclass[-1])), fontsize=20)
plt.axis('off')
report_pdf.savefig(tmpH)
plt.close(tmpH)
results['lrn_times'].append(true_time_vals[-1]-true_time_vals[0])
results['csl_pred_error'].append(valid_misclass[-1])
results['mnp_dists'].append(mnp_dist)
results['mnp_errs'].append(mnp_err)
# Show the MLP training progress.
tmpAx = plt.subplot(1,1,1)
pHandles = tmpAx.plot(time_vals, training_misclass,
time_vals, valid_misclass,
time_vals, test_misclass)
plt.axis('tight')
tmpAx.legend(pHandles, ['Training Err', 'Valid Err', 'Test Err'])
tmpAx.set_xlabel('Epoch')
tmpAx.set_ylabel('Performance')
report_pdf.savefig(tmpAx.figure)
plt.close(tmpAx.figure)
# Display four examples (if present) for each case of
# 1->0, 0->1, 1->1, 0->0.
# Find ids of each class of manipulations.
ids_00 = np.where((results['start_y'][start_id:end_id].\
flatten()==0)*\
(results['end_y'][start_id:end_id].flatten()==0))[0]
ids_01 = np.where((results['start_y'][start_id:end_id].\
flatten()==0)*\
(results['end_y'][start_id:end_id].flatten()==1))[0]
ids_10 = np.where((results['start_y'][start_id:end_id].\
flatten()==1)*\
(results['end_y'][start_id:end_id].flatten()==0))[0]
ids_11 = np.where((results['start_y'][start_id:end_id].\
flatten()==1)*\
(results['end_y'][start_id:end_id].flatten()==1))[0]
freq_00, freq_01 = np.array([ids_00.size, ids_01.size])/\
(float(ids_00.size+ids_01.size))
freq_11, freq_10 = np.array([ids_11.size, ids_10.size])/\
(float(ids_11.size+ids_10.size))
try:
ids_00 = start_id+np.random.choice(ids_00, min(N_EXAMPLE_MNPS,
ids_00.size), replace=False)
except ValueError:
pass
try:
ids_01 = start_id+np.random.choice(ids_01, min(N_EXAMPLE_MNPS,
ids_01.size), replace=False)
except ValueError:
pass
try:
ids_10 = start_id+np.random.choice(ids_10, min(N_EXAMPLE_MNPS,
ids_10.size), replace=False)
except ValueError:
pass
try:
ids_11 = start_id+np.random.choice(ids_11, min(N_EXAMPLE_MNPS,
ids_11.size), replace=False)
except ValueError:
pass
GREEN=(122./255, 229./255, 28./255)
RED=(255./255, 75./255, 0)
tmpH = plt.figure(figsize=(5,5))
matplotlib.rc('text', usetex=True)
matplotlib.rc('font', family='sans-serif')
helpers._plot_column(tmpH, 0, results['start_x'][ids_01],
results['end_x'][ids_01], im_shape=im_shape,
txt=r"0$\Rightarrow$1", clr1=GREEN, clr2=RED,
freqs=[freq_01, freq_00])
helpers._plot_column(tmpH, 0.25, results['start_x'][ids_00],
results['end_x'][ids_00], im_shape=im_shape,
txt=r"0$\Rightarrow$0", clr1=RED)
helpers._plot_column(tmpH, 0.5, results['start_x'][ids_10],
results['end_x'][ids_10], im_shape=im_shape,
txt=r"1$\Rightarrow$0", clr1=GREEN, clr2=RED,
freqs=[freq_10, freq_11])
helpers._plot_column(tmpH, 0.75, results['start_x'][ids_11],
results['end_x'][ids_11], im_shape=im_shape,
txt="1$\Rightarrow$1", clr1=RED)
report_pdf.savefig(tmpH)
plt.close(tmpH)
print(' Causal Error {:.2g}'.format(float(valid_misclass[-1])))
print(' Manipulation Error {:.2g}'.format(mnp_err))
print(' Avg manipulation distance {:.2g}'.format(mnp_dist))
