def evaluate(model='ram', width=60, n_distractors=4, N=10):
"""Tests performance of trained model on larger/noisier mnist images."""
# data
mnist = input_data.read_data_sets('MNIST_data', one_hot=False)
# set parameters
#config.loc_std = 1e-10
config.num_glimpses = FLAGS.num_glimpses
config.n_patches = FLAGS.n_patches
config.use_context = FLAGS.use_context
config.convnet = FLAGS.convnet
config.sensor_size = config.glimpse_size**2 * config.n_patches * config.num_channels
config.N = mnist.train.num_examples # number of training examples
config.new_size = width
config.n_distractors = n_distractors
# init model
print('\n-- Model: {} --'.format(model))
print('Setting samplding SD to {:.4e}'.format(config.loc_std))
if model == 'ram':
net = RAM(config)
elif model == 'dram':
net = DRAM(config)
elif model == 'dram_loc':
net = DRAMl(config)
else:
print('Unknown model {}'.format(model))
exit()
net.load(FLAGS.load) # restore
net.count_params()
#params = net.return_params(['context_network/conv0/w:0'])
#net.plot_filters(params[0], fname=FLAGS.plot_dir + '.pdf')
#exit()
if FLAGS.visualize:
# create plot for current parameters
plot_dir = os.path.join(FLAGS.load, FLAGS.plot_dir)
if not os.path.exists(plot_dir):
os.mkdir(plot_dir)
task = {
'variant': FLAGS.task,
'width': width,
'n_distractors': n_distractors
}
net.visualize(data=mnist,
task=task,
config=config,
N=N,
plot_dir=plot_dir)
# evaluate
#test, val = net.evaluate(data=mnist, task=FLAGS.task)
return test, val
elif FLAGS.model == 'dram':
print('\n\n\nTraining DRAM\n\n\n')
model = DRAM(config, logdir=FLAGS.logdir)
elif FLAGS.model == 'dram_loc':
print('\n\n\nTraining DRAM with location ground truth\n\n\n')
model = DRAMl(config, logdir=FLAGS.logdir)
else:
print(('Unknown model {}'.format(FLAGS.model)))
exit()
# load if specified
if FLAGS.load is not None:
model.load(FLAGS.load)
model.visualize(config=[],
data=mnist,
task={
'variant': 'cluttered',
'width': 60,
'n_distractors': 4
},
plot_dir='.',
N=10,
seed=None)
# display # parameters
model.count_params()
# train
model.train(mnist, FLAGS.task)
model.evaluate(data=mnist, task=FLAGS.task)
def evaluate_numglimpses(model='dram_loc', visualize=False, N=10):
"""Tests performance of trained model on larger/noisier mnist images."""
# data
mnist = input_data.read_data_sets('MNIST_data', one_hot=False)
# set parameters
n_glimpses = [1, 2, 3, 4, 5, 6, 7, 8]
n_reps = N
width, noise = 100, 4
RESULTS = {}
for n in n_glimpses:
# set parameters
config.num_glimpses = n
config.n_patches = FLAGS.n_patches
config.use_context = FLAGS.use_context
config.convnet = FLAGS.convnet
config.sensor_size = config.glimpse_size**2 * config.n_patches * config.num_channels
config.N = mnist.train.num_examples # number of training examples
config.new_size = width
config.n_distractors = noise
# init model
print('\n-- Model: {} --'.format(model))
print('Setting samplding SD to {:.4e}'.format(config.loc_std))
tf.reset_default_graph()
if model == 'ram':
net = RAM(config)
elif model == 'dram':
net = DRAM(config)
elif model == 'dram_loc':
net = DRAMl(config)
else:
print('Unknown model {}'.format(model))
exit()
net.load(FLAGS.load) # restore
if FLAGS.visualize:
n_reps = 1
# create plot for current parameters
subfolder = os.path.join(FLAGS.load, FLAGS.plot_dir)
if not os.path.exists(subfolder):
os.mkdir(subfolder)
plot_dir = os.path.join(
subfolder, 'w={}_n_distractors={}'.format(width, noise))
if not os.path.exists(plot_dir):
os.mkdir(plot_dir)
task = {
'variant': FLAGS.task,
'width': width,
'n_distractors': noise
}
net.visualize(data=mnist,
task=task,
config=config,
plot_dir=plot_dir,
N=N)
# evaluate (n_reps) times
acc, _ = evaluate_repeatedly(ram=net,
data=mnist,
task=FLAGS.task,
n_reps=n_reps)
print(acc)
# store results
RESULTS[n] = acc
# save dictionary
with open(
os.path.join(
FLAGS.load,
'glimpses{}_results.pickle'.format(FLAGS.num_glimpses)),
'wb') as handle:
pickle.dump(RESULTS, handle, protocol=pickle.HIGHEST_PROTOCOL)