def categorical_test(build_cnn_fn, hyperpars, imgdat, runopts, networkstr,
get_eventids_hits_and_targets_fn, get_list_of_hits_fn):
"""
Run tests on the reserved test sample ("trainiing" examples with true
values to check that were not used for learning or validation); read the
data files in chunks into memory.
`get_eventids_hits_and_targets_fn` needs to extract from a data slice
a tuple of (eventids, [inputs], targets), where `[inputs]` might hold
a single view or all three, etc.
`get_list_of_hits_fn` needs to extract from a data slice a list of
`[inputs]` that might hold a single view or all three, etc.
"""
logger.info("Loading data for testing...")
tstamp = get_tstamp_from_model_name(runopts['save_model_file'])
train_sizes, valid_sizes, test_sizes = \
get_and_print_dataset_subsizes(runopts['data_file_list'])
used_sizes, used_data_size = get_used_data_sizes_for_testing(
train_sizes, valid_sizes, test_sizes, runopts['test_all_data'])
# Prepare Theano variables for inputs and targets
inputlist = networkstr['input_list']
target_var = T.ivector('targets')
# Build the model
network = build_cnn_fn(inputlist=inputlist,
imgw=imgdat['imgw'],
imgh=imgdat['imgh'],
convpooldictlist=networkstr['topology'],
nhidden=networkstr['nhidden'],
dropoutp=networkstr['dropoutp'],
noutputs=networkstr['noutputs'],
depth=networkstr['img_depth'])
with np.load(runopts['save_model_file']) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(network, param_values)
# Create a loss expression for testing.
test_prediction = lasagne.layers.get_output(network, deterministic=True)
l2_penalty = lasagne.regularization.regularize_layer_params(
lasagne.layers.get_all_layers(network),
lasagne.regularization.l2) * networkstr['l2_penalty_scale']
test_loss = categorical_crossentropy(test_prediction, target_var) + \
l2_penalty
test_loss = test_loss.mean()
# Also create an expression for the classification accuracy:
test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var),
dtype=theano.config.floatX)
# Look at the classifications
test_prediction_values = T.argmax(test_prediction, axis=1)
# Compute the actual predictions - also instructive is to look at
# `test_prediction` as an output (array of softmax probabilities)
pred_fn = theano.function(inputlist,
[test_prediction, test_prediction_values],
allow_input_downcast=True)
# Compile a function computing the validation loss and accuracy:
inputlist.append(target_var)
val_fn = theano.function(inputlist, [test_loss, test_acc],
allow_input_downcast=True)
logger.info("Starting testing...")
# compute and print the test error and...
test_err = 0
test_acc = 0
test_batches = 0
# look at some concrete predictions
num_poss_segs = networkstr['noutputs']
pred_target = np.zeros(num_poss_segs, dtype='float32')
true_target = np.zeros(num_poss_segs, dtype='float32')
targs_mat = np.zeros(num_poss_segs * num_poss_segs,
dtype='float32').reshape(num_poss_segs, num_poss_segs)
test_slices = []
for tsize in used_sizes:
test_slices.append(slices_maker(tsize, slice_size=50000))
test_set = None
verbose_evt_print_freq = 1
evtcounter = 0
for i, data_file in enumerate(runopts['data_file_list']):
for tslice in test_slices[i]:
t0 = time.time()
test_set = None
if runopts['test_all_data']:
test_set = load_all_datasubsets(data_file, tslice)
else:
test_set = load_datasubset(data_file, 'test', tslice)
_, test_dstream = make_scheme_and_stream(test_set,
1,
shuffle=False)
t1 = time.time()
logger.info(" Loading slice {} from {} took {:.3f}s.".format(
tslice, data_file, t1 - t0))
logger.debug(" dset sources: {}".format(
test_set.provides_sources))
t0 = time.time()
for data in test_dstream.get_epoch_iterator():
eventids, inputlist, targets = \
get_eventids_hits_and_targets_fn(data)
inputlist.append(targets)
err, acc = val_fn(*inputlist)
test_err += err
test_acc += acc
test_batches += 1
hits_list = get_list_of_hits_fn(data)
probs, pred = pred_fn(*hits_list)
pred_targ = zip(pred, targets)
evtcounter += 1
if runopts['be_verbose']:
if evtcounter % verbose_evt_print_freq == 0:
logger.info(
"{}/{} - {}: (prediction, true target): {}, {}".
format(evtcounter, used_data_size, eventids[0],
pred_targ, probs))
for p, t in pred_targ:
targs_mat[t][p] += 1
true_target[t] += 1
if p == t:
pred_target[p] += 1
t1 = time.time()
logger.info(" -Iterating over the slice took {:.3f}s.".format(t1 -
t0))
del test_set
del test_dstream
acc_target = 100.0 * pred_target / true_target.astype('float32')
perf_file = 'perfmat' + tstamp + '.npy'
np.save(perf_file, targs_mat)
logger.info("\nFinal results:"
"\n test loss:\t\t\t{:.6f}"
"\n test accuracy:\t\t{:.2f} %".format(
test_err / test_batches, test_acc / test_batches * 100))
for i, v in enumerate(acc_target):
logger.info(" target {} accuracy:\t\t\t{:.3f} %".format(
i, acc_target[i]))
def categorical_predict(build_cnn_fn, hyperpars, imgdat, runopts, networkstr,
get_eventids_hits_and_targets_fn,
get_id_tagged_inputlist_fn):
"""
Make predictions based on the model _only_ (e.g., this routine should
be used to produce prediction db's quickly or for data)
`get_eventids_hits_and_targets_fn` needs to extract from a data slice
a tuple of (eventids, [inputs], targets), where `[inputs]` might hold
a single view or all three, etc.
"""
logger.info("Loading data for prediction...")
train_sizes, valid_sizes, test_sizes = \
get_and_print_dataset_subsizes(runopts['data_file_list'])
used_sizes, used_data_size = get_used_data_sizes_for_testing(
train_sizes, valid_sizes, test_sizes, runopts['test_all_data'])
metadata = None
try:
import predictiondb
from sqlalchemy import MetaData
except ImportError:
logger.info("Cannot import sqlalchemy...")
write_db = False
if runopts['write_db']:
db_tbl_fun = None
if networkstr['noutputs'] == 67:
db_tbl_fun = predictiondb.get_67segment_prediction_table
elif networkstr['noutputs'] == 11:
db_tbl_fun = predictiondb.get_11segment_prediction_table
else:
raise Exception('Invalid number of outputs for DB tables.')
tstamp = get_tstamp_from_model_name(runopts['save_model_file'])
metadata = MetaData()
dbname = 'prediction' + tstamp
eng = predictiondb.get_engine(dbname)
con = predictiondb.get_connection(eng)
tbl = predictiondb.get_active_table(metadata,
eng,
get_table_fn=db_tbl_fun)
# Prepare Theano variables for inputs
inputlist = networkstr['input_list']
# Build the model
network = build_cnn_fn(inputlist=inputlist,
imgw=imgdat['imgw'],
imgh=imgdat['imgh'],
convpooldictlist=networkstr['topology'],
nhidden=networkstr['nhidden'],
dropoutp=networkstr['dropoutp'],
noutputs=networkstr['noutputs'],
depth=networkstr['img_depth'])
with np.load(runopts['save_model_file']) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(network, param_values)
# Compute the prediction
test_prediction = lasagne.layers.get_output(network, deterministic=True)
test_prediction_values = T.argmax(test_prediction, axis=1)
pred_fn = theano.function(inputlist,
[test_prediction, test_prediction_values],
allow_input_downcast=True)
logger.info("Starting prediction...")
test_slices = []
for tsize in used_sizes:
test_slices.append(slices_maker(tsize, slice_size=50000))
test_set = None
evtcounter = 0
verbose_evt_print_freq = hyperpars['batchsize'] * 4
for i, data_file in enumerate(runopts['data_file_list']):
for tslice in test_slices[i]:
t0 = time.time()
test_set = None
if runopts['test_all_data']:
test_set = load_all_datasubsets(data_file, tslice)
else:
test_set = load_datasubset(data_file, 'test', tslice)
_, test_dstream = make_scheme_and_stream(test_set,
hyperpars['batchsize'],
shuffle=False)
t1 = time.time()
logger.info(" Loading slice {} from {} took {:.3f}s.".format(
tslice, data_file, t1 - t0))
logger.debug(" dset sources: {}".format(
test_set.provides_sources))
t0 = time.time()
for data in test_dstream.get_epoch_iterator():
eventids, hits_list = get_id_tagged_inputlist_fn(data)
probs, pred = pred_fn(*hits_list)
evtcounter += hyperpars['batchsize']
if runopts['write_db']:
for i, evtid in enumerate(eventids):
filldb(tbl, con, evtid, pred[i], probs[i])
if runopts['be_verbose']:
if evtcounter % verbose_evt_print_freq == 0:
logger.info("processed {}/{}".format(
evtcounter, used_data_size))
t1 = time.time()
logger.info(" -Iterating over the slice took {:.3f}s.".format(t1 -
t0))
del test_set
del test_dstream
logger.info("Finished producing predictions!")
def categorical_predict(
build_cnn_fn, hyperpars, imgdat, runopts, networkstr,
get_eventids_hits_and_targets_fn, get_id_tagged_inputlist_fn
):
"""
Make predictions based on the model _only_ (e.g., this routine should
be used to produce prediction db's quickly or for data)
`get_eventids_hits_and_targets_fn` needs to extract from a data slice
a tuple of (eventids, [inputs], targets), where `[inputs]` might hold
a single view or all three, etc.
"""
logger.info("Loading data for prediction...")
train_sizes, valid_sizes, test_sizes = \
get_and_print_dataset_subsizes(runopts['data_file_list'])
used_sizes, used_data_size = get_used_data_sizes_for_testing(
train_sizes, valid_sizes, test_sizes, runopts['test_all_data']
)
metadata = None
try:
import predictiondb
from sqlalchemy import MetaData
except ImportError:
logger.info("Cannot import sqlalchemy...")
write_db = False
if runopts['write_db']:
tstamp = get_tstamp_from_model_name(runopts['save_model_file'])
metadata = MetaData()
dbname = 'prediction' + tstamp
eng = predictiondb.get_engine(dbname)
con = predictiondb.get_connection(eng)
tbl = predictiondb.get_active_table(metadata, eng)
# Prepare Theano variables for inputs
inputlist = networkstr['input_list']
# Build the model
network = build_cnn_fn(inputlist=inputlist,
imgw=imgdat['imgw'], imgh=imgdat['imgh'],
convpooldictlist=networkstr['topology'],
nhidden=networkstr['nhidden'],
dropoutp=networkstr['dropoutp'],
noutputs=networkstr['noutputs'],
depth=networkstr['img_depth']
)
with np.load(runopts['save_model_file']) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(network, param_values)
# Compute the prediction
test_prediction = lasagne.layers.get_output(network, deterministic=True)
test_prediction_values = T.argmax(test_prediction, axis=1)
pred_fn = theano.function(inputlist,
[test_prediction, test_prediction_values],
allow_input_downcast=True)
logger.info("Starting prediction...")
test_slices = []
for tsize in used_sizes:
test_slices.append(slices_maker(tsize, slice_size=50000))
test_set = None
evtcounter = 0
verbose_evt_print_freq = hyperpars['batchsize'] * 4
for i, data_file in enumerate(runopts['data_file_list']):
for tslice in test_slices[i]:
t0 = time.time()
test_set = None
if runopts['test_all_data']:
test_set = load_all_datasubsets(data_file, tslice)
else:
test_set = load_datasubset(data_file, 'test', tslice)
_, test_dstream = make_scheme_and_stream(test_set,
hyperpars['batchsize'],
shuffle=False)
t1 = time.time()
logger.info(" Loading slice {} from {} took {:.3f}s.".format(
tslice, data_file, t1 - t0)
)
logger.debug(
" dset sources: {}".format(test_set.provides_sources)
)
t0 = time.time()
for data in test_dstream.get_epoch_iterator():
eventids, hits_list = get_id_tagged_inputlist_fn(data)
probs, pred = pred_fn(*hits_list)
evtcounter += hyperpars['batchsize']
if runopts['write_db']:
for i, evtid in enumerate(eventids):
filldb(tbl, con, evtid, pred[i], probs[i])
if runopts['be_verbose']:
if evtcounter % verbose_evt_print_freq == 0:
logger.info("processed {}/{}". format(evtcounter,
used_data_size))
t1 = time.time()
logger.info(" -Iterating over the slice took {:.3f}s.".format(t1 - t0))
del test_set
del test_dstream
logger.info("Finished producing predictions!")
def categorical_test(
build_cnn_fn, hyperpars, imgdat, runopts, networkstr,
get_eventids_hits_and_targets_fn, get_list_of_hits_fn
):
"""
Run tests on the reserved test sample ("trainiing" examples with true
values to check that were not used for learning or validation); read the
data files in chunks into memory.
`get_eventids_hits_and_targets_fn` needs to extract from a data slice
a tuple of (eventids, [inputs], targets), where `[inputs]` might hold
a single view or all three, etc.
`get_list_of_hits_fn` needs to extract from a data slice a list of
`[inputs]` that might hold a single view or all three, etc.
"""
logger.info("Loading data for testing...")
tstamp = get_tstamp_from_model_name(runopts['save_model_file'])
train_sizes, valid_sizes, test_sizes = \
get_and_print_dataset_subsizes(runopts['data_file_list'])
used_sizes, used_data_size = get_used_data_sizes_for_testing(
train_sizes, valid_sizes, test_sizes, runopts['test_all_data']
)
# Prepare Theano variables for inputs and targets
inputlist = networkstr['input_list']
target_var = T.ivector('targets')
# Build the model
network = build_cnn_fn(inputlist=inputlist,
imgw=imgdat['imgw'], imgh=imgdat['imgh'],
convpooldictlist=networkstr['topology'],
nhidden=networkstr['nhidden'],
dropoutp=networkstr['dropoutp'],
noutputs=networkstr['noutputs'],
depth=networkstr['img_depth']
)
with np.load(runopts['save_model_file']) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(network, param_values)
# Create a loss expression for testing.
test_prediction = lasagne.layers.get_output(network, deterministic=True)
l2_penalty = lasagne.regularization.regularize_layer_params(
lasagne.layers.get_all_layers(network),
lasagne.regularization.l2) * networkstr['l2_penalty_scale']
test_loss = categorical_crossentropy(test_prediction, target_var) + \
l2_penalty
test_loss = test_loss.mean()
# Also create an expression for the classification accuracy:
test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var),
dtype=theano.config.floatX)
# Look at the classifications
test_prediction_values = T.argmax(test_prediction, axis=1)
# Compute the actual predictions - also instructive is to look at
# `test_prediction` as an output (array of softmax probabilities)
pred_fn = theano.function(inputlist,
[test_prediction, test_prediction_values],
allow_input_downcast=True)
# Compile a function computing the validation loss and accuracy:
inputlist.append(target_var)
val_fn = theano.function(inputlist, [test_loss, test_acc],
allow_input_downcast=True)
logger.info("Starting testing...")
# compute and print the test error and...
test_err = 0
test_acc = 0
test_batches = 0
# look at some concrete predictions
num_poss_segs = networkstr['noutputs']
pred_target = np.zeros(num_poss_segs, dtype='float32')
true_target = np.zeros(num_poss_segs, dtype='float32')
targs_mat = np.zeros(num_poss_segs * num_poss_segs,
dtype='float32').reshape(num_poss_segs, num_poss_segs)
test_slices = []
for tsize in used_sizes:
test_slices.append(slices_maker(tsize, slice_size=50000))
test_set = None
verbose_evt_print_freq = 1
evtcounter = 0
for i, data_file in enumerate(runopts['data_file_list']):
for tslice in test_slices[i]:
t0 = time.time()
test_set = None
if runopts['test_all_data']:
test_set = load_all_datasubsets(data_file, tslice)
else:
test_set = load_datasubset(data_file, 'test', tslice)
_, test_dstream = make_scheme_and_stream(
test_set, 1, shuffle=False
)
t1 = time.time()
logger.info(" Loading slice {} from {} took {:.3f}s.".format(
tslice, data_file, t1 - t0)
)
logger.debug(
" dset sources: {}".format(test_set.provides_sources)
)
t0 = time.time()
for data in test_dstream.get_epoch_iterator():
eventids, inputlist, targets = \
get_eventids_hits_and_targets_fn(data)
inputlist.append(targets)
err, acc = val_fn(*inputlist)
test_err += err
test_acc += acc
test_batches += 1
hits_list = get_list_of_hits_fn(data)
probs, pred = pred_fn(*hits_list)
pred_targ = zip(pred, targets)
evtcounter += 1
if runopts['be_verbose']:
if evtcounter % verbose_evt_print_freq == 0:
logger.info("{}/{} - {}: (prediction, true target): {}, {}".
format(evtcounter,
used_data_size,
eventids[0],
pred_targ, probs))
for p, t in pred_targ:
targs_mat[t][p] += 1
true_target[t] += 1
if p == t:
pred_target[p] += 1
t1 = time.time()
logger.info(" -Iterating over the slice took {:.3f}s.".format(t1 - t0))
del test_set
del test_dstream
acc_target = 100.0 * pred_target / true_target.astype('float32')
perf_file = 'perfmat' + tstamp + '.npy'
np.save(perf_file, targs_mat)
logger.info(
"\nFinal results:"
"\n test loss:\t\t\t{:.6f}"
"\n test accuracy:\t\t{:.2f} %".format(
test_err / test_batches, test_acc / test_batches * 100)
)
for i, v in enumerate(acc_target):
logger.info(" target {} accuracy:\t\t\t{:.3f} %".format(
i, acc_target[i]))
def view_layer_activations(build_cnn=None, data_file_list=None,
imgw=50, imgh=50, views='xuv', target_idx=5,
save_model_file='./params_file.npz',
be_verbose=False, convpooldictlist=None,
nhidden=None, dropoutp=None, write_db=True,
test_all_data=False, debug_print=False):
"""
Run tests on the reserved test sample ("trainiing" examples with true
values to check that were not used for learning or validation); read the
data files in chunks into memory.
"""
print("Loading data for testing...")
train_sizes, valid_sizes, test_sizes = \
get_and_print_dataset_subsizes(data_file_list)
used_sizes, _ = get_used_data_sizes_for_testing(train_sizes,
valid_sizes,
test_sizes,
test_all_data)
# extract timestamp from model file - assume it is the first set of numbers
# otherwise just use "now"
import re
tstamp = str(time.time()).split('.')[0]
m = re.search(r"[0-9]+", save_model_file)
if m:
tstamp = m.group(0)
# Prepare Theano variables for inputs and targets
input_var_x = T.tensor4('inputs')
input_var_u = T.tensor4('inputs')
input_var_v = T.tensor4('inputs')
inputlist = build_inputlist(input_var_x, input_var_u, input_var_v, views)
# Build the model
network = build_cnn(inputlist=inputlist, imgw=imgw, imgh=imgh,
convpooldictlist=convpooldictlist, nhidden=nhidden,
dropoutp=dropoutp, noutputs=noutputs)
with np.load(save_model_file) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(network, param_values)
print(network_repr.get_network_str(
lasagne.layers.get_all_layers(network),
get_network=False, incomings=True, outgoings=True))
layers = lasagne.layers.get_all_layers(network)
# layer assignment is _highly_ network specific...
layer_conv_x1 = lasagne.layers.get_output(layers[1])
layer_conv_u1 = lasagne.layers.get_output(layers[8])
layer_conv_v1 = lasagne.layers.get_output(layers[15])
layer_pool_x1 = lasagne.layers.get_output(layers[2])
layer_pool_u1 = lasagne.layers.get_output(layers[9])
layer_pool_v1 = lasagne.layers.get_output(layers[16])
layer_conv_x2 = lasagne.layers.get_output(layers[3])
layer_conv_u2 = lasagne.layers.get_output(layers[10])
layer_conv_v2 = lasagne.layers.get_output(layers[17])
layer_pool_x2 = lasagne.layers.get_output(layers[4])
layer_pool_u2 = lasagne.layers.get_output(layers[11])
layer_pool_v2 = lasagne.layers.get_output(layers[18])
vis_conv_x1 = theano.function(inputlist,
[layer_conv_x1],
allow_input_downcast=True,
on_unused_input='warn')
vis_conv_u1 = theano.function(inputlist,
[layer_conv_u1],
allow_input_downcast=True,
on_unused_input='warn')
vis_conv_v1 = theano.function(inputlist,
[layer_conv_v1],
allow_input_downcast=True,
on_unused_input='warn')
vis_pool_x1 = theano.function(inputlist,
[layer_pool_x1],
allow_input_downcast=True,
on_unused_input='warn')
vis_pool_u1 = theano.function(inputlist,
[layer_pool_u1],
allow_input_downcast=True,
on_unused_input='warn')
vis_pool_v1 = theano.function(inputlist,
[layer_pool_v1],
allow_input_downcast=True,
on_unused_input='warn')
vis_conv_x2 = theano.function(inputlist,
[layer_conv_x2],
allow_input_downcast=True,
on_unused_input='warn')
vis_conv_u2 = theano.function(inputlist,
[layer_conv_u2],
allow_input_downcast=True,
on_unused_input='warn')
vis_conv_v2 = theano.function(inputlist,
[layer_conv_v2],
allow_input_downcast=True,
on_unused_input='warn')
vis_pool_x2 = theano.function(inputlist,
[layer_pool_x2],
allow_input_downcast=True,
on_unused_input='warn')
vis_pool_u2 = theano.function(inputlist,
[layer_pool_u2],
allow_input_downcast=True,
on_unused_input='warn')
vis_pool_v2 = theano.function(inputlist,
[layer_pool_v2],
allow_input_downcast=True,
on_unused_input='warn')
print("Starting visualization...")
test_slices = []
for tsize in used_sizes:
test_slices.append(slices_maker(tsize, slice_size=50000))
test_set = None
for i, data_file in enumerate(data_file_list):
for tslice in test_slices[i]:
t0 = time.time()
test_set = None
if test_all_data:
test_set = load_all_datasubsets(data_file, tslice)
else:
test_set = load_datasubset(data_file, 'test', tslice)
_, test_dstream = make_scheme_and_stream(test_set, 1,
shuffle=False)
t1 = time.time()
print(" Loading slice {} from {} took {:.3f}s.".format(
tslice, data_file, t1 - t0))
if debug_print:
print(" dset sources:", test_set.provides_sources)
t0 = time.time()
for data in test_dstream.get_epoch_iterator():
# data order in the hdf5 looks like:
# ids, hits-u, hits-v, hits-x, planes, segments, zs
# (Check the file carefully for data names, etc.)
eventids, inputlist, targets = \
get_eventids_hits_and_targets_from_data(
data, views, target_idx)
conv_x1 = vis_conv_x1(*inputlist)
conv_u1 = vis_conv_u1(*inputlist)
conv_v1 = vis_conv_v1(*inputlist)
pool_x1 = vis_pool_x1(*inputlist)
pool_u1 = vis_pool_u1(*inputlist)
pool_v1 = vis_pool_v1(*inputlist)
conv_x2 = vis_conv_x2(*inputlist)
conv_u2 = vis_conv_u2(*inputlist)
conv_v2 = vis_conv_v2(*inputlist)
pool_x2 = vis_pool_x2(*inputlist)
pool_u2 = vis_pool_u2(*inputlist)
pool_v2 = vis_pool_v2(*inputlist)
vis_file = 'vis_' + str(targets[0]) + '_conv_1_' + tstamp + \
'_' + str(eventids[0]) + '.npy'
np.save(vis_file, [conv_x1, conv_u1, conv_v1])
vis_file = 'vis_' + str(targets[0]) + '_pool_1_' + tstamp + \
'_' + str(eventids[0]) + '.npy'
np.save(vis_file, [pool_x1, pool_u1, pool_v1])
vis_file = 'vis_' + str(targets[0]) + '_conv_2_' + tstamp + \
'_' + str(eventids[0]) + '.npy'
np.save(vis_file, [conv_x2, conv_u2, conv_v2])
vis_file = 'vis_' + str(targets[0]) + '_pool_2_' + tstamp + \
'_' + str(eventids[0]) + '.npy'
np.save(vis_file, [pool_x2, pool_u2, pool_v2])
t1 = time.time()
print(" -Iterating over the slice took {:.3f}s.".format(t1 - t0))
del test_set
del test_dstream
def categorical_predict(build_cnn=None, data_file_list=None,
views='xuv', imgw=50, imgh=50, target_idx=5,
save_model_file='./params_file.npz',
be_verbose=False, convpooldictlist=None,
nhidden=None, dropoutp=None, write_db=True,
test_all_data=False, debug_print=False,
noutputs=11):
"""
Make predictions based on the model _only_ (e.g., this routine should
be used to produce prediction db's quickly or for data)
noutputs=11 for zsegments, other vals for planecodes, etc.
"""
print("Loading data for testing...")
train_sizes, valid_sizes, test_sizes = \
get_and_print_dataset_subsizes(data_file_list)
used_sizes, used_data_size = get_used_data_sizes_for_testing(train_sizes,
valid_sizes,
test_sizes,
test_all_data)
metadata = None
try:
import predictiondb
from sqlalchemy import MetaData
except ImportError:
print("Cannot import sqlalchemy...")
write_db = False
if write_db:
tstamp = get_tstamp_from_model_name(save_model_file)
metadata = MetaData()
dbname = 'prediction' + tstamp
eng = predictiondb.get_engine(dbname)
con = predictiondb.get_connection(eng)
tbl = predictiondb.get_active_table(metadata, eng)
# Prepare Theano variables for inputs
input_var_x = T.tensor4('inputs')
input_var_u = T.tensor4('inputs')
input_var_v = T.tensor4('inputs')
inputlist = build_inputlist(input_var_x, input_var_u, input_var_v, views)
# Build the model
network = build_cnn(inputlist=inputlist, imgw=imgw, imgh=imgh,
convpooldictlist=convpooldictlist, nhidden=nhidden,
dropoutp=dropoutp, noutputs=noutputs)
with np.load(save_model_file) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(network, param_values)
# Compute the prediction
test_prediction = lasagne.layers.get_output(network, deterministic=True)
test_prediction_values = T.argmax(test_prediction, axis=1)
pred_fn = theano.function(inputlist,
[test_prediction, test_prediction_values],
allow_input_downcast=True)
print("Starting prediction...")
test_slices = []
for tsize in used_sizes:
test_slices.append(slices_maker(tsize, slice_size=50000))
test_set = None
evtcounter = 0
batch_size = 500
evt_print_freq = batch_size * 4
for i, data_file in enumerate(data_file_list):
for tslice in test_slices[i]:
t0 = time.time()
test_set = None
if test_all_data:
test_set = load_all_datasubsets(data_file, tslice)
else:
test_set = load_datasubset(data_file, 'test', tslice)
_, test_dstream = make_scheme_and_stream(test_set,
batch_size,
shuffle=False)
t1 = time.time()
print(" Loading slice {} from {} took {:.3f}s.".format(
tslice, data_file, t1 - t0))
if debug_print:
print(" dset sources:", test_set.provides_sources)
t0 = time.time()
for data in test_dstream.get_epoch_iterator():
eventids, hits_list = \
get_id_tagged_inputlist_from_data(data, views)
probs, pred = pred_fn(*hits_list)
evtcounter += batch_size
if write_db:
for i, evtid in enumerate(eventids):
filldb(tbl, con, evtid, pred[i], probs[i])
if be_verbose:
if evtcounter % evt_print_freq == 0:
print("processed {}/{}". format(evtcounter,
used_data_size))
t1 = time.time()
print(" -Iterating over the slice took {:.3f}s.".format(t1 - t0))
del test_set
del test_dstream
print("Finished producing predictions!")
def categorical_test(build_cnn=None, data_file_list=None,
l2_penalty_scale=1e-04, views='xuv',
imgw=50, imgh=50, target_idx=5,
save_model_file='./params_file.npz',
be_verbose=False, convpooldictlist=None,
nhidden=None, dropoutp=None,
test_all_data=False, debug_print=False,
noutputs=11):
"""
Run tests on the reserved test sample ("trainiing" examples with true
values to check that were not used for learning or validation); read the
data files in chunks into memory.
noutputs=11 for zsegments, other vals for planecodes, etc.
"""
print("Loading data for testing...")
tstamp = get_tstamp_from_model_name(save_model_file)
train_sizes, valid_sizes, test_sizes = \
get_and_print_dataset_subsizes(data_file_list)
used_sizes, used_data_size = get_used_data_sizes_for_testing(train_sizes,
valid_sizes,
test_sizes,
test_all_data)
# Prepare Theano variables for inputs and targets
input_var_x = T.tensor4('inputs')
input_var_u = T.tensor4('inputs')
input_var_v = T.tensor4('inputs')
target_var = T.ivector('targets')
inputlist = build_inputlist(input_var_x, input_var_u, input_var_v, views)
# Build the model
network = build_cnn(inputlist=inputlist, imgw=imgw, imgh=imgh,
convpooldictlist=convpooldictlist, nhidden=nhidden,
dropoutp=dropoutp, noutputs=noutputs)
with np.load(save_model_file) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(network, param_values)
# Create a loss expression for testing.
test_prediction = lasagne.layers.get_output(network, deterministic=True)
l2_penalty = lasagne.regularization.regularize_layer_params(
lasagne.layers.get_all_layers(network),
lasagne.regularization.l2) * l2_penalty_scale
test_loss = categorical_crossentropy(test_prediction, target_var) + \
l2_penalty
test_loss = test_loss.mean()
# Also create an expression for the classification accuracy:
test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var),
dtype=theano.config.floatX)
# Look at the classifications
test_prediction_values = T.argmax(test_prediction, axis=1)
# Compute the actual predictions - also instructive is to look at
# `test_prediction` as an output (array of softmax probabilities)
pred_fn = theano.function(inputlist,
[test_prediction, test_prediction_values],
allow_input_downcast=True)
# Compile a function computing the validation loss and accuracy:
inputlist.append(target_var)
val_fn = theano.function(inputlist, [test_loss, test_acc],
allow_input_downcast=True)
print("Starting testing...")
# compute and print the test error and...
test_err = 0
test_acc = 0
test_batches = 0
# look at some concrete predictions
num_poss_segs = noutputs
pred_target = np.zeros(num_poss_segs, dtype='float32')
true_target = np.zeros(num_poss_segs, dtype='float32')
targs_mat = np.zeros(num_poss_segs * num_poss_segs,
dtype='float32').reshape(num_poss_segs, num_poss_segs)
test_slices = []
for tsize in used_sizes:
test_slices.append(slices_maker(tsize, slice_size=50000))
test_set = None
evt_print_freq = 1 # TODO: pass this in?
evtcounter = 0
for i, data_file in enumerate(data_file_list):
for tslice in test_slices[i]:
t0 = time.time()
test_set = None
if test_all_data:
test_set = load_all_datasubsets(data_file, tslice)
else:
test_set = load_datasubset(data_file, 'test', tslice)
_, test_dstream = make_scheme_and_stream(test_set, 1,
shuffle=False)
t1 = time.time()
print(" Loading slice {} from {} took {:.3f}s.".format(
tslice, data_file, t1 - t0))
if debug_print:
print(" dset sources:", test_set.provides_sources)
t0 = time.time()
for data in test_dstream.get_epoch_iterator():
eventids, inputlist, targets = \
get_eventids_hits_and_targets_from_data(
data, views, target_idx)
inputlist.append(targets)
err, acc = val_fn(*inputlist)
test_err += err
test_acc += acc
test_batches += 1
hits_list = get_list_of_hits_from_data(data, views)
probs, pred = pred_fn(*hits_list)
pred_targ = zip(pred, targets)
evtcounter += 1
if be_verbose:
if evtcounter % evt_print_freq == 0:
print("{}/{} - {}: (prediction, true target): {}, {}".
format(evtcounter,
used_data_size,
eventids[0],
pred_targ, probs))
for p, t in pred_targ:
targs_mat[t][p] += 1
true_target[t] += 1
if p == t:
pred_target[p] += 1
t1 = time.time()
print(" -Iterating over the slice took {:.3f}s.".format(t1 - t0))
del test_set
del test_dstream
acc_target = 100.0 * pred_target / true_target.astype('float32')
perf_file = 'perfmat' + tstamp + '.npy'
np.save(perf_file, targs_mat)
print("Final results:")
print(" test loss:\t\t\t{:.6f}".format(test_err / test_batches))
print(" test accuracy:\t\t{:.2f} %".format(
test_acc / test_batches * 100))
for i, v in enumerate(acc_target):
print(" target {} accuracy:\t\t\t{:.3f} %".format(
i, acc_target[i]))
