def categorical_test(build_cnn_fn, hyperpars, imgdat, runopts, networkstr, get_eventids_hits_and_targets_fn, get_list_of_hits_fn): """ Run tests on the reserved test sample ("trainiing" examples with true values to check that were not used for learning or validation); read the data files in chunks into memory. `get_eventids_hits_and_targets_fn` needs to extract from a data slice a tuple of (eventids, [inputs], targets), where `[inputs]` might hold a single view or all three, etc. `get_list_of_hits_fn` needs to extract from a data slice a list of `[inputs]` that might hold a single view or all three, etc. """ logger.info("Loading data for testing...") tstamp = get_tstamp_from_model_name(runopts['save_model_file']) train_sizes, valid_sizes, test_sizes = \ get_and_print_dataset_subsizes(runopts['data_file_list']) used_sizes, used_data_size = get_used_data_sizes_for_testing( train_sizes, valid_sizes, test_sizes, runopts['test_all_data']) # Prepare Theano variables for inputs and targets inputlist = networkstr['input_list'] target_var = T.ivector('targets') # Build the model network = build_cnn_fn(inputlist=inputlist, imgw=imgdat['imgw'], imgh=imgdat['imgh'], convpooldictlist=networkstr['topology'], nhidden=networkstr['nhidden'], dropoutp=networkstr['dropoutp'], noutputs=networkstr['noutputs'], depth=networkstr['img_depth']) with np.load(runopts['save_model_file']) as f: param_values = [f['arr_%d' % i] for i in range(len(f.files))] lasagne.layers.set_all_param_values(network, param_values) # Create a loss expression for testing. test_prediction = lasagne.layers.get_output(network, deterministic=True) l2_penalty = lasagne.regularization.regularize_layer_params( lasagne.layers.get_all_layers(network), lasagne.regularization.l2) * networkstr['l2_penalty_scale'] test_loss = categorical_crossentropy(test_prediction, target_var) + \ l2_penalty test_loss = test_loss.mean() # Also create an expression for the classification accuracy: test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var), dtype=theano.config.floatX) # Look at the classifications test_prediction_values = T.argmax(test_prediction, axis=1) # Compute the actual predictions - also instructive is to look at # `test_prediction` as an output (array of softmax probabilities) pred_fn = theano.function(inputlist, [test_prediction, test_prediction_values], allow_input_downcast=True) # Compile a function computing the validation loss and accuracy: inputlist.append(target_var) val_fn = theano.function(inputlist, [test_loss, test_acc], allow_input_downcast=True) logger.info("Starting testing...") # compute and print the test error and... test_err = 0 test_acc = 0 test_batches = 0 # look at some concrete predictions num_poss_segs = networkstr['noutputs'] pred_target = np.zeros(num_poss_segs, dtype='float32') true_target = np.zeros(num_poss_segs, dtype='float32') targs_mat = np.zeros(num_poss_segs * num_poss_segs, dtype='float32').reshape(num_poss_segs, num_poss_segs) test_slices = [] for tsize in used_sizes: test_slices.append(slices_maker(tsize, slice_size=50000)) test_set = None verbose_evt_print_freq = 1 evtcounter = 0 for i, data_file in enumerate(runopts['data_file_list']): for tslice in test_slices[i]: t0 = time.time() test_set = None if runopts['test_all_data']: test_set = load_all_datasubsets(data_file, tslice) else: test_set = load_datasubset(data_file, 'test', tslice) _, test_dstream = make_scheme_and_stream(test_set, 1, shuffle=False) t1 = time.time() logger.info(" Loading slice {} from {} took {:.3f}s.".format( tslice, data_file, t1 - t0)) logger.debug(" dset sources: {}".format( test_set.provides_sources)) t0 = time.time() for data in test_dstream.get_epoch_iterator(): eventids, inputlist, targets = \ get_eventids_hits_and_targets_fn(data) inputlist.append(targets) err, acc = val_fn(*inputlist) test_err += err test_acc += acc test_batches += 1 hits_list = get_list_of_hits_fn(data) probs, pred = pred_fn(*hits_list) pred_targ = zip(pred, targets) evtcounter += 1 if runopts['be_verbose']: if evtcounter % verbose_evt_print_freq == 0: logger.info( "{}/{} - {}: (prediction, true target): {}, {}". format(evtcounter, used_data_size, eventids[0], pred_targ, probs)) for p, t in pred_targ: targs_mat[t][p] += 1 true_target[t] += 1 if p == t: pred_target[p] += 1 t1 = time.time() logger.info(" -Iterating over the slice took {:.3f}s.".format(t1 - t0)) del test_set del test_dstream acc_target = 100.0 * pred_target / true_target.astype('float32') perf_file = 'perfmat' + tstamp + '.npy' np.save(perf_file, targs_mat) logger.info("\nFinal results:" "\n test loss:\t\t\t{:.6f}" "\n test accuracy:\t\t{:.2f} %".format( test_err / test_batches, test_acc / test_batches * 100)) for i, v in enumerate(acc_target): logger.info(" target {} accuracy:\t\t\t{:.3f} %".format( i, acc_target[i]))
def categorical_predict(build_cnn_fn, hyperpars, imgdat, runopts, networkstr, get_eventids_hits_and_targets_fn, get_id_tagged_inputlist_fn): """ Make predictions based on the model _only_ (e.g., this routine should be used to produce prediction db's quickly or for data) `get_eventids_hits_and_targets_fn` needs to extract from a data slice a tuple of (eventids, [inputs], targets), where `[inputs]` might hold a single view or all three, etc. """ logger.info("Loading data for prediction...") train_sizes, valid_sizes, test_sizes = \ get_and_print_dataset_subsizes(runopts['data_file_list']) used_sizes, used_data_size = get_used_data_sizes_for_testing( train_sizes, valid_sizes, test_sizes, runopts['test_all_data']) metadata = None try: import predictiondb from sqlalchemy import MetaData except ImportError: logger.info("Cannot import sqlalchemy...") write_db = False if runopts['write_db']: db_tbl_fun = None if networkstr['noutputs'] == 67: db_tbl_fun = predictiondb.get_67segment_prediction_table elif networkstr['noutputs'] == 11: db_tbl_fun = predictiondb.get_11segment_prediction_table else: raise Exception('Invalid number of outputs for DB tables.') tstamp = get_tstamp_from_model_name(runopts['save_model_file']) metadata = MetaData() dbname = 'prediction' + tstamp eng = predictiondb.get_engine(dbname) con = predictiondb.get_connection(eng) tbl = predictiondb.get_active_table(metadata, eng, get_table_fn=db_tbl_fun) # Prepare Theano variables for inputs inputlist = networkstr['input_list'] # Build the model network = build_cnn_fn(inputlist=inputlist, imgw=imgdat['imgw'], imgh=imgdat['imgh'], convpooldictlist=networkstr['topology'], nhidden=networkstr['nhidden'], dropoutp=networkstr['dropoutp'], noutputs=networkstr['noutputs'], depth=networkstr['img_depth']) with np.load(runopts['save_model_file']) as f: param_values = [f['arr_%d' % i] for i in range(len(f.files))] lasagne.layers.set_all_param_values(network, param_values) # Compute the prediction test_prediction = lasagne.layers.get_output(network, deterministic=True) test_prediction_values = T.argmax(test_prediction, axis=1) pred_fn = theano.function(inputlist, [test_prediction, test_prediction_values], allow_input_downcast=True) logger.info("Starting prediction...") test_slices = [] for tsize in used_sizes: test_slices.append(slices_maker(tsize, slice_size=50000)) test_set = None evtcounter = 0 verbose_evt_print_freq = hyperpars['batchsize'] * 4 for i, data_file in enumerate(runopts['data_file_list']): for tslice in test_slices[i]: t0 = time.time() test_set = None if runopts['test_all_data']: test_set = load_all_datasubsets(data_file, tslice) else: test_set = load_datasubset(data_file, 'test', tslice) _, test_dstream = make_scheme_and_stream(test_set, hyperpars['batchsize'], shuffle=False) t1 = time.time() logger.info(" Loading slice {} from {} took {:.3f}s.".format( tslice, data_file, t1 - t0)) logger.debug(" dset sources: {}".format( test_set.provides_sources)) t0 = time.time() for data in test_dstream.get_epoch_iterator(): eventids, hits_list = get_id_tagged_inputlist_fn(data) probs, pred = pred_fn(*hits_list) evtcounter += hyperpars['batchsize'] if runopts['write_db']: for i, evtid in enumerate(eventids): filldb(tbl, con, evtid, pred[i], probs[i]) if runopts['be_verbose']: if evtcounter % verbose_evt_print_freq == 0: logger.info("processed {}/{}".format( evtcounter, used_data_size)) t1 = time.time() logger.info(" -Iterating over the slice took {:.3f}s.".format(t1 - t0)) del test_set del test_dstream logger.info("Finished producing predictions!")
def categorical_predict( build_cnn_fn, hyperpars, imgdat, runopts, networkstr, get_eventids_hits_and_targets_fn, get_id_tagged_inputlist_fn ): """ Make predictions based on the model _only_ (e.g., this routine should be used to produce prediction db's quickly or for data) `get_eventids_hits_and_targets_fn` needs to extract from a data slice a tuple of (eventids, [inputs], targets), where `[inputs]` might hold a single view or all three, etc. """ logger.info("Loading data for prediction...") train_sizes, valid_sizes, test_sizes = \ get_and_print_dataset_subsizes(runopts['data_file_list']) used_sizes, used_data_size = get_used_data_sizes_for_testing( train_sizes, valid_sizes, test_sizes, runopts['test_all_data'] ) metadata = None try: import predictiondb from sqlalchemy import MetaData except ImportError: logger.info("Cannot import sqlalchemy...") write_db = False if runopts['write_db']: tstamp = get_tstamp_from_model_name(runopts['save_model_file']) metadata = MetaData() dbname = 'prediction' + tstamp eng = predictiondb.get_engine(dbname) con = predictiondb.get_connection(eng) tbl = predictiondb.get_active_table(metadata, eng) # Prepare Theano variables for inputs inputlist = networkstr['input_list'] # Build the model network = build_cnn_fn(inputlist=inputlist, imgw=imgdat['imgw'], imgh=imgdat['imgh'], convpooldictlist=networkstr['topology'], nhidden=networkstr['nhidden'], dropoutp=networkstr['dropoutp'], noutputs=networkstr['noutputs'], depth=networkstr['img_depth'] ) with np.load(runopts['save_model_file']) as f: param_values = [f['arr_%d' % i] for i in range(len(f.files))] lasagne.layers.set_all_param_values(network, param_values) # Compute the prediction test_prediction = lasagne.layers.get_output(network, deterministic=True) test_prediction_values = T.argmax(test_prediction, axis=1) pred_fn = theano.function(inputlist, [test_prediction, test_prediction_values], allow_input_downcast=True) logger.info("Starting prediction...") test_slices = [] for tsize in used_sizes: test_slices.append(slices_maker(tsize, slice_size=50000)) test_set = None evtcounter = 0 verbose_evt_print_freq = hyperpars['batchsize'] * 4 for i, data_file in enumerate(runopts['data_file_list']): for tslice in test_slices[i]: t0 = time.time() test_set = None if runopts['test_all_data']: test_set = load_all_datasubsets(data_file, tslice) else: test_set = load_datasubset(data_file, 'test', tslice) _, test_dstream = make_scheme_and_stream(test_set, hyperpars['batchsize'], shuffle=False) t1 = time.time() logger.info(" Loading slice {} from {} took {:.3f}s.".format( tslice, data_file, t1 - t0) ) logger.debug( " dset sources: {}".format(test_set.provides_sources) ) t0 = time.time() for data in test_dstream.get_epoch_iterator(): eventids, hits_list = get_id_tagged_inputlist_fn(data) probs, pred = pred_fn(*hits_list) evtcounter += hyperpars['batchsize'] if runopts['write_db']: for i, evtid in enumerate(eventids): filldb(tbl, con, evtid, pred[i], probs[i]) if runopts['be_verbose']: if evtcounter % verbose_evt_print_freq == 0: logger.info("processed {}/{}". format(evtcounter, used_data_size)) t1 = time.time() logger.info(" -Iterating over the slice took {:.3f}s.".format(t1 - t0)) del test_set del test_dstream logger.info("Finished producing predictions!")
def categorical_test( build_cnn_fn, hyperpars, imgdat, runopts, networkstr, get_eventids_hits_and_targets_fn, get_list_of_hits_fn ): """ Run tests on the reserved test sample ("trainiing" examples with true values to check that were not used for learning or validation); read the data files in chunks into memory. `get_eventids_hits_and_targets_fn` needs to extract from a data slice a tuple of (eventids, [inputs], targets), where `[inputs]` might hold a single view or all three, etc. `get_list_of_hits_fn` needs to extract from a data slice a list of `[inputs]` that might hold a single view or all three, etc. """ logger.info("Loading data for testing...") tstamp = get_tstamp_from_model_name(runopts['save_model_file']) train_sizes, valid_sizes, test_sizes = \ get_and_print_dataset_subsizes(runopts['data_file_list']) used_sizes, used_data_size = get_used_data_sizes_for_testing( train_sizes, valid_sizes, test_sizes, runopts['test_all_data'] ) # Prepare Theano variables for inputs and targets inputlist = networkstr['input_list'] target_var = T.ivector('targets') # Build the model network = build_cnn_fn(inputlist=inputlist, imgw=imgdat['imgw'], imgh=imgdat['imgh'], convpooldictlist=networkstr['topology'], nhidden=networkstr['nhidden'], dropoutp=networkstr['dropoutp'], noutputs=networkstr['noutputs'], depth=networkstr['img_depth'] ) with np.load(runopts['save_model_file']) as f: param_values = [f['arr_%d' % i] for i in range(len(f.files))] lasagne.layers.set_all_param_values(network, param_values) # Create a loss expression for testing. test_prediction = lasagne.layers.get_output(network, deterministic=True) l2_penalty = lasagne.regularization.regularize_layer_params( lasagne.layers.get_all_layers(network), lasagne.regularization.l2) * networkstr['l2_penalty_scale'] test_loss = categorical_crossentropy(test_prediction, target_var) + \ l2_penalty test_loss = test_loss.mean() # Also create an expression for the classification accuracy: test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var), dtype=theano.config.floatX) # Look at the classifications test_prediction_values = T.argmax(test_prediction, axis=1) # Compute the actual predictions - also instructive is to look at # `test_prediction` as an output (array of softmax probabilities) pred_fn = theano.function(inputlist, [test_prediction, test_prediction_values], allow_input_downcast=True) # Compile a function computing the validation loss and accuracy: inputlist.append(target_var) val_fn = theano.function(inputlist, [test_loss, test_acc], allow_input_downcast=True) logger.info("Starting testing...") # compute and print the test error and... test_err = 0 test_acc = 0 test_batches = 0 # look at some concrete predictions num_poss_segs = networkstr['noutputs'] pred_target = np.zeros(num_poss_segs, dtype='float32') true_target = np.zeros(num_poss_segs, dtype='float32') targs_mat = np.zeros(num_poss_segs * num_poss_segs, dtype='float32').reshape(num_poss_segs, num_poss_segs) test_slices = [] for tsize in used_sizes: test_slices.append(slices_maker(tsize, slice_size=50000)) test_set = None verbose_evt_print_freq = 1 evtcounter = 0 for i, data_file in enumerate(runopts['data_file_list']): for tslice in test_slices[i]: t0 = time.time() test_set = None if runopts['test_all_data']: test_set = load_all_datasubsets(data_file, tslice) else: test_set = load_datasubset(data_file, 'test', tslice) _, test_dstream = make_scheme_and_stream( test_set, 1, shuffle=False ) t1 = time.time() logger.info(" Loading slice {} from {} took {:.3f}s.".format( tslice, data_file, t1 - t0) ) logger.debug( " dset sources: {}".format(test_set.provides_sources) ) t0 = time.time() for data in test_dstream.get_epoch_iterator(): eventids, inputlist, targets = \ get_eventids_hits_and_targets_fn(data) inputlist.append(targets) err, acc = val_fn(*inputlist) test_err += err test_acc += acc test_batches += 1 hits_list = get_list_of_hits_fn(data) probs, pred = pred_fn(*hits_list) pred_targ = zip(pred, targets) evtcounter += 1 if runopts['be_verbose']: if evtcounter % verbose_evt_print_freq == 0: logger.info("{}/{} - {}: (prediction, true target): {}, {}". format(evtcounter, used_data_size, eventids[0], pred_targ, probs)) for p, t in pred_targ: targs_mat[t][p] += 1 true_target[t] += 1 if p == t: pred_target[p] += 1 t1 = time.time() logger.info(" -Iterating over the slice took {:.3f}s.".format(t1 - t0)) del test_set del test_dstream acc_target = 100.0 * pred_target / true_target.astype('float32') perf_file = 'perfmat' + tstamp + '.npy' np.save(perf_file, targs_mat) logger.info( "\nFinal results:" "\n test loss:\t\t\t{:.6f}" "\n test accuracy:\t\t{:.2f} %".format( test_err / test_batches, test_acc / test_batches * 100) ) for i, v in enumerate(acc_target): logger.info(" target {} accuracy:\t\t\t{:.3f} %".format( i, acc_target[i]))
def view_layer_activations(build_cnn=None, data_file_list=None, imgw=50, imgh=50, views='xuv', target_idx=5, save_model_file='./params_file.npz', be_verbose=False, convpooldictlist=None, nhidden=None, dropoutp=None, write_db=True, test_all_data=False, debug_print=False): """ Run tests on the reserved test sample ("trainiing" examples with true values to check that were not used for learning or validation); read the data files in chunks into memory. """ print("Loading data for testing...") train_sizes, valid_sizes, test_sizes = \ get_and_print_dataset_subsizes(data_file_list) used_sizes, _ = get_used_data_sizes_for_testing(train_sizes, valid_sizes, test_sizes, test_all_data) # extract timestamp from model file - assume it is the first set of numbers # otherwise just use "now" import re tstamp = str(time.time()).split('.')[0] m = re.search(r"[0-9]+", save_model_file) if m: tstamp = m.group(0) # Prepare Theano variables for inputs and targets input_var_x = T.tensor4('inputs') input_var_u = T.tensor4('inputs') input_var_v = T.tensor4('inputs') inputlist = build_inputlist(input_var_x, input_var_u, input_var_v, views) # Build the model network = build_cnn(inputlist=inputlist, imgw=imgw, imgh=imgh, convpooldictlist=convpooldictlist, nhidden=nhidden, dropoutp=dropoutp, noutputs=noutputs) with np.load(save_model_file) as f: param_values = [f['arr_%d' % i] for i in range(len(f.files))] lasagne.layers.set_all_param_values(network, param_values) print(network_repr.get_network_str( lasagne.layers.get_all_layers(network), get_network=False, incomings=True, outgoings=True)) layers = lasagne.layers.get_all_layers(network) # layer assignment is _highly_ network specific... layer_conv_x1 = lasagne.layers.get_output(layers[1]) layer_conv_u1 = lasagne.layers.get_output(layers[8]) layer_conv_v1 = lasagne.layers.get_output(layers[15]) layer_pool_x1 = lasagne.layers.get_output(layers[2]) layer_pool_u1 = lasagne.layers.get_output(layers[9]) layer_pool_v1 = lasagne.layers.get_output(layers[16]) layer_conv_x2 = lasagne.layers.get_output(layers[3]) layer_conv_u2 = lasagne.layers.get_output(layers[10]) layer_conv_v2 = lasagne.layers.get_output(layers[17]) layer_pool_x2 = lasagne.layers.get_output(layers[4]) layer_pool_u2 = lasagne.layers.get_output(layers[11]) layer_pool_v2 = lasagne.layers.get_output(layers[18]) vis_conv_x1 = theano.function(inputlist, [layer_conv_x1], allow_input_downcast=True, on_unused_input='warn') vis_conv_u1 = theano.function(inputlist, [layer_conv_u1], allow_input_downcast=True, on_unused_input='warn') vis_conv_v1 = theano.function(inputlist, [layer_conv_v1], allow_input_downcast=True, on_unused_input='warn') vis_pool_x1 = theano.function(inputlist, [layer_pool_x1], allow_input_downcast=True, on_unused_input='warn') vis_pool_u1 = theano.function(inputlist, [layer_pool_u1], allow_input_downcast=True, on_unused_input='warn') vis_pool_v1 = theano.function(inputlist, [layer_pool_v1], allow_input_downcast=True, on_unused_input='warn') vis_conv_x2 = theano.function(inputlist, [layer_conv_x2], allow_input_downcast=True, on_unused_input='warn') vis_conv_u2 = theano.function(inputlist, [layer_conv_u2], allow_input_downcast=True, on_unused_input='warn') vis_conv_v2 = theano.function(inputlist, [layer_conv_v2], allow_input_downcast=True, on_unused_input='warn') vis_pool_x2 = theano.function(inputlist, [layer_pool_x2], allow_input_downcast=True, on_unused_input='warn') vis_pool_u2 = theano.function(inputlist, [layer_pool_u2], allow_input_downcast=True, on_unused_input='warn') vis_pool_v2 = theano.function(inputlist, [layer_pool_v2], allow_input_downcast=True, on_unused_input='warn') print("Starting visualization...") test_slices = [] for tsize in used_sizes: test_slices.append(slices_maker(tsize, slice_size=50000)) test_set = None for i, data_file in enumerate(data_file_list): for tslice in test_slices[i]: t0 = time.time() test_set = None if test_all_data: test_set = load_all_datasubsets(data_file, tslice) else: test_set = load_datasubset(data_file, 'test', tslice) _, test_dstream = make_scheme_and_stream(test_set, 1, shuffle=False) t1 = time.time() print(" Loading slice {} from {} took {:.3f}s.".format( tslice, data_file, t1 - t0)) if debug_print: print(" dset sources:", test_set.provides_sources) t0 = time.time() for data in test_dstream.get_epoch_iterator(): # data order in the hdf5 looks like: # ids, hits-u, hits-v, hits-x, planes, segments, zs # (Check the file carefully for data names, etc.) eventids, inputlist, targets = \ get_eventids_hits_and_targets_from_data( data, views, target_idx) conv_x1 = vis_conv_x1(*inputlist) conv_u1 = vis_conv_u1(*inputlist) conv_v1 = vis_conv_v1(*inputlist) pool_x1 = vis_pool_x1(*inputlist) pool_u1 = vis_pool_u1(*inputlist) pool_v1 = vis_pool_v1(*inputlist) conv_x2 = vis_conv_x2(*inputlist) conv_u2 = vis_conv_u2(*inputlist) conv_v2 = vis_conv_v2(*inputlist) pool_x2 = vis_pool_x2(*inputlist) pool_u2 = vis_pool_u2(*inputlist) pool_v2 = vis_pool_v2(*inputlist) vis_file = 'vis_' + str(targets[0]) + '_conv_1_' + tstamp + \ '_' + str(eventids[0]) + '.npy' np.save(vis_file, [conv_x1, conv_u1, conv_v1]) vis_file = 'vis_' + str(targets[0]) + '_pool_1_' + tstamp + \ '_' + str(eventids[0]) + '.npy' np.save(vis_file, [pool_x1, pool_u1, pool_v1]) vis_file = 'vis_' + str(targets[0]) + '_conv_2_' + tstamp + \ '_' + str(eventids[0]) + '.npy' np.save(vis_file, [conv_x2, conv_u2, conv_v2]) vis_file = 'vis_' + str(targets[0]) + '_pool_2_' + tstamp + \ '_' + str(eventids[0]) + '.npy' np.save(vis_file, [pool_x2, pool_u2, pool_v2]) t1 = time.time() print(" -Iterating over the slice took {:.3f}s.".format(t1 - t0)) del test_set del test_dstream
def categorical_predict(build_cnn=None, data_file_list=None, views='xuv', imgw=50, imgh=50, target_idx=5, save_model_file='./params_file.npz', be_verbose=False, convpooldictlist=None, nhidden=None, dropoutp=None, write_db=True, test_all_data=False, debug_print=False, noutputs=11): """ Make predictions based on the model _only_ (e.g., this routine should be used to produce prediction db's quickly or for data) noutputs=11 for zsegments, other vals for planecodes, etc. """ print("Loading data for testing...") train_sizes, valid_sizes, test_sizes = \ get_and_print_dataset_subsizes(data_file_list) used_sizes, used_data_size = get_used_data_sizes_for_testing(train_sizes, valid_sizes, test_sizes, test_all_data) metadata = None try: import predictiondb from sqlalchemy import MetaData except ImportError: print("Cannot import sqlalchemy...") write_db = False if write_db: tstamp = get_tstamp_from_model_name(save_model_file) metadata = MetaData() dbname = 'prediction' + tstamp eng = predictiondb.get_engine(dbname) con = predictiondb.get_connection(eng) tbl = predictiondb.get_active_table(metadata, eng) # Prepare Theano variables for inputs input_var_x = T.tensor4('inputs') input_var_u = T.tensor4('inputs') input_var_v = T.tensor4('inputs') inputlist = build_inputlist(input_var_x, input_var_u, input_var_v, views) # Build the model network = build_cnn(inputlist=inputlist, imgw=imgw, imgh=imgh, convpooldictlist=convpooldictlist, nhidden=nhidden, dropoutp=dropoutp, noutputs=noutputs) with np.load(save_model_file) as f: param_values = [f['arr_%d' % i] for i in range(len(f.files))] lasagne.layers.set_all_param_values(network, param_values) # Compute the prediction test_prediction = lasagne.layers.get_output(network, deterministic=True) test_prediction_values = T.argmax(test_prediction, axis=1) pred_fn = theano.function(inputlist, [test_prediction, test_prediction_values], allow_input_downcast=True) print("Starting prediction...") test_slices = [] for tsize in used_sizes: test_slices.append(slices_maker(tsize, slice_size=50000)) test_set = None evtcounter = 0 batch_size = 500 evt_print_freq = batch_size * 4 for i, data_file in enumerate(data_file_list): for tslice in test_slices[i]: t0 = time.time() test_set = None if test_all_data: test_set = load_all_datasubsets(data_file, tslice) else: test_set = load_datasubset(data_file, 'test', tslice) _, test_dstream = make_scheme_and_stream(test_set, batch_size, shuffle=False) t1 = time.time() print(" Loading slice {} from {} took {:.3f}s.".format( tslice, data_file, t1 - t0)) if debug_print: print(" dset sources:", test_set.provides_sources) t0 = time.time() for data in test_dstream.get_epoch_iterator(): eventids, hits_list = \ get_id_tagged_inputlist_from_data(data, views) probs, pred = pred_fn(*hits_list) evtcounter += batch_size if write_db: for i, evtid in enumerate(eventids): filldb(tbl, con, evtid, pred[i], probs[i]) if be_verbose: if evtcounter % evt_print_freq == 0: print("processed {}/{}". format(evtcounter, used_data_size)) t1 = time.time() print(" -Iterating over the slice took {:.3f}s.".format(t1 - t0)) del test_set del test_dstream print("Finished producing predictions!")
def categorical_test(build_cnn=None, data_file_list=None, l2_penalty_scale=1e-04, views='xuv', imgw=50, imgh=50, target_idx=5, save_model_file='./params_file.npz', be_verbose=False, convpooldictlist=None, nhidden=None, dropoutp=None, test_all_data=False, debug_print=False, noutputs=11): """ Run tests on the reserved test sample ("trainiing" examples with true values to check that were not used for learning or validation); read the data files in chunks into memory. noutputs=11 for zsegments, other vals for planecodes, etc. """ print("Loading data for testing...") tstamp = get_tstamp_from_model_name(save_model_file) train_sizes, valid_sizes, test_sizes = \ get_and_print_dataset_subsizes(data_file_list) used_sizes, used_data_size = get_used_data_sizes_for_testing(train_sizes, valid_sizes, test_sizes, test_all_data) # Prepare Theano variables for inputs and targets input_var_x = T.tensor4('inputs') input_var_u = T.tensor4('inputs') input_var_v = T.tensor4('inputs') target_var = T.ivector('targets') inputlist = build_inputlist(input_var_x, input_var_u, input_var_v, views) # Build the model network = build_cnn(inputlist=inputlist, imgw=imgw, imgh=imgh, convpooldictlist=convpooldictlist, nhidden=nhidden, dropoutp=dropoutp, noutputs=noutputs) with np.load(save_model_file) as f: param_values = [f['arr_%d' % i] for i in range(len(f.files))] lasagne.layers.set_all_param_values(network, param_values) # Create a loss expression for testing. test_prediction = lasagne.layers.get_output(network, deterministic=True) l2_penalty = lasagne.regularization.regularize_layer_params( lasagne.layers.get_all_layers(network), lasagne.regularization.l2) * l2_penalty_scale test_loss = categorical_crossentropy(test_prediction, target_var) + \ l2_penalty test_loss = test_loss.mean() # Also create an expression for the classification accuracy: test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var), dtype=theano.config.floatX) # Look at the classifications test_prediction_values = T.argmax(test_prediction, axis=1) # Compute the actual predictions - also instructive is to look at # `test_prediction` as an output (array of softmax probabilities) pred_fn = theano.function(inputlist, [test_prediction, test_prediction_values], allow_input_downcast=True) # Compile a function computing the validation loss and accuracy: inputlist.append(target_var) val_fn = theano.function(inputlist, [test_loss, test_acc], allow_input_downcast=True) print("Starting testing...") # compute and print the test error and... test_err = 0 test_acc = 0 test_batches = 0 # look at some concrete predictions num_poss_segs = noutputs pred_target = np.zeros(num_poss_segs, dtype='float32') true_target = np.zeros(num_poss_segs, dtype='float32') targs_mat = np.zeros(num_poss_segs * num_poss_segs, dtype='float32').reshape(num_poss_segs, num_poss_segs) test_slices = [] for tsize in used_sizes: test_slices.append(slices_maker(tsize, slice_size=50000)) test_set = None evt_print_freq = 1 # TODO: pass this in? evtcounter = 0 for i, data_file in enumerate(data_file_list): for tslice in test_slices[i]: t0 = time.time() test_set = None if test_all_data: test_set = load_all_datasubsets(data_file, tslice) else: test_set = load_datasubset(data_file, 'test', tslice) _, test_dstream = make_scheme_and_stream(test_set, 1, shuffle=False) t1 = time.time() print(" Loading slice {} from {} took {:.3f}s.".format( tslice, data_file, t1 - t0)) if debug_print: print(" dset sources:", test_set.provides_sources) t0 = time.time() for data in test_dstream.get_epoch_iterator(): eventids, inputlist, targets = \ get_eventids_hits_and_targets_from_data( data, views, target_idx) inputlist.append(targets) err, acc = val_fn(*inputlist) test_err += err test_acc += acc test_batches += 1 hits_list = get_list_of_hits_from_data(data, views) probs, pred = pred_fn(*hits_list) pred_targ = zip(pred, targets) evtcounter += 1 if be_verbose: if evtcounter % evt_print_freq == 0: print("{}/{} - {}: (prediction, true target): {}, {}". format(evtcounter, used_data_size, eventids[0], pred_targ, probs)) for p, t in pred_targ: targs_mat[t][p] += 1 true_target[t] += 1 if p == t: pred_target[p] += 1 t1 = time.time() print(" -Iterating over the slice took {:.3f}s.".format(t1 - t0)) del test_set del test_dstream acc_target = 100.0 * pred_target / true_target.astype('float32') perf_file = 'perfmat' + tstamp + '.npy' np.save(perf_file, targs_mat) print("Final results:") print(" test loss:\t\t\t{:.6f}".format(test_err / test_batches)) print(" test accuracy:\t\t{:.2f} %".format( test_acc / test_batches * 100)) for i, v in enumerate(acc_target): print(" target {} accuracy:\t\t\t{:.3f} %".format( i, acc_target[i]))