def test_validation(self):
# Specify the parameters for the validation.
base_exp_id = 'training0'
base_val_exp_id = 'validation0'
params = self.setup_params(base_exp_id)
num_models = len(params['model_params'])
params.pop('train_params')
params.pop('learning_rate_params')
params['load_params'] = params['save_params']
params['save_params'] = {'exp_id': base_val_exp_id}
# Actually run the model
base.test_from_params(**params)
# Check that the results are correct.
for i in range(num_models):
exp_id = base_exp_id + '_model_{}'.format(i)
val_exp_id = base_val_exp_id + '_model_{}'.format(i)
# ... specifically, there is now a record containing the validation0 performance results
self.assertEqual(self.collection['files'].find({'exp_id': val_exp_id}).count(), 1)
# ... here's how to load the record:
r = self.collection['files'].find({'exp_id': val_exp_id})[0]
self.asserts_for_record(r, params, train=False)
# ... check that the recorrectly ties to the id information for the
# pre-trained model it was supposed to validate
self.assertTrue(r['validates'])
idval = self.collection['files'].find({'exp_id': exp_id})[50]['_id']
v = self.collection['files'].find({'exp_id': val_exp_id})[0]['validates']
self.assertEqual(idval, v)
def test_ln(steps=None, train_stimulus='whitenoise'):
params = copy.deepcopy(default_params)
for param in ['save_params', 'load_params']:
params[param]['dbname'] = 'ln_model'
params[param]['collname'] = train_stimulus
params[param]['exp_id'] = 'trainval0'
params['model_params']['func'] = ln
# determine time steps
if steps is None:
conn = pm.MongoClient(port=params['load_params']['port'])
coll = conn[params['load_params']['dbname']][train_stimulus + '.files']
steps = [
i['step'] for i in coll.find(
{
'exp_id': 'trainval0',
'train_results': {
'$exists': True
}
},
projection=['step'])
]
for step in steps:
print("Running Step %s" % step)
params['load_params']['query'] = {'step': step}
params['save_params']['exp_id'] = 'testval_step%s' % step
base.test_from_params(**params)
def main():
parser = get_brainscore_parser()
args = parser.parse_args()
args = load_setting(args)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
params = get_params_from_arg(args)
base.test_from_params(**params)
def main():
# Parse arguments
cfg = get_config()
args = cfg.parse_args()
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
# Get params needed, start training
params = get_params_from_arg(args)
if not args.pure_test:
base.train_from_params(**params)
else:
params.pop('learning_rate_params')
params.pop('optimizer_params')
params.pop('loss_params')
params.pop('train_params')
base.test_from_params(**params)
def test_validation(self):
"""Illustrate validation.
This is a test illustrating how to compute performance on a trained model on a new dataset,
using the tfutils.base.test_from_params function. This test assumes that test_training function
has run first (to provide a pre-trained model to validate).
After the test is run, results from the validation are stored in the MongoDB.
(The test shows how the record can be loaded for inspection.)
See the docstring of tfutils.base.test_from_params for more detailed information on usage.
"""
# Specify the parameters for the validation.
exp_id = 'training0'
val_exp_id = 'validation0'
params = self.setup_params(exp_id)
params.pop('train_params')
params.pop('learning_rate_params')
params['load_params'] = params['save_params']
params['save_params'] = {'exp_id': val_exp_id}
# Actually run the model.
base.test_from_params(**params)
# ... specifically, there is now a record containing the validation0 performance results
self.assertEqual(
self.collection['files'].find({
'exp_id': val_exp_id
}).count(), 1)
# ... here's how to load the record:
r = self.collection['files'].find({'exp_id': val_exp_id})[0]
self.asserts_for_record(r, params, train=False)
# ... check that the recorrectly ties to the id information for the
# pre-trained model it was supposed to validate
self.assertTrue(r['validates'])
idval = self.collection['files'].find({'exp_id': exp_id})[50]['_id']
v = self.collection['files'].find({'exp_id':
val_exp_id})[0]['validates']
self.assertEqual(idval, v)
def main():
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu
load_config = get_load_config(
FLAGS.exp_id, FLAGS.step, FLAGS.config_name, seed=0, port=FLAGS.port
)
validations = [FLAGS.config_name]
data_config = get_validation_params(validations, load_config, FLAGS.prep_type)
params = get_params(load_config, data_config)
res = base.test_from_params(**params)
save_base = PROJ_PATH + "/extracted_features/"
save_path = (
save_base
+ FLAGS.exp_id
+ "_step_" + str(FLAGS.step)
+ "_" + FLAGS.config_name
+ FLAGS.identifier
+ ".h5"
)
save_results(res, save_path)
'train_q': None,
'test_q': None,
'split_by': 'labels',
}
res = compute_metric_base(layer_features, meta, category_eval_spec)
res.pop('split_results')
retval['imagenet_%s' % layer] = res
return retval
if __name__ == '__main__':
"""
Illustrates how to run the configured model using tfutils
"""
base.get_params()
m = ImageNetClassificationExperiment()
params = m.setup_params()
base.test_from_params(**params)
"""
exp='exp_reg'
batch=50
crop=224
for iteration in [10000, 20000, 40000]:
print("Running imagenet model at step %s" % iteration)
base.get_params()
m = ImageNetClassificationExperiment('exp_reg', iteration, 32, 224)
params = m.setup_params()
base.test_from_params(**params)
"""
def validate(test_params):
print("Validating only")
print("All params: ")
print(test_params)
base.test_from_params(**test_params)
def eval_responses(target_params={
'func': examine_responses,
'target': 'labels'
},
dbname='deepretina',
collname=stim_type,
expid='trainval1',
step=None,
prefix='eval0',
group='test'):
params = {}
params['model_params'] = {
'func': vgg16,
'target': 'conv2_1',
}
if step is None:
query = None
stepstr = 'recent'
else:
query = {'step': step}
stepstr = str(step)
params['load_params'] = {
'host': 'localhost',
'port': 28887,
'dbname': dbname,
'collname': collname,
'exp_id': expid,
'do_restore': True,
'query': query
}
params['save_params'] = {
'exp_id': prefix + '_step' + stepstr,
'save_intermediate_freq': 1,
'save_to_gfs': ['pred_resp', 'gt_resp']
}
if group == 'train':
assert (N_TRAIN % OUTPUT_BATCH_SIZE == 0)
num_steps = N_TRAIN // OUTPUT_BATCH_SIZE
elif group == 'test':
assert (N_TEST % OUTPUT_BATCH_SIZE == 0)
num_steps = N_TEST // OUTPUT_BATCH_SIZE
filepattern_str = group + '*.tfrecords'
params['validation_params'] = {
prefix: {
'data_params': {
'func':
retinaTF,
'source_dirs': [
os.path.join(DATA_PATH, 'images'),
os.path.join(DATA_PATH, 'labels')
],
'resize':
IMAGE_SIZE_RESIZE,
'batch_size':
INPUT_BATCH_SIZE,
'file_pattern':
filepattern_str,
'n_threads':
1,
'shuffle':
False
},
'queue_params': {
'queue_type': 'fifo',
'batch_size': OUTPUT_BATCH_SIZE,
'capacity': 11 * INPUT_BATCH_SIZE,
'min_after_dequeue': 10 * INPUT_BATCH_SIZE,
'seed': 0,
},
'num_steps': num_steps,
'targets': target_params,
'agg_func': mean_losses_keep_rest
}
}
base.test_from_params(**params)
