def store_two_way(F, two_way_type, train_config, collname, type_tag):
"""
Store one two way result in a database
:param F: Features to run classifier on
:param two_way_type: Name of task
:param train_config: N_test and train and how to select data to train and test on
:param collname: Name to get fs where results are stored via get_results fs
:param type_tag: Name of the set of tasks this belongs to ('subordinate', 'basic')
"""
results_fs = get_results_fs(collname)
query = {'two_way_type': two_way_type}
count = results_fs._GridFS__files.find(query).count()
if count >= 1:
idval = results_fs._GridFS__files.find_one(query)['_id']
print 'Found %s as %s in %s' % (two_way_type, idval, collname)
else:
eval_config = copy.deepcopy(classifier_config)
eval_config.update(train_config)
results = u.compute_metric_base(F,
dataset.meta,
eval_config,
return_splits=True)
info = utils.SONify(
dict(two_way_type=two_way_type,
type_tag=type_tag,
eval_config=eval_config))
blob = cPickle.dumps(results, protocol=cPickle.HIGHEST_PROTOCOL)
count = results_fs._GridFS__files.find(query).count()
if count < 1:
idval = results_fs.put(blob, **info)
print 'Stored %s as %s in %s' % (two_way_type, idval, collname)
def store_two_way(F, two_way_type, train_config, collname, type_tag):
"""
Store one two way result in a database
:param F: Features to run classifier on
:param two_way_type: Name of task
:param train_config: N_test and train and how to select data to train and test on
:param collname: Name to get fs where results are stored via get_results fs
:param type_tag: Name of the set of tasks this belongs to ('subordinate', 'basic')
"""
results_fs = get_results_fs(collname)
query = {'two_way_type': two_way_type}
count = results_fs._GridFS__files.find(query).count()
if count >= 1:
idval = results_fs._GridFS__files.find_one(query)['_id']
print 'Found %s as %s in %s'%(two_way_type, idval, collname)
else:
eval_config = copy.deepcopy(classifier_config)
eval_config.update(train_config)
results = u.compute_metric_base(F, dataset.meta, eval_config,
return_splits=True)
info = utils.SONify(dict(two_way_type=two_way_type, type_tag=type_tag, eval_config=eval_config))
blob = cPickle.dumps(results, protocol=cPickle.HIGHEST_PROTOCOL)
count = results_fs._GridFS__files.find(query).count()
if count <1:
idval = results_fs.put(blob, **info)
print 'Stored %s as %s in %s'%(two_way_type, idval, collname)
def regression_test(self, features, IT_features, meta):
"""
Illustrates how to perform a regression test using dldata
You will need to EDIT this part. Define the specification to
do a regression on the IT neurons using compute_metric_base from dldata.
"""
print('Regression test...')
it_reg_eval_spec = {
'labelfunc': lambda x: (IT_features, None),
'metric_kwargs': {
'model_kwargs': {
'n_components': 25,
'scale': False
},
'model_type': 'pls.PLSRegression'
},
'metric_labels': None,
'metric_screen': 'regression',
'npc_test': 10,
'npc_train': 70,
'npc_validate': 0,
'num_splits': 5,
'split_by': 'obj',
'train_q': {},
'test_q': {},
}
res = compute_metric_base(features, meta, it_reg_eval_spec)
espec = (('all', '', 'IT_regression'), it_reg_eval_spec)
post_process_neural_regression_msplit_preprocessed(
res, self.Config.noise_estimates_path)
res.pop('split_results')
return res
def identification_test(self, features, meta, category, variation=None):
"""
Performs a categorization test using dldata
You will need to EDIT this part. Define the specification to
do a categorization on the neural stimuli using
compute_metric_base from dldata.
"""
print('Within category identification test %s...' % category)
query = {} if variation is None else {'var': [variation]}
query['category'] = [category]
category_eval_spec = {
'npc_train': None,
'npc_test': 2,
'num_splits': 20,
'npc_validate': 0,
'metric_screen': 'classifier',
'metric_labels': None,
'metric_kwargs': {
'model_type': 'svm.LinearSVC',
'model_kwargs': {
'C': 5e-3
}
},
'labelfunc': 'obj',
'train_q': query,
'test_q': query,
'split_by': 'obj'
}
res = compute_metric_base(features, meta, category_eval_spec)
res.pop('split_results')
return res
def categorization_test(self, features, meta, variability=None):
"""
Performs a categorization test using dldata
You will need to EDIT this part. Define the specification to
do a categorization on the neural stimuli using
compute_metric_base from dldata.
"""
print('Categorization test...')
if variability is None:
selection = {},
else:
selection = {'var': variability}
category_eval_spec = {
'npc_train': None,
'npc_test': 2,
'num_splits': 20,
'npc_validate': 0,
'metric_screen': 'classifier',
'metric_labels': None,
'metric_kwargs': {
'model_type': 'svm.LinearSVC',
'model_kwargs': {
'C': 5e-3
}
},
'labelfunc': 'category',
'train_q': selection,
'test_q': selection,
'split_by': 'obj'
}
res = compute_metric_base(features, meta, category_eval_spec)
res.pop('split_results')
return res
def pos_regression_test(self, features, meta, data_subset):
"""
Illustrates how to perform a regression test using dldata
You will need to EDIT this part. Define the specification to
do a regression on the IT neurons using compute_metric_base from dldata.
"""
print('Position regression test...')
pos_reg_eval_spec = {
'labelfunc': 'ty',
'metric_kwargs': {
'model_type': 'linear_model.RidgeCV'
},
'metric_labels': None,
'metric_screen': 'regression',
'npc_test': 2,
'npc_train': None,
'npc_validate': 0,
'num_splits': 20,
'split_by': 'obj',
'test_q': {
'var': data_subset
},
'train_q': {
'var': data_subset
}
}
res = compute_metric_base(features, meta, pos_reg_eval_spec)
res.pop('split_results')
return res
def getClassifierRecord(features_task, meta_task, n_splits=100, classifiertype='svm'):
if len(meta_task) == 1:
meta_task = meta_task[0]
features_task = np.squeeze(features_task)
uobj = list(set(meta_task['obj']))
train_q, test_q = {}, {}
npc_all = [np.sum(list(meta_task['obj'] == o)) for o in uobj]
npc = min(npc_all)
npc_train = npc/2
npc_test = npc/2
metric_kwargs = METRIC_KWARGS[classifiertype]
evalc = {'ctypes': [('dp_standard', 'dp_standard', {'kwargs': {'error': 'std'}})],
'labelfunc': 'obj',
'metric_kwargs': metric_kwargs,
'metric_screen': 'classifier',
'npc_test': npc_test,
'npc_train': npc_train,
'npc_validate': 0,
'num_splits': n_splits,
'split_by': 'obj',
'split_seed': random.seed(),
'test_q': test_q,
'train_q': train_q,
'use_validation': False}
return utils.compute_metric_base(features_task, meta_task, evalc, attach_models=True, return_splits=True)
def meta_regression_test(self, features, meta, variability=None):
"""
Illustrates how to perform a regression test using dldata
You will need to EDIT this part. Define the specification to
do a regression on the IT neurons using compute_metric_base from dldata.
"""
print('Meta Regression test...')
if variability is None:
selection = {},
else:
selection = {'var': variability}
reg_eval_spec = {
'npc_train': None,
'npc_test': 2,
'num_splits': 20,
'npc_validate': 0,
'metric_screen': 'regression',
'metric_labels': None,
'metric_kwargs': {
'model_type': 'linear_model.LassoCV',
'model_kwargs': {}
},
'labelfunc': 'rxz',
'train_q': selection,
'test_q': selection,
'split_by': 'obj'
}
res = compute_metric_base(features, meta, reg_eval_spec)
res.pop('split_results')
return res
def dp_sym_loss(template_sample, dataset, eval_config, features=None, delete_memmaps=True):
"""
:param template_sample: Sample of parameters from template
:param dataset: Dataset object to evaluate loss for
:param eval_config: eval_config passed to compute_metric_base (see that function for reference on this dictionary)
:param features: can be used to pass existing features, features will not be rextracted
:return: float d prime loss score
"""
record = {}
if features is None:
features, record = get_features(template_sample, dataset, record)
#Code to flatten features for computing metrics
features, meta = subset_and_reshape(features, dataset.meta, eval_config)
results = compute_metric_base(features, meta, eval_config)
if delete_memmaps:
if record['basedir'] is not None:
path = os.path.join(record['basedir'],
record['namebase'] + '_img_feat')
else:
path = record['namebase'] + '_img_feat'
print ('removing %s' % path)
os.system('rm -rf %s' % path)
record['loss'] = results['dp_sym_loss']
record['results'] = results
record['status'] = 'ok'
record['desc'] = template_sample
return record
def regression_test(self, features, IT_features, meta, variability=None):
"""
Performs a regression test with IT data using compute_metric_base from dldata.
"""
print('Regression test...')
if variability is None:
selection = {},
else:
selection = {'var': variability}
it_reg_eval_spec = {
'npc_train': None,
'npc_test': 2,
'num_splits': 20,
'npc_validate': 0,
'metric_screen': 'regression',
'metric_labels': None,
'metric_kwargs': {
'model_type': 'pls.PLSRegression',
'model_kwargs': {
'n_components': 25,
'scale': False
}
},
'labelfunc': lambda x: (IT_features, None),
'train_q': selection,
'test_q': selection,
'split_by': 'obj'
}
res = compute_metric_base(features, meta, it_reg_eval_spec)
espec = (('all', '', 'IT_regression'), it_reg_eval_spec)
post_process_neural_regression_msplit_preprocessed(
res, self.Config.noise_estimates_path)
res.pop('split_results')
return res
def continuous_estimation_test(self,
features,
meta,
attribute='ty',
variation=None):
"""
Performs a continuous estimation regression test using dldata
You will need to EDIT this part. Define the specification to
do a categorization on the neural stimuli using
compute_metric_base from dldata.
"""
print('Continuous estimation test %s...' % attribute)
query = {} if variation is None else {'var': [variation]}
category_eval_spec = {
'npc_train': None,
'npc_test': 2,
'num_splits': 20,
'npc_validate': 0,
'metric_screen': 'regression',
'metric_labels': None,
'metric_kwargs': {
'model_type': 'linear_model.RidgeCV'
},
'labelfunc': attribute,
'train_q': query,
'test_q': query,
'split_by': 'obj'
}
res = compute_metric_base(features, meta, category_eval_spec)
res.pop('split_results')
return res
def categorization_test(self, features, meta):
"""
Performs a categorization test using dldata
You will need to EDIT this part. Define the specification to
do a categorization on the neural stimuli using
compute_metric_base from dldata.
"""
print('Categorization test...')
category_eval_spec = {
EDIT_YOUR_SPEC_HERE
}
res = compute_metric_base(features, meta, category_eval_spec)
res.pop('split_results')
return res
def imagenet_classification(self, results):
"""
Performs classification on ImageNet using a linear regression on
feature data from each layer
"""
retval = {}
meta = self.parse_imagenet_meta_data(results)
features = self.get_imagenet_features(results,
num_subsampled_features=1024)
# Subsample to 100 labels
target_labels = np.unique(meta['labels'])[::10]
mask = np.isin(meta['labels'], target_labels)
for layer in features:
features[layer] = features[layer][mask]
meta = tb.tabarray(columns=[list(meta['labels'][mask])],
names=['labels'])
#print "Features:", features['bn1'].shape
print "Labels:", np.unique(meta['labels']).shape
for layer in features:
layer_features = features[layer]
print('%s Imagenet classification test...' % layer)
category_eval_spec = {
'npc_train': None,
'npc_test': 5,
'num_splits': 3,
'npc_validate': 0,
'metric_screen': 'classifier',
'metric_labels': None,
'metric_kwargs': {
'model_type': 'svm.LinearSVC',
'model_kwargs': {
'C': 5e-3
}
},
'labelfunc': 'labels',
'train_q': None,
'test_q': None,
'split_by': 'labels',
}
res = compute_metric_base(layer_features, meta, category_eval_spec)
res.pop('split_results')
retval['imagenet_%s' % layer] = res
return retval
def regression_test(self, features, IT_features, meta):
"""
Illustrates how to perform a regression test using dldata
You will need to EDIT this part. Define the specification to
do a regression on the IT neurons using compute_metric_base from dldata.
"""
print('Regression test...')
it_reg_eval_spec = {
EDIT_YOUR_SPEC_HERE
}
res = compute_metric_base(features, meta, it_reg_eval_spec)
espec = (('all','','IT_regression'), it_reg_eval_spec)
post_process_neural_regression_msplit_preprocessed(
res, self.Config.noise_estimates_path)
res.pop('split_results')
return res
def store_compute_metric_results(F, meta, eval_config, fs, additional_info):
"""
Used to store results of a decoder model (as specified by eval_config) on top of features F
:param F: Features to compute a classifier on
:param meta: meta information about the images
:param eval_config: eval config (see compute metric base)
:param fs: grid fs filesystem for storage
:param additional_info: Additional info about this classifier experiment
:return: tuple of results from compute_metric_base and the id of the record stored
"""
results = compute_metric_base(F, meta, eval_config, return_splits=True, attach_models=True)
additional_info['eval_config'] = SONify(copy.deepcopy(eval_config))
blob = cPickle.dumps(results, protocol=cPickle.HIGHEST_PROTOCOL)
M = get_metric_ready_result(results, meta)
additional_info['metric_ready_result'] = M
additional_info_SON = SONify(additional_info)
idval = fs.put(blob, **additional_info_SON)
print 'Stored results in %s'%idval
return results, idval
def getClassifierRecord(features_task, meta_task, n_splits=1, classifiertype='svm'):
if len(meta_task) == 1:
meta_task = meta_task[0]
features_task = np.squeeze(features_task)
uobj = list(set(meta_task['obj']))
npc = len(meta_task) / len(uobj)
npc_train = npc/2
npc_test = npc/2
metric_kwargs_svm = {'model_type': 'libSVM',
'model_kwargs': {'C': 50000, 'class_weight': 'auto',
'kernel': 'linear'}
}
metric_kwargs_mcc = {'model_type': 'MCC2',
'model_kwargs': {'fnorm': True, 'snorm': False}
}
if classifiertype == 'svm':
metric_kwargs = metric_kwargs_svm
elif classifiertype == 'mcc':
metric_kwargs = metric_kwargs_mcc
evalc = {'ctypes': [('dp_standard', 'dp_standard', {'kwargs': {'error': 'std'}})],
'labelfunc': 'obj',
'metric_kwargs': metric_kwargs,
'metric_screen': 'classifier',
'npc_test': npc_test,
'npc_train': npc_train,
'npc_validate': 0,
'num_splits': n_splits,
'split_by': 'obj',
'test_q': {},
'train_q': {},
'use_validation': False}
return utils.compute_metric_base(features_task, meta_task, evalc, attach_models=True, return_splits=True)
def within_categorization_test(self, features, meta, data_subset, category):
"""
Performs a within-category categorization test using dldata
"""
print('Within-Category Categorization test...')
category_eval_spec = {
'npc_train': None,
'npc_test': 2,
'num_splits': 20,
'npc_validate': 0,
'metric_screen': 'classifier',
'metric_labels': None,
'metric_kwargs': {'model_type': 'svm.LinearSVC',
'model_kwargs': {'C': 5e-3},
},
'labelfunc': 'obj',
'train_q': {'var': data_subset, 'category': category},
'test_q': {'var': data_subset, 'category': category},
'split_by': 'obj'
}
res = compute_metric_base(features, meta, category_eval_spec)
res.pop('split_results')
return res
def store_compute_metric_results(F, meta, eval_config, fs, additional_info):
"""
Used to store results of a decoder model (as specified by eval_config) on top of features F
:param F: Features to compute a classifier on
:param meta: meta information about the images
:param eval_config: eval config (see compute metric base)
:param fs: grid fs filesystem for storage
:param additional_info: Additional info about this classifier experiment
:return: tuple of results from compute_metric_base and the id of the record stored
"""
results = compute_metric_base(F,
meta,
eval_config,
return_splits=True,
attach_models=True)
additional_info['eval_config'] = SONify(copy.deepcopy(eval_config))
blob = cPickle.dumps(results, protocol=cPickle.HIGHEST_PROTOCOL)
M = get_metric_ready_result(results, meta)
additional_info['metric_ready_result'] = M
additional_info_SON = SONify(additional_info)
idval = fs.put(blob, **additional_info_SON)
print 'Stored results in %s' % idval
return results, idval
def compute_performance(mname, lname, bdir):
import dldata.stimulus_sets.hvm as hvm
import dldata.stimulus_sets.synthetic.synthetic_datasets as sd
from dldata.metrics import utils
#Xa = api.assemble_feature_batches(os.path.join(bdir, mname + '_HvMWithDiscfade_' + lname + 'a'), N=1000)
#Xb = api.assemble_feature_batches(os.path.join(bdir, mname + '_HvMWithDiscfade_' + lname + 'b'), N=1000)
#X = np.column_stack([Xa, Xb])
NS = 5
ev = {'npc_train': 35,
'npc_test': 5,
'num_splits': NS,
'npc_validate': 0,
'metric_screen': 'classifier',
'metric_labels': None,
'metric_kwargs': {'model_type': 'MCC2'},
'labelfunc': 'category',
'train_q': {'var': ['V6']},
'test_q': {'var': ['V6']},
'split_by': 'obj'}
#dataset = hvm.HvMWithDiscfade()
#meta = dataset.meta
#meta0 = meta[np.random.RandomState(0).permutation(len(meta))]
#rec = utils.compute_metric_base(X, meta0, ev)
#dataset = sd.TrainingDatasetLarge()
dataset = inet.ChallengeSynsets2013_offline()
meta = dataset.meta
#Xa = api.assemble_feature_batches(os.path.join(bdir, mname + '_TrainingDatasetLarge_' + lname + 'a'))
#Xb = api.assemble_feature_batches(os.path.join(bdir, mname + '_TrainingDatasetLarge_' + lname + 'b'))
Xa = api.assemble_feature_batches(os.path.join(bdir, mname + '_ChallengeSynsets2013_offline_' + lname + 'a'), N=1000)
Xb = api.assemble_feature_batches(os.path.join(bdir, mname + '_ChallengeSynsets2013_offline_' + lname + 'b'), N=1000)
X = np.column_stack([Xa, Xb])
meta0 = meta[np.random.RandomState(0).permutation(len(meta))][: X.shape[0]]
#NS = 5
#ev = {'npc_train': 31,
#'npc_test': 5,
#'num_splits': NS,
#'npc_validate': 0,
#'metric_screen': 'classifier',
#'metric_labels': None,
#'metric_kwargs': {'model_type': 'MCC2'},
#'labelfunc': 'category',
#'train_q': {},
#'test_q': {},
#'split_by': 'obj'}
#rec_t = utils.compute_metric_base(X, meta0, ev)
ev = {'npc_train': 150,
'npc_test': 50,
'num_splits': NS,
'npc_validate': 0,
'metric_screen' : 'classifier', #
'metric_labels': None,
'metric_kwargs': {'model_type': 'MCC2'},
'labelfunc': 'synset',
'train_q': {},
'test_q': {},
'split_by': 'synset'}
rec_c = utils.compute_metric_base(X, meta0, ev)
return {
#'rec_hvm': rec,
#'rec_training': rec_t
'rec_challenge': rec_c}
