def store_two_way(F, two_way_type, train_config, collname, type_tag): """ Store one two way result in a database :param F: Features to run classifier on :param two_way_type: Name of task :param train_config: N_test and train and how to select data to train and test on :param collname: Name to get fs where results are stored via get_results fs :param type_tag: Name of the set of tasks this belongs to ('subordinate', 'basic') """ results_fs = get_results_fs(collname) query = {'two_way_type': two_way_type} count = results_fs._GridFS__files.find(query).count() if count >= 1: idval = results_fs._GridFS__files.find_one(query)['_id'] print 'Found %s as %s in %s' % (two_way_type, idval, collname) else: eval_config = copy.deepcopy(classifier_config) eval_config.update(train_config) results = u.compute_metric_base(F, dataset.meta, eval_config, return_splits=True) info = utils.SONify( dict(two_way_type=two_way_type, type_tag=type_tag, eval_config=eval_config)) blob = cPickle.dumps(results, protocol=cPickle.HIGHEST_PROTOCOL) count = results_fs._GridFS__files.find(query).count() if count < 1: idval = results_fs.put(blob, **info) print 'Stored %s as %s in %s' % (two_way_type, idval, collname)
def store_two_way(F, two_way_type, train_config, collname, type_tag): """ Store one two way result in a database :param F: Features to run classifier on :param two_way_type: Name of task :param train_config: N_test and train and how to select data to train and test on :param collname: Name to get fs where results are stored via get_results fs :param type_tag: Name of the set of tasks this belongs to ('subordinate', 'basic') """ results_fs = get_results_fs(collname) query = {'two_way_type': two_way_type} count = results_fs._GridFS__files.find(query).count() if count >= 1: idval = results_fs._GridFS__files.find_one(query)['_id'] print 'Found %s as %s in %s'%(two_way_type, idval, collname) else: eval_config = copy.deepcopy(classifier_config) eval_config.update(train_config) results = u.compute_metric_base(F, dataset.meta, eval_config, return_splits=True) info = utils.SONify(dict(two_way_type=two_way_type, type_tag=type_tag, eval_config=eval_config)) blob = cPickle.dumps(results, protocol=cPickle.HIGHEST_PROTOCOL) count = results_fs._GridFS__files.find(query).count() if count <1: idval = results_fs.put(blob, **info) print 'Stored %s as %s in %s'%(two_way_type, idval, collname)
def regression_test(self, features, IT_features, meta): """ Illustrates how to perform a regression test using dldata You will need to EDIT this part. Define the specification to do a regression on the IT neurons using compute_metric_base from dldata. """ print('Regression test...') it_reg_eval_spec = { 'labelfunc': lambda x: (IT_features, None), 'metric_kwargs': { 'model_kwargs': { 'n_components': 25, 'scale': False }, 'model_type': 'pls.PLSRegression' }, 'metric_labels': None, 'metric_screen': 'regression', 'npc_test': 10, 'npc_train': 70, 'npc_validate': 0, 'num_splits': 5, 'split_by': 'obj', 'train_q': {}, 'test_q': {}, } res = compute_metric_base(features, meta, it_reg_eval_spec) espec = (('all', '', 'IT_regression'), it_reg_eval_spec) post_process_neural_regression_msplit_preprocessed( res, self.Config.noise_estimates_path) res.pop('split_results') return res
def identification_test(self, features, meta, category, variation=None): """ Performs a categorization test using dldata You will need to EDIT this part. Define the specification to do a categorization on the neural stimuli using compute_metric_base from dldata. """ print('Within category identification test %s...' % category) query = {} if variation is None else {'var': [variation]} query['category'] = [category] category_eval_spec = { 'npc_train': None, 'npc_test': 2, 'num_splits': 20, 'npc_validate': 0, 'metric_screen': 'classifier', 'metric_labels': None, 'metric_kwargs': { 'model_type': 'svm.LinearSVC', 'model_kwargs': { 'C': 5e-3 } }, 'labelfunc': 'obj', 'train_q': query, 'test_q': query, 'split_by': 'obj' } res = compute_metric_base(features, meta, category_eval_spec) res.pop('split_results') return res
def categorization_test(self, features, meta, variability=None): """ Performs a categorization test using dldata You will need to EDIT this part. Define the specification to do a categorization on the neural stimuli using compute_metric_base from dldata. """ print('Categorization test...') if variability is None: selection = {}, else: selection = {'var': variability} category_eval_spec = { 'npc_train': None, 'npc_test': 2, 'num_splits': 20, 'npc_validate': 0, 'metric_screen': 'classifier', 'metric_labels': None, 'metric_kwargs': { 'model_type': 'svm.LinearSVC', 'model_kwargs': { 'C': 5e-3 } }, 'labelfunc': 'category', 'train_q': selection, 'test_q': selection, 'split_by': 'obj' } res = compute_metric_base(features, meta, category_eval_spec) res.pop('split_results') return res
def pos_regression_test(self, features, meta, data_subset): """ Illustrates how to perform a regression test using dldata You will need to EDIT this part. Define the specification to do a regression on the IT neurons using compute_metric_base from dldata. """ print('Position regression test...') pos_reg_eval_spec = { 'labelfunc': 'ty', 'metric_kwargs': { 'model_type': 'linear_model.RidgeCV' }, 'metric_labels': None, 'metric_screen': 'regression', 'npc_test': 2, 'npc_train': None, 'npc_validate': 0, 'num_splits': 20, 'split_by': 'obj', 'test_q': { 'var': data_subset }, 'train_q': { 'var': data_subset } } res = compute_metric_base(features, meta, pos_reg_eval_spec) res.pop('split_results') return res
def getClassifierRecord(features_task, meta_task, n_splits=100, classifiertype='svm'): if len(meta_task) == 1: meta_task = meta_task[0] features_task = np.squeeze(features_task) uobj = list(set(meta_task['obj'])) train_q, test_q = {}, {} npc_all = [np.sum(list(meta_task['obj'] == o)) for o in uobj] npc = min(npc_all) npc_train = npc/2 npc_test = npc/2 metric_kwargs = METRIC_KWARGS[classifiertype] evalc = {'ctypes': [('dp_standard', 'dp_standard', {'kwargs': {'error': 'std'}})], 'labelfunc': 'obj', 'metric_kwargs': metric_kwargs, 'metric_screen': 'classifier', 'npc_test': npc_test, 'npc_train': npc_train, 'npc_validate': 0, 'num_splits': n_splits, 'split_by': 'obj', 'split_seed': random.seed(), 'test_q': test_q, 'train_q': train_q, 'use_validation': False} return utils.compute_metric_base(features_task, meta_task, evalc, attach_models=True, return_splits=True)
def meta_regression_test(self, features, meta, variability=None): """ Illustrates how to perform a regression test using dldata You will need to EDIT this part. Define the specification to do a regression on the IT neurons using compute_metric_base from dldata. """ print('Meta Regression test...') if variability is None: selection = {}, else: selection = {'var': variability} reg_eval_spec = { 'npc_train': None, 'npc_test': 2, 'num_splits': 20, 'npc_validate': 0, 'metric_screen': 'regression', 'metric_labels': None, 'metric_kwargs': { 'model_type': 'linear_model.LassoCV', 'model_kwargs': {} }, 'labelfunc': 'rxz', 'train_q': selection, 'test_q': selection, 'split_by': 'obj' } res = compute_metric_base(features, meta, reg_eval_spec) res.pop('split_results') return res
def dp_sym_loss(template_sample, dataset, eval_config, features=None, delete_memmaps=True): """ :param template_sample: Sample of parameters from template :param dataset: Dataset object to evaluate loss for :param eval_config: eval_config passed to compute_metric_base (see that function for reference on this dictionary) :param features: can be used to pass existing features, features will not be rextracted :return: float d prime loss score """ record = {} if features is None: features, record = get_features(template_sample, dataset, record) #Code to flatten features for computing metrics features, meta = subset_and_reshape(features, dataset.meta, eval_config) results = compute_metric_base(features, meta, eval_config) if delete_memmaps: if record['basedir'] is not None: path = os.path.join(record['basedir'], record['namebase'] + '_img_feat') else: path = record['namebase'] + '_img_feat' print ('removing %s' % path) os.system('rm -rf %s' % path) record['loss'] = results['dp_sym_loss'] record['results'] = results record['status'] = 'ok' record['desc'] = template_sample return record
def regression_test(self, features, IT_features, meta, variability=None): """ Performs a regression test with IT data using compute_metric_base from dldata. """ print('Regression test...') if variability is None: selection = {}, else: selection = {'var': variability} it_reg_eval_spec = { 'npc_train': None, 'npc_test': 2, 'num_splits': 20, 'npc_validate': 0, 'metric_screen': 'regression', 'metric_labels': None, 'metric_kwargs': { 'model_type': 'pls.PLSRegression', 'model_kwargs': { 'n_components': 25, 'scale': False } }, 'labelfunc': lambda x: (IT_features, None), 'train_q': selection, 'test_q': selection, 'split_by': 'obj' } res = compute_metric_base(features, meta, it_reg_eval_spec) espec = (('all', '', 'IT_regression'), it_reg_eval_spec) post_process_neural_regression_msplit_preprocessed( res, self.Config.noise_estimates_path) res.pop('split_results') return res
def continuous_estimation_test(self, features, meta, attribute='ty', variation=None): """ Performs a continuous estimation regression test using dldata You will need to EDIT this part. Define the specification to do a categorization on the neural stimuli using compute_metric_base from dldata. """ print('Continuous estimation test %s...' % attribute) query = {} if variation is None else {'var': [variation]} category_eval_spec = { 'npc_train': None, 'npc_test': 2, 'num_splits': 20, 'npc_validate': 0, 'metric_screen': 'regression', 'metric_labels': None, 'metric_kwargs': { 'model_type': 'linear_model.RidgeCV' }, 'labelfunc': attribute, 'train_q': query, 'test_q': query, 'split_by': 'obj' } res = compute_metric_base(features, meta, category_eval_spec) res.pop('split_results') return res
def categorization_test(self, features, meta): """ Performs a categorization test using dldata You will need to EDIT this part. Define the specification to do a categorization on the neural stimuli using compute_metric_base from dldata. """ print('Categorization test...') category_eval_spec = { EDIT_YOUR_SPEC_HERE } res = compute_metric_base(features, meta, category_eval_spec) res.pop('split_results') return res
def imagenet_classification(self, results): """ Performs classification on ImageNet using a linear regression on feature data from each layer """ retval = {} meta = self.parse_imagenet_meta_data(results) features = self.get_imagenet_features(results, num_subsampled_features=1024) # Subsample to 100 labels target_labels = np.unique(meta['labels'])[::10] mask = np.isin(meta['labels'], target_labels) for layer in features: features[layer] = features[layer][mask] meta = tb.tabarray(columns=[list(meta['labels'][mask])], names=['labels']) #print "Features:", features['bn1'].shape print "Labels:", np.unique(meta['labels']).shape for layer in features: layer_features = features[layer] print('%s Imagenet classification test...' % layer) category_eval_spec = { 'npc_train': None, 'npc_test': 5, 'num_splits': 3, 'npc_validate': 0, 'metric_screen': 'classifier', 'metric_labels': None, 'metric_kwargs': { 'model_type': 'svm.LinearSVC', 'model_kwargs': { 'C': 5e-3 } }, 'labelfunc': 'labels', 'train_q': None, 'test_q': None, 'split_by': 'labels', } res = compute_metric_base(layer_features, meta, category_eval_spec) res.pop('split_results') retval['imagenet_%s' % layer] = res return retval
def regression_test(self, features, IT_features, meta): """ Illustrates how to perform a regression test using dldata You will need to EDIT this part. Define the specification to do a regression on the IT neurons using compute_metric_base from dldata. """ print('Regression test...') it_reg_eval_spec = { EDIT_YOUR_SPEC_HERE } res = compute_metric_base(features, meta, it_reg_eval_spec) espec = (('all','','IT_regression'), it_reg_eval_spec) post_process_neural_regression_msplit_preprocessed( res, self.Config.noise_estimates_path) res.pop('split_results') return res
def store_compute_metric_results(F, meta, eval_config, fs, additional_info): """ Used to store results of a decoder model (as specified by eval_config) on top of features F :param F: Features to compute a classifier on :param meta: meta information about the images :param eval_config: eval config (see compute metric base) :param fs: grid fs filesystem for storage :param additional_info: Additional info about this classifier experiment :return: tuple of results from compute_metric_base and the id of the record stored """ results = compute_metric_base(F, meta, eval_config, return_splits=True, attach_models=True) additional_info['eval_config'] = SONify(copy.deepcopy(eval_config)) blob = cPickle.dumps(results, protocol=cPickle.HIGHEST_PROTOCOL) M = get_metric_ready_result(results, meta) additional_info['metric_ready_result'] = M additional_info_SON = SONify(additional_info) idval = fs.put(blob, **additional_info_SON) print 'Stored results in %s'%idval return results, idval
def getClassifierRecord(features_task, meta_task, n_splits=1, classifiertype='svm'): if len(meta_task) == 1: meta_task = meta_task[0] features_task = np.squeeze(features_task) uobj = list(set(meta_task['obj'])) npc = len(meta_task) / len(uobj) npc_train = npc/2 npc_test = npc/2 metric_kwargs_svm = {'model_type': 'libSVM', 'model_kwargs': {'C': 50000, 'class_weight': 'auto', 'kernel': 'linear'} } metric_kwargs_mcc = {'model_type': 'MCC2', 'model_kwargs': {'fnorm': True, 'snorm': False} } if classifiertype == 'svm': metric_kwargs = metric_kwargs_svm elif classifiertype == 'mcc': metric_kwargs = metric_kwargs_mcc evalc = {'ctypes': [('dp_standard', 'dp_standard', {'kwargs': {'error': 'std'}})], 'labelfunc': 'obj', 'metric_kwargs': metric_kwargs, 'metric_screen': 'classifier', 'npc_test': npc_test, 'npc_train': npc_train, 'npc_validate': 0, 'num_splits': n_splits, 'split_by': 'obj', 'test_q': {}, 'train_q': {}, 'use_validation': False} return utils.compute_metric_base(features_task, meta_task, evalc, attach_models=True, return_splits=True)
def within_categorization_test(self, features, meta, data_subset, category): """ Performs a within-category categorization test using dldata """ print('Within-Category Categorization test...') category_eval_spec = { 'npc_train': None, 'npc_test': 2, 'num_splits': 20, 'npc_validate': 0, 'metric_screen': 'classifier', 'metric_labels': None, 'metric_kwargs': {'model_type': 'svm.LinearSVC', 'model_kwargs': {'C': 5e-3}, }, 'labelfunc': 'obj', 'train_q': {'var': data_subset, 'category': category}, 'test_q': {'var': data_subset, 'category': category}, 'split_by': 'obj' } res = compute_metric_base(features, meta, category_eval_spec) res.pop('split_results') return res
def store_compute_metric_results(F, meta, eval_config, fs, additional_info): """ Used to store results of a decoder model (as specified by eval_config) on top of features F :param F: Features to compute a classifier on :param meta: meta information about the images :param eval_config: eval config (see compute metric base) :param fs: grid fs filesystem for storage :param additional_info: Additional info about this classifier experiment :return: tuple of results from compute_metric_base and the id of the record stored """ results = compute_metric_base(F, meta, eval_config, return_splits=True, attach_models=True) additional_info['eval_config'] = SONify(copy.deepcopy(eval_config)) blob = cPickle.dumps(results, protocol=cPickle.HIGHEST_PROTOCOL) M = get_metric_ready_result(results, meta) additional_info['metric_ready_result'] = M additional_info_SON = SONify(additional_info) idval = fs.put(blob, **additional_info_SON) print 'Stored results in %s' % idval return results, idval
def compute_performance(mname, lname, bdir): import dldata.stimulus_sets.hvm as hvm import dldata.stimulus_sets.synthetic.synthetic_datasets as sd from dldata.metrics import utils #Xa = api.assemble_feature_batches(os.path.join(bdir, mname + '_HvMWithDiscfade_' + lname + 'a'), N=1000) #Xb = api.assemble_feature_batches(os.path.join(bdir, mname + '_HvMWithDiscfade_' + lname + 'b'), N=1000) #X = np.column_stack([Xa, Xb]) NS = 5 ev = {'npc_train': 35, 'npc_test': 5, 'num_splits': NS, 'npc_validate': 0, 'metric_screen': 'classifier', 'metric_labels': None, 'metric_kwargs': {'model_type': 'MCC2'}, 'labelfunc': 'category', 'train_q': {'var': ['V6']}, 'test_q': {'var': ['V6']}, 'split_by': 'obj'} #dataset = hvm.HvMWithDiscfade() #meta = dataset.meta #meta0 = meta[np.random.RandomState(0).permutation(len(meta))] #rec = utils.compute_metric_base(X, meta0, ev) #dataset = sd.TrainingDatasetLarge() dataset = inet.ChallengeSynsets2013_offline() meta = dataset.meta #Xa = api.assemble_feature_batches(os.path.join(bdir, mname + '_TrainingDatasetLarge_' + lname + 'a')) #Xb = api.assemble_feature_batches(os.path.join(bdir, mname + '_TrainingDatasetLarge_' + lname + 'b')) Xa = api.assemble_feature_batches(os.path.join(bdir, mname + '_ChallengeSynsets2013_offline_' + lname + 'a'), N=1000) Xb = api.assemble_feature_batches(os.path.join(bdir, mname + '_ChallengeSynsets2013_offline_' + lname + 'b'), N=1000) X = np.column_stack([Xa, Xb]) meta0 = meta[np.random.RandomState(0).permutation(len(meta))][: X.shape[0]] #NS = 5 #ev = {'npc_train': 31, #'npc_test': 5, #'num_splits': NS, #'npc_validate': 0, #'metric_screen': 'classifier', #'metric_labels': None, #'metric_kwargs': {'model_type': 'MCC2'}, #'labelfunc': 'category', #'train_q': {}, #'test_q': {}, #'split_by': 'obj'} #rec_t = utils.compute_metric_base(X, meta0, ev) ev = {'npc_train': 150, 'npc_test': 50, 'num_splits': NS, 'npc_validate': 0, 'metric_screen' : 'classifier', # 'metric_labels': None, 'metric_kwargs': {'model_type': 'MCC2'}, 'labelfunc': 'synset', 'train_q': {}, 'test_q': {}, 'split_by': 'synset'} rec_c = utils.compute_metric_base(X, meta0, ev) return { #'rec_hvm': rec, #'rec_training': rec_t 'rec_challenge': rec_c}