def test_repeat_same_result(self): model = self.layer_candidate('alexnet', layer='features.12', region='IT', pca_components=1000) score1 = score_model(model_identifier='alexnet-f12-pca_1000', model=model, benchmark_identifier='dicarlo.MajajHong2015.IT-pls') score2 = score_model(model_identifier='alexnet-f12-pca_1000', model=model, benchmark_identifier='dicarlo.MajajHong2015.IT-pls') assert (score1 == score2).all()
def test_Rajalingham2018i2n(self, model_identifier, expected_score): model = brain_translated_pool[model_identifier] score = score_model(model_identifier=model_identifier, model=model, benchmark_identifier='dicarlo.Rajalingham2018-i2n') assert score.raw.sel(aggregation='center') == approx(expected_score, abs=.005)
def test_candidate_Kar2019OST(self, model_identifier, expected_score): model = brain_translated_pool[model_identifier] score = score_model(model_identifier=model_identifier, model=model, benchmark_identifier='dicarlo.Kar2019-ost') if not np.isnan(expected_score): assert score.raw.sel(aggregation='center') == approx(expected_score, abs=.002) else: assert np.isnan(score.raw.sel(aggregation='center'))
def run_benchmark(benchmark_identifier, model_name): print( f'>>>>>Start running model {model_name} on benchmark {benchmark_identifier}' ) model = brain_translated_pool[model_name] score = score_model(model_identifier=model_name, model=model, benchmark_identifier=benchmark_identifier) return score
def test_MajajHong2015ITpls(self, model_identifier, expected_score, attach_hook): model = brain_translated_pool[model_identifier] if attach_hook: activations_model = model.layer_model._layer_model.activations_model LayerPCA.hook(activations_model, n_components=1000) identifier = activations_model.identifier + "-pca_1000" activations_model.identifier = identifier score = score_model(model_identifier, 'dicarlo.MajajHong2015.IT-pls', model=model) assert score.raw.sel(aggregation='center') == approx(expected_score, abs=0.005)
def test_alexnet_conv5_IT(self): model = self.layer_candidate('alexnet', layer='features.12', region='IT', pca_components=1000) score = score_model(model_identifier='alexnet-f12-pca_1000', model=model, benchmark_identifier='dicarlo.Majaj2015.IT-pls') assert score.raw.sel(aggregation='center') == approx(0.590345, abs=0.005)
def test_alexnet_conv2_V4(self): model = self.layer_candidate('alexnet', layer='features.5', region='V4', pca_components=1000) score = score_model(model_identifier='alexnet-f5-pca_1000', model=model, benchmark_identifier='dicarlo.Majaj2015.V4-pls') assert score.raw.sel(aggregation='center').max() == approx(0.633703, abs=0.005)
def test_alexnet_conv3_IT_mask(self): model = self.layer_candidate('alexnet', layer='features.6', region='IT', pca_components=None) np.random.seed(123) score = score_model(model_identifier='alexnet-f6', model=model, benchmark_identifier='dicarlo.Majaj2015.IT-mask') assert score.raw.sel(aggregation='center') == approx(0.607037, abs=0.005)
def test_newmodel_pytorch(self): import torch from torch import nn from model_tools.activations.pytorch import load_preprocess_images class MyModel(nn.Module): def __init__(self): super(MyModel, self).__init__() self.conv1 = torch.nn.Conv2d(in_channels=3, out_channels=2, kernel_size=3) self.relu1 = torch.nn.ReLU() linear_input_size = np.power((224 - 3 + 2 * 0) / 1 + 1, 2) * 2 self.linear = torch.nn.Linear(int(linear_input_size), 1000) self.relu2 = torch.nn.ReLU( ) # can't get named ReLU output otherwise # init weights for reproducibility self.conv1.weight.data.fill_(0.01) self.conv1.bias.data.fill_(0.01) self.linear.weight.data.fill_(0.01) self.linear.bias.data.fill_(0.01) def forward(self, x): x = self.conv1(x) x = self.relu1(x) x = x.view(x.size(0), -1) x = self.linear(x) x = self.relu2(x) return x preprocessing = functools.partial(load_preprocess_images, image_size=224) model_id = 'new_pytorch' activations_model = PytorchWrapper(model=MyModel(), preprocessing=preprocessing, identifier=model_id) layer = 'relu2' candidate = LayerMappedModel(f"{model_id}-{layer}", activations_model=activations_model, visual_degrees=8) candidate.commit('IT', layer) candidate = TemporalIgnore(candidate) ceiled_score = score_model( model_identifier=model_id, model=candidate, benchmark_identifier='dicarlo.MajajHong2015.IT-pls') score = ceiled_score.raw assert score.sel(aggregation='center') == approx(.0820823, abs=.01)
def run_submission(module, test_models, test_benchmarks, submission_entry): ml_brain_pool = get_ml_pool(test_models, module, submission_entry) data = [] success = True try: for model_entry in test_models: model_id = model_entry.name for benchmark_name in test_benchmarks: score_entry = None try: start = datetime.datetime.now() benchmark_entry = get_benchmark_instance(benchmark_name) # Check if the model is already scored on the benchmark score_entry, created = Score.get_or_create(benchmark=benchmark_entry, model=model_entry, defaults={'start_timestamp': start, }) if not created and score_entry.score_raw is not None: logger.warning(f'A score for model {model_id} and benchmark {benchmark_name} already exists') raw = score_entry.score_raw ceiled = score_entry.score_ceiled error = score_entry.error finished = score_entry.end_timestamp comment = score_entry.comment else: if not created: score_entry.start_timestamp = datetime.datetime.now() score_entry.comment = None logger.warning('An entry already exists but was not evaluated successful, we rerun!') logger.info(f"Scoring {model_id}, id {model_entry.id} on benchmark {benchmark_name}") model = ml_brain_pool[model_id] score = score_model(model_id, benchmark_name, model) logger.info(f'Running benchmark {benchmark_name} on model {model_id} (id {model_entry.id}) ' f'produced this score: {score}') if not hasattr(score, 'ceiling'): # many engineering benchmarks do not have a primate ceiling raw = score.sel(aggregation='center').item(0) ceiled = None error = None else: # score has a ceiling. Store ceiled as well as raw value assert score.raw.sel(aggregation='center') is not None raw = score.raw.sel(aggregation='center').item(0) ceiled = score.sel(aggregation='center').item(0) error = score.sel(aggregation='error').item(0) finished = datetime.datetime.now() comment = f"layers: {model.layer_model.region_layer_map}" \ if submission_entry.model_type == 'BaseModel' else '' score_entry.end_timestamp = finished score_entry.error = error score_entry.score_ceiled = ceiled score_entry.score_raw = raw score_entry.comment = comment score_entry.save() result = { 'Model': model_id, 'Benchmark': benchmark_name, 'raw_result': raw, 'ceiled_result': ceiled, 'error': error, 'finished_time': finished, 'comment': comment, } data.append(result) except Exception as e: success = False error = f'Benchmark {benchmark_name} failed for model {model_id} because of this error: {e}' logging.error(f'Could not run model {model_id} because of following error') logging.error(e, exc_info=True) data.append({ 'Model': model_id, 'Benchmark': benchmark_name, 'raw_result': 0, 'ceiled_result': 0, 'error': error, 'finished_time': datetime.datetime.now() }) if score_entry: score_entry.comment = error if len(error) <= SCORE_COMMENT_MAX_LENGTH else \ error[:int(SCORE_COMMENT_MAX_LENGTH / 2) - 5] + ' [...] ' + \ error[-int(SCORE_COMMENT_MAX_LENGTH / 2) + 5:] score_entry.save() finally: if success: submission_entry.status = 'successful' logger.info(f'Submission is stored as successful') else: submission_entry.status = 'failure' logger.info(f'Submission was not entirely successful (some benchmarks could not be executed)') submission_entry.save() return data
# 'mobilenet_v1_1.0_128'] # from tensorflow.compat.v1 import ConfigProto # from tensorflow.compat.v1 import InteractiveSession # # config = ConfigProto() # config.gpu_options.allow_growth = True # session = InteractiveSession(config=config) os.environ['CUDA_VISIBLE_DEVICES'] = '0' from candidate_models.model_commitments import brain_translated_pool model = brain_translated_pool[sys.argv[1]] try: score = score_model(model_identifier=sys.argv[1], model=model, benchmark_identifier='aru.Kuzovkin2018-pls') except Exception as e: print('SOMETHING WENT WRONG. Tried to score {0} and failed. Here is the Exception'.format(sys.argv[1])) print(e) print('Done scoring')
def run_evaluation(config_file, work_dir, jenkins_id, db_secret, models=None, benchmarks=None): secret = get_secret(db_secret) db_configs = json.loads(secret) config_file = Path(config_file).resolve() work_dir = Path(work_dir).resolve() with open(config_file) as file: configs = json.load(file) logger.info(f'Run with following configurations: {str(configs)}') if configs['type'] == 'zip': config_path = config_file.parent logger.info('Start executing models in repo %s' % (configs['zip_filename'])) repo = extract_zip_file(configs, config_path, work_dir) else: logger.info(f'Start executing models in repo {configs["git_url"]}') repo = clone_repo(configs, work_dir) package = 'models.brain_models' if configs['model_type'] == 'BrainModel' else 'models.base_models' module = install_project(repo, package) test_benchmarks = all_benchmarks_list if benchmarks is None or len(benchmarks) == 0 else benchmarks ml_brain_pool = {} test_models = module.get_model_list() if models is None or len(models) == 0 else models if configs['model_type'] == 'BaseModel': logger.info(f"Start working with base models") layers = {} base_model_pool = {} for model in test_models: function = lambda model_inst=model: module.get_model(model_inst) base_model_pool[model] = LazyLoad(function) try: layers[model] = module.get_layers(model) except Exception: logging.warning(f'Could not retrieve layer for model {model} -- skipping model') model_layers = ModelLayers(layers) ml_brain_pool = MLBrainPool(base_model_pool, model_layers) else: logger.info(f"Start working with brain models") for model in test_models: ml_brain_pool[model] = module.get_model(model) data = [] try: for model_id in test_models: model = ml_brain_pool[model_id] for benchmark in test_benchmarks: logger.info(f"Scoring {model_id} on benchmark {benchmark}") try: score = score_model(model_id, benchmark, model) logger.info(f'Running benchmark {benchmark} on model {model_id} produced this score: {score}') if not hasattr(score, 'ceiling'): raw = score.sel(aggregation='center').item(0) ceiled = None error = None else: assert score.raw.sel(aggregation='center') is not None raw = score.raw.sel(aggregation='center').item(0) ceiled = score.sel(aggregation='center').item(0) error = score.sel(aggregation='error').item(0) finished = datetime.datetime.now() result = { 'Model': model_id, 'Benchmark': benchmark, 'raw_result': raw, 'ceiled_result': ceiled, 'error': error, 'finished_time': finished, 'layer' : str(model.layer_model.region_layer_map) } data.append(result) store_score(db_configs, {**result, **{'jenkins_id': jenkins_id, 'email': configs['email'], 'name': configs['name']}}) except Exception as e: error = f'Benchmark {benchmark} failed for model {model_id} because of this error: {e}' logging.error(f'Could not run model {model_id} because of following error') logging.error(e, exc_info=True) data.append({ 'Model': model_id, 'Benchmark': benchmark, 'raw_result': 0, 'ceiled_result': 0, 'error': error, 'finished_time': datetime.datetime.now() }) finally: df = pd.DataFrame(data) # This is the result file we send to the user after the scoring process is done df.to_csv(f'result_{jenkins_id}.csv', index=None, header=True)
# 'pnasnet_large', 'mobilenet_v1_1.0_224', 'mobilenet_v1_1.0_192', 'mobilenet_v1_1.0_160', # 'mobilenet_v1_1.0_128'] # from tensorflow.compat.v1 import ConfigProto # from tensorflow.compat.v1 import InteractiveSession # # config = ConfigProto() # config.gpu_options.allow_growth = True # session = InteractiveSession(config=config) #os.environ['CUDA_VISIBLE_DEVICES'] = '-1' from candidate_models.model_commitments import brain_translated_pool #model = brain_translated_pool[sys.argv[1]] model = brain_translated_pool['alexnet'] try: #score = score_model(model_identifier=sys.argv[1], model=model, benchmark_identifier='aru.Kuzovkin2018-pls') score = score_model(model_identifier='alexnet', model=model, benchmark_identifier='aru.Cichy2019-rdm') except Exception as e: print('SOMETHING WENT WRONG. Tried to score {0} and failed. Here is the Exception'.format(sys.argv[1])) print(e) print('Done scoring')