def test_repeat_same_result(self):
model = self.layer_candidate('alexnet', layer='features.12', region='IT', pca_components=1000)
score1 = score_model(model_identifier='alexnet-f12-pca_1000', model=model,
benchmark_identifier='dicarlo.MajajHong2015.IT-pls')
score2 = score_model(model_identifier='alexnet-f12-pca_1000', model=model,
benchmark_identifier='dicarlo.MajajHong2015.IT-pls')
assert (score1 == score2).all()
def test_Rajalingham2018i2n(self, model_identifier, expected_score):
model = brain_translated_pool[model_identifier]
score = score_model(model_identifier=model_identifier,
model=model,
benchmark_identifier='dicarlo.Rajalingham2018-i2n')
assert score.raw.sel(aggregation='center') == approx(expected_score,
abs=.005)
def test_candidate_Kar2019OST(self, model_identifier, expected_score):
model = brain_translated_pool[model_identifier]
score = score_model(model_identifier=model_identifier, model=model, benchmark_identifier='dicarlo.Kar2019-ost')
if not np.isnan(expected_score):
assert score.raw.sel(aggregation='center') == approx(expected_score, abs=.002)
else:
assert np.isnan(score.raw.sel(aggregation='center'))
def run_benchmark(benchmark_identifier, model_name):
print(
f'>>>>>Start running model {model_name} on benchmark {benchmark_identifier}'
)
model = brain_translated_pool[model_name]
score = score_model(model_identifier=model_name,
model=model,
benchmark_identifier=benchmark_identifier)
return score
def test_MajajHong2015ITpls(self, model_identifier, expected_score, attach_hook):
model = brain_translated_pool[model_identifier]
if attach_hook:
activations_model = model.layer_model._layer_model.activations_model
LayerPCA.hook(activations_model, n_components=1000)
identifier = activations_model.identifier + "-pca_1000"
activations_model.identifier = identifier
score = score_model(model_identifier, 'dicarlo.MajajHong2015.IT-pls', model=model)
assert score.raw.sel(aggregation='center') == approx(expected_score, abs=0.005)
def test_alexnet_conv5_IT(self):
model = self.layer_candidate('alexnet',
layer='features.12',
region='IT',
pca_components=1000)
score = score_model(model_identifier='alexnet-f12-pca_1000',
model=model,
benchmark_identifier='dicarlo.Majaj2015.IT-pls')
assert score.raw.sel(aggregation='center') == approx(0.590345,
abs=0.005)
def test_alexnet_conv2_V4(self):
model = self.layer_candidate('alexnet',
layer='features.5',
region='V4',
pca_components=1000)
score = score_model(model_identifier='alexnet-f5-pca_1000',
model=model,
benchmark_identifier='dicarlo.Majaj2015.V4-pls')
assert score.raw.sel(aggregation='center').max() == approx(0.633703,
abs=0.005)
def test_alexnet_conv3_IT_mask(self):
model = self.layer_candidate('alexnet',
layer='features.6',
region='IT',
pca_components=None)
np.random.seed(123)
score = score_model(model_identifier='alexnet-f6',
model=model,
benchmark_identifier='dicarlo.Majaj2015.IT-mask')
assert score.raw.sel(aggregation='center') == approx(0.607037,
abs=0.005)
def test_newmodel_pytorch(self):
import torch
from torch import nn
from model_tools.activations.pytorch import load_preprocess_images
class MyModel(nn.Module):
def __init__(self):
super(MyModel, self).__init__()
self.conv1 = torch.nn.Conv2d(in_channels=3,
out_channels=2,
kernel_size=3)
self.relu1 = torch.nn.ReLU()
linear_input_size = np.power((224 - 3 + 2 * 0) / 1 + 1, 2) * 2
self.linear = torch.nn.Linear(int(linear_input_size), 1000)
self.relu2 = torch.nn.ReLU(
) # can't get named ReLU output otherwise
# init weights for reproducibility
self.conv1.weight.data.fill_(0.01)
self.conv1.bias.data.fill_(0.01)
self.linear.weight.data.fill_(0.01)
self.linear.bias.data.fill_(0.01)
def forward(self, x):
x = self.conv1(x)
x = self.relu1(x)
x = x.view(x.size(0), -1)
x = self.linear(x)
x = self.relu2(x)
return x
preprocessing = functools.partial(load_preprocess_images,
image_size=224)
model_id = 'new_pytorch'
activations_model = PytorchWrapper(model=MyModel(),
preprocessing=preprocessing,
identifier=model_id)
layer = 'relu2'
candidate = LayerMappedModel(f"{model_id}-{layer}",
activations_model=activations_model,
visual_degrees=8)
candidate.commit('IT', layer)
candidate = TemporalIgnore(candidate)
ceiled_score = score_model(
model_identifier=model_id,
model=candidate,
benchmark_identifier='dicarlo.MajajHong2015.IT-pls')
score = ceiled_score.raw
assert score.sel(aggregation='center') == approx(.0820823, abs=.01)
def run_submission(module, test_models, test_benchmarks, submission_entry):
ml_brain_pool = get_ml_pool(test_models, module, submission_entry)
data = []
success = True
try:
for model_entry in test_models:
model_id = model_entry.name
for benchmark_name in test_benchmarks:
score_entry = None
try:
start = datetime.datetime.now()
benchmark_entry = get_benchmark_instance(benchmark_name)
# Check if the model is already scored on the benchmark
score_entry, created = Score.get_or_create(benchmark=benchmark_entry, model=model_entry,
defaults={'start_timestamp': start, })
if not created and score_entry.score_raw is not None:
logger.warning(f'A score for model {model_id} and benchmark {benchmark_name} already exists')
raw = score_entry.score_raw
ceiled = score_entry.score_ceiled
error = score_entry.error
finished = score_entry.end_timestamp
comment = score_entry.comment
else:
if not created:
score_entry.start_timestamp = datetime.datetime.now()
score_entry.comment = None
logger.warning('An entry already exists but was not evaluated successful, we rerun!')
logger.info(f"Scoring {model_id}, id {model_entry.id} on benchmark {benchmark_name}")
model = ml_brain_pool[model_id]
score = score_model(model_id, benchmark_name, model)
logger.info(f'Running benchmark {benchmark_name} on model {model_id} (id {model_entry.id}) '
f'produced this score: {score}')
if not hasattr(score, 'ceiling'): # many engineering benchmarks do not have a primate ceiling
raw = score.sel(aggregation='center').item(0)
ceiled = None
error = None
else: # score has a ceiling. Store ceiled as well as raw value
assert score.raw.sel(aggregation='center') is not None
raw = score.raw.sel(aggregation='center').item(0)
ceiled = score.sel(aggregation='center').item(0)
error = score.sel(aggregation='error').item(0)
finished = datetime.datetime.now()
comment = f"layers: {model.layer_model.region_layer_map}" \
if submission_entry.model_type == 'BaseModel' else ''
score_entry.end_timestamp = finished
score_entry.error = error
score_entry.score_ceiled = ceiled
score_entry.score_raw = raw
score_entry.comment = comment
score_entry.save()
result = {
'Model': model_id,
'Benchmark': benchmark_name,
'raw_result': raw,
'ceiled_result': ceiled,
'error': error,
'finished_time': finished,
'comment': comment,
}
data.append(result)
except Exception as e:
success = False
error = f'Benchmark {benchmark_name} failed for model {model_id} because of this error: {e}'
logging.error(f'Could not run model {model_id} because of following error')
logging.error(e, exc_info=True)
data.append({
'Model': model_id, 'Benchmark': benchmark_name,
'raw_result': 0, 'ceiled_result': 0,
'error': error, 'finished_time': datetime.datetime.now()
})
if score_entry:
score_entry.comment = error if len(error) <= SCORE_COMMENT_MAX_LENGTH else \
error[:int(SCORE_COMMENT_MAX_LENGTH / 2) - 5] + ' [...] ' + \
error[-int(SCORE_COMMENT_MAX_LENGTH / 2) + 5:]
score_entry.save()
finally:
if success:
submission_entry.status = 'successful'
logger.info(f'Submission is stored as successful')
else:
submission_entry.status = 'failure'
logger.info(f'Submission was not entirely successful (some benchmarks could not be executed)')
submission_entry.save()
return data
# 'mobilenet_v1_1.0_128']
# from tensorflow.compat.v1 import ConfigProto
# from tensorflow.compat.v1 import InteractiveSession
#
# config = ConfigProto()
# config.gpu_options.allow_growth = True
# session = InteractiveSession(config=config)
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
from candidate_models.model_commitments import brain_translated_pool
model = brain_translated_pool[sys.argv[1]]
try:
score = score_model(model_identifier=sys.argv[1], model=model, benchmark_identifier='aru.Kuzovkin2018-pls')
except Exception as e:
print('SOMETHING WENT WRONG. Tried to score {0} and failed. Here is the Exception'.format(sys.argv[1]))
print(e)
print('Done scoring')
def run_evaluation(config_file, work_dir, jenkins_id, db_secret, models=None,
benchmarks=None):
secret = get_secret(db_secret)
db_configs = json.loads(secret)
config_file = Path(config_file).resolve()
work_dir = Path(work_dir).resolve()
with open(config_file) as file:
configs = json.load(file)
logger.info(f'Run with following configurations: {str(configs)}')
if configs['type'] == 'zip':
config_path = config_file.parent
logger.info('Start executing models in repo %s' % (configs['zip_filename']))
repo = extract_zip_file(configs, config_path, work_dir)
else:
logger.info(f'Start executing models in repo {configs["git_url"]}')
repo = clone_repo(configs, work_dir)
package = 'models.brain_models' if configs['model_type'] == 'BrainModel' else 'models.base_models'
module = install_project(repo, package)
test_benchmarks = all_benchmarks_list if benchmarks is None or len(benchmarks) == 0 else benchmarks
ml_brain_pool = {}
test_models = module.get_model_list() if models is None or len(models) == 0 else models
if configs['model_type'] == 'BaseModel':
logger.info(f"Start working with base models")
layers = {}
base_model_pool = {}
for model in test_models:
function = lambda model_inst=model: module.get_model(model_inst)
base_model_pool[model] = LazyLoad(function)
try:
layers[model] = module.get_layers(model)
except Exception:
logging.warning(f'Could not retrieve layer for model {model} -- skipping model')
model_layers = ModelLayers(layers)
ml_brain_pool = MLBrainPool(base_model_pool, model_layers)
else:
logger.info(f"Start working with brain models")
for model in test_models:
ml_brain_pool[model] = module.get_model(model)
data = []
try:
for model_id in test_models:
model = ml_brain_pool[model_id]
for benchmark in test_benchmarks:
logger.info(f"Scoring {model_id} on benchmark {benchmark}")
try:
score = score_model(model_id, benchmark, model)
logger.info(f'Running benchmark {benchmark} on model {model_id} produced this score: {score}')
if not hasattr(score, 'ceiling'):
raw = score.sel(aggregation='center').item(0)
ceiled = None
error = None
else:
assert score.raw.sel(aggregation='center') is not None
raw = score.raw.sel(aggregation='center').item(0)
ceiled = score.sel(aggregation='center').item(0)
error = score.sel(aggregation='error').item(0)
finished = datetime.datetime.now()
result = {
'Model': model_id,
'Benchmark': benchmark,
'raw_result': raw,
'ceiled_result': ceiled,
'error': error,
'finished_time': finished,
'layer' : str(model.layer_model.region_layer_map)
}
data.append(result)
store_score(db_configs, {**result, **{'jenkins_id': jenkins_id,
'email': configs['email'],
'name': configs['name']}})
except Exception as e:
error = f'Benchmark {benchmark} failed for model {model_id} because of this error: {e}'
logging.error(f'Could not run model {model_id} because of following error')
logging.error(e, exc_info=True)
data.append({
'Model': model_id, 'Benchmark': benchmark,
'raw_result': 0, 'ceiled_result': 0,
'error': error, 'finished_time': datetime.datetime.now()
})
finally:
df = pd.DataFrame(data)
# This is the result file we send to the user after the scoring process is done
df.to_csv(f'result_{jenkins_id}.csv', index=None, header=True)
# 'pnasnet_large', 'mobilenet_v1_1.0_224', 'mobilenet_v1_1.0_192', 'mobilenet_v1_1.0_160',
# 'mobilenet_v1_1.0_128']
# from tensorflow.compat.v1 import ConfigProto
# from tensorflow.compat.v1 import InteractiveSession
#
# config = ConfigProto()
# config.gpu_options.allow_growth = True
# session = InteractiveSession(config=config)
#os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
from candidate_models.model_commitments import brain_translated_pool
#model = brain_translated_pool[sys.argv[1]]
model = brain_translated_pool['alexnet']
try:
#score = score_model(model_identifier=sys.argv[1], model=model, benchmark_identifier='aru.Kuzovkin2018-pls')
score = score_model(model_identifier='alexnet', model=model, benchmark_identifier='aru.Cichy2019-rdm')
except Exception as e:
print('SOMETHING WENT WRONG. Tried to score {0} and failed. Here is the Exception'.format(sys.argv[1]))
print(e)
print('Done scoring')
