def __call__(self, candidate):
# The proper `fitting_stimuli` to pass to the candidate would be the imagenet training set.
# For now, since almost all models in our hands were trained with imagenet, we'll just short-cut this
# by telling the candidate to use its pre-trained imagenet weights.
candidate.start_task(BrainModel.Task.label, 'imagenet')
stimulus_set = self._stimulus_set[list(
set(self._stimulus_set.columns) - {'synset'})] # do not show label
stimulus_set = repeat_trials(stimulus_set, number_of_trials=10)
predictions = candidate.look_at(stimulus_set)
predictions = average_trials(predictions)
score = self._similarity_metric(predictions,
self._stimulus_set['synset'].values)
return score
def __call__(self, candidate: BrainModel):
time_bins = [(time_bin_start, time_bin_start + 10)
for time_bin_start in range(70, 250, 10)]
candidate.start_recording('IT', time_bins=time_bins)
stimulus_set = place_on_screen(
self._assembly.stimulus_set,
target_visual_degrees=candidate.visual_degrees(),
source_visual_degrees=self._visual_degrees)
stimulus_set = repeat_trials(stimulus_set,
number_of_trials=self._number_of_trials)
recordings = candidate.look_at(stimulus_set)
recordings = average_trials(recordings)
score = self._similarity_metric(recordings, self._assembly)
score = ceil_score(score, self.ceiling)
return score
def __call__(self, candidate: BrainModel):
candidate.start_recording(self.region, time_bins=self.timebins)
stimulus_set = place_on_screen(
self._assembly.stimulus_set,
target_visual_degrees=candidate.visual_degrees(),
source_visual_degrees=self._visual_degrees)
stimulus_set = repeat_trials(stimulus_set,
number_of_trials=self._number_of_trials)
source_assembly = candidate.look_at(stimulus_set)
if 'time_bin' in source_assembly.dims:
source_assembly = source_assembly.squeeze(
'time_bin') # static case for these benchmarks
source_assembly = average_trials(source_assembly)
raw_score = self._similarity_metric(source_assembly, self._assembly)
return explained_variance(raw_score, self.ceiling)
def __call__(self, candidate: BrainModel):
fitting_stimuli = place_on_screen(
self._fitting_stimuli,
target_visual_degrees=candidate.visual_degrees(),
source_visual_degrees=self._visual_degrees)
candidate.start_task(BrainModel.Task.probabilities, fitting_stimuli)
stimulus_set = place_on_screen(
self._assembly.stimulus_set,
target_visual_degrees=candidate.visual_degrees(),
source_visual_degrees=self._visual_degrees)
stimulus_set = repeat_trials(stimulus_set,
number_of_trials=self._number_of_trials)
probabilities = candidate.look_at(stimulus_set)
probabilities = average_trials(probabilities)
score = self._metric(probabilities, self._assembly)
score = self.ceil_score(score, self.ceiling)
return score
def test_integer_repeat():
stimulus_set = _dummy_stimulus_set()
repeat_stimulus_set = repeat_trials(stimulus_set, number_of_trials=5)
assert len(repeat_stimulus_set) == len(stimulus_set) * 5
original_image_paths = [
stimulus_set.get_image(image_id)
for image_id in stimulus_set['image_id']
]
repeat_image_paths = [
repeat_stimulus_set.get_image(image_id)
for image_id in repeat_stimulus_set['image_id']
]
assert set(repeat_image_paths) == set(original_image_paths)
assert all(
len(group) == 5 and set(group['repetition']) == {0, 1, 2, 3, 4}
for name, group in repeat_stimulus_set.groupby('image_id'))
assert repeat_stimulus_set.identifier == 'dummy-5trials'