def convert_stimuli(stimulus_set_existing, stimulus_set_name_new,
image_dir_new):
Path(image_dir_new).mkdir(parents=True, exist_ok=True)
image_converter = ApplyCosineAperture(target_dir=image_dir_new)
converted_image_paths = {}
converted_image_ids = {}
for image_id in tqdm(stimulus_set_existing['image_id'],
total=len(stimulus_set_existing),
desc='apply cosine aperture'):
converted_image_path = image_converter.convert_image(
image_path=stimulus_set_existing.get_image(image_id))
converted_image_id = kf(converted_image_path).sha1
converted_image_ids[image_id] = converted_image_id
converted_image_paths[converted_image_id] = converted_image_path
_logger.debug(
f"{image_id} -> {converted_image_id}: {converted_image_path}")
converted_stimuli = StimulusSet(stimulus_set_existing.copy(deep=True))
converted_stimuli["image_id_without_aperture"] = converted_stimuli[
"image_id"]
converted_stimuli["image_id"] = converted_stimuli["image_id"].map(
converted_image_ids)
converted_stimuli["image_file_sha1"] = converted_stimuli["image_id"]
converted_stimuli.image_paths = converted_image_paths
converted_stimuli.name = stimulus_set_name_new
converted_stimuli.id_mapping = converted_image_ids
return converted_stimuli
def collect_stimuli(data_dir):
IT_base616 = pickle.load(open(os.path.join(data_dir, 'data_IT_base616.pkl'), 'rb'))
stimuli = IT_base616['meta']
# Add columns
stimuli['image_id'] = ''
stimuli['image_file_name'] = ''
stimuli['image_current_local_file_path'] = ''
stimuli['image_path_within_store'] = ''
stimuli['image_file_sha1'] = ''
for idx, row in stimuli.iterrows():
image_file_name = f'{row.id}.png'
image_file_path = os.path.join(data_dir, 'stimuli', image_file_name)
im_kf = kf(image_file_path)
stimuli.at[idx, 'image_id'] = im_kf.sha1
stimuli.at[idx, 'image_file_name'] = image_file_name
stimuli.at[idx, 'image_current_local_file_path'] = image_file_path
stimuli.at[idx, 'image_path_within_store'] = image_file_name
stimuli.at[idx, 'image_file_sha1'] = im_kf.sha1
stimuli = stimuli.drop(columns='id') # Drop ID column since the information is retained in other columns
stimuli['grp5_bigram_freq'] = stimuli['grp5_bigram_freq'].astype(str) # IntervalIndex not supported by netCDF4
stimuli = stimuli.astype({column_name: 'int32' for column_name
in stimuli.select_dtypes(include=['bool']).keys()}) # Bool not supported by netCDF4
assert len(np.unique(stimuli['image_id'])) == len(stimuli)
stimuli = StimulusSet(stimuli)
stimuli.image_paths = \
{stimuli.at[idx, 'image_id']: stimuli.at[idx, 'image_current_local_file_path'] for idx in range(len(stimuli))}
return stimuli
def collect_stimuli(stimuli_directory):
labels = np.load(stimuli_directory + 'stimgroups.npy') # labels of image
stim_sequence = np.load(
stimuli_directory + 'stimsequence.npy'
) # the names of the files with a b: "b'V12'" (Image ID)
# image file name will be the stimuli_directory + ID.jpg
stimuli = []
for x in range(len(labels)):
stimuli.append({
'image_id':
stim_sequence[x].decode('UTF-8'), # extract just the ID
'image_file_name':
stimuli_directory + "stimuli/" +
str(stim_sequence[x].decode('UTF-8')) + ".jpg",
'image_number':
x,
'label':
labels[x],
})
stimuli = pd.DataFrame(stimuli)
# convert stimuli object into something that can be used with all the packaging functions
stimuli = StimulusSet(stimuli)
# after converted to a type "StimulusSet", you set an attribute of the object, such as "image_paths":
stimuli.image_paths = {
key: stimuli['image_file_name'][i]
for i, key in enumerate(stimuli['image_id'])
}
return stimuli
def test_creates_probabilities(self):
activations_model = pytorch_custom()
brain_model = ModelCommitment(identifier=activations_model.identifier,
activations_model=activations_model,
layers=None,
behavioral_readout_layer='relu2')
fitting_stimuli = StimulusSet({
'image_id': ['rgb1', 'rgb2'],
'image_label': ['label1', 'label2']
})
fitting_stimuli.image_paths = {
'rgb1': os.path.join(os.path.dirname(__file__), 'rgb1.jpg'),
'rgb2': os.path.join(os.path.dirname(__file__), 'rgb2.jpg')
}
fitting_stimuli.identifier = 'test_probabilities_mapping.creates_probabilities'
fitting_stimuli = place_on_screen(
fitting_stimuli,
target_visual_degrees=brain_model.visual_degrees(),
source_visual_degrees=8)
brain_model.start_task(BrainModel.Task.probabilities, fitting_stimuli)
probabilities = brain_model.look_at(fitting_stimuli)
np.testing.assert_array_equal(probabilities.dims,
['presentation', 'choice'])
np.testing.assert_array_equal(probabilities.shape, [2, 2])
np.testing.assert_array_almost_equal(
probabilities.sel(image_id='rgb1', choice='label1').values,
probabilities.sel(image_id='rgb2', choice='label2').values)
assert probabilities.sel(image_id='rgb1', choice='label1') + \
probabilities.sel(image_id='rgb1', choice='label2') == approx(1)
def collect_stimuli(data_dir):
IT_base616 = pickle.load(
open(os.path.join(data_dir, 'data_IT_base616.pkl'), 'rb'))
stimuli = IT_base616['meta']
stimuli = stimuli.rename(columns={'id': 'image_id'})
stimuli['image_file_name'] = ''
stimuli['image_current_local_file_path'] = ''
for idx, row in stimuli.iterrows():
image_file_name = f'{row.image_id}.png'
image_file_path = os.path.join(data_dir, 'stimuli', image_file_name)
stimuli.at[idx, 'image_file_name'] = image_file_name
stimuli.at[idx, 'image_current_local_file_path'] = image_file_path
stimuli['grp5_bigram_freq'] = stimuli['grp5_bigram_freq'].astype(
str) # IntervalIndex not supported by netCDF4
stimuli = stimuli.astype({
column_name: 'int32'
for column_name in stimuli.select_dtypes(include=['bool']).keys()
}) # Bool not supported by netCDF4
assert len(np.unique(stimuli['image_id'])) == len(stimuli)
stimuli = StimulusSet(stimuli)
stimuli.image_paths = \
{stimuli.at[idx, 'image_id']: stimuli.at[idx, 'image_current_local_file_path'] for idx in range(len(stimuli))}
return stimuli
def test_from_stimulus_set(model_ctr, layers, pca_components):
image_names = [
'rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png'
]
stimulus_set = StimulusSet([{
'image_id': image_name,
'some_meta': image_name[::-1]
} for image_name in image_names])
stimulus_set.image_paths = {
image_name: os.path.join(os.path.dirname(__file__), image_name)
for image_name in image_names
}
activations_extractor = model_ctr()
if pca_components:
LayerPCA.hook(activations_extractor, pca_components)
activations = activations_extractor.from_stimulus_set(
stimulus_set, layers=layers, stimuli_identifier=False)
assert activations is not None
assert set(activations['image_id'].values) == set(image_names)
assert all(activations['some_meta'].values ==
[image_name[::-1] for image_name in image_names])
assert len(np.unique(activations['layer'])) == len(layers)
if pca_components is not None:
assert len(activations['neuroid']) == pca_components * len(layers)
def __init__(self):
stimulus_set = pd.read_csv(
os.path.join(os.path.dirname(__file__), 'imagenet2012.csv'))
stimulus_set = StimulusSet(stimulus_set)
stimulus_set.image_paths = {
row.image_id: row.filepath
for row in stimulus_set.itertuples()
}
self._stimulus_set = stimulus_set
self._similarity_metric = Accuracy()
ceiling = Score([1, np.nan],
coords={'aggregation': ['center', 'error']},
dims=['aggregation'])
super(Imagenet2012, self).__init__(identifier='fei-fei.Deng2009-top1',
version=1,
ceiling_func=lambda: ceiling,
parent='ImageNet',
bibtex="""@INPROCEEDINGS{5206848,
author={J. {Deng} and W. {Dong} and R. {Socher} and L. {Li} and {Kai Li} and {Li Fei-Fei}},
booktitle={2009 IEEE Conference on Computer Vision and Pattern Recognition},
title={ImageNet: A large-scale hierarchical image database},
year={2009},
volume={},
number={},
pages={248-255},
url = {https://ieeexplore.ieee.org/document/5206848}
}""")
def get_assembly():
image_names = []
for i in range(1, 21):
image_names.append(f'images/{i}.png')
assembly = NeuroidAssembly(
(np.arange(40 * 5) + np.random.standard_normal(40 * 5)).reshape(
(5, 40, 1)),
coords={
'image_id': ('presentation', image_names * 2),
'object_name': ('presentation', ['a'] * 40),
'repetition': ('presentation', ([1] * 20 + [2] * 20)),
'neuroid_id': ('neuroid', np.arange(5)),
'region': ('neuroid', ['IT'] * 5),
'time_bin_start': ('time_bin', [70]),
'time_bin_end': ('time_bin', [170])
},
dims=['neuroid', 'presentation', 'time_bin'])
labels = ['a'] * 10 + ['b'] * 10
stimulus_set = StimulusSet([{
'image_id': image_names[i],
'object_name': 'a',
'image_label': labels[i]
} for i in range(20)])
stimulus_set.image_paths = {
image_name: os.path.join(os.path.dirname(__file__), image_name)
for image_name in image_names
}
stimulus_set.identifier = 'test'
assembly.attrs['stimulus_set'] = stimulus_set
assembly.attrs['stimulus_set_name'] = stimulus_set.identifier
assembly = assembly.squeeze("time_bin")
return assembly.transpose('presentation', 'neuroid')
def collect_stimuli(stimuli_directory):
stimuli = []
num_images = 128
for x in range(num_images):
stimuli.append({
'image_id':
str(x + 1), # extract just the ID
'image_file_name':
stimuli_directory + "stimuli/stimuli/stim" +
"{0:0=3d}".format(x + 1) + ".png",
'image_number':
x + 1,
'label':
str(x + 1),
})
stimuli = pd.DataFrame(stimuli)
stimuli = StimulusSet(stimuli)
stimuli.image_paths = {
key: stimuli['image_file_name'][i]
for i, key in enumerate(stimuli['image_id'])
}
return stimuli
def get_stimulus_set(name):
stimulus_set_model = StimulusSetModel.get(StimulusSetModel.name == name)
image_paths = fetch_stimulus_set(stimulus_set_model)
pw_query = ImageModel.select() \
.join(StimulusSetImageMap) \
.join(StimulusSetModel) \
.where(StimulusSetModel.name == name)
df_reconstructed = pd.DataFrame(list(pw_query.dicts()))
pw_query_attributes = AttributeModel.select() \
.join(ImageMetaModel) \
.join(ImageModel) \
.join(StimulusSetImageMap) \
.join(StimulusSetModel) \
.where(StimulusSetModel.name == name) \
.distinct()
for a in pw_query_attributes:
pw_query_single_attribute = AttributeModel.select(ImageModel.image_id, ImageMetaModel.value) \
.join(ImageMetaModel) \
.join(ImageModel) \
.join(StimulusSetImageMap) \
.join(StimulusSetModel) \
.where((StimulusSetModel.name == name) & (AttributeModel.name == a.name))
df_single_attribute = pd.DataFrame(
list(pw_query_single_attribute.dicts()))
merged = df_reconstructed.merge(df_single_attribute,
on="image_id",
how="left",
suffixes=("orig_", ""))
df_reconstructed[a.name] = merged["value"].astype(a.type)
stimulus_set = StimulusSet(df_reconstructed)
stimulus_set.image_paths = image_paths
stimulus_set.name = name
return stimulus_set
def collect_synth(h5, data_dir):
protos_stimuli = []
responses_synth_d = {}
for monkey in h5.root.images.synthetic:
for setting in monkey:
for session_images in setting:
session_neural = h5.root.neural.synthetic[monkey._v_name][
setting._v_name][session_images._v_name]
session_target_inds = h5.root.target_inds[monkey._v_name][
setting._v_name][session_images._v_name]
identifier = f"{monkey._v_name[-1]}_{setting._v_name}_{session_images._v_name}"
img_temp_path = data_dir / "images_temp" / "synthetic" / identifier
img_temp_path.mkdir(parents=True, exist_ok=True)
proto_stimuli = np_to_png(session_images, img_temp_path)
proto_stimuli["animal"] = monkey._v_name
proto_stimuli["setting"] = setting._v_name
proto_stimuli["session"] = session_images._v_name
protos_stimuli.append(proto_stimuli)
proto_neural = np_to_xr(monkey, setting, session_neural,
proto_stimuli, session_target_inds,
"synth")
proto_neural = NeuronRecordingAssembly(proto_neural)
responses_synth_d[proto_neural.name] = proto_neural
proto_stimuli_all = pd.concat(protos_stimuli, axis=0)
assert len(np.unique(
proto_stimuli_all['image_id'])) == len(proto_stimuli_all)
stimuli = StimulusSet(proto_stimuli_all)
stimuli.image_paths = {
row.image_id: row.image_current_local_file_path
for row in stimuli.itertuples()
}
return stimuli, responses_synth_d
def collect_stimuli(data_path):
stimuli = []
# search images
for i in range(1, 68):
target_path = os.path.join(data_path / 'stimuli', 's_' + str(i) + '.jpg')
filename = 's_' + str(i) + '.jpg'
image_id = 'klab_vs_waldo_stimuli_' + str(i)
image_label = 'stimuli'
sample_number = i
stimuli.append({
'image_current_local_file_path': target_path,
'image_path_within_store': filename,
'image_label': image_label,
'image_id': image_id,
'sample_number': sample_number,
})
# target images
for i in range(1, 68):
target_path = os.path.join(data_path / 'target', 't_' + str(i) + '.jpg')
filename = 't_' + str(i) + '.jpg'
image_id = 'klab_vs_waldo_target_' + str(i)
image_label = 'target'
sample_number = i
stimuli.append({
'image_current_local_file_path': target_path,
'image_path_within_store': filename,
'image_label': image_label,
'image_id': image_id,
'sample_number': sample_number,
})
# target mask
for i in range(1, 68):
target_path = os.path.join(data_path / 'gt', 'gt_' + str(i) + '.jpg')
filename = 'gt_' + str(i) + '.jpg'
image_id = 'klab_vs_waldo_gt_' + str(i)
image_label = 'gt'
sample_number = i
stimuli.append({
'image_current_local_file_path': target_path,
'image_path_within_store': filename,
'image_label': image_label,
'image_id': image_id,
'sample_number': sample_number,
})
stimuli = StimulusSet(stimuli)
stimuli.image_paths = {row.image_id: row.image_current_local_file_path for row in stimuli.itertuples()}
stimuli['image_file_name']= stimuli['image_path_within_store']
return stimuli
def collect_stimuli(data_dir):
stimulus_set = pd.read_csv(data_dir / 'imagenet2012.csv')
stimulus_set = StimulusSet(stimulus_set)
stimulus_set.image_paths = {row.image_id: row.filepath for row in stimulus_set.itertuples()}
stimulus_set['image_path_within_store'] = stimulus_set['filename'].apply(
lambda filename: os.path.splitext(filename)[0])
stimulus_set = stimulus_set[['image_id', 'label', 'synset', 'image_file_sha1', 'image_path_within_store']]
assert len(np.unique(stimulus_set['image_id'])) == len(stimulus_set), "duplicate entries"
return stimulus_set
def test_image_numbers():
stimulus_set = StimulusSet(DataFrame({'image_id': [0, 1]}))
filenames = ['Nat300_1.png', 'Nat300_2.png']
assert len(stimulus_set) == len(filenames)
stimulus_set.image_paths = {
stimulus_set.at[idx, 'image_id']: filenames[idx]
for idx in range(len(stimulus_set))
}
check_image_numbers(stimulus_set)
def collect_stimuli(data_path):
assert os.path.isdir(data_path)
stimulus_df = pd.read_pickle(os.path.join(data_path, 'info.pkl'))
stimulus_set = StimulusSet(stimulus_df)
stimulus_set.image_paths = {
stimulus_set.at[idx, 'image_id']:
os.path.join(data_path, 'data', stimulus_set.at[idx, 'image_name'])
for idx in range(len(stimulus_set))
}
return stimulus_set
def _build_stimulus_set(image_names):
stimulus_set = StimulusSet([{
'image_id': image_name,
'some_meta': image_name[::-1]
} for image_name in image_names])
stimulus_set.image_paths = {
image_name: os.path.join(os.path.dirname(__file__), image_name)
for image_name in image_names
}
return stimulus_set
def load_stim_info(stim_name, data_dir):
stim = pd.read_csv(os.path.join(data_dir, 'stimulus_set'), dtype={'image_id': str})
image_paths = dict((key, value) for (key, value) in zip(stim['image_id'].values,
[os.path.join(data_dir, image_name) for image_name
in stim['image_file_name'].values]))
stim_set = StimulusSet(stim[stim.columns[:-1]])
stim_set.image_paths = image_paths
stim_set.identifier = stim_name
return stim_set
def load(self):
stimulus_set = pd.read_csv(self.csv_path)
stimulus_set = StimulusSet(stimulus_set)
stimulus_set.image_paths = {
row['image_id']: os.path.join(self.stimuli_directory,
row['filename'])
for _, row in stimulus_set.iterrows()
}
assert all(
os.path.isfile(image_path)
for image_path in stimulus_set.image_paths.values())
return stimulus_set
def collect_stimuli_nat(h5, data_dir):
img_array = h5.root.images.naturalistic
img_temp_path = data_dir / "images_temp" / "naturalistic"
img_temp_path.mkdir(parents=True, exist_ok=True)
proto = np_to_png(img_array, img_temp_path)
assert len(np.unique(proto['image_id'])) == len(proto)
stimuli = StimulusSet(proto)
stimuli.image_paths = {
row.image_id: row.image_current_local_file_path
for row in stimuli.itertuples()
}
return stimuli
def test_creates_synset(self, model_ctr):
np.random.seed(0)
activations_model = model_ctr()
brain_model = ModelCommitment(identifier=activations_model.identifier, activations_model=activations_model,
layers=None, behavioral_readout_layer='dummy') # not needed
stimuli = StimulusSet({'image_id': ['abc123']})
stimuli.image_paths = {'abc123': os.path.join(os.path.dirname(__file__), 'rgb1.jpg')}
stimuli.name = 'test_logits_behavior.creates_synset'
brain_model.start_task(BrainModel.Task.label, 'imagenet')
synsets = brain_model.look_at(stimuli)
assert len(synsets) == 1
assert synsets[0].startswith('n')
def prep_proto_stim():
image_dir = Path(__file__).parent / "images"
csv_path = image_dir / "test_images.csv"
proto = pd.read_csv(csv_path)
proto["image_id"] = [f"{iid}.{now()}" for iid in proto["image_id"]]
proto[f"test_{now()}"] = [f"{iid}.{now()}" for iid in proto["image_id"]]
proto = StimulusSet(proto)
proto.image_paths = {
row.image_id: image_dir / row.image_current_relative_file_path
for row in proto.itertuples()
}
proto['image_file_name'] = proto['image_path_within_store']
return proto
def collect_stimuli(data_dir):
image_dir = data_dir / 'images' / 'bold5000'
assert os.path.isdir(image_dir)
files = sorted(os.listdir(image_dir), key=lambda x: int(os.path.splitext(x)[0].split('_')[-1]))
files = files[:-30] # Discard last 30 images (5 grey and 25 normalizer images)
assert os.path.isdir(data_dir / 'image-metadata')
stimuli = pickle.load(open(data_dir / 'image-metadata' / 'bold5000_metadata.pkl', 'rb'))
# Rename columns
stimuli = stimuli.rename(columns={'image_id': 'id', 'original_dataset': 'source_dataset',
'image_file_name': 'image_id', 'wordnet_id': 'imagenet_synset',
'category': 'coco_category', 'category_id': 'coco_category_id',
'flickr_url': 'coco_flickr_url', 'area': 'coco_area',
'bbox': 'coco_bbox', 'supercategory': 'coco_supercategory',
'label_id': 'coco_label_id', 'segmentation': 'coco_segmentation'})
# Replace all NaNs with None
stimuli = stimuli.where(pd.notnull(stimuli), None)
# Stringify lists and ndarrays
stimuli['label'] = stimuli['label'].apply(lambda x: str(list(x)).strip('[]') if type(x) is np.ndarray else x)
stimuli['coco_label_id'] = stimuli['coco_label_id'].apply(lambda x: str(list(x)) if type(x) is np.ndarray else x)
stimuli['coco_area'] = stimuli['coco_area'].apply(lambda x: str(list(x)) if type(x) is np.ndarray else x)
stimuli['coco_bbox'] = stimuli['coco_bbox'].apply(lambda x: str(list(x)) if type(x) is np.ndarray else x)
stimuli['coco_category_id'] = stimuli['coco_category_id'].apply(lambda x: str(list(x)) if type(x) is np.ndarray else x)
stimuli['coco_category'] = stimuli['coco_category'].apply(lambda x: str(x) if type(x) is list else x)
stimuli['coco_segmentation'] = stimuli['coco_segmentation'].apply(lambda x: str(list(x)) if type(x) is np.ndarray else x)
stimuli['coco_supercategory'] = stimuli['coco_supercategory'].apply(lambda x: str(x) if type(x) is list else x)
# Temporarily drop columns unique to a single source dataset, since filtering method breaks down
# (with NaN/None?), and so does pandas read when fetching assembly from S3
stimuli = stimuli.drop(['coco_url', 'coco_flickr_url', 'coco_category_id', 'coco_category', 'coco_supercategory',
'coco_segmentation', 'coco_label_id', 'coco_area', 'coco_bbox', 'coco_id',
'imagenet_synset'], axis=1)
stimuli['image_file_name'] = ''
stimuli['image_current_local_file_path'] = ''
for idx, image_file_name in enumerate(files):
image_file_path = os.path.join(image_dir, image_file_name)
stimuli.at[idx, 'image_file_name'] = image_file_name
stimuli.at[idx, 'image_current_local_file_path'] = image_file_path
assert len(np.unique(stimuli['image_id'])) == len(stimuli)
stimuli = StimulusSet(stimuli)
stimuli.image_paths = \
{stimuli.at[idx, 'image_id']: stimuli.at[idx, 'image_current_local_file_path'] for idx in range(len(stimuli))}
return stimuli
def load_stimuli(meta_assembly, source_stim_path):
stimuli_paths = list(glob(os.path.join(source_stim_path, '*.png')))
stimuli_paths.sort()
stimuli = StimulusSet({
'image_current_local_file_path':
stimuli_paths,
'image_id': [
os.path.splitext(os.path.basename(filepath))[0]
for filepath in stimuli_paths
],
'image_path_within_store':
[os.path.basename(filepath) for filepath in stimuli_paths]
})
assert all(
meta_assembly['sample_obj'].values == meta_assembly['truth'].values)
image_meta = {
image_id: coord_value
for image_id, coord_value in zip(meta_assembly['image_id'].values,
meta_assembly['sample_obj'].values)
}
meta_values = [
image_meta[image_id] for image_id in stimuli['image_id'].values
]
stimuli['image_sample_obj'] = meta_values
stimuli['image_label'] = stimuli['image_sample_obj']
return stimuli
def test_commit(self, model_ctr, layers, region):
activations_model = model_ctr()
layer_model = LayerMappedModel(identifier=activations_model.identifier,
activations_model=activations_model)
layer_model.commit(region, layers)
layer_model.start_recording(region)
stimulus_set = StimulusSet([{'image_id': 'test'}])
stimulus_set.image_paths = {
'test': os.path.join(os.path.dirname(__file__), 'rgb1.jpg')
}
stimulus_set.name = self.__class__.__name__
predictions = layer_model.look_at(stimulus_set)
assert set(predictions['region'].values) == {region}
assert set(predictions['layer'].values) == {layers} if isinstance(
layers, str) else set(layers)
def _place_on_screen(stimuli_identifier: str,
stimulus_set: StimulusSet,
target_visual_degrees: int,
source_visual_degrees: int = None):
converted_stimuli_id = f"{stimuli_identifier}--target{target_visual_degrees}--source{source_visual_degrees}"
source_visual_degrees = _determine_visual_degrees(source_visual_degrees,
stimulus_set)
target_dir = root_path / converted_stimuli_id
target_dir.mkdir(parents=True, exist_ok=False)
image_converter = ImageConverter(target_dir=target_dir)
converted_image_paths = {}
for image_id, image_degrees in tqdm(zip(stimulus_set['image_id'],
source_visual_degrees),
total=len(stimulus_set),
desc='convert image degrees'):
converted_image_path = image_converter.convert_image(
image_path=stimulus_set.get_image(image_id),
source_degrees=image_degrees,
target_degrees=target_visual_degrees)
converted_image_paths[image_id] = converted_image_path
converted_stimuli = StimulusSet(stimulus_set.copy(
deep=True)) # without copy, it will link to the previous stim set
converted_stimuli.image_paths = converted_image_paths
converted_stimuli.identifier = converted_stimuli_id
converted_stimuli['degrees'] = target_visual_degrees
converted_stimuli.original_paths = copy.deepcopy(stimulus_set.image_paths)
return converted_stimuli
def collect_stimuli(data_dir):
image_dir = data_dir / 'images' / 'things-2'
assert os.path.isdir(image_dir)
files = sorted(os.listdir(image_dir), key=lambda x: int(os.path.splitext(x)[0]))
files = files[:-130] # Discard last 130 images (5 grey and 25x5 normalizer images)
assert os.path.isdir(data_dir / 'image-metadata')
stimuli = pd.read_csv(data_dir / 'image-metadata' / 'things_2_metadata.csv')
stimuli = stimuli.rename(columns={'id': 'image_id'})
stimuli['image_current_local_file_path'] = stimuli.apply(
lambda row: os.path.join(image_dir, str(row.image_id) + '.jpg'), axis=1)
assert len(np.unique(stimuli['image_id'])) == len(stimuli)
stimuli = StimulusSet(stimuli)
stimuli.image_paths = \
{stimuli.at[idx, 'image_id']: stimuli.at[idx, 'image_current_local_file_path'] for idx in range(len(stimuli))}
return stimuli
def __init__(self):
stimulus_set = pd.read_csv(
os.path.join(os.path.dirname(__file__), 'imagenet2012.csv'))
stimulus_set = StimulusSet(stimulus_set)
stimulus_set.image_paths = {
row.image_id: row.filepath
for row in stimulus_set.itertuples()
}
self._stimulus_set = stimulus_set
self._similarity_metric = Accuracy()
ceiling = Score([1, np.nan],
coords={'aggregation': ['center', 'error']},
dims=['aggregation'])
super(Imagenet2012, self).__init__(
identifier='fei-fei.Deng2009-top1',
version=1,
ceiling_func=lambda: ceiling,
parent='ImageNet',
paper_link="https://ieeexplore.ieee.org/abstract/document/5206848")
def _dummy_stimulus_set():
stimulus_set = StimulusSet([
{
'image_id': 'a'
},
{
'image_id': 'b'
},
{
'image_id': 'c'
},
])
stimulus_set.image_paths = {
'a': 'a.png',
'b': 'b.png',
'c': 'c.png',
}
stimulus_set.identifier = 'dummy'
return stimulus_set
def load_assemblies(self, subset=None, fname='dataset.nc'):
if subset is None:
subset = np.arange(len(self.runs))
assemblies = [
xarray.open_dataarray(os.path.join(rd, fname))
for rd in self.runs[subset]
]
self.assemblies = []
loss_w = self.experiment_set[['xent', 'recon']]
for da, xent, recon in zip(assemblies, loss_w.xent[subset],
loss_w.recon[subset]):
stim_df = da.presentation.to_dataframe(
name='stimulus_set').reset_index().drop(
columns=['presentation'])
stim_set = StimulusSet(stim_df)
stim_set.name = "lg.xent{}.recon{}".format(xent, recon)
da = da.assign_attrs({'stimulus_set': stim_set})
da = DataAssembly(da)
da = da.assign_attrs({'xent': xent, 'recon': recon})
self.assemblies.append(da)
return self.assemblies
def test_creates_synset(self, model_ctr):
np.random.seed(0)
activations_model = model_ctr()
brain_model = ModelCommitment(
identifier=activations_model.identifier,
activations_model=activations_model,
layers=None,
behavioral_readout_layer='dummy') # not needed
stimuli = StimulusSet({
'image_id': ['1', '2'],
'filename': ['rgb1', 'rgb2']
})
stimuli.image_paths = {
'1': os.path.join(os.path.dirname(__file__), 'rgb1.jpg'),
'2': os.path.join(os.path.dirname(__file__), 'rgb2.jpg')
}
stimuli.identifier = 'test_logits_behavior.creates_synset'
brain_model.start_task(BrainModel.Task.label, 'imagenet')
behavior = brain_model.look_at(stimuli)
assert isinstance(behavior, BehavioralAssembly)
assert set(behavior['image_id'].values) == {'1', '2'}
assert len(behavior['synset']) == 2
assert behavior['synset'].values[0].startswith('n')
