def main(access):
local_data_path = fetch._local_data_path
name_root = 'movshon.FreemanZiemba2013'
stimulus_set_name_existing = name_root + "-" + access if access != "both" else name_root
stimulus_set_name_new = name_root + ".aperture-" + access if access != "both" else name_root + ".aperture"
data_assembly_name_existing = name_root + "." + access if access != "both" else name_root
data_assembly_name_new = name_root + ".aperture." + access if access != "both" else name_root + ".aperture"
temp_dir = os.path.join(local_data_path,
"temp_" + data_assembly_name_new.replace(".", "_"))
stimulus_set_existing = get_stimulus_set(stimulus_set_name_existing)
stimulus_set_new = convert_stimuli(stimulus_set_existing,
stimulus_set_name_new, temp_dir)
mapping = stimulus_set_new.id_mapping
_logger.debug(f"Packaging stimuli: {stimulus_set_new.name}")
package_stimulus_set(stimulus_set_new,
stimulus_set_name=stimulus_set_new.name,
bucket_name="brainio-contrib")
data_assembly_existing = get_assembly(data_assembly_name_existing)
proto_data_assembly_new = convert_assembly(data_assembly_existing,
data_assembly_name_new,
stimulus_set_new, mapping)
_logger.debug(f"Packaging assembly: {data_assembly_name_new}")
package_data_assembly(proto_data_assembly_new,
data_assembly_name_new,
stimulus_set_name_new,
bucket_name="brainio-contrib")
def test_get_stimulus_set():
stimulus_set = brainio_collection.get_stimulus_set("dicarlo.hvm-public")
assert "image_id" in stimulus_set.columns
assert stimulus_set.shape == (3200, 17)
assert stimulus_set.name == 'dicarlo.hvm-public'
for image_id in stimulus_set['image_id']:
image_path = stimulus_set.get_image(image_id)
assert os.path.exists(image_path)
def test_dicarlohvm(self):
stimulus_set = brainio_collection.get_stimulus_set(
identifier="dicarlo.hvm-public")
paths = stimulus_set.image_paths.values()
for path in paths:
image = imageio.imread(path)
assert isinstance(image, np.ndarray)
assert image.size > 0
def test_klab_Zhang2018search():
stimulus_set = brainio_collection.get_stimulus_set(
'klab.Zhang2018.search_obj_array')
# There are 300 presentation images in the assembly but 606 in the StimulusSet (explanation from @shashikg follows).
# For each of the visual search task out of total 300, you need two images (one - the target image,
# second - the search space image) plus there are 6 different mask images to mask objects
# present at 6 different locations in a specified search image.
# Therefore, a total of 300 * 2 + 6 images are there in the stimulus set.
assert len(stimulus_set) == 606
assert len(set(stimulus_set['image_id'])) == 606
def main():
data_dir = Path(__file__).parents[6] / 'data2' / 'active' / 'users' / 'sachis' / 'database'
assert os.path.isdir(data_dir)
stimuli = brainio_collection.get_stimulus_set('dicarlo.hvm')
assembly = load_responses(data_dir, stimuli)
assembly.name = 'dicarlo.Sanghavi2020'
print(assembly)
print('Packaging assembly')
package_data_assembly(assembly, data_assembly_name=assembly.name, stimulus_set_name=stimuli.name,
bucket_name="brainio-dicarlo")
return
def repackage_stimulus_sets():
for identifier in stimuli.list_stimulus_sets():
stimulus_set = brainio_collection.get_stimulus_set(identifier)
# re-assign bucket
stimulus_set_model = stimuli.StimulusSetModel.get(
stimuli.StimulusSetModel.name == identifier)
location = stimulus_set_model.stimulus_set_image_maps[
0].image.image_image_store_maps[0].image_store.location
bucket = 'brainio-dicarlo' if 'brainio-dicarlo' in location else 'brainio-contrib'
bucket = bucket.replace('-', '.')
# package
package_stimulus_set(stimulus_set,
stimulus_set_identifier=stimulus_set.name,
bucket_name=bucket)
def main():
metric_bins_path = Path("/braintree/home/darren/work/metric_bins.npy")
csv_path = Path(
__file__).parents[2] / "notebooks" / "2020-11-22_hvm_from_dldata.csv"
stimuli = brainio_collection.get_stimulus_set('dicarlo.hvm')
assembly = load_responses(metric_bins_path, csv_path)
assembly.name = 'dicarlo.Seibert2019'
print('Packaging assembly')
package_data_assembly(assembly,
assembly_identifier=assembly.name,
stimulus_set_identifier=stimuli.identifier,
bucket_name='brainio.dicarlo')
def test_get_stimulus_set():
stimulus_set = brainio_collection.get_stimulus_set("dicarlo.hvm-public")
assert "image_id" in stimulus_set.columns
assert set(stimulus_set.columns).issuperset({
'image_id', 'object_name', 'variation', 'category_name',
'image_file_name', 'background_id', 'ty', 'tz', 'size', 'id', 's',
'rxz', 'ryz', 'ryz_semantic', 'rxy', 'rxy_semantic', 'rxz_semantic'
})
assert len(stimulus_set) == 3200
assert stimulus_set.identifier == 'dicarlo.hvm-public'
for image_id in stimulus_set['image_id']:
image_path = stimulus_set.get_image(image_id)
assert os.path.exists(image_path)
extension = os.path.splitext(image_path)[1]
assert extension in ['.png', '.PNG', '.jpg', '.jpeg', '.JPG', '.JPEG']
def test_repeated_path(self):
stimulus_set = brainio_collection.get_stimulus_set(name="dicarlo.hvm")
stimulus_set = stimulus_set.loc[[0]] # check only first image
stimulus_set['degrees'] = 8
model_pixels = 224
pixels_to_degrees1 = PixelsToDegrees(target_pixels=model_pixels)
converted_stimuli1 = pixels_to_degrees1(stimulus_set)
image_path1 = converted_stimuli1.get_image(
converted_stimuli1['image_id'].values[0])
pixels_to_degrees2 = PixelsToDegrees(target_pixels=model_pixels)
converted_stimuli2 = pixels_to_degrees2(stimulus_set)
image_path2 = converted_stimuli2.get_image(
converted_stimuli2['image_id'].values[0])
assert image_path1 == image_path2
def test_image_centered(self):
stimulus_set = brainio_collection.get_stimulus_set(name="dicarlo.hvm")
stimulus_set = stimulus_set.loc[[0]] # check only first image
stimulus_set['degrees'] = 8
model_pixels = 224
pixels_to_degrees = PixelsToDegrees(target_pixels=model_pixels)
converted_stimuli = pixels_to_degrees(stimulus_set)
image_path = converted_stimuli.get_image(
converted_stimuli['image_id'].values[0])
image = scipy.misc.imread(image_path)
gray = [128, 128, 128]
assert (image[48, 48] == gray).all()
assert (image[224 - 48, 224 - 48] == gray).all()
assert (image[48, 224 - 48] == gray).all()
assert (image[224 - 48, 48] == gray).all()
def test_shape(self):
stimulus_set = brainio_collection.get_stimulus_set(name="dicarlo.hvm")
stimulus_set['degrees'] = 8
model_pixels = 224
pixels_to_degrees = PixelsToDegrees(target_pixels=model_pixels)
converted_stimuli = pixels_to_degrees(stimulus_set)
non_degree_columns = list(set(stimulus_set.columns) - {'degrees'})
assert len(converted_stimuli) == len(stimulus_set)
assert converted_stimuli[non_degree_columns].equals(
stimulus_set[non_degree_columns]) # equal metadata
assert (converted_stimuli['degrees'] == 10).all()
for image_id in converted_stimuli['image_id']:
image_path = converted_stimuli.get_image(image_id)
image = scipy.misc.imread(image_path)
np.testing.assert_array_equal(image.shape, [224, 224, 3])
def test_package_stimulus_set(transaction):
proto = prep_proto_stim()
stim_set_name = "dicarlo.test." + now()
test_bucket = "brainio-temp"
stim_model = package_stimulus_set(proto,
stimulus_set_name=stim_set_name,
bucket_name=test_bucket)
assert stim_model
assert stim_model.name == stim_set_name
stim_set_fetched = brainio_collection.get_stimulus_set(stim_set_name)
assert len(proto) == len(stim_set_fetched)
for image in proto.itertuples():
orig = proto.get_image(image.image_id)
fetched = stim_set_fetched.get_image(image.image_id)
assert os.path.basename(orig) == os.path.basename(fetched)
kf_orig = kf(orig)
kf_fetched = kf(fetched)
assert kf_orig.sha1 == kf_fetched.sha1
def main():
stimuli = brainio_collection.get_stimulus_set('dicarlo.Rust2012')
single_nc_path = Path("/Users/jjpr/dev/dldata/scripts/rust_single.nc")
da_single = xr.open_dataarray(single_nc_path)
da_single.name = 'dicarlo.Rust2012.single'
print('Packaging assembly for single-unit')
package_data_assembly(da_single, assembly_identifier=da_single.name, stimulus_set_identifier=stimuli.identifier,
bucket_name='brainio.dicarlo')
array_nc_path = Path("/Users/jjpr/dev/dldata/scripts/rust_array.nc")
da_array = xr.open_dataarray(array_nc_path)
da_array.name = 'dicarlo.Rust2012.array'
print('Packaging assembly for array')
package_data_assembly(da_array, assembly_identifier=da_array.name, stimulus_set_identifier=stimuli.identifier,
bucket_name='brainio.dicarlo')
def repackage_stimulus_sets():
for identifier in stimuli.list_stimulus_sets():
stimulus_set = brainio_collection.get_stimulus_set(identifier)
if identifier.startswith('movshon') or identifier.startswith(
'dicarlo.hvm'):
filenames = stimulus_set['image_file_name'].apply(
lambda s: s.replace('.png', ''))
stimulus_set['image_path_within_store'] = filenames
# re-assign bucket
stimulus_set_model = stimuli.StimulusSetModel.get(
stimuli.StimulusSetModel.name == identifier)
location = stimulus_set_model.stimulus_set_image_maps[
0].image.image_image_store_maps[0].image_store.location
bucket = 'brainio-dicarlo' if 'brainio-dicarlo' in location else 'brainio-contrib'
bucket = bucket.replace('-', '.')
# package
package_stimulus_set(stimulus_set,
stimulus_set_identifier=stimulus_set.name,
bucket_name=bucket)
def test_gray_background(self):
stimulus_set = brainio_collection.get_stimulus_set(name="dicarlo.hvm")
stimulus_set = stimulus_set.loc[[0]] # check only first image
stimulus_set['degrees'] = 8
model_pixels = 224
pixels_to_degrees = PixelsToDegrees(target_pixels=model_pixels)
converted_stimuli = pixels_to_degrees(stimulus_set)
image_path = converted_stimuli.get_image(
converted_stimuli['image_id'].values[0])
image = scipy.misc.imread(image_path)
amount_gray = 0
for index in np.ndindex(image.shape[:2]):
color = image[index]
if (color == [128, 128, 128]).all():
amount_gray += 1
assert amount_gray / image.size == approx(.172041, abs=.0001)
assert amount_gray == 25897
def main():
data_dir = Path(
__file__).parents[6] / 'data2' / 'active' / 'users' / 'sachis'
assert os.path.isdir(data_dir)
import brainio_collection
print(brainio_collection.list_stimulus_sets())
print(brainio_collection.list_assemblies())
stimuli = brainio_collection.get_stimulus_set(
'dicarlo.objectome.public').sort_values(by='image_id')
print(stimuli)
# print(stimuli.columns)
stimuli = collect_stimuli(data_dir)
# stimuli.name = 'dicarlo.Rust2012'
# assembly = load_responses(data_dir, stimuli)
# assembly.name = 'dicarlo.SanghaviKar2020'
# print('Packaging stimuli')
# package_stimulus_set(stimuli, stimulus_set_name=stimuli.name, bucket_name='brainio-dicarlo')
# print('Packaging assembly')
# package_data_assembly(assembly, data_assembly_name=assembly.name, stimulus_set_name=stimuli.name,
# bucket_name='brainio-dicarlo')
return
def test_loadname_dicarlo_hvm():
assert brainio_collection.get_stimulus_set(
identifier="dicarlo.hvm-public") is not None
def test_feifei_Deng2009():
stimulus_set = brainio_collection.get_stimulus_set('fei-fei.Deng2009')
assert len(stimulus_set) == 50_000
assert len(set(stimulus_set['label'])) == 1_000
def test_noise(self):
stimulus_set = brainio_collection.get_stimulus_set(
'dietterich.Hendrycks2019.noise')
assert len(stimulus_set) == 3 * 5 * 50000
assert len(set(stimulus_set['synset'])) == 1000
def test_num_stimuli(self, identifier, num_stimuli):
stimulus_set = brainio_collection.get_stimulus_set(identifier)
assert len(stimulus_set) == num_stimuli
def test_digital(self):
stimulus_set = brainio_collection.get_stimulus_set(
'dietterich.Hendrycks2019.digital')
assert len(stimulus_set) == 4 * 5 * 50000
assert len(set(stimulus_set['synset'])) == 1000
