def main(access): local_data_path = fetch._local_data_path name_root = 'movshon.FreemanZiemba2013' stimulus_set_name_existing = name_root + "-" + access if access != "both" else name_root stimulus_set_name_new = name_root + ".aperture-" + access if access != "both" else name_root + ".aperture" data_assembly_name_existing = name_root + "." + access if access != "both" else name_root data_assembly_name_new = name_root + ".aperture." + access if access != "both" else name_root + ".aperture" temp_dir = os.path.join(local_data_path, "temp_" + data_assembly_name_new.replace(".", "_")) stimulus_set_existing = get_stimulus_set(stimulus_set_name_existing) stimulus_set_new = convert_stimuli(stimulus_set_existing, stimulus_set_name_new, temp_dir) mapping = stimulus_set_new.id_mapping _logger.debug(f"Packaging stimuli: {stimulus_set_new.name}") package_stimulus_set(stimulus_set_new, stimulus_set_name=stimulus_set_new.name, bucket_name="brainio-contrib") data_assembly_existing = get_assembly(data_assembly_name_existing) proto_data_assembly_new = convert_assembly(data_assembly_existing, data_assembly_name_new, stimulus_set_new, mapping) _logger.debug(f"Packaging assembly: {data_assembly_name_new}") package_data_assembly(proto_data_assembly_new, data_assembly_name_new, stimulus_set_name_new, bucket_name="brainio-contrib")
def test_get_stimulus_set(): stimulus_set = brainio_collection.get_stimulus_set("dicarlo.hvm-public") assert "image_id" in stimulus_set.columns assert stimulus_set.shape == (3200, 17) assert stimulus_set.name == 'dicarlo.hvm-public' for image_id in stimulus_set['image_id']: image_path = stimulus_set.get_image(image_id) assert os.path.exists(image_path)
def test_dicarlohvm(self): stimulus_set = brainio_collection.get_stimulus_set( identifier="dicarlo.hvm-public") paths = stimulus_set.image_paths.values() for path in paths: image = imageio.imread(path) assert isinstance(image, np.ndarray) assert image.size > 0
def test_klab_Zhang2018search(): stimulus_set = brainio_collection.get_stimulus_set( 'klab.Zhang2018.search_obj_array') # There are 300 presentation images in the assembly but 606 in the StimulusSet (explanation from @shashikg follows). # For each of the visual search task out of total 300, you need two images (one - the target image, # second - the search space image) plus there are 6 different mask images to mask objects # present at 6 different locations in a specified search image. # Therefore, a total of 300 * 2 + 6 images are there in the stimulus set. assert len(stimulus_set) == 606 assert len(set(stimulus_set['image_id'])) == 606
def main(): data_dir = Path(__file__).parents[6] / 'data2' / 'active' / 'users' / 'sachis' / 'database' assert os.path.isdir(data_dir) stimuli = brainio_collection.get_stimulus_set('dicarlo.hvm') assembly = load_responses(data_dir, stimuli) assembly.name = 'dicarlo.Sanghavi2020' print(assembly) print('Packaging assembly') package_data_assembly(assembly, data_assembly_name=assembly.name, stimulus_set_name=stimuli.name, bucket_name="brainio-dicarlo") return
def repackage_stimulus_sets(): for identifier in stimuli.list_stimulus_sets(): stimulus_set = brainio_collection.get_stimulus_set(identifier) # re-assign bucket stimulus_set_model = stimuli.StimulusSetModel.get( stimuli.StimulusSetModel.name == identifier) location = stimulus_set_model.stimulus_set_image_maps[ 0].image.image_image_store_maps[0].image_store.location bucket = 'brainio-dicarlo' if 'brainio-dicarlo' in location else 'brainio-contrib' bucket = bucket.replace('-', '.') # package package_stimulus_set(stimulus_set, stimulus_set_identifier=stimulus_set.name, bucket_name=bucket)
def main(): metric_bins_path = Path("/braintree/home/darren/work/metric_bins.npy") csv_path = Path( __file__).parents[2] / "notebooks" / "2020-11-22_hvm_from_dldata.csv" stimuli = brainio_collection.get_stimulus_set('dicarlo.hvm') assembly = load_responses(metric_bins_path, csv_path) assembly.name = 'dicarlo.Seibert2019' print('Packaging assembly') package_data_assembly(assembly, assembly_identifier=assembly.name, stimulus_set_identifier=stimuli.identifier, bucket_name='brainio.dicarlo')
def test_get_stimulus_set(): stimulus_set = brainio_collection.get_stimulus_set("dicarlo.hvm-public") assert "image_id" in stimulus_set.columns assert set(stimulus_set.columns).issuperset({ 'image_id', 'object_name', 'variation', 'category_name', 'image_file_name', 'background_id', 'ty', 'tz', 'size', 'id', 's', 'rxz', 'ryz', 'ryz_semantic', 'rxy', 'rxy_semantic', 'rxz_semantic' }) assert len(stimulus_set) == 3200 assert stimulus_set.identifier == 'dicarlo.hvm-public' for image_id in stimulus_set['image_id']: image_path = stimulus_set.get_image(image_id) assert os.path.exists(image_path) extension = os.path.splitext(image_path)[1] assert extension in ['.png', '.PNG', '.jpg', '.jpeg', '.JPG', '.JPEG']
def test_repeated_path(self): stimulus_set = brainio_collection.get_stimulus_set(name="dicarlo.hvm") stimulus_set = stimulus_set.loc[[0]] # check only first image stimulus_set['degrees'] = 8 model_pixels = 224 pixels_to_degrees1 = PixelsToDegrees(target_pixels=model_pixels) converted_stimuli1 = pixels_to_degrees1(stimulus_set) image_path1 = converted_stimuli1.get_image( converted_stimuli1['image_id'].values[0]) pixels_to_degrees2 = PixelsToDegrees(target_pixels=model_pixels) converted_stimuli2 = pixels_to_degrees2(stimulus_set) image_path2 = converted_stimuli2.get_image( converted_stimuli2['image_id'].values[0]) assert image_path1 == image_path2
def test_image_centered(self): stimulus_set = brainio_collection.get_stimulus_set(name="dicarlo.hvm") stimulus_set = stimulus_set.loc[[0]] # check only first image stimulus_set['degrees'] = 8 model_pixels = 224 pixels_to_degrees = PixelsToDegrees(target_pixels=model_pixels) converted_stimuli = pixels_to_degrees(stimulus_set) image_path = converted_stimuli.get_image( converted_stimuli['image_id'].values[0]) image = scipy.misc.imread(image_path) gray = [128, 128, 128] assert (image[48, 48] == gray).all() assert (image[224 - 48, 224 - 48] == gray).all() assert (image[48, 224 - 48] == gray).all() assert (image[224 - 48, 48] == gray).all()
def test_shape(self): stimulus_set = brainio_collection.get_stimulus_set(name="dicarlo.hvm") stimulus_set['degrees'] = 8 model_pixels = 224 pixels_to_degrees = PixelsToDegrees(target_pixels=model_pixels) converted_stimuli = pixels_to_degrees(stimulus_set) non_degree_columns = list(set(stimulus_set.columns) - {'degrees'}) assert len(converted_stimuli) == len(stimulus_set) assert converted_stimuli[non_degree_columns].equals( stimulus_set[non_degree_columns]) # equal metadata assert (converted_stimuli['degrees'] == 10).all() for image_id in converted_stimuli['image_id']: image_path = converted_stimuli.get_image(image_id) image = scipy.misc.imread(image_path) np.testing.assert_array_equal(image.shape, [224, 224, 3])
def test_package_stimulus_set(transaction): proto = prep_proto_stim() stim_set_name = "dicarlo.test." + now() test_bucket = "brainio-temp" stim_model = package_stimulus_set(proto, stimulus_set_name=stim_set_name, bucket_name=test_bucket) assert stim_model assert stim_model.name == stim_set_name stim_set_fetched = brainio_collection.get_stimulus_set(stim_set_name) assert len(proto) == len(stim_set_fetched) for image in proto.itertuples(): orig = proto.get_image(image.image_id) fetched = stim_set_fetched.get_image(image.image_id) assert os.path.basename(orig) == os.path.basename(fetched) kf_orig = kf(orig) kf_fetched = kf(fetched) assert kf_orig.sha1 == kf_fetched.sha1
def main(): stimuli = brainio_collection.get_stimulus_set('dicarlo.Rust2012') single_nc_path = Path("/Users/jjpr/dev/dldata/scripts/rust_single.nc") da_single = xr.open_dataarray(single_nc_path) da_single.name = 'dicarlo.Rust2012.single' print('Packaging assembly for single-unit') package_data_assembly(da_single, assembly_identifier=da_single.name, stimulus_set_identifier=stimuli.identifier, bucket_name='brainio.dicarlo') array_nc_path = Path("/Users/jjpr/dev/dldata/scripts/rust_array.nc") da_array = xr.open_dataarray(array_nc_path) da_array.name = 'dicarlo.Rust2012.array' print('Packaging assembly for array') package_data_assembly(da_array, assembly_identifier=da_array.name, stimulus_set_identifier=stimuli.identifier, bucket_name='brainio.dicarlo')
def repackage_stimulus_sets(): for identifier in stimuli.list_stimulus_sets(): stimulus_set = brainio_collection.get_stimulus_set(identifier) if identifier.startswith('movshon') or identifier.startswith( 'dicarlo.hvm'): filenames = stimulus_set['image_file_name'].apply( lambda s: s.replace('.png', '')) stimulus_set['image_path_within_store'] = filenames # re-assign bucket stimulus_set_model = stimuli.StimulusSetModel.get( stimuli.StimulusSetModel.name == identifier) location = stimulus_set_model.stimulus_set_image_maps[ 0].image.image_image_store_maps[0].image_store.location bucket = 'brainio-dicarlo' if 'brainio-dicarlo' in location else 'brainio-contrib' bucket = bucket.replace('-', '.') # package package_stimulus_set(stimulus_set, stimulus_set_identifier=stimulus_set.name, bucket_name=bucket)
def test_gray_background(self): stimulus_set = brainio_collection.get_stimulus_set(name="dicarlo.hvm") stimulus_set = stimulus_set.loc[[0]] # check only first image stimulus_set['degrees'] = 8 model_pixels = 224 pixels_to_degrees = PixelsToDegrees(target_pixels=model_pixels) converted_stimuli = pixels_to_degrees(stimulus_set) image_path = converted_stimuli.get_image( converted_stimuli['image_id'].values[0]) image = scipy.misc.imread(image_path) amount_gray = 0 for index in np.ndindex(image.shape[:2]): color = image[index] if (color == [128, 128, 128]).all(): amount_gray += 1 assert amount_gray / image.size == approx(.172041, abs=.0001) assert amount_gray == 25897
def main(): data_dir = Path( __file__).parents[6] / 'data2' / 'active' / 'users' / 'sachis' assert os.path.isdir(data_dir) import brainio_collection print(brainio_collection.list_stimulus_sets()) print(brainio_collection.list_assemblies()) stimuli = brainio_collection.get_stimulus_set( 'dicarlo.objectome.public').sort_values(by='image_id') print(stimuli) # print(stimuli.columns) stimuli = collect_stimuli(data_dir) # stimuli.name = 'dicarlo.Rust2012' # assembly = load_responses(data_dir, stimuli) # assembly.name = 'dicarlo.SanghaviKar2020' # print('Packaging stimuli') # package_stimulus_set(stimuli, stimulus_set_name=stimuli.name, bucket_name='brainio-dicarlo') # print('Packaging assembly') # package_data_assembly(assembly, data_assembly_name=assembly.name, stimulus_set_name=stimuli.name, # bucket_name='brainio-dicarlo') return
def test_loadname_dicarlo_hvm(): assert brainio_collection.get_stimulus_set( identifier="dicarlo.hvm-public") is not None
def test_feifei_Deng2009(): stimulus_set = brainio_collection.get_stimulus_set('fei-fei.Deng2009') assert len(stimulus_set) == 50_000 assert len(set(stimulus_set['label'])) == 1_000
def test_noise(self): stimulus_set = brainio_collection.get_stimulus_set( 'dietterich.Hendrycks2019.noise') assert len(stimulus_set) == 3 * 5 * 50000 assert len(set(stimulus_set['synset'])) == 1000
def test_num_stimuli(self, identifier, num_stimuli): stimulus_set = brainio_collection.get_stimulus_set(identifier) assert len(stimulus_set) == num_stimuli
def test_digital(self): stimulus_set = brainio_collection.get_stimulus_set( 'dietterich.Hendrycks2019.digital') assert len(stimulus_set) == 4 * 5 * 50000 assert len(set(stimulus_set['synset'])) == 1000