def package_Movshon_datasets(name):
assembly = load_assembly(name)
assembly.load()
base_assembly = assembly
_, unique_indices = np.unique(base_assembly['image_id'].values, return_index=True)
unique_indices = np.sort(unique_indices) # preserve order
image_ids = base_assembly['image_id'].values[unique_indices]
stratification_values = base_assembly['texture_type'].values[unique_indices]
rng = RandomState(seed=12)
splitter = StratifiedShuffleSplit(n_splits=1, train_size=.3, test_size=None, random_state=rng)
split = next(splitter.split(np.zeros(len(image_ids)), stratification_values))
access_indices = {assembly_type: image_indices
for assembly_type, image_indices in zip(['public', 'private'], split)}
for access in ['public', 'private']:
indices = access_indices[access]
subset_image_ids = image_ids[indices]
assembly = base_assembly[
{'presentation': [image_id in subset_image_ids for image_id in base_assembly['image_id'].values]}]
adapt_stimulus_set(assembly, access)
package_stimulus_set(assembly.attrs['stimulus_set'], stimulus_set_name=assembly.attrs['stimulus_set_name'])
del assembly.attrs['stimulus_set']
package_data_assembly(assembly, f"{name}.{access}", stimulus_set_name=assembly.attrs['stimulus_set_name'])
# not really sure if this is necessary
return assembly
def main(access):
local_data_path = fetch._local_data_path
name_root = 'movshon.FreemanZiemba2013'
stimulus_set_name_existing = name_root + "-" + access if access != "both" else name_root
stimulus_set_name_new = name_root + ".aperture-" + access if access != "both" else name_root + ".aperture"
data_assembly_name_existing = name_root + "." + access if access != "both" else name_root
data_assembly_name_new = name_root + ".aperture." + access if access != "both" else name_root + ".aperture"
temp_dir = os.path.join(local_data_path,
"temp_" + data_assembly_name_new.replace(".", "_"))
stimulus_set_existing = get_stimulus_set(stimulus_set_name_existing)
stimulus_set_new = convert_stimuli(stimulus_set_existing,
stimulus_set_name_new, temp_dir)
mapping = stimulus_set_new.id_mapping
_logger.debug(f"Packaging stimuli: {stimulus_set_new.name}")
package_stimulus_set(stimulus_set_new,
stimulus_set_name=stimulus_set_new.name,
bucket_name="brainio-contrib")
data_assembly_existing = get_assembly(data_assembly_name_existing)
proto_data_assembly_new = convert_assembly(data_assembly_existing,
data_assembly_name_new,
stimulus_set_new, mapping)
_logger.debug(f"Packaging assembly: {data_assembly_name_new}")
package_data_assembly(proto_data_assembly_new,
data_assembly_name_new,
stimulus_set_name_new,
bucket_name="brainio-contrib")
def main():
data_dir = Path(__file__).parent / 'coco'
stimuli = collect_stimuli(data_dir / 'stimuli')
stimuli.name = 'dicarlo.Kar2018cocogray'
assembly = load_responses(data_dir / 'cocoGray_neural.h5', stimuli)
assembly.name = 'dicarlo.Kar2018cocogray'
print("Packaging stimuli")
package_stimulus_set(stimuli, stimulus_set_name=stimuli.name,
bucket_name="brainio-dicarlo")
print("Packaging assembly")
package_data_assembly(assembly, data_assembly_name=assembly.name, stimulus_set_name=stimuli.name,
bucket_name="brainio-dicarlo")
def package_dicarlo_datasets(name):
base_assembly = load_assembly(name)
base_assembly.load()
base_assembly = _filter_erroneous_neuroids(base_assembly)
for variation_name, target_variation in {'public': [0, 3], 'private': [6]}.items():
assembly = base_assembly[
{'presentation': [variation in target_variation for variation in base_assembly['variation'].values]}]
assert hasattr(assembly, 'variation')
adapt_stimulus_set(assembly, name_suffix=variation_name)
package_stimulus_set(assembly.attrs['stimulus_set'], stimulus_set_name=assembly.attrs['stimulus_set_name'],
bucket_name="brainio-dicarlo")
del assembly.attrs['stimulus_set']
package_data_assembly(assembly, f'{name}.{variation_name}',assembly.attrs['stimulus_set_name'], bucket_name='brainio-dicarlo')
return assembly
def main():
data_dir = Path(__file__).parents[3] / 'Rajalingham2020'
assert os.path.isdir(data_dir)
stimuli = collect_stimuli(data_dir)
stimuli.name = 'dicarlo.Rajalingham2020orthographic_IT'
assembly = load_responses(data_dir, stimuli)
assembly.name = 'dicarlo.Rajalingham2020orthographic_IT'
print('Packaging stimuli')
package_stimulus_set(stimuli, stimulus_set_name=stimuli.name, bucket_name="brainio-dicarlo")
print('Packaging assembly')
package_data_assembly(assembly, data_assembly_name=assembly.name, stimulus_set_name=stimuli.name,
bucket_name="brainio-dicarlo")
return
def main():
data_dir = Path(__file__).parent / 'search_datasets'
data_path = data_dir / 'waldo'
# create stimuli
stimuli = collect_stimuli(data_path)
stimuli.name = 'klab.Zhang2018.search_waldo'
# create assembly for different subjects
assembly = collect_data(data_path, [*range(1, 16)])
assembly.name = 'klab.Zhang2018search_waldo'
# package
print("\nPackaging Stimuli ----------")
package_stimulus_set(stimuli, stimulus_set_name=stimuli.name)
print("\nPackaging Assembly ----------")
package_data_assembly(assembly, data_assembly_name=assembly.name, stimulus_set_name=stimuli.name)
def test_package_stimulus_set(transaction):
proto = prep_proto_stim()
stim_set_name = "dicarlo.test." + now()
test_bucket = "brainio-temp"
stim_model = package_stimulus_set(proto,
stimulus_set_name=stim_set_name,
bucket_name=test_bucket)
assert stim_model
assert stim_model.name == stim_set_name
stim_set_fetched = brainio_collection.get_stimulus_set(stim_set_name)
assert len(proto) == len(stim_set_fetched)
for image in proto.itertuples():
orig = proto.get_image(image.image_id)
fetched = stim_set_fetched.get_image(image.image_id)
assert os.path.basename(orig) == os.path.basename(fetched)
kf_orig = kf(orig)
kf_fetched = kf(fetched)
assert kf_orig.sha1 == kf_fetched.sha1
def package(assembly, stimuli):
print("Packaging stimuli")
package_stimulus_set(stimuli, stimulus_set_name=stimuli.name)
print("Packaging assembly")
package_data_assembly(assembly, data_assembly_name=assembly.name, stimulus_set_name=stimuli.name)