def create_dataloaders_rand(file, seed, total_im, n_selected, batch_size, norm=True, cuda=True): dat = StaticImageSet(file, 'images', 'responses') #idx = (dat.neurons.area == 'V1') & (dat.neurons.layer == 'L2/3') dat.transforms = [ToTensor(cuda=cuda)] if norm: dat.transforms.append(Normalized(np.where(dat.tiers == 'train')[0], dat.responses, cuda=cuda)) np.random.seed(seed) selected_indexes = np.random.choice(np.where(dat.tiers == 'train')[0], size=n_selected, replace=False) selected_set = Subset(dat, selected_indexes) train_loader = DataLoader(selected_set, batch_size=batch_size) train_loader.img_shape = dat.img_shape train_loader.n_neurons = dat.n_neurons train_loader.transformed_mean = torch.tensor(dat[()].responses.mean(axis=0)).cuda() #dat.transformed_mean() val_loader = DataLoader(dat, sampler=SubsetRandomSampler(np.where(dat.tiers == 'validation')[0]), batch_size=batch_size) val_loader.img_shape = dat.img_shape val_loader.n_neurons = dat.n_neurons test_loader = DataLoader(dat, sampler=SubsetRandomSampler(np.where(dat.tiers == 'test')[0]), batch_size=batch_size) test_loader.img_shape = dat.img_shape test_loader.n_neurons = dat.n_neurons loaders = {'train': train_loader, 'val': val_loader, 'test': test_loader} return loaders
def create_dataloaders_al(file='', seed=0, selected_idx=set([]), batch_size=64, norm=False, cuda=False): np.random.seed(seed) dat = StaticImageSet(file, 'images', 'responses') idx = (dat.neurons.area == 'V1') & (dat.neurons.layer == 'L2/3') dat.transforms = [Subsample(np.where(idx)[0]), ToTensor(cuda=True)] if norm: dat.transforms.append(Normalized(np.where(dat.tiers == 'train')[0], dat.responses, cuda=cuda)) selected_set = Subset(dat, selected_idx) train_loader = DataLoader(selected_set, batch_size=batch_size) train_loader.img_shape = dat.img_shape train_loader.n_neurons = dat.n_neurons _, train_loader.transformed_mean = dat.transformed_mean() val_loader = DataLoader(dat, sampler=SubsetRandomSampler(np.where(dat.tiers == 'validation')[0]), batch_size=batch_size) val_loader.img_shape = dat.img_shape val_loader.n_neurons = dat.n_neurons test_loader = DataLoader(dat, sampler=SubsetRandomSampler(np.where(dat.tiers == 'test')[0]), batch_size=batch_size) test_loader.img_shape = dat.img_shape test_loader.n_neurons = dat.n_neurons loaders = {'train': train_loader, 'val': val_loader, 'test': test_loader} return loaders
def create_dataloaders_synth(file, seed, selected_idx, batch_size, norm=True, cuda=True): dat = StaticImageSet(file, 'images', 'responses') dat.transforms = [ToTensor(cuda=cuda)] if norm: dat.transforms.append(Normalized(np.where(dat.tiers == 'train')[0], dat.responses, cuda=cuda)) np.random.seed(seed) selected_set = Subset(dat, selected_idx) train_loader = DataLoader(selected_set, batch_size=batch_size) train_loader.img_shape = dat.img_shape train_loader.n_neurons = dat.n_neurons train_loader.img_shape = dat.img_shape train_loader.n_neurons = dat.n_neurons train_loader.transformed_mean = torch.as_tensor(dat[()].responses.mean(axis=0)).cuda() val_loader = DataLoader(dat, sampler=SubsetRandomSampler(np.where(dat.tiers == 'validation')[0]), batch_size=batch_size) val_loader.img_shape = dat.img_shape val_loader.n_neurons = dat.n_neurons test_loader = DataLoader(dat, sampler=SubsetRandomSampler(np.where(dat.tiers == 'test')[0]), batch_size=batch_size) test_loader.img_shape = dat.img_shape test_loader.n_neurons = dat.n_neurons loaders = {'train': train_loader, 'val': val_loader, 'test': test_loader} return loaders
def main(): file = h5py.File('/notebooks/data/static20892-3-14-preproc0.h5', "r") dat = StaticImageSet('/notebooks/data/static20892-3-14-preproc0.h5', 'images', 'responses') img_shape = dat.img_shape[2:] gabor_rf = gen_gabor_RF(img_shape, rf_shape, n=n_neurons, seed=random_seed) images = dat[()].images images = images.reshape(6000, 36, 64) firing_rates = compute_activity_simple(images / 255, gabor_rf) dist = Poisson(torch.tensor(firing_rates)) responses = dist.sample().numpy() data_file = h5py.File("toy_dataset.hdf5", "w") im_set = data_file.create_dataset('images', data=dat[()].images) response_set = data_file.create_dataset('responses', data=responses) tier_set = data_file.create_dataset('tiers', data=file['tiers']) data_file.close()
def load_obj(file): with open('./data/' + file + '.pkl', 'rb') as f: return pickle.load(f) Fabrikant().insert1(dict(fabrikant_name='Matthias Depoortere', email="*****@*****.**", affiliation='sinzlab', dj_username="******"), skip_duplicates=True) Seed().insert([{'seed': 13}], skip_duplicates=True) dat = StaticImageSet('/notebooks/toy_data/toy_dataset.hdf5', 'images', 'responses') TOTAL_IM = np.where(dat.tiers == 'train')[0].size model_config = load_obj('best_model_config') model_config['random_seed'] = 5 model_entry = dict(model_fn="nnsetup.models.create_model", model_config=model_config, model_fabrikant="Matthias Depoortere", model_comment="Best model hp on full dataset") Model().add_entry(**model_entry) trainer_config = load_obj('best_train_config') trainer_entry = dict(trainer_fn="nnsetup.trainer.train_model", trainer_config=trainer_config, trainer_fabrikant="Matthias Depoortere", trainer_comment="best trainer on full dataset")
def load_obj(file): with open('./data/' + file + '.pkl', 'rb') as f: return pickle.load(f) def calc_var(preds): vars = np.stack(preds, axis=0).std(axis=0).mean(axis=0) return vars aa = dict(architect_name="matthias Depoortere", email="depoortere.matthias@gmail;com", affiliation="sinzlab", dj_username="******") Fabrikant().insert1(aa, skip_duplicates=True) dat = StaticImageSet('/notebooks/data/static20892-3-14-preproc0.h5', 'images', 'responses') Seed().insert([{'seed': 13}], skip_duplicates=True) my_dat = LabeledImageSet('/notebooks/data/static20892-3-14-preproc0.h5', 'images', 'responses') idx = (my_dat.neurons.area == 'V1') & (my_dat.neurons.layer == 'L2/3') my_dat.transforms = [ Subsample(np.where(idx)[0]), ToTensor(cuda=True), Normalized(np.where(dat.tiers == 'train')[0], dat.responses, cuda=True) ] TOTAL_IM = np.where(dat.tiers == 'train')[0].size MAX_IM = TOTAL_IM N_AQUIRE = 50 n_im = 500
from nnfabrik.main import * from nnsetup.al_tools import load_latest_model def load_obj(file): with open('./data/' + file + '.pkl', 'rb') as f: return pickle.load(f) aa = dict(architect_name="matthias Depoortere", email="depoortere.matthias@gmail;com", affiliation="sinzlab", dj_username="******") Fabrikant().insert1(aa, skip_duplicates=True) dat = StaticImageSet('/notebooks/data/static20892-3-14-preproc0.h5', 'images', 'responses') Seed().insert([{'seed': 13}], skip_duplicates=True) my_dat = StaticImageSet('/notebooks/data/static20892-3-14-preproc0.h5', 'images', 'responses') idx = (my_dat.neurons.area == 'V1') & (my_dat.neurons.layer == 'L2/3') my_dat.transforms = [ Subsample(np.where(idx)[0]), ToTensor(cuda=True), Normalized(np.where(dat.tiers == 'train')[0], dat.responses, cuda=True) ] test_images = my_dat[()].images[np.where(my_dat.tiers == "test")] model_hashes = [] for i in range(8):