def get_data(dataset, data_path, cutout_length, validation): """ Get torchvision dataset """ dataset = dataset.lower() if dataset == 'cifar10': dset_cls = dset.CIFAR10 n_classes = 10 elif dataset == 'cifar100': dset_cls = dset.CIFAR100 n_classes = 100 elif dataset == 'imagenet': n_classes = 200 elif dataset == 'mnist': dset_cls = dset.MNIST n_classes = 10 elif dataset == 'fashionmnist': dset_cls = dset.FashionMNIST n_classes = 10 else: raise ValueError(dataset) if dataset == "imagenet": trn_transform, val_transform = preproc.data_transforms( dataset, cutout_length) trn_data = dset.ImageFolder( root="/home/LAB/gaoch/asdf/data/imagenet_new/train", transform=trn_transform) input_channels = 3 input_size = trn_data[0][0].shape[1] ret = [input_size, input_channels, n_classes, trn_data] if validation: val_data = dset.ImageFolder(root=os.path.join( data_path, "/home/LAB/gaoch/asdf/data/imagenet_new/valid"), transform=val_transform) ret.append(val_data) return ret trn_transform, val_transform = preproc.data_transforms( dataset, cutout_length) trn_data = dset_cls(root=data_path, train=True, download=True, transform=trn_transform) # assuming shape is NHW or NHWC shape = trn_data.data.shape input_channels = 3 if len(shape) == 4 else 1 assert shape[1] == shape[2], "not expected shape = {}".format(shape) input_size = shape[1] ret = [input_size, input_channels, n_classes, trn_data] if validation: # append validation data ret.append( dset_cls(root=data_path, train=False, download=True, transform=val_transform)) return ret
def get_data(dataset, data_path, cutout_length, validation): """ Get torchvision dataset """ dataset = dataset.lower() if dataset == 'cifar10': dset_cls = dset.CIFAR10 n_classes = 10 elif dataset == 'mnist': dset_cls = dset.MNIST n_classes = 10 elif dataset == 'fashionmnist': dset_cls = dset.FashionMNIST n_classes = 10 else: raise ValueError(dataset) trn_transform, val_transform = preproc.data_transforms(dataset, cutout_length) trn_data = dset_cls(root=data_path, train=True, download=True, transform=trn_transform) # assuming shape is NHW or NHWC shape = trn_data.data.shape input_channels = 3 if len(shape) == 4 else 1 assert shape[1] == shape[2], "not expected shape = {}".format(shape) input_size = shape[1] ret = [input_size, input_channels, n_classes, trn_data] if validation: # append validation data ret.append(dset_cls(root=data_path, train=False, download=True, transform=val_transform)) return ret
def get_data(dataset, data_path,val1_data_path,val2_data_path, cutout_length, validation,validation2 = False,n_class = 3,image_size = 64): """ Get torchvision dataset """ dataset = dataset.lower() if dataset == 'cifar10': dset_cls = dset.CIFAR10 n_classes = 10 elif dataset == 'mnist': dset_cls = dset.MNIST n_classes = 10 elif dataset == 'fashionmnist': dset_cls = dset.FashionMNIST n_classes = 10 elif dataset == 'custom': dset_cls = dset.ImageFolder n_classes = n_class #2 to mama else: raise ValueError(dataset) trn_transform, val_transform = preproc.data_transforms(dataset, cutout_length,image_size) if dataset == 'custom': print("DATA PATH:", data_path) trn_data = dset_cls(root=data_path, transform=trn_transform) #dataset_loader = torch.utils.data.DataLoader(trn_data, # batch_size=16, shuffle=True, # num_workers=1) else: trn_data = dset_cls(root=data_path, train=True, download=True, transform=trn_transform) # assuming shape is NHW or NHWC if dataset == 'custom': shape = [1, image_size, image_size,3] else: shape = trn_data.train_data.shape print(shape) input_channels = 3 if len(shape) == 4 else 1 assert shape[1] == shape[2], "not expected shape = {}".format(shape) input_size = shape[1] print('input_size: uitls',input_size) ret = [input_size, input_channels, n_classes, trn_data] if validation: # append validation data if dataset == 'custom': dset_cls = dset.ImageFolder(val1_data_path,transform=val_transform) ret.append(dset_cls) else: ret.append(dset_cls(root=data_path, train=False, download=True, transform=val_transform)) if validation2: if dataset == 'custom': dset_cls = dset.ImageFolder(val2_data_path,transform=trn_transform) ret.append(dset_cls) return ret
def get_data(dataset, data_path, cutout_length, validation): opener = urllib.request.build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0')] urllib.request.install_opener(opener) """ Get torchvision dataset """ dataset = dataset.lower() if dataset == 'cifar10': dset_cls = dset.CIFAR10 n_classes = 10 elif dataset == 'mnist': dset_cls = dset.MNIST n_classes = 10 elif dataset == 'fashionmnist': dset_cls = dset.FashionMNIST n_classes = 10 elif dataset == 'toy': dset_cls = DsetMock n_classes = 2 else: raise ValueError(dataset) trn_transform, val_transform = preproc.data_transforms( dataset, cutout_length) trn_data = dset_cls(root=data_path, train=True, download=True, transform=trn_transform) # assuming shape is NHW or NHWC try: shape = trn_data.train_data.shape except: shape = trn_data.data.shape input_channels = 3 if len(shape) == 4 else 1 #assert d shape[1] == shape[2], "not expected shape = {}".format(shape) input_size = shape[1] ret = [input_size, input_channels, n_classes, trn_data] if validation: # append validation data ret.append( dset_cls(root=data_path, train=False, download=True, transform=val_transform)) return ret
def get_data(prop_mouse_data_to_use): train_general_transform, train_img_transform, norm_transform, val_transform = preproc.data_transforms() trn_data = NeuronDataset(train_samples, transform_norm=norm_transform, transform_img=train_img_transform, transform_both=train_general_transform) cut_point = round(prop_mouse_data_to_use * len(val_samples)) new_train_samples = train_samples + val_samples[:cut_point] new_val_samples = val_samples[cut_point:] val_data = NeuronDataset(new_val_samples, transform_norm=val_transform) trn_data = NeuronDataset(new_train_samples, transform_norm=norm_transform, transform_img=train_img_transform, transform_both=train_general_transform) #shape is HW or HWC shape = trn_data.shape input_channels = 3 if len(shape) == 3 else 1 assert shape[0] == shape[1], "not expected shape = {}".format(shape) input_size = shape[0] return [{'input_size': input_size, 'input_channels': input_channels, 'num_classes': 1}, trn_data, val_data]
def get_data(dataset, data_root, cutout_length, validation, autoaugment): """ Get torchvision dataset """ dataset = dataset.lower() data_path = data_root if dataset == 'cifar10': dset_cls = dset.CIFAR10 n_classes = 10 elif dataset == 'cifar100': dset_cls = dset.CIFAR100 n_classes = 100 elif dataset == 'mnist': dset_cls = dset.MNIST n_classes = 10 elif dataset == 'fashionmnist': dset_cls = dset.FashionMNIST n_classes = 10 #New Datasets elif dataset == 'mit67': dset_cls = dset.ImageFolder n_classes = 67 data_path = '%s/MIT67/train' % data_root # 'data/MIT67/train' val_path = '%s/MIT67/test' % data_root # 'data/MIT67/val' elif dataset == 'sport8': dset_cls = dset.ImageFolder n_classes = 8 data_path = '%s/Sport8/train' % data_root # 'data/Sport8/train' val_path = '%s/Sport8/test' % data_root # 'data/Sport8/val' elif dataset == 'caltech101': dset_cls = dset.ImageFolder n_classes = 101 data_path = '%s/Caltech101/train' % data_root # 'data/Caltech101/train' val_path = '%s/Caltech101/test' % data_root # 'data/Caltech101/val' else: raise ValueError(dataset) trn_transform, val_transform = preproc.data_transforms( dataset, cutout_length, autoaugment) if dataset in LARGE_DATASETS: trn_data = dset_cls(root=data_path, transform=trn_transform) shape = trn_data[0][0].unsqueeze(0).shape print(shape) assert shape[2] == shape[3], "not expected shape = {}".format(shape) input_size = shape[2] else: trn_data = dset_cls(root=data_path, train=True, download=True, transform=trn_transform) # assuming shape is NHW or NHWC try: shape = trn_data.data.shape except AttributeError: shape = trn_data.train_data.shape assert shape[1] == shape[2], "not expected shape = {}".format(shape) input_size = shape[1] input_channels = 3 if len(shape) == 4 else 1 # print("Number of input channels: ", input_channels) ret = [input_size, input_channels, n_classes, trn_data] if validation: # append validation data if dataset in LARGE_DATASETS: ret.append(dset_cls(root=val_path, transform=val_transform)) else: ret.append( dset_cls(root=data_path, train=False, download=True, transform=val_transform)) return ret