def get_video_dataloader(mode='train', videos_path=os.environ['HOME'] + '/Database/MSR-VTT/train-video/', vocab_path='data/processed/msrvtt_vocab.pkl', captions_path='data/processed/msrvtt_captions.csv', batch_size=32, num_frames=40, max_len=30, embedding_size=2048, num_captions=20, load_features=False, load_captions=False, preload=False, model='resnet152', num_workers=0): """ Generate a dataloader with the specified parameters. Args: mode: Dataset type to load videos_path: Path to MSR-VTT videos dataset vocab_path: Path to MSR-VTT vocab file caption_size: Path to captions vocab file batch_size: Batch size for Dataloader num_frames: Number of frames per video to process max_len: Max caption length embedding_size: Size of image embedding num_captions: Number of captions per image in dataset load_features: Boolean for creating or loading image features load_captions: Boolean for creating or loading image captions preload: Boolean for either preloading data into RAM during construction model: base model for encoderCNN num_workers: Dataloader parameter Return: data_loader: A torch dataloader for the MSR-VTT dataset """ # Ensure specified mode is validate try: assert mode in ['train', 'dev', 'test'] except AssertionError: print('Invalid mode specified: {}'.format(mode)) print(' Defaulting to dev mode') mode = 'dev' # Build dataset data = VideoDataset(mode, videos_path, vocab_path, captions_path, batch_size, num_frames, max_len, embedding_size, num_captions, load_features, load_captions, preload, model) if mode == 'train': # Get all possible video indices indices = data.get_indices() # Initialize a sampler for the indices init_sampler = sampler.SubsetRandomSampler(indices=indices) # Create data loader with dataset and sampler data_loader = DataLoader(dataset=data, num_workers=num_workers, batch_sampler=sampler.BatchSampler( sampler=init_sampler, batch_size=batch_size, drop_last=False)) else: data_loader = DataLoader(dataset=data, batch_size=batch_size, shuffle=True, num_workers=num_workers) return data_loader
def cifar10(datadir, greyscale=False, training_transforms=[], mode='train', transform=True, subset=None, **kwargs): assert mode in ['train', 'test', 'val'] train=mode=='train' root = os.path.join(datadir,'cifar10') gtransf = [transforms.Grayscale()] if greyscale else [] if mode in ['train','test']: if train and transform: tlist = [*gtransf, transforms.RandomCrop(32,padding=4), transforms.RandomHorizontalFlip(), *training_transforms, transforms.ToTensor()] else: tlist = [*gtransf, transforms.ToTensor()] transform = transforms.Compose(tlist) ds = CIFAR10(root, download=True, train=train, transform=transform) sample_size = 50000 if train else 10000 else: filename = 'cifar10.1' label_filename = 'cifar10.1_v6_labels.npy' imagedata_filename = 'cifar10.1_v6_data.npy' label_filepath = os.path.join(root, label_filename) imagedata_filepath = os.path.join(root, imagedata_filename) try: labels = np.load(label_filepath) data = np.load(imagedata_filepath) except FileNotFoundError as e: raise type(e)('Download CIFAR10.1 .npy files from https://github.com/modestyachts/CIFAR-10.1 ' 'and place in %s'%root) ds = tnt.dataset.TensorDataset([data, labels]) augment = transforms.ToTensor() ltrans = lambda x: np.array(x, dtype=np.int_) ds = ds.transform({0:augment, 1:ltrans}) sample_size=2000 # check if we're looking at a range if isinstance(subset, range): indices = np.arange(subset.start, subset.stop) elif isinstance(subset, tuple) and len(subset)==2: indices = np.arange(subset[0], subset[1]) elif isinstance(subset, np.ndarray): indices = subset elif isinstance(subset, float): if (subset > 0. and subset < 1.): num_samples = floor(subset * sample_size) assert num_samples >0 indices = np.random.choice(sample_size, num_samples) else: raise ValueError('subset fraction must be between 0 and 1') elif subset is not None: raise ValueError('Invalid subset parameter.') if subset: # according to Pytorch docs shuffle cannot be true if we are using a sampler # so we're going to turn it off in case that it's on kwargs['shuffle'] = False dataloader = th.utils.data.DataLoader(ds, sampler=sampler.SubsetRandomSampler(indices), **kwargs) dataloader.Nsamples = indices.size else: dataloader = th.utils.data.DataLoader(ds, **kwargs) if mode=='train': dataloader.Nsamples = 50000 elif mode=='test': dataloader.Nsamples = 10000 else: dataloader.Nsamples = 2000 dataloader.classes = 10 dataloader.image_shape = (3, 32, 32) return dataloader
root= r'C:\Users\Administrator\Anaconda3\Lib\site-packages\torchvision\datasets', download=True, train=True, transform=transform) # 地址前加r是因为\u对python来说是特殊字符,所以加一个r代表原生字符 test_data = torchvision.datasets.FashionMNIST( root= r'C:\Users\Administrator\Anaconda3\Lib\site-packages\torchvision\datasets', download=True, train=False, transform=transform) # 对数据进行打包,分批 Loader_train = Dataloader(dataset=train_data, batch_size=HP.Minibatch, sampler=sampler.SubsetRandomSampler(range(500))) Loader_val = Dataloader(dataset=train_data, batch_size=HP.Minibatch, sampler=sampler.SubsetRandomSampler(range(500, 700))) Loader_test = Dataloader(dataset=test_data, batch_size=HP.Minibatch, sampler=sampler.SubsetRandomSampler(range(500))) # 训练模型 cnnmodel = CNN_model() fit = Fit(model=cnnmodel.model, HP=HP, train_data=Loader_train, val_data=Loader_val) fit.train() fit.predict(mode='test', loader=Loader_test)
parser = argparse.ArgumentParser(description='pytorch Dog VS Cat') parser.add_argument('--mode', type=str, default='normal', choices=['normal', 'transfer'], metavar='M', help='use transfer learning or not (default: normal)') parser.add_argument('--data', type=str, default='', metavar='D', help='specify the data folder that contains both training set and test set') args = parser.parse_args() mode = args.mode folder_path = args.data total_size = 20000 val_size = 2000 train_size = total_size - val_size test_size = 4000 train_sampler = sampler.SubsetRandomSampler(range(train_size)) val_sampler = sampler.SubsetRandomSampler(range(val_size)) train_path = folder_path + '/dog-training/*.tif' test_path = folder_path + '/dog-test/*.tif' def transfer(): transform = transforms.Compose([transforms.RandomHorizontalFlip(), transforms.RandomCrop((224, 224)), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) train_data = DogCat(train_path, transform=transform) # 20000 test_data = DogCat(test_path, transform=transform) # 4000
def train(class_tag,model, train_data_set, save, n_epochs=3, batch_size=64, lr=0.001, wd=0.0001, momentum=0.9, seed=None, num=1, train_file=None): class_tag = "all_dset" if seed is not None: torch.manual_seed(seed) global THREADHOLD # # split data with open(train_file,"rb") as fp: train_list = pickle.load(fp) samples_num =len(train_list) split_num = int(configs.splite_rate * samples_num) data_index = train_list np.random.shuffle(data_index) train_index = data_index[:split_num] eval_index = data_index[split_num:] train_samples = sampler.SubsetRandomSampler(train_index) eval_samples = sampler.SubsetRandomSampler(eval_index) # Data loaders train_loader = torch.utils.data.DataLoader(train_data_set, batch_size=batch_size, sampler=train_samples, pin_memory=(torch.cuda.is_available()), num_workers=5, drop_last=False) valid_loader = torch.utils.data.DataLoader(train_data_set, batch_size=batch_size, sampler=eval_samples, pin_memory=(torch.cuda.is_available()), num_workers=5, drop_last=False) # Model on cuda if torch.cuda.is_available(): model = model.cuda() # Wrap model for multi-GPUs, if necessary model_wrapper = model # Optimizer optimizer = torch.optim.Adam(model_wrapper.parameters(), lr=0.001) # Start log with open(os.path.join(save, 'DeepPPI_results.csv'), 'w') as f: f.write('epoch,loss,acc,F_value, precision,recall,auc,aupr,mcc,threadhold\n') # Train model best_F = 0 threadhold = 0 count = 0 for epoch in range(n_epochs): _, train_loss = train_epoch( model=model_wrapper, loader=train_loader, optimizer=optimizer, epoch=epoch, all_epochs=n_epochs, ) _, valid_loss, acc, f_max, p_max, r_max, auc, aupr,t_max,mcc= eval_epoch( model=model_wrapper, loader=valid_loader, is_test=(not valid_loader) ) print( 'epoch:%03d,valid_loss:%0.5f\nacc:%0.6f,F_value:%0.6f, precision:%0.6f,recall:%0.6f,auc:%0.6f,aupr:%0.6f,mcc:%0.6f,threadhold:%0.6f\n' % ( (epoch + 1), valid_loss, acc, f_max, p_max, r_max,auc, aupr,mcc,t_max)) if f_max > best_F: count = 0 best_F = f_max THREADHOLD = t_max print("new best F_value:{0}(threadhold:{1})".format(f_max, THREADHOLD)) torch.save(model.state_dict(), os.path.join(save, 'DeepPPI_model.dat')) else: count += 1 if count>=5: return None # Log results f.write('%03d,%0.6f,%0.6f,%0.6f,%0.6f,%0.6f,%0.6f,%0.6f,%0.6f,%0.6f\n' % ( (epoch + 1), valid_loss, acc, f_max, p_max, r_max, auc, aupr,mcc,t_max))
def train(clargs): metrics = collections.defaultdict(list) save_path = pathlib.Path(clargs.save_path) save_path.resolve() save_path = save_path.absolute() save_path.mkdir(parents=True, exist_ok=True) model_path = save_path / 'model' hyperparameters_path = save_path / 'hyperparameters' metrics_path = save_path / 'metrics' hyperparameters = dict() hyperparameters['clargs'] = vars(clargs) if clargs.pretrained: normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) # From Imagenet. else: # From iMaterialist. normalize = transforms.Normalize(mean=[0.6837, 0.6461, 0.6158], std=[0.2970, 0.3102, 0.3271]) image_transform = transforms.Compose([ transforms.RandomHorizontalFlip(p=0.5), transforms.RandomAffine(degrees=15, scale=(1.0, 1.3), resample=Image.BILINEAR, fillcolor=2**24 - 1), # transforms.ColorJitter(brightness=0.2, contrast=0.8, saturation=0.8, hue=0.3), # transforms.RandomGrayscale(p=0.1), transforms.Resize(224), transforms.RandomCrop(224), transforms.ToTensor(), normalize ]) training_data = datasets.ImageFolder('data/training', transform=image_transform) if clargs.training_subset < len(training_data): subset_indices = random.sample([i for i in range(len(training_data))], clargs.training_subset) training_data_sampler = sampler.SubsetRandomSampler(subset_indices) else: training_data_sampler = sampler.RandomSampler(training_data) training_data_loader = data.DataLoader( training_data, batch_size=clargs.training_batch_size, num_workers=clargs.num_workers, sampler=training_data_sampler) validation_data = datasets.ImageFolder('data/validation', transform=image_transform) validation_data_loader = data.DataLoader( validation_data, batch_size=clargs.validation_batch_size, num_workers=clargs.num_workers) network: models.VGG = models.vgg19_bn(pretrained=clargs.pretrained) network.classifier[6] = nn.Linear(network.classifier[6].in_features, 128) network.cuda() optimizer = optim.SGD(network.parameters(), lr=clargs.learning_rate, weight_decay=0.0001) hyperparameters['optimizer'] = optimizer.state_dict() scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.5, patience=0, verbose=True, eps=0) training_loss_function = functools.partial(functional.cross_entropy, size_average=False) validation_loss_function = functools.partial(functional.cross_entropy, size_average=False) torch.save(hyperparameters, str(hyperparameters_path)) validation_stopwatch = stopwatch.Stopwatch() training_stopwatch = stopwatch.Stopwatch() saving_stopwatch = stopwatch.Stopwatch() best_validation_accuracy = -1.0 with stopwatch.Stopwatch() as total_time_stopwatch, stopwatch.Stopwatch( ) as epoch_stopwatch: for i in range(clargs.epoch_limit): current_lr = optimizer.param_groups[0]['lr'] if current_lr < 10**-8: break metrics['lr'].append(current_lr) LOGGER.debug('Training...') with training_stopwatch: epoch_loss_history, epoch_acccuracy_history = training.train( training_data_loader, network, optimizer, training_loss_function, cuda=True, progress_bar=(clargs.verbose >= 2)) epoch_loss_history = [ batch_loss / clargs.training_batch_size for batch_loss in epoch_loss_history ] training_stopwatch.lap() LOGGER.debug('Validating...') with validation_stopwatch: validation_loss, validation_accuracy = training.evaluate_loss_and_accuracy( validation_data_loader, network, validation_loss_function, cuda=True, progress_bar=(clargs.verbose >= 2)) validation_loss /= len(validation_data) validation_stopwatch.lap() metrics['training_loss'].extend(epoch_loss_history) metrics['validation_loss'].append(validation_loss) metrics['training_accuracy'].extend(epoch_acccuracy_history) metrics['validation_accuracy'].append(validation_accuracy) LOGGER.debug('Saving...') if validation_accuracy > best_validation_accuracy: best_validation_accuracy = validation_accuracy torch.save(network, str(model_path)) saving_stopwatch.lap() torch.save(metrics, metrics_path) epoch_stopwatch.lap() print('epoch {epoch}\n' '- total duration: {total_duration}\n' '- validation loss: {validation_loss}\n' '- validation accuracy: {validation_accuracy}\n' '- average training-batch loss: {avg_training_loss}\n' '- average training-batch accuracy: {avg_training_accuracy}'. format(epoch=i, total_duration=epoch_stopwatch.lap_times()[-1], validation_loss=validation_loss, validation_accuracy=validation_accuracy, avg_training_loss=sum(epoch_loss_history) / len(epoch_loss_history), avg_training_accuracy=sum(epoch_acccuracy_history) / len(epoch_acccuracy_history))) print( '- training duration: {training_duration}\n' '- validation duration: {validation_duration}'. format( training_duration=training_stopwatch.lap_times()[-1], validation_duration=validation_stopwatch.lap_times()[-1])) if not clargs.constant_learning_rate: scheduler.step(1.0 - validation_accuracy) total_time_stopwatch.lap() print('total_time={}'.format(total_time_stopwatch.lap_times()[-1]))
return len(self.annas) # %% Create 'Dataset's and 'DataLoader's transform = T.Compose([ T.Resize((input_size, input_size)), T.ToTensor(), T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) if __name__ == '__main__': dset_train = BoardLocalization(data_dir, anna_name, transform=transform) loader_train = DataLoader(dset_train, batch_size=batch_size, sampler=sampler.SubsetRandomSampler( range(num_train))) dset_val = BoardLocalization(data_dir, anna_name, transform=transform) loader_val = DataLoader(dset_val, batch_size=batch_size, sampler=sampler.SubsetRandomSampler( range(num_train, num_total))) # %% Utility functions def smooth_L1(x, dim=0): """ Inputs: - x: Tensor of size 4xN (by default) or size Nx4 (when dim=1) Returns:
use_cuda = torch.cuda.is_available() device = torch.device("cuda" if use_cuda else "cpu") brats_data = BraTSDataset(data_dir) num_examples = len(brats_data) data_indices = np.arange(num_examples) deterministic_test = True # Fix stochasticity in data sampling if deterministic_test: np.random.seed(0) # TODO: Doesn't really seem to belong here. Make a new # class for handling this or push it to the dataloader? np.random.shuffle(data_indices) split_idx = int(num_examples * train_split) test_sampler = sampler.SubsetRandomSampler(data_indices[split_idx:]) testloader = DataLoader(brats_data, batch_size=1, sampler=test_sampler) model = BraTSSegmentation(input_channels=2) # TODO: continue training from checkpoint checkpoint = torch.load('checkpoints/test') model.load_state_dict(checkpoint['model_state_dict']) model = model.to(device) model.eval() sum_test_dice = 0.0 with torch.no_grad(): for test_ex in tqdm(testloader): test_src, test_target = test_ex
import torch import torch.optim as optim from torch.utils.data import DataLoader from torch.utils.data import sampler from model.DeepFM import DeepFM from data.dataset import CriteoDataset # 900000 items for training, 10000 items for valid, of all 1000000 items Num_train = 9000 # load data train_data = CriteoDataset('./data', train=True) loader_train = DataLoader(train_data, batch_size=100, sampler=sampler.SubsetRandomSampler( range(Num_train))) val_data = CriteoDataset('./data', train=True) loader_val = DataLoader(val_data, batch_size=100, sampler=sampler.SubsetRandomSampler( range(Num_train, 10000))) feature_sizes = np.loadtxt('./data/feature_sizes.txt', delimiter=',') feature_sizes = [int(x) for x in feature_sizes] print(feature_sizes) model = DeepFM(feature_sizes, use_cuda=False) optimizer = optim.Adam(model.parameters(), lr=1e-4, weight_decay=0.0) model.fit(loader_train, loader_val, optimizer, epochs=5, verbose=True)
# Split validation set randomly def split_indices(n, val_percent): # Calculate the size of validation set n_val = int(n*val_percent) # Create random permutation of 0 to n-1 indices_random = np.random.permutation(n) # return train_dataset, val_dataset return indices_random[n_val:], indices_random[:n_val] train_ds, val_ds = split_indices(len(tensor_dataset), val_percent=0.2) batch_size = 100 # Train sampler and data loader train_sampler = sampler.SubsetRandomSampler(train_ds) train_dl = DataLoader(dataset=tensor_dataset, batch_size=batch_size, sampler=train_sampler) # Validation sampler and data loader val_sampler = sampler.SubsetRandomSampler(val_ds) val_dl = DataLoader(tensor_dataset, batch_size, sampler=val_sampler) class MNISTModel(nn.Module): def __init__(self, input_size, output_size): super().__init__() self.linear = nn.Linear(input_size, output_size)
def train(self, data, BATCH=64, VAL=0.1, LR=1e-4, MAX_EPOCH=10, PRE=False, NAME='checkpoint', THRESHOLD=(0.001, 0.001)): if PRE: train_threshold = self.pretrain_threshold valid_threshold = self.prevalid_threshold name = NAME + '_pretrain' else: train_threshold, valid_threshold = THRESHOLD name = NAME + '_' + str(LR) NUM, LEN, DIM = data.shape VAL = int(NUM * VAL) loader_train = DataLoader(data, batch_size=BATCH, sampler=sampler.SubsetRandomSampler( range(NUM - VAL))) loader_valid = DataLoader(data, batch_size=BATCH, sampler=sampler.SubsetRandomSampler( range(NUM - VAL, NUM))) optimizer = torch.optim.Adam([{ 'params': self.encoder.parameters() }, { 'params': self.decoder.parameters() }, { 'params': self.fc.parameters(), 'lr': 1e-4 }], lr=LR, betas=(0.5, 0.999)) train_loss_history = [] valid_loss_history = [] for e in range(MAX_EPOCH): print("train epoch " + str(e) + " starts") count = 0 train_loss = 0 valid_loss = 0 for i, batch in enumerate(loader_train): self.encoder.train() self.decoder.train() self.fc.train() N = batch.shape[0] batch = batch.to(device=self.device, dtype=self.dtype) batch_recon = self.forward(batch, N) loss = torch.nn.functional.mse_loss( batch_recon, batch.contiguous().view(-1, DIM)) train_loss += loss.item() count += 1 optimizer.zero_grad() loss.backward() optimizer.step() if i % 100 == 0: print('Iteration %d, train loss = %.4f' % (i, loss.item())) train_loss /= count count = 0 for i, batch in enumerate(loader_valid): self.encoder.eval() self.decoder.eval() self.fc.eval() with torch.no_grad(): N = batch.shape[0] batch = batch.to(device=self.device, dtype=self.dtype) batch_recon = self.forward(batch, N) loss = torch.nn.functional.mse_loss( batch_recon, batch.contiguous().view(-1, DIM)) valid_loss += loss.item() count += 1 if i % 100 == 0: print('Iteration %d, validation loss = %.4f' % (i, loss.item())) valid_loss /= count train_loss_history.append(train_loss) valid_loss_history.append(valid_loss) print("train_loss ", train_loss) print("valid_loss ", valid_loss) if (train_loss < train_threshold and valid_loss < valid_threshold): break self.save_model(name) return train_loss_history, valid_loss_history
def get_data(dataset, batch_size, _seed, validate, data_dir, shuffle=False): validation_split = 10_000 kwargs = {'num_workers': 16, 'pin_memory': True} if dataset == "MNIST": transform = transforms.Compose([transforms.ToTensor()]) # , transforms.Normalize((0.1307,), (0.3081,))] train_set = datasets.MNIST(root=data_dir, train=True, download=True, transform=transform) test_set = datasets.MNIST(root=data_dir, train=False, download=True, transform=transform) num_train = len(train_set) indices = list(range(num_train)) if not validate: train_sampler = sampler.SubsetRandomSampler(indices) test_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False, **kwargs) else: np.random.seed(_seed) np.random.shuffle(indices) # import pdb; pdb.set_trace() train_idx, valid_idx, = indices[0:-validation_split], indices[ -validation_split:] train_sampler = sampler.SubsetRandomSampler(train_idx) test_sampler = sampler.SubsetRandomSampler(valid_idx) test_loader = DataLoader(train_set, batch_size=batch_size, sampler=test_sampler, **kwargs) elif dataset in ["CIFAR10", "CIFAR10Augmented" ]: # TODO: copy data augmentation from Madry's paper # transform = transforms.Compose([transforms.Grayscale(num_output_channels=1), transforms.ToTensor()]) transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.49137255, 0.48235294, 0.44666667), (0.24705882, 0.24352941, 0.26156863)), ]) if dataset == "CIFAR10": train_set = datasets.CIFAR10(root=data_dir, train=True, download=True, transform=transform) elif dataset == "CIFAR10Augmented": # augmented_transform = transforms.Compose([transforms.Grayscale(num_output_channels=1), # transforms.RandomCrop(32, padding=4), # transforms.RandomHorizontalFlip(), # transforms.ToTensor(),]) augmented_transform = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.49137255, 0.48235294, 0.44666667), (0.24705882, 0.24352941, 0.26156863)), ]) train_set = datasets.CIFAR10(root=data_dir, train=True, download=True, transform=augmented_transform) test_set = datasets.CIFAR10(root=data_dir, train=False, download=True, transform=transform) num_train = len(train_set) indices = list(range(num_train)) if not validate: train_sampler = sampler.SubsetRandomSampler(indices) test_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False, **kwargs) else: np.random.seed(_seed) np.random.shuffle(indices) train_idx, valid_idx, = indices[0:-validation_split], indices[ -validation_split:] train_sampler = sampler.SubsetRandomSampler(train_idx) test_sampler = sampler.SubsetRandomSampler(valid_idx) test_loader = DataLoader(train_set, batch_size=batch_size, sampler=test_sampler, **kwargs) else: raise NotImplementedError train_loader = DataLoader(train_set, batch_size=batch_size, sampler=train_sampler, **kwargs) full_loader = DataLoader(train_set, batch_size=num_train, sampler=train_sampler, **kwargs, shuffle=shuffle) return train_loader, test_loader, full_loader
print(self.csv_data.size()) print(self.label_data.size()) def __len__(self): return len(self.csv_data) def __getitem__(self, idx): data = (self.csv_data[idx], self.label_data[idx]) return data dataset = RideHailingDataset("data.csv", "label.csv") dataset_size = len(dataset) print(dataset[0]) indices = list(range(dataset_size)) split = int(np.floor(validation_split * dataset_size)) if shuffle_dataset: np.random.seed(random_seed) np.random.shuffle(indices) train_indices, val_indices = indices[split:], indices[:split] train_sampler = sampler.SubsetRandomSampler(train_indices) valid_sampler = sampler.SubsetRandomSampler(val_indices) train_loader = DataLoader(dataset, batch_size=batch_size, sampler=train_sampler) validation_loader = DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler)
import torch.utils.data as data import torch.utils.data.sampler as sam import torch import warnings from os.path import dirname, abspath, join warnings.filterwarnings("ignore") torch.manual_seed(12345) LEARNING_RATE = 0.01 BATCH_SIZE = 1 NUM_TRAINING_SAMPLES = 10 NUM_TESTING_SAMPLES = 6 NUM_VAL_SAMPLES = 5 train_sampler = sam.SubsetRandomSampler( np.arange(NUM_TRAINING_SAMPLES, dtype=np.int64)) test_sampler = sam.SubsetRandomSampler( np.arange(NUM_TESTING_SAMPLES, dtype=np.int64)) val_sampler = sam.SubsetRandomSampler( np.arange(NUM_TRAINING_SAMPLES, NUM_VAL_SAMPLES + NUM_TRAINING_SAMPLES, dtype=np.int64)) current_folder = dirname(abspath(__file__)) model_folder = join(current_folder, "saved_models") # net with 2 convolutions, and a binary output class Net(nn.Module): def __init__(self): super(Net, self).__init__()
TRAIN_DATA_SIZE = 50000 TRAIN_EPOCH = 30 #10000 BATCH_SIZE = 128 #%% if True and __name__=='__main__': ''' Load MNIST data ''' mnist_transforms = torchvision.transforms.Compose([torchvision.transforms.ToTensor()]) mnist_train = torchvision.datasets.MNIST(root='./data', train=True, transform=mnist_transforms, download=True) mnist_test = torchvision.datasets.MNIST(root='./data', train=False, transform=mnist_transforms, download=True) indices = list(range(len(mnist_train))) np.random.shuffle(indices) train_idx, valid_idx = indices[:TRAIN_DATA_SIZE], indices[TRAIN_DATA_SIZE:] train_sampler = sampler.SubsetRandomSampler(train_idx) valid_sampler = sampler.SubsetRandomSampler(valid_idx) train_data_loader = torch.utils.data.DataLoader( mnist_train, batch_size=BATCH_SIZE, sampler=train_sampler, num_workers=10) valid_data_loader = torch.utils.data.DataLoader( mnist_train, batch_size=BATCH_SIZE, sampler=valid_sampler, num_workers=10) test_data_loader = torch.utils.data.DataLoader(mnist_test, batch_size=BATCH_SIZE, shuffle=True, num_workers=10) print('Loaded MNIST data, total',len(mnist_train)+len(mnist_test)) normlist=[] for x,_ in train_data_loader: x=x.view(len(x),-1) normlist.append(torch.mean(torch.norm(x,2,1))) print('Mean of Mnist norm (C2) =',torch.mean(torch.Tensor(normlist))) #%%
if CUDA: dtype = torch.cuda.FloatTensor double_type = torch.cuda.LongTensor else: dtype = torch.FloatTensor double_type = torch.LongTensor # Load the data and format them data = dataset_loader.PeptideSequence(FILENAME) idx = np.arange(len(data)) np.random.seed(0) train_data = idx[:(int)(len(data) * TRAIN_RATIO)] test_data = idx[(int)(len(data) * TRAIN_RATIO):] np.random.shuffle(train_data) np.random.shuffle(test_data) train_sampler = sampler.SubsetRandomSampler(train_data) test_sampler = sampler.SubsetRandomSampler(test_data) train_loader = dataloader.DataLoader(data, batch_size=args.batch_size, sampler=train_sampler) test_loader = dataloader.DataLoader(data, batch_size=args.batch_size, sampler=test_sampler) model = model_cnn_tagger.CNN_tagger(N_INPUT, N_OUTPUT1, args.n_kernel, args.kernel_dim) weight = torch.cuda.FloatTensor(2) weight[0] = 0.5 weight[1] = 0.5 loss_function_imbalanced = nn.CrossEntropyLoss(weight) optimizer = optim.RMSprop(model.parameters(), lr=args.lr) if CUDA: model.cuda()
device = torch.device('cpu') dtype = torch.float32 transform = T.Compose([ T.ToTensor(), T.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616)), Pad(size=(3, 36, 36)), RandomCrop(size=(32, 32)), RandomFlip(h=True), ]) dataset = cifar10('./cifar-10-batches-py', transform=transform) loader_train = DataLoader(dataset, batch_size=BATCH_SIZE, sampler=sampler.SubsetRandomSampler( range(NUM_TRAIN))) loader_train_test = DataLoader(dataset, batch_size=BATCH_SIZE, sampler=sampler.SubsetRandomSampler( random.sample(range(NUM_TRAIN), 1000))) loader_val = DataLoader(dataset, batch_size=BATCH_SIZE, sampler=sampler.SubsetRandomSampler( range(NUM_TRAIN, 50000))) loader_test = DataLoader(dataset, batch_size=BATCH_SIZE, sampler=sampler.SubsetRandomSampler( range(50000, 60000))) dropout = 0 hidden1 = 128
def initialize(self, opts, use_cuda, rounds): print('=> loading TinyImageNet data...') normalize = transforms.Normalize(mean=[0.507, 0.487, 0.441], std=[0.267, 0.256, 0.276]) ##### kwargs = {'num_workers': 2, 'pin_memory': True} if use_cuda else {} train_dataset = datasets.ImageFolder(self.root_path_train, transform=transforms.Compose([ transforms.Resize(size=(32,32),interpolation=2), transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) # change labels train_labels = [train_dataset.imgs[i][1] for i in range(train_dataset.__len__())] train_indices = getSubset4(train_labels, 4*rounds, 4*rounds+1, 4*rounds+2, 4*rounds+3) # print('train_indices:',len(train_indices)) for ind in train_indices: if train_labels[ind]==4*rounds: train_dataset.imgs[ind]=tuple_modified(train_dataset.imgs[ind],0) elif train_labels[ind]==4*rounds+1: train_dataset.imgs[ind]=tuple_modified(train_dataset.imgs[ind],1) elif train_labels[ind]==4*rounds+2: train_dataset.imgs[ind]=tuple_modified(train_dataset.imgs[ind],2) else: train_dataset.imgs[ind]=tuple_modified(train_dataset.imgs[ind],3) trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=opts.batch_size, sampler =S.SubsetRandomSampler(train_indices), **kwargs) test_dataset = datasets.ImageFolder(self.root_path_test, transform=transforms.Compose([ transforms.Resize(size=(32,32),interpolation=2), transforms.ToTensor(), normalize, ])) test_labels = [test_dataset.imgs[i][1] for i in range(test_dataset.__len__())] test_indices = getSubset4(test_labels, 4*rounds, 4*rounds+1, 4*rounds+2, 4*rounds+3) for ind in test_indices: if test_labels[ind]==4*rounds: ### test_dataset.imgs[ind]=tuple_modified(test_dataset.imgs[ind],0) elif test_labels[ind]==4*rounds+1: test_dataset.imgs[ind]=tuple_modified(test_dataset.imgs[ind],1) elif test_labels[ind]==4*rounds+2: test_dataset.imgs[ind]=tuple_modified(test_dataset.imgs[ind],2) else: test_dataset.imgs[ind]=tuple_modified(test_dataset.imgs[ind],3) testloader = torch.utils.data.DataLoader(test_dataset, batch_size=100, sampler =S.SubsetRandomSampler(test_indices), **kwargs) return trainloader, testloader
def policy_training(device='cuda'): noiseset = [35, 45, 55] # set random seed for all gpu related things seed_torch(seed=args.seed) # load msdnet and the data model = DnCNN_DS(channels=1, num_of_layers=args.num_of_layers) model = torch.nn.DataParallel(model).cuda() if os.path.exists(os.path.join(args.outf, 'net.pth')): print('Loading denoise model...') model.load_state_dict(torch.load(os.path.join(args.outf, 'net.pth'))) else: print('Need the classification model!') return # need to augment the validation set to generate training set for PolicyNet print('Loading dataset ...\n') # load the original image instead # dataset_train = Dataset(train=True, data_folder=args.data_folder) # total_train = len(dataset_train) # val_size = int(total_train*0.2) # print("Training data for policynet: ", val_size) # # load indices file # indices = np.load(os.path.join(args.outf, 'indices.npy')) # val_idx = indices[:val_size] # train_idx = indices[val_size:] # train_loader = DataLoader(dataset=dataset_train, num_workers=args.num_workers, # sampler=sampler.SubsetRandomSampler(train_idx), # batch_size=args.batch_size, shuffle=False) # val_loader = DataLoader(dataset=dataset_train, num_workers=args.num_workers, # sampler=sampler.SubsetRandomSampler(val_idx), # batch_size=args.batch_size, shuffle=False) # load the original test data dataset_train = load_imgs('train') total_train = len(dataset_train) val_size = int(total_train * args.val_ratio) indices = list(range(total_train)) random.Random(0).shuffle(indices) np.save(os.path.join(args.outf, 'policy_train_indices.npy'), np.array(indices)) val_idx = indices[:val_size] train_idx = indices[val_size:] train_loader = DataLoader(dataset=dataset_train, num_workers=args.num_workers, sampler=sampler.SubsetRandomSampler(train_idx), batch_size=args.batch_size, shuffle=False) val_loader = DataLoader(dataset=dataset_train, num_workers=args.num_workers, sampler=sampler.SubsetRandomSampler(val_idx), batch_size=1, shuffle=False) print('Training data size: ', len(train_loader.dataset)) # print('Validation data size: ', len(val_loader.dataset)) dataset_val = Dataset(train=False) test_loader_12 = DataLoader(dataset=dataset_val, num_workers=4, batch_size=1, shuffle=False) # use Set68 as testdataset dataset_test = load_imgs('Set68') test_loader = DataLoader(dataset=dataset_test, num_workers=4, batch_size=1, shuffle=False) # need to construct the policy network and train the policy net. # the architecture of the policy network need to be designed. ###################################### # need to think about the model of policynet ###################################### model.eval() p_true_all = list() psnr_all = list() np.random.seed(seed=args.seed) test_noiseL = np.random.choice(noiseset, size=len(val_loader.dataset)) # print(test_noiseL) print('Average noise level: ', np.average(test_noiseL)) for i, batch in enumerate(val_loader): # for i in range(1): # batch = next(iter(train_loader)) data = batch data = data.cuda() noise = torch.zeros(data.size()) noise = torch.FloatTensor(data.size()).normal_( mean=0, std=test_noiseL[i] / 255., generator=torch.manual_seed(args.seed)) noise = noise.cuda() with torch.no_grad(): outputs = model(data + noise) p_true, mse_all = PolicyKL.true_posterior(args, outputs, noise) p_true_all.append(p_true) # psnrs = list() # for pred in outputs: # psnr = batch_PSNR(torch.clamp(data+noise-pred, 0., 1.), # data, 1.) # psnrs.append(psnr) # psnr_all.append(np.array(psnrs)) # psnr_all = np.stack(psnr_all) p_true = torch.cat(p_true_all, dim=0) p_det = max_onehot(p_true, dim=-1, device=device) p_true = torch.mean(p_true, dim=0) # find positions with nonzero posterior p_det_index = torch.argmax(p_det, dim=1) print(Counter(list(p_det_index.cpu().numpy()))) p_det = torch.mean(p_det, dim=0) train_post = {} nz_post = {} i = 0 for t in range(len(outputs)): if p_det[t] > 0.001: # if p_det[t] > -1: train_post[i] = t nz_post[i] = t i += 1 del train_post[i - 1] p_str = 'val p true:[' p_str += ','.join(['%0.3f' % p_true[t] for t in nz_post.values()]) print(p_str + ']') p_str = 'val p true det:[' p_str += ','.join(['%0.3f' % p_det[t] for t in nz_post.values()]) print(p_str + ']') print(nz_post) ###################################### # initialize nets with nonzero posterior if args.policy_type == 'multiclass': score_net = MulticlassNet(args, nz_post, 1) elif args.policy_type == 'sequential': score_net = MulticlassNet(args, train_post, 1) else: print('Model not implemented!!') return score_net = torch.nn.DataParallel(score_net) score_net = score_net.cuda() # pdb.set_trace() if args.restart and os.path.exists( os.path.join(args.outf, '{}_policy_net.dump'.format( args.policy_type))): print('Loading previous policynet model...') dump = os.path.join(args.outf, '{}_policy_net.dump'.format(args.policy_type)) score_net.load_state_dict(torch.load(dump)) # train if args.phase == 'train': # start training optimizer = optim.Adam(list(score_net.parameters()), lr=1e-3, weight_decay=args.weight_decay) milestones = [10, 20, 40, 60, 80] # gammas = [0.4, 0.2, 0.2, 0.2, 0.2] gammas = [1, 1, 1, 1, 1] scheduler = MultiStepMultiLR(optimizer, milestones=milestones, gammas=gammas) trainer = PolicyKL(args=args, model=model, score_net=score_net, train_post=train_post, nz_post=nz_post, optimizer=optimizer, train_loader=train_loader, val_loader=val_loader, test_loader=test_loader, device=device, scheduler=scheduler) trainer.train() # test dump = os.path.join(args.outf, '{}_policy_net.dump'.format(args.policy_type)) score_net.load_state_dict(torch.load(dump)) PolicyKL.test(args=args, score_net=score_net, model=model, data_loader=test_loader, nz_post=nz_post, device=device, noiseset=[75]) print(args.outf)
def get_data_loader( transform: tv.transforms, caption_file: str, image_id_file: str, image_folder: str, config: Config, vocab_file: str, mode: str = "train", batch_size: int = 1, vocab_threshold=None, start_word: str = "<start>", end_word: str = "<end>", unk_word: str = "<unk>", vocab_from_file: bool = True, num_workers: int = 0, ): """Returns the data loader :param transform: [description] :type transform: tv.transforms :param mode: [description], defaults to "train" :type mode: str, optional :param batch_size: [description], defaults to 1 :type batch_size: int, optional :param vocab_threshold: [description], defaults to None :type vocab_threshold: [type], optional :param vocab_file: [description], defaults to "output/vocab.pkl" :type vocab_file: str, optional :param start_word: [description], defaults to "<start>" :type start_word: str, optional :param end_word: [description], defaults to "<end>" :type end_word: str, optional :param unk_word: [description], defaults to "<unk>" :type unk_word: str, optional :param vocab_from_file: [description], defaults to True :type vocab_from_file: bool, optional :param num_workers: [description], defaults to 0 :type num_workers: int, optional """ assert mode in [ "train", "validation", "test", ], f"mode: '{mode}' must be one of ['train','validation','test']" if vocab_from_file == False: assert ( mode == "train" ), f"mode: '{mode}', but to generate vocab from caption file, mode must be 'train' " if mode == "train": if vocab_from_file == True: assert os.path.exists( vocab_file ), "vocab_file does not exist. Change vocab_from_file to False to create vocab_file." assert image_id_file.find( "train" ), f"double check image_id_file: {image_id_file}. File name should have the substring 'train'" assert os.path.exists( image_id_file ), f"image id file: {image_id_file} doesn't not exist." assert os.path.exists( caption_file ), f"caption file: {caption_file} doesn't not exist." assert os.path.isdir( config.IMAGE_DATA_DIR ), f"{config.IMAGE_DATA_DIR} not a directory" assert ( len(os.listdir(config.IMAGE_DATA_DIR)) != 0 ), f"{config.IMAGE_DATA_DIR} is empty." if mode == "validation": assert image_id_file.find( "dev" ), f"double check image_id_file: {image_id_file}. File name should have the substring 'dev' " assert os.path.exists( image_id_file ), f"image id file: {image_id_file} doesn't not exist." assert os.path.exists( caption_file ), f"caption file: {caption_file} doesn't not exist." assert os.path.isdir( config.IMAGE_DATA_DIR ), f"{config.IMAGE_DATA_DIR} not a directory" assert ( len(os.listdir(config.IMAGE_DATA_DIR)) != 0 ), f"{config.IMAGE_DATA_DIR} is empty." assert os.path.exists( vocab_file ), f"Must first generate {vocab_file} from training data." assert vocab_from_file == True, "Change vocab_from_file to True." if mode == "test": assert ( batch_size == 1 ), "Please change batch_size to 1 if testing your model." assert image_id_file.find( "test" ), f"double check image_id_file: {image_id_file}. File name should have the substring 'test'" assert os.path.exists( vocab_file ), f"Must first generate {vocab_file} from training data." assert vocab_from_file == True, "Change vocab_from_file to True." img_folder = config.IMAGE_DATA_DIR annotations_file = caption_file # image caption dataset dataset = FlickrDataset( transform, mode, batch_size, vocab_threshold, vocab_file, start_word, end_word, unk_word, caption_file, image_id_file, vocab_from_file, image_folder, ) if mode in ["train", "validation"]: # Randomly sample a caption length, and sample indices with that length. indices = dataset.get_train_indices() # Create and assign a batch sampler to retrieve a batch with the sampled indices. initial_sampler = sampler.SubsetRandomSampler(indices=indices) # data loader for COCO dataset. data_loader = DataLoader( dataset=dataset, num_workers=num_workers, batch_sampler=sampler.BatchSampler( sampler=initial_sampler, batch_size=dataset.batch_size, drop_last=False, ), ) else: data_loader = DataLoader( dataset=dataset, batch_size=dataset.batch_size, shuffle=False, num_workers=num_workers, ) return data_loader
dataset = utils.NumpyDataset(data_shifted, labels, transform=utils.NumpyToTensor()) # Run the experiments for seed in seeds: # Data loaders logger.info('Split data with seed {}'.format(seed)) torch.manual_seed(seed) np.random.seed(seed) train_indices = [] val_indices = [] for cls in np.unique(labels): indices = np.where(labels == cls) indices = np.random.permutation(indices[0]) train_indices.append(indices[:int(len(indices) * (1 - validating_ratio))]) val_indices.append(indices[int(len(indices) * (1 - validating_ratio)):]) train_set_sampler = sampler.SubsetRandomSampler(np.concatenate(train_indices)) validating_set_sampler = sampler.SubsetRandomSampler(np.concatenate(val_indices)) train_loader = DataLoader(dataset, batch_size=batch_size, sampler=train_set_sampler, num_workers=8) val_loader = DataLoader(dataset, batch_size=batch_size, sampler=validating_set_sampler, num_workers=8) # TensorboardX writer writer = SummaryWriter(main_directory + '/runs/' + experiment_name + '_' + str(seed)) # The model torch.manual_seed(0) model = WideNetMasked(len(class_names)).to(device) logger.info('Net parameters number : {}'.format(utils.compute_total_parameter_number(model))) optimizer = optim.SGD(model.parameters(), lr=0.05, momentum=0.9, weight_decay=0.001) scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[50, 100, 150], gamma=0.1)
T.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ]) train_data = MammogramDataset("Mini_DDSM_Upload", "train", transform=transform) test_data = MammogramDataset("Mini_DDSM_Upload", "test") VAL_RATIO = 0.2 NUM_VAL = int(len(train_data) * VAL_RATIO) NUM_TRAIN = len(train_data) - NUM_VAL NUM_TEST = len(test_data) BATCH_SIZE = batch_size loader_train = DataLoader(train_data, batch_size=BATCH_SIZE, sampler=sampler.SubsetRandomSampler( range(NUM_TRAIN)), drop_last=True) loader_val = DataLoader(train_data, batch_size=BATCH_SIZE, sampler=sampler.SubsetRandomSampler( range(NUM_TRAIN, NUM_TRAIN + NUM_VAL))) loader_test = DataLoader(test_data, batch_size=BATCH_SIZE) dtype = torch.float32 device = torch.device('cpu') epoch = 0 loss_list = [] val_acc_list = [] best_accuracy = 0.0
def weighted_loss_train(self, optimizer, BATCH=64, MAX_EPOCH=30, USE_WEIGHT=True): VAL = self.val_data.shape[0] self.model.train() train_loss_history = [] val_loss_history = [] train_recall_history = [] val_recall_history = [] for e in range(MAX_EPOCH): epoch_train_loss = 0 epoch_train_recall = 0 print("training epoch " + str(e) + " starts") if USE_WEIGHT: self.eval(BATCH) weighted_index = self.weighted_sampler.sample_train_data(size) print(data.shape) NUM = data.shape[0] loader_train = DataLoader(data, batch_size=BATCH, sampler=sampler.SubsetRandomSampler( range(NUM))) loader_valid = DataLoader(self.val_data, batch_size=BATCH, sampler=sampler.SubsetRandomSampler( range(VAL))) train_iter = 0 for i, batch in enumerate(loader_train): X = (batch[:, :-2]).to(device=self.device, dtype=self.data_dtype) y = (batch[:, -2]).to(device=self.device, dtype=self.label_dtype) scores = self.model(X) accuracy = np.argmax(scores.cpu().detach().numpy(), axis=1) - y.cpu().detach().numpy() mask = np.where(accuracy == 0) accuracy = len(mask[0]) / X.shape[0] epoch_train_recall += accuracy loss = torch.nn.functional.cross_entropy(scores, y) epoch_train_loss += loss.item() if i % 100 == 0: print("epoch is ", e, ", training loss is ", loss.item()) print("epoch is ", e, ", training accuracy is ", accuracy) optimizer.zero_grad() loss.backward() optimizer.step() train_iter += 1 train_loss_history.append(epoch_train_loss / train_iter) train_recall_history.append(epoch_train_recall / train_iter) epoch_val_loss = 0 epoch_val_recall = 0 val_iter = 0 for i, batch in enumerate(loader_valid): self.model.eval() # put model to evaluation mode X = (batch[:, :-1]).to(device=self.device, dtype=self.data_dtype) y = (batch[:, -1]).to(device=self.device, dtype=self.label_dtype) with torch.no_grad(): scores = self.model(X) accuracy = np.argmax(scores.cpu().detach().numpy(), axis=1) - y.cpu().detach().numpy() mask = np.where(accuracy == 0) accuracy = len(mask[0]) / X.shape[0] epoch_val_recall += accuracy loss = torch.nn.functional.cross_entropy(scores, y) epoch_val_loss += loss.item() if i % 100 == 0: print("epoch is ", e, ", validation loss is ", loss.item()) print("epoch is ", e, ", validation accuracy is ", accuracy) val_iter += 1 val_loss_history.append(epoch_val_loss / val_iter) val_recall_history.append(epoch_val_recall / val_iter) return train_loss_history, train_recall_history, val_loss_history, val_recall_history
T.RandomHorizontalFlip(), T.RandomVerticalFlip(), T.ToTensor() ]) if augment: train_data = MammogramDataset("data", "train", transform=transform) else: train_data = MammogramDataset("data", "train") test_data = MammogramDataset("data", "test") NUM_VAL = int(len(train_data)*VAL_RATIO) NUM_TRAIN = len(train_data) - NUM_VAL NUM_TEST = len(test_data) loader_train = DataLoader(train_data, batch_size=BATCH_SIZE, sampler=sampler.SubsetRandomSampler(range(NUM_TRAIN))) loader_val = DataLoader(train_data, batch_size=BATCH_SIZE, sampler=sampler.SubsetRandomSampler(range(NUM_TRAIN, NUM_TRAIN + NUM_VAL))) loader_test = DataLoader(test_data, batch_size=BATCH_SIZE) loader_tiny_train = DataLoader(train_data, batch_size=BATCH_SIZE, sampler=sampler.SubsetRandomSampler(range(101))) loader_tiny_val = DataLoader(train_data, batch_size=BATCH_SIZE, sampler=sampler.SubsetRandomSampler(range(100, 200))) dtype = torch.float32 if USE_GPU and torch.cuda.is_available(): #Determine whether or not to use GPU device = torch.device('cuda') else: device = torch.device('cpu') print('using device:', device) '''
import torchvision.transforms as T import numpy as np #Preprocessing Data #Substracting the mean RGB value and dividing by the standard deviation of each RGB value NUM_TRAIN = 49000 transform = T.Compose([ T.ToTensor(), T.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ]) cifar10_train = dset.CIFAR10('./datasets', train=True, download=True,transform=transform) loader_train = DataLoader(cifar10_train, batch_size=64, sampler=sampler.SubsetRandomSampler(range(NUM_TRAIN))) cifar10_val = dset.CIFAR10('./datasets', train=True, download=True,transform=transform) loader_val = DataLoader(cifar10_val, batch_size=64, sampler=sampler.SubsetRandomSampler(range(NUM_TRAIN, 50000))) cifar10_test = dset.CIFAR10('./datasets', train=False, download=True, transform=transform) loader_test = DataLoader(cifar10_test, batch_size=64) #Use GPU device USE_GPU = True dtype = torch.float32 if USE_GPU and torch.cuda.is_available(): device = torch.device('cuda') else: device = torch.device('cpu')
# The torchvision.transforms package provides tools for preprocessing data # and for performing data augmentation; here we set up a transform to # preprocess the data by subtracting the mean RGB value and dividing by the # standard deviation of each RGB value; we've hardcoded the mean and std. transform = T.Compose([T.ToTensor()]) # We set up a Dataset object for each split (train / val / test); Datasets load # training examples one at a time, so we wrap each Dataset in a DataLoader which # iterates through the Dataset and forms minibatches. We divide the CIFAR-10 # training set into train and val sets by passing a Sampler object to the # DataLoader telling how it should sample from the underlying Dataset. cbow_train = CBOWDataset('text8.txt', 4) loader_train = DataLoader(cbow_train, batch_size=64, sampler=sampler.SubsetRandomSampler( range(int(len(cbow_train) * 0.8)))) loader_val = DataLoader(cbow_train, batch_size=64, sampler=sampler.SubsetRandomSampler( range(int(len(cbow_train) * 0.8), int(len(cbow_train) * 0.9)))) loader_test = DataLoader(cbow_train, batch_size=64, sampler=sampler.SubsetRandomSampler( range(int(len(cbow_train) * 0.9), len(cbow_train)))) USE_GPU = True dtype = torch.float32 # we will be using float throughout this tutorial if USE_GPU and torch.cuda.is_available():
CF_start_time = time.time() model = CF_knearest(dataset) print(model.predict_score(dataset, 1, 3671)) CF_end_time = time.time() print("the end of training CF, and the time consumption: %d" % (CF_end_time - CF_start_time)) # val_data = PreProcessData("./data/ratings_small.csv", train=True) # loader_val = DataLoader(train_data, batch_size=50, sampler=sampler.SubsetRandomSampler(range(TRAIN_ROUND, 100000))) feature_size = FM_train_data.feature_size print("feature_size is " + str(feature_size)) loader_train = DataLoader(FM_train_data, batch_size=50, sampler=sampler.SubsetRandomSampler( range(FM_TRAIN_DATA_NUMBER))) """ FM """ FM_start_time = time.time() FM_model = FM(feature_sizes=feature_size) print("Now, lets train the model") FM_model.fit(loader_train, epochs=50) FM_end_time = time.time() print("the end of training FM, time consume: %d" % (FM_end_time - FM_start_time)) """ DeepFM """ deepFM_start_time = time.time()
def create_dataloader(loader_type, args_data, do_use_gpu): """ Create a data loader according to the parameters """ kwargs = { 'num_workers': args_data.num_loader_workers, 'pin_memory': True } if do_use_gpu else {} if args_data.dataset_type == DatasetType.NYU: if loader_type == LoaderMode.TRAIN: # Set up sample IDs to sample from ids_train = np.arange(args_data.id_start_train, args_data.id_end_train + 1) ids_train_permuted = args_data.rng.permutation(ids_train) ids_train_labeled = ids_train_permuted[:args_data. num_labeled_samples] ids_train_unlabeled = ids_train_permuted[args_data. num_labeled_samples:] # Ensure a minimum sampling probability for labeled samples ratio_labeled = len(ids_train_labeled) / float(len(ids_train)) prob_labeled = max(args_data.min_sampling_prob_labeled, ratio_labeled) prob_unlabeled = 1.0 - prob_labeled # Set up distribution/weights to sample from (considering un-/labeled samples) scale_weights = float( len(ids_train)) # value to which weights will sum up sample_weight_labeled = prob_labeled * scale_weights / float( len(ids_train_labeled)) sample_weight_unlabeled = prob_unlabeled * scale_weights \ / float(len(ids_train_unlabeled)) \ if len(ids_train_unlabeled) > 0 else 0.0 sampling_weights = np.zeros((args_data.num_all_samples_train)) sampling_weights[ids_train_labeled] = sample_weight_labeled sampling_weights[ids_train_unlabeled] = sample_weight_unlabeled num_samples_used_for_train = np.count_nonzero(sampling_weights) loader = torch.utils.data.DataLoader( NyuHandPoseMultiViewDataset( args_data.nyu_data_basepath, train=True, cropSize=args_data.in_crop_size, doJitterCom=args_data.do_jitter_com, sigmaCom=args_data.sigma_com, doAddWhiteNoise=args_data.do_add_white_noise, sigmaNoise=args_data.sigma_noise, doLoadRealSamples=args_data.do_use_real_samples, unlabeledSampleIds=ids_train_unlabeled, transform=transforms.ToTensor(), useCache=args_data.use_pickled_cache, cacheDir=args_data.nyu_data_basepath_pickled, annoType=args_data.anno_type, neededCamIds=args_data.needed_cam_ids_train, randomSeed=args_data.seed, cropSize3D=args_data.crop_size_3d_tuple, args_data=args_data), batch_size=args_data.batch_size, sampler=smpl.WeightedRandomSampler( sampling_weights, num_samples=num_samples_used_for_train, replacement=True), **kwargs) print("Using {} samples for training".format( num_samples_used_for_train)) if sample_weight_labeled > 0.: print(" {} labeled".format(len(ids_train_labeled))) if sample_weight_unlabeled > 0.: print(" {} unlabeled".format(len(ids_train_unlabeled))) elif loader_type == LoaderMode.VAL: num_samples_val = min( int( round(args_data.max_val_train_ratio * args_data.num_labeled_samples)), args_data.max_num_samples_val) ids_val = np.arange(args_data.id_start_val, args_data.id_end_val + 1) ids_val = args_data.rng.permutation(ids_val) ids_val = ids_val[:num_samples_val] loader = torch.utils.data.DataLoader( NyuHandPoseMultiViewDataset( args_data.nyu_data_basepath, train=False, cropSize=args_data.in_crop_size, doJitterCom=args_data.do_jitter_com_test, sigmaCom=args_data.sigma_com, doAddWhiteNoise=args_data.do_add_white_noise_test, sigmaNoise=args_data.sigma_noise, doLoadRealSamples=args_data.do_use_real_samples, transform=transforms.ToTensor(), useCache=args_data.use_pickled_cache, cacheDir=args_data.nyu_data_basepath_pickled, annoType=args_data.anno_type, neededCamIds=args_data.needed_cam_ids_test, randomSeed=args_data.seed, cropSize3D=args_data.crop_size_3d_tuple, args_data=args_data), batch_size=args_data.batch_size, sampler=smpl.SubsetRandomSampler(ids_val), **kwargs) print("Using {} samples for validation".format(len(ids_val))) elif loader_type == LoaderMode.TEST: ids_test = np.arange(args_data.id_start_test, args_data.id_end_test + 1) loader = torch.utils.data.DataLoader( NyuHandPoseMultiViewDataset( args_data.nyu_data_basepath, train=False, cropSize=args_data.in_crop_size, doJitterCom=args_data.do_jitter_com_test, sigmaCom=args_data.sigma_com, doAddWhiteNoise=args_data.do_add_white_noise_test, sigmaNoise=args_data.sigma_noise, doLoadRealSamples=args_data.do_use_real_samples, transform=transforms.ToTensor(), useCache=args_data.use_pickled_cache, cacheDir=args_data.nyu_data_basepath_pickled, annoType=args_data.anno_type, neededCamIds=args_data.needed_cam_ids_test, randomSeed=args_data.seed, cropSize3D=args_data.crop_size_3d_tuple, args_data=args_data), batch_size=args_data.batch_size, sampler=smpl.SubsetRandomSampler(ids_test), **kwargs) print("Using {} samples for test".format(len(ids_test))) else: raise UserWarning("LoaderMode unknown.") elif args_data.dataset_type == DatasetType.ICG: args_data = set_loader_type_specific_settings_icg( args_data, loader_type) dataset = IcgHandPoseMultiViewDataset(args_data) if loader_type == LoaderMode.TRAIN: num_samples = len(dataset) num_samples_labeled_all = dataset.get_num_samples_labeled() num_samples_unlabeled = dataset.get_num_samples_unlabeled() num_samples_labeled_used = min(num_samples_labeled_all, args_data.num_labeled_samples) num_samples_used = num_samples_labeled_used + num_samples_unlabeled # Set up sample IDs to sample from ids_train = np.arange(num_samples) ids_train_labeled_all = ids_train[:num_samples_labeled_all] ids_train_labeled_perm = args_data.rng.permutation( ids_train_labeled_all) ids_train_labeled = ids_train_labeled_perm[: num_samples_labeled_used] ids_train_unlabeled = ids_train[num_samples_labeled_all:] # Ensure a minimum sampling probability for labeled samples ratio_labeled = len(ids_train_labeled) / float(num_samples_used) prob_labeled = max(args_data.min_sampling_prob_labeled, ratio_labeled) prob_unlabeled = 1.0 - prob_labeled # Set up distribution/weights to sample from (considering un-/labeled samples) scale_weights = float( num_samples_used) # value to which weights will sum up sample_weight_labeled = prob_labeled * scale_weights / float( len(ids_train_labeled)) sample_weight_unlabeled = prob_unlabeled * scale_weights \ / float(len(ids_train_unlabeled)) \ if len(ids_train_unlabeled) > 0 else 0.0 sampling_weights = np.zeros(len(ids_train)) sampling_weights[ids_train_labeled] = sample_weight_labeled sampling_weights[ids_train_unlabeled] = sample_weight_unlabeled num_samples_used_for_train = np.count_nonzero(sampling_weights) loader = torch.utils.data.DataLoader( dataset, batch_size=args_data.batch_size, sampler=smpl.WeightedRandomSampler( sampling_weights, num_samples=num_samples_used_for_train, replacement=True), **kwargs) print("Using {} samples for training".format( num_samples_used_for_train)) if sample_weight_labeled > 0.: print(" {} labeled".format(len(ids_train_labeled))) if sample_weight_unlabeled > 0.: print(" {} unlabeled".format(len(ids_train_unlabeled))) elif loader_type == LoaderMode.VAL: # Prepare val. sample IDs ids_val = np.arange(len(dataset)) if args_data.do_val_only_with_labeled: num_samples_labeled_all = dataset.get_num_samples_labeled() ids_val = np.arange(num_samples_labeled_all) # Use subset? max_num_samples_val = int( round(args_data.max_val_train_ratio * args_data.num_labeled_samples)) num_samples_val = min(max_num_samples_val, len(ids_val)) ids_val = args_data.rng.permutation(ids_val) ids_val = ids_val[:num_samples_val] loader = torch.utils.data.DataLoader( dataset, batch_size=args_data.batch_size, sampler=smpl.SubsetRandomSampler(ids_val), **kwargs) print("Using {} samples for validation".format(len(ids_val))) print( " {} labeled (might be incorrect if a subset is used (e.g., wrt. train set size))" .format(dataset.get_num_samples_labeled())) if not args_data.do_val_only_with_labeled: print( " {} unlabeled (might be incorrect if a subset is used (e.g., wrt. train set size))" .format(dataset.get_num_samples_unlabeled())) elif loader_type == LoaderMode.TEST: # Prepare test sample IDs ids_test = np.arange(len(dataset)) if args_data.do_test_only_with_labeled: num_samples_labeled_all = dataset.get_num_samples_labeled() ids_test = np.arange(num_samples_labeled_all) loader = torch.utils.data.DataLoader( dataset, batch_size=args_data.batch_size, sampler=smpl.SubsetRandomSampler(ids_test), **kwargs) print("Using {} samples for test".format(len(ids_test))) print(" {} labeled".format(dataset.get_num_samples_labeled())) if not args_data.do_test_only_with_labeled: print(" {} unlabeled".format( dataset.get_num_samples_unlabeled())) else: raise UserWarning("DatasetType unknown.") return loader
T.ToTensor(), T.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ]) # We set up a Dataset object for each split (train / val / test); Datasets load # training examples one at a time, so we wrap each Dataset in a DataLoader which # iterates through the Dataset and forms mini batches. We divide the CIFAR-10 # training set into train and val sets by passing a Sampler object to the # DataLoader telling how it should sample from the underlying Dataset. cifar10_train = dset.CIFAR10(cifar10_dir, train=True, download=True, transform=transform) loader_train = DataLoader(cifar10_train, batch_size=64, sampler=sampler.SubsetRandomSampler( range(NUM_TRAIN))) cifar10_val = dset.CIFAR10(cifar10_dir, train=True, download=True, transform=transform) loader_val = DataLoader(cifar10_val, batch_size=64, sampler=sampler.SubsetRandomSampler( range(NUM_TRAIN, 50000))) cifar10_test = dset.CIFAR10(cifar10_dir, train=False, download=True, transform=transform) loader_test = DataLoader(cifar10_test, batch_size=64)
def reset_dataloader(data_loader): """Reset sampler for dataloader.""" indices = data_loader.dataset.get_indices() new_sampler = sampler.SubsetRandomSampler(indices=indices) data_loader.batch_sampler.sampler = new_sampler