def train(LOOP): # setup model model.train() model.to(DEVICE) # load model ckpt from the previous trained LOOP if LOOP: ckpt = os.path.join(EXPT_DIR, 'ckpt_{}.pth'.format(LOOP - 1)) model.load_state_dict(torch.load(ckpt)) # load selected indices labeled = os.path.join(EXPT_DIR, 'labeled.txt') f = open(labeled, 'r') indices = f.readlines() indices = [int(x.strip()) for x in indices] f.close() # setup dataloader dataloader = DataLoader(cifar_train, batch_size=64, sampler=SubsetRandomSampler(indices)) loss_fn = nn.CrossEntropyLoss() opt = optim.Adam(model.parameters(), lr=params['learning_rate'], weight_decay=params['weight_decay']) for epoch in range(epochs): for x, y in dataloader: x, y = x.to(DEVICE), y.to(DEVICE) y_ = model(x) loss = loss_fn(y_, y) opt.zero_grad() loss.backward() opt.step() # save ckpt ckpt = os.path.join(EXPT_DIR, 'ckpt_{}.pth'.format(LOOP)) torch.save(model.state_dict(), ckpt) return
def get_sampler_classifier(dataloader, seed, proportion_training): if proportion_training == 1: return None labels = np.array(dataloader.dataset.train_labels) unique_labels = np.unique(labels) n_labels = len(unique_labels) num_training_samples_per_label = int(proportion_training * len(labels) / n_labels) training_subset_idx = [] for label in unique_labels: label_set = np.where(labels == label)[0] label_set_len = len(label_set) # Subset indices for training np.random.seed(seed) label_training_subset = np.random.choice( label_set, size=num_training_samples_per_label, replace=False) training_subset_idx += list(label_training_subset) return SubsetRandomSampler(training_subset_idx)
def make_data_loader(*tensors, N, batch_size, num_batches) -> torch.utils.data.DataLoader: """ Creates DataLoader, which samples num_batches batches of sequences with shape (batch_size, N) from tensors :param tensors: tensors to sample batches from :param N: length of sampled subsequences :param batch_size: batch size :param num_batches: amount of batches to sample :return: DataLoader """ assert tensors[0].shape[ 0] % N == 0, "length of tensors has to be dividable by N!" M = tensors[0].shape[0] // N dataset = SequenceDataset(*tensors, N=N) indices = np.random.choice(N * (M - 1), size=batch_size * num_batches, replace=False) base_sampler = SubsetRandomSampler(indices) loader = DataLoader(dataset, sampler=base_sampler, batch_size=batch_size) return loader
def create_dataloader(args, dataset, batch_size, eval_slice_dataset=None, world_size=None, rank=None): logger = logging_utils.get_logger(args) if eval_slice_dataset is not None and not args.run_config.distributed: indices = get_eval_slice_subset_indices(args, eval_slice_dataset=eval_slice_dataset, dataset=dataset) # Form sampler with for indices from eval_slice_dataset sampler = SubsetRandomSampler(indices) elif args.run_config.distributed: # wrap dataset object to use a subsetsampler with distributed if eval_slice_dataset is not None: indices = get_eval_slice_subset_indices(args, eval_slice_dataset=eval_slice_dataset, dataset=dataset) dataset = DistributedIndicesWrapper(dataset, torch.tensor(indices)) sampler = DistributedSampler(dataset, num_replicas=world_size, rank=rank) else: sampler = None dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=(sampler is None), sampler=sampler, num_workers=args.run_config.dataloader_threads, pin_memory=False) return dataloader, sampler
def get_restrictedImageNetValidationTestSplit(val_split=True, batch_size=128, shuffle=False, augm_type='test', num_workers=8, size=224): idxs = np.loadtxt('idxs_rimgnet.txt', dtype=int) if not val_split: all_idcs = np.arange(10150) idxs = np.setdiff1d(all_idcs, idxs) #test idcs if shuffle: sampler = SubsetRandomSampler(idxs) else: sampler = SubsetSampler(idxs) transform = get_imageNet_augmentation(type=augm_type, out_size=size) path = get_imagenet_path() dataset = RestrictedImageNet(path=path, split='val', transform=transform, balanced=False) loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=sampler, num_workers=num_workers) return loader
def __init__(self, concat_dataset, batch_size, drop_last, patial_batches_ok): super(MTBatchSampler, self).__init__(concat_dataset) self.task_idx_bins = [] offset = 0 for ds in concat_dataset.datasets: self.task_idx_bins.append(np.arange(offset, offset + len(ds))) offset += len(ds) if not isinstance(batch_size, int) or isinstance(batch_size, bool) or \ batch_size <= 0: raise ValueError("batch_size should be a positive integer value, " "but got batch_size={}".format(batch_size)) if not isinstance(drop_last, bool): raise ValueError("drop_last should be a boolean value, but got " "drop_last={}".format(drop_last)) self.samplers = [SubsetRandomSampler(idx) for idx in self.task_idx_bins] self.batch_size = batch_size self.drop_last = drop_last self.partial_batches_ok = patial_batches_ok
def fit(self, zeta, xu, nb_iter=100, batch_size=None, lr=1e-3): if self.prior and 'l2_penalty' in self.prior: self.optim = Adam(self.parameters(), lr=lr, weight_decay=self.prior['l2_penalty']) else: self.optim = Adam(self.parameters(), lr=lr) set_size = xu.shape[0] batch_size = set_size if batch_size is None else batch_size batches = list( BatchSampler(SubsetRandomSampler(range(set_size)), batch_size, True)) for n in range(nb_iter): for batch in batches: self.optim.zero_grad() loss = -self.elbo(zeta[batch], xu[batch], batch_size, set_size) loss.backward() self.optim.step()
def get_dataloader(is_train=True, indeces=None, batch_size=32, num_workers=0, data_path=DATA_FOLDER): # https://discuss.pytorch.org/t/train-on-a-fraction-of-the-data-set/16743/6 dataset = get_dataset(is_train, data_path) if indeces is None: data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=is_train, num_workers=num_workers) else: data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=is_train, num_workers=num_workers, sampler=SubsetRandomSampler(indeces)) return data_loader
def _update_offpolicy(self): if len(self.memory) > self.batch_size: for _ in range(self.epochs): indices = torch.randperm(len(self.memory))[:self.batch_size] indices = len(self.memory) - indices - 1 dataloader = DataLoader(self.memory, self.batch_size, sampler=SubsetRandomSampler(indices), num_workers=8) if self.q_update == 2: self.q_update = 1 for states, actions, rewards, dones, next_states in dataloader: self._training_q(states.to(self.device), actions.to(self.device), rewards.to(self.device), dones.to(self.device), next_states.to(self.device)) self._training_policy(states.to(self.device)) self.target_soft_q1 = copy_parameters( self.soft_q1, self.target_soft_q1, self.soft_tau) self.target_soft_q2 = copy_parameters( self.soft_q2, self.target_soft_q2, self.soft_tau) self.target_policy = copy_parameters( self.policy, self.target_policy, self.soft_tau) else: self.q_update = 2 for states, actions, rewards, dones, next_states in dataloader: self._training_q(states.to(self.device), actions.to(self.device), rewards.to(self.device), dones.to(self.device), next_states.to(self.device)) self.target_soft_q1 = copy_parameters( self.soft_q1, self.target_soft_q1, self.soft_tau) self.target_soft_q2 = copy_parameters( self.soft_q2, self.target_soft_q2, self.soft_tau)
def train(): # setup model model.train() model.to(DEVICE) if RESUME_FROM: resume_from = os.path.join(EXPT_DIR, RESUME_FROM) model.load_state_dict(torch.load(resume_from)) # load selected indices labeled = load(os.path.join(EXPT_DIR, 'labeled.pkl')) indices = list(labeled.keys()) # change dataset object's target attribute to labels # this is used for pollution study for ix in labeled: mnist_train.targets[ix] = labeled[ix] # setup dataloader dataloader = DataLoader(mnist_train, batch_size=64, sampler=SubsetRandomSampler(indices)) loss_fn = nn.CrossEntropyLoss() opt = optim.Adam(model.parameters(), lr=params['learning_rate'], weight_decay=params['weight_decay']) for epoch in range(epochs): for x, y in dataloader: x, y = x.to(DEVICE), y.to(DEVICE) y_ = model(x) loss = loss_fn(y_, y) opt.zero_grad() loss.backward() opt.step() # save ckpt ckpt = os.path.join(EXPT_DIR, CKPT_FILE) torch.save(model.state_dict(), ckpt) return
def get_dataloader(synthetic_dataset, real_dataset, height, width, batch_size, workers, is_train, keep_ratio): num_synthetic_dataset = len(synthetic_dataset) num_real_dataset = len(real_dataset) synthetic_indices = list(np.random.permutation(num_synthetic_dataset)) synthetic_indices = synthetic_indices[num_real_dataset:] real_indices = list(np.random.permutation( num_real_dataset) + num_synthetic_dataset) concated_indices = synthetic_indices + real_indices assert len(concated_indices) == num_synthetic_dataset sampler = SubsetRandomSampler(concated_indices) concated_dataset = ConcatDataset([synthetic_dataset, real_dataset]) print('total image: ', len(concated_dataset)) data_loader = DataLoader(concated_dataset, batch_size=batch_size, num_workers=workers, shuffle=False, pin_memory=True, drop_last=True, sampler=sampler, collate_fn=AlignCollate(imgH=height, imgW=width, keep_ratio=keep_ratio)) return concated_dataset, data_loader
def main(): trainset = torchvision.datasets.FashionMNIST('../data', train=True, download=True, transform=transforms.Compose([ lambda x: transforms.functional.to_grayscale(x, num_output_channels=3), transforms.Resize(32), transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])) dataset_indices = list(range(len(trainset))) dataset_indices = dataset_indices[2000:3000] sampler = SubsetRandomSampler(dataset_indices) trainloader = torch.utils.data.DataLoader(trainset, batch_size=defs.batch_size, drop_last=False, num_workers=4, pin_memory=True, sampler=sampler) exit(0) if opt.data == 'cifar100': downloaded_list = [ ['train', '16019d7e3df5f24257cddd939b257f8d'], ] root = os.path.join(os.getenv("HOME"), 'data') base_folder = 'cifar-100-python' # now load the picked numpy arrays data = list() targets = list() for file_name, checksum in downloaded_list: file_path = os.path.join(root, base_folder, file_name) with open(file_path, 'rb') as f: if sys.version_info[0] == 2: entry = pickle.load(f) else: entry = pickle.load(f, encoding='latin1') data.append(entry['data']) if 'labels' in entry: targets.extend(entry['labels']) else: targets.extend(entry['fine_labels'])
def fit(self, target, input, nb_epochs=1000, batch_size=32, lr=1e-3, l2=1e-32, verbose=True, preprocess=True): if preprocess: self.init_preprocess(target, input) target = transform(target, self.target_trans) input = transform(input, self.input_trans) self.optim = Adam(self.parameters(), lr=lr, weight_decay=l2) set_size = input.shape[0] batch_size = set_size if batch_size is None else batch_size for n in range(nb_epochs): batches = list( BatchSampler(SubsetRandomSampler(range(set_size)), batch_size, True)) for batch in batches: self.optim.zero_grad() _output = self.forward(input[batch]) loss = self.criterion(_output, target[batch]) loss.backward() self.optim.step() if verbose: if n % 50 == 0: output = self.forward(input) print('Epoch: {}/{}.............'.format(n, nb_epochs), end=' ') print("Loss: {:.4f}".format(self.criterion(output, target)))
def data_loaders_split(params: dict): """ Creats data loaders for train, validation and test sections of data Args: params: dictionary of params to input to data loader: dataset, batch_size, num_workers, collate_fn Returns: data_loaders: dictionary of data loaders for train, val, test splits of data """ # Create a dictionary to collect index_names = get_split_index_names() data_loaders = dict.fromkeys(index_names) for key in data_loaders: # Make sampler and data loader cur_prop_name = index_names[key] cur_split_indices = getattr(params['dataset'], cur_prop_name) cur_sampler = SubsetRandomSampler(cur_split_indices) data_loaders[key] = torch.utils.data.DataLoader(sampler=cur_sampler, **params) return data_loaders
def __init__( self, root_dir, dataset=None, k_shot=None, transform=None, batch_size=64, online_crop=False, ): self.traindataset = CustomDataset( root_dir, dataset_selection=dataset, k_shot=k_shot, split=True, train_valid="train", transform=transform, ) self.validdataset = CustomDataset( root_dir, dataset_selection=dataset, k_shot=k_shot, split=True, train_valid="valid", transform=transform, ) self.batch_size = batch_size self.k_shot = k_shot self.images_indices = self.traindataset.selectKshots() self.meta_sampler = SubsetRandomSampler(self.images_indices) self.train_loader = DataLoader(self.traindataset, batch_size=self.batch_size, sampler=self.meta_sampler) self.val_loader = DataLoader(self.validdataset, batch_size=self.batch_size)
def get_mnist_subset_loader(train: bool, path: str, c1: int, c2: int) -> Tuple[DataLoader, int]: """Return an MNIST dataloader for the two specified classes. Args: train (bool): Should this be a training set or validation set path (str): The directory in which to store/find the MNIST dataset c1 (int): a number in [0, 9] denoting a MNIST class/number c2 (int): a number in [0, 9] denoting a MNIST class/number Returns: Tuple[DataLoader, int]: Return a dataloader and its size """ # All inputs must be converted into torch tensors, and the normalization values # have been precomputed and provided below. mnist_transforms = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )), ]) dataset = MNIST(root=path, train=train, download=True, transform=mnist_transforms) # Grab indices for the two classes we care about idx_class1 = [i for i, t in enumerate(dataset.targets) if t == c1] idx_class2 = [i for i, t in enumerate(dataset.targets) if t == c2] idxs = idx_class1 + idx_class2 size = len(idxs) loader = DataLoader(dataset, sampler=SubsetRandomSampler(idxs), batch_size=size) return loader, size
def get_dataloader(csv_files, csv_labels, name, batch_size, shuffle, split=None, directory="", num_workers=4): data_transforms = { "train": transforms.Compose(( transforms.Resize((256, 256)), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), )), "valid": transforms.Compose(( transforms.Resize((256, 256)), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), )), } sampler = None dataset = MURA(directory, csv_files, csv_labels, data_transforms[name]) if split is not None: dataset_size = len(dataset) indices = list(range(dataset_size)) new_dataset_size = int(np.floor(split * dataset_size)) np.random.shuffle(indices) splited = indices[:new_dataset_size] sampler = SubsetRandomSampler(splited) return DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=shuffle, sampler=sampler)
def get_mnist_dataloaders(batch_size=128, character_classes=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]): """ MNIST dataloader with (32, 32) sized images. """ # Resize images so they are a power of 2 all_transforms = transforms.Compose( [transforms.Resize(32), transforms.ToTensor()]) # Get train and test data train_data = datasets.MNIST('./data', train=True, download=True, transform=all_transforms) test_data = datasets.MNIST('./data', train=False, transform=all_transforms) # Use SubsetRandomSampler, given a set of indices, to be the dataloader. character_indices = {} for i in range(len(train_data)): label = train_data[i][1] if label in character_indices: character_indices[label].append(i) else: character_indices[label] = [i] train_loaders = [] for value in character_classes: data_loader = DataLoader(train_data, batch_size=batch_size, sampler=SubsetRandomSampler( character_indices[value])) train_loaders.append(data_loader) test_loader = DataLoader(test_data, batch_size=batch_size, shuffle=True) all_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True) return train_loaders, test_loader, all_loader
def dataset_loader(data_dir, batch_size, test_batch_size, train_num=100, test_num=100): path = os.path.join(data_dir, "mnist_correct/label_correct_index.npy") label_correct_indices = list(np.load(path)) random.seed(1234) random.shuffle(label_correct_indices) train_indices = label_correct_indices[:train_num] test_indices = label_correct_indices[5000:5000 + test_num] train_loader = torch.utils.data.DataLoader( datasets.MNIST(data_dir, train=False, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=batch_size, shuffle=False, sampler=SubsetRandomSampler(train_indices), drop_last=True) test_loader = torch.utils.data.DataLoader( Subset(datasets.MNIST(data_dir, train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), test_indices), batch_size=test_batch_size, shuffle=False) return train_loader, test_loader
def train(): forecast_length = 3 backcast_length = 24 batch_size = 1 # greater than 4 for viz f_b_dim = (forecast_length, backcast_length) num_samples = 0 lr = 1e-3 # If the user specified a checkpoint, load the data # and model from that checkpoint/run if CHECKPOINT_NAME == "": data_gen = data_generator(batch_size, backcast_length, forecast_length, signal_type='seasonality', random=True) ts = [] # user did not specify a previous checkpoint, generate new data for grad_step, x, in enumerate(data_gen): ts.append(x) num_samples += len(x) if num_samples >= 100000: break ts = np.concatenate(ts) save_ts(ts) else: ts = np.load("data/" + RUN_NAME + "/dataset/timeseries.npy") data = DatasetTS(ts, forecast_length, backcast_length) net = NBeats(stacks=[TrendBlock, SeasonalityBlock, GenericNBeatsBlock], f_b_dim=f_b_dim, num_blocks_per_stack=[2,4,4], thetas_dims=[[2,2], [8,8], [2,8]], hidden_layer_dim=64) optimiser = optim.Adam(net.parameters(), lr=lr) print('--- Training ---') epochs = 50 initial_epoch_step = load_model(net, optimiser) ds_len = len(data) train_length = int(np.ceil(ds_len*0.95)) train_sampler = SubsetRandomSampler(list(range(train_length))) validation_sampler = SubsetRandomSampler(list(range(train_length + 1, ds_len))) train_loader = torch.utils.data.DataLoader(data, batch_size=100, sampler=train_sampler) validation_loader = torch.utils.data.DataLoader(data, batch_size=4, sampler=validation_sampler) backcast_validation_data, target_validation = iter(validation_loader).next() writer = SummaryWriter("data/" + RUN_NAME) if TENSORBOARD_ENABLE else None for epoch in range(initial_epoch_step, epochs): for grad_step, (x, target) in enumerate(train_loader): optimiser.zero_grad() net.train() forecast, backcast = net(x) loss = F.smooth_l1_loss(forecast, target) temp = loss # loss = loss * 100 #loss = F.smooth_l1_loss(forecast, target) loss.backward() optimiser.step() if writer: writer.add_scalar("loss/training_loss", temp, epoch) with torch.no_grad(): # save_model(net, optimiser, epoch) if not DISABLE_PLOT: plot(net, backcast_validation_data, target_validation, validation_loader, backcast_length, forecast_length, epoch, writer) plt.close()
print("\nTarget : %s" % targets[i]) print("\nPredicted : %s" % dec[i]) print( "=====================================================================\n" ) # -------------------------- Eval on Test -------------------------- # print('\n\n==================== Eval on Test ====================\n\n') dataset = MyDataset(tokenizer, args.data_dir, 'test', max_len=args.max_seq_length) # loader = DataLoader(dataset, batch_size=26, shuffle=True) from torch.utils.data import SubsetRandomSampler splr = SubsetRandomSampler([i for i in range(len(dataset))]) loader = DataLoader( dataset, batch_size=args.eval_batch_size, # 16, sampler=splr, shuffle=False, # sampler is mutually exclusive with shuffle drop_last=True) metric = datasets.load_metric('seqeval') for i in range(len(loader)): it = iter(loader) batch = next(it) outs = model.generate(input_ids=batch['source_ids'].cuda(), attention_mask=batch['source_mask'].cuda(), max_length=args.max_seq_length)
def train(self, batch_size=6): # PPO algorithm # Unroll rewards rewards = np.array(self.rewards) reward = 0 for i in reversed(range(len(self.rewards))): rewards[i] += self.gamma * reward reward = rewards[i] states = torch.tensor(self.states, dtype=torch.float) actions = torch.tensor(self.actions, dtype=torch.long).view(-1, 1) all_old_probs = torch.tensor(self.a_probs, dtype=torch.float).view(-1, 1) rewards = rewards.reshape(-1, 1) actor_entropies = [] actor_losses = [] for batch in BatchSampler(SubsetRandomSampler(range(len(self.states))), batch_size, drop_last=False): states_batch = states[batch].numpy() actions_batch = actions[batch].numpy() old_action_probs = all_old_probs[batch].numpy() rewards_batch = rewards[batch] actor_loss = [] actor_entropy = [] for state, action, reward, old_action_probs in zip( states_batch, actions_batch, rewards_batch, old_action_probs): state = state.reshape((-1, 1)) V = self.critic.forward(state) advantage = reward - V probs = self.actor.forward(state) actor_entropy.append(-np.sum(np.log2(probs) * probs)) action_prob = probs[action] action_prob_ratio = action_prob / old_action_probs surr = action_prob_ratio * advantage if action_prob_ratio < (1 - self.clip_e): clamp_prob = 1 - self.clip_e elif action_prob_ratio > (1 + self.clip_e): clamp_prob = 1 + self.clip_e else: clamp_prob = action_prob_ratio clipped_surr = clamp_prob * advantage actor_loss.append(-np.minimum(surr, clipped_surr)) if surr < clipped_surr: actor_dLoss = 1 / old_action_probs * advantage else: if action_prob_ratio < 1 - self.clip_e or action_prob_ratio > 1 + self.clip_e: actor_dLoss = 0 else: actor_dLoss = 1 / old_action_probs * advantage self.actor.backward(actor_dLoss, action) critic_loss = np.square(advantage) critic_dloss = 2 * advantage self.critic.backward(critic_dloss, action) self.actor.mean_grads(batch_size) self.actor.update() actor_losses.append(np.mean(actor_loss)) actor_entropies.append(np.mean(actor_entropy)) self.critic.mean_grads(batch_size) self.critic.update() self._clear_buffers() return np.mean(actor_losses), np.mean(actor_entropies), self.a_lr
#DEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu' DEVICE = torch.device("cuda:1" if torch.cuda.is_available() else "cpu") collate_fn = BertCollator(device='cpu') if __name__ == '__main__': dataset = AmazonZiser17(ds=SOURCE, dl=0, labeled=True, cldata=False) dataset_size = len(dataset) indices = list(range(dataset_size)) perm = torch.randperm(len(indices)) val_size = 0.2 val_split = int(np.floor(val_size * dataset_size)) train_indices = perm[val_split:] val_indices = perm[:val_split] train_sampler = SubsetRandomSampler(train_indices) val_sampler = SubsetRandomSampler(val_indices) train_loader = DataLoader(dataset, batch_size=4, sampler=train_sampler, drop_last=False, collate_fn=collate_fn) val_loader = DataLoader(dataset, batch_size=4, sampler=val_sampler, drop_last=False, collate_fn=collate_fn) if TARGET == "books": pre = './sbooks' elif TARGET == "dvd":
def train_val_split( dataset: Dataset, batch_size: int, val_split: float, shuffle: bool = True, num_workers: int = 4, collate_fn: Callable = None, pin_memory: bool = True, drop_last: bool = False, timeout: float = 0, worker_init_fn: Callable = None, val_transform: Callable = None, target_val_transform: Callable = None, use_default_val_transform: bool = False, ) -> Tuple[DataLoader, DataLoader]: """ Splits the dataset into train and validation dataloaders. Args: batch_size (int): The batch size dataset (Dataset): The dataset to split val_split (float): The amount of data from the original dataset to be used \ in the validation dataloader. Should be a value between 0 and 1. shuffle (bool, optional): If ``True``, the train and validation data are chosen at \ random. Defaults to ``True``. num_workers (int, optional): How many subprocesses to use for data loading. Defaults to 4. collate_fn (Callable, optional): Merges a list of samples to form a mini-batch of Tensor(s). \ Used when using batched loading from a map-style dataset. Defaults to ``None``. pin_memory (bool, optional): If ``True``, copies Tensors to cuda pinned memory. Defaults to ``True``. drop_last (bool, optional): Set to ``True`` to drop the last incomplete batch. Defaults to ``False``. timeout (float, optional): If positive, the timeout value for collecting a batch from workers. \ Should always be non-negative. Defaults to 0. worker_init_fn (Callable, optional): If not None, this will be called on each worker subprocess with \ the worker id (an int in [0, num_workers - 1]) as input, after seeding and before data loading. val_transform (Callable, optional): The transform to be used in the val dataset. Defaults to None. target_val_transform (Callable, optional): The transform to be used for the targets in the val dataset. \ Defaults to None. use_default_val_transform (bool, optional): Whether to use the default val transforms or not. \ Defaults to False. Returns: Tuple[DataLoader, DataLoader]: The train dataloader and the validation dataloader """ # getting dataset size dataset_size = len(dataset) # generating indices indices = np.arange(dataset_size) if shuffle: np.random.shuffle(indices) # getting amount of data in validation split split = np.floor(val_split * dataset_size).astype(np.int) # generating samplers train_sampler = SubsetRandomSampler(indices[split:]) val_sampler = SubsetRandomSampler(indices[:split]) # creating val dataset val_dataset = deepcopy(dataset) # setting the transform for the val dataset if use_default_val_transform: val_transform = val_dataset.default_val_transform() val_dataset.train = False val_dataset.transform = val_transform val_dataset.target_transform = target_val_transform # creating and returning dataloaders return ( DataLoader( dataset, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers, collate_fn=collate_fn, pin_memory=pin_memory, drop_last=drop_last, timeout=timeout, worker_init_fn=worker_init_fn, ), DataLoader( val_dataset, batch_size=batch_size, sampler=val_sampler, num_workers=num_workers, collate_fn=collate_fn, pin_memory=pin_memory, drop_last=drop_last, timeout=timeout, worker_init_fn=worker_init_fn, ), )
def main(): parsed_args = parse_args() init_logging(parsed_args.logger) # dataset related davis_dir = parsed_args.dataset year = parsed_args.year dataset_mode = parsed_args.set seq_names = parsed_args.sequences shuffle = not parsed_args.no_shuffle # model related mode = parsed_args.mode cuda_dev = parsed_args.cuda_device model_save_path = parsed_args.model_save model_load_path = parsed_args.model_load # training related batch_size = parsed_args.batch_size epochs = parsed_args.epochs iters = parsed_args.iters augment = not parsed_args.no_augment lr = parsed_args.learning_rate weight_decay = parsed_args.weight_decay validation_size = parsed_args.validation_size loss_function = parsed_args.loss_function # videomatch related img_shape = parsed_args.input_image_shape seg_shape = parsed_args.segmentation_shape remove_outliers = not parsed_args.leave_outliers fg_thresh = parsed_args.fg_thresh encoder = parsed_args.encoder upsample_fac = parsed_args.upsample_factor # misc val_report_iter = parsed_args.val_report visualize = parsed_args.visualize results_dir = parsed_args.results_dir loss_visualize = parsed_args.loss_visualization # args checks if mode == 'train' and batch_size != 1: logger.warning("Batch size > 1 is only applicable to 'eval' mode.") if iters != -1 and epochs > 1: logger.warning("Iters is set to {} and not to -1 (full dataset), but epoch is > 1".format(iters)) if mode == 'eval' and shuffle: logger.warning("Dataset shuffle can't be set to True in 'eval' mode, setting it to False!") shuffle = False if mode == 'train' and not shuffle: logger.warning("Dataset shuffle is off, consider turning it on when training, " "to avoid overfitting on starting sequences") if mode != 'eval' and visualize: logger.warning("Visualize is set to True, but mode isn't 'eval'") device = None if cuda_dev is None else "cuda:{:d}".format(cuda_dev) dataset = Davis(davis_dir, dataset_mode, seq_names) vm = VideoMatch(out_shape=seg_shape, device=device, encoder=encoder, upsample_fac=upsample_fac) if model_load_path is not None: logger.info("Loading model from path {}".format(model_load_path)) vm.load_model(model_load_path) if mode == 'train': pair_sampler = PairSampler(dataset, randomize=shuffle) indices = np.arange(len(pair_sampler)) if shuffle: np.random.shuffle(indices) split = int(np.floor(validation_size * len(pair_sampler))) val_indices = indices[:split] train_indices = indices[split:] train_loader = DataLoader(dataset, batch_sampler=SubsetRandomSampler(pair_sampler.get_indexes(train_indices)), collate_fn=collate_pairs) val_loader = DataLoader(dataset, batch_sampler=SubsetRandomSampler(pair_sampler.get_indexes(val_indices)), collate_fn=collate_pairs) logger.debug("Train set size: {}, Validation set size: {}".format(len(pair_sampler) - split, split)) iters = len(train_loader) if iters == -1 else iters fp = FrameAugmentor(img_shape, augment) train_vm(train_loader, val_loader, vm, fp, device, lr, weight_decay, iters, epochs, val_report_iter, model_save_path, loss_visualize, loss_function) elif mode == 'eval': multiframe_sampler = MultiFrameSampler(dataset) data_loader = DataLoader(dataset, sampler=multiframe_sampler, collate_fn=collate_multiframes, batch_size=batch_size, num_workers=batch_size) if results_dir is not None and not isdir(results_dir): mkdir(results_dir) eval_vm(data_loader, vm, img_shape, fg_thresh, remove_outliers, visualize, results_dir)
kwargs['node_feature_func'] = 'property_prediction' kwargs['edge_feature_func'] = 'default' if args.task in ['hiv']: kwargs['k_fold'] = 'StratifiedKFold' else: kwargs['k_fold'] = 'KFold' dataset = config_task2dataset[args.task](**kwargs) train_indices, test_indices = split_into_KFold(dataset=dataset, k=args.k_fold, index=args.running_index, **kwargs) train_sampler = SubsetRandomSampler(train_indices) test_sampler = SubsetRandomSampler(test_indices) train_dataloader = DataLoader(dataset, sampler=train_sampler, batch_size=args.batch_size) test_dataloader = DataLoader(dataset, sampler=test_sampler, batch_size=args.batch_size) if args.model == 'ECFP': model = config_model[args.model](ECFP_dim=args.fp_length, hidden_dim=args.fp_hiddden_dim, output_dim=1) elif args.model == 'GIN': model = config_model[args.model](dataset.node_feature_dim, [256, 256, 256])
def train(self, batch_size=128): # PPO algorithm # Unroll rewards rewards = np.array(self.rewards) reward = 0 for i in reversed(range(len(self.rewards))): rewards[i] += self.gamma * reward reward = rewards[i] states = torch.tensor(self.states, dtype=torch.float) actions = torch.tensor(self.actions, dtype=torch.float).view(-1, self.action_space) all_old_probs = torch.tensor(self.a_probs, dtype=torch.float).view( -1, self.action_space) rewards = rewards.reshape(-1, 1) actor_entropies = [] actor_losses = [] for batch in BatchSampler(SubsetRandomSampler(range(len(self.states))), batch_size, drop_last=False): states_batch = states[batch].numpy() actions_batch = actions[batch].numpy() old_action_probs = all_old_probs[batch].numpy() rewards_batch = rewards[batch] actor_grads_batch = [] critic_grads_batch = [] actor_loss = [] actor_entropy = [] for state, action, reward, old_action_probs in zip( states_batch, actions_batch, rewards_batch, old_action_probs): state = state.reshape( (-1, 1)) # TODO maybe check shape? Does not coincide with act V, critic_cache = self.critic.full_forward_propagation(state) advantage = reward - V _, action_prob, entropy, actor_cache = self.act(state) actor_entropy.append(np.sum(entropy.numpy())) old_action_probs = old_action_probs.reshape(-1, 1) action_prob_ratio = action_prob / old_action_probs surr = action_prob_ratio * advantage clamp_prob = action_prob_ratio.copy() clamp_prob[action_prob_ratio < (1 - self.clip_e)] = 1 - self.clip_e clamp_prob[action_prob_ratio > (1 + self.clip_e)] = 1 + self.clip_e clipped_surr = clamp_prob * advantage actor_loss.append(-np.minimum(surr, clipped_surr)) actor_dLoss = np.zeros(shape=(self.action_space, 1)) for i in range(self.action_space): if surr[i] < clipped_surr[i]: actor_dLoss[i, 0] = 1 / old_action_probs[i] * advantage else: if action_prob_ratio[ i] < 1 - self.clip_e or action_prob_ratio[ i] > 1 + self.clip_e: actor_dLoss[i, 0] = 0 else: actor_dLoss[ i, 0] = 1 / old_action_probs[i] * advantage actor_grads = self.actor.full_backward_propagation( actor_dLoss, actor_cache, action) actor_grads_batch.append(actor_grads) critic_loss = np.square(advantage) critic_dloss = 2 * advantage critic_grads = self.critic.full_backward_propagation( critic_dloss, critic_cache, action) critic_grads_batch.append(critic_grads) actor_grads_values_mean = self.actor.mean_grads( actor_grads_batch, batch_size) self.actor.update(actor_grads_values_mean, self.a_lr) actor_losses.append(np.mean(actor_loss)) actor_entropies.append(np.mean(actor_entropy)) critic_grads_values_mean = self.critic.mean_grads( critic_grads_batch, batch_size) self.critic.update(critic_grads_values_mean, self.c_lr) self._clear_buffers() return np.mean(actor_losses), np.mean(actor_entropies), self.a_lr
def get_dataloader(module_name, module_args): if module_args['dataset']['img_type'] == 'cv': from opencv_transforms import opencv_transforms as transforms else: from torchvision import transforms train_transfroms = transforms.Compose([ transforms.ColorJitter(brightness=0.5), transforms.ToTensor() ]) val_transfroms = transforms.ToTensor() # 创建数据集 dataset_args = module_args['dataset'] train_data_path = dataset_args.pop('train_data_path') val_data_path = dataset_args.pop('val_data_path') train_dataset = get_dataset(data_path=train_data_path, module_name=module_name, transform=train_transfroms, phase='train', dataset_args=dataset_args) val_dataset = get_dataset(data_path=val_data_path, module_name=module_name, transform=val_transfroms, phase='test', dataset_args=dataset_args) if val_dataset is None: val_dataset = copy.deepcopy(train_dataset) val_dataset.transform = transforms.ToTensor() val_dataset.phase = 'test' # 数据集切分 indices = list(range(len(train_dataset))) random.shuffle(indices) val_len = int(module_args['loader']['validation_split'] * len(train_dataset)) train_sampler = SubsetRandomSampler(indices[val_len:]) val_sampler = SubsetRandomSampler(indices[:val_len]) train_loader = DataLoader(dataset=train_dataset, sampler=train_sampler, batch_size=module_args['loader']['train_batch_size'], pin_memory=module_args['loader']['pin_memory'], num_workers=module_args['loader']['num_workers']) val_loader = DataLoader(dataset=val_dataset, sampler=val_sampler, batch_size=module_args['loader']['val_batch_size'], pin_memory=module_args['loader']['pin_memory'], num_workers=module_args['loader']['num_workers']) else: train_loader = DataLoader(dataset=train_dataset, batch_size=module_args['loader']['train_batch_size'], shuffle=module_args['loader']['shuffle'], pin_memory=module_args['loader']['pin_memory'], num_workers=module_args['loader']['num_workers']) val_loader = DataLoader(dataset=val_dataset, batch_size=module_args['loader']['val_batch_size'], shuffle=module_args['loader']['shuffle'], pin_memory=module_args['loader']['pin_memory'], num_workers=module_args['loader']['num_workers']) return train_loader, val_loader
def get_clusters(cluster_set): nDescriptors = 50000 nPerImage = 100 nIm = ceil(nDescriptors / nPerImage) sampler = SubsetRandomSampler( np.random.choice(len(cluster_set), nIm, replace=False)) data_loader = DataLoader(dataset=cluster_set, num_workers=opt.threads, batch_size=opt.cacheBatchSize, shuffle=False, pin_memory=cuda, sampler=sampler) if not exists(join(opt.dataPath, 'centroids')): makedirs(join(opt.dataPath, 'centroids')) initcache = join( opt.dataPath, 'centroids', opt.arch + '_' + cluster_set.dataset + '_' + str(opt.num_clusters) + '_desc_cen.hdf5') with h5py.File(initcache, mode='w') as h5: with torch.no_grad(): model.eval() print('====> Extracting Descriptors') dbFeat = h5.create_dataset("descriptors", [nDescriptors, encoder_dim], dtype=np.float32) for iteration, (input, indices) in enumerate(data_loader, 1): input = input.to(device) print(input.size()) #torch.Size([8, 3, 480, 640]) image_descriptors = model.encoder(input).view( input.size(0), encoder_dim, -1).permute(0, 2, 1) print(image_descriptors.shape) #torch.Size([8, 1200, 512]) print(image_descriptors.size(1)) #1200 batchix = (iteration - 1) * opt.cacheBatchSize * nPerImage for ix in range(image_descriptors.size(0)): # sample different location for each image in batch sample = np.random.choice(image_descriptors.size(1), nPerImage, replace=False) startix = batchix + ix * nPerImage dbFeat[startix:startix + nPerImage, :] = image_descriptors[ ix, sample, :].detach().cpu().numpy() if iteration % 50 == 0 or len(data_loader) <= 10: print("==> Batch ({}/{})".format( iteration, ceil(nIm / opt.cacheBatchSize)), flush=True) del input, image_descriptors print('====> Clustering..') niter = 100 kmeans = faiss.Kmeans(encoder_dim, opt.num_clusters, niter=niter, verbose=False) kmeans.train(dbFeat[...]) print('====> Storing centroids', kmeans.centroids.shape) h5.create_dataset('centroids', data=kmeans.centroids) print('====> Done!')
def main(options): model_dir = os.path.join('experiments', 'models') trg_size = (224, 224) transformations = transforms.Compose([CustomResize(trg_size), CustomToTensor() ]) batch_size = options.batch_size train_set = SliceSet2D(transform=transformations) validation_split = .4 shuffle_dataset = True random_seed = 42 set_size = len(train_set) indices = list(range(set_size)) split = int(np.floor(validation_split * set_size)) if shuffle_dataset: np.random.seed(random_seed) np.random.shuffle(indices) train_indices, val_indices = indices[split:], indices[:split] print("Set Size: {}|Train Size: {}| Validation Size: {}".format(len(indices), len(train_indices), len(val_indices))) print("Model will be saved in \" ./experiments/model\". Start training...") # Creating data samplers and loaders: train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(val_indices) train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, sampler=train_sampler) validation_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, sampler=valid_sampler) # Initiate the model model = models.resnet18(pretrained=True) model.fc = Linear(in_features=512, out_features=3) for para in list(model.parameters())[:-2]: para.requires_grad = False optimizer = Adam(params=[model.fc.weight, model.fc.bias], lr=3e-4) print('the training layer is:') for name, param in model.named_parameters(): # 查看可优化的参数有哪些 if param.requires_grad: print(name) # model.load_state_dict(torch.load(options.load)) if torch.cuda.is_available(): model.cuda() else: model.cpu() # Binary cross-entropy loss criterion = torch.nn.CrossEntropyLoss() # criterion = torch.nn.CrossEntropyLoss() lr = options.learning_rate optimizer = eval("torch.optim." + options.optimizer)(filter(lambda x: x.requires_grad, model.parameters()), lr) best_accuracy = float("-inf") for epoch_i in range(options.epochs): logging.info("At {0}-th epoch.".format(epoch_i)) train_loss, correct_cnt = train(model, train_loader, criterion, optimizer) # each instance in one batch has 3 views train_avg_loss = train_loss / len(train_indices)*3/options.batch_size train_avg_acu = float(correct_cnt) / (len(train_indices) * 3) logging.info( "Average training loss is {0:.5f} at the end of epoch {1}".format(train_avg_loss.data, epoch_i)) logging.info("Average training accuracy is {0:.5f} at the end of epoch {1}".format(train_avg_acu, epoch_i)) with torch.no_grad(): correct_cnt = validate(model, validation_loader) dev_avg_acu = float(correct_cnt) / len(val_indices) logging.info("Average validation accuracy is {0:.5f} at the end of epoch {1}".format(dev_avg_acu, epoch_i)) if dev_avg_acu > best_accuracy: best_accuracy = dev_avg_acu torch.save(model, os.path.join('experiments', 'models/mymodel.pth'))