def _make_enqueued_generator(generator, workers=1, use_multiprocessing=False, max_queue_size=10, shuffle=False): """Create a buffered queue of next elements of the generator.""" is_sequence = isinstance(generator, data_utils.Sequence) enqueuer = None if workers > 0: if is_sequence: enqueuer = data_utils.OrderedEnqueuer( generator, use_multiprocessing=use_multiprocessing, shuffle=shuffle) else: enqueuer = data_utils.GeneratorEnqueuer( generator, use_multiprocessing=use_multiprocessing) enqueuer.start(workers=workers, max_queue_size=max_queue_size) output_generator = enqueuer.get() else: if is_sequence: output_generator = data_utils.iter_sequence_infinite(generator) else: output_generator = generator return output_generator, enqueuer
def train_generator(self, generator, num_worker): total_c_loss = 0.0 total_accuracy = 0.0 # optimizer = self._create_optimizer(self.matchNet, self.lr) # traindata = self.data.get_trainset(batch_size, 0, shuffle=True) traindata = generator train_enqueuer = data_utils.GeneratorEnqueuer(traindata) train_enqueuer.start(workers=num_worker, max_queue_size=traindata.batch_size * 2) train_generator = train_enqueuer.get() total_train_batches = len(traindata) with tqdm.tqdm(total=total_train_batches) as pbar: for i in range(total_train_batches): # (x_support_set, y_support_set, x_target, y_target) = next(train_generator) # batch = [] # for b in range(batch_size): batch = next(train_generator) x_support_set, y_support_set, x_target, y_target = self.numpy2tensor( batch) x_support_set = x_support_set.permute(0, 1, 4, 2, 3) x_target = x_target.permute(0, 3, 1, 2) y_support_set = y_support_set.float() y_target = y_target.long() x_support_set = x_support_set.float() x_target = x_target.float() acc, c_loss = self.matchNet(x_support_set, y_support_set, x_target, y_target) # optimize process self.optimizer.zero_grad() c_loss.backward() self.optimizer.step() total_c_loss += c_loss.data[0] total_accuracy += acc.data[0] i += 1 iter_out = f"loss: {total_c_loss / i:.{3}}, acc: {total_accuracy / i:.{3}}" pbar.set_description(iter_out) pbar.update(1) # self.total_train_iter+=1 total_c_loss = total_c_loss / total_train_batches total_accuracy = total_accuracy / total_train_batches self.scheduler.step(total_c_loss) return total_c_loss, total_accuracy
def validate_generator(self, generator, num_worker): total_c_loss = 0.0 total_accuracy = 0.0 # optimizer = self._create_optimizer(self.matchNet, self.lr) # traindata = self.data.get_trainset(batch_size, 0, shuffle=True) valdata = generator val_enqueuer = data_utils.GeneratorEnqueuer(valdata) val_enqueuer.start(workers=num_worker, max_queue_size=valdata.batch_size * 2) val_generator = val_enqueuer.get() total_val_batches = len(valdata) with tqdm.tqdm(total=total_val_batches) as pbar: with torch.no_grad(): for i in range(total_val_batches): # (x_support_set, y_support_set, x_target, y_target) = next(train_generator) batch = next(val_generator) x_support_set, y_support_set, x_target, y_target = self.numpy2tensor( batch) x_support_set = x_support_set.permute(0, 1, 4, 2, 3) x_target = x_target.permute(0, 3, 1, 2) y_support_set = y_support_set.float() y_target = y_target.long() x_support_set = x_support_set.float() x_target = x_target.float() # with torch.no_grad(): acc, c_loss = self.matchNet(x_support_set, y_support_set, x_target, y_target) total_c_loss += c_loss.data[0] total_accuracy += acc.data[0] i += 1 iter_out = f"v_loss: {total_c_loss / i:.{3}}, v_acc: {total_accuracy / i:.{3}}" pbar.set_description(iter_out) pbar.update(1) # self.total_train_iter+=1 total_c_loss = total_c_loss / total_val_batches total_accuracy = total_accuracy / total_val_batches return total_c_loss, total_accuracy