def _test(self): self.encoder.eval() with torch.no_grad(): macro_f1s = [] test_losses = [] for input_tensor, target_tensor in logger.iterate( "Test", self.test_loader): encoder_hidden = self.encoder.init_hidden( self.device).double().to(self.device) input_tensor = input_tensor.to(self.device).unsqueeze(1) target_tensor = target_tensor.to(self.device).double() encoder_output, encoder_hidden = self.encoder( input_tensor, encoder_hidden) test_loss = self.loss(encoder_output, target_tensor) macro_f1 = f1_score( y_true=target_tensor.cpu().detach().numpy().ravel(), y_pred=encoder_output.cpu().detach().to( torch.int32).numpy().ravel(), average='macro') test_losses.append(test_loss) macro_f1s.append(macro_f1) logger.store(test_loss=np.mean(test_losses)) logger.store(accuracy=np.mean(macro_f1s)) logger.write()
def loop(): logger.info(a=2, b=1) logger.add_indicator('loss_ma', IndicatorType.queue, IndicatorOptions(queue_size=10)) for i in range(10): logger.add_global_step(1) logger.store(loss=100 / (i + 1), loss_ma=100 / (i + 1)) logger.write() if (i + 1) % 2 == 0: logger.new_line() time.sleep(2)
def _test(self): self.model.eval() test_loss = 0 correct = 0 with torch.no_grad(): for data, target in logger.iterate("Test", self.test_loader): data, target = data.to(self.device), target.to(self.device) output = self.model(data) test_loss += F.cross_entropy(output, target, reduction='sum').item() pred = output.argmax(dim=1, keepdim=True) correct += pred.eq(target.view_as(pred)).sum().item() logger.store(test_loss=test_loss / len(self.test_loader.dataset)) logger.store(accuracy=correct / len(self.test_loader.dataset))
def _test(self): self.model.eval() test_loss = 0 correct = 0 idx = 0 with torch.no_grad(): for data, target in logger.iterate("Test", self.test_loader): data, target = data.to(self.device), target.to(self.device) output = self.model(data) loss = F.nll_loss(output, target, reduction='none') indexes = [idx + i for i in range(self.test_loader.batch_size)] values = list(loss.cpu().numpy()) logger.store('test_sample_loss', (indexes, values)) test_loss += float(np.sum(loss.cpu().numpy())) pred = output.argmax(dim=1, keepdim=True) values = list(pred.cpu().numpy()) logger.store('test_sample_pred', (indexes, values)) correct += pred.eq(target.view_as(pred)).sum().item() idx += self.test_loader.batch_size # Add test loss and accuracy to logger logger.store(test_loss=test_loss / len(self.test_loader.dataset)) logger.store(accuracy=correct / len(self.test_loader.dataset))
def _train(self): self.model.train() for i, (data, target) in logger.enum("Train", self.train_loader): data, target = data.to(self.device), target.to(self.device) self.optimizer.zero_grad() output = self.model(data) loss = F.cross_entropy(output, target) loss.backward() self.optimizer.step() logger.store(train_loss=loss) logger.add_global_step() if i % self.train_log_interval == 0: logger.write()
def test(model, device, test_loader): with logger.section("Test", total_steps=len(test_loader)): model.eval() test_loss = 0 correct = 0 with torch.no_grad(): for batch_idx, (data, target) in enumerate(test_loader): data, target = data.to(device), target.to(device) output = model(data) test_loss += F.nll_loss(output, target, reduction='sum').item() pred = output.argmax(dim=1, keepdim=True) correct += pred.eq(target.view_as(pred)).sum().item() logger.progress(batch_idx + 1) # Add test loss and accuracy to logger logger.store(test_loss=test_loss / len(test_loader.dataset)) logger.store(accuracy=correct / len(test_loader.dataset))
def test(session: tf.Session, loss_value, accuracy_value, batches): with logger.section("Test", total_steps=batches): test_loss = 0 correct = 0 batch_idx = -1 while True: batch_idx += 1 try: l, a = session.run([loss_value, accuracy_value]) test_loss += l correct += a except tf.errors.OutOfRangeError: break logger.progress(batch_idx + 1) logger.store(test_loss=test_loss / batches) logger.store(accuracy=correct / batches)
def _train(self): self.model.train() for i, (data, target) in logger.enumerator("Train", self.train_loader): data, target = data.to(self.device), target.to(self.device) self.optimizer.zero_grad() output = self.model(data) loss = F.nll_loss(output, target) loss.backward() self.optimizer.step() # Add training loss to the logger. # The logger will queue the values and output the mean logger.store(train_loss=loss.item()) logger.add_global_step() # Print output to the console if i % self.train_log_interval == 0: # Output the indicators logger.write()
def train(args, session: tf.Session, loss_value, train_op, batches, epoch): with logger.section("Train", total_steps=batches): batch_idx = -1 while True: batch_idx += 1 try: l, _ = session.run([loss_value, train_op]) except tf.errors.OutOfRangeError: break # Add training loss to the logger. # The logger will queue the values and output the mean logger.store(train_loss=l) logger.progress(batch_idx + 1) logger.set_global_step(epoch * batches + batch_idx) # Print output to the console if batch_idx % args.log_interval == 0: # Output the indicators logger.write()
def train(args, model, device, train_loader, optimizer, epoch): with logger.section("Train", total_steps=len(train_loader)): model.train() for batch_idx, (data, target) in enumerate(train_loader): data, target = data.to(device), target.to(device) optimizer.zero_grad() output = model(data) loss = F.nll_loss(output, target) loss.backward() optimizer.step() # Add training loss to the logger. # The logger will queue the values and output the mean logger.store(train_loss=loss.item()) logger.progress(batch_idx + 1) logger.set_global_step(epoch * len(train_loader) + batch_idx) # Print output to the console if batch_idx % args.log_interval == 0: # Output the indicators logger.write()
def _train(self): for i, (input_tensor, target_tensor) in logger.enum("Train", self.train_loader): encoder_hidden = self.encoder.init_hidden(self.device).double().to( self.device) input_tensor = input_tensor.to(self.device).unsqueeze(1) target_tensor = target_tensor.to(self.device).double() self.optimizer.zero_grad() encoder_output, encoder_hidden = self.encoder( input_tensor, encoder_hidden) train_loss = self.loss(encoder_output, target_tensor) train_loss.backward() self.optimizer.step() logger.store(loss=train_loss.item()) logger.add_global_step() logger.write()
def _train(self): for i, (images, _) in logger.enum("Train", self.train_loader): targets_real = torch.empty(images.size(0), 1, device=self.device).uniform_(0.8, 1.0) targets_fake = torch.empty(images.size(0), 1, device=self.device).uniform_(0.0, 0.2) images = images.to(self.device) self.optimizer_D.zero_grad() logits_real = self.discriminator(images) fake_images = self.generator( noise(self.device, self.batch_size, self.noise_dim)).detach() logits_fake = self.discriminator(fake_images) discriminator_loss = DLoss(logits_real, logits_fake, targets_real, targets_fake) discriminator_loss.backward() self.optimizer_D.step() self.optimizer_G.zero_grad() fake_images = self.generator(noise(self.device, self.batch_size, self.noise_dim)) logits_fake = self.discriminator(fake_images) generator_loss = GLoss(logits_fake, targets_real) generator_loss.backward() self.optimizer_G.step() logger.store(G_Loss=generator_loss.item()) logger.store(D_Loss=discriminator_loss.item()) logger.add_global_step() for j in range(1, 10): img = fake_images[j].squeeze() logger.store('generated', img)
def main(): args = parse_args() use_cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) device = torch.device("cuda" if use_cuda else "cpu") kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} # Loading data with logger.section("Loading data"): train_loader = torch.utils.data.DataLoader(datasets.MNIST( './data', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])), batch_size=args.batch_size, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader( datasets.MNIST('./data', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])), batch_size=args.test_batch_size, shuffle=True, **kwargs) # Model creation with logger.section("Create model"): model = Net().to(device) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) # Specify indicators logger.add_indicator("train_loss", queue_limit=10, is_print=True) logger.add_indicator("test_loss", is_histogram=False, is_print=True) logger.add_indicator("accuracy", is_histogram=False, is_print=True) for name, param in model.named_parameters(): if param.requires_grad: logger.add_indicator(name, is_histogram=True, is_print=False) logger.add_indicator(f"{name}_grad", is_histogram=True, is_print=False) # Start the experiment EXPERIMENT.start_train() # Loop through the monitored iterator for epoch in logger.loop(range(0, args.epochs)): # Delayed keyboard interrupt handling to use # keyboard interrupts to end the loop. # This will capture interrupts and finish # the loop at the end of processing the iteration; # i.e. the loop won't stop in the middle of an epoch. try: with logger.delayed_keyboard_interrupt(): # Training and testing train(args, model, device, train_loader, optimizer, epoch) test(model, device, test_loader) # Add histograms with model parameter values and gradients for name, param in model.named_parameters(): if param.requires_grad: logger.store(name, param.data.cpu().numpy()) logger.store(f"{name}_grad", param.grad.cpu().numpy()) # Clear line and output to console logger.write() # Output the progress summaries to `trial.yaml` and # to the python file header logger.save_progress() # Clear line and go to the next line; # that is, we add a new line to the output # at the end of each epoch logger.new_line() # Handled delayed interrupt except KeyboardInterrupt: logger.finish_loop() logger.new_line() logger.log("\nKilling loop...") break
def store_model_indicators(model: torch.nn.Module, model_name: str = "model"): for name, param in model.named_parameters(): if param.requires_grad: logger.store(f"{model_name}.{name}", param) logger.store(f"{model_name}.{name}.grad", param.grad)
for global_step in logger.loop(range(50)): # You can set the global step explicitly with # 'logger.set_global_step(global_step)' # Handle Keyboard Interrupts try: with logger.delayed_keyboard_interrupt(): # A sample monitored section inside iterator with logger.section("sample"): time.sleep(0.5) # A silent section is used only to organize code. # It produces no output with logger.section("logging", is_silent=True): # Store values logger.store(reward=global_step / 3.0, fps=12) # Store more values for i in range(global_step, global_step + 10): logger.store('loss', i) logger.store(advantage_reward=(i, i * 2)) # Another silent section with logger.section("process_samples", is_silent=True): time.sleep(0.5) # A third section with an inner loop with logger.section("train", total_steps=100): # Let it run for multiple iterations. # We'll track the progress of that too for i in range(100): time.sleep(0.01)