def train(self, dataloader, module, optimizer): # Set module to train module.train() # Shuffle dataloader sampler if hasattr(dataloader.sampler, "set_epoch"): dataloader.sampler.set_epoch(module.current_epoch) # Prepare progress bar progress_bar = self.train_progress_bar(dataloader, module.config.datasets.train) # Start training loop outputs = [] # For all batches for i, batch in progress_bar: # Reset optimizer optimizer.zero_grad() # Send samples to GPU and take a training step batch = sample_to_cuda(batch) output = module.training_step(batch, i) # Backprop through loss and take an optimizer step output['loss'].backward() optimizer.step() # Append output to list of outputs output['loss'] = output['loss'].detach() outputs.append(output) print('###model_wrapper.py output', output) # Update progress bar if in rank 0 if self.is_rank_0: progress_bar.set_description( 'Epoch {} | Avg.Loss {:.4f}'.format( module.current_epoch, self.avg_loss(output['loss'].item()))) # Return outputs for epoch end return module.training_epoch_end(outputs)
def train(self, dataloader, module, optimizer): global idx_log # Set module to train module.train() # Shuffle dataloader sampler if hasattr(dataloader.sampler, "set_epoch"): dataloader.sampler.set_epoch(module.current_epoch) # Prepare progress bar progress_bar = self.train_progress_bar(dataloader, module.config.datasets.train) # Start training loop outputs = [] # For all batches for i, batch in progress_bar: # Reset optimizer optimizer.zero_grad() # Send samples to GPU and take a training step batch = sample_to_cuda(batch) output = module.training_step(batch, i) # Backprop through loss and take an optimizer step output['loss'].backward() optimizer.step() # Append output to list of outputs output['loss'] = output['loss'].detach() outputs.append(output) # Update progress bar if in rank 0 if self.is_rank_0: progress_bar.set_description( 'Epoch {} | Avg.Loss {:.4f}'.format( module.current_epoch, self.avg_loss(output['loss'].item()))) if i % 20 == 0 and i > 0: idx_log += 1 metrics = module.training_epoch_end(outputs[-20:]) avg_train_loss = metrics['avg_train-loss'] avg_train_photometric_loss = metrics[ 'avg_train-photometric_loss'] avg_train_smoothness_loss = metrics[ 'avg_train-smoothness_loss'] avg_train_supervised_loss = metrics[ 'avg_train-supervised_loss'] print('##new_trainer', i, " ", idx_log) # idx = (module.current_epoch+1) * 48 + i self.writer.add_scalar('Loss/avg_train_loss', avg_train_loss, idx_log) self.writer.add_scalar('Loss/avg_train_photometric_loss', avg_train_photometric_loss, idx_log) self.writer.add_scalar('Loss/avg_train_smoothness_loss', avg_train_smoothness_loss, idx_log) self.writer.add_scalar('Loss/avg_train_supervised_loss', avg_train_supervised_loss, idx_log) # Return outputs for epoch end return module.training_epoch_end(outputs)
def validate(self, dataloaders, module): # Set module to eval module.eval() # Start validation loop all_outputs = [] # For all validation datasets for n, dataloader in enumerate(dataloaders): # Prepare progress bar for that dataset progress_bar = self.val_progress_bar( dataloader, module.config.datasets.validation, n) outputs = [] # For all batches for i, batch in progress_bar: # Send batch to GPU and take a validation step batch = sample_to_cuda(batch) output = module.validation_step(batch, i, n) # Append output to list of outputs outputs.append(output) # Append dataset outputs to list of all outputs all_outputs.append(outputs) # Return all outputs for epoch end return module.validation_epoch_end(all_outputs)
def validate(self, dataloaders, module): # Set module to eval module.eval() # Start validation loop all_outputs = [] print('#'* 50) print(len(dataloaders)) print(type(dataloaders[0])) # For all validation datasets for n, dataloader in enumerate(dataloaders): # Prepare progress bar for that dataset #progress_bar = self.val_progress_bar( # dataloader, module.config.datasets.validation, n) outputs = [] #print(type(dataloader)) #print(len(dataloader)) #print(dataloader[0].keys()) # For all batches for i, batch in enumerate(dataloader): print(i) #print(batch.keys()) #print(dataloader[0].keys()) # Send batch to GPU and take a validation step batch = sample_to_cuda(batch) #print('bak') #print(module.config.datasets.train) #print(type(module.config.datasets.train)) #print(type(module.config.datasets.train.split)) #print(module.config.datasets.train.split[0]) output = module.validation_step(batch, i, module.config.datasets.train.split[0][:-4], self.model_name_temp, i, n) # Append output to list of outputs outputs.append(output) # Append dataset outputs to list of all outputs all_outputs.append(outputs) # Return all outputs for epoch end return all_outputs #module.validation_epoch_end(all_outputs)
def train(self, dataloader, module, optimizer): # Set module to train module.train() # Shuffle dataloader sampler if hasattr(dataloader.sampler, "set_epoch"): dataloader.sampler.set_epoch(module.current_epoch) # Prepare progress bar progress_bar = self.train_progress_bar(dataloader, module.config.datasets.train) # Start training loop outputs = [] # For all batches for i, batch in progress_bar: # Reset optimizer optimizer.zero_grad() # Send samples to GPU and take a training step batch = sample_to_cuda(batch) output = module.training_step(batch) # Backprop through loss and take an optimizer step output['loss'].backward() optimizer.step() # Append output to list of outputs output['loss'] = output['loss'].detach() output['inv_depth'] = [t.detach() for t in output['inv_depth']] # outputs.append(output) # Mike's # Update progress bar if in rank 0 if module.current_step % module.config.datasets.train.log_freq == 0: module.log_epoch_step(output, batch, module.current_step, self.checkpoint, self.is_rank_0) if self.is_rank_0: progress_bar.set_description( 'Epoch {} | Avg.Loss {:.4f}'.format( module.current_epoch, self.avg_loss(output['loss'].item()))) module.current_step += 1