def train(self, dataloader, module, optimizer):
# Set module to train
module.train()
# Shuffle dataloader sampler
if hasattr(dataloader.sampler, "set_epoch"):
dataloader.sampler.set_epoch(module.current_epoch)
# Prepare progress bar
progress_bar = self.train_progress_bar(dataloader,
module.config.datasets.train)
# Start training loop
outputs = []
# For all batches
for i, batch in progress_bar:
# Reset optimizer
optimizer.zero_grad()
# Send samples to GPU and take a training step
batch = sample_to_cuda(batch)
output = module.training_step(batch, i)
# Backprop through loss and take an optimizer step
output['loss'].backward()
optimizer.step()
# Append output to list of outputs
output['loss'] = output['loss'].detach()
outputs.append(output)
print('###model_wrapper.py output', output)
# Update progress bar if in rank 0
if self.is_rank_0:
progress_bar.set_description(
'Epoch {} | Avg.Loss {:.4f}'.format(
module.current_epoch,
self.avg_loss(output['loss'].item())))
# Return outputs for epoch end
return module.training_epoch_end(outputs)
def train(self, dataloader, module, optimizer):
global idx_log
# Set module to train
module.train()
# Shuffle dataloader sampler
if hasattr(dataloader.sampler, "set_epoch"):
dataloader.sampler.set_epoch(module.current_epoch)
# Prepare progress bar
progress_bar = self.train_progress_bar(dataloader,
module.config.datasets.train)
# Start training loop
outputs = []
# For all batches
for i, batch in progress_bar:
# Reset optimizer
optimizer.zero_grad()
# Send samples to GPU and take a training step
batch = sample_to_cuda(batch)
output = module.training_step(batch, i)
# Backprop through loss and take an optimizer step
output['loss'].backward()
optimizer.step()
# Append output to list of outputs
output['loss'] = output['loss'].detach()
outputs.append(output)
# Update progress bar if in rank 0
if self.is_rank_0:
progress_bar.set_description(
'Epoch {} | Avg.Loss {:.4f}'.format(
module.current_epoch,
self.avg_loss(output['loss'].item())))
if i % 20 == 0 and i > 0:
idx_log += 1
metrics = module.training_epoch_end(outputs[-20:])
avg_train_loss = metrics['avg_train-loss']
avg_train_photometric_loss = metrics[
'avg_train-photometric_loss']
avg_train_smoothness_loss = metrics[
'avg_train-smoothness_loss']
avg_train_supervised_loss = metrics[
'avg_train-supervised_loss']
print('##new_trainer', i, " ", idx_log)
# idx = (module.current_epoch+1) * 48 + i
self.writer.add_scalar('Loss/avg_train_loss', avg_train_loss,
idx_log)
self.writer.add_scalar('Loss/avg_train_photometric_loss',
avg_train_photometric_loss, idx_log)
self.writer.add_scalar('Loss/avg_train_smoothness_loss',
avg_train_smoothness_loss, idx_log)
self.writer.add_scalar('Loss/avg_train_supervised_loss',
avg_train_supervised_loss, idx_log)
# Return outputs for epoch end
return module.training_epoch_end(outputs)
def validate(self, dataloaders, module):
# Set module to eval
module.eval()
# Start validation loop
all_outputs = []
# For all validation datasets
for n, dataloader in enumerate(dataloaders):
# Prepare progress bar for that dataset
progress_bar = self.val_progress_bar(
dataloader, module.config.datasets.validation, n)
outputs = []
# For all batches
for i, batch in progress_bar:
# Send batch to GPU and take a validation step
batch = sample_to_cuda(batch)
output = module.validation_step(batch, i, n)
# Append output to list of outputs
outputs.append(output)
# Append dataset outputs to list of all outputs
all_outputs.append(outputs)
# Return all outputs for epoch end
return module.validation_epoch_end(all_outputs)
def validate(self, dataloaders, module):
# Set module to eval
module.eval()
# Start validation loop
all_outputs = []
print('#'* 50)
print(len(dataloaders))
print(type(dataloaders[0]))
# For all validation datasets
for n, dataloader in enumerate(dataloaders):
# Prepare progress bar for that dataset
#progress_bar = self.val_progress_bar(
# dataloader, module.config.datasets.validation, n)
outputs = []
#print(type(dataloader))
#print(len(dataloader))
#print(dataloader[0].keys())
# For all batches
for i, batch in enumerate(dataloader):
print(i)
#print(batch.keys())
#print(dataloader[0].keys())
# Send batch to GPU and take a validation step
batch = sample_to_cuda(batch)
#print('bak')
#print(module.config.datasets.train)
#print(type(module.config.datasets.train))
#print(type(module.config.datasets.train.split))
#print(module.config.datasets.train.split[0])
output = module.validation_step(batch, i, module.config.datasets.train.split[0][:-4], self.model_name_temp, i, n)
# Append output to list of outputs
outputs.append(output)
# Append dataset outputs to list of all outputs
all_outputs.append(outputs)
# Return all outputs for epoch end
return all_outputs #module.validation_epoch_end(all_outputs)
def train(self, dataloader, module, optimizer):
# Set module to train
module.train()
# Shuffle dataloader sampler
if hasattr(dataloader.sampler, "set_epoch"):
dataloader.sampler.set_epoch(module.current_epoch)
# Prepare progress bar
progress_bar = self.train_progress_bar(dataloader,
module.config.datasets.train)
# Start training loop
outputs = []
# For all batches
for i, batch in progress_bar:
# Reset optimizer
optimizer.zero_grad()
# Send samples to GPU and take a training step
batch = sample_to_cuda(batch)
output = module.training_step(batch)
# Backprop through loss and take an optimizer step
output['loss'].backward()
optimizer.step()
# Append output to list of outputs
output['loss'] = output['loss'].detach()
output['inv_depth'] = [t.detach() for t in output['inv_depth']]
# outputs.append(output) # Mike's
# Update progress bar if in rank 0
if module.current_step % module.config.datasets.train.log_freq == 0:
module.log_epoch_step(output, batch, module.current_step,
self.checkpoint, self.is_rank_0)
if self.is_rank_0:
progress_bar.set_description(
'Epoch {} | Avg.Loss {:.4f}'.format(
module.current_epoch,
self.avg_loss(output['loss'].item())))
module.current_step += 1
