def edge_detection(depth):
if use_cuda:
get_edge = sobel.Sobel().cuda()
else:
get_edge = sobel.Sobel()
edge_xy = get_edge(depth)
edge_sobel = torch.pow(edge_xy[:, 0, :, :], 2) + \
torch.pow(edge_xy[:, 1, :, :], 2)
edge_sobel = torch.sqrt(edge_sobel)
return edge_sobel
def train(train_loader, model, optimizer, epoch):
criterion = nn.L1Loss()
batch_time = AverageMeter()
losses = AverageMeter()
model.train()
cos = nn.CosineSimilarity(dim=1, eps=0)
get_gradient = sobel.Sobel().cuda()
end = time.time()
for i, sample_batched in enumerate(train_loader):
image, depth = sample_batched['image'], sample_batched['depth']
#depth = depth.cuda(async=True)
depth = depth.cuda()
image = image.cuda()
image = torch.autograd.Variable(image)
depth = torch.autograd.Variable(depth)
ones = torch.ones(depth.size(0), 1, depth.size(2),depth.size(3)).float().cuda()
ones = torch.autograd.Variable(ones)
optimizer.zero_grad()
output = model(image)
depth_grad = get_gradient(depth)
output_grad = get_gradient(output)
depth_grad_dx = depth_grad[:, 0, :, :].contiguous().view_as(depth)
depth_grad_dy = depth_grad[:, 1, :, :].contiguous().view_as(depth)
output_grad_dx = output_grad[:, 0, :, :].contiguous().view_as(depth)
output_grad_dy = output_grad[:, 1, :, :].contiguous().view_as(depth)
depth_normal = torch.cat((-depth_grad_dx, -depth_grad_dy, ones), 1)
output_normal = torch.cat((-output_grad_dx, -output_grad_dy, ones), 1)
# depth_normal = F.normalize(depth_normal, p=2, dim=1)
# output_normal = F.normalize(output_normal, p=2, dim=1)
loss_depth = torch.log(torch.abs(output - depth) + 0.5).mean()
loss_dx = torch.log(torch.abs(output_grad_dx - depth_grad_dx) + 0.5).mean()
loss_dy = torch.log(torch.abs(output_grad_dy - depth_grad_dy) + 0.5).mean()
loss_normal = torch.abs(1 - cos(output_normal, depth_normal)).mean()
loss = loss_depth + loss_normal + (loss_dx + loss_dy)
#losses.update(loss.data[0], image.size(0))
losses.update(loss.data.item(), image.size(0))
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
batchSize = depth.size(0)
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.sum:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})'
.format(epoch, i, len(train_loader), batch_time=batch_time, loss=losses))
def get_loss_function():
get_gradient = sobel.Sobel().get_gradient
def loss_function(disp, pred):
# disp loss
loss_disp = K.mean(K.abs(pred - disp))
# grad loss
pred_grad = get_gradient(pred)
disp_grad = get_gradient(disp)
pred_grad_dx = K.expand_dims(pred_grad[..., 0], axis=-1)
pred_grad_dy = K.expand_dims(pred_grad[..., 1], axis=-1)
disp_grad_dx = K.expand_dims(disp_grad[..., 0], axis=-1)
disp_grad_dy = K.expand_dims(disp_grad[..., 1], axis=-1)
loss_dx = K.mean(K.abs(pred_grad_dx - disp_grad_dx))
loss_dy = K.mean(K.abs(pred_grad_dy - disp_grad_dy))
# normal loss
ones = K.ones_like(disp_grad_dx)
pred_normal = K.concatenate([-pred_grad_dx, -pred_grad_dy, ones],
axis=-1)
disp_normal = K.concatenate([-disp_grad_dx, -disp_grad_dy, ones],
axis=-1)
loss_normal = K.mean(
K.abs(1 - cos_similarity(pred_normal, disp_normal, axis=-1)))
return loss_disp + (loss_dx + loss_dy) + loss_normal
return loss_function
def testing_loss(depth , output, losses, batchSize):
ones = torch.ones(depth.size(0), 1, depth.size(2),depth.size(3)).float().cuda()
get_gradient = sobel.Sobel().cuda()
cos = nn.CosineSimilarity(dim=1, eps=0)
depth_grad = get_gradient(depth)
output_grad = get_gradient(output)
depth_grad_dx = depth_grad[:, 0, :, :].contiguous().view_as(depth)
depth_grad_dy = depth_grad[:, 1, :, :].contiguous().view_as(depth)
output_grad_dx = output_grad[:, 0, :, :].contiguous().view_as(depth)
output_grad_dy = output_grad[:, 1, :, :].contiguous().view_as(depth)
depth_normal = torch.cat((-depth_grad_dx, -depth_grad_dy, ones), 1)
output_normal = torch.cat((-output_grad_dx, -output_grad_dy, ones), 1)
loss_depth = torch.log(torch.abs(output - depth) + 0.5).mean()
loss_dx = torch.log(torch.abs(output_grad_dx - depth_grad_dx) + 0.5).mean()
loss_dy = torch.log(torch.abs(output_grad_dy - depth_grad_dy) + 0.5).mean()
loss_normal = torch.abs(1 - cos(output_normal, depth_normal)).mean()
loss = loss_depth + loss_normal + (loss_dx + loss_dy)
losses.update(loss.data, batchSize)
def train(train_loader, model, optimizer, epoch):
criterion = nn.L1Loss()
batch_time = AverageMeter()
losses = AverageMeter()
model.train()
cos = nn.CosineSimilarity(dim=1, eps=0)
get_gradient = sobel.Sobel().cuda()
global args
args = parser.parse_args()
end = time.time()
for i, sample_batched in enumerate(train_loader):
image, depth = sample_batched['image'], sample_batched['depth']
depth = depth.cuda(async=True)
image = image.cuda()
image = torch.autograd.Variable(image)
depth = torch.autograd.Variable(depth)
ones = torch.ones(depth.size(0), 1, depth.size(2),
depth.size(3)).float().cuda()
ones = torch.autograd.Variable(ones)
optimizer.zero_grad()
output = model(image)
if i % 200 == 0:
x = output[0]
x = x.view([220, 220])
x = x.cpu().detach().numpy()
x = x * 100000
x2 = depth[0]
print(x)
x2 = x2.view([220, 220])
x2 = x2.cpu().detach().numpy()
x2 = x2 * 100000
print(x2)
x = x.astype('uint16')
cv2.imwrite(args.data + str(i) + '_out.png', x)
x2 = x2.astype('uint16')
cv2.imwrite(args.data + str(i) + '_out2.png', x2)
depth_grad = get_gradient(depth)
output_grad = get_gradient(output)
depth_grad_dx = depth_grad[:, 0, :, :].contiguous().view_as(depth)
depth_grad_dy = depth_grad[:, 1, :, :].contiguous().view_as(depth)
output_grad_dx = output_grad[:, 0, :, :].contiguous().view_as(depth)
output_grad_dy = output_grad[:, 1, :, :].contiguous().view_as(depth)
depth_normal = torch.cat((-depth_grad_dx, -depth_grad_dy, ones), 1)
output_normal = torch.cat((-output_grad_dx, -output_grad_dy, ones), 1)
loss_depth = torch.log(torch.abs(output - depth) + 0.5).mean()
loss_dx = torch.log(torch.abs(output_grad_dx - depth_grad_dx) +
0.5).mean()
loss_dy = torch.log(torch.abs(output_grad_dy - depth_grad_dy) +
0.5).mean()
loss_normal = torch.abs(1 - cos(output_normal, depth_normal)).mean()
loss = loss_depth + loss_normal + (loss_dx + loss_dy)
losses.update(loss.data, image.size(0))
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
batchSize = depth.size(0)
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.sum:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
loss=losses))
log_value('training loss', losses.avg, epoch)
def train(train_loader, model, optimizer, epoch):
criterion = nn.L1Loss()
batch_time = AverageMeter()
losses = AverageMeter()
model.train()
cos = nn.CosineSimilarity(dim=1, eps=0)
get_gradient = sobel.Sobel().cuda()
end = time.time()
for i, sample_batched in enumerate(train_loader):
image, depth = sample_batched['image'], sample_batched['depth']
# print('--------Input information:')
# print('image0 = {0}, image1 = {1}, image2 = {2}, image3 = {3}'.format(image.size(0), image.size(1), image.size(2), image.size(3)))
depth = depth.cuda(async=True) # This can be used to overlap data transfers with computation.
image = image.cuda()
image = torch.autograd.Variable(image) # torch.autograd provides classes and functions implementing automatic differentiation of arbitrary scalar valued functions.
depth = torch.autograd.Variable(depth)
ones = torch.ones(depth.size(0), 1, depth.size(2),depth.size(3)).float().cuda() # batchSize = depth.size(0)
ones = torch.autograd.Variable(ones)
optimizer.zero_grad() # we need to set the gradients to zero before starting to do backpropragation because PyTorch accumulates the gradients on subsequent backward passes.
# the default action is to accumulate (i.e. sum) the gradients on every loss.backward() call.
output = model(image)
# print('--------Output information:')
# print('output0 = {0}, output1 = {1}, output2 = {2}, output3 = {3}'.format(output.size(0), output.size(1), output.size(2), output.size(3)))
depth_grad = get_gradient(depth)
output_grad = get_gradient(output)
depth_grad_dx = depth_grad[:, 0, :, :].contiguous().view_as(depth) # depth_grad.shape = (-1, 2, x.size(2), x.size(3))
depth_grad_dy = depth_grad[:, 1, :, :].contiguous().view_as(depth)
output_grad_dx = output_grad[:, 0, :, :].contiguous().view_as(depth)
output_grad_dy = output_grad[:, 1, :, :].contiguous().view_as(depth)
# print('--------Output_grad_dx information:')
# print('output_grad_dx0 = {0}, output_grad_dx1 = {1}, output_grad_dx2 = {2}, output_grad_dx3 = {3}'.format(output_grad_dx.size(0), output_grad_dx.size(1), output_grad_dx.size(2), output_grad_dx.size(3)))
depth_normal = torch.cat((-depth_grad_dx, -depth_grad_dy, ones), 1)
output_normal = torch.cat((-output_grad_dx, -output_grad_dy, ones), 1)
# print('--------output_normal information:')
# print('output_normal0 = {0}, output_normal1 = {1}, output_normal2 = {2}, output_normal3 = {3}'.format(output_normal.size(0), output_normal.size(1), output_normal.size(2), output_normal.size(3)))
# depth_normal = F.normalize(depth_normal, p=2, dim=1)
# output_normal = F.normalize(output_normal, p=2, dim=1)
loss_depth = torch.log(torch.abs(output - depth) + 0.5).mean()
loss_dx = torch.log(torch.abs(output_grad_dx - depth_grad_dx) + 0.5).mean()
loss_dy = torch.log(torch.abs(output_grad_dy - depth_grad_dy) + 0.5).mean()
loss_normal = torch.abs(1 - cos(output_normal, depth_normal)).mean()
# print('--------loss information:')
# print('loss_depth.info = ', loss_depth)
# print('loss_dx.info = ', loss_dx)
# print('loss_dy.info = ', loss_dy)
# print('loss_normal.info = ', loss_normal)
loss = loss_depth + loss_normal + (loss_dx + loss_dy)
# print('loss.info = ', loss)
# print('+++++++')
# print('loss.data[0] = ', loss.data)
# print('loss.data[0].item = ', loss.item)
# print('image.size(0) = ', image.size(0))
# losses.update(loss.data[0], image.size(0)) # bai2 old PyTorch=0.4.1
losses.update(loss.data, image.size(0)) # bai2 new PyTorch>=0.5
loss.backward() # The graph is used by loss.backward() to compute gradients.
optimizer.step() # Performs a parameter update based on the current gradient (stored in .grad attribute of a parameter) and the update rule
batch_time.update(time.time() - end)
end = time.time()
batchSize = depth.size(0)
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.sum:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})'
.format(epoch, i, len(train_loader), batch_time=batch_time, loss=losses))
def train(train_loader, model,model_final, optimizer, epoch,logger ):
batch_time = AverageMeter()
losses = AverageMeter()
model.train()
#梯度与预先相似度
cos = nn.CosineSimilarity(dim=1, eps=0)
get_gradient = sobel.Sobel().cuda()
end = time.time()
for i, sample_batched in enumerate(train_loader):
total_step = len(train_loader)
image, depth,focal,_ = sample_batched['image'], sample_batched['depth'], sample_batched['focal'],sample_batched['depth1']
depth = depth.cuda()
image = image.cuda()
focal = focal .cuda()
focal = torch.cat(torch.chunk(focal, 12, dim=1), dim=0) # 2*12*3*256*256
focal = focal.squeeze(1)
image = torch.autograd.Variable(image)
depth = torch.autograd.Variable(depth)
ones = torch.ones(depth.size(0), 1, depth.size(2), depth.size(3)).float().cuda()
ones = torch.autograd.Variable(ones)
optimizer.zero_grad()
out = model(image)
output = model_final(out, focal)
# 梯度
depth_grad = get_gradient(depth)
output_grad = get_gradient(output)
depth_grad_dx = depth_grad[:, 0, :, :].contiguous().view_as(depth)
depth_grad_dy = depth_grad[:, 1, :, :].contiguous().view_as(depth)
output_grad_dx = output_grad[:, 0, :, :].contiguous().view_as(depth)
output_grad_dy = output_grad[:, 1, :, :].contiguous().view_as(depth)
# 法线
depth_normal = torch.cat((-depth_grad_dx, -depth_grad_dy, ones), 1)
output_normal = torch.cat((-output_grad_dx, -output_grad_dy, ones), 1)
# loss
loss_depth = torch.log(torch.abs(output - depth) + 1.0).mean()
loss_dx = torch.log(torch.abs(output_grad_dx - depth_grad_dx) + 1.0).mean()
loss_dy = torch.log(torch.abs(output_grad_dy - depth_grad_dy) +1.0).mean()
loss_normal = torch.abs(1 - cos(output_normal, depth_normal)).mean()
loss = loss_depth + loss_normal + (loss_dx + loss_dy)
logger.log_value('loss',loss,step=i+(epoch-1)*total_step)
losses.update(loss.item(), image.size(0))
loss.backward()
torch.cuda.empty_cache()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.sum:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
.format(epoch, i, len(train_loader), batch_time=batch_time, loss=losses))
def train(train_loader, model, model2, optimizer, epoch):
batch_time = AverageMeter()
losses = AverageMeter()
totalNumber = 0
errorSum = {
'MSE': 0,
'RMSE': 0,
'ABS_REL': 0,
'LG10': 0,
'MAE': 0,
'DELTA1': 0,
'DELTA2': 0,
'DELTA3': 0
}
model.eval()
model2.train()
cos = nn.CosineSimilarity(dim=1, eps=0)
get_gradient = sobel.Sobel().cuda()
end = time.time()
for i, sample_batched in enumerate(train_loader):
image, depth_ = sample_batched['image'], sample_batched['depth']
image = torch.autograd.Variable(image).cuda()
depth_ = torch.autograd.Variable(depth_).cuda()
ones = torch.ones(depth_.size(0), 1, depth_.size(2),
depth_.size(3)).float().cuda()
ones = torch.autograd.Variable(ones)
depth = model(image.clone()).detach()
optimizer.zero_grad()
mask = model2(image)
output = model(image * mask)
depth_grad = get_gradient(depth)
output_grad = get_gradient(output)
depth_grad_dx = depth_grad[:, 0, :, :].contiguous().view_as(depth)
depth_grad_dy = depth_grad[:, 1, :, :].contiguous().view_as(depth)
output_grad_dx = output_grad[:, 0, :, :].contiguous().view_as(depth)
output_grad_dy = output_grad[:, 1, :, :].contiguous().view_as(depth)
depth_normal = torch.cat((-depth_grad_dx, -depth_grad_dy, ones), 1)
output_normal = torch.cat((-output_grad_dx, -output_grad_dy, ones), 1)
loss_depth = torch.log(torch.abs(output - depth) + 0.5).mean()
loss_dx = torch.log(torch.abs(output_grad_dx - depth_grad_dx) +
0.5).mean()
loss_dy = torch.log(torch.abs(output_grad_dy - depth_grad_dy) +
0.5).mean()
loss_normal = torch.abs(1 - cos(output_normal, depth_normal)).mean()
loss_rec = loss_depth + loss_normal + (loss_dx + loss_dy)
loss_sparse = mask.mean()
loss = loss_rec + loss_sparse * 5
losses.update(loss_sparse.data[0], image.size(0))
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
batchSize = depth.size(0)
errors = util.evaluateError(output, depth)
errorSum = util.addErrors(errorSum, errors, batchSize)
totalNumber = totalNumber + batchSize
averageError = util.averageErrors(errorSum, totalNumber)
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.sum:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
loss=losses))
print('errors: ', averageError)
def train_model(
train_ds: tf.data.Dataset,
dev_ds: tf.data.Dataset,
model: nn.Module,
optimizer: optim.Optimizer,
lr_scheduler: optim.lr_scheduler._LRScheduler,
args: argparse.Namespace,
) -> nn.Module:
device = model_utils.get_device()
loss_fn = model_utils.depth_proportional_loss
val_loss_fn = model_utils.l1_norm_loss
best_val_loss = torch.tensor(float('inf'))
saved_checkpoints = []
writer = SummaryWriter(log_dir=f'{args.log_dir}/{args.experiment}')
cos = nn.CosineSimilarity(dim=1, eps=0)
get_gradient: nn.Module = sobel.Sobel().to(device)
for e in range(1, args.train_epochs + 1):
print(f'Training epoch {e}...')
if args.use_scheduler:
lr_scheduler.step()
# Training portion
torch.cuda.empty_cache()
torch.set_grad_enabled(True)
with tqdm(total=args.train_batch_size * len(train_ds)) as progress_bar:
model.train()
for i, (x_batch_orig,
y_batch) in enumerate(train_ds.as_numpy_iterator()):
x_batch, y_batch = model_utils.preprocess_training_example(
x_batch_orig, y_batch)
y_blurred = model_utils.blur_depth_map(y_batch)
ones = torch.ones(y_batch.shape,
dtype=torch.float32,
device=device)
# Forward pass on model
optimizer.zero_grad()
y_pred = model(x_batch)
depth_grad = get_gradient(y_blurred)
output_grad = get_gradient(y_pred)
depth_grad_dx = depth_grad[:, 0, :, :].contiguous().view_as(
y_blurred)
depth_grad_dy = depth_grad[:, 1, :, :].contiguous().view_as(
y_batch)
output_grad_dx = output_grad[:, 0, :, :].contiguous().view_as(
y_blurred)
output_grad_dy = output_grad[:, 1, :, :].contiguous().view_as(
y_batch)
depth_normal = torch.cat(
(-depth_grad_dx, -depth_grad_dy, ones), 1)
output_normal = torch.cat(
(-output_grad_dx, -output_grad_dy, ones), 1)
loss_depth = torch.log(torch.abs(y_pred - y_batch) +
0.5).mean()
loss_dx = torch.log(
torch.abs(output_grad_dx - depth_grad_dx) + 0.5).mean()
loss_dy = torch.log(
torch.abs(output_grad_dy - depth_grad_dy) + 0.5).mean()
loss_normal = torch.abs(
1 - cos(output_normal, depth_normal)).mean()
loss = loss_depth + loss_normal + (loss_dx + loss_dy)
# Backward pass and optimization
loss.backward()
optimizer.step()
progress_bar.update(len(x_batch))
progress_bar.set_postfix(loss=loss.item())
writer.add_scalar("train/Loss", loss,
((e - 1) * len(train_ds) + i) *
args.train_batch_size)
# Periodically save a diagram
if (i + 1) % args.picture_frequency == 0:
model_utils.make_diagram(
np.transpose(x_batch_orig, (0, 3, 1, 2)),
x_batch.cpu().numpy(),
y_batch.cpu().numpy(),
y_pred.cpu().detach().numpy(),
f'{args.save_path}/{args.experiment}/diagram_{e}_{i+1}.png',
)
del x_batch
del y_batch
del y_blurred
del y_pred
del loss
# Validation portion
torch.cuda.empty_cache()
torch.set_grad_enabled(False)
with tqdm(total=args.dev_batch_size * len(dev_ds)) as progress_bar:
model.eval()
val_loss = 0.0
num_batches_processed = 0
total_pixels = 0
total_examples = 0
squared_error = 0
rel_error = 0
log_error = 0
threshold1 = 0 # 1.25
threshold2 = 0 # 1.25^2
threshold3 = 0 # corresponds to 1.25^3
for i, (x_batch, y_batch) in enumerate(dev_ds.as_numpy_iterator()):
x_batch, y_batch = model_utils.preprocess_test_example(
x_batch, y_batch)
# Forward pass on model in validation environment
y_pred = model(x_batch)
# TODO: Process y_pred in whatever way inference requires.
loss = val_loss_fn(y_pred, y_batch)
val_loss += loss.item()
num_batches_processed += 1
nanmask = getNanMask(y_batch)
total_pixels = torch.sum(~nanmask)
total_examples += x_batch.shape[0]
# RMS, REL, LOG10, threshold calculation
squared_error += (
torch.sum(torch.pow(y_pred - y_batch, 2)).item() /
total_pixels)**0.5
rel_error += torch.sum(
removeNans(torch.abs(y_pred - y_batch) /
y_batch)).item() / total_pixels
log_error += torch.sum(
torch.abs(
removeNans(torch.log10(y_pred)) - removeNans(
torch.log10(y_batch)))).item() / total_pixels
threshold1 += torch.sum(
torch.max(y_pred / y_batch, y_batch /
y_pred) < 1.25).item() / total_pixels
threshold2 += torch.sum(
torch.max(y_pred / y_batch, y_batch /
y_pred) < 1.25**2).item() / total_pixels
threshold3 += torch.sum(
torch.max(y_pred / y_batch, y_batch /
y_pred) < 1.25**3).item() / total_pixels
progress_bar.update(len(x_batch))
progress_bar.set_postfix(val_loss=val_loss /
num_batches_processed)
writer.add_scalar("Val/Loss", loss,
((e - 1) * len(dev_ds) + i) *
args.dev_batch_size)
del x_batch
del y_batch
del y_pred
del loss
writer.add_scalar("Val/RMS", squared_error / total_examples, e)
writer.add_scalar("Val/REL", rel_error / total_examples, e)
writer.add_scalar("Val/LOG10", log_error / total_examples, e)
writer.add_scalar("Val/delta1", threshold1 / total_examples, e)
writer.add_scalar("Val/delta2", threshold2 / total_examples, e)
writer.add_scalar("Val/delta3", threshold3 / total_examples, e)
# Save model if it's the best one yet.
if val_loss / num_batches_processed < best_val_loss:
best_val_loss = val_loss / num_batches_processed
filename = f'{args.save_path}/{args.experiment}/{model.__class__.__name__}_best_val.checkpoint'
model_utils.save_model(model, filename)
print(f'Model saved!')
print(f'Best validation loss yet: {best_val_loss}')
# Save model on checkpoints.
if e % args.checkpoint_freq == 0:
filename = f'{args.save_path}/{args.experiment}/{model.__class__.__name__}_epoch_{e}.checkpoint'
model_utils.save_model(model, filename)
print(f'Model checkpoint reached!')
saved_checkpoints.append(filename)
# Delete checkpoints if there are too many
while len(saved_checkpoints) > args.num_checkpoints:
os.remove(saved_checkpoints.pop(0))
return model
def train_model(dataloader, rmodel, model, optimizer, start_epoch, epochs,
output_dir):
# # initialize grid for point selection
# step = 5
# grid = torch.zeros([114,152],dtype=torch.uint8)
# x = np.arange(0,114,step)
# y = np.arange(0,152,step)
# X,Y = np.meshgrid(x,y)
# grid[X.ravel(),Y.ravel()]=1
# grid = grid.view(-1).cuda()
if not os.path.exists(output_dir):
os.makedirs(output_dir)
best_acc = np.inf
valfile = open(output_dir + 'validation.log', 'w')
trainfile = open(output_dir + 'training.log', 'w')
resultsfile = open(output_dir + 'results.log', 'w')
# d1 = 114
# d2 = 152
# xx, yy = torch.meshgrid([torch.arange(d1), torch.arange(d2)])
# xx = xx.cuda()
# yy = yy.cuda()
for epoch in range(start_epoch, epochs):
# adjust_learning_rate(optimizer, epoch)
for phase in ['train', 'val']:
if phase == 'train':
rmodel.train() # Set model to training mode
model.eval() # Set model to training mode
else:
continue
rmodel.eval() # Set model to evaluate mode
model.eval() # Set model to evaluate mode
since = time.time()
threshold_percent = AverageMeter()
batch_time = AverageMeter()
losses = AverageMeter()
totalNumber = 0
acc = AverageMeter()
errorSum = {
'MSE': 0,
'RMSE': 0,
'ABS_REL': 0,
'LG10': 0,
'MAE': 0,
'DELTA1': 0,
'DELTA2': 0,
'DELTA3': 0
}
print('Epoch {}/{}'.format(epoch, epochs - 1))
print('-' * 10)
cos = nn.CosineSimilarity(dim=1, eps=0)
get_gradient = sobel.Sobel().cuda()
mu = 1
lamda = 1
alpha = 0.5
# Iterate over data.
end = time.time()
for i, sample_batched in enumerate(dataloader[phase]):
image, depth, mask, imagedepth, imagemask = \
sample_batched['image'].cuda(), sample_batched['depth'].cuda(), sample_batched['mask'].cuda(), sample_batched['imagedepth'].cuda(), sample_batched['imagemask'].cuda()
bs = image.size(0)
depth_mean = 0 #1.8698
depth_std = 10 #1.6716
# mi, _ = depth.view(bs,-1).min(1)
# mi = mi.view(-1,1,1,1)
# ma, _ = depth.view(bs,-1).max(1)
# ma = ma.view(-1,1,1,1)
# depth = (depth - mi)/(ma-mi)
depth = (depth - depth_mean) / depth_std
# zero the parameter gradients
optimizer.zero_grad()
zero_points = torch.zeros_like(imagemask)
base_input = image #torch.cat((image,imagemask, zero_points),dim=1)
with torch.set_grad_enabled(False):
output = model(base_input)
# output2x = nn.functional.interpolate(output, size=None, scale_factor=2)
# output = output.squeeze().view(114,152).data.cpu().float().numpy()
# matplotlib.image.imsave(output_dir+'base'+'.png', output)
# # normalize
# mi, _ = output.view(bs,-1).min(1)
# mi = mi.view(-1,1,1,1)
# ma, _ = output.view(bs,-1).max(1)
# ma = ma.view(-1,1,1,1)
# output = (output - mi)/(ma-mi)
output = (output - depth_mean) / depth_std
# # imagemask = image[:,3,:,:].unsqueeze(1)
P = torch.zeros_like(output)
# diff_map = (output/depth - 1)*mask # < 0 if output point is relatively farther
# adj_diff_map = torch.abs(diff_map)>0.5
# # 3 levels of ordinal relatinos
# # abs_diff_map = torch.abs(diff_map)
# # K = 3
diff_map = (output / depth - 1) * mask
diff_map_reverse = (depth / output - 1) * mask
adj_diff_map = torch.max(
torch.abs(diff_map),
torch.abs(diff_map_reverse)) > 0.25
threshold_percent.update(adj_diff_map.sum() / mask.sum(),
1)
# # diff_map = output - depth
# # adj_diff_map = torch.zeros_like(diff_map).byte()
# # for j in range(bs):
# # subdiff = diff_map[j,:,:,:]
# # adj_diff_map[j,:,:,:] = torch.abs(subdiff)>0.1*(subdiff.max())
# selection_percent = 0.25
# r = (torch.rand_like(mask) <= selection_percent) * adj_diff_map
# P = torch.sign(diff_map) * mask * r.float()
# m = r.view(bs,-1).data.nonzero()
# sig = 5
# d1 = r.size(2)
# d2 = r.size(3)
# # xx, yy = torch.meshgrid([torch.arange(d1), torch.arange(d2)])
# # xx = xx.cuda()
# # yy = yy.cuda()
# F = torch.zeros_like(r).float()
# g = F[0,0,:,:]
# old_idx = 0
# for idx in m:
# mask_idx = idx[0]
# if (mask_idx != old_idx):
# F[mask_idx-1,0,:,:] = g/g.max()
# g = F[mask_idx,0,:,:]
# one_idx = idx[1]
# x0 = one_idx // d2
# y0 = one_idx % d2
# t = -0.5 * ((xx-x0)**2 + (yy-y0)**2).float() / sig**2
# kernel = torch.exp(t)
# g += kernel
# F[mask_idx,0,:,:] = g/g.max()
# r = F
D = diff_map.view(bs, -1)
m = torch.max(torch.abs(D),
torch.abs(diff_map_reverse).view(
bs, -1)).argmax(1).view(-1, 1)
# P is -1 at point where
updates = torch.sign(D.gather(1, m))
P.view(bs, -1).scatter_(1, m, updates)
# new_input = torch.cat((image, P),dim=1)
# d = torch.abs(diff_map[0,:,:,:])
# depth = d.squeeze().view(228,304).data.cpu().float().numpy()
# implot = plt.imshow(depth)
# d2 = diff_map.size(3)
# indices = torch.cat((m // d2, m % d2), dim=1).data.cpu()
# plt.scatter(indices[0,1],indices[0,0],c='r')
# plt.savefig(output_dir+'point'+'.png')
# mask = mask[0,:,:,:]
# mask = mask.squeeze().view(114,152).data.cpu().float().numpy()
# matplotlib.image.imsave(output_dir+str(i)+'mask'+'.png', mask)
# for j in range(bs): # batch size
# # submask = mask[j,:,:,:].view(-1).byte()
# submask = mask[j,:,:,:].view(-1).byte()
# z = output[j,:,:,:].view(-1)
# gt_depth = depth[j,:,:,:].view(-1)
# subP = P[j,:,:,:].view(-1)
# num_sampled = 1 # number of randomly sample points pairs in mask
# NZ = submask.nonzero()
# sample_idx = torch.multinomial(torch.ones(submask.nonzero().size(0)),num_sampled*2)
# randomly_sampled = NZ[sample_idx,:]
# J = randomly_sampled.view(num_sampled,2)
# # # choose
# # for ik in randomly_sampled:
# # diff = gt_depth[ik]-z[i]
# # select point-pair with greatest discrepancy
# best_pair = None
# max_diff = 0
# for (ik,jk) in J: #combinations(randomly_sampled, 2):
# gt_diff = gt_depth[ik]/gt_depth[jk]
# z_diff = z[ik]/z[jk]
# diff = gt_diff - z_diff
# if torch.abs(diff) > max_diff:
# best_pair = (ik,jk)
# max_diff = torch.abs(diff)
# ik = best_pair[0]
# jk = best_pair[1]
# if max_diff<0: # predicted P1 should be relatively closer, P2 relatively further
# subP[ik] = 1
# subP[jk] = -1
# else:
# subP[ik] = -1
# subP[jk] = 1
new_input = torch.cat((output, mask, P), dim=1)
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
# output = model(image).detach()
# P = torch.zeros_like(output).cuda()
refined_output = rmodel(new_input)
# P.zero_()
# diff_map = (refined_output.detach()/depth - 1)*mask # < 0 if output point is relatively farther
# D = diff_map.view(bs, -1)
# m = torch.abs(D).argmax(1).view(-1,1)
# updates = torch.sign(D.gather(1,m))
# P.view(bs,-1).scatter_(1,m,updates)
# new_input = torch.cat((refined_output, mask, P),dim=1)
# refined_output = rmodel(new_input)
# P.zero_()
# diff_map = (refined_output.detach()/depth - 1)*mask # < 0 if output point is relatively farther
# D = diff_map.view(bs, -1)
# m = torch.abs(D).argmax(1).view(-1,1)
# updates = torch.sign(D.gather(1,m))
# P.view(bs,-1).scatter_(1,m,updates)
# new_input = torch.cat((refined_output, mask, P),dim=1)
# refined_output = rmodel(new_input)
# routput_im = refined_output.squeeze().view(114,152).data.cpu().float().numpy()
# matplotlib.image.imsave(output_dir+'refined'+'.png', routput_im)
# output_im = refined_output.squeeze().view(114,152).data.cpu().float().numpy()
# matplotlib.image.imsave(output_dir+'output'+'.png', output_im)
byte_mask = mask.byte()
masked_depth = depth[byte_mask]
masked_output = output[byte_mask]
masked_refined_output = refined_output[byte_mask]
depth_ = depth
output_ = refined_output
ones = torch.ones(depth_.size(0), 1, depth_.size(2),
depth_.size(3)).float().cuda()
depth_grad = get_gradient(depth_)
output_grad = get_gradient(output_)
depth_grad_dx = depth_grad[:,
0, :, :].contiguous().view_as(
depth_)
depth_grad_dy = depth_grad[:,
1, :, :].contiguous().view_as(
depth_)
output_grad_dx = output_grad[:,
0, :, :].contiguous().view_as(
depth_)
output_grad_dy = output_grad[:,
1, :, :].contiguous().view_as(
depth_)
depth_normal = torch.cat(
(-depth_grad_dx, -depth_grad_dy, ones), 1)
output_normal = torch.cat(
(-output_grad_dx, -output_grad_dy, ones), 1)
loss_depth = torch.log(torch.abs(output_ - depth_) + alpha)
loss_dx = torch.log(
torch.abs(output_grad_dx - depth_grad_dx) + alpha)
loss_dy = torch.log(
torch.abs(output_grad_dy - depth_grad_dy) + alpha)
loss_grad = loss_dx + loss_dy
loss_normal = torch.abs(1 -
cos(output_normal, depth_normal))
loss_normal.unsqueeze_(1)
loss_depth_masked = loss_depth[byte_mask].mean()
loss_grad_masked = loss_grad[byte_mask].mean()
loss_normal_masked = loss_normal[byte_mask].mean()
loss = loss_depth_masked + mu * loss_normal_masked + lamda * loss_grad_masked
# loss_point_depth_masked = loss_depth.view(bs,-1).gather(1,m).mean()
# loss_point_grad_masked = loss_grad.view(bs,-1).gather(1,m).mean()
# loss_point_normal_masked = loss_normal.view(bs,-1).gather(1,m).mean()
# loss_point_depth_masked = (loss_depth)[byte_mask*r].mean()
# loss_point_grad_masked = (loss_grad)[byte_mask* r].mean()
# loss_point_normal_masked = (loss_normal)[byte_mask* r].mean()
# point_loss = loss_point_depth_masked + mu*loss_point_grad_masked + lamda*loss_point_normal_masked
# loss += 0.5*point_loss
# # #berHu loss (Deeper Depth Prediction with Fully Convolutional Residual Networks)
# diff = torch.abs(masked_refined_output - masked_depth)
# c = torch.max(diff)/5
# bh = diff.where(diff<=c, (diff**2+c**2)/(2*c))
# bh_loss = bh.sum()/mask.sum()
# # point loss
# diff_map = torch.abs((refined_output/depth - 1)*mask)
# D = diff_map.view(bs, -1)
# point_loss = torch.sum(D.gather(1,m))/bs
# loss = bh_loss + 3*point_loss
# loss = diff[diff<=c] + ((diff[diff>c])**2)/(2*c)
# # loss function
# rankingloss = 0
# for j in range(bs): # batch size
# submask = mask[j,:,:,:].view(-1).byte()
# z = refined_output[j,:,:,:].view(-1)
# gt_depth = depth[j,:,:,:].view(-1)
# # selection_points = torch.mul(submask,grid)
# # NZ = selection_points.nonzero()
# NZ = submask.nonzero()
# M = 10 # number of pairs of points selected k = 1..M-1
# sample_idx = torch.multinomial(torch.ones(NZ.size(0)),2*M)
# # if NZ.size(0) < 2*M:
# # sample_idx = torch.multinomial(torch.ones(submask.nonzero().size(0)),2*M)
# # # M = NZ.size(0)//2
# # else:
# # sample_idx = torch.multinomial(torch.ones(NZ.size(0)),2*M)
# J = NZ[sample_idx,:]
# J = J.view(M,2)
# # r = torch.zeros(M,).cuda()
# # k = 0
# tau = 0.02
# rloss = 0
# for ik, jk in J: # M*(M-1)/2 loop iterations (so keep M small!)
# if (gt_depth[ik]/gt_depth[jk] > 1 + tau):
# rloss += torch.log(1+torch.exp(-z[ik]+z[jk])) # jk closer than ik
# # r[k] = 1
# elif (gt_depth[jk]/gt_depth[ik] > 1 + tau): # ik closer than jk
# rloss += torch.log(1+torch.exp(z[ik]-z[jk]))
# # r[k] = -1
# else: # equal
# rloss += (z[ik]-z[jk])**2
# # r[k] = 0
# # k = k + 1
# rankingloss += rloss/M
# rl = rankingloss/bs
# loss += 2*rl
losses.update(loss.item(), bs)
if phase == 'train':
loss.backward()
optimizer.step()
# masked_depth = depth[mask.byte()]
# masked_output = output[mask.byte()]
# if phase == 'val':
# depth_val = masked_depth.data.cpu().numpy()
# output_val = masked_output.data.cpu().numpy()
# indices = np.argsort(depth_val,kind='stable')
# idx = np.argsort(output_val[indices],kind='stable')
# n = idx.size
# num_swaps = countSwaps(idx, n)
# acc.update(num_swaps/n)
# statistics
batch_time.update(time.time() - end)
end = time.time()
errors = util.evaluateError(masked_refined_output,
masked_output)
errorSum = util.addErrors(errorSum, errors, bs)
totalNumber = totalNumber + bs
averageError = util.averageErrors(errorSum, totalNumber)
if i % 100 == 0:
out_str = (
'Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.sum:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})'.format(
epoch,
i,
len(dataloader[phase]),
batch_time=batch_time,
loss=losses))
print(out_str)
trainfile.write(out_str + '\n')
# get accuracy as RMSE
# epoch_acc = np.sqrt(averageError['MSE'])
# epoch_acc = acc.avg
# deep copy the model
if phase == 'val':
epoch_acc = np.sqrt(averageError['MSE'])
valfile.write('epoch: ' + str(epoch) + ', rmse: ' +
str(epoch_acc) + '\n')
if epoch_acc < best_acc:
best_acc = epoch_acc
is_best = True
else:
is_best = False
save_checkpoint(rmodel, optimizer, loss, False, epoch, output_dir)
time_elapsed = time.time() - since
s1 = 'Testing complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60)
s2 = 'rmse:' + str(np.sqrt(averageError['MSE']))
s3 = 'abs_rel:' + str(averageError['ABS_REL'])
s4 = 'mae:' + str(averageError['MAE'])
print(s1)
print(s2)
print(s3)
print(s4)
resultsfile.write(phase + '\n')
resultsfile.write(s1 + '\n')
resultsfile.write(s2 + '\n')
resultsfile.write(s3 + '\n')
resultsfile.write(s4 + '\n')
print(threshold_percent.avg)
valfile.write('\n')
trainfile.write('\n')
resultsfile.write('\n')
# print('avg. num swaps:',epoch_acc)
print()
print('errors: ', averageError)
