def iterate(mode, args, loader, model, optimizer, logger, epoch):
block_average_meter = AverageMeter()
average_meter = AverageMeter()
meters = [block_average_meter, average_meter]
# switch to appropriate mode
assert mode in ["train", "val", "eval", "test_prediction", "test_completion"], \
"unsupported mode: {}".format(mode)
if mode == 'train':
model.train()
lr = helper.adjust_learning_rate(args.lr, optimizer, epoch)
else:
model.eval()
lr = 0
for i, batch_data in enumerate(loader):
print("The batch data keys are {}".format(batch_data.keys()))
start = time.time()
batch_data = {
key: val.to(device)
for key, val in batch_data.items() if val is not None
}
gt = batch_data[
'gt'] if mode != 'test_prediction' and mode != 'test_completion' else None
data_time = time.time() - start
start = time.time()
temp_d = batch_data['d']
temp_gt = batch_data['gt']
temp_g = batch_data['g']
print("The depth min:{}, max:{}, shape:{}, dtype:{}".format(
torch.min(temp_d), torch.max(temp_d), temp_d.shape, temp_d.dtype))
print("The groundtruth min:{}, max:{}, shape:{}, dtype:{}".format(
torch.min(temp_gt), torch.max(temp_gt), temp_gt.shape,
temp_gt.dtype))
print("The greyscale min:{}, max:{}, shape:{}, dtype:{}".format(
torch.min(temp_g), torch.max(temp_g), temp_g.shape, temp_g.dtype))
pred = model(batch_data)
temp_out = pred.detach().cpu()
print("The output min:{}, max:{}, shape:{}, dtype:{}".format(
torch.min(temp_out), torch.max(temp_out), temp_out.shape,
temp_out.dtype))
depth_loss, photometric_loss, smooth_loss, mask = 0, 0, 0, None
if mode == 'train':
# Loss 1: the direct depth supervision from ground truth label
# mask=1 indicates that a pixel does not ground truth labels
if 'sparse' in args.train_mode:
depth_loss = depth_criterion(pred, batch_data['d'])
mask = (batch_data['d'] < 1e-3).float()
elif 'dense' in args.train_mode:
depth_loss = depth_criterion(pred, gt)
mask = (gt < 1e-3).float()
# Loss 2: the self-supervised photometric loss
if args.use_pose:
# create multi-scale pyramids
pred_array = helper.multiscale(pred)
rgb_curr_array = helper.multiscale(batch_data['rgb'])
rgb_near_array = helper.multiscale(batch_data['rgb_near'])
if mask is not None:
mask_array = helper.multiscale(mask)
num_scales = len(pred_array)
# compute photometric loss at multiple scales
for scale in range(len(pred_array)):
pred_ = pred_array[scale]
rgb_curr_ = rgb_curr_array[scale]
rgb_near_ = rgb_near_array[scale]
mask_ = None
if mask is not None:
mask_ = mask_array[scale]
# compute the corresponding intrinsic parameters
height_, width_ = pred_.size(2), pred_.size(3)
intrinsics_ = kitti_intrinsics.scale(height_, width_)
# inverse warp from a nearby frame to the current frame
warped_ = homography_from(rgb_near_, pred_,
batch_data['r_mat'],
batch_data['t_vec'], intrinsics_)
photometric_loss += photometric_criterion(
rgb_curr_, warped_, mask_) * (2**(scale - num_scales))
# Loss 3: the depth smoothness loss
smooth_loss = smoothness_criterion(pred) if args.w2 > 0 else 0
# backprop
loss = depth_loss + args.w1 * photometric_loss + args.w2 * smooth_loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
gpu_time = time.time() - start
# measure accuracy and record loss
with torch.no_grad():
mini_batch_size = next(iter(batch_data.values())).size(0)
result = Result()
if mode != 'test_prediction' and mode != 'test_completion':
result.evaluate(pred.data, gt.data, photometric_loss)
[
m.update(result, gpu_time, data_time, mini_batch_size)
for m in meters
]
logger.conditional_print(mode, i, epoch, lr, len(loader),
block_average_meter, average_meter)
logger.conditional_save_img_comparison(mode, i, batch_data, pred,
epoch)
logger.conditional_save_pred(mode, i, pred, epoch)
avg = logger.conditional_save_info(mode, average_meter, epoch)
is_best = logger.rank_conditional_save_best(mode, avg, epoch)
if is_best and not (mode == "train"):
logger.save_img_comparison_as_best(mode, epoch)
logger.conditional_summarize(mode, avg, is_best)
return avg, is_best
def iterate(mode, args, loader, model, optimizer, logger, epoch):
block_average_meter = AverageMeter()
average_meter = AverageMeter()
meters = [block_average_meter, average_meter]
# switch to appropriate mode
assert mode in ["train", "val", "eval", "test_prediction", "test_completion"], \
"unsupported mode: {}".format(mode)
if mode == 'train':
model.train()
lr = helper.adjust_learning_rate(args.lr, optimizer, epoch)
else:
model.eval(
) # batchnorm or dropout layers will work in eval mode instead of training mode
lr = 0
for i, batch_data in enumerate(
loader
): # batch_data keys: 'd' (depth), 'gt' (ground truth), 'g' (gray)
start = time.time()
batch_data = {
key: val.to(device)
for key, val in batch_data.items() if val is not None
}
gt = batch_data[
'gt'] if mode != 'test_prediction' and mode != 'test_completion' else None
data_time = time.time() - start
start = time.time()
pred = model(batch_data)
if args.save_images: # save depth predictions
pred_out_dir = max(glob.glob('../outputs/var_final_NN/var.test*'),
key=os.path.getmtime) + '/dense_depth_images'
pred1 = pred.cpu().detach().numpy()[:, 0, :, :]
for im_idx, pred_im in enumerate(pred1):
pred_out_dir1 = os.path.abspath(pred_out_dir)
cur_path = os.path.abspath((loader.dataset.paths)['d'][i])
basename = os.path.basename(cur_path)
cur_dir = os.path.abspath(os.path.dirname(cur_path))
cur_dir = cur_dir.split('var_final_NN/')[1]
new_dir = os.path.abspath(pred_out_dir1 + '/' + cur_dir)
new_path = os.path.abspath(new_dir + '/' + basename)
if os.path.isdir(new_dir) == False:
os.makedirs(new_dir)
depth_write(new_path, pred_im)
depth_loss, photometric_loss, smooth_loss, mask = 0, 0, 0, None
if mode == 'train':
# Loss 1: the direct depth supervision from ground truth label
# mask=1 indicates that a pixel does not ground truth labels
if 'sparse' in args.train_mode:
depth_loss = depth_criterion(pred, batch_data['d'])
mask = (batch_data['d'] < 1e-3).float()
elif 'dense' in args.train_mode:
depth_loss = depth_criterion(pred, gt)
mask = (gt < 1e-3).float()
# Loss 2: the self-supervised photometric loss
if args.use_pose:
# create multi-scale pyramids
pred_array = helper.multiscale(pred)
rgb_curr_array = helper.multiscale(batch_data['rgb'])
rgb_near_array = helper.multiscale(batch_data['rgb_near'])
if mask is not None:
mask_array = helper.multiscale(mask)
num_scales = len(pred_array)
# compute photometric loss at multiple scales
for scale in range(len(pred_array)):
pred_ = pred_array[scale]
rgb_curr_ = rgb_curr_array[scale]
rgb_near_ = rgb_near_array[scale]
mask_ = None
if mask is not None:
mask_ = mask_array[scale]
# compute the corresponding intrinsic parameters
height_, width_ = pred_.size(2), pred_.size(3)
intrinsics_ = kitti_intrinsics.scale(height_, width_)
# inverse warp from a nearby frame to the current frame
warped_ = homography_from(rgb_near_, pred_,
batch_data['r_mat'],
batch_data['t_vec'], intrinsics_)
photometric_loss += photometric_criterion(
rgb_curr_, warped_, mask_) * (2**(scale - num_scales))
# Loss 3: the depth smoothness loss
smooth_loss = smoothness_criterion(pred) if args.w2 > 0 else 0
# backprop
loss = depth_loss + args.w1 * photometric_loss + args.w2 * smooth_loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
gpu_time = time.time() - start
# measure accuracy and record loss of each batch
with torch.no_grad(
): # impacts the autograd engine and deactivate it (will reduce memory usage and speed up computations)
mini_batch_size = next(iter(batch_data.values())).size(0)
result = Result() # metrics
if mode != 'test_prediction' and mode != 'test_completion':
result.evaluate(pred.data, gt.data, photometric_loss)
[
m.update(result, gpu_time, data_time, mini_batch_size)
for m in meters
]
logger.conditional_print(mode, i, epoch, args.epochs, lr,
len(loader), block_average_meter,
average_meter)
logger.conditional_save_img_comparison(mode, i, batch_data, pred,
epoch)
logger.conditional_save_pred(mode, i, pred, epoch)
del pred
avg = logger.conditional_save_info(
mode, average_meter,
epoch) # take the avg of all the batches, to get the epoch metrics
is_best = logger.rank_conditional_save_best(mode, avg, epoch, args.epochs)
if is_best and not (mode == "train"):
logger.save_img_comparison_as_best(mode, epoch)
logger.conditional_summarize(mode, avg, is_best)
return avg, is_best
def iterate(mode, args, loader, model, optimizer, logger, epoch):
block_average_meter = AverageMeter()
average_meter = AverageMeter()
meters = [block_average_meter, average_meter]
# switch to appropriate mode
assert mode in ["train", "val", "eval", "test_prediction", "test_completion"], \
"unsupported mode: {}".format(mode)
if mode == 'train':
model.train()
lr = helper.adjust_learning_rate(args.lr, optimizer, epoch)
else:
model.eval()
lr = 0
torch.set_printoptions(profile="full")
for i, batch_data in enumerate(loader):
name = batch_data['name'][0]
print(name)
del batch_data['name']
print("i: ", i)
# each batch data is 1 and has three keys d, gt, g and dim [1, 352, 1216]
start = time.time()
batch_data = {
key: val.to(device)
for key, val in batch_data.items() if val is not None
}
gt = batch_data[
'gt'] if mode != 'test_prediction' and mode != 'test_completion' else None
# if args.type_feature=="sq":
# depth_adjustment(gt, False)
data_time = time.time() - start
start = time.time()
if mode == "train":
pred = model(batch_data)
else:
with torch.no_grad():
pred = model(batch_data)
vis=False
if vis:
im = batch_data['gt'].detach().cpu().numpy()
im_sq = im.squeeze()
plt.figure()
plt.imshow(im_sq)
plt.show()
# for i in range(im_sq.shape[0]):
# print(f"{i} - {np.sum(im_sq[i])}")
depth_loss, photometric_loss, smooth_loss, mask = 0, 0, 0, None
if mode == 'train':
# Loss 1: the direct depth supervision from ground truth label
# mask=1 indicates that a pixel does not ground truth labels
if 'sparse' in args.train_mode:
depth_loss = depth_criterion(pred, batch_data['d'])
print("d pts: ", len(torch.where(batch_data['d']>0)[0]))
mask = (batch_data['d'] < 1e-3).float()
elif 'dense' in args.train_mode:
depth_loss = depth_criterion(pred, gt)
mask = (gt < 1e-3).float()
# Loss 2: the self-supervised photometric loss
if args.use_pose:
# create multi-scale pyramids
pred_array = helper.multiscale(pred)
rgb_curr_array = helper.multiscale(batch_data['rgb'])
rgb_near_array = helper.multiscale(batch_data['rgb_near'])
if mask is not None:
mask_array = helper.multiscale(mask)
num_scales = len(pred_array)
# compute photometric loss at multiple scales
for scale in range(len(pred_array)):
pred_ = pred_array[scale]
rgb_curr_ = rgb_curr_array[scale]
rgb_near_ = rgb_near_array[scale]
mask_ = None
if mask is not None:
mask_ = mask_array[scale]
# compute the corresponding intrinsic parameters
height_, width_ = pred_.size(2), pred_.size(3)
intrinsics_ = kitti_intrinsics.scale(height_, width_)
# inverse warp from a nearby frame to the current frame
warped_ = homography_from(rgb_near_, pred_,
batch_data['r_mat'],
batch_data['t_vec'], intrinsics_)
photometric_loss += photometric_criterion(
rgb_curr_, warped_, mask_) * (2**(scale - num_scales))
# Loss 3: the depth smoothness loss
smooth_loss = smoothness_criterion(pred) if args.w2 > 0 else 0
# backprop
loss = depth_loss + args.w1 * photometric_loss + args.w2 * smooth_loss
optimizer.zero_grad()
loss.backward()
zero_params(model)
optimizer.step()
gpu_time = time.time() - start
# counting pixels in each bin
#binned_pixels = np.load("value.npy", allow_pickle=True)
#print(len(binned_pixels))
if (i % 1 == 0 and args.evaluate and args.instancewise) or\
(i % args.every == 0 and not args.evaluate and not args.instancewise): # global training
# print(model.module.conv4[5].conv1.weight[0])
# print(model.conv4.5.bn2.weight)
# print(model.module.parameter.grad)
#print("*************swiches:")
torch.set_printoptions(precision=7, sci_mode=False)
if model.module.phi is not None:
mmp = 1000 * model.module.phi
phi = F.softplus(mmp)
S = phi / torch.sum(phi)
#print("S", S[1, -10:])
S_numpy= S.detach().cpu().numpy()
if args.instancewise:
global Ss
if "Ss" not in globals():
Ss = []
Ss.append(S_numpy)
else:
Ss.append(S_numpy)
# GLOBAL
if (i % args.every ==0 and not args.evaluate and not args.instancewise and model.module.phi is not None):
np.set_printoptions(precision=4)
switches_2d_argsort = np.argsort(S_numpy, None) # 2d to 1d sort torch.Size([9, 31])
switches_2d_sort = np.sort(S_numpy, None)
print("Switches: ")
print(switches_2d_argsort[:10])
print(switches_2d_sort[:10])
print("and")
print(switches_2d_argsort[-10:])
print(switches_2d_sort[-10:])
##### saving global ranks
global_ranks_path = lambda \
ii: f"ranks/{args.type_feature}/global/{folder_and_name[0]}/Ss_val_{folder_and_name[1]}_iter_{ii}.npy"
global old_i
if ("old_i" in globals()):
print("old_i")
if os.path.isfile(global_ranks_path(old_i)):
os.remove(global_ranks_path(old_i))
folder_and_name = args.resume.split(os.sep)[-2:]
os.makedirs(f"ranks/{args.type_feature}/global/{folder_and_name[0]}", exist_ok=True)
np.save(global_ranks_path(i), S_numpy)
old_i = i
print("saving ranks")
if args.type_feature == "sq":
hor = switches_2d_argsort % S_numpy.shape[1]
ver = np.floor(switches_2d_argsort // S_numpy.shape[1])
print(ver[:10],hor[:10])
print("and")
print(ver[-10:], hor[-10:])
# measure accuracy and record loss
with torch.no_grad():
mini_batch_size = next(iter(batch_data.values())).size(0)
result = Result()
if mode != 'test_prediction' and mode != 'test_completion':
result.evaluate(pred.data, gt.data, photometric_loss)
[
m.update(result, gpu_time, data_time, mini_batch_size)
for m in meters
]
logger.conditional_print(mode, i, epoch, lr, len(loader),
block_average_meter, average_meter)
logger.conditional_save_img_comparison(mode, i, batch_data, pred,
epoch)
logger.conditional_save_pred(mode, i, pred, epoch)
draw=False
if draw:
ma = batch_data['rgb'].detach().cpu().numpy().squeeze()
ma = np.transpose(ma, axes=[1, 2, 0])
# ma = np.uint8(ma)
#ma2 = Image.fromarray(ma)
ma2 = Image.fromarray(np.uint8(ma)).convert('RGB')
# create rectangle image
img1 = ImageDraw.Draw(ma2)
if args.type_feature == "sq":
size=40
print_square_num = 20
for ii in range(print_square_num):
s_hor=hor[-ii].detach().cpu().numpy()
s_ver=ver[-ii].detach().cpu().numpy()
shape = [(s_hor * size, s_ver * size), ((s_hor + 1) * size, (s_ver + 1) * size)]
img1.rectangle(shape, outline="red")
tim = time.time()
lala = ma2.save(f"switches_photos/squares/squares_{tim}.jpg")
print("saving")
elif args.type_feature == "lines":
print_square_num = 20
r=1
parameter_mask = np.load("../kitti_pixels_to_lines.npy", allow_pickle=True)
# for m in range(10,50):
# im = Image.fromarray(parameter_mask[m]*155)
# im = im.convert('1') # convert image to black and white
# im.save(f"switches_photos/lala_{m}.jpg")
for ii in range(print_square_num):
points = parameter_mask[ii]
y = np.where(points==1)[0]
x = np.where(points == 1)[1]
for p in range(len(x)):
img1.ellipse((x[p] - r, y[p] - r, x[p] + r, y[p] + r), fill=(255, 0, 0, 0))
tim = time.time()
lala = ma2.save(f"switches_photos/lines/lines_{tim}.jpg")
print("saving")
every = args.every
if i % every ==0:
print("saving")
avg = logger.conditional_save_info(mode, average_meter, epoch)
is_best = logger.rank_conditional_save_best(mode, avg, epoch)
#is_best = True #saving all the checkpoints
if is_best and not (mode == "train"):
logger.save_img_comparison_as_best(mode, epoch)
logger.conditional_summarize(mode, avg, is_best)
if mode != "val":
helper.save_checkpoint({ # save checkpoint
'epoch': epoch,
'model': model.module.state_dict(),
'best_result': logger.best_result,
'optimizer': optimizer.state_dict(),
'args': args,
}, is_best, epoch, logger.output_directory, args.type_feature, i, every, qnet=True)
if args.evaluate and args.instancewise:
#filename = os.path.split(args.evaluate)[1]
Ss_numpy = np.array(Ss)
folder_and_name = args.evaluate.split(os.sep)[-3:]
os.makedirs(f"ranks/instance/{folder_and_name[0]}", exist_ok=True)
os.makedirs(f"ranks/instance/{folder_and_name[0]}/{folder_and_name[1]}", exist_ok=True)
np.save(f"ranks/instance/{folder_and_name[0]}/{folder_and_name[1]}/Ss_val_{folder_and_name[2]}.npy", Ss)
return avg, is_best
def iterate(mode, args, loader, model, optimizer, logger, epoch):
block_average_meter = AverageMeter()
average_meter = AverageMeter()
meters = [block_average_meter, average_meter]
# switch to appropriate mode
assert mode in ["train", "val", "eval", "test_prediction", "test_completion"], \
"unsupported mode: {}".format(mode)
if mode == 'train':
model.train()
lr = helper.adjust_learning_rate(args.lr, optimizer, epoch)
else:
model.eval()
lr = 0
torch.set_printoptions(profile="full")
table_is = np.zeros(400)
for i, batch_data in enumerate(loader):
sparse_depth_pathname = batch_data['d_path'][0]
print(sparse_depth_pathname)
del batch_data['d_path']
print("i: ", i)
start = time.time()
batch_data = {
key: val.to(device)
for key, val in batch_data.items() if val is not None
}
gt = batch_data[
'gt'] if mode != 'test_prediction' and mode != 'test_completion' else None
# adjust depth for features
depth_adjust = args.depth_adjust
adjust_features = False
if depth_adjust and args.use_d:
if args.type_feature == "sq":
if args.use_rgb:
depth_new, alg_mode, feat_mode, features, shape = depth_adjustment(
batch_data['d'], args.test_mode, args.feature_mode,
args.feature_num, args.rank_file_global_sq,
adjust_features, i, model_orig, args.seed,
batch_data['rgb'])
else:
depth_new, alg_mode, feat_mode, features, shape = depth_adjustment(
batch_data['d'], args.test_mode, args.feature_mode,
args.feature_num, args.rank_file_global_sq,
adjust_features, i, model_orig, args.seed)
elif args.type_feature == "lines":
depth_new, alg_mode, feat_mode, features = depth_adjustment_lines(
batch_data['d'], args.test_mode, args.feature_mode,
args.feature_num, args.rank_file_global_sq, i, model_orig,
args.seed)
batch_data['d'] = torch.Tensor(depth_new).unsqueeze(0).unsqueeze(
1).to(device)
data_time = time.time() - start
start = time.time()
if mode == "train":
pred = model(batch_data)
else:
with torch.no_grad():
pred = model(batch_data)
# im = batch_data['d'].detach().cpu().numpy()
# im_sq = im.squeeze()
# plt.figure()
# plt.imshow(im_sq)
# plt.show()
# for i in range(im_sq.shape[0]):
# print(f"{i} - {np.sum(im_sq[i])}")
# pred = pred +0.155
# gt = gt+0.155
# compute loss
depth_loss, photometric_loss, smooth_loss, mask = 0, 0, 0, None
if mode == 'train':
# Loss 1: the direct depth supervision from ground truth label
# mask=1 indicates that a pixel does not ground truth labels
if 'sparse' in args.train_mode:
depth_loss = depth_criterion(pred, batch_data['d'])
mask = (batch_data['d'] < 1e-3).float()
elif 'dense' in args.train_mode:
depth_loss = depth_criterion(pred, gt)
mask = (gt < 1e-3).float()
# Loss 2: the self-supervised photometric loss
if args.use_pose:
# create multi-scale pyramids
pred_array = helper.multiscale(pred)
rgb_curr_array = helper.multiscale(batch_data['rgb'])
rgb_near_array = helper.multiscale(batch_data['rgb_near'])
if mask is not None:
mask_array = helper.multiscale(mask)
num_scales = len(pred_array)
# compute photometric loss at multiple scales
for scale in range(len(pred_array)):
pred_ = pred_array[scale]
rgb_curr_ = rgb_curr_array[scale]
rgb_near_ = rgb_near_array[scale]
mask_ = None
if mask is not None:
mask_ = mask_array[scale]
# compute the corresponding intrinsic parameters
height_, width_ = pred_.size(2), pred_.size(3)
intrinsics_ = kitti_intrinsics.scale(height_, width_)
# inverse warp from a nearby frame to the current frame
warped_ = homography_from(rgb_near_, pred_,
batch_data['r_mat'],
batch_data['t_vec'], intrinsics_)
photometric_loss += photometric_criterion(
rgb_curr_, warped_, mask_) * (2**(scale - num_scales))
# Loss 3: the depth smoothness loss
smooth_loss = smoothness_criterion(pred) if args.w2 > 0 else 0
# backprop
loss = depth_loss + args.w1 * photometric_loss + args.w2 * smooth_loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
gpu_time = time.time() - start
# measure accuracy and record loss
with torch.no_grad():
mini_batch_size = next(iter(batch_data.values())).size(0)
result = Result()
if mode != 'test_prediction' and mode != 'test_completion':
result.evaluate(pred.data, gt.data, photometric_loss)
[
m.update(result, gpu_time, data_time, mini_batch_size)
for m in meters
]
print(f"rmse: {result.rmse:,}")
if result.rmse < 6000:
print("good rmse")
elif result.rmse > 12000:
print("bad rmse")
logger.conditional_print(mode, i, epoch, lr, len(loader),
block_average_meter, average_meter)
logger.conditional_save_img_comparison(mode, i, batch_data, pred,
epoch)
logger.conditional_save_pred(mode, i, pred, epoch)
# save log and checkpoint
every = 999 if mode == "val" else 200
if i % every == 0 and i != 0:
print(
f"test settings (main_orig eval): {args.type_feature} {args.test_mode} {args.feature_mode} {args.feature_num}"
)
avg = logger.conditional_save_info(mode, average_meter, epoch)
is_best = logger.rank_conditional_save_best(mode, avg, epoch)
if is_best and not (mode == "train"):
logger.save_img_comparison_as_best(mode, epoch)
logger.conditional_summarize(mode, avg, is_best)
if mode != "val":
#if 1:
helper.save_checkpoint({ # save checkpoint
'epoch': epoch,
'model': model.module.state_dict(),
'best_result': logger.best_result,
'optimizer': optimizer.state_dict(),
'args': args,
}, is_best, epoch, logger.output_directory, args.type_feature, args.test_mode, args.feature_num, args.feature_mode, args.depth_adjust, i, every, "scratch")
# draw features
# run_info = [args.type_feature, alg_mode, feat_mode, model_orig]
# if batch_data['rgb'] != None and 1 and (i % 1) == 0:
# draw("sq", batch_data['rgb'], batch_data['d'], features, shape[1], run_info, i, result)
return avg, is_best
def iterate(mode, args, loader, model, optimizer, logger, epoch):
block_average_meter = AverageMeter()
average_meter = AverageMeter()
meters = [block_average_meter, average_meter]
block_average_meter_intensity = AverageIntensity()
average_meter_intensity = AverageIntensity()
meters_intensity = [block_average_meter_intensity, average_meter_intensity]
block_average_meter_intensity_pure = AverageIntensity()
average_meter_intensity_pure = AverageIntensity()
meters_intensity_pure = [
block_average_meter_intensity_pure, average_meter_intensity_pure
]
# switch to appropriate mode
assert mode in ["train", "val", "eval", "test_prediction", "test_completion"], \
"unsupported mode: {}".format(mode)
if mode == 'train':
model.train()
# lr = args.lr #helper.adjust_learning_rate(args.lr, optimizer, epoch)
if args.lradj > 0:
lr = helper.adjust_learning_rate(args.lr, optimizer, epoch,
args.lradj)
else:
lr = args.lr
else:
model.eval()
lr = 0
global batch_num
for i, batch_data in enumerate(loader):
if mode == "train":
batch_num += 1
start = time.time()
file_name = batch_data['filename']
batch_data = {
key: val.cuda()
for key, val in batch_data.items()
if (key != "filename" and val is not None)
}
# batch_data = {key:val.cuda() for key,val in batch_data.items() if val is not None}
gt = batch_data[
'gt'] if mode != 'test_prediction' and mode != 'test_completion' else None
# Ireal as gt_intensity
gt_intensity = batch_data[
'gt_intensity'] if mode != 'test_prediction' and mode != 'test_completion' else None
gt_intensity_pure = batch_data[
'gt_intensity_pure'] if mode != 'test_prediction' and mode != 'test_completion' else None
data_time = time.time() - start
start = time.time()
pred = model(batch_data)
pred_intensity_pure = pred[0][:, 1, :, :].unsqueeze(1)
pred_intensity = pred[1]
pred = pred[0][:, 0, :, :].unsqueeze(1)
depth_loss, intensity_loss, photometric_loss, smooth_loss, pure_loss, mask = 0, 0, 0, 0, 0, None
if mode == 'train':
# Loss 1: the direct depth supervision from ground truth label
# mask=1 indicates that a pixel does not ground truth labels
if 'sparse' in args.train_mode:
depth_loss = depth_criterion(pred, batch_data['d'])
mask = (batch_data['d'] < 1e-3).float()
elif 'dense' in args.train_mode:
depth_loss = depth_criterion(pred, gt)
pure_loss = intensity_pure_criterion(pred_intensity_pure,
gt_intensity_pure)
intensity_loss = intensity_criterion(pred_intensity,
gt_intensity)
mask = (gt < 1e-3).float()
# Loss 2: the self-supervised photometric loss
if args.use_pose:
# create multi-scale pyramids
pred_array = helper.multiscale(pred)
rgb_curr_array = helper.multiscale(batch_data['rgb'])
rgb_near_array = helper.multiscale(batch_data['rgb_near'])
if mask is not None:
mask_array = helper.multiscale(mask)
num_scales = len(pred_array)
# compute photometric loss at multiple scales
for scale in range(len(pred_array)):
pred_ = pred_array[scale]
rgb_curr_ = rgb_curr_array[scale]
rgb_near_ = rgb_near_array[scale]
mask_ = None
if mask is not None:
mask_ = mask_array[scale]
# compute the corresponding intrinsic parameters
height_, width_ = pred_.size(2), pred_.size(3)
intrinsics_ = kitti_intrinsics.scale(height_, width_)
# inverse warp from a nearby frame to the current frame
warped_ = homography_from(rgb_near_, pred_,
batch_data['r_mat'],
batch_data['t_vec'], intrinsics_)
photometric_loss += photometric_criterion(
rgb_curr_, warped_, mask_) * (2**(scale - num_scales))
# Loss 3: the depth smoothness loss
smooth_loss = smoothness_criterion(pred) if args.w2 > 0 else 0
# backprop
loss = depth_loss + args.wi * intensity_loss + args.wpure * pure_loss
# loss = depth_loss + wi * intensity_loss + args.w1*photometric_loss + args.w2*smooth_loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
#log train
if batch_num % 25 == 0 and mode == "train":
logger.writer.add_scalar('train/loss_total', loss, batch_num)
logger.writer.add_scalar('train/loss_depth', depth_loss,
batch_num)
logger.writer.add_scalar('train/loss_intensity',
intensity_loss, batch_num)
logger.writer.add_scalar('train/loss_pure', pure_loss,
batch_num)
logger.writer.add_scalar('train/lr', lr, batch_num)
#if batch_num % 200 == 0:
# logger.writer.add_image('train/Ipure', pred_intensity_pure[0], batch_num)
gpu_time = time.time() - start
# measure accuracy and record loss
with torch.no_grad():
mini_batch_size = next(iter(batch_data.values())).size(0)
result = Result()
# result_intensity = Result()
result_intensity_pure = Result_intensity()
result_intensity = Result_intensity()
if mode != 'test_prediction' and mode != 'test_completion':
# result.evaluate(pred.data, gt.data, photometric_loss)
result.evaluate(pred.data, gt.data, photometric_loss)
result_intensity_pure.evaluate(pred_intensity_pure.data,
gt_intensity_pure.data)
result_intensity.evaluate(pred_intensity.data,
gt_intensity.data)
[
m.update(result, gpu_time, data_time, mini_batch_size)
for m in meters
]
[
m_intensity.update(result_intensity, gpu_time, data_time,
mini_batch_size)
for m_intensity in meters_intensity
]
[
m_intensity_pure.update(result_intensity_pure, gpu_time,
data_time, mini_batch_size)
for m_intensity_pure in meters_intensity_pure
]
logger.conditional_print(mode, i, epoch, lr, len(loader),
block_average_meter, average_meter)
# logger.conditional_save_img_comparison(mode, i, batch_data, pred, epoch)
logger.conditional_save_pred_named(mode, file_name[0], pred, epoch)
logger.conditional_print(mode, i, epoch, lr, len(loader),
block_average_meter_intensity_pure,
average_meter_intensity_pure, "Ipure")
logger.conditional_print(mode, i, epoch, lr, len(loader),
block_average_meter_intensity,
average_meter_intensity, "Intensity")
# logger.conditional_save_img_comparison_with_intensity(mode, i, batch_data, pred, pred_intensity, epoch)
logger.conditional_save_img_comparison_with_intensity2(
mode, i, batch_data, pred, pred_intensity_pure, pred_intensity,
epoch)
# run when eval, bt always = 1
#logger.conditional_save_pred_named_with_intensity(mode, file_name[0], pred, pred_intensity, epoch)
avg = logger.conditional_save_info(mode, average_meter, epoch)
avg_intensity = logger.conditional_save_info_intensity(
mode, average_meter_intensity, epoch)
# is_best = logger.rank_conditional_save_best(mode, avg, epoch)
is_best = logger.rank_conditional_save_best_with_intensity(
mode, avg, avg_intensity, epoch)
if is_best and not (mode == "train"):
logger.save_img_comparison_as_best(mode, epoch)
logger.conditional_summarize(mode, avg, is_best)
logger.conditional_summarize_intensity(mode, avg_intensity)
return avg, avg_intensity, is_best
def iterate(mode, args, loader, model, optimizer, logger, epoch):
block_average_meter = AverageMeter()
average_meter = AverageMeter()
meters = [block_average_meter, average_meter]
# switch to appropriate mode
assert mode in ["train", "val", "eval", "test_prediction", "test_completion"], \
"unsupported mode: {}".format(mode)
if mode == 'train':
model.train()
lr = helper.adjust_learning_rate(args.lr, optimizer, epoch)
else:
model.eval()
lr = 0
for i, batch_data in enumerate(loader):
#print(i)
start = time.time()
batch_data = {
key: val.to(device)
for key, val in batch_data.items() if val is not None
}
gt = batch_data[
'gt'] if mode != 'test_prediction' and mode != 'test_completion' else None
data_time = time.time() - start
start = time.time()
pred = model(batch_data)
#im = batch_data['d'].detach().cpu().numpy()
#im_sq = im.squeeze()
#plt.figure()
#plt.imshow(im_sq)
#plt.show()
#for i in range(im_sq.shape[0]):
# print(f"{i} - {np.sum(im_sq[i])}")
depth_loss, photometric_loss, smooth_loss, mask = 0, 0, 0, None
if mode == 'train':
# Loss 1: the direct depth supervision from ground truth label
# mask=1 indicates that a pixel does not ground truth labels
if 'sparse' in args.train_mode:
depth_loss = depth_criterion(pred, batch_data['d'])
mask = (batch_data['d'] < 1e-3).float()
elif 'dense' in args.train_mode:
depth_loss = depth_criterion(pred, gt)
mask = (gt < 1e-3).float()
# Loss 2: the self-supervised photometric loss
if args.use_pose:
# create multi-scale pyramids
pred_array = helper.multiscale(pred)
rgb_curr_array = helper.multiscale(batch_data['rgb'])
rgb_near_array = helper.multiscale(batch_data['rgb_near'])
if mask is not None:
mask_array = helper.multiscale(mask)
num_scales = len(pred_array)
# compute photometric loss at multiple scales
for scale in range(len(pred_array)):
pred_ = pred_array[scale]
rgb_curr_ = rgb_curr_array[scale]
rgb_near_ = rgb_near_array[scale]
mask_ = None
if mask is not None:
mask_ = mask_array[scale]
# compute the corresponding intrinsic parameters
height_, width_ = pred_.size(2), pred_.size(3)
intrinsics_ = kitti_intrinsics.scale(height_, width_)
# inverse warp from a nearby frame to the current frame
warped_ = homography_from(rgb_near_, pred_,
batch_data['r_mat'],
batch_data['t_vec'], intrinsics_)
photometric_loss += photometric_criterion(
rgb_curr_, warped_, mask_) * (2**(scale - num_scales))
# Loss 3: the depth smoothness loss
smooth_loss = smoothness_criterion(pred) if args.w2 > 0 else 0
# backprop
loss = depth_loss + args.w1 * photometric_loss + args.w2 * smooth_loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
gpu_time = time.time() - start
if i % 50 ==0:
# print(model.module.conv4[5].conv1.weight[0])
#print(model.conv4.5.bn2.weight)
#print(model.module.parameter.grad)
print(model.module.parameter)
print(torch.argsort(model.module.parameter))
# measure accuracy and record loss
with torch.no_grad():
mini_batch_size = next(iter(batch_data.values())).size(0)
result = Result()
if mode != 'test_prediction' and mode != 'test_completion':
result.evaluate(pred.data, gt.data, photometric_loss)
[
m.update(result, gpu_time, data_time, mini_batch_size)
for m in meters
]
logger.conditional_print(mode, i, epoch, lr, len(loader),
block_average_meter, average_meter)
logger.conditional_save_img_comparison(mode, i, batch_data, pred,
epoch)
logger.conditional_save_pred(mode, i, pred, epoch)
if i % 20 == 0:
avg = logger.conditional_save_info(mode, average_meter, epoch)
is_best = logger.rank_conditional_save_best(mode, avg, epoch)
if is_best and not (mode == "train"):
logger.save_img_comparison_as_best(mode, epoch)
logger.conditional_summarize(mode, avg, is_best)
helper.save_checkpoint({ # save checkpoint
'epoch': epoch,
'model': model.module.state_dict(),
'best_result': logger.best_result,
'optimizer': optimizer.state_dict(),
'args': args,
}, is_best, epoch, logger.output_directory)
return avg, is_best
def iterate(mode, args, loader, model, optimizer, logger, epoch):
block_average_meter = AverageMeter()
average_meter = AverageMeter()
meters = [block_average_meter, average_meter]
# switch to appropriate mode
assert mode in ["train", "val", "eval", "test_prediction", "test_completion"], \
"unsupported mode: {}".format(mode)
if mode == 'train':
model.train()
lr = helper.adjust_learning_rate(args.lr, optimizer, epoch)
else:
model.eval()
lr = 0
print("\nTraining")
prune_type = "sq" # sq, vlines, nothing
square_choice = args.training_sparse_opt
if prune_type == "sq":
print(f"Features: squares\n Square choice: {square_choice}")
for i, batch_data in enumerate(loader):
start = time.time()
batch_data = {
key: val.to(device)
for key, val in batch_data.items() if val is not None
}
gt = batch_data[
'gt'] if mode != 'test_prediction' and mode != 'test_completion' else None
data_time = time.time() - start
start = time.time()
if prune_type == "vlines":
np.random.seed(10)
lines_unmasked = np.random.choice(352, 20, replace=False)
lines_unmasked = np.arange(352)
lines_all = np.arange(352)
lines_masked = [x for x in lines_all if x not in lines_unmasked]
batch_data['d'][:, :, lines_masked] = 0
print(batch_data['d'].shape)
print("lines unmasked", lines_unmasked)
elif prune_type == "sq":
A = np.load("ranks/switches_2D_equal_iter_390.npy",
allow_pickle=True)
# with np.printoptions(precision=5):
# print("switches", A)
# get the ver and hor coordinates of the most important squares
A_2d_argsort = np.argsort(A, None)[::-1]
if square_choice == "most":
squares_top_file = "ranks/sq/global/squares_most.npy"
A_2d_argsort = np.load(squares_top_file)[::-1]
ver = np.floor(A_2d_argsort // A.shape[1])
hor = A_2d_argsort % A.shape[1]
A_list = np.stack([ver, hor]).transpose()
square_size = 40
squares_top_num = 50
if square_choice == "full":
squares_top = A_list
if square_choice == "most":
squares_top = A_list[:squares_top_num]
if square_choice == "best_sw":
squares_top = A_list[:squares_top_num]
if square_choice == "latin_sw":
# creating latin grid (with big squares/blocks)
hor_large = np.linspace(0, 30, 7)
ver_larger = np.arange(10)
all_squares = np.arange(len(A_list))
bins_2d_latin = binned_statistic_2d(
ver, hor, all_squares, 'min', bins=[ver_larger, hor_large])
bins_2d_latin.statistic
best_latin = bins_2d_latin.statistic[-3:].flatten().astype(int)
best_latin_coors = list(A_list[best_latin])
for i1 in A_list:
elem_in = False # check if the block already contains a small square
for i2 in best_latin_coors:
if i1[0] == i2[0] and i1[1] == i2[1]:
elem_in = True
if not elem_in:
best_latin_coors.append(i1)
if len(best_latin_coors) == squares_top_num:
break
squares_top = np.array(best_latin_coors)
elif square_choice == "latin":
np.random.seed(12)
squares_latin_evenlyspaced = []
# create blocks, choose k random blocks and have fixed first block
hor_large = np.linspace(0, 30, 7)
ver_large = np.arange(10)
# random sample from positive blocks
hor_large_rand = np.random.choice(len(hor_large),
squares_top_num)
ver_large_rand = np.random.choice([6, 7, 8], squares_top_num)
# selecting a small square from A_list with given corrdinates within a block
for j in range(len(hor_large_rand)):
elem = \
np.where((A_list[:, 0] == ver_large_rand[j]) & (A_list[:, 1] == hor_large[hor_large_rand[j]]))[0][0]
squares_latin_evenlyspaced.append(elem)
squares_top = A_list[squares_latin_evenlyspaced]
elif square_choice == "random_all":
np.random.seed(12)
rand_idx = np.random.choice(len(A_list), squares_top_num)
print(rand_idx)
squares_top = A_list[rand_idx]
elif square_choice == "random_pos": # from squares which include depth points
np.random.seed(12)
# choose from the squares which have roughly positive number of depth points
rand_idx = np.random.choice(len(A_list[:93]), squares_top_num)
print(rand_idx)
squares_top = A_list[rand_idx]
# after selecting indices of the squares save in squares_top
squares_top_scaled = np.array(squares_top) * square_size
mask = np.zeros((352, 1216))
bin_ver = np.arange(0, 352, square_size)
bin_ver = np.append(bin_ver, oheight)
bin_hor = np.arange(0, 1216, square_size)
bin_hor = np.append(bin_hor, owidth)
# filling in the mask with selected squares up to squares_top_num (e.g. 20)
# print("Number of squares selected: ", len(squares_top))
# print(squares_top)
for it in range(
len(squares_top)
): # in all but full should be equal to squares_top_num
ver = int(squares_top[it][0])
hor = int(squares_top[it][1])
# print("ver", bin_ver[ver], bin_ver[ver+1], "hor", bin_hor[hor], bin_hor[hor+1] )
mask[bin_ver[ver]:bin_ver[ver + 1],
bin_hor[hor]:bin_hor[hor + 1]] = 1
aaa1 = batch_data['d'].detach().cpu().numpy()
batch_data['d'] = torch.einsum(
"abcd, cd->abcd",
[batch_data['d'],
torch.Tensor(mask).to(device)])
aaa2 = batch_data['d'].detach().cpu().numpy()
#
# # from PIL import Image
# # img = Image.fromarray(aaa[0, :, :, :], 'RGB')
# # #img.save('my.png')
# # img.show()
pred = model(batch_data)
# im = batch_data['d'].detach().cpu().numpy()
# im_sq = im.squeeze()
# plt.figure()
# plt.imshow(im_sq)
# plt.show()
# for i in range(im_sq.shape[0]):
# print(f"{i} - {np.sum(im_sq[i])}")
depth_loss, photometric_loss, smooth_loss, mask = 0, 0, 0, None
if mode == 'train':
# Loss 1: the direct depth supervision from ground truth label
# mask=1 indicates that a pixel does not ground truth labels
if 'sparse' in args.train_mode:
depth_loss = depth_criterion(pred, batch_data['d'])
mask = (batch_data['d'] < 1e-3).float()
elif 'dense' in args.train_mode:
depth_loss = depth_criterion(pred, gt)
mask = (gt < 1e-3).float()
# Loss 2: the self-supervised photometric loss
if args.use_pose:
# create multi-scale pyramids
pred_array = helper.multiscale(pred)
rgb_curr_array = helper.multiscale(batch_data['rgb'])
rgb_near_array = helper.multiscale(batch_data['rgb_near'])
if mask is not None:
mask_array = helper.multiscale(mask)
num_scales = len(pred_array)
# compute photometric loss at multiple scales
for scale in range(len(pred_array)):
pred_ = pred_array[scale]
rgb_curr_ = rgb_curr_array[scale]
rgb_near_ = rgb_near_array[scale]
mask_ = None
if mask is not None:
mask_ = mask_array[scale]
# compute the corresponding intrinsic parameters
height_, width_ = pred_.size(2), pred_.size(3)
intrinsics_ = kitti_intrinsics.scale(height_, width_)
# inverse warp from a nearby frame to the current frame
warped_ = homography_from(rgb_near_, pred_,
batch_data['r_mat'],
batch_data['t_vec'], intrinsics_)
photometric_loss += photometric_criterion(
rgb_curr_, warped_, mask_) * (2**(scale - num_scales))
# Loss 3: the depth smoothness loss
smooth_loss = smoothness_criterion(pred) if args.w2 > 0 else 0
# backprop
loss = depth_loss + args.w1 * photometric_loss + args.w2 * smooth_loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
gpu_time = time.time() - start
# measure accuracy and record loss
with torch.no_grad():
mini_batch_size = next(iter(batch_data.values())).size(0)
result = Result()
if mode != 'test_prediction' and mode != 'test_completion':
result.evaluate(pred.data, gt.data, photometric_loss)
[
m.update(result, gpu_time, data_time, mini_batch_size)
for m in meters
]
logger.conditional_print(mode, i, epoch, lr, len(loader),
block_average_meter, average_meter)
logger.conditional_save_img_comparison(mode, i, batch_data, pred,
epoch)
logger.conditional_save_pred(mode, i, pred, epoch)
every = 100
if i % 500 == 0: # every 100 batches/images (before it was after the entire dataset - two tabs/on if statement)
avg = logger.conditional_save_info(mode, average_meter, epoch)
is_best = logger.rank_conditional_save_best(mode, avg, epoch)
if is_best and not (mode == "train"):
logger.save_img_comparison_as_best(mode, epoch)
logger.conditional_summarize(mode, avg, is_best)
helper.save_checkpoint({ # save checkpoint
'epoch': epoch,
'model': model.module.state_dict(),
'best_result': logger.best_result,
'optimizer': optimizer.state_dict(),
'args': args,
}, is_best, epoch, logger.output_directory, args.type_feature, i, every, "scratch")
return avg, is_best