def train(train_loader, model, criterion, optimizer, epoch):
average_meter = AverageMeter()
model.train() # switch to train mode
end = time.time()
for i, (input, target) in enumerate(train_loader):
input, target = input.cuda(), target.cuda()
torch.cuda.synchronize()
data_time = time.time() - end
# compute pred
end = time.time()
pred = model(input)
loss = criterion(pred, target)
optimizer.zero_grad()
loss.backward() # compute gradient and do SGD step
optimizer.step()
torch.cuda.synchronize()
gpu_time = time.time() - end
# measure accuracy and record loss
result = Result()
result.evaluate(pred.data, target.data)
average_meter.update(result, gpu_time, data_time, input.size(0))
end = time.time()
if (i + 1) % args.print_freq == 0:
print('=> output: {}'.format(output_directory))
print('Train Epoch: {0} [{1}/{2}]\t'
't_Data={data_time:.3f}({average.data_time:.3f}) '
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'RMSE={result.rmse:.2f}({average.rmse:.2f}) '
'MAE={result.mae:.2f}({average.mae:.2f}) '
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'REL={result.absrel:.3f}({average.absrel:.3f}) '
'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format(
epoch,
i + 1,
len(train_loader),
data_time=data_time,
gpu_time=gpu_time,
result=result,
average=average_meter.average()))
avg = average_meter.average()
with open(train_csv, 'a') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow({
'mse': avg.mse,
'rmse': avg.rmse,
'absrel': avg.absrel,
'lg10': avg.lg10,
'mae': avg.mae,
'delta1': avg.delta1,
'delta2': avg.delta2,
'delta3': avg.delta3,
'gpu_time': avg.gpu_time,
'data_time': avg.data_time
})
def __init__(self, args, prepare=True):
self.args = args
self.args.save_pred = True
output_directory = get_folder_name(args)
self.output_directory = output_directory
self.best_result = Result()
self.best_result.set_to_worst()
if not prepare:
return
if not os.path.exists(output_directory):
os.makedirs(output_directory)
self.train_csv = os.path.join(output_directory, 'train.csv')
self.val_csv = os.path.join(output_directory, 'val.csv')
self.best_txt = os.path.join(output_directory, 'best.txt')
# backup the source code
if args.resume == '':
print("=> creating source code backup ...")
backup_directory = os.path.join(output_directory, "code_backup")
self.backup_directory = backup_directory
print("=> stop source code backup ...")
# backup_source_code(backup_directory)
# create new csv files with only header
with open(self.train_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
with open(self.val_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
print("=> finished creating source code backup.")
def train(train_loader, model, criterion, optimizer, epoch, logger):
average_meter = AverageMeter()
model.train() # switch to train mode
end = time.time()
batch_num = len(train_loader)
for i, (input, target) in enumerate(train_loader):
# itr_count += 1
input, target = input.cuda(), target.cuda()
# print('input size = ', input.size())
# print('target size = ', target.size())
torch.cuda.synchronize()
data_time = time.time() - end
# compute pred
end = time.time()
pred = model(input) # @wx 注意输出
# print('pred size = ', pred.size())
# print('target size = ', target.size())
loss = criterion(pred, target)
optimizer.zero_grad()
loss.backward() # compute gradient and do SGD step
optimizer.step()
torch.cuda.synchronize()
gpu_time = time.time() - end
# measure accuracy and record loss
result = Result()
result.evaluate(pred.data, target.data)
average_meter.update(result, gpu_time, data_time, input.size(0))
end = time.time()
if (i + 1) % args.print_freq == 0:
print('=> output: {}'.format(output_directory))
print('Train Epoch: {0} [{1}/{2}]\t'
't_Data={data_time:.3f}({average.data_time:.3f}) '
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'Loss={Loss:.5f} '
'RMSE={result.rmse:.2f}({average.rmse:.2f}) '
'RML={result.absrel:.2f}({average.absrel:.2f}) '
'Log10={result.lg10:.3f}({average.lg10:.3f}) '
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'Delta2={result.delta2:.3f}({average.delta2:.3f}) '
'Delta3={result.delta3:.3f}({average.delta3:.3f})'.format(
epoch, i + 1, len(train_loader), data_time=data_time,
gpu_time=gpu_time, Loss=loss.item(), result=result, average=average_meter.average()))
current_step = epoch * batch_num + i
logger.add_scalar('Train/RMSE', result.rmse, current_step)
logger.add_scalar('Train/rml', result.absrel, current_step)
logger.add_scalar('Train/Log10', result.lg10, current_step)
logger.add_scalar('Train/Delta1', result.delta1, current_step)
logger.add_scalar('Train/Delta2', result.delta2, current_step)
logger.add_scalar('Train/Delta3', result.delta3, current_step)
avg = average_meter.average()
def validate(epoch, valData, model, logger):
average_meter = AverageMeter()
model.eval() # switch to evaluate mode
end = time.time()
# skip = len(valData) // 9 # save images every skip iters
for i, (image, depth) in enumerate(valData):
image = image.cuda()
depth = depth.cuda()
# normal = normal.cuda()
torch.cuda.synchronize()
data_time = time.time() - end
end = time.time()
with torch.no_grad():
pred = model(image)
torch.cuda.synchronize()
gpu_time = time.time() - end
result = Result()
result.evaluate(pred.data, depth.data)
average_meter.update(result, gpu_time, data_time, image.size(0))
end = time.time()
if (i + 1) % 10 == 0:
print('Test Epoch: [{0}/{1}]\t'
't_Data={data_time:.3f}({average.data_time:.3f}) '
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'RMSE={result.rmse:.2f}({average.rmse:.2f}) '
'RML={result.absrel:.2f}({average.absrel:.2f}) '
'Log10={result.lg10:.3f}({average.lg10:.3f}) '
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'Delta2={result.delta2:.3f}({average.delta2:.3f}) '
'Delta3={result.delta3:.3f}({average.delta3:.3f})'.format(
i + 1, len(valData), data_time=data_time,
gpu_time=gpu_time, result=result, average=average_meter.average()))
avg = average_meter.average()
print('\n*\n'
'RMSE={average.rmse:.3f}\n'
'Rel={average.absrel:.3f}\n'
'Log10={average.lg10:.3f}\n'
'Delta1={average.delta1:.3f}\n'
'Delta2={average.delta2:.3f}\n'
'Delta3={average.delta3:.3f}\n'
't_GPU={time:.3f}\n'.format(
average=avg, time=avg.gpu_time))
logger.add_scalar('Test/rmse', avg.rmse, epoch)
logger.add_scalar('Test/Rel', avg.absrel, epoch)
logger.add_scalar('Test/log10', avg.lg10, epoch)
logger.add_scalar('Test/Delta1', avg.delta1, epoch)
logger.add_scalar('Test/Delta2', avg.delta2, epoch)
logger.add_scalar('Test/Delta3', avg.delta3, epoch)
return avg
def train(train_loader, model, criterion, optimizer, epoch):
average_meter = AverageMeter()
model.train() # switch to train mode
end = time.time()
for i, (input, target) in enumerate(train_loader):
input, target = input.cuda(), target.cuda()
torch.cuda.synchronize()
data_time = time.time() - end
# compute pred
end = time.time()
pred = model(input)
loss = criterion(pred, target)
optimizer.zero_grad()
loss.backward() # compute gradient and do SGD step
optimizer.step()
torch.cuda.synchronize()
gpu_time = time.time() - end
depth_in = np.hstack(input.data.cpu().numpy()[:4, 3] / 10.)
depth_in = cv2.applyColorMap((depth_in * 255).astype(np.uint8), cv2.COLORMAP_HOT)
tgt_out = np.hstack(np.squeeze(target[:4].data.cpu().numpy())) / 10.
tgt_out = cv2.applyColorMap((tgt_out * 255).astype(np.uint8), cv2.COLORMAP_HOT)
out = np.hstack(np.squeeze(pred[:4].data.cpu().numpy()))
out = np.clip(out / 10., 0., 1.)
out = cv2.applyColorMap((out * 255).astype(np.uint8), cv2.COLORMAP_HOT)
if i % 20 == 0:
cv2.imshow("Training Results", np.vstack([depth_in, tgt_out, out]))
cv2.waitKey(1)
# measure accuracy and record loss
result = Result()
result.evaluate(pred.data, target.data)
average_meter.update(result, gpu_time, data_time, input.size(0))
end = time.time()
if (i + 1) % args.print_freq == 0:
print('=> output: {}'.format(output_directory))
print('Train Epoch: {0} [{1}/{2}]\t'
't_Data={data_time:.3f}({average.data_time:.3f}) '
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'RMSE={result.rmse:.2f}({average.rmse:.2f}) '
'MAE={result.mae:.2f}({average.mae:.2f}) '
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'REL={result.absrel:.3f}({average.absrel:.3f}) '
'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format(
epoch, i+1, len(train_loader), data_time=data_time,
gpu_time=gpu_time, result=result, average=average_meter.average()))
avg = average_meter.average()
with open(train_csv, 'a') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow({'mse': avg.mse, 'rmse': avg.rmse, 'absrel': avg.absrel, 'lg10': avg.lg10,
'mae': avg.mae, 'delta1': avg.delta1, 'delta2': avg.delta2, 'delta3': avg.delta3,
'gpu_time': avg.gpu_time, 'data_time': avg.data_time})
def iterate(mode, args, loader, model, optimizer, logger, epoch):
block_average_meter = AverageMeter()
average_meter = AverageMeter()
meters = [block_average_meter, average_meter]
# switch to appropriate mode
assert mode in ["train", "val", "eval", "test_prediction", "test_completion"], \
"unsupported mode: {}".format(mode)
if mode == 'train':
model.train()
lr = helper.adjust_learning_rate(args.lr, optimizer, epoch)
else:
model.eval()
lr = 0
for i, batch_data in enumerate(loader):
start = time.time()
batch_data = {
key: val.to(device)
for key, val in batch_data.items() if val is not None
}
gt = batch_data[
'gt'] if mode != 'test_prediction' and mode != 'test_completion' else None
data_time = time.time() - start
start = time.time()
with torch.no_grad(): # 自己加的
pred = model(batch_data)
depth_loss, photometric_loss, smooth_loss, mask = 0, 0, 0, None
gpu_time = time.time() - start
# measure accuracy and record loss
with torch.no_grad():
mini_batch_size = next(iter(batch_data.values())).size(0)
result = Result()
if mode != 'test_prediction' and mode != 'test_completion':
#result.evaluate(pred.data, gt.data, photometric_loss)
result.evaluate(pred.data.cpu(), gt.data.cpu(),
photometric_loss)
[
m.update(result, gpu_time, data_time, mini_batch_size)
for m in meters
]
logger.conditional_print(mode, i, epoch, lr, len(loader),
block_average_meter, average_meter)
logger.conditional_save_img_comparison(mode, i, batch_data, pred,
epoch)
logger.conditional_save_pred(mode, i, pred, epoch)
avg = logger.conditional_save_info(mode, average_meter, epoch)
is_best = logger.rank_conditional_save_best(mode, avg, epoch)
if is_best and not (mode == "train"):
logger.save_img_comparison_as_best(mode, epoch)
logger.conditional_summarize(mode, avg, is_best)
return avg, is_best
def __init__(self, args, prepare=True):
self.args = args
output_directory = get_folder_name(args)
self.output_directory = output_directory
self.best_result = Result()
self.best_result.set_to_worst()
self.best_result_intensity = Result_intensity()
self.best_result_intensity.set_to_worst()
#visual
# self.viz = Visdom(server='http://10.5.40.31', port=11207)
# assert self.viz.check_connection()
from datetime import datetime, timedelta, timezone
utc_dt = datetime.utcnow().replace(tzinfo=timezone.utc)
cn_dt = utc_dt.astimezone(timezone(timedelta(hours=8)))
TIMESTAMP = "{0:%Y-%m-%dT%H-%M-%S/}".format(datetime.now())
name_prefix = "log_lr{}bt{}decay{}wi{}wpure{}lradj{}/".format(
args.lr, args.batch_size, args.weight_decay, args.wi, args.wpure,
args.lradj)
#name_prefix = "log_lr{}bt{}decay{}_ws_{}_alphabeta_{}_{}/".format(args.lr ,args.batch_size, args.weight_decay, args.ws, args.alpha, args.beta)
print(name_prefix + TIMESTAMP)
self.writer = SummaryWriter(name_prefix + TIMESTAMP)
if not prepare:
return
if not os.path.exists(output_directory):
os.makedirs(output_directory)
self.train_csv = os.path.join(output_directory, 'train.csv')
self.val_csv = os.path.join(output_directory, 'val.csv')
self.best_txt = os.path.join(output_directory, 'best.txt')
self.train_csv_intensity = os.path.join(output_directory,
'train_intensity.csv')
self.val_csv_intensity = os.path.join(output_directory,
'val_intensity.csv')
# backup the source code
if args.resume == '':
print("=> creating source code backup ...")
backup_directory = os.path.join(output_directory, "code_backup")
self.backup_directory = backup_directory
backup_source_code(backup_directory)
# create new csv files with only header
with open(self.train_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
with open(self.val_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
with open(self.train_csv_intensity, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
with open(self.val_csv_intensity, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
print("=> finished creating source code backup.")
def write_results(txt: str, res: Result, res_inside: Result,
res_outside: Result, epoch: int):
res.name = "result"
res_inside.name = "result inside"
res_outside.name = "result outside"
with open(txt, 'w') as txtfile:
txtfile.write(str(res) + "\n")
txtfile.write(str(res_inside) + "\n")
txtfile.write(str(res_outside) + "\n")
txtfile.write(f"epoch: {epoch}")
def train(epoch, trainData, model, crite, optimizer, logger):
average_meter = AverageMeter()
model.train() # switch to train mode
end = time.time()
for i, (image, depth) in enumerate(trainData):
image = image.cuda()
depth = depth.cuda()
# normal = normal.cuda()
# image = torch.autograd.Variable(image)
# depth = torch.autograd.Variable(depth)
torch.cuda.synchronize()
data_time = time.time() - end
end = time.time()
optimizer.zero_grad()
pred = model(image)
loss = crite(pred, depth)
loss.backward()
optimizer.step()
torch.cuda.synchronize()
gpu_time = time.time() - end
result = Result()
result.evaluate(pred.data, depth.data)
average_meter.update(result, gpu_time, data_time, image.size(0))
end = time.time()
if (i + 1) % 10 == 0:
print('=> output: {}'.format(opt.output_dir))
print('Train Epoch: {0} [{1}/{2}]\t'
't_Data={data_time:.3f}({average.data_time:.3f}) '
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'Loss={Loss:.5f} '
'RMSE={result.rmse:.2f}({average.rmse:.2f}) '
'RML={result.absrel:.2f}({average.absrel:.2f}) '
'Log10={result.lg10:.3f}({average.lg10:.3f}) '
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'Delta2={result.delta2:.3f}({average.delta2:.3f}) '
'Delta3={result.delta3:.3f}({average.delta3:.3f})'.format(
epoch, i + 1, len(trainData), data_time=data_time,
gpu_time=gpu_time, Loss=loss.item(), result=result, average=average_meter.average()))
current_step = epoch * len(trainData) + i
logger.add_scalar('Train/loss', loss, current_step)
logger.add_scalar('Train/RMSE', result.rmse, current_step)
logger.add_scalar('Train/rml', result.absrel, current_step)
logger.add_scalar('Train/Log10', result.lg10, current_step)
logger.add_scalar('Train/Delta1', result.delta1, current_step)
logger.add_scalar('Train/Delta2', result.delta2, current_step)
logger.add_scalar('Train/Delta3', result.delta3, current_step)
def compute_depth_metrics(self, verbose=True) -> Result:
"""Computes metrics on the difference between raw and fixed depth values"""
avg = AverageMeter()
for i, path in enumerate(self.paths):
_, depth_raw, depth_fix = self.load_images(path)
depth_raw = torch.tensor(depth_raw)
depth_fix = torch.tensor(depth_fix)
res = Result()
res.evaluate(depth_raw, depth_fix)
avg.update(res, 0, 0, 1)
if verbose:
stdout.write(f"=> computing img {i}/{len(self)}\r")
if verbose:
stdout.write("\n")
return avg.average()
def train_coarse(train_loader, model, criterion, optimizer, epoch):
average_meter = AverageMeter()
model.train() # switch to train mode
end = time.time()
for i, (input, target) in enumerate(train_loader):
input, target = input.cuda(), target.cuda()
torch.cuda.synchronize()
data_time = time.time() - end
# compute pred
end = time.time()
pred = model(input)
loss = criterion(pred, target)
optimizer.zero_grad()
loss.backward() # compute gradient and do SGD step
optimizer.step()
torch.cuda.synchronize()
gpu_time = time.time() - end
# measure accuracy and record loss
end = time.time()
result = Result()
result.evaluate(pred.data, target.data)
average_meter.update(result, gpu_time, data_time, input.size(0))
eval_time = time.time() - end
if (i + 1) % args.print_freq == 0:
history_loss.append(loss.item())
print('=> output: {}'.format(output_directory))
print('Train Epoch: {0} [{1}/{2}]\t'
't_Data={data_time:.3f}({average.data_time:.3f}) '
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'RMSE={result.rmse:.2f}({average.rmse:.2f}) '
'MAE={result.mae:.2f}({average.mae:.2f}) '
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'REL={result.absrel:.3f}({average.absrel:.3f}) '
'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format(
epoch,
i + 1,
len(train_loader),
data_time=data_time,
gpu_time=gpu_time,
result=result,
average=average_meter.average()))
def evaluate(params, loader, model, experiment):
print("Testing...")
with experiment.test() and torch.no_grad():
average = AverageMeter()
end = time.time()
for i, (inputs, targets) in enumerate(loader):
inputs, targets = inputs.to(params["device"]), targets.to(
params["device"])
data_time = time.time() - end
# Predict
end = time.time()
outputs = model(inputs)
gpu_time = time.time() - end
# Clip prediction
outputs[outputs > params["depth_max"]] = params["depth_max"]
outputs[outputs < params["depth_min"]] = params["depth_min"]
result = Result()
result.evaluate(outputs.data, targets.data)
average.update(result, gpu_time, data_time, inputs.size(0))
# Log images to comet
img_merged = utils.log_image_to_comet(inputs[0],
targets[0],
outputs[0],
epoch=0,
id=i,
experiment=experiment,
result=result,
prefix="visual_test")
if params["save_test_images"]:
filename = os.path.join(
params["experiment_dir"],
"image_{}_epoch_{}.png".format(i,
str(params["start_epoch"])))
utils.save_image(img_merged, filename)
def __load_result(path):
# Loads results from the file at the specified path and returns a corresponding Result object
labels = datapoint_features # List of all feature labels
# Split path on '\' and find the index of 'result' in the resulting list
substrings = (path[:-4]).split("\\")
begin = len(substrings) - 1
while substrings[begin] != 'result':
begin -= 1
# Extract information about the used dataset, situated at specific offsets from the 'begin' index
baseline = substrings[begin + 1] == 'baseline'
model = substrings[begin + 2]
imp_split = substrings[begin + 3] != 'imp_full'
dos_type = substrings[begin + 4]
# Split the remaining string on '_' and discard 'mixed'
file_split = substrings[begin + 5].split("_")[1:]
is_test = file_split[0] == 'test'
window_ms = int((file_split[2])[:-2])
stride_ms = int((file_split[3])[:-2])
subset = []
# Use each substring from index 4 as an index into the list of all feature labels, to establish a feature subset
for substring in file_split[4:]:
index = int(substring)
subset.append(labels[index])
# Use the gathered information to load the metrics and times
metrics = load_metrics(window_ms, stride_ms, imp_split, dos_type, model, baseline, subset, is_test)
times = load_times(window_ms, stride_ms, imp_split, dos_type, model, baseline, subset, is_test)
return Result(window_ms, stride_ms, model, imp_split, dos_type, baseline, subset, is_test, metrics, times)
def validate(val_loader, model, epoch, write_to_file=True):
average_meter = AverageMeter()
model.eval() # switch to evaluate mode
end = time.time()
for i, (input, target) in enumerate(val_loader):
input, target = input.cuda(), target.cuda()
torch.cuda.synchronize()
data_time = time.time() - end
# compute output
end = time.time()
##############################################################
## Start of PnP-Depth modification ##
##############################################################
# Original inference
with torch.no_grad():
ori_pred = model.pnp_forward_front(model.pnp_forward_rear(
input)) # equivalent to `ori_pred = model(input)`
# Inference with PnP
sparse_target = input[:, -1:] # NOTE: written for rgbd input
criterion = criteria.MaskedL1Loss().cuda(
) # NOTE: criterion function defined here only for clarity
pnp_iters = 5 # number of iterations
pnp_alpha = 0.01 # update/learning rate
pnp_z = model.pnp_forward_front(input)
for pnp_i in range(pnp_iters):
if pnp_i != 0:
pnp_z = pnp_z - pnp_alpha * torch.sign(pnp_z_grad) # iFGM
pnp_z = Variable(pnp_z, requires_grad=True)
pred = model.pnp_forward_rear(pnp_z)
if pnp_i < pnp_iters - 1:
pnp_loss = criterion(pred, sparse_target)
pnp_z_grad = Grad([pnp_loss], [pnp_z], create_graph=True)[0]
##############################################################
## End of PnP-Depth modification ##
##############################################################
torch.cuda.synchronize()
gpu_time = time.time() - end
# measure accuracy and record loss
result = Result()
result.evaluate(pred.data, target.data)
average_meter.update(result, gpu_time, data_time, input.size(0))
end = time.time()
# save 8 images for visualization
skip = 50
if args.modality == 'd':
img_merge = None
else:
if args.modality == 'rgb':
rgb = input
elif args.modality == 'rgbd':
rgb = input[:, :3, :, :]
depth = input[:, 3:, :, :]
if i == 0:
if args.modality == 'rgbd':
img_merge = utils.merge_into_row_with_gt(
rgb, depth, target, pred)
else:
img_merge = utils.merge_into_row(rgb, target, pred)
elif (i < 8 * skip) and (i % skip == 0):
if args.modality == 'rgbd':
row = utils.merge_into_row_with_gt(rgb, depth, target,
pred)
else:
row = utils.merge_into_row(rgb, target, pred)
img_merge = utils.add_row(img_merge, row)
elif i == 8 * skip:
filename = output_directory + '/comparison_' + str(
epoch) + '.png'
utils.save_image(img_merge, filename)
if (i + 1) % args.print_freq == 0:
print('Test: [{0}/{1}]\t'
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'RMSE={result.rmse:.2f}({average.rmse:.2f}) '
'MAE={result.mae:.2f}({average.mae:.2f}) '
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'REL={result.absrel:.3f}({average.absrel:.3f}) '
'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format(
i + 1,
len(val_loader),
gpu_time=gpu_time,
result=result,
average=average_meter.average()))
avg = average_meter.average()
print('\n*\n'
'RMSE={average.rmse:.3f}\n'
'MAE={average.mae:.3f}\n'
'Delta1={average.delta1:.3f}\n'
'REL={average.absrel:.3f}\n'
'Lg10={average.lg10:.3f}\n'
't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time))
if write_to_file:
with open(test_csv, 'a') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow({
'mse': avg.mse,
'rmse': avg.rmse,
'absrel': avg.absrel,
'lg10': avg.lg10,
'mae': avg.mae,
'delta1': avg.delta1,
'delta2': avg.delta2,
'delta3': avg.delta3,
'data_time': avg.data_time,
'gpu_time': avg.gpu_time
})
return avg, img_merge
def train(train_loader, val_loader, model, criterion, optimizer, epoch, lr):
average_meter = AverageMeter()
model.train() # switch to train mode
global batch_num, best_result
end = time.time()
#every batch
for i, (Y, Y_1_2, Y_1_4, Y_1_8, LR, LR_8, HR,
name) in enumerate(train_loader): #处理被train_loader进去的每一个数据
batch_num = batch_num + 1
Y = Y.cuda()
Y_1_2 = Y_1_2.cuda()
Y_1_4 = Y_1_4.cuda()
Y_1_8 = Y_1_8.cuda()
LR = LR.cuda()
LR_8 = LR_8.cuda()
HR = HR.cuda()
torch.cuda.synchronize()
data_time = time.time() - end
end = time.time()
if args.arch == 'VDSR_16':
pred_HR = model(LR)
loss = criterion(pred_HR, HR, Y)
elif args.arch == 'VDSR_16_2':
pred_HR = model(Y, LR)
loss = criterion(pred_HR, HR, Y)
elif args.arch == 'VDSR':
pred_HR, residule = model(LR_8, Y)
loss = criterion(pred_HR, HR, Y)
elif args.arch == 'ResNet_bicubic':
pred_HR, residule = model(LR_8, Y)
loss = criterion(pred_HR, HR, Y)
elif args.arch == 'resnet50_15_6' or 'resnet50_15_11' or 'resnet50_15_12':
pred_HR = model(Y_1_2, LR)
loss = criterion(pred_HR, HR, Y)
elif args.arch == 'resnet50_15_2' or 'resnet50_15_3' or 'resnet50_15_5' or 'resnet50_15_8' or 'resnet50_15_9':
pred_HR, residule = model(Y, LR, LR_8)
loss = criterion(pred_HR, HR, Y)
else:
if config.loss_num == 2:
if config.LOSS_1 == 'l2':
# 均方差
criterion1 = criteria.MaskedMSELoss().cuda()
elif config.LOSS_1 == 'l1':
criterion1 = criteria.MaskedL1Loss().cuda()
elif config.LOSS_1 == 'l1_canny':
# 均方差
criterion1 = criteria.MaskedL1_cannyLoss().cuda()
elif config.LOSS_1 == 'l1_from_rgb_sobel':
# 均方差
criterion1 = criteria.MaskedL1_from_rgb_sobel_Loss().cuda()
elif aconfig.LOSS_1 == 'l1_canny_from_GT_canny':
criterion1 = criteria.MaskedL1_canny_from_GT_Loss().cuda()
elif aconfig.LOSS_1 == 'l1_from_GT_sobel':
criterion1 = criteria.MaskedL1_from_GT_sobel_Loss().cuda()
elif config.LOSS_1 == 'l2_from_GT_sobel_Loss':
criterion1 = criteria.MaskedL2_from_GT_sobel_Loss().cuda()
if config.use_different_size_Y == 1:
pred_HR, pred_thermal0 = model(Y, Y_1_2, Y_1_4, Y_1_8, LR)
else:
pred_HR, pred_thermal0 = model(Y, LR)
#final loss
loss0 = criterion(pred_HR, HR, Y)
#therma upsample loss
loss1 = criterion1(pred_thermal0, HR, Y)
loss = config.LOSS_0_weight * loss0 + config.LOSS_1_weight * loss1
else:
if config.use_different_size_Y == 1:
pred_HR, pred_thermal0 = model(Y, Y_1_2, Y_1_4, Y_1_8, LR)
#writer = SummaryWriter(log_dir='logs')
#writer.add_graph(model, input_to_model=(Y,Y_1_2,Y_1_4,Y_1_8,LR,))
else:
pred_HR, pred_thermal0 = model(Y, LR)
loss = criterion(pred_HR, HR, Y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
torch.cuda.synchronize()
gpu_time = time.time() - end
# measure accuracy and record loss
result = Result()
result.evaluate(pred_HR, HR, loss.cpu().detach().numpy())
average_meter.update(result, gpu_time, data_time, Y.size(0))
end = time.time()
if (i + 1) % args.print_freq == 0:
print('=> output: {}'.format(output_directory))
print('Dataset Epoch: {0} [{1}/{2}]\t'
'Batch Epoch: {3} \t'
't_Data={data_time:.3f}({average.data_time:.3f}) '
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n'
'PSNR={result.psnr:.5f}({average.psnr:.5f}) '
'MSE={result.mse:.3f}({average.mse:.3f}) '
'RMSE={result.rmse:.3f}({average.rmse:.3f}) '
'MAE={result.mae:.3f}({average.mae:.3f}) '
'Delta1={result.delta1:.4f}({average.delta1:.4f}) '
'REL={result.absrel:.4f}({average.absrel:.4f}) '
'Lg10={result.lg10:.4f}({average.lg10:.4f}) '
'Loss={result.loss:}({average.loss:}) '.format(
epoch,
i + 1,
len(train_loader),
batch_num,
data_time=data_time,
gpu_time=gpu_time,
result=result,
average=average_meter.average()))
else:
pass
if (batch_num + 1) % config.save_fc == 0:
print("==============Time to evaluate=================")
utils.adjust_learning_rate(optimizer, batch_num, lr)
print("==============SAVE_MODEL=================")
avg = average_meter.average()
average_meter = AverageMeter()
with open(train_csv, 'a') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow({
'dataset epoch': epoch,
'batch epoch': batch_num + 1,
'psnr': 10 * math.log(1 / (avg.mse), 10),
'mse': result.mse,
'rmse': result.rmse,
'absrel': result.absrel,
'lg10': result.lg10,
'mae': result.mae,
'delta1': result.delta1,
'delta2': result.delta2,
'delta3': result.delta3,
'gpu_time': result.gpu_time,
'data_time': result.data_time,
'loss': result.loss
})
#------------------#
# VALIDATION #
#------------------#
result_val, img_merge = validate(
val_loader, model, epoch,
batch_num) # evaluate on validation set，每次训练完以后都要测试一下
#------------------#
# SAVE BEST MODEL #
#------------------#
is_best = result_val.rmse < best_result.rmse
if is_best:
best_result = result_val
with open(best_txt, 'w') as txtfile:
txtfile.write(
"dataset epoch={}\nbatch epoch={}\npsnr={:.5f}\nmse={:.3f}\nrmse={:.3f}\nabsrel={:.3f}\nlg10={:.3f}\nmae={:.3f}\ndelta1={:.3f}\nt_gpu={:.4f}\n"
.format(epoch, batch_num + 1,
10 * math.log(1 / (best_result.mse), 10),
best_result.mse, best_result.rmse,
best_result.absrel, best_result.lg10,
best_result.mae, best_result.delta1,
best_result.gpu_time))
if img_merge is not None:
img_filename = output_directory + '/comparison_best.png'
utils.save_image(img_merge, img_filename)
utils.save_checkpoint(
{
'args': args,
'epoch': epoch,
'batch_epoch': batch_num,
'arch': args.arch,
'model': model,
'best_result': best_result,
'optimizer': optimizer,
}, is_best, epoch, batch_num, output_directory)
def evaluate(params, loader, model, experiment):
print("Testing...")
with experiment.test():
with torch.no_grad():
average = AverageMeter()
img_idxs = np.random.randint(0,
len(loader),
size=min(len(loader), 50))
end = time.time()
for i, (inputs, targets) in enumerate(loader):
inputs, targets = inputs.to(params["device"]), targets.to(
params["device"])
data_time = time.time() - end
# Predict
end = time.time()
outputs = model(inputs)
gpu_time = time.time() - end
result = Result()
result.evaluate(outputs.data, targets.data)
average.update(result, gpu_time, data_time, inputs.size(0))
end = time.time()
# Log images to comet
if i in img_idxs:
img_merged = utils.log_image_to_comet(
inputs[0],
targets[0],
outputs[0],
epoch=0,
id=i,
experiment=experiment,
result=result,
prefix="test")
if params["save_test_images"]:
filename = os.path.join(
params["experiment_dir"],
"comparison_epoch_{}_{}.png".format(
str(params["start_epoch"]),
np.where(img_idxs == i)[0][0]))
utils.save_image(img_merged, filename)
if (i + 1) % params["stats_frequency"] == 0 and i != 0:
print('Test: [{0}/{1}]\t'
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'RMSE={result.rmse:.2f}({average.rmse:.2f}) '
'MAE={result.mae:.2f}({average.mae:.2f}) '
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'REL={result.absrel:.3f}({average.absrel:.3f}) '
'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format(
i + 1,
len(loader),
gpu_time=gpu_time,
result=result,
average=average.average()))
# Mean validation loss
avg = average.average()
utils.log_comet_metrics(experiment, avg, None)
print('\n*\n'
'RMSE={average.rmse:.3f}\n'
'MAE={average.mae:.3f}\n'
'Delta1={average.delta1:.3f}\n'
'REL={average.absrel:.3f}\n'
'Lg10={average.lg10:.3f}\n'
't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time))
if params["save_test_metrics"]:
filename = os.path.join(params["experiment_dir"],
"results.csv")
with open(filename, 'a') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow({
'mse': avg.mse,
'rmse': avg.rmse,
'absrel': avg.absrel,
'lg10': avg.lg10,
'mae': avg.mae,
'delta1': avg.delta1,
'delta2': avg.delta2,
'delta3': avg.delta3,
'irmse': avg.irmse,
'imae': avg.imae,
'data_time': avg.data_time,
'gpu_time': avg.gpu_time
})
def iterate(mode, args, loader, model, optimizer, logger, epoch):
block_average_meter = AverageMeter()
average_meter = AverageMeter()
meters = [block_average_meter, average_meter]
# switch to appropriate mode
assert mode in ["train", "val", "eval", "test_prediction", "test_completion"], \
"unsupported mode: {}".format(mode)
if mode == 'train':
model.train()
lr = helper.adjust_learning_rate(args.lr, optimizer, epoch)
else:
model.eval()
lr = 0
torch.set_printoptions(profile="full")
for i, batch_data in enumerate(loader):
name = batch_data['name'][0]
print(name)
del batch_data['name']
print("i: ", i)
# each batch data is 1 and has three keys d, gt, g and dim [1, 352, 1216]
start = time.time()
batch_data = {
key: val.to(device)
for key, val in batch_data.items() if val is not None
}
gt = batch_data[
'gt'] if mode != 'test_prediction' and mode != 'test_completion' else None
# if args.type_feature=="sq":
# depth_adjustment(gt, False)
data_time = time.time() - start
start = time.time()
if mode == "train":
pred = model(batch_data)
else:
with torch.no_grad():
pred = model(batch_data)
vis=False
if vis:
im = batch_data['gt'].detach().cpu().numpy()
im_sq = im.squeeze()
plt.figure()
plt.imshow(im_sq)
plt.show()
# for i in range(im_sq.shape[0]):
# print(f"{i} - {np.sum(im_sq[i])}")
depth_loss, photometric_loss, smooth_loss, mask = 0, 0, 0, None
if mode == 'train':
# Loss 1: the direct depth supervision from ground truth label
# mask=1 indicates that a pixel does not ground truth labels
if 'sparse' in args.train_mode:
depth_loss = depth_criterion(pred, batch_data['d'])
print("d pts: ", len(torch.where(batch_data['d']>0)[0]))
mask = (batch_data['d'] < 1e-3).float()
elif 'dense' in args.train_mode:
depth_loss = depth_criterion(pred, gt)
mask = (gt < 1e-3).float()
# Loss 2: the self-supervised photometric loss
if args.use_pose:
# create multi-scale pyramids
pred_array = helper.multiscale(pred)
rgb_curr_array = helper.multiscale(batch_data['rgb'])
rgb_near_array = helper.multiscale(batch_data['rgb_near'])
if mask is not None:
mask_array = helper.multiscale(mask)
num_scales = len(pred_array)
# compute photometric loss at multiple scales
for scale in range(len(pred_array)):
pred_ = pred_array[scale]
rgb_curr_ = rgb_curr_array[scale]
rgb_near_ = rgb_near_array[scale]
mask_ = None
if mask is not None:
mask_ = mask_array[scale]
# compute the corresponding intrinsic parameters
height_, width_ = pred_.size(2), pred_.size(3)
intrinsics_ = kitti_intrinsics.scale(height_, width_)
# inverse warp from a nearby frame to the current frame
warped_ = homography_from(rgb_near_, pred_,
batch_data['r_mat'],
batch_data['t_vec'], intrinsics_)
photometric_loss += photometric_criterion(
rgb_curr_, warped_, mask_) * (2**(scale - num_scales))
# Loss 3: the depth smoothness loss
smooth_loss = smoothness_criterion(pred) if args.w2 > 0 else 0
# backprop
loss = depth_loss + args.w1 * photometric_loss + args.w2 * smooth_loss
optimizer.zero_grad()
loss.backward()
zero_params(model)
optimizer.step()
gpu_time = time.time() - start
# counting pixels in each bin
#binned_pixels = np.load("value.npy", allow_pickle=True)
#print(len(binned_pixels))
if (i % 1 == 0 and args.evaluate and args.instancewise) or\
(i % args.every == 0 and not args.evaluate and not args.instancewise): # global training
# print(model.module.conv4[5].conv1.weight[0])
# print(model.conv4.5.bn2.weight)
# print(model.module.parameter.grad)
#print("*************swiches:")
torch.set_printoptions(precision=7, sci_mode=False)
if model.module.phi is not None:
mmp = 1000 * model.module.phi
phi = F.softplus(mmp)
S = phi / torch.sum(phi)
#print("S", S[1, -10:])
S_numpy= S.detach().cpu().numpy()
if args.instancewise:
global Ss
if "Ss" not in globals():
Ss = []
Ss.append(S_numpy)
else:
Ss.append(S_numpy)
# GLOBAL
if (i % args.every ==0 and not args.evaluate and not args.instancewise and model.module.phi is not None):
np.set_printoptions(precision=4)
switches_2d_argsort = np.argsort(S_numpy, None) # 2d to 1d sort torch.Size([9, 31])
switches_2d_sort = np.sort(S_numpy, None)
print("Switches: ")
print(switches_2d_argsort[:10])
print(switches_2d_sort[:10])
print("and")
print(switches_2d_argsort[-10:])
print(switches_2d_sort[-10:])
##### saving global ranks
global_ranks_path = lambda \
ii: f"ranks/{args.type_feature}/global/{folder_and_name[0]}/Ss_val_{folder_and_name[1]}_iter_{ii}.npy"
global old_i
if ("old_i" in globals()):
print("old_i")
if os.path.isfile(global_ranks_path(old_i)):
os.remove(global_ranks_path(old_i))
folder_and_name = args.resume.split(os.sep)[-2:]
os.makedirs(f"ranks/{args.type_feature}/global/{folder_and_name[0]}", exist_ok=True)
np.save(global_ranks_path(i), S_numpy)
old_i = i
print("saving ranks")
if args.type_feature == "sq":
hor = switches_2d_argsort % S_numpy.shape[1]
ver = np.floor(switches_2d_argsort // S_numpy.shape[1])
print(ver[:10],hor[:10])
print("and")
print(ver[-10:], hor[-10:])
# measure accuracy and record loss
with torch.no_grad():
mini_batch_size = next(iter(batch_data.values())).size(0)
result = Result()
if mode != 'test_prediction' and mode != 'test_completion':
result.evaluate(pred.data, gt.data, photometric_loss)
[
m.update(result, gpu_time, data_time, mini_batch_size)
for m in meters
]
logger.conditional_print(mode, i, epoch, lr, len(loader),
block_average_meter, average_meter)
logger.conditional_save_img_comparison(mode, i, batch_data, pred,
epoch)
logger.conditional_save_pred(mode, i, pred, epoch)
draw=False
if draw:
ma = batch_data['rgb'].detach().cpu().numpy().squeeze()
ma = np.transpose(ma, axes=[1, 2, 0])
# ma = np.uint8(ma)
#ma2 = Image.fromarray(ma)
ma2 = Image.fromarray(np.uint8(ma)).convert('RGB')
# create rectangle image
img1 = ImageDraw.Draw(ma2)
if args.type_feature == "sq":
size=40
print_square_num = 20
for ii in range(print_square_num):
s_hor=hor[-ii].detach().cpu().numpy()
s_ver=ver[-ii].detach().cpu().numpy()
shape = [(s_hor * size, s_ver * size), ((s_hor + 1) * size, (s_ver + 1) * size)]
img1.rectangle(shape, outline="red")
tim = time.time()
lala = ma2.save(f"switches_photos/squares/squares_{tim}.jpg")
print("saving")
elif args.type_feature == "lines":
print_square_num = 20
r=1
parameter_mask = np.load("../kitti_pixels_to_lines.npy", allow_pickle=True)
# for m in range(10,50):
# im = Image.fromarray(parameter_mask[m]*155)
# im = im.convert('1') # convert image to black and white
# im.save(f"switches_photos/lala_{m}.jpg")
for ii in range(print_square_num):
points = parameter_mask[ii]
y = np.where(points==1)[0]
x = np.where(points == 1)[1]
for p in range(len(x)):
img1.ellipse((x[p] - r, y[p] - r, x[p] + r, y[p] + r), fill=(255, 0, 0, 0))
tim = time.time()
lala = ma2.save(f"switches_photos/lines/lines_{tim}.jpg")
print("saving")
every = args.every
if i % every ==0:
print("saving")
avg = logger.conditional_save_info(mode, average_meter, epoch)
is_best = logger.rank_conditional_save_best(mode, avg, epoch)
#is_best = True #saving all the checkpoints
if is_best and not (mode == "train"):
logger.save_img_comparison_as_best(mode, epoch)
logger.conditional_summarize(mode, avg, is_best)
if mode != "val":
helper.save_checkpoint({ # save checkpoint
'epoch': epoch,
'model': model.module.state_dict(),
'best_result': logger.best_result,
'optimizer': optimizer.state_dict(),
'args': args,
}, is_best, epoch, logger.output_directory, args.type_feature, i, every, qnet=True)
if args.evaluate and args.instancewise:
#filename = os.path.split(args.evaluate)[1]
Ss_numpy = np.array(Ss)
folder_and_name = args.evaluate.split(os.sep)[-3:]
os.makedirs(f"ranks/instance/{folder_and_name[0]}", exist_ok=True)
os.makedirs(f"ranks/instance/{folder_and_name[0]}/{folder_and_name[1]}", exist_ok=True)
np.save(f"ranks/instance/{folder_and_name[0]}/{folder_and_name[1]}/Ss_val_{folder_and_name[2]}.npy", Ss)
return avg, is_best
def iterate(mode, args, loader, model, optimizer, logger, epoch):
block_average_meter = AverageMeter()
average_meter = AverageMeter()
meters = [block_average_meter, average_meter]
# switch to appropriate mode
assert mode in ["train", "val", "eval", "test_prediction", "test_completion"], \
"unsupported mode: {}".format(mode)
if mode == 'train':
model.train()
lr = helper.adjust_learning_rate(args.lr, optimizer, epoch)
else:
model.eval()
lr = 0
for i, batch_data in enumerate(loader):
print("The batch data keys are {}".format(batch_data.keys()))
start = time.time()
batch_data = {
key: val.to(device)
for key, val in batch_data.items() if val is not None
}
gt = batch_data[
'gt'] if mode != 'test_prediction' and mode != 'test_completion' else None
data_time = time.time() - start
start = time.time()
temp_d = batch_data['d']
temp_gt = batch_data['gt']
temp_g = batch_data['g']
print("The depth min:{}, max:{}, shape:{}, dtype:{}".format(
torch.min(temp_d), torch.max(temp_d), temp_d.shape, temp_d.dtype))
print("The groundtruth min:{}, max:{}, shape:{}, dtype:{}".format(
torch.min(temp_gt), torch.max(temp_gt), temp_gt.shape,
temp_gt.dtype))
print("The greyscale min:{}, max:{}, shape:{}, dtype:{}".format(
torch.min(temp_g), torch.max(temp_g), temp_g.shape, temp_g.dtype))
pred = model(batch_data)
temp_out = pred.detach().cpu()
print("The output min:{}, max:{}, shape:{}, dtype:{}".format(
torch.min(temp_out), torch.max(temp_out), temp_out.shape,
temp_out.dtype))
depth_loss, photometric_loss, smooth_loss, mask = 0, 0, 0, None
if mode == 'train':
# Loss 1: the direct depth supervision from ground truth label
# mask=1 indicates that a pixel does not ground truth labels
if 'sparse' in args.train_mode:
depth_loss = depth_criterion(pred, batch_data['d'])
mask = (batch_data['d'] < 1e-3).float()
elif 'dense' in args.train_mode:
depth_loss = depth_criterion(pred, gt)
mask = (gt < 1e-3).float()
# Loss 2: the self-supervised photometric loss
if args.use_pose:
# create multi-scale pyramids
pred_array = helper.multiscale(pred)
rgb_curr_array = helper.multiscale(batch_data['rgb'])
rgb_near_array = helper.multiscale(batch_data['rgb_near'])
if mask is not None:
mask_array = helper.multiscale(mask)
num_scales = len(pred_array)
# compute photometric loss at multiple scales
for scale in range(len(pred_array)):
pred_ = pred_array[scale]
rgb_curr_ = rgb_curr_array[scale]
rgb_near_ = rgb_near_array[scale]
mask_ = None
if mask is not None:
mask_ = mask_array[scale]
# compute the corresponding intrinsic parameters
height_, width_ = pred_.size(2), pred_.size(3)
intrinsics_ = kitti_intrinsics.scale(height_, width_)
# inverse warp from a nearby frame to the current frame
warped_ = homography_from(rgb_near_, pred_,
batch_data['r_mat'],
batch_data['t_vec'], intrinsics_)
photometric_loss += photometric_criterion(
rgb_curr_, warped_, mask_) * (2**(scale - num_scales))
# Loss 3: the depth smoothness loss
smooth_loss = smoothness_criterion(pred) if args.w2 > 0 else 0
# backprop
loss = depth_loss + args.w1 * photometric_loss + args.w2 * smooth_loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
gpu_time = time.time() - start
# measure accuracy and record loss
with torch.no_grad():
mini_batch_size = next(iter(batch_data.values())).size(0)
result = Result()
if mode != 'test_prediction' and mode != 'test_completion':
result.evaluate(pred.data, gt.data, photometric_loss)
[
m.update(result, gpu_time, data_time, mini_batch_size)
for m in meters
]
logger.conditional_print(mode, i, epoch, lr, len(loader),
block_average_meter, average_meter)
logger.conditional_save_img_comparison(mode, i, batch_data, pred,
epoch)
logger.conditional_save_pred(mode, i, pred, epoch)
avg = logger.conditional_save_info(mode, average_meter, epoch)
is_best = logger.rank_conditional_save_best(mode, avg, epoch)
if is_best and not (mode == "train"):
logger.save_img_comparison_as_best(mode, epoch)
logger.conditional_summarize(mode, avg, is_best)
return avg, is_best
class logger:
def __init__(self, args, prepare=True):
self.args = args
self.args.save_pred = True
output_directory = get_folder_name(args)
self.output_directory = output_directory
self.best_result = Result()
self.best_result.set_to_worst()
if not prepare:
return
if not os.path.exists(output_directory):
os.makedirs(output_directory)
self.train_csv = os.path.join(output_directory, 'train.csv')
self.val_csv = os.path.join(output_directory, 'val.csv')
self.best_txt = os.path.join(output_directory, 'best.txt')
# backup the source code
if args.resume == '':
print("=> creating source code backup ...")
backup_directory = os.path.join(output_directory, "code_backup")
self.backup_directory = backup_directory
print("=> stop source code backup ...")
# backup_source_code(backup_directory)
# create new csv files with only header
with open(self.train_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
with open(self.val_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
print("=> finished creating source code backup.")
def conditional_print(self, split, i, epoch, lr, n_set, blk_avg_meter,
avg_meter):
if (i + 1) % self.args.print_freq == 0:
avg = avg_meter.average()
blk_avg = blk_avg_meter.average()
print('=> output: {}'.format(self.output_directory))
print(
'{split} Epoch: {0} [{1}/{2}]\tlr={lr} '
't_Data={blk_avg.data_time:.3f}({average.data_time:.3f}) '
't_GPU={blk_avg.gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'RMSE={blk_avg.rmse:.2f}({average.rmse:.2f}) '
'MAE={blk_avg.mae:.2f}({average.mae:.2f}) '
'iRMSE={blk_avg.irmse:.2f}({average.irmse:.2f}) '
'iMAE={blk_avg.imae:.2f}({average.imae:.2f})\n\t'
'silog={blk_avg.silog:.2f}({average.silog:.2f}) '
'squared_rel={blk_avg.squared_rel:.2f}({average.squared_rel:.2f}) '
'Delta1={blk_avg.delta1:.3f}({average.delta1:.3f}) '
'REL={blk_avg.absrel:.3f}({average.absrel:.3f})\n\t'
'Lg10={blk_avg.lg10:.3f}({average.lg10:.3f}) '
'Photometric={blk_avg.photometric:.3f}({average.photometric:.3f}) '
.format(epoch,
i + 1,
n_set,
lr=lr,
blk_avg=blk_avg,
average=avg,
split=split.capitalize()))
blk_avg_meter.reset()
def conditional_save_info(self, split, average_meter, epoch):
avg = average_meter.average()
if split == "train":
csvfile_name = self.train_csv
elif split == "val":
csvfile_name = self.val_csv
elif split == "eval":
eval_filename = os.path.join(self.output_directory, 'eval.txt')
self.save_single_txt(eval_filename, avg, epoch)
return avg
elif "test" in split:
return avg
else:
raise ValueError("wrong split provided to logger")
with open(csvfile_name, 'a') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow({
'epoch': epoch,
'rmse': avg.rmse,
'photo': avg.photometric,
'mae': avg.mae,
'irmse': avg.irmse,
'imae': avg.imae,
'mse': avg.mse,
'silog': avg.silog,
'squared_rel': avg.squared_rel,
'absrel': avg.absrel,
'lg10': avg.lg10,
'delta1': avg.delta1,
'delta2': avg.delta2,
'delta3': avg.delta3,
'gpu_time': avg.gpu_time,
'data_time': avg.data_time
})
return avg
def save_single_txt(self, filename, result, epoch):
with open(filename, 'w') as txtfile:
txtfile.write(
("rank_metric={}\n" + "epoch={}\n" + "rmse={:.3f}\n" +
"mae={:.3f}\n" + "silog={:.3f}\n" + "squared_rel={:.3f}\n" +
"irmse={:.3f}\n" + "imae={:.3f}\n" + "mse={:.3f}\n" +
"absrel={:.3f}\n" + "lg10={:.3f}\n" + "delta1={:.3f}\n" +
"t_gpu={:.4f}").format(self.args.rank_metric, epoch,
result.rmse, result.mae, result.silog,
result.squared_rel, result.irmse,
result.imae, result.mse, result.absrel,
result.lg10, result.delta1,
result.gpu_time))
def save_best_txt(self, result, epoch):
self.save_single_txt(self.best_txt, result, epoch)
def _get_img_comparison_name(self, mode, epoch, is_best=False):
if mode == 'eval':
return self.output_directory + '/comparison_eval.png'
if mode == 'val':
if is_best:
return self.output_directory + '/comparison_best.png'
else:
return self.output_directory + '/comparison_' + str(
epoch) + '.png'
def conditional_save_img_comparison(self, mode, i, ele, pred, epoch):
# save 8 images for visualization
if mode == 'val' or mode == 'eval':
skip = 100
if i == 0:
self.img_merge = vis_utils.merge_into_row(ele, pred)
elif i % skip == 0 and i < 8 * skip:
row = vis_utils.merge_into_row(ele, pred)
self.img_merge = vis_utils.add_row(self.img_merge, row)
elif i == 8 * skip:
filename = self._get_img_comparison_name(mode, epoch)
vis_utils.save_image(self.img_merge, filename)
def save_img_comparison_as_best(self, mode, epoch):
if mode == 'val':
filename = self._get_img_comparison_name(mode, epoch, is_best=True)
vis_utils.save_image(self.img_merge, filename)
def get_ranking_error(self, result):
return getattr(result, self.args.rank_metric)
def rank_conditional_save_best(self, mode, result, epoch):
error = self.get_ranking_error(result)
best_error = self.get_ranking_error(self.best_result)
is_best = error < best_error
if is_best and mode == "val":
self.old_best_result = self.best_result
self.best_result = result
self.save_best_txt(result, epoch)
return is_best
def conditional_save_pred(self, mode, file_name, pred, epoch):
if ("test" in mode or mode == "eval") and self.args.save_pred:
# save images for visualization/ testing
image_folder = os.path.join(self.output_directory,
mode + "_output")
if not os.path.exists(image_folder):
os.makedirs(image_folder)
img = torch.squeeze(pred.data.cpu()).numpy()
file_path = os.path.join(image_folder, file_name)
vis_utils.save_depth_as_uint16png(img, file_path)
def conditional_summarize(self, mode, avg, is_best):
print("\n*\nSummary of ", mode, "round")
print(''
'RMSE={average.rmse:.3f}\n'
'MAE={average.mae:.3f}\n'
'Photo={average.photometric:.3f}\n'
'iRMSE={average.irmse:.3f}\n'
'iMAE={average.imae:.3f}\n'
'squared_rel={average.squared_rel}\n'
'silog={average.silog}\n'
'Delta1={average.delta1:.3f}\n'
'REL={average.absrel:.3f}\n'
'Lg10={average.lg10:.3f}\n'
't_GPU={time:.3f}'.format(average=avg, time=avg.gpu_time))
if is_best and mode == "val":
print("New best model by %s (was %.3f)" %
(self.args.rank_metric,
self.get_ranking_error(self.old_best_result)))
elif mode == "val":
print("(best %s is %.3f)" %
(self.args.rank_metric,
self.get_ranking_error(self.best_result)))
print("*\n")
def validate(val_loader, model, epoch, write_to_file=True):
average_meter = AverageMeter()
model.eval() # switch to evaluate mode
end = time.time()
eval_file = output_directory + '/evaluation.csv'
f = open(eval_file, "w+")
f.write("Max_Error,Depth,RMSE,GPU_TIME,Number_Of_Frame\r\n")
for i, (input, target) in enumerate(val_loader):
input, target = input.cuda(), target.cuda()
# torch.cuda.synchronize()
data_time = time.time() - end
# compute output
end = time.time()
with torch.no_grad():
pred = model(input)
# torch.cuda.synchronize()
gpu_time = time.time() - end
abs_err = (target.data - pred.data).abs().cpu()
max_err_ind = np.unravel_index(np.argmax(abs_err, axis=None),
abs_err.shape)
max_err_depth = target.data[max_err_ind]
max_err = abs_err[max_err_ind]
# measure accuracy and record loss
result = Result()
result.evaluate(pred.data, target.data)
average_meter.update(result, gpu_time, data_time, input.size(0))
end = time.time()
f.write(
f'{max_err},{max_err_depth},{result.rmse:.2f},{gpu_time},{i+1}\r\n'
)
# save 8 images for visualization
skip = 50
if args.modality == 'rgb':
rgb = input
if i == 0:
img_merge = utils.merge_into_row_with_gt(rgb, target, pred,
(target - pred).abs())
elif (i < 8 * skip) and (i % skip == 0):
row = utils.merge_into_row_with_gt(rgb, target, pred,
(target - pred).abs())
img_merge = utils.add_row(img_merge, row)
elif i == 8 * skip:
filename = output_directory + '/comparison_' + str(epoch) + '.png'
utils.save_image(img_merge, filename)
if (i + 1) % args.print_freq == 0:
print('Test: [{0}/{1}]\t'
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'RMSE={result.rmse:.2f}({average.rmse:.2f}) '
'MAE={result.mae:.2f}({average.mae:.2f}) '
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'REL={result.absrel:.3f}({average.absrel:.3f}) '
'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format(
i + 1,
len(val_loader),
gpu_time=gpu_time,
result=result,
average=average_meter.average()))
f.close()
avg = average_meter.average()
print('\n*\n'
'RMSE={average.rmse:.3f}\n'
'MAE={average.mae:.3f}\n'
'Delta1={average.delta1:.3f}\n'
'REL={average.absrel:.3f}\n'
'Lg10={average.lg10:.3f}\n'
't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time))
if write_to_file:
with open(test_csv, 'a') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow({
'mse': avg.mse,
'rmse': avg.rmse,
'absrel': avg.absrel,
'lg10': avg.lg10,
'mae': avg.mae,
'delta1': avg.delta1,
'delta2': avg.delta2,
'delta3': avg.delta3,
'data_time': avg.data_time,
'gpu_time': avg.gpu_time
})
return avg, img_merge
def train(train_loader, model, criterion,smoothloss,photometric_loss,optimizer, epoch):
average_meter = AverageMeter()
model.train() # switch to train mode
end = time.time()
iheight, iwidth = 480, 640 # raw image size
fx_rgb = 5.1885790117450188e+02;
fy_rgb = 5.1946961112127485e+02;
cx_rgb = 3.2558244941119034e+02;
cy_rgb = 2.5373616633400465e+02;
new_intrinsics = Intrinsics(304,228,fx_rgb*(250.0/iheight),fy_rgb*(250.0/iheight),155.1, 121.0).cuda()
for i, (batch_data, intrinsics) in enumerate(train_loader):
#input, target = input.cuda(), target.cuda()
batch_data = {key:val.cuda() for key,val in batch_data.items() if val is not None}
oheight, owidth = intrinsics["output_size"]
#new_intrinsics = Intrinsics(owidth,oheight,intrinsics["fx"],intrinsics["fy"],intrinsics["cx"],intrinsics["cy"]).cuda()
target = batch_data['gt']
torch.cuda.synchronize()
data_time = time.time() - end
# compute pred
end = time.time()
#pred = model(input[:,3:,:,:],input[:,:3,:,:]) # (depth,image)
#candidates = {"rgb":input[:,:3,:,:], "d":input[:,3:,:,:], "gt":input[:,3:,:,:]}
#batch_data = {key:val.cuda() for key,val in candidates.items() if val is not None}
pred = model(batch_data) # (depth,image)
#loss = criterion(pred, input[:,3:,:,:]) + 0.01*smoothloss(pred)
photoloss = 0.0
if args.use_pose:
# warp near frame to current frame
#hh, ww = pred.size(2), pred.size(3)
#new_intrinsics = new_intrinsics.scale(hh,ww)
mask = (batch_data['d'] < 1e-3).float()
pred_array = multiscale(pred)
rgb_curr_array = multiscale(batch_data['rgb'])
rgb_near_array = multiscale(batch_data['rgb_near'])
mask_array = multiscale(mask)
num_scales = len(pred_array)
# compute photometric loss at multiple scales
for scale in range(len(pred_array)):
pred_ = pred_array[scale]
rgb_curr_ = rgb_curr_array[scale]
rgb_near_ = rgb_near_array[scale]
mask_ = None
if mask is not None:
mask_ = mask_array[scale]
# compute the corresponding intrinsic parameters
height_, width_ = pred_.size(2), pred_.size(3)
intrinsics_ = new_intrinsics.scale(height_, width_)
warped_ = homography_from(rgb_near_,pred_,batch_data["r_mat"],batch_data["t_vec"],intrinsics_)
#warped = homography_from(batch_data["rgb_near"],pred,batch_data["r_mat"],batch_data["t_vec"],new_intrinsics)
#photoloss = photometric_loss(batch_data["rgb"],warped,mask)
photoloss += photometric_loss(rgb_curr_,warped_,mask_)*(2**(scale-num_scales))
loss = criterion(pred, target) + 0.01*smoothloss(pred) + 0.1*photoloss
optimizer.zero_grad()
loss.backward() # compute gradient and do SGD step
optimizer.step()
torch.cuda.synchronize()
gpu_time = time.time() - end
# measure accuracy and record loss
result = Result()
result.evaluate(pred.data, target.data)
average_meter.update(result, gpu_time, data_time, batch_data['rgb'].size(0))
end = time.time()
if (i + 1) % args.print_freq == 0:
print('=> output: {}'.format(output_directory))
print('Train Epoch: {0} [{1}/{2}]\t'
'LOSS={loss:.3f} '
't_Data={data_time:.3f}({average.data_time:.3f}) '
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'RMSE={result.rmse:.2f}({average.rmse:.2f}) '
'MAE={result.mae:.2f}({average.mae:.2f}) '
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'REL={result.absrel:.3f}({average.absrel:.3f}) '
'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format(
epoch, i+1, len(train_loader),loss=loss.item(), data_time=data_time,
gpu_time=gpu_time, result=result, average=average_meter.average()))
avg = average_meter.average()
with open(train_csv, 'a') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow({'mse': avg.mse, 'rmse': avg.rmse, 'absrel': avg.absrel, 'lg10': avg.lg10,
'mae': avg.mae, 'delta1': avg.delta1, 'delta2': avg.delta2, 'delta3': avg.delta3,
'gpu_time': avg.gpu_time, 'data_time': avg.data_time})
def depth_estimate(model, rgb_frame, depth, save=False,
switch=False):
to_tensor = transforms.ToTensor()
cmap_depth = plt.cm.viridis
cmap_error = plt.cm.inferno
rgb_np, depth_np = image_transform(rgb_frame, depth)
##creat sparse depth
mask_keep = sampler(depth_np, args.sample_spacing)
sample_number = np.count_nonzero(
mask_keep.astype('int'))
if args.modality == 'rgb':
sample_number = 0
if switch:
print('Total samples = ' + str(sample_number))
sparse_depth = np.zeros(depth_np.shape)
sparse_depth[mask_keep] = depth_np[mask_keep]
sparse_depth_np = sparse_depth
##choose input
if args.modality == 'rgb':
input_np = rgb_np
elif args.modality == 'rgbd':
rgbd = np.append(rgb_np,
np.expand_dims(sparse_depth_np,
axis=2), axis=2)
input_np = np.asfarray(rgbd, dtype='float')
elif args.modality == 'd':
input_np = sparse_depth_np
input_tensor = to_tensor(input_np)
while input_tensor.dim() < 4:
input_tensor = input_tensor.unsqueeze(0)
input_tensor = input_tensor.cuda()
torch.cuda.synchronize()
##get prediction
end = time.time()
with torch.no_grad():
pred = model(input_tensor)
torch.cuda.synchronize()
gpu_time = time.time() - end
##get result
target_tensor = to_tensor(depth_np)
while target_tensor.dim() < 4:
target_tensor = target_tensor.unsqueeze(0)
target_tensor = target_tensor.cuda()
torch.cuda.synchronize()
result = Result()
result.evaluate(pred.data, target_tensor.data)
pred_depth = np.squeeze(pred.data.cpu().numpy())
##convert depth to colour map
d_min = min(np.min(pred_depth), np.min(depth_np))
d_max = max(np.max(pred_depth), np.max(depth_np))
# # d_min = float(0.5)
# # d_max = float(6)
pred_depth_rgb_map = color_map(pred_depth, d_min, d_max,
cmap_depth)
input_depth_color_map = color_map(depth_np, d_min,
d_max, cmap_depth)
sparse_depth_color_map = color_map(sparse_depth, d_min,
d_max, cmap_depth)
##get error map
mask = depth_np <= 0
combined_map = depth_np
combined_map[mask] = pred_depth[mask]
abs_error_map = np.absolute(combined_map - pred_depth)
# error_min = min(np.min(combined_map), np.min(
# pred_depth))
# error_max = max(np.max(combined_map), np.max(
# pred_depth))
error_min = np.min(abs_error_map)
error_max = np.max(abs_error_map)
error_map_color = color_map(abs_error_map, error_min,
error_max, cmap_error)
##show images
rgb_image = rgb_np * 255
merged_image = np.hstack([rgb_image, pred_depth_rgb_map,
input_depth_color_map,
sparse_depth_color_map,
error_map_color])
##save image
if args.write and save:
images_save(sample_number, pred_depth_rgb_map,
sparse_depth_color_map, error_map_color,
rgb_image, input_depth_color_map,
result)
return merged_image, result.rmse
def validate(val_loader, model, epoch, logger):
average_meter = AverageMeter()
model.eval() # switch to evaluate mode
end = time.time()
skip = len(val_loader) // 9 # save images every skip iters
for i, (input, target) in enumerate(val_loader):
input, target = input.cuda(), target.cuda()
torch.cuda.synchronize()
data_time = time.time() - end
# compute output
end = time.time()
with torch.no_grad():
pred = model(input)
torch.cuda.synchronize()
gpu_time = time.time() - end
# measure accuracy and record loss
result = Result()
result.evaluate(pred.data, target.data)
average_meter.update(result, gpu_time, data_time, input.size(0))
end = time.time()
# save 8 images for visualization
if args.dataset == 'kitti':
rgb = input[0]
pred = pred[0]
target = target[0]
else:
rgb = input
if i == 0:
img_merge = utils.merge_into_row(rgb, target, pred)
elif (i < 8 * skip) and (i % skip == 0):
row = utils.merge_into_row(rgb, target, pred)
img_merge = utils.add_row(img_merge, row)
elif i == 8 * skip:
filename = output_directory + '/comparison_' + str(epoch) + '.png'
utils.save_image(img_merge, filename)
if (i + 1) % args.print_freq == 0:
print('Test: [{0}/{1}]\t'
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'RMSE={result.rmse:.2f}({average.rmse:.2f}) '
'RML={result.absrel:.2f}({average.absrel:.2f}) '
'Log10={result.lg10:.3f}({average.lg10:.3f}) '
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'Delta2={result.delta2:.3f}({average.delta2:.3f}) '
'Delta3={result.delta3:.3f}({average.delta3:.3f})'.format(
i + 1,
len(val_loader),
gpu_time=gpu_time,
result=result,
average=average_meter.average()))
avg = average_meter.average()
print('\n*\n'
'RMSE={average.rmse:.3f}\n'
'Rel={average.absrel:.3f}\n'
'Log10={average.lg10:.3f}\n'
'Delta1={average.delta1:.3f}\n'
'Delta2={average.delta2:.3f}\n'
'Delta3={average.delta3:.3f}\n'
't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time))
logger.add_scalar('Test/rmse', avg.rmse, epoch)
logger.add_scalar('Test/Rel', avg.absrel, epoch)
logger.add_scalar('Test/log10', avg.lg10, epoch)
logger.add_scalar('Test/Delta1', avg.delta1, epoch)
logger.add_scalar('Test/Delta2', avg.delta2, epoch)
logger.add_scalar('Test/Delta3', avg.delta3, epoch)
return avg, img_merge
def main():
torch.cuda.set_device(config.cuda_id)
global args, best_result, output_directory, train_csv, test_csv, batch_num, best_txt
best_result = Result()
best_result.set_to_worst()
batch_num = 0
output_directory = utils.get_output_directory(args)
#-----------------#
# pytorch version #
#-----------------#
try:
torch._utils._rebuild_tensor_v2
except AttributeError:
def _rebuild_tensor_v2(storage, storage_offset, size, stride,
requires_grad, backward_hooks):
tensor = torch._utils._rebuild_tensor(storage, storage_offset,
size, stride)
tensor.requires_grad = requires_grad
tensor._backward_hooks = backward_hooks
return tensor
torch._utils._rebuild_tensor_v2 = _rebuild_tensor_v2
if not os.path.exists(output_directory):
os.makedirs(output_directory)
train_csv = os.path.join(output_directory, 'train.csv')
test_csv = os.path.join(output_directory, 'test.csv')
best_txt = os.path.join(output_directory, 'best.txt')
nowTime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
file = open(namefile, 'a+')
file.writelines(
str("====================================================") +
str(nowTime) + '\n')
file.writelines(str("Cuda_id: ") + str(config.cuda_id) + '\n')
file.writelines(str("NAME: ") + str(config.name) + '\n')
file.writelines(str("Description: ") + str(config.description) + '\n')
file.writelines(
str("model: ") + str(args.arch) + '\n' + str("loss_final: ") +
str(args.criterion) + '\n' + str("loss_1: ") + str(config.LOSS_1) +
'\n' + str("batch_size:") + str(args.batch_size) + '\n')
file.writelines(str("zoom_scale: ") + str(config.zoom_scale) + '\n')
file.writelines(str("------------------------") + '\n')
file.writelines(str("Train_dataste: ") + str(config.train_dir) + '\n')
file.writelines(str("Validation_dataste: ") + str(config.val_dir) + '\n')
file.writelines(str("------------------------") + '\n')
file.writelines(str("Input_type: ") + str(config.input) + '\n')
file.writelines(str("target_type: ") + str(config.target) + '\n')
file.writelines(str("LOSS--------------------") + '\n')
file.writelines(str("Loss_num: ") + str(config.loss_num) + '\n')
file.writelines(
str("loss_final: ") + str(args.criterion) + '\n' + str("loss_1: ") +
str(config.LOSS_1) + '\n')
file.writelines(
str("loss_0_weight: ") + str(config.LOSS_0_weight) + '\n' +
str("loss_1_weight: ") + str(config.LOSS_1_weight) + '\n')
file.writelines(
str("weight_GT_canny: ") + str(config.weight_GT_canny_loss) + '\n' +
str("weight_GT_sobel: ") + str(config.weight_GT_sobel_loss) + '\n' +
str("weight_rgb_sobel: ") + str(config.weight_rgb_sobel_loss) + '\n')
file.writelines(str("------------------------") + '\n')
file.writelines(str("target: ") + str(config.target) + '\n')
file.writelines(str("data_loader_type: ") + str(config.data_loader) + '\n')
file.writelines(str("lr: ") + str(config.Init_lr) + '\n')
file.writelines(str("save_fc: ") + str(config.save_fc) + '\n')
file.writelines(str("Max epoch: ") + str(config.epoch) + '\n')
file.close()
# define loss function (criterion) and optimizer,定义误差函数和优化器
if args.criterion == 'l2':
criterion = criteria.MaskedMSELoss().cuda()
elif args.criterion == 'l1':
criterion = criteria.MaskedL1Loss().cuda()
elif args.criterion == 'l1_canny':
criterion = criteria.MaskedL1_cannyLoss().cuda()
#SOBEL
elif args.criterion == 'l1_from_rgb_sobel':
criterion = criteria.MaskedL1_from_rgb_sobel_Loss().cuda()
elif args.criterion == 'l1_from_GT_rgb_sobel':
criterion = criteria.MaskedL1_from_GT_rgb_sobel_Loss().cuda()
elif args.criterion == 'l1_from_GT_sobel':
criterion = criteria.MaskedL1_from_GT_sobel_Loss().cuda()
elif args.criterion == 'l2_from_GT_sobel_Loss':
criterion = criteria.MaskedL2_from_GT_sobel_Loss().cuda()
#CANNY
elif args.criterion == 'l1_canny_from_GT_canny':
criterion = criteria.MaskedL1_canny_from_GT_Loss().cuda()
# Data loading code
print("=> creating data loaders ...")
train_dir = config.train_dir
val_dir = config.val_dir
train_dataset = YT_dataset(train_dir, config, is_train_set=True)
val_dataset = YT_dataset(val_dir, config, is_train_set=False)
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=config.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True,
sampler=None,
worker_init_fn=lambda work_id: np.random.seed(work_id))
# worker_init_fn ensures different sampling patterns for each data loading thread
# set batch size to be 1 for validation
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
print("=> data loaders created.")
if args.evaluate:
best_model_filename = os.path.join(output_directory,
'model_best.pth.tar')
assert os.path.isfile(best_model_filename), \
"=> no best model found at '{}'".format(best_model_filename)
print("=> loading best model '{}'".format(best_model_filename))
checkpoint = torch.load(best_model_filename)
args.start_epoch = checkpoint['epoch']
best_result = checkpoint['best_result']
model = checkpoint['model']
print("=> loaded best model (epoch {})".format(checkpoint['epoch']))
validate(val_loader,
model,
checkpoint['epoch'],
1,
write_to_file=False)
return
elif args.test:
print("testing...")
best_model_filename = best_model_dir
assert os.path.isfile(best_model_filename), \
"=> no best model found at '{}'".format(best_model_filename)
print("=> loading best model '{}'".format(best_model_filename))
checkpoint = torch.load(best_model_filename)
args.start_epoch = checkpoint['epoch']
best_result = checkpoint['best_result']
model = checkpoint['model']
print("=> loaded best model (epoch {})".format(checkpoint['epoch']))
optimizer = checkpoint['optimizer']
for state in optimizer.state.values():
for k, v in state.items():
print(type(v))
if torch.is_tensor(v):
state[k] = v.cuda()
#test(val_loader, model, checkpoint['epoch'], write_to_file=False)
test(model)
return
elif args.resume:
assert os.path.isfile(config.resume_model_dir), \
"=> no checkpoint found at '{}'".format(config.resume_model_dir)
print("=> loading checkpoint '{}'".format(config.resume_model_dir))
best_model_filename = config.resume_model_dir
checkpoint = torch.load(best_model_filename)
args.start_epoch = checkpoint['epoch'] + 1
best_result = checkpoint['best_result']
model = checkpoint['model']
optimizer = checkpoint['optimizer']
for state in optimizer.state.values():
for k, v in state.items():
#print(type(v))
if torch.is_tensor(v):
state[k] = v.cuda(config.cuda_id)
print("=> loaded checkpoint (epoch {})".format(checkpoint['epoch']))
else:
print("=> creating Model ({}-{}) ...".format(args.arch, args.decoder))
if config.input == 'RGBT':
in_channels = 4
elif config.input == 'YT':
in_channels = 2
else:
print("Input type is wrong !")
return 0
if args.arch == 'resnet50': #调用ResNet的定义实例化model，这里的in_channels是
model = ResNet(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_deconv1_loss0': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_with_deconv(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_deconv1_loss1': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_with_deconv_loss(
layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_direct_deconv1_loss1': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_with_direct_deconv(
layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_1': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_1(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_2': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_2(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_3': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_3(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_3_1': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_3_1(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_3_2': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_3_2(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_3_3': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_3_3(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_4': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_4(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_5': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_5(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_7': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_7(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_8': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_8(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_9': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_9(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_10': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_10(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_11': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_11(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_11_1': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_11_1(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_11_without_pretrain': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_11_without_pretrain(
layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_12': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_12(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_13': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_13(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_14': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_14(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_15': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_15(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_16': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_16(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_17': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_17(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_18': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet50_18(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_30': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_30(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_31': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_31(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_32': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_32(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_33': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_33(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_40': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_40(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_15_1': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_15_1(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_15_2': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_15_2(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_15_3': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_15_3(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_15_4': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_15_4(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_15_5': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_15_5(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_15_6': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_15_6(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_15_8': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_15_8(layers=34,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_15_9': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_15_9(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_15_10': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_15_10(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_15_11': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_15_11(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_15_12': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_15_12(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet18':
model = ResNet(layers=18,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'resnet50_20':
model = ResNet50_20(Bottleneck, [3, 4, 6, 3])
elif args.arch == 'UNet':
model = UNet()
elif args.arch == 'UP_only':
model = UP_only()
elif args.arch == 'ResNet_bicubic': # 调用ResNet的定义实例化model，这里的in_channels是
model = ResNet_bicubic(layers=50,
decoder=args.decoder,
output_size=train_dataset.output_size,
in_channels=in_channels,
pretrained=args.pretrained)
elif args.arch == 'VDSR':
model = VDSR()
elif args.arch == 'VDSR_without_res':
model = VDSR_without_res()
elif args.arch == 'VDSR_16':
model = VDSR_16()
elif args.arch == 'VDSR_16_2':
model = VDSR_16_2()
elif args.arch == 'Leon_resnet50':
model = Leon_resnet50()
elif args.arch == 'Leon_resnet101':
model = Leon_resnet101()
elif args.arch == 'Leon_resnet18':
model = Leon_resnet18()
elif args.arch == 'Double_resnet50':
model = Double_resnet50()
print("=> model created.")
if args.finetune:
print("===============loading finetune model=====================")
assert os.path.isfile(config.fitune_model_dir), \
"=> no checkpoint found at '{}'".format(config.fitune_model_dir)
print("=> loading checkpoint '{}'".format(config.fitune_model_dir))
best_model_filename = config.fitune_model_dir
checkpoint = torch.load(best_model_filename)
args.start_epoch = checkpoint['epoch'] + 1
#best_result = checkpoint['best_result']
model_fitune = checkpoint['model']
model_fitune_dict = model_fitune.state_dict()
model_dict = model.state_dict()
for k in model_fitune_dict:
if k in model_dict:
#print("There is model k: ",k)
model_dict[k] = model_fitune_dict[k]
#model_dict={k:v for k,v in model_fitune_dict.items() if k in model_dict}
model_dict.update(model_fitune_dict)
model.load_state_dict(model_dict)
#optimizer = checkpoint['optimizer']
print("=> loaded checkpoint (epoch {})".format(
checkpoint['epoch']))
#optimizer = torch.optim.SGD(model.parameters(), args.lr,momentum=args.momentum, weight_decay=args.weight_decay)
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
amsgrad=True,
weight_decay=args.weight_decay)
'''
optimizer = torch.optim.Adam(
[
#{'params':model.base.parameters()}, 3
{'params': model.re_conv_Y_1.parameters(),'lr':0.0001},
{'params': model.re_conv_Y_2.parameters(), 'lr': 0.0001},
{'params': model.re_conv_Y_3.parameters(), 'lr': 0.0001},
#3
{'params': model.re_deconv_up0.parameters(), 'lr': 0.0001},
{'params': model.re_deconv_up1.parameters(), 'lr': 0.0001},
{'params': model.re_deconv_up2.parameters(), 'lr': 0.0001},
#3
{'params': model.re_conv1.parameters(), 'lr': 0.0001},
{'params': model.re_bn1.parameters(), 'lr': 0.0001},
{'params': model.re_conv4.parameters(), 'lr': 0.0001},
#5
{'params': model.re_ResNet50_layer1.parameters(), 'lr': 0.0001},
{'params': model.re_ResNet50_layer2.parameters(), 'lr': 0.0001},
{'params': model.re_ResNet50_layer3.parameters(), 'lr': 0.0001},
{'params': model.re_ResNet50_layer4.parameters(), 'lr': 0.0001},
{'params': model.re_bn2.parameters(), 'lr': 0.0001},
#5
{'params': model.re_deconcv_res_up1.parameters(), 'lr': 0.0001},
{'params': model.re_deconcv_res_up2.parameters(), 'lr': 0.0001},
{'params': model.re_deconcv_res_up3.parameters(), 'lr': 0.0001},
{'params': model.re_deconcv_res_up4.parameters(), 'lr': 0.0001},
{'params': model.re_deconv_last.parameters(), 'lr': 0.0001},
#denoise net 3
{'params': model.conv_denoise_1.parameters(), 'lr': 0},
{'params': model.conv_denoise_2.parameters(), 'lr': 0},
{'params': model.conv_denoise_3.parameters(), 'lr': 0}
]
, lr=args.lr, amsgrad=True, weight_decay=args.weight_decay)
'''
for state in optimizer.state.values():
for k, v in state.items():
print(type(v))
if torch.is_tensor(v):
state[k] = v.cuda(config.cuda_id)
print(optimizer)
# create new csv files with only header
with open(train_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
with open(test_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
# writer = SummaryWriter(log_dir='logs')
model = model.cuda(config.cuda_id)
#torch.save(model, './net1.pkl')
for state in optimizer.state.values():
for k, v in state.items():
print(type(v))
if torch.is_tensor(v):
state[k] = v.cuda()
print("=> model transferred to GPU.")
for epoch in range(args.start_epoch, args.epochs):
train(train_loader, val_loader, model, criterion, optimizer, epoch,
args.lr) # train for one epoch
parser.set_defaults(pretrained=True)
args = parser.parse_args()
if args.modality == 'rgb' and args.num_samples != 0:
print("number of samples is forced to be 0 when input modality is rgb")
args.num_samples = 0
if args.modality == 'rgb' and args.max_depth != 0.0:
print("max depth is forced to be 0.0 when input modality is rgb/rgbd")
args.max_depth = 0.0
print(args)
fieldnames = [
'mse', 'rmse', 'absrel', 'lg10', 'mae', 'delta1', 'delta2', 'delta3',
'data_time', 'gpu_time'
]
best_result = Result()
best_result.set_to_worst()
def main():
global args, best_result, output_directory, train_csv, test_csv
sparsifier = None
max_depth = args.max_depth if args.max_depth >= 0.0 else np.inf
if args.sparsifier == UniformSampling.name:
sparsifier = UniformSampling(num_samples=args.num_samples,
max_depth=max_depth)
elif args.sparsifier == SimulatedStereo.name:
sparsifier = SimulatedStereo(num_samples=args.num_samples,
max_depth=max_depth)
action="store_true",
help='use ImageNet pre-trained weights')
parser.add_argument('--val',
type=str,
default="select",
choices=["select", "full"],
help='full or select validation set')
parser.add_argument('--jitter',
type=float,
default=0.1,
help='color jitter for images')
parser.add_argument(
'--rank-metric',
type=str,
default='rmse',
choices=[m for m in dir(Result()) if not m.startswith('_')],
help='metrics for which best result is sbatch_datacted')
parser.add_argument(
'-m',
'--train-mode',
type=str,
default="dense",
choices=["dense", "sparse", "photo", "sparse+photo", "dense+photo"],
help='dense | sparse | photo | sparse+photo | dense+photo')
parser.add_argument('-e', '--evaluate', default='', type=str, metavar='PATH')
parser.add_argument('--cpu', action="store_true", help='run on cpu')
parser.add_argument(
'--partial-train',
default="no",
type=str,
help=
def iterate(mode, args, loader, model, optimizer, logger, epoch):
block_average_meter = AverageMeter()
average_meter = AverageMeter()
meters = [block_average_meter, average_meter]
# switch to appropriate mode
assert mode in ["train", "val", "eval", "test_prediction", "test_completion"], \
"unsupported mode: {}".format(mode)
if mode == 'train':
model.train()
lr = helper.adjust_learning_rate(args.lr, optimizer, epoch)
else:
model.eval(
) # batchnorm or dropout layers will work in eval mode instead of training mode
lr = 0
for i, batch_data in enumerate(
loader
): # batch_data keys: 'd' (depth), 'gt' (ground truth), 'g' (gray)
start = time.time()
batch_data = {
key: val.to(device)
for key, val in batch_data.items() if val is not None
}
gt = batch_data[
'gt'] if mode != 'test_prediction' and mode != 'test_completion' else None
data_time = time.time() - start
start = time.time()
pred = model(batch_data)
if args.save_images: # save depth predictions
pred_out_dir = max(glob.glob('../outputs/var_final_NN/var.test*'),
key=os.path.getmtime) + '/dense_depth_images'
pred1 = pred.cpu().detach().numpy()[:, 0, :, :]
for im_idx, pred_im in enumerate(pred1):
pred_out_dir1 = os.path.abspath(pred_out_dir)
cur_path = os.path.abspath((loader.dataset.paths)['d'][i])
basename = os.path.basename(cur_path)
cur_dir = os.path.abspath(os.path.dirname(cur_path))
cur_dir = cur_dir.split('var_final_NN/')[1]
new_dir = os.path.abspath(pred_out_dir1 + '/' + cur_dir)
new_path = os.path.abspath(new_dir + '/' + basename)
if os.path.isdir(new_dir) == False:
os.makedirs(new_dir)
depth_write(new_path, pred_im)
depth_loss, photometric_loss, smooth_loss, mask = 0, 0, 0, None
if mode == 'train':
# Loss 1: the direct depth supervision from ground truth label
# mask=1 indicates that a pixel does not ground truth labels
if 'sparse' in args.train_mode:
depth_loss = depth_criterion(pred, batch_data['d'])
mask = (batch_data['d'] < 1e-3).float()
elif 'dense' in args.train_mode:
depth_loss = depth_criterion(pred, gt)
mask = (gt < 1e-3).float()
# Loss 2: the self-supervised photometric loss
if args.use_pose:
# create multi-scale pyramids
pred_array = helper.multiscale(pred)
rgb_curr_array = helper.multiscale(batch_data['rgb'])
rgb_near_array = helper.multiscale(batch_data['rgb_near'])
if mask is not None:
mask_array = helper.multiscale(mask)
num_scales = len(pred_array)
# compute photometric loss at multiple scales
for scale in range(len(pred_array)):
pred_ = pred_array[scale]
rgb_curr_ = rgb_curr_array[scale]
rgb_near_ = rgb_near_array[scale]
mask_ = None
if mask is not None:
mask_ = mask_array[scale]
# compute the corresponding intrinsic parameters
height_, width_ = pred_.size(2), pred_.size(3)
intrinsics_ = kitti_intrinsics.scale(height_, width_)
# inverse warp from a nearby frame to the current frame
warped_ = homography_from(rgb_near_, pred_,
batch_data['r_mat'],
batch_data['t_vec'], intrinsics_)
photometric_loss += photometric_criterion(
rgb_curr_, warped_, mask_) * (2**(scale - num_scales))
# Loss 3: the depth smoothness loss
smooth_loss = smoothness_criterion(pred) if args.w2 > 0 else 0
# backprop
loss = depth_loss + args.w1 * photometric_loss + args.w2 * smooth_loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
gpu_time = time.time() - start
# measure accuracy and record loss of each batch
with torch.no_grad(
): # impacts the autograd engine and deactivate it (will reduce memory usage and speed up computations)
mini_batch_size = next(iter(batch_data.values())).size(0)
result = Result() # metrics
if mode != 'test_prediction' and mode != 'test_completion':
result.evaluate(pred.data, gt.data, photometric_loss)
[
m.update(result, gpu_time, data_time, mini_batch_size)
for m in meters
]
logger.conditional_print(mode, i, epoch, args.epochs, lr,
len(loader), block_average_meter,
average_meter)
logger.conditional_save_img_comparison(mode, i, batch_data, pred,
epoch)
logger.conditional_save_pred(mode, i, pred, epoch)
del pred
avg = logger.conditional_save_info(
mode, average_meter,
epoch) # take the avg of all the batches, to get the epoch metrics
is_best = logger.rank_conditional_save_best(mode, avg, epoch, args.epochs)
if is_best and not (mode == "train"):
logger.save_img_comparison_as_best(mode, epoch)
logger.conditional_summarize(mode, avg, is_best)
return avg, is_best
def validate(val_loader, model, epoch, batch_epoch, write_to_file=True):
average_meter = AverageMeter()
model.eval() # switch to evaluate mode
end = time.time()
for i, (Y, Y_1_2, Y_1_4, Y_1_8, LR, LR_8, HR,
name) in enumerate(val_loader):
Y = Y.cuda()
Y_1_2 = Y_1_2.cuda()
Y_1_4 = Y_1_4.cuda()
Y_1_8 = Y_1_8.cuda()
LR_8 = LR_8.cuda()
LR = LR.cuda()
HR = HR.cuda()
torch.cuda.synchronize()
data_time = time.time() - end
# compute output
end = time.time()
with torch.no_grad():
# print("I am for validation in main 342")
if args.arch == 'VDSR_16':
pred_HR = model(LR)
elif args.arch == 'VDSR_16_2':
pred_HR = model(Y, LR)
elif args.arch == 'VDSR':
pred_HR, residule = model(LR_8, Y)
elif args.arch == 'ResNet_bicubic':
pred_HR, residule = model(LR_8, Y)
elif args.arch == 'resnet50_15_6' or 'resnet50_15_11' or 'resnet50_15_12':
pred_HR = model(Y_1_2, LR)
elif args.arch == 'resnet50_15_2' or 'resnet50_15_3' or 'resnet50_15_5' or 'resnet50_15_8' or 'resnet50_15_9':
pred_HR, residule = model(Y, LR, LR_8)
else:
if config.use_different_size_Y == 1:
pred_HR, pred_thermal0 = model(Y, Y_1_2, Y_1_4, Y_1_8, LR)
else:
pred_HR, pred_thermal = model(Y, LR)
torch.cuda.synchronize()
gpu_time = time.time() - end
# measure accuracy and record loss
result = Result()
result.evaluate(pred_HR, HR)
average_meter.update(result, gpu_time, data_time,
Y.size(0)) #Y.size(0) batch_size
end = time.time()
# save 8 images for visualization,对验证集合生产图片
skip = config.skip
if i == 0:
img_merge = utils.merge_into_row_with_YT(Y, LR, pred_HR, HR)
elif (i < 8 * skip) and (i % skip == 0):
row = utils.merge_into_row_with_YT(Y, LR, pred_HR, HR)
img_merge = utils.add_row(img_merge, row)
elif i == 8 * skip: #储存最终的图片
filename = output_directory + '/Compair_data_epoch_' + str(
epoch) + '_batch_eopch_' + str(batch_epoch + 1) + '.png'
utils.save_image(img_merge, filename)
print("生成第" + str(batch_epoch + 1) + "图片")
if (i + 1) % args.print_freq == 0:
print('Test: [{0}/{1}]\t'
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'RMSE={result.rmse:.2f}({average.rmse:.2f}) '
'MAE={result.mae:.2f}({average.mae:.2f}) '
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'REL={result.absrel:.3f}({average.absrel:.3f}) '
'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format(
i + 1,
len(val_loader),
gpu_time=gpu_time,
result=result,
average=average_meter.average()))
avg = average_meter.average()
print('\n*\n'
'RMSE={average.rmse:.3f}\n'
'MAE={average.mae:.3f}\n'
'Delta1={average.delta1:.3f}\n'
'REL={average.absrel:.3f}\n'
'Lg10={average.lg10:.3f}\n'
't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time))
if write_to_file:
with open(test_csv, 'a') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow({
'dataset epoch': epoch,
'batch epoch': batch_num + 1,
'psnr': 10 * math.log(1 / (avg.mse), 10),
'mse': avg.mse,
'rmse': avg.rmse,
'absrel': avg.absrel,
'lg10': avg.lg10,
'mae': avg.mae,
'delta1': avg.delta1,
'delta2': avg.delta2,
'delta3': avg.delta3,
'data_time': avg.data_time,
'gpu_time': avg.gpu_time
})
return avg, img_merge
def validate(val_loader, model, epoch, write_to_file=True):
average_meter = AverageMeter()
# switch to evaluate mode
model.eval()
end = time.time()
for i, (input, target) in enumerate(val_loader):
input, target = input.cuda(), target.cuda()
# torch.cuda.synchronize()
data_time = time.time() - end
# compute output
end = time.time()
with torch.no_grad():
pred = model(input)
# torch.cuda.synchronize()
gpu_time = time.time() - end
# measure accuracy and record loss
result = Result()
result.evaluate(pred.data, target.data)
average_meter.update(result, gpu_time, data_time, input.size(0))
end = time.time()
# save 8 images for visualization
skip = 50
if args.modality == 'd':
img_merge = None
else:
if args.modality == 'rgb':
rgb = input
elif args.modality == 'rgbd':
rgb = input[:, :3, :, :]
depth = input[:, 3:, :, :]
if i == 0:
if args.modality == 'rgbd':
img_merge = utils.merge_into_row_with_gt(
rgb, depth, target, pred)
else:
img_merge = utils.merge_into_row(rgb, target, pred)
elif (i < 8 * skip) and (i % skip == 0):
if args.modality == 'rgbd':
row = utils.merge_into_row_with_gt(rgb, depth, target,
pred)
else:
row = utils.merge_into_row(rgb, target, pred)
img_merge = utils.add_row(img_merge, row)
elif i == 8 * skip:
filename = output_directory + '/comparison_' + str(
epoch) + '.png'
utils.save_image(img_merge, filename)
if (i + 1) % args.print_freq == 0:
print('Test: [{0}/{1}]\t'
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'RMSE={result.rmse:.2f}({average.rmse:.2f}) '
'MAE={result.mae:.2f}({average.mae:.2f})\n\t'
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'REL={result.absrel:.3f}({average.absrel:.3f}) '
'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format(
i + 1,
len(val_loader),
gpu_time=gpu_time,
result=result,
average=average_meter.average()))
avg = average_meter.average()
print('\n*\n'
'RMSE={average.rmse:.3f}\n'
'MAE={average.mae:.3f}\n'
'Delta1={average.delta1:.3f}\n'
'REL={average.absrel:.3f}\n'
'Lg10={average.lg10:.3f}\n'
't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time))
if write_to_file:
with open(test_csv, 'a') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow({
'mse': avg.mse,
'rmse': avg.rmse,
'absrel': avg.absrel,
'lg10': avg.lg10,
'mae': avg.mae,
'delta1': avg.delta1,
'delta2': avg.delta2,
'delta3': avg.delta3,
'data_time': avg.data_time,
'gpu_time': avg.gpu_time
})
return avg, img_merge
def test(model):
global name
test_dir = config.test_dir
test_dataset = YT_dataset(test_dir, config, is_train_set=False)
test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
file_name = config.test_result_name + str('.csv')
test_csv_2 = os.path.join(config.test_output_dir, file_name)
#print(test_csv_2)
average_meter = AverageMeter()
model.eval() # switch to evaluate mode
end = time.time()
num = 0
for i, (Y, Y_1_2, Y_1_4, Y_1_8, LR, LR_8, HR,
name) in enumerate(test_loader):
Y = Y.cuda()
LR = LR.cuda()
LR_8 = LR_8.cuda()
HR = HR.cuda()
torch.cuda.synchronize()
data_time = time.time() - end
end = time.time()
with torch.no_grad():
# print("I am for validation in main 342")
num = num + 1
if args.arch == 'VDSR_16':
pred_HR = model(LR)
elif args.arch == 'VDSR_16_2':
pred_HR = model(Y, LR)
elif args.arch == 'VDSR':
pred_HR, residule = model(LR_8, Y)
elif args.arch == 'ResNet_bicubic':
pred_HR, residule = model(LR_8, Y)
elif args.arch == 'resnet50_15_2' or 'resnet50_15_3':
pred_HR, residule = model(Y, LR, LR_8)
else:
pred_HR, thermal0 = model(Y, LR)
gpu_time = time.time() - end
image_pred_HR = trans_norm_tensor_into_T(pred_HR)
#image_pred_HR = trans_norm_tensor_into_T(thermal0)
image_LR = trans_norm_tensor_into_rgb(LR)
image_HR = trans_norm_tensor_into_rgb(HR)
#image_HR=trans_norm_tensor_into_T(HR)
name_HR = str(
config.test_output_dir_HR) + str(num) + str('_HR') + str(".png")
name_pred_HR = str(
config.test_output_dir_pred_HR) + str(num) + str(".png")
name_HR_bicubic_HR_res = str(
config.test_output_dir_res) + str(num) + str(".png")
#name_bicubic_HR = str(config.test_output_dir_bicubic_HR) + str(num) + str(".png")
#cv2.imwrite(name_HR,image_HR[...,::-1])
#print(name_pred_HR)
#cv2.imwrite(name_pred_HR,image_pred_HR[...,::-1])
#cv2.waitKey(0)
torch.cuda.synchronize()
# measure accuracy and record loss
result = Result()
result.evaluate(pred_HR, HR)
average_meter.update(result, gpu_time, data_time,
Y.size(0)) # Y.size(0) batch_size
end = time.time()
# save 8 images for visualization,对验证集合生产图片
dir_small = '/home1/sas/datasets/crop/smallthan7/'
dir_big = '/home1/sas/datasets/crop/bigthan/'
dir_out = '/home/zju231/sas/data/old_val_resnet50_15_1/'
#dir_out='/home1/sas/results_model/pytorch/YT-SD/resnet50_30_thermal0/'
#dir_out=test_dir
name1 = dir_out + str(name[0][:-4]) + str('.png')
cv2.imwrite(name1, image_pred_HR)
name2 = dir_out + str(name[0][:-4]) + str('_HR.png')
#cv2.imwrite(name2, image_HR[..., ::-1])
name3 = dir_out + str(name[0][:-4]) + str('_Y.png')
#cv2.imwrite(name3, np.squeeze(np.array(Y.cpu())) * 255)
name4 = dir_out + str(name[0][:-4]) + str('_LR.png')
#cv2.imwrite(name4, image_LR[..., ::-1])
print('Test: [{0}/{1}]\t'
't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t'
'PSNR={result.psnr:.5f}({average.psnr:.5f}) '
'RMSE={result.rmse:.5f}({average.rmse:.5f}) '
'MAE={result.mae:.5f}({average.mae:.5f}) '
'Delta1={result.delta1:.3f}({average.delta1:.3f}) '
'REL={result.absrel:.3f}({average.absrel:.3f}) '
'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format(
i + 1,
len(test_loader),
gpu_time=gpu_time,
result=result,
average=average_meter.average()))
avg = average_meter.average()
print('\n*\n'
'PSNR={average.psnr:.5f}\n'
'RMSE={average.rmse:.3f}\n'
'MAE={average.mae:.3f}\n'
'Delta1={average.delta1:.3f}\n'
'REL={average.absrel:.3f}\n'
'Lg10={average.lg10:.3f}\n'
't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time))
if 1:
with open(test_csv_2, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
writer.writerow({
'psnr': 10 * math.log(1 / (avg.mse), 10),
'mse': avg.mse,
'rmse': avg.rmse,
'absrel': avg.absrel,
'lg10': avg.lg10,
'mae': avg.mae,
'delta1': avg.delta1,
'delta2': avg.delta2,
'delta3': avg.delta3,
'data_time': avg.data_time,
'gpu_time': avg.gpu_time
})
