def train(train_loader, model, criterion, optimizer, epoch): average_meter = AverageMeter() model.train() # switch to train mode end = time.time() for i, (input, target) in enumerate(train_loader): input, target = input.cuda(), target.cuda() torch.cuda.synchronize() data_time = time.time() - end # compute pred end = time.time() pred = model(input) loss = criterion(pred, target) optimizer.zero_grad() loss.backward() # compute gradient and do SGD step optimizer.step() torch.cuda.synchronize() gpu_time = time.time() - end # measure accuracy and record loss result = Result() result.evaluate(pred.data, target.data) average_meter.update(result, gpu_time, data_time, input.size(0)) end = time.time() if (i + 1) % args.print_freq == 0: print('=> output: {}'.format(output_directory)) print('Train Epoch: {0} [{1}/{2}]\t' 't_Data={data_time:.3f}({average.data_time:.3f}) ' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'RMSE={result.rmse:.2f}({average.rmse:.2f}) ' 'MAE={result.mae:.2f}({average.mae:.2f}) ' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'REL={result.absrel:.3f}({average.absrel:.3f}) ' 'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format( epoch, i + 1, len(train_loader), data_time=data_time, gpu_time=gpu_time, result=result, average=average_meter.average())) avg = average_meter.average() with open(train_csv, 'a') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writerow({ 'mse': avg.mse, 'rmse': avg.rmse, 'absrel': avg.absrel, 'lg10': avg.lg10, 'mae': avg.mae, 'delta1': avg.delta1, 'delta2': avg.delta2, 'delta3': avg.delta3, 'gpu_time': avg.gpu_time, 'data_time': avg.data_time })
def __init__(self, args, prepare=True): self.args = args self.args.save_pred = True output_directory = get_folder_name(args) self.output_directory = output_directory self.best_result = Result() self.best_result.set_to_worst() if not prepare: return if not os.path.exists(output_directory): os.makedirs(output_directory) self.train_csv = os.path.join(output_directory, 'train.csv') self.val_csv = os.path.join(output_directory, 'val.csv') self.best_txt = os.path.join(output_directory, 'best.txt') # backup the source code if args.resume == '': print("=> creating source code backup ...") backup_directory = os.path.join(output_directory, "code_backup") self.backup_directory = backup_directory print("=> stop source code backup ...") # backup_source_code(backup_directory) # create new csv files with only header with open(self.train_csv, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() with open(self.val_csv, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() print("=> finished creating source code backup.")
def train(train_loader, model, criterion, optimizer, epoch, logger): average_meter = AverageMeter() model.train() # switch to train mode end = time.time() batch_num = len(train_loader) for i, (input, target) in enumerate(train_loader): # itr_count += 1 input, target = input.cuda(), target.cuda() # print('input size = ', input.size()) # print('target size = ', target.size()) torch.cuda.synchronize() data_time = time.time() - end # compute pred end = time.time() pred = model(input) # @wx 注意输出 # print('pred size = ', pred.size()) # print('target size = ', target.size()) loss = criterion(pred, target) optimizer.zero_grad() loss.backward() # compute gradient and do SGD step optimizer.step() torch.cuda.synchronize() gpu_time = time.time() - end # measure accuracy and record loss result = Result() result.evaluate(pred.data, target.data) average_meter.update(result, gpu_time, data_time, input.size(0)) end = time.time() if (i + 1) % args.print_freq == 0: print('=> output: {}'.format(output_directory)) print('Train Epoch: {0} [{1}/{2}]\t' 't_Data={data_time:.3f}({average.data_time:.3f}) ' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'Loss={Loss:.5f} ' 'RMSE={result.rmse:.2f}({average.rmse:.2f}) ' 'RML={result.absrel:.2f}({average.absrel:.2f}) ' 'Log10={result.lg10:.3f}({average.lg10:.3f}) ' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'Delta2={result.delta2:.3f}({average.delta2:.3f}) ' 'Delta3={result.delta3:.3f}({average.delta3:.3f})'.format( epoch, i + 1, len(train_loader), data_time=data_time, gpu_time=gpu_time, Loss=loss.item(), result=result, average=average_meter.average())) current_step = epoch * batch_num + i logger.add_scalar('Train/RMSE', result.rmse, current_step) logger.add_scalar('Train/rml', result.absrel, current_step) logger.add_scalar('Train/Log10', result.lg10, current_step) logger.add_scalar('Train/Delta1', result.delta1, current_step) logger.add_scalar('Train/Delta2', result.delta2, current_step) logger.add_scalar('Train/Delta3', result.delta3, current_step) avg = average_meter.average()
def validate(epoch, valData, model, logger): average_meter = AverageMeter() model.eval() # switch to evaluate mode end = time.time() # skip = len(valData) // 9 # save images every skip iters for i, (image, depth) in enumerate(valData): image = image.cuda() depth = depth.cuda() # normal = normal.cuda() torch.cuda.synchronize() data_time = time.time() - end end = time.time() with torch.no_grad(): pred = model(image) torch.cuda.synchronize() gpu_time = time.time() - end result = Result() result.evaluate(pred.data, depth.data) average_meter.update(result, gpu_time, data_time, image.size(0)) end = time.time() if (i + 1) % 10 == 0: print('Test Epoch: [{0}/{1}]\t' 't_Data={data_time:.3f}({average.data_time:.3f}) ' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'RMSE={result.rmse:.2f}({average.rmse:.2f}) ' 'RML={result.absrel:.2f}({average.absrel:.2f}) ' 'Log10={result.lg10:.3f}({average.lg10:.3f}) ' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'Delta2={result.delta2:.3f}({average.delta2:.3f}) ' 'Delta3={result.delta3:.3f}({average.delta3:.3f})'.format( i + 1, len(valData), data_time=data_time, gpu_time=gpu_time, result=result, average=average_meter.average())) avg = average_meter.average() print('\n*\n' 'RMSE={average.rmse:.3f}\n' 'Rel={average.absrel:.3f}\n' 'Log10={average.lg10:.3f}\n' 'Delta1={average.delta1:.3f}\n' 'Delta2={average.delta2:.3f}\n' 'Delta3={average.delta3:.3f}\n' 't_GPU={time:.3f}\n'.format( average=avg, time=avg.gpu_time)) logger.add_scalar('Test/rmse', avg.rmse, epoch) logger.add_scalar('Test/Rel', avg.absrel, epoch) logger.add_scalar('Test/log10', avg.lg10, epoch) logger.add_scalar('Test/Delta1', avg.delta1, epoch) logger.add_scalar('Test/Delta2', avg.delta2, epoch) logger.add_scalar('Test/Delta3', avg.delta3, epoch) return avg
def train(train_loader, model, criterion, optimizer, epoch): average_meter = AverageMeter() model.train() # switch to train mode end = time.time() for i, (input, target) in enumerate(train_loader): input, target = input.cuda(), target.cuda() torch.cuda.synchronize() data_time = time.time() - end # compute pred end = time.time() pred = model(input) loss = criterion(pred, target) optimizer.zero_grad() loss.backward() # compute gradient and do SGD step optimizer.step() torch.cuda.synchronize() gpu_time = time.time() - end depth_in = np.hstack(input.data.cpu().numpy()[:4, 3] / 10.) depth_in = cv2.applyColorMap((depth_in * 255).astype(np.uint8), cv2.COLORMAP_HOT) tgt_out = np.hstack(np.squeeze(target[:4].data.cpu().numpy())) / 10. tgt_out = cv2.applyColorMap((tgt_out * 255).astype(np.uint8), cv2.COLORMAP_HOT) out = np.hstack(np.squeeze(pred[:4].data.cpu().numpy())) out = np.clip(out / 10., 0., 1.) out = cv2.applyColorMap((out * 255).astype(np.uint8), cv2.COLORMAP_HOT) if i % 20 == 0: cv2.imshow("Training Results", np.vstack([depth_in, tgt_out, out])) cv2.waitKey(1) # measure accuracy and record loss result = Result() result.evaluate(pred.data, target.data) average_meter.update(result, gpu_time, data_time, input.size(0)) end = time.time() if (i + 1) % args.print_freq == 0: print('=> output: {}'.format(output_directory)) print('Train Epoch: {0} [{1}/{2}]\t' 't_Data={data_time:.3f}({average.data_time:.3f}) ' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'RMSE={result.rmse:.2f}({average.rmse:.2f}) ' 'MAE={result.mae:.2f}({average.mae:.2f}) ' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'REL={result.absrel:.3f}({average.absrel:.3f}) ' 'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format( epoch, i+1, len(train_loader), data_time=data_time, gpu_time=gpu_time, result=result, average=average_meter.average())) avg = average_meter.average() with open(train_csv, 'a') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writerow({'mse': avg.mse, 'rmse': avg.rmse, 'absrel': avg.absrel, 'lg10': avg.lg10, 'mae': avg.mae, 'delta1': avg.delta1, 'delta2': avg.delta2, 'delta3': avg.delta3, 'gpu_time': avg.gpu_time, 'data_time': avg.data_time})
def iterate(mode, args, loader, model, optimizer, logger, epoch): block_average_meter = AverageMeter() average_meter = AverageMeter() meters = [block_average_meter, average_meter] # switch to appropriate mode assert mode in ["train", "val", "eval", "test_prediction", "test_completion"], \ "unsupported mode: {}".format(mode) if mode == 'train': model.train() lr = helper.adjust_learning_rate(args.lr, optimizer, epoch) else: model.eval() lr = 0 for i, batch_data in enumerate(loader): start = time.time() batch_data = { key: val.to(device) for key, val in batch_data.items() if val is not None } gt = batch_data[ 'gt'] if mode != 'test_prediction' and mode != 'test_completion' else None data_time = time.time() - start start = time.time() with torch.no_grad(): # 自己加的 pred = model(batch_data) depth_loss, photometric_loss, smooth_loss, mask = 0, 0, 0, None gpu_time = time.time() - start # measure accuracy and record loss with torch.no_grad(): mini_batch_size = next(iter(batch_data.values())).size(0) result = Result() if mode != 'test_prediction' and mode != 'test_completion': #result.evaluate(pred.data, gt.data, photometric_loss) result.evaluate(pred.data.cpu(), gt.data.cpu(), photometric_loss) [ m.update(result, gpu_time, data_time, mini_batch_size) for m in meters ] logger.conditional_print(mode, i, epoch, lr, len(loader), block_average_meter, average_meter) logger.conditional_save_img_comparison(mode, i, batch_data, pred, epoch) logger.conditional_save_pred(mode, i, pred, epoch) avg = logger.conditional_save_info(mode, average_meter, epoch) is_best = logger.rank_conditional_save_best(mode, avg, epoch) if is_best and not (mode == "train"): logger.save_img_comparison_as_best(mode, epoch) logger.conditional_summarize(mode, avg, is_best) return avg, is_best
def __init__(self, args, prepare=True): self.args = args output_directory = get_folder_name(args) self.output_directory = output_directory self.best_result = Result() self.best_result.set_to_worst() self.best_result_intensity = Result_intensity() self.best_result_intensity.set_to_worst() #visual # self.viz = Visdom(server='http://10.5.40.31', port=11207) # assert self.viz.check_connection() from datetime import datetime, timedelta, timezone utc_dt = datetime.utcnow().replace(tzinfo=timezone.utc) cn_dt = utc_dt.astimezone(timezone(timedelta(hours=8))) TIMESTAMP = "{0:%Y-%m-%dT%H-%M-%S/}".format(datetime.now()) name_prefix = "log_lr{}bt{}decay{}wi{}wpure{}lradj{}/".format( args.lr, args.batch_size, args.weight_decay, args.wi, args.wpure, args.lradj) #name_prefix = "log_lr{}bt{}decay{}_ws_{}_alphabeta_{}_{}/".format(args.lr ,args.batch_size, args.weight_decay, args.ws, args.alpha, args.beta) print(name_prefix + TIMESTAMP) self.writer = SummaryWriter(name_prefix + TIMESTAMP) if not prepare: return if not os.path.exists(output_directory): os.makedirs(output_directory) self.train_csv = os.path.join(output_directory, 'train.csv') self.val_csv = os.path.join(output_directory, 'val.csv') self.best_txt = os.path.join(output_directory, 'best.txt') self.train_csv_intensity = os.path.join(output_directory, 'train_intensity.csv') self.val_csv_intensity = os.path.join(output_directory, 'val_intensity.csv') # backup the source code if args.resume == '': print("=> creating source code backup ...") backup_directory = os.path.join(output_directory, "code_backup") self.backup_directory = backup_directory backup_source_code(backup_directory) # create new csv files with only header with open(self.train_csv, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() with open(self.val_csv, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() with open(self.train_csv_intensity, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() with open(self.val_csv_intensity, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() print("=> finished creating source code backup.")
def write_results(txt: str, res: Result, res_inside: Result, res_outside: Result, epoch: int): res.name = "result" res_inside.name = "result inside" res_outside.name = "result outside" with open(txt, 'w') as txtfile: txtfile.write(str(res) + "\n") txtfile.write(str(res_inside) + "\n") txtfile.write(str(res_outside) + "\n") txtfile.write(f"epoch: {epoch}")
def train(epoch, trainData, model, crite, optimizer, logger): average_meter = AverageMeter() model.train() # switch to train mode end = time.time() for i, (image, depth) in enumerate(trainData): image = image.cuda() depth = depth.cuda() # normal = normal.cuda() # image = torch.autograd.Variable(image) # depth = torch.autograd.Variable(depth) torch.cuda.synchronize() data_time = time.time() - end end = time.time() optimizer.zero_grad() pred = model(image) loss = crite(pred, depth) loss.backward() optimizer.step() torch.cuda.synchronize() gpu_time = time.time() - end result = Result() result.evaluate(pred.data, depth.data) average_meter.update(result, gpu_time, data_time, image.size(0)) end = time.time() if (i + 1) % 10 == 0: print('=> output: {}'.format(opt.output_dir)) print('Train Epoch: {0} [{1}/{2}]\t' 't_Data={data_time:.3f}({average.data_time:.3f}) ' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'Loss={Loss:.5f} ' 'RMSE={result.rmse:.2f}({average.rmse:.2f}) ' 'RML={result.absrel:.2f}({average.absrel:.2f}) ' 'Log10={result.lg10:.3f}({average.lg10:.3f}) ' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'Delta2={result.delta2:.3f}({average.delta2:.3f}) ' 'Delta3={result.delta3:.3f}({average.delta3:.3f})'.format( epoch, i + 1, len(trainData), data_time=data_time, gpu_time=gpu_time, Loss=loss.item(), result=result, average=average_meter.average())) current_step = epoch * len(trainData) + i logger.add_scalar('Train/loss', loss, current_step) logger.add_scalar('Train/RMSE', result.rmse, current_step) logger.add_scalar('Train/rml', result.absrel, current_step) logger.add_scalar('Train/Log10', result.lg10, current_step) logger.add_scalar('Train/Delta1', result.delta1, current_step) logger.add_scalar('Train/Delta2', result.delta2, current_step) logger.add_scalar('Train/Delta3', result.delta3, current_step)
def compute_depth_metrics(self, verbose=True) -> Result: """Computes metrics on the difference between raw and fixed depth values""" avg = AverageMeter() for i, path in enumerate(self.paths): _, depth_raw, depth_fix = self.load_images(path) depth_raw = torch.tensor(depth_raw) depth_fix = torch.tensor(depth_fix) res = Result() res.evaluate(depth_raw, depth_fix) avg.update(res, 0, 0, 1) if verbose: stdout.write(f"=> computing img {i}/{len(self)}\r") if verbose: stdout.write("\n") return avg.average()
def train_coarse(train_loader, model, criterion, optimizer, epoch): average_meter = AverageMeter() model.train() # switch to train mode end = time.time() for i, (input, target) in enumerate(train_loader): input, target = input.cuda(), target.cuda() torch.cuda.synchronize() data_time = time.time() - end # compute pred end = time.time() pred = model(input) loss = criterion(pred, target) optimizer.zero_grad() loss.backward() # compute gradient and do SGD step optimizer.step() torch.cuda.synchronize() gpu_time = time.time() - end # measure accuracy and record loss end = time.time() result = Result() result.evaluate(pred.data, target.data) average_meter.update(result, gpu_time, data_time, input.size(0)) eval_time = time.time() - end if (i + 1) % args.print_freq == 0: history_loss.append(loss.item()) print('=> output: {}'.format(output_directory)) print('Train Epoch: {0} [{1}/{2}]\t' 't_Data={data_time:.3f}({average.data_time:.3f}) ' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'RMSE={result.rmse:.2f}({average.rmse:.2f}) ' 'MAE={result.mae:.2f}({average.mae:.2f}) ' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'REL={result.absrel:.3f}({average.absrel:.3f}) ' 'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format( epoch, i + 1, len(train_loader), data_time=data_time, gpu_time=gpu_time, result=result, average=average_meter.average()))
def evaluate(params, loader, model, experiment): print("Testing...") with experiment.test() and torch.no_grad(): average = AverageMeter() end = time.time() for i, (inputs, targets) in enumerate(loader): inputs, targets = inputs.to(params["device"]), targets.to( params["device"]) data_time = time.time() - end # Predict end = time.time() outputs = model(inputs) gpu_time = time.time() - end # Clip prediction outputs[outputs > params["depth_max"]] = params["depth_max"] outputs[outputs < params["depth_min"]] = params["depth_min"] result = Result() result.evaluate(outputs.data, targets.data) average.update(result, gpu_time, data_time, inputs.size(0)) # Log images to comet img_merged = utils.log_image_to_comet(inputs[0], targets[0], outputs[0], epoch=0, id=i, experiment=experiment, result=result, prefix="visual_test") if params["save_test_images"]: filename = os.path.join( params["experiment_dir"], "image_{}_epoch_{}.png".format(i, str(params["start_epoch"]))) utils.save_image(img_merged, filename)
def __load_result(path): # Loads results from the file at the specified path and returns a corresponding Result object labels = datapoint_features # List of all feature labels # Split path on '\' and find the index of 'result' in the resulting list substrings = (path[:-4]).split("\\") begin = len(substrings) - 1 while substrings[begin] != 'result': begin -= 1 # Extract information about the used dataset, situated at specific offsets from the 'begin' index baseline = substrings[begin + 1] == 'baseline' model = substrings[begin + 2] imp_split = substrings[begin + 3] != 'imp_full' dos_type = substrings[begin + 4] # Split the remaining string on '_' and discard 'mixed' file_split = substrings[begin + 5].split("_")[1:] is_test = file_split[0] == 'test' window_ms = int((file_split[2])[:-2]) stride_ms = int((file_split[3])[:-2]) subset = [] # Use each substring from index 4 as an index into the list of all feature labels, to establish a feature subset for substring in file_split[4:]: index = int(substring) subset.append(labels[index]) # Use the gathered information to load the metrics and times metrics = load_metrics(window_ms, stride_ms, imp_split, dos_type, model, baseline, subset, is_test) times = load_times(window_ms, stride_ms, imp_split, dos_type, model, baseline, subset, is_test) return Result(window_ms, stride_ms, model, imp_split, dos_type, baseline, subset, is_test, metrics, times)
def validate(val_loader, model, epoch, write_to_file=True): average_meter = AverageMeter() model.eval() # switch to evaluate mode end = time.time() for i, (input, target) in enumerate(val_loader): input, target = input.cuda(), target.cuda() torch.cuda.synchronize() data_time = time.time() - end # compute output end = time.time() ############################################################## ## Start of PnP-Depth modification ## ############################################################## # Original inference with torch.no_grad(): ori_pred = model.pnp_forward_front(model.pnp_forward_rear( input)) # equivalent to `ori_pred = model(input)` # Inference with PnP sparse_target = input[:, -1:] # NOTE: written for rgbd input criterion = criteria.MaskedL1Loss().cuda( ) # NOTE: criterion function defined here only for clarity pnp_iters = 5 # number of iterations pnp_alpha = 0.01 # update/learning rate pnp_z = model.pnp_forward_front(input) for pnp_i in range(pnp_iters): if pnp_i != 0: pnp_z = pnp_z - pnp_alpha * torch.sign(pnp_z_grad) # iFGM pnp_z = Variable(pnp_z, requires_grad=True) pred = model.pnp_forward_rear(pnp_z) if pnp_i < pnp_iters - 1: pnp_loss = criterion(pred, sparse_target) pnp_z_grad = Grad([pnp_loss], [pnp_z], create_graph=True)[0] ############################################################## ## End of PnP-Depth modification ## ############################################################## torch.cuda.synchronize() gpu_time = time.time() - end # measure accuracy and record loss result = Result() result.evaluate(pred.data, target.data) average_meter.update(result, gpu_time, data_time, input.size(0)) end = time.time() # save 8 images for visualization skip = 50 if args.modality == 'd': img_merge = None else: if args.modality == 'rgb': rgb = input elif args.modality == 'rgbd': rgb = input[:, :3, :, :] depth = input[:, 3:, :, :] if i == 0: if args.modality == 'rgbd': img_merge = utils.merge_into_row_with_gt( rgb, depth, target, pred) else: img_merge = utils.merge_into_row(rgb, target, pred) elif (i < 8 * skip) and (i % skip == 0): if args.modality == 'rgbd': row = utils.merge_into_row_with_gt(rgb, depth, target, pred) else: row = utils.merge_into_row(rgb, target, pred) img_merge = utils.add_row(img_merge, row) elif i == 8 * skip: filename = output_directory + '/comparison_' + str( epoch) + '.png' utils.save_image(img_merge, filename) if (i + 1) % args.print_freq == 0: print('Test: [{0}/{1}]\t' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'RMSE={result.rmse:.2f}({average.rmse:.2f}) ' 'MAE={result.mae:.2f}({average.mae:.2f}) ' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'REL={result.absrel:.3f}({average.absrel:.3f}) ' 'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format( i + 1, len(val_loader), gpu_time=gpu_time, result=result, average=average_meter.average())) avg = average_meter.average() print('\n*\n' 'RMSE={average.rmse:.3f}\n' 'MAE={average.mae:.3f}\n' 'Delta1={average.delta1:.3f}\n' 'REL={average.absrel:.3f}\n' 'Lg10={average.lg10:.3f}\n' 't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time)) if write_to_file: with open(test_csv, 'a') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writerow({ 'mse': avg.mse, 'rmse': avg.rmse, 'absrel': avg.absrel, 'lg10': avg.lg10, 'mae': avg.mae, 'delta1': avg.delta1, 'delta2': avg.delta2, 'delta3': avg.delta3, 'data_time': avg.data_time, 'gpu_time': avg.gpu_time }) return avg, img_merge
def train(train_loader, val_loader, model, criterion, optimizer, epoch, lr): average_meter = AverageMeter() model.train() # switch to train mode global batch_num, best_result end = time.time() #every batch for i, (Y, Y_1_2, Y_1_4, Y_1_8, LR, LR_8, HR, name) in enumerate(train_loader): #处理被train_loader进去的每一个数据 batch_num = batch_num + 1 Y = Y.cuda() Y_1_2 = Y_1_2.cuda() Y_1_4 = Y_1_4.cuda() Y_1_8 = Y_1_8.cuda() LR = LR.cuda() LR_8 = LR_8.cuda() HR = HR.cuda() torch.cuda.synchronize() data_time = time.time() - end end = time.time() if args.arch == 'VDSR_16': pred_HR = model(LR) loss = criterion(pred_HR, HR, Y) elif args.arch == 'VDSR_16_2': pred_HR = model(Y, LR) loss = criterion(pred_HR, HR, Y) elif args.arch == 'VDSR': pred_HR, residule = model(LR_8, Y) loss = criterion(pred_HR, HR, Y) elif args.arch == 'ResNet_bicubic': pred_HR, residule = model(LR_8, Y) loss = criterion(pred_HR, HR, Y) elif args.arch == 'resnet50_15_6' or 'resnet50_15_11' or 'resnet50_15_12': pred_HR = model(Y_1_2, LR) loss = criterion(pred_HR, HR, Y) elif args.arch == 'resnet50_15_2' or 'resnet50_15_3' or 'resnet50_15_5' or 'resnet50_15_8' or 'resnet50_15_9': pred_HR, residule = model(Y, LR, LR_8) loss = criterion(pred_HR, HR, Y) else: if config.loss_num == 2: if config.LOSS_1 == 'l2': # 均方差 criterion1 = criteria.MaskedMSELoss().cuda() elif config.LOSS_1 == 'l1': criterion1 = criteria.MaskedL1Loss().cuda() elif config.LOSS_1 == 'l1_canny': # 均方差 criterion1 = criteria.MaskedL1_cannyLoss().cuda() elif config.LOSS_1 == 'l1_from_rgb_sobel': # 均方差 criterion1 = criteria.MaskedL1_from_rgb_sobel_Loss().cuda() elif aconfig.LOSS_1 == 'l1_canny_from_GT_canny': criterion1 = criteria.MaskedL1_canny_from_GT_Loss().cuda() elif aconfig.LOSS_1 == 'l1_from_GT_sobel': criterion1 = criteria.MaskedL1_from_GT_sobel_Loss().cuda() elif config.LOSS_1 == 'l2_from_GT_sobel_Loss': criterion1 = criteria.MaskedL2_from_GT_sobel_Loss().cuda() if config.use_different_size_Y == 1: pred_HR, pred_thermal0 = model(Y, Y_1_2, Y_1_4, Y_1_8, LR) else: pred_HR, pred_thermal0 = model(Y, LR) #final loss loss0 = criterion(pred_HR, HR, Y) #therma upsample loss loss1 = criterion1(pred_thermal0, HR, Y) loss = config.LOSS_0_weight * loss0 + config.LOSS_1_weight * loss1 else: if config.use_different_size_Y == 1: pred_HR, pred_thermal0 = model(Y, Y_1_2, Y_1_4, Y_1_8, LR) #writer = SummaryWriter(log_dir='logs') #writer.add_graph(model, input_to_model=(Y,Y_1_2,Y_1_4,Y_1_8,LR,)) else: pred_HR, pred_thermal0 = model(Y, LR) loss = criterion(pred_HR, HR, Y) optimizer.zero_grad() loss.backward() optimizer.step() torch.cuda.synchronize() gpu_time = time.time() - end # measure accuracy and record loss result = Result() result.evaluate(pred_HR, HR, loss.cpu().detach().numpy()) average_meter.update(result, gpu_time, data_time, Y.size(0)) end = time.time() if (i + 1) % args.print_freq == 0: print('=> output: {}'.format(output_directory)) print('Dataset Epoch: {0} [{1}/{2}]\t' 'Batch Epoch: {3} \t' 't_Data={data_time:.3f}({average.data_time:.3f}) ' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n' 'PSNR={result.psnr:.5f}({average.psnr:.5f}) ' 'MSE={result.mse:.3f}({average.mse:.3f}) ' 'RMSE={result.rmse:.3f}({average.rmse:.3f}) ' 'MAE={result.mae:.3f}({average.mae:.3f}) ' 'Delta1={result.delta1:.4f}({average.delta1:.4f}) ' 'REL={result.absrel:.4f}({average.absrel:.4f}) ' 'Lg10={result.lg10:.4f}({average.lg10:.4f}) ' 'Loss={result.loss:}({average.loss:}) '.format( epoch, i + 1, len(train_loader), batch_num, data_time=data_time, gpu_time=gpu_time, result=result, average=average_meter.average())) else: pass if (batch_num + 1) % config.save_fc == 0: print("==============Time to evaluate=================") utils.adjust_learning_rate(optimizer, batch_num, lr) print("==============SAVE_MODEL=================") avg = average_meter.average() average_meter = AverageMeter() with open(train_csv, 'a') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writerow({ 'dataset epoch': epoch, 'batch epoch': batch_num + 1, 'psnr': 10 * math.log(1 / (avg.mse), 10), 'mse': result.mse, 'rmse': result.rmse, 'absrel': result.absrel, 'lg10': result.lg10, 'mae': result.mae, 'delta1': result.delta1, 'delta2': result.delta2, 'delta3': result.delta3, 'gpu_time': result.gpu_time, 'data_time': result.data_time, 'loss': result.loss }) #------------------# # VALIDATION # #------------------# result_val, img_merge = validate( val_loader, model, epoch, batch_num) # evaluate on validation set,每次训练完以后都要测试一下 #------------------# # SAVE BEST MODEL # #------------------# is_best = result_val.rmse < best_result.rmse if is_best: best_result = result_val with open(best_txt, 'w') as txtfile: txtfile.write( "dataset epoch={}\nbatch epoch={}\npsnr={:.5f}\nmse={:.3f}\nrmse={:.3f}\nabsrel={:.3f}\nlg10={:.3f}\nmae={:.3f}\ndelta1={:.3f}\nt_gpu={:.4f}\n" .format(epoch, batch_num + 1, 10 * math.log(1 / (best_result.mse), 10), best_result.mse, best_result.rmse, best_result.absrel, best_result.lg10, best_result.mae, best_result.delta1, best_result.gpu_time)) if img_merge is not None: img_filename = output_directory + '/comparison_best.png' utils.save_image(img_merge, img_filename) utils.save_checkpoint( { 'args': args, 'epoch': epoch, 'batch_epoch': batch_num, 'arch': args.arch, 'model': model, 'best_result': best_result, 'optimizer': optimizer, }, is_best, epoch, batch_num, output_directory)
def evaluate(params, loader, model, experiment): print("Testing...") with experiment.test(): with torch.no_grad(): average = AverageMeter() img_idxs = np.random.randint(0, len(loader), size=min(len(loader), 50)) end = time.time() for i, (inputs, targets) in enumerate(loader): inputs, targets = inputs.to(params["device"]), targets.to( params["device"]) data_time = time.time() - end # Predict end = time.time() outputs = model(inputs) gpu_time = time.time() - end result = Result() result.evaluate(outputs.data, targets.data) average.update(result, gpu_time, data_time, inputs.size(0)) end = time.time() # Log images to comet if i in img_idxs: img_merged = utils.log_image_to_comet( inputs[0], targets[0], outputs[0], epoch=0, id=i, experiment=experiment, result=result, prefix="test") if params["save_test_images"]: filename = os.path.join( params["experiment_dir"], "comparison_epoch_{}_{}.png".format( str(params["start_epoch"]), np.where(img_idxs == i)[0][0])) utils.save_image(img_merged, filename) if (i + 1) % params["stats_frequency"] == 0 and i != 0: print('Test: [{0}/{1}]\t' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'RMSE={result.rmse:.2f}({average.rmse:.2f}) ' 'MAE={result.mae:.2f}({average.mae:.2f}) ' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'REL={result.absrel:.3f}({average.absrel:.3f}) ' 'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format( i + 1, len(loader), gpu_time=gpu_time, result=result, average=average.average())) # Mean validation loss avg = average.average() utils.log_comet_metrics(experiment, avg, None) print('\n*\n' 'RMSE={average.rmse:.3f}\n' 'MAE={average.mae:.3f}\n' 'Delta1={average.delta1:.3f}\n' 'REL={average.absrel:.3f}\n' 'Lg10={average.lg10:.3f}\n' 't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time)) if params["save_test_metrics"]: filename = os.path.join(params["experiment_dir"], "results.csv") with open(filename, 'a') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writerow({ 'mse': avg.mse, 'rmse': avg.rmse, 'absrel': avg.absrel, 'lg10': avg.lg10, 'mae': avg.mae, 'delta1': avg.delta1, 'delta2': avg.delta2, 'delta3': avg.delta3, 'irmse': avg.irmse, 'imae': avg.imae, 'data_time': avg.data_time, 'gpu_time': avg.gpu_time })
def iterate(mode, args, loader, model, optimizer, logger, epoch): block_average_meter = AverageMeter() average_meter = AverageMeter() meters = [block_average_meter, average_meter] # switch to appropriate mode assert mode in ["train", "val", "eval", "test_prediction", "test_completion"], \ "unsupported mode: {}".format(mode) if mode == 'train': model.train() lr = helper.adjust_learning_rate(args.lr, optimizer, epoch) else: model.eval() lr = 0 torch.set_printoptions(profile="full") for i, batch_data in enumerate(loader): name = batch_data['name'][0] print(name) del batch_data['name'] print("i: ", i) # each batch data is 1 and has three keys d, gt, g and dim [1, 352, 1216] start = time.time() batch_data = { key: val.to(device) for key, val in batch_data.items() if val is not None } gt = batch_data[ 'gt'] if mode != 'test_prediction' and mode != 'test_completion' else None # if args.type_feature=="sq": # depth_adjustment(gt, False) data_time = time.time() - start start = time.time() if mode == "train": pred = model(batch_data) else: with torch.no_grad(): pred = model(batch_data) vis=False if vis: im = batch_data['gt'].detach().cpu().numpy() im_sq = im.squeeze() plt.figure() plt.imshow(im_sq) plt.show() # for i in range(im_sq.shape[0]): # print(f"{i} - {np.sum(im_sq[i])}") depth_loss, photometric_loss, smooth_loss, mask = 0, 0, 0, None if mode == 'train': # Loss 1: the direct depth supervision from ground truth label # mask=1 indicates that a pixel does not ground truth labels if 'sparse' in args.train_mode: depth_loss = depth_criterion(pred, batch_data['d']) print("d pts: ", len(torch.where(batch_data['d']>0)[0])) mask = (batch_data['d'] < 1e-3).float() elif 'dense' in args.train_mode: depth_loss = depth_criterion(pred, gt) mask = (gt < 1e-3).float() # Loss 2: the self-supervised photometric loss if args.use_pose: # create multi-scale pyramids pred_array = helper.multiscale(pred) rgb_curr_array = helper.multiscale(batch_data['rgb']) rgb_near_array = helper.multiscale(batch_data['rgb_near']) if mask is not None: mask_array = helper.multiscale(mask) num_scales = len(pred_array) # compute photometric loss at multiple scales for scale in range(len(pred_array)): pred_ = pred_array[scale] rgb_curr_ = rgb_curr_array[scale] rgb_near_ = rgb_near_array[scale] mask_ = None if mask is not None: mask_ = mask_array[scale] # compute the corresponding intrinsic parameters height_, width_ = pred_.size(2), pred_.size(3) intrinsics_ = kitti_intrinsics.scale(height_, width_) # inverse warp from a nearby frame to the current frame warped_ = homography_from(rgb_near_, pred_, batch_data['r_mat'], batch_data['t_vec'], intrinsics_) photometric_loss += photometric_criterion( rgb_curr_, warped_, mask_) * (2**(scale - num_scales)) # Loss 3: the depth smoothness loss smooth_loss = smoothness_criterion(pred) if args.w2 > 0 else 0 # backprop loss = depth_loss + args.w1 * photometric_loss + args.w2 * smooth_loss optimizer.zero_grad() loss.backward() zero_params(model) optimizer.step() gpu_time = time.time() - start # counting pixels in each bin #binned_pixels = np.load("value.npy", allow_pickle=True) #print(len(binned_pixels)) if (i % 1 == 0 and args.evaluate and args.instancewise) or\ (i % args.every == 0 and not args.evaluate and not args.instancewise): # global training # print(model.module.conv4[5].conv1.weight[0]) # print(model.conv4.5.bn2.weight) # print(model.module.parameter.grad) #print("*************swiches:") torch.set_printoptions(precision=7, sci_mode=False) if model.module.phi is not None: mmp = 1000 * model.module.phi phi = F.softplus(mmp) S = phi / torch.sum(phi) #print("S", S[1, -10:]) S_numpy= S.detach().cpu().numpy() if args.instancewise: global Ss if "Ss" not in globals(): Ss = [] Ss.append(S_numpy) else: Ss.append(S_numpy) # GLOBAL if (i % args.every ==0 and not args.evaluate and not args.instancewise and model.module.phi is not None): np.set_printoptions(precision=4) switches_2d_argsort = np.argsort(S_numpy, None) # 2d to 1d sort torch.Size([9, 31]) switches_2d_sort = np.sort(S_numpy, None) print("Switches: ") print(switches_2d_argsort[:10]) print(switches_2d_sort[:10]) print("and") print(switches_2d_argsort[-10:]) print(switches_2d_sort[-10:]) ##### saving global ranks global_ranks_path = lambda \ ii: f"ranks/{args.type_feature}/global/{folder_and_name[0]}/Ss_val_{folder_and_name[1]}_iter_{ii}.npy" global old_i if ("old_i" in globals()): print("old_i") if os.path.isfile(global_ranks_path(old_i)): os.remove(global_ranks_path(old_i)) folder_and_name = args.resume.split(os.sep)[-2:] os.makedirs(f"ranks/{args.type_feature}/global/{folder_and_name[0]}", exist_ok=True) np.save(global_ranks_path(i), S_numpy) old_i = i print("saving ranks") if args.type_feature == "sq": hor = switches_2d_argsort % S_numpy.shape[1] ver = np.floor(switches_2d_argsort // S_numpy.shape[1]) print(ver[:10],hor[:10]) print("and") print(ver[-10:], hor[-10:]) # measure accuracy and record loss with torch.no_grad(): mini_batch_size = next(iter(batch_data.values())).size(0) result = Result() if mode != 'test_prediction' and mode != 'test_completion': result.evaluate(pred.data, gt.data, photometric_loss) [ m.update(result, gpu_time, data_time, mini_batch_size) for m in meters ] logger.conditional_print(mode, i, epoch, lr, len(loader), block_average_meter, average_meter) logger.conditional_save_img_comparison(mode, i, batch_data, pred, epoch) logger.conditional_save_pred(mode, i, pred, epoch) draw=False if draw: ma = batch_data['rgb'].detach().cpu().numpy().squeeze() ma = np.transpose(ma, axes=[1, 2, 0]) # ma = np.uint8(ma) #ma2 = Image.fromarray(ma) ma2 = Image.fromarray(np.uint8(ma)).convert('RGB') # create rectangle image img1 = ImageDraw.Draw(ma2) if args.type_feature == "sq": size=40 print_square_num = 20 for ii in range(print_square_num): s_hor=hor[-ii].detach().cpu().numpy() s_ver=ver[-ii].detach().cpu().numpy() shape = [(s_hor * size, s_ver * size), ((s_hor + 1) * size, (s_ver + 1) * size)] img1.rectangle(shape, outline="red") tim = time.time() lala = ma2.save(f"switches_photos/squares/squares_{tim}.jpg") print("saving") elif args.type_feature == "lines": print_square_num = 20 r=1 parameter_mask = np.load("../kitti_pixels_to_lines.npy", allow_pickle=True) # for m in range(10,50): # im = Image.fromarray(parameter_mask[m]*155) # im = im.convert('1') # convert image to black and white # im.save(f"switches_photos/lala_{m}.jpg") for ii in range(print_square_num): points = parameter_mask[ii] y = np.where(points==1)[0] x = np.where(points == 1)[1] for p in range(len(x)): img1.ellipse((x[p] - r, y[p] - r, x[p] + r, y[p] + r), fill=(255, 0, 0, 0)) tim = time.time() lala = ma2.save(f"switches_photos/lines/lines_{tim}.jpg") print("saving") every = args.every if i % every ==0: print("saving") avg = logger.conditional_save_info(mode, average_meter, epoch) is_best = logger.rank_conditional_save_best(mode, avg, epoch) #is_best = True #saving all the checkpoints if is_best and not (mode == "train"): logger.save_img_comparison_as_best(mode, epoch) logger.conditional_summarize(mode, avg, is_best) if mode != "val": helper.save_checkpoint({ # save checkpoint 'epoch': epoch, 'model': model.module.state_dict(), 'best_result': logger.best_result, 'optimizer': optimizer.state_dict(), 'args': args, }, is_best, epoch, logger.output_directory, args.type_feature, i, every, qnet=True) if args.evaluate and args.instancewise: #filename = os.path.split(args.evaluate)[1] Ss_numpy = np.array(Ss) folder_and_name = args.evaluate.split(os.sep)[-3:] os.makedirs(f"ranks/instance/{folder_and_name[0]}", exist_ok=True) os.makedirs(f"ranks/instance/{folder_and_name[0]}/{folder_and_name[1]}", exist_ok=True) np.save(f"ranks/instance/{folder_and_name[0]}/{folder_and_name[1]}/Ss_val_{folder_and_name[2]}.npy", Ss) return avg, is_best
def iterate(mode, args, loader, model, optimizer, logger, epoch): block_average_meter = AverageMeter() average_meter = AverageMeter() meters = [block_average_meter, average_meter] # switch to appropriate mode assert mode in ["train", "val", "eval", "test_prediction", "test_completion"], \ "unsupported mode: {}".format(mode) if mode == 'train': model.train() lr = helper.adjust_learning_rate(args.lr, optimizer, epoch) else: model.eval() lr = 0 for i, batch_data in enumerate(loader): print("The batch data keys are {}".format(batch_data.keys())) start = time.time() batch_data = { key: val.to(device) for key, val in batch_data.items() if val is not None } gt = batch_data[ 'gt'] if mode != 'test_prediction' and mode != 'test_completion' else None data_time = time.time() - start start = time.time() temp_d = batch_data['d'] temp_gt = batch_data['gt'] temp_g = batch_data['g'] print("The depth min:{}, max:{}, shape:{}, dtype:{}".format( torch.min(temp_d), torch.max(temp_d), temp_d.shape, temp_d.dtype)) print("The groundtruth min:{}, max:{}, shape:{}, dtype:{}".format( torch.min(temp_gt), torch.max(temp_gt), temp_gt.shape, temp_gt.dtype)) print("The greyscale min:{}, max:{}, shape:{}, dtype:{}".format( torch.min(temp_g), torch.max(temp_g), temp_g.shape, temp_g.dtype)) pred = model(batch_data) temp_out = pred.detach().cpu() print("The output min:{}, max:{}, shape:{}, dtype:{}".format( torch.min(temp_out), torch.max(temp_out), temp_out.shape, temp_out.dtype)) depth_loss, photometric_loss, smooth_loss, mask = 0, 0, 0, None if mode == 'train': # Loss 1: the direct depth supervision from ground truth label # mask=1 indicates that a pixel does not ground truth labels if 'sparse' in args.train_mode: depth_loss = depth_criterion(pred, batch_data['d']) mask = (batch_data['d'] < 1e-3).float() elif 'dense' in args.train_mode: depth_loss = depth_criterion(pred, gt) mask = (gt < 1e-3).float() # Loss 2: the self-supervised photometric loss if args.use_pose: # create multi-scale pyramids pred_array = helper.multiscale(pred) rgb_curr_array = helper.multiscale(batch_data['rgb']) rgb_near_array = helper.multiscale(batch_data['rgb_near']) if mask is not None: mask_array = helper.multiscale(mask) num_scales = len(pred_array) # compute photometric loss at multiple scales for scale in range(len(pred_array)): pred_ = pred_array[scale] rgb_curr_ = rgb_curr_array[scale] rgb_near_ = rgb_near_array[scale] mask_ = None if mask is not None: mask_ = mask_array[scale] # compute the corresponding intrinsic parameters height_, width_ = pred_.size(2), pred_.size(3) intrinsics_ = kitti_intrinsics.scale(height_, width_) # inverse warp from a nearby frame to the current frame warped_ = homography_from(rgb_near_, pred_, batch_data['r_mat'], batch_data['t_vec'], intrinsics_) photometric_loss += photometric_criterion( rgb_curr_, warped_, mask_) * (2**(scale - num_scales)) # Loss 3: the depth smoothness loss smooth_loss = smoothness_criterion(pred) if args.w2 > 0 else 0 # backprop loss = depth_loss + args.w1 * photometric_loss + args.w2 * smooth_loss optimizer.zero_grad() loss.backward() optimizer.step() gpu_time = time.time() - start # measure accuracy and record loss with torch.no_grad(): mini_batch_size = next(iter(batch_data.values())).size(0) result = Result() if mode != 'test_prediction' and mode != 'test_completion': result.evaluate(pred.data, gt.data, photometric_loss) [ m.update(result, gpu_time, data_time, mini_batch_size) for m in meters ] logger.conditional_print(mode, i, epoch, lr, len(loader), block_average_meter, average_meter) logger.conditional_save_img_comparison(mode, i, batch_data, pred, epoch) logger.conditional_save_pred(mode, i, pred, epoch) avg = logger.conditional_save_info(mode, average_meter, epoch) is_best = logger.rank_conditional_save_best(mode, avg, epoch) if is_best and not (mode == "train"): logger.save_img_comparison_as_best(mode, epoch) logger.conditional_summarize(mode, avg, is_best) return avg, is_best
class logger: def __init__(self, args, prepare=True): self.args = args self.args.save_pred = True output_directory = get_folder_name(args) self.output_directory = output_directory self.best_result = Result() self.best_result.set_to_worst() if not prepare: return if not os.path.exists(output_directory): os.makedirs(output_directory) self.train_csv = os.path.join(output_directory, 'train.csv') self.val_csv = os.path.join(output_directory, 'val.csv') self.best_txt = os.path.join(output_directory, 'best.txt') # backup the source code if args.resume == '': print("=> creating source code backup ...") backup_directory = os.path.join(output_directory, "code_backup") self.backup_directory = backup_directory print("=> stop source code backup ...") # backup_source_code(backup_directory) # create new csv files with only header with open(self.train_csv, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() with open(self.val_csv, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() print("=> finished creating source code backup.") def conditional_print(self, split, i, epoch, lr, n_set, blk_avg_meter, avg_meter): if (i + 1) % self.args.print_freq == 0: avg = avg_meter.average() blk_avg = blk_avg_meter.average() print('=> output: {}'.format(self.output_directory)) print( '{split} Epoch: {0} [{1}/{2}]\tlr={lr} ' 't_Data={blk_avg.data_time:.3f}({average.data_time:.3f}) ' 't_GPU={blk_avg.gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'RMSE={blk_avg.rmse:.2f}({average.rmse:.2f}) ' 'MAE={blk_avg.mae:.2f}({average.mae:.2f}) ' 'iRMSE={blk_avg.irmse:.2f}({average.irmse:.2f}) ' 'iMAE={blk_avg.imae:.2f}({average.imae:.2f})\n\t' 'silog={blk_avg.silog:.2f}({average.silog:.2f}) ' 'squared_rel={blk_avg.squared_rel:.2f}({average.squared_rel:.2f}) ' 'Delta1={blk_avg.delta1:.3f}({average.delta1:.3f}) ' 'REL={blk_avg.absrel:.3f}({average.absrel:.3f})\n\t' 'Lg10={blk_avg.lg10:.3f}({average.lg10:.3f}) ' 'Photometric={blk_avg.photometric:.3f}({average.photometric:.3f}) ' .format(epoch, i + 1, n_set, lr=lr, blk_avg=blk_avg, average=avg, split=split.capitalize())) blk_avg_meter.reset() def conditional_save_info(self, split, average_meter, epoch): avg = average_meter.average() if split == "train": csvfile_name = self.train_csv elif split == "val": csvfile_name = self.val_csv elif split == "eval": eval_filename = os.path.join(self.output_directory, 'eval.txt') self.save_single_txt(eval_filename, avg, epoch) return avg elif "test" in split: return avg else: raise ValueError("wrong split provided to logger") with open(csvfile_name, 'a') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writerow({ 'epoch': epoch, 'rmse': avg.rmse, 'photo': avg.photometric, 'mae': avg.mae, 'irmse': avg.irmse, 'imae': avg.imae, 'mse': avg.mse, 'silog': avg.silog, 'squared_rel': avg.squared_rel, 'absrel': avg.absrel, 'lg10': avg.lg10, 'delta1': avg.delta1, 'delta2': avg.delta2, 'delta3': avg.delta3, 'gpu_time': avg.gpu_time, 'data_time': avg.data_time }) return avg def save_single_txt(self, filename, result, epoch): with open(filename, 'w') as txtfile: txtfile.write( ("rank_metric={}\n" + "epoch={}\n" + "rmse={:.3f}\n" + "mae={:.3f}\n" + "silog={:.3f}\n" + "squared_rel={:.3f}\n" + "irmse={:.3f}\n" + "imae={:.3f}\n" + "mse={:.3f}\n" + "absrel={:.3f}\n" + "lg10={:.3f}\n" + "delta1={:.3f}\n" + "t_gpu={:.4f}").format(self.args.rank_metric, epoch, result.rmse, result.mae, result.silog, result.squared_rel, result.irmse, result.imae, result.mse, result.absrel, result.lg10, result.delta1, result.gpu_time)) def save_best_txt(self, result, epoch): self.save_single_txt(self.best_txt, result, epoch) def _get_img_comparison_name(self, mode, epoch, is_best=False): if mode == 'eval': return self.output_directory + '/comparison_eval.png' if mode == 'val': if is_best: return self.output_directory + '/comparison_best.png' else: return self.output_directory + '/comparison_' + str( epoch) + '.png' def conditional_save_img_comparison(self, mode, i, ele, pred, epoch): # save 8 images for visualization if mode == 'val' or mode == 'eval': skip = 100 if i == 0: self.img_merge = vis_utils.merge_into_row(ele, pred) elif i % skip == 0 and i < 8 * skip: row = vis_utils.merge_into_row(ele, pred) self.img_merge = vis_utils.add_row(self.img_merge, row) elif i == 8 * skip: filename = self._get_img_comparison_name(mode, epoch) vis_utils.save_image(self.img_merge, filename) def save_img_comparison_as_best(self, mode, epoch): if mode == 'val': filename = self._get_img_comparison_name(mode, epoch, is_best=True) vis_utils.save_image(self.img_merge, filename) def get_ranking_error(self, result): return getattr(result, self.args.rank_metric) def rank_conditional_save_best(self, mode, result, epoch): error = self.get_ranking_error(result) best_error = self.get_ranking_error(self.best_result) is_best = error < best_error if is_best and mode == "val": self.old_best_result = self.best_result self.best_result = result self.save_best_txt(result, epoch) return is_best def conditional_save_pred(self, mode, file_name, pred, epoch): if ("test" in mode or mode == "eval") and self.args.save_pred: # save images for visualization/ testing image_folder = os.path.join(self.output_directory, mode + "_output") if not os.path.exists(image_folder): os.makedirs(image_folder) img = torch.squeeze(pred.data.cpu()).numpy() file_path = os.path.join(image_folder, file_name) vis_utils.save_depth_as_uint16png(img, file_path) def conditional_summarize(self, mode, avg, is_best): print("\n*\nSummary of ", mode, "round") print('' 'RMSE={average.rmse:.3f}\n' 'MAE={average.mae:.3f}\n' 'Photo={average.photometric:.3f}\n' 'iRMSE={average.irmse:.3f}\n' 'iMAE={average.imae:.3f}\n' 'squared_rel={average.squared_rel}\n' 'silog={average.silog}\n' 'Delta1={average.delta1:.3f}\n' 'REL={average.absrel:.3f}\n' 'Lg10={average.lg10:.3f}\n' 't_GPU={time:.3f}'.format(average=avg, time=avg.gpu_time)) if is_best and mode == "val": print("New best model by %s (was %.3f)" % (self.args.rank_metric, self.get_ranking_error(self.old_best_result))) elif mode == "val": print("(best %s is %.3f)" % (self.args.rank_metric, self.get_ranking_error(self.best_result))) print("*\n")
def validate(val_loader, model, epoch, write_to_file=True): average_meter = AverageMeter() model.eval() # switch to evaluate mode end = time.time() eval_file = output_directory + '/evaluation.csv' f = open(eval_file, "w+") f.write("Max_Error,Depth,RMSE,GPU_TIME,Number_Of_Frame\r\n") for i, (input, target) in enumerate(val_loader): input, target = input.cuda(), target.cuda() # torch.cuda.synchronize() data_time = time.time() - end # compute output end = time.time() with torch.no_grad(): pred = model(input) # torch.cuda.synchronize() gpu_time = time.time() - end abs_err = (target.data - pred.data).abs().cpu() max_err_ind = np.unravel_index(np.argmax(abs_err, axis=None), abs_err.shape) max_err_depth = target.data[max_err_ind] max_err = abs_err[max_err_ind] # measure accuracy and record loss result = Result() result.evaluate(pred.data, target.data) average_meter.update(result, gpu_time, data_time, input.size(0)) end = time.time() f.write( f'{max_err},{max_err_depth},{result.rmse:.2f},{gpu_time},{i+1}\r\n' ) # save 8 images for visualization skip = 50 if args.modality == 'rgb': rgb = input if i == 0: img_merge = utils.merge_into_row_with_gt(rgb, target, pred, (target - pred).abs()) elif (i < 8 * skip) and (i % skip == 0): row = utils.merge_into_row_with_gt(rgb, target, pred, (target - pred).abs()) img_merge = utils.add_row(img_merge, row) elif i == 8 * skip: filename = output_directory + '/comparison_' + str(epoch) + '.png' utils.save_image(img_merge, filename) if (i + 1) % args.print_freq == 0: print('Test: [{0}/{1}]\t' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'RMSE={result.rmse:.2f}({average.rmse:.2f}) ' 'MAE={result.mae:.2f}({average.mae:.2f}) ' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'REL={result.absrel:.3f}({average.absrel:.3f}) ' 'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format( i + 1, len(val_loader), gpu_time=gpu_time, result=result, average=average_meter.average())) f.close() avg = average_meter.average() print('\n*\n' 'RMSE={average.rmse:.3f}\n' 'MAE={average.mae:.3f}\n' 'Delta1={average.delta1:.3f}\n' 'REL={average.absrel:.3f}\n' 'Lg10={average.lg10:.3f}\n' 't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time)) if write_to_file: with open(test_csv, 'a') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writerow({ 'mse': avg.mse, 'rmse': avg.rmse, 'absrel': avg.absrel, 'lg10': avg.lg10, 'mae': avg.mae, 'delta1': avg.delta1, 'delta2': avg.delta2, 'delta3': avg.delta3, 'data_time': avg.data_time, 'gpu_time': avg.gpu_time }) return avg, img_merge
def train(train_loader, model, criterion,smoothloss,photometric_loss,optimizer, epoch): average_meter = AverageMeter() model.train() # switch to train mode end = time.time() iheight, iwidth = 480, 640 # raw image size fx_rgb = 5.1885790117450188e+02; fy_rgb = 5.1946961112127485e+02; cx_rgb = 3.2558244941119034e+02; cy_rgb = 2.5373616633400465e+02; new_intrinsics = Intrinsics(304,228,fx_rgb*(250.0/iheight),fy_rgb*(250.0/iheight),155.1, 121.0).cuda() for i, (batch_data, intrinsics) in enumerate(train_loader): #input, target = input.cuda(), target.cuda() batch_data = {key:val.cuda() for key,val in batch_data.items() if val is not None} oheight, owidth = intrinsics["output_size"] #new_intrinsics = Intrinsics(owidth,oheight,intrinsics["fx"],intrinsics["fy"],intrinsics["cx"],intrinsics["cy"]).cuda() target = batch_data['gt'] torch.cuda.synchronize() data_time = time.time() - end # compute pred end = time.time() #pred = model(input[:,3:,:,:],input[:,:3,:,:]) # (depth,image) #candidates = {"rgb":input[:,:3,:,:], "d":input[:,3:,:,:], "gt":input[:,3:,:,:]} #batch_data = {key:val.cuda() for key,val in candidates.items() if val is not None} pred = model(batch_data) # (depth,image) #loss = criterion(pred, input[:,3:,:,:]) + 0.01*smoothloss(pred) photoloss = 0.0 if args.use_pose: # warp near frame to current frame #hh, ww = pred.size(2), pred.size(3) #new_intrinsics = new_intrinsics.scale(hh,ww) mask = (batch_data['d'] < 1e-3).float() pred_array = multiscale(pred) rgb_curr_array = multiscale(batch_data['rgb']) rgb_near_array = multiscale(batch_data['rgb_near']) mask_array = multiscale(mask) num_scales = len(pred_array) # compute photometric loss at multiple scales for scale in range(len(pred_array)): pred_ = pred_array[scale] rgb_curr_ = rgb_curr_array[scale] rgb_near_ = rgb_near_array[scale] mask_ = None if mask is not None: mask_ = mask_array[scale] # compute the corresponding intrinsic parameters height_, width_ = pred_.size(2), pred_.size(3) intrinsics_ = new_intrinsics.scale(height_, width_) warped_ = homography_from(rgb_near_,pred_,batch_data["r_mat"],batch_data["t_vec"],intrinsics_) #warped = homography_from(batch_data["rgb_near"],pred,batch_data["r_mat"],batch_data["t_vec"],new_intrinsics) #photoloss = photometric_loss(batch_data["rgb"],warped,mask) photoloss += photometric_loss(rgb_curr_,warped_,mask_)*(2**(scale-num_scales)) loss = criterion(pred, target) + 0.01*smoothloss(pred) + 0.1*photoloss optimizer.zero_grad() loss.backward() # compute gradient and do SGD step optimizer.step() torch.cuda.synchronize() gpu_time = time.time() - end # measure accuracy and record loss result = Result() result.evaluate(pred.data, target.data) average_meter.update(result, gpu_time, data_time, batch_data['rgb'].size(0)) end = time.time() if (i + 1) % args.print_freq == 0: print('=> output: {}'.format(output_directory)) print('Train Epoch: {0} [{1}/{2}]\t' 'LOSS={loss:.3f} ' 't_Data={data_time:.3f}({average.data_time:.3f}) ' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'RMSE={result.rmse:.2f}({average.rmse:.2f}) ' 'MAE={result.mae:.2f}({average.mae:.2f}) ' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'REL={result.absrel:.3f}({average.absrel:.3f}) ' 'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format( epoch, i+1, len(train_loader),loss=loss.item(), data_time=data_time, gpu_time=gpu_time, result=result, average=average_meter.average())) avg = average_meter.average() with open(train_csv, 'a') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writerow({'mse': avg.mse, 'rmse': avg.rmse, 'absrel': avg.absrel, 'lg10': avg.lg10, 'mae': avg.mae, 'delta1': avg.delta1, 'delta2': avg.delta2, 'delta3': avg.delta3, 'gpu_time': avg.gpu_time, 'data_time': avg.data_time})
def depth_estimate(model, rgb_frame, depth, save=False, switch=False): to_tensor = transforms.ToTensor() cmap_depth = plt.cm.viridis cmap_error = plt.cm.inferno rgb_np, depth_np = image_transform(rgb_frame, depth) ##creat sparse depth mask_keep = sampler(depth_np, args.sample_spacing) sample_number = np.count_nonzero( mask_keep.astype('int')) if args.modality == 'rgb': sample_number = 0 if switch: print('Total samples = ' + str(sample_number)) sparse_depth = np.zeros(depth_np.shape) sparse_depth[mask_keep] = depth_np[mask_keep] sparse_depth_np = sparse_depth ##choose input if args.modality == 'rgb': input_np = rgb_np elif args.modality == 'rgbd': rgbd = np.append(rgb_np, np.expand_dims(sparse_depth_np, axis=2), axis=2) input_np = np.asfarray(rgbd, dtype='float') elif args.modality == 'd': input_np = sparse_depth_np input_tensor = to_tensor(input_np) while input_tensor.dim() < 4: input_tensor = input_tensor.unsqueeze(0) input_tensor = input_tensor.cuda() torch.cuda.synchronize() ##get prediction end = time.time() with torch.no_grad(): pred = model(input_tensor) torch.cuda.synchronize() gpu_time = time.time() - end ##get result target_tensor = to_tensor(depth_np) while target_tensor.dim() < 4: target_tensor = target_tensor.unsqueeze(0) target_tensor = target_tensor.cuda() torch.cuda.synchronize() result = Result() result.evaluate(pred.data, target_tensor.data) pred_depth = np.squeeze(pred.data.cpu().numpy()) ##convert depth to colour map d_min = min(np.min(pred_depth), np.min(depth_np)) d_max = max(np.max(pred_depth), np.max(depth_np)) # # d_min = float(0.5) # # d_max = float(6) pred_depth_rgb_map = color_map(pred_depth, d_min, d_max, cmap_depth) input_depth_color_map = color_map(depth_np, d_min, d_max, cmap_depth) sparse_depth_color_map = color_map(sparse_depth, d_min, d_max, cmap_depth) ##get error map mask = depth_np <= 0 combined_map = depth_np combined_map[mask] = pred_depth[mask] abs_error_map = np.absolute(combined_map - pred_depth) # error_min = min(np.min(combined_map), np.min( # pred_depth)) # error_max = max(np.max(combined_map), np.max( # pred_depth)) error_min = np.min(abs_error_map) error_max = np.max(abs_error_map) error_map_color = color_map(abs_error_map, error_min, error_max, cmap_error) ##show images rgb_image = rgb_np * 255 merged_image = np.hstack([rgb_image, pred_depth_rgb_map, input_depth_color_map, sparse_depth_color_map, error_map_color]) ##save image if args.write and save: images_save(sample_number, pred_depth_rgb_map, sparse_depth_color_map, error_map_color, rgb_image, input_depth_color_map, result) return merged_image, result.rmse
def validate(val_loader, model, epoch, logger): average_meter = AverageMeter() model.eval() # switch to evaluate mode end = time.time() skip = len(val_loader) // 9 # save images every skip iters for i, (input, target) in enumerate(val_loader): input, target = input.cuda(), target.cuda() torch.cuda.synchronize() data_time = time.time() - end # compute output end = time.time() with torch.no_grad(): pred = model(input) torch.cuda.synchronize() gpu_time = time.time() - end # measure accuracy and record loss result = Result() result.evaluate(pred.data, target.data) average_meter.update(result, gpu_time, data_time, input.size(0)) end = time.time() # save 8 images for visualization if args.dataset == 'kitti': rgb = input[0] pred = pred[0] target = target[0] else: rgb = input if i == 0: img_merge = utils.merge_into_row(rgb, target, pred) elif (i < 8 * skip) and (i % skip == 0): row = utils.merge_into_row(rgb, target, pred) img_merge = utils.add_row(img_merge, row) elif i == 8 * skip: filename = output_directory + '/comparison_' + str(epoch) + '.png' utils.save_image(img_merge, filename) if (i + 1) % args.print_freq == 0: print('Test: [{0}/{1}]\t' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'RMSE={result.rmse:.2f}({average.rmse:.2f}) ' 'RML={result.absrel:.2f}({average.absrel:.2f}) ' 'Log10={result.lg10:.3f}({average.lg10:.3f}) ' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'Delta2={result.delta2:.3f}({average.delta2:.3f}) ' 'Delta3={result.delta3:.3f}({average.delta3:.3f})'.format( i + 1, len(val_loader), gpu_time=gpu_time, result=result, average=average_meter.average())) avg = average_meter.average() print('\n*\n' 'RMSE={average.rmse:.3f}\n' 'Rel={average.absrel:.3f}\n' 'Log10={average.lg10:.3f}\n' 'Delta1={average.delta1:.3f}\n' 'Delta2={average.delta2:.3f}\n' 'Delta3={average.delta3:.3f}\n' 't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time)) logger.add_scalar('Test/rmse', avg.rmse, epoch) logger.add_scalar('Test/Rel', avg.absrel, epoch) logger.add_scalar('Test/log10', avg.lg10, epoch) logger.add_scalar('Test/Delta1', avg.delta1, epoch) logger.add_scalar('Test/Delta2', avg.delta2, epoch) logger.add_scalar('Test/Delta3', avg.delta3, epoch) return avg, img_merge
def main(): torch.cuda.set_device(config.cuda_id) global args, best_result, output_directory, train_csv, test_csv, batch_num, best_txt best_result = Result() best_result.set_to_worst() batch_num = 0 output_directory = utils.get_output_directory(args) #-----------------# # pytorch version # #-----------------# try: torch._utils._rebuild_tensor_v2 except AttributeError: def _rebuild_tensor_v2(storage, storage_offset, size, stride, requires_grad, backward_hooks): tensor = torch._utils._rebuild_tensor(storage, storage_offset, size, stride) tensor.requires_grad = requires_grad tensor._backward_hooks = backward_hooks return tensor torch._utils._rebuild_tensor_v2 = _rebuild_tensor_v2 if not os.path.exists(output_directory): os.makedirs(output_directory) train_csv = os.path.join(output_directory, 'train.csv') test_csv = os.path.join(output_directory, 'test.csv') best_txt = os.path.join(output_directory, 'best.txt') nowTime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') file = open(namefile, 'a+') file.writelines( str("====================================================") + str(nowTime) + '\n') file.writelines(str("Cuda_id: ") + str(config.cuda_id) + '\n') file.writelines(str("NAME: ") + str(config.name) + '\n') file.writelines(str("Description: ") + str(config.description) + '\n') file.writelines( str("model: ") + str(args.arch) + '\n' + str("loss_final: ") + str(args.criterion) + '\n' + str("loss_1: ") + str(config.LOSS_1) + '\n' + str("batch_size:") + str(args.batch_size) + '\n') file.writelines(str("zoom_scale: ") + str(config.zoom_scale) + '\n') file.writelines(str("------------------------") + '\n') file.writelines(str("Train_dataste: ") + str(config.train_dir) + '\n') file.writelines(str("Validation_dataste: ") + str(config.val_dir) + '\n') file.writelines(str("------------------------") + '\n') file.writelines(str("Input_type: ") + str(config.input) + '\n') file.writelines(str("target_type: ") + str(config.target) + '\n') file.writelines(str("LOSS--------------------") + '\n') file.writelines(str("Loss_num: ") + str(config.loss_num) + '\n') file.writelines( str("loss_final: ") + str(args.criterion) + '\n' + str("loss_1: ") + str(config.LOSS_1) + '\n') file.writelines( str("loss_0_weight: ") + str(config.LOSS_0_weight) + '\n' + str("loss_1_weight: ") + str(config.LOSS_1_weight) + '\n') file.writelines( str("weight_GT_canny: ") + str(config.weight_GT_canny_loss) + '\n' + str("weight_GT_sobel: ") + str(config.weight_GT_sobel_loss) + '\n' + str("weight_rgb_sobel: ") + str(config.weight_rgb_sobel_loss) + '\n') file.writelines(str("------------------------") + '\n') file.writelines(str("target: ") + str(config.target) + '\n') file.writelines(str("data_loader_type: ") + str(config.data_loader) + '\n') file.writelines(str("lr: ") + str(config.Init_lr) + '\n') file.writelines(str("save_fc: ") + str(config.save_fc) + '\n') file.writelines(str("Max epoch: ") + str(config.epoch) + '\n') file.close() # define loss function (criterion) and optimizer,定义误差函数和优化器 if args.criterion == 'l2': criterion = criteria.MaskedMSELoss().cuda() elif args.criterion == 'l1': criterion = criteria.MaskedL1Loss().cuda() elif args.criterion == 'l1_canny': criterion = criteria.MaskedL1_cannyLoss().cuda() #SOBEL elif args.criterion == 'l1_from_rgb_sobel': criterion = criteria.MaskedL1_from_rgb_sobel_Loss().cuda() elif args.criterion == 'l1_from_GT_rgb_sobel': criterion = criteria.MaskedL1_from_GT_rgb_sobel_Loss().cuda() elif args.criterion == 'l1_from_GT_sobel': criterion = criteria.MaskedL1_from_GT_sobel_Loss().cuda() elif args.criterion == 'l2_from_GT_sobel_Loss': criterion = criteria.MaskedL2_from_GT_sobel_Loss().cuda() #CANNY elif args.criterion == 'l1_canny_from_GT_canny': criterion = criteria.MaskedL1_canny_from_GT_Loss().cuda() # Data loading code print("=> creating data loaders ...") train_dir = config.train_dir val_dir = config.val_dir train_dataset = YT_dataset(train_dir, config, is_train_set=True) val_dataset = YT_dataset(val_dir, config, is_train_set=False) train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=config.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True, sampler=None, worker_init_fn=lambda work_id: np.random.seed(work_id)) # worker_init_fn ensures different sampling patterns for each data loading thread # set batch size to be 1 for validation val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=1, shuffle=False, num_workers=args.workers, pin_memory=True) print("=> data loaders created.") if args.evaluate: best_model_filename = os.path.join(output_directory, 'model_best.pth.tar') assert os.path.isfile(best_model_filename), \ "=> no best model found at '{}'".format(best_model_filename) print("=> loading best model '{}'".format(best_model_filename)) checkpoint = torch.load(best_model_filename) args.start_epoch = checkpoint['epoch'] best_result = checkpoint['best_result'] model = checkpoint['model'] print("=> loaded best model (epoch {})".format(checkpoint['epoch'])) validate(val_loader, model, checkpoint['epoch'], 1, write_to_file=False) return elif args.test: print("testing...") best_model_filename = best_model_dir assert os.path.isfile(best_model_filename), \ "=> no best model found at '{}'".format(best_model_filename) print("=> loading best model '{}'".format(best_model_filename)) checkpoint = torch.load(best_model_filename) args.start_epoch = checkpoint['epoch'] best_result = checkpoint['best_result'] model = checkpoint['model'] print("=> loaded best model (epoch {})".format(checkpoint['epoch'])) optimizer = checkpoint['optimizer'] for state in optimizer.state.values(): for k, v in state.items(): print(type(v)) if torch.is_tensor(v): state[k] = v.cuda() #test(val_loader, model, checkpoint['epoch'], write_to_file=False) test(model) return elif args.resume: assert os.path.isfile(config.resume_model_dir), \ "=> no checkpoint found at '{}'".format(config.resume_model_dir) print("=> loading checkpoint '{}'".format(config.resume_model_dir)) best_model_filename = config.resume_model_dir checkpoint = torch.load(best_model_filename) args.start_epoch = checkpoint['epoch'] + 1 best_result = checkpoint['best_result'] model = checkpoint['model'] optimizer = checkpoint['optimizer'] for state in optimizer.state.values(): for k, v in state.items(): #print(type(v)) if torch.is_tensor(v): state[k] = v.cuda(config.cuda_id) print("=> loaded checkpoint (epoch {})".format(checkpoint['epoch'])) else: print("=> creating Model ({}-{}) ...".format(args.arch, args.decoder)) if config.input == 'RGBT': in_channels = 4 elif config.input == 'YT': in_channels = 2 else: print("Input type is wrong !") return 0 if args.arch == 'resnet50': #调用ResNet的定义实例化model,这里的in_channels是 model = ResNet(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_deconv1_loss0': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_with_deconv(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_deconv1_loss1': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_with_deconv_loss( layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_direct_deconv1_loss1': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_with_direct_deconv( layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_1': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_1(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_2': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_2(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_3': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_3(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_3_1': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_3_1(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_3_2': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_3_2(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_3_3': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_3_3(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_4': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_4(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_5': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_5(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_7': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_7(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_8': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_8(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_9': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_9(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_10': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_10(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_11': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_11(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_11_1': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_11_1(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_11_without_pretrain': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_11_without_pretrain( layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_12': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_12(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_13': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_13(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_14': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_14(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_15': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_15(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_16': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_16(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_17': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_17(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_18': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet50_18(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_30': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_30(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_31': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_31(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_32': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_32(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_33': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_33(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_40': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_40(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_15_1': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_15_1(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_15_2': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_15_2(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_15_3': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_15_3(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_15_4': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_15_4(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_15_5': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_15_5(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_15_6': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_15_6(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_15_8': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_15_8(layers=34, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_15_9': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_15_9(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_15_10': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_15_10(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_15_11': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_15_11(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_15_12': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_15_12(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet18': model = ResNet(layers=18, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'resnet50_20': model = ResNet50_20(Bottleneck, [3, 4, 6, 3]) elif args.arch == 'UNet': model = UNet() elif args.arch == 'UP_only': model = UP_only() elif args.arch == 'ResNet_bicubic': # 调用ResNet的定义实例化model,这里的in_channels是 model = ResNet_bicubic(layers=50, decoder=args.decoder, output_size=train_dataset.output_size, in_channels=in_channels, pretrained=args.pretrained) elif args.arch == 'VDSR': model = VDSR() elif args.arch == 'VDSR_without_res': model = VDSR_without_res() elif args.arch == 'VDSR_16': model = VDSR_16() elif args.arch == 'VDSR_16_2': model = VDSR_16_2() elif args.arch == 'Leon_resnet50': model = Leon_resnet50() elif args.arch == 'Leon_resnet101': model = Leon_resnet101() elif args.arch == 'Leon_resnet18': model = Leon_resnet18() elif args.arch == 'Double_resnet50': model = Double_resnet50() print("=> model created.") if args.finetune: print("===============loading finetune model=====================") assert os.path.isfile(config.fitune_model_dir), \ "=> no checkpoint found at '{}'".format(config.fitune_model_dir) print("=> loading checkpoint '{}'".format(config.fitune_model_dir)) best_model_filename = config.fitune_model_dir checkpoint = torch.load(best_model_filename) args.start_epoch = checkpoint['epoch'] + 1 #best_result = checkpoint['best_result'] model_fitune = checkpoint['model'] model_fitune_dict = model_fitune.state_dict() model_dict = model.state_dict() for k in model_fitune_dict: if k in model_dict: #print("There is model k: ",k) model_dict[k] = model_fitune_dict[k] #model_dict={k:v for k,v in model_fitune_dict.items() if k in model_dict} model_dict.update(model_fitune_dict) model.load_state_dict(model_dict) #optimizer = checkpoint['optimizer'] print("=> loaded checkpoint (epoch {})".format( checkpoint['epoch'])) #optimizer = torch.optim.SGD(model.parameters(), args.lr,momentum=args.momentum, weight_decay=args.weight_decay) optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, amsgrad=True, weight_decay=args.weight_decay) ''' optimizer = torch.optim.Adam( [ #{'params':model.base.parameters()}, 3 {'params': model.re_conv_Y_1.parameters(),'lr':0.0001}, {'params': model.re_conv_Y_2.parameters(), 'lr': 0.0001}, {'params': model.re_conv_Y_3.parameters(), 'lr': 0.0001}, #3 {'params': model.re_deconv_up0.parameters(), 'lr': 0.0001}, {'params': model.re_deconv_up1.parameters(), 'lr': 0.0001}, {'params': model.re_deconv_up2.parameters(), 'lr': 0.0001}, #3 {'params': model.re_conv1.parameters(), 'lr': 0.0001}, {'params': model.re_bn1.parameters(), 'lr': 0.0001}, {'params': model.re_conv4.parameters(), 'lr': 0.0001}, #5 {'params': model.re_ResNet50_layer1.parameters(), 'lr': 0.0001}, {'params': model.re_ResNet50_layer2.parameters(), 'lr': 0.0001}, {'params': model.re_ResNet50_layer3.parameters(), 'lr': 0.0001}, {'params': model.re_ResNet50_layer4.parameters(), 'lr': 0.0001}, {'params': model.re_bn2.parameters(), 'lr': 0.0001}, #5 {'params': model.re_deconcv_res_up1.parameters(), 'lr': 0.0001}, {'params': model.re_deconcv_res_up2.parameters(), 'lr': 0.0001}, {'params': model.re_deconcv_res_up3.parameters(), 'lr': 0.0001}, {'params': model.re_deconcv_res_up4.parameters(), 'lr': 0.0001}, {'params': model.re_deconv_last.parameters(), 'lr': 0.0001}, #denoise net 3 {'params': model.conv_denoise_1.parameters(), 'lr': 0}, {'params': model.conv_denoise_2.parameters(), 'lr': 0}, {'params': model.conv_denoise_3.parameters(), 'lr': 0} ] , lr=args.lr, amsgrad=True, weight_decay=args.weight_decay) ''' for state in optimizer.state.values(): for k, v in state.items(): print(type(v)) if torch.is_tensor(v): state[k] = v.cuda(config.cuda_id) print(optimizer) # create new csv files with only header with open(train_csv, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() with open(test_csv, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() # writer = SummaryWriter(log_dir='logs') model = model.cuda(config.cuda_id) #torch.save(model, './net1.pkl') for state in optimizer.state.values(): for k, v in state.items(): print(type(v)) if torch.is_tensor(v): state[k] = v.cuda() print("=> model transferred to GPU.") for epoch in range(args.start_epoch, args.epochs): train(train_loader, val_loader, model, criterion, optimizer, epoch, args.lr) # train for one epoch
parser.set_defaults(pretrained=True) args = parser.parse_args() if args.modality == 'rgb' and args.num_samples != 0: print("number of samples is forced to be 0 when input modality is rgb") args.num_samples = 0 if args.modality == 'rgb' and args.max_depth != 0.0: print("max depth is forced to be 0.0 when input modality is rgb/rgbd") args.max_depth = 0.0 print(args) fieldnames = [ 'mse', 'rmse', 'absrel', 'lg10', 'mae', 'delta1', 'delta2', 'delta3', 'data_time', 'gpu_time' ] best_result = Result() best_result.set_to_worst() def main(): global args, best_result, output_directory, train_csv, test_csv sparsifier = None max_depth = args.max_depth if args.max_depth >= 0.0 else np.inf if args.sparsifier == UniformSampling.name: sparsifier = UniformSampling(num_samples=args.num_samples, max_depth=max_depth) elif args.sparsifier == SimulatedStereo.name: sparsifier = SimulatedStereo(num_samples=args.num_samples, max_depth=max_depth)
action="store_true", help='use ImageNet pre-trained weights') parser.add_argument('--val', type=str, default="select", choices=["select", "full"], help='full or select validation set') parser.add_argument('--jitter', type=float, default=0.1, help='color jitter for images') parser.add_argument( '--rank-metric', type=str, default='rmse', choices=[m for m in dir(Result()) if not m.startswith('_')], help='metrics for which best result is sbatch_datacted') parser.add_argument( '-m', '--train-mode', type=str, default="dense", choices=["dense", "sparse", "photo", "sparse+photo", "dense+photo"], help='dense | sparse | photo | sparse+photo | dense+photo') parser.add_argument('-e', '--evaluate', default='', type=str, metavar='PATH') parser.add_argument('--cpu', action="store_true", help='run on cpu') parser.add_argument( '--partial-train', default="no", type=str, help=
def iterate(mode, args, loader, model, optimizer, logger, epoch): block_average_meter = AverageMeter() average_meter = AverageMeter() meters = [block_average_meter, average_meter] # switch to appropriate mode assert mode in ["train", "val", "eval", "test_prediction", "test_completion"], \ "unsupported mode: {}".format(mode) if mode == 'train': model.train() lr = helper.adjust_learning_rate(args.lr, optimizer, epoch) else: model.eval( ) # batchnorm or dropout layers will work in eval mode instead of training mode lr = 0 for i, batch_data in enumerate( loader ): # batch_data keys: 'd' (depth), 'gt' (ground truth), 'g' (gray) start = time.time() batch_data = { key: val.to(device) for key, val in batch_data.items() if val is not None } gt = batch_data[ 'gt'] if mode != 'test_prediction' and mode != 'test_completion' else None data_time = time.time() - start start = time.time() pred = model(batch_data) if args.save_images: # save depth predictions pred_out_dir = max(glob.glob('../outputs/var_final_NN/var.test*'), key=os.path.getmtime) + '/dense_depth_images' pred1 = pred.cpu().detach().numpy()[:, 0, :, :] for im_idx, pred_im in enumerate(pred1): pred_out_dir1 = os.path.abspath(pred_out_dir) cur_path = os.path.abspath((loader.dataset.paths)['d'][i]) basename = os.path.basename(cur_path) cur_dir = os.path.abspath(os.path.dirname(cur_path)) cur_dir = cur_dir.split('var_final_NN/')[1] new_dir = os.path.abspath(pred_out_dir1 + '/' + cur_dir) new_path = os.path.abspath(new_dir + '/' + basename) if os.path.isdir(new_dir) == False: os.makedirs(new_dir) depth_write(new_path, pred_im) depth_loss, photometric_loss, smooth_loss, mask = 0, 0, 0, None if mode == 'train': # Loss 1: the direct depth supervision from ground truth label # mask=1 indicates that a pixel does not ground truth labels if 'sparse' in args.train_mode: depth_loss = depth_criterion(pred, batch_data['d']) mask = (batch_data['d'] < 1e-3).float() elif 'dense' in args.train_mode: depth_loss = depth_criterion(pred, gt) mask = (gt < 1e-3).float() # Loss 2: the self-supervised photometric loss if args.use_pose: # create multi-scale pyramids pred_array = helper.multiscale(pred) rgb_curr_array = helper.multiscale(batch_data['rgb']) rgb_near_array = helper.multiscale(batch_data['rgb_near']) if mask is not None: mask_array = helper.multiscale(mask) num_scales = len(pred_array) # compute photometric loss at multiple scales for scale in range(len(pred_array)): pred_ = pred_array[scale] rgb_curr_ = rgb_curr_array[scale] rgb_near_ = rgb_near_array[scale] mask_ = None if mask is not None: mask_ = mask_array[scale] # compute the corresponding intrinsic parameters height_, width_ = pred_.size(2), pred_.size(3) intrinsics_ = kitti_intrinsics.scale(height_, width_) # inverse warp from a nearby frame to the current frame warped_ = homography_from(rgb_near_, pred_, batch_data['r_mat'], batch_data['t_vec'], intrinsics_) photometric_loss += photometric_criterion( rgb_curr_, warped_, mask_) * (2**(scale - num_scales)) # Loss 3: the depth smoothness loss smooth_loss = smoothness_criterion(pred) if args.w2 > 0 else 0 # backprop loss = depth_loss + args.w1 * photometric_loss + args.w2 * smooth_loss optimizer.zero_grad() loss.backward() optimizer.step() gpu_time = time.time() - start # measure accuracy and record loss of each batch with torch.no_grad( ): # impacts the autograd engine and deactivate it (will reduce memory usage and speed up computations) mini_batch_size = next(iter(batch_data.values())).size(0) result = Result() # metrics if mode != 'test_prediction' and mode != 'test_completion': result.evaluate(pred.data, gt.data, photometric_loss) [ m.update(result, gpu_time, data_time, mini_batch_size) for m in meters ] logger.conditional_print(mode, i, epoch, args.epochs, lr, len(loader), block_average_meter, average_meter) logger.conditional_save_img_comparison(mode, i, batch_data, pred, epoch) logger.conditional_save_pred(mode, i, pred, epoch) del pred avg = logger.conditional_save_info( mode, average_meter, epoch) # take the avg of all the batches, to get the epoch metrics is_best = logger.rank_conditional_save_best(mode, avg, epoch, args.epochs) if is_best and not (mode == "train"): logger.save_img_comparison_as_best(mode, epoch) logger.conditional_summarize(mode, avg, is_best) return avg, is_best
def validate(val_loader, model, epoch, batch_epoch, write_to_file=True): average_meter = AverageMeter() model.eval() # switch to evaluate mode end = time.time() for i, (Y, Y_1_2, Y_1_4, Y_1_8, LR, LR_8, HR, name) in enumerate(val_loader): Y = Y.cuda() Y_1_2 = Y_1_2.cuda() Y_1_4 = Y_1_4.cuda() Y_1_8 = Y_1_8.cuda() LR_8 = LR_8.cuda() LR = LR.cuda() HR = HR.cuda() torch.cuda.synchronize() data_time = time.time() - end # compute output end = time.time() with torch.no_grad(): # print("I am for validation in main 342") if args.arch == 'VDSR_16': pred_HR = model(LR) elif args.arch == 'VDSR_16_2': pred_HR = model(Y, LR) elif args.arch == 'VDSR': pred_HR, residule = model(LR_8, Y) elif args.arch == 'ResNet_bicubic': pred_HR, residule = model(LR_8, Y) elif args.arch == 'resnet50_15_6' or 'resnet50_15_11' or 'resnet50_15_12': pred_HR = model(Y_1_2, LR) elif args.arch == 'resnet50_15_2' or 'resnet50_15_3' or 'resnet50_15_5' or 'resnet50_15_8' or 'resnet50_15_9': pred_HR, residule = model(Y, LR, LR_8) else: if config.use_different_size_Y == 1: pred_HR, pred_thermal0 = model(Y, Y_1_2, Y_1_4, Y_1_8, LR) else: pred_HR, pred_thermal = model(Y, LR) torch.cuda.synchronize() gpu_time = time.time() - end # measure accuracy and record loss result = Result() result.evaluate(pred_HR, HR) average_meter.update(result, gpu_time, data_time, Y.size(0)) #Y.size(0) batch_size end = time.time() # save 8 images for visualization,对验证集合生产图片 skip = config.skip if i == 0: img_merge = utils.merge_into_row_with_YT(Y, LR, pred_HR, HR) elif (i < 8 * skip) and (i % skip == 0): row = utils.merge_into_row_with_YT(Y, LR, pred_HR, HR) img_merge = utils.add_row(img_merge, row) elif i == 8 * skip: #储存最终的图片 filename = output_directory + '/Compair_data_epoch_' + str( epoch) + '_batch_eopch_' + str(batch_epoch + 1) + '.png' utils.save_image(img_merge, filename) print("生成第" + str(batch_epoch + 1) + "图片") if (i + 1) % args.print_freq == 0: print('Test: [{0}/{1}]\t' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'RMSE={result.rmse:.2f}({average.rmse:.2f}) ' 'MAE={result.mae:.2f}({average.mae:.2f}) ' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'REL={result.absrel:.3f}({average.absrel:.3f}) ' 'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format( i + 1, len(val_loader), gpu_time=gpu_time, result=result, average=average_meter.average())) avg = average_meter.average() print('\n*\n' 'RMSE={average.rmse:.3f}\n' 'MAE={average.mae:.3f}\n' 'Delta1={average.delta1:.3f}\n' 'REL={average.absrel:.3f}\n' 'Lg10={average.lg10:.3f}\n' 't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time)) if write_to_file: with open(test_csv, 'a') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writerow({ 'dataset epoch': epoch, 'batch epoch': batch_num + 1, 'psnr': 10 * math.log(1 / (avg.mse), 10), 'mse': avg.mse, 'rmse': avg.rmse, 'absrel': avg.absrel, 'lg10': avg.lg10, 'mae': avg.mae, 'delta1': avg.delta1, 'delta2': avg.delta2, 'delta3': avg.delta3, 'data_time': avg.data_time, 'gpu_time': avg.gpu_time }) return avg, img_merge
def validate(val_loader, model, epoch, write_to_file=True): average_meter = AverageMeter() # switch to evaluate mode model.eval() end = time.time() for i, (input, target) in enumerate(val_loader): input, target = input.cuda(), target.cuda() # torch.cuda.synchronize() data_time = time.time() - end # compute output end = time.time() with torch.no_grad(): pred = model(input) # torch.cuda.synchronize() gpu_time = time.time() - end # measure accuracy and record loss result = Result() result.evaluate(pred.data, target.data) average_meter.update(result, gpu_time, data_time, input.size(0)) end = time.time() # save 8 images for visualization skip = 50 if args.modality == 'd': img_merge = None else: if args.modality == 'rgb': rgb = input elif args.modality == 'rgbd': rgb = input[:, :3, :, :] depth = input[:, 3:, :, :] if i == 0: if args.modality == 'rgbd': img_merge = utils.merge_into_row_with_gt( rgb, depth, target, pred) else: img_merge = utils.merge_into_row(rgb, target, pred) elif (i < 8 * skip) and (i % skip == 0): if args.modality == 'rgbd': row = utils.merge_into_row_with_gt(rgb, depth, target, pred) else: row = utils.merge_into_row(rgb, target, pred) img_merge = utils.add_row(img_merge, row) elif i == 8 * skip: filename = output_directory + '/comparison_' + str( epoch) + '.png' utils.save_image(img_merge, filename) if (i + 1) % args.print_freq == 0: print('Test: [{0}/{1}]\t' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'RMSE={result.rmse:.2f}({average.rmse:.2f}) ' 'MAE={result.mae:.2f}({average.mae:.2f})\n\t' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'REL={result.absrel:.3f}({average.absrel:.3f}) ' 'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format( i + 1, len(val_loader), gpu_time=gpu_time, result=result, average=average_meter.average())) avg = average_meter.average() print('\n*\n' 'RMSE={average.rmse:.3f}\n' 'MAE={average.mae:.3f}\n' 'Delta1={average.delta1:.3f}\n' 'REL={average.absrel:.3f}\n' 'Lg10={average.lg10:.3f}\n' 't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time)) if write_to_file: with open(test_csv, 'a') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writerow({ 'mse': avg.mse, 'rmse': avg.rmse, 'absrel': avg.absrel, 'lg10': avg.lg10, 'mae': avg.mae, 'delta1': avg.delta1, 'delta2': avg.delta2, 'delta3': avg.delta3, 'data_time': avg.data_time, 'gpu_time': avg.gpu_time }) return avg, img_merge
def test(model): global name test_dir = config.test_dir test_dataset = YT_dataset(test_dir, config, is_train_set=False) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=args.workers, pin_memory=True) file_name = config.test_result_name + str('.csv') test_csv_2 = os.path.join(config.test_output_dir, file_name) #print(test_csv_2) average_meter = AverageMeter() model.eval() # switch to evaluate mode end = time.time() num = 0 for i, (Y, Y_1_2, Y_1_4, Y_1_8, LR, LR_8, HR, name) in enumerate(test_loader): Y = Y.cuda() LR = LR.cuda() LR_8 = LR_8.cuda() HR = HR.cuda() torch.cuda.synchronize() data_time = time.time() - end end = time.time() with torch.no_grad(): # print("I am for validation in main 342") num = num + 1 if args.arch == 'VDSR_16': pred_HR = model(LR) elif args.arch == 'VDSR_16_2': pred_HR = model(Y, LR) elif args.arch == 'VDSR': pred_HR, residule = model(LR_8, Y) elif args.arch == 'ResNet_bicubic': pred_HR, residule = model(LR_8, Y) elif args.arch == 'resnet50_15_2' or 'resnet50_15_3': pred_HR, residule = model(Y, LR, LR_8) else: pred_HR, thermal0 = model(Y, LR) gpu_time = time.time() - end image_pred_HR = trans_norm_tensor_into_T(pred_HR) #image_pred_HR = trans_norm_tensor_into_T(thermal0) image_LR = trans_norm_tensor_into_rgb(LR) image_HR = trans_norm_tensor_into_rgb(HR) #image_HR=trans_norm_tensor_into_T(HR) name_HR = str( config.test_output_dir_HR) + str(num) + str('_HR') + str(".png") name_pred_HR = str( config.test_output_dir_pred_HR) + str(num) + str(".png") name_HR_bicubic_HR_res = str( config.test_output_dir_res) + str(num) + str(".png") #name_bicubic_HR = str(config.test_output_dir_bicubic_HR) + str(num) + str(".png") #cv2.imwrite(name_HR,image_HR[...,::-1]) #print(name_pred_HR) #cv2.imwrite(name_pred_HR,image_pred_HR[...,::-1]) #cv2.waitKey(0) torch.cuda.synchronize() # measure accuracy and record loss result = Result() result.evaluate(pred_HR, HR) average_meter.update(result, gpu_time, data_time, Y.size(0)) # Y.size(0) batch_size end = time.time() # save 8 images for visualization,对验证集合生产图片 dir_small = '/home1/sas/datasets/crop/smallthan7/' dir_big = '/home1/sas/datasets/crop/bigthan/' dir_out = '/home/zju231/sas/data/old_val_resnet50_15_1/' #dir_out='/home1/sas/results_model/pytorch/YT-SD/resnet50_30_thermal0/' #dir_out=test_dir name1 = dir_out + str(name[0][:-4]) + str('.png') cv2.imwrite(name1, image_pred_HR) name2 = dir_out + str(name[0][:-4]) + str('_HR.png') #cv2.imwrite(name2, image_HR[..., ::-1]) name3 = dir_out + str(name[0][:-4]) + str('_Y.png') #cv2.imwrite(name3, np.squeeze(np.array(Y.cpu())) * 255) name4 = dir_out + str(name[0][:-4]) + str('_LR.png') #cv2.imwrite(name4, image_LR[..., ::-1]) print('Test: [{0}/{1}]\t' 't_GPU={gpu_time:.3f}({average.gpu_time:.3f})\n\t' 'PSNR={result.psnr:.5f}({average.psnr:.5f}) ' 'RMSE={result.rmse:.5f}({average.rmse:.5f}) ' 'MAE={result.mae:.5f}({average.mae:.5f}) ' 'Delta1={result.delta1:.3f}({average.delta1:.3f}) ' 'REL={result.absrel:.3f}({average.absrel:.3f}) ' 'Lg10={result.lg10:.3f}({average.lg10:.3f}) '.format( i + 1, len(test_loader), gpu_time=gpu_time, result=result, average=average_meter.average())) avg = average_meter.average() print('\n*\n' 'PSNR={average.psnr:.5f}\n' 'RMSE={average.rmse:.3f}\n' 'MAE={average.mae:.3f}\n' 'Delta1={average.delta1:.3f}\n' 'REL={average.absrel:.3f}\n' 'Lg10={average.lg10:.3f}\n' 't_GPU={time:.3f}\n'.format(average=avg, time=avg.gpu_time)) if 1: with open(test_csv_2, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() writer.writerow({ 'psnr': 10 * math.log(1 / (avg.mse), 10), 'mse': avg.mse, 'rmse': avg.rmse, 'absrel': avg.absrel, 'lg10': avg.lg10, 'mae': avg.mae, 'delta1': avg.delta1, 'delta2': avg.delta2, 'delta3': avg.delta3, 'data_time': avg.data_time, 'gpu_time': avg.gpu_time })