def train(args):
seqmap, entries = generate_seqmapping()
eval_dataset = KITTI_eigen(root=args.dataset_root,
inheight=args.evalheight,
inwidth=args.evalwidth,
entries=entries,
depth_root=args.depthvlsgt_root,
depthvls_root=args.depthvlsgt_root,
maxinsnum=args.maxinsnum,
istrain=False,
isgarg=True,
flowPred_root=args.flowPred_root,
ins_root=args.ins_root,
mdPred_root=args.mdPred_root)
eval_loader = data.DataLoader(eval_dataset,
batch_size=1,
pin_memory=True,
num_workers=args.num_workers,
drop_last=False,
shuffle=False)
print("Test splits contain %d images" % (eval_dataset.__len__()))
validate_RANSAC_odom_relpose(args,
eval_loader,
banins=args.banins,
bangrad=args.bangrad,
samplenum=args.samplenum)
return
def train(gpu, ngpus_per_node, args):
print("Using GPU %d for training" % gpu)
args.gpu = gpu
if args.distributed:
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=ngpus_per_node,
rank=args.gpu)
seqmap, entries = generate_seqmapping()
import random
random.shuffle(entries)
entries = entries[0:5000]
# entries = entries[0:5000]
eval_dataset = KITTI_eigen(root=args.dataset_root,
inheight=args.evalheight,
inwidth=args.evalwidth,
entries=entries,
depth_root=args.depth_root,
depthvls_root=args.depthvlsgt_root,
maxinsnum=args.maxinsnum,
istrain=False,
isgarg=True,
flowPred_root=args.flowPred_root,
ins_root=args.ins_root,
mdPred_root=args.mdPred_root,
deepv2dpred_root=args.deepv2dpred_root)
eval_sampler = torch.utils.data.distributed.DistributedSampler(
eval_dataset) if args.distributed else None
eval_loader = data.DataLoader(eval_dataset,
batch_size=1,
pin_memory=True,
num_workers=3,
drop_last=False,
sampler=eval_sampler,
shuffle=False)
print("Test splits contain %d images" % (eval_dataset.__len__()))
if args.distributed:
group = dist.new_group([i for i in range(ngpus_per_node)])
validate_RANSAC_odom_relpose(args, eval_loader, group, seqmap)
return
def train(gpu, ngpus_per_node, args):
print("Using GPU %d for training" % gpu)
args.gpu = gpu
if args.distributed:
dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=ngpus_per_node, rank=args.gpu)
model = EppFlowNet(args=args)
if args.distributed:
torch.cuda.set_device(args.gpu)
args.batch_size = int(args.batch_size / ngpus_per_node)
model = nn.SyncBatchNorm.convert_sync_batchnorm(module=model)
model = model.to(f'cuda:{args.gpu}')
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], find_unused_parameters=True, output_device=args.gpu)
else:
model = torch.nn.DataParallel(model)
model.cuda()
logroot = os.path.join(args.logroot, args.name)
print("Parameter Count: %d, saving location: %s" % (count_parameters(model), logroot))
if args.restore_ckpt is not None:
print("=> loading checkpoint '{}'".format(args.restore_ckpt))
loc = 'cuda:{}'.format(args.gpu)
checkpoint = torch.load(args.restore_ckpt, map_location=loc)
model.load_state_dict(checkpoint, strict=False)
train_entries, evaluation_entries = read_splits()
eval_dataset = KITTI_eigen(root=args.dataset_root, inheight=args.evalheight, inwidth=args.evalwidth, entries=evaluation_entries, maxinsnum=args.maxinsnum,
depth_root=args.depth_root, depthvls_root=args.depthvlsgt_root, prediction_root=args.prediction_root, deepv2dpred_root=args.deepv2dpred_root,
mdPred_root=args.mdPred_root, ins_root=args.ins_root, istrain=False, isgarg=True, RANSACPose_root=args.RANSACPose_root)
eval_sampler = torch.utils.data.distributed.DistributedSampler(eval_dataset) if args.distributed else None
eval_loader = data.DataLoader(eval_dataset, batch_size=1, pin_memory=True, num_workers=3, drop_last=True, sampler=eval_sampler)
print("Test splits contain %d images" % (eval_dataset.__len__()))
if args.distributed:
group = dist.new_group([i for i in range(ngpus_per_node)])
validate_kitti(model.module, args, eval_loader, None, group, None, isdeepv2d=False)
# validate_kitti(model.module, args, eval_loader, None, group, None, isdeepv2d=True)
return
def train(gpu, ngpus_per_node, args):
print("Using GPU %d for training" % gpu)
args.gpu = gpu
if args.distributed:
dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=ngpus_per_node, rank=args.gpu)
model = EppFlowNet(args=args)
if args.distributed:
torch.cuda.set_device(args.gpu)
args.batch_size = int(args.batch_size / ngpus_per_node)
model = nn.SyncBatchNorm.convert_sync_batchnorm(module=model)
model = model.to(f'cuda:{args.gpu}')
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], find_unused_parameters=True, output_device=args.gpu)
else:
model = torch.nn.DataParallel(model)
model.cuda()
ssim = SSIM()
logroot = os.path.join(args.logroot, args.name)
print("Parameter Count: %d, saving location: %s" % (count_parameters(model), logroot))
if args.restore_ckpt is not None:
print("=> loading checkpoint '{}'".format(args.restore_ckpt))
loc = 'cuda:{}'.format(args.gpu)
checkpoint = torch.load(args.restore_ckpt, map_location=loc)
model.load_state_dict(checkpoint, strict=False)
model.train()
train_entries, evaluation_entries = read_splits()
train_dataset = KITTI_eigen(root=args.dataset_root, inheight=args.inheight, inwidth=args.inwidth, entries=train_entries, maxinsnum=args.maxinsnum,
depth_root=args.depth_root, depthvls_root=args.depthvlsgt_root, prediction_root=args.prediction_root, ins_root=args.ins_root,
istrain=True, muteaug=False, banremovedup=False)
train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) if args.distributed else None
train_loader = data.DataLoader(train_dataset, batch_size=args.batch_size, pin_memory=True, num_workers=int(args.num_workers / ngpus_per_node), drop_last=True, sampler=train_sampler)
eval_dataset = KITTI_eigen(root=args.dataset_root, inheight=args.evalheight, inwidth=args.evalwidth, entries=evaluation_entries, maxinsnum=args.maxinsnum,
depth_root=args.depth_root, depthvls_root=args.depthvlsgt_root, prediction_root=args.prediction_root, ins_root=args.ins_root, istrain=False)
eval_sampler = torch.utils.data.distributed.DistributedSampler(eval_dataset) if args.distributed else None
eval_loader = data.DataLoader(eval_dataset, batch_size=args.batch_size, pin_memory=True, num_workers=3, drop_last=True, sampler=eval_sampler)
print("Training splits contain %d images while test splits contain %d images" % (train_dataset.__len__(), eval_dataset.__len__()))
if args.distributed:
group = dist.new_group([i for i in range(ngpus_per_node)])
optimizer, scheduler = fetch_optimizer(args, model)
total_steps = 0
if args.gpu == 0:
logger = Logger(logroot)
logger_evaluation = Logger(os.path.join(args.logroot, 'evaluation_eigen_background', args.name))
logger_evaluation_org = Logger(os.path.join(args.logroot, 'evaluation_eigen_background', "{}_org".format(args.name)))
logger.create_summarywriter()
logger_evaluation.create_summarywriter()
logger_evaluation_org.create_summarywriter()
VAL_FREQ = 5000
epoch = 0
minreld = 100
st = time.time()
should_keep_training = True
while should_keep_training:
train_sampler.set_epoch(epoch)
for i_batch, data_blob in enumerate(train_loader):
optimizer.zero_grad()
image1 = data_blob['img1'].cuda(gpu) / 255.0
image2 = data_blob['img2'].cuda(gpu) / 255.0
intrinsic = data_blob['intrinsic'].cuda(gpu)
insmap = data_blob['insmap'].cuda(gpu)
depthgt = data_blob['depthmap'].cuda(gpu)
depthpred = data_blob['depthpred'].cuda(gpu)
posepred = data_blob['posepred'].cuda(gpu)
selfpose_gt = data_blob['rel_pose'].cuda(gpu)
reldepth_gt = torch.log(depthgt + 1e-10) - torch.log(torch.sqrt(torch.sum(selfpose_gt[:, 0:3, 3] ** 2, dim=1, keepdim=True))).unsqueeze(-1).unsqueeze(-1).expand([-1, -1, args.inheight, args.inwidth])
outputs = model(image1, image2, depthpred, intrinsic, posepred, insmap)
depthloss, depthselector = get_rdepth_loss(reldepth_gt=reldepth_gt, depthgt=depthgt, outputs=outputs, insmap=insmap)
ssimloss, reprojselector = get_reprojection_loss(image1, insmap, outputs, ssim)
metrics = dict()
metrics['depthloss'] = depthloss.item()
metrics['ssimloss'] = ssimloss.item()
loss = depthloss + ssimloss * 0
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip)
optimizer.step()
scheduler.step()
if args.gpu == 0:
logger.write_dict(metrics, step=total_steps)
if total_steps % SUM_FREQ == 0:
dr = time.time() - st
resths = (args.num_steps - total_steps) * dr / (total_steps + 1) / 60 / 60
print("Step: %d, rest hour: %f, depthloss: %f, ssimloss: %f" % (total_steps, resths, depthloss.item(), ssimloss.item()))
logger.write_vls(data_blob, outputs, depthselector, reprojselector, total_steps)
if total_steps % VAL_FREQ == 1:
if args.gpu == 0:
results = validate_kitti(model.module, args, eval_loader, logger, group, total_steps, isdeepv2dpred=True)
else:
results = validate_kitti(model.module, args, eval_loader, None, group, None, isdeepv2dpred=True)
if args.gpu == 0:
logger_evaluation.write_dict(results, total_steps)
if minreld > results['reld']:
minreld = results['reld']
PATH = os.path.join(logroot, 'minreld.pth')
torch.save(model.state_dict(), PATH)
print("model saved to %s" % PATH)
if args.gpu == 0:
results = validate_kitti(model.module, args, eval_loader, logger, group, total_steps, isdeepv2dpred=False)
else:
results = validate_kitti(model.module, args, eval_loader, None, group, None, isdeepv2dpred=False)
if args.gpu == 0:
logger_evaluation_org.write_dict(results, total_steps)
model.train()
total_steps += 1
if total_steps > args.num_steps:
should_keep_training = False
break
epoch = epoch + 1
if args.gpu == 0:
logger.close()
PATH = os.path.join(logroot, 'final.pth')
torch.save(model.state_dict(), PATH)
return PATH
def train(gpu, ngpus_per_node, args):
print("Using GPU %d for training" % gpu)
args.gpu = gpu
model = EppFlowNet(args=args)
model = torch.nn.DataParallel(model)
model.cuda()
if args.restore_ckpt is not None:
print("=> loading checkpoint '{}'".format(args.restore_ckpt))
loc = 'cuda:{}'.format(args.gpu)
checkpoint = torch.load(args.restore_ckpt, map_location=loc)
model.load_state_dict(checkpoint, strict=False)
train_entries, evaluation_entries = read_splits()
eval_dataset = KITTI_eigen(root=args.dataset_root,
inheight=args.evalheight,
inwidth=args.evalwidth,
entries=train_entries,
maxinsnum=args.maxinsnum,
depth_root=args.depth_root,
depthvls_root=args.depthvlsgt_root,
prediction_root=args.prediction_root,
mdPred_root=args.mdPred_root,
ins_root=args.ins_root,
istrain=False,
isgarg=False,
RANSACPose_root=args.RANSACPose_root,
baninsmap=args.baninsmap)
eval_loader = data.DataLoader(eval_dataset,
batch_size=1,
pin_memory=True,
num_workers=0,
drop_last=True)
model.eval()
gpu = args.gpu
totnum = 0
dr = 0
with torch.no_grad():
for val_id, data_blob in enumerate(eval_loader):
image1 = data_blob['img1'].cuda(gpu) / 255.0
image2 = data_blob['img2'].cuda(gpu) / 255.0
intrinsic = data_blob['intrinsic'].cuda(gpu)
insmap = data_blob['insmap'].cuda(gpu)
posepred = data_blob['posepred'].cuda(gpu)
mD_pred = data_blob['mdDepth_pred'].cuda(gpu)
mD_pred_clipped = torch.clamp_min(mD_pred, min=args.min_depth_pred)
st = time.time()
outputs = model(image1, image2, mD_pred_clipped, intrinsic,
posepred, insmap)
dr += time.time() - st
totnum += 1
print("%d Samples, Ave sec/frame: %f, Mem: %f Gb" %
(totnum, dr / totnum,
float(torch.cuda.memory_allocated() / 1024 / 1024 / 1024)))
return
def train(gpu, ngpus_per_node, args):
print("Using GPU %d for training" % gpu)
args.gpu = gpu
if args.distributed:
dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=ngpus_per_node, rank=args.gpu)
model = RAFT(args=args)
if args.distributed:
torch.cuda.set_device(args.gpu)
args.batch_size = int(args.batch_size / ngpus_per_node)
model = nn.SyncBatchNorm.convert_sync_batchnorm(module=model)
model = model.to(f'cuda:{args.gpu}')
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], find_unused_parameters=True, output_device=args.gpu)
else:
model = torch.nn.DataParallel(model)
model.cuda()
logroot = os.path.join(args.logroot, args.name)
print("Parameter Count: %d, saving location: %s" % (count_parameters(model), logroot))
if args.restore_ckpt is not None:
print("=> loading checkpoint '{}'".format(args.restore_ckpt))
loc = 'cuda:{}'.format(args.gpu)
checkpoint = torch.load(args.restore_ckpt, map_location=loc)
model.load_state_dict(checkpoint, strict=False)
model.train()
train_entries, evaluation_entries = read_splits()
train_dataset = KITTI_eigen(root=args.dataset_root, inheight=args.inheight, inwidth=args.inwidth, entries=train_entries, maxinsnum=args.maxinsnum,
depth_root=args.depth_root, depthvls_root=args.depthvlsgt_root, prediction_root=args.prediction_root, ins_root=args.ins_root,
istrain=True, muteaug=False, banremovedup=False, isgarg=True)
train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) if args.distributed else None
train_loader = data.DataLoader(train_dataset, batch_size=args.batch_size, pin_memory=True, num_workers=int(args.num_workers / ngpus_per_node), drop_last=True, sampler=train_sampler)
eval_dataset = KITTI_eigen(root=args.dataset_root, inheight=args.evalheight, inwidth=args.evalwidth, entries=evaluation_entries, maxinsnum=args.maxinsnum,
depth_root=args.depth_root, depthvls_root=args.depthvlsgt_root, prediction_root=args.prediction_root, ins_root=args.ins_root, istrain=False, isgarg=True)
eval_sampler = torch.utils.data.distributed.DistributedSampler(eval_dataset) if args.distributed else None
eval_loader = data.DataLoader(eval_dataset, batch_size=1, pin_memory=True, num_workers=3, drop_last=True, sampler=eval_sampler)
print("Training splits contain %d images while test splits contain %d images" % (train_dataset.__len__(), eval_dataset.__len__()))
if args.distributed:
group = dist.new_group([i for i in range(ngpus_per_node)])
optimizer, scheduler = fetch_optimizer(args, model, int(train_dataset.__len__() / 2))
total_steps = 0
if args.gpu == 0:
logger = Logger(logroot)
logger_evaluation = Logger(os.path.join(args.logroot, 'evaluation_eigen_background', args.name))
logger.create_summarywriter()
logger_evaluation.create_summarywriter()
VAL_FREQ = 5000
epoch = 0
minout = 1
st = time.time()
should_keep_training = True
while should_keep_training:
train_sampler.set_epoch(epoch)
for i_batch, data_blob in enumerate(train_loader):
optimizer.zero_grad()
image1 = data_blob['img1'].cuda(gpu) / 255.0
image2 = data_blob['img2'].cuda(gpu) / 255.0
flowmap = data_blob['flowmap'].cuda(gpu)
outputs = model(image1, image2)
selector = (flowmap[:, 0, :, :] != 0)
flow_loss = sequence_loss(outputs, flowmap, selector, gamma=args.gamma, max_flow=MAX_FLOW)
metrics = dict()
metrics['flow_loss'] = flow_loss
loss = flow_loss
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip)
optimizer.step()
scheduler.step()
if args.gpu == 0:
logger.write_dict(metrics, step=total_steps)
if total_steps % SUM_FREQ == 0:
dr = time.time() - st
resths = (args.num_steps - total_steps) * dr / (total_steps + 1) / 60 / 60
print("Step: %d, rest hour: %f, flowloss: %f" % (total_steps, resths, flow_loss.item()))
logger.write_vls(data_blob, outputs, selector.unsqueeze(1), total_steps)
if total_steps % VAL_FREQ == 1:
if args.gpu == 0:
results = validate_kitti(model.module, args, eval_loader, logger, group, total_steps)
else:
results = validate_kitti(model.module, args, eval_loader, None, group, None)
if args.gpu == 0:
logger_evaluation.write_dict(results, total_steps)
if minout > results['out']:
minout = results['out']
PATH = os.path.join(logroot, 'minout.pth')
torch.save(model.state_dict(), PATH)
print("model saved to %s" % PATH)
model.train()
total_steps += 1
if total_steps > args.num_steps:
should_keep_training = False
break
epoch = epoch + 1
if args.gpu == 0:
logger.close()
PATH = os.path.join(logroot, 'final.pth')
torch.save(model.state_dict(), PATH)
return
def train(gpu, ngpus_per_node, args):
print("Using GPU %d for training" % gpu)
args.gpu = gpu
if args.distributed:
dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=ngpus_per_node, rank=args.gpu)
model = EppFlowNet(args=args)
if args.distributed:
torch.cuda.set_device(args.gpu)
args.batch_size = int(args.batch_size / ngpus_per_node)
model = nn.SyncBatchNorm.convert_sync_batchnorm(module=model)
model = model.to(f'cuda:{args.gpu}')
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], find_unused_parameters=True, output_device=args.gpu)
else:
model = torch.nn.DataParallel(model)
model.cuda()
logroot = os.path.join(args.logroot, args.name)
print("Parameter Count: %d, saving location: %s" % (count_parameters(model), logroot))
if args.restore_ckpt is not None:
print("=> loading checkpoint '{}'".format(args.restore_ckpt))
loc = 'cuda:{}'.format(args.gpu)
checkpoint = torch.load(args.restore_ckpt, map_location=loc)
model.load_state_dict(checkpoint, strict=False)
train_entries, evaluation_entries = read_splits()
if args.distributed:
group = dist.new_group([i for i in range(ngpus_per_node)])
if args.single_test:
eval_dataset = KITTI_eigen(root=args.dataset_root, inheight=args.evalheight, inwidth=args.evalwidth,
entries=evaluation_entries, maxinsnum=args.maxinsnum,
depth_root=args.depth_root, depthvls_root=args.depthvlsgt_root,
prediction_root=args.prediction_root, deepv2dpred_root=args.deepv2dpred_root,
mdPred_root=args.mdPred_root, ins_root=args.ins_root, istrain=False, isgarg=True,
RANSACPose_root=args.RANSACPose_root, baninsmap=args.baninsmap)
eval_sampler = torch.utils.data.distributed.DistributedSampler(eval_dataset) if args.distributed else None
eval_loader = data.DataLoader(eval_dataset, batch_size=1, pin_memory=True, num_workers=3, drop_last=False,
sampler=eval_sampler)
print("Test splits contain %d images" % (eval_dataset.__len__()))
validate_kitti(model.module, args, eval_loader, group, ismean=True)
else:
pose_folds = glob.glob(os.path.join(args.RANSACPose_root, '*/'))
pose_folds.sort()
eval_measures_depth_rec = list()
for pose_fold in pose_folds:
if '_txt' in pose_fold:
continue
args.RANSACPose_root = pose_fold
eval_dataset = KITTI_eigen(root=args.dataset_root, inheight=args.evalheight, inwidth=args.evalwidth, entries=evaluation_entries, maxinsnum=args.maxinsnum,
depth_root=args.depth_root, depthvls_root=args.depthvlsgt_root, prediction_root=args.prediction_root, deepv2dpred_root=args.deepv2dpred_root,
mdPred_root=args.mdPred_root, ins_root=args.ins_root, istrain=False, isgarg=True, RANSACPose_root=args.RANSACPose_root, baninsmap=args.baninsmap)
eval_sampler = torch.utils.data.distributed.DistributedSampler(eval_dataset) if args.distributed else None
eval_loader = data.DataLoader(eval_dataset, batch_size=1, pin_memory=True, num_workers=3, drop_last=False, sampler=eval_sampler)
print("Test splits contain %d images" % (eval_dataset.__len__()))
eval_measures_depth = validate_kitti(model.module, args, eval_loader, group, ismean=True)
if args.gpu == 0:
eval_measures_depth_rec.append(eval_measures_depth)
if args.gpu == 0:
eval_measures_depth_rec = np.stack(eval_measures_depth_rec, axis=1)
eval_measures_depth_mean = np.mean(eval_measures_depth_rec, axis=1)
eval_measures_depth_std = np.std(eval_measures_depth_rec, axis=1)
print("=============AVE Mean Scaling====================")
print("{:>18}, {:>18}, {:>18}, {:>18}, {:>18}, {:>18}, {:>18}, {:>18}, {:>18}".format('silog', 'abs_rel', 'log10', 'rms', 'sq_rel', 'log_rms', 'd1', 'd2', 'd3'))
for i in range(8):
print('({:7.3f}, {:7.3f}), '.format(eval_measures_depth_mean[i], eval_measures_depth_std[i]), end='')
print('({:7.3f}, {:7.3f})'.format(eval_measures_depth_mean[8], eval_measures_depth_std[i]))
print("=================================================")
eval_measures_depth_rec = list()
for pose_fold in pose_folds:
if '_txt' in pose_fold:
continue
args.RANSACPose_root = pose_fold
eval_dataset = KITTI_eigen(root=args.dataset_root, inheight=args.evalheight, inwidth=args.evalwidth, entries=evaluation_entries, maxinsnum=args.maxinsnum,
depth_root=args.depth_root, depthvls_root=args.depthvlsgt_root, prediction_root=args.prediction_root, deepv2dpred_root=args.deepv2dpred_root,
mdPred_root=args.mdPred_root, ins_root=args.ins_root, istrain=False, isgarg=True, RANSACPose_root=args.RANSACPose_root, baninsmap=args.baninsmap)
eval_sampler = torch.utils.data.distributed.DistributedSampler(eval_dataset) if args.distributed else None
eval_loader = data.DataLoader(eval_dataset, batch_size=1, pin_memory=True, num_workers=3, drop_last=True, sampler=eval_sampler)
print("Test splits contain %d images" % (eval_dataset.__len__()))
eval_measures_depth = validate_kitti(model.module, args, eval_loader, group, ismean=False)
if args.gpu == 0:
eval_measures_depth_rec.append(eval_measures_depth)
if args.gpu == 0:
eval_measures_depth_rec = np.stack(eval_measures_depth_rec, axis=1)
eval_measures_depth_mean = np.mean(eval_measures_depth_rec, axis=1)
eval_measures_depth_std = np.std(eval_measures_depth_rec, axis=1)
print("=============No Mean Scaling====================")
print("{:>18}, {:>18}, {:>18}, {:>18}, {:>18}, {:>18}, {:>18}, {:>18}, {:>18}".format('silog', 'abs_rel', 'log10', 'rms', 'sq_rel', 'log_rms', 'd1', 'd2', 'd3'))
for i in range(8):
print('({:7.3f}, {:7.3f}), '.format(eval_measures_depth_mean[i], eval_measures_depth_std[i]), end='')
print('({:7.3f}, {:7.3f})'.format(eval_measures_depth_mean[8], eval_measures_depth_std[i]))
print("=================================================")
return