def test_data_structure(self):
if os.path.exists('/home/mlt/mot/fafe/cfg/adams_computer'):
config_path = 'cfg/cfg_pp_mini.yml'
elif os.path.exists('/Users/erikbohnsack'):
config_path = 'cfg/cfg_mac.yml'
else:
config_path = 'cfg/cfg_pp.yml'
print('Using config: \n\t{}\n'.format(config_path))
config = load_config(config_path)
input_config = InputConfig(config['INPUT_CONFIG'])
post_config = PostConfig(config['POST_CONFIG'])
data = torch.load('inference0.pt')
data1 = torch.load('inference1.pt')
data2 = torch.load('inference2.pt')
device = torch.device("cpu")
post_fafe = PostFafe(input_config, post_config, device)
print(post_fafe.object_state)
post_fafe(data[0])
print(post_fafe.object_state.keys())
post_fafe(data1[0])
print(post_fafe.object_state.keys())
post_fafe(data2[0])
print(post_fafe.object_state.keys())
def test_plot_bevs(self):
config_path = 'cfg_mini.yml'
config = load_config(config_path)
input_config = InputConfig(config['INPUT_CONFIG'])
train_config = TrainConfig(config['TRAIN_CONFIG'])
basepath = "/Users/erikbohnsack/data/training"
tracking = pykitti.tracking(base_path=basepath, sequence="0000")
points = tracking.get_velo(0)
bev_map = np.zeros(input_config.bev_shape)
print("BEV map size: {}".format(bev_map.shape))
grid_limits = np.array([[input_config.x_min, input_config.x_max],
[input_config.y_min, input_config.y_max],
[input_config.z_min, input_config.z_max]])
grid_sizes = np.array([
input_config.x_grid_size, input_config.y_grid_size,
input_config.z_grid_size
])
start_jit = time.time()
kitti_dataset.format_to_BEV_jit(bev_map, points, grid_limits,
grid_sizes)
end_jit = time.time()
plot_BEV(bev_map)
print("Elapsed time: {}".format(end_jit - start_jit))
def eval(model_path, data_path):
config_path = 'cfg/cfg_mini.yml'
config = load_config(config_path)
input_config = InputConfig(config['INPUT_CONFIG'])
eval_config = EvalConfig(config['EVAL_CONFIG'])
model = FafeNet(input_config)
model.load_state_dict(torch.load(model_path)["model_state_dict"])
model.eval()
print("Model loaded")
eval_head = FafePredict(input_config, eval_config)
testing_seqs = [0] # TODO: Fix
testing_datasets = [
TestDataset(input_config,
root_dir=data_path,
split='testing',
sequence=seq) for seq in testing_seqs
]
testing_dataset = ConcatDataset(testing_datasets)
testing_loader = DataLoader(dataset=testing_dataset,
batch_size=eval_config.batch_size,
num_workers=eval_config.num_workers)
for batch_index, batch in enumerate(testing_loader):
input, info = batch
out_det, out_reg = model(input)
inference = eval_head(out_det, out_reg)
def test_concat_dataset(self):
config_path = 'cfg_mini.yml'
config = load_config(config_path)
input_config = InputConfig(config['INPUT_CONFIG'])
train_config = TrainConfig(config['TRAIN_CONFIG'])
if os.path.exists("/Users/erikbohnsack/data/"):
root_dir = "/Users/erikbohnsack/data/"
sequence = 0
else:
root_dir = "/home/mlt/data/"
sequence = 3
training_datasets = [
TemporalBEVsDataset(input_config,
root_dir,
split='training',
sequence=seq)
for seq in train_config.training_seqs
]
validation_datasets = [
TemporalBEVsDataset(input_config,
root_dir,
split='training',
sequence=seq)
for seq in train_config.validation_seqs
] # Split still training due to data structure
training_dataset = ConcatDataset(training_datasets)
validation_dataset = ConcatDataset(validation_datasets)
training_dataloader = DataLoader(training_dataset,
batch_size=train_config.batch_size,
shuffle=train_config.shuffle,
num_workers=train_config.num_workers)
validation_dataloader = DataLoader(
validation_dataset,
batch_size=train_config.batch_size,
shuffle=train_config.shuffle,
num_workers=train_config.num_workers)
print("----------------------- TRAINING -----------------------")
for i_batch, batch_sample in enumerate(training_dataloader):
print(i_batch)
input, target, info = batch_sample
print("Input shape: {}".format(input.shape))
print("Sequence: {}, real index: {}".format(
info["sequence"], info["Current_index"]))
print("----------------------- VALIDATION -----------------------")
for i_batch, batch_sample in enumerate(validation_dataloader):
print(i_batch)
input, target, info = batch_sample
print("Input shape: {}".format(input.shape))
print("Sequence: {}, real index: {}".format(
info["sequence"], info["Current_index"]))
def test_eval(self):
save_filename = 'foo'
config_path = 'cfg.yml'
config = load_config(config_path)
input_config = InputConfig(config['INPUT_CONFIG'])
model = FafeNet(config)
torch.save({'model_state_dict': model.state_dict()}, save_filename)
model_path = save_filename
data_path = '/Users/erikbohnsack/data'
eval(model_path=model_path, data_path=data_path)
def test_train_with_model(self):
root_dir = get_root_dir()
if os.path.exists(
'/Users/erikbohnsack/Code/MOT/fafe/trained_models/weights_2019-04-12_13_43_epoch_85'
):
model_path = '/Users/erikbohnsack/Code/MOT/fafe/trained_models/weights_2019-04-12_13_43_epoch_85'
config_path = 'cfg/cfg_mac.yml'
else:
model_path = '/home/mlt/mot/fafe/trained_models/test_train_2019-04-11_14_01_epoch_40'
config_path = 'cfg/cfg_mini.yml'
config = load_config(config_path)
input_config = InputConfig(config['INPUT_CONFIG'])
train_config = TrainConfig(config['TRAIN_CONFIG'])
verbose = train_config.verbose
net = FafeNet(input_config)
net.load_state_dict(
torch.load(
model_path,
map_location=lambda storage, loc: storage)["model_state_dict"])
#########################
# Set which device run on
#########################
if train_config.use_cuda:
if train_config.cuda_device == 0:
device = torch.device("cuda:0")
print('Using CUDA:{}\n'.format(0))
elif train_config.cuda_device == 1:
device = torch.device("cuda:1")
print('Using CUDA:{}\n'.format(1))
else:
print(
'Functionality for CUDA device cuda:{} not yet implemented.'
.format(train_config.cuda_device))
print('Using cuda:0 instead...\n')
device = torch.device("cuda:0")
net = net.to(device)
else:
device = torch.device("cpu")
print('Using CPU\n')
#########################
# Define optimizer
#########################
optimizer = optim.Adam(net.parameters(),
lr=train_config.learning_rate,
weight_decay=train_config.weight_decay)
print('Adams Optimizer set up with\n\tlr = {}\n\twd = {}\n'.format(
train_config.learning_rate, train_config.weight_decay))
loss_func = FafeLoss(input_config, train_config, device)
training_datasets = [
TemporalBEVsDataset(input_config,
root_dir,
split='training',
sequence=seq)
for seq in train_config.training_seqs
]
validation_datasets = [
TemporalBEVsDataset(input_config,
root_dir,
split='training',
sequence=seq)
for seq in train_config.validation_seqs
] # Split still training due to data structure
training_dataset = ConcatDataset(training_datasets)
validation_dataset = ConcatDataset(validation_datasets)
training_dataloader = DataLoader(training_dataset,
batch_size=train_config.batch_size,
shuffle=train_config.shuffle,
num_workers=train_config.num_workers)
validation_dataloader = DataLoader(
validation_dataset,
batch_size=train_config.batch_size,
shuffle=train_config.shuffle,
num_workers=train_config.num_workers)
###############################
# Start training and evaluation
###############################
print('\nTraining initiated [' + strftime("%Y-%m-%d %H:%M") + ']')
for epoch in range(train_config.max_epochs):
train_mean_loss, train_mean_recall, train_mean_precision, train_num_samples = 0, 0, 0, 0
eval_mean_loss, eval_mean_recall, eval_mean_precision, eval_num_samples = 0, 0, 0, 0
train_scaled_L1_mean, train_classification_loss = 0, 0
eval_scaled_L1_mean, eval_classification_loss = 0, 0
#########################
# TRAINING
#########################
tic = time()
net.train()
for i_batch, sample_batched in enumerate(training_dataloader):
input, target, _ = sample_batched
if train_config.use_cuda:
input = input.to(device)
target = target.to(device)
# Always reset optimizer's gradient each iteration
optimizer.zero_grad()
if verbose:
print('{} i: {} {}'.format('~' * 10, i_batch, '~' * 10))
print('Input shape: {}'.format(input.shape))
print('Target shape: {}'.format(target.shape))
print('Target: {}'.format(target[0][0][0][0]))
# Forward propagation
out_detection, out_regression = net.forward(input)
if verbose:
print('Output shape det: {}'.format(out_detection.shape))
print('Output shape reg: {}'.format(out_regression.shape))
# Calculate the loss
loss, recall, precision, scaled_L1, scaled_euler, classification_loss = loss_func(
out_detection, out_regression, target, verbose)
# Back propagate
loss.backward()
# Update the weights
optimizer.step()
train_mean_loss += loss
train_mean_recall += recall
train_mean_precision += precision
train_scaled_L1_mean += scaled_L1
train_classification_loss += classification_loss
train_num_samples += 1
# Calculate the actual averages
train_mean_loss /= train_num_samples
train_mean_recall /= train_num_samples
train_mean_precision /= train_num_samples
train_scaled_L1_mean /= train_num_samples
train_classification_loss /= train_num_samples
training_time = time() - tic
def train():
if os.path.exists('/home/mlt/mot/fafe/cfg/adams_computer'):
config_path = 'cfg/cfg_pp_mini.yml'
elif os.path.exists('/Users/erikbohnsack'):
config_path = 'cfg/cfg_mac.yml'
else:
config_path = 'cfg/cfg_pp.yml'
print('Using config: \n\t{}\n'.format(config_path))
config = load_config(config_path)
input_config = InputConfig(config["INPUT_CONFIG"])
train_config = TrainConfig(config["TRAIN_CONFIG"])
loss_config = LossConfig(config['LOSS_CONFIG'])
model_config = ModelConfig(config['MODEL_CONFIG'])
verbose = train_config.verbose
time_str = strftime("%Y-%m-%d_%H-%M")
weights_filename = 'trained_models/' + time_str + '/weights_' + time_str
showroom_path = get_showroom_path(model_path="_".join(
('weights', time_str)),
full_path_bool=False)
if not os.path.exists('trained_models/' + time_str):
os.mkdir('trained_models/' + time_str)
print(
'Training weights will be saved to:\n\t{}\n'.format(weights_filename))
config_filename = 'trained_models/' + time_str + '/config_' + time_str + '.yml'
save_config(config_filename, config)
print('Config file saved to:\n\t{}\n'.format(config_filename))
if train_config.use_visdom:
print(
'Dont forget to run "visdom" in a terminal in parallel to this in order to start the Visdom server'
)
print('Choose port with "python -m visdom.server -port {}" \n'.format(
train_config.visdom_port))
vis = visdom.Visdom(
port=train_config.visdom_port) # port 8097 is default
loss_window, sub_loss_window, recall_window, precision_window = viz.get_windows(
vis, time_str)
print("~" * 20)
print("Setting up net")
pillar = PillarOfFafe(input_config=input_config,
batch_size=train_config.batch_size,
verbose=input_config.pp_verbose)
if model_config.model == 'little_fafe':
fafe = LittleFafe(input_config=input_config)
else:
fafe = FafeNet(input_config=input_config)
#########################
# Set which device run on
#########################
if train_config.use_cuda:
# "If you load your samples in the Dataset on CPU and would like to push it during training to the GPU,
# you can speed up the host to device transfer by enabling pin_memory."
# - ptrblck [https://discuss.pytorch.org/t/when-to-set-pin-memory-to-true/19723]
pin_memory = False
device = torch.device("cuda:" + str(train_config.cuda_device))
print('\nUsing device {}\n'.format(device))
pillar = pillar.to(device)
fafe = fafe.to(device)
else:
pin_memory = False
device = torch.device("cpu")
print('Using CPU\n')
loss_func = FafeLoss(input_config, train_config, loss_config, device)
if train_config.use_cuda:
loss_func = loss_func.to(device)
print("Net set up successfully!")
pp_pytorch_total_params = sum(p.numel() for p in pillar.parameters())
pp_pytorch_trainable_params = sum(p.numel() for p in pillar.parameters()
if p.requires_grad)
print("~" * 20)
print(
"PP:\n\tNumber of parameters: {}\n\tNumber of trainable parameters: {}"
.format(pp_pytorch_total_params, pp_pytorch_trainable_params))
pytorch_total_params = sum(p.numel() for p in fafe.parameters())
pytorch_trainable_params = sum(p.numel() for p in fafe.parameters()
if p.requires_grad)
print(
"FAFE:\n\tNumber of parameters: {}\n\tNumber of trainable parameters: {}"
.format(pytorch_total_params, pytorch_trainable_params))
root_dir = get_root_dir()
#########################
# Define optimizer
#########################
params = list(pillar.parameters()) + list(fafe.parameters()) + list(
loss_func.parameters())
optimizer = optim.Adam(params,
lr=train_config.learning_rate,
weight_decay=train_config.weight_decay)
print('Adams Optimizer set up with\n\tlr = {}\n\twd = {}\n'.format(
train_config.learning_rate, train_config.weight_decay))
#########################
# Get Datasets
#########################
print('Training Data:')
# fafe_sampler = FafeSampler(data_source=dataset, input_config=input_config)
# TODO: Sampler needs a *data_source* as input to know the length of objects it can iterate over.
# TODO: When concatenating
training_dataloader = DataLoader(ConcatDataset([
VoxelDataset(input_config, root_dir, split='training', sequence=seq)
for seq in train_config.training_seqs
]),
batch_size=train_config.batch_size,
shuffle=train_config.shuffle,
num_workers=train_config.num_workers,
pin_memory=pin_memory)
print('Validation Data:')
validation_dataloader = DataLoader(ConcatDataset([
VoxelDataset(input_config, root_dir, split='training', sequence=seq)
for seq in train_config.validation_seqs
]),
batch_size=train_config.batch_size,
shuffle=train_config.shuffle,
num_workers=train_config.num_workers,
pin_memory=pin_memory)
print('Data Loaders set up with:\n\tBatch size: {}\n\tNum Workers: {}'.
format(train_config.batch_size, train_config.num_workers))
###############################
# Start training and evaluation
###############################
print('\nTraining initiated [' + strftime("%Y-%m-%d %H:%M") + ']')
for epoch in range(train_config.max_epochs):
train_mean_loss, train_mean_recall, train_mean_precision, train_num_samples = 0, 0, 0, 0
eval_mean_loss, eval_mean_recall, eval_mean_precision, eval_num_samples = 0, 0, 0, 0
train_scaled_reg_mean, train_scaled_euler_mean, train_classification_loss = 0, 0, 0
eval_scaled_reg_mean, eval_scaled_euler_mean, eval_classification_loss = 0, 0, 0
#########################
# TRAINING
#########################
tic = time()
pillar.train()
fafe.train()
torch.set_grad_enabled(True)
for i_batch, batch in enumerate(training_dataloader):
# Always reset optimizer's gradient each iteration
optimizer.zero_grad()
# Create Pillar Pseudo Img
voxel_stack, coord_stack, num_points_stack, num_nonempty_voxels, target, info = batch
# Move all input data to the correct device if not using CPU
if train_config.use_cuda:
voxel_stack = voxel_stack.to(device)
coord_stack = coord_stack.to(device)
num_points_stack = num_points_stack.to(device)
num_nonempty_voxels = num_nonempty_voxels.to(device)
target = target.to(device)
pseudo_stack = []
for time_iter in range(input_config.num_conseq_frames):
if input_config.pp_verbose:
print("~" * 20)
print("time iter: {}".format(time_iter))
print("voxels \n\tshape: {}".format(
voxel_stack[:, time_iter].shape))
print("coord \n\tshape: {}".format(
coord_stack[:, time_iter].shape))
print("num_points \n\tshape: {}".format(
num_points_stack[:, time_iter].shape))
print("num_nonempty_voxels \n\tshape: {}".format(
num_nonempty_voxels[:, time_iter].shape))
pseudo_image = pillar(voxel_stack[:, time_iter],
num_points_stack[:, time_iter],
coord_stack[:, time_iter],
num_nonempty_voxels[:, time_iter])
if input_config.pp_verbose:
print("Pseudo_img: {}".format(
pseudo_image.unsqueeze(1).shape))
pseudo_stack.append(pseudo_image.unsqueeze(1))
pseudo_torch = torch.cat(pseudo_stack, dim=1)
if input_config.pp_verbose:
print("Pseudo stacked over time: \n\t{}".format(
pseudo_torch.shape))
if train_config.use_cuda:
pseudo_torch = pseudo_torch.to(device)
target = target.to(device)
# Forward propagation. Reshape by squishing together Time and Channel dimensions.
# Reshaping basically as we do with BEV, stacking them on top of eachother.
out_detection, out_regression = fafe.forward(
pseudo_torch.reshape(
-1, pseudo_torch.shape[1] * pseudo_torch.shape[2],
pseudo_torch.shape[-2], pseudo_torch.shape[-1]))
# Calculate the loss
loss, recall, precision, scaled_reg, scaled_euler, classification_loss = loss_func(
out_detection, out_regression, target, verbose)
# Back propagate
loss.backward()
if train_config.plot_grad_flow:
plot_grad_flow(
pillar.named_parameters(),
os.path.join(
showroom_path, 'grad_flow_pillar', "".join(
("epoch_", str(epoch).zfill(4), "_batch_",
str(i_batch).zfill(4), ".png"))))
plot_grad_flow(
fafe.named_parameters(),
os.path.join(
showroom_path, 'grad_flow_fafe', "".join(
("epoch_", str(epoch).zfill(4), "_batch_",
str(i_batch).zfill(4), ".png"))))
# Update the weights
optimizer.step()
train_mean_loss += loss
train_mean_recall += recall
train_mean_precision += precision
train_scaled_reg_mean += scaled_reg
train_scaled_euler_mean += scaled_euler
train_classification_loss += classification_loss
train_num_samples += 1
# Calculate the actual averages
train_mean_loss /= train_num_samples
train_mean_recall /= train_num_samples
train_mean_precision /= train_num_samples
train_scaled_reg_mean /= train_num_samples
train_scaled_euler_mean /= train_num_samples
train_classification_loss /= train_num_samples
training_time = time() - tic
#########################
# EVALUATION
#########################
tic2 = time()
pillar.eval()
fafe.eval()
with torch.no_grad():
for i_batch, batch in enumerate(validation_dataloader):
# Create Pillar Pseudo Img
voxel_stack, coord_stack, num_points_stack, num_nonempty_voxels, target, index = batch
# Move all input data to the correct device if not using CPU
if train_config.use_cuda:
voxel_stack = voxel_stack.to(device)
coord_stack = coord_stack.to(device)
num_points_stack = num_points_stack.to(device)
num_nonempty_voxels = num_nonempty_voxels.to(device)
target = target.to(device)
pseudo_stack = []
for time_iter in range(input_config.num_conseq_frames):
if train_config.verbose:
print("~" * 20)
print("time iter: {}".format(time_iter))
print("voxels \n\tshape: {}".format(
voxel_stack[:, time_iter].shape))
print("coord \n\tshape: {}".format(
coord_stack[:, time_iter].shape))
print("num_points \n\tshape: {}".format(
num_points_stack[:, time_iter].shape))
print("num_nonempty_voxels \n\tshape: {}".format(
num_nonempty_voxels[:, time_iter].shape))
pseudo_image = pillar(voxel_stack[:, time_iter],
num_points_stack[:, time_iter],
coord_stack[:, time_iter],
num_nonempty_voxels[:, time_iter])
if train_config.verbose:
print("Pseudo_img: {}".format(
pseudo_image.unsqueeze(1).shape))
pseudo_stack.append(pseudo_image.unsqueeze(1))
pseudo_torch = torch.cat(pseudo_stack, dim=1)
if train_config.use_cuda:
pseudo_torch = pseudo_torch.to(device)
target = target.to(device)
# Forward propagation
out_detection, out_regression = fafe.forward(
pseudo_torch.reshape(
-1, pseudo_torch.shape[1] * pseudo_torch.shape[2],
pseudo_torch.shape[-2], pseudo_torch.shape[-1]))
# Calculate the loss
loss, recall, precision, scaled_reg, scaled_euler, classification_loss = loss_func(
out_detection, out_regression, target, verbose)
eval_mean_loss += loss
eval_mean_recall += recall
eval_mean_precision += precision
eval_scaled_reg_mean += scaled_reg
eval_scaled_euler_mean += scaled_euler
eval_classification_loss += classification_loss
eval_num_samples += 1
eval_mean_loss /= eval_num_samples
eval_mean_recall /= eval_num_samples
eval_mean_precision /= eval_num_samples
eval_scaled_reg_mean /= eval_num_samples
eval_scaled_euler_mean /= eval_num_samples
eval_classification_loss /= eval_num_samples
eval_time = time() - tic2
total_time = time() - tic
#########################
# PRINT STUFF ON SCREEN
#########################
print('\nEpoch {} / {}\n{}\nCurrent time: {}'.format(
epoch, train_config.max_epochs - 1, '-' * 12,
strftime("%Y-%m-%d %H:%M")))
print('Epoch Total Time: {} s ({} + {})'.format(
round(total_time, 2), round(training_time, 2), round(eval_time,
2)))
print('Next Epoch ETA: ' +
format(datetime.now() +
timedelta(seconds=total_time), '%Y-%m-%d %H:%M'))
print('Training ETA: ' + format(
datetime.now() + timedelta(
seconds=total_time *
(train_config.max_epochs - epoch - 1)), '%Y-%m-%d %H:%M'))
print('Train\n\tLoss: \t\t{}'
'\n\t\tL1: \t{}'
'\n\t\tEuler:\t{}'
'\n\t\tCL: \t{}'
'\n\tRecall: \t{}'
'\n\tPrecision:\t{}'.format(train_mean_loss,
train_scaled_reg_mean,
train_scaled_euler_mean,
train_classification_loss,
train_mean_recall,
train_mean_precision))
print('Validation\n\tLoss: \t\t{}'
'\n\t\tL1: \t{}'
'\n\t\tEuler:\t{}'
'\n\t\tCL: \t{}'
'\n\tRecall: \t{}'
'\n\tPrecision:\t{}'.format(eval_mean_loss, eval_scaled_reg_mean,
eval_scaled_euler_mean,
eval_classification_loss,
eval_mean_recall,
eval_mean_precision))
if train_config.use_visdom:
# Visualize Loss
viz.push_data(epoch, vis, loss_window, sub_loss_window,
recall_window, precision_window, train_mean_loss,
eval_mean_loss, train_scaled_reg_mean,
train_classification_loss, train_scaled_euler_mean,
eval_scaled_euler_mean, eval_scaled_reg_mean,
eval_classification_loss, train_mean_recall,
eval_mean_recall, train_mean_precision,
eval_mean_precision)
####################################################
# SAVE WEIGHTS (every save_weights_modulus th epoch)
####################################################
if epoch % train_config.save_weights_modulus == 0 or epoch == train_config.max_epochs - 1:
save_filename = weights_filename + '_epoch_' + str(epoch)
pp_fn = save_filename + '_pp'
torch.save(
{
'epoch': epoch,
'model_state_dict': pillar.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'loss': loss
}, pp_fn)
fafe_fn = save_filename + '_fafe'
torch.save(
{
'epoch': epoch,
'model_state_dict': fafe.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'loss': loss
}, fafe_fn)
print('Training Complete [' + strftime("%Y-%m-%d %H:%M") + ']')
def eval_with_GT(model_path, data_path, config_path, v2=False):
timestr = strftime("%Y-%m-%d_%H:%M")
showroom_path = get_showroom_path(model_path, full_path_bool=True)
print('Images will be saved to:\n\t{}'.format(showroom_path))
velo_path = os.path.join(data_path, 'training', 'velodyne')
config = load_config(config_path)
start_time = time()
input_config = InputConfig(config['INPUT_CONFIG'])
eval_config = EvalConfig(config['EVAL_CONFIG'])
if v2:
model = HaveFafe4Eval(input_config)
else:
model = FafeNet(input_config)
if eval_config.use_cuda:
if eval_config.cuda_device == 0:
device = torch.device("cuda:0")
print('Using CUDA:{}\n'.format(0))
elif eval_config.cuda_device == 1:
device = torch.device("cuda:1")
print('Using CUDA:{}\n'.format(1))
else:
print('Functionality for CUDA device cuda:{} not yet implemented.'.
format(eval_config.cuda_device))
print('Using cuda:0 instead...\n')
device = torch.device("cuda:0")
model = model.to(device)
else:
device = torch.device("cpu")
print('Using CPU\n')
imu_path = os.path.join(data_path, 'training', 'oxts')
model.load_state_dict(
torch.load(
model_path,
map_location=lambda storage, loc: storage)["model_state_dict"])
model.eval()
print("Model loaded, loading time: {}".format(round(
time() - start_time, 2)))
timestr = strftime("%Y-%m-%d_%H:%M")
eval_head = FafePredict(input_config, eval_config)
eval_head.eval()
print("Evalutation Model Loaded!")
testing_datasets = [
TemporalBEVsDataset(input_config,
root_dir=data_path,
split='training',
sequence=seq)
for seq in eval_config.validation_seqs
]
testing_dataset = ConcatDataset(testing_datasets)
testing_loader = DataLoader(dataset=testing_dataset,
batch_size=eval_config.batch_size,
num_workers=eval_config.num_workers,
shuffle=False)
print("Starting to iterate over batches")
if eval_config.use_gospa:
gospa_scores_dict = {}
for batch_idx, batch in enumerate(testing_loader):
input, target, info = batch
if eval_config.use_cuda:
input = input.to(device)
target = target.to(device)
timeit = time()
with torch.no_grad():
out_det, out_reg = model(input)
inference = eval_head(out_det, out_reg)
print("\nTime to propagate through network: {}".format(
round(time() - timeit, 2)))
nB = target.shape[0]
nT = input_config.num_conseq_frames
tic = time()
for batch_index in range(nB):
current_sequence = info["sequence"][batch_index]
imud = load_imu(imu_path, current_sequence)
print("Batch: {} \t| Seq: {}".format(
batch_index, info["sequence"][batch_index]),
end='')
if eval_config.save_figs:
frame = info["GT_indices"][0]
file = os.path.join(velo_path, info["sequence"][batch_index],
str(frame.item()).zfill(6) + '.bin')
pc = np.fromfile(file, dtype=np.float32).reshape((-1, 4))
fig = draw_lidar(
pc, off_screen_rendering=eval_config.off_screen_rendering)
if eval_config.use_gospa:
current_frame = info["Current_index"][batch_index].item()
gospa_scores_dict[int(current_sequence),
current_frame] = np.zeros(nT)
for time_index in range(nT):
# Get ground truths
non_zero_gt_mask = target[batch_index][time_index].sum(
dim=1) != 0
non_zero_gt = target[batch_index][time_index][non_zero_gt_mask]
# Transform gt to 2D bbox x1y1x2y2 form
point_form_gt = point_form_fafe(non_zero_gt)
# Transform gt to 3D bbox form
point_form_gt_3d = point_form_3d(point_form_gt,
input_config.z_center,
input_config.z_height,
device=device)
# Rotate gt 3D bboxes
gt_center = center_size_3d(non_zero_gt, input_config.z_center,
device)
point_form_gt_3d_rot = rotate_3d_bbx(point_form_gt_3d,
non_zero_gt[:, 4],
gt_center, device)
# Get inference
inference_reg = inference[batch_index][time_index]
# Calculate GOSPA if ordered to do so
if eval_config.use_gospa:
if inference_reg.is_cuda:
gospa_gt = non_zero_gt[:, 0:2].cpu()
gospa_inf = inference_reg[:, 0:2].cpu()
else:
gospa_gt = non_zero_gt[:, 0:2]
gospa_inf = inference_reg[:, 0:2]
gospa_score = GOSPA(gospa_gt, gospa_inf)
gospa_scores_dict[int(current_sequence),
current_frame][time_index] = gospa_score
if eval_config.save_figs and eval_config.draw_gospa:
fig = draw_gospa(gospa_score, fig, time_index)
# Get probabilities of objects
inference_probabilities = inference_reg[
..., -1] if eval_config.show_confidence else None
# Transform inference to 2D bbox x1y1x2y2
inf_points_2d = point_form_fafe(inference_reg)
# Transform inference to 3D bbox
inference_points = point_form_3d(inf_points_2d,
input_config.z_center,
input_config.z_height, device)
# Rotate inference 3D bboxes
inference_center = center_size_3d(inference_reg,
input_config.z_center,
device)
inference_points_rot = rotate_3d_bbx(inference_points,
inference_reg[:, 4],
inference_center, device)
# Translate center points according to time_index
translated_gt_c = translate_center(gt_center,
imu_data=imud[frame.item()],
timestep=time_index,
dt=input_config.dt,
device=device)
translated_infer_c = translate_center(
inference_center,
imu_data=imud[frame.item()],
timestep=time_index,
dt=input_config.dt,
device=device)
# TODO: Move these if-statements further up to reduce computation
if eval_config.save_figs:
if time_index == 0:
fig = draw_gt_boxes3d(point_form_gt_3d_rot,
fig,
color=eval_config.gt_color)
fig = draw_gt_boxes3d(
inference_points_rot,
fig,
color=eval_config.infer_color,
probabilities=inference_probabilities)
else:
fig = draw_points_3d(translated_gt_c,
fig,
color=eval_config.gt_color)
fig = draw_points_3d(translated_infer_c,
fig,
color=eval_config.infer_color)
if eval_config.save_figs:
filename = "_".join(
("Infer", "seq", info["sequence"][batch_index], "f",
str(frame.item()), timestr + ".png"))
filepath = os.path.join(showroom_path, 'oracle_view', filename)
filename_top = "_".join(
("Top_Infer", "seq", info["sequence"][batch_index], "f",
str(frame.item()), timestr + ".png"))
filepath_top = os.path.join(showroom_path, 'top_view',
filename_top)
mayavi.mlab.view(azimuth=90,
elevation=0,
focalpoint=[24., 0, 0],
distance=120.0,
figure=fig)
mayavi.mlab.savefig(filepath_top, figure=fig)
mayavi.mlab.view(
azimuth=180,
elevation=70,
focalpoint=[12.0909996, -1.04700089, -2.03249991],
distance=62.0,
figure=fig)
mayavi.mlab.savefig(filepath, figure=fig)
print('\t | time: {}'.format(round(time() - tic, 2)))
print(gospa_scores_dict)
def eval_with_GT(fafe_model_path, pp_model_path, data_path, config_path):
timestr = strftime("%Y-%m-%d_%H:%M")
showroom_path = get_showroom_path(fafe_model_path, full_path_bool=True)
print('Images will be saved to:\n\t{}'.format(showroom_path))
velo_path = os.path.join(data_path, 'training', 'velodyne')
config = load_config(config_path)
start_time = time()
input_config = InputConfig(config['INPUT_CONFIG'])
train_config = TrainConfig(config['TRAIN_CONFIG'])
eval_config = EvalConfig(config['EVAL_CONFIG'])
model_config = ModelConfig(config['MODEL_CONFIG'])
post_config = PostConfig(config['POST_CONFIG'])
pillar = PillarOfFafe(input_config, train_config.batch_size, train_config.verbose)
if model_config.model == 'little_fafe':
fafe = LittleFafe(input_config)
else:
fafe = FafeNet(input_config)
if eval_config.use_cuda:
if eval_config.cuda_device == 0:
device = torch.device("cuda:0")
print('Using CUDA:{}\n'.format(0))
elif eval_config.cuda_device == 1:
device = torch.device("cuda:1")
print('Using CUDA:{}\n'.format(1))
else:
print('Functionality for CUDA device cuda:{} not yet implemented.'.format(eval_config.cuda_device))
print('Using cuda:0 instead...\n')
device = torch.device("cuda:0")
pillar = pillar.to(device)
fafe = fafe.to(device)
else:
device = torch.device("cpu")
print('Using CPU\n')
imu_path = os.path.join(data_path, 'training', 'oxts')
pillar.load_state_dict(torch.load(pp_model_path, map_location=lambda storage, loc: storage)["model_state_dict"])
fafe.load_state_dict(torch.load(fafe_model_path, map_location=lambda storage, loc: storage)["model_state_dict"])
pillar.eval()
fafe.eval()
print("Model loaded, loading time: {}".format(round(time() - start_time, 2)))
timestr = strftime("%Y-%m-%d_%H:%M")
eval_head = FafePredict(input_config, eval_config)
eval_head.eval()
print("Evalutation Model Loaded!")
testing_datasets = [VoxelDataset(input_config, root_dir=data_path, split='training', sequence=seq) for seq in
eval_config.validation_seqs]
testing_dataset = ConcatDataset(testing_datasets)
testing_loader = DataLoader(dataset=testing_dataset,
batch_size=eval_config.batch_size,
num_workers=eval_config.num_workers,
shuffle=False)
print("Starting to iterate over batches")
if eval_config.use_gospa:
gospa_scores_dict = {}
for batch_idx, batch in enumerate(testing_loader):
# Create Pillar Pseudo Img
voxel_stack, coord_stack, num_points_stack, num_nonempty_voxels, target, info = batch
# Move all input data to the correct device if not using CPU
if train_config.use_cuda:
voxel_stack = voxel_stack.to(device)
coord_stack = coord_stack.to(device)
num_points_stack = num_points_stack.to(device)
num_nonempty_voxels = num_nonempty_voxels.to(device)
target = target.to(device)
timeit = time()
with torch.no_grad():
pseudo_stack = []
for time_iter in range(input_config.num_conseq_frames):
pseudo_image = pillar(voxel_stack[:, time_iter], num_points_stack[:, time_iter],
coord_stack[:, time_iter], num_nonempty_voxels[:, time_iter])
if input_config.pp_verbose:
print("Pseudo_img: {}".format(pseudo_image.unsqueeze(1).shape))
pseudo_stack.append(pseudo_image.unsqueeze(1))
pseudo_torch = torch.cat(pseudo_stack, dim=1)
if train_config.use_cuda:
pseudo_torch = pseudo_torch.to(device)
target = target.to(device)
# Forward propagation. Reshape by squishing together Time and Channel dimensions.
# Reshaping basically as we do with BEV, stacking them on top of eachother.
out_detection, out_regression = fafe.forward(
pseudo_torch.reshape(-1, pseudo_torch.shape[1] * pseudo_torch.shape[2], pseudo_torch.shape[-2],
pseudo_torch.shape[-1]))
inference = eval_head(out_detection, out_regression)
print("\nTime to propagate through network: {}".format(round(time() - timeit, 2)))
nB = target.shape[0]
nT = input_config.num_conseq_frames
tic = time()
for batch_index in range(nB):
current_sequence = info["sequence"][batch_index]
frame = info["GT_indices"][0]
if torch.is_tensor(current_sequence):
current_sequence = current_sequence.item()
imud = load_imu(imu_path, current_sequence)
print("Batch: {} \t| Seq: {} Frame: {}".format(batch_index, current_sequence, frame.item()), end='')
if eval_config.save_raw:
if not os.path.exists("showroom/detections_" +str(current_sequence).zfill(4)):
os.mkdir("showroom/detections_" +str(current_sequence).zfill(4))
filename = os.path.join("showroom", "detections_{}".format(str(current_sequence).zfill(4)), str(int(frame)).zfill(4) + ".pt")
torch.save(inference, filename)
if eval_config.save_figs:
file = os.path.join(velo_path, str(current_sequence).zfill(4), str(frame.item()).zfill(6) + '.bin')
pc = np.fromfile(file, dtype=np.float32).reshape((-1, 4))
fig = draw_lidar(pc, off_screen_rendering=eval_config.off_screen_rendering)
if eval_config.use_gospa:
current_frame = info["Current_index"][batch_index].item()
gospa_scores_dict[int(current_sequence), current_frame] = np.zeros(nT)
for time_index in range(nT):
# Get ground truths
non_zero_gt_mask = target[batch_index][time_index].sum(dim=1) != 0
non_zero_gt = target[batch_index][time_index][non_zero_gt_mask]
# Transform gt to 2D bbox x1y1x2y2 form
point_form_gt = point_form_fafe(non_zero_gt)
# Transform gt to 3D bbox form
point_form_gt_3d = point_form_3d(point_form_gt, input_config.z_center, input_config.z_height,
device=device)
# Rotate gt 3D bboxes
gt_center = center_size_3d(non_zero_gt, input_config.z_center, device)
point_form_gt_3d_rot = rotate_3d_bbx(point_form_gt_3d, non_zero_gt[:, 4], gt_center, device)
# Get inference
inference_reg = inference[batch_index][time_index]
# Filter out if outside FOV
filtered_inference_reg = []
for inf_reg in inference_reg:
np_inf_reg = inf_reg.cpu().data.numpy()
if within_fov(np_inf_reg[0:2], min_angle=0.78, max_angle=2.45, max_radius=100):
filtered_inference_reg.append(np_inf_reg)
# Filter out if too closely located
filtered_inference_reg = too_close(filtered_inference_reg, eval_config.distance_threshold)
if eval_config.save_detections_as_measurements and time_index == 0:
for fir in filtered_inference_reg:
file1 = open(showroom_path + "/detections_" + str(current_sequence).zfill(4) + ".txt", "a")
# Save it on the format PMBM wants it... p = [x, y, rot] with x defined right wards and y forward
file1.write(str(frame.item()) + ' Car:' + str(fir[5]) + ' ' + str(- fir[1]) + ' ' + str(
fir[0]) + ' ' + str(fir[4]) + "\n")
file1.close()
# Make filtered_inference_reg numpy because GOSPA needs it to be
filtered_inference_reg = np.array(filtered_inference_reg)
# Calculate GOSPA if ordered to do so
if eval_config.use_gospa:
if inference_reg.is_cuda:
gospa_gt = non_zero_gt[:, 0:2].cpu()
# gospa_inf = inference_reg[:, 0:2].cpu()
gospa_inf = filtered_inference_reg[:, 0:2] if len(filtered_inference_reg) > 0 else np.array([])
else:
gospa_gt = non_zero_gt[:, 0:2]
# gospa_inf = inference_reg[:, 0:2]
gospa_inf = filtered_inference_reg[:, 0:2]
gospa_score = GOSPA(gospa_gt, gospa_inf)
gospa_scores_dict[int(current_sequence), current_frame][time_index] = gospa_score
if eval_config.save_figs and eval_config.draw_gospa:
fig = draw_gospa(gospa_score, fig, time_index)
# Make filtered_inference_reg torch tensor because point forms need it to be
filtered_inference_reg = torch.from_numpy(filtered_inference_reg).to(device)
# If no outputs -> plot ev. gt and continue
if len(filtered_inference_reg) == 0:
if time_index == 0:
fig = draw_gt_boxes3d(point_form_gt_3d_rot, fig, color=eval_config.gt_color)
continue
else:
# Translate center points according to time_index
translated_gt_c = translate_center(gt_center, imu_data=imud[frame.item()], timestep=time_index,
dt=input_config.dt, device=device)
fig = draw_points_3d(translated_gt_c, fig, color=eval_config.gt_color)
continue
# Get probabilities of objects
inference_probabilities = filtered_inference_reg[..., -1] if eval_config.show_confidence else None
# Transform inference to 2D bbox x1y1x2y2
inf_points_2d = point_form_fafe(filtered_inference_reg)
# Transform inference to 3D bbox
inference_points = point_form_3d(inf_points_2d, input_config.z_center, input_config.z_height, device)
# Rotate inference 3D bboxes
inference_center = center_size_3d(filtered_inference_reg, input_config.z_center, device)
inference_points_rot = rotate_3d_bbx(inference_points, filtered_inference_reg[:, 4], inference_center,
device)
# Translate center points according to time_index
translated_gt_c = translate_center(gt_center, imu_data=imud[frame.item()], timestep=time_index,
dt=input_config.dt, device=device)
translated_infer_c = translate_center(inference_center, imu_data=imud[frame.item()],
timestep=time_index, dt=input_config.dt, device=device)
# TODO: Move these if-statements further up to reduce computation
if eval_config.save_figs:
if time_index == 0:
fig = draw_gt_boxes3d(point_form_gt_3d_rot, fig, color=eval_config.gt_color)
fig = draw_gt_boxes3d(inference_points_rot, fig, color=eval_config.infer_color,
probabilities=inference_probabilities)
else:
fig = draw_points_3d(translated_gt_c, fig, color=eval_config.gt_color)
fig = draw_points_3d(translated_infer_c, fig, color=eval_config.infer_color)
if eval_config.save_figs:
filename = "_".join(
("Infer", "seq", str(current_sequence), "f", str(frame.item()), timestr + ".png"))
filepath = os.path.join(showroom_path, 'oracle_view', filename)
filename_top = "_".join(
("Top_Infer", "seq", str(current_sequence), "f", str(frame.item()), timestr + ".png"))
filepath_top = os.path.join(showroom_path, 'top_view', filename_top)
mayavi.mlab.view(azimuth=90, elevation=0, focalpoint=[24., 0, 0],
distance=120.0, figure=fig)
mayavi.mlab.savefig(filepath_top, figure=fig)
mayavi.mlab.view(azimuth=180, elevation=70, focalpoint=[12.0909996, -1.04700089, -2.03249991],
distance=62.0, figure=fig)
mayavi.mlab.savefig(filepath, figure=fig)
if eval_config.use_gospa:
# Save gospa scores in txt file
file2 = open(showroom_path + "/gospa_scores_" + str(current_sequence).zfill(4) + ".txt", "a")
row_str = str(frame.item())
for time_index in range(nT):
row_str += (' ' + str(gospa_scores_dict[int(current_sequence), current_frame][time_index]))
file2.write(row_str + "\n")
file2.close()
print('\t | time: {}'.format(round(time() - tic, 2)))
def train():
if os.path.exists('/home/mlt/mot/fafe/cfg/adams_computer'):
config_path = 'cfg/cfg_mini.yml'
else:
config_path = 'cfg/cfg.yml'
print('Using config: \n\t{}\n'.format(config_path))
config = load_config(config_path)
input_config = InputConfig(config['INPUT_CONFIG'])
train_config = TrainConfig(config['TRAIN_CONFIG'])
loss_config = LossConfig(config['LOSS_CONFIG'])
model_config = ModelConfig(config['MODEL_CONFIG'])
verbose = train_config.verbose
time_str = strftime("%Y-%m-%d_%H-%M")
weights_filename = 'trained_models/' + time_str + '/weights_' + time_str
if not os.path.exists('trained_models/' + time_str):
os.mkdir('trained_models/' + time_str)
print(
'Training weights will be saved to:\n\t{}\n'.format(weights_filename))
config_filename = 'trained_models/' + time_str + '/config_' + time_str + '.yml'
save_config(config_filename, config)
print('Config file saved to:\n\t{}\n'.format(config_filename))
if train_config.use_visdom:
print(
'Dont forget to run "visdom" in a terminal in parallel to this in order to start the Visdom server'
)
print('Choose port with "python -m visdom.server -port {}" \n'.format(
train_config.visdom_port))
vis = visdom.Visdom(
port=train_config.visdom_port) # port 8097 is default
loss_window, sub_loss_window, recall_window, precision_window = viz.get_windows(
vis, time_str)
# Get root directory depending on which computer is running...
# Don't forget to add your own path in 'fafe_utils.config_stuff.get_root_dir'
root_dir = get_root_dir()
#########################
# Define network
#########################
if model_config.model == 'little_fafe':
net = LittleFafe(input_config=input_config)
else:
net = FafeNet(input_config=input_config)
print('Net set up successfully!')
pytorch_total_params = sum(p.numel() for p in net.parameters())
pytorch_trainable_params = sum(p.numel() for p in net.parameters()
if p.requires_grad)
print("\tNumber of parameters: {}\n\tNumber of trainable parameters: {}".
format(pytorch_total_params, pytorch_trainable_params))
# TODO: add posibility to load old weights
#########################
# Set which device run on
#########################
if train_config.use_cuda:
# "If you load your samples in the Dataset on CPU and would like to push it during training to the GPU,
# you can speed up the host to device transfer by enabling pin_memory."
# - ptrblck [https://discuss.pytorch.org/t/when-to-set-pin-memory-to-true/19723]
pin_memory = True
if train_config.multi_gpu:
device_ids = list(range(torch.cuda.device_count()))
device = torch.device("cuda:" + str(device_ids[0]))
net = net.to(device)
net = nn.DataParallel(net)
print(
'\nUsing multiple GPUs.\n\tDevices: {}\n\tOutput device: {}\n'.
format(device_ids, device))
else:
device = torch.device("cuda:" + str(train_config.cuda_device))
print('\nUsing device {}\n'.format(device))
net = net.to(device)
else:
pin_memory = False
device = torch.device("cpu")
print('Using CPU\n')
#########################
# Define loss function
#########################
loss_func = FafeLoss(input_config, train_config, loss_config, device)
if train_config.use_cuda:
loss_func = loss_func.to(device)
if train_config.multi_gpu:
loss_func = nn.DataParallel(loss_func)
#########################
# Define optimizer
#########################
params = list(net.parameters()) + list(loss_func.parameters())
optimizer = optim.Adam(params,
lr=train_config.learning_rate,
weight_decay=train_config.weight_decay)
print('Adams Optimizer set up with\n\tlr = {}\n\twd = {}\n'.format(
train_config.learning_rate, train_config.weight_decay))
#########################
# Get Datasets
#########################
print('Training Data:')
training_dataloader = DataLoader(ConcatDataset([
TemporalBEVsDataset(input_config,
root_dir,
split='training',
sequence=seq) for seq in train_config.training_seqs
]),
batch_size=train_config.batch_size,
shuffle=train_config.shuffle,
num_workers=train_config.num_workers,
pin_memory=pin_memory)
print('Validation Data:')
validation_dataloader = DataLoader(ConcatDataset([
TemporalBEVsDataset(input_config,
root_dir,
split='training',
sequence=seq)
for seq in train_config.validation_seqs
]),
batch_size=train_config.batch_size,
shuffle=train_config.shuffle,
num_workers=train_config.num_workers,
pin_memory=pin_memory)
print('Data Loaders set up with:\n\tBatch size: {}\n\tNum Workers: {}'.
format(train_config.batch_size, train_config.num_workers))
###############################
# Start training and evaluation
###############################
print('\nTraining initiated [' + strftime("%Y-%m-%d %H:%M") + ']')
for epoch in range(train_config.max_epochs):
train_mean_loss, train_mean_recall, train_mean_precision, train_num_samples = 0, 0, 0, 0
eval_mean_loss, eval_mean_recall, eval_mean_precision, eval_num_samples = 0, 0, 0, 0
train_scaled_L1_mean, train_scaled_euler_mean, train_classification_loss = 0, 0, 0
eval_scaled_L1_mean, eval_scaled_euler_mean, eval_classification_loss = 0, 0, 0
#########################
# TRAINING
#########################
tic = time()
net.train()
for i_batch, sample_batched in enumerate(training_dataloader):
input, target, _ = sample_batched
if train_config.use_cuda:
input = input.to(device)
target = target.to(device)
# Always reset optimizer's gradient each iteration
optimizer.zero_grad()
if verbose:
print('{} i: {} {}'.format('~' * 10, i_batch, '~' * 10))
print('Input shape: {}'.format(input.shape))
print('Target shape: {}'.format(target.shape))
print('Target: {}'.format(target[0][0][0][0]))
# Forward propagation
out_detection, out_regression = net.forward(input)
if verbose:
print('Output shape det: {}'.format(out_detection.shape))
print('Output shape reg: {}'.format(out_regression.shape))
# Calculate the loss
loss, recall, precision, scaled_l1, scaled_euler, classification_loss = loss_func(
out_detection, out_regression, target, verbose)
# Back propagate
loss.backward()
# Update the weights
optimizer.step()
train_mean_loss += loss
train_mean_recall += recall
train_mean_precision += precision
train_scaled_L1_mean += scaled_l1
train_scaled_euler_mean += scaled_euler
train_classification_loss += classification_loss
train_num_samples += 1
# Calculate the actual averages
train_mean_loss /= train_num_samples
train_mean_recall /= train_num_samples
train_mean_precision /= train_num_samples
train_scaled_L1_mean /= train_num_samples
train_scaled_euler_mean /= train_num_samples
train_classification_loss /= train_num_samples
training_time = time() - tic
#########################
# EVALUATION
#########################
tic2 = time()
net.eval()
with torch.no_grad():
for i_batch, sample_batched in enumerate(validation_dataloader):
input, target, _ = sample_batched
if train_config.use_cuda:
input = input.to(device)
target = target.to(device)
# Forward propagation
out_detection, out_regression = net.forward(input)
# Calculate the loss
loss, recall, precision, scaled_l1, scaled_euler, classification_loss = loss_func(
out_detection, out_regression, target, verbose)
eval_mean_loss += loss
eval_mean_recall += recall
eval_mean_precision += precision
eval_scaled_L1_mean += scaled_l1
eval_scaled_euler_mean += scaled_euler
eval_classification_loss += classification_loss
eval_num_samples += 1
eval_mean_loss /= eval_num_samples
eval_mean_recall /= eval_num_samples
eval_mean_precision /= eval_num_samples
eval_scaled_L1_mean /= eval_num_samples
eval_scaled_euler_mean /= eval_num_samples
eval_classification_loss /= eval_num_samples
eval_time = time() - tic2
total_time = time() - tic
#########################
# PRINT STUFF ON SCREEN
#########################
print('\nEpoch {} / {}\n{}\nCurrent time: {}'.format(
epoch, train_config.max_epochs - 1, '-' * 12,
strftime("%Y-%m-%d %H:%M")))
print('Epoch Total Time: {} s ({} + {})'.format(
round(total_time, 2), round(training_time, 2), round(eval_time,
2)))
print('Next Epoch ETA: ' +
format(datetime.now() +
timedelta(seconds=total_time), '%Y-%m-%d %H:%M'))
print('Training ETA: ' + format(
datetime.now() + timedelta(
seconds=total_time *
(train_config.max_epochs - epoch - 1)), '%Y-%m-%d %H:%M'))
print('Train\n\tLoss: \t\t{}'
'\n\t\tL1: \t{}'
'\n\t\tEuler:\t{}'
'\n\t\tCL: \t{}'
'\n\tRecall: \t{}'
'\n\tPrecision:\t{}'.format(train_mean_loss,
train_scaled_L1_mean,
train_scaled_euler_mean,
train_classification_loss,
train_mean_recall,
train_mean_precision))
print('Validation\n\tLoss: \t\t{}'
'\n\t\tL1: \t{}'
'\n\t\tEuler:\t{}'
'\n\t\tCL: \t{}'
'\n\tRecall: \t{}'
'\n\tPrecision:\t{}'.format(eval_mean_loss, eval_scaled_L1_mean,
eval_scaled_euler_mean,
eval_classification_loss,
eval_mean_recall,
eval_mean_precision))
if train_config.use_visdom:
# Visualize Loss
viz.push_data(epoch, vis, loss_window, sub_loss_window,
recall_window, precision_window, train_mean_loss,
eval_mean_loss, train_scaled_L1_mean,
train_classification_loss, train_scaled_euler_mean,
eval_scaled_euler_mean, eval_scaled_L1_mean,
eval_classification_loss, train_mean_recall,
eval_mean_recall, train_mean_precision,
eval_mean_precision)
#########################
# SAVE WEIGHTS (every save_weights_modulus th epoch)
#########################
if epoch % train_config.save_weights_modulus == 0 or epoch == train_config.max_epochs - 1:
save_filename = weights_filename + '_epoch_' + str(epoch)
torch.save(
{
'epoch': epoch,
'model_state_dict': net.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'loss': loss
}, save_filename)
print('Training Complete [' + strftime("%Y-%m-%d %H:%M") + ']')
def eval_post_fafe_pp(fafe_model_path, pp_model_path, data_path, config_path,
sequence, kitti):
"""
:param self:
:param fafe_model_path:
:param pp_model_path:
:param data_path:
:param config_path:
:return:
"""
timestr = strftime("%Y-%m-%d_%H:%M")
velo_path = os.path.join(data_path, 'training', 'velodyne')
config = load_config(config_path)
start_time = time()
input_config = InputConfig(config['INPUT_CONFIG'])
train_config = TrainConfig(config['TRAIN_CONFIG'])
eval_config = EvalConfig(config['EVAL_CONFIG'])
model_config = ModelConfig(config['MODEL_CONFIG'])
post_config = PostConfig(config['POST_CONFIG'])
pillar = PillarOfFafe(input_config, train_config.batch_size,
train_config.verbose)
if model_config.model == 'little_fafe':
fafe = LittleFafe(input_config)
else:
fafe = FafeNet(input_config)
if eval_config.use_cuda:
if eval_config.cuda_device == 0:
device = torch.device("cuda:0")
print('Using CUDA:{}\n'.format(0))
elif eval_config.cuda_device == 1:
device = torch.device("cuda:1")
print('Using CUDA:{}\n'.format(1))
else:
print('Functionality for CUDA device cuda:{} not yet implemented.'.
format(eval_config.cuda_device))
print('Using cuda:0 instead...\n')
device = torch.device("cuda:0")
pillar = pillar.to(device)
fafe = fafe.to(device)
else:
device = torch.device("cpu")
print('Using CPU\n')
imu_path = os.path.join(data_path, 'training', 'oxts')
pillar.load_state_dict(
torch.load(
pp_model_path,
map_location=lambda storage, loc: storage)["model_state_dict"])
fafe.load_state_dict(
torch.load(
fafe_model_path,
map_location=lambda storage, loc: storage)["model_state_dict"])
post_fafe = PostFafe(input_config, post_config, device)
pillar.eval()
fafe.eval()
print("Model loaded, loading time: {}".format(round(
time() - start_time, 2)))
timestr = strftime("%Y-%m-%d_%H:%M")
eval_head = FafePredict(input_config, eval_config)
eval_head.eval()
print("Evalutation Model Loaded!")
# for seq in eval_config.validation_seqs:
testing_dataset = VoxelDataset(input_config,
root_dir=data_path,
split='training',
sequence=sequence)
testing_loader = DataLoader(dataset=testing_dataset,
batch_size=eval_config.batch_size,
num_workers=eval_config.num_workers,
shuffle=False)
data = []
frames = []
total_time_per_iteration = []
for batch_idx, batch in enumerate(testing_loader):
# Create Pillar Pseudo Img
voxel_stack, coord_stack, num_points_stack, num_nonempty_voxels, target, info = batch
nB = target.shape[0]
nT = input_config.num_conseq_frames
current_sequence = sequence
frame = info["GT_indices"][0]
if type(frame) == torch.Tensor:
frame = frame.item()
d = {}
d['current_time'] = frame
d['frame_id'] = frame # To match pmbm's data structure...
frames.append(frame)
imud = load_imu(imu_path, current_sequence)
true_states = kitti.get_bev_states(frame,
classes_to_track=['Car', 'Van'])
d['true_states'] = true_states
d['state_dims'] = 3
# Move all input data to the correct device if not using CPU
if train_config.use_cuda:
voxel_stack = voxel_stack.to(device)
coord_stack = coord_stack.to(device)
num_points_stack = num_points_stack.to(device)
num_nonempty_voxels = num_nonempty_voxels.to(device)
target = target.to(device)
timeit = time()
with torch.no_grad():
pseudo_stack = []
for time_iter in range(input_config.num_conseq_frames):
pseudo_image = pillar(voxel_stack[:, time_iter],
num_points_stack[:, time_iter],
coord_stack[:, time_iter],
num_nonempty_voxels[:, time_iter])
if input_config.pp_verbose:
print("Pseudo_img: {}".format(
pseudo_image.unsqueeze(1).shape))
pseudo_stack.append(pseudo_image.unsqueeze(1))
pseudo_torch = torch.cat(pseudo_stack, dim=1)
if train_config.use_cuda:
pseudo_torch = pseudo_torch.to(device)
target = target.to(device)
# Forward propagation. Reshape by squishing together Time and Channel dimensions.
# Reshaping basically as we do with BEV, stacking them on top of eachother.
out_detection, out_regression = fafe.forward(
pseudo_torch.reshape(
-1, pseudo_torch.shape[1] * pseudo_torch.shape[2],
pseudo_torch.shape[-2], pseudo_torch.shape[-1]))
inference = eval_head(out_detection, out_regression)
inference_time = round(time() - timeit, 2)
if torch.is_tensor(current_sequence):
current_sequence = current_sequence.item()
meas_ = inference[0][0]
meas = []
for row in range(meas_.shape[0]):
meas.append(transf(meas_[0].to('cpu').numpy()))
d['measurements'] = meas
if len(inference) > 1:
raise ValueError("Batch index > 1. Do something god damnit")
# Call post_fafe to infer the inference
post_fafe(inference[0])
imu_data = imud[frame]
# Add targets and predictions to data dict
# TID = Target ID
estimated_targets = []
for tid, tensor_state in post_fafe.object_state.items():
continue_bool = 0
if post_config.verbose:
print("Object: {} \n\t{}".format(tid, tensor_state))
_temp = {}
_temp['target_idx'] = tid
_temp['object_class'] = 'car'
_state = tensor_state[0].numpy()
single_target = SingleTargetHypothesis(transf(_state))
_temp['single_target'] = single_target
_temp['state_predictions'] = []
_temp['var_predictions'] = []
for i, step in enumerate(tensor_state):
if step is None or continue_bool:
continue_bool = 1
continue
# Coordinate transform predictions
preds = transf(step.numpy())
if post_config.coordinate_transform:
preds = translate_center(preds,
imu_data,
timestep=i,
dt=input_config.dt)
_temp['state_predictions'].append(preds)
if post_config.verbose:
print("Step: {}".format(transf(step.numpy())))
estimated_targets.append(_temp)
d['estimated_targets'] = estimated_targets
data.append(d)
total_time_per_iteration.append(inference_time)
print("{}({}s,{}#), ".format(frame, inference_time,
len(estimated_targets)),
end='')
return data, total_time_per_iteration
def eval_post_fafe(fafe_model_path, data_path, config_path, sequence, kitti):
"""
:param self:
:param fafe_model_path:
:param pp_model_path:
:param data_path:
:param config_path:
:return:
"""
timestr = strftime("%Y-%m-%d_%H:%M")
config = load_config(config_path)
start_time = time()
input_config = InputConfig(config['INPUT_CONFIG'])
train_config = TrainConfig(config['TRAIN_CONFIG'])
eval_config = EvalConfig(config['EVAL_CONFIG'])
model_config = ModelConfig(config['MODEL_CONFIG'])
post_config = PostConfig(config['POST_CONFIG'])
pillar = PillarOfFafe(input_config, train_config.batch_size, train_config.verbose)
if model_config.model == 'little_fafe':
fafe = LittleFafe(input_config)
else:
fafe = FafeNet(input_config)
if eval_config.use_cuda:
if eval_config.cuda_device == 0:
device = torch.device("cuda:0")
print('Using CUDA:{}\n'.format(0))
elif eval_config.cuda_device == 1:
device = torch.device("cuda:1")
print('Using CUDA:{}\n'.format(1))
else:
print('Functionality for CUDA device cuda:{} not yet implemented.'.format(eval_config.cuda_device))
print('Using cuda:0 instead...\n')
device = torch.device("cuda:0")
pillar = pillar.to(device)
fafe = fafe.to(device)
else:
device = torch.device("cpu")
print('Using CPU\n')
imu_path = os.path.join(data_path, 'training', 'oxts')
fafe.load_state_dict(torch.load(fafe_model_path, map_location=lambda storage, loc: storage)["model_state_dict"])
post_fafe = PostFafe(input_config, post_config, device)
pillar.eval()
fafe.eval()
print("Model loaded, loading time: {}".format(round(time() - start_time, 2)))
timestr = strftime("%Y-%m-%d_%H:%M")
eval_head = FafePredict(input_config, eval_config)
eval_head.eval()
print("Evalutation Model Loaded!")
# for seq in eval_config.validation_seqs:
testing_dataset = TemporalBEVsDataset(input_config, root_dir=data_path, split='training', sequence=sequence)
testing_loader = DataLoader(dataset=testing_dataset,
batch_size=eval_config.batch_size,
num_workers=eval_config.num_workers,
shuffle=False)
data = []
frames = []
print("Starting to run over sequence {}. Total number of frames: {}".format(sequence, len(testing_loader)))
total_time_per_iteration = []
for batch_idx, batch in enumerate(testing_loader):
input, target, info = batch
if eval_config.use_cuda:
input = input.to(device)
target = target.to(device)
timeit = time()
with torch.no_grad():
out_det, out_reg = fafe(input)
inference = eval_head(out_det, out_reg)
inference_time = round(time() - timeit, 2)
nB = target.shape[0]
nT = input_config.num_conseq_frames
current_sequence = sequence
frame = info["GT_indices"][0]
if type(frame) == torch.Tensor:
frame = frame.item()
d = {}
d['current_time'] = frame
d['frame_id'] = frame # To match pmbm's data structure...
frames.append(frame)
imud = load_imu(imu_path, current_sequence)
true_states = kitti.get_bev_states(frame, classes_to_track=['Car', 'Van'])
d['true_states'] = true_states
d['state_dims'] = 3
if torch.is_tensor(current_sequence):
current_sequence = current_sequence.item()
meas_ = inference[0][0]
meas = []
for row in range(meas_.shape[0]):
meas.append(transf(meas_[0].to('cpu').numpy()))
d['measurements'] = meas
if len(inference) > 1:
raise ValueError("Batch index > 1. Do something god damnit")
# Call post_fafe to infer the inference
post_fafe(inference[0])
imu_data = imud[frame]
# Add targets and predictions to data dict
# TID = Target ID
estimated_targets = []
for tid, tensor_state in post_fafe.object_state.items():
continue_bool = 0
if post_config.verbose:
print("Object: {} \n\t{}".format(tid, tensor_state))
_temp = {}
_temp['target_idx'] = tid
_temp['object_class'] = 'car'
_state = tensor_state[0].numpy()
single_target = SingleTargetHypothesis(transf(_state))
_temp['single_target'] = single_target
_temp['state_predictions'] = []
_temp['var_predictions'] = []
for i, step in enumerate(tensor_state):
if step is None or continue_bool:
continue_bool = 1
continue
# Coordinate transform predictions
preds = transf(step.numpy())
if post_config.coordinate_transform:
preds = translate_center(preds, imu_data, timestep=i, dt=input_config.dt )
_temp['state_predictions'].append(preds)
if post_config.verbose:
print("Step: {}".format(transf(step.numpy())))
estimated_targets.append(_temp)
d['estimated_targets'] = estimated_targets
data.append(d)
total_time_per_iteration.append(inference_time)
print("{}({}s,{}#), ".format(frame, inference_time, len(estimated_targets)),end='')
return data, total_time_per_iteration