def test_data_structure(self): if os.path.exists('/home/mlt/mot/fafe/cfg/adams_computer'): config_path = 'cfg/cfg_pp_mini.yml' elif os.path.exists('/Users/erikbohnsack'): config_path = 'cfg/cfg_mac.yml' else: config_path = 'cfg/cfg_pp.yml' print('Using config: \n\t{}\n'.format(config_path)) config = load_config(config_path) input_config = InputConfig(config['INPUT_CONFIG']) post_config = PostConfig(config['POST_CONFIG']) data = torch.load('inference0.pt') data1 = torch.load('inference1.pt') data2 = torch.load('inference2.pt') device = torch.device("cpu") post_fafe = PostFafe(input_config, post_config, device) print(post_fafe.object_state) post_fafe(data[0]) print(post_fafe.object_state.keys()) post_fafe(data1[0]) print(post_fafe.object_state.keys()) post_fafe(data2[0]) print(post_fafe.object_state.keys())
def test_plot_bevs(self): config_path = 'cfg_mini.yml' config = load_config(config_path) input_config = InputConfig(config['INPUT_CONFIG']) train_config = TrainConfig(config['TRAIN_CONFIG']) basepath = "/Users/erikbohnsack/data/training" tracking = pykitti.tracking(base_path=basepath, sequence="0000") points = tracking.get_velo(0) bev_map = np.zeros(input_config.bev_shape) print("BEV map size: {}".format(bev_map.shape)) grid_limits = np.array([[input_config.x_min, input_config.x_max], [input_config.y_min, input_config.y_max], [input_config.z_min, input_config.z_max]]) grid_sizes = np.array([ input_config.x_grid_size, input_config.y_grid_size, input_config.z_grid_size ]) start_jit = time.time() kitti_dataset.format_to_BEV_jit(bev_map, points, grid_limits, grid_sizes) end_jit = time.time() plot_BEV(bev_map) print("Elapsed time: {}".format(end_jit - start_jit))
def eval(model_path, data_path): config_path = 'cfg/cfg_mini.yml' config = load_config(config_path) input_config = InputConfig(config['INPUT_CONFIG']) eval_config = EvalConfig(config['EVAL_CONFIG']) model = FafeNet(input_config) model.load_state_dict(torch.load(model_path)["model_state_dict"]) model.eval() print("Model loaded") eval_head = FafePredict(input_config, eval_config) testing_seqs = [0] # TODO: Fix testing_datasets = [ TestDataset(input_config, root_dir=data_path, split='testing', sequence=seq) for seq in testing_seqs ] testing_dataset = ConcatDataset(testing_datasets) testing_loader = DataLoader(dataset=testing_dataset, batch_size=eval_config.batch_size, num_workers=eval_config.num_workers) for batch_index, batch in enumerate(testing_loader): input, info = batch out_det, out_reg = model(input) inference = eval_head(out_det, out_reg)
def test_concat_dataset(self): config_path = 'cfg_mini.yml' config = load_config(config_path) input_config = InputConfig(config['INPUT_CONFIG']) train_config = TrainConfig(config['TRAIN_CONFIG']) if os.path.exists("/Users/erikbohnsack/data/"): root_dir = "/Users/erikbohnsack/data/" sequence = 0 else: root_dir = "/home/mlt/data/" sequence = 3 training_datasets = [ TemporalBEVsDataset(input_config, root_dir, split='training', sequence=seq) for seq in train_config.training_seqs ] validation_datasets = [ TemporalBEVsDataset(input_config, root_dir, split='training', sequence=seq) for seq in train_config.validation_seqs ] # Split still training due to data structure training_dataset = ConcatDataset(training_datasets) validation_dataset = ConcatDataset(validation_datasets) training_dataloader = DataLoader(training_dataset, batch_size=train_config.batch_size, shuffle=train_config.shuffle, num_workers=train_config.num_workers) validation_dataloader = DataLoader( validation_dataset, batch_size=train_config.batch_size, shuffle=train_config.shuffle, num_workers=train_config.num_workers) print("----------------------- TRAINING -----------------------") for i_batch, batch_sample in enumerate(training_dataloader): print(i_batch) input, target, info = batch_sample print("Input shape: {}".format(input.shape)) print("Sequence: {}, real index: {}".format( info["sequence"], info["Current_index"])) print("----------------------- VALIDATION -----------------------") for i_batch, batch_sample in enumerate(validation_dataloader): print(i_batch) input, target, info = batch_sample print("Input shape: {}".format(input.shape)) print("Sequence: {}, real index: {}".format( info["sequence"], info["Current_index"]))
def test_eval(self): save_filename = 'foo' config_path = 'cfg.yml' config = load_config(config_path) input_config = InputConfig(config['INPUT_CONFIG']) model = FafeNet(config) torch.save({'model_state_dict': model.state_dict()}, save_filename) model_path = save_filename data_path = '/Users/erikbohnsack/data' eval(model_path=model_path, data_path=data_path)
def test_train_with_model(self): root_dir = get_root_dir() if os.path.exists( '/Users/erikbohnsack/Code/MOT/fafe/trained_models/weights_2019-04-12_13_43_epoch_85' ): model_path = '/Users/erikbohnsack/Code/MOT/fafe/trained_models/weights_2019-04-12_13_43_epoch_85' config_path = 'cfg/cfg_mac.yml' else: model_path = '/home/mlt/mot/fafe/trained_models/test_train_2019-04-11_14_01_epoch_40' config_path = 'cfg/cfg_mini.yml' config = load_config(config_path) input_config = InputConfig(config['INPUT_CONFIG']) train_config = TrainConfig(config['TRAIN_CONFIG']) verbose = train_config.verbose net = FafeNet(input_config) net.load_state_dict( torch.load( model_path, map_location=lambda storage, loc: storage)["model_state_dict"]) ######################### # Set which device run on ######################### if train_config.use_cuda: if train_config.cuda_device == 0: device = torch.device("cuda:0") print('Using CUDA:{}\n'.format(0)) elif train_config.cuda_device == 1: device = torch.device("cuda:1") print('Using CUDA:{}\n'.format(1)) else: print( 'Functionality for CUDA device cuda:{} not yet implemented.' .format(train_config.cuda_device)) print('Using cuda:0 instead...\n') device = torch.device("cuda:0") net = net.to(device) else: device = torch.device("cpu") print('Using CPU\n') ######################### # Define optimizer ######################### optimizer = optim.Adam(net.parameters(), lr=train_config.learning_rate, weight_decay=train_config.weight_decay) print('Adams Optimizer set up with\n\tlr = {}\n\twd = {}\n'.format( train_config.learning_rate, train_config.weight_decay)) loss_func = FafeLoss(input_config, train_config, device) training_datasets = [ TemporalBEVsDataset(input_config, root_dir, split='training', sequence=seq) for seq in train_config.training_seqs ] validation_datasets = [ TemporalBEVsDataset(input_config, root_dir, split='training', sequence=seq) for seq in train_config.validation_seqs ] # Split still training due to data structure training_dataset = ConcatDataset(training_datasets) validation_dataset = ConcatDataset(validation_datasets) training_dataloader = DataLoader(training_dataset, batch_size=train_config.batch_size, shuffle=train_config.shuffle, num_workers=train_config.num_workers) validation_dataloader = DataLoader( validation_dataset, batch_size=train_config.batch_size, shuffle=train_config.shuffle, num_workers=train_config.num_workers) ############################### # Start training and evaluation ############################### print('\nTraining initiated [' + strftime("%Y-%m-%d %H:%M") + ']') for epoch in range(train_config.max_epochs): train_mean_loss, train_mean_recall, train_mean_precision, train_num_samples = 0, 0, 0, 0 eval_mean_loss, eval_mean_recall, eval_mean_precision, eval_num_samples = 0, 0, 0, 0 train_scaled_L1_mean, train_classification_loss = 0, 0 eval_scaled_L1_mean, eval_classification_loss = 0, 0 ######################### # TRAINING ######################### tic = time() net.train() for i_batch, sample_batched in enumerate(training_dataloader): input, target, _ = sample_batched if train_config.use_cuda: input = input.to(device) target = target.to(device) # Always reset optimizer's gradient each iteration optimizer.zero_grad() if verbose: print('{} i: {} {}'.format('~' * 10, i_batch, '~' * 10)) print('Input shape: {}'.format(input.shape)) print('Target shape: {}'.format(target.shape)) print('Target: {}'.format(target[0][0][0][0])) # Forward propagation out_detection, out_regression = net.forward(input) if verbose: print('Output shape det: {}'.format(out_detection.shape)) print('Output shape reg: {}'.format(out_regression.shape)) # Calculate the loss loss, recall, precision, scaled_L1, scaled_euler, classification_loss = loss_func( out_detection, out_regression, target, verbose) # Back propagate loss.backward() # Update the weights optimizer.step() train_mean_loss += loss train_mean_recall += recall train_mean_precision += precision train_scaled_L1_mean += scaled_L1 train_classification_loss += classification_loss train_num_samples += 1 # Calculate the actual averages train_mean_loss /= train_num_samples train_mean_recall /= train_num_samples train_mean_precision /= train_num_samples train_scaled_L1_mean /= train_num_samples train_classification_loss /= train_num_samples training_time = time() - tic
def train(): if os.path.exists('/home/mlt/mot/fafe/cfg/adams_computer'): config_path = 'cfg/cfg_pp_mini.yml' elif os.path.exists('/Users/erikbohnsack'): config_path = 'cfg/cfg_mac.yml' else: config_path = 'cfg/cfg_pp.yml' print('Using config: \n\t{}\n'.format(config_path)) config = load_config(config_path) input_config = InputConfig(config["INPUT_CONFIG"]) train_config = TrainConfig(config["TRAIN_CONFIG"]) loss_config = LossConfig(config['LOSS_CONFIG']) model_config = ModelConfig(config['MODEL_CONFIG']) verbose = train_config.verbose time_str = strftime("%Y-%m-%d_%H-%M") weights_filename = 'trained_models/' + time_str + '/weights_' + time_str showroom_path = get_showroom_path(model_path="_".join( ('weights', time_str)), full_path_bool=False) if not os.path.exists('trained_models/' + time_str): os.mkdir('trained_models/' + time_str) print( 'Training weights will be saved to:\n\t{}\n'.format(weights_filename)) config_filename = 'trained_models/' + time_str + '/config_' + time_str + '.yml' save_config(config_filename, config) print('Config file saved to:\n\t{}\n'.format(config_filename)) if train_config.use_visdom: print( 'Dont forget to run "visdom" in a terminal in parallel to this in order to start the Visdom server' ) print('Choose port with "python -m visdom.server -port {}" \n'.format( train_config.visdom_port)) vis = visdom.Visdom( port=train_config.visdom_port) # port 8097 is default loss_window, sub_loss_window, recall_window, precision_window = viz.get_windows( vis, time_str) print("~" * 20) print("Setting up net") pillar = PillarOfFafe(input_config=input_config, batch_size=train_config.batch_size, verbose=input_config.pp_verbose) if model_config.model == 'little_fafe': fafe = LittleFafe(input_config=input_config) else: fafe = FafeNet(input_config=input_config) ######################### # Set which device run on ######################### if train_config.use_cuda: # "If you load your samples in the Dataset on CPU and would like to push it during training to the GPU, # you can speed up the host to device transfer by enabling pin_memory." # - ptrblck [https://discuss.pytorch.org/t/when-to-set-pin-memory-to-true/19723] pin_memory = False device = torch.device("cuda:" + str(train_config.cuda_device)) print('\nUsing device {}\n'.format(device)) pillar = pillar.to(device) fafe = fafe.to(device) else: pin_memory = False device = torch.device("cpu") print('Using CPU\n') loss_func = FafeLoss(input_config, train_config, loss_config, device) if train_config.use_cuda: loss_func = loss_func.to(device) print("Net set up successfully!") pp_pytorch_total_params = sum(p.numel() for p in pillar.parameters()) pp_pytorch_trainable_params = sum(p.numel() for p in pillar.parameters() if p.requires_grad) print("~" * 20) print( "PP:\n\tNumber of parameters: {}\n\tNumber of trainable parameters: {}" .format(pp_pytorch_total_params, pp_pytorch_trainable_params)) pytorch_total_params = sum(p.numel() for p in fafe.parameters()) pytorch_trainable_params = sum(p.numel() for p in fafe.parameters() if p.requires_grad) print( "FAFE:\n\tNumber of parameters: {}\n\tNumber of trainable parameters: {}" .format(pytorch_total_params, pytorch_trainable_params)) root_dir = get_root_dir() ######################### # Define optimizer ######################### params = list(pillar.parameters()) + list(fafe.parameters()) + list( loss_func.parameters()) optimizer = optim.Adam(params, lr=train_config.learning_rate, weight_decay=train_config.weight_decay) print('Adams Optimizer set up with\n\tlr = {}\n\twd = {}\n'.format( train_config.learning_rate, train_config.weight_decay)) ######################### # Get Datasets ######################### print('Training Data:') # fafe_sampler = FafeSampler(data_source=dataset, input_config=input_config) # TODO: Sampler needs a *data_source* as input to know the length of objects it can iterate over. # TODO: When concatenating training_dataloader = DataLoader(ConcatDataset([ VoxelDataset(input_config, root_dir, split='training', sequence=seq) for seq in train_config.training_seqs ]), batch_size=train_config.batch_size, shuffle=train_config.shuffle, num_workers=train_config.num_workers, pin_memory=pin_memory) print('Validation Data:') validation_dataloader = DataLoader(ConcatDataset([ VoxelDataset(input_config, root_dir, split='training', sequence=seq) for seq in train_config.validation_seqs ]), batch_size=train_config.batch_size, shuffle=train_config.shuffle, num_workers=train_config.num_workers, pin_memory=pin_memory) print('Data Loaders set up with:\n\tBatch size: {}\n\tNum Workers: {}'. format(train_config.batch_size, train_config.num_workers)) ############################### # Start training and evaluation ############################### print('\nTraining initiated [' + strftime("%Y-%m-%d %H:%M") + ']') for epoch in range(train_config.max_epochs): train_mean_loss, train_mean_recall, train_mean_precision, train_num_samples = 0, 0, 0, 0 eval_mean_loss, eval_mean_recall, eval_mean_precision, eval_num_samples = 0, 0, 0, 0 train_scaled_reg_mean, train_scaled_euler_mean, train_classification_loss = 0, 0, 0 eval_scaled_reg_mean, eval_scaled_euler_mean, eval_classification_loss = 0, 0, 0 ######################### # TRAINING ######################### tic = time() pillar.train() fafe.train() torch.set_grad_enabled(True) for i_batch, batch in enumerate(training_dataloader): # Always reset optimizer's gradient each iteration optimizer.zero_grad() # Create Pillar Pseudo Img voxel_stack, coord_stack, num_points_stack, num_nonempty_voxels, target, info = batch # Move all input data to the correct device if not using CPU if train_config.use_cuda: voxel_stack = voxel_stack.to(device) coord_stack = coord_stack.to(device) num_points_stack = num_points_stack.to(device) num_nonempty_voxels = num_nonempty_voxels.to(device) target = target.to(device) pseudo_stack = [] for time_iter in range(input_config.num_conseq_frames): if input_config.pp_verbose: print("~" * 20) print("time iter: {}".format(time_iter)) print("voxels \n\tshape: {}".format( voxel_stack[:, time_iter].shape)) print("coord \n\tshape: {}".format( coord_stack[:, time_iter].shape)) print("num_points \n\tshape: {}".format( num_points_stack[:, time_iter].shape)) print("num_nonempty_voxels \n\tshape: {}".format( num_nonempty_voxels[:, time_iter].shape)) pseudo_image = pillar(voxel_stack[:, time_iter], num_points_stack[:, time_iter], coord_stack[:, time_iter], num_nonempty_voxels[:, time_iter]) if input_config.pp_verbose: print("Pseudo_img: {}".format( pseudo_image.unsqueeze(1).shape)) pseudo_stack.append(pseudo_image.unsqueeze(1)) pseudo_torch = torch.cat(pseudo_stack, dim=1) if input_config.pp_verbose: print("Pseudo stacked over time: \n\t{}".format( pseudo_torch.shape)) if train_config.use_cuda: pseudo_torch = pseudo_torch.to(device) target = target.to(device) # Forward propagation. Reshape by squishing together Time and Channel dimensions. # Reshaping basically as we do with BEV, stacking them on top of eachother. out_detection, out_regression = fafe.forward( pseudo_torch.reshape( -1, pseudo_torch.shape[1] * pseudo_torch.shape[2], pseudo_torch.shape[-2], pseudo_torch.shape[-1])) # Calculate the loss loss, recall, precision, scaled_reg, scaled_euler, classification_loss = loss_func( out_detection, out_regression, target, verbose) # Back propagate loss.backward() if train_config.plot_grad_flow: plot_grad_flow( pillar.named_parameters(), os.path.join( showroom_path, 'grad_flow_pillar', "".join( ("epoch_", str(epoch).zfill(4), "_batch_", str(i_batch).zfill(4), ".png")))) plot_grad_flow( fafe.named_parameters(), os.path.join( showroom_path, 'grad_flow_fafe', "".join( ("epoch_", str(epoch).zfill(4), "_batch_", str(i_batch).zfill(4), ".png")))) # Update the weights optimizer.step() train_mean_loss += loss train_mean_recall += recall train_mean_precision += precision train_scaled_reg_mean += scaled_reg train_scaled_euler_mean += scaled_euler train_classification_loss += classification_loss train_num_samples += 1 # Calculate the actual averages train_mean_loss /= train_num_samples train_mean_recall /= train_num_samples train_mean_precision /= train_num_samples train_scaled_reg_mean /= train_num_samples train_scaled_euler_mean /= train_num_samples train_classification_loss /= train_num_samples training_time = time() - tic ######################### # EVALUATION ######################### tic2 = time() pillar.eval() fafe.eval() with torch.no_grad(): for i_batch, batch in enumerate(validation_dataloader): # Create Pillar Pseudo Img voxel_stack, coord_stack, num_points_stack, num_nonempty_voxels, target, index = batch # Move all input data to the correct device if not using CPU if train_config.use_cuda: voxel_stack = voxel_stack.to(device) coord_stack = coord_stack.to(device) num_points_stack = num_points_stack.to(device) num_nonempty_voxels = num_nonempty_voxels.to(device) target = target.to(device) pseudo_stack = [] for time_iter in range(input_config.num_conseq_frames): if train_config.verbose: print("~" * 20) print("time iter: {}".format(time_iter)) print("voxels \n\tshape: {}".format( voxel_stack[:, time_iter].shape)) print("coord \n\tshape: {}".format( coord_stack[:, time_iter].shape)) print("num_points \n\tshape: {}".format( num_points_stack[:, time_iter].shape)) print("num_nonempty_voxels \n\tshape: {}".format( num_nonempty_voxels[:, time_iter].shape)) pseudo_image = pillar(voxel_stack[:, time_iter], num_points_stack[:, time_iter], coord_stack[:, time_iter], num_nonempty_voxels[:, time_iter]) if train_config.verbose: print("Pseudo_img: {}".format( pseudo_image.unsqueeze(1).shape)) pseudo_stack.append(pseudo_image.unsqueeze(1)) pseudo_torch = torch.cat(pseudo_stack, dim=1) if train_config.use_cuda: pseudo_torch = pseudo_torch.to(device) target = target.to(device) # Forward propagation out_detection, out_regression = fafe.forward( pseudo_torch.reshape( -1, pseudo_torch.shape[1] * pseudo_torch.shape[2], pseudo_torch.shape[-2], pseudo_torch.shape[-1])) # Calculate the loss loss, recall, precision, scaled_reg, scaled_euler, classification_loss = loss_func( out_detection, out_regression, target, verbose) eval_mean_loss += loss eval_mean_recall += recall eval_mean_precision += precision eval_scaled_reg_mean += scaled_reg eval_scaled_euler_mean += scaled_euler eval_classification_loss += classification_loss eval_num_samples += 1 eval_mean_loss /= eval_num_samples eval_mean_recall /= eval_num_samples eval_mean_precision /= eval_num_samples eval_scaled_reg_mean /= eval_num_samples eval_scaled_euler_mean /= eval_num_samples eval_classification_loss /= eval_num_samples eval_time = time() - tic2 total_time = time() - tic ######################### # PRINT STUFF ON SCREEN ######################### print('\nEpoch {} / {}\n{}\nCurrent time: {}'.format( epoch, train_config.max_epochs - 1, '-' * 12, strftime("%Y-%m-%d %H:%M"))) print('Epoch Total Time: {} s ({} + {})'.format( round(total_time, 2), round(training_time, 2), round(eval_time, 2))) print('Next Epoch ETA: ' + format(datetime.now() + timedelta(seconds=total_time), '%Y-%m-%d %H:%M')) print('Training ETA: ' + format( datetime.now() + timedelta( seconds=total_time * (train_config.max_epochs - epoch - 1)), '%Y-%m-%d %H:%M')) print('Train\n\tLoss: \t\t{}' '\n\t\tL1: \t{}' '\n\t\tEuler:\t{}' '\n\t\tCL: \t{}' '\n\tRecall: \t{}' '\n\tPrecision:\t{}'.format(train_mean_loss, train_scaled_reg_mean, train_scaled_euler_mean, train_classification_loss, train_mean_recall, train_mean_precision)) print('Validation\n\tLoss: \t\t{}' '\n\t\tL1: \t{}' '\n\t\tEuler:\t{}' '\n\t\tCL: \t{}' '\n\tRecall: \t{}' '\n\tPrecision:\t{}'.format(eval_mean_loss, eval_scaled_reg_mean, eval_scaled_euler_mean, eval_classification_loss, eval_mean_recall, eval_mean_precision)) if train_config.use_visdom: # Visualize Loss viz.push_data(epoch, vis, loss_window, sub_loss_window, recall_window, precision_window, train_mean_loss, eval_mean_loss, train_scaled_reg_mean, train_classification_loss, train_scaled_euler_mean, eval_scaled_euler_mean, eval_scaled_reg_mean, eval_classification_loss, train_mean_recall, eval_mean_recall, train_mean_precision, eval_mean_precision) #################################################### # SAVE WEIGHTS (every save_weights_modulus th epoch) #################################################### if epoch % train_config.save_weights_modulus == 0 or epoch == train_config.max_epochs - 1: save_filename = weights_filename + '_epoch_' + str(epoch) pp_fn = save_filename + '_pp' torch.save( { 'epoch': epoch, 'model_state_dict': pillar.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'loss': loss }, pp_fn) fafe_fn = save_filename + '_fafe' torch.save( { 'epoch': epoch, 'model_state_dict': fafe.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'loss': loss }, fafe_fn) print('Training Complete [' + strftime("%Y-%m-%d %H:%M") + ']')
def eval_with_GT(model_path, data_path, config_path, v2=False): timestr = strftime("%Y-%m-%d_%H:%M") showroom_path = get_showroom_path(model_path, full_path_bool=True) print('Images will be saved to:\n\t{}'.format(showroom_path)) velo_path = os.path.join(data_path, 'training', 'velodyne') config = load_config(config_path) start_time = time() input_config = InputConfig(config['INPUT_CONFIG']) eval_config = EvalConfig(config['EVAL_CONFIG']) if v2: model = HaveFafe4Eval(input_config) else: model = FafeNet(input_config) if eval_config.use_cuda: if eval_config.cuda_device == 0: device = torch.device("cuda:0") print('Using CUDA:{}\n'.format(0)) elif eval_config.cuda_device == 1: device = torch.device("cuda:1") print('Using CUDA:{}\n'.format(1)) else: print('Functionality for CUDA device cuda:{} not yet implemented.'. format(eval_config.cuda_device)) print('Using cuda:0 instead...\n') device = torch.device("cuda:0") model = model.to(device) else: device = torch.device("cpu") print('Using CPU\n') imu_path = os.path.join(data_path, 'training', 'oxts') model.load_state_dict( torch.load( model_path, map_location=lambda storage, loc: storage)["model_state_dict"]) model.eval() print("Model loaded, loading time: {}".format(round( time() - start_time, 2))) timestr = strftime("%Y-%m-%d_%H:%M") eval_head = FafePredict(input_config, eval_config) eval_head.eval() print("Evalutation Model Loaded!") testing_datasets = [ TemporalBEVsDataset(input_config, root_dir=data_path, split='training', sequence=seq) for seq in eval_config.validation_seqs ] testing_dataset = ConcatDataset(testing_datasets) testing_loader = DataLoader(dataset=testing_dataset, batch_size=eval_config.batch_size, num_workers=eval_config.num_workers, shuffle=False) print("Starting to iterate over batches") if eval_config.use_gospa: gospa_scores_dict = {} for batch_idx, batch in enumerate(testing_loader): input, target, info = batch if eval_config.use_cuda: input = input.to(device) target = target.to(device) timeit = time() with torch.no_grad(): out_det, out_reg = model(input) inference = eval_head(out_det, out_reg) print("\nTime to propagate through network: {}".format( round(time() - timeit, 2))) nB = target.shape[0] nT = input_config.num_conseq_frames tic = time() for batch_index in range(nB): current_sequence = info["sequence"][batch_index] imud = load_imu(imu_path, current_sequence) print("Batch: {} \t| Seq: {}".format( batch_index, info["sequence"][batch_index]), end='') if eval_config.save_figs: frame = info["GT_indices"][0] file = os.path.join(velo_path, info["sequence"][batch_index], str(frame.item()).zfill(6) + '.bin') pc = np.fromfile(file, dtype=np.float32).reshape((-1, 4)) fig = draw_lidar( pc, off_screen_rendering=eval_config.off_screen_rendering) if eval_config.use_gospa: current_frame = info["Current_index"][batch_index].item() gospa_scores_dict[int(current_sequence), current_frame] = np.zeros(nT) for time_index in range(nT): # Get ground truths non_zero_gt_mask = target[batch_index][time_index].sum( dim=1) != 0 non_zero_gt = target[batch_index][time_index][non_zero_gt_mask] # Transform gt to 2D bbox x1y1x2y2 form point_form_gt = point_form_fafe(non_zero_gt) # Transform gt to 3D bbox form point_form_gt_3d = point_form_3d(point_form_gt, input_config.z_center, input_config.z_height, device=device) # Rotate gt 3D bboxes gt_center = center_size_3d(non_zero_gt, input_config.z_center, device) point_form_gt_3d_rot = rotate_3d_bbx(point_form_gt_3d, non_zero_gt[:, 4], gt_center, device) # Get inference inference_reg = inference[batch_index][time_index] # Calculate GOSPA if ordered to do so if eval_config.use_gospa: if inference_reg.is_cuda: gospa_gt = non_zero_gt[:, 0:2].cpu() gospa_inf = inference_reg[:, 0:2].cpu() else: gospa_gt = non_zero_gt[:, 0:2] gospa_inf = inference_reg[:, 0:2] gospa_score = GOSPA(gospa_gt, gospa_inf) gospa_scores_dict[int(current_sequence), current_frame][time_index] = gospa_score if eval_config.save_figs and eval_config.draw_gospa: fig = draw_gospa(gospa_score, fig, time_index) # Get probabilities of objects inference_probabilities = inference_reg[ ..., -1] if eval_config.show_confidence else None # Transform inference to 2D bbox x1y1x2y2 inf_points_2d = point_form_fafe(inference_reg) # Transform inference to 3D bbox inference_points = point_form_3d(inf_points_2d, input_config.z_center, input_config.z_height, device) # Rotate inference 3D bboxes inference_center = center_size_3d(inference_reg, input_config.z_center, device) inference_points_rot = rotate_3d_bbx(inference_points, inference_reg[:, 4], inference_center, device) # Translate center points according to time_index translated_gt_c = translate_center(gt_center, imu_data=imud[frame.item()], timestep=time_index, dt=input_config.dt, device=device) translated_infer_c = translate_center( inference_center, imu_data=imud[frame.item()], timestep=time_index, dt=input_config.dt, device=device) # TODO: Move these if-statements further up to reduce computation if eval_config.save_figs: if time_index == 0: fig = draw_gt_boxes3d(point_form_gt_3d_rot, fig, color=eval_config.gt_color) fig = draw_gt_boxes3d( inference_points_rot, fig, color=eval_config.infer_color, probabilities=inference_probabilities) else: fig = draw_points_3d(translated_gt_c, fig, color=eval_config.gt_color) fig = draw_points_3d(translated_infer_c, fig, color=eval_config.infer_color) if eval_config.save_figs: filename = "_".join( ("Infer", "seq", info["sequence"][batch_index], "f", str(frame.item()), timestr + ".png")) filepath = os.path.join(showroom_path, 'oracle_view', filename) filename_top = "_".join( ("Top_Infer", "seq", info["sequence"][batch_index], "f", str(frame.item()), timestr + ".png")) filepath_top = os.path.join(showroom_path, 'top_view', filename_top) mayavi.mlab.view(azimuth=90, elevation=0, focalpoint=[24., 0, 0], distance=120.0, figure=fig) mayavi.mlab.savefig(filepath_top, figure=fig) mayavi.mlab.view( azimuth=180, elevation=70, focalpoint=[12.0909996, -1.04700089, -2.03249991], distance=62.0, figure=fig) mayavi.mlab.savefig(filepath, figure=fig) print('\t | time: {}'.format(round(time() - tic, 2))) print(gospa_scores_dict)
def eval_with_GT(fafe_model_path, pp_model_path, data_path, config_path): timestr = strftime("%Y-%m-%d_%H:%M") showroom_path = get_showroom_path(fafe_model_path, full_path_bool=True) print('Images will be saved to:\n\t{}'.format(showroom_path)) velo_path = os.path.join(data_path, 'training', 'velodyne') config = load_config(config_path) start_time = time() input_config = InputConfig(config['INPUT_CONFIG']) train_config = TrainConfig(config['TRAIN_CONFIG']) eval_config = EvalConfig(config['EVAL_CONFIG']) model_config = ModelConfig(config['MODEL_CONFIG']) post_config = PostConfig(config['POST_CONFIG']) pillar = PillarOfFafe(input_config, train_config.batch_size, train_config.verbose) if model_config.model == 'little_fafe': fafe = LittleFafe(input_config) else: fafe = FafeNet(input_config) if eval_config.use_cuda: if eval_config.cuda_device == 0: device = torch.device("cuda:0") print('Using CUDA:{}\n'.format(0)) elif eval_config.cuda_device == 1: device = torch.device("cuda:1") print('Using CUDA:{}\n'.format(1)) else: print('Functionality for CUDA device cuda:{} not yet implemented.'.format(eval_config.cuda_device)) print('Using cuda:0 instead...\n') device = torch.device("cuda:0") pillar = pillar.to(device) fafe = fafe.to(device) else: device = torch.device("cpu") print('Using CPU\n') imu_path = os.path.join(data_path, 'training', 'oxts') pillar.load_state_dict(torch.load(pp_model_path, map_location=lambda storage, loc: storage)["model_state_dict"]) fafe.load_state_dict(torch.load(fafe_model_path, map_location=lambda storage, loc: storage)["model_state_dict"]) pillar.eval() fafe.eval() print("Model loaded, loading time: {}".format(round(time() - start_time, 2))) timestr = strftime("%Y-%m-%d_%H:%M") eval_head = FafePredict(input_config, eval_config) eval_head.eval() print("Evalutation Model Loaded!") testing_datasets = [VoxelDataset(input_config, root_dir=data_path, split='training', sequence=seq) for seq in eval_config.validation_seqs] testing_dataset = ConcatDataset(testing_datasets) testing_loader = DataLoader(dataset=testing_dataset, batch_size=eval_config.batch_size, num_workers=eval_config.num_workers, shuffle=False) print("Starting to iterate over batches") if eval_config.use_gospa: gospa_scores_dict = {} for batch_idx, batch in enumerate(testing_loader): # Create Pillar Pseudo Img voxel_stack, coord_stack, num_points_stack, num_nonempty_voxels, target, info = batch # Move all input data to the correct device if not using CPU if train_config.use_cuda: voxel_stack = voxel_stack.to(device) coord_stack = coord_stack.to(device) num_points_stack = num_points_stack.to(device) num_nonempty_voxels = num_nonempty_voxels.to(device) target = target.to(device) timeit = time() with torch.no_grad(): pseudo_stack = [] for time_iter in range(input_config.num_conseq_frames): pseudo_image = pillar(voxel_stack[:, time_iter], num_points_stack[:, time_iter], coord_stack[:, time_iter], num_nonempty_voxels[:, time_iter]) if input_config.pp_verbose: print("Pseudo_img: {}".format(pseudo_image.unsqueeze(1).shape)) pseudo_stack.append(pseudo_image.unsqueeze(1)) pseudo_torch = torch.cat(pseudo_stack, dim=1) if train_config.use_cuda: pseudo_torch = pseudo_torch.to(device) target = target.to(device) # Forward propagation. Reshape by squishing together Time and Channel dimensions. # Reshaping basically as we do with BEV, stacking them on top of eachother. out_detection, out_regression = fafe.forward( pseudo_torch.reshape(-1, pseudo_torch.shape[1] * pseudo_torch.shape[2], pseudo_torch.shape[-2], pseudo_torch.shape[-1])) inference = eval_head(out_detection, out_regression) print("\nTime to propagate through network: {}".format(round(time() - timeit, 2))) nB = target.shape[0] nT = input_config.num_conseq_frames tic = time() for batch_index in range(nB): current_sequence = info["sequence"][batch_index] frame = info["GT_indices"][0] if torch.is_tensor(current_sequence): current_sequence = current_sequence.item() imud = load_imu(imu_path, current_sequence) print("Batch: {} \t| Seq: {} Frame: {}".format(batch_index, current_sequence, frame.item()), end='') if eval_config.save_raw: if not os.path.exists("showroom/detections_" +str(current_sequence).zfill(4)): os.mkdir("showroom/detections_" +str(current_sequence).zfill(4)) filename = os.path.join("showroom", "detections_{}".format(str(current_sequence).zfill(4)), str(int(frame)).zfill(4) + ".pt") torch.save(inference, filename) if eval_config.save_figs: file = os.path.join(velo_path, str(current_sequence).zfill(4), str(frame.item()).zfill(6) + '.bin') pc = np.fromfile(file, dtype=np.float32).reshape((-1, 4)) fig = draw_lidar(pc, off_screen_rendering=eval_config.off_screen_rendering) if eval_config.use_gospa: current_frame = info["Current_index"][batch_index].item() gospa_scores_dict[int(current_sequence), current_frame] = np.zeros(nT) for time_index in range(nT): # Get ground truths non_zero_gt_mask = target[batch_index][time_index].sum(dim=1) != 0 non_zero_gt = target[batch_index][time_index][non_zero_gt_mask] # Transform gt to 2D bbox x1y1x2y2 form point_form_gt = point_form_fafe(non_zero_gt) # Transform gt to 3D bbox form point_form_gt_3d = point_form_3d(point_form_gt, input_config.z_center, input_config.z_height, device=device) # Rotate gt 3D bboxes gt_center = center_size_3d(non_zero_gt, input_config.z_center, device) point_form_gt_3d_rot = rotate_3d_bbx(point_form_gt_3d, non_zero_gt[:, 4], gt_center, device) # Get inference inference_reg = inference[batch_index][time_index] # Filter out if outside FOV filtered_inference_reg = [] for inf_reg in inference_reg: np_inf_reg = inf_reg.cpu().data.numpy() if within_fov(np_inf_reg[0:2], min_angle=0.78, max_angle=2.45, max_radius=100): filtered_inference_reg.append(np_inf_reg) # Filter out if too closely located filtered_inference_reg = too_close(filtered_inference_reg, eval_config.distance_threshold) if eval_config.save_detections_as_measurements and time_index == 0: for fir in filtered_inference_reg: file1 = open(showroom_path + "/detections_" + str(current_sequence).zfill(4) + ".txt", "a") # Save it on the format PMBM wants it... p = [x, y, rot] with x defined right wards and y forward file1.write(str(frame.item()) + ' Car:' + str(fir[5]) + ' ' + str(- fir[1]) + ' ' + str( fir[0]) + ' ' + str(fir[4]) + "\n") file1.close() # Make filtered_inference_reg numpy because GOSPA needs it to be filtered_inference_reg = np.array(filtered_inference_reg) # Calculate GOSPA if ordered to do so if eval_config.use_gospa: if inference_reg.is_cuda: gospa_gt = non_zero_gt[:, 0:2].cpu() # gospa_inf = inference_reg[:, 0:2].cpu() gospa_inf = filtered_inference_reg[:, 0:2] if len(filtered_inference_reg) > 0 else np.array([]) else: gospa_gt = non_zero_gt[:, 0:2] # gospa_inf = inference_reg[:, 0:2] gospa_inf = filtered_inference_reg[:, 0:2] gospa_score = GOSPA(gospa_gt, gospa_inf) gospa_scores_dict[int(current_sequence), current_frame][time_index] = gospa_score if eval_config.save_figs and eval_config.draw_gospa: fig = draw_gospa(gospa_score, fig, time_index) # Make filtered_inference_reg torch tensor because point forms need it to be filtered_inference_reg = torch.from_numpy(filtered_inference_reg).to(device) # If no outputs -> plot ev. gt and continue if len(filtered_inference_reg) == 0: if time_index == 0: fig = draw_gt_boxes3d(point_form_gt_3d_rot, fig, color=eval_config.gt_color) continue else: # Translate center points according to time_index translated_gt_c = translate_center(gt_center, imu_data=imud[frame.item()], timestep=time_index, dt=input_config.dt, device=device) fig = draw_points_3d(translated_gt_c, fig, color=eval_config.gt_color) continue # Get probabilities of objects inference_probabilities = filtered_inference_reg[..., -1] if eval_config.show_confidence else None # Transform inference to 2D bbox x1y1x2y2 inf_points_2d = point_form_fafe(filtered_inference_reg) # Transform inference to 3D bbox inference_points = point_form_3d(inf_points_2d, input_config.z_center, input_config.z_height, device) # Rotate inference 3D bboxes inference_center = center_size_3d(filtered_inference_reg, input_config.z_center, device) inference_points_rot = rotate_3d_bbx(inference_points, filtered_inference_reg[:, 4], inference_center, device) # Translate center points according to time_index translated_gt_c = translate_center(gt_center, imu_data=imud[frame.item()], timestep=time_index, dt=input_config.dt, device=device) translated_infer_c = translate_center(inference_center, imu_data=imud[frame.item()], timestep=time_index, dt=input_config.dt, device=device) # TODO: Move these if-statements further up to reduce computation if eval_config.save_figs: if time_index == 0: fig = draw_gt_boxes3d(point_form_gt_3d_rot, fig, color=eval_config.gt_color) fig = draw_gt_boxes3d(inference_points_rot, fig, color=eval_config.infer_color, probabilities=inference_probabilities) else: fig = draw_points_3d(translated_gt_c, fig, color=eval_config.gt_color) fig = draw_points_3d(translated_infer_c, fig, color=eval_config.infer_color) if eval_config.save_figs: filename = "_".join( ("Infer", "seq", str(current_sequence), "f", str(frame.item()), timestr + ".png")) filepath = os.path.join(showroom_path, 'oracle_view', filename) filename_top = "_".join( ("Top_Infer", "seq", str(current_sequence), "f", str(frame.item()), timestr + ".png")) filepath_top = os.path.join(showroom_path, 'top_view', filename_top) mayavi.mlab.view(azimuth=90, elevation=0, focalpoint=[24., 0, 0], distance=120.0, figure=fig) mayavi.mlab.savefig(filepath_top, figure=fig) mayavi.mlab.view(azimuth=180, elevation=70, focalpoint=[12.0909996, -1.04700089, -2.03249991], distance=62.0, figure=fig) mayavi.mlab.savefig(filepath, figure=fig) if eval_config.use_gospa: # Save gospa scores in txt file file2 = open(showroom_path + "/gospa_scores_" + str(current_sequence).zfill(4) + ".txt", "a") row_str = str(frame.item()) for time_index in range(nT): row_str += (' ' + str(gospa_scores_dict[int(current_sequence), current_frame][time_index])) file2.write(row_str + "\n") file2.close() print('\t | time: {}'.format(round(time() - tic, 2)))
def train(): if os.path.exists('/home/mlt/mot/fafe/cfg/adams_computer'): config_path = 'cfg/cfg_mini.yml' else: config_path = 'cfg/cfg.yml' print('Using config: \n\t{}\n'.format(config_path)) config = load_config(config_path) input_config = InputConfig(config['INPUT_CONFIG']) train_config = TrainConfig(config['TRAIN_CONFIG']) loss_config = LossConfig(config['LOSS_CONFIG']) model_config = ModelConfig(config['MODEL_CONFIG']) verbose = train_config.verbose time_str = strftime("%Y-%m-%d_%H-%M") weights_filename = 'trained_models/' + time_str + '/weights_' + time_str if not os.path.exists('trained_models/' + time_str): os.mkdir('trained_models/' + time_str) print( 'Training weights will be saved to:\n\t{}\n'.format(weights_filename)) config_filename = 'trained_models/' + time_str + '/config_' + time_str + '.yml' save_config(config_filename, config) print('Config file saved to:\n\t{}\n'.format(config_filename)) if train_config.use_visdom: print( 'Dont forget to run "visdom" in a terminal in parallel to this in order to start the Visdom server' ) print('Choose port with "python -m visdom.server -port {}" \n'.format( train_config.visdom_port)) vis = visdom.Visdom( port=train_config.visdom_port) # port 8097 is default loss_window, sub_loss_window, recall_window, precision_window = viz.get_windows( vis, time_str) # Get root directory depending on which computer is running... # Don't forget to add your own path in 'fafe_utils.config_stuff.get_root_dir' root_dir = get_root_dir() ######################### # Define network ######################### if model_config.model == 'little_fafe': net = LittleFafe(input_config=input_config) else: net = FafeNet(input_config=input_config) print('Net set up successfully!') pytorch_total_params = sum(p.numel() for p in net.parameters()) pytorch_trainable_params = sum(p.numel() for p in net.parameters() if p.requires_grad) print("\tNumber of parameters: {}\n\tNumber of trainable parameters: {}". format(pytorch_total_params, pytorch_trainable_params)) # TODO: add posibility to load old weights ######################### # Set which device run on ######################### if train_config.use_cuda: # "If you load your samples in the Dataset on CPU and would like to push it during training to the GPU, # you can speed up the host to device transfer by enabling pin_memory." # - ptrblck [https://discuss.pytorch.org/t/when-to-set-pin-memory-to-true/19723] pin_memory = True if train_config.multi_gpu: device_ids = list(range(torch.cuda.device_count())) device = torch.device("cuda:" + str(device_ids[0])) net = net.to(device) net = nn.DataParallel(net) print( '\nUsing multiple GPUs.\n\tDevices: {}\n\tOutput device: {}\n'. format(device_ids, device)) else: device = torch.device("cuda:" + str(train_config.cuda_device)) print('\nUsing device {}\n'.format(device)) net = net.to(device) else: pin_memory = False device = torch.device("cpu") print('Using CPU\n') ######################### # Define loss function ######################### loss_func = FafeLoss(input_config, train_config, loss_config, device) if train_config.use_cuda: loss_func = loss_func.to(device) if train_config.multi_gpu: loss_func = nn.DataParallel(loss_func) ######################### # Define optimizer ######################### params = list(net.parameters()) + list(loss_func.parameters()) optimizer = optim.Adam(params, lr=train_config.learning_rate, weight_decay=train_config.weight_decay) print('Adams Optimizer set up with\n\tlr = {}\n\twd = {}\n'.format( train_config.learning_rate, train_config.weight_decay)) ######################### # Get Datasets ######################### print('Training Data:') training_dataloader = DataLoader(ConcatDataset([ TemporalBEVsDataset(input_config, root_dir, split='training', sequence=seq) for seq in train_config.training_seqs ]), batch_size=train_config.batch_size, shuffle=train_config.shuffle, num_workers=train_config.num_workers, pin_memory=pin_memory) print('Validation Data:') validation_dataloader = DataLoader(ConcatDataset([ TemporalBEVsDataset(input_config, root_dir, split='training', sequence=seq) for seq in train_config.validation_seqs ]), batch_size=train_config.batch_size, shuffle=train_config.shuffle, num_workers=train_config.num_workers, pin_memory=pin_memory) print('Data Loaders set up with:\n\tBatch size: {}\n\tNum Workers: {}'. format(train_config.batch_size, train_config.num_workers)) ############################### # Start training and evaluation ############################### print('\nTraining initiated [' + strftime("%Y-%m-%d %H:%M") + ']') for epoch in range(train_config.max_epochs): train_mean_loss, train_mean_recall, train_mean_precision, train_num_samples = 0, 0, 0, 0 eval_mean_loss, eval_mean_recall, eval_mean_precision, eval_num_samples = 0, 0, 0, 0 train_scaled_L1_mean, train_scaled_euler_mean, train_classification_loss = 0, 0, 0 eval_scaled_L1_mean, eval_scaled_euler_mean, eval_classification_loss = 0, 0, 0 ######################### # TRAINING ######################### tic = time() net.train() for i_batch, sample_batched in enumerate(training_dataloader): input, target, _ = sample_batched if train_config.use_cuda: input = input.to(device) target = target.to(device) # Always reset optimizer's gradient each iteration optimizer.zero_grad() if verbose: print('{} i: {} {}'.format('~' * 10, i_batch, '~' * 10)) print('Input shape: {}'.format(input.shape)) print('Target shape: {}'.format(target.shape)) print('Target: {}'.format(target[0][0][0][0])) # Forward propagation out_detection, out_regression = net.forward(input) if verbose: print('Output shape det: {}'.format(out_detection.shape)) print('Output shape reg: {}'.format(out_regression.shape)) # Calculate the loss loss, recall, precision, scaled_l1, scaled_euler, classification_loss = loss_func( out_detection, out_regression, target, verbose) # Back propagate loss.backward() # Update the weights optimizer.step() train_mean_loss += loss train_mean_recall += recall train_mean_precision += precision train_scaled_L1_mean += scaled_l1 train_scaled_euler_mean += scaled_euler train_classification_loss += classification_loss train_num_samples += 1 # Calculate the actual averages train_mean_loss /= train_num_samples train_mean_recall /= train_num_samples train_mean_precision /= train_num_samples train_scaled_L1_mean /= train_num_samples train_scaled_euler_mean /= train_num_samples train_classification_loss /= train_num_samples training_time = time() - tic ######################### # EVALUATION ######################### tic2 = time() net.eval() with torch.no_grad(): for i_batch, sample_batched in enumerate(validation_dataloader): input, target, _ = sample_batched if train_config.use_cuda: input = input.to(device) target = target.to(device) # Forward propagation out_detection, out_regression = net.forward(input) # Calculate the loss loss, recall, precision, scaled_l1, scaled_euler, classification_loss = loss_func( out_detection, out_regression, target, verbose) eval_mean_loss += loss eval_mean_recall += recall eval_mean_precision += precision eval_scaled_L1_mean += scaled_l1 eval_scaled_euler_mean += scaled_euler eval_classification_loss += classification_loss eval_num_samples += 1 eval_mean_loss /= eval_num_samples eval_mean_recall /= eval_num_samples eval_mean_precision /= eval_num_samples eval_scaled_L1_mean /= eval_num_samples eval_scaled_euler_mean /= eval_num_samples eval_classification_loss /= eval_num_samples eval_time = time() - tic2 total_time = time() - tic ######################### # PRINT STUFF ON SCREEN ######################### print('\nEpoch {} / {}\n{}\nCurrent time: {}'.format( epoch, train_config.max_epochs - 1, '-' * 12, strftime("%Y-%m-%d %H:%M"))) print('Epoch Total Time: {} s ({} + {})'.format( round(total_time, 2), round(training_time, 2), round(eval_time, 2))) print('Next Epoch ETA: ' + format(datetime.now() + timedelta(seconds=total_time), '%Y-%m-%d %H:%M')) print('Training ETA: ' + format( datetime.now() + timedelta( seconds=total_time * (train_config.max_epochs - epoch - 1)), '%Y-%m-%d %H:%M')) print('Train\n\tLoss: \t\t{}' '\n\t\tL1: \t{}' '\n\t\tEuler:\t{}' '\n\t\tCL: \t{}' '\n\tRecall: \t{}' '\n\tPrecision:\t{}'.format(train_mean_loss, train_scaled_L1_mean, train_scaled_euler_mean, train_classification_loss, train_mean_recall, train_mean_precision)) print('Validation\n\tLoss: \t\t{}' '\n\t\tL1: \t{}' '\n\t\tEuler:\t{}' '\n\t\tCL: \t{}' '\n\tRecall: \t{}' '\n\tPrecision:\t{}'.format(eval_mean_loss, eval_scaled_L1_mean, eval_scaled_euler_mean, eval_classification_loss, eval_mean_recall, eval_mean_precision)) if train_config.use_visdom: # Visualize Loss viz.push_data(epoch, vis, loss_window, sub_loss_window, recall_window, precision_window, train_mean_loss, eval_mean_loss, train_scaled_L1_mean, train_classification_loss, train_scaled_euler_mean, eval_scaled_euler_mean, eval_scaled_L1_mean, eval_classification_loss, train_mean_recall, eval_mean_recall, train_mean_precision, eval_mean_precision) ######################### # SAVE WEIGHTS (every save_weights_modulus th epoch) ######################### if epoch % train_config.save_weights_modulus == 0 or epoch == train_config.max_epochs - 1: save_filename = weights_filename + '_epoch_' + str(epoch) torch.save( { 'epoch': epoch, 'model_state_dict': net.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'loss': loss }, save_filename) print('Training Complete [' + strftime("%Y-%m-%d %H:%M") + ']')
def eval_post_fafe_pp(fafe_model_path, pp_model_path, data_path, config_path, sequence, kitti): """ :param self: :param fafe_model_path: :param pp_model_path: :param data_path: :param config_path: :return: """ timestr = strftime("%Y-%m-%d_%H:%M") velo_path = os.path.join(data_path, 'training', 'velodyne') config = load_config(config_path) start_time = time() input_config = InputConfig(config['INPUT_CONFIG']) train_config = TrainConfig(config['TRAIN_CONFIG']) eval_config = EvalConfig(config['EVAL_CONFIG']) model_config = ModelConfig(config['MODEL_CONFIG']) post_config = PostConfig(config['POST_CONFIG']) pillar = PillarOfFafe(input_config, train_config.batch_size, train_config.verbose) if model_config.model == 'little_fafe': fafe = LittleFafe(input_config) else: fafe = FafeNet(input_config) if eval_config.use_cuda: if eval_config.cuda_device == 0: device = torch.device("cuda:0") print('Using CUDA:{}\n'.format(0)) elif eval_config.cuda_device == 1: device = torch.device("cuda:1") print('Using CUDA:{}\n'.format(1)) else: print('Functionality for CUDA device cuda:{} not yet implemented.'. format(eval_config.cuda_device)) print('Using cuda:0 instead...\n') device = torch.device("cuda:0") pillar = pillar.to(device) fafe = fafe.to(device) else: device = torch.device("cpu") print('Using CPU\n') imu_path = os.path.join(data_path, 'training', 'oxts') pillar.load_state_dict( torch.load( pp_model_path, map_location=lambda storage, loc: storage)["model_state_dict"]) fafe.load_state_dict( torch.load( fafe_model_path, map_location=lambda storage, loc: storage)["model_state_dict"]) post_fafe = PostFafe(input_config, post_config, device) pillar.eval() fafe.eval() print("Model loaded, loading time: {}".format(round( time() - start_time, 2))) timestr = strftime("%Y-%m-%d_%H:%M") eval_head = FafePredict(input_config, eval_config) eval_head.eval() print("Evalutation Model Loaded!") # for seq in eval_config.validation_seqs: testing_dataset = VoxelDataset(input_config, root_dir=data_path, split='training', sequence=sequence) testing_loader = DataLoader(dataset=testing_dataset, batch_size=eval_config.batch_size, num_workers=eval_config.num_workers, shuffle=False) data = [] frames = [] total_time_per_iteration = [] for batch_idx, batch in enumerate(testing_loader): # Create Pillar Pseudo Img voxel_stack, coord_stack, num_points_stack, num_nonempty_voxels, target, info = batch nB = target.shape[0] nT = input_config.num_conseq_frames current_sequence = sequence frame = info["GT_indices"][0] if type(frame) == torch.Tensor: frame = frame.item() d = {} d['current_time'] = frame d['frame_id'] = frame # To match pmbm's data structure... frames.append(frame) imud = load_imu(imu_path, current_sequence) true_states = kitti.get_bev_states(frame, classes_to_track=['Car', 'Van']) d['true_states'] = true_states d['state_dims'] = 3 # Move all input data to the correct device if not using CPU if train_config.use_cuda: voxel_stack = voxel_stack.to(device) coord_stack = coord_stack.to(device) num_points_stack = num_points_stack.to(device) num_nonempty_voxels = num_nonempty_voxels.to(device) target = target.to(device) timeit = time() with torch.no_grad(): pseudo_stack = [] for time_iter in range(input_config.num_conseq_frames): pseudo_image = pillar(voxel_stack[:, time_iter], num_points_stack[:, time_iter], coord_stack[:, time_iter], num_nonempty_voxels[:, time_iter]) if input_config.pp_verbose: print("Pseudo_img: {}".format( pseudo_image.unsqueeze(1).shape)) pseudo_stack.append(pseudo_image.unsqueeze(1)) pseudo_torch = torch.cat(pseudo_stack, dim=1) if train_config.use_cuda: pseudo_torch = pseudo_torch.to(device) target = target.to(device) # Forward propagation. Reshape by squishing together Time and Channel dimensions. # Reshaping basically as we do with BEV, stacking them on top of eachother. out_detection, out_regression = fafe.forward( pseudo_torch.reshape( -1, pseudo_torch.shape[1] * pseudo_torch.shape[2], pseudo_torch.shape[-2], pseudo_torch.shape[-1])) inference = eval_head(out_detection, out_regression) inference_time = round(time() - timeit, 2) if torch.is_tensor(current_sequence): current_sequence = current_sequence.item() meas_ = inference[0][0] meas = [] for row in range(meas_.shape[0]): meas.append(transf(meas_[0].to('cpu').numpy())) d['measurements'] = meas if len(inference) > 1: raise ValueError("Batch index > 1. Do something god damnit") # Call post_fafe to infer the inference post_fafe(inference[0]) imu_data = imud[frame] # Add targets and predictions to data dict # TID = Target ID estimated_targets = [] for tid, tensor_state in post_fafe.object_state.items(): continue_bool = 0 if post_config.verbose: print("Object: {} \n\t{}".format(tid, tensor_state)) _temp = {} _temp['target_idx'] = tid _temp['object_class'] = 'car' _state = tensor_state[0].numpy() single_target = SingleTargetHypothesis(transf(_state)) _temp['single_target'] = single_target _temp['state_predictions'] = [] _temp['var_predictions'] = [] for i, step in enumerate(tensor_state): if step is None or continue_bool: continue_bool = 1 continue # Coordinate transform predictions preds = transf(step.numpy()) if post_config.coordinate_transform: preds = translate_center(preds, imu_data, timestep=i, dt=input_config.dt) _temp['state_predictions'].append(preds) if post_config.verbose: print("Step: {}".format(transf(step.numpy()))) estimated_targets.append(_temp) d['estimated_targets'] = estimated_targets data.append(d) total_time_per_iteration.append(inference_time) print("{}({}s,{}#), ".format(frame, inference_time, len(estimated_targets)), end='') return data, total_time_per_iteration
def eval_post_fafe(fafe_model_path, data_path, config_path, sequence, kitti): """ :param self: :param fafe_model_path: :param pp_model_path: :param data_path: :param config_path: :return: """ timestr = strftime("%Y-%m-%d_%H:%M") config = load_config(config_path) start_time = time() input_config = InputConfig(config['INPUT_CONFIG']) train_config = TrainConfig(config['TRAIN_CONFIG']) eval_config = EvalConfig(config['EVAL_CONFIG']) model_config = ModelConfig(config['MODEL_CONFIG']) post_config = PostConfig(config['POST_CONFIG']) pillar = PillarOfFafe(input_config, train_config.batch_size, train_config.verbose) if model_config.model == 'little_fafe': fafe = LittleFafe(input_config) else: fafe = FafeNet(input_config) if eval_config.use_cuda: if eval_config.cuda_device == 0: device = torch.device("cuda:0") print('Using CUDA:{}\n'.format(0)) elif eval_config.cuda_device == 1: device = torch.device("cuda:1") print('Using CUDA:{}\n'.format(1)) else: print('Functionality for CUDA device cuda:{} not yet implemented.'.format(eval_config.cuda_device)) print('Using cuda:0 instead...\n') device = torch.device("cuda:0") pillar = pillar.to(device) fafe = fafe.to(device) else: device = torch.device("cpu") print('Using CPU\n') imu_path = os.path.join(data_path, 'training', 'oxts') fafe.load_state_dict(torch.load(fafe_model_path, map_location=lambda storage, loc: storage)["model_state_dict"]) post_fafe = PostFafe(input_config, post_config, device) pillar.eval() fafe.eval() print("Model loaded, loading time: {}".format(round(time() - start_time, 2))) timestr = strftime("%Y-%m-%d_%H:%M") eval_head = FafePredict(input_config, eval_config) eval_head.eval() print("Evalutation Model Loaded!") # for seq in eval_config.validation_seqs: testing_dataset = TemporalBEVsDataset(input_config, root_dir=data_path, split='training', sequence=sequence) testing_loader = DataLoader(dataset=testing_dataset, batch_size=eval_config.batch_size, num_workers=eval_config.num_workers, shuffle=False) data = [] frames = [] print("Starting to run over sequence {}. Total number of frames: {}".format(sequence, len(testing_loader))) total_time_per_iteration = [] for batch_idx, batch in enumerate(testing_loader): input, target, info = batch if eval_config.use_cuda: input = input.to(device) target = target.to(device) timeit = time() with torch.no_grad(): out_det, out_reg = fafe(input) inference = eval_head(out_det, out_reg) inference_time = round(time() - timeit, 2) nB = target.shape[0] nT = input_config.num_conseq_frames current_sequence = sequence frame = info["GT_indices"][0] if type(frame) == torch.Tensor: frame = frame.item() d = {} d['current_time'] = frame d['frame_id'] = frame # To match pmbm's data structure... frames.append(frame) imud = load_imu(imu_path, current_sequence) true_states = kitti.get_bev_states(frame, classes_to_track=['Car', 'Van']) d['true_states'] = true_states d['state_dims'] = 3 if torch.is_tensor(current_sequence): current_sequence = current_sequence.item() meas_ = inference[0][0] meas = [] for row in range(meas_.shape[0]): meas.append(transf(meas_[0].to('cpu').numpy())) d['measurements'] = meas if len(inference) > 1: raise ValueError("Batch index > 1. Do something god damnit") # Call post_fafe to infer the inference post_fafe(inference[0]) imu_data = imud[frame] # Add targets and predictions to data dict # TID = Target ID estimated_targets = [] for tid, tensor_state in post_fafe.object_state.items(): continue_bool = 0 if post_config.verbose: print("Object: {} \n\t{}".format(tid, tensor_state)) _temp = {} _temp['target_idx'] = tid _temp['object_class'] = 'car' _state = tensor_state[0].numpy() single_target = SingleTargetHypothesis(transf(_state)) _temp['single_target'] = single_target _temp['state_predictions'] = [] _temp['var_predictions'] = [] for i, step in enumerate(tensor_state): if step is None or continue_bool: continue_bool = 1 continue # Coordinate transform predictions preds = transf(step.numpy()) if post_config.coordinate_transform: preds = translate_center(preds, imu_data, timestep=i, dt=input_config.dt ) _temp['state_predictions'].append(preds) if post_config.verbose: print("Step: {}".format(transf(step.numpy()))) estimated_targets.append(_temp) d['estimated_targets'] = estimated_targets data.append(d) total_time_per_iteration.append(inference_time) print("{}({}s,{}#), ".format(frame, inference_time, len(estimated_targets)),end='') return data, total_time_per_iteration