Exemplo n.º 1
0
    def test_data_structure(self):
        if os.path.exists('/home/mlt/mot/fafe/cfg/adams_computer'):
            config_path = 'cfg/cfg_pp_mini.yml'
        elif os.path.exists('/Users/erikbohnsack'):
            config_path = 'cfg/cfg_mac.yml'
        else:
            config_path = 'cfg/cfg_pp.yml'

        print('Using config: \n\t{}\n'.format(config_path))
        config = load_config(config_path)
        input_config = InputConfig(config['INPUT_CONFIG'])
        post_config = PostConfig(config['POST_CONFIG'])
        data = torch.load('inference0.pt')
        data1 = torch.load('inference1.pt')
        data2 = torch.load('inference2.pt')

        device = torch.device("cpu")

        post_fafe = PostFafe(input_config, post_config, device)
        print(post_fafe.object_state)
        post_fafe(data[0])
        print(post_fafe.object_state.keys())
        post_fafe(data1[0])
        print(post_fafe.object_state.keys())
        post_fafe(data2[0])
        print(post_fafe.object_state.keys())
Exemplo n.º 2
0
    def test_plot_bevs(self):
        config_path = 'cfg_mini.yml'
        config = load_config(config_path)

        input_config = InputConfig(config['INPUT_CONFIG'])
        train_config = TrainConfig(config['TRAIN_CONFIG'])
        basepath = "/Users/erikbohnsack/data/training"

        tracking = pykitti.tracking(base_path=basepath, sequence="0000")
        points = tracking.get_velo(0)
        bev_map = np.zeros(input_config.bev_shape)
        print("BEV map size: {}".format(bev_map.shape))

        grid_limits = np.array([[input_config.x_min, input_config.x_max],
                                [input_config.y_min, input_config.y_max],
                                [input_config.z_min, input_config.z_max]])
        grid_sizes = np.array([
            input_config.x_grid_size, input_config.y_grid_size,
            input_config.z_grid_size
        ])

        start_jit = time.time()
        kitti_dataset.format_to_BEV_jit(bev_map, points, grid_limits,
                                        grid_sizes)
        end_jit = time.time()

        plot_BEV(bev_map)
        print("Elapsed time: {}".format(end_jit - start_jit))
Exemplo n.º 3
0
def eval(model_path, data_path):
    config_path = 'cfg/cfg_mini.yml'
    config = load_config(config_path)

    input_config = InputConfig(config['INPUT_CONFIG'])
    eval_config = EvalConfig(config['EVAL_CONFIG'])
    model = FafeNet(input_config)

    model.load_state_dict(torch.load(model_path)["model_state_dict"])
    model.eval()
    print("Model loaded")
    eval_head = FafePredict(input_config, eval_config)
    testing_seqs = [0]  # TODO: Fix

    testing_datasets = [
        TestDataset(input_config,
                    root_dir=data_path,
                    split='testing',
                    sequence=seq) for seq in testing_seqs
    ]

    testing_dataset = ConcatDataset(testing_datasets)
    testing_loader = DataLoader(dataset=testing_dataset,
                                batch_size=eval_config.batch_size,
                                num_workers=eval_config.num_workers)

    for batch_index, batch in enumerate(testing_loader):
        input, info = batch
        out_det, out_reg = model(input)
        inference = eval_head(out_det, out_reg)
Exemplo n.º 4
0
    def test_concat_dataset(self):
        config_path = 'cfg_mini.yml'
        config = load_config(config_path)

        input_config = InputConfig(config['INPUT_CONFIG'])
        train_config = TrainConfig(config['TRAIN_CONFIG'])

        if os.path.exists("/Users/erikbohnsack/data/"):
            root_dir = "/Users/erikbohnsack/data/"
            sequence = 0
        else:
            root_dir = "/home/mlt/data/"
            sequence = 3

        training_datasets = [
            TemporalBEVsDataset(input_config,
                                root_dir,
                                split='training',
                                sequence=seq)
            for seq in train_config.training_seqs
        ]
        validation_datasets = [
            TemporalBEVsDataset(input_config,
                                root_dir,
                                split='training',
                                sequence=seq)
            for seq in train_config.validation_seqs
        ]  # Split still training due to data structure

        training_dataset = ConcatDataset(training_datasets)
        validation_dataset = ConcatDataset(validation_datasets)

        training_dataloader = DataLoader(training_dataset,
                                         batch_size=train_config.batch_size,
                                         shuffle=train_config.shuffle,
                                         num_workers=train_config.num_workers)
        validation_dataloader = DataLoader(
            validation_dataset,
            batch_size=train_config.batch_size,
            shuffle=train_config.shuffle,
            num_workers=train_config.num_workers)
        print("----------------------- TRAINING -----------------------")
        for i_batch, batch_sample in enumerate(training_dataloader):
            print(i_batch)
            input, target, info = batch_sample
            print("Input shape: {}".format(input.shape))
            print("Sequence: {}, real index: {}".format(
                info["sequence"], info["Current_index"]))
        print("----------------------- VALIDATION -----------------------")
        for i_batch, batch_sample in enumerate(validation_dataloader):
            print(i_batch)
            input, target, info = batch_sample
            print("Input shape: {}".format(input.shape))
            print("Sequence: {}, real index: {}".format(
                info["sequence"], info["Current_index"]))
Exemplo n.º 5
0
    def test_eval(self):
        save_filename = 'foo'
        config_path = 'cfg.yml'
        config = load_config(config_path)

        input_config = InputConfig(config['INPUT_CONFIG'])
        model = FafeNet(config)
        torch.save({'model_state_dict': model.state_dict()}, save_filename)

        model_path = save_filename
        data_path = '/Users/erikbohnsack/data'
        eval(model_path=model_path, data_path=data_path)
Exemplo n.º 6
0
    def test_train_with_model(self):

        root_dir = get_root_dir()

        if os.path.exists(
                '/Users/erikbohnsack/Code/MOT/fafe/trained_models/weights_2019-04-12_13_43_epoch_85'
        ):
            model_path = '/Users/erikbohnsack/Code/MOT/fafe/trained_models/weights_2019-04-12_13_43_epoch_85'
            config_path = 'cfg/cfg_mac.yml'
        else:
            model_path = '/home/mlt/mot/fafe/trained_models/test_train_2019-04-11_14_01_epoch_40'
            config_path = 'cfg/cfg_mini.yml'

        config = load_config(config_path)

        input_config = InputConfig(config['INPUT_CONFIG'])
        train_config = TrainConfig(config['TRAIN_CONFIG'])

        verbose = train_config.verbose

        net = FafeNet(input_config)
        net.load_state_dict(
            torch.load(
                model_path,
                map_location=lambda storage, loc: storage)["model_state_dict"])

        #########################
        # Set which device run on
        #########################
        if train_config.use_cuda:
            if train_config.cuda_device == 0:
                device = torch.device("cuda:0")
                print('Using CUDA:{}\n'.format(0))
            elif train_config.cuda_device == 1:
                device = torch.device("cuda:1")
                print('Using CUDA:{}\n'.format(1))
            else:
                print(
                    'Functionality for CUDA device cuda:{} not yet implemented.'
                    .format(train_config.cuda_device))
                print('Using cuda:0 instead...\n')
                device = torch.device("cuda:0")
            net = net.to(device)
        else:
            device = torch.device("cpu")
            print('Using CPU\n')

        #########################
        # Define optimizer
        #########################
        optimizer = optim.Adam(net.parameters(),
                               lr=train_config.learning_rate,
                               weight_decay=train_config.weight_decay)
        print('Adams Optimizer set up with\n\tlr = {}\n\twd = {}\n'.format(
            train_config.learning_rate, train_config.weight_decay))

        loss_func = FafeLoss(input_config, train_config, device)

        training_datasets = [
            TemporalBEVsDataset(input_config,
                                root_dir,
                                split='training',
                                sequence=seq)
            for seq in train_config.training_seqs
        ]
        validation_datasets = [
            TemporalBEVsDataset(input_config,
                                root_dir,
                                split='training',
                                sequence=seq)
            for seq in train_config.validation_seqs
        ]  # Split still training due to data structure

        training_dataset = ConcatDataset(training_datasets)
        validation_dataset = ConcatDataset(validation_datasets)

        training_dataloader = DataLoader(training_dataset,
                                         batch_size=train_config.batch_size,
                                         shuffle=train_config.shuffle,
                                         num_workers=train_config.num_workers)
        validation_dataloader = DataLoader(
            validation_dataset,
            batch_size=train_config.batch_size,
            shuffle=train_config.shuffle,
            num_workers=train_config.num_workers)
        ###############################
        # Start training and evaluation
        ###############################
        print('\nTraining initiated [' + strftime("%Y-%m-%d %H:%M") + ']')
        for epoch in range(train_config.max_epochs):
            train_mean_loss, train_mean_recall, train_mean_precision, train_num_samples = 0, 0, 0, 0
            eval_mean_loss, eval_mean_recall, eval_mean_precision, eval_num_samples = 0, 0, 0, 0
            train_scaled_L1_mean, train_classification_loss = 0, 0
            eval_scaled_L1_mean, eval_classification_loss = 0, 0

            #########################
            # TRAINING
            #########################
            tic = time()
            net.train()
            for i_batch, sample_batched in enumerate(training_dataloader):
                input, target, _ = sample_batched

                if train_config.use_cuda:
                    input = input.to(device)
                    target = target.to(device)

                # Always reset optimizer's gradient each iteration
                optimizer.zero_grad()

                if verbose:
                    print('{} i: {} {}'.format('~' * 10, i_batch, '~' * 10))
                    print('Input shape: {}'.format(input.shape))
                    print('Target shape: {}'.format(target.shape))
                    print('Target: {}'.format(target[0][0][0][0]))

                # Forward propagation
                out_detection, out_regression = net.forward(input)

                if verbose:
                    print('Output shape det: {}'.format(out_detection.shape))
                    print('Output shape reg: {}'.format(out_regression.shape))

                # Calculate the loss
                loss, recall, precision, scaled_L1, scaled_euler, classification_loss = loss_func(
                    out_detection, out_regression, target, verbose)

                # Back propagate
                loss.backward()

                # Update the weights
                optimizer.step()

                train_mean_loss += loss
                train_mean_recall += recall
                train_mean_precision += precision
                train_scaled_L1_mean += scaled_L1
                train_classification_loss += classification_loss
                train_num_samples += 1
            # Calculate the actual averages
            train_mean_loss /= train_num_samples
            train_mean_recall /= train_num_samples
            train_mean_precision /= train_num_samples
            train_scaled_L1_mean /= train_num_samples
            train_classification_loss /= train_num_samples
            training_time = time() - tic
Exemplo n.º 7
0
def train():
    if os.path.exists('/home/mlt/mot/fafe/cfg/adams_computer'):
        config_path = 'cfg/cfg_pp_mini.yml'
    elif os.path.exists('/Users/erikbohnsack'):
        config_path = 'cfg/cfg_mac.yml'
    else:
        config_path = 'cfg/cfg_pp.yml'

    print('Using config: \n\t{}\n'.format(config_path))
    config = load_config(config_path)
    input_config = InputConfig(config["INPUT_CONFIG"])
    train_config = TrainConfig(config["TRAIN_CONFIG"])
    loss_config = LossConfig(config['LOSS_CONFIG'])
    model_config = ModelConfig(config['MODEL_CONFIG'])

    verbose = train_config.verbose

    time_str = strftime("%Y-%m-%d_%H-%M")
    weights_filename = 'trained_models/' + time_str + '/weights_' + time_str

    showroom_path = get_showroom_path(model_path="_".join(
        ('weights', time_str)),
                                      full_path_bool=False)
    if not os.path.exists('trained_models/' + time_str):
        os.mkdir('trained_models/' + time_str)

    print(
        'Training weights will be saved to:\n\t{}\n'.format(weights_filename))

    config_filename = 'trained_models/' + time_str + '/config_' + time_str + '.yml'
    save_config(config_filename, config)
    print('Config file saved to:\n\t{}\n'.format(config_filename))

    if train_config.use_visdom:
        print(
            'Dont forget to run "visdom" in a terminal in parallel to this in order to start the Visdom server'
        )
        print('Choose port with "python -m visdom.server -port {}" \n'.format(
            train_config.visdom_port))
        vis = visdom.Visdom(
            port=train_config.visdom_port)  # port 8097 is default
        loss_window, sub_loss_window, recall_window, precision_window = viz.get_windows(
            vis, time_str)
    print("~" * 20)
    print("Setting up net")
    pillar = PillarOfFafe(input_config=input_config,
                          batch_size=train_config.batch_size,
                          verbose=input_config.pp_verbose)

    if model_config.model == 'little_fafe':
        fafe = LittleFafe(input_config=input_config)
    else:
        fafe = FafeNet(input_config=input_config)

    #########################
    # Set which device run on
    #########################
    if train_config.use_cuda:
        # "If you load your samples in the Dataset on CPU and would like to push it during training to the GPU,
        # you can speed up the host to device transfer by enabling pin_memory."
        # - ptrblck [https://discuss.pytorch.org/t/when-to-set-pin-memory-to-true/19723]
        pin_memory = False
        device = torch.device("cuda:" + str(train_config.cuda_device))
        print('\nUsing device {}\n'.format(device))
        pillar = pillar.to(device)
        fafe = fafe.to(device)
    else:
        pin_memory = False
        device = torch.device("cpu")
        print('Using CPU\n')

    loss_func = FafeLoss(input_config, train_config, loss_config, device)
    if train_config.use_cuda:
        loss_func = loss_func.to(device)

    print("Net set up successfully!")
    pp_pytorch_total_params = sum(p.numel() for p in pillar.parameters())
    pp_pytorch_trainable_params = sum(p.numel() for p in pillar.parameters()
                                      if p.requires_grad)
    print("~" * 20)
    print(
        "PP:\n\tNumber of parameters: {}\n\tNumber of trainable parameters: {}"
        .format(pp_pytorch_total_params, pp_pytorch_trainable_params))

    pytorch_total_params = sum(p.numel() for p in fafe.parameters())
    pytorch_trainable_params = sum(p.numel() for p in fafe.parameters()
                                   if p.requires_grad)
    print(
        "FAFE:\n\tNumber of parameters: {}\n\tNumber of trainable parameters: {}"
        .format(pytorch_total_params, pytorch_trainable_params))
    root_dir = get_root_dir()

    #########################
    # Define optimizer
    #########################
    params = list(pillar.parameters()) + list(fafe.parameters()) + list(
        loss_func.parameters())
    optimizer = optim.Adam(params,
                           lr=train_config.learning_rate,
                           weight_decay=train_config.weight_decay)
    print('Adams Optimizer set up with\n\tlr = {}\n\twd = {}\n'.format(
        train_config.learning_rate, train_config.weight_decay))

    #########################
    # Get Datasets
    #########################
    print('Training Data:')
    # fafe_sampler = FafeSampler(data_source=dataset, input_config=input_config)
    # TODO: Sampler needs a *data_source* as input to know the length of objects it can iterate over.
    # TODO: When concatenating

    training_dataloader = DataLoader(ConcatDataset([
        VoxelDataset(input_config, root_dir, split='training', sequence=seq)
        for seq in train_config.training_seqs
    ]),
                                     batch_size=train_config.batch_size,
                                     shuffle=train_config.shuffle,
                                     num_workers=train_config.num_workers,
                                     pin_memory=pin_memory)
    print('Validation Data:')
    validation_dataloader = DataLoader(ConcatDataset([
        VoxelDataset(input_config, root_dir, split='training', sequence=seq)
        for seq in train_config.validation_seqs
    ]),
                                       batch_size=train_config.batch_size,
                                       shuffle=train_config.shuffle,
                                       num_workers=train_config.num_workers,
                                       pin_memory=pin_memory)
    print('Data Loaders set up with:\n\tBatch size:  {}\n\tNum Workers: {}'.
          format(train_config.batch_size, train_config.num_workers))

    ###############################
    # Start training and evaluation
    ###############################
    print('\nTraining initiated [' + strftime("%Y-%m-%d %H:%M") + ']')
    for epoch in range(train_config.max_epochs):
        train_mean_loss, train_mean_recall, train_mean_precision, train_num_samples = 0, 0, 0, 0
        eval_mean_loss, eval_mean_recall, eval_mean_precision, eval_num_samples = 0, 0, 0, 0
        train_scaled_reg_mean, train_scaled_euler_mean, train_classification_loss = 0, 0, 0
        eval_scaled_reg_mean, eval_scaled_euler_mean, eval_classification_loss = 0, 0, 0

        #########################
        # TRAINING
        #########################
        tic = time()
        pillar.train()
        fafe.train()
        torch.set_grad_enabled(True)
        for i_batch, batch in enumerate(training_dataloader):

            # Always reset optimizer's gradient each iteration
            optimizer.zero_grad()

            # Create Pillar Pseudo Img
            voxel_stack, coord_stack, num_points_stack, num_nonempty_voxels, target, info = batch

            # Move all input data to the correct device if not using CPU
            if train_config.use_cuda:
                voxel_stack = voxel_stack.to(device)
                coord_stack = coord_stack.to(device)
                num_points_stack = num_points_stack.to(device)
                num_nonempty_voxels = num_nonempty_voxels.to(device)
                target = target.to(device)

            pseudo_stack = []
            for time_iter in range(input_config.num_conseq_frames):
                if input_config.pp_verbose:
                    print("~" * 20)
                    print("time iter: {}".format(time_iter))
                    print("voxels \n\tshape: {}".format(
                        voxel_stack[:, time_iter].shape))
                    print("coord \n\tshape: {}".format(
                        coord_stack[:, time_iter].shape))
                    print("num_points \n\tshape: {}".format(
                        num_points_stack[:, time_iter].shape))
                    print("num_nonempty_voxels \n\tshape: {}".format(
                        num_nonempty_voxels[:, time_iter].shape))

                pseudo_image = pillar(voxel_stack[:, time_iter],
                                      num_points_stack[:, time_iter],
                                      coord_stack[:, time_iter],
                                      num_nonempty_voxels[:, time_iter])

                if input_config.pp_verbose:
                    print("Pseudo_img: {}".format(
                        pseudo_image.unsqueeze(1).shape))
                pseudo_stack.append(pseudo_image.unsqueeze(1))

            pseudo_torch = torch.cat(pseudo_stack, dim=1)
            if input_config.pp_verbose:
                print("Pseudo stacked over time: \n\t{}".format(
                    pseudo_torch.shape))
            if train_config.use_cuda:
                pseudo_torch = pseudo_torch.to(device)
                target = target.to(device)

            # Forward propagation. Reshape by squishing together Time and Channel dimensions.
            # Reshaping basically as we do with BEV, stacking them on top of eachother.
            out_detection, out_regression = fafe.forward(
                pseudo_torch.reshape(
                    -1, pseudo_torch.shape[1] * pseudo_torch.shape[2],
                    pseudo_torch.shape[-2], pseudo_torch.shape[-1]))

            # Calculate the loss
            loss, recall, precision, scaled_reg, scaled_euler, classification_loss = loss_func(
                out_detection, out_regression, target, verbose)
            # Back propagate
            loss.backward()

            if train_config.plot_grad_flow:
                plot_grad_flow(
                    pillar.named_parameters(),
                    os.path.join(
                        showroom_path, 'grad_flow_pillar', "".join(
                            ("epoch_", str(epoch).zfill(4), "_batch_",
                             str(i_batch).zfill(4), ".png"))))
                plot_grad_flow(
                    fafe.named_parameters(),
                    os.path.join(
                        showroom_path, 'grad_flow_fafe', "".join(
                            ("epoch_", str(epoch).zfill(4), "_batch_",
                             str(i_batch).zfill(4), ".png"))))

            # Update the weights
            optimizer.step()

            train_mean_loss += loss
            train_mean_recall += recall
            train_mean_precision += precision
            train_scaled_reg_mean += scaled_reg
            train_scaled_euler_mean += scaled_euler
            train_classification_loss += classification_loss
            train_num_samples += 1

        # Calculate the actual averages
        train_mean_loss /= train_num_samples
        train_mean_recall /= train_num_samples
        train_mean_precision /= train_num_samples
        train_scaled_reg_mean /= train_num_samples
        train_scaled_euler_mean /= train_num_samples
        train_classification_loss /= train_num_samples
        training_time = time() - tic

        #########################
        # EVALUATION
        #########################
        tic2 = time()
        pillar.eval()
        fafe.eval()
        with torch.no_grad():
            for i_batch, batch in enumerate(validation_dataloader):
                # Create Pillar Pseudo Img
                voxel_stack, coord_stack, num_points_stack, num_nonempty_voxels, target, index = batch

                # Move all input data to the correct device if not using CPU
                if train_config.use_cuda:
                    voxel_stack = voxel_stack.to(device)
                    coord_stack = coord_stack.to(device)
                    num_points_stack = num_points_stack.to(device)
                    num_nonempty_voxels = num_nonempty_voxels.to(device)
                    target = target.to(device)

                pseudo_stack = []
                for time_iter in range(input_config.num_conseq_frames):
                    if train_config.verbose:
                        print("~" * 20)
                        print("time iter: {}".format(time_iter))
                        print("voxels \n\tshape: {}".format(
                            voxel_stack[:, time_iter].shape))
                        print("coord \n\tshape: {}".format(
                            coord_stack[:, time_iter].shape))
                        print("num_points \n\tshape: {}".format(
                            num_points_stack[:, time_iter].shape))
                        print("num_nonempty_voxels \n\tshape: {}".format(
                            num_nonempty_voxels[:, time_iter].shape))

                    pseudo_image = pillar(voxel_stack[:, time_iter],
                                          num_points_stack[:, time_iter],
                                          coord_stack[:, time_iter],
                                          num_nonempty_voxels[:, time_iter])

                    if train_config.verbose:
                        print("Pseudo_img: {}".format(
                            pseudo_image.unsqueeze(1).shape))
                    pseudo_stack.append(pseudo_image.unsqueeze(1))

                pseudo_torch = torch.cat(pseudo_stack, dim=1)
                if train_config.use_cuda:
                    pseudo_torch = pseudo_torch.to(device)
                    target = target.to(device)

                # Forward propagation
                out_detection, out_regression = fafe.forward(
                    pseudo_torch.reshape(
                        -1, pseudo_torch.shape[1] * pseudo_torch.shape[2],
                        pseudo_torch.shape[-2], pseudo_torch.shape[-1]))
                # Calculate the loss
                loss, recall, precision, scaled_reg, scaled_euler, classification_loss = loss_func(
                    out_detection, out_regression, target, verbose)
                eval_mean_loss += loss
                eval_mean_recall += recall
                eval_mean_precision += precision
                eval_scaled_reg_mean += scaled_reg
                eval_scaled_euler_mean += scaled_euler
                eval_classification_loss += classification_loss
                eval_num_samples += 1
        eval_mean_loss /= eval_num_samples
        eval_mean_recall /= eval_num_samples
        eval_mean_precision /= eval_num_samples
        eval_scaled_reg_mean /= eval_num_samples
        eval_scaled_euler_mean /= eval_num_samples
        eval_classification_loss /= eval_num_samples

        eval_time = time() - tic2
        total_time = time() - tic

        #########################
        # PRINT STUFF ON SCREEN
        #########################
        print('\nEpoch {} / {}\n{}\nCurrent time: {}'.format(
            epoch, train_config.max_epochs - 1, '-' * 12,
            strftime("%Y-%m-%d %H:%M")))
        print('Epoch Total Time: {} s ({} + {})'.format(
            round(total_time, 2), round(training_time, 2), round(eval_time,
                                                                 2)))
        print('Next Epoch ETA: ' +
              format(datetime.now() +
                     timedelta(seconds=total_time), '%Y-%m-%d %H:%M'))
        print('Training ETA: ' + format(
            datetime.now() + timedelta(
                seconds=total_time *
                (train_config.max_epochs - epoch - 1)), '%Y-%m-%d %H:%M'))
        print('Train\n\tLoss: \t\t{}'
              '\n\t\tL1:  \t{}'
              '\n\t\tEuler:\t{}'
              '\n\t\tCL:  \t{}'
              '\n\tRecall: \t{}'
              '\n\tPrecision:\t{}'.format(train_mean_loss,
                                          train_scaled_reg_mean,
                                          train_scaled_euler_mean,
                                          train_classification_loss,
                                          train_mean_recall,
                                          train_mean_precision))
        print('Validation\n\tLoss: \t\t{}'
              '\n\t\tL1:  \t{}'
              '\n\t\tEuler:\t{}'
              '\n\t\tCL:  \t{}'
              '\n\tRecall: \t{}'
              '\n\tPrecision:\t{}'.format(eval_mean_loss, eval_scaled_reg_mean,
                                          eval_scaled_euler_mean,
                                          eval_classification_loss,
                                          eval_mean_recall,
                                          eval_mean_precision))
        if train_config.use_visdom:
            # Visualize Loss
            viz.push_data(epoch, vis, loss_window, sub_loss_window,
                          recall_window, precision_window, train_mean_loss,
                          eval_mean_loss, train_scaled_reg_mean,
                          train_classification_loss, train_scaled_euler_mean,
                          eval_scaled_euler_mean, eval_scaled_reg_mean,
                          eval_classification_loss, train_mean_recall,
                          eval_mean_recall, train_mean_precision,
                          eval_mean_precision)

        ####################################################
        # SAVE WEIGHTS (every save_weights_modulus th epoch)
        ####################################################
        if epoch % train_config.save_weights_modulus == 0 or epoch == train_config.max_epochs - 1:
            save_filename = weights_filename + '_epoch_' + str(epoch)

            pp_fn = save_filename + '_pp'
            torch.save(
                {
                    'epoch': epoch,
                    'model_state_dict': pillar.state_dict(),
                    'optimizer_state_dict': optimizer.state_dict(),
                    'loss': loss
                }, pp_fn)

            fafe_fn = save_filename + '_fafe'
            torch.save(
                {
                    'epoch': epoch,
                    'model_state_dict': fafe.state_dict(),
                    'optimizer_state_dict': optimizer.state_dict(),
                    'loss': loss
                }, fafe_fn)

    print('Training Complete [' + strftime("%Y-%m-%d %H:%M") + ']')
Exemplo n.º 8
0
def eval_with_GT(model_path, data_path, config_path, v2=False):
    timestr = strftime("%Y-%m-%d_%H:%M")

    showroom_path = get_showroom_path(model_path, full_path_bool=True)
    print('Images will be saved to:\n\t{}'.format(showroom_path))

    velo_path = os.path.join(data_path, 'training', 'velodyne')

    config = load_config(config_path)
    start_time = time()
    input_config = InputConfig(config['INPUT_CONFIG'])
    eval_config = EvalConfig(config['EVAL_CONFIG'])

    if v2:
        model = HaveFafe4Eval(input_config)
    else:
        model = FafeNet(input_config)

    if eval_config.use_cuda:
        if eval_config.cuda_device == 0:
            device = torch.device("cuda:0")
            print('Using CUDA:{}\n'.format(0))
        elif eval_config.cuda_device == 1:
            device = torch.device("cuda:1")
            print('Using CUDA:{}\n'.format(1))
        else:
            print('Functionality for CUDA device cuda:{} not yet implemented.'.
                  format(eval_config.cuda_device))
            print('Using cuda:0 instead...\n')
            device = torch.device("cuda:0")
        model = model.to(device)
    else:
        device = torch.device("cpu")
        print('Using CPU\n')

    imu_path = os.path.join(data_path, 'training', 'oxts')

    model.load_state_dict(
        torch.load(
            model_path,
            map_location=lambda storage, loc: storage)["model_state_dict"])
    model.eval()

    print("Model loaded, loading time: {}".format(round(
        time() - start_time, 2)))
    timestr = strftime("%Y-%m-%d_%H:%M")
    eval_head = FafePredict(input_config, eval_config)
    eval_head.eval()
    print("Evalutation Model Loaded!")

    testing_datasets = [
        TemporalBEVsDataset(input_config,
                            root_dir=data_path,
                            split='training',
                            sequence=seq)
        for seq in eval_config.validation_seqs
    ]

    testing_dataset = ConcatDataset(testing_datasets)

    testing_loader = DataLoader(dataset=testing_dataset,
                                batch_size=eval_config.batch_size,
                                num_workers=eval_config.num_workers,
                                shuffle=False)

    print("Starting to iterate over batches")

    if eval_config.use_gospa:
        gospa_scores_dict = {}

    for batch_idx, batch in enumerate(testing_loader):
        input, target, info = batch

        if eval_config.use_cuda:
            input = input.to(device)
            target = target.to(device)

        timeit = time()
        with torch.no_grad():
            out_det, out_reg = model(input)
            inference = eval_head(out_det, out_reg)
        print("\nTime to propagate through network: {}".format(
            round(time() - timeit, 2)))

        nB = target.shape[0]
        nT = input_config.num_conseq_frames

        tic = time()
        for batch_index in range(nB):
            current_sequence = info["sequence"][batch_index]
            imud = load_imu(imu_path, current_sequence)
            print("Batch: {} \t| Seq: {}".format(
                batch_index, info["sequence"][batch_index]),
                  end='')

            if eval_config.save_figs:
                frame = info["GT_indices"][0]
                file = os.path.join(velo_path, info["sequence"][batch_index],
                                    str(frame.item()).zfill(6) + '.bin')
                pc = np.fromfile(file, dtype=np.float32).reshape((-1, 4))
                fig = draw_lidar(
                    pc, off_screen_rendering=eval_config.off_screen_rendering)

            if eval_config.use_gospa:
                current_frame = info["Current_index"][batch_index].item()
                gospa_scores_dict[int(current_sequence),
                                  current_frame] = np.zeros(nT)

            for time_index in range(nT):
                # Get ground truths
                non_zero_gt_mask = target[batch_index][time_index].sum(
                    dim=1) != 0
                non_zero_gt = target[batch_index][time_index][non_zero_gt_mask]

                # Transform gt to 2D bbox x1y1x2y2 form
                point_form_gt = point_form_fafe(non_zero_gt)

                # Transform gt to 3D bbox form
                point_form_gt_3d = point_form_3d(point_form_gt,
                                                 input_config.z_center,
                                                 input_config.z_height,
                                                 device=device)

                # Rotate gt 3D bboxes
                gt_center = center_size_3d(non_zero_gt, input_config.z_center,
                                           device)
                point_form_gt_3d_rot = rotate_3d_bbx(point_form_gt_3d,
                                                     non_zero_gt[:, 4],
                                                     gt_center, device)

                # Get inference
                inference_reg = inference[batch_index][time_index]

                # Calculate GOSPA if ordered to do so
                if eval_config.use_gospa:
                    if inference_reg.is_cuda:
                        gospa_gt = non_zero_gt[:, 0:2].cpu()
                        gospa_inf = inference_reg[:, 0:2].cpu()
                    else:
                        gospa_gt = non_zero_gt[:, 0:2]
                        gospa_inf = inference_reg[:, 0:2]
                    gospa_score = GOSPA(gospa_gt, gospa_inf)
                    gospa_scores_dict[int(current_sequence),
                                      current_frame][time_index] = gospa_score
                    if eval_config.save_figs and eval_config.draw_gospa:
                        fig = draw_gospa(gospa_score, fig, time_index)

                        # Get probabilities of objects
                inference_probabilities = inference_reg[
                    ..., -1] if eval_config.show_confidence else None

                # Transform inference to 2D bbox x1y1x2y2
                inf_points_2d = point_form_fafe(inference_reg)

                # Transform inference to 3D bbox
                inference_points = point_form_3d(inf_points_2d,
                                                 input_config.z_center,
                                                 input_config.z_height, device)

                # Rotate inference 3D bboxes
                inference_center = center_size_3d(inference_reg,
                                                  input_config.z_center,
                                                  device)
                inference_points_rot = rotate_3d_bbx(inference_points,
                                                     inference_reg[:, 4],
                                                     inference_center, device)

                # Translate center points according to time_index
                translated_gt_c = translate_center(gt_center,
                                                   imu_data=imud[frame.item()],
                                                   timestep=time_index,
                                                   dt=input_config.dt,
                                                   device=device)
                translated_infer_c = translate_center(
                    inference_center,
                    imu_data=imud[frame.item()],
                    timestep=time_index,
                    dt=input_config.dt,
                    device=device)

                # TODO: Move these if-statements further up to reduce computation
                if eval_config.save_figs:
                    if time_index == 0:
                        fig = draw_gt_boxes3d(point_form_gt_3d_rot,
                                              fig,
                                              color=eval_config.gt_color)
                        fig = draw_gt_boxes3d(
                            inference_points_rot,
                            fig,
                            color=eval_config.infer_color,
                            probabilities=inference_probabilities)
                    else:
                        fig = draw_points_3d(translated_gt_c,
                                             fig,
                                             color=eval_config.gt_color)
                        fig = draw_points_3d(translated_infer_c,
                                             fig,
                                             color=eval_config.infer_color)
            if eval_config.save_figs:
                filename = "_".join(
                    ("Infer", "seq", info["sequence"][batch_index], "f",
                     str(frame.item()), timestr + ".png"))
                filepath = os.path.join(showroom_path, 'oracle_view', filename)
                filename_top = "_".join(
                    ("Top_Infer", "seq", info["sequence"][batch_index], "f",
                     str(frame.item()), timestr + ".png"))
                filepath_top = os.path.join(showroom_path, 'top_view',
                                            filename_top)
                mayavi.mlab.view(azimuth=90,
                                 elevation=0,
                                 focalpoint=[24., 0, 0],
                                 distance=120.0,
                                 figure=fig)
                mayavi.mlab.savefig(filepath_top, figure=fig)
                mayavi.mlab.view(
                    azimuth=180,
                    elevation=70,
                    focalpoint=[12.0909996, -1.04700089, -2.03249991],
                    distance=62.0,
                    figure=fig)
                mayavi.mlab.savefig(filepath, figure=fig)
            print('\t | time: {}'.format(round(time() - tic, 2)))

        print(gospa_scores_dict)
Exemplo n.º 9
0
def eval_with_GT(fafe_model_path, pp_model_path, data_path, config_path):
    timestr = strftime("%Y-%m-%d_%H:%M")

    showroom_path = get_showroom_path(fafe_model_path, full_path_bool=True)
    print('Images will be saved to:\n\t{}'.format(showroom_path))

    velo_path = os.path.join(data_path, 'training', 'velodyne')

    config = load_config(config_path)
    start_time = time()
    input_config = InputConfig(config['INPUT_CONFIG'])
    train_config = TrainConfig(config['TRAIN_CONFIG'])
    eval_config = EvalConfig(config['EVAL_CONFIG'])
    model_config = ModelConfig(config['MODEL_CONFIG'])
    post_config = PostConfig(config['POST_CONFIG'])

    pillar = PillarOfFafe(input_config, train_config.batch_size, train_config.verbose)

    if model_config.model == 'little_fafe':
        fafe = LittleFafe(input_config)
    else:
        fafe = FafeNet(input_config)

    if eval_config.use_cuda:
        if eval_config.cuda_device == 0:
            device = torch.device("cuda:0")
            print('Using CUDA:{}\n'.format(0))
        elif eval_config.cuda_device == 1:
            device = torch.device("cuda:1")
            print('Using CUDA:{}\n'.format(1))
        else:
            print('Functionality for CUDA device cuda:{} not yet implemented.'.format(eval_config.cuda_device))
            print('Using cuda:0 instead...\n')
            device = torch.device("cuda:0")
        pillar = pillar.to(device)
        fafe = fafe.to(device)
    else:
        device = torch.device("cpu")
        print('Using CPU\n')

    imu_path = os.path.join(data_path, 'training', 'oxts')

    pillar.load_state_dict(torch.load(pp_model_path, map_location=lambda storage, loc: storage)["model_state_dict"])
    fafe.load_state_dict(torch.load(fafe_model_path, map_location=lambda storage, loc: storage)["model_state_dict"])
    pillar.eval()
    fafe.eval()

    print("Model loaded, loading time: {}".format(round(time() - start_time, 2)))
    timestr = strftime("%Y-%m-%d_%H:%M")
    eval_head = FafePredict(input_config, eval_config)
    eval_head.eval()
    print("Evalutation Model Loaded!")

    testing_datasets = [VoxelDataset(input_config, root_dir=data_path, split='training', sequence=seq) for seq in
                        eval_config.validation_seqs]

    testing_dataset = ConcatDataset(testing_datasets)

    testing_loader = DataLoader(dataset=testing_dataset,
                                batch_size=eval_config.batch_size,
                                num_workers=eval_config.num_workers,
                                shuffle=False)

    print("Starting to iterate over batches")

    if eval_config.use_gospa:
        gospa_scores_dict = {}

    for batch_idx, batch in enumerate(testing_loader):

        # Create Pillar Pseudo Img
        voxel_stack, coord_stack, num_points_stack, num_nonempty_voxels, target, info = batch

        # Move all input data to the correct device if not using CPU
        if train_config.use_cuda:
            voxel_stack = voxel_stack.to(device)
            coord_stack = coord_stack.to(device)
            num_points_stack = num_points_stack.to(device)
            num_nonempty_voxels = num_nonempty_voxels.to(device)
            target = target.to(device)

        timeit = time()
        with torch.no_grad():
            pseudo_stack = []
            for time_iter in range(input_config.num_conseq_frames):
                pseudo_image = pillar(voxel_stack[:, time_iter], num_points_stack[:, time_iter],
                                      coord_stack[:, time_iter], num_nonempty_voxels[:, time_iter])

                if input_config.pp_verbose:
                    print("Pseudo_img: {}".format(pseudo_image.unsqueeze(1).shape))
                pseudo_stack.append(pseudo_image.unsqueeze(1))

            pseudo_torch = torch.cat(pseudo_stack, dim=1)

            if train_config.use_cuda:
                pseudo_torch = pseudo_torch.to(device)
                target = target.to(device)

            # Forward propagation. Reshape by squishing together Time and Channel dimensions.
            # Reshaping basically as we do with BEV, stacking them on top of eachother.
            out_detection, out_regression = fafe.forward(
                pseudo_torch.reshape(-1, pseudo_torch.shape[1] * pseudo_torch.shape[2], pseudo_torch.shape[-2],
                                     pseudo_torch.shape[-1]))
            inference = eval_head(out_detection, out_regression)
        print("\nTime to propagate through network: {}".format(round(time() - timeit, 2)))

        nB = target.shape[0]
        nT = input_config.num_conseq_frames

        tic = time()
        for batch_index in range(nB):
            current_sequence = info["sequence"][batch_index]
            frame = info["GT_indices"][0]

            if torch.is_tensor(current_sequence):
                current_sequence = current_sequence.item()

            imud = load_imu(imu_path, current_sequence)
            print("Batch: {} \t| Seq: {} Frame: {}".format(batch_index, current_sequence, frame.item()), end='')

            if eval_config.save_raw:
                if not os.path.exists("showroom/detections_" +str(current_sequence).zfill(4)):
                    os.mkdir("showroom/detections_" +str(current_sequence).zfill(4))
                filename = os.path.join("showroom", "detections_{}".format(str(current_sequence).zfill(4)), str(int(frame)).zfill(4) + ".pt")
                torch.save(inference, filename)

            if eval_config.save_figs:
                file = os.path.join(velo_path, str(current_sequence).zfill(4), str(frame.item()).zfill(6) + '.bin')
                pc = np.fromfile(file, dtype=np.float32).reshape((-1, 4))
                fig = draw_lidar(pc, off_screen_rendering=eval_config.off_screen_rendering)

            if eval_config.use_gospa:
                current_frame = info["Current_index"][batch_index].item()
                gospa_scores_dict[int(current_sequence), current_frame] = np.zeros(nT)

            for time_index in range(nT):
                # Get ground truths
                non_zero_gt_mask = target[batch_index][time_index].sum(dim=1) != 0
                non_zero_gt = target[batch_index][time_index][non_zero_gt_mask]

                # Transform gt to 2D bbox x1y1x2y2 form
                point_form_gt = point_form_fafe(non_zero_gt)

                # Transform gt to 3D bbox form
                point_form_gt_3d = point_form_3d(point_form_gt, input_config.z_center, input_config.z_height,
                                                 device=device)

                # Rotate gt 3D bboxes
                gt_center = center_size_3d(non_zero_gt, input_config.z_center, device)
                point_form_gt_3d_rot = rotate_3d_bbx(point_form_gt_3d, non_zero_gt[:, 4], gt_center, device)

                # Get inference
                inference_reg = inference[batch_index][time_index]

                # Filter out if outside FOV
                filtered_inference_reg = []
                for inf_reg in inference_reg:
                    np_inf_reg = inf_reg.cpu().data.numpy()
                    if within_fov(np_inf_reg[0:2], min_angle=0.78, max_angle=2.45, max_radius=100):
                        filtered_inference_reg.append(np_inf_reg)

                # Filter out if too closely located
                filtered_inference_reg = too_close(filtered_inference_reg, eval_config.distance_threshold)

                if eval_config.save_detections_as_measurements and time_index == 0:
                    for fir in filtered_inference_reg:
                        file1 = open(showroom_path + "/detections_" + str(current_sequence).zfill(4) + ".txt", "a")
                        # Save it on the format PMBM wants it... p = [x, y, rot] with x defined right wards and y forward
                        file1.write(str(frame.item()) + ' Car:' + str(fir[5]) + ' ' + str(- fir[1]) + ' ' + str(
                            fir[0]) + ' ' + str(fir[4]) + "\n")
                        file1.close()

                # Make filtered_inference_reg numpy because GOSPA needs it to be
                filtered_inference_reg = np.array(filtered_inference_reg)

                # Calculate GOSPA if ordered to do so
                if eval_config.use_gospa:
                    if inference_reg.is_cuda:
                        gospa_gt = non_zero_gt[:, 0:2].cpu()
                        # gospa_inf = inference_reg[:, 0:2].cpu()
                        gospa_inf = filtered_inference_reg[:, 0:2] if len(filtered_inference_reg) > 0 else np.array([])
                    else:
                        gospa_gt = non_zero_gt[:, 0:2]
                        # gospa_inf = inference_reg[:, 0:2]
                        gospa_inf = filtered_inference_reg[:, 0:2]

                    gospa_score = GOSPA(gospa_gt, gospa_inf)
                    gospa_scores_dict[int(current_sequence), current_frame][time_index] = gospa_score
                    if eval_config.save_figs and eval_config.draw_gospa:
                        fig = draw_gospa(gospa_score, fig, time_index)

                # Make filtered_inference_reg torch tensor because point forms need it to be
                filtered_inference_reg = torch.from_numpy(filtered_inference_reg).to(device)

                # If no outputs -> plot ev. gt and continue
                if len(filtered_inference_reg) == 0:
                    if time_index == 0:
                        fig = draw_gt_boxes3d(point_form_gt_3d_rot, fig, color=eval_config.gt_color)
                        continue
                    else:
                        # Translate center points according to time_index
                        translated_gt_c = translate_center(gt_center, imu_data=imud[frame.item()], timestep=time_index,
                                                           dt=input_config.dt, device=device)
                        fig = draw_points_3d(translated_gt_c, fig, color=eval_config.gt_color)
                        continue

                # Get probabilities of objects
                inference_probabilities = filtered_inference_reg[..., -1] if eval_config.show_confidence else None

                # Transform inference to 2D bbox x1y1x2y2
                inf_points_2d = point_form_fafe(filtered_inference_reg)

                # Transform inference to 3D bbox
                inference_points = point_form_3d(inf_points_2d, input_config.z_center, input_config.z_height, device)

                # Rotate inference 3D bboxes
                inference_center = center_size_3d(filtered_inference_reg, input_config.z_center, device)
                inference_points_rot = rotate_3d_bbx(inference_points, filtered_inference_reg[:, 4], inference_center,
                                                     device)

                # Translate center points according to time_index
                translated_gt_c = translate_center(gt_center, imu_data=imud[frame.item()], timestep=time_index,
                                                   dt=input_config.dt, device=device)
                translated_infer_c = translate_center(inference_center, imu_data=imud[frame.item()],
                                                      timestep=time_index, dt=input_config.dt, device=device)

                # TODO: Move these if-statements further up to reduce computation
                if eval_config.save_figs:
                    if time_index == 0:
                        fig = draw_gt_boxes3d(point_form_gt_3d_rot, fig, color=eval_config.gt_color)
                        fig = draw_gt_boxes3d(inference_points_rot, fig, color=eval_config.infer_color,
                                              probabilities=inference_probabilities)
                    else:
                        fig = draw_points_3d(translated_gt_c, fig, color=eval_config.gt_color)
                        fig = draw_points_3d(translated_infer_c, fig, color=eval_config.infer_color)
            if eval_config.save_figs:
                filename = "_".join(
                    ("Infer", "seq", str(current_sequence), "f", str(frame.item()), timestr + ".png"))
                filepath = os.path.join(showroom_path, 'oracle_view', filename)
                filename_top = "_".join(
                    ("Top_Infer", "seq", str(current_sequence), "f", str(frame.item()), timestr + ".png"))
                filepath_top = os.path.join(showroom_path, 'top_view', filename_top)
                mayavi.mlab.view(azimuth=90, elevation=0, focalpoint=[24., 0, 0],
                                 distance=120.0, figure=fig)
                mayavi.mlab.savefig(filepath_top, figure=fig)
                mayavi.mlab.view(azimuth=180, elevation=70, focalpoint=[12.0909996, -1.04700089, -2.03249991],
                                 distance=62.0, figure=fig)
                mayavi.mlab.savefig(filepath, figure=fig)

            if eval_config.use_gospa:
            # Save gospa scores in txt file
                file2 = open(showroom_path + "/gospa_scores_" + str(current_sequence).zfill(4) + ".txt", "a")
                row_str = str(frame.item())
                for time_index in range(nT):
                    row_str += (' ' + str(gospa_scores_dict[int(current_sequence), current_frame][time_index]))
                file2.write(row_str + "\n")
                file2.close()

            print('\t | time: {}'.format(round(time() - tic, 2)))
Exemplo n.º 10
0
def train():
    if os.path.exists('/home/mlt/mot/fafe/cfg/adams_computer'):
        config_path = 'cfg/cfg_mini.yml'
    else:
        config_path = 'cfg/cfg.yml'

    print('Using config: \n\t{}\n'.format(config_path))
    config = load_config(config_path)

    input_config = InputConfig(config['INPUT_CONFIG'])
    train_config = TrainConfig(config['TRAIN_CONFIG'])
    loss_config = LossConfig(config['LOSS_CONFIG'])
    model_config = ModelConfig(config['MODEL_CONFIG'])

    verbose = train_config.verbose

    time_str = strftime("%Y-%m-%d_%H-%M")
    weights_filename = 'trained_models/' + time_str + '/weights_' + time_str

    if not os.path.exists('trained_models/' + time_str):
        os.mkdir('trained_models/' + time_str)

    print(
        'Training weights will be saved to:\n\t{}\n'.format(weights_filename))

    config_filename = 'trained_models/' + time_str + '/config_' + time_str + '.yml'
    save_config(config_filename, config)
    print('Config file saved to:\n\t{}\n'.format(config_filename))

    if train_config.use_visdom:
        print(
            'Dont forget to run "visdom" in a terminal in parallel to this in order to start the Visdom server'
        )
        print('Choose port with "python -m visdom.server -port {}" \n'.format(
            train_config.visdom_port))
        vis = visdom.Visdom(
            port=train_config.visdom_port)  # port 8097 is default
        loss_window, sub_loss_window, recall_window, precision_window = viz.get_windows(
            vis, time_str)

    # Get root directory depending on which computer is running...
    # Don't forget to add your own path in 'fafe_utils.config_stuff.get_root_dir'
    root_dir = get_root_dir()

    #########################
    # Define network
    #########################
    if model_config.model == 'little_fafe':
        net = LittleFafe(input_config=input_config)
    else:
        net = FafeNet(input_config=input_config)
    print('Net set up successfully!')
    pytorch_total_params = sum(p.numel() for p in net.parameters())
    pytorch_trainable_params = sum(p.numel() for p in net.parameters()
                                   if p.requires_grad)
    print("\tNumber of parameters: {}\n\tNumber of trainable parameters: {}".
          format(pytorch_total_params, pytorch_trainable_params))
    # TODO: add posibility to load old weights

    #########################
    # Set which device run on
    #########################
    if train_config.use_cuda:
        # "If you load your samples in the Dataset on CPU and would like to push it during training to the GPU,
        # you can speed up the host to device transfer by enabling pin_memory."
        # - ptrblck [https://discuss.pytorch.org/t/when-to-set-pin-memory-to-true/19723]
        pin_memory = True
        if train_config.multi_gpu:
            device_ids = list(range(torch.cuda.device_count()))
            device = torch.device("cuda:" + str(device_ids[0]))
            net = net.to(device)
            net = nn.DataParallel(net)
            print(
                '\nUsing multiple GPUs.\n\tDevices: {}\n\tOutput device: {}\n'.
                format(device_ids, device))
        else:
            device = torch.device("cuda:" + str(train_config.cuda_device))
            print('\nUsing device {}\n'.format(device))
            net = net.to(device)
    else:
        pin_memory = False
        device = torch.device("cpu")
        print('Using CPU\n')

    #########################
    # Define loss function
    #########################
    loss_func = FafeLoss(input_config, train_config, loss_config, device)
    if train_config.use_cuda:
        loss_func = loss_func.to(device)
        if train_config.multi_gpu:
            loss_func = nn.DataParallel(loss_func)

    #########################
    # Define optimizer
    #########################
    params = list(net.parameters()) + list(loss_func.parameters())
    optimizer = optim.Adam(params,
                           lr=train_config.learning_rate,
                           weight_decay=train_config.weight_decay)
    print('Adams Optimizer set up with\n\tlr = {}\n\twd = {}\n'.format(
        train_config.learning_rate, train_config.weight_decay))

    #########################
    # Get Datasets
    #########################
    print('Training Data:')
    training_dataloader = DataLoader(ConcatDataset([
        TemporalBEVsDataset(input_config,
                            root_dir,
                            split='training',
                            sequence=seq) for seq in train_config.training_seqs
    ]),
                                     batch_size=train_config.batch_size,
                                     shuffle=train_config.shuffle,
                                     num_workers=train_config.num_workers,
                                     pin_memory=pin_memory)
    print('Validation Data:')
    validation_dataloader = DataLoader(ConcatDataset([
        TemporalBEVsDataset(input_config,
                            root_dir,
                            split='training',
                            sequence=seq)
        for seq in train_config.validation_seqs
    ]),
                                       batch_size=train_config.batch_size,
                                       shuffle=train_config.shuffle,
                                       num_workers=train_config.num_workers,
                                       pin_memory=pin_memory)
    print('Data Loaders set up with:\n\tBatch size:  {}\n\tNum Workers: {}'.
          format(train_config.batch_size, train_config.num_workers))

    ###############################
    # Start training and evaluation
    ###############################
    print('\nTraining initiated [' + strftime("%Y-%m-%d %H:%M") + ']')
    for epoch in range(train_config.max_epochs):
        train_mean_loss, train_mean_recall, train_mean_precision, train_num_samples = 0, 0, 0, 0
        eval_mean_loss, eval_mean_recall, eval_mean_precision, eval_num_samples = 0, 0, 0, 0
        train_scaled_L1_mean, train_scaled_euler_mean, train_classification_loss = 0, 0, 0
        eval_scaled_L1_mean, eval_scaled_euler_mean, eval_classification_loss = 0, 0, 0

        #########################
        # TRAINING
        #########################
        tic = time()
        net.train()
        for i_batch, sample_batched in enumerate(training_dataloader):
            input, target, _ = sample_batched

            if train_config.use_cuda:
                input = input.to(device)
                target = target.to(device)

            # Always reset optimizer's gradient each iteration
            optimizer.zero_grad()

            if verbose:
                print('{} i: {} {}'.format('~' * 10, i_batch, '~' * 10))
                print('Input shape: {}'.format(input.shape))
                print('Target shape: {}'.format(target.shape))
                print('Target: {}'.format(target[0][0][0][0]))

            # Forward propagation
            out_detection, out_regression = net.forward(input)
            if verbose:
                print('Output shape det: {}'.format(out_detection.shape))
                print('Output shape reg: {}'.format(out_regression.shape))

            # Calculate the loss
            loss, recall, precision, scaled_l1, scaled_euler, classification_loss = loss_func(
                out_detection, out_regression, target, verbose)
            # Back propagate
            loss.backward()

            # Update the weights
            optimizer.step()

            train_mean_loss += loss
            train_mean_recall += recall
            train_mean_precision += precision
            train_scaled_L1_mean += scaled_l1
            train_scaled_euler_mean += scaled_euler
            train_classification_loss += classification_loss
            train_num_samples += 1

        # Calculate the actual averages
        train_mean_loss /= train_num_samples
        train_mean_recall /= train_num_samples
        train_mean_precision /= train_num_samples
        train_scaled_L1_mean /= train_num_samples
        train_scaled_euler_mean /= train_num_samples
        train_classification_loss /= train_num_samples
        training_time = time() - tic

        #########################
        # EVALUATION
        #########################
        tic2 = time()
        net.eval()
        with torch.no_grad():
            for i_batch, sample_batched in enumerate(validation_dataloader):
                input, target, _ = sample_batched
                if train_config.use_cuda:
                    input = input.to(device)
                    target = target.to(device)
                # Forward propagation
                out_detection, out_regression = net.forward(input)
                # Calculate the loss
                loss, recall, precision, scaled_l1, scaled_euler, classification_loss = loss_func(
                    out_detection, out_regression, target, verbose)
                eval_mean_loss += loss
                eval_mean_recall += recall
                eval_mean_precision += precision
                eval_scaled_L1_mean += scaled_l1
                eval_scaled_euler_mean += scaled_euler
                eval_classification_loss += classification_loss
                eval_num_samples += 1
        eval_mean_loss /= eval_num_samples
        eval_mean_recall /= eval_num_samples
        eval_mean_precision /= eval_num_samples
        eval_scaled_L1_mean /= eval_num_samples
        eval_scaled_euler_mean /= eval_num_samples
        eval_classification_loss /= eval_num_samples

        eval_time = time() - tic2
        total_time = time() - tic

        #########################
        # PRINT STUFF ON SCREEN
        #########################
        print('\nEpoch {} / {}\n{}\nCurrent time: {}'.format(
            epoch, train_config.max_epochs - 1, '-' * 12,
            strftime("%Y-%m-%d %H:%M")))
        print('Epoch Total Time: {} s ({} + {})'.format(
            round(total_time, 2), round(training_time, 2), round(eval_time,
                                                                 2)))
        print('Next Epoch ETA: ' +
              format(datetime.now() +
                     timedelta(seconds=total_time), '%Y-%m-%d %H:%M'))
        print('Training ETA: ' + format(
            datetime.now() + timedelta(
                seconds=total_time *
                (train_config.max_epochs - epoch - 1)), '%Y-%m-%d %H:%M'))
        print('Train\n\tLoss: \t\t{}'
              '\n\t\tL1:  \t{}'
              '\n\t\tEuler:\t{}'
              '\n\t\tCL:  \t{}'
              '\n\tRecall: \t{}'
              '\n\tPrecision:\t{}'.format(train_mean_loss,
                                          train_scaled_L1_mean,
                                          train_scaled_euler_mean,
                                          train_classification_loss,
                                          train_mean_recall,
                                          train_mean_precision))
        print('Validation\n\tLoss: \t\t{}'
              '\n\t\tL1:  \t{}'
              '\n\t\tEuler:\t{}'
              '\n\t\tCL:  \t{}'
              '\n\tRecall: \t{}'
              '\n\tPrecision:\t{}'.format(eval_mean_loss, eval_scaled_L1_mean,
                                          eval_scaled_euler_mean,
                                          eval_classification_loss,
                                          eval_mean_recall,
                                          eval_mean_precision))
        if train_config.use_visdom:
            # Visualize Loss
            viz.push_data(epoch, vis, loss_window, sub_loss_window,
                          recall_window, precision_window, train_mean_loss,
                          eval_mean_loss, train_scaled_L1_mean,
                          train_classification_loss, train_scaled_euler_mean,
                          eval_scaled_euler_mean, eval_scaled_L1_mean,
                          eval_classification_loss, train_mean_recall,
                          eval_mean_recall, train_mean_precision,
                          eval_mean_precision)

        #########################
        # SAVE WEIGHTS (every save_weights_modulus th epoch)
        #########################
        if epoch % train_config.save_weights_modulus == 0 or epoch == train_config.max_epochs - 1:
            save_filename = weights_filename + '_epoch_' + str(epoch)
            torch.save(
                {
                    'epoch': epoch,
                    'model_state_dict': net.state_dict(),
                    'optimizer_state_dict': optimizer.state_dict(),
                    'loss': loss
                }, save_filename)

    print('Training Complete [' + strftime("%Y-%m-%d %H:%M") + ']')
Exemplo n.º 11
0
def eval_post_fafe_pp(fafe_model_path, pp_model_path, data_path, config_path,
                      sequence, kitti):
    """

    :param self:
    :param fafe_model_path:
    :param pp_model_path:
    :param data_path:
    :param config_path:
    :return:
    """
    timestr = strftime("%Y-%m-%d_%H:%M")

    velo_path = os.path.join(data_path, 'training', 'velodyne')

    config = load_config(config_path)
    start_time = time()
    input_config = InputConfig(config['INPUT_CONFIG'])
    train_config = TrainConfig(config['TRAIN_CONFIG'])
    eval_config = EvalConfig(config['EVAL_CONFIG'])
    model_config = ModelConfig(config['MODEL_CONFIG'])
    post_config = PostConfig(config['POST_CONFIG'])

    pillar = PillarOfFafe(input_config, train_config.batch_size,
                          train_config.verbose)

    if model_config.model == 'little_fafe':
        fafe = LittleFafe(input_config)
    else:
        fafe = FafeNet(input_config)

    if eval_config.use_cuda:
        if eval_config.cuda_device == 0:
            device = torch.device("cuda:0")
            print('Using CUDA:{}\n'.format(0))
        elif eval_config.cuda_device == 1:
            device = torch.device("cuda:1")
            print('Using CUDA:{}\n'.format(1))
        else:
            print('Functionality for CUDA device cuda:{} not yet implemented.'.
                  format(eval_config.cuda_device))
            print('Using cuda:0 instead...\n')
            device = torch.device("cuda:0")
        pillar = pillar.to(device)
        fafe = fafe.to(device)
    else:
        device = torch.device("cpu")
        print('Using CPU\n')

    imu_path = os.path.join(data_path, 'training', 'oxts')
    pillar.load_state_dict(
        torch.load(
            pp_model_path,
            map_location=lambda storage, loc: storage)["model_state_dict"])
    fafe.load_state_dict(
        torch.load(
            fafe_model_path,
            map_location=lambda storage, loc: storage)["model_state_dict"])
    post_fafe = PostFafe(input_config, post_config, device)

    pillar.eval()
    fafe.eval()

    print("Model loaded, loading time: {}".format(round(
        time() - start_time, 2)))
    timestr = strftime("%Y-%m-%d_%H:%M")
    eval_head = FafePredict(input_config, eval_config)
    eval_head.eval()
    print("Evalutation Model Loaded!")

    # for seq in eval_config.validation_seqs:
    testing_dataset = VoxelDataset(input_config,
                                   root_dir=data_path,
                                   split='training',
                                   sequence=sequence)
    testing_loader = DataLoader(dataset=testing_dataset,
                                batch_size=eval_config.batch_size,
                                num_workers=eval_config.num_workers,
                                shuffle=False)

    data = []
    frames = []
    total_time_per_iteration = []
    for batch_idx, batch in enumerate(testing_loader):

        # Create Pillar Pseudo Img
        voxel_stack, coord_stack, num_points_stack, num_nonempty_voxels, target, info = batch

        nB = target.shape[0]
        nT = input_config.num_conseq_frames
        current_sequence = sequence
        frame = info["GT_indices"][0]
        if type(frame) == torch.Tensor:
            frame = frame.item()

        d = {}
        d['current_time'] = frame
        d['frame_id'] = frame  # To match pmbm's data structure...
        frames.append(frame)
        imud = load_imu(imu_path, current_sequence)

        true_states = kitti.get_bev_states(frame,
                                           classes_to_track=['Car', 'Van'])
        d['true_states'] = true_states
        d['state_dims'] = 3

        # Move all input data to the correct device if not using CPU
        if train_config.use_cuda:
            voxel_stack = voxel_stack.to(device)
            coord_stack = coord_stack.to(device)
            num_points_stack = num_points_stack.to(device)
            num_nonempty_voxels = num_nonempty_voxels.to(device)
            target = target.to(device)

        timeit = time()
        with torch.no_grad():
            pseudo_stack = []
            for time_iter in range(input_config.num_conseq_frames):
                pseudo_image = pillar(voxel_stack[:, time_iter],
                                      num_points_stack[:, time_iter],
                                      coord_stack[:, time_iter],
                                      num_nonempty_voxels[:, time_iter])

                if input_config.pp_verbose:
                    print("Pseudo_img: {}".format(
                        pseudo_image.unsqueeze(1).shape))
                pseudo_stack.append(pseudo_image.unsqueeze(1))

            pseudo_torch = torch.cat(pseudo_stack, dim=1)

            if train_config.use_cuda:
                pseudo_torch = pseudo_torch.to(device)
                target = target.to(device)

            # Forward propagation. Reshape by squishing together Time and Channel dimensions.
            # Reshaping basically as we do with BEV, stacking them on top of eachother.
            out_detection, out_regression = fafe.forward(
                pseudo_torch.reshape(
                    -1, pseudo_torch.shape[1] * pseudo_torch.shape[2],
                    pseudo_torch.shape[-2], pseudo_torch.shape[-1]))
            inference = eval_head(out_detection, out_regression)
        inference_time = round(time() - timeit, 2)

        if torch.is_tensor(current_sequence):
            current_sequence = current_sequence.item()

        meas_ = inference[0][0]
        meas = []
        for row in range(meas_.shape[0]):
            meas.append(transf(meas_[0].to('cpu').numpy()))
        d['measurements'] = meas
        if len(inference) > 1:
            raise ValueError("Batch index > 1. Do something god damnit")

        # Call post_fafe to infer the inference
        post_fafe(inference[0])
        imu_data = imud[frame]
        # Add targets and predictions to data dict
        # TID = Target ID
        estimated_targets = []
        for tid, tensor_state in post_fafe.object_state.items():
            continue_bool = 0
            if post_config.verbose:
                print("Object: {} \n\t{}".format(tid, tensor_state))
            _temp = {}
            _temp['target_idx'] = tid
            _temp['object_class'] = 'car'

            _state = tensor_state[0].numpy()

            single_target = SingleTargetHypothesis(transf(_state))
            _temp['single_target'] = single_target
            _temp['state_predictions'] = []
            _temp['var_predictions'] = []
            for i, step in enumerate(tensor_state):
                if step is None or continue_bool:
                    continue_bool = 1
                    continue

                # Coordinate transform predictions
                preds = transf(step.numpy())

                if post_config.coordinate_transform:
                    preds = translate_center(preds,
                                             imu_data,
                                             timestep=i,
                                             dt=input_config.dt)

                _temp['state_predictions'].append(preds)
                if post_config.verbose:
                    print("Step: {}".format(transf(step.numpy())))
            estimated_targets.append(_temp)

        d['estimated_targets'] = estimated_targets
        data.append(d)
        total_time_per_iteration.append(inference_time)
        print("{}({}s,{}#), ".format(frame, inference_time,
                                     len(estimated_targets)),
              end='')

    return data, total_time_per_iteration
Exemplo n.º 12
0
def eval_post_fafe(fafe_model_path, data_path, config_path, sequence, kitti):
    """

    :param self:
    :param fafe_model_path:
    :param pp_model_path:
    :param data_path:
    :param config_path:
    :return:
    """
    timestr = strftime("%Y-%m-%d_%H:%M")

    config = load_config(config_path)
    start_time = time()
    input_config = InputConfig(config['INPUT_CONFIG'])
    train_config = TrainConfig(config['TRAIN_CONFIG'])
    eval_config = EvalConfig(config['EVAL_CONFIG'])
    model_config = ModelConfig(config['MODEL_CONFIG'])
    post_config = PostConfig(config['POST_CONFIG'])

    pillar = PillarOfFafe(input_config, train_config.batch_size, train_config.verbose)

    if model_config.model == 'little_fafe':
        fafe = LittleFafe(input_config)
    else:
        fafe = FafeNet(input_config)

    if eval_config.use_cuda:
        if eval_config.cuda_device == 0:
            device = torch.device("cuda:0")
            print('Using CUDA:{}\n'.format(0))
        elif eval_config.cuda_device == 1:
            device = torch.device("cuda:1")
            print('Using CUDA:{}\n'.format(1))
        else:
            print('Functionality for CUDA device cuda:{} not yet implemented.'.format(eval_config.cuda_device))
            print('Using cuda:0 instead...\n')
            device = torch.device("cuda:0")
        pillar = pillar.to(device)
        fafe = fafe.to(device)
    else:
        device = torch.device("cpu")
        print('Using CPU\n')

    imu_path = os.path.join(data_path, 'training', 'oxts')
    fafe.load_state_dict(torch.load(fafe_model_path, map_location=lambda storage, loc: storage)["model_state_dict"])
    post_fafe = PostFafe(input_config, post_config, device)

    pillar.eval()
    fafe.eval()

    print("Model loaded, loading time: {}".format(round(time() - start_time, 2)))
    timestr = strftime("%Y-%m-%d_%H:%M")
    eval_head = FafePredict(input_config, eval_config)
    eval_head.eval()
    print("Evalutation Model Loaded!")

    # for seq in eval_config.validation_seqs:
    testing_dataset = TemporalBEVsDataset(input_config, root_dir=data_path, split='training', sequence=sequence)
    testing_loader = DataLoader(dataset=testing_dataset,
                                batch_size=eval_config.batch_size,
                                num_workers=eval_config.num_workers,
                                shuffle=False)

    data = []
    frames = []
    print("Starting to run over sequence {}. Total number of frames: {}".format(sequence, len(testing_loader)))
    total_time_per_iteration = []
    for batch_idx, batch in enumerate(testing_loader):

        input, target, info = batch

        if eval_config.use_cuda:
            input = input.to(device)
            target = target.to(device)

        timeit = time()
        with torch.no_grad():
            out_det, out_reg = fafe(input)
            inference = eval_head(out_det, out_reg)
        inference_time = round(time() - timeit, 2)
        nB = target.shape[0]
        nT = input_config.num_conseq_frames
        current_sequence = sequence
        frame = info["GT_indices"][0]
        if type(frame) == torch.Tensor:
            frame = frame.item()
        d = {}
        d['current_time'] = frame
        d['frame_id'] = frame       # To match pmbm's data structure...

        frames.append(frame)
        imud = load_imu(imu_path, current_sequence)

        true_states = kitti.get_bev_states(frame, classes_to_track=['Car', 'Van'])
        d['true_states'] = true_states
        d['state_dims'] = 3

        if torch.is_tensor(current_sequence):
            current_sequence = current_sequence.item()

        meas_ = inference[0][0]
        meas = []
        for row in range(meas_.shape[0]):
            meas.append(transf(meas_[0].to('cpu').numpy()))
        d['measurements'] = meas
        if len(inference) > 1:
            raise ValueError("Batch index > 1. Do something god damnit")

        # Call post_fafe to infer the inference
        post_fafe(inference[0])
        imu_data = imud[frame]
        # Add targets and predictions to data dict
        # TID = Target ID
        estimated_targets = []
        for tid, tensor_state in post_fafe.object_state.items():
            continue_bool = 0
            if post_config.verbose:
                print("Object: {} \n\t{}".format(tid, tensor_state))
            _temp = {}
            _temp['target_idx'] = tid
            _temp['object_class'] = 'car'

            _state = tensor_state[0].numpy()

            single_target = SingleTargetHypothesis(transf(_state))
            _temp['single_target'] = single_target
            _temp['state_predictions'] = []
            _temp['var_predictions'] = []
            for i, step in enumerate(tensor_state):
                if step is None or continue_bool:
                    continue_bool = 1
                    continue

                # Coordinate transform predictions
                preds = transf(step.numpy())

                if post_config.coordinate_transform:
                    preds = translate_center(preds, imu_data, timestep=i, dt=input_config.dt )

                _temp['state_predictions'].append(preds)
                if post_config.verbose:
                    print("Step: {}".format(transf(step.numpy())))
            estimated_targets.append(_temp)

        d['estimated_targets'] = estimated_targets
        data.append(d)
        total_time_per_iteration.append(inference_time)
        print("{}({}s,{}#), ".format(frame, inference_time, len(estimated_targets)),end='')

    return data, total_time_per_iteration