Example #1
0
    def __init__(self):
        # general
        self.device = None
        # self.sr = 16000
        # self.clip_len = 32768

        # experiment paths
        self.proj_dir = OUTPUT_ROOT
        self.exp_name = EXPERIMENT_NAME
        self.exp_dir = EXPERIMENT_DIR

        # create soft link to experiment log directory
        if not os.path.exists('train_log'):
            os.symlink(self.exp_dir, 'train_log')

        self.log_dir, self.model_dir = self.set_exp_paths()
        utils.ensure_dirs([self.log_dir, self.model_dir])

        # network configuration
        self.netType = 'context'
        self.set_network_info()

        # training configuration
        self.nr_epochs = 100
        self.batch_size = 15  # GPU memory usage
        self.num_workers = 60  #32 #multiprocessing.cpu_count()    # RAM usage
        self.lr = 1e-3  #1e-4
        self.lr_step_size = 15
        self.lr_decay = 0.999

        self.save_frequency = 1
        self.val_frequency = 10
        self.visualize_frequency = 100

        self.points_batch_size = None
Example #2
0
def split_seq_dataset(root_dset, obj_category, num_expr, test_ins):
    """
    write train_seq.txt, test_seq.txt
    """

    path = pjoin(root_dset, "render_seq", obj_category)
    train_ins = [
        ins for ins in os.listdir(path)
        if ins not in test_ins and not ins.startswith('.')
    ]
    print(train_ins)
    print(test_ins)
    print(f'len train_ins = {len(train_ins)}, len test_ins = {len(test_ins)}')

    train_list, test_list = [], []
    for data_list, ins_list in zip([train_list, test_list],
                                   [train_ins, test_ins]):
        for instance in ins_list:
            for track_dir in glob.glob(pjoin(path, instance, '*')):
                frames = glob.glob(pjoin(track_dir, 'cloud', '*'))
                cloud_list = [file for file in frames if file.endswith('.npz')]
                cloud_list.sort(
                    key=lambda str: int(str.split('.')[-2].split('/')[-1]))
                data_list += cloud_list

    output_path = pjoin(root_dset, "splits", obj_category, num_expr)
    ensure_dirs(output_path)

    name_list = ['train_seq.txt', 'test_seq.txt']
    for name, content in zip(name_list, [train_list, test_list]):
        print(f'{name}: {len(content)}')
        with open(pjoin(output_path, name), 'w') as f:
            for item in content:
                f.write('{}\n'.format(item))
Example #3
0
    def write(self, context, fname, overwrite=True):
        if fname is None:
            fname = self.fname

        fname = Template(fname).safe_substitute(config)

        if fname.startswith(
                '/'
        ):  # assume the caller know what they are doing, and it's a full path
            pth = fname
        else:
            pth = os.path.join(config.templates_path, fname)
            ensure_dirs(pth)

        if os.path.exists(
                pth
        ) and not overwrite:  # os.path.join (config.templates_path, fname)):
            if trace: print 'Not overwritten:', fname
            return

        content = context or self.content

        with open(pth, 'w') as f:
            f.write(content.encode('UTF-8'))
            if trace: print fname, 'WRITE OK'
Example #4
0
    def __init__(self):
        # general
        self.device = None

        # experiment paths
        self.proj_dir = "some_path"
        self.exp_name = os.getcwd().split('/')[-1]
        self.exp_dir = os.path.join(self.proj_dir, self.exp_name)

        self.log_dir, self.model_dir = self.set_exp_paths()
        utils.ensure_dirs([self.log_dir, self.model_dir])

        # network configuration
        self.set_network_info()

        # training configuration
        self.nr_epochs = 1000
        self.batch_size = 64
        self.num_workers = 8
        self.lr = 1e-4
        self.lr_step_size = 400

        self.save_frequency = 100
        self.val_frequency = 100
        self.visualize_frequency = 100

        self.points_batch_size = None
Example #5
0
    def __init__(self, cfg, logger=None):
        super(Trainer, self).__init__()
        self.ckpt_dir = pjoin(cfg['experiment_dir'], 'ckpt')
        ensure_dirs(self.ckpt_dir)

        self.device = cfg['device']
        if cfg['network']['type'] == 'canon_coord':
            self.model = CanonCoordModel(cfg)
        elif cfg['network']['type'] == 'rot':
            self.model = RotationModel(cfg)
        elif cfg['network']['type'] == 'rot_coord_track':
            self.model = EvalTrackModel(cfg)

        self.network_type = cfg['network']['type']

        if self.network_type in ['rot_coord_track']:
            self.coord_exp_dir = pjoin(cfg['coord_exp']['dir'], 'ckpt')
            self.coord_resume_epoch = cfg['coord_exp']['resume_epoch']
        else:
            self.coord_exp_dir = None

        self.optimizer = get_optimizer(
            [p for p in self.model.parameters() if p.requires_grad], cfg)
        self.scheduler = get_scheduler(self.optimizer, cfg)

        self.apply(weights_init(cfg['weight_init']))
        self.epoch = 0
        self.iteration = 0
        self.loss_dict = {}
        self.cfg = cfg
        self.logger = logger

        self.to(self.device)
Example #6
0
def result_logger_ids18(fingerprint, cm_ids, cm_tuple,fold_index):
    cmdir = join(fingerprint,'cm')
    recalldir = join(fingerprint,'recall')
    evaldir = join(fingerprint,'eval')
    ensure_dirs([cmdir,recalldir,evaldir])
    (cm_any, cm_majority, cm_all) = cm_tuple
    
    _, id_to_label, _ = get_ids18_mappers()
    cm_labels = np.array([id_to_label[cm_id] for cm_id in cm_ids])
    #logging, plotting
    
    plot_confusion_matrix(join(cmdir,'any_{}.jpg'.format(fold_index)), [], [],cm=cm_any, classes=cm_labels, id_to_label=id_to_label)
    plot_confusion_matrix(join(cmdir,'majority_{}.jpg'.format(fold_index)), [], [],cm=cm_majority, classes=cm_labels, id_to_label=id_to_label)
    plot_confusion_matrix(join(cmdir,'all_{}.jpg'.format(fold_index)), [], [],cm=cm_all, classes=cm_labels, id_to_label=id_to_label)

    plot_confusion_matrix(join(cmdir,'any_norm_{}.jpg'.format(fold_index)), [], [],cm=cm_any, classes=cm_labels,id_to_label=id_to_label, normalize=True)
    plot_confusion_matrix(join(cmdir,'majority_norm_{}.jpg'.format(fold_index)), [], [],cm=cm_majority, classes=cm_labels,id_to_label=id_to_label, normalize=True)
    plot_confusion_matrix(join(cmdir,'all_norm_{}.jpg'.format(fold_index)), [], [],cm=cm_all, classes=cm_labels,id_to_label=id_to_label, normalize=True)


    print_evaluation(cm_any, cm_labels, evaldir, fold_index, 'any')
    print_evaluation(cm_majority, cm_labels, evaldir, fold_index, 'majority')
    print_evaluation(cm_all, cm_labels, evaldir, fold_index, 'all')

    print_absolute_recall(cm_any, cm_labels, recalldir, fold_index, 'any')
    print_absolute_recall(cm_majority, cm_labels, recalldir, fold_index, 'majority')
    print_absolute_recall(cm_all, cm_labels, recalldir, fold_index, 'all')
Example #7
0
def split_dataset(root_dset, obj_category, num_expr, test_ins, temporal=False):
    """
    write train.txt, val.txt, test.txt
    """

    path = pjoin(root_dset, "render", obj_category)
    train_ins = [
        ins for ins in os.listdir(path)
        if ins not in test_ins and not ins.startswith('.')
    ]
    print(train_ins)
    print(test_ins)
    print(f'len train_ins = {len(train_ins)}, len test_ins = {len(test_ins)}')

    train_list, val_list, test_list = [], [], []
    for instance in train_ins:
        all_tracks = []
        for track_dir in glob.glob(pjoin(path, instance, '*')):
            frames = glob.glob(pjoin(track_dir, 'cloud', '*'))
            cloud_list = [file for file in frames if file.endswith('.npz')]
            cloud_list.sort(
                key=lambda str: int(str.split('.')[-2].split('/')[-1]))
            if not temporal:  # pick a random view point for each art
                train_list += cloud_list[:-1]
                val_list += cloud_list[-1:]
            else:
                all_tracks.append(cloud_list)
        if temporal:
            train_list += [
                item for sublist in all_tracks[:-2] for item in sublist
            ]
            val_list += [
                item for sublist in all_tracks[-2:] for item in sublist
            ]

    for instance in test_ins:
        for track_dir in glob.glob(pjoin(path, instance, '*')):
            frames = glob.glob(pjoin(track_dir, 'cloud', '*'))
            cloud_list = [file for file in frames if file.endswith('.npz')]
            cloud_list.sort(
                key=lambda str: int(str.split('.')[-2].split('/')[-1]))
            test_list += cloud_list

    output_path = pjoin(root_dset, "splits", obj_category, num_expr)
    ensure_dirs(output_path)
    name_list = ['train.txt', 'val.txt', 'test.txt']
    for name, content in zip(name_list, [train_list, val_list, test_list]):
        print(f'{name}: {len(content)}')
        with open(pjoin(output_path, name), 'w') as f:
            for item in content:
                f.write('{}\n'.format(item))
Example #8
0
def get_config(args, save=True):
    base_path = os.path.dirname(__file__)
    f = open(pjoin(base_path, 'all_config', args.config), 'r')
    cfg = yaml.load(f, Loader=yaml.FullLoader)

    """ Update info from command line """
    args = vars(args)  # convert to a dictionary!
    args.pop('config')
    for key, item in args.items():
        if item is not None:
            key_path = key.split('/')
            overwrite_config(cfg, key, key_path, item)

    """ Load object config """
    f = open(pjoin(base_path, 'obj_config', cfg['obj_config']), 'r')
    obj_cfg = yaml.load(f, Loader=yaml.FullLoader)

    """ Load pointnet config """
    point_cfg_path = pjoin(base_path, 'pointnet_config')
    pointnet_cfgs = cfg['pointnet_cfg']
    cfg['pointnet'] = {}
    for key, value in pointnet_cfgs.items():
        f = open(pjoin(point_cfg_path, value), 'r')
        cfg['pointnet'][key] = yaml.load(f, Loader=yaml.FullLoader)

    """ Save config """
    root_dir = cfg['experiment_dir']
    ensure_dirs(root_dir)
    cfg['num_expr'] = root_dir.split('/')[-1]

    if save:
        yml_cfg = pjoin(root_dir, 'config.yml')
        yml_obj = pjoin(root_dir, cfg['obj_config'])
        print('Saving config and obj_config at {} and {}'.format(yml_cfg, yml_obj))
        for yml_file, content in zip([yml_cfg, yml_obj], [cfg, obj_cfg]):
            with open(yml_file, 'w') as f:
                yaml.dump(content, f, default_flow_style=False)

    """ Fill in additional info """
    obj_cat = cfg["obj_category"]
    cfg["num_parts"] = obj_cfg[obj_cat]["num_parts"]
    cfg["num_joints"] = obj_cfg[obj_cat]["num_joints"]
    cfg["obj_tree"] = obj_cfg[obj_cat]["tree"]
    cfg["obj_sym"] = obj_cfg[obj_cat]["sym"]
    cfg["obj"] = obj_cfg
    cfg["obj_info"] = obj_cfg[obj_cat]
    cfg["root_dset"] = obj_cfg['basepath']
    cfg["device"] = torch.device("cuda:%d" % cfg['cuda_id']) if torch.cuda.is_available() else "cpu"
    print("Running on ", cfg["device"])

    return cfg
Example #9
0
def generate_full_data(root_dset, obj_category, instance, track_num, frame_i, num_points):
    preproc_path = pjoin(root_dset, 'preproc',
                         obj_category, instance, f'{track_num}', 'full')
    ensure_dirs(preproc_path)
    cur_preproc_path = pjoin(preproc_path, f'{frame_i}.npz')
    if os.path.exists(cur_preproc_path):
        all = np.load(cur_preproc_path, allow_pickle=True)
        points = all['point']
        ret = {'points': points}
        if 'pose' in all:
            ret.update({'nocs2camera': [pose for pose in all['pose']], 'nocs_corners': all['corners']})

    cloud_path = pjoin(root_dset, 'render',
                 obj_category, instance, f'{track_num}', 'cloud', f'{frame_i}.npz')

    points = np.load(cloud_path, allow_pickle=True)['point']

    while len(points) < num_points:
        points = np.concatenate([points, points], axis=0)
    fps_idx = farthest_point_sample(points, num_points, device='cuda:0')
    points = points[fps_idx]

    ret = {'points': points}

    pose_path = pjoin(root_dset, 'real_pose', obj_category, instance, f'{track_num}.json')
    meta_path = pjoin(root_dset, 'real_pose', obj_category, instance, 'meta.json')
    if os.path.exists(pose_path) and os.path.exists(meta_path):
        with open(pose_path, 'r') as f:
            all_pose = json.load(f)
        with open(meta_path, 'r') as f:
            meta = json.load(f)
        if obj_category == 'drawers':
            num_parts = 4
            name2num = {'drawer3': 0, 'drawer2': 1, 'drawer1': 2, 'body': 3}
        num2name = {value: key for key, value in name2num.items()}
        extents = np.stack([meta[num2name[p]]['size'] for p in range(num_parts)], axis=0)  # [P, 3]
        radius = np.sqrt(np.sum(extents ** 2, axis=-1))  # [P]
        extents /= radius.reshape(num_parts, 1)
        corners = np.stack([-extents * 0.5, extents * 0.5], axis=1)  # [P, 2, 3]

        mat = np.array([[0, 0, 1], [-1, 0, 0], [0, -1, 0]])
        nocs2camera = [{'rotation': np.matmul(mat, np.array(all_pose[int(frame_i)][num2name[p]]['R']).reshape(3, 3)),
                        'translation': np.matmul(mat, np.array(all_pose[int(frame_i)][num2name[p]]['t']).reshape(3, 1)),
                        'scale': float(radius[p])} for p in range(num_parts)]
        np.savez_compressed(cur_preproc_path, point=points, pose=nocs2camera, corners=corners)
        ret.update({'nocs2camera': nocs2camera, 'nocs_corners': corners})
    else:
        np.savez_compressed(cur_preproc_path, point=points)
    return ret
Example #10
0
  def write (self, context, fname):
    if fname is None:
      fname = self.fname

    fname = Template (fname).safe_substitute (config)

    if fname.startswith ('/'):  # assume the caller know what they are doing, and it's a full path
      pth = fname
    else:
      ensure_dirs (templates_path + fname)
      pth = templates_path + fname

    content = context or self.content

    with open (pth, 'w') as f:
      f.write (content.encode ('UTF-8'))
      if trace: print fname, 'WRITE OK'
Example #11
0
  def write (self, context, fname):
    if fname is None:
      fname = self.fname

    fname = Template (fname).safe_substitute (config)

    if fname.startswith ('/'):  # assume the caller know what they are doing, and it's a full path
      pth = fname
    else:
      pth = os.path.join (config.templates_path, fname)
      ensure_dirs (pth)

    content = context or self.content

    with open (pth, 'w') as f:
      f.write (content.encode ('UTF-8'))
      if trace: print fname, 'WRITE OK'
Example #12
0
def main(args):
    if args.data_type in ['real_train', 'real_test']:
        intrinsics = np.array([[591.0125, 0, 322.525],
                               [0, 590.16775, 244.11084], [0, 0, 1]])
    else:
        intrinsics = np.array([[577.5, 0, 319.5], [0., 577.5, 239.5],
                               [0., 0., 1.]])

    data_path = pjoin(args.data_path, args.data_type)
    list_path = pjoin(args.list_path, args.data_type, str(args.category))
    model_path = args.model_path
    output_path = pjoin(args.output_path, args.data_type, str(args.category))
    ensure_dirs(output_path)
    instances = list(map(lambda s: s.split('.')[0], os.listdir(list_path)))
    instances.sort()

    if not args.parallel:
        gather_instances(list_path,
                         data_path,
                         model_path,
                         output_path,
                         instances,
                         intrinsics,
                         flip=args.data_type
                         not in ['real_train', 'real_test'],
                         real=args.data_type in ['real_train', 'real_test'])
    else:
        processes = []
        proc_cnt = args.num_proc
        num_per_proc = int((len(instances) - 1) / proc_cnt) + 1

        for k in range(proc_cnt):
            s_ind = num_per_proc * k
            e_ind = min(num_per_proc * (k + 1), len(instances))
            p = Process(target=gather_instances,
                        args=(list_path, data_path, model_path, output_path,
                              instances[s_ind:e_ind], intrinsics,
                              args.data_type
                              not in ['real_train',
                                      'real_test'], args.data_type
                              in ['real_train', 'real_test']))
            processes.append(p)
            p.start()
        """
Example #13
0
def process_pkl(pkl_path, output_path):
    for folder in ['rgb', 'cloud']:
        ensure_dirs(pjoin(output_path, folder))
    with open(pkl_path, 'rb') as f:
        all_dict = pickle.load(f)  # 'point_cloud', 'image', 'time'
    points_list, image_list = all_dict['point_cloud'], all_dict['image']
    num_frames = len(points_list)

    for i in range(num_frames):
        img = image_list[i]
        raw_point = points_list[i]
        cv2.imwrite(pjoin(output_path, 'rgb', f'{i}.png'), img)
        point = np.stack(
            [raw_point[..., 2], -raw_point[..., 0], -raw_point[..., 1]],
            axis=-1)
        # [[0, 0, 1], [-1, 0, 0], [0, -1, 0]]
        # plot3d_pts([[point]], show_fig=True)
        np.savez_compressed(pjoin(output_path, 'cloud', f'{i}.npz'),
                            point=point)
Example #14
0
def result_logger_ddos19(fingerprint, y_test_perflowid_list, pred_list_tuples,
                         id_to_label):
    cmdir = join(fingerprint, 'cm')
    recalldir = join(fingerprint, 'recall')
    evaldir = join(fingerprint, 'eval')
    ensure_dirs([cmdir, recalldir, evaldir])

    (pred_any_list, pred_majority_list, pred_all_list) = pred_list_tuples

    # cm
    cm_any = confusion_matrix(np.array(y_test_perflowid_list),
                              np.array(pred_any_list))
    cm_majority = confusion_matrix(np.array(y_test_perflowid_list),
                                   np.array(pred_majority_list))
    cm_all = confusion_matrix(np.array(y_test_perflowid_list),
                              np.array(pred_all_list))

    cm_ids = np.unique(y_test_perflowid_list)
    cm_labels = np.array([id_to_label[cm_id] for cm_id in cm_ids])

    #logging, plotting
    plot_confusion_matrix(join(cmdir, 'any_norm.jpg'), [], [],
                          cm=cm_any,
                          classes=cm_labels,
                          normalize=True)
    plot_confusion_matrix(join(cmdir, 'majority_norm.jpg'), [], [],
                          cm=cm_majority,
                          classes=cm_labels,
                          normalize=True)
    plot_confusion_matrix(join(cmdir, 'all_norm.jpg'), [], [],
                          cm=cm_all,
                          classes=cm_labels,
                          normalize=True)

    print_evaluation_ddos19(cm_any, cm_labels, join(evaldir, 'any'))
    print_evaluation_ddos19(cm_majority, cm_labels, join(evaldir, 'majority'))
    print_evaluation_ddos19(cm_all, cm_labels, join(evaldir, 'all'))

    print_absolute_recall_ddos19(cm_any, cm_labels, join(recalldir, 'any'))
    print_absolute_recall_ddos19(cm_majority, cm_labels,
                                 join(recalldir, 'majority'))
    print_absolute_recall_ddos19(cm_all, cm_labels, join(recalldir, 'all'))
    def __init__(self, phase):
        self.is_train = phase == "train"

        # init hyperparameters and parse from command-line
        parser, args = self.parse()

        # set as attributes
        print("----Experiment Configuration-----")
        for k, v in args.__dict__.items():
            print("{0:20}".format(k), v)
            self.__setattr__(k, v)

        # experiment paths
        self.exp_dir = os.path.join(self.proj_dir, self.exp_name)
        if phase == "train" and args.cont is not True and os.path.exists(
                self.exp_dir):
            response = input(
                'Experiment log/model already exists, overwrite? (y/n) ')
            if response != 'y':
                exit()
            shutil.rmtree(self.exp_dir)

        self.log_dir = os.path.join(self.exp_dir, 'log')
        self.model_dir = os.path.join(self.exp_dir, 'model')
        ensure_dirs([self.log_dir, self.model_dir])

        # GPU usage
        if args.gpu_ids is not None:
            os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu_ids)

        # create soft link to experiment log directory
        if not os.path.exists('train_log'):
            os.symlink(self.exp_dir, 'train_log')

        # save this configuration
        if self.is_train:
            with open(os.path.join(self.exp_dir, 'config.json'), 'w') as f:
                json.dump(args.__dict__, f, indent=2)
            copy_code_dir = os.path.join(self.exp_dir, "code")
            ensure_dirs(copy_code_dir)
            os.system("cp *.py {}".format(copy_code_dir))
Example #16
0
def data_split(dataset_dir, train_percent, valid_percent, filelists_dir,
               train_file, valid_file):
    if not pt.exists(dataset_dir):
        print(f"Error: Dataset directory '{dataset_dir}' does not exist.")
    utils.ensure_dirs(filelists_dir)
    dataset_file = pt.join(dataset_dir, "metadata.csv")
    wavs_dir = pt.join(dataset_dir, "wavs")
    train_filepath = pt.join(filelists_dir, train_file)
    valid_filepath = pt.join(filelists_dir, valid_file)

    data = [l for l in open(dataset_file, "r")]

    train_file = open(train_filepath, "w")
    valid_file = open(valid_filepath, "w")

    num_of_data = len(data)
    num_train = int((train_percent / 100.0) * num_of_data)
    num_valid = int((valid_percent / 100.0) * num_of_data)

    data_fractions = [num_train, num_valid]
    split_data = [[], [], []]

    rand_data_ind = 0

    for split_ind, fraction in enumerate(data_fractions):
        for _i in range(fraction):
            rand_data_ind = random.randint(0, len(data) - 1)
            file, text = data[rand_data_ind].split("|")[:2]
            file = utils.strip(pt.basename(file), ".wav")
            l = pt.join(wavs_dir, file + ".wav") + "|" + text.strip() + "\n"
            split_data[split_ind].append(l)
            data.pop(rand_data_ind)

    for l in split_data[0]:
        train_file.write(l)

    for l in split_data[1]:
        valid_file.write(l)

    train_file.close()
    valid_file.close()
Example #17
0
def main(args):
    root_path = pjoin(args.data_path, args.data_type)
    folders = os.listdir(
        root_path
    )  # [folder for folder in os.listdir(root_path) if os.path.isdir(pjoin(root_path, folder))]
    folders.sort()
    output_path = pjoin(args.list_path, args.data_type)
    ensure_dirs(output_path)

    data_list = get_valid_instance(root_path,
                                   folders,
                                   real=args.data_type
                                   in ['real_train', 'real_test'])

    for cls_id in data_list:
        cur_path = pjoin(output_path, str(cls_id))
        ensure_dirs(cur_path)
        for instance_id in data_list[cls_id]:
            with open(pjoin(cur_path, f'{instance_id}.txt'), 'w') as f:
                for line in data_list[cls_id][instance_id]:
                    print(line, file=f)
Example #18
0
def split_real_dataset(root_dset, obj_category, num_expr, test_ins):
    """
    write real_test.txt
    """

    path = pjoin(root_dset, "render", obj_category)

    test_list = []
    for instance in test_ins:
        for track_dir in glob.glob(pjoin(path, instance, '*')):
            frames = glob.glob(pjoin(track_dir, 'cloud', '*'))
            cloud_list = [file for file in frames if file.endswith('.npz')]
            cloud_list.sort(
                key=lambda str: int(str.split('.')[-2].split('/')[-1]))
            test_list += cloud_list

    output_path = pjoin(root_dset, "splits", obj_category, num_expr)
    ensure_dirs(output_path)
    with open(pjoin(output_path, 'real_test.txt'), 'w') as f:
        for item in test_list:
            f.write('{}\n'.format(item))
Example #19
0
class Config:
    proj_dir = '/data1/wurundi/CycleGAN_motion/'
    exp_name = os.getcwd().split('/')[-1]
    exp_dir = os.path.join(proj_dir, exp_name)
    log_dir = os.path.join(exp_dir, 'log/')
    model_dir = os.path.join(exp_dir, 'model/')

    stat_path = os.path.join(proj_dir, 'statistic.csv')

    device = None
    isTrain = None

    # data
    base_id = {"h": 0, "nh": 0}

    # model parameters
    input_nc = 63
    output_nc = 45
    ngf = 64
    ndf = 64
    n_layers_G = 3
    n_layers_D = 2
    G_en_ks = 8
    G_de_ks = 7
    D_ks = 8

    direction = 'AtoB'

    # training parameters
    niter = 50  # 100
    niter_decay = 50  # 100
    beta1 = 0.5
    lr = 0.0002
    gan_mode = 'lsgan'
    pool_size = 50
    lr_policy = 'linear'
    lr_decay_iters = 50

    # loss weight
    lambda_A = 10.0  # weight for cycle loss (A -> B -> A)
    lambda_B = 10.0  # weight for cycle loss (B -> A -> B)

    nr_epochs = niter + niter_decay
    batch_size = 64
    save_frequency = 10
    val_frequency = 50
    visualize_frequency = 200

    utils.ensure_dirs([proj_dir, log_dir, exp_dir, model_dir])
def setup():
    ensure_dirs(["./speakers", "./speakers/LJ"])
def classify(dataroot,classifier_name):
        global K
        K=5
        #fraction = 1
        fraction = 0.001
        
        #total_records_in_whole = 6907723;
        total_records = 6907705; # in fold
        folds_df = []
        fold_root = join(dataroot,'folds_fraction_{}'.format(fraction))
        
        ds_list = []
        for fold_index in range(K):
            df = pd.read_csv(join(fold_root,'fold_{}.csv'.format(fold_index))) 
            folds_df.append(df)
            ds_list.append(df.Label)
        total_label_df = pd.concat(ds_list,sort=False)
        labels,labels_d = get_labels(total_label_df.unique())
        class_weight = get_class_weights(encode_label(total_label_df.values,labels_d))
        
        #balance = 'sample_per_batch'
        #balance = 'with_loss'
        balance = 'explicit'
 
        input_dim = folds_df[0].shape[1]-2 # because we remove Label and FlowID columns from X
        labels,labels_d = get_labels(folds_df[0].Label.unique())

        num_class = len(labels)

        if classifier_name in ['cnn','softmax']:
            batch_size =256
            num_iters = 0.1*(total_records*.8*.9)//batch_size # 10 epochs for total dataset
            optim='Adam'  
            if classifier_name=='cnn':
                lr =1e-3
                reg = 0
                device = [0,1]

            elif classifier_name=='softmax':
                lr = 1e-3
                reg =0 
                device = 'cuda:0'
            classifier_args = {'classifier_name':classifier_name,'optim':optim,'lr':lr,'reg':reg,'batch_size':batch_size,'input_dim':input_dim,'num_class':num_class,'num_iters':num_iters,'device':device, 'balance':balance, 'class_weight':class_weight}
            config =  '_optim_{}_lr_{}_reg_{}_bs_{}_b_{}'.format(optim,lr,reg,batch_size,balance)
        else:
            lr = None
            reg = None
            batch_size=None
            device=None
            classifier_args = {'classifier_name':classifier_name,'balance':balance}
            config = '_b_{}'.format(balance)
        
        
        pre_fingerprint = join(dataroot,'classifiers','kfold', 'r_{}_c_{}_k_{}'.format(fraction,classifier_name,str(K)))
            
        fingerprint = pre_fingerprint + config
        print("Running experiment \n ",fingerprint)
        logdir = join(fingerprint,'log')
        cmdir = join(fingerprint,'cm')
        recalldir = join(fingerprint,'recall')
        evaldir = join(fingerprint,'eval')

        ensure_dirs(fingerprint,logdir,cmdir,recalldir,evaldir)       

        confusion_matrix_sum = np.zeros((num_class, num_class),dtype=float)
        majority_confusion_matrix_sum = np.zeros((num_class, num_class),dtype=float)
        kfold_pred_time = 0
        kfold_feature_importance = np.zeros(input_dim,dtype=np.float)
        skf = StratifiedKFold(n_splits=K,random_state=SEED)
        for fold_index in range(K):
            print("Fold ",fold_index)
            test_df = folds_df[fold_index]
            train_df = pd.concat([folds_df[i] for i in range(K) if i!=fold_index],sort=False)
            
            X_train, y_train = df_to_array(train_df)
            y_train = encode_label(y_train,labels_d)

            runs_dir=join(logdir,'fold_{}'.format(fold_index))
            clf,duration = train_fold(X_train,y_train,fold_index,classifier_args,runs_dir)
            if classifier_name=='forest':
                kfold_feature_importance+=clf.feature_importances_
            
            flowids_test,y_flowid_test = group_data(test_df)
            y_flowid_test = encode_label(y_flowid_test,labels_d)
            pred_any, pred_majority = predict_fold(classifier_name,clf,test_df, flowids_test, y_flowid_test)
            
            assert pred_any.shape==pred_majority.shape,"any and majority shapes should be same {},{}".format(pred_any.shape,pred_majority.shape)
            #assert pred.shape==y_flowid_test.shape, "y_true={} and pred.shape={} should be same ".format(y_flowid_test.shape,pred.shape)
            acc_pred_any = metrics.balanced_accuracy_score(y_flowid_test,pred_any)
            acc_pred_majority = metrics.balanced_accuracy_score(y_flowid_test,pred_majority)
            print("Fold Balanced accuracy(any,majority): ({:.2f},{:.2f})".format(acc_pred_any,acc_pred_majority))
            kfold_pred_time+=duration

            plot_confusion_matrix(join(cmdir,'any_fold_{}.jpg'.format(fold_index)), y_flowid_test, pred_any, classes=labels,normalize=False, title='Confusion matrix, with normalization')
            plot_confusion_matrix(join(cmdir,'any_norm_fold_{}.jpg'.format(fold_index)), y_flowid_test, pred_any, classes=labels,normalize=True, title='Confusion matrix, with normalization')         
            cm_i = confusion_matrix(y_flowid_test,pred_any)
            print_absolute_recall(cm_i, labels, join(recalldir,'any_fold_{}.csv'.format(fold_index)),fold_root)
            print_evaluation(cm_i, labels, join(evaldir,'any_fold_{}.csv'.format(fold_index)))
            confusion_matrix_sum+=cm_i
       

            plot_confusion_matrix(join(cmdir,'majority_fold_{}.jpg'.format(fold_index)), y_flowid_test, pred_majority, classes=labels,normalize=False, title='Confusion matrix, with normalization')
            plot_confusion_matrix(join(cmdir,'majority_norm_fold_{}.jpg'.format(fold_index)), y_flowid_test, pred_majority, classes=labels,normalize=True, title='Confusion matrix, with normalization')                       
            majority_cm_i = confusion_matrix(y_flowid_test,pred_majority)
            print_absolute_recall(majority_cm_i, labels, join(recalldir,'majority_fold_{}.csv'.format(fold_index)),fold_root)
            print_evaluation(majority_cm_i, labels, join(evaldir,'majority_fold_{}.csv'.format(fold_index)))
            majority_confusion_matrix_sum+=majority_cm_i

        if classifier_name=='forest':
            print_feature_importance(kfold_feature_importance,join(dataroot,'folds_fraction_{}'.format(fraction),'feature_selection.csv'))


        cm = confusion_matrix_sum
        cm_majority = majority_confusion_matrix_sum
        print(dataroot,classifier_name)

 
        plot_confusion_matrix(join(cmdir,'avg_any_fold.jpg'), [], [],cm=cm, classes=labels, title='Confusion matrix, without normalization')
        plot_confusion_matrix(join(cmdir,'avg_any_norm_fold.jpg'), [], [],cm=cm, classes=labels, normalize=True, title='Confusion matrix, with normalization')

        
        plot_confusion_matrix(join(cmdir,'avg_majority_fold.jpg'), [], [],cm=cm_majority, classes=labels, title='Confusion matrix, without normalization')
        plot_confusion_matrix(join(cmdir,'avg_majority_norm_fold.jpg'), [], [],cm=cm_majority, classes=labels, normalize=True, title='Confusion matrix, with normalization')
        
        print_evaluation(cm, labels, join(fingerprint,'evaluation_any.csv'))
        print_evaluation(cm_majority, labels, join(fingerprint,'evaluation_majority.csv'))
        
        print_absolute_recall(cm, labels, join(fingerprint,'recall_any.csv'),fold_root,kfold=True)
        print_absolute_recall(cm_majority, labels, join(fingerprint,'recall_majority.csv'),fold_root,kfold=True)
def setup():
    ensure_dirs(["./speakers", "./speakers/TIMIT"])
Example #23
0
def main(args):
    def log_string(str):
        logger.info(str)
        print(str)

    cfg = get_config(args)
    '''LOG'''
    log_dir = pjoin(cfg['experiment_dir'], 'log')
    ensure_dirs(log_dir)

    logger = logging.getLogger("TrainModel")
    logger.setLevel(logging.INFO)
    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    file_handler = logging.FileHandler('%s/log.txt' % (log_dir))
    file_handler.setLevel(logging.INFO)
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)
    log_string('PARAMETER ...')
    log_string(cfg)
    '''DATA'''
    train_dataloader = get_dataloader(cfg, 'train', shuffle=True)
    test_dataloader = get_dataloader(cfg, 'test')

    if args.use_val is not None:
        val_dataloader = get_dataloader(cfg, args.use_val)
    else:
        val_dataloader = None
    '''TRAINER'''
    trainer = Trainer(cfg, logger)
    start_epoch = trainer.resume()

    def test_all():
        '''testing'''
        test_loss = {}
        for i, data in enumerate(test_dataloader):
            pred_dict, loss_dict = trainer.test(data)
            loss_dict['cnt'] = 1
            add_dict(test_loss, loss_dict)

        cnt = test_loss.pop('cnt')
        log_loss_summary(test_loss, cnt,
                         lambda x, y: log_string('Test {} is {}'.format(x, y)))

        if val_dataloader is not None:
            val_loss = {}
            for i, data in enumerate(val_dataloader):
                pred_dict, loss_dict = trainer.test(data)
                loss_dict['cnt'] = 1
                add_dict(val_loss, loss_dict)

            cnt = val_loss.pop('cnt')
            log_loss_summary(
                val_loss, cnt, lambda x, y: log_string('{} {} is {}'.format(
                    args.use_val, x, y)))

    for epoch in range(start_epoch, cfg['total_epoch']):
        trainer.step_epoch()
        train_loss = {}
        '''training'''
        for i, data in enumerate(train_dataloader):
            loss_dict = trainer.update(data)
            loss_dict['cnt'] = 1
            add_dict(train_loss, loss_dict)

        cnt = train_loss.pop('cnt')
        log_loss_summary(
            train_loss, cnt,
            lambda x, y: log_string('Train {} is {}'.format(x, y)))

        if (epoch + 1) % cfg['freq']['save'] == 0:
            trainer.save()

        test_all()
def setup():
    ensure_dirs(["./noise"])
Example #25
0
 def __init__(self, cfg, mode='train', downsampling=None):
     super(PointData, self).__init__()
     self.cfg = cfg
     self.mode = mode
     obj_cfg = cfg['obj']
     root_dir = obj_cfg['basepath']
     ctgy = cfg['obj_category']
     obj_info = obj_cfg[ctgy]
     num_expr = cfg['num_expr']
     num_points = cfg['num_points']
     truncate_length = cfg['dataset_length']
     synthetic = obj_cfg['synthetic']
     perturb = 'perturb' in obj_cfg and obj_cfg['perturb']
     preproc_dir = pjoin(root_dir, 'preproc', ctgy)
     ensure_dirs(preproc_dir)
     self.synthetic = synthetic
     self.nocs_data = 'nocs_data' in obj_cfg and obj_cfg['nocs_data']
     self.real_data = self.mode in ['real_test'] and not self.nocs_data
     self.bmvc_data = 'bmvc' in self.mode
     if self.nocs_data:
         self.dataset = NOCSDataset(root_dset=root_dir,
                                    obj_category=ctgy,
                                    obj_info=obj_info,
                                    num_expr=num_expr,
                                    num_points=num_points,
                                    mode=mode,
                                    truncate_length=truncate_length,
                                    synthetic=synthetic,
                                    radius=cfg['data_radius'],
                                    perturb_cfg=cfg['pose_perturb'],
                                    device=cfg['device'],
                                    downsampling=downsampling)
     elif self.real_data:
         self.dataset = SAPIENRealDataset(root_dset=root_dir,
                                          obj_category=ctgy,
                                          obj_info=obj_info,
                                          num_expr=num_expr,
                                          num_points=num_points,
                                          truncate_length=truncate_length)
     elif self.bmvc_data:
         self.dataset = BMVCDataset(root_dset=root_dir,
                                    obj_category=ctgy,
                                    track=int(self.mode.split('_')[-1]),
                                    truncate_length=truncate_length)
     else:
         self.dataset = SAPIENDataset(root_dset=root_dir,
                                      obj_category=ctgy,
                                      obj_info=obj_info,
                                      num_expr=num_expr,
                                      num_points=num_points,
                                      mode=mode,
                                      truncate_length=truncate_length,
                                      synthetic=synthetic,
                                      perturb=perturb,
                                      device=cfg['device'])
     self.ins_info = self.dataset.ins_info
     self.data = {}
     self.num_points = cfg['num_points']
     self.tree = cfg['obj_tree']
     self.root = [p for p in range(len(self.tree)) if self.tree[p] == -1][0]
     self.num_parts = len(self.tree)
Example #26
0
def preprocess_zeta(source_dir, output_dir, include_path, metadata_filename):
    source_dir = pt.normpath(source_dir)
    output_dir = pt.normpath(output_dir)

    wavs_dir = pt.join(output_dir, "wavs")
    metadata_csv_path = pt.join(output_dir, metadata_filename)

    utils.safe_remove(output_dir)
    utils.ensure_dirs(source_dir, output_dir, wavs_dir)
    audio_paths = []
    for ext in AUDIO_EXTS:
        audio_paths.extend(glob(pt.join(source_dir, "*" + ext)))
    rows = []
    for ap in audio_paths:
        apb = pt.basename(ap)
        stripped = utils.strip_audio_ext(ap)
        stripped_apb = utils.strip_audio_ext(apb)
        big_wav_path = stripped + ".wav"

        sub_path = stripped + SUBTITLE_EXT
        cut_path = stripped + IGNORE_EXT

        if not pt.exists(sub_path):
            continue
        idx = 1
        cuts = process_cuts(cut_path)
        if not pt.exists(big_wav_path):
            subprocess.run(["ffmpeg", "-i", ap, big_wav_path])
            subprocess.run([
                "sox",
                big_wav_path,
                "-b",
                "16",
                "tmp.wav",
                "rate",
                "22050",
                "channels",
                "1",
            ])
            utils.safe_remove(big_wav_path)
            os.rename("tmp.wav", big_wav_path)

        print(sub_path)
        for (line, start, end) in gen_subs(sub_path):
            in_cut = False
            for (cut_start, cut_end) in cuts:
                if cut_start < end < cut_end or cut_start < start < cut_end:
                    in_cut = True
                    break

            if not in_cut and 7000 > (end - start) > 100:
                if "[" in line and "]" in line:
                    continue
                p = pt.join(wavs_dir, stripped_apb + "_" + str(idx) + ".wav")
                subprocess.run([
                    "sox",
                    big_wav_path,
                    p,
                    "trim",
                    str(start / 1000),
                    "=" + str(end / 1000),
                    "silence",
                    "1",
                    "0.1",
                    "0.5%",
                    "reverse",
                    "silence",
                    "1",
                    "0.1",
                    "0.5%",
                    "reverse",
                ])
                if not pt.exists(p):
                    raise Exception("Clip wasn't made??")
                sound = AudioSegment.from_file(p)
                if len(sound) < 2000:
                    utils.safe_remove(p)
                    continue
                rows.append([
                    p if include_path else pt.splitext(pt.basename(p))[0],
                    line, line
                ])
                idx += 1
                print("Finished " + p)
            utils.safe_remove(big_wav_path)
    with open(metadata_csv_path, "w", newline="") as csvfile:
        writer = csv.writer(csvfile, delimiter="|", quoting=csv.QUOTE_MINIMAL)
        writer.writerows(rows)
Example #27
0
def gather_instance(list_path,
                    data_path,
                    model_path,
                    output_path,
                    instance,
                    intrinsics,
                    flip=True,
                    real=False,
                    img_per_folder=100,
                    render_rgb=False):
    corners = np.load(pjoin(model_path, f'{instance}.npy'))
    bbox = bbox_from_corners(corners)
    bbox *= 1.4
    meta_path = pjoin(list_path, f'{instance}.txt')
    with open(meta_path, 'r') as f:
        lines = f.readlines()

    inst_output_path = pjoin(output_path, instance)
    if not real:
        folder_num, img_num = 0, -1
        cur_folder_path = pjoin(inst_output_path, f'{folder_num:04d}')
        ensure_dirs([
            pjoin(cur_folder_path, name)
            for name in (['data'] if not render_rgb else ['rgb', 'data'])
        ])

    meta_dict = {}

    for line in tqdm(lines, desc=f'Instance {instance}'):
        track_name, prefix = line.strip().split('/')[:2]
        file_path = pjoin(data_path, track_name)
        if real and track_name not in meta_dict:
            meta_dict[track_name] = file_path
        suffix = 'depth' if real else 'composed'
        try:
            depth = cv2.imread(pjoin(file_path, f'{prefix}_{suffix}.png'), -1)
            mask = cv2.imread(pjoin(file_path, f'{prefix}_mask.png'))[:, :, 2]
            rgb = cv2.imread(pjoin(file_path, f'{prefix}_color.png'))
            with open(pjoin(file_path, f'{prefix}_meta.txt'), 'r') as f:
                meta_lines = f.readlines()
            with open(pjoin(file_path, f'{prefix}_pose.pkl'), 'rb') as f:
                pose_dict = pickle.load(f)
        except:
            continue
        if flip:
            depth, mask, rgb = depth[:, ::-1], mask[:, ::-1], rgb[:, ::-1]
        inst_num = -1
        for meta_line in meta_lines:
            inst_num = int(meta_line.split()[0])
            inst_id = meta_line.split()[-1]
            if inst_id == instance:
                break
        if inst_num not in pose_dict:
            continue
        pose = pose_dict[inst_num]
        posed_bbox = (np.matmul(bbox, pose['rotation'].swapaxes(-1, -2)) *
                      np.expand_dims(pose['scale'], (-1, -2)) +
                      pose['translation'].swapaxes(-1, -2))
        center = posed_bbox.mean(axis=0)
        radius = np.sqrt(np.sum((posed_bbox[0] - center)**2)) + 0.1
        aa_corner = get_corners(
            [center - np.ones(3) * radius, center + np.ones(3) * radius])
        aabb = bbox_from_corners(aa_corner)
        height, width = mask.shape
        projected_corners = project(aabb, intrinsics).astype(np.int32)[:,
                                                                       [1, 0]]
        projected_corners[:, 0] = height - projected_corners[:, 0]
        corner_2d = np.stack([
            np.min(projected_corners, axis=0),
            np.max(projected_corners, axis=0)
        ],
                             axis=0)
        corner_2d[0, :] = np.maximum(corner_2d[0, :], 0)
        corner_2d[1, :] = np.minimum(corner_2d[1, :],
                                     np.array([height - 1, width - 1]))
        corner_mask = np.zeros_like(mask)
        corner_mask[corner_2d[0, 0]:corner_2d[1, 0] + 1,
                    corner_2d[0, 1]:corner_2d[1, 1] + 1] = 1
        cropped_rgb = rgb[corner_2d[0, 0]:corner_2d[1, 0] + 1,
                          corner_2d[0, 1]:corner_2d[1, 1] + 1]

        raw_pts, raw_idx = backproject(depth,
                                       intrinsics=intrinsics,
                                       mask=corner_mask)
        raw_mask = (mask == inst_num)[raw_idx[0], raw_idx[1]]

        def filter_box(pts, corner):
            mask = np.prod(
                np.concatenate([pts >= corner[0], pts <= corner[1]],
                               axis=1).astype(np.int8),  # [N, 6]
                axis=1)
            idx = np.where(mask == 1)[0]
            return pts[idx]

        def filter_ball(pts, center, radius):
            distance = np.sqrt(np.sum((pts - center)**2, axis=-1))  # [N]
            idx = np.where(distance <= radius)
            return pts[idx], idx

        pts, idx = filter_ball(raw_pts, center, radius)
        obj_mask = raw_mask[idx]

        data_dict = {
            'points': pts,
            'labels': obj_mask,
            'pose': pose,
            'path': pjoin(file_path, f'{prefix}_{suffix}.png')
        }
        if not real:
            img_num += 1
            if img_num >= img_per_folder:
                folder_num += 1
                cur_folder_path = pjoin(inst_output_path, f'{folder_num:04d}')
                ensure_dirs([
                    pjoin(cur_folder_path, name) for name in (
                        ['data'] if not render_rgb else ['rgb', 'data'])
                ])
                img_num = 0
            np.savez_compressed(pjoin(cur_folder_path, 'data',
                                      f'{img_num:02d}.npz'),
                                all_dict=data_dict)
            if render_rgb:
                cv2.imwrite(
                    pjoin(cur_folder_path, 'rgb', f'{img_num:02d}.png'),
                    cropped_rgb)
        else:
            cur_folder_path = pjoin(inst_output_path, track_name)
            ensure_dirs(pjoin(cur_folder_path, 'data'))
            np.savez_compressed(pjoin(cur_folder_path, 'data',
                                      f'{prefix}.npz'),
                                all_dict=data_dict)

    if real:
        cur_folder_path = pjoin(inst_output_path, track_name)
        ensure_dirs([cur_folder_path])
        for track_name in meta_dict:
            with open(pjoin(cur_folder_path, 'meta.txt'), 'w') as f:
                print(meta_dict[track_name], file=f)
Example #28
0
from keras.layers.recurrent import GRU
from keras.layers import Dense, Conv1D, LeakyReLU, Dropout, BatchNormalization, TimeDistributed, ZeroPadding1D, CuDNNGRU
from keras.optimizers import Adam
from keras.callbacks import TensorBoard, ModelCheckpoint
import datetime
from matplotlib import pyplot

from utils import ensure_dirs
from VADSequence import VADSequence
from dataset_loader import vad_voice_train, vad_noise_train, vad_voice_test, vad_noise_test

model_name = "vad2"
run = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M")
gpu = True

ensure_dirs(["./models", "./models/" + run])

opt = Adam(lr=0.0001, decay=0.00005)

batch_size = 100
timeseries_length = 100
nb_epochs = 60

voice_train = vad_voice_train()
noise_train = vad_noise_train()
print("Voice train set shape", voice_train.shape)
print("Noise train set shape", noise_train.shape)
train_generator = VADSequence(voice_train,
                              noise_train,
                              timeseries_length=timeseries_length,
                              batch_size=batch_size,
Example #29
0
def safe_save_fig(fig, save_path):
    ensure_dirs(os.path.dirname(save_path))
    fig.savefig(save_path)
Example #30
0
    def forward(self, save=False):
        self.timer.tick()

        pred_poses = []
        gt_part = self.feed_dict[0]['gt_part']
        if self.gt_init:
            pred_poses.append(gt_part)
        else:
            part = add_noise_to_part_dof(gt_part, self.pose_perturb_cfg)
            if 'crop_pose' in self.feed_dict[0]['meta']:
                crop_pose = part_model_batch_to_part(
                    cvt_torch(self.feed_dict[0]['meta']['crop_pose'],
                              self.device), self.num_parts, self.device)
                for key in ['translation', 'scale']:
                    part[key] = crop_pose[key]
            pred_poses.append(part)

        self.timer.tick()

        time_dict = {'crop': 0.0, 'npcs_net': 0.0, 'rot_all': 0.0}

        frame_nums = []
        npcs_pred = []
        with torch.no_grad():
            for i, input in enumerate(self.feed_dict):
                frame_nums.append([
                    path.split('.')[-2].split('/')[-1]
                    for path in input['meta']['path']
                ])
                if i == 0:
                    npcs_pred.append(None)
                    continue
                perturbed_part = add_noise_to_part_dof(
                    self.feed_dict[i - 1]['gt_part'], self.pose_perturb_cfg)
                if 'crop_pose' in self.feed_dict[i]['meta']:
                    crop_pose = part_model_batch_to_part(
                        cvt_torch(self.feed_dict[i]['meta']['crop_pose'],
                                  self.device), self.num_parts, self.device)
                    for key in ['translation', 'scale']:
                        perturbed_part[key] = crop_pose[key]

                last_pose = {
                    key: value.clone()
                    for key, value in pred_poses[-1].items()
                }

                self.timer.tick()
                if self.nocs_otf:
                    center = last_pose['translation'].reshape(
                        3).detach().cpu().numpy()  # [3]
                    scale = last_pose['scale'].reshape(1).detach().cpu().item()
                    depth_path = input['meta']['ori_path'][0]
                    category, instance = input['meta']['path'][0].split(
                        '/')[-4:-2]
                    pre_fetched = input['meta']['pre_fetched']
                    pre_fetched = {
                        key: value.reshape(value.shape[1:])
                        for key, value in pre_fetched.items()
                    }

                    pose = {
                        key:
                        value.squeeze(0).squeeze(0).detach().cpu().numpy()
                        for key, value in input['gt_part'].items()
                    }
                    full_data = full_data_from_depth_image(
                        depth_path,
                        category,
                        instance,
                        center,
                        self.radius * scale,
                        pose,
                        num_points=input['points'].shape[-1],
                        device=self.device,
                        mask_from_nocs2d=self.track_cfg['nocs2d_label'],
                        nocs2d_path=self.track_cfg['nocs2d_path'],
                        pre_fetched=pre_fetched)

                    points, nocs, labels = full_data['points'], full_data[
                        'nocs'], full_data['labels']

                    points = cvt_torch(points, self.device)
                    points -= self.npcs_feed_dict[i]['points_mean'].reshape(
                        1, 3)
                    input['points'] = points.transpose(-1,
                                                       -2).reshape(1, 3, -1)
                    input['labels'] = torch.tensor(full_data['labels']).to(
                        self.device).long().reshape(1, -1)
                    nocs = cvt_torch(nocs, self.device)
                    self.npcs_feed_dict[i]['points'] = input['points']
                    self.npcs_feed_dict[i]['labels'] = input['labels']
                    self.npcs_feed_dict[i]['nocs'] = nocs.transpose(
                        -1, -2).reshape(1, 3, -1)

                    time_dict['crop'] += self.timer.tick()

                state = {'part': last_pose}
                input['state'] = state

                npcs_canon_pose = {
                    key: last_pose[key][:, self.root].clone()
                    for key in ['rotation', 'translation', 'scale']
                }
                npcs_input = self.npcs_feed_dict[i]
                npcs_input['canon_pose'] = npcs_canon_pose
                npcs_input['init_part'] = last_pose
                cur_npcs_pred = self.npcs_net(
                    npcs_input)  # seg: [B, P, N], npcs: [B, P * 3, N]
                npcs_pred.append(cur_npcs_pred)
                pred_npcs, pred_seg = cur_npcs_pred['nocs'], cur_npcs_pred[
                    'seg']
                pred_npcs = pred_npcs.reshape(len(pred_npcs), self.num_parts,
                                              3, -1)  # [B, P, 3, N]
                pred_labels = torch.max(pred_seg,
                                        dim=-2)[1]  # [B, P, N] -> [B, N]

                time_dict['npcs_net'] += self.timer.tick()

                input['pred_labels'], input[
                    'pred_nocs'] = pred_labels, pred_npcs
                input['pred_label_conf'] = pred_seg[:, 0]  # [B, P, N]
                if self.track_cfg['gt_label'] or self.track_cfg['nocs2d_label']:
                    input['pred_labels'] = npcs_input['labels']

                pred_dict = self.net(input, test_mode=True)
                pred_poses.append(pred_dict['part'])

                time_dict['rot_all'] += self.timer.tick()

        self.pred_dict = {'poses': pred_poses, 'npcs_pred': npcs_pred}

        if save:
            gt_corners = self.feed_dict[0]['meta']['nocs_corners'].cpu().numpy(
            )
            corner_list = []
            for i, pred_pose in enumerate(self.pred_dict['poses']):
                if i == 0:
                    corner_list.append(None)
                    continue
                pred_labels = torch.max(self.pred_dict['npcs_pred'][i]['seg'],
                                        dim=-2)[1]  # [B, P, N] -> [B, N]
                pred_nocs = choose_coord_by_label(
                    self.pred_dict['npcs_pred'][i]['nocs'].transpose(-1, -2),
                    pred_labels)
                pred_corners = get_pred_nocs_corners(pred_labels, pred_nocs,
                                                     self.num_parts)
                corner_list.append(pred_corners)

            gt_poses = [{
                key: value.detach().cpu().numpy()
                for key, value in frame[f'gt_part'].items()
            } for frame in self.feed_dict]
            save_dict = {
                'pred': {
                    'poses': [{
                        key: value.detach().cpu().numpy()
                        for key, value in pred_pose.items()
                    } for pred_pose in pred_poses],
                    'corners':
                    corner_list
                },
                'gt': {
                    'poses': gt_poses,
                    'corners': gt_corners
                },
                'frame_nums': frame_nums
            }

            save_path = pjoin(self.cfg['experiment_dir'], 'results', 'data')
            ensure_dirs([save_path])
            for i, path in enumerate(self.feed_dict[0]['meta']['path']):
                instance, track_num = path.split('.')[-2].split('/')[-3:-1]
                with open(pjoin(save_path, f'{instance}_{track_num}.pkl'),
                          'wb') as f:
                    cur_dict = get_ith_from_batch(save_dict,
                                                  i,
                                                  to_single=False)
                    pickle.dump(cur_dict, f)
def setup():
    ensure_dirs(["./speakers", "./speakers/LIBRISPEECH"])