Beispiel #1
0
def par_generate_gt(config, pair_rec, flow_depth_rendered=None):
    from lib.pair_matching.flow import calc_flow
    target_size, max_size = config.SCALES[0][0], config.SCALES[0][1]

    if flow_depth_rendered is None:
        flow_depth_rendered = cv2.imread(pair_rec['depth_rendered'],
                                     cv2.IMREAD_UNCHANGED).astype(np.float32)
        flow_depth_rendered /= config.dataset.DEPTH_FACTOR

        flow_depth_rendered, _ = resize(flow_depth_rendered, target_size, max_size)

    if 'depth_render_real' in pair_rec:
        flow_depth_real = cv2.imread(pair_rec['depth_render_real'],
                                 cv2.IMREAD_UNCHANGED).astype(np.float32)
        flow_depth_real /= config.dataset.DEPTH_FACTOR
    else:
        print('not using render_real depth in par_generate_gt')
        flow_depth_real = cv2.imread(pair_rec['depth_real'],
                                 cv2.IMREAD_UNCHANGED).astype(np.float32)
        flow_depth_real /= config.dataset.DEPTH_FACTOR

    flow_depth_real, _ = resize(flow_depth_real, target_size, max_size)

    if 'mask_real_gt' or 'mask_real_est' in pair_rec:
        mask_real_path = pair_rec['mask_real_gt']
        assert os.path.exists(mask_real_path), '%s does not exist'.format(pair_rec['mask_real_gt'])
        mask_real = cv2.imread(mask_real_path, cv2.IMREAD_UNCHANGED)
        mask_real, _ = resize(mask_real, target_size, max_size)
        flow_depth_real[mask_real != pair_rec['mask_idx']] = 0

    if config.network.FLOW_I2R:
        if config.FLOW_CLASS_AGNOSTIC:
            flow_i2r, visible, _ = calc_flow(flow_depth_rendered, pair_rec['pose_est'], pair_rec['pose_real'],
                                             config.dataset.INTRINSIC_MATRIX, flow_depth_real,
                                             standard_rep=config.network.STANDARD_FLOW_REP)
            flow_i2r_list = [flow_i2r, visible, np.logical_and(visible == 0, flow_depth_rendered == 0)]
        else:
            raise Exception('NOT_IMPLEMENTED')
    else:
        flow_i2r_list = None

    if config.network.FLOW_R2I:
        if config.FLOW_CLASS_AGNOSTIC:
            flow_r2i, visible, _ = calc_flow(flow_depth_real, pair_rec['pose_real'], pair_rec['pose_est'],
                                             config.dataset.INTRINSIC_MATRIX, flow_depth_rendered,
                                             standard_rep=config.network.STANDARD_FLOW_REP)
            flow_r2i_list = [flow_r2i, visible, np.logical_and(visible == 0, flow_depth_real == 0)]
        else:
            raise Exception('NOT_IMPLEMENTED')
    else:
        flow_r2i_list = None
    return {'flow_i2r': flow_i2r_list, 'flow_r2i': flow_r2i_list}
Beispiel #2
0
def par_generate_gt(config, pair_rec, flow_depth_rendered=None):
    from lib.pair_matching.flow import calc_flow

    target_size, max_size = config.SCALES[0][0], config.SCALES[0][1]

    if flow_depth_rendered is None:
        flow_depth_rendered = cv2.imread(pair_rec["depth_rendered"],
                                         cv2.IMREAD_UNCHANGED).astype(
                                             np.float32)
        flow_depth_rendered /= config.dataset.DEPTH_FACTOR

        flow_depth_rendered, _ = resize(flow_depth_rendered, target_size,
                                        max_size)

    if "depth_gt_observed" in pair_rec:
        flow_depth_observed = cv2.imread(pair_rec["depth_gt_observed"],
                                         cv2.IMREAD_UNCHANGED).astype(
                                             np.float32)
        flow_depth_observed /= config.dataset.DEPTH_FACTOR
    else:
        logger.info("not using gt_observed depth in par_generate_gt")
        flow_depth_observed = cv2.imread(pair_rec["depth_observed"],
                                         cv2.IMREAD_UNCHANGED).astype(
                                             np.float32)
        flow_depth_observed /= config.dataset.DEPTH_FACTOR

    flow_depth_observed, _ = resize(flow_depth_observed, target_size, max_size)

    if "mask_gt_observed" or "mask_observed" in pair_rec:
        mask_observed_path = pair_rec["mask_gt_observed"]
        assert os.path.exists(mask_observed_path), "%s does not exist".format(
            pair_rec["mask_gt_observed"])
        mask_observed = cv2.imread(mask_observed_path, cv2.IMREAD_UNCHANGED)
        mask_observed, _ = resize(mask_observed, target_size, max_size)
        flow_depth_observed[mask_observed != pair_rec["mask_idx"]] = 0

    if config.network.PRED_FLOW:
        flow_i2r, visible, _ = calc_flow(
            flow_depth_rendered,
            pair_rec["pose_rendered"],
            pair_rec["pose_observed"],
            config.dataset.INTRINSIC_MATRIX,
            flow_depth_observed,
            standard_rep=config.network.STANDARD_FLOW_REP,
        )
        flow_i2r_list = [
            flow_i2r, visible,
            np.logical_and(visible == 0, flow_depth_rendered == 0)
        ]

    return {"flow": flow_i2r_list}
Beispiel #3
0
def generate_batch(im):
    """
    preprocess image, return batch
    :param im: cv2.imread returns [height, width, channel] in BGR
    :return:
    data_batch: MXNet input batch
    data_names: names in data_batch
    im_scale: float number
    """
    SHORT_SIDE = config.SCALES[0][0]
    LONG_SIDE = config.SCALES[0][1]
    PIXEL_MEANS = config.network.PIXEL_MEANS
    DATA_NAMES = ['data', 'im_info']

    im_array, im_scale = resize(im, SHORT_SIDE, LONG_SIDE)
    im_array = transform(im_array, PIXEL_MEANS)
    im_info = np.array([[im_array.shape[2], im_array.shape[3], im_scale]],
                       dtype=np.float32)
    data = [[mx.nd.array(im_array), mx.nd.array(im_info)]]
    data_shapes = [[('data', im_array.shape), ('im_info', im_info.shape)]]
    data_batch = mx.io.DataBatch(data=data,
                                 label=[None],
                                 provide_data=data_shapes,
                                 provide_label=[None])
    return data_batch, DATA_NAMES, [im_scale]
Beispiel #4
0
def generate_batch(im):
    """
    preprocess image, return batch
    :param im: cv2.imread returns [height, width, channel] in BGR
    :return:
    data_batch: MXNet input batch
    im_scale: float number
    """
    SHORT_SIDE = config.SCALES[0][0]
    LONG_SIDE = config.SCALES[0][1]
    PIXEL_MEANS = config.network.PIXEL_MEANS

    im_array, im_scale = resize(im, SHORT_SIDE, LONG_SIDE)
    im_array = transform(im_array, PIXEL_MEANS)
    im_array = im_array.astype(np.float32)
    im_info = np.array([[im_array.shape[2], im_array.shape[3], im_scale]],
                       dtype=np.float32)
    data_shapes = [[('data', im_array.shape), ('im_info', im_info.shape)]]
    return {
        'data_shapes': data_shapes,
        'im_scale': [im_scale],
        'im_array': im_array,
        'im_info': im_info
    }
Beispiel #5
0
def update_data_batch(config, data_batch, update_package):
    import mxnet.ndarray as nd

    for ctx_idx, data in enumerate(data_batch.data):
        package = update_package[ctx_idx]
        for blob_idx, blob in enumerate(data):
            blob_name = data_batch.provide_data[ctx_idx][blob_idx][0]
            blob_shape = data_batch.provide_data[ctx_idx][blob_idx][1]
            if blob_name not in package:
                continue
            update_data = package[blob_name]
            if blob_name.startswith("image"):
                target_size = min(blob_shape[2:])
                max_size = max(blob_shape[2:])
                image_update, _ = image.resize(update_data, target_size,
                                               max_size)
                image_update = image.transform(image_update,
                                               config.network.PIXEL_MEANS)
                image_update = nd.array(image_update)
                data_batch.data[ctx_idx][blob_idx] = image_update
            elif blob_name.startswith("src_pose"):
                src_pose_update = nd.array(update_data[np.newaxis, :, :])
                data_batch.data[ctx_idx][blob_idx] = src_pose_update
            elif blob_name.startswith("depth"):
                target_size = min(blob_shape[2:])
                max_size = max(blob_shape[2:])
                depth_update, _ = image.resize(update_data, target_size,
                                               max_size)
                depth_update = nd.array(depth_update[np.newaxis,
                                                     np.newaxis, :, :])
                data_batch.data[ctx_idx][blob_idx] = depth_update
            elif blob_name.startswith("mask_observed"):
                if config.TEST.UPDATE_MASK == "box_rendered":
                    update_data = np.copy(package["mask_rendered"])
                    mask_observed = np.zeros(update_data.shape)
                    x_max = np.max(update_data, 0)
                    y_max = np.max(update_data, 1)
                    nz_x = np.nonzero(x_max)[0]
                    nz_y = np.nonzero(y_max)[0]
                    x_start = np.min(nz_x)
                    x_end = np.max(nz_x)
                    y_start = np.min(nz_y)
                    y_end = np.max(nz_y)
                    mask_observed[y_start:y_end,
                                  x_start:x_end] = 1.0  # rectangle
                elif config.TEST.UPDATE_MASK == "box_observed":
                    mask_observed = np.zeros(update_data.shape)
                    x_max = np.max(update_data, 0)
                    y_max = np.max(update_data, 1)
                    nz_x = np.nonzero(x_max)[0]
                    nz_y = np.nonzero(y_max)[0]
                    if len(nz_x) == 0 or len(nz_y) == 0:
                        raise Exception("no point valid in mask_observed")

                    x_start = np.min(nz_x)
                    x_end = np.max(nz_x)
                    y_start = np.min(nz_y)
                    y_end = np.max(nz_y)
                    mask_observed[y_start:y_end,
                                  x_start:x_end] = 1.0  # rectangle
                else:
                    mask_observed = update_data
                mask_update = nd.array(mask_observed[np.newaxis,
                                                     np.newaxis, :, :])
                data_batch.data[ctx_idx][blob_idx] = mask_update
            elif blob_name.startswith("mask_rendered"):
                mask_update = nd.array(update_data[np.newaxis,
                                                   np.newaxis, :, :])
                data_batch.data[ctx_idx][blob_idx] = mask_update
            else:
                raise Exception("NOT_IMPLEMENTED")
    return data_batch
def main():
    # get symbol
    ctx_id = [int(i) for i in config.gpus.split(',')]
    # ctx_id = [mx.cpu()]
    print(ctx_id)
    pprint.pprint(config)
    sym_instance = eval(config.symbol)()
    sym = sym_instance.get_symbol(config, is_train=False)

    # set up class names
    num_classes = 2
    classes = ['nuclei']

    # load demo data
    # image_root = '/home/daiab/machine_disk/work/kaggle_nuclei/data/stage1_test_images'
    image_root = '/home/daiab/machine_disk/work/kaggle_nuclei/data/LikeVOC/img'
    image_names = glob.glob(os.path.join(image_root, '*.png'))
    data = []
    for im_name in image_names:
        im = cv2.imread(im_name,
                        cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
        target_size = config.SCALES[0][0]
        max_size = config.SCALES[0][1]
        im, im_scale = resize(im,
                              target_size,
                              max_size,
                              stride=config.network.IMAGE_STRIDE)
        im_tensor = transform(im, config.network.PIXEL_MEANS)
        im_info = np.array(
            [[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
            dtype=np.float32)
        data.append({'data': im_tensor, 'im_info': im_info})

    # get predictor
    data_names = ['data', 'im_info']
    label_names = []
    data = [[mx.nd.array(data[i][name]) for name in data_names]
            for i in range(len(data))]
    max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
                                 max([v[1] for v in config.SCALES])))]]
    provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
                    for i in range(len(data))]
    provide_label = [None for i in range(len(data))]
    arg_params, aux_params = load_param(
        '/home/daiab/machine_disk/projects/FCIS/'
        'output/fcis/nuclei/nuclei_end2end/SDS_train/e2e',
        2,
        process=True)
    data = data
    max_data_shape = max_data_shape
    provide_data = provide_data
    predictor = Predictor(sym,
                          data_names,
                          label_names,
                          context=[mx.gpu(ctx_id[0])],
                          max_data_shapes=max_data_shape,
                          provide_data=provide_data,
                          provide_label=provide_label,
                          arg_params=arg_params,
                          aux_params=aux_params)

    # warm up
    for i in range(2):
        data_batch = mx.io.DataBatch(data=[data[0]],
                                     label=[],
                                     pad=0,
                                     index=0,
                                     provide_data=[[
                                         (k, v.shape)
                                         for k, v in zip(data_names, data[0])
                                     ]],
                                     provide_label=[None])
        scales = [
            data_batch.data[i][1].asnumpy()[0, 2]
            for i in range(len(data_batch.data))
        ]
        # print('-----------', data_batch)
        _, _, _, _ = im_detect(predictor, data_batch, data_names, scales,
                               config)

    # test
    for idx, im_name in enumerate(image_names):
        data_batch = mx.io.DataBatch(
            data=[data[idx]],
            label=[],
            pad=0,
            index=idx,
            provide_data=[[(k, v.shape)
                           for k, v in zip(data_names, data[idx])]],
            provide_label=[None])
        scales = [
            data_batch.data[i][1].asnumpy()[0, 2]
            for i in range(len(data_batch.data))
        ]

        tic()
        scores, boxes, masks, data_dict = im_detect(predictor, data_batch,
                                                    data_names, scales, config)
        im_shapes = [
            data_batch.data[i][0].shape[2:4]
            for i in range(len(data_batch.data))
        ]

        if not config.TEST.USE_MASK_MERGE:
            all_boxes = [[] for _ in range(num_classes)]
            all_masks = [[] for _ in range(num_classes)]
            nms = py_nms_wrapper(config.TEST.NMS)
            for j in range(1, num_classes):
                indexes = np.where(scores[0][:, j] > 0.7)[0]
                cls_scores = scores[0][indexes, j, np.newaxis]
                cls_masks = masks[0][indexes, 1, :, :]
                try:
                    if config.CLASS_AGNOSTIC:
                        cls_boxes = boxes[0][indexes, :]
                    else:
                        raise Exception()
                except:
                    cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4]

                cls_dets = np.hstack((cls_boxes, cls_scores))
                keep = nms(cls_dets)
                all_boxes[j] = cls_dets[keep, :]
                all_masks[j] = cls_masks[keep, :]
            dets = [all_boxes[j] for j in range(1, num_classes)]
            masks = [all_masks[j] for j in range(1, num_classes)]
        else:
            masks = masks[0][:, 1:, :, :]
            im_height = np.round(im_shapes[0][0] / scales[0]).astype('int')
            im_width = np.round(im_shapes[0][1] / scales[0]).astype('int')
            print(im_height, im_width)
            boxes = clip_boxes(boxes[0], (im_height, im_width))
            result_masks, result_dets = gpu_mask_voting(
                masks, boxes, scores[0], num_classes, 100, im_width, im_height,
                config.TEST.NMS, config.TEST.MASK_MERGE_THRESH,
                config.BINARY_THRESH, ctx_id[0])
            dets = [result_dets[j] for j in range(1, num_classes)]
            masks = [
                result_masks[j][:, 0, :, :] for j in range(1, num_classes)
            ]
            # print(dets)
            # print(masks)
        print('testing {} {:.4f}s'.format(im_name, toc()))
        # visualize
        for i in range(len(dets)):
            keep = np.where(dets[i][:, -1] > 0.7)
            dets[i] = dets[i][keep]
            masks[i] = masks[i][keep]
        im = cv2.imread(im_name)
        im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
        show_masks(im, dets, masks, classes, config)

    print('done')
Beispiel #7
0
def main():
    # get symbol
    ctx_id = [int(i) for i in config.gpus.split(',')]
    # ctx_id = [mx.cpu()]
    print(ctx_id)
    pprint.pprint(config)
    sym_instance = eval(config.symbol)()
    sym = sym_instance.get_symbol(config, is_train=False)

    # set up class names
    num_classes = 81
    classes = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat',
               'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse',
               'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie',
               'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove',
               'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon',
               'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut',
               'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse',
               'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book',
               'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush']

    # load demo data
    image_names = ['COCO_test2015_000000000275.jpg', 'COCO_test2015_000000001412.jpg', 'COCO_test2015_000000073428.jpg',
                    'COCO_test2015_000000393281.jpg']
    data = []
    for im_name in image_names:
        assert os.path.exists(cur_path + '/../demo/' + im_name), ('%s does not exist'.format('../demo/' + im_name))
        im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
        target_size = config.SCALES[0][0]
        max_size = config.SCALES[0][1]
        im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE)
        im_tensor = transform(im, config.network.PIXEL_MEANS)
        im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32)
        data.append({'data': im_tensor, 'im_info': im_info})

    # get predictor
    data_names = ['data', 'im_info']
    label_names = []
    data = [[mx.nd.array(data[i][name]) for name in data_names] for i in range(len(data))]
    max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]]
    provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in range(len(data))]
    provide_label = [None for i in range(len(data))]
    arg_params, aux_params = load_param(cur_path + '/../model/fcis_coco', 0, process=True)
    data = data
    max_data_shape = max_data_shape
    provide_data = provide_data
    predictor = Predictor(sym, data_names, label_names,
                          context=[mx.gpu(ctx_id[0])],
                          max_data_shapes=max_data_shape,
                          provide_data=provide_data,
                          provide_label=provide_label,
                          arg_params=arg_params,
                          aux_params=aux_params)

    # warm up
    for i in range(2):
        data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0,
                                     provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]],
                                     provide_label=[None])
        scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data))]
        # print('-----------', data_batch)
        _, _, _, _ = im_detect(predictor, data_batch, data_names, scales, config)

    # test
    for idx, im_name in enumerate(image_names):
        data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx,
                                     provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]],
                                     provide_label=[None])
        scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data))]

        tic()
        scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config)
        im_shapes = [data_batch.data[i][0].shape[2:4] for i in range(len(data_batch.data))]

        if not config.TEST.USE_MASK_MERGE:
            all_boxes = [[] for _ in range(num_classes)]
            all_masks = [[] for _ in range(num_classes)]
            nms = py_nms_wrapper(config.TEST.NMS)
            for j in range(1, num_classes):
                indexes = np.where(scores[0][:, j] > 0.7)[0]
                cls_scores = scores[0][indexes, j, np.newaxis]
                cls_masks = masks[0][indexes, 1, :, :]
                try:
                    if config.CLASS_AGNOSTIC:
                        cls_boxes = boxes[0][indexes, :]
                    else:
                        raise Exception()
                except:
                    cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4]

                cls_dets = np.hstack((cls_boxes, cls_scores))
                keep = nms(cls_dets)
                all_boxes[j] = cls_dets[keep, :]
                all_masks[j] = cls_masks[keep, :]
            dets = [all_boxes[j] for j in range(1, num_classes)]
            masks = [all_masks[j] for j in range(1, num_classes)]
        else:
            print('=============')
            masks = masks[0][:, 1:, :, :]
            im_height = np.round(im_shapes[0][0] / scales[0]).astype('int')
            im_width = np.round(im_shapes[0][1] / scales[0]).astype('int')
            print(im_height, im_width)
            boxes = clip_boxes(boxes[0], (im_height, im_width))
            result_masks, result_dets = gpu_mask_voting(masks, boxes, scores[0], num_classes,
                                                        100, im_width, im_height,
                                                        config.TEST.NMS, config.TEST.MASK_MERGE_THRESH,
                                                        config.BINARY_THRESH, ctx_id[0])
            dets = [result_dets[j] for j in range(1, num_classes)]
            masks = [result_masks[j][:, 0, :, :] for j in range(1, num_classes)]
            # print(dets)
            # print(masks)
        print('testing {} {:.4f}s'.format(im_name, toc()))
        # visualize
        for i in range(len(dets)):
            keep = np.where(dets[i][:,-1]>0.7)
            dets[i] = dets[i][keep]
            masks[i] = masks[i][keep]
        im = cv2.imread(cur_path + '/../demo/' + im_name)
        im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
        show_masks(im, dets, masks, classes, config)

    print('done')
Beispiel #8
0
def par_generate_gt(config, pair_rec, flow_depth_rendered=None):
    from lib.pair_matching.flow import calc_flow

    target_size, max_size = config.SCALES[0][0], config.SCALES[0][1]

    if flow_depth_rendered is None:
        flow_depth_rendered = cv2.imread(
            pair_rec["depth_rendered"], cv2.IMREAD_UNCHANGED
        ).astype(np.float32)
        flow_depth_rendered /= config.dataset.DEPTH_FACTOR

        flow_depth_rendered, _ = resize(flow_depth_rendered, target_size, max_size)

    if "depth_render_real" in pair_rec:
        flow_depth_real = cv2.imread(
            pair_rec["depth_render_real"], cv2.IMREAD_UNCHANGED
        ).astype(np.float32)
        flow_depth_real /= config.dataset.DEPTH_FACTOR
    else:
        print("not using render_real depth in par_generate_gt")
        flow_depth_real = cv2.imread(
            pair_rec["depth_real"], cv2.IMREAD_UNCHANGED
        ).astype(np.float32)
        flow_depth_real /= config.dataset.DEPTH_FACTOR

    flow_depth_real, _ = resize(flow_depth_real, target_size, max_size)

    if "mask_gt_observed" or "mask_observed" in pair_rec:
        mask_observed_path = pair_rec["mask_gt_observed"]
        assert os.path.exists(mask_observed_path), "%s does not exist".format(
            pair_rec["mask_gt_observed"]
        )
        mask_observed = cv2.imread(mask_observed_path, cv2.IMREAD_UNCHANGED)
        mask_observed, _ = resize(mask_observed, target_size, max_size)
        flow_depth_real[mask_observed != pair_rec["mask_idx"]] = 0

    if config.network.FLOW_I2R:
        if config.FLOW_CLASS_AGNOSTIC:
            flow_i2r, visible, _ = calc_flow(
                flow_depth_rendered,
                pair_rec["pose_rendered"],
                pair_rec["pose_observed"],
                config.dataset.INTRINSIC_MATRIX,
                flow_depth_real,
                standard_rep=config.network.STANDARD_FLOW_REP,
            )
            flow_i2r_list = [
                flow_i2r,
                visible,
                np.logical_and(visible == 0, flow_depth_rendered == 0),
            ]
        else:
            raise Exception("NOT_IMPLEMENTED")
    else:
        flow_i2r_list = None

    if config.network.FLOW_R2I:
        if config.FLOW_CLASS_AGNOSTIC:
            flow_r2i, visible, _ = calc_flow(
                flow_depth_real,
                pair_rec["pose_observed"],
                pair_rec["pose_rendered"],
                config.dataset.INTRINSIC_MATRIX,
                flow_depth_rendered,
                standard_rep=config.network.STANDARD_FLOW_REP,
            )
            flow_r2i_list = [
                flow_r2i,
                visible,
                np.logical_and(visible == 0, flow_depth_real == 0),
            ]
        else:
            raise Exception("NOT_IMPLEMENTED")
    else:
        flow_r2i_list = None
    return {"flow_i2r": flow_i2r_list, "flow_r2i": flow_r2i_list}