예제 #1
0
def read_charness_histograms(path):
    """
    ['pigraph_norm_factor', 'pigraph_histogram_params',
     'pigraph_histogram_charness', 'pigraph_pose_charness',
     'pigraph_scenelet_names', 'categories']
    """
    hash_mat_file_current = hash_path_md5(path)
    hists = None

    path_pickle = "%s.pickle" % path
    if os.path.exists(path_pickle):
        with open(path_pickle, 'rb') as f:
            # hists, hash_mat_file = pickle_load(f)
            tmp = pickle_load(f)
            if tmp[-1] != hash_mat_file_current:
                hists = None
                lg.warning("Hashes don't match, reloading hists")
            else:
                if len(tmp) == 3:
                    hists = tmp[1]  # pose_charness, hists, hash
                else:
                    hists = tmp[0]  # hists, hash
                lg.info("Loaded hists from\n\t%s!!!" % split_path(path_pickle))

    if hists is None:
        dmat = scipy.io.loadmat(path)
        hists = parse_charness_histograms(dmat)
        with open(path_pickle, 'wb') as f:
            pickle.dump((hists, hash_mat_file_current), f, -1)
            lg.info("Saved hists to %s" % path_pickle)
    # print(hists.keys())
    # key = list(hists.keys())[0]
    # logging.info("key: %s, %s" % (key, hists[key].volume))
    return hists
예제 #2
0
def main(argv):
    np.set_printoptions(suppress=True, linewidth=200)
    pjoin = os.path.join

    parser = argparse.ArgumentParser("matcher")
    parser.add_argument("d_scenelets",
                        type=argparse_check_exists,
                        help="Folder containing scenelets")
    parser.add_argument("video", type=argparse_check_exists, help="Input path")
    parser.add_argument("--gap-size-limit",
                        type=int,
                        help="Smallest gap size to still explain")
    args = parser.parse_args(argv)
    d_query = os.path.dirname(args.video)

    # 2d keypoint rescale
    p_im = pjoin(d_query, 'origjpg', 'color_00100.jpg')
    im_ = cv2.imread(p_im)
    shape_orig = im_.shape
    scale_2d = shape_orig[0] / float(INPUT_SIZE)

    query = Scenelet.load(args.video, no_obj=True)
    tr_ground = np.array(query.aux_info['ground'], dtype=np.float32)
    print("tr: %s" % tr_ground)

    name_query = os.path.basename(args.video).split('_')[1]
    query_2d = Scenelet.load(pjoin(d_query,
                                   "skel_%s_2d_00.json" % name_query)).skeleton
    p_intr = pjoin(d_query, 'intrinsics.json')
    intr = np.array(json.load(open(p_intr, 'r')), dtype=np.float32)
    lg.debug("intr: %s" % intr)

    gaps = find_gaps(query.skeleton, min_pad=1)

    p_scenelets_pickle = pjoin(args.d_scenelets, 'match_gap_scenelets.pickle')
    if os.path.exists(p_scenelets_pickle):
        scenelets = pickle_load(open(p_scenelets_pickle, 'rb'))
    else:
        scenelets = read_scenelets(args.d_scenelets, limit=0)
        pickle.dump(scenelets, open(p_scenelets_pickle, 'wb'))

    p_out_sclts = pjoin(d_query, 'fill')
    if os.path.exists(p_out_sclts):
        shutil.rmtree(p_out_sclts)
    os.makedirs(p_out_sclts)
    times = []
    for gap_id, gap in enumerate(gaps):
        lg.debug("gap is %s" % repr(gap))
        if gap[1] - gap[0] < args.gap_size_limit:
            continue
        with Timer("gap %d" % gap_id) as timer:
            chosen = []
            for sc_id, sclt in enumerate(scenelets):
                lg.info("scenelet %d / %d" % (sc_id, len(scenelets)))
                sclt = scenelets[sc_id]
                ground_obj = next(ob for ob in sclt.objects.values()
                                  if ob.label == 'floor')
                ground_part = ground_obj.get_part_by_name('floor')
                lg.debug("floor: %s" % ground_part)
                ground_transform = ground_part.obb.as_transform()
                lg.debug("floor: %s" % ground_transform)
                # sys.exit(0)
                # chosen.extend(
                out_sclts = match(query,
                                  d_query,
                                  query_2d,
                                  sclt,
                                  intr,
                                  gap,
                                  tr_ground,
                                  scale=scale_2d)
                if not len(out_sclts):
                    continue
                # pick best from scene
                chosen.append(
                    [out_sclts[0][i]
                     for i in range(len(out_sclts[0]))] + [sc_id])
                # break

            # os.system("rm %s/skel_%s_fill_%03d_%03d__*.json"
            #           % (p_out_sclts, name_query, gap[0], gap[1]))
            chosen = sorted(chosen, key=lambda score_sclt: score_sclt[0])
            for sid, (score, out_sclt, sc_id) in enumerate(chosen):
                p_out = pjoin(
                    p_out_sclts, "skel_%s_fill_%03d_%03d__%02d.json" %
                    (name_query, gap[0], gap[1], sid))
                out_sclt.save(p_out)
                if sid > 5:
                    break
        times.append(timer.get_elapsed_ms())
    lg.info("mean time per gap: %s" % np.mean(times))
예제 #3
0
def main(argv):
    conf = Conf.get()
    parser = argparse.ArgumentParser("Denis pose converter")
    parser.add_argument('camera_name',
                        help="Camera name ('G15', 'S6')",
                        type=str)
    parser.add_argument(
        '-d',
        dest='dir',
        required=True,
        help="Path to the <scene folder>/denis containing skeletons.json")
    parser.add_argument(
        '-filter',
        dest='with_filtering',
        action="store_true",
        help="Should we do post-filtering (1-euro) on the pelvis positions")
    parser.add_argument('-huber',
                        required=False,
                        help="Should we do huber loss?",
                        action='store_true')
    parser.add_argument('-smooth',
                        type=float,
                        default=0.005,
                        help="Should we have a smoothness term (l2/huber)?")
    parser.add_argument(
        '--winsorize-limit',
        type=float,
        default=conf.optimize_path.winsorize_limit,
        help='Threshold for filtering too large jumps of the 2D centroid')
    parser.add_argument('--no-resample',
                        action='store_true',
                        help="add resampled frames")
    parser.add_argument('--n-actors',
                        type=int,
                        default=1,
                        help="How many skeletons to track.")
    parser.add_argument('-n-actors',
                        type=int,
                        default=1,
                        help="Max number of people in scene.")
    # parser.add_argument(
    #     '-r', type=float,
    #     help='Video rate. Default: 1, if avconv -r 5. '
    #          'Original video sampling rate (no subsampling) should be '
    #          '24/5=4.8. avconv -r 10 leads to 24/10=2.4.',
    #     required=True)
    parser.add_argument('--person_height',
                        type=float,
                        help='Assumed height of human(s) in video.',
                        default=Conf.get().optimize_path.person_height)
    parser.add_argument(
        '--forwards-window-size',
        type=int,
        help='How many poses in time to look before AND after to '
        'average forward direction. 0 means no averaging. Default: 0.',
        default=0)
    parser.add_argument('--no-img',
                        action='store_true',
                        help='Read and write images (vis reproj error)')
    parser.add_argument('--postfix',
                        type=str,
                        help="output file postfix.",
                        default='unannot')
    args = parser.parse_args(argv)
    show = False
    args.resample = not args.no_resample
    # assert not args.resample, "resample should be off"
    assert os.path.exists(args.dir), "Source does not exist: %s" % args.dir
    p_scene = os.path.normpath(os.path.join(args.dir, os.pardir))  # type: str
    p_video_params = os.path.join(p_scene, 'video_params.json')
    assert os.path.exists(p_video_params), "Need video_params.json for rate"
    if 'r' not in args or args.r is None:
        args.r = json.load(open(p_video_params, 'r'))['rate-avconv']

    # manual parameters (depth initialization, number of actors)
    p_scene_params = os.path.join(args.dir, os.pardir, 'scene_params.json')
    if not os.path.exists(p_scene_params):
        scene_params = {
            'depth_init': 10.,
            'actors': args.n_actors,
            'ground_rot': [0., 0., 0.]
        }
        json.dump(scene_params, open(p_scene_params, 'w'))
        raise RuntimeError("Inited scene_params.json, please check: %s" %
                           p_scene_params)
    else:
        scene_params = json.load(open(p_scene_params, 'r'))
        lg.warning("Will work with %d actors and init depth to %g" %
                   (scene_params['actors'], scene_params['depth_init']))
        assert '--n-actors' not in argv \
               or args.n_actors == scene_params['actors'], \
            "Actor count mismatch, remove %d from args, because " \
            "scene_params.json says %d?" \
            % (args.n_actors, scene_params['actors'])
        args.n_actors = scene_params['actors']
        ground_rot = scene_params['ground_rot'] or [0., 0., 0.]

    # load images
    path_images = os.path.abspath(os.path.join(args.dir, os.pardir, 'origjpg'))
    images = {}
    shape_orig = None
    if not args.no_img:
        images, shape_orig = load_images(path_images)

    path_skeleton = \
        max((f for f in os.listdir(os.path.join(args.dir))
             if f.startswith('skeletons') and f.endswith('json')),
            key=lambda s: int(os.path.splitext(s)[0].split('_')[1]))
    print("path_skeleton: %s" % path_skeleton)
    data = json.load(open(os.path.join(args.dir, path_skeleton), 'r'))
    # data, pose_constraints, first_run = \
    #     cleanup(data, p_dir=os.path.join(args.dir, os.pardir))
    # poses_2d = []
    # plt.figure()
    # show_images(images, data)
    if False:
        # pose_ids = identify_actors_multi(data, n_actors=1)
        p_segm_pickle = os.path.join(args.dir, os.pardir,
                                     "label_skeletons.pickle")
        problem = None
        if False and os.path.exists(p_segm_pickle):
            lg.warning("Loading skeleton segmentation from pickle %s" %
                       p_segm_pickle)
            pose_ids, problem = pickle_load(open(p_segm_pickle, 'rb'))
        if not problem or problem._n_actors != args.n_actors:
            pose_ids, problem, data = more_actors_gurobi(
                data,
                n_actors=args.n_actors,
                constraints=pose_constraints,
                first_run=first_run)
            if True or show:
                show_multi(images,
                           data,
                           pose_ids,
                           problem,
                           p_dir=os.path.join(args.dir, os.pardir),
                           first_run=first_run,
                           n_actors=args.n_actors)
            pickle.dump((pose_ids, problem), open(p_segm_pickle, 'wb'), -1)
    else:
        pose_ids = greedy_actors(data, n_actors=args.n_actors)
        data = DataPosesWrapper(data=data)

    visible_f = {a: {} for a in range(args.n_actors)}
    visible_f_max = 0.
    if show:
        plt.ion()
        fig = None
        axe = None
        scatters = dict()

    # how many images we have
    min_frame_id = min(f for f in pose_ids)
    frames_mod = max(f for f in pose_ids) - min_frame_id + 1
    skel_ours = Skeleton(frames_mod=frames_mod,
                         n_actors=args.n_actors,
                         min_frame_id=min_frame_id)
    skel_ours_2d = Skeleton(frames_mod=frames_mod,
                            n_actors=args.n_actors,
                            min_frame_id=min_frame_id)

    # assert len(images) == 0 or max(f for f in images) + 1 == frames_mod, \
    #     "Assumed image count is %d, but max_frame_id is %d" \
    #     % (len(images), frames_mod-1)
    if isinstance(data, DataPosesWrapper):
        frames = data.get_frames()
    else:
        frames = []
        for frame_str in sorted(data.get_frames()):
            try:
                frame_id = int(frame_str.split('_')[1])
            except ValueError:
                print("skipping key %s" % frame_id)
                continue
            frames.append(frame_id)
    my_visibilities = [[], []]
    for frame_id in frames:
        frame_str = DataPosesWrapper._to_frame_str(frame_id)
        pose_in = data.get_poses_3d(frame_id=frame_id)
        # np.asarray(data[frame_str][u'centered_3d'])
        # pose_in_2d = np.asarray(data[frame_str][u'pose_2d'])
        pose_in_2d = data.get_poses_2d(frame_id=frame_id)
        # visible = np.asarray(data[frame_str][u'visible'])

        if False and len(pose_in.shape) > 2:
            pose_id = pose_ids[frame_id]
            if not args.no_img:
                im = cv2.cvtColor(images[frame_id], cv2.COLOR_RGB2BGR)
                for i in range(pose_in.shape[0]):
                    c = (1., 0., 0., 1.)
                    if i == pose_id:
                        c = (0., 1., 0., 1.)
                    color = tuple(int(c_ * 255) for c_ in c[:3])
                    for p2d in pose_in_2d[i, :, :]:
                        # color = (c[0] * 255, c[1] * 255., c[2] * 255.)
                        cv2.circle(im, (p2d[1], p2d[0]),
                                   radius=3,
                                   color=color,
                                   thickness=-1)
                    center = np.mean(pose_in_2d[i, :, :],
                                     axis=0).round().astype('i4').tolist()
                    cv2.putText(im, "%d" % i, (center[1], center[0]), 1, 1,
                                color)
                if show:
                    cv2.imshow("im", im)
                    cv2.waitKey(100)
            # if sid not in scatters:
            #     scatters[sid] = axe.scatter(pose_in_2d[i, :, 1], pose_in_2d[i, :, 0], c=c)
            # else:
            #     scatters[sid].set_offsets(pose_in_2d[i, :, [1, 0]])
            #     scatters[sid].set_array(np.tile(np.array(c), pose_in_2d.shape[1]))
            # scatter.set_color(c)
            # plt.draw()
            # plt.pause(1.)
            pose_in = pose_in[pose_id, :, :]
            pose_in_2d = pose_in_2d[pose_id, :, :]
            visible = visible[pose_id]
        # else:
        # pose_id = 0
        # pose_id = pose_ids[frame_id]

        for actor_id in range(args.n_actors):
            # if actor_id in (2, 3, 4, 5, 8, 9)
            # expanded frame_id
            frame_id2 = Skeleton.unmod_frame_id(frame_id=frame_id,
                                                actor_id=actor_id,
                                                frames_mod=frames_mod)
            assert (actor_id != 0) ^ (frame_id2 == frame_id), "no"
            frame_id_mod = skel_ours.mod_frame_id(frame_id=frame_id2)

            assert frame_id_mod == frame_id, \
                "No: %d %d %d" % (frame_id, frame_id2, frame_id_mod)
            actor_id2 = skel_ours.get_actor_id(frame_id2)
            assert actor_id2 == actor_id, "no: %s %s" % (actor_id, actor_id2)

            # which pose explains this actor in this frame
            pose_id = pose_ids[frame_id][actor_id]
            # check, if actor found
            if pose_id < 0:
                continue

            # 3D pose
            pose = pose_in[pose_id, :, JointDenis.revmap].T
            # added by Aron on 4/4/2018 (Denis' pelvis is too high up)
            pose[:, Joint.PELV] = (pose[:, Joint.LHIP] + pose[:, Joint.RHIP]) \
                                  / 2.
            skel_ours.set_pose(frame_id2, pose)

            # 2D pose
            pose_2d = pose_in_2d[pose_id, :, :]
            arr = np.array(JointDenis.pose_2d_to_ours(pose_2d),
                           dtype=np.float32).T
            skel_ours_2d.set_pose(frame_id2, arr)

            #
            # visibility (binary) and confidence (float)
            #

            # np.asarray(data[frame_str][u'visible'][pose_id])
            vis_i = data.get_visibilities(frame_id)[pose_id]

            # vis_f = np.asarray(data[frame_str][u'visible_float'][pose_id])
            vis_f = data.get_confidences(frame_id)[pose_id]
            for jid, visible in enumerate(vis_i):  # for each joint
                # binary visibility
                jid_ours = JointDenis.to_ours_2d(jid)
                skel_ours_2d.set_visible(frame_id2, jid_ours, visible)

                # confidence (fractional visibility)
                if np.isnan(vis_f[jid]):
                    continue

                try:
                    visible_f[actor_id][frame_id2][jid_ours] = vis_f[jid]
                except KeyError:
                    visible_f[actor_id][frame_id2] = {jid_ours: vis_f[jid]}
                visible_f_max = max(visible_f_max, vis_f[jid])
                conf_ = get_conf_thresholded(vis_f[jid],
                                             thresh_log_conf=None,
                                             dtype_np=np.float32)
                skel_ours_2d.set_confidence(frame_id=frame_id2,
                                            joint=jid_ours,
                                            confidence=conf_)
                my_visibilities[0].append(vis_f[jid])
                my_visibilities[1].append(conf_)
            skel_ours_2d._confidence_normalized = True

    plt.figure()
    plt.plot(my_visibilities[0], my_visibilities[1], 'o')
    plt.savefig('confidences.pdf')

    assert skel_ours.n_actors == args.n_actors, "no"
    assert skel_ours_2d.n_actors == args.n_actors, "no"
    # align to room
    min_z = np.min(skel_ours.poses[:, 2, :])
    print("min_max: %s, %s" % (min_z, np.max(skel_ours.poses[:, 2, :])))
    skel_ours.poses[:, 2, :] += min_z
    skel_ours.poses /= 1000.
    # The output is scaled to 2m by Denis.
    # We change this to 1.8 * a scale in order to correct for
    # the skeletons being a bit too high still.
    skel_ours.poses *= \
        args.person_height * conf.optimize_path.height_correction / 2.
    skel_ours.poses[:, 2, :] *= -1.
    skel_ours.poses = skel_ours.poses[:, [0, 2, 1], :]

    # refine
    name_video = args.dir.split(os.sep)[-2]
    out_path = os.path.join(args.dir, os.pardir,
                            "skel_%s_%s.json" % (name_video, args.postfix))
    out_path_orig = os.path.join(args.dir, os.pardir,
                                 "skel_%s_lfd_orig.json" % name_video)
    sclt_orig = Scenelet(skeleton=copy.deepcopy(skel_ours))
    sclt_orig.save(out_path_orig)

    skel_ours_2d_all = copy.deepcopy(skel_ours_2d)
    assert len(skel_ours_2d_all.get_frames()), skel_ours_2d_all.get_frames()

    #
    # Optimize
    #

    # frames_ignore = [(282, 372), (516, 1000)]

    skel_ours, skel_ours_2d, intrinsics, \
    frame_ids_filled_in = prepare(
      args.camera_name,
      winsorize_limit=args.winsorize_limit,
      shape_orig=shape_orig,
      path_scene=p_scene,
      skel_ours_2d=skel_ours_2d,
      skel_ours=skel_ours,
      resample=args.resample,
    path_skel=path_skeleton)
    frames_ignore = []
    tr_ground = np.eye(4, dtype=np.float32)
    skel_opt, out_images, K = \
        optimize_path(
          skel_ours, skel_ours_2d, images, intrinsics=intrinsics,
          path_skel=out_path, shape_orig=shape_orig,
          use_huber=args.huber, weight_smooth=args.smooth,
          frames_ignore=frames_ignore, resample=args.resample,
          depth_init=scene_params['depth_init'],
          ground_rot=ground_rot)

    for frame_id in skel_opt.get_frames():
        skel_opt.set_time(frame_id=frame_id, time=float(frame_id) / args.r)

    skel_opt_raw = copy.deepcopy(skel_opt)
    skel_opt_resampled = Skeleton.resample(skel_opt)

    # Filter pelvis
    if args.with_filtering:
        out_filter_path = os.path.join(args.dir, os.pardir, "vis_filtering")
        skel_opt = filter_(skel_opt_resampled,
                           out_filter_path=out_filter_path,
                           skel_orig=skel_opt,
                           weight_smooth=args.smooth,
                           forwards_window_size=args.forwards_window_size)
    else:
        skel_opt.estimate_forwards(k=args.forwards_window_size)
        skel_opt_resampled.estimate_forwards(k=args.forwards_window_size)

    # if len(images):
    #     skel_opt.fill_with_closest(images.keys()[0], images.keys()[-1])

    min_y, max_y = skel_opt.get_min_y(tr_ground)
    print("min_y: %s, max_y: %s" % (min_y, max_y))

    #
    # save
    #
    frame_ids_old = set(skel_opt.get_frames())
    if args.resample:
        skel_opt = skel_opt_resampled
        frame_ids_filled_in.update(
            set(skel_opt.get_frames()).difference(frame_ids_old))
        lg.warning("Saving resampled scenelet!")
    scenelet = Scenelet(skel_opt)
    del skel_opt
    # skel_dict = skel_opt.to_json()
    tr_ground[1, 3] = min_y
    scenelet.aux_info['ground'] = tr_ground.tolist()
    assert isinstance(ground_rot, list) and len(ground_rot) == 3
    scenelet.add_aux_info('ground_rot', ground_rot)
    scenelet.add_aux_info(
        'path_opt_params', {
            'rate': args.r,
            'w-smooth': args.smooth,
            'winsorize-limit': args.winsorize_limit,
            'camera': args.camera_name,
            'huber': args.huber,
            'height_correction': conf.optimize_path.height_correction,
            'focal_correction': conf.optimize_path.focal_correction
        })
    scenelet.add_aux_info('frame_ids_filled_in', list(frame_ids_filled_in))

    # To MATLAB
    # _skeleton.get_min_y(_tr_ground)
    # with skel_opt as skeleton:
    # skeleton = skel_opt
    # skeleton_name = os.path.split(args.dir)[0]
    # skeleton_name = skeleton_name[skeleton_name.rfind('/')+1:]
    # mdict = skeleton.to_mdict(skeleton_name)
    # mdict['room_transform'] = tr_ground
    # mdict['room_transform'][1, 3] *= -1.
    # print(mdict)
    # print("scene_name?: %s" % os.path.split(args.dir)[0])
    # skeleton.save_matlab(
    #     os.path.join(os.path.dirname(args.dir), "skeleton_opt.mat"),
    #     mdict=mdict)

    assert scenelet.skeleton.has_forwards(), "No forwards??"
    scenelet.save(out_path)
    if show:
        # save path plot
        out_path_path = os.path.join(args.dir, os.pardir,
                                     "%s_path.jpg" % name_video)
        path_fig = plot_path(scenelet.skeleton)
        legend = ["smooth %g" % args.smooth]

    # hack debug
    # path_skel2 = os.path.join(args.dir, os.pardir, 'skel_lobby7_nosmooth.json')
    # if os.path.exists(path_skel2):
    #     skel2 = Skeleton.load(path_skel2)
    #     path_fig = plot_path(skel2, path_fig)
    #     legend.append('no smooth')
    if show:
        plt.legend(legend)
        path_fig.savefig(out_path_path)

    # backup args
    path_args = os.path.join(args.dir, os.pardir, 'args_denis.txt')
    with open(path_args, 'a') as f_args:
        f_args.write("%s %s\n" %
                     (os.path.basename(sys.executable), " ".join(argv)))

    # save 2D detections to file
    if args.postfix == 'unannot':
        path_skel_ours_2d = os.path.join(
            args.dir, os.pardir, "skel_%s_2d_%02d.json" % (name_video, 0))
        sclt_2d = Scenelet(skel_ours_2d_all)
        print('Saving {} to {}'.format(len(skel_ours_2d_all.get_frames()),
                                       path_skel_ours_2d))
        sclt_2d.skeleton.aux_info = {}
        sclt_2d.save(path_skel_ours_2d)
    else:
        print(args.postfix)

    logging.info("Saving images...")
    if len(images) and len(out_images):
        path_out_images = os.path.join(args.dir, os.pardir, 'color')
        try:
            os.makedirs(path_out_images)
        except OSError:
            pass
        visible_f_max_log = np.log(visible_f_max)
        frames = list(out_images.keys())
        for frame_id in range(frames[0], frames[-1] + 1):
            im = out_images[frame_id] if frame_id in out_images \
                else cv2.cvtColor(images[frame_id], cv2.COLOR_BGR2RGB)
            for actor_id in range(args.n_actors):
                if frame_id in visible_f[actor_id]:
                    frame_id2 = skel_ours_2d_all.unmod_frame_id(
                        frame_id=frame_id,
                        actor_id=actor_id,
                        frames_mod=skel_ours_2d_all.frames_mod)
                    for joint, is_vis in visible_f[actor_id][frame_id].items():
                        p2d = skel_ours_2d_all.get_joint_3d(joint,
                                                            frame_id=frame_id2)
                        # radius = np.log(is_vis) / visible_f_max_log
                        # lg.debug("r0: %g" % radius)
                        # radius = np.exp(np.log(is_vis) / visible_f_max_log)
                        # lg.debug("radius is %g" % radius)
                        vis_bool = True
                        if skel_ours_2d_all.has_visible(frame_id=frame_id2,
                                                        joint_id=joint):
                            vis_bool &= skel_ours_2d_all.is_visible(
                                frame_id2, joint)
                        radius = abs(np.log(is_vis / 0.1 + 1e-6))
                        if not np.isnan(radius):
                            p2d = (int(round(p2d[0])), int(round(p2d[1])))
                            cv2.circle(im,
                                       center=p2d,
                                       radius=int(round(radius)),
                                       color=(1., 1., 1., 0.5),
                                       thickness=1)
                            conf = get_conf_thresholded(conf=is_vis,
                                                        thresh_log_conf=None,
                                                        dtype_np=np.float32)
                            if conf > 0.5:
                                cv2.putText(img=im,
                                            text=Joint(joint).get_name(),
                                            org=p2d,
                                            fontFace=1,
                                            fontScale=1,
                                            color=(10., 150., 10., 100.))
                    # lg.debug("set visibility to %g, radius %g" % (is_vis, radius))
            # if frame_id in out_images:
            scale = (shape_orig[1] / float(im.shape[1]),
                     shape_orig[0] / float(im.shape[0]))
            cv2.imwrite(
                os.path.join(path_out_images, "color_%05d.jpg" % frame_id),
                cv2.resize(im, (0, 0),
                           fx=scale[0],
                           fy=scale[1],
                           interpolation=cv2.INTER_CUBIC))
            # else:
            #     fname = "color_%05d.jpg" % frame_id
            #     shutil.copyfile(
            #         os.path.join(path_images, fname),
            #         os.path.join(path_out_images, fname))
        lg.info("Wrote images to %s/" % path_out_images)
예제 #4
0
def main(argv=None):
    np.set_printoptions(suppress=True)
    parser = argparse.ArgumentParser()
    parser.add_argument('d', help="Folder of scene")
    parser.add_argument('-resolution',
                        help='Target resolution for occupancy map',
                        default=0.1)
    parser.add_argument(
        '-thresh-area',
        help='Ratio of occupancy map cell area that has to be occupied '
        'for it to count as occupied',
        default=0.1)
    parser.add_argument('-postfix',
                        type=str,
                        help="Scene postfix for augmentation",
                        default="")
    args = parser.parse_args(argv if argv is not None else sys.argv)
    res_target = args.resolution
    if args.postfix and len(args.postfix) and not args.postfix.startswith('_'):
        args.postfix = "_%s" % args.postfix

    path_parent, name_input = os.path.split(os.path.abspath(args.d))
    lg.warning("name input: %s" % name_input)
    path_for_tf = os.path.abspath(
        os.path.join(path_parent, os.pardir, 'dataset'))
    # if 'video' not in path_parent else os.path.join(path_parent, 'dataset')
    if not os.path.exists(path_for_tf):
        os.makedirs(path_for_tf, mode=0o0775)

    lg.debug("Loading scenelet...")
    path_scenelet = os.path.join(args.d, "skel_%s.json" % name_input)
    scenelet = Scenelet.load(path_scenelet)
    lg.debug("Scenelet: %s" % scenelet)

    path_state_pickle = os.path.join(args.d, "state%s.pickle" % args.postfix)
    if not os.path.exists(path_state_pickle):
        lg.error("Does not exist: %s" % path_state_pickle)
        return False

    # assert os.path.exists(path_state_pickle), \
    #     "Does not exist: %s" % path_state_pickle
    lg.debug("Loading volume...")
    state = pickle_load(open(path_state_pickle, 'rb'))
    lg.debug("Loaded volume...")

    lg.debug("Creating scene from scenelet")
    if not no_vis:
        vis = Visualizer(win_size=(1024, 1024))
        vis.add_coords()
    else:
        vis = None
    # scene = Scene(scenelet.name_scenelet)
    # colors = {0: (200., 0., 0.), 1: (0., 200., 0.), 2: (0., 0., 200.)}
    # unit_x = np.array((1., 0., 0.))

    occup = State(room=state.room,
                  tr_ground_inv=None,
                  res_theta=state.resolution[3],
                  resolution=[res_target, res_target, res_target])
    occup.get_volume(labels_to_lin_ids_arg=state.get_labels_to_lin_ids())
    occup_angle = np.ones(shape=(len(
        occup.volume), occup.volume[0].shape[0], occup.volume[0].shape[1], 1),
                          dtype=np.float32) * -1.
    assert np.min(occup_angle) < 0. and np.max(occup_angle) < 0., "Not empty"

    grid_polys = get_grid_shapely(occup=occup, res_orig=state.resolution)
    occup.volume.flags.writeable = True
    volume_occp = occup.volume
    angles = sorted(state.get_angles())
    labels_to_lin_ids = occup.get_labels_to_lin_ids()
    had_vtk_problem = no_vis

    plt.figure()

    rects = []
    for oid, ob in scenelet.objects.items():
        assert oid >= 0, "Need positive here"
        label = ob.label
        if label in TRANSLATIONS_CATEGORIES:
            label = TRANSLATIONS_CATEGORIES[label]

        if label not in labels_to_lin_ids:
            continue

        try:
            poly = get_poly([part.obb for part in ob.parts.values()])
        except ValueError as e:
            print("\n===========\n\nShapely error: %s for %s\n\n" %
                  (e, (label, oid, ob)))
            with open('error.log', 'a') as f:
                f.write("[%s] %d, %s, %s\n" % (args.d, oid, label, ob))
            continue

        ob_angle = ob.get_angle(positive_only=True)
        assert 0. <= ob_angle <= 2 * np.pi, "No: %g" % ob_angle

        rect = get_rectangle(poly, ob_angle)
        rect.extend([oid, CATEGORIES[label]])
        rects.append(rect)

        cat_id = labels_to_lin_ids[label]  # cat_id in volume, not categories
        for gp in grid_polys:
            # skip, if not occupied enough
            if gp.poly.intersection(poly).area / gp.area < args.thresh_area:
                continue
            # save occupancy
            gp.occupancy = 1.
            id_angle_lower = None
            id_angle_upper = None
            if ob_angle > angles[-1]:
                id_angle_lower = len(angles) - 1
                id_angle_upper = 0
            else:
                for id_angle, angle in enumerate(angles):
                    if ob_angle < angle:
                        id_angle_upper = id_angle
                        id_angle_lower = id_angle - 1
                        break
            assert id_angle_lower is not None \
                   and id_angle_upper is not None, \
                "Wrong?"
            assert id_angle_upper != id_angle_lower, \
                "? %s %s" % (id_angle_lower, id_angle_upper)

            # cache
            xy = gp.xy

            # zero means empty in occupancy,
            # so object ids are shifted with 1
            # we need object ids to filter "untouched" objects
            # in tfrecords_create
            if volume_occp[cat_id, xy[0], xy[1], id_angle_lower] == 0 \
               or label in CATEGORIES_DOMINANT:
                volume_occp[cat_id, xy[0], xy[1], id_angle_lower] = oid + 1
            if volume_occp[cat_id, xy[0], xy[1], id_angle_upper] == 0 \
               or label in CATEGORIES_DOMINANT:
                volume_occp[cat_id, xy[0], xy[1], id_angle_upper] = oid + 1

            # angles are right now not per-category, but per-scene
            # hence, an object can only overwrite, if it's usually "above"
            # other objects, e.g. a table
            # this is a hack for a z-test
            if occup_angle[cat_id, xy[0], xy[1], 0] < 0. \
               or label in CATEGORIES_DOMINANT:
                occup_angle[cat_id, xy[0], xy[1], 0] = ob_angle

        if not had_vtk_problem:
            color = COLORS_CATEGORIES[label] if label in COLORS_CATEGORIES \
                else (200., 200., 200.)
            try:
                for id_part, part in ob.parts.items():
                    vis.add_mesh(MeshOBJ.from_obb(part.obb),
                                 name="ob_%02d_part_%02d" % (oid, id_part),
                                 color=color)
            except AttributeError:
                print("VTK problem...")
                had_vtk_problem = True
    #plt.savefig()
    plt.close()
    if not had_vtk_problem:
        vis.set_camera_pos(pos=(0., -1., 0.))
        vis.camera().SetFocalPoint(0., 0., 0.)
        vis.camera().SetViewUp(-1., 0., 0.)
        vis.set_camera_type(is_ortho=True)
        vis.camera().SetParallelScale(3.)
    # vis.show()

    name_recording = "%s_%s" % (os.path.basename(args.d), args.postfix) \
        if args.postfix else os.path.basename(args.d)
    lg.info("name_recording: %s" % name_recording)

    path_out_occp = os.path.join(os.path.dirname(args.d), os.pardir,
                                 'occupancy', name_recording)
    if not os.path.exists(path_out_occp):
        os.makedirs(path_out_occp)

    # prepare www storage
    www_grid = {'evidence': {}, 'occ': {}}

    # normalize evidence maps
    vmax = 0.
    ims = {}
    for cat, cat_id in labels_to_lin_ids.items():
        ims[cat] = np.squeeze(
            np.sum(state.volume[cat_id, :, :, :], axis=2, keepdims=True))
        vmax = max(vmax, np.max(ims[cat]))

    # gather joined occupancy map
    im_sum = None

    # for each evidence category
    for cat, cat_id in labels_to_lin_ids.items():
        im = ims[cat] / vmax * 255.
        path_out_im = os.path.join(path_out_occp, "e_%s.jpg" % cat)
        cv2.imwrite(path_out_im, im)
        # lg.debug("wrote to %s" % path_out_im)
        www_grid['evidence'][cat] = path_out_im

        im = np.squeeze(volume_occp[cat_id, :, :, 0])
        path_out_im = os.path.join(path_out_occp, "o_%s.jpg" % cat)
        cv2.imwrite(path_out_im, im * 255.)
        # lg.debug("wrote to %s" % path_out_im)
        www_grid['occ'][cat] = path_out_im

        if im_sum is None:
            im_sum = im.copy()
        else:
            im_sum = np.maximum(im, im_sum)

    #
    # save dataset
    #
    name_input_old = name_input
    if args.postfix is not None and len(args.postfix):
        name_input = "%s_%s" % (name_input, args.postfix)

    # state
    path_state_dest = os.path.join(path_for_tf, "state_%s.pickle" % name_input)
    shutil.copyfile(path_state_pickle, path_state_dest)
    lg.info("Copied\n\t%s to\n\t%s" % (path_state_pickle, path_state_dest))

    # occupancy
    path_occup_dest = os.path.join(path_for_tf, "occup_%s.pickle" % name_input)
    pickle.dump(occup, open(path_occup_dest, 'wb'), -1)
    lg.info("Wrote to %s" % path_occup_dest)

    # occupancy_angle
    path_occup_angle_dest = os.path.join(path_for_tf,
                                         "angle_%s.npy" % name_input)
    min_angle = np.min(occup_angle)
    assert min_angle < 0., "No empty cells??"
    lg.debug("min angle is %s" % min_angle)
    np.save(open(path_occup_angle_dest, 'wb'), occup_angle)
    lg.info("Wrote to %s" % path_occup_angle_dest)

    # skeleton
    path_copied = shutil.copy2(path_scenelet, path_for_tf)
    lg.info("Copied\n\t%s to \n\t%s" % (path_scenelet, path_copied))

    # charness skeleton
    name_skeleton_charness = "skel_%s-charness.json" % name_input_old
    path_scenelet_charness = os.path.join(args.d, name_skeleton_charness)
    assert os.path.exists(path_scenelet_charness), \
        "Does not exist: %s" % path_scenelet_charness
    shutil.copy2(path_scenelet_charness, path_for_tf)
    assert os.path.exists(os.path.join(path_for_tf, name_skeleton_charness)), \
        "Does not exist: %s" % os.path.join(path_for_tf,
                                            name_skeleton_charness)

    # rectangles
    name_rectangles = "rectangles_%s.npy" % name_input_old
    path_rectangles = os.path.join(path_for_tf, name_rectangles)
    np.save(open(path_rectangles, 'wb'), rects)

    #
    # visualize
    #

    path_out_im = os.path.join(path_out_occp, '3d.png')
    if not had_vtk_problem:
        vis.save_png(path_out_im)
    www_grid['3d'] = path_out_im

    path_out_im = os.path.join(path_out_occp, 'o_sum.png')
    max_im_sum = np.max(im_sum)
    if max_im_sum > 0.:
        cv2.imwrite(path_out_im, im_sum / max_im_sum * 255.)
    else:
        cv2.imwrite(path_out_im, im_sum * 255.)
    www_grid['o_sum'] = path_out_im

    path_www = os.path.join(path_out_occp, os.pardir)
    with open(os.path.join(path_www, 'index.html'), 'a') as f:
        f.write("<style> img {image-rendering: pixelated; } </style>\n")
        f.write("<script>\n")
        f.write("</script>\n")
        f.write("<h3>%s</h3>" % os.path.basename(args.d))
        f.write('<table>\n')

        f.write("<tr>\n")
        f.write("<th>3d</th>")
        f.write("<th>Occupancy sum</th>")
        for cat in www_grid['evidence']:
            f.write("\t<th>%s</th>\n" % cat)
        f.write("<th></th>\n")  # titles
        f.write("</tr>\n")

        f.write("<tr>\n")
        # 3D
        f.write("\t<td rowspan=\"2\">\n")
        path_im = os.path.relpath(www_grid['3d'], path_www)
        f.write("\t<a href=\"%s\">\n"
                "\t\t<img src=\"%s\" height=\"400\" />\n"
                "\t</a>\n" % (path_im, path_im))

        # Evidence sum
        f.write("\t<td rowspan=\"2\">\n")
        path_im = os.path.relpath(www_grid['o_sum'], path_www)
        f.write("\t<a href=\"%s\">\n"
                "\t\t<img src=\"%s\" height=\"400\" />\n"
                "\t</a>\n" % (path_im, path_im))
        # Evidence
        for cat in www_grid['evidence']:
            f.write("<td style=\"padding-bottom: 2px\">\n")
            path_im = os.path.relpath(www_grid['evidence'][cat], path_www)
            f.write("\t<a href=\"%s\">\n"
                    "\t\t<img src=\"%s\" height=\"200\" />\n"
                    "\t</a>\n" % (path_im, path_im))
            f.write("</td>\n")
        f.write("<td>Evidence</td>\n")

        f.write("\t</td>\n")
        f.write("</tr>\n")

        f.write("<tr>\n")
        for cat in www_grid['occ']:
            f.write("<td>\n")
            path_im = os.path.relpath(www_grid['occ'][cat], path_www)
            f.write("\t<a href=\"%s\">\n"
                    "\t\t<img src=\"%s\" height=\"200\" />\n"
                    "</a>\n" % (path_im, path_im))
            f.write("</td>\n")
        f.write("<td>Occupancy map</td>\n")
        f.write("</tr>")

        f.write('</table>')

    return True
예제 #5
0
def main(argv):
    pjoin = os.path.join  # cache long name
    parser = argparse.ArgumentParser(
        "Find characteristic scene times",
        description="Scans a directory of short scenelets (exported from "
        "Matlab), and looks up their full version in the original "
        "scenes. It exports a new scenelet containing all poses "
        "between the start and end times of the input short "
        "scenelets. Matching is done by name."
        "Scenelets below time length limit and not enough objects "
        "are thrown away."
        "It also saves the scenelet characteristicness into the "
        "output scenelet files.")
    parser.add_argument(
        'd',
        type=argparse_check_exists,
        help="Folder containing PiGraphs scenelets. E.g. "
        "/mnt/thorin_data/stealth/shared/"
        "pigraph_scenelets__linterval_squarehist_large_radiusx2")
    parser.add_argument('s',
                        type=argparse_check_exists,
                        help="Folder containing PiGraphs full scenes. E.g. "
                        "/mnt/thorin_data/stealth/shared/scenes_pigraphs")
    parser.add_argument('-l',
                        '--limit-len',
                        type=int,
                        help="Minimum length for a scenelet",
                        default=10)  # changed from `5` on 15/1/2018
    parser.add_argument(
        '--dist-thresh',
        type=float,
        help='Distance threshold for object pruning. Typically: 0.2 or 0.5.',
        default=.5)
    # parse arguments
    args = parser.parse_args(argv)
    parts_to_remove = ['sidetable']
    lg.warning("Will remove all parts named %s" % parts_to_remove)

    # read scenes and scenelets
    p_pickle = pjoin(args.d, 'scenes_and_scenelets.pickle')
    if os.path.exists(p_pickle):
        lg.info("reading from %s" % p_pickle)
        scenes, scenelets = pickle_load(open(p_pickle, 'rb'))
        lg.info("read from %s" % p_pickle)
    else:
        scenelets = read_scenelets(args.d)
        scenes = read_scenelets(args.s)
        scenes = {scene.name_scene: scene for scene in scenes}
        pickle.dump((scenes, scenelets), open(p_pickle, 'wb'), protocol=-1)
        lg.info("wrote to %s" % p_pickle)

    # Read characteristicnesses (to put them into the scenelet).
    p_charness = pjoin(args.d, "charness__gaussian.mat")
    pose_charness, scenelet_names = read_charness(p_charness,
                                                  return_hists=False,
                                                  return_names=True)

    # output folder
    d_scenelets_parent = os.path.dirname(args.d)
    d_dest = pjoin(d_scenelets_parent, 'deb',
                   "%s_full_sampling" % args.d.split(os.sep)[-1])
    # makedirs_backed
    if os.path.exists(d_dest):
        i = 0
        while i < 100:
            try:
                os.rename(d_dest, "%s.bak.%02d" % (d_dest, i))
                break
            except OSError:
                i += 1
    os.makedirs(d_dest)

    # _is_close = is_close  # cache namespace lookup
    # processing
    for sclt in scenelets:
        # cache skeleton
        skeleton = sclt.skeleton

        if 'scene09' in sclt.name_scenelet or 'scene10' in sclt.name_scenelet:
            lg.debug("here")
        else:
            continue

        # prune objects
        per_cat = {}
        cnt = 0
        for oid, scene_obj in sclt.objects.items():
            close_, dist = is_close(scene_obj,
                                    skeleton,
                                    args.dist_thresh,
                                    return_dist=True)
            label = scene_obj.label
            if 'chair' in label or 'couch' in label or 'stool' in label:
                label = 'sittable'

            try:
                per_cat[label].append((dist, oid))
            except KeyError:
                per_cat[label] = [(dist, oid)]
            if scene_obj.label != 'floor':
                cnt += 1

        per_cat = {k: sorted(v) for k, v in per_cat.items()}

        name_scene = sclt.name_scene.split('__')[0]
        if '-no-coffeetable' in name_scene:
            name_scene = name_scene[:name_scene.find('-no-coffeetable')]
        scene = scenes[name_scene]

        if 'shelf' not in per_cat:
            for oid, ob in scene.objects.items():
                if ob.label == 'shelf':
                    close_, dist = is_close(ob,
                                            skeleton,
                                            args.dist_thresh,
                                            return_dist=True)
                    oid_ = oid
                    while oid_ in sclt.objects:
                        oid_ += 1
                    sclt.add_object(oid_, ob)
                    cnt += 1
                    try:
                        per_cat['shelf'].append((dist, oid_))
                    except KeyError:
                        per_cat['shelf'] = [(dist, oid_)]

        if 'shelf' in per_cat:
            assert len(per_cat['shelf']) == 1, "TODO: keep all shelves"

        oids_to_keep = [
            v[0][1] for v in per_cat.values() if v[0][0] < args.dist_thresh
        ]

        if not len(oids_to_keep):  # there is always a floor
            lg.warning("Skipping %s, not enough objects: %s" %
                       (sclt.name_scenelet, per_cat))
            continue

        # if 'gates392_mati3_2014-04-30-21-13-46__scenelet_25' \
        #     == sclt.name_scenelet:
        #     lg.debug("here")
        # else:
        #     continue

        # copy skeleton with dense sampling in time
        mn, mx = skeleton.get_frames_min_max()
        # assert mn == 0, "This usually starts indexing from 0, " \
        #                 "no explicit problem, just flagging the change."
        time_mn = floor(skeleton.get_time(mn))
        time_mx = ceil(skeleton.get_time(mx))

        # artificially prolong mocap scenes
        if 'scene' in name_scene and (time_mx - time_mn < 60):
            d = (time_mx - time_mn) // 2 + 1
            time_mn -= d
            time_mx += d
        # lookup original scene name
        # mn_frame_id_scene, mx_frame_id_scene = \
        #     scene.skeleton.get_frames_min_max()

        frame_ids_old = skeleton.get_frames()
        times_old = [skeleton.get_time(fid) for fid in frame_ids_old]
        for frame_id in frame_ids_old:
            skeleton.remove_pose(frame_id)

        for frame_id in range(time_mn, time_mx + 1):
            if not scene.skeleton.has_pose(frame_id):
                continue
            pose = scene.skeleton.get_pose(frame_id=frame_id)
            # scale mocap skeletons
            fw = scene.skeleton.get_forward(frame_id=frame_id,
                                            estimate_ok=False)
            sclt.set_pose(frame_id=frame_id,
                          angles=None,
                          pose=pose,
                          forward=fw,
                          clone_forward=True)

        if 'scene0' in name_scene or 'scene10' in name_scene:
            mx_old = np.max(sclt.skeleton.poses[:, 1, :])
            sclt.skeleton.poses *= 0.8
            mx_new = np.max(sclt.skeleton.poses[:, 1, :])
            sclt.skeleton.poses[1, :] += mx_new - mx_old + 0.05
        _frames = sclt.skeleton.get_frames()

        # check length
        if len(_frames) < args.limit_len:
            lg.warning("Skipping %s, because not enough frames: %s" %
                       (sclt.name_scene, _frames))
            continue

        # save charness
        try:
            id_charness = next(i for i in range(len(scenelet_names))
                               if scenelet_names[i] == sclt.name_scenelet)
            sclt.charness = pose_charness[id_charness]
        except StopIteration:
            lg.error("Something is wrong, can't find %s, %s in charness db "
                     "containing names such as %s." %
                     (sclt.name_scene, sclt.name_scenelet, scenelet_names[0]))
            sclt.charness = 0.4111111

        _mn, _mx = (_frames[0], _frames[-1])
        assert _mn >= time_mn, "not inside? %s < %s" % (_mn, time_mn)
        assert _mx <= time_mx, "not inside? %s < %s" % (_mx, time_mx)
        if len(_frames) < len(frame_ids_old):
            lg.warning("Not more frames than interpolated "
                       "scenelet?\n%s\n%s\n%s" %
                       (_frames, frame_ids_old, times_old))

        oids = list(sclt.objects.keys())
        for oid in oids:
            if oid not in oids_to_keep:
                lg.debug("removed %s" % sclt.objects[oid])
                sclt.objects.pop(oid)
            else:
                obj = sclt.objects[oid]
                part_ids_to_remove = [
                    part_id for part_id, part in obj.parts.items()
                    if part.label in parts_to_remove
                ]
                if len(part_ids_to_remove) == len(obj.parts):
                    sclt.objects.pop(oid)
                else:
                    for part_id in part_ids_to_remove:
                        lg.debug("removed %s" %
                                 sclt.objects[obj].parts[part_id])
                        obj.parts.pop(part_id)
        if len(sclt.objects) < 2 and next(iter(
                sclt.objects.values())).label == 'floor':
            lg.debug("finally removing scenelet: %s" % sclt.objects)
            continue

        # save in the scene folder
        d_dest_scene = pjoin(d_dest, name_scene)
        if not os.path.exists(d_dest_scene):
            os.makedirs(d_dest_scene)
        sclt.save(pjoin(d_dest_scene, "skel_%s" % sclt.name_scenelet))