def read_charness_histograms(path): """ ['pigraph_norm_factor', 'pigraph_histogram_params', 'pigraph_histogram_charness', 'pigraph_pose_charness', 'pigraph_scenelet_names', 'categories'] """ hash_mat_file_current = hash_path_md5(path) hists = None path_pickle = "%s.pickle" % path if os.path.exists(path_pickle): with open(path_pickle, 'rb') as f: # hists, hash_mat_file = pickle_load(f) tmp = pickle_load(f) if tmp[-1] != hash_mat_file_current: hists = None lg.warning("Hashes don't match, reloading hists") else: if len(tmp) == 3: hists = tmp[1] # pose_charness, hists, hash else: hists = tmp[0] # hists, hash lg.info("Loaded hists from\n\t%s!!!" % split_path(path_pickle)) if hists is None: dmat = scipy.io.loadmat(path) hists = parse_charness_histograms(dmat) with open(path_pickle, 'wb') as f: pickle.dump((hists, hash_mat_file_current), f, -1) lg.info("Saved hists to %s" % path_pickle) # print(hists.keys()) # key = list(hists.keys())[0] # logging.info("key: %s, %s" % (key, hists[key].volume)) return hists
def main(argv): np.set_printoptions(suppress=True, linewidth=200) pjoin = os.path.join parser = argparse.ArgumentParser("matcher") parser.add_argument("d_scenelets", type=argparse_check_exists, help="Folder containing scenelets") parser.add_argument("video", type=argparse_check_exists, help="Input path") parser.add_argument("--gap-size-limit", type=int, help="Smallest gap size to still explain") args = parser.parse_args(argv) d_query = os.path.dirname(args.video) # 2d keypoint rescale p_im = pjoin(d_query, 'origjpg', 'color_00100.jpg') im_ = cv2.imread(p_im) shape_orig = im_.shape scale_2d = shape_orig[0] / float(INPUT_SIZE) query = Scenelet.load(args.video, no_obj=True) tr_ground = np.array(query.aux_info['ground'], dtype=np.float32) print("tr: %s" % tr_ground) name_query = os.path.basename(args.video).split('_')[1] query_2d = Scenelet.load(pjoin(d_query, "skel_%s_2d_00.json" % name_query)).skeleton p_intr = pjoin(d_query, 'intrinsics.json') intr = np.array(json.load(open(p_intr, 'r')), dtype=np.float32) lg.debug("intr: %s" % intr) gaps = find_gaps(query.skeleton, min_pad=1) p_scenelets_pickle = pjoin(args.d_scenelets, 'match_gap_scenelets.pickle') if os.path.exists(p_scenelets_pickle): scenelets = pickle_load(open(p_scenelets_pickle, 'rb')) else: scenelets = read_scenelets(args.d_scenelets, limit=0) pickle.dump(scenelets, open(p_scenelets_pickle, 'wb')) p_out_sclts = pjoin(d_query, 'fill') if os.path.exists(p_out_sclts): shutil.rmtree(p_out_sclts) os.makedirs(p_out_sclts) times = [] for gap_id, gap in enumerate(gaps): lg.debug("gap is %s" % repr(gap)) if gap[1] - gap[0] < args.gap_size_limit: continue with Timer("gap %d" % gap_id) as timer: chosen = [] for sc_id, sclt in enumerate(scenelets): lg.info("scenelet %d / %d" % (sc_id, len(scenelets))) sclt = scenelets[sc_id] ground_obj = next(ob for ob in sclt.objects.values() if ob.label == 'floor') ground_part = ground_obj.get_part_by_name('floor') lg.debug("floor: %s" % ground_part) ground_transform = ground_part.obb.as_transform() lg.debug("floor: %s" % ground_transform) # sys.exit(0) # chosen.extend( out_sclts = match(query, d_query, query_2d, sclt, intr, gap, tr_ground, scale=scale_2d) if not len(out_sclts): continue # pick best from scene chosen.append( [out_sclts[0][i] for i in range(len(out_sclts[0]))] + [sc_id]) # break # os.system("rm %s/skel_%s_fill_%03d_%03d__*.json" # % (p_out_sclts, name_query, gap[0], gap[1])) chosen = sorted(chosen, key=lambda score_sclt: score_sclt[0]) for sid, (score, out_sclt, sc_id) in enumerate(chosen): p_out = pjoin( p_out_sclts, "skel_%s_fill_%03d_%03d__%02d.json" % (name_query, gap[0], gap[1], sid)) out_sclt.save(p_out) if sid > 5: break times.append(timer.get_elapsed_ms()) lg.info("mean time per gap: %s" % np.mean(times))
def main(argv): conf = Conf.get() parser = argparse.ArgumentParser("Denis pose converter") parser.add_argument('camera_name', help="Camera name ('G15', 'S6')", type=str) parser.add_argument( '-d', dest='dir', required=True, help="Path to the <scene folder>/denis containing skeletons.json") parser.add_argument( '-filter', dest='with_filtering', action="store_true", help="Should we do post-filtering (1-euro) on the pelvis positions") parser.add_argument('-huber', required=False, help="Should we do huber loss?", action='store_true') parser.add_argument('-smooth', type=float, default=0.005, help="Should we have a smoothness term (l2/huber)?") parser.add_argument( '--winsorize-limit', type=float, default=conf.optimize_path.winsorize_limit, help='Threshold for filtering too large jumps of the 2D centroid') parser.add_argument('--no-resample', action='store_true', help="add resampled frames") parser.add_argument('--n-actors', type=int, default=1, help="How many skeletons to track.") parser.add_argument('-n-actors', type=int, default=1, help="Max number of people in scene.") # parser.add_argument( # '-r', type=float, # help='Video rate. Default: 1, if avconv -r 5. ' # 'Original video sampling rate (no subsampling) should be ' # '24/5=4.8. avconv -r 10 leads to 24/10=2.4.', # required=True) parser.add_argument('--person_height', type=float, help='Assumed height of human(s) in video.', default=Conf.get().optimize_path.person_height) parser.add_argument( '--forwards-window-size', type=int, help='How many poses in time to look before AND after to ' 'average forward direction. 0 means no averaging. Default: 0.', default=0) parser.add_argument('--no-img', action='store_true', help='Read and write images (vis reproj error)') parser.add_argument('--postfix', type=str, help="output file postfix.", default='unannot') args = parser.parse_args(argv) show = False args.resample = not args.no_resample # assert not args.resample, "resample should be off" assert os.path.exists(args.dir), "Source does not exist: %s" % args.dir p_scene = os.path.normpath(os.path.join(args.dir, os.pardir)) # type: str p_video_params = os.path.join(p_scene, 'video_params.json') assert os.path.exists(p_video_params), "Need video_params.json for rate" if 'r' not in args or args.r is None: args.r = json.load(open(p_video_params, 'r'))['rate-avconv'] # manual parameters (depth initialization, number of actors) p_scene_params = os.path.join(args.dir, os.pardir, 'scene_params.json') if not os.path.exists(p_scene_params): scene_params = { 'depth_init': 10., 'actors': args.n_actors, 'ground_rot': [0., 0., 0.] } json.dump(scene_params, open(p_scene_params, 'w')) raise RuntimeError("Inited scene_params.json, please check: %s" % p_scene_params) else: scene_params = json.load(open(p_scene_params, 'r')) lg.warning("Will work with %d actors and init depth to %g" % (scene_params['actors'], scene_params['depth_init'])) assert '--n-actors' not in argv \ or args.n_actors == scene_params['actors'], \ "Actor count mismatch, remove %d from args, because " \ "scene_params.json says %d?" \ % (args.n_actors, scene_params['actors']) args.n_actors = scene_params['actors'] ground_rot = scene_params['ground_rot'] or [0., 0., 0.] # load images path_images = os.path.abspath(os.path.join(args.dir, os.pardir, 'origjpg')) images = {} shape_orig = None if not args.no_img: images, shape_orig = load_images(path_images) path_skeleton = \ max((f for f in os.listdir(os.path.join(args.dir)) if f.startswith('skeletons') and f.endswith('json')), key=lambda s: int(os.path.splitext(s)[0].split('_')[1])) print("path_skeleton: %s" % path_skeleton) data = json.load(open(os.path.join(args.dir, path_skeleton), 'r')) # data, pose_constraints, first_run = \ # cleanup(data, p_dir=os.path.join(args.dir, os.pardir)) # poses_2d = [] # plt.figure() # show_images(images, data) if False: # pose_ids = identify_actors_multi(data, n_actors=1) p_segm_pickle = os.path.join(args.dir, os.pardir, "label_skeletons.pickle") problem = None if False and os.path.exists(p_segm_pickle): lg.warning("Loading skeleton segmentation from pickle %s" % p_segm_pickle) pose_ids, problem = pickle_load(open(p_segm_pickle, 'rb')) if not problem or problem._n_actors != args.n_actors: pose_ids, problem, data = more_actors_gurobi( data, n_actors=args.n_actors, constraints=pose_constraints, first_run=first_run) if True or show: show_multi(images, data, pose_ids, problem, p_dir=os.path.join(args.dir, os.pardir), first_run=first_run, n_actors=args.n_actors) pickle.dump((pose_ids, problem), open(p_segm_pickle, 'wb'), -1) else: pose_ids = greedy_actors(data, n_actors=args.n_actors) data = DataPosesWrapper(data=data) visible_f = {a: {} for a in range(args.n_actors)} visible_f_max = 0. if show: plt.ion() fig = None axe = None scatters = dict() # how many images we have min_frame_id = min(f for f in pose_ids) frames_mod = max(f for f in pose_ids) - min_frame_id + 1 skel_ours = Skeleton(frames_mod=frames_mod, n_actors=args.n_actors, min_frame_id=min_frame_id) skel_ours_2d = Skeleton(frames_mod=frames_mod, n_actors=args.n_actors, min_frame_id=min_frame_id) # assert len(images) == 0 or max(f for f in images) + 1 == frames_mod, \ # "Assumed image count is %d, but max_frame_id is %d" \ # % (len(images), frames_mod-1) if isinstance(data, DataPosesWrapper): frames = data.get_frames() else: frames = [] for frame_str in sorted(data.get_frames()): try: frame_id = int(frame_str.split('_')[1]) except ValueError: print("skipping key %s" % frame_id) continue frames.append(frame_id) my_visibilities = [[], []] for frame_id in frames: frame_str = DataPosesWrapper._to_frame_str(frame_id) pose_in = data.get_poses_3d(frame_id=frame_id) # np.asarray(data[frame_str][u'centered_3d']) # pose_in_2d = np.asarray(data[frame_str][u'pose_2d']) pose_in_2d = data.get_poses_2d(frame_id=frame_id) # visible = np.asarray(data[frame_str][u'visible']) if False and len(pose_in.shape) > 2: pose_id = pose_ids[frame_id] if not args.no_img: im = cv2.cvtColor(images[frame_id], cv2.COLOR_RGB2BGR) for i in range(pose_in.shape[0]): c = (1., 0., 0., 1.) if i == pose_id: c = (0., 1., 0., 1.) color = tuple(int(c_ * 255) for c_ in c[:3]) for p2d in pose_in_2d[i, :, :]: # color = (c[0] * 255, c[1] * 255., c[2] * 255.) cv2.circle(im, (p2d[1], p2d[0]), radius=3, color=color, thickness=-1) center = np.mean(pose_in_2d[i, :, :], axis=0).round().astype('i4').tolist() cv2.putText(im, "%d" % i, (center[1], center[0]), 1, 1, color) if show: cv2.imshow("im", im) cv2.waitKey(100) # if sid not in scatters: # scatters[sid] = axe.scatter(pose_in_2d[i, :, 1], pose_in_2d[i, :, 0], c=c) # else: # scatters[sid].set_offsets(pose_in_2d[i, :, [1, 0]]) # scatters[sid].set_array(np.tile(np.array(c), pose_in_2d.shape[1])) # scatter.set_color(c) # plt.draw() # plt.pause(1.) pose_in = pose_in[pose_id, :, :] pose_in_2d = pose_in_2d[pose_id, :, :] visible = visible[pose_id] # else: # pose_id = 0 # pose_id = pose_ids[frame_id] for actor_id in range(args.n_actors): # if actor_id in (2, 3, 4, 5, 8, 9) # expanded frame_id frame_id2 = Skeleton.unmod_frame_id(frame_id=frame_id, actor_id=actor_id, frames_mod=frames_mod) assert (actor_id != 0) ^ (frame_id2 == frame_id), "no" frame_id_mod = skel_ours.mod_frame_id(frame_id=frame_id2) assert frame_id_mod == frame_id, \ "No: %d %d %d" % (frame_id, frame_id2, frame_id_mod) actor_id2 = skel_ours.get_actor_id(frame_id2) assert actor_id2 == actor_id, "no: %s %s" % (actor_id, actor_id2) # which pose explains this actor in this frame pose_id = pose_ids[frame_id][actor_id] # check, if actor found if pose_id < 0: continue # 3D pose pose = pose_in[pose_id, :, JointDenis.revmap].T # added by Aron on 4/4/2018 (Denis' pelvis is too high up) pose[:, Joint.PELV] = (pose[:, Joint.LHIP] + pose[:, Joint.RHIP]) \ / 2. skel_ours.set_pose(frame_id2, pose) # 2D pose pose_2d = pose_in_2d[pose_id, :, :] arr = np.array(JointDenis.pose_2d_to_ours(pose_2d), dtype=np.float32).T skel_ours_2d.set_pose(frame_id2, arr) # # visibility (binary) and confidence (float) # # np.asarray(data[frame_str][u'visible'][pose_id]) vis_i = data.get_visibilities(frame_id)[pose_id] # vis_f = np.asarray(data[frame_str][u'visible_float'][pose_id]) vis_f = data.get_confidences(frame_id)[pose_id] for jid, visible in enumerate(vis_i): # for each joint # binary visibility jid_ours = JointDenis.to_ours_2d(jid) skel_ours_2d.set_visible(frame_id2, jid_ours, visible) # confidence (fractional visibility) if np.isnan(vis_f[jid]): continue try: visible_f[actor_id][frame_id2][jid_ours] = vis_f[jid] except KeyError: visible_f[actor_id][frame_id2] = {jid_ours: vis_f[jid]} visible_f_max = max(visible_f_max, vis_f[jid]) conf_ = get_conf_thresholded(vis_f[jid], thresh_log_conf=None, dtype_np=np.float32) skel_ours_2d.set_confidence(frame_id=frame_id2, joint=jid_ours, confidence=conf_) my_visibilities[0].append(vis_f[jid]) my_visibilities[1].append(conf_) skel_ours_2d._confidence_normalized = True plt.figure() plt.plot(my_visibilities[0], my_visibilities[1], 'o') plt.savefig('confidences.pdf') assert skel_ours.n_actors == args.n_actors, "no" assert skel_ours_2d.n_actors == args.n_actors, "no" # align to room min_z = np.min(skel_ours.poses[:, 2, :]) print("min_max: %s, %s" % (min_z, np.max(skel_ours.poses[:, 2, :]))) skel_ours.poses[:, 2, :] += min_z skel_ours.poses /= 1000. # The output is scaled to 2m by Denis. # We change this to 1.8 * a scale in order to correct for # the skeletons being a bit too high still. skel_ours.poses *= \ args.person_height * conf.optimize_path.height_correction / 2. skel_ours.poses[:, 2, :] *= -1. skel_ours.poses = skel_ours.poses[:, [0, 2, 1], :] # refine name_video = args.dir.split(os.sep)[-2] out_path = os.path.join(args.dir, os.pardir, "skel_%s_%s.json" % (name_video, args.postfix)) out_path_orig = os.path.join(args.dir, os.pardir, "skel_%s_lfd_orig.json" % name_video) sclt_orig = Scenelet(skeleton=copy.deepcopy(skel_ours)) sclt_orig.save(out_path_orig) skel_ours_2d_all = copy.deepcopy(skel_ours_2d) assert len(skel_ours_2d_all.get_frames()), skel_ours_2d_all.get_frames() # # Optimize # # frames_ignore = [(282, 372), (516, 1000)] skel_ours, skel_ours_2d, intrinsics, \ frame_ids_filled_in = prepare( args.camera_name, winsorize_limit=args.winsorize_limit, shape_orig=shape_orig, path_scene=p_scene, skel_ours_2d=skel_ours_2d, skel_ours=skel_ours, resample=args.resample, path_skel=path_skeleton) frames_ignore = [] tr_ground = np.eye(4, dtype=np.float32) skel_opt, out_images, K = \ optimize_path( skel_ours, skel_ours_2d, images, intrinsics=intrinsics, path_skel=out_path, shape_orig=shape_orig, use_huber=args.huber, weight_smooth=args.smooth, frames_ignore=frames_ignore, resample=args.resample, depth_init=scene_params['depth_init'], ground_rot=ground_rot) for frame_id in skel_opt.get_frames(): skel_opt.set_time(frame_id=frame_id, time=float(frame_id) / args.r) skel_opt_raw = copy.deepcopy(skel_opt) skel_opt_resampled = Skeleton.resample(skel_opt) # Filter pelvis if args.with_filtering: out_filter_path = os.path.join(args.dir, os.pardir, "vis_filtering") skel_opt = filter_(skel_opt_resampled, out_filter_path=out_filter_path, skel_orig=skel_opt, weight_smooth=args.smooth, forwards_window_size=args.forwards_window_size) else: skel_opt.estimate_forwards(k=args.forwards_window_size) skel_opt_resampled.estimate_forwards(k=args.forwards_window_size) # if len(images): # skel_opt.fill_with_closest(images.keys()[0], images.keys()[-1]) min_y, max_y = skel_opt.get_min_y(tr_ground) print("min_y: %s, max_y: %s" % (min_y, max_y)) # # save # frame_ids_old = set(skel_opt.get_frames()) if args.resample: skel_opt = skel_opt_resampled frame_ids_filled_in.update( set(skel_opt.get_frames()).difference(frame_ids_old)) lg.warning("Saving resampled scenelet!") scenelet = Scenelet(skel_opt) del skel_opt # skel_dict = skel_opt.to_json() tr_ground[1, 3] = min_y scenelet.aux_info['ground'] = tr_ground.tolist() assert isinstance(ground_rot, list) and len(ground_rot) == 3 scenelet.add_aux_info('ground_rot', ground_rot) scenelet.add_aux_info( 'path_opt_params', { 'rate': args.r, 'w-smooth': args.smooth, 'winsorize-limit': args.winsorize_limit, 'camera': args.camera_name, 'huber': args.huber, 'height_correction': conf.optimize_path.height_correction, 'focal_correction': conf.optimize_path.focal_correction }) scenelet.add_aux_info('frame_ids_filled_in', list(frame_ids_filled_in)) # To MATLAB # _skeleton.get_min_y(_tr_ground) # with skel_opt as skeleton: # skeleton = skel_opt # skeleton_name = os.path.split(args.dir)[0] # skeleton_name = skeleton_name[skeleton_name.rfind('/')+1:] # mdict = skeleton.to_mdict(skeleton_name) # mdict['room_transform'] = tr_ground # mdict['room_transform'][1, 3] *= -1. # print(mdict) # print("scene_name?: %s" % os.path.split(args.dir)[0]) # skeleton.save_matlab( # os.path.join(os.path.dirname(args.dir), "skeleton_opt.mat"), # mdict=mdict) assert scenelet.skeleton.has_forwards(), "No forwards??" scenelet.save(out_path) if show: # save path plot out_path_path = os.path.join(args.dir, os.pardir, "%s_path.jpg" % name_video) path_fig = plot_path(scenelet.skeleton) legend = ["smooth %g" % args.smooth] # hack debug # path_skel2 = os.path.join(args.dir, os.pardir, 'skel_lobby7_nosmooth.json') # if os.path.exists(path_skel2): # skel2 = Skeleton.load(path_skel2) # path_fig = plot_path(skel2, path_fig) # legend.append('no smooth') if show: plt.legend(legend) path_fig.savefig(out_path_path) # backup args path_args = os.path.join(args.dir, os.pardir, 'args_denis.txt') with open(path_args, 'a') as f_args: f_args.write("%s %s\n" % (os.path.basename(sys.executable), " ".join(argv))) # save 2D detections to file if args.postfix == 'unannot': path_skel_ours_2d = os.path.join( args.dir, os.pardir, "skel_%s_2d_%02d.json" % (name_video, 0)) sclt_2d = Scenelet(skel_ours_2d_all) print('Saving {} to {}'.format(len(skel_ours_2d_all.get_frames()), path_skel_ours_2d)) sclt_2d.skeleton.aux_info = {} sclt_2d.save(path_skel_ours_2d) else: print(args.postfix) logging.info("Saving images...") if len(images) and len(out_images): path_out_images = os.path.join(args.dir, os.pardir, 'color') try: os.makedirs(path_out_images) except OSError: pass visible_f_max_log = np.log(visible_f_max) frames = list(out_images.keys()) for frame_id in range(frames[0], frames[-1] + 1): im = out_images[frame_id] if frame_id in out_images \ else cv2.cvtColor(images[frame_id], cv2.COLOR_BGR2RGB) for actor_id in range(args.n_actors): if frame_id in visible_f[actor_id]: frame_id2 = skel_ours_2d_all.unmod_frame_id( frame_id=frame_id, actor_id=actor_id, frames_mod=skel_ours_2d_all.frames_mod) for joint, is_vis in visible_f[actor_id][frame_id].items(): p2d = skel_ours_2d_all.get_joint_3d(joint, frame_id=frame_id2) # radius = np.log(is_vis) / visible_f_max_log # lg.debug("r0: %g" % radius) # radius = np.exp(np.log(is_vis) / visible_f_max_log) # lg.debug("radius is %g" % radius) vis_bool = True if skel_ours_2d_all.has_visible(frame_id=frame_id2, joint_id=joint): vis_bool &= skel_ours_2d_all.is_visible( frame_id2, joint) radius = abs(np.log(is_vis / 0.1 + 1e-6)) if not np.isnan(radius): p2d = (int(round(p2d[0])), int(round(p2d[1]))) cv2.circle(im, center=p2d, radius=int(round(radius)), color=(1., 1., 1., 0.5), thickness=1) conf = get_conf_thresholded(conf=is_vis, thresh_log_conf=None, dtype_np=np.float32) if conf > 0.5: cv2.putText(img=im, text=Joint(joint).get_name(), org=p2d, fontFace=1, fontScale=1, color=(10., 150., 10., 100.)) # lg.debug("set visibility to %g, radius %g" % (is_vis, radius)) # if frame_id in out_images: scale = (shape_orig[1] / float(im.shape[1]), shape_orig[0] / float(im.shape[0])) cv2.imwrite( os.path.join(path_out_images, "color_%05d.jpg" % frame_id), cv2.resize(im, (0, 0), fx=scale[0], fy=scale[1], interpolation=cv2.INTER_CUBIC)) # else: # fname = "color_%05d.jpg" % frame_id # shutil.copyfile( # os.path.join(path_images, fname), # os.path.join(path_out_images, fname)) lg.info("Wrote images to %s/" % path_out_images)
def main(argv=None): np.set_printoptions(suppress=True) parser = argparse.ArgumentParser() parser.add_argument('d', help="Folder of scene") parser.add_argument('-resolution', help='Target resolution for occupancy map', default=0.1) parser.add_argument( '-thresh-area', help='Ratio of occupancy map cell area that has to be occupied ' 'for it to count as occupied', default=0.1) parser.add_argument('-postfix', type=str, help="Scene postfix for augmentation", default="") args = parser.parse_args(argv if argv is not None else sys.argv) res_target = args.resolution if args.postfix and len(args.postfix) and not args.postfix.startswith('_'): args.postfix = "_%s" % args.postfix path_parent, name_input = os.path.split(os.path.abspath(args.d)) lg.warning("name input: %s" % name_input) path_for_tf = os.path.abspath( os.path.join(path_parent, os.pardir, 'dataset')) # if 'video' not in path_parent else os.path.join(path_parent, 'dataset') if not os.path.exists(path_for_tf): os.makedirs(path_for_tf, mode=0o0775) lg.debug("Loading scenelet...") path_scenelet = os.path.join(args.d, "skel_%s.json" % name_input) scenelet = Scenelet.load(path_scenelet) lg.debug("Scenelet: %s" % scenelet) path_state_pickle = os.path.join(args.d, "state%s.pickle" % args.postfix) if not os.path.exists(path_state_pickle): lg.error("Does not exist: %s" % path_state_pickle) return False # assert os.path.exists(path_state_pickle), \ # "Does not exist: %s" % path_state_pickle lg.debug("Loading volume...") state = pickle_load(open(path_state_pickle, 'rb')) lg.debug("Loaded volume...") lg.debug("Creating scene from scenelet") if not no_vis: vis = Visualizer(win_size=(1024, 1024)) vis.add_coords() else: vis = None # scene = Scene(scenelet.name_scenelet) # colors = {0: (200., 0., 0.), 1: (0., 200., 0.), 2: (0., 0., 200.)} # unit_x = np.array((1., 0., 0.)) occup = State(room=state.room, tr_ground_inv=None, res_theta=state.resolution[3], resolution=[res_target, res_target, res_target]) occup.get_volume(labels_to_lin_ids_arg=state.get_labels_to_lin_ids()) occup_angle = np.ones(shape=(len( occup.volume), occup.volume[0].shape[0], occup.volume[0].shape[1], 1), dtype=np.float32) * -1. assert np.min(occup_angle) < 0. and np.max(occup_angle) < 0., "Not empty" grid_polys = get_grid_shapely(occup=occup, res_orig=state.resolution) occup.volume.flags.writeable = True volume_occp = occup.volume angles = sorted(state.get_angles()) labels_to_lin_ids = occup.get_labels_to_lin_ids() had_vtk_problem = no_vis plt.figure() rects = [] for oid, ob in scenelet.objects.items(): assert oid >= 0, "Need positive here" label = ob.label if label in TRANSLATIONS_CATEGORIES: label = TRANSLATIONS_CATEGORIES[label] if label not in labels_to_lin_ids: continue try: poly = get_poly([part.obb for part in ob.parts.values()]) except ValueError as e: print("\n===========\n\nShapely error: %s for %s\n\n" % (e, (label, oid, ob))) with open('error.log', 'a') as f: f.write("[%s] %d, %s, %s\n" % (args.d, oid, label, ob)) continue ob_angle = ob.get_angle(positive_only=True) assert 0. <= ob_angle <= 2 * np.pi, "No: %g" % ob_angle rect = get_rectangle(poly, ob_angle) rect.extend([oid, CATEGORIES[label]]) rects.append(rect) cat_id = labels_to_lin_ids[label] # cat_id in volume, not categories for gp in grid_polys: # skip, if not occupied enough if gp.poly.intersection(poly).area / gp.area < args.thresh_area: continue # save occupancy gp.occupancy = 1. id_angle_lower = None id_angle_upper = None if ob_angle > angles[-1]: id_angle_lower = len(angles) - 1 id_angle_upper = 0 else: for id_angle, angle in enumerate(angles): if ob_angle < angle: id_angle_upper = id_angle id_angle_lower = id_angle - 1 break assert id_angle_lower is not None \ and id_angle_upper is not None, \ "Wrong?" assert id_angle_upper != id_angle_lower, \ "? %s %s" % (id_angle_lower, id_angle_upper) # cache xy = gp.xy # zero means empty in occupancy, # so object ids are shifted with 1 # we need object ids to filter "untouched" objects # in tfrecords_create if volume_occp[cat_id, xy[0], xy[1], id_angle_lower] == 0 \ or label in CATEGORIES_DOMINANT: volume_occp[cat_id, xy[0], xy[1], id_angle_lower] = oid + 1 if volume_occp[cat_id, xy[0], xy[1], id_angle_upper] == 0 \ or label in CATEGORIES_DOMINANT: volume_occp[cat_id, xy[0], xy[1], id_angle_upper] = oid + 1 # angles are right now not per-category, but per-scene # hence, an object can only overwrite, if it's usually "above" # other objects, e.g. a table # this is a hack for a z-test if occup_angle[cat_id, xy[0], xy[1], 0] < 0. \ or label in CATEGORIES_DOMINANT: occup_angle[cat_id, xy[0], xy[1], 0] = ob_angle if not had_vtk_problem: color = COLORS_CATEGORIES[label] if label in COLORS_CATEGORIES \ else (200., 200., 200.) try: for id_part, part in ob.parts.items(): vis.add_mesh(MeshOBJ.from_obb(part.obb), name="ob_%02d_part_%02d" % (oid, id_part), color=color) except AttributeError: print("VTK problem...") had_vtk_problem = True #plt.savefig() plt.close() if not had_vtk_problem: vis.set_camera_pos(pos=(0., -1., 0.)) vis.camera().SetFocalPoint(0., 0., 0.) vis.camera().SetViewUp(-1., 0., 0.) vis.set_camera_type(is_ortho=True) vis.camera().SetParallelScale(3.) # vis.show() name_recording = "%s_%s" % (os.path.basename(args.d), args.postfix) \ if args.postfix else os.path.basename(args.d) lg.info("name_recording: %s" % name_recording) path_out_occp = os.path.join(os.path.dirname(args.d), os.pardir, 'occupancy', name_recording) if not os.path.exists(path_out_occp): os.makedirs(path_out_occp) # prepare www storage www_grid = {'evidence': {}, 'occ': {}} # normalize evidence maps vmax = 0. ims = {} for cat, cat_id in labels_to_lin_ids.items(): ims[cat] = np.squeeze( np.sum(state.volume[cat_id, :, :, :], axis=2, keepdims=True)) vmax = max(vmax, np.max(ims[cat])) # gather joined occupancy map im_sum = None # for each evidence category for cat, cat_id in labels_to_lin_ids.items(): im = ims[cat] / vmax * 255. path_out_im = os.path.join(path_out_occp, "e_%s.jpg" % cat) cv2.imwrite(path_out_im, im) # lg.debug("wrote to %s" % path_out_im) www_grid['evidence'][cat] = path_out_im im = np.squeeze(volume_occp[cat_id, :, :, 0]) path_out_im = os.path.join(path_out_occp, "o_%s.jpg" % cat) cv2.imwrite(path_out_im, im * 255.) # lg.debug("wrote to %s" % path_out_im) www_grid['occ'][cat] = path_out_im if im_sum is None: im_sum = im.copy() else: im_sum = np.maximum(im, im_sum) # # save dataset # name_input_old = name_input if args.postfix is not None and len(args.postfix): name_input = "%s_%s" % (name_input, args.postfix) # state path_state_dest = os.path.join(path_for_tf, "state_%s.pickle" % name_input) shutil.copyfile(path_state_pickle, path_state_dest) lg.info("Copied\n\t%s to\n\t%s" % (path_state_pickle, path_state_dest)) # occupancy path_occup_dest = os.path.join(path_for_tf, "occup_%s.pickle" % name_input) pickle.dump(occup, open(path_occup_dest, 'wb'), -1) lg.info("Wrote to %s" % path_occup_dest) # occupancy_angle path_occup_angle_dest = os.path.join(path_for_tf, "angle_%s.npy" % name_input) min_angle = np.min(occup_angle) assert min_angle < 0., "No empty cells??" lg.debug("min angle is %s" % min_angle) np.save(open(path_occup_angle_dest, 'wb'), occup_angle) lg.info("Wrote to %s" % path_occup_angle_dest) # skeleton path_copied = shutil.copy2(path_scenelet, path_for_tf) lg.info("Copied\n\t%s to \n\t%s" % (path_scenelet, path_copied)) # charness skeleton name_skeleton_charness = "skel_%s-charness.json" % name_input_old path_scenelet_charness = os.path.join(args.d, name_skeleton_charness) assert os.path.exists(path_scenelet_charness), \ "Does not exist: %s" % path_scenelet_charness shutil.copy2(path_scenelet_charness, path_for_tf) assert os.path.exists(os.path.join(path_for_tf, name_skeleton_charness)), \ "Does not exist: %s" % os.path.join(path_for_tf, name_skeleton_charness) # rectangles name_rectangles = "rectangles_%s.npy" % name_input_old path_rectangles = os.path.join(path_for_tf, name_rectangles) np.save(open(path_rectangles, 'wb'), rects) # # visualize # path_out_im = os.path.join(path_out_occp, '3d.png') if not had_vtk_problem: vis.save_png(path_out_im) www_grid['3d'] = path_out_im path_out_im = os.path.join(path_out_occp, 'o_sum.png') max_im_sum = np.max(im_sum) if max_im_sum > 0.: cv2.imwrite(path_out_im, im_sum / max_im_sum * 255.) else: cv2.imwrite(path_out_im, im_sum * 255.) www_grid['o_sum'] = path_out_im path_www = os.path.join(path_out_occp, os.pardir) with open(os.path.join(path_www, 'index.html'), 'a') as f: f.write("<style> img {image-rendering: pixelated; } </style>\n") f.write("<script>\n") f.write("</script>\n") f.write("<h3>%s</h3>" % os.path.basename(args.d)) f.write('<table>\n') f.write("<tr>\n") f.write("<th>3d</th>") f.write("<th>Occupancy sum</th>") for cat in www_grid['evidence']: f.write("\t<th>%s</th>\n" % cat) f.write("<th></th>\n") # titles f.write("</tr>\n") f.write("<tr>\n") # 3D f.write("\t<td rowspan=\"2\">\n") path_im = os.path.relpath(www_grid['3d'], path_www) f.write("\t<a href=\"%s\">\n" "\t\t<img src=\"%s\" height=\"400\" />\n" "\t</a>\n" % (path_im, path_im)) # Evidence sum f.write("\t<td rowspan=\"2\">\n") path_im = os.path.relpath(www_grid['o_sum'], path_www) f.write("\t<a href=\"%s\">\n" "\t\t<img src=\"%s\" height=\"400\" />\n" "\t</a>\n" % (path_im, path_im)) # Evidence for cat in www_grid['evidence']: f.write("<td style=\"padding-bottom: 2px\">\n") path_im = os.path.relpath(www_grid['evidence'][cat], path_www) f.write("\t<a href=\"%s\">\n" "\t\t<img src=\"%s\" height=\"200\" />\n" "\t</a>\n" % (path_im, path_im)) f.write("</td>\n") f.write("<td>Evidence</td>\n") f.write("\t</td>\n") f.write("</tr>\n") f.write("<tr>\n") for cat in www_grid['occ']: f.write("<td>\n") path_im = os.path.relpath(www_grid['occ'][cat], path_www) f.write("\t<a href=\"%s\">\n" "\t\t<img src=\"%s\" height=\"200\" />\n" "</a>\n" % (path_im, path_im)) f.write("</td>\n") f.write("<td>Occupancy map</td>\n") f.write("</tr>") f.write('</table>') return True
def main(argv): pjoin = os.path.join # cache long name parser = argparse.ArgumentParser( "Find characteristic scene times", description="Scans a directory of short scenelets (exported from " "Matlab), and looks up their full version in the original " "scenes. It exports a new scenelet containing all poses " "between the start and end times of the input short " "scenelets. Matching is done by name." "Scenelets below time length limit and not enough objects " "are thrown away." "It also saves the scenelet characteristicness into the " "output scenelet files.") parser.add_argument( 'd', type=argparse_check_exists, help="Folder containing PiGraphs scenelets. E.g. " "/mnt/thorin_data/stealth/shared/" "pigraph_scenelets__linterval_squarehist_large_radiusx2") parser.add_argument('s', type=argparse_check_exists, help="Folder containing PiGraphs full scenes. E.g. " "/mnt/thorin_data/stealth/shared/scenes_pigraphs") parser.add_argument('-l', '--limit-len', type=int, help="Minimum length for a scenelet", default=10) # changed from `5` on 15/1/2018 parser.add_argument( '--dist-thresh', type=float, help='Distance threshold for object pruning. Typically: 0.2 or 0.5.', default=.5) # parse arguments args = parser.parse_args(argv) parts_to_remove = ['sidetable'] lg.warning("Will remove all parts named %s" % parts_to_remove) # read scenes and scenelets p_pickle = pjoin(args.d, 'scenes_and_scenelets.pickle') if os.path.exists(p_pickle): lg.info("reading from %s" % p_pickle) scenes, scenelets = pickle_load(open(p_pickle, 'rb')) lg.info("read from %s" % p_pickle) else: scenelets = read_scenelets(args.d) scenes = read_scenelets(args.s) scenes = {scene.name_scene: scene for scene in scenes} pickle.dump((scenes, scenelets), open(p_pickle, 'wb'), protocol=-1) lg.info("wrote to %s" % p_pickle) # Read characteristicnesses (to put them into the scenelet). p_charness = pjoin(args.d, "charness__gaussian.mat") pose_charness, scenelet_names = read_charness(p_charness, return_hists=False, return_names=True) # output folder d_scenelets_parent = os.path.dirname(args.d) d_dest = pjoin(d_scenelets_parent, 'deb', "%s_full_sampling" % args.d.split(os.sep)[-1]) # makedirs_backed if os.path.exists(d_dest): i = 0 while i < 100: try: os.rename(d_dest, "%s.bak.%02d" % (d_dest, i)) break except OSError: i += 1 os.makedirs(d_dest) # _is_close = is_close # cache namespace lookup # processing for sclt in scenelets: # cache skeleton skeleton = sclt.skeleton if 'scene09' in sclt.name_scenelet or 'scene10' in sclt.name_scenelet: lg.debug("here") else: continue # prune objects per_cat = {} cnt = 0 for oid, scene_obj in sclt.objects.items(): close_, dist = is_close(scene_obj, skeleton, args.dist_thresh, return_dist=True) label = scene_obj.label if 'chair' in label or 'couch' in label or 'stool' in label: label = 'sittable' try: per_cat[label].append((dist, oid)) except KeyError: per_cat[label] = [(dist, oid)] if scene_obj.label != 'floor': cnt += 1 per_cat = {k: sorted(v) for k, v in per_cat.items()} name_scene = sclt.name_scene.split('__')[0] if '-no-coffeetable' in name_scene: name_scene = name_scene[:name_scene.find('-no-coffeetable')] scene = scenes[name_scene] if 'shelf' not in per_cat: for oid, ob in scene.objects.items(): if ob.label == 'shelf': close_, dist = is_close(ob, skeleton, args.dist_thresh, return_dist=True) oid_ = oid while oid_ in sclt.objects: oid_ += 1 sclt.add_object(oid_, ob) cnt += 1 try: per_cat['shelf'].append((dist, oid_)) except KeyError: per_cat['shelf'] = [(dist, oid_)] if 'shelf' in per_cat: assert len(per_cat['shelf']) == 1, "TODO: keep all shelves" oids_to_keep = [ v[0][1] for v in per_cat.values() if v[0][0] < args.dist_thresh ] if not len(oids_to_keep): # there is always a floor lg.warning("Skipping %s, not enough objects: %s" % (sclt.name_scenelet, per_cat)) continue # if 'gates392_mati3_2014-04-30-21-13-46__scenelet_25' \ # == sclt.name_scenelet: # lg.debug("here") # else: # continue # copy skeleton with dense sampling in time mn, mx = skeleton.get_frames_min_max() # assert mn == 0, "This usually starts indexing from 0, " \ # "no explicit problem, just flagging the change." time_mn = floor(skeleton.get_time(mn)) time_mx = ceil(skeleton.get_time(mx)) # artificially prolong mocap scenes if 'scene' in name_scene and (time_mx - time_mn < 60): d = (time_mx - time_mn) // 2 + 1 time_mn -= d time_mx += d # lookup original scene name # mn_frame_id_scene, mx_frame_id_scene = \ # scene.skeleton.get_frames_min_max() frame_ids_old = skeleton.get_frames() times_old = [skeleton.get_time(fid) for fid in frame_ids_old] for frame_id in frame_ids_old: skeleton.remove_pose(frame_id) for frame_id in range(time_mn, time_mx + 1): if not scene.skeleton.has_pose(frame_id): continue pose = scene.skeleton.get_pose(frame_id=frame_id) # scale mocap skeletons fw = scene.skeleton.get_forward(frame_id=frame_id, estimate_ok=False) sclt.set_pose(frame_id=frame_id, angles=None, pose=pose, forward=fw, clone_forward=True) if 'scene0' in name_scene or 'scene10' in name_scene: mx_old = np.max(sclt.skeleton.poses[:, 1, :]) sclt.skeleton.poses *= 0.8 mx_new = np.max(sclt.skeleton.poses[:, 1, :]) sclt.skeleton.poses[1, :] += mx_new - mx_old + 0.05 _frames = sclt.skeleton.get_frames() # check length if len(_frames) < args.limit_len: lg.warning("Skipping %s, because not enough frames: %s" % (sclt.name_scene, _frames)) continue # save charness try: id_charness = next(i for i in range(len(scenelet_names)) if scenelet_names[i] == sclt.name_scenelet) sclt.charness = pose_charness[id_charness] except StopIteration: lg.error("Something is wrong, can't find %s, %s in charness db " "containing names such as %s." % (sclt.name_scene, sclt.name_scenelet, scenelet_names[0])) sclt.charness = 0.4111111 _mn, _mx = (_frames[0], _frames[-1]) assert _mn >= time_mn, "not inside? %s < %s" % (_mn, time_mn) assert _mx <= time_mx, "not inside? %s < %s" % (_mx, time_mx) if len(_frames) < len(frame_ids_old): lg.warning("Not more frames than interpolated " "scenelet?\n%s\n%s\n%s" % (_frames, frame_ids_old, times_old)) oids = list(sclt.objects.keys()) for oid in oids: if oid not in oids_to_keep: lg.debug("removed %s" % sclt.objects[oid]) sclt.objects.pop(oid) else: obj = sclt.objects[oid] part_ids_to_remove = [ part_id for part_id, part in obj.parts.items() if part.label in parts_to_remove ] if len(part_ids_to_remove) == len(obj.parts): sclt.objects.pop(oid) else: for part_id in part_ids_to_remove: lg.debug("removed %s" % sclt.objects[obj].parts[part_id]) obj.parts.pop(part_id) if len(sclt.objects) < 2 and next(iter( sclt.objects.values())).label == 'floor': lg.debug("finally removing scenelet: %s" % sclt.objects) continue # save in the scene folder d_dest_scene = pjoin(d_dest, name_scene) if not os.path.exists(d_dest_scene): os.makedirs(d_dest_scene) sclt.save(pjoin(d_dest_scene, "skel_%s" % sclt.name_scenelet))