def create_dataset(videos_list=None, savepath=joints_path(), im_regularizer=reg.Regularizer(), heat_regularizer=reg.Regularizer(), fillgaps=False, cross_radius=3, enlarge=0.2, shade=False): """reads the videos specified as parameter and for each frame produces and saves a .mat file containing the frame, the corresponding heatmap indicating the position of the hand and the modified depth. :param fillgaps: set to True to also get interpolated frames :param im_regularizer: object used to regularize the images :param heat_regularizer: object used to regularize the heatmaps :param savepath: path of the folder where the produces .mat files will be saved. If left to the default value None, the /resources/hands_bounding_dataset/hands_rgbd_transformed folder will be used :param videos_list: list of videos you need the .mat files of. If left to the default value None, all videos will be exploited :param cross_radius: radius of the crosses of the heatmaps :param enlarge: crops enlarge factor :param shade: set to true to shade the pixels that identify a junction in a heatmap according to their distance with the center (real position of junction""" if savepath is None: basedir = joints_path() else: basedir = savepath if not os.path.exists(basedir): os.makedirs(basedir) framesdir = resources_path("framedata") if videos_list is None: vids = os.listdir(framesdir) vids = [x for x in vids if os.path.isdir(os.path.join(framesdir, x))] else: vids = videos_list for vid in tqdm.tqdm(vids): frames, labels = load_labelled_videos(vid, fillgaps=fillgaps) # depths, _ = load_labelled_videos(vid, getdepth=True, fillgaps=fillgaps) fr_num = frames.shape[0] for i in tqdm.tqdm(range(0, fr_num)): if labels[i] is not None: try: frame = frames[i] label = labels[i][:, 0:2] visible = labels[i][:, 2:3] label *= [frame.shape[1], frame.shape[0]] label = np.array(label, dtype=np.int32).tolist() label = [[p[1], p[0]] for p in label] coords = __get_coord_from_labels(label) cut = u.crop_from_coords(frame, coords, enlarge) heatmaps = __create_21_heatmaps(label, coords, np.shape(frame), cross_radius, enlarge, shade) cut = im_regularizer.apply(cut) heatmaps = heat_regularizer.apply_on_batch(heatmaps) # heatmaps = __heatmaps_dim_reducer(heatmaps) heatmaps = __stack_heatmaps(heatmaps) path = os.path.join(basedir, vid + "_" + str(i)) __persist_frame(path, cut, heatmaps, visible) except ValueError as e: print("Error " + e + " on vid " + vid + str(i))
def create_joint_dataset(): img_reg = regularizer.Regularizer() img_reg.fixresize(200, 200) hm_reg = regularizer.Regularizer() hm_reg.fixresize(100, 100) hm_reg.heatmaps_threshold(.5) jointscreate(savepath=joints_path(), fillgaps=False, im_regularizer=img_reg, heat_regularizer=hm_reg, enlarge=.5, cross_radius=5)
def create_dataset_shaded_heatmaps_synth(dspath=None, savepath=jsonhands_path(), heigth_shrink_rate=4, width_shrink_rate=4, enlarge_heat=0.3, im_reg=reg.Regularizer(), he_r=reg.Regularizer(), resize_rate=1.0): if dspath is None: dspath = resources_path("jsonHands") if not os.path.exists(savepath): os.makedirs(savepath) dspath = resources_path(dspath) labels_dir = os.path.join(dspath, "labels") framesdir = os.path.join(dspath, "images") frameslist = os.listdir(framesdir) final_base_path = resources_path(savepath) for frame in tqdm.tqdm(frameslist): try: fr_to_save = {} f_pruned_name = __remove_ext(frame) frame_read = __read_frame(os.path.join(framesdir, frame)) res_h = frame_read.shape[0] / 480 res_w = frame_read.shape[1] / 640 frame_read = imresize(frame_read, (480, 640)) label_l = __read_label(os.path.join(labels_dir, f_pruned_name + ".json")) frame_read = imresize(frame_read, resize_rate) label_l = [[p[1] * resize_rate / res_h, p[0] * resize_rate / res_w] for p in label_l] frame_l = __add_padding(frame_read, frame_read.shape[1] - (frame_read.shape[1] // width_shrink_rate) * width_shrink_rate, frame_read.shape[0] - ( frame_read.shape[0] // heigth_shrink_rate) * heigth_shrink_rate) frame_l = im_reg.apply(frame_l) fr_to_save['frame'] = frame_l coords_l = [__get_coord_from_labels(label_l)] heat1 = np.zeros([frame_l.shape[0] // heigth_shrink_rate, frame_l.shape[1] // width_shrink_rate]) coords_l = coords_l[0] res_coords_l = [[l[0] // heigth_shrink_rate, l[1] // width_shrink_rate] for l in coords_l] res_coords_l = __enlarge_coords(res_coords_l, enlarge_heat, np.shape(heat1)) res_labels_l = [[l[0] // heigth_shrink_rate, l[1] // width_shrink_rate] for l in label_l] heat1 = __shade_heatmap(heat1, res_coords_l, res_labels_l) heat = heat1 heat[heat > 1] = 1 heat = he_r.apply(heat) heat = __heatmap_to_uint8(heat) fr_to_save['heatmap'] = heat path = os.path.join(final_base_path, f_pruned_name + ".mat") scio.savemat(path, fr_to_save) except Exception: print(frame)
def create_dataset(videos_list=None, savepath=None, im_regularizer=reg.Regularizer(), fillgaps=False, enlarge=0.5): """reads the videos specified as parameter and for each frame produces and saves a .mat file containing the frame, the corresponding heatmap indicating the position of the hand and the modified depth. :param fillgaps: set to True to also get interpolated frames :param im_regularizer: object used to regularize the cuts :param savepath: path of the folder where the produces .mat files will be saved. If left to the default value None, the /resources/hands_bounding_dataset/hands_rgbd_transformed folder will be used :param videos_list: list of videos you need the .mat files of. If left to the default value None, all videos will be exploited :param enlarge: crops enlarge factor""" if savepath is None: basedir = resources_path("palm_back_classification_dataset") else: basedir = savepath if not os.path.exists(basedir): os.makedirs(basedir) framesdir = resources_path("framedata") if videos_list is None: vids = os.listdir(framesdir) vids = [x for x in vids if os.path.isdir(os.path.join(framesdir, x))] else: vids = videos_list for vid in tqdm.tqdm(vids): frames, labels = load_labelled_videos(vid, fillgaps=fillgaps) fr_num = frames.shape[0] lr = get_right_left(vid) if lr == LEFT: lr = -1 for i in tqdm.tqdm(range(0, fr_num)): if labels[i] is not None: try: frame = frames[i] label = labels[i][:, 0:2] # conf is a real in [-1.0, 1.0] such that -1.0 is full back, +1.0 is full palm # middle values express partial confidence, but all info is in one single value conf = pb.leftright_to_palmback( hand=hand_format(label), side=pb.RIGHT if lr == RIGHT else pb.LEFT) # if you want the crisp result, there it is: result = PALM if conf >= 0 else BACK if result < 0: result = 0 # and the confidence on that result is in [0..1]: conf = abs(conf) label *= [frame.shape[1], frame.shape[0]] label = np.array(label, dtype=np.int32).tolist() label = [[p[1], p[0]] for p in label] coords = __get_coord_from_labels(label) cut = u.crop_from_coords(frame, coords, enlarge) cut = im_regularizer.apply(cut) path = os.path.join(basedir, vid + "_" + str(i)) __persist_frame(path, cut, result, conf) except ValueError as e: print("Error " + str(e) + " on vid " + vid + str(i))
def create_dataset(videos_list=None, savepath=None, im_regularizer=reg.Regularizer(), fillgaps=False, enlarge=0.5, data_augment=False): """reads the videos specified as parameter and for each frame produces and saves a .mat file containing the frame, the corresponding heatmap indicating the position of the hand and the modified depth. :param fillgaps: set to True to also get interpolated frames :param im_regularizer: object used to regularize the cuts :param savepath: path of the folder where the produces .mat files will be saved. If left to the default value None, the /resources/hands_bounding_dataset/hands_rgbd_transformed folder will be used :param videos_list: list of videos you need the .mat files of. If left to the default value None, all videos will be exploited :param enlarge: crops enlarge factor""" if savepath is None: basedir = resources_path("left_right_classification_dataset") else: basedir = savepath if not os.path.exists(basedir): os.makedirs(basedir) framesdir = resources_path("framedata") if videos_list is None: vids = os.listdir(framesdir) vids = [x for x in vids if os.path.isdir(os.path.join(framesdir, x))] else: vids = videos_list for vid in tqdm.tqdm(vids): frames, labels = load_labelled_videos(vid, fillgaps=fillgaps) fr_num = frames.shape[0] result = get_right_left(vid) for i in tqdm.tqdm(range(0, fr_num)): if labels[i] is not None: try: frame = frames[i] label = labels[i][:, 0:2] label *= [frame.shape[1], frame.shape[0]] label = np.array(label, dtype=np.int32).tolist() label = [[p[1], p[0]] for p in label] coords = __get_coord_from_labels(label) cut = u.crop_from_coords(frame, coords, enlarge) cut = im_regularizer.apply(cut) path = os.path.join(basedir, vid + "_" + str(i)) __persist_frame(path, cut, result) if data_augment: path = os.path.join(basedir, vid + "_t_" + str(i)) transp = cut.squeeze().transpose() transp = transp.reshape( [transp.shape[0], transp.shape[1], 1]) __persist_frame(path, transp, 1 - result) except ValueError as e: print("Error " + str(e) + " on vid " + vid + str(i))
fr_to_save = { 'cut': cut, 'heatmap_array': np.array(heatmaps * 255, dtype=np.uint8), 'visible': np.array(visible_flags, dtype=np.float32) } scio.savemat(path, fr_to_save) def __read_frame(path): matcontent = scio.loadmat(path) return matcontent['cut'], matcontent['heatmap_array'] / 255.0, matcontent[ 'visible'] if __name__ == '__main__': im_r = reg.Regularizer() im_r.fixresize(200, 200) im_r.percresize(0.5) h_r = reg.Regularizer() h_r.fixresize(200, 200) h_r.percresize(0.5) create_dataset(im_regularizer=im_r, heat_regularizer=h_r, enlarge=0.5, cross_radius=10, shade=True) c, h, v = read_dataset_random(number=2) u.showimage(c[1]) print(np.shape(h)) # show heatmap for the first junction of the second item u.showimage(u.heatmap_to_rgb(h[1][:, :, 0:1]))
def create_weights(gt): outs = [] for i in range(len(gt)): try: outs = outs + list(np.array(gt[i]['OUT_0000']).squeeze()) except Exception: continue outs = np.array(outs).flatten() weights = count_ones_zeros(outs, outs) return weights if __name__ == '__main__': # NO NEED TO TOUCH ANYTHING AFTER THIS, IT'S DELICATE regularizer = reg.Regularizer() regularizer.fixresize(200, 200) if createdataset: create_dataset(savepath=path, im_regularizer=regularizer) formatting = confidence_filtered_pb_format(minconf) generator = DatasetManager(train_samples=train_samples, valid_samples=valid_samples, batch_size=batch_size, dataset_dir=path, formatting=formatting) gt = generator.train() cw = create_weights(gt) data_processing_plan = ProcessingPlan()
n = len(ims) ris = [] for i in range(n): prod = prod_im_heat(ims[i], heats[i]) ris.append(prod) return ris if __name__ == '__main__': LOAD_MODEL = False TRAIN_MODEL = True regularizer = reg.Regularizer() regularizer.fixresize(200, 200) regularizer_h = reg.Regularizer() regularizer_h.fixresize(200, 200) #create_dataset_w_heatmaps(savepath=path, im_regularizer=regularizer, h_r=regularizer_h) x_train, y_train, c_train, h_train, x_test, y_test, c_test, h_test = read_dataset_h(path=path, leave_out=['handsMaddalena2', 'handsGianpy', 'handsMatteo']) # x_train = prod_im_heat_batch(x_train, h_train) # x_test = prod_im_heat_batch(x_test, h_test) x_train = np.array(x_train) y_train = np.array(y_train) c_train = np.array(c_train)
def create_dataset(videos_list=None, savepath=None, resize_rate=1.0, heigth_shrink_rate=10, width_shrink_rate=10, im_reg=reg.Regularizer(), heat_reg=reg.Regularizer()): """reads the videos specified as parameter and for each frame produces and saves a .mat file containing the frame, the corresponding heatmap indicating the position of the hand(s) THE PROCESS: - image is resized - heatmap is produced with dimensions resized w.r.t. the resized image - regularizers are applied (BE CAREFUL WITH REGULARIZERS, YOU MAY ALTER DIMENSION RATIOS BETWEEN IMAGES AND HEATMAPS) :param width_shrink_rate: shrink rate of heatmaps width wrt the resized image :param heigth_shrink_rate: shrink rate of heatmaps height wrt the resized image :param resize_rate: resize rate of the images (1 (default) is the original image) :param savepath: path of the folder where the produces .mat files will be saved. If left to the default value None, the /resources/hands_bounding_dataset/egohands_transformed folder will be used :param videos_list: list of videos you need the .mat files of. If left to the default value None, all videos will be exploited :param im_reg: object used to regularize the images :param heat_reg: object used to regularize the heatmaps""" # approximation_ratio = resize_rate / max(heigth_shrink_rate, width_shrink_rate) approximation_ratio = 0.05 if savepath is None: basedir = resources_path( os.path.join("hands_bounding_dataset", "egohands_tranformed")) else: basedir = savepath if not os.path.exists(basedir): os.makedirs(basedir) framesdir = resources_path( os.path.join("hands_bounding_dataset", "egohands")) if videos_list is None: vids = os.listdir(framesdir) vids = [x for x in vids if os.path.isdir(os.path.join(framesdir, x))] else: vids = videos_list for vid in tqdm(vids): frames, labels = __load_egohand_video( os.path.join(framesdir, vid), one_out_of=int(min(width_shrink_rate, heigth_shrink_rate)) / resize_rate) fr_num = len(frames) for i in tqdm(range(0, fr_num)): fr_to_save = {} frame = frames[i] frame = imresize(frame, [480, 640]) frame = imresize(frame, resize_rate) frame = __add_padding( frame, frame.shape[1] - (frame.shape[1] // width_shrink_rate) * width_shrink_rate, frame.shape[0] - (frame.shape[0] // heigth_shrink_rate) * heigth_shrink_rate) heat = __create_ego_heatmap(frame, labels[i], heigth_shrink_rate, width_shrink_rate, resize_rate, approximation_ratio) frame = im_reg.apply(frame) heat = heat_reg.apply(heat) fr_to_save['frame'] = frame fr_to_save['heatmap'] = __heatmap_to_uint8(heat) path = os.path.join(basedir, vid + "_" + str(i)) scio.savemat(path, fr_to_save)
def create_palmback_dataset(): img_reg = regularizer.Regularizer() img_reg.fixresize(200, 200) pbcreate(savepath=palmback_path(), im_regularizer=img_reg)
def create_dataset_shaded_heatmaps(videos_list=None, savepath=crops_path(), resize_rate=1.0, heigth_shrink_rate=10, width_shrink_rate=10, overlapping_penalty=0.9, fillgaps=False, toofar=1500, tooclose=500, enlarge_heat=0.3, im_reg=reg.Regularizer(), he_r=reg.Regularizer()): """reads the videos specified as parameter and for each frame produces and saves a .mat file containing the frame, the corresponding heatmap indicating the position of the hand and the modified depth. :param tooclose: threshold value used to eliminate too close objects/values in the depth :param toofar: threshold value used to eliminate too far objects/values in the depth :param fillgaps: set to True to also get interpolated frames :param overlapping_penalty: penalty "inflicted" to the overlapping hands area in the images :param width_shrink_rate: shrink rate of heatmaps width wrt the resized image :param heigth_shrink_rate: shrink rate of heatmaps height wrt the resized image :param resize_rate: resize rate of the images (1 (default) is the original image) :param savepath: path of the folder where the produces .mat files will be saved. If left to the default value None, the /resources/hands_bounding_dataset/hands_rgbd_transformed folder will be used :param videos_list: list of videos you need the .mat files of. If left to the default value None, all videos will be exploited""" if savepath is None: basedir = crops_path() else: basedir = savepath if not os.path.exists(basedir): os.makedirs(basedir) framesdir = resources_path("framedata") if videos_list is None: vids = os.listdir(framesdir) vids = [x for x in vids if os.path.isdir(os.path.join(framesdir, x))] else: vids = videos_list for vid in tqdm.tqdm(vids): frames, labels = load_labelled_videos(vid, fillgaps=fillgaps) if labels is None: continue depths, _ = load_labelled_videos(vid, getdepth=True, fillgaps=fillgaps) fr_num = frames.shape[0] for i in tqdm.tqdm(range(0, fr_num)): if labels[i] is not None: try: fr_to_save = {} frame = frames[i] depth = depths[i] frame, depth = transorm_rgd_depth(frame, depth, toofar=toofar, tooclose=tooclose) frame = imresize(frame, resize_rate) depth = depth_resize(depth, resize_rate) label = labels[i][:, 0:2] label *= [frame.shape[1], frame.shape[0]] label = np.array(label, dtype=np.int32).tolist() label = [[p[1], p[0]] for p in label] frame = __add_padding( frame, frame.shape[1] - (frame.shape[1] // width_shrink_rate) * width_shrink_rate, frame.shape[0] - (frame.shape[0] // heigth_shrink_rate) * heigth_shrink_rate) depth = __add_padding( depth, depth.shape[1] - (depth.shape[1] // width_shrink_rate) * width_shrink_rate, depth.shape[0] - (depth.shape[0] // heigth_shrink_rate) * heigth_shrink_rate) depth = depth.squeeze() depth = np.uint8(depth) frame = im_reg.apply(frame) fr_to_save['frame'] = frame coords = [__get_coord_from_labels(label)] heat = u.get_heatmap_from_coords(frame, heigth_shrink_rate, width_shrink_rate, coords, overlapping_penalty) coords = coords[0] res_coords = [[ l[0] // heigth_shrink_rate, l[1] // width_shrink_rate ] for l in coords] res_coords = __enlarge_coords(res_coords, enlarge_heat, np.shape(heat)) res_labels = [[ l[0] // heigth_shrink_rate, l[1] // width_shrink_rate ] for l in label] heat = __shade_heatmap(heat, res_coords, res_labels) heat = he_r.apply(heat) heat = __heatmap_to_uint8(heat) fr_to_save['heatmap'] = heat depth = he_r.apply(depth) fr_to_save['depth'] = depth path = os.path.join(basedir, vid + "_" + str(i)) scio.savemat(path, fr_to_save) except ValueError as e: print(vid + str(i) + " => " + e)