def create_dataset(videos_list=None,
savepath=joints_path(),
im_regularizer=reg.Regularizer(),
heat_regularizer=reg.Regularizer(),
fillgaps=False,
cross_radius=3,
enlarge=0.2,
shade=False):
"""reads the videos specified as parameter and for each frame produces and saves a .mat file containing
the frame, the corresponding heatmap indicating the position of the hand and the modified depth.
:param fillgaps: set to True to also get interpolated frames
:param im_regularizer: object used to regularize the images
:param heat_regularizer: object used to regularize the heatmaps
:param savepath: path of the folder where the produces .mat files will be saved. If left to the default value None,
the /resources/hands_bounding_dataset/hands_rgbd_transformed folder will be used
:param videos_list: list of videos you need the .mat files of. If left to the default value None, all videos will
be exploited
:param cross_radius: radius of the crosses of the heatmaps
:param enlarge: crops enlarge factor
:param shade: set to true to shade the pixels that identify a junction in a heatmap according to their
distance with the center (real position of junction"""
if savepath is None:
basedir = joints_path()
else:
basedir = savepath
if not os.path.exists(basedir):
os.makedirs(basedir)
framesdir = resources_path("framedata")
if videos_list is None:
vids = os.listdir(framesdir)
vids = [x for x in vids if os.path.isdir(os.path.join(framesdir, x))]
else:
vids = videos_list
for vid in tqdm.tqdm(vids):
frames, labels = load_labelled_videos(vid, fillgaps=fillgaps)
# depths, _ = load_labelled_videos(vid, getdepth=True, fillgaps=fillgaps)
fr_num = frames.shape[0]
for i in tqdm.tqdm(range(0, fr_num)):
if labels[i] is not None:
try:
frame = frames[i]
label = labels[i][:, 0:2]
visible = labels[i][:, 2:3]
label *= [frame.shape[1], frame.shape[0]]
label = np.array(label, dtype=np.int32).tolist()
label = [[p[1], p[0]] for p in label]
coords = __get_coord_from_labels(label)
cut = u.crop_from_coords(frame, coords, enlarge)
heatmaps = __create_21_heatmaps(label, coords,
np.shape(frame),
cross_radius, enlarge,
shade)
cut = im_regularizer.apply(cut)
heatmaps = heat_regularizer.apply_on_batch(heatmaps)
# heatmaps = __heatmaps_dim_reducer(heatmaps)
heatmaps = __stack_heatmaps(heatmaps)
path = os.path.join(basedir, vid + "_" + str(i))
__persist_frame(path, cut, heatmaps, visible)
except ValueError as e:
print("Error " + e + " on vid " + vid + str(i))
def create_joint_dataset():
img_reg = regularizer.Regularizer()
img_reg.fixresize(200, 200)
hm_reg = regularizer.Regularizer()
hm_reg.fixresize(100, 100)
hm_reg.heatmaps_threshold(.5)
jointscreate(savepath=joints_path(),
fillgaps=False,
im_regularizer=img_reg,
heat_regularizer=hm_reg,
enlarge=.5,
cross_radius=5)
def create_dataset_shaded_heatmaps_synth(dspath=None, savepath=jsonhands_path(), heigth_shrink_rate=4,
width_shrink_rate=4,
enlarge_heat=0.3, im_reg=reg.Regularizer(), he_r=reg.Regularizer(),
resize_rate=1.0):
if dspath is None:
dspath = resources_path("jsonHands")
if not os.path.exists(savepath):
os.makedirs(savepath)
dspath = resources_path(dspath)
labels_dir = os.path.join(dspath, "labels")
framesdir = os.path.join(dspath, "images")
frameslist = os.listdir(framesdir)
final_base_path = resources_path(savepath)
for frame in tqdm.tqdm(frameslist):
try:
fr_to_save = {}
f_pruned_name = __remove_ext(frame)
frame_read = __read_frame(os.path.join(framesdir, frame))
res_h = frame_read.shape[0] / 480
res_w = frame_read.shape[1] / 640
frame_read = imresize(frame_read, (480, 640))
label_l = __read_label(os.path.join(labels_dir, f_pruned_name + ".json"))
frame_read = imresize(frame_read, resize_rate)
label_l = [[p[1] * resize_rate / res_h, p[0] * resize_rate / res_w] for p in label_l]
frame_l = __add_padding(frame_read,
frame_read.shape[1] - (frame_read.shape[1] // width_shrink_rate) * width_shrink_rate,
frame_read.shape[0] - (
frame_read.shape[0] // heigth_shrink_rate) * heigth_shrink_rate)
frame_l = im_reg.apply(frame_l)
fr_to_save['frame'] = frame_l
coords_l = [__get_coord_from_labels(label_l)]
heat1 = np.zeros([frame_l.shape[0] // heigth_shrink_rate, frame_l.shape[1] // width_shrink_rate])
coords_l = coords_l[0]
res_coords_l = [[l[0] // heigth_shrink_rate, l[1] // width_shrink_rate] for l in coords_l]
res_coords_l = __enlarge_coords(res_coords_l, enlarge_heat, np.shape(heat1))
res_labels_l = [[l[0] // heigth_shrink_rate, l[1] // width_shrink_rate] for l in label_l]
heat1 = __shade_heatmap(heat1, res_coords_l, res_labels_l)
heat = heat1
heat[heat > 1] = 1
heat = he_r.apply(heat)
heat = __heatmap_to_uint8(heat)
fr_to_save['heatmap'] = heat
path = os.path.join(final_base_path, f_pruned_name + ".mat")
scio.savemat(path, fr_to_save)
except Exception:
print(frame)
def create_dataset(videos_list=None,
savepath=None,
im_regularizer=reg.Regularizer(),
fillgaps=False,
enlarge=0.5):
"""reads the videos specified as parameter and for each frame produces and saves a .mat file containing
the frame, the corresponding heatmap indicating the position of the hand and the modified depth.
:param fillgaps: set to True to also get interpolated frames
:param im_regularizer: object used to regularize the cuts
:param savepath: path of the folder where the produces .mat files will be saved. If left to the default value None,
the /resources/hands_bounding_dataset/hands_rgbd_transformed folder will be used
:param videos_list: list of videos you need the .mat files of. If left to the default value None, all videos will
be exploited
:param enlarge: crops enlarge factor"""
if savepath is None:
basedir = resources_path("palm_back_classification_dataset")
else:
basedir = savepath
if not os.path.exists(basedir):
os.makedirs(basedir)
framesdir = resources_path("framedata")
if videos_list is None:
vids = os.listdir(framesdir)
vids = [x for x in vids if os.path.isdir(os.path.join(framesdir, x))]
else:
vids = videos_list
for vid in tqdm.tqdm(vids):
frames, labels = load_labelled_videos(vid, fillgaps=fillgaps)
fr_num = frames.shape[0]
lr = get_right_left(vid)
if lr == LEFT:
lr = -1
for i in tqdm.tqdm(range(0, fr_num)):
if labels[i] is not None:
try:
frame = frames[i]
label = labels[i][:, 0:2]
# conf is a real in [-1.0, 1.0] such that -1.0 is full back, +1.0 is full palm
# middle values express partial confidence, but all info is in one single value
conf = pb.leftright_to_palmback(
hand=hand_format(label),
side=pb.RIGHT if lr == RIGHT else pb.LEFT)
# if you want the crisp result, there it is:
result = PALM if conf >= 0 else BACK
if result < 0:
result = 0
# and the confidence on that result is in [0..1]:
conf = abs(conf)
label *= [frame.shape[1], frame.shape[0]]
label = np.array(label, dtype=np.int32).tolist()
label = [[p[1], p[0]] for p in label]
coords = __get_coord_from_labels(label)
cut = u.crop_from_coords(frame, coords, enlarge)
cut = im_regularizer.apply(cut)
path = os.path.join(basedir, vid + "_" + str(i))
__persist_frame(path, cut, result, conf)
except ValueError as e:
print("Error " + str(e) + " on vid " + vid + str(i))
def create_dataset(videos_list=None,
savepath=None,
im_regularizer=reg.Regularizer(),
fillgaps=False,
enlarge=0.5,
data_augment=False):
"""reads the videos specified as parameter and for each frame produces and saves a .mat file containing
the frame, the corresponding heatmap indicating the position of the hand and the modified depth.
:param fillgaps: set to True to also get interpolated frames
:param im_regularizer: object used to regularize the cuts
:param savepath: path of the folder where the produces .mat files will be saved. If left to the default value None,
the /resources/hands_bounding_dataset/hands_rgbd_transformed folder will be used
:param videos_list: list of videos you need the .mat files of. If left to the default value None, all videos will
be exploited
:param enlarge: crops enlarge factor"""
if savepath is None:
basedir = resources_path("left_right_classification_dataset")
else:
basedir = savepath
if not os.path.exists(basedir):
os.makedirs(basedir)
framesdir = resources_path("framedata")
if videos_list is None:
vids = os.listdir(framesdir)
vids = [x for x in vids if os.path.isdir(os.path.join(framesdir, x))]
else:
vids = videos_list
for vid in tqdm.tqdm(vids):
frames, labels = load_labelled_videos(vid, fillgaps=fillgaps)
fr_num = frames.shape[0]
result = get_right_left(vid)
for i in tqdm.tqdm(range(0, fr_num)):
if labels[i] is not None:
try:
frame = frames[i]
label = labels[i][:, 0:2]
label *= [frame.shape[1], frame.shape[0]]
label = np.array(label, dtype=np.int32).tolist()
label = [[p[1], p[0]] for p in label]
coords = __get_coord_from_labels(label)
cut = u.crop_from_coords(frame, coords, enlarge)
cut = im_regularizer.apply(cut)
path = os.path.join(basedir, vid + "_" + str(i))
__persist_frame(path, cut, result)
if data_augment:
path = os.path.join(basedir, vid + "_t_" + str(i))
transp = cut.squeeze().transpose()
transp = transp.reshape(
[transp.shape[0], transp.shape[1], 1])
__persist_frame(path, transp, 1 - result)
except ValueError as e:
print("Error " + str(e) + " on vid " + vid + str(i))
fr_to_save = {
'cut': cut,
'heatmap_array': np.array(heatmaps * 255, dtype=np.uint8),
'visible': np.array(visible_flags, dtype=np.float32)
}
scio.savemat(path, fr_to_save)
def __read_frame(path):
matcontent = scio.loadmat(path)
return matcontent['cut'], matcontent['heatmap_array'] / 255.0, matcontent[
'visible']
if __name__ == '__main__':
im_r = reg.Regularizer()
im_r.fixresize(200, 200)
im_r.percresize(0.5)
h_r = reg.Regularizer()
h_r.fixresize(200, 200)
h_r.percresize(0.5)
create_dataset(im_regularizer=im_r,
heat_regularizer=h_r,
enlarge=0.5,
cross_radius=10,
shade=True)
c, h, v = read_dataset_random(number=2)
u.showimage(c[1])
print(np.shape(h))
# show heatmap for the first junction of the second item
u.showimage(u.heatmap_to_rgb(h[1][:, :, 0:1]))
def create_weights(gt):
outs = []
for i in range(len(gt)):
try:
outs = outs + list(np.array(gt[i]['OUT_0000']).squeeze())
except Exception:
continue
outs = np.array(outs).flatten()
weights = count_ones_zeros(outs, outs)
return weights
if __name__ == '__main__':
# NO NEED TO TOUCH ANYTHING AFTER THIS, IT'S DELICATE
regularizer = reg.Regularizer()
regularizer.fixresize(200, 200)
if createdataset:
create_dataset(savepath=path, im_regularizer=regularizer)
formatting = confidence_filtered_pb_format(minconf)
generator = DatasetManager(train_samples=train_samples,
valid_samples=valid_samples,
batch_size=batch_size,
dataset_dir=path,
formatting=formatting)
gt = generator.train()
cw = create_weights(gt)
data_processing_plan = ProcessingPlan()
n = len(ims)
ris = []
for i in range(n):
prod = prod_im_heat(ims[i], heats[i])
ris.append(prod)
return ris
if __name__ == '__main__':
LOAD_MODEL = False
TRAIN_MODEL = True
regularizer = reg.Regularizer()
regularizer.fixresize(200, 200)
regularizer_h = reg.Regularizer()
regularizer_h.fixresize(200, 200)
#create_dataset_w_heatmaps(savepath=path, im_regularizer=regularizer, h_r=regularizer_h)
x_train, y_train, c_train, h_train, x_test, y_test, c_test, h_test = read_dataset_h(path=path,
leave_out=['handsMaddalena2',
'handsGianpy',
'handsMatteo'])
# x_train = prod_im_heat_batch(x_train, h_train)
# x_test = prod_im_heat_batch(x_test, h_test)
x_train = np.array(x_train)
y_train = np.array(y_train)
c_train = np.array(c_train)
def create_dataset(videos_list=None,
savepath=None,
resize_rate=1.0,
heigth_shrink_rate=10,
width_shrink_rate=10,
im_reg=reg.Regularizer(),
heat_reg=reg.Regularizer()):
"""reads the videos specified as parameter and for each frame produces and saves a .mat file containing
the frame, the corresponding heatmap indicating the position of the hand(s)
THE PROCESS:
- image is resized
- heatmap is produced with dimensions resized w.r.t. the resized image
- regularizers are applied (BE CAREFUL WITH REGULARIZERS, YOU MAY ALTER DIMENSION RATIOS BETWEEN IMAGES AND
HEATMAPS)
:param width_shrink_rate: shrink rate of heatmaps width wrt the resized image
:param heigth_shrink_rate: shrink rate of heatmaps height wrt the resized image
:param resize_rate: resize rate of the images (1 (default) is the original image)
:param savepath: path of the folder where the produces .mat files will be saved. If left to the default value None,
the /resources/hands_bounding_dataset/egohands_transformed folder will be used
:param videos_list: list of videos you need the .mat files of. If left to the default value None, all videos will
be exploited
:param im_reg: object used to regularize the images
:param heat_reg: object used to regularize the heatmaps"""
# approximation_ratio = resize_rate / max(heigth_shrink_rate, width_shrink_rate)
approximation_ratio = 0.05
if savepath is None:
basedir = resources_path(
os.path.join("hands_bounding_dataset", "egohands_tranformed"))
else:
basedir = savepath
if not os.path.exists(basedir):
os.makedirs(basedir)
framesdir = resources_path(
os.path.join("hands_bounding_dataset", "egohands"))
if videos_list is None:
vids = os.listdir(framesdir)
vids = [x for x in vids if os.path.isdir(os.path.join(framesdir, x))]
else:
vids = videos_list
for vid in tqdm(vids):
frames, labels = __load_egohand_video(
os.path.join(framesdir, vid),
one_out_of=int(min(width_shrink_rate, heigth_shrink_rate)) /
resize_rate)
fr_num = len(frames)
for i in tqdm(range(0, fr_num)):
fr_to_save = {}
frame = frames[i]
frame = imresize(frame, [480, 640])
frame = imresize(frame, resize_rate)
frame = __add_padding(
frame, frame.shape[1] -
(frame.shape[1] // width_shrink_rate) * width_shrink_rate,
frame.shape[0] -
(frame.shape[0] // heigth_shrink_rate) * heigth_shrink_rate)
heat = __create_ego_heatmap(frame, labels[i], heigth_shrink_rate,
width_shrink_rate, resize_rate,
approximation_ratio)
frame = im_reg.apply(frame)
heat = heat_reg.apply(heat)
fr_to_save['frame'] = frame
fr_to_save['heatmap'] = __heatmap_to_uint8(heat)
path = os.path.join(basedir, vid + "_" + str(i))
scio.savemat(path, fr_to_save)
def create_palmback_dataset():
img_reg = regularizer.Regularizer()
img_reg.fixresize(200, 200)
pbcreate(savepath=palmback_path(), im_regularizer=img_reg)
def create_dataset_shaded_heatmaps(videos_list=None,
savepath=crops_path(),
resize_rate=1.0,
heigth_shrink_rate=10,
width_shrink_rate=10,
overlapping_penalty=0.9,
fillgaps=False,
toofar=1500,
tooclose=500,
enlarge_heat=0.3,
im_reg=reg.Regularizer(),
he_r=reg.Regularizer()):
"""reads the videos specified as parameter and for each frame produces and saves a .mat file containing
the frame, the corresponding heatmap indicating the position of the hand and the modified depth.
:param tooclose: threshold value used to eliminate too close objects/values in the depth
:param toofar: threshold value used to eliminate too far objects/values in the depth
:param fillgaps: set to True to also get interpolated frames
:param overlapping_penalty: penalty "inflicted" to the overlapping hands area in the images
:param width_shrink_rate: shrink rate of heatmaps width wrt the resized image
:param heigth_shrink_rate: shrink rate of heatmaps height wrt the resized image
:param resize_rate: resize rate of the images (1 (default) is the original image)
:param savepath: path of the folder where the produces .mat files will be saved. If left to the default value None,
the /resources/hands_bounding_dataset/hands_rgbd_transformed folder will be used
:param videos_list: list of videos you need the .mat files of. If left to the default value None, all videos will
be exploited"""
if savepath is None:
basedir = crops_path()
else:
basedir = savepath
if not os.path.exists(basedir):
os.makedirs(basedir)
framesdir = resources_path("framedata")
if videos_list is None:
vids = os.listdir(framesdir)
vids = [x for x in vids if os.path.isdir(os.path.join(framesdir, x))]
else:
vids = videos_list
for vid in tqdm.tqdm(vids):
frames, labels = load_labelled_videos(vid, fillgaps=fillgaps)
if labels is None:
continue
depths, _ = load_labelled_videos(vid, getdepth=True, fillgaps=fillgaps)
fr_num = frames.shape[0]
for i in tqdm.tqdm(range(0, fr_num)):
if labels[i] is not None:
try:
fr_to_save = {}
frame = frames[i]
depth = depths[i]
frame, depth = transorm_rgd_depth(frame,
depth,
toofar=toofar,
tooclose=tooclose)
frame = imresize(frame, resize_rate)
depth = depth_resize(depth, resize_rate)
label = labels[i][:, 0:2]
label *= [frame.shape[1], frame.shape[0]]
label = np.array(label, dtype=np.int32).tolist()
label = [[p[1], p[0]] for p in label]
frame = __add_padding(
frame, frame.shape[1] -
(frame.shape[1] // width_shrink_rate) *
width_shrink_rate, frame.shape[0] -
(frame.shape[0] // heigth_shrink_rate) *
heigth_shrink_rate)
depth = __add_padding(
depth, depth.shape[1] -
(depth.shape[1] // width_shrink_rate) *
width_shrink_rate, depth.shape[0] -
(depth.shape[0] // heigth_shrink_rate) *
heigth_shrink_rate)
depth = depth.squeeze()
depth = np.uint8(depth)
frame = im_reg.apply(frame)
fr_to_save['frame'] = frame
coords = [__get_coord_from_labels(label)]
heat = u.get_heatmap_from_coords(frame, heigth_shrink_rate,
width_shrink_rate, coords,
overlapping_penalty)
coords = coords[0]
res_coords = [[
l[0] // heigth_shrink_rate, l[1] // width_shrink_rate
] for l in coords]
res_coords = __enlarge_coords(res_coords, enlarge_heat,
np.shape(heat))
res_labels = [[
l[0] // heigth_shrink_rate, l[1] // width_shrink_rate
] for l in label]
heat = __shade_heatmap(heat, res_coords, res_labels)
heat = he_r.apply(heat)
heat = __heatmap_to_uint8(heat)
fr_to_save['heatmap'] = heat
depth = he_r.apply(depth)
fr_to_save['depth'] = depth
path = os.path.join(basedir, vid + "_" + str(i))
scio.savemat(path, fr_to_save)
except ValueError as e:
print(vid + str(i) + " => " + e)
