def plot_halnet_joints_from_heatmaps_crop(halnet_main_out, img_numpy, filenamebase, plot=True):
labels_colorspace = conv.heatmaps_to_joints_colorspace(halnet_main_out)
data_crop, crop_coords, labels_heatmaps, labels_colorspace = \
converter.crop_image_get_labels(img_numpy, labels_colorspace, range(21))
if plot:
fig = visualize.create_fig()
visualize.plot_image(data_crop, title=filenamebase, fig=fig)
visualize.plot_joints_from_colorspace(labels_colorspace, title=filenamebase, fig=fig, data=data_crop)
visualize.title('HALNet (joints from heatmaps - cropped): ' + filenamebase)
visualize.show()
return data_crop
depth_filepath = '/'.join([curr_data_folder, 'depth', 'depth_' + frame_num + '.png'])
try:
depth_img = fpa_io.read_depth_img(depth_filepath)
depth_imgs.append(depth_img)
except FileNotFoundError as e:
break
frame_idx = 0
for depth_img in depth_imgs:
frame_idx += 1
depth_img = depth_img.reshape((1, 1, depth_img.shape[0], depth_img.shape[1]))
depth_img = torch.from_numpy(depth_img).float()
if args.use_cuda:
depth_img = depth_img.cuda()
output = model(depth_img)
output_bbox = np.zeros((2, 2))
out_heatmaps = output[3].detach().cpu().numpy()[0, :, :, :]
output_bbox[0, :] = np.unravel_index(np.argmax(out_heatmaps[0]), (640, 480))
output_bbox[1, :] = np.unravel_index(np.argmax(out_heatmaps[1]), (640, 480))
title = "Subj: " + subject + ", Action: " + action + ", Seq: " + seq_str + ", Frame: " + str(frame_idx)
visualize.plot_image(depth_img.cpu().numpy()[0, 0, :, :], fig=fig, title=title)
visualize.plot_bound_box(output_bbox, fig=fig, color='red')
visualize.pause(0.001)
visualize.clear_plot()
visualize.show()
def validate(valid_loader, model, optimizer, valid_vars, control_vars, verbose=True):
curr_epoch_iter = 1
for batch_idx, (data, target) in enumerate(valid_loader):
control_vars['batch_idx'] = batch_idx
if batch_idx < control_vars['iter_size']:
print_verbose("\rPerforming first iteration; current mini-batch: " +
str(batch_idx + 1) + "/" + str(control_vars['iter_size']), verbose, n_tabs=0, erase_line=True)
# start time counter
start = time.time()
# get data and targetas cuda variables
target_heatmaps, target_joints, target_joints_z = target
data, target_heatmaps = Variable(data), Variable(target_heatmaps)
if valid_vars['use_cuda']:
data = data.cuda()
target_heatmaps = target_heatmaps.cuda()
# visualize if debugging
# get model output
output = model(data)
# accumulate loss for sub-mini-batch
if valid_vars['cross_entropy']:
loss_func = my_losses.cross_entropy_loss_p_logq
else:
loss_func = my_losses.euclidean_loss
loss = my_losses.calculate_loss_HALNet(loss_func,
output, target_heatmaps, model.joint_ixs, model.WEIGHT_LOSS_INTERMED1,
model.WEIGHT_LOSS_INTERMED2, model.WEIGHT_LOSS_INTERMED3,
model.WEIGHT_LOSS_MAIN, control_vars['iter_size'])
if DEBUG_VISUALLY:
for i in range(control_vars['max_mem_batch']):
filenamebase_idx = (batch_idx * control_vars['max_mem_batch']) + i
filenamebase = valid_loader.dataset.get_filenamebase(filenamebase_idx)
fig = visualize.create_fig()
#visualize.plot_joints_from_heatmaps(output[3][i].data.numpy(), fig=fig,
# title=filenamebase, data=data[i].data.numpy())
#visualize.plot_image_and_heatmap(output[3][i][8].data.numpy(),
# data=data[i].data.numpy(),
# title=filenamebase)
#visualize.savefig('/home/paulo/' + filenamebase.replace('/', '_') + '_heatmap')
labels_colorspace = conv.heatmaps_to_joints_colorspace(output[3][i].data.numpy())
data_crop, crop_coords, labels_heatmaps, labels_colorspace = \
converter.crop_image_get_labels(data[i].data.numpy(), labels_colorspace, range(21))
visualize.plot_image(data_crop, title=filenamebase, fig=fig)
visualize.plot_joints_from_colorspace(labels_colorspace, title=filenamebase, fig=fig, data=data_crop)
#visualize.savefig('/home/paulo/' + filenamebase.replace('/', '_') + '_crop')
visualize.show()
#loss.backward()
valid_vars['total_loss'] += loss
# accumulate pixel dist loss for sub-mini-batch
valid_vars['total_pixel_loss'] = my_losses.accumulate_pixel_dist_loss_multiple(
valid_vars['total_pixel_loss'], output[3], target_heatmaps, control_vars['batch_size'])
if valid_vars['cross_entropy']:
valid_vars['total_pixel_loss_sample'] = my_losses.accumulate_pixel_dist_loss_from_sample_multiple(
valid_vars['total_pixel_loss_sample'], output[3], target_heatmaps, control_vars['batch_size'])
else:
valid_vars['total_pixel_loss_sample'] = [-1] * len(model.joint_ixs)
# get boolean variable stating whether a mini-batch has been completed
minibatch_completed = (batch_idx+1) % control_vars['iter_size'] == 0
if minibatch_completed:
# append total loss
valid_vars['losses'].append(valid_vars['total_loss'].item())
# erase total loss
total_loss = valid_vars['total_loss'].item()
valid_vars['total_loss'] = 0
# append dist loss
valid_vars['pixel_losses'].append(valid_vars['total_pixel_loss'])
# erase pixel dist loss
valid_vars['total_pixel_loss'] = [0] * len(model.joint_ixs)
# append dist loss of sample from output
valid_vars['pixel_losses_sample'].append(valid_vars['total_pixel_loss_sample'])
# erase dist loss of sample from output
valid_vars['total_pixel_loss_sample'] = [0] * len(model.joint_ixs)
# check if loss is better
if valid_vars['losses'][-1] < valid_vars['best_loss']:
valid_vars['best_loss'] = valid_vars['losses'][-1]
#print_verbose(" This is a best loss found so far: " + str(valid_vars['losses'][-1]), verbose)
# log checkpoint
if control_vars['curr_iter'] % control_vars['log_interval'] == 0:
trainer.print_log_info(model, optimizer, 1, total_loss, valid_vars, control_vars)
model_dict = {
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'control_vars': control_vars,
'train_vars': valid_vars,
}
trainer.save_checkpoint(model_dict,
filename=valid_vars['checkpoint_filenamebase'] +
str(control_vars['num_iter']) + '.pth.tar')
# print time lapse
prefix = 'Validating (Epoch #' + str(1) + ' ' + str(control_vars['curr_epoch_iter']) + '/' +\
str(control_vars['tot_iter']) + ')' + ', (Batch ' + str(control_vars['batch_idx']+1) +\
'(' + str(control_vars['iter_size']) + ')' + '/' +\
str(control_vars['num_batches']) + ')' + ', (Iter #' + str(control_vars['curr_iter']) +\
'(' + str(control_vars['batch_size']) + ')' +\
' - log every ' + str(control_vars['log_interval']) + ' iter): '
control_vars['tot_toc'] = display_est_time_loop(control_vars['tot_toc'] + time.time() - start,
control_vars['curr_iter'], control_vars['num_iter'],
prefix=prefix)
control_vars['curr_iter'] += 1
control_vars['start_iter'] = control_vars['curr_iter'] + 1
control_vars['curr_epoch_iter'] += 1
return valid_vars, control_vars
def validate(valid_loader,
model,
optimizer,
valid_vars,
control_vars,
verbose=True):
curr_epoch_iter = 1
for batch_idx, (data, target) in enumerate(valid_loader):
control_vars['batch_idx'] = batch_idx
if batch_idx < control_vars['iter_size']:
print_verbose(
"\rPerforming first iteration; current mini-batch: " +
str(batch_idx + 1) + "/" + str(control_vars['iter_size']),
verbose,
n_tabs=0,
erase_line=True)
# start time counter
start = time.time()
# get data and targetas cuda variables
target_heatmaps, target_joints, target_handroot = target
# make target joints be relative
target_joints = target_joints[:, 3:]
data, target_heatmaps = Variable(data), Variable(target_heatmaps)
if valid_vars['use_cuda']:
data = data.cuda()
target_joints = target_joints.cuda()
target_heatmaps = target_heatmaps.cuda()
target_handroot = target_handroot.cuda()
# visualize if debugging
# get model output
output = model(data)
# accumulate loss for sub-mini-batch
if model.cross_entropy:
loss_func = my_losses.cross_entropy_loss_p_logq
else:
loss_func = my_losses.euclidean_loss
weights_heatmaps_loss, weights_joints_loss = get_loss_weights(
control_vars['curr_iter'])
loss, loss_heatmaps, loss_joints = my_losses.calculate_loss_JORNet(
loss_func, output, target_heatmaps, target_joints,
valid_vars['joint_ixs'], weights_heatmaps_loss,
weights_joints_loss, control_vars['iter_size'])
valid_vars['total_loss'] += loss
valid_vars['total_joints_loss'] += loss_joints
valid_vars['total_heatmaps_loss'] += loss_heatmaps
# accumulate pixel dist loss for sub-mini-batch
valid_vars[
'total_pixel_loss'] = my_losses.accumulate_pixel_dist_loss_multiple(
valid_vars['total_pixel_loss'], output[3], target_heatmaps,
control_vars['batch_size'])
valid_vars[
'total_pixel_loss_sample'] = my_losses.accumulate_pixel_dist_loss_from_sample_multiple(
valid_vars['total_pixel_loss_sample'], output[3],
target_heatmaps, control_vars['batch_size'])
# get boolean variable stating whether a mini-batch has been completed
for i in range(control_vars['max_mem_batch']):
filenamebase_idx = (batch_idx * control_vars['max_mem_batch']) + i
filenamebase = valid_loader.dataset.get_filenamebase(
filenamebase_idx)
print('')
print(filenamebase)
visualize.plot_image(data[i].data.numpy())
visualize.show()
output_batch_numpy = output[7][i].data.cpu().numpy()
print('\n-------------------------------')
reshaped_out = output_batch_numpy.reshape((20, 3))
for j in range(20):
print('[{}, {}, {}],'.format(reshaped_out[j, 0],
reshaped_out[j, 1],
reshaped_out[j, 2]))
print('-------------------------------')
fig, ax = visualize.plot_3D_joints(target_joints[i])
visualize.plot_3D_joints(output_batch_numpy,
fig=fig,
ax=ax,
color='C6')
visualize.title(filenamebase)
visualize.show()
temp = np.zeros((21, 3))
output_batch_numpy_abs = output_batch_numpy.reshape((20, 3))
temp[1:, :] = output_batch_numpy_abs
output_batch_numpy_abs = temp
output_joints_colorspace = camera.joints_depth2color(
output_batch_numpy_abs,
depth_intr_matrix=synthhands_handler.DEPTH_INTR_MTX,
handroot=target_handroot[i].data.cpu().numpy())
visualize.plot_3D_joints(output_joints_colorspace)
visualize.show()
aa1 = target_joints[i].data.cpu().numpy().reshape((20, 3))
aa2 = output[7][i].data.cpu().numpy().reshape((20, 3))
print('\n----------------------------------')
print(np.sum(np.abs(aa1 - aa2)) / 60)
print('----------------------------------')
#loss.backward()
valid_vars['total_loss'] += loss
valid_vars['total_joints_loss'] += loss_joints
valid_vars['total_heatmaps_loss'] += loss_heatmaps
# accumulate pixel dist loss for sub-mini-batch
valid_vars[
'total_pixel_loss'] = my_losses.accumulate_pixel_dist_loss_multiple(
valid_vars['total_pixel_loss'], output[3], target_heatmaps,
control_vars['batch_size'])
valid_vars[
'total_pixel_loss_sample'] = my_losses.accumulate_pixel_dist_loss_from_sample_multiple(
valid_vars['total_pixel_loss_sample'], output[3],
target_heatmaps, control_vars['batch_size'])
# get boolean variable stating whether a mini-batch has been completed
minibatch_completed = (batch_idx + 1) % control_vars['iter_size'] == 0
if minibatch_completed:
# append total loss
valid_vars['losses'].append(valid_vars['total_loss'].data[0])
# erase total loss
total_loss = valid_vars['total_loss'].data[0]
valid_vars['total_loss'] = 0
# append total joints loss
valid_vars['losses_joints'].append(
valid_vars['total_joints_loss'].data[0])
# erase total joints loss
valid_vars['total_joints_loss'] = 0
# append total joints loss
valid_vars['losses_heatmaps'].append(
valid_vars['total_heatmaps_loss'].data[0])
# erase total joints loss
valid_vars['total_heatmaps_loss'] = 0
# append dist loss
valid_vars['pixel_losses'].append(valid_vars['total_pixel_loss'])
# erase pixel dist loss
valid_vars['total_pixel_loss'] = [0] * len(model.joint_ixs)
# append dist loss of sample from output
valid_vars['pixel_losses_sample'].append(
valid_vars['total_pixel_loss_sample'])
# erase dist loss of sample from output
valid_vars['total_pixel_loss_sample'] = [0] * len(model.joint_ixs)
# check if loss is better
#if valid_vars['losses'][-1] < valid_vars['best_loss']:
# valid_vars['best_loss'] = valid_vars['losses'][-1]
# print_verbose(" This is a best loss found so far: " + str(valid_vars['losses'][-1]), verbose)
# log checkpoint
if control_vars['curr_iter'] % control_vars['log_interval'] == 0:
trainer.print_log_info(model, optimizer, 1, total_loss,
valid_vars, control_vars)
model_dict = {
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'control_vars': control_vars,
'train_vars': valid_vars,
}
trainer.save_checkpoint(
model_dict,
filename=valid_vars['checkpoint_filenamebase'] +
str(control_vars['num_iter']) + '.pth.tar')
# print time lapse
prefix = 'Validating (Epoch #' + str(1) + ' ' + str(control_vars['curr_epoch_iter']) + '/' +\
str(control_vars['tot_iter']) + ')' + ', (Batch ' + str(control_vars['batch_idx']+1) +\
'(' + str(control_vars['iter_size']) + ')' + '/' +\
str(control_vars['num_batches']) + ')' + ', (Iter #' + str(control_vars['curr_iter']) +\
'(' + str(control_vars['batch_size']) + ')' +\
' - log every ' + str(control_vars['log_interval']) + ' iter): '
control_vars['tot_toc'] = display_est_time_loop(
control_vars['tot_toc'] + time.time() - start,
control_vars['curr_iter'],
control_vars['num_iter'],
prefix=prefix)
control_vars['curr_iter'] += 1
control_vars['start_iter'] = control_vars['curr_iter'] + 1
control_vars['curr_epoch_iter'] += 1
return valid_vars, control_vars
def read_image_from_image_grid(img_filepath, img_res, grid_x, grid_y):
img_grid = io_image.read_RGB_image((img_filepath))
img = np.zeros(img_res)
for i in range(3):
start_y = grid_x * img_res[0]
start_x = grid_y * img_res[1]
img += img_grid[start_x:start_x + img_res[0],
start_y:start_y + img_res[1], i] * np.power(255, i)
img = img / 255
return img
def read_image_column_from_image_grid(img_filepath, img_res, column, n_rows):
imgs = []
for i in range(n_rows):
imgs.append(
read_image_from_image_grid(img_filepath, img_res, i, column))
return imgs
images = read_image_column_from_image_grid(img_filepath, img_res, 4, n_rows)
for i in range(3):
vis.plot_image(images[i])
vis.show()
loss_func, output, label_heatmaps, model.joint_ixs,
model.WEIGHT_LOSS_INTERMED1, model.WEIGHT_LOSS_INTERMED2,
model.WEIGHT_LOSS_INTERMED3, model.WEIGHT_LOSS_MAIN,
train_vars['iter_size'])
loss_pixel, label_bbox, out_bbox = calculate_pixel_loss(
output[3].detach().cpu().numpy()[0, :, :, :], label_heatmaps)
losses_pixel.append(loss_pixel)
if batch_idx == 4:
depth_img = data.cpu().numpy()[0, 0, :, :]
depth_img_crop = depth_img[label_bbox[0, 0]:label_bbox[1, 0],
label_bbox[0, 1]:label_bbox[1, 1]]
depth_img_crop = io_image.change_res_image(depth_img_crop,
((200, 200)))
vis.plot_image(depth_img_crop,
title=train_loader.dataset.get_img_title(batch_idx))
vis.show()
fig = vis.plot_image(
depth_img, title=train_loader.dataset.get_img_title(batch_idx))
#vis.plot_bound_box(label_bbox, fig=fig, color='blue')
#vis.plot_bound_box(out_bbox, fig=fig, color='red')
vis.show()
a = 0
train_vars['total_loss'] = loss.item()
print('Batch {} / {}'.format(batch_idx, len_dataset))
print('Loss (pixel): {}'.format(loss_pixel))
print('\tMean loss (pixel): {}'.format(np.mean(losses_pixel)))
print('\tStddev loss (pixel): {}'.format(np.std(losses_pixel)))
'color_{:04d}.jpeg'.format(int(frame_num)))
print('Loading image from {}'.format(img_path))
img = Image.open(img_path)
img2_path = 'C:/Users/Administrator/Documents/Datasets/fpa_benchmark/gen_objs/' \
'Subject_1/close_juice_bottle/1/' \
'2_depth.csv'
img2_depth_array = np.loadtxt(open(img2_path, "rb"), delimiter=",")
print(np.max(img2_depth_array.reshape((640 * 480, ))))
print(np.min(img2_depth_array.reshape((640 * 480, ))))
print(np.mean(img2_depth_array.reshape((640 * 480, ))))
print(np.std(img2_depth_array.reshape((640 * 480, ))))
img2_path2 = 'C:/Users/Administrator/Documents/Datasets/fpa_benchmark/gen_objs/' \
'Subject_1/close_juice_bottle/1/' \
'2_mask.jpg'
img = Image.open(img2_path2)
img_numpy = np.array(img).T
vis.plot_image(img_numpy)
vis.show()
ax = plt.gca()
ax.imshow(img, alpha=0.5)
ax.scatter(verts_proj[:, 0], verts_proj[:, 1], c='r')
#ax.scatter(verts_camcoords[:, 0], verts_camcoords[:, 1], s=1)
plt.pause(0.001)
plt.clf()
plt.show()