def Preprocessing(d, stage='train'):
height, width = cfg.data_shape
imgs = []
labels = []
valids = []
if cfg.use_seg:
segms = []
vis = False
img = cv2.imread(os.path.join(cfg.img_path, d['imgpath']))
#hack(multiprocessing data provider)
while img is None:
print('read none image')
time.sleep(np.random.rand() * 5)
img = cv2.imread(os.path.join(cfg.img_path, d['imgpath']))
add = max(img.shape[0], img.shape[1])
bimg = cv2.copyMakeBorder(img,
add,
add,
add,
add,
borderType=cv2.BORDER_CONSTANT,
value=cfg.pixel_means.reshape(-1))
bbox = np.array(d['bbox']).reshape(4, ).astype(np.float32)
bbox[:2] += add
if 'joints' in d:
joints = np.array(d['joints']).reshape(cfg.nr_skeleton,
3).astype(np.float32)
joints[:, :2] += add
inds = np.where(joints[:, -1] == 0)
joints[inds, :2] = -1000000
crop_width = bbox[2] * (1 + cfg.imgExtXBorder * 2)
crop_height = bbox[3] * (1 + cfg.imgExtYBorder * 2)
objcenter = np.array([bbox[0] + bbox[2] / 2., bbox[1] + bbox[3] / 2.])
if stage == 'train':
crop_width = crop_width * (1 + 0.25)
crop_height = crop_height * (1 + 0.25)
if crop_height / height > crop_width / width:
crop_size = crop_height
min_shape = height
else:
crop_size = crop_width
min_shape = width
crop_size = min(crop_size, objcenter[0] / width * min_shape * 2. - 1.)
crop_size = min(
crop_size, (bimg.shape[1] - objcenter[0]) / width * min_shape * 2. - 1)
crop_size = min(crop_size, objcenter[1] / height * min_shape * 2. - 1.)
crop_size = min(crop_size,
(bimg.shape[0] - objcenter[1]) / height * min_shape * 2. -
1)
min_x = int(objcenter[0] - crop_size / 2. / min_shape * width)
max_x = int(objcenter[0] + crop_size / 2. / min_shape * width)
min_y = int(objcenter[1] - crop_size / 2. / min_shape * height)
max_y = int(objcenter[1] + crop_size / 2. / min_shape * height)
x_ratio = float(width) / (max_x - min_x)
y_ratio = float(height) / (max_y - min_y)
if 'joints' in d:
joints[:, 0] = joints[:, 0] - min_x
joints[:, 1] = joints[:, 1] - min_y
joints[:, 0] *= x_ratio
joints[:, 1] *= y_ratio
label = joints[:, :2].copy()
valid = joints[:, 2].copy()
img = cv2.resize(bimg[min_y:max_y, min_x:max_x, :], (width, height))
if stage != 'train':
details = np.asarray(
[min_x - add, min_y - add, max_x - add, max_y - add])
if cfg.use_seg is True and 'segmentation' in d:
seg = get_seg(ori_img.shape[0], ori_img.shape[1], d['segmentation'])
add = max(seg.shape[0], seg.shape[1])
bimg = cv2.copyMakeBorder(seg,
add,
add,
add,
add,
borderType=cv2.BORDER_CONSTANT,
value=(0, 0, 0))
seg = cv2.resize(bimg[min_y:max_y, min_x:max_x], (width, height))
segms.append(seg)
if vis:
img2 = img.copy()
from visualization import draw_skeleton
draw_skeleton(img, label.astype(int))
cv2.imshow('1', ori_img)
cv2.imshow('2', img2)
cv2.imshow('', img)
cv2.waitKey()
# from vis_detection import visualize
# visualize(ori_img, np.array(d['bbox']).reshape(4,))
# visualize(img, keypoints=joints)
img = img - cfg.pixel_means
if cfg.pixel_norm:
img = img / 255.
img = img.transpose(2, 0, 1)
imgs.append(img)
if 'joints' in d:
labels.append(label.reshape(-1))
valids.append(valid.reshape(-1))
if stage == 'train':
imgs, labels, valids = data_augmentation(imgs, labels, valids)
heatmaps15 = joints_heatmap_gen(imgs,
labels,
cfg.output_shape,
cfg.data_shape,
return_valid=False,
gaussian_kernel=cfg.gk15)
heatmaps11 = joints_heatmap_gen(imgs,
labels,
cfg.output_shape,
cfg.data_shape,
return_valid=False,
gaussian_kernel=cfg.gk11)
heatmaps9 = joints_heatmap_gen(imgs,
labels,
cfg.output_shape,
cfg.data_shape,
return_valid=False,
gaussian_kernel=cfg.gk9)
heatmaps7 = joints_heatmap_gen(imgs,
labels,
cfg.output_shape,
cfg.data_shape,
return_valid=False,
gaussian_kernel=cfg.gk7)
return [
imgs.astype(np.float32).transpose(0, 2, 3, 1),
heatmaps15.astype(np.float32).transpose(0, 2, 3, 1),
heatmaps11.astype(np.float32).transpose(0, 2, 3, 1),
heatmaps9.astype(np.float32).transpose(0, 2, 3, 1),
heatmaps7.astype(np.float32).transpose(0, 2, 3, 1),
valids.astype(np.float32)
]
else:
return [np.asarray(imgs).astype(np.float32), details]
'discriminator_state_dict': discriminator.state_dict(),
'optimizer_G_state_dict': optimizer_G.state_dict(),
'optimizer_D_state_dict': optimizer_D.state_dict(),
'epoch': epoch,
'loss_D': loss_D.item(),
'loss_G': loss_G.item(),
'total_invalid_frame': total_invalid_frame
}, model_saving_dir)
if valid_error is False:
gen_skeleton_np = gen_skeleton.transpose(0, 1)
gen_skeleton_np = gen_skeleton_np.cpu().detach().numpy()
# gen_skeleton_np = np.expand_dims(gen_skeleton_np, axis=0)
firstSkeleton = gen_skeleton_np[0:1, :, :, 0:2]
secondSkeleton = gen_skeleton_np[1:2, :, :, 0:2]
fig, ax = vs.draw_skeleton(firstSkeleton, step=2)
plt.savefig('simple_wgan_exp/fig/fig_2p_0/epoch%d_%3f_%3f_p%d.jpg' %
(epoch, loss_D.item(), loss_G.item(), 0))
fig, ax = vs.draw_skeleton(secondSkeleton, step=2)
plt.savefig('simple_wgan_exp/fig/fig_2p_1/epoch%d_%3f_%3f_p%d.jpg' %
(epoch, loss_D.item(), loss_G.item(), 1))
fig, ax = vs.draw_skeleton(gen_skeleton_np[:, :, :, 0:2],
step=2,
max_bodies=2)
plt.savefig('simple_wgan_exp/fig/fig_2p_2/epoch%d_%3f_%3f_p%d.jpg' %
(epoch, loss_D.item(), loss_G.item(), 2))
else:
valid_error = False
print("total invalid frame: ", total_invalid_frame)
def Preprocessing(d, stage='train'):
height, width = cfg.data_shape
imgs = []
labels = []
valids = []
vis = False
img = cv2.imread(os.path.join(d['imgpath']))
# ori_height, ori_width, ori_channel = img.shape
while img is None:
print('read none image')
time.sleep(np.random.rand() * 5)
img = cv2.imread(os.path.join(d['imgpath']))
bbox = np.array(d['bbox']).reshape(4, ).astype(np.float32)
if 'joints' in d:
joints = np.array(d['joints']).reshape(cfg.nr_skeleton,
3).astype(np.float32)
inds = np.where(joints[:, -1] == 0)
joints[inds, :2] = -1000000
min_x = int(bbox[0])
max_x = int(bbox[2])
min_y = int(bbox[1])
max_y = int(bbox[3])
x_ratio = float(width) / (max_x - min_x + 1)
y_ratio = float(height) / (max_y - min_y + 1)
if 'joints' in d:
joints[:, 0] = joints[:, 0] - min_x
joints[:, 1] = joints[:, 1] - min_y
joints[:, 0] *= x_ratio
joints[:, 1] *= y_ratio
label = joints[:, :2].copy()
valid = joints[:, 2].copy()
img = cv2.resize(img[min_y:max_y + 1, min_x:max_x + 1, :], (width, height))
if stage != 'train':
details = np.asarray([x_ratio, y_ratio, min_x, min_y])
if vis:
img2 = img.copy()
from visualization import draw_skeleton
draw_skeleton(img, label.astype(int))
cv2.imshow('1', ori_img)
cv2.imshow('2', img2)
cv2.imshow('', img)
cv2.waitKey()
# from vis_detection import visualize
# visualize(ori_img, np.array(d['bbox']).reshape(4,))
# visualize(img, keypoints=joints)
#img = img - cfg.pixel_means
if cfg.pixel_norm:
img = img / 255.
img = img.transpose(2, 0, 1)
imgs.append(img)
if 'joints' in d:
labels.append(label.reshape(-1))
valids.append(valid.reshape(-1))
if stage == 'train':
imgs, labels, valids = data_augmentation(imgs, labels, valids)
heatmaps15 = joints_heatmap_gen(imgs,
labels,
cfg.output_shape,
cfg.data_shape,
return_valid=False,
gaussian_kernel=cfg.gk15)
heatmaps11 = joints_heatmap_gen(imgs,
labels,
cfg.output_shape,
cfg.data_shape,
return_valid=False,
gaussian_kernel=cfg.gk11)
heatmaps9 = joints_heatmap_gen(imgs,
labels,
cfg.output_shape,
cfg.data_shape,
return_valid=False,
gaussian_kernel=cfg.gk9)
heatmaps7 = joints_heatmap_gen(imgs,
labels,
cfg.output_shape,
cfg.data_shape,
return_valid=False,
gaussian_kernel=cfg.gk7)
return [
imgs.astype(np.float32).transpose(1, 2, 0),
heatmaps15.astype(np.float32).transpose(1, 2, 0),
heatmaps11.astype(np.float32).transpose(1, 2, 0),
heatmaps9.astype(np.float32).transpose(1, 2, 0),
heatmaps7.astype(np.float32).transpose(1, 2, 0),
valids.astype(np.float32)
]
else:
return [np.asarray(imgs).astype(np.float32), details]
def gen_single_frames(model, in_dim, max_frames=(4, 4), fname='samples_sf'):
z = torch.randn(max_frames[0] * max_frames[1], in_dim)
x = model(z=z)
x = x.reshape((1, -1, 25, 2)).detach().numpy()
fig, ax = draw_skeleton(x, step=1)
fig.savefig(fname + '.png')
def preprocessing(d, config, stage='train', debug=False):
height, width = config.DATA_SHAPE
imgs = []
labels = []
valids = []
# read image
img_ori = cv2.imread(d['imgpath'])
if img_ori is None:
print('read none image')
return None
# crop based on bbox
img = img_ori.copy()
add = max(img.shape[0], img.shape[1])
## make border and avoid crop size exceed image size
bimg = cv2.copyMakeBorder(img, add, add, add, add, borderType=cv2.BORDER_CONSTANT,\
value=config.PIXEL_MEANS.reshape(-1))
bbox = np.array(d['bbox']).reshape(4, ).astype(np.float32)
## coordinate should also translate of 'add'
bbox[:2] += add
if 'joints' in d:
joints = np.array(d['joints']).reshape(config.KEYPOINTS_NUM, 3).astype(np.float32)
joints[:, :2] += add
idx = np.where(joints[:, -1] == 0)
joints[idx, :2] = -1000000
## preprare crop size
crop_width = bbox[2] * (1 + config.imgExtXBorder * 2)
crop_height = bbox[3] * (1 + config.imgExtYBorder * 2)
objcenter = np.array([bbox[0] + bbox[2] / 2., bbox[1] + bbox[3] / 2.])
## in training stage, crop size should extend 0.25
if stage == 'train':
crop_width = crop_width * (1 + 0.25)
crop_height = crop_height * (1 + 0.25)
if crop_height / height > crop_width / width:
crop_size = crop_height
min_shape = height
else:
crop_size = crop_width
min_shape = width
crop_size = min(crop_size, objcenter[0] / width * min_shape * 2. - 1.)
crop_size = min(crop_size, (bimg.shape[1] - objcenter[0]) / width * min_shape * 2. - 1)
crop_size = min(crop_size, objcenter[1] / height * min_shape * 2. - 1.)
crop_size = min(crop_size, (bimg.shape[0] - objcenter[1]) / height * min_shape * 2. - 1)
min_x = int(objcenter[0] - crop_size / 2. / min_shape * width)
max_x = int(objcenter[0] + crop_size / 2. / min_shape * width)
min_y = int(objcenter[1] - crop_size / 2. / min_shape * height)
max_y = int(objcenter[1] + crop_size / 2. / min_shape * height)
x_ratio = float(width) / (max_x - min_x)
y_ratio = float(height) / (max_y - min_y)
if 'joints' in d:
joints[:, 0] = joints[:, 0] - min_x
joints[:, 1] = joints[:, 1] - min_y
joints[:, 0] *= x_ratio
joints[:, 1] *= y_ratio
label = joints[:, :2].copy()
valid = joints[:, 2].copy()
img = cv2.resize(bimg[min_y:max_y, min_x:max_x, :], (width, height))
if stage != 'train':
details = np.asarray([min_x - add, min_y - add, max_x - add, max_y - add])
if debug:
from visualization import draw_skeleton
img2 = img.copy()
draw_skeleton(img2, joints.astype(int))
cv2.imshow('', img2)
cv2.imshow('1', img_ori)
cv2.waitKey()
if 'joints' in d:
labels.append(label.reshape(-1))
valids.append(valid.reshape(-1))
imgs.append(img)
if stage == 'train':
imgs, labels, valids = data_augmentation(imgs, labels, valids, config)
heatmaps15 = joints_heatmap_gen(imgs, labels, config, return_valid=False, \
gaussian_kernel=config.GK15)
heatmaps11 = joints_heatmap_gen(imgs, labels, config, return_valid=False, \
gaussian_kernel=config.GK11)
heatmaps9 = joints_heatmap_gen(imgs, labels, config, return_valid=False, \
gaussian_kernel=config.GK9)
heatmaps7 = joints_heatmap_gen(imgs, labels, config, return_valid=False, \
gaussian_kernel=config.GK7)
imgs = imgs.astype(np.float32)
for index_ in range(len(imgs)):
imgs[index_] = image_preprocessing(imgs[index_], config)
return_args = [imgs.astype(np.float32),
heatmaps15.astype(np.float32).transpose(0, 2, 3, 1),
heatmaps11.astype(np.float32).transpose(0, 2, 3, 1),
heatmaps9.astype(np.float32).transpose(0, 2, 3, 1),
heatmaps7.astype(np.float32).transpose(0, 2, 3, 1),
valids.astype(np.float32)]
return return_args
else:
for index_ in range(len(imgs)):
imgs[index_] = image_preprocessing(imgs[index_], config)
return [np.asarray(imgs).astype(np.float32), details]
