def hist_similarity(img1, img2, PLOT=False):
img1 = adjust_support(img1, "0->255", "0->1")
img2 = adjust_support(img2, "0->255", "0->1")
OPENCV_METHODS = (("Correlation", cv2.HISTCMP_CORREL),
("Chi-Squared", cv2.HISTCMP_CHISQR),
("Intersection", cv2.HISTCMP_INTERSECT),
("Hellinger", cv2.HISTCMP_BHATTACHARYYA))
hist1 = cv2.calcHist([img1], [0, 1, 2], None, [8, 8, 8],
[0, 256, 0, 256, 0, 256])
hist1 = cv2.normalize(hist1, hist1).flatten()
hist2 = cv2.calcHist([img2], [0, 1, 2], None, [8, 8, 8],
[0, 256, 0, 256, 0, 256])
hist2 = cv2.normalize(hist2, hist2).flatten()
hist_compare = cv2.compareHist(hist1, hist2, cv2.HISTCMP_CORREL)
if PLOT:
fig, (ax1, ax2) = plt.subplots(1, 2)
plt.title(f"histcompare: {hist_compare}")
ax1.imshow(img1)
ax2.imshow(img2)
plt.show()
return hist_compare
def make_video(D, q_idxs, opt, crop_name, full_name):
import tempfile
from PIL import Image
from edflow.data.util import adjust_support
D.expand = True
with tempfile.TemporaryDirectory() as tmpd:
for i, [q, o] in enumerate(zip(q_idxs, opt)):
imq = Image.fromarray(adjust_support(D[q][crop_name], '0->255'))
imo = Image.fromarray(adjust_support(D[o][crop_name], '0->255'))
imq = _add_text(imq, 'Q')
imo = _add_text(imo, 'R')
qpath = os.path.join(tmpd, f'q_{i:0>4d}.png')
imq.save(qpath)
imo.save(os.path.join(tmpd, f'o_{i:0>4d}.png'))
pat_q = os.path.join(tmpd, 'q_%04d.png')
pat_o = os.path.join(tmpd, 'o_%04d.png')
name_q = os.path.join(tmpd, 'q.mp4')
name_o = os.path.join(tmpd, 'o.mp4')
out_name = f'{full_name}.mp4'
vid_command = 'ffmpeg -i {im_pat} -vf fps=25 -vcodec libx264 -crf 18 {name}'
qcommand = vid_command.format(im_pat=pat_q, name=name_q)
os.system(qcommand)
os.system(vid_command.format(im_pat=pat_o, name=name_o))
stack_command = 'ffmpeg -y -i {top} -i {bot} -vcodec libx264 -crf 18 -filter_complex hstack {name}'
os.system(stack_command.format(top=name_q, bot=name_o, name=out_name))
def crop(image, box):
'''Arguments:
image (np.ndarray or PIL.Image): Image to crop.
box (list): Box specifying ``[x, y, width, height]``
points (np.ndarray): Optional set of points in image coordinate, which
are translated to box coordinates. Shape: ``[(*), 2]``.
Returns:
np.ndarray: Cropped image with shape ``[W, H, C]`` and same support
as :attr:`image`.
If points is not None:
np.ndarray: The translated point coordinates.
'''
is_image = True
if not isinstance(image, Image.Image):
in_support = get_support(image)
image = adjust_support(image, '0->255')
image = Image.fromarray(image)
is_image = False
box[2:] = box[:2] + box[2:]
image = image.crop(box)
if not is_image:
image = adjust_support(np.array(image), in_support)
return image
def log_op():
PCK_THRESH = [0.01, 0.025, 0.05, 0.1, 0.125, 0.15, 0.175, 0.2, 0.25, 0.5]
HM_THRESH = [0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]
if self.config['pck_alpha'] not in PCK_THRESH: PCK_THRESH.append(self.config["pck_alpha"])
coords, _ = heatmaps_to_coords(predictions.clone(), thresh=self.config["hm"]["thresh"])
pck = {t: percentage_correct_keypoints(kwargs["kps"], coords, t, self.config["pck"]["type"]) for t in
PCK_THRESH}
gridded_outputs = np.expand_dims(sure_to_numpy(torchvision.utils.make_grid(
predictions[0], nrow=10)), 0).transpose(1, 2, 3, 0)
gridded_targets = np.expand_dims(sure_to_numpy(torchvision.utils.make_grid(
sure_to_torch(kwargs["targets"])[0], nrow=10)), 0).transpose(1, 2, 3, 0)
logs = {
"images": {
# Image input not needed, because stick animal is printed on input image
# "image_input": adjust_support(torch2numpy(inputs).transpose(0, 2, 3, 1), "-1->1"),
"first_pred": adjust_support(gridded_outputs, "-1->1", "0->1"),
"first_targets": adjust_support(gridded_targets, "-1->1", "0->1"),
"outputs": adjust_support(heatmaps_to_image(torch2numpy(predictions)).transpose(0, 2, 3, 1),
"-1->1", "0->1"),
"targets": adjust_support(heatmaps_to_image(kwargs["targets"]).transpose(0, 2, 3, 1), "-1->1"),
"inputs_with_stick": make_stickanimal(torch2numpy(inputs).transpose(0, 2, 3, 1), kwargs["kps"]),
"stickanimal": make_stickanimal(torch2numpy(inputs).transpose(0, 2, 3, 1), predictions.clone(),
thresh=self.config["hm"]["thresh"]),
},
"scalars": {
"loss": losses["total"],
"learning_rate": self.optimizer.state_dict()["param_groups"][0]["lr"],
f"PCK@{self.config['pck_alpha']}": np.around(pck[self.config['pck_alpha']][0], 5),
},
"figures": {
"Keypoint Mapping": plot_input_target_keypoints(torch2numpy(inputs).transpose(0, 2, 3, 1),
# get BHWC
torch2numpy(predictions), # stay BCHW
kwargs["kps"], coords),
}
}
if self.config["losses"]["L2"]:
logs["scalars"]["L2"] = losses["L2"]
if self.config["losses"]["L1"]:
logs["scalars"]["L1"] = losses["L1"]
if self.config["losses"]["perceptual"]:
logs["scalars"]["perceptual"] = losses["perceptual"]
if self.config["pck"]["pck_multi"]:
for key, val in pck.items():
# get mean value for pck at given threshold
logs["scalars"][f"PCK@{key}"] = np.around(val[0], 5)
for idx, part in enumerate(val[1]):
logs["scalars"][f"PCK@{key}_{self.dataset.get_idx_parts(idx)}"] = np.around(part, 5)
return logs
def log_op():
is_train = self.get_split() == "train"
logs = {
"images": {
"input": adjust_support(torch2numpy(inputs0).transpose(0, 2, 3, 1), "-1->1", "0->1"),
"outputs": adjust_support(torch2numpy(predictions).transpose(0, 2, 3, 1), "-1->1", "0->1"),
},
"scalars": {
"loss": loss,
},
}
return logs
def render_image(image_array, id):
'''Displays an image. Allows zooming!
Based on https://plot.ly/python/imshow/ and https://plot.ly/python/images/
Parameters
----------
image_array : np.ndarray
Image to display. Shape: ``[W, H, 3 or 4]``. Will be modified for
display using :func:`adjust_support`.
'''
plot_im = adjust_support(image_array, '0->255')
# Create figure and add image
fig = px.imshow(plot_im)
info_dict = {
'min': image_array.min(),
'max': image_array.max(),
'dtype': image_array.dtype,
'dimensions': '{}x{}'.format(*image_array.shape[:2])
}
return dcc.Graph(id=id, figure=fig), info_dict
def vid(start, length=250, K=100):
save_root = f'./video/{start}/'
query_root = os.path.join(save_root, 'query')
ref_root = os.path.join(save_root, 'reference')
os.makedirs(save_root, exist_ok=True)
os.makedirs(query_root, exist_ok=True)
os.makedirs(ref_root, exist_ok=True)
N = length
HG = HumanGaitFixedBox({'data_split': 'train'})
vidps = HG.labels['video_path']
vidns = np.char.rpartition(vidps, '/')[:, -1]
healths = np.char.rpartition(vidns, 'H')[:, 1]
healthy = healths == 'H'
unhealthy = healths == ''
h_indices = np.arange(len(HG))[healthy]
i_indices = np.arange(len(HG))[unhealthy]
kps = HG.labels['kps_fixed_rel'][..., :2].astype('float32')
kp_hidden = kps[healthy]
R = ReferenceSampler(kp_hidden, k=K)
unhealthy_kps = kps[unhealthy]
end = start + N
q_idxs = list(range(start, end))
q = unhealthy_kps[start:end]
opt = R(q)
print(opt)
HG.expand = True
vids = []
for which, indices in [['query', q_idxs], ['reference', opt]]:
save = os.path.join(save_root, which)
for i, idx in enumerate(indices):
if which == 'reference':
im = HG[h_indices[idx]]['target']
else:
im = HG[i_indices[idx]]['target']
im = adjust_support(im, '0->255')
Image.fromarray(im).save(os.path.join(save, f'{i:0>3}.png'))
vid = os.path.join(save_root, f'{which}.mp4')
vids += [vid]
command = f'ffmpeg -y -i {save}/%3d.png {vid}'
os.system(f'{command}')
stack = os.path.join(save_root, f'query_vs_ref_{start}-{length}-{K}.mp4')
command = f'ffmpeg -y -i {vids[0]} -i {vids[1]} -filter_complex hstack {stack}'
os.system(f'{command}')
def image_saver(savepath, image):
im_adjust = adjust_support(image, "0->255", clip=True)
mode = "RGB" if im_adjust.shape[-1] in [1, 3] else "RGBA"
im = Image.fromarray(im_adjust, mode)
im.save(savepath)
def get_example(self, idx: object) -> object:
"""
Args:
idx: integer indicating index of dataset
Returns: example element from dataset
"""
example = super().get_example(idx)
# Images are loaded with support from 0->255
output = {}
if self.config.get("image_type", "") == "mask":
image_p0a0 = example["p0a0_masked_frames"]()
image_p0a1 = example["p0a1_masked_frames"]()
image_p1a0 = example["p1a0_masked_frames"]()
image_p1a1 = example["p1a1_masked_frames"]()
elif self.config.get("image_type", "") == "white":
image_p0a0 = example["p0a0_whitened_frames"]()
image_p0a1 = example["p0a1_whitened_frames"]()
image_p1a0 = example["p1a0_whitened_frames"]()
image_p1a1 = example["p1a1_whitened_frames"]()
else:
image_p0a0 = example["p0a0_frames"]()
image_p0a1 = example["p0a1_frames"]()
image_p1a0 = example["p1a0_frames"]()
image_p1a1 = example["p1a1_frames"]()
if self.augmentation:
# randomly perform some augmentations on the image, keypoints and bboxes
image_p0a0 = self.seq(image=image_p0a0)
image_p0a1 = self.seq(image=image_p0a1)
image_p1a1 = self.seq(image=image_p1a0)
image_p1a1 = self.seq(image=image_p1a1)
image_p0a0 = self.resize(image=image_p0a0)
image_p0a1 = self.resize(image=image_p0a1)
image_p1a0 = self.resize(image=image_p1a0)
image_p1a1 = self.resize(image=image_p1a1)
output["inp0"] = adjust_support(image_p0a0, "0->1") # p0a0
output["inp1"] = adjust_support(image_p0a1, "0->1") # p0a1
output["inp2"] = adjust_support(image_p1a0, "0->1") # p1a0
output["inp3"] = adjust_support(image_p1a1, "0->1") # p1a1
output["animal_class"] = np.array(animal_class[self.data.data.animal])
return output
def rgb2openpose(image):
'''Prepares an image to be interpreted by openpose:
RGB->BGR, supp.: 0->1
'''
image = np.stack([image[..., 2], image[..., 1], image[..., 0]], axis=-1)
image = adjust_support(image, '0->255')
return image
def plot_input_target_keypoints(inputs: np.ndarray, targets, gt_coords,
coords):
"""
Remember to clip output numpy array to [0, 255] range and cast it to uint8.
Otherwise matplot.pyplot.imshow would show weird results.
Args:
inputs:
targets:
gt_coords:
Returns:
"""
fig = plt.figure(figsize=(10, 10))
# heatmaps_to_coords needs [batch_size, num_joints, height, width]
# coords, _ = heatmaps_to_coords(targets)
coords = sure_to_numpy(coords.clone())
for idx in range(8):
fig.add_subplot(4, 2, idx + 1)
fig.suptitle('Blue: GT, Red: Predicted')
if inputs[idx].shape[-1] == 1:
plt.imshow(adjust_support(inputs[idx].squeeze(-1), "0->255"))
else:
plt.imshow(adjust_support(inputs[idx], "0->255"))
mask = np.ones(20).astype(bool)
for kpt in range(0, len(coords[0])):
if (gt_coords[idx][:, :2][kpt] == [0, 0]).all():
mask[kpt] = False
# If gt_coords are 0,0 meaning not present in the dataset, don't draw them.
continue
plt.plot([
np.array(gt_coords[idx][:, :2][kpt][0]),
np.array(coords[idx][kpt][0])
], [
np.array(gt_coords[idx][:, :2][kpt][1]),
np.array(coords[idx][kpt][1])
],
'bx-',
alpha=0.3)
plt.scatter(gt_coords[idx][mask][:, 0],
gt_coords[idx][mask][:, 1],
c="blue")
plt.scatter(coords[idx][mask][:, 0], coords[idx][mask][:, 1], c="red")
return fig
def ssim(root,
data_in,
data_out,
config,
im_in_key='image',
im_out_key='image',
name='ssim'):
data_range = retrieve(config, 'ssim_cb/data_range', default='None')
if data_range == 'None':
data_range = None
im_shape = np.shape(retrieve(data_in[0], im_in_key))
multichannel = len(im_shape) == 3 and im_shape[-1] > 1
ssims = []
for i in trange(len(data_in), desc=name):
im_targ = retrieve(data_in[i], im_in_key)
im_gen = retrieve(data_out[i], im_out_key)
im_targ = adjust_support(im_targ, '0->1')
im_gen = adjust_support(im_gen, '0->1')
similiarity = sk_ssim(im_targ,
im_gen,
data_range=data_range,
multichannel=multichannel)
ssims += [similiarity]
ssims = np.array(ssims)
mean_ssim = np.mean(ssims)
std_ssim = np.std(ssims)
print('\n{}: {:4.3f} +- {:4.3}'.format(name, mean_ssim, std_ssim))
save_root = os.path.join(root, name)
os.makedirs(save_root, exist_ok=True)
save_name = os.path.join(save_root, 'vals.npz')
np.savez(save_name, ssims=ssims, mean=mean_ssim, std=std_ssim)
def log_op():
logs = {
"images": {
"image_input_0": adjust_support(torch2numpy(inputs0).transpose(0, 2, 3, 1), "-1->1", "0->1"),
"disentanglement": adjust_support(np.expand_dims(generate_samples(inputs0, model), 0), "-1->1",
"0->1"),
"pose_random": adjust_support(np.expand_dims(generate_samples(inputs0, model, pose_random=True), 0),
"-1->1",
"0->1"),
"appearance_random": adjust_support(
np.expand_dims(generate_samples(inputs0, model, appreance_random=True), 0), "-1->1",
"0->1"),
"outputs": adjust_support(torch2numpy(predictions).transpose(0, 2, 3, 1), "-1->1", "0->1"),
},
"scalars": {
"loss": loss,
},
}
logs["images"]["image_input_1_flipped"] = adjust_support(
torch2numpy(inputs1_flipped).transpose(0, 2, 3, 1), "-1->1", "0->1")
if self.get_global_step() >= self.config["LossConstrained"]["no_kl_for"] and \
self.config["LossConstrained"]["active"]:
logs["scalars"]["lambda_"] = log["scalars"]["lambda_"]
logs["scalars"]["gain"] = log["scalars"]["gain"]
logs["scalars"]["active"] = log["scalars"]["active"]
logs["scalars"]["kl_loss"] = log["scalars"]["kl_loss"]
logs["scalars"]["nll_loss"] = log["scalars"]["nll_loss"]
logs["scalars"]["rec_loss"] = log["scalars"]["rec_loss"]
logs["scalars"]["mu"] = log["scalars"]["mu"]
logs["scalars"]["eps"] = log["scalars"]["eps"]
if self.encoder_2:
logs["images"]["image_input_1"] = adjust_support(torch2numpy(inputs1).transpose(0, 2, 3, 1), "-1->1",
"0->1")
logs["images"]["image_input_1_flipped"] = adjust_support(
torch2numpy(inputs1_flipped).transpose(0, 2, 3, 1), "-1->1", "0->1")
logs["images"]["pose_reconstruction"] = adjust_support(
torch2numpy(kl_test_preds).transpose(0, 2, 3, 1), "-1->1", "0->1")
if retrieve(self.config, "classifier/active", default=False):
logs["scalars"]["accuracy"] = accuracy
if self.encoder_2:
logs["scalars"]["hog"] = np.array(hog_values).mean()
logs["scalars"]["hog_std"] = np.array(hog_values).std()
logs["scalars"]["hist"] = np.array(hist_values).mean()
logs["scalars"]["hist_std"] = np.array(hist_values).std()
return logs
def resize(image, size, points=None, prev_size=None):
'''Arguments:
image (np.ndarray or PIL.Image): Image to crop.
size (int or list): Size of the image after resize.
points (np.ndarray): Optional set of points in image coordinate, which
are translated to box coordinates. Shape: ``[(*), 2]``.
prev_size (int or list): Used to calculate the scaling for the points.
If not given, this is estimated from the shape of the image.
Returns:
np.ndarray: Resized image with shape ``[W, H, C]`` and same support
as :attr:`image`.
If points is not None:
np.ndarray: The translated point coordinates.
'''
in_support = get_support(image)
if prev_size is None:
prev_size = image.shape[:2]
else:
if isinstance(prev_size, int):
prev_size = [prev_size] * 2
prev_size = np.array(prev_size)
if not isinstance(image, Image.Image):
image = adjust_support(image, '0->255')
image = Image.fromarray(image)
if isinstance(size, int):
size = [size] * 2
image = image.resize(size)
size = np.array(size)
image = adjust_support(np.array(image), in_support)
if points is not None:
points[..., :2] = points[..., :2] * size / prev_size
return image, points
return image
def plot_kps(datum, idx=-1):
import matplotlib.pyplot as plt
im = datum['image']
kps = datum['keypoints']
f, ax = plt.subplots(1, 1)
ax.imshow(adjust_support(im, '0->1', clip=True))
ax.scatter(kps[:, 0], kps[:, 1])
f.savefig('kps-{}.png'.format(idx))
def get_example(self, idx):
example = self.MP[idx]
example['fid'] = self.labels['fid'][idx]
example['pid'] = self.labels['pid'][idx]
example['vid'] = example['video_path']
example['box'] = example['bbox']
example['orig_path'] = example['frame_path']
example['keypoints'] = self.labels['keypoints'][idx]
if self.load_images:
crop_im = Image.open(example['crop_path'])
crop_im = crop_im.resize(self.crop_size)
crop_im = np.array(crop_im)
crop_im = adjust_support(crop_im, '-1->1', '0->255')
example['crop_im'] = crop_im
m_path = example['mask_path']
if os.path.exists(m_path):
mask_im = Image.open(m_path)
mask_im = mask_im.resize(self.crop_size)
mask_im = np.array(mask_im)
mask_im = adjust_support(mask_im, '-1->1')
else:
mask_im = np.ones(crop_im.shape[:2])
if len(mask_im.shape) == 2:
mask_im = np.expand_dims(mask_im, -1)
example['mask'] = mask_im
if self.apply_mask:
example['target'] = np.concatenate([crop_im, mask_im], axis=-1)
else:
example['target'] = crop_im
example['flow'] = None
return example
def get_example(self, idx):
target = self.base[idx]['crop']()
pose = self.base.labels['kps_rel'][idx]
pose = pose * np.array(target.shape[:2])[None]
stickman = kp2stick(pose, size=[256, 256], kp_model=OPENPOSE_18)
stickman = adjust_support(stickman, '-1->1', '0->255')
app_idx = self.prng.choice(len(self.base))
appearance = self.base[app_idx]['crop']
return {'stickman': stickman, 'appearance': appearance, 'target': target}
def log_op():
logs = {
"images": {
"p0a0": adjust_support(torch2numpy(inputs0).transpose(0, 2, 3, 1), "-1->1", "0->1"),
"p0a1": adjust_support(torch2numpy(inputs1).transpose(0, 2, 3, 1), "-1->1", "0->1"),
"p1a0": adjust_support(torch2numpy(inputs2).transpose(0, 2, 3, 1), "-1->1", "0->1"),
"p1a1": adjust_support(torch2numpy(inputs3).transpose(0, 2, 3, 1), "-1->1", "0->1"),
"disentanglement": adjust_support(torch2numpy(predictions).transpose(0, 2, 3, 1), "-1->1", "0->1"),
"testing": adjust_support(torch2numpy(testing).transpose(0, 2, 3, 1), "-1->1", "0->1"),
},
"scalars": {
"loss": loss,
"loss_dis": loss_dis,
},
}
logs["scalars"]["lambda_"] = log["scalars"]["lambda_"]
logs["scalars"]["gain"] = log["scalars"]["gain"]
logs["scalars"]["active"] = log["scalars"]["active"]
logs["scalars"]["kl_loss"] = log["scalars"]["kl_loss"]
logs["scalars"]["nll_loss"] = log["scalars"]["nll_loss"]
logs["scalars"]["rec_loss"] = log["scalars"]["rec_loss"]
logs["scalars"]["mu"] = log["scalars"]["mu"]
logs["scalars"]["eps"] = log["scalars"]["eps"]
return logs
def eval_op(): # np.concatenate((np.expand_dims(generate_samples(inputs0, model)),0),)*5)
from datetime import datetime
now = datetime.now()
mystring = now.strftime("%m-%d-%Y-%H-%M-%S")
n_samples = 4
out, hog, cosine_distances, hist_similarities, cosine_row, hist_column = generate_samples(inputs0, model,
n_samples)
out_pose, hog_pose, cosine_distances_pose, hist_similarities_pose, cosine_row_pose, hist_column_pose = generate_samples(
inputs0, model, n_samples, pose_random=True, )
out_appearance, hog_appearance, cosine_distances_appearance, hist_similarities_appearance, cosine_row_appearance, hist_column_appearance = generate_samples(
inputs0, model, n_samples, appreance_random=True)
export_path = "/export/home/ffeldman/tempfolder/"
save_hog_image(hog, out, n_samples, cosine_distances, cosine_row, export_path, mystring)
save_hist_image(out, n_samples, hist_similarities, hist_column, export_path, mystring)
logs = {
#"image_input_0": adjust_support(torch2numpy(inputs0).transpose(0, 2, 3, 1), "-1->1", "0->1"),
#"inputs1_flipped": adjust_support(torch2numpy(inputs1_flipped).transpose(0, 2, 3, 1), "-1->1", "0->1"),
"disentanglement": adjust_support(
np.concatenate(
(np.expand_dims(out.transpose(1, 2, 0), 0),) * inputs0.size(0)),
"-1->1", "0->1"),
"pose_random": adjust_support(
np.concatenate((
np.expand_dims(
out_pose.transpose(1, 2, 0),
0),) * inputs0.size(0)),
"-1->1", "0->1"),
"appearance_random": adjust_support(
np.concatenate((np.expand_dims(
out_appearance.transpose(1, 2, 0), 0),) * inputs0.size(
0)),
"-1->1", "0->1"),
#"test_preds": adjust_support(torch2numpy(kl_test_preds).transpose(0, 2, 3, 1), "-1->1", "0->1"),
}
return logs
def hog_similarity(img1, img2=False, PLOT=False):
orientations = 8
pixels_per_cell = (16, 16)
img1 = adjust_support(img1, "0->255", "0->1")
hog_inp, hog_img_inp = skimage.feature.hog(img1,
orientations=orientations,
pixels_per_cell=pixels_per_cell,
visualize=True,
feature_vector=True)
hog_img_inp = skimage.exposure.rescale_intensity(hog_img_inp,
in_range=(0, 10))
if type(img2) == np.ndarray:
img2 = adjust_support(img2, "0->255", "0->1")
hog_dis, hog_img_dis = skimage.feature.hog(
img2,
orientations=orientations,
pixels_per_cell=pixels_per_cell,
# cells_per_block=(1, 1),
visualize=True,
feature_vector=True)
hog_img_dis = skimage.exposure.rescale_intensity(hog_img_dis,
in_range=(0, 10))
value = 1 - scipy.spatial.distance.cosine(hog_inp, hog_dis)
if PLOT:
fig, (ax1, ax2, ax3, ax4) = plt.subplots(1, 4)
plt.title(value)
ax1.imshow(
np.stack((hog_img_inp, ) * 3, axis=0).transpose(1, 2, 0)[:, :,
0])
ax2.imshow(hog_img_dis)
ax3.imshow(adjust_support(img1, "0->1"))
ax4.imshow(adjust_support(img2, "0->1"))
plt.show()
return value, [hog_img_inp, hog_img_dis]
return hog_img_inp
def loader(support=support, resize_to=resize_to):
im = Image.open(path)
if resize_to is not None:
if isinstance(resize_to, int):
resize_to = [resize_to] * 2
im = im.resize(resize_to)
im = np.array(im)
if support == "0->255":
return im
else:
return adjust_support(im, support, "0->255")
def run_through_dataset(
dataset, support: str = "-1->1", batch_size: int = 50, image_key: str = "image", model_name=""
):
"""
:param dataset: DatasetMixin which contains the images.
:param support: Support of images. One of '-1->1', '0->1' or '0->255'
:param batch_size: The images numpy array is split into batches with batch size
batch_size. A reasonable batch size depends on the disposable hardware.
:param image_key: Dataset key containing the image to be embedded
:return: np.ndarray embedding.
"""
dataset_length = len(dataset)
if batch_size > dataset_length:
print(Warning("Setting batch size to length of the dataset.."))
batch_size = dataset_length
batches = make_batches(dataset, batch_size, shuffle=False)
n_batches = len(batches)
n_used_imgs = n_batches * batch_size
embeddings = [] # np.empty((n_used_imgs, 128))
labels = []
sess_config = tf.compat.v1.ConfigProto()
sess_config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=sess_config)
model = reIdModel(model_name=model_name)
if os.path.basename(TRIP_CHECK) == "checkpoint-25000":
initialize_model(model, TRIP_CHECK, session)
for i, batch in enumerate(tqdm(batches, desc="reID")):
if i >= n_batches:
break
images = retrieve(batch, image_key)
labels_batch = retrieve(batch, "pose_pid")
images = adjust_support(
np.array(images), future_support="0->255", current_support=support, clip=True
)
images = images.astype(np.float32)[..., :3]
batch_embeddings = get_embedding(model, session=session, image=images)["emb"]
embeddings += [batch_embeddings.reshape(batch_size, -1)]
labels += [labels_batch]
batches.finalize()
return np.array(embeddings), np.array(labels)
def log_op():
from AnimalPose.utils.log_utils import plot_pred_figure
from edflow.data.util import adjust_support
logs = {
"images": {
"image_input": adjust_support(torch2numpy(inputs).transpose(0, 2, 3, 1), "-1->1"),
},
"scalars": {
"loss": losses["batch"]["total"],
"accuracy": accuracy.cpu().numpy(),
},
"figures": {
"predictions": plot_pred_figure(inputs, preds.cpu().detach().numpy(), labels)
}
}
if self.config["losses"]["CEL"]:
logs["scalars"]["CrossEntropyLoss"] = losses["batch"]["CEL"]
return logs
def image_plot(start):
N = 20
K = 100
HG = HumanGaitFixedBox({'data_split': 'train'})
edprint(HG.labels)
kps = HG.labels['kps_fixed_rel'][..., :2].astype('float32')
print(kps)
kp_hidden = kps[:int(0.84 * len(HG))]
R = ReferenceSampler(kp_hidden, k=K)
start_ = int(0.84 * len(HG))
start += start_
end = start + N
q_idxs = list(range(start, end))
print(q_idxs)
q = kps[start:end]
opt = R(q)
print(opt)
f, AX = plt.subplots(2,
len(opt),
figsize=[N / 10 * 12.8, 7.2],
dpi=100,
constrained_layout=True)
HG.expand = True
for Ax, indices in zip(AX, [q_idxs, opt]):
for ax, idx in zip(Ax, indices):
im = adjust_support(HG[idx]['target'], '0->1')
ax.imshow(im)
ax.axis('off')
f.savefig(f'viterbi_{start}.pdf')
def log_op():
from edflow.data.util import adjust_support
logs = {
"images": {
"inputs0":
adjust_support(
torch2numpy(inputs0).transpose(0, 2, 3, 1), "-1->1",
"0->1"),
"inputs1":
adjust_support(
torch2numpy(inputs1).transpose(0, 2, 3, 1), "-1->1",
"0->1"),
"pred0":
adjust_support(
torch2numpy(pred0).transpose(0, 2, 3, 1), "-1->1",
"0->1"),
"pred1":
adjust_support(
torch2numpy(pred1).transpose(0, 2, 3, 1), "-1->1",
"0->1"),
"mixed_reconstruction":
adjust_support(
torch2numpy(mixed_reconstruction).transpose(
0, 2, 3, 1), "-1->1", "0->1"),
"input0_flipped":
adjust_support(
torch2numpy(torch.flip(inputs0,
[0])).transpose(0, 2, 3, 1),
"-1->1", "0->1"),
"flip_test":
adjust_support(
torch2numpy(flip_test).transpose(0, 2, 3, 1), "-1->1",
"0->1"),
},
"scalars": {
"loss":
loss_appearance + loss_pose + loss_mixed_reconstruction,
"loss_appearance": loss_appearance,
"loss_pose": loss_pose,
"loss_mixed_reconstruction": loss_mixed_reconstruction,
},
}
return logs
def test_new_crop(d, idx):
from multiperson_dataset import square_bbox, get_kps_rel
kps = d['keypoints_abs']
box = square_bbox(kps)
kps_rel = get_kps_rel(kps, box)
image = np.array(Image.open(d['frame_path']))
im_crop = crop(image, box)
im_crop = adjust_support(im_crop, '0->255')
im_crop = Image.fromarray(im_crop)
im_crop.save('croptest_{}.png'.format(idx), 'PNG')
d['keypoints'] = kps
d['im_crop'] = im_crop
plot_kps(d, idx)
def plot_outcome(D, opt, q_idxs, crop_key, full_name):
import matplotlib.pyplot as plt
from edflow.data.util import adjust_support
f, AX = plt.subplots(2, len(opt),
figsize=[len(opt)/10*12.8, 7.2],
dpi=100,
constrained_layout=True)
D.expand = True
for i, [Ax, indices] in enumerate(zip(AX, [q_idxs, opt])):
for ax, idx in zip(Ax, indices):
D.base.loader_kwargs['target']['root'] = '/home/jhaux/remote/cg2'
ex = D[idx]
crop = D[idx][crop_key]
im = adjust_support(crop, '0->1')
ax.imshow(im)
ax.axis('off')
ax.set_title(f'{"Q" if i == 0 else "R"}: {idx}')
f.savefig(f'{full_name}.pdf')
def plot_pred_figure(images, predictions, labels=None):
"""
Remember to clip output numpy array to [0, 255] range and cast it to uint8.
Otherwise matplot.pyplot.imshow would show weird results.
Args:
images:
predictions:
Returns:
"""
from AnimalPose.data.animals_VOC2011 import animal_class
idx_to_animal = {v: k for k, v in animal_class.items()}
fig = plt.figure(figsize=(10, 10))
for idx in range(8):
fig.add_subplot(4, 2, idx + 1)
fig.suptitle('Input, Prediction')
if labels != None:
plt.title(f"GT:{labels[idx]}, Pred: {predictions[idx]}")
else:
plt.title(f"{idx_to_animal[predictions[idx]]}")
plt.imshow(
adjust_support(images[idx].cpu().numpy().transpose(1, 2, 0),
"0->1", "0->1"))
plt.tight_layout()
return fig
def image_saver(savepath, image):
"""
Parameters
----------
savepath :
image :
Returns
-------
"""
im_adjust = adjust_support(image, "0->255", clip=True)
modes = {1: "L", 3: "RGB", 4: "RGBA"}
mode = modes[im_adjust.shape[-1]]
if mode == "L":
im_adjust = np.squeeze(im_adjust, -1)
im = Image.fromarray(im_adjust, mode)
im.save(savepath)
def log_op():
from edflow.data.util import adjust_support
#PCK_THRESH = [0.01, 0.025, 0.05, 0.1, 0.125, 0.15, 0.175, 0.2, 0.25, 0.5]
#if self.config['pck_alpha'] not in PCK_THRESH: PCK_THRESH.append(self.config["pck_alpha"])
#coords, _ = heatmaps_to_coords(pose_predictions.clone(), thresh=self.config["hm"]["thresh"])
#pck = {t: percentage_correct_keypoints(kwargs["kps"], coords, t, self.config["pck"]["type"]) for t in
# PCK_THRESH}
logs = {
"images": {
"inputs0":
adjust_support(
torch2numpy(inputs0).transpose(0, 2, 3, 1), "-1->1",
"0->1"),
"inputs1":
adjust_support(
torch2numpy(inputs1).transpose(0, 2, 3, 1), "-1->1",
"0->1"),
"pred0":
adjust_support(
torch2numpy(pred0).transpose(0, 2, 3, 1), "-1->1",
"0->1"),
"pred1":
adjust_support(
torch2numpy(pred1).transpose(0, 2, 3, 1), "-1->1",
"0->1"),
"mixed_reconstruction":
adjust_support(
torch2numpy(mixed_reconstruction).transpose(
0, 2, 3, 1), "-1->1", "0->1"),
"cycle0":
adjust_support(
torch2numpy(pose_recon).transpose(0, 2, 3, 1), "-1->1",
"0->1"),
"cycle1":
adjust_support(
torch2numpy(appearance_recon).transpose(0, 2, 3, 1),
"-1->1", "0->1"),
#"targets": adjust_support(heatmaps_to_image(kwargs["targets0"]).transpose(0, 2, 3, 1), "-1->1"),
#"inputs_with_stick": make_stickanimal(torch2numpy(inputs).transpose(0, 2, 3, 1), kwargs["kps0"]),
#"stickanimal": make_stickanimal(torch2numpy(inputs).transpose(0, 2, 3, 1), pose_predictions.clone(),
# thresh=self.config["hm"]["thresh"]),
#"pred_hm": adjust_support(heatmaps_to_image(torch2numpy(pose_predictions)).transpose(0, 2, 3, 1),
# "-1->1", "0->1"),
},
"scalars": {
"loss": autoencoder_loss + cycle_loss, # + pose_loss,
"loss_autoencoder": autoencoder_loss,
"recon_loss": cycle_loss,
#"pose_loss": pose_loss,
#f"PCK@{self.config['pck_alpha']}": np.around(pck[self.config['pck_alpha']][0], 5),
},
}
# if self.config["pck"]["pck_multi"]:
# for key, val in pck.items():
# # get mean value for pck at given threshold
# logs["scalars"][f"PCK@{key}"] = np.around(val[0], 5)
# for idx, part in enumerate(val[1]):
# logs["scalars"][f"PCK@{key}_{self.dataset.get_idx_parts(idx)}"] = np.around(part, 5)
# gridded_outputs = np.expand_dims(sure_to_numpy(torchvision.utils.make_grid(
# predictions[0], nrow=10)), 0).transpose(1, 2, 3, 0)
# gridded_targets = np.expand_dims(sure_to_numpy(torchvision.utils.make_grid(
# sure_to_torch(kwargs["targets"])[0], nrow=10)), 0).transpose(1, 2, 3, 0)
#
# logs = {
# "images": {
# # Image input not needed, because stick animal is printed on input image
# # "image_input": adjust_support(torch2numpy(inputs).transpose(0, 2, 3, 1), "-1->1"),
# #"first_pred": adjust_support(gridded_outputs, "-1->1", "0->1"),
# #"first_targets": adjust_support(gridded_targets, "-1->1", "0->1"),
# "outputs": adjust_support(heatmaps_to_image(torch2numpy(predictions)).transpose(0, 2, 3, 1),
# "-1->1", "0->1"),
# },
# "scalars": {
# "loss": losses["total"],
# "learning_rate": self.optimizer.state_dict()["param_groups"][0]["lr"],
# f"PCK@{self.config['pck_alpha']}": np.around(pck[self.config['pck_alpha']][0], 5),
# },
# "figures": {
# "Keypoint Mapping": plot_input_target_keypoints(torch2numpy(inputs).transpose(0, 2, 3, 1),
# # get BHWC
# torch2numpy(predictions), # stay BCHW
# kwargs["kps"], coords),
# }
# }
# if self.config["losses"]["L2"]:
# logs["scalars"]["L2"] = losses["L2"]
# if self.config["losses"]["L1"]:
# logs["scalars"]["L1"] = losses["L1"]
# if self.config["losses"]["perceptual"]:
# logs["scalars"]["perceptual"] = losses["perceptual"]
#
return logs
