def __init__(self):
self.model_pose = PoseEstimationModel()
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.model_pose.to(self.device, dtype=torch.float)
self.model_pose.load_ckpt(allow_new=False)
self.model_pose.eval()
def __init__(self, batch_size, is_unittest=False):
self.is_unittest = is_unittest
self.epochs = 5
self.val_step = 500
self.batch_size = batch_size
self.vis = visdom.Visdom()
self.img_key = HK.NORM_IMAGE
self.pcm_key = HK.PCM_ALL
self.paf_key = HK.PAF_ALL
if torch.cuda.is_available():
print("GPU available.")
else:
print("GPU not available. running with CPU.")
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.model_pose = PoseEstimationModel()
self.model_optimizer = optim.Adam(self.model_pose.parameters(),
lr=1e-3)
self.loss = PAFsLoss()
train_dataset = AicNorm(Path.home() / "AI_challenger_keypoint",
is_train=True,
resize_img_size=(512, 512),
heat_size=(64, 64))
self.train_loader = DataLoader(train_dataset,
self.batch_size,
shuffle=True,
num_workers=settings.num_workers,
pin_memory=True,
drop_last=True)
def __init__(self, batch_size, debug_mode):
# self.debug_mode: Set num_worker to 0. Otherwise pycharm debug won't work due to multithreading.
self.debug_mode = debug_mode
self.epochs = 5
self.val_step = 500
self.batch_size = batch_size
self.vis = visdom.Visdom()
self.img_key = HK.NORM_IMAGE
self.pcm_key = HK.PCM_ALL
self.paf_key = HK.PAF_ALL
if torch.cuda.is_available():
print("GPU available.")
else:
print("GPU not available. running with CPU.")
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.model_pose = PoseEstimationModel()
self.model_optimizer = optim.Adam(self.model_pose.parameters(),
lr=1e-3)
self.loss = PAFsLoss()
if self.debug_mode:
# self.batch_size = 1
workers = 0
else:
# self.batch_size = batch_size
workers = 4
train_dataset = AicNorm(Path.home() / "AI_challenger_keypoint",
is_train=True,
resize_img_size=(512, 512),
heat_size=(64, 64))
self.train_loader = DataLoader(train_dataset,
self.batch_size,
shuffle=True,
num_workers=workers,
pin_memory=True,
drop_last=True)
test_dataset = AicNorm(Path.home() / "AI_challenger_keypoint",
is_train=False,
resize_img_size=(512, 512),
heat_size=(64, 64))
self.val_loader = DataLoader(test_dataset,
self.batch_size,
shuffle=False,
num_workers=workers,
pin_memory=True,
drop_last=True)
self.val_iter = iter(self.val_loader)
class HumanKeypointPredict:
def __init__(self):
self.model_pose = PoseEstimationModel()
self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
self.model_pose.to(self.device, dtype=torch.float)
self.model_pose.load_ckpt(allow_new=False)
self.model_pose.eval()
def get_heatmaps(self, u8_image: np.ndarray):
assert len(u8_image.shape) == 3, "expect npy image of shape (H, W, C)"
assert u8_image.dtype == np.uint8, "expect uint8 image"
norm_img = u8_image.astype(np.float32) / 255.
norm_img = np.transpose(norm_img, axes=(2, 0, 1))[np.newaxis]
norm_img = torch.from_numpy(norm_img)
norm_img = norm_img.to(self.device)
with torch.no_grad():
res = self.model_pose(norm_img)
del res[HK.B1_SUPERVISION]
del res[HK.B2_SUPERVISION]
return res
def get_coordinates(self, norm_img: np.ndarray):
res = self.get_heatmaps(norm_img)
b1, b2 = res[HK.B1_OUT][0].cpu().numpy(), res[HK.B2_OUT][0].cpu().numpy()
chw_shape = b1.shape
max_indices = [np.argmax(hw, axis=None) for hw in b1]
ys, xs = np.unravel_index(max_indices, b1[0].shape)
xs_norm, ys_norm = xs / chw_shape[2], ys / chw_shape[1]
results = {PG.COORD_NATIVE: np.array((xs, ys)), PG.COORD_NORM: np.array((xs_norm, ys_norm))}
return results
class HumanKeypointPredict:
def __init__(self):
self.model_pose = PoseEstimationModel()
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.model_pose.to(self.device, dtype=torch.float)
self.model_pose.load_ckpt(allow_new=False)
self.model_pose.eval()
def get_heatmaps(self, u8_image: np.ndarray):
"""
Get PCM and PAF heatmap
:param u8_image:
:return: {HK.B1_OUT, HK.B2_OUT}
"""
assert len(u8_image.shape) == 3, "expect npy image of shape (H, W, C)"
assert u8_image.dtype == np.uint8, "expect uint8 image"
norm_img = u8_image.astype(np.float32) / 255.
norm_img = np.transpose(norm_img,
axes=(2, 0, 1))[np.newaxis] # HWC->NCHW
norm_img = torch.from_numpy(norm_img)
norm_img = norm_img.to(self.device)
with torch.no_grad():
res = self.model_pose(norm_img)
del res[HK.B1_SUPERVISION]
del res[HK.B2_SUPERVISION]
return res
def get_coordinates(self, norm_img: np.ndarray):
# Output coordinates and norm_coordinates
res = self.get_heatmaps(norm_img)
b1, b2 = res[HK.B1_OUT][0].cpu().numpy(), res[
HK.B2_OUT][0].cpu().numpy()
chw_shape = b1.shape
# TODO: Separate points for multiple person and compute mass center:
# TODO: x = 1/(w*h) * integral_0^h integral_0^w u*Val(u,v) du dv
max_indices = [np.argmax(hw, axis=None) for hw in b1]
ys, xs = np.unravel_index(
max_indices, b1[0].shape) # array([y, y, y])), (array([x, x, x])
xs_norm, ys_norm = xs / chw_shape[2], ys / chw_shape[1]
# PG.COORD_NATIVE shape: (xy(2), num_keypoints)
results = {
PG.COORD_NATIVE: np.array((xs, ys)),
PG.COORD_NORM: np.array((xs_norm, ys_norm))
}
return results
class Trainer:
def __init__(self, batch_size, is_unittest=False):
self.is_unittest = is_unittest
self.epochs = 5
self.val_step = 500
self.batch_size = batch_size
self.vis = visdom.Visdom()
self.img_key = HK.NORM_IMAGE
self.pcm_key = HK.PCM_ALL
self.paf_key = HK.PAF_ALL
if torch.cuda.is_available():
print("GPU available.")
else:
print("GPU not available. running with CPU.")
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.model_pose = PoseEstimationModel()
self.model_optimizer = optim.Adam(self.model_pose.parameters(),
lr=1e-3)
self.loss = PAFsLoss()
train_dataset = AicNorm(Path.home() / "AI_challenger_keypoint",
is_train=True,
resize_img_size=(512, 512),
heat_size=(64, 64))
self.train_loader = DataLoader(train_dataset,
self.batch_size,
shuffle=True,
num_workers=settings.num_workers,
pin_memory=True,
drop_last=True)
def set_train(self):
self.model_pose.train()
def set_eval(self):
self.model_pose.eval()
def train(self):
self.epoch = 0
self.step = 0
self.model_pose.load_ckpt()
for self.epoch in range(self.epochs):
print("Epoch:{}".format(self.epoch))
self.run_epoch()
if self.is_unittest:
break
def run_epoch(self):
self.set_train()
for batch_idx, inputs in enumerate(self.train_loader):
inputs[self.img_key] = inputs[self.img_key].to(self.device,
dtype=torch.float32)
inputs[self.pcm_key] = inputs[self.pcm_key].to(self.device,
dtype=torch.float32)
inputs[self.paf_key] = inputs[self.paf_key].to(self.device,
dtype=torch.float32)
loss, b1_out, b2_out = self.process_batch(inputs)
self.model_optimizer.zero_grad()
loss.backward()
self.model_optimizer.step()
if self.step == 0:
self.vis.close()
if self.step % self.val_step == 0 and self.step != 0:
self.model_pose.save_ckpt()
if self.step % 50 == 0:
print("step {}; Loss {}".format(self.step, loss.item()))
if self.step % self.val_step == 0:
pcm_CHW = b1_out[0].cpu().detach().numpy()
paf_CHW = b2_out[0].cpu().detach().numpy()
img_CHW = inputs[self.img_key][0].cpu().detach().numpy()[::-1,
...]
pred_pcm_amax = np.amax(pcm_CHW, axis=0)
gt_pcm_amax = np.amax(
inputs[self.pcm_key][0].cpu().detach().numpy(), axis=0)
pred_paf_amax = np.amax(paf_CHW, axis=0)
gt_paf_amax = np.amax(
inputs[self.paf_key][0].cpu().detach().numpy(), axis=0)
self.vis.image(img_CHW, win="Input", opts={'title': "Input"})
self.vis.heatmap(np.flipud(pred_pcm_amax),
win="Pred-PCM",
opts={'title': "Pred-PCM"})
self.vis.heatmap(np.flipud(gt_pcm_amax),
win="GT-PCM",
opts={'title': "GT-PCM"})
self.vis.heatmap(np.flipud(pred_paf_amax),
win="Pred-PAF",
opts={'title': "Pred-PAF"})
self.vis.heatmap(np.flipud(gt_paf_amax),
win="GT-PAF",
opts={'title': "GT-PAF"})
self.vis.line(X=np.array([self.step]),
Y=loss.cpu().detach().numpy()[np.newaxis],
win='Loss',
update='append')
if self.is_unittest:
break
self.step += 1
def process_batch(self, inputs):
res = self.model_pose(inputs[self.img_key])
gt_pcm = inputs[self.pcm_key]
gt_pcm = gt_pcm.unsqueeze(1)
gt_paf = inputs[self.paf_key]
gt_paf = gt_paf.unsqueeze(1)
b1_stack = torch.stack(res[HK.B1_SUPERVISION], dim=1)
b2_stack = torch.stack(res[HK.B2_SUPERVISION], dim=1)
loss = self.loss(b1_stack, b2_stack, gt_pcm, gt_paf)
return loss, res[HK.B1_OUT], res[HK.B2_OUT]
class Trainer:
def __init__(self, batch_size, is_unittest=False):
# self.debug_mode: Set num_worker to 0. Otherwise pycharm debug won't work due to multithreading.
self.is_unittest = is_unittest
self.epochs = 5
self.val_step = 500
self.batch_size = batch_size
self.vis = visdom.Visdom()
self.img_key = HK.NORM_IMAGE
self.pcm_key = HK.PCM_ALL
self.paf_key = HK.PAF_ALL
if torch.cuda.is_available():
print("GPU available.")
else:
print("GPU not available. running with CPU.")
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.model_pose = PoseEstimationModel()
self.model_optimizer = optim.Adam(self.model_pose.parameters(),
lr=1e-3)
self.loss = PAFsLoss()
train_dataset = AicNorm(Path.home() / "AI_challenger_keypoint",
is_train=True,
resize_img_size=(512, 512),
heat_size=(64, 64))
self.train_loader = DataLoader(train_dataset,
self.batch_size,
shuffle=True,
num_workers=settings.num_workers,
pin_memory=True,
drop_last=True)
# test_dataset = AicNorm(Path.home() / "AI_challenger_keypoint", is_train=False,
# resize_img_size=(512, 512), heat_size=(64, 64))
# self.val_loader = DataLoader(test_dataset, self.batch_size, shuffle=False, num_workers=workers, pin_memory=True,
# drop_last=True)
# self.val_iter = iter(self.val_loader)
def set_train(self):
"""Convert models to training mode
"""
self.model_pose.train()
def set_eval(self):
"""Convert models to testing/evaluation mode
"""
self.model_pose.eval()
def train(self):
self.epoch = 0
self.step = 0
self.model_pose.load_ckpt()
for self.epoch in range(self.epochs):
print("Epoch:{}".format(self.epoch))
self.run_epoch()
if self.is_unittest:
break
def run_epoch(self):
self.set_train()
for batch_idx, inputs in enumerate(self.train_loader):
inputs[self.img_key] = inputs[self.img_key].to(self.device,
dtype=torch.float32)
inputs[self.pcm_key] = inputs[self.pcm_key].to(self.device,
dtype=torch.float32)
inputs[self.paf_key] = inputs[self.paf_key].to(self.device,
dtype=torch.float32)
loss, b1_out, b2_out = self.process_batch(inputs)
self.model_optimizer.zero_grad()
loss.backward()
self.model_optimizer.step()
# Clear visdom environment
if self.step == 0:
self.vis.close()
# Validate
if self.step % self.val_step == 0 and self.step != 0:
# self.val()
self.model_pose.save_ckpt()
if self.step % 50 == 0:
print("step {}; Loss {}".format(self.step, loss.item()))
# Show training materials
if self.step % self.val_step == 0:
pcm_CHW = b1_out[0].cpu().detach().numpy()
paf_CHW = b2_out[0].cpu().detach().numpy()
img_CHW = inputs[self.img_key][0].cpu().detach().numpy()[::-1,
...]
pred_pcm_amax = np.amax(pcm_CHW, axis=0) # HW
gt_pcm_amax = np.amax(
inputs[self.pcm_key][0].cpu().detach().numpy(), axis=0)
pred_paf_amax = np.amax(paf_CHW, axis=0)
gt_paf_amax = np.amax(
inputs[self.paf_key][0].cpu().detach().numpy(), axis=0)
self.vis.image(img_CHW, win="Input", opts={'title': "Input"})
self.vis.heatmap(np.flipud(pred_pcm_amax),
win="Pred-PCM",
opts={'title': "Pred-PCM"})
self.vis.heatmap(np.flipud(gt_pcm_amax),
win="GT-PCM",
opts={'title': "GT-PCM"})
self.vis.heatmap(np.flipud(pred_paf_amax),
win="Pred-PAF",
opts={'title': "Pred-PAF"})
self.vis.heatmap(np.flipud(gt_paf_amax),
win="GT-PAF",
opts={'title': "GT-PAF"})
self.vis.line(X=np.array([self.step]),
Y=loss.cpu().detach().numpy()[np.newaxis],
win='Loss',
update='append')
if self.is_unittest:
break
self.step += 1
# def val(self, ):
# """Validate the model on a single minibatch
# """
# self.set_eval()
#
# try:
# inputs = next(self.val_iter)
# except StopIteration:
# self.val_iter = iter(self.val_loader)
# inputs = next(self.val_iter)
#
# inputs_gpu = {self.img_key: inputs[self.img_key].to(self.device, dtype=torch.float32),
# self.pcm_key: inputs[self.pcm_key].to(self.device, dtype=torch.float32),
# self.paf_key: inputs[self.paf_key].to(self.device, dtype=torch.float32)}
# with torch.no_grad():
# loss, b1_out, b2_out = self.process_batch(inputs_gpu)
# pred_pcm_amax = np.amax(b1_out[0].cpu().numpy(), axis=0) # HW
# gt_pcm_amax = np.amax(inputs[self.pcm_key][0].cpu().numpy(), axis=0)
# pred_paf_amax = np.amax(b2_out[0].cpu().numpy(), axis=0)
# gt_paf_amax = np.amax(inputs[self.paf_key][0].cpu().numpy(), axis=0)
# # Image augmentation disabled due to pred phase
# self.vis.image(inputs[self.img_key][0].cpu().numpy()[::-1, ...], win="Input", opts={'title': "Input"})
# self.vis.heatmap(np.flipud(pred_pcm_amax), win="Pred-PCM", opts={'title': "Pred-PCM"})
# self.vis.heatmap(np.flipud(gt_pcm_amax), win="GT-PCM", opts={'title': "GT-PCM"})
# self.vis.heatmap(np.flipud(pred_paf_amax), win="Pred-PAF", opts={'title': "Pred-PAF"})
# self.vis.heatmap(np.flipud(gt_paf_amax), win="GT-PAF", opts={'title': "GT-PAF"})
# self.vis.line(X=np.array([self.step]), Y=loss.cpu().numpy()[np.newaxis], win='Loss', update='append')
# self.set_train()
def process_batch(self, inputs):
"""Pass a minibatch through the network and generate images and losses
"""
res = self.model_pose(inputs[self.img_key])
gt_pcm = inputs[
self.pcm_key] # ["heatmap"]: {"vis_or_not": NJHW, "visible": NJHW}
gt_pcm = gt_pcm.unsqueeze(1)
gt_paf = inputs[self.paf_key]
gt_paf = gt_paf.unsqueeze(1)
b1_stack = torch.stack(res[HK.B1_SUPERVISION],
dim=1) # Shape (N, Stage, C, H, W)
b2_stack = torch.stack(res[HK.B2_SUPERVISION], dim=1)
loss = self.loss(b1_stack, b2_stack, gt_pcm, gt_paf)
return loss, res[HK.B1_OUT], res[HK.B2_OUT]