"""Load datasets"""
mpii = MpiiSinglePerson(datasetpath('MPII'), dataconf=mpii_dataconf,
poselayout=pa17j3d)
# h36m = Human36M(datasetpath('Human3.6M'), dataconf=human36m_dataconf,
# poselayout=pa17j3d, topology='frames')f
penn_sf = PennAction(datasetpath('Penn_Action'), pennaction_dataconf,
poselayout=pa17j3d, topology='frames', use_gt_bbox=True)
ntu_sf = Ntu(datasetpath('NTU'), ntu_pe_dataconf, poselayout=pa17j3d,
topology='frames', use_gt_bbox=True)
"""Create an object to load data from all datasets."""
data_tr = BatchLoader([mpii, penn_sf, ntu_sf], ['frame'], ['pose'],
TRAIN_MODE, batch_size=[batch_size_mpii, batch_size_ar,
batch_size_ar], num_predictions=num_predictions, shuffle=True)
"""MPII validation samples."""
mpii_val = BatchLoader(mpii, ['frame'], ['pose', 'afmat', 'headsize'],
VALID_MODE, batch_size=mpii.get_length(VALID_MODE), shuffle=True)
printcn(OKBLUE, 'Pre-loading MPII validation data...')
[x_val], [p_val, afmat_val, head_val] = mpii_val[0]
mpii_callback = MpiiEvalCallback(x_val, p_val, afmat_val, head_val,
map_to_pa16j=pa17j3d.map_to_pa16j, logdir=logdir)
# """Human3.6H validation samples."""
# h36m_val = BatchLoader(h36m, ['frame'],
# ['pose_w', 'pose_uvd', 'afmat', 'camera', 'action'], VALID_MODE,
# batch_size=h36m.get_length(VALID_MODE), shuffle=True)
# printcn(OKBLUE, 'Preloading Human3.6M validation samples...')
"""Merge pose and visibility as a single output."""
outputs = []
for b in range(int(len(model.outputs) / 2)):
outputs.append(
concatenate([model.outputs[2 * b], model.outputs[2 * b + 1]],
name='blk%d' % (b + 1)))
model = Model(model.input, outputs, name=model.name)
logger.debug("DATASETS")
"""Load the MPII dataset."""
mpii = MpiiSinglePerson('datasets/MPII', dataconf=mpii_sp_dataconf)
logger.debug(mpii.dataset_path)
"""Pre-load validation samples and generate the eval. callback."""
mpii_val = BatchLoader(mpii,
x_dictkeys=['frame'],
y_dictkeys=['pose', 'afmat', 'headsize'],
mode=VALID_MODE,
batch_size=mpii.get_length(VALID_MODE),
num_predictions=1,
shuffle=False)
logger.debug(mpii_val.datasets)
printcn(OKBLUE, 'Pre-loading MPII validation data...')
[x_val], [p_val, afmat_val, head_val] = mpii_val[0]
pose_pred = np.zeros_like(p_val)
scores, y_pred = eval_singleperson_pckh(model, x_val, p_val[:, :, 0:2],
afmat_val, head_val)
logger.info("SCORES ")
logger.debug(scores)
pose_pred = get_pred_data(mpii, pose_pred=pose_pred, pred=y_pred, mode=mode)
model.load_weights(weights_path)
"""Merge pose and visibility as a single output."""
outputs = []
for b in range(int(len(model.outputs) / 2)):
outputs.append(
concatenate([model.outputs[2 * b], model.outputs[2 * b + 1]],
name='blk%d' % (b + 1)))
model = Model(model.input, outputs, name=model.name)
"""Load Human3.6M dataset."""
h36m = Human36M('datasets/Human3.6M',
dataconf=human36m_dataconf,
poselayout=pa17j3d,
topology='frames')
h36m_val = BatchLoader(h36m, ['frame'],
['pose_w', 'pose_uvd', 'afmat', 'camera', 'action'],
VALID_MODE,
batch_size=h36m.get_length(VALID_MODE),
shuffle=False)
printcn(OKBLUE, 'Preloading Human3.6M validation samples...')
[x_val], [pw_val, puvd_val, afmat_val, scam_val, action] = h36m_val[0]
eval_human36m_sc_error(model,
x_val,
pw_val,
afmat_val,
puvd_val[:, 0, 2],
scam_val,
action,
batch_size=24)
pose_net_version='v1',
full_trainable=False)
weights_path = "weights_pennaction_010.h5"
model.load_weights(weights_path)
"""Load PennAction dataset."""
# """
penn_seq = PennAction('datasets/PennAction',
pennaction_dataconf,
poselayout=pa16j2d,
topology='sequences',
use_gt_bbox=use_bbox,
clip_size=num_frames)
penn_te = BatchLoader(penn_seq, ['frame'], ['pennaction'],
TEST_MODE,
batch_size=1,
shuffle=False)
# =================================================================
import numpy as np
import json
import matplotlib.pyplot as plt
frame_list = penn_seq.get_clip_index(0, TEST_MODE, subsamples=[2])
with open('datasets/PennAction/penn_pred_bboxes_16f.json', 'r') as fid:
bboxes_data = json.load(fid)
i = 0
subsampling = 2
f = 0
hflip = 0
penn_seq.dataconf.fixed_hflip = hflip
num_joints = 16
print("***********************")
print(input_shape)
model = reception.build(input_shape,
num_joints,
dim=2,
num_blocks=num_blocks,
num_context_per_joint=2,
ksize=(5, 5))
"""Load the MPII dataset."""
mpii = MpiiSinglePerson('datasets/MPII', dataconf=mpii_sp_dataconf)
data_tr = BatchLoader(mpii, ['frame'], ['pose'],
TRAIN_MODE,
batch_size=batch_size,
num_predictions=num_blocks,
shuffle=True)
"""Pre-load validation samples and generate the eval. callback."""
mpii_val = BatchLoader(mpii,
x_dictkeys=['frame'],
y_dictkeys=['pose', 'afmat', 'headsize'],
mode=VALID_MODE,
batch_size=mpii.get_length(VALID_MODE),
num_predictions=1,
shuffle=False)
printcn(OKBLUE, 'Pre-loading MPII validation data...')
[x_val], [p_val, afmat_val, head_val] = mpii_val[0]
eval_callback = MpiiEvalCallback(x_val,
p_val,
afmat_val,
"""Load pre-trained weights from pose estimation and copy replica layers."""
full_model.load_weights(
# 'output/ntu_spnet_trial-03-ft_replica_0ae2bf7/weights_3dp+ntu_ar_062.hdf5',
'output/ntu_spnet_trial_06_nopose_g_512a239/weights_3dp+ntu_ar_030.hdf5',
by_name=True)
"""Split model to simplify evaluation."""
models = split_model(full_model, cfg, interlaced=False,
model_names=['3DPose', '3DAction'])
"""Human3.6H validation samples."""
# h36m_val = BatchLoader(h36m, ['frame'],
# ['pose_w', 'pose_uvd', 'afmat', 'camera', 'action'], VALID_MODE,
# batch_size=h36m.get_length(VALID_MODE), shuffle=False)
# printcn(OKBLUE, 'Preloading Human3.6M validation samples...')
# [h36m_x_val], [h36m_pw_val, h36m_puvd_val, h36m_afmat_val, h36m_scam_val, \
# h36m_action] = h36m_val[0]
"""NTU subset of testing samples"""
ntu_te = BatchLoader(ntu, ['frame'], ['ntuaction'], TEST_MODE, batch_size=1,
shuffle=False)
"""Evaluate on Human3.6M using 3D poses."""
# s = eval_human36m_sc_error(models[0], h36m_x_val, h36m_pw_val, h36m_afmat_val,
# h36m_puvd_val[:,0,2], h36m_scam_val, h36m_action, batch_size=2)
s = eval_multiclip_dataset(models[1], ntu,
subsampling=ntu_dataconf.fixed_subsampling, logdir=logdir)
# weights_path = "weights_merlaction_1_003.h5"
# model.load_weights(weights_path)
# model.summary()
# """
# anno_path = "/mnt/hdd10tb/Users/andang/actions/train_2.json"
# videos_dict_path = "/home/son/lightweight-human-pose-estimation/settings/3/train_merl.json"
# videos_dict_path = "/mnt/hdd10tb/Users/pminhtamnb/deephar/settings/setting_5/train.txt"
merl_seq = MERL5Action(anno_path,
pennaction_dataconf,
poselayout=pa16j2d,
clip_size=num_frames)
merl_te = BatchLoader(merl_seq, ['frame'], ['merlaction'],
TRAIN_MODE,
batch_size=batch_size,
shuffle=False,
num_predictions=11)
printcn(WARNING, "Data train " + str(merl_seq.get_length()))
printcn(OKGREEN, "Load data train done")
# val_anno_path = "/mnt/hdd10tb/Users/andang/actions/test_2.json"
val_merl_seq = MERL5Action(val_anno_path,
pennaction_dataconf,
poselayout=pa16j2d,
clip_size=num_frames)
val_merl_te = BatchLoader(val_merl_seq, ['frame'], ['merlaction'],
TRAIN_MODE,
batch_size=batch_size,
shuffle=False,
f'\nPlease download it from https://drive.google.com/file/d/106yIhqNN-TrI34SX81q2xbU-NczcQj6I/view?usp=sharing'
)
sys.stdout.flush()
sys.exit()
"""Load pre-trained weights from pose estimation and copy replica layers."""
full_model.load_weights(weights_file, by_name=True)
"""This call splits the model into its parts: pose estimation and action
recognition, so we can evaluate each part separately on its respective datasets.
"""
models = split_model(full_model,
cfg,
interlaced=False,
model_names=['2DPose', '2DAction'])
"""Trick to pre-load validation samples from MPII."""
mpii_val = BatchLoader(mpii, ['frame'], ['pose', 'afmat', 'headsize'],
VALID_MODE,
batch_size=mpii.get_length(VALID_MODE),
shuffle=False)
logger.debug('Pre-loading MPII validation data...')
[x_val], [p_val, afmat_val, head_val] = mpii_val[0]
"""Define a loader for PennAction test samples. """
penn_te = BatchLoader(penn_seq, ['frame'], ['pennaction'],
TEST_MODE,
batch_size=1,
shuffle=False)
"""Evaluate on 2D action recognition (PennAction)."""
s = eval_singleclip_generator(models[1], penn_te, logdir=logdir)
logger.debug('Best score on PennAction (single-clip): ')
logger.debug(str(s))
s = eval_multiclip_dataset(models[1],
penn_seq,
by_name=True)
"""Save model callback."""
sys.path.append(os.path.join(os.getcwd(), 'benset'))
from benset_batchloader_ar import *
from benset_dataloader_ar import *
dataset_path = "E:\\Bachelorarbeit-SS20\\datasets\\Benset256_testdataset"
#dataset_path="D:\\sortout_Benset"
num_action_predictions = 6
benset = Benset(dataset_path, num_action_predictions)
batch_size = 1
benset_test = BatchLoader(benset, benset.get_dataset_keys(),
benset.get_dataset_annotations(), batch_size,
num_frames)
#benset_seq = BatchLoader(benset, benset.get_dataset_keys(),benset.get_dataset_annotations(),batch_size,num_frames, mode=1, random_hflip=1, random_brightness=1, random_channel_shift=0, random_zoom=1, random_subsampling=1, random_rot=1, random_blur=1)
from sklearn.metrics import confusion_matrix
y_actu = []
y_pred = []
from collections import Counter
predictions = []
wrong = [0, 0, 0, 0]
while True:
x, y = benset_test.__next__()
sys.path.append(os.path.join(os.getcwd(), 'exp/common'))
from mpii_tools import eval_singleperson_pckh
sys.path.append(os.path.join(os.getcwd(), 'datasets'))
import annothelper
annothelper.check_mpii_dataset()
"""Load the MPII dataset."""
mpii = MpiiSinglePerson('datasets/MPII', dataconf=mpii_sp_dataconf)
"""Pre-load validation samples and generate the eval. callback."""
mpii_val = BatchLoader(
mpii,
x_dictkeys=['frame'],
y_dictkeys=['pose'],
mode=TRAIN_MODE,
# batch_size=mpii.get_length(VALID_MODE), num_predictions=1,
batch_size=1,
num_predictions=1,
shuffle=False)
printcn(OKBLUE, 'Pre-loading MPII validation data...')
# img 256x256
[x_val], [p_val] = mpii_val[10]
print(p_val)
# print(p_val)
# plot data
import matplotlib.pyplot as plt
plt.imshow(x_val[0])
print(p_val.shape)
ntu_dataconf,
poselayout=pa17j3d,
topology='sequences',
use_gt_bbox=True,
clip_size=num_frames)
ntu_s1 = Ntu(datasetpath('NTU'),
ntu_dataconf,
poselayout=pa17j3d,
topology='sequences',
use_gt_bbox=True,
clip_size=num_frames)
# topology='sequences', use_gt_bbox=True, clip_size=num_frames, num_S=1)
pe_data_tr = BatchLoader([ntu_sf], ['frame'], ['pose'],
TRAIN_MODE,
batch_size=[batch_size_ntu],
shuffle=True)
pe_data_tr = BatchLoader(pe_data_tr, ['frame'], ['pose'],
TRAIN_MODE,
batch_size=batch_clips,
num_predictions=num_predictions,
shuffle=False)
ar_data_tr = BatchLoader(ntu, ['frame'], ['ntuaction'],
TRAIN_MODE,
batch_size=batch_clips,
num_predictions=num_action_predictions,
shuffle=True)
"""Build the full model"""
full_model = spnet.build(cfg)
"""Load pre-trained weights from pose estimation and copy replica layers."""
poselayout=pa16j2d,
topology='frames',
use_gt_bbox=True)
penn_seq = PennAction("D:\\PennAction",
pennaction_dataconf,
poselayout=pa16j2d,
topology='sequences',
use_gt_bbox=True,
clip_size=num_frames)
# pe_data_tr = BatchLoader([mpii, penn_sf], ['frame'], ['pose'], TRAIN_MODE,
pe_data_tr = BatchLoader(
[mpii],
['frame'],
['pose'],
TRAIN_MODE,
# batch_size=[batch_size_mpii, batch_size_penn], shuffle=True)
batch_size=[batch_size_penn],
shuffle=True)
pe_data_tr = BatchLoader(pe_data_tr, ['frame'], ['pose'],
TRAIN_MODE,
batch_size=batch_clips,
num_predictions=num_predictions,
shuffle=False)
ar_data_tr = BatchLoader(penn_seq, ['frame'], ['pennaction'],
TRAIN_MODE,
batch_size=batch_clips,
num_predictions=num_action_predictions,
shuffle=True)
"""Build the full model"""
sys.stdout = open(str(logdir) + '/log_merl.txt', 'w')
model = reception.build(input_shape, num_joints, dim=2,
num_blocks=num_blocks, num_context_per_joint=2, ksize=(5, 5))
# weights_path = 'weights_merl_021.h5'
weights_path = args.weights_path
if weights_path:
printcn(WARNING, 'load weights from %s' % (weights_path))
model.load_weights(weights_path)
"""Load the dataset."""
merl = MERLSinglePerson(image_path,anno_path,dataconf=mpii_sp_dataconf)
data_tr = BatchLoader(merl, ['image'], ['pose'], TRAIN_MODE,
batch_size=batch_size, num_predictions=num_blocks, shuffle=True)
# """
loss = pose_regression_loss('l1l2bincross', 0.01)
model.compile(loss=loss, optimizer=RMSprop())
# model.summary()
def lr_scheduler(epoch, lr):
if epoch in [300, 350] or epoch in [50,60]:
newlr = 0.99*lr
printcn(WARNING, 'lr_scheduler: lr %g -> %g @ %d' % (lr, newlr, epoch))
else:
newlr = lr
printcn(OKBLUE, 'lr_scheduler: lr %g @ %d' % (newlr, epoch))
# cache_subdir='models')
# print("wwwww xxxxxxxx c zzz ",weights_path)
# model.load_weights(weights_path)
"""Load PennAction dataset."""
penn_seq = PennAction('datasets/PennAction',
pennaction_dataconf,
poselayout=pa16j2d,
topology='sequences',
use_gt_bbox=use_bbox,
clip_size=num_frames)
penn_te = BatchLoader(penn_seq, ['frame'], ['pennaction'],
TRAIN_MODE,
batch_size=1,
shuffle=False,
num_predictions=11)
print(penn_te.get_length(TRAIN_MODE))
# print(penn_te.get_length(TEST_MODE))
# data = penn_te.__getitem__(1)
# print(data)
# print(data[0])
# print(data[1])
# x = data[0]
# y = data[1]
# """
printcn(OKGREEN, 'Evaluation on PennAction multi-clip using predicted bboxes')
from keras.optimizers import SGD
benset_dataloader = Benset(dataset_path_green,
num_action_predictions,
use_backgrounds=use_backgrounds)
batch_size = 3
num_frames = 8
mode = 1
green_screen = 1
augmentation = 1
benset_train_batchloader = BatchLoader(
dataloader=benset_dataloader,
x_set=benset_dataloader.get_train_data_keys(),
y_set=benset_dataloader.get_train_annotations(),
batch_size=batch_size,
num_frames=num_frames,
mode=mode,
green_screen=green_screen,
backgrounds=benset_dataloader.get_backgrounds(),
augmentation=augmentation)
batch_size = 1
augmentation = 0
benset_val_batchloader = BatchLoader(
dataloader=benset_dataloader,
x_set=benset_dataloader.get_val_data_keys(),
y_set=benset_dataloader.get_val_annotations(),
batch_size=batch_size,
num_frames=num_frames,
mode=mode,
augmentation=augmentation)