def preprocessing(self, input_length, file_names):
human_data = list()
robot_data = list()
third_data = list()
pbar = tqdm(total=len(file_names))
for file in file_names:
with np.load(file, allow_pickle=True) as data:
human_data.append([norm_features(human, self.norm_method) for human in data['human_info']])
robot_data.append([norm_features(robot, self.norm_method) for robot in data['robot_info']])
third_data.append(data['third_info'])
pbar.update(1)
pbar.close()
step = round(KINECT_FRAME_RATE / TARGET_FRAME_RATE)
inputs = list()
for idx, third in enumerate(third_data):
if all(v == 1.0 for v in third):
continue
sampled_human_seq = human_data[idx][::step]
sampled_third_seq = third_data[idx][::step]
for human_seq, third_seq in zip(gen_sequence(sampled_human_seq, input_length),
gen_sequence(sampled_third_seq, input_length)):
inputs.append(np.concatenate((third_seq, human_seq), axis=1))
return self.make_dataframe(inputs, input_length)
def test():
# action list to test
# actions = ["A001", "A003", "A004", "A005", "A006", "A008"]
actions = ["A004", "A005"]
# show all test data
data_files = list()
for action in actions:
data_files.extend(glob.glob(os.path.join(TRAIN_PATH, F"*{action}*.npz")))
data_files.sort()
n_data = len(data_files)
print('There are %d data.' % n_data)
for data_idx in range(n_data):
print('%d: %s' % (data_idx, os.path.basename(data_files[data_idx])))
# select data name to draw
artist = Artist(n_plot=1)
while True:
var = int(input("Input data number to display: "))
data_file = data_files[var]
with np.load(data_file, allow_pickle=True) as data:
# action class mapping
print(os.path.basename(data_file))
action = os.path.basename(data_file)[4:8]
kmeans, sub_action_mapping = load_proper_model(action)
km_model = kmeans.km_model
# extract inputs from data file
human_data = [norm_features(human, NORM_METHOD) for human in data['human_info']]
third_data = data['third_info']
sampled_human_data = human_data[::3]
sampled_third_data = third_data[::3]
# draw data from start start
for f in range(SEQ_LENGTH - 1):
features = denorm_features(sampled_human_data[f], NORM_METHOD)
action_info = "None"
frame_info = f"{f+1}/{len(sampled_human_data)}"
artist.update([features], [action_info], [frame_info], fps=10)
# recognize sub-action
for human_seq, third_seq in zip(gen_sequence(sampled_human_data, SEQ_LENGTH),
gen_sequence(sampled_third_data, SEQ_LENGTH)):
seq = np.concatenate((third_seq, human_seq), axis=1)
df = kmeans.make_dataframe([seq], SEQ_LENGTH)
sub_action = km_model.predict(df)
action_name = ALL_SUBACTION_NAMES[sub_action_mapping[sub_action[0]]]
print(action_name)
f += 1
features = denorm_features(human_seq[-1], NORM_METHOD)
frame_info = f"{f+1}/{len(sampled_human_data)}"
artist.update([features], [action_name], [frame_info], fps=10)
def test_all():
# action list to test
actions = ["A008"]
# show all test data
data_files = list()
for action in actions:
data_files.extend(glob.glob(os.path.join(TRAIN_PATH, F"*{action}*.npz")))
for action in actions:
data_files.extend(glob.glob(os.path.join(TEST_PATH, F"*{action}*.npz")))
data_files.sort()
n_data = len(data_files)
print('There are %d data.' % n_data)
# test each data
for data_file in data_files:
with np.load(data_file, allow_pickle=True) as data:
results = list()
# action class mapping
# print(os.path.basename(data_file))
action = os.path.basename(data_file)[4:8]
kmeans, sub_action_mapping = load_proper_model(action)
km_model = kmeans.km_model
# extract inputs from data file
human_data = [norm_features(human, NORM_METHOD) for human in data['human_info']]
third_data = data['third_info']
sampled_human_data = human_data[::3]
sampled_third_data = third_data[::3]
# recognize sub-action
for human_seq, third_seq in zip(gen_sequence(sampled_human_data, SEQ_LENGTH),
gen_sequence(sampled_third_data, SEQ_LENGTH)):
seq = np.concatenate((third_seq, human_seq), axis=1)
df = kmeans.make_dataframe([seq], SEQ_LENGTH)
sub_action = km_model.predict(df)
results.append(sub_action_mapping[sub_action[0]])
# print results
# if not all(result == 0 for result in results):
# if not any(result == 1 for result in results):
print(os.path.basename(data_file))
print(results)
def test_with_webcam(model):
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, W_VIDEO)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, H_VIDEO)
last = time.time()
inputs = list()
while True:
if time.time() - last > 0.1:
last = time.time()
# 2d skeleton from video
ret, frame = cap.read()
key_points, output_data = pose_keypoints(frame)
output_data = cv2.flip(output_data, 3)
cv2.imshow(f'{W_VIDEO}x{H_VIDEO}', output_data)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# extract skeleton features
if len(key_points.shape) != 3:
continue
user_key_points = key_points[0]
if not user_key_points[4].any() or not user_key_points[7].any():
continue
skel = user_key_points[:, :2]
body = skel_to_AIR(skel)
features = norm_features(body, NORM_METHOD, INPUT_DATA_TYPE,
B_HANDS)
# stack joint information
inputs.append(features)
inputs = inputs[-lstm_input_length:]
if len(inputs) < lstm_input_length:
continue
behaviors, behavior_names = classify(model, [inputs])
print("user:", behavior_names[0])
yield body, behaviors[0]
def test_with_kinect(model):
game = BodyGameRuntime()
last = time.time()
first = True
inputs = list()
artist = Artist(n_plot=1)
start_time = time.time()
for joints in game.run():
if time.time() - last > 0.1 and (len(joints) > 0 or not first):
first = False
last = time.time()
if len(joints) > 0:
depths = [joint[20].Position.z
for joint in joints] # 20: spineShoulder
idx = depths.index(min(depths))
body = pose_to_AIR(joints[idx])
else:
body = null_body_3D()
# stack joint information
distance = body[0]['z'] / MAX_DISTANCE
inputs.append(
np.hstack([
distance if distance != 0.0 else 1.0,
norm_features(body, NORM_METHOD, INPUT_DATA_TYPE, B_HANDS)
]))
inputs = inputs[-lstm_input_length:]
if len(inputs) < lstm_input_length:
continue
behaviors, behavior_names = classify(model, [inputs])
cur_features = denorm_features(inputs[-1][1:], NORM_METHOD,
INPUT_DATA_TYPE, B_HANDS)
elapsed_time = time_to_string(time.time() - start_time)
artist.update([cur_features], [behavior_names[0]], [elapsed_time])
yield body, behaviors[0]
data_files.extend(glob.glob(os.path.join(TEST_PATH, F"*{action}*.npz")))
data_files.extend(glob.glob(os.path.join(TRAIN_PATH, F"*{action}*.npz")))
data_files.sort()
# label action classes
pbar = tqdm(total=len(data_files))
for data_file in data_files:
with np.load(data_file, allow_pickle=True) as data:
# action class mapping
action = os.path.basename(data_file)[4:8]
kmeans, sub_action_mapping = load_proper_model(action)
km_model = kmeans.km_model
# extract inputs from data file
human_data = [
norm_features(human, method=NORM_METHOD, type='3D', b_hand=True)
for human in data['human_info']
]
third_data = data['third_info']
step = round(KINECT_FRAME_RATE / TARGET_FRAME_RATE)
sampled_human_data = human_data[::step]
sampled_third_data = third_data[::step]
# label "None"
sampled_labels = list()
for f in range(SEQ_LENGTH - 1):
sampled_labels.append("None")
# label recognized action class by k-means clustering
for human_seq, third_seq in zip(
def __init__(self,
data_path,
dim_input,
dim_output,
data_name=None,
b_add_noise=False,
b_connect_sequence=False):
# load data from files
self.data_path = data_path
self.data_name = data_name
self.file_names = list()
if data_name is not None:
self.file_names.append(os.path.join(self.data_path,
self.data_name))
else:
for action in ACTIONS:
self.file_names.extend(
glob.glob(os.path.join(self.data_path, f"*{action}*.npz")))
path = data_path if data_name is None else os.path.join(
data_path, data_name)
print(f'Data loading... ({path})')
print(f'Total {len(self.file_names)} files.')
self.human_data = list()
self.robot_data = list()
self.third_data = list()
human_actions = list()
step = round(KINECT_FRAME_RATE / TARGET_FRAME_RATE)
for file in self.file_names:
with np.load(file, allow_pickle=True) as data:
self.human_data.append([
norm_features(human, NORM_METHOD, INPUT_DATA_TYPE, B_HANDS)
for human in data['human_info']
][::step])
self.robot_data.append([
norm_features(robot, NORM_METHOD, INPUT_DATA_TYPE, B_HANDS)
for robot in data['robot_info']
][::step])
self.third_data.append(data['third_info'][::step])
human_actions.append(data['human_action'][::step])
# extract training data
self.dim_input = dim_input
self.dim_output = dim_output
self.b_add_noise = b_add_noise
self.b_connect_sequence = b_connect_sequence
self.inputs = list()
self.outputs = list()
seq_length = self.dim_input[0]
pbar = tqdm(total=len(self.third_data))
for idx, third in enumerate(self.third_data):
if all(v == 1.0 for v in third):
continue
f = seq_length - 1
for human_seq, third_seq in zip(
gen_sequence(self.human_data[idx], seq_length),
gen_sequence(self.third_data[idx], seq_length)):
action_name = human_actions[idx][f]
if action_name not in SUBACTION_NAMES:
continue
seq = np.concatenate(
(third_seq, human_seq),
axis=1) if INPUT_DATA_TYPE == '3D' else np.array(human_seq)
self.inputs.append(seq)
cur_action = SUBACTION_NAMES.index(action_name)
self.outputs.append(cur_action)
f += 1
pbar.update(1)
pbar.close()