def create_cnn_image_pose(param, instance_nn):
list_poses = param['listPoses']
max_confidence = 1
image_height = cnn_image_height
image_width = len(list_poses)
image_np = np.zeros((image_height, image_width), dtype=np.uint8)
for index_pose, pose in enumerate(list_poses):
angles = pose['angles']
transformed_points = pose['transformedPoints']
list_desc = list()
list_desc += angles
list_desc += ClassUtils.get_flat_list(transformed_points)
list_desc_np = np.asanyarray(list_desc, dtype=np.float)
res = instance_nn.predict_model_fast(list_desc_np)
probabilities = res['probabilities']
for index, value in enumerate(probabilities):
pixel_value = int(value * 255 / max_confidence)
image_np[index, index_pose] = pixel_value
# Resizing image
image_res = cv2.resize(image_np, (cnn_image_height, cnn_image_width))
return image_res
def evaluating_fast_nn():
print('Initializing evaluating fast nn')
classes = 8
hidden_layers = 40
instance_nn = ClassNN(model_dir=ClassNN.model_dir_pose, classes=classes, hidden_number=hidden_layers)
instance_pose = ClassOpenPose()
info = ClassDescriptors.load_images_comparision_ext(instance_pose, min_score=0.05, load_one_img=True)
pose1 = info['pose1']
items = ClassDescriptors.get_person_descriptors(pose1, 0.05)
# Valid pose for detection
data_to_add = list()
data_to_add += items['angles']
data_to_add += ClassUtils.get_flat_list(items['transformedPoints'])
data_np = np.asanyarray(data_to_add, dtype=np.float)
result = instance_nn.predict_model_fast(data_np)
print(result)
key_pose = result['classes']
probability = result['probabilities'][key_pose]
print('Key pose: {0}'.format(key_pose))
print('Probability: {0}'.format(probability))
print('Done!')
def main():
print('Initializing main function')
print('Loading elements')
list_elems = [[1, 2, 3], [4, 5, 6]]
list_flat = ClassUtils.get_flat_list(list_elems)
print('list_flat: {0}'.format(list_flat))
print('Done!')
def calculate_poses(option: Option, nn_classifier: ClassNN, svm_classifier: ClassSVM):
print('Calculating poses using nn')
# Recalculate all poses and get confidence
for classInfo in list_classes:
folder = classInfo['folderPath']
for root, _, files in os.walk(folder):
for file in files:
full_path = os.path.join(root, file)
ext = ClassUtils.get_filename_extension(full_path)
if '_rawdata' in file and ext == '.json':
print('Processing file: {0}'.format(full_path))
with open(full_path, 'r') as f:
file_txt = f.read()
file_json = json.loads(file_txt)
list_poses = file_json['listPoses']
for pose in list_poses:
angles = pose['angles']
transformed_points = pose['transformedPoints']
# Using transformed points and angles
list_desc = list()
list_desc += ClassUtils.get_flat_list(transformed_points)
# Convert to numpy
list_desc_np = np.asanyarray(list_desc, np.float)
if option == Option.NN:
result = nn_classifier.predict_model_fast(list_desc_np)
else:
result = svm_classifier.predict_model(list_desc_np)
pose['class'] = int(result['classes'])
pose['probabilities'] = result['probabilities'].tolist()
# Writing again into file
file_txt = json.dumps(file_json, indent=4)
new_full_path = ClassUtils.change_ext_training(full_path,
'{0}_posedata'.format(option.value))
with open(new_full_path, 'w') as f:
f.write(file_txt)
# Done
print('Done processing elements')
def _get_descriptor_list(cls, angles, transformed_points,
type_desc: EnumDesc):
if type_desc == EnumDesc.ANGLES:
return angles
elif type_desc == EnumDesc.POINTS:
return ClassUtils.get_flat_list(transformed_points)
elif type_desc == EnumDesc.ALL:
data_to_add = list()
data_to_add += angles
data_to_add += ClassUtils.get_flat_list(transformed_points)
return data_to_add
elif type_desc == EnumDesc.ANGLES_TRANSFORMED:
data_to_add = list()
for angle in angles:
sin_angle = math.sin(angle)
cos_angle = math.cos(angle)
data_to_add.append(sin_angle)
data_to_add.append(cos_angle)
return data_to_add
elif type_desc == EnumDesc.ALL_TRANSFORMED:
data_to_add = list()
for angle in angles:
sin_angle = math.sin(angle)
cos_angle = math.cos(angle)
data_to_add.append(sin_angle)
data_to_add.append(cos_angle)
data_to_add += ClassUtils.get_flat_list(transformed_points)
return data_to_add
else:
raise Exception('Invalid option: {0}'.format(type_desc))
def cnn_reprocess_images():
print('Re processing images')
list_folders = list()
list_folders.append(ClassUtils.cnn_class_folder)
# Loading instances
instance_nn = ClassNN(ClassNN.model_dir_pose, classes_number, hidden_layers)
# File walk
count = 0
for folder in list_folders:
for root, _, files in os.walk(folder):
for file in files:
full_path = os.path.join(root, file)
extension = ClassUtils.get_filename_extension(full_path)
if extension == '.json':
print('Processing file: {0}'.format(full_path))
with open(full_path, 'r') as f:
json_txt = f.read()
json_data = json.loads(json_txt)
list_poses = json_data['listPoses']
for pose in list_poses:
angles = pose['angles']
transformed_points = pose['transformedPoints']
list_desc = list()
list_desc += angles
list_desc += ClassUtils.get_flat_list(transformed_points)
list_desc_np = np.asanyarray(list_desc, dtype=np.float)
res = instance_nn.predict_model_fast(list_desc_np)
pose['keyPose'] = int(res['classes'])
pose['probability'] = 1
# Writing data again
data_txt = json.dumps(json_data, indent=2)
with open(full_path, 'w') as f:
f.write(data_txt)
count += 1
print('Done')
print('Total files processed: {0}'.format(count))
def create_cnn_image_pos_black_rem(param):
# For now
# Only consider transformed points
list_poses = param['listPoses']
# Last item - movement vector
# Total elements: 27
image_height = cnn_image_height
image_width = len(list_poses)
if len(ClassUtils.get_flat_list(list_poses[0]['transformedPoints'])) > image_height:
raise Exception('Invalid image height: {0}'.format(image_height))
image_np = np.zeros((image_height, image_width), dtype=np.uint8)
for index_pose, pose in enumerate(list_poses):
points = pose['transformedPoints']
for index, item in enumerate(points):
# X Value
pixel_value = 128 + int(item[0] * 255 / 3)
if pixel_value > 255:
print('Truncate value 255 x')
pixel_value = 255
if pixel_value < 0:
print('Truncate value 0 x')
pixel_value = 0
image_np[index, index_pose] = pixel_value
for index, item in enumerate(points):
# Y Calc
pixel_value = 32 + int(item[1] * 255 / 3)
if pixel_value > 255:
print('Truncate value 255 y')
pixel_value = 255
if pixel_value < 0:
print('Truncate value 0 y')
pixel_value = 0
image_np[index + len(points), index_pose] = pixel_value
# Resizing image
image_res = cv2.resize(image_np, (cnn_image_height, cnn_image_width))
# Return created image
return image_res
def load_descriptors(instance_nn_train: ClassNN, instance_nn_pose: ClassNN,
pose_type: Desc):
training_data = list()
training_labels = list()
eval_data = list()
eval_labels = list()
training_files = list()
eval_files = list()
for index, item in enumerate(list_folder_data):
folder = item['folderPath']
label = item['label']
print('Processing folder path: {0}'.format(folder))
num_file = 0
list_paths = list()
for root, _, files in os.walk(folder):
for file in files:
full_path = os.path.join(root, file)
extension = ClassUtils.get_filename_extension(full_path)
if extension == '.json':
list_paths.append(full_path)
total_samples = len(list_paths)
total_train = int(total_samples * 80 / 100)
# Shuffle samples
random.Random(seed).shuffle(list_paths)
for full_path in list_paths:
# Reading data
with open(full_path, 'r') as f:
json_txt = f.read()
json_data = json.loads(json_txt)
list_poses = json_data['listPoses']
# Sampling data
descriptor = list()
for index_size in range(samples_size):
index_pose = int(len(list_poses) * index_size / samples_size)
pose = list_poses[index_pose]
transformed_points = pose['transformedPoints']
angles = pose['angles']
list_desc = list()
list_desc += angles
list_desc += ClassUtils.get_flat_list(transformed_points)
if pose_type == Desc.POSES:
list_desc_np = np.asanyarray(list_desc, dtype=np.float)
res = instance_nn_pose.predict_model_fast(list_desc_np)
# Add descriptor with probabilities
for elem in res['probabilities']:
descriptor.append(elem)
elif pose_type == Desc.ALL:
for elem in list_desc:
descriptor.append(elem)
elif pose_type == Desc.POINTS:
list_flat = ClassUtils.get_flat_list(transformed_points)
for elem in list_flat:
descriptor.append(elem)
else:
raise Exception(
'Pose type not recognized: {0}'.format(pose_type))
if num_file < total_train:
training_data.append(descriptor)
training_labels.append(label)
training_files.append(full_path)
else:
eval_data.append(descriptor)
eval_labels.append(label)
eval_files.append(full_path)
num_file += 1
# Convert data to numpy array
training_data_np = np.asanyarray(training_data, dtype=np.float)
training_labels_np = np.asanyarray(training_labels, dtype=int)
eval_data_np = np.asanyarray(eval_data, dtype=np.float)
eval_labels_np = np.asanyarray(eval_labels, dtype=int)
print('Shape images training: {0}'.format(training_data_np.shape))
print('Shape labels training: {0}'.format(training_labels_np.shape))
if training_data_np.shape[0] == 0:
raise Exception('No files found!')
res = input('Press 1 to train - 2 to eval: ')
if res == '1':
train_model(training_data_np,
training_labels_np,
eval_data_np,
eval_labels_np,
instance_nn_train,
steps=30000)
elif res == '2':
eval_model(eval_data_np, eval_labels_np, instance_nn_train)
else:
raise Exception('Option not implemented!')
def get_cnn_descriptor_pos(list_poses):
# For now
# Only consider transformed points
min_x = 0
max_x = 0
min_y = 0
max_y = 0
first = True
for index, pose in enumerate(list_poses):
points = pose['transformedPoints']
for point in points:
if first:
min_x = point[0]
min_y = point[1]
max_x = min_x
max_y = min_y
first = False
if point[0] < min_x:
min_x = point[0]
if point[1] < min_y:
min_y = point[1]
if point[0] > max_x:
max_x = point[0]
if point[1] > max_y:
max_y = point[1]
# Last item - movement vector
# Total elements: 27
image_height = cnn_image_height
image_width = len(list_poses)
image_np = np.zeros((image_height, image_width), dtype=np.uint8)
for index_pose, pose in enumerate(list_poses):
points = pose['transformedPoints']
vector_points = ClassUtils.get_flat_list(points)
for index, item in enumerate(vector_points):
if index % 2 == 0:
min_value = min_x
max_value = max_x
else:
min_value = min_y
max_value = max_y
delta = max_value - min_value
if delta != 0:
pixel_value = int((item - min_value) * 255 / delta)
else:
pixel_value = 0
image_np[index, index_pose] = pixel_value
# Resizing image
image_res = cv2.resize(image_np, (cnn_image_height, cnn_image_width))
image_res = np.resize(image_res, (cnn_image_height, cnn_image_width, 1))
"""
# Only for debugging purposes
cv2.namedWindow('main_window', cv2.WINDOW_AUTOSIZE)
cv2.imshow('main_window', image_res)
cv2.waitKey()
cv2.destroyAllWindows()
"""
# Return created image
return image_res
def create_cnn_image_pos_black(param):
# For now
# Only consider transformed points
list_poses = param['listPoses']
min_x = 0
max_x = 0
min_y = 0
max_y = 0
first = True
for index, pose in enumerate(list_poses):
points = pose['transformedPoints']
for point in points:
if first:
min_x = point[0]
min_y = point[1]
max_x = min_x
max_y = min_y
first = False
if point[0] < min_x:
min_x = point[0]
if point[1] < min_y:
min_y = point[1]
if point[0] > max_x:
max_x = point[0]
if point[1] > max_y:
max_y = point[1]
# Last item - movement vector
# Total elements: 27
image_height = cnn_image_height
image_width = len(list_poses)
if len(ClassUtils.get_flat_list(list_poses[0]['transformedPoints'])) > image_height:
raise Exception('Invalid image height: {0}'.format(image_height))
image_np = np.zeros((image_height, image_width), dtype=np.uint8)
for index_pose, pose in enumerate(list_poses):
points = pose['transformedPoints']
vector_points = ClassUtils.get_flat_list(points)
for index, item in enumerate(vector_points):
if index % 2 == 0:
min_value = min_x
max_value = max_x
else:
min_value = min_y
max_value = max_y
delta = max_value - min_value
if delta != 0:
pixel_value = int((item - min_value) * 255 / delta)
else:
pixel_value = 0
image_np[index, index_pose] = pixel_value
# Resizing image
image_res = cv2.resize(image_np, (cnn_image_height, cnn_image_width))
# Return created image
return image_res
def main():
print('Initializing main function')
# Prompt for user input
cam_number_str = input('Insert camera number to process: ')
cam_number = int(cam_number_str)
# Open video from opencv
cap = cv2.VideoCapture(cam_number)
# Initializing open pose distance
instance_pose = ClassOpenPose()
# Initializing variables
model_dir = '/home/mauricio/models/nn_classifier'
instance_nn = ClassNN.load_from_params(model_dir)
while True:
# Capture frame-by-frame
ret, frame = cap.read()
# Processing frame with openpose
arr, frame = instance_pose.recognize_image_tuple(frame)
# Check if there is one frame with vector integrity
arr_pass = list()
min_score = 0.05
# Checking vector integrity for all elements
# Verify there is at least one arm and one leg
for elem in arr:
if ClassUtils.check_vector_integrity_pos(elem, min_score):
arr_pass.append(elem)
if len(arr_pass) != 1:
print('Invalid len for arr_pass: {0}'.format(arr_pass))
else:
person_array = arr_pass[0]
# Getting person descriptors
results = ClassDescriptors.get_person_descriptors(
person_array, min_score)
# Descriptors
data_to_add = results['angles']
data_to_add += ClassUtils.get_flat_list(
results['transformedPoints'])
data_np = np.asanyarray(data_to_add, dtype=np.float)
# Getting result predict
result_predict = instance_nn.predict_model(data_np)
detected_class = result_predict['classes']
label_class = get_label_name(instance_nn.label_names,
detected_class)
print('Detected: {0} - Label: {1}'.format(detected_class,
label_class))
# Draw text class into image - Evaluation purposes
font = cv2.FONT_HERSHEY_SIMPLEX
font_scale = 0.6
font_color = (255, 255, 255)
line_type = 2
cv2.putText(frame, '{0}'.format(label_class), (0, 0), font,
font_scale, font_color, line_type)
# Display the resulting frame
cv2.imshow('frame', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()
print('Done!')