def data_frame_to_object_annotations(annotations,
model,
has_z=False,
frame_id=0,
resolution=None):
is_valid_model(model)
if annotations.empty:
return ObjectAnnotations()
n_model_joints = int(model.strip('joints'))
joints_values = len(
annotations.drop(['sample_id', 'person_id'], axis=1).columns)
n_annotations_joints = int(joints_values / (4 if has_z else 3))
n_joints = min(n_model_joints, n_annotations_joints)
annotations_pb = ObjectAnnotations()
annotations_pb.frame_id = frame_id
if resolution is not None:
annotations_pb.resolution.width = resolution[0]
annotations_pb.resolution.height = resolution[1]
for _, annotation in annotations.iterrows():
skeleton = annotations_pb.objects.add()
skeleton.id = int(annotation['person_id'])
for joint_id in range(n_joints):
x = annotation['j{:d}x'.format(joint_id)]
y = annotation['j{:d}y'.format(joint_id)]
z = annotation['j{:d}z'.format(joint_id)] if has_z else 0.0
c = annotation['j{:d}c'.format(joint_id)]
human_keypoint = index_to_human_keypoint(joint_id, model)
# check for invalid joint
if (x == 0.0 and y == 0.0 and z == 0.0) or c < 0.0:
continue
if human_keypoint == HKP.Value('UNKNOWN_HUMAN_KEYPOINT'):
continue
keypoint = skeleton.keypoints.add()
keypoint.position.x = x
keypoint.position.y = y
keypoint.position.z = z
keypoint.score = c
keypoint.id = human_keypoint
return annotations_pb
def left_leg(skeletons):
skeletons_pb = ParseDict(skeletons, ObjectAnnotations())
for skeletons in skeletons_pb.objects:
left_hip = None
left_knee = None
left_ankle = None
parts = {}
for part in skeletons.keypoints:
parts[part.id] = (part.position.x, part.position.y,
part.position.z)
if part.id == 13:
left_hip = parts[13]
if part.id == 14:
left_knee = parts[14]
if part.id == 15:
left_ankle = parts[15]
if left_hip and left_knee and left_ankle:
a = np.sqrt((left_ankle[0] - left_knee[0])**2 +
(left_ankle[1] - left_knee[1])**2 +
(left_ankle[2] - left_knee[2])**2)
b = np.sqrt((left_knee[0] - left_hip[0])**2 +
(left_knee[1] - left_hip[1])**2 +
(left_knee[2] - left_hip[2])**2)
left_leg = a + b
return left_leg
else:
left_leg = 0
return left_leg
break
def render_skeletons(images, annotations, it, links, colors):
for cam_id, image in images.items():
deteccoes = 0 # Detections in each frame
skeletons = ParseDict(annotations[cam_id][it], ObjectAnnotations())
for ob in skeletons.objects:
parts = {}
for part in ob.keypoints:
deteccoes += 1
juntas[cam_id] += 1
parts[part.id] = (int(part.position.x), int(part.position.y))
for link_parts, color in zip(links, colors):
begin, end = link_parts
if begin in parts and end in parts:
cv2.line(image,
parts[begin],
parts[end],
color=color,
thickness=4)
for _, center in parts.items():
cv2.circle(image,
center=center,
radius=4,
color=(255, 255, 255),
thickness=-1)
if deteccoes < 10:
juntas[cam_id] -= deteccoes
# perdidas[cam_id] = 0
else:
perdidas[cam_id] += 15 - deteccoes
return juntas, perdidas
def render_skeletons_3d(ax, skeletons, links, colors, juntas_3d, perdidas_3d):
deteccoes_3d = 0
skeletons_pb = ParseDict(skeletons, ObjectAnnotations())
#print(skeletons_pb)
for skeleton in skeletons_pb.objects:
parts = {}
for part in skeleton.keypoints:
deteccoes_3d += 1
juntas_3d += 1
parts[part.id] = (part.position.x, part.position.y, part.position.z)
for link_parts, color in zip(links, colors):
begin, end = link_parts
if begin in parts and end in parts:
x_pair = [parts[begin][0], parts[end][0]]
y_pair = [parts[begin][1], parts[end][1]]
z_pair = [parts[begin][2], parts[end][2]]
ax.plot(
x_pair,
y_pair,
zs=z_pair,
linewidth=3,
color='#{:02X}{:02X}{:02X}'.format(*reversed(color)))
if deteccoes_3d < 10:
juntas_3d -= deteccoes_3d
else:
perdidas_3d += 15 - deteccoes_3d
return juntas_3d, perdidas_3d
def calc_waistline(skeletons):
right_hip = None
left_hip = None
skeletons_pb = ParseDict(skeletons, ObjectAnnotations())
for skeleton in skeletons_pb.objects:
parts = {}
for part in skeleton.keypoints:
parts[part.id] = (part.position.x, part.position.y,
part.position.z)
if part.id == 10:
right_hip = parts[10]
if part.id == 13:
left_hip = parts[13]
if right_hip and left_hip:
mid_hip = ((right_hip[0] + left_hip[0]) / 2,
(right_hip[1] + left_hip[1]) / 2,
(right_hip[2] + left_hip[2]) / 2)
# log.info("Head: {}", parts[1])
# log.info("Right hip: {}", right_hip)
# log.info("Left hip: {}", left_hip)
return mid_hip
else:
mid_hip = None
return mid_hip
break
def transform_object_annotations(objs, tf, ref_id):
new_objs = ObjectAnnotations(frame_id=ref_id)
for obj in objs.objects:
new_obj = new_objs.objects.add()
for keypoint in obj.keypoints:
new_keypoint = new_obj.keypoints.add()
new_keypoint.id = keypoint.id
new_keypoint.position.CopyFrom(transform_vertex(keypoint.position, tf))
return new_objs
def altura_da_pessoa(skeletons):
skeletons_pb = ParseDict(skeletons, ObjectAnnotations())
for skeletons in skeletons_pb.objects:
parts = {}
for part in skeletons.keypoints:
parts[part.id] = (part.position.x, part.position.y,
part.position.z)
altura_da_pessoa = parts[1][2]
break
return altura_da_pessoa
def skeletons_localization(skeletons, initial_foot):
skeletons_pb = ParseDict(skeletons, ObjectAnnotations())
for skeleton in skeletons_pb.objects:
parts = {}
for part in skeleton.keypoints:
parts[part.id] = (part.position.x, part.position.y,
part.position.z)
position = (parts[12], parts[15]) # (pe_direito, pe_esquerdo)
log.info("Foot position {}", position)
break
return position
def _to_object_annotations(self, faces, image_shape):
obs = ObjectAnnotations()
for x, y, width, height in faces:
ob = obs.objects.add()
v1 = ob.region.vertices.add()
v1.x = x
v1.y = y
v2 = ob.region.vertices.add()
v2.x = x + width
v2.y = y + height
ob.label = "human_face"
obs.resolution.width = image_shape[1]
obs.resolution.height = image_shape[0]
return obs
def send_information(skeletons):
skeletons_pb = ParseDict(skeletons, ObjectAnnotations())
connection = pika.BlockingConnection(
pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='Receive_information')
for skeleton in skeletons_pb.objects:
parts = {}
for part in skeleton.keypoints:
parts[part.id] = (part.position.x, part.position.y, part.position.z)
channel.basic_publish(exchange='', routing_key='Receive_information', body = json.dumps({'dict': skeletons}).encode('utf-8'))
#channel.basic_consume(queue='Receive_information', on_message_callback=callback, auto_ack=True)
#channel.start_consuming()
#print("Enviado")
connection.close()
def get_dataset_range(basedir, dataset, cameras, model):
filename = os.path.join(basedir, dataset, '{}_2d_dataset'.format(model))
reader = ProtobufReader(filename)
begin, end, first_read = 0, 0, True
while True:
sequence_id = reader.next(Int64Value())
for camera in cameras:
_ = reader.next(ObjectAnnotations())
if not sequence_id:
break
if first_read:
begin = sequence_id.value
first_read = False
end = sequence_id.value
return begin, end
def __to_object_annotations(self, humans, im_width, im_height):
obs = ObjectAnnotations()
for human in humans:
ob = obs.objects.add()
for part_id, bp in human.body_parts.items():
part = ob.keypoints.add()
part.id = self.__to_sks_part[part_id]
part.position.x = bp.x * im_width
part.position.y = bp.y * im_height
part.score = bp.score
ob.label = "human_skeleton"
ob.id = ObjectLabels.Value('HUMAN_SKELETON')
obs.resolution.width = im_width
obs.resolution.height = im_height
return obs
def render_skeletons_3d(ax, skeletons, links, colors, juntas_3d, perdidas_3d):
deteccoes_3d = 0
skeletons_pb = ParseDict(skeletons, ObjectAnnotations())
# msg = Message(reply_to=subscription, content_type=ContentType.JSON)
# body = json.dumps({'dict': skeletons_pb}).encode('utf-8')
# msg.body = body
# msg.timeout = 5.0
# channel.publish(msg, topic='SkeletonsGrouper.Localization')
# requests[msg.correlation_id] = skeletons_pb
# {
# 'body': body,
# 'person_id': person_id,
# 'gesture_id': gesture_id,
# 'pos': pos,
# 'requested_at': time.time()
# }
for skeleton in skeletons_pb.objects:
parts = {}
for part in skeleton.keypoints:
deteccoes_3d += 1
juntas_3d += 1
parts[part.id] = (part.position.x, part.position.y,
part.position.z)
# msg = Message(reply_to=subscription, content_type=ContentType.JSON)
# body = json.dumps({'dict': parts}).encode('utf-8')
# msg.body = body
# msg.timeout = time.time()
# channel.publish(msg, topic='SkeletonsGrouper.Localize')
# # msg = channel.consume(timeout=5.0)
# print(skeleton)
for link_parts, color in zip(links, colors):
begin, end = link_parts
if begin in parts and end in parts:
x_pair = [parts[begin][0], parts[end][0]]
y_pair = [parts[begin][1], parts[end][1]]
z_pair = [parts[begin][2], parts[end][2]]
ax.plot(x_pair,
y_pair,
zs=z_pair,
linewidth=3,
color='#{:02X}{:02X}{:02X}'.format(*reversed(color)))
if deteccoes_3d < 10:
juntas_3d -= deteccoes_3d
else:
perdidas_3d += 15 - deteccoes_3d
return juntas_3d, perdidas_3d
def perdas_3d(ax, skeletons, links, colors):
skeletons_pb = ParseDict(skeletons, ObjectAnnotations())
for skeleton in skeletons_pb.objects:
parts = {}
for part in skeleton.keypoints:
parts[part.id] = (part.position.x, part.position.y,
part.position.z)
for link_parts, color in zip(links, colors):
begin, end = link_parts
if begin in parts and end in parts:
x_pair = [parts[begin][0], parts[end][0]]
y_pair = [parts[begin][1], parts[end][1]]
z_pair = [parts[begin][2], parts[end][2]]
perdas = (15 - len(parts)) / 15.0
perdas = perdas * 100.0
#print("Perdas na detecção: " + str(perdas) + "%")
return perdas
def render_skeletons_3d(ax, skeletons, links, colors):
skeletons_pb = ParseDict(skeletons, ObjectAnnotations())
for skeleton in skeletons_pb.objects:
parts = {}
for part in skeleton.keypoints:
parts[part.id] = (part.position.x, part.position.y,
part.position.z)
for link_parts, color in zip(links, colors):
begin, end = link_parts
if begin in parts and end in parts:
x_pair = [parts[begin][0], parts[end][0]]
y_pair = [parts[begin][1], parts[end][1]]
z_pair = [parts[begin][2], parts[end][2]]
ax.plot(x_pair,
y_pair,
zs=z_pair,
linewidth=3,
color='#{:02X}{:02X}{:02X}'.format(*reversed(color)))
def render_skeletons(images, annotations, it, links, colors):
for cam_id, image in images.items():
skeletons = ParseDict(annotations[cam_id][it], ObjectAnnotations())
for ob in skeletons.objects:
parts = {}
for part in ob.keypoints:
parts[part.id] = (int(part.position.x), int(part.position.y))
for link_parts, color in zip(links, colors):
begin, end = link_parts
if begin in parts and end in parts:
cv2.line(image,
parts[begin],
parts[end],
color=color,
thickness=4)
for _, center in parts.items():
cv2.circle(image,
center=center,
radius=4,
color=(255, 255, 255),
thickness=-1)
def right_leg(skeletons):
skeletons_pb = ParseDict(skeletons, ObjectAnnotations())
right_hip = None
right_ankle = None
right_knee = None
right_leg = None
mid_point_ankle = None
left_ankle = None
for skeletons in skeletons_pb.objects:
parts = {}
for part in skeletons.keypoints:
parts[part.id] = (part.position.x, part.position.y,
part.position.z)
if part.id == 10:
right_hip = parts[10]
if part.id == 11:
right_knee = parts[11]
if part.id == 12:
right_ankle = parts[12]
if part.id == 15:
left_ankle = parts[15]
if right_ankle and right_knee and right_hip and left_ankle:
a = np.sqrt((right_ankle[0] - right_knee[0])**2 +
(right_ankle[1] - right_knee[1])**2 +
(right_ankle[2] - right_knee[2])**2)
b = np.sqrt((right_knee[0] - right_hip[0])**2 +
(right_knee[1] - right_hip[1])**2 +
(right_knee[2] - right_hip[2])**2)
right_leg = a + b
mid_point_ankle = (left_ankle[2] + right_ankle[2]) / 2
return right_leg, mid_point_ankle
else:
mid_point_ankle = 0
right_leg = 0
return right_leg, mid_point_ankle
break
def render_skeletons_3d(ax, skeletons, links, colors):
skeletons_pb = ParseDict(skeletons, ObjectAnnotations())
for skeleton in skeletons_pb.objects:
parts = {}
for part in skeleton.keypoints:
parts[part.id] = (part.position.x, part.position.y,
part.position.z)
for link_parts, color in zip(links, colors):
begin, end = link_parts
if begin in parts and end in parts:
x_pair = [parts[begin][0], parts[end][0]]
y_pair = [parts[begin][1], parts[end][1]]
z_pair = [parts[begin][2], parts[end][2]]
#perdas=(15-len(parts))/15.0
#perdas=perdas*100.0
#print("Perdas na detecção: " + str(perdas) + "%")
#print(skeleton.keypoints)
ax.plot(x_pair,
y_pair,
zs=z_pair,
linewidth=3,
color='#{:02X}{:02X}{:02X}'.format(*reversed(color)))
break
detections_ids = list(map(lambda f: get_detection_id(f), detections_files))
detections_ids.sort()
output_file = os.path.join(dataset_folder, '{}_2d_dataset'.format(model))
wirter_2D = ProtobufWriter(output_file)
output_file = os.path.join(dataset_folder, '{}_3d_dataset'.format(model))
wirter_3D = ProtobufWriter(output_file)
log.info("[{}] {} sequences", dataset, len(detections_ids))
for detection_id in detections_ids:
detections_file = os.path.join(detections_folder,
DETECTION_FILE.format(detection_id))
with open(detections_file, 'r') as f:
detections = json.load(f)
sks_3D = ObjectAnnotations()
sks_3D.frame_id = referencial
sks_2D = {}
for camera in cameras:
sks_2D[camera] = ObjectAnnotations()
sks_2D[camera].resolution.width = calibs[camera].resolution.width
sks_2D[camera].resolution.height = calibs[camera].resolution.height
for body in detections['bodies']:
joints_3D = np.array(body[joints_key]).reshape(
(-1, 4)).transpose()[0:3, :]
sk_3D, invalid_joints = joints_to_object_annotation(
joints_3D, model)
sk_3D.id = body[
'id'] # on sk_3D, ObjectAnnotation::id stores person id from dataset
def detect(image, ctx):
# simulate error
time.sleep(randint(mean_time - var_time, mean_time + var_time) / 1000.0)
reply = ObjectAnnotations(frame_id=randint(0, 4),
objects=[ObjectAnnotation()] * randint(1, 3))
return reply
dataset_folder,
'{}_pose_2d_detected_00_{:02d}'.format(model, camera))
reader = ProtobufReader(filename)
filename = os.path.join(dataset_folder,
'{}_2d_detector_{}'.format(model, camera))
writer = ProtobufWriter(filename)
sid = 0
while True:
sks = reader.next(Skeletons())
if not sks:
log.info("[Done][{}] camera \'{}\' with {} sequences", dataset,
camera, sid + 1)
break
objs = ObjectAnnotations()
# panoptic dataset HD cameras resolution
objs.resolution.width = width
objs.resolution.height = height
for sk in sks.skeletons:
obj = objs.objects.add()
for part in sk.parts:
type_str = SkeletonPartType.Name(part.type)
if type_str == 'UNKNOWN' or type_str == 'BACKGROUND':
continue
keypoint = obj.keypoints.add()
keypoint.id = HumanKeypoints.Value(type_str)
keypoint.score = part.score
keypoint.position.x = part.x
keypoint.position.y = part.y
from is_msgs.image_pb2 import ObjectAnnotations
from google.protobuf.wrappers_pb2 import Int64Value
from utils.io import ProtobufReader
from utils.options import load_options
from is_wire.core import Logger
log = Logger()
options = load_options()
with open('panoptic_datasets.json', 'r') as f:
panoptic_data = json.load(f)
datasets = panoptic_data['datasets']
model = panoptic_data['model']
source = 'dataset' # can be either 'detector' or 'dataset'
for dataset in datasets:
dataset_folder = os.path.join(options['data_folder'], dataset)
# read 3D information
filename = os.path.join(dataset_folder, '{}_3d_{}'.format(model, source))
reader = ProtobufReader(filename)
while True:
sequence_id = reader.next(Int64Value())
if not sequence_id:
break
objs = reader.next(ObjectAnnotations())
if not objs:
break
n_skeletons = len(objs.objects)
log.info("[{}] Sequence {} with {} skeletons", dataset, sequence_id,
n_skeletons)
from utils.io import ProtobufReader
from utils.options import load_options
from is_wire.core import Logger
log = Logger()
options = load_options()
with open('panoptic_datasets.json', 'r') as f:
panoptic_data = json.load(f)
datasets = panoptic_data['datasets']
model = panoptic_data['model']
source = 'detector' # can be either 'detector' or 'dataset'
for dataset in datasets:
dataset_folder = os.path.join(options['data_folder'], dataset)
# read 3D information
filename = os.path.join(dataset_folder, '{}_3d_dataset'.format(model))
reader = ProtobufReader(filename)
filename = os.path.join(dataset_folder, '{}_3d_{}_grouped'.format(model, source))
detections_reader = ProtobufReader(filename)
while True:
sid = reader.next(Int64Value())
sid_detections = detections_reader.next(Int64Value())
if not sid or not sid_detections:
break
objs = reader.next(ObjectAnnotations())
objs_detected = detections_reader.next(ObjectAnnotations())
log.info("{} -> {}", sid.value, sid_detections.value)
begin, end = get_dataset_range(options['data_folder'], dataset, cameras,
model) if get_range else (-1, -1)
log.info("[Starting][{}]{} -> {}", dataset, begin, end)
reading = True
while reading:
detections = {}
sequence_id = Int64Value()
write = False
for camera in cameras:
sequence_id = readers[camera].next(Int64Value())
if not sequence_id:
reading = False
break
objs = readers[camera].next(ObjectAnnotations())
if (sequence_id.value >= begin
and sequence_id.value <= end) or not get_range:
write = True
detections[camera] = objs
log.info('[{}][{}][{}][{} skeletons]', dataset, camera,
sequence_id.value, len(detections[camera].objects))
elif sequence_id.value > end:
reading = False
break
if write:
writer.insert(sequence_id)
for camera in cameras:
writer.insert(detections[camera])