def _create_consumer(competition, competition_config):
''' Starts the consumer for streams of predictions sent by users.'''
options = (('grpc.so_reuseport', 1),)
grpc_server = StreamServer('0.0.0.0:50051', options=options)
try:
streamer = DataStreamerServicer(SERVER_HOST, competition, competition_config)
grpc_server.add_server(streamer, competition)
grpc_server.start_server()
grpc_server._wait_forever()
except Exception as e:
logging.debug("Inside consumer process: {}".format(e))
class SocialDistancing:
colors = [(0, 255, 0), (0, 0, 255)]
nd_color = [(153, 0, 51), (153, 0, 0), (153, 51, 0), (153, 102, 0),
(153, 153, 0), (102, 153, 0), (51, 153, 0), (0, 153, 0),
(0, 102, 153), (0, 153, 51), (0, 153, 102), (0, 153, 153),
(0, 102, 153), (0, 51, 153), (0, 0, 153), (153, 0, 102),
(102, 0, 153), (153, 0, 153), (102, 0, 153), (0, 0, 153),
(0, 0, 153), (0, 0, 153), (0, 153, 153), (0, 153, 153),
(0, 153, 153)]
connections = [(0, 16), (0, 15), (16, 18), (15, 17), (0, 1), (1, 2),
(2, 3), (3, 4), (1, 5), (5, 6), (6, 7), (1, 8), (8, 9),
(9, 10), (10, 11), (8, 12), (12, 13), (13, 14), (11, 24),
(11, 22), (22, 23), (14, 21), (14, 19), (19, 20)]
'''
Initialize Object
'''
def __init__(self, args):
# Ratio params
horizontal_ratio = float(args[0].horizontal_ratio)
vertical_ratio = float(args[0].vertical_ratio)
# Check video
if args[0].video != "enabled" and args[0].video != "disabled":
print("Error: set correct video mode, enabled or disabled")
sys.exit(-1)
# Check video
if args[0].image != "enabled" and args[0].image != "disabled":
print("Error: set correct image mode, enabled or disabled")
sys.exit(-1)
# Convert args to boolean
self.use_video = True if args[0].video == "enabled" else False
self.use_image = True if args[0].image == "enabled" else False
self.use_preview = True if args[0].preview == "enabled" else False
# Unable to use video and image mode at same time
if self.use_video and self.use_image:
print(
"Error: unable to use video and image mode at the same time!")
sys.exit(-1)
# Unable to not use or video or image mode at same time
if self.use_video and self.use_image:
print("Error: enable or video or image mode!")
sys.exit(-1)
self.streaming = True if args[0].streaming == "enabled" else False
if self.use_video:
# Open video capture
self.cap = cv2.VideoCapture(args[0].stream_in)
if not self.cap.isOpened():
print("Error: Opening video stream or file {0}".format(
args[0].stream_in))
sys.exit(-1)
# Get input size
width = int(self.cap.get(3))
height = int(self.cap.get(4))
if not self.streaming:
# Open video output (if output is not an image)
self.out = cv2.VideoWriter(args[0].stream_out,
cv2.VideoWriter_fourcc(*'XVID'),
int(self.cap.get(cv2.CAP_PROP_FPS)),
(width, height))
if self.out is None:
print("Error: Unable to open output video file {0}".format(
args[0].stream_out))
sys.exit(-1)
# Get image size
im_size = (width, height)
if self.use_image:
self.image = cv2.imread(args[0].image_in)
if self.image is None:
print("Error: Unable to open input image file {0}".format(
args[0].image_in))
sys.exit(-1)
self.image_out = args[0].image_out
# Get image size
im_size = (self.image.shape[1], self.image.shape[0])
# Compute Homograpy
self.homography_matrix = self.compute_homography(
horizontal_ratio, vertical_ratio, im_size)
self.background_masked = False
# Open image backgrouns, if it is necessary
if args[0].masked == "enabled":
# Set masked flag
self.background_masked = True
# Load static background
self.background_image = cv2.imread(args[0].background_in)
# Close, if no background, but required
if self.background_image is None:
print("Error: Unable to load background image (flag enabled)")
sys.exit(-1)
# Custom Params (refer to include/openpose/flags.hpp for more parameters)
params = dict()
# Openpose params
# Model path
params["model_folder"] = args[0].openpose_folder
# Face disabled
params["face"] = False
# Hand disabled
params["hand"] = False
# Net Resolution
params["net_resolution"] = args[0].net_size
# Gpu number
params["num_gpu"] = 1 # Set GPU number
# Gpu Id
# Set GPU start id (not considering previous)
params["num_gpu_start"] = 0
# Starting OpenPose
self.opWrapper = op.WrapperPython()
self.opWrapper.configure(params)
self.opWrapper.start()
# Process Image
self.datum = op.Datum()
# Json server
self.dt_vector = {}
# Client list
self.stream_list = []
if self.streaming:
# Initialize video server
self.video_server = StreamServer(int(args[0].video_port),
self.stream_list, "image/jpeg")
self.video_server.activate()
# Initialize json server
self.js_server = ResponseServer(int(args[0].js_port),
"application/json")
self.js_server.activate()
# turbo jpeg initialization
self.jpeg = TurboJPEG()
# Calibrate heigh value
self.calibrate = float(args[0].calibration)
# Actually unused
self.ellipse_angle = 0
# Body confidence threshold
self.body_th = float(args[0].body_threshold)
# Show confidence
self.show_confidence = True if args[
0].show_confidence == "enabled" else False
'''
Draw Skelethon
'''
def draw_skeleton(self, frame, keypoints, colour):
for keypoint_id1, keypoint_id2 in self.connections:
x1, y1 = keypoints[keypoint_id1]
x2, y2 = keypoints[keypoint_id2]
if 0 in (x1, y1, x2, y2):
continue
pt1 = int(round(x1)), int(round(y1))
pt2 = int(round(x2)), int(round(y2))
cv2.circle(frame,
center=pt1,
radius=4,
color=self.nd_color[keypoint_id2],
thickness=-1)
cv2.line(frame,
pt1=pt1,
pt2=pt2,
color=self.nd_color[keypoint_id2],
thickness=2)
'''
Compute skelethon bounding box
'''
def compute_simple_bounding_box(self, skeleton):
x = skeleton[::2]
x = np.where(x == 0.0, np.nan, x)
left, right = int(round(np.nanmin(x))), int(round(np.nanmax(x)))
y = skeleton[1::2]
y = np.where(y == 0.0, np.nan, y)
top, bottom = int(round(np.nanmin(y))), int(round(np.nanmax(y)))
return left, right, top, bottom
'''
Compute Homograpy
'''
def compute_homography(self, H_ratio, V_ratio, im_size):
rationed_hight = im_size[1] * V_ratio
rationed_width = im_size[0] * H_ratio
src = np.array([[0, 0], [0, im_size[1]], [im_size[0], im_size[1]],
[im_size[0], 0]])
dst = np.array([[0 + rationed_width / 2, 0 + rationed_hight],
[0, im_size[1]], [im_size[0], im_size[1]],
[im_size[0] - rationed_width / 2, 0 + rationed_hight]],
np.int32)
h, status = cv2.findHomography(src, dst)
return h
'''
Compute overlap
'''
def compute_overlap(self, rect_1, rect_2):
x_overlap = max(0,
min(rect_1[1], rect_2[1]) - max(rect_1[0], rect_2[0]))
y_overlap = max(0,
min(rect_1[3], rect_2[3]) - max(rect_1[2], rect_2[2]))
overlapArea = x_overlap * y_overlap
if overlapArea:
overlaps = True
else:
overlaps = False
return overlaps
'''
Trace results
'''
def trace(self, image, skeletal_coordinates, draw_ellipse_requirements,
is_skeletal_overlapped):
bodys = []
# Trace ellipses and body on target image
i = 0
for skeletal_coordinate in skeletal_coordinates[0]:
if float(skeletal_coordinates[1][i]) < self.body_th:
continue
# Trace ellipse
cv2.ellipse(image, (int(draw_ellipse_requirements[i][0]),
int(draw_ellipse_requirements[i][1])),
(int(draw_ellipse_requirements[i][2]),
int(draw_ellipse_requirements[i][3])), 0, 0, 360,
self.colors[int(is_skeletal_overlapped[i])], 3)
# Trace skelethon
skeletal_coordinate = np.array(skeletal_coordinate)
self.draw_skeleton(image, skeletal_coordinate.reshape(-1, 2),
(255, 0, 0))
if int(skeletal_coordinate[2]) != 0 and int(
skeletal_coordinate[3]) != 0 and self.show_confidence:
cv2.putText(
image, "{0:.2f}".format(skeletal_coordinates[1][i]),
(int(skeletal_coordinate[2]), int(skeletal_coordinate[3])),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2)
# Append json body data, joints coordinates, ground ellipses
bodys.append([[round(x) for x in skeletal_coordinate],
draw_ellipse_requirements[i],
int(is_skeletal_overlapped[i])])
i += 1
self.dt_vector["bodys"] = bodys
'''
Evaluate skelethon height
'''
def evaluate_height(self, skeletal_coordinate):
# Calculate skeleton height
calculated_height = 0
pointer = -1
# Left side body joints from top to down
#joint_set = [0, 1, 8, 12, 13, 14]
# Left leg
joint_set = [12, 13, 14]
# Check if leg is complete
left_leg = True
for k in joint_set:
x = int(skeletal_coordinate[k * 2])
y = int(skeletal_coordinate[k * 2 + 1])
if x == 0 or y == 0:
# No left leg, try right_leg
joint_set = [9, 10, 11]
left_leg = False
break
if not left_leg:
joint_set = [9, 10, 11]
# Check if leg is complete
for k in joint_set:
x = int(skeletal_coordinate[k * 2])
y = int(skeletal_coordinate[k * 2 + 1])
if x == 0 or y == 0:
# No left leg, no right leg, unable to evaluate ellipse
return 0
# Evaluate leg height
pointer = -1
for k in joint_set[:-1]:
pointer += 1
if skeletal_coordinate[joint_set[pointer]*2]\
and skeletal_coordinate[joint_set[pointer+1]*2]\
and skeletal_coordinate[joint_set[pointer]*2+1]\
and skeletal_coordinate[joint_set[pointer+1]*2+1]:
calculated_height = calculated_height +\
math.sqrt(((skeletal_coordinate[joint_set[pointer]*2] -
skeletal_coordinate[joint_set[pointer+1]*2])**2) +
((skeletal_coordinate[joint_set[pointer]*2+1] -
skeletal_coordinate[joint_set[pointer+1]*2+1])**2))
return calculated_height * self.calibrate
'''
Evaluate overlapping
'''
def evaluate_overlapping(self, ellipse_boxes, is_skeletal_overlapped,
ellipse_pool):
# checks for overlaps between people's ellipses, to determine risky or not
for ind1, ind2 in itertools.combinations(
list(range(0, len(ellipse_pool))), 2):
is_overlap = cv2.bitwise_and(ellipse_pool[ind1],
ellipse_pool[ind2])
if is_overlap.any() and (not is_skeletal_overlapped[ind1]
or not is_skeletal_overlapped[ind2]):
is_skeletal_overlapped[ind1] = 1
is_skeletal_overlapped[ind2] = 1
'''
Create Joint Array
'''
def create_joint_array(self, skeletal_coordinates):
# Get joints sequence
bodys_sequence = []
bodys_probability = []
for body in skeletal_coordinates:
body_sequence = []
body_probability = 0.0
# For each joint put it in vetcor list
for joint in body:
body_sequence.append(joint[0])
body_sequence.append(joint[1])
# Sum joints probability to find body probability
body_probability += joint[2]
body_probability = body_probability / len(body)
# Add body sequence to list
bodys_sequence.append(body_sequence)
bodys_probability.append(body_probability)
# Assign coordiates sequence
return [bodys_sequence, bodys_probability]
'''
Evaluate ellipses shadow, for each body
'''
def evaluate_ellipses(self, skeletal_coordinates,
draw_ellipse_requirements, ellipse_boxes,
ellipse_pool):
for skeletal_coordinate in skeletal_coordinates:
# Evaluate skeleton bounding box
left, right, top, bottom = self.compute_simple_bounding_box(
np.array(skeletal_coordinate))
bb_center = np.array([(left + right) / 2, (top + bottom) / 2],
np.int32)
calculated_height = self.evaluate_height(skeletal_coordinate)
# computing how the height of the circle varies in perspective
pts = np.array([[bb_center[0], top], [bb_center[0], bottom]],
np.float32)
pts1 = pts.reshape(-1, 1, 2).astype(np.float32) # (n, 1, 2)
dst1 = cv2.perspectiveTransform(pts1, self.homography_matrix)
# height of the ellipse in perspective
width = int(dst1[1, 0][1] - dst1[0, 0][1])
# Bounding box surrending the ellipses, useful to compute whether there is any overlap between two ellipses
ellipse_bbx = [
bb_center[0] - calculated_height,
bb_center[0] + calculated_height, bottom - width,
bottom + width
]
# Add boundig box to ellipse list
ellipse_boxes.append(ellipse_bbx)
ellipse = [
int(bb_center[0]),
int(bottom),
int(calculated_height),
int(width)
]
mask_copy = self.mask.copy()
ellipse_pool.append(
cv2.ellipse(mask_copy, (bb_center[0], bottom),
(int(calculated_height), width), 0, 0, 360,
(255, 255, 255), -1))
draw_ellipse_requirements.append(ellipse)
'''
Analyze image and evaluate distances
'''
def distances_evaluate(self, image, background):
ellipse_boxes = []
draw_ellipse_requirements = []
ellipse_pool = []
# Assign input image to openpose
self.datum.cvInputData = image
# Start wrapper
self.opWrapper.emplaceAndPop([self.datum])
# Get openpose coordinates (rounding values)
skeletal_coordinates = self.datum.poseKeypoints.tolist()
# Trace on background
if self.background_masked:
image = background
self.dt_vector['ts'] = int(round(time.time() * 1000))
self.dt_vector['bodys'] = []
if type(skeletal_coordinates) is list:
# Remove probability from joints and get a joint position list
skeletal_coordinates = self.create_joint_array(
skeletal_coordinates)
# Initialize overlapped buffer
is_skeletal_overlapped = np.zeros(
np.shape(skeletal_coordinates[0])[0])
# Evaluate ellipses for each body detected by openpose
self.evaluate_ellipses(skeletal_coordinates[0],
draw_ellipse_requirements, ellipse_boxes,
ellipse_pool)
# Evaluate overlapping
self.evaluate_overlapping(ellipse_boxes, is_skeletal_overlapped,
ellipse_pool)
# Trace results over output image
self.trace(image, skeletal_coordinates, draw_ellipse_requirements,
is_skeletal_overlapped)
if self.streaming:
# Send video to client queues
self.send_image(self.stream_list, image, int(self.dt_vector['ts']))
# Put json vector availble to rest requests
self.js_server.put(bytes(json.dumps(self.dt_vector), "UTF-8"))
return image
'''
Send image over queue list and then over http mjpeg stream
'''
def send_image(self, queue_list, image, ts):
encoded_image = self.jpeg.encode(image, quality=80)
# Put image into queue for each server thread
for q in queue_list:
try:
block = (ts, encoded_image)
q.put(block, True, 0.02)
except queue.Full:
pass
'''
Analyze video
'''
def analyze_video(self):
while self.cap.isOpened():
# Capture from image/video
ret, image = self.cap.read()
# Check image
if image is None or not ret:
os._exit(0)
self.mask = np.zeros(image.shape, dtype=np.uint8)
# Get openpose output
if self.background_masked:
background = self.background_image.copy()
else:
background = image
image = self.distances_evaluate(image, background)
# Write image
if not self.streaming:
self.out.write(image)
# Show image and wait some time
if self.use_preview:
cv2.imshow('Social Distance', image)
cv2.waitKey(1)
'''
Analyze image
'''
def analyze_image(self):
# Get openpose output
if self.background_masked:
background = self.background_image.copy()
else:
background = self.image
self.mask = np.zeros(self.image.shape, dtype=np.uint8)
self.image = self.distances_evaluate(self.image, background)
# Write image
cv2.imwrite(self.image_out, self.image)
# Show image and wait some time
if self.use_preview:
cv2.imshow('Social Distance', self.image)
cv2.waitKey(1000)
'''
Analyze image/video
'''
def analyze(self):
if self.use_image:
self.analyze_image()
if self.use_video:
self.analyze_video()
def __init__(self, args):
# Ratio params
horizontal_ratio = float(args[0].horizontal_ratio)
vertical_ratio = float(args[0].vertical_ratio)
# Check video
if args[0].video != "enabled" and args[0].video != "disabled":
print("Error: set correct video mode, enabled or disabled")
sys.exit(-1)
# Check video
if args[0].image != "enabled" and args[0].image != "disabled":
print("Error: set correct image mode, enabled or disabled")
sys.exit(-1)
# Convert args to boolean
self.use_video = True if args[0].video == "enabled" else False
self.use_image = True if args[0].image == "enabled" else False
self.use_preview = True if args[0].preview == "enabled" else False
# Unable to use video and image mode at same time
if self.use_video and self.use_image:
print(
"Error: unable to use video and image mode at the same time!")
sys.exit(-1)
# Unable to not use or video or image mode at same time
if self.use_video and self.use_image:
print("Error: enable or video or image mode!")
sys.exit(-1)
self.streaming = True if args[0].streaming == "enabled" else False
if self.use_video:
# Open video capture
self.cap = cv2.VideoCapture(args[0].stream_in)
if not self.cap.isOpened():
print("Error: Opening video stream or file {0}".format(
args[0].stream_in))
sys.exit(-1)
# Get input size
width = int(self.cap.get(3))
height = int(self.cap.get(4))
if not self.streaming:
# Open video output (if output is not an image)
self.out = cv2.VideoWriter(args[0].stream_out,
cv2.VideoWriter_fourcc(*'XVID'),
int(self.cap.get(cv2.CAP_PROP_FPS)),
(width, height))
if self.out is None:
print("Error: Unable to open output video file {0}".format(
args[0].stream_out))
sys.exit(-1)
# Get image size
im_size = (width, height)
if self.use_image:
self.image = cv2.imread(args[0].image_in)
if self.image is None:
print("Error: Unable to open input image file {0}".format(
args[0].image_in))
sys.exit(-1)
self.image_out = args[0].image_out
# Get image size
im_size = (self.image.shape[1], self.image.shape[0])
# Compute Homograpy
self.homography_matrix = self.compute_homography(
horizontal_ratio, vertical_ratio, im_size)
self.background_masked = False
# Open image backgrouns, if it is necessary
if args[0].masked == "enabled":
# Set masked flag
self.background_masked = True
# Load static background
self.background_image = cv2.imread(args[0].background_in)
# Close, if no background, but required
if self.background_image is None:
print("Error: Unable to load background image (flag enabled)")
sys.exit(-1)
# Custom Params (refer to include/openpose/flags.hpp for more parameters)
params = dict()
# Openpose params
# Model path
params["model_folder"] = args[0].openpose_folder
# Face disabled
params["face"] = False
# Hand disabled
params["hand"] = False
# Net Resolution
params["net_resolution"] = args[0].net_size
# Gpu number
params["num_gpu"] = 1 # Set GPU number
# Gpu Id
# Set GPU start id (not considering previous)
params["num_gpu_start"] = 0
# Starting OpenPose
self.opWrapper = op.WrapperPython()
self.opWrapper.configure(params)
self.opWrapper.start()
# Process Image
self.datum = op.Datum()
# Json server
self.dt_vector = {}
# Client list
self.stream_list = []
if self.streaming:
# Initialize video server
self.video_server = StreamServer(int(args[0].video_port),
self.stream_list, "image/jpeg")
self.video_server.activate()
# Initialize json server
self.js_server = ResponseServer(int(args[0].js_port),
"application/json")
self.js_server.activate()
# turbo jpeg initialization
self.jpeg = TurboJPEG()
# Calibrate heigh value
self.calibrate = float(args[0].calibration)
# Actually unused
self.ellipse_angle = 0
# Body confidence threshold
self.body_th = float(args[0].body_threshold)
# Show confidence
self.show_confidence = True if args[
0].show_confidence == "enabled" else False
class ProcessSource:
'''
Initialize
'''
def __init__(self, args):
# Social Distancing arguments
arguments = {}
# Arguments
arguments["horizontal_ratio"] = args[0].horizontal_ratio
arguments["vertical_ratio"] = args[0].vertical_ratio
arguments["calibration"] = args[0].calibration
arguments["body_threshold"] = args[0].body_threshold
arguments["show_confidence"] = args[0].show_confidence
arguments["show_sketch"] = args[0].show_sketch
# Initialize social distancing
self.social_distancing = SocialDistancing(arguments)
# Initialize Openpose
self.initialize_openpose(args)
# Initialize file opening/writing and streaming
self.initialize_others(args)
'''
Initialize openpose
'''
def initialize_openpose(self, args):
# Custom Params (refer to include/openpose/flags.hpp for more parameters)
params = dict()
# Openpose params
# Model path
params["model_folder"] = args[0].openpose_folder
# Face disabled
params["face"] = False
# Hand disabled
params["hand"] = False
# Net Resolution
params["net_resolution"] = args[0].net_size
# Gpu number
params["num_gpu"] = 1 # Set GPU number
# Gpu Id
# Set GPU start id (not considering previous)
params["num_gpu_start"] = 0
# Starting OpenPose
self.opWrapper = op.WrapperPython()
self.opWrapper.configure(params)
self.opWrapper.start()
self.datum = op.Datum()
'''
Initialize acquiring methods (video, mjpeg preprocessed json, jetson, etc), sockets, output files
'''
def initialize_others(self, args):
# Convert args to boolean
self.use_video = True if args[0].video == "enabled" else False
# Process image
self.use_image = True if args[0].image == "enabled" else False
# Use preview
self.use_preview = True if args[0].preview == "enabled" else False
# Jetson internal camera enabled
self.jetson_video = True if args[0].jetson_video == "enabled" else False
# Mjpeg video reader
self.use_mjpeg = True if args[0].use_mjpeg == "enabled" else False
# Enable streaming ption
self.streaming = True if args[0].streaming == "enabled" else False
# Use json as input
self.use_js = True if args[0].use_js == "enabled" else False
# Json input file
self.js_in = args[0].js_in
if self.use_video:
# Open video capture
if not self.jetson_video:
# Use standard cv2 capture library
self.cap = cv2.VideoCapture(args[0].stream_in)
else:
# Connect Standard cv2 capture library to gstreamer
print(gstreamer_pipeline(flip_method=0))
self.cap = cv2.VideoCapture(gstreamer_pipeline(flip_method=0),
cv2.CAP_GSTREAMER)
if not self.cap.isOpened():
print("Error: Opening video stream or file {0}".format(
args[0].stream_in),
flush=True)
sys.exit(-1)
# Get input size
width = int(self.cap.get(3))
height = int(self.cap.get(4))
self.mask_in = cv2.imread(args[0].mask_in)
if not self.streaming:
# Open video output (if output is not an image)
self.out = cv2.VideoWriter(
args[0].stream_out,
cv2.VideoWriter_fourcc(*args[0].encoding_codec), 25,
(width, height))
if self.out is None:
print("Error: Unable to open output video file {0}".format(
args[0].stream_out),
flush=True)
sys.exit(-1)
# Get image size
im_size = (width, height)
if self.use_image:
self.mask_in = cv2.imread(args[0].mask_in)
self.image = cv2.imread(args[0].image_in)
if self.image is None:
print("Error: Unable to open input image file {0}".format(
args[0].image_in),
flush=True)
sys.exit(-1)
self.image_out = args[0].image_out
# Get image size
im_size = (self.image.shape[1], self.image.shape[0])
if self.use_mjpeg:
# Create mjpeg reader
self.mjpeg_reader = MjpegReader(args[0].stream_in)
# Read first image to get image size
image = self.mjpeg_reader.get_image()
if not self.mjpeg_reader.is_opened():
print("Error: Unable to open input image file {0}".format(
args[0].image_in),
flush=True)
exit(-1)
# Get input size
width = int(image.shape[1])
height = int(image.shape[0])
if not self.streaming:
# Open video output (if output is not an image)
self.out = cv2.VideoWriter(
args[0].stream_out,
cv2.VideoWriter_fourcc(*args[0].encoding_codec),
int(args[0].dummy_fps), (width, height))
if self.out is None:
print("Error: Unable to open output video file {0}".format(
args[0].stream_out),
flush=True)
sys.exit(-1)
print("Mjpeg multipart file:{0}x{1}".format(width, height))
# Get image size
im_size = (width, height)
if not self.use_js:
# Compute Homograpy
self.social_distancing.compute_homography(im_size)
self.background_masked = False
# Open image backgrouns, if it is necessary
if args[0].masked == "enabled":
# Set masked flag
self.background_masked = True
# Load static background
self.background_image = cv2.imread(args[0].background_in)
# Close, if no background, but required
if self.background_image is None:
print("Error: Unable to load background image (flag enabled)",
flush=True)
sys.exit(-1)
if self.use_js:
im_size = (self.background_image.shape[1],
self.background_image.shape[0])
self.out = cv2.VideoWriter(
args[0].stream_out,
cv2.VideoWriter_fourcc(*args[0].encoding_codec),
int(args[0].dummy_fps), im_size)
self.social_distancing.compute_homography(im_size)
# Json server
self.dt_vector = {}
# Client list
self.stream_list = []
self.js_list = []
if self.streaming:
# Initialize video server
self.video_server = StreamServer(int(args[0].video_port),
self.stream_list, "image/jpeg")
self.video_server.activate()
# Initialize stream server
self.stream_server = StreamServer(int(args[0].stream_port),
self.js_list, "application/json")
self.stream_server.activate()
# Initialize json server
self.js_server = ResponseServer(int(args[0].js_port),
"application/json")
self.js_server.activate()
# turbo jpeg initialization
self.jpeg = TurboJPEG()
# Json recorder
self.js_recording = False
if args[0].js_out != "":
self.js_recording = True
self.js_out = open(args[0].js_out, "w")
# Mjpeg recorder
self.mjpeg_recorder = False
if args[0].mjpeg_out != "":
self.mjpeg_recorder = True
self.mjpeg_out = open(args[0].mjpeg_out, "wb")
# Json version
self.dt_vector["vs"] = 1
# Fps evaluation init
self.millis = 0
self.frames = 0
'''
Process source and save on image/video/js file, distribuite on network
'''
def process_source(self, source, background):
start = round(time.time() * 1000)
if self.mask_in is not None:
source = cv2.bitwise_and(source, self.mask_in)
# Check if pre-processed json is used
if not self.use_js:
# Assign input image to openpose
self.datum.cvInputData = source
# Use Openpose to extract poses
self.opWrapper.emplaceAndPop([self.datum])
# Get openpose coordinates (rounding values)
skeletals = np.around(
np.array(self.datum.poseKeypoints).tolist(), 2).tolist()
else:
# Copy json data
skeletals = source
# Trace on background
if self.background_masked:
source = background
if type(skeletals) is not list:
return background
# Evaluate distances, draw body and ellipses and get json bodies and ellipses list
image, bodies, ellipses = self.social_distancing.distances_calculate(
source, skeletals, [1 for k in range(len(skeletals))])
# Save data to json vector
self.dt_vector["bodies"] = bodies
self.dt_vector["ellipses"] = ellipses
if self.streaming:
# Send video to client queues
self.send_image(self.stream_list, image, int(self.dt_vector['ts']))
# Put json vector availble to rest requests
self.js_server.put(bytes(json.dumps(self.dt_vector), "UTF-8"))
# Send json vestor available to streaming
self.send_js(self.js_list,
bytes(json.dumps(self.dt_vector), "UTF-8"),
int(self.dt_vector['ts']))
# Write json data
if self.js_recording:
self.js_out.write(json.dumps(self.dt_vector) + "\n")
stop = round(time.time() * 1000)
if self.millis > 1000:
print("Analyzing at {0} Fps".format(self.frames),
end="\r",
flush=True)
self.millis = 0
self.frames = 0
self.millis += stop - start
self.frames += 1
return image
'''
Send image over queue list and then over http mjpeg stream
'''
def send_image(self, queue_list, image, ts):
encoded_image = self.jpeg.encode(image, quality=80)
# Put image into queue for each server thread
for q in queue_list:
try:
block = (ts, encoded_image)
q.put(block, True, 0.02)
except queue.Full:
pass
'''
Send json over queue list and then over http multipart stream
'''
def send_js(self, queue_list, js, ts):
# Put json into queue for each server thread
for q in queue_list:
try:
block = (ts, js)
q.put(block, True, 0.02)
except queue.Full:
pass
'''
Analyze video
'''
def analyze_video(self):
first_frame = True
counter = 0
while self.cap.isOpened():
# Get a global image ts
self.dt_vector['ts'] = int(round(time.time() * 1000))
# Capture from image/video
ret, image = self.cap.read()
# Check image
if image is None or not ret:
os._exit(0)
# Record image
if self.mjpeg_recorder:
encoded_image = self.jpeg.encode(image, quality=80)
header = "--myboundary\r\n" \
"X-TimeStamp: " + str(self.dt_vector['ts']) + "\r\n" \
"Content-Type: image/jpeg\r\n" \
"Content-Length: " + \
str(len(encoded_image)) + "\r\n\r\n"
self.mjpeg_out.write(bytes(header, "UTF-8"))
self.mjpeg_out.write(encoded_image)
# Get openpose output
if self.background_masked:
background = self.background_image.copy()
else:
background = image
image = self.process_source(image, background)
# Write image
if not self.streaming:
self.out.write(image)
# Show image and wait some time
if self.use_preview:
cv2.imshow('Social Distance', image)
cv2.waitKey(1)
#print(counter, end="\r", flush=True)
counter += 1
'''
Analyze image
'''
def analyze_image(self):
# Get openpose output
if self.background_masked:
background = self.background_image.copy()
else:
background = self.image
self.image = self.process_source(self.image, background)
# Write image
cv2.imwrite(self.image_out, self.image)
# Show image and wait some time
if self.use_preview:
cv2.imshow('Social Distance', self.image)
cv2.waitKey(1000)
'''
Analyze json data
'''
def analyze_js(self):
# Read json files
lines = open(self.js_in, "r").read().split("\n")
# While there are lines
for line in lines[:-1]:
js_line = json.loads(line)
# Create
background = self.background_image.copy()
if 'vs' in js_line.keys():
self.image = self.process_source(js_line['bodies'], background)
else:
self.image = self.process_source(js_line['bodys'], background)
# Write image
if not self.streaming:
self.out.write(self.image)
# Show image and wait some time
if self.use_preview:
cv2.imshow('Social Distance', self.image)
cv2.waitKey(1)
'''
Analyze mjpeg (timestamped jpeg sequence)
'''
def analyze_mjpeg(self):
first_frame = True
counter = 0
old_timestamp = self.mjpeg_reader.get_ts()
while True:
# Capture from image/video
image = self.mjpeg_reader.get_image()
# Get a global image ts
self.dt_vector['ts'] = self.mjpeg_reader.get_ts()
# Check image
if image is None:
os._exit(0)
# Record image
if self.mjpeg_recorder:
encoded_image = self.jpeg.encode(image, quality=80)
header = "--myboundary\r\n" \
"X-TimeStamp: " + str(self.dt_vector['ts']) + "\r\n" \
"Content-Type: image/jpeg\r\n" \
"Content-Length: " + \
str(len(encoded_image)) + "\r\n\r\n"
self.mjpeg_out.write(bytes(header, "UTF-8"))
self.mjpeg_out.write(encoded_image)
# Get openpose output
if self.background_masked:
background = self.background_image.copy()
else:
background = image
image = self.process_source(image, background)
# Write image
if not self.streaming:
self.out.write(image)
# Show image and wait some time
if self.use_preview:
cv2.imshow('Social Distance', image)
cv2.waitKey(1)
# Wait timestamp difference
time.sleep((self.mjpeg_reader.get_ts() - old_timestamp) / 1000)
# print(counter, end = "\n", flush=True)
# Store old timestamp
old_timestamp = self.mjpeg_reader.get_ts()
counter += 1
'''
Analyze image/video/json/mjpeg
'''
def analyze(self):
if self.use_image:
self.analyze_image()
if self.use_video:
self.analyze_video()
if self.use_js:
self.analyze_js()
if self.use_mjpeg:
self.analyze_mjpeg()
def initialize_others(self, args):
# Convert args to boolean
self.use_video = True if args[0].video == "enabled" else False
# Process image
self.use_image = True if args[0].image == "enabled" else False
# Use preview
self.use_preview = True if args[0].preview == "enabled" else False
# Jetson internal camera enabled
self.jetson_video = True if args[0].jetson_video == "enabled" else False
# Mjpeg video reader
self.use_mjpeg = True if args[0].use_mjpeg == "enabled" else False
# Enable streaming ption
self.streaming = True if args[0].streaming == "enabled" else False
# Use json as input
self.use_js = True if args[0].use_js == "enabled" else False
# Json input file
self.js_in = args[0].js_in
if self.use_video:
# Open video capture
if not self.jetson_video:
# Use standard cv2 capture library
self.cap = cv2.VideoCapture(args[0].stream_in)
else:
# Connect Standard cv2 capture library to gstreamer
print(gstreamer_pipeline(flip_method=0))
self.cap = cv2.VideoCapture(gstreamer_pipeline(flip_method=0),
cv2.CAP_GSTREAMER)
if not self.cap.isOpened():
print("Error: Opening video stream or file {0}".format(
args[0].stream_in),
flush=True)
sys.exit(-1)
# Get input size
width = int(self.cap.get(3))
height = int(self.cap.get(4))
self.mask_in = cv2.imread(args[0].mask_in)
if not self.streaming:
# Open video output (if output is not an image)
self.out = cv2.VideoWriter(
args[0].stream_out,
cv2.VideoWriter_fourcc(*args[0].encoding_codec), 25,
(width, height))
if self.out is None:
print("Error: Unable to open output video file {0}".format(
args[0].stream_out),
flush=True)
sys.exit(-1)
# Get image size
im_size = (width, height)
if self.use_image:
self.mask_in = cv2.imread(args[0].mask_in)
self.image = cv2.imread(args[0].image_in)
if self.image is None:
print("Error: Unable to open input image file {0}".format(
args[0].image_in),
flush=True)
sys.exit(-1)
self.image_out = args[0].image_out
# Get image size
im_size = (self.image.shape[1], self.image.shape[0])
if self.use_mjpeg:
# Create mjpeg reader
self.mjpeg_reader = MjpegReader(args[0].stream_in)
# Read first image to get image size
image = self.mjpeg_reader.get_image()
if not self.mjpeg_reader.is_opened():
print("Error: Unable to open input image file {0}".format(
args[0].image_in),
flush=True)
exit(-1)
# Get input size
width = int(image.shape[1])
height = int(image.shape[0])
if not self.streaming:
# Open video output (if output is not an image)
self.out = cv2.VideoWriter(
args[0].stream_out,
cv2.VideoWriter_fourcc(*args[0].encoding_codec),
int(args[0].dummy_fps), (width, height))
if self.out is None:
print("Error: Unable to open output video file {0}".format(
args[0].stream_out),
flush=True)
sys.exit(-1)
print("Mjpeg multipart file:{0}x{1}".format(width, height))
# Get image size
im_size = (width, height)
if not self.use_js:
# Compute Homograpy
self.social_distancing.compute_homography(im_size)
self.background_masked = False
# Open image backgrouns, if it is necessary
if args[0].masked == "enabled":
# Set masked flag
self.background_masked = True
# Load static background
self.background_image = cv2.imread(args[0].background_in)
# Close, if no background, but required
if self.background_image is None:
print("Error: Unable to load background image (flag enabled)",
flush=True)
sys.exit(-1)
if self.use_js:
im_size = (self.background_image.shape[1],
self.background_image.shape[0])
self.out = cv2.VideoWriter(
args[0].stream_out,
cv2.VideoWriter_fourcc(*args[0].encoding_codec),
int(args[0].dummy_fps), im_size)
self.social_distancing.compute_homography(im_size)
# Json server
self.dt_vector = {}
# Client list
self.stream_list = []
self.js_list = []
if self.streaming:
# Initialize video server
self.video_server = StreamServer(int(args[0].video_port),
self.stream_list, "image/jpeg")
self.video_server.activate()
# Initialize stream server
self.stream_server = StreamServer(int(args[0].stream_port),
self.js_list, "application/json")
self.stream_server.activate()
# Initialize json server
self.js_server = ResponseServer(int(args[0].js_port),
"application/json")
self.js_server.activate()
# turbo jpeg initialization
self.jpeg = TurboJPEG()
# Json recorder
self.js_recording = False
if args[0].js_out != "":
self.js_recording = True
self.js_out = open(args[0].js_out, "w")
# Mjpeg recorder
self.mjpeg_recorder = False
if args[0].mjpeg_out != "":
self.mjpeg_recorder = True
self.mjpeg_out = open(args[0].mjpeg_out, "wb")
# Json version
self.dt_vector["vs"] = 1
# Fps evaluation init
self.millis = 0
self.frames = 0