def __init__(self, server_address):
'''init func'''
self.channel_manager = ChannelManager(["image", "video"])
super(DisplayServer, self).__init__(server_address)
self.app_manager = AppManager()
self.register_dict = {}
def __init__(self):
"""
init method
"""
self.channel_mgr = ChannelManager(["image", "video"])
self.request_list = set()
self.lock = threading.Lock()
def stop_thread(self):
"""
Description: clean thread when process exit.
Input: NA
Returns: NA
"""
channel_manager = ChannelManager([])
channel_manager.close_all_thread()
self.set_exit_switch()
self.app_manager.set_thread_switch()
def close_all_thread(signum, frame):
'''close all thread of the process, and exit.'''
logging.info("receive signal, signum:%s, frame:%s", signum, frame)
webapp.stop_webapp()
SOCKET_SERVER.stop_thread()
channel_manager = ChannelManager()
channel_manager.close_all_thread()
logging.info("presenter server exit by Ctrl + c")
sys.exit()
def __init__(self):
"""
init method
"""
self.channel_mgr = ChannelManager(["image", "video"])
self.request_list = set()
self.lock = threading.Lock()
self.anti = antispoof()
print (self.anti.get_ans())
def __init__(self, server_address):
"""
Args:
server_address: server listen address,
include an ipv4 address and a port.
"""
# thread exit switch, if set true, thread must exit immediately.
self._thread_exit_switch = False
self._config = config_parser.ConfigParser()
self._channel_manager = ChannelManager()
self._create_socket_server(server_address)
def __init__(self):
"""
init method
"""
self.channel_mgr = ChannelManager(["image", "video"])
self.request_list = set()
self.lock = threading.Lock()
self.display_manager = display_server.DisplayServerManager()
self.thread = None
self._create_sending_thread()
def __init__(self, channel_name, media_type):
self._channel_name = channel_name
self._media_type = media_type
self._img_data = None
self._thread = None
# last time the channel receive data.
self._heartbeat = time.time()
self._web_event = ThreadEvent(timeout=WEB_EVENT_TIMEOUT)
self._image_event = ThreadEvent(timeout=IMAGE_EVENT_TIMEOUT)
self._frame = None
self._lock = threading.Lock()
self._channel_manager = ChannelManager()
if media_type == "video":
self._thread_name = "videothread-{}".format(self._channel_name)
self._create_thread()
def __new__(cls):
"""ensure only a single instance created. """
if cls.__instance is None:
cls.__instance = object.__new__(cls)
cls.channel_manager = ChannelManager([])
cls._create_thread()
return cls.__instance
def __init__(self, config):
"""
Description: class init func
Input:
config: config information
Returns: NA
"""
server_address = (config.presenter_server_ip,
int(config.presenter_server_port))
super(FacialRecognitionServer, self).__init__(server_address)
self.storage_dir = config.storage_dir
self.max_face_num = int(config.max_face_num)
self.face_match_threshold = float(config.face_match_threshold)
self.register_dict = {}
self.app_manager = AppManager()
self.channel_manager = ChannelManager()
self.face_register_file = os.path.join(self.storage_dir,
"registered_faces.json")
self._init_face_database()
def __init__(self, channel_name, media_type):
self.channel_name = channel_name
self.media_type = media_type
self.img_data = None
self._frame = None
self.thread = None
self._frame = None
# last time the channel receive data.
self.heartbeat = time.time()
self.web_event = ThreadEvent(timeout=WEB_EVENT_TIMEOUT)
self.image_event = ThreadEvent(timeout=IMAGE_EVENT_TIMEOUT)
self.lock = threading.Lock()
self.channel_manager = ChannelManager([])
self.rectangle_list = None
if media_type == "video":
self.thread_name = "videothread-{}".format(self.channel_name)
self.heartbeat = time.time()
self.close_thread_switch = False
self.fps = 0
self.image_number = 0
self.time_list = []
self._create_thread()
def __init__(self, server_address):
'''init func'''
self.channel_manager = ChannelManager(["image", "video"])
super(CarRecognitionServer, self).__init__(server_address)
class FacialRecognitionServer(PresenterSocketServer):
'''A server for face recognition'''
def __init__(self, config):
"""
Description: class init func
Input:
config: config information
Returns: NA
"""
server_address = (config.presenter_server_ip,
int(config.presenter_server_port))
super(FacialRecognitionServer, self).__init__(server_address)
self.storage_dir = config.storage_dir
self.max_face_num = int(config.max_face_num)
self.face_match_threshold = float(config.face_match_threshold)
self.register_dict = {}
self.app_manager = AppManager()
self.channel_manager = ChannelManager()
self.face_register_file = os.path.join(self.storage_dir,
"registered_faces.json")
self._init_face_database()
def _init_face_database(self):
"""
Description: Init face recognition database,
read information from face_register_file
Input: NA
Returns: NA
"""
if not os.path.isfile(self.face_register_file):
with open(self.face_register_file, "w", encoding="utf-8") as f:
f.write("{}")
with open(self.face_register_file, "r") as f:
self.face_lock = threading.Lock()
self.registered_faces = json.load(f)
self._filter_registration_data()
def _filter_registration_data(self):
face_dict = self.registered_faces.copy()
for i in face_dict:
image_path = os.path.join(self.storage_dir, i + ".jpg")
if not os.path.isfile(image_path):
del self.registered_faces[i]
def get_all_face(self):
"""
Description: get registered face list.
Input: NA
Returns: NA
"""
with self.face_lock:
return [i for i in self.registered_faces]
def save_face_image(self, name, image):
"""
Description: save face image.
Input:
name face name
image: face image
Returns: True or False
"""
image_file = os.path.join(self.storage_dir, name + ".jpg")
try:
#image = image.decode("utf-8")
with open(image_file, "wb") as f:
f.write(image)
return True
except (OSError, TypeError) as exp:
logging.error(exp)
return False
def get_app_socket(self, app_id):
"""
Description: get a socket which is bound to the app.
Input:
app_id: id of the app
Returns: socket
"""
return self.app_manager.get_socket_by_app_id(app_id)
def list_registered_apps(self):
"""
Description: get registered apps list.
Input: NA
Returns: app list
"""
return self.app_manager.list_app()
def delete_faces(self, name_list):
"""
Description: delete registered faces in name_list
Input:
name_list: a name list
Returns: True or False
"""
with self.face_lock:
for i in name_list:
if self.registered_faces.get(i):
backup = self.registered_faces[i]
del self.registered_faces[i]
try:
with open(self.face_register_file, "w") as f:
json.dump(self.registered_faces, f)
image_file = os.path.join(self.storage_dir, i + ".jpg")
os.remove(image_file)
except (OSError, JSONDecodeError) as exp:
logging.error(exp)
self.registered_faces[i] = backup
return False
return True
def _clean_connect(self, sock_fileno, epoll, conns, msgs):
"""
Description: close socket, and clean local variables
Input:
sock_fileno: a socket fileno, return value of socket.fileno()
epoll: a set of select.epoll.
conns: all socket connections registered in epoll
msgs: msg read from a socket
"""
logging.info("clean fd:%s, conns:%s", sock_fileno, conns)
self.app_manager.unregister_app_by_fd(sock_fileno)
epoll.unregister(sock_fileno)
conns[sock_fileno].close()
del conns[sock_fileno]
del msgs[sock_fileno]
def _process_msg(self, conn, msg_name, msg_data):
"""
Total entrance to process protobuf msg
Input:
conn: a socket connection
msg_name: name of a msg.
msg_data: msg body, serialized by protobuf
Returns:
False:somme error occured
True:succeed
"""
# process open channel request
if msg_name == pb2._REGISTERAPP.full_name:
ret = self._process_register_app(conn, msg_data)
# process image request, receive an image data from presenter agent
elif msg_name == pb2._FACERESULT.full_name:
ret = self._process_face_result(msg_data)
elif msg_name == pb2._FRAMEINFO.full_name:
ret = self._process_frame_info(conn, msg_data)
elif msg_name == presenter_message_pb2._OPENCHANNELREQUEST.full_name:
ret = self._process_open_channel(conn, msg_data)
# process heartbeat request, it used to keepalive a channel path
elif msg_name == presenter_message_pb2._HEARTBEATMESSAGE.full_name:
ret = self._process_heartbeat(conn)
else:
logging.error("Not recognized msg type %s", msg_name)
ret = False
return ret
def _process_heartbeat(self, conn):
'''
set heartbeat
Input:
conn: a socket connection
Returns:
True: set heartbeat ok.
'''
sock_fileno = conn.fileno()
if self.app_manager.get_app_id_by_socket(sock_fileno):
self.app_manager.set_heartbeat(sock_fileno)
handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno)
if handler is not None:
handler.set_heartbeat()
return True
def _parse_protobuf(self, protobuf, msg_data):
"""
Description: parse protobuf
Input:
protobuf: a struct defined by protobuf
msg_data: msg body, serialized by protobuf
Returns: True or False
"""
try:
protobuf.ParseFromString(msg_data)
return True
except DecodeError as exp:
logging.error(exp)
return False
def _process_register_app(self, conn, msg_data):
"""
Description: process register_app message
Input:
conn: a socket connection
msg_data: msg body, serialized by protobuf
Returns: True or False
"""
request = pb2.RegisterApp()
response = pb2.CommonResponse()
msg_name = pb2._COMMONRESPONSE.full_name
if not self._parse_protobuf(request, msg_data):
response.ret = pb2.kErrorOther
response.message = "ParseFromString exception"
self.send_message(conn, response, msg_name)
return False
app_id = request.id
app_type = request.type
# check app id if exist
if self.app_manager.is_app_exist(app_id):
logging.error("App %s is already exist.", app_id)
response.ret = pb2.kErrorAppRegisterExist
response.message = "App {} is already exist.".format(app_id)
self.send_message(conn, response, msg_name)
elif self.app_manager.get_app_num() >= MAX_APP_NUM:
logging.error("App number reach the upper limit")
response.ret = pb2.kErrorAppRegisterLimit
response.message = "App number reach the upper limit"
self.send_message(conn, response, msg_name)
elif app_type != SERVER_TYPE:
logging.error("App type %s error", app_type)
response.ret = pb2.kErrorAppRegisterType
response.message = "App type {} error".format(app_type)
self.send_message(conn, response, msg_name)
elif len(app_id) > APP_ID_MAX_LENGTH:
logging.error("App id %s is too long", app_id)
response.ret = pb2.kErrorOther
response.message = "App id: {} is too long".format(app_id)
self.send_message(conn, response, msg_name)
else:
self.app_manager.register_app(app_id, conn)
response.ret = pb2.kErrorNone
response.message = "Register app {} succeed".format(app_id)
self.send_message(conn, response, msg_name)
return True
return False
def _process_face_result(self, msg_data):
"""
Description: process face_result message
Input:
msg_data: msg body, serialized by protobuf
Returns: True or False
"""
face_result = pb2.FaceResult()
if not self._parse_protobuf(face_result, msg_data):
return False
face_id = face_result.id
if not self.register_dict.get(face_id):
logging.warning("face id %s is already deleted", face_id)
return True
ret = face_result.response.ret
if ret != pb2.kErrorNone:
err_msg = face_result.response.message
logging.error("get face feature error message: %s", err_msg)
status = FACE_REGISTER_STATUS_FAILED
message = "Get face feature failed"
self._update_register_dict(face_id, status, message)
return True
face_num = len(face_result.feature)
if face_num == 0:
status = FACE_REGISTER_STATUS_FAILED
message = "No face recognized"
self._update_register_dict(face_id, status, message)
elif face_num > 1:
status = FACE_REGISTER_STATUS_FAILED
message = "{} faces recognized".format(face_num)
self._update_register_dict(face_id, status, message)
else:
box = face_result.feature[0].box
face_coordinate = [box.lt_x, box.lt_y, box.rb_x, box.rb_x]
feature_vector = [i for i in face_result.feature[0].vector]
if len(feature_vector) != FEATURE_VECTOR_LENGTH:
logging.error("feature_vector length not equal 1024")
status = FACE_REGISTER_STATUS_FAILED
message = "Face feature vector length invalid"
self._update_register_dict(face_id, status, message)
return True
return self._save_face_feature(face_id, face_coordinate,
feature_vector)
return True
def _update_register_dict(self, face_id, status, message):
"""
Description: update register_dict
Input:
face_id: id of face
status: status of face register
message: message of status of face register
Returns: True or False
"""
if self.register_dict.get(face_id):
self.register_dict[face_id]["status"] = status
self.register_dict[face_id]["message"] = message
self.register_dict[face_id]["event"].set()
def _save_face_feature(self, face_id, face_coordinate, feature_vector):
"""
Description: save face_feature
Input:
face_id: id of face
face_coordinate: face coordinates
feature_vector: face feature vector
Returns: True or False
"""
with self.face_lock:
self.registered_faces[face_id] = {
"coordinate": face_coordinate,
"feature": feature_vector
}
try:
with open(self.face_register_file, "w") as f:
json.dump(self.registered_faces, f)
status = FACE_REGISTER_STATUS_SUCCEED
message = "Successful registration"
self._update_register_dict(face_id, status, message)
return True
except (OSError, JSONDecodeError) as exp:
logging.error(exp)
del self.registered_faces[face_id]
status = FACE_REGISTER_STATUS_FAILED
message = "save face feature to json file failed"
self._update_register_dict(face_id, status, message)
return False
def _process_open_channel(self, conn, msg_data):
"""
Description: process open channel message
Input:
conn: a socket connection
msg_data: msg body, serialized by protobuf
Returns: True or False
"""
request = presenter_message_pb2.OpenChannelRequest()
response = presenter_message_pb2.OpenChannelResponse()
if not self._parse_protobuf(request, msg_data):
channel_name = "unknown channel"
err_code = presenter_message_pb2.kOpenChannelErrorOther
return self._response_open_channel(conn, channel_name, response,
err_code)
channel_name = request.channel_name
# check channel name if exist
if not self.channel_manager.is_channel_exist(channel_name):
logging.error("channel name %s is not exist.", channel_name)
err_code = presenter_message_pb2.kOpenChannelErrorNoSuchChannel
return self._response_open_channel(conn, channel_name, response,
err_code)
# check channel path if busy
if self.channel_manager.is_channel_busy(channel_name):
logging.error("channel path %s is busy.", channel_name)
err = presenter_message_pb2.kOpenChannelErrorChannelAlreadyOpened
return self._response_open_channel(conn, channel_name, response,
err)
content_type = presenter_message_pb2.kChannelContentTypeVideo
if request.content_type == content_type:
media_type = "video"
else:
logging.error("media type %s is not recognized.",
request.content_type)
err_code = presenter_message_pb2.kOpenChannelErrorOther
return self._response_open_channel(conn, channel_name, response,
err_code)
handler = FacialRecognitionHandler(channel_name, media_type)
sock = conn.fileno()
self.channel_manager.create_channel_resource(channel_name, sock,
media_type, handler)
err_code = presenter_message_pb2.kOpenChannelErrorNone
return self._response_open_channel(conn, channel_name, response,
err_code)
def _process_frame_info(self, conn, msg_data):
"""
Description: process frame info message
Input:
conn: a socket connection
msg_data: msg body, serialized by protobuf
Returns: True or False
"""
request = pb2.FrameInfo()
response = pb2.CommonResponse()
msg_name = pb2._COMMONRESPONSE.full_name
if not self._parse_protobuf(request, msg_data):
return False
sock_fileno = conn.fileno()
handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno)
if handler is None:
logging.error("get channel handler failed")
response.ret = pb2.kErrorOther
response.message = "channel error."
self.send_message(conn, response, msg_name)
return False
face_list = self._recognize_face(request.feature)
handler.save_frame(request.image, face_list)
response.ret = pb2.kErrorNone
response.message = "process frame info suceed."
self.send_message(conn, response, msg_name)
return True
def _recognize_face(self, face_feature):
"""
Description: recognize which face it is.
Input:
face_feature: face feature
Returns: face list
"""
face_list = []
for i in face_feature:
face_info = {}
box = i.box
coordinate = [box.lt_x, box.lt_y, box.rb_x, box.rb_y]
feature_vector = i.vector
if len(feature_vector) != FEATURE_VECTOR_LENGTH:
logging.error("feature_vector length not equal 1024")
continue
(name, score, allmsg) = self._compute_face_feature(feature_vector)
face_info["coordinate"] = coordinate
face_info["name"] = name
face_info["confidence"] = score
face_info["allmsg"] = allmsg
face_list.append(face_info)
return face_list
def _compute_face_feature(self, feture_vector):
"""
Description: compute score of the feture_vector
Input:
feture_vector: face feature vector
Returns: face name and score
"""
highest_score_face = "Unknown"
highest_score = 0
allmsg = ""
with self.face_lock:
for i in self.registered_faces:
feature = self.registered_faces[i]["feature"]
score = self._compute_similar_degree(feature, feture_vector)
if score < self.face_match_threshold:
continue
allmsg += str(i) + ":" + str(round(score, 2)) + ","
if score > highest_score:
highest_score = score
highest_score_face = i
if allmsg != "":
allmsg = allmsg[:-1]
return (highest_score_face, highest_score, allmsg)
def _compute_similar_degree(self, feture_vector1, feture_vector2):
"""
Description: compute cosine similarity of two vectors
Input:
feture_vector1: face feature vector
feture_vector2: face feature vector
Returns: score
"""
vector1 = np.array(feture_vector1)
vector2 = np.array(feture_vector2)
square_diff = ((np.linalg.norm(vector1)) * (np.linalg.norm(vector2)))
score = np.dot(vector1, vector2) / square_diff
return score
def stop_thread(self):
"""
Description: clean thread when process exit.
Input: NA
Returns: NA
"""
channel_manager = ChannelManager([])
channel_manager.close_all_thread()
self.set_exit_switch()
self.app_manager.set_thread_switch()
def __init__(self, server_address):
'''init func'''
self.channel_manager = ChannelManager(["image", "video"])
super(crowdcountingServer, self).__init__(server_address)
def __init__(self, server_address):
'''init func'''
self.channel_manager = ChannelManager(["image", "video"])
super(peppapigdetectionServer, self).__init__(server_address)
class peppapigdetectionServer(PresenterSocketServer):
'''A server for face detection'''
def __init__(self, server_address):
'''init func'''
self.channel_manager = ChannelManager(["image", "video"])
super(peppapigdetectionServer, self).__init__(server_address)
def _clean_connect(self, sock_fileno, epoll, conns, msgs):
"""
close socket, and clean local variables
Args:
sock_fileno: a socket fileno, return value of socket.fileno()
epoll: a set of select.epoll.
conns: all socket connections registered in epoll
msgs: msg read from a socket
"""
logging.info("clean fd:%s, conns:%s", sock_fileno, conns)
self.channel_manager.clean_channel_resource_by_fd(sock_fileno)
epoll.unregister(sock_fileno)
conns[sock_fileno].close()
del conns[sock_fileno]
del msgs[sock_fileno]
def _process_msg(self, conn, msg_name, msg_data):
"""
Total entrance to process protobuf msg
Args:
conn: a socket connection
msg_name: name of a msg.
msg_data: msg body, serialized by protobuf
Returns:
False:somme error occured
True:succeed
"""
# process open channel request
if msg_name == pb2._OPENCHANNELREQUEST.full_name:
ret = self._process_open_channel(conn, msg_data)
# process image request, receive an image data from presenter agent
elif msg_name == pb2._PRESENTIMAGEREQUEST.full_name:
ret = self._process_image_request(conn, msg_data)
# process heartbeat request, it used to keepalive a channel path
elif msg_name == pb2._HEARTBEATMESSAGE.full_name:
ret = self._process_heartbeat(conn)
else:
logging.error("Not recognized msg type %s", msg_name)
ret = False
return ret
def _response_image_request(self, conn, response, err_code):
"""
Assemble protobuf to response image_request
Message structure like this:
--------------------------------------------------------------------
|total message len | int | 4 bytes |
|-------------------------------------------------------------------
|message name len | byte | 1 byte |
|-------------------------------------------------------------------
|message name | string | xx bytes |
|-------------------------------------------------------------------
|message body | protobuf | xx bytes |
--------------------------------------------------------------------
protobuf structure like this:
--------------------------------------------------------------------
|error_code | enum | PresentDataErrorCode |
|-------------------------------------------------------------------
|error_message | string | xx bytes |
|-------------------------------------------------------------------
enum PresentDataErrorCode {
kPresentDataErrorNone = 0;
kPresentDataErrorUnsupportedType = 1;
kPresentDataErrorUnsupportedFormat = 2;
kPresentDataErrorOther = -1;
}
"""
response.error_code = err_code
ret_code = True
if err_code == pb2.kPresentDataErrorUnsupportedFormat:
response.error_message = "Present data not support format."
logging.error("Present data not support format.")
ret_code = False
elif err_code == pb2.kPresentDataErrorNone:
response.error_message = "Present data ok"
ret_code = True
else:
response.error_message = "Present data not known error."
logging.error("Present data not known error.")
ret_code = False
self.send_message(conn, response, pb2._PRESENTIMAGERESPONSE.full_name)
return ret_code
def _process_image_request(self, conn, msg_data):
"""
Deserialization protobuf and process image_request
Args:
conn: a socket connection
msg_data: a protobuf struct, include image request.
Returns:
protobuf structure like this:
------------------------------------
|format | ImageFormat |
|------------------------------------
|width | uint32 |
|------------------------------------
|height | uint32 |
|------------------------------------
|data | bytes |
------------------------------------
enum ImageFormat {
kImageFormatJpeg = 0;
}
"""
request = pb2.PresentImageRequest()
response = pb2.PresentImageResponse()
# Parse msg_data from protobuf
try:
request.ParseFromString(msg_data)
except DecodeError:
logging.error("ParseFromString exception: Error parsing message")
err_code = pb2.kPresentDataErrorOther
return self._response_image_request(conn, response, err_code)
sock_fileno = conn.fileno()
handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno)
if handler is None:
logging.error("get channel handler failed")
err_code = pb2.kPresentDataErrorOther
return self._response_image_request(conn, response, err_code)
# Currently, image format only support jpeg
if request.format != pb2.kImageFormatJpeg:
logging.error("image format %s not support", request.format)
err_code = pb2.kPresentDataErrorUnsupportedFormat
return self._response_image_request(conn, response, err_code)
rectangle_list = []
if request.rectangle_list:
for one_rectangle in request.rectangle_list:
rectangle = []
rectangle.append(one_rectangle.left_top.x)
rectangle.append(one_rectangle.left_top.y)
rectangle.append(one_rectangle.right_bottom.x)
rectangle.append(one_rectangle.right_bottom.y)
rectangle.append(one_rectangle.label_text)
# add the detection result to list
rectangle_list.append(rectangle)
handler.save_image(request.data, request.width, request.height,
rectangle_list)
return self._response_image_request(conn, response,
pb2.kPresentDataErrorNone)
def stop_thread(self):
channel_manager = ChannelManager([])
channel_manager.close_all_thread()
self.set_exit_switch()
def stop_thread(self):
channel_manager = ChannelManager([])
channel_manager.close_all_thread()
self.set_exit_switch()
class TestFacialRecognitionHandler(unittest.TestCase):
"""TestFacialRecognitionHandler"""
channel_name = "facial_recognition"
media_type = "video"
channel_manager = ChannelManager()
handler = None
def func_end(self):
self.handler.close_thread_switch = True
channel_name = TestFacialRecognitionHandler.channel_name
TestFacialRecognitionHandler.channel_manager.unregister_one_channel(channel_name)
def func_begin(self):
channel_name = TestFacialRecognitionHandler.channel_name
media_type = TestFacialRecognitionHandler.media_type
TestFacialRecognitionHandler.channel_manager.register_one_channel(channel_name)
self.handler = FacialRecognitionHandler(channel_name, media_type)
@classmethod
def tearDownClass(cls):
pass
@classmethod
def setUpClass(cls):
pass
def run_thread(self, func):
thread = threading.Thread(target=func)
thread.start()
def set_img_data(self):
time.sleep(0.5)
self.handler.img_data = b'1234'
def set_img_data_none(self):
time.sleep(1.5)
self.handler.img_data = None
@patch('threading.Event.clear', return_value = True)
@patch('threading.Event.wait', return_value = True)
def test_save_frame1(self, mock1, mock2):
self.func_begin()
image = b'1234'
face_list = []
self.handler.img_data = b'12'
self.handler.save_frame(image, face_list)
self.func_end()
def test_save_frame2(self):
self.func_begin()
image = b'1234'
face_list = []
self.handler.img_data = b'12'
self.run_thread(self.set_img_data_none)
self.handler.save_frame(image, face_list)
self.func_end()
@patch('threading.Event.clear', return_value = True)
@patch('threading.Event.wait', return_value = True)
def test_frames(self, mock1, mock2):
self.func_begin()
self.handler.close_thread_switch = True
time.sleep(0.5) # wait thread exit
self.handler.img_data = None
backup_heartbeat = facial_recognition_handler.HEARTBEAT_TIMEOUT
facial_recognition_handler.HEARTBEAT_TIMEOUT = 0
self.handler.close_thread_switch = False
for frame in self.handler.frames():
self.assertEqual(frame, None)
break
facial_recognition_handler.HEARTBEAT_TIMEOUT = backup_heartbeat
self.func_end()
@patch('threading.Event.clear', return_value = True)
@patch('threading.Event.wait')
def test_get_frame(self, mock1, mock2):
self.func_begin()
self.handler.frame_data = b'123'
self.handler.face_list = []
self.handler.fps = 5
mock1.return_value = True
ret = self.handler.get_frame()
self.assertNotEqual(ret, {})
mock1.return_value = False
ret = self.handler.get_frame()
self.assertEqual(ret, {})
mock1.return_value = True
self.func_end()
class WebApp:
"""
web application
"""
__instance = None
def __init__(self):
"""
init method
"""
self.channel_mgr = ChannelManager(["image", "video"])
self.request_list = set()
self.lock = threading.Lock()
def __new__(cls, *args, **kwargs):
# if instance is None than create one
if cls.__instance is None:
cls.__instance = object.__new__(cls, *args, **kwargs)
return cls.__instance
def add_channel(self, channel_name):
"""
add channel
@param channel_name name of channel
@return: return add status and message (for error status)
"""
ret = {"ret": "error", "msg": ""}
# check channel_name validate,
# channel_name can not be None or length = 0
if channel_name is None:
logging.info("Channel name is None , add channel failed")
ret["msg"] = "Channel name can not be empty"
return ret
# strip channel name
channel_name = channel_name.strip()
# check channel_name emtpy or not
if channel_name == "":
logging.info("Channel name is emtpy , add channel failed")
ret["msg"] = "Channel name can not be empty"
return ret
# length of channel name can not over 25
if len(channel_name) > 25:
logging.info("Length of channel name %s > 25 , add channel failed",
channel_name)
ret["msg"] = "Length of channel name should less than 25"
return ret
# define pattern support a-z A-Z and /
pattern = re.compile(r"[a-z]|[A-Z]|[0-9]|/")
tmp = pattern.findall(channel_name)
# check reuslt changed or not
if len(tmp) != len(channel_name):
logging.info("%s contain invalidate character, add channel failed",
channel_name)
ret["msg"] = "Channel name only support 0-9, a-z, A-Z /"
return ret
# register channel
flag = self.channel_mgr.register_one_channel(channel_name)
# check register result
if self.channel_mgr.err_code_too_many_channel == flag:
logging.info("Only supports up to 10 channels, add channel failed")
ret["msg"] = "Only supports up to 10 channels"
elif self.channel_mgr.err_code_repeat_channel == flag:
logging.info("%s already exist, add channel failed", channel_name)
ret["msg"] = "Channel %s already exist" % channel_name
else:
logging.info("add channel %s succeed", channel_name)
ret["ret"] = "success"
return ret
def del_channel(self, names):
"""
delete channel
@param names: channel name to be deleted, separated by ','
@return: return add status and message (for error status)
"""
# init ret for return
ret = {"ret": "error", "msg": ""}
# check length of names
if names.strip() == "":
logging.info("Channel name is empty, delete channel failed")
ret["msg"] = "Channel name should not be empty"
return ret
# split name for multi name
listname = names.split(",")
# unregister name
for item in listname:
item = item.strip()
# if name is emtpy continu
if item == "":
continue
self.channel_mgr.unregister_one_channel(item)
logging.info("delete channel %s succeed", item)
ret["ret"] = "success"
return ret
def list_channels(self):
"""
list all channels information
"""
# list register channels
ret = self.channel_mgr.list_channels()
# id for every channel item , start with 1
idx = 1
# set id for channel
for item in ret:
item['id'] = idx
idx = idx + 1
return ret
def is_channel_exists(self, name):
"""
view channel content via browser.
@param name : channel name
@return return True if exists. otherwise return False.
"""
return self.channel_mgr.is_channel_exist(name)
def add_requst(self, request):
"""
add request
@param requst: request item to be stored
@note: request can not be same with other request.
request is identified by (channel name ,random number)
so this method do not return value.
"""
with self.lock:
self.request_list.add(request)
def has_request(self, request):
"""
whether request exist or not
@param request: request to be checked.
@return: return True if exists, otherwise return False.
"""
with self.lock:
for item in self.request_list:
# check request equal
if item[0] == request[0] and item[1] == request[1]:
return True
return False
def get_media_data(self, channel_name):
"""
get media data by channel name
@param channel_name: channel to be quest data.
@return return dictionary which have for item
type: identify channel type, for image or video.
image: data to be returned.
fps: just for video type
status: can be error, ok, or loading.
"""
# channel exists or not
if self.is_channel_exists(channel_name) is False:
return {'type': 'unkown', 'image': '', 'fps': 0, 'status': 'error'}
image_data = self.channel_mgr.get_channel_image(channel_name)
# only for image type.
if image_data is not None:
image_data = base64.b64encode(image_data).decode('utf-8')
return {
'type': 'image',
'image': image_data,
'fps': 0,
'status': 'ok'
}
fps = 0 # fps for video
image = None # image for video & image
rectangle_list = None
handler = self.channel_mgr.get_channel_handler_by_name(channel_name)
if handler is not None:
media_type = handler.get_media_type()
# if type is image then get image data
if media_type == "image":
image = handler.get_image_data()
# for video
else:
frame_info = handler.get_frame()
image = frame_info[0]
fps = frame_info[1]
rectangle_list = frame_info[4]
status = "loading"
# decode binary to utf-8 when image is not None
if image is not None:
status = "ok"
image = base64.b64encode(image).decode('utf-8')
return {
'type': media_type,
'image': image,
'fps': fps,
'status': status,
'rectangle_list': rectangle_list
}
else:
return {
'type': 'unkown',
'image': None,
'fps': 0,
'status': 'loading'
}
def __init__(self, server_address):
'''init func'''
self.channel_manager = ChannelManager(["image", "video"])
super(HeadPoseServer, self).__init__(server_address)
class ChannelHandler():
"""A set of channel handlers, process data received from channel"""
def __init__(self, channel_name, media_type):
self.channel_name = channel_name
self.media_type = media_type
self.img_data = None
self._frame = None
self.thread = None
self._frame = None
# last time the channel receive data.
self.heartbeat = time.time()
self.web_event = ThreadEvent(timeout=WEB_EVENT_TIMEOUT)
self.image_event = ThreadEvent(timeout=IMAGE_EVENT_TIMEOUT)
self.lock = threading.Lock()
self.channel_manager = ChannelManager([])
self.rectangle_list = None
if media_type == "video":
self.thread_name = "videothread-{}".format(self.channel_name)
self.heartbeat = time.time()
self.close_thread_switch = False
self.fps = 0
self.image_number = 0
self.time_list = []
self._create_thread()
def close_thread(self):
"""close thread if object has created"""
if self.thread is None:
return
self.set_thread_switch()
self.image_event.set()
logging.info("%s set _close_thread_switch True", self.thread_name)
def set_heartbeat(self):
"""record heartbeat"""
self.heartbeat = time.time()
def set_thread_switch(self):
"""record heartbeat"""
self.close_thread_switch = True
def save_image(self, data, width, height, rectangle_list):
"""save image receive from socket"""
self.width = width
self.height = height
self.rectangle_list = rectangle_list
# compute fps if type is video
if self.media_type == "video":
while self.img_data:
time.sleep(0.01)
self.time_list.append(self.heartbeat)
self.image_number += 1
while self.time_list[0] + 1 < time.time():
self.time_list.pop(0)
self.image_number -= 1
if self.image_number == 0:
break
self.fps = len(self.time_list)
self.img_data = data
self.image_event.set()
else:
self.img_data = data
self.channel_manager.save_channel_image(self.channel_name,
self.img_data,
self.rectangle_list)
self.heartbeat = time.time()
def get_media_type(self):
"""get media_type, support image or video"""
return self.media_type
def get_image(self):
"""get image_data"""
return self.img_data
def _create_thread(self):
"""Start the background video thread if it isn't running yet."""
if self.thread is not None and self.thread.isAlive():
return
# start background frame thread
self.thread = threading.Thread(target=self._video_thread)
self.thread.start()
def get_frame(self):
"""Return the current video frame."""
# wait util receive a frame data, and push it to your browser.
ret = self.web_event.wait()
self.web_event.clear()
# True: _web_event return because set()
# False: _web_event return because timeout
if ret:
return (self._frame, self.fps, self.width, self.height,
self.rectangle_list)
return (None, None, None, None, None)
def frames(self):
"""a generator generates image"""
while True:
self.image_event.wait()
self.image_event.clear()
if self.img_data:
yield self.img_data
self.img_data = None
# if set _close_thread_switch, return immediately
if self.close_thread_switch:
yield None
# if no frames or heartbeat coming in the last 100 seconds,
# stop the thread and close socket
if time.time() - self.heartbeat > HEARTBEAT_TIMEOUT:
self.set_thread_switch()
self.img_data = None
yield None
def _video_thread(self):
"""background thread to process video"""
logging.info('create %s...', (self.thread_name))
for frame in self.frames():
if frame:
# send signal to clients
self._frame = frame
self.web_event.set()
# exit thread
if self.close_thread_switch:
self.channel_manager.clean_channel_resource_by_name(
self.channel_name)
logging.info('Stop thread:%s.', (self.thread_name))
break
class WebApp:
"""
web application
"""
__instance = None
def __init__(self):
"""
init method
"""
self.channel_mgr = ChannelManager(["image", "video"])
self.request_list = set()
self.lock = threading.Lock()
self.display_manager = display_server.DisplayServerManager()
self.thread = None
self._create_sending_thread()
def __new__(cls, *args, **kwargs):
# if instance is None than create one
if cls.__instance is None:
cls.__instance = object.__new__(cls, *args, **kwargs)
return cls.__instance
def add_channel(self, channel_name):
"""
add channel
@param channel_name name of channel
@return: return add status and message (for error status)
"""
ret = {"ret": "error", "msg": ""}
# check channel_name validate,
# channel_name can not be None or length = 0
if channel_name is None:
logging.info("Channel name is None , add channel failed")
ret["msg"] = "Channel name can not be empty"
return ret
# strip channel name
channel_name = channel_name.strip()
# check channel_name emtpy or not
if channel_name == "":
logging.info("Channel name is emtpy , add channel failed")
ret["msg"] = "Channel name can not be empty"
return ret
# length of channel name can not over 25
if len(channel_name) > 25:
logging.info("Length of channel name %s > 25 , add channel failed",
channel_name)
ret["msg"] = "Length of channel name should less than 25"
return ret
# define pattern support a-z A-Z and /
pattern = re.compile(r"[a-z]|[A-Z]|[0-9]|/")
tmp = pattern.findall(channel_name)
# check reuslt changed or not
if len(tmp) != len(channel_name):
logging.info("%s contain invalidate character, add channel failed",
channel_name)
ret["msg"] = "Channel name only support 0-9, a-z, A-Z /"
return ret
# register channel
flag = self.channel_mgr.register_one_channel(channel_name)
# check register result
if self.channel_mgr.err_code_too_many_channel == flag:
logging.info("Only supports up to 10 channels, add channel failed")
ret["msg"] = "Only supports up to 10 channels"
elif self.channel_mgr.err_code_repeat_channel == flag:
logging.info("%s already exist, add channel failed", channel_name)
ret["msg"] = "Channel %s already exist" % channel_name
else:
logging.info("add channel %s succeed", channel_name)
ret["ret"] = "success"
return ret
def del_channel(self, names):
"""
delete channel
@param names: channel name to be deleted, separated by ','
@return: return add status and message (for error status)
"""
# init ret for return
ret = {"ret": "error", "msg": ""}
# check length of names
if names.strip() == "":
logging.info("Channel name is empty, delete channel failed")
ret["msg"] = "Channel name should not be empty"
return ret
# split name for multi name
listname = names.split(",")
# unregister name
for item in listname:
item = item.strip()
# if name is emtpy continu
if item == "":
continue
self.channel_mgr.unregister_one_channel(item)
logging.info("delete channel %s succeed", item)
ret["ret"] = "success"
return ret
def list_channels(self):
"""
list all channels information
"""
# list register channels
ret = self.channel_mgr.list_channels()
# id for every channel item , start with 1
idx = 1
# set id for channel
for item in ret:
item['id'] = idx
idx = idx + 1
return ret
def is_channel_exists(self, name):
"""
view channel content via browser.
@param name : channel name
@return return True if exists. otherwise return False.
"""
return self.channel_mgr.is_channel_exist(name)
def add_requst(self, request):
"""
add request
@param requst: request item to be stored
@note: request can not be same with other request.
request is identified by (channel name ,random number)
so this method do not return value.
"""
with self.lock:
self.request_list.add(request)
def has_request(self, request):
"""
whether request exist or not
@param request: request to be checked.
@return: return True if exists, otherwise return False.
"""
with self.lock:
for item in self.request_list:
# check request equal
if item[0] == request[0] and item[1] == request[1]:
return True
return False
def get_media_data(self, channel_name):
"""
get media data by channel name
@param channel_name: channel to be quest data.
@return return dictionary which have for item
type: identify channel type, for image or video.
image: data to be returned.
fps: just for video type
status: can be error, ok, or loading.
"""
# channel exists or not
if self.is_channel_exists(channel_name) is False:
return {'type': 'unkown', 'image': '', 'fps': 0, 'status': 'error'}
image_data = self.channel_mgr.get_channel_image(channel_name)
# only for image type.
if image_data is not None:
image_data = base64.b64encode(image_data).decode('utf-8')
return {
'type': 'image',
'image': image_data,
'fps': 0,
'status': 'ok'
}
fps = 0 # fps for video
image = None # image for video & image
rectangle_list = None
view = None
handler = self.channel_mgr.get_channel_handler_by_name(channel_name)
if handler is not None:
media_type = handler.get_media_type()
# if type is image then get image data
if media_type == "image":
image = handler.get_image_data()
# for video
else:
frame_info = handler.get_frame()
image = frame_info[0]
fps = frame_info[1]
rectangle_list = frame_info[4]
width, height = frame_info[2], frame_info[2]
if width == 256:
view = 'ColorMap'
else:
view = 'ResultImage'
status = "loading"
# decode binary to utf-8 when image is not None
if image is not None:
status = "ok"
image = base64.b64encode(image).decode('utf-8')
return {
'view': view,
'type': media_type,
'image': image,
'fps': fps,
'status': status,
'rectangle_list': rectangle_list
}
else:
return {
'view': view,
'type': 'unkown',
'image': None,
'fps': 0,
'status': 'loading'
}
def _create_sending_thread(self):
"""Start the sending thread if it isn't running yet."""
if self.thread is not None and self.thread.isAlive():
return
# start style type thread
self.thread = threading.Thread(target=self.sending_thread_func,
args=(self.display_manager, ))
self.thread.start()
def sending_thread_func(self, app_manager):
global g_object_bytes, g_layout_bytes
while True:
event.wait()
flag = app_manager.send_model_input_data(g_object_bytes,
g_layout_bytes)
print("data package sending msg:", flag[1])
if flag[0] is True:
logging.info("Send model input data package success")
else:
logging.info("Send model input data package failed")
event.clear()
def update_model_input_data(self, objects_bytes, layout_bytes):
''' send model data to agent'''
### check if the input changed same input
### type change, trigger the wait thread to register
global g_object_bytes, g_layout_bytes
g_object_bytes = b''
g_layout_bytes = b''
g_object_bytes = objects_bytes
g_layout_bytes = layout_bytes
if g_layout_bytes == b'' or g_object_bytes == b'':
return ""
# Here, you may not use thread event method to trigger the transfer manager
# to register style type directly, it maybe ok. But here i use thread event instead.
print("send model input data thread event set")
event.set()
class PresenterSocketServer():
"""a socket server communication with presenter agent.
"""
def __init__(self, server_address):
"""
Args:
server_address: server listen address,
include an ipv4 address and a port.
"""
# thread exit switch, if set true, thread must exit immediately.
self._thread_exit_switch = False
self._config = config_parser.ConfigParser()
self._channel_manager = ChannelManager()
self._create_socket_server(server_address)
def _create_socket_server(self, server_address):
"""
create a socket server
Args:
server_address: server listen address,
include an ipv4 address and a port.
"""
# Create a socket server.
self._sock_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._sock_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._sock_server.bind(server_address)
self._sock_server.listen(SOCKET_WAIT_QUEUE)
self._sock_server.setblocking(False)
# Get server host name and port
host, port = self._sock_server.getsockname()[:2]
# Start presenter socket server thread.
threading.Thread(target=self._server_listen_thread).start()
# Display directly on the screen
print('Presenter socket server listen on %s:%s\n' % (host, port))
def stop_thread(self):
"""set switch True to stop presenter socket server thread."""
self._thread_exit_switch = True
def _read_socket(self, conn, read_len):
'''
Read fixed length data
Args:
conn: a socket connection
read_len: read fix byte.
Returns:
ret: PRESENTER_OK or PRESENTER_ERR
buf: read fix byte buf.
'''
has_read_len = 0
read_buf = SOCK_RECV_NULL
total_buf = SOCK_RECV_NULL
recv_num = 0
while has_read_len != read_len:
#Troubleshoot CPU problems when Network blocking
if recv_num % 10 == 5:
time.sleep(0.01)
try:
read_buf = conn.recv(read_len - has_read_len)
except socket.error:
logging.error("socket %u exception:socket.error",
conn.fileno())
return PRESENTER_ERR, None
if read_buf == SOCK_RECV_NULL:
return PRESENTER_ERR, None
total_buf += read_buf
has_read_len = len(total_buf)
recv_num += 1
return PRESENTER_OK, total_buf
def _read_msg_head(self, sock_fileno, conns):
'''
Args:
sock_fileno: a socket fileno
conns: all socket connections which created by server.
Returns:
msg_total_len: total message length.
msg_name_len: message name length.
'''
ret, msg_head = self._read_socket(conns[sock_fileno], MSG_HEAD_LENGTH)
if ret == PRESENTER_ERR:
logging.error("socket %u receive msg head null", sock_fileno)
return None, None
# in Struct(), 'I' is unsigned int, 'B' is unsigned char
msg_head_data = struct.Struct('IB')
(msg_total_len, msg_name_len) = msg_head_data.unpack(msg_head)
msg_total_len = socket.ntohl(msg_total_len)
return msg_total_len, msg_name_len
def _read_msg_name(self, sock_fd, conns, msg_name_len):
'''
Args:
sock_fd: a socket fileno
conns: all socket connections which created by server.
msg_name_len: message name length.
Returns:
ret: PRESENTER_OK or PRESENTER_ERR
msg_name: message name.
'''
ret, msg_name = self._read_socket(conns[sock_fd], msg_name_len)
if ret == PRESENTER_ERR:
logging.error("socket %u receive msg name null", sock_fd)
return PRESENTER_ERR, None
try:
msg_name = msg_name.decode("utf-8")
except UnicodeDecodeError:
logging.error("msg name decode to utf-8 error")
return PRESENTER_ERR, None
return PRESENTER_OK, msg_name
def _read_msg_body(self, sock_fd, conns, msg_body_len, msgs):
'''
Args:
sock_fd: a socket fileno
conns: all socket connections which created by server.
msg_name_len: message name length.
msgs: msg read from a socket
Returns:
ret: PRESENTER_OK or PRESENTER_ERR
'''
ret, msg_body = self._read_socket(conns[sock_fd], msg_body_len)
if ret == PRESENTER_ERR:
logging.error("socket %u receive msg body null", sock_fd)
return PRESENTER_ERR
msgs[sock_fd] = msg_body
return PRESENTER_OK
def _read_sock_and_process_msg(self, sock_fileno, conns, msgs):
'''
Args:
sock_fileno: a socket fileno, return value of socket.fileno()
conns: all socket connections registered in epoll
msgs: msg read from a socket
Returns:
ret: PRESENTER_OK or PRESENTER_ERR
'''
# Step1: read msg head
msg_total_len, msg_name_len = self._read_msg_head(sock_fileno, conns)
if msg_total_len is None:
return PRESENTER_ERR
# Step2: read msg name
ret, msg_name = self._read_msg_name(sock_fileno, conns, msg_name_len)
if ret == PRESENTER_ERR:
return ret
# Step3: read msg body
msg_body_len = msg_total_len - MSG_HEAD_LENGTH - msg_name_len
if msg_body_len < 0:
logging.error("msg_total_len:%u, msg_name_len:%u, msg_body_len:%u",
msg_total_len, msg_name_len, msg_body_len)
return PRESENTER_ERR
ret = self._read_msg_body(sock_fileno, conns, msg_body_len, msgs)
if ret == PRESENTER_ERR:
return ret
# Step4: process msg
ret = self._process_msg(conns[sock_fileno], msg_name,
msgs[sock_fileno])
return ret
def _process_epollin(self, sock_fileno, epoll, conns, msgs):
'''
Args:
sock_fileno: a socket fileno, return value of socket.fileno()
epoll: a set of select.epoll.
conns: all socket connections registered in epoll
msgs: msg read from a socket
'''
msgs[sock_fileno] = b''
try:
ret = self._read_sock_and_process_msg(sock_fileno, conns, msgs)
if not ret:
self._clean_connect(sock_fileno, epoll, conns, msgs)
except socket.error:
logging.error("receive socket error.")
self._clean_connect(sock_fileno, epoll, conns, msgs)
def _accept_new_socket(self, epoll, conns):
'''
Args:
epoll: a set of select.epoll.
conns: all socket connections registered in epoll
'''
try:
new_conn, address = self._sock_server.accept()
new_conn.setblocking(True)
epoll.register(new_conn.fileno(), select.EPOLLIN | select.EPOLLHUP)
conns[new_conn.fileno()] = new_conn
logging.info("create new connection:client-ip:%s, client-port:%s",
address[0], address[1])
except socket.error:
logging.error("socket.error exception when sock.accept()")
def _server_listen_thread(self):
"""socket server thread, epoll listening all the socket events"""
epoll = select.epoll()
epoll.register(self._sock_server.fileno(),
select.EPOLLIN | select.EPOLLHUP)
try:
conns = {}
msgs = {}
while True:
# thread must exit immediately
if self._thread_exit_switch:
break
events = epoll.poll(EPOLL_TIMEOUT)
# timeout, but no event come, continue waiting
if not events:
continue
for sock_fileno, event in events:
# new connection request from presenter agent
if self._sock_server.fileno() == sock_fileno:
self._accept_new_socket(epoll, conns)
# remote connection closed
# it means presenter agent exit withot close socket.
elif event & select.EPOLLHUP:
logging.info("receive event EPOLLHUP")
self._clean_connect(sock_fileno, epoll, conns, msgs)
# new data coming in a socket connection
elif event & select.EPOLLIN:
self._process_epollin(sock_fileno, epoll, conns, msgs)
# receive event not recognize
else:
logging.error("not recognize event %f", event)
self._clean_connect(sock_fileno, epoll, conns, msgs)
finally:
logging.info("conns:%s", conns)
logging.info("presenter agent listen thread exit.")
epoll.unregister(self._sock_server.fileno())
epoll.close()
self._sock_server.close()
def _clean_connect(self, sock_fileno, epoll, conns, msgs):
"""
close socket, and clean local variables
Args:
sock_fileno: a socket fileno, return value of socket.fileno()
epoll: a set of select.epoll.
conns: all socket connections registered in epoll
msgs: msg read from a socket
"""
logging.info("clean sock_fd:%s, conns:%s", sock_fileno, conns)
self._channel_manager.clean_channel_resource_by_fd(sock_fileno)
epoll.unregister(sock_fileno)
conns[sock_fileno].close()
del conns[sock_fileno]
del msgs[sock_fileno]
def _process_msg(self, conn, msg_name, msg_data):
"""
Total entrance to process protobuf msg
Args:
conn: a socket connection
msg_name: name of a msg.
msg_data: msg body, serialized by protobuf
Returns:
False:somme error occured
True:succeed
"""
# process open channel request
if msg_name == OPEN_CHANNEL_REQUEST_FULL_NAME:
ret = self._process_open_channel(conn, msg_data)
# process image request, receive an image data from presenter agent
elif msg_name == PRESENT_IMAGE_REQUEST_FULL_NAME:
ret = self._process_image_request(conn, msg_data)
# process heartbeat request, it used to keepalive a channel path
elif msg_name == HEART_BEAT_MESSAGE_FULL_NAME:
ret = self._process_heartbeat(conn)
else:
logging.error("Not recognized msg type %s", msg_name)
ret = PRESENTER_ERR
return ret
def _response_open_channel(self, conn, channel_name, response, err_code):
"""
Assemble protobuf to response open_channel request
Args:
conn: a socket connection
channel_name: name of a channel.
response: a protobuf response to presenter agent
err_code: part of the response
Returns:
ret_code:PRESENTER_OK or PRESENTER_ERR
Message structure like this:
--------------------------------------------------------------------
|total message len | int | 4 bytes |
|-------------------------------------------------------------------
|message name len | byte | 1 byte |
|-------------------------------------------------------------------
|message name | string | xx bytes |
|-------------------------------------------------------------------
|message body | protobuf | xx bytes |
--------------------------------------------------------------------
protobuf structure like this:
--------------------------------------------------------------------
|error_code | enum | OpenChannelErrorCode |
|-------------------------------------------------------------------
|error_message | string | xx bytes |
|-------------------------------------------------------------------
enum OpenChannelErrorCode {
kOpenChannelErrorNone = 0;
kOpenChannelErrorNoSuchChannel = 1;
kOpenChannelErrorChannelAlreadyOpened = 2;
kOpenChannelErrorOther = -1;
}
"""
response.error_code = err_code
ret_code = PRESENTER_ERR
if err_code == OPEN_CHANNEL_NO_CHANNEL_ERR:
response.error_message = "channel {} not exist." \
.format(channel_name)
elif err_code == OPEN_CHANNEL_BUSY_ERR:
response.error_message = "channel {} is busy.".format(channel_name)
elif err_code == OPEN_CHANNEL_OK:
response.error_message = "open channel succeed"
ret_code = PRESENTER_OK
else:
response.error_message = "Unknown err open channel {}." \
.format(channel_name)
self._send_response(conn, response, OPEN_CHANNEL_RESPONSE_FULL_NAME)
return ret_code
def _process_open_channel(self, conn, msg_data):
"""
Deserialization protobuf and process open_channel request
Args:
conn: a socket connection
msg_data: a protobuf struct, include open channel request.
Returns:
protobuf structure like this:
----------------------------------------------
|channel_name | string |
|----------------------------------------------
|content_type | ChannelContentType |
|----------------------------------------------
enum ChannelContentType {
kChannelContentTypeImage = 0;
kChannelContentTypeVideo = 1;
}
"""
request = pb.OpenChannelRequest()
response = pb.OpenChannelResponse()
try:
request.ParseFromString(msg_data)
except message.DecodeError:
logging.error("ParseFromString exception: Error parsing message")
channel_name = "unknown channel"
return self._response_open_channel(conn, channel_name, response,
OPEN_CHANNEL_UNKOWN_ERR)
channel_name = request.channel_name
# check channel name if exist
if not self._channel_manager.is_channel_exist(channel_name):
logging.error("channel name %s is not exist.", channel_name)
return self._response_open_channel(conn, channel_name, response,
OPEN_CHANNEL_NO_CHANNEL_ERR)
# check channel path if busy
if self._channel_manager.is_channel_busy(channel_name):
logging.error("channel path %s is busy.", channel_name)
return self._response_open_channel(conn, channel_name, response,
OPEN_CHANNEL_BUSY_ERR)
# if channel type is image, need clean image if exist
self._channel_manager.clean_channel_image(channel_name)
if request.content_type == IMAGE_TYPE:
media_type = "image"
elif request.content_type == VIDEO_TYPE:
media_type = "video"
else:
logging.error("media type %s is not recognized.",
request.content_type)
return self._response_open_channel(conn, channel_name, response,
OPEN_CHANNEL_UNKOWN_ERR)
handler = ChannelHandler(channel_name, media_type)
self._channel_manager.create_channel_resource(channel_name,
conn.fileno(),
media_type, handler)
return self._response_open_channel(conn, channel_name, response,
OPEN_CHANNEL_OK)
def _send_response(self, conn, response, msg_name):
response_data = response.SerializeToString()
response_len = len(response_data)
msg_name_size = len(msg_name)
msg_total_size = MSG_HEAD_LENGTH + msg_name_size + response_len
# in Struct(), 'I' is unsigned int, 'B' is unsigned char
struct_head = struct.Struct('IB')
msg_head = (socket.htonl(msg_total_size), msg_name_size)
packed_msg_head = struct_head.pack(*msg_head)
msg_data = packed_msg_head + \
bytes(msg_name, encoding="utf-8") + response_data
conn.sendall(msg_data)
def _response_image_request(self, conn, response, err_code):
"""
Assemble protobuf to response image_request
Message structure like this:
--------------------------------------------------------------------
|total message len | int | 4 bytes |
|-------------------------------------------------------------------
|message name len | byte | 1 byte |
|-------------------------------------------------------------------
|message name | string | xx bytes |
|-------------------------------------------------------------------
|message body | protobuf | xx bytes |
--------------------------------------------------------------------
protobuf structure like this:
--------------------------------------------------------------------
|error_code | enum | PresentDataErrorCode |
|-------------------------------------------------------------------
|error_message | string | xx bytes |
|-------------------------------------------------------------------
enum PresentDataErrorCode {
kPresentDataErrorNone = 0;
kPresentDataErrorUnsupportedType = 1;
kPresentDataErrorUnsupportedFormat = 2;
kPresentDataErrorOther = -1;
}
"""
response.error_code = err_code
ret_code = PRESENTER_OK
if err_code == PRESENT_DATA_UNSUPPORTED_FORMAT_ERR:
response.error_message = "Present data not support format."
logging.error("Present data not support format.")
ret_code = PRESENTER_ERR
elif err_code == PRESENT_DATA_OK:
response.error_message = "Present data ok"
ret_code = PRESENTER_OK
else:
response.error_message = "Present data not known error."
logging.error("Present data not known error.")
ret_code = PRESENTER_ERR
self._send_response(conn, response, PRESENT_IMAGE_RESPONSE_FULL_NAME)
return ret_code
def _process_image_request(self, conn, msg_data):
"""
Deserialization protobuf and process image_request
Args:
conn: a socket connection
msg_data: a protobuf struct, include image request.
Returns:
protobuf structure like this:
------------------------------------
|format | ImageFormat |
|------------------------------------
|width | uint32 |
|------------------------------------
|height | uint32 |
|------------------------------------
|data | bytes |
------------------------------------
enum ImageFormat {
kImageFormatJpeg = 0;
}
"""
request = pb.PresentImageRequest()
response = pb.PresentImageResponse()
# Parse msg_data from protobuf
try:
request.ParseFromString(msg_data)
except message.DecodeError:
logging.error("ParseFromString exception: Error parsing message")
err_code = PRESENT_DATA_UNKOWN_ERR
return self._response_image_request(conn, response, err_code)
sock_fileno = conn.fileno()
handler = self._channel_manager.get_channel_handler_by_fd(sock_fileno)
if handler is None:
logging.error("get channel handler failed")
err_code = PRESENT_DATA_UNKOWN_ERR
return self._response_image_request(conn, response, err_code)
# Currently, image format only support jpeg
if request.format != FORMAT_JPEG:
logging.error("image format %s not support", request.format)
err_code = PRESENT_DATA_UNSUPPORTED_FORMAT_ERR
return self._response_image_request(conn, response, err_code)
handler.save_image(request.data, request.width, request.height)
return self._response_image_request(conn, response, PRESENT_DATA_OK)
def _process_heartbeat(self, conn):
'''
set heartbeat
Args:
conn: a socket connection
Returns:
True: set heartbeat ok.
'''
sock_fileno = conn.fileno()
handler = self._channel_manager.get_channel_handler_by_fd(sock_fileno)
if handler is not None:
handler.setheartbeat()
return PRESENTER_OK
class FaceDetectionServer(PresenterSocketServer):
'''A server for face detection'''
def __init__(self, server_address):
'''init func'''
self.channel_manager = ChannelManager(["image", "video"])
super(FaceDetectionServer, self).__init__(server_address)
def _clean_connect(self, sock_fileno, epoll, conns, msgs):
"""
close socket, and clean local variables
Args:
sock_fileno: a socket fileno, return value of socket.fileno()
epoll: a set of select.epoll.
conns: all socket connections registered in epoll
msgs: msg read from a socket
"""
logging.info("clean fd:%s, conns:%s", sock_fileno, conns)
self.channel_manager.clean_channel_resource_by_fd(sock_fileno)
epoll.unregister(sock_fileno)
conns[sock_fileno].close()
del conns[sock_fileno]
del msgs[sock_fileno]
def _process_msg(self, conn, msg_name, msg_data):
"""
Total entrance to process protobuf msg
Args:
conn: a socket connection
msg_name: name of a msg.
msg_data: msg body, serialized by protobuf
Returns:
False:somme error occured
True:succeed
"""
# process open channel request
if msg_name == pb2._OPENCHANNELREQUEST.full_name:
ret = self._process_open_channel(conn, msg_data)
# process image request, receive an image data from presenter agent
elif msg_name == pb2._PRESENTIMAGEREQUEST.full_name:
ret = self._process_image_request(conn, msg_data)
# process heartbeat request, it used to keepalive a channel path
elif msg_name == pb2._HEARTBEATMESSAGE.full_name:
ret = self._process_heartbeat(conn)
else:
logging.error("Not recognized msg type %s", msg_name)
ret = False
return ret
def _response_image_request(self, conn, response, err_code):
"""
Assemble protobuf to response image_request
Message structure like this:
--------------------------------------------------------------------
|total message len | int | 4 bytes |
|-------------------------------------------------------------------
|message name len | byte | 1 byte |
|-------------------------------------------------------------------
|message name | string | xx bytes |
|-------------------------------------------------------------------
|message body | protobuf | xx bytes |
--------------------------------------------------------------------
protobuf structure like this:
--------------------------------------------------------------------
|error_code | enum | PresentDataErrorCode |
|-------------------------------------------------------------------
|error_message | string | xx bytes |
|-------------------------------------------------------------------
enum PresentDataErrorCode {
kPresentDataErrorNone = 0;
kPresentDataErrorUnsupportedType = 1;
kPresentDataErrorUnsupportedFormat = 2;
kPresentDataErrorOther = -1;
}
"""
response.error_code = err_code
ret_code = True
if err_code == pb2.kPresentDataErrorUnsupportedFormat:
response.error_message = "Present data not support format."
logging.error("Present data not support format.")
ret_code = False
elif err_code == pb2.kPresentDataErrorNone:
response.error_message = "Present data ok"
ret_code = True
else:
response.error_message = "Present data not known error."
logging.error("Present data not known error.")
ret_code = False
self.send_message(conn, response, pb2._PRESENTIMAGERESPONSE.full_name)
return ret_code
def _process_image_request(self, conn, msg_data):
"""
Deserialization protobuf and process image_request
Args:
conn: a socket connection
msg_data: a protobuf struct, include image request.
Returns:
protobuf structure like this:
------------------------------------
|format | ImageFormat |
|------------------------------------
|width | uint32 |
|------------------------------------
|height | uint32 |
|------------------------------------
|data | bytes |
------------------------------------
enum ImageFormat {
kImageFormatJpeg = 0;
}
"""
request = pb2.PresentImageRequest()
response = pb2.PresentImageResponse()
# Parse msg_data from protobuf
try:
request.ParseFromString(msg_data)
except DecodeError:
logging.error("ParseFromString exception: Error parsing message")
err_code = pb2.kPresentDataErrorOther
return self._response_image_request(conn, response, err_code)
sock_fileno = conn.fileno()
handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno)
if handler is None:
logging.error("get channel handler failed")
err_code = pb2.kPresentDataErrorOther
return self._response_image_request(conn, response, err_code)
# Currently, image format only support jpeg
if request.format != pb2.kImageFormatJpeg:
logging.error("image format %s not support", request.format)
err_code = pb2.kPresentDataErrorUnsupportedFormat
return self._response_image_request(conn, response, err_code)
rectangle_list = []
if request.rectangle_list:
for one_rectangle in request.rectangle_list:
rectangle = []
rectangle.append(one_rectangle.left_top.x)
rectangle.append(one_rectangle.left_top.y)
rectangle.append(one_rectangle.right_bottom.x)
rectangle.append(one_rectangle.right_bottom.y)
rectangle.append(one_rectangle.label_text)
# add the detection result to list
rectangle_list.append(rectangle)
imageData = request.data[:1382400]
maskData = request.data[-100352:]
# convert yuv420sp(nv12) to jpg
yuvNum = len(
[lists for lists in os.listdir('yuv') if lists.endswith(".yuv")])
with open(os.path.join('yuv', str(yuvNum) + '.yuv'), 'wb') as f:
f.write(imageData)
ff = ffmpy.FFmpeg(
inputs={
os.path.join('yuv',
str(yuvNum) + '.yuv'):
'-s 1280*720 -pix_fmt nv12'
},
outputs={os.path.join('yuv',
str(yuvNum) + '.jpg'): None})
ff.run()
# read data
imgNow = cv2.imread(os.path.join('yuv', str(yuvNum) + '.jpg'))
background = cv2.imread("background.jpg")
maskarr = np.fromstring(maskData, np.uint8).astype(np.float32)
maskarr = np.reshape(maskarr, (224, 224, 2))
alphargb = cv2.resize(maskarr[:, :, 1],
(request.width, request.height))
# output the mask to compare result with the output of pb model
cv2.imwrite(os.path.join('yuv', str(yuvNum) + 'alpha.jpg'), alphargb)
alphargb = alphargb / 255
alphargb = np.repeat(alphargb[..., np.newaxis], 3, 2)
result = np.uint8(imgNow * alphargb + background * (1 - alphargb))
img_decode = cv2.imencode('.jpg', result)[1].tostring()
handler.save_image(img_decode, request.width, request.height,
rectangle_list)
return self._response_image_request(conn, response,
pb2.kPresentDataErrorNone)
def stop_thread(self):
channel_manager = ChannelManager([])
channel_manager.close_all_thread()
self.set_exit_switch()
def __init__(self, server_address):
'''init func'''
self.channel_manager = ChannelManager(["image", "video"])
self._image_cnt = 0
super(DisplayServer, self).__init__(server_address)
class DisplayServer(PresenterSocketServer):
def __init__(self, server_address):
'''init func'''
self.channel_manager = ChannelManager(["image", "video"])
super(DisplayServer, self).__init__(server_address)
self.app_manager = AppManager()
self.register_dict = {}
def _clean_connect(self, sock_fileno, epoll, conns, msgs):
"""
close socket, and clean local variables
Args:
sock_fileno: a socket fileno, return value of socket.fileno()
epoll: a set of select.epoll.
conns: all socket connections registered in epoll
msgs: msg read from a socket
"""
logging.info("clean fd:%s, conns:%s", sock_fileno, conns)
self.channel_manager.clean_channel_resource_by_fd(sock_fileno)
epoll.unregister(sock_fileno)
conns[sock_fileno].close()
del conns[sock_fileno]
del msgs[sock_fileno]
def _process_msg(self, conn, msg_name, msg_data):
"""
Total entrance to process protobuf msg
Args:
conn: a socket connection
msg_name: name of a msg.
msg_data: msg body, serialized by protobuf
Returns:
False:somme error occured
True:succeed
"""
## process_register_app
if msg_name == painting_message_pb2._REGISTERAPP.full_name:
ret = self._process_register_app(conn, msg_data)
# process open channel request
elif msg_name == pb2._OPENCHANNELREQUEST.full_name:
ret = self._process_open_channel(conn, msg_data)
# process image request, receive an image data from presenter agent
elif msg_name == pb2._PRESENTIMAGEREQUEST.full_name:
ret = self._process_image_request(conn, msg_data)
# process heartbeat request, it used to keepalive a channel path
elif msg_name == pb2._HEARTBEATMESSAGE.full_name:
ret = self._process_heartbeat(conn)
else:
logging.error("Not recognized msg type %s", msg_name)
ret = False
return ret
# 重载_process_heartbeat
def _process_heartbeat(self, conn):
'''
set heartbeat
Input:
conn: a socket connection
Returns:
True: set heartbeat ok.
'''
sock_fileno = conn.fileno()
if self.app_manager.get_app_id_by_socket(sock_fileno):
self.app_manager.set_heartbeat(sock_fileno)
handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno)
if handler is not None:
handler.set_heartbeat()
return True
def _parse_protobuf(self, protobuf, msg_data):
"""
Description: parse protobuf
Input:
protobuf: a struct defined by protobuf
msg_data: msg body, serialized by protobuf
Returns: True or False
"""
try:
protobuf.ParseFromString(msg_data)
return True
except DecodeError as exp:
logging.error(exp)
return False
def _process_register_app(self, conn, msg_data):
"""
Description: process register_app message
Input:
conn: a socket connection
msg_data: msg body, serialized by protobuf
Returns: True or False
"""
request = painting_message_pb2.RegisterApp()
response = painting_message_pb2.CommonResponse()
msg_name = painting_message_pb2._COMMONRESPONSE.full_name
if not self._parse_protobuf(request, msg_data):
response.ret = painting_message_pb2.kErrorOther
response.message = "ParseFromString exception"
self.send_message(conn, response, msg_name)
return False
app_id = request.id
app_type = request.type
# check app id if exist
if self.app_manager.is_app_exist(app_id):
logging.error("App %s is already exist.", app_id)
response.ret = painting_message_pb2.kErrorAppRegisterExist
response.message = "App {} is already exist.".format(app_id)
self.send_message(conn, response, msg_name)
elif self.app_manager.get_app_num() >= MAX_APP_NUM:
logging.error("App number reach the upper limit")
response.ret = painting_message_pb2.kErrorAppRegisterLimit
response.message = "App number reach the upper limit"
self.send_message(conn, response, msg_name)
elif app_type != SERVER_TYPE:
logging.error("App type %s error", app_type)
response.ret = painting_message_pb2.kErrorAppRegisterType
response.message = "App type {} error".format(app_type)
self.send_message(conn, response, msg_name)
elif len(app_id) > APP_ID_MAX_LENGTH:
logging.error("App id %s is too long", app_id)
response.ret = painting_message_pb2.kErrorOther
response.message = "App id: {} is too long".format(app_id)
self.send_message(conn, response, msg_name)
else:
self.app_manager.register_app(app_id, conn)
response.ret = painting_message_pb2.kErrorNone
response.message = "Register app {} succeed".format(app_id)
self.send_message(conn, response, msg_name)
return True
return False
def get_app_socket(self, app_id):
"""
Description: get a socket which is bound to the app.
Input:
app_id: id of the app
Returns: socket
"""
return self.app_manager.get_socket_by_app_id(app_id)
def list_registered_apps(self):
"""
Description: get registered apps list.
Input: NA
Returns: app list
"""
return self.app_manager.list_app()
def _response_image_request(self, conn, response, err_code):
"""
Assemble protobuf to response image_request
Message structure like this:
--------------------------------------------------------------------
|total message len | int | 4 bytes |
|-------------------------------------------------------------------
|message name len | byte | 1 byte |
|-------------------------------------------------------------------
|message name | string | xx bytes |
|-------------------------------------------------------------------
|message body | protobuf | xx bytes |
--------------------------------------------------------------------
protobuf structure like this:
--------------------------------------------------------------------
|error_code | enum | PresentDataErrorCode |
|-------------------------------------------------------------------
|error_message | string | xx bytes |
|-------------------------------------------------------------------
enum PresentDataErrorCode {
kPresentDataErrorNone = 0;
kPresentDataErrorUnsupportedType = 1;
kPresentDataErrorUnsupportedFormat = 2;
kPresentDataErrorOther = -1;
}
"""
response.error_code = err_code
ret_code = True
if err_code == pb2.kPresentDataErrorUnsupportedFormat:
response.error_message = "Present data not support format."
logging.error("Present data not support format.")
ret_code = False
elif err_code == pb2.kPresentDataErrorNone:
response.error_message = "Present data ok"
ret_code = True
else:
response.error_message = "Present data not known error."
logging.error("Present data not known error.")
ret_code = False
self.send_message(conn, response, pb2._PRESENTIMAGERESPONSE.full_name)
return ret_code
def _process_image_request(self, conn, msg_data):
"""
Deserialization protobuf and process display image request
Args:
conn: a socket connection
msg_data: a protobuf struct, include image request.
Returns:
protobuf structure like this:
------------------------------------
|data | byts |
|------------------------------------
|width | uint32 |
|------------------------------------
enum ImageFormat {
kImageFormatJpeg = 0;
}
"""
request = pb2.PresentImageRequest()
response = pb2.PresentImageResponse()
# Parse msg_data from protobuf
try:
request.ParseFromString(msg_data)
except DecodeError:
logging.error("ParseFromString exception: Error parsing message")
err_code = pb2.kPresentDataErrorOther
return self._response_image_request(conn, response, err_code)
sock_fileno = conn.fileno()
handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno)
if handler is None:
logging.error("get channel handler failed")
err_code = pb2.kPresentDataErrorOther
return self._response_image_request(conn, response, err_code)
rectangle_list = []
if request.rectangle_list:
for one_rectangle in request.rectangle_list:
rectangle = []
rectangle.append(one_rectangle.left_top.x)
rectangle.append(one_rectangle.left_top.y)
rectangle.append(one_rectangle.right_bottom.x)
rectangle.append(one_rectangle.right_bottom.y)
rectangle.append(one_rectangle.label_text)
# add the detection result to list
rectangle_list.append(rectangle)
handler.save_image(request.data, request.width, request.height,
rectangle_list)
return self._response_image_request(conn, response,
pb2.kPresentDataErrorNone)
def stop_thread(self):
channel_manager = ChannelManager([])
channel_manager.close_all_thread()
self.set_exit_switch()
