def __init__(self, server_address): '''init func''' self.channel_manager = ChannelManager(["image", "video"]) super(DisplayServer, self).__init__(server_address) self.app_manager = AppManager() self.register_dict = {}
def __init__(self): """ init method """ self.channel_mgr = ChannelManager(["image", "video"]) self.request_list = set() self.lock = threading.Lock()
def stop_thread(self): """ Description: clean thread when process exit. Input: NA Returns: NA """ channel_manager = ChannelManager([]) channel_manager.close_all_thread() self.set_exit_switch() self.app_manager.set_thread_switch()
def close_all_thread(signum, frame): '''close all thread of the process, and exit.''' logging.info("receive signal, signum:%s, frame:%s", signum, frame) webapp.stop_webapp() SOCKET_SERVER.stop_thread() channel_manager = ChannelManager() channel_manager.close_all_thread() logging.info("presenter server exit by Ctrl + c") sys.exit()
def __init__(self): """ init method """ self.channel_mgr = ChannelManager(["image", "video"]) self.request_list = set() self.lock = threading.Lock() self.anti = antispoof() print (self.anti.get_ans())
def __init__(self, server_address): """ Args: server_address: server listen address, include an ipv4 address and a port. """ # thread exit switch, if set true, thread must exit immediately. self._thread_exit_switch = False self._config = config_parser.ConfigParser() self._channel_manager = ChannelManager() self._create_socket_server(server_address)
def __init__(self): """ init method """ self.channel_mgr = ChannelManager(["image", "video"]) self.request_list = set() self.lock = threading.Lock() self.display_manager = display_server.DisplayServerManager() self.thread = None self._create_sending_thread()
def __init__(self, channel_name, media_type): self._channel_name = channel_name self._media_type = media_type self._img_data = None self._thread = None # last time the channel receive data. self._heartbeat = time.time() self._web_event = ThreadEvent(timeout=WEB_EVENT_TIMEOUT) self._image_event = ThreadEvent(timeout=IMAGE_EVENT_TIMEOUT) self._frame = None self._lock = threading.Lock() self._channel_manager = ChannelManager() if media_type == "video": self._thread_name = "videothread-{}".format(self._channel_name) self._create_thread()
def __new__(cls): """ensure only a single instance created. """ if cls.__instance is None: cls.__instance = object.__new__(cls) cls.channel_manager = ChannelManager([]) cls._create_thread() return cls.__instance
def __init__(self, config): """ Description: class init func Input: config: config information Returns: NA """ server_address = (config.presenter_server_ip, int(config.presenter_server_port)) super(FacialRecognitionServer, self).__init__(server_address) self.storage_dir = config.storage_dir self.max_face_num = int(config.max_face_num) self.face_match_threshold = float(config.face_match_threshold) self.register_dict = {} self.app_manager = AppManager() self.channel_manager = ChannelManager() self.face_register_file = os.path.join(self.storage_dir, "registered_faces.json") self._init_face_database()
def __init__(self, channel_name, media_type): self.channel_name = channel_name self.media_type = media_type self.img_data = None self._frame = None self.thread = None self._frame = None # last time the channel receive data. self.heartbeat = time.time() self.web_event = ThreadEvent(timeout=WEB_EVENT_TIMEOUT) self.image_event = ThreadEvent(timeout=IMAGE_EVENT_TIMEOUT) self.lock = threading.Lock() self.channel_manager = ChannelManager([]) self.rectangle_list = None if media_type == "video": self.thread_name = "videothread-{}".format(self.channel_name) self.heartbeat = time.time() self.close_thread_switch = False self.fps = 0 self.image_number = 0 self.time_list = [] self._create_thread()
def __init__(self, server_address): '''init func''' self.channel_manager = ChannelManager(["image", "video"]) super(CarRecognitionServer, self).__init__(server_address)
class FacialRecognitionServer(PresenterSocketServer): '''A server for face recognition''' def __init__(self, config): """ Description: class init func Input: config: config information Returns: NA """ server_address = (config.presenter_server_ip, int(config.presenter_server_port)) super(FacialRecognitionServer, self).__init__(server_address) self.storage_dir = config.storage_dir self.max_face_num = int(config.max_face_num) self.face_match_threshold = float(config.face_match_threshold) self.register_dict = {} self.app_manager = AppManager() self.channel_manager = ChannelManager() self.face_register_file = os.path.join(self.storage_dir, "registered_faces.json") self._init_face_database() def _init_face_database(self): """ Description: Init face recognition database, read information from face_register_file Input: NA Returns: NA """ if not os.path.isfile(self.face_register_file): with open(self.face_register_file, "w", encoding="utf-8") as f: f.write("{}") with open(self.face_register_file, "r") as f: self.face_lock = threading.Lock() self.registered_faces = json.load(f) self._filter_registration_data() def _filter_registration_data(self): face_dict = self.registered_faces.copy() for i in face_dict: image_path = os.path.join(self.storage_dir, i + ".jpg") if not os.path.isfile(image_path): del self.registered_faces[i] def get_all_face(self): """ Description: get registered face list. Input: NA Returns: NA """ with self.face_lock: return [i for i in self.registered_faces] def save_face_image(self, name, image): """ Description: save face image. Input: name face name image: face image Returns: True or False """ image_file = os.path.join(self.storage_dir, name + ".jpg") try: #image = image.decode("utf-8") with open(image_file, "wb") as f: f.write(image) return True except (OSError, TypeError) as exp: logging.error(exp) return False def get_app_socket(self, app_id): """ Description: get a socket which is bound to the app. Input: app_id: id of the app Returns: socket """ return self.app_manager.get_socket_by_app_id(app_id) def list_registered_apps(self): """ Description: get registered apps list. Input: NA Returns: app list """ return self.app_manager.list_app() def delete_faces(self, name_list): """ Description: delete registered faces in name_list Input: name_list: a name list Returns: True or False """ with self.face_lock: for i in name_list: if self.registered_faces.get(i): backup = self.registered_faces[i] del self.registered_faces[i] try: with open(self.face_register_file, "w") as f: json.dump(self.registered_faces, f) image_file = os.path.join(self.storage_dir, i + ".jpg") os.remove(image_file) except (OSError, JSONDecodeError) as exp: logging.error(exp) self.registered_faces[i] = backup return False return True def _clean_connect(self, sock_fileno, epoll, conns, msgs): """ Description: close socket, and clean local variables Input: sock_fileno: a socket fileno, return value of socket.fileno() epoll: a set of select.epoll. conns: all socket connections registered in epoll msgs: msg read from a socket """ logging.info("clean fd:%s, conns:%s", sock_fileno, conns) self.app_manager.unregister_app_by_fd(sock_fileno) epoll.unregister(sock_fileno) conns[sock_fileno].close() del conns[sock_fileno] del msgs[sock_fileno] def _process_msg(self, conn, msg_name, msg_data): """ Total entrance to process protobuf msg Input: conn: a socket connection msg_name: name of a msg. msg_data: msg body, serialized by protobuf Returns: False:somme error occured True:succeed """ # process open channel request if msg_name == pb2._REGISTERAPP.full_name: ret = self._process_register_app(conn, msg_data) # process image request, receive an image data from presenter agent elif msg_name == pb2._FACERESULT.full_name: ret = self._process_face_result(msg_data) elif msg_name == pb2._FRAMEINFO.full_name: ret = self._process_frame_info(conn, msg_data) elif msg_name == presenter_message_pb2._OPENCHANNELREQUEST.full_name: ret = self._process_open_channel(conn, msg_data) # process heartbeat request, it used to keepalive a channel path elif msg_name == presenter_message_pb2._HEARTBEATMESSAGE.full_name: ret = self._process_heartbeat(conn) else: logging.error("Not recognized msg type %s", msg_name) ret = False return ret def _process_heartbeat(self, conn): ''' set heartbeat Input: conn: a socket connection Returns: True: set heartbeat ok. ''' sock_fileno = conn.fileno() if self.app_manager.get_app_id_by_socket(sock_fileno): self.app_manager.set_heartbeat(sock_fileno) handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is not None: handler.set_heartbeat() return True def _parse_protobuf(self, protobuf, msg_data): """ Description: parse protobuf Input: protobuf: a struct defined by protobuf msg_data: msg body, serialized by protobuf Returns: True or False """ try: protobuf.ParseFromString(msg_data) return True except DecodeError as exp: logging.error(exp) return False def _process_register_app(self, conn, msg_data): """ Description: process register_app message Input: conn: a socket connection msg_data: msg body, serialized by protobuf Returns: True or False """ request = pb2.RegisterApp() response = pb2.CommonResponse() msg_name = pb2._COMMONRESPONSE.full_name if not self._parse_protobuf(request, msg_data): response.ret = pb2.kErrorOther response.message = "ParseFromString exception" self.send_message(conn, response, msg_name) return False app_id = request.id app_type = request.type # check app id if exist if self.app_manager.is_app_exist(app_id): logging.error("App %s is already exist.", app_id) response.ret = pb2.kErrorAppRegisterExist response.message = "App {} is already exist.".format(app_id) self.send_message(conn, response, msg_name) elif self.app_manager.get_app_num() >= MAX_APP_NUM: logging.error("App number reach the upper limit") response.ret = pb2.kErrorAppRegisterLimit response.message = "App number reach the upper limit" self.send_message(conn, response, msg_name) elif app_type != SERVER_TYPE: logging.error("App type %s error", app_type) response.ret = pb2.kErrorAppRegisterType response.message = "App type {} error".format(app_type) self.send_message(conn, response, msg_name) elif len(app_id) > APP_ID_MAX_LENGTH: logging.error("App id %s is too long", app_id) response.ret = pb2.kErrorOther response.message = "App id: {} is too long".format(app_id) self.send_message(conn, response, msg_name) else: self.app_manager.register_app(app_id, conn) response.ret = pb2.kErrorNone response.message = "Register app {} succeed".format(app_id) self.send_message(conn, response, msg_name) return True return False def _process_face_result(self, msg_data): """ Description: process face_result message Input: msg_data: msg body, serialized by protobuf Returns: True or False """ face_result = pb2.FaceResult() if not self._parse_protobuf(face_result, msg_data): return False face_id = face_result.id if not self.register_dict.get(face_id): logging.warning("face id %s is already deleted", face_id) return True ret = face_result.response.ret if ret != pb2.kErrorNone: err_msg = face_result.response.message logging.error("get face feature error message: %s", err_msg) status = FACE_REGISTER_STATUS_FAILED message = "Get face feature failed" self._update_register_dict(face_id, status, message) return True face_num = len(face_result.feature) if face_num == 0: status = FACE_REGISTER_STATUS_FAILED message = "No face recognized" self._update_register_dict(face_id, status, message) elif face_num > 1: status = FACE_REGISTER_STATUS_FAILED message = "{} faces recognized".format(face_num) self._update_register_dict(face_id, status, message) else: box = face_result.feature[0].box face_coordinate = [box.lt_x, box.lt_y, box.rb_x, box.rb_x] feature_vector = [i for i in face_result.feature[0].vector] if len(feature_vector) != FEATURE_VECTOR_LENGTH: logging.error("feature_vector length not equal 1024") status = FACE_REGISTER_STATUS_FAILED message = "Face feature vector length invalid" self._update_register_dict(face_id, status, message) return True return self._save_face_feature(face_id, face_coordinate, feature_vector) return True def _update_register_dict(self, face_id, status, message): """ Description: update register_dict Input: face_id: id of face status: status of face register message: message of status of face register Returns: True or False """ if self.register_dict.get(face_id): self.register_dict[face_id]["status"] = status self.register_dict[face_id]["message"] = message self.register_dict[face_id]["event"].set() def _save_face_feature(self, face_id, face_coordinate, feature_vector): """ Description: save face_feature Input: face_id: id of face face_coordinate: face coordinates feature_vector: face feature vector Returns: True or False """ with self.face_lock: self.registered_faces[face_id] = { "coordinate": face_coordinate, "feature": feature_vector } try: with open(self.face_register_file, "w") as f: json.dump(self.registered_faces, f) status = FACE_REGISTER_STATUS_SUCCEED message = "Successful registration" self._update_register_dict(face_id, status, message) return True except (OSError, JSONDecodeError) as exp: logging.error(exp) del self.registered_faces[face_id] status = FACE_REGISTER_STATUS_FAILED message = "save face feature to json file failed" self._update_register_dict(face_id, status, message) return False def _process_open_channel(self, conn, msg_data): """ Description: process open channel message Input: conn: a socket connection msg_data: msg body, serialized by protobuf Returns: True or False """ request = presenter_message_pb2.OpenChannelRequest() response = presenter_message_pb2.OpenChannelResponse() if not self._parse_protobuf(request, msg_data): channel_name = "unknown channel" err_code = presenter_message_pb2.kOpenChannelErrorOther return self._response_open_channel(conn, channel_name, response, err_code) channel_name = request.channel_name # check channel name if exist if not self.channel_manager.is_channel_exist(channel_name): logging.error("channel name %s is not exist.", channel_name) err_code = presenter_message_pb2.kOpenChannelErrorNoSuchChannel return self._response_open_channel(conn, channel_name, response, err_code) # check channel path if busy if self.channel_manager.is_channel_busy(channel_name): logging.error("channel path %s is busy.", channel_name) err = presenter_message_pb2.kOpenChannelErrorChannelAlreadyOpened return self._response_open_channel(conn, channel_name, response, err) content_type = presenter_message_pb2.kChannelContentTypeVideo if request.content_type == content_type: media_type = "video" else: logging.error("media type %s is not recognized.", request.content_type) err_code = presenter_message_pb2.kOpenChannelErrorOther return self._response_open_channel(conn, channel_name, response, err_code) handler = FacialRecognitionHandler(channel_name, media_type) sock = conn.fileno() self.channel_manager.create_channel_resource(channel_name, sock, media_type, handler) err_code = presenter_message_pb2.kOpenChannelErrorNone return self._response_open_channel(conn, channel_name, response, err_code) def _process_frame_info(self, conn, msg_data): """ Description: process frame info message Input: conn: a socket connection msg_data: msg body, serialized by protobuf Returns: True or False """ request = pb2.FrameInfo() response = pb2.CommonResponse() msg_name = pb2._COMMONRESPONSE.full_name if not self._parse_protobuf(request, msg_data): return False sock_fileno = conn.fileno() handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is None: logging.error("get channel handler failed") response.ret = pb2.kErrorOther response.message = "channel error." self.send_message(conn, response, msg_name) return False face_list = self._recognize_face(request.feature) handler.save_frame(request.image, face_list) response.ret = pb2.kErrorNone response.message = "process frame info suceed." self.send_message(conn, response, msg_name) return True def _recognize_face(self, face_feature): """ Description: recognize which face it is. Input: face_feature: face feature Returns: face list """ face_list = [] for i in face_feature: face_info = {} box = i.box coordinate = [box.lt_x, box.lt_y, box.rb_x, box.rb_y] feature_vector = i.vector if len(feature_vector) != FEATURE_VECTOR_LENGTH: logging.error("feature_vector length not equal 1024") continue (name, score, allmsg) = self._compute_face_feature(feature_vector) face_info["coordinate"] = coordinate face_info["name"] = name face_info["confidence"] = score face_info["allmsg"] = allmsg face_list.append(face_info) return face_list def _compute_face_feature(self, feture_vector): """ Description: compute score of the feture_vector Input: feture_vector: face feature vector Returns: face name and score """ highest_score_face = "Unknown" highest_score = 0 allmsg = "" with self.face_lock: for i in self.registered_faces: feature = self.registered_faces[i]["feature"] score = self._compute_similar_degree(feature, feture_vector) if score < self.face_match_threshold: continue allmsg += str(i) + ":" + str(round(score, 2)) + "," if score > highest_score: highest_score = score highest_score_face = i if allmsg != "": allmsg = allmsg[:-1] return (highest_score_face, highest_score, allmsg) def _compute_similar_degree(self, feture_vector1, feture_vector2): """ Description: compute cosine similarity of two vectors Input: feture_vector1: face feature vector feture_vector2: face feature vector Returns: score """ vector1 = np.array(feture_vector1) vector2 = np.array(feture_vector2) square_diff = ((np.linalg.norm(vector1)) * (np.linalg.norm(vector2))) score = np.dot(vector1, vector2) / square_diff return score def stop_thread(self): """ Description: clean thread when process exit. Input: NA Returns: NA """ channel_manager = ChannelManager([]) channel_manager.close_all_thread() self.set_exit_switch() self.app_manager.set_thread_switch()
def __init__(self, server_address): '''init func''' self.channel_manager = ChannelManager(["image", "video"]) super(crowdcountingServer, self).__init__(server_address)
def __init__(self, server_address): '''init func''' self.channel_manager = ChannelManager(["image", "video"]) super(peppapigdetectionServer, self).__init__(server_address)
class peppapigdetectionServer(PresenterSocketServer): '''A server for face detection''' def __init__(self, server_address): '''init func''' self.channel_manager = ChannelManager(["image", "video"]) super(peppapigdetectionServer, self).__init__(server_address) def _clean_connect(self, sock_fileno, epoll, conns, msgs): """ close socket, and clean local variables Args: sock_fileno: a socket fileno, return value of socket.fileno() epoll: a set of select.epoll. conns: all socket connections registered in epoll msgs: msg read from a socket """ logging.info("clean fd:%s, conns:%s", sock_fileno, conns) self.channel_manager.clean_channel_resource_by_fd(sock_fileno) epoll.unregister(sock_fileno) conns[sock_fileno].close() del conns[sock_fileno] del msgs[sock_fileno] def _process_msg(self, conn, msg_name, msg_data): """ Total entrance to process protobuf msg Args: conn: a socket connection msg_name: name of a msg. msg_data: msg body, serialized by protobuf Returns: False:somme error occured True:succeed """ # process open channel request if msg_name == pb2._OPENCHANNELREQUEST.full_name: ret = self._process_open_channel(conn, msg_data) # process image request, receive an image data from presenter agent elif msg_name == pb2._PRESENTIMAGEREQUEST.full_name: ret = self._process_image_request(conn, msg_data) # process heartbeat request, it used to keepalive a channel path elif msg_name == pb2._HEARTBEATMESSAGE.full_name: ret = self._process_heartbeat(conn) else: logging.error("Not recognized msg type %s", msg_name) ret = False return ret def _response_image_request(self, conn, response, err_code): """ Assemble protobuf to response image_request Message structure like this: -------------------------------------------------------------------- |total message len | int | 4 bytes | |------------------------------------------------------------------- |message name len | byte | 1 byte | |------------------------------------------------------------------- |message name | string | xx bytes | |------------------------------------------------------------------- |message body | protobuf | xx bytes | -------------------------------------------------------------------- protobuf structure like this: -------------------------------------------------------------------- |error_code | enum | PresentDataErrorCode | |------------------------------------------------------------------- |error_message | string | xx bytes | |------------------------------------------------------------------- enum PresentDataErrorCode { kPresentDataErrorNone = 0; kPresentDataErrorUnsupportedType = 1; kPresentDataErrorUnsupportedFormat = 2; kPresentDataErrorOther = -1; } """ response.error_code = err_code ret_code = True if err_code == pb2.kPresentDataErrorUnsupportedFormat: response.error_message = "Present data not support format." logging.error("Present data not support format.") ret_code = False elif err_code == pb2.kPresentDataErrorNone: response.error_message = "Present data ok" ret_code = True else: response.error_message = "Present data not known error." logging.error("Present data not known error.") ret_code = False self.send_message(conn, response, pb2._PRESENTIMAGERESPONSE.full_name) return ret_code def _process_image_request(self, conn, msg_data): """ Deserialization protobuf and process image_request Args: conn: a socket connection msg_data: a protobuf struct, include image request. Returns: protobuf structure like this: ------------------------------------ |format | ImageFormat | |------------------------------------ |width | uint32 | |------------------------------------ |height | uint32 | |------------------------------------ |data | bytes | ------------------------------------ enum ImageFormat { kImageFormatJpeg = 0; } """ request = pb2.PresentImageRequest() response = pb2.PresentImageResponse() # Parse msg_data from protobuf try: request.ParseFromString(msg_data) except DecodeError: logging.error("ParseFromString exception: Error parsing message") err_code = pb2.kPresentDataErrorOther return self._response_image_request(conn, response, err_code) sock_fileno = conn.fileno() handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is None: logging.error("get channel handler failed") err_code = pb2.kPresentDataErrorOther return self._response_image_request(conn, response, err_code) # Currently, image format only support jpeg if request.format != pb2.kImageFormatJpeg: logging.error("image format %s not support", request.format) err_code = pb2.kPresentDataErrorUnsupportedFormat return self._response_image_request(conn, response, err_code) rectangle_list = [] if request.rectangle_list: for one_rectangle in request.rectangle_list: rectangle = [] rectangle.append(one_rectangle.left_top.x) rectangle.append(one_rectangle.left_top.y) rectangle.append(one_rectangle.right_bottom.x) rectangle.append(one_rectangle.right_bottom.y) rectangle.append(one_rectangle.label_text) # add the detection result to list rectangle_list.append(rectangle) handler.save_image(request.data, request.width, request.height, rectangle_list) return self._response_image_request(conn, response, pb2.kPresentDataErrorNone) def stop_thread(self): channel_manager = ChannelManager([]) channel_manager.close_all_thread() self.set_exit_switch()
def stop_thread(self): channel_manager = ChannelManager([]) channel_manager.close_all_thread() self.set_exit_switch()
class TestFacialRecognitionHandler(unittest.TestCase): """TestFacialRecognitionHandler""" channel_name = "facial_recognition" media_type = "video" channel_manager = ChannelManager() handler = None def func_end(self): self.handler.close_thread_switch = True channel_name = TestFacialRecognitionHandler.channel_name TestFacialRecognitionHandler.channel_manager.unregister_one_channel(channel_name) def func_begin(self): channel_name = TestFacialRecognitionHandler.channel_name media_type = TestFacialRecognitionHandler.media_type TestFacialRecognitionHandler.channel_manager.register_one_channel(channel_name) self.handler = FacialRecognitionHandler(channel_name, media_type) @classmethod def tearDownClass(cls): pass @classmethod def setUpClass(cls): pass def run_thread(self, func): thread = threading.Thread(target=func) thread.start() def set_img_data(self): time.sleep(0.5) self.handler.img_data = b'1234' def set_img_data_none(self): time.sleep(1.5) self.handler.img_data = None @patch('threading.Event.clear', return_value = True) @patch('threading.Event.wait', return_value = True) def test_save_frame1(self, mock1, mock2): self.func_begin() image = b'1234' face_list = [] self.handler.img_data = b'12' self.handler.save_frame(image, face_list) self.func_end() def test_save_frame2(self): self.func_begin() image = b'1234' face_list = [] self.handler.img_data = b'12' self.run_thread(self.set_img_data_none) self.handler.save_frame(image, face_list) self.func_end() @patch('threading.Event.clear', return_value = True) @patch('threading.Event.wait', return_value = True) def test_frames(self, mock1, mock2): self.func_begin() self.handler.close_thread_switch = True time.sleep(0.5) # wait thread exit self.handler.img_data = None backup_heartbeat = facial_recognition_handler.HEARTBEAT_TIMEOUT facial_recognition_handler.HEARTBEAT_TIMEOUT = 0 self.handler.close_thread_switch = False for frame in self.handler.frames(): self.assertEqual(frame, None) break facial_recognition_handler.HEARTBEAT_TIMEOUT = backup_heartbeat self.func_end() @patch('threading.Event.clear', return_value = True) @patch('threading.Event.wait') def test_get_frame(self, mock1, mock2): self.func_begin() self.handler.frame_data = b'123' self.handler.face_list = [] self.handler.fps = 5 mock1.return_value = True ret = self.handler.get_frame() self.assertNotEqual(ret, {}) mock1.return_value = False ret = self.handler.get_frame() self.assertEqual(ret, {}) mock1.return_value = True self.func_end()
class WebApp: """ web application """ __instance = None def __init__(self): """ init method """ self.channel_mgr = ChannelManager(["image", "video"]) self.request_list = set() self.lock = threading.Lock() def __new__(cls, *args, **kwargs): # if instance is None than create one if cls.__instance is None: cls.__instance = object.__new__(cls, *args, **kwargs) return cls.__instance def add_channel(self, channel_name): """ add channel @param channel_name name of channel @return: return add status and message (for error status) """ ret = {"ret": "error", "msg": ""} # check channel_name validate, # channel_name can not be None or length = 0 if channel_name is None: logging.info("Channel name is None , add channel failed") ret["msg"] = "Channel name can not be empty" return ret # strip channel name channel_name = channel_name.strip() # check channel_name emtpy or not if channel_name == "": logging.info("Channel name is emtpy , add channel failed") ret["msg"] = "Channel name can not be empty" return ret # length of channel name can not over 25 if len(channel_name) > 25: logging.info("Length of channel name %s > 25 , add channel failed", channel_name) ret["msg"] = "Length of channel name should less than 25" return ret # define pattern support a-z A-Z and / pattern = re.compile(r"[a-z]|[A-Z]|[0-9]|/") tmp = pattern.findall(channel_name) # check reuslt changed or not if len(tmp) != len(channel_name): logging.info("%s contain invalidate character, add channel failed", channel_name) ret["msg"] = "Channel name only support 0-9, a-z, A-Z /" return ret # register channel flag = self.channel_mgr.register_one_channel(channel_name) # check register result if self.channel_mgr.err_code_too_many_channel == flag: logging.info("Only supports up to 10 channels, add channel failed") ret["msg"] = "Only supports up to 10 channels" elif self.channel_mgr.err_code_repeat_channel == flag: logging.info("%s already exist, add channel failed", channel_name) ret["msg"] = "Channel %s already exist" % channel_name else: logging.info("add channel %s succeed", channel_name) ret["ret"] = "success" return ret def del_channel(self, names): """ delete channel @param names: channel name to be deleted, separated by ',' @return: return add status and message (for error status) """ # init ret for return ret = {"ret": "error", "msg": ""} # check length of names if names.strip() == "": logging.info("Channel name is empty, delete channel failed") ret["msg"] = "Channel name should not be empty" return ret # split name for multi name listname = names.split(",") # unregister name for item in listname: item = item.strip() # if name is emtpy continu if item == "": continue self.channel_mgr.unregister_one_channel(item) logging.info("delete channel %s succeed", item) ret["ret"] = "success" return ret def list_channels(self): """ list all channels information """ # list register channels ret = self.channel_mgr.list_channels() # id for every channel item , start with 1 idx = 1 # set id for channel for item in ret: item['id'] = idx idx = idx + 1 return ret def is_channel_exists(self, name): """ view channel content via browser. @param name : channel name @return return True if exists. otherwise return False. """ return self.channel_mgr.is_channel_exist(name) def add_requst(self, request): """ add request @param requst: request item to be stored @note: request can not be same with other request. request is identified by (channel name ,random number) so this method do not return value. """ with self.lock: self.request_list.add(request) def has_request(self, request): """ whether request exist or not @param request: request to be checked. @return: return True if exists, otherwise return False. """ with self.lock: for item in self.request_list: # check request equal if item[0] == request[0] and item[1] == request[1]: return True return False def get_media_data(self, channel_name): """ get media data by channel name @param channel_name: channel to be quest data. @return return dictionary which have for item type: identify channel type, for image or video. image: data to be returned. fps: just for video type status: can be error, ok, or loading. """ # channel exists or not if self.is_channel_exists(channel_name) is False: return {'type': 'unkown', 'image': '', 'fps': 0, 'status': 'error'} image_data = self.channel_mgr.get_channel_image(channel_name) # only for image type. if image_data is not None: image_data = base64.b64encode(image_data).decode('utf-8') return { 'type': 'image', 'image': image_data, 'fps': 0, 'status': 'ok' } fps = 0 # fps for video image = None # image for video & image rectangle_list = None handler = self.channel_mgr.get_channel_handler_by_name(channel_name) if handler is not None: media_type = handler.get_media_type() # if type is image then get image data if media_type == "image": image = handler.get_image_data() # for video else: frame_info = handler.get_frame() image = frame_info[0] fps = frame_info[1] rectangle_list = frame_info[4] status = "loading" # decode binary to utf-8 when image is not None if image is not None: status = "ok" image = base64.b64encode(image).decode('utf-8') return { 'type': media_type, 'image': image, 'fps': fps, 'status': status, 'rectangle_list': rectangle_list } else: return { 'type': 'unkown', 'image': None, 'fps': 0, 'status': 'loading' }
def __init__(self, server_address): '''init func''' self.channel_manager = ChannelManager(["image", "video"]) super(HeadPoseServer, self).__init__(server_address)
class ChannelHandler(): """A set of channel handlers, process data received from channel""" def __init__(self, channel_name, media_type): self.channel_name = channel_name self.media_type = media_type self.img_data = None self._frame = None self.thread = None self._frame = None # last time the channel receive data. self.heartbeat = time.time() self.web_event = ThreadEvent(timeout=WEB_EVENT_TIMEOUT) self.image_event = ThreadEvent(timeout=IMAGE_EVENT_TIMEOUT) self.lock = threading.Lock() self.channel_manager = ChannelManager([]) self.rectangle_list = None if media_type == "video": self.thread_name = "videothread-{}".format(self.channel_name) self.heartbeat = time.time() self.close_thread_switch = False self.fps = 0 self.image_number = 0 self.time_list = [] self._create_thread() def close_thread(self): """close thread if object has created""" if self.thread is None: return self.set_thread_switch() self.image_event.set() logging.info("%s set _close_thread_switch True", self.thread_name) def set_heartbeat(self): """record heartbeat""" self.heartbeat = time.time() def set_thread_switch(self): """record heartbeat""" self.close_thread_switch = True def save_image(self, data, width, height, rectangle_list): """save image receive from socket""" self.width = width self.height = height self.rectangle_list = rectangle_list # compute fps if type is video if self.media_type == "video": while self.img_data: time.sleep(0.01) self.time_list.append(self.heartbeat) self.image_number += 1 while self.time_list[0] + 1 < time.time(): self.time_list.pop(0) self.image_number -= 1 if self.image_number == 0: break self.fps = len(self.time_list) self.img_data = data self.image_event.set() else: self.img_data = data self.channel_manager.save_channel_image(self.channel_name, self.img_data, self.rectangle_list) self.heartbeat = time.time() def get_media_type(self): """get media_type, support image or video""" return self.media_type def get_image(self): """get image_data""" return self.img_data def _create_thread(self): """Start the background video thread if it isn't running yet.""" if self.thread is not None and self.thread.isAlive(): return # start background frame thread self.thread = threading.Thread(target=self._video_thread) self.thread.start() def get_frame(self): """Return the current video frame.""" # wait util receive a frame data, and push it to your browser. ret = self.web_event.wait() self.web_event.clear() # True: _web_event return because set() # False: _web_event return because timeout if ret: return (self._frame, self.fps, self.width, self.height, self.rectangle_list) return (None, None, None, None, None) def frames(self): """a generator generates image""" while True: self.image_event.wait() self.image_event.clear() if self.img_data: yield self.img_data self.img_data = None # if set _close_thread_switch, return immediately if self.close_thread_switch: yield None # if no frames or heartbeat coming in the last 100 seconds, # stop the thread and close socket if time.time() - self.heartbeat > HEARTBEAT_TIMEOUT: self.set_thread_switch() self.img_data = None yield None def _video_thread(self): """background thread to process video""" logging.info('create %s...', (self.thread_name)) for frame in self.frames(): if frame: # send signal to clients self._frame = frame self.web_event.set() # exit thread if self.close_thread_switch: self.channel_manager.clean_channel_resource_by_name( self.channel_name) logging.info('Stop thread:%s.', (self.thread_name)) break
class WebApp: """ web application """ __instance = None def __init__(self): """ init method """ self.channel_mgr = ChannelManager(["image", "video"]) self.request_list = set() self.lock = threading.Lock() self.display_manager = display_server.DisplayServerManager() self.thread = None self._create_sending_thread() def __new__(cls, *args, **kwargs): # if instance is None than create one if cls.__instance is None: cls.__instance = object.__new__(cls, *args, **kwargs) return cls.__instance def add_channel(self, channel_name): """ add channel @param channel_name name of channel @return: return add status and message (for error status) """ ret = {"ret": "error", "msg": ""} # check channel_name validate, # channel_name can not be None or length = 0 if channel_name is None: logging.info("Channel name is None , add channel failed") ret["msg"] = "Channel name can not be empty" return ret # strip channel name channel_name = channel_name.strip() # check channel_name emtpy or not if channel_name == "": logging.info("Channel name is emtpy , add channel failed") ret["msg"] = "Channel name can not be empty" return ret # length of channel name can not over 25 if len(channel_name) > 25: logging.info("Length of channel name %s > 25 , add channel failed", channel_name) ret["msg"] = "Length of channel name should less than 25" return ret # define pattern support a-z A-Z and / pattern = re.compile(r"[a-z]|[A-Z]|[0-9]|/") tmp = pattern.findall(channel_name) # check reuslt changed or not if len(tmp) != len(channel_name): logging.info("%s contain invalidate character, add channel failed", channel_name) ret["msg"] = "Channel name only support 0-9, a-z, A-Z /" return ret # register channel flag = self.channel_mgr.register_one_channel(channel_name) # check register result if self.channel_mgr.err_code_too_many_channel == flag: logging.info("Only supports up to 10 channels, add channel failed") ret["msg"] = "Only supports up to 10 channels" elif self.channel_mgr.err_code_repeat_channel == flag: logging.info("%s already exist, add channel failed", channel_name) ret["msg"] = "Channel %s already exist" % channel_name else: logging.info("add channel %s succeed", channel_name) ret["ret"] = "success" return ret def del_channel(self, names): """ delete channel @param names: channel name to be deleted, separated by ',' @return: return add status and message (for error status) """ # init ret for return ret = {"ret": "error", "msg": ""} # check length of names if names.strip() == "": logging.info("Channel name is empty, delete channel failed") ret["msg"] = "Channel name should not be empty" return ret # split name for multi name listname = names.split(",") # unregister name for item in listname: item = item.strip() # if name is emtpy continu if item == "": continue self.channel_mgr.unregister_one_channel(item) logging.info("delete channel %s succeed", item) ret["ret"] = "success" return ret def list_channels(self): """ list all channels information """ # list register channels ret = self.channel_mgr.list_channels() # id for every channel item , start with 1 idx = 1 # set id for channel for item in ret: item['id'] = idx idx = idx + 1 return ret def is_channel_exists(self, name): """ view channel content via browser. @param name : channel name @return return True if exists. otherwise return False. """ return self.channel_mgr.is_channel_exist(name) def add_requst(self, request): """ add request @param requst: request item to be stored @note: request can not be same with other request. request is identified by (channel name ,random number) so this method do not return value. """ with self.lock: self.request_list.add(request) def has_request(self, request): """ whether request exist or not @param request: request to be checked. @return: return True if exists, otherwise return False. """ with self.lock: for item in self.request_list: # check request equal if item[0] == request[0] and item[1] == request[1]: return True return False def get_media_data(self, channel_name): """ get media data by channel name @param channel_name: channel to be quest data. @return return dictionary which have for item type: identify channel type, for image or video. image: data to be returned. fps: just for video type status: can be error, ok, or loading. """ # channel exists or not if self.is_channel_exists(channel_name) is False: return {'type': 'unkown', 'image': '', 'fps': 0, 'status': 'error'} image_data = self.channel_mgr.get_channel_image(channel_name) # only for image type. if image_data is not None: image_data = base64.b64encode(image_data).decode('utf-8') return { 'type': 'image', 'image': image_data, 'fps': 0, 'status': 'ok' } fps = 0 # fps for video image = None # image for video & image rectangle_list = None view = None handler = self.channel_mgr.get_channel_handler_by_name(channel_name) if handler is not None: media_type = handler.get_media_type() # if type is image then get image data if media_type == "image": image = handler.get_image_data() # for video else: frame_info = handler.get_frame() image = frame_info[0] fps = frame_info[1] rectangle_list = frame_info[4] width, height = frame_info[2], frame_info[2] if width == 256: view = 'ColorMap' else: view = 'ResultImage' status = "loading" # decode binary to utf-8 when image is not None if image is not None: status = "ok" image = base64.b64encode(image).decode('utf-8') return { 'view': view, 'type': media_type, 'image': image, 'fps': fps, 'status': status, 'rectangle_list': rectangle_list } else: return { 'view': view, 'type': 'unkown', 'image': None, 'fps': 0, 'status': 'loading' } def _create_sending_thread(self): """Start the sending thread if it isn't running yet.""" if self.thread is not None and self.thread.isAlive(): return # start style type thread self.thread = threading.Thread(target=self.sending_thread_func, args=(self.display_manager, )) self.thread.start() def sending_thread_func(self, app_manager): global g_object_bytes, g_layout_bytes while True: event.wait() flag = app_manager.send_model_input_data(g_object_bytes, g_layout_bytes) print("data package sending msg:", flag[1]) if flag[0] is True: logging.info("Send model input data package success") else: logging.info("Send model input data package failed") event.clear() def update_model_input_data(self, objects_bytes, layout_bytes): ''' send model data to agent''' ### check if the input changed same input ### type change, trigger the wait thread to register global g_object_bytes, g_layout_bytes g_object_bytes = b'' g_layout_bytes = b'' g_object_bytes = objects_bytes g_layout_bytes = layout_bytes if g_layout_bytes == b'' or g_object_bytes == b'': return "" # Here, you may not use thread event method to trigger the transfer manager # to register style type directly, it maybe ok. But here i use thread event instead. print("send model input data thread event set") event.set()
class PresenterSocketServer(): """a socket server communication with presenter agent. """ def __init__(self, server_address): """ Args: server_address: server listen address, include an ipv4 address and a port. """ # thread exit switch, if set true, thread must exit immediately. self._thread_exit_switch = False self._config = config_parser.ConfigParser() self._channel_manager = ChannelManager() self._create_socket_server(server_address) def _create_socket_server(self, server_address): """ create a socket server Args: server_address: server listen address, include an ipv4 address and a port. """ # Create a socket server. self._sock_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._sock_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self._sock_server.bind(server_address) self._sock_server.listen(SOCKET_WAIT_QUEUE) self._sock_server.setblocking(False) # Get server host name and port host, port = self._sock_server.getsockname()[:2] # Start presenter socket server thread. threading.Thread(target=self._server_listen_thread).start() # Display directly on the screen print('Presenter socket server listen on %s:%s\n' % (host, port)) def stop_thread(self): """set switch True to stop presenter socket server thread.""" self._thread_exit_switch = True def _read_socket(self, conn, read_len): ''' Read fixed length data Args: conn: a socket connection read_len: read fix byte. Returns: ret: PRESENTER_OK or PRESENTER_ERR buf: read fix byte buf. ''' has_read_len = 0 read_buf = SOCK_RECV_NULL total_buf = SOCK_RECV_NULL recv_num = 0 while has_read_len != read_len: #Troubleshoot CPU problems when Network blocking if recv_num % 10 == 5: time.sleep(0.01) try: read_buf = conn.recv(read_len - has_read_len) except socket.error: logging.error("socket %u exception:socket.error", conn.fileno()) return PRESENTER_ERR, None if read_buf == SOCK_RECV_NULL: return PRESENTER_ERR, None total_buf += read_buf has_read_len = len(total_buf) recv_num += 1 return PRESENTER_OK, total_buf def _read_msg_head(self, sock_fileno, conns): ''' Args: sock_fileno: a socket fileno conns: all socket connections which created by server. Returns: msg_total_len: total message length. msg_name_len: message name length. ''' ret, msg_head = self._read_socket(conns[sock_fileno], MSG_HEAD_LENGTH) if ret == PRESENTER_ERR: logging.error("socket %u receive msg head null", sock_fileno) return None, None # in Struct(), 'I' is unsigned int, 'B' is unsigned char msg_head_data = struct.Struct('IB') (msg_total_len, msg_name_len) = msg_head_data.unpack(msg_head) msg_total_len = socket.ntohl(msg_total_len) return msg_total_len, msg_name_len def _read_msg_name(self, sock_fd, conns, msg_name_len): ''' Args: sock_fd: a socket fileno conns: all socket connections which created by server. msg_name_len: message name length. Returns: ret: PRESENTER_OK or PRESENTER_ERR msg_name: message name. ''' ret, msg_name = self._read_socket(conns[sock_fd], msg_name_len) if ret == PRESENTER_ERR: logging.error("socket %u receive msg name null", sock_fd) return PRESENTER_ERR, None try: msg_name = msg_name.decode("utf-8") except UnicodeDecodeError: logging.error("msg name decode to utf-8 error") return PRESENTER_ERR, None return PRESENTER_OK, msg_name def _read_msg_body(self, sock_fd, conns, msg_body_len, msgs): ''' Args: sock_fd: a socket fileno conns: all socket connections which created by server. msg_name_len: message name length. msgs: msg read from a socket Returns: ret: PRESENTER_OK or PRESENTER_ERR ''' ret, msg_body = self._read_socket(conns[sock_fd], msg_body_len) if ret == PRESENTER_ERR: logging.error("socket %u receive msg body null", sock_fd) return PRESENTER_ERR msgs[sock_fd] = msg_body return PRESENTER_OK def _read_sock_and_process_msg(self, sock_fileno, conns, msgs): ''' Args: sock_fileno: a socket fileno, return value of socket.fileno() conns: all socket connections registered in epoll msgs: msg read from a socket Returns: ret: PRESENTER_OK or PRESENTER_ERR ''' # Step1: read msg head msg_total_len, msg_name_len = self._read_msg_head(sock_fileno, conns) if msg_total_len is None: return PRESENTER_ERR # Step2: read msg name ret, msg_name = self._read_msg_name(sock_fileno, conns, msg_name_len) if ret == PRESENTER_ERR: return ret # Step3: read msg body msg_body_len = msg_total_len - MSG_HEAD_LENGTH - msg_name_len if msg_body_len < 0: logging.error("msg_total_len:%u, msg_name_len:%u, msg_body_len:%u", msg_total_len, msg_name_len, msg_body_len) return PRESENTER_ERR ret = self._read_msg_body(sock_fileno, conns, msg_body_len, msgs) if ret == PRESENTER_ERR: return ret # Step4: process msg ret = self._process_msg(conns[sock_fileno], msg_name, msgs[sock_fileno]) return ret def _process_epollin(self, sock_fileno, epoll, conns, msgs): ''' Args: sock_fileno: a socket fileno, return value of socket.fileno() epoll: a set of select.epoll. conns: all socket connections registered in epoll msgs: msg read from a socket ''' msgs[sock_fileno] = b'' try: ret = self._read_sock_and_process_msg(sock_fileno, conns, msgs) if not ret: self._clean_connect(sock_fileno, epoll, conns, msgs) except socket.error: logging.error("receive socket error.") self._clean_connect(sock_fileno, epoll, conns, msgs) def _accept_new_socket(self, epoll, conns): ''' Args: epoll: a set of select.epoll. conns: all socket connections registered in epoll ''' try: new_conn, address = self._sock_server.accept() new_conn.setblocking(True) epoll.register(new_conn.fileno(), select.EPOLLIN | select.EPOLLHUP) conns[new_conn.fileno()] = new_conn logging.info("create new connection:client-ip:%s, client-port:%s", address[0], address[1]) except socket.error: logging.error("socket.error exception when sock.accept()") def _server_listen_thread(self): """socket server thread, epoll listening all the socket events""" epoll = select.epoll() epoll.register(self._sock_server.fileno(), select.EPOLLIN | select.EPOLLHUP) try: conns = {} msgs = {} while True: # thread must exit immediately if self._thread_exit_switch: break events = epoll.poll(EPOLL_TIMEOUT) # timeout, but no event come, continue waiting if not events: continue for sock_fileno, event in events: # new connection request from presenter agent if self._sock_server.fileno() == sock_fileno: self._accept_new_socket(epoll, conns) # remote connection closed # it means presenter agent exit withot close socket. elif event & select.EPOLLHUP: logging.info("receive event EPOLLHUP") self._clean_connect(sock_fileno, epoll, conns, msgs) # new data coming in a socket connection elif event & select.EPOLLIN: self._process_epollin(sock_fileno, epoll, conns, msgs) # receive event not recognize else: logging.error("not recognize event %f", event) self._clean_connect(sock_fileno, epoll, conns, msgs) finally: logging.info("conns:%s", conns) logging.info("presenter agent listen thread exit.") epoll.unregister(self._sock_server.fileno()) epoll.close() self._sock_server.close() def _clean_connect(self, sock_fileno, epoll, conns, msgs): """ close socket, and clean local variables Args: sock_fileno: a socket fileno, return value of socket.fileno() epoll: a set of select.epoll. conns: all socket connections registered in epoll msgs: msg read from a socket """ logging.info("clean sock_fd:%s, conns:%s", sock_fileno, conns) self._channel_manager.clean_channel_resource_by_fd(sock_fileno) epoll.unregister(sock_fileno) conns[sock_fileno].close() del conns[sock_fileno] del msgs[sock_fileno] def _process_msg(self, conn, msg_name, msg_data): """ Total entrance to process protobuf msg Args: conn: a socket connection msg_name: name of a msg. msg_data: msg body, serialized by protobuf Returns: False:somme error occured True:succeed """ # process open channel request if msg_name == OPEN_CHANNEL_REQUEST_FULL_NAME: ret = self._process_open_channel(conn, msg_data) # process image request, receive an image data from presenter agent elif msg_name == PRESENT_IMAGE_REQUEST_FULL_NAME: ret = self._process_image_request(conn, msg_data) # process heartbeat request, it used to keepalive a channel path elif msg_name == HEART_BEAT_MESSAGE_FULL_NAME: ret = self._process_heartbeat(conn) else: logging.error("Not recognized msg type %s", msg_name) ret = PRESENTER_ERR return ret def _response_open_channel(self, conn, channel_name, response, err_code): """ Assemble protobuf to response open_channel request Args: conn: a socket connection channel_name: name of a channel. response: a protobuf response to presenter agent err_code: part of the response Returns: ret_code:PRESENTER_OK or PRESENTER_ERR Message structure like this: -------------------------------------------------------------------- |total message len | int | 4 bytes | |------------------------------------------------------------------- |message name len | byte | 1 byte | |------------------------------------------------------------------- |message name | string | xx bytes | |------------------------------------------------------------------- |message body | protobuf | xx bytes | -------------------------------------------------------------------- protobuf structure like this: -------------------------------------------------------------------- |error_code | enum | OpenChannelErrorCode | |------------------------------------------------------------------- |error_message | string | xx bytes | |------------------------------------------------------------------- enum OpenChannelErrorCode { kOpenChannelErrorNone = 0; kOpenChannelErrorNoSuchChannel = 1; kOpenChannelErrorChannelAlreadyOpened = 2; kOpenChannelErrorOther = -1; } """ response.error_code = err_code ret_code = PRESENTER_ERR if err_code == OPEN_CHANNEL_NO_CHANNEL_ERR: response.error_message = "channel {} not exist." \ .format(channel_name) elif err_code == OPEN_CHANNEL_BUSY_ERR: response.error_message = "channel {} is busy.".format(channel_name) elif err_code == OPEN_CHANNEL_OK: response.error_message = "open channel succeed" ret_code = PRESENTER_OK else: response.error_message = "Unknown err open channel {}." \ .format(channel_name) self._send_response(conn, response, OPEN_CHANNEL_RESPONSE_FULL_NAME) return ret_code def _process_open_channel(self, conn, msg_data): """ Deserialization protobuf and process open_channel request Args: conn: a socket connection msg_data: a protobuf struct, include open channel request. Returns: protobuf structure like this: ---------------------------------------------- |channel_name | string | |---------------------------------------------- |content_type | ChannelContentType | |---------------------------------------------- enum ChannelContentType { kChannelContentTypeImage = 0; kChannelContentTypeVideo = 1; } """ request = pb.OpenChannelRequest() response = pb.OpenChannelResponse() try: request.ParseFromString(msg_data) except message.DecodeError: logging.error("ParseFromString exception: Error parsing message") channel_name = "unknown channel" return self._response_open_channel(conn, channel_name, response, OPEN_CHANNEL_UNKOWN_ERR) channel_name = request.channel_name # check channel name if exist if not self._channel_manager.is_channel_exist(channel_name): logging.error("channel name %s is not exist.", channel_name) return self._response_open_channel(conn, channel_name, response, OPEN_CHANNEL_NO_CHANNEL_ERR) # check channel path if busy if self._channel_manager.is_channel_busy(channel_name): logging.error("channel path %s is busy.", channel_name) return self._response_open_channel(conn, channel_name, response, OPEN_CHANNEL_BUSY_ERR) # if channel type is image, need clean image if exist self._channel_manager.clean_channel_image(channel_name) if request.content_type == IMAGE_TYPE: media_type = "image" elif request.content_type == VIDEO_TYPE: media_type = "video" else: logging.error("media type %s is not recognized.", request.content_type) return self._response_open_channel(conn, channel_name, response, OPEN_CHANNEL_UNKOWN_ERR) handler = ChannelHandler(channel_name, media_type) self._channel_manager.create_channel_resource(channel_name, conn.fileno(), media_type, handler) return self._response_open_channel(conn, channel_name, response, OPEN_CHANNEL_OK) def _send_response(self, conn, response, msg_name): response_data = response.SerializeToString() response_len = len(response_data) msg_name_size = len(msg_name) msg_total_size = MSG_HEAD_LENGTH + msg_name_size + response_len # in Struct(), 'I' is unsigned int, 'B' is unsigned char struct_head = struct.Struct('IB') msg_head = (socket.htonl(msg_total_size), msg_name_size) packed_msg_head = struct_head.pack(*msg_head) msg_data = packed_msg_head + \ bytes(msg_name, encoding="utf-8") + response_data conn.sendall(msg_data) def _response_image_request(self, conn, response, err_code): """ Assemble protobuf to response image_request Message structure like this: -------------------------------------------------------------------- |total message len | int | 4 bytes | |------------------------------------------------------------------- |message name len | byte | 1 byte | |------------------------------------------------------------------- |message name | string | xx bytes | |------------------------------------------------------------------- |message body | protobuf | xx bytes | -------------------------------------------------------------------- protobuf structure like this: -------------------------------------------------------------------- |error_code | enum | PresentDataErrorCode | |------------------------------------------------------------------- |error_message | string | xx bytes | |------------------------------------------------------------------- enum PresentDataErrorCode { kPresentDataErrorNone = 0; kPresentDataErrorUnsupportedType = 1; kPresentDataErrorUnsupportedFormat = 2; kPresentDataErrorOther = -1; } """ response.error_code = err_code ret_code = PRESENTER_OK if err_code == PRESENT_DATA_UNSUPPORTED_FORMAT_ERR: response.error_message = "Present data not support format." logging.error("Present data not support format.") ret_code = PRESENTER_ERR elif err_code == PRESENT_DATA_OK: response.error_message = "Present data ok" ret_code = PRESENTER_OK else: response.error_message = "Present data not known error." logging.error("Present data not known error.") ret_code = PRESENTER_ERR self._send_response(conn, response, PRESENT_IMAGE_RESPONSE_FULL_NAME) return ret_code def _process_image_request(self, conn, msg_data): """ Deserialization protobuf and process image_request Args: conn: a socket connection msg_data: a protobuf struct, include image request. Returns: protobuf structure like this: ------------------------------------ |format | ImageFormat | |------------------------------------ |width | uint32 | |------------------------------------ |height | uint32 | |------------------------------------ |data | bytes | ------------------------------------ enum ImageFormat { kImageFormatJpeg = 0; } """ request = pb.PresentImageRequest() response = pb.PresentImageResponse() # Parse msg_data from protobuf try: request.ParseFromString(msg_data) except message.DecodeError: logging.error("ParseFromString exception: Error parsing message") err_code = PRESENT_DATA_UNKOWN_ERR return self._response_image_request(conn, response, err_code) sock_fileno = conn.fileno() handler = self._channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is None: logging.error("get channel handler failed") err_code = PRESENT_DATA_UNKOWN_ERR return self._response_image_request(conn, response, err_code) # Currently, image format only support jpeg if request.format != FORMAT_JPEG: logging.error("image format %s not support", request.format) err_code = PRESENT_DATA_UNSUPPORTED_FORMAT_ERR return self._response_image_request(conn, response, err_code) handler.save_image(request.data, request.width, request.height) return self._response_image_request(conn, response, PRESENT_DATA_OK) def _process_heartbeat(self, conn): ''' set heartbeat Args: conn: a socket connection Returns: True: set heartbeat ok. ''' sock_fileno = conn.fileno() handler = self._channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is not None: handler.setheartbeat() return PRESENTER_OK
class FaceDetectionServer(PresenterSocketServer): '''A server for face detection''' def __init__(self, server_address): '''init func''' self.channel_manager = ChannelManager(["image", "video"]) super(FaceDetectionServer, self).__init__(server_address) def _clean_connect(self, sock_fileno, epoll, conns, msgs): """ close socket, and clean local variables Args: sock_fileno: a socket fileno, return value of socket.fileno() epoll: a set of select.epoll. conns: all socket connections registered in epoll msgs: msg read from a socket """ logging.info("clean fd:%s, conns:%s", sock_fileno, conns) self.channel_manager.clean_channel_resource_by_fd(sock_fileno) epoll.unregister(sock_fileno) conns[sock_fileno].close() del conns[sock_fileno] del msgs[sock_fileno] def _process_msg(self, conn, msg_name, msg_data): """ Total entrance to process protobuf msg Args: conn: a socket connection msg_name: name of a msg. msg_data: msg body, serialized by protobuf Returns: False:somme error occured True:succeed """ # process open channel request if msg_name == pb2._OPENCHANNELREQUEST.full_name: ret = self._process_open_channel(conn, msg_data) # process image request, receive an image data from presenter agent elif msg_name == pb2._PRESENTIMAGEREQUEST.full_name: ret = self._process_image_request(conn, msg_data) # process heartbeat request, it used to keepalive a channel path elif msg_name == pb2._HEARTBEATMESSAGE.full_name: ret = self._process_heartbeat(conn) else: logging.error("Not recognized msg type %s", msg_name) ret = False return ret def _response_image_request(self, conn, response, err_code): """ Assemble protobuf to response image_request Message structure like this: -------------------------------------------------------------------- |total message len | int | 4 bytes | |------------------------------------------------------------------- |message name len | byte | 1 byte | |------------------------------------------------------------------- |message name | string | xx bytes | |------------------------------------------------------------------- |message body | protobuf | xx bytes | -------------------------------------------------------------------- protobuf structure like this: -------------------------------------------------------------------- |error_code | enum | PresentDataErrorCode | |------------------------------------------------------------------- |error_message | string | xx bytes | |------------------------------------------------------------------- enum PresentDataErrorCode { kPresentDataErrorNone = 0; kPresentDataErrorUnsupportedType = 1; kPresentDataErrorUnsupportedFormat = 2; kPresentDataErrorOther = -1; } """ response.error_code = err_code ret_code = True if err_code == pb2.kPresentDataErrorUnsupportedFormat: response.error_message = "Present data not support format." logging.error("Present data not support format.") ret_code = False elif err_code == pb2.kPresentDataErrorNone: response.error_message = "Present data ok" ret_code = True else: response.error_message = "Present data not known error." logging.error("Present data not known error.") ret_code = False self.send_message(conn, response, pb2._PRESENTIMAGERESPONSE.full_name) return ret_code def _process_image_request(self, conn, msg_data): """ Deserialization protobuf and process image_request Args: conn: a socket connection msg_data: a protobuf struct, include image request. Returns: protobuf structure like this: ------------------------------------ |format | ImageFormat | |------------------------------------ |width | uint32 | |------------------------------------ |height | uint32 | |------------------------------------ |data | bytes | ------------------------------------ enum ImageFormat { kImageFormatJpeg = 0; } """ request = pb2.PresentImageRequest() response = pb2.PresentImageResponse() # Parse msg_data from protobuf try: request.ParseFromString(msg_data) except DecodeError: logging.error("ParseFromString exception: Error parsing message") err_code = pb2.kPresentDataErrorOther return self._response_image_request(conn, response, err_code) sock_fileno = conn.fileno() handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is None: logging.error("get channel handler failed") err_code = pb2.kPresentDataErrorOther return self._response_image_request(conn, response, err_code) # Currently, image format only support jpeg if request.format != pb2.kImageFormatJpeg: logging.error("image format %s not support", request.format) err_code = pb2.kPresentDataErrorUnsupportedFormat return self._response_image_request(conn, response, err_code) rectangle_list = [] if request.rectangle_list: for one_rectangle in request.rectangle_list: rectangle = [] rectangle.append(one_rectangle.left_top.x) rectangle.append(one_rectangle.left_top.y) rectangle.append(one_rectangle.right_bottom.x) rectangle.append(one_rectangle.right_bottom.y) rectangle.append(one_rectangle.label_text) # add the detection result to list rectangle_list.append(rectangle) imageData = request.data[:1382400] maskData = request.data[-100352:] # convert yuv420sp(nv12) to jpg yuvNum = len( [lists for lists in os.listdir('yuv') if lists.endswith(".yuv")]) with open(os.path.join('yuv', str(yuvNum) + '.yuv'), 'wb') as f: f.write(imageData) ff = ffmpy.FFmpeg( inputs={ os.path.join('yuv', str(yuvNum) + '.yuv'): '-s 1280*720 -pix_fmt nv12' }, outputs={os.path.join('yuv', str(yuvNum) + '.jpg'): None}) ff.run() # read data imgNow = cv2.imread(os.path.join('yuv', str(yuvNum) + '.jpg')) background = cv2.imread("background.jpg") maskarr = np.fromstring(maskData, np.uint8).astype(np.float32) maskarr = np.reshape(maskarr, (224, 224, 2)) alphargb = cv2.resize(maskarr[:, :, 1], (request.width, request.height)) # output the mask to compare result with the output of pb model cv2.imwrite(os.path.join('yuv', str(yuvNum) + 'alpha.jpg'), alphargb) alphargb = alphargb / 255 alphargb = np.repeat(alphargb[..., np.newaxis], 3, 2) result = np.uint8(imgNow * alphargb + background * (1 - alphargb)) img_decode = cv2.imencode('.jpg', result)[1].tostring() handler.save_image(img_decode, request.width, request.height, rectangle_list) return self._response_image_request(conn, response, pb2.kPresentDataErrorNone) def stop_thread(self): channel_manager = ChannelManager([]) channel_manager.close_all_thread() self.set_exit_switch()
def __init__(self, server_address): '''init func''' self.channel_manager = ChannelManager(["image", "video"]) self._image_cnt = 0 super(DisplayServer, self).__init__(server_address)
class DisplayServer(PresenterSocketServer): def __init__(self, server_address): '''init func''' self.channel_manager = ChannelManager(["image", "video"]) super(DisplayServer, self).__init__(server_address) self.app_manager = AppManager() self.register_dict = {} def _clean_connect(self, sock_fileno, epoll, conns, msgs): """ close socket, and clean local variables Args: sock_fileno: a socket fileno, return value of socket.fileno() epoll: a set of select.epoll. conns: all socket connections registered in epoll msgs: msg read from a socket """ logging.info("clean fd:%s, conns:%s", sock_fileno, conns) self.channel_manager.clean_channel_resource_by_fd(sock_fileno) epoll.unregister(sock_fileno) conns[sock_fileno].close() del conns[sock_fileno] del msgs[sock_fileno] def _process_msg(self, conn, msg_name, msg_data): """ Total entrance to process protobuf msg Args: conn: a socket connection msg_name: name of a msg. msg_data: msg body, serialized by protobuf Returns: False:somme error occured True:succeed """ ## process_register_app if msg_name == painting_message_pb2._REGISTERAPP.full_name: ret = self._process_register_app(conn, msg_data) # process open channel request elif msg_name == pb2._OPENCHANNELREQUEST.full_name: ret = self._process_open_channel(conn, msg_data) # process image request, receive an image data from presenter agent elif msg_name == pb2._PRESENTIMAGEREQUEST.full_name: ret = self._process_image_request(conn, msg_data) # process heartbeat request, it used to keepalive a channel path elif msg_name == pb2._HEARTBEATMESSAGE.full_name: ret = self._process_heartbeat(conn) else: logging.error("Not recognized msg type %s", msg_name) ret = False return ret # 重载_process_heartbeat def _process_heartbeat(self, conn): ''' set heartbeat Input: conn: a socket connection Returns: True: set heartbeat ok. ''' sock_fileno = conn.fileno() if self.app_manager.get_app_id_by_socket(sock_fileno): self.app_manager.set_heartbeat(sock_fileno) handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is not None: handler.set_heartbeat() return True def _parse_protobuf(self, protobuf, msg_data): """ Description: parse protobuf Input: protobuf: a struct defined by protobuf msg_data: msg body, serialized by protobuf Returns: True or False """ try: protobuf.ParseFromString(msg_data) return True except DecodeError as exp: logging.error(exp) return False def _process_register_app(self, conn, msg_data): """ Description: process register_app message Input: conn: a socket connection msg_data: msg body, serialized by protobuf Returns: True or False """ request = painting_message_pb2.RegisterApp() response = painting_message_pb2.CommonResponse() msg_name = painting_message_pb2._COMMONRESPONSE.full_name if not self._parse_protobuf(request, msg_data): response.ret = painting_message_pb2.kErrorOther response.message = "ParseFromString exception" self.send_message(conn, response, msg_name) return False app_id = request.id app_type = request.type # check app id if exist if self.app_manager.is_app_exist(app_id): logging.error("App %s is already exist.", app_id) response.ret = painting_message_pb2.kErrorAppRegisterExist response.message = "App {} is already exist.".format(app_id) self.send_message(conn, response, msg_name) elif self.app_manager.get_app_num() >= MAX_APP_NUM: logging.error("App number reach the upper limit") response.ret = painting_message_pb2.kErrorAppRegisterLimit response.message = "App number reach the upper limit" self.send_message(conn, response, msg_name) elif app_type != SERVER_TYPE: logging.error("App type %s error", app_type) response.ret = painting_message_pb2.kErrorAppRegisterType response.message = "App type {} error".format(app_type) self.send_message(conn, response, msg_name) elif len(app_id) > APP_ID_MAX_LENGTH: logging.error("App id %s is too long", app_id) response.ret = painting_message_pb2.kErrorOther response.message = "App id: {} is too long".format(app_id) self.send_message(conn, response, msg_name) else: self.app_manager.register_app(app_id, conn) response.ret = painting_message_pb2.kErrorNone response.message = "Register app {} succeed".format(app_id) self.send_message(conn, response, msg_name) return True return False def get_app_socket(self, app_id): """ Description: get a socket which is bound to the app. Input: app_id: id of the app Returns: socket """ return self.app_manager.get_socket_by_app_id(app_id) def list_registered_apps(self): """ Description: get registered apps list. Input: NA Returns: app list """ return self.app_manager.list_app() def _response_image_request(self, conn, response, err_code): """ Assemble protobuf to response image_request Message structure like this: -------------------------------------------------------------------- |total message len | int | 4 bytes | |------------------------------------------------------------------- |message name len | byte | 1 byte | |------------------------------------------------------------------- |message name | string | xx bytes | |------------------------------------------------------------------- |message body | protobuf | xx bytes | -------------------------------------------------------------------- protobuf structure like this: -------------------------------------------------------------------- |error_code | enum | PresentDataErrorCode | |------------------------------------------------------------------- |error_message | string | xx bytes | |------------------------------------------------------------------- enum PresentDataErrorCode { kPresentDataErrorNone = 0; kPresentDataErrorUnsupportedType = 1; kPresentDataErrorUnsupportedFormat = 2; kPresentDataErrorOther = -1; } """ response.error_code = err_code ret_code = True if err_code == pb2.kPresentDataErrorUnsupportedFormat: response.error_message = "Present data not support format." logging.error("Present data not support format.") ret_code = False elif err_code == pb2.kPresentDataErrorNone: response.error_message = "Present data ok" ret_code = True else: response.error_message = "Present data not known error." logging.error("Present data not known error.") ret_code = False self.send_message(conn, response, pb2._PRESENTIMAGERESPONSE.full_name) return ret_code def _process_image_request(self, conn, msg_data): """ Deserialization protobuf and process display image request Args: conn: a socket connection msg_data: a protobuf struct, include image request. Returns: protobuf structure like this: ------------------------------------ |data | byts | |------------------------------------ |width | uint32 | |------------------------------------ enum ImageFormat { kImageFormatJpeg = 0; } """ request = pb2.PresentImageRequest() response = pb2.PresentImageResponse() # Parse msg_data from protobuf try: request.ParseFromString(msg_data) except DecodeError: logging.error("ParseFromString exception: Error parsing message") err_code = pb2.kPresentDataErrorOther return self._response_image_request(conn, response, err_code) sock_fileno = conn.fileno() handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is None: logging.error("get channel handler failed") err_code = pb2.kPresentDataErrorOther return self._response_image_request(conn, response, err_code) rectangle_list = [] if request.rectangle_list: for one_rectangle in request.rectangle_list: rectangle = [] rectangle.append(one_rectangle.left_top.x) rectangle.append(one_rectangle.left_top.y) rectangle.append(one_rectangle.right_bottom.x) rectangle.append(one_rectangle.right_bottom.y) rectangle.append(one_rectangle.label_text) # add the detection result to list rectangle_list.append(rectangle) handler.save_image(request.data, request.width, request.height, rectangle_list) return self._response_image_request(conn, response, pb2.kPresentDataErrorNone) def stop_thread(self): channel_manager = ChannelManager([]) channel_manager.close_all_thread() self.set_exit_switch()