class FacialRecognitionServer(PresenterSocketServer): '''A server for face recognition''' def __init__(self, config): """ Description: class init func Input: config: config information Returns: NA """ server_address = (config.presenter_server_ip, int(config.presenter_server_port)) super(FacialRecognitionServer, self).__init__(server_address) self.storage_dir = config.storage_dir self.max_face_num = int(config.max_face_num) self.face_match_threshold = float(config.face_match_threshold) self.register_dict = {} self.app_manager = AppManager() self.channel_manager = ChannelManager() self.face_register_file = os.path.join(self.storage_dir, "registered_faces.json") self._init_face_database() def _init_face_database(self): """ Description: Init face recognition database, read information from face_register_file Input: NA Returns: NA """ if not os.path.isfile(self.face_register_file): with open(self.face_register_file, "w", encoding="utf-8") as f: f.write("{}") with open(self.face_register_file, "r") as f: self.face_lock = threading.Lock() self.registered_faces = json.load(f) self._filter_registration_data() def _filter_registration_data(self): face_dict = self.registered_faces.copy() for i in face_dict: image_path = os.path.join(self.storage_dir, i + ".jpg") if not os.path.isfile(image_path): del self.registered_faces[i] def get_all_face(self): """ Description: get registered face list. Input: NA Returns: NA """ with self.face_lock: return [i for i in self.registered_faces] def save_face_image(self, name, image): """ Description: save face image. Input: name face name image: face image Returns: True or False """ image_file = os.path.join(self.storage_dir, name + ".jpg") try: #image = image.decode("utf-8") with open(image_file, "wb") as f: f.write(image) return True except (OSError, TypeError) as exp: logging.error(exp) return False def get_app_socket(self, app_id): """ Description: get a socket which is bound to the app. Input: app_id: id of the app Returns: socket """ return self.app_manager.get_socket_by_app_id(app_id) def list_registered_apps(self): """ Description: get registered apps list. Input: NA Returns: app list """ return self.app_manager.list_app() def delete_faces(self, name_list): """ Description: delete registered faces in name_list Input: name_list: a name list Returns: True or False """ with self.face_lock: for i in name_list: if self.registered_faces.get(i): backup = self.registered_faces[i] del self.registered_faces[i] try: with open(self.face_register_file, "w") as f: json.dump(self.registered_faces, f) image_file = os.path.join(self.storage_dir, i + ".jpg") os.remove(image_file) except (OSError, JSONDecodeError) as exp: logging.error(exp) self.registered_faces[i] = backup return False return True def _clean_connect(self, sock_fileno, epoll, conns, msgs): """ Description: close socket, and clean local variables Input: sock_fileno: a socket fileno, return value of socket.fileno() epoll: a set of select.epoll. conns: all socket connections registered in epoll msgs: msg read from a socket """ logging.info("clean fd:%s, conns:%s", sock_fileno, conns) self.app_manager.unregister_app_by_fd(sock_fileno) epoll.unregister(sock_fileno) conns[sock_fileno].close() del conns[sock_fileno] del msgs[sock_fileno] def _process_msg(self, conn, msg_name, msg_data): """ Total entrance to process protobuf msg Input: conn: a socket connection msg_name: name of a msg. msg_data: msg body, serialized by protobuf Returns: False:somme error occured True:succeed """ # process open channel request if msg_name == pb2._REGISTERAPP.full_name: ret = self._process_register_app(conn, msg_data) # process image request, receive an image data from presenter agent elif msg_name == pb2._FACERESULT.full_name: ret = self._process_face_result(msg_data) elif msg_name == pb2._FRAMEINFO.full_name: ret = self._process_frame_info(conn, msg_data) elif msg_name == presenter_message_pb2._OPENCHANNELREQUEST.full_name: ret = self._process_open_channel(conn, msg_data) # process heartbeat request, it used to keepalive a channel path elif msg_name == presenter_message_pb2._HEARTBEATMESSAGE.full_name: ret = self._process_heartbeat(conn) else: logging.error("Not recognized msg type %s", msg_name) ret = False return ret def _process_heartbeat(self, conn): ''' set heartbeat Input: conn: a socket connection Returns: True: set heartbeat ok. ''' sock_fileno = conn.fileno() if self.app_manager.get_app_id_by_socket(sock_fileno): self.app_manager.set_heartbeat(sock_fileno) handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is not None: handler.set_heartbeat() return True def _parse_protobuf(self, protobuf, msg_data): """ Description: parse protobuf Input: protobuf: a struct defined by protobuf msg_data: msg body, serialized by protobuf Returns: True or False """ try: protobuf.ParseFromString(msg_data) return True except DecodeError as exp: logging.error(exp) return False def _process_register_app(self, conn, msg_data): """ Description: process register_app message Input: conn: a socket connection msg_data: msg body, serialized by protobuf Returns: True or False """ request = pb2.RegisterApp() response = pb2.CommonResponse() msg_name = pb2._COMMONRESPONSE.full_name if not self._parse_protobuf(request, msg_data): response.ret = pb2.kErrorOther response.message = "ParseFromString exception" self.send_message(conn, response, msg_name) return False app_id = request.id app_type = request.type # check app id if exist if self.app_manager.is_app_exist(app_id): logging.error("App %s is already exist.", app_id) response.ret = pb2.kErrorAppRegisterExist response.message = "App {} is already exist.".format(app_id) self.send_message(conn, response, msg_name) elif self.app_manager.get_app_num() >= MAX_APP_NUM: logging.error("App number reach the upper limit") response.ret = pb2.kErrorAppRegisterLimit response.message = "App number reach the upper limit" self.send_message(conn, response, msg_name) elif app_type != SERVER_TYPE: logging.error("App type %s error", app_type) response.ret = pb2.kErrorAppRegisterType response.message = "App type {} error".format(app_type) self.send_message(conn, response, msg_name) elif len(app_id) > APP_ID_MAX_LENGTH: logging.error("App id %s is too long", app_id) response.ret = pb2.kErrorOther response.message = "App id: {} is too long".format(app_id) self.send_message(conn, response, msg_name) else: self.app_manager.register_app(app_id, conn) response.ret = pb2.kErrorNone response.message = "Register app {} succeed".format(app_id) self.send_message(conn, response, msg_name) return True return False def _process_face_result(self, msg_data): """ Description: process face_result message Input: msg_data: msg body, serialized by protobuf Returns: True or False """ face_result = pb2.FaceResult() if not self._parse_protobuf(face_result, msg_data): return False face_id = face_result.id if not self.register_dict.get(face_id): logging.warning("face id %s is already deleted", face_id) return True ret = face_result.response.ret if ret != pb2.kErrorNone: err_msg = face_result.response.message logging.error("get face feature error message: %s", err_msg) status = FACE_REGISTER_STATUS_FAILED message = "Get face feature failed" self._update_register_dict(face_id, status, message) return True face_num = len(face_result.feature) if face_num == 0: status = FACE_REGISTER_STATUS_FAILED message = "No face recognized" self._update_register_dict(face_id, status, message) elif face_num > 1: status = FACE_REGISTER_STATUS_FAILED message = "{} faces recognized".format(face_num) self._update_register_dict(face_id, status, message) else: box = face_result.feature[0].box face_coordinate = [box.lt_x, box.lt_y, box.rb_x, box.rb_x] feature_vector = [i for i in face_result.feature[0].vector] if len(feature_vector) != FEATURE_VECTOR_LENGTH: logging.error("feature_vector length not equal 1024") status = FACE_REGISTER_STATUS_FAILED message = "Face feature vector length invalid" self._update_register_dict(face_id, status, message) return True return self._save_face_feature(face_id, face_coordinate, feature_vector) return True def _update_register_dict(self, face_id, status, message): """ Description: update register_dict Input: face_id: id of face status: status of face register message: message of status of face register Returns: True or False """ if self.register_dict.get(face_id): self.register_dict[face_id]["status"] = status self.register_dict[face_id]["message"] = message self.register_dict[face_id]["event"].set() def _save_face_feature(self, face_id, face_coordinate, feature_vector): """ Description: save face_feature Input: face_id: id of face face_coordinate: face coordinates feature_vector: face feature vector Returns: True or False """ with self.face_lock: self.registered_faces[face_id] = { "coordinate": face_coordinate, "feature": feature_vector } try: with open(self.face_register_file, "w") as f: json.dump(self.registered_faces, f) status = FACE_REGISTER_STATUS_SUCCEED message = "Successful registration" self._update_register_dict(face_id, status, message) return True except (OSError, JSONDecodeError) as exp: logging.error(exp) del self.registered_faces[face_id] status = FACE_REGISTER_STATUS_FAILED message = "save face feature to json file failed" self._update_register_dict(face_id, status, message) return False def _process_open_channel(self, conn, msg_data): """ Description: process open channel message Input: conn: a socket connection msg_data: msg body, serialized by protobuf Returns: True or False """ request = presenter_message_pb2.OpenChannelRequest() response = presenter_message_pb2.OpenChannelResponse() if not self._parse_protobuf(request, msg_data): channel_name = "unknown channel" err_code = presenter_message_pb2.kOpenChannelErrorOther return self._response_open_channel(conn, channel_name, response, err_code) channel_name = request.channel_name # check channel name if exist if not self.channel_manager.is_channel_exist(channel_name): logging.error("channel name %s is not exist.", channel_name) err_code = presenter_message_pb2.kOpenChannelErrorNoSuchChannel return self._response_open_channel(conn, channel_name, response, err_code) # check channel path if busy if self.channel_manager.is_channel_busy(channel_name): logging.error("channel path %s is busy.", channel_name) err = presenter_message_pb2.kOpenChannelErrorChannelAlreadyOpened return self._response_open_channel(conn, channel_name, response, err) content_type = presenter_message_pb2.kChannelContentTypeVideo if request.content_type == content_type: media_type = "video" else: logging.error("media type %s is not recognized.", request.content_type) err_code = presenter_message_pb2.kOpenChannelErrorOther return self._response_open_channel(conn, channel_name, response, err_code) handler = FacialRecognitionHandler(channel_name, media_type) sock = conn.fileno() self.channel_manager.create_channel_resource(channel_name, sock, media_type, handler) err_code = presenter_message_pb2.kOpenChannelErrorNone return self._response_open_channel(conn, channel_name, response, err_code) def _process_frame_info(self, conn, msg_data): """ Description: process frame info message Input: conn: a socket connection msg_data: msg body, serialized by protobuf Returns: True or False """ request = pb2.FrameInfo() response = pb2.CommonResponse() msg_name = pb2._COMMONRESPONSE.full_name if not self._parse_protobuf(request, msg_data): return False sock_fileno = conn.fileno() handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is None: logging.error("get channel handler failed") response.ret = pb2.kErrorOther response.message = "channel error." self.send_message(conn, response, msg_name) return False face_list = self._recognize_face(request.feature) handler.save_frame(request.image, face_list) response.ret = pb2.kErrorNone response.message = "process frame info suceed." self.send_message(conn, response, msg_name) return True def _recognize_face(self, face_feature): """ Description: recognize which face it is. Input: face_feature: face feature Returns: face list """ face_list = [] for i in face_feature: face_info = {} box = i.box coordinate = [box.lt_x, box.lt_y, box.rb_x, box.rb_y] feature_vector = i.vector if len(feature_vector) != FEATURE_VECTOR_LENGTH: logging.error("feature_vector length not equal 1024") continue (name, score, allmsg) = self._compute_face_feature(feature_vector) face_info["coordinate"] = coordinate face_info["name"] = name face_info["confidence"] = score face_info["allmsg"] = allmsg face_list.append(face_info) return face_list def _compute_face_feature(self, feture_vector): """ Description: compute score of the feture_vector Input: feture_vector: face feature vector Returns: face name and score """ highest_score_face = "Unknown" highest_score = 0 allmsg = "" with self.face_lock: for i in self.registered_faces: feature = self.registered_faces[i]["feature"] score = self._compute_similar_degree(feature, feture_vector) if score < self.face_match_threshold: continue allmsg += str(i) + ":" + str(round(score, 2)) + "," if score > highest_score: highest_score = score highest_score_face = i if allmsg != "": allmsg = allmsg[:-1] return (highest_score_face, highest_score, allmsg) def _compute_similar_degree(self, feture_vector1, feture_vector2): """ Description: compute cosine similarity of two vectors Input: feture_vector1: face feature vector feture_vector2: face feature vector Returns: score """ vector1 = np.array(feture_vector1) vector2 = np.array(feture_vector2) square_diff = ((np.linalg.norm(vector1)) * (np.linalg.norm(vector2))) score = np.dot(vector1, vector2) / square_diff return score def stop_thread(self): """ Description: clean thread when process exit. Input: NA Returns: NA """ channel_manager = ChannelManager([]) channel_manager.close_all_thread() self.set_exit_switch() self.app_manager.set_thread_switch()
class peppapigdetectionServer(PresenterSocketServer): '''A server for face detection''' def __init__(self, server_address): '''init func''' self.channel_manager = ChannelManager(["image", "video"]) super(peppapigdetectionServer, self).__init__(server_address) def _clean_connect(self, sock_fileno, epoll, conns, msgs): """ close socket, and clean local variables Args: sock_fileno: a socket fileno, return value of socket.fileno() epoll: a set of select.epoll. conns: all socket connections registered in epoll msgs: msg read from a socket """ logging.info("clean fd:%s, conns:%s", sock_fileno, conns) self.channel_manager.clean_channel_resource_by_fd(sock_fileno) epoll.unregister(sock_fileno) conns[sock_fileno].close() del conns[sock_fileno] del msgs[sock_fileno] def _process_msg(self, conn, msg_name, msg_data): """ Total entrance to process protobuf msg Args: conn: a socket connection msg_name: name of a msg. msg_data: msg body, serialized by protobuf Returns: False:somme error occured True:succeed """ # process open channel request if msg_name == pb2._OPENCHANNELREQUEST.full_name: ret = self._process_open_channel(conn, msg_data) # process image request, receive an image data from presenter agent elif msg_name == pb2._PRESENTIMAGEREQUEST.full_name: ret = self._process_image_request(conn, msg_data) # process heartbeat request, it used to keepalive a channel path elif msg_name == pb2._HEARTBEATMESSAGE.full_name: ret = self._process_heartbeat(conn) else: logging.error("Not recognized msg type %s", msg_name) ret = False return ret def _response_image_request(self, conn, response, err_code): """ Assemble protobuf to response image_request Message structure like this: -------------------------------------------------------------------- |total message len | int | 4 bytes | |------------------------------------------------------------------- |message name len | byte | 1 byte | |------------------------------------------------------------------- |message name | string | xx bytes | |------------------------------------------------------------------- |message body | protobuf | xx bytes | -------------------------------------------------------------------- protobuf structure like this: -------------------------------------------------------------------- |error_code | enum | PresentDataErrorCode | |------------------------------------------------------------------- |error_message | string | xx bytes | |------------------------------------------------------------------- enum PresentDataErrorCode { kPresentDataErrorNone = 0; kPresentDataErrorUnsupportedType = 1; kPresentDataErrorUnsupportedFormat = 2; kPresentDataErrorOther = -1; } """ response.error_code = err_code ret_code = True if err_code == pb2.kPresentDataErrorUnsupportedFormat: response.error_message = "Present data not support format." logging.error("Present data not support format.") ret_code = False elif err_code == pb2.kPresentDataErrorNone: response.error_message = "Present data ok" ret_code = True else: response.error_message = "Present data not known error." logging.error("Present data not known error.") ret_code = False self.send_message(conn, response, pb2._PRESENTIMAGERESPONSE.full_name) return ret_code def _process_image_request(self, conn, msg_data): """ Deserialization protobuf and process image_request Args: conn: a socket connection msg_data: a protobuf struct, include image request. Returns: protobuf structure like this: ------------------------------------ |format | ImageFormat | |------------------------------------ |width | uint32 | |------------------------------------ |height | uint32 | |------------------------------------ |data | bytes | ------------------------------------ enum ImageFormat { kImageFormatJpeg = 0; } """ request = pb2.PresentImageRequest() response = pb2.PresentImageResponse() # Parse msg_data from protobuf try: request.ParseFromString(msg_data) except DecodeError: logging.error("ParseFromString exception: Error parsing message") err_code = pb2.kPresentDataErrorOther return self._response_image_request(conn, response, err_code) sock_fileno = conn.fileno() handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is None: logging.error("get channel handler failed") err_code = pb2.kPresentDataErrorOther return self._response_image_request(conn, response, err_code) # Currently, image format only support jpeg if request.format != pb2.kImageFormatJpeg: logging.error("image format %s not support", request.format) err_code = pb2.kPresentDataErrorUnsupportedFormat return self._response_image_request(conn, response, err_code) rectangle_list = [] if request.rectangle_list: for one_rectangle in request.rectangle_list: rectangle = [] rectangle.append(one_rectangle.left_top.x) rectangle.append(one_rectangle.left_top.y) rectangle.append(one_rectangle.right_bottom.x) rectangle.append(one_rectangle.right_bottom.y) rectangle.append(one_rectangle.label_text) # add the detection result to list rectangle_list.append(rectangle) handler.save_image(request.data, request.width, request.height, rectangle_list) return self._response_image_request(conn, response, pb2.kPresentDataErrorNone) def stop_thread(self): channel_manager = ChannelManager([]) channel_manager.close_all_thread() self.set_exit_switch()
class FaceDetectionServer(PresenterSocketServer): '''A server for face detection''' def __init__(self, server_address): '''init func''' self.channel_manager = ChannelManager(["image", "video"]) super(FaceDetectionServer, self).__init__(server_address) def _clean_connect(self, sock_fileno, epoll, conns, msgs): """ close socket, and clean local variables Args: sock_fileno: a socket fileno, return value of socket.fileno() epoll: a set of select.epoll. conns: all socket connections registered in epoll msgs: msg read from a socket """ logging.info("clean fd:%s, conns:%s", sock_fileno, conns) self.channel_manager.clean_channel_resource_by_fd(sock_fileno) epoll.unregister(sock_fileno) conns[sock_fileno].close() del conns[sock_fileno] del msgs[sock_fileno] def _process_msg(self, conn, msg_name, msg_data): """ Total entrance to process protobuf msg Args: conn: a socket connection msg_name: name of a msg. msg_data: msg body, serialized by protobuf Returns: False:somme error occured True:succeed """ # process open channel request if msg_name == pb2._OPENCHANNELREQUEST.full_name: ret = self._process_open_channel(conn, msg_data) # process image request, receive an image data from presenter agent elif msg_name == pb2._PRESENTIMAGEREQUEST.full_name: ret = self._process_image_request(conn, msg_data) # process heartbeat request, it used to keepalive a channel path elif msg_name == pb2._HEARTBEATMESSAGE.full_name: ret = self._process_heartbeat(conn) else: logging.error("Not recognized msg type %s", msg_name) ret = False return ret def _response_image_request(self, conn, response, err_code): """ Assemble protobuf to response image_request Message structure like this: -------------------------------------------------------------------- |total message len | int | 4 bytes | |------------------------------------------------------------------- |message name len | byte | 1 byte | |------------------------------------------------------------------- |message name | string | xx bytes | |------------------------------------------------------------------- |message body | protobuf | xx bytes | -------------------------------------------------------------------- protobuf structure like this: -------------------------------------------------------------------- |error_code | enum | PresentDataErrorCode | |------------------------------------------------------------------- |error_message | string | xx bytes | |------------------------------------------------------------------- enum PresentDataErrorCode { kPresentDataErrorNone = 0; kPresentDataErrorUnsupportedType = 1; kPresentDataErrorUnsupportedFormat = 2; kPresentDataErrorOther = -1; } """ response.error_code = err_code ret_code = True if err_code == pb2.kPresentDataErrorUnsupportedFormat: response.error_message = "Present data not support format." logging.error("Present data not support format.") ret_code = False elif err_code == pb2.kPresentDataErrorNone: response.error_message = "Present data ok" ret_code = True else: response.error_message = "Present data not known error." logging.error("Present data not known error.") ret_code = False self.send_message(conn, response, pb2._PRESENTIMAGERESPONSE.full_name) return ret_code def _process_image_request(self, conn, msg_data): """ Deserialization protobuf and process image_request Args: conn: a socket connection msg_data: a protobuf struct, include image request. Returns: protobuf structure like this: ------------------------------------ |format | ImageFormat | |------------------------------------ |width | uint32 | |------------------------------------ |height | uint32 | |------------------------------------ |data | bytes | ------------------------------------ enum ImageFormat { kImageFormatJpeg = 0; } """ request = pb2.PresentImageRequest() response = pb2.PresentImageResponse() # Parse msg_data from protobuf try: request.ParseFromString(msg_data) except DecodeError: logging.error("ParseFromString exception: Error parsing message") err_code = pb2.kPresentDataErrorOther return self._response_image_request(conn, response, err_code) sock_fileno = conn.fileno() handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is None: logging.error("get channel handler failed") err_code = pb2.kPresentDataErrorOther return self._response_image_request(conn, response, err_code) # Currently, image format only support jpeg if request.format != pb2.kImageFormatJpeg: logging.error("image format %s not support", request.format) err_code = pb2.kPresentDataErrorUnsupportedFormat return self._response_image_request(conn, response, err_code) rectangle_list = [] if request.rectangle_list: for one_rectangle in request.rectangle_list: rectangle = [] rectangle.append(one_rectangle.left_top.x) rectangle.append(one_rectangle.left_top.y) rectangle.append(one_rectangle.right_bottom.x) rectangle.append(one_rectangle.right_bottom.y) rectangle.append(one_rectangle.label_text) # add the detection result to list rectangle_list.append(rectangle) imageData = request.data[:1382400] maskData = request.data[-100352:] # convert yuv420sp(nv12) to jpg yuvNum = len( [lists for lists in os.listdir('yuv') if lists.endswith(".yuv")]) with open(os.path.join('yuv', str(yuvNum) + '.yuv'), 'wb') as f: f.write(imageData) ff = ffmpy.FFmpeg( inputs={ os.path.join('yuv', str(yuvNum) + '.yuv'): '-s 1280*720 -pix_fmt nv12' }, outputs={os.path.join('yuv', str(yuvNum) + '.jpg'): None}) ff.run() # read data imgNow = cv2.imread(os.path.join('yuv', str(yuvNum) + '.jpg')) background = cv2.imread("background.jpg") maskarr = np.fromstring(maskData, np.uint8).astype(np.float32) maskarr = np.reshape(maskarr, (224, 224, 2)) alphargb = cv2.resize(maskarr[:, :, 1], (request.width, request.height)) # output the mask to compare result with the output of pb model cv2.imwrite(os.path.join('yuv', str(yuvNum) + 'alpha.jpg'), alphargb) alphargb = alphargb / 255 alphargb = np.repeat(alphargb[..., np.newaxis], 3, 2) result = np.uint8(imgNow * alphargb + background * (1 - alphargb)) img_decode = cv2.imencode('.jpg', result)[1].tostring() handler.save_image(img_decode, request.width, request.height, rectangle_list) return self._response_image_request(conn, response, pb2.kPresentDataErrorNone) def stop_thread(self): channel_manager = ChannelManager([]) channel_manager.close_all_thread() self.set_exit_switch()
class DisplayServer(PresenterSocketServer): def __init__(self, server_address): '''init func''' self.channel_manager = ChannelManager(["image", "video"]) super(DisplayServer, self).__init__(server_address) self.app_manager = AppManager() self.register_dict = {} def _clean_connect(self, sock_fileno, epoll, conns, msgs): """ close socket, and clean local variables Args: sock_fileno: a socket fileno, return value of socket.fileno() epoll: a set of select.epoll. conns: all socket connections registered in epoll msgs: msg read from a socket """ logging.info("clean fd:%s, conns:%s", sock_fileno, conns) self.channel_manager.clean_channel_resource_by_fd(sock_fileno) epoll.unregister(sock_fileno) conns[sock_fileno].close() del conns[sock_fileno] del msgs[sock_fileno] def _process_msg(self, conn, msg_name, msg_data): """ Total entrance to process protobuf msg Args: conn: a socket connection msg_name: name of a msg. msg_data: msg body, serialized by protobuf Returns: False:somme error occured True:succeed """ ## process_register_app if msg_name == painting_message_pb2._REGISTERAPP.full_name: ret = self._process_register_app(conn, msg_data) # process open channel request elif msg_name == pb2._OPENCHANNELREQUEST.full_name: ret = self._process_open_channel(conn, msg_data) # process image request, receive an image data from presenter agent elif msg_name == pb2._PRESENTIMAGEREQUEST.full_name: ret = self._process_image_request(conn, msg_data) # process heartbeat request, it used to keepalive a channel path elif msg_name == pb2._HEARTBEATMESSAGE.full_name: ret = self._process_heartbeat(conn) else: logging.error("Not recognized msg type %s", msg_name) ret = False return ret # 重载_process_heartbeat def _process_heartbeat(self, conn): ''' set heartbeat Input: conn: a socket connection Returns: True: set heartbeat ok. ''' sock_fileno = conn.fileno() if self.app_manager.get_app_id_by_socket(sock_fileno): self.app_manager.set_heartbeat(sock_fileno) handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is not None: handler.set_heartbeat() return True def _parse_protobuf(self, protobuf, msg_data): """ Description: parse protobuf Input: protobuf: a struct defined by protobuf msg_data: msg body, serialized by protobuf Returns: True or False """ try: protobuf.ParseFromString(msg_data) return True except DecodeError as exp: logging.error(exp) return False def _process_register_app(self, conn, msg_data): """ Description: process register_app message Input: conn: a socket connection msg_data: msg body, serialized by protobuf Returns: True or False """ request = painting_message_pb2.RegisterApp() response = painting_message_pb2.CommonResponse() msg_name = painting_message_pb2._COMMONRESPONSE.full_name if not self._parse_protobuf(request, msg_data): response.ret = painting_message_pb2.kErrorOther response.message = "ParseFromString exception" self.send_message(conn, response, msg_name) return False app_id = request.id app_type = request.type # check app id if exist if self.app_manager.is_app_exist(app_id): logging.error("App %s is already exist.", app_id) response.ret = painting_message_pb2.kErrorAppRegisterExist response.message = "App {} is already exist.".format(app_id) self.send_message(conn, response, msg_name) elif self.app_manager.get_app_num() >= MAX_APP_NUM: logging.error("App number reach the upper limit") response.ret = painting_message_pb2.kErrorAppRegisterLimit response.message = "App number reach the upper limit" self.send_message(conn, response, msg_name) elif app_type != SERVER_TYPE: logging.error("App type %s error", app_type) response.ret = painting_message_pb2.kErrorAppRegisterType response.message = "App type {} error".format(app_type) self.send_message(conn, response, msg_name) elif len(app_id) > APP_ID_MAX_LENGTH: logging.error("App id %s is too long", app_id) response.ret = painting_message_pb2.kErrorOther response.message = "App id: {} is too long".format(app_id) self.send_message(conn, response, msg_name) else: self.app_manager.register_app(app_id, conn) response.ret = painting_message_pb2.kErrorNone response.message = "Register app {} succeed".format(app_id) self.send_message(conn, response, msg_name) return True return False def get_app_socket(self, app_id): """ Description: get a socket which is bound to the app. Input: app_id: id of the app Returns: socket """ return self.app_manager.get_socket_by_app_id(app_id) def list_registered_apps(self): """ Description: get registered apps list. Input: NA Returns: app list """ return self.app_manager.list_app() def _response_image_request(self, conn, response, err_code): """ Assemble protobuf to response image_request Message structure like this: -------------------------------------------------------------------- |total message len | int | 4 bytes | |------------------------------------------------------------------- |message name len | byte | 1 byte | |------------------------------------------------------------------- |message name | string | xx bytes | |------------------------------------------------------------------- |message body | protobuf | xx bytes | -------------------------------------------------------------------- protobuf structure like this: -------------------------------------------------------------------- |error_code | enum | PresentDataErrorCode | |------------------------------------------------------------------- |error_message | string | xx bytes | |------------------------------------------------------------------- enum PresentDataErrorCode { kPresentDataErrorNone = 0; kPresentDataErrorUnsupportedType = 1; kPresentDataErrorUnsupportedFormat = 2; kPresentDataErrorOther = -1; } """ response.error_code = err_code ret_code = True if err_code == pb2.kPresentDataErrorUnsupportedFormat: response.error_message = "Present data not support format." logging.error("Present data not support format.") ret_code = False elif err_code == pb2.kPresentDataErrorNone: response.error_message = "Present data ok" ret_code = True else: response.error_message = "Present data not known error." logging.error("Present data not known error.") ret_code = False self.send_message(conn, response, pb2._PRESENTIMAGERESPONSE.full_name) return ret_code def _process_image_request(self, conn, msg_data): """ Deserialization protobuf and process display image request Args: conn: a socket connection msg_data: a protobuf struct, include image request. Returns: protobuf structure like this: ------------------------------------ |data | byts | |------------------------------------ |width | uint32 | |------------------------------------ enum ImageFormat { kImageFormatJpeg = 0; } """ request = pb2.PresentImageRequest() response = pb2.PresentImageResponse() # Parse msg_data from protobuf try: request.ParseFromString(msg_data) except DecodeError: logging.error("ParseFromString exception: Error parsing message") err_code = pb2.kPresentDataErrorOther return self._response_image_request(conn, response, err_code) sock_fileno = conn.fileno() handler = self.channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is None: logging.error("get channel handler failed") err_code = pb2.kPresentDataErrorOther return self._response_image_request(conn, response, err_code) rectangle_list = [] if request.rectangle_list: for one_rectangle in request.rectangle_list: rectangle = [] rectangle.append(one_rectangle.left_top.x) rectangle.append(one_rectangle.left_top.y) rectangle.append(one_rectangle.right_bottom.x) rectangle.append(one_rectangle.right_bottom.y) rectangle.append(one_rectangle.label_text) # add the detection result to list rectangle_list.append(rectangle) handler.save_image(request.data, request.width, request.height, rectangle_list) return self._response_image_request(conn, response, pb2.kPresentDataErrorNone) def stop_thread(self): channel_manager = ChannelManager([]) channel_manager.close_all_thread() self.set_exit_switch()
class PresenterSocketServer(): """a socket server communication with presenter agent. """ def __init__(self, server_address): """ Args: server_address: server listen address, include an ipv4 address and a port. """ # thread exit switch, if set true, thread must exit immediately. self._thread_exit_switch = False self._config = config_parser.ConfigParser() self._channel_manager = ChannelManager() self._create_socket_server(server_address) def _create_socket_server(self, server_address): """ create a socket server Args: server_address: server listen address, include an ipv4 address and a port. """ # Create a socket server. self._sock_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._sock_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self._sock_server.bind(server_address) self._sock_server.listen(SOCKET_WAIT_QUEUE) self._sock_server.setblocking(False) # Get server host name and port host, port = self._sock_server.getsockname()[:2] # Start presenter socket server thread. threading.Thread(target=self._server_listen_thread).start() # Display directly on the screen print('Presenter socket server listen on %s:%s\n' % (host, port)) def stop_thread(self): """set switch True to stop presenter socket server thread.""" self._thread_exit_switch = True def _read_socket(self, conn, read_len): ''' Read fixed length data Args: conn: a socket connection read_len: read fix byte. Returns: ret: PRESENTER_OK or PRESENTER_ERR buf: read fix byte buf. ''' has_read_len = 0 read_buf = SOCK_RECV_NULL total_buf = SOCK_RECV_NULL recv_num = 0 while has_read_len != read_len: #Troubleshoot CPU problems when Network blocking if recv_num % 10 == 5: time.sleep(0.01) try: read_buf = conn.recv(read_len - has_read_len) except socket.error: logging.error("socket %u exception:socket.error", conn.fileno()) return PRESENTER_ERR, None if read_buf == SOCK_RECV_NULL: return PRESENTER_ERR, None total_buf += read_buf has_read_len = len(total_buf) recv_num += 1 return PRESENTER_OK, total_buf def _read_msg_head(self, sock_fileno, conns): ''' Args: sock_fileno: a socket fileno conns: all socket connections which created by server. Returns: msg_total_len: total message length. msg_name_len: message name length. ''' ret, msg_head = self._read_socket(conns[sock_fileno], MSG_HEAD_LENGTH) if ret == PRESENTER_ERR: logging.error("socket %u receive msg head null", sock_fileno) return None, None # in Struct(), 'I' is unsigned int, 'B' is unsigned char msg_head_data = struct.Struct('IB') (msg_total_len, msg_name_len) = msg_head_data.unpack(msg_head) msg_total_len = socket.ntohl(msg_total_len) return msg_total_len, msg_name_len def _read_msg_name(self, sock_fd, conns, msg_name_len): ''' Args: sock_fd: a socket fileno conns: all socket connections which created by server. msg_name_len: message name length. Returns: ret: PRESENTER_OK or PRESENTER_ERR msg_name: message name. ''' ret, msg_name = self._read_socket(conns[sock_fd], msg_name_len) if ret == PRESENTER_ERR: logging.error("socket %u receive msg name null", sock_fd) return PRESENTER_ERR, None try: msg_name = msg_name.decode("utf-8") except UnicodeDecodeError: logging.error("msg name decode to utf-8 error") return PRESENTER_ERR, None return PRESENTER_OK, msg_name def _read_msg_body(self, sock_fd, conns, msg_body_len, msgs): ''' Args: sock_fd: a socket fileno conns: all socket connections which created by server. msg_name_len: message name length. msgs: msg read from a socket Returns: ret: PRESENTER_OK or PRESENTER_ERR ''' ret, msg_body = self._read_socket(conns[sock_fd], msg_body_len) if ret == PRESENTER_ERR: logging.error("socket %u receive msg body null", sock_fd) return PRESENTER_ERR msgs[sock_fd] = msg_body return PRESENTER_OK def _read_sock_and_process_msg(self, sock_fileno, conns, msgs): ''' Args: sock_fileno: a socket fileno, return value of socket.fileno() conns: all socket connections registered in epoll msgs: msg read from a socket Returns: ret: PRESENTER_OK or PRESENTER_ERR ''' # Step1: read msg head msg_total_len, msg_name_len = self._read_msg_head(sock_fileno, conns) if msg_total_len is None: return PRESENTER_ERR # Step2: read msg name ret, msg_name = self._read_msg_name(sock_fileno, conns, msg_name_len) if ret == PRESENTER_ERR: return ret # Step3: read msg body msg_body_len = msg_total_len - MSG_HEAD_LENGTH - msg_name_len if msg_body_len < 0: logging.error("msg_total_len:%u, msg_name_len:%u, msg_body_len:%u", msg_total_len, msg_name_len, msg_body_len) return PRESENTER_ERR ret = self._read_msg_body(sock_fileno, conns, msg_body_len, msgs) if ret == PRESENTER_ERR: return ret # Step4: process msg ret = self._process_msg(conns[sock_fileno], msg_name, msgs[sock_fileno]) return ret def _process_epollin(self, sock_fileno, epoll, conns, msgs): ''' Args: sock_fileno: a socket fileno, return value of socket.fileno() epoll: a set of select.epoll. conns: all socket connections registered in epoll msgs: msg read from a socket ''' msgs[sock_fileno] = b'' try: ret = self._read_sock_and_process_msg(sock_fileno, conns, msgs) if not ret: self._clean_connect(sock_fileno, epoll, conns, msgs) except socket.error: logging.error("receive socket error.") self._clean_connect(sock_fileno, epoll, conns, msgs) def _accept_new_socket(self, epoll, conns): ''' Args: epoll: a set of select.epoll. conns: all socket connections registered in epoll ''' try: new_conn, address = self._sock_server.accept() new_conn.setblocking(True) epoll.register(new_conn.fileno(), select.EPOLLIN | select.EPOLLHUP) conns[new_conn.fileno()] = new_conn logging.info("create new connection:client-ip:%s, client-port:%s", address[0], address[1]) except socket.error: logging.error("socket.error exception when sock.accept()") def _server_listen_thread(self): """socket server thread, epoll listening all the socket events""" epoll = select.epoll() epoll.register(self._sock_server.fileno(), select.EPOLLIN | select.EPOLLHUP) try: conns = {} msgs = {} while True: # thread must exit immediately if self._thread_exit_switch: break events = epoll.poll(EPOLL_TIMEOUT) # timeout, but no event come, continue waiting if not events: continue for sock_fileno, event in events: # new connection request from presenter agent if self._sock_server.fileno() == sock_fileno: self._accept_new_socket(epoll, conns) # remote connection closed # it means presenter agent exit withot close socket. elif event & select.EPOLLHUP: logging.info("receive event EPOLLHUP") self._clean_connect(sock_fileno, epoll, conns, msgs) # new data coming in a socket connection elif event & select.EPOLLIN: self._process_epollin(sock_fileno, epoll, conns, msgs) # receive event not recognize else: logging.error("not recognize event %f", event) self._clean_connect(sock_fileno, epoll, conns, msgs) finally: logging.info("conns:%s", conns) logging.info("presenter agent listen thread exit.") epoll.unregister(self._sock_server.fileno()) epoll.close() self._sock_server.close() def _clean_connect(self, sock_fileno, epoll, conns, msgs): """ close socket, and clean local variables Args: sock_fileno: a socket fileno, return value of socket.fileno() epoll: a set of select.epoll. conns: all socket connections registered in epoll msgs: msg read from a socket """ logging.info("clean sock_fd:%s, conns:%s", sock_fileno, conns) self._channel_manager.clean_channel_resource_by_fd(sock_fileno) epoll.unregister(sock_fileno) conns[sock_fileno].close() del conns[sock_fileno] del msgs[sock_fileno] def _process_msg(self, conn, msg_name, msg_data): """ Total entrance to process protobuf msg Args: conn: a socket connection msg_name: name of a msg. msg_data: msg body, serialized by protobuf Returns: False:somme error occured True:succeed """ # process open channel request if msg_name == OPEN_CHANNEL_REQUEST_FULL_NAME: ret = self._process_open_channel(conn, msg_data) # process image request, receive an image data from presenter agent elif msg_name == PRESENT_IMAGE_REQUEST_FULL_NAME: ret = self._process_image_request(conn, msg_data) # process heartbeat request, it used to keepalive a channel path elif msg_name == HEART_BEAT_MESSAGE_FULL_NAME: ret = self._process_heartbeat(conn) else: logging.error("Not recognized msg type %s", msg_name) ret = PRESENTER_ERR return ret def _response_open_channel(self, conn, channel_name, response, err_code): """ Assemble protobuf to response open_channel request Args: conn: a socket connection channel_name: name of a channel. response: a protobuf response to presenter agent err_code: part of the response Returns: ret_code:PRESENTER_OK or PRESENTER_ERR Message structure like this: -------------------------------------------------------------------- |total message len | int | 4 bytes | |------------------------------------------------------------------- |message name len | byte | 1 byte | |------------------------------------------------------------------- |message name | string | xx bytes | |------------------------------------------------------------------- |message body | protobuf | xx bytes | -------------------------------------------------------------------- protobuf structure like this: -------------------------------------------------------------------- |error_code | enum | OpenChannelErrorCode | |------------------------------------------------------------------- |error_message | string | xx bytes | |------------------------------------------------------------------- enum OpenChannelErrorCode { kOpenChannelErrorNone = 0; kOpenChannelErrorNoSuchChannel = 1; kOpenChannelErrorChannelAlreadyOpened = 2; kOpenChannelErrorOther = -1; } """ response.error_code = err_code ret_code = PRESENTER_ERR if err_code == OPEN_CHANNEL_NO_CHANNEL_ERR: response.error_message = "channel {} not exist." \ .format(channel_name) elif err_code == OPEN_CHANNEL_BUSY_ERR: response.error_message = "channel {} is busy.".format(channel_name) elif err_code == OPEN_CHANNEL_OK: response.error_message = "open channel succeed" ret_code = PRESENTER_OK else: response.error_message = "Unknown err open channel {}." \ .format(channel_name) self._send_response(conn, response, OPEN_CHANNEL_RESPONSE_FULL_NAME) return ret_code def _process_open_channel(self, conn, msg_data): """ Deserialization protobuf and process open_channel request Args: conn: a socket connection msg_data: a protobuf struct, include open channel request. Returns: protobuf structure like this: ---------------------------------------------- |channel_name | string | |---------------------------------------------- |content_type | ChannelContentType | |---------------------------------------------- enum ChannelContentType { kChannelContentTypeImage = 0; kChannelContentTypeVideo = 1; } """ request = pb.OpenChannelRequest() response = pb.OpenChannelResponse() try: request.ParseFromString(msg_data) except message.DecodeError: logging.error("ParseFromString exception: Error parsing message") channel_name = "unknown channel" return self._response_open_channel(conn, channel_name, response, OPEN_CHANNEL_UNKOWN_ERR) channel_name = request.channel_name # check channel name if exist if not self._channel_manager.is_channel_exist(channel_name): logging.error("channel name %s is not exist.", channel_name) return self._response_open_channel(conn, channel_name, response, OPEN_CHANNEL_NO_CHANNEL_ERR) # check channel path if busy if self._channel_manager.is_channel_busy(channel_name): logging.error("channel path %s is busy.", channel_name) return self._response_open_channel(conn, channel_name, response, OPEN_CHANNEL_BUSY_ERR) # if channel type is image, need clean image if exist self._channel_manager.clean_channel_image(channel_name) if request.content_type == IMAGE_TYPE: media_type = "image" elif request.content_type == VIDEO_TYPE: media_type = "video" else: logging.error("media type %s is not recognized.", request.content_type) return self._response_open_channel(conn, channel_name, response, OPEN_CHANNEL_UNKOWN_ERR) handler = ChannelHandler(channel_name, media_type) self._channel_manager.create_channel_resource(channel_name, conn.fileno(), media_type, handler) return self._response_open_channel(conn, channel_name, response, OPEN_CHANNEL_OK) def _send_response(self, conn, response, msg_name): response_data = response.SerializeToString() response_len = len(response_data) msg_name_size = len(msg_name) msg_total_size = MSG_HEAD_LENGTH + msg_name_size + response_len # in Struct(), 'I' is unsigned int, 'B' is unsigned char struct_head = struct.Struct('IB') msg_head = (socket.htonl(msg_total_size), msg_name_size) packed_msg_head = struct_head.pack(*msg_head) msg_data = packed_msg_head + \ bytes(msg_name, encoding="utf-8") + response_data conn.sendall(msg_data) def _response_image_request(self, conn, response, err_code): """ Assemble protobuf to response image_request Message structure like this: -------------------------------------------------------------------- |total message len | int | 4 bytes | |------------------------------------------------------------------- |message name len | byte | 1 byte | |------------------------------------------------------------------- |message name | string | xx bytes | |------------------------------------------------------------------- |message body | protobuf | xx bytes | -------------------------------------------------------------------- protobuf structure like this: -------------------------------------------------------------------- |error_code | enum | PresentDataErrorCode | |------------------------------------------------------------------- |error_message | string | xx bytes | |------------------------------------------------------------------- enum PresentDataErrorCode { kPresentDataErrorNone = 0; kPresentDataErrorUnsupportedType = 1; kPresentDataErrorUnsupportedFormat = 2; kPresentDataErrorOther = -1; } """ response.error_code = err_code ret_code = PRESENTER_OK if err_code == PRESENT_DATA_UNSUPPORTED_FORMAT_ERR: response.error_message = "Present data not support format." logging.error("Present data not support format.") ret_code = PRESENTER_ERR elif err_code == PRESENT_DATA_OK: response.error_message = "Present data ok" ret_code = PRESENTER_OK else: response.error_message = "Present data not known error." logging.error("Present data not known error.") ret_code = PRESENTER_ERR self._send_response(conn, response, PRESENT_IMAGE_RESPONSE_FULL_NAME) return ret_code def _process_image_request(self, conn, msg_data): """ Deserialization protobuf and process image_request Args: conn: a socket connection msg_data: a protobuf struct, include image request. Returns: protobuf structure like this: ------------------------------------ |format | ImageFormat | |------------------------------------ |width | uint32 | |------------------------------------ |height | uint32 | |------------------------------------ |data | bytes | ------------------------------------ enum ImageFormat { kImageFormatJpeg = 0; } """ request = pb.PresentImageRequest() response = pb.PresentImageResponse() # Parse msg_data from protobuf try: request.ParseFromString(msg_data) except message.DecodeError: logging.error("ParseFromString exception: Error parsing message") err_code = PRESENT_DATA_UNKOWN_ERR return self._response_image_request(conn, response, err_code) sock_fileno = conn.fileno() handler = self._channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is None: logging.error("get channel handler failed") err_code = PRESENT_DATA_UNKOWN_ERR return self._response_image_request(conn, response, err_code) # Currently, image format only support jpeg if request.format != FORMAT_JPEG: logging.error("image format %s not support", request.format) err_code = PRESENT_DATA_UNSUPPORTED_FORMAT_ERR return self._response_image_request(conn, response, err_code) handler.save_image(request.data, request.width, request.height) return self._response_image_request(conn, response, PRESENT_DATA_OK) def _process_heartbeat(self, conn): ''' set heartbeat Args: conn: a socket connection Returns: True: set heartbeat ok. ''' sock_fileno = conn.fileno() handler = self._channel_manager.get_channel_handler_by_fd(sock_fileno) if handler is not None: handler.setheartbeat() return PRESENTER_OK