def __init__(self, connection, publish_snapshot_function):
super().__init__()
self._is_valid_connection = True
# Network
self.connection = connection
self.files_handler = _FileHandler()
# Protocol
self.supported_fields = DEFAULT_SUPPORTED_FIELDS
self.protocol = Protocol()
# Messages
self.messages = MessageQueueMessages()
# Publish functions
self.publish_snapshot_function = publish_snapshot_function
def __init__(self,
database_type=None,
database_host=None,
database_port=None):
# DataBase
self._database = _DataBaseCortex(database_type, database_host,
database_port)
# Messages
self.messages = MessageQueueMessages()
# Save methods
self.SAVE_METHODS = \
{ \
'pose' : self.save_parsed_pose, \
'user_feelings' : self.save_user_feelings, \
'color_image' : self.save_color_image, \
'depth_image' : self.save_depth_image, \
}
def snapshot_publish(message):
snapshot = MessageQueueMessages().get_message(
MessageQueueMessagesTyeps.RAW_SNAPSHOT_MESSAGE).deserialize(
message)
user_id = snapshot.user_info.user_id
test_server_snapshot_published_ids.append(user_id)
# Update shared-memory published snapshots counter
test_server_snapshot_published_counter.value += 1
def __init__(self,
parser_type,
message_queue_type=None,
message_queue_host=None,
message_queue_port=None):
"""
Initializes a new parser of given type, which consumes, parses and published to a messages-queue.
:param parser_type: The parser type.
:param message_queue_type: Message queue type, if empty - default will be selected.
:param message_queue_host: Message queue host, if empty - default will be selected.
:param message_queue_port: Message queue port, if empty - default will be selected.
"""
self.parser_type = parser_type
self.parser_name = ParserService.get_parser_name(parser_type)
self.parser_handler = ParserHandler(parser_type)
self.initialized = self.parser_handler.initialized
# Message Queue
self.message_queue_type = message_queue_type
self.message_queue_host = message_queue_host
self.message_queue_port = message_queue_port
# Messages
self.messages = MessageQueueMessages()
class ParserService:
"""
Server Parser class, runs the micro-service logic
:ivar ParserService.SERVICE_TYPE: name of the service
"""
SERVICE_TYPE = 'parser'
@staticmethod
def get_parser_name(parser_type):
"""
Returns parser name.
:param parser_type: The parser type.
"""
return f'{parser_type}.parser'
def __init__(self,
parser_type,
message_queue_type=None,
message_queue_host=None,
message_queue_port=None):
"""
Initializes a new parser of given type, which consumes, parses and published to a messages-queue.
:param parser_type: The parser type.
:param message_queue_type: Message queue type, if empty - default will be selected.
:param message_queue_host: Message queue host, if empty - default will be selected.
:param message_queue_port: Message queue port, if empty - default will be selected.
"""
self.parser_type = parser_type
self.parser_name = ParserService.get_parser_name(parser_type)
self.parser_handler = ParserHandler(parser_type)
self.initialized = self.parser_handler.initialized
# Message Queue
self.message_queue_type = message_queue_type
self.message_queue_host = message_queue_host
self.message_queue_port = message_queue_port
# Messages
self.messages = MessageQueueMessages()
# Snapshot Methods Section
def desrialize_raw_snapshot_message(self, incoming_snapshot_message):
"""
Desrializes a raw snapshot message, as received from the message-queue.
:param parser_type: The parser type.
:param message_queue_type: Message queue type, if empty - default will be selected.
:param message_queue_host: Message queue host, if empty - default will be selected.
:param message_queue_port: Message queue port, if empty - default will be selected.
"""
raw_snapshot_message = \
self.messages.get_message( \
MessageQueueMessagesTyeps.RAW_SNAPSHOT_MESSAGE).deserialize(incoming_snapshot_message)
return raw_snapshot_message
def _get_context(self, raw_snapshot_message):
"""
Gets `ParserContext` from a raw snapshot message.
:param raw_snapshot_message: The raw snapshot message.
"""
return ParserContext(raw_snapshot_message.user_info,
raw_snapshot_message.snapshot_uuid,
self.parser_type)
# Parse Message Section
def parse_message(self, incoming_snapshot_message):
"""
Parses an incoming snapshot message.
:param incoming_snapshot_message: The incoming snapshot message to parse.
:return: If error occurs during parsing, None would be returned, If parsed successfully, a parsed snapshot message would be returned.
"""
try:
raw_snapshot_message = self.desrialize_raw_snapshot_message(
incoming_snapshot_message)
except Exception as e:
logger.error(f'error deserializing raw snapshot message : {e}')
return None
context = self._get_context(raw_snapshot_message)
export_result = self.parser_handler.export_parse(
context, raw_snapshot_message)
if not export_result:
logger.info(f'{self.parser_name} Failed to Parse message')
return None
is_uri, result, snapshot, metadata = export_result
parsed_snapshot_message = \
self.messages.get_message( \
MessageQueueMessagesTyeps.PARSED_SNAPSHOT_MESSAGE)( \
context.user_info, \
context.snapshot_uuid, \
snapshot.timestamp, \
self.parser_type, \
result, \
is_uri, \
metadata)
logger.info(f'{self.parser_name} Parsed message')
return parsed_snapshot_message
# Generates parse callback with custom arguments - by this currying function
def publish_parsed_callback(self):
"""
Decorator for callback used to parse and publish incoming snapshot message.
"""
def parse_and_publish(incoming_snapshot_message):
logger.info(f'{self.parser_name} Received message')
parsed_snapshot_message = self.parse_message(
incoming_snapshot_message)
if parsed_snapshot_message:
self.publish_function(parsed_snapshot_message.serialize(),
publisher_name=self.parser_name)
logger.info(f'{self.parser_name} Sent message')
return parse_and_publish
# Core Logic Method Section
def run(self):
"""
Runs the parser micro-service logic, consuming messages from the message queue, parsing them and publishing the results to the message queue.
"""
if not self.initialized:
logger.error(ERROR_DURING_INITIALIZATION_CANT_RUN_ERROR_MESSAGE)
return
mq_context_factory = MessageQueueContextFactory(
self.message_queue_type)
self.register_publish_parsed(mq_context_factory)
self.consume_messages(mq_context_factory)
# Message Queue Methods Section
# Consume snapshot messages
def consume_messages(self, mq_context_factory):
"""
Consumes messages from the message-queue, and calling the parse callbakc to parse them.
:param mq_context_factory: The context factory used to create context for the consumer.
"""
message_queue_context = \
mq_context_factory.get_mq_context(ParserService.SERVICE_TYPE, 'consumers', 'snapshots')
message_queue_consumer = \
MessageQueueConsumer( \
self.publish_parsed_callback(), \
message_queue_context, \
self.message_queue_type, \
self.message_queue_host, \
self.message_queue_port, \
)
message_queue_consumer.run()
# Publish parsed
def register_publish_parsed(self, mq_context_factory):
"""
Registers publisher to the message-queue, to be called when publishing to the message-queue is needed.
:param mq_context_factory: The context factory used to create context for the publisher.
"""
message_queue_context = \
mq_context_factory.get_mq_context(ParserService.SERVICE_TYPE, 'publishers', 'parsed_snapshot', name=self.parser_name)
self.message_queue_publisher = MessageQueuePublisherThread(
message_queue_context)
self.publish_function = self.message_queue_publisher.get_publish_function(
)
self.message_queue_publisher.run()
class SaverMessagesHandler:
DEFAULT_ENCODING = 'utf-8'
def __init__(self,
database_type=None,
database_host=None,
database_port=None):
# DataBase
self._database = _DataBaseCortex(database_type, database_host,
database_port)
# Messages
self.messages = MessageQueueMessages()
# Save methods
self.SAVE_METHODS = \
{ \
'pose' : self.save_parsed_pose, \
'user_feelings' : self.save_user_feelings, \
'color_image' : self.save_color_image, \
'depth_image' : self.save_depth_image, \
}
# Message Handling Methods
def handle(self, message):
message = message if isinstance(message, str) else message.decode(
SaverMessagesHandler.DEFAULT_ENCODING)
parsed_snapshot_message = self.messages.get_message( \
MessageQueueMessagesTyeps.PARSED_SNAPSHOT_MESSAGE).deserialize(message)
# Fetch user id and path
user_info = parsed_snapshot_message.user_info
self.save_user(user_info)
self.save_snapshot(parsed_snapshot_message.snapshot_uuid,
user_info.user_id,
parsed_snapshot_message.snapshot_timestamp)
# Fetch parsed field
field = parsed_snapshot_message.field
snapshot_uuid = parsed_snapshot_message.snapshot_uuid
result = parsed_snapshot_message.result
is_uri = parsed_snapshot_message.is_uri
metadata = parsed_snapshot_message.metadata
self.save_parsed(field, snapshot_uuid, result, is_uri, metadata)
return snapshot_uuid
# Save Methods
def save_parsed(self, field, snapshot_uuid, result, is_uri, metadata):
if field not in self.SAVE_METHODS.keys():
logger.error(f'saving field {field} is unknown')
return
self.SAVE_METHODS[field](snapshot_uuid, result, is_uri, metadata)
# User Saving Method
def save_user(self, user_info):
if self._database.has_user(user_id=user_info.user_id):
return
user_id = user_info.user_id
username = user_info.username
birth_date = TimeUtils.timestamp_to_dateime(user_info.birth_date)
gender = user_info.gender
creation_status = \
self._database.create_user( \
user_id = user_id, \
username = username, \
birth_date = birth_date, \
gender = gender, \
)
if creation_status:
logger.info(f'user {user_id} added successfully!')
else:
logger.error(f'failed to add user {user_id}')
# Snapshot Saving Method
def save_snapshot(self, snapshot_uuid, user_id, snapshot_timestamp):
if self._database.has_snapshot(snapshot_uuid=snapshot_uuid):
return
creation_status = \
self._database.create_snapshot( \
snapshot_uuid = snapshot_uuid, \
user_id = user_id, \
timestamp = snapshot_timestamp \
)
if creation_status:
logger.info(f'snapshot {snapshot_uuid} added successfully!')
else:
logger.error(f'failed to add snapshot {snapshot_uuid}')
# Fields Saving Methods
def save_parsed_pose(self, snapshot_uuid, result, is_uri, metadata):
snapshot_uuid = snapshot_uuid
pose = json_to_object(result)
creation_status = \
self._database.create_pose( \
snapshot_uuid = snapshot_uuid, \
translation_x = pose.translation.x, \
translation_y = pose.translation.y, \
translation_z = pose.translation.z, \
rotation_x = pose.rotation.x, \
rotation_y = pose.rotation.y, \
rotation_z = pose.rotation.z, \
rotation_w = pose.rotation.w, \
)
if creation_status:
logger.info(
f'pose of snapshot {snapshot_uuid} added successfully!')
else:
logger.error(f'failed to add pose of snapshot {snapshot_uuid}')
def save_user_feelings(self, snapshot_uuid, result, is_uri, metadata):
snapshot_uuid = snapshot_uuid
user_feelings = json_to_object(result)
creation_status = \
self._database.create_user_feelings( \
snapshot_uuid = snapshot_uuid, \
hunger = user_feelings.hunger, \
thirst = user_feelings.thirst, \
exhaustion = user_feelings.exhaustion, \
happiness = user_feelings.happiness, \
)
if creation_status:
logger.info(
f'user_feelings of snapshot {snapshot_uuid} added successfully!'
)
else:
logger.error(
f'failed to add user_feelings of snapshot {snapshot_uuid}')
def save_color_image(self, snapshot_uuid, result, is_uri, metadata):
snapshot_uuid = snapshot_uuid
uri = expand_file_path_relative_to_project_root(result)
width, height = metadata
creation_status = \
self._database.create_color_image( \
snapshot_uuid = snapshot_uuid, \
uri = uri, \
width = width, \
height = height, \
)
if creation_status:
logger.info(
f'color_image of snapshot {snapshot_uuid} added successfully!')
else:
logger.error(
f'failed to add color_image of snapshot {snapshot_uuid}')
def save_depth_image(self, snapshot_uuid, result, is_uri, metadata):
snapshot_uuid = snapshot_uuid
uri = expand_file_path_relative_to_project_root(result)
width, height = metadata
creation_status = \
self._database.create_depth_image( \
snapshot_uuid = snapshot_uuid, \
uri = uri, \
width = width, \
height = height, \
)
if creation_status:
logger.info(
f'depth_image of snapshot {snapshot_uuid} added successfully!')
else:
logger.error(
f'failed to add depth_image of snapshot {snapshot_uuid}')
class ServerHandler(threading.Thread):
# Constructor Section
def __init__(self, connection, publish_snapshot_function):
super().__init__()
self._is_valid_connection = True
# Network
self.connection = connection
self.files_handler = _FileHandler()
# Protocol
self.supported_fields = DEFAULT_SUPPORTED_FIELDS
self.protocol = Protocol()
# Messages
self.messages = MessageQueueMessages()
# Publish functions
self.publish_snapshot_function = publish_snapshot_function
# Methods Section
# Protocol Methods Section
# Receives hello message from client
def receive_hello_message(self):
try:
hello_message_bytes = self.connection.receive_message()
except Exception as e:
logger.error(f'error receiving hello_message : {e}')
self._is_valid_connection = False
return None
try:
hello_message = self.protocol.get_message(
ProtocolMessagesTyeps.HELLO_MESSAGE).read(hello_message_bytes)
except Exception as e:
logger.error(f'error parsing hello_message : {e}')
self._is_valid_connection = False
return None
return hello_message
# Sends configuration message to client
def send_config_message(self):
config_message = self.protocol.get_message(
ProtocolMessagesTyeps.CONFIG_MESSAGE)(self.supported_fields)
try:
self.connection.send_message(config_message.serialize())
except Exception as e:
logger.error(
f'error sending config_message to user {self.context.user_info.user_id}: {e}'
)
self._is_valid_connection = False
return
# Receives snapshot message from client
def receive_snapshot_message_bytes(self):
try:
snapshot_message_bytes = self.connection.receive_message()
except EOFError as e:
raise e
except Exception as e:
logger.error(
f'error receiving snapshot_message of user {self.context.user_info.user_id}: {e}'
)
self._is_valid_connection = False
raise EOFError
return snapshot_message_bytes
# UUID Methods Section
def _get_uuid(self):
return generate_uuid()
def _get_hash(self, data):
return get_data_hash(data)
# File Handling Methods Section
# Generate path to save file
@staticmethod
def _get_save_path(*pathsegments, fname=None, extension=None):
return _FileHandler.to_safe_file_path(*pathsegments,
fname=fname,
extension=extension)
# Save file
def _save_file(self, path, data):
return self.files_handler.save(path, data)
# Messages Handling Methods Section
# Sets context
def _set_context(self, hello_message):
try:
self.context = UserContext(hello_message.user_info)
except Exception as e:
logger.error(f'error parsing hello_message : {e}')
self._is_valid_connection = False
@staticmethod
def get_user_snapshots_path(user_id):
return ServerHandler._get_save_path(
ConstantPathes.get_snapshots_path(), user_id)
# Saves snapshot
def _get_snapshot_save_path(self, snapshot_uuid):
snapshot_path = \
ServerHandler._get_save_path( \
ServerHandler.get_user_snapshots_path(self.context.user_info.user_id), \
snapshot_uuid, \
SNAPSHOT_FILE_NAME)
return snapshot_path
def _save_snapshot(self, snapshot_uuid, snapshot_message_bytes):
snapshot_path = self._get_snapshot_save_path(snapshot_uuid)
# Deduping snapshots (to avoid multiple saving and handling of existing snapshots)
if self.files_handler.is_file_exists(snapshot_path):
return None
# Save snapshot
is_saved = self._save_file(snapshot_path, snapshot_message_bytes)
if not is_saved:
logger.error(
f'error saving snapshot of user {self.context.user_info.user_id}'
)
return None
return snapshot_path
# Publish snapshot message
def publish_snapshot_message(self, snapshot_uuid, raw_snapshot_path):
# Creating raw snapshot message
raw_snapshot_message = \
self.messages.get_message( \
MessageQueueMessagesTyeps.RAW_SNAPSHOT_MESSAGE)( \
self.context.user_info, \
snapshot_uuid, \
raw_snapshot_path)
# Publish raw snapshot message
self.publish_snapshot_function(raw_snapshot_message.serialize())
# Handle snapshot message
def handle_snapshot_message(self, snapshot_message_bytes):
# Gets snapshot uuid
snapshot_uuid = self._get_hash(snapshot_message_bytes)
# Saving snapshot
raw_snapshot_path = self._save_snapshot(snapshot_uuid,
snapshot_message_bytes)
# If error occurred while saving snapshot
if not raw_snapshot_path:
return
# Publishing snapshot message
self.publish_snapshot_message(snapshot_uuid, raw_snapshot_path)
# Core Logic Methods Section
# Runs client handling loop
def run(self):
# Receives hello_message from client
hello_message = self.receive_hello_message()
if not self._is_valid_connection:
return
# Sets client context based on hello message
self._set_context(hello_message)
if not self._is_valid_connection:
return
# Sends configuration message to client
self.send_config_message()
if not self._is_valid_connection:
return
logger.info(
f'start receiving snapshots from user {self.context.user_info.user_id}'
)
# As long as connection is valid
while self._is_valid_connection:
try:
# Receives snapshot_message from client
snapshot_message_bytes = self.receive_snapshot_message_bytes()
# Handle snapshot message
self.handle_snapshot_message(snapshot_message_bytes)
except EOFError:
logger.info(
f'finished snapshots from user {self.context.user_info.user_id}'
)
break