def pending_demands(event: ThinkerEvent):
thinker: Thinker = event.parent
demands_waiting_reply: MsgDict = thinker.demands_waiting_reply
info_msg(event,
'STARTED',
extra=f' of {thinker.name} with tick {event.tick}')
while event.active:
sleep(0.001)
if not event.paused and demands_waiting_reply:
try:
sleep(event.tick)
for key, item in demands_waiting_reply.items():
pending: PendingReply = item
if (time() -
event.time) > event.tick and pending.attempt < 3:
pending.attempt += 1
info_msg(
event, 'INFO',
f'Msg {pending.message.id}, com {pending.message.com} waits '
f'{pending.attempt}.')
elif (time() -
event.time) > event.tick and pending.attempt >= 3:
try:
msg = pending.message
del demands_waiting_reply[key]
info_msg(event, 'INFO',
f'Reply timeout for msg: {msg.short()}')
info_msg(event, 'INFO', f'Msg {msg.id} is deleted')
except KeyError:
error_logger(
event, pending_demands,
f'Cannot delete Msg {msg.id}, com {msg.com} from '
f'demand_waiting_reply')
except ThinkerErrorReact as e:
error_logger(event, pending_demands, e)
def subscribe_sub(self, address=None, filter_opt=b""):
try:
self.sockets['sub'].connect(address)
self.sockets['sub'].setsockopt(zmq.SUBSCRIBE, filter_opt)
self.sockets['sub'].setsockopt(zmq.RCVHWM, 3)
except (zmq.ZMQError, Exception) as e:
error_logger(self, self.subscribe_sub, e)
def _create_sockets(self):
try:
self.context = zmq.Context()
frontend = self.context.socket(zmq.ROUTER)
frontend.bind(self.addresses[FRONTEND_Server])
backend = self.context.socket(zmq.ROUTER)
backend.bind(self.addresses[BACKEND_Server])
publisher = self.context.socket(zmq.PUB)
publisher.bind(self.addresses[PUB_Socket_Server])
# SOCKET SUBSCRIBER
sub = self.context.socket(zmq.SUB)
self.poller = zmq.Poller()
self.poller.register(frontend, zmq.POLLIN)
self.poller.register(backend, zmq.POLLIN)
self.poller.register(sub, zmq.POLLIN)
self.sockets = {
FRONTEND_Server: frontend,
BACKEND_Server: backend,
PUB_Socket_Server: publisher,
SUB_Socket: sub
}
self.public_sockets = {
PUB_Socket_Server: self.addresses[PUB_Socket_Server],
FRONTEND_Server: self.addresses[FRONTEND_Server],
BACKEND_Server: self.addresses[BACKEND_Server]
}
except (WrongAddress, KeyError, zmq.ZMQError) as e:
error_logger(self, self._create_sockets, e)
raise e
def task_out_reaction(event: ThinkerEvent):
thinker: Thinker = event.parent
tasks_out: MsgDict = thinker.tasks_out
demand_waiting_reply: MsgDict = thinker.demands_waiting_reply
info_msg(event,
'STARTED',
extra=f' of {thinker.name} with tick {event.tick}')
while event.active:
sleep(0.001)
if not event.paused and tasks_out:
try:
msg: MessageExt = tasks_out.popitem(last=False)[1]
react = True
if msg.receiver_id != '' and msg.reply_to == '':
# If msg is not reply, than add to pending demand
info_msg(
event, 'INFO',
f'Msg id={msg.id}, com {msg.com} is considered to get a reply'
)
thinker.add_demand_waiting_reply(msg)
elif msg.reply_to != '':
if msg.reply_to in demand_waiting_reply:
msg_awaited: MessageExt = thinker.demands_waiting_reply[
msg.reply_to].message
del demand_waiting_reply[msg.reply_to]
info_msg(
event, 'INFO',
f'Msg id={msg.reply_to} {msg_awaited.com} is deleted from '
f'demand_waiting_reply')
if react:
thinker.parent.messenger.add_msg_out(msg)
except (ThinkerErrorReact, KeyError) as e:
error_logger(event, task_out_reaction, f'{e}: {msg.short()}')
def _create_sockets(self):
try:
self.context = zmq.Context()
# SOCKET DEALER
dealer = self.context.socket(zmq.DEALER)
dealer.setsockopt_unicode(zmq.IDENTITY, self.id)
# SOCKET SUBSCRIBER
subscriber = self.context.socket(zmq.SUB)
subscriber.setsockopt(zmq.RCVHWM, 10)
self.sockets = {DEALER_Socket: dealer, SUB_Socket: subscriber}
# SOCKET PUBLISHER
if self.pub_option:
publisher = self.context.socket(zmq.PUB)
publisher.setsockopt_unicode(zmq.IDENTITY,
self.addresses[PUB_Socket])
self.sockets[PUB_Socket] = publisher
self.public_sockets = {PUB_Socket: self.addresses[PUB_Socket]}
# POLLER
self.poller = zmq.Poller()
self.poller.register(dealer, zmq.POLLIN)
self.poller.register(subscriber, zmq.POLLIN)
except (WrongAddress, KeyError, zmq.ZMQError) as e:
error_logger(self, self._create_sockets, e)
raise e
def send_msg(self, msg: MessageExt):
try:
crypted = str(int(msg.crypted)).encode('utf-8')
msg_bytes = self.encrypt_with_session_key(msg)
if msg.receiver_id == '':
self.sockets[PUB_Socket_Server].send_multipart(
[msg_bytes, crypted])
else:
if msg.receiver_id in self._frontendpool:
self.sockets[FRONTEND_Server].send_multipart(
[msg.receiver_id.encode('utf-8'), msg_bytes, crypted])
info_msg(
self, 'INFO',
f'Msg {msg.id}, com {msg.com} is send from frontend to {msg.receiver_id}.'
)
elif msg.receiver_id in self._backendpool:
self.sockets[BACKEND_Server].send_multipart(
[msg.receiver_id.encode('utf-8'), msg_bytes, crypted])
info_msg(
self, 'INFO',
f'Msg {msg.id}, com {msg.com} is send from backend to {msg.receiver_id}.'
)
else:
error_logger(
self, self.send_msg,
f'ReceiverID {msg.receiver_id} is not present in Server pool.'
)
except zmq.ZMQError as e:
error_logger(self, self.send_msg, e)
def execute_com(self, msg: MessageExt):
error = False
com: str = msg.info.com
input: FuncInput = msg.info
if com in self.available_public_functions_names:
f = getattr(self, com)
func_input_type = signature(f).parameters['func_input'].annotation
if func_input_type == type(input):
try:
result: FuncOutput = f(input)
msg_r = self.generate_msg(msg_com=MsgComExt.DONE_IT,
receiver_id=msg.sender_id,
func_output=result,
reply_to=msg.id)
except Exception as e: # TODO: replace Exception
error_logger(self, self.execute_com, e)
error = True
comments = str(e)
else:
error = True
comments = f'Device {self.id} function: execute_com: Input type: {type(input)} do not match to ' \
f'func_input type : {func_input_type}'
else:
error = True
comments = f'com: {com} is not available for Service {self.id}. See {self.available_public_functions()}'
if error:
msg_r = self.generate_msg(msg_com=MsgComExt.ERROR,
comments=comments,
receiver_id=msg.sender_id,
reply_to=msg.id)
self.send_msg_externally(msg_r)
def __init__(self, parent):
from devices.devices import Device
Thinker.n_instance += 1
self.logger = logging.getLogger(__name__ + '.' +
self.__class__.__name__)
self.name = f'{self.__class__.__name__}:{parent.name}:{Thinker.n_instance}'
self.id = f'{self.name}:{unique_id(self.name)}'
self.parent: Device = parent
self.msg_counter = 0
self.events = Events_Dict()
msg_dict_size_limit = 10000
self._tasks_in = MsgDict(name='tasks_in',
size_limit=msg_dict_size_limit,
dict_parent=self)
self.tasks_in_test = MsgDict(name='tasks_in_test',
size_limit=msg_dict_size_limit,
dict_parent=self)
self._tasks_out = MsgDict(name='tasks_out',
size_limit=msg_dict_size_limit,
dict_parent=self)
self.tasks_out_test = MsgDict(name='tasks_out_test',
size_limit=msg_dict_size_limit,
dict_parent=self)
self._demands_waiting_reply = MsgDict(name='demands_waiting_reply',
size_limit=msg_dict_size_limit,
dict_parent=self)
# TODO: add slow thread to track after forwarded messages
self._forwarded = MsgDict(name='forwarded_messages',
size_limit=msg_dict_size_limit,
dict_parent=self)
self.paused = False
info_msg(self, 'CREATING')
try:
self.timeout = int(self.parent.get_general_settings()['timeout'])
pending_demands_tick = float(
self.parent.get_general_settings()['pending_demands']) / 1000.
except KeyError as e:
error_logger(self, self.__init__, e)
self.timeout = 10
pending_demands_tick = 0.2
try:
from communication.logic.logic_functions import (task_in_reaction,
task_out_reaction,
pending_demands)
self.register_event(name='task_in_reaction',
logic_func=task_in_reaction,
tick=None)
self.register_event(name='task_out_reaction',
logic_func=task_out_reaction,
tick=None)
self.register_event(name='demands_waiting_reply',
logic_func=pending_demands,
tick=pending_demands_tick)
info_msg(self, 'CREATED')
except (ThinkerEventError, ThinkerEventFuncError, TypeError) as e:
error_logger(self, self.register_event, e)
info_msg(self, 'NOT CREATED')
raise ThinkerError(str(e))
def add_task_out(self, msg: MessageExt):
try:
self._tasks_out[msg.id] = msg
if self.parent.test and not (msg.com
== MsgComExt.HEARTBEAT.msg_name):
self.tasks_out_test[msg.id] = msg
except KeyError as e:
error_logger(self, self.add_task_out, e)
def connect(self):
try:
self.sockets[DEALER_Socket].connect(
self.addresses[FRONTEND_Server])
if self.pub_option:
self.bind_pub()
except (WrongAddress, zmq.error.ZMQBindError) as e:
error_logger(self, self.connect, e)
def subscribe_sub(self, address=None, filter_opt=b""):
try:
self.sockets[SUB_Socket].connect(address)
self.sockets[SUB_Socket].setsockopt(zmq.SUBSCRIBE, filter_opt)
self.sockets[SUB_Socket].setsockopt(zmq.RCVHWM, 3)
self.logger.info(f'socket sub is connected to {address}')
except (zmq.ZMQError, Exception) as e:
error_logger(self, self.subscribe_sub, e)
def __setitem__(self, key, value):
if key not in self:
self._check_size_limit()
super().__setitem__(key, value)
else:
error = f'Key: {key} already exists in {self.name} {self}'
if self.dict_parent:
error_logger(self.dict_parent, self, error)
raise KeyError(error)
def run(self):
self.active = True
self.paused = self.parent.paused
try:
self.time = time()
info_msg(self, 'STARTING', extra=f' of {self.parent.name}')
self.logic_func(self)
except Exception as e:
error_logger(self, self.run, f'{self.name}. Error: {e}')
finally:
info_msg(self, 'STOPPED', extra=f' of {self.parent.name}')
def react_broadcast(self, msg: MessageExt):
if msg.com == MsgComExt.HEARTBEAT.msg_name:
try:
self.events[msg.info.event_id].time = time()
self.events[msg.info.event_id].n = msg.info.event_n
if self.parent.pyqtsignal_connected:
self.parent.signal.emit(msg.ext_to_int())
except (KeyError, TypeError) as e:
error_logger(self, self.react_broadcast, e)
elif msg.com == MsgComExt.SHUTDOWN.msg_name: # When one of devices shutdowns
self.remove_device_from_connections(msg.sender_id)
self.parent.send_status_pyqt()
def msg_out(self, msg_out: Union[MessageExt, List[MessageExt]]):
if msg_out:
if isinstance(msg_out, list):
for msg in msg_out:
self.msg_out(msg)
elif isinstance(msg_out, MessageExt):
info_msg(self, 'INFO', msg_out.short())
self.add_task_out(msg_out)
else:
error_logger(
self, self.msg_out,
f'Union[MessageExt, List[MessageExt]] was not passed to msg_out, but'
f'{msg_out}')
def bind_pub(self):
try:
self.sockets[PUB_Socket].setsockopt_unicode(
zmq.IDENTITY, self.addresses[PUB_Socket])
self.sockets[PUB_Socket].bind(self.addresses[PUB_Socket])
except (WrongAddress, zmq.error.ZMQError) as e:
# TODO: potential recursion depth violation
error_logger(self, self.connect,
f'{e}: {self.addresses[PUB_Socket]}')
port = get_free_port(scope=None)
local_ip = get_local_ip()
self.addresses[PUB_Socket] = f'tcp://{local_ip}:{port}'
self.public_sockets = {PUB_Socket: self.addresses[PUB_Socket]}
self.bind_pub()
def _deal_with_reaceived_msg(self, msgs: List[MsgTuple]):
for msg, device_id, socket_name, crypted in msgs:
try:
if int(crypted):
msg: bytes = self.decrypt_with_session_key(msg, device_id)
mes: MessageExt = MessageExt.bytes_to_msg(msg)
self.parent.thinker.add_task_in(mes)
except (MessengerError, MessageError, ThinkerError,
Exception) as e:
error_logger(self, self.run, e)
msg_r = self.parent.generate_msg(msg_com=MsgComExt.ERROR,
error_comments=str(e),
reply_to='',
receiver_id=device_id)
self.add_msg_out(msg_r)
def run(self):
super().run()
try:
info_msg(self, 'STARTED')
self._receive_msgs()
except zmq.error.ZMQError as e: # Bad kind of error!
error_logger(self, self.run, e)
self.stop()
finally:
for _, soc in self.sockets.items():
soc.close()
self.context.destroy()
self.active = False
self.paused = True
info_msg(self, 'STOPPED')
def react_internal(self, event: ThinkerEvent):
if 'heartbeat' in event.name:
if event.counter_timeout > 5:
try:
connections = self.parent.connections
self.logger.info(f"""{connections[event.original_owner].device_info.type} {event.name}
was away for too long...removing it from active services, deleting its tasks""")
self.unregister_event(event.id)
del connections[event.original_owner]
# TODO: tasks should be deleted here
except KeyError as e:
error_logger(self, self.react_internal, e)
self.unregister_event(event.id)
self.parent.send_status_pyqt(com='status_server_info_full')
else:
self.logger.info(f'react_internal: I do not know what to do, {event.name} is not known')
def remove_device_from_connections(self, device_id):
# TODO: the service_info is not deleted from _frontend sockets or backend sockets
connections = self.parent.connections
if device_id in connections:
info_msg(self, 'INFO',
f'Procedure to delete {device_id} is started')
for key, event in list(self.events.items()):
if event.original_owner == device_id:
self.unregister_event(key)
del self.parent.connections[device_id]
info_msg(self, 'INFO', f'Device {device_id} is deleted')
else:
error_logger(
self, self.remove_device_from_connections,
f'remove_device_from_connections: Wrong device_id {device_id} is passed'
)
def task_in_reaction(event: ThinkerEvent):
thinker: Thinker = event.parent
tasks_in: MsgDict = thinker.tasks_in
info_msg(event,
'STARTED',
extra=f' of {thinker.name} with tick {event.tick}')
exclude_msgs = [
MsgComExt.HEARTBEAT.msg_name, MsgComExt.HEARTBEAT_FULL.msg_name
]
while event.active:
sleep(
0.001
) # Any small interruption is necessary not to overuse processor time
if not event.paused and tasks_in:
try:
msg: MessageExt = tasks_in.popitem(last=False)[1]
thinker.msg_counter += 1
react = True
if msg.com not in exclude_msgs:
info_msg(event, 'INFO', f'Received: {msg.short()}')
if msg.reply_to == '' and msg.receiver_id != '': # If message is not a reply, it must be a demand one
thinker.add_demand_waiting_reply(msg)
info_msg(
event, 'INFO',
f'Expect a reply to {msg.id} com={msg.com}. Adding to waiting list.'
)
elif msg.reply_to != '':
if msg.reply_to in thinker.demands_waiting_reply:
# TODO: should it have else clause or not?
msg_awaited: MessageExt = thinker.demands_waiting_reply[
msg.reply_to].message
del thinker.demands_waiting_reply[msg.reply_to]
info_msg(
event, 'INFO',
f'REPLY to Msg {msg.reply_to} {msg_awaited.com} is obtained.'
)
else:
react = False
info_msg(
event, 'INFO',
f'Reply to msg {msg.reply_to} arrived too late.')
if react:
thinker.react_external(msg)
except (ThinkerErrorReact, KeyError) as e:
error_logger(event, task_in_reaction, f'{e}: {msg.short()}')
def register_event(self,
name: str,
logic_func: Callable,
event_id='',
external_name='',
original_owner='',
start_now=False,
**kwargs):
# TODO: to complicated. Need to optimize.
try:
if 'tick' in kwargs:
tick = kwargs['tick']
else:
tick = float(self.parent.get_general_settings()[name.split(':')
[0]]) / 1000
except KeyError as e:
error_logger(self, self.register_event,
f'{e}. Tick value is set to {tick}s')
finally:
print_every_n = int(
self.parent.get_general_settings()['print_every_n'])
try:
if not external_name:
external_name = name
if not event_id:
event_id = f'{external_name}:{self.parent.id}'
if original_owner == '':
original_owner = self.parent.id
self.events[event_id] = ThinkerEvent(
name=name,
external_name=external_name,
parent=self,
logic_func=logic_func,
tick=tick,
print_every_n=print_every_n,
event_id=event_id,
original_owner=original_owner)
if start_now:
self.events[event_id].start()
except ThinkerEventFuncError as e:
raise ThinkerEventError(str(e))
def send_msg(self, msg: MessageExt):
try:
crypted = str(int(msg.crypted)).encode('utf-8')
msg_bytes = self.encrypt_with_session_key(msg)
if msg.receiver_id != '':
self.sockets[DEALER_Socket].send_multipart(
[msg_bytes, crypted])
info_msg(
self, 'INFO',
f'Msg {msg.id}, msg_com {msg.com} is send to {msg.receiver_id}.'
)
else:
if self.pub_option:
self.sockets[PUB_Socket].send_multipart(
[msg_bytes, crypted])
else:
info_msg(
self, 'INFO',
f'Publisher socket is not available for {self.name}.')
except zmq.ZMQError as e:
error_logger(self, self.send_msg, e)
def run(self):
super().run()
try:
for adr in self.addresses[PUB_Socket_Server]:
self.subscribe_sub(address=adr)
msg = self._wait_server_hb()
if self.active:
self.connect()
self.parent.thinker.react_heartbeat_full(msg)
info_msg(self, 'STARTED')
self._receive_msgs()
except (zmq.error.ZMQError, MessengerError) as e: # Bad type of error
error_logger(self, self.run, e)
self.stop()
finally:
self.sockets[DEALER_Socket].close()
if self.sockets[SUB_Socket]:
self.sockets[SUB_Socket].close()
if self.sockets[PUB_Socket]:
self.sockets[PUB_Socket].close()
self.context.destroy()
self.active = False
self.paused = True
info_msg(self, 'STOPPED')
def gen_msg(self, msg_com: Union[MsgComInt, MsgComExt],
**kwargs) -> Union[MessageExt, MessageInt, None]:
try:
message_info: MessageInfoExt = msg_com.value
if not message_info.must_have_param.issubset(set(
kwargs.keys())):
error_logger(
self, self.generate_msg,
f'Not all required parameters are given for {msg_com}'
f'{message_info.must_have_param}, only {kwargs.keys()}'
)
raise DeviceError(
f'Not all parameters are passed to device.generate_msg'
)
else:
if msg_com is MsgComExt.AVAILABLE_SERVICES:
info = AvailableServices(
available_services=kwargs['available_services'])
elif msg_com is MsgComExt.DO_IT:
info = kwargs['func_input']
elif msg_com is MsgComExt.DONE_IT:
info = kwargs['func_output']
elif msg_com is MsgComExt.HEARTBEAT or msg_com is MsgComInt.HEARTBEAT:
event = kwargs['event']
info = HeartBeat(device_id=self.id,
event_n=event.n,
event_id=event.id)
elif msg_com is MsgComExt.HEARTBEAT_FULL:
event = kwargs['event']
info = HeartBeatFull(
event_n=event.n,
event_name=event.external_name,
event_tick=event.tick,
event_id=event.id,
device_id=self.id,
device_name=self.name,
device_type=self.type,
device_public_key=self.messenger.public_key,
device_public_sockets=self.messenger.public_sockets
)
elif msg_com is MsgComExt.ERROR:
info = MsgError(
error_comments=kwargs['error_comments'])
elif msg_com is MsgComInt.DEVICE_INFO_INT:
info = DeviceInfoInt(
active_connections=self.active_connections(),
available_public_functions=self.
available_public_functions(),
device_id=self.id,
device_status=self.device_status,
device_description=self.description(),
events_running=list(
self.thinker.events.name_id.keys()))
elif msg_com is MsgComExt.SHUTDOWN:
info = ShutDown(device_id=self.id,
reason=kwargs['reason'])
elif msg_com is MsgComExt.WELCOME_INFO_SERVER:
try:
session_key = self.connections[
kwargs['receiver_id']].session_key
device_public_key = self.connections[
kwargs['receiver_id']].device_public_key
# Session key Server-Device is crypted with device public key, message is not crypted
session_key_crypted = self.messenger.encrypt_with_public(
session_key, device_public_key)
except KeyError:
session_key_crypted = b''
finally:
info = WelcomeInfoServer(
session_key=session_key_crypted)
elif msg_com is MsgComExt.WELCOME_INFO_DEVICE:
try:
server_public_key = self.connections[
self.server_id].device_public_key
pub_key = self.messenger.public_key
device_public_key_crypted = self.messenger.encrypt_with_public(
pub_key, server_public_key)
except KeyError:
device_public_key_crypted = self.messenger.public_key
event = kwargs['event']
info = WelcomeInfoDevice(
event_name=event.external_name,
event_tick=event.tick,
event_id=event.id,
device_id=self.id,
device_name=self.name,
device_type=self.type,
device_public_key=device_public_key_crypted,
device_public_sockets=self.messenger.public_sockets
)
except Exception as e: # TODO: replace Exception, after all it is needed for development
error_logger(self, self.generate_msg, f'{msg_com}: {e}')
raise e
finally:
if isinstance(msg_com, MsgComExt):
if msg_com.msg_type is MsgType.BROADCASTED:
reply_to = ''
receiver_id = ''
elif msg_com.msg_type is MsgType.DIRECTED:
try:
reply_to = kwargs['reply_to']
except KeyError:
reply_to = ''
receiver_id = kwargs['receiver_id']
return MessageExt(com=msg_com.msg_name,
crypted=msg_com.msg_crypted,
info=info,
receiver_id=receiver_id,
reply_to=reply_to,
sender_id=self.messenger.id)
else:
return MessageInt(com=msg_com.msg_name,
info=info,
sender_id=self.messenger.id)
def add_demand_waiting_reply(self, msg: MessageExt):
try:
self._demands_waiting_reply[msg.id] = PendingDemand(message=msg)
except KeyError as e:
error_logger(self, self.add_demand_waiting_reply, e)
def add_to_forwarded(self, msg_forwarded: MessageExt,
msg_arrived: MessageExt):
try:
self._forwarded[msg_forwarded.id] = msg_arrived
except KeyError as e:
error_logger(self, self.add_demand_waiting_reply, e)
def add_msg_out(self, msg: MessageExt):
try:
self._msg_out[msg.id] = msg
except KeyError as e:
error_logger(self, self.add_msg_out, e)
