def pending_demands(event: ThinkerEvent): thinker: Thinker = event.parent demands_waiting_reply: MsgDict = thinker.demands_waiting_reply info_msg(event, 'STARTED', extra=f' of {thinker.name} with tick {event.tick}') while event.active: sleep(0.001) if not event.paused and demands_waiting_reply: try: sleep(event.tick) for key, item in demands_waiting_reply.items(): pending: PendingReply = item if (time() - event.time) > event.tick and pending.attempt < 3: pending.attempt += 1 info_msg( event, 'INFO', f'Msg {pending.message.id}, com {pending.message.com} waits ' f'{pending.attempt}.') elif (time() - event.time) > event.tick and pending.attempt >= 3: try: msg = pending.message del demands_waiting_reply[key] info_msg(event, 'INFO', f'Reply timeout for msg: {msg.short()}') info_msg(event, 'INFO', f'Msg {msg.id} is deleted') except KeyError: error_logger( event, pending_demands, f'Cannot delete Msg {msg.id}, com {msg.com} from ' f'demand_waiting_reply') except ThinkerErrorReact as e: error_logger(event, pending_demands, e)
def subscribe_sub(self, address=None, filter_opt=b""): try: self.sockets['sub'].connect(address) self.sockets['sub'].setsockopt(zmq.SUBSCRIBE, filter_opt) self.sockets['sub'].setsockopt(zmq.RCVHWM, 3) except (zmq.ZMQError, Exception) as e: error_logger(self, self.subscribe_sub, e)
def _create_sockets(self): try: self.context = zmq.Context() frontend = self.context.socket(zmq.ROUTER) frontend.bind(self.addresses[FRONTEND_Server]) backend = self.context.socket(zmq.ROUTER) backend.bind(self.addresses[BACKEND_Server]) publisher = self.context.socket(zmq.PUB) publisher.bind(self.addresses[PUB_Socket_Server]) # SOCKET SUBSCRIBER sub = self.context.socket(zmq.SUB) self.poller = zmq.Poller() self.poller.register(frontend, zmq.POLLIN) self.poller.register(backend, zmq.POLLIN) self.poller.register(sub, zmq.POLLIN) self.sockets = { FRONTEND_Server: frontend, BACKEND_Server: backend, PUB_Socket_Server: publisher, SUB_Socket: sub } self.public_sockets = { PUB_Socket_Server: self.addresses[PUB_Socket_Server], FRONTEND_Server: self.addresses[FRONTEND_Server], BACKEND_Server: self.addresses[BACKEND_Server] } except (WrongAddress, KeyError, zmq.ZMQError) as e: error_logger(self, self._create_sockets, e) raise e
def task_out_reaction(event: ThinkerEvent): thinker: Thinker = event.parent tasks_out: MsgDict = thinker.tasks_out demand_waiting_reply: MsgDict = thinker.demands_waiting_reply info_msg(event, 'STARTED', extra=f' of {thinker.name} with tick {event.tick}') while event.active: sleep(0.001) if not event.paused and tasks_out: try: msg: MessageExt = tasks_out.popitem(last=False)[1] react = True if msg.receiver_id != '' and msg.reply_to == '': # If msg is not reply, than add to pending demand info_msg( event, 'INFO', f'Msg id={msg.id}, com {msg.com} is considered to get a reply' ) thinker.add_demand_waiting_reply(msg) elif msg.reply_to != '': if msg.reply_to in demand_waiting_reply: msg_awaited: MessageExt = thinker.demands_waiting_reply[ msg.reply_to].message del demand_waiting_reply[msg.reply_to] info_msg( event, 'INFO', f'Msg id={msg.reply_to} {msg_awaited.com} is deleted from ' f'demand_waiting_reply') if react: thinker.parent.messenger.add_msg_out(msg) except (ThinkerErrorReact, KeyError) as e: error_logger(event, task_out_reaction, f'{e}: {msg.short()}')
def _create_sockets(self): try: self.context = zmq.Context() # SOCKET DEALER dealer = self.context.socket(zmq.DEALER) dealer.setsockopt_unicode(zmq.IDENTITY, self.id) # SOCKET SUBSCRIBER subscriber = self.context.socket(zmq.SUB) subscriber.setsockopt(zmq.RCVHWM, 10) self.sockets = {DEALER_Socket: dealer, SUB_Socket: subscriber} # SOCKET PUBLISHER if self.pub_option: publisher = self.context.socket(zmq.PUB) publisher.setsockopt_unicode(zmq.IDENTITY, self.addresses[PUB_Socket]) self.sockets[PUB_Socket] = publisher self.public_sockets = {PUB_Socket: self.addresses[PUB_Socket]} # POLLER self.poller = zmq.Poller() self.poller.register(dealer, zmq.POLLIN) self.poller.register(subscriber, zmq.POLLIN) except (WrongAddress, KeyError, zmq.ZMQError) as e: error_logger(self, self._create_sockets, e) raise e
def send_msg(self, msg: MessageExt): try: crypted = str(int(msg.crypted)).encode('utf-8') msg_bytes = self.encrypt_with_session_key(msg) if msg.receiver_id == '': self.sockets[PUB_Socket_Server].send_multipart( [msg_bytes, crypted]) else: if msg.receiver_id in self._frontendpool: self.sockets[FRONTEND_Server].send_multipart( [msg.receiver_id.encode('utf-8'), msg_bytes, crypted]) info_msg( self, 'INFO', f'Msg {msg.id}, com {msg.com} is send from frontend to {msg.receiver_id}.' ) elif msg.receiver_id in self._backendpool: self.sockets[BACKEND_Server].send_multipart( [msg.receiver_id.encode('utf-8'), msg_bytes, crypted]) info_msg( self, 'INFO', f'Msg {msg.id}, com {msg.com} is send from backend to {msg.receiver_id}.' ) else: error_logger( self, self.send_msg, f'ReceiverID {msg.receiver_id} is not present in Server pool.' ) except zmq.ZMQError as e: error_logger(self, self.send_msg, e)
def execute_com(self, msg: MessageExt): error = False com: str = msg.info.com input: FuncInput = msg.info if com in self.available_public_functions_names: f = getattr(self, com) func_input_type = signature(f).parameters['func_input'].annotation if func_input_type == type(input): try: result: FuncOutput = f(input) msg_r = self.generate_msg(msg_com=MsgComExt.DONE_IT, receiver_id=msg.sender_id, func_output=result, reply_to=msg.id) except Exception as e: # TODO: replace Exception error_logger(self, self.execute_com, e) error = True comments = str(e) else: error = True comments = f'Device {self.id} function: execute_com: Input type: {type(input)} do not match to ' \ f'func_input type : {func_input_type}' else: error = True comments = f'com: {com} is not available for Service {self.id}. See {self.available_public_functions()}' if error: msg_r = self.generate_msg(msg_com=MsgComExt.ERROR, comments=comments, receiver_id=msg.sender_id, reply_to=msg.id) self.send_msg_externally(msg_r)
def __init__(self, parent): from devices.devices import Device Thinker.n_instance += 1 self.logger = logging.getLogger(__name__ + '.' + self.__class__.__name__) self.name = f'{self.__class__.__name__}:{parent.name}:{Thinker.n_instance}' self.id = f'{self.name}:{unique_id(self.name)}' self.parent: Device = parent self.msg_counter = 0 self.events = Events_Dict() msg_dict_size_limit = 10000 self._tasks_in = MsgDict(name='tasks_in', size_limit=msg_dict_size_limit, dict_parent=self) self.tasks_in_test = MsgDict(name='tasks_in_test', size_limit=msg_dict_size_limit, dict_parent=self) self._tasks_out = MsgDict(name='tasks_out', size_limit=msg_dict_size_limit, dict_parent=self) self.tasks_out_test = MsgDict(name='tasks_out_test', size_limit=msg_dict_size_limit, dict_parent=self) self._demands_waiting_reply = MsgDict(name='demands_waiting_reply', size_limit=msg_dict_size_limit, dict_parent=self) # TODO: add slow thread to track after forwarded messages self._forwarded = MsgDict(name='forwarded_messages', size_limit=msg_dict_size_limit, dict_parent=self) self.paused = False info_msg(self, 'CREATING') try: self.timeout = int(self.parent.get_general_settings()['timeout']) pending_demands_tick = float( self.parent.get_general_settings()['pending_demands']) / 1000. except KeyError as e: error_logger(self, self.__init__, e) self.timeout = 10 pending_demands_tick = 0.2 try: from communication.logic.logic_functions import (task_in_reaction, task_out_reaction, pending_demands) self.register_event(name='task_in_reaction', logic_func=task_in_reaction, tick=None) self.register_event(name='task_out_reaction', logic_func=task_out_reaction, tick=None) self.register_event(name='demands_waiting_reply', logic_func=pending_demands, tick=pending_demands_tick) info_msg(self, 'CREATED') except (ThinkerEventError, ThinkerEventFuncError, TypeError) as e: error_logger(self, self.register_event, e) info_msg(self, 'NOT CREATED') raise ThinkerError(str(e))
def add_task_out(self, msg: MessageExt): try: self._tasks_out[msg.id] = msg if self.parent.test and not (msg.com == MsgComExt.HEARTBEAT.msg_name): self.tasks_out_test[msg.id] = msg except KeyError as e: error_logger(self, self.add_task_out, e)
def connect(self): try: self.sockets[DEALER_Socket].connect( self.addresses[FRONTEND_Server]) if self.pub_option: self.bind_pub() except (WrongAddress, zmq.error.ZMQBindError) as e: error_logger(self, self.connect, e)
def subscribe_sub(self, address=None, filter_opt=b""): try: self.sockets[SUB_Socket].connect(address) self.sockets[SUB_Socket].setsockopt(zmq.SUBSCRIBE, filter_opt) self.sockets[SUB_Socket].setsockopt(zmq.RCVHWM, 3) self.logger.info(f'socket sub is connected to {address}') except (zmq.ZMQError, Exception) as e: error_logger(self, self.subscribe_sub, e)
def __setitem__(self, key, value): if key not in self: self._check_size_limit() super().__setitem__(key, value) else: error = f'Key: {key} already exists in {self.name} {self}' if self.dict_parent: error_logger(self.dict_parent, self, error) raise KeyError(error)
def run(self): self.active = True self.paused = self.parent.paused try: self.time = time() info_msg(self, 'STARTING', extra=f' of {self.parent.name}') self.logic_func(self) except Exception as e: error_logger(self, self.run, f'{self.name}. Error: {e}') finally: info_msg(self, 'STOPPED', extra=f' of {self.parent.name}')
def react_broadcast(self, msg: MessageExt): if msg.com == MsgComExt.HEARTBEAT.msg_name: try: self.events[msg.info.event_id].time = time() self.events[msg.info.event_id].n = msg.info.event_n if self.parent.pyqtsignal_connected: self.parent.signal.emit(msg.ext_to_int()) except (KeyError, TypeError) as e: error_logger(self, self.react_broadcast, e) elif msg.com == MsgComExt.SHUTDOWN.msg_name: # When one of devices shutdowns self.remove_device_from_connections(msg.sender_id) self.parent.send_status_pyqt()
def msg_out(self, msg_out: Union[MessageExt, List[MessageExt]]): if msg_out: if isinstance(msg_out, list): for msg in msg_out: self.msg_out(msg) elif isinstance(msg_out, MessageExt): info_msg(self, 'INFO', msg_out.short()) self.add_task_out(msg_out) else: error_logger( self, self.msg_out, f'Union[MessageExt, List[MessageExt]] was not passed to msg_out, but' f'{msg_out}')
def bind_pub(self): try: self.sockets[PUB_Socket].setsockopt_unicode( zmq.IDENTITY, self.addresses[PUB_Socket]) self.sockets[PUB_Socket].bind(self.addresses[PUB_Socket]) except (WrongAddress, zmq.error.ZMQError) as e: # TODO: potential recursion depth violation error_logger(self, self.connect, f'{e}: {self.addresses[PUB_Socket]}') port = get_free_port(scope=None) local_ip = get_local_ip() self.addresses[PUB_Socket] = f'tcp://{local_ip}:{port}' self.public_sockets = {PUB_Socket: self.addresses[PUB_Socket]} self.bind_pub()
def _deal_with_reaceived_msg(self, msgs: List[MsgTuple]): for msg, device_id, socket_name, crypted in msgs: try: if int(crypted): msg: bytes = self.decrypt_with_session_key(msg, device_id) mes: MessageExt = MessageExt.bytes_to_msg(msg) self.parent.thinker.add_task_in(mes) except (MessengerError, MessageError, ThinkerError, Exception) as e: error_logger(self, self.run, e) msg_r = self.parent.generate_msg(msg_com=MsgComExt.ERROR, error_comments=str(e), reply_to='', receiver_id=device_id) self.add_msg_out(msg_r)
def run(self): super().run() try: info_msg(self, 'STARTED') self._receive_msgs() except zmq.error.ZMQError as e: # Bad kind of error! error_logger(self, self.run, e) self.stop() finally: for _, soc in self.sockets.items(): soc.close() self.context.destroy() self.active = False self.paused = True info_msg(self, 'STOPPED')
def react_internal(self, event: ThinkerEvent): if 'heartbeat' in event.name: if event.counter_timeout > 5: try: connections = self.parent.connections self.logger.info(f"""{connections[event.original_owner].device_info.type} {event.name} was away for too long...removing it from active services, deleting its tasks""") self.unregister_event(event.id) del connections[event.original_owner] # TODO: tasks should be deleted here except KeyError as e: error_logger(self, self.react_internal, e) self.unregister_event(event.id) self.parent.send_status_pyqt(com='status_server_info_full') else: self.logger.info(f'react_internal: I do not know what to do, {event.name} is not known')
def remove_device_from_connections(self, device_id): # TODO: the service_info is not deleted from _frontend sockets or backend sockets connections = self.parent.connections if device_id in connections: info_msg(self, 'INFO', f'Procedure to delete {device_id} is started') for key, event in list(self.events.items()): if event.original_owner == device_id: self.unregister_event(key) del self.parent.connections[device_id] info_msg(self, 'INFO', f'Device {device_id} is deleted') else: error_logger( self, self.remove_device_from_connections, f'remove_device_from_connections: Wrong device_id {device_id} is passed' )
def task_in_reaction(event: ThinkerEvent): thinker: Thinker = event.parent tasks_in: MsgDict = thinker.tasks_in info_msg(event, 'STARTED', extra=f' of {thinker.name} with tick {event.tick}') exclude_msgs = [ MsgComExt.HEARTBEAT.msg_name, MsgComExt.HEARTBEAT_FULL.msg_name ] while event.active: sleep( 0.001 ) # Any small interruption is necessary not to overuse processor time if not event.paused and tasks_in: try: msg: MessageExt = tasks_in.popitem(last=False)[1] thinker.msg_counter += 1 react = True if msg.com not in exclude_msgs: info_msg(event, 'INFO', f'Received: {msg.short()}') if msg.reply_to == '' and msg.receiver_id != '': # If message is not a reply, it must be a demand one thinker.add_demand_waiting_reply(msg) info_msg( event, 'INFO', f'Expect a reply to {msg.id} com={msg.com}. Adding to waiting list.' ) elif msg.reply_to != '': if msg.reply_to in thinker.demands_waiting_reply: # TODO: should it have else clause or not? msg_awaited: MessageExt = thinker.demands_waiting_reply[ msg.reply_to].message del thinker.demands_waiting_reply[msg.reply_to] info_msg( event, 'INFO', f'REPLY to Msg {msg.reply_to} {msg_awaited.com} is obtained.' ) else: react = False info_msg( event, 'INFO', f'Reply to msg {msg.reply_to} arrived too late.') if react: thinker.react_external(msg) except (ThinkerErrorReact, KeyError) as e: error_logger(event, task_in_reaction, f'{e}: {msg.short()}')
def register_event(self, name: str, logic_func: Callable, event_id='', external_name='', original_owner='', start_now=False, **kwargs): # TODO: to complicated. Need to optimize. try: if 'tick' in kwargs: tick = kwargs['tick'] else: tick = float(self.parent.get_general_settings()[name.split(':') [0]]) / 1000 except KeyError as e: error_logger(self, self.register_event, f'{e}. Tick value is set to {tick}s') finally: print_every_n = int( self.parent.get_general_settings()['print_every_n']) try: if not external_name: external_name = name if not event_id: event_id = f'{external_name}:{self.parent.id}' if original_owner == '': original_owner = self.parent.id self.events[event_id] = ThinkerEvent( name=name, external_name=external_name, parent=self, logic_func=logic_func, tick=tick, print_every_n=print_every_n, event_id=event_id, original_owner=original_owner) if start_now: self.events[event_id].start() except ThinkerEventFuncError as e: raise ThinkerEventError(str(e))
def send_msg(self, msg: MessageExt): try: crypted = str(int(msg.crypted)).encode('utf-8') msg_bytes = self.encrypt_with_session_key(msg) if msg.receiver_id != '': self.sockets[DEALER_Socket].send_multipart( [msg_bytes, crypted]) info_msg( self, 'INFO', f'Msg {msg.id}, msg_com {msg.com} is send to {msg.receiver_id}.' ) else: if self.pub_option: self.sockets[PUB_Socket].send_multipart( [msg_bytes, crypted]) else: info_msg( self, 'INFO', f'Publisher socket is not available for {self.name}.') except zmq.ZMQError as e: error_logger(self, self.send_msg, e)
def run(self): super().run() try: for adr in self.addresses[PUB_Socket_Server]: self.subscribe_sub(address=adr) msg = self._wait_server_hb() if self.active: self.connect() self.parent.thinker.react_heartbeat_full(msg) info_msg(self, 'STARTED') self._receive_msgs() except (zmq.error.ZMQError, MessengerError) as e: # Bad type of error error_logger(self, self.run, e) self.stop() finally: self.sockets[DEALER_Socket].close() if self.sockets[SUB_Socket]: self.sockets[SUB_Socket].close() if self.sockets[PUB_Socket]: self.sockets[PUB_Socket].close() self.context.destroy() self.active = False self.paused = True info_msg(self, 'STOPPED')
def gen_msg(self, msg_com: Union[MsgComInt, MsgComExt], **kwargs) -> Union[MessageExt, MessageInt, None]: try: message_info: MessageInfoExt = msg_com.value if not message_info.must_have_param.issubset(set( kwargs.keys())): error_logger( self, self.generate_msg, f'Not all required parameters are given for {msg_com}' f'{message_info.must_have_param}, only {kwargs.keys()}' ) raise DeviceError( f'Not all parameters are passed to device.generate_msg' ) else: if msg_com is MsgComExt.AVAILABLE_SERVICES: info = AvailableServices( available_services=kwargs['available_services']) elif msg_com is MsgComExt.DO_IT: info = kwargs['func_input'] elif msg_com is MsgComExt.DONE_IT: info = kwargs['func_output'] elif msg_com is MsgComExt.HEARTBEAT or msg_com is MsgComInt.HEARTBEAT: event = kwargs['event'] info = HeartBeat(device_id=self.id, event_n=event.n, event_id=event.id) elif msg_com is MsgComExt.HEARTBEAT_FULL: event = kwargs['event'] info = HeartBeatFull( event_n=event.n, event_name=event.external_name, event_tick=event.tick, event_id=event.id, device_id=self.id, device_name=self.name, device_type=self.type, device_public_key=self.messenger.public_key, device_public_sockets=self.messenger.public_sockets ) elif msg_com is MsgComExt.ERROR: info = MsgError( error_comments=kwargs['error_comments']) elif msg_com is MsgComInt.DEVICE_INFO_INT: info = DeviceInfoInt( active_connections=self.active_connections(), available_public_functions=self. available_public_functions(), device_id=self.id, device_status=self.device_status, device_description=self.description(), events_running=list( self.thinker.events.name_id.keys())) elif msg_com is MsgComExt.SHUTDOWN: info = ShutDown(device_id=self.id, reason=kwargs['reason']) elif msg_com is MsgComExt.WELCOME_INFO_SERVER: try: session_key = self.connections[ kwargs['receiver_id']].session_key device_public_key = self.connections[ kwargs['receiver_id']].device_public_key # Session key Server-Device is crypted with device public key, message is not crypted session_key_crypted = self.messenger.encrypt_with_public( session_key, device_public_key) except KeyError: session_key_crypted = b'' finally: info = WelcomeInfoServer( session_key=session_key_crypted) elif msg_com is MsgComExt.WELCOME_INFO_DEVICE: try: server_public_key = self.connections[ self.server_id].device_public_key pub_key = self.messenger.public_key device_public_key_crypted = self.messenger.encrypt_with_public( pub_key, server_public_key) except KeyError: device_public_key_crypted = self.messenger.public_key event = kwargs['event'] info = WelcomeInfoDevice( event_name=event.external_name, event_tick=event.tick, event_id=event.id, device_id=self.id, device_name=self.name, device_type=self.type, device_public_key=device_public_key_crypted, device_public_sockets=self.messenger.public_sockets ) except Exception as e: # TODO: replace Exception, after all it is needed for development error_logger(self, self.generate_msg, f'{msg_com}: {e}') raise e finally: if isinstance(msg_com, MsgComExt): if msg_com.msg_type is MsgType.BROADCASTED: reply_to = '' receiver_id = '' elif msg_com.msg_type is MsgType.DIRECTED: try: reply_to = kwargs['reply_to'] except KeyError: reply_to = '' receiver_id = kwargs['receiver_id'] return MessageExt(com=msg_com.msg_name, crypted=msg_com.msg_crypted, info=info, receiver_id=receiver_id, reply_to=reply_to, sender_id=self.messenger.id) else: return MessageInt(com=msg_com.msg_name, info=info, sender_id=self.messenger.id)
def add_demand_waiting_reply(self, msg: MessageExt): try: self._demands_waiting_reply[msg.id] = PendingDemand(message=msg) except KeyError as e: error_logger(self, self.add_demand_waiting_reply, e)
def add_to_forwarded(self, msg_forwarded: MessageExt, msg_arrived: MessageExt): try: self._forwarded[msg_forwarded.id] = msg_arrived except KeyError as e: error_logger(self, self.add_demand_waiting_reply, e)
def add_msg_out(self, msg: MessageExt): try: self._msg_out[msg.id] = msg except KeyError as e: error_logger(self, self.add_msg_out, e)