def send_checkup(): def checkup_timeout(): global srv, responding, timeout timeout.stop() if not responding: # we've timed out, restart # TODO: provide config var for how many times to attempt start before exiting print('{0} not responding, attempting start.'.format(MODULE)) srv = launch_service() def recv_checkup(msg): global responding responding = True # access globals global timeout, checkup, responding, loop # listen for ping back checkup.on_recv(recv_checkup) # do what's needed to rescue on timeout responding = False timeout = DelayedCallback(checkup_timeout, CHECKUP_TIMEOUT, io_loop=loop) timeout.start() # send ping checkup.send('You alive?')
def send_checkup(): def checkup_timeout(): global service, responding, timeout timeout.stop() if not responding: # we've timed out, restart print('Mongrel2 not responding, attempting restart.') # since control port isn't responding, do a dirty kill just in case kill_mongrel_with_pid(M2_PID_PATH) # start her back up # TODO: add configurable delay here before starting again start_mongrel() def recv_response(msg): global responding responding = True # access globals global timeout, control_port, responding # listen for ping back control_port.on_recv(recv_response) # do what's needed to rescue on timeout responding = False timeout = DelayedCallback(checkup_timeout, CHECKUP_TIMEOUT, io_loop=loop) timeout.start() # send status request control_port.send(tnetstrings.dump(['status', {'what': 'net'}]))
def call(self, proc_name, args=[], kwargs={}, result='async', timeout=None): #{ """ Call the remote method with *args and **kwargs. Parameters ---------- proc_name : <str> name of the remote procedure to call args : <tuple> positional arguments of the procedure kwargs : <dict> keyword arguments of the procedure ignore : <bool> whether to ignore result or wait for it timeout : <float> | None Number of seconds to wait for a reply. RPCTimeoutError is set as the future result in case of timeout. Set to None, 0 or a negative number to disable. Returns ------- <Future> if result is 'async' None if result is 'ignore' If remote call fails: - sets <RemoteRPCError> into the <Future> if result is 'async' """ assert result in ('async', 'ignore'), \ 'expected any of "async", "ignore" -- got %r' % result if not (timeout is None or isinstance(timeout, (int, float))): raise TypeError("timeout param: <float> or None expected, got %r" % timeout) ignore = result == 'ignore' req_id, msg_list = self._build_request(proc_name, args, kwargs, ignore) self.socket.send_multipart(msg_list) if ignore: return None # The following logic assumes that the reply won't come back too # quickly, otherwise the callbacks won't be in place in time. It should # be fine as this code should run very fast. This approach improves # latency we send the request ASAP. def _abort_request(): future_tout = self._futures.pop(req_id, None) if future_tout: future, _ = future_tout tout_msg = "Request %s timed out after %s sec" % (req_id, timeout) #self.logger.debug(tout_msg) future.set_exception(RPCTimeoutError(tout_msg)) timeout = timeout or 0 if timeout > 0: tout_cb = DelayedCallback(_abort_request, int(timeout*1000), self.ioloop) tout_cb.start() else: tout_cb = None future = Future() self._futures[req_id] = (future, tout_cb) return future
def stop(signum, frame): global loop, checkup_periodic, control_port, stop_timeout def stop_timeout(): print('Terminate request timed out, mongrel2 might be orphaned.') kill_mongrel_with_pid(M2_PID_PATH) shutdown() def terminate_resp(msg): print('Mongrel2 control port confirmed SIGTERM sent.') shutdown() def shutdown(): print('Shutting down.') remove_hosts(HOSTS) control_port.close() loop.stop() print('\nStopping.') # make sure checkup doesn't happen during termination checkup_periodic.stop() # register terminate response callback control_port.on_recv(terminate_resp) # prepare timeout stop_timeout = DelayedCallback(stop_timeout, STOP_TIMEOUT, io_loop=loop) stop_timeout.start() # send terminate request control_port.send(tnetstrings.dump(['terminate', {}]))
def call(self, proc_name, args=[], kwargs={}, ignore=False, timeout=None): #{ """ Call the remote method with *args and **kwargs. Parameters ---------- proc_name : <str> name of the remote procedure to call args : <tuple> positional arguments of the procedure kwargs : <dict> keyword arguments of the procedure ignore : <bool> whether to ignore result or wait for it timeout : <float> | None Number of seconds to wait for a reply. RPCTimeoutError is set as the future result in case of timeout. Set to None, 0 or a negative number to disable. Returns None or a <Future> representing a remote call result """ if not (timeout is None or isinstance(timeout, (int, float))): raise TypeError("timeout param: <float> or None expected, got %r" % timeout) req_id, msg_list = self._build_request(proc_name, args, kwargs, ignore) self.socket.send_multipart(msg_list) if ignore: return None # The following logic assumes that the reply won't come back too # quickly, otherwise the callbacks won't be in place in time. It should # be fine as this code should run very fast. This approach improves # latency we send the request ASAP. def _abort_request(): future_tout = self._futures.pop(req_id, None) if future_tout: future, _ = future_tout tout_msg = "Request %s timed out after %s sec" % (req_id, timeout) self.logger.debug(tout_msg) future.set_exception(RPCTimeoutError(tout_msg)) timeout = timeout or 0 if timeout > 0: tout_cb = DelayedCallback(_abort_request, int(timeout * 1000), self.ioloop) tout_cb.start() else: tout_cb = None future = Future() self._futures[req_id] = (future, tout_cb) return future
def _start_timeout(self, timeout): """Helper for starting the timeout. :param timeout: the time to wait in milliseconds. :type timeout: int """ self._tmo = DelayedCallback(self._on_timeout, timeout) self._tmo.start() return
def call(self, method, callback, errback, timeout, *args, **kwargs): """Call the remote method with *args and **kwargs. Parameters ---------- method : str The name of the remote method to call. callback : callable The callable to call upon success or None. The result of the RPC call is passed as the single argument to the callback: `callback(result)`. errback : callable The callable to call upon a remote exception or None, The signature of this method is `errback(ename, evalue, tb)` where the arguments are passed as strings. timeout : int The number of milliseconds to wait before aborting the request. When a request is aborted, the errback will be called with an RPCTimeoutError. Set to 0 or a negative number to use an infinite timeout. args : tuple The tuple of arguments to pass as `*args` to the RPC method. kwargs : dict The dict of arguments to pass as `**kwargs` to the RPC method. """ if not isinstance(timeout, int): raise TypeError("int expected, got %r" % timeout) if not (callback is None or callable(callback)): raise TypeError("callable or None expected, got %r" % callback) if not (errback is None or callable(errback)): raise TypeError("callable or None expected, got %r" % errback) msg_id, msg_list = self._build_request(method, args, kwargs) self.stream.send_multipart(msg_list) # The following logic assumes that the reply won't come back too # quickly, otherwise the callbacks won't be in place in time. It should # be fine as this code should run very fast. This approach improves # latency we send the request ASAP. def _abort_request(): cb_eb_dc = self._callbacks.pop(msg_id, None) if cb_eb_dc is not None: eb = cb_eb_dc[1] if eb is not None: try: raise RPCTimeoutError() except: etype, evalue, tb = sys.exc_info() eb(etype.__name__, evalue, traceback.format_exc(tb)) if timeout > 0: dc = DelayedCallback(_abort_request, timeout, self.loop) dc.start() else: dc = None self._callbacks[msg_id] = (callback, errback, dc)
def test_01_simple_01(self): """Test MDPWorker simple req/reply. """ self._start_broker() time.sleep(0.2) worker = MyWorker(self.context, self.endpoint, self.service) sender = DelayedCallback(self.send_req, 1000) sender.start() IOLoop.instance().start() worker.shutdown() self._stop_broker() return
def restart_service(): global srv def restart_checkup(): global checkup_periodic checkup_periodic.start() global loop checkup_restart = DelayedCallback(restart_checkup, PAUSE_AFTER_RESTART, io_loop=loop) service = launch_service() checkup_restart.start()
def _tick(self): """Method called every HB_INTERVAL milliseconds. """ self.curr_liveness -= 1 ## print '%.3f tick - %d' % (time.time(), self.curr_liveness) self.send_hb() if self.curr_liveness >= 0: return ## print '%.3f lost connection' % time.time() # ouch, connection seems to be dead self.shutdown() # try to recreate it self._delayed_cb = DelayedCallback(self._create_stream, 5000) self._delayed_cb.start() return
def __init__(self, broker, service, io_loop=None): """Create and setup an MDP worker. @param broker A string containing the broker's URL @param service A string containing the service name @param io_loop An existing I/O loop object. If None, the default will be used. """ self.service=service self._broker = broker self.ctx = zmq.Context() sock = self.ctx.socket(zmq.DEALER) ZMQStream.__init__(self, sock, io_loop) # last watchdog timer tick self.watchdog = 0 # connection callback one-shot self._conncb = DelayedCallback(self.send_ready, 3000, self.io_loop) # heartbeat callback..runs continuous when connected self._hbcb = PeriodicCallback(self.send_heartbeat, 2000, self.io_loop) # number of connection attempts self._conn_attempt = 0 # waiting to connect state self._waiting_to_connect = True # have we been disconnected? (flags a reconnect attempt) self.disconnected = False # connect the socket and send a READY when the io_loop starts self.connect(self._broker) self._conncb.start()
def _tick(self): """Method called every HB_INTERVAL milliseconds. """ self.curr_retries -= 1 self.send_hb() if self.curr_retries >= 0: return # connection seems to be dead self.shutdown() # try to recreate it # self._delayed_cb = IOLoop.call_later(self._create_stream, 5000) # self._delayed_cb = IOLoop.add_timeout(self._create_stream, 5000) self._delayed_cb = DelayedCallback(self._create_stream, self.HB_INTERVAL) self._delayed_cb.start() return
def check_for_change(): global checkup, loop, checksums, responding, checkup_periodic, KEY curr_sums = check(path, watch) changed, deleted = get_diff(curr_sums, checksums) if len(changed) or len(deleted): checksums = curr_sums print('restarting {0}.'.format(MODULE)) checkup_periodic.stop() command.send(json.dumps({ 'key': KEY, 'command': 'die' })) delay = DelayedCallback(restart_service, PAUSE_BEFORE_RESTART + 300, io_loop=loop) delay.start()
def _recreate_stream(self): logging.debug('Worker trying to recreate stream') if self.endpoint_callback is not None: # # Check, maybe the ip of the proxy changed. # try: self.endpoint, self.hb_endpoint = self.endpoint_callback() except: # # Probably some problem in accessing the server. # self._delayed_reconnect = DelayedCallback( self._recreate_stream, self.RECONNECT_PERIOD) self._delayed_reconnect.start() return self._create_stream()
def quit_worker(raw_msg): """ When the worker should quit, stop the io_loop after 2 seconds. """ msg = MgmtMessage(raw_msg) if ZMQ_SPYDER_MGMT_WORKER_QUIT == msg.data: logger.info("process::We have been asked to shutdown, do so") DelayedCallback(io_loop.stop, 2000, io_loop).start() ack = MgmtMessage(topic=ZMQ_SPYDER_MGMT_WORKER, identity=identity, data=ZMQ_SPYDER_MGMT_WORKER_QUIT_ACK) mgmt._out_stream.send_multipart(ack.serialize())
def test_simple_worker(self): """Test MNWorker simple req/reply. """ self._start_broker() time.sleep(0.2) worker = WorkerRunner(self.context, self.endpoint, self.service) sender = DelayedCallback(self.send_req, 500) stopper = DelayedCallback(self.stop_loop, 2500) sender.start() stopper.start() IOLoop.instance().start() worker.shutdown() self._stop_broker() return
def __init__(self, context, device_id, address): self.context = context self.address = address self.device_id = device_id self.socket = None # delay = seconds_till_next('hour', duration=1) + 10 # 10 second buffer delay = 120 # 2 minutes after startup self.updater = DelayedCallback(self.update, delay * 1000) self.updater.start() return
def _tick(self): """Method called every HB_INTERVAL milliseconds. """ self.curr_liveness -= 1 self.send_hb() if self.curr_liveness >= 0: return # ouch, connection seems to be dead self.on_log_event('broker.timeout', 'Connection to broker timeouted, disconnecting') self.shutdown(False) # try to recreate it self._delayed_cb = DelayedCallback(self._create_stream, 5000) self._delayed_cb.start() return
def _tick(self): """Method called every HB_INTERVAL milliseconds. """ self.curr_liveness -= 1 logging.debug('Worker HB tick, current liveness: %d' % self.curr_liveness) self.send_hb() if self.curr_liveness >= 0: return # # Ouch, connection seems to be dead # logging.debug('Worker lost connection') self.shutdown() # # try to recreate the connection # self._delayed_reconnect = DelayedCallback(self._recreate_stream, self.RECONNECT_PERIOD) self._delayed_reconnect.start()
def send_request(self, request, args, kwargs, handler, timeout): """Send a request to the service.""" req = {} req['method'] = request.method req['uri'] = request.uri req['version'] = request.version req['headers'] = dict(request.headers) body = request.body req['remote_ip'] = request.remote_ip req['protocol'] = request.protocol req['host'] = request.host req['files'] = request.files req['arguments'] = request.arguments req['args'] = args req['kwargs'] = kwargs msg_id = bytes(uuid.uuid4()) msg_list = [b'|', msg_id, jsonapi.dumps(req)] if body: msg_list.append(body) logging.debug('Sending request: %r', msg_list) self.stream.send_multipart(msg_list) if timeout > 0: def _handle_timeout(): handler.send_error(504) # Gateway timeout try: self._callbacks.pop(msg_id) except KeyError: logging.error('Unexpected error removing callbacks') dc = DelayedCallback(_handle_timeout, timeout, self.loop) dc.start() else: dc = None self._callbacks[msg_id] = (handler, dc) return msg_id
def _tick(self): """Method called every HB_INTERVAL milliseconds. """ self.curr_liveness -= 1 ## print '%.3f tick - %d' % (time.time(), self.curr_liveness) self.send_hb() if self.curr_liveness >= 0: return print '%.3f lost connection' % time.time() # ouch, connection seems to be dead self.shutdown() # try to recreate it self._delayed_cb = DelayedCallback(self._create_stream, 5000) self._delayed_cb.start() return
def _tick(self): """Method called every HB_INTERVAL milliseconds. """ if DEBUG: print("MQREP > _tick") self.curr_liveness -= 1 if DEBUG: print('MQREP > _tick - {0} tick = {1}'.format(time.time(), self.curr_liveness)) self.send_hb() if self.curr_liveness >= 0: return if DEBUG: print('MQREP > _tick - {0} lost connection'.format(time.time())) # ouch, connection seems to be dead self.shutdown() # try to recreate it self._delayed_cb = DelayedCallback(self._create_stream, self.HB_INTERVAL) self._delayed_cb.start() return
def test_01_simple_01(self): """Test MDPWorker simple req/reply. """ self._start_broker() time.sleep(0.2) worker = MyWorker(self.context, self.endpoint, self.service) sender = DelayedCallback(self.send_req, 500) stopper = DelayedCallback(self.stop_test, 2500) sender.start() stopper.start() IOLoop.instance().start() worker.shutdown() self._stop_broker() return
def update(self): self.updater.stop() if self.device_state == 'connected': logger.info("Sending update for device") update = DeviceUpdate(self.device_id, 'farm_monitor') update.create() reply = self.send([pickle.dumps(update)]) if reply: message = pickle.loads(reply[1]) logger.debug("Update response from {0} :{1}".format(message.source, message.reply)) # delay = seconds_till_next('hour', duration=1) + 2 # 2 second buffer delay = 60 * 60 # send next update in 60 minutes self.updater = DelayedCallback(self.update, delay * 1000) self.updater.start() return
def _restart(self): self.shutdown() self.socket = self.context.socket(zmq.REQ) self.socket.setsockopt(zmq.LINGER, 0) self.socket.connect(self.address) self.can_send = True self.connection_attempts = 0 self.device_state = 'unknown' self.ticker = PeriodicCallback(self.heartbeat, DeviceConnection.HB_INTERVAL) self.ticker.start() # delay = seconds_till_next('hour', duration=1) + 2 # 2 second buffer delay = 60 # send update one minute after startup self.updater = DelayedCallback(self.update, delay * 1000) self.updater.start() return
def update(self): self.updater.stop() if internal_dealer.check_connection(): logger.info("Sending update for all grainbins") socket = self.context.socket(zmq.REQ) socket.setsockopt(zmq.LINGER, 0) socket.connect(self.address) update = GrainbinUpdate(self.device_id, 'farm_monitor') update.create() message = [pickle.dumps(update)] reply = mdp_request(socket, 'grainbin', message, 5) if reply: message = pickle.loads(reply[1]) logger.debug("reply from farm monitor for update: {0}".format(message.reply)) # delay = seconds_till_next('hour', duration=1) + 10 # 10 second buffer delay = 60 * 60 # send next update in one hour self.updater = DelayedCallback(self.update, delay * 1000) self.updater.start() return
class MDPClient(object): """Class for the MDP client side. Thin asynchronous encapsulation of a zmq.REQ socket. Provides a :func:`request` method with optional timeout. Objects of this class are ment to be integrated into the asynchronous IOLoop of pyzmq. :param context: the ZeroMQ context to create the socket in. :type context: zmq.Context :param endpoint: the enpoint to connect to. :type endpoint: str :param service: the service the client should use :type service: str """ _proto_version = C_CLIENT def __init__(self, context, endpoint): """Initialize the MDPClient. """ self.context = context self.endpoint = endpoint def start(self): """ Initialize the zmq sockets on a ioloop stream. The separation of this part from the init is useful if we start the client on a separate thread with a new ioloop (for example to enable use in an ipython notebook) """ socket = self.context.socket(zmq.DEALER) ioloop = IOLoop.instance() self.stream = ZMQStream(socket, ioloop) self.stream.on_recv(self._on_message) self._proto_prefix = [EMPTY_FRAME, self._proto_version] self._delayed_timeout = None self.timed_out = False socket.connect(self.endpoint) def shutdown(self): """Method to deactivate the client connection completely. Will delete the stream and the underlying socket. .. warning:: The instance MUST not be used after :func:`shutdown` has been called. :rtype: None """ if not self.stream: return self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.socket.close() self.stream.close() self.stream = None def request(self, service, msg, msg_extra=STANDARD, timeout=None): """Send the given message. :param msg: message parts to send. :type msg: list of str :param msg_extra: Extra message flags (e.g. STANDARD or BROADCAST) :type msg_extra: int :param timeout: time to wait in milliseconds. :type timeout: int :rtype None: """ if type(msg) in (bytes, unicode): msg = [msg] # # prepare full message # to_send = [chr(msg_extra)] + self._proto_prefix[:] to_send.extend([service]) to_send.extend(msg) self.stream.send_multipart(to_send) if timeout: self._start_timeout(timeout) def _on_timeout(self): """Helper called after timeout. """ self.timed_out = True self._delayed_timeout = None self.on_timeout() def _start_timeout(self, timeout): """Helper for starting the timeout. :param timeout: the time to wait in milliseconds. :type timeout: int """ self._delayed_timeout = DelayedCallback(self._on_timeout, timeout) self._delayed_timeout.start() def _on_message(self, msg): """Helper method called on message receive. :param msg: list of message parts. :type msg: list of str """ if self._delayed_timeout: # # disable timout # self._delayed_timeout.stop() self._delayed_timeout = None self.on_message(msg) def on_message(self, msg): """Public method called when a message arrived. .. note:: Does nothing. Should be overloaded! """ raise NotImplementedError('on_message must be implemented by the subclass.') def on_timeout(self): """Public method called when a timeout occured. .. note:: Does nothing. Should be overloaded! """ raise NotImplementedError('on_timeout must be implemented by the subclass.')
class MqAsyncReq(object): """Class for the MDP client side. Thin asynchronous encapsulation of a zmq.REQ socket. Provides a :func:`request` method with optional timeout. Objects of this class are ment to be integrated into the asynchronous IOLoop of pyzmq. :param context: the ZeroMQ context to create the socket in. :type context: zmq.Context :param endpoint: the enpoint to connect to. :type endpoint: str :param service: the service the client should use :type service: str """ _proto_version = b'MDPC01' def __init__(self, context, service): """Initialize the MDPClient. """ if ("domogik.common.configloader" in sys.modules): cfg = Loader('mq').load() confi = dict(cfg[1]) self.endpoint = "tcp://{0}:{1}".format(config['ip'], config['req_rep_port']) else: ip = Parameter.objects.get(key='mq-ip') port = Parameter.objects.get(key='mq-req_rep_port') self.endpoint = "tcp://{0}:{1}".format(ip.value, port.value) socket = ZmqSocket(context, zmq.REQ) ioloop = IOLoop.instance() self.service = service self.stream = ZMQStream(socket, ioloop) self.stream.on_recv(self._on_message) self.can_send = True self._proto_prefix = [ PROTO_VERSION, service] self._tmo = None self.timed_out = False socket.connect(self.endpoint) return def shutdown(self): """Method to deactivate the client connection completely. Will delete the stream and the underlying socket. .. warning:: The instance MUST not be used after :func:`shutdown` has been called. :rtype: None """ if not self.stream: return self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.socket.close() self.stream.close() self.stream = None return def request(self, msg, timeout=None): """Send the given message. :param msg: message parts to send. :type msg: list of str :param timeout: time to wait in milliseconds. :type timeout: int :rtype None: """ if not self.can_send: raise InvalidStateError() if type(msg) in (bytes, str): msg = [msg] # prepare full message to_send = self._proto_prefix[:] to_send.extend(msg) self.stream.send_multipart(to_send) self.can_send = False if timeout: self._start_timeout(timeout) return def _on_timeout(self): """Helper called after timeout. """ self.timed_out = True self._tmo = None self.on_timeout() return def _start_timeout(self, timeout): """Helper for starting the timeout. :param timeout: the time to wait in milliseconds. :type timeout: int """ self._tmo = DelayedCallback(self._on_timeout, timeout) self._tmo.start() return def _on_message(self, msg): """Helper method called on message receive. :param msg: list of message parts. :type msg: list of str """ if self._tmo: # disable timout self._tmo.stop() self._tmo = None # setting state before invoking on_message, so we can request from there self.can_send = True self.on_message(msg) return def on_message(self, msg): """Public method called when a message arrived. .. note:: Does nothing. Should be overloaded! """ pass def on_timeout(self): """Public method called when a timeout occured. .. note:: Does nothing. Should be overloaded! """ pass
class MDPWorker(object): """Class for the MDP worker side. Thin encapsulation of a zmq.DEALER socket. Provides a send method with optional timeout parameter. Will use a timeout to indicate a broker failure. """ _proto_version = b'MDPW01' # TODO: integrate that into API HB_INTERVAL = 1000 # in milliseconds HB_LIVENESS = 3 # HBs to miss before connection counts as dead def __init__(self, context, endpoint, service): """Initialize the MDPWorker. context is the zmq context to create the socket from. service is a byte-string with the service name. """ self.context = context self.endpoint = endpoint self.service = service self.stream = None self._tmo = None self.need_handshake = True self.ticker = None self._delayed_cb = None self._create_stream() return def _create_stream(self): """Helper to create the socket and the stream. """ socket = self.context.socket(zmq.DEALER) ioloop = IOLoop.instance() self.stream = ZMQStream(socket, ioloop) self.stream.on_recv(self._on_message) self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.connect(self.endpoint) self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL) self._send_ready() self.ticker.start() return def _send_ready(self): """Helper method to prepare and send the workers READY message. """ ready_msg = [b'', self._proto_version, b'\x01', self.service] self.stream.send_multipart(ready_msg) self.curr_liveness = self.HB_LIVENESS return def _tick(self): """Method called every HB_INTERVAL milliseconds. """ self.curr_liveness -= 1 ## print '%.3f tick - %d' % (time.time(), self.curr_liveness) self.send_hb() if self.curr_liveness >= 0: return ## print '%.3f lost connection' % time.time() # ouch, connection seems to be dead self.shutdown() # try to recreate it self._delayed_cb = DelayedCallback(self._create_stream, 5000) self._delayed_cb.start() return def send_hb(self): """Construct and send HB message to broker. """ msg = [b'', self._proto_version, b'\x04'] self.stream.send_multipart(msg) return def shutdown(self): """Method to deactivate the worker connection completely. Will delete the stream and the underlying socket. """ if self.ticker: self.ticker.stop() self.ticker = None if not self.stream: return self.stream.socket.close() self.stream.close() self.stream = None self.timed_out = False self.need_handshake = True self.connected = False return def reply(self, msg): """Send the given message. msg can either be a byte-string or a list of byte-strings. """ ## if self.need_handshake: ## raise ConnectionNotReadyError() # prepare full message to_send = self.envelope self.envelope = None if isinstance(msg, list): to_send.extend(msg) else: to_send.append(msg) self.stream.send_multipart(to_send) return def _on_message(self, msg): """Helper method called on message receive. msg is a list w/ the message parts """ # 1st part is empty msg.pop(0) # 2nd part is protocol version # TODO: version check proto = msg.pop(0) # 3rd part is message type msg_type = msg.pop(0) # XXX: hardcoded message types! # any message resets the liveness counter self.need_handshake = False self.curr_liveness = self.HB_LIVENESS if msg_type == b'\x05': # disconnect self.curr_liveness = 0 # reconnect will be triggered by hb timer elif msg_type == b'\x02': # request # remaining parts are the user message envelope, msg = split_address(msg) envelope.append(b'') envelope = [b'', self._proto_version, b'\x03'] + envelope # REPLY self.envelope = envelope self.on_request(msg) else: # invalid message # ignored pass return def on_request(self, msg): """Public method called when a request arrived. Must be overloaded! """ pass
class MDPWorker(object): """Class for the MDP worker side. Thin encapsulation of a zmq.DEALER socket. Provides a send method with optional timeout parameter. Will use a timeout to indicate a broker failure. """ _proto_version = W_WORKER # TODO: integrate that into API HB_INTERVAL = 2500 # in milliseconds HB_LIVENESS = 5 # HBs to miss before connection counts as dead RECONNECT_PERIOD = 5000 def __init__(self, context, endpoint, hb_endpoint, service, endpoint_callback=None): """Initialize the MDPWorker. context is the zmq context to create the socket from. service is a byte-string with the service name. """ self.context = context self.endpoint = endpoint self.hb_endpoint = hb_endpoint self.service = service self.endpoint_callback = endpoint_callback self.stream = None self.hb_stream = None self.ticker = None self._delayed_reconnect = None self._unique_id = '' self._create_stream() def _create_stream(self): """Helper to create the socket and the stream. """ logging.debug('Worker creating stream') ioloop = IOLoop.instance() socket = self.context.socket(zmq.DEALER) self.stream = ZMQStream(socket, ioloop) self.stream.on_recv(self._on_message) self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.connect(self.endpoint) socket = self.context.socket(zmq.DEALER) self.hb_stream = ZMQStream(socket, ioloop) self.hb_stream.on_recv(self._on_message) self.hb_stream.socket.setsockopt(zmq.LINGER, 0) self.hb_stream.connect(self.hb_endpoint) self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL) self._send_ready() self.ticker.start() def _send_ready(self): """Helper method to prepare and send the workers READY message. """ ready_msg = [EMPTY_FRAME, self._proto_version, W_READY, self.service] self.stream.send_multipart(ready_msg) self.curr_liveness = self.HB_LIVENESS def _tick(self): """Method called every HB_INTERVAL milliseconds. """ self.curr_liveness -= 1 logging.debug('Worker HB tick, current liveness: %d' % self.curr_liveness) self.send_hb() if self.curr_liveness >= 0: return # # Ouch, connection seems to be dead # logging.debug('Worker lost connection') self.shutdown() # # try to recreate the connection # self._delayed_reconnect = DelayedCallback(self._recreate_stream, self.RECONNECT_PERIOD) self._delayed_reconnect.start() def _recreate_stream(self): logging.debug('Worker trying to recreate stream') if self.endpoint_callback is not None: # # Check, maybe the ip of the proxy changed. # try: self.endpoint, self.hb_endpoint = self.endpoint_callback() except: # # Probably some problem in accessing the server. # self._delayed_reconnect = DelayedCallback( self._recreate_stream, self.RECONNECT_PERIOD) self._delayed_reconnect.start() return self._create_stream() def send_hb(self): """Construct and send HB message to broker. """ msg = [EMPTY_FRAME, self._proto_version, W_HEARTBEAT, self._unique_id] self.hb_stream.send_multipart(msg) def shutdown(self): """Method to deactivate the worker connection completely. Will delete the stream and the underlying socket. """ logging.debug('Shutdown of the worker') if self.ticker: logging.debug('Stopping the HB ticker') self.ticker.stop() self.ticker = None if not self.stream: return logging.debug('Closing the stream') self.stream.socket.close() self.stream.close() self.stream = None self.hb_stream.socket.close() self.hb_stream.close() self.hb_stream = None self.timed_out = False self.connected = False def reply(self, msg): """Send the given message. msg can either be a byte-string or a list of byte-strings. """ # # prepare full message # to_send = self.envelope self.envelope = None if isinstance(msg, list): to_send.extend(msg) else: to_send.append(msg) self.stream.send_multipart(to_send) def _on_message(self, msg): """Helper method called on message receive. msg is a list w/ the message parts """ logging.debug('Received message: {}'.format(msg)) # # 1st part is empty # msg.pop(0) # # 2nd part is protocol version # TODO: version check # proto = msg.pop(0) # # 3rd part is message type # msg_type = msg.pop(0) # # XXX: hardcoded message types! # any message resets the liveness counter # self.curr_liveness = self.HB_LIVENESS if msg_type == W_DISCONNECT: # # Disconnect. Reconnection will be triggered by hb timer # self.curr_liveness = 0 elif msg_type == W_READY: # # The message contains the unique id attached to the worker. # if len(msg) > 0: # # This above check is used for supporting older version of # the code. # self._unique_id = msg[0] elif msg_type == W_REQUEST: # # Request. Remaining parts are the user message # envelope, msg = split_address(msg) envelope.append(EMPTY_FRAME) envelope = [EMPTY_FRAME, self._proto_version, W_REPLY] + envelope self.envelope = envelope self.on_request(msg) else: # # invalid message # ignored # pass def on_request(self, msg): """Public method called when a request arrived. Must be overloaded! """ raise NotImplementedError( 'on_request must be implemented by the subclass.')
class MNWorker(MN_object): """Class for the MN worker side. Thin encapsulation of a zmq.DEALER socket. Provides a send method with optional timeout parameter. Will use a timeout to indicate a broker failure. :param context: the context to use for socket creation. :type context: zmq.Context :param endpoint: endpoint to connect to. :type endpoint: str :param service: the name of the service we support. :type service: byte-string """ _proto_version = b'MNPW01' # worker protocol version def __init__(self, context, endpoint, service, worker_type, address, protocols): """Initialize the MNWorker. """ self.context = context self.endpoint = endpoint self.service = service self.type = worker_type self.address = address self.protocols = protocols self.envelope = None self.HB_RETRIES = HB_RETRIES self.HB_INTERVAL = HB_INTERVAL self._data = {} self.stream = None self._tmo = None self.timed_out = False self.need_handshake = True self.connected = False self.ticker = None self._delayed_cb = None self._create_stream() _LOG.info("Worker initialized and can be found at '%s'" % endpoint) return def _create_stream(self): """Helper to create the socket and the stream. """ socket = self.context.socket(zmq.DEALER) ioloop = IOLoop.instance() self.stream = ZMQStream(socket, ioloop) self.stream.on_recv(self._on_message) self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.connect(self.endpoint) self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL) self._send_ready() self.ticker.start() return def _send_ready(self): """Helper method to prepare and send the workers READY message. """ _LOG.debug("Informing broker I am ready") ready_msg = [ b'', WORKER_PROTO, MSG_READY, self.service, self.type, self.address, self.protocols ] if self.stream.closed(): self.shutdown() self.stream.send_multipart(ready_msg) self.curr_retries = self.HB_RETRIES return def _tick(self): """Method called every HB_INTERVAL milliseconds. """ self.curr_retries -= 1 self.send_hb() if self.curr_retries >= 0: return # connection seems to be dead self.shutdown() # try to recreate it # self._delayed_cb = IOLoop.call_later(self._create_stream, 5000) # self._delayed_cb = IOLoop.add_timeout(self._create_stream, 5000) self._delayed_cb = DelayedCallback(self._create_stream, self.HB_INTERVAL) self._delayed_cb.start() return def send_hb(self): """Construct and send HB message to broker. """ _LOG.debug("Sending heartbeat") msg = [b'', WORKER_PROTO, MSG_HEARTBEAT] if self.stream.closed(): self.shutdown() self.stream.send_multipart(msg) return def shutdown(self): """Method to deactivate the worker connection completely. Will delete the stream and the underlying socket. """ if self.ticker: self.ticker.stop() self.ticker = None if not self.stream: return self.stream.socket.close() self.stream.close() self.stream = None self.timed_out = False self.need_handshake = True self.connected = False return def reply(self, msg): """Send the given message. :param msg: full message to send. :type msg: can either be a byte-string or a list of byte-strings """ if self.need_handshake: raise ConnectionNotReadyError() to_send = self.envelope self.envelope = None if isinstance(msg, list): to_send.extend(msg) else: to_send.append(msg) if self.stream.closed(): self.shutdown() self.stream.send_multipart(to_send) return def _on_message(self, msg): """Helper method called on message receive. :param msg: a list w/ the message parts :type msg: a list of byte-strings """ _LOG.debug("Received: %s." % msg) # 1st part is empty msg.pop(0) # 2nd part is protocol version proto = msg.pop(0) if proto != WORKER_PROTO: # ignore message from not supported protocol pass # 3rd part is message type msg_type = msg.pop(0) # XXX: hardcoded message types! # any message resets the retries counter self.need_handshake = False self.curr_retries = self.HB_RETRIES if msg_type == MSG_DISCONNECT: # disconnect _LOG.info("Broker wants us to disconnect.") self.curr_retries = 0 # reconnect will be triggered by hb timer elif msg_type == MSG_QUERY: # request # remaining parts are the user message _LOG.debug("Received new request: %s." % msg) envelope, msg = split_address(msg) envelope.append(b'') envelope = [b'', WORKER_PROTO, MSG_REPLY] + envelope # reply self.envelope = envelope self.on_request(msg) else: # invalid message # ignored _LOG.debug('ignoring message with invalid id') pass return def on_request(self, msg): """Public method called when a request arrived. :param msg: a list w/ the message parts :type msg: a list of byte-strings Must be overloaded to provide support for various services! """ pass
class MNClient(MN_object): """Class for the MN client side. Thin asynchronous encapsulation of a zmq.REQ socket. Provides a :func:`request` method with optional timeout. :param context: the ZeroMQ context to create the socket in. :type context: zmq.Context :param endpoint: the endpoint to connect to. :type endpoint: str :param service: the service the client should use :type service: str """ _proto_version = CLIENT_PROTO def __init__(self, context, endpoint, service): """Initialize the MNClient. """ self.context = context self.service = service self.endpoint = endpoint self.can_send = True self._proto_prefix = ['', CLIENT_PROTO, service] self._tmo = None self.timed_out = False self._create_stream() return def _create_stream(self): """Helper for opening a stream. """ socket = self.context.socket(zmq.DEALER) ioloop = IOLoop.instance() self.stream = ZMQStream(socket, ioloop) self.stream.on_recv(self._on_message) self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.connect(self.endpoint) def shutdown(self): """Method to deactivate the client connection completely. Will delete the stream and the underlying socket. .. warning:: The instance MUST not be used after :func:`shutdown` has been called. :rtype: None """ if not self.stream: return self.stream.socket.close() self.stream.close() self.stream = None return def request(self, msg, timeout=None): """Send the given message. :param msg: message parts to send. :type msg: list of str :param timeout: time to wait in milliseconds. :type timeout: int :rtype None: """ if not self.can_send: raise InvalidStateError() if isinstance(msg, bytes): msg = [msg] # prepare full message # to_send = self._proto_prefix[:] # to_send.extend(msg) to_send = msg if self.stream.closed(): self._create_stream() # TODO check this self.stream.send_multipart(to_send) self.can_send = False if timeout: self._start_timeout(timeout) return def _on_timeout(self): """Helper called after timeout. """ self.timed_out = True self._tmo = None self.on_timeout() return def _start_timeout(self, timeout): """Helper for starting the timeout. :param timeout: the time to wait in milliseconds. :type timeout: int """ self._tmo = DelayedCallback(self._on_timeout, timeout) self._tmo.start() return def _on_message(self, msg): """Helper method called on message receive. :param msg: list of message parts. :type msg: list of str """ if self._tmo: # disable timout self._tmo.stop() self._tmo = None # setting state before invoking on_message, so we can request from there self.can_send = True self.on_message(msg) return def on_message(self, msg): """Public method called when a message arrived. .. note:: Does nothing. Should be overloaded! """ pass def on_timeout(self): """Public method called when a timeout occurred. .. note:: Does nothing. Should be overloaded! """ pass
class GrainBinService(object): SERVICE_NAME = 'grain_bin' def __init__(self, context, device_id, address): self.context = context self.address = address self.device_id = device_id self.socket = None # delay = seconds_till_next('hour', duration=1) + 10 # 10 second buffer delay = 120 # 2 minutes after startup self.updater = DelayedCallback(self.update, delay * 1000) self.updater.start() return def shutdown(self): self.updater.stop() return def update(self): self.updater.stop() if internal_dealer.check_connection(): logger.info("Sending update for all grainbins") socket = self.context.socket(zmq.REQ) socket.setsockopt(zmq.LINGER, 0) socket.connect(self.address) update = GrainbinUpdate(self.device_id, 'farm_monitor') update.create() message = [pickle.dumps(update)] reply = mdp_request(socket, 'grainbin', message, 5) if reply: message = pickle.loads(reply[1]) logger.debug("reply from farm monitor for update: {0}".format(message.reply)) # delay = seconds_till_next('hour', duration=1) + 10 # 10 second buffer delay = 60 * 60 # send next update in one hour self.updater = DelayedCallback(self.update, delay * 1000) self.updater.start() return def update_individual(self, bus_number): if internal_dealer.check_connection(): logger.info("Sending update for grainbin: {0}".format(bus_number)) socket = self.context.socket(zmq.REQ) socket.setsockopt(zmq.LINGER, 0) socket.connect(self.address) update = GrainbinUpdate(self.device_id, 'farm_monitor') update.create(bus_number=bus_number) message = [pickle.dumps(update)] reply = mdp_request(socket, 'grainbin', message, 5) if reply: message = pickle.loads(reply[1]) logger.debug("reply from farm monitor for update: {0}".format(message.reply)) return def rescan(self, bus_number): logger.info("sending rescan for grainbin: {0}".format(bus_number)) info = GrainbinInfo(self.device_id, 'farm_monitor') info.create() message = [pickle.dumps(info)] socket = self.context.socket(zmq.REQ) socket.setsockopt(zmq.LINGER, 0) socket.connect(self.address) reply = mdp_request(socket, 'grainbin', message, 5) if reply: message = pickle.loads(reply[1]) logger.debug("reply from farm monitor for rescan: {0}".format(message.reply)) return
class MDPWorker(object): """Class for the MDP worker side. Thin encapsulation of a zmq.XREQ socket. Provides a send method with optional timeout parameter. Will use a timeout to indicate a broker failure. """ _proto_version = b'MDPW01' # TODO: integrate that into API HB_INTERVAL = 1000 # in milliseconds HB_LIVENESS = 3 # HBs to miss before connection counts as dead def __init__(self, context, endpoint, service): """Initialize the MDPWorker. context is the zmq context to create the socket from. service is a byte-string with the service name. """ self.context = context self.endpoint = endpoint self.service = service self.stream = None self._tmo = None self.need_handshake = True self.ticker = None self._delayed_cb = None self._create_stream() return def _create_stream(self): """Helper to create the socket and the stream. """ socket = self.context.socket(zmq.XREQ) ioloop = IOLoop.instance() self.stream = ZMQStream(socket, ioloop) self.stream.on_recv(self._on_message) self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.connect(self.endpoint) self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL) self._send_ready() self.ticker.start() return def _send_ready(self): """Helper method to prepare and send the workers READY message. """ ready_msg = [ b'', self._proto_version, chr(1), self.service ] self.stream.send_multipart(ready_msg) self.curr_liveness = self.HB_LIVENESS return def _tick(self): """Method called every HB_INTERVAL milliseconds. """ self.curr_liveness -= 1 ## print '%.3f tick - %d' % (time.time(), self.curr_liveness) self.send_hb() if self.curr_liveness >= 0: return print '%.3f lost connection' % time.time() # ouch, connection seems to be dead self.shutdown() # try to recreate it self._delayed_cb = DelayedCallback(self._create_stream, 5000) self._delayed_cb.start() return def send_hb(self): """Construct and send HB message to broker. """ msg = [ b'', self._proto_version, chr(4) ] self.stream.send_multipart(msg) return def shutdown(self): """Method to deactivate the worker connection completely. Will delete the stream and the underlying socket. """ if self.ticker: self.ticker.stop() self.ticker = None if not self.stream: return self.stream.socket.close() self.stream.close() self.stream = None self.timed_out = False self.need_handshake = True self.connected = False return def reply(self, msg): """Send the given message. msg can either be a byte-string or a list of byte-strings. """ ## if self.need_handshake: ## raise ConnectionNotReadyError() # prepare full message to_send = self.envelope self.envelope = None if isinstance(msg, list): to_send.extend(msg) else: to_send.append(msg) self.stream.send_multipart(to_send) return def _on_message(self, msg): """Helper method called on message receive. msg is a list w/ the message parts """ # 1st part is empty msg.pop(0) # 2nd part is protocol version # TODO: version check proto = msg.pop(0) # 3nd part is message type msg_type = msg.pop(0) # XXX: hardcoded message types! # any message resets the liveness counter self.need_handshake = False self.curr_liveness = self.HB_LIVENESS if msg_type == '\x05': # disconnect print ' DISC' self.curr_liveness = 0 # reconnect will be triggered by hb timer elif msg_type == '\x02': # request # remaining parts are the user message envelope, msg = split_address(msg) envelope.append(b'') envelope = [ b'', self._proto_version, '\x03'] + envelope # REPLY self.envelope = envelope self.on_request(msg) else: # invalid message # ignored pass return def on_request(self, msg): """Public method called when a request arrived. Must be overloaded! """ pass
class MDPWorker(ZMQStream): def __init__(self, broker, service, io_loop=None): """Create and setup an MDP worker. @param broker A string containing the broker's URL @param service A string containing the service name @param io_loop An existing I/O loop object. If None, the default will be used. """ self.service=service self._broker = broker self.ctx = zmq.Context() sock = self.ctx.socket(zmq.DEALER) ZMQStream.__init__(self, sock, io_loop) # last watchdog timer tick self.watchdog = 0 # connection callback one-shot self._conncb = DelayedCallback(self.send_ready, 3000, self.io_loop) # heartbeat callback..runs continuous when connected self._hbcb = PeriodicCallback(self.send_heartbeat, 2000, self.io_loop) # number of connection attempts self._conn_attempt = 0 # waiting to connect state self._waiting_to_connect = True # have we been disconnected? (flags a reconnect attempt) self.disconnected = False # connect the socket and send a READY when the io_loop starts self.connect(self._broker) self._conncb.start() def reset_watchdog(self): """Private method used to reset the HEARTBEAT watchdog """ self.watchdog = time.time() def disconnect(self): """Disconnect from the broker. """ logging.info("Disconnected from broker") self.on_recv(None) # stop message processing self._conncb.stop() # stop any pending reconnect self._hbcb.stop() # stop heartbeats self.disconnected = True self.io_loop.stop() # stop the I/O loop. If it's used by something else, the caller can restart it def reconnect(self): """Try to reconnect to the broker. """ if self.disconnected: # don't try and reconnect, we got an explicit disconnect return logging.info("Attempting to reconnect to broker") self._hbcb.stop() self._conn_attempt = 0 self._waiting_to_connect = True try: self.connect(self._broker) except ZMQError: logging.exception() self.io_loop.stop() return self._conncb.start() def send_ready(self): """Send a READY message. """ if not self._waiting_to_connect: # connected already return if self.disconnected: # don't try and connect, we got an explicit disconnect return logging.debug("Sending READY") if self._conn_attempt >= 10: logging.error("10 connection attempts have failed. Giving up.") return self._conn_attempt += 1 logging.debug("Connection attempt %i" % self._conn_attempt) rdy = [b'', MDPW_VER, b'\x01', self.service] self.on_recv(self.on_message) self.send_multipart(rdy) # There is no reply to READY so # we must assume we are connected unless we see a DISCONNECT self._waiting_to_connect = False self._disconed = False self.reset_watchdog() self._hbcb.start() def send_reply(self, client, msg, partial=False): """Send a reply to a client. This is typically called from on_request() @param client The client identifier as passed to on_request() @param msg The message to send to the client. If this is a list, it's appended to the multipart; otherwise it is converted to a string and sent as a single frame. @param partial If this is True, the message is sent as a PARTIAL and at least one more call must be made to send_reply(). Otherwise a FINAL is sent and not more calls should be made to send_reply() until another request is processed. """ self._hbcb.stop() # stop while sending other messages if partial: rply = [b'', MDPW_VER, b'\x03', client, b''] else: rply = [b'', MDPW_VER, b'\x04', client, b''] if isinstance(msg, list): rply.extend(msg) else: rply.append(msg) try: logging.debug("Sending reply: %s" % msg_str(rply)) self.send_multipart(rply) except BaseException,e: logging.error("Error sending reply: " % e) self._hbcb.start() # restart heartbeats
class DeviceConnection(object): SERVICE_NAME = 'device' # service to connect to TIMEOUT = 5 # time to wait for answer in seconds # Number of connections to try before restarting when in incorrect state CONNECTION_ATTEMPS = 3 HB_INTERVAL = 1000 * 10 # in milliseconds def __init__(self, context, device_id, address): self.context = context self.device_id = device_id self.address = address self.ticker = None self.updater = None self.socket = None self.can_send = False self._restart() self.heartbeat() return def _restart(self): self.shutdown() self.socket = self.context.socket(zmq.REQ) self.socket.setsockopt(zmq.LINGER, 0) self.socket.connect(self.address) self.can_send = True self.connection_attempts = 0 self.device_state = 'unknown' self.ticker = PeriodicCallback(self.heartbeat, DeviceConnection.HB_INTERVAL) self.ticker.start() # delay = seconds_till_next('hour', duration=1) + 2 # 2 second buffer delay = 60 # send update one minute after startup self.updater = DelayedCallback(self.update, delay * 1000) self.updater.start() return def shutdown(self): if self.ticker: self.ticker.stop() self.ticker = None if self.updater: self.updater.stop() if self.socket: self.socket.close() self.socket = None self.can_send = False return def send(self, message): if not self.can_send: self.connection_attempts += 1 logger.error("DeviceConnection is not in state to send") return None else: self.connection_attempts = 0 self.can_send = False logger.debug("DeviceConnection sending {0}".format(message)) reply = mdp_request(self.socket, DeviceConnection.SERVICE_NAME, message, DeviceConnection.TIMEOUT) if reply: logger.debug("DeviceConnection reply received: {0}".format(reply)) self.can_send = True return reply else: # Timeout! Will be restarted at next heartbeat logger.warn("DeviceConnection timeout. Will be restarted at next heartbeat") self.connection_attempts = DeviceConnection.CONNECTION_ATTEMPS return None def heartbeat(self): if self.connection_attempts >= DeviceConnection.CONNECTION_ATTEMPS: logger.warn("DeviceConnection attempts max reached. Restarting connection.") self._restart() message = HeartbeatMessage(self.device_id) reply = self.send([pickle.dumps(message)]) if reply: message = pickle.loads(reply[1]) if message.response == self.device_state: # device state has not changed pass elif message.response == 'unknown': logger.info("connection status set to 'unknown'") self.device_state = 'unknown' elif message.response == 'joined': logger.info("connection status set to 'joined'") self.device_state = 'joined' elif message.response == 'connected': logger.info("connection status set to 'connected'") self.device_state = 'connected' else: self._restart() return def create(self): logger.info("Sending create details for device") device_details = DeviceInfo(self.device_id, 'farm_monitor') device_details.create() message = [pickle.dumps(device_details)] reply = self.send(message) if reply: message = pickle.loads(reply[1]) logger.debug("Update response from {0} :{1}".format(message.source, message.reply)) return def update(self): self.updater.stop() if self.device_state == 'connected': logger.info("Sending update for device") update = DeviceUpdate(self.device_id, 'farm_monitor') update.create() reply = self.send([pickle.dumps(update)]) if reply: message = pickle.loads(reply[1]) logger.debug("Update response from {0} :{1}".format(message.source, message.reply)) # delay = seconds_till_next('hour', duration=1) + 2 # 2 second buffer delay = 60 * 60 # send next update in 60 minutes self.updater = DelayedCallback(self.update, delay * 1000) self.updater.start() return def backup(self): backup_index = create_backup() session = Session() backup = session.query(Backup).filter_by(index=backup_index).first() standalone_configuration = session.query(SystemSetup.standalone_configuration).scalar() if self.device_state == 'connected': failed_attempts = 0 # don't send the file if we are a combined configuration if standalone_configuration: logger.info("Sending device backup file to FarmMonitor") file_size = os.path.getsize(backup.filepath) bytes_sent = 0 zip_file = open(backup.filepath, 'r') while bytes_sent < file_size and failed_attempts < 3: zip_file.seek(bytes_sent) data = zip_file.read(512) logger.debug("Sending {0} bytes of backup file".format(len(data))) message = FileMessage(self.device_id, 'farm_monitor', 'backup') message.set_file(backup.filepath, data) reply = self.send([pickle.dumps(message)]) if reply: # only mark the data received if a reply is recieved bytes_sent += len(data) failed_attempts = 0 reply_message = pickle.loads(reply[1]) logger.debug("Update reply: {0}".format(reply_message.reply)) else: logger.warn("Failed attempt in sending backup file") failed_attempts += 1 zip_file.close() # only send the database object if the file was sent correctly if failed_attempts == 0: logger.info("Sending device backup database object to FarmMonitor") message = FileMessage(self.device_id, 'farm_monitor', 'backup') message.set_db_object(pickle.dumps(backup)) reply = self.send([pickle.dumps(message)]) if reply: reply_message = pickle.loads(reply[1]) logger.debug("Update response: {0}".format(reply_message.reply)) else: logger.warn("To many failed attempts. Backup failed to send to Farm Monitor") session.close() return def perform_update(self, update_id): session = Session() standalone_configuration = session.query(SystemSetup.standalone_configuration).scalar() if self.device_state == 'connected': # request the update db_object logger.info("Requesting update with id: {0} from FarmMonitor".format(update_id)) message = [self.device_id, 'perform_update', 'db_object', update_id] message = FileMessage(self.device_id, 'farm_monitor', 'perform_update') message.set_db_object(update_id) reply = self.send([pickle.dumps(message)]) if reply: reply_message = pickle.loads(reply[1]) logger.debug("Update response from farm_monitor") if reply_message.data: update = pickle.loads(reply_message.data) else: logger.warn("incorrect response in perform_update: {0}".format(reply[0])) session.close() return failed_attempts = 0 # don't request the file if we are a combined configuration if standalone_configuration: if not os.path.exists(update.filepath): os.makedirs(update.filepath) bytes_received = 0 more_to_send = True new_file = open(update.filepath + update.filename, 'a') while more_to_send and failed_attempts < 3: logger.info("Requesting update file from FarmMonitor") message = FileMessage(self.device_id, 'farm_monitor', 'perform_update') message.request_file(update_id, str(bytes_received)) reply = self.send([pickle.dumps(message)]) if reply: reply_message = pickle.loads(reply[1]) logger.debug("Update response from FarmMonitor") failed_attempts = 0 # in this case, reply_message.filepath is really a boolean. # reusing the FileMessage object more_to_send = reply_message.filepath data = reply_message.data new_file.seek(bytes_received) new_file.write(data) bytes_received += len(data) else: logger.warn("Failed attempt in requesting update file") failed_attempts += 1 new_file.close() # compare the hash of the file sha256 = hashlib.sha256() with open(update.filepath + update.filename, 'rb') as f: for block in iter(lambda: f.read(2048), b''): sha256.update(block) if update.sha256 == sha256.hexdigest(): logger.info("Update file successfully received") print('update received good') session.merge(update) session.commit() else: logger.warn("Update file hash does not match. Update failed") session.close() return # now perform the actual update run_update(update.id, 'farm_device') else: logger.warn("No reply when requesting update db_object from FarMonitor.") session.close() return
def _add_callback(future): success, _, _ = future.result() if not success: cb = self.uart_cb[path] = DelayedCallback( _check_device, 500, self.ioloop) cb.start()
class Client(object): def __init__(self, context, endpoint, service): """Initialize the Client. """ socket = context.socket(zmq.DEALER) self.ioloop = IOLoop.instance() self.service = service.encode('utf-8') self.endpoint = endpoint self.stream = ZMQStream(socket, self.ioloop) self.stream.on_recv(self._on_message) self.can_send = True self._proto_prefix = [MDP_CLIENT_VERSION, b'\x01', service.encode('utf-8')] self._tmo = None self.timed_out = False self.multicast_services = None self.multicast_already_received = [] socket.connect(endpoint) return def shutdown(self): """Method to deactivate the client connection completely. Will delete the stream and the underlying socket. .. warning:: The instance MUST not be used after :func:`shutdown` has been called. :rtype: None """ if not self.stream: return self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.socket.close() self.stream.close() self.stream = None return def request(self, msg, timeout=None): """Send the given message. :param msg: message parts to send. :type msg: list of str :param timeout: time to wait in milliseconds. :type timeout: int | float :rtype None: """ if not self.can_send: raise InvalidStateError() try: if type(msg) in (bytes, unicode): # python 2 msg = [msg] except NameError: if type(msg) == bytes: # python 3 msg = [msg] # prepare full message to_send = [b''] to_send.extend(self._proto_prefix[:]) to_send.extend(msg) self.stream.send_multipart(to_send) self.can_send = False self.multicast_services = None if timeout: self._start_timeout(timeout) return def rpc(self, method, args=None, kwargs=None, timeout=None): """Call RPC method :param method: name of method :type method: str :param args: list of args :type args: list :param kwargs: list of kwargs :type kwargs: dict :param timeout: time to wait in milliseconds. :type timeout: int | float :rtype None: """ data = [method.encode('utf-8'), msgpack.packb([] if args is None else args), msgpack.packb({} if kwargs is None else kwargs)] return self.request(data, timeout) def _on_message(self, message): """Helper method called on message receive. :param message: list of message parts. :type message: list of str """ message.pop(0) # remove empty string protocol_version = message.pop(0) if protocol_version != MDP_WORKER_VERSION: # version check, ignore old versions return message_type = message.pop(0) if message_type == b'\x02': # partial message worker = message.pop(0) try: msg = msgpack.unpackb(message[0]) except: msg = message[0] self.on_partial_message(worker, msg) elif message_type == b'\x03': # final message worker = message.pop(0) try: msg = msgpack.unpackb(message[0]) except: msg = message[0] self.on_message(worker, msg) if message[0] not in self.multicast_already_received: self.multicast_already_received.append(worker) if self.multicast_services is None or len(self.multicast_already_received) == len(self.multicast_services): self.stop_waiting() elif message_type == b'\x04': # error message worker = message.pop(0) self.on_error_message(worker, message[0]) self.stop_waiting() elif message_type == b'\x05': # multicast start, load list of multicast services self.multicast_services = message self.multicast_already_received = [] self.on_multicast_start(message) elif message_type == b'\x06': # exception message worker = message.pop(0) try: msg = msgpack.unpackb(message[0]) except: msg = {b'class': 'Exception', b'message': 'parsing failed', b'traceback': message[0]} self.on_exception_message(worker, msg[b'class'], msg[b'message'], msg[b'traceback']) if message[0] not in self.multicast_already_received: self.multicast_already_received.append(worker) if self.multicast_services is None or len(self.multicast_already_received) == len(self.multicast_services): self.stop_waiting() #else: unknown type - do nothing return def wait_for_reply(self): """Start waiting for replies """ self.ioloop.start() def stop_waiting(self): """Stop waiting for replies and cancel timeout """ self.ioloop.stop() self.can_send = True self._stop_timeout() def _on_timeout(self): """Helper called after timeout. """ self.timed_out = True self._tmo = None self.ioloop.stop() self.on_timeout() return def _start_timeout(self, timeout): """Helper for starting the timeout. :param timeout: the time to wait in milliseconds. :type timeout: int """ self._tmo = DelayedCallback(self._on_timeout, timeout) self._tmo.start() return def _stop_timeout(self): """Helper for stopping timeout event """ if self._tmo: self._tmo.stop() self._tmo = None def on_message(self, service, msg): """Public method called when a message arrived. .. note:: Does nothing. Should be overloaded! :param service: name of the worker :type service: str :param msg: message :type msg: mixed """ pass def on_partial_message(self, service, msg): """Public method called when a partial message arrived. .. note:: Does nothing. Should be overloaded! :param service: name of the worker :type service: str :param msg: message :type msg: mixed """ pass def on_error_message(self, worker, msg): """Public method called when an error message arrived. .. note:: Does nothing. Should be overloaded! :param service: name of the worker :type service: str :param msg: message :type msg: str """ pass def on_exception_message(self, service, cls, message, traceback): """Public method called when an exception arrived. .. note:: Does nothing. Should be overloaded! :param service: name of the worker :type service: str :param cls: exception class name :type cls: str :param message: error message :type message: str :param traceback: traceback :type traceback: str """ pass def on_timeout(self): """Public method called when a timeout occured. .. note:: Does nothing. Should be overloaded! """ pass def on_multicast_start(self, services): """Public method called when multicast request started .. note:: Does nothing. Should be overloaded! :param services: list of services in multicast group :type services: list """ pass
class Worker(object): """Class for the MDP worker side. Thin encapsulation of a zmq.DEALER socket. Provides a send method with optional timeout parameter. Will use a timeout to indicate a broker failure. """ max_forks = 10 ipc = 'ipc:///tmp/zmq-rpc-'+str(uuid4()) HB_INTERVAL = 1000 # in milliseconds HB_LIVENESS = 3 # HBs to miss before connection counts as dead def __init__(self, context, endpoint, service, multicasts=()): """Initialize the MDPWorker. :param context: is the zmq context to create the socket from :type context: zmq.Context :param service: service name - you can put hostname here :type service: str :param multicasts: list of groups to subscribe :type multicasts: list """ self.context = context self.endpoint = endpoint self.service = service.encode('utf-8') # convert to byte-string - required in python 3 self.multicasts = [m.encode('utf-8') for m in multicasts] # convert to byte-string self.stream = None self._tmo = None self.need_handshake = True self.ticker = None self._delayed_cb = None self._create_stream() self.forks = [] self.curr_liveness = self.HB_LIVENESS socket = self.context.socket(zmq.ROUTER) socket.bind(self.ipc) self.stream_w = ZMQStream(socket) self.stream_w.on_recv(self._on_fork_response) self.reply_socket = None return def _create_stream(self): """Helper to create the socket and the stream. """ self.on_log_event('broker.connect', 'Trying to connect do broker') socket = self.context.socket(zmq.DEALER) ioloop = IOLoop.instance() self.stream = ZMQStream(socket, ioloop) self.stream.on_recv(self._on_message) self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.connect(self.endpoint) self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL) self._send_ready() for m in self.multicasts: self._register_worker_to_multicast(m) self.ticker.start() return def _tick(self): """Method called every HB_INTERVAL milliseconds. """ self.curr_liveness -= 1 self.send_hb() if self.curr_liveness >= 0: return # ouch, connection seems to be dead self.on_log_event('broker.timeout', 'Connection to broker timeouted, disconnecting') self.shutdown(False) # try to recreate it self._delayed_cb = DelayedCallback(self._create_stream, 5000) self._delayed_cb.start() return def send_hb(self): """Construct and send HB message to broker. """ msg = [b'', MDP_WORKER_VERSION, b'\x05'] self.stream.send_multipart(msg) return def shutdown(self, final=True): """Method to deactivate the worker connection completely. Will delete the stream and the underlying socket. :param final: if shutdown is final and we want to close all sockets :type final: bool """ if self.ticker: self.ticker.stop() self.ticker = None if not self.stream: return self.stream.on_recv(None) self.disconnect() self.stream.socket.close() self.stream.close() self.stream = None self.need_handshake = True if final: self.stream_w.socket.close() self.stream_w.close() self.stream = None return def disconnect(self): """Helper method to send the workers DISCONNECT message. """ self.stream.socket.send_multipart([b'', MDP_WORKER_VERSION, b'\x06' ]) self.curr_liveness = self.HB_LIVENESS return def _send_ready(self): """Helper method to prepare and send the workers READY message. """ self.on_log_event('broker.ready', 'Sending ready to broker.') ready_msg = [b'', MDP_WORKER_VERSION, b'\x01', self.service] self.stream.send_multipart(ready_msg) self.curr_liveness = self.HB_LIVENESS return def _register_worker_to_multicast(self, name): """Helper method to register worker to multicast group :param name: group name :type name: str """ self.on_log_event('broker.register-group', 'Subscribing to group \'%s\'.' % name) reg_msg = [b'', MDP_WORKER_VERSION, b'\x07', name] self.stream.send_multipart(reg_msg) self.curr_liveness = self.HB_LIVENESS return def _on_message(self, msg): """Helper method called on message receive. :param msg: message parts :type msg: list """ # 1st part is empty msg.pop(0) # 2nd part is protocol version protocol_version = msg.pop(0) if protocol_version != MDP_WORKER_VERSION: # version check, ignore old versions return # 3rd part is message type msg_type = msg.pop(0) # any message resets the liveness counter self.need_handshake = False self.curr_liveness = self.HB_LIVENESS if msg_type == b'\x06': # disconnect self.curr_liveness = 0 # reconnect will be triggered by hb timer elif msg_type == b'\x02': # request # remaining parts are the user message addresses, msg = self.split_address(msg) self._on_request(addresses, msg) elif msg_type == b'\x05': # received hardbeat - timer handled above pass else: # invalid message ignored pass return def _on_fork_response(self, to_send): """Helper method to send message from forked worker. This message will be received by main worker process and resend to broker. :param to_send address and data to send :type to_send list """ self.stream.send_multipart(to_send) return def send_reply(self, addresses, msg, partial=False, exception=False): """Send reply from forked worker process. This method can be called only from do_work() method! This method will send messages to main worker listening on local socket in /tmp/zmq-rpc-... :param addresses: return address stack :type addresses: list of str :param msg: return value from called method :type msg: mixed :param partial: if the message is partial or final :type partial: bool :param exception: if the message is exception, msg format is: {'class':'c', 'message':'m', 'traceback':'t'} :type exception: bool """ if not self.reply_socket: context = zmq.Context() self.reply_socket = context.socket(zmq.DEALER) self.reply_socket.connect(self.ipc) msg = msgpack.Packer().pack(msg) if exception: to_send = [b'', MDP_WORKER_VERSION, b'\x08'] elif partial: to_send = [b'', MDP_WORKER_VERSION, b'\x03'] else: to_send = [b'', MDP_WORKER_VERSION, b'\x04'] to_send.extend(addresses) to_send.append(b'') if isinstance(msg, list): to_send.extend(msg) else: to_send.append(msg) m = self.reply_socket.send_multipart(to_send, track=True, copy=False) m.wait() if not partial: self.reply_socket.close() self.reply_socket = None return def send_message(self, addresses, msg, partial=False, error=False): """Send response message from main worker process. Please do not call this method from do_work() :param addresses: return address stack :type addresses: list of str :param msg: return value from called method :type msg: mixed :param partial: if the message is partial or final :type partial: bool :param error: if the message is error :type error: bool """ to_send = [b'', MDP_WORKER_VERSION] if partial: to_send.append(b'\x03') elif error: to_send.append(b'\x09') else: to_send.append(b'\x04') to_send.extend(addresses) to_send.append(b'') if isinstance(msg, list): to_send.extend(msg) else: to_send.append(msg) self.stream.send_multipart(to_send) return def _on_request(self, addresses, message): """Helper method called on RPC message receive. """ # remove finished forks self._remove_finished_processes() # test max forks if len(self.forks) >= self.max_forks: self.send_message(addresses, b'max workers limit exceeded', error=True) self.on_max_forks(addresses, message) return name = message[0] args = msgpack.unpackb(message[1]) kwargs = msgpack.unpackb(message[2]) p = Process(target=self.do_work, args=(addresses, name, args, kwargs)) p.start() p._args = None # free memory self.forks.append(p) return def _remove_finished_processes(self): """Helper method dedicated to cleaning list of forked workers """ for f in [f for f in self.forks if not f.is_alive()]: self.forks.remove(f) return def split_address(self, msg): """Function to split return Id and message received by ROUTER socket. Returns 2-tuple with return Id and remaining message parts. Empty frames after the Id are stripped. """ ret_ids = [] for i, p in enumerate(msg): if p: ret_ids.append(p) else: break return ret_ids, msg[i + 1:] def on_log_event(self, event, message): """Override this method if you want to log events from broker :type event: str :param event: event type - used for filtering :type message: str :param message: log message :rtype: None """ pass def on_max_forks(self, addresses, message): """This method is called when max_forks limit is reached You can override this method. """ pass def do_work(self, addresses, name, args, kwargs): """Main method responsible for handling rpc calls, and sending response messages. Please override this method! :param addresses: return address stack :type addresses: list of str :param name: name of task :type name: str :param args: positional task arguments :type args: list :param kwargs: key-value task arguments :type kwargs: dict """ # this is example of simple response message self.send_reply(addresses, 'method not implemented') # and send message to main worker # you can also send partial message and exception - read 'send_reply' docs return
class MQRep(object): """Class for the MDP worker side. Thin encapsulation of a zmq.DEALER socket. Provides a send method with optional timeout parameter. Will use a timeout to indicate a broker failure. """ _proto_version = b'MDPW01' # TODO: integrate that into API HB_INTERVAL = 1000 # in milliseconds HB_LIVENESS = 3 # HBs to miss before connection counts as dead def __init__(self, context, service): """Initialize the MDPWorker. context is the zmq context to create the socket from. service is a byte-string with the service name. """ if DEBUG: print("MQRep > __init__") cfg = Loader('mq').load() config = dict(cfg[1]) if config['ip'].strip() == "*": config['ip'] = get_ip() self.endpoint = "tcp://{0}:{1}".format(config['ip'], config['req_rep_port']) self.context = context self.service = service self.stream = None self._tmo = None self.need_handshake = True self.ticker = None self._delayed_cb = None self._create_stream() ### patch fritz self._reconnect_in_progress = False ### end patch fritz return def _create_stream(self): """Helper to create the socket and the stream. """ if DEBUG: print("MQRep > _create_stream") socket = ZmqSocket(self.context, zmq.DEALER) ioloop = IOLoop.instance() self.stream = ZMQStream(socket, ioloop) self.stream.on_recv(self._on_mpd_message) self.stream.socket.setsockopt(zmq.LINGER, 0) self.stream.connect(self.endpoint) if self.ticker != None: if DEBUG: print("MQRep > _create_stream - stop ticker") self.ticker.stop() self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL) self._send_ready() self.ticker.start() return def _send_ready(self): """Helper method to prepare and send the workers READY message. """ if DEBUG: print("MQREP > _send_ready") ready_msg = [ b'', self._proto_version, b'\x01', self.service ] self.stream.send_multipart(ready_msg) self.curr_liveness = self.HB_LIVENESS if DEBUG: print("MQREP > _send_ready > curr_liveness <= {0}".format(self.HB_LIVENESS)) return def _tick(self): """Method called every HB_INTERVAL milliseconds. """ if DEBUG: print("MQREP > _tick") self.curr_liveness -= 1 if DEBUG: print('MQREP > _tick - {0} tick = {1}'.format(time.time(), self.curr_liveness)) self.send_hb() if self.curr_liveness >= 0: return if DEBUG: print('MQREP > _tick - {0} lost connection'.format(time.time())) # ouch, connection seems to be dead self.shutdown() # try to recreate it self._delayed_cb = DelayedCallback(self._create_stream, self.HB_INTERVAL) self._delayed_cb.start() return def send_hb(self): """Construct and send HB message to broker. """ msg = [ b'', self._proto_version, b'\x04' ] self.stream.send_multipart(msg) return def shutdown(self): """Method to deactivate the worker connection completely. Will delete the stream and the underlying socket. """ if self.ticker: self.ticker.stop() self.ticker = None if not self.stream: return self.stream.socket.close() self.stream.close() self.stream = None self.timed_out = False self.need_handshake = True self.connected = False return def reply(self, msg): """Send the given message. msg can either be a byte-string or a list of byte-strings. """ ## if self.need_handshake: ## raise ConnectionNotReadyError() # prepare full message to_send = self.envelope self.envelope = None if isinstance(msg, list): to_send.extend(msg) else: to_send.append(msg) self.stream.send_multipart(to_send) return def _on_mpd_message(self, msg): """Helper method called on message receive. msg is a list w/ the message parts """ if DEBUG: print("MQRep > _on_mpd_message : {0} - {1}".format(time.strftime("%H:%M:%S"), msg)) # 1st part is empty msg.pop(0) # 2nd part is protocol version # TODO: version check proto = msg.pop(0) # 3rd part is message type msg_type = msg.pop(0) # XXX: hardcoded message types! # any message resets the liveness counter self.need_handshake = False self.curr_liveness = self.HB_LIVENESS if DEBUG: print("MQREP > _on_mpd_message > curr_liveness <= {0}".format(self.HB_LIVENESS)) if msg_type == b'\x05': # disconnect if DEBUG: print("MQREP > _on_mpd_message > type x05 : disconnect") self.curr_liveness = 0 # reconnect will be triggered by hb timer elif msg_type == b'\x02': # request if DEBUG: print("MQREP > _on_mpd_message > type x02 : request") # remaining parts are the user message envelope, msg = split_address(msg) envelope.append(b'') envelope = [ b'', self._proto_version, b'\x03'] + envelope # REPLY self.envelope = envelope mes = MQMessage() mes.set(msg) #print("MQRep > before self.on_mdp_request") #print(self.on_mdp_request) #print(mes) try: self.on_mdp_request(mes) except: print("ERROR {0}".format(traceback.format_exc())) else: if DEBUG: print("MQREP > _on_mpd_message > type ??? : invalid or hbeat") # invalid message # ignored # if \x04, this is a hbeat message pass return def on_mdp_request(self, msg): """Public method called when a request arrived. Must be overloaded! """ pass