def send_checkup():
def checkup_timeout():
global srv, responding, timeout
timeout.stop()
if not responding:
# we've timed out, restart
# TODO: provide config var for how many times to attempt start before exiting
print('{0} not responding, attempting start.'.format(MODULE))
srv = launch_service()
def recv_checkup(msg):
global responding
responding = True
# access globals
global timeout, checkup, responding, loop
# listen for ping back
checkup.on_recv(recv_checkup)
# do what's needed to rescue on timeout
responding = False
timeout = DelayedCallback(checkup_timeout, CHECKUP_TIMEOUT, io_loop=loop)
timeout.start()
# send ping
checkup.send('You alive?')
def send_checkup():
def checkup_timeout():
global service, responding, timeout
timeout.stop()
if not responding:
# we've timed out, restart
print('Mongrel2 not responding, attempting restart.')
# since control port isn't responding, do a dirty kill just in case
kill_mongrel_with_pid(M2_PID_PATH)
# start her back up
# TODO: add configurable delay here before starting again
start_mongrel()
def recv_response(msg):
global responding
responding = True
# access globals
global timeout, control_port, responding
# listen for ping back
control_port.on_recv(recv_response)
# do what's needed to rescue on timeout
responding = False
timeout = DelayedCallback(checkup_timeout, CHECKUP_TIMEOUT, io_loop=loop)
timeout.start()
# send status request
control_port.send(tnetstrings.dump(['status', {'what': 'net'}]))
def call(self, proc_name, args=[], kwargs={}, result='async', timeout=None): #{
"""
Call the remote method with *args and **kwargs.
Parameters
----------
proc_name : <str> name of the remote procedure to call
args : <tuple> positional arguments of the procedure
kwargs : <dict> keyword arguments of the procedure
ignore : <bool> whether to ignore result or wait for it
timeout : <float> | None
Number of seconds to wait for a reply.
RPCTimeoutError is set as the future result in case of timeout.
Set to None, 0 or a negative number to disable.
Returns
-------
<Future> if result is 'async'
None if result is 'ignore'
If remote call fails:
- sets <RemoteRPCError> into the <Future> if result is 'async'
"""
assert result in ('async', 'ignore'), \
'expected any of "async", "ignore" -- got %r' % result
if not (timeout is None or isinstance(timeout, (int, float))):
raise TypeError("timeout param: <float> or None expected, got %r" % timeout)
ignore = result == 'ignore'
req_id, msg_list = self._build_request(proc_name, args, kwargs, ignore)
self.socket.send_multipart(msg_list)
if ignore:
return None
# The following logic assumes that the reply won't come back too
# quickly, otherwise the callbacks won't be in place in time. It should
# be fine as this code should run very fast. This approach improves
# latency we send the request ASAP.
def _abort_request():
future_tout = self._futures.pop(req_id, None)
if future_tout:
future, _ = future_tout
tout_msg = "Request %s timed out after %s sec" % (req_id, timeout)
#self.logger.debug(tout_msg)
future.set_exception(RPCTimeoutError(tout_msg))
timeout = timeout or 0
if timeout > 0:
tout_cb = DelayedCallback(_abort_request, int(timeout*1000), self.ioloop)
tout_cb.start()
else:
tout_cb = None
future = Future()
self._futures[req_id] = (future, tout_cb)
return future
def stop(signum, frame):
global loop, checkup_periodic, control_port, stop_timeout
def stop_timeout():
print('Terminate request timed out, mongrel2 might be orphaned.')
kill_mongrel_with_pid(M2_PID_PATH)
shutdown()
def terminate_resp(msg):
print('Mongrel2 control port confirmed SIGTERM sent.')
shutdown()
def shutdown():
print('Shutting down.')
remove_hosts(HOSTS)
control_port.close()
loop.stop()
print('\nStopping.')
# make sure checkup doesn't happen during termination
checkup_periodic.stop()
# register terminate response callback
control_port.on_recv(terminate_resp)
# prepare timeout
stop_timeout = DelayedCallback(stop_timeout, STOP_TIMEOUT, io_loop=loop)
stop_timeout.start()
# send terminate request
control_port.send(tnetstrings.dump(['terminate', {}]))
def call(self,
proc_name,
args=[],
kwargs={},
ignore=False,
timeout=None): #{
"""
Call the remote method with *args and **kwargs.
Parameters
----------
proc_name : <str> name of the remote procedure to call
args : <tuple> positional arguments of the procedure
kwargs : <dict> keyword arguments of the procedure
ignore : <bool> whether to ignore result or wait for it
timeout : <float> | None
Number of seconds to wait for a reply.
RPCTimeoutError is set as the future result in case of timeout.
Set to None, 0 or a negative number to disable.
Returns None or a <Future> representing a remote call result
"""
if not (timeout is None or isinstance(timeout, (int, float))):
raise TypeError("timeout param: <float> or None expected, got %r" %
timeout)
req_id, msg_list = self._build_request(proc_name, args, kwargs, ignore)
self.socket.send_multipart(msg_list)
if ignore:
return None
# The following logic assumes that the reply won't come back too
# quickly, otherwise the callbacks won't be in place in time. It should
# be fine as this code should run very fast. This approach improves
# latency we send the request ASAP.
def _abort_request():
future_tout = self._futures.pop(req_id, None)
if future_tout:
future, _ = future_tout
tout_msg = "Request %s timed out after %s sec" % (req_id,
timeout)
self.logger.debug(tout_msg)
future.set_exception(RPCTimeoutError(tout_msg))
timeout = timeout or 0
if timeout > 0:
tout_cb = DelayedCallback(_abort_request, int(timeout * 1000),
self.ioloop)
tout_cb.start()
else:
tout_cb = None
future = Future()
self._futures[req_id] = (future, tout_cb)
return future
def _start_timeout(self, timeout):
"""Helper for starting the timeout.
:param timeout: the time to wait in milliseconds.
:type timeout: int
"""
self._tmo = DelayedCallback(self._on_timeout, timeout)
self._tmo.start()
return
def call(self, method, callback, errback, timeout, *args, **kwargs):
"""Call the remote method with *args and **kwargs.
Parameters
----------
method : str
The name of the remote method to call.
callback : callable
The callable to call upon success or None. The result of the RPC
call is passed as the single argument to the callback:
`callback(result)`.
errback : callable
The callable to call upon a remote exception or None, The
signature of this method is `errback(ename, evalue, tb)` where
the arguments are passed as strings.
timeout : int
The number of milliseconds to wait before aborting the request.
When a request is aborted, the errback will be called with an
RPCTimeoutError. Set to 0 or a negative number to use an infinite
timeout.
args : tuple
The tuple of arguments to pass as `*args` to the RPC method.
kwargs : dict
The dict of arguments to pass as `**kwargs` to the RPC method.
"""
if not isinstance(timeout, int):
raise TypeError("int expected, got %r" % timeout)
if not (callback is None or callable(callback)):
raise TypeError("callable or None expected, got %r" % callback)
if not (errback is None or callable(errback)):
raise TypeError("callable or None expected, got %r" % errback)
msg_id, msg_list = self._build_request(method, args, kwargs)
self.stream.send_multipart(msg_list)
# The following logic assumes that the reply won't come back too
# quickly, otherwise the callbacks won't be in place in time. It should
# be fine as this code should run very fast. This approach improves
# latency we send the request ASAP.
def _abort_request():
cb_eb_dc = self._callbacks.pop(msg_id, None)
if cb_eb_dc is not None:
eb = cb_eb_dc[1]
if eb is not None:
try:
raise RPCTimeoutError()
except:
etype, evalue, tb = sys.exc_info()
eb(etype.__name__, evalue, traceback.format_exc(tb))
if timeout > 0:
dc = DelayedCallback(_abort_request, timeout, self.loop)
dc.start()
else:
dc = None
self._callbacks[msg_id] = (callback, errback, dc)
def call(self, method, callback, errback, timeout, *args, **kwargs):
"""Call the remote method with *args and **kwargs.
Parameters
----------
method : str
The name of the remote method to call.
callback : callable
The callable to call upon success or None. The result of the RPC
call is passed as the single argument to the callback:
`callback(result)`.
errback : callable
The callable to call upon a remote exception or None, The
signature of this method is `errback(ename, evalue, tb)` where
the arguments are passed as strings.
timeout : int
The number of milliseconds to wait before aborting the request.
When a request is aborted, the errback will be called with an
RPCTimeoutError. Set to 0 or a negative number to use an infinite
timeout.
args : tuple
The tuple of arguments to pass as `*args` to the RPC method.
kwargs : dict
The dict of arguments to pass as `**kwargs` to the RPC method.
"""
if not isinstance(timeout, int):
raise TypeError("int expected, got %r" % timeout)
if not (callback is None or callable(callback)):
raise TypeError("callable or None expected, got %r" % callback)
if not (errback is None or callable(errback)):
raise TypeError("callable or None expected, got %r" % errback)
msg_id, msg_list = self._build_request(method, args, kwargs)
self.stream.send_multipart(msg_list)
# The following logic assumes that the reply won't come back too
# quickly, otherwise the callbacks won't be in place in time. It should
# be fine as this code should run very fast. This approach improves
# latency we send the request ASAP.
def _abort_request():
cb_eb_dc = self._callbacks.pop(msg_id, None)
if cb_eb_dc is not None:
eb = cb_eb_dc[1]
if eb is not None:
try:
raise RPCTimeoutError()
except:
etype, evalue, tb = sys.exc_info()
eb(etype.__name__, evalue, traceback.format_exc(tb))
if timeout > 0:
dc = DelayedCallback(_abort_request, timeout, self.loop)
dc.start()
else:
dc = None
self._callbacks[msg_id] = (callback, errback, dc)
def test_01_simple_01(self):
"""Test MDPWorker simple req/reply.
"""
self._start_broker()
time.sleep(0.2)
worker = MyWorker(self.context, self.endpoint, self.service)
sender = DelayedCallback(self.send_req, 1000)
sender.start()
IOLoop.instance().start()
worker.shutdown()
self._stop_broker()
return
def restart_service():
global srv
def restart_checkup():
global checkup_periodic
checkup_periodic.start()
global loop
checkup_restart = DelayedCallback(restart_checkup,
PAUSE_AFTER_RESTART,
io_loop=loop)
service = launch_service()
checkup_restart.start()
def _tick(self):
"""Method called every HB_INTERVAL milliseconds.
"""
self.curr_liveness -= 1
## print '%.3f tick - %d' % (time.time(), self.curr_liveness)
self.send_hb()
if self.curr_liveness >= 0:
return
## print '%.3f lost connection' % time.time()
# ouch, connection seems to be dead
self.shutdown()
# try to recreate it
self._delayed_cb = DelayedCallback(self._create_stream, 5000)
self._delayed_cb.start()
return
def __init__(self, broker, service, io_loop=None):
"""Create and setup an MDP worker.
@param broker A string containing the broker's URL
@param service A string containing the service name
@param io_loop An existing I/O loop object. If None, the default will be used.
"""
self.service=service
self._broker = broker
self.ctx = zmq.Context()
sock = self.ctx.socket(zmq.DEALER)
ZMQStream.__init__(self, sock, io_loop)
# last watchdog timer tick
self.watchdog = 0
# connection callback one-shot
self._conncb = DelayedCallback(self.send_ready, 3000, self.io_loop)
# heartbeat callback..runs continuous when connected
self._hbcb = PeriodicCallback(self.send_heartbeat, 2000, self.io_loop)
# number of connection attempts
self._conn_attempt = 0
# waiting to connect state
self._waiting_to_connect = True
# have we been disconnected? (flags a reconnect attempt)
self.disconnected = False
# connect the socket and send a READY when the io_loop starts
self.connect(self._broker)
self._conncb.start()
def _tick(self):
"""Method called every HB_INTERVAL milliseconds.
"""
self.curr_retries -= 1
self.send_hb()
if self.curr_retries >= 0:
return
# connection seems to be dead
self.shutdown()
# try to recreate it
# self._delayed_cb = IOLoop.call_later(self._create_stream, 5000)
# self._delayed_cb = IOLoop.add_timeout(self._create_stream, 5000)
self._delayed_cb = DelayedCallback(self._create_stream,
self.HB_INTERVAL)
self._delayed_cb.start()
return
def _start_timeout(self, timeout):
"""Helper for starting the timeout.
:param timeout: the time to wait in milliseconds.
:type timeout: int
"""
self._tmo = DelayedCallback(self._on_timeout, timeout)
self._tmo.start()
return
def check_for_change():
global checkup, loop, checksums, responding, checkup_periodic, KEY
curr_sums = check(path, watch)
changed, deleted = get_diff(curr_sums, checksums)
if len(changed) or len(deleted):
checksums = curr_sums
print('restarting {0}.'.format(MODULE))
checkup_periodic.stop()
command.send(json.dumps({
'key': KEY,
'command': 'die'
}))
delay = DelayedCallback(restart_service,
PAUSE_BEFORE_RESTART + 300,
io_loop=loop)
delay.start()
def _recreate_stream(self):
logging.debug('Worker trying to recreate stream')
if self.endpoint_callback is not None:
#
# Check, maybe the ip of the proxy changed.
#
try:
self.endpoint, self.hb_endpoint = self.endpoint_callback()
except:
#
# Probably some problem in accessing the server.
#
self._delayed_reconnect = DelayedCallback(
self._recreate_stream, self.RECONNECT_PERIOD)
self._delayed_reconnect.start()
return
self._create_stream()
def quit_worker(raw_msg):
"""
When the worker should quit, stop the io_loop after 2 seconds.
"""
msg = MgmtMessage(raw_msg)
if ZMQ_SPYDER_MGMT_WORKER_QUIT == msg.data:
logger.info("process::We have been asked to shutdown, do so")
DelayedCallback(io_loop.stop, 2000, io_loop).start()
ack = MgmtMessage(topic=ZMQ_SPYDER_MGMT_WORKER, identity=identity,
data=ZMQ_SPYDER_MGMT_WORKER_QUIT_ACK)
mgmt._out_stream.send_multipart(ack.serialize())
def test_simple_worker(self):
"""Test MNWorker simple req/reply.
"""
self._start_broker()
time.sleep(0.2)
worker = WorkerRunner(self.context, self.endpoint, self.service)
sender = DelayedCallback(self.send_req, 500)
stopper = DelayedCallback(self.stop_loop, 2500)
sender.start()
stopper.start()
IOLoop.instance().start()
worker.shutdown()
self._stop_broker()
return
def __init__(self, context, device_id, address):
self.context = context
self.address = address
self.device_id = device_id
self.socket = None
# delay = seconds_till_next('hour', duration=1) + 10 # 10 second buffer
delay = 120 # 2 minutes after startup
self.updater = DelayedCallback(self.update, delay * 1000)
self.updater.start()
return
def _tick(self):
"""Method called every HB_INTERVAL milliseconds.
"""
self.curr_liveness -= 1
self.send_hb()
if self.curr_liveness >= 0:
return
# ouch, connection seems to be dead
self.on_log_event('broker.timeout', 'Connection to broker timeouted, disconnecting')
self.shutdown(False)
# try to recreate it
self._delayed_cb = DelayedCallback(self._create_stream, 5000)
self._delayed_cb.start()
return
def _tick(self):
"""Method called every HB_INTERVAL milliseconds.
"""
self.curr_liveness -= 1
logging.debug('Worker HB tick, current liveness: %d' %
self.curr_liveness)
self.send_hb()
if self.curr_liveness >= 0:
return
#
# Ouch, connection seems to be dead
#
logging.debug('Worker lost connection')
self.shutdown()
#
# try to recreate the connection
#
self._delayed_reconnect = DelayedCallback(self._recreate_stream,
self.RECONNECT_PERIOD)
self._delayed_reconnect.start()
def send_request(self, request, args, kwargs, handler, timeout):
"""Send a request to the service."""
req = {}
req['method'] = request.method
req['uri'] = request.uri
req['version'] = request.version
req['headers'] = dict(request.headers)
body = request.body
req['remote_ip'] = request.remote_ip
req['protocol'] = request.protocol
req['host'] = request.host
req['files'] = request.files
req['arguments'] = request.arguments
req['args'] = args
req['kwargs'] = kwargs
msg_id = bytes(uuid.uuid4())
msg_list = [b'|', msg_id, jsonapi.dumps(req)]
if body:
msg_list.append(body)
logging.debug('Sending request: %r', msg_list)
self.stream.send_multipart(msg_list)
if timeout > 0:
def _handle_timeout():
handler.send_error(504) # Gateway timeout
try:
self._callbacks.pop(msg_id)
except KeyError:
logging.error('Unexpected error removing callbacks')
dc = DelayedCallback(_handle_timeout, timeout, self.loop)
dc.start()
else:
dc = None
self._callbacks[msg_id] = (handler, dc)
return msg_id
def _tick(self):
"""Method called every HB_INTERVAL milliseconds.
"""
self.curr_liveness -= 1
## print '%.3f tick - %d' % (time.time(), self.curr_liveness)
self.send_hb()
if self.curr_liveness >= 0:
return
print '%.3f lost connection' % time.time()
# ouch, connection seems to be dead
self.shutdown()
# try to recreate it
self._delayed_cb = DelayedCallback(self._create_stream, 5000)
self._delayed_cb.start()
return
def _tick(self):
"""Method called every HB_INTERVAL milliseconds.
"""
if DEBUG:
print("MQREP > _tick")
self.curr_liveness -= 1
if DEBUG:
print('MQREP > _tick - {0} tick = {1}'.format(time.time(), self.curr_liveness))
self.send_hb()
if self.curr_liveness >= 0:
return
if DEBUG:
print('MQREP > _tick - {0} lost connection'.format(time.time()))
# ouch, connection seems to be dead
self.shutdown()
# try to recreate it
self._delayed_cb = DelayedCallback(self._create_stream, self.HB_INTERVAL)
self._delayed_cb.start()
return
def test_01_simple_01(self):
"""Test MDPWorker simple req/reply.
"""
self._start_broker()
time.sleep(0.2)
worker = MyWorker(self.context, self.endpoint, self.service)
sender = DelayedCallback(self.send_req, 500)
stopper = DelayedCallback(self.stop_test, 2500)
sender.start()
stopper.start()
IOLoop.instance().start()
worker.shutdown()
self._stop_broker()
return
def update(self):
self.updater.stop()
if self.device_state == 'connected':
logger.info("Sending update for device")
update = DeviceUpdate(self.device_id, 'farm_monitor')
update.create()
reply = self.send([pickle.dumps(update)])
if reply:
message = pickle.loads(reply[1])
logger.debug("Update response from {0} :{1}".format(message.source, message.reply))
# delay = seconds_till_next('hour', duration=1) + 2 # 2 second buffer
delay = 60 * 60 # send next update in 60 minutes
self.updater = DelayedCallback(self.update, delay * 1000)
self.updater.start()
return
def _restart(self):
self.shutdown()
self.socket = self.context.socket(zmq.REQ)
self.socket.setsockopt(zmq.LINGER, 0)
self.socket.connect(self.address)
self.can_send = True
self.connection_attempts = 0
self.device_state = 'unknown'
self.ticker = PeriodicCallback(self.heartbeat,
DeviceConnection.HB_INTERVAL)
self.ticker.start()
# delay = seconds_till_next('hour', duration=1) + 2 # 2 second buffer
delay = 60 # send update one minute after startup
self.updater = DelayedCallback(self.update, delay * 1000)
self.updater.start()
return
def update(self):
self.updater.stop()
if internal_dealer.check_connection():
logger.info("Sending update for all grainbins")
socket = self.context.socket(zmq.REQ)
socket.setsockopt(zmq.LINGER, 0)
socket.connect(self.address)
update = GrainbinUpdate(self.device_id, 'farm_monitor')
update.create()
message = [pickle.dumps(update)]
reply = mdp_request(socket, 'grainbin', message, 5)
if reply:
message = pickle.loads(reply[1])
logger.debug("reply from farm monitor for update: {0}".format(message.reply))
# delay = seconds_till_next('hour', duration=1) + 10 # 10 second buffer
delay = 60 * 60 # send next update in one hour
self.updater = DelayedCallback(self.update, delay * 1000)
self.updater.start()
return
class MDPClient(object):
"""Class for the MDP client side.
Thin asynchronous encapsulation of a zmq.REQ socket.
Provides a :func:`request` method with optional timeout.
Objects of this class are ment to be integrated into the
asynchronous IOLoop of pyzmq.
:param context: the ZeroMQ context to create the socket in.
:type context: zmq.Context
:param endpoint: the enpoint to connect to.
:type endpoint: str
:param service: the service the client should use
:type service: str
"""
_proto_version = C_CLIENT
def __init__(self, context, endpoint):
"""Initialize the MDPClient.
"""
self.context = context
self.endpoint = endpoint
def start(self):
"""
Initialize the zmq sockets on a ioloop stream.
The separation of this part from the init is useful if
we start the client on a separate thread with a new ioloop
(for example to enable use in an ipython notebook)
"""
socket = self.context.socket(zmq.DEALER)
ioloop = IOLoop.instance()
self.stream = ZMQStream(socket, ioloop)
self.stream.on_recv(self._on_message)
self._proto_prefix = [EMPTY_FRAME, self._proto_version]
self._delayed_timeout = None
self.timed_out = False
socket.connect(self.endpoint)
def shutdown(self):
"""Method to deactivate the client connection completely.
Will delete the stream and the underlying socket.
.. warning:: The instance MUST not be used after :func:`shutdown` has been called.
:rtype: None
"""
if not self.stream:
return
self.stream.socket.setsockopt(zmq.LINGER, 0)
self.stream.socket.close()
self.stream.close()
self.stream = None
def request(self, service, msg, msg_extra=STANDARD, timeout=None):
"""Send the given message.
:param msg: message parts to send.
:type msg: list of str
:param msg_extra: Extra message flags (e.g. STANDARD or BROADCAST)
:type msg_extra: int
:param timeout: time to wait in milliseconds.
:type timeout: int
:rtype None:
"""
if type(msg) in (bytes, unicode):
msg = [msg]
#
# prepare full message
#
to_send = [chr(msg_extra)] + self._proto_prefix[:]
to_send.extend([service])
to_send.extend(msg)
self.stream.send_multipart(to_send)
if timeout:
self._start_timeout(timeout)
def _on_timeout(self):
"""Helper called after timeout.
"""
self.timed_out = True
self._delayed_timeout = None
self.on_timeout()
def _start_timeout(self, timeout):
"""Helper for starting the timeout.
:param timeout: the time to wait in milliseconds.
:type timeout: int
"""
self._delayed_timeout = DelayedCallback(self._on_timeout, timeout)
self._delayed_timeout.start()
def _on_message(self, msg):
"""Helper method called on message receive.
:param msg: list of message parts.
:type msg: list of str
"""
if self._delayed_timeout:
#
# disable timout
#
self._delayed_timeout.stop()
self._delayed_timeout = None
self.on_message(msg)
def on_message(self, msg):
"""Public method called when a message arrived.
.. note:: Does nothing. Should be overloaded!
"""
raise NotImplementedError('on_message must be implemented by the subclass.')
def on_timeout(self):
"""Public method called when a timeout occured.
.. note:: Does nothing. Should be overloaded!
"""
raise NotImplementedError('on_timeout must be implemented by the subclass.')
class MqAsyncReq(object):
"""Class for the MDP client side.
Thin asynchronous encapsulation of a zmq.REQ socket.
Provides a :func:`request` method with optional timeout.
Objects of this class are ment to be integrated into the
asynchronous IOLoop of pyzmq.
:param context: the ZeroMQ context to create the socket in.
:type context: zmq.Context
:param endpoint: the enpoint to connect to.
:type endpoint: str
:param service: the service the client should use
:type service: str
"""
_proto_version = b'MDPC01'
def __init__(self, context, service):
"""Initialize the MDPClient.
"""
if ("domogik.common.configloader" in sys.modules):
cfg = Loader('mq').load()
confi = dict(cfg[1])
self.endpoint = "tcp://{0}:{1}".format(config['ip'], config['req_rep_port'])
else:
ip = Parameter.objects.get(key='mq-ip')
port = Parameter.objects.get(key='mq-req_rep_port')
self.endpoint = "tcp://{0}:{1}".format(ip.value, port.value)
socket = ZmqSocket(context, zmq.REQ)
ioloop = IOLoop.instance()
self.service = service
self.stream = ZMQStream(socket, ioloop)
self.stream.on_recv(self._on_message)
self.can_send = True
self._proto_prefix = [ PROTO_VERSION, service]
self._tmo = None
self.timed_out = False
socket.connect(self.endpoint)
return
def shutdown(self):
"""Method to deactivate the client connection completely.
Will delete the stream and the underlying socket.
.. warning:: The instance MUST not be used after :func:`shutdown` has been called.
:rtype: None
"""
if not self.stream:
return
self.stream.socket.setsockopt(zmq.LINGER, 0)
self.stream.socket.close()
self.stream.close()
self.stream = None
return
def request(self, msg, timeout=None):
"""Send the given message.
:param msg: message parts to send.
:type msg: list of str
:param timeout: time to wait in milliseconds.
:type timeout: int
:rtype None:
"""
if not self.can_send:
raise InvalidStateError()
if type(msg) in (bytes, str):
msg = [msg]
# prepare full message
to_send = self._proto_prefix[:]
to_send.extend(msg)
self.stream.send_multipart(to_send)
self.can_send = False
if timeout:
self._start_timeout(timeout)
return
def _on_timeout(self):
"""Helper called after timeout.
"""
self.timed_out = True
self._tmo = None
self.on_timeout()
return
def _start_timeout(self, timeout):
"""Helper for starting the timeout.
:param timeout: the time to wait in milliseconds.
:type timeout: int
"""
self._tmo = DelayedCallback(self._on_timeout, timeout)
self._tmo.start()
return
def _on_message(self, msg):
"""Helper method called on message receive.
:param msg: list of message parts.
:type msg: list of str
"""
if self._tmo:
# disable timout
self._tmo.stop()
self._tmo = None
# setting state before invoking on_message, so we can request from there
self.can_send = True
self.on_message(msg)
return
def on_message(self, msg):
"""Public method called when a message arrived.
.. note:: Does nothing. Should be overloaded!
"""
pass
def on_timeout(self):
"""Public method called when a timeout occured.
.. note:: Does nothing. Should be overloaded!
"""
pass
class MDPWorker(object):
"""Class for the MDP worker side.
Thin encapsulation of a zmq.DEALER socket.
Provides a send method with optional timeout parameter.
Will use a timeout to indicate a broker failure.
"""
_proto_version = b'MDPW01'
# TODO: integrate that into API
HB_INTERVAL = 1000 # in milliseconds
HB_LIVENESS = 3 # HBs to miss before connection counts as dead
def __init__(self, context, endpoint, service):
"""Initialize the MDPWorker.
context is the zmq context to create the socket from.
service is a byte-string with the service name.
"""
self.context = context
self.endpoint = endpoint
self.service = service
self.stream = None
self._tmo = None
self.need_handshake = True
self.ticker = None
self._delayed_cb = None
self._create_stream()
return
def _create_stream(self):
"""Helper to create the socket and the stream.
"""
socket = self.context.socket(zmq.DEALER)
ioloop = IOLoop.instance()
self.stream = ZMQStream(socket, ioloop)
self.stream.on_recv(self._on_message)
self.stream.socket.setsockopt(zmq.LINGER, 0)
self.stream.connect(self.endpoint)
self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
self._send_ready()
self.ticker.start()
return
def _send_ready(self):
"""Helper method to prepare and send the workers READY message.
"""
ready_msg = [b'', self._proto_version, b'\x01', self.service]
self.stream.send_multipart(ready_msg)
self.curr_liveness = self.HB_LIVENESS
return
def _tick(self):
"""Method called every HB_INTERVAL milliseconds.
"""
self.curr_liveness -= 1
## print '%.3f tick - %d' % (time.time(), self.curr_liveness)
self.send_hb()
if self.curr_liveness >= 0:
return
## print '%.3f lost connection' % time.time()
# ouch, connection seems to be dead
self.shutdown()
# try to recreate it
self._delayed_cb = DelayedCallback(self._create_stream, 5000)
self._delayed_cb.start()
return
def send_hb(self):
"""Construct and send HB message to broker.
"""
msg = [b'', self._proto_version, b'\x04']
self.stream.send_multipart(msg)
return
def shutdown(self):
"""Method to deactivate the worker connection completely.
Will delete the stream and the underlying socket.
"""
if self.ticker:
self.ticker.stop()
self.ticker = None
if not self.stream:
return
self.stream.socket.close()
self.stream.close()
self.stream = None
self.timed_out = False
self.need_handshake = True
self.connected = False
return
def reply(self, msg):
"""Send the given message.
msg can either be a byte-string or a list of byte-strings.
"""
## if self.need_handshake:
## raise ConnectionNotReadyError()
# prepare full message
to_send = self.envelope
self.envelope = None
if isinstance(msg, list):
to_send.extend(msg)
else:
to_send.append(msg)
self.stream.send_multipart(to_send)
return
def _on_message(self, msg):
"""Helper method called on message receive.
msg is a list w/ the message parts
"""
# 1st part is empty
msg.pop(0)
# 2nd part is protocol version
# TODO: version check
proto = msg.pop(0)
# 3rd part is message type
msg_type = msg.pop(0)
# XXX: hardcoded message types!
# any message resets the liveness counter
self.need_handshake = False
self.curr_liveness = self.HB_LIVENESS
if msg_type == b'\x05': # disconnect
self.curr_liveness = 0 # reconnect will be triggered by hb timer
elif msg_type == b'\x02': # request
# remaining parts are the user message
envelope, msg = split_address(msg)
envelope.append(b'')
envelope = [b'', self._proto_version, b'\x03'] + envelope # REPLY
self.envelope = envelope
self.on_request(msg)
else:
# invalid message
# ignored
pass
return
def on_request(self, msg):
"""Public method called when a request arrived.
Must be overloaded!
"""
pass
class MDPWorker(object):
"""Class for the MDP worker side.
Thin encapsulation of a zmq.DEALER socket.
Provides a send method with optional timeout parameter.
Will use a timeout to indicate a broker failure.
"""
_proto_version = W_WORKER
# TODO: integrate that into API
HB_INTERVAL = 2500 # in milliseconds
HB_LIVENESS = 5 # HBs to miss before connection counts as dead
RECONNECT_PERIOD = 5000
def __init__(self,
context,
endpoint,
hb_endpoint,
service,
endpoint_callback=None):
"""Initialize the MDPWorker.
context is the zmq context to create the socket from.
service is a byte-string with the service name.
"""
self.context = context
self.endpoint = endpoint
self.hb_endpoint = hb_endpoint
self.service = service
self.endpoint_callback = endpoint_callback
self.stream = None
self.hb_stream = None
self.ticker = None
self._delayed_reconnect = None
self._unique_id = ''
self._create_stream()
def _create_stream(self):
"""Helper to create the socket and the stream.
"""
logging.debug('Worker creating stream')
ioloop = IOLoop.instance()
socket = self.context.socket(zmq.DEALER)
self.stream = ZMQStream(socket, ioloop)
self.stream.on_recv(self._on_message)
self.stream.socket.setsockopt(zmq.LINGER, 0)
self.stream.connect(self.endpoint)
socket = self.context.socket(zmq.DEALER)
self.hb_stream = ZMQStream(socket, ioloop)
self.hb_stream.on_recv(self._on_message)
self.hb_stream.socket.setsockopt(zmq.LINGER, 0)
self.hb_stream.connect(self.hb_endpoint)
self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
self._send_ready()
self.ticker.start()
def _send_ready(self):
"""Helper method to prepare and send the workers READY message.
"""
ready_msg = [EMPTY_FRAME, self._proto_version, W_READY, self.service]
self.stream.send_multipart(ready_msg)
self.curr_liveness = self.HB_LIVENESS
def _tick(self):
"""Method called every HB_INTERVAL milliseconds.
"""
self.curr_liveness -= 1
logging.debug('Worker HB tick, current liveness: %d' %
self.curr_liveness)
self.send_hb()
if self.curr_liveness >= 0:
return
#
# Ouch, connection seems to be dead
#
logging.debug('Worker lost connection')
self.shutdown()
#
# try to recreate the connection
#
self._delayed_reconnect = DelayedCallback(self._recreate_stream,
self.RECONNECT_PERIOD)
self._delayed_reconnect.start()
def _recreate_stream(self):
logging.debug('Worker trying to recreate stream')
if self.endpoint_callback is not None:
#
# Check, maybe the ip of the proxy changed.
#
try:
self.endpoint, self.hb_endpoint = self.endpoint_callback()
except:
#
# Probably some problem in accessing the server.
#
self._delayed_reconnect = DelayedCallback(
self._recreate_stream, self.RECONNECT_PERIOD)
self._delayed_reconnect.start()
return
self._create_stream()
def send_hb(self):
"""Construct and send HB message to broker.
"""
msg = [EMPTY_FRAME, self._proto_version, W_HEARTBEAT, self._unique_id]
self.hb_stream.send_multipart(msg)
def shutdown(self):
"""Method to deactivate the worker connection completely.
Will delete the stream and the underlying socket.
"""
logging.debug('Shutdown of the worker')
if self.ticker:
logging.debug('Stopping the HB ticker')
self.ticker.stop()
self.ticker = None
if not self.stream:
return
logging.debug('Closing the stream')
self.stream.socket.close()
self.stream.close()
self.stream = None
self.hb_stream.socket.close()
self.hb_stream.close()
self.hb_stream = None
self.timed_out = False
self.connected = False
def reply(self, msg):
"""Send the given message.
msg can either be a byte-string or a list of byte-strings.
"""
#
# prepare full message
#
to_send = self.envelope
self.envelope = None
if isinstance(msg, list):
to_send.extend(msg)
else:
to_send.append(msg)
self.stream.send_multipart(to_send)
def _on_message(self, msg):
"""Helper method called on message receive.
msg is a list w/ the message parts
"""
logging.debug('Received message: {}'.format(msg))
#
# 1st part is empty
#
msg.pop(0)
#
# 2nd part is protocol version
# TODO: version check
#
proto = msg.pop(0)
#
# 3rd part is message type
#
msg_type = msg.pop(0)
#
# XXX: hardcoded message types!
# any message resets the liveness counter
#
self.curr_liveness = self.HB_LIVENESS
if msg_type == W_DISCONNECT:
#
# Disconnect. Reconnection will be triggered by hb timer
#
self.curr_liveness = 0
elif msg_type == W_READY:
#
# The message contains the unique id attached to the worker.
#
if len(msg) > 0:
#
# This above check is used for supporting older version of
# the code.
#
self._unique_id = msg[0]
elif msg_type == W_REQUEST:
#
# Request. Remaining parts are the user message
#
envelope, msg = split_address(msg)
envelope.append(EMPTY_FRAME)
envelope = [EMPTY_FRAME, self._proto_version, W_REPLY] + envelope
self.envelope = envelope
self.on_request(msg)
else:
#
# invalid message
# ignored
#
pass
def on_request(self, msg):
"""Public method called when a request arrived.
Must be overloaded!
"""
raise NotImplementedError(
'on_request must be implemented by the subclass.')
class MNWorker(MN_object):
"""Class for the MN worker side.
Thin encapsulation of a zmq.DEALER socket.
Provides a send method with optional timeout parameter.
Will use a timeout to indicate a broker failure.
:param context: the context to use for socket creation.
:type context: zmq.Context
:param endpoint: endpoint to connect to.
:type endpoint: str
:param service: the name of the service we support.
:type service: byte-string
"""
_proto_version = b'MNPW01' # worker protocol version
def __init__(self, context, endpoint, service, worker_type, address,
protocols):
"""Initialize the MNWorker.
"""
self.context = context
self.endpoint = endpoint
self.service = service
self.type = worker_type
self.address = address
self.protocols = protocols
self.envelope = None
self.HB_RETRIES = HB_RETRIES
self.HB_INTERVAL = HB_INTERVAL
self._data = {}
self.stream = None
self._tmo = None
self.timed_out = False
self.need_handshake = True
self.connected = False
self.ticker = None
self._delayed_cb = None
self._create_stream()
_LOG.info("Worker initialized and can be found at '%s'" % endpoint)
return
def _create_stream(self):
"""Helper to create the socket and the stream.
"""
socket = self.context.socket(zmq.DEALER)
ioloop = IOLoop.instance()
self.stream = ZMQStream(socket, ioloop)
self.stream.on_recv(self._on_message)
self.stream.socket.setsockopt(zmq.LINGER, 0)
self.stream.connect(self.endpoint)
self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
self._send_ready()
self.ticker.start()
return
def _send_ready(self):
"""Helper method to prepare and send the workers READY message.
"""
_LOG.debug("Informing broker I am ready")
ready_msg = [
b'', WORKER_PROTO, MSG_READY, self.service, self.type,
self.address, self.protocols
]
if self.stream.closed():
self.shutdown()
self.stream.send_multipart(ready_msg)
self.curr_retries = self.HB_RETRIES
return
def _tick(self):
"""Method called every HB_INTERVAL milliseconds.
"""
self.curr_retries -= 1
self.send_hb()
if self.curr_retries >= 0:
return
# connection seems to be dead
self.shutdown()
# try to recreate it
# self._delayed_cb = IOLoop.call_later(self._create_stream, 5000)
# self._delayed_cb = IOLoop.add_timeout(self._create_stream, 5000)
self._delayed_cb = DelayedCallback(self._create_stream,
self.HB_INTERVAL)
self._delayed_cb.start()
return
def send_hb(self):
"""Construct and send HB message to broker.
"""
_LOG.debug("Sending heartbeat")
msg = [b'', WORKER_PROTO, MSG_HEARTBEAT]
if self.stream.closed():
self.shutdown()
self.stream.send_multipart(msg)
return
def shutdown(self):
"""Method to deactivate the worker connection completely.
Will delete the stream and the underlying socket.
"""
if self.ticker:
self.ticker.stop()
self.ticker = None
if not self.stream:
return
self.stream.socket.close()
self.stream.close()
self.stream = None
self.timed_out = False
self.need_handshake = True
self.connected = False
return
def reply(self, msg):
"""Send the given message.
:param msg: full message to send.
:type msg: can either be a byte-string or a list of byte-strings
"""
if self.need_handshake:
raise ConnectionNotReadyError()
to_send = self.envelope
self.envelope = None
if isinstance(msg, list):
to_send.extend(msg)
else:
to_send.append(msg)
if self.stream.closed():
self.shutdown()
self.stream.send_multipart(to_send)
return
def _on_message(self, msg):
"""Helper method called on message receive.
:param msg: a list w/ the message parts
:type msg: a list of byte-strings
"""
_LOG.debug("Received: %s." % msg)
# 1st part is empty
msg.pop(0)
# 2nd part is protocol version
proto = msg.pop(0)
if proto != WORKER_PROTO:
# ignore message from not supported protocol
pass
# 3rd part is message type
msg_type = msg.pop(0)
# XXX: hardcoded message types!
# any message resets the retries counter
self.need_handshake = False
self.curr_retries = self.HB_RETRIES
if msg_type == MSG_DISCONNECT: # disconnect
_LOG.info("Broker wants us to disconnect.")
self.curr_retries = 0 # reconnect will be triggered by hb timer
elif msg_type == MSG_QUERY: # request
# remaining parts are the user message
_LOG.debug("Received new request: %s." % msg)
envelope, msg = split_address(msg)
envelope.append(b'')
envelope = [b'', WORKER_PROTO, MSG_REPLY] + envelope # reply
self.envelope = envelope
self.on_request(msg)
else:
# invalid message
# ignored
_LOG.debug('ignoring message with invalid id')
pass
return
def on_request(self, msg):
"""Public method called when a request arrived.
:param msg: a list w/ the message parts
:type msg: a list of byte-strings
Must be overloaded to provide support for various services!
"""
pass
class MNClient(MN_object):
"""Class for the MN client side.
Thin asynchronous encapsulation of a zmq.REQ socket.
Provides a :func:`request` method with optional timeout.
:param context: the ZeroMQ context to create the socket in.
:type context: zmq.Context
:param endpoint: the endpoint to connect to.
:type endpoint: str
:param service: the service the client should use
:type service: str
"""
_proto_version = CLIENT_PROTO
def __init__(self, context, endpoint, service):
"""Initialize the MNClient.
"""
self.context = context
self.service = service
self.endpoint = endpoint
self.can_send = True
self._proto_prefix = ['', CLIENT_PROTO, service]
self._tmo = None
self.timed_out = False
self._create_stream()
return
def _create_stream(self):
"""Helper for opening a stream.
"""
socket = self.context.socket(zmq.DEALER)
ioloop = IOLoop.instance()
self.stream = ZMQStream(socket, ioloop)
self.stream.on_recv(self._on_message)
self.stream.socket.setsockopt(zmq.LINGER, 0)
self.stream.connect(self.endpoint)
def shutdown(self):
"""Method to deactivate the client connection completely.
Will delete the stream and the underlying socket.
.. warning:: The instance MUST not be used after :func:`shutdown` has been called.
:rtype: None
"""
if not self.stream:
return
self.stream.socket.close()
self.stream.close()
self.stream = None
return
def request(self, msg, timeout=None):
"""Send the given message.
:param msg: message parts to send.
:type msg: list of str
:param timeout: time to wait in milliseconds.
:type timeout: int
:rtype None:
"""
if not self.can_send:
raise InvalidStateError()
if isinstance(msg, bytes):
msg = [msg]
# prepare full message
# to_send = self._proto_prefix[:]
# to_send.extend(msg)
to_send = msg
if self.stream.closed():
self._create_stream()
# TODO check this
self.stream.send_multipart(to_send)
self.can_send = False
if timeout:
self._start_timeout(timeout)
return
def _on_timeout(self):
"""Helper called after timeout.
"""
self.timed_out = True
self._tmo = None
self.on_timeout()
return
def _start_timeout(self, timeout):
"""Helper for starting the timeout.
:param timeout: the time to wait in milliseconds.
:type timeout: int
"""
self._tmo = DelayedCallback(self._on_timeout, timeout)
self._tmo.start()
return
def _on_message(self, msg):
"""Helper method called on message receive.
:param msg: list of message parts.
:type msg: list of str
"""
if self._tmo:
# disable timout
self._tmo.stop()
self._tmo = None
# setting state before invoking on_message, so we can request from there
self.can_send = True
self.on_message(msg)
return
def on_message(self, msg):
"""Public method called when a message arrived.
.. note:: Does nothing. Should be overloaded!
"""
pass
def on_timeout(self):
"""Public method called when a timeout occurred.
.. note:: Does nothing. Should be overloaded!
"""
pass
class GrainBinService(object):
SERVICE_NAME = 'grain_bin'
def __init__(self, context, device_id, address):
self.context = context
self.address = address
self.device_id = device_id
self.socket = None
# delay = seconds_till_next('hour', duration=1) + 10 # 10 second buffer
delay = 120 # 2 minutes after startup
self.updater = DelayedCallback(self.update, delay * 1000)
self.updater.start()
return
def shutdown(self):
self.updater.stop()
return
def update(self):
self.updater.stop()
if internal_dealer.check_connection():
logger.info("Sending update for all grainbins")
socket = self.context.socket(zmq.REQ)
socket.setsockopt(zmq.LINGER, 0)
socket.connect(self.address)
update = GrainbinUpdate(self.device_id, 'farm_monitor')
update.create()
message = [pickle.dumps(update)]
reply = mdp_request(socket, 'grainbin', message, 5)
if reply:
message = pickle.loads(reply[1])
logger.debug("reply from farm monitor for update: {0}".format(message.reply))
# delay = seconds_till_next('hour', duration=1) + 10 # 10 second buffer
delay = 60 * 60 # send next update in one hour
self.updater = DelayedCallback(self.update, delay * 1000)
self.updater.start()
return
def update_individual(self, bus_number):
if internal_dealer.check_connection():
logger.info("Sending update for grainbin: {0}".format(bus_number))
socket = self.context.socket(zmq.REQ)
socket.setsockopt(zmq.LINGER, 0)
socket.connect(self.address)
update = GrainbinUpdate(self.device_id, 'farm_monitor')
update.create(bus_number=bus_number)
message = [pickle.dumps(update)]
reply = mdp_request(socket, 'grainbin', message, 5)
if reply:
message = pickle.loads(reply[1])
logger.debug("reply from farm monitor for update: {0}".format(message.reply))
return
def rescan(self, bus_number):
logger.info("sending rescan for grainbin: {0}".format(bus_number))
info = GrainbinInfo(self.device_id, 'farm_monitor')
info.create()
message = [pickle.dumps(info)]
socket = self.context.socket(zmq.REQ)
socket.setsockopt(zmq.LINGER, 0)
socket.connect(self.address)
reply = mdp_request(socket, 'grainbin', message, 5)
if reply:
message = pickle.loads(reply[1])
logger.debug("reply from farm monitor for rescan: {0}".format(message.reply))
return
class MDPWorker(object):
"""Class for the MDP worker side.
Thin encapsulation of a zmq.XREQ socket.
Provides a send method with optional timeout parameter.
Will use a timeout to indicate a broker failure.
"""
_proto_version = b'MDPW01'
# TODO: integrate that into API
HB_INTERVAL = 1000 # in milliseconds
HB_LIVENESS = 3 # HBs to miss before connection counts as dead
def __init__(self, context, endpoint, service):
"""Initialize the MDPWorker.
context is the zmq context to create the socket from.
service is a byte-string with the service name.
"""
self.context = context
self.endpoint = endpoint
self.service = service
self.stream = None
self._tmo = None
self.need_handshake = True
self.ticker = None
self._delayed_cb = None
self._create_stream()
return
def _create_stream(self):
"""Helper to create the socket and the stream.
"""
socket = self.context.socket(zmq.XREQ)
ioloop = IOLoop.instance()
self.stream = ZMQStream(socket, ioloop)
self.stream.on_recv(self._on_message)
self.stream.socket.setsockopt(zmq.LINGER, 0)
self.stream.connect(self.endpoint)
self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
self._send_ready()
self.ticker.start()
return
def _send_ready(self):
"""Helper method to prepare and send the workers READY message.
"""
ready_msg = [ b'', self._proto_version, chr(1), self.service ]
self.stream.send_multipart(ready_msg)
self.curr_liveness = self.HB_LIVENESS
return
def _tick(self):
"""Method called every HB_INTERVAL milliseconds.
"""
self.curr_liveness -= 1
## print '%.3f tick - %d' % (time.time(), self.curr_liveness)
self.send_hb()
if self.curr_liveness >= 0:
return
print '%.3f lost connection' % time.time()
# ouch, connection seems to be dead
self.shutdown()
# try to recreate it
self._delayed_cb = DelayedCallback(self._create_stream, 5000)
self._delayed_cb.start()
return
def send_hb(self):
"""Construct and send HB message to broker.
"""
msg = [ b'', self._proto_version, chr(4) ]
self.stream.send_multipart(msg)
return
def shutdown(self):
"""Method to deactivate the worker connection completely.
Will delete the stream and the underlying socket.
"""
if self.ticker:
self.ticker.stop()
self.ticker = None
if not self.stream:
return
self.stream.socket.close()
self.stream.close()
self.stream = None
self.timed_out = False
self.need_handshake = True
self.connected = False
return
def reply(self, msg):
"""Send the given message.
msg can either be a byte-string or a list of byte-strings.
"""
## if self.need_handshake:
## raise ConnectionNotReadyError()
# prepare full message
to_send = self.envelope
self.envelope = None
if isinstance(msg, list):
to_send.extend(msg)
else:
to_send.append(msg)
self.stream.send_multipart(to_send)
return
def _on_message(self, msg):
"""Helper method called on message receive.
msg is a list w/ the message parts
"""
# 1st part is empty
msg.pop(0)
# 2nd part is protocol version
# TODO: version check
proto = msg.pop(0)
# 3nd part is message type
msg_type = msg.pop(0)
# XXX: hardcoded message types!
# any message resets the liveness counter
self.need_handshake = False
self.curr_liveness = self.HB_LIVENESS
if msg_type == '\x05': # disconnect
print ' DISC'
self.curr_liveness = 0 # reconnect will be triggered by hb timer
elif msg_type == '\x02': # request
# remaining parts are the user message
envelope, msg = split_address(msg)
envelope.append(b'')
envelope = [ b'', self._proto_version, '\x03'] + envelope # REPLY
self.envelope = envelope
self.on_request(msg)
else:
# invalid message
# ignored
pass
return
def on_request(self, msg):
"""Public method called when a request arrived.
Must be overloaded!
"""
pass
class MDPWorker(ZMQStream):
def __init__(self, broker, service, io_loop=None):
"""Create and setup an MDP worker.
@param broker A string containing the broker's URL
@param service A string containing the service name
@param io_loop An existing I/O loop object. If None, the default will be used.
"""
self.service=service
self._broker = broker
self.ctx = zmq.Context()
sock = self.ctx.socket(zmq.DEALER)
ZMQStream.__init__(self, sock, io_loop)
# last watchdog timer tick
self.watchdog = 0
# connection callback one-shot
self._conncb = DelayedCallback(self.send_ready, 3000, self.io_loop)
# heartbeat callback..runs continuous when connected
self._hbcb = PeriodicCallback(self.send_heartbeat, 2000, self.io_loop)
# number of connection attempts
self._conn_attempt = 0
# waiting to connect state
self._waiting_to_connect = True
# have we been disconnected? (flags a reconnect attempt)
self.disconnected = False
# connect the socket and send a READY when the io_loop starts
self.connect(self._broker)
self._conncb.start()
def reset_watchdog(self):
"""Private method used to reset the HEARTBEAT watchdog
"""
self.watchdog = time.time()
def disconnect(self):
"""Disconnect from the broker.
"""
logging.info("Disconnected from broker")
self.on_recv(None) # stop message processing
self._conncb.stop() # stop any pending reconnect
self._hbcb.stop() # stop heartbeats
self.disconnected = True
self.io_loop.stop() # stop the I/O loop. If it's used by something else, the caller can restart it
def reconnect(self):
"""Try to reconnect to the broker.
"""
if self.disconnected: # don't try and reconnect, we got an explicit disconnect
return
logging.info("Attempting to reconnect to broker")
self._hbcb.stop()
self._conn_attempt = 0
self._waiting_to_connect = True
try:
self.connect(self._broker)
except ZMQError:
logging.exception()
self.io_loop.stop()
return
self._conncb.start()
def send_ready(self):
"""Send a READY message.
"""
if not self._waiting_to_connect:
# connected already
return
if self.disconnected: # don't try and connect, we got an explicit disconnect
return
logging.debug("Sending READY")
if self._conn_attempt >= 10:
logging.error("10 connection attempts have failed. Giving up.")
return
self._conn_attempt += 1
logging.debug("Connection attempt %i" % self._conn_attempt)
rdy = [b'', MDPW_VER, b'\x01', self.service]
self.on_recv(self.on_message)
self.send_multipart(rdy)
# There is no reply to READY so
# we must assume we are connected unless we see a DISCONNECT
self._waiting_to_connect = False
self._disconed = False
self.reset_watchdog()
self._hbcb.start()
def send_reply(self, client, msg, partial=False):
"""Send a reply to a client. This is typically called from on_request()
@param client The client identifier as passed to on_request()
@param msg The message to send to the client. If this is a list, it's appended to the multipart;
otherwise it is converted to a string and sent as a single frame.
@param partial If this is True, the message is sent as a PARTIAL and at least one
more call must be made to send_reply(). Otherwise a FINAL is sent and
not more calls should be made to send_reply() until another request is processed.
"""
self._hbcb.stop() # stop while sending other messages
if partial:
rply = [b'', MDPW_VER, b'\x03', client, b'']
else:
rply = [b'', MDPW_VER, b'\x04', client, b'']
if isinstance(msg, list):
rply.extend(msg)
else:
rply.append(msg)
try:
logging.debug("Sending reply: %s" % msg_str(rply))
self.send_multipart(rply)
except BaseException,e:
logging.error("Error sending reply: " % e)
self._hbcb.start() # restart heartbeats
class DeviceConnection(object):
SERVICE_NAME = 'device' # service to connect to
TIMEOUT = 5 # time to wait for answer in seconds
# Number of connections to try before restarting when in incorrect state
CONNECTION_ATTEMPS = 3
HB_INTERVAL = 1000 * 10 # in milliseconds
def __init__(self, context, device_id, address):
self.context = context
self.device_id = device_id
self.address = address
self.ticker = None
self.updater = None
self.socket = None
self.can_send = False
self._restart()
self.heartbeat()
return
def _restart(self):
self.shutdown()
self.socket = self.context.socket(zmq.REQ)
self.socket.setsockopt(zmq.LINGER, 0)
self.socket.connect(self.address)
self.can_send = True
self.connection_attempts = 0
self.device_state = 'unknown'
self.ticker = PeriodicCallback(self.heartbeat,
DeviceConnection.HB_INTERVAL)
self.ticker.start()
# delay = seconds_till_next('hour', duration=1) + 2 # 2 second buffer
delay = 60 # send update one minute after startup
self.updater = DelayedCallback(self.update, delay * 1000)
self.updater.start()
return
def shutdown(self):
if self.ticker:
self.ticker.stop()
self.ticker = None
if self.updater:
self.updater.stop()
if self.socket:
self.socket.close()
self.socket = None
self.can_send = False
return
def send(self, message):
if not self.can_send:
self.connection_attempts += 1
logger.error("DeviceConnection is not in state to send")
return None
else:
self.connection_attempts = 0
self.can_send = False
logger.debug("DeviceConnection sending {0}".format(message))
reply = mdp_request(self.socket,
DeviceConnection.SERVICE_NAME,
message,
DeviceConnection.TIMEOUT)
if reply:
logger.debug("DeviceConnection reply received: {0}".format(reply))
self.can_send = True
return reply
else:
# Timeout! Will be restarted at next heartbeat
logger.warn("DeviceConnection timeout. Will be restarted at next heartbeat")
self.connection_attempts = DeviceConnection.CONNECTION_ATTEMPS
return None
def heartbeat(self):
if self.connection_attempts >= DeviceConnection.CONNECTION_ATTEMPS:
logger.warn("DeviceConnection attempts max reached. Restarting connection.")
self._restart()
message = HeartbeatMessage(self.device_id)
reply = self.send([pickle.dumps(message)])
if reply:
message = pickle.loads(reply[1])
if message.response == self.device_state:
# device state has not changed
pass
elif message.response == 'unknown':
logger.info("connection status set to 'unknown'")
self.device_state = 'unknown'
elif message.response == 'joined':
logger.info("connection status set to 'joined'")
self.device_state = 'joined'
elif message.response == 'connected':
logger.info("connection status set to 'connected'")
self.device_state = 'connected'
else:
self._restart()
return
def create(self):
logger.info("Sending create details for device")
device_details = DeviceInfo(self.device_id, 'farm_monitor')
device_details.create()
message = [pickle.dumps(device_details)]
reply = self.send(message)
if reply:
message = pickle.loads(reply[1])
logger.debug("Update response from {0} :{1}".format(message.source, message.reply))
return
def update(self):
self.updater.stop()
if self.device_state == 'connected':
logger.info("Sending update for device")
update = DeviceUpdate(self.device_id, 'farm_monitor')
update.create()
reply = self.send([pickle.dumps(update)])
if reply:
message = pickle.loads(reply[1])
logger.debug("Update response from {0} :{1}".format(message.source, message.reply))
# delay = seconds_till_next('hour', duration=1) + 2 # 2 second buffer
delay = 60 * 60 # send next update in 60 minutes
self.updater = DelayedCallback(self.update, delay * 1000)
self.updater.start()
return
def backup(self):
backup_index = create_backup()
session = Session()
backup = session.query(Backup).filter_by(index=backup_index).first()
standalone_configuration = session.query(SystemSetup.standalone_configuration).scalar()
if self.device_state == 'connected':
failed_attempts = 0
# don't send the file if we are a combined configuration
if standalone_configuration:
logger.info("Sending device backup file to FarmMonitor")
file_size = os.path.getsize(backup.filepath)
bytes_sent = 0
zip_file = open(backup.filepath, 'r')
while bytes_sent < file_size and failed_attempts < 3:
zip_file.seek(bytes_sent)
data = zip_file.read(512)
logger.debug("Sending {0} bytes of backup file".format(len(data)))
message = FileMessage(self.device_id, 'farm_monitor', 'backup')
message.set_file(backup.filepath, data)
reply = self.send([pickle.dumps(message)])
if reply:
# only mark the data received if a reply is recieved
bytes_sent += len(data)
failed_attempts = 0
reply_message = pickle.loads(reply[1])
logger.debug("Update reply: {0}".format(reply_message.reply))
else:
logger.warn("Failed attempt in sending backup file")
failed_attempts += 1
zip_file.close()
# only send the database object if the file was sent correctly
if failed_attempts == 0:
logger.info("Sending device backup database object to FarmMonitor")
message = FileMessage(self.device_id, 'farm_monitor', 'backup')
message.set_db_object(pickle.dumps(backup))
reply = self.send([pickle.dumps(message)])
if reply:
reply_message = pickle.loads(reply[1])
logger.debug("Update response: {0}".format(reply_message.reply))
else:
logger.warn("To many failed attempts. Backup failed to send to Farm Monitor")
session.close()
return
def perform_update(self, update_id):
session = Session()
standalone_configuration = session.query(SystemSetup.standalone_configuration).scalar()
if self.device_state == 'connected':
# request the update db_object
logger.info("Requesting update with id: {0} from FarmMonitor".format(update_id))
message = [self.device_id, 'perform_update', 'db_object', update_id]
message = FileMessage(self.device_id, 'farm_monitor', 'perform_update')
message.set_db_object(update_id)
reply = self.send([pickle.dumps(message)])
if reply:
reply_message = pickle.loads(reply[1])
logger.debug("Update response from farm_monitor")
if reply_message.data:
update = pickle.loads(reply_message.data)
else:
logger.warn("incorrect response in perform_update: {0}".format(reply[0]))
session.close()
return
failed_attempts = 0
# don't request the file if we are a combined configuration
if standalone_configuration:
if not os.path.exists(update.filepath):
os.makedirs(update.filepath)
bytes_received = 0
more_to_send = True
new_file = open(update.filepath + update.filename, 'a')
while more_to_send and failed_attempts < 3:
logger.info("Requesting update file from FarmMonitor")
message = FileMessage(self.device_id, 'farm_monitor', 'perform_update')
message.request_file(update_id, str(bytes_received))
reply = self.send([pickle.dumps(message)])
if reply:
reply_message = pickle.loads(reply[1])
logger.debug("Update response from FarmMonitor")
failed_attempts = 0
# in this case, reply_message.filepath is really a boolean.
# reusing the FileMessage object
more_to_send = reply_message.filepath
data = reply_message.data
new_file.seek(bytes_received)
new_file.write(data)
bytes_received += len(data)
else:
logger.warn("Failed attempt in requesting update file")
failed_attempts += 1
new_file.close()
# compare the hash of the file
sha256 = hashlib.sha256()
with open(update.filepath + update.filename, 'rb') as f:
for block in iter(lambda: f.read(2048), b''):
sha256.update(block)
if update.sha256 == sha256.hexdigest():
logger.info("Update file successfully received")
print('update received good')
session.merge(update)
session.commit()
else:
logger.warn("Update file hash does not match. Update failed")
session.close()
return
# now perform the actual update
run_update(update.id, 'farm_device')
else:
logger.warn("No reply when requesting update db_object from FarMonitor.")
session.close()
return
class MqAsyncReq(object):
"""Class for the MDP client side.
Thin asynchronous encapsulation of a zmq.REQ socket.
Provides a :func:`request` method with optional timeout.
Objects of this class are ment to be integrated into the
asynchronous IOLoop of pyzmq.
:param context: the ZeroMQ context to create the socket in.
:type context: zmq.Context
:param endpoint: the enpoint to connect to.
:type endpoint: str
:param service: the service the client should use
:type service: str
"""
_proto_version = b'MDPC01'
def __init__(self, context, service):
"""Initialize the MDPClient.
"""
if ("domogik.common.configloader" in sys.modules):
cfg = Loader('mq').load()
confi = dict(cfg[1])
self.endpoint = "tcp://{0}:{1}".format(config['ip'], config['req_rep_port'])
else:
ip = Parameter.objects.get(key='mq-ip')
port = Parameter.objects.get(key='mq-req_rep_port')
self.endpoint = "tcp://{0}:{1}".format(ip.value, port.value)
socket = ZmqSocket(context, zmq.REQ)
ioloop = IOLoop.instance()
self.service = service
self.stream = ZMQStream(socket, ioloop)
self.stream.on_recv(self._on_message)
self.can_send = True
self._proto_prefix = [ PROTO_VERSION, service]
self._tmo = None
self.timed_out = False
socket.connect(self.endpoint)
return
def shutdown(self):
"""Method to deactivate the client connection completely.
Will delete the stream and the underlying socket.
.. warning:: The instance MUST not be used after :func:`shutdown` has been called.
:rtype: None
"""
if not self.stream:
return
self.stream.socket.setsockopt(zmq.LINGER, 0)
self.stream.socket.close()
self.stream.close()
self.stream = None
return
def request(self, msg, timeout=None):
"""Send the given message.
:param msg: message parts to send.
:type msg: list of str
:param timeout: time to wait in milliseconds.
:type timeout: int
:rtype None:
"""
if not self.can_send:
raise InvalidStateError()
if type(msg) in (bytes, str):
msg = [msg]
# prepare full message
to_send = self._proto_prefix[:]
to_send.extend(msg)
self.stream.send_multipart(to_send)
self.can_send = False
if timeout:
self._start_timeout(timeout)
return
def _on_timeout(self):
"""Helper called after timeout.
"""
self.timed_out = True
self._tmo = None
self.on_timeout()
return
def _start_timeout(self, timeout):
"""Helper for starting the timeout.
:param timeout: the time to wait in milliseconds.
:type timeout: int
"""
self._tmo = DelayedCallback(self._on_timeout, timeout)
self._tmo.start()
return
def _on_message(self, msg):
"""Helper method called on message receive.
:param msg: list of message parts.
:type msg: list of str
"""
if self._tmo:
# disable timout
self._tmo.stop()
self._tmo = None
# setting state before invoking on_message, so we can request from there
self.can_send = True
self.on_message(msg)
return
def on_message(self, msg):
"""Public method called when a message arrived.
.. note:: Does nothing. Should be overloaded!
"""
pass
def on_timeout(self):
"""Public method called when a timeout occured.
.. note:: Does nothing. Should be overloaded!
"""
pass
def _add_callback(future):
success, _, _ = future.result()
if not success:
cb = self.uart_cb[path] = DelayedCallback(
_check_device, 500, self.ioloop)
cb.start()
class Client(object):
def __init__(self, context, endpoint, service):
"""Initialize the Client.
"""
socket = context.socket(zmq.DEALER)
self.ioloop = IOLoop.instance()
self.service = service.encode('utf-8')
self.endpoint = endpoint
self.stream = ZMQStream(socket, self.ioloop)
self.stream.on_recv(self._on_message)
self.can_send = True
self._proto_prefix = [MDP_CLIENT_VERSION, b'\x01', service.encode('utf-8')]
self._tmo = None
self.timed_out = False
self.multicast_services = None
self.multicast_already_received = []
socket.connect(endpoint)
return
def shutdown(self):
"""Method to deactivate the client connection completely.
Will delete the stream and the underlying socket.
.. warning:: The instance MUST not be used after :func:`shutdown` has been called.
:rtype: None
"""
if not self.stream:
return
self.stream.socket.setsockopt(zmq.LINGER, 0)
self.stream.socket.close()
self.stream.close()
self.stream = None
return
def request(self, msg, timeout=None):
"""Send the given message.
:param msg: message parts to send.
:type msg: list of str
:param timeout: time to wait in milliseconds.
:type timeout: int | float
:rtype None:
"""
if not self.can_send:
raise InvalidStateError()
try:
if type(msg) in (bytes, unicode): # python 2
msg = [msg]
except NameError:
if type(msg) == bytes: # python 3
msg = [msg]
# prepare full message
to_send = [b'']
to_send.extend(self._proto_prefix[:])
to_send.extend(msg)
self.stream.send_multipart(to_send)
self.can_send = False
self.multicast_services = None
if timeout:
self._start_timeout(timeout)
return
def rpc(self, method, args=None, kwargs=None, timeout=None):
"""Call RPC method
:param method: name of method
:type method: str
:param args: list of args
:type args: list
:param kwargs: list of kwargs
:type kwargs: dict
:param timeout: time to wait in milliseconds.
:type timeout: int | float
:rtype None:
"""
data = [method.encode('utf-8'), msgpack.packb([] if args is None else args), msgpack.packb({} if kwargs is None else kwargs)]
return self.request(data, timeout)
def _on_message(self, message):
"""Helper method called on message receive.
:param message: list of message parts.
:type message: list of str
"""
message.pop(0) # remove empty string
protocol_version = message.pop(0)
if protocol_version != MDP_WORKER_VERSION: # version check, ignore old versions
return
message_type = message.pop(0)
if message_type == b'\x02': # partial message
worker = message.pop(0)
try:
msg = msgpack.unpackb(message[0])
except:
msg = message[0]
self.on_partial_message(worker, msg)
elif message_type == b'\x03': # final message
worker = message.pop(0)
try:
msg = msgpack.unpackb(message[0])
except:
msg = message[0]
self.on_message(worker, msg)
if message[0] not in self.multicast_already_received:
self.multicast_already_received.append(worker)
if self.multicast_services is None or len(self.multicast_already_received) == len(self.multicast_services):
self.stop_waiting()
elif message_type == b'\x04': # error message
worker = message.pop(0)
self.on_error_message(worker, message[0])
self.stop_waiting()
elif message_type == b'\x05': # multicast start, load list of multicast services
self.multicast_services = message
self.multicast_already_received = []
self.on_multicast_start(message)
elif message_type == b'\x06': # exception message
worker = message.pop(0)
try:
msg = msgpack.unpackb(message[0])
except:
msg = {b'class': 'Exception', b'message': 'parsing failed', b'traceback': message[0]}
self.on_exception_message(worker, msg[b'class'], msg[b'message'], msg[b'traceback'])
if message[0] not in self.multicast_already_received:
self.multicast_already_received.append(worker)
if self.multicast_services is None or len(self.multicast_already_received) == len(self.multicast_services):
self.stop_waiting()
#else: unknown type - do nothing
return
def wait_for_reply(self):
"""Start waiting for replies
"""
self.ioloop.start()
def stop_waiting(self):
"""Stop waiting for replies and cancel timeout
"""
self.ioloop.stop()
self.can_send = True
self._stop_timeout()
def _on_timeout(self):
"""Helper called after timeout.
"""
self.timed_out = True
self._tmo = None
self.ioloop.stop()
self.on_timeout()
return
def _start_timeout(self, timeout):
"""Helper for starting the timeout.
:param timeout: the time to wait in milliseconds.
:type timeout: int
"""
self._tmo = DelayedCallback(self._on_timeout, timeout)
self._tmo.start()
return
def _stop_timeout(self):
"""Helper for stopping timeout event
"""
if self._tmo:
self._tmo.stop()
self._tmo = None
def on_message(self, service, msg):
"""Public method called when a message arrived.
.. note:: Does nothing. Should be overloaded!
:param service: name of the worker
:type service: str
:param msg: message
:type msg: mixed
"""
pass
def on_partial_message(self, service, msg):
"""Public method called when a partial message arrived.
.. note:: Does nothing. Should be overloaded!
:param service: name of the worker
:type service: str
:param msg: message
:type msg: mixed
"""
pass
def on_error_message(self, worker, msg):
"""Public method called when an error message arrived.
.. note:: Does nothing. Should be overloaded!
:param service: name of the worker
:type service: str
:param msg: message
:type msg: str
"""
pass
def on_exception_message(self, service, cls, message, traceback):
"""Public method called when an exception arrived.
.. note:: Does nothing. Should be overloaded!
:param service: name of the worker
:type service: str
:param cls: exception class name
:type cls: str
:param message: error message
:type message: str
:param traceback: traceback
:type traceback: str
"""
pass
def on_timeout(self):
"""Public method called when a timeout occured.
.. note:: Does nothing. Should be overloaded!
"""
pass
def on_multicast_start(self, services):
"""Public method called when multicast request started
.. note:: Does nothing. Should be overloaded!
:param services: list of services in multicast group
:type services: list
"""
pass
class Worker(object):
"""Class for the MDP worker side.
Thin encapsulation of a zmq.DEALER socket.
Provides a send method with optional timeout parameter.
Will use a timeout to indicate a broker failure.
"""
max_forks = 10
ipc = 'ipc:///tmp/zmq-rpc-'+str(uuid4())
HB_INTERVAL = 1000 # in milliseconds
HB_LIVENESS = 3 # HBs to miss before connection counts as dead
def __init__(self, context, endpoint, service, multicasts=()):
"""Initialize the MDPWorker.
:param context: is the zmq context to create the socket from
:type context: zmq.Context
:param service: service name - you can put hostname here
:type service: str
:param multicasts: list of groups to subscribe
:type multicasts: list
"""
self.context = context
self.endpoint = endpoint
self.service = service.encode('utf-8') # convert to byte-string - required in python 3
self.multicasts = [m.encode('utf-8') for m in multicasts] # convert to byte-string
self.stream = None
self._tmo = None
self.need_handshake = True
self.ticker = None
self._delayed_cb = None
self._create_stream()
self.forks = []
self.curr_liveness = self.HB_LIVENESS
socket = self.context.socket(zmq.ROUTER)
socket.bind(self.ipc)
self.stream_w = ZMQStream(socket)
self.stream_w.on_recv(self._on_fork_response)
self.reply_socket = None
return
def _create_stream(self):
"""Helper to create the socket and the stream.
"""
self.on_log_event('broker.connect', 'Trying to connect do broker')
socket = self.context.socket(zmq.DEALER)
ioloop = IOLoop.instance()
self.stream = ZMQStream(socket, ioloop)
self.stream.on_recv(self._on_message)
self.stream.socket.setsockopt(zmq.LINGER, 0)
self.stream.connect(self.endpoint)
self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
self._send_ready()
for m in self.multicasts:
self._register_worker_to_multicast(m)
self.ticker.start()
return
def _tick(self):
"""Method called every HB_INTERVAL milliseconds.
"""
self.curr_liveness -= 1
self.send_hb()
if self.curr_liveness >= 0:
return
# ouch, connection seems to be dead
self.on_log_event('broker.timeout', 'Connection to broker timeouted, disconnecting')
self.shutdown(False)
# try to recreate it
self._delayed_cb = DelayedCallback(self._create_stream, 5000)
self._delayed_cb.start()
return
def send_hb(self):
"""Construct and send HB message to broker.
"""
msg = [b'', MDP_WORKER_VERSION, b'\x05']
self.stream.send_multipart(msg)
return
def shutdown(self, final=True):
"""Method to deactivate the worker connection completely.
Will delete the stream and the underlying socket.
:param final: if shutdown is final and we want to close all sockets
:type final: bool
"""
if self.ticker:
self.ticker.stop()
self.ticker = None
if not self.stream:
return
self.stream.on_recv(None)
self.disconnect()
self.stream.socket.close()
self.stream.close()
self.stream = None
self.need_handshake = True
if final:
self.stream_w.socket.close()
self.stream_w.close()
self.stream = None
return
def disconnect(self):
"""Helper method to send the workers DISCONNECT message.
"""
self.stream.socket.send_multipart([b'', MDP_WORKER_VERSION, b'\x06' ])
self.curr_liveness = self.HB_LIVENESS
return
def _send_ready(self):
"""Helper method to prepare and send the workers READY message.
"""
self.on_log_event('broker.ready', 'Sending ready to broker.')
ready_msg = [b'', MDP_WORKER_VERSION, b'\x01', self.service]
self.stream.send_multipart(ready_msg)
self.curr_liveness = self.HB_LIVENESS
return
def _register_worker_to_multicast(self, name):
"""Helper method to register worker to multicast group
:param name: group name
:type name: str
"""
self.on_log_event('broker.register-group', 'Subscribing to group \'%s\'.' % name)
reg_msg = [b'', MDP_WORKER_VERSION, b'\x07', name]
self.stream.send_multipart(reg_msg)
self.curr_liveness = self.HB_LIVENESS
return
def _on_message(self, msg):
"""Helper method called on message receive.
:param msg: message parts
:type msg: list
"""
# 1st part is empty
msg.pop(0)
# 2nd part is protocol version
protocol_version = msg.pop(0)
if protocol_version != MDP_WORKER_VERSION: # version check, ignore old versions
return
# 3rd part is message type
msg_type = msg.pop(0)
# any message resets the liveness counter
self.need_handshake = False
self.curr_liveness = self.HB_LIVENESS
if msg_type == b'\x06': # disconnect
self.curr_liveness = 0 # reconnect will be triggered by hb timer
elif msg_type == b'\x02': # request
# remaining parts are the user message
addresses, msg = self.split_address(msg)
self._on_request(addresses, msg)
elif msg_type == b'\x05':
# received hardbeat - timer handled above
pass
else:
# invalid message ignored
pass
return
def _on_fork_response(self, to_send):
"""Helper method to send message from forked worker.
This message will be received by main worker process and resend to broker.
:param to_send address and data to send
:type to_send list
"""
self.stream.send_multipart(to_send)
return
def send_reply(self, addresses, msg, partial=False, exception=False):
"""Send reply from forked worker process.
This method can be called only from do_work() method!
This method will send messages to main worker listening on local socket in /tmp/zmq-rpc-...
:param addresses: return address stack
:type addresses: list of str
:param msg: return value from called method
:type msg: mixed
:param partial: if the message is partial or final
:type partial: bool
:param exception: if the message is exception, msg format is: {'class':'c', 'message':'m', 'traceback':'t'}
:type exception: bool
"""
if not self.reply_socket:
context = zmq.Context()
self.reply_socket = context.socket(zmq.DEALER)
self.reply_socket.connect(self.ipc)
msg = msgpack.Packer().pack(msg)
if exception:
to_send = [b'', MDP_WORKER_VERSION, b'\x08']
elif partial:
to_send = [b'', MDP_WORKER_VERSION, b'\x03']
else:
to_send = [b'', MDP_WORKER_VERSION, b'\x04']
to_send.extend(addresses)
to_send.append(b'')
if isinstance(msg, list):
to_send.extend(msg)
else:
to_send.append(msg)
m = self.reply_socket.send_multipart(to_send, track=True, copy=False)
m.wait()
if not partial:
self.reply_socket.close()
self.reply_socket = None
return
def send_message(self, addresses, msg, partial=False, error=False):
"""Send response message from main worker process.
Please do not call this method from do_work()
:param addresses: return address stack
:type addresses: list of str
:param msg: return value from called method
:type msg: mixed
:param partial: if the message is partial or final
:type partial: bool
:param error: if the message is error
:type error: bool
"""
to_send = [b'', MDP_WORKER_VERSION]
if partial:
to_send.append(b'\x03')
elif error:
to_send.append(b'\x09')
else:
to_send.append(b'\x04')
to_send.extend(addresses)
to_send.append(b'')
if isinstance(msg, list):
to_send.extend(msg)
else:
to_send.append(msg)
self.stream.send_multipart(to_send)
return
def _on_request(self, addresses, message):
"""Helper method called on RPC message receive.
"""
# remove finished forks
self._remove_finished_processes()
# test max forks
if len(self.forks) >= self.max_forks:
self.send_message(addresses, b'max workers limit exceeded', error=True)
self.on_max_forks(addresses, message)
return
name = message[0]
args = msgpack.unpackb(message[1])
kwargs = msgpack.unpackb(message[2])
p = Process(target=self.do_work, args=(addresses, name, args, kwargs))
p.start()
p._args = None # free memory
self.forks.append(p)
return
def _remove_finished_processes(self):
"""Helper method dedicated to cleaning list of forked workers
"""
for f in [f for f in self.forks if not f.is_alive()]:
self.forks.remove(f)
return
def split_address(self, msg):
"""Function to split return Id and message received by ROUTER socket.
Returns 2-tuple with return Id and remaining message parts.
Empty frames after the Id are stripped.
"""
ret_ids = []
for i, p in enumerate(msg):
if p:
ret_ids.append(p)
else:
break
return ret_ids, msg[i + 1:]
def on_log_event(self, event, message):
"""Override this method if you want to log events from broker
:type event: str
:param event: event type - used for filtering
:type message: str
:param message: log message
:rtype: None
"""
pass
def on_max_forks(self, addresses, message):
"""This method is called when max_forks limit is reached
You can override this method.
"""
pass
def do_work(self, addresses, name, args, kwargs):
"""Main method responsible for handling rpc calls, and sending response messages.
Please override this method!
:param addresses: return address stack
:type addresses: list of str
:param name: name of task
:type name: str
:param args: positional task arguments
:type args: list
:param kwargs: key-value task arguments
:type kwargs: dict
"""
# this is example of simple response message
self.send_reply(addresses, 'method not implemented') # and send message to main worker
# you can also send partial message and exception - read 'send_reply' docs
return
class MQRep(object):
"""Class for the MDP worker side.
Thin encapsulation of a zmq.DEALER socket.
Provides a send method with optional timeout parameter.
Will use a timeout to indicate a broker failure.
"""
_proto_version = b'MDPW01'
# TODO: integrate that into API
HB_INTERVAL = 1000 # in milliseconds
HB_LIVENESS = 3 # HBs to miss before connection counts as dead
def __init__(self, context, service):
"""Initialize the MDPWorker.
context is the zmq context to create the socket from.
service is a byte-string with the service name.
"""
if DEBUG:
print("MQRep > __init__")
cfg = Loader('mq').load()
config = dict(cfg[1])
if config['ip'].strip() == "*":
config['ip'] = get_ip()
self.endpoint = "tcp://{0}:{1}".format(config['ip'], config['req_rep_port'])
self.context = context
self.service = service
self.stream = None
self._tmo = None
self.need_handshake = True
self.ticker = None
self._delayed_cb = None
self._create_stream()
### patch fritz
self._reconnect_in_progress = False
### end patch fritz
return
def _create_stream(self):
"""Helper to create the socket and the stream.
"""
if DEBUG:
print("MQRep > _create_stream")
socket = ZmqSocket(self.context, zmq.DEALER)
ioloop = IOLoop.instance()
self.stream = ZMQStream(socket, ioloop)
self.stream.on_recv(self._on_mpd_message)
self.stream.socket.setsockopt(zmq.LINGER, 0)
self.stream.connect(self.endpoint)
if self.ticker != None:
if DEBUG:
print("MQRep > _create_stream - stop ticker")
self.ticker.stop()
self.ticker = PeriodicCallback(self._tick, self.HB_INTERVAL)
self._send_ready()
self.ticker.start()
return
def _send_ready(self):
"""Helper method to prepare and send the workers READY message.
"""
if DEBUG:
print("MQREP > _send_ready")
ready_msg = [ b'', self._proto_version, b'\x01', self.service ]
self.stream.send_multipart(ready_msg)
self.curr_liveness = self.HB_LIVENESS
if DEBUG:
print("MQREP > _send_ready > curr_liveness <= {0}".format(self.HB_LIVENESS))
return
def _tick(self):
"""Method called every HB_INTERVAL milliseconds.
"""
if DEBUG:
print("MQREP > _tick")
self.curr_liveness -= 1
if DEBUG:
print('MQREP > _tick - {0} tick = {1}'.format(time.time(), self.curr_liveness))
self.send_hb()
if self.curr_liveness >= 0:
return
if DEBUG:
print('MQREP > _tick - {0} lost connection'.format(time.time()))
# ouch, connection seems to be dead
self.shutdown()
# try to recreate it
self._delayed_cb = DelayedCallback(self._create_stream, self.HB_INTERVAL)
self._delayed_cb.start()
return
def send_hb(self):
"""Construct and send HB message to broker.
"""
msg = [ b'', self._proto_version, b'\x04' ]
self.stream.send_multipart(msg)
return
def shutdown(self):
"""Method to deactivate the worker connection completely.
Will delete the stream and the underlying socket.
"""
if self.ticker:
self.ticker.stop()
self.ticker = None
if not self.stream:
return
self.stream.socket.close()
self.stream.close()
self.stream = None
self.timed_out = False
self.need_handshake = True
self.connected = False
return
def reply(self, msg):
"""Send the given message.
msg can either be a byte-string or a list of byte-strings.
"""
## if self.need_handshake:
## raise ConnectionNotReadyError()
# prepare full message
to_send = self.envelope
self.envelope = None
if isinstance(msg, list):
to_send.extend(msg)
else:
to_send.append(msg)
self.stream.send_multipart(to_send)
return
def _on_mpd_message(self, msg):
"""Helper method called on message receive.
msg is a list w/ the message parts
"""
if DEBUG:
print("MQRep > _on_mpd_message : {0} - {1}".format(time.strftime("%H:%M:%S"), msg))
# 1st part is empty
msg.pop(0)
# 2nd part is protocol version
# TODO: version check
proto = msg.pop(0)
# 3rd part is message type
msg_type = msg.pop(0)
# XXX: hardcoded message types!
# any message resets the liveness counter
self.need_handshake = False
self.curr_liveness = self.HB_LIVENESS
if DEBUG:
print("MQREP > _on_mpd_message > curr_liveness <= {0}".format(self.HB_LIVENESS))
if msg_type == b'\x05': # disconnect
if DEBUG:
print("MQREP > _on_mpd_message > type x05 : disconnect")
self.curr_liveness = 0 # reconnect will be triggered by hb timer
elif msg_type == b'\x02': # request
if DEBUG:
print("MQREP > _on_mpd_message > type x02 : request")
# remaining parts are the user message
envelope, msg = split_address(msg)
envelope.append(b'')
envelope = [ b'', self._proto_version, b'\x03'] + envelope # REPLY
self.envelope = envelope
mes = MQMessage()
mes.set(msg)
#print("MQRep > before self.on_mdp_request")
#print(self.on_mdp_request)
#print(mes)
try:
self.on_mdp_request(mes)
except:
print("ERROR {0}".format(traceback.format_exc()))
else:
if DEBUG:
print("MQREP > _on_mpd_message > type ??? : invalid or hbeat")
# invalid message
# ignored
# if \x04, this is a hbeat message
pass
return
def on_mdp_request(self, msg):
"""Public method called when a request arrived.
Must be overloaded!
"""
pass