def _sync(self):
"""
Send output data and receive input data.
Notes
-----
Assumes that the attributes used for input and output already
exist.
Each message is a tuple containing a module ID and data; for
outbound messages, the ID is that of the destination module.
for inbound messages, the ID is that of the source module.
Data is serialized before being sent and unserialized when
received.
"""
if self.net in ['none', 'ctrl']:
self.logger.info('not synchronizing with network')
else:
self.logger.info('synchronizing with network')
# Send outbound data:
if self.net in ['out', 'full']:
# Send all data in outbound buffer:
send_ids = [out_id for out_id in self._out_ids]
for out_id, data in self._out_data:
self.sock_data.send(msgpack.packb((out_id, data)))
send_ids.remove(out_id)
self.logger.info('sent to %s: %s' % (out_id, str(data)))
# Send data tuples containing None to those modules for which no
# actual data was generated to satisfy the barrier condition:
for out_id in send_ids:
self.sock_data.send(msgpack.packb((out_id, None)))
self.logger.info('sent to %s: %s' % (out_id, None))
# All output IDs should be sent data by this point:
self.logger.info('sent data to all output IDs')
# Receive inbound data:
if self.net in ['in', 'full']:
# Wait until inbound data is received from all source modules:
while not all((q for q in self._in_data.itervalues())):
# Use poller to avoid blocking:
if is_poll_in(self.sock_data, self.data_poller):
in_id, data = msgpack.unpackb(self.sock_data.recv())
self.logger.info('recv from %s: %s ' %
(in_id, str(data)))
# Ignore incoming data containing None:
if data is not None:
self._in_data[in_id].append(data)
# Stop the synchronization if a quit message has been received:
if not self.running:
self.logger.info('run loop stopped - stopping sync')
break
self.logger.info('recv data from all input IDs')
def _sync(self):
"""
Send output data and receive input data.
Notes
-----
Assumes that the attributes used for input and output already
exist.
Each message is a tuple containing a module ID and data; for
outbound messages, the ID is that of the destination module.
for inbound messages, the ID is that of the source module.
Data is serialized before being sent and unserialized when
received.
"""
if self.net in ['none', 'ctrl']:
self.logger.info('not synchronizing with network')
else:
self.logger.info('synchronizing with network')
# Send outbound data:
if self.net in ['out', 'full']:
# Send all data in outbound buffer:
send_ids = self.out_ids
for out_id, data in self._out_data:
self.sock_data.send(msgpack.packb((out_id, data)))
send_ids.remove(out_id)
self.logger.info('sent to %s: %s' % (out_id, str(data)))
# Send data tuples containing None to those modules for which no
# actual data was generated to satisfy the barrier condition:
for out_id in send_ids:
self.sock_data.send(msgpack.packb((out_id, None)))
self.logger.info('sent to %s: %s' % (out_id, None))
# All output IDs should be sent data by this point:
self.logger.info('sent data to all output IDs')
# Receive inbound data:
if self.net in ['in', 'full']:
# Wait until inbound data is received from all source modules:
while not all((q for q in self._in_data.itervalues())):
# Use poller to avoid blocking:
if is_poll_in(self.sock_data, self.data_poller):
in_id, data = msgpack.unpackb(self.sock_data.recv())
self.logger.info('recv from %s: %s ' % (in_id, str(data)))
# Ignore incoming data containing None:
if data is not None:
self._in_data[in_id].append(data)
# Stop the synchronization if a quit message has been received:
if not self.running:
self.logger.info('run loop stopped - stopping sync')
break
self.logger.info('recv data from all input IDs')
def send_ctrl_msg(self, i, *msg):
"""
Send control message(s) to a module.
"""
self.sock_ctrl.send_multipart([i]+msg)
self.logger.info('sent to %s: %s' % (i, msg))
while True:
if is_poll_in(self.sock_ctrl, self.ctrl_poller):
j, data = self.sock_ctrl.recv_multipart()
self.logger.info('recv from %s: ack' % j)
break
def send_ctrl_msg(self, i, *msg):
"""
Send control message(s) to a module.
"""
self.sock_ctrl.send_multipart([i] + msg)
self.logger.info('sent to %s: %s' % (i, msg))
while True:
if is_poll_in(self.sock_ctrl, self.ctrl_poller):
j, data = self.sock_ctrl.recv_multipart()
self.logger.info('recv from %s: ack' % j)
break
def join_modules(self, send_quit=False):
"""
Wait until all modules have stopped.
Parameters
----------
send_quit : bool
If True, send quit messages to all modules.
"""
self.logger.info('waiting for modules to shut down')
recv_ids = self.mod_dict.keys()
while recv_ids:
i = recv_ids[0]
# Send quit messages to all live modules:
if send_quit:
self.logger.info(str(recv_ids))
self.logger.info('sent to %s: quit' % i)
self.sock_ctrl.send_multipart([i, 'quit'])
# If a module acknowledges receiving a quit message,
# wait for it to shutdown:
if is_poll_in(self.sock_ctrl, self.ctrl_poller):
j, data = self.sock_ctrl.recv_multipart()
self.logger.info('recv from %s: %s' % (j, data))
if j in recv_ids and data == 'shutdown':
self.logger.info('waiting for module %s to shut down' % j)
recv_ids.remove(j)
self.mod_dict[j].join(1)
self.logger.info('module %s shut down' % j)
# Sometimes quit messages are received but the acknowledgements are
# lost; if so, the module will eventually shutdown:
# XXX this shouldn't be necessary XXX
if not self.mod_dict[i].is_alive() and i in recv_ids:
self.logger.info('%s shutdown without ack' % i)
recv_ids.remove(i)
self.logger.info('all modules stopped')
def join_modules(self, send_quit=False):
"""
Wait until all modules have stopped.
Parameters
----------
send_quit : bool
If True, send quit messages to all modules.
"""
self.logger.info('waiting for modules to shut down')
recv_ids = self.modules.keys()
while recv_ids:
i = recv_ids[0]
# Send quit messages to all live modules:
if send_quit:
self.logger.info('live modules: ' + str(recv_ids))
self.logger.info('sent to %s: quit' % i)
self.sock_ctrl.send_multipart([i, 'quit'])
# If a module acknowledges receiving a quit message,
# wait for it to shutdown:
if is_poll_in(self.sock_ctrl, self.ctrl_poller):
j, data = self.sock_ctrl.recv_multipart()
self.logger.info('recv from %s: %s' % (j, data))
if j in recv_ids and data == 'shutdown':
self.logger.info('waiting for module %s to shut down' % j)
recv_ids.remove(j)
self.modules[j].join(1)
self.logger.info('module %s shut down' % j)
# Sometimes quit messages are received but the acknowledgements are
# lost; if so, the module will eventually shutdown:
# XXX this shouldn't be necessary XXX
if not self.modules[i].is_alive() and i in recv_ids:
self.logger.info('%s shutdown without ack' % i)
recv_ids.remove(i)
self.logger.info('all modules stopped')
