def testDistributeTasksComplex(self):
"""test dispatch work between several gateways (more complex case)"""
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write('[Main]\n')
tmpfile.write('admin[0-2]: gw[0-1]\n')
tmpfile.write('gw0: n[0-9]\n')
tmpfile.write('gw1: gwa[0-1]\n')
tmpfile.write('gwa0: n[10-19]\n')
tmpfile.write('gwa1: n[20-29]\n')
tmpfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
tree = parser.tree('admin1')
wtree = WorkerTree('dummy',
None,
0,
command=':',
topology=tree,
newroot='admin1')
dist = wtree._distribute(5, NodeSet('n[0-29]'))
self.assertEquals(str(dist['gw0']), 'n[0-9]')
self.assertEquals(str(dist['gw1']), 'n[10-29]')
def testDistributeTasksSimple(self):
"""test dispatch work between several gateways (simple case)"""
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write('[Main]\n')
tmpfile.write('admin[0-2]: gw[0-3]\n')
tmpfile.write('gw[0-1]: node[0-9]\n')
tmpfile.write('gw[2-3]: node[10-19]\n')
tmpfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
tree = parser.tree('admin1')
wtree = WorkerTree('dummy', None, 0, command=':', topology=tree,
newroot='admin1')
dist = wtree._distribute(128, NodeSet('node[2-18]'))
self.assertEquals(dist['gw0'], NodeSet('node[2-8/2]'))
self.assertEquals(dist['gw2'], NodeSet('node[10-18/2]'))
def testDistributeTasksComplex(self):
"""test dispatch work between several gateways (more complex case)"""
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write('[Main]\n')
tmpfile.write('admin[0-2]: gw[0-1]\n')
tmpfile.write('gw0: n[0-9]\n')
tmpfile.write('gw1: gwa[0-1]\n')
tmpfile.write('gwa0: n[10-19]\n')
tmpfile.write('gwa1: n[20-29]\n')
tmpfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
tree = parser.tree('admin1')
wtree = WorkerTree('dummy', None, 0, command=':', topology=tree,
newroot='admin1')
dist = wtree._distribute(5, NodeSet('n[0-29]'))
self.assertEquals(str(dist['gw0']), 'n[0-9]')
self.assertEquals(str(dist['gw1']), 'n[10-29]')
def testDistributeTasksSimple(self):
"""test dispatch work between several gateways (simple case)"""
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write('[Main]\n')
tmpfile.write('admin[0-2]: gw[0-3]\n')
tmpfile.write('gw[0-1]: node[0-9]\n')
tmpfile.write('gw[2-3]: node[10-19]\n')
tmpfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
tree = parser.tree('admin1')
wtree = WorkerTree('dummy',
None,
0,
command=':',
topology=tree,
newroot='admin1')
dist = wtree._distribute(128, NodeSet('node[2-18]'))
self.assertEquals(dist['gw0'], NodeSet('node[2-8/2]'))
self.assertEquals(dist['gw2'], NodeSet('node[10-18/2]'))
def testExecuteTasksOnNeighbors(self):
"""test execute tasks on directly connected machines"""
tmpfile = tempfile.NamedTemporaryFile()
myhost = my_node()
cfgparser = load_cfg('topology1.conf')
neighbor = cfgparser.get('CONFIG', 'NEIGHBOR')
gateways = cfgparser.get('CONFIG', 'GATEWAYS')
targets = cfgparser.get('CONFIG', 'TARGETS')
tmpfile.write('[Main]\n')
tmpfile.write('%s: %s\n' % (myhost, neighbor))
tmpfile.write('%s: %s\n' % (neighbor, gateways))
tmpfile.write('%s: %s\n' % (gateways, targets))
tmpfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
tree = parser.tree(myhost)
wtree = WorkerTree(NodeSet(targets),
None,
0,
command='echo ok',
topology=tree,
newroot=myhost)
# XXX Need to propagate topology for this to work in tests
raise RuntimeError
task = task_self()
task.set_info('debug', True)
task.schedule(wtree)
task.resume()
for buf, nodes in task.iter_buffers():
print '-' * 15
print str(nodes)
print '-' * 15
print buf
print ''
def _check_channel_ctl_shell(self,
command,
target,
stderr,
remote,
reply_msg_class,
reply_pattern,
write_string=None,
timeout=-1,
replycnt=1,
reply_rc=0):
"""helper to check channel shell action"""
self.channel_send_start()
msg = self.recvxml(StartMessage)
self.channel_send_cfg()
msg = self.recvxml(ACKMessage)
# prepare a remote shell command request...
workertree = WorkerTree(nodes=target,
handler=None,
timeout=timeout,
command=command)
# code snippet from PropagationChannel.shell()
ctl = ControlMessage(id(workertree))
ctl.action = 'shell'
ctl.target = NodeSet(target)
info = task_self()._info.copy()
info['debug'] = False
ctl_data = {
'cmd': command,
'invoke_gateway': workertree.invoke_gateway,
'taskinfo': info,
'stderr': stderr,
'timeout': timeout,
'remote': remote
}
ctl.data_encode(ctl_data)
self.gateway.send(ctl.xml())
self.recvxml(ACKMessage)
if write_string:
ctl = ControlMessage(id(workertree))
ctl.action = 'write'
ctl.target = NodeSet(target)
ctl_data = {
'buf': write_string,
}
# Send write message
ctl.data_encode(ctl_data)
self.gateway.send(ctl.xml())
self.recvxml(ACKMessage)
# Send EOF message
ctl = ControlMessage(id(workertree))
ctl.action = 'eof'
ctl.target = NodeSet(target)
self.gateway.send(ctl.xml())
self.recvxml(ACKMessage)
while replycnt > 0:
msg = self.recvxml(reply_msg_class)
replycnt -= len(NodeSet(msg.nodes))
self.assertTrue(msg.nodes in ctl.target)
if msg.has_payload or reply_pattern:
msg_data = msg.data_decode()
try:
if not reply_pattern.search(msg_data):
self.assertEqual(
msg.data, reply_pattern,
'Pattern "%s" not found in data="%s"' %
(reply_pattern.pattern, msg_data))
except AttributeError:
# not a regexp
self.assertEqual(msg_data, reply_pattern)
if timeout <= 0:
msg = self.recvxml(RetcodeMessage)
self.assertEqual(msg.retcode, reply_rc)
self.channel_send_stop()
self.gateway.wait()
self.gateway.close()
def recv_ctl(self, msg):
"""receive control message with actions to perform"""
if msg.type == ControlMessage.ident:
self.logger.debug('GatewayChannel._state_ctl')
if msg.action == 'shell':
data = msg.data_decode()
cmd = data['cmd']
stderr = data['stderr']
timeout = data['timeout']
remote = data['remote']
#self.propagation.invoke_gateway = data['invoke_gateway']
self.logger.debug('decoded gw invoke (%s)',
data['invoke_gateway'])
taskinfo = data['taskinfo']
self.logger.debug('assigning task infos (%s)',
data['taskinfo'])
task = task_self()
task._info.update(taskinfo)
task.set_info('print_debug', _gw_print_debug)
if task.info('debug'):
self.logger.setLevel(logging.DEBUG)
self.logger.debug('inherited fanout value=%d',
task.info("fanout"))
self.logger.debug('launching execution/enter gathering state')
responder = WorkerTreeResponder(task, self, msg.srcid)
self.propagation = WorkerTree(msg.target,
responder,
timeout,
command=cmd,
topology=self.topology,
newroot=self.nodename,
stderr=stderr,
remote=remote)
# FIXME ev_start-not-called workaround
responder.worker = self.propagation
self.propagation.upchannel = self
task.schedule(self.propagation)
self.logger.debug("WorkerTree scheduled")
self._ack(msg)
elif msg.action == 'write':
data = msg.data_decode()
self.logger.debug('GatewayChannel write: %d bytes',
len(data['buf']))
self.propagation.write(data['buf'])
self._ack(msg)
elif msg.action == 'eof':
self.logger.debug('GatewayChannel eof')
self.propagation.set_write_eof()
self._ack(msg)
else:
self.logger.error('unexpected CTL action: %s', msg.action)
else:
self.logger.error('unexpected message: %s', str(msg))
class GatewayChannel(Channel):
"""high level logic for gateways"""
def __init__(self, task):
Channel.__init__(self, error_response=True)
self.task = task
self.nodename = None
self.topology = None
self.propagation = None
self.logger = logging.getLogger(__name__)
def start(self):
"""initialization"""
# prepare communication
self._init()
self.logger.debug('ready to accept channel communication')
def close(self):
"""close gw channel"""
self.logger.debug('closing gateway channel')
self._close()
def recv(self, msg):
"""handle incoming message"""
try:
self.logger.debug('handling incoming message: %s', str(msg))
if msg.type == EndMessage.ident:
self.logger.debug('recv: got EndMessage')
self._close()
elif self.setup:
self.recv_ctl(msg)
elif self.opened:
self.recv_cfg(msg)
elif msg.type == StartMessage.ident:
self.logger.debug('got start message %s', msg)
self.opened = True
self._open()
self.logger.debug('channel started (version %s on remote end)',
self._xml_reader.version)
else:
self.logger.error('unexpected message: %s', str(msg))
raise MessageProcessingError('unexpected message: %s' % msg)
except MessageProcessingError as ex:
self.logger.error('on recv(): %s', str(ex))
self.send(ErrorMessage(str(ex)))
self._close()
except EngineAbortException:
# gateway task abort: don't handle like other exceptions
raise
except Exception as ex:
self.logger.exception('on recv(): %s', str(ex))
self.send(ErrorMessage(str(ex)))
self._close()
def recv_cfg(self, msg):
"""receive cfg/topology configuration"""
if msg.type != ConfigurationMessage.ident:
raise MessageProcessingError('unexpected message: %s' % msg)
self.logger.debug('got channel configuration')
# gw node name
hostname = _getshorthostname()
if not msg.gateway:
self.nodename = hostname
self.logger.warn('gw name not provided, using system hostname %s',
self.nodename)
else:
self.nodename = msg.gateway
self.logger.debug('using gateway node name %s', self.nodename)
if self.nodename.lower() != hostname.lower():
self.logger.debug('gw name %s does not match system hostname %s',
self.nodename, hostname)
# topology
task_self().topology = self.topology = msg.data_decode()
self.logger.debug('decoded propagation tree')
self.logger.debug('\n%s', self.topology)
self.setup = True
self._ack(msg)
def recv_ctl(self, msg):
"""receive control message with actions to perform"""
if msg.type == ControlMessage.ident:
self.logger.debug('GatewayChannel._state_ctl')
if msg.action == 'shell':
data = msg.data_decode()
cmd = data['cmd']
stderr = data['stderr']
timeout = data['timeout']
remote = data['remote']
#self.propagation.invoke_gateway = data['invoke_gateway']
self.logger.debug('decoded gw invoke (%s)',
data['invoke_gateway'])
taskinfo = data['taskinfo']
self.logger.debug('assigning task infos (%s)',
data['taskinfo'])
task = task_self()
task._info.update(taskinfo)
task.set_info('print_debug', _gw_print_debug)
if task.info('debug'):
self.logger.setLevel(logging.DEBUG)
self.logger.debug('inherited fanout value=%d',
task.info("fanout"))
self.logger.debug('launching execution/enter gathering state')
responder = WorkerTreeResponder(task, self, msg.srcid)
self.propagation = WorkerTree(msg.target,
responder,
timeout,
command=cmd,
topology=self.topology,
newroot=self.nodename,
stderr=stderr,
remote=remote)
# FIXME ev_start-not-called workaround
responder.worker = self.propagation
self.propagation.upchannel = self
task.schedule(self.propagation)
self.logger.debug("WorkerTree scheduled")
self._ack(msg)
elif msg.action == 'write':
data = msg.data_decode()
self.logger.debug('GatewayChannel write: %d bytes',
len(data['buf']))
self.propagation.write(data['buf'])
self._ack(msg)
elif msg.action == 'eof':
self.logger.debug('GatewayChannel eof')
self.propagation.set_write_eof()
self._ack(msg)
else:
self.logger.error('unexpected CTL action: %s', msg.action)
else:
self.logger.error('unexpected message: %s', str(msg))
def _ack(self, msg):
"""acknowledge a received message"""
self.send(ACKMessage(msg.msgid))
def ev_close(self, worker):
"""Gateway (parent) channel is closing.
We abort the whole gateway task to stop other running workers.
This avoids any unwanted remaining processes on gateways.
"""
self.logger.debug('GatewayChannel: ev_close')
self.worker.task.abort()
