def testExecuteTasksOnNeighbors(self):
"""test execute tasks on directly connected machines"""
tmpfile = tempfile.NamedTemporaryFile()
myhost = my_node()
cfgparser = load_cfg('topology1.conf')
neighbor = cfgparser.get('CONFIG', 'NEIGHBOR')
gateways = cfgparser.get('CONFIG', 'GATEWAYS')
targets = cfgparser.get('CONFIG', 'TARGETS')
tmpfile.write('[Main]\n')
tmpfile.write('%s: %s\n' % (myhost, neighbor))
tmpfile.write('%s: %s\n' % (neighbor, gateways))
tmpfile.write('%s: %s\n' % (gateways, targets))
tmpfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
tree = parser.tree(myhost)
wtree = WorkerTree(NodeSet(targets), None, 0, command='echo ok',
topology=tree, newroot=myhost)
# XXX Need to propagate topology for this to work in tests
raise RuntimeError
task = task_self()
task.set_info('debug', True)
task.schedule(wtree)
task.resume()
for buf, nodes in task.iter_buffers():
print '-' * 15
print str(nodes)
print '-' * 15
print buf
print ''
def testDistributeTasksComplex(self):
"""test dispatch work between several gateways (more complex case)"""
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write('[Main]\n')
tmpfile.write('admin[0-2]: gw[0-1]\n')
tmpfile.write('gw0: n[0-9]\n')
tmpfile.write('gw1: gwa[0-1]\n')
tmpfile.write('gwa0: n[10-19]\n')
tmpfile.write('gwa1: n[20-29]\n')
tmpfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
tree = parser.tree('admin1')
wtree = WorkerTree('dummy',
None,
0,
command=':',
topology=tree,
newroot='admin1')
dist = wtree._distribute(5, NodeSet('n[0-29]'))
self.assertEquals(str(dist['gw0']), 'n[0-9]')
self.assertEquals(str(dist['gw1']), 'n[10-29]')
def setUp(self):
"""generate a sample topology tree"""
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write('[Main]\n')
tmpfile.write('admin[0-2]: proxy\n')
tmpfile.write('proxy: STA[0-400]\n')
tmpfile.write('STA[0-400]: STB[0-2000]\n')
tmpfile.write('STB[0-2000]: node[0-11000]\n')
tmpfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
self.topology = parser.tree('admin1')
# XXX
os.environ["PYTHONPATH"] = "%s/../lib" % os.getcwd()
def setUp(self):
"""generate a sample topology tree"""
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write('[Main]\n')
tmpfile.write('admin[0-2]: proxy\n')
tmpfile.write('proxy: STA[0-400]\n')
tmpfile.write('STA[0-400]: STB[0-2000]\n')
tmpfile.write('STB[0-2000]: node[0-11000]\n')
tmpfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
self.topology = parser.tree('admin1')
# XXX
os.environ["PYTHONPATH"] = "%s/../lib" % os.getcwd()
def testDistributeTasksSimple(self):
"""test dispatch work between several gateways (simple case)"""
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write('[Main]\n')
tmpfile.write('admin[0-2]: gw[0-3]\n')
tmpfile.write('gw[0-1]: node[0-9]\n')
tmpfile.write('gw[2-3]: node[10-19]\n')
tmpfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
tree = parser.tree('admin1')
wtree = WorkerTree('dummy', None, 0, command=':', topology=tree,
newroot='admin1')
dist = wtree._distribute(128, NodeSet('node[2-18]'))
self.assertEquals(dist['gw0'], NodeSet('node[2-8/2]'))
self.assertEquals(dist['gw2'], NodeSet('node[10-18/2]'))
def testDistributeTasksComplex(self):
"""test dispatch work between several gateways (more complex case)"""
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write('[Main]\n')
tmpfile.write('admin[0-2]: gw[0-1]\n')
tmpfile.write('gw0: n[0-9]\n')
tmpfile.write('gw1: gwa[0-1]\n')
tmpfile.write('gwa0: n[10-19]\n')
tmpfile.write('gwa1: n[20-29]\n')
tmpfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
tree = parser.tree('admin1')
wtree = WorkerTree('dummy', None, 0, command=':', topology=tree,
newroot='admin1')
dist = wtree._distribute(5, NodeSet('n[0-29]'))
self.assertEquals(str(dist['gw0']), 'n[0-9]')
self.assertEquals(str(dist['gw1']), 'n[10-29]')
def testPropagationDriver(self):
"""test propagation logic"""
## --------
# This test use a tricky topology, whose next hop for any admin node is
# localhost. This way, the test machine will connect to itself as if it
# was a remote gateway.
# Then, instead of talking to a gateway instance, it will load a
# specifically crafted file, that contains XML messages as those sent by
# an actual gateway.
# -----------------------
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write('[Main]\n')
tmpfile.write('admin[0-2]: localhost\n')
tmpfile.write('localhost: node[0-500]\n')
gwfile = tempfile.NamedTemporaryFile()
gwfile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
gwfile.write('<channel>\n')
gwfile.write('<message type="ACK" msgid="0" ack="0"></message>\n')
gwfile.write('<message type="ACK" msgid="1" ack="1"></message>\n')
gwfile.write('<message type="ACK" msgid="2" ack="2"></message>\n')
gwfile.write('<message type="CTL" target="node[0-500]" msgid="3" ' \
'action="res">' \
'UydMaW51eCAyLjYuMTgtOTIuZWw1ICMxIFNNUCBUdWUgQXByIDI5IDEzOjE2OjE1IEVEVCAyMDA4' \
'IHg4Nl82NCB4ODZfNjQgeDg2XzY0IEdOVS9MaW51eCcKcDEKLg==' \
'</message>\n')
gwfile.write('</channel>\n')
tmpfile.flush()
gwfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
"""
XXX need a way to override the way the gateway is remotely launch
"""
raise RuntimeError
"""
def testExecuteTasksOnNeighbors(self):
"""test execute tasks on directly connected machines"""
tmpfile = tempfile.NamedTemporaryFile()
myhost = my_node()
cfgparser = load_cfg('topology1.conf')
neighbor = cfgparser.get('CONFIG', 'NEIGHBOR')
gateways = cfgparser.get('CONFIG', 'GATEWAYS')
targets = cfgparser.get('CONFIG', 'TARGETS')
tmpfile.write('[Main]\n')
tmpfile.write('%s: %s\n' % (myhost, neighbor))
tmpfile.write('%s: %s\n' % (neighbor, gateways))
tmpfile.write('%s: %s\n' % (gateways, targets))
tmpfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
tree = parser.tree(myhost)
wtree = WorkerTree(NodeSet(targets),
None,
0,
command='echo ok',
topology=tree,
newroot=myhost)
# XXX Need to propagate topology for this to work in tests
raise RuntimeError
task = task_self()
task.set_info('debug', True)
task.schedule(wtree)
task.resume()
for buf, nodes in task.iter_buffers():
print '-' * 15
print str(nodes)
print '-' * 15
print buf
print ''
def testPropagationDriver(self):
"""test propagation logic"""
## --------
# This test use a tricky topology, whose next hop for any admin node is
# localhost. This way, the test machine will connect to itself as if it
# was a remote gateway.
# Then, instead of talking to a gateway instance, it will load a
# specifically crafted file, that contains XML messages as those sent by
# an actual gateway.
# -----------------------
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write('[Main]\n')
tmpfile.write('admin[0-2]: localhost\n')
tmpfile.write('localhost: node[0-500]\n')
gwfile = tempfile.NamedTemporaryFile()
gwfile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
gwfile.write('<channel>\n')
gwfile.write('<message type="ACK" msgid="0" ack="0"></message>\n')
gwfile.write('<message type="ACK" msgid="1" ack="1"></message>\n')
gwfile.write('<message type="ACK" msgid="2" ack="2"></message>\n')
gwfile.write('<message type="CTL" target="node[0-500]" msgid="3" ' \
'action="res">' \
'UydMaW51eCAyLjYuMTgtOTIuZWw1ICMxIFNNUCBUdWUgQXByIDI5IDEzOjE2OjE1IEVEVCAyMDA4' \
'IHg4Nl82NCB4ODZfNjQgeDg2XzY0IEdOVS9MaW51eCcKcDEKLg==' \
'</message>\n')
gwfile.write('</channel>\n')
tmpfile.flush()
gwfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
"""
XXX need a way to override the way the gateway is remotely launch
"""
raise RuntimeError
"""
def testDistributeTasksSimple(self):
"""test dispatch work between several gateways (simple case)"""
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write('[Main]\n')
tmpfile.write('admin[0-2]: gw[0-3]\n')
tmpfile.write('gw[0-1]: node[0-9]\n')
tmpfile.write('gw[2-3]: node[10-19]\n')
tmpfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
tree = parser.tree('admin1')
wtree = WorkerTree('dummy',
None,
0,
command=':',
topology=tree,
newroot='admin1')
dist = wtree._distribute(128, NodeSet('node[2-18]'))
self.assertEquals(dist['gw0'], NodeSet('node[2-8/2]'))
self.assertEquals(dist['gw2'], NodeSet('node[10-18/2]'))
def testCompletePropagation(self):
"""test a complete command propagation trip"""
#
# This test relies on configured parameters (topology2.conf)
tmpfile = tempfile.NamedTemporaryFile()
logging.basicConfig(
level=logging.DEBUG
)
logging.debug("STARTING")
hostname = my_node()
cfgparser = load_cfg('topology2.conf')
neighbors = cfgparser.get('CONFIG', 'NEIGHBORS')
targets = cfgparser.get('CONFIG', 'TARGETS')
tmpfile.write('[DEFAULT]\n')
tmpfile.write('%s: %s\n' % (hostname, neighbors))
tmpfile.write('%s: %s\n' % (neighbors, targets))
tmpfile.flush()
parser = TopologyParser()
parser.load(tmpfile.name)
tmpfile.close()
nfs_tmpdir = os.path.expanduser('~/.clustershell/tests/tmp')
tree = parser.tree(hostname)
print tree
ptree = PropagationTree(tree, hostname)
ptree.upchannel = None
ptree.edgehandler = DirectHandler()
ptree.fanout = 20
ptree.invoke_gateway = \
'cd %s; PYTHONPATH=../lib python -m ClusterShell/Gateway -Bu' % \
os.getcwd()
#print ptree.invoke_gateway
## delete remaining files from previous tests
for filename in os.listdir(nfs_tmpdir):
if filename.startswith("fortoy"):
os.remove(os.path.join(nfs_tmpdir, filename))
dst = NodeSet(targets)
task = ptree.execute('python -c "import time; print time.time()" > ' + \
os.path.join(nfs_tmpdir, '$(hostname)'), dst, 20)
#task = ptree.execute('sleep 2; echo "output from $(hostname)"', \
# dst, 20)
self.assert_(task)
res = NodeSet()
times = []
for filename in os.listdir(nfs_tmpdir):
for k in dst:
if filename.startswith(str(k)):
res.add(k)
fd = open(os.path.join(nfs_tmpdir, filename))
times.append(float(fd.read()))
fd.close()
self.assertEquals(str(res), str(dst))
print "Complete propagation time: %fs for %d nodes" % \
(max(times) - min(times), len(dst))
