if not skw.get('launch'):
if len(args) > 0:
skw['launch'] = sshcmd
skw['nodeList'] = args
if skw.has_key('workerCount'):
print >> sys.stderr, 'warning: ignoring -c option since workers are specified'
else:
if len(args) > 0:
print >> sys.stderr, \
'warning: ignoring unused arguments:', " ".join(args)
print >> sys.stderr, 'starting sleigh in web launch mode'
s = Sleigh(**skw)
s.eachWorker(worker, blocking=False)
for i in range(numtasks):
s.userNws.store('task', i)
for i in range(numtasks):
result = s.userNws.fetch('result')
print result[0], 'times 2 is', result[1]
except getopt.GetoptError, e:
print >> sys.stderr, e.msg
sys.exit(1)
except ValueError, e:
print >> sys.stderr, "option %s requires an integer argument" % opt
sys.exit(1)
except (KeyboardInterrupt, SystemExit):
if len(args) > 0:
print >> sys.stderr, \
'warning: ignoring unused arguments:', " ".join(args)
print >> sys.stderr, 'starting sleigh in web launch mode'
# create a Sleigh, and compute the number of workers.
# this is necessary because of web launch.
s = Sleigh(**skw)
numworkers, complete = s.status(True, timeout)
# include the master as one of the workers (which works even
# when no workers join)
numworkers += 1
# tell the workers to execute the ring function defined in this file
s.eachWorker(ring, numworkers, numtasks, blocking=False)
# the master becomes the last worker
global SleighRank, SleighUserNws
SleighRank = numworkers - 1
SleighUserNws = s.userNws
print "Master assigned rank %d" % SleighRank
# time how long it takes the token to go all the way around the
# ring numtask times.
start = time.time()
s.userNws.store('worker_0', 0)
ring(numworkers, numtasks)
finish = time.time()
token = s.userNws.fetch('worker_0')
if len(args) > 0:
skw['launch'] = sshcmd
skw['nodeList'] = args
if skw.has_key('workerCount'):
print >> sys.stderr, 'warning: ignoring -c option since workers are specified'
# create a Sleigh, and compute the number of workers.
# this is necessary because of web launch.
s = Sleigh(**skw)
s.userNws.declare('store', 'single')
print "Running with %d workers and %d tasks" % \
(s.workerCount, numtasks)
# tell the workers to execute the store function defined in this file
s.eachWorker(store, numtasks)
print "Finished"
except getopt.GetoptError, e:
print >> sys.stderr, e.msg
sys.exit(1)
except ValueError, e:
print >> sys.stderr, "option %s requires an integer argument" % opt
sys.exit(1)
except (KeyboardInterrupt, SystemExit):
pass
except:
ex = sys.exc_info()
print >> sys.stderr, ex[0], ex[1]
#!/usr/bin/env python
import os
from nws.sleigh import Sleigh, sshcmd
from socket import getfqdn
slargs = {}
try: slargs['nwsHost'] = os.environ['NWS_HOST']
except: pass
try: slargs['nwsPort'] = int(os.environ['NWS_PORT'])
except: pass
try:
slargs['nodeList'] = os.environ['NWS_NODES'].split()
slargs['launch'] = sshcmd
print "using ssh to launch workers on machines:", slargs['nodeList']
except:
print "using default local workers"
s = Sleigh(**slargs)
# use os.getloadavg if they have it (it's not in Python 2.2 or on Windows)
# because socket.getfqdn runs terribly slowly on Mac OS X
if hasattr(os, 'getloadavg'):
for i in xrange(1000): print i, s.eachWorker(os.getloadavg)
else:
for i in xrange(1000): print i, s.eachWorker(getfqdn)
s.stop()
# create a Sleigh, and compute the number of workers.
# this is necessary because of web launch.
s = Sleigh(**skw)
fobj = open(fname, 'rb')
print "Storing %s to server..." % fname
s.userNws.declare('longvalue', 'single')
s.userNws.storeFile('longvalue', fobj)
fobj.close()
print "Running with %d workers and %d tasks" % \
(s.workerCount, numtasks)
# tell the workers to execute the longvalue function
# defined in this file
print s.eachWorker(longvalue, numtasks, fname)
raw_input('Hit return to finish ')
print "Finished"
except getopt.GetoptError, e:
print >> sys.stderr, e.msg
sys.exit(1)
except ValueError, e:
print >> sys.stderr, "option %s requires an integer argument" % opt
sys.exit(1)
except (KeyboardInterrupt, SystemExit):
pass
except:
ex = sys.exc_info()
print >> sys.stderr, ex[0], ex[1]
class SleighTest(unittest.TestCase):
log = getLogger('SleighTest')
wsname = 'unit test ws'
def setUp(self):
self.log.info('Setting up')
self.s = Sleigh(**slargs)
def tearDown(self):
self.log.info('Tearing down')
try: self.s.stop()
except: pass
def testMonitor(self):
"""Test one-to-all broadcast with a single mode variable"""
for n, s in [(1000, 0.0), (100, 0.1), (10, 1.0)]:
pending = self.s.eachWorker(monitorWorker, blocking=False)
time.sleep(3) # not necessary for test to pass
self.s.nws.declare('monitor', SINGLE)
for i in xrange(n):
self.s.nws.store('monitor', i)
time.sleep(s) # not necessary for test to pass
time.sleep(3) # not necessary for test to pass
self.s.nws.deleteVar('monitor')
results = pending.wait()
self.assertEquals(len(results), self.s.workerCount)
for r in results:
self.assert_(len(r) <= n)
if len(r) > 0:
p = r.pop(0)
for t in r:
self.assert_(t > p)
p = t
else:
print 'WARNING: got zero length result'
def testString(self):
"""Test when elementArgs is a string"""
s = "Hello, world"
act = self.s.eachElem(ord, s)
exp = map(ord, s)
self.assertEquals(act, exp)
def testListOfString(self):
"""Test when elementArgs is a list containing a string"""
s = "This is another string."
act = self.s.eachElem(ord, [s])
exp = map(ord, s)
self.assertEquals(act, exp)
def testIterables(self):
"""Test when elementArgs is a list of unequal length iterables"""
act = self.s.eachElem(cmp, [xrange(10), xrange(20)])
exp = [0] * 10
self.assertEquals(act, exp)
def testDefaultElementArgs(self):
"""Test the default value of elementArgs"""
act = self.s.eachElem(cmp)
exp = []
self.assertEquals(act, exp)
def testElementArgsIFind(self):
"""Test using ifindTry to specify elementArgs for eachElem"""
n = 10
for i in xrange(n):
self.s.userNws.store('i', i)
exp = map(lambda x: x + x, xrange(n))
for i in xrange(3):
act = self.s.eachElem(lambda x: x + x, self.s.userNws.ifindTry('i'))
self.assertEqual(act, exp)
def testElementArgsStoreFile(self):
"""Test the use of storeFile with an elementArgs ifindTry"""
f = open("test_sleigh2.py", "rb")
try:
# write values of up to 100 bytes until EOF
while self.s.userNws.storeFile('test_sleigh2.py', f, n=100):
pass
finally:
f.close()
exp = string.upper(open("test_sleigh2.py", "rb").read())
for i in xrange(3):
act = "".join(self.s.eachElem(string.upper, self.s.userNws.ifindTry('test_sleigh2.py')))
self.assertEqual(act, exp)
def testAccumulator(self):
"""Test the use of the accumulator argument to eachElem"""
exp = string.uppercase
# do it the standard way
act = "".join(self.s.eachElem(string.upper, string.lowercase))
self.assertEqual(act, exp)
# do it with an accumulator
accum = Accum_1()
self.s.eachElem(string.upper, string.lowercase, accumulator=accum.setresults)
act = "".join(accum.getresults())
self.assertEqual(act, exp)
def testEachWorkerAccum(self):
"""Test the use of the accumulator argument to eachWorker"""
n = 42
exp = [n] * self.s.workerCount
accum = Accum_1()
self.s.eachWorker(nothing, n, accumulator=accum.setresults)
act = accum.getresults()
self.assertEqual(act, exp)
def testNewStyleAccum(self):
"""Test the new-style accumulators"""
exp = sum(xrange(300))
accum = Accum_2()
act = self.s.eachElem(sum,
[[range(100), range(100, 200), range(200, 300)]],
accumulator=accum)
self.assertEqual(act, exp)
def testImap(self):
"""Test imap method"""
self.assertEqual(list(self.s.imap(sqrt, xrange(10))), map(sqrt, xrange(10)))
self.assertEqual(list(self.s.imap(mul, xrange(10), repeat(6))),
list(imap(mul, xrange(10), repeat(6))))
axpy = lambda a, x, y: a * x + y
self.assertEqual(list(self.s.imap(axpy, repeat(10), xrange(100), cycle(xrange(10)))),
list(imap(axpy, repeat(10), xrange(100), cycle(xrange(10)))))
def testStarmap(self):
"""Test starmap method"""
def gen(a, x, y):
x = iter(x)
y = iter(y)
try:
while True:
yield (a, x.next(), y.next())
except StopIteration:
pass
axpy = lambda a, x, y: a * x + y
it = gen(10, xrange(100), cycle(xrange(10)))
self.assertEqual(list(self.s.starmap(axpy, it)),
list(imap(axpy, repeat(10), xrange(100), cycle(xrange(10)))))
else:
raise 'internal error: out-of-sync with getopt'
if len(args) == 0:
print >> sys.stderr, 'no workers specified'
sys.exit(1)
s = Sleigh(launch=sshcmd, nodeList=args, nwsHost=host, nwsPort=port, verbose=verbose)
while 1:
print '>',
cmd = raw_input()
cmd = cmd.strip()
if cmd:
try:
results = s.eachWorker(cmd)
for i in results:
if i:
print i
except SyntaxError, e:
print >> sys.stderr, str(e)
except getopt.GetoptError, e:
print >> sys.stderr, e.msg
sys.exit(1)
except ValueError, e:
print >> sys.stderr, "option %s requires an integer argument" % opt
sys.exit(1)
except (KeyboardInterrupt, SystemExit, EOFError):
print
except:
elif skw.get('launch') == 'service':
skw['nodeList'] = defaultNodeList
print "warning: using default nodeList:", defaultNodeList
# create a Sleigh
s = Sleigh(**skw)
numworkers, complete = s.status(True, timeout)
if numworkers == 0:
print >> sys.stderr, "no workers successfully started " \
"within %d seconds" % timeout
sys.exit(1)
# tell the workers to execute the ring function defined in this file
s.eachWorker(pong, numtasks, size, blocking=False)
totaltasks = numworkers * numtasks
start = time.time()
ping(s.userNws, totaltasks)
totaltime = time.time() - start
print "Seconds per operation:", totaltime / (4 * totaltasks)
print "Payload size is approximately %d bytes" % size
except getopt.GetoptError, e:
print >> sys.stderr, e.msg
sys.exit(1)
except ValueError, e:
print >> sys.stderr, "option %s requires an integer argument" % opt
sys.exit(1)
except (KeyboardInterrupt, SystemExit):
