def test(verbose=0):
s = Sleigh(verbose=verbose, **slargs)
r = s.eachElem(exp, range(100))
q = [exp(i) for i in range(100)]
s.stop()
if r == q:
sys.stderr.write('.')
sys.stderr.flush()
return 0
else:
sys.stderr.write('E')
sys.stderr.flush()
return 1
else:
raise 'internal error: out-of-sync with getopt'
if not skw.get('launch'):
if len(args) > 0:
skw['launch'] = sshcmd
skw['nodeList'] = args
if skw.has_key('workerCount'):
print >> sys.stderr, 'warning: ignoring -c option since workers are specified'
else:
if len(args) > 0:
print >> sys.stderr, \
'warning: ignoring unused arguments:', " ".join(args)
print >> sys.stderr, 'starting sleigh in web launch mode'
s = Sleigh(**skw)
s.eachWorker(worker, blocking=False)
for i in range(numtasks):
s.userNws.store('task', i)
for i in range(numtasks):
result = s.userNws.fetch('result')
print result[0], 'times 2 is', result[1]
except getopt.GetoptError, e:
print >> sys.stderr, e.msg
sys.exit(1)
except ValueError, e:
print >> sys.stderr, "option %s requires an integer argument" % opt
elif opt == '-c':
skw['workerCount'] = int(arg)
elif opt == '-v':
skw['verbose'] = 1
else:
raise 'internal error: out-of-sync with getopt'
if len(args) > 0:
skw['launch'] = sshcmd
skw['nodeList'] = args
if skw.has_key('workerCount'):
print >> sys.stderr, 'warning: ignoring -c option since workers are specified'
# create a Sleigh, and compute the number of workers.
# this is necessary because of web launch.
s = Sleigh(**skw)
s.userNws.declare('store', 'single')
print "Running with %d workers and %d tasks" % \
(s.workerCount, numtasks)
# tell the workers to execute the store function defined in this file
s.eachWorker(store, numtasks)
print "Finished"
except getopt.GetoptError, e:
print >> sys.stderr, e.msg
sys.exit(1)
except ValueError, e:
print >> sys.stderr, "option %s requires an integer argument" % opt
sys.exit(1)
if not skw.get('launch'):
if len(args) > 0:
skw['launch'] = sshcmd
skw['nodeList'] = args
if skw.has_key('workerCount'):
print >> sys.stderr, 'warning: ignoring -c option since workers are specified'
else:
if len(args) > 0:
print >> sys.stderr, \
'warning: ignoring unused arguments:', " ".join(args)
print >> sys.stderr, 'starting sleigh in web launch mode'
# create a Sleigh, and compute the number of workers.
# this is necessary because of web launch.
s = Sleigh(**skw)
numworkers, complete = s.status(True, timeout)
# include the master as one of the workers (which works even
# when no workers join)
numworkers += 1
# tell the workers to execute the ring function defined in this file
s.eachWorker(ring, numworkers, numtasks, blocking=False)
# the master becomes the last worker
global SleighRank, SleighUserNws
SleighRank = numworkers - 1
SleighUserNws = s.userNws
print "Master assigned rank %d" % SleighRank
#!/usr/bin/env python
import os
from nws.sleigh import Sleigh, sshcmd
from socket import getfqdn
slargs = {}
try: slargs['nwsHost'] = os.environ['NWS_HOST']
except: pass
try: slargs['nwsPort'] = int(os.environ['NWS_PORT'])
except: pass
try:
slargs['nodeList'] = os.environ['NWS_NODES'].split()
slargs['launch'] = sshcmd
print "using ssh to launch workers on machines:", slargs['nodeList']
except:
print "using default local workers"
s = Sleigh(**slargs)
# use os.getloadavg if they have it (it's not in Python 2.2 or on Windows)
# because socket.getfqdn runs terribly slowly on Mac OS X
if hasattr(os, 'getloadavg'):
for i in xrange(1000): print i, s.eachWorker(os.getloadavg)
else:
for i in xrange(1000): print i, s.eachWorker(getfqdn)
s.stop()
from nws.sleigh import Sleigh
from math import sqrt
# start the local nws server
srv = NwsLocalServer(daemon=False)
srv.wait_until_started()
# try an nws example
ws = NetWorkSpace('foo', serverPort=srv.port)
print ws
for i in range(10):
ws.store('x', 'Value %d' % i)
print list(ws.ifetchTry('x'))
# try a sleigh example
s = Sleigh(nwsPort=srv.port)
print s
for i in xrange(10):
print s.eachElem(sqrt, range(10))
# shutdown the local nws server
print 'shutting it down'
srv.shutdown()
assert not srv.isAlive()
try:
ws.store('y', 1)
print 'did not get expected exception'
except:
print 'got expected exception'
else:
raise 'internal error: out-of-sync with getopt'
if not fname:
print >> sys.stderr, 'error: must use -f option to specify a file'
sys.exit(1)
if len(args) > 0:
skw['launch'] = sshcmd
skw['nodeList'] = args
if skw.has_key('workerCount'):
print >> sys.stderr, 'warning: ignoring -c option since workers are specified'
# create a Sleigh, and compute the number of workers.
# this is necessary because of web launch.
s = Sleigh(**skw)
fobj = open(fname, 'rb')
print "Storing %s to server..." % fname
s.userNws.declare('longvalue', 'single')
s.userNws.storeFile('longvalue', fobj)
fobj.close()
print "Running with %d workers and %d tasks" % \
(s.workerCount, numtasks)
# tell the workers to execute the longvalue function
# defined in this file
print s.eachWorker(longvalue, numtasks, fname)
raw_input('Hit return to finish ')
port = int(arg)
elif opt == '-v':
verbose = 1
else:
raise 'internal error: out-of-sync with getopt'
if not host:
print >> sys.stderr, 'the -h option is required'
sys.exit(1)
if len(args) > 0:
print >> sys.stderr, 'ignoring unused arguments:', ' '.join(args)
nodeList = getNodeList()
print "Executing on nodes:", " ".join(nodeList)
s = Sleigh(launch=sshcmd, nodeList=nodeList, nwsHost=host, nwsPort=port, verbose=verbose)
# here's a personal favorite of mine...
from math import exp
r = s.eachElem(exp, range(100))
print "The results are:", r
except getopt.GetoptError, e:
print >> sys.stderr, e.msg
sys.exit(1)
except ValueError, e:
print >> sys.stderr, "option %s requires an integer argument" % opt
sys.exit(1)
except SystemExit, e:
sys.exit(e.code)
except:
scriptDir = arg
elif opt == '-v':
verbose = 1
elif opt == '-w':
launch = 'web'
else:
print >> sys.stderr, 'error: getopt should have prevented this'
sys.exit(1)
print "scriptDir =", scriptDir
print "verbose =", verbose
print "launch =", launch
if launch == 'web':
print "waiting for web launch workers..."
s = Sleigh(scriptDir=scriptDir, launch=launch, verbose=verbose)
t = map(math.exp, range(100))
print "testing a simple list:"
r = s.eachElem(math.exp, range(100))
if r != t:
print "error:", r
else:
print "success"
print "testing list of one list:"
r = s.eachElem(math.exp, [range(100)])
if r != t:
print "error:", r
else:
def setUp(self):
self.log.info('Setting up')
self.s = Sleigh(**slargs)
class SleighTest(unittest.TestCase):
log = getLogger('SleighTest')
wsname = 'unit test ws'
def setUp(self):
self.log.info('Setting up')
self.s = Sleigh(**slargs)
def tearDown(self):
self.log.info('Tearing down')
try: self.s.stop()
except: pass
def testMonitor(self):
"""Test one-to-all broadcast with a single mode variable"""
for n, s in [(1000, 0.0), (100, 0.1), (10, 1.0)]:
pending = self.s.eachWorker(monitorWorker, blocking=False)
time.sleep(3) # not necessary for test to pass
self.s.nws.declare('monitor', SINGLE)
for i in xrange(n):
self.s.nws.store('monitor', i)
time.sleep(s) # not necessary for test to pass
time.sleep(3) # not necessary for test to pass
self.s.nws.deleteVar('monitor')
results = pending.wait()
self.assertEquals(len(results), self.s.workerCount)
for r in results:
self.assert_(len(r) <= n)
if len(r) > 0:
p = r.pop(0)
for t in r:
self.assert_(t > p)
p = t
else:
print 'WARNING: got zero length result'
def testString(self):
"""Test when elementArgs is a string"""
s = "Hello, world"
act = self.s.eachElem(ord, s)
exp = map(ord, s)
self.assertEquals(act, exp)
def testListOfString(self):
"""Test when elementArgs is a list containing a string"""
s = "This is another string."
act = self.s.eachElem(ord, [s])
exp = map(ord, s)
self.assertEquals(act, exp)
def testIterables(self):
"""Test when elementArgs is a list of unequal length iterables"""
act = self.s.eachElem(cmp, [xrange(10), xrange(20)])
exp = [0] * 10
self.assertEquals(act, exp)
def testDefaultElementArgs(self):
"""Test the default value of elementArgs"""
act = self.s.eachElem(cmp)
exp = []
self.assertEquals(act, exp)
def testElementArgsIFind(self):
"""Test using ifindTry to specify elementArgs for eachElem"""
n = 10
for i in xrange(n):
self.s.userNws.store('i', i)
exp = map(lambda x: x + x, xrange(n))
for i in xrange(3):
act = self.s.eachElem(lambda x: x + x, self.s.userNws.ifindTry('i'))
self.assertEqual(act, exp)
def testElementArgsStoreFile(self):
"""Test the use of storeFile with an elementArgs ifindTry"""
f = open("test_sleigh2.py", "rb")
try:
# write values of up to 100 bytes until EOF
while self.s.userNws.storeFile('test_sleigh2.py', f, n=100):
pass
finally:
f.close()
exp = string.upper(open("test_sleigh2.py", "rb").read())
for i in xrange(3):
act = "".join(self.s.eachElem(string.upper, self.s.userNws.ifindTry('test_sleigh2.py')))
self.assertEqual(act, exp)
def testAccumulator(self):
"""Test the use of the accumulator argument to eachElem"""
exp = string.uppercase
# do it the standard way
act = "".join(self.s.eachElem(string.upper, string.lowercase))
self.assertEqual(act, exp)
# do it with an accumulator
accum = Accum_1()
self.s.eachElem(string.upper, string.lowercase, accumulator=accum.setresults)
act = "".join(accum.getresults())
self.assertEqual(act, exp)
def testEachWorkerAccum(self):
"""Test the use of the accumulator argument to eachWorker"""
n = 42
exp = [n] * self.s.workerCount
accum = Accum_1()
self.s.eachWorker(nothing, n, accumulator=accum.setresults)
act = accum.getresults()
self.assertEqual(act, exp)
def testNewStyleAccum(self):
"""Test the new-style accumulators"""
exp = sum(xrange(300))
accum = Accum_2()
act = self.s.eachElem(sum,
[[range(100), range(100, 200), range(200, 300)]],
accumulator=accum)
self.assertEqual(act, exp)
def testImap(self):
"""Test imap method"""
self.assertEqual(list(self.s.imap(sqrt, xrange(10))), map(sqrt, xrange(10)))
self.assertEqual(list(self.s.imap(mul, xrange(10), repeat(6))),
list(imap(mul, xrange(10), repeat(6))))
axpy = lambda a, x, y: a * x + y
self.assertEqual(list(self.s.imap(axpy, repeat(10), xrange(100), cycle(xrange(10)))),
list(imap(axpy, repeat(10), xrange(100), cycle(xrange(10)))))
def testStarmap(self):
"""Test starmap method"""
def gen(a, x, y):
x = iter(x)
y = iter(y)
try:
while True:
yield (a, x.next(), y.next())
except StopIteration:
pass
axpy = lambda a, x, y: a * x + y
it = gen(10, xrange(100), cycle(xrange(10)))
self.assertEqual(list(self.s.starmap(axpy, it)),
list(imap(axpy, repeat(10), xrange(100), cycle(xrange(10)))))
for opt, arg in opts:
if opt == '-h':
host = arg
elif opt == '-p':
port = int(arg)
elif opt == '-v':
verbose = 1
else:
raise 'internal error: out-of-sync with getopt'
if len(args) == 0:
print >> sys.stderr, 'no workers specified'
sys.exit(1)
s = Sleigh(launch=sshcmd, nodeList=args, nwsHost=host, nwsPort=port, verbose=verbose)
while 1:
print '>',
cmd = raw_input()
cmd = cmd.strip()
if cmd:
try:
results = s.eachWorker(cmd)
for i in results:
if i:
print i
except SyntaxError, e:
print >> sys.stderr, str(e)
except getopt.GetoptError, e:
elif opt == '-e':
enableNwsUtility = True
elif opt == '-c':
skw['workerCount'] = int(arg)
else:
raise 'internal error: out-of-sync with getopt'
# if no nodes are specified, start three on the local machine
if len(args) > 0:
skw['nodeList'] = args
elif skw.get('launch') == 'service':
skw['nodeList'] = defaultNodeList
print "warning: using default nodeList:", defaultNodeList
# create a sleigh on the specified nodes
s = Sleigh(**skw)
if enableNwsUtility:
s.nws.declare('enableNwsUtility', SINGLE)
s.nws.declare('MainTitle', SINGLE)
s.nws.store('MainTitle', description)
s.nws.declare('SubTitle', SINGLE)
submitted = s.workerCount
# do the work
for i in xrange(numtasks):
submitted += numtasks
s.nws.store('SubTitle', '%d tasks submitted' % submitted)
s.eachElem(worker, range(numtasks), **eo)