if not skw.get('launch'): if len(args) > 0: skw['launch'] = sshcmd skw['nodeList'] = args if skw.has_key('workerCount'): print >> sys.stderr, 'warning: ignoring -c option since workers are specified' else: if len(args) > 0: print >> sys.stderr, \ 'warning: ignoring unused arguments:', " ".join(args) print >> sys.stderr, 'starting sleigh in web launch mode' s = Sleigh(**skw) s.eachWorker(worker, blocking=False) for i in range(numtasks): s.userNws.store('task', i) for i in range(numtasks): result = s.userNws.fetch('result') print result[0], 'times 2 is', result[1] except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except (KeyboardInterrupt, SystemExit):
if len(args) > 0: print >> sys.stderr, \ 'warning: ignoring unused arguments:', " ".join(args) print >> sys.stderr, 'starting sleigh in web launch mode' # create a Sleigh, and compute the number of workers. # this is necessary because of web launch. s = Sleigh(**skw) numworkers, complete = s.status(True, timeout) # include the master as one of the workers (which works even # when no workers join) numworkers += 1 # tell the workers to execute the ring function defined in this file s.eachWorker(ring, numworkers, numtasks, blocking=False) # the master becomes the last worker global SleighRank, SleighUserNws SleighRank = numworkers - 1 SleighUserNws = s.userNws print "Master assigned rank %d" % SleighRank # time how long it takes the token to go all the way around the # ring numtask times. start = time.time() s.userNws.store('worker_0', 0) ring(numworkers, numtasks) finish = time.time() token = s.userNws.fetch('worker_0')
if len(args) > 0: skw['launch'] = sshcmd skw['nodeList'] = args if skw.has_key('workerCount'): print >> sys.stderr, 'warning: ignoring -c option since workers are specified' # create a Sleigh, and compute the number of workers. # this is necessary because of web launch. s = Sleigh(**skw) s.userNws.declare('store', 'single') print "Running with %d workers and %d tasks" % \ (s.workerCount, numtasks) # tell the workers to execute the store function defined in this file s.eachWorker(store, numtasks) print "Finished" except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except (KeyboardInterrupt, SystemExit): pass except: ex = sys.exc_info() print >> sys.stderr, ex[0], ex[1]
#!/usr/bin/env python import os from nws.sleigh import Sleigh, sshcmd from socket import getfqdn slargs = {} try: slargs['nwsHost'] = os.environ['NWS_HOST'] except: pass try: slargs['nwsPort'] = int(os.environ['NWS_PORT']) except: pass try: slargs['nodeList'] = os.environ['NWS_NODES'].split() slargs['launch'] = sshcmd print "using ssh to launch workers on machines:", slargs['nodeList'] except: print "using default local workers" s = Sleigh(**slargs) # use os.getloadavg if they have it (it's not in Python 2.2 or on Windows) # because socket.getfqdn runs terribly slowly on Mac OS X if hasattr(os, 'getloadavg'): for i in xrange(1000): print i, s.eachWorker(os.getloadavg) else: for i in xrange(1000): print i, s.eachWorker(getfqdn) s.stop()
# create a Sleigh, and compute the number of workers. # this is necessary because of web launch. s = Sleigh(**skw) fobj = open(fname, 'rb') print "Storing %s to server..." % fname s.userNws.declare('longvalue', 'single') s.userNws.storeFile('longvalue', fobj) fobj.close() print "Running with %d workers and %d tasks" % \ (s.workerCount, numtasks) # tell the workers to execute the longvalue function # defined in this file print s.eachWorker(longvalue, numtasks, fname) raw_input('Hit return to finish ') print "Finished" except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except (KeyboardInterrupt, SystemExit): pass except: ex = sys.exc_info() print >> sys.stderr, ex[0], ex[1]
class SleighTest(unittest.TestCase): log = getLogger('SleighTest') wsname = 'unit test ws' def setUp(self): self.log.info('Setting up') self.s = Sleigh(**slargs) def tearDown(self): self.log.info('Tearing down') try: self.s.stop() except: pass def testMonitor(self): """Test one-to-all broadcast with a single mode variable""" for n, s in [(1000, 0.0), (100, 0.1), (10, 1.0)]: pending = self.s.eachWorker(monitorWorker, blocking=False) time.sleep(3) # not necessary for test to pass self.s.nws.declare('monitor', SINGLE) for i in xrange(n): self.s.nws.store('monitor', i) time.sleep(s) # not necessary for test to pass time.sleep(3) # not necessary for test to pass self.s.nws.deleteVar('monitor') results = pending.wait() self.assertEquals(len(results), self.s.workerCount) for r in results: self.assert_(len(r) <= n) if len(r) > 0: p = r.pop(0) for t in r: self.assert_(t > p) p = t else: print 'WARNING: got zero length result' def testString(self): """Test when elementArgs is a string""" s = "Hello, world" act = self.s.eachElem(ord, s) exp = map(ord, s) self.assertEquals(act, exp) def testListOfString(self): """Test when elementArgs is a list containing a string""" s = "This is another string." act = self.s.eachElem(ord, [s]) exp = map(ord, s) self.assertEquals(act, exp) def testIterables(self): """Test when elementArgs is a list of unequal length iterables""" act = self.s.eachElem(cmp, [xrange(10), xrange(20)]) exp = [0] * 10 self.assertEquals(act, exp) def testDefaultElementArgs(self): """Test the default value of elementArgs""" act = self.s.eachElem(cmp) exp = [] self.assertEquals(act, exp) def testElementArgsIFind(self): """Test using ifindTry to specify elementArgs for eachElem""" n = 10 for i in xrange(n): self.s.userNws.store('i', i) exp = map(lambda x: x + x, xrange(n)) for i in xrange(3): act = self.s.eachElem(lambda x: x + x, self.s.userNws.ifindTry('i')) self.assertEqual(act, exp) def testElementArgsStoreFile(self): """Test the use of storeFile with an elementArgs ifindTry""" f = open("test_sleigh2.py", "rb") try: # write values of up to 100 bytes until EOF while self.s.userNws.storeFile('test_sleigh2.py', f, n=100): pass finally: f.close() exp = string.upper(open("test_sleigh2.py", "rb").read()) for i in xrange(3): act = "".join(self.s.eachElem(string.upper, self.s.userNws.ifindTry('test_sleigh2.py'))) self.assertEqual(act, exp) def testAccumulator(self): """Test the use of the accumulator argument to eachElem""" exp = string.uppercase # do it the standard way act = "".join(self.s.eachElem(string.upper, string.lowercase)) self.assertEqual(act, exp) # do it with an accumulator accum = Accum_1() self.s.eachElem(string.upper, string.lowercase, accumulator=accum.setresults) act = "".join(accum.getresults()) self.assertEqual(act, exp) def testEachWorkerAccum(self): """Test the use of the accumulator argument to eachWorker""" n = 42 exp = [n] * self.s.workerCount accum = Accum_1() self.s.eachWorker(nothing, n, accumulator=accum.setresults) act = accum.getresults() self.assertEqual(act, exp) def testNewStyleAccum(self): """Test the new-style accumulators""" exp = sum(xrange(300)) accum = Accum_2() act = self.s.eachElem(sum, [[range(100), range(100, 200), range(200, 300)]], accumulator=accum) self.assertEqual(act, exp) def testImap(self): """Test imap method""" self.assertEqual(list(self.s.imap(sqrt, xrange(10))), map(sqrt, xrange(10))) self.assertEqual(list(self.s.imap(mul, xrange(10), repeat(6))), list(imap(mul, xrange(10), repeat(6)))) axpy = lambda a, x, y: a * x + y self.assertEqual(list(self.s.imap(axpy, repeat(10), xrange(100), cycle(xrange(10)))), list(imap(axpy, repeat(10), xrange(100), cycle(xrange(10))))) def testStarmap(self): """Test starmap method""" def gen(a, x, y): x = iter(x) y = iter(y) try: while True: yield (a, x.next(), y.next()) except StopIteration: pass axpy = lambda a, x, y: a * x + y it = gen(10, xrange(100), cycle(xrange(10))) self.assertEqual(list(self.s.starmap(axpy, it)), list(imap(axpy, repeat(10), xrange(100), cycle(xrange(10)))))
else: raise 'internal error: out-of-sync with getopt' if len(args) == 0: print >> sys.stderr, 'no workers specified' sys.exit(1) s = Sleigh(launch=sshcmd, nodeList=args, nwsHost=host, nwsPort=port, verbose=verbose) while 1: print '>', cmd = raw_input() cmd = cmd.strip() if cmd: try: results = s.eachWorker(cmd) for i in results: if i: print i except SyntaxError, e: print >> sys.stderr, str(e) except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except (KeyboardInterrupt, SystemExit, EOFError): print except:
elif skw.get('launch') == 'service': skw['nodeList'] = defaultNodeList print "warning: using default nodeList:", defaultNodeList # create a Sleigh s = Sleigh(**skw) numworkers, complete = s.status(True, timeout) if numworkers == 0: print >> sys.stderr, "no workers successfully started " \ "within %d seconds" % timeout sys.exit(1) # tell the workers to execute the ring function defined in this file s.eachWorker(pong, numtasks, size, blocking=False) totaltasks = numworkers * numtasks start = time.time() ping(s.userNws, totaltasks) totaltime = time.time() - start print "Seconds per operation:", totaltime / (4 * totaltasks) print "Payload size is approximately %d bytes" % size except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except (KeyboardInterrupt, SystemExit):