def test(verbose=0): s = Sleigh(verbose=verbose, **slargs) r = s.eachElem(exp, range(100)) q = [exp(i) for i in range(100)] s.stop() if r == q: sys.stderr.write('.') sys.stderr.flush() return 0 else: sys.stderr.write('E') sys.stderr.flush() return 1
# start the local nws server srv = NwsLocalServer(daemon=False) srv.wait_until_started() # try an nws example ws = NetWorkSpace('foo', serverPort=srv.port) print ws for i in range(10): ws.store('x', 'Value %d' % i) print list(ws.ifetchTry('x')) # try a sleigh example s = Sleigh(nwsPort=srv.port) print s for i in xrange(10): print s.eachElem(sqrt, range(10)) # shutdown the local nws server print 'shutting it down' srv.shutdown() assert not srv.isAlive() try: ws.store('y', 1) print 'did not get expected exception' except: print 'got expected exception' # This hangs on shutdown if False:
else: raise 'internal error: out-of-sync with getopt' if not host: print >> sys.stderr, 'the -h option is required' sys.exit(1) if len(args) > 0: print >> sys.stderr, 'ignoring unused arguments:', ' '.join(args) nodeList = getNodeList() print "Executing on nodes:", " ".join(nodeList) s = Sleigh(launch=sshcmd, nodeList=nodeList, nwsHost=host, nwsPort=port, verbose=verbose) # here's a personal favorite of mine... from math import exp r = s.eachElem(exp, range(100)) print "The results are:", r except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except SystemExit, e: sys.exit(e.code) except: ex = sys.exc_info() print >> sys.stderr, ex[0], ex[1]
else: print >> sys.stderr, 'error: getopt should have prevented this' sys.exit(1) print "scriptDir =", scriptDir print "verbose =", verbose print "launch =", launch if launch == 'web': print "waiting for web launch workers..." s = Sleigh(scriptDir=scriptDir, launch=launch, verbose=verbose) t = map(math.exp, range(100)) print "testing a simple list:" r = s.eachElem(math.exp, range(100)) if r != t: print "error:", r else: print "success" print "testing list of one list:" r = s.eachElem(math.exp, [range(100)]) if r != t: print "error:", r else: print "success" print "testing a list of two lists:" r = s.eachElem(operator.add, [range(100), range(100)]) if r != range(0, 200, 2):
class SleighTest(unittest.TestCase): log = getLogger('SleighTest') wsname = 'unit test ws' def setUp(self): self.log.info('Setting up') self.s = Sleigh(**slargs) def tearDown(self): self.log.info('Tearing down') try: self.s.stop() except: pass def testMonitor(self): """Test one-to-all broadcast with a single mode variable""" for n, s in [(1000, 0.0), (100, 0.1), (10, 1.0)]: pending = self.s.eachWorker(monitorWorker, blocking=False) time.sleep(3) # not necessary for test to pass self.s.nws.declare('monitor', SINGLE) for i in xrange(n): self.s.nws.store('monitor', i) time.sleep(s) # not necessary for test to pass time.sleep(3) # not necessary for test to pass self.s.nws.deleteVar('monitor') results = pending.wait() self.assertEquals(len(results), self.s.workerCount) for r in results: self.assert_(len(r) <= n) if len(r) > 0: p = r.pop(0) for t in r: self.assert_(t > p) p = t else: print 'WARNING: got zero length result' def testString(self): """Test when elementArgs is a string""" s = "Hello, world" act = self.s.eachElem(ord, s) exp = map(ord, s) self.assertEquals(act, exp) def testListOfString(self): """Test when elementArgs is a list containing a string""" s = "This is another string." act = self.s.eachElem(ord, [s]) exp = map(ord, s) self.assertEquals(act, exp) def testIterables(self): """Test when elementArgs is a list of unequal length iterables""" act = self.s.eachElem(cmp, [xrange(10), xrange(20)]) exp = [0] * 10 self.assertEquals(act, exp) def testDefaultElementArgs(self): """Test the default value of elementArgs""" act = self.s.eachElem(cmp) exp = [] self.assertEquals(act, exp) def testElementArgsIFind(self): """Test using ifindTry to specify elementArgs for eachElem""" n = 10 for i in xrange(n): self.s.userNws.store('i', i) exp = map(lambda x: x + x, xrange(n)) for i in xrange(3): act = self.s.eachElem(lambda x: x + x, self.s.userNws.ifindTry('i')) self.assertEqual(act, exp) def testElementArgsStoreFile(self): """Test the use of storeFile with an elementArgs ifindTry""" f = open("test_sleigh2.py", "rb") try: # write values of up to 100 bytes until EOF while self.s.userNws.storeFile('test_sleigh2.py', f, n=100): pass finally: f.close() exp = string.upper(open("test_sleigh2.py", "rb").read()) for i in xrange(3): act = "".join(self.s.eachElem(string.upper, self.s.userNws.ifindTry('test_sleigh2.py'))) self.assertEqual(act, exp) def testAccumulator(self): """Test the use of the accumulator argument to eachElem""" exp = string.uppercase # do it the standard way act = "".join(self.s.eachElem(string.upper, string.lowercase)) self.assertEqual(act, exp) # do it with an accumulator accum = Accum_1() self.s.eachElem(string.upper, string.lowercase, accumulator=accum.setresults) act = "".join(accum.getresults()) self.assertEqual(act, exp) def testEachWorkerAccum(self): """Test the use of the accumulator argument to eachWorker""" n = 42 exp = [n] * self.s.workerCount accum = Accum_1() self.s.eachWorker(nothing, n, accumulator=accum.setresults) act = accum.getresults() self.assertEqual(act, exp) def testNewStyleAccum(self): """Test the new-style accumulators""" exp = sum(xrange(300)) accum = Accum_2() act = self.s.eachElem(sum, [[range(100), range(100, 200), range(200, 300)]], accumulator=accum) self.assertEqual(act, exp) def testImap(self): """Test imap method""" self.assertEqual(list(self.s.imap(sqrt, xrange(10))), map(sqrt, xrange(10))) self.assertEqual(list(self.s.imap(mul, xrange(10), repeat(6))), list(imap(mul, xrange(10), repeat(6)))) axpy = lambda a, x, y: a * x + y self.assertEqual(list(self.s.imap(axpy, repeat(10), xrange(100), cycle(xrange(10)))), list(imap(axpy, repeat(10), xrange(100), cycle(xrange(10))))) def testStarmap(self): """Test starmap method""" def gen(a, x, y): x = iter(x) y = iter(y) try: while True: yield (a, x.next(), y.next()) except StopIteration: pass axpy = lambda a, x, y: a * x + y it = gen(10, xrange(100), cycle(xrange(10))) self.assertEqual(list(self.s.starmap(axpy, it)), list(imap(axpy, repeat(10), xrange(100), cycle(xrange(10)))))
# create a sleigh on the specified nodes s = Sleigh(**skw) if enableNwsUtility: s.nws.declare('enableNwsUtility', SINGLE) s.nws.declare('MainTitle', SINGLE) s.nws.store('MainTitle', description) s.nws.declare('SubTitle', SINGLE) submitted = s.workerCount # do the work for i in xrange(numtasks): submitted += numtasks s.nws.store('SubTitle', '%d tasks submitted' % submitted) s.eachElem(worker, range(numtasks), **eo) raw_input('Hit return to exit sleigh: ') s.stop() except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except (KeyboardInterrupt, SystemExit): pass except: ex = sys.exc_info() print >> sys.stderr, ex[0], ex[1]