Beispiel #1
0
def test(verbose=0):
    s = Sleigh(verbose=verbose, **slargs)
    r = s.eachElem(exp, range(100))
    q = [exp(i) for i in range(100)]
    s.stop()
    if r == q:
        sys.stderr.write('.')
        sys.stderr.flush()
        return 0
    else:
        sys.stderr.write('E')
        sys.stderr.flush()
        return 1
Beispiel #2
0
            else:
                raise 'internal error: out-of-sync with getopt'

        if not skw.get('launch'):
            if len(args) > 0:
                skw['launch'] = sshcmd
                skw['nodeList'] = args
                if skw.has_key('workerCount'):
                    print >> sys.stderr, 'warning: ignoring -c option since workers are specified'
        else:
            if len(args) > 0:
                print >> sys.stderr, \
                        'warning: ignoring unused arguments:', " ".join(args)
            print >> sys.stderr, 'starting sleigh in web launch mode'

        s = Sleigh(**skw)

        s.eachWorker(worker, blocking=False)

        for i in range(numtasks):
            s.userNws.store('task', i)

        for i in range(numtasks):
            result = s.userNws.fetch('result')
            print result[0], 'times 2 is', result[1]

    except getopt.GetoptError, e:
        print >> sys.stderr, e.msg
        sys.exit(1)
    except ValueError, e:
        print >> sys.stderr, "option %s requires an integer argument" % opt
Beispiel #3
0
            elif opt == '-c':
                skw['workerCount'] = int(arg)
            elif opt == '-v':
                skw['verbose'] = 1
            else:
                raise 'internal error: out-of-sync with getopt'

        if len(args) > 0:
            skw['launch'] = sshcmd
            skw['nodeList'] = args
            if skw.has_key('workerCount'):
                print >> sys.stderr, 'warning: ignoring -c option since workers are specified'

        # create a Sleigh, and compute the number of workers.
        # this is necessary because of web launch.
        s = Sleigh(**skw)
	s.userNws.declare('store', 'single')

	print "Running with %d workers and %d tasks" % \
	        (s.workerCount, numtasks)

        # tell the workers to execute the store function defined in this file
        s.eachWorker(store, numtasks)

        print "Finished"
    except getopt.GetoptError, e:
        print >> sys.stderr, e.msg
        sys.exit(1)
    except ValueError, e:
        print >> sys.stderr, "option %s requires an integer argument" % opt
        sys.exit(1)
Beispiel #4
0
        if not skw.get('launch'):
            if len(args) > 0:
                skw['launch'] = sshcmd
                skw['nodeList'] = args
                if skw.has_key('workerCount'):
                    print >> sys.stderr, 'warning: ignoring -c option since workers are specified'
        else:
            if len(args) > 0:
                print >> sys.stderr, \
                        'warning: ignoring unused arguments:', " ".join(args)
            print >> sys.stderr, 'starting sleigh in web launch mode'

        # create a Sleigh, and compute the number of workers.
        # this is necessary because of web launch.
        s = Sleigh(**skw)
        numworkers, complete = s.status(True, timeout)

        # include the master as one of the workers (which works even
        # when no workers join)
        numworkers += 1

        # tell the workers to execute the ring function defined in this file
        s.eachWorker(ring, numworkers, numtasks, blocking=False)

        # the master becomes the last worker
        global SleighRank, SleighUserNws
        SleighRank = numworkers - 1
        SleighUserNws = s.userNws

        print "Master assigned rank %d" % SleighRank
#!/usr/bin/env python

import os
from nws.sleigh import Sleigh, sshcmd
from socket import getfqdn

slargs = {}
try: slargs['nwsHost'] = os.environ['NWS_HOST']
except: pass
try: slargs['nwsPort'] = int(os.environ['NWS_PORT'])
except: pass

try:
    slargs['nodeList'] = os.environ['NWS_NODES'].split()
    slargs['launch'] = sshcmd
    print "using ssh to launch workers on machines:", slargs['nodeList']
except:
    print "using default local workers"

s = Sleigh(**slargs)

# use os.getloadavg if they have it (it's not in Python 2.2 or on Windows)
# because socket.getfqdn runs terribly slowly on Mac OS X
if hasattr(os, 'getloadavg'):
    for i in xrange(1000): print i, s.eachWorker(os.getloadavg)
else:
    for i in xrange(1000): print i, s.eachWorker(getfqdn)

s.stop()
Beispiel #6
0
from nws.sleigh import Sleigh
from math import sqrt

# start the local nws server
srv = NwsLocalServer(daemon=False)
srv.wait_until_started()

# try an nws example
ws = NetWorkSpace('foo', serverPort=srv.port)
print ws
for i in range(10):
    ws.store('x', 'Value %d' % i)
print list(ws.ifetchTry('x'))

# try a sleigh example
s = Sleigh(nwsPort=srv.port)
print s
for i in xrange(10):
    print s.eachElem(sqrt, range(10))

# shutdown the local nws server
print 'shutting it down'
srv.shutdown()

assert not srv.isAlive()

try:
    ws.store('y', 1)
    print 'did not get expected exception'
except:
    print 'got expected exception'
Beispiel #7
0
            else:
                raise 'internal error: out-of-sync with getopt'

        if not fname:
            print >> sys.stderr, 'error: must use -f option to specify a file'
            sys.exit(1)

        if len(args) > 0:
            skw['launch'] = sshcmd
            skw['nodeList'] = args
            if skw.has_key('workerCount'):
                print >> sys.stderr, 'warning: ignoring -c option since workers are specified'

        # create a Sleigh, and compute the number of workers.
        # this is necessary because of web launch.
        s = Sleigh(**skw)

        fobj = open(fname, 'rb')
        print "Storing %s to server..." % fname
        s.userNws.declare('longvalue', 'single')
        s.userNws.storeFile('longvalue', fobj)
        fobj.close()

        print "Running with %d workers and %d tasks" % \
                (s.workerCount, numtasks)

        # tell the workers to execute the longvalue function
        # defined in this file
        print s.eachWorker(longvalue, numtasks, fname)

        raw_input('Hit return to finish ')
                port = int(arg)
            elif opt == '-v':
                verbose = 1
            else:
                raise 'internal error: out-of-sync with getopt'

        if not host:
            print >> sys.stderr, 'the -h option is required'
            sys.exit(1)

        if len(args) > 0:
            print >> sys.stderr, 'ignoring unused arguments:', ' '.join(args)

        nodeList = getNodeList()
        print "Executing on nodes:", " ".join(nodeList)
        s = Sleigh(launch=sshcmd, nodeList=nodeList, nwsHost=host, nwsPort=port, verbose=verbose)

        # here's a personal favorite of mine...
        from math import exp
        r = s.eachElem(exp, range(100))
        print "The results are:", r

    except getopt.GetoptError, e:
        print >> sys.stderr, e.msg
        sys.exit(1)
    except ValueError, e:
        print >> sys.stderr, "option %s requires an integer argument" % opt
        sys.exit(1)
    except SystemExit, e:
        sys.exit(e.code)
    except:
        scriptDir = arg
    elif opt == '-v':
        verbose = 1
    elif opt == '-w':
        launch = 'web'
    else:
        print >> sys.stderr, 'error: getopt should have prevented this'
        sys.exit(1)

print "scriptDir =", scriptDir
print "verbose =", verbose
print "launch =", launch
if launch == 'web':
    print "waiting for web launch workers..."

s = Sleigh(scriptDir=scriptDir, launch=launch, verbose=verbose)

t = map(math.exp, range(100))

print "testing a simple list:"
r = s.eachElem(math.exp, range(100))
if r != t:
    print "error:", r
else:
    print "success"

print "testing list of one list:"
r = s.eachElem(math.exp, [range(100)])
if r != t:
    print "error:", r
else:
Beispiel #10
0
 def setUp(self):
     self.log.info('Setting up')
     self.s = Sleigh(**slargs)
Beispiel #11
0
class SleighTest(unittest.TestCase):
    log = getLogger('SleighTest')
    wsname = 'unit test ws'

    def setUp(self):
        self.log.info('Setting up')
        self.s = Sleigh(**slargs)

    def tearDown(self):
        self.log.info('Tearing down')
        try: self.s.stop()
        except: pass

    def testMonitor(self):
        """Test one-to-all broadcast with a single mode variable"""
        for n, s in [(1000, 0.0), (100, 0.1), (10, 1.0)]:
            pending = self.s.eachWorker(monitorWorker, blocking=False)
            time.sleep(3) # not necessary for test to pass
            self.s.nws.declare('monitor', SINGLE)
            for i in xrange(n):
                self.s.nws.store('monitor', i)
                time.sleep(s) # not necessary for test to pass
            time.sleep(3) # not necessary for test to pass
            self.s.nws.deleteVar('monitor')
            results = pending.wait()
            self.assertEquals(len(results), self.s.workerCount)
            for r in results:
                self.assert_(len(r) <= n)
                if len(r) > 0:
                    p = r.pop(0)
                    for t in r:
                        self.assert_(t > p)
                        p = t
                else:
                    print 'WARNING: got zero length result'

    def testString(self):
        """Test when elementArgs is a string"""
        s = "Hello, world"
        act = self.s.eachElem(ord, s)
        exp = map(ord, s)
        self.assertEquals(act, exp)

    def testListOfString(self):
        """Test when elementArgs is a list containing a string"""
        s = "This is another string."
        act = self.s.eachElem(ord, [s])
        exp = map(ord, s)
        self.assertEquals(act, exp)

    def testIterables(self):
        """Test when elementArgs is a list of unequal length iterables"""
        act = self.s.eachElem(cmp, [xrange(10), xrange(20)])
        exp = [0] * 10
        self.assertEquals(act, exp)

    def testDefaultElementArgs(self):
        """Test the default value of elementArgs"""
        act = self.s.eachElem(cmp)
        exp = []
        self.assertEquals(act, exp)

    def testElementArgsIFind(self):
        """Test using ifindTry to specify elementArgs for eachElem"""
        n = 10
        for i in xrange(n):
            self.s.userNws.store('i', i)

        exp = map(lambda x: x + x, xrange(n))
        for i in xrange(3):
            act = self.s.eachElem(lambda x: x + x, self.s.userNws.ifindTry('i'))
            self.assertEqual(act, exp)

    def testElementArgsStoreFile(self):
        """Test the use of storeFile with an elementArgs ifindTry"""
        f = open("test_sleigh2.py", "rb")
        try:
            # write values of up to 100 bytes until EOF
            while self.s.userNws.storeFile('test_sleigh2.py', f, n=100):
                pass
        finally:
            f.close()

        exp = string.upper(open("test_sleigh2.py", "rb").read())
        for i in xrange(3):
            act = "".join(self.s.eachElem(string.upper, self.s.userNws.ifindTry('test_sleigh2.py')))
            self.assertEqual(act, exp)

    def testAccumulator(self):
        """Test the use of the accumulator argument to eachElem"""
        exp = string.uppercase

        # do it the standard way
        act = "".join(self.s.eachElem(string.upper, string.lowercase))
        self.assertEqual(act, exp)

        # do it with an accumulator
        accum = Accum_1()
        self.s.eachElem(string.upper, string.lowercase, accumulator=accum.setresults)
        act = "".join(accum.getresults())
        self.assertEqual(act, exp)

    def testEachWorkerAccum(self):
        """Test the use of the accumulator argument to eachWorker"""
        n = 42
        exp = [n] * self.s.workerCount
        accum = Accum_1()
        self.s.eachWorker(nothing, n, accumulator=accum.setresults)
        act = accum.getresults()
        self.assertEqual(act, exp)

    def testNewStyleAccum(self):
        """Test the new-style accumulators"""
        exp = sum(xrange(300))
        accum = Accum_2()
        act = self.s.eachElem(sum,
                [[range(100), range(100, 200), range(200, 300)]],
                accumulator=accum)
        self.assertEqual(act, exp)

    def testImap(self):
        """Test imap method"""
        self.assertEqual(list(self.s.imap(sqrt, xrange(10))), map(sqrt, xrange(10)))
        self.assertEqual(list(self.s.imap(mul, xrange(10), repeat(6))),
                         list(imap(mul, xrange(10), repeat(6))))
        axpy = lambda a, x, y: a * x + y
        self.assertEqual(list(self.s.imap(axpy, repeat(10), xrange(100), cycle(xrange(10)))),
                         list(imap(axpy, repeat(10), xrange(100), cycle(xrange(10)))))

    def testStarmap(self):
        """Test starmap method"""
        def gen(a, x, y):
            x = iter(x)
            y = iter(y)
            try:
                while True:
                    yield (a, x.next(), y.next())
            except StopIteration:
                pass

        axpy = lambda a, x, y: a * x + y
        it = gen(10, xrange(100), cycle(xrange(10)))
        self.assertEqual(list(self.s.starmap(axpy, it)),
                         list(imap(axpy, repeat(10), xrange(100), cycle(xrange(10)))))
Beispiel #12
0
        for opt, arg in opts:
            if opt == '-h':
                host = arg
            elif opt == '-p':
                port = int(arg)
            elif opt == '-v':
                verbose = 1
            else:
                raise 'internal error: out-of-sync with getopt'

        if len(args) == 0:
            print >> sys.stderr, 'no workers specified'
            sys.exit(1)

        s = Sleigh(launch=sshcmd, nodeList=args, nwsHost=host, nwsPort=port, verbose=verbose)

        while 1:
            print '>',
            cmd = raw_input()
            cmd = cmd.strip()
            if cmd:
                try:
                    results = s.eachWorker(cmd)
                    for i in results:
                        if i:
                            print i
                except SyntaxError, e:
                    print >> sys.stderr, str(e)

    except getopt.GetoptError, e:
Beispiel #13
0
            elif opt == '-e':
                enableNwsUtility = True
            elif opt == '-c':
                skw['workerCount'] = int(arg)
            else:
                raise 'internal error: out-of-sync with getopt'

        # if no nodes are specified, start three on the local machine
        if len(args) > 0:
            skw['nodeList'] = args
        elif skw.get('launch') == 'service':
            skw['nodeList'] = defaultNodeList
            print "warning: using default nodeList:", defaultNodeList

        # create a sleigh on the specified nodes
        s = Sleigh(**skw)

        if enableNwsUtility:
            s.nws.declare('enableNwsUtility', SINGLE)
        s.nws.declare('MainTitle', SINGLE)
        s.nws.store('MainTitle', description)
        s.nws.declare('SubTitle', SINGLE)

        submitted = s.workerCount

        # do the work
        for i in xrange(numtasks):
            submitted += numtasks
            s.nws.store('SubTitle', '%d tasks submitted' % submitted)
            s.eachElem(worker, range(numtasks), **eo)