def run(self): while True: try: try: ws = NetWorkSpace(self.wsName, self.nwsHost, self.nwsPort, persistent=True) ws.declare(self.varName, FIFO) while True: s = ws.fetch(self.varName) try: # split the request using the first character x = s[1:].split(s[0]) r = {'PythonSleighNwsHost': self.nwsHost, 'PythonSleighNwsPort': str(self.nwsPort)} r['PythonSleighNwsName'], r['PythonSleighWorkerCount'], \ r['PythonSleighID'], r['PythonSleighWorkingDir'], \ r['PythonSleighWorkerOut'], r['PythonSleighLogDir'], \ r['PythonSleighUserName'], r['PythonSleighModulePath'] = x self.gotRequest(r) except: # bad request: ignore it pass except NwsServerException: # server is probably down. sleep a bit, and try again time.sleep(10) except NwsOperationException: # maybe someone deleted my variable to signal a shutdown? return finally: # ws may not be defined try:ws.server.close() except: pass ws = None
def main(): # initialize variables from environment modulePath = Env.get('PythonSleighModulePath', '') logDir = Env.get('PythonSleighLogDir') if not logDir or not os.path.isdir(logDir): logDir = _TMPDIR outfile = Env.get('PythonSleighWorkerOut') if outfile: outfile = os.path.join(logDir, os.path.split(outfile)[1]) else: outfile = _NULFILE Env['PythonSleighLogFile'] = outfile verbose = Env.has_key('PythonSleighWorkerOut') and 1 or 0 nwsName = Env['PythonSleighNwsName'] nwsHost = Env.get('PythonSleighNwsHost', 'localhost') nwsPort = int(Env.get('PythonSleighNwsPort', '8765')) if Env.has_key('PythonSleighName'): name = Env['PythonSleighName'] if Env.has_key('PythonSleighID'): name = '%s@%s' % (name, Env['PythonSleighID']) logger.setName(name) logger.logDebug(nwsName, nwsHost, nwsPort) nws = NetWorkSpace(nwsName, nwsHost, nwsPort, useUse=True, create=False) newProtocol = nws.findTry('version') is not None if newProtocol: worker_id = nws.fetch('worker_ids') Env['PythonSleighRank'] = worker_id # create the script file for the worker to execute script = '''\ import sys, os try: os.setpgid(0, 0) except: pass try: sys.stdout = sys.stderr = open(%s, 'w', 0) except: pass sys.path[1:1] = %s.split(os.pathsep) from nws.sleigh import cmdLaunch print "entering worker loop" cmdLaunch(%d) ''' % (repr(outfile), repr(modulePath), verbose) fd, tmpname = tempfile.mkstemp(suffix='.py', prefix='__nws', text=True) tmpfile = os.fdopen(fd, 'w') tmpfile.write(script) tmpfile.close() logger.logDebug("executing Python worker") argv = [sys.executable, tmpname] out = open(outfile, 'w') try: p = subprocess.Popen(argv, stdin=open(tmpname), stdout=out, stderr=subprocess.STDOUT) except OSError, e: logger.logError("error executing command:", argv) raise e
def babelfish(host, port): ws = NetWorkSpace('Python babelfish', host, port) while 1: try: v = ws.fetch('food') t = str(v) if len(t) > MAXLEN: t = t[:MAXLEN] + '[WARNING: output truncated]' elif not t: t = repr(v) except NwsUnpicklingException, e: t = 'Unpickling error: ' + str(e.exception) ws.store('doof', t)
ws.store('x', 1) print 'should now see x.' print ws.listVars() print 'find (but don\'t consume) x.' print ws.find('x') print 'check that it is still there.' print ws.listVars() print 'associate another value with x.' ws.store('x', 2) print ws.listVars() print 'consume values for x, should see them in order saved.' print ws.fetch('x') print ws.fetch('x') print 'no more values for x... .' print ws.listVars() print 'so try to fetch and see what happens... .' print ws.fetchTry('x', 'no go') print 'create a single-value variable.' ws.declare('pi', 'single') print ws.listVars() print 'get rid of x.' ws.deleteVar('x') print ws.listVars()
import sys, socket host = "localhost" port = 8765 wsname = "hello" try: from nws.client import NetWorkSpace nws = NetWorkSpace(wsname, host, port) count = int(raw_input("Number of times to loop? ")) print "hello: iterations:", count nws.store("hello example", count) for i in range(count): nws.store("hello", i) j = nws.fetch("hello") nws.fetch("hello example") print "Success" except KeyboardInterrupt: print "hello exiting" except ImportError: print "Unable to import the nwsClient module" print "You may need to set or correct your PYTHONPATH" except socket.error: print "NWS server not running on %s, port %d" % (host, port) except: print "An unexpected error occurred:", sys.exc_info()[1]
if len(sys.argv) == 2: wsname = sys.argv[1] elif len(sys.argv) != 1: print >> sys.stderr, "Usage: %s [<wsname>]" % sys.argv[0] sys.exit(1) try: from nws.client import NetWorkSpace nws = NetWorkSpace(wsname, host, port) nws.store("game", 0) print "Ping-pong server '%s' starting" % wsname while 1: request = nws.fetch("ping") pong = request["replyto"] request["time"] = time.asctime() print "Got a ping from", pong nws.store(pong, request) except KeyboardInterrupt: print "Ping-pong server '%s' exiting" % wsname except ImportError: print "Unable to import the nwsClient module" print "You may need to set or correct your PYTHONPATH" except socket.error: print "NWS server not running on %s, port %d" % (host, port) except: print "An unexpected error occurred:", sys.exc_info()[1]
def main(): # initialize variables from environment modulePath = Env.get("PythonSleighModulePath", "") logDir = Env.get("RSleighLogDir") if not logDir or not os.path.isdir(logDir): logDir = _TMPDIR verbose = Env.has_key("RSleighWorkerOut") and "TRUE" or "FALSE" nwsName = Env["RSleighNwsName"] nwsHost = Env.get("RSleighNwsHost", "localhost") nwsPort = int(Env.get("RSleighNwsPort", "8765")) print nwsName, nwsHost, nwsPort # create the script file for the worker to execute script = ( """\ # use RSleighScriptDir to add the directory that contains # the nws package to the library path if possible # (and without modifying the worker's global environment) local({ scriptDir <- Sys.getenv('RSleighScriptDir') if (basename(scriptDir) == 'bin') { nwsDir <- dirname(scriptDir) nwsPkg <- basename(nwsDir) if (nwsPkg == 'Rbig') { libDir <- dirname(nwsDir) oldPaths <- .libPaths() newPaths <- c(libDir, oldPaths) cat("setting library paths to", newPaths, "\n") .libPaths(newPaths) } } }) library(Rbig) cmdLaunch(%s) """ % verbose ) fd, tmpname = tempfile.mkstemp(suffix=".R", prefix="__nws", dir=_TMPDIR, text=True) tmpfile = os.fdopen(fd, "w") tmpfile.write(script) tmpfile.close() print "executing R worker(s)" rprog = Env.get("RProg", "R") argv = [rprog, "--vanilla", "--slave"] if NetWorkSpace: nws = NetWorkSpace(nwsName, nwsHost, nwsPort, useUse=True, create=False) newProtocol = nws.findTry("version") is not None else: nws = None newProtocol = False numProcs = int(Env.get("RSleighNumProcs", "1")) workers = [] sleighID = int(Env["RSleighID"]) # only used for non-sleigh plugin for i in xrange(numProcs): outfile = Env.get("RSleighWorkerOut") if outfile: prefix = "sleigh_ride_%d_" % i fd, outfile = tempfile.mkstemp(suffix=".txt", prefix=prefix, dir=logDir, text=True) out = os.fdopen(fd, "w") else: outfile = _NULFILE out = open(outfile, "w") Env["RSleighLogFile"] = outfile if newProtocol: worker_id = nws.fetch("worker_ids") else: worker_id = str(i + sleighID) Env["RSleighRank"] = worker_id Env["RSleighLocalID"] = str(i) try: if sys.platform.startswith("win"): p = subprocess.Popen(argv, stdin=open(tmpname), stdout=out, stderr=subprocess.STDOUT) wpid = int(p._handle) else: p = subprocess.Popen(argv, stdin=open(tmpname), stdout=out, stderr=subprocess.STDOUT, preexec_fn=setpg) wpid = p.pid # attempt to make the child process a process group leader try: os.setpgid(wpid, 0) except: pass except OSError, e: print >> sys.stderr, "error executing command:", argv if not os.path.exists(rprog): print >> sys.stderr, rprog, "does not exist" raise e print "started worker %s, pid: %s" % (worker_id, wpid) workers.append((p, wpid, worker_id))
try: loops = int(sys.argv[1]) except: print >> sys.stderr, "Usage: %s [<loops> [<wsname>]]" % sys.argv[0] sys.exit(1) if len(sys.argv) == 3: wsname = sys.argv[2] elif len(sys.argv) != 1: print >> sys.stderr, "Usage: %s [<loops> [<wsname>]]" % sys.argv[0] sys.exit(1) try: from nws.client import NetWorkSpace nws = NetWorkSpace(wsname, host, port, useUse=True) game = nws.fetch("game") nws.store("game", game + 1) pong = "pong_%d" % game request = {'replyto': pong} print "Starting a ping-pong game", game for i in xrange(loops): request['i'] = i nws.store("ping", request) reply = nws.fetch(pong) print reply if i != reply['i']: print "Error: expected %d, received %d" % (i, reply['i']) nws.deleteVar(pong)