def main(): # initialize variables from environment modulePath = Env.get('PythonSleighModulePath', '') logDir = Env.get('PythonSleighLogDir') if not logDir or not os.path.isdir(logDir): logDir = _TMPDIR outfile = Env.get('PythonSleighWorkerOut') if outfile: outfile = os.path.join(logDir, os.path.split(outfile)[1]) else: outfile = _NULFILE Env['PythonSleighLogFile'] = outfile verbose = Env.has_key('PythonSleighWorkerOut') and 1 or 0 nwsName = Env['PythonSleighNwsName'] nwsHost = Env.get('PythonSleighNwsHost', 'localhost') nwsPort = int(Env.get('PythonSleighNwsPort', '8765')) if Env.has_key('PythonSleighName'): name = Env['PythonSleighName'] if Env.has_key('PythonSleighID'): name = '%s@%s' % (name, Env['PythonSleighID']) logger.setName(name) logger.logDebug(nwsName, nwsHost, nwsPort) nws = NetWorkSpace(nwsName, nwsHost, nwsPort, useUse=True, create=False) newProtocol = nws.findTry('version') is not None if newProtocol: worker_id = nws.fetch('worker_ids') Env['PythonSleighRank'] = worker_id # create the script file for the worker to execute script = '''\ import sys, os try: os.setpgid(0, 0) except: pass try: sys.stdout = sys.stderr = open(%s, 'w', 0) except: pass sys.path[1:1] = %s.split(os.pathsep) from nws.sleigh import cmdLaunch print "entering worker loop" cmdLaunch(%d) ''' % (repr(outfile), repr(modulePath), verbose) fd, tmpname = tempfile.mkstemp(suffix='.py', prefix='__nws', text=True) tmpfile = os.fdopen(fd, 'w') tmpfile.write(script) tmpfile.close() logger.logDebug("executing Python worker") argv = [sys.executable, tmpname] out = open(outfile, 'w') try: p = subprocess.Popen(argv, stdin=open(tmpname), stdout=out, stderr=subprocess.STDOUT) except OSError, e: logger.logError("error executing command:", argv) raise e
def main(): # initialize variables from environment modulePath = Env.get("PythonSleighModulePath", "") logDir = Env.get("RSleighLogDir") if not logDir or not os.path.isdir(logDir): logDir = _TMPDIR verbose = Env.has_key("RSleighWorkerOut") and "TRUE" or "FALSE" nwsName = Env["RSleighNwsName"] nwsHost = Env.get("RSleighNwsHost", "localhost") nwsPort = int(Env.get("RSleighNwsPort", "8765")) print nwsName, nwsHost, nwsPort # create the script file for the worker to execute script = ( """\ # use RSleighScriptDir to add the directory that contains # the nws package to the library path if possible # (and without modifying the worker's global environment) local({ scriptDir <- Sys.getenv('RSleighScriptDir') if (basename(scriptDir) == 'bin') { nwsDir <- dirname(scriptDir) nwsPkg <- basename(nwsDir) if (nwsPkg == 'Rbig') { libDir <- dirname(nwsDir) oldPaths <- .libPaths() newPaths <- c(libDir, oldPaths) cat("setting library paths to", newPaths, "\n") .libPaths(newPaths) } } }) library(Rbig) cmdLaunch(%s) """ % verbose ) fd, tmpname = tempfile.mkstemp(suffix=".R", prefix="__nws", dir=_TMPDIR, text=True) tmpfile = os.fdopen(fd, "w") tmpfile.write(script) tmpfile.close() print "executing R worker(s)" rprog = Env.get("RProg", "R") argv = [rprog, "--vanilla", "--slave"] if NetWorkSpace: nws = NetWorkSpace(nwsName, nwsHost, nwsPort, useUse=True, create=False) newProtocol = nws.findTry("version") is not None else: nws = None newProtocol = False numProcs = int(Env.get("RSleighNumProcs", "1")) workers = [] sleighID = int(Env["RSleighID"]) # only used for non-sleigh plugin for i in xrange(numProcs): outfile = Env.get("RSleighWorkerOut") if outfile: prefix = "sleigh_ride_%d_" % i fd, outfile = tempfile.mkstemp(suffix=".txt", prefix=prefix, dir=logDir, text=True) out = os.fdopen(fd, "w") else: outfile = _NULFILE out = open(outfile, "w") Env["RSleighLogFile"] = outfile if newProtocol: worker_id = nws.fetch("worker_ids") else: worker_id = str(i + sleighID) Env["RSleighRank"] = worker_id Env["RSleighLocalID"] = str(i) try: if sys.platform.startswith("win"): p = subprocess.Popen(argv, stdin=open(tmpname), stdout=out, stderr=subprocess.STDOUT) wpid = int(p._handle) else: p = subprocess.Popen(argv, stdin=open(tmpname), stdout=out, stderr=subprocess.STDOUT, preexec_fn=setpg) wpid = p.pid # attempt to make the child process a process group leader try: os.setpgid(wpid, 0) except: pass except OSError, e: print >> sys.stderr, "error executing command:", argv if not os.path.exists(rprog): print >> sys.stderr, rprog, "does not exist" raise e print "started worker %s, pid: %s" % (worker_id, wpid) workers.append((p, wpid, worker_id))