def main():
# initialize variables from environment
modulePath = Env.get('PythonSleighModulePath', '')
logDir = Env.get('PythonSleighLogDir')
if not logDir or not os.path.isdir(logDir):
logDir = _TMPDIR
outfile = Env.get('PythonSleighWorkerOut')
if outfile:
outfile = os.path.join(logDir, os.path.split(outfile)[1])
else:
outfile = _NULFILE
Env['PythonSleighLogFile'] = outfile
verbose = Env.has_key('PythonSleighWorkerOut') and 1 or 0
nwsName = Env['PythonSleighNwsName']
nwsHost = Env.get('PythonSleighNwsHost', 'localhost')
nwsPort = int(Env.get('PythonSleighNwsPort', '8765'))
if Env.has_key('PythonSleighName'):
name = Env['PythonSleighName']
if Env.has_key('PythonSleighID'):
name = '%s@%s' % (name, Env['PythonSleighID'])
logger.setName(name)
logger.logDebug(nwsName, nwsHost, nwsPort)
nws = NetWorkSpace(nwsName, nwsHost, nwsPort, useUse=True, create=False)
newProtocol = nws.findTry('version') is not None
if newProtocol:
worker_id = nws.fetch('worker_ids')
Env['PythonSleighRank'] = worker_id
# create the script file for the worker to execute
script = '''\
import sys, os
try: os.setpgid(0, 0)
except: pass
try: sys.stdout = sys.stderr = open(%s, 'w', 0)
except: pass
sys.path[1:1] = %s.split(os.pathsep)
from nws.sleigh import cmdLaunch
print "entering worker loop"
cmdLaunch(%d)
''' % (repr(outfile), repr(modulePath), verbose)
fd, tmpname = tempfile.mkstemp(suffix='.py', prefix='__nws', text=True)
tmpfile = os.fdopen(fd, 'w')
tmpfile.write(script)
tmpfile.close()
logger.logDebug("executing Python worker")
argv = [sys.executable, tmpname]
out = open(outfile, 'w')
try:
p = subprocess.Popen(argv, stdin=open(tmpname), stdout=out,
stderr=subprocess.STDOUT)
except OSError, e:
logger.logError("error executing command:", argv)
raise e
def main():
# initialize variables from environment
modulePath = Env.get("PythonSleighModulePath", "")
logDir = Env.get("RSleighLogDir")
if not logDir or not os.path.isdir(logDir):
logDir = _TMPDIR
verbose = Env.has_key("RSleighWorkerOut") and "TRUE" or "FALSE"
nwsName = Env["RSleighNwsName"]
nwsHost = Env.get("RSleighNwsHost", "localhost")
nwsPort = int(Env.get("RSleighNwsPort", "8765"))
print nwsName, nwsHost, nwsPort
# create the script file for the worker to execute
script = (
"""\
# use RSleighScriptDir to add the directory that contains
# the nws package to the library path if possible
# (and without modifying the worker's global environment)
local({
scriptDir <- Sys.getenv('RSleighScriptDir')
if (basename(scriptDir) == 'bin') {
nwsDir <- dirname(scriptDir)
nwsPkg <- basename(nwsDir)
if (nwsPkg == 'Rbig') {
libDir <- dirname(nwsDir)
oldPaths <- .libPaths()
newPaths <- c(libDir, oldPaths)
cat("setting library paths to", newPaths, "\n")
.libPaths(newPaths)
}
}
})
library(Rbig)
cmdLaunch(%s)
"""
% verbose
)
fd, tmpname = tempfile.mkstemp(suffix=".R", prefix="__nws", dir=_TMPDIR, text=True)
tmpfile = os.fdopen(fd, "w")
tmpfile.write(script)
tmpfile.close()
print "executing R worker(s)"
rprog = Env.get("RProg", "R")
argv = [rprog, "--vanilla", "--slave"]
if NetWorkSpace:
nws = NetWorkSpace(nwsName, nwsHost, nwsPort, useUse=True, create=False)
newProtocol = nws.findTry("version") is not None
else:
nws = None
newProtocol = False
numProcs = int(Env.get("RSleighNumProcs", "1"))
workers = []
sleighID = int(Env["RSleighID"]) # only used for non-sleigh plugin
for i in xrange(numProcs):
outfile = Env.get("RSleighWorkerOut")
if outfile:
prefix = "sleigh_ride_%d_" % i
fd, outfile = tempfile.mkstemp(suffix=".txt", prefix=prefix, dir=logDir, text=True)
out = os.fdopen(fd, "w")
else:
outfile = _NULFILE
out = open(outfile, "w")
Env["RSleighLogFile"] = outfile
if newProtocol:
worker_id = nws.fetch("worker_ids")
else:
worker_id = str(i + sleighID)
Env["RSleighRank"] = worker_id
Env["RSleighLocalID"] = str(i)
try:
if sys.platform.startswith("win"):
p = subprocess.Popen(argv, stdin=open(tmpname), stdout=out, stderr=subprocess.STDOUT)
wpid = int(p._handle)
else:
p = subprocess.Popen(argv, stdin=open(tmpname), stdout=out, stderr=subprocess.STDOUT, preexec_fn=setpg)
wpid = p.pid
# attempt to make the child process a process group leader
try:
os.setpgid(wpid, 0)
except:
pass
except OSError, e:
print >> sys.stderr, "error executing command:", argv
if not os.path.exists(rprog):
print >> sys.stderr, rprog, "does not exist"
raise e
print "started worker %s, pid: %s" % (worker_id, wpid)
workers.append((p, wpid, worker_id))
