def _runWorker(self, worker_id, logDir):
iteration = 0
while worker_id in self.threads:
iteration += 1
log_path = self._workerLogFile(worker_id, iteration, logDir)
logging.info("adding worker. script: %s, log: %s",
self.worker_path, log_path)
with open(log_path, 'a') as logfile:
def writeline(msg):
logfile.write(msg + '\n')
env = dict(os.environ)
env['UFORA_WORKER_BASE_PORT'] = str(30009 +
2 * self.num_ever_started)
proc = SubprocessRunner.SubprocessRunner(
[sys.executable, '-u', self.worker_path],
writeline,
writeline,
env=env)
logfile.write("***Starting worker***\n")
logging.info("Starting worker %s", worker_id)
logfile.flush()
proc.start()
self.processes[worker_id] = proc
proc.wait()
logging.info("Worker exited: %s", worker_id)
def test_tracebacks(self):
traceback_test_notebook_path = \
os.path.join(self.cur_dir, "traceback_test.ipynb")
returnCode, output, _ = SubprocessRunner.callAndReturnResultAndOutput(
['jupyter', 'nbconvert', '--stdout', '--to=markdown',
'--ExecutePreprocessor.enabled=True', '--allow-errors',
traceback_test_notebook_path])
outputString = "".join(output)
self.assertEqual(returnCode, 0)
pattern = ".*ValueError\\s*Traceback \\(most recent call last\\)" \
".*<ipython-input-4-[a-f0-9]{12}> in inner\\(\\)" \
"\\s*1 with e\\.remotely\\.downloadAll\\(\\):" \
"\\s*2\\s* 1\\+2\\+3" \
"\\s*----> 3\\s*res = f\\(0\\)" \
"\\s*<ipython-input-3-[a-f0-9]{12}> in f\\(\\)" \
"\\s*1 def f\\(x\\):" \
"\\s*----> 2\\s*return g\\(x\\) \\+ 1" \
"\\s*<ipython-input-2-[a-f0-9]{12}> in g\\(\\)" \
"\\s*1 def g\\(x\\):" \
"\\s*----> 2\\s* return 1 / math\\.sqrt\\(x - 1\\)" \
"\\s*/volumes/src/packages/python/pyfora/pure_modules/pure_math.py" \
" in __call__\\(\\)" \
"\\s*21\\s*def __call__\\(self, val\\):" \
"\\s*22\\s*if val < 0\\.0:" \
"\\s*---> 23\\s*raise ValueError\\(\"math domain error\"\\)" \
"\\s*24" \
"\\s*25\\s* return val \\*\\* 0.5" \
"\\s*ValueError: math domain error"
match = re.match(pattern, outputString, re.DOTALL)
self.assertIsNotNone(match)
def validateTimingsForSubprocessCall(
self,
testName,
subprocessArgs,
meta,
timeout = 600.0
):
resultCode, out, err = SubprocessRunner.callAndReturnResultAndOutput(
subprocessArgs,
timeout = timeout
)
if resultCode != 0:
meta.update({"failure": "subprocess call returned error"})
if PerformanceTestReporter.isCurrentlyTesting():
PerformanceTestReporter.recordTest(
testName,
None,
meta
)
assert resultCode == 0, err
logging.info("Actual time was %s for %s", out[0], subprocessArgs)
measuredTiming = float(out[0]) / self.baseTiming
if PerformanceTestReporter.isCurrentlyTesting():
PerformanceTestReporter.recordTest(
"fora_lang." + testName,
float(out[0]),
meta
)
def startSharedState(self):
cacheDir = Setup.config().getConfigValue(
"SHARED_STATE_CACHE",
os.path.join(Setup.config().fakeAwsBaseDir, 'ss_cache'))
logging.info(
"Starting shared state with cache dir '%s' and log file '%s'",
cacheDir, self.sharedStateLogFile)
with DirectoryScope.DirectoryScope(self.sharedStatePath):
args = [
'forever', '--killSignal', 'SIGTERM', '-l',
self.sharedStateLogFile, 'start', '-c', 'python',
self.sharedStateMainline, '--cacheDir', cacheDir, '--logging',
'info'
]
def sharedStateStdout(msg):
logging.info("SHARED STATE OUT> %s", msg)
def sharedStateStderr(msg):
logging.info("SHARED STATE ERR> %s", msg)
startSharedState = SubprocessRunner.SubprocessRunner(
args, sharedStateStdout, sharedStateStderr, dict(os.environ))
startSharedState.start()
startSharedState.wait(60.0)
startSharedState.stop()
def tryToStartRelayProcess(self, relayScript):
env = dict(os.environ)
env['NODE_ENV'] = 'test'
hasStartedEvent = threading.Event()
def onStdOut(msg):
hasStartedEvent.set()
logging.critical("RELAY STDOUT> %s", msg)
def onStdErr(msg):
logging.critical("RELAY STDERR> %s", msg)
coffeeCommand = './run_coffee.sh'
if 'UFORA_DEBUG_RELAY' in os.environ:
coffeeCommand = 'coffee-debug'
nodejsOptions = []
if 'UFORA_PROFILE_RELAY' in os.environ:
nodejsOptions = ['--nodejs', '--prof']
args = [
relayScript, '--port',
str(self.relayPort), '--gatewayport',
str(self.subscribableWebObjectsPort)
]
command = [
'forever', '-f', '-l', self.relayLogFile, '--workingDir',
self.webPath, 'start', '-c', coffeeCommand
] + nodejsOptions + args
SubprocessRunner.SubprocessRunner(command, onStdOut, onStdErr,
env).start()
return hasStartedEvent
def pidAndProcessNameHoldingPorts(portStart, portStop):
output = SubprocessRunner.callAndReturnOutput(
["/bin/bash", "-c", "netstat -tulpn 2> /dev/null"],
timeout = 5
)
if output is None:
#the netstat subprocess must have timed out
return None
outputLines = output.split("\n")
allOutputLinesMatching = []
for port in range(portStart, portStop):
for line in output.split("\n"):
if (":" + str(port)) in line:
allOutputLinesMatching.append(line)
if not allOutputLinesMatching:
return None
for line in allOutputLinesMatching:
match = re.match(r'.*LISTEN *([0-9]+)/(.*)', line)
if match is not None:
logging.warn("Found process: %s", line)
pidToKill = int(match.group(1))
processName = match.group(2)
return pidToKill, processName
return None
def start(self):
assert not self.hasStarted
if self.actuallyRunOutOfProcess:
def onStdout(msg):
logging.info("OutOfProcessDownloader Out> %s", msg)
def onStderr(msg):
logging.info("OutOfProcessDownloader Err> %s", msg)
self.childSubprocess = SubprocessRunner.SubprocessRunner(
[sys.executable, __file__,
str(self.childSocket.fileno())], onStdout, onStderr)
self.childSubprocess.start()
self.hasStarted = True
self.backgroundThread = ManagedThread.ManagedThread(
target=self.watchChild_)
self.backgroundThread.start()
else:
self.hasStarted = True
self.backgroundThread = ManagedThread.ManagedThread(
target=self.executeChild_)
self.backgroundThread.start()
def pidAndProcessNameHoldingPorts(portStart, portStop):
output = SubprocessRunner.callAndReturnOutput(
["/bin/bash", "-c", "netstat -tulpn 2> /dev/null"], timeout=5)
if output is None:
#the netstat subprocess must have timed out
return None
outputLines = output.split("\n")
allOutputLinesMatching = []
for port in range(portStart, portStop):
for line in output.split("\n"):
if (":" + str(port)) in line:
allOutputLinesMatching.append(line)
if not allOutputLinesMatching:
return None
for line in allOutputLinesMatching:
match = re.match(r'.*LISTEN *([0-9]+)/(.*)', line)
if match is not None:
logging.warn("Found process: %s", line)
pidToKill = int(match.group(1))
processName = match.group(2)
return pidToKill, processName
return None
def runSomeFora():
args = [
"/usr/local/bin/python", ufora.rootPythonPath + "/test.py", "-lang",
"-langfilter=classTests.fora"
]
returnCode, stdOut, stdErr = SubprocessRunner.callAndReturnResultAndOutput(
args)
assert returnCode == 0
def assertPrints(expression, expectedResult):
returnCode, output, err = SubprocessRunner.callAndReturnResultAndOutput(
["fora", "-e", expression])
output = [l.strip() for l in output if l.strip()]
assert returnCode == 0
assert output == [expectedResult], \
"Evaluating:\n\t%s\n\nExpected %s, got %s" % (expression, [expectedResult], output)
def assertPrints(expression, expectedResult):
returnCode, output, err = SubprocessRunner.callAndReturnResultAndOutput(
["fora", "-e", expression ]
)
output = [l.strip() for l in output if l.strip()]
assert returnCode == 0
assert output == [expectedResult], \
"Evaluating:\n\t%s\n\nExpected %s, got %s" % (expression, [expectedResult], output)
def setUpClass(cls):
cls.shrinkwrapPath = tempfile.mkdtemp()
result, stdOut, stdErr = SubprocessRunner.callAndReturnResultAndOutput(
["ufora/scripts/shrinkwrap.py", "--source", rootPythonPath,
"--dest", cls.shrinkwrapPath],
timeout = 240.0,
env=None
)
assert result == 0, (result, stdOut, stdErr)
def setUpClass(cls):
cls.shrinkwrapPath = tempfile.mkdtemp()
result, stdOut, stdErr = SubprocessRunner.callAndReturnResultAndOutput(
[
"ufora/scripts/shrinkwrap.py", "--source", rootPythonPath,
"--dest", cls.shrinkwrapPath
],
timeout=240.0,
env=None)
assert result == 0, (result, stdOut, stdErr)
def runCommandAndLogOutput(path, command, argument=None, env=None):
toRun = [os.path.join(path, command)]
if argument:
toRun.append(argument)
logging.info("Running command: %s", toRun)
exitCode, stdOut, stdErr = SubprocessRunner.callAndReturnResultAndOutput(
toRun, env=env)
logging.info("Exit code: %s", exitCode)
logging.info("%s STDOUT> %s", command, stdOut)
logging.info("%s STDERR> %s", command, stdErr)
return exitCode
def runCommandAndLogOutput(path, command, argument=None, env=None):
toRun = [os.path.join(path, command)]
if argument:
toRun.append(argument)
logging.info("Running command: %s", toRun)
exitCode, stdOut, stdErr = SubprocessRunner.callAndReturnResultAndOutput(
toRun,
env=env
)
logging.info("Exit code: %s", exitCode)
logging.info("%s STDOUT> %s", command, stdOut)
logging.info("%s STDERR> %s", command, stdErr)
return exitCode
def runForeverCommand(script, foreverCommand, timeout=60.0):
args = ['forever', foreverCommand, script]
def foreverStdOut(msg):
logging.info("FOREVER(%s) OUT> %s", script, msg)
def foreverStdErr(msg):
logging.info("FOREVER(%s) ERR> %s", script, msg)
subprocess = SubprocessRunner.SubprocessRunner(args, foreverStdOut,
foreverStdErr,
dict(os.environ))
subprocess.start()
subprocess.wait(timeout)
subprocess.stop()
def test_subprocess_runner_exception(self):
err = []
out = []
runner = SubprocessRunner.SubprocessRunner(
["/bin/bash", "-c", "echo toErr 1>&2; echo toNormal"],
out.append,
err.append
)
runner.start()
self.assertEqual(runner.wait(10), 0)
self.assertEqual(out, ['toNormal'])
self.assertEqual(err, ["toErr"])
runner.stop()
def test_subprocess_runner_runner(self):
err = []
out = []
runner = SubprocessRunner.SubprocessRunner(
["/bin/echo", "asdf"],
out.append,
err.append
)
runner.start()
self.assertEqual(runner.wait(10), 0)
self.assertEqual(out, ["asdf"])
self.assertEqual(err, [])
runner.stop()
def executeInDocker(self, dockerTag, args):
standardPath = "/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/sbin:/bin"
shrinkwrappedPathVariable = ":".join([os.path.abspath(self.shrinkwrapPath + "/dependencies/" + p) for p in standardPath.split(":")])
return SubprocessRunner.callAndReturnResultAndOutput(
["docker","run"] +
["-e","PATH=" + shrinkwrappedPathVariable + ":" + standardPath] +
#so that we get 'ufora'
["-e","PYTHONPATH=" + self.shrinkwrapPath] +
["-w",self.shrinkwrapPath] +
["-v",self.shrinkwrapPath + ":" + self.shrinkwrapPath] +
[dockerTag] +
args
)
def executeInDocker(self, dockerTag, args):
standardPath = "/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/sbin:/bin"
shrinkwrappedPathVariable = ":".join([
os.path.abspath(self.shrinkwrapPath + "/dependencies/" + p)
for p in standardPath.split(":")
])
return SubprocessRunner.callAndReturnResultAndOutput(
["docker", "run"] +
["-e", "PATH=" + shrinkwrappedPathVariable + ":" + standardPath] +
#so that we get 'ufora'
["-e", "PYTHONPATH=" + self.shrinkwrapPath] +
["-w", self.shrinkwrapPath] +
["-v", self.shrinkwrapPath + ":" + self.shrinkwrapPath] +
[dockerTag] + args)
def startGatewayService(self):
args = [
'forever', '--killSignal', 'SIGTERM', '-l', self.gatewayLogFile,
'start', '-c', 'python', self.gatewayServiceMainline,
'--cluster-name', 'test'
]
def gatewayStdout(msg):
logging.info("GATEWAY OUT> %s", msg)
def gatewayStderr(msg):
logging.info("GATEWAY ERR> %s", msg)
gatewayProc = SubprocessRunner.SubprocessRunner(
args, gatewayStdout, gatewayStderr, dict(os.environ))
gatewayProc.start()
gatewayProc.wait(60.0)
gatewayProc.stop()
def onSubscribableConnection(self, sock, _):
logging.info("creating a new process to handle connection")
handlerPid = None
with self._lock:
if self.shouldStop():
return
scriptName = os.path.join(
os.path.split(__file__)[0],
'handleBackendGatewayConnection.py')
def onStdOut(line):
logging.info("%s > %s", handlerPid, line)
def onStdErr(line):
logging.error("%s > %s", handlerPid, line)
connectProc = SubprocessRunner.SubprocessRunner(
[sys.executable, scriptName], onStdOut, onStdErr)
self.socketsToDisconnectOnExit.append(sock)
connectProc.start()
handlerPid = connectProc.pid
with self._lock:
self.procsToKillOnExit.add(connectProc)
toWrite = pickle.dumps({
'socketFd': sock.fileno(),
'sharedStateAddress': self.sharedStateAddress
})
connectProc.write(struct.pack('I', len(toWrite)))
connectProc.write(toWrite)
connectProc.flush()
def waitForProcToFinish():
connectProc.wait()
connectProc.stop()
sock.close()
threading.Thread(target=waitForProcToFinish).start()
def __call__(self):
args = ['forever', self.foreverCommand, self.script]
response = []
def foreverStdOut(msg):
response.append("FOREVER(%s) OUT> %s" % (self.script, msg))
def foreverStdErr(msg):
response.append("FOREVER(%s) ERR> %s" % (self.script, msg))
subprocess = SubprocessRunner.SubprocessRunner(args, foreverStdOut,
foreverStdErr,
self.environ)
subprocess.start()
subprocess.wait(self.timeout)
subprocess.stop()
return "\n".join(response)
def testUsingCoffeescript(self):
args = [
"mocha", "--reporter", "spec", "--compilers",
"coffee:coffee-script/register",
"testSubscribableWebObjects.coffee", "-b"
]
def onOut(l):
logging.info("Mocha Out> %s", l)
def onErr(l):
logging.info("Mocha Err> %s", l)
subprocess = SubprocessRunner.SubprocessRunner(args, onOut, onErr)
subprocess.start()
result = subprocess.wait(720)
subprocess.stop()
self.assertEqual(result, 0)
def _runWorker(self, worker_id, logDir):
iteration = 0
while worker_id in self.threads:
iteration += 1
log_path = self._workerLogFile(worker_id, iteration, logDir)
logging.info("adding worker. script: %s, log: %s",
self.worker_path, log_path)
with open(log_path, 'a') as logfile:
def writeline(msg):
logfile.write(msg + '\n')
env = dict(os.environ)
env['UFORA_WORKER_BASE_PORT'] = str(30009 +
2 * self.num_ever_started)
core_log = log_path.split('.')
core_log.insert(-1, 'core')
env['UFORA_WORKER_CORE_LOG_FILE'] = '.'.join(core_log)
env['CUMULUS_VECTOR_MB'] = "400"
env['CUMULUS_MAX_MB'] = "500"
env['CUMULUS_TRACK_TCMALLOC'] = '0'
env['CUMULUS_DISK_STORAGE_SUBDIR'] = str(worker_id)
self.num_ever_started += 1
proc = SubprocessRunner.SubprocessRunner(
[sys.executable, '-u', self.worker_path],
writeline,
writeline,
env=env)
logfile.write("***Starting worker***\n")
logging.info("Starting worker %s", worker_id)
logfile.flush()
proc.start()
self.processes[worker_id] = proc
proc.wait()
logging.info("Worker exited: %s", worker_id)
def validateTimingsForSubprocessCall(self,
testName,
subprocessArgs,
meta,
timeout=600.0):
resultCode, out, err = SubprocessRunner.callAndReturnResultAndOutput(
subprocessArgs, timeout=timeout)
if resultCode != 0:
meta.update({"failure": "subprocess call returned error"})
if PerformanceTestReporter.isCurrentlyTesting():
PerformanceTestReporter.recordTest(testName, None, meta)
assert resultCode == 0, err
logging.info("Actual time was %s for %s", out[0], subprocessArgs)
measuredTiming = float(out[0]) / self.baseTiming
if PerformanceTestReporter.isCurrentlyTesting():
PerformanceTestReporter.recordTest("fora_lang." + testName,
float(out[0]), meta)
def evaluateNotebookInSubprocess(self, notebookPath):
return SubprocessRunner.callAndReturnResultAndOutput([
'jupyter', 'nbconvert', '--stdout',
'--ExecutePreprocessor.enabled=True', notebookPath
])
def runScript_(self, script):
print
print "Running %s" % script
print "with a timeout of %s sec"% self.getTimeout(script)
if sys.platform == 'linux2':
directory, filename = os.path.split(script)
args = [sys.executable, "-u", '-c', "print 'started'; execfile('%s')" % filename]
with DirectoryScope(directory):
tries = 0
runner = None
while tries < 5 and runner is None:
startedEvent = threading.Event()
def printOutput(line):
if line == 'started':
startedEvent.set()
print "Script %s started" % filename
else:
print "OUT> %s\n" % line,
def printErr(line):
print "ERR> %s\n" % line,
runner = SubprocessRunner.SubprocessRunner(
args,
printOutput,
printErr,
self.envVars
)
runner.start()
startedEvent.wait(5)
if not startedEvent.isSet():
runner.terminate()
runner = None
tries = tries + 1
print "Retrying script ", filename, " as python failed to start."
if runner is None:
print "Test %s failed to start a python process in 5 tries" % filename
return False
else:
result = runner.wait(self.getTimeout(script))
if result is None:
try:
runner.terminate()
except:
print "Failed to terminate test runner: ", traceback.format_exc()
print "Test %s timed out" % filename,
return False
runner.stop()
if result != 0:
print "Test %s failed" % filename,
return False
return True
else:
subprocess.check_call('cd "%s" & c:\python27\python.exe %s '
% os.path.split(script),
shell = True
)
return True
def runSomeFora():
args = ["/usr/local/bin/python", ufora.rootPythonPath + "/test.py", "-lang", "-langfilter=classTests.fora"]
returnCode, stdOut, stdErr = SubprocessRunner.callAndReturnResultAndOutput(args)
assert returnCode == 0
def evaluateNotebookInSubprocess(self, notebookPath):
return SubprocessRunner.callAndReturnResultAndOutput(
['jupyter', 'nbconvert', '--stdout',
'--ExecutePreprocessor.enabled=True', notebookPath]
)
