def _runWorker(self, worker_id, logDir): iteration = 0 while worker_id in self.threads: iteration += 1 log_path = self._workerLogFile(worker_id, iteration, logDir) logging.info("adding worker. script: %s, log: %s", self.worker_path, log_path) with open(log_path, 'a') as logfile: def writeline(msg): logfile.write(msg + '\n') env = dict(os.environ) env['UFORA_WORKER_BASE_PORT'] = str(30009 + 2 * self.num_ever_started) proc = SubprocessRunner.SubprocessRunner( [sys.executable, '-u', self.worker_path], writeline, writeline, env=env) logfile.write("***Starting worker***\n") logging.info("Starting worker %s", worker_id) logfile.flush() proc.start() self.processes[worker_id] = proc proc.wait() logging.info("Worker exited: %s", worker_id)
def test_tracebacks(self): traceback_test_notebook_path = \ os.path.join(self.cur_dir, "traceback_test.ipynb") returnCode, output, _ = SubprocessRunner.callAndReturnResultAndOutput( ['jupyter', 'nbconvert', '--stdout', '--to=markdown', '--ExecutePreprocessor.enabled=True', '--allow-errors', traceback_test_notebook_path]) outputString = "".join(output) self.assertEqual(returnCode, 0) pattern = ".*ValueError\\s*Traceback \\(most recent call last\\)" \ ".*<ipython-input-4-[a-f0-9]{12}> in inner\\(\\)" \ "\\s*1 with e\\.remotely\\.downloadAll\\(\\):" \ "\\s*2\\s* 1\\+2\\+3" \ "\\s*----> 3\\s*res = f\\(0\\)" \ "\\s*<ipython-input-3-[a-f0-9]{12}> in f\\(\\)" \ "\\s*1 def f\\(x\\):" \ "\\s*----> 2\\s*return g\\(x\\) \\+ 1" \ "\\s*<ipython-input-2-[a-f0-9]{12}> in g\\(\\)" \ "\\s*1 def g\\(x\\):" \ "\\s*----> 2\\s* return 1 / math\\.sqrt\\(x - 1\\)" \ "\\s*/volumes/src/packages/python/pyfora/pure_modules/pure_math.py" \ " in __call__\\(\\)" \ "\\s*21\\s*def __call__\\(self, val\\):" \ "\\s*22\\s*if val < 0\\.0:" \ "\\s*---> 23\\s*raise ValueError\\(\"math domain error\"\\)" \ "\\s*24" \ "\\s*25\\s* return val \\*\\* 0.5" \ "\\s*ValueError: math domain error" match = re.match(pattern, outputString, re.DOTALL) self.assertIsNotNone(match)
def validateTimingsForSubprocessCall( self, testName, subprocessArgs, meta, timeout = 600.0 ): resultCode, out, err = SubprocessRunner.callAndReturnResultAndOutput( subprocessArgs, timeout = timeout ) if resultCode != 0: meta.update({"failure": "subprocess call returned error"}) if PerformanceTestReporter.isCurrentlyTesting(): PerformanceTestReporter.recordTest( testName, None, meta ) assert resultCode == 0, err logging.info("Actual time was %s for %s", out[0], subprocessArgs) measuredTiming = float(out[0]) / self.baseTiming if PerformanceTestReporter.isCurrentlyTesting(): PerformanceTestReporter.recordTest( "fora_lang." + testName, float(out[0]), meta )
def startSharedState(self): cacheDir = Setup.config().getConfigValue( "SHARED_STATE_CACHE", os.path.join(Setup.config().fakeAwsBaseDir, 'ss_cache')) logging.info( "Starting shared state with cache dir '%s' and log file '%s'", cacheDir, self.sharedStateLogFile) with DirectoryScope.DirectoryScope(self.sharedStatePath): args = [ 'forever', '--killSignal', 'SIGTERM', '-l', self.sharedStateLogFile, 'start', '-c', 'python', self.sharedStateMainline, '--cacheDir', cacheDir, '--logging', 'info' ] def sharedStateStdout(msg): logging.info("SHARED STATE OUT> %s", msg) def sharedStateStderr(msg): logging.info("SHARED STATE ERR> %s", msg) startSharedState = SubprocessRunner.SubprocessRunner( args, sharedStateStdout, sharedStateStderr, dict(os.environ)) startSharedState.start() startSharedState.wait(60.0) startSharedState.stop()
def tryToStartRelayProcess(self, relayScript): env = dict(os.environ) env['NODE_ENV'] = 'test' hasStartedEvent = threading.Event() def onStdOut(msg): hasStartedEvent.set() logging.critical("RELAY STDOUT> %s", msg) def onStdErr(msg): logging.critical("RELAY STDERR> %s", msg) coffeeCommand = './run_coffee.sh' if 'UFORA_DEBUG_RELAY' in os.environ: coffeeCommand = 'coffee-debug' nodejsOptions = [] if 'UFORA_PROFILE_RELAY' in os.environ: nodejsOptions = ['--nodejs', '--prof'] args = [ relayScript, '--port', str(self.relayPort), '--gatewayport', str(self.subscribableWebObjectsPort) ] command = [ 'forever', '-f', '-l', self.relayLogFile, '--workingDir', self.webPath, 'start', '-c', coffeeCommand ] + nodejsOptions + args SubprocessRunner.SubprocessRunner(command, onStdOut, onStdErr, env).start() return hasStartedEvent
def pidAndProcessNameHoldingPorts(portStart, portStop): output = SubprocessRunner.callAndReturnOutput( ["/bin/bash", "-c", "netstat -tulpn 2> /dev/null"], timeout = 5 ) if output is None: #the netstat subprocess must have timed out return None outputLines = output.split("\n") allOutputLinesMatching = [] for port in range(portStart, portStop): for line in output.split("\n"): if (":" + str(port)) in line: allOutputLinesMatching.append(line) if not allOutputLinesMatching: return None for line in allOutputLinesMatching: match = re.match(r'.*LISTEN *([0-9]+)/(.*)', line) if match is not None: logging.warn("Found process: %s", line) pidToKill = int(match.group(1)) processName = match.group(2) return pidToKill, processName return None
def start(self): assert not self.hasStarted if self.actuallyRunOutOfProcess: def onStdout(msg): logging.info("OutOfProcessDownloader Out> %s", msg) def onStderr(msg): logging.info("OutOfProcessDownloader Err> %s", msg) self.childSubprocess = SubprocessRunner.SubprocessRunner( [sys.executable, __file__, str(self.childSocket.fileno())], onStdout, onStderr) self.childSubprocess.start() self.hasStarted = True self.backgroundThread = ManagedThread.ManagedThread( target=self.watchChild_) self.backgroundThread.start() else: self.hasStarted = True self.backgroundThread = ManagedThread.ManagedThread( target=self.executeChild_) self.backgroundThread.start()
def pidAndProcessNameHoldingPorts(portStart, portStop): output = SubprocessRunner.callAndReturnOutput( ["/bin/bash", "-c", "netstat -tulpn 2> /dev/null"], timeout=5) if output is None: #the netstat subprocess must have timed out return None outputLines = output.split("\n") allOutputLinesMatching = [] for port in range(portStart, portStop): for line in output.split("\n"): if (":" + str(port)) in line: allOutputLinesMatching.append(line) if not allOutputLinesMatching: return None for line in allOutputLinesMatching: match = re.match(r'.*LISTEN *([0-9]+)/(.*)', line) if match is not None: logging.warn("Found process: %s", line) pidToKill = int(match.group(1)) processName = match.group(2) return pidToKill, processName return None
def runSomeFora(): args = [ "/usr/local/bin/python", ufora.rootPythonPath + "/test.py", "-lang", "-langfilter=classTests.fora" ] returnCode, stdOut, stdErr = SubprocessRunner.callAndReturnResultAndOutput( args) assert returnCode == 0
def assertPrints(expression, expectedResult): returnCode, output, err = SubprocessRunner.callAndReturnResultAndOutput( ["fora", "-e", expression]) output = [l.strip() for l in output if l.strip()] assert returnCode == 0 assert output == [expectedResult], \ "Evaluating:\n\t%s\n\nExpected %s, got %s" % (expression, [expectedResult], output)
def assertPrints(expression, expectedResult): returnCode, output, err = SubprocessRunner.callAndReturnResultAndOutput( ["fora", "-e", expression ] ) output = [l.strip() for l in output if l.strip()] assert returnCode == 0 assert output == [expectedResult], \ "Evaluating:\n\t%s\n\nExpected %s, got %s" % (expression, [expectedResult], output)
def setUpClass(cls): cls.shrinkwrapPath = tempfile.mkdtemp() result, stdOut, stdErr = SubprocessRunner.callAndReturnResultAndOutput( ["ufora/scripts/shrinkwrap.py", "--source", rootPythonPath, "--dest", cls.shrinkwrapPath], timeout = 240.0, env=None ) assert result == 0, (result, stdOut, stdErr)
def setUpClass(cls): cls.shrinkwrapPath = tempfile.mkdtemp() result, stdOut, stdErr = SubprocessRunner.callAndReturnResultAndOutput( [ "ufora/scripts/shrinkwrap.py", "--source", rootPythonPath, "--dest", cls.shrinkwrapPath ], timeout=240.0, env=None) assert result == 0, (result, stdOut, stdErr)
def runCommandAndLogOutput(path, command, argument=None, env=None): toRun = [os.path.join(path, command)] if argument: toRun.append(argument) logging.info("Running command: %s", toRun) exitCode, stdOut, stdErr = SubprocessRunner.callAndReturnResultAndOutput( toRun, env=env) logging.info("Exit code: %s", exitCode) logging.info("%s STDOUT> %s", command, stdOut) logging.info("%s STDERR> %s", command, stdErr) return exitCode
def runCommandAndLogOutput(path, command, argument=None, env=None): toRun = [os.path.join(path, command)] if argument: toRun.append(argument) logging.info("Running command: %s", toRun) exitCode, stdOut, stdErr = SubprocessRunner.callAndReturnResultAndOutput( toRun, env=env ) logging.info("Exit code: %s", exitCode) logging.info("%s STDOUT> %s", command, stdOut) logging.info("%s STDERR> %s", command, stdErr) return exitCode
def runForeverCommand(script, foreverCommand, timeout=60.0): args = ['forever', foreverCommand, script] def foreverStdOut(msg): logging.info("FOREVER(%s) OUT> %s", script, msg) def foreverStdErr(msg): logging.info("FOREVER(%s) ERR> %s", script, msg) subprocess = SubprocessRunner.SubprocessRunner(args, foreverStdOut, foreverStdErr, dict(os.environ)) subprocess.start() subprocess.wait(timeout) subprocess.stop()
def test_subprocess_runner_exception(self): err = [] out = [] runner = SubprocessRunner.SubprocessRunner( ["/bin/bash", "-c", "echo toErr 1>&2; echo toNormal"], out.append, err.append ) runner.start() self.assertEqual(runner.wait(10), 0) self.assertEqual(out, ['toNormal']) self.assertEqual(err, ["toErr"]) runner.stop()
def test_subprocess_runner_runner(self): err = [] out = [] runner = SubprocessRunner.SubprocessRunner( ["/bin/echo", "asdf"], out.append, err.append ) runner.start() self.assertEqual(runner.wait(10), 0) self.assertEqual(out, ["asdf"]) self.assertEqual(err, []) runner.stop()
def executeInDocker(self, dockerTag, args): standardPath = "/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/sbin:/bin" shrinkwrappedPathVariable = ":".join([os.path.abspath(self.shrinkwrapPath + "/dependencies/" + p) for p in standardPath.split(":")]) return SubprocessRunner.callAndReturnResultAndOutput( ["docker","run"] + ["-e","PATH=" + shrinkwrappedPathVariable + ":" + standardPath] + #so that we get 'ufora' ["-e","PYTHONPATH=" + self.shrinkwrapPath] + ["-w",self.shrinkwrapPath] + ["-v",self.shrinkwrapPath + ":" + self.shrinkwrapPath] + [dockerTag] + args )
def executeInDocker(self, dockerTag, args): standardPath = "/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/sbin:/bin" shrinkwrappedPathVariable = ":".join([ os.path.abspath(self.shrinkwrapPath + "/dependencies/" + p) for p in standardPath.split(":") ]) return SubprocessRunner.callAndReturnResultAndOutput( ["docker", "run"] + ["-e", "PATH=" + shrinkwrappedPathVariable + ":" + standardPath] + #so that we get 'ufora' ["-e", "PYTHONPATH=" + self.shrinkwrapPath] + ["-w", self.shrinkwrapPath] + ["-v", self.shrinkwrapPath + ":" + self.shrinkwrapPath] + [dockerTag] + args)
def startGatewayService(self): args = [ 'forever', '--killSignal', 'SIGTERM', '-l', self.gatewayLogFile, 'start', '-c', 'python', self.gatewayServiceMainline, '--cluster-name', 'test' ] def gatewayStdout(msg): logging.info("GATEWAY OUT> %s", msg) def gatewayStderr(msg): logging.info("GATEWAY ERR> %s", msg) gatewayProc = SubprocessRunner.SubprocessRunner( args, gatewayStdout, gatewayStderr, dict(os.environ)) gatewayProc.start() gatewayProc.wait(60.0) gatewayProc.stop()
def onSubscribableConnection(self, sock, _): logging.info("creating a new process to handle connection") handlerPid = None with self._lock: if self.shouldStop(): return scriptName = os.path.join( os.path.split(__file__)[0], 'handleBackendGatewayConnection.py') def onStdOut(line): logging.info("%s > %s", handlerPid, line) def onStdErr(line): logging.error("%s > %s", handlerPid, line) connectProc = SubprocessRunner.SubprocessRunner( [sys.executable, scriptName], onStdOut, onStdErr) self.socketsToDisconnectOnExit.append(sock) connectProc.start() handlerPid = connectProc.pid with self._lock: self.procsToKillOnExit.add(connectProc) toWrite = pickle.dumps({ 'socketFd': sock.fileno(), 'sharedStateAddress': self.sharedStateAddress }) connectProc.write(struct.pack('I', len(toWrite))) connectProc.write(toWrite) connectProc.flush() def waitForProcToFinish(): connectProc.wait() connectProc.stop() sock.close() threading.Thread(target=waitForProcToFinish).start()
def __call__(self): args = ['forever', self.foreverCommand, self.script] response = [] def foreverStdOut(msg): response.append("FOREVER(%s) OUT> %s" % (self.script, msg)) def foreverStdErr(msg): response.append("FOREVER(%s) ERR> %s" % (self.script, msg)) subprocess = SubprocessRunner.SubprocessRunner(args, foreverStdOut, foreverStdErr, self.environ) subprocess.start() subprocess.wait(self.timeout) subprocess.stop() return "\n".join(response)
def testUsingCoffeescript(self): args = [ "mocha", "--reporter", "spec", "--compilers", "coffee:coffee-script/register", "testSubscribableWebObjects.coffee", "-b" ] def onOut(l): logging.info("Mocha Out> %s", l) def onErr(l): logging.info("Mocha Err> %s", l) subprocess = SubprocessRunner.SubprocessRunner(args, onOut, onErr) subprocess.start() result = subprocess.wait(720) subprocess.stop() self.assertEqual(result, 0)
def _runWorker(self, worker_id, logDir): iteration = 0 while worker_id in self.threads: iteration += 1 log_path = self._workerLogFile(worker_id, iteration, logDir) logging.info("adding worker. script: %s, log: %s", self.worker_path, log_path) with open(log_path, 'a') as logfile: def writeline(msg): logfile.write(msg + '\n') env = dict(os.environ) env['UFORA_WORKER_BASE_PORT'] = str(30009 + 2 * self.num_ever_started) core_log = log_path.split('.') core_log.insert(-1, 'core') env['UFORA_WORKER_CORE_LOG_FILE'] = '.'.join(core_log) env['CUMULUS_VECTOR_MB'] = "400" env['CUMULUS_MAX_MB'] = "500" env['CUMULUS_TRACK_TCMALLOC'] = '0' env['CUMULUS_DISK_STORAGE_SUBDIR'] = str(worker_id) self.num_ever_started += 1 proc = SubprocessRunner.SubprocessRunner( [sys.executable, '-u', self.worker_path], writeline, writeline, env=env) logfile.write("***Starting worker***\n") logging.info("Starting worker %s", worker_id) logfile.flush() proc.start() self.processes[worker_id] = proc proc.wait() logging.info("Worker exited: %s", worker_id)
def validateTimingsForSubprocessCall(self, testName, subprocessArgs, meta, timeout=600.0): resultCode, out, err = SubprocessRunner.callAndReturnResultAndOutput( subprocessArgs, timeout=timeout) if resultCode != 0: meta.update({"failure": "subprocess call returned error"}) if PerformanceTestReporter.isCurrentlyTesting(): PerformanceTestReporter.recordTest(testName, None, meta) assert resultCode == 0, err logging.info("Actual time was %s for %s", out[0], subprocessArgs) measuredTiming = float(out[0]) / self.baseTiming if PerformanceTestReporter.isCurrentlyTesting(): PerformanceTestReporter.recordTest("fora_lang." + testName, float(out[0]), meta)
def evaluateNotebookInSubprocess(self, notebookPath): return SubprocessRunner.callAndReturnResultAndOutput([ 'jupyter', 'nbconvert', '--stdout', '--ExecutePreprocessor.enabled=True', notebookPath ])
def runScript_(self, script): print print "Running %s" % script print "with a timeout of %s sec"% self.getTimeout(script) if sys.platform == 'linux2': directory, filename = os.path.split(script) args = [sys.executable, "-u", '-c', "print 'started'; execfile('%s')" % filename] with DirectoryScope(directory): tries = 0 runner = None while tries < 5 and runner is None: startedEvent = threading.Event() def printOutput(line): if line == 'started': startedEvent.set() print "Script %s started" % filename else: print "OUT> %s\n" % line, def printErr(line): print "ERR> %s\n" % line, runner = SubprocessRunner.SubprocessRunner( args, printOutput, printErr, self.envVars ) runner.start() startedEvent.wait(5) if not startedEvent.isSet(): runner.terminate() runner = None tries = tries + 1 print "Retrying script ", filename, " as python failed to start." if runner is None: print "Test %s failed to start a python process in 5 tries" % filename return False else: result = runner.wait(self.getTimeout(script)) if result is None: try: runner.terminate() except: print "Failed to terminate test runner: ", traceback.format_exc() print "Test %s timed out" % filename, return False runner.stop() if result != 0: print "Test %s failed" % filename, return False return True else: subprocess.check_call('cd "%s" & c:\python27\python.exe %s ' % os.path.split(script), shell = True ) return True
def runSomeFora(): args = ["/usr/local/bin/python", ufora.rootPythonPath + "/test.py", "-lang", "-langfilter=classTests.fora"] returnCode, stdOut, stdErr = SubprocessRunner.callAndReturnResultAndOutput(args) assert returnCode == 0
def evaluateNotebookInSubprocess(self, notebookPath): return SubprocessRunner.callAndReturnResultAndOutput( ['jupyter', 'nbconvert', '--stdout', '--ExecutePreprocessor.enabled=True', notebookPath] )