def perform_build(self, history_line):
import pdb
pdb.set_trace()
self.build_number += 1
self.start_time = datetime.now()
log.debug("Performing build %d on history line: %s" % (self.build_number, history_line))
build_proc = ProcessHandler(cmd = ['/home/ctalbert/projects/b2g-hamachi/build.sh'],
cwd = self.build_info['workdir'],
env=self.build_info['env'],
processOutputLine=[self.notify_status],
kill_on_timeout=True,
onTimeout=[self.notify_timeout],
onFinish=[self.notify_finished],
shell=True)
try:
sys.stdout.write("Starting Build %d:" % self.build_number)
build_proc.run(timeout=7200)
build_proc.processOutput()
exitcode = build_proc.wait()
except (KeyboardInterrupt, SystemExit):
print "User Canceled Operation!"
log.debug("Build canceled by user")
raise
finally:
self.build_log.close()
if exitcode == 0:
print "Build %d Completed Successfully" % self.build_number
log.debug("Build %d for history line: %s completed successfully" % (self.build_number, history_line))
else:
print "Build %d Failed" % self.build_number
log.debug("Build %d for history line: %s FAILED" % (self.build_number, history_line))
def build_script(self, modules=None, debug=False, profiling=False,
noftu=False, noopt=False, valgrind=False):
command = [os.path.join(self.b2g_home, 'build.sh')]
if modules:
command.extend(modules)
if debug:
command.insert(0, 'B2G_DEBUG=1')
if profiling:
command.insert(0, 'MOZ_PROFILING=1')
if noftu:
command.insert(0, 'NOFTU=1')
if noopt:
if profiling:
print("Can't perform profiling if optimizer is disabled")
return 1
command.insert(0, 'B2G_NOOPT=1')
if valgrind:
command.insert(0, 'B2G_VALGRIND=1')
p = ProcessHandler(command)
p.run()
#TODO: Error checking.
return p.wait()
def _run_profile(self):
command_args = utils.GenerateBrowserCommandLine(
self.browser_config["browser_path"],
self.browser_config["extra_args"],
self.profile_dir,
self.browser_config["init_url"]
)
def browser_log(line):
LOG.process_output(browser.pid, line)
browser = ProcessHandler(command_args, env=self.env,
processOutputLine=browser_log)
browser.run()
LOG.process_start(browser.pid, ' '.join(command_args))
try:
exit_code = browser.wait()
except KeyboardInterrupt:
browser.kill()
raise
LOG.process_exit(browser.pid, exit_code)
results_raw = '\n'.join(browser.output)
if not self.PROFILE_REGEX.search(results_raw):
LOG.info("Could not find %s in browser output"
% self.PROFILE_REGEX.pattern)
LOG.info("Raw results:%s" % results_raw)
raise TalosError("browser failed to close after being initialized")
def _runCmd(self, args, retryLimit=None):
"""
Runs a command using adb
returns: instance of ProcessHandler
"""
retryLimit = retryLimit or self.retryLimit
finalArgs = [self._adbPath]
if self._serverHost is not None:
finalArgs.extend(['-H', self._serverHost])
if self._serverPort is not None:
finalArgs.extend(['-P', str(self._serverPort)])
if self._deviceSerial:
finalArgs.extend(['-s', self._deviceSerial])
finalArgs.extend(args)
self._logger.debug("_runCmd - command: %s" % ' '.join(finalArgs))
retries = 0
while retries < retryLimit:
proc = ProcessHandler(finalArgs, storeOutput=True,
processOutputLine=self._log)
proc.run()
proc.returncode = proc.wait()
if proc.returncode == None:
proc.kill()
retries += 1
else:
return proc
def _run_profile(self):
command_args = utils.GenerateBrowserCommandLine(
self.browser_config["browser_path"],
self.browser_config["extra_args"],
self.profile_dir,
self.browser_config["init_url"]
)
def browser_log(line):
logging.debug('BROWSER_OUTPUT: %s', line)
browser = ProcessHandler(command_args, env=self.env,
processOutputLine=browser_log)
browser.run()
try:
browser.wait()
except KeyboardInterrupt:
browser.kill()
raise
results_raw = '\n'.join(browser.output)
if not self.PROFILE_REGEX.search(results_raw):
logging.info("Could not find %s in browser output",
self.PROFILE_REGEX.pattern)
logging.info("Raw results:%s", results_raw)
raise TalosError("browser failed to close after being initialized")
def flash(self):
command = os.path.join(self.b2g_home, 'flash.sh')
p = ProcessHandler(command)
if _is_device_attached():
p.run()
return p.wait()
return 1
def setup_avds(self):
'''
If tooltool cache mechanism is enabled, the cached version is used by
the fetch command. If the manifest includes an "unpack" field, tooltool
will unpack all compressed archives mentioned in the manifest.
'''
c = self.config
dirs = self.query_abs_dirs()
# FIXME
# Clobbering and re-unpacking would not be needed if we had a way to
# check whether the unpacked content already present match the
# contents of the tar ball
self.rmtree(dirs['abs_avds_dir'])
self.mkdir_p(dirs['abs_avds_dir'])
if 'avd_url' in c:
# Intended for experimental setups to evaluate an avd prior to
# tooltool deployment.
url = c['avd_url']
self.download_unpack(url, dirs['abs_avds_dir'])
else:
url = self._get_repo_url(c["tooltool_manifest_path"])
self._tooltool_fetch(url, dirs['abs_avds_dir'])
avd_home_dir = self.abs_dirs['abs_avds_dir']
if avd_home_dir != "/home/cltbld/.android":
# Modify the downloaded avds to point to the right directory.
cmd = [
'bash', '-c',
'sed -i "s|/home/cltbld/.android|%s|" %s/test-*.ini' %
(avd_home_dir, os.path.join(avd_home_dir, 'avd'))
]
proc = ProcessHandler(cmd)
proc.run()
proc.wait()
def test_relative_path(self):
tempdir = tempfile.mkdtemp()
# make a dummy profile
profile = FirefoxProfile(os.path.join(tempdir, 'testprofilepath'),
restore=False)
self.assertTrue(os.path.exists(os.path.join(tempdir,
'testprofilepath',
'user.js')))
# make a dummy test
test = """var test = function () { };"""
f = file(os.path.join(tempdir, 'test_dummy.js'), 'w')
f.write(test)
f.close()
# run mozmill on it
process = ProcessHandler(['mozmill',
'-t', 'test_dummy.js',
'--profile=testprofilepath'],
cwd=tempdir)
process.run()
process.waitForFinish()
self.assertNotEqual(process.proc.poll(), None)
# cleanup
shutil.rmtree(tempdir)
def test_all_js(tests, options):
print "Running JS Tests"
# We run each test in its own instance since these are harness tests.
# That just seems safer, no opportunity for cross-talk since
# we are sorta using the framework to test itself
results = JSResults()
for t in tests:
# write a temporary manifest
manifest = TestManifest()
manifest.tests = [t]
fd, filename = tempfile.mkstemp(suffix=".ini")
os.close(fd)
fp = file(filename, "w")
manifest.write(fp=fp)
fp.close()
# get CLI arguments to mozmill
args = ["-b", options.binary]
args.append("--console-level=DEBUG")
args.append("-m")
args.append(filename)
# run the test
proc = ProcessHandler("mozmill", args=args)
proc.run()
status = proc.waitForFinish(timeout=300)
command = proc.commandline
results.acquire(t["name"], proc.output, status, command)
# remove the temporary manifest
os.remove(filename)
return results
def _runCmd(self, args, timeout=None, retryLimit=None):
"""
Runs a command using adb
If timeout is specified, the process is killed after <timeout> seconds.
returns: instance of ProcessHandler
"""
retryLimit = retryLimit or self.retryLimit
finalArgs = [self._adbPath]
if self._serverHost is not None:
finalArgs.extend(['-H', self._serverHost])
if self._serverPort is not None:
finalArgs.extend(['-P', str(self._serverPort)])
if self._deviceSerial:
finalArgs.extend(['-s', self._deviceSerial])
finalArgs.extend(args)
self._logger.debug("_runCmd - command: %s" % ' '.join(finalArgs))
if not timeout:
timeout = self.default_timeout
def _timeout():
self._logger.error("Timeout exceeded for _runCmd call '%s'" % ' '.join(finalArgs))
retries = 0
while retries < retryLimit:
proc = ProcessHandler(finalArgs, storeOutput=True,
processOutputLine=self._log, onTimeout=_timeout)
proc.run(timeout=timeout)
proc.returncode = proc.wait()
if proc.returncode == None:
proc.kill()
retries += 1
else:
return proc
def lint(files, config, **kwargs):
tests_dir = os.path.join(kwargs['root'], 'testing', 'web-platform', 'tests')
def process_line(line):
try:
data = json.loads(line)
except ValueError:
return
data["level"] = "error"
data["path"] = os.path.relpath(os.path.join(tests_dir, data["path"]), kwargs['root'])
results.append(result.from_config(config, **data))
if files == [tests_dir]:
print >> sys.stderr, ("No specific files specified, running the full wpt lint"
" (this is slow)")
files = ["--all"]
cmd = [os.path.join(tests_dir, 'wpt'), 'lint', '--json'] + files
if platform.system() == 'Windows':
cmd.insert(0, sys.executable)
proc = ProcessHandler(cmd, env=os.environ, processOutputLine=process_line)
proc.run()
try:
proc.wait()
if proc.returncode != 0:
results.append(
result.from_config(config,
message="Lint process exited with return code %s" %
proc.returncode))
except KeyboardInterrupt:
proc.kill()
return results
def _checkCmd(self, args, timeout=None, retryLimit=None):
"""
Runs a command using adb and waits for the command to finish.
If timeout is specified, the process is killed after <timeout> seconds.
returns: returncode from process
"""
retryLimit = retryLimit or self.retryLimit
finalArgs = [self._adbPath]
if self._deviceSerial:
finalArgs.extend(['-s', self._deviceSerial])
finalArgs.extend(args)
self._logger.debug("_checkCmd - command: %s" % ' '.join(finalArgs))
if not timeout:
# We are asserting that all commands will complete in this
# time unless otherwise specified
timeout = self.default_timeout
timeout = int(timeout)
retries = 0
while retries < retryLimit:
proc = ProcessHandler(finalArgs, processOutputLine=self._log)
proc.run(timeout=timeout)
ret_code = proc.wait()
if ret_code == None:
proc.kill()
retries += 1
else:
return ret_code
raise DMError("Timeout exceeded for _checkCmd call after %d retries." % retries)
def setup_avds(self):
'''
If tooltool cache mechanism is enabled, the cached version is used by
the fetch command. If the manifest includes an "unpack" field, tooltool
will unpack all compressed archives mentioned in the manifest.
'''
c = self.config
dirs = self.query_abs_dirs()
# Always start with a clean AVD: AVD includes Android images
# which can be stateful.
self.rmtree(dirs['abs_avds_dir'])
self.mkdir_p(dirs['abs_avds_dir'])
if 'avd_url' in c:
# Intended for experimental setups to evaluate an avd prior to
# tooltool deployment.
url = c['avd_url']
self.download_unpack(url, dirs['abs_avds_dir'])
else:
url = self._get_repo_url(c["tooltool_manifest_path"])
self._tooltool_fetch(url, dirs['abs_avds_dir'])
avd_home_dir = self.abs_dirs['abs_avds_dir']
if avd_home_dir != "/home/cltbld/.android":
# Modify the downloaded avds to point to the right directory.
cmd = [
'bash', '-c',
'sed -i "s|/home/cltbld/.android|%s|" %s/test-*.ini' %
(avd_home_dir, os.path.join(avd_home_dir, 'avd'))
]
proc = ProcessHandler(cmd)
proc.run()
proc.wait()
def test_relative_path(self):
tempdir = tempfile.mkdtemp()
# make a dummy profile
profile = FirefoxProfile(os.path.join(tempdir, 'testprofilepath'),
restore=False)
self.assertTrue(os.path.exists(os.path.join(tempdir,
'testprofilepath',
'user.js')))
# make a dummy test
test = """function test() { };"""
f = file(os.path.join(tempdir, 'test_dummy.js'), 'w')
f.write(test)
f.close()
# run mozmill on it
process = ProcessHandler(['mozmill',
'-t', 'test_dummy.js',
'--profile=testprofilepath'],
cwd=tempdir,
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertNotEqual(process.proc.poll(), None)
# cleanup
shutil.rmtree(tempdir)
class ServoTestharnessExecutor(ProcessTestExecutor):
convert_result = testharness_result_converter
def __init__(self, *args, **kwargs):
ProcessTestExecutor.__init__(self, *args, **kwargs)
self.result_data = None
self.result_flag = None
def run_test(self, test):
self.result_data = None
self.result_flag = threading.Event()
self.command = [self.binary, "--cpu", "--hard-fail",
urlparse.urljoin(self.http_server_url, test.url)]
if self.debug_args:
self.command = list(self.debug_args) + self.command
self.proc = ProcessHandler(self.command,
processOutputLine=[self.on_output],
onFinish=self.on_finish)
self.proc.run()
timeout = test.timeout * self.timeout_multiplier
# Now wait to get the output we expect, or until we reach the timeout
self.result_flag.wait(timeout + 5)
if self.result_flag.is_set() and self.result_data is not None:
self.result_data["test"] = test.url
result = self.convert_result(test, self.result_data)
self.proc.kill()
else:
if self.proc.proc.poll() is not None:
result = (test.result_cls("CRASH", None), [])
else:
self.proc.kill()
result = (test.result_cls("TIMEOUT", None), [])
self.runner.send_message("test_ended", test, result)
def on_output(self, line):
prefix = "ALERT: RESULT: "
line = line.decode("utf8", "replace")
if line.startswith(prefix):
self.result_data = json.loads(line[len(prefix):])
self.result_flag.set()
else:
if self.interactive:
print line
else:
self.logger.process_output(self.proc.pid,
line,
" ".join(self.command))
def on_finish(self):
self.result_flag.set()
def run_process():
path = os.path.join(tests_dir, "lint")
proc = ProcessHandler([path, "--json"], env=os.environ,
processOutputLine=process_line)
proc.run()
try:
proc.wait()
except KeyboardInterrupt:
proc.kill()
def gdb(self):
command = os.path.join(self.b2g_home, 'run-gdb.sh')
p = ProcessHandler(command)
p.run()
#TODO The emulator requires adb to run, we should check if that is
#running, catch that error or better yet, start adb.
return p.wait()
class ServoRefTestExecutor(ProcessTestExecutor):
convert_result = reftest_result_converter
def __init__(self, browser, http_server_url, binary=None, timeout_multiplier=1,
screenshot_cache=None, debug_args=None, pause_after_test=False):
ProcessTestExecutor.__init__(self,
browser,
http_server_url,
timeout_multiplier=timeout_multiplier,
debug_args=debug_args)
self.protocol = Protocol(self, browser, http_server_url)
self.screenshot_cache = screenshot_cache
self.implementation = RefTestImplementation(self)
self.tempdir = tempfile.mkdtemp()
def teardown(self):
os.rmdir(self.tempdir)
ProcessTestExecutor.teardown(self)
def screenshot(self, url, timeout):
full_url = urlparse.urljoin(self.http_server_url, url)
with TempFilename(self.tempdir) as output_path:
self.command = [self.binary, "--cpu", "--hard-fail", "--exit",
"--output=%s" % output_path, full_url]
self.proc = ProcessHandler(self.command,
processOutputLine=[self.on_output])
self.proc.run()
rv = self.proc.wait(timeout=timeout)
if rv is None:
self.proc.kill()
return False, ("EXTERNAL-TIMEOUT", None)
if rv < 0:
return False, ("CRASH", None)
with open(output_path) as f:
# Might need to strip variable headers or something here
data = f.read()
return True, data
def do_test(self, test):
result = self.implementation.run_test(test)
return self.convert_result(test, result)
def on_output(self, line):
line = line.decode("utf8", "replace")
if self.interactive:
print line
else:
self.logger.process_output(self.proc.pid,
line,
" ".join(self.command))
def _isLocalZipAvailable(self):
def _noOutput(line):
# suppress output from zip ProcessHandler
pass
try:
proc = ProcessHandler(["zip", "-?"], storeOutput=False, processOutputLine=_noOutput)
proc.run()
proc.wait()
except:
return False
return True
def pushDir(self, localDir, remoteDir, retryLimit=None, timeout=None):
# adb "push" accepts a directory as an argument, but if the directory
# contains symbolic links, the links are pushed, rather than the linked
# files; we either zip/unzip or re-copy the directory into a temporary
# one to get around this limitation
retryLimit = retryLimit or self.retryLimit
if self._useZip:
self.removeDir(remoteDir)
self.mkDirs(remoteDir + "/x")
try:
localZip = tempfile.mktemp() + ".zip"
remoteZip = remoteDir + "/adbdmtmp.zip"
proc = ProcessHandler(["zip", "-r", localZip, '.'], cwd=localDir,
processOutputLine=self._log)
proc.run()
proc.wait()
self.pushFile(localZip, remoteZip, retryLimit=retryLimit, createDir=False)
mozfile.remove(localZip)
data = self._runCmd(["shell", "unzip", "-o", remoteZip,
"-d", remoteDir]).output[0]
self._checkCmd(["shell", "rm", remoteZip],
retryLimit=retryLimit, timeout=self.short_timeout)
if re.search("unzip: exiting", data) or re.search("Operation not permitted", data):
raise Exception("unzip failed, or permissions error")
except Exception:
self._logger.warning(traceback.format_exc())
self._logger.warning("zip/unzip failure: falling back to normal push")
self._useZip = False
self.pushDir(localDir, remoteDir, retryLimit=retryLimit, timeout=timeout)
else:
localDir = os.path.normpath(localDir)
remoteDir = os.path.normpath(remoteDir)
tempParent = tempfile.mkdtemp()
remoteName = os.path.basename(remoteDir)
newLocal = os.path.join(tempParent, remoteName)
dir_util.copy_tree(localDir, newLocal)
# See do_sync_push in
# https://android.googlesource.com/platform/system/core/+/master/adb/file_sync_client.cpp
# Work around change in behavior in adb 1.0.36 where if
# the remote destination directory exists, adb push will
# copy the source directory *into* the destination
# directory otherwise it will copy the source directory
# *onto* the destination directory.
if self._adb_version >= '1.0.36':
remoteDir = '/'.join(remoteDir.rstrip('/').split('/')[:-1])
try:
if self._checkCmd(["push", newLocal, remoteDir],
retryLimit=retryLimit, timeout=timeout):
raise DMError("failed to push %s (copy of %s) to %s" %
(newLocal, localDir, remoteDir))
except BaseException:
raise
finally:
mozfile.remove(tempParent)
def _tooltool_fetch(self):
def outputHandler(line):
self._log_debug(line)
command = ["python", "tooltool.py", "fetch", "-m", "releng.manifest"]
proc = ProcessHandler(command, processOutputLine=outputHandler, storeOutput=False, cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def _tooltool_fetch():
def outputHandler(line):
_log_debug(line)
_download_file(TOOLTOOL_URL, "tooltool.py", EMULATOR_HOME_DIR)
command = [sys.executable, "tooltool.py", "fetch", "-o", "-m", "releng.manifest"]
proc = ProcessHandler(command, processOutputLine=outputHandler, storeOutput=False, cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def run_process(cmdargs):
# flake8 seems to handle SIGINT poorly. Handle it here instead
# so we can kill the process without a cryptic traceback.
orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
proc = ProcessHandler(cmdargs, env=os.environ,
processOutputLine=process_line)
proc.run()
signal.signal(signal.SIGINT, orig)
try:
proc.wait()
except KeyboardInterrupt:
proc.kill()
def _tooltool_fetch():
def outputHandler(line):
_log_debug(line)
command = ['python', 'tooltool.py', 'fetch', '-o', '-m', 'releng.manifest']
proc = ProcessHandler(
command, processOutputLine=outputHandler, storeOutput=False,
cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def start_logcat(self, serial, logfile=None, stream=None, filterspec=None):
logcat_args = [self.app_ctx.adb, '-s', '%s' % serial,
'logcat', '-v', 'time', '-b', 'main', '-b', 'radio']
# only log filterspec
if filterspec:
logcat_args.extend(['-s', filterspec])
process_args = {}
if logfile:
process_args['logfile'] = logfile
elif stream:
process_args['stream'] = stream
proc = ProcessHandler(logcat_args, **process_args)
proc.run()
return proc
def test_no_option(self):
process = ProcessHandler(['mozmill',
'-b', os.environ['BROWSER_PATH'],
'-t', os.path.join(testdir,
'useMozmill',
'testServerRoot.js')
],
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertEqual(process.proc.poll(), 1,
'Test failed')
def test_options(self):
absdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
testdir = os.path.join(absdir, 'js-tests')
process = ProcessHandler(['mozmill',
'-b', os.environ['BROWSER_PATH'],
'-t', os.path.join(testdir,
'test_module1.js'),
'-m', os.path.join(testdir, 'example.ini')
])
process.run()
process.waitForFinish()
self.assertNotEqual(process.proc.poll(), 0,
'Parser error due to -t and -m are mutually exclusive')
def test_option(self):
process = ProcessHandler(['mozmill',
'-b', os.environ['BROWSER_PATH'],
'-t', os.path.join(testdir,
'useMozmill',
'testServerRoot.js'),
'--server-root', os.path.join(testdir,
'../../data'),
],
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertEqual(process.proc.poll(), 0,
'Test was run successful')
def _runCmd(self, args):
"""
Runs a command using adb
returns: instance of ProcessHandler
"""
finalArgs = [self._adbPath]
if self._deviceSerial:
finalArgs.extend(['-s', self._deviceSerial])
finalArgs.extend(args)
self._logger.debug("_runCmd - command: %s" % ' '.join(finalArgs))
proc = ProcessHandler(finalArgs, storeOutput=True,
processOutputLine=self._log)
proc.run()
proc.returncode = proc.wait()
return proc
def test_options(self):
absdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
testdir = os.path.join(absdir, 'js-tests')
process = ProcessHandler(['mozmill',
'-b', os.environ['BROWSER_PATH'],
'-t', os.path.join(testdir,
'testUsemozmillControllerOpen.js'),
'-m', os.path.join(testdir, 'tests-null.ini')
],
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertNotEqual(process.proc.poll(), 0,
'Parser error due to -t and -m are mutually exclusive')
def run(self):
self.reset()
# Update the environment variables
env = os.environ.copy()
process_args = {
"cmd": self.command,
"cwd": self.path,
"onFinish": self.collect_results,
"processOutputLine": self.process_line,
"stream": sys.stdout,
"env": env,
"universal_newlines": True,
}
proc = ProcessHandler(**process_args)
proc.run()
return proc.wait()
def _tooltool_fetch():
def outputHandler(line):
_log_debug(line)
_download_file(TOOLTOOL_URL, 'tooltool.py', EMULATOR_HOME_DIR)
command = [
sys.executable, 'tooltool.py', 'fetch', '-o', '-m', 'releng.manifest'
]
proc = ProcessHandler(command,
processOutputLine=outputHandler,
storeOutput=False,
cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except Exception:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def test_options(self):
absdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
testdir = os.path.join(absdir, 'js-modules')
process = ProcessHandler(
[
'mozmill', '-b', os.environ['BROWSER_PATH'], '-t',
os.path.join(testdir, 'useMozmill', 'testTestPass.js'), '-m',
os.path.join(testdir, 'manifest-empty.ini')
],
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertNotEqual(
process.proc.poll(), 0,
'Parser error due to -t and -m are mutually exclusive')
def _run_python_test(self, test_path):
from mozprocess import ProcessHandler
output = []
def _log(line):
# Buffer messages if more than one worker to avoid interleaving
if self.jobs > 1:
output.append(line)
else:
self.log(logging.INFO, 'python-test', {'line': line.rstrip()}, '{line}')
file_displayed_test = [] # used as boolean
def _line_handler(line):
if not file_displayed_test:
output = ('Ran' in line or 'collected' in line or
line.startswith('TEST-'))
if output:
file_displayed_test.append(True)
_log(line)
_log(test_path)
cmd = [self.virtualenv_manager.python_path, test_path]
env = os.environ.copy()
env[b'PYTHONDONTWRITEBYTECODE'] = b'1'
proc = ProcessHandler(cmd, env=env, processOutputLine=_line_handler, storeOutput=False)
proc.run()
return_code = proc.wait()
if not file_displayed_test:
_log('TEST-UNEXPECTED-FAIL | No test output (missing mozunit.main() '
'call?): {}'.format(test_path))
if self.verbose:
if return_code != 0:
_log('Test failed: {}'.format(test_path))
else:
_log('Test passed: {}'.format(test_path))
return output, return_code
def start_mitmproxy_playback(
self,
mitmdump_path,
browser_path,
):
"""Startup mitmproxy and replay the specified flow file"""
LOG.info("mitmdump path: %s" % mitmdump_path)
LOG.info("browser path: %s" % browser_path)
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
env["PATH"] = os.path.dirname(browser_path) + ";" + env["PATH"]
command = [mitmdump_path, "-k"]
if "playback_tool_args" in self.config:
command.extend(self.config["playback_tool_args"])
LOG.info("Starting mitmproxy playback using env path: %s" %
env["PATH"])
LOG.info("Starting mitmproxy playback using command: %s" %
" ".join(command))
# to turn off mitmproxy log output, use these params for Popen:
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
mitmproxy_proc = ProcessHandler(command,
logfile=os.path.join(
self.upload_dir, "mitmproxy.log"),
env=env)
mitmproxy_proc.run()
# XXX replace the code below with a loop with a connection attempt
# Bug 1532557
time.sleep(MITMDUMP_SLEEP)
data = mitmproxy_proc.poll()
if data is None: # None value indicates process hasn't terminated
LOG.info("Mitmproxy playback successfully started as pid %d" %
mitmproxy_proc.pid)
return mitmproxy_proc
# cannot continue as we won't be able to playback the pages
LOG.error(
"Aborting: mitmproxy playback process failed to start, poll returned: %s"
% data)
# XXX here we might end up with a ghost mitmproxy
sys.exit()
def lint(files, config, **kwargs):
log = kwargs['log']
tests_dir = os.path.join(kwargs['root'], 'testing', 'web-platform',
'tests')
def process_line(line):
try:
data = json.loads(line)
except ValueError:
return
data["level"] = "error"
data["path"] = os.path.relpath(os.path.join(tests_dir, data["path"]),
kwargs['root'])
data.setdefault("lineno", 0)
results.append(result.from_config(config, **data))
if files == [tests_dir]:
print(
"No specific files specified, running the full wpt lint"
" (this is slow)",
file=sys.stderr)
files = ["--all"]
cmd = ['python2', os.path.join(tests_dir, 'wpt'), 'lint', '--json'] + files
log.debug("Command: {}".format(' '.join(cmd)))
proc = ProcessHandler(cmd,
env=os.environ,
processOutputLine=process_line,
universal_newlines=True)
proc.run()
try:
proc.wait()
if proc.returncode != 0:
results.append(
result.from_config(
config,
message="Lint process exited with return code %s" %
proc.returncode))
except KeyboardInterrupt:
proc.kill()
return results
def _tooltool_fetch():
def outputHandler(line):
_log_debug(line)
tooltool_full_path = os.path.abspath(TOOLTOOL_PATH)
command = [
sys.executable, tooltool_full_path, 'fetch', '-o', '-m',
'releng.manifest'
]
proc = ProcessHandler(command,
processOutputLine=outputHandler,
storeOutput=False,
cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except Exception:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def _checkCmd(self, args, timeout=None, retryLimit=None):
"""
Runs a command using adb and waits for the command to finish.
If timeout is specified, the process is killed after <timeout> seconds.
returns: returncode from process
"""
retryLimit = retryLimit or self.retryLimit
finalArgs = [self._adbPath]
if self._serverHost is not None:
finalArgs.extend(['-H', self._serverHost])
if self._serverPort is not None:
finalArgs.extend(['-P', str(self._serverPort)])
if self._deviceSerial:
finalArgs.extend(['-s', self._deviceSerial])
finalArgs.extend(args)
self._logger.debug("_checkCmd - command: %s" % ' '.join(finalArgs))
if not timeout:
# We are asserting that all commands will complete in this
# time unless otherwise specified
timeout = self.default_timeout
def _timeout():
self._logger.error("Timeout exceeded for _checkCmd call '%s'" %
' '.join(finalArgs))
timeout = int(timeout)
retries = 0
while retries < retryLimit:
proc = ProcessHandler(finalArgs,
processOutputLine=self._log,
onTimeout=_timeout)
proc.run(timeout=timeout)
ret_code = proc.wait()
if ret_code == None:
proc.kill()
retries += 1
else:
return ret_code
raise DMError("Timeout exceeded for _checkCmd call after %d retries." %
retries)
def _runCmd(self, args, timeout=None, retryLimit=None):
"""
Runs a command using adb
If timeout is specified, the process is killed after <timeout> seconds.
returns: instance of ProcessHandler
"""
retryLimit = retryLimit or self.retryLimit
finalArgs = [self._adbPath]
if self._serverHost is not None:
finalArgs.extend(['-H', self._serverHost])
if self._serverPort is not None:
finalArgs.extend(['-P', str(self._serverPort)])
if self._deviceSerial:
finalArgs.extend(['-s', self._deviceSerial])
finalArgs.extend(args)
self._logger.debug("_runCmd - command: %s" % ' '.join(finalArgs))
if not timeout:
timeout = self.default_timeout
def _timeout():
self._logger.error("Timeout exceeded for _runCmd call '%s'" %
' '.join(finalArgs))
retries = 0
proc = None
while retries < retryLimit:
proc = ProcessHandler(finalArgs,
storeOutput=True,
processOutputLine=self._log,
onTimeout=_timeout)
proc.run(timeout=timeout)
proc.returncode = proc.wait()
if proc.returncode is not None:
break
proc.kill()
self._logger.warning("_runCmd failed for '%s'" %
' '.join(finalArgs))
retries += 1
if retries >= retryLimit:
self._logger.warning("_runCmd exceeded all retries")
return proc
def tooltool_download(manifest, run_local, raptor_dir):
"""Download a file from tooltool using the provided tooltool manifest"""
def outputHandler(line):
LOG.info(line)
if run_local:
command = [
sys.executable, TOOLTOOL_PATH, "fetch", "-o", "-m", manifest
]
else:
# we want to use the tooltool cache in production
if os.environ.get("TOOLTOOLCACHE") is not None:
_cache = os.environ["TOOLTOOLCACHE"]
else:
# XXX top level dir? really?
# that gets run locally on any platform
# when you call ./mach python-test
_cache = "/builds/tooltool_cache"
command = [
sys.executable,
TOOLTOOL_PATH,
"fetch",
"-o",
"-m",
manifest,
"-c",
_cache,
]
proc = ProcessHandler(command,
processOutputLine=outputHandler,
storeOutput=False,
cwd=raptor_dir)
proc.run()
try:
proc.wait()
except Exception:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def pushDir(self, localDir, remoteDir, retryLimit=None, timeout=None):
# adb "push" accepts a directory as an argument, but if the directory
# contains symbolic links, the links are pushed, rather than the linked
# files; we either zip/unzip or re-copy the directory into a temporary
# one to get around this limitation
retryLimit = retryLimit or self.retryLimit
if self._useZip:
self.removeDir(remoteDir)
self.mkDirs(remoteDir+"/x")
try:
localZip = tempfile.mktemp() + ".zip"
remoteZip = remoteDir + "/adbdmtmp.zip"
proc = ProcessHandler(["zip", "-r", localZip, '.'], cwd=localDir,
processOutputLine=self._log)
proc.run()
proc.wait()
self.pushFile(localZip, remoteZip, retryLimit=retryLimit, createDir=False)
mozfile.remove(localZip)
data = self._runCmd(["shell", "unzip", "-o", remoteZip,
"-d", remoteDir]).output[0]
self._checkCmd(["shell", "rm", remoteZip],
retryLimit=retryLimit, timeout=self.short_timeout)
if re.search("unzip: exiting", data) or re.search("Operation not permitted", data):
raise Exception("unzip failed, or permissions error")
except:
self._logger.warning(traceback.format_exc())
self._logger.warning("zip/unzip failure: falling back to normal push")
self._useZip = False
self.pushDir(localDir, remoteDir, retryLimit=retryLimit, timeout=timeout)
else:
# If the remote directory exists, newer implementations of
# "adb push" will create a sub-directory, while older versions
# will not! Bug 1285040
self.mkDirs(remoteDir+"/x")
self.removeDir(remoteDir)
tmpDir = tempfile.mkdtemp()
# copytree's target dir must not already exist, so create a subdir
tmpDirTarget = os.path.join(tmpDir, "tmp")
shutil.copytree(localDir, tmpDirTarget)
self._checkCmd(["push", tmpDirTarget, remoteDir],
retryLimit=retryLimit, timeout=timeout)
mozfile.remove(tmpDir)
def setup_avds(self):
'''
If tooltool cache mechanism is enabled, the cached version is used by
the fetch command. If the manifest includes an "unpack" field, tooltool
will unpack all compressed archives mentioned in the manifest.
'''
c = self.config
dirs = self.query_abs_dirs()
# FIXME
# Clobbering and re-unpacking would not be needed if we had a way to
# check whether the unpacked content already present match the
# contents of the tar ball
self.rmtree(dirs['abs_avds_dir'])
self.mkdir_p(dirs['abs_avds_dir'])
if not self.buildbot_config:
# XXX until we figure out how to determine the repo_path, revision
url = 'https://hg.mozilla.org/%s/raw-file/%s/%s' % (
"try", "default", c["tooltool_manifest_path"])
self._tooltool_fetch(url)
elif self.buildbot_config and 'properties' in self.buildbot_config:
url = 'https://hg.mozilla.org/%s/raw-file/%s/%s' % (
self.buildbot_config['properties']['repo_path'],
self.buildbot_config['properties']['revision'],
c["tooltool_manifest_path"])
self._tooltool_fetch(url)
else:
self.fatal("properties in self.buildbot_config are required to "
"retrieve tooltool manifest to be used for avds setup")
if self.config.get("developer_mode"):
# For developer mode we have to modify the downloaded avds to
# point to the right directory.
avd_home_dir = self.abs_dirs['abs_avds_dir']
cmd = [
'bash', '-c',
'sed -i "s|/home/cltbld/.android|%s|" %s/test-*.ini' %
(avd_home_dir, os.path.join(avd_home_dir, 'avd'))
]
proc = ProcessHandler(cmd)
proc.run()
proc.wait()
def run(self):
self.reset()
# Update the environment variables
env = os.environ.copy()
# disable "GC poisoning" Bug# 1499043
env['JSGC_DISABLE_POISONING'] = '1'
process_args = {
'cmd': self.command,
'cwd': self.path,
'onFinish': self.collect_results,
'processOutputLine': self.process_line,
'stream': sys.stdout,
'env': env,
}
proc = ProcessHandler(**process_args)
proc.run()
return proc.wait()
def _tooltool_fetch(self, manifest):
def outputHandler(line):
LOG.info(line)
command = [
sys.executable, TOOLTOOL_PATH, 'fetch', '-o', '-m', manifest
]
proc = ProcessHandler(command,
processOutputLine=outputHandler,
storeOutput=False,
cwd=self.bindir)
proc.run()
try:
proc.wait()
except Exception:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def _verify_kvm(substs):
# 'emulator -accel-check' should produce output like:
# accel:
# 0
# KVM (version 12) is installed and usable
# accel
emulator_path = _find_sdk_exe(substs, 'emulator', True)
if not emulator_path:
emulator_path = 'emulator'
command = [emulator_path, '-accel-check']
try:
p = ProcessHandler(command, storeOutput=True)
p.run()
p.wait()
out = p.output
if 'is installed and usable' in ''.join(out):
return
except Exception as e:
_log_warning(str(e))
_log_warning("Unable to verify kvm acceleration!")
_log_warning("The x86 emulator may fail to start without kvm.")
def run(self):
self.reset()
# Update the environment variables
env = os.environ.copy()
# disable "GC poisoning" Bug# 1499043
env["JSGC_DISABLE_POISONING"] = "1"
process_args = {
"cmd": self.command,
"cwd": self.path,
"onFinish": self.collect_results,
"processOutputLine": self.process_line,
"stream": sys.stdout,
"env": env,
"universal_newlines": True,
}
proc = ProcessHandler(**process_args)
proc.run()
return proc.wait()
def lint(files, config, **kwargs):
tests_dir = os.path.join(kwargs['root'], 'testing', 'web-platform',
'tests')
def process_line(line):
try:
data = json.loads(line)
except ValueError:
return
data["level"] = "error"
data["path"] = os.path.relpath(os.path.join(tests_dir, data["path"]),
kwargs['root'])
data.setdefault("lineno", 0)
results.append(result.from_config(config, **data))
if files == [tests_dir]:
print(
"No specific files specified, running the full wpt lint"
" (this is slow)",
file=sys.stderr)
files = ["--all"]
cmd = [os.path.join(tests_dir, 'wpt'), 'lint', '--json'] + files
if platform.system() == 'Windows':
cmd.insert(0, sys.executable)
proc = ProcessHandler(cmd, env=os.environ, processOutputLine=process_line)
proc.run()
try:
proc.wait()
if proc.returncode != 0:
results.append(
result.from_config(
config,
message="Lint process exited with return code %s" %
proc.returncode))
except KeyboardInterrupt:
proc.kill()
return results
def _runCmd(self, args, retryLimit=None):
"""
Runs a command using adb
returns: instance of ProcessHandler
"""
retryLimit = retryLimit or self.retryLimit
finalArgs = [self._adbPath]
if self._deviceSerial:
finalArgs.extend(['-s', self._deviceSerial])
finalArgs.extend(args)
self._logger.debug("_runCmd - command: %s" % ' '.join(finalArgs))
retries = 0
while retries < retryLimit:
proc = ProcessHandler(finalArgs, storeOutput=True,
processOutputLine=self._log)
proc.run()
proc.returncode = proc.wait()
if proc.returncode == None:
proc.kill()
retries += 1
else:
return proc
def run_process(self, cmd, cwd=None, dump=False):
def _processOutput(line):
if self.verbose or dump:
print(line)
if self.verbose:
self.build_obj.log(logging.INFO, "autophone", {},
"Running '%s' in '%s'" % (cmd, cwd))
proc = ProcessHandler(cmd, cwd=cwd, processOutputLine=_processOutput,
processStderrLine=_processOutput)
proc.run()
proc_complete = False
try:
proc.wait()
if proc.proc.returncode == 0:
proc_complete = True
except Exception:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
if not proc_complete:
if not self.verbose:
print(proc.output)
return proc_complete
def test_all_js(tests, options):
print "Running JS Tests"
# We run each test in its own instance since these are harness tests.
# That just seems safer, no opportunity for cross-talk since
# we are sorta using the framework to test itself
results = JSResults()
for t in tests:
# write a temporary manifest
manifest = TestManifest()
manifest.tests = [t]
fd, filename = tempfile.mkstemp(suffix='.ini')
os.close(fd)
fp = file(filename, 'w')
manifest.write(fp=fp)
fp.close()
# get CLI arguments to mozmill
args = []
if options.binary:
args.extend(['-b', options.binary])
args.append('--console-level=DEBUG')
args.append('-m')
args.append(filename)
# run the test
proc = ProcessHandler("mozmill", args=args)
proc.run()
status = proc.waitForFinish(timeout=300)
command = proc.commandline
results.acquire(t['name'], proc.output, status, command)
# remove the temporary manifest
os.remove(filename)
return results
def lint(files, config, **kwargs):
tests_dir = os.path.join(kwargs['root'], 'testing', 'web-platform', 'tests')
def process_line(line):
try:
data = json.loads(line)
except ValueError:
return
data["level"] = "error"
data["path"] = os.path.relpath(os.path.join(tests_dir, data["path"]), kwargs['root'])
results.append(result.from_config(config, **data))
cmd = [os.path.join(tests_dir, 'wpt'), 'lint', '--json'] + files
if platform.system() == 'Windows':
cmd.insert(0, sys.executable)
proc = ProcessHandler(cmd, env=os.environ, processOutputLine=process_line)
proc.run()
try:
proc.wait()
except KeyboardInterrupt:
proc.kill()
return results
def test_run(install_mozproxy):
build = install_mozproxy
output_handler = OutputHandler()
p = ProcessHandler(
["mozproxy",
"--local",
"--binary=firefox",
"--topsrcdir=" + build.topsrcdir,
"--objdir=" + build.topobjdir,
os.path.join(here, "example.dump")],
processOutputLine=output_handler,
onFinish=output_handler.finished,
)
p.run()
# The first time we run mozproxy, we need to fetch mitmproxy, which can
# take a while...
assert output_handler.port_event.wait(120) is True
# Give mitmproxy a bit of time to start up so we can verify that it's
# actually running before we kill mozproxy.
time.sleep(5)
_kill_mozproxy(p.pid)
assert p.wait(10) == 0
assert output_handler.port is not None
def verify_android_device(build_obj, install=False, xre=False, debugger=False):
"""
Determine if any Android device is connected via adb.
If no device is found, prompt to start an emulator.
If a device is found or an emulator started and 'install' is
specified, also check whether Firefox is installed on the
device; if not, prompt to install Firefox.
If 'xre' is specified, also check with MOZ_HOST_BIN is set
to a valid xre/host-utils directory; if not, prompt to set
one up.
If 'debugger' is specified, also check that JimDB is installed;
if JimDB is not found, prompt to set up JimDB.
Returns True if the emulator was started or another device was
already connected.
"""
device_verified = False
emulator = AndroidEmulator('*', substs=build_obj.substs)
devices = emulator.dm.devices()
if (len(devices) > 0) and ('device' in [d[1] for d in devices]):
device_verified = True
elif emulator.is_available():
response = raw_input(
"No Android devices connected. Start an emulator? (Y/n) ").strip()
if response.lower().startswith('y') or response == '':
if not emulator.check_avd():
_log_info("Fetching AVD. This may take a while...")
emulator.update_avd()
_log_info("Starting emulator running %s..." %
emulator.get_avd_description())
emulator.start()
emulator.wait_for_start()
device_verified = True
if device_verified and install:
# Determine if Firefox is installed on the device; if not,
# prompt to install. This feature allows a test command to
# launch an emulator, install Firefox, and proceed with testing
# in one operation. It is also a basic safeguard against other
# cases where testing is requested but Firefox installation has
# been forgotten.
# If Firefox is installed, there is no way to determine whether
# the current build is installed, and certainly no way to
# determine if the installed build is the desired build.
# Installing every time is problematic because:
# - it prevents testing against other builds (downloaded apk)
# - installation may take a couple of minutes.
installed = emulator.dm.shellCheckOutput(
['pm', 'list', 'packages', 'org.mozilla.'])
if not 'fennec' in installed and not 'firefox' in installed:
response = raw_input(
"It looks like Firefox is not installed on this device.\n"
"Install Firefox? (Y/n) ").strip()
if response.lower().startswith('y') or response == '':
_log_info("Installing Firefox. This may take a while...")
build_obj._run_make(directory=".",
target='install',
ensure_exit_code=False)
if device_verified and xre:
# Check whether MOZ_HOST_BIN has been set to a valid xre; if not,
# prompt to install one.
xre_path = os.environ.get('MOZ_HOST_BIN')
err = None
if not xre_path:
err = 'environment variable MOZ_HOST_BIN is not set to a directory containing host xpcshell'
elif not os.path.isdir(xre_path):
err = '$MOZ_HOST_BIN does not specify a directory'
elif not os.path.isfile(os.path.join(xre_path, 'xpcshell')):
err = '$MOZ_HOST_BIN/xpcshell does not exist'
if err:
xre_path = glob.glob(os.path.join(EMULATOR_HOME_DIR,
'host-utils*'))
for path in xre_path:
if os.path.isdir(path) and os.path.isfile(
os.path.join(path, 'xpcshell')):
os.environ['MOZ_HOST_BIN'] = path
err = None
break
if err:
_log_info("Host utilities not found: %s" % err)
response = raw_input(
"Download and setup your host utilities? (Y/n) ").strip()
if response.lower().startswith('y') or response == '':
_log_info(
"Installing host utilities. This may take a while...")
_download_file(TOOLTOOL_URL, 'tooltool.py', EMULATOR_HOME_DIR)
host_platform = _get_host_platform()
if host_platform:
path = os.path.join(MANIFEST_PATH, host_platform,
'hostutils.manifest')
_get_tooltool_manifest(build_obj.substs, path,
EMULATOR_HOME_DIR,
'releng.manifest')
_tooltool_fetch()
xre_path = glob.glob(
os.path.join(EMULATOR_HOME_DIR, 'host-utils*'))
for path in xre_path:
if os.path.isdir(path) and os.path.isfile(
os.path.join(path, 'xpcshell')):
os.environ['MOZ_HOST_BIN'] = path
err = None
break
if err:
_log_warning("Unable to install host utilities.")
else:
_log_warning(
"Unable to install host utilities -- your platform is not supported!"
)
if debugger:
# Optionally set up JimDB. See https://wiki.mozilla.org/Mobile/Fennec/Android/GDB.
build_platform = _get_build_platform(build_obj.substs)
jimdb_path = os.path.join(EMULATOR_HOME_DIR,
'jimdb-%s' % build_platform)
jimdb_utils_path = os.path.join(jimdb_path, 'utils')
gdb_path = os.path.join(jimdb_path, 'bin', 'gdb')
err = None
if not os.path.isdir(jimdb_path):
err = '%s does not exist' % jimdb_path
elif not os.path.isfile(gdb_path):
err = '%s not found' % gdb_path
if err:
_log_info("JimDB (%s) not found: %s" % (build_platform, err))
response = raw_input("Download and setup JimDB (%s)? (Y/n) " %
build_platform).strip()
if response.lower().startswith('y') or response == '':
host_platform = _get_host_platform()
if host_platform:
_log_info(
"Installing JimDB (%s/%s). This may take a while..." %
(host_platform, build_platform))
path = os.path.join(MANIFEST_PATH, host_platform,
'jimdb-%s.manifest' % build_platform)
_get_tooltool_manifest(build_obj.substs, path,
EMULATOR_HOME_DIR,
'releng.manifest')
_tooltool_fetch()
if os.path.isfile(gdb_path):
# Get JimDB utilities from git repository
proc = ProcessHandler(['git', 'pull'],
cwd=jimdb_utils_path)
proc.run()
git_pull_complete = False
try:
proc.wait()
if proc.proc.returncode == 0:
git_pull_complete = True
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
if not git_pull_complete:
_log_warning(
"Unable to update JimDB utils from git -- some JimDB features may be unavailable."
)
else:
_log_warning(
"Unable to install JimDB -- unable to fetch from tooltool."
)
else:
_log_warning(
"Unable to install JimDB -- your platform is not supported!"
)
if os.path.isfile(gdb_path):
# sync gdbinit.local with build settings
_update_gdbinit(build_obj.substs,
os.path.join(jimdb_utils_path, "gdbinit.local"))
# ensure JimDB is in system path, so that mozdebug can find it
bin_path = os.path.join(jimdb_path, 'bin')
os.environ['PATH'] = "%s:%s" % (bin_path, os.environ['PATH'])
return device_verified
class Mitmproxy(Playback):
def __init__(self, config):
self.config = config
self.host = ("127.0.0.1" if "localhost" in self.config["host"] else
self.config["host"])
self.port = None
self.mitmproxy_proc = None
self.mitmdump_path = None
self.browser_path = ""
if config.get("binary", None):
self.browser_path = os.path.normpath(config.get("binary"))
self.policies_dir = None
self.ignore_mitmdump_exit_failure = config.get(
"ignore_mitmdump_exit_failure", False)
self.recording_paths = None
if self.config.get("playback_version") is None:
LOG.info("mitmproxy was not provided with a 'playback_version' "
"Using default playback version: 4.0.4")
self.config["playback_version"] = "4.0.4"
if self.config.get("playback_binary_manifest") is None:
LOG.info(
"mitmproxy was not provided with a 'playback_binary_manifest' "
"Using default playback_binary_manifest")
self.config["playback_binary_manifest"] = (
"mitmproxy-rel-bin-%s-{platform}.manifest" %
self.config["playback_version"])
# mozproxy_dir is where we will download all mitmproxy required files
# when running locally it comes from obj_path via mozharness/mach
if self.config.get("obj_path") is not None:
self.mozproxy_dir = self.config.get("obj_path")
else:
# in production it is ../tasks/task_N/build/, in production that dir
# is not available as an envvar, however MOZ_UPLOAD_DIR is set as
# ../tasks/task_N/build/blobber_upload_dir so take that and go up 1 level
self.mozproxy_dir = os.path.dirname(
os.path.dirname(os.environ["MOZ_UPLOAD_DIR"]))
self.mozproxy_dir = os.path.join(self.mozproxy_dir, "testing",
"mozproxy")
self.upload_dir = os.environ.get("MOZ_UPLOAD_DIR", self.mozproxy_dir)
LOG.info(
"mozproxy_dir used for mitmproxy downloads and exe files: %s" %
self.mozproxy_dir)
# setting up the MOZPROXY_DIR env variable so custom scripts know
# where to get the data
os.environ["MOZPROXY_DIR"] = self.mozproxy_dir
LOG.info("Playback tool: %s" % self.config["playback_tool"])
LOG.info("Playback tool version: %s" % self.config["playback_version"])
def start(self):
# go ahead and download and setup mitmproxy
self.download()
# mitmproxy must be started before setup, so that the CA cert is available
self.start_mitmproxy_playback(self.mitmdump_path, self.browser_path)
# In case the setup fails, we want to stop the process before raising.
try:
self.setup()
except Exception:
self.stop()
raise
def download(self):
"""Download and unpack mitmproxy binary and pageset using tooltool"""
if not os.path.exists(self.mozproxy_dir):
os.makedirs(self.mozproxy_dir)
_manifest = os.path.join(here, self.config["playback_binary_manifest"])
transformed_manifest = transform_platform(_manifest,
self.config["platform"])
# generate the mitmdump_path
self.mitmdump_path = os.path.normpath(
os.path.join(
self.mozproxy_dir,
"mitmdump-%s" % self.config["playback_version"],
"mitmdump",
))
# Check if mitmproxy bin exists
if os.path.exists(self.mitmdump_path):
LOG.info("mitmproxy binary already exists. Skipping download")
else:
# Download and unpack mitmproxy binary
download_path = os.path.dirname(self.mitmdump_path)
LOG.info("create mitmproxy %s dir" %
self.config["playback_version"])
if not os.path.exists(download_path):
os.makedirs(download_path)
LOG.info("downloading mitmproxy binary")
tooltool_download(transformed_manifest, self.config["run_local"],
download_path)
if "playback_pageset_manifest" in self.config:
# we use one pageset for all platforms
LOG.info("downloading mitmproxy pageset")
_manifest = self.config["playback_pageset_manifest"]
transformed_manifest = transform_platform(_manifest,
self.config["platform"])
tooltool_download(transformed_manifest, self.config["run_local"],
self.mozproxy_dir)
if "playback_artifacts" in self.config:
artifacts = self.config["playback_artifacts"].split(",")
for artifact in artifacts:
artifact = artifact.strip()
if not artifact:
continue
artifact_name = artifact.split("/")[-1]
if artifact_name.endswith(".manifest"):
tooltool_download(artifact, self.config["run_local"],
self.mozproxy_dir)
else:
dest = os.path.join(self.mozproxy_dir, artifact_name)
download_file_from_url(artifact, dest, extract=True)
def stop(self):
self.stop_mitmproxy_playback()
def start_mitmproxy_playback(self, mitmdump_path, browser_path):
"""Startup mitmproxy and replay the specified flow file"""
if self.mitmproxy_proc is not None:
raise Exception("Proxy already started.")
self.port = get_available_port()
LOG.info("mitmdump path: %s" % mitmdump_path)
LOG.info("browser path: %s" % browser_path)
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
env["PATH"] = os.path.dirname(browser_path) + os.pathsep + env["PATH"]
command = [mitmdump_path]
# add proxy host and port options
command.extend(
["--listen-host", self.host, "--listen-port",
str(self.port)])
if "playback_tool_args" in self.config:
LOG.info("Staring Proxy using provided command line!")
command.extend(self.config["playback_tool_args"])
elif "playback_files" in self.config:
script = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"scripts",
"alternate-server-replay.py",
)
self.recording_paths = [
normalize_path(recording_path)
for recording_path in self.config["playback_files"]
]
if self.config["playback_version"] in ["4.0.4", "5.0.1"]:
args = [
"-v",
"--set",
"upstream_cert=false",
"--set",
"upload_dir=" + normalize_path(self.upload_dir),
"--set",
"websocket=false",
"--set",
"server_replay_files={}".format(",".join(
self.recording_paths)),
"--scripts",
normalize_path(script),
]
command.extend(args)
else:
raise Exception("Mitmproxy version is unknown!")
else:
raise Exception(
"Mitmproxy can't start playback! Playback settings missing.")
LOG.info("Starting mitmproxy playback using env path: %s" %
env["PATH"])
LOG.info("Starting mitmproxy playback using command: %s" %
" ".join(command))
# to turn off mitmproxy log output, use these params for Popen:
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
self.mitmproxy_proc = ProcessHandler(
command,
logfile=os.path.join(self.upload_dir, "mitmproxy.log"),
env=env,
processStderrLine=LOG.error,
storeOutput=False,
)
self.mitmproxy_proc.run()
end_time = time.time() + MITMDUMP_COMMAND_TIMEOUT
ready = False
while time.time() < end_time:
ready = self.check_proxy(host=self.host, port=self.port)
if ready:
LOG.info(
"Mitmproxy playback successfully started on %s:%d as pid %d"
% (self.host, self.port, self.mitmproxy_proc.pid))
return
time.sleep(0.25)
# cannot continue as we won't be able to playback the pages
LOG.error("Aborting: Mitmproxy process did not startup")
self.stop_mitmproxy_playback()
sys.exit() # XXX why do we need to do that? a raise is not enough?
def stop_mitmproxy_playback(self):
"""Stop the mitproxy server playback"""
if self.mitmproxy_proc is None or self.mitmproxy_proc.poll(
) is not None:
return
LOG.info("Stopping mitmproxy playback, killing process %d" %
self.mitmproxy_proc.pid)
# On Windows, mozprocess brutally kills mitmproxy with TerminateJobObject
# The process has no chance to gracefully shutdown.
# Here, we send the process a break event to give it a chance to wrapup.
# See the signal handler in the alternate-server-replay-4.0.4.py script
if mozinfo.os == "win":
LOG.info("Sending CTRL_BREAK_EVENT to mitmproxy")
os.kill(self.mitmproxy_proc.pid, signal.CTRL_BREAK_EVENT)
time.sleep(2)
exit_code = self.mitmproxy_proc.kill()
self.mitmproxy_proc = None
if exit_code != 0:
if exit_code is None:
LOG.error("Failed to kill the mitmproxy playback process")
return
if mozinfo.os == "win":
from mozprocess.winprocess import ERROR_CONTROL_C_EXIT # noqa
if exit_code == ERROR_CONTROL_C_EXIT:
LOG.info(
"Successfully killed the mitmproxy playback process"
" with exit code %d" % exit_code)
return
log_func = LOG.error
if self.ignore_mitmdump_exit_failure:
log_func = LOG.info
log_func("Mitmproxy exited with error code %d" % exit_code)
else:
LOG.info("Successfully killed the mitmproxy playback process")
def check_proxy(self, host, port):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((host, port))
s.shutdown(socket.SHUT_RDWR)
s.close()
return True
except socket.error:
return False
def confidence(self):
file_name = "mitm_netlocs_%s.json" % os.path.splitext(
os.path.basename(self.recording_paths[0]))[0]
path = os.path.normpath(os.path.join(self.upload_dir, file_name))
if os.path.exists(path):
try:
LOG.info("Reading confidence values from: %s" % path)
with open(path, "r") as f:
data = json.load(f)
return {
"confidence": data["confidence"],
"not-replayed": data["not-replayed"],
"replayed": data["replayed"]
}
except Exception:
LOG.info("Can't read netlocs file!", exc_info=True)
return None
else:
LOG.info("Netlocs file is not available! Cant find %s" % path)
return None
class Mitmproxy(Playback):
def __init__(self, config):
self.config = config
self.host = ("127.0.0.1" if "localhost" in self.config["host"] else
self.config["host"])
self.port = None
self.mitmproxy_proc = None
self.mitmdump_path = None
self.record_mode = config.get("record", False)
self.recording = None
self.playback_files = []
self.browser_path = ""
if config.get("binary", None):
self.browser_path = os.path.normpath(config.get("binary"))
self.policies_dir = None
self.ignore_mitmdump_exit_failure = config.get(
"ignore_mitmdump_exit_failure", False)
if self.record_mode:
if "recording_file" not in self.config:
LOG.error(
"recording_file value was not provided. Proxy service wont' start "
)
raise Exception("Please provide a playback_files list.")
if not isinstance(self.config.get("recording_file"),
six.string_types):
LOG.error("recording_file argument type is not str!")
raise Exception("recording_file argument type invalid!")
if not os.path.splitext(
self.config.get("recording_file"))[1] == ".zip":
LOG.error("Recording file type (%s) should be a zip. "
"Please provide a valid file type!" %
self.config.get("recording_file"))
raise Exception("Recording file type should be a zip")
if os.path.exists(self.config.get("recording_file")):
LOG.error("Recording file (%s) already exists."
"Please provide a valid file path!" %
self.config.get("recording_file"))
raise Exception("Recording file already exists.")
if self.config.get("playback_files", False):
LOG.error(
"Record mode is True and playback_files where provided!")
raise Exception("playback_files specified during record!")
if self.config.get("playback_version") is None:
LOG.error("mitmproxy was not provided with a 'playback_version' "
"Please provide a valid playback version")
raise Exception("playback_version not specified!")
# mozproxy_dir is where we will download all mitmproxy required files
# when running locally it comes from obj_path via mozharness/mach
if self.config.get("obj_path") is not None:
self.mozproxy_dir = self.config.get("obj_path")
else:
# in production it is ../tasks/task_N/build/, in production that dir
# is not available as an envvar, however MOZ_UPLOAD_DIR is set as
# ../tasks/task_N/build/blobber_upload_dir so take that and go up 1 level
self.mozproxy_dir = os.path.dirname(
os.path.dirname(os.environ["MOZ_UPLOAD_DIR"]))
self.mozproxy_dir = os.path.join(self.mozproxy_dir, "testing",
"mozproxy")
self.upload_dir = os.environ.get("MOZ_UPLOAD_DIR", self.mozproxy_dir)
LOG.info(
"mozproxy_dir used for mitmproxy downloads and exe files: %s" %
self.mozproxy_dir)
# setting up the MOZPROXY_DIR env variable so custom scripts know
# where to get the data
os.environ["MOZPROXY_DIR"] = self.mozproxy_dir
LOG.info("Playback tool: %s" % self.config["playback_tool"])
LOG.info("Playback tool version: %s" % self.config["playback_version"])
def download_mitm_bin(self):
# Download and setup mitm binaries
manifest = os.path.join(
here,
"manifests",
"mitmproxy-rel-bin-%s-{platform}.manifest" %
self.config["playback_version"],
)
transformed_manifest = transform_platform(manifest,
self.config["platform"])
# generate the mitmdump_path
self.mitmdump_path = os.path.normpath(
os.path.join(
self.mozproxy_dir,
"mitmdump-%s" % self.config["playback_version"],
"mitmdump",
))
# Check if mitmproxy bin exists
if os.path.exists(self.mitmdump_path):
LOG.info("mitmproxy binary already exists. Skipping download")
else:
# Download and unpack mitmproxy binary
download_path = os.path.dirname(self.mitmdump_path)
LOG.info("create mitmproxy %s dir" %
self.config["playback_version"])
if not os.path.exists(download_path):
os.makedirs(download_path)
LOG.info("downloading mitmproxy binary")
tooltool_download(transformed_manifest, self.config["run_local"],
download_path)
def download_manifest_file(self, manifest_path):
# Manifest File
# we use one pageset for all platforms
LOG.info("downloading mitmproxy pageset")
tooltool_download(manifest_path, self.config["run_local"],
self.mozproxy_dir)
with open(manifest_path) as manifest_file:
manifest = json.load(manifest_file)
for file in manifest:
zip_path = os.path.join(self.mozproxy_dir, file["filename"])
LOG.info("Adding %s to recording list" % zip_path)
self.playback_files.append(RecordingFile(zip_path))
def download_playback_files(self):
# Detect type of file from playback_files and download accordingly
if "playback_files" not in self.config:
LOG.error(
"playback_files value was not provided. Proxy service wont' start "
)
raise Exception("Please provide a playback_files list.")
if not isinstance(self.config["playback_files"], list):
LOG.error("playback_files should be a list")
raise Exception("playback_files should be a list")
for playback_file in self.config["playback_files"]:
if playback_file.startswith(
"https://") and "mozilla.com" in playback_file:
# URL provided
dest = os.path.join(self.mozproxy_dir,
os.path.basename(playback_file))
download_file_from_url(playback_file,
self.mozproxy_dir,
extract=False)
# Add Downloaded file to playback_files list
LOG.info("Adding %s to recording list" % dest)
self.playback_files.append(RecordingFile(dest))
continue
if not os.path.exists(playback_file):
LOG.error(
"Zip or manifest file path (%s) does not exist. Please provide a valid path!"
% playback_file)
raise Exception("Zip or manifest file path does not exist")
if os.path.splitext(playback_file)[1] == ".zip":
# zip file path provided
LOG.info("Adding %s to recording list" % playback_file)
self.playback_files.append(RecordingFile(playback_file))
elif os.path.splitext(playback_file)[1] == ".manifest":
# manifest file path provided
self.download_manifest_file(playback_file)
def download(self):
"""Download and unpack mitmproxy binary and pageset using tooltool"""
if not os.path.exists(self.mozproxy_dir):
os.makedirs(self.mozproxy_dir)
self.download_mitm_bin()
if self.record_mode:
self.recording = RecordingFile(self.config["recording_file"])
else:
self.download_playback_files()
def stop(self):
LOG.info("Mitmproxy stop!!")
self.stop_mitmproxy_playback()
if self.record_mode:
LOG.info("Record mode ON. Generating zip file ")
self.recording.generate_zip_file()
def wait(self, timeout=1):
"""Wait until the mitmproxy process has terminated."""
# We wait using this method to allow Windows to respond to the Ctrl+Break
# signal so that we can exit cleanly from the command-line driver.
while True:
returncode = self.mitmproxy_proc.wait(timeout)
if returncode is not None:
return returncode
def start(self):
# go ahead and download and setup mitmproxy
self.download()
# mitmproxy must be started before setup, so that the CA cert is available
self.start_mitmproxy(self.mitmdump_path, self.browser_path)
# In case the setup fails, we want to stop the process before raising.
try:
self.setup()
except Exception:
try:
self.stop()
except Exception:
LOG.error("MitmProxy failed to STOP.", exc_info=True)
LOG.error("Setup of MitmProxy failed.", exc_info=True)
raise
def start_mitmproxy(self, mitmdump_path, browser_path):
"""Startup mitmproxy and replay the specified flow file"""
if self.mitmproxy_proc is not None:
raise Exception("Proxy already started.")
self.port = get_available_port()
LOG.info("mitmdump path: %s" % mitmdump_path)
LOG.info("browser path: %s" % browser_path)
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
env["PATH"] = os.path.dirname(browser_path) + os.pathsep + env["PATH"]
command = [mitmdump_path]
if self.config.get("verbose", False):
# Generate mitmproxy verbose logs
command.extend(["-v"])
# add proxy host and port options
command.extend(
["--listen-host", self.host, "--listen-port",
str(self.port)])
# record mode
if self.record_mode:
# generate recording script paths
inject_deterministic = os.path.join(
mitm_folder,
"scripts",
"inject-deterministic.py",
)
http_protocol_extractor = os.path.join(
mitm_folder,
"scripts",
"http_protocol_extractor.py",
)
args = [
"--save-stream-file",
normalize_path(self.recording.recording_path),
"--set",
"websocket=false",
"--scripts",
inject_deterministic,
"--scripts",
http_protocol_extractor,
]
command.extend(args)
self.recording.set_metadata("proxy_version",
self.config["playback_version"])
else:
# playback mode
if len(self.playback_files) > 0:
script = os.path.join(
mitm_folder,
"scripts",
"alternate-server-replay.py",
)
if self.config["playback_version"] in [
"4.0.4", "5.1.1", "6.0.2"
]:
args = [
"--set",
"upstream_cert=false",
"--set",
"upload_dir=" + normalize_path(self.upload_dir),
"--set",
"websocket=false",
"--set",
"server_replay_files={}".format(",".join([
normalize_path(playback_file.recording_path)
for playback_file in self.playback_files
])),
"--scripts",
normalize_path(script),
]
command.extend(args)
else:
raise Exception("Mitmproxy version is unknown!")
else:
raise Exception(
"Mitmproxy can't start playback! Playback settings missing."
)
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
if not os.path.dirname(self.browser_path) in env["PATH"]:
env["PATH"] = os.path.dirname(
self.browser_path) + os.pathsep + env["PATH"]
LOG.info("Starting mitmproxy playback using env path: %s" %
env["PATH"])
LOG.info("Starting mitmproxy playback using command: %s" %
" ".join(command))
# to turn off mitmproxy log output, use these params for Popen:
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
self.mitmproxy_proc = ProcessHandler(
command,
logfile=os.path.join(self.upload_dir, "mitmproxy.log"),
env=env,
processStderrLine=LOG.error,
storeOutput=False,
)
self.mitmproxy_proc.run()
end_time = time.time() + MITMDUMP_COMMAND_TIMEOUT
ready = False
while time.time() < end_time:
ready = self.check_proxy(host=self.host, port=self.port)
if ready:
LOG.info(
"Mitmproxy playback successfully started on %s:%d as pid %d"
% (self.host, self.port, self.mitmproxy_proc.pid))
return
time.sleep(0.25)
# cannot continue as we won't be able to playback the pages
LOG.error("Aborting: Mitmproxy process did not startup")
self.stop_mitmproxy_playback()
sys.exit(1) # XXX why do we need to do that? a raise is not enough?
def stop_mitmproxy_playback(self):
"""Stop the mitproxy server playback"""
if self.mitmproxy_proc is None or self.mitmproxy_proc.poll(
) is not None:
return
LOG.info("Stopping mitmproxy playback, killing process %d" %
self.mitmproxy_proc.pid)
# On Windows, mozprocess brutally kills mitmproxy with TerminateJobObject
# The process has no chance to gracefully shutdown.
# Here, we send the process a break event to give it a chance to wrapup.
# See the signal handler in the alternate-server-replay-4.0.4.py script
if mozinfo.os == "win":
LOG.info("Sending CTRL_BREAK_EVENT to mitmproxy")
os.kill(self.mitmproxy_proc.pid, signal.CTRL_BREAK_EVENT)
time.sleep(2)
exit_code = self.mitmproxy_proc.kill()
self.mitmproxy_proc = None
if exit_code != 0:
if exit_code is None:
LOG.error("Failed to kill the mitmproxy playback process")
return
if mozinfo.os == "win":
from mozprocess.winprocess import ERROR_CONTROL_C_EXIT # noqa
if exit_code == ERROR_CONTROL_C_EXIT:
LOG.info(
"Successfully killed the mitmproxy playback process"
" with exit code %d" % exit_code)
return
log_func = LOG.error
if self.ignore_mitmdump_exit_failure:
log_func = LOG.info
log_func("Mitmproxy exited with error code %d" % exit_code)
else:
LOG.info("Successfully killed the mitmproxy playback process")
def check_proxy(self, host, port):
"""Check that mitmproxy process is working by doing a socket call using the proxy settings
:param host: Host of the proxy server
:param port: Port of the proxy server
:return: True if the proxy service is working
"""
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((host, port))
s.shutdown(socket.SHUT_RDWR)
s.close()
return True
except socket.error:
return False
def _run_python_test(self, test):
from mozprocess import ProcessHandler
if test.get('requirements'):
self.virtualenv_manager.install_pip_requirements(
test['requirements'], quiet=True)
output = []
def _log(line):
# Buffer messages if more than one worker to avoid interleaving
if self.jobs > 1:
output.append(line)
else:
self.log(logging.INFO, 'python-test', {'line': line.rstrip()},
'{line}')
file_displayed_test = [] # used as boolean
def _line_handler(line):
if not file_displayed_test:
output = ('Ran' in line or 'collected' in line
or line.startswith('TEST-'))
if output:
file_displayed_test.append(True)
# Hack to make sure treeherder highlights pytest failures
if b'FAILED' in line.rsplit(b' ', 1)[-1]:
line = line.replace(b'FAILED', b'TEST-UNEXPECTED-FAIL')
_log(line)
_log(test['path'])
python = self.virtualenv_manager.python_path
cmd = [python, test['path']]
env = os.environ.copy()
env[b'PYTHONDONTWRITEBYTECODE'] = b'1'
# Homebrew on OS X will change Python's sys.executable to a custom value
# which messes with mach's virtualenv handling code. Override Homebrew's
# changes with the correct sys.executable value.
env[b'PYTHONEXECUTABLE'] = python.encode('utf-8')
proc = ProcessHandler(cmd,
env=env,
processOutputLine=_line_handler,
storeOutput=False)
proc.run()
return_code = proc.wait()
if not file_displayed_test:
_log(
'TEST-UNEXPECTED-FAIL | No test output (missing mozunit.main() '
'call?): {}'.format(test['path']))
if self.verbose:
if return_code != 0:
_log('Test failed: {}'.format(test['path']))
else:
_log('Test passed: {}'.format(test['path']))
return output, return_code, test['path']
class ServoTestharnessExecutor(ProcessTestExecutor):
convert_result = testharness_result_converter
def __init__(self,
browser,
server_config,
timeout_multiplier=1,
debug_info=None,
pause_after_test=False,
**kwargs):
ProcessTestExecutor.__init__(self,
browser,
server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.pause_after_test = pause_after_test
self.result_data = None
self.result_flag = None
self.protocol = ConnectionlessProtocol(self, browser)
self.hosts_path = write_hosts_file(server_config)
def teardown(self):
try:
os.unlink(self.hosts_path)
except OSError:
pass
ProcessTestExecutor.teardown(self)
def do_test(self, test):
self.result_data = None
self.result_flag = threading.Event()
args = [
"--hard-fail",
"-u",
"Servo/wptrunner",
"-Z",
"replace-surrogates",
"-z",
self.test_url(test),
]
for stylesheet in self.browser.user_stylesheets:
args += ["--user-stylesheet", stylesheet]
for pref, value in test.environment.get('prefs', {}).iteritems():
args += ["--pref", "%s=%s" % (pref, value)]
if self.browser.ca_certificate_path:
args += ["--certificate-path", self.browser.ca_certificate_path]
args += self.browser.binary_args
debug_args, command = browser_command(self.binary, args,
self.debug_info)
self.command = command
if self.pause_after_test:
self.command.remove("-z")
self.command = debug_args + self.command
env = os.environ.copy()
env["HOST_FILE"] = self.hosts_path
env["RUST_BACKTRACE"] = "1"
if not self.interactive:
self.proc = ProcessHandler(self.command,
processOutputLine=[self.on_output],
onFinish=self.on_finish,
env=env,
storeOutput=False)
self.proc.run()
else:
self.proc = subprocess.Popen(self.command, env=env)
try:
timeout = test.timeout * self.timeout_multiplier
# Now wait to get the output we expect, or until we reach the timeout
if not self.interactive and not self.pause_after_test:
wait_timeout = timeout + 5
self.result_flag.wait(wait_timeout)
else:
wait_timeout = None
self.proc.wait()
proc_is_running = True
if self.result_flag.is_set():
if self.result_data is not None:
result = self.convert_result(test, self.result_data)
else:
self.proc.wait()
result = (test.result_cls("CRASH", None), [])
proc_is_running = False
else:
result = (test.result_cls("TIMEOUT", None), [])
if proc_is_running:
if self.pause_after_test:
self.logger.info("Pausing until the browser exits")
self.proc.wait()
else:
self.proc.kill()
except: # noqa
self.proc.kill()
raise
return result
def on_output(self, line):
prefix = "ALERT: RESULT: "
line = line.decode("utf8", "replace")
if line.startswith(prefix):
self.result_data = json.loads(line[len(prefix):])
self.result_flag.set()
else:
if self.interactive:
print(line)
else:
self.logger.process_output(self.proc.pid, line,
" ".join(self.command))
def on_finish(self):
self.result_flag.set()
class ServoRefTestExecutor(ProcessTestExecutor):
convert_result = reftest_result_converter
def __init__(self,
browser,
server_config,
binary=None,
timeout_multiplier=1,
screenshot_cache=None,
debug_info=None,
pause_after_test=False,
**kwargs):
ProcessTestExecutor.__init__(self,
browser,
server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.protocol = ConnectionlessProtocol(self, browser)
self.screenshot_cache = screenshot_cache
self.implementation = RefTestImplementation(self)
self.tempdir = tempfile.mkdtemp()
self.hosts_path = write_hosts_file(server_config)
def teardown(self):
try:
os.unlink(self.hosts_path)
except OSError:
pass
os.rmdir(self.tempdir)
ProcessTestExecutor.teardown(self)
def screenshot(self, test, viewport_size, dpi):
full_url = self.test_url(test)
with TempFilename(self.tempdir) as output_path:
debug_args, command = browser_command(self.binary, [
"--hard-fail", "--exit", "-u", "Servo/wptrunner", "-Z",
"disable-text-aa,load-webfonts-synchronously,replace-surrogates",
"--output=%s" % output_path, full_url
] + self.browser.binary_args, self.debug_info)
for stylesheet in self.browser.user_stylesheets:
command += ["--user-stylesheet", stylesheet]
for pref, value in test.environment.get('prefs', {}).iteritems():
command += ["--pref", "%s=%s" % (pref, value)]
command += ["--resolution", viewport_size or "800x600"]
if self.browser.ca_certificate_path:
command += [
"--certificate-path", self.browser.ca_certificate_path
]
if dpi:
command += ["--device-pixel-ratio", dpi]
# Run ref tests in headless mode
command += ["-z"]
self.command = debug_args + command
env = os.environ.copy()
env["HOST_FILE"] = self.hosts_path
env["RUST_BACKTRACE"] = "1"
if not self.interactive:
self.proc = ProcessHandler(self.command,
processOutputLine=[self.on_output],
env=env)
try:
self.proc.run()
timeout = test.timeout * self.timeout_multiplier + 5
rv = self.proc.wait(timeout=timeout)
except KeyboardInterrupt:
self.proc.kill()
raise
else:
self.proc = subprocess.Popen(self.command, env=env)
try:
rv = self.proc.wait()
except KeyboardInterrupt:
self.proc.kill()
raise
if rv is None:
self.proc.kill()
return False, ("EXTERNAL-TIMEOUT", None)
if rv != 0 or not os.path.exists(output_path):
return False, ("CRASH", None)
with open(output_path) as f:
# Might need to strip variable headers or something here
data = f.read()
return True, base64.b64encode(data)
def do_test(self, test):
result = self.implementation.run_test(test)
return self.convert_result(test, result)
def on_output(self, line):
line = line.decode("utf8", "replace")
if self.interactive:
print(line)
else:
self.logger.process_output(self.proc.pid, line,
" ".join(self.command))
def run_browser(command,
minidump_dir,
timeout=None,
on_started=None,
debug=None,
debugger=None,
debugger_args=None,
**kwargs):
"""
Run the browser using the given `command`.
After the browser prints __endTimestamp, we give it 5
seconds to quit and kill it if it's still alive at that point.
Note that this method ensure that the process is killed at
the end. If this is not possible, an exception will be raised.
:param command: the commad (as a string list) to run the browser
:param minidump_dir: a path where to extract minidumps in case the
browser hang. This have to be the same value
used in `mozcrash.check_for_crashes`.
:param timeout: if specified, timeout to wait for the browser before
we raise a :class:`TalosError`
:param on_started: a callback that can be used to do things just after
the browser has been started. The callback must takes
an argument, which is the psutil.Process instance
:param kwargs: additional keyword arguments for the :class:`ProcessHandler`
instance
Returns a ProcessContext instance, with available output and pid used.
"""
debugger_info = find_debugger_info(debug, debugger, debugger_args)
if debugger_info is not None:
return run_in_debug_mode(command,
debugger_info,
on_started=on_started,
env=kwargs.get('env'))
is_launcher = sys.platform.startswith(
'win') and '-wait-for-browser' in command
context = ProcessContext(is_launcher)
first_time = int(time.time()) * 1000
wait_for_quit_timeout = 20
event = Event()
reader = Reader(event)
LOG.info("Using env: %s" % pprint.pformat(kwargs['env']))
kwargs['storeOutput'] = False
kwargs['processOutputLine'] = reader
kwargs['onFinish'] = event.set
proc = ProcessHandler(command, **kwargs)
reader.proc = proc
proc.run()
LOG.process_start(proc.pid, ' '.join(command))
try:
context.process = psutil.Process(proc.pid)
if on_started:
on_started(context.process)
# wait until we saw __endTimestamp in the proc output,
# or the browser just terminated - or we have a timeout
if not event.wait(timeout):
LOG.info("Timeout waiting for test completion; killing browser...")
# try to extract the minidump stack if the browser hangs
kill_and_get_minidump(context, minidump_dir)
raise TalosError("timeout")
if reader.got_end_timestamp:
for i in range(1, wait_for_quit_timeout):
if proc.wait(1) is not None:
break
if proc.poll() is None:
LOG.info(
"Browser shutdown timed out after {0} seconds, killing"
" process.".format(wait_for_quit_timeout))
kill_and_get_minidump(context, minidump_dir)
raise TalosError(
"Browser shutdown timed out after {0} seconds, killed"
" process.".format(wait_for_quit_timeout))
elif reader.got_timeout:
raise TalosError('TIMEOUT: %s' % reader.timeout_message)
elif reader.got_error:
raise TalosError("unexpected error")
finally:
# this also handle KeyboardInterrupt
# ensure early the process is really terminated
return_code = None
try:
return_code = context.kill_process()
if return_code is None:
return_code = proc.wait(1)
except Exception:
# Maybe killed by kill_and_get_minidump(), maybe ended?
LOG.info("Unable to kill process")
LOG.info(traceback.format_exc())
reader.output.append(
"__startBeforeLaunchTimestamp%d__endBeforeLaunchTimestamp" %
first_time)
reader.output.append(
"__startAfterTerminationTimestamp%d__endAfterTerminationTimestamp" %
(int(time.time()) * 1000))
if return_code is not None:
LOG.process_exit(proc.pid, return_code)
else:
LOG.debug("Unable to detect exit code of the process %s." % proc.pid)
context.output = reader.output
return context
class BaseEmulator(Device):
port = None
proc = None
telnet = None
def __init__(self, app_ctx, **kwargs):
self.arch = ArchContext(kwargs.pop('arch', 'arm'),
app_ctx,
binary=kwargs.pop('binary', None),
avd=kwargs.pop('avd', None))
super(BaseEmulator, self).__init__(app_ctx, **kwargs)
self.tmpdir = tempfile.mkdtemp()
# These rely on telnet
self.battery = EmulatorBattery(self)
self.geo = EmulatorGeo(self)
self.screen = EmulatorScreen(self)
@property
def args(self):
"""
Arguments to pass into the emulator binary.
"""
return [self.arch.binary]
def start(self):
"""
Starts a new emulator.
"""
if self.proc:
return
original_devices = set(self._get_online_devices())
# QEMU relies on atexit() to remove temporary files, which does not
# work since mozprocess uses SIGKILL to kill the emulator process.
# Use a customized temporary directory so we can clean it up.
os.environ['ANDROID_TMP'] = self.tmpdir
qemu_log = None
qemu_proc_args = {}
if self.logdir:
# save output from qemu to logfile
qemu_log = os.path.join(self.logdir, 'qemu.log')
if os.path.isfile(qemu_log):
self._rotate_log(qemu_log)
qemu_proc_args['logfile'] = qemu_log
else:
qemu_proc_args['processOutputLine'] = lambda line: None
self.proc = ProcessHandler(self.args, **qemu_proc_args)
self.proc.run()
devices = set(self._get_online_devices())
now = datetime.datetime.now()
while (devices - original_devices) == set([]):
time.sleep(1)
# Sometimes it takes more than 60s to launch emulator, so we
# increase timeout value to 180s. Please see bug 1143380.
if datetime.datetime.now() - now > datetime.timedelta(seconds=180):
raise TimeoutException(
'timed out waiting for emulator to start')
devices = set(self._get_online_devices())
devices = devices - original_devices
self.serial = devices.pop()
self.connect()
def _get_online_devices(self):
return [
d[0] for d in self.dm.devices() if d[1] != 'offline'
if d[0].startswith('emulator')
]
def connect(self):
"""
Connects to a running device. If no serial was specified in the
constructor, defaults to the first entry in `adb devices`.
"""
if self.connected:
return
super(BaseEmulator, self).connect()
serial = self.serial or self.dm._deviceSerial
self.port = int(serial[serial.rindex('-') + 1:])
def cleanup(self):
"""
Cleans up and kills the emulator, if it was started by mozrunner.
"""
super(BaseEmulator, self).cleanup()
if self.proc:
self.proc.kill()
self.proc = None
self.connected = False
# Remove temporary files
if os.path.isdir(self.tmpdir):
shutil.rmtree(self.tmpdir)
def _get_telnet_response(self, command=None):
output = []
assert self.telnet
if command is not None:
self.telnet.write('%s\n' % command)
while True:
line = self.telnet.read_until('\n')
output.append(line.rstrip())
if line.startswith('OK'):
return output
elif line.startswith('KO:'):
raise Exception('bad telnet response: %s' % line)
def _run_telnet(self, command):
if not self.telnet:
self.telnet = Telnet('localhost', self.port)
self._get_telnet_response()
return self._get_telnet_response(command)
def __del__(self):
if self.telnet:
self.telnet.write('exit\n')
self.telnet.read_all()
