def test_relative_path(self):
tempdir = tempfile.mkdtemp()
# make a dummy profile
profile = FirefoxProfile(os.path.join(tempdir, 'testprofilepath'),
restore=False)
self.assertTrue(
os.path.exists(os.path.join(tempdir, 'testprofilepath',
'user.js')))
# make a dummy test
test = """function test() { };"""
f = file(os.path.join(tempdir, 'test_dummy.js'), 'w')
f.write(test)
f.close()
# run mozmill on it
process = ProcessHandler(
['mozmill', '-t', 'test_dummy.js', '--profile=testprofilepath'],
cwd=tempdir,
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertNotEqual(process.proc.poll(), None)
# cleanup
shutil.rmtree(tempdir)
def _run_profile(self):
command_args = utils.GenerateBrowserCommandLine(
self.browser_config["browser_path"],
self.browser_config["extra_args"], self.profile_dir,
self.browser_config["init_url"])
def browser_log(line):
logging.debug('BROWSER_OUTPUT: %s', line)
browser = ProcessHandler(command_args,
env=self.env,
processOutputLine=browser_log)
browser.run()
try:
browser.wait()
except KeyboardInterrupt:
browser.kill()
raise
results_raw = '\n'.join(browser.output)
if not self.PROFILE_REGEX.search(results_raw):
logging.info("Could not find %s in browser output",
self.PROFILE_REGEX.pattern)
logging.info("Raw results:%s", results_raw)
raise TalosError("browser failed to close after being initialized")
def tooltool_download(manifest, run_local, raptor_dir):
"""Download a file from tooltool using the provided tooltool manifest"""
def outputHandler(line):
LOG.info(line)
if run_local:
command = [
sys.executable, TOOLTOOL_PATH, 'fetch', '-o', '-m', manifest
]
else:
# we want to use the tooltool cache in production
if os.environ.get('TOOLTOOLCACHE', None) is not None:
_cache = os.environ['TOOLTOOLCACHE']
else:
_cache = "/builds/tooltool_cache"
command = [
sys.executable, TOOLTOOL_PATH, 'fetch', '-o', '-m', manifest, '-c',
_cache
]
proc = ProcessHandler(command,
processOutputLine=outputHandler,
storeOutput=False,
cwd=raptor_dir)
proc.run()
try:
proc.wait()
except Exception:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def run_process(self, cmd, cwd=None, dump=False):
def _processOutput(line):
if self.verbose or dump:
print(line)
if self.verbose:
self.build_obj.log(logging.INFO, "autophone", {},
"Running '%s' in '%s'" % (cmd, cwd))
proc = ProcessHandler(cmd,
cwd=cwd,
processOutputLine=_processOutput,
processStderrLine=_processOutput)
proc.run()
proc_complete = False
try:
proc.wait()
if proc.proc.returncode == 0:
proc_complete = True
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
if not proc_complete:
if not self.verbose:
print(proc.output)
return proc_complete
def test_relative_path(self):
tempdir = tempfile.mkdtemp()
# make a dummy profile
profile = FirefoxProfile(os.path.join(tempdir, 'testprofilepath'),
restore=False)
self.assertTrue(os.path.exists(os.path.join(tempdir,
'testprofilepath',
'user.js')))
# make a dummy test
test = """function test() { };"""
f = file(os.path.join(tempdir, 'test_dummy.js'), 'w')
f.write(test)
f.close()
# run mozmill on it
process = ProcessHandler(['mozmill',
'-t', 'test_dummy.js',
'--profile=testprofilepath'],
cwd=tempdir,
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertNotEqual(process.proc.poll(), None)
# cleanup
shutil.rmtree(tempdir)
def setup_avds(self):
'''
If tooltool cache mechanism is enabled, the cached version is used by
the fetch command. If the manifest includes an "unpack" field, tooltool
will unpack all compressed archives mentioned in the manifest.
'''
c = self.config
dirs = self.query_abs_dirs()
# FIXME
# Clobbering and re-unpacking would not be needed if we had a way to
# check whether the unpacked content already present match the
# contents of the tar ball
self.rmtree(dirs['abs_avds_dir'])
self.mkdir_p(dirs['abs_avds_dir'])
if 'avd_url' in c:
# Intended for experimental setups to evaluate an avd prior to
# tooltool deployment.
url = c['avd_url']
self.download_unpack(url, dirs['abs_avds_dir'])
else:
url = self._get_repo_url(c["tooltool_manifest_path"])
self._tooltool_fetch(url, dirs['abs_avds_dir'])
avd_home_dir = self.abs_dirs['abs_avds_dir']
if avd_home_dir != "/home/cltbld/.android":
# Modify the downloaded avds to point to the right directory.
cmd = [
'bash', '-c',
'sed -i "s|/home/cltbld/.android|%s|" %s/test-*.ini' %
(avd_home_dir, os.path.join(avd_home_dir, 'avd'))
]
proc = ProcessHandler(cmd)
proc.run()
proc.wait()
def _run_profile(self):
command_args = utils.GenerateBrowserCommandLine(
self.browser_config["browser_path"],
self.browser_config["extra_args"],
self.profile_dir,
self.browser_config["init_url"]
)
def browser_log(line):
logging.debug('BROWSER_OUTPUT: %s', line)
browser = ProcessHandler(command_args, env=self.env,
processOutputLine=browser_log)
browser.run()
try:
browser.wait()
except KeyboardInterrupt:
browser.kill()
raise
results_raw = '\n'.join(browser.output)
if not self.PROFILE_REGEX.search(results_raw):
logging.info("Could not find %s in browser output",
self.PROFILE_REGEX.pattern)
logging.info("Raw results:%s", results_raw)
raise TalosError("browser failed to close after being initialized")
def setup_avds(self):
'''
If tooltool cache mechanism is enabled, the cached version is used by
the fetch command. If the manifest includes an "unpack" field, tooltool
will unpack all compressed archives mentioned in the manifest.
'''
c = self.config
dirs = self.query_abs_dirs()
self.mkdir_p(dirs['abs_work_dir'])
self.mkdir_p(dirs['abs_blob_upload_dir'])
# Always start with a clean AVD: AVD includes Android images
# which can be stateful.
self.rmtree(dirs['abs_avds_dir'])
self.mkdir_p(dirs['abs_avds_dir'])
if 'avd_url' in c:
# Intended for experimental setups to evaluate an avd prior to
# tooltool deployment.
url = c['avd_url']
self.download_unpack(url, dirs['abs_avds_dir'])
else:
url = self._get_repo_url(c["tooltool_manifest_path"])
self._tooltool_fetch(url, dirs['abs_avds_dir'])
avd_home_dir = self.abs_dirs['abs_avds_dir']
if avd_home_dir != "/home/cltbld/.android":
# Modify the downloaded avds to point to the right directory.
cmd = [
'bash', '-c',
'sed -i "s|/home/cltbld/.android|%s|" %s/test-*.ini' %
(avd_home_dir, os.path.join(avd_home_dir, 'avd'))
]
proc = ProcessHandler(cmd)
proc.run()
proc.wait()
def setup_avds(self):
'''
If tooltool cache mechanism is enabled, the cached version is used by
the fetch command. If the manifest includes an "unpack" field, tooltool
will unpack all compressed archives mentioned in the manifest.
'''
c = self.config
dirs = self.query_abs_dirs()
# Always start with a clean AVD: AVD includes Android images
# which can be stateful.
self.rmtree(dirs['abs_avds_dir'])
self.mkdir_p(dirs['abs_avds_dir'])
if 'avd_url' in c:
# Intended for experimental setups to evaluate an avd prior to
# tooltool deployment.
url = c['avd_url']
self.download_unpack(url, dirs['abs_avds_dir'])
else:
url = self._get_repo_url(c["tooltool_manifest_path"])
self._tooltool_fetch(url, dirs['abs_avds_dir'])
avd_home_dir = self.abs_dirs['abs_avds_dir']
if avd_home_dir != "/home/cltbld/.android":
# Modify the downloaded avds to point to the right directory.
cmd = [
'bash', '-c',
'sed -i "s|/home/cltbld/.android|%s|" %s/test-*.ini' %
(avd_home_dir, os.path.join(avd_home_dir, 'avd'))
]
proc = ProcessHandler(cmd)
proc.run()
proc.wait()
def lint(files, config, **kwargs):
tests_dir = os.path.join(kwargs['root'], 'testing', 'web-platform', 'tests')
def process_line(line):
try:
data = json.loads(line)
except ValueError:
return
data["level"] = "error"
data["path"] = os.path.relpath(os.path.join(tests_dir, data["path"]), kwargs['root'])
data.setdefault("lineno", 0)
results.append(result.from_config(config, **data))
if files == [tests_dir]:
print >> sys.stderr, ("No specific files specified, running the full wpt lint"
" (this is slow)")
files = ["--all"]
cmd = [os.path.join(tests_dir, 'wpt'), 'lint', '--json'] + files
if platform.system() == 'Windows':
cmd.insert(0, sys.executable)
proc = ProcessHandler(cmd, env=os.environ, processOutputLine=process_line)
proc.run()
try:
proc.wait()
if proc.returncode != 0:
results.append(
result.from_config(config,
message="Lint process exited with return code %s" %
proc.returncode))
except KeyboardInterrupt:
proc.kill()
return results
def setup_avds(self):
'''
If tooltool cache mechanism is enabled, the cached version is used by
the fetch command. If the manifest includes an "unpack" field, tooltool
will unpack all compressed archives mentioned in the manifest.
'''
c = self.config
dirs = self.query_abs_dirs()
# FIXME
# Clobbering and re-unpacking would not be needed if we had a way to
# check whether the unpacked content already present match the
# contents of the tar ball
self.rmtree(dirs['abs_avds_dir'])
self.mkdir_p(dirs['abs_avds_dir'])
url = self._get_repo_url(c["tooltool_manifest_path"])
self._tooltool_fetch(url, dirs['abs_avds_dir'])
avd_home_dir = self.abs_dirs['abs_avds_dir']
if avd_home_dir != "/home/cltbld/.android":
# Modify the downloaded avds to point to the right directory.
cmd = [
'bash', '-c',
'sed -i "s|/home/cltbld/.android|%s|" %s/test-*.ini' %
(avd_home_dir, os.path.join(avd_home_dir, 'avd'))
]
proc = ProcessHandler(cmd)
proc.run()
proc.wait()
def lint(files, config, **kwargs):
tests_dir = os.path.join(kwargs['root'], 'testing', 'web-platform', 'tests')
def process_line(line):
try:
data = json.loads(line)
except ValueError:
return
data["level"] = "error"
data["path"] = os.path.relpath(os.path.join(tests_dir, data["path"]), kwargs['root'])
results.append(result.from_config(config, **data))
if files == [tests_dir]:
print >> sys.stderr, ("No specific files specified, running the full wpt lint"
" (this is slow)")
files = ["--all"]
cmd = [os.path.join(tests_dir, 'wpt'), 'lint', '--json'] + files
if platform.system() == 'Windows':
cmd.insert(0, sys.executable)
proc = ProcessHandler(cmd, env=os.environ, processOutputLine=process_line)
proc.run()
try:
proc.wait()
if proc.returncode != 0:
results.append(
result.from_config(config,
message="Lint process exited with return code %s" %
proc.returncode))
except KeyboardInterrupt:
proc.kill()
return results
def lint(files, config, **kwargs):
tests_dir = os.path.join(kwargs['root'], 'testing', 'web-platform',
'tests')
def process_line(line):
try:
data = json.loads(line)
except ValueError:
return
data["level"] = "error"
data["path"] = os.path.relpath(os.path.join(tests_dir, data["path"]),
kwargs['root'])
results.append(result.from_config(config, **data))
path = os.path.join(tests_dir, "lint")
proc = ProcessHandler([path, "--json"] + files,
env=os.environ,
processOutputLine=process_line)
proc.run()
try:
proc.wait()
except KeyboardInterrupt:
proc.kill()
return results
def run_thing(self):
thing = os.path.abspath(
os.path.join(self.install_dir, self.config["thing_to_run"]))
# thing_args is a LockedTuple, which mozprocess doesn't like
args = list(self.config["thing_args"])
timeout = self.config["run_for"]
self.log(f"Running {thing} with args {args}")
p = ProcessHandler(
thing,
args=args,
shell=False,
storeOutput=True,
kill_on_timeout=True,
stream=False,
)
p.run(timeout)
# Wait for the timeout + a grace period (to make sure we don't interrupt
# process tear down).
# Without this, this script could potentially hang
p.wait(timeout + 10)
if not p.timedOut:
# It crashed, oh no!
self.critical(
f"TEST-UNEXPECTED-FAIL: {thing} did not run for {timeout} seconds"
)
self.critical("Output was:")
for l in p.output:
self.critical(l)
self.fatal("fail")
else:
self.info(f"PASS: {thing} ran successfully for {timeout} seconds")
def run_process():
path = os.path.join(tests_dir, "lint")
proc = ProcessHandler([path, "--json"], env=os.environ,
processOutputLine=process_line)
proc.run()
try:
proc.wait()
except KeyboardInterrupt:
proc.kill()
def tooltool_download(manifest, run_local, raptor_dir):
"""Download a file from tooltool using the provided tooltool manifest"""
def outputHandler(line):
LOG.info(line)
tooltool_path = None
for path in TOOLTOOL_PATHS:
if os.path.exists(os.path.dirname(path)):
tooltool_path = path
break
if tooltool_path is None:
raise Exception("Could not find tooltool path!")
if run_local:
command = [
sys.executable, tooltool_path, "fetch", "-o", "-m", manifest
]
else:
# Attempt to determine the tooltool cache path:
# - TOOLTOOLCACHE is used by Raptor tests
# - TOOLTOOL_CACHE is automatically set for taskcluster jobs
# - fallback to a hardcoded path
_cache = next(x for x in (
os.environ.get("TOOLTOOLCACHE"),
os.environ.get("TOOLTOOL_CACHE"),
"/builds/tooltool_cache",
) if x is not None)
command = [
sys.executable,
tooltool_path,
"fetch",
"-o",
"-m",
manifest,
"-c",
_cache,
]
try:
proc = ProcessHandler(command,
processOutputLine=outputHandler,
storeOutput=False,
cwd=raptor_dir)
proc.run()
proc.wait()
except Exception as e:
LOG.critical(
"Error while downloading the hostutils from tooltool: {}".format(
str(e)))
if proc.poll() is None:
proc.kill(signal.SIGTERM)
raise
def _isLocalZipAvailable(self):
def _noOutput(line):
# suppress output from zip ProcessHandler
pass
try:
proc = ProcessHandler(["zip", "-?"], storeOutput=False, processOutputLine=_noOutput)
proc.run()
proc.wait()
except:
return False
return True
def _isLocalZipAvailable(self):
def _noOutput(line):
# suppress output from zip ProcessHandler
pass
try:
proc = ProcessHandler(["zip", "-?"], storeOutput=False, processOutputLine=_noOutput)
proc.run()
proc.wait()
except:
return False
return True
def pushDir(self, localDir, remoteDir, retryLimit=None, timeout=None):
# adb "push" accepts a directory as an argument, but if the directory
# contains symbolic links, the links are pushed, rather than the linked
# files; we either zip/unzip or re-copy the directory into a temporary
# one to get around this limitation
retryLimit = retryLimit or self.retryLimit
if self._useZip:
self.removeDir(remoteDir)
self.mkDirs(remoteDir + "/x")
try:
localZip = tempfile.mktemp() + ".zip"
remoteZip = remoteDir + "/adbdmtmp.zip"
proc = ProcessHandler(["zip", "-r", localZip, '.'],
cwd=localDir,
processOutputLine=self._log)
proc.run()
proc.wait()
self.pushFile(localZip,
remoteZip,
retryLimit=retryLimit,
createDir=False)
mozfile.remove(localZip)
data = self._runCmd(
["shell", "unzip", "-o", remoteZip, "-d",
remoteDir]).output[0]
self._checkCmd(["shell", "rm", remoteZip],
retryLimit=retryLimit,
timeout=self.short_timeout)
if re.search("unzip: exiting", data) or re.search(
"Operation not permitted", data):
raise Exception("unzip failed, or permissions error")
except:
self._logger.warning(traceback.format_exc())
self._logger.warning(
"zip/unzip failure: falling back to normal push")
self._useZip = False
self.pushDir(localDir,
remoteDir,
retryLimit=retryLimit,
timeout=timeout)
else:
# If the remote directory exists, newer implementations of
# "adb push" will create a sub-directory, while older versions
# will not! Bug 1285040
self.mkDirs(remoteDir + "/x")
self.removeDir(remoteDir)
tmpDir = tempfile.mkdtemp()
# copytree's target dir must not already exist, so create a subdir
tmpDirTarget = os.path.join(tmpDir, "tmp")
shutil.copytree(localDir, tmpDirTarget)
self._checkCmd(["push", tmpDirTarget, remoteDir],
retryLimit=retryLimit,
timeout=timeout)
mozfile.remove(tmpDir)
def pushDir(self, localDir, remoteDir, retryLimit=None, timeout=None):
# adb "push" accepts a directory as an argument, but if the directory
# contains symbolic links, the links are pushed, rather than the linked
# files; we either zip/unzip or re-copy the directory into a temporary
# one to get around this limitation
retryLimit = retryLimit or self.retryLimit
if self._useZip:
self.removeDir(remoteDir)
self.mkDirs(remoteDir + "/x")
try:
localZip = tempfile.mktemp() + ".zip"
remoteZip = remoteDir + "/adbdmtmp.zip"
proc = ProcessHandler(["zip", "-r", localZip, '.'], cwd=localDir,
processOutputLine=self._log)
proc.run()
proc.wait()
self.pushFile(localZip, remoteZip, retryLimit=retryLimit, createDir=False)
mozfile.remove(localZip)
data = self._runCmd(["shell", "unzip", "-o", remoteZip,
"-d", remoteDir]).output[0]
self._checkCmd(["shell", "rm", remoteZip],
retryLimit=retryLimit, timeout=self.short_timeout)
if re.search("unzip: exiting", data) or re.search("Operation not permitted", data):
raise Exception("unzip failed, or permissions error")
except Exception:
self._logger.warning(traceback.format_exc())
self._logger.warning("zip/unzip failure: falling back to normal push")
self._useZip = False
self.pushDir(localDir, remoteDir, retryLimit=retryLimit, timeout=timeout)
else:
localDir = os.path.normpath(localDir)
remoteDir = os.path.normpath(remoteDir)
tempParent = tempfile.mkdtemp()
remoteName = os.path.basename(remoteDir)
newLocal = os.path.join(tempParent, remoteName)
dir_util.copy_tree(localDir, newLocal)
# See do_sync_push in
# https://android.googlesource.com/platform/system/core/+/master/adb/file_sync_client.cpp
# Work around change in behavior in adb 1.0.36 where if
# the remote destination directory exists, adb push will
# copy the source directory *into* the destination
# directory otherwise it will copy the source directory
# *onto* the destination directory.
if self._adb_version >= '1.0.36':
remoteDir = '/'.join(remoteDir.rstrip('/').split('/')[:-1])
try:
if self._checkCmd(["push", newLocal, remoteDir],
retryLimit=retryLimit, timeout=timeout):
raise DMError("failed to push %s (copy of %s) to %s" %
(newLocal, localDir, remoteDir))
except BaseException:
raise
finally:
mozfile.remove(tempParent)
def pushDir(self, localDir, remoteDir, retryLimit=None, timeout=None):
# adb "push" accepts a directory as an argument, but if the directory
# contains symbolic links, the links are pushed, rather than the linked
# files; we either zip/unzip or re-copy the directory into a temporary
# one to get around this limitation
retryLimit = retryLimit or self.retryLimit
if self._useZip:
self.removeDir(remoteDir)
self.mkDirs(remoteDir + "/x")
try:
localZip = tempfile.mktemp() + ".zip"
remoteZip = remoteDir + "/adbdmtmp.zip"
proc = ProcessHandler(["zip", "-r", localZip, '.'], cwd=localDir,
processOutputLine=self._log)
proc.run()
proc.wait()
self.pushFile(localZip, remoteZip, retryLimit=retryLimit, createDir=False)
mozfile.remove(localZip)
data = self._runCmd(["shell", "unzip", "-o", remoteZip,
"-d", remoteDir]).output[0]
self._checkCmd(["shell", "rm", remoteZip],
retryLimit=retryLimit, timeout=self.short_timeout)
if re.search("unzip: exiting", data) or re.search("Operation not permitted", data):
raise Exception("unzip failed, or permissions error")
except:
self._logger.warning(traceback.format_exc())
self._logger.warning("zip/unzip failure: falling back to normal push")
self._useZip = False
self.pushDir(localDir, remoteDir, retryLimit=retryLimit, timeout=timeout)
else:
localDir = os.path.normpath(localDir)
remoteDir = os.path.normpath(remoteDir)
tempParent = tempfile.mkdtemp()
remoteName = os.path.basename(remoteDir)
newLocal = os.path.join(tempParent, remoteName)
dir_util.copy_tree(localDir, newLocal)
# See do_sync_push in
# https://android.googlesource.com/platform/system/core/+/master/adb/file_sync_client.cpp
# Work around change in behavior in adb 1.0.36 where if
# the remote destination directory exists, adb push will
# copy the source directory *into* the destination
# directory otherwise it will copy the source directory
# *onto* the destination directory.
if self._adb_version >= '1.0.36':
remoteDir = '/'.join(remoteDir.rstrip('/').split('/')[:-1])
try:
if self._checkCmd(["push", newLocal, remoteDir],
retryLimit=retryLimit, timeout=timeout):
raise DMError("failed to push %s (copy of %s) to %s" %
(newLocal, localDir, remoteDir))
except:
raise
finally:
mozfile.remove(tempParent)
def test_no_option(self):
process = ProcessHandler(
[
'mozmill', '-b', os.environ['BROWSER_PATH'], '-t',
os.path.join(testdir, 'useMozmill', 'testServerRoot.js')
],
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertEqual(process.proc.poll(), 1, 'Test failed')
def _tooltool_fetch(self):
def outputHandler(line):
self._log_debug(line)
command = ["python", "tooltool.py", "fetch", "-m", "releng.manifest"]
proc = ProcessHandler(command, processOutputLine=outputHandler, storeOutput=False, cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def _tooltool_fetch():
def outputHandler(line):
_log_debug(line)
_download_file(TOOLTOOL_URL, "tooltool.py", EMULATOR_HOME_DIR)
command = [sys.executable, "tooltool.py", "fetch", "-o", "-m", "releng.manifest"]
proc = ProcessHandler(command, processOutputLine=outputHandler, storeOutput=False, cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def run_process(cmdargs):
# flake8 seems to handle SIGINT poorly. Handle it here instead
# so we can kill the process without a cryptic traceback.
orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
proc = ProcessHandler(cmdargs, env=os.environ,
processOutputLine=process_line)
proc.run()
signal.signal(signal.SIGINT, orig)
try:
proc.wait()
except KeyboardInterrupt:
proc.kill()
def _tooltool_fetch():
def outputHandler(line):
_log_debug(line)
command = ['python', 'tooltool.py', 'fetch', '-o', '-m', 'releng.manifest']
proc = ProcessHandler(
command, processOutputLine=outputHandler, storeOutput=False,
cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def test_no_option(self):
process = ProcessHandler(['mozmill',
'-b', os.environ['BROWSER_PATH'],
'-t', os.path.join(testdir,
'useMozmill',
'testServerRoot.js')
],
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertEqual(process.proc.poll(), 1,
'Test failed')
def run_process(cmdargs):
# flake8 seems to handle SIGINT poorly. Handle it here instead
# so we can kill the process without a cryptic traceback.
orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
proc = ProcessHandler(cmdargs,
env=os.environ,
processOutputLine=process_line)
proc.run()
signal.signal(signal.SIGINT, orig)
try:
proc.wait()
except KeyboardInterrupt:
proc.kill()
def _tooltool_fetch():
def outputHandler(line):
_log_debug(line)
_download_file(TOOLTOOL_URL, 'tooltool.py', EMULATOR_HOME_DIR)
command = ['python', 'tooltool.py', 'fetch', '-o', '-m', 'releng.manifest']
proc = ProcessHandler(
command, processOutputLine=outputHandler, storeOutput=False,
cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def _tooltool_fetch():
def outputHandler(line):
_log_debug(line)
tooltool_full_path = os.path.abspath(TOOLTOOL_PATH)
command = [sys.executable, tooltool_full_path,
'fetch', '-o', '-m', 'releng.manifest']
proc = ProcessHandler(
command, processOutputLine=outputHandler, storeOutput=False,
cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except Exception:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def test_run(install_mozproxy):
build = install_mozproxy
output_handler = OutputHandler()
p = ProcessHandler(
[
"mozproxy",
"--local",
"--binary=firefox",
"--topsrcdir=" + build.topsrcdir,
"--objdir=" + build.topobjdir,
os.path.join(here, "files", "mitm5-linux-firefox-amazon.zip"),
],
processOutputLine=output_handler,
onFinish=output_handler.finished,
)
p.run()
# The first time we run mozproxy, we need to fetch mitmproxy, which can
# take a while...
assert output_handler.port_event.wait(120) is True
# Give mitmproxy a bit of time to start up so we can verify that it's
# actually running before we kill mozproxy.
time.sleep(5)
_kill_mozproxy(p.pid)
assert p.wait(10) == 0
assert output_handler.port is not None
def _runCmd(self, args, retryLimit=None):
"""
Runs a command using adb
returns: instance of ProcessHandler
"""
retryLimit = retryLimit or self.retryLimit
finalArgs = [self._adbPath]
if self._serverHost is not None:
finalArgs.extend(['-H', self._serverHost])
if self._serverPort is not None:
finalArgs.extend(['-P', str(self._serverPort)])
if self._deviceSerial:
finalArgs.extend(['-s', self._deviceSerial])
finalArgs.extend(args)
self._logger.debug("_runCmd - command: %s" % ' '.join(finalArgs))
retries = 0
while retries < retryLimit:
proc = ProcessHandler(finalArgs,
storeOutput=True,
processOutputLine=self._log)
proc.run()
proc.returncode = proc.wait()
if proc.returncode == None:
proc.kill()
retries += 1
else:
return proc
def _runCmd(self, args, timeout=None, retryLimit=None):
"""
Runs a command using adb
If timeout is specified, the process is killed after <timeout> seconds.
returns: instance of ProcessHandler
"""
retryLimit = retryLimit or self.retryLimit
finalArgs = [self._adbPath]
if self._serverHost is not None:
finalArgs.extend(['-H', self._serverHost])
if self._serverPort is not None:
finalArgs.extend(['-P', str(self._serverPort)])
if self._deviceSerial:
finalArgs.extend(['-s', self._deviceSerial])
finalArgs.extend(args)
self._logger.debug("_runCmd - command: %s" % ' '.join(finalArgs))
if not timeout:
timeout = self.default_timeout
def _timeout():
self._logger.error("Timeout exceeded for _runCmd call '%s'" % ' '.join(finalArgs))
retries = 0
while retries < retryLimit:
proc = ProcessHandler(finalArgs, storeOutput=True,
processOutputLine=self._log, onTimeout=_timeout)
proc.run(timeout=timeout)
proc.returncode = proc.wait()
if proc.returncode is None:
proc.kill()
retries += 1
else:
return proc
def _run_profile(self):
command_args = utils.GenerateBrowserCommandLine(
self.browser_config["browser_path"],
self.browser_config["extra_args"],
self.profile_dir,
self.browser_config["init_url"]
)
def browser_log(line):
LOG.process_output(browser.pid, line)
browser = ProcessHandler(command_args, env=self.env,
processOutputLine=browser_log)
browser.run()
LOG.process_start(browser.pid, ' '.join(command_args))
try:
exit_code = browser.wait()
except KeyboardInterrupt:
browser.kill()
raise
LOG.process_exit(browser.pid, exit_code)
results_raw = '\n'.join(browser.output)
if not self.PROFILE_REGEX.search(results_raw):
LOG.info("Could not find %s in browser output"
% self.PROFILE_REGEX.pattern)
LOG.info("Raw results:%s" % results_raw)
raise TalosError("browser failed to close after being initialized")
def flash(self):
command = os.path.join(self.b2g_home, 'flash.sh')
p = ProcessHandler(command)
if _is_device_attached():
p.run()
return p.wait()
return 1
def _runCmd(self, args, retryLimit=None):
"""
Runs a command using adb
returns: instance of ProcessHandler
"""
retryLimit = retryLimit or self.retryLimit
finalArgs = [self._adbPath]
if self._serverHost is not None:
finalArgs.extend(['-H', self._serverHost])
if self._serverPort is not None:
finalArgs.extend(['-P', str(self._serverPort)])
if self._deviceSerial:
finalArgs.extend(['-s', self._deviceSerial])
finalArgs.extend(args)
self._logger.debug("_runCmd - command: %s" % ' '.join(finalArgs))
retries = 0
while retries < retryLimit:
proc = ProcessHandler(finalArgs, storeOutput=True,
processOutputLine=self._log)
proc.run()
proc.returncode = proc.wait()
if proc.returncode == None:
proc.kill()
retries += 1
else:
return proc
def _runCmd(self, args, timeout=None, retryLimit=None):
"""
Runs a command using adb
If timeout is specified, the process is killed after <timeout> seconds.
returns: instance of ProcessHandler
"""
retryLimit = retryLimit or self.retryLimit
finalArgs = [self._adbPath]
if self._serverHost is not None:
finalArgs.extend(['-H', self._serverHost])
if self._serverPort is not None:
finalArgs.extend(['-P', str(self._serverPort)])
if self._deviceSerial:
finalArgs.extend(['-s', self._deviceSerial])
finalArgs.extend(args)
self._logger.debug("_runCmd - command: %s" % ' '.join(finalArgs))
if not timeout:
timeout = self.default_timeout
def _timeout():
self._logger.error("Timeout exceeded for _runCmd call '%s'" % ' '.join(finalArgs))
retries = 0
while retries < retryLimit:
proc = ProcessHandler(finalArgs, storeOutput=True,
processOutputLine=self._log, onTimeout=_timeout)
proc.run(timeout=timeout)
proc.returncode = proc.wait()
if proc.returncode == None:
proc.kill()
retries += 1
else:
return proc
def test_pref(self):
absdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
testdir = os.path.join(absdir, 'js-modules')
process = ProcessHandler(
[
'mozmill', '-b', os.environ['BROWSER_PATH'], '-t',
os.path.join(testdir, 'useMozmill', 'testPref.js'),
'--pref=abc:123'
],
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertEqual(process.proc.poll(), 0, 'Test passed')
def test_option(self):
process = ProcessHandler(['mozmill',
'-b', os.environ['BROWSER_PATH'],
'-t', os.path.join(testdir,
'useMozmill',
'testServerRoot.js'),
'--server-root', os.path.join(testdir,
'../../data'),
],
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertEqual(process.proc.poll(), 0,
'Test was run successful')
def build_script(self, modules=None, debug=False, profiling=False,
noftu=False, noopt=False, valgrind=False):
command = [os.path.join(self.b2g_home, 'build.sh')]
if modules:
command.extend(modules)
if debug:
command.insert(0, 'B2G_DEBUG=1')
if profiling:
command.insert(0, 'MOZ_PROFILING=1')
if noftu:
command.insert(0, 'NOFTU=1')
if noopt:
if profiling:
print("Can't perform profiling if optimizer is disabled")
return 1
command.insert(0, 'B2G_NOOPT=1')
if valgrind:
command.insert(0, 'B2G_VALGRIND=1')
p = ProcessHandler(command)
p.run()
#TODO: Error checking.
return p.wait()
def _checkCmd(self, args, timeout=None, retryLimit=None):
"""
Runs a command using adb and waits for the command to finish.
If timeout is specified, the process is killed after <timeout> seconds.
returns: returncode from process
"""
retryLimit = retryLimit or self.retryLimit
finalArgs = [self._adbPath]
if self._deviceSerial:
finalArgs.extend(['-s', self._deviceSerial])
finalArgs.extend(args)
self._logger.debug("_checkCmd - command: %s" % ' '.join(finalArgs))
if not timeout:
# We are asserting that all commands will complete in this
# time unless otherwise specified
timeout = self.default_timeout
timeout = int(timeout)
retries = 0
while retries < retryLimit:
proc = ProcessHandler(finalArgs, processOutputLine=self._log)
proc.run(timeout=timeout)
ret_code = proc.wait()
if ret_code == None:
proc.kill()
retries += 1
else:
return ret_code
raise DMError("Timeout exceeded for _checkCmd call after %d retries." % retries)
def _run_profile(self):
command_args = utils.GenerateBrowserCommandLine(
self.browser_config["browser_path"],
self.browser_config["extra_args"], self.profile_dir,
self.browser_config["init_url"])
def browser_log(line):
LOG.process_output(browser.pid, line)
browser = ProcessHandler(command_args,
env=self.env,
processOutputLine=browser_log)
browser.run()
LOG.process_start(browser.pid, ' '.join(command_args))
try:
exit_code = browser.wait()
except KeyboardInterrupt:
browser.kill()
raise
LOG.process_exit(browser.pid, exit_code)
results_raw = '\n'.join(browser.output)
if not self.PROFILE_REGEX.search(results_raw):
LOG.info("Could not find %s in browser output" %
self.PROFILE_REGEX.pattern)
LOG.info("Raw results:%s" % results_raw)
raise TalosError("browser failed to close after being initialized")
def _checkCmd(self, args, timeout=None, retryLimit=None):
"""
Runs a command using adb and waits for the command to finish.
If timeout is specified, the process is killed after <timeout> seconds.
returns: returncode from process
"""
retryLimit = retryLimit or self.retryLimit
finalArgs = [self._adbPath]
if self._deviceSerial:
finalArgs.extend(['-s', self._deviceSerial])
finalArgs.extend(args)
self._logger.debug("_checkCmd - command: %s" % ' '.join(finalArgs))
if not timeout:
# We are asserting that all commands will complete in this
# time unless otherwise specified
timeout = self.default_timeout
timeout = int(timeout)
retries = 0
while retries < retryLimit:
proc = ProcessHandler(finalArgs, processOutputLine=self._log)
proc.run(timeout=timeout)
ret_code = proc.wait()
if ret_code == None:
proc.kill()
retries += 1
else:
return ret_code
raise DMError("Timeout exceeded for _checkCmd call after %d retries." % retries)
def _tooltool_fetch(self, manifest):
def outputHandler(line):
LOG.info(line)
command = [sys.executable, TOOLTOOL_PATH, 'fetch', '-o', '-m', manifest]
proc = ProcessHandler(
command, processOutputLine=outputHandler, storeOutput=False,
cwd=self.raptor_dir)
proc.run()
try:
proc.wait()
except Exception:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def perform_build(self, history_line):
import pdb
pdb.set_trace()
self.build_number += 1
self.start_time = datetime.now()
log.debug("Performing build %d on history line: %s" % (self.build_number, history_line))
build_proc = ProcessHandler(cmd = ['/home/ctalbert/projects/b2g-hamachi/build.sh'],
cwd = self.build_info['workdir'],
env=self.build_info['env'],
processOutputLine=[self.notify_status],
kill_on_timeout=True,
onTimeout=[self.notify_timeout],
onFinish=[self.notify_finished],
shell=True)
try:
sys.stdout.write("Starting Build %d:" % self.build_number)
build_proc.run(timeout=7200)
build_proc.processOutput()
exitcode = build_proc.wait()
except (KeyboardInterrupt, SystemExit):
print "User Canceled Operation!"
log.debug("Build canceled by user")
raise
finally:
self.build_log.close()
if exitcode == 0:
print "Build %d Completed Successfully" % self.build_number
log.debug("Build %d for history line: %s completed successfully" % (self.build_number, history_line))
else:
print "Build %d Failed" % self.build_number
log.debug("Build %d for history line: %s FAILED" % (self.build_number, history_line))
def test_run_record_no_files(install_mozproxy):
build = install_mozproxy
output_handler = OutputHandler()
p = ProcessHandler(
[
"mozproxy",
"--local",
"--mode=record",
"--binary=firefox",
"--topsrcdir=" + build.topsrcdir,
"--objdir=" + build.topobjdir,
],
processOutputLine=output_handler,
onFinish=output_handler.finished,
)
p.run()
# The first time we run mozproxy, we need to fetch mitmproxy, which can
# take a while...
assert output_handler.port_event.wait(120) is True
# Give mitmproxy a bit of time to start up so we can verify that it's
# actually running before we kill mozproxy.
time.sleep(5)
_kill_mozproxy(p.pid)
# Assert process raises error
assert p.wait(10) == 2
assert output_handler.port is None
def test_options(self):
absdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
testdir = os.path.join(absdir, 'js-tests')
process = ProcessHandler(['mozmill',
'-b', os.environ['BROWSER_PATH'],
'-t', os.path.join(testdir,
'test_module1.js'),
'-m', os.path.join(testdir, 'example.ini')
],
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertNotEqual(process.proc.poll(), 0,
'Parser error due to -t and -m are mutually exclusive')
def test_pref(self):
absdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
testdir = os.path.join(absdir, 'js-modules')
process = ProcessHandler(['mozmill',
'-b', os.environ['BROWSER_PATH'],
'-t', os.path.join(testdir,
'useMozmill',
'testPref.js'),
'--pref=abc:123'
],
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertEqual(process.proc.poll(), 0, 'Test passed')
def test_options(self):
absdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
testdir = os.path.join(absdir, 'js-tests')
process = ProcessHandler(['mozmill',
'-b', os.environ['BROWSER_PATH'],
'-t', os.path.join(testdir,
'testUsemozmillControllerOpen.js'),
'-m', os.path.join(testdir, 'tests-null.ini')
],
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertNotEqual(process.proc.poll(), 0,
'Parser error due to -t and -m are mutually exclusive')
def test_option(self):
process = ProcessHandler(
[
'mozmill',
'-b',
os.environ['BROWSER_PATH'],
'-t',
os.path.join(testdir, 'useMozmill', 'testServerRoot.js'),
'--server-root',
os.path.join(testdir, '../../data'),
],
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertEqual(process.proc.poll(), 0, 'Test was run successful')
def _run_python_test(self, test):
from mozprocess import ProcessHandler
output = []
def _log(line):
# Buffer messages if more than one worker to avoid interleaving
if self.jobs > 1:
output.append(line)
else:
self.log(logging.INFO, "python-test", {"line": line.rstrip()},
"{line}")
file_displayed_test = [] # used as boolean
def _line_handler(line):
line = six.ensure_str(line)
if not file_displayed_test:
output = ("Ran" in line or "collected" in line
or line.startswith("TEST-"))
if output:
file_displayed_test.append(True)
# Hack to make sure treeherder highlights pytest failures
if "FAILED" in line.rsplit(" ", 1)[-1]:
line = line.replace("FAILED", "TEST-UNEXPECTED-FAIL")
_log(line)
_log(test["path"])
python = self.virtualenv_manager.python_path
cmd = [python, test["path"]]
env = os.environ.copy()
if six.PY2:
env[b"PYTHONDONTWRITEBYTECODE"] = b"1"
else:
env["PYTHONDONTWRITEBYTECODE"] = "1"
proc = ProcessHandler(cmd,
env=env,
processOutputLine=_line_handler,
storeOutput=False)
proc.run()
return_code = proc.wait()
if not file_displayed_test:
_log(
"TEST-UNEXPECTED-FAIL | No test output (missing mozunit.main() "
"call?): {}".format(test["path"]))
if self.verbose:
if return_code != 0:
_log("Test failed: {}".format(test["path"]))
else:
_log("Test passed: {}".format(test["path"]))
return output, return_code, test["path"]
def _run_python_safety(self, test_path):
from mozprocess import ProcessHandler
output = []
self.logger.test_start(test_path)
def _line_handler(line):
output.append(line.decode('UTF-8'))
cmd = ['safety', 'check', '--cache', '--json', '-r', test_path]
env = os.environ.copy()
env['PYTHONDONTWRITEBYTECODE'] = '1'
proc = ProcessHandler(cmd,
env=env,
processOutputLine=_line_handler,
storeOutput=False)
proc.run()
return_code = proc.wait()
"""
Example output for an error in json.
[
"pycrypto",
"<=2.6.1",
"2.6",
"Heap-based buffer overflow in the ALGnew...",
"35015"
]
"""
# Warnings are currently interleaved with json, see
# https://github.com/pyupio/safety/issues/133
for warning in output:
if warning.startswith('Warning'):
self.logger.warning(warning)
output = [line for line in output if not line.startswith('Warning')]
if output:
output_json = json.loads("".join(output))
affected = set()
for entry in output_json:
affected.add(entry[0])
message = "{0} installed:{2} affected:{1} description:{3}\n".format(
*entry)
self.logger.test_status(test=test_path,
subtest=entry[0],
status='FAIL',
message=message)
if return_code != 0:
status = 'FAIL'
else:
status = 'PASS'
self.logger.test_end(test_path,
status=status,
expected='PASS',
message=" ".join(affected))
return return_code
def setup_avds(self):
"""
If tooltool cache mechanism is enabled, the cached version is used by
the fetch command. If the manifest includes an "unpack" field, tooltool
will unpack all compressed archives mentioned in the manifest.
"""
c = self.config
dirs = self.query_abs_dirs()
# FIXME
# Clobbering and re-unpacking would not be needed if we had a way to
# check whether the unpacked content already present match the
# contents of the tar ball
self.rmtree(dirs["abs_avds_dir"])
self.mkdir_p(dirs["abs_avds_dir"])
if not self.buildbot_config:
# XXX until we figure out how to determine the repo_path, revision
url = "https://hg.mozilla.org/%s/raw-file/%s/%s" % ("try", "default", c["tooltool_manifest_path"])
self._tooltool_fetch(url)
elif self.buildbot_config and "properties" in self.buildbot_config:
url = "https://hg.mozilla.org/%s/raw-file/%s/%s" % (
self.buildbot_config["properties"]["repo_path"],
self.buildbot_config["properties"]["revision"],
c["tooltool_manifest_path"],
)
self._tooltool_fetch(url)
else:
self.fatal(
"properties in self.buildbot_config are required to "
"retrieve tooltool manifest to be used for avds setup"
)
if self.config.get("developer_mode"):
# For developer mode we have to modify the downloaded avds to
# point to the right directory.
avd_home_dir = self.abs_dirs["abs_avds_dir"]
cmd = [
"bash",
"-c",
'sed -i "s|/home/cltbld/.android|%s|" %s/test-*.ini'
% (avd_home_dir, os.path.join(avd_home_dir, "avd")),
]
proc = ProcessHandler(cmd)
proc.run()
proc.wait()
def _launch_emulator(self, emulator_index):
emulator = self.emulators[emulator_index]
env = self.query_env()
# Set $LD_LIBRARY_PATH to self.dirs['abs_work_dir'] so that
# the emulator picks up the symlink to libGL.so.1 that we
# constructed in start_emulators.
env['LD_LIBRARY_PATH'] = self.abs_dirs['abs_work_dir']
if self.config.get("developer_mode"):
avds_dir = os.path.join(self.abs_dirs['abs_avds_dir'], 'avd')
# IIUC when using a directory different than $HOME/.android/avd
# we need to set this value. This is important since we're running
# in a developer's machine
env['ANDROID_AVD_HOME'] = avds_dir
cmd = ['bash', '-c',
'sed -i "s|/home/cltbld/.android/avd|%s|" %s/test-*.ini' %
(avds_dir, avds_dir)]
print cmd
proc = ProcessHandler(cmd)
proc.run()
proc.wait()
command = [
"emulator", "-avd", emulator["name"],
"-debug", "init,console,gles,memcheck,adbserver,adbclient,adb,avd_config,socket",
"-port", str(emulator["emulator_port"]),
# Enable kvm; -qemu arguments must be at the end of the command
"-qemu", "-m", "1024"
]
if "emulator_cpu" in self.config:
command += ["-cpu", self.config["emulator_cpu"]]
else:
command += ["-enable-kvm"]
tmp_file = tempfile.NamedTemporaryFile(mode='w')
tmp_stdout = open(tmp_file.name, 'w')
self.info("Created temp file %s." % tmp_file.name)
self.info("Trying to start the emulator with this command: %s" % ' '.join(command))
proc = ProcessHandler(command, env=env)
proc.run()
return {
"process": proc,
"tmp_file": tmp_file,
"tmp_stdout": tmp_stdout
}
def gdb(self):
command = os.path.join(self.b2g_home, 'run-gdb.sh')
p = ProcessHandler(command)
p.run()
#TODO The emulator requires adb to run, we should check if that is
#running, catch that error or better yet, start adb.
return p.wait()
class ServoRefTestExecutor(ProcessTestExecutor):
convert_result = reftest_result_converter
def __init__(self, browser, http_server_url, binary=None, timeout_multiplier=1,
screenshot_cache=None, debug_args=None, pause_after_test=False):
ProcessTestExecutor.__init__(self,
browser,
http_server_url,
timeout_multiplier=timeout_multiplier,
debug_args=debug_args)
self.protocol = Protocol(self, browser, http_server_url)
self.screenshot_cache = screenshot_cache
self.implementation = RefTestImplementation(self)
self.tempdir = tempfile.mkdtemp()
def teardown(self):
os.rmdir(self.tempdir)
ProcessTestExecutor.teardown(self)
def screenshot(self, url, timeout):
full_url = urlparse.urljoin(self.http_server_url, url)
with TempFilename(self.tempdir) as output_path:
self.command = [self.binary, "--cpu", "--hard-fail", "--exit",
"--output=%s" % output_path, full_url]
self.proc = ProcessHandler(self.command,
processOutputLine=[self.on_output])
self.proc.run()
rv = self.proc.wait(timeout=timeout)
if rv is None:
self.proc.kill()
return False, ("EXTERNAL-TIMEOUT", None)
if rv < 0:
return False, ("CRASH", None)
with open(output_path) as f:
# Might need to strip variable headers or something here
data = f.read()
return True, data
def do_test(self, test):
result = self.implementation.run_test(test)
return self.convert_result(test, result)
def on_output(self, line):
line = line.decode("utf8", "replace")
if self.interactive:
print line
else:
self.logger.process_output(self.proc.pid,
line,
" ".join(self.command))
def pushDir(self, localDir, remoteDir, retryLimit=None, timeout=None):
# adb "push" accepts a directory as an argument, but if the directory
# contains symbolic links, the links are pushed, rather than the linked
# files; we either zip/unzip or re-copy the directory into a temporary
# one to get around this limitation
retryLimit = retryLimit or self.retryLimit
if self._useZip:
self.removeDir(remoteDir)
self.mkDirs(remoteDir + "/x")
try:
localZip = tempfile.mktemp() + ".zip"
remoteZip = remoteDir + "/adbdmtmp.zip"
proc = ProcessHandler(["zip", "-r", localZip, '.'], cwd=localDir,
processOutputLine=self._log)
proc.run()
proc.wait()
self.pushFile(localZip, remoteZip, retryLimit=retryLimit, createDir=False)
mozfile.remove(localZip)
data = self._runCmd(["shell", "unzip", "-o", remoteZip,
"-d", remoteDir]).output[0]
self._checkCmd(["shell", "rm", remoteZip],
retryLimit=retryLimit, timeout=self.short_timeout)
if re.search("unzip: exiting", data) or re.search("Operation not permitted", data):
raise Exception("unzip failed, or permissions error")
except:
self._logger.warning(traceback.format_exc())
self._logger.warning("zip/unzip failure: falling back to normal push")
self._useZip = False
self.pushDir(localDir, remoteDir, retryLimit=retryLimit, timeout=timeout)
else:
# If the remote directory exists, newer implementations of
# "adb push" will create a sub-directory, while older versions
# will not! Bug 1285040
self.mkDirs(remoteDir + "/x")
self.removeDir(remoteDir)
tmpDir = tempfile.mkdtemp()
# copytree's target dir must not already exist, so create a subdir
tmpDirTarget = os.path.join(tmpDir, "tmp")
shutil.copytree(localDir, tmpDirTarget)
self._checkCmd(["push", tmpDirTarget, remoteDir],
retryLimit=retryLimit, timeout=timeout)
mozfile.remove(tmpDir)
def setup_avds(self):
'''
If tooltool cache mechanism is enabled, the cached version is used by
the fetch command. If the manifest includes an "unpack" field, tooltool
will unpack all compressed archives mentioned in the manifest.
'''
c = self.config
dirs = self.query_abs_dirs()
# FIXME
# Clobbering and re-unpacking would not be needed if we had a way to
# check whether the unpacked content already present match the
# contents of the tar ball
self.rmtree(dirs['abs_avds_dir'])
self.mkdir_p(dirs['abs_avds_dir'])
if not self.buildbot_config:
# XXX until we figure out how to determine the repo_path, revision
url = 'https://hg.mozilla.org/%s/raw-file/%s/%s' % (
"try", "default", c["tooltool_manifest_path"])
self._tooltool_fetch(url, dirs['abs_avds_dir'])
elif self.buildbot_config and 'properties' in self.buildbot_config:
url = 'https://hg.mozilla.org/%s/raw-file/%s/%s' % (
self.buildbot_config['properties']['repo_path'],
self.buildbot_config['properties']['revision'],
c["tooltool_manifest_path"])
self._tooltool_fetch(url, dirs['abs_avds_dir'])
else:
self.fatal("properties in self.buildbot_config are required to "
"retrieve tooltool manifest to be used for avds setup")
avd_home_dir = self.abs_dirs['abs_avds_dir']
if avd_home_dir != "/home/cltbld/.android":
# Modify the downloaded avds to point to the right directory.
cmd = [
'bash', '-c',
'sed -i "s|/home/cltbld/.android|%s|" %s/test-*.ini' %
(avd_home_dir, os.path.join(avd_home_dir, 'avd'))
]
proc = ProcessHandler(cmd)
proc.run()
proc.wait()
def run_process(self, cmd, cwd=None, dump=False):
def _processOutput(line):
if self.verbose or dump:
print(line)
if self.verbose:
self.build_obj.log(logging.INFO, "autophone", {},
"Running '%s' in '%s'" % (cmd, cwd))
proc = ProcessHandler(cmd, cwd=cwd, processOutputLine=_processOutput,
processStderrLine=_processOutput)
proc.run()
proc_complete = False
try:
proc.wait()
if proc.proc.returncode == 0:
proc_complete = True
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
if not proc_complete:
if not self.verbose:
print(proc.output)
return proc_complete
