def run_process(self, cmd, cwd=None, dump=False):
def _processOutput(line):
if self.verbose or dump:
print(line)
if self.verbose:
self.build_obj.log(logging.INFO, "autophone", {},
"Running '%s' in '%s'" % (cmd, cwd))
proc = ProcessHandler(cmd,
cwd=cwd,
processOutputLine=_processOutput,
processStderrLine=_processOutput)
proc.run()
proc_complete = False
try:
proc.wait()
if proc.proc.returncode == 0:
proc_complete = True
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
if not proc_complete:
if not self.verbose:
print(proc.output)
return proc_complete
def tooltool_download(manifest, run_local, raptor_dir):
"""Download a file from tooltool using the provided tooltool manifest"""
def outputHandler(line):
LOG.info(line)
if run_local:
command = [
sys.executable, TOOLTOOL_PATH, 'fetch', '-o', '-m', manifest
]
else:
# we want to use the tooltool cache in production
if os.environ.get('TOOLTOOLCACHE', None) is not None:
_cache = os.environ['TOOLTOOLCACHE']
else:
_cache = "/builds/tooltool_cache"
command = [
sys.executable, TOOLTOOL_PATH, 'fetch', '-o', '-m', manifest, '-c',
_cache
]
proc = ProcessHandler(command,
processOutputLine=outputHandler,
storeOutput=False,
cwd=raptor_dir)
proc.run()
try:
proc.wait()
except Exception:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def tooltool_download(manifest, run_local, raptor_dir):
"""Download a file from tooltool using the provided tooltool manifest"""
def outputHandler(line):
LOG.info(line)
tooltool_path = None
for path in TOOLTOOL_PATHS:
if os.path.exists(os.path.dirname(path)):
tooltool_path = path
break
if tooltool_path is None:
raise Exception("Could not find tooltool path!")
if run_local:
command = [sys.executable, tooltool_path, "fetch", "-o", "-m", manifest]
else:
# Attempt to determine the tooltool cache path:
# - TOOLTOOLCACHE is used by Raptor tests
# - TOOLTOOL_CACHE is automatically set for taskcluster jobs
# - fallback to a hardcoded path
_cache = next(
x
for x in (
os.environ.get("TOOLTOOLCACHE"),
os.environ.get("TOOLTOOL_CACHE"),
"/builds/tooltool_cache",
)
if x is not None
)
command = [
sys.executable,
tooltool_path,
"fetch",
"-o",
"-m",
manifest,
"-c",
_cache,
]
try:
proc = ProcessHandler(
command, processOutputLine=outputHandler, storeOutput=False, cwd=raptor_dir
)
proc.run()
if proc.wait() != 0:
raise Exception("Command failed")
except Exception as e:
LOG.critical(
"Error while downloading {} from tooltool:{}".format(manifest, str(e))
)
if proc.poll() is None:
proc.kill(signal.SIGTERM)
raise
def _tooltool_fetch(self):
def outputHandler(line):
self._log_debug(line)
command = ["python", "tooltool.py", "fetch", "-m", "releng.manifest"]
proc = ProcessHandler(command, processOutputLine=outputHandler, storeOutput=False, cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def _tooltool_fetch():
def outputHandler(line):
_log_debug(line)
command = ['python', 'tooltool.py', 'fetch', '-o', '-m', 'releng.manifest']
proc = ProcessHandler(
command, processOutputLine=outputHandler, storeOutput=False,
cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def _tooltool_fetch():
def outputHandler(line):
_log_debug(line)
_download_file(TOOLTOOL_URL, "tooltool.py", EMULATOR_HOME_DIR)
command = [sys.executable, "tooltool.py", "fetch", "-o", "-m", "releng.manifest"]
proc = ProcessHandler(command, processOutputLine=outputHandler, storeOutput=False, cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def _tooltool_fetch():
def outputHandler(line):
_log_debug(line)
_download_file(TOOLTOOL_URL, 'tooltool.py', EMULATOR_HOME_DIR)
command = ['python', 'tooltool.py', 'fetch', '-o', '-m', 'releng.manifest']
proc = ProcessHandler(
command, processOutputLine=outputHandler, storeOutput=False,
cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def _tooltool_fetch():
def outputHandler(line):
_log_debug(line)
tooltool_full_path = os.path.abspath(TOOLTOOL_PATH)
command = [sys.executable, tooltool_full_path,
'fetch', '-o', '-m', 'releng.manifest']
proc = ProcessHandler(
command, processOutputLine=outputHandler, storeOutput=False,
cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except Exception:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def _tooltool_fetch(self, manifest):
def outputHandler(line):
LOG.info(line)
command = [sys.executable, TOOLTOOL_PATH, 'fetch', '-o', '-m', manifest]
proc = ProcessHandler(
command, processOutputLine=outputHandler, storeOutput=False,
cwd=self.raptor_dir)
proc.run()
try:
proc.wait()
except Exception:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def start_mitmproxy_playback(
self,
mitmdump_path,
browser_path,
):
"""Startup mitmproxy and replay the specified flow file"""
LOG.info("mitmdump path: %s" % mitmdump_path)
LOG.info("browser path: %s" % browser_path)
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
env["PATH"] = os.path.dirname(browser_path) + ";" + env["PATH"]
command = [mitmdump_path, "-k"]
if "playback_tool_args" in self.config:
command.extend(self.config["playback_tool_args"])
LOG.info("Starting mitmproxy playback using env path: %s" %
env["PATH"])
LOG.info("Starting mitmproxy playback using command: %s" %
" ".join(command))
# to turn off mitmproxy log output, use these params for Popen:
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
mitmproxy_proc = ProcessHandler(command,
logfile=os.path.join(
self.upload_dir, "mitmproxy.log"),
env=env)
mitmproxy_proc.run()
# XXX replace the code below with a loop with a connection attempt
# Bug 1532557
time.sleep(MITMDUMP_SLEEP)
data = mitmproxy_proc.poll()
if data is None: # None value indicates process hasn't terminated
LOG.info("Mitmproxy playback successfully started as pid %d" %
mitmproxy_proc.pid)
return mitmproxy_proc
# cannot continue as we won't be able to playback the pages
LOG.error(
"Aborting: mitmproxy playback process failed to start, poll returned: %s"
% data)
# XXX here we might end up with a ghost mitmproxy
sys.exit()
def tooltool_download(manifest, run_local, raptor_dir):
"""Download a file from tooltool using the provided tooltool manifest"""
def outputHandler(line):
LOG.info(line)
if run_local:
command = [
sys.executable, TOOLTOOL_PATH, "fetch", "-o", "-m", manifest
]
else:
# we want to use the tooltool cache in production
if os.environ.get("TOOLTOOLCACHE") is not None:
_cache = os.environ["TOOLTOOLCACHE"]
else:
# XXX top level dir? really?
# that gets run locally on any platform
# when you call ./mach python-test
_cache = "/builds/tooltool_cache"
command = [
sys.executable,
TOOLTOOL_PATH,
"fetch",
"-o",
"-m",
manifest,
"-c",
_cache,
]
proc = ProcessHandler(command,
processOutputLine=outputHandler,
storeOutput=False,
cwd=raptor_dir)
proc.run()
try:
proc.wait()
except Exception:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def run_process(self, cmd, cwd=None, dump=False):
def _processOutput(line):
if self.verbose or dump:
print(line)
if self.verbose:
self.build_obj.log(logging.INFO, "autophone", {},
"Running '%s' in '%s'" % (cmd, cwd))
proc = ProcessHandler(cmd, cwd=cwd, processOutputLine=_processOutput,
processStderrLine=_processOutput)
proc.run()
proc_complete = False
try:
proc.wait()
if proc.proc.returncode == 0:
proc_complete = True
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
if not proc_complete:
if not self.verbose:
print(proc.output)
return proc_complete
def verify_android_device(build_obj, install=False, xre=False, debugger=False, verbose=False):
"""
Determine if any Android device is connected via adb.
If no device is found, prompt to start an emulator.
If a device is found or an emulator started and 'install' is
specified, also check whether Firefox is installed on the
device; if not, prompt to install Firefox.
If 'xre' is specified, also check with MOZ_HOST_BIN is set
to a valid xre/host-utils directory; if not, prompt to set
one up.
If 'debugger' is specified, also check that JimDB is installed;
if JimDB is not found, prompt to set up JimDB.
Returns True if the emulator was started or another device was
already connected.
"""
device_verified = False
emulator = AndroidEmulator('*', substs=build_obj.substs, verbose=verbose)
devices = emulator.dm.devices()
if (len(devices) > 0) and ('device' in [d[1] for d in devices]):
device_verified = True
elif emulator.is_available():
response = raw_input(
"No Android devices connected. Start an emulator? (Y/n) ").strip()
if response.lower().startswith('y') or response == '':
if not emulator.check_avd():
_log_info("Fetching AVD. This may take a while...")
emulator.update_avd()
_log_info("Starting emulator running %s..." %
emulator.get_avd_description())
emulator.start()
emulator.wait_for_start()
device_verified = True
if device_verified and install:
# Determine if Firefox is installed on the device; if not,
# prompt to install. This feature allows a test command to
# launch an emulator, install Firefox, and proceed with testing
# in one operation. It is also a basic safeguard against other
# cases where testing is requested but Firefox installation has
# been forgotten.
# If Firefox is installed, there is no way to determine whether
# the current build is installed, and certainly no way to
# determine if the installed build is the desired build.
# Installing every time is problematic because:
# - it prevents testing against other builds (downloaded apk)
# - installation may take a couple of minutes.
installed = emulator.dm.shellCheckOutput(['pm', 'list',
'packages', 'org.mozilla.'])
if 'fennec' not in installed and 'firefox' not in installed:
response = raw_input(
"It looks like Firefox is not installed on this device.\n"
"Install Firefox? (Y/n) ").strip()
if response.lower().startswith('y') or response == '':
_log_info("Installing Firefox. This may take a while...")
build_obj._run_make(directory=".", target='install',
ensure_exit_code=False)
if device_verified and xre:
# Check whether MOZ_HOST_BIN has been set to a valid xre; if not,
# prompt to install one.
xre_path = os.environ.get('MOZ_HOST_BIN')
err = None
if not xre_path:
err = "environment variable MOZ_HOST_BIN is not set to a directory" \
"containing host xpcshell"
elif not os.path.isdir(xre_path):
err = '$MOZ_HOST_BIN does not specify a directory'
elif not os.path.isfile(os.path.join(xre_path, 'xpcshell')):
err = '$MOZ_HOST_BIN/xpcshell does not exist'
if err:
xre_path = glob.glob(os.path.join(EMULATOR_HOME_DIR, 'host-utils*'))
for path in xre_path:
if os.path.isdir(path) and os.path.isfile(os.path.join(path, 'xpcshell')):
os.environ['MOZ_HOST_BIN'] = path
err = None
break
if err:
_log_info("Host utilities not found: %s" % err)
response = raw_input(
"Download and setup your host utilities? (Y/n) ").strip()
if response.lower().startswith('y') or response == '':
_log_info("Installing host utilities. This may take a while...")
host_platform = _get_host_platform()
if host_platform:
path = os.path.join(MANIFEST_PATH, host_platform, 'hostutils.manifest')
_get_tooltool_manifest(build_obj.substs, path, EMULATOR_HOME_DIR,
'releng.manifest')
_tooltool_fetch()
xre_path = glob.glob(os.path.join(EMULATOR_HOME_DIR, 'host-utils*'))
for path in xre_path:
if os.path.isdir(path) and os.path.isfile(os.path.join(path, 'xpcshell')):
os.environ['MOZ_HOST_BIN'] = path
err = None
break
if err:
_log_warning("Unable to install host utilities.")
else:
_log_warning(
"Unable to install host utilities -- your platform is not supported!")
if debugger:
# Optionally set up JimDB. See https://wiki.mozilla.org/Mobile/Fennec/Android/GDB.
build_platform = _get_device_platform(build_obj.substs)
jimdb_path = os.path.join(EMULATOR_HOME_DIR, 'jimdb-%s' % build_platform)
jimdb_utils_path = os.path.join(jimdb_path, 'utils')
gdb_path = os.path.join(jimdb_path, 'bin', 'gdb')
err = None
if not os.path.isdir(jimdb_path):
err = '%s does not exist' % jimdb_path
elif not os.path.isfile(gdb_path):
err = '%s not found' % gdb_path
if err:
_log_info("JimDB (%s) not found: %s" % (build_platform, err))
response = raw_input(
"Download and setup JimDB (%s)? (Y/n) " % build_platform).strip()
if response.lower().startswith('y') or response == '':
host_platform = _get_host_platform()
if host_platform:
_log_info(
"Installing JimDB (%s/%s). This may take a while..." % (host_platform,
build_platform))
path = os.path.join(MANIFEST_PATH, host_platform,
'jimdb-%s.manifest' % build_platform)
_get_tooltool_manifest(build_obj.substs, path,
EMULATOR_HOME_DIR, 'releng.manifest')
_tooltool_fetch()
if os.path.isfile(gdb_path):
# Get JimDB utilities from git repository
proc = ProcessHandler(['git', 'pull'], cwd=jimdb_utils_path)
proc.run()
git_pull_complete = False
try:
proc.wait()
if proc.proc.returncode == 0:
git_pull_complete = True
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
if not git_pull_complete:
_log_warning("Unable to update JimDB utils from git -- "
"some JimDB features may be unavailable.")
else:
_log_warning("Unable to install JimDB -- unable to fetch from tooltool.")
else:
_log_warning("Unable to install JimDB -- your platform is not supported!")
if os.path.isfile(gdb_path):
# sync gdbinit.local with build settings
_update_gdbinit(build_obj.substs, os.path.join(jimdb_utils_path, "gdbinit.local"))
# ensure JimDB is in system path, so that mozdebug can find it
bin_path = os.path.join(jimdb_path, 'bin')
os.environ['PATH'] = "%s:%s" % (bin_path, os.environ['PATH'])
return device_verified
class AndroidEmulator(object):
"""
Support running the Android emulator with an AVD from Mozilla
test automation.
Example usage:
emulator = AndroidEmulator()
if not emulator.is_running() and emulator.is_available():
if not emulator.check_avd():
warn("this may take a while...")
emulator.update_avd()
emulator.start()
emulator.wait_for_start()
emulator.wait()
"""
def __init__(self, avd_type='4.3', verbose=False, substs=None, device_serial=None):
global verbose_logging
self.emulator_log = None
self.emulator_path = 'emulator'
verbose_logging = verbose
self.substs = substs
self.avd_type = self._get_avd_type(avd_type)
self.avd_info = AVD_DICT[self.avd_type]
self.gpu = True
self.restarted = False
adb_path = _find_sdk_exe(substs, 'adb', False)
if not adb_path:
adb_path = 'adb'
self.dm = DeviceManagerADB(autoconnect=False, adbPath=adb_path, retryLimit=1,
deviceSerial=device_serial)
self.dm.default_timeout = 10
_log_debug("Running on %s" % platform.platform())
_log_debug("Emulator created with type %s" % self.avd_type)
def __del__(self):
if self.emulator_log:
self.emulator_log.close()
def is_running(self):
"""
Returns True if the Android emulator is running.
"""
for proc in psutil.process_iter():
name = proc.name()
# On some platforms, "emulator" may start an emulator with
# process name "emulator64-arm" or similar.
if name and name.startswith('emulator'):
return True
return False
def is_available(self):
"""
Returns True if an emulator executable is found.
"""
found = False
emulator_path = _find_sdk_exe(self.substs, 'emulator', True)
if emulator_path:
self.emulator_path = emulator_path
found = True
return found
def check_avd(self, force=False):
"""
Determine if the AVD is already installed locally.
(This is usually used to determine if update_avd() is likely
to require a download; it is a convenient way of determining
whether a 'this may take a while' warning is warranted.)
Returns True if the AVD is installed.
"""
avd = os.path.join(
EMULATOR_HOME_DIR, 'avd', self.avd_info.name + '.avd')
if force and os.path.exists(avd):
shutil.rmtree(avd)
if os.path.exists(avd):
_log_debug("AVD found at %s" % avd)
return True
return False
def update_avd(self, force=False):
"""
If required, update the AVD via tooltool.
If the AVD directory is not found, or "force" is requested,
download the tooltool manifest associated with the AVD and then
invoke tooltool.py on the manifest. tooltool.py will download the
required archive (unless already present in the local tooltool
cache) and install the AVD.
"""
avd = os.path.join(
EMULATOR_HOME_DIR, 'avd', self.avd_info.name + '.avd')
ini_file = os.path.join(
EMULATOR_HOME_DIR, 'avd', self.avd_info.name + '.ini')
if force and os.path.exists(avd):
shutil.rmtree(avd)
if not os.path.exists(avd):
if os.path.exists(ini_file):
os.remove(ini_file)
path = self.avd_info.tooltool_manifest
_get_tooltool_manifest(self.substs, path, EMULATOR_HOME_DIR, 'releng.manifest')
_tooltool_fetch()
self._update_avd_paths()
def start(self):
"""
Launch the emulator.
"""
if os.path.exists(EMULATOR_AUTH_FILE):
os.remove(EMULATOR_AUTH_FILE)
_log_debug("deleted %s" % EMULATOR_AUTH_FILE)
# create an empty auth file to disable emulator authentication
auth_file = open(EMULATOR_AUTH_FILE, 'w')
auth_file.close()
def outputHandler(line):
self.emulator_log.write("<%s>\n" % line)
if "Invalid value for -gpu" in line or "Invalid GPU mode" in line:
self.gpu = False
env = os.environ
env['ANDROID_AVD_HOME'] = os.path.join(EMULATOR_HOME_DIR, "avd")
command = [self.emulator_path, "-avd",
self.avd_info.name, "-port", "5554"]
if self.gpu:
command += ['-gpu', 'swiftshader']
if self.avd_info.extra_args:
# -enable-kvm option is not valid on OSX
if _get_host_platform() == 'macosx64' and '-enable-kvm' in self.avd_info.extra_args:
self.avd_info.extra_args.remove('-enable-kvm')
command += self.avd_info.extra_args
log_path = os.path.join(EMULATOR_HOME_DIR, 'emulator.log')
self.emulator_log = open(log_path, 'w')
_log_debug("Starting the emulator with this command: %s" %
' '.join(command))
_log_debug("Emulator output will be written to '%s'" %
log_path)
self.proc = ProcessHandler(
command, storeOutput=False, processOutputLine=outputHandler,
env=env)
self.proc.run()
_log_debug("Emulator started with pid %d" %
int(self.proc.proc.pid))
def wait_for_start(self):
"""
Verify that the emulator is running, the emulator device is visible
to adb, and Android has booted.
"""
if not self.proc:
_log_warning("Emulator not started!")
return False
if self.check_completed():
return False
_log_debug("Waiting for device status...")
while(('emulator-5554', 'device') not in self.dm.devices()):
time.sleep(10)
if self.check_completed():
return False
_log_debug("Device status verified.")
_log_debug("Checking that Android has booted...")
complete = False
while(not complete):
output = ''
try:
output = self.dm.shellCheckOutput(
['getprop', 'sys.boot_completed'], timeout=5)
except DMError:
# adb not yet responding...keep trying
pass
if output.strip() == '1':
complete = True
else:
time.sleep(10)
if self.check_completed():
return False
_log_debug("Android boot status verified.")
if not self._verify_emulator():
return False
if self.avd_info.x86:
_log_info("Running the x86 emulator; be sure to install an x86 APK!")
else:
_log_info("Running the arm emulator; be sure to install an arm APK!")
return True
def check_completed(self):
if self.proc.proc.poll() is not None:
if not self.gpu and not self.restarted:
_log_warning("Emulator failed to start. Your emulator may be out of date.")
_log_warning("Trying to restart the emulator without -gpu argument.")
self.restarted = True
self.start()
return False
_log_warning("Emulator has already completed!")
log_path = os.path.join(EMULATOR_HOME_DIR, 'emulator.log')
_log_warning("See log at %s and/or use --verbose for more information." % log_path)
return True
return False
def wait(self):
"""
Wait for the emulator to close. If interrupted, close the emulator.
"""
try:
self.proc.wait()
except:
if self.proc.poll() is None:
self.cleanup()
return self.proc.poll()
def cleanup(self):
"""
Close the emulator.
"""
self.proc.kill(signal.SIGTERM)
def get_avd_description(self):
"""
Return the human-friendly description of this AVD.
"""
return self.avd_info.description
def _update_avd_paths(self):
avd_path = os.path.join(EMULATOR_HOME_DIR, "avd")
ini_file = os.path.join(avd_path, "test-1.ini")
ini_file_new = os.path.join(avd_path, self.avd_info.name + ".ini")
os.rename(ini_file, ini_file_new)
avd_dir = os.path.join(avd_path, "test-1.avd")
avd_dir_new = os.path.join(avd_path, self.avd_info.name + ".avd")
os.rename(avd_dir, avd_dir_new)
self._replace_ini_contents(ini_file_new)
def _replace_ini_contents(self, path):
with open(path, "r") as f:
lines = f.readlines()
with open(path, "w") as f:
for line in lines:
if line.startswith('path='):
avd_path = os.path.join(EMULATOR_HOME_DIR, "avd")
f.write('path=%s/%s.avd\n' %
(avd_path, self.avd_info.name))
elif line.startswith('path.rel='):
f.write('path.rel=avd/%s.avd\n' % self.avd_info.name)
else:
f.write(line)
def _telnet_cmd(self, telnet, command):
_log_debug(">>> " + command)
telnet.write('%s\n' % command)
result = telnet.read_until('OK', 10)
_log_debug("<<< " + result)
return result
def _verify_emulator(self):
telnet_ok = False
tn = None
while(not telnet_ok):
try:
tn = telnetlib.Telnet('localhost', 5554, 10)
if tn is not None:
tn.read_until('OK', 10)
self._telnet_cmd(tn, 'avd status')
self._telnet_cmd(tn, 'redir list')
self._telnet_cmd(tn, 'network status')
tn.write('quit\n')
tn.read_all()
telnet_ok = True
else:
_log_warning("Unable to connect to port 5554")
except:
_log_warning("Trying again after unexpected exception")
finally:
if tn is not None:
tn.close()
if not telnet_ok:
time.sleep(10)
if self.proc.proc.poll() is not None:
_log_warning("Emulator has already completed!")
return False
return telnet_ok
def _get_avd_type(self, requested):
if requested in AVD_DICT.keys():
return requested
if self.substs:
if not self.substs['TARGET_CPU'].startswith('arm'):
return 'x86'
return '4.3'
def start_mitmproxy_playback(self, mitmdump_path, mitmproxy_recording_path,
mitmproxy_recordings_list, browser_path):
"""Startup mitmproxy and replay the specified flow file"""
LOG.info("mitmdump path: %s" % mitmdump_path)
LOG.info("recording path: %s" % mitmproxy_recording_path)
LOG.info("recordings list: %s" % mitmproxy_recordings_list)
LOG.info("browser path: %s" % browser_path)
mitmproxy_recordings = []
# recording names can be provided in comma-separated list; build py list including path
for recording in mitmproxy_recordings_list:
if not os.path.isfile(
os.path.join(mitmproxy_recording_path, recording)):
LOG.critical('Recording file {} cannot be found!'.format(
os.path.join(mitmproxy_recording_path, recording)))
raise Exception('Recording file {} cannot be found!'.format(
os.path.join(mitmproxy_recording_path, recording)))
mitmproxy_recordings.append(
os.path.join(mitmproxy_recording_path, recording))
# cmd line to start mitmproxy playback using custom playback script is as follows:
# <path>/mitmdump -s "<path>mitmdump-alternate-server-replay/alternate-server-replay.py
# <path>recording-1.mp <path>recording-2.mp..."
param = os.path.join(here, 'alternate-server-replay.py')
env = os.environ.copy()
# this part is platform-specific
if mozinfo.os == 'win':
param2 = '""' + param.replace('\\', '\\\\\\') + ' ' + \
' '.join(mitmproxy_recordings).replace('\\', '\\\\\\') + '""'
sys.path.insert(1, mitmdump_path)
else:
# mac and linux
param2 = param + ' ' + ' '.join(mitmproxy_recordings)
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env["PATH"] = os.path.dirname(browser_path) + ";" + env["PATH"]
command = [mitmdump_path, '-k', '-q', '-s', param2]
LOG.info("Starting mitmproxy playback using env path: %s" %
env["PATH"])
LOG.info("Starting mitmproxy playback using command: %s" %
' '.join(command))
# to turn off mitmproxy log output, use these params for Popen:
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
mitmproxy_proc = ProcessHandler(command, env=env)
mitmproxy_proc.run()
time.sleep(MITMDUMP_SLEEP)
data = mitmproxy_proc.poll()
if data is None: # None value indicates process hasn't terminated
LOG.info("Mitmproxy playback successfully started as pid %d" %
mitmproxy_proc.pid)
return mitmproxy_proc
# cannot continue as we won't be able to playback the pages
LOG.error(
'Aborting: mitmproxy playback process failed to start, poll returned: %s'
% data)
sys.exit()
class ServoTestharnessExecutor(ProcessTestExecutor):
convert_result = testharness_result_converter
def __init__(self,
browser,
server_config,
timeout_multiplier=1,
debug_info=None,
pause_after_test=False):
ProcessTestExecutor.__init__(self,
browser,
server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.pause_after_test = pause_after_test
self.result_data = None
self.result_flag = None
self.protocol = Protocol(self, browser)
self.hosts_path = make_hosts_file()
def teardown(self):
try:
os.unlink(self.hosts_path)
except OSError:
pass
ProcessTestExecutor.teardown(self)
def do_test(self, test):
self.result_data = None
self.result_flag = threading.Event()
debug_args, command = browser_command(
self.binary, ["--cpu", "--hard-fail", "-z",
self.test_url(test)], self.debug_info)
self.command = command
if self.pause_after_test:
self.command.remove("-z")
self.command = debug_args + self.command
env = os.environ.copy()
env["HOST_FILE"] = self.hosts_path
if not self.interactive:
self.proc = ProcessHandler(self.command,
processOutputLine=[self.on_output],
onFinish=self.on_finish,
env=env,
storeOutput=False)
self.proc.run()
else:
self.proc = subprocess.Popen(self.command, env=env)
try:
timeout = test.timeout * self.timeout_multiplier
# Now wait to get the output we expect, or until we reach the timeout
if not self.interactive and not self.pause_after_test:
wait_timeout = timeout + 5
self.result_flag.wait(wait_timeout)
else:
wait_timeout = None
self.proc.wait()
proc_is_running = True
if self.result_flag.is_set() and self.result_data is not None:
self.result_data["test"] = test.url
result = self.convert_result(test, self.result_data)
else:
if self.proc.poll() is not None:
result = (test.result_cls("CRASH", None), [])
proc_is_running = False
else:
result = (test.result_cls("TIMEOUT", None), [])
if proc_is_running:
if self.pause_after_test:
self.logger.info("Pausing until the browser exits")
self.proc.wait()
else:
self.proc.kill()
except KeyboardInterrupt:
self.proc.kill()
raise
return result
def on_output(self, line):
prefix = "ALERT: RESULT: "
line = line.decode("utf8", "replace")
if line.startswith(prefix):
self.result_data = json.loads(line[len(prefix):])
self.result_flag.set()
else:
if self.interactive:
print line
else:
self.logger.process_output(self.proc.pid, line,
" ".join(self.command))
def on_finish(self):
self.result_flag.set()
class AndroidEmulator(object):
"""
Support running the Android emulator with an AVD from Mozilla
test automation.
Example usage:
emulator = AndroidEmulator()
if not emulator.is_running() and emulator.is_available():
if not emulator.check_avd():
warn("this may take a while...")
emulator.update_avd()
emulator.start()
emulator.wait_for_start()
emulator.wait()
"""
def __init__(self, avd_type="4.3", verbose=False, substs=None):
self.emulator_log = None
self.emulator_path = "emulator"
self.verbose = verbose
self.substs = substs
self.avd_type = self._get_avd_type(avd_type)
self.avd_info = AVD_DICT[self.avd_type]
adb_path = self._find_sdk_exe("adb", False)
if not adb_path:
adb_path = "adb"
self.dm = DeviceManagerADB(autoconnect=False, adbPath=adb_path, retryLimit=1)
self.dm.default_timeout = 10
self._log_debug("Emulator created with type %s" % self.avd_type)
def __del__(self):
if self.emulator_log:
self.emulator_log.close()
def is_running(self):
"""
Returns True if the Android emulator is running.
"""
for proc in psutil.process_iter():
name = proc.name()
# On some platforms, "emulator" may start an emulator with
# process name "emulator64-arm" or similar.
if name and name.startswith("emulator"):
return True
return False
def is_available(self):
"""
Returns True if an emulator executable is found.
"""
found = False
emulator_path = self._find_sdk_exe("emulator", True)
if emulator_path:
self.emulator_path = emulator_path
found = True
return found
def check_avd(self, force=False):
"""
Determine if the AVD is already installed locally.
(This is usually used to determine if update_avd() is likely
to require a download; it is a convenient way of determining
whether a 'this may take a while' warning is warranted.)
Returns True if the AVD is installed.
"""
avd = os.path.join(EMULATOR_HOME_DIR, "avd", self.avd_info.name + ".avd")
if force and os.path.exists(avd):
shutil.rmtree(avd)
if os.path.exists(avd):
self._log_debug("AVD found at %s" % avd)
return True
return False
def update_avd(self, force=False):
"""
If required, update the AVD via tooltool.
If the AVD directory is not found, or "force" is requested,
download the tooltool manifest associated with the AVD and then
invoke tooltool.py on the manifest. tooltool.py will download the
required archive (unless already present in the local tooltool
cache) and install the AVD.
"""
avd = os.path.join(EMULATOR_HOME_DIR, "avd", self.avd_info.name + ".avd")
if force and os.path.exists(avd):
shutil.rmtree(avd)
if not os.path.exists(avd):
self._fetch_tooltool()
self._fetch_tooltool_manifest()
self._tooltool_fetch()
self._update_avd_paths()
def start(self):
"""
Launch the emulator.
"""
def outputHandler(line):
self.emulator_log.write("<%s>\n" % line)
env = os.environ
env["ANDROID_AVD_HOME"] = os.path.join(EMULATOR_HOME_DIR, "avd")
command = [self.emulator_path, "-avd", self.avd_info.name, "-port", "5554"]
if self.avd_info.extra_args:
command += self.avd_info.extra_args
log_path = os.path.join(EMULATOR_HOME_DIR, "emulator.log")
self.emulator_log = open(log_path, "w")
self._log_debug("Starting the emulator with this command: %s" % " ".join(command))
self._log_debug("Emulator output will be written to '%s'" % log_path)
self.proc = ProcessHandler(command, storeOutput=False, processOutputLine=outputHandler, env=env)
self.proc.run()
self._log_debug("Emulator started with pid %d" % int(self.proc.proc.pid))
def wait_for_start(self):
"""
Verify that the emulator is running, the emulator device is visible
to adb, and Android has booted.
"""
if not self.proc:
self._log_warning("Emulator not started!")
return False
if self.proc.proc.poll() is not None:
self._log_warning("Emulator has already completed!")
return False
self._log_debug("Waiting for device status...")
while ("emulator-5554", "device") not in self.dm.devices():
time.sleep(10)
if self.proc.proc.poll() is not None:
self._log_warning("Emulator has already completed!")
return False
self._log_debug("Device status verified.")
self._log_debug("Checking that Android has booted...")
complete = False
while not complete:
output = ""
try:
output = self.dm.shellCheckOutput(["getprop", "sys.boot_completed"], timeout=5)
except DMError:
# adb not yet responding...keep trying
pass
if output.strip() == "1":
complete = True
else:
time.sleep(10)
if self.proc.proc.poll() is not None:
self._log_warning("Emulator has already completed!")
return False
self._log_debug("Android boot status verified.")
if not self._verify_emulator():
return False
if self.avd_info.uses_sut:
if not self._verify_sut():
return False
return True
def wait(self):
"""
Wait for the emulator to close. If interrupted, close the emulator.
"""
try:
self.proc.wait()
except:
if self.proc.poll() is None:
self.cleanup()
return self.proc.poll()
def cleanup(self):
"""
Close the emulator.
"""
self.proc.kill(signal.SIGTERM)
def get_avd_description(self):
"""
Return the human-friendly description of this AVD.
"""
return self.avd_info.description
def _log_debug(self, text):
if self.verbose:
print "DEBUG: %s" % text
def _log_warning(self, text):
print "WARNING: %s" % text
def _fetch_tooltool(self):
self._download_file(TOOLTOOL_URL, "tooltool.py", EMULATOR_HOME_DIR)
def _fetch_tooltool_manifest(self):
url = "https://hg.mozilla.org/%s/raw-file/%s/%s" % ("try", "default", self.avd_info.tooltool_manifest)
self._download_file(url, "releng.manifest", EMULATOR_HOME_DIR)
def _tooltool_fetch(self):
def outputHandler(line):
self._log_debug(line)
command = ["python", "tooltool.py", "fetch", "-m", "releng.manifest"]
proc = ProcessHandler(command, processOutputLine=outputHandler, storeOutput=False, cwd=EMULATOR_HOME_DIR)
proc.run()
try:
proc.wait()
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def _update_avd_paths(self):
avd_path = os.path.join(EMULATOR_HOME_DIR, "avd")
ini_file = os.path.join(avd_path, "test-1.ini")
ini_file_new = os.path.join(avd_path, self.avd_info.name + ".ini")
os.rename(ini_file, ini_file_new)
avd_dir = os.path.join(avd_path, "test-1.avd")
avd_dir_new = os.path.join(avd_path, self.avd_info.name + ".avd")
os.rename(avd_dir, avd_dir_new)
self._replace_ini_contents(ini_file_new)
def _download_file(self, url, filename, path):
f = urllib2.urlopen(url)
if not os.path.isdir(path):
try:
os.makedirs(path)
except Exception, e:
self._log_warning(str(e))
return False
local_file = open(os.path.join(path, filename), "wb")
local_file.write(f.read())
local_file.close()
self._log_debug("Downloaded %s to %s/%s" % (url, path, filename))
return True
class AndroidEmulator(object):
"""
Support running the Android emulator with an AVD from Mozilla
test automation.
Example usage:
emulator = AndroidEmulator()
if not emulator.is_running() and emulator.is_available():
if not emulator.check_avd():
warn("this may take a while...")
emulator.update_avd()
emulator.start()
emulator.wait_for_start()
emulator.wait()
"""
def __init__(self,
avd_type='4.3',
verbose=False,
substs=None,
device_serial=None):
global verbose_logging
self.emulator_log = None
self.emulator_path = 'emulator'
verbose_logging = verbose
self.substs = substs
self.avd_type = self._get_avd_type(avd_type)
self.avd_info = AVD_DICT[self.avd_type]
adb_path = _find_sdk_exe(substs, 'adb', False)
if not adb_path:
adb_path = 'adb'
self.dm = DeviceManagerADB(autoconnect=False,
adbPath=adb_path,
retryLimit=1,
deviceSerial=device_serial)
self.dm.default_timeout = 10
_log_debug("Emulator created with type %s" % self.avd_type)
def __del__(self):
if self.emulator_log:
self.emulator_log.close()
def is_running(self):
"""
Returns True if the Android emulator is running.
"""
for proc in psutil.process_iter():
name = proc.name()
# On some platforms, "emulator" may start an emulator with
# process name "emulator64-arm" or similar.
if name and name.startswith('emulator'):
return True
return False
def is_available(self):
"""
Returns True if an emulator executable is found.
"""
found = False
emulator_path = _find_sdk_exe(self.substs, 'emulator', True)
if emulator_path:
self.emulator_path = emulator_path
found = True
return found
def check_avd(self, force=False):
"""
Determine if the AVD is already installed locally.
(This is usually used to determine if update_avd() is likely
to require a download; it is a convenient way of determining
whether a 'this may take a while' warning is warranted.)
Returns True if the AVD is installed.
"""
avd = os.path.join(EMULATOR_HOME_DIR, 'avd',
self.avd_info.name + '.avd')
if force and os.path.exists(avd):
shutil.rmtree(avd)
if os.path.exists(avd):
_log_debug("AVD found at %s" % avd)
return True
return False
def update_avd(self, force=False):
"""
If required, update the AVD via tooltool.
If the AVD directory is not found, or "force" is requested,
download the tooltool manifest associated with the AVD and then
invoke tooltool.py on the manifest. tooltool.py will download the
required archive (unless already present in the local tooltool
cache) and install the AVD.
"""
avd = os.path.join(EMULATOR_HOME_DIR, 'avd',
self.avd_info.name + '.avd')
if force and os.path.exists(avd):
shutil.rmtree(avd)
if not os.path.exists(avd):
_download_file(TOOLTOOL_URL, 'tooltool.py', EMULATOR_HOME_DIR)
url = '%s/%s' % (TRY_URL, self.avd_info.tooltool_manifest)
_download_file(url, 'releng.manifest', EMULATOR_HOME_DIR)
_tooltool_fetch()
self._update_avd_paths()
def start(self):
"""
Launch the emulator.
"""
def outputHandler(line):
self.emulator_log.write("<%s>\n" % line)
env = os.environ
env['ANDROID_AVD_HOME'] = os.path.join(EMULATOR_HOME_DIR, "avd")
command = [
self.emulator_path, "-avd", self.avd_info.name, "-port", "5554"
]
if self.avd_info.extra_args:
# -enable-kvm option is not valid on OSX
if _get_host_platform(
) == 'macosx64' and '-enable-kvm' in self.avd_info.extra_args:
self.avd_info.extra_args.remove('-enable-kvm')
command += self.avd_info.extra_args
log_path = os.path.join(EMULATOR_HOME_DIR, 'emulator.log')
self.emulator_log = open(log_path, 'w')
_log_debug("Starting the emulator with this command: %s" %
' '.join(command))
_log_debug("Emulator output will be written to '%s'" % log_path)
self.proc = ProcessHandler(command,
storeOutput=False,
processOutputLine=outputHandler,
env=env)
self.proc.run()
_log_debug("Emulator started with pid %d" % int(self.proc.proc.pid))
def wait_for_start(self):
"""
Verify that the emulator is running, the emulator device is visible
to adb, and Android has booted.
"""
if not self.proc:
_log_warning("Emulator not started!")
return False
if self.proc.proc.poll() is not None:
_log_warning("Emulator has already completed!")
return False
_log_debug("Waiting for device status...")
while (('emulator-5554', 'device') not in self.dm.devices()):
time.sleep(10)
if self.proc.proc.poll() is not None:
_log_warning("Emulator has already completed!")
return False
_log_debug("Device status verified.")
_log_debug("Checking that Android has booted...")
complete = False
while (not complete):
output = ''
try:
output = self.dm.shellCheckOutput(
['getprop', 'sys.boot_completed'], timeout=5)
except DMError:
# adb not yet responding...keep trying
pass
if output.strip() == '1':
complete = True
else:
time.sleep(10)
if self.proc.proc.poll() is not None:
_log_warning("Emulator has already completed!")
return False
_log_debug("Android boot status verified.")
if not self._verify_emulator():
return False
if self.avd_info.uses_sut:
if not self._verify_sut():
return False
return True
def wait(self):
"""
Wait for the emulator to close. If interrupted, close the emulator.
"""
try:
self.proc.wait()
except:
if self.proc.poll() is None:
self.cleanup()
return self.proc.poll()
def cleanup(self):
"""
Close the emulator.
"""
self.proc.kill(signal.SIGTERM)
def get_avd_description(self):
"""
Return the human-friendly description of this AVD.
"""
return self.avd_info.description
def _update_avd_paths(self):
avd_path = os.path.join(EMULATOR_HOME_DIR, "avd")
ini_file = os.path.join(avd_path, "test-1.ini")
ini_file_new = os.path.join(avd_path, self.avd_info.name + ".ini")
os.rename(ini_file, ini_file_new)
avd_dir = os.path.join(avd_path, "test-1.avd")
avd_dir_new = os.path.join(avd_path, self.avd_info.name + ".avd")
os.rename(avd_dir, avd_dir_new)
self._replace_ini_contents(ini_file_new)
def _replace_ini_contents(self, path):
with open(path, "r") as f:
lines = f.readlines()
with open(path, "w") as f:
for line in lines:
if line.startswith('path='):
avd_path = os.path.join(EMULATOR_HOME_DIR, "avd")
f.write('path=%s/%s.avd\n' %
(avd_path, self.avd_info.name))
elif line.startswith('path.rel='):
f.write('path.rel=avd/%s.avd\n' % self.avd_info.name)
else:
f.write(line)
def _telnet_cmd(self, telnet, command):
_log_debug(">>> " + command)
telnet.write('%s\n' % command)
result = telnet.read_until('OK', 10)
_log_debug("<<< " + result)
return result
def _verify_emulator(self):
telnet_ok = False
tn = None
while (not telnet_ok):
try:
tn = telnetlib.Telnet('localhost', self.avd_info.port, 10)
if tn is not None:
res = tn.read_until('OK', 10)
self._telnet_cmd(tn, 'avd status')
if self.avd_info.uses_sut:
cmd = 'redir add tcp:%s:%s' % \
(str(self.avd_info.sut_port),
str(self.avd_info.sut_port))
self._telnet_cmd(tn, cmd)
cmd = 'redir add tcp:%s:%s' % \
(str(self.avd_info.sut_port2),
str(self.avd_info.sut_port2))
self._telnet_cmd(tn, cmd)
self._telnet_cmd(tn, 'redir list')
self._telnet_cmd(tn, 'network status')
tn.write('quit\n')
tn.read_all()
telnet_ok = True
else:
_log_warning("Unable to connect to port %d" % port)
except:
_log_warning("Trying again after unexpected exception")
finally:
if tn is not None:
tn.close()
if not telnet_ok:
time.sleep(10)
if self.proc.proc.poll() is not None:
_log_warning("Emulator has already completed!")
return False
return telnet_ok
def _verify_sut(self):
sut_ok = False
while (not sut_ok):
try:
tn = telnetlib.Telnet('localhost', self.avd_info.sut_port, 10)
if tn is not None:
_log_debug("Connected to port %d" % self.avd_info.sut_port)
res = tn.read_until('$>', 10)
if res.find('$>') == -1:
_log_debug("Unexpected SUT response: %s" % res)
else:
_log_debug("SUT response: %s" % res)
sut_ok = True
tn.write('quit\n')
tn.read_all()
except:
_log_debug("Caught exception while verifying sutagent")
finally:
if tn is not None:
tn.close()
if not sut_ok:
time.sleep(10)
if self.proc.proc.poll() is not None:
_log_warning("Emulator has already completed!")
return False
return sut_ok
def _get_avd_type(self, requested):
if requested in AVD_DICT.keys():
return requested
if self.substs:
if not self.substs['TARGET_CPU'].startswith('arm'):
return 'x86'
if self.substs['MOZ_ANDROID_MIN_SDK_VERSION'] == '9':
return '2.3'
return '4.3'
def start_mitmproxy_playback(
self,
mitmdump_path,
mitmproxy_recording_path,
mitmproxy_recordings_list,
browser_path,
):
"""Startup mitmproxy and replay the specified flow file"""
LOG.info("mitmdump path: %s" % mitmdump_path)
LOG.info("recording path: %s" % mitmproxy_recording_path)
LOG.info("recordings list: %s" % mitmproxy_recordings_list)
LOG.info("browser path: %s" % browser_path)
mitmproxy_recordings = []
# recording names can be provided in comma-separated list; build py list including path
for recording in mitmproxy_recordings_list:
if not os.path.isfile(
os.path.join(mitmproxy_recording_path, recording)):
LOG.critical("Recording file {} cannot be found!".format(
os.path.join(mitmproxy_recording_path, recording)))
raise Exception("Recording file {} cannot be found!".format(
os.path.join(mitmproxy_recording_path, recording)))
mitmproxy_recordings.append(
os.path.join(mitmproxy_recording_path, recording))
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
env["PATH"] = os.path.dirname(browser_path) + ";" + env["PATH"]
command = [mitmdump_path, "-k", "-q"]
if "custom_script" in self.config:
# cmd line to start mitmproxy playback using custom playback script is as follows:
# <path>/mitmdump -s "<path>/alternate-server-replay.py
# <path>recording-1.mp <path>recording-2.mp..."
custom_script = self.config["custom_script"] + " " + " ".join(
mitmproxy_recordings)
# this part is platform-specific
if mozinfo.os == "win":
custom_script = '""' + custom_script.replace("\\",
"\\\\\\") + '""'
sys.path.insert(1, mitmdump_path)
command.extend(["-s", custom_script])
LOG.info("Starting mitmproxy playback using env path: %s" %
env["PATH"])
LOG.info("Starting mitmproxy playback using command: %s" %
" ".join(command))
# to turn off mitmproxy log output, use these params for Popen:
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
mitmproxy_proc = ProcessHandler(command, env=env)
mitmproxy_proc.run()
# XXX replace the code below with a loop with a connection attempt
# Bug 1532557
time.sleep(MITMDUMP_SLEEP)
data = mitmproxy_proc.poll()
if data is None: # None value indicates process hasn't terminated
LOG.info("Mitmproxy playback successfully started as pid %d" %
mitmproxy_proc.pid)
return mitmproxy_proc
# cannot continue as we won't be able to playback the pages
LOG.error(
"Aborting: mitmproxy playback process failed to start, poll returned: %s"
% data)
# XXX here we might end up with a ghost mitmproxy
sys.exit()
def run_browser(command, minidump_dir, timeout=None, on_started=None,
**kwargs):
"""
Run the browser using the given `command`.
After the browser prints __endTimestamp, we give it 5
seconds to quit and kill it if it's still alive at that point.
Note that this method ensure that the process is killed at
the end. If this is not possible, an exception will be raised.
:param command: the commad (as a string list) to run the browser
:param minidump_dir: a path where to extract minidumps in case the
browser hang. This have to be the same value
used in `mozcrash.check_for_crashes`.
:param timeout: if specified, timeout to wait for the browser before
we raise a :class:`TalosError`
:param on_started: a callback that can be used to do things just after
the browser has been started. The callback must takes
an argument, which is the psutil.Process instance
:param kwargs: additional keyword arguments for the :class:`ProcessHandler`
instance
Returns a ProcessContext instance, with available output and pid used.
"""
context = ProcessContext()
first_time = int(time.time()) * 1000
wait_for_quit_timeout = 5
event = Event()
reader = Reader(event)
kwargs['storeOutput'] = False
kwargs['processOutputLine'] = reader
kwargs['onFinish'] = event.set
proc = ProcessHandler(command, **kwargs)
reader.proc = proc
proc.run()
LOG.process_start(proc.pid, ' '.join(command))
try:
context.process = psutil.Process(proc.pid)
if on_started:
on_started(context.process)
# wait until we saw __endTimestamp in the proc output,
# or the browser just terminated - or we have a timeout
if not event.wait(timeout):
# try to extract the minidump stack if the browser hangs
mozcrash.kill_and_get_minidump(proc.pid, minidump_dir)
raise TalosError("timeout")
if reader.got_end_timestamp:
for i in range(1, wait_for_quit_timeout):
if proc.wait(1) is not None:
break
if proc.poll() is None:
LOG.info(
"Browser shutdown timed out after {0} seconds, terminating"
" process.".format(wait_for_quit_timeout)
)
elif reader.got_timeout:
raise TalosError('TIMEOUT: %s' % reader.timeout_message)
finally:
# this also handle KeyboardInterrupt
# ensure early the process is really terminated
return_code = context.kill_process()
if return_code is None:
return_code = proc.wait(1)
reader.output.append(
"__startBeforeLaunchTimestamp%d__endBeforeLaunchTimestamp"
% first_time)
reader.output.append(
"__startAfterTerminationTimestamp%d__endAfterTerminationTimestamp"
% (int(time.time()) * 1000))
if return_code is not None:
LOG.process_exit(proc.pid, return_code)
else:
LOG.debug("Unable to detect exit code of the process %s." % proc.pid)
context.output = reader.output
return context
class Mitmproxy(Playback):
def __init__(self, config):
self.config = config
self.host = ("127.0.0.1" if "localhost" in self.config["host"] else
self.config["host"])
self.port = None
self.mitmproxy_proc = None
self.mitmdump_path = None
self.browser_path = ""
if config.get("binary", None):
self.browser_path = os.path.normpath(config.get("binary"))
self.policies_dir = None
self.ignore_mitmdump_exit_failure = config.get(
"ignore_mitmdump_exit_failure", False)
self.recording_paths = None
if self.config.get("playback_version") is None:
LOG.info("mitmproxy was not provided with a 'playback_version' "
"Using default playback version: 4.0.4")
self.config["playback_version"] = "4.0.4"
if self.config.get("playback_binary_manifest") is None:
LOG.info(
"mitmproxy was not provided with a 'playback_binary_manifest' "
"Using default playback_binary_manifest")
self.config["playback_binary_manifest"] = (
"mitmproxy-rel-bin-%s-{platform}.manifest" %
self.config["playback_version"])
# mozproxy_dir is where we will download all mitmproxy required files
# when running locally it comes from obj_path via mozharness/mach
if self.config.get("obj_path") is not None:
self.mozproxy_dir = self.config.get("obj_path")
else:
# in production it is ../tasks/task_N/build/, in production that dir
# is not available as an envvar, however MOZ_UPLOAD_DIR is set as
# ../tasks/task_N/build/blobber_upload_dir so take that and go up 1 level
self.mozproxy_dir = os.path.dirname(
os.path.dirname(os.environ["MOZ_UPLOAD_DIR"]))
self.mozproxy_dir = os.path.join(self.mozproxy_dir, "testing",
"mozproxy")
self.upload_dir = os.environ.get("MOZ_UPLOAD_DIR", self.mozproxy_dir)
LOG.info(
"mozproxy_dir used for mitmproxy downloads and exe files: %s" %
self.mozproxy_dir)
# setting up the MOZPROXY_DIR env variable so custom scripts know
# where to get the data
os.environ["MOZPROXY_DIR"] = self.mozproxy_dir
LOG.info("Playback tool: %s" % self.config["playback_tool"])
LOG.info("Playback tool version: %s" % self.config["playback_version"])
def start(self):
# go ahead and download and setup mitmproxy
self.download()
# mitmproxy must be started before setup, so that the CA cert is available
self.start_mitmproxy_playback(self.mitmdump_path, self.browser_path)
# In case the setup fails, we want to stop the process before raising.
try:
self.setup()
except Exception:
self.stop()
raise
def download(self):
"""Download and unpack mitmproxy binary and pageset using tooltool"""
if not os.path.exists(self.mozproxy_dir):
os.makedirs(self.mozproxy_dir)
_manifest = os.path.join(here, self.config["playback_binary_manifest"])
transformed_manifest = transform_platform(_manifest,
self.config["platform"])
# generate the mitmdump_path
self.mitmdump_path = os.path.normpath(
os.path.join(
self.mozproxy_dir,
"mitmdump-%s" % self.config["playback_version"],
"mitmdump",
))
# Check if mitmproxy bin exists
if os.path.exists(self.mitmdump_path):
LOG.info("mitmproxy binary already exists. Skipping download")
else:
# Download and unpack mitmproxy binary
download_path = os.path.dirname(self.mitmdump_path)
LOG.info("create mitmproxy %s dir" %
self.config["playback_version"])
if not os.path.exists(download_path):
os.makedirs(download_path)
LOG.info("downloading mitmproxy binary")
tooltool_download(transformed_manifest, self.config["run_local"],
download_path)
if "playback_pageset_manifest" in self.config:
# we use one pageset for all platforms
LOG.info("downloading mitmproxy pageset")
_manifest = self.config["playback_pageset_manifest"]
transformed_manifest = transform_platform(_manifest,
self.config["platform"])
tooltool_download(transformed_manifest, self.config["run_local"],
self.mozproxy_dir)
if "playback_artifacts" in self.config:
artifacts = self.config["playback_artifacts"].split(",")
for artifact in artifacts:
artifact = artifact.strip()
if not artifact:
continue
artifact_name = artifact.split("/")[-1]
if artifact_name.endswith(".manifest"):
tooltool_download(artifact, self.config["run_local"],
self.mozproxy_dir)
else:
dest = os.path.join(self.mozproxy_dir, artifact_name)
download_file_from_url(artifact, dest, extract=True)
def stop(self):
self.stop_mitmproxy_playback()
def start_mitmproxy_playback(self, mitmdump_path, browser_path):
"""Startup mitmproxy and replay the specified flow file"""
if self.mitmproxy_proc is not None:
raise Exception("Proxy already started.")
self.port = get_available_port()
LOG.info("mitmdump path: %s" % mitmdump_path)
LOG.info("browser path: %s" % browser_path)
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
env["PATH"] = os.path.dirname(browser_path) + os.pathsep + env["PATH"]
command = [mitmdump_path]
# add proxy host and port options
command.extend(
["--listen-host", self.host, "--listen-port",
str(self.port)])
if "playback_tool_args" in self.config:
LOG.info("Staring Proxy using provided command line!")
command.extend(self.config["playback_tool_args"])
elif "playback_files" in self.config:
script = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"scripts",
"alternate-server-replay.py",
)
self.recording_paths = [
normalize_path(recording_path)
for recording_path in self.config["playback_files"]
]
if self.config["playback_version"] in ["4.0.4", "5.0.1"]:
args = [
"-v",
"--set",
"upstream_cert=false",
"--set",
"upload_dir=" + normalize_path(self.upload_dir),
"--set",
"websocket=false",
"--set",
"server_replay_files={}".format(",".join(
self.recording_paths)),
"--scripts",
normalize_path(script),
]
command.extend(args)
else:
raise Exception("Mitmproxy version is unknown!")
else:
raise Exception(
"Mitmproxy can't start playback! Playback settings missing.")
LOG.info("Starting mitmproxy playback using env path: %s" %
env["PATH"])
LOG.info("Starting mitmproxy playback using command: %s" %
" ".join(command))
# to turn off mitmproxy log output, use these params for Popen:
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
self.mitmproxy_proc = ProcessHandler(
command,
logfile=os.path.join(self.upload_dir, "mitmproxy.log"),
env=env,
processStderrLine=LOG.error,
storeOutput=False,
)
self.mitmproxy_proc.run()
end_time = time.time() + MITMDUMP_COMMAND_TIMEOUT
ready = False
while time.time() < end_time:
ready = self.check_proxy(host=self.host, port=self.port)
if ready:
LOG.info(
"Mitmproxy playback successfully started on %s:%d as pid %d"
% (self.host, self.port, self.mitmproxy_proc.pid))
return
time.sleep(0.25)
# cannot continue as we won't be able to playback the pages
LOG.error("Aborting: Mitmproxy process did not startup")
self.stop_mitmproxy_playback()
sys.exit() # XXX why do we need to do that? a raise is not enough?
def stop_mitmproxy_playback(self):
"""Stop the mitproxy server playback"""
if self.mitmproxy_proc is None or self.mitmproxy_proc.poll(
) is not None:
return
LOG.info("Stopping mitmproxy playback, killing process %d" %
self.mitmproxy_proc.pid)
# On Windows, mozprocess brutally kills mitmproxy with TerminateJobObject
# The process has no chance to gracefully shutdown.
# Here, we send the process a break event to give it a chance to wrapup.
# See the signal handler in the alternate-server-replay-4.0.4.py script
if mozinfo.os == "win":
LOG.info("Sending CTRL_BREAK_EVENT to mitmproxy")
os.kill(self.mitmproxy_proc.pid, signal.CTRL_BREAK_EVENT)
time.sleep(2)
exit_code = self.mitmproxy_proc.kill()
self.mitmproxy_proc = None
if exit_code != 0:
if exit_code is None:
LOG.error("Failed to kill the mitmproxy playback process")
return
if mozinfo.os == "win":
from mozprocess.winprocess import ERROR_CONTROL_C_EXIT # noqa
if exit_code == ERROR_CONTROL_C_EXIT:
LOG.info(
"Successfully killed the mitmproxy playback process"
" with exit code %d" % exit_code)
return
log_func = LOG.error
if self.ignore_mitmdump_exit_failure:
log_func = LOG.info
log_func("Mitmproxy exited with error code %d" % exit_code)
else:
LOG.info("Successfully killed the mitmproxy playback process")
def check_proxy(self, host, port):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((host, port))
s.shutdown(socket.SHUT_RDWR)
s.close()
return True
except socket.error:
return False
def confidence(self):
file_name = "mitm_netlocs_%s.json" % os.path.splitext(
os.path.basename(self.recording_paths[0]))[0]
path = os.path.normpath(os.path.join(self.upload_dir, file_name))
if os.path.exists(path):
try:
LOG.info("Reading confidence values from: %s" % path)
with open(path, "r") as f:
data = json.load(f)
return {
"confidence": data["confidence"],
"not-replayed": data["not-replayed"],
"replayed": data["replayed"]
}
except Exception:
LOG.info("Can't read netlocs file!", exc_info=True)
return None
else:
LOG.info("Netlocs file is not available! Cant find %s" % path)
return None
class Mitmproxy(Playback):
def __init__(self, config):
self.config = config
self.host = ("127.0.0.1" if "localhost" in self.config["host"] else
self.config["host"])
self.port = None
self.mitmproxy_proc = None
self.mitmdump_path = None
self.record_mode = config.get("record", False)
self.recording = None
self.playback_files = []
self.browser_path = ""
if config.get("binary", None):
self.browser_path = os.path.normpath(config.get("binary"))
self.policies_dir = None
self.ignore_mitmdump_exit_failure = config.get(
"ignore_mitmdump_exit_failure", False)
if self.record_mode:
if "recording_file" not in self.config:
LOG.error(
"recording_file value was not provided. Proxy service wont' start "
)
raise Exception("Please provide a playback_files list.")
if not isinstance(self.config.get("recording_file"),
six.string_types):
LOG.error("recording_file argument type is not str!")
raise Exception("recording_file argument type invalid!")
if not os.path.splitext(
self.config.get("recording_file"))[1] == ".zip":
LOG.error("Recording file type (%s) should be a zip. "
"Please provide a valid file type!" %
self.config.get("recording_file"))
raise Exception("Recording file type should be a zip")
if os.path.exists(self.config.get("recording_file")):
LOG.error("Recording file (%s) already exists."
"Please provide a valid file path!" %
self.config.get("recording_file"))
raise Exception("Recording file already exists.")
if self.config.get("playback_files", False):
LOG.error(
"Record mode is True and playback_files where provided!")
raise Exception("playback_files specified during record!")
if self.config.get("playback_version") is None:
LOG.error("mitmproxy was not provided with a 'playback_version' "
"Please provide a valid playback version")
raise Exception("playback_version not specified!")
# mozproxy_dir is where we will download all mitmproxy required files
# when running locally it comes from obj_path via mozharness/mach
if self.config.get("obj_path") is not None:
self.mozproxy_dir = self.config.get("obj_path")
else:
# in production it is ../tasks/task_N/build/, in production that dir
# is not available as an envvar, however MOZ_UPLOAD_DIR is set as
# ../tasks/task_N/build/blobber_upload_dir so take that and go up 1 level
self.mozproxy_dir = os.path.dirname(
os.path.dirname(os.environ["MOZ_UPLOAD_DIR"]))
self.mozproxy_dir = os.path.join(self.mozproxy_dir, "testing",
"mozproxy")
self.upload_dir = os.environ.get("MOZ_UPLOAD_DIR", self.mozproxy_dir)
LOG.info(
"mozproxy_dir used for mitmproxy downloads and exe files: %s" %
self.mozproxy_dir)
# setting up the MOZPROXY_DIR env variable so custom scripts know
# where to get the data
os.environ["MOZPROXY_DIR"] = self.mozproxy_dir
LOG.info("Playback tool: %s" % self.config["playback_tool"])
LOG.info("Playback tool version: %s" % self.config["playback_version"])
def download_mitm_bin(self):
# Download and setup mitm binaries
manifest = os.path.join(
here,
"manifests",
"mitmproxy-rel-bin-%s-{platform}.manifest" %
self.config["playback_version"],
)
transformed_manifest = transform_platform(manifest,
self.config["platform"])
# generate the mitmdump_path
self.mitmdump_path = os.path.normpath(
os.path.join(
self.mozproxy_dir,
"mitmdump-%s" % self.config["playback_version"],
"mitmdump",
))
# Check if mitmproxy bin exists
if os.path.exists(self.mitmdump_path):
LOG.info("mitmproxy binary already exists. Skipping download")
else:
# Download and unpack mitmproxy binary
download_path = os.path.dirname(self.mitmdump_path)
LOG.info("create mitmproxy %s dir" %
self.config["playback_version"])
if not os.path.exists(download_path):
os.makedirs(download_path)
LOG.info("downloading mitmproxy binary")
tooltool_download(transformed_manifest, self.config["run_local"],
download_path)
def download_manifest_file(self, manifest_path):
# Manifest File
# we use one pageset for all platforms
LOG.info("downloading mitmproxy pageset")
tooltool_download(manifest_path, self.config["run_local"],
self.mozproxy_dir)
with open(manifest_path) as manifest_file:
manifest = json.load(manifest_file)
for file in manifest:
zip_path = os.path.join(self.mozproxy_dir, file["filename"])
LOG.info("Adding %s to recording list" % zip_path)
self.playback_files.append(RecordingFile(zip_path))
def download_playback_files(self):
# Detect type of file from playback_files and download accordingly
if "playback_files" not in self.config:
LOG.error(
"playback_files value was not provided. Proxy service wont' start "
)
raise Exception("Please provide a playback_files list.")
if not isinstance(self.config["playback_files"], list):
LOG.error("playback_files should be a list")
raise Exception("playback_files should be a list")
for playback_file in self.config["playback_files"]:
if playback_file.startswith(
"https://") and "mozilla.com" in playback_file:
# URL provided
dest = os.path.join(self.mozproxy_dir,
os.path.basename(playback_file))
download_file_from_url(playback_file,
self.mozproxy_dir,
extract=False)
# Add Downloaded file to playback_files list
LOG.info("Adding %s to recording list" % dest)
self.playback_files.append(RecordingFile(dest))
continue
if not os.path.exists(playback_file):
LOG.error(
"Zip or manifest file path (%s) does not exist. Please provide a valid path!"
% playback_file)
raise Exception("Zip or manifest file path does not exist")
if os.path.splitext(playback_file)[1] == ".zip":
# zip file path provided
LOG.info("Adding %s to recording list" % playback_file)
self.playback_files.append(RecordingFile(playback_file))
elif os.path.splitext(playback_file)[1] == ".manifest":
# manifest file path provided
self.download_manifest_file(playback_file)
def download(self):
"""Download and unpack mitmproxy binary and pageset using tooltool"""
if not os.path.exists(self.mozproxy_dir):
os.makedirs(self.mozproxy_dir)
self.download_mitm_bin()
if self.record_mode:
self.recording = RecordingFile(self.config["recording_file"])
else:
self.download_playback_files()
def stop(self):
LOG.info("Mitmproxy stop!!")
self.stop_mitmproxy_playback()
if self.record_mode:
LOG.info("Record mode ON. Generating zip file ")
self.recording.generate_zip_file()
def wait(self, timeout=1):
"""Wait until the mitmproxy process has terminated."""
# We wait using this method to allow Windows to respond to the Ctrl+Break
# signal so that we can exit cleanly from the command-line driver.
while True:
returncode = self.mitmproxy_proc.wait(timeout)
if returncode is not None:
return returncode
def start(self):
# go ahead and download and setup mitmproxy
self.download()
# mitmproxy must be started before setup, so that the CA cert is available
self.start_mitmproxy(self.mitmdump_path, self.browser_path)
# In case the setup fails, we want to stop the process before raising.
try:
self.setup()
except Exception:
try:
self.stop()
except Exception:
LOG.error("MitmProxy failed to STOP.", exc_info=True)
LOG.error("Setup of MitmProxy failed.", exc_info=True)
raise
def start_mitmproxy(self, mitmdump_path, browser_path):
"""Startup mitmproxy and replay the specified flow file"""
if self.mitmproxy_proc is not None:
raise Exception("Proxy already started.")
self.port = get_available_port()
LOG.info("mitmdump path: %s" % mitmdump_path)
LOG.info("browser path: %s" % browser_path)
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
env["PATH"] = os.path.dirname(browser_path) + os.pathsep + env["PATH"]
command = [mitmdump_path]
if self.config.get("verbose", False):
# Generate mitmproxy verbose logs
command.extend(["-v"])
# add proxy host and port options
command.extend(
["--listen-host", self.host, "--listen-port",
str(self.port)])
# record mode
if self.record_mode:
# generate recording script paths
inject_deterministic = os.path.join(
mitm_folder,
"scripts",
"inject-deterministic.py",
)
http_protocol_extractor = os.path.join(
mitm_folder,
"scripts",
"http_protocol_extractor.py",
)
args = [
"--save-stream-file",
normalize_path(self.recording.recording_path),
"--set",
"websocket=false",
"--scripts",
inject_deterministic,
"--scripts",
http_protocol_extractor,
]
command.extend(args)
self.recording.set_metadata("proxy_version",
self.config["playback_version"])
else:
# playback mode
if len(self.playback_files) > 0:
script = os.path.join(
mitm_folder,
"scripts",
"alternate-server-replay.py",
)
if self.config["playback_version"] in [
"4.0.4", "5.1.1", "6.0.2"
]:
args = [
"--set",
"upstream_cert=false",
"--set",
"upload_dir=" + normalize_path(self.upload_dir),
"--set",
"websocket=false",
"--set",
"server_replay_files={}".format(",".join([
normalize_path(playback_file.recording_path)
for playback_file in self.playback_files
])),
"--scripts",
normalize_path(script),
]
command.extend(args)
else:
raise Exception("Mitmproxy version is unknown!")
else:
raise Exception(
"Mitmproxy can't start playback! Playback settings missing."
)
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
if not os.path.dirname(self.browser_path) in env["PATH"]:
env["PATH"] = os.path.dirname(
self.browser_path) + os.pathsep + env["PATH"]
LOG.info("Starting mitmproxy playback using env path: %s" %
env["PATH"])
LOG.info("Starting mitmproxy playback using command: %s" %
" ".join(command))
# to turn off mitmproxy log output, use these params for Popen:
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
self.mitmproxy_proc = ProcessHandler(
command,
logfile=os.path.join(self.upload_dir, "mitmproxy.log"),
env=env,
processStderrLine=LOG.error,
storeOutput=False,
)
self.mitmproxy_proc.run()
end_time = time.time() + MITMDUMP_COMMAND_TIMEOUT
ready = False
while time.time() < end_time:
ready = self.check_proxy(host=self.host, port=self.port)
if ready:
LOG.info(
"Mitmproxy playback successfully started on %s:%d as pid %d"
% (self.host, self.port, self.mitmproxy_proc.pid))
return
time.sleep(0.25)
# cannot continue as we won't be able to playback the pages
LOG.error("Aborting: Mitmproxy process did not startup")
self.stop_mitmproxy_playback()
sys.exit(1) # XXX why do we need to do that? a raise is not enough?
def stop_mitmproxy_playback(self):
"""Stop the mitproxy server playback"""
if self.mitmproxy_proc is None or self.mitmproxy_proc.poll(
) is not None:
return
LOG.info("Stopping mitmproxy playback, killing process %d" %
self.mitmproxy_proc.pid)
# On Windows, mozprocess brutally kills mitmproxy with TerminateJobObject
# The process has no chance to gracefully shutdown.
# Here, we send the process a break event to give it a chance to wrapup.
# See the signal handler in the alternate-server-replay-4.0.4.py script
if mozinfo.os == "win":
LOG.info("Sending CTRL_BREAK_EVENT to mitmproxy")
os.kill(self.mitmproxy_proc.pid, signal.CTRL_BREAK_EVENT)
time.sleep(2)
exit_code = self.mitmproxy_proc.kill()
self.mitmproxy_proc = None
if exit_code != 0:
if exit_code is None:
LOG.error("Failed to kill the mitmproxy playback process")
return
if mozinfo.os == "win":
from mozprocess.winprocess import ERROR_CONTROL_C_EXIT # noqa
if exit_code == ERROR_CONTROL_C_EXIT:
LOG.info(
"Successfully killed the mitmproxy playback process"
" with exit code %d" % exit_code)
return
log_func = LOG.error
if self.ignore_mitmdump_exit_failure:
log_func = LOG.info
log_func("Mitmproxy exited with error code %d" % exit_code)
else:
LOG.info("Successfully killed the mitmproxy playback process")
def check_proxy(self, host, port):
"""Check that mitmproxy process is working by doing a socket call using the proxy settings
:param host: Host of the proxy server
:param port: Port of the proxy server
:return: True if the proxy service is working
"""
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((host, port))
s.shutdown(socket.SHUT_RDWR)
s.close()
return True
except socket.error:
return False
def run_browser(command,
minidump_dir,
timeout=None,
on_started=None,
**kwargs):
"""
Run the browser using the given `command`.
After the browser prints __endTimestamp, we give it 5
seconds to quit and kill it if it's still alive at that point.
Note that this method ensure that the process is killed at
the end. If this is not possible, an exception will be raised.
:param command: the commad (as a string list) to run the browser
:param minidump_dir: a path where to extract minidumps in case the
browser hang. This have to be the same value
used in `mozcrash.check_for_crashes`.
:param timeout: if specified, timeout to wait for the browser before
we raise a :class:`TalosError`
:param on_started: a callback that can be used to do things just after
the browser has been started. The callback must takes
an argument, which is the psutil.Process instance
:param kwargs: additional keyword arguments for the :class:`ProcessHandler`
instance
Returns a ProcessContext instance, with available output and pid used.
"""
context = ProcessContext()
first_time = int(time.time()) * 1000
wait_for_quit_timeout = 5
event = Event()
reader = Reader(event)
kwargs['storeOutput'] = False
kwargs['processOutputLine'] = reader
kwargs['onFinish'] = event.set
proc = ProcessHandler(command, **kwargs)
proc.run()
try:
context.process = psutil.Process(proc.pid)
if on_started:
on_started(context.process)
# wait until we saw __endTimestamp in the proc output,
# or the browser just terminated - or we have a timeout
if not event.wait(timeout):
# try to extract the minidump stack if the browser hangs
mozcrash.kill_and_get_minidump(proc.pid, minidump_dir)
raise TalosError("timeout")
if reader.got_end_timestamp:
for i in range(1, wait_for_quit_timeout):
if proc.wait(1) is not None:
break
if proc.poll() is None:
logging.info(
"Browser shutdown timed out after {0} seconds, terminating"
" process.".format(wait_for_quit_timeout))
elif reader.got_timeout:
raise TalosError('TIMEOUT: %s' % reader.timeout_message)
finally:
# this also handle KeyboardInterrupt
# ensure early the process is really terminated
context.kill_process()
return_code = proc.wait(1)
reader.output.append(
"__startBeforeLaunchTimestamp%d__endBeforeLaunchTimestamp" %
first_time)
reader.output.append(
"__startAfterTerminationTimestamp%d__endAfterTerminationTimestamp" %
(int(time.time()) * 1000))
logging.info("Browser exited with error code: {0}".format(return_code))
context.output = reader.output
return context
def verify_android_device(build_obj,
install=False,
xre=False,
debugger=False,
network=False,
verbose=False,
app=None,
device_serial=None):
"""
Determine if any Android device is connected via adb.
If no device is found, prompt to start an emulator.
If a device is found or an emulator started and 'install' is
specified, also check whether Firefox is installed on the
device; if not, prompt to install Firefox.
If 'xre' is specified, also check with MOZ_HOST_BIN is set
to a valid xre/host-utils directory; if not, prompt to set
one up.
If 'debugger' is specified, also check that JimDB is installed;
if JimDB is not found, prompt to set up JimDB.
If 'network' is specified, also check that the device has basic
network connectivity.
Returns True if the emulator was started or another device was
already connected.
"""
device_verified = False
emulator = AndroidEmulator('*', substs=build_obj.substs, verbose=verbose)
adb_path = _find_sdk_exe(build_obj.substs, 'adb', False)
if not adb_path:
adb_path = 'adb'
adbhost = ADBHost(adb=adb_path, verbose=verbose, timeout=10)
devices = adbhost.devices(timeout=10)
if 'device' in [d['state'] for d in devices]:
device_verified = True
elif emulator.is_available():
response = raw_input(
"No Android devices connected. Start an emulator? (Y/n) ").strip()
if response.lower().startswith('y') or response == '':
if not emulator.check_avd():
_log_info("Fetching AVD. This may take a while...")
emulator.update_avd()
_log_info("Starting emulator running %s..." %
emulator.get_avd_description())
emulator.start()
emulator.wait_for_start()
device_verified = True
if device_verified and "DEVICE_SERIAL" not in os.environ:
devices = adbhost.devices(timeout=10)
for d in devices:
if d['state'] == 'device':
os.environ["DEVICE_SERIAL"] = d['device_serial']
break
if device_verified and install:
# Determine if Firefox is installed on the device; if not,
# prompt to install. This feature allows a test command to
# launch an emulator, install Firefox, and proceed with testing
# in one operation. It is also a basic safeguard against other
# cases where testing is requested but Firefox installation has
# been forgotten.
# If Firefox is installed, there is no way to determine whether
# the current build is installed, and certainly no way to
# determine if the installed build is the desired build.
# Installing every time is problematic because:
# - it prevents testing against other builds (downloaded apk)
# - installation may take a couple of minutes.
if not app:
app = build_obj.substs["ANDROID_PACKAGE_NAME"]
device = _get_device(build_obj.substs, device_serial)
response = ''
while not device.is_app_installed(app):
try:
if 'fennec' in app or 'firefox' in app:
response = response = raw_input(
"It looks like %s is not installed on this device.\n"
"Install Firefox? (Y/n) or quit to exit " %
app).strip()
if response.lower().startswith('y') or response == '':
_log_info(
"Installing Firefox. This may take a while...")
build_obj._run_make(directory=".",
target='install',
ensure_exit_code=False)
elif app == 'org.mozilla.geckoview.test':
response = response = raw_input(
"It looks like %s is not installed on this device.\n"
"Install geckoview AndroidTest? (Y/n) or quit to exit "
% app).strip()
if response.lower().startswith('y') or response == '':
_log_info(
"Installing geckoview AndroidTest. This may take a while..."
)
sub = 'geckoview:installWithGeckoBinariesDebugAndroidTest'
build_obj._mach_context.commands.dispatch(
'gradle',
args=[sub],
context=build_obj._mach_context)
elif app == 'org.mozilla.geckoview_example':
response = response = raw_input(
"It looks like %s is not installed on this device.\n"
"Install geckoview_example? (Y/n) or quit to exit " %
app).strip()
if response.lower().startswith('y') or response == '':
_log_info(
"Installing geckoview_example. This may take a while..."
)
sub = 'install-geckoview_example'
build_obj._mach_context.commands.dispatch(
'android',
subcommand=sub,
args=[],
context=build_obj._mach_context)
else:
response = raw_input(
"It looks like %s is not installed on this device,\n"
"but I don't know how to install it.\n"
"Install it now, then hit Enter or quit to exit " %
app)
except EOFError:
response = 'quit'
if response == 'quit':
device_verified = False
break
if device_verified and xre:
# Check whether MOZ_HOST_BIN has been set to a valid xre; if not,
# prompt to install one.
xre_path = os.environ.get('MOZ_HOST_BIN')
err = None
if not xre_path:
err = "environment variable MOZ_HOST_BIN is not set to a directory " \
"containing host xpcshell"
elif not os.path.isdir(xre_path):
err = '$MOZ_HOST_BIN does not specify a directory'
elif not os.path.isfile(os.path.join(xre_path, 'xpcshell')):
err = '$MOZ_HOST_BIN/xpcshell does not exist'
if err:
_maybe_update_host_utils(build_obj)
xre_path = glob.glob(os.path.join(EMULATOR_HOME_DIR,
'host-utils*'))
for path in xre_path:
if os.path.isdir(path) and os.path.isfile(
os.path.join(path, 'xpcshell')):
os.environ['MOZ_HOST_BIN'] = path
err = None
break
if err:
_log_info("Host utilities not found: %s" % err)
response = raw_input(
"Download and setup your host utilities? (Y/n) ").strip()
if response.lower().startswith('y') or response == '':
_install_host_utils(build_obj)
if device_verified and network:
# Optionally check the network: If on a device that does not look like
# an emulator, verify that the device IP address can be obtained
# and check that this host can ping the device.
serial = device_serial or os.environ.get('DEVICE_SERIAL')
if not serial or ('emulator' not in serial):
device = _get_device(build_obj.substs, serial)
try:
addr = device.get_ip_address()
if not addr:
_log_warning("unable to get Android device's IP address!")
_log_warning(
"tests may fail without network connectivity to the device!"
)
else:
_log_info("Android device's IP address: %s" % addr)
response = subprocess.check_output(
["ping", "-c", "1", addr])
_log_debug(response)
except Exception as e:
_log_warning(
"unable to verify network connection to device: %s" %
str(e))
_log_warning(
"tests may fail without network connectivity to the device!"
)
else:
_log_debug("network check skipped on emulator")
if debugger:
# Optionally set up JimDB. See https://wiki.mozilla.org/Mobile/Fennec/Android/GDB.
build_platform = _get_device_platform(build_obj.substs)
jimdb_path = os.path.join(EMULATOR_HOME_DIR,
'jimdb-%s' % build_platform)
jimdb_utils_path = os.path.join(jimdb_path, 'utils')
gdb_path = os.path.join(jimdb_path, 'bin', 'gdb')
err = None
if not os.path.isdir(jimdb_path):
err = '%s does not exist' % jimdb_path
elif not os.path.isfile(gdb_path):
err = '%s not found' % gdb_path
if err:
_log_info("JimDB (%s) not found: %s" % (build_platform, err))
response = raw_input("Download and setup JimDB (%s)? (Y/n) " %
build_platform).strip()
if response.lower().startswith('y') or response == '':
host_platform = _get_host_platform()
if host_platform:
_log_info(
"Installing JimDB (%s/%s). This may take a while..." %
(host_platform, build_platform))
path = os.path.join(MANIFEST_PATH, host_platform,
'jimdb-%s.manifest' % build_platform)
_get_tooltool_manifest(build_obj.substs, path,
EMULATOR_HOME_DIR,
'releng.manifest')
_tooltool_fetch()
if os.path.isfile(gdb_path):
# Get JimDB utilities from git repository
proc = ProcessHandler(['git', 'pull'],
cwd=jimdb_utils_path)
proc.run()
git_pull_complete = False
try:
proc.wait()
if proc.proc.returncode == 0:
git_pull_complete = True
except Exception:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
if not git_pull_complete:
_log_warning(
"Unable to update JimDB utils from git -- "
"some JimDB features may be unavailable.")
else:
_log_warning(
"Unable to install JimDB -- unable to fetch from tooltool."
)
else:
_log_warning(
"Unable to install JimDB -- your platform is not supported!"
)
if os.path.isfile(gdb_path):
# sync gdbinit.local with build settings
_update_gdbinit(build_obj.substs,
os.path.join(jimdb_utils_path, "gdbinit.local"))
# ensure JimDB is in system path, so that mozdebug can find it
bin_path = os.path.join(jimdb_path, 'bin')
os.environ['PATH'] = "%s:%s" % (bin_path, os.environ['PATH'])
return device_verified
class ServoCrashtestExecutor(ProcessTestExecutor):
convert_result = crashtest_result_converter
def __init__(self,
logger,
browser,
server_config,
binary=None,
timeout_multiplier=1,
screenshot_cache=None,
debug_info=None,
pause_after_test=False,
**kwargs):
ProcessTestExecutor.__init__(self,
logger,
browser,
server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.pause_after_test = pause_after_test
self.protocol = ConnectionlessProtocol(self, browser)
self.tempdir = tempfile.mkdtemp()
self.hosts_path = write_hosts_file(server_config)
def do_test(self, test):
timeout = (test.timeout * self.timeout_multiplier
if self.debug_info is None else None)
test_url = self.test_url(test)
# We want to pass the full test object into build_servo_command,
# so stash it in the class
self.test = test
success, data = ServoTimedRunner(self.logger, self.do_crashtest,
self.protocol, test_url, timeout,
self.extra_timeout).run()
# Ensure that no processes hang around if they timeout.
self.proc.kill()
if success:
return self.convert_result(test, data)
return (test.result_cls(*data), [])
def do_crashtest(self, protocol, url, timeout):
env = os.environ.copy()
env["HOST_FILE"] = self.hosts_path
env["RUST_BACKTRACE"] = "1"
command = build_servo_command(self.test,
self.test_url,
self.browser,
self.binary,
False,
self.debug_info,
extra_args=["-x"])
if not self.interactive:
self.proc = ProcessHandler(command, env=env, storeOutput=False)
self.proc.run()
else:
self.proc = subprocess.Popen(command, env=env)
self.proc.wait()
if self.proc.poll() >= 0:
return {"status": "PASS", "message": None}
return {"status": "CRASH", "message": None}
class ServoTestharnessExecutor(ProcessTestExecutor):
convert_result = testharness_result_converter
def __init__(self, browser, server_config, timeout_multiplier=1, debug_info=None,
pause_after_test=False):
ProcessTestExecutor.__init__(self, browser, server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.pause_after_test = pause_after_test
self.result_data = None
self.result_flag = None
self.protocol = Protocol(self, browser)
self.hosts_path = make_hosts_file()
def teardown(self):
try:
os.unlink(self.hosts_path)
except OSError:
pass
ProcessTestExecutor.teardown(self)
def do_test(self, test):
self.result_data = None
self.result_flag = threading.Event()
debug_args, command = browser_command(self.binary, ["--cpu", "--hard-fail", "-z", self.test_url(test)],
self.debug_info)
self.command = command
if self.pause_after_test:
self.command.remove("-z")
self.command = debug_args + self.command
env = os.environ.copy()
env["HOST_FILE"] = self.hosts_path
if not self.interactive:
self.proc = ProcessHandler(self.command,
processOutputLine=[self.on_output],
onFinish=self.on_finish,
env=env,
storeOutput=False)
self.proc.run()
else:
self.proc = subprocess.Popen(self.command, env=env)
try:
timeout = test.timeout * self.timeout_multiplier
# Now wait to get the output we expect, or until we reach the timeout
if not self.interactive and not self.pause_after_test:
wait_timeout = timeout + 5
self.result_flag.wait(wait_timeout)
else:
wait_timeout = None
self.proc.wait()
proc_is_running = True
if self.result_flag.is_set() and self.result_data is not None:
self.result_data["test"] = test.url
result = self.convert_result(test, self.result_data)
else:
if self.proc.poll() is not None:
result = (test.result_cls("CRASH", None), [])
proc_is_running = False
else:
result = (test.result_cls("TIMEOUT", None), [])
if proc_is_running:
if self.pause_after_test:
self.logger.info("Pausing until the browser exits")
self.proc.wait()
else:
self.proc.kill()
except KeyboardInterrupt:
self.proc.kill()
raise
return result
def on_output(self, line):
prefix = "ALERT: RESULT: "
line = line.decode("utf8", "replace")
if line.startswith(prefix):
self.result_data = json.loads(line[len(prefix):])
self.result_flag.set()
else:
if self.interactive:
print line
else:
self.logger.process_output(self.proc.pid,
line,
" ".join(self.command))
def on_finish(self):
self.result_flag.set()
class AndroidEmulator(object):
"""
Support running the Android emulator with an AVD from Mozilla
test automation.
Example usage:
emulator = AndroidEmulator()
if not emulator.is_running() and emulator.is_available():
if not emulator.check_avd():
warn("this may take a while...")
emulator.update_avd()
emulator.start()
emulator.wait_for_start()
emulator.wait()
"""
def __init__(self, avd_type='4.3', verbose=False, substs=None):
global verbose_logging
self.emulator_log = None
self.emulator_path = 'emulator'
verbose_logging = verbose
self.substs = substs
self.avd_type = self._get_avd_type(avd_type)
self.avd_info = AVD_DICT[self.avd_type]
adb_path = self._find_sdk_exe('adb', False)
if not adb_path:
adb_path = 'adb'
self.dm = DeviceManagerADB(autoconnect=False, adbPath=adb_path, retryLimit=1)
self.dm.default_timeout = 10
_log_debug("Emulator created with type %s" % self.avd_type)
def __del__(self):
if self.emulator_log:
self.emulator_log.close()
def is_running(self):
"""
Returns True if the Android emulator is running.
"""
for proc in psutil.process_iter():
name = proc.name()
# On some platforms, "emulator" may start an emulator with
# process name "emulator64-arm" or similar.
if name and name.startswith('emulator'):
return True
return False
def is_available(self):
"""
Returns True if an emulator executable is found.
"""
found = False
emulator_path = self._find_sdk_exe('emulator', True)
if emulator_path:
self.emulator_path = emulator_path
found = True
return found
def check_avd(self, force=False):
"""
Determine if the AVD is already installed locally.
(This is usually used to determine if update_avd() is likely
to require a download; it is a convenient way of determining
whether a 'this may take a while' warning is warranted.)
Returns True if the AVD is installed.
"""
avd = os.path.join(
EMULATOR_HOME_DIR, 'avd', self.avd_info.name + '.avd')
if force and os.path.exists(avd):
shutil.rmtree(avd)
if os.path.exists(avd):
_log_debug("AVD found at %s" % avd)
return True
return False
def update_avd(self, force=False):
"""
If required, update the AVD via tooltool.
If the AVD directory is not found, or "force" is requested,
download the tooltool manifest associated with the AVD and then
invoke tooltool.py on the manifest. tooltool.py will download the
required archive (unless already present in the local tooltool
cache) and install the AVD.
"""
avd = os.path.join(
EMULATOR_HOME_DIR, 'avd', self.avd_info.name + '.avd')
if force and os.path.exists(avd):
shutil.rmtree(avd)
if not os.path.exists(avd):
_download_file(TOOLTOOL_URL, 'tooltool.py', EMULATOR_HOME_DIR)
url = '%s/%s' % (TRY_URL, self.avd_info.tooltool_manifest)
_download_file(url, 'releng.manifest', EMULATOR_HOME_DIR)
_tooltool_fetch()
self._update_avd_paths()
def start(self):
"""
Launch the emulator.
"""
def outputHandler(line):
self.emulator_log.write("<%s>\n" % line)
env = os.environ
env['ANDROID_AVD_HOME'] = os.path.join(EMULATOR_HOME_DIR, "avd")
command = [self.emulator_path, "-avd",
self.avd_info.name, "-port", "5554"]
if self.avd_info.extra_args:
command += self.avd_info.extra_args
log_path = os.path.join(EMULATOR_HOME_DIR, 'emulator.log')
self.emulator_log = open(log_path, 'w')
_log_debug("Starting the emulator with this command: %s" %
' '.join(command))
_log_debug("Emulator output will be written to '%s'" %
log_path)
self.proc = ProcessHandler(
command, storeOutput=False, processOutputLine=outputHandler,
env=env)
self.proc.run()
_log_debug("Emulator started with pid %d" %
int(self.proc.proc.pid))
def wait_for_start(self):
"""
Verify that the emulator is running, the emulator device is visible
to adb, and Android has booted.
"""
if not self.proc:
_log_warning("Emulator not started!")
return False
if self.proc.proc.poll() is not None:
_log_warning("Emulator has already completed!")
return False
_log_debug("Waiting for device status...")
while(('emulator-5554', 'device') not in self.dm.devices()):
time.sleep(10)
if self.proc.proc.poll() is not None:
_log_warning("Emulator has already completed!")
return False
_log_debug("Device status verified.")
_log_debug("Checking that Android has booted...")
complete = False
while(not complete):
output = ''
try:
output = self.dm.shellCheckOutput(
['getprop', 'sys.boot_completed'], timeout=5)
except DMError:
# adb not yet responding...keep trying
pass
if output.strip() == '1':
complete = True
else:
time.sleep(10)
if self.proc.proc.poll() is not None:
_log_warning("Emulator has already completed!")
return False
_log_debug("Android boot status verified.")
if not self._verify_emulator():
return False
if self.avd_info.uses_sut:
if not self._verify_sut():
return False
return True
def wait(self):
"""
Wait for the emulator to close. If interrupted, close the emulator.
"""
try:
self.proc.wait()
except:
if self.proc.poll() is None:
self.cleanup()
return self.proc.poll()
def cleanup(self):
"""
Close the emulator.
"""
self.proc.kill(signal.SIGTERM)
def get_avd_description(self):
"""
Return the human-friendly description of this AVD.
"""
return self.avd_info.description
def _update_avd_paths(self):
avd_path = os.path.join(EMULATOR_HOME_DIR, "avd")
ini_file = os.path.join(avd_path, "test-1.ini")
ini_file_new = os.path.join(avd_path, self.avd_info.name + ".ini")
os.rename(ini_file, ini_file_new)
avd_dir = os.path.join(avd_path, "test-1.avd")
avd_dir_new = os.path.join(avd_path, self.avd_info.name + ".avd")
os.rename(avd_dir, avd_dir_new)
self._replace_ini_contents(ini_file_new)
def _replace_ini_contents(self, path):
with open(path, "r") as f:
lines = f.readlines()
with open(path, "w") as f:
for line in lines:
if line.startswith('path='):
avd_path = os.path.join(EMULATOR_HOME_DIR, "avd")
f.write('path=%s/%s.avd\n' %
(avd_path, self.avd_info.name))
elif line.startswith('path.rel='):
f.write('path.rel=avd/%s.avd\n' % self.avd_info.name)
else:
f.write(line)
def _telnet_cmd(self, telnet, command):
_log_debug(">>> " + command)
telnet.write('%s\n' % command)
result = telnet.read_until('OK', 10)
_log_debug("<<< " + result)
return result
def _verify_emulator(self):
telnet_ok = False
tn = None
while(not telnet_ok):
try:
tn = telnetlib.Telnet('localhost', self.avd_info.port, 10)
if tn is not None:
res = tn.read_until('OK', 10)
self._telnet_cmd(tn, 'avd status')
if self.avd_info.uses_sut:
cmd = 'redir add tcp:%s:%s' % \
(str(self.avd_info.sut_port),
str(self.avd_info.sut_port))
self._telnet_cmd(tn, cmd)
cmd = 'redir add tcp:%s:%s' % \
(str(self.avd_info.sut_port2),
str(self.avd_info.sut_port2))
self._telnet_cmd(tn, cmd)
self._telnet_cmd(tn, 'redir list')
self._telnet_cmd(tn, 'network status')
tn.write('quit\n')
tn.read_all()
telnet_ok = True
else:
_log_warning("Unable to connect to port %d" % port)
except:
_log_warning("Trying again after unexpected exception")
finally:
if tn is not None:
tn.close()
if not telnet_ok:
time.sleep(10)
if self.proc.proc.poll() is not None:
_log_warning("Emulator has already completed!")
return False
return telnet_ok
def _verify_sut(self):
sut_ok = False
while(not sut_ok):
try:
tn = telnetlib.Telnet('localhost', self.avd_info.sut_port, 10)
if tn is not None:
_log_debug(
"Connected to port %d" % self.avd_info.sut_port)
res = tn.read_until('$>', 10)
if res.find('$>') == -1:
_log_debug("Unexpected SUT response: %s" % res)
else:
_log_debug("SUT response: %s" % res)
sut_ok = True
tn.write('quit\n')
tn.read_all()
except:
_log_debug("Caught exception while verifying sutagent")
finally:
if tn is not None:
tn.close()
if not sut_ok:
time.sleep(10)
if self.proc.proc.poll() is not None:
_log_warning("Emulator has already completed!")
return False
return sut_ok
def _get_avd_type(self, requested):
if requested in AVD_DICT.keys():
return requested
if self.substs:
if not self.substs['TARGET_CPU'].startswith('arm'):
return 'x86'
if self.substs['MOZ_ANDROID_MIN_SDK_VERSION'] == '9':
return '2.3'
return '4.3'
def _find_sdk_exe(self, exe, tools):
if tools:
subdir = 'tools'
var = 'ANDROID_TOOLS'
else:
subdir = 'platform-tools'
var = 'ANDROID_PLATFORM_TOOLS'
found = False
# Can exe be found in the Android SDK?
try:
android_sdk_root = os.environ['ANDROID_SDK_ROOT']
exe_path = os.path.join(
android_sdk_root, subdir, exe)
if os.path.exists(exe_path):
found = True
else:
_log_debug(
"Unable to find executable at %s" % exe_path)
except KeyError:
_log_debug("ANDROID_SDK_ROOT not set")
if not found and self.substs:
# Can exe be found in ANDROID_TOOLS/ANDROID_PLATFORM_TOOLS?
try:
exe_path = os.path.join(
self.substs[var], exe)
if os.path.exists(exe_path):
found = True
else:
_log_debug(
"Unable to find executable at %s" % exe_path)
except KeyError:
_log_debug("%s not set" % var)
if not found:
# Can exe be found in the default bootstrap location?
mozbuild_path = os.environ.get('MOZBUILD_STATE_PATH',
os.path.expanduser(os.path.join('~', '.mozbuild')))
exe_path = os.path.join(
mozbuild_path, 'android-sdk-linux', subdir, exe)
if os.path.exists(exe_path):
found = True
else:
_log_debug(
"Unable to find executable at %s" % exe_path)
if not found:
# Is exe on PATH?
exe_path = find_executable(exe)
if exe_path:
found = True
else:
_log_debug("Unable to find executable on PATH")
if found:
_log_debug("%s found at %s" % (exe, exe_path))
else:
exe_path = None
return exe_path
def verify_android_device(build_obj, install=False, xre=False, debugger=False):
"""
Determine if any Android device is connected via adb.
If no device is found, prompt to start an emulator.
If a device is found or an emulator started and 'install' is
specified, also check whether Firefox is installed on the
device; if not, prompt to install Firefox.
If 'xre' is specified, also check with MOZ_HOST_BIN is set
to a valid xre/host-utils directory; if not, prompt to set
one up.
If 'debugger' is specified, also check that JimDB is installed;
if JimDB is not found, prompt to set up JimDB.
Returns True if the emulator was started or another device was
already connected.
"""
device_verified = False
emulator = AndroidEmulator('*', substs=build_obj.substs)
devices = emulator.dm.devices()
if (len(devices) > 0) and ('device' in [d[1] for d in devices]):
device_verified = True
elif emulator.is_available():
response = raw_input(
"No Android devices connected. Start an emulator? (Y/n) ").strip()
if response.lower().startswith('y') or response == '':
if not emulator.check_avd():
_log_info("Fetching AVD. This may take a while...")
emulator.update_avd()
_log_info("Starting emulator running %s..." %
emulator.get_avd_description())
emulator.start()
emulator.wait_for_start()
device_verified = True
if device_verified and install:
# Determine if Firefox is installed on the device; if not,
# prompt to install. This feature allows a test command to
# launch an emulator, install Firefox, and proceed with testing
# in one operation. It is also a basic safeguard against other
# cases where testing is requested but Firefox installation has
# been forgotten.
# If Firefox is installed, there is no way to determine whether
# the current build is installed, and certainly no way to
# determine if the installed build is the desired build.
# Installing every time is problematic because:
# - it prevents testing against other builds (downloaded apk)
# - installation may take a couple of minutes.
installed = emulator.dm.shellCheckOutput(
['pm', 'list', 'packages', 'org.mozilla.'])
if not 'fennec' in installed and not 'firefox' in installed:
response = raw_input(
"It looks like Firefox is not installed on this device.\n"
"Install Firefox? (Y/n) ").strip()
if response.lower().startswith('y') or response == '':
_log_info("Installing Firefox. This may take a while...")
build_obj._run_make(directory=".",
target='install',
ensure_exit_code=False)
if device_verified and xre:
# Check whether MOZ_HOST_BIN has been set to a valid xre; if not,
# prompt to install one.
xre_path = os.environ.get('MOZ_HOST_BIN')
err = None
if not xre_path:
err = 'environment variable MOZ_HOST_BIN is not set to a directory containing host xpcshell'
elif not os.path.isdir(xre_path):
err = '$MOZ_HOST_BIN does not specify a directory'
elif not os.path.isfile(os.path.join(xre_path, 'xpcshell')):
err = '$MOZ_HOST_BIN/xpcshell does not exist'
if err:
xre_path = glob.glob(os.path.join(EMULATOR_HOME_DIR,
'host-utils*'))
for path in xre_path:
if os.path.isdir(path) and os.path.isfile(
os.path.join(path, 'xpcshell')):
os.environ['MOZ_HOST_BIN'] = path
err = None
break
if err:
_log_info("Host utilities not found: %s" % err)
response = raw_input(
"Download and setup your host utilities? (Y/n) ").strip()
if response.lower().startswith('y') or response == '':
_log_info(
"Installing host utilities. This may take a while...")
_download_file(TOOLTOOL_URL, 'tooltool.py', EMULATOR_HOME_DIR)
host_platform = _get_host_platform()
if host_platform:
path = os.path.join(MANIFEST_PATH, host_platform,
'hostutils.manifest')
_get_tooltool_manifest(build_obj.substs, path,
EMULATOR_HOME_DIR,
'releng.manifest')
_tooltool_fetch()
xre_path = glob.glob(
os.path.join(EMULATOR_HOME_DIR, 'host-utils*'))
for path in xre_path:
if os.path.isdir(path) and os.path.isfile(
os.path.join(path, 'xpcshell')):
os.environ['MOZ_HOST_BIN'] = path
err = None
break
if err:
_log_warning("Unable to install host utilities.")
else:
_log_warning(
"Unable to install host utilities -- your platform is not supported!"
)
if debugger:
# Optionally set up JimDB. See https://wiki.mozilla.org/Mobile/Fennec/Android/GDB.
build_platform = _get_build_platform(build_obj.substs)
jimdb_path = os.path.join(EMULATOR_HOME_DIR,
'jimdb-%s' % build_platform)
jimdb_utils_path = os.path.join(jimdb_path, 'utils')
gdb_path = os.path.join(jimdb_path, 'bin', 'gdb')
err = None
if not os.path.isdir(jimdb_path):
err = '%s does not exist' % jimdb_path
elif not os.path.isfile(gdb_path):
err = '%s not found' % gdb_path
if err:
_log_info("JimDB (%s) not found: %s" % (build_platform, err))
response = raw_input("Download and setup JimDB (%s)? (Y/n) " %
build_platform).strip()
if response.lower().startswith('y') or response == '':
host_platform = _get_host_platform()
if host_platform:
_log_info(
"Installing JimDB (%s/%s). This may take a while..." %
(host_platform, build_platform))
path = os.path.join(MANIFEST_PATH, host_platform,
'jimdb-%s.manifest' % build_platform)
_get_tooltool_manifest(build_obj.substs, path,
EMULATOR_HOME_DIR,
'releng.manifest')
_tooltool_fetch()
if os.path.isfile(gdb_path):
# Get JimDB utilities from git repository
proc = ProcessHandler(['git', 'pull'],
cwd=jimdb_utils_path)
proc.run()
git_pull_complete = False
try:
proc.wait()
if proc.proc.returncode == 0:
git_pull_complete = True
except:
if proc.poll() is None:
proc.kill(signal.SIGTERM)
if not git_pull_complete:
_log_warning(
"Unable to update JimDB utils from git -- some JimDB features may be unavailable."
)
else:
_log_warning(
"Unable to install JimDB -- unable to fetch from tooltool."
)
else:
_log_warning(
"Unable to install JimDB -- your platform is not supported!"
)
if os.path.isfile(gdb_path):
# sync gdbinit.local with build settings
_update_gdbinit(build_obj.substs,
os.path.join(jimdb_utils_path, "gdbinit.local"))
# ensure JimDB is in system path, so that mozdebug can find it
bin_path = os.path.join(jimdb_path, 'bin')
os.environ['PATH'] = "%s:%s" % (bin_path, os.environ['PATH'])
return device_verified
class AndroidEmulator(object):
"""
Support running the Android emulator with an AVD from Mozilla
test automation.
Example usage:
emulator = AndroidEmulator()
if not emulator.is_running() and emulator.is_available():
if not emulator.check_avd():
warn("this may take a while...")
emulator.update_avd()
emulator.start()
emulator.wait_for_start()
emulator.wait()
"""
def __init__(self,
avd_type=None,
verbose=False,
substs=None,
device_serial=None):
global verbose_logging
self.emulator_log = None
self.emulator_path = 'emulator'
verbose_logging = verbose
self.substs = substs
self.avd_type = self._get_avd_type(avd_type)
self.avd_info = AVD_DICT[self.avd_type]
self.gpu = True
self.restarted = False
self.device_serial = device_serial
_log_debug("Running on %s" % platform.platform())
_log_debug("Emulator created with type %s" % self.avd_type)
def __del__(self):
if self.emulator_log:
self.emulator_log.close()
def is_running(self):
"""
Returns True if the Android emulator is running.
"""
for proc in psutil.process_iter():
name = proc.name()
# On some platforms, "emulator" may start an emulator with
# process name "emulator64-arm" or similar.
if name and name.startswith('emulator'):
return True
return False
def is_available(self):
"""
Returns True if an emulator executable is found.
"""
found = False
emulator_path = _find_sdk_exe(self.substs, 'emulator', True)
if emulator_path:
self.emulator_path = emulator_path
found = True
return found
def check_avd(self, force=False):
"""
Determine if the AVD is already installed locally.
(This is usually used to determine if update_avd() is likely
to require a download; it is a convenient way of determining
whether a 'this may take a while' warning is warranted.)
Returns True if the AVD is installed.
"""
avd = os.path.join(EMULATOR_HOME_DIR, 'avd',
self.avd_info.name + '.avd')
if force and os.path.exists(avd):
shutil.rmtree(avd)
if os.path.exists(avd):
_log_debug("AVD found at %s" % avd)
return True
return False
def update_avd(self, force=False):
"""
If required, update the AVD via tooltool.
If the AVD directory is not found, or "force" is requested,
download the tooltool manifest associated with the AVD and then
invoke tooltool.py on the manifest. tooltool.py will download the
required archive (unless already present in the local tooltool
cache) and install the AVD.
"""
avd = os.path.join(EMULATOR_HOME_DIR, 'avd',
self.avd_info.name + '.avd')
ini_file = os.path.join(EMULATOR_HOME_DIR, 'avd',
self.avd_info.name + '.ini')
if force and os.path.exists(avd):
shutil.rmtree(avd)
if not os.path.exists(avd):
if os.path.exists(ini_file):
os.remove(ini_file)
path = self.avd_info.tooltool_manifest
_get_tooltool_manifest(self.substs, path, EMULATOR_HOME_DIR,
'releng.manifest')
_tooltool_fetch()
self._update_avd_paths()
def start(self):
"""
Launch the emulator.
"""
if self.avd_info.x86 and 'linux' in _get_host_platform():
_verify_kvm(self.substs)
if os.path.exists(EMULATOR_AUTH_FILE):
os.remove(EMULATOR_AUTH_FILE)
_log_debug("deleted %s" % EMULATOR_AUTH_FILE)
# create an empty auth file to disable emulator authentication
auth_file = open(EMULATOR_AUTH_FILE, 'w')
auth_file.close()
def outputHandler(line):
self.emulator_log.write("<%s>\n" % line)
if "Invalid value for -gpu" in line or "Invalid GPU mode" in line:
self.gpu = False
env = os.environ
env['ANDROID_AVD_HOME'] = os.path.join(EMULATOR_HOME_DIR, "avd")
command = [self.emulator_path, "-avd", self.avd_info.name]
if self.gpu:
command += ['-gpu', 'swiftshader']
if self.avd_info.extra_args:
# -enable-kvm option is not valid on OSX
if _get_host_platform(
) == 'macosx64' and '-enable-kvm' in self.avd_info.extra_args:
self.avd_info.extra_args.remove('-enable-kvm')
command += self.avd_info.extra_args
log_path = os.path.join(EMULATOR_HOME_DIR, 'emulator.log')
self.emulator_log = open(log_path, 'w')
_log_debug("Starting the emulator with this command: %s" %
' '.join(command))
_log_debug("Emulator output will be written to '%s'" % log_path)
self.proc = ProcessHandler(command,
storeOutput=False,
processOutputLine=outputHandler,
env=env,
ignore_children=True)
self.proc.run()
_log_debug("Emulator started with pid %d" % int(self.proc.proc.pid))
def wait_for_start(self):
"""
Verify that the emulator is running, the emulator device is visible
to adb, and Android has booted.
"""
if not self.proc:
_log_warning("Emulator not started!")
return False
if self.check_completed():
return False
_log_debug("Waiting for device status...")
adb_path = _find_sdk_exe(self.substs, 'adb', False)
if not adb_path:
adb_path = 'adb'
adbhost = ADBHost(adb=adb_path, verbose=verbose_logging, timeout=10)
devs = adbhost.devices(timeout=10)
devs = [(d['device_serial'], d['state']) for d in devs]
while ('emulator-5554', 'device') not in devs:
time.sleep(10)
if self.check_completed():
return False
devs = adbhost.devices(timeout=10)
devs = [(d['device_serial'], d['state']) for d in devs]
_log_debug("Device status verified.")
_log_debug("Checking that Android has booted...")
device = _get_device(self.substs, self.device_serial)
complete = False
while not complete:
output = ''
try:
output = device.get_prop('sys.boot_completed', timeout=5)
except Exception:
# adb not yet responding...keep trying
pass
if output.strip() == '1':
complete = True
else:
time.sleep(10)
if self.check_completed():
return False
_log_debug("Android boot status verified.")
if not self._verify_emulator():
return False
if self.avd_info.x86:
_log_info(
"Running the x86 emulator; be sure to install an x86 APK!")
else:
_log_info(
"Running the arm emulator; be sure to install an arm APK!")
return True
def check_completed(self):
if self.proc.proc.poll() is not None:
if not self.gpu and not self.restarted:
_log_warning(
"Emulator failed to start. Your emulator may be out of date."
)
_log_warning(
"Trying to restart the emulator without -gpu argument.")
self.restarted = True
self.start()
return False
_log_warning("Emulator has already completed!")
log_path = os.path.join(EMULATOR_HOME_DIR, 'emulator.log')
_log_warning(
"See log at %s and/or use --verbose for more information." %
log_path)
return True
return False
def wait(self):
"""
Wait for the emulator to close. If interrupted, close the emulator.
"""
try:
self.proc.wait()
except Exception:
if self.proc.poll() is None:
self.cleanup()
return self.proc.poll()
def cleanup(self):
"""
Close the emulator.
"""
self.proc.kill(signal.SIGTERM)
def get_avd_description(self):
"""
Return the human-friendly description of this AVD.
"""
return self.avd_info.description
def _update_avd_paths(self):
avd_path = os.path.join(EMULATOR_HOME_DIR, "avd")
ini_file = os.path.join(avd_path, "test-1.ini")
ini_file_new = os.path.join(avd_path, self.avd_info.name + ".ini")
os.rename(ini_file, ini_file_new)
avd_dir = os.path.join(avd_path, "test-1.avd")
avd_dir_new = os.path.join(avd_path, self.avd_info.name + ".avd")
os.rename(avd_dir, avd_dir_new)
self._replace_ini_contents(ini_file_new)
def _replace_ini_contents(self, path):
with open(path, "r") as f:
lines = f.readlines()
with open(path, "w") as f:
for line in lines:
if line.startswith('path='):
avd_path = os.path.join(EMULATOR_HOME_DIR, "avd")
f.write('path=%s/%s.avd\n' %
(avd_path, self.avd_info.name))
elif line.startswith('path.rel='):
f.write('path.rel=avd/%s.avd\n' % self.avd_info.name)
else:
f.write(line)
def _telnet_cmd(self, telnet, command):
_log_debug(">>> " + command)
telnet.write('%s\n' % command)
result = telnet.read_until('OK', 10)
_log_debug("<<< " + result)
return result
def _verify_emulator(self):
telnet_ok = False
tn = None
while (not telnet_ok):
try:
tn = telnetlib.Telnet('localhost', 5554, 10)
if tn is not None:
tn.read_until('OK', 10)
self._telnet_cmd(tn, 'avd status')
self._telnet_cmd(tn, 'redir list')
self._telnet_cmd(tn, 'network status')
tn.write('quit\n')
tn.read_all()
telnet_ok = True
else:
_log_warning("Unable to connect to port 5554")
except Exception:
_log_warning("Trying again after unexpected exception")
finally:
if tn is not None:
tn.close()
if not telnet_ok:
time.sleep(10)
if self.proc.proc.poll() is not None:
_log_warning("Emulator has already completed!")
return False
return telnet_ok
def _get_avd_type(self, requested):
if requested in AVD_DICT.keys():
return requested
if self.substs:
if not self.substs['TARGET_CPU'].startswith('arm'):
return 'x86-7.0'
else:
return '7.0'
return 'x86-7.0'
def run_browser(command, timeout=None, on_started=None, **kwargs):
"""
Run the browser using the given `command`.
After the browser prints __endTimestamp, we give it 5
seconds to quit and kill it if it's still alive at that point.
Note that this method ensure that the process is killed at
the end. If this is not possible, an exception will be raised.
:param command: the commad (as a string list) to run the browser
:param timeout: if specified, timeout to wait for the browser before
we raise a :class:`TalosError`
:param on_started: a callback that can be used to do things just after
the browser has been started
:param kwargs: additional keyword arguments for the :class:`ProcessHandler`
instance
Returns a ProcessContext instance, with available output and pid used.
"""
context = ProcessContext()
first_time = int(time.time()) * 1000
wait_for_quit_timeout = 5
event = Event()
reader = Reader(event)
kwargs['storeOutput'] = False
kwargs['processOutputLine'] = reader
kwargs['onFinish'] = event.set
proc = ProcessHandler(command, **kwargs)
proc.run()
try:
context.process = psutil.Process(proc.pid)
if on_started:
on_started()
# wait until we saw __endTimestamp in the proc output,
# or the browser just terminated - or we have a timeout
if not event.wait(timeout):
# try to extract the minidump stack if the browser hangs
mozcrash.kill_and_get_minidump(proc.pid)
raise TalosError("timeout")
if reader.got_end_timestamp:
for i in range(1, wait_for_quit_timeout):
if proc.wait(1) is not None:
break
if proc.poll() is None:
logging.info(
"Browser shutdown timed out after {0} seconds, terminating"
" process.".format(wait_for_quit_timeout)
)
finally:
# this also handle KeyboardInterrupt
# ensure early the process is really terminated
context.kill_process()
reader.output.append(
"__startBeforeLaunchTimestamp%d__endBeforeLaunchTimestamp"
% first_time)
reader.output.append(
"__startAfterTerminationTimestamp%d__endAfterTerminationTimestamp"
% (int(time.time()) * 1000))
logging.info("Browser exited with error code: {0}".format(proc.returncode))
context.output = reader.output
return context
class Mitmproxy(Playback):
def __init__(self, config):
self.config = config
self.mitmproxy_proc = None
self.mitmdump_path = None
self.browser_path = config.get("binary")
self.policies_dir = None
self.ignore_mitmdump_exit_failure = config.get(
"ignore_mitmdump_exit_failure", False)
if self.config.get("playback_version") is None:
LOG.info(
"mitmproxy was not provided with a 'playback_version' "
"getting 'playback_version' from 'playback_binary_manifest'")
if "4.0.4" in self.config["playback_binary_manifest"]:
self.config["playback_version"] = "4.0.4"
else:
self.config["playback_version"] = "2.0.2"
# mozproxy_dir is where we will download all mitmproxy required files
# when running locally it comes from obj_path via mozharness/mach
if self.config.get("obj_path") is not None:
self.mozproxy_dir = self.config.get("obj_path")
else:
# in production it is ../tasks/task_N/build/, in production that dir
# is not available as an envvar, however MOZ_UPLOAD_DIR is set as
# ../tasks/task_N/build/blobber_upload_dir so take that and go up 1 level
self.mozproxy_dir = os.path.dirname(
os.path.dirname(os.environ["MOZ_UPLOAD_DIR"]))
self.mozproxy_dir = os.path.join(self.mozproxy_dir, "testing",
"mozproxy")
self.upload_dir = os.environ.get("MOZ_UPLOAD_DIR", self.mozproxy_dir)
LOG.info(
"mozproxy_dir used for mitmproxy downloads and exe files: %s" %
self.mozproxy_dir)
# setting up the MOZPROXY_DIR env variable so custom scripts know
# where to get the data
os.environ["MOZPROXY_DIR"] = self.mozproxy_dir
def start(self):
# go ahead and download and setup mitmproxy
self.download()
# mitmproxy must be started before setup, so that the CA cert is available
self.start_mitmproxy_playback(self.mitmdump_path, self.browser_path)
# In case the setup fails, we want to stop the process before raising.
try:
self.setup()
except Exception:
self.stop()
raise
def download(self):
"""Download and unpack mitmproxy binary and pageset using tooltool"""
if not os.path.exists(self.mozproxy_dir):
os.makedirs(self.mozproxy_dir)
_manifest = os.path.join(here, self.config["playback_binary_manifest"])
transformed_manifest = transform_platform(_manifest,
self.config["platform"])
# generate the mitmdump_path
self.mitmdump_path = os.path.join(
self.mozproxy_dir, "mitmdump-%s" % self.config["playback_version"],
"mitmdump")
# Check if mitmproxy bin exists
if os.path.exists(self.mitmdump_path):
LOG.info("mitmproxy binary already exists. Skipping download")
else:
# Download and unpack mitmproxy binary
download_path = os.path.dirname(self.mitmdump_path)
LOG.info("create mitmproxy %s dir" %
self.config["playback_version"])
if not os.path.exists(download_path):
os.makedirs(download_path)
LOG.info("downloading mitmproxy binary")
tooltool_download(transformed_manifest, self.config["run_local"],
download_path)
if "playback_pageset_manifest" in self.config:
# we use one pageset for all platforms
LOG.info("downloading mitmproxy pageset")
_manifest = self.config["playback_pageset_manifest"]
transformed_manifest = transform_platform(_manifest,
self.config["platform"])
tooltool_download(transformed_manifest, self.config["run_local"],
self.mozproxy_dir)
if "playback_artifacts" in self.config:
artifacts = self.config["playback_artifacts"].split(",")
for artifact in artifacts:
artifact = artifact.strip()
if not artifact:
continue
artifact_name = artifact.split("/")[-1]
if artifact_name.endswith(".manifest"):
tooltool_download(artifact, self.config["run_local"],
self.mozproxy_dir)
else:
dest = os.path.join(self.mozproxy_dir, artifact_name)
download_file_from_url(artifact, dest, extract=True)
def stop(self):
self.stop_mitmproxy_playback()
def start_mitmproxy_playback(self, mitmdump_path, browser_path):
"""Startup mitmproxy and replay the specified flow file"""
if self.mitmproxy_proc is not None:
raise Exception("Proxy already started.")
LOG.info("mitmdump path: %s" % mitmdump_path)
LOG.info("browser path: %s" % browser_path)
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
env["PATH"] = os.path.dirname(browser_path) + os.pathsep + env["PATH"]
command = [mitmdump_path]
if "playback_tool_args" in self.config:
command.extend(self.config["playback_tool_args"])
LOG.info("Starting mitmproxy playback using env path: %s" %
env["PATH"])
LOG.info("Starting mitmproxy playback using command: %s" %
" ".join(command))
# to turn off mitmproxy log output, use these params for Popen:
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
self.mitmproxy_proc = ProcessHandler(command,
logfile=os.path.join(
self.upload_dir,
"mitmproxy.log"),
env=env)
self.mitmproxy_proc.run()
end_time = time.time() + MITMDUMP_COMMAND_TIMEOUT
ready = False
while time.time() < end_time:
ready = self.check_proxy()
if ready:
LOG.info("Mitmproxy playback successfully started as pid %d" %
self.mitmproxy_proc.pid)
return
time.sleep(0.25)
# cannot continue as we won't be able to playback the pages
LOG.error("Aborting: Mitmproxy process did not startup")
self.stop_mitmproxy_playback()
sys.exit() # XXX why do we need to do that? a raise is not enough?
def stop_mitmproxy_playback(self):
"""Stop the mitproxy server playback"""
if self.mitmproxy_proc is None or self.mitmproxy_proc.poll(
) is not None:
return
LOG.info("Stopping mitmproxy playback, killing process %d" %
self.mitmproxy_proc.pid)
exit_code = self.mitmproxy_proc.kill()
if exit_code != 0:
# I *think* we can still continue, as process will be automatically
# killed anyway when mozharness is done (?) if not, we won't be able
# to startup mitmxproy next time if it is already running
if exit_code is None:
LOG.error("Failed to kill the mitmproxy playback process")
else:
log_func = LOG.error
if self.ignore_mitmdump_exit_failure:
log_func = LOG.info
log_func("Mitmproxy exited with error code %d" % exit_code)
else:
LOG.info("Successfully killed the mitmproxy playback process")
self.mitmproxy_proc = None
def check_proxy(self, host="localhost", port=8080):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((host, port))
s.shutdown(socket.SHUT_RDWR)
s.close()
return True
except socket.error:
return False
def run_browser(command,
minidump_dir,
timeout=None,
on_started=None,
debug=None,
debugger=None,
debugger_args=None,
**kwargs):
"""
Run the browser using the given `command`.
After the browser prints __endTimestamp, we give it 5
seconds to quit and kill it if it's still alive at that point.
Note that this method ensure that the process is killed at
the end. If this is not possible, an exception will be raised.
:param command: the commad (as a string list) to run the browser
:param minidump_dir: a path where to extract minidumps in case the
browser hang. This have to be the same value
used in `mozcrash.check_for_crashes`.
:param timeout: if specified, timeout to wait for the browser before
we raise a :class:`TalosError`
:param on_started: a callback that can be used to do things just after
the browser has been started. The callback must takes
an argument, which is the psutil.Process instance
:param kwargs: additional keyword arguments for the :class:`ProcessHandler`
instance
Returns a ProcessContext instance, with available output and pid used.
"""
debugger_info = find_debugger_info(debug, debugger, debugger_args)
if debugger_info is not None:
return run_in_debug_mode(command,
debugger_info,
on_started=on_started,
env=kwargs.get('env'))
is_launcher = sys.platform.startswith(
'win') and '-wait-for-browser' in command
context = ProcessContext(is_launcher)
first_time = int(time.time()) * 1000
wait_for_quit_timeout = 20
event = Event()
reader = Reader(event)
LOG.info("Using env: %s" % pprint.pformat(kwargs['env']))
kwargs['storeOutput'] = False
kwargs['processOutputLine'] = reader
kwargs['onFinish'] = event.set
proc = ProcessHandler(command, **kwargs)
reader.proc = proc
proc.run()
LOG.process_start(proc.pid, ' '.join(command))
try:
context.process = psutil.Process(proc.pid)
if on_started:
on_started(context.process)
# wait until we saw __endTimestamp in the proc output,
# or the browser just terminated - or we have a timeout
if not event.wait(timeout):
LOG.info("Timeout waiting for test completion; killing browser...")
# try to extract the minidump stack if the browser hangs
kill_and_get_minidump(context, minidump_dir)
raise TalosError("timeout")
if reader.got_end_timestamp:
for i in range(1, wait_for_quit_timeout):
if proc.wait(1) is not None:
break
if proc.poll() is None:
LOG.info(
"Browser shutdown timed out after {0} seconds, killing"
" process.".format(wait_for_quit_timeout))
kill_and_get_minidump(context, minidump_dir)
raise TalosError(
"Browser shutdown timed out after {0} seconds, killed"
" process.".format(wait_for_quit_timeout))
elif reader.got_timeout:
raise TalosError('TIMEOUT: %s' % reader.timeout_message)
elif reader.got_error:
raise TalosError("unexpected error")
finally:
# this also handle KeyboardInterrupt
# ensure early the process is really terminated
return_code = None
try:
return_code = context.kill_process()
if return_code is None:
return_code = proc.wait(1)
except Exception:
# Maybe killed by kill_and_get_minidump(), maybe ended?
LOG.info("Unable to kill process")
LOG.info(traceback.format_exc())
reader.output.append(
"__startBeforeLaunchTimestamp%d__endBeforeLaunchTimestamp" %
first_time)
reader.output.append(
"__startAfterTerminationTimestamp%d__endAfterTerminationTimestamp" %
(int(time.time()) * 1000))
if return_code is not None:
LOG.process_exit(proc.pid, return_code)
else:
LOG.debug("Unable to detect exit code of the process %s." % proc.pid)
context.output = reader.output
return context
class ServoWebDriverBrowser(Browser):
init_timeout = 300 # Large timeout for cases where we're booting an Android emulator
def __init__(self,
logger,
binary,
debug_info=None,
webdriver_host="127.0.0.1",
server_config=None,
binary_args=None,
user_stylesheets=None,
headless=None):
Browser.__init__(self, logger)
self.binary = binary
self.binary_args = binary_args or []
self.webdriver_host = webdriver_host
self.webdriver_port = None
self.proc = None
self.debug_info = debug_info
self.hosts_path = write_hosts_file(server_config)
self.server_ports = server_config.ports if server_config else {}
self.command = None
self.user_stylesheets = user_stylesheets if user_stylesheets else []
self.headless = headless if headless else False
self.ca_certificate_path = server_config.ssl_config["ca_cert_path"]
def start(self, **kwargs):
self.webdriver_port = get_free_port()
env = os.environ.copy()
env["HOST_FILE"] = self.hosts_path
env["RUST_BACKTRACE"] = "1"
env["EMULATOR_REVERSE_FORWARD_PORTS"] = ",".join(
str(port) for _protocol, ports in self.server_ports.items()
for port in ports if port)
debug_args, command = browser_command(
self.binary, self.binary_args + [
"--hard-fail",
"--webdriver=%s" % self.webdriver_port,
"about:blank",
], self.debug_info)
if self.headless:
command += ["--headless"]
if self.ca_certificate_path:
command += ["--certificate-path", self.ca_certificate_path]
for stylesheet in self.user_stylesheets:
command += ["--user-stylesheet", stylesheet]
self.command = command
self.command = debug_args + self.command
if not self.debug_info or not self.debug_info.interactive:
self.proc = ProcessHandler(self.command,
processOutputLine=[self.on_output],
env=cast_env(env),
storeOutput=False)
self.proc.run()
else:
self.proc = subprocess.Popen(self.command, env=cast_env(env))
self.logger.debug("Servo Started")
def stop(self, force=False):
self.logger.debug("Stopping browser")
if self.proc is not None:
try:
self.proc.kill()
except OSError:
# This can happen on Windows if the process is already dead
pass
def pid(self):
if self.proc is None:
return None
try:
return self.proc.pid
except AttributeError:
return None
def on_output(self, line):
"""Write a line of output from the process to the log"""
self.logger.process_output(self.pid(),
line.decode("utf8", "replace"),
command=" ".join(self.command))
def is_alive(self):
return self.proc.poll() is None
def cleanup(self):
self.stop()
os.remove(self.hosts_path)
def executor_browser(self):
assert self.webdriver_port is not None
return ExecutorBrowser, {
"webdriver_host": self.webdriver_host,
"webdriver_port": self.webdriver_port,
"init_timeout": self.init_timeout
}
