def certutil(self, args, raise_exception=True):
cmd = [self.certutil_path] + list(args)
LOG.info("Certutil: Running command: %s" % " ".join(cmd))
try:
cmd_proc = subprocess.Popen(cmd,
stdout=PIPE,
stderr=PIPE,
env=os.environ.copy())
cmd_output, errs = cmd_proc.communicate()
except subprocess.SubprocessError:
LOG.critical("could not run the certutil command")
raise
if cmd_proc.returncode == 0:
# Debug purpose only remove if stable
LOG.info("Certutil returncode: %s" % cmd_proc.returncode)
LOG.info("Certutil output: %s" % cmd_output)
return cmd_output
else:
if raise_exception:
LOG.critical("Certutil command failed!!")
LOG.info("Certutil returncode: %s" % cmd_proc.returncode)
LOG.info("Certutil output: %s" % cmd_output)
LOG.info("Certutil error: %s" % errs)
raise Exception("Certutil command failed!!")
else:
return False
def is_mitmproxy_cert_installed(self):
"""Verify mitmxproy CA cert was added to Firefox on android"""
LOG.info(
"verifying that the mitmproxy ca cert is installed on android")
# list the certifcates that are in the nss cert db (inside the browser profile dir)
LOG.info(
"getting the list of certs in the nss cert db in the android browser profile"
)
param1 = "sql:%s/" % self.config["local_profile_dir"]
command = [self.certutil, "-d", param1, "-L"]
try:
cmd_output = subprocess.check_output(command,
env=os.environ.copy())
except subprocess.CalledProcessError:
# cmd itself failed
LOG.critical("certutil command failed")
raise
# check output from the certutil command, see if 'mitmproxy-cert' is listed
time.sleep(self.CERTUTIL_SLEEP)
LOG.info(cmd_output)
if "mitmproxy-cert" in cmd_output:
LOG.info(
"verfied the mitmproxy-cert is installed in the nss cert db on android"
)
return True
return False
def import_certificate_in_cert_db(self, cert_db_location, local_cert_path):
# import mitmproxy cert into the db
command = [
self.certutil,
"-A",
"-d",
cert_db_location,
"-n",
"mitmproxy-cert",
"-t",
"TC,,",
"-a",
"-i",
local_cert_path,
]
LOG.info("importing mitmproxy cert into db using command: %s" %
" ".join(command))
cmd_proc = subprocess.Popen(command, env=os.environ.copy())
time.sleep(self.certutil_sleep_seconds)
cmd_terminated = cmd_proc.poll()
if cmd_terminated is None: # None value indicates process hasn't terminated
LOG.critical(
"command to import mitmproxy cert into cert db failed to complete"
)
def install_mitmproxy_cert(self, mitmproxy_proc, browser_path):
"""Install the CA certificate generated by mitmproxy, into geckoview android
If running locally:
1. Will use the `certutil` tool from the local Firefox desktop build
If running in production:
1. Get the tooltools manifest file for downloading hostutils (contains certutil)
2. Get the `certutil` tool by downloading hostutils using the tooltool manifest
Then, both locally and in production:
1. Create an NSS certificate database in the geckoview browser profile dir, only
if it doesn't already exist. Use this certutil command:
`certutil -N -d sql:<path to profile> --empty-password`
2. Import the mitmproxy certificate into the database, i.e.:
`certutil -A -d sql:<path to profile> -n "some nickname" -t TC,, -a -i <path to CA.pem>`
"""
self.CERTUTIL_SLEEP = 10
if self.config['run_local']:
# when running locally, it is found in the Firefox desktop build (..obj../dist/bin)
self.certutil = os.path.join(self.config['obj_path'], 'dist',
'bin')
os.environ['LD_LIBRARY_PATH'] = self.certutil
else:
# must download certutil inside hostutils via tooltool; use this manifest:
# mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest
# after it will be found here inside the worker/bitbar container:
# /builds/worker/workspace/build/hostutils/host-utils-66.0a1.en-US.linux-x86_64
LOG.info("downloading certutil binary (hostutils)")
# get path to the hostutils tooltool manifest; was set earlier in
# mozharness/configs/raptor/android_hw_config.py, to the path i.e.
# mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest
# the bitbar container is always linux64
if os.environ.get('GECKO_HEAD_REPOSITORY', None) is None:
LOG.critical('Abort: unable to get GECKO_HEAD_REPOSITORY')
raise
if os.environ.get('GECKO_HEAD_REV', None) is None:
LOG.critical('Abort: unable to get GECKO_HEAD_REV')
raise
if os.environ.get('HOSTUTILS_MANIFEST_PATH', None) is not None:
manifest_url = os.path.join(
os.environ['GECKO_HEAD_REPOSITORY'], "raw-file",
os.environ['GECKO_HEAD_REV'],
os.environ['HOSTUTILS_MANIFEST_PATH'])
else:
LOG.critical("Abort: unable to get HOSTUTILS_MANIFEST_PATH!")
raise
# first need to download the hostutils tooltool manifest file itself
_dest = os.path.join(self.mozproxy_dir, 'hostutils.manifest')
have_manifest = download_file_from_url(manifest_url, _dest)
if not have_manifest:
LOG.critical(
'failed to download the hostutils tooltool manifest')
raise
# now use the manifest to download hostutils so we can get certutil
tooltool_download(_dest, self.config['run_local'],
self.mozproxy_dir)
# the production bitbar container host is always linux
self.certutil = os.path.join(
self.mozproxy_dir, 'host-utils-67.0a1.en-US.linux-x86_64')
# must add hostutils/certutil to the path
os.environ['LD_LIBRARY_PATH'] = self.certutil
bin_suffix = mozinfo.info.get('bin_suffix', '')
self.certutil = os.path.join(self.certutil, "certutil" + bin_suffix)
if os.path.isfile(self.certutil):
LOG.info("certutil is found at: %s" % self.certutil)
else:
LOG.critical("unable to find certutil at %s" % self.certutil)
raise
# DEFAULT_CERT_PATH has local path and name of mitmproxy cert i.e.
# /home/cltbld/.mitmproxy/mitmproxy-ca-cert.cer
self.local_cert_path = DEFAULT_CERT_PATH
# check if the nss ca cert db already exists in the device profile
LOG.info(
"checking if the nss cert db already exists in the android browser profile"
)
param1 = "sql:%s/" % self.config["local_profile_dir"]
command = [self.certutil, "-d", param1, "-L"]
try:
subprocess.check_output(command, env=os.environ.copy())
LOG.info("the nss cert db already exists")
cert_db_exists = True
except subprocess.CalledProcessError:
# this means the nss cert db doesn't exist yet
LOG.info("nss cert db doesn't exist yet")
cert_db_exists = False
# try a forced pause between certutil cmds; possibly reduce later
time.sleep(self.CERTUTIL_SLEEP)
if not cert_db_exists:
# create cert db if it doesn't already exist; it may exist already
# if a previous pageload test ran in the same test suite
param1 = "sql:%s/" % self.config["local_profile_dir"]
command = [
self.certutil, "-N", "-v", "-d", param1, "--empty-password"
]
LOG.info("creating nss cert database using command: %s" %
" ".join(command))
cmd_proc = subprocess.Popen(command, env=os.environ.copy())
time.sleep(self.CERTUTIL_SLEEP)
cmd_terminated = cmd_proc.poll()
if cmd_terminated is None: # None value indicates process hasn't terminated
LOG.critical("nss cert db creation command failed to complete")
raise
# import mitmproxy cert into the db
command = [
self.certutil,
"-A",
"-d",
param1,
"-n",
"mitmproxy-cert",
"-t",
"TC,,",
"-a",
"-i",
self.local_cert_path,
]
LOG.info("importing mitmproxy cert into db using command: %s" %
" ".join(command))
cmd_proc = subprocess.Popen(command, env=os.environ.copy())
time.sleep(self.CERTUTIL_SLEEP)
cmd_terminated = cmd_proc.poll()
if cmd_terminated is None: # None value indicates process hasn't terminated
LOG.critical(
"command to import mitmproxy cert into cert db failed to complete"
)
# cannot continue if failed to add CA cert to Firefox, need to check
if not self.is_mitmproxy_cert_installed():
LOG.error(
"Aborting: failed to install mitmproxy CA cert into Firefox")
self.stop_mitmproxy_playback()
sys.exit()
def start_mitmproxy_playback(
self,
mitmdump_path,
mitmproxy_recording_path,
mitmproxy_recordings_list,
browser_path,
):
"""Startup mitmproxy and replay the specified flow file"""
LOG.info("mitmdump path: %s" % mitmdump_path)
LOG.info("recording path: %s" % mitmproxy_recording_path)
LOG.info("recordings list: %s" % mitmproxy_recordings_list)
LOG.info("browser path: %s" % browser_path)
mitmproxy_recordings = []
# recording names can be provided in comma-separated list; build py list including path
for recording in mitmproxy_recordings_list:
if not os.path.isfile(
os.path.join(mitmproxy_recording_path, recording)):
LOG.critical("Recording file {} cannot be found!".format(
os.path.join(mitmproxy_recording_path, recording)))
raise Exception("Recording file {} cannot be found!".format(
os.path.join(mitmproxy_recording_path, recording)))
mitmproxy_recordings.append(
os.path.join(mitmproxy_recording_path, recording))
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
env["PATH"] = os.path.dirname(browser_path) + ";" + env["PATH"]
command = [mitmdump_path, "-k", "-q"]
if "custom_script" in self.config:
# cmd line to start mitmproxy playback using custom playback script is as follows:
# <path>/mitmdump -s "<path>/alternate-server-replay.py
# <path>recording-1.mp <path>recording-2.mp..."
custom_script = self.config["custom_script"] + " " + " ".join(
mitmproxy_recordings)
# this part is platform-specific
if mozinfo.os == "win":
custom_script = '""' + custom_script.replace("\\",
"\\\\\\") + '""'
sys.path.insert(1, mitmdump_path)
command.extend(["-s", custom_script])
LOG.info("Starting mitmproxy playback using env path: %s" %
env["PATH"])
LOG.info("Starting mitmproxy playback using command: %s" %
" ".join(command))
# to turn off mitmproxy log output, use these params for Popen:
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
mitmproxy_proc = ProcessHandler(command, env=env)
mitmproxy_proc.run()
# XXX replace the code below with a loop with a connection attempt
# Bug 1532557
time.sleep(MITMDUMP_SLEEP)
data = mitmproxy_proc.poll()
if data is None: # None value indicates process hasn't terminated
LOG.info("Mitmproxy playback successfully started as pid %d" %
mitmproxy_proc.pid)
return mitmproxy_proc
# cannot continue as we won't be able to playback the pages
LOG.error(
"Aborting: mitmproxy playback process failed to start, poll returned: %s"
% data)
# XXX here we might end up with a ghost mitmproxy
sys.exit()
