def download(self):
"""Download and unpack mitmproxy binary and pageset using tooltool"""
if not os.path.exists(self.mozproxy_dir):
os.makedirs(self.mozproxy_dir)
LOG.info("downloading mitmproxy binary")
_manifest = os.path.join(here, self.config["playback_binary_manifest"])
transformed_manifest = transform_platform(_manifest,
self.config["platform"])
tooltool_download(transformed_manifest, self.config["run_local"],
self.mozproxy_dir)
if "playback_pageset_manifest" in self.config:
# we use one pageset for all platforms
LOG.info("downloading mitmproxy pageset")
_manifest = self.config["playback_pageset_manifest"]
transformed_manifest = transform_platform(_manifest,
self.config["platform"])
tooltool_download(transformed_manifest, self.config["run_local"],
self.mozproxy_dir)
if "playback_artifacts" in self.config:
artifacts = self.config["playback_artifacts"].split(",")
for artifact in artifacts:
artifact = artifact.strip()
if not artifact:
continue
artifact_name = artifact.split("/")[-1]
dest = os.path.join(self.mozproxy_dir, artifact_name)
download_file_from_url(artifact, dest, extract=True)
def download(self):
"""Download and unpack mitmproxy binary and pageset using tooltool"""
if not os.path.exists(self.mozproxy_dir):
os.makedirs(self.mozproxy_dir)
_manifest = os.path.join(here, self.config["playback_binary_manifest"])
transformed_manifest = transform_platform(_manifest,
self.config["platform"])
# generate the mitmdump_path
self.mitmdump_path = os.path.normpath(
os.path.join(
self.mozproxy_dir,
"mitmdump-%s" % self.config["playback_version"],
"mitmdump",
))
# Check if mitmproxy bin exists
if os.path.exists(self.mitmdump_path):
LOG.info("mitmproxy binary already exists. Skipping download")
else:
# Download and unpack mitmproxy binary
download_path = os.path.dirname(self.mitmdump_path)
LOG.info("create mitmproxy %s dir" %
self.config["playback_version"])
if not os.path.exists(download_path):
os.makedirs(download_path)
LOG.info("downloading mitmproxy binary")
tooltool_download(transformed_manifest, self.config["run_local"],
download_path)
if "playback_pageset_manifest" in self.config:
# we use one pageset for all platforms
LOG.info("downloading mitmproxy pageset")
_manifest = self.config["playback_pageset_manifest"]
transformed_manifest = transform_platform(_manifest,
self.config["platform"])
tooltool_download(transformed_manifest, self.config["run_local"],
self.mozproxy_dir)
if "playback_artifacts" in self.config:
artifacts = self.config["playback_artifacts"].split(",")
for artifact in artifacts:
artifact = artifact.strip()
if not artifact:
continue
artifact_name = artifact.split("/")[-1]
if artifact_name.endswith(".manifest"):
tooltool_download(artifact, self.config["run_local"],
self.mozproxy_dir)
else:
dest = os.path.join(self.mozproxy_dir, artifact_name)
download_file_from_url(artifact, dest, extract=True)
def download_playback_files(self):
# Detect type of file from playback_files and download accordingly
if "playback_files" not in self.config:
LOG.error(
"playback_files value was not provided. Proxy service wont' start "
)
raise Exception("Please provide a playback_files list.")
if not isinstance(self.config["playback_files"], list):
LOG.error("playback_files should be a list")
raise Exception("playback_files should be a list")
for playback_file in self.config["playback_files"]:
if playback_file.startswith(
"https://") and "mozilla.com" in playback_file:
# URL provided
dest = os.path.join(self.mozproxy_dir,
os.path.basename(playback_file))
download_file_from_url(playback_file,
self.mozproxy_dir,
extract=False)
# Add Downloaded file to recordings list
LOG.info("Adding %s to recording list" % dest)
self.recordings.append(RecordingFile(dest))
continue
if not os.path.exists(playback_file):
LOG.error(
"Zip or manifest file path (%s) does not exist. Please provide a valid path!"
% playback_file)
raise Exception("Zip or manifest file path does not exist")
if os.path.splitext(playback_file)[1] == ".zip":
# zip file path provided
LOG.info("Adding %s to recording list" % playback_file)
self.recordings.append(RecordingFile(playback_file))
elif os.path.splitext(playback_file)[1] == ".manifest":
# manifest file path provided
self.download_manifest_file(playback_file)
def download_and_install_host_utils(self):
"""
If running locally:
1. Will use the `certutil` tool from the local Firefox desktop build
If running in production:
1. Get the tooltools manifest file for downloading hostutils (contains certutil)
2. Get the `certutil` tool by downloading hostutils using the tooltool manifest
"""
if self.config["run_local"]:
# when running locally, it is found in the Firefox desktop build (..obj../dist/bin)
self.certutil_path = os.path.join(os.environ["MOZ_HOST_BIN"],
"certutil")
if not (os.path.isfile(self.certutil_path)
and os.access(self.certutil_path, os.X_OK)):
raise Exception("Abort: unable to execute certutil: {}".format(
self.certutil_path))
self.certutil_path = os.environ["MOZ_HOST_BIN"]
os.environ["LD_LIBRARY_PATH"] = self.certutil_path
else:
# must download certutil inside hostutils via tooltool; use this manifest:
# mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest
# after it will be found here inside the worker/bitbar container:
# /builds/worker/workspace/build/hostutils/host-utils-66.0a1.en-US.linux-x86_64
LOG.info("downloading certutil binary (hostutils)")
# get path to the hostutils tooltool manifest; was set earlier in
# mozharness/configs/raptor/android_hw_config.py, to the path i.e.
# mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest
# the bitbar container is always linux64
if os.environ.get("GECKO_HEAD_REPOSITORY", None) is None:
raise Exception("Abort: unable to get GECKO_HEAD_REPOSITORY")
if os.environ.get("GECKO_HEAD_REV", None) is None:
raise Exception("Abort: unable to get GECKO_HEAD_REV")
if os.environ.get("HOSTUTILS_MANIFEST_PATH", None) is not None:
manifest_url = os.path.join(
os.environ["GECKO_HEAD_REPOSITORY"],
"raw-file",
os.environ["GECKO_HEAD_REV"],
os.environ["HOSTUTILS_MANIFEST_PATH"],
)
else:
raise Exception(
"Abort: unable to get HOSTUTILS_MANIFEST_PATH!")
# first need to download the hostutils tooltool manifest file itself
_dest = os.path.join(self.mozproxy_dir, "hostutils.manifest")
have_manifest = download_file_from_url(manifest_url, _dest)
if not have_manifest:
raise Exception(
"failed to download the hostutils tooltool manifest")
# now use the manifest to download hostutils so we can get certutil
tooltool_download(_dest, self.config["run_local"],
self.mozproxy_dir)
# the production bitbar container host is always linux
self.certutil_path = glob.glob(
os.path.join(self.mozproxy_dir, "host-utils*[!z|checksum]"))[0]
# must add hostutils/certutil to the path
os.environ["LD_LIBRARY_PATH"] = self.certutil_path
bin_suffix = mozinfo.info.get("bin_suffix", "")
self.certutil_path = os.path.join(self.certutil_path,
"certutil" + bin_suffix)
if os.path.isfile(self.certutil_path):
LOG.info("certutil is found at: %s" % self.certutil_path)
else:
raise Exception("unable to find certutil at %s" %
self.certutil_path)
def install_mitmproxy_cert(self, mitmproxy_proc, browser_path):
"""Install the CA certificate generated by mitmproxy, into geckoview android
If running locally:
1. Will use the `certutil` tool from the local Firefox desktop build
If running in production:
1. Get the tooltools manifest file for downloading hostutils (contains certutil)
2. Get the `certutil` tool by downloading hostutils using the tooltool manifest
Then, both locally and in production:
1. Create an NSS certificate database in the geckoview browser profile dir, only
if it doesn't already exist. Use this certutil command:
`certutil -N -d sql:<path to profile> --empty-password`
2. Import the mitmproxy certificate into the database, i.e.:
`certutil -A -d sql:<path to profile> -n "some nickname" -t TC,, -a -i <path to CA.pem>`
"""
self.CERTUTIL_SLEEP = 10
if self.config['run_local']:
# when running locally, it is found in the Firefox desktop build (..obj../dist/bin)
self.certutil = os.path.join(self.config['obj_path'], 'dist',
'bin')
os.environ['LD_LIBRARY_PATH'] = self.certutil
else:
# must download certutil inside hostutils via tooltool; use this manifest:
# mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest
# after it will be found here inside the worker/bitbar container:
# /builds/worker/workspace/build/hostutils/host-utils-66.0a1.en-US.linux-x86_64
LOG.info("downloading certutil binary (hostutils)")
# get path to the hostutils tooltool manifest; was set earlier in
# mozharness/configs/raptor/android_hw_config.py, to the path i.e.
# mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest
# the bitbar container is always linux64
if os.environ.get('GECKO_HEAD_REPOSITORY', None) is None:
LOG.critical('Abort: unable to get GECKO_HEAD_REPOSITORY')
raise
if os.environ.get('GECKO_HEAD_REV', None) is None:
LOG.critical('Abort: unable to get GECKO_HEAD_REV')
raise
if os.environ.get('HOSTUTILS_MANIFEST_PATH', None) is not None:
manifest_url = os.path.join(
os.environ['GECKO_HEAD_REPOSITORY'], "raw-file",
os.environ['GECKO_HEAD_REV'],
os.environ['HOSTUTILS_MANIFEST_PATH'])
else:
LOG.critical("Abort: unable to get HOSTUTILS_MANIFEST_PATH!")
raise
# first need to download the hostutils tooltool manifest file itself
_dest = os.path.join(self.mozproxy_dir, 'hostutils.manifest')
have_manifest = download_file_from_url(manifest_url, _dest)
if not have_manifest:
LOG.critical(
'failed to download the hostutils tooltool manifest')
raise
# now use the manifest to download hostutils so we can get certutil
tooltool_download(_dest, self.config['run_local'],
self.mozproxy_dir)
# the production bitbar container host is always linux
self.certutil = os.path.join(
self.mozproxy_dir, 'host-utils-67.0a1.en-US.linux-x86_64')
# must add hostutils/certutil to the path
os.environ['LD_LIBRARY_PATH'] = self.certutil
bin_suffix = mozinfo.info.get('bin_suffix', '')
self.certutil = os.path.join(self.certutil, "certutil" + bin_suffix)
if os.path.isfile(self.certutil):
LOG.info("certutil is found at: %s" % self.certutil)
else:
LOG.critical("unable to find certutil at %s" % self.certutil)
raise
# DEFAULT_CERT_PATH has local path and name of mitmproxy cert i.e.
# /home/cltbld/.mitmproxy/mitmproxy-ca-cert.cer
self.local_cert_path = DEFAULT_CERT_PATH
# check if the nss ca cert db already exists in the device profile
LOG.info(
"checking if the nss cert db already exists in the android browser profile"
)
param1 = "sql:%s/" % self.config["local_profile_dir"]
command = [self.certutil, "-d", param1, "-L"]
try:
subprocess.check_output(command, env=os.environ.copy())
LOG.info("the nss cert db already exists")
cert_db_exists = True
except subprocess.CalledProcessError:
# this means the nss cert db doesn't exist yet
LOG.info("nss cert db doesn't exist yet")
cert_db_exists = False
# try a forced pause between certutil cmds; possibly reduce later
time.sleep(self.CERTUTIL_SLEEP)
if not cert_db_exists:
# create cert db if it doesn't already exist; it may exist already
# if a previous pageload test ran in the same test suite
param1 = "sql:%s/" % self.config["local_profile_dir"]
command = [
self.certutil, "-N", "-v", "-d", param1, "--empty-password"
]
LOG.info("creating nss cert database using command: %s" %
" ".join(command))
cmd_proc = subprocess.Popen(command, env=os.environ.copy())
time.sleep(self.CERTUTIL_SLEEP)
cmd_terminated = cmd_proc.poll()
if cmd_terminated is None: # None value indicates process hasn't terminated
LOG.critical("nss cert db creation command failed to complete")
raise
# import mitmproxy cert into the db
command = [
self.certutil,
"-A",
"-d",
param1,
"-n",
"mitmproxy-cert",
"-t",
"TC,,",
"-a",
"-i",
self.local_cert_path,
]
LOG.info("importing mitmproxy cert into db using command: %s" %
" ".join(command))
cmd_proc = subprocess.Popen(command, env=os.environ.copy())
time.sleep(self.CERTUTIL_SLEEP)
cmd_terminated = cmd_proc.poll()
if cmd_terminated is None: # None value indicates process hasn't terminated
LOG.critical(
"command to import mitmproxy cert into cert db failed to complete"
)
# cannot continue if failed to add CA cert to Firefox, need to check
if not self.is_mitmproxy_cert_installed():
LOG.error(
"Aborting: failed to install mitmproxy CA cert into Firefox")
self.stop_mitmproxy_playback()
sys.exit()
def test_download_file(*args):
with tempdir() as dest_dir:
dest = os.path.join(dest_dir, "archive.tar.gz")
download_file_from_url("http://example.com/archive.tar.gz", dest, extract=True)
# archive.tar.gz contains hey.txt, if it worked we should see it
assert os.path.exists(os.path.join(dest_dir, "hey.txt"))
