def install_mitmproxy_cert(self, browser_path):
"""Install the CA certificate generated by mitmproxy, into geckoview android
1. Create an NSS certificate database in the geckoview browser profile dir, only
if it doesn't already exist. Use this certutil command:
`certutil -N -d sql:<path to profile> --empty-password`
2. Import the mitmproxy certificate into the database, i.e.:
`certutil -A -d sql:<path to profile> -n "some nickname" -t TC,, -a -i <path to CA.pem>`
"""
cert_db_location = "sql:%s/" % self.config["local_profile_dir"]
if not self.cert_db_exists(cert_db_location):
self.create_cert_db(cert_db_location)
# DEFAULT_CERT_PATH has local path and name of mitmproxy cert i.e.
# /home/cltbld/.mitmproxy/mitmproxy-ca-cert.cer
self.import_certificate_in_cert_db(cert_db_location, DEFAULT_CERT_PATH)
# cannot continue if failed to add CA cert to Firefox, need to check
if not self.is_mitmproxy_cert_installed(cert_db_location):
LOG.error(
"Aborting: failed to install mitmproxy CA cert into Firefox")
self.stop_mitmproxy_playback()
sys.exit()
def is_mitmproxy_cert_installed(self, cert_db_location):
"""Verify mitmxproy CA cert was added to Firefox on android"""
LOG.info(
"verifying that the mitmproxy ca cert is installed on android")
# list the certifcates that are in the nss cert db (inside the browser profile dir)
LOG.info(
"getting the list of certs in the nss cert db in the android browser profile"
)
command = [self.certutil, "-d", cert_db_location, "-L"]
try:
cmd_output = subprocess.check_output(command,
env=os.environ.copy())
except subprocess.CalledProcessError:
# cmd itself failed
LOG.error(cmd_output)
raise Exception("certutil command failed")
# check output from the certutil command, see if 'mitmproxy-cert' is listed
time.sleep(self.certutil_sleep_seconds)
LOG.info(cmd_output)
if "mitmproxy-cert" in cmd_output:
LOG.info(
"verfied the mitmproxy-cert is installed in the nss cert db on android"
)
return True
return False
def stop_mitmproxy_playback(self):
"""Stop the mitproxy server playback"""
if self.mitmproxy_proc is None or self.mitmproxy_proc.poll() is not None:
return
LOG.info(
"Stopping mitmproxy playback, killing process %d" % self.mitmproxy_proc.pid
)
exit_code = self.mitmproxy_proc.kill()
if exit_code != 0:
# I *think* we can still continue, as process will be automatically
# killed anyway when mozharness is done (?) if not, we won't be able
# to startup mitmxproy next time if it is already running
if exit_code is None:
LOG.error("Failed to kill the mitmproxy playback process")
elif exit_code == 572 and mozinfo.os == "win":
LOG.info("Successfully killed the mitmproxy playback process with exit code 572")
else:
log_func = LOG.error
if self.ignore_mitmdump_exit_failure:
log_func = LOG.info
log_func("Mitmproxy exited with error code %d" % exit_code)
else:
LOG.info("Successfully killed the mitmproxy playback process")
self.mitmproxy_proc = None
def __init__(self, config):
self.config = config
self.host = (
"127.0.0.1" if "localhost" in self.config["host"] else self.config["host"]
)
self.port = None
self.mitmproxy_proc = None
self.mitmdump_path = None
self.recordings = []
self.browser_path = ""
if config.get("binary", None):
self.browser_path = os.path.normpath(config.get("binary"))
self.policies_dir = None
self.ignore_mitmdump_exit_failure = config.get(
"ignore_mitmdump_exit_failure", False
)
self.recording_paths = None
if self.config.get("playback_version") is None:
LOG.error(
"mitmproxy was not provided with a 'playback_version' "
"Please provide a valid playback version"
)
raise Exception("playback_version not specified!")
# mozproxy_dir is where we will download all mitmproxy required files
# when running locally it comes from obj_path via mozharness/mach
if self.config.get("obj_path") is not None:
self.mozproxy_dir = self.config.get("obj_path")
else:
# in production it is ../tasks/task_N/build/, in production that dir
# is not available as an envvar, however MOZ_UPLOAD_DIR is set as
# ../tasks/task_N/build/blobber_upload_dir so take that and go up 1 level
self.mozproxy_dir = os.path.dirname(
os.path.dirname(os.environ["MOZ_UPLOAD_DIR"])
)
self.mozproxy_dir = os.path.join(self.mozproxy_dir, "testing", "mozproxy")
self.upload_dir = os.environ.get("MOZ_UPLOAD_DIR", self.mozproxy_dir)
LOG.info(
"mozproxy_dir used for mitmproxy downloads and exe files: %s"
% self.mozproxy_dir
)
# setting up the MOZPROXY_DIR env variable so custom scripts know
# where to get the data
os.environ["MOZPROXY_DIR"] = self.mozproxy_dir
LOG.info("Playback tool: %s" % self.config["playback_tool"])
LOG.info("Playback tool version: %s" % self.config["playback_version"])
def start(self):
# go ahead and download and setup mitmproxy
self.download()
# mitmproxy must be started before setup, so that the CA cert is available
self.start_mitmproxy(self.mitmdump_path, self.browser_path)
# In case the setup fails, we want to stop the process before raising.
try:
self.setup()
except Exception:
try:
self.stop()
except Exception:
LOG.error("MitmProxy failed to STOP.", exc_info=True)
LOG.error("Setup of MitmProxy failed.", exc_info=True)
raise
def stop_mitmproxy_playback(self):
"""Stop the mitproxy server playback"""
mitmproxy_proc = self.mitmproxy_proc
LOG.info("Stopping mitmproxy playback, killing process %d" %
mitmproxy_proc.pid)
mitmproxy_proc.kill()
time.sleep(MITMDUMP_SLEEP)
status = mitmproxy_proc.poll()
if status is None: # None value indicates process hasn't terminated
# I *think* we can still continue, as process will be automatically
# killed anyway when mozharness is done (?) if not, we won't be able
# to startup mitmxproy next time if it is already running
LOG.error("Failed to kill the mitmproxy playback process")
LOG.info(str(status))
else:
LOG.info("Successfully killed the mitmproxy playback process")
def install_mitmproxy_cert(self, browser_path):
"""Install the CA certificate generated by mitmproxy, into Firefox
1. Create a dir called 'distribution' in the same directory as the Firefox executable
2. Create the policies.json file inside that folder; which points to the certificate
location, and turns on the the browser proxy settings
"""
LOG.info("Installing mitmproxy CA certficate into Firefox")
# browser_path is the exe, we want the folder
self.policies_dir = os.path.dirname(browser_path)
# on macosx we need to remove the last folders 'MacOS'
# and the policies json needs to go in ../Content/Resources/
if "mac" in self.config["platform"]:
self.policies_dir = os.path.join(self.policies_dir[:-6],
"Resources")
# for all platforms the policies json goes in a 'distribution' dir
self.policies_dir = os.path.join(self.policies_dir, "distribution")
self.cert_path = DEFAULT_CERT_PATH
# for windows only
if mozinfo.os == "win":
self.cert_path = self.cert_path.replace("\\", "\\\\")
if not os.path.exists(self.policies_dir):
LOG.info("creating folder: %s" % self.policies_dir)
os.makedirs(self.policies_dir)
else:
LOG.info("folder already exists: %s" % self.policies_dir)
self.write_policies_json(
self.policies_dir,
policies_content=POLICIES_CONTENT_ON % {
"cert": self.cert_path,
"host": self.host,
"port": self.port
},
)
# cannot continue if failed to add CA cert to Firefox, need to check
if not self.is_mitmproxy_cert_installed():
LOG.error(
"Aborting: failed to install mitmproxy CA cert into Firefox desktop"
)
self.stop_mitmproxy_playback()
sys.exit()
def start_mitmproxy_playback(
self,
mitmdump_path,
browser_path,
):
"""Startup mitmproxy and replay the specified flow file"""
LOG.info("mitmdump path: %s" % mitmdump_path)
LOG.info("browser path: %s" % browser_path)
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
env["PATH"] = os.path.dirname(browser_path) + ";" + env["PATH"]
command = [mitmdump_path, "-k"]
if "playback_tool_args" in self.config:
command.extend(self.config["playback_tool_args"])
LOG.info("Starting mitmproxy playback using env path: %s" %
env["PATH"])
LOG.info("Starting mitmproxy playback using command: %s" %
" ".join(command))
# to turn off mitmproxy log output, use these params for Popen:
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
mitmproxy_proc = ProcessHandler(command,
logfile=os.path.join(
self.upload_dir, "mitmproxy.log"),
env=env)
mitmproxy_proc.run()
# XXX replace the code below with a loop with a connection attempt
# Bug 1532557
time.sleep(MITMDUMP_SLEEP)
data = mitmproxy_proc.poll()
if data is None: # None value indicates process hasn't terminated
LOG.info("Mitmproxy playback successfully started as pid %d" %
mitmproxy_proc.pid)
return mitmproxy_proc
# cannot continue as we won't be able to playback the pages
LOG.error(
"Aborting: mitmproxy playback process failed to start, poll returned: %s"
% data)
# XXX here we might end up with a ghost mitmproxy
sys.exit()
def start_mitmproxy_playback(self, mitmdump_path, browser_path):
"""Startup mitmproxy and replay the specified flow file"""
if self.mitmproxy_proc is not None:
raise Exception("Proxy already started.")
LOG.info("mitmdump path: %s" % mitmdump_path)
LOG.info("browser path: %s" % browser_path)
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
env["PATH"] = os.path.dirname(browser_path) + os.pathsep + env["PATH"]
command = [mitmdump_path]
if "playback_tool_args" in self.config:
command.extend(self.config["playback_tool_args"])
LOG.info("Starting mitmproxy playback using env path: %s" %
env["PATH"])
LOG.info("Starting mitmproxy playback using command: %s" %
" ".join(command))
# to turn off mitmproxy log output, use these params for Popen:
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
self.mitmproxy_proc = ProcessHandler(command,
logfile=os.path.join(
self.upload_dir,
"mitmproxy.log"),
env=env)
self.mitmproxy_proc.run()
end_time = time.time() + MITMDUMP_COMMAND_TIMEOUT
ready = False
while time.time() < end_time:
ready = self.check_proxy()
if ready:
LOG.info("Mitmproxy playback successfully started as pid %d" %
self.mitmproxy_proc.pid)
return
time.sleep(0.25)
# cannot continue as we won't be able to playback the pages
LOG.error("Aborting: Mitmproxy process did not startup")
self.stop_mitmproxy_playback()
sys.exit() # XXX why do we need to do that? a raise is not enough?
def stop_mitmproxy_playback(self):
"""Stop the mitproxy server playback"""
if self.mitmproxy_proc is None or self.mitmproxy_proc.poll() is not None:
return
LOG.info(
"Stopping mitmproxy playback, killing process %d" % self.mitmproxy_proc.pid
)
# On Windows, mozprocess brutally kills mitmproxy with TerminateJobObject
# The process has no chance to gracefully shutdown.
# Here, we send the process a break event to give it a chance to wrapup.
# See the signal handler in the alternate-server-replay-4.0.4.py script
if mozinfo.os == "win":
LOG.info("Sending CTRL_BREAK_EVENT to mitmproxy")
os.kill(self.mitmproxy_proc.pid, signal.CTRL_BREAK_EVENT)
time.sleep(2)
exit_code = self.mitmproxy_proc.kill()
self.mitmproxy_proc = None
if exit_code != 0:
if exit_code is None:
LOG.error("Failed to kill the mitmproxy playback process")
return
if mozinfo.os == "win":
from mozprocess.winprocess import ERROR_CONTROL_C_EXIT # noqa
if exit_code == ERROR_CONTROL_C_EXIT:
LOG.info(
"Successfully killed the mitmproxy playback process"
" with exit code %d" % exit_code
)
return
log_func = LOG.error
if self.ignore_mitmdump_exit_failure:
log_func = LOG.info
log_func("Mitmproxy exited with error code %d" % exit_code)
else:
LOG.info("Successfully killed the mitmproxy playback process")
def download_playback_files(self):
# Detect type of file from playback_files and download accordingly
if "playback_files" not in self.config:
LOG.error(
"playback_files value was not provided. Proxy service wont' start "
)
raise Exception("Please provide a playback_files list.")
if not isinstance(self.config["playback_files"], list):
LOG.error("playback_files should be a list")
raise Exception("playback_files should be a list")
for playback_file in self.config["playback_files"]:
if playback_file.startswith(
"https://") and "mozilla.com" in playback_file:
# URL provided
dest = os.path.join(self.mozproxy_dir,
os.path.basename(playback_file))
download_file_from_url(playback_file,
self.mozproxy_dir,
extract=False)
# Add Downloaded file to recordings list
LOG.info("Adding %s to recording list" % dest)
self.recordings.append(RecordingFile(dest))
continue
if not os.path.exists(playback_file):
LOG.error(
"Zip or manifest file path (%s) does not exist. Please provide a valid path!"
% playback_file)
raise Exception("Zip or manifest file path does not exist")
if os.path.splitext(playback_file)[1] == ".zip":
# zip file path provided
LOG.info("Adding %s to recording list" % playback_file)
self.recordings.append(RecordingFile(playback_file))
elif os.path.splitext(playback_file)[1] == ".manifest":
# manifest file path provided
self.download_manifest_file(playback_file)
def start_mitmproxy_playback(self, mitmdump_path, browser_path):
"""Startup mitmproxy and replay the specified flow file"""
if self.mitmproxy_proc is not None:
raise Exception("Proxy already started.")
self.port = get_available_port()
LOG.info("mitmdump path: %s" % mitmdump_path)
LOG.info("browser path: %s" % browser_path)
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
env["PATH"] = os.path.dirname(browser_path) + os.pathsep + env["PATH"]
command = [mitmdump_path]
# add proxy host and port options
command.extend(
["--listen-host", self.host, "--listen-port",
str(self.port)])
if "playback_tool_args" in self.config:
LOG.info("Staring Proxy using provided command line!")
command.extend(self.config["playback_tool_args"])
elif "playback_files" in self.config:
script = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"scripts",
"alternate-server-replay.py",
)
self.recording_paths = [
normalize_path(recording_path)
for recording_path in self.config["playback_files"]
]
if self.config["playback_version"] in ["4.0.4", "5.0.1"]:
args = [
"-v",
"--set",
"upstream_cert=false",
"--set",
"upload_dir=" + normalize_path(self.upload_dir),
"--set",
"websocket=false",
"--set",
"server_replay_files={}".format(",".join(
self.recording_paths)),
"--scripts",
normalize_path(script),
]
command.extend(args)
else:
raise Exception("Mitmproxy version is unknown!")
else:
raise Exception(
"Mitmproxy can't start playback! Playback settings missing.")
LOG.info("Starting mitmproxy playback using env path: %s" %
env["PATH"])
LOG.info("Starting mitmproxy playback using command: %s" %
" ".join(command))
# to turn off mitmproxy log output, use these params for Popen:
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
self.mitmproxy_proc = ProcessHandler(
command,
logfile=os.path.join(self.upload_dir, "mitmproxy.log"),
env=env,
processStderrLine=LOG.error,
storeOutput=False,
)
self.mitmproxy_proc.run()
end_time = time.time() + MITMDUMP_COMMAND_TIMEOUT
ready = False
while time.time() < end_time:
ready = self.check_proxy(host=self.host, port=self.port)
if ready:
LOG.info(
"Mitmproxy playback successfully started on %s:%d as pid %d"
% (self.host, self.port, self.mitmproxy_proc.pid))
return
time.sleep(0.25)
# cannot continue as we won't be able to playback the pages
LOG.error("Aborting: Mitmproxy process did not startup")
self.stop_mitmproxy_playback()
sys.exit() # XXX why do we need to do that? a raise is not enough?
def __init__(self, config):
self.config = config
self.host = ("127.0.0.1" if "localhost" in self.config["host"] else
self.config["host"])
self.port = None
self.mitmproxy_proc = None
self.mitmdump_path = None
self.record_mode = config.get("record", False)
self.recording = None
self.playback_files = []
self.browser_path = ""
if config.get("binary", None):
self.browser_path = os.path.normpath(config.get("binary"))
self.policies_dir = None
self.ignore_mitmdump_exit_failure = config.get(
"ignore_mitmdump_exit_failure", False)
if self.record_mode:
if "recording_file" not in self.config:
LOG.error(
"recording_file value was not provided. Proxy service wont' start "
)
raise Exception("Please provide a playback_files list.")
if not isinstance(self.config.get("recording_file"),
six.string_types):
LOG.error("recording_file argument type is not str!")
raise Exception("recording_file argument type invalid!")
if not os.path.splitext(
self.config.get("recording_file"))[1] == ".zip":
LOG.error("Recording file type (%s) should be a zip. "
"Please provide a valid file type!" %
self.config.get("recording_file"))
raise Exception("Recording file type should be a zip")
if os.path.exists(self.config.get("recording_file")):
LOG.error("Recording file (%s) already exists."
"Please provide a valid file path!" %
self.config.get("recording_file"))
raise Exception("Recording file already exists.")
if self.config.get("playback_files", False):
LOG.error(
"Record mode is True and playback_files where provided!")
raise Exception("playback_files specified during record!")
if self.config.get("playback_version") is None:
LOG.error("mitmproxy was not provided with a 'playback_version' "
"Please provide a valid playback version")
raise Exception("playback_version not specified!")
# mozproxy_dir is where we will download all mitmproxy required files
# when running locally it comes from obj_path via mozharness/mach
if self.config.get("obj_path") is not None:
self.mozproxy_dir = self.config.get("obj_path")
else:
# in production it is ../tasks/task_N/build/, in production that dir
# is not available as an envvar, however MOZ_UPLOAD_DIR is set as
# ../tasks/task_N/build/blobber_upload_dir so take that and go up 1 level
self.mozproxy_dir = os.path.dirname(
os.path.dirname(os.environ["MOZ_UPLOAD_DIR"]))
self.mozproxy_dir = os.path.join(self.mozproxy_dir, "testing",
"mozproxy")
self.upload_dir = os.environ.get("MOZ_UPLOAD_DIR", self.mozproxy_dir)
LOG.info(
"mozproxy_dir used for mitmproxy downloads and exe files: %s" %
self.mozproxy_dir)
# setting up the MOZPROXY_DIR env variable so custom scripts know
# where to get the data
os.environ["MOZPROXY_DIR"] = self.mozproxy_dir
LOG.info("Playback tool: %s" % self.config["playback_tool"])
LOG.info("Playback tool version: %s" % self.config["playback_version"])
def start_mitmproxy(self, mitmdump_path, browser_path):
"""Startup mitmproxy and replay the specified flow file"""
if self.mitmproxy_proc is not None:
raise Exception("Proxy already started.")
self.port = get_available_port()
LOG.info("mitmdump path: %s" % mitmdump_path)
LOG.info("browser path: %s" % browser_path)
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
env["PATH"] = os.path.dirname(browser_path) + os.pathsep + env["PATH"]
command = [mitmdump_path]
if self.config.get("verbose", False):
# Generate mitmproxy verbose logs
command.extend(["-v"])
# add proxy host and port options
command.extend(
["--listen-host", self.host, "--listen-port",
str(self.port)])
# record mode
if self.record_mode:
# generate recording script paths
inject_deterministic = os.path.join(
mitm_folder,
"scripts",
"inject-deterministic.py",
)
http_protocol_extractor = os.path.join(
mitm_folder,
"scripts",
"http_protocol_extractor.py",
)
args = [
"--save-stream-file",
normalize_path(self.recording.recording_path),
"--set",
"websocket=false",
"--scripts",
inject_deterministic,
"--scripts",
http_protocol_extractor,
]
command.extend(args)
self.recording.set_metadata("proxy_version",
self.config["playback_version"])
else:
# playback mode
if len(self.playback_files) > 0:
script = os.path.join(
mitm_folder,
"scripts",
"alternate-server-replay.py",
)
if self.config["playback_version"] in [
"4.0.4", "5.1.1", "6.0.2"
]:
args = [
"--set",
"upstream_cert=false",
"--set",
"upload_dir=" + normalize_path(self.upload_dir),
"--set",
"websocket=false",
"--set",
"server_replay_files={}".format(",".join([
normalize_path(playback_file.recording_path)
for playback_file in self.playback_files
])),
"--scripts",
normalize_path(script),
]
command.extend(args)
else:
raise Exception("Mitmproxy version is unknown!")
else:
raise Exception(
"Mitmproxy can't start playback! Playback settings missing."
)
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
if not os.path.dirname(self.browser_path) in env["PATH"]:
env["PATH"] = os.path.dirname(
self.browser_path) + os.pathsep + env["PATH"]
LOG.info("Starting mitmproxy playback using env path: %s" %
env["PATH"])
LOG.info("Starting mitmproxy playback using command: %s" %
" ".join(command))
# to turn off mitmproxy log output, use these params for Popen:
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
self.mitmproxy_proc = ProcessHandler(
command,
logfile=os.path.join(self.upload_dir, "mitmproxy.log"),
env=env,
processStderrLine=LOG.error,
storeOutput=False,
)
self.mitmproxy_proc.run()
end_time = time.time() + MITMDUMP_COMMAND_TIMEOUT
ready = False
while time.time() < end_time:
ready = self.check_proxy(host=self.host, port=self.port)
if ready:
LOG.info(
"Mitmproxy playback successfully started on %s:%d as pid %d"
% (self.host, self.port, self.mitmproxy_proc.pid))
return
time.sleep(0.25)
# cannot continue as we won't be able to playback the pages
LOG.error("Aborting: Mitmproxy process did not startup")
self.stop_mitmproxy_playback()
sys.exit(1) # XXX why do we need to do that? a raise is not enough?
def install_mitmproxy_cert(self, mitmproxy_proc, browser_path):
"""Install the CA certificate generated by mitmproxy, into geckoview android
If running locally:
1. Will use the `certutil` tool from the local Firefox desktop build
If running in production:
1. Get the tooltools manifest file for downloading hostutils (contains certutil)
2. Get the `certutil` tool by downloading hostutils using the tooltool manifest
Then, both locally and in production:
1. Create an NSS certificate database in the geckoview browser profile dir, only
if it doesn't already exist. Use this certutil command:
`certutil -N -d sql:<path to profile> --empty-password`
2. Import the mitmproxy certificate into the database, i.e.:
`certutil -A -d sql:<path to profile> -n "some nickname" -t TC,, -a -i <path to CA.pem>`
"""
self.CERTUTIL_SLEEP = 10
if self.config['run_local']:
# when running locally, it is found in the Firefox desktop build (..obj../dist/bin)
self.certutil = os.path.join(self.config['obj_path'], 'dist',
'bin')
os.environ['LD_LIBRARY_PATH'] = self.certutil
else:
# must download certutil inside hostutils via tooltool; use this manifest:
# mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest
# after it will be found here inside the worker/bitbar container:
# /builds/worker/workspace/build/hostutils/host-utils-66.0a1.en-US.linux-x86_64
LOG.info("downloading certutil binary (hostutils)")
# get path to the hostutils tooltool manifest; was set earlier in
# mozharness/configs/raptor/android_hw_config.py, to the path i.e.
# mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest
# the bitbar container is always linux64
if os.environ.get('GECKO_HEAD_REPOSITORY', None) is None:
LOG.critical('Abort: unable to get GECKO_HEAD_REPOSITORY')
raise
if os.environ.get('GECKO_HEAD_REV', None) is None:
LOG.critical('Abort: unable to get GECKO_HEAD_REV')
raise
if os.environ.get('HOSTUTILS_MANIFEST_PATH', None) is not None:
manifest_url = os.path.join(
os.environ['GECKO_HEAD_REPOSITORY'], "raw-file",
os.environ['GECKO_HEAD_REV'],
os.environ['HOSTUTILS_MANIFEST_PATH'])
else:
LOG.critical("Abort: unable to get HOSTUTILS_MANIFEST_PATH!")
raise
# first need to download the hostutils tooltool manifest file itself
_dest = os.path.join(self.mozproxy_dir, 'hostutils.manifest')
have_manifest = download_file_from_url(manifest_url, _dest)
if not have_manifest:
LOG.critical(
'failed to download the hostutils tooltool manifest')
raise
# now use the manifest to download hostutils so we can get certutil
tooltool_download(_dest, self.config['run_local'],
self.mozproxy_dir)
# the production bitbar container host is always linux
self.certutil = os.path.join(
self.mozproxy_dir, 'host-utils-67.0a1.en-US.linux-x86_64')
# must add hostutils/certutil to the path
os.environ['LD_LIBRARY_PATH'] = self.certutil
bin_suffix = mozinfo.info.get('bin_suffix', '')
self.certutil = os.path.join(self.certutil, "certutil" + bin_suffix)
if os.path.isfile(self.certutil):
LOG.info("certutil is found at: %s" % self.certutil)
else:
LOG.critical("unable to find certutil at %s" % self.certutil)
raise
# DEFAULT_CERT_PATH has local path and name of mitmproxy cert i.e.
# /home/cltbld/.mitmproxy/mitmproxy-ca-cert.cer
self.local_cert_path = DEFAULT_CERT_PATH
# check if the nss ca cert db already exists in the device profile
LOG.info(
"checking if the nss cert db already exists in the android browser profile"
)
param1 = "sql:%s/" % self.config["local_profile_dir"]
command = [self.certutil, "-d", param1, "-L"]
try:
subprocess.check_output(command, env=os.environ.copy())
LOG.info("the nss cert db already exists")
cert_db_exists = True
except subprocess.CalledProcessError:
# this means the nss cert db doesn't exist yet
LOG.info("nss cert db doesn't exist yet")
cert_db_exists = False
# try a forced pause between certutil cmds; possibly reduce later
time.sleep(self.CERTUTIL_SLEEP)
if not cert_db_exists:
# create cert db if it doesn't already exist; it may exist already
# if a previous pageload test ran in the same test suite
param1 = "sql:%s/" % self.config["local_profile_dir"]
command = [
self.certutil, "-N", "-v", "-d", param1, "--empty-password"
]
LOG.info("creating nss cert database using command: %s" %
" ".join(command))
cmd_proc = subprocess.Popen(command, env=os.environ.copy())
time.sleep(self.CERTUTIL_SLEEP)
cmd_terminated = cmd_proc.poll()
if cmd_terminated is None: # None value indicates process hasn't terminated
LOG.critical("nss cert db creation command failed to complete")
raise
# import mitmproxy cert into the db
command = [
self.certutil,
"-A",
"-d",
param1,
"-n",
"mitmproxy-cert",
"-t",
"TC,,",
"-a",
"-i",
self.local_cert_path,
]
LOG.info("importing mitmproxy cert into db using command: %s" %
" ".join(command))
cmd_proc = subprocess.Popen(command, env=os.environ.copy())
time.sleep(self.CERTUTIL_SLEEP)
cmd_terminated = cmd_proc.poll()
if cmd_terminated is None: # None value indicates process hasn't terminated
LOG.critical(
"command to import mitmproxy cert into cert db failed to complete"
)
# cannot continue if failed to add CA cert to Firefox, need to check
if not self.is_mitmproxy_cert_installed():
LOG.error(
"Aborting: failed to install mitmproxy CA cert into Firefox")
self.stop_mitmproxy_playback()
sys.exit()
def start_mitmproxy_playback(
self,
mitmdump_path,
mitmproxy_recording_path,
mitmproxy_recordings_list,
browser_path,
):
"""Startup mitmproxy and replay the specified flow file"""
LOG.info("mitmdump path: %s" % mitmdump_path)
LOG.info("recording path: %s" % mitmproxy_recording_path)
LOG.info("recordings list: %s" % mitmproxy_recordings_list)
LOG.info("browser path: %s" % browser_path)
mitmproxy_recordings = []
# recording names can be provided in comma-separated list; build py list including path
for recording in mitmproxy_recordings_list:
if not os.path.isfile(
os.path.join(mitmproxy_recording_path, recording)):
LOG.critical("Recording file {} cannot be found!".format(
os.path.join(mitmproxy_recording_path, recording)))
raise Exception("Recording file {} cannot be found!".format(
os.path.join(mitmproxy_recording_path, recording)))
mitmproxy_recordings.append(
os.path.join(mitmproxy_recording_path, recording))
# mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
env = os.environ.copy()
env["PATH"] = os.path.dirname(browser_path) + ";" + env["PATH"]
command = [mitmdump_path, "-k", "-q"]
if "custom_script" in self.config:
# cmd line to start mitmproxy playback using custom playback script is as follows:
# <path>/mitmdump -s "<path>/alternate-server-replay.py
# <path>recording-1.mp <path>recording-2.mp..."
custom_script = self.config["custom_script"] + " " + " ".join(
mitmproxy_recordings)
# this part is platform-specific
if mozinfo.os == "win":
custom_script = '""' + custom_script.replace("\\",
"\\\\\\") + '""'
sys.path.insert(1, mitmdump_path)
command.extend(["-s", custom_script])
LOG.info("Starting mitmproxy playback using env path: %s" %
env["PATH"])
LOG.info("Starting mitmproxy playback using command: %s" %
" ".join(command))
# to turn off mitmproxy log output, use these params for Popen:
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
mitmproxy_proc = ProcessHandler(command, env=env)
mitmproxy_proc.run()
# XXX replace the code below with a loop with a connection attempt
# Bug 1532557
time.sleep(MITMDUMP_SLEEP)
data = mitmproxy_proc.poll()
if data is None: # None value indicates process hasn't terminated
LOG.info("Mitmproxy playback successfully started as pid %d" %
mitmproxy_proc.pid)
return mitmproxy_proc
# cannot continue as we won't be able to playback the pages
LOG.error(
"Aborting: mitmproxy playback process failed to start, poll returned: %s"
% data)
# XXX here we might end up with a ghost mitmproxy
sys.exit()
