def download(self): """Download and unpack mitmproxy binary and pageset using tooltool""" if not os.path.exists(self.mozproxy_dir): os.makedirs(self.mozproxy_dir) LOG.info("downloading mitmproxy binary") _manifest = os.path.join(here, self.config["playback_binary_manifest"]) transformed_manifest = transform_platform(_manifest, self.config["platform"]) tooltool_download(transformed_manifest, self.config["run_local"], self.mozproxy_dir) if "playback_pageset_manifest" in self.config: # we use one pageset for all platforms LOG.info("downloading mitmproxy pageset") _manifest = self.config["playback_pageset_manifest"] transformed_manifest = transform_platform(_manifest, self.config["platform"]) tooltool_download(transformed_manifest, self.config["run_local"], self.mozproxy_dir) if "playback_artifacts" in self.config: artifacts = self.config["playback_artifacts"].split(",") for artifact in artifacts: artifact = artifact.strip() if not artifact: continue artifact_name = artifact.split("/")[-1] dest = os.path.join(self.mozproxy_dir, artifact_name) download_file_from_url(artifact, dest, extract=True)
def import_certificate_in_cert_db(self, cert_db_location, local_cert_path): # import mitmproxy cert into the db command = [ self.certutil, "-A", "-d", cert_db_location, "-n", "mitmproxy-cert", "-t", "TC,,", "-a", "-i", local_cert_path, ] LOG.info("importing mitmproxy cert into db using command: %s" % " ".join(command)) cmd_proc = subprocess.Popen(command, env=os.environ.copy()) time.sleep(self.certutil_sleep_seconds) cmd_terminated = cmd_proc.poll() if cmd_terminated is None: # None value indicates process hasn't terminated LOG.critical( "command to import mitmproxy cert into cert db failed to complete" )
def __init__(self, config): self.config = config self.mitmproxy_proc = None self.mitmdump_path = None self.browser_path = config.get("binary") self.policies_dir = None # mozproxy_dir is where we will download all mitmproxy required files # when running locally it comes from obj_path via mozharness/mach if self.config.get("obj_path") is not None: self.mozproxy_dir = self.config.get("obj_path") else: # in production it is ../tasks/task_N/build/, in production that dir # is not available as an envvar, however MOZ_UPLOAD_DIR is set as # ../tasks/task_N/build/blobber_upload_dir so take that and go up 1 level self.mozproxy_dir = os.path.dirname( os.path.dirname(os.environ["MOZ_UPLOAD_DIR"])) self.mozproxy_dir = os.path.join(self.mozproxy_dir, "testing", "mozproxy") self.upload_dir = os.environ.get("MOZ_UPLOAD_DIR", self.mozproxy_dir) LOG.info( "mozproxy_dir used for mitmproxy downloads and exe files: %s" % self.mozproxy_dir) # setting up the MOZPROXY_DIR env variable so custom scripts know # where to get the data os.environ["MOZPROXY_DIR"] = self.mozproxy_dir
def turn_off_browser_proxy(self): """Turn off the browser proxy that was used for mitmproxy playback. In Firefox we need to change the autoconfig files to revert the proxy; for Chromium the proxy was setup on the cmd line, so nothing is required here.""" if self.config["app"] == "firefox" and self.policies_dir is not None: LOG.info("Turning off the browser proxy") self.write_policies_json(self.policies_dir, policies_content=POLICIES_CONTENT_OFF)
def create_cert_db(self, cert_db_location): # create cert db if it doesn't already exist; it may exist already # if a previous pageload test ran in the same test suite args = ["-d", cert_db_location, "-N", "--empty-password"] LOG.info("creating nss cert database") self.certutil(args) if not self.cert_db_exists(cert_db_location): raise Exception("nss cert db creation command failed. Cert db not created.")
def __init__(self, config): self.config = config self.host = ("127.0.0.1" if "localhost" in self.config["host"] else self.config["host"]) self.port = None self.mitmproxy_proc = None self.mitmdump_path = None self.browser_path = "" if config.get("binary", None): self.browser_path = os.path.normpath(config.get("binary")) self.policies_dir = None self.ignore_mitmdump_exit_failure = config.get( "ignore_mitmdump_exit_failure", False) self.recording_paths = None if self.config.get("playback_version") is None: LOG.info("mitmproxy was not provided with a 'playback_version' " "Using default playback version: 4.0.4") self.config["playback_version"] = "4.0.4" if self.config.get("playback_binary_manifest") is None: LOG.info( "mitmproxy was not provided with a 'playback_binary_manifest' " "Using default playback_binary_manifest") self.config["playback_binary_manifest"] = ( "mitmproxy-rel-bin-%s-{platform}.manifest" % self.config["playback_version"]) # mozproxy_dir is where we will download all mitmproxy required files # when running locally it comes from obj_path via mozharness/mach if self.config.get("obj_path") is not None: self.mozproxy_dir = self.config.get("obj_path") else: # in production it is ../tasks/task_N/build/, in production that dir # is not available as an envvar, however MOZ_UPLOAD_DIR is set as # ../tasks/task_N/build/blobber_upload_dir so take that and go up 1 level self.mozproxy_dir = os.path.dirname( os.path.dirname(os.environ["MOZ_UPLOAD_DIR"])) self.mozproxy_dir = os.path.join(self.mozproxy_dir, "testing", "mozproxy") self.upload_dir = os.environ.get("MOZ_UPLOAD_DIR", self.mozproxy_dir) LOG.info( "mozproxy_dir used for mitmproxy downloads and exe files: %s" % self.mozproxy_dir) # setting up the MOZPROXY_DIR env variable so custom scripts know # where to get the data os.environ["MOZPROXY_DIR"] = self.mozproxy_dir LOG.info("Playback tool: %s" % self.config["playback_tool"]) LOG.info("Playback tool version: %s" % self.config["playback_version"])
def download_manifest_file(self, manifest_path): # Manifest File # we use one pageset for all platforms LOG.info("downloading mitmproxy pageset") tooltool_download(manifest_path, self.config["run_local"], self.mozproxy_dir) with open(manifest_path) as manifest_file: manifest = json.load(manifest_file) for file in manifest: zip_path = os.path.join(self.mozproxy_dir, file["filename"]) LOG.info("Adding %s to recording list" % zip_path) self.recordings.append(RecordingFile(zip_path))
def is_mitmproxy_cert_installed(self, cert_db_location): """Verify mitmxproy CA cert was added to Firefox on android""" LOG.info( "verifying that the mitmproxy ca cert is installed on android") # list the certifcates that are in the nss cert db (inside the browser profile dir) LOG.info( "getting the list of certs in the nss cert db in the android browser profile" ) command = [self.certutil, "-d", cert_db_location, "-L"] try: cmd_output = subprocess.check_output(command, env=os.environ.copy()) except subprocess.CalledProcessError: # cmd itself failed LOG.error(cmd_output) raise Exception("certutil command failed") # check output from the certutil command, see if 'mitmproxy-cert' is listed time.sleep(self.certutil_sleep_seconds) LOG.info(cmd_output) if "mitmproxy-cert" in cmd_output: LOG.info( "verfied the mitmproxy-cert is installed in the nss cert db on android" ) return True return False
def is_mitmproxy_cert_installed(self): """Verify mitmxproy CA cert was added to Firefox on android""" LOG.info( "verifying that the mitmproxy ca cert is installed on android") # list the certifcates that are in the nss cert db (inside the browser profile dir) LOG.info( "getting the list of certs in the nss cert db in the android browser profile" ) param1 = "sql:%s/" % self.config["local_profile_dir"] command = [self.certutil, "-d", param1, "-L"] try: cmd_output = subprocess.check_output(command, env=os.environ.copy()) except subprocess.CalledProcessError: # cmd itself failed LOG.critical("certutil command failed") raise # check output from the certutil command, see if 'mitmproxy-cert' is listed time.sleep(self.CERTUTIL_SLEEP) LOG.info(cmd_output) if "mitmproxy-cert" in cmd_output: LOG.info( "verfied the mitmproxy-cert is installed in the nss cert db on android" ) return True return False
def import_certificate_in_cert_db(self, cert_db_location, local_cert_path): # import mitmproxy cert into the db args = [ "-A", "-d", cert_db_location, "-n", "mitmproxy-cert", "-t", "TC,,", "-a", "-i", local_cert_path, ] LOG.info("importing mitmproxy cert into db using command") self.certutil(args)
def create_cert_db(self, cert_db_location): # create cert db if it doesn't already exist; it may exist already # if a previous pageload test ran in the same test suite command = [ self.certutil, "-d", cert_db_location, "-N", "--empty-password" ] LOG.info("creating nss cert database using command: %s" % " ".join(command)) cmd_proc = subprocess.Popen(command, env=os.environ.copy()) time.sleep(self.certutil_sleep_seconds) cmd_terminated = cmd_proc.poll() if cmd_terminated is None: # None value indicates process hasn't terminated raise Exception("nss cert db creation command failed to complete") if not self.cert_db_exists(cert_db_location): raise Exception( "nss cert db creation command failed. Cert db not created.")
def start_mitmproxy_playback( self, mitmdump_path, browser_path, ): """Startup mitmproxy and replay the specified flow file""" LOG.info("mitmdump path: %s" % mitmdump_path) LOG.info("browser path: %s" % browser_path) # mitmproxy needs some DLL's that are a part of Firefox itself, so add to path env = os.environ.copy() env["PATH"] = os.path.dirname(browser_path) + ";" + env["PATH"] command = [mitmdump_path, "-k"] if "playback_tool_args" in self.config: command.extend(self.config["playback_tool_args"]) LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"]) LOG.info("Starting mitmproxy playback using command: %s" % " ".join(command)) # to turn off mitmproxy log output, use these params for Popen: # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) mitmproxy_proc = ProcessHandler(command, logfile=os.path.join( self.upload_dir, "mitmproxy.log"), env=env) mitmproxy_proc.run() # XXX replace the code below with a loop with a connection attempt # Bug 1532557 time.sleep(MITMDUMP_SLEEP) data = mitmproxy_proc.poll() if data is None: # None value indicates process hasn't terminated LOG.info("Mitmproxy playback successfully started as pid %d" % mitmproxy_proc.pid) return mitmproxy_proc # cannot continue as we won't be able to playback the pages LOG.error( "Aborting: mitmproxy playback process failed to start, poll returned: %s" % data) # XXX here we might end up with a ghost mitmproxy sys.exit()
def download(self): """Download and unpack mitmproxy binary and pageset using tooltool""" if not os.path.exists(self.mozproxy_dir): os.makedirs(self.mozproxy_dir) _manifest = os.path.join(here, self.config["playback_binary_manifest"]) transformed_manifest = transform_platform(_manifest, self.config["platform"]) # generate the mitmdump_path self.mitmdump_path = os.path.normpath( os.path.join( self.mozproxy_dir, "mitmdump-%s" % self.config["playback_version"], "mitmdump", )) # Check if mitmproxy bin exists if os.path.exists(self.mitmdump_path): LOG.info("mitmproxy binary already exists. Skipping download") else: # Download and unpack mitmproxy binary download_path = os.path.dirname(self.mitmdump_path) LOG.info("create mitmproxy %s dir" % self.config["playback_version"]) if not os.path.exists(download_path): os.makedirs(download_path) LOG.info("downloading mitmproxy binary") tooltool_download(transformed_manifest, self.config["run_local"], download_path) if "playback_pageset_manifest" in self.config: # we use one pageset for all platforms LOG.info("downloading mitmproxy pageset") _manifest = self.config["playback_pageset_manifest"] transformed_manifest = transform_platform(_manifest, self.config["platform"]) tooltool_download(transformed_manifest, self.config["run_local"], self.mozproxy_dir) if "playback_artifacts" in self.config: artifacts = self.config["playback_artifacts"].split(",") for artifact in artifacts: artifact = artifact.strip() if not artifact: continue artifact_name = artifact.split("/")[-1] if artifact_name.endswith(".manifest"): tooltool_download(artifact, self.config["run_local"], self.mozproxy_dir) else: dest = os.path.join(self.mozproxy_dir, artifact_name) download_file_from_url(artifact, dest, extract=True)
def download(self): """Download and unpack mitmproxy binary and pageset using tooltool""" if not os.path.exists(self.mozproxy_dir): os.makedirs(self.mozproxy_dir) LOG.info("downloading mitmproxy binary") _manifest = os.path.join(here, self.config["playback_binary_manifest"]) transformed_manifest = transform_platform(_manifest, self.config["platform"]) tooltool_download(transformed_manifest, self.config["run_local"], self.mozproxy_dir) # we use one pageset for all platforms LOG.info("downloading mitmproxy pageset") _manifest = self.config["playback_pageset_manifest"] transformed_manifest = transform_platform(_manifest, self.config["platform"]) tooltool_download(transformed_manifest, self.config["run_local"], self.mozproxy_dir)
def download_playback_files(self): # Detect type of file from playback_files and download accordingly if "playback_files" not in self.config: LOG.error( "playback_files value was not provided. Proxy service wont' start " ) raise Exception("Please provide a playback_files list.") if not isinstance(self.config["playback_files"], list): LOG.error("playback_files should be a list") raise Exception("playback_files should be a list") for playback_file in self.config["playback_files"]: if playback_file.startswith( "https://") and "mozilla.com" in playback_file: # URL provided dest = os.path.join(self.mozproxy_dir, os.path.basename(playback_file)) download_file_from_url(playback_file, self.mozproxy_dir, extract=False) # Add Downloaded file to recordings list LOG.info("Adding %s to recording list" % dest) self.recordings.append(RecordingFile(dest)) continue if not os.path.exists(playback_file): LOG.error( "Zip or manifest file path (%s) does not exist. Please provide a valid path!" % playback_file) raise Exception("Zip or manifest file path does not exist") if os.path.splitext(playback_file)[1] == ".zip": # zip file path provided LOG.info("Adding %s to recording list" % playback_file) self.recordings.append(RecordingFile(playback_file)) elif os.path.splitext(playback_file)[1] == ".manifest": # manifest file path provided self.download_manifest_file(playback_file)
def stop_mitmproxy_playback(self): """Stop the mitproxy server playback""" if self.mitmproxy_proc is None or self.mitmproxy_proc.poll( ) is not None: return LOG.info("Stopping mitmproxy playback, killing process %d" % self.mitmproxy_proc.pid) exit_code = self.mitmproxy_proc.kill() if exit_code != 0: # I *think* we can still continue, as process will be automatically # killed anyway when mozharness is done (?) if not, we won't be able # to startup mitmxproy next time if it is already running if exit_code is None: LOG.error("Failed to kill the mitmproxy playback process") else: LOG.error("Mitmproxy exited with error code %d" % exit_code) else: LOG.info("Successfully killed the mitmproxy playback process") self.mitmproxy_proc = None
def start_mitmproxy_playback(self, mitmdump_path, browser_path): """Startup mitmproxy and replay the specified flow file""" if self.mitmproxy_proc is not None: raise Exception("Proxy already started.") LOG.info("mitmdump path: %s" % mitmdump_path) LOG.info("browser path: %s" % browser_path) # mitmproxy needs some DLL's that are a part of Firefox itself, so add to path env = os.environ.copy() env["PATH"] = os.path.dirname(browser_path) + os.pathsep + env["PATH"] command = [mitmdump_path] if "playback_tool_args" in self.config: command.extend(self.config["playback_tool_args"]) LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"]) LOG.info("Starting mitmproxy playback using command: %s" % " ".join(command)) # to turn off mitmproxy log output, use these params for Popen: # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) self.mitmproxy_proc = ProcessHandler(command, logfile=os.path.join( self.upload_dir, "mitmproxy.log"), env=env) self.mitmproxy_proc.run() end_time = time.time() + MITMDUMP_COMMAND_TIMEOUT ready = False while time.time() < end_time: ready = self.check_proxy() if ready: LOG.info("Mitmproxy playback successfully started as pid %d" % self.mitmproxy_proc.pid) return time.sleep(0.25) # cannot continue as we won't be able to playback the pages LOG.error("Aborting: Mitmproxy process did not startup") self.stop_mitmproxy_playback() sys.exit() # XXX why do we need to do that? a raise is not enough?
def confidence(self): file_name = "mitm_netlocs_%s.json" % os.path.splitext( os.path.basename(self.recording_paths[0]))[0] path = os.path.normpath(os.path.join(self.upload_dir, file_name)) if os.path.exists(path): try: LOG.info("Reading confidence values from: %s" % path) with open(path, "r") as f: data = json.load(f) return { "confidence": { "values": data["confidence"], "subtest-prefix-type": False, "unit": "%", "lowerIsBetter": False }, "not-replayed": { "values": data["not-replayed"], "subtest-prefix-type": False, "unit": "a.u." }, "replayed": { "values": data["replayed"], "subtest-prefix-type": False, "unit": "a.u.", "lowerIsBetter": False } } except Exception: LOG.info("Can't read netlocs file!", exc_info=True) return None else: LOG.info("Netlocs file is not available! Cant find %s" % path) return None
def __init__(self, config): self.config = config self.mitmproxy_proc = None self.mitmdump_path = None self.browser_path = config.get("binary") self.policies_dir = None self.ignore_mitmdump_exit_failure = config.get( "ignore_mitmdump_exit_failure", False ) if self.config.get("playback_version") is None: LOG.info("mitmproxy was not provided with a 'playback_version' " "getting 'playback_version' from 'playback_binary_manifest'") if "4.0.4" in self.config["playback_binary_manifest"]: self.config["playback_version"] = "4.0.4" else: self.config["playback_version"] = "2.0.2" # mozproxy_dir is where we will download all mitmproxy required files # when running locally it comes from obj_path via mozharness/mach if self.config.get("obj_path") is not None: self.mozproxy_dir = self.config.get("obj_path") else: # in production it is ../tasks/task_N/build/, in production that dir # is not available as an envvar, however MOZ_UPLOAD_DIR is set as # ../tasks/task_N/build/blobber_upload_dir so take that and go up 1 level self.mozproxy_dir = os.path.dirname( os.path.dirname(os.environ["MOZ_UPLOAD_DIR"]) ) self.mozproxy_dir = os.path.join(self.mozproxy_dir, "testing", "mozproxy") self.upload_dir = os.environ.get("MOZ_UPLOAD_DIR", self.mozproxy_dir) LOG.info( "mozproxy_dir used for mitmproxy downloads and exe files: %s" % self.mozproxy_dir ) # setting up the MOZPROXY_DIR env variable so custom scripts know # where to get the data os.environ["MOZPROXY_DIR"] = self.mozproxy_dir
def download_mitm_bin(self): # Download and setup mitm binaries manifest = os.path.join( here, "manifests", "mitmproxy-rel-bin-%s-{platform}.manifest" % self.config["playback_version"], ) transformed_manifest = transform_platform(manifest, self.config["platform"]) # generate the mitmdump_path self.mitmdump_path = os.path.normpath( os.path.join( self.mozproxy_dir, "mitmdump-%s" % self.config["playback_version"], "mitmdump", )) # Check if mitmproxy bin exists if os.path.exists(self.mitmdump_path): LOG.info("mitmproxy binary already exists. Skipping download") else: # Download and unpack mitmproxy binary download_path = os.path.dirname(self.mitmdump_path) LOG.info("create mitmproxy %s dir" % self.config["playback_version"]) if not os.path.exists(download_path): os.makedirs(download_path) LOG.info("downloading mitmproxy binary") tooltool_download(transformed_manifest, self.config["run_local"], download_path)
def __init__(self, config): self.config = config self.mitmproxy_proc = None self.mitmdump_path = None self.recordings = config.get("playback_recordings") self.browser_path = config.get("binary") self.policies_dir = None # mozproxy_dir is where we will download all mitmproxy required files # when running locally it comes from obj_path via mozharness/mach if self.config.get("obj_path") is not None: self.mozproxy_dir = self.config.get("obj_path") else: # in production it is ../tasks/task_N/build/, in production that dir # is not available as an envvar, however MOZ_UPLOAD_DIR is set as # ../tasks/task_N/build/blobber_upload_dir so take that and go up 1 level self.mozproxy_dir = os.path.dirname( os.path.dirname(os.environ["MOZ_UPLOAD_DIR"])) self.mozproxy_dir = os.path.join(self.mozproxy_dir, "testing", "mozproxy") self.recordings_path = self.mozproxy_dir LOG.info( "mozproxy_dir used for mitmproxy downloads and exe files: %s" % self.mozproxy_dir) # go ahead and download and setup mitmproxy self.download() # mitmproxy must be started before setup, so that the CA cert is available self.start() # In case the setup fails, we want to stop the process before raising. try: self.setup() except Exception: self.stop() raise
def confidence(self): """Extract confidence metrics from the netlocs file and convert them to perftest results """ if len(self.recordings) == 0: LOG.warning( "Proxy service did not load a recording file. " "Confidence metrics will nt be generated" ) return file_name = ( "mitm_netlocs_%s.json" % os.path.splitext(os.path.basename(self.recordings[0].recording_path))[0] ) path = os.path.normpath(os.path.join(self.upload_dir, file_name)) if os.path.exists(path): try: LOG.info("Reading confidence values from: %s" % path) with open(path, "r") as f: data = json.load(f) return { "replay-confidence": { "values": data["replay-confidence"], "subtest-prefix-type": False, "unit": "%", "shouldAlert": False, "lowerIsBetter": False, }, "recording-proportion-used": { "values": data["recording-proportion-used"], "subtest-prefix-type": False, "unit": "%", "shouldAlert": False, "lowerIsBetter": False, }, "not-replayed": { "values": data["not-replayed"], "subtest-prefix-type": False, "shouldAlert": False, "unit": "a.u.", }, "replayed": { "values": data["replayed"], "subtest-prefix-type": False, "unit": "a.u.", "shouldAlert": False, "lowerIsBetter": False, }, } except Exception: LOG.info("Can't read netlocs file!", exc_info=True) return None else: LOG.info("Netlocs file is not available! Cant find %s" % path) return None
def is_mitmproxy_cert_installed(self): """Verify mitmxproy CA cert was added to Firefox""" try: # read autoconfig file, confirm mitmproxy cert is in there contents = self.read_policies_json(self.policies_dir) LOG.info("Firefox policies file contents:") LOG.info(contents) if ( POLICIES_CONTENT_ON % {"cert": self.cert_path, "host": self.host, "port": self.port} ) in contents: LOG.info("Verified mitmproxy CA certificate is installed in Firefox") else: return False except Exception as e: LOG.info("failed to read Firefox policies file, exeption: %s" % e) return False return True
def stop_mitmproxy_playback(self): """Stop the mitproxy server playback""" if self.mitmproxy_proc is None or self.mitmproxy_proc.poll() is not None: return LOG.info( "Stopping mitmproxy playback, killing process %d" % self.mitmproxy_proc.pid ) # On Windows, mozprocess brutally kills mitmproxy with TerminateJobObject # The process has no chance to gracefully shutdown. # Here, we send the process a break event to give it a chance to wrapup. # See the signal handler in the alternate-server-replay-4.0.4.py script if mozinfo.os == "win": LOG.info("Sending CTRL_BREAK_EVENT to mitmproxy") os.kill(self.mitmproxy_proc.pid, signal.CTRL_BREAK_EVENT) time.sleep(2) exit_code = self.mitmproxy_proc.kill() self.mitmproxy_proc = None if exit_code != 0: if exit_code is None: LOG.error("Failed to kill the mitmproxy playback process") return if mozinfo.os == "win": from mozprocess.winprocess import ERROR_CONTROL_C_EXIT # noqa if exit_code == ERROR_CONTROL_C_EXIT: LOG.info( "Successfully killed the mitmproxy playback process" " with exit code %d" % exit_code ) return log_func = LOG.error if self.ignore_mitmdump_exit_failure: log_func = LOG.info log_func("Mitmproxy exited with error code %d" % exit_code) else: LOG.info("Successfully killed the mitmproxy playback process")
def cert_db_exists(self, cert_db_location): # check if the nss ca cert db already exists in the device profile LOG.info( "checking if the nss cert db already exists in the android browser profile" ) args = ["-d", cert_db_location, "-L"] cert_db_exists = self.certutil(args, raise_exception=False) if cert_db_exists: LOG.info("the nss cert db exists") return True else: LOG.info("nss cert db doesn't exist yet.") return False
def stop_mitmproxy_playback(self): """Stop the mitproxy server playback""" mitmproxy_proc = self.mitmproxy_proc LOG.info("Stopping mitmproxy playback, killing process %d" % mitmproxy_proc.pid) mitmproxy_proc.kill() time.sleep(MITMDUMP_SLEEP) status = mitmproxy_proc.poll() if status is None: # None value indicates process hasn't terminated # I *think* we can still continue, as process will be automatically # killed anyway when mozharness is done (?) if not, we won't be able # to startup mitmxproy next time if it is already running LOG.error("Failed to kill the mitmproxy playback process") LOG.info(str(status)) else: LOG.info("Successfully killed the mitmproxy playback process")
def install_mitmproxy_cert(self, browser_path): """Install the CA certificate generated by mitmproxy, into Firefox 1. Create a dir called 'distribution' in the same directory as the Firefox executable 2. Create the policies.json file inside that folder; which points to the certificate location, and turns on the the browser proxy settings """ LOG.info("Installing mitmproxy CA certficate into Firefox") # browser_path is the exe, we want the folder self.policies_dir = os.path.dirname(browser_path) # on macosx we need to remove the last folders 'MacOS' # and the policies json needs to go in ../Content/Resources/ if "mac" in self.config["platform"]: self.policies_dir = os.path.join(self.policies_dir[:-6], "Resources") # for all platforms the policies json goes in a 'distribution' dir self.policies_dir = os.path.join(self.policies_dir, "distribution") self.cert_path = DEFAULT_CERT_PATH # for windows only if mozinfo.os == "win": self.cert_path = self.cert_path.replace("\\", "\\\\") if not os.path.exists(self.policies_dir): LOG.info("creating folder: %s" % self.policies_dir) os.makedirs(self.policies_dir) else: LOG.info("folder already exists: %s" % self.policies_dir) self.write_policies_json( self.policies_dir, policies_content=POLICIES_CONTENT_ON % { "cert": self.cert_path, "host": self.host, "port": self.port }, ) # cannot continue if failed to add CA cert to Firefox, need to check if not self.is_mitmproxy_cert_installed(): LOG.error( "Aborting: failed to install mitmproxy CA cert into Firefox desktop" ) self.stop_mitmproxy_playback() sys.exit()
def is_mitmproxy_cert_installed(self, cert_db_location): """Verify mitmxproy CA cert was added to Firefox on android""" LOG.info("verifying that the mitmproxy ca cert is installed on android") # list the certifcates that are in the nss cert db (inside the browser profile dir) LOG.info( "getting the list of certs in the nss cert db in the android browser profile" ) args = ["-d", cert_db_location, "-L"] cmd_output = self.certutil(args) if "mitmproxy-cert" in cmd_output: LOG.info( "verfied the mitmproxy-cert is installed in the nss cert db on android" ) return True return False
def cert_db_exists(self, cert_db_location): # check if the nss ca cert db already exists in the device profile LOG.info( "checking if the nss cert db already exists in the android browser profile" ) command = [self.certutil, "-d", cert_db_location, "-L"] try: subprocess.check_call(command, env=os.environ.copy()) LOG.info("the nss cert db exists") cert_db_exists = True except subprocess.CalledProcessError: # this means the nss cert db doesn't exist yet LOG.info( "nss cert db doesn't exist yet. Note: certutil error is expected!!!" ) cert_db_exists = False # try a forced pause between certutil cmds; possibly reduce later time.sleep(self.certutil_sleep_seconds) return cert_db_exists
def certutil(self, args, raise_exception=True): cmd = [self.certutil_path] + list(args) LOG.info("Certutil: Running command: %s" % " ".join(cmd)) try: cmd_proc = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE, env=os.environ.copy()) cmd_output, errs = cmd_proc.communicate() except subprocess.SubprocessError: LOG.critical("could not run the certutil command") raise if cmd_proc.returncode == 0: # Debug purpose only remove if stable LOG.info("Certutil returncode: %s" % cmd_proc.returncode) LOG.info("Certutil output: %s" % cmd_output) return cmd_output else: if raise_exception: LOG.critical("Certutil command failed!!") LOG.info("Certutil returncode: %s" % cmd_proc.returncode) LOG.info("Certutil output: %s" % cmd_output) LOG.info("Certutil error: %s" % errs) raise Exception("Certutil command failed!!") else: return False