def certutil(self, args, raise_exception=True): cmd = [self.certutil_path] + list(args) LOG.info("Certutil: Running command: %s" % " ".join(cmd)) try: cmd_proc = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE, env=os.environ.copy()) cmd_output, errs = cmd_proc.communicate() except subprocess.SubprocessError: LOG.critical("could not run the certutil command") raise if cmd_proc.returncode == 0: # Debug purpose only remove if stable LOG.info("Certutil returncode: %s" % cmd_proc.returncode) LOG.info("Certutil output: %s" % cmd_output) return cmd_output else: if raise_exception: LOG.critical("Certutil command failed!!") LOG.info("Certutil returncode: %s" % cmd_proc.returncode) LOG.info("Certutil output: %s" % cmd_output) LOG.info("Certutil error: %s" % errs) raise Exception("Certutil command failed!!") else: return False
def is_mitmproxy_cert_installed(self): """Verify mitmxproy CA cert was added to Firefox on android""" LOG.info( "verifying that the mitmproxy ca cert is installed on android") # list the certifcates that are in the nss cert db (inside the browser profile dir) LOG.info( "getting the list of certs in the nss cert db in the android browser profile" ) param1 = "sql:%s/" % self.config["local_profile_dir"] command = [self.certutil, "-d", param1, "-L"] try: cmd_output = subprocess.check_output(command, env=os.environ.copy()) except subprocess.CalledProcessError: # cmd itself failed LOG.critical("certutil command failed") raise # check output from the certutil command, see if 'mitmproxy-cert' is listed time.sleep(self.CERTUTIL_SLEEP) LOG.info(cmd_output) if "mitmproxy-cert" in cmd_output: LOG.info( "verfied the mitmproxy-cert is installed in the nss cert db on android" ) return True return False
def import_certificate_in_cert_db(self, cert_db_location, local_cert_path): # import mitmproxy cert into the db command = [ self.certutil, "-A", "-d", cert_db_location, "-n", "mitmproxy-cert", "-t", "TC,,", "-a", "-i", local_cert_path, ] LOG.info("importing mitmproxy cert into db using command: %s" % " ".join(command)) cmd_proc = subprocess.Popen(command, env=os.environ.copy()) time.sleep(self.certutil_sleep_seconds) cmd_terminated = cmd_proc.poll() if cmd_terminated is None: # None value indicates process hasn't terminated LOG.critical( "command to import mitmproxy cert into cert db failed to complete" )
def install_mitmproxy_cert(self, mitmproxy_proc, browser_path): """Install the CA certificate generated by mitmproxy, into geckoview android If running locally: 1. Will use the `certutil` tool from the local Firefox desktop build If running in production: 1. Get the tooltools manifest file for downloading hostutils (contains certutil) 2. Get the `certutil` tool by downloading hostutils using the tooltool manifest Then, both locally and in production: 1. Create an NSS certificate database in the geckoview browser profile dir, only if it doesn't already exist. Use this certutil command: `certutil -N -d sql:<path to profile> --empty-password` 2. Import the mitmproxy certificate into the database, i.e.: `certutil -A -d sql:<path to profile> -n "some nickname" -t TC,, -a -i <path to CA.pem>` """ self.CERTUTIL_SLEEP = 10 if self.config['run_local']: # when running locally, it is found in the Firefox desktop build (..obj../dist/bin) self.certutil = os.path.join(self.config['obj_path'], 'dist', 'bin') os.environ['LD_LIBRARY_PATH'] = self.certutil else: # must download certutil inside hostutils via tooltool; use this manifest: # mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest # after it will be found here inside the worker/bitbar container: # /builds/worker/workspace/build/hostutils/host-utils-66.0a1.en-US.linux-x86_64 LOG.info("downloading certutil binary (hostutils)") # get path to the hostutils tooltool manifest; was set earlier in # mozharness/configs/raptor/android_hw_config.py, to the path i.e. # mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest # the bitbar container is always linux64 if os.environ.get('GECKO_HEAD_REPOSITORY', None) is None: LOG.critical('Abort: unable to get GECKO_HEAD_REPOSITORY') raise if os.environ.get('GECKO_HEAD_REV', None) is None: LOG.critical('Abort: unable to get GECKO_HEAD_REV') raise if os.environ.get('HOSTUTILS_MANIFEST_PATH', None) is not None: manifest_url = os.path.join( os.environ['GECKO_HEAD_REPOSITORY'], "raw-file", os.environ['GECKO_HEAD_REV'], os.environ['HOSTUTILS_MANIFEST_PATH']) else: LOG.critical("Abort: unable to get HOSTUTILS_MANIFEST_PATH!") raise # first need to download the hostutils tooltool manifest file itself _dest = os.path.join(self.mozproxy_dir, 'hostutils.manifest') have_manifest = download_file_from_url(manifest_url, _dest) if not have_manifest: LOG.critical( 'failed to download the hostutils tooltool manifest') raise # now use the manifest to download hostutils so we can get certutil tooltool_download(_dest, self.config['run_local'], self.mozproxy_dir) # the production bitbar container host is always linux self.certutil = os.path.join( self.mozproxy_dir, 'host-utils-67.0a1.en-US.linux-x86_64') # must add hostutils/certutil to the path os.environ['LD_LIBRARY_PATH'] = self.certutil bin_suffix = mozinfo.info.get('bin_suffix', '') self.certutil = os.path.join(self.certutil, "certutil" + bin_suffix) if os.path.isfile(self.certutil): LOG.info("certutil is found at: %s" % self.certutil) else: LOG.critical("unable to find certutil at %s" % self.certutil) raise # DEFAULT_CERT_PATH has local path and name of mitmproxy cert i.e. # /home/cltbld/.mitmproxy/mitmproxy-ca-cert.cer self.local_cert_path = DEFAULT_CERT_PATH # check if the nss ca cert db already exists in the device profile LOG.info( "checking if the nss cert db already exists in the android browser profile" ) param1 = "sql:%s/" % self.config["local_profile_dir"] command = [self.certutil, "-d", param1, "-L"] try: subprocess.check_output(command, env=os.environ.copy()) LOG.info("the nss cert db already exists") cert_db_exists = True except subprocess.CalledProcessError: # this means the nss cert db doesn't exist yet LOG.info("nss cert db doesn't exist yet") cert_db_exists = False # try a forced pause between certutil cmds; possibly reduce later time.sleep(self.CERTUTIL_SLEEP) if not cert_db_exists: # create cert db if it doesn't already exist; it may exist already # if a previous pageload test ran in the same test suite param1 = "sql:%s/" % self.config["local_profile_dir"] command = [ self.certutil, "-N", "-v", "-d", param1, "--empty-password" ] LOG.info("creating nss cert database using command: %s" % " ".join(command)) cmd_proc = subprocess.Popen(command, env=os.environ.copy()) time.sleep(self.CERTUTIL_SLEEP) cmd_terminated = cmd_proc.poll() if cmd_terminated is None: # None value indicates process hasn't terminated LOG.critical("nss cert db creation command failed to complete") raise # import mitmproxy cert into the db command = [ self.certutil, "-A", "-d", param1, "-n", "mitmproxy-cert", "-t", "TC,,", "-a", "-i", self.local_cert_path, ] LOG.info("importing mitmproxy cert into db using command: %s" % " ".join(command)) cmd_proc = subprocess.Popen(command, env=os.environ.copy()) time.sleep(self.CERTUTIL_SLEEP) cmd_terminated = cmd_proc.poll() if cmd_terminated is None: # None value indicates process hasn't terminated LOG.critical( "command to import mitmproxy cert into cert db failed to complete" ) # cannot continue if failed to add CA cert to Firefox, need to check if not self.is_mitmproxy_cert_installed(): LOG.error( "Aborting: failed to install mitmproxy CA cert into Firefox") self.stop_mitmproxy_playback() sys.exit()
def start_mitmproxy_playback( self, mitmdump_path, mitmproxy_recording_path, mitmproxy_recordings_list, browser_path, ): """Startup mitmproxy and replay the specified flow file""" LOG.info("mitmdump path: %s" % mitmdump_path) LOG.info("recording path: %s" % mitmproxy_recording_path) LOG.info("recordings list: %s" % mitmproxy_recordings_list) LOG.info("browser path: %s" % browser_path) mitmproxy_recordings = [] # recording names can be provided in comma-separated list; build py list including path for recording in mitmproxy_recordings_list: if not os.path.isfile( os.path.join(mitmproxy_recording_path, recording)): LOG.critical("Recording file {} cannot be found!".format( os.path.join(mitmproxy_recording_path, recording))) raise Exception("Recording file {} cannot be found!".format( os.path.join(mitmproxy_recording_path, recording))) mitmproxy_recordings.append( os.path.join(mitmproxy_recording_path, recording)) # mitmproxy needs some DLL's that are a part of Firefox itself, so add to path env = os.environ.copy() env["PATH"] = os.path.dirname(browser_path) + ";" + env["PATH"] command = [mitmdump_path, "-k", "-q"] if "custom_script" in self.config: # cmd line to start mitmproxy playback using custom playback script is as follows: # <path>/mitmdump -s "<path>/alternate-server-replay.py # <path>recording-1.mp <path>recording-2.mp..." custom_script = self.config["custom_script"] + " " + " ".join( mitmproxy_recordings) # this part is platform-specific if mozinfo.os == "win": custom_script = '""' + custom_script.replace("\\", "\\\\\\") + '""' sys.path.insert(1, mitmdump_path) command.extend(["-s", custom_script]) LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"]) LOG.info("Starting mitmproxy playback using command: %s" % " ".join(command)) # to turn off mitmproxy log output, use these params for Popen: # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) mitmproxy_proc = ProcessHandler(command, env=env) mitmproxy_proc.run() # XXX replace the code below with a loop with a connection attempt # Bug 1532557 time.sleep(MITMDUMP_SLEEP) data = mitmproxy_proc.poll() if data is None: # None value indicates process hasn't terminated LOG.info("Mitmproxy playback successfully started as pid %d" % mitmproxy_proc.pid) return mitmproxy_proc # cannot continue as we won't be able to playback the pages LOG.error( "Aborting: mitmproxy playback process failed to start, poll returned: %s" % data) # XXX here we might end up with a ghost mitmproxy sys.exit()