def setup(self): global logger self.app = self.get_test_candidate(self.args.base) self.profile = self.make_profile("base_profile") tmp_zip_name = os.path.join(self.tmp_dir, "top.zip") logger.info("Fetching unfiltered top sites data from the `Tranco Top 1M` online database") get_to_file(self.top_sites_location, tmp_zip_name) try: zipped = zipfile.ZipFile(tmp_zip_name) if len(zipped.filelist) != 1 or not zipped.filelist[0].orig_filename.lower().endswith(".csv"): logger.critical("Top sites zip file has unexpected content") sys.exit(5) tmp_csv_name = zipped.extract(zipped.filelist[0], self.tmp_dir) except zipfile.BadZipfile: logger.critical("Error opening top sites zip archive") sys.exit(5) self.db = sdb.SourcesDB(self.args) is_default = self.args.source == self.db.default self.sources = sdb.Sources(self.args.source, is_default) with open(tmp_csv_name) as f: cr = csv.DictReader(f, fieldnames=["rank", "hostname"]) self.sources.rows = [row for row in cr] # A mild sanity check to see whether the downloaded data is valid. if len(self.sources) < 900000: logger.warning("Top sites is surprisingly small, just %d hosts" % len(self.sources)) if "hostname" not in self.sources.rows[0] or "rank" not in self.sources.rows[0] \ or self.sources.rows[0]["rank"] != 1: logger.warning("Top sites data looks weird. First line: `%s`" % self.sources.rows[0])
def setup(self): global logger # Argument validation logic to make sure user has specified only test build if self.args.test is None: logger.critical("Must specify test build for scan") sys.exit(5) elif self.args.base is not None: logger.debug("Ignoring base build parameter") # Download app and extract metadata self.test_app = self.get_test_candidate(self.args.test) self.test_metadata = self.collect_worker_info(self.test_app) logger.info( "Testing Firefox %s %s scan run" % (self.test_metadata["app_version"], self.test_metadata["branch"])) # Create custom profile self.test_profile = self.make_profile("test_profile") # Compile the set of hosts to test db = sdb.SourcesDB(self.args) logger.info("Reading `%s` host database" % self.args.source) self.sources = db.read(self.args.source) logger.info("%d hosts in test set" % len(self.sources))
def main(argv=None): global logger, tmp_dir, module_dir module_dir = os.path.split(__file__)[0] args = parse_args(argv) if args.debug: coloredlogs.install(level='DEBUG') logger.debug("Command arguments: %s" % args) cleanup.init() fix_terminal_encoding() tmp_dir = __create_tempdir() # If 'list' is specified as test, list available test sets, builds, and platforms if "source" in args and args.source == "list": coloredlogs.install(level='ERROR') db = sdb.SourcesDB(args) build_list, platform_list, _, _ = fd.FirefoxDownloader.list() print("Available builds: %s" % ' '.join(build_list)) print("Available platforms: %s" % ' '.join(platform_list)) print("Available test sets:") for handle in db.list(): test_set = db.read(handle) if handle == db.default: default = " (default)" else: default = "" print(" - %s [%d hosts]%s" % (handle, len(test_set), default)) return 0 # Create workdir (usually ~/.tlscanary, used for caching etc.) # Assumes that no previous code must write to it. if not os.path.exists(args.workdir): logger.debug('Creating working directory %s' % args.workdir) os.makedirs(args.workdir) # Load the specified test mode try: loader.run(args, module_dir, tmp_dir) except KeyboardInterrupt: logger.critical("\nUser interrupt. Quitting...") return 10 if len(threading.enumerate()) > 1: logger.info("Waiting for background threads to finish") while len(threading.enumerate()) > 1: logger.debug("Remaining threads: %s" % threading.enumerate()) time.sleep(2) return 0
def one_crl_sanity_check(self): global logger # Query host(s) with a known revoked cert and examine the results # These hosts must be revoked via OCSP and/or OneCRL db = sdb.SourcesDB(self.args) self.revoked_source = db.read("revoked") logger.debug("%d host(s) in revoked test set" % len(self.revoked_source)) next_chunk = self.revoked_source.iter_chunks(chunk_size=1 / 50, min_chunk_size=1000) host_set_chunk = next_chunk(as_set=True) # Note: turn off OCSP for this test, to factor out that mechanism self.custom_ocsp_pref = ["security.OCSP.enabled;0"] # First, use the test build and profile as-is # This should return errors, which means OneCRL is working test_result = self.run_test(self.test_app, url_list=host_set_chunk, profile=self.test_profile, prefs=self.custom_ocsp_pref, num_workers=1, n_per_worker=1) # Second, use the test build with a profile that is missing OneCRL entries # This should NOT return errors, which means we've turned off protection self.altered_profile = self.make_profile("altered_profile", "none") base_result = self.run_test(self.test_app, url_list=host_set_chunk, profile=self.altered_profile, prefs=self.custom_ocsp_pref, num_workers=1, n_per_worker=1) logger.debug("Length of first OneCRL check, with revocation: %d" % len(test_result)) logger.debug("Length of second OneCRL check, without revocation: %d" % len(base_result)) # If our list of revoked sites are all blocked, and we can verify # that they can be unblocked, this confirms that OneCRL is working if len(test_result) == len( self.revoked_source) and len(base_result) == 0: return True else: logger.warning( "OneCRL check failed. This is expected, so continuing") return True
def test_sources_db_write_and_override(tmpdir): """SourcesDB databases can be written and overridden""" db = sdb.SourcesDB(ArgsMock(workdir=tmpdir)) old = db.read("debug") old_default = db.default override = sdb.Sources("debug", True) row_one = {"foo": "bar", "baz": "bang", "boom": "bang"} row_two = {"foo": "bar2", "baz": "bang2", "boom": "bang2"} override.append(row_one) override.append(row_two) db.write(override) # New SourcesDB instance required to detect overrides db = sdb.SourcesDB(ArgsMock(workdir=tmpdir)) assert os.path.exists(tmpdir.join("sources", "debug.csv")), "override file is written" assert db.default == "debug", "overriding the default works" assert old_default != db.default, "overridden default actually changes" new = db.read("debug") assert len(new) == 2, "number of overridden rows is correct" assert new[0] == row_one and new[ 1] == row_two, "new rows are written as expected" assert old[0] != new[0], "overridden rows actually change"
def setup(self): global logger # Argument validation logic # Make sure user has test build if self.args.test is None: logger.critical("Must specify test build for regression testing") sys.exit(5) elif self.args.base is None: logger.critical("Must specify base build for regression testing") sys.exit(5) if self.args.scans < 2: logger.critical( "Must specify minimum of 2 scans for regression testing") sys.exit(5) if self.args.prefs is not None: if self.args.prefs_test is not None or self.args.prefs_base is not None: logger.warning( "Detected both global prefs and individual build prefs.") self.test_app = self.get_test_candidate(self.args.test) self.base_app = self.get_test_candidate(self.args.base) self.test_metadata = self.collect_worker_info(self.test_app) self.base_metadata = self.collect_worker_info(self.base_app) # Setup custom profiles self.test_profile = self.make_profile("test_profile", self.args.onecrl) self.base_profile = self.make_profile("base_profile", "production") # Compile the set of hosts to test db = sdb.SourcesDB(self.args) logger.info("Reading `%s` host database" % self.args.source) self.sources = db.read(self.args.source) logger.info("%d hosts in test set" % len(self.sources)) # Sanity check for OneCRL - if it fails, abort run if not self.one_crl_sanity_check(): logger.critical("OneCRL sanity check failed, aborting run") sys.exit(5)
def setup_args(cls, parser): """ Add subparser for the mode's specific arguments. This definition serves as default, but modes are free to override it. :param parser: parent argparser to add to :return: None """ # By nature of workdir being undetermined at this point, user-defined test sets in # the override directory can not override the default test set. The defaulting logic # needs to move behind the argument parser for that to happen. src = sdb.SourcesDB() testset_default = src.default release_choice, _, test_default, base_default = fd.FirefoxDownloader.list() group = parser.add_argument_group("test candidates selection") group.add_argument('-t', '--test', help=("Firefox version to test. It can be one of {%s}, a package file, " "or a build directory (default: `%s`)") % (",".join(release_choice), test_default), action='store', default=test_default) group.add_argument("-b", "--base", help=("Firefox base version to compare against. It can be one of {%s}, a package file, " "or a build directory (default: `%s`)") % (",".join(release_choice), base_default), action='store', default=base_default) group = parser.add_argument_group("profile setup") group.add_argument("-c", "--cache", help='Allow profiles to cache web content', action="store_true", default=False) group.add_argument("-o", "--onecrl", help="OneCRL set to test. Can be production|stage|custom|none or a file name " "(default: production)", action="store", default="production") group.add_argument("--onecrlpin", help="OneCRL-Tools git branch, tag, or commit to use (default: stable)", action="store", default=os.environ.get("ONECRLPIN", "stable")) group.add_argument("-p", "--prefs", help="Prefs to apply to all builds", type=str, action="append", default=None) group.add_argument("-p1", "--prefs_test", help="Prefs to apply to test build", type=str, action="append", default=None) group.add_argument("-p2", "--prefs_base", help="Prefs to apply to base build", type=str, action="append", default=None) group = parser.add_argument_group("host database selection") group.add_argument("-s", "--source", help="Test set to run. Use `list` for info. (default: `%s`)" % testset_default, action="store", default=testset_default) group.add_argument("-l", "--limit", help="Limit for number of hosts to test (default: no limit)", type=int, action="store", default=None) group = parser.add_argument_group("worker configuration") group.add_argument("-j", "--parallel", help="Number of parallel worker instances (default: 4)", type=int, action="store", default=4) group.add_argument("-m", "--timeout", help="Timeout for worker requests (default: 10)", type=float, action="store", default=10) group.add_argument("-n", "--requestsperworker", help="Number of requests per worker (default: 50)", type=int, action="store", default=50) group.add_argument("-u", "--max_timeout", help="Maximum timeout for worker requests (default: 20)", type=float, action="store", default=20) group.add_argument("-x", "--scans", help="Number of scans per host (default: 3)", type=int, action="store", default=3) group = parser.add_argument_group("result processing") group.add_argument("-f", "--filter", help="Filter level for results 0:none 1:timeouts (default: 1)", type=int, choices=[0, 1], action="store", default=1) group.add_argument("-r", "--remove_certs", help="Filter certificate data from results", action="store_true")
def sources_db(tmpdir): """A SourcesDB fixture""" return sdb.SourcesDB(ArgsMock(workdir=tmpdir))