def setup(self):
global logger
self.app = self.get_test_candidate(self.args.base)
self.profile = self.make_profile("base_profile")
tmp_zip_name = os.path.join(self.tmp_dir, "top.zip")
logger.info("Fetching unfiltered top sites data from the `Tranco Top 1M` online database")
get_to_file(self.top_sites_location, tmp_zip_name)
try:
zipped = zipfile.ZipFile(tmp_zip_name)
if len(zipped.filelist) != 1 or not zipped.filelist[0].orig_filename.lower().endswith(".csv"):
logger.critical("Top sites zip file has unexpected content")
sys.exit(5)
tmp_csv_name = zipped.extract(zipped.filelist[0], self.tmp_dir)
except zipfile.BadZipfile:
logger.critical("Error opening top sites zip archive")
sys.exit(5)
self.db = sdb.SourcesDB(self.args)
is_default = self.args.source == self.db.default
self.sources = sdb.Sources(self.args.source, is_default)
with open(tmp_csv_name) as f:
cr = csv.DictReader(f, fieldnames=["rank", "hostname"])
self.sources.rows = [row for row in cr]
# A mild sanity check to see whether the downloaded data is valid.
if len(self.sources) < 900000:
logger.warning("Top sites is surprisingly small, just %d hosts" % len(self.sources))
if "hostname" not in self.sources.rows[0] or "rank" not in self.sources.rows[0] \
or self.sources.rows[0]["rank"] != 1:
logger.warning("Top sites data looks weird. First line: `%s`" % self.sources.rows[0])
def setup(self):
global logger
# Argument validation logic to make sure user has specified only test build
if self.args.test is None:
logger.critical("Must specify test build for scan")
sys.exit(5)
elif self.args.base is not None:
logger.debug("Ignoring base build parameter")
# Download app and extract metadata
self.test_app = self.get_test_candidate(self.args.test)
self.test_metadata = self.collect_worker_info(self.test_app)
logger.info(
"Testing Firefox %s %s scan run" %
(self.test_metadata["app_version"], self.test_metadata["branch"]))
# Create custom profile
self.test_profile = self.make_profile("test_profile")
# Compile the set of hosts to test
db = sdb.SourcesDB(self.args)
logger.info("Reading `%s` host database" % self.args.source)
self.sources = db.read(self.args.source)
logger.info("%d hosts in test set" % len(self.sources))
def main(argv=None):
global logger, tmp_dir, module_dir
module_dir = os.path.split(__file__)[0]
args = parse_args(argv)
if args.debug:
coloredlogs.install(level='DEBUG')
logger.debug("Command arguments: %s" % args)
cleanup.init()
fix_terminal_encoding()
tmp_dir = __create_tempdir()
# If 'list' is specified as test, list available test sets, builds, and platforms
if "source" in args and args.source == "list":
coloredlogs.install(level='ERROR')
db = sdb.SourcesDB(args)
build_list, platform_list, _, _ = fd.FirefoxDownloader.list()
print("Available builds: %s" % ' '.join(build_list))
print("Available platforms: %s" % ' '.join(platform_list))
print("Available test sets:")
for handle in db.list():
test_set = db.read(handle)
if handle == db.default:
default = " (default)"
else:
default = ""
print(" - %s [%d hosts]%s" % (handle, len(test_set), default))
return 0
# Create workdir (usually ~/.tlscanary, used for caching etc.)
# Assumes that no previous code must write to it.
if not os.path.exists(args.workdir):
logger.debug('Creating working directory %s' % args.workdir)
os.makedirs(args.workdir)
# Load the specified test mode
try:
loader.run(args, module_dir, tmp_dir)
except KeyboardInterrupt:
logger.critical("\nUser interrupt. Quitting...")
return 10
if len(threading.enumerate()) > 1:
logger.info("Waiting for background threads to finish")
while len(threading.enumerate()) > 1:
logger.debug("Remaining threads: %s" % threading.enumerate())
time.sleep(2)
return 0
def one_crl_sanity_check(self):
global logger
# Query host(s) with a known revoked cert and examine the results
# These hosts must be revoked via OCSP and/or OneCRL
db = sdb.SourcesDB(self.args)
self.revoked_source = db.read("revoked")
logger.debug("%d host(s) in revoked test set" %
len(self.revoked_source))
next_chunk = self.revoked_source.iter_chunks(chunk_size=1 / 50,
min_chunk_size=1000)
host_set_chunk = next_chunk(as_set=True)
# Note: turn off OCSP for this test, to factor out that mechanism
self.custom_ocsp_pref = ["security.OCSP.enabled;0"]
# First, use the test build and profile as-is
# This should return errors, which means OneCRL is working
test_result = self.run_test(self.test_app,
url_list=host_set_chunk,
profile=self.test_profile,
prefs=self.custom_ocsp_pref,
num_workers=1,
n_per_worker=1)
# Second, use the test build with a profile that is missing OneCRL entries
# This should NOT return errors, which means we've turned off protection
self.altered_profile = self.make_profile("altered_profile", "none")
base_result = self.run_test(self.test_app,
url_list=host_set_chunk,
profile=self.altered_profile,
prefs=self.custom_ocsp_pref,
num_workers=1,
n_per_worker=1)
logger.debug("Length of first OneCRL check, with revocation: %d" %
len(test_result))
logger.debug("Length of second OneCRL check, without revocation: %d" %
len(base_result))
# If our list of revoked sites are all blocked, and we can verify
# that they can be unblocked, this confirms that OneCRL is working
if len(test_result) == len(
self.revoked_source) and len(base_result) == 0:
return True
else:
logger.warning(
"OneCRL check failed. This is expected, so continuing")
return True
def test_sources_db_write_and_override(tmpdir):
"""SourcesDB databases can be written and overridden"""
db = sdb.SourcesDB(ArgsMock(workdir=tmpdir))
old = db.read("debug")
old_default = db.default
override = sdb.Sources("debug", True)
row_one = {"foo": "bar", "baz": "bang", "boom": "bang"}
row_two = {"foo": "bar2", "baz": "bang2", "boom": "bang2"}
override.append(row_one)
override.append(row_two)
db.write(override)
# New SourcesDB instance required to detect overrides
db = sdb.SourcesDB(ArgsMock(workdir=tmpdir))
assert os.path.exists(tmpdir.join("sources",
"debug.csv")), "override file is written"
assert db.default == "debug", "overriding the default works"
assert old_default != db.default, "overridden default actually changes"
new = db.read("debug")
assert len(new) == 2, "number of overridden rows is correct"
assert new[0] == row_one and new[
1] == row_two, "new rows are written as expected"
assert old[0] != new[0], "overridden rows actually change"
def setup(self):
global logger
# Argument validation logic
# Make sure user has test build
if self.args.test is None:
logger.critical("Must specify test build for regression testing")
sys.exit(5)
elif self.args.base is None:
logger.critical("Must specify base build for regression testing")
sys.exit(5)
if self.args.scans < 2:
logger.critical(
"Must specify minimum of 2 scans for regression testing")
sys.exit(5)
if self.args.prefs is not None:
if self.args.prefs_test is not None or self.args.prefs_base is not None:
logger.warning(
"Detected both global prefs and individual build prefs.")
self.test_app = self.get_test_candidate(self.args.test)
self.base_app = self.get_test_candidate(self.args.base)
self.test_metadata = self.collect_worker_info(self.test_app)
self.base_metadata = self.collect_worker_info(self.base_app)
# Setup custom profiles
self.test_profile = self.make_profile("test_profile", self.args.onecrl)
self.base_profile = self.make_profile("base_profile", "production")
# Compile the set of hosts to test
db = sdb.SourcesDB(self.args)
logger.info("Reading `%s` host database" % self.args.source)
self.sources = db.read(self.args.source)
logger.info("%d hosts in test set" % len(self.sources))
# Sanity check for OneCRL - if it fails, abort run
if not self.one_crl_sanity_check():
logger.critical("OneCRL sanity check failed, aborting run")
sys.exit(5)
def setup_args(cls, parser):
"""
Add subparser for the mode's specific arguments.
This definition serves as default, but modes are free to
override it.
:param parser: parent argparser to add to
:return: None
"""
# By nature of workdir being undetermined at this point, user-defined test sets in
# the override directory can not override the default test set. The defaulting logic
# needs to move behind the argument parser for that to happen.
src = sdb.SourcesDB()
testset_default = src.default
release_choice, _, test_default, base_default = fd.FirefoxDownloader.list()
group = parser.add_argument_group("test candidates selection")
group.add_argument('-t', '--test',
help=("Firefox version to test. It can be one of {%s}, a package file, "
"or a build directory (default: `%s`)") % (",".join(release_choice), test_default),
action='store',
default=test_default)
group.add_argument("-b", "--base",
help=("Firefox base version to compare against. It can be one of {%s}, a package file, "
"or a build directory (default: `%s`)") % (",".join(release_choice), base_default),
action='store',
default=base_default)
group = parser.add_argument_group("profile setup")
group.add_argument("-c", "--cache",
help='Allow profiles to cache web content',
action="store_true",
default=False)
group.add_argument("-o", "--onecrl",
help="OneCRL set to test. Can be production|stage|custom|none or a file name "
"(default: production)",
action="store",
default="production")
group.add_argument("--onecrlpin",
help="OneCRL-Tools git branch, tag, or commit to use (default: stable)",
action="store",
default=os.environ.get("ONECRLPIN", "stable"))
group.add_argument("-p", "--prefs",
help="Prefs to apply to all builds",
type=str,
action="append",
default=None)
group.add_argument("-p1", "--prefs_test",
help="Prefs to apply to test build",
type=str,
action="append",
default=None)
group.add_argument("-p2", "--prefs_base",
help="Prefs to apply to base build",
type=str,
action="append",
default=None)
group = parser.add_argument_group("host database selection")
group.add_argument("-s", "--source",
help="Test set to run. Use `list` for info. (default: `%s`)" % testset_default,
action="store",
default=testset_default)
group.add_argument("-l", "--limit",
help="Limit for number of hosts to test (default: no limit)",
type=int,
action="store",
default=None)
group = parser.add_argument_group("worker configuration")
group.add_argument("-j", "--parallel",
help="Number of parallel worker instances (default: 4)",
type=int,
action="store",
default=4)
group.add_argument("-m", "--timeout",
help="Timeout for worker requests (default: 10)",
type=float,
action="store",
default=10)
group.add_argument("-n", "--requestsperworker",
help="Number of requests per worker (default: 50)",
type=int,
action="store",
default=50)
group.add_argument("-u", "--max_timeout",
help="Maximum timeout for worker requests (default: 20)",
type=float,
action="store",
default=20)
group.add_argument("-x", "--scans",
help="Number of scans per host (default: 3)",
type=int,
action="store",
default=3)
group = parser.add_argument_group("result processing")
group.add_argument("-f", "--filter",
help="Filter level for results 0:none 1:timeouts (default: 1)",
type=int,
choices=[0, 1],
action="store",
default=1)
group.add_argument("-r", "--remove_certs",
help="Filter certificate data from results",
action="store_true")
def sources_db(tmpdir):
"""A SourcesDB fixture"""
return sdb.SourcesDB(ArgsMock(workdir=tmpdir))
