def test_one_host(self): results = crlite_query.parse_hosts_file( io.StringIO( """ example.com:112 """ ) ) self.assertEqual(results, ["example.com:112"])
def test_one_host_and_comments(self): results = crlite_query.parse_hosts_file( io.StringIO( """ # this is a comment ; and so is this example.com:999 # and that was a blank line """ ) ) self.assertEqual(results, ["example.com:999"])
def test_only_comments(self): results = crlite_query.parse_hosts_file( io.StringIO( """ # this is a comment ; and so is this # and that was a blank line """ ) ) self.assertFalse(results)
def test_two_hosts_and_comments(self): results = crlite_query.parse_hosts_file( io.StringIO( """ example.net \t# this is a comment ; \tand so is this example.com:999 # okay we should be done now """ ) ) self.assertEqual(results, ["example.net", "example.com:999"])
def test_empty_file(self): results = crlite_query.parse_hosts_file(io.StringIO("")) self.assertFalse(results)
def main(): parser = argparse.ArgumentParser( description="Query CRLite data", epilog=""" The --db option should point to a folder containing a single filter file of the form "YYYYMMDDnn.filter" along with a collection of files of the form "YYYYMMDDnn.stash" which contain updates from that original filter. By default, if this tool believes it is out-of-date based on the local database, it will attempt to update itself before performing its checks. To avoid that behavior, pass --no-update on the command line. """, ) parser.add_argument( "--hosts", help="Hosts to check, in the form host[:port] where " + "port is assumed 443 if not provided. Can be specified multiple times.", action="append", nargs="+", default=[], metavar="host[:port]", ) parser.add_argument( "--hosts-file", help="File of hosts to check, in the form of 'host[:port]' each line, " + "where port is assumed 443 if not provided. Can be specified multiple " + " times.", action="append", default=[], type=Path, ) parser.add_argument("files", help="PEM files to load", type=argparse.FileType("r"), nargs="*") parser.add_argument( "--db", type=Path, default=Path("~/.crlite_db"), help="Path to CRLite database folder", ) parser.add_argument("--no-update", help="Do not attempt to update the database", action="store_true") group = parser.add_mutually_exclusive_group() group.add_argument("--force-update", help="Force an update to the database", action="store_true") group.add_argument( "--use-filter", help="Use this specific filter file, ignoring the database", type=Path, ) parser.add_argument( "--check-freshness", help="Set exit code 0 if the database is more than this many hours old", type=int, ) parser.add_argument( "--check-not-revoked", help="Set exit code 0 if none of the supplied certificates are revoked", action="store_true", ) parser.add_argument( "--no-delete", help="Do not attempt to delete old database files", action="store_true", ) group = parser.add_mutually_exclusive_group() group.add_argument( "--crlite-url", default=crlite_collection_prod, help="URL to the CRLite records at Remote Settings.", ) group.add_argument( "--crlite-staging", action="store_true", help="Use the staging URL for CRLite", ) parser.add_argument( "--intermediates-url", default=intermediates_collection_prod, help="URL to the CRLite records at Remote Settings.", ) parser.add_argument( "--download-intermediates", action="store_true", help="Download all intermediate PEM files to the database", ) parser.add_argument("--verbose", "-v", help="Be more verbose", action="count", default=0) parser.add_argument( "--structured", help="Emit log entries intended for structured loggers", action="store_true", ) args = parser.parse_args() if args.crlite_staging: args.crlite_url = crlite_collection_stage args.intermediates_url = intermediates_collection_stage if args.verbose > 1: logging.basicConfig(level=logging.DEBUG) if args.verbose > 2: from pyasn1 import debug debug.setLogger(debug.Debug("all")) else: logging.basicConfig(level=logging.INFO) db_dir = args.db.expanduser() if not db_dir.is_dir(): db_dir.expanduser().mkdir() last_updated_file = (db_dir / ".last_updated").expanduser() if last_updated_file.exists() and not args.force_update: updated_file_timestamp = datetime.fromtimestamp( last_updated_file.stat().st_mtime) grace_time = datetime.now() - timedelta(hours=6) if last_updated_file.is_file() and updated_file_timestamp > grace_time: log.info( f"Database was updated at {updated_file_timestamp}, skipping.") log.debug( f"Database was last updated {datetime.now() - updated_file_timestamp} ago." ) args.no_update = True attachments_base_url = find_attachments_base_url(args.crlite_url) intermediates_db = IntermediatesDB( db_path=db_dir, download_pems=args.download_intermediates) crlite_db = CRLiteDB(db_path=args.db) try: if args.force_update or not args.no_update: if args.download_intermediates: log.info( "Downloading all intermediate certificates. Look in " + f"{intermediates_db.intermediates_path}") intermediates_db.update( collection_url=args.intermediates_url, attachments_base_url=attachments_base_url, ) crlite_db.update( collection_url=args.crlite_url, attachments_base_url=attachments_base_url, ) last_updated_file.touch() except KeyboardInterrupt: log.warning("Interrupted.") sys.exit(1) if args.use_filter: crlite_db.load_filter(path=args.use_filter) if not args.no_delete: crlite_db.cleanup() log.info(f"Status: {intermediates_db}, {crlite_db}") if args.check_freshness: freshness_limit = timedelta(hours=args.check_freshness) if crlite_db.age() > freshness_limit: log.error( f"Database age is {crlite_db.age()}, which is larger than {freshness_limit}, " + "aborting!") sys.exit(1) query = CRLiteQuery(intermediates_db=intermediates_db, crlite_db=crlite_db) if not args.files and not args.hosts and not args.hosts_file: log.info( "No PEM files or hosts specified to load. Run with --help for usage." ) to_test = list() for file in args.files: to_test.append((file.name, query.gen_from_pem(file))) host_strings = [] for host_list in args.hosts: host_strings.extend(host_list) for path in args.hosts_file: with path.open("r") as fd: host_strings.extend(parse_hosts_file(fd)) for host_str in host_strings: parts = host_str.split(":") hostname = parts[0] port = 443 if len(parts) > 1: port = int(parts[1]) to_test.append( (f"{hostname}:{port}", query.gen_from_host(hostname, port))) failures = list() for (name, generator) in to_test: for result in query.query(name=name, generator=generator): if args.structured: result.log_query_result() else: result.print_query_result(verbose=args.verbose) if args.check_not_revoked and result.is_revoked(): failures.append(result) if failures: log.error(f"{len(failures)} failures logged:") for result in failures: log.error(result) sys.exit(1)