def queue(command): try: return scanner.queue_scan_command(server_info, command) except OSError as err: text = ("OSError - likely too many processes and open files.") data['errors'].append(text) logging.warn("%s\n%s" % (text, utils.format_last_exception())) return None, None, None, None, None, None except Exception as err: text = ("Unknown exception queueing sslyze command.\n%s" % utils.format_last_exception()) data['errors'].append(text) logging.warn(text) return None, None, None, None, None, None
def gather(suffix, options): url = options.get("url") if url is None: logging.warn("A --url is required. (Can be a local path.)") exit(1) # remote URL if url.startswith("http:") or url.startswith("https:"): # Though it's saved in cache/, it will be downloaded every time. remote_path = os.path.join(utils.cache_dir(), "url.csv") try: response = requests.get(url) utils.write(response.text, remote_path) except: logging.error("Remote URL not downloaded successfully.") print(utils.format_last_exception()) exit(1) # local path else: remote_path = url for domain in utils.load_domains(remote_path): yield domain
def scan_serial(scanner, server_info, data, options): logging.debug("\tRunning scans in serial.") logging.debug("\t\tSSLv2 scan.") sslv2 = scanner.run_scan_command(server_info, Sslv20ScanCommand()) logging.debug("\t\tSSLv3 scan.") sslv3 = scanner.run_scan_command(server_info, Sslv30ScanCommand()) logging.debug("\t\tTLSv1.0 scan.") tlsv1 = scanner.run_scan_command(server_info, Tlsv10ScanCommand()) logging.debug("\t\tTLSv1.1 scan.") tlsv1_1 = scanner.run_scan_command(server_info, Tlsv11ScanCommand()) logging.debug("\t\tTLSv1.2 scan.") tlsv1_2 = scanner.run_scan_command(server_info, Tlsv12ScanCommand()) certs = None if options.get("sslyze-certs", True) is True: try: logging.debug("\t\tCertificate information scan.") certs = scanner.run_scan_command(server_info, CertificateInfoScanCommand()) # Let generic exceptions bubble up. except idna.core.InvalidCodepoint: logging.warn(utils.format_last_exception()) data['errors'].append("Invalid certificate/OCSP for this domain.") certs = None else: certs = None logging.debug("\tDone scanning.") return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs
def gather(suffixes, options, extra={}): # Returns a parsed, processed Google service credentials object. credentials = load_credentials() if credentials is None: logging.warn("No BigQuery credentials provided.") logging.warn( "Set BIGQUERY_CREDENTIALS or BIGQUERY_CREDENTIALS_PATH environment variables." ) exit(1) # When using this form of instantiation, the client won't pull # the project_id out of the creds, has to be set explicitly. client = bigquery.Client(project=credentials.project_id, credentials=credentials) # Allow override of default timeout (in seconds). timeout = int(options.get("timeout", default_timeout)) # Construct the query. query = query_for(suffixes) logging.debug("Censys query:\n%s\n" % query) # Plan to store in cache/censys/export.csv. download_path = utils.cache_path("export", "censys", ext="csv") # Reuse of cached data can be turned on with --cache. cache = options.get("cache", False) if (cache is True) and os.path.exists(download_path): logging.warn("Using cached download data.") # But by default, fetch new data from the BigQuery API, # and write it to the expected download location. else: logging.warn("Kicking off SQL query job.") rows = None # Actually execute the query. try: # Executes query and loads all results into memory. query_job = client.query(query) iterator = query_job.result(timeout=timeout) rows = list(iterator) except google.api_core.exceptions.Forbidden: logging.warn("Access denied to Censys' BigQuery tables.") except: logging.warn(utils.format_last_exception()) logging.warn("Error talking to BigQuery, aborting.") # At this point, the job is complete and we need to download # the resulting CSV URL in results_url. logging.warn("Caching results of SQL query.") download_file = open(download_path, 'w', newline='') download_writer = csv.writer(download_file) download_writer.writerow(["Domain"]) # will be skipped on read # Parse the rows and write them out as they were returned (dupes # and all), to be de-duped by the central gathering script. for row in rows: domains = row['common_name'] + row['dns_names'] for domain in domains: download_writer.writerow([domain]) # End CSV writing. download_file.close() # Whether we downloaded it fresh or not, read from the cached data. for domain in utils.load_domains(download_path): if domain: yield domain
def paginated_mode(suffix, options, uid, api_key): # Cache hostnames in a dict for de-duping. hostnames_map = {} certificate_api = certificates.CensysCertificates(uid, api_key) if 'query' in options and options['query']: query = options['query'] else: query = "parsed.subject.common_name:\"%s\" or parsed.extensions.subject_alt_name.dns_names:\"%s\"" % ( suffix, suffix) logging.debug("Censys query:\n%s\n" % query) # time to sleep between requests (defaults to 5s) delay = int(options.get("delay", 5)) # Censys page size, fixed page_size = 100 # Start page defaults to 1. start_page = int(options.get("start", 1)) # End page defaults to whatever the API says is the last one. end_page = options.get("end", None) if end_page is None: end_page = get_end_page(query, certificate_api) if end_page is None: logging.warn("Error looking up number of pages.") exit(1) else: end_page = int(end_page) max_records = ((end_page - start_page) + 1) * page_size fields = [ "parsed.subject.common_name", "parsed.extensions.subject_alt_name.dns_names" ] current_page = start_page logging.warn("Fetching up to %i records, starting at page %i." % (max_records, start_page)) last_cached = False force = options.get("force", False) while current_page <= end_page: if (not last_cached) and (current_page > start_page): logging.debug("(Waiting %is before fetching page %i.)" % (delay, current_page)) last_cached = False time.sleep(delay) logging.debug("Fetching page %i." % current_page) cache_page = utils.cache_path(str(current_page), "censys") if (force is False) and (os.path.exists(cache_page)): logging.warn("\t[%i] Cached page." % current_page) last_cached = True certs_raw = open(cache_page).read() certs = json.loads(certs_raw) if (certs.__class__ is dict) and certs.get('invalid'): continue else: try: certs = list( certificate_api.search(query, fields=fields, page=current_page, max_records=page_size)) utils.write(utils.json_for(certs), cache_page) except censys.base.CensysException: logging.warn(utils.format_last_exception()) logging.warn("Censys error, skipping page %i." % current_page) utils.write(utils.invalid({}), cache_page) continue except: logging.warn(utils.format_last_exception()) logging.warn("Unexpected error, skipping page %i." % current_page) utils.write(utils.invalid({}), cache_page) exit(1) for cert in certs: # Common name + SANs names = cert.get('parsed.subject.common_name', []) + cert.get( 'parsed.extensions.subject_alt_name.dns_names', []) logging.debug(names) for name in names: hostnames_map[sanitize_name(name)] = None current_page += 1 logging.debug("Done fetching from API.") return hostnames_map
def export_mode(suffix, options, uid, api_key): # Cache hostnames in a dict for de-duping. hostnames_map = {} # Default timeout to 20 minutes. timeout = int(options.get("timeout", (60 * 60 * 20))) # Wait 5 seconds between checking on the job. between_jobs = 5 try: export_api = export.CensysExport(uid, api_key) except censys.base.CensysUnauthorizedException: logging.warn( "The Censys.io Export API rejected the provided Censys credentials. The credentials may be inaccurate, or you may need to request access from the Censys.io team." ) exit(1) # Uses a FLATTEN command in order to work around a BigQuery # error around multiple "repeated" fields. *shrug* query = "SELECT parsed.subject.common_name, parsed.extensions.subject_alt_name.dns_names from FLATTEN([certificates.certificates], parsed.extensions.subject_alt_name.dns_names) where parsed.subject.common_name LIKE \"%%%s\" OR parsed.extensions.subject_alt_name.dns_names LIKE \"%%%s\";" % ( suffix, suffix) logging.debug("Censys query:\n%s\n" % query) download_file = utils.cache_path("export", "censys", ext="csv") force = options.get("force", False) if (force is False) and os.path.exists(download_file): logging.warn("Using cached download data.") else: logging.warn("Kicking off SQL query job.") results_url = None try: job = export_api.new_job(query, format='csv', flatten=True) job_id = job['job_id'] started = datetime.datetime.now() while True: elapsed = (datetime.datetime.now() - started).seconds status = export_api.check_job(job_id) if status['status'] == 'error': logging.warn("Error from Censys: %s" % status['error']) exit(1) # Not expected, but better to explicitly handle. elif status['status'] == 'expired': logging.warn("Results are somehow expired, bailing.") exit(1) elif status['status'] == 'pending': logging.debug("[%is] Job still pending." % elapsed) time.sleep(between_jobs) elif status['status'] == 'success': logging.warn("[%is] Job complete!" % elapsed) results_url = status['download_paths'][0] break if (elapsed > timeout): logging.warn("Timeout waiting for job to complete.") exit(1) except censys.base.CensysException: logging.warn(utils.format_last_exception()) logging.warn("Censys error, aborting.") # At this point, the job is complete and we need to download # the resulting CSV URL in results_url. logging.warn("Downloading results of SQL query.") utils.download(results_url, download_file) # Read in downloaded CSV file, run any hostnames in each line # through the sanitizer, and de-dupe using the map. with open(download_file, newline='') as csvfile: for row in csv.reader(csvfile): if (not row[0]) or ( row[0].lower().startswith("parsed_subject_common_name")): continue names = [row[0].lower(), row[1].lower()] # logging.debug(names) for name in names: if name: hostnames_map[sanitize_name(name)] = None return hostnames_map
def main(): options = utils.options() debug = options.get('debug', False) encoding = options.get('encoding', 'latin-1') name = options.get('name', 'hostnames') filter_name = options.get('filter', name) filter = filters.get(filter_name, None) if filter is None: print("No filter by that name. Specify one with --filter.") exit(1) # Drop output in a directory next to the script. this_dir = os.path.dirname(__file__) output = os.path.join(this_dir, "hostnames") utils.mkdir_p(output) out_filename = "%s.csv" % name out_file = open(os.path.join(output, out_filename), 'w', newline='') out_writer = csv.writer(out_file) if len(options["_"]) < 1: print("Provide the name to an input file.") exit(1) input_filename = options["_"][0] if not os.path.exists(input_filename): print("Input file doesn't exist.") exit(1) suffix = options.get("suffix", ".gov") # if it has a ., make sure the . is escaped if suffix.startswith("."): suffix = "\\%s" % suffix pattern = re.compile("%s\n?$" % suffix) max = int(options.get("max", -1)) # Proceed missed = 0 matched = 0 name_map = {} curr = 0 with open(input_filename, encoding=encoding) as f: try: for line in f: if pattern.search(line): hostname = filter(line) if debug: print("Match!!!! %s" % hostname) matched += 1 name_map[hostname] = None else: if debug: print("Didn't match: %s" % line.strip()) missed += 1 curr += 1 if (max > 0) and (curr >= max): print("Stopping at %i." % curr) break if (curr % 1000000) == 0: print("Processing: %i" % curr) except UnicodeDecodeError as e: print(curr) print(utils.format_last_exception()) exit(1) hostnames = list(name_map.keys()) hostnames.sort() print("Matched %i (%i unique), missed on %i." % (matched, len(hostnames), missed)) print("Writing out CSV.") for hostname in hostnames: out_writer.writerow([hostname]) print("Done.")
def gather(suffix, options): # Register a (free) Censys.io account to get a UID and API key. uid = options.get("censys_id", None) api_key = options.get("censys_key", None) if (uid is None) or (api_key is None): uid = os.environ.get("CENSYS_UID", None) api_key = os.environ.get("CENSYS_API_KEY", None) if (uid is None) or (api_key is None): logging.warn( "No Censys credentials set. API key required to use the Censys API." ) exit(1) certificate_api = certificates.CensysCertificates(uid, api_key) query = "parsed.subject.common_name:\"%s\" or parsed.extensions.subject_alt_name.dns_names:\"%s\"" % ( suffix, suffix) logging.debug("Censys query:\n%s\n" % query) # Hostnames beginning with a wildcard prefix will have the prefix stripped. wildcard_pattern = re.compile("^\*\.") redacted_pattern = re.compile("^(\?\.)+") # time to sleep between requests (defaults to 5s) delay = int(options.get("delay", 5)) # Censys page size, fixed page_size = 100 # Start page defaults to 1. start_page = int(options.get("start", 1)) # End page defaults to whatever the API says is the last one. end_page = options.get("end", None) if end_page is None: end_page = get_end_page(query, certificate_api) if end_page is None: logging.warn("Error looking up number of pages.") exit(1) else: end_page = int(end_page) max_records = ((end_page - start_page) + 1) * page_size # Cache hostnames in a dict for de-duping. hostnames_map = {} fields = [ "parsed.subject.common_name", "parsed.extensions.subject_alt_name.dns_names" ] current_page = start_page logging.warn("Fetching up to %i records, starting at page %i." % (max_records, start_page)) last_cached = False force = options.get("force", False) while current_page <= end_page: if (not last_cached) and (current_page > start_page): logging.debug("(Waiting %is before fetching page %i.)" % (delay, current_page)) last_cached = False time.sleep(delay) logging.debug("Fetching page %i." % current_page) cache_page = utils.cache_path(str(current_page), "censys") if (force is False) and (os.path.exists(cache_page)): logging.warn("\t[%i] Cached page." % current_page) last_cached = True certs_raw = open(cache_page).read() certs = json.loads(certs_raw) if (certs.__class__ is dict) and certs.get('invalid'): continue else: try: certs = list( certificate_api.search(query, fields=fields, page=current_page, max_records=page_size)) utils.write(utils.json_for(certs), cache_page) except censys.base.CensysException: logging.warn(utils.format_last_exception()) logging.warn("Censys error, skipping page %i." % current_page) utils.write(utils.invalid({}), cache_page) continue except: logging.warn(utils.format_last_exception()) logging.warn("Unexpected error, skipping page %i." % current_page) utils.write(utils.invalid({}), cache_page) exit(1) for cert in certs: # Common name + SANs names = cert.get('parsed.subject.common_name', []) + cert.get( 'parsed.extensions.subject_alt_name.dns_names', []) logging.debug(names) for name in names: # Strip off any wildcard prefix. name = re.sub(wildcard_pattern, '', name).lower().strip() # Strip off any redacted ? prefixes. (Ugh.) name = re.sub(redacted_pattern, '', name).lower().strip() hostnames_map[name] = None current_page += 1 logging.debug("Done fetching from API.") # Iterator doesn't buy much efficiency, since we paginated already. # Necessary evil to de-dupe before returning hostnames, though. for hostname in hostnames_map.keys(): yield hostname
def scan_parallel(scanner, server_info, data, options): logging.debug("\tRunning scans in parallel.") def queue(command): try: return scanner.queue_scan_command(server_info, command) except OSError as err: text = ("OSError - likely too many processes and open files.") data['errors'].append(text) logging.warn("%s\n%s" % (text, utils.format_last_exception())) return None, None, None, None, None, None except Exception as err: text = ("Unknown exception queueing sslyze command.\n%s" % utils.format_last_exception()) data['errors'].append(text) logging.warn(text) return None, None, None, None, None, None # Initialize commands and result containers sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs = None, None, None, None, None, None # Queue them all up queue(Sslv20ScanCommand()) queue(Sslv30ScanCommand()) queue(Tlsv10ScanCommand()) queue(Tlsv11ScanCommand()) queue(Tlsv12ScanCommand()) if options.get("sslyze-certs", True) is True: queue(CertificateInfoScanCommand()) # Reassign them back to predictable places after they're all done was_error = False for result in scanner.get_results(): try: if isinstance(result, PluginRaisedExceptionScanResult): error = ("Scan command failed: %s" % result.as_text()) logging.warn(error) data['errors'].append(error) return None, None, None, None, None, None if type(result.scan_command) == Sslv20ScanCommand: sslv2 = result elif type(result.scan_command) == Sslv30ScanCommand: sslv3 = result elif type(result.scan_command) == Tlsv10ScanCommand: tlsv1 = result elif type(result.scan_command) == Tlsv11ScanCommand: tlsv1_1 = result elif type(result.scan_command) == Tlsv12ScanCommand: tlsv1_2 = result elif type(result.scan_command) == CertificateInfoScanCommand: certs = result else: error = "Couldn't match scan result with command! %s" % result logging.warn("\t%s" % error) data['errors'].append(error) was_error = True except Exception as err: was_error = True text = ("Exception inside async scanner result processing.\n%s" % utils.format_last_exception()) data['errors'].append(text) logging.warn("\t%s" % text) # There was an error during async processing. if was_error: return None, None, None, None, None, None logging.debug("\tDone scanning.") return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, certs
def main(): options = utils.options() debug = options.get('debug', False) encoding = options.get('encoding', 'latin-1') name = options.get('name', 'hostnames') filter_name = options.get('filter', name) filter = filters.get(filter_name, None) if filter is None: print("No filter by that name. Specify one with --filter.") exit(1) # Drop output in a directory next to the script. this_dir = os.path.dirname(__file__) output = os.path.join(this_dir, "hostnames") utils.mkdir_p(output) out_filename = "%s.csv" % name out_file = open(os.path.join(output, out_filename), 'w', newline='') out_writer = csv.writer(out_file) if len(options["_"]) < 1: print("Provide the name to an input file.") exit(1) input_filename = options["_"][0] if not os.path.exists(input_filename): print("Input file doesn't exist.") exit(1) suffix = options.get("suffix", ".gov") # if it has a ., make sure the . is escaped if suffix.startswith("."): suffix = "\\%s" % suffix pattern = re.compile("%s\n?$" % suffix) max = int(options.get("max", -1)) # Proceed missed = 0 matched = 0 name_map = {} curr = 0 with open(input_filename, encoding=encoding) as f: try: for line in f: if pattern.search(line): hostname = filter(line) if debug: print("Match!!!! %s" % hostname) matched += 1 name_map[hostname] = None else: if debug: print("Didn't match: %s" % line.strip()) missed += 1 curr += 1 if (max > 0) and (curr >= max): print("Stopping at %i." % curr) break if (curr % 1000000) == 0: print("Processing: %i" % curr) except UnicodeDecodeError as e: print(curr) print(utils.format_last_exception()) exit(1) hostnames = list(name_map.keys()) hostnames.sort() print("Matched %i (%i unique), missed on %i." % (matched, len(hostnames), missed)) print("Writing out CSV.") for hostname in hostnames: out_writer.writerow([hostname]) print("Done.")