def queue(command):
try:
return scanner.queue_scan_command(server_info, command)
except OSError:
text = ("OSError - likely too many processes and open files.")
data['errors'].append(text)
logging.exception("%s\n%s" % (text, utils.format_last_exception()))
return None, None, None, None, None, None, None
except Exception:
text = ("Unknown exception queueing sslyze command.\n%s" % utils.format_last_exception())
data['errors'].append(text)
logging.exception(text)
return None, None, None, None, None, None, None
def queue(command):
try:
return scanner.queue_scan_command(server_info, command)
except OSError:
text = ("OSError - likely too many processes and open files.")
data['errors'].append(text)
logging.warning("%s\n%s" % (text, utils.format_last_exception()))
return None, None, None, None, None, None, None
except Exception:
text = ("Unknown exception queueing sslyze command.\n%s" % utils.format_last_exception())
data['errors'].append(text)
logging.warning(text)
return None, None, None, None, None, None, None
def gather(self):
# Defaults to --url, but can be overridden.
name = self.extra.get("name", "url")
url = self.options.get(name)
if url is None:
logging.warn("A --url is required. (Can be a local path.)")
exit(1)
# remote URL
if url.startswith("http:") or url.startswith("https:"):
# Though it's saved in cache/, it will be downloaded every time.
remote_path = os.path.join(self.cache_dir, "url.csv")
try:
response = requests.get(url)
utils.write(response.text, remote_path)
except:
logging.error("Remote URL not downloaded successfully.")
print(utils.format_last_exception())
exit(1)
# local path
else:
remote_path = url
for domain in utils.load_domains(remote_path):
yield domain
def scan_serial(scanner, server_info, data, options):
logging.debug("\tRunning scans in serial.")
logging.debug("\t\tSSLv2 scan.")
sslv2 = scanner.run_scan_command(server_info, Sslv20ScanCommand())
logging.debug("\t\tSSLv3 scan.")
sslv3 = scanner.run_scan_command(server_info, Sslv30ScanCommand())
logging.debug("\t\tTLSv1.0 scan.")
tlsv1 = scanner.run_scan_command(server_info, Tlsv10ScanCommand())
logging.debug("\t\tTLSv1.1 scan.")
tlsv1_1 = scanner.run_scan_command(server_info, Tlsv11ScanCommand())
logging.debug("\t\tTLSv1.2 scan.")
tlsv1_2 = scanner.run_scan_command(server_info, Tlsv12ScanCommand())
logging.debug("\t\tTLSv1.3 scan.")
tlsv1_3 = scanner.run_scan_command(server_info, Tlsv13ScanCommand())
certs = None
if options.get("sslyze_certs", True) is True:
try:
logging.debug("\t\tCertificate information scan.")
certs = scanner.run_scan_command(server_info,
CertificateInfoScanCommand())
# Let generic exceptions bubble up.
except idna.core.InvalidCodepoint:
logging.warn(utils.format_last_exception())
data['errors'].append("Invalid certificate/OCSP for this domain.")
certs = None
else:
certs = None
logging.debug("\tDone scanning.")
return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs
def gather(self):
# Defaults to --url, but can be overridden.
name = self.extra.get("name", "url")
url = self.options.get(name)
if url is None:
logging.warning("A --url is required. (Can be a local path.)")
exit(1)
# remote URL
if url.startswith("http:") or url.startswith("https:"):
# Though it's saved in cache/, it will be downloaded every time.
remote_path = os.path.join(self.cache_dir, "url.csv")
try:
response = requests.get(url)
utils.write(response.text, remote_path)
except:
logging.error("Remote URL not downloaded successfully.")
print(utils.format_last_exception())
exit(1)
# local path
else:
remote_path = url
for domain in utils.load_domains(remote_path):
yield domain
def init(environment, options):
global analytics_domains
analytics_file = options.get("analytics")
if (not analytics_file) or (not analytics_file.endswith(".csv")):
no_csv = "--analytics should point to the file path or URL to a CSV of participating domains."
logging.error(no_csv)
return False
# It's a URL, download it first.
if analytics_file.startswith("http:") or analytics_file.startswith(
"https:"):
analytics_path = os.path.join(utils.cache_dir(), "analytics.csv")
try:
utils.download(analytics_file, analytics_path)
except:
logging.error(utils.format_last_exception())
no_csv = "--analytics URL not downloaded successfully."
logging.error(no_csv)
return False
# Otherwise, read it off the disk
else:
analytics_path = analytics_file
if (not os.path.exists(analytics_path)):
no_csv = "--analytics file not found."
logging.error(no_csv)
return False
analytics_domains = utils.load_domains(analytics_path)
return {'analytics_domains': analytics_domains}
def scan_serial(scanner, server_info, data, options):
errors = 0
def run_scan(scan_type, command, errors):
if (errors >= 2):
return None, errors
logging.debug("\t\t{} scan.".format(scan_type))
result = None
try:
result = scanner.run_scan_command(server_info, command)
except Exception as err:
logging.warning("{}: Error during {} scan.".format(
server_info.hostname, scan_type))
logging.debug("{}: Exception during {} scan: {}".format(
server_info.hostname, scan_type, err))
errors = errors + 1
return result, errors
logging.debug("\tRunning scans in serial.")
sslv2, errors = run_scan("SSLv2", Sslv20ScanCommand(), errors)
sslv3, errors = run_scan("SSLv3", Sslv30ScanCommand(), errors)
tlsv1, errors = run_scan("TLSv1.0", Tlsv10ScanCommand(), errors)
tlsv1_1, errors = run_scan("TLSv1.1", Tlsv11ScanCommand(), errors)
tlsv1_2, errors = run_scan("TLSv1.2", Tlsv12ScanCommand(), errors)
tlsv1_3, errors = run_scan("TLSv1.3", Tlsv13ScanCommand(), errors)
certs = None
if errors < 2 and options.get("sslyze_certs", True) is True:
try:
logging.debug("\t\tCertificate information scan.")
certs = scanner.run_scan_command(
server_info, CertificateInfoScanCommand(ca_file=CA_FILE))
except idna.core.InvalidCodepoint:
logging.warning(utils.format_last_exception())
data['errors'].append("Invalid certificate/OCSP for this domain.")
certs = None
except Exception as err:
logging.warning(
"{}: Error during certificate information scan.".format(
server_info.hostname))
logging.debug(
"{}: Exception during certificate information scan: {}".format(
server_info.hostname, err))
else:
certs = None
reneg = None
if options.get("sslyze_reneg", True) is True:
reneg, errors = run_scan("Renegotiation",
SessionRenegotiationScanCommand(), errors)
else:
reneg = None
logging.debug("\tDone scanning.")
return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs, reneg
def handle_scanner_args(args, opts) -> Tuple[dict, list]:
"""
--analytics: file path or URL to a CSV of participating domains.
This function also handles checking for the existence of the file,
downloading it succesfully, and reading the file in order to populate the
list of analytics domains.
"""
parser = scan_utils.ArgumentParser(prefix_chars="--")
parser.add_argument("--analytics", nargs=1, required=True)
parsed, unknown = parser.parse_known_args(args)
dicted = vars(parsed)
should_be_single = ["analytics"]
dicted = scan_utils.make_values_single(dicted, should_be_single)
resource = dicted.get("analytics")
if not resource.endswith(".csv"):
no_csv = "".join([
"--analytics should be the file path or URL to a CSV of participating",
" domains and end with .csv, which '%s' does not" % resource
])
logging.error(no_csv)
raise argparse.ArgumentTypeError(no_csv)
try:
parsed_url = urlparse(resource)
except:
raise
if parsed_url.scheme and parsed_url.scheme in ("http", "https"):
analytics_path = Path(opts["_"]["cache_dir"],
"analytics.csv").resolve()
try:
utils.download(resource, str(analytics_path))
except:
logging.error(utils.format_last_exception())
no_csv = "--analytics URL %s not downloaded successfully." % resource
logging.error(no_csv)
raise argparse.ArgumentTypeError(no_csv)
else:
if not os.path.exists(resource):
no_csv = "--analytics file %s not found." % resource
logging.error(no_csv)
raise FileNotFoundError(no_csv)
else:
analytics_path = resource
analytics_domains = utils.load_domains(analytics_path)
dicted["analytics_domains"] = analytics_domains
del dicted["analytics"]
return (dicted, unknown)
def scan_serial(scanner, server_info, data, options):
errors = 0
def run_scan(scan_type, command, errors):
if(errors >= 2):
return None, errors
logging.debug("\t\t{} scan.".format(scan_type))
result = None
try:
result = scanner.run_scan_command(server_info, command)
except Exception as err:
logging.warning("{}: Error during {} scan.".format(server_info.hostname, scan_type))
logging.debug("{}: Exception during {} scan: {}".format(server_info.hostname, scan_type, err))
errors = errors + 1
return result, errors
logging.debug("\tRunning scans in serial.")
sslv2, errors = run_scan("SSLv2", Sslv20ScanCommand(), errors)
sslv3, errors = run_scan("SSLv3", Sslv30ScanCommand(), errors)
tlsv1, errors = run_scan("TLSv1.0", Tlsv10ScanCommand(), errors)
tlsv1_1, errors = run_scan("TLSv1.1", Tlsv11ScanCommand(), errors)
tlsv1_2, errors = run_scan("TLSv1.2", Tlsv12ScanCommand(), errors)
tlsv1_3, errors = run_scan("TLSv1.3", Tlsv13ScanCommand(), errors)
certs = None
if errors < 2 and options.get("sslyze_certs", True) is True:
try:
logging.debug("\t\tCertificate information scan.")
certs = scanner.run_scan_command(server_info, CertificateInfoScanCommand(ca_file=CA_FILE))
except idna.core.InvalidCodepoint:
logging.warning(utils.format_last_exception())
data['errors'].append("Invalid certificate/OCSP for this domain.")
certs = None
except Exception as err:
logging.warning("{}: Error during certificate information scan.".format(server_info.hostname))
logging.debug("{}: Exception during certificate information scan: {}".format(server_info.hostname, err))
else:
certs = None
reneg = None
if options.get("sslyze_reneg", True) is True:
reneg, errors = run_scan("Renegotiation", SessionRenegotiationScanCommand(), errors)
else:
reneg = None
logging.debug("\tDone scanning.")
return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs, reneg
def handle_scanner_args(args, opts) -> Tuple[dict, list]:
"""
--analytics: file path or URL to a CSV of participating domains.
This function also handles checking for the existence of the file,
downloading it succesfully, and reading the file in order to populate the
list of analytics domains.
"""
parser = scan_utils.ArgumentParser(prefix_chars="--")
parser.add_argument("--analytics", nargs=1, required=True)
parsed, unknown = parser.parse_known_args(args)
dicted = vars(parsed)
should_be_single = ["analytics"]
dicted = scan_utils.make_values_single(dicted, should_be_single)
resource = dicted.get("analytics")
if not resource.endswith(".csv"):
no_csv = "".join([
"--analytics should be the file path or URL to a CSV of participating",
" domains and end with .csv, which '%s' does not" % resource
])
logging.error(no_csv)
raise argparse.ArgumentTypeError(no_csv)
try:
parsed_url = urlparse(resource)
except:
raise
if parsed_url.scheme and parsed_url.scheme in ("http", "https"):
analytics_path = Path(opts["_"]["cache_dir"], "analytics.csv").resolve()
try:
utils.download(resource, str(analytics_path))
except:
logging.error(utils.format_last_exception())
no_csv = "--analytics URL %s not downloaded successfully." % resource
logging.error(no_csv)
raise argparse.ArgumentTypeError(no_csv)
else:
if not os.path.exists(resource):
no_csv = "--analytics file %s not found." % resource
logging.error(no_csv)
raise FileNotFoundError(no_csv)
else:
analytics_path = resource
analytics_domains = utils.load_domains(analytics_path)
dicted["analytics_domains"] = analytics_domains
del dicted["analytics"]
return (dicted, unknown)
def gather(suffixes, options, extra={}):
# Returns a parsed, processed Google service credentials object.
credentials = load_credentials()
if credentials is None:
logging.warn("No BigQuery credentials provided.")
logging.warn("Set BIGQUERY_CREDENTIALS or BIGQUERY_CREDENTIALS_PATH environment variables.")
exit(1)
# When using this form of instantiation, the client won't pull
# the project_id out of the creds, has to be set explicitly.
client = bigquery.Client(
project=credentials.project_id,
credentials=credentials
)
# Allow override of default timeout (in seconds).
timeout = int(options.get("timeout", default_timeout))
# Construct the query.
query = query_for(suffixes)
logging.debug("Censys query:\n%s\n" % query)
# Plan to store in cache/censys/export.csv.
download_path = utils.cache_path("export", "censys", ext="csv")
# Reuse of cached data can be turned on with --cache.
cache = options.get("cache", False)
if (cache is True) and os.path.exists(download_path):
logging.warn("Using cached download data.")
# But by default, fetch new data from the BigQuery API,
# and write it to the expected download location.
else:
logging.warn("Kicking off SQL query job.")
rows = None
# Actually execute the query.
try:
# Executes query and loads all results into memory.
query_job = client.query(query)
iterator = query_job.result(timeout=timeout)
rows = list(iterator)
except google.api_core.exceptions.Forbidden:
logging.warn("Access denied to Censys' BigQuery tables.")
except:
logging.warn(utils.format_last_exception())
logging.warn("Error talking to BigQuery, aborting.")
# At this point, the job is complete and we need to download
# the resulting CSV URL in results_url.
logging.warn("Caching results of SQL query.")
download_file = open(download_path, 'w', newline='')
download_writer = csv.writer(download_file)
download_writer.writerow(["Domain"]) # will be skipped on read
# Parse the rows and write them out as they were returned (dupes
# and all), to be de-duped by the central gathering script.
for row in rows:
domains = row['common_name'] + row['dns_names']
for domain in domains:
download_writer.writerow([domain])
# End CSV writing.
download_file.close()
# Whether we downloaded it fresh or not, read from the cached data.
for domain in utils.load_domains(download_path):
if domain:
yield domain
def scan_parallel(scanner, server_info, data, options):
logging.debug("\tRunning scans in parallel.")
def queue(command):
try:
return scanner.queue_scan_command(server_info, command)
except OSError as err:
text = ("OSError - likely too many processes and open files.")
data['errors'].append(text)
logging.warn("%s\n%s" % (text, utils.format_last_exception()))
return None, None, None, None, None, None, None
except Exception as err:
text = ("Unknown exception queueing sslyze command.\n%s" %
utils.format_last_exception())
data['errors'].append(text)
logging.warn(text)
return None, None, None, None, None, None, None
# Initialize commands and result containers
sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs = None, None, None, None, None, None
# Queue them all up
queue(Sslv20ScanCommand())
queue(Sslv30ScanCommand())
queue(Tlsv10ScanCommand())
queue(Tlsv11ScanCommand())
queue(Tlsv12ScanCommand())
queue(Tlsv13ScanCommand())
if options.get("sslyze-certs", True) is True:
queue(CertificateInfoScanCommand())
# Reassign them back to predictable places after they're all done
was_error = False
for result in scanner.get_results():
try:
if isinstance(result, PluginRaisedExceptionScanResult):
error = ("Scan command failed: %s" % result.as_text())
logging.warn(error)
data['errors'].append(error)
return None, None, None, None, None, None, None
if type(result.scan_command) == Sslv20ScanCommand:
sslv2 = result
elif type(result.scan_command) == Sslv30ScanCommand:
sslv3 = result
elif type(result.scan_command) == Tlsv10ScanCommand:
tlsv1 = result
elif type(result.scan_command) == Tlsv11ScanCommand:
tlsv1_1 = result
elif type(result.scan_command) == Tlsv12ScanCommand:
tlsv1_2 = result
elif type(result.scan_command) == Tlsv13ScanCommand:
tlsv1_3 = result
elif type(result.scan_command) == CertificateInfoScanCommand:
certs = result
else:
error = "Couldn't match scan result with command! %s" % result
logging.warn("\t%s" % error)
data['errors'].append(error)
was_error = True
except Exception as err:
was_error = True
text = ("Exception inside async scanner result processing.\n%s" %
utils.format_last_exception())
data['errors'].append(text)
logging.warn("\t%s" % text)
# There was an error during async processing.
if was_error:
return None, None, None, None, None, None, None
logging.debug("\tDone scanning.")
return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs
def gather(self):
# Returns a parsed, processed Google service credentials object.
credentials = load_credentials()
if credentials is None:
logging.warning("No BigQuery credentials provided.")
logging.warning("Set BIGQUERY_CREDENTIALS or BIGQUERY_CREDENTIALS_PATH environment variables.")
exit(1)
# When using this form of instantiation, the client won't pull
# the project_id out of the creds, has to be set explicitly.
client = bigquery.Client(
project=credentials.project_id,
credentials=credentials
)
# Allow override of default timeout (in seconds).
timeout = int(self.options.get("timeout", default_timeout))
# Construct the query.
query = query_for(self.suffixes)
logging.debug("Censys query:\n%s\n" % query)
# Plan to store in cache/censys/export.csv.
download_path = utils.cache_path(
"export", "censys", ext="csv", cache_dir=self.cache_dir)
# Reuse of cached data can be turned on with --cache.
cache = self.options.get("cache", False)
if (cache is True) and os.path.exists(download_path):
logging.warning("Using cached download data.")
# But by default, fetch new data from the BigQuery API,
# and write it to the expected download location.
else:
# Ensure cache destination exists.
utils.mkdir_p(os.path.dirname(download_path))
logging.warning("Kicking off SQL query job.")
rows = None
# Actually execute the query.
try:
# Executes query and loads all results into memory.
query_job = client.query(query)
iterator = query_job.result(timeout=timeout)
rows = list(iterator)
except google.api_core.exceptions.Forbidden:
logging.warning("Access denied to Censys' BigQuery tables.")
except:
logging.warning(utils.format_last_exception())
logging.warning("Error talking to BigQuery, aborting.")
# At this point, the job is complete and we need to download
# the resulting CSV URL in results_url.
logging.warning("Caching results of SQL query.")
download_file = open(download_path, 'w', newline='')
download_writer = csv.writer(download_file)
download_writer.writerow(["Domain"]) # will be skipped on read
# Parse the rows and write them out as they were returned (dupes
# and all), to be de-duped by the central gathering script.
for row in rows:
domains = row['common_name'] + row['dns_names']
for domain in domains:
download_writer.writerow([domain])
# End CSV writing.
download_file.close()
# Whether we downloaded it fresh or not, read from the cached data.
for domain in utils.load_domains(download_path):
if domain:
yield domain
def scan_parallel(scanner, server_info, data, options):
logging.debug("\tRunning scans in parallel.")
def queue(command):
try:
return scanner.queue_scan_command(server_info, command)
except OSError:
text = ("OSError - likely too many processes and open files.")
data['errors'].append(text)
logging.warning("%s\n%s" % (text, utils.format_last_exception()))
return None, None, None, None, None, None, None
except Exception:
text = ("Unknown exception queueing sslyze command.\n%s" % utils.format_last_exception())
data['errors'].append(text)
logging.warning(text)
return None, None, None, None, None, None, None
# Initialize commands and result containers
sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs, reneg = None, None, None, None, None, None, None, None
# Queue them all up
queue(Sslv20ScanCommand())
queue(Sslv30ScanCommand())
queue(Tlsv10ScanCommand())
queue(Tlsv11ScanCommand())
queue(Tlsv12ScanCommand())
queue(Tlsv13ScanCommand())
if options.get("sslyze-certs", True) is True:
queue(CertificateInfoScanCommand())
if options.get("sslyze-reneg", True) is True:
queue(CertificateInfoScanCommand())
# Reassign them back to predictable places after they're all done
was_error = False
for result in scanner.get_results():
try:
if isinstance(result, PluginRaisedExceptionScanResult):
error = ("Scan command failed: %s" % result.as_text())
logging.warning(error)
data['errors'].append(error)
return None, None, None, None, None, None, None
if type(result.scan_command) == Sslv20ScanCommand:
sslv2 = result
elif type(result.scan_command) == Sslv30ScanCommand:
sslv3 = result
elif type(result.scan_command) == Tlsv10ScanCommand:
tlsv1 = result
elif type(result.scan_command) == Tlsv11ScanCommand:
tlsv1_1 = result
elif type(result.scan_command) == Tlsv12ScanCommand:
tlsv1_2 = result
elif type(result.scan_command) == Tlsv13ScanCommand:
tlsv1_3 = result
elif type(result.scan_command) == CertificateInfoScanCommand:
certs = result
elif type(result.scan_command) == SessionRenegotiationScanCommand:
reneg = result
else:
error = "Couldn't match scan result with command! %s" % result
logging.warning("\t%s" % error)
data['errors'].append(error)
was_error = True
except Exception:
was_error = True
text = ("Exception inside async scanner result processing.\n%s" % utils.format_last_exception())
data['errors'].append(text)
logging.warning("\t%s" % text)
# There was an error during async processing.
if was_error:
return None, None, None, None, None, None, None, None
logging.debug("\tDone scanning.")
return sslv2, sslv3, tlsv1, tlsv1_1, tlsv1_2, tlsv1_3, certs, reneg
