def scan(domain, options):
logging.debug("[%s][pageload]" % domain)
inspection = utils.data_for(domain, "inspect")
# If we have data from inspect, skip if it's not a live domain.
if inspection and (not inspection.get("up")):
logging.debug("\tSkipping, domain not reachable during inspection.")
return None
# If we have data from inspect, skip if it's just a redirector.
if inspection and (inspection.get("redirect") is True):
logging.debug(
"\tSkipping, domain seen as just a redirector during inspection.")
return None
# phantomas needs a URL, not just a domain.
if not (domain.startswith('http://') or domain.startswith('https://')):
# If we have data from inspect, use the canonical endpoint.
if inspection and inspection.get("canonical"):
url = inspection.get("canonical")
# Otherwise, well, whatever.
else:
url = 'http://' + domain
else:
url = domain
# We'll cache prettified JSON from the output.
cache = utils.cache_path(domain, "pageload")
# If we've got it cached, use that.
if (options.get("force", False) is False) and (os.path.exists(cache)):
logging.debug("\tCached.")
raw = open(cache).read()
data = json.loads(raw)
if data.get('invalid'):
return None
# If no cache, or we should run anyway, do the scan.
else:
logging.debug("\t %s %s --reporter=json --ignore-ssl-errors" %
(command, url))
raw = utils.scan(
[command, url, "--reporter=json", "--ignore-ssl-errors"])
if not raw:
utils.write(utils.invalid({}), cache)
return None
# It had better be JSON, which we can cache in prettified form.
data = json.loads(raw)
utils.write(utils.json_for(data), cache)
yield [data['metrics'][metric] for metric in interesting_metrics]
def scan(domain, options):
logging.debug("[%s][pageload]" % domain)
inspection = utils.data_for(domain, "inspect")
# If we have data from inspect, skip if it's not a live domain.
if inspection and (not inspection.get("up")):
logging.debug("\tSkipping, domain not reachable during inspection.")
return None
# If we have data from inspect, skip if it's just a redirector.
if inspection and (inspection.get("redirect") is True):
logging.debug("\tSkipping, domain seen as just a redirector during inspection.")
return None
# phantomas needs a URL, not just a domain.
if not (domain.startswith('http://') or domain.startswith('https://')):
# If we have data from inspect, use the canonical endpoint.
if inspection and inspection.get("canonical"):
url = inspection.get("canonical")
# Otherwise, well, whatever.
else:
url = 'http://' + domain
else:
url = domain
# We'll cache prettified JSON from the output.
cache = utils.cache_path(domain, "pageload")
# If we've got it cached, use that.
if (options.get("force", False) is False) and (os.path.exists(cache)):
logging.debug("\tCached.")
raw = open(cache).read()
data = json.loads(raw)
if data.get('invalid'):
return None
# If no cache, or we should run anyway, do the scan.
else:
logging.debug("\t %s %s --reporter=json --ignore-ssl-errors" % (command, url))
raw = utils.scan([command, url, "--reporter=json", "--ignore-ssl-errors"])
if not raw:
utils.write(utils.invalid({}), cache)
return None
# It had better be JSON, which we can cache in prettified form.
data = json.loads(raw)
utils.write(utils.json_for(data), cache)
yield [data['metrics'][metric] for metric in interesting_metrics]
def scan(domain, options):
logging.debug("[%s][sslyze]" % domain)
# Optional: skip domains which don't support HTTPS in prior inspection
inspection = utils.data_for(domain, "inspect")
if inspection and (not inspection.get("support_https")):
logging.debug("\tSkipping, HTTPS not supported in inspection.")
return None
# Optional: if inspect data says canonical endpoint uses www and this domain
# doesn't have it, add it.
if inspection and (inspection.get("canonical_endpoint") == "www") and (not domain.startswith("www.")):
scan_domain = "www.%s" % domain
else:
scan_domain = domain
# cache XML from sslyze
cache_xml = utils.cache_path(domain, "sslyze", ext="xml")
# because sslyze manages its own output (can't yet print to stdout),
# we have to mkdir_p the path ourselves
utils.mkdir_p(os.path.dirname(cache_xml))
force = options.get("force", False)
if (force is False) and (os.path.exists(cache_xml)):
logging.debug("\tCached.")
xml = open(cache_xml).read()
else:
logging.debug("\t %s %s" % (command, domain))
# use scan_domain (possibly www-prefixed) to do actual scan
raw = utils.scan([command, "--regular", "--quiet", scan_domain, "--xml_out=%s" % cache_xml], env=command_env)
if raw is None:
# TODO: save standard invalid XML data...?
logging.warn("\tBad news scanning, sorry!")
return None
xml = utils.scan(["cat", cache_xml])
if not xml:
logging.warn("\tBad news reading XML, sorry!")
return None
utils.write(xml, cache_xml)
data = parse_sslyze(xml)
if data is None:
logging.warn("\tNo valid target for scanning, couldn't connect.")
return None
utils.write(utils.json_for(data), utils.cache_path(domain, "sslyze"))
yield [
data['protocols']['sslv2'], data['protocols']['sslv3'],
data['protocols']['tlsv1.0'], data['protocols']['tlsv1.1'],
data['protocols']['tlsv1.2'],
data['config'].get('any_dhe'), data['config'].get('all_dhe'),
data['config'].get('weakest_dh'),
data['config'].get('any_rc4'), data['config'].get('all_rc4'),
data['config'].get('ocsp_stapling'),
data['certs'].get('key_type'), data['certs'].get('key_length'),
data['certs'].get('leaf_signature'), data['certs'].get('any_sha1'),
data['certs'].get('not_before'), data['certs'].get('not_after'),
data['certs'].get('served_issuer'),
data.get('errors')
]
def scan(domain, options):
logging.debug("[%s][subdomains]" % domain)
base_original = utils.base_domain_for(domain)
sub_original = domain
base_metadata = domain_map.get(base_original, None)
if domain in exclude_list:
logging.debug("\tSkipping, excluded through manual review.")
return None
# This only looks at subdomains, remove second-level root's and www's.
if re.sub("^www.", "", domain) == base_original:
logging.debug("\tSkipping, second-level domain.")
return None
# If inspection data exists, check to see if we can skip.
inspection = utils.data_for(domain, "inspect")
if not inspection:
logging.debug("\tSkipping, wasn't inspected.")
return None
if not inspection.get("up"):
logging.debug("\tSkipping, subdomain wasn't up during inspection.")
return None
# Default to canonical endpoint, but if that didn't detect right, find the others
endpoint = inspection["endpoints"][inspection.get(
"canonical_protocol")]["root"]
protocol = inspection.get("canonical_protocol")
prefix = inspection.get("canonical_endpoint")
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["http"]["www"]
protocol = "http"
prefix = "www"
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["https"]["root"]
protocol = "https"
prefix = "root"
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["https"]["www"]
protocol = "https"
prefix = "www"
# this should have been the default default, but check anyway
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["http"]["root"]
protocol = "http"
prefix = "root"
# If it's a 0 status code, I guess it's down.
# If it's non-200, we filter out by default.
status = endpoint.get("status", None)
if prefix == "root":
real_prefix = ""
else:
real_prefix = "www."
if status == 0:
logging.debug(
"\tSkipping, really down somehow, status code 0 for all.")
return None
# If the subdomain redirects anywhere, see if it redirects within the domain
if endpoint.get("redirect_to"):
sub_redirect = urllib.parse.urlparse(endpoint["redirect_to"]).hostname
sub_redirect = re.sub("^www.", "",
sub_redirect) # discount www redirects
base_redirect = utils.base_domain_for(sub_redirect)
redirected_external = base_original != base_redirect
redirected_subdomain = ((base_original == base_redirect)
and (sub_original != sub_redirect))
else:
redirected_external = False
redirected_subdomain = False
status_code = endpoint.get("status", None)
# Hit the network for DNS reads and content
endpoint_url = "%s://%s%s" % (protocol, real_prefix, sub_original)
network = network_check(sub_original, endpoint_url, options)
matched_wild = network['matched_wild']
content = network['content']
if content:
try:
hashed = hashlib.sha256(bytearray(content, "utf-8")).hexdigest()
except:
hashed = None
else:
hashed = None
# If it matches a wildcard domain, and the status code we found was non-200,
# the signal-to-noise is just too low to include it.
if matched_wild and (not str(status).startswith('2')):
logging.debug("\tSkipping, wildcard DNS match with %i status code." %
status)
return None
yield [
base_metadata, redirected_external, redirected_subdomain, status_code,
matched_wild, hashed
]
def scan(domain, options):
logging.debug("[%s][subdomains]" % domain)
base_original = utils.base_domain_for(domain)
sub_original = domain
base_metadata = domain_map.get(base_original, None)
if domain in exclude_list:
logging.debug("\tSkipping, excluded through manual review.")
return None
# This only looks at subdomains, remove second-level root's and www's.
if re.sub("^www.", "", domain) == base_original:
logging.debug("\tSkipping, second-level domain.")
return None
# If inspection data exists, check to see if we can skip.
inspection = utils.data_for(domain, "inspect")
if not inspection:
logging.debug("\tSkipping, wasn't inspected.")
return None
if not inspection.get("up"):
logging.debug("\tSkipping, subdomain wasn't up during inspection.")
return None
# Default to canonical endpoint, but if that didn't detect right, find the others
endpoint = inspection["endpoints"][inspection.get("canonical_protocol")]["root"]
protocol = inspection.get("canonical_protocol")
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["http"]["www"]
protocol = "http"
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["https"]["root"]
protocol = "https"
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["https"]["www"]
protocol = "https"
# this should have been the default default, but check anyway
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["http"]["root"]
protocol = "http"
# If it's a 0 status code, I guess it's down.
# If it's non-200, we filter out by default.
status = endpoint.get("status", None)
if status == 0:
logging.debug("\tSkipping, really down somehow, status code 0 for all.")
return None
# bad hostname for cert?
if (protocol == "https") and (endpoint.get("https_bad_name", False) == True):
bad_cert_name = True
else:
bad_cert_name = False
# If the subdomain redirects anywhere, see if it redirects within the domain
if endpoint.get("redirect_to"):
sub_redirect = urllib.parse.urlparse(endpoint["redirect_to"]).hostname
sub_redirect = re.sub("^www.", "", sub_redirect) # discount www redirects
base_redirect = utils.base_domain_for(sub_redirect)
redirected_external = base_original != base_redirect
redirected_subdomain = (
(base_original == base_redirect) and
(sub_original != sub_redirect)
)
else:
redirected_external = False
redirected_subdomain = False
status_code = endpoint.get("status", None)
wildcard = check_wildcard(domain, options)
if (wildcard['wild']) and (wildcard['wild'] == wildcard['itself']):
matched_wild = True
else:
matched_wild = False
# If it matches a wildcard domain, and the status code we found was non-200,
# the signal-to-noise is just too low to include it.
if matched_wild and (not str(status).startswith('2')):
logging.debug("\tSkipping, wildcard DNS match with %i status code." % status)
return None
yield [
base_metadata,
redirected_external,
redirected_subdomain,
status_code,
matched_wild
]
def scan(domain, options):
logging.debug("[%s][tls]" % domain)
# If inspection data exists, check to see if we can skip.
inspection = utils.data_for(domain, "inspect")
if inspection and (not inspection.get("support_https")):
logging.debug("\tSkipping, HTTPS not supported in inspection.")
return None
else:
# cache reformatted JSON from ssllabs
cache = utils.cache_path(domain, "tls")
force = options.get("force", False)
if (force is False) and (os.path.exists(cache)):
logging.debug("\tCached.")
raw = open(cache).read()
data = json.loads(raw)
if data.get("invalid"):
return None
else:
logging.debug("\t %s %s" % (command, domain))
usecache = str(not force).lower()
if options.get("debug"):
cmd = [command, "--usecache=%s" % usecache, "--verbosity=debug", domain]
else:
cmd = [command, "--usecache=%s" % usecache, "--quiet", domain]
raw = utils.scan(cmd)
if raw:
data = json.loads(raw)
# we only give ssllabs-scan one at a time,
# so we can de-pluralize this
data = data[0]
# if SSL Labs had an error hitting the site, cache this
# as an invalid entry.
if data["status"] == "ERROR":
utils.write(utils.invalid(data), cache)
return None
utils.write(utils.json_for(data), cache)
else:
return None
# raise Exception("Invalid data from ssllabs-scan: %s" % raw)
# can return multiple rows, one for each 'endpoint'
for endpoint in data["endpoints"]:
# this meant it couldn't connect to the endpoint
if not endpoint.get("grade"):
continue
sslv3 = False
tlsv12 = False
for protocol in endpoint["details"]["protocols"]:
if (protocol["name"] == "SSL") and (protocol["version"] == "3.0"):
sslv3 = True
if (protocol["name"] == "TLS") and (protocol["version"] == "1.2"):
tlsv12 = True
spdy = False
h2 = False
npn = endpoint["details"].get("npnProtocols", None)
if npn:
spdy = "spdy" in npn
h2 = "h2-" in npn
yield [
endpoint["grade"],
endpoint["details"]["cert"]["sigAlg"],
endpoint["details"]["key"]["alg"],
endpoint["details"]["key"]["size"],
endpoint["details"]["forwardSecrecy"],
endpoint["details"]["ocspStapling"],
endpoint["details"].get("fallbackScsv", "N/A"),
endpoint["details"]["supportsRc4"],
sslv3,
tlsv12,
spdy,
endpoint["details"]["sniRequired"],
h2,
]
def scan(domain, options):
logging.debug("[%s][tls]" % domain)
# If inspection data exists, check to see if we can skip.
inspection = utils.data_for(domain, "inspect")
if inspection and (not inspection.get("support_https")):
logging.debug("\tSkipping, HTTPS not supported in inspection.")
yield None
else:
# cache reformatted JSON from ssllabs
cache = utils.cache_path(domain, "tls")
force = options.get("force", False)
if (force is False) and (os.path.exists(cache)):
logging.debug("\tCached.")
raw = open(cache).read()
data = json.loads(raw)
if data.get('invalid'):
return None
else:
logging.debug("\t %s %s" % (command, domain))
usecache = str(not force).lower()
if options.get("debug"):
cmd = [command, "--usecache=%s" % usecache,
"--verbosity=debug", domain]
else:
cmd = [command, "--usecache=%s" % usecache,
"--quiet", domain]
raw = utils.scan(cmd)
if raw:
data = json.loads(raw)
# we only give ssllabs-scan one at a time,
# so we can de-pluralize this
data = data[0]
# if SSL Labs had an error hitting the site, cache this
# as an invalid entry.
if data["status"] == "ERROR":
utils.write(utils.invalid(data), cache)
return None
utils.write(utils.json_for(data), cache)
else:
return None
# raise Exception("Invalid data from ssllabs-scan: %s" % raw)
# can return multiple rows, one for each 'endpoint'
for endpoint in data['endpoints']:
# this meant it couldn't connect to the endpoint
if not endpoint.get("grade"):
continue
sslv3 = False
tlsv12 = False
for protocol in endpoint['details']['protocols']:
if ((protocol['name'] == "SSL") and
(protocol['version'] == '3.0')):
sslv3 = True
if ((protocol['name'] == "TLS") and
(protocol['version'] == '1.2')):
tlsv12 = True
spdy = False
h2 = False
npn = endpoint['details'].get('npnProtocols', None)
if npn:
spdy = ("spdy" in npn)
h2 = ("h2-" in npn)
def ccs_map(n):
return {
-1: "N/A (Error)",
0: "N/A (Unknown)",
1: "No (not vulnerable)",
2: "No (not exploitable)",
3: "Yes"
}[n]
def fs_map(n):
return {
0: "0 - No",
1: "1 - Some",
2: "2 - Modern",
4: "3 - Robust"
}[n]
yield [
endpoint['grade'],
endpoint['details']['cert']['sigAlg'],
endpoint['details']['key']['alg'],
endpoint['details']['key']['size'],
fs_map(endpoint['details']['forwardSecrecy']),
endpoint['details']['ocspStapling'],
endpoint['details'].get('fallbackScsv', "N/A"),
endpoint['details'].get('freak'),
ccs_map(endpoint['details']['openSslCcs']),
sslv3,
tlsv12,
spdy,
endpoint['details']['sniRequired'],
h2
]
def scan(domain, options):
logging.debug("[%s][subdomains]" % domain)
base_original = utils.base_domain_for(domain)
sub_original = domain
base_metadata = domain_map.get(base_original, None)
if domain in exclude_list:
logging.debug("\tSkipping, excluded through manual review.")
return None
# This only looks at subdomains, remove second-level root's and www's.
if re.sub("^www.", "", domain) == base_original:
logging.debug("\tSkipping, second-level domain.")
return None
# If inspection data exists, check to see if we can skip.
inspection = utils.data_for(domain, "inspect")
if not inspection:
logging.debug("\tSkipping, wasn't inspected.")
return None
if not inspection.get("up"):
logging.debug("\tSkipping, subdomain wasn't up during inspection.")
return None
# Default to canonical endpoint, but if that didn't detect right, find the others
endpoint = inspection["endpoints"][inspection.get(
"canonical_protocol")]["root"]
protocol = inspection.get("canonical_protocol")
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["http"]["www"]
protocol = "http"
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["https"]["root"]
protocol = "https"
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["https"]["www"]
protocol = "https"
# this should have been the default default, but check anyway
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["http"]["root"]
protocol = "http"
# If it's a 0 status code, I guess it's down.
# If it's non-200, we filter out by default.
status = endpoint.get("status", None)
if status == 0:
logging.debug(
"\tSkipping, really down somehow, status code 0 for all.")
return None
# bad hostname for cert?
if (protocol == "https") and (endpoint.get("https_bad_name", False)
== True):
bad_cert_name = True
else:
bad_cert_name = False
# If the subdomain redirects anywhere, see if it redirects within the domain
if endpoint.get("redirect_to"):
sub_redirect = urllib.parse.urlparse(endpoint["redirect_to"]).hostname
sub_redirect = re.sub("^www.", "",
sub_redirect) # discount www redirects
base_redirect = utils.base_domain_for(sub_redirect)
redirected_external = base_original != base_redirect
redirected_subdomain = ((base_original == base_redirect)
and (sub_original != sub_redirect))
else:
redirected_external = False
redirected_subdomain = False
status_code = endpoint.get("status", None)
wildcard = check_wildcard(domain, options)
if (wildcard['wild']) and (wildcard['wild'] == wildcard['itself']):
matched_wild = True
else:
matched_wild = False
# If it matches a wildcard domain, and the status code we found was non-200,
# the signal-to-noise is just too low to include it.
if matched_wild and (not str(status).startswith('2')):
logging.debug("\tSkipping, wildcard DNS match with %i status code." %
status)
return None
yield [
base_metadata, redirected_external, redirected_subdomain, status_code,
matched_wild
]
def scan(domain, options):
logging.debug("[%s][tls]" % domain)
# If inspection data exists, check to see if we can skip.
inspection = utils.data_for(domain, "inspect")
if inspection and (not inspection.get("support_https")):
logging.debug("\tSkipping, HTTPS not supported in inspection.")
return None
else:
# cache reformatted JSON from ssllabs
cache = utils.cache_path(domain, "tls")
force = options.get("force", False)
if (force is False) and (os.path.exists(cache)):
logging.debug("\tCached.")
raw = open(cache).read()
data = json.loads(raw)
if data.get('invalid'):
return None
else:
logging.debug("\t %s %s" % (command, domain))
usecache = str(not force).lower()
if options.get("debug"):
cmd = [
command,
"--usecache=%s" % usecache, "--verbosity=debug", domain
]
else:
cmd = [command, "--usecache=%s" % usecache, "--quiet", domain]
raw = utils.scan(cmd)
if raw:
data = json.loads(raw)
# we only give ssllabs-scan one at a time,
# so we can de-pluralize this
data = data[0]
# if SSL Labs had an error hitting the site, cache this
# as an invalid entry.
if data["status"] == "ERROR":
utils.write(utils.invalid(data), cache)
return None
utils.write(utils.json_for(data), cache)
else:
return None
# raise Exception("Invalid data from ssllabs-scan: %s" % raw)
# can return multiple rows, one for each 'endpoint'
for endpoint in data['endpoints']:
# this meant it couldn't connect to the endpoint
if not endpoint.get("grade"):
continue
sslv3 = False
tlsv12 = False
for protocol in endpoint['details']['protocols']:
if ((protocol['name'] == "SSL")
and (protocol['version'] == '3.0')):
sslv3 = True
if ((protocol['name'] == "TLS")
and (protocol['version'] == '1.2')):
tlsv12 = True
spdy = False
h2 = False
npn = endpoint['details'].get('npnProtocols', None)
if npn:
spdy = ("spdy" in npn)
h2 = ("h2-" in npn)
yield [
endpoint['grade'], endpoint['details']['cert']['sigAlg'],
endpoint['details']['key']['alg'],
endpoint['details']['key']['size'],
endpoint['details']['forwardSecrecy'],
endpoint['details']['ocspStapling'], endpoint['details'].get(
'fallbackScsv', "N/A"), endpoint['details']['supportsRc4'],
sslv3, tlsv12, spdy, endpoint['details']['sniRequired'], h2
]
def scan(domain, options):
logging.debug("[%s][subdomains]" % domain)
base_original = utils.base_domain_for(domain)
sub_original = domain
base_metadata = domain_map.get(base_original, None)
if domain in exclude_list:
logging.debug("\tSkipping, excluded through manual review.")
return None
# This only looks at subdomains, remove second-level root's and www's.
if re.sub("^www.", "", domain) == base_original:
logging.debug("\tSkipping, second-level domain.")
return None
# If inspection data exists, check to see if we can skip.
inspection = utils.data_for(domain, "inspect")
if not inspection:
logging.debug("\tSkipping, wasn't inspected.")
return None
if not inspection.get("up"):
logging.debug("\tSkipping, subdomain wasn't up during inspection.")
return None
# Default to canonical endpoint, but if that didn't detect right, find the others
endpoint = inspection["endpoints"][inspection.get("canonical_protocol")]["root"]
protocol = inspection.get("canonical_protocol")
prefix = inspection.get("canonical_endpoint")
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["http"]["www"]
protocol = "http"
prefix = "www"
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["https"]["root"]
protocol = "https"
prefix = "root"
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["https"]["www"]
protocol = "https"
prefix = "www"
# this should have been the default default, but check anyway
if endpoint.get("status", None) == 0:
endpoint = inspection["endpoints"]["http"]["root"]
protocol = "http"
prefix = "root"
# If it's a 0 status code, I guess it's down.
# If it's non-200, we filter out by default.
status = endpoint.get("status", None)
if prefix == "root":
real_prefix = ""
else:
real_prefix = "www."
if status == 0:
logging.debug("\tSkipping, really down somehow, status code 0 for all.")
return None
# bad hostname for cert?
if (protocol == "https") and (endpoint.get("https_bad_name", False) is True):
bad_cert_name = True # nopep8
else:
bad_cert_name = False # nopep8
# If the subdomain redirects anywhere, see if it redirects within the domain
if endpoint.get("redirect_to"):
sub_redirect = urllib.parse.urlparse(endpoint["redirect_to"]).hostname
sub_redirect = re.sub("^www.", "", sub_redirect) # discount www redirects
base_redirect = utils.base_domain_for(sub_redirect)
redirected_external = base_original != base_redirect
redirected_subdomain = (
(base_original == base_redirect) and
(sub_original != sub_redirect)
)
else:
redirected_external = False
redirected_subdomain = False
status_code = endpoint.get("status", None)
# Hit the network for DNS reads and content
endpoint_url = "%s://%s%s" % (protocol, real_prefix, sub_original)
network = network_check(sub_original, endpoint_url, options)
matched_wild = network['matched_wild']
content = network['content']
if content:
try:
hashed = hashlib.sha256(bytearray(content, "utf-8")).hexdigest()
except:
hashed = None
else:
hashed = None
# If it matches a wildcard domain, and the status code we found was non-200,
# the signal-to-noise is just too low to include it.
if matched_wild and (not str(status).startswith('2')):
logging.debug("\tSkipping, wildcard DNS match with %i status code." % status)
return None
yield [
base_metadata,
redirected_external,
redirected_subdomain,
status_code,
matched_wild,
hashed
]
