def scan(domain, options):
logging.debug("[%s][pshtt]" % domain)
# cache output from pshtt
cache_pshtt = utils.cache_path(domain, "pshtt", ext="json")
force = options.get("force", False)
data = None
if (force is False) and (os.path.exists(cache_pshtt)):
logging.debug("\tCached.")
raw = open(cache_pshtt).read()
data = json.loads(raw)
if (data.__class__ is dict) and data.get('invalid'):
return None
else:
logging.debug("\t %s %s" % (command, domain))
flags = "--json --user-agent \"%s\" --timeout %i --preload-cache %s" % (
user_agent, timeout, preload_cache)
# Only useful when debugging interaction between projects.
# flags = "%s --debug" % flags
# Give the Python shell environment a pyenv environment.
pyenv_init = "eval \"$(pyenv init -)\" && pyenv shell %s" % pyenv_version
# Really un-ideal, but calling out to Python2 from Python 3 is a nightmare.
# I don't think this tool's threat model includes untrusted CSV, either.
raw = utils.unsafe_execute("%s && %s %s %s" %
(pyenv_init, command, domain, flags))
if not raw:
utils.write(utils.invalid({}), cache_pshtt)
logging.warn("\tBad news scanning, sorry!")
return None
data = json.loads(raw)
utils.write(utils.json_for(data), utils.cache_path(domain, "pshtt"))
# pshtt scanner uses JSON arrays, even for single items
data = data[0]
row = []
for field in headers:
value = data[field]
# TODO: Fix this upstream
if (field != "HSTS Header") and (field != "HSTS Max Age") and (
field != "Redirect To"):
if value is None:
value = False
row.append(value)
yield row
def scan(domain, options):
logging.debug("[%s][pshtt]" % domain)
# cache output from pshtt
cache_pshtt = utils.cache_path(domain, "pshtt", ext="json")
force = options.get("force", False)
data = None
if (force is False) and (os.path.exists(cache_pshtt)):
logging.debug("\tCached.")
raw = open(cache_pshtt).read()
data = json.loads(raw)
if (data.__class__ is dict) and data.get('invalid'):
return None
else:
logging.debug("\t %s %s" % (command, domain))
flags = "--json --user-agent \"%s\" --timeout %i --preload-cache %s" % (user_agent, timeout, preload_cache)
# Only useful when debugging interaction between projects.
# flags = "%s --debug" % flags
# Give the Python shell environment a pyenv environment.
pyenv_init = "eval \"$(pyenv init -)\" && pyenv shell %s" % pyenv_version
# Really un-ideal, but calling out to Python2 from Python 3 is a nightmare.
# I don't think this tool's threat model includes untrusted CSV, either.
raw = utils.unsafe_execute("%s && %s %s %s" % (pyenv_init, command, domain, flags))
if not raw:
utils.write(utils.invalid({}), cache_pshtt)
logging.warn("\tBad news scanning, sorry!")
return None
data = json.loads(raw)
utils.write(utils.json_for(data), utils.cache_path(domain, "pshtt"))
# pshtt scanner uses JSON arrays, even for single items
data = data[0]
row = []
for field in headers:
value = data[field]
# TODO: Fix this upstream
if (field != "HSTS Header") and (field != "HSTS Max Age") and (field != "Redirect To"):
if value is None:
value = False
row.append(value)
yield row
def check_wildcard(subdomain, options):
wildcard = wildcard_for(subdomain)
cache = utils.cache_path(subdomain, "subdomains")
if (options.get("force", False) is False) and (os.path.exists(cache)):
logging.debug("\tDNS info cached.")
raw = open(cache).read()
data = json.loads(raw)
else:
logging.debug("\t dig +short '%s'" % wildcard)
raw_wild = utils.unsafe_execute("dig +short '%s'" % wildcard)
if raw_wild == "":
raw_wild = None
raw_self = None
else:
logging.debug("\t dig +short '%s'" % subdomain)
raw_self = utils.unsafe_execute("dig +short '%s'" % subdomain)
if raw_wild:
parsed_wild = raw_wild.split("\n")
parsed_wild.sort()
else:
parsed_wild = None
if raw_self:
parsed_self = raw_self.split("\n")
parsed_self.sort()
else:
parsed_self = None
data = {'response': {'wild': parsed_wild, 'itself': parsed_self}}
utils.write(
utils.json_for(data),
cache
)
return data['response']
def check_wildcard(subdomain, options):
wildcard = wildcard_for(subdomain)
cache = utils.cache_path(subdomain, "subdomains")
if (options.get("force", False) is False) and (os.path.exists(cache)):
logging.debug("\tDNS info cached.")
raw = open(cache).read()
data = json.loads(raw)
else:
logging.debug("\t dig +short '%s'" % wildcard)
raw_wild = utils.unsafe_execute("dig +short '%s'" % wildcard)
if raw_wild == "":
raw_wild = None
raw_self = None
else:
logging.debug("\t dig +short '%s'" % subdomain)
raw_self = utils.unsafe_execute("dig +short '%s'" % subdomain)
if raw_wild:
parsed_wild = raw_wild.split("\n")
parsed_wild.sort()
else:
parsed_wild = None
if raw_self:
parsed_self = raw_self.split("\n")
parsed_self.sort()
else:
parsed_self = None
data = {'response': {'wild': parsed_wild, 'itself': parsed_self}}
utils.write(utils.json_for(data), cache)
return data['response']
def network_check(subdomain, endpoint, options):
cache = utils.cache_path(subdomain, "subdomains")
wildcard = wildcard_for(subdomain)
if (options.get("force", False) is False) and (os.path.exists(cache)):
logging.debug("\tDNS and content cached.")
raw = open(cache).read()
data = json.loads(raw)
# Hit DNS and HTTP.
else:
# HTTP content: just use curl.
#
# Turn on --insecure because we want to see the content even at sites
# where the certificate isn't right or proper.
logging.debug("\t curl --silent --insecure %s" % endpoint)
content = utils.scan(["curl", "--silent", "--insecure", endpoint])
# DNS content: just use dig.
#
# Not awesome - uses an unsafe shell execution of `dig` to look up DNS,
# as I couldn't figure out a way to get "+short" to play nice with
# the more secure execution methods available to me. Since this system
# isn't expected to process untrusted input, this should be okay.
logging.debug("\t dig +short '%s'" % wildcard)
raw_wild = utils.unsafe_execute("dig +short '%s'" % wildcard)
if raw_wild == "":
raw_wild = None
raw_self = None
else:
logging.debug("\t dig +short '%s'" % subdomain)
raw_self = utils.unsafe_execute("dig +short '%s'" % subdomain)
if raw_wild:
parsed_wild = raw_wild.split("\n")
parsed_wild.sort()
else:
parsed_wild = None
if raw_self:
parsed_self = raw_self.split("\n")
parsed_self.sort()
else:
parsed_self = None
# Cache HTTP and DNS data to disk.
data = {
'response': {
'content': content,
'wildcard_dns': parsed_wild,
'self_dns': parsed_self
}
}
if (parsed_wild) and (parsed_wild == parsed_self):
data['response']['matched_wild'] = True
else:
data['response']['matched_wild'] = False
utils.write(utils.json_for(data), cache)
return data['response']
def scan(domain, options):
logging.debug("[%s][sslyze]" % domain)
# Optional: skip domains which don't support HTTPS in prior inspection
if utils.domain_doesnt_support_https(domain):
logging.debug("\tSkipping, HTTPS not supported in inspection.")
return None
# Optional: if pshtt data says canonical endpoint uses www and this domain
# doesn't have it, add it.
if utils.domain_uses_www(domain):
scan_domain = "www.%s" % domain
else:
scan_domain = domain
# cache XML from sslyze
cache_xml = utils.cache_path(domain, "sslyze", ext="xml")
# because sslyze manages its own output (can't yet print to stdout),
# we have to mkdir_p the path ourselves
utils.mkdir_p(os.path.dirname(cache_xml))
force = options.get("force", False)
if (force is False) and (os.path.exists(cache_xml)):
logging.debug("\tCached.")
xml = open(cache_xml).read()
else:
logging.debug("\t %s %s" % (command, scan_domain))
# use scan_domain (possibly www-prefixed) to do actual scan
# Give the Python shell environment a pyenv environment.
pyenv_init = "eval \"$(pyenv init -)\" && pyenv shell %s" % pyenv_version
# Really un-ideal, but calling out to Python2 from Python 3 is a nightmare.
# I don't think this tool's threat model includes untrusted CSV, either.
raw = utils.unsafe_execute(
"%s && %s --regular --quiet %s --xml_out=%s" %
(pyenv_init, command, scan_domain, cache_xml))
if raw is None:
# TODO: save standard invalid XML data...?
logging.warn("\tBad news scanning, sorry!")
return None
xml = utils.scan(["cat", cache_xml])
if not xml:
logging.warn("\tBad news reading XML, sorry!")
return None
utils.write(xml, cache_xml)
data = parse_sslyze(xml)
if data is None:
logging.warn("\tNo valid target for scanning, couldn't connect.")
return None
utils.write(utils.json_for(data), utils.cache_path(domain, "sslyze"))
yield [
data['protocols']['sslv2'], data['protocols']['sslv3'],
data['protocols']['tlsv1.0'], data['protocols']['tlsv1.1'],
data['protocols']['tlsv1.2'], data['config'].get('any_dhe'),
data['config'].get('all_dhe'), data['config'].get('weakest_dh'),
data['config'].get('any_rc4'), data['config'].get('all_rc4'),
data['config'].get('ocsp_stapling'), data['certs'].get('key_type'),
data['certs'].get('key_length'), data['certs'].get('leaf_signature'),
data['certs'].get('any_sha1'), data['certs'].get('not_before'),
data['certs'].get('not_after'), data['certs'].get('served_issuer'),
data.get('errors')
]
def scan(domain, options):
logging.debug("[%s][sslyze]" % domain)
# Optional: skip domains which don't support HTTPS in prior inspection
if utils.domain_doesnt_support_https(domain):
logging.debug("\tSkipping, HTTPS not supported in inspection.")
return None
# Optional: if pshtt data says canonical endpoint uses www and this domain
# doesn't have it, add it.
if utils.domain_uses_www(domain):
scan_domain = "www.%s" % domain
else:
scan_domain = domain
# cache XML from sslyze
cache_xml = utils.cache_path(domain, "sslyze", ext="xml")
# because sslyze manages its own output (can't yet print to stdout),
# we have to mkdir_p the path ourselves
utils.mkdir_p(os.path.dirname(cache_xml))
force = options.get("force", False)
if (force is False) and (os.path.exists(cache_xml)):
logging.debug("\tCached.")
xml = open(cache_xml).read()
else:
logging.debug("\t %s %s" % (command, scan_domain))
# use scan_domain (possibly www-prefixed) to do actual scan
# Give the Python shell environment a pyenv environment.
pyenv_init = "eval \"$(pyenv init -)\" && pyenv shell %s" % pyenv_version
# Really un-ideal, but calling out to Python2 from Python 3 is a nightmare.
# I don't think this tool's threat model includes untrusted CSV, either.
raw = utils.unsafe_execute("%s && %s --regular --quiet %s --xml_out=%s" % (pyenv_init, command, scan_domain, cache_xml))
if raw is None:
# TODO: save standard invalid XML data...?
logging.warn("\tBad news scanning, sorry!")
return None
xml = utils.scan(["cat", cache_xml])
if not xml:
logging.warn("\tBad news reading XML, sorry!")
return None
utils.write(xml, cache_xml)
data = parse_sslyze(xml)
if data is None:
logging.warn("\tNo valid target for scanning, couldn't connect.")
return None
utils.write(utils.json_for(data), utils.cache_path(domain, "sslyze"))
yield [
data['protocols']['sslv2'], data['protocols']['sslv3'],
data['protocols']['tlsv1.0'], data['protocols']['tlsv1.1'],
data['protocols']['tlsv1.2'],
data['config'].get('any_dhe'), data['config'].get('all_dhe'),
data['config'].get('weakest_dh'),
data['config'].get('any_rc4'), data['config'].get('all_rc4'),
data['config'].get('ocsp_stapling'),
data['certs'].get('key_type'), data['certs'].get('key_length'),
data['certs'].get('leaf_signature'), data['certs'].get('any_sha1'),
data['certs'].get('not_before'), data['certs'].get('not_after'),
data['certs'].get('served_issuer'),
data.get('errors')
]
def network_check(subdomain, endpoint, options):
cache = utils.cache_path(subdomain, "subdomains")
wildcard = wildcard_for(subdomain)
if (options.get("force", False) is False) and (os.path.exists(cache)):
logging.debug("\tDNS and content cached.")
raw = open(cache).read()
data = json.loads(raw)
# Hit DNS and HTTP.
else:
# HTTP content: just use curl.
#
# Turn on --insecure because we want to see the content even at sites
# where the certificate isn't right or proper.
logging.debug("\t curl --silent --insecure %s" % endpoint)
content = utils.scan(["curl", "--silent", "--insecure", endpoint])
# DNS content: just use dig.
#
# Not awesome - uses an unsafe shell execution of `dig` to look up DNS,
# as I couldn't figure out a way to get "+short" to play nice with
# the more secure execution methods available to me. Since this system
# isn't expected to process untrusted input, this should be okay.
logging.debug("\t dig +short '%s'" % wildcard)
raw_wild = utils.unsafe_execute("dig +short '%s'" % wildcard)
if raw_wild == "":
raw_wild = None
raw_self = None
else:
logging.debug("\t dig +short '%s'" % subdomain)
raw_self = utils.unsafe_execute("dig +short '%s'" % subdomain)
if raw_wild:
parsed_wild = raw_wild.split("\n")
parsed_wild.sort()
else:
parsed_wild = None
if raw_self:
parsed_self = raw_self.split("\n")
parsed_self.sort()
else:
parsed_self = None
# Cache HTTP and DNS data to disk.
data = {'response': {
'content': content,
'wildcard_dns': parsed_wild,
'self_dns': parsed_self
}}
if (parsed_wild) and (parsed_wild == parsed_self):
data['response']['matched_wild'] = True
else:
data['response']['matched_wild'] = False
utils.write(utils.json_for(data), cache)
return data['response']
