def scan(domain, options): logging.debug("[%s][pageload]" % domain) inspection = utils.data_for(domain, "inspect") # If we have data from inspect, skip if it's not a live domain. if inspection and (not inspection.get("up")): logging.debug("\tSkipping, domain not reachable during inspection.") return None # If we have data from inspect, skip if it's just a redirector. if inspection and (inspection.get("redirect") is True): logging.debug( "\tSkipping, domain seen as just a redirector during inspection.") return None # phantomas needs a URL, not just a domain. if not (domain.startswith('http://') or domain.startswith('https://')): # If we have data from inspect, use the canonical endpoint. if inspection and inspection.get("canonical"): url = inspection.get("canonical") # Otherwise, well, whatever. else: url = 'http://' + domain else: url = domain # We'll cache prettified JSON from the output. cache = utils.cache_path(domain, "pageload") # If we've got it cached, use that. if (options.get("force", False) is False) and (os.path.exists(cache)): logging.debug("\tCached.") raw = open(cache).read() data = json.loads(raw) if data.get('invalid'): return None # If no cache, or we should run anyway, do the scan. else: logging.debug("\t %s %s --reporter=json --ignore-ssl-errors" % (command, url)) raw = utils.scan( [command, url, "--reporter=json", "--ignore-ssl-errors"]) if not raw: utils.write(utils.invalid({}), cache) return None # It had better be JSON, which we can cache in prettified form. data = json.loads(raw) utils.write(utils.json_for(data), cache) yield [data['metrics'][metric] for metric in interesting_metrics]
def scan(domain, options): logging.debug("[%s][pageload]" % domain) inspection = utils.data_for(domain, "inspect") # If we have data from inspect, skip if it's not a live domain. if inspection and (not inspection.get("up")): logging.debug("\tSkipping, domain not reachable during inspection.") return None # If we have data from inspect, skip if it's just a redirector. if inspection and (inspection.get("redirect") is True): logging.debug("\tSkipping, domain seen as just a redirector during inspection.") return None # phantomas needs a URL, not just a domain. if not (domain.startswith('http://') or domain.startswith('https://')): # If we have data from inspect, use the canonical endpoint. if inspection and inspection.get("canonical"): url = inspection.get("canonical") # Otherwise, well, whatever. else: url = 'http://' + domain else: url = domain # We'll cache prettified JSON from the output. cache = utils.cache_path(domain, "pageload") # If we've got it cached, use that. if (options.get("force", False) is False) and (os.path.exists(cache)): logging.debug("\tCached.") raw = open(cache).read() data = json.loads(raw) if data.get('invalid'): return None # If no cache, or we should run anyway, do the scan. else: logging.debug("\t %s %s --reporter=json --ignore-ssl-errors" % (command, url)) raw = utils.scan([command, url, "--reporter=json", "--ignore-ssl-errors"]) if not raw: utils.write(utils.invalid({}), cache) return None # It had better be JSON, which we can cache in prettified form. data = json.loads(raw) utils.write(utils.json_for(data), cache) yield [data['metrics'][metric] for metric in interesting_metrics]
def scan(domain, options): logging.debug("[%s][sslyze]" % domain) # Optional: skip domains which don't support HTTPS in prior inspection inspection = utils.data_for(domain, "inspect") if inspection and (not inspection.get("support_https")): logging.debug("\tSkipping, HTTPS not supported in inspection.") return None # Optional: if inspect data says canonical endpoint uses www and this domain # doesn't have it, add it. if inspection and (inspection.get("canonical_endpoint") == "www") and (not domain.startswith("www.")): scan_domain = "www.%s" % domain else: scan_domain = domain # cache XML from sslyze cache_xml = utils.cache_path(domain, "sslyze", ext="xml") # because sslyze manages its own output (can't yet print to stdout), # we have to mkdir_p the path ourselves utils.mkdir_p(os.path.dirname(cache_xml)) force = options.get("force", False) if (force is False) and (os.path.exists(cache_xml)): logging.debug("\tCached.") xml = open(cache_xml).read() else: logging.debug("\t %s %s" % (command, domain)) # use scan_domain (possibly www-prefixed) to do actual scan raw = utils.scan([command, "--regular", "--quiet", scan_domain, "--xml_out=%s" % cache_xml], env=command_env) if raw is None: # TODO: save standard invalid XML data...? logging.warn("\tBad news scanning, sorry!") return None xml = utils.scan(["cat", cache_xml]) if not xml: logging.warn("\tBad news reading XML, sorry!") return None utils.write(xml, cache_xml) data = parse_sslyze(xml) if data is None: logging.warn("\tNo valid target for scanning, couldn't connect.") return None utils.write(utils.json_for(data), utils.cache_path(domain, "sslyze")) yield [ data['protocols']['sslv2'], data['protocols']['sslv3'], data['protocols']['tlsv1.0'], data['protocols']['tlsv1.1'], data['protocols']['tlsv1.2'], data['config'].get('any_dhe'), data['config'].get('all_dhe'), data['config'].get('weakest_dh'), data['config'].get('any_rc4'), data['config'].get('all_rc4'), data['config'].get('ocsp_stapling'), data['certs'].get('key_type'), data['certs'].get('key_length'), data['certs'].get('leaf_signature'), data['certs'].get('any_sha1'), data['certs'].get('not_before'), data['certs'].get('not_after'), data['certs'].get('served_issuer'), data.get('errors') ]
def scan(domain, options): logging.debug("[%s][subdomains]" % domain) base_original = utils.base_domain_for(domain) sub_original = domain base_metadata = domain_map.get(base_original, None) if domain in exclude_list: logging.debug("\tSkipping, excluded through manual review.") return None # This only looks at subdomains, remove second-level root's and www's. if re.sub("^www.", "", domain) == base_original: logging.debug("\tSkipping, second-level domain.") return None # If inspection data exists, check to see if we can skip. inspection = utils.data_for(domain, "inspect") if not inspection: logging.debug("\tSkipping, wasn't inspected.") return None if not inspection.get("up"): logging.debug("\tSkipping, subdomain wasn't up during inspection.") return None # Default to canonical endpoint, but if that didn't detect right, find the others endpoint = inspection["endpoints"][inspection.get( "canonical_protocol")]["root"] protocol = inspection.get("canonical_protocol") prefix = inspection.get("canonical_endpoint") if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["http"]["www"] protocol = "http" prefix = "www" if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["https"]["root"] protocol = "https" prefix = "root" if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["https"]["www"] protocol = "https" prefix = "www" # this should have been the default default, but check anyway if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["http"]["root"] protocol = "http" prefix = "root" # If it's a 0 status code, I guess it's down. # If it's non-200, we filter out by default. status = endpoint.get("status", None) if prefix == "root": real_prefix = "" else: real_prefix = "www." if status == 0: logging.debug( "\tSkipping, really down somehow, status code 0 for all.") return None # If the subdomain redirects anywhere, see if it redirects within the domain if endpoint.get("redirect_to"): sub_redirect = urllib.parse.urlparse(endpoint["redirect_to"]).hostname sub_redirect = re.sub("^www.", "", sub_redirect) # discount www redirects base_redirect = utils.base_domain_for(sub_redirect) redirected_external = base_original != base_redirect redirected_subdomain = ((base_original == base_redirect) and (sub_original != sub_redirect)) else: redirected_external = False redirected_subdomain = False status_code = endpoint.get("status", None) # Hit the network for DNS reads and content endpoint_url = "%s://%s%s" % (protocol, real_prefix, sub_original) network = network_check(sub_original, endpoint_url, options) matched_wild = network['matched_wild'] content = network['content'] if content: try: hashed = hashlib.sha256(bytearray(content, "utf-8")).hexdigest() except: hashed = None else: hashed = None # If it matches a wildcard domain, and the status code we found was non-200, # the signal-to-noise is just too low to include it. if matched_wild and (not str(status).startswith('2')): logging.debug("\tSkipping, wildcard DNS match with %i status code." % status) return None yield [ base_metadata, redirected_external, redirected_subdomain, status_code, matched_wild, hashed ]
def scan(domain, options): logging.debug("[%s][subdomains]" % domain) base_original = utils.base_domain_for(domain) sub_original = domain base_metadata = domain_map.get(base_original, None) if domain in exclude_list: logging.debug("\tSkipping, excluded through manual review.") return None # This only looks at subdomains, remove second-level root's and www's. if re.sub("^www.", "", domain) == base_original: logging.debug("\tSkipping, second-level domain.") return None # If inspection data exists, check to see if we can skip. inspection = utils.data_for(domain, "inspect") if not inspection: logging.debug("\tSkipping, wasn't inspected.") return None if not inspection.get("up"): logging.debug("\tSkipping, subdomain wasn't up during inspection.") return None # Default to canonical endpoint, but if that didn't detect right, find the others endpoint = inspection["endpoints"][inspection.get("canonical_protocol")]["root"] protocol = inspection.get("canonical_protocol") if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["http"]["www"] protocol = "http" if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["https"]["root"] protocol = "https" if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["https"]["www"] protocol = "https" # this should have been the default default, but check anyway if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["http"]["root"] protocol = "http" # If it's a 0 status code, I guess it's down. # If it's non-200, we filter out by default. status = endpoint.get("status", None) if status == 0: logging.debug("\tSkipping, really down somehow, status code 0 for all.") return None # bad hostname for cert? if (protocol == "https") and (endpoint.get("https_bad_name", False) == True): bad_cert_name = True else: bad_cert_name = False # If the subdomain redirects anywhere, see if it redirects within the domain if endpoint.get("redirect_to"): sub_redirect = urllib.parse.urlparse(endpoint["redirect_to"]).hostname sub_redirect = re.sub("^www.", "", sub_redirect) # discount www redirects base_redirect = utils.base_domain_for(sub_redirect) redirected_external = base_original != base_redirect redirected_subdomain = ( (base_original == base_redirect) and (sub_original != sub_redirect) ) else: redirected_external = False redirected_subdomain = False status_code = endpoint.get("status", None) wildcard = check_wildcard(domain, options) if (wildcard['wild']) and (wildcard['wild'] == wildcard['itself']): matched_wild = True else: matched_wild = False # If it matches a wildcard domain, and the status code we found was non-200, # the signal-to-noise is just too low to include it. if matched_wild and (not str(status).startswith('2')): logging.debug("\tSkipping, wildcard DNS match with %i status code." % status) return None yield [ base_metadata, redirected_external, redirected_subdomain, status_code, matched_wild ]
def scan(domain, options): logging.debug("[%s][tls]" % domain) # If inspection data exists, check to see if we can skip. inspection = utils.data_for(domain, "inspect") if inspection and (not inspection.get("support_https")): logging.debug("\tSkipping, HTTPS not supported in inspection.") return None else: # cache reformatted JSON from ssllabs cache = utils.cache_path(domain, "tls") force = options.get("force", False) if (force is False) and (os.path.exists(cache)): logging.debug("\tCached.") raw = open(cache).read() data = json.loads(raw) if data.get("invalid"): return None else: logging.debug("\t %s %s" % (command, domain)) usecache = str(not force).lower() if options.get("debug"): cmd = [command, "--usecache=%s" % usecache, "--verbosity=debug", domain] else: cmd = [command, "--usecache=%s" % usecache, "--quiet", domain] raw = utils.scan(cmd) if raw: data = json.loads(raw) # we only give ssllabs-scan one at a time, # so we can de-pluralize this data = data[0] # if SSL Labs had an error hitting the site, cache this # as an invalid entry. if data["status"] == "ERROR": utils.write(utils.invalid(data), cache) return None utils.write(utils.json_for(data), cache) else: return None # raise Exception("Invalid data from ssllabs-scan: %s" % raw) # can return multiple rows, one for each 'endpoint' for endpoint in data["endpoints"]: # this meant it couldn't connect to the endpoint if not endpoint.get("grade"): continue sslv3 = False tlsv12 = False for protocol in endpoint["details"]["protocols"]: if (protocol["name"] == "SSL") and (protocol["version"] == "3.0"): sslv3 = True if (protocol["name"] == "TLS") and (protocol["version"] == "1.2"): tlsv12 = True spdy = False h2 = False npn = endpoint["details"].get("npnProtocols", None) if npn: spdy = "spdy" in npn h2 = "h2-" in npn yield [ endpoint["grade"], endpoint["details"]["cert"]["sigAlg"], endpoint["details"]["key"]["alg"], endpoint["details"]["key"]["size"], endpoint["details"]["forwardSecrecy"], endpoint["details"]["ocspStapling"], endpoint["details"].get("fallbackScsv", "N/A"), endpoint["details"]["supportsRc4"], sslv3, tlsv12, spdy, endpoint["details"]["sniRequired"], h2, ]
def scan(domain, options): logging.debug("[%s][tls]" % domain) # If inspection data exists, check to see if we can skip. inspection = utils.data_for(domain, "inspect") if inspection and (not inspection.get("support_https")): logging.debug("\tSkipping, HTTPS not supported in inspection.") yield None else: # cache reformatted JSON from ssllabs cache = utils.cache_path(domain, "tls") force = options.get("force", False) if (force is False) and (os.path.exists(cache)): logging.debug("\tCached.") raw = open(cache).read() data = json.loads(raw) if data.get('invalid'): return None else: logging.debug("\t %s %s" % (command, domain)) usecache = str(not force).lower() if options.get("debug"): cmd = [command, "--usecache=%s" % usecache, "--verbosity=debug", domain] else: cmd = [command, "--usecache=%s" % usecache, "--quiet", domain] raw = utils.scan(cmd) if raw: data = json.loads(raw) # we only give ssllabs-scan one at a time, # so we can de-pluralize this data = data[0] # if SSL Labs had an error hitting the site, cache this # as an invalid entry. if data["status"] == "ERROR": utils.write(utils.invalid(data), cache) return None utils.write(utils.json_for(data), cache) else: return None # raise Exception("Invalid data from ssllabs-scan: %s" % raw) # can return multiple rows, one for each 'endpoint' for endpoint in data['endpoints']: # this meant it couldn't connect to the endpoint if not endpoint.get("grade"): continue sslv3 = False tlsv12 = False for protocol in endpoint['details']['protocols']: if ((protocol['name'] == "SSL") and (protocol['version'] == '3.0')): sslv3 = True if ((protocol['name'] == "TLS") and (protocol['version'] == '1.2')): tlsv12 = True spdy = False h2 = False npn = endpoint['details'].get('npnProtocols', None) if npn: spdy = ("spdy" in npn) h2 = ("h2-" in npn) def ccs_map(n): return { -1: "N/A (Error)", 0: "N/A (Unknown)", 1: "No (not vulnerable)", 2: "No (not exploitable)", 3: "Yes" }[n] def fs_map(n): return { 0: "0 - No", 1: "1 - Some", 2: "2 - Modern", 4: "3 - Robust" }[n] yield [ endpoint['grade'], endpoint['details']['cert']['sigAlg'], endpoint['details']['key']['alg'], endpoint['details']['key']['size'], fs_map(endpoint['details']['forwardSecrecy']), endpoint['details']['ocspStapling'], endpoint['details'].get('fallbackScsv', "N/A"), endpoint['details'].get('freak'), ccs_map(endpoint['details']['openSslCcs']), sslv3, tlsv12, spdy, endpoint['details']['sniRequired'], h2 ]
def scan(domain, options): logging.debug("[%s][subdomains]" % domain) base_original = utils.base_domain_for(domain) sub_original = domain base_metadata = domain_map.get(base_original, None) if domain in exclude_list: logging.debug("\tSkipping, excluded through manual review.") return None # This only looks at subdomains, remove second-level root's and www's. if re.sub("^www.", "", domain) == base_original: logging.debug("\tSkipping, second-level domain.") return None # If inspection data exists, check to see if we can skip. inspection = utils.data_for(domain, "inspect") if not inspection: logging.debug("\tSkipping, wasn't inspected.") return None if not inspection.get("up"): logging.debug("\tSkipping, subdomain wasn't up during inspection.") return None # Default to canonical endpoint, but if that didn't detect right, find the others endpoint = inspection["endpoints"][inspection.get( "canonical_protocol")]["root"] protocol = inspection.get("canonical_protocol") if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["http"]["www"] protocol = "http" if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["https"]["root"] protocol = "https" if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["https"]["www"] protocol = "https" # this should have been the default default, but check anyway if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["http"]["root"] protocol = "http" # If it's a 0 status code, I guess it's down. # If it's non-200, we filter out by default. status = endpoint.get("status", None) if status == 0: logging.debug( "\tSkipping, really down somehow, status code 0 for all.") return None # bad hostname for cert? if (protocol == "https") and (endpoint.get("https_bad_name", False) == True): bad_cert_name = True else: bad_cert_name = False # If the subdomain redirects anywhere, see if it redirects within the domain if endpoint.get("redirect_to"): sub_redirect = urllib.parse.urlparse(endpoint["redirect_to"]).hostname sub_redirect = re.sub("^www.", "", sub_redirect) # discount www redirects base_redirect = utils.base_domain_for(sub_redirect) redirected_external = base_original != base_redirect redirected_subdomain = ((base_original == base_redirect) and (sub_original != sub_redirect)) else: redirected_external = False redirected_subdomain = False status_code = endpoint.get("status", None) wildcard = check_wildcard(domain, options) if (wildcard['wild']) and (wildcard['wild'] == wildcard['itself']): matched_wild = True else: matched_wild = False # If it matches a wildcard domain, and the status code we found was non-200, # the signal-to-noise is just too low to include it. if matched_wild and (not str(status).startswith('2')): logging.debug("\tSkipping, wildcard DNS match with %i status code." % status) return None yield [ base_metadata, redirected_external, redirected_subdomain, status_code, matched_wild ]
def scan(domain, options): logging.debug("[%s][tls]" % domain) # If inspection data exists, check to see if we can skip. inspection = utils.data_for(domain, "inspect") if inspection and (not inspection.get("support_https")): logging.debug("\tSkipping, HTTPS not supported in inspection.") return None else: # cache reformatted JSON from ssllabs cache = utils.cache_path(domain, "tls") force = options.get("force", False) if (force is False) and (os.path.exists(cache)): logging.debug("\tCached.") raw = open(cache).read() data = json.loads(raw) if data.get('invalid'): return None else: logging.debug("\t %s %s" % (command, domain)) usecache = str(not force).lower() if options.get("debug"): cmd = [ command, "--usecache=%s" % usecache, "--verbosity=debug", domain ] else: cmd = [command, "--usecache=%s" % usecache, "--quiet", domain] raw = utils.scan(cmd) if raw: data = json.loads(raw) # we only give ssllabs-scan one at a time, # so we can de-pluralize this data = data[0] # if SSL Labs had an error hitting the site, cache this # as an invalid entry. if data["status"] == "ERROR": utils.write(utils.invalid(data), cache) return None utils.write(utils.json_for(data), cache) else: return None # raise Exception("Invalid data from ssllabs-scan: %s" % raw) # can return multiple rows, one for each 'endpoint' for endpoint in data['endpoints']: # this meant it couldn't connect to the endpoint if not endpoint.get("grade"): continue sslv3 = False tlsv12 = False for protocol in endpoint['details']['protocols']: if ((protocol['name'] == "SSL") and (protocol['version'] == '3.0')): sslv3 = True if ((protocol['name'] == "TLS") and (protocol['version'] == '1.2')): tlsv12 = True spdy = False h2 = False npn = endpoint['details'].get('npnProtocols', None) if npn: spdy = ("spdy" in npn) h2 = ("h2-" in npn) yield [ endpoint['grade'], endpoint['details']['cert']['sigAlg'], endpoint['details']['key']['alg'], endpoint['details']['key']['size'], endpoint['details']['forwardSecrecy'], endpoint['details']['ocspStapling'], endpoint['details'].get( 'fallbackScsv', "N/A"), endpoint['details']['supportsRc4'], sslv3, tlsv12, spdy, endpoint['details']['sniRequired'], h2 ]
def scan(domain, options): logging.debug("[%s][subdomains]" % domain) base_original = utils.base_domain_for(domain) sub_original = domain base_metadata = domain_map.get(base_original, None) if domain in exclude_list: logging.debug("\tSkipping, excluded through manual review.") return None # This only looks at subdomains, remove second-level root's and www's. if re.sub("^www.", "", domain) == base_original: logging.debug("\tSkipping, second-level domain.") return None # If inspection data exists, check to see if we can skip. inspection = utils.data_for(domain, "inspect") if not inspection: logging.debug("\tSkipping, wasn't inspected.") return None if not inspection.get("up"): logging.debug("\tSkipping, subdomain wasn't up during inspection.") return None # Default to canonical endpoint, but if that didn't detect right, find the others endpoint = inspection["endpoints"][inspection.get("canonical_protocol")]["root"] protocol = inspection.get("canonical_protocol") prefix = inspection.get("canonical_endpoint") if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["http"]["www"] protocol = "http" prefix = "www" if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["https"]["root"] protocol = "https" prefix = "root" if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["https"]["www"] protocol = "https" prefix = "www" # this should have been the default default, but check anyway if endpoint.get("status", None) == 0: endpoint = inspection["endpoints"]["http"]["root"] protocol = "http" prefix = "root" # If it's a 0 status code, I guess it's down. # If it's non-200, we filter out by default. status = endpoint.get("status", None) if prefix == "root": real_prefix = "" else: real_prefix = "www." if status == 0: logging.debug("\tSkipping, really down somehow, status code 0 for all.") return None # bad hostname for cert? if (protocol == "https") and (endpoint.get("https_bad_name", False) is True): bad_cert_name = True # nopep8 else: bad_cert_name = False # nopep8 # If the subdomain redirects anywhere, see if it redirects within the domain if endpoint.get("redirect_to"): sub_redirect = urllib.parse.urlparse(endpoint["redirect_to"]).hostname sub_redirect = re.sub("^www.", "", sub_redirect) # discount www redirects base_redirect = utils.base_domain_for(sub_redirect) redirected_external = base_original != base_redirect redirected_subdomain = ( (base_original == base_redirect) and (sub_original != sub_redirect) ) else: redirected_external = False redirected_subdomain = False status_code = endpoint.get("status", None) # Hit the network for DNS reads and content endpoint_url = "%s://%s%s" % (protocol, real_prefix, sub_original) network = network_check(sub_original, endpoint_url, options) matched_wild = network['matched_wild'] content = network['content'] if content: try: hashed = hashlib.sha256(bytearray(content, "utf-8")).hexdigest() except: hashed = None else: hashed = None # If it matches a wildcard domain, and the status code we found was non-200, # the signal-to-noise is just too low to include it. if matched_wild and (not str(status).startswith('2')): logging.debug("\tSkipping, wildcard DNS match with %i status code." % status) return None yield [ base_metadata, redirected_external, redirected_subdomain, status_code, matched_wild, hashed ]