Example #1
0
def get_chrome_preload_list(options):

    preload_cache = utils.cache_single("preload-list.json")
    preload_json = None

    if (not options.get("force", False)) and os.path.exists(preload_cache):
        logging.debug("Using cached Chrome preload list.")
        preload_json = json.loads(open(preload_cache).read())
    else:
        logging.debug("Fetching Chrome preload list from source...")

        preload_list_url = 'https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json'
        preload_list_url_as_text = preload_list_url + '?format=text'
        with urllib.request.urlopen(preload_list_url_as_text) as response:
            raw = response.read()

        # To avoid parsing the contents of the file out of the source tree viewer's
        # HTML, we download it as a raw file. googlesource.com Base64-encodes the
        # file to avoid potential content injection issues, so we need to decode it
        # before using it. https://code.google.com/p/gitiles/issues/detail?id=7
        raw = base64.b64decode(raw).decode('utf-8')

        # The .json file contains '//' comments, which are not actually valid JSON,
        # and confuse Python's JSON decoder. Begone, foul comments!
        raw = ''.join([re.sub(r'^\s*//.*$', '', line)
                       for line in raw.splitlines()])

        preload_json = json.loads(raw)
        utils.write(utils.json_for(preload_json), preload_cache)

    return {entry['name'] for entry in preload_json['entries']}
Example #2
0
def get_chrome_preload_list(options):

    preload_cache = utils.cache_single("preload-list.json")
    preload_json = None

    if (not options.get("force", False)) and os.path.exists(preload_cache):
        logging.debug("Using cached Chrome preload list.")
        preload_json = json.loads(open(preload_cache).read())
    else:
        logging.debug("Fetching Chrome preload list from source...")

        preload_list_url = 'https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json'
        preload_list_url_as_text = preload_list_url + '?format=text'
        with urllib.request.urlopen(preload_list_url_as_text) as response:
            raw = response.read()

        # To avoid parsing the contents of the file out of the source tree viewer's
        # HTML, we download it as a raw file. googlesource.com Base64-encodes the
        # file to avoid potential content injection issues, so we need to decode it
        # before using it. https://code.google.com/p/gitiles/issues/detail?id=7
        raw = base64.b64decode(raw).decode('utf-8')

        # The .json file contains '//' comments, which are not actually valid JSON,
        # and confuse Python's JSON decoder. Begone, foul comments!
        raw = ''.join(
            [re.sub(r'^\s*//.*$', '', line) for line in raw.splitlines()])

        preload_json = json.loads(raw)
        utils.write(utils.json_for(preload_json), preload_cache)

    return {entry['name'] for entry in preload_json['entries']}
Example #3
0
def scan(domain, options):
    logging.debug("[%s][pshtt]" % domain)

    # cache output from pshtt
    cache_pshtt = utils.cache_path(domain, "pshtt", ext="json")

    force = options.get("force", False)
    data = None

    if (force is False) and (os.path.exists(cache_pshtt)):
        logging.debug("\tCached.")
        raw = open(cache_pshtt).read()
        data = json.loads(raw)
        if (data.__class__ is dict) and data.get('invalid'):
            return None

    else:
        logging.debug("\t %s %s" % (command, domain))

        flags = "--json --user-agent \"%s\" --timeout %i --preload-cache %s" % (
            user_agent, timeout, preload_cache)

        # Only useful when debugging interaction between projects.
        # flags = "%s --debug" % flags

        # Give the Python shell environment a pyenv environment.
        pyenv_init = "eval \"$(pyenv init -)\" && pyenv shell %s" % pyenv_version
        # Really un-ideal, but calling out to Python2 from Python 3 is a nightmare.
        # I don't think this tool's threat model includes untrusted CSV, either.
        raw = utils.unsafe_execute("%s && %s %s %s" %
                                   (pyenv_init, command, domain, flags))

        if not raw:
            utils.write(utils.invalid({}), cache_pshtt)
            logging.warn("\tBad news scanning, sorry!")
            return None

        data = json.loads(raw)
        utils.write(utils.json_for(data), utils.cache_path(domain, "pshtt"))

    # pshtt scanner uses JSON arrays, even for single items
    data = data[0]

    row = []
    for field in headers:
        value = data[field]

        # TODO: Fix this upstream
        if (field != "HSTS Header") and (field != "HSTS Max Age") and (
                field != "Redirect To"):
            if value is None:
                value = False

        row.append(value)

    yield row
Example #4
0
def scan(domain, options):
    logging.debug("[%s][analytics]" % domain)
    logging.debug("\tChecking file.")

    data = {'participating': (domain in analytics_domains)}

    cache = utils.cache_path(domain, "analytics")
    utils.write(utils.json_for(data), cache)

    yield [data['participating']]
Example #5
0
def scan(domain, options):
    logging.debug("[%s][pageload]" % domain)

    inspection = utils.data_for(domain, "inspect")

    # If we have data from inspect, skip if it's not a live domain.
    if inspection and (not inspection.get("up")):
        logging.debug("\tSkipping, domain not reachable during inspection.")
        return None

    # If we have data from inspect, skip if it's just a redirector.
    if inspection and (inspection.get("redirect") is True):
        logging.debug(
            "\tSkipping, domain seen as just a redirector during inspection.")
        return None

    # phantomas needs a URL, not just a domain.
    if not (domain.startswith('http://') or domain.startswith('https://')):

        # If we have data from inspect, use the canonical endpoint.
        if inspection and inspection.get("canonical"):
            url = inspection.get("canonical")

        # Otherwise, well, whatever.
        else:
            url = 'http://' + domain
    else:
        url = domain

    # We'll cache prettified JSON from the output.
    cache = utils.cache_path(domain, "pageload")

    # If we've got it cached, use that.
    if (options.get("force", False) is False) and (os.path.exists(cache)):
        logging.debug("\tCached.")
        raw = open(cache).read()
        data = json.loads(raw)
        if data.get('invalid'):
            return None

    # If no cache, or we should run anyway, do the scan.
    else:
        logging.debug("\t %s %s --reporter=json --ignore-ssl-errors" %
                      (command, url))
        raw = utils.scan(
            [command, url, "--reporter=json", "--ignore-ssl-errors"])
        if not raw:
            utils.write(utils.invalid({}), cache)
            return None

        # It had better be JSON, which we can cache in prettified form.
        data = json.loads(raw)
        utils.write(utils.json_for(data), cache)

    yield [data['metrics'][metric] for metric in interesting_metrics]
Example #6
0
def scan(domain, options):
    logging.debug("[%s][pshtt]" % domain)

    # cache output from pshtt
    cache_pshtt = utils.cache_path(domain, "pshtt", ext="json")

    force = options.get("force", False)
    data = None

    if (force is False) and (os.path.exists(cache_pshtt)):
        logging.debug("\tCached.")
        raw = open(cache_pshtt).read()
        data = json.loads(raw)
        if (data.__class__ is dict) and data.get('invalid'):
            return None

    else:
        logging.debug("\t %s %s" % (command, domain))

        flags = "--json --user-agent \"%s\" --timeout %i --preload-cache %s" % (user_agent, timeout, preload_cache)

        # Only useful when debugging interaction between projects.
        # flags = "%s --debug" % flags

        # Give the Python shell environment a pyenv environment.
        pyenv_init = "eval \"$(pyenv init -)\" && pyenv shell %s" % pyenv_version
        # Really un-ideal, but calling out to Python2 from Python 3 is a nightmare.
        # I don't think this tool's threat model includes untrusted CSV, either.
        raw = utils.unsafe_execute("%s && %s %s %s" % (pyenv_init, command, domain, flags))

        if not raw:
            utils.write(utils.invalid({}), cache_pshtt)
            logging.warn("\tBad news scanning, sorry!")
            return None

        data = json.loads(raw)
        utils.write(utils.json_for(data), utils.cache_path(domain, "pshtt"))

    # pshtt scanner uses JSON arrays, even for single items
    data = data[0]

    row = []
    for field in headers:
        value = data[field]

        # TODO: Fix this upstream
        if (field != "HSTS Header") and (field != "HSTS Max Age") and (field != "Redirect To"):
            if value is None:
                value = False

        row.append(value)

    yield row
Example #7
0
def scan(domain, options):
    logging.debug("[%s][analytics]" % domain)
    logging.debug("\tChecking file.")

    data = {
        'participating': (domain in analytics_domains)
    }

    cache = utils.cache_path(domain, "analytics")
    utils.write(utils.json_for(data), cache)

    yield [data['participating']]
Example #8
0
def scan(domain, options):
    logging.debug("[%s][pageload]" % domain)

    inspection = utils.data_for(domain, "inspect")

    # If we have data from inspect, skip if it's not a live domain.
    if inspection and (not inspection.get("up")):
        logging.debug("\tSkipping, domain not reachable during inspection.")
        return None

    # If we have data from inspect, skip if it's just a redirector.
    if inspection and (inspection.get("redirect") is True):
        logging.debug("\tSkipping, domain seen as just a redirector during inspection.")
        return None

    # phantomas needs a URL, not just a domain.
    if not (domain.startswith('http://') or domain.startswith('https://')):

        # If we have data from inspect, use the canonical endpoint.
        if inspection and inspection.get("canonical"):
            url = inspection.get("canonical")

        # Otherwise, well, whatever.
        else:
            url = 'http://' + domain
    else:
        url = domain

    # We'll cache prettified JSON from the output.
    cache = utils.cache_path(domain, "pageload")

    # If we've got it cached, use that.
    if (options.get("force", False) is False) and (os.path.exists(cache)):
        logging.debug("\tCached.")
        raw = open(cache).read()
        data = json.loads(raw)
        if data.get('invalid'):
            return None

    # If no cache, or we should run anyway, do the scan.
    else:
        logging.debug("\t %s %s --reporter=json --ignore-ssl-errors" % (command, url))
        raw = utils.scan([command, url, "--reporter=json", "--ignore-ssl-errors"])
        if not raw:
            utils.write(utils.invalid({}), cache)
            return None

        # It had better be JSON, which we can cache in prettified form.
        data = json.loads(raw)
        utils.write(utils.json_for(data), cache)

    yield [data['metrics'][metric] for metric in interesting_metrics]
Example #9
0
def scan(domain, options):
    logging.debug("[%s][pshtt]" % domain)

    # cache output from pshtt
    cache_pshtt = utils.cache_path(domain, "pshtt", ext="json")

    force = options.get("force", False)
    data = None

    if (force is False) and (os.path.exists(cache_pshtt)):
        logging.debug("\tCached.")
        raw = open(cache_pshtt).read()
        data = json.loads(raw)
        if (data.__class__ is dict) and data.get('invalid'):
            return None

    else:
        logging.debug("\t %s %s" % (command, domain))

        raw = utils.scan([
            command,
            domain,
            '--json',
            '--user-agent', '\"%s\"' % user_agent,
            '--timeout', str(timeout),
            '--preload-cache', preload_cache
        ])

        if not raw:
            utils.write(utils.invalid({}), cache_pshtt)
            logging.warn("\tBad news scanning, sorry!")
            return None

        data = json.loads(raw)
        utils.write(utils.json_for(data), utils.cache_path(domain, "pshtt"))

    # pshtt scanner uses JSON arrays, even for single items
    data = data[0]

    row = []
    for field in headers:
        value = data[field]

        # TODO: Fix this upstream
        if (field != "HSTS Header") and (field != "HSTS Max Age") and (field != "Redirect To"):
            if value is None:
                value = False

        row.append(value)

    yield row
Example #10
0
def handler(event, context):
    start_time = utils.local_now()

    domain = event.get('domain')
    options = event.get('options')
    name = event.get('scanner')
    environment = event.get('environment')

    # Log all sent events, for the record.
    utils.configure_logging(options)
    logging.warn(event)

    # Might be acceptable to let this crash the module, in Lambda.
    try:
        scanner = importlib.import_module("scanners.%s" % name)
    except ImportError:
        exc_type, exc_value, exc_traceback = sys.exc_info()
        logging.error(
            "[%s] Scanner not found, or had an error during loading.\n\tERROR: %s\n\t%s"
            % (name, exc_type, exc_value))
        exit(1)  # ?

    # Same method call as when run locally.
    data = scanner.scan(domain, environment, options)

    # We capture start and end times locally as well, but it's
    # useful to know the start/end from Lambda's vantage point.
    end_time = utils.local_now()
    duration = end_time - start_time
    response = {
        'lambda': {
            'log_group_name': context.log_group_name,
            'log_stream_name': context.log_stream_name,
            'request_id': context.aws_request_id,
            'memory_limit': context.memory_limit_in_mb,
            'start_time': start_time,
            'end_time': end_time,
            'measured_duration': duration
        },
        'data': data
    }

    # Serialize and re-parse the JSON, so that we run our own
    # date transform functions in one place, before Amazon's built-in
    # JSON serialization prepares the data for transport.
    return utils.from_json(utils.json_for(response))
Example #11
0
def scan(domain, options):
    logging.debug("[%s][trustymail]" % domain)

    # cache output from pshtt
    cache_trustymail = utils.cache_path(domain, "trustymail", ext="json")

    force = options.get("force", False)

    if (force is False) and (os.path.exists(cache_trustymail)):
        logging.debug("\tCached.")
        raw = open(cache_trustymail).read()
        data = json.loads(raw)
        if (data.__class__ is dict) and data.get('invalid'):
            return None

    else:
        logging.debug("\t %s %s" % (command, domain))

        raw = utils.scan([
            command,
            domain,
            '--json',
            '--timeout',
            str(timeout),
        ])

        if not raw:
            utils.write(utils.invalid({}), cache_trustymail)
            logging.warn("\tBad news scanning, sorry!")
            return None

        data = json.loads(raw)
        utils.write(utils.json_for(data),
                    utils.cache_path(domain, "trustymail"))

    # trustymail scanner follows pshtt in  using JSON arrays, even for single items
    data = data[0]

    row = []
    for field in headers:
        value = data[field]

        row.append(value)

    yield row
Example #12
0
def check_wildcard(subdomain, options):

    wildcard = wildcard_for(subdomain)

    cache = utils.cache_path(subdomain, "subdomains")
    if (options.get("force", False) is False) and (os.path.exists(cache)):
        logging.debug("\tDNS info cached.")
        raw = open(cache).read()
        data = json.loads(raw)

    else:
        logging.debug("\t dig +short '%s'" % wildcard)
        raw_wild = utils.unsafe_execute("dig +short '%s'" % wildcard)

        if raw_wild == "":
            raw_wild = None
            raw_self = None
        else:
            logging.debug("\t dig +short '%s'" % subdomain)
            raw_self = utils.unsafe_execute("dig +short '%s'" % subdomain)

        if raw_wild:
            parsed_wild = raw_wild.split("\n")
            parsed_wild.sort()
        else:
            parsed_wild = None

        if raw_self:
            parsed_self = raw_self.split("\n")
            parsed_self.sort()
        else:
            parsed_self = None

        data = {'response': {'wild': parsed_wild, 'itself': parsed_self}}
        utils.write(
            utils.json_for(data),
            cache
        )

    return data['response']
Example #13
0
def check_wildcard(subdomain, options):

    wildcard = wildcard_for(subdomain)

    cache = utils.cache_path(subdomain, "subdomains")
    if (options.get("force", False) is False) and (os.path.exists(cache)):
        logging.debug("\tDNS info cached.")
        raw = open(cache).read()
        data = json.loads(raw)

    else:
        logging.debug("\t dig +short '%s'" % wildcard)
        raw_wild = utils.unsafe_execute("dig +short '%s'" % wildcard)

        if raw_wild == "":
            raw_wild = None
            raw_self = None
        else:
            logging.debug("\t dig +short '%s'" % subdomain)
            raw_self = utils.unsafe_execute("dig +short '%s'" % subdomain)

        if raw_wild:
            parsed_wild = raw_wild.split("\n")
            parsed_wild.sort()
        else:
            parsed_wild = None

        if raw_self:
            parsed_self = raw_self.split("\n")
            parsed_self.sort()
        else:
            parsed_self = None

        data = {'response': {'wild': parsed_wild, 'itself': parsed_self}}
        utils.write(utils.json_for(data), cache)

    return data['response']
Example #14
0
def scan(domain, options):
	logging.debug("[%s][sslyze]" % domain)

	# Optional: skip domains which don't support HTTPS in prior inspection
	inspection = utils.data_for(domain, "inspect")
	if inspection and (not inspection.get("support_https")):
		logging.debug("\tSkipping, HTTPS not supported in inspection.")
		return None

	# Optional: if inspect data says canonical endpoint uses www and this domain
	# doesn't have it, add it.
	if inspection and (inspection.get("canonical_endpoint") == "www") and (not domain.startswith("www.")):
		scan_domain = "www.%s" % domain
	else:
		scan_domain = domain

	# cache XML from sslyze
	cache_xml = utils.cache_path(domain, "sslyze", ext="xml")
	# because sslyze manages its own output (can't yet print to stdout),
	# we have to mkdir_p the path ourselves
	utils.mkdir_p(os.path.dirname(cache_xml))

	force = options.get("force", False)

	if (force is False) and (os.path.exists(cache_xml)):
		logging.debug("\tCached.")
		xml = open(cache_xml).read()

	else:
		logging.debug("\t %s %s" % (command, domain))
		# use scan_domain (possibly www-prefixed) to do actual scan
		raw = utils.scan([command, "--regular", "--quiet", scan_domain, "--xml_out=%s" % cache_xml], env=command_env)
		
		if raw is None:
			# TODO: save standard invalid XML data...?
			logging.warn("\tBad news scanning, sorry!")
			return None

		xml = utils.scan(["cat", cache_xml])
		if not xml:
			logging.warn("\tBad news reading XML, sorry!")
			return None

		utils.write(xml, cache_xml)

	data = parse_sslyze(xml)

	if data is None:
		logging.warn("\tNo valid target for scanning, couldn't connect.")
		return None

	utils.write(utils.json_for(data), utils.cache_path(domain, "sslyze"))

	yield [
		data['protocols']['sslv2'], data['protocols']['sslv3'], 
		data['protocols']['tlsv1.0'], data['protocols']['tlsv1.1'], 
		data['protocols']['tlsv1.2'], 

		data['config'].get('any_dhe'), data['config'].get('all_dhe'),
		data['config'].get('weakest_dh'),
		data['config'].get('any_rc4'), data['config'].get('all_rc4'),

		data['config'].get('ocsp_stapling'),
		
		data['certs'].get('key_type'), data['certs'].get('key_length'),
		data['certs'].get('leaf_signature'), data['certs'].get('any_sha1'),
		data['certs'].get('not_before'), data['certs'].get('not_after'),
		data['certs'].get('served_issuer'), 

		data.get('errors')
	]
Example #15
0
def network_check(subdomain, endpoint, options):
    cache = utils.cache_path(subdomain, "subdomains")

    wildcard = wildcard_for(subdomain)

    if (options.get("force", False) is False) and (os.path.exists(cache)):
        logging.debug("\tDNS and content cached.")
        raw = open(cache).read()
        data = json.loads(raw)

    # Hit DNS and HTTP.
    else:
        # HTTP content: just use curl.
        #
        # Turn on --insecure because we want to see the content even at sites
        # where the certificate isn't right or proper.
        logging.debug("\t curl --silent --insecure %s" % endpoint)
        content = utils.scan(["curl", "--silent", "--insecure", endpoint])

        # DNS content: just use dig.
        #
        # Not awesome - uses an unsafe shell execution of `dig` to look up DNS,
        # as I couldn't figure out a way to get "+short" to play nice with
        # the more secure execution methods available to me. Since this system
        # isn't expected to process untrusted input, this should be okay.
        logging.debug("\t dig +short '%s'" % wildcard)
        raw_wild = utils.unsafe_execute("dig +short '%s'" % wildcard)

        if raw_wild == "":
            raw_wild = None
            raw_self = None
        else:
            logging.debug("\t dig +short '%s'" % subdomain)
            raw_self = utils.unsafe_execute("dig +short '%s'" % subdomain)

        if raw_wild:
            parsed_wild = raw_wild.split("\n")
            parsed_wild.sort()
        else:
            parsed_wild = None

        if raw_self:
            parsed_self = raw_self.split("\n")
            parsed_self.sort()
        else:
            parsed_self = None

        # Cache HTTP and DNS data to disk.
        data = {
            'response': {
                'content': content,
                'wildcard_dns': parsed_wild,
                'self_dns': parsed_self
            }
        }

        if (parsed_wild) and (parsed_wild == parsed_self):
            data['response']['matched_wild'] = True
        else:
            data['response']['matched_wild'] = False

        utils.write(utils.json_for(data), cache)

    return data['response']
Example #16
0
def paginated_mode(suffix, options, uid, api_key):
    # Cache hostnames in a dict for de-duping.
    hostnames_map = {}

    certificate_api = certificates.CensysCertificates(uid, api_key)

    if 'query' in options and options['query']:
        query = options['query']
    else:
        query = "parsed.subject.common_name:\"%s\" or parsed.extensions.subject_alt_name.dns_names:\"%s\"" % (
            suffix, suffix)
    logging.debug("Censys query:\n%s\n" % query)

    # time to sleep between requests (defaults to 5s)
    delay = int(options.get("delay", 5))

    # Censys page size, fixed
    page_size = 100

    # Start page defaults to 1.
    start_page = int(options.get("start", 1))

    # End page defaults to whatever the API says is the last one.
    end_page = options.get("end", None)
    if end_page is None:
        end_page = get_end_page(query, certificate_api)
        if end_page is None:
            logging.warn("Error looking up number of pages.")
            exit(1)
    else:
        end_page = int(end_page)

    max_records = ((end_page - start_page) + 1) * page_size

    fields = [
        "parsed.subject.common_name",
        "parsed.extensions.subject_alt_name.dns_names"
    ]

    current_page = start_page

    logging.warn("Fetching up to %i records, starting at page %i." %
                 (max_records, start_page))
    last_cached = False
    force = options.get("force", False)

    while current_page <= end_page:
        if (not last_cached) and (current_page > start_page):
            logging.debug("(Waiting %is before fetching page %i.)" %
                          (delay, current_page))
            last_cached = False
            time.sleep(delay)

        logging.debug("Fetching page %i." % current_page)

        cache_page = utils.cache_path(str(current_page), "censys")
        if (force is False) and (os.path.exists(cache_page)):
            logging.warn("\t[%i] Cached page." % current_page)
            last_cached = True

            certs_raw = open(cache_page).read()
            certs = json.loads(certs_raw)
            if (certs.__class__ is dict) and certs.get('invalid'):
                continue
        else:
            try:
                certs = list(
                    certificate_api.search(query,
                                           fields=fields,
                                           page=current_page,
                                           max_records=page_size))
                utils.write(utils.json_for(certs), cache_page)
            except censys.base.CensysException:
                logging.warn(utils.format_last_exception())
                logging.warn("Censys error, skipping page %i." % current_page)
                utils.write(utils.invalid({}), cache_page)
                continue
            except:
                logging.warn(utils.format_last_exception())
                logging.warn("Unexpected error, skipping page %i." %
                             current_page)
                utils.write(utils.invalid({}), cache_page)
                exit(1)

        for cert in certs:
            # Common name + SANs
            names = cert.get('parsed.subject.common_name', []) + cert.get(
                'parsed.extensions.subject_alt_name.dns_names', [])
            logging.debug(names)

            for name in names:
                hostnames_map[sanitize_name(name)] = None

        current_page += 1

    logging.debug("Done fetching from API.")

    return hostnames_map
Example #17
0
def scan(domain, options):
    logging.debug("[%s][sslyze]" % domain)

    # Optional: skip domains which don't support HTTPS in prior inspection
    if utils.domain_doesnt_support_https(domain):
        logging.debug("\tSkipping, HTTPS not supported in inspection.")
        return None

    # Optional: if pshtt data says canonical endpoint uses www and this domain
    # doesn't have it, add it.
    if utils.domain_uses_www(domain):
        scan_domain = "www.%s" % domain
    else:
        scan_domain = domain

    # cache XML from sslyze
    cache_xml = utils.cache_path(domain, "sslyze", ext="xml")
    # because sslyze manages its own output (can't yet print to stdout),
    # we have to mkdir_p the path ourselves
    utils.mkdir_p(os.path.dirname(cache_xml))

    force = options.get("force", False)

    if (force is False) and (os.path.exists(cache_xml)):
        logging.debug("\tCached.")
        xml = open(cache_xml).read()

    else:
        logging.debug("\t %s %s" % (command, scan_domain))
        # use scan_domain (possibly www-prefixed) to do actual scan

        # Give the Python shell environment a pyenv environment.
        pyenv_init = "eval \"$(pyenv init -)\" && pyenv shell %s" % pyenv_version
        # Really un-ideal, but calling out to Python2 from Python 3 is a nightmare.
        # I don't think this tool's threat model includes untrusted CSV, either.
        raw = utils.unsafe_execute(
            "%s && %s --regular --quiet %s --xml_out=%s" %
            (pyenv_init, command, scan_domain, cache_xml))

        if raw is None:
            # TODO: save standard invalid XML data...?
            logging.warn("\tBad news scanning, sorry!")
            return None

        xml = utils.scan(["cat", cache_xml])
        if not xml:
            logging.warn("\tBad news reading XML, sorry!")
            return None

        utils.write(xml, cache_xml)

    data = parse_sslyze(xml)

    if data is None:
        logging.warn("\tNo valid target for scanning, couldn't connect.")
        return None

    utils.write(utils.json_for(data), utils.cache_path(domain, "sslyze"))

    yield [
        data['protocols']['sslv2'], data['protocols']['sslv3'],
        data['protocols']['tlsv1.0'], data['protocols']['tlsv1.1'],
        data['protocols']['tlsv1.2'], data['config'].get('any_dhe'),
        data['config'].get('all_dhe'), data['config'].get('weakest_dh'),
        data['config'].get('any_rc4'), data['config'].get('all_rc4'),
        data['config'].get('ocsp_stapling'), data['certs'].get('key_type'),
        data['certs'].get('key_length'), data['certs'].get('leaf_signature'),
        data['certs'].get('any_sha1'), data['certs'].get('not_before'),
        data['certs'].get('not_after'), data['certs'].get('served_issuer'),
        data.get('errors')
    ]
Example #18
0
def scan(domain, options):
    logging.debug("[%s][tls]" % domain)

    # If inspection data exists, check to see if we can skip.
    if utils.domain_doesnt_support_https(domain):
        logging.debug("\tSkipping, HTTPS not supported in inspection.")
        return None

    # cache reformatted JSON from ssllabs
    cache = utils.cache_path(domain, "tls")

    # Optional: if pshtt data says canonical endpoint uses www and this domain
    # doesn't have it, add it.
    if utils.domain_uses_www(domain):
        scan_domain = "www.%s" % domain
    else:
        scan_domain = domain

    force = options.get("force", False)

    if (force is False) and (os.path.exists(cache)):
        logging.debug("\tCached.")
        raw = open(cache).read()
        data = json.loads(raw)

        if data.get('invalid'):
            return None
    else:
        logging.debug("\t %s %s" % (command, scan_domain))

        usecache = str(not force).lower()

        if options.get("debug"):
            cmd = [command, "--usecache=%s" % usecache,
                   "--verbosity=debug", scan_domain]
        else:
            cmd = [command, "--usecache=%s" % usecache,
                   "--quiet", scan_domain]

        raw = utils.scan(cmd)
        if raw:
            data = json.loads(raw)

            # if SSL Labs gave us back an error response, cache this
            # as an invalid entry.
            if len(data) < 1:
                utils.write(utils.invalid({'response': data}), cache)
                return None

            # we only give ssllabs-scan one at a time,
            # so we can de-pluralize this
            data = data[0]

            # if SSL Labs had an error hitting the site, cache this
            # as an invalid entry.
            if data["status"] == "ERROR":
                utils.write(utils.invalid(data), cache)
                return None

            utils.write(utils.json_for(data), cache)
        else:
            return None
            # raise Exception("Invalid data from ssllabs-scan: %s" % raw)

    # can return multiple rows, one for each 'endpoint'
    for endpoint in data['endpoints']:

        # this meant it couldn't connect to the endpoint
        if not endpoint.get("grade"):
            continue

        sslv3 = False
        tlsv12 = False
        for protocol in endpoint['details']['protocols']:
            if ((protocol['name'] == "SSL") and
                    (protocol['version'] == '3.0')):
                sslv3 = True
            if ((protocol['name'] == "TLS") and
                    (protocol['version'] == '1.2')):
                tlsv12 = True

        spdy = False
        h2 = False
        npn = endpoint['details'].get('npnProtocols', None)
        if npn:
            spdy = ("spdy" in npn)
            h2 = ("h2" in npn)

        yield [
            endpoint['grade'],
            endpoint['details']['cert']['sigAlg'],
            endpoint['details']['key']['alg'],
            endpoint['details']['key']['size'],
            endpoint['details']['forwardSecrecy'],
            endpoint['details']['ocspStapling'],
            endpoint['details'].get('fallbackScsv', "N/A"),
            endpoint['details']['supportsRc4'],
            sslv3,
            tlsv12,
            spdy,
            endpoint['details']['sniRequired'],
            h2
        ]
Example #19
0
def scan(domain, options):
    logging.debug("[%s][tls]" % domain)

    # If inspection data exists, check to see if we can skip.
    inspection = utils.data_for(domain, "inspect")
    if inspection and (not inspection.get("support_https")):
        logging.debug("\tSkipping, HTTPS not supported in inspection.")
        yield None

    else:
        # cache reformatted JSON from ssllabs
        cache = utils.cache_path(domain, "tls")

        force = options.get("force", False)

        if (force is False) and (os.path.exists(cache)):
            logging.debug("\tCached.")
            raw = open(cache).read()
            data = json.loads(raw)

            if data.get('invalid'):
                return None
        else:
            logging.debug("\t %s %s" % (command, domain))

            usecache = str(not force).lower()

            if options.get("debug"):
                cmd = [command, "--usecache=%s" % usecache,
                       "--verbosity=debug", domain]
            else:
                cmd = [command, "--usecache=%s" % usecache,
                       "--quiet", domain]
            raw = utils.scan(cmd)
            if raw:
                data = json.loads(raw)

                # we only give ssllabs-scan one at a time,
                # so we can de-pluralize this
                data = data[0]

                # if SSL Labs had an error hitting the site, cache this
                # as an invalid entry.
                if data["status"] == "ERROR":
                    utils.write(utils.invalid(data), cache)
                    return None

                utils.write(utils.json_for(data), cache)
            else:
                return None
                # raise Exception("Invalid data from ssllabs-scan: %s" % raw)

        # can return multiple rows, one for each 'endpoint'
        for endpoint in data['endpoints']:

            # this meant it couldn't connect to the endpoint
            if not endpoint.get("grade"):
                continue

            sslv3 = False
            tlsv12 = False
            for protocol in endpoint['details']['protocols']:
                if ((protocol['name'] == "SSL") and
                        (protocol['version'] == '3.0')):
                    sslv3 = True
                if ((protocol['name'] == "TLS") and
                        (protocol['version'] == '1.2')):
                    tlsv12 = True

            spdy = False
            h2 = False
            npn = endpoint['details'].get('npnProtocols', None)
            if npn:
                spdy = ("spdy" in npn)
                h2 = ("h2-" in npn)

            def ccs_map(n):
                return {
                    -1: "N/A (Error)",
                    0: "N/A (Unknown)",
                    1: "No (not vulnerable)",
                    2: "No (not exploitable)",
                    3: "Yes"
                }[n]

            def fs_map(n):
                return {
                    0: "0 - No",
                    1: "1 - Some",
                    2: "2 - Modern",
                    4: "3 - Robust"
                }[n]

            yield [
                endpoint['grade'],
                endpoint['details']['cert']['sigAlg'],
                endpoint['details']['key']['alg'],
                endpoint['details']['key']['size'],
                fs_map(endpoint['details']['forwardSecrecy']),
                endpoint['details']['ocspStapling'],
                endpoint['details'].get('fallbackScsv', "N/A"),
                endpoint['details'].get('freak'),
                ccs_map(endpoint['details']['openSslCcs']),
                sslv3,
                tlsv12,
                spdy,
                endpoint['details']['sniRequired'],
                h2
            ]
def scan(domain, options):
    logging.debug("[%s][third_parties]" % domain)

    # Default timeout is 15s, too little.
    timeout = int(options.get("timeout", 60))

    # If we have data from pshtt, skip if it's not a live domain.
    if utils.domain_not_live(domain):
        logging.debug("\tSkipping, domain not reachable during inspection.")
        return None

    # If we have data from pshtt, skip if it's just a redirector.
    if utils.domain_is_redirect(domain):
        logging.debug(
            "\tSkipping, domain seen as just an external redirector during inspection."
        )
        return None

    # phantomas needs a URL, not just a domain.
    if not (domain.startswith('http://') or domain.startswith('https://')):

        # If we have data from pshtt, use the canonical endpoint.
        if utils.domain_canonical(domain):
            url = utils.domain_canonical(domain)

        # Otherwise, well, whatever.
        else:
            url = 'http://' + domain
    else:
        url = domain

    # calculated_domain = re.sub("https?:\/\/", "", url)

    # We'll cache prettified JSON from the output.
    cache = utils.cache_path(domain, "third_parties")

    # If we've got it cached, use that.
    if (options.get("force", False) is False) and (os.path.exists(cache)):
        logging.debug("\tCached.")
        raw = open(cache).read()
        data = json.loads(raw)
        if data.get('invalid'):
            return None

    # If no cache, or we should run anyway, do the scan.
    else:
        logging.debug(
            "\t %s %s --modules=domains --reporter=json --timeout=%i --ignore-ssl-errors"
            % (command, url, timeout))
        raw = utils.scan([
            command, url, "--modules=domains", "--reporter=json",
            "--timeout=%i" % timeout, "--ignore-ssl-errors"
        ],
                         allowed_return_codes=[252])
        if not raw:
            utils.write(utils.invalid({}), cache)
            return None

        # It had better be JSON, which we can cache in prettified form.
        data = json.loads(raw)
        utils.write(utils.json_for(data), cache)

    services = services_for(data, domain, options)

    # Convert to CSV row
    known_names = list(known_services.keys())
    known_names.sort()
    known_matches = [
        'Yes' if host in services['known'] else 'No' for host in known_names
    ]

    yield [
        len(services['external']),
        len(services['internal']),
        services['external_requests'],
        services['internal_requests'],
        serialize(services['external']),
        serialize(services['internal']),
        # services['affiliated'],
        # services['unknown']
    ] + known_matches
Example #21
0
def gather(suffix, options):
    # Register a (free) Censys.io account to get a UID and API key.
    uid = options.get("censys_id", None)
    api_key = options.get("censys_key", None)

    if (uid is None) or (api_key is None):
        uid = os.environ.get("CENSYS_UID", None)
        api_key = os.environ.get("CENSYS_API_KEY", None)

    if (uid is None) or (api_key is None):
        logging.warn(
            "No Censys credentials set. API key required to use the Censys API."
        )
        exit(1)

    certificate_api = certificates.CensysCertificates(uid, api_key)

    query = "parsed.subject.common_name:\"%s\" or parsed.extensions.subject_alt_name.dns_names:\"%s\"" % (
        suffix, suffix)
    logging.debug("Censys query:\n%s\n" % query)

    # Hostnames beginning with a wildcard prefix will have the prefix stripped.
    wildcard_pattern = re.compile("^\*\.")
    redacted_pattern = re.compile("^(\?\.)+")

    # time to sleep between requests (defaults to 5s)
    delay = int(options.get("delay", 5))

    # Censys page size, fixed
    page_size = 100

    # Start page defaults to 1.
    start_page = int(options.get("start", 1))

    # End page defaults to whatever the API says is the last one.
    end_page = options.get("end", None)
    if end_page is None:
        end_page = get_end_page(query, certificate_api)
        if end_page is None:
            logging.warn("Error looking up number of pages.")
            exit(1)
    else:
        end_page = int(end_page)

    max_records = ((end_page - start_page) + 1) * page_size

    # Cache hostnames in a dict for de-duping.
    hostnames_map = {}

    fields = [
        "parsed.subject.common_name",
        "parsed.extensions.subject_alt_name.dns_names"
    ]

    current_page = start_page

    logging.warn("Fetching up to %i records, starting at page %i." %
                 (max_records, start_page))
    last_cached = False
    force = options.get("force", False)

    while current_page <= end_page:
        if (not last_cached) and (current_page > start_page):
            logging.debug("(Waiting %is before fetching page %i.)" %
                          (delay, current_page))
            last_cached = False
            time.sleep(delay)

        logging.debug("Fetching page %i." % current_page)

        cache_page = utils.cache_path(str(current_page), "censys")
        if (force is False) and (os.path.exists(cache_page)):
            logging.warn("\t[%i] Cached page." % current_page)
            last_cached = True

            certs_raw = open(cache_page).read()
            certs = json.loads(certs_raw)
            if (certs.__class__ is dict) and certs.get('invalid'):
                continue
        else:
            try:
                certs = list(
                    certificate_api.search(query,
                                           fields=fields,
                                           page=current_page,
                                           max_records=page_size))
                utils.write(utils.json_for(certs), cache_page)
            except censys.base.CensysException:
                logging.warn(utils.format_last_exception())
                logging.warn("Censys error, skipping page %i." % current_page)
                utils.write(utils.invalid({}), cache_page)
                continue
            except:
                logging.warn(utils.format_last_exception())
                logging.warn("Unexpected error, skipping page %i." %
                             current_page)
                utils.write(utils.invalid({}), cache_page)
                exit(1)

        for cert in certs:
            # Common name + SANs
            names = cert.get('parsed.subject.common_name', []) + cert.get(
                'parsed.extensions.subject_alt_name.dns_names', [])
            logging.debug(names)

            for name in names:
                # Strip off any wildcard prefix.
                name = re.sub(wildcard_pattern, '', name).lower().strip()
                # Strip off any redacted ? prefixes. (Ugh.)
                name = re.sub(redacted_pattern, '', name).lower().strip()
                hostnames_map[name] = None

        current_page += 1

    logging.debug("Done fetching from API.")

    # Iterator doesn't buy much efficiency, since we paginated already.
    # Necessary evil to de-dupe before returning hostnames, though.
    for hostname in hostnames_map.keys():
        yield hostname
Example #22
0
def scan(domain, options):
    logging.debug("[%s][tls]" % domain)

    # If inspection data exists, check to see if we can skip.
    inspection = utils.data_for(domain, "inspect")
    if inspection and (not inspection.get("support_https")):
        logging.debug("\tSkipping, HTTPS not supported in inspection.")
        return None

    else:
        # cache reformatted JSON from ssllabs
        cache = utils.cache_path(domain, "tls")

        force = options.get("force", False)

        if (force is False) and (os.path.exists(cache)):
            logging.debug("\tCached.")
            raw = open(cache).read()
            data = json.loads(raw)

            if data.get("invalid"):
                return None
        else:
            logging.debug("\t %s %s" % (command, domain))

            usecache = str(not force).lower()

            if options.get("debug"):
                cmd = [command, "--usecache=%s" % usecache, "--verbosity=debug", domain]
            else:
                cmd = [command, "--usecache=%s" % usecache, "--quiet", domain]
            raw = utils.scan(cmd)
            if raw:
                data = json.loads(raw)

                # we only give ssllabs-scan one at a time,
                # so we can de-pluralize this
                data = data[0]

                # if SSL Labs had an error hitting the site, cache this
                # as an invalid entry.
                if data["status"] == "ERROR":
                    utils.write(utils.invalid(data), cache)
                    return None

                utils.write(utils.json_for(data), cache)
            else:
                return None
                # raise Exception("Invalid data from ssllabs-scan: %s" % raw)

        # can return multiple rows, one for each 'endpoint'
        for endpoint in data["endpoints"]:

            # this meant it couldn't connect to the endpoint
            if not endpoint.get("grade"):
                continue

            sslv3 = False
            tlsv12 = False
            for protocol in endpoint["details"]["protocols"]:
                if (protocol["name"] == "SSL") and (protocol["version"] == "3.0"):
                    sslv3 = True
                if (protocol["name"] == "TLS") and (protocol["version"] == "1.2"):
                    tlsv12 = True

            spdy = False
            h2 = False
            npn = endpoint["details"].get("npnProtocols", None)
            if npn:
                spdy = "spdy" in npn
                h2 = "h2-" in npn

            yield [
                endpoint["grade"],
                endpoint["details"]["cert"]["sigAlg"],
                endpoint["details"]["key"]["alg"],
                endpoint["details"]["key"]["size"],
                endpoint["details"]["forwardSecrecy"],
                endpoint["details"]["ocspStapling"],
                endpoint["details"].get("fallbackScsv", "N/A"),
                endpoint["details"]["supportsRc4"],
                sslv3,
                tlsv12,
                spdy,
                endpoint["details"]["sniRequired"],
                h2,
            ]
Example #23
0
def network_check(subdomain, endpoint, options):
    cache = utils.cache_path(subdomain, "subdomains")

    wildcard = wildcard_for(subdomain)

    if (options.get("force", False) is False) and (os.path.exists(cache)):
        logging.debug("\tDNS and content cached.")
        raw = open(cache).read()
        data = json.loads(raw)

    # Hit DNS and HTTP.
    else:
        # HTTP content: just use curl.
        #
        # Turn on --insecure because we want to see the content even at sites
        # where the certificate isn't right or proper.
        logging.debug("\t curl --silent --insecure %s" % endpoint)
        content = utils.scan(["curl", "--silent", "--insecure", endpoint])

        # DNS content: just use dig.
        #
        # Not awesome - uses an unsafe shell execution of `dig` to look up DNS,
        # as I couldn't figure out a way to get "+short" to play nice with
        # the more secure execution methods available to me. Since this system
        # isn't expected to process untrusted input, this should be okay.
        logging.debug("\t dig +short '%s'" % wildcard)
        raw_wild = utils.unsafe_execute("dig +short '%s'" % wildcard)

        if raw_wild == "":
            raw_wild = None
            raw_self = None
        else:
            logging.debug("\t dig +short '%s'" % subdomain)
            raw_self = utils.unsafe_execute("dig +short '%s'" % subdomain)

        if raw_wild:
            parsed_wild = raw_wild.split("\n")
            parsed_wild.sort()
        else:
            parsed_wild = None

        if raw_self:
            parsed_self = raw_self.split("\n")
            parsed_self.sort()
        else:
            parsed_self = None

        # Cache HTTP and DNS data to disk.
        data = {'response': {
            'content': content,
            'wildcard_dns': parsed_wild,
            'self_dns': parsed_self
        }}

        if (parsed_wild) and (parsed_wild == parsed_self):
            data['response']['matched_wild'] = True
        else:
            data['response']['matched_wild'] = False

        utils.write(utils.json_for(data), cache)

    return data['response']
Example #24
0
def scan(domain, options):
    logging.debug("[%s][tls]" % domain)

    # If pshtt data exists, check to see if we can skip.
    if utils.domain_doesnt_support_https(domain):
        logging.debug("\tSkipping, HTTPS not supported.")
        return None

    # cache reformatted JSON from ssllabs
    cache = utils.cache_path(domain, "tls")

    # Optional: if pshtt data says canonical endpoint uses www and this domain
    # doesn't have it, add it.
    if utils.domain_uses_www(domain):
        scan_domain = "www.%s" % domain
    else:
        scan_domain = domain

    force = options.get("force", False)

    if (force is False) and (os.path.exists(cache)):
        logging.debug("\tCached.")
        raw = open(cache).read()
        data = json.loads(raw)

        if data.get('invalid'):
            return None
    else:
        logging.debug("\t %s %s" % (command, scan_domain))

        usecache = str(not force).lower()

        if options.get("debug"):
            cmd = [
                command,
                "--usecache=%s" % usecache, "--verbosity=debug", scan_domain
            ]
        else:
            cmd = [command, "--usecache=%s" % usecache, "--quiet", scan_domain]

        raw = utils.scan(cmd)
        if raw:
            data = json.loads(raw)

            # if SSL Labs gave us back an error response, cache this
            # as an invalid entry.
            if len(data) < 1:
                utils.write(utils.invalid({'response': data}), cache)
                return None

            # we only give ssllabs-scan one at a time,
            # so we can de-pluralize this
            data = data[0]

            # if SSL Labs had an error hitting the site, cache this
            # as an invalid entry.
            if data["status"] == "ERROR":
                utils.write(utils.invalid(data), cache)
                return None

            utils.write(utils.json_for(data), cache)
        else:
            return None
            # raise Exception("Invalid data from ssllabs-scan: %s" % raw)

    # can return multiple rows, one for each 'endpoint'
    for endpoint in data['endpoints']:

        # this meant it couldn't connect to the endpoint
        if not endpoint.get("grade"):
            continue

        sslv3 = False
        tlsv12 = False
        for protocol in endpoint['details']['protocols']:
            if ((protocol['name'] == "SSL")
                    and (protocol['version'] == '3.0')):
                sslv3 = True
            if ((protocol['name'] == "TLS")
                    and (protocol['version'] == '1.2')):
                tlsv12 = True

        spdy = False
        h2 = False
        npn = endpoint['details'].get('npnProtocols', None)
        if npn:
            spdy = ("spdy" in npn)
            h2 = ("h2" in npn)

        yield [
            endpoint['grade'], endpoint['details']['cert']['sigAlg'],
            endpoint['details']['key']['alg'],
            endpoint['details']['key']['size'],
            endpoint['details']['forwardSecrecy'],
            endpoint['details']['ocspStapling'],
            endpoint['details'].get('fallbackScsv',
                                    "N/A"), endpoint['details']['supportsRc4'],
            sslv3, tlsv12, spdy, endpoint['details']['sniRequired'], h2
        ]
Example #25
0
def scan(domain, options):
    logging.debug("[%s][sslyze]" % domain)

    # Optional: skip domains which don't support HTTPS in prior inspection
    if utils.domain_doesnt_support_https(domain):
        logging.debug("\tSkipping, HTTPS not supported in inspection.")
        return None

    # Optional: if pshtt data says canonical endpoint uses www and this domain
    # doesn't have it, add it.
    if utils.domain_uses_www(domain):
        scan_domain = "www.%s" % domain
    else:
        scan_domain = domain

    # cache XML from sslyze
    cache_xml = utils.cache_path(domain, "sslyze", ext="xml")
    # because sslyze manages its own output (can't yet print to stdout),
    # we have to mkdir_p the path ourselves
    utils.mkdir_p(os.path.dirname(cache_xml))

    force = options.get("force", False)

    if (force is False) and (os.path.exists(cache_xml)):
        logging.debug("\tCached.")
        xml = open(cache_xml).read()

    else:
        logging.debug("\t %s %s" % (command, scan_domain))
        # use scan_domain (possibly www-prefixed) to do actual scan

        # Give the Python shell environment a pyenv environment.
        pyenv_init = "eval \"$(pyenv init -)\" && pyenv shell %s" % pyenv_version
        # Really un-ideal, but calling out to Python2 from Python 3 is a nightmare.
        # I don't think this tool's threat model includes untrusted CSV, either.
        raw = utils.unsafe_execute("%s && %s --regular --quiet %s --xml_out=%s" % (pyenv_init, command, scan_domain, cache_xml))

        if raw is None:
            # TODO: save standard invalid XML data...?
            logging.warn("\tBad news scanning, sorry!")
            return None

        xml = utils.scan(["cat", cache_xml])
        if not xml:
            logging.warn("\tBad news reading XML, sorry!")
            return None

        utils.write(xml, cache_xml)

    data = parse_sslyze(xml)

    if data is None:
        logging.warn("\tNo valid target for scanning, couldn't connect.")
        return None

    utils.write(utils.json_for(data), utils.cache_path(domain, "sslyze"))

    yield [
        data['protocols']['sslv2'], data['protocols']['sslv3'],
        data['protocols']['tlsv1.0'], data['protocols']['tlsv1.1'],
        data['protocols']['tlsv1.2'],

        data['config'].get('any_dhe'), data['config'].get('all_dhe'),
        data['config'].get('weakest_dh'),
        data['config'].get('any_rc4'), data['config'].get('all_rc4'),

        data['config'].get('ocsp_stapling'),

        data['certs'].get('key_type'), data['certs'].get('key_length'),
        data['certs'].get('leaf_signature'), data['certs'].get('any_sha1'),
        data['certs'].get('not_before'), data['certs'].get('not_after'),
        data['certs'].get('served_issuer'),

        data.get('errors')
    ]