Exemple #1
0
def _prepare_rec(spec, ignorenets, neverignore):
    # First of all, let's see if we are supposed to ignore this spec,
    # and if so, do so.
    if 'addr' in spec and \
       spec.get('source') not in neverignore.get(spec['recontype'], []):
        for start, stop in ignorenets.get(spec['recontype'], ()):
            if start <= utils.force_ip2int(spec['addr']) <= stop:
                return None
    # Then, let's clean up the records.
    # Change Symantec's random user agents (matching SYMANTEC_UA) to
    # the constant string 'SymantecRandomUserAgent'.
    if spec['recontype'] == 'HTTP_CLIENT_HEADER' and \
       spec.get('source') == 'USER-AGENT':
        if SYMANTEC_UA.match(spec['value']):
            spec['value'] = 'SymantecRandomUserAgent'
        elif KASPERSKY_UA.match(spec['value']):
            spec['value'] = 'KasperskyWeirdUserAgent'
        else:
            match = SYMANTEC_SEP_UA.match(spec['value'])
            if match is not None:
                spec['value'] = '%s%s' % match.groups()
    # Change any Digest authorization header to remove non-constant
    # information. On one hand we loose the necessary information to
    # try to recover the passwords, but on the other hand we store
    # specs with different challenges but the same username, realm,
    # host and sensor in the same records.
    elif (spec['recontype']
          in {'HTTP_CLIENT_HEADER', 'HTTP_CLIENT_HEADER_SERVER'}
          and spec.get('source') in {'AUTHORIZATION', 'PROXY-AUTHORIZATION'}):
        value = spec['value']
        if value:
            authtype = value.split(None, 1)[0]
            if authtype.lower() == 'digest':
                try:
                    # we only keep relevant info
                    spec['value'] = '%s %s' % (authtype, ','.join(
                        val for val in _split_digest_auth(value[6:].strip())
                        if DIGEST_AUTH_INFOS.match(val)))
                except Exception:
                    utils.LOGGER.warning("Cannot parse digest error for %r",
                                         spec,
                                         exc_info=True)
            elif ntlm._is_ntlm_message(value):
                # NTLM_NEGOTIATE and NTLM_AUTHENTICATE
                try:
                    auth = utils.decode_b64(value.split(None, 1)[1].encode())
                except (UnicodeDecodeError, TypeError, ValueError,
                        binascii.Error):
                    pass
                spec['value'] = "%s %s" % \
                    (value.split(None, 1)[0],
                     ntlm._ntlm_dict2string(ntlm.ntlm_extract_info(auth)))
            elif authtype.lower() in {'negotiate', 'kerberos', 'oauth'}:
                spec['value'] = authtype
    elif (spec['recontype'] == 'HTTP_SERVER_HEADER' and spec.get('source')
          in {'WWW-AUTHENTICATE', 'PROXY-AUTHENTICATE'}):
        value = spec['value']
        if value:
            authtype = value.split(None, 1)[0]
            if authtype.lower() == 'digest':
                try:
                    # we only keep relevant info
                    spec['value'] = '%s %s' % (authtype, ','.join(
                        val for val in _split_digest_auth(value[6:].strip())
                        if DIGEST_AUTH_INFOS.match(val)))
                except Exception:
                    utils.LOGGER.warning("Cannot parse digest error for %r",
                                         spec,
                                         exc_info=True)
            elif ntlm._is_ntlm_message(value):
                # NTLM_CHALLENGE
                try:
                    auth = utils.decode_b64(value.split(None, 1)[1].encode())
                except (UnicodeDecodeError, TypeError, ValueError,
                        binascii.Error):
                    pass
                spec['value'] = "%s %s" % \
                    (value.split(None, 1)[0],
                     ntlm._ntlm_dict2string(ntlm.ntlm_extract_info(auth)))
            elif authtype.lower() in {'negotiate', 'kerberos', 'oauth'}:
                spec['value'] = authtype
    # TCP server banners: try to normalize data
    elif spec['recontype'] == 'TCP_SERVER_BANNER':
        newvalue = value = utils.nmap_decode_data(spec['value'])
        for pattern, replace in TCP_SERVER_PATTERNS:
            if pattern.search(newvalue):
                newvalue = pattern.sub(replace, newvalue)
        if newvalue != value:
            spec['value'] = utils.nmap_encode_data(newvalue)
    elif spec['recontype'] == 'TCP_CLIENT_BANNER':
        probe = utils.get_nmap_probes('tcp').get(
            utils.nmap_decode_data(spec['value']))
        if probe is not None:
            spec.setdefault('infos', {}).update({
                'service_name':
                'scanner',
                'service_product':
                'Nmap',
                'service_extrainfo':
                'TCP probe %s' % probe,
            })
    elif spec['recontype'] == 'UDP_HONEYPOT_HIT':
        data = utils.nmap_decode_data(spec['value'])
        probe = utils.get_nmap_probes('udp').get(data)
        if probe is not None:
            spec.setdefault('infos', {}).update({
                'service_name':
                'scanner',
                'service_product':
                'Nmap',
                'service_extrainfo':
                'UDP probe %s' % probe,
            })
        else:
            payload = utils.get_nmap_udp_payloads().get(data)
            if payload is not None:
                spec.setdefault('infos', {}).update({
                    'service_name':
                    'scanner',
                    'service_product':
                    'Nmap',
                    'service_extrainfo':
                    'UDP payload %s' % payload,
                })
    # SSL_{CLIENT,SERVER} JA3
    elif ((spec['recontype'] == 'SSL_CLIENT' and spec['source'] == 'ja3')
          or (spec['recontype'] == 'SSL_SERVER'
              and spec['source'].startswith('ja3-'))):
        value = spec['value']
        spec.setdefault('infos', {})['raw'] = value
        spec['value'] = hashlib.new("md5", value.encode()).hexdigest()
        if spec['recontype'] == 'SSL_SERVER':
            clientvalue = spec['source'][4:]
            spec['infos'].setdefault('client', {})['raw'] = clientvalue
            spec['source'] = 'ja3-%s' % hashlib.new(
                "md5",
                clientvalue.encode(),
            ).hexdigest()
    # SSH_{CLIENT,SERVER}_HASSH
    elif spec['recontype'] in ['SSH_CLIENT_HASSH', 'SSH_SERVER_HASSH']:
        value = spec['value']
        spec.setdefault('infos', {})['raw'] = value
        spec['value'] = hashlib.new("md5", value.encode()).hexdigest()
    # Check DNS Blacklist answer
    elif spec['recontype'] == 'DNS_ANSWER':
        if any((spec.get('value') or "").endswith(dnsbl)
               for dnsbl in config.DNS_BLACKLIST_DOMAINS):
            dnsbl_val = spec['value']
            match = DNSBL_START.search(dnsbl_val)
            if match is not None:
                spec['recontype'] = 'DNS_BLACKLIST'
                spec['value'] = spec.get('addr')
                spec.update({
                    'source':
                    "%s-%s" % (dnsbl_val[match.end():], spec['source'])
                })
                addr = match.group()
                # IPv4
                if addr.count('.') == 4:
                    spec['addr'] = '.'.join(addr.split('.')[3::-1])
                # IPv6
                else:
                    spec['addr'] = utils.int2ip6(
                        int(addr.replace('.', '')[::-1], 16))
    return spec
Exemple #2
0
def _prepare_rec(spec, ignorenets, neverignore):
    # First of all, let's see if we are supposed to ignore this spec,
    # and if so, do so.
    if "addr" in spec and spec.get("source") not in neverignore.get(
            spec["recontype"], []):
        for start, stop in ignorenets.get(spec["recontype"], []):
            if start <= utils.force_ip2int(spec["addr"]) <= stop:
                return
    # Then, let's clean up the records.
    # Change Symantec's random user agents (matching SYMANTEC_UA) to
    # the constant string "SymantecRandomUserAgent".
    if spec["recontype"] == "HTTP_CLIENT_HEADER" and spec.get(
            "source") == "USER-AGENT":
        if SYMANTEC_UA.match(spec["value"]):
            spec["value"] = "SymantecRandomUserAgent"
        elif KASPERSKY_UA.match(spec["value"]):
            spec["value"] = "KasperskyWeirdUserAgent"
        else:
            match = SYMANTEC_SEP_UA.match(spec["value"])
            if match is not None:
                spec["value"] = "%s%s" % match.groups()
    # Change any Digest authorization header to remove non-constant
    # information. On one hand we loose the necessary information to
    # try to recover the passwords, but on the other hand we store
    # specs with different challenges but the same username, realm,
    # host and sensor in the same records.
    elif spec["recontype"] in {
            "HTTP_CLIENT_HEADER",
            "HTTP_CLIENT_HEADER_SERVER",
    } and spec.get("source") in {"AUTHORIZATION", "PROXY-AUTHORIZATION"}:
        value = spec["value"]
        if value:
            authtype = value.split(None, 1)[0]
            if authtype.lower() == "digest":
                try:
                    # we only keep relevant info
                    spec["value"] = "%s %s" % (
                        authtype,
                        ",".join(
                            val
                            for val in _split_digest_auth(value[6:].strip())
                            if DIGEST_AUTH_INFOS.match(val)),
                    )
                except Exception:
                    utils.LOGGER.warning("Cannot parse digest error for %r",
                                         spec,
                                         exc_info=True)
            elif ntlm._is_ntlm_message(value):
                # NTLM_NEGOTIATE and NTLM_AUTHENTICATE
                yield from _prepare_rec_ntlm(spec, "NTLM_CLIENT_FLAGS")
                return
            elif authtype.lower() in {"negotiate", "kerberos", "oauth"}:
                spec["value"] = authtype
    elif spec["recontype"] == "HTTP_SERVER_HEADER" and spec.get("source") in {
            "WWW-AUTHENTICATE",
            "PROXY-AUTHENTICATE",
    }:
        value = spec["value"]
        if value:
            authtype = value.split(None, 1)[0]
            if authtype.lower() == "digest":
                try:
                    # we only keep relevant info
                    spec["value"] = "%s %s" % (
                        authtype,
                        ",".join(
                            val
                            for val in _split_digest_auth(value[6:].strip())
                            if DIGEST_AUTH_INFOS.match(val)),
                    )
                except Exception:
                    utils.LOGGER.warning("Cannot parse digest error for %r",
                                         spec,
                                         exc_info=True)
            elif ntlm._is_ntlm_message(value):
                # NTLM_CHALLENGE
                yield from _prepare_rec_ntlm(spec, "NTLM_SERVER_FLAGS")
                return
            elif authtype.lower() in {"negotiate", "kerberos", "oauth"}:
                spec["value"] = authtype
    # TCP server banners: try to normalize data
    elif spec["recontype"] == "TCP_SERVER_BANNER":
        newvalue = value = utils.nmap_decode_data(spec["value"])
        for pattern, replace in TCP_SERVER_PATTERNS:
            if pattern.search(newvalue):
                newvalue = pattern.sub(replace, newvalue)
        if newvalue != value:
            spec["value"] = utils.nmap_encode_data(newvalue)
    elif spec["recontype"] in {"TCP_CLIENT_BANNER", "TCP_HONEYPOT_HIT"}:
        if spec["value"]:
            data = utils.nmap_decode_data(spec["value"])
            if data in scanners.TCP_PROBES:
                scanner, probe = scanners.TCP_PROBES[data]
                info = {
                    "service_name": "scanner",
                    "service_product": scanner,
                }
                if probe is not None:
                    info["service_extrainfo"] = "TCP probe %s" % probe
                spec.setdefault("infos", {}).update(info)
            else:
                probe = utils.get_nmap_probes("tcp").get(data)
                if probe is not None:
                    spec.setdefault("infos", {}).update({
                        "service_name":
                        "scanner",
                        "service_product":
                        "Nmap",
                        "service_extrainfo":
                        "TCP probe %s" % probe,
                    })
    elif spec["recontype"] == "UDP_HONEYPOT_HIT":
        data = utils.nmap_decode_data(spec["value"])
        if data in scanners.UDP_PROBES:
            scanner, probe = scanners.UDP_PROBES[data]
            info = {
                "service_name": "scanner",
                "service_product": scanner,
            }
            if probe is not None:
                info["service_extrainfo"] = "UDP probe %s" % probe
            spec.setdefault("infos", {}).update(info)
        else:
            probe = utils.get_nmap_probes("udp").get(data)
            if probe is not None:
                spec.setdefault("infos", {}).update({
                    "service_name":
                    "scanner",
                    "service_product":
                    "Nmap",
                    "service_extrainfo":
                    "UDP probe %s" % probe,
                })
            else:
                payload = utils.get_nmap_udp_payloads().get(data)
                if payload is not None:
                    spec.setdefault("infos", {}).update({
                        "service_name":
                        "scanner",
                        "service_product":
                        "Nmap",
                        "service_extrainfo":
                        "UDP payload %s" % payload,
                    })
    elif spec["recontype"] == "STUN_HONEYPOT_REQUEST":
        spec["value"] = utils.nmap_decode_data(spec["value"])
    # SSL_{CLIENT,SERVER} JA3
    elif (spec["recontype"] == "SSL_CLIENT" and spec["source"]
          == "ja3") or (spec["recontype"] == "SSL_SERVER"
                        and spec["source"].startswith("ja3-")):
        value = spec["value"]
        spec.setdefault("infos", {})["raw"] = value
        spec["value"] = hashlib.new("md5", value.encode()).hexdigest()
        if spec["recontype"] == "SSL_SERVER":
            clientvalue = spec["source"][4:]
            spec["infos"].setdefault("client", {})["raw"] = clientvalue
            spec["source"] = ("ja3-%s" % hashlib.new(
                "md5",
                clientvalue.encode(),
            ).hexdigest())
    # SSH_{CLIENT,SERVER}_HASSH
    elif spec["recontype"] in ["SSH_CLIENT_HASSH", "SSH_SERVER_HASSH"]:
        value = spec["value"]
        spec.setdefault("infos", {})["raw"] = value
        spec["value"] = hashlib.new("md5", value.encode()).hexdigest()
    # Check DNS Blacklist answer
    elif spec["recontype"] == "DNS_ANSWER":
        if any((spec.get("value") or "").endswith(dnsbl)
               for dnsbl in config.DNS_BLACKLIST_DOMAINS):
            dnsbl_val = spec["value"]
            match = DNSBL_START.search(dnsbl_val)
            if match is not None:
                spec["recontype"] = "DNS_BLACKLIST"
                spec["value"] = spec.get("addr")
                spec["source"] = "%s-%s" % (dnsbl_val[match.end():],
                                            spec["source"])
                addr = match.group()
                # IPv4
                if addr.count(".") == 4:
                    spec["addr"] = ".".join(addr.split(".")[3::-1])
                # IPv6
                else:
                    spec["addr"] = utils.int2ip6(
                        int(addr.replace(".", "")[::-1], 16))
    yield spec