def _getinfos_http_client_authorization(spec): """Extract (for now) the usernames and passwords from Basic authorization headers """ infos = {} data = spec["value"].split(None, 1) value = spec["value"] if data[1:]: if data[0].lower() == "basic": try: infos["username"], infos["password"] = ( utils.nmap_encode_data(v) for v in utils.decode_b64(data[1].strip().encode()).split( b":", 1)) except Exception: pass elif data[0].lower() == "digest": try: infos = dict( value.split("=", 1) if "=" in value else [value, None] for value in _split_digest_auth(data[1].strip())) for key, value in list(infos.items()): if value.startswith('"') and value.endswith('"'): infos[key] = value[1:-1] except Exception: pass elif (value[:4].lower() == "ntlm" and value[4:].strip()) or (value[:9].lower() == "negotiate" and value[9:].strip()): return _getinfos_ntlm(spec) res = {} if infos: res["infos"] = infos return res
def _prepare_rec_ntlm(spec, new_recontype): """ Decode NTLM messages in HTTP headers and split fingerprint from the other NTLM info in the spec """ try: auth = utils.decode_b64(spec['value'].split(None, 1)[1].encode()) except (UnicodeDecodeError, TypeError, ValueError, binascii.Error): pass spec['value'] = "%s %s" % \ (spec['value'].split(None, 1)[0], ntlm._ntlm_dict2string(ntlm.ntlm_extract_info(auth))) # Separate the NTLM flags from the rest of the message's info # for NTLMSSP_NEGOTIAGE and NTLMSSP_CHALLENGE messages if spec['value'].startswith('NTLM ntlm-fingerprint'): fingerprint = spec.copy() fingerprint['recontype'] = new_recontype try: fingerprint['value'], spec['value'] = \ spec['value'].split(',', 1) except ValueError: spec['value'] = '' else: spec['value'] = "NTLM %s" % spec['value'] fingerprint['value'] = fingerprint['value'][5:] yield fingerprint yield spec
def _getinfos_http_client_authorization(spec): """Extract (for now) the usernames and passwords from Basic authorization headers """ infos = {} data = spec['value'].split(None, 1) value = spec['value'] if data[1:]: if data[0].lower() == 'basic': try: infos['username'], infos['password'] = ( utils.nmap_encode_data(v) for v in utils.decode_b64(data[1].strip().encode()).split( b':', 1)) except Exception: pass elif data[0].lower() == 'digest': try: infos = dict( value.split('=', 1) if '=' in value else [value, None] for value in _split_digest_auth(data[1].strip())) for key, value in list(viewitems(infos)): if value.startswith('"') and value.endswith('"'): infos[key] = value[1:-1] except Exception: pass elif (value[:4].lower() == 'ntlm' and value[4:].strip()) or \ (value[:9].lower() == 'negotiate' and value[9:].strip()): spec['value'] = spec['value'].split(None, 1)[1] return _getinfos_ntlm(spec) res = {} if infos: res['infos'] = infos return res
def _getinfos_ntlm(spec): """ Get information from NTLMSSP messages """ info = {} try: for k, v in (item.split(":", 1) for item in spec["value"].split(",")): if k == "NTLM_Version": try: info[k] = int(v) except ValueError: utils.LOGGER.warning( "Incorrect value for field %r in record %r", k, spec) else: try: info[k] = utils.nmap_encode_data( utils.decode_b64(v.encode())) except (UnicodeDecodeError, TypeError, ValueError, binascii.Error): utils.LOGGER.warning( "Incorrect value for field %r in record %r", k, spec) except ValueError: utils.LOGGER.warning("Incorrect value in message: %r", spec) return {} return {"infos": info}
def _getinfos_cert(spec): """Extract info from a certificate (hash values, issuer, subject, algorithm) in an handy-to-index-and-query form. """ infos = {} fullinfos = {} try: cert = utils.decode_b64(spec.get('fullvalue', spec['value'])) except Exception: return {} for hashtype in ['md5', 'sha1']: infos['%shash' % hashtype] = hashlib.new(hashtype, cert).hexdigest() proc = subprocess.Popen(['openssl', 'x509', '-noout', '-text', '-inform', 'DER'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) proc.stdin.write(cert) proc.stdin.close() try: newinfos = _CERTINFOS.search(proc.stdout.read()).groupdict() newfullinfos = {} for field in newinfos: if len(newinfos[field]) > utils.MAXVALLEN: newfullinfos[field] = newinfos[field] newinfos[field] = newinfos[field][:utils.MAXVALLEN] infos.update(newinfos) fullinfos.update(newfullinfos) except Exception: pass res = {} if infos: res['infos'] = infos if fullinfos: res['fullinfos'] = fullinfos return res
def _getinfos_ntlm(spec): """ Get information from NTLM_CHALLENGE and NTLM_AUTHENTICATE messages """ info = {} try: for k, v in (item.split(':', 1) for item in spec['value'].split(',')): if k == 'NTLM_Version': try: info[k] = int(v) except ValueError: utils.LOGGER.warning( "Incorrect value for field %r in record %r", k, spec) else: try: info[k] = utils.nmap_encode_data( utils.decode_b64(v.encode())) except (UnicodeDecodeError, TypeError, ValueError, binascii.Error): utils.LOGGER.warning( "Incorrect value for field %r in record %r", k, spec) except ValueError: utils.LOGGER.warning("Incorrect value in message: %r", spec) return {} return {'infos': info}
def _getinfos_http_client_authorization(spec): """Extract (for now) the usernames and passwords from Basic authorization headers """ infos = {} data = spec['value'].split(None, 1) if data[1:]: if data[0].lower() == b'basic': try: infos['username'], infos['password'] = utils.decode_b64( data[1].strip() ).decode('latin-1').split(':', 1) except Exception: pass elif data[0].lower() == 'digest': try: infos = dict( value.split('=', 1) if '=' in value else [value, None] for value in _split_digest_auth(data[1].strip()) ) for key, value in list(viewitems(infos)): if value.startswith('"') and value.endswith('"'): infos[key] = value[1:-1] except Exception: pass res = {} if infos: res['infos'] = infos return res
def _getinfos_cert(spec): """Extract info from a certificate (hash values, issuer, subject, algorithm) in an handy-to-index-and-query form. """ try: cert = utils.decode_b64(spec.get('fullvalue', spec['value']).encode()) except Exception: utils.LOGGER.info("Cannot parse certificate for record %r", spec, exc_info=True) return {} info = utils.get_cert_info(cert) fullinfo = {} for key, value in list(viewitems(info)): if key in ['issuer', 'subject']: for skey, svalue in list(viewitems(value)): if len(svalue) > utils.MAXVALLEN: fullinfo.setdefault(key, {})[skey] = svalue info[key][skey] = svalue[:utils.MAXVALLEN] elif len(value) > utils.MAXVALLEN: fullinfo[key] = value info[key] = value[:utils.MAXVALLEN] res = {} if info: res['infos'] = info if fullinfo: res['fullinfos'] = fullinfo return res
def _prepare_rec_ntlm(spec, new_recontype): """ Decode NTLM messages in HTTP headers and split fingerprint from the other NTLM info in the spec """ try: auth = utils.decode_b64(spec["value"].split(None, 1)[1].encode()) except (UnicodeDecodeError, TypeError, ValueError, binascii.Error): utils.LOGGER.warning("_prepare_rec_ntlm(): cannot decode %r", spec["value"], exc_info=True) return spec["value"] = "%s %s" % ( spec["value"].split(None, 1)[0], ntlm._ntlm_dict2string(ntlm.ntlm_extract_info(auth)), ) # Separate the NTLM flags from the rest of the message's info # for NTLMSSP_NEGOTIAGE and NTLMSSP_CHALLENGE messages if spec["value"].startswith("NTLM ntlm-fingerprint"): fingerprint = spec.copy() fingerprint["recontype"] = new_recontype try: fingerprint["value"], spec["value"] = spec["value"].split(",", 1) except ValueError: spec["value"] = "" else: spec["value"] = "NTLM %s" % spec["value"] fingerprint["value"] = fingerprint["value"][5:] yield fingerprint yield spec
def _getinfos_smb(spec): """ Get information on an OS from SMB `Session Setup Request` and `Session Setup Response` """ info = {} try: for k, v in (item.split(':', 1) for item in spec['value'].split(',')): if k == 'is_guest': try: info[k] = v == 'true' except ValueError: utils.LOGGER.warning( "Incorrect value for field %r in record %r", k, spec) else: try: info[k] = utils.nmap_encode_data( utils.decode_b64(v.encode())) except (UnicodeDecodeError, TypeError, ValueError, binascii.Error): utils.LOGGER.warning( "Incorrect value for field %r in record %r", k, spec) except ValueError: utils.LOGGER.warning("Incorrect value in message: %r", spec) return {} return {'infos': info}
def _getinfos_cert(spec): """Extract info from a certificate (hash values, issuer, subject, algorithm) in an handy-to-index-and-query form. """ infos = {} fullinfos = {} try: cert = utils.decode_b64(spec.get('fullvalue', spec['value'])) except Exception: return {} for hashtype in ['md5', 'sha1']: infos['%shash' % hashtype] = hashlib.new(hashtype, cert).hexdigest() proc = subprocess.Popen(['openssl', 'x509', '-noout', '-text', '-inform', 'DER'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) proc.stdin.write(cert) proc.stdin.close() try: newinfos = _CERTINFOS.search(proc.stdout.read()).groupdict() newfullinfos = {} for field in newinfos: if len(newinfos[field]) > utils.MAXVALLEN: newfullinfos[field] = newinfos[field] newinfos[field] = newinfos[field][:utils.MAXVALLEN] infos.update(newinfos) fullinfos.update(newfullinfos) except Exception: pass res = {} if infos: res['infos'] = infos if fullinfos: res['fullinfos'] = fullinfos return res
def getkeys(self, host): for script in self.getscripts(host): for key in script['script'][self.scriptid]: if key['type'][4:] == self.keytype: data = utils.decode_b64(key['key']) # Handle bug (in Nmap?) where data gets encoded # twice. if data[0] != b'\x00': data = utils.decode_b64(data) yield Key( utils.int2ip(host['addr']), script["port"], "ssh", key['type'][4:], int(key['bits']), self.data2key(data), utils.decode_hex(key['fingerprint']), )
def _getinfos_http_client_authorization(spec): """Extract (for now) the usernames and passwords from Basic authorization headers """ infos = {} data = spec['value'].split(None, 1) if data[1:]: if data[0].lower() == b'basic': try: infos['username'], infos['password'] = utils.decode_b64( data[1].strip() ).decode('latin-1').split(':', 1) except Exception: pass elif data[0].lower() == 'digest': try: infos = dict( value.split('=', 1) if '=' in value else [value, None] for value in _split_digest_auth(data[1].strip()) ) for key, value in list(viewitems(infos)): if value.startswith('"') and value.endswith('"'): infos[key] = value[1:-1] except Exception: pass res = {} if infos: res['infos'] = infos return res
def _getinfos_ntlm(spec): """ Get information from NTLMSSP messages """ value = spec["value"] try: val1, val2 = value.split(None, 1) except ValueError: pass else: if val1.lower() in {"ntlm", "negotiate"} and val2: value = val2 info = {} try: for k, v in (item.split(":", 1) for item in value.split(",")): if k == "NTLM_Version": info[k] = v else: try: info[k] = utils.nmap_encode_data( utils.decode_b64(v.encode())) except (UnicodeDecodeError, TypeError, ValueError, binascii.Error): utils.LOGGER.warning( "Incorrect value for field %r in record %r", k, spec) except ValueError: utils.LOGGER.warning("Incorrect value in message: %r", spec) return {} return {"infos": info}
def getkeys(self, host): for script in self.getscripts(host): for key in script['script'][self.scriptid]: if key['type'][4:] == self.keytype: data = utils.decode_b64(key['key'].encode()) # Handle bug (in Nmap?) where data gets encoded # twice. if data[:1] != b'\x00': data = utils.decode_b64(data) yield Key( host['addr'], script["port"], "ssh", key['type'][4:], int(float(key['bits'])), # for some reason, # Nmap sometimes # outputs 1024.0 self.data2key(data), utils.decode_hex(key['fingerprint']), )
def _getinfos_ssh_hostkey(spec): """Parse SSH host keys.""" infos = {} data = utils.decode_b64(spec["value"].encode()) for hashtype in ["md5", "sha1", "sha256"]: infos[hashtype] = hashlib.new(hashtype, data).hexdigest() info = utils.parse_ssh_key(data) return {"infos": info}
def read_pem(self, pem): pem = utils.decode_b64(self.pem_borders.sub(b"", pem)) proc = subprocess.Popen(['openssl', 'x509', '-noout', '-text', '-inform', 'DER'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) proc.stdin.write(pem) proc.stdin.close() return proc.stdout.read()
def getkeys(self, host): for script in self.getscripts(host): for key in script['script'][self.scriptid]: if key['type'][4:] == self.keytype: data = utils.decode_b64(key['key'].encode()) # Handle bug (in Nmap?) where data gets encoded # twice. if data[:1] != b'\x00': data = utils.decode_b64(data) yield Key( host['addr'], script["port"], "ssh", key['type'][4:], int(float(key['bits'])), # for some reason, # Nmap sometimes # outputs 1024.0 self.data2key(data), utils.decode_hex(key['fingerprint']), )
def read_pem(self, pem): pem = utils.decode_b64(self.pem_borders.sub(b"", pem)) proc = subprocess.Popen( ['openssl', 'x509', '-noout', '-text', '-inform', 'DER'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) proc.stdin.write(pem) proc.stdin.close() return proc.stdout.read()
def getkeys(self, host): for script in self.getscripts(host): for key in script['script'][self.scriptid]: if key['type'][4:] == self.keytype: data = utils.decode_b64(key['key']) # Handle bug (in Nmap?) where data gets encoded # twice. if data[0] != b'\x00': data = utils.decode_b64(data) yield Key( utils.int2ip(host['addr']), script["port"], "ssh", key['type'][4:], int(key['bits']), self.data2key(data), utils.decode_hex(key['fingerprint']), )
def getkeys(self, record: Record) -> Generator[Key, None, None]: for script in self.getscripts(record): assert isinstance(script["script"], dict) for key in script["script"][self.scriptid]: if key["type"][4:] == self.keytype: data = utils.decode_b64(key["key"].encode()) # Handle bug (in Nmap?) where data gets encoded # twice. if data[:1] != b"\x00": data = utils.decode_b64(data) yield Key( record["addr"], script["port"], "ssh", key["type"][4:], int(float(key["bits"])), # for some reason, # Nmap sometimes # outputs 1024.0 self.data2key(data), utils.decode_hex(key["fingerprint"]), )
def read_pem(cls, pem): try: pem = pem.encode() except AttributeError: pass pem = utils.decode_b64(cls.pem_borders.sub(b"", pem)) proc = subprocess.Popen([config.OPENSSL_CMD, 'x509', '-noout', '-text', '-inform', 'DER'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) proc.stdin.write(pem) proc.stdin.close() return proc.stdout.read()
def read_pem(cls, pem): try: pem = pem.encode() except AttributeError: pass pem = utils.decode_b64(cls.pem_borders.sub(b"", pem)) proc = subprocess.Popen([config.OPENSSL_CMD, 'x509', '-noout', '-text', '-inform', 'DER'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) proc.stdin.write(pem) proc.stdin.close() return proc.stdout.read()
def _getinfos_ntlm(spec): """ Get infos from the NTLMSSP_CHALLENGE matching the smb-os-discovery scripts form Masscan and Nmap """ return { 'infos': { k: (int(v) if k == 'ntlm-version' else utils.decode_b64( v.encode()).decode()) for k, v in (item.split(':', 1) for item in spec['value'].split(',')) if v } }
def _is_ntlm_message(message): """ Checks whether the given string is an NTLM message """ if message[:4].lower() == 'ntlm' and message[4:].strip(): return True if message[:9].lower() == 'negotiate': message = message.split(None, 1) if message[1:]: try: return utils.decode_b64(message[1].encode())[:7] == b'NTLMSSP' except (UnicodeDecodeError, TypeError, ValueError, binascii.Error): pass return False
def _getinfos_cert(spec): """Extract info from a certificate (hash values, issuer, subject, algorithm) in an handy-to-index-and-query form. """ # TODO: move to mongodb specific functions. try: cert = utils.decode_b64(spec["value"].encode()) except Exception: utils.LOGGER.info("Cannot parse certificate for record %r", spec, exc_info=True) return {} info = utils.get_cert_info(cert) res = {} if info: res["infos"] = info return res
def read_pem(cls, pem: AnyStr) -> bytes: if isinstance(pem, str): raw_cert = pem.encode() else: raw_cert = pem raw_cert = utils.decode_b64(cls.pem_borders.sub(b"", raw_cert)) with subprocess.Popen( [config.OPENSSL_CMD, "x509", "-noout", "-text", "-inform", "DER"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, ) as proc: assert proc.stdin is not None assert proc.stdout is not None proc.stdin.write(raw_cert) proc.stdin.close() return proc.stdout.read()
def _getinfos_cert(spec): """Extract info from a certificate (hash values, issuer, subject, algorithm) in an handy-to-index-and-query form. """ # TODO: move to mongodb specific functions. try: cert = utils.decode_b64(spec['value'].encode()) except Exception: utils.LOGGER.info("Cannot parse certificate for record %r", spec, exc_info=True) return {} info = utils.get_cert_info(cert) res = {} if info: res['infos'] = info return res
def getkeys(self, record: Filter) -> Generator[Key, None, None]: certtext = self._der2key(utils.decode_b64(record["value"].encode())) if certtext is None: return yield Key( record["addr"], record["port"], "ssl", certtext["type"], int(certtext["len"]), _rsa_construct( int(certtext["exponent"]), int(MODULUS_BADCHARS.sub(b"", certtext["modulus"]), 16), ), utils.decode_hex(record["infos"]["md5"]), )
def _is_ntlm_message(message): """ Checks whether the given string is an NTLM message """ try: val1, val2 = message.split(None, 1) except ValueError: return False else: if not val2: return False if val1.lower() == "ntlm": return True if val1.lower() == "negotiate": try: return utils.decode_b64(val2.encode())[:7] == b"NTLMSSP" except (UnicodeDecodeError, TypeError, ValueError, binascii.Error): pass return False
def _getinfos_cert(spec, to_binary): """Extract info from a certificate (hash values, issuer, subject, algorithm) in an handy-to-index-and-query form. """ try: cert = utils.decode_b64(spec.pop('fullvalue', spec['value']).encode()) except Exception: utils.LOGGER.info("Cannot parse certificate for record %r", spec, exc_info=True) return {} if len(cert) > utils.MAXVALLEN: spec['fullvalue'] = to_binary(cert) spec['value'] = hashlib.sha1(cert).hexdigest() else: spec['value'] = to_binary(cert) info = utils.get_cert_info(cert) fullinfo = {} for key, value in list(viewitems(info)): if key in ['issuer', 'subject']: for skey, svalue in list(viewitems(value)): # We enforce a utils.MAXVALLEN // 10 size limits for # subkey values in subject and issuer; this is because # MongoDB cannot index values longer than 1024 bytes, # and subject and issuer fields may have more than one # key. if len(svalue) > utils.MAXVALLEN // 10: fullinfo.setdefault(key, {})[skey] = svalue info[key][skey] = svalue[:utils.MAXVALLEN // 10] elif len(value) > utils.MAXVALLEN: fullinfo[key] = value info[key] = value[:utils.MAXVALLEN] res = {} if info: res['infos'] = info if fullinfo: res['fullinfos'] = fullinfo return res
def zgrap_parser_http(data, hostrec, port=None): """This function handles data from `{"data": {"http": [...]}}` records. `data` should be the content, i.e. the `[...]`. It should consist of simple dictionary, that may contain a `"response"` key and/or a `"redirect_response_chain"` key. The output is a port dict (i.e., the content of the "ports" key of an `nmap` of `view` record in IVRE), that may be empty. """ if not data: return {} # for zgrab2 results if "result" in data: data.update(data.pop("result")) if "response" not in data: utils.LOGGER.warning('Missing "response" field in zgrab HTTP result') return {} resp = data["response"] needed_fields = set(["request", "status_code", "status_line"]) missing_fields = needed_fields.difference(resp) if missing_fields: utils.LOGGER.warning( "Missing field%s %s in zgrab HTTP result", "s" if len(missing_fields) > 1 else "", ", ".join(repr(fld) for fld in missing_fields), ) return {} req = resp["request"] url = req.get("url") res = { "service_name": "http", "service_method": "probed", "state_state": "open", "state_reason": "response", "protocol": "tcp", } tls = None try: tls = req["tls_handshake"] except KeyError: # zgrab2 try: tls = req["tls_log"]["handshake_log"] except KeyError: pass if tls is not None: res["service_tunnel"] = "ssl" try: cert = tls["server_certificates"]["certificate"]["raw"] except KeyError: pass else: output, info = create_ssl_cert(cert.encode(), b64encoded=True) if info: res.setdefault("scripts", []).append( { "id": "ssl-cert", "output": output, "ssl-cert": info, } ) for cert in info: add_cert_hostnames(cert, hostrec.setdefault("hostnames", [])) if url: guessed_port = None if ":" in url.get("host", ""): try: guessed_port = int(url["host"].split(":", 1)[1]) except ValueError: pass if port is None: if guessed_port is None: if url.get("scheme") == "https": port = 443 else: port = 80 else: port = guessed_port elif port != guessed_port: utils.LOGGER.warning( "Port %d found from the URL %s differs from the provided port " "value %d", guessed_port, url.get("path"), port, ) port = guessed_port # Specific paths if url.get("path").endswith("/.git/index"): if resp.get("status_code") != 200: return {} if not resp.get("body", "").startswith("DIRC"): return {} # Due to an issue with ZGrab2 output, we cannot, for now, # process the content of the file. See # <https://github.com/zmap/zgrab2/issues/263>. repository = "%s:%d%s" % (hostrec["addr"], port, url["path"][:-5]) res["port"] = port res.setdefault("scripts", []).append( { "id": "http-git", "output": "\n %s\n Git repository found!\n" % repository, "http-git": [ {"repository": repository, "files-found": [".git/index"]} ], } ) return res if url.get("path").endswith("/owa/auth/logon.aspx"): if resp.get("status_code") != 200: return {} version = set( m.group(1) for m in _EXPR_OWA_VERSION.finditer(resp.get("body", "")) ) if not version: return {} version = sorted(version, key=lambda v: [int(x) for x in v.split(".")]) res["port"] = port path = url["path"][:-15] if len(version) > 1: output = "OWA: path %s, version %s (multiple versions found!)" % ( path, " / ".join(version), ) else: output = "OWA: path %s, version %s" % (path, version[0]) res.setdefault("scripts", []).append( { "id": "http-app", "output": output, "http-app": [ {"path": path, "application": "OWA", "version": version[0]} ], } ) return res if url.get("path").endswith("/centreon/"): if resp.get("status_code") != 200: return {} if not resp.get("body"): return {} body = resp["body"] res["port"] = port path = url["path"] match = _EXPR_TITLE.search(body) if match is None: return {} if match.groups()[0] != "Centreon - IT & Network Monitoring": return {} match = _EXPR_CENTREON_VERSION.search(body) if match is None: version = None else: version = match.group(1) or match.group(2) res.setdefault("scripts", []).append( { "id": "http-app", "output": "Centreon: path %s%s" % ( path, "" if version is None else (", version %s" % version), ), "http-app": [ dict( {"path": path, "application": "Centreon"}, **({} if version is None else {"version": version}), ) ], } ) return res if url.get("path") != "/": utils.LOGGER.warning("URL path not supported yet: %s", url.get("path")) return {} elif port is None: if req.get("tls_handshake") or req.get("tls_log"): port = 443 else: port = 80 res["port"] = port # Since Zgrab does not preserve the order of the headers, we need # to reconstruct a banner to use Nmap fingerprints banner = ( utils.nmap_decode_data(resp["protocol"]["name"]) + b" " + utils.nmap_decode_data(resp["status_line"]) + b"\r\n" ) if resp.get("headers"): headers = resp["headers"] # Check the Authenticate header first: if we requested it with # an Authorization header, we don't want to gather other information if headers.get("www_authenticate"): auths = headers.get("www_authenticate") for auth in auths: if ntlm._is_ntlm_message(auth): try: infos = ntlm.ntlm_extract_info( utils.decode_b64(auth.split(None, 1)[1].encode()) ) except (UnicodeDecodeError, TypeError, ValueError, binascii.Error): pass keyvals = zip(ntlm_values, [infos.get(k) for k in ntlm_values]) output = "\n".join("{}: {}".format(k, v) for k, v in keyvals if v) res.setdefault("scripts", []).append( {"id": "http-ntlm-info", "output": output, "ntlm-info": infos} ) if "DNS_Computer_Name" in infos: add_hostname( infos["DNS_Computer_Name"], "ntlm", hostrec.setdefault("hostnames", []), ) if any( val.lower().startswith("ntlm") for val in req.get("headers", {}).get("authorization", []) ): return res # the order will be incorrect! line = "%s %s" % (resp["protocol"]["name"], resp["status_line"]) http_hdrs = [{"name": "_status", "value": line}] output = [line] for unk in headers.pop("unknown", []): headers[unk["key"]] = unk["value"] for hdr, values in headers.items(): hdr = hdr.replace("_", "-") for val in values: http_hdrs.append({"name": hdr, "value": val}) output.append("%s: %s" % (hdr, val)) if http_hdrs: method = req.get("method") if method: output.append("") output.append("(Request type: %s)" % method) res.setdefault("scripts", []).append( { "id": "http-headers", "output": "\n".join(output), "http-headers": http_hdrs, } ) handle_http_headers(hostrec, res, http_hdrs, path=url.get("path")) if headers.get("server"): banner += ( b"Server: " + utils.nmap_decode_data(headers["server"][0]) + b"\r\n\r\n" ) info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest") if info: add_cpe_values(hostrec, "ports.port:%s" % port, info.pop("cpe", [])) res.update(info) if resp.get("body"): body = resp["body"] res.setdefault("scripts", []).append( { "id": "http-content", "output": utils.nmap_encode_data(body.encode()), } ) match = _EXPR_TITLE.search(body) if match is not None: title = match.groups()[0] res["scripts"].append( { "id": "http-title", "output": title, "http-title": {"title": title}, } ) script_http_ls = create_http_ls(body, url=url) if script_http_ls is not None: res.setdefault("scripts", []).append(script_http_ls) service_elasticsearch = create_elasticsearch_service(body) if service_elasticsearch: if "hostname" in service_elasticsearch: add_hostname( service_elasticsearch.pop("hostname"), "service", hostrec.setdefault("hostnames", []), ) add_cpe_values( hostrec, "ports.port:%s" % port, service_elasticsearch.pop("cpe", []) ) res.update(service_elasticsearch) return res
def from_binary(data): return utils.decode_b64(data.encode())
def _prepare_rec(spec, ignorenets, neverignore): # First of all, let's see if we are supposed to ignore this spec, # and if so, do so. if 'addr' in spec and \ spec.get('source') not in neverignore.get(spec['recontype'], []): for start, stop in ignorenets.get(spec['recontype'], ()): if start <= utils.force_ip2int(spec['addr']) <= stop: return None # Then, let's clean up the records. # Change Symantec's random user agents (matching SYMANTEC_UA) to # the constant string 'SymantecRandomUserAgent'. if spec['recontype'] == 'HTTP_CLIENT_HEADER' and \ spec.get('source') == 'USER-AGENT': if SYMANTEC_UA.match(spec['value']): spec['value'] = 'SymantecRandomUserAgent' elif KASPERSKY_UA.match(spec['value']): spec['value'] = 'KasperskyWeirdUserAgent' else: match = SYMANTEC_SEP_UA.match(spec['value']) if match is not None: spec['value'] = '%s%s' % match.groups() # Change any Digest authorization header to remove non-constant # information. On one hand we loose the necessary information to # try to recover the passwords, but on the other hand we store # specs with different challenges but the same username, realm, # host and sensor in the same records. elif (spec['recontype'] in {'HTTP_CLIENT_HEADER', 'HTTP_CLIENT_HEADER_SERVER'} and spec.get('source') in {'AUTHORIZATION', 'PROXY-AUTHORIZATION'}): value = spec['value'] if value: authtype = value.split(None, 1)[0] if authtype.lower() == 'digest': try: # we only keep relevant info spec['value'] = '%s %s' % (authtype, ','.join( val for val in _split_digest_auth(value[6:].strip()) if DIGEST_AUTH_INFOS.match(val))) except Exception: utils.LOGGER.warning("Cannot parse digest error for %r", spec, exc_info=True) elif ntlm._is_ntlm_message(value): # NTLM_NEGOTIATE and NTLM_AUTHENTICATE try: auth = utils.decode_b64(value.split(None, 1)[1].encode()) except (UnicodeDecodeError, TypeError, ValueError, binascii.Error): pass spec['value'] = "%s %s" % \ (value.split(None, 1)[0], ntlm._ntlm_dict2string(ntlm.ntlm_extract_info(auth))) elif authtype.lower() in {'negotiate', 'kerberos', 'oauth'}: spec['value'] = authtype elif (spec['recontype'] == 'HTTP_SERVER_HEADER' and spec.get('source') in {'WWW-AUTHENTICATE', 'PROXY-AUTHENTICATE'}): value = spec['value'] if value: authtype = value.split(None, 1)[0] if authtype.lower() == 'digest': try: # we only keep relevant info spec['value'] = '%s %s' % (authtype, ','.join( val for val in _split_digest_auth(value[6:].strip()) if DIGEST_AUTH_INFOS.match(val))) except Exception: utils.LOGGER.warning("Cannot parse digest error for %r", spec, exc_info=True) elif ntlm._is_ntlm_message(value): # NTLM_CHALLENGE try: auth = utils.decode_b64(value.split(None, 1)[1].encode()) except (UnicodeDecodeError, TypeError, ValueError, binascii.Error): pass spec['value'] = "%s %s" % \ (value.split(None, 1)[0], ntlm._ntlm_dict2string(ntlm.ntlm_extract_info(auth))) elif authtype.lower() in {'negotiate', 'kerberos', 'oauth'}: spec['value'] = authtype # TCP server banners: try to normalize data elif spec['recontype'] == 'TCP_SERVER_BANNER': newvalue = value = utils.nmap_decode_data(spec['value']) for pattern, replace in TCP_SERVER_PATTERNS: if pattern.search(newvalue): newvalue = pattern.sub(replace, newvalue) if newvalue != value: spec['value'] = utils.nmap_encode_data(newvalue) elif spec['recontype'] == 'TCP_CLIENT_BANNER': probe = utils.get_nmap_probes('tcp').get( utils.nmap_decode_data(spec['value'])) if probe is not None: spec.setdefault('infos', {}).update({ 'service_name': 'scanner', 'service_product': 'Nmap', 'service_extrainfo': 'TCP probe %s' % probe, }) elif spec['recontype'] == 'UDP_HONEYPOT_HIT': data = utils.nmap_decode_data(spec['value']) probe = utils.get_nmap_probes('udp').get(data) if probe is not None: spec.setdefault('infos', {}).update({ 'service_name': 'scanner', 'service_product': 'Nmap', 'service_extrainfo': 'UDP probe %s' % probe, }) else: payload = utils.get_nmap_udp_payloads().get(data) if payload is not None: spec.setdefault('infos', {}).update({ 'service_name': 'scanner', 'service_product': 'Nmap', 'service_extrainfo': 'UDP payload %s' % payload, }) # SSL_{CLIENT,SERVER} JA3 elif ((spec['recontype'] == 'SSL_CLIENT' and spec['source'] == 'ja3') or (spec['recontype'] == 'SSL_SERVER' and spec['source'].startswith('ja3-'))): value = spec['value'] spec.setdefault('infos', {})['raw'] = value spec['value'] = hashlib.new("md5", value.encode()).hexdigest() if spec['recontype'] == 'SSL_SERVER': clientvalue = spec['source'][4:] spec['infos'].setdefault('client', {})['raw'] = clientvalue spec['source'] = 'ja3-%s' % hashlib.new( "md5", clientvalue.encode(), ).hexdigest() # SSH_{CLIENT,SERVER}_HASSH elif spec['recontype'] in ['SSH_CLIENT_HASSH', 'SSH_SERVER_HASSH']: value = spec['value'] spec.setdefault('infos', {})['raw'] = value spec['value'] = hashlib.new("md5", value.encode()).hexdigest() # Check DNS Blacklist answer elif spec['recontype'] == 'DNS_ANSWER': if any((spec.get('value') or "").endswith(dnsbl) for dnsbl in config.DNS_BLACKLIST_DOMAINS): dnsbl_val = spec['value'] match = DNSBL_START.search(dnsbl_val) if match is not None: spec['recontype'] = 'DNS_BLACKLIST' spec['value'] = spec.get('addr') spec.update({ 'source': "%s-%s" % (dnsbl_val[match.end():], spec['source']) }) addr = match.group() # IPv4 if addr.count('.') == 4: spec['addr'] = '.'.join(addr.split('.')[3::-1]) # IPv6 else: spec['addr'] = utils.int2ip6( int(addr.replace('.', '')[::-1], 16)) return spec
def zgrap_parser_http(data: Dict[str, Any], hostrec: NmapHost, port: Optional[int] = None) -> NmapPort: """This function handles data from `{"data": {"http": [...]}}` records. `data` should be the content, i.e. the `[...]`. It should consist of simple dictionary, that may contain a `"response"` key and/or a `"redirect_response_chain"` key. The output is a port dict (i.e., the content of the "ports" key of an `nmap` of `view` record in IVRE), that may be empty. """ if not data: return {} # for zgrab2 results if "result" in data: data.update(data.pop("result")) if "response" not in data: utils.LOGGER.warning('Missing "response" field in zgrab HTTP result') return {} resp = data["response"] needed_fields = set(["request", "status_code", "status_line"]) missing_fields = needed_fields.difference(resp) if missing_fields: utils.LOGGER.warning( "Missing field%s %s in zgrab HTTP result", "s" if len(missing_fields) > 1 else "", ", ".join(repr(fld) for fld in missing_fields), ) return {} req = resp["request"] url = req.get("url") res: NmapPort = { "service_name": "http", "service_method": "probed", "state_state": "open", "state_reason": "response", "protocol": "tcp", } tls = None try: tls = req["tls_handshake"] except KeyError: # zgrab2 try: tls = req["tls_log"]["handshake_log"] except KeyError: pass if tls is not None: res["service_tunnel"] = "ssl" try: cert = tls["server_certificates"]["certificate"]["raw"] except KeyError: pass else: output, info_cert = create_ssl_cert(cert.encode(), b64encoded=True) if info_cert: res.setdefault("scripts", []).append({ "id": "ssl-cert", "output": output, "ssl-cert": info_cert, }) for cert in info_cert: add_cert_hostnames(cert, hostrec.setdefault("hostnames", [])) if url: try: _, guessed_port = utils.url2hostport("%(scheme)s://%(host)s" % url) except ValueError: utils.LOGGER.warning("Cannot guess port from url %r", url) guessed_port = 80 # because reasons else: if port is not None and port != guessed_port: utils.LOGGER.warning( "Port %d found from the URL %s differs from the provided port " "value %d", guessed_port, url.get("path"), port, ) port = guessed_port if port is None: port = guessed_port # Specific paths if url.get("path").endswith("/.git/index"): if resp.get("status_code") != 200: return {} if not resp.get("body", "").startswith("DIRC"): return {} # Due to an issue with ZGrab2 output, we cannot, for now, # process the content of the file. See # <https://github.com/zmap/zgrab2/issues/263>. repository = "%s:%d%s" % (hostrec["addr"], port, url["path"][:-5]) res["port"] = port res.setdefault("scripts", []).append({ "id": "http-git", "output": "\n %s\n Git repository found!\n" % repository, "http-git": [ { "repository": repository, "files-found": [".git/index"] }, ], }) return res if url.get("path").endswith("/owa/auth/logon.aspx"): if resp.get("status_code") != 200: return {} version_set = set( m.group(1) for m in _EXPR_OWA_VERSION.finditer(resp.get("body", ""))) if not version_set: return {} version_list = sorted(version_set, key=lambda v: [int(x) for x in v.split(".")]) res["port"] = port path = url["path"][:-15] if version_list: parsed_version = EXCHANGE_BUILDS.get(version_list[0], "unknown build number") if len(version_list) > 1: version_list = [ "%s (%s)" % (vers, EXCHANGE_BUILDS.get(vers, "unknown build number")) for vers in version_list ] output = "OWA: path %s, version %s (multiple versions found!)" % ( path, " / ".join(version_list), ) else: output = "OWA: path %s, version %s (%s)" % ( path, version_list[0], parsed_version, ) res.setdefault("scripts", []).append({ "id": "http-app", "output": output, "http-app": [{ "path": path, "application": "OWA", "version": version_list[0], "parsed_version": parsed_version, }], }) return res if url.get("path").endswith("/centreon/"): if resp.get("status_code") != 200: return {} if not resp.get("body"): return {} body = resp["body"] res["port"] = port path = url["path"] match = _EXPR_TITLE.search(body) if match is None: return {} if match.groups()[0] != "Centreon - IT & Network Monitoring": return {} match = _EXPR_CENTREON_VERSION.search(body) version: Optional[str] if match is None: version = None else: version = match.group(1) or match.group(2) res.setdefault("scripts", []).append({ "id": "http-app", "output": "Centreon: path %s%s" % ( path, "" if version is None else (", version %s" % version), ), "http-app": [ dict( { "path": path, "application": "Centreon" }, **({} if version is None else { "version": version }), ) ], }) return res if url.get("path").endswith("/.well-known/security.txt"): if resp.get("status_code") != 200: return {} if not resp.get("headers"): return {} if not any( ctype.split(";", 1)[0].lower() == "text/plain" for ctype in resp["headers"].get("content_type", [])): return {} if not resp.get("body"): return {} body = resp["body"] res["port"] = port parsed: Dict[str, List[str]] = {} for line in body.splitlines(): line = line.strip().split("#", 1)[0] if not line: continue if ":" not in line: utils.LOGGER.warning( "Invalid line in security.txt file [%r]", line) continue key, value = line.split(":", 1) parsed.setdefault(key.strip().lower(), []).append(value.strip()) res.setdefault("scripts", []).append({ "id": "http-securitytxt", "output": body, "http-securitytxt": {key: " / ".join(value) for key, value in parsed.items()}, }) return res if url.get("path") != "/": utils.LOGGER.warning("URL path not supported yet: %s", url.get("path")) return {} elif port is None: if req.get("tls_handshake") or req.get("tls_log"): port = 443 else: port = 80 res["port"] = port # Since Zgrab does not preserve the order of the headers, we need # to reconstruct a banner to use Nmap fingerprints if resp.get("headers"): headers = resp["headers"] # Check the Authenticate header first: if we requested it with # an Authorization header, we don't want to gather other information if headers.get("www_authenticate"): auths = headers.get("www_authenticate") for auth in auths: if ntlm._is_ntlm_message(auth): try: infos = ntlm.ntlm_extract_info( utils.decode_b64(auth.split(None, 1)[1].encode())) except (UnicodeDecodeError, TypeError, ValueError, binascii.Error): continue if not infos: continue keyvals = zip(ntlm_values, [infos.get(k) for k in ntlm_values]) output = "\n".join("{}: {}".format(k, v) for k, v in keyvals if v) res.setdefault("scripts", []).append({ "id": "ntlm-info", "output": output, "ntlm-info": dict(infos, protocol="http"), }) if "DNS_Computer_Name" in infos: add_hostname( infos["DNS_Computer_Name"], "ntlm", hostrec.setdefault("hostnames", []), ) if any(val.lower().startswith("ntlm") for val in req.get("headers", {}).get("authorization", [])): return res # If we have headers_raw value, let's use it. Else, let's fake it as well as we can. http_hdrs: List[HttpHeader] = [] output_list: List[str] = [] has_raw_value = False if resp.get("headers_raw"): try: banner = utils.decode_b64(resp.get("headers_raw").encode()) except Exception: utils.LOGGER.warning( "Cannot decode raw headers, using parsed result") else: output_list = [ utils.nmap_encode_data(line) for line in re.split(b"\r?\n", banner) ] banner_split = banner.split(b"\n") http_hdrs = [{ "name": "_status", "value": utils.nmap_encode_data(banner_split[0].strip()), }] http_hdrs.extend( { "name": utils.nmap_encode_data(hdrname).lower(), "value": utils.nmap_encode_data(hdrval), } for hdrname, hdrval in (m.groups() for m in ( utils.RAW_HTTP_HEADER.search(part.strip()) for part in banner_split) if m)) has_raw_value = True if not has_raw_value: # no headers_raw or decoding failed # The order will be incorrect! banner = (utils.nmap_decode_data(resp["protocol"]["name"]) + b" " + utils.nmap_decode_data(resp["status_line"]) + b"\r\n") line = "%s %s" % (resp["protocol"]["name"], resp["status_line"]) http_hdrs = [{"name": "_status", "value": line}] output_list = [line] for unk in headers.pop("unknown", []): headers[unk["key"]] = unk["value"] for hdr, values in headers.items(): hdr = hdr.replace("_", "-") for val in values: http_hdrs.append({"name": hdr, "value": val}) output_list.append("%s: %s" % (hdr, val)) if headers.get("server"): banner += (b"Server: " + utils.nmap_decode_data(headers["server"][0]) + b"\r\n\r\n") if http_hdrs: method = req.get("method") if method: output_list.append("") output_list.append("(Request type: %s)" % method) script: NmapScript = { "id": "http-headers", "output": "\n".join(output_list), "http-headers": http_hdrs, } if has_raw_value: script["masscan"] = {"raw": utils.encode_b64(banner).decode()} res.setdefault("scripts", []).append(script) handle_http_headers(hostrec, res, http_hdrs, path=url.get("path")) info: NmapServiceMatch = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest") if info: add_cpe_values(hostrec, "ports.port:%s" % port, info.pop("cpe", [])) res.update(cast(NmapPort, info)) add_service_hostname(info, hostrec.setdefault("hostnames", [])) if resp.get("body"): body = resp["body"] res.setdefault("scripts", []).append({ "id": "http-content", "output": utils.nmap_encode_data(body.encode()), }) handle_http_content(hostrec, res, body.encode()) return res
def zgrap_parser_http(data, hostrec, port=None): """This function handles data from `{"data": {"http": [...]}}` records. `data` should be the content, i.e. the `[...]`. It should consist of simple dictionary, that may contain a `"response"` key and/or a `"redirect_response_chain"` key. The output is a port dict (i.e., the content of the "ports" key of an `nmap` of `view` record in IVRE), that may be empty. """ if not data: return {} # for zgrab2 results if 'result' in data: data.update(data.pop('result')) if 'response' not in data: utils.LOGGER.warning('Missing "response" field in zgrab HTTP result') return {} resp = data['response'] needed_fields = set(["request", "status_code", "status_line"]) missing_fields = needed_fields.difference(resp) if missing_fields: utils.LOGGER.warning( 'Missing field%s %s in zgrab HTTP result', 's' if len(missing_fields) > 1 else '', ', '.join(repr(fld) for fld in missing_fields), ) return {} req = resp['request'] url = req.get('url') res = { "service_name": "http", "service_method": "probed", "state_state": "open", "state_reason": "response", "protocol": "tcp" } tls = None try: tls = req['tls_handshake'] except KeyError: # zgrab2 try: tls = req['tls_log']['handshake_log'] except KeyError: pass if tls is not None: res['service_tunnel'] = 'ssl' try: cert = tls['server_certificates']['certificate']['raw'] except KeyError: pass else: output, info = create_ssl_cert(cert.encode(), b64encoded=True) if info: res.setdefault('scripts', []).append({ 'id': 'ssl-cert', 'output': output, 'ssl-cert': info, }) for cert in info: add_cert_hostnames(cert, hostrec.setdefault('hostnames', [])) if url: guessed_port = None if ':' in url.get('host', ''): try: guessed_port = int(url['host'].split(':', 1)[1]) except ValueError: pass if port is None: if guessed_port is None: if url.get('scheme') == 'https': port = 443 else: port = 80 else: port = guessed_port elif port != guessed_port: utils.LOGGER.warning( 'Port %d found from the URL %s differs from the provided port ' 'value %d', guessed_port, url.get('path'), port) port = guessed_port # Specific paths if url.get('path').endswith('/.git/index'): if resp.get('status_code') != 200: return {} if not resp.get('body', '').startswith('DIRC'): return {} # Due to an issue with ZGrab2 output, we cannot, for now, # process the content of the file. See # <https://github.com/zmap/zgrab2/issues/263>. repository = '%s:%d%s' % (hostrec['addr'], port, url['path'][:-5]) res['port'] = port res.setdefault('scripts', []).append({ 'id': 'http-git', 'output': '\n %s\n Git repository found!\n' % repository, 'http-git': [{ 'repository': repository, 'files-found': [".git/index"] }], }) return res if url.get('path').endswith('/owa/auth/logon.aspx'): if resp.get('status_code') != 200: return {} version = set( m.group(1) for m in _EXPR_OWA_VERSION.finditer(resp.get('body', ''))) if not version: return {} version = sorted(version, key=lambda v: [int(x) for x in v.split('.')]) res['port'] = port path = url['path'][:-15] if len(version) > 1: output = ( 'OWA: path %s, version %s (multiple versions found!)' % ( path, ' / '.join(version), )) else: output = 'OWA: path %s, version %s' % (path, version[0]) res.setdefault('scripts', []).append({ 'id': 'http-app', 'output': output, 'http-app': [{ 'path': path, 'application': 'OWA', 'version': version[0] }], }) return res if url.get('path').endswith('/centreon/'): if resp.get('status_code') != 200: return {} if not resp.get('body'): return {} body = resp['body'] res['port'] = port path = url['path'] match = _EXPR_TITLE.search(body) if match is None: return {} if match.groups()[0] != "Centreon - IT & Network Monitoring": return {} match = _EXPR_CENTREON_VERSION.search(body) if match is None: version = None else: version = match.group(1) or match.group(2) res.setdefault('scripts', []).append({ 'id': 'http-app', 'output': 'Centreon: path %s%s' % ( path, '' if version is None else (', version %s' % version), ), 'http-app': [ dict({ 'path': path, 'application': 'Centreon' }, **({} if version is None else { 'version': version })) ], }) return res if url.get('path') != '/': utils.LOGGER.warning('URL path not supported yet: %s', url.get('path')) return {} elif port is None: if req.get('tls_handshake') or req.get('tls_log'): port = 443 else: port = 80 res['port'] = port # Since Zgrab does not preserve the order of the headers, we need # to reconstruct a banner to use Nmap fingerprints banner = (utils.nmap_decode_data(resp['protocol']['name']) + b' ' + utils.nmap_decode_data(resp['status_line']) + b"\r\n") if resp.get('headers'): headers = resp['headers'] # Check the Authenticate header first: if we requested it with # an Authorization header, we don't want to gather other information if headers.get('www_authenticate'): auths = headers.get('www_authenticate') for auth in auths: if ntlm._is_ntlm_message(auth): try: infos = ntlm.ntlm_extract_info( utils.decode_b64(auth.split(None, 1)[1].encode())) except (UnicodeDecodeError, TypeError, ValueError, binascii.Error): pass keyvals = zip(ntlm_values, [infos.get(k) for k in ntlm_values]) output = '\n'.join("{}: {}".format(k, v) for k, v in keyvals if v) res.setdefault('scripts', []).append({ 'id': 'http-ntlm-info', 'output': output, 'ntlm-info': infos }) if 'DNS_Computer_Name' in infos: add_hostname(infos['DNS_Computer_Name'], 'ntlm', hostrec.setdefault('hostnames', [])) if any(val.lower().startswith('ntlm') for val in req.get('headers', {}).get('authorization', [])): return res # the order will be incorrect! line = '%s %s' % (resp['protocol']['name'], resp['status_line']) http_hdrs = [{'name': '_status', 'value': line}] output = [line] for unk in headers.pop('unknown', []): headers[unk['key']] = unk['value'] for hdr, values in headers.items(): hdr = hdr.replace('_', '-') for val in values: http_hdrs.append({'name': hdr, 'value': val}) output.append('%s: %s' % (hdr, val)) if http_hdrs: method = req.get('method') if method: output.append('') output.append('(Request type: %s)' % method) res.setdefault('scripts', []).append({ 'id': 'http-headers', 'output': '\n'.join(output), 'http-headers': http_hdrs, }) handle_http_headers(hostrec, res, http_hdrs, path=url.get('path')) if headers.get('server'): banner += (b"Server: " + utils.nmap_decode_data(headers['server'][0]) + b"\r\n\r\n") info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest") if info: add_cpe_values(hostrec, 'ports.port:%s' % port, info.pop('cpe', [])) res.update(info) if resp.get('body'): body = resp['body'] res.setdefault('scripts', []).append({ 'id': 'http-content', 'output': utils.nmap_encode_data(body.encode()), }) match = _EXPR_TITLE.search(body) if match is not None: title = match.groups()[0] res['scripts'].append({ 'id': 'http-title', 'output': title, 'http-title': { 'title': title }, }) script_http_ls = create_http_ls(body, url=url) if script_http_ls is not None: res.setdefault('scripts', []).append(script_http_ls) service_elasticsearch = create_elasticsearch_service(body) if service_elasticsearch: if 'hostname' in service_elasticsearch: add_hostname(service_elasticsearch.pop('hostname'), 'service', hostrec.setdefault('hostnames', [])) add_cpe_values(hostrec, 'ports.port:%s' % port, service_elasticsearch.pop('cpe', [])) res.update(service_elasticsearch) return res