def zgrap_parser_http(data, hostrec, port=None): """This function handles data from `{"data": {"http": [...]}}` records. `data` should be the content, i.e. the `[...]`. It should consist of simple dictionary, that may contain a `"response"` key and/or a `"redirect_response_chain"` key. The output is a port dict (i.e., the content of the "ports" key of an `nmap` of `view` record in IVRE), that may be empty. """ if not data: return {} # for zgrab2 results if "result" in data: data.update(data.pop("result")) if "response" not in data: utils.LOGGER.warning('Missing "response" field in zgrab HTTP result') return {} resp = data["response"] needed_fields = set(["request", "status_code", "status_line"]) missing_fields = needed_fields.difference(resp) if missing_fields: utils.LOGGER.warning( "Missing field%s %s in zgrab HTTP result", "s" if len(missing_fields) > 1 else "", ", ".join(repr(fld) for fld in missing_fields), ) return {} req = resp["request"] url = req.get("url") res = { "service_name": "http", "service_method": "probed", "state_state": "open", "state_reason": "response", "protocol": "tcp", } tls = None try: tls = req["tls_handshake"] except KeyError: # zgrab2 try: tls = req["tls_log"]["handshake_log"] except KeyError: pass if tls is not None: res["service_tunnel"] = "ssl" try: cert = tls["server_certificates"]["certificate"]["raw"] except KeyError: pass else: output, info = create_ssl_cert(cert.encode(), b64encoded=True) if info: res.setdefault("scripts", []).append( { "id": "ssl-cert", "output": output, "ssl-cert": info, } ) for cert in info: add_cert_hostnames(cert, hostrec.setdefault("hostnames", [])) if url: guessed_port = None if ":" in url.get("host", ""): try: guessed_port = int(url["host"].split(":", 1)[1]) except ValueError: pass if port is None: if guessed_port is None: if url.get("scheme") == "https": port = 443 else: port = 80 else: port = guessed_port elif port != guessed_port: utils.LOGGER.warning( "Port %d found from the URL %s differs from the provided port " "value %d", guessed_port, url.get("path"), port, ) port = guessed_port # Specific paths if url.get("path").endswith("/.git/index"): if resp.get("status_code") != 200: return {} if not resp.get("body", "").startswith("DIRC"): return {} # Due to an issue with ZGrab2 output, we cannot, for now, # process the content of the file. See # <https://github.com/zmap/zgrab2/issues/263>. repository = "%s:%d%s" % (hostrec["addr"], port, url["path"][:-5]) res["port"] = port res.setdefault("scripts", []).append( { "id": "http-git", "output": "\n %s\n Git repository found!\n" % repository, "http-git": [ {"repository": repository, "files-found": [".git/index"]} ], } ) return res if url.get("path").endswith("/owa/auth/logon.aspx"): if resp.get("status_code") != 200: return {} version = set( m.group(1) for m in _EXPR_OWA_VERSION.finditer(resp.get("body", "")) ) if not version: return {} version = sorted(version, key=lambda v: [int(x) for x in v.split(".")]) res["port"] = port path = url["path"][:-15] if len(version) > 1: output = "OWA: path %s, version %s (multiple versions found!)" % ( path, " / ".join(version), ) else: output = "OWA: path %s, version %s" % (path, version[0]) res.setdefault("scripts", []).append( { "id": "http-app", "output": output, "http-app": [ {"path": path, "application": "OWA", "version": version[0]} ], } ) return res if url.get("path").endswith("/centreon/"): if resp.get("status_code") != 200: return {} if not resp.get("body"): return {} body = resp["body"] res["port"] = port path = url["path"] match = _EXPR_TITLE.search(body) if match is None: return {} if match.groups()[0] != "Centreon - IT & Network Monitoring": return {} match = _EXPR_CENTREON_VERSION.search(body) if match is None: version = None else: version = match.group(1) or match.group(2) res.setdefault("scripts", []).append( { "id": "http-app", "output": "Centreon: path %s%s" % ( path, "" if version is None else (", version %s" % version), ), "http-app": [ dict( {"path": path, "application": "Centreon"}, **({} if version is None else {"version": version}), ) ], } ) return res if url.get("path") != "/": utils.LOGGER.warning("URL path not supported yet: %s", url.get("path")) return {} elif port is None: if req.get("tls_handshake") or req.get("tls_log"): port = 443 else: port = 80 res["port"] = port # Since Zgrab does not preserve the order of the headers, we need # to reconstruct a banner to use Nmap fingerprints banner = ( utils.nmap_decode_data(resp["protocol"]["name"]) + b" " + utils.nmap_decode_data(resp["status_line"]) + b"\r\n" ) if resp.get("headers"): headers = resp["headers"] # Check the Authenticate header first: if we requested it with # an Authorization header, we don't want to gather other information if headers.get("www_authenticate"): auths = headers.get("www_authenticate") for auth in auths: if ntlm._is_ntlm_message(auth): try: infos = ntlm.ntlm_extract_info( utils.decode_b64(auth.split(None, 1)[1].encode()) ) except (UnicodeDecodeError, TypeError, ValueError, binascii.Error): pass keyvals = zip(ntlm_values, [infos.get(k) for k in ntlm_values]) output = "\n".join("{}: {}".format(k, v) for k, v in keyvals if v) res.setdefault("scripts", []).append( {"id": "http-ntlm-info", "output": output, "ntlm-info": infos} ) if "DNS_Computer_Name" in infos: add_hostname( infos["DNS_Computer_Name"], "ntlm", hostrec.setdefault("hostnames", []), ) if any( val.lower().startswith("ntlm") for val in req.get("headers", {}).get("authorization", []) ): return res # the order will be incorrect! line = "%s %s" % (resp["protocol"]["name"], resp["status_line"]) http_hdrs = [{"name": "_status", "value": line}] output = [line] for unk in headers.pop("unknown", []): headers[unk["key"]] = unk["value"] for hdr, values in headers.items(): hdr = hdr.replace("_", "-") for val in values: http_hdrs.append({"name": hdr, "value": val}) output.append("%s: %s" % (hdr, val)) if http_hdrs: method = req.get("method") if method: output.append("") output.append("(Request type: %s)" % method) res.setdefault("scripts", []).append( { "id": "http-headers", "output": "\n".join(output), "http-headers": http_hdrs, } ) handle_http_headers(hostrec, res, http_hdrs, path=url.get("path")) if headers.get("server"): banner += ( b"Server: " + utils.nmap_decode_data(headers["server"][0]) + b"\r\n\r\n" ) info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest") if info: add_cpe_values(hostrec, "ports.port:%s" % port, info.pop("cpe", [])) res.update(info) if resp.get("body"): body = resp["body"] res.setdefault("scripts", []).append( { "id": "http-content", "output": utils.nmap_encode_data(body.encode()), } ) match = _EXPR_TITLE.search(body) if match is not None: title = match.groups()[0] res["scripts"].append( { "id": "http-title", "output": title, "http-title": {"title": title}, } ) script_http_ls = create_http_ls(body, url=url) if script_http_ls is not None: res.setdefault("scripts", []).append(script_http_ls) service_elasticsearch = create_elasticsearch_service(body) if service_elasticsearch: if "hostname" in service_elasticsearch: add_hostname( service_elasticsearch.pop("hostname"), "service", hostrec.setdefault("hostnames", []), ) add_cpe_values( hostrec, "ports.port:%s" % port, service_elasticsearch.pop("cpe", []) ) res.update(service_elasticsearch) return res
def zgrap_parser_http(data, hostrec): """This function handles data from `{"data": {"http": [...]}}` records. `data` should be the content, i.e. the `[...]`. It should consist of simple dictionary, that may contain a `"response"` key and/or a `"redirect_response_chain"` key. The output is a port dict (i.e., the content of the "ports" key of an `nmap` of `view` record in IVRE), that may be empty. """ if not data: return {} # for zgrab2 results if 'result' in data: data.update(data.pop('result')) if 'response' not in data: utils.LOGGER.warning('Missing "response" field in zgrab HTTP result') return {} resp = data['response'] needed_fields = set(["request", "status_code", "status_line"]) missing_fields = needed_fields.difference(resp) if missing_fields: utils.LOGGER.warning( 'Missing field%s %s in zgrab HTTP result', 's' if len(missing_fields) > 1 else '', ', '.join(repr(fld) for fld in missing_fields), ) return {} req = resp['request'] url = req.get('url') res = {"service_name": "http", "service_method": "probed", "state_state": "open", "state_reason": "response", "protocol": "tcp"} tls = None try: tls = req['tls_handshake'] except KeyError: # zgrab2 try: tls = req['tls_log']['handshake_log'] except KeyError: pass if tls is not None: res['service_tunnel'] = 'ssl' try: cert = tls['server_certificates']['certificate']['raw'] except KeyError: pass else: output, info = create_ssl_cert(cert.encode(), b64encoded=True) if info: res.setdefault('scripts', []).append({ 'id': 'ssl-cert', 'output': output, 'ssl-cert': info, }) for cert in info: add_cert_hostnames(cert, hostrec.setdefault('hostnames', [])) if url: port = None if ':' in url.get('host', ''): try: port = int(url['host'].split(':', 1)[1]) except ValueError: pass if port is None: if url.get('scheme') == 'https': port = 443 else: port = 80 # Specific paths if url.get('path').endswith('/.git/index'): if resp.get('status_code') != 200: return {} if not resp.get('body', '').startswith('DIRC'): return {} # Due to an issue with ZGrab2 output, we cannot, for now, # process the content of the file. See # <https://github.com/zmap/zgrab2/issues/263>. repository = '%s:%d%s' % (hostrec['addr'], port, url['path'][:-5]) res['port'] = port res.setdefault('scripts', []).append({ 'id': 'http-git', 'output': '\n %s\n Git repository found!\n' % repository, 'http-git': [{'repository': repository, 'files-found': [".git/index"]}], }) return res if url.get('path').endswith('/owa/auth/logon.aspx'): if resp.get('status_code') != 200: return {} version = set( m.group(1) for m in _EXPR_OWA_VERSION.finditer(resp.get('body', '')) ) if not version: return {} version = sorted(version, key=lambda v: [int(x) for x in v.split('.')]) res['port'] = port path = url['path'][:-15] if len(version) > 1: output = ( 'OWA: path %s, version %s (multiple versions found!)' % ( path, ' / '.join(version), ) ) else: output = 'OWA: path %s, version %s' % (path, version[0]) res.setdefault('scripts', []).append({ 'id': 'http-app', 'output': output, 'http-app': [{'path': path, 'application': 'OWA', 'version': version[0]}], }) return res elif req.get('tls_handshake') or req.get('tls_log'): # zgrab / zgrab2 port = 443 else: port = 80 res['port'] = port # Since Zgrab does not preserve the order of the headers, we need # to reconstruct a banner to use Nmap fingerprints banner = (utils.nmap_decode_data(resp['protocol']['name']) + b' ' + utils.nmap_decode_data(resp['status_line']) + b"\r\n") if resp.get('headers'): headers = resp['headers'] # the order will be incorrect! line = '%s %s' % (resp['protocol']['name'], resp['status_line']) http_hdrs = [{'name': '_status', 'value': line}] output = [line] if 'unknown' in headers: for unk in headers.pop('unknown'): headers[unk['key']] = unk['value'] for hdr, values in viewitems(headers): hdr = hdr.replace('_', '-') for val in values: http_hdrs.append({'name': hdr, 'value': val}) output.append('%s: %s' % (hdr, val)) if http_hdrs: method = req.get('method') if method: output.append('') output.append('(Request type: %s)' % method) res.setdefault('scripts', []).append({ 'id': 'http-headers', 'output': '\n'.join(output), 'http-headers': http_hdrs, }) if headers.get('server'): server = resp['headers']['server'] res.setdefault('scripts', []).append({ 'id': 'http-server-header', 'output': server[0], 'http-server-header': server, }) banner += (b"Server: " + utils.nmap_decode_data(server[0]) + b"\r\n\r\n") info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest") if info: res.update(info) if resp.get('body'): body = resp['body'] res.setdefault('scripts', []).append({ 'id': 'http-content', 'output': utils.nmap_encode_data(body.encode()), }) match = _EXPR_TITLE.search(body) if match is not None: title = match.groups()[0] res['scripts'].append({ 'id': 'http-title', 'output': title, 'http-title': {'title': title}, }) script_http_ls = create_http_ls(body, url=url) if script_http_ls is not None: res.setdefault('scripts', []).append(script_http_ls) return res
def zgrap_parser_http(data, hostrec, port=None): """This function handles data from `{"data": {"http": [...]}}` records. `data` should be the content, i.e. the `[...]`. It should consist of simple dictionary, that may contain a `"response"` key and/or a `"redirect_response_chain"` key. The output is a port dict (i.e., the content of the "ports" key of an `nmap` of `view` record in IVRE), that may be empty. """ if not data: return {} # for zgrab2 results if 'result' in data: data.update(data.pop('result')) if 'response' not in data: utils.LOGGER.warning('Missing "response" field in zgrab HTTP result') return {} resp = data['response'] needed_fields = set(["request", "status_code", "status_line"]) missing_fields = needed_fields.difference(resp) if missing_fields: utils.LOGGER.warning( 'Missing field%s %s in zgrab HTTP result', 's' if len(missing_fields) > 1 else '', ', '.join(repr(fld) for fld in missing_fields), ) return {} req = resp['request'] url = req.get('url') res = { "service_name": "http", "service_method": "probed", "state_state": "open", "state_reason": "response", "protocol": "tcp" } tls = None try: tls = req['tls_handshake'] except KeyError: # zgrab2 try: tls = req['tls_log']['handshake_log'] except KeyError: pass if tls is not None: res['service_tunnel'] = 'ssl' try: cert = tls['server_certificates']['certificate']['raw'] except KeyError: pass else: output, info = create_ssl_cert(cert.encode(), b64encoded=True) if info: res.setdefault('scripts', []).append({ 'id': 'ssl-cert', 'output': output, 'ssl-cert': info, }) for cert in info: add_cert_hostnames(cert, hostrec.setdefault('hostnames', [])) if url: guessed_port = None if ':' in url.get('host', ''): try: guessed_port = int(url['host'].split(':', 1)[1]) except ValueError: pass if port is None: if guessed_port is None: if url.get('scheme') == 'https': port = 443 else: port = 80 else: port = guessed_port elif port != guessed_port: utils.LOGGER.warning( 'Port %d found from the URL %s differs from the provided port ' 'value %d', guessed_port, url.get('path'), port) port = guessed_port # Specific paths if url.get('path').endswith('/.git/index'): if resp.get('status_code') != 200: return {} if not resp.get('body', '').startswith('DIRC'): return {} # Due to an issue with ZGrab2 output, we cannot, for now, # process the content of the file. See # <https://github.com/zmap/zgrab2/issues/263>. repository = '%s:%d%s' % (hostrec['addr'], port, url['path'][:-5]) res['port'] = port res.setdefault('scripts', []).append({ 'id': 'http-git', 'output': '\n %s\n Git repository found!\n' % repository, 'http-git': [{ 'repository': repository, 'files-found': [".git/index"] }], }) return res if url.get('path').endswith('/owa/auth/logon.aspx'): if resp.get('status_code') != 200: return {} version = set( m.group(1) for m in _EXPR_OWA_VERSION.finditer(resp.get('body', ''))) if not version: return {} version = sorted(version, key=lambda v: [int(x) for x in v.split('.')]) res['port'] = port path = url['path'][:-15] if len(version) > 1: output = ( 'OWA: path %s, version %s (multiple versions found!)' % ( path, ' / '.join(version), )) else: output = 'OWA: path %s, version %s' % (path, version[0]) res.setdefault('scripts', []).append({ 'id': 'http-app', 'output': output, 'http-app': [{ 'path': path, 'application': 'OWA', 'version': version[0] }], }) return res if url.get('path').endswith('/centreon/'): if resp.get('status_code') != 200: return {} if not resp.get('body'): return {} body = resp['body'] res['port'] = port path = url['path'] match = _EXPR_TITLE.search(body) if match is None: return {} if match.groups()[0] != "Centreon - IT & Network Monitoring": return {} match = _EXPR_CENTREON_VERSION.search(body) if match is None: version = None else: version = match.group(1) or match.group(2) res.setdefault('scripts', []).append({ 'id': 'http-app', 'output': 'Centreon: path %s%s' % ( path, '' if version is None else (', version %s' % version), ), 'http-app': [ dict({ 'path': path, 'application': 'Centreon' }, **({} if version is None else { 'version': version })) ], }) return res if url.get('path') != '/': utils.LOGGER.warning('URL path not supported yet: %s', url.get('path')) return {} elif port is None: if req.get('tls_handshake') or req.get('tls_log'): port = 443 else: port = 80 res['port'] = port # Since Zgrab does not preserve the order of the headers, we need # to reconstruct a banner to use Nmap fingerprints banner = (utils.nmap_decode_data(resp['protocol']['name']) + b' ' + utils.nmap_decode_data(resp['status_line']) + b"\r\n") if resp.get('headers'): headers = resp['headers'] # Check the Authenticate header first: if we requested it with # an Authorization header, we don't want to gather other information if headers.get('www_authenticate'): auths = headers.get('www_authenticate') for auth in auths: if ntlm._is_ntlm_message(auth): try: infos = ntlm.ntlm_extract_info( utils.decode_b64(auth.split(None, 1)[1].encode())) except (UnicodeDecodeError, TypeError, ValueError, binascii.Error): pass keyvals = zip(ntlm_values, [infos.get(k) for k in ntlm_values]) output = '\n'.join("{}: {}".format(k, v) for k, v in keyvals if v) res.setdefault('scripts', []).append({ 'id': 'http-ntlm-info', 'output': output, 'ntlm-info': infos }) if 'DNS_Computer_Name' in infos: add_hostname(infos['DNS_Computer_Name'], 'ntlm', hostrec.setdefault('hostnames', [])) if any(val.lower().startswith('ntlm') for val in req.get('headers', {}).get('authorization', [])): return res # the order will be incorrect! line = '%s %s' % (resp['protocol']['name'], resp['status_line']) http_hdrs = [{'name': '_status', 'value': line}] output = [line] for unk in headers.pop('unknown', []): headers[unk['key']] = unk['value'] for hdr, values in headers.items(): hdr = hdr.replace('_', '-') for val in values: http_hdrs.append({'name': hdr, 'value': val}) output.append('%s: %s' % (hdr, val)) if http_hdrs: method = req.get('method') if method: output.append('') output.append('(Request type: %s)' % method) res.setdefault('scripts', []).append({ 'id': 'http-headers', 'output': '\n'.join(output), 'http-headers': http_hdrs, }) handle_http_headers(hostrec, res, http_hdrs, path=url.get('path')) if headers.get('server'): banner += (b"Server: " + utils.nmap_decode_data(headers['server'][0]) + b"\r\n\r\n") info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest") if info: add_cpe_values(hostrec, 'ports.port:%s' % port, info.pop('cpe', [])) res.update(info) if resp.get('body'): body = resp['body'] res.setdefault('scripts', []).append({ 'id': 'http-content', 'output': utils.nmap_encode_data(body.encode()), }) match = _EXPR_TITLE.search(body) if match is not None: title = match.groups()[0] res['scripts'].append({ 'id': 'http-title', 'output': title, 'http-title': { 'title': title }, }) script_http_ls = create_http_ls(body, url=url) if script_http_ls is not None: res.setdefault('scripts', []).append(script_http_ls) service_elasticsearch = create_elasticsearch_service(body) if service_elasticsearch: if 'hostname' in service_elasticsearch: add_hostname(service_elasticsearch.pop('hostname'), 'service', hostrec.setdefault('hostnames', [])) add_cpe_values(hostrec, 'ports.port:%s' % port, service_elasticsearch.pop('cpe', [])) res.update(service_elasticsearch) return res
def zgrap_parser_http(data): """This function handles data from `{"data": {"http": [...]}}` records. `data` should be the content, i.e. the `[...]`. It should consist of simple dictionary, that may contain a `"response"` key and/or a `"redirect_response_chain"` key. The output is a port dict (i.e., the content of the "ports" key of an `nmap` of `view` record in IVRE), that may be empty. """ if not data: return {} # for zgrab2 results if 'result' in data: data.update(data.pop('result')) if 'response' not in data: utils.LOGGER.warning('Missing "response" field in zgrab HTTP result') return {} resp = data['response'] needed_fields = set(["request", "status_code", "status_line"]) missing_fields = needed_fields.difference(resp) if missing_fields: utils.LOGGER.warning( 'Missing field%s %s in zgrab HTTP result', 's' if len(missing_fields) > 1 else '', ', '.join(repr(fld) for fld in missing_fields), ) return {} req = resp['request'] res = { "service_name": "http", "service_method": "probed", "state_state": "open", "state_reason": "syn-ack", "protocol": "tcp" } url = req.get('url') if url: port = None if ':' in url.get('host', ''): try: port = int(url['host'].split(':', 1)[1]) except ValueError: pass if port is None: if url.get('scheme') == 'https': port = 443 else: port = 80 elif req.get('tls_handshake') or req.get('tls_log'): # zgrab / zgrab2 port = 443 else: port = 80 res['port'] = port # Since Zgrab does not preserve the order of the headers, we need # to reconstruct a banner to use Nmap fingerprints banner = (utils.nmap_decode_data(resp['protocol']['name']) + b' ' + utils.nmap_decode_data(resp['status_line']) + b"\r\n") if resp.get('headers'): headers = resp['headers'] # the order will be incorrect! http_hdrs = [{'name': '_status', 'value': resp['status_line']}] output = [resp['status_line']] if 'unknown' in headers: for unk in headers.pop('unknown'): headers[unk['key']] = unk['value'] for hdr, values in viewitems(headers): hdr = hdr.replace('_', '-') for val in values: http_hdrs.append({'name': hdr, 'value': val}) output.append('%s: %s' % (hdr, val)) if http_hdrs: method = req.get('method') if method: output.append('') output.append('(Request type: %s)' % method) res.setdefault('scripts', []).append({ 'id': 'http-headers', 'output': '\n'.join(output), 'http-headers': http_hdrs, }) if headers.get('server'): server = resp['headers']['server'] res.setdefault('scripts', []).append({ 'id': 'http-server-header', 'output': server[0], 'http-server-header': server, }) banner += (b"Server: " + utils.nmap_decode_data(server[0]) + b"\r\n\r\n") info = utils.match_nmap_svc_fp(banner, proto="tcp", probe="GetRequest") if info: res.update(info) if resp.get('body'): body = resp['body'] res.setdefault('scripts', []).append({ 'id': 'http-content', 'output': utils.nmap_encode_data(body.encode()), }) match = _EXPR_TITLE.search(body) if match is not None: title = match.groups()[0] res['scripts'].append({ 'id': 'http-title', 'output': title, 'http-title': { 'title': title }, }) script_http_ls = create_http_ls(body, url=url) if script_http_ls is not None: res.setdefault('scripts', []).append(script_http_ls) tls = None try: tls = req['tls_handshake'] except KeyError: # zgrab2 try: tls = req['tls_log']['handshake_log'] except KeyError: pass if tls is not None: res['service_tunnel'] = 'ssl' try: cert = tls['server_certificates']['certificate']['raw'] except KeyError: pass else: output, info = create_ssl_cert(cert.encode(), b64encoded=True) if info: res.setdefault('scripts', []).append({ 'id': 'ssl-cert', 'output': "\n".join(output), 'ssl-cert': info, }) return res