def _transform_object(self, obj): zout = ZMapTransformOutput() out = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") ecdh = wrapped['data']['tls']['server_key_exchange']['ecdh_params'][ 'curve_id'] ecdh_name = ecdh['name'].resolve() ecdh_id = ecdh['id'].resolve() if ecdh_name or ecdh_id is not None: out['ecdh_params'] = dict() out['ecdh_params']['curve_id'] = dict() if ecdh_name is not None: out['ecdh_params']['curve_id']['name'] = ecdh_name if ecdh_id is not None: out['ecdh_params']['curve_id']['id'] = ecdh_id out["support"] = bool(out) zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() out = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") rsa = wrapped['data']['tls']['server_key_exchange']['rsa_params'] rsa_exponent = rsa['exponent'].resolve() rsa_modulus = rsa['modulus'].resolve() rsa_length = rsa['length'].resolve() if rsa_exponent or rsa_modulus is not None: out['rsa_params'] = dict() if rsa_exponent is not None: out['rsa_params']['exponent'] = rsa_exponent if rsa_modulus is not None: out['rsa_params']['modulus'] = rsa_modulus if rsa_length is not None: out['rsa_params']['length'] = rsa_length out["support"] = bool(out) zout.transformed = out return zout
def _transform_object(self, obj): http = Transformable(obj) http_response = http['data']['http']['response'] zout = ZMapTransformOutput() out = dict() error_component = http['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("connection error") if http_response is not None: status_line = http_response['status_line'].resolve() status_code = http_response['status_code'].resolve() body = http_response['body'].resolve() headers = http_response['headers'].resolve() if status_line is not None: out['status_line'] = status_line if status_code is not None: out['status_code'] = status_code if body is not None: out['body'] = body if headers is not None: out['headers'] = headers if len(out) == 0: raise errors.IgnoreObject("Empty output dict") zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() out = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") dh = wrapped['data']['tls']['server_key_exchange']['dh_params'] dh_prime_value = dh['prime']['value'].resolve() dh_prime_length = dh['prime']['length'].resolve() dh_generator_value = dh['generator']['value'].resolve() dh_generator_length = dh['generator']['length'].resolve() if dh_prime_value or dh_generator_value is not None: out['dh_params'] = dict() if dh_prime_value is not None: out['dh_params']['prime'] = dict() if dh_generator_value is not None: out['dh_params']['generator'] = dict() if dh_prime_value is not None: out['dh_params']['prime']['value'] = dh_prime_value if dh_prime_length is not None: out['dh_params']['prime']['length'] = dh_prime_length if dh_generator_value is not None: out['dh_params']['generator']['value'] = dh_generator_value if dh_generator_length is not None: out['dh_params']['generator']['length'] = dh_generator_length out["support"] = bool(out) zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() out = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") heartbleed = wrapped['data']['heartbleed'] heartbleed = wrapped['data']['heartbleed'] heartbleed_enabled = heartbleed['heartbeat_enabled'].resolve() heartbleed_vulnerable = heartbleed['heartbleed_vulnerable'].resolve() if heartbleed_enabled is not None: out['heartbeat_enabled'] = heartbleed_enabled if heartbleed_vulnerable is not None: out['heartbleed_vulnerable'] = heartbleed_vulnerable if len(out) == 0: raise errors.IgnoreObject("Empty Output dict") zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") banner = wrapped['data']['banner'].resolve() try: tls_handshake = obj['data']['tls'] out, certificates = https.HTTPSTransform.make_tls_obj( tls_handshake) zout.transformed['tls'] = out zout.certificates = certificates except (TypeError, KeyError, IndexError): pass if banner is not None: zout.transformed['banner'] = self.clean_banner(banner) if len(zout.transformed) == 0: raise errors.IgnoreObject("Empty Dict output") return zout
def _transform_object(self, obj): wrapped = Transformable(obj['data']) ciphers = wrapped['sslv2']['server_hello']['ciphers'].resolve() certificate = wrapped['sslv2']['server_hello']['certificate'].resolve() sslv2_support = bool(wrapped['sslv2']['server_verify'].resolve()) sslv2_export = bool(wrapped['sslv2_export']['server_verify'].resolve()) sslv2_extra_clear = bool(wrapped['sslv2_extra_clear']['server_verify'] ['extra_clear'].resolve()) out = { 'support': sslv2_support, 'export': sslv2_export, 'extra_clear': sslv2_extra_clear, } if ciphers is not None: out['ciphers'] = ciphers if certificate is not None: out['certificate'] = {'parsed': certificate['parsed']} certificates = [certificate] else: certificates = list() zout = ZMapTransformOutput() zout.transformed = out zout.certificates = certificates return zout
def _transform_object(self, obj): http = Transformable(obj) connect_response = http['data']['http']['connect_response'] get_response = http['data']['http']['response'] zout = ZMapTransformOutput() out = dict() error_component = http['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("connection error") if connect_response is not None: status_line = connect_response['status_line'].resolve() status_code = connect_response['status_code'].resolve() body = connect_response['body'].resolve() headers = connect_response['headers'].resolve() if status_line or status_code or body or headers is not None: out['connect'] = dict() if status_line is not None: out['connect']['status_line'] = status_line if status_code is not None: out['connect']['status_code'] = status_code if body is not None: out['connect']['body'] = body if headers is not None: out['connect']['headers'] = headers if get_response is not None: status_line = get_response['status_line'].resolve() status_code = get_response['status_code'].resolve() body = get_response['body'].resolve() headers = get_response['headers'].resolve() body_sha256 = get_response['body_sha256'].resolve() out['get'] = dict() if body: random_present = "Uh2Qn8Y7NPRm6h3xqEXUq4EhtW7Po4gy" in body else: random_present = False out['get']['random_present'] = random_present if status_line is not None: out['get']['status_line'] = status_line if status_code is not None: out['get']['status_code'] = status_code if body is not None: out['get']['body'] = body if headers is not None: out['get']['headers'] = headers if body_sha256: out['get']['body_sha256'] = body_sha256 if len(out) == 0: raise errors.IgnoreObject("Empty output dict") zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() wrapped = Transformable(obj) modbus = wrapped['data']['modbus'] if not modbus['raw_response'].resolve(): raise errors.IgnoreObject() out = dict() function_code = modbus['function_code'].resolve() mei_response = modbus['mei_response'] if function_code: out['function_code'] = function_code if mei_response: conformity_level = mei_response['conformity_level'].resolve() objects = mei_response['objects'] vendor = objects['vendor'].resolve() product_code = objects['product_code'].resolve() revision = objects['revision'].resolve() vendor_url = objects['vendor_url'].resolve() product_name = objects['product_name'].resolve() model_name = objects['model_name'].resolve() user_application_name = objects['user_application_name'].resolve() if conformity_level or vendor or product_code or revision or \ vendor_url or product_name or model_name or user_application_name: out['mei_response'] = dict() if vendor or product_code or revision or vendor_url or product_name \ or model_name or user_application_name: out['mei_response']['objects'] = dict() if conformity_level: out['mei_response']['conformity_level'] = conformity_level if vendor: out['mei_response']['objects']['vendor'] = vendor if product_code: out['mei_response']['objects']['product_code'] = product_code if revision: out['mei_response']['objects']['revision'] = revision if vendor_url: out['mei_response']['objects']['vendor_url'] = vendor_url if product_name: out['mei_response']['objects']['product_name'] = product_name if model_name: out['mei_response']['objects']['model_name'] = model_name if user_application_name: out['mei_response']['objects']['user_application_name'] = \ user_application_name if len(out) == 0: raise errors.IgnoreObject("Empty output dict") zout.transformed = out return zout
def _transform_object(self, obj): http = Transformable(obj) http_response = http['data']['http']['response'] zout = ZMapTransformOutput() out = dict() zout.transformed = out error_component = http['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("connection error") if http_response is not None: status_line = http_response['status_line'].resolve() status_code = http_response['status_code'].resolve() body = http_response['body'].resolve() body_sha256 = http_response['body_sha256'].resolve() headers = http_response['headers'].resolve() if status_line is not None: out['status_line'] = status_line if status_code is not None: out['status_code'] = status_code if body is not None: out['body'] = body m = title_regex.search(body) if m: title = m.group(1) if len(title) > 1024: title = title[0:1024] out['title'] = title.strip() if headers: # FIXME: This modifies the input? if "set_cookie" in headers: del headers["set_cookie"] if "date" in headers: del headers["date"] for k, v, in headers.iteritems(): if k == "unknown": for d in v: if len(d["value"]) < 1: continue d["value"] = d["value"][0] elif v: headers[k] = v[0] else: del headers[k] out['headers'] = headers if body_sha256: out['body_sha256'] = body_sha256 if len(out) == 0: raise errors.IgnoreObject("Empty output dict") return zout
def _transform_object(self, obj): classification = obj['classification'] if classification != "upnp": raise errors.IgnoreObject(classification) success = int(obj['success']) if success != 1: raise errors.IgnoreObject("unsuccessful zmap") wrapped = Transformable(obj) transformed = {} for key in self.KEYS: transformed[key] = wrapped[key].resolve() out = ZMapTransformOutput() out.transformed = transformed return out
def _transform_object(self, obj): zout = ZMapTransformOutput() out = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") server_hello = wrapped['data']['tls']['server_hello'] if server_hello.resolve() is None: raise IgnoreObject("missing server hello") exr = server_hello['extended_random'].resolve() if exr is not None: out['extended_random_support'] = True else: out['extended_random_support'] = False zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() tempout = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") tls_handshake = wrapped['data']['tls'] value = tls_handshake['server_hello']['version']['value'].resolve() out = dict() if value is not None: version = int(value) out['support'] = True if version == 772 else False else: raise errors.IgnoreObject("Empty output dict") zout.transformed = out return zout
def _transform_object(self, obj): ftp_banner = obj ftp = Transformable(obj) zout = ZMapTransformOutput() error = ftp['error'].resolve() if error is not None: raise errors.IgnoreObject("Error") out = dict() banner = ftp['data']['banner'].resolve() if banner is not None: out['banner'] = self.clean_banner(banner) if len(out) == 0: raise errors.IgnoreObject("Empty output dict") out['ip_address'] = obj['ip'] out['timestamp'] = obj['timestamp'] zout.transformed = out return zout
def _transform_object(self, obj): ssh = Transformable(obj) server_protocol = dict() sp = ssh['data']['ssh']['server_protocol'] if sp.resolve() is None: raise errors.IgnoreObject("no ssh server protocol") raw_banner = sp['raw_banner'].resolve() if raw_banner is not None: server_protocol['raw_banner'] = raw_banner protocol_version = sp['protocol_version'].resolve() if protocol_version is not None: server_protocol['protocol_version'] = protocol_version software_version = sp['software_version'].resolve() if software_version is not None: server_protocol['software_version'] = software_version comment = sp['comment'].resolve() if comment is not None: server_protocol['comment'] = comment if len(server_protocol) == 0: raise errors.IgnoreObject("Empty server protocol output dict") zout = ZMapTransformOutput() zout.transformed = server_protocol return zout
def _transform_object(self, obj): http = Transformable(obj) http_response = http['data']['http']['response'] zout = ZMapTransformOutput() out = dict() error_component = http['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("connection error") if http_response is not None: status_line = http_response['status_line'].resolve() status_code = http_response['status_code'].resolve() body = http_response['body'].resolve() body_sha256 = http_response['body_sha256'].resolve() headers = http_response['headers'].resolve() if status_line is not None: out['status_line'] = status_line if status_code is not None: out['status_code'] = status_code if body is not None: out['body'] = body m = title_regex.search(body) if m: title = m.group(1) if len(title) > 1024: title = title[0:1024] out['title'] = title if headers is not None: out['headers'] = headers if body_sha256: out['body_sha256'] = body_sha256 if len(out) == 0: raise errors.IgnoreObject("Empty output dict") zout.transformed = out return zout
def make_tls_obj(tls): out = dict() wrapped = Transformable(tls) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") certificates = [] hello = wrapped['server_hello'] server_certificates = wrapped['server_certificates'] cipher_suite = hello['cipher_suite'] validation = server_certificates['validation'] server_key_exchange = wrapped['server_key_exchange'] ecdh = server_key_exchange['ecdh_params']['curve_id'] dh = server_key_exchange['dh_params'] rsa = server_key_exchange['rsa_params'] signature = server_key_exchange['signature'] signature_hash = signature['signature_and_hash_type'] version = hello['version']['name'].resolve() if version is not None: out['version'] = version session_ticket_len = wrapped['session_ticket']['length'].resolve() session_ticket_hint = wrapped['session_ticket'][ 'lifetime_hint'].resolve() if session_ticket_len is not None or session_ticket_hint is not None: out['session_ticket'] = dict() if session_ticket_len is not None: out['session_ticket']['length'] = session_ticket_len if session_ticket_hint is not None: out['session_ticket']['lifetime_hint'] = session_ticket_hint scts = hello["scts"].resolve() if scts: out["scts"] = [{ "log_id": sct["parsed"]["log_id"], "timestamp": sct["parsed"]["timestamp"] / 1000, "signature": sct["parsed"]["signature"], "version": sct["parsed"]["version"] } for sct in scts] cipher_id = cipher_suite['hex'].resolve() cipher_name = cipher_suite['name'].resolve() ocsp_stapling = hello['ocsp_stapling'].resolve() secure_renegotiation = hello['secure_renegotiation'].resolve() if cipher_id or cipher_name is not None: out['cipher_suite'] = dict() if cipher_id is not None: out['cipher_suite']['id'] = cipher_id if cipher_name is not None: out['cipher_suite']['name'] = cipher_name if ocsp_stapling is not None: out['ocsp_stapling'] = ocsp_stapling cert = server_certificates['certificate'].resolve() if cert is not None: out['certificate'] = { 'parsed': cert['parsed'], } certificates.append(cert) chain = wrapped['server_certificates']['chain'].resolve() if chain is not None: out['chain'] = [{'parsed': c['parsed']} for c in chain] cert['parents'] = list() for c in chain: certificates.append(c) cert['parents'].append(c['parsed']['fingerprint_sha256']) browser_trusted = validation['browser_trusted'].resolve() browser_error = validation['browser_error'].resolve() matches_domain = validation['matches_domain'].resolve() if browser_trusted is not None or browser_error is not None \ or matches_domain is not None: out['validation'] = dict() if browser_trusted is not None: out['validation']['browser_trusted'] = browser_trusted cert['nss_trusted'] = browser_trusted if browser_error is not None: out['validation']['browser_error'] = browser_error if matches_domain is not None: out['validation']['matches_domain'] = matches_domain ecdh_name = ecdh['name'].resolve() ecdh_id = ecdh['id'].resolve() dh_prime_value = dh['prime']['value'].resolve() dh_prime_length = dh['prime']['length'].resolve() dh_generator_value = dh['generator']['value'].resolve() dh_generator_length = dh['generator']['length'].resolve() rsa_exponent = rsa['exponent'].resolve() rsa_modulus = rsa['modulus'].resolve() rsa_length = rsa['length'].resolve() ecdh_name = ecdh['name'].resolve() ecdh_id = ecdh['id'].resolve() if ecdh_name or dh_prime_value or dh_generator_value or rsa_exponent is not None: out['server_key_exchange'] = dict() if ecdh_name or ecdh_id is not None: out['server_key_exchange']['ecdh_params'] = dict() out['server_key_exchange']['ecdh_params']['curve_id'] = dict() if ecdh_name is not None: out['server_key_exchange']['ecdh_params']['curve_id'][ 'name'] = ecdh_name if ecdh_id is not None: out['server_key_exchange']['ecdh_params']['curve_id'][ 'id'] = ecdh_id if dh_prime_value or dh_generator_value is not None: out['server_key_exchange']['dh_params'] = dict() if dh_prime_value is not None: out['server_key_exchange']['dh_params']['prime'] = dict() if dh_generator_value is not None: out['server_key_exchange']['dh_params']['generator'] = dict() if dh_prime_value is not None: out['server_key_exchange']['dh_params']['prime'][ 'value'] = dh_prime_value if dh_prime_length is not None: out['server_key_exchange']['dh_params']['prime'][ 'length'] = dh_prime_length if dh_generator_value is not None: out['server_key_exchange']['dh_params']['generator'][ 'value'] = dh_generator_value if dh_generator_length is not None: out['server_key_exchange']['dh_params']['generator'][ 'length'] = dh_generator_length if rsa_exponent or rsa_modulus is not None: out['server_key_exchange']['rsa_params'] = dict() if rsa_exponent is not None: out['server_key_exchange']['rsa_params']['exponent'] = rsa_exponent if rsa_modulus is not None: out['server_key_exchange']['rsa_params']['modulus'] = rsa_modulus if rsa_length is not None: out['server_key_exchange']['rsa_params']['length'] = rsa_length signature_valid = signature['valid'].resolve() signature_error = signature['signature_error'].resolve() signature_algorithm = signature_hash['signature_algorithm'].resolve() signature_hash = signature_hash['hash_algorithm'].resolve() if signature_valid or signature_error is not None: out['signature'] = dict() if signature_valid is not None: out['signature']['valid'] = signature_valid if signature_error is not None: out['signature']['signature_error'] = signature_error if signature_algorithm is not None: out['signature']['signature_algorithm'] = signature_algorithm if signature_hash is not None: out['signature']['hash_algorithm'] = signature_hash if len(out) == 0: raise errors.IgnoreObject("Empty output dict") return out, certificates
def _transform_object(self, obj): grab = Transformable(obj)['data']['xssh'] out = dict() # banner: set_value(out, 'banner', grab['server_id'].resolve()) # support: support = dict() set_value(support, 'kex_algorithms', grab['server_key_exchange']['kex_algorithms'].resolve()) set_value(support, 'host_key_algorithms', grab['server_key_exchange']['host_key_algorithms'].resolve()) set_value(support, 'first_kex_follows', grab['server_key_exchange']['first_kex_follows'].resolve()) client_to_server = dict() set_value( client_to_server, 'ciphers', grab['server_key_exchange']['client_to_server_ciphers'].resolve()) set_value( client_to_server, 'macs', grab['server_key_exchange']['client_to_server_macs'].resolve()) set_value( client_to_server, 'compressions', grab['server_key_exchange'] ['client_to_server_compression'].resolve()) set_value( client_to_server, 'languages', grab['server_key_exchange'] ['client_to_server_languages'].resolve()) set_value(support, 'client_to_server', client_to_server) server_to_client = dict() set_value( server_to_client, 'ciphers', grab['server_key_exchange']['server_to_client_ciphers'].resolve()) set_value( server_to_client, 'macs', grab['server_key_exchange']['server_to_client_macs'].resolve()) set_value( server_to_client, 'compressions', grab['server_key_exchange'] ['server_to_client_compression'].resolve()) set_value( server_to_client, 'languages', grab['server_key_exchange'] ['server_to_client_languages'].resolve()) set_value(support, 'server_to_client', server_to_client) set_value(out, 'support', support) # selected: selected = grab['algorithm_selection'].resolve() if selected is not None: rename_key(selected, 'dh_kex_algorithm', 'kex_algorithm') rename_key(selected, 'client_to_server_alg_group', 'client_to_server') rename_key(selected, 'server_to_client_alg_group', 'server_to_client') set_value(out, 'selected', selected) # key_exchange: key_exchange = dict() set_value(key_exchange, 'ecdh_params', grab['key_exchange']['ecdh_params'].resolve()) set_value(key_exchange, 'dh_params', grab['key_exchange']['dh_params'].resolve()) try: del key_exchange['dh_params']['server_public'] except KeyError: pass set_value(out, 'key_exchange', key_exchange) # server_host_key: host_key = dict() for clone_key in [ 'fingerprint_sha256', 'rsa_public_key', 'dsa_public_key', 'ecdsa_public_key', 'ed25519_public_key' ]: # a bunch of keys we just need to copy verbatim from the grab set_value( host_key, clone_key, grab['key_exchange']['server_host_key'][clone_key].resolve()) set_value( host_key, 'key_algorithm', grab['key_exchange']['server_host_key']['algorithm'].resolve()) certkey_public_key = grab['key_exchange']['server_host_key'][ 'certkey_public_key'].resolve() if certkey_public_key is not None: rename_key(certkey_public_key, 'cert_type', 'type') # 2018/09/11: Workaround for mis-typed CertType.id field in ES; actual type is uint32, # current ES type is keyword (string). Added 'exclude={elasticsearch}' to the schema, # but to prevent attempted insertions of mistyped values, we need to actually remove the # id field as well. if 'type' in certkey_public_key: cert_type = certkey_public_key['type'] del_key(cert_type, 'id') del_key(certkey_public_key, 'reserved') signature_key = certkey_public_key.get('signature_key') if signature_key is not None: del_key(signature_key, 'raw') rename_key(signature_key, 'algorithm', 'key_algorithm') certkey_public_key['signature_key'] = signature_key # rewrite extensions and critical_options to maps of string -> bool: extensions = certkey_public_key.get('extensions') if extensions is not None: certkey_public_key['extensions'] = rewrite_known(extensions) critical_options = certkey_public_key.get('critical_options') if critical_options is not None: certkey_public_key['critical_options'] = rewrite_known( critical_options) key = certkey_public_key.get('key') if key is not None: del_key(key, 'raw') certkey_public_key['key'] = key sig = certkey_public_key.get('signature') if sig is not None: formatted_sig = dict() parsed = sig.get('parsed') if parsed is not None: set_value(formatted_sig, 'value', parsed.get('value')) alg = dict() set_value(alg, 'name', parsed.get('algorithm')) set_value(formatted_sig, 'signature_algorithm', alg) certkey_public_key['signature'] = formatted_sig set_value(host_key, 'certkey_public_key', certkey_public_key) set_value(out, 'server_host_key', host_key) if len(out) == 0: raise errors.IgnoreObject("Empty SSH protocol output dict") zout = ZMapTransformOutput() zout.transformed = out return zout