def _transform_object(self, obj): if 'tls' not in obj['data']: raise errors.IgnoreObject("Not a TLS response") tls = obj['data']['tls'] out, certificates = HTTPSTransform.make_tls_obj(tls) zout = ZMapTransformOutput() zout.transformed = out zout.certificates = certificates return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() tempout = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") tls_handshake = wrapped['data']['tls'] value = tls_handshake['server_hello']['version']['value'].resolve() out = dict() if value is not None: version = int(value) out['support'] = True if version == 772 else False else: raise errors.IgnoreObject("Empty output dict") zout.transformed = out return zout
def _transform_object(self, obj): ftp_banner = obj ftp = Transformable(obj) zout = ZMapTransformOutput() error = ftp['error'].resolve() if error is not None: raise errors.IgnoreObject("Error") out = dict() banner = ftp['data']['banner'].resolve() if banner is not None: out['banner'] = self.clean_banner(banner) if len(out) == 0: raise errors.IgnoreObject("Empty output dict") out['ip_address'] = obj['ip'] out['timestamp'] = obj['timestamp'] zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() wrapped = Transformable(obj) s = wrapped['data']['s7'] if not s['is_s7'].resolve() or not s.resolve(): raise errors.IgnoreObject() out = s.resolve() out['support'] = True del out['is_s7'] zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() wrapped = Transformable(obj) dnp3 = wrapped['data']['dnp3'] if not dnp3['is_dnp3'].resolve(): raise errors.IgnoreObject() out = { "support": True, "raw_response": dnp3["raw_response"].resolve(), } zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() wrapped = Transformable(obj) modbus = wrapped['data']['modbus'] if not modbus['raw_response'].resolve(): raise errors.IgnoreObject() out = dict() out["support"] = True function_code = modbus['function_code'].resolve() if function_code: out['function_code'] = function_code mei_response = modbus['mei_response'] if mei_response: conformity_level = mei_response['conformity_level'].resolve() objects = mei_response['objects'] vendor = objects['vendor'].resolve() product_code = objects['product_code'].resolve() revision = objects['revision'].resolve() vendor_url = objects['vendor_url'].resolve() product_name = objects['product_name'].resolve() model_name = objects['model_name'].resolve() user_application_name = objects['user_application_name'].resolve() if conformity_level or vendor or product_code or revision or \ vendor_url or product_name or model_name or user_application_name: out['mei_response'] = dict() if vendor or product_code or revision or vendor_url or product_name \ or model_name or user_application_name: out['mei_response']['objects'] = dict() if conformity_level: out['mei_response']['conformity_level'] = conformity_level if vendor: out['mei_response']['objects']['vendor'] = vendor if product_code: out['mei_response']['objects']['product_code'] = product_code if revision: out['mei_response']['objects']['revision'] = revision if vendor_url: out['mei_response']['objects']['vendor_url'] = vendor_url if product_name: out['mei_response']['objects']['product_name'] = product_name if model_name: out['mei_response']['objects']['model_name'] = model_name if user_application_name: out['mei_response']['objects']['user_application_name'] = \ user_application_name zout.transformed = out return zout
def _transform_object(self, obj): ssh = Transformable(obj) server_protocol = dict() sp = ssh['data']['ssh']['server_protocol'] if sp.resolve() is None: raise errors.IgnoreObject("no ssh server protocol") raw_banner = sp['raw_banner'].resolve() if raw_banner is not None: server_protocol['raw_banner'] = raw_banner protocol_version = sp['protocol_version'].resolve() if protocol_version is not None: server_protocol['protocol_version'] = protocol_version software_version = sp['software_version'].resolve() if software_version is not None: server_protocol['software_version'] = software_version comment = sp['comment'].resolve() if comment is not None: server_protocol['comment'] = comment if len(server_protocol) == 0: raise errors.IgnoreObject("Empty server protocol output dict") zout = ZMapTransformOutput() zout.transformed = server_protocol return zout
def _transform_object(self, obj): http = Transformable(obj) http_response = http['data']['http']['response'] zout = ZMapTransformOutput() out = dict() error_component = http['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("connection error") if http_response is not None: status_line = http_response['status_line'].resolve() status_code = http_response['status_code'].resolve() body = http_response['body'].resolve() body_sha256 = http_response['body_sha256'].resolve() headers = http_response['headers'].resolve() if status_line is not None: out['status_line'] = status_line if status_code is not None: out['status_code'] = status_code if body is not None: out['body'] = body m = title_regex.search(body) if m: title = m.group(1) if len(title) > 1024: title = title[0:1024] out['title'] = title if headers is not None: out['headers'] = headers if body_sha256: out['body_sha256'] = body_sha256 if len(out) == 0: raise errors.IgnoreObject("Empty output dict") zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() out = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") server_hello = wrapped['data']['tls']['server_hello'] if server_hello.resolve() is None: raise IgnoreObject("missing server hello") exr = server_hello['extended_random'].resolve() if exr is not None: out['extended_random_support'] = True else: out['extended_random_support'] = False zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() wrapped = Transformable(obj) fox = wrapped['data']['fox'] if not fox['is_fox'].resolve(): raise errors.IgnoreObject() instance_number = fox['instance_number'].resolve() version = fox['version'].resolve() id_ = fox['id'].resolve() hostname = fox['hostname'].resolve() host_address = fox['host_address'].resolve() app_name = fox['app_name'].resolve() app_version = fox['app_version'].resolve() vm_name = fox['vm_name'].resolve() vm_version = fox['vm_version'].resolve() os_name = fox['os_name'].resolve() os_version = fox['os_version'].resolve() station_name = fox['station_name'].resolve() language = fox['language'].resolve() time_zone = fox['time_zone'].resolve() host_id = fox['host_id'].resolve() vm_uuid = fox['vm_uuid'].resolve() brand_id = fox['brand_id'].resolve() sys_info = fox['sys_info'].resolve() auth_agent_type = fox['auth_agent_type'].resolve() out = dict() out["support"] = True if instance_number: out["instance_number"] = instance_number if version: out["version"] = version if id_: out["id"] = id_ if hostname: out["hostname"] = hostname if host_address: out["host_address"] = host_address if app_name: out["app_name"] = app_name if app_version: out["app_version"] = app_version if vm_name: out["vm_name"] = vm_name if vm_version: out["vm_version"] = vm_version if os_name: out["os_name"] = os_name if os_version: out["os_version"] = os_version if station_name: out["station_name"] = station_name if language: out["language"] = language if time_zone: out["time_zone"] = time_zone if host_id: out["host_id"] = host_id if vm_uuid: out["vm_uuid"] = vm_uuid if brand_id: out["brand_id"] = brand_id if sys_info: out["sys_info"] = sys_info if auth_agent_type: out["auth_agent_type"] = auth_agent_type zout.transformed = out return zout
def _transform_object(self, obj): grab = Transformable(obj)['data']['xssh'] out = dict() # banner: set_value(out, 'banner', grab['server_id'].resolve()) # support: support = dict() set_value(support, 'kex_algorithms', grab['server_key_exchange']['kex_algorithms'].resolve()) set_value(support, 'host_key_algorithms', grab['server_key_exchange']['host_key_algorithms'].resolve()) set_value(support, 'first_kex_follows', grab['server_key_exchange']['first_kex_follows'].resolve()) client_to_server = dict() set_value( client_to_server, 'ciphers', grab['server_key_exchange']['client_to_server_ciphers'].resolve()) set_value( client_to_server, 'macs', grab['server_key_exchange']['client_to_server_macs'].resolve()) set_value( client_to_server, 'compressions', grab['server_key_exchange'] ['client_to_server_compression'].resolve()) set_value( client_to_server, 'languages', grab['server_key_exchange'] ['client_to_server_languages'].resolve()) set_value(support, 'client_to_server', client_to_server) server_to_client = dict() set_value( server_to_client, 'ciphers', grab['server_key_exchange']['server_to_client_ciphers'].resolve()) set_value( server_to_client, 'macs', grab['server_key_exchange']['server_to_client_macs'].resolve()) set_value( server_to_client, 'compressions', grab['server_key_exchange'] ['server_to_client_compression'].resolve()) set_value( server_to_client, 'languages', grab['server_key_exchange'] ['server_to_client_languages'].resolve()) set_value(support, 'server_to_client', server_to_client) set_value(out, 'support', support) # selected: selected = grab['algorithm_selection'].resolve() if selected is not None: rename_key(selected, 'dh_kex_algorithm', 'kex_algorithm') rename_key(selected, 'client_to_server_alg_group', 'client_to_server') rename_key(selected, 'server_to_client_alg_group', 'server_to_client') set_value(out, 'selected', selected) # key_exchange: key_exchange = dict() set_value(key_exchange, 'ecdh_params', grab['key_exchange']['ecdh_params'].resolve()) set_value(key_exchange, 'dh_params', grab['key_exchange']['dh_params'].resolve()) try: del key_exchange['dh_params']['server_public'] except KeyError: pass set_value(out, 'key_exchange', key_exchange) # server_host_key: host_key = dict() for clone_key in [ 'fingerprint_sha256', 'rsa_public_key', 'dsa_public_key', 'ecdsa_public_key', 'ed25519_public_key' ]: # a bunch of keys we just need to copy verbatim from the grab set_value( host_key, clone_key, grab['key_exchange']['server_host_key'][clone_key].resolve()) set_value( host_key, 'key_algorithm', grab['key_exchange']['server_host_key']['algorithm'].resolve()) certkey_public_key = grab['key_exchange']['server_host_key'][ 'certkey_public_key'].resolve() if certkey_public_key is not None: rename_key(certkey_public_key, 'cert_type', 'type') # 2018/09/11: Workaround for mis-typed CertType.id field in ES; actual type is uint32, # current ES type is keyword (string). Added 'exclude={elasticsearch}' to the schema, # but to prevent attempted insertions of mistyped values, we need to actually remove the # id field as well. if 'type' in certkey_public_key: cert_type = certkey_public_key['type'] del_key(cert_type, 'id') del_key(certkey_public_key, 'reserved') signature_key = certkey_public_key.get('signature_key') if signature_key is not None: del_key(signature_key, 'raw') rename_key(signature_key, 'algorithm', 'key_algorithm') certkey_public_key['signature_key'] = signature_key # rewrite extensions and critical_options to maps of string -> bool: extensions = certkey_public_key.get('extensions') if extensions is not None: certkey_public_key['extensions'] = rewrite_known(extensions) critical_options = certkey_public_key.get('critical_options') if critical_options is not None: certkey_public_key['critical_options'] = rewrite_known( critical_options) key = certkey_public_key.get('key') if key is not None: del_key(key, 'raw') certkey_public_key['key'] = key sig = certkey_public_key.get('signature') if sig is not None: formatted_sig = dict() parsed = sig.get('parsed') if parsed is not None: set_value(formatted_sig, 'value', parsed.get('value')) alg = dict() set_value(alg, 'name', parsed.get('algorithm')) set_value(formatted_sig, 'signature_algorithm', alg) certkey_public_key['signature'] = formatted_sig set_value(host_key, 'certkey_public_key', certkey_public_key) set_value(out, 'server_host_key', host_key) if len(out) == 0: raise errors.IgnoreObject("Empty SSH protocol output dict") zout = ZMapTransformOutput() zout.transformed = out return zout
def make_tls_obj(tls): out = dict() wrapped = Transformable(tls) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") certificates = [] hello = wrapped['server_hello'] server_certificates = wrapped['server_certificates'] cipher_suite = hello['cipher_suite'] validation = server_certificates['validation'] server_key_exchange = wrapped['server_key_exchange'] ecdh = server_key_exchange['ecdh_params']['curve_id'] dh = server_key_exchange['dh_params'] rsa = server_key_exchange['rsa_params'] signature = server_key_exchange['signature'] signature_hash = signature['signature_and_hash_type'] version = hello['version']['name'].resolve() if version is not None: out['version'] = version session_ticket_len = wrapped['session_ticket']['length'].resolve() session_ticket_hint = wrapped['session_ticket'][ 'lifetime_hint'].resolve() if session_ticket_len is not None or session_ticket_hint is not None: out['session_ticket'] = dict() if session_ticket_len is not None: out['session_ticket']['length'] = session_ticket_len if session_ticket_hint is not None: out['session_ticket']['lifetime_hint'] = session_ticket_hint scts = hello["scts"].resolve() if scts: out["scts"] = [{ "log_id": sct["parsed"]["log_id"], "timestamp": sct["parsed"]["timestamp"] / 1000, "signature": sct["parsed"]["signature"], "version": sct["parsed"]["version"] } for sct in scts] cipher_id = cipher_suite['hex'].resolve() cipher_name = cipher_suite['name'].resolve() ocsp_stapling = hello['ocsp_stapling'].resolve() secure_renegotiation = hello['secure_renegotiation'].resolve() if cipher_id or cipher_name is not None: out['cipher_suite'] = dict() if cipher_id is not None: out['cipher_suite']['id'] = cipher_id if cipher_name is not None: out['cipher_suite']['name'] = cipher_name if ocsp_stapling is not None: out['ocsp_stapling'] = ocsp_stapling cert = server_certificates['certificate'].resolve() if cert is not None: out['certificate'] = { 'parsed': cert['parsed'], } certificates.append(cert) chain = wrapped['server_certificates']['chain'].resolve() if chain is not None: out['chain'] = [{'parsed': c['parsed']} for c in chain] cert['parents'] = list() for c in chain: certificates.append(c) cert['parents'].append(c['parsed']['fingerprint_sha256']) browser_trusted = validation['browser_trusted'].resolve() browser_error = validation['browser_error'].resolve() matches_domain = validation['matches_domain'].resolve() if browser_trusted is not None or browser_error is not None \ or matches_domain is not None: out['validation'] = dict() if browser_trusted is not None: out['validation']['browser_trusted'] = browser_trusted cert['nss_trusted'] = browser_trusted if browser_error is not None: out['validation']['browser_error'] = browser_error if matches_domain is not None: out['validation']['matches_domain'] = matches_domain ecdh_name = ecdh['name'].resolve() ecdh_id = ecdh['id'].resolve() dh_prime_value = dh['prime']['value'].resolve() dh_prime_length = dh['prime']['length'].resolve() dh_generator_value = dh['generator']['value'].resolve() dh_generator_length = dh['generator']['length'].resolve() rsa_exponent = rsa['exponent'].resolve() rsa_modulus = rsa['modulus'].resolve() rsa_length = rsa['length'].resolve() ecdh_name = ecdh['name'].resolve() ecdh_id = ecdh['id'].resolve() if ecdh_name or dh_prime_value or dh_generator_value or rsa_exponent is not None: out['server_key_exchange'] = dict() if ecdh_name or ecdh_id is not None: out['server_key_exchange']['ecdh_params'] = dict() out['server_key_exchange']['ecdh_params']['curve_id'] = dict() if ecdh_name is not None: out['server_key_exchange']['ecdh_params']['curve_id'][ 'name'] = ecdh_name if ecdh_id is not None: out['server_key_exchange']['ecdh_params']['curve_id'][ 'id'] = ecdh_id if dh_prime_value or dh_generator_value is not None: out['server_key_exchange']['dh_params'] = dict() if dh_prime_value is not None: out['server_key_exchange']['dh_params']['prime'] = dict() if dh_generator_value is not None: out['server_key_exchange']['dh_params']['generator'] = dict() if dh_prime_value is not None: out['server_key_exchange']['dh_params']['prime'][ 'value'] = dh_prime_value if dh_prime_length is not None: out['server_key_exchange']['dh_params']['prime'][ 'length'] = dh_prime_length if dh_generator_value is not None: out['server_key_exchange']['dh_params']['generator'][ 'value'] = dh_generator_value if dh_generator_length is not None: out['server_key_exchange']['dh_params']['generator'][ 'length'] = dh_generator_length if rsa_exponent or rsa_modulus is not None: out['server_key_exchange']['rsa_params'] = dict() if rsa_exponent is not None: out['server_key_exchange']['rsa_params']['exponent'] = rsa_exponent if rsa_modulus is not None: out['server_key_exchange']['rsa_params']['modulus'] = rsa_modulus if rsa_length is not None: out['server_key_exchange']['rsa_params']['length'] = rsa_length signature_valid = signature['valid'].resolve() signature_error = signature['signature_error'].resolve() signature_algorithm = signature_hash['signature_algorithm'].resolve() signature_hash = signature_hash['hash_algorithm'].resolve() if signature_valid or signature_error is not None: out['signature'] = dict() if signature_valid is not None: out['signature']['valid'] = signature_valid if signature_error is not None: out['signature']['signature_error'] = signature_error if signature_algorithm is not None: out['signature']['signature_algorithm'] = signature_algorithm if signature_hash is not None: out['signature']['hash_algorithm'] = signature_hash if len(out) == 0: raise errors.IgnoreObject("Empty output dict") return out, certificates