def _transform_object(self, obj): zout = ZMapTransformOutput() out = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") rsa = wrapped['server_key_exchange']['rsa_params'] rsa_exponent = rsa['exponent'].resolve() rsa_modulus = rsa['modulus'].resolve() rsa_length = rsa['length'].resolve() if rsa_exponent or rsa_modulus is not None: out['rsa_params'] = dict() if rsa_exponent is not None: out['rsa_params']['exponent'] = rsa_exponent if rsa_modulus is not None: out['rsa_params']['modulus'] = rsa_modulus if rsa_length is not None: out['rsa_params']['length'] = rsa_length out["support"] = True if len(out) > 0 else False zout.transformed = out return zout
def _transform_object(self, obj): tls = obj['data']['tls'] out, certificates = HTTPSTransform.make_tls_obj(tls) zout = ZMapTransformOutput() zout.transformed = out zout.certificates = certificates return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() out = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") ecdh = wrapped['data']['tls']['server_key_exchange']['ecdh_params'][ 'curve_id'] ecdh_name = ecdh['name'].resolve() ecdh_id = ecdh['id'].resolve() if ecdh_name or ecdh_id is not None: out['ecdh_params'] = dict() out['ecdh_params']['curve_id'] = dict() if ecdh_name is not None: out['ecdh_params']['curve_id']['name'] = ecdh_name if ecdh_id is not None: out['ecdh_params']['curve_id']['id'] = ecdh_id out["support"] = bool(out) zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() wrapped = Transformable(obj) modbus = wrapped["data"]["modbus"] if not modbus["raw_response"].resolve(): raise errors.IgnoreObject() out = dict() function_code = modbus["function_code"].resolve() mei_response = modbus["mei_response"] if function_code: out["function_code"] = function_code if mei_response: conformity_level = mei_response["conformity_level"].resolve() objects = mei_response["objects"] vendor = objects["vendor"].resolve() product_code = objects["product_code"].resolve() revision = objects["revision"].resolve() vendor_url = objects["vendor_url"].resolve() product_name = objects["product_name"].resolve() model_name = objects["model_name"].resolve() user_application_name = objects["user_application_name"].resolve() if ( conformity_level or vendor or product_code or revision or vendor_url or product_name or model_name or user_application_name ): out["mei_response"] = dict() if vendor or product_code or revision or vendor_url or product_name or model_name or user_application_name: out["mei_response"]["objects"] = dict() if conformity_level: out["mei_response"]["conformity_level"] = conformity_level if vendor: out["mei_response"]["objects"]["vendor"] = vendor if product_code: out["mei_response"]["objects"]["product_code"] = product_code if revision: out["mei_response"]["objects"]["revision"] = revision if vendor_url: out["mei_response"]["objects"]["vendor_url"] = vendor_url if product_name: out["mei_response"]["objects"]["product_name"] = product_name if model_name: out["mei_response"]["objects"]["model_name"] = model_name if user_application_name: out["mei_response"]["objects"]["user_application_name"] = user_application_name if len(out) == 0: raise errors.IgnoreObject("Empty output dict") zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") banner = wrapped['data']['banner'].resolve() try: tls_handshake = obj['data']['tls'] out, certificates = https.HTTPSTransform.make_tls_obj( tls_handshake) zout.transformed['tls'] = out zout.certificates = certificates except (TypeError, KeyError, IndexError): pass if banner is not None: zout.transformed['banner'] = self.clean_banner(banner) if len(zout.transformed) == 0: raise errors.IgnoreObject("Empty Dict output") return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() out = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") heartbleed = wrapped['data']['heartbleed'] heartbleed = wrapped['data']['heartbleed'] heartbleed_enabled = heartbleed['heartbeat_enabled'].resolve() heartbleed_vulnerable = heartbleed['heartbleed_vulnerable'].resolve() if heartbleed_enabled is not None: out['heartbeat_enabled'] = heartbleed_enabled if heartbleed_vulnerable is not None: out['heartbleed_vulnerable'] = heartbleed_vulnerable if len(out) == 0: raise errors.IgnoreObject("Empty Output dict") zout.transformed = out return zout
def _transform_object(self, obj): http = Transformable(obj) http_response = http['data']['http']['response'] zout = ZMapTransformOutput() out = dict() error_component = http['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("connection error") if http_response is not None: status_line = http_response['status_line'].resolve() status_code = http_response['status_code'].resolve() body = http_response['body'].resolve() headers = http_response['headers'].resolve() if status_line is not None: out['status_line'] = status_line if status_code is not None: out['status_code'] = status_code if body is not None: out['body'] = body if headers is not None: out['headers'] = headers if len(out) == 0: raise errors.IgnoreObject("Empty output dict") zout.transformed = out return zout
def _transform_object(self, obj): wrapped = Transformable(obj['data']) ciphers = wrapped['sslv2']['server_hello']['ciphers'].resolve() certificate = wrapped['sslv2']['server_hello']['certificate'].resolve() sslv2_support = bool(wrapped['sslv2']['server_verify'].resolve()) sslv2_export = bool(wrapped['sslv2_export']['server_verify'].resolve()) sslv2_extra_clear = bool(wrapped['sslv2_extra_clear']['server_verify'] ['extra_clear'].resolve()) out = { 'support': sslv2_support, 'export': sslv2_export, 'extra_clear': sslv2_extra_clear, } if ciphers is not None: out['ciphers'] = ciphers if certificate is not None: out['certificate'] = { 'parsed': certificate['parsed'] } certificates = [certificate] else: certificates = list() zout = ZMapTransformOutput() zout.transformed = out zout.certificates = certificates return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() out = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") rsa = wrapped['data']['tls']['server_key_exchange']['rsa_params'] rsa_exponent = rsa['exponent'].resolve() rsa_modulus = rsa['modulus'].resolve() rsa_length = rsa['length'].resolve() if rsa_exponent or rsa_modulus is not None: out['rsa_params'] = dict() if rsa_exponent is not None: out['rsa_params']['exponent'] = rsa_exponent if rsa_modulus is not None: out['rsa_params']['modulus'] = rsa_modulus if rsa_length is not None: out['rsa_params']['length'] = rsa_length out["support"] = bool(out) zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() out = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") dh = wrapped['data']['tls']['server_key_exchange']['dh_params'] dh_prime_value = dh['prime']['value'].resolve() dh_prime_length = dh['prime']['length'].resolve() dh_generator_value = dh['generator']['value'].resolve() dh_generator_length = dh['generator']['length'].resolve() if dh_prime_value or dh_generator_value is not None: out['dh_params'] = dict() if dh_prime_value is not None: out['dh_params']['prime'] = dict() if dh_generator_value is not None: out['dh_params']['generator'] = dict() if dh_prime_value is not None: out['dh_params']['prime']['value'] = dh_prime_value if dh_prime_length is not None: out['dh_params']['prime']['length'] = dh_prime_length if dh_generator_value is not None: out['dh_params']['generator']['value'] = dh_generator_value if dh_generator_length is not None: out['dh_params']['generator']['length'] = dh_generator_length out["support"] = bool(out) zout.transformed = out return zout
def _transform_object(self, obj): wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") banner = wrapped['data']['banner'].resolve() zout = ZMapTransformOutput() try: tls_handshake = obj['data']['tls'] out, certificates = https.HTTPSTransform.make_tls_obj(tls_handshake) zout.transformed['tls'] = out zout.certificates = certificates except (TypeError, KeyError, IndexError): pass if banner is not None: zout.transformed['banner'] = self.clean_banner(banner) if len(zout.transformed) == 0: raise errors.IgnoreObject("Empty output dict") return zout
def _transform_object(self, obj): wrapped = Transformable(obj['data']) ciphers = wrapped['sslv2']['server_hello']['ciphers'].resolve() certificate = wrapped['sslv2']['server_hello']['certificate'].resolve() sslv2_support = bool(wrapped['sslv2']['server_verify'].resolve()) sslv2_export = bool(wrapped['sslv2_export']['server_verify'].resolve()) sslv2_extra_clear = bool(wrapped['sslv2_extra_clear']['server_verify'] ['extra_clear'].resolve()) out = { 'support': sslv2_support, 'export': sslv2_export, 'extra_clear': sslv2_extra_clear, } if ciphers is not None: out['ciphers'] = ciphers if certificate is not None: out['certificate'] = {'parsed': certificate['parsed']} certificates = [certificate] else: certificates = list() zout = ZMapTransformOutput() zout.transformed = out zout.certificates = certificates return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() out = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") dh = wrapped['server_key_exchange']['dh_params'] dh_prime_value = dh['prime']['value'].resolve() dh_prime_length = dh['prime']['length'].resolve() dh_generator_value = dh['generator']['value'].resolve() dh_generator_length = dh['generator']['length'].resolve() if dh_prime_value or dh_generator_value is not None: out['dh_params'] = dict() if dh_prime_value is not None: out['dh_params']['prime'] = dict() if dh_generator_value is not None: out['dh_params']['generator'] = dict() if dh_prime_value is not None: out['dh_params']['prime']['value'] = dh_prime_value if dh_prime_length is not None: out['dh_params']['prime']['length'] = dh_prime_length if dh_generator_value is not None: out['dh_params']['generator']['value'] = dh_generator_value if dh_generator_length is not None: out['dh_params']['generator']['length'] = dh_generator_length out["support"] = True if len(out) > 0 else False zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() out = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") ecdh = wrapped['server_key_exchange']['ecdh_params']['curve_id'] ecdh_name = ecdh['name'].resolve() ecdh_id = ecdh['id'].resolve() if ecdh_name or ecdh_id is not None: out['ecdh_params'] = dict() out['ecdh_params']['curve_id'] = dict() if ecdh_name is not None: out['ecdh_params']['curve_id']['name'] = ecdh_name if ecdh_id is not None: out['ecdh_params']['curve_id']['id'] = ecdh_id out["support"] = True if len(out) > 0 else False zout.transformed = out return zout
def encode(self, obj): pb_out = super(RecordEncoder, self).encode(obj) out = ZMapTransformOutput() out.transformed = pb_out.transformed.SerializeToString() out.certificates = [c.SerializeToString() for c in pb_out.certificates] out.public_keys = [pk.SerializeToString() for pk in pb_out.public_keys] return out
def _transform_object(self, obj): if 'tls' not in obj['data']: raise errors.IgnoreObject("Not a TLS response") tls = obj['data']['tls'] out, certificates = HTTPSTransform.make_tls_obj(tls) zout = ZMapTransformOutput() zout.transformed = out zout.certificates = certificates return zout
def _transform_object(self, obj): http = Transformable(obj) connect_response = http['data']['http']['connect_response'] get_response = http['data']['http']['response'] zout = ZMapTransformOutput() out = dict() error_component = http['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("connection error") if connect_response is not None: status_line = connect_response['status_line'].resolve() status_code = connect_response['status_code'].resolve() body = connect_response['body'].resolve() headers = connect_response['headers'].resolve() if status_line or status_code or body or headers is not None: out['connect'] = dict() if status_line is not None: out['connect']['status_line'] = status_line if status_code is not None: out['connect']['status_code'] = status_code if body is not None: out['connect']['body'] = body if headers is not None: out['connect']['headers'] = headers if get_response is not None: status_line = get_response['status_line'].resolve() status_code = get_response['status_code'].resolve() body = get_response['body'].resolve() headers = get_response['headers'].resolve() body_sha256 = get_response['body_sha256'].resolve() out['get'] = dict() if body: random_present = "Uh2Qn8Y7NPRm6h3xqEXUq4EhtW7Po4gy" in body else: random_present = False out['get']['random_present'] = random_present if status_line is not None: out['get']['status_line'] = status_line if status_code is not None: out['get']['status_code'] = status_code if body is not None: out['get']['body'] = body if headers is not None: out['get']['headers'] = headers if body_sha256: out['get']['body_sha256'] = body_sha256 if len(out) == 0: raise errors.IgnoreObject("Empty output dict") zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() wrapped = Transformable(obj) modbus = wrapped['data']['modbus'] if not modbus['raw_response'].resolve(): raise errors.IgnoreObject() out = dict() function_code = modbus['function_code'].resolve() mei_response = modbus['mei_response'] if function_code: out['function_code'] = function_code if mei_response: conformity_level = mei_response['conformity_level'].resolve() objects = mei_response['objects'] vendor = objects['vendor'].resolve() product_code = objects['product_code'].resolve() revision = objects['revision'].resolve() vendor_url = objects['vendor_url'].resolve() product_name = objects['product_name'].resolve() model_name = objects['model_name'].resolve() user_application_name = objects['user_application_name'].resolve() if conformity_level or vendor or product_code or revision or \ vendor_url or product_name or model_name or user_application_name: out['mei_response'] = dict() if vendor or product_code or revision or vendor_url or product_name \ or model_name or user_application_name: out['mei_response']['objects'] = dict() if conformity_level: out['mei_response']['conformity_level'] = conformity_level if vendor: out['mei_response']['objects']['vendor'] = vendor if product_code: out['mei_response']['objects']['product_code'] = product_code if revision: out['mei_response']['objects']['revision'] = revision if vendor_url: out['mei_response']['objects']['vendor_url'] = vendor_url if product_name: out['mei_response']['objects']['product_name'] = product_name if model_name: out['mei_response']['objects']['model_name'] = model_name if user_application_name: out['mei_response']['objects']['user_application_name'] = \ user_application_name if len(out) == 0: raise errors.IgnoreObject("Empty output dict") zout.transformed = out return zout
def _transform_object(self, obj): http = Transformable(obj) http_response = http['data']['http']['response'] zout = ZMapTransformOutput() out = dict() zout.transformed = out error_component = http['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("connection error") if http_response is not None: status_line = http_response['status_line'].resolve() status_code = http_response['status_code'].resolve() body = http_response['body'].resolve() body_sha256 = http_response['body_sha256'].resolve() headers = http_response['headers'].resolve() if status_line is not None: out['status_line'] = status_line if status_code is not None: out['status_code'] = status_code if body is not None: out['body'] = body m = title_regex.search(body) if m: title = m.group(1) if len(title) > 1024: title = title[0:1024] out['title'] = title.strip() if headers: # FIXME: This modifies the input? if "set_cookie" in headers: del headers["set_cookie"] if "date" in headers: del headers["date"] for k, v, in headers.iteritems(): if k == "unknown": for d in v: if len(d["value"]) < 1: continue d["value"] = d["value"][0] elif v: headers[k] = v[0] else: del headers[k] out['headers'] = headers if body_sha256: out['body_sha256'] = body_sha256 if len(out) == 0: raise errors.IgnoreObject("Empty output dict") return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() wrapped = Transformable(obj) modbus = wrapped['data']['modbus'] if not modbus['raw_response'].resolve(): raise errors.IgnoreObject() out = dict() out["support"] = True function_code = modbus['function_code'].resolve() if function_code: out['function_code'] = function_code mei_response = modbus['mei_response'] if mei_response: conformity_level = mei_response['conformity_level'].resolve() objects = mei_response['objects'] vendor = objects['vendor'].resolve() product_code = objects['product_code'].resolve() revision = objects['revision'].resolve() vendor_url = objects['vendor_url'].resolve() product_name = objects['product_name'].resolve() model_name = objects['model_name'].resolve() user_application_name = objects['user_application_name'].resolve() if conformity_level or vendor or product_code or revision or \ vendor_url or product_name or model_name or user_application_name: out['mei_response'] = dict() if vendor or product_code or revision or vendor_url or product_name \ or model_name or user_application_name: out['mei_response']['objects'] = dict() if conformity_level: out['mei_response']['conformity_level'] = conformity_level if vendor: out['mei_response']['objects']['vendor'] = vendor if product_code: out['mei_response']['objects']['product_code'] = product_code if revision: out['mei_response']['objects']['revision'] = revision if vendor_url: out['mei_response']['objects']['vendor_url'] = vendor_url if product_name: out['mei_response']['objects']['product_name'] = product_name if model_name: out['mei_response']['objects']['model_name'] = model_name if user_application_name: out['mei_response']['objects']['user_application_name'] = \ user_application_name zout.transformed = out return zout
def _transform_object(self, obj): classification = obj['classification'] if classification != "upnp": raise errors.IgnoreObject(classification) success = int(obj['success']) if success != 1: raise errors.IgnoreObject("unsuccessful zmap") wrapped = Transformable(obj) transformed = {} for key in self.KEYS: transformed[key] = wrapped[key].resolve() out = ZMapTransformOutput() out.transformed = transformed return out
def _transform_object(self, obj): zout = ZMapTransformOutput() out = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") server_hello = wrapped['data']['tls']['server_hello'] if server_hello.resolve() is None: raise IgnoreObject("missing server hello") exr = server_hello['extended_random'].resolve() if exr is not None: out['extended_random_support'] = True else: out['extended_random_support'] = False zout.transformed = out return zout
def encode(self, zout): out = ZMapTransformOutput() obj = zout.transformed ip = obj['ip_address'] ts = obj['timestamp'] domain = obj.get('domain', None) if zout.metadata.local_metadata is not None: obj['metadata'] = zout.metadata.local_metadata.to_dict() m = zout.metadata.global_metadata.to_dict() tags = list(zout.metadata.tags) data = {x: obj[x] for x in obj if x not in self.DROP_KEYS} m = {k: simple_strip(v) for k, v in m.iteritems()} metadata = self.Metadatum.from_dict(m) atom = self.ProtocolAtom(tags=tags, metadata=metadata, data=data) record = self.Record(ip, self.port, self.protocol.value, self.subprotocol.value, protocol_atom=atom, domain=domain, timestamp=ts, scan_id=self.scan_id) out.transformed = record.protobuf out.certificates = [] for cert_dict in zout.certificates: ar = self.zsearch_definitions.anonstore_pb2.AnonymousRecord() c = ar.certificate c.parsed = json.dumps(cert_dict["parsed"], sort_keys=True) c.raw = base64.b64decode(cert_dict["raw"]) c.sha1fp = cert_dict["parsed"]["fingerprint_sha1"].decode("hex") c.sha256fp = cert_dict["parsed"]["fingerprint_sha256"].decode("hex") valid_nss = cert_dict.get("nss_trusted", None) if valid_nss is not None: c.valid_nss = valid_nss c.validation_timestamp = record.timestamp parents_sha256 = cert_dict.get("parents", None) if parents_sha256 is not None: c.parents.extend([ sha_value.decode("hex") for sha_value in parents_sha256 ]) ar.sha256fp = c.sha256fp ar.scan_id = self.scan_id out.certificates.append(ar) return out
def _transform_object(self, obj): classification = obj['classification'] if classification != "dns": raise errors.IgnoreObject(classification) if int(obj['success']) != 1: raise errors.IgnoreObject("Not a DNS resposne") zout = ZMapTransformOutput() out = dict() out["support"] = True errors_present = False if obj["dns_parse_err"] == True: errors_present = True out["questions"] = [] for question in obj["dns_questions"]: q = dict() q["name"] = question["name"] q["type"] = question["qtype_str"] out["questions"].append(q) response_types = (("answers", "dns_answers"), ("authorities", "dns_authorities"),\ ("additionals", "dns_additionals")) for out_field, obj_field in response_types: (response_errors, responses) = self._transform_responses(obj[obj_field]) if response_errors: errors_present = True out[out_field] = responses out["errors"] = errors_present out["open_resolver"] = bool(len(out["answers"]) + len(out["additionals"]) + \ len(out["authorities"])) out["resolves_correctly"] = False for answer in out["answers"]: if answer["type"] == "A" and answer[ "response"] == self.CORRECT_RESPONSE: out["resolves_correctly"] = True break zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() tempout = dict() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") tls_handshake = wrapped['data']['tls'] value = tls_handshake['server_hello']['version']['value'].resolve() out = dict() if value is not None: version = int(value) out['support'] = True if version == 772 else False else: raise errors.IgnoreObject("Empty output dict") zout.transformed = out return zout
def encode(self, zout): out = ZMapTransformOutput() obj = zout.transformed ip = obj['ip_address'] ts = obj['timestamp'] domain = obj.get('domain', None) if zout.metadata.local_metadata is not None: obj['metadata'] = zout.metadata.local_metadata.to_dict() m = zout.metadata.global_metadata.to_dict() tags = list(zout.metadata.tags) data = {x: obj[x] for x in obj if x not in self.DROP_KEYS} m = {k: simple_strip(v) for k, v in m.iteritems()} metadata = self.Metadatum.from_dict(m) atom = self.ProtocolAtom(tags=tags, metadata=metadata, data=data) record = self.Record(ip, self.port, self.protocol.value, self.subprotocol.value, protocol_atom=atom, domain=domain, timestamp=ts, scan_id=self.scan_id) out.transformed = record.protobuf out.certificates = [] # The chain may be helpful in validating this certificate later on # if some of the parents haven't been previously been by the cert # daemon. Therefore, pass along all raw certificates in the chain. # If chains were guaranteed to be presented in a rasonable order, we # could just pass up [n+1:], but people get this wrong all the time, # so we might as well just pass up the entire chain along with every # certificate. We will not store this to disk inside of zdb. if len(zout.certificates) > 1: presented_chain = [base64.b64decode(c["raw"]) for c in zout.certificates[1:]] else: presented_chain = [] for cert_dict in zout.certificates: ar = self.zsearch_definitions.anonstore_pb2.AnonymousRecord() c = ar.certificate c.raw = base64.b64decode(cert_dict["raw"]) out.certificates.append(ar) return out
def _transform_object(self, obj): ftp_banner = obj ftp = Transformable(obj) zout = ZMapTransformOutput() error = ftp['error'].resolve() if error is not None: raise errors.IgnoreObject("Error") out = dict() banner = ftp['data']['banner'].resolve() if banner is not None: out['banner'] = self.clean_banner(banner) if len(out) == 0: raise errors.IgnoreObject("Empty output dict") out['ip_address'] = obj['ip'] out['timestamp'] = obj['timestamp'] zout.transformed = out return zout
def _transform_object(self, obj): classification = obj['classification'] if classification != "dns": raise errors.IgnoreObject(classification) if int(obj['success']) != 1: raise errors.IgnoreObject("Not a DNS resposne") zout = ZMapTransformOutput() out = dict() out["support"] = True errors_present = False if obj["dns_parse_err"] == True: errors_present = True out["questions"] = [] for question in obj["dns_questions"]: q = dict() q["name"] = question["name"] q["type"] = question["qtype_str"] out["questions"].append(q) response_types = (("answers", "dns_answers"), ("authorities", "dns_authorities"),\ ("additionals", "dns_additionals")) for out_field, obj_field in response_types: (response_errors, responses) = self._transform_responses(obj[obj_field]) if response_errors: errors_present = True out[out_field] = responses out["errors"] = errors_present out["open_resolver"] = bool(len(out["answers"]) + len(out["additionals"]) + \ len(out["authorities"])) out["resolves_correctly"] = False for answer in out["answers"]: if answer["type"] == "A" and answer["response"] == self.CORRECT_RESPONSE: out["resolves_correctly"] = True break zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() wrapped = Transformable(obj) bacnet = wrapped['data']['bacnet'] if not bacnet['is_bacnet'].resolve(): raise errors.IgnoreObject() instance_number = bacnet['instance_number'].resolve() vendor_id = bacnet['vendor_id'].resolve() vendor_name = bacnet['vendor_name'].resolve() firmware_revision = bacnet['firmware_revision'].resolve() app_software_rev = bacnet['application_software_revision'].resolve() object_name = bacnet['object_name'].resolve() model_name = bacnet['model_name'].resolve() description = bacnet['description'].resolve() out = dict() if instance_number: out['instance_number'] = instance_number if vendor_id or vendor_name: vendor = dict() if vendor_id: vendor['id'] = vendor_id vendor['official_name'] = _vendors.get(vendor_id, 'unknown') if vendor_name: vendor['reported_name'] = vendor_name out['vendor'] = vendor if firmware_revision: out['firmware_revision'] = firmware_revision.strip() if app_software_rev: out['application_software_revision'] = app_software_rev.strip() if object_name: out['object_name'] = object_name.strip() if model_name: out['model_name'] = model_name.strip() if description: out['description'] = description if not bool(out): raise errors.IgnoreObject("Empty output dict") zout.transformed = out return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() wrapped = Transformable(obj) bacnet = wrapped["data"]["bacnet"] if not bacnet["is_bacnet"].resolve(): raise errors.IgnoreObject() instance_number = bacnet["instance_number"].resolve() vendor_id = bacnet["vendor_id"].resolve() vendor_name = bacnet["vendor_name"].resolve() firmware_revision = bacnet["firmware_revision"].resolve() app_software_rev = bacnet["application_software_revision"].resolve() object_name = bacnet["object_name"].resolve() model_name = bacnet["model_name"].resolve() description = bacnet["description"].resolve() out = dict() out["support"] = True if instance_number: out["instance_number"] = instance_number if vendor_id or vendor_name: vendor = dict() if vendor_id: vendor["id"] = vendor_id vendor["official_name"] = _vendors.get(vendor_id, "unknown") if vendor_name: vendor["reported_name"] = vendor_name out["vendor"] = vendor if firmware_revision: out["firmware_revision"] = firmware_revision.strip() if app_software_rev: out["application_software_revision"] = app_software_rev.strip() if object_name: out["object_name"] = object_name.strip() if model_name: out["model_name"] = model_name.strip() if description: out["description"] = description zout.transformed = out return zout
def _transform_object(self, obj): http = Transformable(obj) http_response = http['data']['http']['response'] zout = ZMapTransformOutput() out = dict() error_component = http['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("connection error") if http_response is not None: status_line = http_response['status_line'].resolve() status_code = http_response['status_code'].resolve() body = http_response['body'].resolve() body_sha256 = http_response['body_sha256'].resolve() headers = http_response['headers'].resolve() if status_line is not None: out['status_line'] = status_line if status_code is not None: out['status_code'] = status_code if body is not None: out['body'] = body m = title_regex.search(body) if m: title = m.group(1) if len(title) > 1024: title = title[0:1024] out['title'] = title if headers is not None: out['headers'] = headers if body_sha256: out['body_sha256'] = body_sha256 if len(out) == 0: raise errors.IgnoreObject("Empty output dict") zout.transformed = out return zout
def _transform_object(self, obj): ssh = Transformable(obj) server_protocol = dict() sp = ssh['data']['ssh']['server_protocol'] if sp.resolve() is None: raise errors.IgnoreObject("no ssh server protocol") raw_banner = sp['raw_banner'].resolve() if raw_banner is not None: server_protocol['raw_banner'] = raw_banner protocol_version = sp['protocol_version'].resolve() if protocol_version is not None: server_protocol['protocol_version'] = protocol_version software_version = sp['software_version'].resolve() if software_version is not None: server_protocol['software_version'] = software_version comment = sp['comment'].resolve() if comment is not None: server_protocol['comment'] = comment if len(server_protocol) == 0: raise errors.IgnoreObject("Empty server protocol output dict") zout = ZMapTransformOutput() zout.transformed = server_protocol return zout
def _transform_object(self, obj): classification = obj['classification'] if classification != "dns": raise errors.IgnoreObject(classification) if int(obj['success']) != 1 or int(obj['app_success']) != 1: raise errors.IgnoreObject("Not a DNS resposne") zout = ZMapTransformOutput() out = dict() out["support"] = True out["recursive_resolver"] = bool(len(obj["dns_answers"])) out["iterative_resolver"] = bool(len(obj["dns_authorities"])) if out["recursive_resolver"]: answers = [] errors_present = False for answer in obj["dns_answers"]: if answer["rdata_is_parsed"] == 1: answers.append(answer["rdata"]) else: errors_present = True break if not errors_present: out["answers"] = answers if out["iterative_resolver"]: authorities = [] errors_present = False for authority in obj["dns_authorities"]: if authority["rdata_is_parsed"] == 1: authorities.append(authority["rdata"]) else: errors_present = True break if not errors_present: out["authorities"] = authorities out['ip_address'] = obj['saddr'] out['timestamp'] = obj['timestamp-str'] zout.transformed = out return zout
def _transform_object(self, obj): grab = Transformable(obj)['data']['xssh'] out = dict() # banner: set_value(out, 'banner', grab['server_id'].resolve()) # support: support = dict() set_value(support, 'kex_algorithms', grab['server_key_exchange']['kex_algorithms'].resolve()) set_value(support, 'host_key_algorithms', grab['server_key_exchange']['host_key_algorithms'].resolve()) set_value(support, 'first_kex_follows', grab['server_key_exchange']['first_kex_follows'].resolve()) client_to_server = dict() set_value( client_to_server, 'ciphers', grab['server_key_exchange']['client_to_server_ciphers'].resolve()) set_value( client_to_server, 'macs', grab['server_key_exchange']['client_to_server_macs'].resolve()) set_value( client_to_server, 'compressions', grab['server_key_exchange'] ['client_to_server_compression'].resolve()) set_value( client_to_server, 'languages', grab['server_key_exchange'] ['client_to_server_languages'].resolve()) set_value(support, 'client_to_server', client_to_server) server_to_client = dict() set_value( server_to_client, 'ciphers', grab['server_key_exchange']['server_to_client_ciphers'].resolve()) set_value( server_to_client, 'macs', grab['server_key_exchange']['server_to_client_macs'].resolve()) set_value( server_to_client, 'compressions', grab['server_key_exchange'] ['server_to_client_compression'].resolve()) set_value( server_to_client, 'languages', grab['server_key_exchange'] ['server_to_client_languages'].resolve()) set_value(support, 'server_to_client', server_to_client) set_value(out, 'support', support) # selected: selected = grab['algorithm_selection'].resolve() if selected is not None: rename_key(selected, 'dh_kex_algorithm', 'kex_algorithm') rename_key(selected, 'client_to_server_alg_group', 'client_to_server') rename_key(selected, 'server_to_client_alg_group', 'server_to_client') set_value(out, 'selected', selected) # key_exchange: key_exchange = dict() set_value(key_exchange, 'ecdh_params', grab['key_exchange']['ecdh_params'].resolve()) set_value(key_exchange, 'dh_params', grab['key_exchange']['dh_params'].resolve()) try: del key_exchange['dh_params']['server_public'] except KeyError: pass set_value(out, 'key_exchange', key_exchange) # server_host_key: host_key = dict() for clone_key in [ 'fingerprint_sha256', 'rsa_public_key', 'dsa_public_key', 'ecdsa_public_key', 'ed25519_public_key' ]: # a bunch of keys we just need to copy verbatim from the grab set_value( host_key, clone_key, grab['key_exchange']['server_host_key'][clone_key].resolve()) set_value( host_key, 'key_algorithm', grab['key_exchange']['server_host_key']['algorithm'].resolve()) certkey_public_key = grab['key_exchange']['server_host_key'][ 'certkey_public_key'].resolve() if certkey_public_key is not None: rename_key(certkey_public_key, 'cert_type', 'type') # 2018/09/11: Workaround for mis-typed CertType.id field in ES; actual type is uint32, # current ES type is keyword (string). Added 'exclude={elasticsearch}' to the schema, # but to prevent attempted insertions of mistyped values, we need to actually remove the # id field as well. if 'type' in certkey_public_key: cert_type = certkey_public_key['type'] del_key(cert_type, 'id') del_key(certkey_public_key, 'reserved') signature_key = certkey_public_key.get('signature_key') if signature_key is not None: del_key(signature_key, 'raw') rename_key(signature_key, 'algorithm', 'key_algorithm') certkey_public_key['signature_key'] = signature_key # rewrite extensions and critical_options to maps of string -> bool: extensions = certkey_public_key.get('extensions') if extensions is not None: certkey_public_key['extensions'] = rewrite_known(extensions) critical_options = certkey_public_key.get('critical_options') if critical_options is not None: certkey_public_key['critical_options'] = rewrite_known( critical_options) key = certkey_public_key.get('key') if key is not None: del_key(key, 'raw') certkey_public_key['key'] = key sig = certkey_public_key.get('signature') if sig is not None: formatted_sig = dict() parsed = sig.get('parsed') if parsed is not None: set_value(formatted_sig, 'value', parsed.get('value')) alg = dict() set_value(alg, 'name', parsed.get('algorithm')) set_value(formatted_sig, 'signature_algorithm', alg) certkey_public_key['signature'] = formatted_sig set_value(host_key, 'certkey_public_key', certkey_public_key) set_value(out, 'server_host_key', host_key) if len(out) == 0: raise errors.IgnoreObject("Empty SSH protocol output dict") zout = ZMapTransformOutput() zout.transformed = out return zout
def _transform_object(self, obj): grab = Transformable(obj)['data']['xssh'] out = dict() # banner: set_value(out, 'banner', grab['server_id'].resolve()) # support: support = dict() set_value(support, 'kex_algorithms', grab['server_key_exchange']['kex_algorithms'].resolve()) set_value(support, 'host_key_algorithms', grab['server_key_exchange']['host_key_algorithms'].resolve()) set_value(support, 'first_kex_follows', grab['server_key_exchange']['first_kex_follows'].resolve()) client_to_server = dict() set_value(client_to_server, 'ciphers', grab['server_key_exchange']['client_to_server_ciphers'].resolve()) set_value(client_to_server, 'macs', grab['server_key_exchange']['client_to_server_macs'].resolve()) set_value(client_to_server, 'compressions', grab['server_key_exchange']['client_to_server_compression'].resolve()) set_value(client_to_server, 'languages', grab['server_key_exchange']['client_to_server_languages'].resolve()) set_value(support, 'client_to_server', client_to_server) server_to_client = dict() set_value(server_to_client, 'ciphers', grab['server_key_exchange']['server_to_client_ciphers'].resolve()) set_value(server_to_client, 'macs', grab['server_key_exchange']['server_to_client_macs'].resolve()) set_value(server_to_client, 'compressions', grab['server_key_exchange']['server_to_client_compression'].resolve()) set_value(server_to_client, 'languages', grab['server_key_exchange']['server_to_client_languages'].resolve()) set_value(support, 'server_to_client', server_to_client) set_value(out, 'support', support) # selected: selected = grab['algorithm_selection'].resolve() if selected is not None: rename_key(selected, 'dh_kex_algorithm', 'kex_algorithm') rename_key(selected, 'client_to_server_alg_group', 'client_to_server') rename_key(selected, 'server_to_client_alg_group', 'server_to_client') set_value(out, 'selected', selected) # key_exchange: key_exchange = dict() set_value(key_exchange, 'ecdh_params', grab['key_exchange']['ecdh_params'].resolve()) set_value(key_exchange, 'dh_params', grab['key_exchange']['dh_params'].resolve()) try: del key_exchange['dh_params']['server_public'] except KeyError: pass set_value(out, 'key_exchange', key_exchange) # server_host_key: host_key = dict() for clone_key in ['fingerprint_sha256', 'rsa_public_key', 'dsa_public_key', 'ecdsa_public_key', 'ed25519_public_key']: # a bunch of keys we just need to copy verbatim from the grab set_value(host_key, clone_key, grab['key_exchange']['server_host_key'][clone_key].resolve()) set_value(host_key, 'key_algorithm', grab['key_exchange']['server_host_key']['algorithm'].resolve()) certkey_public_key = grab['key_exchange']['server_host_key']['certkey_public_key'].resolve() if certkey_public_key is not None: rename_key(certkey_public_key, 'cert_type', 'type') # 2018/09/11: Workaround for mis-typed CertType.id field in ES; actual type is uint32, # current ES type is keyword (string). Added 'exclude={elasticsearch}' to the schema, # but to prevent attempted insertions of mistyped values, we need to actually remove the # id field as well. if 'type' in certkey_public_key: cert_type = certkey_public_key['type'] del_key(cert_type, 'id') del_key(certkey_public_key, 'reserved') signature_key = certkey_public_key.get('signature_key') if signature_key is not None: del_key(signature_key, 'raw') rename_key(signature_key, 'algorithm', 'key_algorithm') certkey_public_key['signature_key'] = signature_key # rewrite extensions and critical_options to maps of string -> bool: extensions = certkey_public_key.get('extensions') if extensions is not None: certkey_public_key['extensions'] = rewrite_known(extensions) critical_options = certkey_public_key.get('critical_options') if critical_options is not None: certkey_public_key['critical_options'] = rewrite_known(critical_options) key = certkey_public_key.get('key') if key is not None: del_key(key, 'raw') certkey_public_key['key'] = key sig = certkey_public_key.get('signature') if sig is not None: formatted_sig = dict() parsed = sig.get('parsed') if parsed is not None: set_value(formatted_sig, 'value', parsed.get('value')) alg = dict() set_value(alg, 'name', parsed.get('algorithm')) set_value(formatted_sig, 'signature_algorithm', alg) certkey_public_key['signature'] = formatted_sig set_value(host_key, 'certkey_public_key', certkey_public_key) set_value(out, 'server_host_key', host_key) if len(out) == 0: raise errors.IgnoreObject("Empty SSH protocol output dict") zout = ZMapTransformOutput() zout.transformed = out return zout