def _transform_object(self, obj):
zout = ZMapTransformOutput()
out = dict()
wrapped = Transformable(obj)
error_component = wrapped['error_component'].resolve()
if error_component is not None and error_component == 'connect':
raise errors.IgnoreObject("Error connecting")
heartbleed = wrapped['data']['heartbleed']
heartbleed = wrapped['data']['heartbleed']
heartbleed_enabled = heartbleed['heartbeat_enabled'].resolve()
heartbleed_vulnerable = heartbleed['heartbleed_vulnerable'].resolve()
if heartbleed_enabled is not None:
out['heartbeat_enabled'] = heartbleed_enabled
if heartbleed_vulnerable is not None:
out['heartbleed_vulnerable'] = heartbleed_vulnerable
if len(out) == 0:
raise errors.IgnoreObject("Empty Output dict")
zout.transformed = out
return zout
def _transform_object(self, obj):
zout = ZMapTransformOutput()
wrapped = Transformable(obj)
error_component = wrapped['error_component'].resolve()
if error_component is not None and error_component == 'connect':
raise errors.IgnoreObject("Error connecting")
banner = wrapped['data']['banner'].resolve()
try:
tls_handshake = obj['data']['tls']
out, certificates = https.HTTPSTransform.make_tls_obj(
tls_handshake)
zout.transformed['tls'] = out
zout.certificates = certificates
except (TypeError, KeyError, IndexError):
pass
if banner is not None:
zout.transformed['banner'] = self.clean_banner(banner)
if len(zout.transformed) == 0:
raise errors.IgnoreObject("Empty Dict output")
return zout
def _transform_object(self, obj):
zout = ZMapTransformOutput()
out = dict()
wrapped = Transformable(obj)
error_component = wrapped['error_component'].resolve()
if error_component is not None and error_component == 'connect':
raise errors.IgnoreObject("Error connecting")
dh = wrapped['data']['tls']['server_key_exchange']['dh_params']
dh_prime_value = dh['prime']['value'].resolve()
dh_prime_length = dh['prime']['length'].resolve()
dh_generator_value = dh['generator']['value'].resolve()
dh_generator_length = dh['generator']['length'].resolve()
if dh_prime_value or dh_generator_value is not None:
out['dh_params'] = dict()
if dh_prime_value is not None:
out['dh_params']['prime'] = dict()
if dh_generator_value is not None:
out['dh_params']['generator'] = dict()
if dh_prime_value is not None:
out['dh_params']['prime']['value'] = dh_prime_value
if dh_prime_length is not None:
out['dh_params']['prime']['length'] = dh_prime_length
if dh_generator_value is not None:
out['dh_params']['generator']['value'] = dh_generator_value
if dh_generator_length is not None:
out['dh_params']['generator']['length'] = dh_generator_length
out["support"] = bool(out)
zout.transformed = out
return zout
def _transform_object(self, obj):
zout = ZMapTransformOutput()
out = dict()
wrapped = Transformable(obj)
error_component = wrapped['error_component'].resolve()
if error_component is not None and error_component == 'connect':
raise errors.IgnoreObject("Error connecting")
ecdh = wrapped['data']['tls']['server_key_exchange']['ecdh_params'][
'curve_id']
ecdh_name = ecdh['name'].resolve()
ecdh_id = ecdh['id'].resolve()
if ecdh_name or ecdh_id is not None:
out['ecdh_params'] = dict()
out['ecdh_params']['curve_id'] = dict()
if ecdh_name is not None:
out['ecdh_params']['curve_id']['name'] = ecdh_name
if ecdh_id is not None:
out['ecdh_params']['curve_id']['id'] = ecdh_id
out["support"] = bool(out)
zout.transformed = out
return zout
def _transform_object(self, obj):
tls = obj['data']['tls']
out, certificates = HTTPSTransform.make_tls_obj(tls)
zout = ZMapTransformOutput()
zout.transformed = out
zout.certificates = certificates
return zout
def _transform_object(self, obj):
zout = ZMapTransformOutput()
out = dict()
wrapped = Transformable(obj)
error_component = wrapped['error_component'].resolve()
if error_component is not None and error_component == 'connect':
raise errors.IgnoreObject("Error connecting")
rsa = wrapped['data']['tls']['server_key_exchange']['rsa_params']
rsa_exponent = rsa['exponent'].resolve()
rsa_modulus = rsa['modulus'].resolve()
rsa_length = rsa['length'].resolve()
if rsa_exponent or rsa_modulus is not None:
out['rsa_params'] = dict()
if rsa_exponent is not None:
out['rsa_params']['exponent'] = rsa_exponent
if rsa_modulus is not None:
out['rsa_params']['modulus'] = rsa_modulus
if rsa_length is not None:
out['rsa_params']['length'] = rsa_length
out["support"] = bool(out)
zout.transformed = out
return zout
def _transform_object(self, obj):
wrapped = Transformable(obj['data'])
ciphers = wrapped['sslv2']['server_hello']['ciphers'].resolve()
certificate = wrapped['sslv2']['server_hello']['certificate'].resolve()
sslv2_support = bool(wrapped['sslv2']['server_verify'].resolve())
sslv2_export = bool(wrapped['sslv2_export']['server_verify'].resolve())
sslv2_extra_clear = bool(wrapped['sslv2_extra_clear']['server_verify']
['extra_clear'].resolve())
out = {
'support': sslv2_support,
'export': sslv2_export,
'extra_clear': sslv2_extra_clear,
}
if ciphers is not None:
out['ciphers'] = ciphers
if certificate is not None:
out['certificate'] = {'parsed': certificate['parsed']}
certificates = [certificate]
else:
certificates = list()
zout = ZMapTransformOutput()
zout.transformed = out
zout.certificates = certificates
return zout
def _transform_object(self, obj):
http = Transformable(obj)
http_response = http['data']['http']['response']
zout = ZMapTransformOutput()
out = dict()
error_component = http['error_component'].resolve()
if error_component is not None and error_component == 'connect':
raise errors.IgnoreObject("connection error")
if http_response is not None:
status_line = http_response['status_line'].resolve()
status_code = http_response['status_code'].resolve()
body = http_response['body'].resolve()
headers = http_response['headers'].resolve()
if status_line is not None:
out['status_line'] = status_line
if status_code is not None:
out['status_code'] = status_code
if body is not None:
out['body'] = body
if headers is not None:
out['headers'] = headers
if len(out) == 0:
raise errors.IgnoreObject("Empty output dict")
zout.transformed = out
return zout
def encode(self, obj):
pb_out = super(RecordEncoder, self).encode(obj)
out = ZMapTransformOutput()
out.transformed = pb_out.transformed.SerializeToString()
out.certificates = [c.SerializeToString() for c in pb_out.certificates]
out.public_keys = [pk.SerializeToString() for pk in pb_out.public_keys]
return out
def _transform_object(self, obj):
if 'tls' not in obj['data']:
raise errors.IgnoreObject("Not a TLS response")
tls = obj['data']['tls']
out, certificates = HTTPSTransform.make_tls_obj(tls)
zout = ZMapTransformOutput()
zout.transformed = out
zout.certificates = certificates
return zout
def _transform_object(self, obj):
http = Transformable(obj)
connect_response = http['data']['http']['connect_response']
get_response = http['data']['http']['response']
zout = ZMapTransformOutput()
out = dict()
error_component = http['error_component'].resolve()
if error_component is not None and error_component == 'connect':
raise errors.IgnoreObject("connection error")
if connect_response is not None:
status_line = connect_response['status_line'].resolve()
status_code = connect_response['status_code'].resolve()
body = connect_response['body'].resolve()
headers = connect_response['headers'].resolve()
if status_line or status_code or body or headers is not None:
out['connect'] = dict()
if status_line is not None:
out['connect']['status_line'] = status_line
if status_code is not None:
out['connect']['status_code'] = status_code
if body is not None:
out['connect']['body'] = body
if headers is not None:
out['connect']['headers'] = headers
if get_response is not None:
status_line = get_response['status_line'].resolve()
status_code = get_response['status_code'].resolve()
body = get_response['body'].resolve()
headers = get_response['headers'].resolve()
body_sha256 = get_response['body_sha256'].resolve()
out['get'] = dict()
if body:
random_present = "Uh2Qn8Y7NPRm6h3xqEXUq4EhtW7Po4gy" in body
else:
random_present = False
out['get']['random_present'] = random_present
if status_line is not None:
out['get']['status_line'] = status_line
if status_code is not None:
out['get']['status_code'] = status_code
if body is not None:
out['get']['body'] = body
if headers is not None:
out['get']['headers'] = headers
if body_sha256:
out['get']['body_sha256'] = body_sha256
if len(out) == 0:
raise errors.IgnoreObject("Empty output dict")
zout.transformed = out
return zout
def _transform_object(self, obj):
zout = ZMapTransformOutput()
wrapped = Transformable(obj)
modbus = wrapped['data']['modbus']
if not modbus['raw_response'].resolve():
raise errors.IgnoreObject()
out = dict()
function_code = modbus['function_code'].resolve()
mei_response = modbus['mei_response']
if function_code:
out['function_code'] = function_code
if mei_response:
conformity_level = mei_response['conformity_level'].resolve()
objects = mei_response['objects']
vendor = objects['vendor'].resolve()
product_code = objects['product_code'].resolve()
revision = objects['revision'].resolve()
vendor_url = objects['vendor_url'].resolve()
product_name = objects['product_name'].resolve()
model_name = objects['model_name'].resolve()
user_application_name = objects['user_application_name'].resolve()
if conformity_level or vendor or product_code or revision or \
vendor_url or product_name or model_name or user_application_name:
out['mei_response'] = dict()
if vendor or product_code or revision or vendor_url or product_name \
or model_name or user_application_name:
out['mei_response']['objects'] = dict()
if conformity_level:
out['mei_response']['conformity_level'] = conformity_level
if vendor:
out['mei_response']['objects']['vendor'] = vendor
if product_code:
out['mei_response']['objects']['product_code'] = product_code
if revision:
out['mei_response']['objects']['revision'] = revision
if vendor_url:
out['mei_response']['objects']['vendor_url'] = vendor_url
if product_name:
out['mei_response']['objects']['product_name'] = product_name
if model_name:
out['mei_response']['objects']['model_name'] = model_name
if user_application_name:
out['mei_response']['objects']['user_application_name'] = \
user_application_name
if len(out) == 0:
raise errors.IgnoreObject("Empty output dict")
zout.transformed = out
return zout
def _transform_object(self, obj):
http = Transformable(obj)
http_response = http['data']['http']['response']
zout = ZMapTransformOutput()
out = dict()
zout.transformed = out
error_component = http['error_component'].resolve()
if error_component is not None and error_component == 'connect':
raise errors.IgnoreObject("connection error")
if http_response is not None:
status_line = http_response['status_line'].resolve()
status_code = http_response['status_code'].resolve()
body = http_response['body'].resolve()
body_sha256 = http_response['body_sha256'].resolve()
headers = http_response['headers'].resolve()
if status_line is not None:
out['status_line'] = status_line
if status_code is not None:
out['status_code'] = status_code
if body is not None:
out['body'] = body
m = title_regex.search(body)
if m:
title = m.group(1)
if len(title) > 1024:
title = title[0:1024]
out['title'] = title.strip()
if headers:
# FIXME: This modifies the input?
if "set_cookie" in headers:
del headers["set_cookie"]
if "date" in headers:
del headers["date"]
for k, v, in headers.iteritems():
if k == "unknown":
for d in v:
if len(d["value"]) < 1:
continue
d["value"] = d["value"][0]
elif v:
headers[k] = v[0]
else:
del headers[k]
out['headers'] = headers
if body_sha256:
out['body_sha256'] = body_sha256
if len(out) == 0:
raise errors.IgnoreObject("Empty output dict")
return zout
def _transform_object(self, obj):
classification = obj['classification']
if classification != "upnp":
raise errors.IgnoreObject(classification)
success = int(obj['success'])
if success != 1:
raise errors.IgnoreObject("unsuccessful zmap")
wrapped = Transformable(obj)
transformed = {}
for key in self.KEYS:
transformed[key] = wrapped[key].resolve()
out = ZMapTransformOutput()
out.transformed = transformed
return out
def _transform_object(self, obj):
zout = ZMapTransformOutput()
out = dict()
wrapped = Transformable(obj)
error_component = wrapped['error_component'].resolve()
if error_component is not None and error_component == 'connect':
raise errors.IgnoreObject("Error connecting")
server_hello = wrapped['data']['tls']['server_hello']
if server_hello.resolve() is None:
raise IgnoreObject("missing server hello")
exr = server_hello['extended_random'].resolve()
if exr is not None:
out['extended_random_support'] = True
else:
out['extended_random_support'] = False
zout.transformed = out
return zout
def encode(self, zout):
out = ZMapTransformOutput()
obj = zout.transformed
ip = obj['ip_address']
ts = obj['timestamp']
domain = obj.get('domain', None)
if zout.metadata.local_metadata is not None:
obj['metadata'] = zout.metadata.local_metadata.to_dict()
m = zout.metadata.global_metadata.to_dict()
tags = list(zout.metadata.tags)
data = {x: obj[x] for x in obj if x not in self.DROP_KEYS}
m = {k: simple_strip(v) for k, v in m.iteritems()}
metadata = self.Metadatum.from_dict(m)
atom = self.ProtocolAtom(tags=tags, metadata=metadata, data=data)
record = self.Record(ip, self.port, self.protocol.value, self.subprotocol.value,
protocol_atom=atom, domain=domain,
timestamp=ts, scan_id=self.scan_id)
out.transformed = record.protobuf
out.certificates = []
for cert_dict in zout.certificates:
ar = self.zsearch_definitions.anonstore_pb2.AnonymousRecord()
c = ar.certificate
c.parsed = json.dumps(cert_dict["parsed"], sort_keys=True)
c.raw = base64.b64decode(cert_dict["raw"])
c.sha1fp = cert_dict["parsed"]["fingerprint_sha1"].decode("hex")
c.sha256fp = cert_dict["parsed"]["fingerprint_sha256"].decode("hex")
valid_nss = cert_dict.get("nss_trusted", None)
if valid_nss is not None:
c.valid_nss = valid_nss
c.validation_timestamp = record.timestamp
parents_sha256 = cert_dict.get("parents", None)
if parents_sha256 is not None:
c.parents.extend([
sha_value.decode("hex") for sha_value in parents_sha256
])
ar.sha256fp = c.sha256fp
ar.scan_id = self.scan_id
out.certificates.append(ar)
return out
def _transform_object(self, obj):
classification = obj['classification']
if classification != "dns":
raise errors.IgnoreObject(classification)
if int(obj['success']) != 1:
raise errors.IgnoreObject("Not a DNS resposne")
zout = ZMapTransformOutput()
out = dict()
out["support"] = True
errors_present = False
if obj["dns_parse_err"] == True:
errors_present = True
out["questions"] = []
for question in obj["dns_questions"]:
q = dict()
q["name"] = question["name"]
q["type"] = question["qtype_str"]
out["questions"].append(q)
response_types = (("answers", "dns_answers"), ("authorities", "dns_authorities"),\
("additionals", "dns_additionals"))
for out_field, obj_field in response_types:
(response_errors,
responses) = self._transform_responses(obj[obj_field])
if response_errors:
errors_present = True
out[out_field] = responses
out["errors"] = errors_present
out["open_resolver"] = bool(len(out["answers"]) + len(out["additionals"]) + \
len(out["authorities"]))
out["resolves_correctly"] = False
for answer in out["answers"]:
if answer["type"] == "A" and answer[
"response"] == self.CORRECT_RESPONSE:
out["resolves_correctly"] = True
break
zout.transformed = out
return zout
def _transform_object(self, obj):
zout = ZMapTransformOutput()
tempout = dict()
wrapped = Transformable(obj)
error_component = wrapped['error_component'].resolve()
if error_component is not None and error_component == 'connect':
raise errors.IgnoreObject("Error connecting")
tls_handshake = wrapped['data']['tls']
value = tls_handshake['server_hello']['version']['value'].resolve()
out = dict()
if value is not None:
version = int(value)
out['support'] = True if version == 772 else False
else:
raise errors.IgnoreObject("Empty output dict")
zout.transformed = out
return zout
def encode(self, zout):
out = ZMapTransformOutput()
obj = zout.transformed
ip = obj['ip_address']
ts = obj['timestamp']
domain = obj.get('domain', None)
if zout.metadata.local_metadata is not None:
obj['metadata'] = zout.metadata.local_metadata.to_dict()
m = zout.metadata.global_metadata.to_dict()
tags = list(zout.metadata.tags)
data = {x: obj[x] for x in obj if x not in self.DROP_KEYS}
m = {k: simple_strip(v) for k, v in m.iteritems()}
metadata = self.Metadatum.from_dict(m)
atom = self.ProtocolAtom(tags=tags, metadata=metadata, data=data)
record = self.Record(ip, self.port, self.protocol.value, self.subprotocol.value,
protocol_atom=atom, domain=domain,
timestamp=ts, scan_id=self.scan_id)
out.transformed = record.protobuf
out.certificates = []
# The chain may be helpful in validating this certificate later on
# if some of the parents haven't been previously been by the cert
# daemon. Therefore, pass along all raw certificates in the chain.
# If chains were guaranteed to be presented in a rasonable order, we
# could just pass up [n+1:], but people get this wrong all the time,
# so we might as well just pass up the entire chain along with every
# certificate. We will not store this to disk inside of zdb.
if len(zout.certificates) > 1:
presented_chain = [base64.b64decode(c["raw"]) for c in
zout.certificates[1:]]
else:
presented_chain = []
for cert_dict in zout.certificates:
ar = self.zsearch_definitions.anonstore_pb2.AnonymousRecord()
c = ar.certificate
c.raw = base64.b64decode(cert_dict["raw"])
out.certificates.append(ar)
return out
def _transform_object(self, obj):
ftp_banner = obj
ftp = Transformable(obj)
zout = ZMapTransformOutput()
error = ftp['error'].resolve()
if error is not None:
raise errors.IgnoreObject("Error")
out = dict()
banner = ftp['data']['banner'].resolve()
if banner is not None:
out['banner'] = self.clean_banner(banner)
if len(out) == 0:
raise errors.IgnoreObject("Empty output dict")
out['ip_address'] = obj['ip']
out['timestamp'] = obj['timestamp']
zout.transformed = out
return zout
def _transform_object(self, obj):
ssh = Transformable(obj)
server_protocol = dict()
sp = ssh['data']['ssh']['server_protocol']
if sp.resolve() is None:
raise errors.IgnoreObject("no ssh server protocol")
raw_banner = sp['raw_banner'].resolve()
if raw_banner is not None:
server_protocol['raw_banner'] = raw_banner
protocol_version = sp['protocol_version'].resolve()
if protocol_version is not None:
server_protocol['protocol_version'] = protocol_version
software_version = sp['software_version'].resolve()
if software_version is not None:
server_protocol['software_version'] = software_version
comment = sp['comment'].resolve()
if comment is not None:
server_protocol['comment'] = comment
if len(server_protocol) == 0:
raise errors.IgnoreObject("Empty server protocol output dict")
zout = ZMapTransformOutput()
zout.transformed = server_protocol
return zout
def _transform_object(self, obj):
http = Transformable(obj)
http_response = http['data']['http']['response']
zout = ZMapTransformOutput()
out = dict()
error_component = http['error_component'].resolve()
if error_component is not None and error_component == 'connect':
raise errors.IgnoreObject("connection error")
if http_response is not None:
status_line = http_response['status_line'].resolve()
status_code = http_response['status_code'].resolve()
body = http_response['body'].resolve()
body_sha256 = http_response['body_sha256'].resolve()
headers = http_response['headers'].resolve()
if status_line is not None:
out['status_line'] = status_line
if status_code is not None:
out['status_code'] = status_code
if body is not None:
out['body'] = body
m = title_regex.search(body)
if m:
title = m.group(1)
if len(title) > 1024:
title = title[0:1024]
out['title'] = title
if headers is not None:
out['headers'] = headers
if body_sha256:
out['body_sha256'] = body_sha256
if len(out) == 0:
raise errors.IgnoreObject("Empty output dict")
zout.transformed = out
return zout
def _transform_object(self, obj):
grab = Transformable(obj)['data']['xssh']
out = dict()
# banner:
set_value(out, 'banner', grab['server_id'].resolve())
# support:
support = dict()
set_value(support, 'kex_algorithms',
grab['server_key_exchange']['kex_algorithms'].resolve())
set_value(support, 'host_key_algorithms',
grab['server_key_exchange']['host_key_algorithms'].resolve())
set_value(support, 'first_kex_follows',
grab['server_key_exchange']['first_kex_follows'].resolve())
client_to_server = dict()
set_value(
client_to_server, 'ciphers',
grab['server_key_exchange']['client_to_server_ciphers'].resolve())
set_value(
client_to_server, 'macs',
grab['server_key_exchange']['client_to_server_macs'].resolve())
set_value(
client_to_server, 'compressions', grab['server_key_exchange']
['client_to_server_compression'].resolve())
set_value(
client_to_server, 'languages', grab['server_key_exchange']
['client_to_server_languages'].resolve())
set_value(support, 'client_to_server', client_to_server)
server_to_client = dict()
set_value(
server_to_client, 'ciphers',
grab['server_key_exchange']['server_to_client_ciphers'].resolve())
set_value(
server_to_client, 'macs',
grab['server_key_exchange']['server_to_client_macs'].resolve())
set_value(
server_to_client, 'compressions', grab['server_key_exchange']
['server_to_client_compression'].resolve())
set_value(
server_to_client, 'languages', grab['server_key_exchange']
['server_to_client_languages'].resolve())
set_value(support, 'server_to_client', server_to_client)
set_value(out, 'support', support)
# selected:
selected = grab['algorithm_selection'].resolve()
if selected is not None:
rename_key(selected, 'dh_kex_algorithm', 'kex_algorithm')
rename_key(selected, 'client_to_server_alg_group',
'client_to_server')
rename_key(selected, 'server_to_client_alg_group',
'server_to_client')
set_value(out, 'selected', selected)
# key_exchange:
key_exchange = dict()
set_value(key_exchange, 'ecdh_params',
grab['key_exchange']['ecdh_params'].resolve())
set_value(key_exchange, 'dh_params',
grab['key_exchange']['dh_params'].resolve())
try:
del key_exchange['dh_params']['server_public']
except KeyError:
pass
set_value(out, 'key_exchange', key_exchange)
# server_host_key:
host_key = dict()
for clone_key in [
'fingerprint_sha256', 'rsa_public_key', 'dsa_public_key',
'ecdsa_public_key', 'ed25519_public_key'
]:
# a bunch of keys we just need to copy verbatim from the grab
set_value(
host_key, clone_key,
grab['key_exchange']['server_host_key'][clone_key].resolve())
set_value(
host_key, 'key_algorithm',
grab['key_exchange']['server_host_key']['algorithm'].resolve())
certkey_public_key = grab['key_exchange']['server_host_key'][
'certkey_public_key'].resolve()
if certkey_public_key is not None:
rename_key(certkey_public_key, 'cert_type', 'type')
# 2018/09/11: Workaround for mis-typed CertType.id field in ES; actual type is uint32,
# current ES type is keyword (string). Added 'exclude={elasticsearch}' to the schema,
# but to prevent attempted insertions of mistyped values, we need to actually remove the
# id field as well.
if 'type' in certkey_public_key:
cert_type = certkey_public_key['type']
del_key(cert_type, 'id')
del_key(certkey_public_key, 'reserved')
signature_key = certkey_public_key.get('signature_key')
if signature_key is not None:
del_key(signature_key, 'raw')
rename_key(signature_key, 'algorithm', 'key_algorithm')
certkey_public_key['signature_key'] = signature_key
# rewrite extensions and critical_options to maps of string -> bool:
extensions = certkey_public_key.get('extensions')
if extensions is not None:
certkey_public_key['extensions'] = rewrite_known(extensions)
critical_options = certkey_public_key.get('critical_options')
if critical_options is not None:
certkey_public_key['critical_options'] = rewrite_known(
critical_options)
key = certkey_public_key.get('key')
if key is not None:
del_key(key, 'raw')
certkey_public_key['key'] = key
sig = certkey_public_key.get('signature')
if sig is not None:
formatted_sig = dict()
parsed = sig.get('parsed')
if parsed is not None:
set_value(formatted_sig, 'value', parsed.get('value'))
alg = dict()
set_value(alg, 'name', parsed.get('algorithm'))
set_value(formatted_sig, 'signature_algorithm', alg)
certkey_public_key['signature'] = formatted_sig
set_value(host_key, 'certkey_public_key', certkey_public_key)
set_value(out, 'server_host_key', host_key)
if len(out) == 0:
raise errors.IgnoreObject("Empty SSH protocol output dict")
zout = ZMapTransformOutput()
zout.transformed = out
return zout
