def encode(self, obj):
pb_out = super(RecordEncoder, self).encode(obj)
out = ZMapTransformOutput()
out.transformed = pb_out.transformed.SerializeToString()
out.certificates = [c.SerializeToString() for c in pb_out.certificates]
out.public_keys = [pk.SerializeToString() for pk in pb_out.public_keys]
return out
def _transform_object(self, obj):
tls = obj['data']['tls']
out, certificates = HTTPSTransform.make_tls_obj(tls)
zout = ZMapTransformOutput()
zout.transformed = out
zout.certificates = certificates
return zout
def _transform_object(self, obj):
wrapped = Transformable(obj['data'])
ciphers = wrapped['sslv2']['server_hello']['ciphers'].resolve()
certificate = wrapped['sslv2']['server_hello']['certificate'].resolve()
sslv2_support = bool(wrapped['sslv2']['server_verify'].resolve())
sslv2_export = bool(wrapped['sslv2_export']['server_verify'].resolve())
sslv2_extra_clear = bool(wrapped['sslv2_extra_clear']['server_verify']
['extra_clear'].resolve())
out = {
'support': sslv2_support,
'export': sslv2_export,
'extra_clear': sslv2_extra_clear,
}
if ciphers is not None:
out['ciphers'] = ciphers
if certificate is not None:
out['certificate'] = {
'parsed': certificate['parsed']
}
certificates = [certificate]
else:
certificates = list()
zout = ZMapTransformOutput()
zout.transformed = out
zout.certificates = certificates
return zout
def _transform_object(self, obj):
zout = ZMapTransformOutput()
wrapped = Transformable(obj)
error_component = wrapped['error_component'].resolve()
if error_component is not None and error_component == 'connect':
raise errors.IgnoreObject("Error connecting")
banner = wrapped['data']['banner'].resolve()
try:
tls_handshake = obj['data']['tls']
out, certificates = https.HTTPSTransform.make_tls_obj(
tls_handshake)
zout.transformed['tls'] = out
zout.certificates = certificates
except (TypeError, KeyError, IndexError):
pass
if banner is not None:
zout.transformed['banner'] = self.clean_banner(banner)
if len(zout.transformed) == 0:
raise errors.IgnoreObject("Empty Dict output")
return zout
def _transform_object(self, obj):
wrapped = Transformable(obj)
error_component = wrapped['error_component'].resolve()
if error_component is not None and error_component == 'connect':
raise errors.IgnoreObject("Error connecting")
banner = wrapped['data']['banner'].resolve()
ehlo = wrapped['data']['ehlo'].resolve()
starttls = wrapped['data']['starttls'].resolve()
zout = ZMapTransformOutput()
try:
tls_handshake = obj['data']['tls']
out, certificates = https.HTTPSTransform.make_tls_obj(tls_handshake)
zout.transformed['tls'] = out
zout.certificates = certificates
except (KeyError, TypeError, IndexError):
pass
if banner is not None:
zout.transformed['banner'] = self.clean_banner(banner)
if ehlo is not None:
zout.transformed['ehlo'] = self.clean_banner(ehlo)
if starttls is not None:
zout.transformed['starttls'] = self.clean_banner(starttls)
if len(zout.transformed) == 0:
raise errors.IgnoreObject("Empty output dict")
return zout
def _transform_object(self, obj):
tls = obj['data']['tls']
out, certificates = HTTPSTransform.make_tls_obj(tls)
zout = ZMapTransformOutput()
zout.transformed = out
zout.certificates = certificates
return zout
def _transform_object(self, obj):
wrapped = Transformable(obj['data'])
ciphers = wrapped['sslv2']['server_hello']['ciphers'].resolve()
certificate = wrapped['sslv2']['server_hello']['certificate'].resolve()
sslv2_support = bool(wrapped['sslv2']['server_verify'].resolve())
sslv2_export = bool(wrapped['sslv2_export']['server_verify'].resolve())
sslv2_extra_clear = bool(wrapped['sslv2_extra_clear']['server_verify']
['extra_clear'].resolve())
out = {
'support': sslv2_support,
'export': sslv2_export,
'extra_clear': sslv2_extra_clear,
}
if ciphers is not None:
out['ciphers'] = ciphers
if certificate is not None:
out['certificate'] = {'parsed': certificate['parsed']}
certificates = [certificate]
else:
certificates = list()
zout = ZMapTransformOutput()
zout.transformed = out
zout.certificates = certificates
return zout
def _transform_object(self, obj):
if 'tls' not in obj['data']:
raise errors.IgnoreObject("Not a TLS response")
tls = obj['data']['tls']
out, certificates = HTTPSTransform.make_tls_obj(tls)
zout = ZMapTransformOutput()
zout.transformed = out
zout.certificates = certificates
return zout
def _transform_object(self, obj):
if 'tls' not in obj['data']:
raise errors.IgnoreObject("Not a TLS response")
tls = obj['data']['tls']
out, certificates = HTTPSTransform.make_tls_obj(tls)
zout = ZMapTransformOutput()
zout.transformed = out
zout.certificates = certificates
return zout
def encode(self, zout):
out = ZMapTransformOutput()
obj = zout.transformed
ip = obj['ip_address']
ts = obj['timestamp']
domain = obj.get('domain', None)
if zout.metadata.local_metadata is not None:
obj['metadata'] = zout.metadata.local_metadata.to_dict()
m = zout.metadata.global_metadata.to_dict()
tags = list(zout.metadata.tags)
data = {x: obj[x] for x in obj if x not in self.DROP_KEYS}
m = {k: simple_strip(v) for k, v in m.iteritems()}
metadata = self.Metadatum.from_dict(m)
atom = self.ProtocolAtom(tags=tags, metadata=metadata, data=data)
record = self.Record(ip, self.port, self.protocol.value, self.subprotocol.value,
protocol_atom=atom, domain=domain,
timestamp=ts, scan_id=self.scan_id)
out.transformed = record.protobuf
out.certificates = []
for cert_dict in zout.certificates:
ar = self.zsearch_definitions.anonstore_pb2.AnonymousRecord()
c = ar.certificate
c.parsed = json.dumps(cert_dict["parsed"], sort_keys=True)
c.raw = base64.b64decode(cert_dict["raw"])
c.sha1fp = cert_dict["parsed"]["fingerprint_sha1"].decode("hex")
c.sha256fp = cert_dict["parsed"]["fingerprint_sha256"].decode("hex")
valid_nss = cert_dict.get("nss_trusted", None)
if valid_nss is not None:
c.valid_nss = valid_nss
c.validation_timestamp = record.timestamp
parents_sha256 = cert_dict.get("parents", None)
if parents_sha256 is not None:
c.parents.extend([
sha_value.decode("hex") for sha_value in parents_sha256
])
ar.sha256fp = c.sha256fp
ar.scan_id = self.scan_id
out.certificates.append(ar)
return out
def encode(self, zout):
out = ZMapTransformOutput()
obj = zout.transformed
ip = obj['ip_address']
ts = obj['timestamp']
domain = obj.get('domain', None)
if zout.metadata.local_metadata is not None:
obj['metadata'] = zout.metadata.local_metadata.to_dict()
m = zout.metadata.global_metadata.to_dict()
tags = list(zout.metadata.tags)
data = {x: obj[x] for x in obj if x not in self.DROP_KEYS}
m = {k: simple_strip(v) for k, v in m.iteritems()}
metadata = self.Metadatum.from_dict(m)
atom = self.ProtocolAtom(tags=tags, metadata=metadata, data=data)
record = self.Record(ip, self.port, self.protocol.value, self.subprotocol.value,
protocol_atom=atom, domain=domain,
timestamp=ts, scan_id=self.scan_id)
out.transformed = record.protobuf
out.certificates = []
# The chain may be helpful in validating this certificate later on
# if some of the parents haven't been previously been by the cert
# daemon. Therefore, pass along all raw certificates in the chain.
# If chains were guaranteed to be presented in a rasonable order, we
# could just pass up [n+1:], but people get this wrong all the time,
# so we might as well just pass up the entire chain along with every
# certificate. We will not store this to disk inside of zdb.
if len(zout.certificates) > 1:
presented_chain = [base64.b64decode(c["raw"]) for c in
zout.certificates[1:]]
else:
presented_chain = []
for cert_dict in zout.certificates:
ar = self.zsearch_definitions.anonstore_pb2.AnonymousRecord()
c = ar.certificate
c.raw = base64.b64decode(cert_dict["raw"])
out.certificates.append(ar)
return out
