def encode(self, obj): pb_out = super(RecordEncoder, self).encode(obj) out = ZMapTransformOutput() out.transformed = pb_out.transformed.SerializeToString() out.certificates = [c.SerializeToString() for c in pb_out.certificates] out.public_keys = [pk.SerializeToString() for pk in pb_out.public_keys] return out
def _transform_object(self, obj): tls = obj['data']['tls'] out, certificates = HTTPSTransform.make_tls_obj(tls) zout = ZMapTransformOutput() zout.transformed = out zout.certificates = certificates return zout
def _transform_object(self, obj): wrapped = Transformable(obj['data']) ciphers = wrapped['sslv2']['server_hello']['ciphers'].resolve() certificate = wrapped['sslv2']['server_hello']['certificate'].resolve() sslv2_support = bool(wrapped['sslv2']['server_verify'].resolve()) sslv2_export = bool(wrapped['sslv2_export']['server_verify'].resolve()) sslv2_extra_clear = bool(wrapped['sslv2_extra_clear']['server_verify'] ['extra_clear'].resolve()) out = { 'support': sslv2_support, 'export': sslv2_export, 'extra_clear': sslv2_extra_clear, } if ciphers is not None: out['ciphers'] = ciphers if certificate is not None: out['certificate'] = { 'parsed': certificate['parsed'] } certificates = [certificate] else: certificates = list() zout = ZMapTransformOutput() zout.transformed = out zout.certificates = certificates return zout
def _transform_object(self, obj): zout = ZMapTransformOutput() wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") banner = wrapped['data']['banner'].resolve() try: tls_handshake = obj['data']['tls'] out, certificates = https.HTTPSTransform.make_tls_obj( tls_handshake) zout.transformed['tls'] = out zout.certificates = certificates except (TypeError, KeyError, IndexError): pass if banner is not None: zout.transformed['banner'] = self.clean_banner(banner) if len(zout.transformed) == 0: raise errors.IgnoreObject("Empty Dict output") return zout
def _transform_object(self, obj): wrapped = Transformable(obj) error_component = wrapped['error_component'].resolve() if error_component is not None and error_component == 'connect': raise errors.IgnoreObject("Error connecting") banner = wrapped['data']['banner'].resolve() ehlo = wrapped['data']['ehlo'].resolve() starttls = wrapped['data']['starttls'].resolve() zout = ZMapTransformOutput() try: tls_handshake = obj['data']['tls'] out, certificates = https.HTTPSTransform.make_tls_obj(tls_handshake) zout.transformed['tls'] = out zout.certificates = certificates except (KeyError, TypeError, IndexError): pass if banner is not None: zout.transformed['banner'] = self.clean_banner(banner) if ehlo is not None: zout.transformed['ehlo'] = self.clean_banner(ehlo) if starttls is not None: zout.transformed['starttls'] = self.clean_banner(starttls) if len(zout.transformed) == 0: raise errors.IgnoreObject("Empty output dict") return zout
def _transform_object(self, obj): wrapped = Transformable(obj['data']) ciphers = wrapped['sslv2']['server_hello']['ciphers'].resolve() certificate = wrapped['sslv2']['server_hello']['certificate'].resolve() sslv2_support = bool(wrapped['sslv2']['server_verify'].resolve()) sslv2_export = bool(wrapped['sslv2_export']['server_verify'].resolve()) sslv2_extra_clear = bool(wrapped['sslv2_extra_clear']['server_verify'] ['extra_clear'].resolve()) out = { 'support': sslv2_support, 'export': sslv2_export, 'extra_clear': sslv2_extra_clear, } if ciphers is not None: out['ciphers'] = ciphers if certificate is not None: out['certificate'] = {'parsed': certificate['parsed']} certificates = [certificate] else: certificates = list() zout = ZMapTransformOutput() zout.transformed = out zout.certificates = certificates return zout
def _transform_object(self, obj): if 'tls' not in obj['data']: raise errors.IgnoreObject("Not a TLS response") tls = obj['data']['tls'] out, certificates = HTTPSTransform.make_tls_obj(tls) zout = ZMapTransformOutput() zout.transformed = out zout.certificates = certificates return zout
def encode(self, zout): out = ZMapTransformOutput() obj = zout.transformed ip = obj['ip_address'] ts = obj['timestamp'] domain = obj.get('domain', None) if zout.metadata.local_metadata is not None: obj['metadata'] = zout.metadata.local_metadata.to_dict() m = zout.metadata.global_metadata.to_dict() tags = list(zout.metadata.tags) data = {x: obj[x] for x in obj if x not in self.DROP_KEYS} m = {k: simple_strip(v) for k, v in m.iteritems()} metadata = self.Metadatum.from_dict(m) atom = self.ProtocolAtom(tags=tags, metadata=metadata, data=data) record = self.Record(ip, self.port, self.protocol.value, self.subprotocol.value, protocol_atom=atom, domain=domain, timestamp=ts, scan_id=self.scan_id) out.transformed = record.protobuf out.certificates = [] for cert_dict in zout.certificates: ar = self.zsearch_definitions.anonstore_pb2.AnonymousRecord() c = ar.certificate c.parsed = json.dumps(cert_dict["parsed"], sort_keys=True) c.raw = base64.b64decode(cert_dict["raw"]) c.sha1fp = cert_dict["parsed"]["fingerprint_sha1"].decode("hex") c.sha256fp = cert_dict["parsed"]["fingerprint_sha256"].decode("hex") valid_nss = cert_dict.get("nss_trusted", None) if valid_nss is not None: c.valid_nss = valid_nss c.validation_timestamp = record.timestamp parents_sha256 = cert_dict.get("parents", None) if parents_sha256 is not None: c.parents.extend([ sha_value.decode("hex") for sha_value in parents_sha256 ]) ar.sha256fp = c.sha256fp ar.scan_id = self.scan_id out.certificates.append(ar) return out
def encode(self, zout): out = ZMapTransformOutput() obj = zout.transformed ip = obj['ip_address'] ts = obj['timestamp'] domain = obj.get('domain', None) if zout.metadata.local_metadata is not None: obj['metadata'] = zout.metadata.local_metadata.to_dict() m = zout.metadata.global_metadata.to_dict() tags = list(zout.metadata.tags) data = {x: obj[x] for x in obj if x not in self.DROP_KEYS} m = {k: simple_strip(v) for k, v in m.iteritems()} metadata = self.Metadatum.from_dict(m) atom = self.ProtocolAtom(tags=tags, metadata=metadata, data=data) record = self.Record(ip, self.port, self.protocol.value, self.subprotocol.value, protocol_atom=atom, domain=domain, timestamp=ts, scan_id=self.scan_id) out.transformed = record.protobuf out.certificates = [] # The chain may be helpful in validating this certificate later on # if some of the parents haven't been previously been by the cert # daemon. Therefore, pass along all raw certificates in the chain. # If chains were guaranteed to be presented in a rasonable order, we # could just pass up [n+1:], but people get this wrong all the time, # so we might as well just pass up the entire chain along with every # certificate. We will not store this to disk inside of zdb. if len(zout.certificates) > 1: presented_chain = [base64.b64decode(c["raw"]) for c in zout.certificates[1:]] else: presented_chain = [] for cert_dict in zout.certificates: ar = self.zsearch_definitions.anonstore_pb2.AnonymousRecord() c = ar.certificate c.raw = base64.b64decode(cert_dict["raw"]) out.certificates.append(ar) return out