def generate_mlbf(stats, blocked, not_blocked): log.info('Starting to generating bloomfilter') cascade = FilterCascade( defaultHashAlg=HashAlgorithm.SHA256, salt=secrets.token_bytes(16), ) error_rates = sorted((len(blocked), len(not_blocked))) cascade.set_crlite_error_rates(include_len=error_rates[0], exclude_len=error_rates[1]) stats['mlbf_blocked_count'] = len(blocked) stats['mlbf_notblocked_count'] = len(not_blocked) cascade.initialize(include=blocked, exclude=not_blocked) stats['mlbf_version'] = cascade.version stats['mlbf_layers'] = cascade.layerCount() stats['mlbf_bits'] = cascade.bitCount() log.info(f'Filter cascade layers: {cascade.layerCount()}, ' f'bit: {cascade.bitCount()}') cascade.verify(include=blocked, exclude=not_blocked) return cascade
def generateMLBF(args, stats, *, revoked_certs, nonrevoked_certs, nonrevoked_certs_len): revoked_certs_len = len(revoked_certs) log.info("Generating filter") cascade = FilterCascade([], version=1, defaultHashAlg=fileformats.HashAlgorithm.MURMUR3) cascade.set_crlite_error_rates(include_len=revoked_certs_len, exclude_len=nonrevoked_certs_len) cascade.initialize(include=revoked_certs, exclude=nonrevoked_certs) stats["mlbf_fprs"] = cascade.error_rates stats["mlbf_version"] = cascade.version stats["mlbf_layers"] = cascade.layerCount() stats["mlbf_bits"] = cascade.bitCount() log.debug("Filter cascade layers: {layers}, bit: {bits}".format( layers=cascade.layerCount(), bits=cascade.bitCount())) metrics.gauge("GenerateMLBF.BitCount", cascade.bitCount()) metrics.gauge("GenerateMLBF.LayerCount", cascade.layerCount()) return cascade