def generate_mlbf(cls, stats, blocked, not_blocked):
"""Originally based on:
https://github.com/mozilla/crlite/blob/master/create_filter_cascade/certs_to_crlite.py
(not so much any longer, apart from the fprs calculation)
"""
salt = secrets.token_bytes(16)
stats['mlbf_blocked_count'] = len(blocked)
stats['mlbf_notblocked_count'] = len(not_blocked)
fprs = [len(blocked) / (math.sqrt(2) * len(not_blocked)), 0.5]
log.info("Generating filter")
cascade = FilterCascade(
error_rates=fprs,
defaultHashAlg=HashAlgorithm.SHA256,
salt=salt,
)
cascade.initialize(include=blocked, exclude=not_blocked)
stats['mlbf_fprs'] = fprs
stats['mlbf_version'] = cascade.version
stats['mlbf_layers'] = cascade.layerCount()
stats['mlbf_bits'] = cascade.bitCount()
log.debug("Filter cascade layers: {layers}, bit: {bits}".format(
layers=cascade.layerCount(), bits=cascade.bitCount()))
cascade.verify(include=blocked, exclude=not_blocked)
return cascade
def generate_mlbf(stats, blocked, not_blocked):
log.info('Starting to generating bloomfilter')
cascade = FilterCascade(
defaultHashAlg=HashAlgorithm.SHA256,
salt=secrets.token_bytes(16),
)
error_rates = sorted((len(blocked), len(not_blocked)))
cascade.set_crlite_error_rates(include_len=error_rates[0],
exclude_len=error_rates[1])
stats['mlbf_blocked_count'] = len(blocked)
stats['mlbf_notblocked_count'] = len(not_blocked)
cascade.initialize(include=blocked, exclude=not_blocked)
stats['mlbf_version'] = cascade.version
stats['mlbf_layers'] = cascade.layerCount()
stats['mlbf_bits'] = cascade.bitCount()
log.info(f'Filter cascade layers: {cascade.layerCount()}, '
f'bit: {cascade.bitCount()}')
cascade.verify(include=blocked, exclude=not_blocked)
return cascade
def generateMLBF(args, stats, *, revoked_certs, nonrevoked_certs,
nonrevoked_certs_len):
revoked_certs_len = len(revoked_certs)
log.info("Generating filter")
cascade = FilterCascade([],
version=1,
defaultHashAlg=fileformats.HashAlgorithm.MURMUR3)
cascade.set_crlite_error_rates(include_len=revoked_certs_len,
exclude_len=nonrevoked_certs_len)
cascade.initialize(include=revoked_certs, exclude=nonrevoked_certs)
stats["mlbf_fprs"] = cascade.error_rates
stats["mlbf_version"] = cascade.version
stats["mlbf_layers"] = cascade.layerCount()
stats["mlbf_bits"] = cascade.bitCount()
log.debug("Filter cascade layers: {layers}, bit: {bits}".format(
layers=cascade.layerCount(), bits=cascade.bitCount()))
metrics.gauge("GenerateMLBF.BitCount", cascade.bitCount())
metrics.gauge("GenerateMLBF.LayerCount", cascade.layerCount())
return cascade