def generate_mlbf(stats, blocked, not_blocked):
log.info('Starting to generating bloomfilter')
cascade = FilterCascade(
defaultHashAlg=HashAlgorithm.SHA256,
salt=secrets.token_bytes(16),
)
error_rates = sorted((len(blocked), len(not_blocked)))
cascade.set_crlite_error_rates(include_len=error_rates[0],
exclude_len=error_rates[1])
stats['mlbf_blocked_count'] = len(blocked)
stats['mlbf_notblocked_count'] = len(not_blocked)
cascade.initialize(include=blocked, exclude=not_blocked)
stats['mlbf_version'] = cascade.version
stats['mlbf_layers'] = cascade.layerCount()
stats['mlbf_bits'] = cascade.bitCount()
log.info(f'Filter cascade layers: {cascade.layerCount()}, '
f'bit: {cascade.bitCount()}')
cascade.verify(include=blocked, exclude=not_blocked)
return cascade
def generate_mlbf(cls, stats, blocked, not_blocked):
"""Originally based on:
https://github.com/mozilla/crlite/blob/master/create_filter_cascade/certs_to_crlite.py
(not so much any longer, apart from the fprs calculation)
"""
salt = secrets.token_bytes(16)
stats['mlbf_blocked_count'] = len(blocked)
stats['mlbf_notblocked_count'] = len(not_blocked)
fprs = [len(blocked) / (math.sqrt(2) * len(not_blocked)), 0.5]
log.info("Generating filter")
cascade = FilterCascade(
error_rates=fprs,
defaultHashAlg=HashAlgorithm.SHA256,
salt=salt,
)
cascade.initialize(include=blocked, exclude=not_blocked)
stats['mlbf_fprs'] = fprs
stats['mlbf_version'] = cascade.version
stats['mlbf_layers'] = cascade.layerCount()
stats['mlbf_bits'] = cascade.bitCount()
log.debug("Filter cascade layers: {layers}, bit: {bits}".format(
layers=cascade.layerCount(), bits=cascade.bitCount()))
cascade.verify(include=blocked, exclude=not_blocked)
return cascade
