def generateMLBF(stats, *, blocked, not_blocked, capacity, diffMetaFile=None):
"""Based on:
https://github.com/mozilla/crlite/blob/master/create_filter_cascade/certs_to_crlite.py
"""
fprs = [len(blocked) / (math.sqrt(2) * len(not_blocked)), 0.5]
if diffMetaFile is not None:
log.info(
"Generating filter with characteristics from mlbf base file {}".
format(diffMetaFile))
mlbf_meta_file = open(diffMetaFile, 'rb')
cascade = FilterCascade.loadDiffMeta(mlbf_meta_file)
cascade.error_rates = fprs
else:
log.info("Generating filter")
cascade = FilterCascade.cascade_with_characteristics(
int(len(blocked) * capacity), fprs)
cascade.version = 1
cascade.initialize(include=blocked, exclude=not_blocked)
stats['mlbf_fprs'] = fprs
stats['mlbf_version'] = cascade.version
stats['mlbf_layers'] = cascade.layerCount()
stats['mlbf_bits'] = cascade.bitCount()
log.debug("Filter cascade layers: {layers}, bit: {bits}".format(
layers=cascade.layerCount(), bits=cascade.bitCount()))
return cascade
def generate_mlbf(stats, key_format, *, blocked=None, not_blocked=None):
"""Based on:
https://github.com/mozilla/crlite/blob/master/create_filter_cascade/certs_to_crlite.py
"""
blocked = hash_filter_inputs(
blocked or get_blocked_guids(), key_format)
not_blocked = hash_filter_inputs(
not_blocked or get_all_guids(), key_format)
not_blocked = list(set(not_blocked) - set(blocked))
stats['mlbf_blocked_count'] = len(blocked)
stats['mlbf_unblocked_count'] = len(not_blocked)
fprs = [len(blocked) / (math.sqrt(2) * len(not_blocked)), 0.5]
log.info("Generating filter")
cascade = FilterCascade.cascade_with_characteristics(
int(len(blocked) * 1.1), fprs)
cascade.version = 1
cascade.initialize(include=blocked, exclude=not_blocked)
stats['mlbf_fprs'] = fprs
stats['mlbf_version'] = cascade.version
stats['mlbf_layers'] = cascade.layerCount()
stats['mlbf_bits'] = cascade.bitCount()
log.debug("Filter cascade layers: {layers}, bit: {bits}".format(
layers=cascade.layerCount(), bits=cascade.bitCount()))
cascade.check(entries=blocked, exclusions=not_blocked)
return cascade
