def test_generate_and_write_filter(self):
self.setup_data()
mlbf = MLBF.generate_from_db(123456)
mlbf.generate_and_write_filter()
with open(mlbf.filter_path, 'rb') as filter_file:
buffer = filter_file.read()
bfilter = FilterCascade.from_buf(buffer)
blocked_versions = fetch_blocked_from_db()
blocked_guids = blocked_versions.values()
for guid, version_str in blocked_guids:
key = MLBF.KEY_FORMAT.format(guid=guid, version=version_str)
assert key in bfilter
all_addons = fetch_all_versions_from_db(blocked_versions.keys())
for guid, version_str in all_addons:
# edge case where a version_str exists in both
if (guid, version_str) in blocked_guids:
continue
key = MLBF.KEY_FORMAT.format(guid=guid, version=version_str)
assert key not in bfilter
# Occasionally a combination of salt generated with secrets.token_bytes
# and the version str generated in version_factory results in a
# collision in layer 1 of the bloomfilter, leading to a second layer
# being generated. When this happens the bitCount and size is larger.
expected_size, expected_bit_count = (
(203, 1384) if bfilter.layerCount() == 1 else (393, 2824)
)
assert os.stat(mlbf.filter_path).st_size == expected_size, (
blocked_guids,
all_addons,
)
assert bfilter.bitCount() == expected_bit_count, (blocked_guids, all_addons)
def test_generate_and_write_mlbf(self):
self.setup_data()
mlbf = MLBF(123456)
mlbf.generate_and_write_mlbf()
with open(mlbf.filter_path, 'rb') as filter_file:
buffer = filter_file.read()
bfilter = FilterCascade.from_buf(buffer)
blocked_versions = mlbf.fetch_blocked_from_db()
blocked_guids = blocked_versions.values()
for guid, version_str in blocked_guids:
key = mlbf.KEY_FORMAT.format(guid=guid, version=version_str)
assert key in bfilter
all_addons = mlbf.fetch_all_versions_from_db(blocked_versions.keys())
for guid, version_str in all_addons:
# edge case where a version_str exists in both
if (guid, version_str) in blocked_guids:
continue
key = mlbf.KEY_FORMAT.format(guid=guid, version=version_str)
assert key not in bfilter
assert os.stat(mlbf.filter_path).st_size == 203, (
blocked_guids, all_addons)
assert bfilter.bitCount() == 1384, (
blocked_guids, all_addons)
def load_filter(self, *, filter_path, coverage_path):
self.filter_file = filter_path
self.coverage_file = coverage_path
self.filtercascade = FilterCascade.from_buf(
self.filter_file.read_bytes())
self.issuer_to_revocations = collections.defaultdict(list)
self.stash_files = list()
self.coverage = {}
with open(coverage_path, "r") as f:
for ct_log in json.load(f):
log_id = base64.b64decode(ct_log["logID"])
self.coverage[log_id] = (ct_log["minTimestamp"],
ct_log["maxTimestamp"])
def test_generate_and_write_mlbf(self):
mlbf = MLBF(123456)
mlbf.generate_and_write_mlbf()
with open(mlbf.filter_path, 'rb') as filter_file:
buffer = filter_file.read()
bfilter = FilterCascade.from_buf(buffer)
assert bfilter.bitCount() == 3008
blocked_versions = mlbf.get_blocked_versions()
for guid, version_str in blocked_versions.values():
key = mlbf.KEY_FORMAT.format(guid=guid, version=version_str)
assert key in bfilter
for guid, version_str in mlbf.get_all_guids(blocked_versions.keys()):
key = mlbf.KEY_FORMAT.format(guid=guid, version=version_str)
assert key not in bfilter
assert os.stat(mlbf.filter_path).st_size == 406