def test_generate_and_write_mlbf(self):
self.setup_data()
mlbf = MLBF(123456)
mlbf.generate_and_write_mlbf()
with open(mlbf.filter_path, 'rb') as filter_file:
buffer = filter_file.read()
bfilter = FilterCascade.from_buf(buffer)
blocked_versions = mlbf.fetch_blocked_from_db()
blocked_guids = blocked_versions.values()
for guid, version_str in blocked_guids:
key = mlbf.KEY_FORMAT.format(guid=guid, version=version_str)
assert key in bfilter
all_addons = mlbf.fetch_all_versions_from_db(blocked_versions.keys())
for guid, version_str in all_addons:
# edge case where a version_str exists in both
if (guid, version_str) in blocked_guids:
continue
key = mlbf.KEY_FORMAT.format(guid=guid, version=version_str)
assert key not in bfilter
assert os.stat(mlbf.filter_path).st_size == 203, (
blocked_guids, all_addons)
assert bfilter.bitCount() == 1384, (
blocked_guids, all_addons)
def test_fetch_all_versions_from_db(self):
self.setup_data()
all_versions = MLBF.fetch_all_versions_from_db()
assert len(all_versions) == File.objects.count() == 5 + 10 + 5
assert (self.five_ver_block.guid, '123.40') in all_versions
assert (self.five_ver_block.guid, '123.5') in all_versions
assert (self.five_ver_block.guid, '123.45.1') in all_versions
assert (self.five_ver_block.guid, '123.5.1') in all_versions
assert (self.five_ver_block.guid, '123.5.2') in all_versions
over_tuple = (self.over.guid, self.over.addon.current_version.version)
under_tuple = (
self.under.guid, self.under.addon.current_version.version)
assert over_tuple in all_versions
assert under_tuple in all_versions
assert (self.addon_deleted_before_block.guid, '2') in all_versions
# this is fine; test_hash_filter_inputs removes duplicates.
assert all_versions.count(
(self.addon_deleted_before_block.guid, '2.1')) == 2
assert (self.addon_deleted_before_block.guid, '2.5') in all_versions
assert (self.addon_deleted_before_block.guid, '3.0') in all_versions
# repeat, but with excluded version ids
excludes = [self.five_ver_123_40.id, self.five_ver_123_5.id]
all_versions = MLBF.fetch_all_versions_from_db(excludes)
assert len(all_versions) == 18
assert (self.five_ver_block.guid, '123.40') not in all_versions
assert (self.five_ver_block.guid, '123.5') not in all_versions
assert (self.five_ver_block.guid, '123.45.1') in all_versions
assert (self.five_ver_block.guid, '123.5.1') in all_versions
assert (self.five_ver_block.guid, '123.5.2') in all_versions
over_tuple = (self.over.guid, self.over.addon.current_version.version)
under_tuple = (
self.under.guid, self.under.addon.current_version.version)
assert over_tuple in all_versions
assert under_tuple in all_versions
def test_generate_and_write_mlbf(self):
mlbf = MLBF(123456)
mlbf.generate_and_write_mlbf()
with open(mlbf.filter_path, 'rb') as filter_file:
buffer = filter_file.read()
bfilter = FilterCascade.from_buf(buffer)
assert bfilter.bitCount() == 3008
blocked_versions = mlbf.get_blocked_versions()
for guid, version_str in blocked_versions.values():
key = mlbf.KEY_FORMAT.format(guid=guid, version=version_str)
assert key in bfilter
for guid, version_str in mlbf.get_all_guids(blocked_versions.keys()):
key = mlbf.KEY_FORMAT.format(guid=guid, version=version_str)
assert key not in bfilter
assert os.stat(mlbf.filter_path).st_size == 406
def test_write_stash(self):
old_mlbf = MLBF('old')
old_mlbf.generate_and_write_mlbf()
new_mlbf = MLBF('new_no_change')
new_mlbf.generate_and_write_mlbf()
new_mlbf.write_stash('old')
with open(new_mlbf.stash_path) as stash_file:
assert json.load(stash_file) == {'blocked': [], 'unblocked': []}
# add a new Block and delete one
Block.objects.create(addon=addon_factory(guid='fooo@baaaa',
version_kw={'version': '999'},
file_kw={
'is_signed': True,
'is_webextension': True
}),
updated_by=user_factory())
self.five_ver_123_5.delete(hard=True)
newer_mlbf = MLBF('new_one_change')
newer_mlbf.generate_and_write_mlbf()
newer_mlbf.write_stash('new_no_change')
with open(newer_mlbf.stash_path) as stash_file:
assert json.load(stash_file) == {
'blocked': ['fooo@baaaa:999'],
'unblocked': [f'{self.five_ver.guid}:123.5']
}
def test_should_reset_base_filter_and_blocks_changed_since_previous(self):
base_mlbf = MLBF('base')
# should handle the files not existing
assert MLBF('no_files').should_reset_base_filter(base_mlbf)
assert MLBF('no_files').blocks_changed_since_previous(base_mlbf)
base_mlbf.generate_and_write_mlbf()
no_change_mlbf = MLBF('no_change')
no_change_mlbf.generate_and_write_mlbf()
assert not no_change_mlbf.should_reset_base_filter(base_mlbf)
assert not no_change_mlbf.blocks_changed_since_previous(base_mlbf)
# make some changes
Block.objects.create(addon=addon_factory(guid='fooo@baaaa',
version_kw={'version': '999'},
file_kw={
'is_signed': True,
'is_webextension': True
}),
updated_by=user_factory())
self.five_ver_123_5.delete(hard=True)
small_change_mlbf = MLBF('small_change')
small_change_mlbf.generate_and_write_mlbf()
# but the changes were small (less than threshold) so no need for new
# base filter
assert not small_change_mlbf.should_reset_base_filter(base_mlbf)
# there _were_ changes though
assert small_change_mlbf.blocks_changed_since_previous(base_mlbf)
# double check what the differences were
diffs = MLBF.generate_diffs(previous=base_mlbf.blocked_json,
current=small_change_mlbf.blocked_json)
assert diffs == ({'fooo@baaaa:999'},
{f'{self.five_ver_block.guid}:123.5'})
# so lower the threshold
to_patch = 'olympia.blocklist.mlbf.BASE_REPLACE_THRESHOLD'
with mock.patch(to_patch, 1):
assert small_change_mlbf.should_reset_base_filter(base_mlbf)
assert small_change_mlbf.blocks_changed_since_previous(base_mlbf)