def test_stash_because_previous_mlbf(self):
set_config(MLBF_TIME_CONFIG_KEY, 123456, json_value=True)
set_config(MLBF_BASE_ID_CONFIG_KEY, 123456, json_value=True)
prev_blocked_path = os.path.join(
settings.MLBF_STORAGE_PATH, '123456', 'blocked.json')
with storage.open(prev_blocked_path, 'w') as blocked_file:
json.dump(['madeup@guid:123'], blocked_file)
upload_mlbf_to_remote_settings()
generation_time = int(
datetime.datetime(2020, 1, 1, 12, 34, 56).timestamp() * 1000)
self.publish_attachment_mock.assert_called_with(
{'key_format': MLBF.KEY_FORMAT,
'generation_time': generation_time,
'attachment_type': 'bloomfilter-full'},
('filter.bin', mock.ANY, 'application/octet-stream'))
self.publish_record_mock.assert_called_with({
'key_format': MLBF.KEY_FORMAT,
'stash_time': generation_time,
'stash': {
'blocked': [
f'{self.block.guid}:'
f'{self.block.addon.current_version.version}'],
'unblocked': ['madeup@guid:123']}
})
self.delete_mock.assert_not_called()
assert (
get_config(MLBF_TIME_CONFIG_KEY, json_value=True) ==
generation_time)
assert (
get_config(MLBF_BASE_ID_CONFIG_KEY, json_value=True) ==
123456)
def test_no_previous_mlbf(self):
upload_mlbf_to_remote_settings()
generation_time = int(
datetime.datetime(2020, 1, 1, 12, 34, 56).timestamp() * 1000)
self.publish_attachment_mock.assert_called_with(
{
'key_format': MLBF.KEY_FORMAT,
'generation_time': generation_time,
'attachment_type': 'bloomfilter-base'
}, ('filter.bin', mock.ANY, 'application/octet-stream'))
assert (get_config(MLBF_TIME_CONFIG_KEY,
json_value=True) == generation_time)
assert (get_config(MLBF_BASE_ID_CONFIG_KEY,
json_value=True) == generation_time)
self.publish_record_mock.assert_not_called()
self.delete_mock.assert_called_once()
gen_path = os.path.join(settings.MLBF_STORAGE_PATH,
str(generation_time))
assert os.path.getsize(os.path.join(gen_path, 'filter'))
assert os.path.getsize(os.path.join(gen_path, 'blocked.json'))
assert os.path.getsize(os.path.join(gen_path, 'notblocked.json'))
# no stash because no previous mlbf
assert not os.path.exists(os.path.join(gen_path, 'stash.json'))
self.statsd_incr_mock.assert_has_calls([
mock.call(f'{STATSD_PREFIX}blocked_changed', 1),
mock.call(f'{STATSD_PREFIX}blocked_count', 1),
mock.call(f'{STATSD_PREFIX}not_blocked_count', 3),
mock.call('blocklist.tasks.upload_filter.reset_collection'),
mock.call('blocklist.tasks.upload_filter.upload_mlbf'),
mock.call('blocklist.tasks.upload_filter.upload_mlbf.base'),
mock.call(f'{STATSD_PREFIX}success'),
])
def test_force_base_option(self, should_reset_mock):
should_reset_mock.return_value = False
# set the times to now
now = datetime.datetime.now()
now_timestamp = now.timestamp() * 1000
set_config(MLBF_TIME_CONFIG_KEY, now_timestamp, json_value=True)
self.block.update(modified=now)
prev_blocked_path = os.path.join(
settings.MLBF_STORAGE_PATH, str(now_timestamp), 'blocked.json')
with storage.open(prev_blocked_path, 'w') as blocked_file:
json.dump([f'{self.block.guid}:1.2b3'], blocked_file)
# without force_base nothing happens
upload_mlbf_to_remote_settings()
self.publish_attachment_mock.assert_not_called()
self.publish_record_mock.assert_not_called()
# but with force_base=True we generate a filter
upload_mlbf_to_remote_settings(force_base=True)
self.publish_attachment_mock.assert_called_once() # the mlbf
self.publish_record_mock.assert_not_called() # no stash
self.delete_mock.assert_called() # the collection was cleared
# doublecheck no stash
gen_path = os.path.join(
settings.MLBF_STORAGE_PATH,
str(get_config(MLBF_TIME_CONFIG_KEY, json_value=True)))
# no stash because we're starting with a new base mlbf
assert not os.path.exists(os.path.join(gen_path, 'stash.json'))
def test_no_statsd_ping_when_switch_off(self, inner_mock):
with override_switch('blocklist_mlbf_submit', active=False):
upload_mlbf_to_remote_settings()
self.statsd_incr_mock.assert_not_called()
with override_switch('blocklist_mlbf_submit', active=True):
upload_mlbf_to_remote_settings()
self.statsd_incr_mock.assert_called()
def test_waffle_off_disables_publishing(self):
upload_mlbf_to_remote_settings()
self.publish_attachment_mock.assert_not_called()
self.publish_record_mock.assert_not_called()
assert not get_config(MLBF_TIME_CONFIG_KEY)
# except when 'bypass_switch' kwarg is passed
upload_mlbf_to_remote_settings(bypass_switch=True)
self.publish_attachment_mock.assert_called()
assert get_config(MLBF_TIME_CONFIG_KEY)
def test_reset_base_because_over_reset_threshold(self, should_reset_mock):
should_reset_mock.return_value = True
set_config(MLBF_TIME_CONFIG_KEY, 123456, json_value=True)
set_config(MLBF_BASE_ID_CONFIG_KEY, 987654, json_value=True)
prev_blocked_path = os.path.join(
settings.MLBF_STORAGE_PATH, '123456', 'blocked.json'
)
with storage.open(prev_blocked_path, 'w') as blocked_file:
json.dump(['madeup@guid:12345'], blocked_file)
base_blocked_path = os.path.join(
settings.MLBF_STORAGE_PATH, '987654', 'blocked.json'
)
with storage.open(base_blocked_path, 'w') as blocked_file:
json.dump([], blocked_file)
upload_mlbf_to_remote_settings()
generation_time = int(
datetime.datetime(2020, 1, 1, 12, 34, 56).timestamp() * 1000
)
self.publish_attachment_mock.assert_called_with(
{
'key_format': MLBF.KEY_FORMAT,
'generation_time': generation_time,
'attachment_type': 'bloomfilter-base',
},
('filter.bin', mock.ANY, 'application/octet-stream'),
)
self.publish_record_mock.assert_not_called()
self.delete_mock.assert_called_once()
assert get_config(MLBF_TIME_CONFIG_KEY, json_value=True) == generation_time
assert get_config(MLBF_BASE_ID_CONFIG_KEY, json_value=True) == generation_time
gen_path = os.path.join(settings.MLBF_STORAGE_PATH, str(generation_time))
# no stash because we're starting with a new base mlbf
assert not os.path.exists(os.path.join(gen_path, 'stash.json'))
self.statsd_incr_mock.assert_has_calls(
[
mock.call(f'{STATSD_PREFIX}blocked_changed', 2),
mock.call(f'{STATSD_PREFIX}blocked_count', 1),
mock.call(f'{STATSD_PREFIX}not_blocked_count', 3),
mock.call('blocklist.tasks.upload_filter.reset_collection'),
mock.call('blocklist.tasks.upload_filter.upload_mlbf'),
mock.call('blocklist.tasks.upload_filter.upload_mlbf.base'),
mock.call(f'{STATSD_PREFIX}success'),
]
)
def test_stash_because_many_mlbf(self):
set_config(MLBF_TIME_CONFIG_KEY, 123456, json_value=True)
set_config(MLBF_BASE_ID_CONFIG_KEY, 987654, json_value=True)
prev_blocked_path = os.path.join(settings.MLBF_STORAGE_PATH, '123456',
'blocked.json')
with storage.open(prev_blocked_path, 'w') as blocked_file:
json.dump(['madeup@guid:12345'], blocked_file)
base_blocked_path = os.path.join(settings.MLBF_STORAGE_PATH, '987654',
'blocked.json')
with storage.open(base_blocked_path, 'w') as blocked_file:
json.dump([], blocked_file)
upload_mlbf_to_remote_settings()
generation_time = int(
datetime.datetime(2020, 1, 1, 12, 34, 56).timestamp() * 1000)
self.publish_attachment_mock.assert_called_with(
{
'key_format': MLBF.KEY_FORMAT,
'generation_time': generation_time,
'attachment_type': 'bloomfilter-full'
}, ('filter.bin', mock.ANY, 'application/octet-stream'))
self.publish_record_mock.assert_called_with({
'key_format': MLBF.KEY_FORMAT,
'stash_time': generation_time,
'stash': {
'blocked': [
f'{self.block.guid}:'
f'{self.block.addon.current_version.version}'
],
'unblocked': ['madeup@guid:12345']
}
})
self.delete_mock.assert_not_called()
assert (get_config(MLBF_TIME_CONFIG_KEY,
json_value=True) == generation_time)
assert (get_config(MLBF_BASE_ID_CONFIG_KEY, json_value=True) == 987654)
self.statsd_incr_mock.assert_has_calls([
mock.call(f'{STATSD_PREFIX}blocked_changed', 2),
mock.call(f'{STATSD_PREFIX}blocked_count', 1),
mock.call(f'{STATSD_PREFIX}not_blocked_count', 3),
mock.call('blocklist.tasks.upload_filter.upload_stash'),
mock.call('blocklist.tasks.upload_filter.upload_mlbf'),
mock.call('blocklist.tasks.upload_filter.upload_mlbf.full'),
mock.call(f'{STATSD_PREFIX}success'),
])
def test_stash_because_previous_mlbf(self):
set_config(MLBF_TIME_CONFIG_KEY, 123456, json_value=True)
set_config(MLBF_BASE_ID_CONFIG_KEY, 123456, json_value=True)
prev_blocked_path = os.path.join(
settings.MLBF_STORAGE_PATH, '123456', 'blocked.json'
)
with storage.open(prev_blocked_path, 'w') as blocked_file:
json.dump(['madeup@guid:123'], blocked_file)
upload_mlbf_to_remote_settings()
generation_time = int(
datetime.datetime(2020, 1, 1, 12, 34, 56).timestamp() * 1000
)
self.publish_attachment_mock.assert_not_called()
self.publish_record_mock.assert_called_with(
{
'key_format': MLBF.KEY_FORMAT,
'stash_time': generation_time,
'stash': {
'blocked': [
f'{self.block.guid}:'
f'{self.block.addon.current_version.version}'
],
'unblocked': ['madeup@guid:123'],
},
}
)
self.delete_mock.assert_not_called()
assert get_config(MLBF_TIME_CONFIG_KEY, json_value=True) == generation_time
assert get_config(MLBF_BASE_ID_CONFIG_KEY, json_value=True) == 123456
self.statsd_incr_mock.assert_has_calls(
[
mock.call(f'{STATSD_PREFIX}blocked_changed', 2),
mock.call(f'{STATSD_PREFIX}blocked_count', 1),
mock.call(f'{STATSD_PREFIX}not_blocked_count', 3),
mock.call('blocklist.tasks.upload_filter.upload_stash'),
mock.call(f'{STATSD_PREFIX}success'),
]
)
self.cleanup_files_mock.assert_called_with(base_filter_id=123456)
def test_no_block_changes(frozen_time, self):
# This was the last time the mlbf was generated
last_time = int(
(frozen_time() - timedelta(seconds=1)).timestamp() * 1000)
# And the Block was modified just that before so would be included
self.block.update(modified=(frozen_time() - timedelta(seconds=2)))
set_config(MLBF_TIME_CONFIG_KEY, last_time, json_value=True)
set_config(MLBF_BASE_ID_CONFIG_KEY, last_time, json_value=True)
prev_blocked_path = os.path.join(settings.MLBF_STORAGE_PATH,
str(last_time), 'blocked.json')
with storage.open(prev_blocked_path, 'w') as blocked_file:
json.dump([f'{self.block.guid}:1.2b3'], blocked_file)
upload_mlbf_to_remote_settings()
# So no need for a new bloomfilter
self.publish_attachment_mock.assert_not_called()
self.publish_record_mock.assert_not_called()
self.cleanup_files_mock.assert_not_called()
# But if we add a new Block a new filter is needed
Block.objects.create(
addon=addon_factory(file_kw={
'is_signed': True,
'is_webextension': True
}),
updated_by=user_factory(),
)
upload_mlbf_to_remote_settings()
self.publish_attachment_mock.assert_not_called()
self.publish_record_mock.assert_called_once()
self.cleanup_files_mock.assert_called_once()
assert get_config(MLBF_TIME_CONFIG_KEY, json_value=True) == int(
datetime.datetime(2020, 1, 1, 12, 34, 56).timestamp() * 1000)
self.statsd_incr_mock.reset_mock()
frozen_time.tick()
# If the first block is deleted the last_modified date won't have
# changed, but the number of blocks will, so trigger a new filter.
last_modified = get_blocklist_last_modified_time()
self.block.delete()
assert last_modified == get_blocklist_last_modified_time()
upload_mlbf_to_remote_settings()
self.publish_attachment_mock.assert_not_called()
self.publish_record_mock.call_count == 2
self.cleanup_files_mock.call_count == 2
self.statsd_incr_mock.assert_has_calls([
mock.call(f'{STATSD_PREFIX}blocked_changed', 1),
mock.call(f'{STATSD_PREFIX}blocked_count', 1),
mock.call(f'{STATSD_PREFIX}not_blocked_count', 4),
mock.call('blocklist.tasks.upload_filter.upload_stash'),
mock.call(f'{STATSD_PREFIX}success'),
])
self.cleanup_files_mock.assert_called_with(base_filter_id=last_time)
