def upload_filter(generation_time, is_base=True, upload_stash=False):
bucket = settings.REMOTE_SETTINGS_WRITER_BUCKET
server = RemoteSettings(
bucket, REMOTE_SETTINGS_COLLECTION_MLBF, sign_off_needed=False)
mlbf = MLBF(generation_time)
if is_base:
# clear the collection for the base - we want to be the only filter
server.delete_all_records()
# Deal with possible stashes first
if upload_stash:
# If we have a stash, write that
stash_data = {
'key_format': MLBF.KEY_FORMAT,
'stash_time': generation_time,
'stash': mlbf.stash_json,
}
server.publish_record(stash_data)
# Then the bloomfilter
data = {
'key_format': MLBF.KEY_FORMAT,
'generation_time': generation_time,
'attachment_type':
BLOCKLIST_RECORD_MLBF_BASE if is_base else
BLOCKLIST_RECORD_MLBF_UPDATE,
}
with storage.open(mlbf.filter_path, 'rb') as filter_file:
attachment = ('filter.bin', filter_file, 'application/octet-stream')
server.publish_attachment(data, attachment)
server.complete_session()
set_config(MLBF_TIME_CONFIG_KEY, generation_time, json_value=True)
if is_base:
set_config(MLBF_BASE_ID_CONFIG_KEY, generation_time, json_value=True)
def upload_filter(generation_time, is_base=True):
bucket = settings.REMOTE_SETTINGS_WRITER_BUCKET
server = RemoteSettings(bucket,
REMOTE_SETTINGS_COLLECTION_MLBF,
sign_off_needed=False)
mlbf = MLBF.load_from_storage(generation_time)
if is_base:
# clear the collection for the base - we want to be the only filter
server.delete_all_records()
statsd.incr('blocklist.tasks.upload_filter.reset_collection')
# Then the bloomfilter
data = {
'key_format': MLBF.KEY_FORMAT,
'generation_time': generation_time,
'attachment_type': BLOCKLIST_RECORD_MLBF_BASE,
}
storage = SafeStorage(user_media='mlbf_storage')
with storage.open(mlbf.filter_path, 'rb') as filter_file:
attachment = ('filter.bin', filter_file,
'application/octet-stream')
server.publish_attachment(data, attachment)
statsd.incr('blocklist.tasks.upload_filter.upload_mlbf')
statsd.incr('blocklist.tasks.upload_filter.upload_mlbf.base')
else:
# If we have a stash, write that
stash_data = {
'key_format': MLBF.KEY_FORMAT,
'stash_time': generation_time,
'stash': mlbf.stash_json,
}
server.publish_record(stash_data)
statsd.incr('blocklist.tasks.upload_filter.upload_stash')
server.complete_session()
set_config(MLBF_TIME_CONFIG_KEY, generation_time, json_value=True)
if is_base:
set_config(MLBF_BASE_ID_CONFIG_KEY, generation_time, json_value=True)
def test_publish_attachment(self, uuidmock):
uuidmock.uuid4.return_value = 1234567890
server = RemoteSettings('foo', 'baa')
server._setup_done = True
assert not server._changes
url = (settings.REMOTE_SETTINGS_WRITER_URL +
'buckets/foo/collections/baa/records/1234567890/attachment')
responses.add(responses.POST, url, json={'data': {'id': '1234567890'}})
with tempfile.TemporaryFile() as attachment:
record = server.publish_attachment({'something': 'somevalue'},
('file', attachment))
assert server._changes
assert record == {'id': '1234567890'}
url = (settings.REMOTE_SETTINGS_WRITER_URL +
'buckets/foo/collections/baa/records/an-id/attachment')
responses.add(responses.POST, url, json={'data': {'id': 'an-id'}})
with tempfile.TemporaryFile() as attachment:
record = server.publish_attachment({'something': 'somevalue'},
('otherfile', attachment),
'an-id')
assert record == {'id': 'an-id'}