def upload_filter_to_kinto(generation_time, is_base=True, upload_stash=False): server = KintoServer(KINTO_BUCKET, KINTO_COLLECTION_MLBF, kinto_sign_off_needed=False) mlbf = MLBF(generation_time) if is_base: # clear the collection for the base - we want to be the only filter server.delete_all_records() # Deal with possible stashes first if upload_stash: # If we have a stash, write that stash_data = { 'key_format': MLBF.KEY_FORMAT, 'stash_time': generation_time, 'stash': mlbf.stash_json, } server.publish_record(stash_data) # Then the bloomfilter data = { 'key_format': MLBF.KEY_FORMAT, 'generation_time': generation_time, 'attachment_type': BLOCKLIST_RECORD_MLBF_BASE if is_base else BLOCKLIST_RECORD_MLBF_UPDATE, } with storage.open(mlbf.filter_path, 'rb') as filter_file: attachment = ('filter.bin', filter_file, 'application/octet-stream') server.publish_attachment(data, attachment) server.complete_session() set_config(MLBF_TIME_CONFIG_KEY, generation_time, json_value=True) if is_base: set_config(MLBF_BASE_ID_CONFIG_KEY, generation_time, json_value=True)
def test_publish_attachment(self, uuidmock): uuidmock.uuid4.return_value = 1234567890 server = KintoServer('foo', 'baa') server._setup_done = True assert not server._changes url = ( settings.REMOTE_SETTINGS_WRITER_URL + 'buckets/foo/collections/baa/records/1234567890/attachment') responses.add( responses.POST, url, json={'data': {'id': '1234567890'}}) with tempfile.TemporaryFile() as attachment: record = server.publish_attachment( {'something': 'somevalue'}, ('file', attachment)) assert server._changes assert record == {'id': '1234567890'} url = ( settings.REMOTE_SETTINGS_WRITER_URL + 'buckets/foo/collections/baa/records/an-id/attachment') responses.add( responses.POST, url, json={'data': {'id': 'an-id'}}) with tempfile.TemporaryFile() as attachment: record = server.publish_attachment( {'something': 'somevalue'}, ('otherfile', attachment), 'an-id') assert record == {'id': 'an-id'}
def upload_filter_to_kinto(generation_time): server = KintoServer(KINTO_BUCKET, KINTO_COLLECTION_MLBF, kinto_sign_off_needed=False) data = { 'key_format': MLBF.KEY_FORMAT, 'generation_time': generation_time, } mlbf_path = MLBF(generation_time).filter_path with storage.open(mlbf_path) as filter_file: attachment = ('filter.bin', filter_file, 'application/octet-stream') server.publish_attachment(data, attachment) server.complete_session() set_config(MLBF_TIME_CONFIG_KEY, generation_time, json_value=True)
def upload_mlbf_to_kinto(): if not waffle.switch_is_active('blocklist_mlbf_submit'): log.info('Upload MLBF to kinto cron job disabled.') return log.info('Starting Upload MLBF to kinto cron job.') server = KintoServer(KINTO_BUCKET, KINTO_COLLECTION_MLBF) stats = {} key_format = get_mlbf_key_format() bloomfilter = generate_mlbf(stats, key_format) with tempfile.NamedTemporaryFile() as filter_file: bloomfilter.tofile(filter_file) filter_file.seek(0) # TODO: sign filter blob data = {'key_format': key_format} attachment = ('filter.bin', filter_file, 'application/octet-stream') server.publish_attachment(data, attachment) log.info(json.dumps(stats))
def upload_mlbf_to_kinto(): if not waffle.switch_is_active('blocklist_mlbf_submit'): log.info('Upload MLBF to kinto cron job disabled.') return last_generation_time = get_config(MLBF_TIME_CONFIG_KEY, 0, json_value=True) if last_generation_time > _get_blocklist_last_modified_time(): log.info( 'No new/modified Blocks in database; skipping MLBF generation') return log.info('Starting Upload MLBF to kinto cron job.') server = KintoServer(KINTO_BUCKET, KINTO_COLLECTION_MLBF, kinto_sign_off_needed=False) stats = {} key_format = get_mlbf_key_format() # This timestamp represents the point in time when all previous addon # guid + versions and blocks were used to generate the bloomfilter. # An add-on version/file from before this time will definitely be accounted # for in the bloomfilter so we can reliably assert if it's blocked or not. # An add-on version/file from after this time can't be reliably asserted - # there may be false positives or false negatives. # https://github.com/mozilla/addons-server/issues/13695 generation_time = int(time.time() * 1000) bloomfilter = generate_mlbf(stats, key_format) with tempfile.NamedTemporaryFile() as filter_file: bloomfilter.tofile(filter_file) filter_file.seek(0) data = { 'key_format': key_format, 'generation_time': generation_time, } attachment = ('filter.bin', filter_file, 'application/octet-stream') server.publish_attachment(data, attachment) server.complete_session() set_config(MLBF_TIME_CONFIG_KEY, generation_time, json_value=True) log.info(json.dumps(stats))