def test_3_s3_sources_in_list_instantiation(self): # Basically the same tests in test_0_get_list and test_2_delta above dumdum = dummy(body='4:4\n%s' % self.hg[:4], path='/gethash') for list_ in ('mozpub-track-digest256', 'testpub-bananas-digest256'): sblist = get_list(dumdum, list_) self.assertIsInstance(sblist, Digest256) self.assertEqual(sblist.delta([1, 2], [3]), ([4, 5], [6]))
def test_3_s3_sources_in_list_instantiation(self): # Basically the same tests in test_0_get_list and test_2_delta above dumdum = dummy(body='4:4\n%s' % self.hg[:4], path='/gethash') for list_ in ('mozpub-track-digest256', 'testpub-bananas-digest256'): sblist = get_list(dumdum, list_) self.assertIsInstance(sblist, Digest256) self.assertEqual(sblist.delta([1, 2], [3]), ([4, 5], [6]))
def test_2_delta(self): dumdum = dummy(body='4:4\n%s' % self.hg[:4], path='/gethash') sblist = get_list(dumdum, 'mozpub-track-digest256') # By way of explanation: # # In the data file. # Chunks 1, 2, 4, and 5 are "add" chunks # Chunks 3 and 6 are "sub" chunks # # So delta([1, 2], [3]) should return # ([4, 5], [6]) self.assertEqual(sblist.delta([1, 2], [3]), ([4, 5], [6]))
def test_2_delta(self): dumdum = dummy(body='4:4\n%s' % self.hg[:4], path='/gethash') sblist = get_list(dumdum, 'mozpub-track-digest256') # By way of explanation: # # In the data file. # Chunks 1, 2, 4, and 5 are "add" chunks # Chunks 3 and 6 are "sub" chunks # # So delta([1, 2], [3]) should return # ([4, 5], [6]) self.assertEqual(sblist.delta([1, 2], [3]), ([4, 5], [6]))
def test_0_get_list(self): dumdum = dummy(body='4:4\n%s' % self.hg[:4], path='/gethash') sblist = get_list(dumdum, 'mozpub-track-digest256') self.assertIsInstance(sblist, Digest256)
def test_9_get_list_version_not_specified(self): dumdum = dummy(body='4:4\n%s' % self.hg[:4], path='/gethash') sblist, list_ver = get_list(dumdum, 'mozpub-track-digest256') self.assertIsNone(list_ver)
for list_info in parsed: # Do we even serve that list? if list_info.name not in _setting(request, 'shavar', 'lists_served', tuple()): logger.warn('Unknown list "%s" reported; ignoring' % list_info.name) annotate_request(request, "shavar.downloads.unknown.list", 1) continue provider, type_, format_ = list_info.name.split('-', 2) if not provider or not type_ or not format_: s = 'Unknown list format for "%s"; ignoring' % list_info.name logger.error(s) annotate_request(request, "shavar.downloads.unknown.format", 1) raise HTTPBadRequest(s) sblist = get_list(request, list_info.name) # Calculate delta to_add, to_sub = sblist.delta(list_info.adds, list_info.subs) # No delta? No response, I think. Spec doesn't actually say. if not to_add and not to_sub: continue # Fetch the appropriate chunks resp_payload['lists'][list_info.name] = sblist.fetch(to_add, to_sub) # Not publishing deltas for this list? Delete all previous chunks to # make way for the new corpus if _setting(request, list_info.name, 'not_publishing_deltas'): # Raise hell if we have suspicious data with this flag set
def test_0_get_list(self): dumdum = dummy(body='4:4\n%s' % self.hg[:4], path='/gethash') sblist = get_list(dumdum, 'mozpub-track-digest256') self.assertIsInstance(sblist, Digest256)
def downloads_view(request): # Use the new config variable name but support the old one for default_interval = _setting(request, 'shavar', 'default_interval', None) backoff_delay = _setting(request, 'shavar', 'client_backoff_delay', None) # Throw a fit if both are specified if default_interval is not None and backoff_delay is not None: raise ConfigurationError("Specify either default_interval or " "client_backoff_delay in the [shavar] " "section of your config but not both.\n" "client_backoff_delay is preferred.") delay = backoff_delay or default_interval or 30 * 60 resp_payload = {'interval': delay, 'lists': {}} try: parsed = parse_downloads(request) except ParseError as e: capture_exception(e) raise HTTPBadRequest(e) for list_info in parsed: # Do we even serve that list? if list_info.name not in _setting(request, 'shavar', 'list_names_served', tuple()): logger.warn('Unknown list "%s" reported; ignoring' % list_info.name) annotate_request(request, "shavar.downloads.unknown.list", 1) continue provider, type_, format_ = list_info.name.split('-', 2) if not provider or not type_ or not format_: s = 'Unknown list format for "%s"; ignoring' % list_info.name logger.error(s) annotate_request(request, "shavar.downloads.unknown.format", 1) raise HTTPBadRequest(s) app_ver = str(request.GET['appver']) sblist, list_ver = get_list(request, list_info.name, app_ver) # Calculate delta to_add, to_sub = sblist.delta(list_info.adds, list_info.subs) # No delta? No response, I think. Spec doesn't actually say. if not to_add and not to_sub: continue # Fetch the appropriate chunks resp_payload['lists'][list_info.name] = { 'sblist': sblist, 'ldata': sblist.fetch(to_add, to_sub), 'list_ver': list_ver } # Not publishing deltas for this list? Delete all previous chunks to # make way for the new corpus # if _setting(request, list_info.name, 'not_publishing_deltas'): if sblist.settings.get('not_publishing_deltas'): # Raise hell if we have suspicious data with this flag set if (len(to_add) != 1 or len(to_sub) != 0): logger.error("Configuration error! Mismatch between " "{0}'s configuration has " "'not_publishing_deltas' enabled but its data" "file has more than one chunk to serve.".format( list_info.name)) raise HTTPInternalServerError() resp_payload['lists'][list_info.name]['adddels'] = list_info.adds return HTTPOk(content_type="application/octet-stream", body=format_downloads(request, resp_payload))
if list_info.name not in _setting( request, 'shavar', 'list_names_served', tuple() ): logger.warn('Unknown list "%s" reported; ignoring' % list_info.name) annotate_request(request, "shavar.downloads.unknown.list", 1) continue provider, type_, format_ = list_info.name.split('-', 2) if not provider or not type_ or not format_: s = 'Unknown list format for "%s"; ignoring' % list_info.name logger.error(s) annotate_request(request, "shavar.downloads.unknown.format", 1) raise HTTPBadRequest(s) app_ver = str(request.GET['appver']) sblist, list_ver = get_list(request, list_info.name, app_ver) # Calculate delta to_add, to_sub = sblist.delta(list_info.adds, list_info.subs) # No delta? No response, I think. Spec doesn't actually say. if not to_add and not to_sub: continue # Fetch the appropriate chunks resp_payload['lists'][list_info.name] = { 'sblist': sblist, 'ldata': sblist.fetch(to_add, to_sub), 'list_ver': list_ver }
for list_info in parsed: # Do we even serve that list? if list_info.name not in _setting(request, 'shavar', 'list_names_served', tuple()): logger.warn('Unknown list "%s" reported; ignoring' % list_info.name) annotate_request(request, "shavar.downloads.unknown.list", 1) continue provider, type_, format_ = list_info.name.split('-', 2) if not provider or not type_ or not format_: s = 'Unknown list format for "%s"; ignoring' % list_info.name logger.error(s) annotate_request(request, "shavar.downloads.unknown.format", 1) raise HTTPBadRequest(s) sblist = get_list(request, list_info.name) # Calculate delta to_add, to_sub = sblist.delta(list_info.adds, list_info.subs) # No delta? No response, I think. Spec doesn't actually say. if not to_add and not to_sub: continue # Fetch the appropriate chunks resp_payload['lists'][list_info.name] = { 'sblist': sblist, 'ldata': sblist.fetch(to_add, to_sub) } # Not publishing deltas for this list? Delete all previous chunks to