def test_3_s3_sources_in_list_instantiation(self):
# Basically the same tests in test_0_get_list and test_2_delta above
dumdum = dummy(body='4:4\n%s' % self.hg[:4], path='/gethash')
for list_ in ('mozpub-track-digest256', 'testpub-bananas-digest256'):
sblist = get_list(dumdum, list_)
self.assertIsInstance(sblist, Digest256)
self.assertEqual(sblist.delta([1, 2], [3]), ([4, 5], [6]))
def test_3_s3_sources_in_list_instantiation(self):
# Basically the same tests in test_0_get_list and test_2_delta above
dumdum = dummy(body='4:4\n%s' % self.hg[:4], path='/gethash')
for list_ in ('mozpub-track-digest256', 'testpub-bananas-digest256'):
sblist = get_list(dumdum, list_)
self.assertIsInstance(sblist, Digest256)
self.assertEqual(sblist.delta([1, 2], [3]), ([4, 5], [6]))
def test_2_delta(self):
dumdum = dummy(body='4:4\n%s' % self.hg[:4], path='/gethash')
sblist = get_list(dumdum, 'mozpub-track-digest256')
# By way of explanation:
#
# In the data file.
# Chunks 1, 2, 4, and 5 are "add" chunks
# Chunks 3 and 6 are "sub" chunks
#
# So delta([1, 2], [3]) should return
# ([4, 5], [6])
self.assertEqual(sblist.delta([1, 2], [3]), ([4, 5], [6]))
def test_2_delta(self):
dumdum = dummy(body='4:4\n%s' % self.hg[:4], path='/gethash')
sblist = get_list(dumdum, 'mozpub-track-digest256')
# By way of explanation:
#
# In the data file.
# Chunks 1, 2, 4, and 5 are "add" chunks
# Chunks 3 and 6 are "sub" chunks
#
# So delta([1, 2], [3]) should return
# ([4, 5], [6])
self.assertEqual(sblist.delta([1, 2], [3]), ([4, 5], [6]))
def test_0_get_list(self):
dumdum = dummy(body='4:4\n%s' % self.hg[:4], path='/gethash')
sblist = get_list(dumdum, 'mozpub-track-digest256')
self.assertIsInstance(sblist, Digest256)
def test_9_get_list_version_not_specified(self):
dumdum = dummy(body='4:4\n%s' % self.hg[:4], path='/gethash')
sblist, list_ver = get_list(dumdum, 'mozpub-track-digest256')
self.assertIsNone(list_ver)
for list_info in parsed:
# Do we even serve that list?
if list_info.name not in _setting(request, 'shavar', 'lists_served',
tuple()):
logger.warn('Unknown list "%s" reported; ignoring'
% list_info.name)
annotate_request(request, "shavar.downloads.unknown.list", 1)
continue
provider, type_, format_ = list_info.name.split('-', 2)
if not provider or not type_ or not format_:
s = 'Unknown list format for "%s"; ignoring' % list_info.name
logger.error(s)
annotate_request(request, "shavar.downloads.unknown.format", 1)
raise HTTPBadRequest(s)
sblist = get_list(request, list_info.name)
# Calculate delta
to_add, to_sub = sblist.delta(list_info.adds, list_info.subs)
# No delta? No response, I think. Spec doesn't actually say.
if not to_add and not to_sub:
continue
# Fetch the appropriate chunks
resp_payload['lists'][list_info.name] = sblist.fetch(to_add, to_sub)
# Not publishing deltas for this list? Delete all previous chunks to
# make way for the new corpus
if _setting(request, list_info.name, 'not_publishing_deltas'):
# Raise hell if we have suspicious data with this flag set
def test_0_get_list(self):
dumdum = dummy(body='4:4\n%s' % self.hg[:4], path='/gethash')
sblist = get_list(dumdum, 'mozpub-track-digest256')
self.assertIsInstance(sblist, Digest256)
def downloads_view(request):
# Use the new config variable name but support the old one for
default_interval = _setting(request, 'shavar', 'default_interval', None)
backoff_delay = _setting(request, 'shavar', 'client_backoff_delay', None)
# Throw a fit if both are specified
if default_interval is not None and backoff_delay is not None:
raise ConfigurationError("Specify either default_interval or "
"client_backoff_delay in the [shavar] "
"section of your config but not both.\n"
"client_backoff_delay is preferred.")
delay = backoff_delay or default_interval or 30 * 60
resp_payload = {'interval': delay, 'lists': {}}
try:
parsed = parse_downloads(request)
except ParseError as e:
capture_exception(e)
raise HTTPBadRequest(e)
for list_info in parsed:
# Do we even serve that list?
if list_info.name not in _setting(request, 'shavar',
'list_names_served', tuple()):
logger.warn('Unknown list "%s" reported; ignoring' %
list_info.name)
annotate_request(request, "shavar.downloads.unknown.list", 1)
continue
provider, type_, format_ = list_info.name.split('-', 2)
if not provider or not type_ or not format_:
s = 'Unknown list format for "%s"; ignoring' % list_info.name
logger.error(s)
annotate_request(request, "shavar.downloads.unknown.format", 1)
raise HTTPBadRequest(s)
app_ver = str(request.GET['appver'])
sblist, list_ver = get_list(request, list_info.name, app_ver)
# Calculate delta
to_add, to_sub = sblist.delta(list_info.adds, list_info.subs)
# No delta? No response, I think. Spec doesn't actually say.
if not to_add and not to_sub:
continue
# Fetch the appropriate chunks
resp_payload['lists'][list_info.name] = {
'sblist': sblist,
'ldata': sblist.fetch(to_add, to_sub),
'list_ver': list_ver
}
# Not publishing deltas for this list? Delete all previous chunks to
# make way for the new corpus
# if _setting(request, list_info.name, 'not_publishing_deltas'):
if sblist.settings.get('not_publishing_deltas'):
# Raise hell if we have suspicious data with this flag set
if (len(to_add) != 1 or len(to_sub) != 0):
logger.error("Configuration error! Mismatch between "
"{0}'s configuration has "
"'not_publishing_deltas' enabled but its data"
"file has more than one chunk to serve.".format(
list_info.name))
raise HTTPInternalServerError()
resp_payload['lists'][list_info.name]['adddels'] = list_info.adds
return HTTPOk(content_type="application/octet-stream",
body=format_downloads(request, resp_payload))
if list_info.name not in _setting(
request, 'shavar', 'list_names_served', tuple()
):
logger.warn('Unknown list "%s" reported; ignoring'
% list_info.name)
annotate_request(request, "shavar.downloads.unknown.list", 1)
continue
provider, type_, format_ = list_info.name.split('-', 2)
if not provider or not type_ or not format_:
s = 'Unknown list format for "%s"; ignoring' % list_info.name
logger.error(s)
annotate_request(request, "shavar.downloads.unknown.format", 1)
raise HTTPBadRequest(s)
app_ver = str(request.GET['appver'])
sblist, list_ver = get_list(request, list_info.name, app_ver)
# Calculate delta
to_add, to_sub = sblist.delta(list_info.adds, list_info.subs)
# No delta? No response, I think. Spec doesn't actually say.
if not to_add and not to_sub:
continue
# Fetch the appropriate chunks
resp_payload['lists'][list_info.name] = {
'sblist': sblist,
'ldata': sblist.fetch(to_add, to_sub),
'list_ver': list_ver
}
for list_info in parsed:
# Do we even serve that list?
if list_info.name not in _setting(request, 'shavar',
'list_names_served', tuple()):
logger.warn('Unknown list "%s" reported; ignoring' %
list_info.name)
annotate_request(request, "shavar.downloads.unknown.list", 1)
continue
provider, type_, format_ = list_info.name.split('-', 2)
if not provider or not type_ or not format_:
s = 'Unknown list format for "%s"; ignoring' % list_info.name
logger.error(s)
annotate_request(request, "shavar.downloads.unknown.format", 1)
raise HTTPBadRequest(s)
sblist = get_list(request, list_info.name)
# Calculate delta
to_add, to_sub = sblist.delta(list_info.adds, list_info.subs)
# No delta? No response, I think. Spec doesn't actually say.
if not to_add and not to_sub:
continue
# Fetch the appropriate chunks
resp_payload['lists'][list_info.name] = {
'sblist': sblist,
'ldata': sblist.fetch(to_add, to_sub)
}
# Not publishing deltas for this list? Delete all previous chunks to
