def downloads_view(request):
# Use the new config variable name but support the old one for
default_interval = _setting(request, 'shavar', 'default_interval', None)
backoff_delay = _setting(request, 'shavar', 'client_backoff_delay', None)
# Throw a fit if both are specified
if default_interval is not None and backoff_delay is not None:
raise ConfigurationError("Specify either default_interval or "
"client_backoff_delay in the [shavar] "
"section of your config but not both.\n"
"client_backoff_delay is preferred.")
delay = backoff_delay or default_interval or 30 * 60
resp_payload = {'interval': delay, 'lists': {}}
try:
parsed = parse_downloads(request)
except ParseError, e:
logger.error(e)
raise HTTPBadRequest(e)
def downloads_view(request):
# Use the new config variable name but support the old one for
default_interval = _setting(request, 'shavar', 'default_interval', None)
backoff_delay = _setting(request, 'shavar', 'client_backoff_delay', None)
# Throw a fit if both are specified
if default_interval is not None and backoff_delay is not None:
raise ConfigurationError("Specify either default_interval or "
"client_backoff_delay in the [shavar] "
"section of your config but not both.\n"
"client_backoff_delay is preferred.")
delay = backoff_delay or default_interval or 30 * 60
resp_payload = {'interval': delay, 'lists': {}}
try:
parsed = parse_downloads(request)
except ParseError, e:
logger.error(e)
raise HTTPBadRequest(e)
def downloads_view(request):
# Use the new config variable name but support the old one for
default_interval = _setting(request, 'shavar', 'default_interval', None)
backoff_delay = _setting(request, 'shavar', 'client_backoff_delay', None)
# Throw a fit if both are specified
if default_interval is not None and backoff_delay is not None:
raise ConfigurationError("Specify either default_interval or "
"client_backoff_delay in the [shavar] "
"section of your config but not both.\n"
"client_backoff_delay is preferred.")
delay = backoff_delay or default_interval or 30 * 60
resp_payload = {'interval': delay, 'lists': {}}
try:
parsed = parse_downloads(request)
except ParseError as e:
capture_exception(e)
raise HTTPBadRequest(e)
for list_info in parsed:
# Do we even serve that list?
if list_info.name not in _setting(request, 'shavar',
'list_names_served', tuple()):
logger.warn('Unknown list "%s" reported; ignoring' %
list_info.name)
annotate_request(request, "shavar.downloads.unknown.list", 1)
continue
provider, type_, format_ = list_info.name.split('-', 2)
if not provider or not type_ or not format_:
s = 'Unknown list format for "%s"; ignoring' % list_info.name
logger.error(s)
annotate_request(request, "shavar.downloads.unknown.format", 1)
raise HTTPBadRequest(s)
app_ver = str(request.GET['appver'])
sblist, list_ver = get_list(request, list_info.name, app_ver)
# Calculate delta
to_add, to_sub = sblist.delta(list_info.adds, list_info.subs)
# No delta? No response, I think. Spec doesn't actually say.
if not to_add and not to_sub:
continue
# Fetch the appropriate chunks
resp_payload['lists'][list_info.name] = {
'sblist': sblist,
'ldata': sblist.fetch(to_add, to_sub),
'list_ver': list_ver
}
# Not publishing deltas for this list? Delete all previous chunks to
# make way for the new corpus
# if _setting(request, list_info.name, 'not_publishing_deltas'):
if sblist.settings.get('not_publishing_deltas'):
# Raise hell if we have suspicious data with this flag set
if (len(to_add) != 1 or len(to_sub) != 0):
logger.error("Configuration error! Mismatch between "
"{0}'s configuration has "
"'not_publishing_deltas' enabled but its data"
"file has more than one chunk to serve.".format(
list_info.name))
raise HTTPInternalServerError()
resp_payload['lists'][list_info.name]['adddels'] = list_info.adds
return HTTPOk(content_type="application/octet-stream",
body=format_downloads(request, resp_payload))
def test_parse_download(self):
"""
Test bodies taken from
https://developers.google.com/safe-browsing/developers_guide_v2
"""
# empty list
p = parse_downloads(dummy("acme-malware-shavar;"))
d = Downloads()
d.append(DownloadsListInfo("acme-malware-shavar"))
self.assertEqual(p, d)
# empty list w/ MAC
p = parse_downloads(dummy("acme-malware-shavar;mac"))
d = Downloads()
d.append(DownloadsListInfo("acme-malware-shavar", wants_mac=True))
self.assertEqual(p, d)
# with size
p = parse_downloads(dummy("s;200\nacme-malware-shavar;"))
d = Downloads(200)
d.append(DownloadsListInfo("acme-malware-shavar"))
self.assertEqual(p, d)
# with chunks
p = parse_downloads(dummy("googpub-phish-shavar;a:1,2,3,4,5"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 1, 5)
self.assertEqual(p, d)
# chunks w/ MAC
p = parse_downloads(dummy("googpub-phish-shavar;a:1,2,3:mac"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar", wants_mac=True)
d.append(dli)
dli.add_range_claim('a', 1, 3)
self.assertEqual(p, d)
# chunks w/ ranges
p = parse_downloads(dummy("googpub-phish-shavar;a:1-5,10,12"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 1, 5)
dli.add_claim('a', 10)
dli.add_claim('a', 12)
self.assertEqual(p, d)
# with add & subtract chunks
p = parse_downloads(dummy("googpub-phish-shavar;a:1-5,10:s:3-8"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 1, 5)
dli.add_claim('a', 10)
dli.add_range_claim('s', 3, 8)
self.assertEqual(p, d)
# with add & subtract chunks out of order
p = parse_downloads(dummy("googpub-phish-shavar;a:3-5,1,10"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 3, 5)
dli.add_claim('a', 1)
dli.add_claim('a', 10)
self.assertEqual(p, d)
# with multiple lists
s = "googpub-phish-shavar;a:1-3,5:s:4-5\n"
s += "acme-white-shavar;a:1-7:s:1-2"
p = parse_downloads(dummy(s))
d = Downloads()
dli0 = DownloadsListInfo("googpub-phish-shavar")
d.append(dli0)
dli0.add_range_claim('a', 1, 3)
dli0.add_claim('a', 5)
dli0.add_range_claim('s', 4, 5)
dli1 = DownloadsListInfo("acme-white-shavar")
d.append(dli1)
dli1.add_range_claim('a', 1, 7)
dli1.add_range_claim('s', 1, 2)
self.assertEqual(p, d)
# with multiple lists, at least one empty
# See https://github.com/mozilla-services/shavar/issues/56
s = "googpub-phish-shavar;\n"
s += "acme-white-shavar;a:1-7:s:1-2"
p = parse_downloads(dummy(s))
d = Downloads()
dli0 = DownloadsListInfo("googpub-phish-shavar")
d.append(dli0)
dli1 = DownloadsListInfo("acme-white-shavar")
d.append(dli1)
dli1.add_range_claim('a', 1, 7)
dli1.add_range_claim('s', 1, 2)
self.assertEqual(p, d)
def test_parse_download(self):
"""
Test bodies taken from
https://developers.google.com/safe-browsing/developers_guide_v2
"""
# empty list
p = parse_downloads(dummy("acme-malware-shavar;"))
d = Downloads()
d.append(DownloadsListInfo("acme-malware-shavar"))
self.assertEqual(p, d)
# empty list w/ MAC
p = parse_downloads(dummy("acme-malware-shavar;mac"))
d = Downloads()
d.append(DownloadsListInfo("acme-malware-shavar", wants_mac=True))
self.assertEqual(p, d)
# with size
p = parse_downloads(dummy("s;200\nacme-malware-shavar;"))
d = Downloads(200)
d.append(DownloadsListInfo("acme-malware-shavar"))
self.assertEqual(p, d)
# with chunks
p = parse_downloads(dummy("googpub-phish-shavar;a:1,2,3,4,5"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 1, 5)
self.assertEqual(p, d)
# chunks w/ MAC
p = parse_downloads(dummy("googpub-phish-shavar;a:1,2,3:mac"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar", wants_mac=True)
d.append(dli)
dli.add_range_claim('a', 1, 3)
self.assertEqual(p, d)
# chunks w/ ranges
p = parse_downloads(dummy("googpub-phish-shavar;a:1-5,10,12"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 1, 5)
dli.add_claim('a', 10)
dli.add_claim('a', 12)
self.assertEqual(p, d)
# with add & subtract chunks
p = parse_downloads(dummy("googpub-phish-shavar;a:1-5,10:s:3-8"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 1, 5)
dli.add_claim('a', 10)
dli.add_range_claim('s', 3, 8)
self.assertEqual(p, d)
# with add & subtract chunks out of order
p = parse_downloads(dummy("googpub-phish-shavar;a:3-5,1,10"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 3, 5)
dli.add_claim('a', 1)
dli.add_claim('a', 10)
self.assertEqual(p, d)
# with multiple lists
s = "googpub-phish-shavar;a:1-3,5:s:4-5\n"
s += "acme-white-shavar;a:1-7:s:1-2"
p = parse_downloads(dummy(s))
d = Downloads()
dli0 = DownloadsListInfo("googpub-phish-shavar")
d.append(dli0)
dli0.add_range_claim('a', 1, 3)
dli0.add_claim('a', 5)
dli0.add_range_claim('s', 4, 5)
dli1 = DownloadsListInfo("acme-white-shavar")
d.append(dli1)
dli1.add_range_claim('a', 1, 7)
dli1.add_range_claim('s', 1, 2)
self.assertEqual(p, d)
# with multiple lists, at least one empty
# See https://github.com/mozilla-services/shavar/issues/56
s = "googpub-phish-shavar;\n"
s += "acme-white-shavar;a:1-7:s:1-2"
p = parse_downloads(dummy(s))
d = Downloads()
dli0 = DownloadsListInfo("googpub-phish-shavar")
d.append(dli0)
dli1 = DownloadsListInfo("acme-white-shavar")
d.append(dli1)
dli1.add_range_claim('a', 1, 7)
dli1.add_range_claim('s', 1, 2)
self.assertEqual(p, d)
