def parse_downloads(request):
parsed = Downloads()
limit = request.registry.settings.get("shavar.max_downloads_chunks",
10000)
for lineno, line in enumerate(request.body_file):
line = line.strip()
if not line or line.isspace():
continue
# Did client provide max size preference?
if line.startswith("s;"):
if lineno != 0:
return ParseError("Size request can only be the first line!")
req_size = line.split(";", 1)[1]
# Almost certainly redundant due to stripping the line above
req_size = req_size.strip()
try:
req_size = int(req_size)
except ValueError:
raise ParseError("Invalid requested size")
parsed.req_size = req_size
continue
if ";" not in line:
return ParseError("Bad downloads request: no semi-colon")
lname, chunklist = line.split(";", 1)
if not lname or '-' not in lname:
raise ParseError("Invalid list name: \"%s\"" % lname)
info = DownloadsListInfo(lname, limit=limit)
chunks = chunklist.split(":")
# Check for MAC
if len(chunks) >= 1 and chunks[-1] == "mac":
if request.GET.get('pver') == '3.0':
raise ParseError('MAC not supported in protocol version 3')
info.wants_mac = True
chunks.pop(-1)
# Client claims to have chunks for this list
if not chunks or (len(chunks) == 1 and not chunks[0]):
parsed.append(info)
continue
# Uneven number of chunks should only occur if 'mac' was specified
if len(chunks) % 2 != 0:
raise ParseError("Invalid LISTINFO for %s" % lname)
while chunks:
ctype = chunks.pop(0)
if ctype not in ('a', 's'):
raise ParseError("Invalid CHUNKTYPE \"%s\" for %s" % (ctype,
lname))
list_of_chunks = chunks.pop(0)
for chunk in list_of_chunks.split(','):
try:
chunk = int(chunk)
except ValueError:
if chunk.find('-'):
low, high = chunk.split('-', 1)
# FIXME should probably be stricter about testing for
# pure integers only on the input
try:
low = int(low)
high = int(high)
except ValueError:
raise ParseError("Invalid RANGE \"%s\" for %s" %
(chunk, lname))
if low >= high:
raise ParseError("Invalid RANGE \"%s\" for %s" %
(chunk, lname))
info.add_range_claim(ctype, low, high)
else: # Resist temptation to indent! It's a try/except/else
info.add_claim(ctype, chunk)
parsed.append(info)
return parsed
def test_parse_download(self):
"""
Test bodies taken from
https://developers.google.com/safe-browsing/developers_guide_v2
"""
# empty list
p = parse_downloads(dummy("acme-malware-shavar;"))
d = Downloads()
d.append(DownloadsListInfo("acme-malware-shavar"))
self.assertEqual(p, d)
# empty list w/ MAC
p = parse_downloads(dummy("acme-malware-shavar;mac"))
d = Downloads()
d.append(DownloadsListInfo("acme-malware-shavar", wants_mac=True))
self.assertEqual(p, d)
# with size
p = parse_downloads(dummy("s;200\nacme-malware-shavar;"))
d = Downloads(200)
d.append(DownloadsListInfo("acme-malware-shavar"))
self.assertEqual(p, d)
# with chunks
p = parse_downloads(dummy("googpub-phish-shavar;a:1,2,3,4,5"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 1, 5)
self.assertEqual(p, d)
# chunks w/ MAC
p = parse_downloads(dummy("googpub-phish-shavar;a:1,2,3:mac"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar", wants_mac=True)
d.append(dli)
dli.add_range_claim('a', 1, 3)
self.assertEqual(p, d)
# chunks w/ ranges
p = parse_downloads(dummy("googpub-phish-shavar;a:1-5,10,12"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 1, 5)
dli.add_claim('a', 10)
dli.add_claim('a', 12)
self.assertEqual(p, d)
# with add & subtract chunks
p = parse_downloads(dummy("googpub-phish-shavar;a:1-5,10:s:3-8"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 1, 5)
dli.add_claim('a', 10)
dli.add_range_claim('s', 3, 8)
self.assertEqual(p, d)
# with add & subtract chunks out of order
p = parse_downloads(dummy("googpub-phish-shavar;a:3-5,1,10"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 3, 5)
dli.add_claim('a', 1)
dli.add_claim('a', 10)
self.assertEqual(p, d)
# with multiple lists
s = "googpub-phish-shavar;a:1-3,5:s:4-5\n"
s += "acme-white-shavar;a:1-7:s:1-2"
p = parse_downloads(dummy(s))
d = Downloads()
dli0 = DownloadsListInfo("googpub-phish-shavar")
d.append(dli0)
dli0.add_range_claim('a', 1, 3)
dli0.add_claim('a', 5)
dli0.add_range_claim('s', 4, 5)
dli1 = DownloadsListInfo("acme-white-shavar")
d.append(dli1)
dli1.add_range_claim('a', 1, 7)
dli1.add_range_claim('s', 1, 2)
self.assertEqual(p, d)
# with multiple lists, at least one empty
# See https://github.com/mozilla-services/shavar/issues/56
s = "googpub-phish-shavar;\n"
s += "acme-white-shavar;a:1-7:s:1-2"
p = parse_downloads(dummy(s))
d = Downloads()
dli0 = DownloadsListInfo("googpub-phish-shavar")
d.append(dli0)
dli1 = DownloadsListInfo("acme-white-shavar")
d.append(dli1)
dli1.add_range_claim('a', 1, 7)
dli1.add_range_claim('s', 1, 2)
self.assertEqual(p, d)
def parse_downloads(request):
parsed = Downloads()
limit = request.registry.settings.get("shavar.max_downloads_chunks", 10000)
for lineno, line in enumerate(request.body_file):
line = line.strip()
if not line or line.isspace():
continue
line = line.decode()
# Did client provide max size preference?
if line.startswith("s;"):
if lineno != 0:
return ParseError("Size request can only be the first line!")
req_size = line.split(";", 1)[1]
# Almost certainly redundant due to stripping the line above
req_size = req_size.strip()
try:
req_size = int(req_size)
except ValueError:
raise ParseError("Invalid requested size")
parsed.req_size = req_size
continue
if ";" not in line:
raise ParseError("Bad downloads request: no semi-colon")
lname, chunklist = line.split(";", 1)
if not lname or '-' not in lname:
raise ParseError("Invalid list name: \"%s\"" % lname)
info = DownloadsListInfo(lname, limit=limit)
chunks = chunklist.split(":")
# Check for MAC
if len(chunks) >= 1 and chunks[-1] == "mac":
if request.GET.get('pver') == '3.0':
raise ParseError('MAC not supported in protocol version 3')
info.wants_mac = True
chunks.pop(-1)
# Client claims to have chunks for this list
if not chunks or (len(chunks) == 1 and not chunks[0]):
parsed.append(info)
continue
# Uneven number of chunks should only occur if 'mac' was specified
if len(chunks) % 2 != 0:
raise ParseError("Invalid LISTINFO for %s" % lname)
while chunks:
ctype = chunks.pop(0)
if ctype not in ('a', 's'):
raise ParseError("Invalid CHUNKTYPE \"%s\" for %s" %
(ctype, lname))
list_of_chunks = chunks.pop(0)
for chunk in list_of_chunks.split(','):
try:
chunk = int(chunk)
except ValueError:
if chunk.find('-'):
low, high = chunk.split('-', 1)
# FIXME should probably be stricter about testing for
# pure integers only on the input
try:
low = int(low)
high = int(high)
except ValueError:
raise ParseError("Invalid RANGE \"%s\" for %s" %
(chunk, lname))
if low >= high:
raise ParseError("Invalid RANGE \"%s\" for %s" %
(chunk, lname))
info.add_range_claim(ctype, low, high)
else: # Resist temptation to indent! It's a try/except/else
info.add_claim(ctype, chunk)
parsed.append(info)
return parsed
def test_parse_download(self):
"""
Test bodies taken from
https://developers.google.com/safe-browsing/developers_guide_v2
"""
# empty list
p = parse_downloads(dummy("acme-malware-shavar;"))
d = Downloads()
d.append(DownloadsListInfo("acme-malware-shavar"))
self.assertEqual(p, d)
# empty list w/ MAC
p = parse_downloads(dummy("acme-malware-shavar;mac"))
d = Downloads()
d.append(DownloadsListInfo("acme-malware-shavar", wants_mac=True))
self.assertEqual(p, d)
# with size
p = parse_downloads(dummy("s;200\nacme-malware-shavar;"))
d = Downloads(200)
d.append(DownloadsListInfo("acme-malware-shavar"))
self.assertEqual(p, d)
# with chunks
p = parse_downloads(dummy("googpub-phish-shavar;a:1,2,3,4,5"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 1, 5)
self.assertEqual(p, d)
# chunks w/ MAC
p = parse_downloads(dummy("googpub-phish-shavar;a:1,2,3:mac"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar", wants_mac=True)
d.append(dli)
dli.add_range_claim('a', 1, 3)
self.assertEqual(p, d)
# chunks w/ ranges
p = parse_downloads(dummy("googpub-phish-shavar;a:1-5,10,12"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 1, 5)
dli.add_claim('a', 10)
dli.add_claim('a', 12)
self.assertEqual(p, d)
# with add & subtract chunks
p = parse_downloads(dummy("googpub-phish-shavar;a:1-5,10:s:3-8"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 1, 5)
dli.add_claim('a', 10)
dli.add_range_claim('s', 3, 8)
self.assertEqual(p, d)
# with add & subtract chunks out of order
p = parse_downloads(dummy("googpub-phish-shavar;a:3-5,1,10"))
d = Downloads()
dli = DownloadsListInfo("googpub-phish-shavar")
d.append(dli)
dli.add_range_claim('a', 3, 5)
dli.add_claim('a', 1)
dli.add_claim('a', 10)
self.assertEqual(p, d)
# with multiple lists
s = "googpub-phish-shavar;a:1-3,5:s:4-5\n"
s += "acme-white-shavar;a:1-7:s:1-2"
p = parse_downloads(dummy(s))
d = Downloads()
dli0 = DownloadsListInfo("googpub-phish-shavar")
d.append(dli0)
dli0.add_range_claim('a', 1, 3)
dli0.add_claim('a', 5)
dli0.add_range_claim('s', 4, 5)
dli1 = DownloadsListInfo("acme-white-shavar")
d.append(dli1)
dli1.add_range_claim('a', 1, 7)
dli1.add_range_claim('s', 1, 2)
self.assertEqual(p, d)
# with multiple lists, at least one empty
# See https://github.com/mozilla-services/shavar/issues/56
s = "googpub-phish-shavar;\n"
s += "acme-white-shavar;a:1-7:s:1-2"
p = parse_downloads(dummy(s))
d = Downloads()
dli0 = DownloadsListInfo("googpub-phish-shavar")
d.append(dli0)
dli1 = DownloadsListInfo("acme-white-shavar")
d.append(dli1)
dli1.add_range_claim('a', 1, 7)
dli1.add_range_claim('s', 1, 2)
self.assertEqual(p, d)