Beispiel #1
0
def parse_downloads(request):
    parsed = Downloads()

    limit = request.registry.settings.get("shavar.max_downloads_chunks",
                                          10000)

    for lineno, line in enumerate(request.body_file):
        line = line.strip()

        if not line or line.isspace():
            continue

        # Did client provide max size preference?
        if line.startswith("s;"):
            if lineno != 0:
                return ParseError("Size request can only be the first line!")
            req_size = line.split(";", 1)[1]
            # Almost certainly redundant due to stripping the line above
            req_size = req_size.strip()
            try:
                req_size = int(req_size)
            except ValueError:
                raise ParseError("Invalid requested size")
            parsed.req_size = req_size
            continue

        if ";" not in line:
            return ParseError("Bad downloads request: no semi-colon")

        lname, chunklist = line.split(";", 1)
        if not lname or '-' not in lname:
            raise ParseError("Invalid list name: \"%s\"" % lname)
        info = DownloadsListInfo(lname, limit=limit)

        chunks = chunklist.split(":")
        # Check for MAC
        if len(chunks) >= 1 and chunks[-1] == "mac":
            if request.GET.get('pver') == '3.0':
                raise ParseError('MAC not supported in protocol version 3')
            info.wants_mac = True
            chunks.pop(-1)
        # Client claims to have chunks for this list
        if not chunks or (len(chunks) == 1 and not chunks[0]):
            parsed.append(info)
            continue
        # Uneven number of chunks should only occur if 'mac' was specified
        if len(chunks) % 2 != 0:
            raise ParseError("Invalid LISTINFO for %s" % lname)

        while chunks:
            ctype = chunks.pop(0)
            if ctype not in ('a', 's'):
                raise ParseError("Invalid CHUNKTYPE \"%s\" for %s" % (ctype,
                                                                      lname))

            list_of_chunks = chunks.pop(0)
            for chunk in list_of_chunks.split(','):
                try:
                    chunk = int(chunk)
                except ValueError:
                    if chunk.find('-'):
                        low, high = chunk.split('-', 1)
                        # FIXME should probably be stricter about testing for
                        #       pure integers only on the input
                        try:
                            low = int(low)
                            high = int(high)
                        except ValueError:
                            raise ParseError("Invalid RANGE \"%s\" for %s" %
                                             (chunk, lname))
                        if low >= high:
                            raise ParseError("Invalid RANGE \"%s\" for %s" %
                                             (chunk, lname))

                        info.add_range_claim(ctype, low, high)
                else:  # Resist temptation to indent!  It's a try/except/else
                    info.add_claim(ctype, chunk)
        parsed.append(info)
    return parsed
Beispiel #2
0
    def test_parse_download(self):
        """
        Test bodies taken from
        https://developers.google.com/safe-browsing/developers_guide_v2
        """
        # empty list
        p = parse_downloads(dummy("acme-malware-shavar;"))
        d = Downloads()
        d.append(DownloadsListInfo("acme-malware-shavar"))
        self.assertEqual(p, d)

        # empty list w/ MAC
        p = parse_downloads(dummy("acme-malware-shavar;mac"))
        d = Downloads()
        d.append(DownloadsListInfo("acme-malware-shavar", wants_mac=True))
        self.assertEqual(p, d)

        # with size
        p = parse_downloads(dummy("s;200\nacme-malware-shavar;"))
        d = Downloads(200)
        d.append(DownloadsListInfo("acme-malware-shavar"))
        self.assertEqual(p, d)

        # with chunks
        p = parse_downloads(dummy("googpub-phish-shavar;a:1,2,3,4,5"))
        d = Downloads()
        dli = DownloadsListInfo("googpub-phish-shavar")
        d.append(dli)
        dli.add_range_claim('a', 1, 5)
        self.assertEqual(p, d)

        # chunks w/ MAC
        p = parse_downloads(dummy("googpub-phish-shavar;a:1,2,3:mac"))
        d = Downloads()
        dli = DownloadsListInfo("googpub-phish-shavar", wants_mac=True)
        d.append(dli)
        dli.add_range_claim('a', 1, 3)
        self.assertEqual(p, d)

        # chunks w/ ranges
        p = parse_downloads(dummy("googpub-phish-shavar;a:1-5,10,12"))
        d = Downloads()
        dli = DownloadsListInfo("googpub-phish-shavar")
        d.append(dli)
        dli.add_range_claim('a', 1, 5)
        dli.add_claim('a', 10)
        dli.add_claim('a', 12)
        self.assertEqual(p, d)

        # with add & subtract chunks
        p = parse_downloads(dummy("googpub-phish-shavar;a:1-5,10:s:3-8"))
        d = Downloads()
        dli = DownloadsListInfo("googpub-phish-shavar")
        d.append(dli)
        dli.add_range_claim('a', 1, 5)
        dli.add_claim('a', 10)
        dli.add_range_claim('s', 3, 8)
        self.assertEqual(p, d)

        # with add & subtract chunks out of order
        p = parse_downloads(dummy("googpub-phish-shavar;a:3-5,1,10"))
        d = Downloads()
        dli = DownloadsListInfo("googpub-phish-shavar")
        d.append(dli)
        dli.add_range_claim('a', 3, 5)
        dli.add_claim('a', 1)
        dli.add_claim('a', 10)
        self.assertEqual(p, d)

        # with multiple lists
        s = "googpub-phish-shavar;a:1-3,5:s:4-5\n"
        s += "acme-white-shavar;a:1-7:s:1-2"
        p = parse_downloads(dummy(s))

        d = Downloads()
        dli0 = DownloadsListInfo("googpub-phish-shavar")
        d.append(dli0)
        dli0.add_range_claim('a', 1, 3)
        dli0.add_claim('a', 5)
        dli0.add_range_claim('s', 4, 5)

        dli1 = DownloadsListInfo("acme-white-shavar")
        d.append(dli1)
        dli1.add_range_claim('a', 1, 7)
        dli1.add_range_claim('s', 1, 2)
        self.assertEqual(p, d)

        # with multiple lists, at least one empty
        # See https://github.com/mozilla-services/shavar/issues/56
        s = "googpub-phish-shavar;\n"
        s += "acme-white-shavar;a:1-7:s:1-2"
        p = parse_downloads(dummy(s))

        d = Downloads()
        dli0 = DownloadsListInfo("googpub-phish-shavar")
        d.append(dli0)

        dli1 = DownloadsListInfo("acme-white-shavar")
        d.append(dli1)
        dli1.add_range_claim('a', 1, 7)
        dli1.add_range_claim('s', 1, 2)
        self.assertEqual(p, d)
Beispiel #3
0
def parse_downloads(request):
    parsed = Downloads()

    limit = request.registry.settings.get("shavar.max_downloads_chunks", 10000)

    for lineno, line in enumerate(request.body_file):
        line = line.strip()

        if not line or line.isspace():
            continue

        line = line.decode()
        # Did client provide max size preference?
        if line.startswith("s;"):
            if lineno != 0:
                return ParseError("Size request can only be the first line!")
            req_size = line.split(";", 1)[1]
            # Almost certainly redundant due to stripping the line above
            req_size = req_size.strip()
            try:
                req_size = int(req_size)
            except ValueError:
                raise ParseError("Invalid requested size")
            parsed.req_size = req_size
            continue

        if ";" not in line:
            raise ParseError("Bad downloads request: no semi-colon")

        lname, chunklist = line.split(";", 1)
        if not lname or '-' not in lname:
            raise ParseError("Invalid list name: \"%s\"" % lname)
        info = DownloadsListInfo(lname, limit=limit)

        chunks = chunklist.split(":")
        # Check for MAC
        if len(chunks) >= 1 and chunks[-1] == "mac":
            if request.GET.get('pver') == '3.0':
                raise ParseError('MAC not supported in protocol version 3')
            info.wants_mac = True
            chunks.pop(-1)
        # Client claims to have chunks for this list
        if not chunks or (len(chunks) == 1 and not chunks[0]):
            parsed.append(info)
            continue
        # Uneven number of chunks should only occur if 'mac' was specified
        if len(chunks) % 2 != 0:
            raise ParseError("Invalid LISTINFO for %s" % lname)

        while chunks:
            ctype = chunks.pop(0)
            if ctype not in ('a', 's'):
                raise ParseError("Invalid CHUNKTYPE \"%s\" for %s" %
                                 (ctype, lname))

            list_of_chunks = chunks.pop(0)
            for chunk in list_of_chunks.split(','):
                try:
                    chunk = int(chunk)
                except ValueError:
                    if chunk.find('-'):
                        low, high = chunk.split('-', 1)
                        # FIXME should probably be stricter about testing for
                        #       pure integers only on the input
                        try:
                            low = int(low)
                            high = int(high)
                        except ValueError:
                            raise ParseError("Invalid RANGE \"%s\" for %s" %
                                             (chunk, lname))
                        if low >= high:
                            raise ParseError("Invalid RANGE \"%s\" for %s" %
                                             (chunk, lname))

                        info.add_range_claim(ctype, low, high)
                else:  # Resist temptation to indent!  It's a try/except/else
                    info.add_claim(ctype, chunk)
        parsed.append(info)
    return parsed
Beispiel #4
0
    def test_parse_download(self):
        """
        Test bodies taken from
        https://developers.google.com/safe-browsing/developers_guide_v2
        """
        # empty list
        p = parse_downloads(dummy("acme-malware-shavar;"))
        d = Downloads()
        d.append(DownloadsListInfo("acme-malware-shavar"))
        self.assertEqual(p, d)

        # empty list w/ MAC
        p = parse_downloads(dummy("acme-malware-shavar;mac"))
        d = Downloads()
        d.append(DownloadsListInfo("acme-malware-shavar", wants_mac=True))
        self.assertEqual(p, d)

        # with size
        p = parse_downloads(dummy("s;200\nacme-malware-shavar;"))
        d = Downloads(200)
        d.append(DownloadsListInfo("acme-malware-shavar"))
        self.assertEqual(p, d)

        # with chunks
        p = parse_downloads(dummy("googpub-phish-shavar;a:1,2,3,4,5"))
        d = Downloads()
        dli = DownloadsListInfo("googpub-phish-shavar")
        d.append(dli)
        dli.add_range_claim('a', 1, 5)
        self.assertEqual(p, d)

        # chunks w/ MAC
        p = parse_downloads(dummy("googpub-phish-shavar;a:1,2,3:mac"))
        d = Downloads()
        dli = DownloadsListInfo("googpub-phish-shavar", wants_mac=True)
        d.append(dli)
        dli.add_range_claim('a', 1, 3)
        self.assertEqual(p, d)

        # chunks w/ ranges
        p = parse_downloads(dummy("googpub-phish-shavar;a:1-5,10,12"))
        d = Downloads()
        dli = DownloadsListInfo("googpub-phish-shavar")
        d.append(dli)
        dli.add_range_claim('a', 1, 5)
        dli.add_claim('a', 10)
        dli.add_claim('a', 12)
        self.assertEqual(p, d)

        # with add & subtract chunks
        p = parse_downloads(dummy("googpub-phish-shavar;a:1-5,10:s:3-8"))
        d = Downloads()
        dli = DownloadsListInfo("googpub-phish-shavar")
        d.append(dli)
        dli.add_range_claim('a', 1, 5)
        dli.add_claim('a', 10)
        dli.add_range_claim('s', 3, 8)
        self.assertEqual(p, d)

        # with add & subtract chunks out of order
        p = parse_downloads(dummy("googpub-phish-shavar;a:3-5,1,10"))
        d = Downloads()
        dli = DownloadsListInfo("googpub-phish-shavar")
        d.append(dli)
        dli.add_range_claim('a', 3, 5)
        dli.add_claim('a', 1)
        dli.add_claim('a', 10)
        self.assertEqual(p, d)

        # with multiple lists
        s = "googpub-phish-shavar;a:1-3,5:s:4-5\n"
        s += "acme-white-shavar;a:1-7:s:1-2"
        p = parse_downloads(dummy(s))

        d = Downloads()
        dli0 = DownloadsListInfo("googpub-phish-shavar")
        d.append(dli0)
        dli0.add_range_claim('a', 1, 3)
        dli0.add_claim('a', 5)
        dli0.add_range_claim('s', 4, 5)

        dli1 = DownloadsListInfo("acme-white-shavar")
        d.append(dli1)
        dli1.add_range_claim('a', 1, 7)
        dli1.add_range_claim('s', 1, 2)
        self.assertEqual(p, d)

        # with multiple lists, at least one empty
        # See https://github.com/mozilla-services/shavar/issues/56
        s = "googpub-phish-shavar;\n"
        s += "acme-white-shavar;a:1-7:s:1-2"
        p = parse_downloads(dummy(s))

        d = Downloads()
        dli0 = DownloadsListInfo("googpub-phish-shavar")
        d.append(dli0)

        dli1 = DownloadsListInfo("acme-white-shavar")
        d.append(dli1)
        dli1.add_range_claim('a', 1, 7)
        dli1.add_range_claim('s', 1, 2)
        self.assertEqual(p, d)