def downloads_view(request): # Use the new config variable name but support the old one for default_interval = _setting(request, 'shavar', 'default_interval', None) backoff_delay = _setting(request, 'shavar', 'client_backoff_delay', None) # Throw a fit if both are specified if default_interval is not None and backoff_delay is not None: raise ConfigurationError("Specify either default_interval or " "client_backoff_delay in the [shavar] " "section of your config but not both.\n" "client_backoff_delay is preferred.") delay = backoff_delay or default_interval or 30 * 60 resp_payload = {'interval': delay, 'lists': {}} try: parsed = parse_downloads(request) except ParseError, e: logger.error(e) raise HTTPBadRequest(e)
def downloads_view(request): # Use the new config variable name but support the old one for default_interval = _setting(request, 'shavar', 'default_interval', None) backoff_delay = _setting(request, 'shavar', 'client_backoff_delay', None) # Throw a fit if both are specified if default_interval is not None and backoff_delay is not None: raise ConfigurationError("Specify either default_interval or " "client_backoff_delay in the [shavar] " "section of your config but not both.\n" "client_backoff_delay is preferred.") delay = backoff_delay or default_interval or 30 * 60 resp_payload = {'interval': delay, 'lists': {}} try: parsed = parse_downloads(request) except ParseError as e: capture_exception(e) raise HTTPBadRequest(e) for list_info in parsed: # Do we even serve that list? if list_info.name not in _setting(request, 'shavar', 'list_names_served', tuple()): logger.warn('Unknown list "%s" reported; ignoring' % list_info.name) annotate_request(request, "shavar.downloads.unknown.list", 1) continue provider, type_, format_ = list_info.name.split('-', 2) if not provider or not type_ or not format_: s = 'Unknown list format for "%s"; ignoring' % list_info.name logger.error(s) annotate_request(request, "shavar.downloads.unknown.format", 1) raise HTTPBadRequest(s) app_ver = str(request.GET['appver']) sblist, list_ver = get_list(request, list_info.name, app_ver) # Calculate delta to_add, to_sub = sblist.delta(list_info.adds, list_info.subs) # No delta? No response, I think. Spec doesn't actually say. if not to_add and not to_sub: continue # Fetch the appropriate chunks resp_payload['lists'][list_info.name] = { 'sblist': sblist, 'ldata': sblist.fetch(to_add, to_sub), 'list_ver': list_ver } # Not publishing deltas for this list? Delete all previous chunks to # make way for the new corpus # if _setting(request, list_info.name, 'not_publishing_deltas'): if sblist.settings.get('not_publishing_deltas'): # Raise hell if we have suspicious data with this flag set if (len(to_add) != 1 or len(to_sub) != 0): logger.error("Configuration error! Mismatch between " "{0}'s configuration has " "'not_publishing_deltas' enabled but its data" "file has more than one chunk to serve.".format( list_info.name)) raise HTTPInternalServerError() resp_payload['lists'][list_info.name]['adddels'] = list_info.adds return HTTPOk(content_type="application/octet-stream", body=format_downloads(request, resp_payload))
def test_parse_download(self): """ Test bodies taken from https://developers.google.com/safe-browsing/developers_guide_v2 """ # empty list p = parse_downloads(dummy("acme-malware-shavar;")) d = Downloads() d.append(DownloadsListInfo("acme-malware-shavar")) self.assertEqual(p, d) # empty list w/ MAC p = parse_downloads(dummy("acme-malware-shavar;mac")) d = Downloads() d.append(DownloadsListInfo("acme-malware-shavar", wants_mac=True)) self.assertEqual(p, d) # with size p = parse_downloads(dummy("s;200\nacme-malware-shavar;")) d = Downloads(200) d.append(DownloadsListInfo("acme-malware-shavar")) self.assertEqual(p, d) # with chunks p = parse_downloads(dummy("googpub-phish-shavar;a:1,2,3,4,5")) d = Downloads() dli = DownloadsListInfo("googpub-phish-shavar") d.append(dli) dli.add_range_claim('a', 1, 5) self.assertEqual(p, d) # chunks w/ MAC p = parse_downloads(dummy("googpub-phish-shavar;a:1,2,3:mac")) d = Downloads() dli = DownloadsListInfo("googpub-phish-shavar", wants_mac=True) d.append(dli) dli.add_range_claim('a', 1, 3) self.assertEqual(p, d) # chunks w/ ranges p = parse_downloads(dummy("googpub-phish-shavar;a:1-5,10,12")) d = Downloads() dli = DownloadsListInfo("googpub-phish-shavar") d.append(dli) dli.add_range_claim('a', 1, 5) dli.add_claim('a', 10) dli.add_claim('a', 12) self.assertEqual(p, d) # with add & subtract chunks p = parse_downloads(dummy("googpub-phish-shavar;a:1-5,10:s:3-8")) d = Downloads() dli = DownloadsListInfo("googpub-phish-shavar") d.append(dli) dli.add_range_claim('a', 1, 5) dli.add_claim('a', 10) dli.add_range_claim('s', 3, 8) self.assertEqual(p, d) # with add & subtract chunks out of order p = parse_downloads(dummy("googpub-phish-shavar;a:3-5,1,10")) d = Downloads() dli = DownloadsListInfo("googpub-phish-shavar") d.append(dli) dli.add_range_claim('a', 3, 5) dli.add_claim('a', 1) dli.add_claim('a', 10) self.assertEqual(p, d) # with multiple lists s = "googpub-phish-shavar;a:1-3,5:s:4-5\n" s += "acme-white-shavar;a:1-7:s:1-2" p = parse_downloads(dummy(s)) d = Downloads() dli0 = DownloadsListInfo("googpub-phish-shavar") d.append(dli0) dli0.add_range_claim('a', 1, 3) dli0.add_claim('a', 5) dli0.add_range_claim('s', 4, 5) dli1 = DownloadsListInfo("acme-white-shavar") d.append(dli1) dli1.add_range_claim('a', 1, 7) dli1.add_range_claim('s', 1, 2) self.assertEqual(p, d) # with multiple lists, at least one empty # See https://github.com/mozilla-services/shavar/issues/56 s = "googpub-phish-shavar;\n" s += "acme-white-shavar;a:1-7:s:1-2" p = parse_downloads(dummy(s)) d = Downloads() dli0 = DownloadsListInfo("googpub-phish-shavar") d.append(dli0) dli1 = DownloadsListInfo("acme-white-shavar") d.append(dli1) dli1.add_range_claim('a', 1, 7) dli1.add_range_claim('s', 1, 2) self.assertEqual(p, d)