Example #1
0
 def chunks(self):
     "Generator iterating through the server respones chunk by chunk"
     log.debug("accessing chunks")
     for list_name, chunk_urls in list(self.lists_data.items()):
         for chunk_url in chunk_urls:
             log.debug("processing chunk url: %s", util.format_max_len(chunk_url, max_len=45))
             packed_chunks = self._fetch_chunks(chunk_url)
             for chunk_data in self._unpack_chunks(packed_chunks):
                 # log.debug("chunk_data: {data}".format(data=chunk_data))
                 chunk = Chunk(chunk_data, list_name)
                 # log.debug("yielding {chunk}".format(chunk=chunk))
                 yield chunk
Example #2
0
 def api_call(self, url, payload=None):
     "Perform a call to Safe Browsing API"
     log.debug("performing api call to %s", util.format_max_len(url, 35))
     if payload is None:
         payload = b''
     if isinstance(payload, str):
         payload = bytes(payload.encode("ascii"))
     request = urllib.request.Request(url, data=BytesIO(payload),
                                      headers={'Content-Length': len(payload)})
     try:
         response = urllib.request.urlopen(request)
     except urllib.error.HTTPError:
         self._error_count += 1
         raise
     self._error_count = 0
     return response.read()
Example #3
0
 def test_format_max_len(self):
     eq_(util.format_max_len('abcdefghijklmnopqrstuvwxyz'), 'abcde[...]vwxyz')
     eq_(util.format_max_len('abcdefghijklmnop'), 'abcde[...]lmnop')
     eq_(util.format_max_len('abcdefghijklmno'), 'abcdefghijklmno')
     eq_(util.format_max_len('a'), 'a')
Example #4
0
 def _fetch_chunks(self, url):
     "Download chunks of data containing hash prefixes"
     log.debug("fetching chunk %s", util.format_max_len(url, max_len=45))
     response = urllib.request.urlopen(url)
     return response