def chunks(self):
"Generator iterating through the server respones chunk by chunk"
log.debug("accessing chunks")
for list_name, chunk_urls in list(self.lists_data.items()):
for chunk_url in chunk_urls:
log.debug("processing chunk url: %s", util.format_max_len(chunk_url, max_len=45))
packed_chunks = self._fetch_chunks(chunk_url)
for chunk_data in self._unpack_chunks(packed_chunks):
# log.debug("chunk_data: {data}".format(data=chunk_data))
chunk = Chunk(chunk_data, list_name)
# log.debug("yielding {chunk}".format(chunk=chunk))
yield chunk
def api_call(self, url, payload=None):
"Perform a call to Safe Browsing API"
log.debug("performing api call to %s", util.format_max_len(url, 35))
if payload is None:
payload = b''
if isinstance(payload, str):
payload = bytes(payload.encode("ascii"))
request = urllib.request.Request(url, data=BytesIO(payload),
headers={'Content-Length': len(payload)})
try:
response = urllib.request.urlopen(request)
except urllib.error.HTTPError:
self._error_count += 1
raise
self._error_count = 0
return response.read()
def test_format_max_len(self):
eq_(util.format_max_len('abcdefghijklmnopqrstuvwxyz'), 'abcde[...]vwxyz')
eq_(util.format_max_len('abcdefghijklmnop'), 'abcde[...]lmnop')
eq_(util.format_max_len('abcdefghijklmno'), 'abcdefghijklmno')
eq_(util.format_max_len('a'), 'a')
def _fetch_chunks(self, url):
"Download chunks of data containing hash prefixes"
log.debug("fetching chunk %s", util.format_max_len(url, max_len=45))
response = urllib.request.urlopen(url)
return response