def chunks(self): "Generator iterating through the server respones chunk by chunk" log.debug("accessing chunks") for list_name, chunk_urls in list(self.lists_data.items()): for chunk_url in chunk_urls: log.debug("processing chunk url: %s", util.format_max_len(chunk_url, max_len=45)) packed_chunks = self._fetch_chunks(chunk_url) for chunk_data in self._unpack_chunks(packed_chunks): # log.debug("chunk_data: {data}".format(data=chunk_data)) chunk = Chunk(chunk_data, list_name) # log.debug("yielding {chunk}".format(chunk=chunk)) yield chunk
def api_call(self, url, payload=None): "Perform a call to Safe Browsing API" log.debug("performing api call to %s", util.format_max_len(url, 35)) if payload is None: payload = b'' if isinstance(payload, str): payload = bytes(payload.encode("ascii")) request = urllib.request.Request(url, data=BytesIO(payload), headers={'Content-Length': len(payload)}) try: response = urllib.request.urlopen(request) except urllib.error.HTTPError: self._error_count += 1 raise self._error_count = 0 return response.read()
def test_format_max_len(self): eq_(util.format_max_len('abcdefghijklmnopqrstuvwxyz'), 'abcde[...]vwxyz') eq_(util.format_max_len('abcdefghijklmnop'), 'abcde[...]lmnop') eq_(util.format_max_len('abcdefghijklmno'), 'abcdefghijklmno') eq_(util.format_max_len('a'), 'a')
def _fetch_chunks(self, url): "Download chunks of data containing hash prefixes" log.debug("fetching chunk %s", util.format_max_len(url, max_len=45)) response = urllib.request.urlopen(url) return response