def decompress(input, type=None):
if type == None: type = guesstype(input)
if type == 'gzip':
magic1, magic2, method, flags, mtime, xf, os = unpack('<BBBBIBB', input[:10])
if magic1 != 0x1F or magic2 != 0x8B: raise IOError('Not a gzipped file')
if method != 8: raise IOError('Unknown compression method')
if flags & 0xE0: raise IOError('Unknown flags')
off = unpack('<H', input[10:12])[0] + 12 if flags & FEXTRA else 10
if flag & FNAME: off = input.index('\x00', off) + 1
if flag & FCOMMENT: off = input.index('\x00', off) + 1
if flags & FHCRC:
if unpack('<H', input[off:off+2])[0] != (crc32(input[:off]) & 0xffff): raise IOError('Header corrupted')
off += 2
crc32, isize = unpack('<II', input[-8:])
s = zdecompress(input[off:-8], -MAX_WBITS, isize)
checksum = crc32(s)
if crc32 != checksum: raise IOError("CRC32 check failed %08x != %08x" % (crc32, checksum))
if isize != (len(s) & 0xffffffffL): raise IOError("Incorrect length of data produced")
return s
elif type == 'zlib':
header = unpack('>H', input[:2])[0]
method = (header >> 8) & 0xF
windowsize = ((header >> 12) & 0xF) + 8
fdict = (header & 0x20) != 0
if method != 8 or windowsize > MAX_WBITS or fdict: raise IOError('Unknown compression method')
if header % 31 != 0: raise IOError('Header corrupted')
s = zdecompress(input[2:-4], -windowsize)
a32 = unpack('>I', input[-4:])[0]
checksum = adler32(s)
if a32 != checksum: raise IOError("Adler32 check failed %08x != %08x" % (a32, checksum))
return s
elif type == 'deflate':
return zdecompress(input)
else:
raise ValueError('Compression type must be one of deflate, zlib, gzip, or None')
def _decompress(self, body):
try:
resp_encoding = self.headers['Content-Encoding'].strip()
except KeyError:
return body
if resp_encoding == 'gzip':
return gdecompress(body)
elif resp_encoding == 'deflate':
return zdecompress(body)
return body
def remote_api_data(self):
## 0. memcache hit -> response
v = self.cache_get(self.cache_key)
if v:
return v
## 1. mmemcache miss -> url fetch
url = self.remote_url
try:
if not url:
raise Exception('bad url')
data = self.load(urlopen(url).read(self.read_size))
except (Exception, ), exc:
error('error: %s, url: %s', exc, url)
## 1a. fetch failure -> history lookup
storage = History.all().filter('url =', url).get()
if storage:
## this assumes that the value has already been
## parsed and cooked at least one time.
data = self.load(zdecompress(storage.payload.encode('ISO-8859-1')))
self.cache_set(data)
else:
## 1b. total failure
data = self.fail_value
def decompress(data):
if ZLIB:
return zdecompress(data)
else:
return data
def decompress(data):
if ZLIB:
return zdecompress(data)
else:
return data
def pklfile2data(_filename: str) -> object:
"""Open the specified file and load the contained compressed+pickled base64url (RFC 4648) data"""
ensurefileexists(_filename)
return loads(zdecompress(b64decode(bytes(getfile(_filename), encoding=r'utf-8'), altchars=br'-_', validate=True)))
