def compress(input, level=9, type=None):
level = int(level)
if type == 'gzip' or type == None:
xf = 2 if level >= 7 else (4 if level <= 2 else 0)
s = b'\x1F\x8B\x08\x02' + pack('<LB', int(time()), xf) + b'\xFF'
s += pack('<H', crc32(s) & 0xffff)
s += zcompress(input, level)
s += pack('<LL', crc32(input) & 0xffffffffL, len(input) & 0xffffffffL)
return s
elif type == 'zlib':
header = 0x7800 + (((level+1)//3) << 6)
mod31 = header % 31
if mod31 != 0: header += (31 - mod31)
s += pack('>H', header)
s += zcompress(input, level)
s += pack('<L', adler32(input) & 0xffffffffL)
return s
elif type == 'deflate':
return zcompress(input, level)
else:
raise ValueError('Compression type must be one of deflate, zlib, or gzip')
def compress(data):
if ZLIB:
return zcompress(data)
else:
return data
def data2pklfile(_data: object, _filename: str) -> None:
"""Pickle, compress (using Zlib), and encode the data in base64url (RFC 4648), then write it to a file"""
write2file(_filename, str(b64encode(zcompress(dumps(_data), level=9), altchars=br'-_'), encoding=r'utf-8'))
def compress(data):
if ZLIB:
return zcompress(data)
else:
return data
## this assumes that the value has already been
## parsed and cooked at least one time.
data = self.load(zdecompress(storage.payload.encode('ISO-8859-1')))
self.cache_set(data)
else:
## 1b. total failure
data = self.fail_value
else:
## 2a. cook the new value before storing and sending
data = self.cook(data)
## 2b. store in cache and history
storage = History.all().filter('url =', url).get()
if storage is None:
storage = History(url=url)
# reparse because it may have changed
storage.payload = unicode(zcompress(json_dumps(data, indent=4)), 'ISO-8859-1')
storage.put()
self.cache_set(data, self.cache_key)
return data
@property
def request_lang(self):
lang = 'en'
try:
values = parse_qs(self.request.query_string).get('lang', (lang, ))
lang = values[0].lower()
except:
pass
return lang
def write_json(self, value, seconds):