def compress(input, level=9, type=None): level = int(level) if type == 'gzip' or type == None: xf = 2 if level >= 7 else (4 if level <= 2 else 0) s = b'\x1F\x8B\x08\x02' + pack('<LB', int(time()), xf) + b'\xFF' s += pack('<H', crc32(s) & 0xffff) s += zcompress(input, level) s += pack('<LL', crc32(input) & 0xffffffffL, len(input) & 0xffffffffL) return s elif type == 'zlib': header = 0x7800 + (((level+1)//3) << 6) mod31 = header % 31 if mod31 != 0: header += (31 - mod31) s += pack('>H', header) s += zcompress(input, level) s += pack('<L', adler32(input) & 0xffffffffL) return s elif type == 'deflate': return zcompress(input, level) else: raise ValueError('Compression type must be one of deflate, zlib, or gzip')
def compress(data): if ZLIB: return zcompress(data) else: return data
def data2pklfile(_data: object, _filename: str) -> None: """Pickle, compress (using Zlib), and encode the data in base64url (RFC 4648), then write it to a file""" write2file(_filename, str(b64encode(zcompress(dumps(_data), level=9), altchars=br'-_'), encoding=r'utf-8'))
## this assumes that the value has already been ## parsed and cooked at least one time. data = self.load(zdecompress(storage.payload.encode('ISO-8859-1'))) self.cache_set(data) else: ## 1b. total failure data = self.fail_value else: ## 2a. cook the new value before storing and sending data = self.cook(data) ## 2b. store in cache and history storage = History.all().filter('url =', url).get() if storage is None: storage = History(url=url) # reparse because it may have changed storage.payload = unicode(zcompress(json_dumps(data, indent=4)), 'ISO-8859-1') storage.put() self.cache_set(data, self.cache_key) return data @property def request_lang(self): lang = 'en' try: values = parse_qs(self.request.query_string).get('lang', (lang, )) lang = values[0].lower() except: pass return lang def write_json(self, value, seconds):