def test_compression(compression_method): for N in range(100): flts = np.array(range(N), dtype=np.float32).reshape( (N, 1, 1, 1)).tostring() compressed = compress(flts, compression_method) assert compressed != flts decompressed = decompress(compressed, compression_method) assert decompressed == flts flts = np.array([], dtype=np.float32).tobytes() try: decompress(flts, compression_method) assert False except DecompressionError: pass
def test_compression(compression_method): for N in range(100): flts = np.array(range(N), dtype=np.float32).reshape( (N,1,1,1) ).tostring() compressed = compress(flts, compression_method) assert compressed != flts decompressed = decompress(compressed, compression_method) assert decompressed == flts
def get_file(self, file_path, start=None, end=None): """ Get the binary contents of a file. Optionally, specify the inclusive byte range to request. """ with self.get_connection() as conn: content, encoding = conn.get_file(file_path, start=start, end=end) return compression.decompress(content, encoding, filename=file_path)
def test_gzip(): for N in range(100): flts = np.array(range(N), dtype=np.float32).reshape( (N, 1, 1, 1)).tostring() compressed = compress(flts, 'gzip') assert compressed != flts decompressed = decompress(compressed, 'gzip') assert decompressed == flts
def get_file(self, file_path, start=None, end=None): """ Get the binary contents of a file. Optionally, specify the inclusive byte range to request. """ content, encoding = self._interface.get_file(file_path, start=start, end=end) content = compression.decompress(content, encoding, filename=file_path) return content
def before_request(): current_app.request_start_time = time.time() current_app.request_start_date = datetime.utcnow() current_app.user_id = None current_app.table_id = None current_app.request_type = None content_encoding = request.headers.get('Content-Encoding', '') if "gzip" in content_encoding.lower(): request.data = compression.decompress(request.data, "gzip")
def get_file_thunk(path, interface): result = error = None try: result = interface.get_file(path) except Exception as err: error = err # important to print immediately because # errors are collected at the end print(err) content, encoding = result content = compression.decompress(content, encoding) results.append({ "filename": path, "content": content, "error": error, })
def test_compress_level(compression_method): N = 10000 x = np.array(range(N), dtype=np.float32).reshape( (N, 1, 1, 1) ) content = np.ascontiguousarray(x, dtype=np.float32).tostring() compr_rate = [] compress_levels = (1, 8) for compress_level in compress_levels: print(compress_level) compressed = compress(content, compression_method, compress_level=compress_level) assert compressed != content decompressed = decompress(compressed, compression_method) assert decompressed == content compr_rate.append(float(len(compressed)) / float(len(content))) # make sure we get better compression at highest level than lowest level assert compr_rate[-1] < compr_rate[0]
def getfn(path): result = error = None conn = self.get_connection() try: result = conn.get_file(path) except Exception as err: error = err # important to print immediately because # errors are collected at the end print(err) del conn else: conn.release_connection() content, encoding = result content = compression.decompress(content, encoding) return { "filename": path, "content": content, "error": error, }
def get_file(self, file_path): content, encoding = self._interface.get_file(file_path) content = compression.decompress(content, encoding, filename=file_path) return content
def get_file(self, file_path): with self.get_connection() as conn: content, encoding = conn.get_file(file_path) return compression.decompress(content, encoding, filename=file_path)