def test_compression(compression_method):
for N in range(100):
flts = np.array(range(N), dtype=np.float32).reshape(
(N, 1, 1, 1)).tostring()
compressed = compress(flts, compression_method)
assert compressed != flts
decompressed = decompress(compressed, compression_method)
assert decompressed == flts
flts = np.array([], dtype=np.float32).tobytes()
try:
decompress(flts, compression_method)
assert False
except DecompressionError:
pass
def test_compression(compression_method):
for N in range(100):
flts = np.array(range(N), dtype=np.float32).reshape( (N,1,1,1) ).tostring()
compressed = compress(flts, compression_method)
assert compressed != flts
decompressed = decompress(compressed, compression_method)
assert decompressed == flts
def get_file(self, file_path, start=None, end=None):
"""
Get the binary contents of a file. Optionally, specify
the inclusive byte range to request.
"""
with self.get_connection() as conn:
content, encoding = conn.get_file(file_path, start=start, end=end)
return compression.decompress(content, encoding, filename=file_path)
def test_gzip():
for N in range(100):
flts = np.array(range(N), dtype=np.float32).reshape(
(N, 1, 1, 1)).tostring()
compressed = compress(flts, 'gzip')
assert compressed != flts
decompressed = decompress(compressed, 'gzip')
assert decompressed == flts
def get_file(self, file_path, start=None, end=None): """ Get the binary contents of a file. Optionally, specify the inclusive byte range to request. """ content, encoding = self._interface.get_file(file_path, start=start, end=end) content = compression.decompress(content, encoding, filename=file_path) return content
def before_request():
current_app.request_start_time = time.time()
current_app.request_start_date = datetime.utcnow()
current_app.user_id = None
current_app.table_id = None
current_app.request_type = None
content_encoding = request.headers.get('Content-Encoding', '')
if "gzip" in content_encoding.lower():
request.data = compression.decompress(request.data, "gzip")
def get_file_thunk(path, interface):
result = error = None
try:
result = interface.get_file(path)
except Exception as err:
error = err
# important to print immediately because
# errors are collected at the end
print(err)
content, encoding = result
content = compression.decompress(content, encoding)
results.append({
"filename": path,
"content": content,
"error": error,
})
def test_compress_level(compression_method):
N = 10000
x = np.array(range(N), dtype=np.float32).reshape( (N, 1, 1, 1) )
content = np.ascontiguousarray(x, dtype=np.float32).tostring()
compr_rate = []
compress_levels = (1, 8)
for compress_level in compress_levels:
print(compress_level)
compressed = compress(content, compression_method, compress_level=compress_level)
assert compressed != content
decompressed = decompress(compressed, compression_method)
assert decompressed == content
compr_rate.append(float(len(compressed)) / float(len(content)))
# make sure we get better compression at highest level than lowest level
assert compr_rate[-1] < compr_rate[0]
def getfn(path):
result = error = None
conn = self.get_connection()
try:
result = conn.get_file(path)
except Exception as err:
error = err
# important to print immediately because
# errors are collected at the end
print(err)
del conn
else:
conn.release_connection()
content, encoding = result
content = compression.decompress(content, encoding)
return {
"filename": path,
"content": content,
"error": error,
}
def get_file(self, file_path):
content, encoding = self._interface.get_file(file_path)
content = compression.decompress(content, encoding, filename=file_path)
return content
def get_file(self, file_path):
with self.get_connection() as conn:
content, encoding = conn.get_file(file_path)
return compression.decompress(content, encoding, filename=file_path)
