def put_files(self,
files,
content_type=None,
compress=None,
cache_control=None,
block=True):
"""
Put lots of files at once and get a nice progress bar. It'll also wait
for the upload to complete, just like get_files.
Required:
files: [ (filepath, content), .... ]
"""
def base_uploadfn(path, content, interface):
interface.put_file(path,
content,
content_type,
compress,
cache_control=cache_control)
for path, content in files:
content = compression.compress(content, method=compress)
uploadfn = partial(base_uploadfn, path, content)
if len(self._threads):
self.put(uploadfn)
else:
uploadfn(self._interface)
if block:
desc = 'Uploading' if self.progress else None
self.wait(desc)
return self
def put_files(self,
files,
content_type=None,
compress=None,
compress_level=None,
cache_control=None,
block=True):
"""
Put lots of files at once and get a nice progress bar. It'll also wait
for the upload to complete, just like get_files.
Required:
files: [ (filepath, content), .... ]
"""
for path, content in tqdm(files,
disable=(not self.progress),
desc="Uploading"):
content = compression.compress(content,
method=compress,
compress_level=compress_level)
self._interface.put_file(path,
content,
content_type,
compress,
cache_control=cache_control)
return self
def test_compression(compression_method):
for N in range(100):
flts = np.array(range(N), dtype=np.float32).reshape( (N,1,1,1) ).tostring()
compressed = compress(flts, compression_method)
assert compressed != flts
decompressed = decompress(compressed, compression_method)
assert decompressed == flts
def put_files(self, files, content_type=None, compress=None, compress_level=None, cache_control=None, block=True):
"""
Put lots of files at once and get a nice progress bar. It'll also wait
for the upload to complete, just like get_files.
Required:
files: [ (filepath, content), .... ]
"""
def base_uploadfn(path, content, interface):
interface.put_file(path, content, content_type, compress, cache_control=cache_control)
seen = set()
for path, content in files:
if path in seen:
raise ValueError("Cannot write the same file multiple times in one pass. This causes a race condition. File: " + path)
seen.add(path)
content = compression.compress(content, method=compress, compress_level=compress_level)
uploadfn = partial(base_uploadfn, path, content)
if len(self._threads):
self.put(uploadfn)
else:
uploadfn(self._interface)
if block:
desc = self.progress_description('Uploading')
self.wait(desc)
return self
def test_gzip():
for N in range(100):
flts = np.array(range(N), dtype=np.float32).reshape(
(N, 1, 1, 1)).tostring()
compressed = compress(flts, 'gzip')
assert compressed != flts
decompressed = decompress(compressed, 'gzip')
assert decompressed == flts
def uploadfn(path, content):
with self.get_connection() as conn:
content = compression.compress(content, method=compress)
conn.put_file(
file_path=path,
content=content,
content_type=content_type,
compress=compress,
cache_control=cache_control,
)
def test_compression(compression_method):
for N in range(100):
flts = np.array(range(N), dtype=np.float32).reshape(
(N, 1, 1, 1)).tostring()
compressed = compress(flts, compression_method)
assert compressed != flts
decompressed = decompress(compressed, compression_method)
assert decompressed == flts
flts = np.array([], dtype=np.float32).tobytes()
try:
decompress(flts, compression_method)
assert False
except DecompressionError:
pass
def test_compress_level(compression_method):
N = 10000
x = np.array(range(N), dtype=np.float32).reshape( (N, 1, 1, 1) )
content = np.ascontiguousarray(x, dtype=np.float32).tostring()
compr_rate = []
compress_levels = (1, 8)
for compress_level in compress_levels:
print(compress_level)
compressed = compress(content, compression_method, compress_level=compress_level)
assert compressed != content
decompressed = decompress(compressed, compression_method)
assert decompressed == content
compr_rate.append(float(len(compressed)) / float(len(content)))
# make sure we get better compression at highest level than lowest level
assert compr_rate[-1] < compr_rate[0]