def put_files(self, files, content_type=None, compress=None, cache_control=None, block=True): """ Put lots of files at once and get a nice progress bar. It'll also wait for the upload to complete, just like get_files. Required: files: [ (filepath, content), .... ] """ def base_uploadfn(path, content, interface): interface.put_file(path, content, content_type, compress, cache_control=cache_control) for path, content in files: content = compression.compress(content, method=compress) uploadfn = partial(base_uploadfn, path, content) if len(self._threads): self.put(uploadfn) else: uploadfn(self._interface) if block: desc = 'Uploading' if self.progress else None self.wait(desc) return self
def put_files(self, files, content_type=None, compress=None, compress_level=None, cache_control=None, block=True): """ Put lots of files at once and get a nice progress bar. It'll also wait for the upload to complete, just like get_files. Required: files: [ (filepath, content), .... ] """ for path, content in tqdm(files, disable=(not self.progress), desc="Uploading"): content = compression.compress(content, method=compress, compress_level=compress_level) self._interface.put_file(path, content, content_type, compress, cache_control=cache_control) return self
def test_compression(compression_method): for N in range(100): flts = np.array(range(N), dtype=np.float32).reshape( (N,1,1,1) ).tostring() compressed = compress(flts, compression_method) assert compressed != flts decompressed = decompress(compressed, compression_method) assert decompressed == flts
def put_files(self, files, content_type=None, compress=None, compress_level=None, cache_control=None, block=True): """ Put lots of files at once and get a nice progress bar. It'll also wait for the upload to complete, just like get_files. Required: files: [ (filepath, content), .... ] """ def base_uploadfn(path, content, interface): interface.put_file(path, content, content_type, compress, cache_control=cache_control) seen = set() for path, content in files: if path in seen: raise ValueError("Cannot write the same file multiple times in one pass. This causes a race condition. File: " + path) seen.add(path) content = compression.compress(content, method=compress, compress_level=compress_level) uploadfn = partial(base_uploadfn, path, content) if len(self._threads): self.put(uploadfn) else: uploadfn(self._interface) if block: desc = self.progress_description('Uploading') self.wait(desc) return self
def test_gzip(): for N in range(100): flts = np.array(range(N), dtype=np.float32).reshape( (N, 1, 1, 1)).tostring() compressed = compress(flts, 'gzip') assert compressed != flts decompressed = decompress(compressed, 'gzip') assert decompressed == flts
def uploadfn(path, content): with self.get_connection() as conn: content = compression.compress(content, method=compress) conn.put_file( file_path=path, content=content, content_type=content_type, compress=compress, cache_control=cache_control, )
def test_compression(compression_method): for N in range(100): flts = np.array(range(N), dtype=np.float32).reshape( (N, 1, 1, 1)).tostring() compressed = compress(flts, compression_method) assert compressed != flts decompressed = decompress(compressed, compression_method) assert decompressed == flts flts = np.array([], dtype=np.float32).tobytes() try: decompress(flts, compression_method) assert False except DecompressionError: pass
def test_compress_level(compression_method): N = 10000 x = np.array(range(N), dtype=np.float32).reshape( (N, 1, 1, 1) ) content = np.ascontiguousarray(x, dtype=np.float32).tostring() compr_rate = [] compress_levels = (1, 8) for compress_level in compress_levels: print(compress_level) compressed = compress(content, compression_method, compress_level=compress_level) assert compressed != content decompressed = decompress(compressed, compression_method) assert decompressed == content compr_rate.append(float(len(compressed)) / float(len(content))) # make sure we get better compression at highest level than lowest level assert compr_rate[-1] < compr_rate[0]