def benchmark_upload(voltype):
global CHUNK_SIZES
global N
originvol = CloudVolume('gs://seunglab-test/test_v0/{}'.format(voltype))
originvol.reset_scales()
info = deepcopy(originvol.info)
img = originvol[:]
for chunk_size in CHUNK_SIZES[::-1]:
cloudpath = 'gs://seunglab-test/test_v0/{}_upload_{}_{}_{}'.format(voltype, *chunk_size)
info['scales'][0]['chunk_sizes'] = [ list(chunk_size) ]
vol = CloudVolume(cloudpath, progress=True, info=info, compress='gzip')
def upload():
vol[:] = img
stats = benchmark(upload, N)
log({
"direction": "upload",
"compression": "gzip",
"image_type": voltype,
"N": N,
"mean": stats['mean'],
"fastest": stats['fastest'],
"slowest": stats['slowest'],
"cloudpath": cloudpath,
"chunk_size": chunk_size,
"dtype": vol.dtype,
"hostname": socket.gethostname(),
})
vol.delete(vol.bounds)
for chunk_size in CHUNK_SIZES[::-1]:
cloudpath = 'gs://seunglab-test/test_v0/{}_upload_{}_{}_{}'.format(voltype, *chunk_size)
info['scales'][0]['chunk_sizes'] = [ list(chunk_size) ]
vol = CloudVolume(cloudpath, progress=True, info=info, compress='')
def upload():
vol[:] = img
stats = benchmark(upload, N)
log({
"direction": "upload",
"compression": "none",
"image_type": voltype,
"N": N,
"mean": stats['mean'],
"fastest": stats['fastest'],
"slowest": stats['slowest'],
"cloudpath": cloudpath,
"chunk_size": chunk_size,
"dtype": vol.dtype,
"hostname": socket.gethostname(),
})
vol.delete(vol.bounds)
[1024, 1024, 100], # max size of SNEMI3D
)
imgvol = CloudVolume('gs://seunglab-test/test_v0/black', progress=True)
img = imgvol[:]
for chunksize in sizes:
info['scales'][0]['chunk_sizes'] = [chunksize]
layer = 'black_{}_{}_{}'.format(*chunksize)
vol = CloudVolume('gs://seunglab-test/test_v0/' + layer,
info=info,
compress='gzip',
progress=True)
vol.reset_scales()
vol.commit_info()
# vol[:] = np.zeros(shape=vol.shape, dtype=vol.dtype)
vol[:] = img
layer = 'black_uncompressed_{}_{}_{}'.format(*chunksize)
vol = CloudVolume('gs://seunglab-test/test_v0/' + layer,
info=info,
compress='',
progress=True)
vol.reset_scales()
vol.commit_info()
vol[:] = img
