def test_cache_compression_setting():
image = np.zeros(shape=(128,128,128,1), dtype=np.uint8)
dirpath = '/tmp/cloudvolume/caching-validity-' + str(TEST_NUMBER)
layer_path = 'file://' + dirpath
vol = create_volume_from_image(
image=image,
offset=(1,1,1),
layer_path=layer_path,
layer_type='image',
resolution=(1,1,1),
encoding='raw'
)
vol.cache.enabled = True
vol.cache.flush()
vol.commit_info()
vol.cache.compress = None
vol[:] = image
assert all([ os.path.splitext(x)[1] == '.gz' for x in vol.cache.list() ])
vol.cache.flush()
vol.cache.compress = True
vol[:] = image
assert all([ os.path.splitext(x)[1] == '.gz' for x in vol.cache.list() ])
vol.cache.flush()
vol.cache.compress = False
vol[:] = image
assert all([ os.path.splitext(x)[1] == '' for x in vol.cache.list() ])
vol.cache.flush()
delete_layer(dirpath)
def test_info_provenance_cache():
image = np.zeros(shape=(128, 128, 128, 1), dtype=np.uint8)
vol = create_volume_from_image(
image=image,
offset=(0, 0, 0),
layer_path='gs://seunglab-test/cloudvolume/caching',
layer_type='image',
resolution=(1, 1, 1),
encoding='raw')
# Test Info
vol.cache.enabled = True
vol.cache.flush()
info = vol.refresh_info()
assert info is not None
with open(os.path.join(vol.cache.path, 'info'), 'r') as infof:
info = infof.read()
info = json.loads(info)
with open(os.path.join(vol.cache.path, 'info'), 'w') as infof:
infof.write(json.dumps({'wow': 'amaze'}))
info = vol.refresh_info()
assert info == {'wow': 'amaze'}
vol.cache.enabled = False
info = vol.refresh_info()
assert info != {'wow': 'amaze'}
infopath = os.path.join(vol.cache.path, 'info')
assert os.path.exists(infopath)
vol.cache.flush_info()
assert not os.path.exists(infopath)
vol.cache.flush_info() # assert no error by double delete
# Test Provenance
vol.cache.enabled = True
vol.cache.flush()
prov = vol.refresh_provenance()
assert prov is not None
with open(os.path.join(vol.cache.path, 'provenance'), 'r') as provf:
prov = provf.read()
prov = json.loads(prov)
with open(os.path.join(vol.cache.path, 'provenance'), 'w') as provf:
prov['description'] = 'wow'
provf.write(json.dumps(prov))
prov = vol.refresh_provenance()
assert prov['description'] == 'wow'
vol.cache.enabled = False
prov = vol.refresh_provenance()
assert prov['description'] == ''
provpath = os.path.join(vol.cache.path, 'provenance')
vol.cache.flush_provenance()
assert not os.path.exists(provpath)
vol.cache.flush_provenance() # assert no error by double delete
def test_cache_validity():
image = np.zeros(shape=(128,128,128,1), dtype=np.uint8)
dirpath = '/tmp/cloudvolume/caching-validity-' + str(TEST_NUMBER)
layer_path = 'file://' + dirpath
vol = create_volume_from_image(
image=image,
offset=(1,1,1),
layer_path=layer_path,
layer_type='image',
resolution=(1,1,1),
encoding='raw'
)
vol.cache.enabled = True
vol.cache.flush()
vol.commit_info()
def test_with_mock_cache_info(info, shoulderror):
finfo = os.path.join(vol.cache.path, 'info')
with open(finfo, 'w') as f:
f.write(json.dumps(info))
if shoulderror:
try:
CloudVolume(vol.layer_cloudpath, cache=True)
except ValueError:
pass
else:
assert False
else:
CloudVolume(vol.layer_cloudpath, cache=True)
test_with_mock_cache_info(vol.info, shoulderror=False)
info = vol.info.copy()
info['scales'][0]['size'][0] = 666
test_with_mock_cache_info(info, shoulderror=False)
test_with_mock_cache_info({ 'zomg': 'wow' }, shoulderror=True)
def tiny_change(key, val):
info = vol.info.copy()
info[key] = val
test_with_mock_cache_info(info, shoulderror=True)
tiny_change('type', 'zoolander')
tiny_change('data_type', 'uint32')
tiny_change('num_channels', 2)
tiny_change('mesh', 'mesh')
def scale_change(key, val, mip=0):
info = vol.info.copy()
info['scales'][mip][key] = val
test_with_mock_cache_info(info, shoulderror=True)
scale_change('voxel_offset', [ 1, 2, 3 ])
scale_change('resolution', [ 1, 2, 3 ])
scale_change('encoding', 'npz')
vol.cache.flush()
# Test no info file at all
CloudVolume(vol.layer_cloudpath, cache=True)
vol.cache.flush()
def test_caching():
image = np.zeros(shape=(128,128,128,1), dtype=np.uint8)
image[0:64,0:64,0:64] = 1
image[64:128,0:64,0:64] = 2
image[0:64,64:128,0:64] = 3
image[0:64,0:64,64:128] = 4
image[64:128,64:128,0:64] = 5
image[64:128,0:64,64:128] = 6
image[0:64,64:128,64:128] = 7
image[64:128,64:128,64:128] = 8
dirpath = '/tmp/cloudvolume/caching-volume-' + str(TEST_NUMBER)
layer_path = 'file://' + dirpath
vol = create_volume_from_image(
image=image,
offset=(0,0,0),
layer_path=layer_path,
layer_type='image',
resolution=(1,1,1),
encoding='raw'
)
vol.cache.enabled = True
vol.cache.flush()
# Test that reading populates the cache
read1 = vol[:,:,:]
assert np.all(read1 == image)
read2 = vol[:,:,:]
assert np.all(read2 == image)
assert len(vol.cache.list()) > 0
files = vol.cache.list()
validation_set = [
'0-64_0-64_0-64',
'64-128_0-64_0-64',
'0-64_64-128_0-64',
'0-64_0-64_64-128',
'64-128_64-128_0-64',
'64-128_0-64_64-128',
'0-64_64-128_64-128',
'64-128_64-128_64-128'
]
assert set([ os.path.splitext(fname)[0] for fname in files ]) == set(validation_set)
for i in range(8):
fname = os.path.join(vol.cache.path, vol.key, validation_set[i]) + '.gz'
with gzip.GzipFile(fname, mode='rb') as gfile:
chunk = gfile.read()
img3d = chunks.decode(
chunk, 'raw', (64,64,64,1), np.uint8
)
assert np.all(img3d == (i+1))
vol.cache.flush()
assert not os.path.exists(vol.cache.path)
# Test that writing populates the cache
vol[:,:,:] = image
assert os.path.exists(vol.cache.path)
assert np.all(vol[:,:,:] == image)
vol.cache.flush()
# Test that partial reads work too
result = vol[0:64,0:64,:]
assert np.all(result == image[0:64,0:64,:])
files = vol.cache.list()
assert len(files) == 2
result = vol[:,:,:]
assert np.all(result == image)
files = vol.cache.list()
assert len(files) == 8
vol.cache.flush()
# Test Non-standard Cache Destination
dirpath = '/tmp/cloudvolume/caching-cache-' + str(TEST_NUMBER)
vol.cache.enabled = dirpath
vol[:,:,:] = image
assert len(os.listdir(os.path.join(dirpath, vol.key))) == 8
vol.cache.flush()
# Test that caching doesn't occur when cache is not set
vol.cache.enabled = False
result = vol[:,:,:]
if os.path.exists(vol.cache.path):
files = vol.cache.list()
assert len(files) == 0
vol[:,:,:] = image
if os.path.exists(vol.cache.path):
files = vol.cache.list()
assert len(files) == 0
vol.cache.flush()
# Test that deletion works too
vol.cache.enabled = True
vol[:,:,:] = image
files = vol.cache.list()
assert len(files) == 8
vol.delete( np.s_[:,:,:] )
files = vol.cache.list()
assert len(files) == 0
vol.cache.flush()
vol[:,:,:] = image
files = vol.cache.list()
assert len(files) == 8
vol.cache.flush(preserve=np.s_[:,:,:])
files = vol.cache.list()
assert len(files) == 8
vol.cache.flush(preserve=np.s_[:64,:64,:])
files = vol.cache.list()
assert len(files) == 2
vol.cache.flush()
vol[:,:,:] = image
files = vol.cache.list()
assert len(files) == 8
vol.cache.flush_region(Bbox( (50, 50, 0), (100, 100, 10) ))
files = vol.cache.list()
assert len(files) == 4
vol.cache.flush()
vol[:,:,:] = image
files = vol.cache.list()
assert len(files) == 8
vol.cache.flush_region(np.s_[50:100, 50:100, 0:10])
files = vol.cache.list()
assert len(files) == 4
vol.cache.flush()