def test_default_reader(resources_dir, filename, expected_shape, expected_dims,
expected_chunksize, expected_task_count):
# Get file
f = resources_dir / filename
# Read file
img = DefaultReader(f)
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
with Profiler() as prof:
assert img.dims == expected_dims
assert img.metadata
assert img.dask_data.shape == expected_shape
assert img.dask_data.chunksize == expected_chunksize
# Check that basic details don't require task computation
assert len(prof.results) == 0
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
# Check computed type is numpy array, computed shape is expected shape, and task count is expected
with Profiler() as prof:
assert isinstance(img.data, np.ndarray)
assert img.data.shape == expected_shape
assert len(prof.results) == expected_task_count
# Check that there are no open file pointers after retrieval
assert str(f) not in [f.path for f in proc.open_files()]
def test_imread_use_dask_false(resources_dir):
# Load image as delayed dask array then as numpy array
# Check computed task count
with dask_utils.cluster_and_client(processes=False) as (cluster, client):
# Get filepath
f = resources_dir / BIG_OME_FILE
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check that a client does exist
get_client()
# Don't use dask for reads
use_dask(False)
# Read image without dask
img = AICSImage(f)
assert img.data.shape == (3, 1, 3, 5, 325, 475)
# Check that the file was read with base reader then rechunked with dask
# Normally the task count for this file is 90
assert len(optimize(img.dask_data)[0].__dask_graph__()) == 3
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
def test_size_functions(resources_dir, filename, s, t, c, z, y, x):
# Get file
f = resources_dir / filename
# Check that there are no open file pointers
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Init reader
img = LifReader(f)
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check sizes
assert img.size_s() == s
assert img.size_t() == t
assert img.size_c() == c
assert img.size_z() == z
assert img.size_y() == y
assert img.size_x() == x
# Check that there are no open file pointers
assert str(f) not in [f.path for f in proc.open_files()]
def test_czi_reader(
resources_dir,
filename,
expected_shape,
expected_dims,
expected_dtype,
select_scene,
chunk_dims,
):
# Get file
f = resources_dir / filename
# Read file
img = CziReader(f, chunk_by_dims=chunk_dims, S=select_scene)
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
assert img.dims == expected_dims
assert img.metadata is not None
assert img.dask_data.shape == expected_shape
assert img.dtype() == expected_dtype
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
# Check array
assert isinstance(img.data, np.ndarray)
assert img.data.shape == expected_shape
# Check that there are no open file pointers after retrieval
assert str(f) not in [f.path for f in proc.open_files()]
def test_default_reader(
resources_dir,
filename,
expected_shape,
expected_dims,
):
# Get file
f = resources_dir / filename
# Read file
img = DefaultReader(f)
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
assert img.dims == expected_dims
assert img.metadata
assert img.dask_data.shape == expected_shape
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
# Check array
assert isinstance(img.data, np.ndarray)
assert img.data.shape == expected_shape
# Check that there are no open file pointers after retrieval
assert str(f) not in [f.path for f in proc.open_files()]
def test_dims_setting(resources_dir, expected_starting_dims, set_dims,
expected_ending_dims):
# Get file
f = resources_dir / "example.png"
# Read file
img = DefaultReader(f)
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
assert img.dims == expected_starting_dims
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
# Update dims
img.dims = set_dims
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
# Check expected
assert img.dims == expected_ending_dims
# Check that there are no open file pointers after retrieval
assert str(f) not in [f.path for f in proc.open_files()]
def test_get_channel_names(resources_dir, filename, scene, expected):
f = resources_dir / filename
# Check that there are no open file pointers
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
assert LifReader(resources_dir / filename).get_channel_names(scene) == expected
# Check that there are no open file pointers
assert str(f) not in [f.path for f in proc.open_files()]
def test_lif_image_data_two(resources_dir, filename, scene, expected):
f = resources_dir / filename
# Check that there are no open file pointers
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
img = LifReader(f)
assert img._chunk_offsets[0][0, 0, 0] == expected
# Check that there are no open file pointers
assert str(f) not in [f.path for f in proc.open_files()]
def test_get_physical_pixel_size(resources_dir, filename, scene, expected):
f = resources_dir / filename
# Check that there are no open file pointers
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
assert LifReader(resources_dir /
filename).get_physical_pixel_size(scene) == pytest.approx(
expected, rel=0.001)
# Check that there are no open file pointers
assert str(f) not in [f.path for f in proc.open_files()]
def test_ome_tiff_reader(
resources_dir,
filename,
expected_shape,
expected_dims,
select_scene,
):
# Get file
f = resources_dir / filename
# Read file
img = OmeTiffReader(f, S=select_scene)
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
# Check that OME Metadata matches the dask data array shape and dims order
dim_size_getters = {
Dimensions.Scene: img.size_s,
Dimensions.Time: img.size_t,
Dimensions.Channel: img.size_c,
Dimensions.SpatialZ: img.size_z,
Dimensions.SpatialY: img.size_y,
Dimensions.SpatialX: img.size_x,
}
for d, getter in dim_size_getters.items():
if d in expected_dims:
assert getter() == img.dask_data.shape[img.dims.index(d)]
assert img.dims == expected_dims
assert img.is_ome()
assert img.metadata
assert img.shape == expected_shape
assert img.dask_data.shape == expected_shape
assert img.size(expected_dims) == expected_shape
# Will error because those dimensions don't exist in the file
with pytest.raises(exceptions.InvalidDimensionOrderingError):
assert img.size("ABCDEFG") == expected_shape
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
# Check array
assert isinstance(img.data, np.ndarray)
assert img.data.shape == expected_shape
# Check that there are no open file pointers after retrieval
assert str(f) not in [f.path for f in proc.open_files()]
def test_file_passed_was_directory(resources_dir):
# Get filepath
f = resources_dir
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
with pytest.raises(IsADirectoryError):
AICSImage(resources_dir)
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
def test_aicsimage_serialize(
resources_dir,
tmpdir,
filename,
expected_shape,
expected_metadata_type,
expected_task_count,
):
"""
Test that the entire AICSImage object can be serialized - a requirement to distribute on dask clusters.
https://distributed.dask.org/en/latest/serialization.html
"""
# Get file
f = resources_dir / filename
# Read file
img = AICSImage(f)
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
with Profiler() as prof:
assert img.shape == expected_shape
assert isinstance(img.metadata, expected_metadata_type)
# Check that basic details don't require task computation
assert len(prof.results) == 0
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
# Serialize object
serialized = pickle.dumps(img)
# Reload
img = pickle.loads(serialized)
# Check computed type is numpy array, computed shape is expected shape, and task count is expected
with Profiler() as prof:
assert isinstance(img.data, np.ndarray)
assert img.shape == expected_shape
assert img.data.shape == expected_shape
assert isinstance(img.metadata, expected_metadata_type)
assert len(prof.results) == expected_task_count
# Check that there are no open file pointers after retrieval
assert str(f) not in [f.path for f in proc.open_files()]
def test_physical_pixel_size(resources_dir, filename, expected_sizes):
# Get filepath
f = resources_dir / filename
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
img = AICSImage(f)
assert img.get_physical_pixel_size() == expected_sizes
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
def test_imread(resources_dir, filename, expected_shape):
# Get filepath
f = resources_dir / filename
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
img = imread(f)
assert img.shape == expected_shape
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
def test_typing(filename, expected_reader, resources_dir):
# Get filepath
f = resources_dir / filename
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
actual_reader = AICSImage.determine_reader(f)
assert actual_reader == expected_reader
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
def test_metadata(resources_dir, filename, expected_metadata_type):
# Get filepath
f = resources_dir / filename
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
img = AICSImage(f)
assert isinstance(img.metadata, expected_metadata_type)
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
def test_channel_names(resources_dir, filename, expected_channel_names):
# Get filepath
f = resources_dir / filename
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
img = AICSImage(f)
assert img.get_channel_names() == expected_channel_names
assert len(img.get_channel_names()) == img.size_c
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
def test_get_channel_names(resources_dir, filename, scene, expected_channel_names):
# Get file
f = resources_dir / filename
# Read file
img = OmeTiffReader(f)
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check channel names
assert img.get_channel_names(scene) == expected_channel_names
# Check that there are no open file pointers after check
assert str(f) not in [f.path for f in proc.open_files()]
def test_ome_tiff_reader(resources_dir, filename, expected_shape,
expected_dims, select_scene, expected_chunksize,
expected_task_count):
# Get file
f = resources_dir / filename
# Read file
img = OmeTiffReader(f, S=select_scene)
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
with Profiler() as prof:
# Check that OME Metadata matches the dask data array shape and dims order
dim_size_getters = {
Dimensions.Scene: img.size_s,
Dimensions.Time: img.size_t,
Dimensions.Channel: img.size_c,
Dimensions.SpatialZ: img.size_z,
Dimensions.SpatialY: img.size_y,
Dimensions.SpatialX: img.size_x
}
for d, getter in dim_size_getters.items():
if d in expected_dims:
assert getter() == img.dask_data.shape[img.dims.index(d)]
assert img.dims == expected_dims
assert img.is_ome()
assert img.metadata
assert img.dask_data.shape == expected_shape
assert img.dask_data.chunksize == expected_chunksize
# Check that basic details don't require task computation
assert len(prof.results) == 0
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
# Check computed type is numpy array, computed shape is expected shape, and task count is expected
with Profiler() as prof:
assert isinstance(img.data, np.ndarray)
assert img.data.shape == expected_shape
assert len(prof.results) == expected_task_count
# Check that there are no open file pointers after retrieval
assert str(f) not in [f.path for f in proc.open_files()]
def test_file_passed_was_directory(resources_dir):
# Get filepath
f = resources_dir
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
with Profiler() as prof:
with pytest.raises(IsADirectoryError):
AICSImage(resources_dir)
# Check that basic details don't require task computation
assert len(prof.results) == 0
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
def test_typing(filename, expected_reader, resources_dir):
# Get filepath
f = resources_dir / filename
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
with Profiler() as prof:
actual_reader = AICSImage.determine_reader(f)
assert actual_reader == expected_reader
# Check that basic details don't require task computation
assert len(prof.results) == 0
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
def test_large_imread_dask(resources_dir, filename, expected_shape,
expected_task_count):
# Get filepath
f = resources_dir / filename
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
with Profiler() as prof:
img = imread_dask(f)
assert img.shape == expected_shape
assert len(prof.results) == expected_task_count
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
def test_physical_pixel_size(resources_dir, filename, expected_sizes):
# Get filepath
f = resources_dir / filename
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
with Profiler() as prof:
img = AICSImage(f)
assert img.get_physical_pixel_size() == expected_sizes
# Check that basic details don't require task computation
assert len(prof.results) == 0
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
def test_metadata(resources_dir, filename, expected_metadata_type):
# Get filepath
f = resources_dir / filename
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
with Profiler() as prof:
img = AICSImage(f)
assert isinstance(img.metadata, expected_metadata_type)
# Check that basic details don't require task computation
assert len(prof.results) == 0
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
def test_imread_from_delayed(resources_dir, filename, expected_shape, expected_tasks):
# Load image as delayed dask array then as numpy array
# Check computed task count
with dask_utils.cluster_and_client(processes=False) as (cluster, client):
# Get filepath
f = resources_dir / filename
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Read image
img = imread_dask(f)
assert img.shape == expected_shape
assert len(optimize(img)[0].__dask_graph__()) == expected_tasks
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
def test_imread(resources_dir, filename, expected_shape):
# Get filepath
f = resources_dir / filename
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
with Profiler() as prof:
img = imread(f)
assert img.shape == expected_shape
# Reshape and transpose are required so there should be two tasks in the graph
assert len(prof.results) == 2
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
def test_lif_reader(
resources_dir,
filename,
expected_shape,
expected_dims,
expected_dtype,
select_scene,
chunk_dims,
):
# Get file
f = resources_dir / filename
# Check that there are no open file pointers
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Read file
img = LifReader(f, chunk_by_dims=chunk_dims, S=select_scene)
# Check that there are no open file pointers after init
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
assert img.dims == expected_dims
assert img.metadata
assert img.shape == expected_shape
assert img.dask_data.shape == expected_shape
assert img.size(expected_dims) == expected_shape
assert img.dtype() == expected_dtype
# Will error because those dimensions don't exist in the file
with pytest.raises(exceptions.InvalidDimensionOrderingError):
assert img.size("ABCDEFG") == expected_shape
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
# Check array
assert isinstance(img.data, np.ndarray)
assert img.data.shape == expected_shape
# Check that there are no open file pointers after retrieval
assert str(f) not in [f.path for f in proc.open_files()]
def _restart_program(args):
"""Restarts the current program, with file objects and descriptors cleanup"""
try:
p = Process(getpid())
for handler in p.open_files() + p.connections():
close(handler.fd)
except Exception as e:
print("error found > ", e)
python3 = executable
print(args)
execv(python3, ["python3"] + [__file__] + args)
def getOpenFilesList(offset=4):
"""
Returns all currently open files.
Problem: #41 - Too many open files (fdtd side)
"""
myPid = os.getpid()
proc = Process(myPid)
files = proc.open_files()
filesStr = "\n".join(
["%s%s (fd=%s)" % (offset * ' ', f.path, f.fd) for f in files])
numFiles = len(files)
return numFiles, filesStr
def test_dims_setting(resources_dir, expected_starting_dims, set_dims,
expected_ending_dims):
# Get file
f = resources_dir / "example.png"
# Read file
img = DefaultReader(f)
# Check that there are no open file pointers after init
proc = Process()
assert str(f) not in [f.path for f in proc.open_files()]
# Check basics
with Profiler() as prof:
assert img.dims == expected_starting_dims
# Check that basic details don't require task computation
assert len(prof.results) == 0
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
# Check no tasks happen during dims setting
with Profiler() as prof:
img.dims = set_dims
# Check that basic details don't require task computation
assert len(prof.results) == 0
# Check that there are no open file pointers after basics
assert str(f) not in [f.path for f in proc.open_files()]
# Check no tasks happen during dims getting
with Profiler() as prof:
assert img.dims == expected_ending_dims
# Check that basic details don't require task computation
assert len(prof.results) == 0
# Check that there are no open file pointers after retrieval
assert str(f) not in [f.path for f in proc.open_files()]
put(data_dir, os.path.dirname(data_dir))
def restore(options):
''' Restores process state on the destination machine '''
# Err: Restore detach doesn't work
cmd = ("{0} restore -D {1}".format(criu_exec, data_dir).split())
cmd += options
if env.user != 'root':
print ' '.join(cmd)
sudo(' '.join(cmd))
else:
print ' '.join(cmd)
run(' '.join(cmd))
if __name__ == '__main__':
args = parse_arguments()
env.user = args.user
env.host_string = args.host
proc = Process(args.pid)
proc_files = proc.open_files()
options = common_option(args.pid)
print options
dump(args.pid, options)
# Transfer the dump and files to the destination node
transfer_files(proc_files)
restore(options)
