def test_default_reader(resources_dir, filename, expected_shape, expected_dims, expected_chunksize, expected_task_count): # Get file f = resources_dir / filename # Read file img = DefaultReader(f) # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics with Profiler() as prof: assert img.dims == expected_dims assert img.metadata assert img.dask_data.shape == expected_shape assert img.dask_data.chunksize == expected_chunksize # Check that basic details don't require task computation assert len(prof.results) == 0 # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()] # Check computed type is numpy array, computed shape is expected shape, and task count is expected with Profiler() as prof: assert isinstance(img.data, np.ndarray) assert img.data.shape == expected_shape assert len(prof.results) == expected_task_count # Check that there are no open file pointers after retrieval assert str(f) not in [f.path for f in proc.open_files()]
def test_imread_use_dask_false(resources_dir): # Load image as delayed dask array then as numpy array # Check computed task count with dask_utils.cluster_and_client(processes=False) as (cluster, client): # Get filepath f = resources_dir / BIG_OME_FILE # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check that a client does exist get_client() # Don't use dask for reads use_dask(False) # Read image without dask img = AICSImage(f) assert img.data.shape == (3, 1, 3, 5, 325, 475) # Check that the file was read with base reader then rechunked with dask # Normally the task count for this file is 90 assert len(optimize(img.dask_data)[0].__dask_graph__()) == 3 # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()]
def test_size_functions(resources_dir, filename, s, t, c, z, y, x): # Get file f = resources_dir / filename # Check that there are no open file pointers proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Init reader img = LifReader(f) # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check sizes assert img.size_s() == s assert img.size_t() == t assert img.size_c() == c assert img.size_z() == z assert img.size_y() == y assert img.size_x() == x # Check that there are no open file pointers assert str(f) not in [f.path for f in proc.open_files()]
def test_czi_reader( resources_dir, filename, expected_shape, expected_dims, expected_dtype, select_scene, chunk_dims, ): # Get file f = resources_dir / filename # Read file img = CziReader(f, chunk_by_dims=chunk_dims, S=select_scene) # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics assert img.dims == expected_dims assert img.metadata is not None assert img.dask_data.shape == expected_shape assert img.dtype() == expected_dtype # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()] # Check array assert isinstance(img.data, np.ndarray) assert img.data.shape == expected_shape # Check that there are no open file pointers after retrieval assert str(f) not in [f.path for f in proc.open_files()]
def test_default_reader( resources_dir, filename, expected_shape, expected_dims, ): # Get file f = resources_dir / filename # Read file img = DefaultReader(f) # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics assert img.dims == expected_dims assert img.metadata assert img.dask_data.shape == expected_shape # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()] # Check array assert isinstance(img.data, np.ndarray) assert img.data.shape == expected_shape # Check that there are no open file pointers after retrieval assert str(f) not in [f.path for f in proc.open_files()]
def test_dims_setting(resources_dir, expected_starting_dims, set_dims, expected_ending_dims): # Get file f = resources_dir / "example.png" # Read file img = DefaultReader(f) # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics assert img.dims == expected_starting_dims # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()] # Update dims img.dims = set_dims # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()] # Check expected assert img.dims == expected_ending_dims # Check that there are no open file pointers after retrieval assert str(f) not in [f.path for f in proc.open_files()]
def test_get_channel_names(resources_dir, filename, scene, expected): f = resources_dir / filename # Check that there are no open file pointers proc = Process() assert str(f) not in [f.path for f in proc.open_files()] assert LifReader(resources_dir / filename).get_channel_names(scene) == expected # Check that there are no open file pointers assert str(f) not in [f.path for f in proc.open_files()]
def test_lif_image_data_two(resources_dir, filename, scene, expected): f = resources_dir / filename # Check that there are no open file pointers proc = Process() assert str(f) not in [f.path for f in proc.open_files()] img = LifReader(f) assert img._chunk_offsets[0][0, 0, 0] == expected # Check that there are no open file pointers assert str(f) not in [f.path for f in proc.open_files()]
def test_get_physical_pixel_size(resources_dir, filename, scene, expected): f = resources_dir / filename # Check that there are no open file pointers proc = Process() assert str(f) not in [f.path for f in proc.open_files()] assert LifReader(resources_dir / filename).get_physical_pixel_size(scene) == pytest.approx( expected, rel=0.001) # Check that there are no open file pointers assert str(f) not in [f.path for f in proc.open_files()]
def test_ome_tiff_reader( resources_dir, filename, expected_shape, expected_dims, select_scene, ): # Get file f = resources_dir / filename # Read file img = OmeTiffReader(f, S=select_scene) # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics # Check that OME Metadata matches the dask data array shape and dims order dim_size_getters = { Dimensions.Scene: img.size_s, Dimensions.Time: img.size_t, Dimensions.Channel: img.size_c, Dimensions.SpatialZ: img.size_z, Dimensions.SpatialY: img.size_y, Dimensions.SpatialX: img.size_x, } for d, getter in dim_size_getters.items(): if d in expected_dims: assert getter() == img.dask_data.shape[img.dims.index(d)] assert img.dims == expected_dims assert img.is_ome() assert img.metadata assert img.shape == expected_shape assert img.dask_data.shape == expected_shape assert img.size(expected_dims) == expected_shape # Will error because those dimensions don't exist in the file with pytest.raises(exceptions.InvalidDimensionOrderingError): assert img.size("ABCDEFG") == expected_shape # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()] # Check array assert isinstance(img.data, np.ndarray) assert img.data.shape == expected_shape # Check that there are no open file pointers after retrieval assert str(f) not in [f.path for f in proc.open_files()]
def test_file_passed_was_directory(resources_dir): # Get filepath f = resources_dir # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics with pytest.raises(IsADirectoryError): AICSImage(resources_dir) # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()]
def test_aicsimage_serialize( resources_dir, tmpdir, filename, expected_shape, expected_metadata_type, expected_task_count, ): """ Test that the entire AICSImage object can be serialized - a requirement to distribute on dask clusters. https://distributed.dask.org/en/latest/serialization.html """ # Get file f = resources_dir / filename # Read file img = AICSImage(f) # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics with Profiler() as prof: assert img.shape == expected_shape assert isinstance(img.metadata, expected_metadata_type) # Check that basic details don't require task computation assert len(prof.results) == 0 # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()] # Serialize object serialized = pickle.dumps(img) # Reload img = pickle.loads(serialized) # Check computed type is numpy array, computed shape is expected shape, and task count is expected with Profiler() as prof: assert isinstance(img.data, np.ndarray) assert img.shape == expected_shape assert img.data.shape == expected_shape assert isinstance(img.metadata, expected_metadata_type) assert len(prof.results) == expected_task_count # Check that there are no open file pointers after retrieval assert str(f) not in [f.path for f in proc.open_files()]
def test_physical_pixel_size(resources_dir, filename, expected_sizes): # Get filepath f = resources_dir / filename # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics img = AICSImage(f) assert img.get_physical_pixel_size() == expected_sizes # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()]
def test_imread(resources_dir, filename, expected_shape): # Get filepath f = resources_dir / filename # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics img = imread(f) assert img.shape == expected_shape # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()]
def test_typing(filename, expected_reader, resources_dir): # Get filepath f = resources_dir / filename # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics actual_reader = AICSImage.determine_reader(f) assert actual_reader == expected_reader # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()]
def test_metadata(resources_dir, filename, expected_metadata_type): # Get filepath f = resources_dir / filename # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics img = AICSImage(f) assert isinstance(img.metadata, expected_metadata_type) # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()]
def test_channel_names(resources_dir, filename, expected_channel_names): # Get filepath f = resources_dir / filename # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics img = AICSImage(f) assert img.get_channel_names() == expected_channel_names assert len(img.get_channel_names()) == img.size_c # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()]
def test_get_channel_names(resources_dir, filename, scene, expected_channel_names): # Get file f = resources_dir / filename # Read file img = OmeTiffReader(f) # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check channel names assert img.get_channel_names(scene) == expected_channel_names # Check that there are no open file pointers after check assert str(f) not in [f.path for f in proc.open_files()]
def test_ome_tiff_reader(resources_dir, filename, expected_shape, expected_dims, select_scene, expected_chunksize, expected_task_count): # Get file f = resources_dir / filename # Read file img = OmeTiffReader(f, S=select_scene) # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics with Profiler() as prof: # Check that OME Metadata matches the dask data array shape and dims order dim_size_getters = { Dimensions.Scene: img.size_s, Dimensions.Time: img.size_t, Dimensions.Channel: img.size_c, Dimensions.SpatialZ: img.size_z, Dimensions.SpatialY: img.size_y, Dimensions.SpatialX: img.size_x } for d, getter in dim_size_getters.items(): if d in expected_dims: assert getter() == img.dask_data.shape[img.dims.index(d)] assert img.dims == expected_dims assert img.is_ome() assert img.metadata assert img.dask_data.shape == expected_shape assert img.dask_data.chunksize == expected_chunksize # Check that basic details don't require task computation assert len(prof.results) == 0 # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()] # Check computed type is numpy array, computed shape is expected shape, and task count is expected with Profiler() as prof: assert isinstance(img.data, np.ndarray) assert img.data.shape == expected_shape assert len(prof.results) == expected_task_count # Check that there are no open file pointers after retrieval assert str(f) not in [f.path for f in proc.open_files()]
def test_file_passed_was_directory(resources_dir): # Get filepath f = resources_dir # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics with Profiler() as prof: with pytest.raises(IsADirectoryError): AICSImage(resources_dir) # Check that basic details don't require task computation assert len(prof.results) == 0 # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()]
def test_typing(filename, expected_reader, resources_dir): # Get filepath f = resources_dir / filename # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics with Profiler() as prof: actual_reader = AICSImage.determine_reader(f) assert actual_reader == expected_reader # Check that basic details don't require task computation assert len(prof.results) == 0 # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()]
def test_large_imread_dask(resources_dir, filename, expected_shape, expected_task_count): # Get filepath f = resources_dir / filename # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics with Profiler() as prof: img = imread_dask(f) assert img.shape == expected_shape assert len(prof.results) == expected_task_count # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()]
def test_physical_pixel_size(resources_dir, filename, expected_sizes): # Get filepath f = resources_dir / filename # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics with Profiler() as prof: img = AICSImage(f) assert img.get_physical_pixel_size() == expected_sizes # Check that basic details don't require task computation assert len(prof.results) == 0 # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()]
def test_metadata(resources_dir, filename, expected_metadata_type): # Get filepath f = resources_dir / filename # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics with Profiler() as prof: img = AICSImage(f) assert isinstance(img.metadata, expected_metadata_type) # Check that basic details don't require task computation assert len(prof.results) == 0 # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()]
def test_imread_from_delayed(resources_dir, filename, expected_shape, expected_tasks): # Load image as delayed dask array then as numpy array # Check computed task count with dask_utils.cluster_and_client(processes=False) as (cluster, client): # Get filepath f = resources_dir / filename # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Read image img = imread_dask(f) assert img.shape == expected_shape assert len(optimize(img)[0].__dask_graph__()) == expected_tasks # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()]
def test_imread(resources_dir, filename, expected_shape): # Get filepath f = resources_dir / filename # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics with Profiler() as prof: img = imread(f) assert img.shape == expected_shape # Reshape and transpose are required so there should be two tasks in the graph assert len(prof.results) == 2 # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()]
def test_lif_reader( resources_dir, filename, expected_shape, expected_dims, expected_dtype, select_scene, chunk_dims, ): # Get file f = resources_dir / filename # Check that there are no open file pointers proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Read file img = LifReader(f, chunk_by_dims=chunk_dims, S=select_scene) # Check that there are no open file pointers after init assert str(f) not in [f.path for f in proc.open_files()] # Check basics assert img.dims == expected_dims assert img.metadata assert img.shape == expected_shape assert img.dask_data.shape == expected_shape assert img.size(expected_dims) == expected_shape assert img.dtype() == expected_dtype # Will error because those dimensions don't exist in the file with pytest.raises(exceptions.InvalidDimensionOrderingError): assert img.size("ABCDEFG") == expected_shape # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()] # Check array assert isinstance(img.data, np.ndarray) assert img.data.shape == expected_shape # Check that there are no open file pointers after retrieval assert str(f) not in [f.path for f in proc.open_files()]
def _restart_program(args): """Restarts the current program, with file objects and descriptors cleanup""" try: p = Process(getpid()) for handler in p.open_files() + p.connections(): close(handler.fd) except Exception as e: print("error found > ", e) python3 = executable print(args) execv(python3, ["python3"] + [__file__] + args)
def getOpenFilesList(offset=4): """ Returns all currently open files. Problem: #41 - Too many open files (fdtd side) """ myPid = os.getpid() proc = Process(myPid) files = proc.open_files() filesStr = "\n".join( ["%s%s (fd=%s)" % (offset * ' ', f.path, f.fd) for f in files]) numFiles = len(files) return numFiles, filesStr
def test_dims_setting(resources_dir, expected_starting_dims, set_dims, expected_ending_dims): # Get file f = resources_dir / "example.png" # Read file img = DefaultReader(f) # Check that there are no open file pointers after init proc = Process() assert str(f) not in [f.path for f in proc.open_files()] # Check basics with Profiler() as prof: assert img.dims == expected_starting_dims # Check that basic details don't require task computation assert len(prof.results) == 0 # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()] # Check no tasks happen during dims setting with Profiler() as prof: img.dims = set_dims # Check that basic details don't require task computation assert len(prof.results) == 0 # Check that there are no open file pointers after basics assert str(f) not in [f.path for f in proc.open_files()] # Check no tasks happen during dims getting with Profiler() as prof: assert img.dims == expected_ending_dims # Check that basic details don't require task computation assert len(prof.results) == 0 # Check that there are no open file pointers after retrieval assert str(f) not in [f.path for f in proc.open_files()]
put(data_dir, os.path.dirname(data_dir)) def restore(options): ''' Restores process state on the destination machine ''' # Err: Restore detach doesn't work cmd = ("{0} restore -D {1}".format(criu_exec, data_dir).split()) cmd += options if env.user != 'root': print ' '.join(cmd) sudo(' '.join(cmd)) else: print ' '.join(cmd) run(' '.join(cmd)) if __name__ == '__main__': args = parse_arguments() env.user = args.user env.host_string = args.host proc = Process(args.pid) proc_files = proc.open_files() options = common_option(args.pid) print options dump(args.pid, options) # Transfer the dump and files to the destination node transfer_files(proc_files) restore(options)