def test_pf_export_non_scalar_filtered_issue_166():
h5path = retrieve_data("fmt-hdf5_image-bg_2020.zip")
# initialize PlugInFeature instance
info = example_plugin_info_non_scalar_feature()
info["feature shapes"] = [(80, 250)]
pf = PlugInFeature("image_gauss_filter", info)
with dclab.new_dataset(h5path) as ds:
# extract the feature information from the dataset
assert pf in PlugInFeature.features
image_gauss_filter = ds[pf.feature_name]
assert np.all(ds.filter.manual) # just checking the default
ds.filter.manual[0] = False
ds.filter.manual[2] = False
image_gauss_filter_filtered = np.array(
image_gauss_filter[ds.filter.manual], copy=True)
ds.apply_filter()
# export the data to a new file
expath = h5path.with_name("exported.rtdc")
ds.export.hdf5(expath, features=[pf.feature_name], filtered=True)
# make sure that worked
with h5py.File(expath, "r") as h5:
assert pf.feature_name in h5["events"]
assert np.allclose(h5["events"][pf.feature_name],
image_gauss_filter_filtered)
def test_pf_export_non_scalar_single_event():
h5path = retrieve_data("fmt-hdf5_image-bg_2020.zip")
# initialize PlugInFeature instance
info = example_plugin_info_non_scalar_feature()
info["feature shapes"] = [(80, 250)]
pf = PlugInFeature("image_gauss_filter", info)
with dclab.new_dataset(h5path) as ds:
# extract the feature information from the dataset
assert pf in PlugInFeature.features
image_gauss_filter = ds[pf.feature_name]
# export the data to a new file
expath = h5path.with_name("exported.rtdc")
ds.export.hdf5(expath, features=["image", pf.feature_name])
# write another single event
with dclab.RTDCWriter(expath) as hw:
hw.store_feature(pf.feature_name, ds["image"][0])
hw.store_feature("image", ds["image"][0])
# make sure that worked
with h5py.File(expath, "r") as h5:
assert pf.feature_name in h5["events"]
assert np.allclose(h5["events"][pf.feature_name][:-1],
image_gauss_filter)
assert np.allclose(h5["events"][pf.feature_name][-1],
h5["events/image"][0])
def test_pf_export_and_load():
"""Check that exported and loaded hdf5 file will keep a plugin feature"""
h5path = retrieve_data("fmt-hdf5_fl_2018.zip")
# initialize PlugInFeature instance
info = example_plugin_info_single_feature()
pf = PlugInFeature("circ_per_area", info)
with dclab.new_dataset(h5path) as ds:
# extract the feature information from the dataset
assert pf in PlugInFeature.features
circ_per_area = ds[pf.feature_name]
# export the data to a new file
expath = h5path.with_name("exported.rtdc")
ds.export.hdf5(expath, features=ds.features_innate + [pf.feature_name])
# make sure that worked
with h5py.File(expath, "r") as h5:
assert pf.feature_name in h5["events"]
assert np.allclose(h5["events"][pf.feature_name], circ_per_area)
# now check again with dclab
with dclab.new_dataset(expath) as ds2:
assert pf in PlugInFeature.features
assert pf.feature_name in ds2
assert pf.feature_name in ds2.features_innate
assert np.allclose(ds2[pf.feature_name], circ_per_area)
# and a control check
remove_plugin_feature(pf)
assert pf.feature_name not in ds2
def test_pf_incorrect_input_method():
"""Raise error when method is not callable"""
info = example_plugin_info_single_feature()
# set `info["method"]` to something that isn't callable
info["method"] = "this_is_a_string"
with pytest.raises(ValueError, match="is not callable"):
PlugInFeature("circ_per_area", info)
def test_pf_export_non_scalar_filtered_from_file_issue_166():
h5path = retrieve_data("fmt-hdf5_image-bg_2020.zip")
expath = h5path.with_name("exported.rtdc")
# initialize PlugInFeature instance
info = example_plugin_info_non_scalar_feature()
info["feature shapes"] = [(80, 250)]
pf = PlugInFeature("image_gauss_filter", info)
# write the plugin feature data to an HDF5 file
with dclab.new_dataset(h5path) as ds:
# extract the feature information from the dataset
ds.export.hdf5(expath, features=[pf.feature_name])
# remove all plugin features and work with temporary feature
remove_all_plugin_features()
# try to load the plugin feature data from that HDF5 file and
# export it again (this time, the exporter has to get the data
# from the H5File object).
expath2 = h5path.with_name("exported2.rtdc")
dclab.register_temporary_feature("image_gauss_filter", is_scalar=False)
with dclab.new_dataset(expath) as ds2:
ds2.export.hdf5(expath2,
features=["image_gauss_filter"],
filtered=True)
# make sure that worked
with h5py.File(expath2, "r") as h5:
assert "image_gauss_filter" in h5["events"]
def test_pf_initialize_plugin_features_multiple():
"""Check multiple plugin features exist independant of loaded dataset"""
ds = dclab.new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip"))
assert "circ_per_area" not in ds.features_innate
assert "circ_times_area" not in ds.features_innate
info = example_plugin_info_multiple_feature()
PlugInFeature("circ_per_area", info)
PlugInFeature("circ_times_area", info)
assert "circ_per_area" in ds
assert "circ_times_area" in ds
assert dclab.dfn.feature_exists("circ_per_area")
assert dclab.dfn.feature_exists("circ_times_area")
circ_per_area = ds["circ_per_area"]
circ_times_area = ds["circ_times_area"]
assert np.allclose(circ_per_area, ds["circ"] / ds["area_um"])
assert np.allclose(circ_times_area, ds["circ"] * ds["area_um"])
def test_pf_attribute_ancill_info():
"""Check the plugin feature attribute input to AncillaryFeature"""
info = example_plugin_info_single_feature()
pf = PlugInFeature("circ_per_area", info)
assert pf.plugin_feature_info["feature name"] == "circ_per_area"
assert pf.plugin_feature_info["method"] is compute_single_plugin_feature
assert pf.plugin_feature_info["config required"] == []
assert pf.plugin_feature_info["features required"] == ["circ", "area_um"]
def test_pf_wrong_data_shape_1():
h5path = retrieve_data("fmt-hdf5_fl_2018.zip")
with dclab.new_dataset(h5path) as ds:
info = example_plugin_info_single_feature()
info["scalar feature"] = [False]
pf = PlugInFeature("circ_per_area", info)
with pytest.raises(ValueError, match="is not a scalar feature"):
ds[pf.feature_name]
def test_pf_with_feature_label():
"""Check that a plugin feature label is added to definitions"""
info = example_plugin_info_single_feature()
info["feature labels"] = ["Circ / Area [1/µm²]"]
feature_name = "circ_per_area"
PlugInFeature(feature_name, info)
assert dclab.dfn.feature_exists("circ_per_area")
label = dclab.dfn.get_feature_label("circ_per_area")
assert label == "Circ / Area [1/µm²]"
def test_pf_wrong_data_shape_2():
h5path = retrieve_data("fmt-hdf5_fl_2018.zip")
with dclab.new_dataset(h5path) as ds:
info = example_plugin_info_single_feature()
info["scalar feature"] = [True]
info["method"] = lambda x: np.arange(len(ds) * 2).reshape(-1, 2)
pf = PlugInFeature("circ_per_area", info)
with pytest.raises(ValueError, match="is a scalar feature"):
ds[pf.feature_name]
def test_pf_input_no_feature_labels():
"""Check that feature labels are populated even if not given"""
info = example_plugin_info_single_feature()
info.pop("feature labels")
feature_name = "circ_per_area"
pf = PlugInFeature(feature_name, info)
assert dclab.dfn.feature_exists(feature_name)
label = dclab.dfn.get_feature_label(feature_name)
assert label == "Plugin feature {}".format(feature_name)
assert label == pf.plugin_feature_info["feature label"]
def test_pf_initialize_plugin_feature_non_scalar():
"""Check that the non-scalar plugin feature works"""
ds = dclab.new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip"))
info = example_plugin_info_non_scalar_feature()
PlugInFeature("image_gauss_filter", info)
assert "image_gauss_filter" in ds
image_gauss_filter = ds["image_gauss_filter"]
assert np.allclose(image_gauss_filter,
gaussian_filter(ds["image"], sigma=(0, 1, 1)))
def test_pf_inherited_scalar():
"""Scalar inherited PluginFeatures should be a 1D np.ndarray"""
info = example_plugin_info_single_feature()
PlugInFeature("circ_per_area", info)
with dclab.new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip")) as ds:
ds.filter.manual[2] = False
ch = dclab.new_dataset(ds)
assert "circ_per_area" in ch
assert isinstance(ch["circ_per_area"], np.ndarray)
assert ch["circ_per_area"].ndim == 1
def test_pf_inherited_non_scalar():
"""Non scalar inherited plugin features should be of class ChildNDArray"""
info = example_plugin_info_non_scalar_feature()
PlugInFeature("image_gauss_filter", info)
with dclab.new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip")) as ds:
ds.filter.manual[2] = False
ch = dclab.new_dataset(ds)
assert "image_gauss_filter" in ch
assert isinstance(ch["image_gauss_filter"], ChildNDArray)
assert len(ch["image_gauss_filter"].shape) == 3
def test_pf_exists_in_hierarchy():
"""Test that RTDCHierarchy works with PlugInFeature"""
info = example_plugin_info_single_feature()
pf = PlugInFeature("circ_per_area", info)
h5path = retrieve_data("fmt-hdf5_fl_2018.zip")
with dclab.new_dataset(h5path) as ds:
assert pf.feature_name in ds
assert dclab.dfn.feature_exists(pf.feature_name)
child = dclab.new_dataset(ds)
assert pf.feature_name in child
def test_pf_wrong_length_2():
"""plugin feature should have same length"""
h5path = retrieve_data("fmt-hdf5_fl_2018.zip")
with dclab.new_dataset(h5path) as ds:
info = example_plugin_info_single_feature()
info["method"] = lambda x: np.arange(len(ds) * 2)
pf = PlugInFeature("circ_per_area", info)
with pytest.warns(BadFeatureSizeWarning,
match="to match event number"):
ds[pf.feature_name]
def test_pf_filtering_with_plugin_feature():
"""Filtering with plugin feature"""
h5path = retrieve_data("fmt-hdf5_fl_2018.zip")
with dclab.new_dataset(h5path) as ds:
info = example_plugin_info_single_feature()
pf = PlugInFeature("circ_per_area", info)
ds.config["filtering"][f"{pf.feature_name} min"] = 0.030
ds.config["filtering"][f"{pf.feature_name} max"] = 0.031
ds.apply_filter()
assert np.sum(ds.filter.all) == 1
assert ds.filter.all[4]
def test_pf_initialize_plugin_feature_single():
"""Check that single plugin feature exists independant of loaded dataset"""
ds = dclab.new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip"))
info = example_plugin_info_single_feature()
PlugInFeature("circ_per_area", info)
assert "circ_per_area" in ds
circ_per_area = ds["circ_per_area"]
assert np.allclose(circ_per_area, ds["circ"] / ds["area_um"])
# check that PlugInFeature exists independent of loaded ds
ds2 = dclab.new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip"))
assert "circ_per_area" in ds2
def test_pf_initialize_plugin_after_loading():
"""plugin feature loads correctly after feature added to hdf5 file"""
h5path = retrieve_data("fmt-hdf5_fl_2018.zip")
with dclab.new_dataset(h5path) as ds:
circ_per_area = compute_single_plugin_feature(ds)
with h5py.File(h5path, "a") as h5:
h5["events"]["circ_per_area"] = circ_per_area
with dclab.new_dataset(h5path) as ds:
assert "circ_per_area" not in ds
info = example_plugin_info_single_feature()
PlugInFeature("circ_per_area", info)
assert "circ_per_area" in ds
assert "circ_per_area" in ds.features_innate
def test_pf_with_no_feature_label():
"""A feature label of None is replaced with a real feature label
Show that `feature_label=None` will still give a descriptive
feature label. See `dclab.dfn._add_feature_to_definitions` for details.
"""
info = example_plugin_info_single_feature()
info["feature labels"] = [None]
feature_name = "circ_per_area"
PlugInFeature(feature_name, info)
assert dclab.dfn.feature_exists("circ_per_area")
label = dclab.dfn.get_feature_label("circ_per_area")
assert label is not None
assert label == "Plugin feature {}".format(feature_name)
def test_pf_with_empty_feature_label_string():
"""An empty string is replaced with a real feature label
Show that an empty `feature_label` will still give a descriptive
feature label. See `dclab.dfn._add_feature_to_definitions` for details.
"""
info = example_plugin_info_single_feature()
info["feature labels"] = [""]
feature_name = "circ_per_area"
PlugInFeature(feature_name, info)
assert dclab.dfn.feature_exists("circ_per_area")
label = dclab.dfn.get_feature_label("circ_per_area")
assert label != ""
assert label == "Plugin feature {}".format(feature_name)
def test_pf_load_scalar_plugin_data():
"""Test loading scalar plugin data return np.ndarray"""
ds = dclab.new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip"))
info = example_plugin_info_single_feature()
PlugInFeature("circ_per_area", info)
assert isinstance(ds["circ_per_area"], np.ndarray)
# Exporting the rtdc-file including the plugin-feature and then reloading
# it should still load scalar plugin data as np.ndarray
tdir = tempfile.mkdtemp()
pdir = pathlib.Path(tdir)
pfile = pdir / "tmp.rtdc"
features = ds.features + ["circ_per_area"]
ds.export.hdf5(pfile, features=features)
ds2 = dclab.new_dataset(pfile)
assert isinstance(ds2["circ_per_area"], np.ndarray)
def test_pf_export_non_scalar():
h5path = retrieve_data("fmt-hdf5_image-bg_2020.zip")
# initialize PlugInFeature instance
info = example_plugin_info_non_scalar_feature()
pf = PlugInFeature("image_gauss_filter", info)
with dclab.new_dataset(h5path) as ds:
# extract the feature information from the dataset
assert pf in PlugInFeature.features
image_gauss_filter = ds[pf.feature_name]
# export the data to a new file
expath = h5path.with_name("exported.rtdc")
ds.export.hdf5(expath, features=[pf.feature_name])
# make sure that worked
with h5py.File(expath, "r") as h5:
assert pf.feature_name in h5["events"]
assert np.allclose(h5["events"][pf.feature_name], image_gauss_filter)
def test_pf_with_user_config_section_fails():
"""Use a plugin feature with the user defined config section"""
info = {
"method": compute_with_user_section,
"feature names": ["area_of_region"],
"config required": [["user", ["n_constrictions"]]]
}
PlugInFeature("area_of_region", info)
ds = dclab.new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip"))
# show that the plugin feature is not available before setting the
# user metadata
ds.config["user"].clear()
with pytest.raises(KeyError,
match=r"Feature \'area_of_region\' does not exist"):
ds["area_of_region"]
# show that the plugin fails when the user metadata type is wrong
ds.config["user"]["n_constrictions"] = 4.99
with pytest.raises(AssertionError, match="should be an integer value"):
ds["area_of_region"]
def test_pf_load_non_scalar_plugin_data():
"""Test loading non-scalar plugin data format"""
ds = dclab.new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip"))
info = example_plugin_info_non_scalar_feature()
info["feature shapes"] = [(80, 250)]
PlugInFeature("image_gauss_filter", info)
# Accessing non-scalar plugin data without prior saving and storing data in
# HDF5-format should return already computed data as np.ndarray
assert isinstance(ds["image_gauss_filter"], np.ndarray)
# Exporting the rtdc-file including the plugin-feature and then reloading
# it should load the non-scalar plugin data as h5py.Dataset
tdir = tempfile.mkdtemp()
pdir = pathlib.Path(tdir)
pfile = pdir / "tmp.rtdc"
features = ds.features + ["image_gauss_filter"]
ds.export.hdf5(pfile, features=features)
ds2 = dclab.new_dataset(pfile)
assert isinstance(ds2["image_gauss_filter"], h5py.Dataset)
def test_pf_with_user_config_section():
"""Use a plugin feature with the user defined config section"""
info = {
"method": compute_with_user_section,
"feature names": ["area_of_region"],
"config required": [["user", ["n_constrictions"]]]
}
PlugInFeature("area_of_region", info)
ds = dclab.new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip"))
assert "area_of_region" not in ds, "not available b/c missing metadata"
# add some metadata to the user config section
metadata = {"channel": True, "n_constrictions": 3}
ds.config["user"].update(metadata)
assert ds.config["user"] == metadata
assert "area_of_region" in ds, "available b/c metadata is set"
area_of_region1 = ds["area_of_region"]
area_of_region1_calc = (ds["area_um"] *
ds.config["user"]["n_constrictions"])
assert np.allclose(area_of_region1, area_of_region1_calc)
def test_pf_minimum_info_input():
"""Only method and feature names are required to create PlugInFeature"""
info = {
"method": compute_single_plugin_feature,
"feature names": ["circ_per_area"]
}
pf = PlugInFeature("circ_per_area", info)
# check that all other plugin_feature_info is populated
assert "method" in pf.plugin_feature_info
assert callable(pf.plugin_feature_info["method"])
assert "description" in pf.plugin_feature_info
assert "long description" in pf.plugin_feature_info
assert "feature name" in pf.plugin_feature_info
assert "feature label" in pf.plugin_feature_info
assert "features required" in pf.plugin_feature_info
assert "config required" in pf.plugin_feature_info
assert "method check required" in pf.plugin_feature_info
assert "scalar feature" in pf.plugin_feature_info
assert "version" in pf.plugin_feature_info
assert "plugin path" in pf.plugin_feature_info
def test_pf_attribute_plugin_feature_info():
"""Check the plugin feature info attribute"""
info = example_plugin_info_single_feature()
# comparing lambda functions fails due to differing memory locations
info.pop("method check required")
pf = PlugInFeature("circ_per_area", info)
pf.plugin_feature_info.pop("method check required")
plugin_feature_info = {
"method": compute_single_plugin_feature,
"description": "This plugin will compute a feature",
"long description": "Even longer description that "
"can span multiple lines",
"feature name": "circ_per_area",
"feature label": "Circularity per Area",
"feature shape": (1, ),
"features required": ["circ", "area_um"],
"config required": [],
"scalar feature": True,
"version": "0.1.0",
"plugin path": None,
"identifier": "3a3e72c4cb015424ebbe6d4af63f2170",
}
assert pf.plugin_feature_info == plugin_feature_info
def test_pf_input_no_scalar_feature():
"""Check that scalar feature bools are populated even if not given"""
info = example_plugin_info_single_feature()
info.pop("scalar feature")
pf = PlugInFeature("circ_per_area", info)
assert pf.plugin_feature_info["scalar feature"]
def test_pf_try_existing_feature_fails():
"""An existing feature name is not allowed"""
info = example_plugin_info_single_feature()
info["feature names"] = ["deform"]
with pytest.raises(ValueError, match="Feature 'deform' already exists"):
PlugInFeature("deform", info)