def test_ic_fl_max_ctc(): # Testing dataset with negative fl_max_ctc values ddict = example_data_dict(size=8472, keys=["fl1_max_ctc"]) ddict["fl1_max_ctc"] -= min(ddict["fl1_max_ctc"]) + 1 ds = new_dataset(ddict) with check.IntegrityChecker(ds) as ic: cues = ic.check_fl_max_ctc_positive() assert cues[0].level == "alert" assert cues[0].category == "feature data" # Testing dataset with fl_max_ctc values of 0.1 ddict = example_data_dict(size=8472, keys=["fl1_max_ctc"]) ddict["fl1_max_ctc"] -= min(ddict["fl1_max_ctc"]) - 0.1 ds = new_dataset(ddict) with check.IntegrityChecker(ds) as ic: cues = ic.check_fl_max_ctc_positive() assert cues[0].level == "alert" assert cues[0].category == "feature data" # Testing dataset with fl_max_ctc values > 0.1 ddict = example_data_dict(size=8472, keys=["fl1_max_ctc"]) ddict["fl1_max_ctc"] -= min(ddict["fl1_max_ctc"]) - 1 ds = new_dataset(ddict) with check.IntegrityChecker(ds) as ic: cues = ic.check_fl_max_ctc_positive() assert not cues
def test_ic_expand_section(): ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds1 = new_dataset(ddict) ds2 = new_dataset(ddict) with check.IntegrityChecker(ds1) as ic: cues1 = ic.check_metadata_missing(expand_section=True) with check.IntegrityChecker(ds2) as ic: cues2 = ic.check_metadata_missing(expand_section=False) assert len(cues1) > len(cues2)
def test_user_section_set_save_reload_empty_dict(): """The 'user' config section as an empty dict will not save""" h5path = retrieve_data("fmt-hdf5_fl_2018.zip") with new_dataset(h5path) as ds: ds.config.update({"user": {}}) expath = h5path.with_name("exported.rtdc") ds.export.hdf5(expath, features=ds.features_innate) # nothing "user"-like is written to the HDF5 attributes with h5py.File(expath, "r") as h5: for ak in h5.attrs: assert not ak.startswith("user") # works for dclab because "user" added when checked with new_dataset(expath) as ds2: assert ds2.config["user"] == {}
def test_contour_from_hdf5(): ds1 = new_dataset(retrieve_data("fmt-hdf5_image-bg_2020.zip")) assert ds1["contour"].shape == (5, np.nan, 2) edest = tempfile.mkdtemp() f1 = join(edest, "dclab_test_export_hdf5_image.rtdc") with RTDCWriter(f1) as hw: hw.store_metadata({"setup": ds1.config["setup"], "experiment": ds1.config["experiment"]}) hw.store_feature("deform", ds1["deform"]) hw.store_feature("contour", ds1["contour"]) ds2 = new_dataset(f1) assert ds2["contour"].shape == (5, np.nan, 2)
def test_lower_case_conversion_issue_139(): """Check that chip region is lower-case""" h5path = retrieve_data("fmt-hdf5_polygon_gate_2021.zip") with h5py.File(h5path, "a") as h5: h5.attrs["setup:chip region"] = "Channel" with new_dataset(h5path) as ds: assert ds.config["setup"]["chip region"] == "channel"
def test_user_section_exists(): """Check that the user config section exists""" ds = new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip")) assert ds.config["user"] == {} # control: nonsense sections don't exist with pytest.raises(KeyError): ds.config["Oh I seem to have lost my key"]
def test_user_section_set_save_reload_fmt_dcor(): """Check that 'user' section metadata works for RTDC_DCOR""" # create temp directory for storing outputted file tpath = pathlib.Path(tempfile.mkdtemp()) with new_dataset("fb719fb2-bd9f-817a-7d70-f4002af916f0") as ds: metadata = {"some metadata": 12} ds.config.update({"user": metadata}) assert ds.config["user"] == metadata expath = tpath / "exported.rtdc" ds.export.hdf5(expath, features=["deform", "area_um"]) # make sure that worked with h5py.File(expath, "r") as h5: assert h5.attrs["user:some metadata"] == 12 # check it again with dclab with new_dataset(expath) as ds2: assert ds2.config["user"] == metadata
def test_filter_manual(): # make sure min/max values are filtered ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) ds.filter.manual[[0, 8471]] = False ds.apply_filter() assert len(ds["deform"][ds.filter.all]) == 8470 assert ds["deform"][1] == ds["deform"][ds.filter.all][0]
def test_ic_fl_metadata_channel_names(): ddict = example_data_dict(size=8472, keys=["area_um", "deform", "fl1_max"]) ddict["trace"] = {"fl1_raw": [range(10)] * 1000} ds = new_dataset(ddict) with check.IntegrityChecker(ds) as ic: cues = ic.check_fl_metadata_channel_names() assert cues[0].category == "metadata missing" assert cues[0].cfg_section == "fluorescence" assert cues[0].cfg_key == "channel 1 name"
def test_disable_filters(): """Disabling the filters should only affect RTDCBase.filter.all""" ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) ds.filter.manual[[0, 8471]] = False ds.apply_filter() ds.config["filtering"]["enable filters"] = False ds.apply_filter() assert np.alltrue(ds.filter.all)
def test_temperature(): # there are probably a million things wrong with this dataset, but # we are only looking for the temperature thing ddict = example_data_dict(size=8472, keys=["area_um", "deform", "temp"]) ds = new_dataset(ddict) sstr = "Metadata: Missing key [setup] 'temperature', " \ + "because the 'temp' feature is given" _, aler, _ = check_dataset(ds) assert sstr in aler
def test_shapein_issue3_bad_medium_control(si_version): h5path = retrieve_data("fmt-hdf5_fl_2018.zip") with h5py.File(h5path, "a") as h5: h5.attrs["setup:software version"] = si_version h5.attrs["setup:medium"] = "CellCarrierB" ds = new_dataset(h5path) with check.IntegrityChecker(ds) as ic: cues = ic.check_shapein_issue3_bad_medium() assert len(cues) == 0
def test_ic_fl_metadata_channel_names_2(): ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) ds.config["fluorescence"]["channel 1 name"] = "peter" with check.IntegrityChecker(ds) as ic: cues = ic.check_fl_metadata_channel_names() assert cues[0].category == "metadata invalid" assert cues[0].cfg_section == "fluorescence" assert cues[0].cfg_key == "channel 1 name"
def test_ic_metadata_choices_medium(): ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) ds.config["setup"]["medium"] = "honey" with check.IntegrityChecker(ds) as ic: cues = ic.check_metadata_choices() # changed in 0.29.1: medium can now be an arbitrary string # except for an empty string. assert len(cues) == 0
def test_ic_flow_rate_not_zero(): ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) ds.config["setup"]["flow rate"] = 0 with check.IntegrityChecker(ds) as ic: cues = ic.check_metadata_bad_greater_zero() assert cues[0].category == "metadata wrong" assert cues[0].cfg_section == "setup" assert cues[0].cfg_key == "flow rate"
def test_ic_fl_num_channels(): ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) ds.config["fluorescence"]["channel count"] = 3 ds.config["fluorescence"]["channel 1 name"] = "hans" with check.IntegrityChecker(ds) as ic: cues = ic.check_fl_num_channels() assert cues[0].category == "metadata wrong" assert cues[0].cfg_section == "fluorescence" assert cues[0].cfg_key == "channel count"
def test_ic_metadata_bad(): ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) # Since version 0.35, metadata are checked in `Configuration` class with pytest.warns(dclab.rtdc_dataset.config.WrongConfigurationTypeWarning, match="run index"): ds.config["experiment"]["run index"] = "1" with check.IntegrityChecker(ds) as ic: cues = ic.check_metadata_bad() assert len(cues) == 0
def test_ic_invalid_dataset(): # Testing if IC throws NotImplementedError for hierarchy datasets ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) ds_child = new_dataset(ds) with check.IntegrityChecker(ds_child) as ic: with pytest.raises(NotImplementedError): ic.check() # Testing if IC throws NotImplementedError for raw-datasets with # applied filters ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) ds.config["filtering"]["area_um max"] = 100 ds.config["filtering"]["area_um min"] = 1 ds.apply_filter() with check.IntegrityChecker(ds) as ic: with pytest.raises(NotImplementedError): ic.check()
def test_user_section_basic(): """Add information to the user section of config""" ds = new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip")) metadata = { "channel area": 100.5, "inlet": True, "n_constrictions": 3, "channel information": "other information" } ds.config.update({"user": metadata}) assert ds.config["user"] == metadata
def test_ml_class(): """Test score data outside boundary""" data = { "ml_score_001": [.1, 10, -10, 0.01, .89], "ml_score_002": [.2, .1, .4, 0, .4], } ds = new_dataset(data) with check.IntegrityChecker(ds) as ic: cues = ic.check_ml_class() assert len(cues) == 1 assert "ml_score_001" in cues[0].msg
def test_ic_fl_num_lasers(): ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) ds.config["fluorescence"]["laser count"] = 3 ds.config["fluorescence"]["laser 1 lambda"] = 550 ds.config["fluorescence"]["laser 1 power"] = 20 with check.IntegrityChecker(ds) as ic: cues = ic.check_fl_num_lasers() assert cues[0].category == "metadata wrong" assert cues[0].cfg_section == "fluorescence" assert cues[0].cfg_key == "laser count"
def test_index_increment(): rtdc_file = tempfile.mktemp(suffix=".rtdc", prefix="dclab_test_error_") with RTDCWriter(rtdc_file) as hw: hw.store_metadata({"experiment": {"sample": "test", "run index": 1}}) hw.store_feature("index", np.arange(1, 11)) hw.store_feature("index", np.arange(1, 11)) with new_dataset(rtdc_file) as ds: assert np.all(ds["index"] == np.arange(1, 21))
def test_user_section_set_save_reload_fmt_hdf5_containers(): """Check that 'user' section metadata works for container data types""" h5path = retrieve_data("fmt-hdf5_fl_2018.zip") channel_area = [0, 100] inlet = (1, 20, 40) outlet = np.array(inlet) metadata = {"channel area": channel_area, "inlet": inlet, "outlet": outlet} with new_dataset(h5path) as ds: ds.config.update({"user": metadata}) expath = h5path.with_name("exported.rtdc") ds.export.hdf5(expath, features=ds.features_innate) # make sure that worked with h5py.File(expath, "r") as h5: assert all(h5.attrs["user:channel area"] == channel_area) assert all(h5.attrs["user:inlet"] == inlet) assert all(h5.attrs["user:outlet"] == outlet) # now check again with dclab with new_dataset(expath) as ds2: for k1 in metadata: assert all(metadata[k1] == ds2.config["user"][k1])
def test_user_section_set_and_overwrite(): """Add information to the user section of config via dict.__setitem__""" ds = new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip")) ds.config["user"]["some metadata"] = 42 assert ds.config["user"] == {"some metadata": 42} metadata = {"more metadata": True} ds.config.update({"user": metadata}) assert ds.config["user"] == {"some metadata": 42, "more metadata": True} # overwrite the previous keys and values ds.config["user"] = {} assert ds.config["user"] == {}
def test_shapein_issue3_bad_medium(si_version): h5path = retrieve_data("fmt-hdf5_fl_2018.zip") with h5py.File(h5path, "a") as h5: h5.attrs["setup:software version"] = si_version h5.attrs["setup:medium"] = "CellCarrierB" ds = new_dataset(h5path) with check.IntegrityChecker(ds) as ic: cues = ic.check_shapein_issue3_bad_medium() assert len(cues) == 1 assert cues[0].cfg_key == "medium" assert cues[0].cfg_section == "setup" assert cues[0].category == "metadata wrong"
def test_only_one_boundary_error(): ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ddict["area_um"][[1, 4, 6]] = np.nan ds = new_dataset(ddict) amin, amax = np.nanmin(ds["area_um"]), np.nanmax(ds["area_um"]) ds.config["filtering"]["area_um min"] = (amax + amin) / 2 try: ds.apply_filter() except ValueError: pass else: assert False, "setting only half of a box filter should not work"
def test_config_invalid_key(): pytest.importorskip("nptdms") ds = new_dataset(retrieve_data("fmt-tdms_fl-image_2016.zip")) with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") # Trigger a warning. ds.config["setup"]["invalid_key"] = "picard" # Verify some things assert len(w) == 1 assert issubclass(w[-1].category, dccfg.UnknownConfigurationKeyWarning) assert "invalid_key" in str(w[-1].message)
def test_user_section_set_save_reload_fmt_dict(): """Check that 'user' section metadata works for RTDC_Dict""" # create temp directory for storing outputted file tpath = pathlib.Path(tempfile.mkdtemp()) ddict = example_data_dict(size=67, keys=["area_um", "deform"]) ds = new_dataset(ddict) metadata = {"some metadata": 42} ds.config.update({"user": metadata}) assert ds.config["user"] == metadata # must add some metadata to the "experiment" key for loading with dclab ds.config["experiment"]["sample"] = "test" ds.config["experiment"]["run index"] = 1 expath = tpath / "exported.rtdc" with expath as exp: ds.export.hdf5(exp, features=["deform", "area_um"]) # make sure that worked with h5py.File(expath, "r") as h5: assert h5.attrs["user:some metadata"] == 42 # check again with dclab with new_dataset(expath) as ds2: assert ds2.config["user"] == metadata
def test_config_save_load(): # Download and extract data tdms_path = retrieve_data(example_data_sets[0]) ds = new_dataset(tdms_path) cfg_file = tempfile.mktemp(prefix="test_dclab_rtdc_config_") ds.config.save(cfg_file) loaded = Configuration(files=[cfg_file]) assert equals(loaded, ds.config) cleanup() try: os.remove(cfg_file) except OSError: pass
def test_config_save_load(): pytest.importorskip("nptdms") # Download and extract data tdms_path = retrieve_data("fmt-tdms_minimal_2016.zip") ds = new_dataset(tdms_path) cfg_file = tempfile.mktemp(prefix="test_dclab_rtdc_config_") ds.config.save(cfg_file) loaded = dccfg.Configuration(files=[cfg_file]) assert equals(loaded, ds.config) try: os.remove(cfg_file) except OSError: pass
def test_user_section_set_with_update(): """Add information to the user section of config with .update""" ds = new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip")) metadata = {"some metadata": 42} ds.config.update({"user": metadata}) assert ds.config["user"] == {"some metadata": 42} metadata2 = {"channel information": "information"} ds.config["user"].update(metadata2) assert ds.config["user"] == { "some metadata": 42, "channel information": "information" }
def test_filter_min_max(): # make sure min/max values are filtered ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) amin, amax = ds["area_um"].min(), ds["area_um"].max() ds.config["filtering"]["area_um min"] = (amax + amin) / 2 ds.config["filtering"]["area_um max"] = amax ds.apply_filter() assert np.sum(ds.filter.all) == 4256 # make sure data is not filtered before calling ds.apply_filter dmin, dmax = ds["deform"].min(), ds["deform"].max() ds.config["filtering"]["deform min"] = (dmin + dmax) / 2 ds.config["filtering"]["deform max"] = dmax assert np.sum(ds.filter.all) == 4256
def test_config_basic(): ds = new_dataset(retrieve_data(example_data_sets[1])) assert ds.config["imaging"]["roi size y"] == 96. cleanup()