def test_task_compress_and_repack_strip_issue19(qtbot, monkeypatch): """Same test as above, only tests whether repack and strip logs works""" path = retrieve_data("rtdc_data_hdf5_rtfdc.zip") path_out = path.with_name("compressed") path_out.mkdir() # Monkeypatch monkeypatch.setattr(QDialog, "exec_", lambda *args: QMessageBox.Ok) monkeypatch.setattr(QMessageBox, "exec_", lambda *args: QMessageBox.Ok) monkeypatch.setattr(QFileDialog, "getExistingDirectory", lambda *args: str(path_out)) mw = DCKit(check_update=False) qtbot.addWidget(mw) mw.append_paths([path]) mw.checkBox_repack.setChecked(True) pouts, invalid = mw.on_task_compress() assert len(pouts) == 1 assert len(invalid) == 0 with dclab.new_dataset(pouts[0]) as ds, dclab.new_dataset(path) as ds0: assert len(ds) == len(ds0) assert len(ds.logs) == 0 scf = list(set(ds.features_scalar + ds0.features_scalar)) for feat in scf: assert feat in ds0 assert feat in ds assert np.all(ds[feat] == ds0[feat])
def test_tdms2rtdc_bulk(): pytest.importorskip("nptdms") path_data = retrieve_data("fmt-tdms_shapein-2.0.1-no-image_2017.zip") path_wd = pathlib.Path( tempfile.mkdtemp(prefix="tdms2rtdc_bulk_")).resolve() path_in = path_wd / "input" path_in.mkdir() shutil.copytree(path_data.parent, path_in / "data_1") shutil.copytree(path_data.parent, path_in / "data_2") shutil.copytree(path_data.parent, path_in / "data_3") (path_in / "data_nested").mkdir() shutil.copytree(path_data.parent, path_in / "data_nested" / "data_4") # same directory (will be cleaned up with path_in) path_out = path_wd / "output" path_out.mkdir() cli.tdms2rtdc(path_tdms=path_in, path_rtdc=path_out, compute_features=False) for pp in [path_out / "data_1" / "M1_data.rtdc", path_out / "data_2" / "M1_data.rtdc", path_out / "data_3" / "M1_data.rtdc", path_out / "data_nested" / "data_4" / "M1_data.rtdc"]: assert pp.exists() with new_dataset(pp) as ds2, new_dataset(path_data) as ds1: assert len(ds2) assert set(ds1.features) == set(ds2.features) # not all features are computed assert set(ds2._events.keys()) < set(ds1.features) for feat in ds1: assert np.all(ds1[feat] == ds2[feat])
def test_af_emodulus_visc_only_2(): keys = ["area_um", "deform"] ddict = example_data_dict(size=8472, keys=keys) visc = dclab.features.emodulus.viscosity.get_viscosity( medium="CellCarrier", channel_width=30, flow_rate=0.16, temperature=23.0) # legacy ds = dclab.new_dataset(ddict) ds.config["setup"]["flow rate"] = 0.16 ds.config["setup"]["channel width"] = 30 ds.config["imaging"]["pixel size"] = .34 ds.config["calculation"] = {"emodulus lut": "LE-2D-FEM-19", "emodulus medium": "other", "emodulus temperature": 47.0, # irrelevant "emodulus viscosity": visc } # visc-only ds2 = dclab.new_dataset(ddict) ds2.config["setup"]["flow rate"] = 0.16 ds2.config["setup"]["channel width"] = 30 ds2.config["imaging"]["pixel size"] = .34 ds2.config["calculation"] = {"emodulus lut": "LE-2D-FEM-19", "emodulus viscosity": visc } assert np.sum(~np.isnan(ds["emodulus"])) > 0 assert np.allclose(ds["emodulus"], ds2["emodulus"], equal_nan=True, rtol=0, atol=1e-15)
def test_af_emodulus_reservoir(): """Reservoir measurements should not have emodulus""" keys = ["area_um", "deform"] ddict = example_data_dict(size=8472, keys=keys) # legacy ds = dclab.new_dataset(ddict) ds.config["setup"]["flow rate"] = 0.16 ds.config["setup"]["channel width"] = 30 ds.config["imaging"]["pixel size"] = .34 ds.config["calculation"] = {"emodulus lut": "LE-2D-FEM-19", "emodulus medium": "CellCarrier", "emodulus temperature": 23.0, "emodulus viscosity": 0.5 } assert "emodulus" in ds ds2 = dclab.new_dataset(ddict) ds2.config["setup"]["flow rate"] = 0.16 ds2.config["setup"]["channel width"] = 30 ds2.config["imaging"]["pixel size"] = .34 ds2.config["calculation"] = {"emodulus lut": "LE-2D-FEM-19", "emodulus medium": "CellCarrier", "emodulus temperature": 23.0, "emodulus viscosity": 0.5 } ds2.config["setup"]["chip region"] = "reservoir" assert "emodulus" not in ds2
def test_logs(): path_in = retrieve_data("fmt-hdf5_mask-contour_2018.zip") with new_dataset(path_in) as ds: assert not ds.logs # write some logs with h5py.File(path_in, "a") as h5: hw = rtdc_dataset.RTDCWriter(h5) hw.store_log("test_log", ["peter", "hans"]) with new_dataset(path_in) as ds: assert ds.logs assert ds.logs["test_log"][0] == "peter" # remove logs with h5py.File(path_in, "a") as h5: del h5["logs"] with new_dataset(path_in) as ds: assert not ds.logs try: ds.logs["test_log"] except KeyError: # no log data pass
def test_register_external_lut_and_get_emodulus(): keys = ["area_um", "deform"] ddict = example_data_dict(size=8472, keys=keys) # from internal LUT ds = dclab.new_dataset(ddict) ds.config["setup"]["flow rate"] = 0.16 ds.config["setup"]["channel width"] = 30 ds.config["imaging"]["pixel size"] = .34 ds.config["calculation"] = {"emodulus lut": "LE-2D-FEM-19", "emodulus medium": "CellCarrier", "emodulus temperature": 23.0 } assert np.sum(~np.isnan(ds["emodulus"])) > 0 # from external LUT identifier = "test-test_register_external_lut" path = resource_filename("dclab.features.emodulus", emodulus.load.INTERNAL_LUTS["LE-2D-FEM-19"]) emodulus.register_lut(path, identifier=identifier) ds2 = dclab.new_dataset(ddict) ds2.config["setup"]["flow rate"] = 0.16 ds2.config["setup"]["channel width"] = 30 ds2.config["imaging"]["pixel size"] = .34 ds2.config["calculation"] = {"emodulus lut": identifier, "emodulus medium": "CellCarrier", "emodulus temperature": 23.0 } assert np.sum(~np.isnan(ds2["emodulus"])) > 0 assert np.allclose(ds["emodulus"], ds2["emodulus"], equal_nan=True, rtol=0, atol=1e-15)
def test_export_and_load(): h5path = retrieve_data("fmt-hdf5_fl_2018.zip") # register temporary feature dclab.register_temporary_feature(feature="fl1_mean") with dclab.new_dataset(h5path) as ds: # extract the feature information from the dataset fl1_mean = np.array( [np.mean(ds["trace"]["fl1_raw"][ii]) for ii in range(len(ds))]) # set the data dclab.set_temporary_feature(rtdc_ds=ds, feature="fl1_mean", data=fl1_mean) # export the data to a new file expath = h5path.with_name("exported.rtdc") ds.export.hdf5(expath, features=ds.features_innate + ["fl1_mean"]) # make sure that worked with h5py.File(expath, "r") as h5: assert "fl1_mean" in h5["events"] assert np.allclose(h5["events"]["fl1_mean"], fl1_mean) # now check again with dclab with dclab.new_dataset(expath) as ds2: assert "fl1_mean" in ds2 assert np.allclose(ds2["fl1_mean"], fl1_mean) # and a control check deregister_all() assert "fl1_mean" not in ds2
def test_manual_exclude(): data = example_data_dict(42, keys=["area_um", "deform"]) p = new_dataset(data) c1 = new_dataset(p) c2 = new_dataset(c1) c3 = new_dataset(c2) c1.filter.manual[0] = False c2.apply_filter() c2.filter.manual[1] = False c3.apply_filter() # simple exclusion of few events assert len(c3) == len(p) - 2 # removing same event in parent removes the event from the # child altogether, including the manual filter c3.filter.manual[0] = False c2.filter.manual[0] = False c3.apply_filter() assert np.alltrue(c3.filter.manual) # reinserting the event in the parent, retrieves back # the manual filter in the child c2.filter.manual[0] = True c3.apply_filter() assert not c3.filter.manual[0]
def test_get_min_max_inf(): # generate fake dataset path = pathlib.Path(__file__).parent / "data" / "calibration_beads_47.rtdc" with dclab.new_dataset(path) as ds: config = copy.deepcopy(ds.config) tmp = tempfile.mktemp(".rtdc", prefix="example_filter_inf_") ds2 = dclab.new_dataset({"deform": np.linspace(0, .01, 100), "area_um": np.linspace(20, 200, 100), "area_ratio": np.linspace(1, 1.1, 100) }) ds2.config.update(config) ds2["area_ratio"][0] = np.inf ds2["area_ratio"][1] = np.nan ds2.export.hdf5(tmp, features=["area_um", "deform", "area_ratio"]) # initiate the pipeline pl = pipeline.Pipeline() pl.add_slot(path=tmp) pl.add_filter() # get the current min/max values amin, amax = pl.get_min_max("area_ratio") assert amin == ds2["area_ratio"][2] assert amax == 1.1 try: pathlib.Path(tmp).unlink() except BaseException: pass
def test_dcor_hierarchy(monkeypatch): monkeypatch.setattr(dclab.rtdc_dataset.fmt_dcor, "APIHandler", MockAPIHandler) dso = dclab.new_dataset("https://example.com/api/3/action/dcserv?id=1") dsh = dclab.new_dataset(dso) assert np.all(dso["area_um"] == dsh["area_um"])
def test_hierarchy_shape_mask(): ds = new_dataset(retrieve_data("fmt-hdf5_image-bg_2020.zip")) ds.filter.manual[0] = False ch = new_dataset(ds) assert "mask" in ch.features_innate assert len(ch["mask"]) == 4 assert ch["mask"].shape == (4, 80, 250)
def test_hdf5_traces_filter2(): """Length of traces was wrong when filters were applied #112 Test dataset lenght only with trace. """ ds = new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip")) # applying some filters ds.config["filtering"]["deform min"] = 0.01 ds.config["filtering"]["deform max"] = 0.1 ds.apply_filter() # sanity check assert np.sum(ds.filter.all) == 3 edest = tempfile.mkdtemp() f1 = join(edest, "test.rtdc") ds.export.hdf5(f1, ["deform", "trace"]) ds2 = new_dataset(f1) assert len(ds2) == 3 assert len(ds2["deform"]) == 3 assert len(ds2["trace"]["fl1_median"]) == 3 assert np.all(ds["trace"]["fl1_raw"][3] == ds2["trace"]["fl1_raw"][0]) assert np.all(ds["trace"]["fl1_raw"][5] == ds2["trace"]["fl1_raw"][1]) assert np.all(ds["trace"]["fl1_raw"][6] == ds2["trace"]["fl1_raw"][2])
def test_pf_export_and_load(): """Check that exported and loaded hdf5 file will keep a plugin feature""" h5path = retrieve_data("fmt-hdf5_fl_2018.zip") # initialize PlugInFeature instance info = example_plugin_info_single_feature() pf = PlugInFeature("circ_per_area", info) with dclab.new_dataset(h5path) as ds: # extract the feature information from the dataset assert pf in PlugInFeature.features circ_per_area = ds[pf.feature_name] # export the data to a new file expath = h5path.with_name("exported.rtdc") ds.export.hdf5(expath, features=ds.features_innate + [pf.feature_name]) # make sure that worked with h5py.File(expath, "r") as h5: assert pf.feature_name in h5["events"] assert np.allclose(h5["events"][pf.feature_name], circ_per_area) # now check again with dclab with dclab.new_dataset(expath) as ds2: assert pf in PlugInFeature.features assert pf.feature_name in ds2 assert pf.feature_name in ds2.features_innate assert np.allclose(ds2[pf.feature_name], circ_per_area) # and a control check remove_plugin_feature(pf) assert pf.feature_name not in ds2
def test_pf_export_non_scalar_filtered_from_file_issue_166(): h5path = retrieve_data("fmt-hdf5_image-bg_2020.zip") expath = h5path.with_name("exported.rtdc") # initialize PlugInFeature instance info = example_plugin_info_non_scalar_feature() info["feature shapes"] = [(80, 250)] pf = PlugInFeature("image_gauss_filter", info) # write the plugin feature data to an HDF5 file with dclab.new_dataset(h5path) as ds: # extract the feature information from the dataset ds.export.hdf5(expath, features=[pf.feature_name]) # remove all plugin features and work with temporary feature remove_all_plugin_features() # try to load the plugin feature data from that HDF5 file and # export it again (this time, the exporter has to get the data # from the H5File object). expath2 = h5path.with_name("exported2.rtdc") dclab.register_temporary_feature("image_gauss_filter", is_scalar=False) with dclab.new_dataset(expath) as ds2: ds2.export.hdf5(expath2, features=["image_gauss_filter"], filtered=True) # make sure that worked with h5py.File(expath2, "r") as h5: assert "image_gauss_filter" in h5["events"]
def test_af_ml_class_bad_feature(): data = { "ml_score_0-1": [.1, .3, .1, 0.01, .59], } with pytest.raises(ValueError, match="Invalid feature name 'ml_score_0-1'"): dclab.new_dataset(data)
def test_tdms2rtdc_update_roi_size(): pytest.importorskip("nptdms") path_in = retrieve_data("fmt-tdms_fl-image_2016.zip") # set wrong roi sizes camin = path_in.with_name("M1_camera.ini") with camin.open("r") as fd: lines = fd.readlines() lines = lines[:-2] lines.append("width = 23\n") lines.append("height = 24\n") with camin.open("w") as fd: fd.writelines(lines) # same directory (will be cleaned up with path_in) path_out = path_in.with_name("out.rtdc") cli.tdms2rtdc(path_tdms=path_in, path_rtdc=path_out, compute_features=False, skip_initial_empty_image=True) with new_dataset(path_out) as dsj, new_dataset(path_in) as ds0: assert ds0.config["imaging"]["roi size x"] == 23 assert ds0.config["imaging"]["roi size y"] == 24 assert dsj.config["imaging"]["roi size x"] == 256 assert dsj.config["imaging"]["roi size y"] == 96 wlog = "dclab-tdms2rtdc-warnings" assert "LimitingExportSizeWarning" in dsj.logs[wlog]
def test_hierarchy_shape_contour(): ds = new_dataset(retrieve_data("fmt-hdf5_image-bg_2020.zip")) assert ds["contour"].shape == (5, np.nan, 2) ds.filter.manual[0] = False ch = new_dataset(ds) assert ch["contour"].shape == (4, np.nan, 2) assert len(ch["contour"]) == 4
def test_join_frame(): path_in1 = retrieve_data("fmt-hdf5_mask-contour_2018.zip") path_in2 = retrieve_data("fmt-hdf5_mask-contour_2018.zip") # same directory (will be cleaned up with path_in) path_out = path_in1.with_name("out.rtdc") # modify acquisition times with h5py.File(path_in1, mode="a") as h1: h1.attrs["experiment:date"] = "2019-11-04" h1.attrs["experiment:time"] = "15:00:00" with h5py.File(path_in2, mode="a") as h2: h2.attrs["experiment:date"] = "2019-11-05" h2.attrs["experiment:time"] = "16:01:15.050" offset = 24 * 60 * 60 + 60 * 60 + 1 * 60 + 15 + .05 cli.join(path_out=path_out, paths_in=[path_in1, path_in2]) with new_dataset(path_out) as dsj, new_dataset(path_in1) as ds0: fr = ds0.config["imaging"]["frame rate"] assert np.allclose(dsj["frame"], np.concatenate((ds0["frame"], ds0["frame"] + offset * fr)), rtol=0, atol=.0001)
def test_manual_exclude_parent_changed(): data = example_data_dict(42, keys=["area_um", "tilt"]) p = new_dataset(data) p.filter.manual[4] = False c = new_dataset(p) c.filter.manual[5] = False c.apply_filter() p.config["filtering"]["tilt min"] = 0 p.config["filtering"]["tilt max"] = .5 p.apply_filter() assert np.sum(p.filter.all) == 21 # size of child is directly determined from parent assert len(c) == 21 # filters have not yet been updated assert len(c.filter.all) == 41 assert c.filter.parent_changed # the initially excluded event assert c.filter.retrieve_manual_indices(c) == [6] # try to change the excluded events try: c.filter.apply_manual_indices(c, [1, 2]) except fmt_hierarchy.HierarchyFilterError: pass else: assert False, "expected HierarchyFilterError" # this can be resolved by applying the filter c.apply_filter() c.filter.apply_manual_indices(c, [1, 2]) assert c.filter.retrieve_manual_indices(c) == [1, 2]
def test_hdf5_filtered(): N = 10 keys = ["area_um", "image"] ddict = example_data_dict(size=N, keys=keys) ddict["image"][3] = np.arange(10 * 20, dtype=np.uint8).reshape(10, 20) + 22 ds1 = dclab.new_dataset(ddict) ds1.config["experiment"]["sample"] = "test" ds1.config["experiment"]["run index"] = 1 ds1.filter.manual[2] = False ds1.apply_filter() fta = ds1.filter.manual.copy() edest = tempfile.mkdtemp() f1 = join(edest, "dclab_test_export_hdf5_filtered.rtdc") ds1.export.hdf5(f1, keys) ds2 = dclab.new_dataset(f1) assert ds1 != ds2 assert np.allclose(ds2["area_um"], ds1["area_um"][fta]) assert np.allclose(ds2["image"][2], ds1["image"][3]) assert np.all(ds2["image"][2] != ds1["image"][2]) # cleanup shutil.rmtree(edest, ignore_errors=True)
def test_dcor_base(monkeypatch): monkeypatch.setattr(dclab.rtdc_dataset.fmt_dcor, "APIHandler", MockAPIHandler) with dclab.new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip")) as ds: dso = dclab.new_dataset("https://example.com/api/3/action/dcserv?id=1") assert len(dso) == len(ds) assert dso.config["setup"]["channel width"] == \ ds.config["setup"]["channel width"] assert np.all(dso["area_um"] == ds["area_um"]) assert np.all(dso["area_um"] == ds["area_um"]) # test cache assert np.all(dso["image"][4] == ds["image"][4]) assert len(dso["image"]) == len(ds) for key in dso._events: assert key in ds for m, n in zip(dso["mask"], ds["mask"]): assert np.all(m == n) # compute an ancillary feature assert np.all(dso["volume"] == ds["volume"]) assert np.all(dso["volume"] == ds["volume"]) # test cache # trace assert sorted(dso["trace"].keys()) == sorted(ds["trace"].keys()) assert len(dso["trace"]["fl1_raw"]) == len(ds["trace"]["fl1_raw"]) assert np.all(dso["trace"]["fl1_raw"][1] == ds["trace"]["fl1_raw"][1]) for t1, t2 in zip(dso["trace"]["fl1_raw"], ds["trace"]["fl1_raw"]): assert np.all(t1 == t2)
def test_load_nonexistent_file_issue81(): """https://github.com/ZELLMECHANIK-DRESDEN/dclab/issues/81""" try: dclab.new_dataset("path/does/not/exist.rtdc") except FileNotFoundError: pass else: assert False, "Non-existent files should raise FileNotFoundError"
def test_feat_mask(): path = retrieve_data("fmt-hdf5_mask-contour_2018.zip") ds = new_dataset(path) ds.filter.manual[0] = False ds.filter.manual[2] = False ch = new_dataset(ds) assert np.all(ch["mask"][0] == ds["mask"][1]) assert np.all(ch["mask"][1] == ds["mask"][3])
def test_features(): path = retrieve_data("fmt-hdf5_fl_2017.zip") ds = new_dataset(path) ch = new_dataset(ds) assert ds.features == ch.features assert ds.features_innate == ch.features_innate assert ds.features_loaded == ch.features_loaded assert ds.features_scalar == ch.features_scalar
def test_feat_contour(): path = retrieve_data("fmt-hdf5_fl_2017.zip") ds = new_dataset(path) ds.filter.manual[0] = False ds.filter.manual[2] = False ch = new_dataset(ds) assert np.all(ch["contour"][0] == ds["contour"][1]) assert np.all(ch["contour"][1] == ds["contour"][3])
def test_index_deep_contour(): data = example_data_dict(42, keys=["area_um", "contour", "deform"]) ds = new_dataset(data) ds.filter.manual[3] = False c1 = new_dataset(ds) c1.filter.manual[1] = False c2 = new_dataset(c1) assert np.all(c2["contour"][3] == ds["contour"][5])
def test_feat_trace(): path = retrieve_data("fmt-hdf5_fl_2017.zip") ds = new_dataset(path) ds.filter.manual[0] = False ds.filter.manual[2] = False ch = new_dataset(ds) assert np.all(ch["trace"]["fl1_median"][0] == ds["trace"]["fl1_median"][1]) assert np.all(ch["trace"]["fl1_median"][1] == ds["trace"]["fl1_median"][3])
def test_hdf5_contour_from_hdf5(): ds1 = new_dataset(retrieve_data("fmt-hdf5_image-bg_2020.zip")) assert ds1["contour"].shape == (5, np.nan, 2) edest = tempfile.mkdtemp() f1 = join(edest, "dclab_test_export_hdf5_image.rtdc") ds1.export.hdf5(f1, ["contour"], filtered=False) ds2 = dclab.new_dataset(f1) assert ds2["contour"].shape == (5, np.nan, 2)
def test_event_count(): pytest.importorskip("nptdms") tdms_path = retrieve_data("fmt-tdms_fl-image_2016.zip") ds = new_dataset(tdms_path) ds.filter.manual[0] = False ch = new_dataset(ds) assert ds.config["experiment"]["event count"] == len(ds) assert ch.config["experiment"]["event count"] == len(ch) assert len(ds) == len(ch) + 1
def test_trace_import_fail(): # make sure undefined trace data does not raise an error tdms_path = retrieve_data("fmt-tdms_fl-image_2016.zip") dclab.definitions.FLUOR_TRACES.append("peter") dclab.rtdc_dataset.fmt_tdms.naming.tr_data_map["peter"] = [u'ukwn', u'ha'] new_dataset(tdms_path) # clean up dclab.rtdc_dataset.fmt_tdms.naming.tr_data_map.pop("peter") dclab.definitions.FLUOR_TRACES.pop(-1)
def test_pf_inherited_scalar(): """Scalar inherited PluginFeatures should be a 1D np.ndarray""" info = example_plugin_info_single_feature() PlugInFeature("circ_per_area", info) with dclab.new_dataset(retrieve_data("fmt-hdf5_fl_2018.zip")) as ds: ds.filter.manual[2] = False ch = dclab.new_dataset(ds) assert "circ_per_area" in ch assert isinstance(ch["circ_per_area"], np.ndarray) assert ch["circ_per_area"].ndim == 1
def test_trace_import_fail(): # make sure undefined trace data does not raise an error tdms_path = retrieve_data(example_data_sets[1]) dclab.definitions.FLUOR_TRACES.append("peter") dclab.rtdc_dataset.fmt_tdms.naming.tr_data_map["peter"] = [u'ukwn', u'ha'] new_dataset(tdms_path) # clean up dclab.rtdc_dataset.fmt_tdms.naming.tr_data_map.pop("peter") dclab.definitions.FLUOR_TRACES.pop(-1) cleanup()
def test_time(): ds = dclab.new_dataset(retrieve_data("rtdc_data_minimal.zip")) tt = ds["time"] assert tt[0] == 0 assert np.allclose(tt[1], 0.0385) assert np.all(np.diff(tt) > 0) cleanup()
def test_volume(): ds = dclab.new_dataset(retrieve_data("rtdc_data_minimal.zip")) vol = ds["volume"] # There are a lot of nans, because the contour is not given everywhere vol = vol[~np.isnan(vol)] assert np.allclose(vol[0], 574.60368907528346) assert np.allclose(vol[12], 1010.5669523203878)
def test_aspect(): # Aspect ratio of the data ds = dclab.new_dataset(retrieve_data("rtdc_data_traces_video_bright.zip")) aspect = ds["aspect"] assert np.sum(aspect > 1) == 904 assert np.sum(aspect < 1) == 48 cleanup()
def test_area_ratio(): ds = dclab.new_dataset(retrieve_data("rtdc_data_traces_video.zip")) comp_ratio = ds["area_ratio"] # The convex area is always >= the raw area assert np.all(comp_ratio >= 1) assert np.allclose(comp_ratio[0], 1.0196464) cleanup()
def test_image_basic(): ds = new_dataset(retrieve_data(example_data_sets[1])) # Transition image assert np.allclose(ds["image"][0], 0) # Real image assert np.allclose(np.average(ds["image"][1]), 45.1490478515625) cleanup()
def test_compatibility_minimal(): ds = new_dataset(retrieve_data("rtdc_data_minimal.zip")) assert ds.config["setup"]["channel width"] == 20 assert ds.config["setup"]["chip region"].lower() == "channel" assert ds.config["setup"]["flow rate"] == 0.12 assert ds.config["imaging"]["pixel size"] == 0.34 cleanup()
def test_fl_crosstalk_3chanvs2chan(): data = {"fl1_max": np.linspace(1, 1.1, 10), "fl2_max": np.linspace(0, 4.1, 10), "fl3_max": np.linspace(3, 2.5, 10), } ds = dclab.new_dataset(data) analysis = {"calculation": {"crosstalk fl12": .4, "crosstalk fl21": .05, }} ds.config.update(analysis) assert "fl2_max_ctc" in ds try: ds["fl2_max_ctc"] except ancillaries.af_fl_max_ctc.MissingCrosstalkMatrixElementsError: pass else: assert False, "Crosstalk correction from missing data should not work" # add missing matrix elements analysis = {"calculation": {"crosstalk fl13": .1, "crosstalk fl23": .7, "crosstalk fl31": .2, "crosstalk fl32": .2, }} ds.config.update(analysis) ds["fl1_max_ctc"] ds["fl2_max_ctc"] ds["fl3_max_ctc"] ds.config.update(analysis)
def test_classify_treatment_repetition_simple(): measurements = [] dd = {"area_um": np.linspace(40, 50, 10), "deform": np.linspace(.1, .2, 10)} for ii in range(5): ctl = dclab.new_dataset(dd) ctl.title = "donor {} control".format(ii) ctl.config["setup"]["chip region"] = "channel" trt = dclab.new_dataset(dd) trt.title = "donor {}".format(ii) trt.config["setup"]["chip region"] = "channel" measurements += [ctl, trt] ana = Analysis(measurements) treatment, repetition = classify_treatment_repetition(ana, id_ctl="control") assert treatment == ["Control", "Treatment"] * 5 assert np.all(repetition == np.repeat(np.arange(5), 2) + 1)
def test_kde_log_scatter(): ddict = example_data_dict(size=300, keys=["area_um", "deform"]) ddict["deform"][:20] = .1 ddict["area_um"][:20] = .5 ds = dclab.new_dataset(ddict) a = ds.get_kde_scatter(yscale="log") assert np.all(a[:20] == a[0])
def test_collect_data_tree(): features = ["area_um", "deform", "time"] edest = pathlib.Path(tempfile.mkdtemp(prefix="shapeout_test")) for ii in range(1, 4): dat = new_dataset(data=example_data_dict(ii + 10, keys=features)) cfg = {"experiment": {"sample": "test sample", "run index": ii}, "imaging": {"pixel size": 0.34}, "setup": {"channel width": 20, "chip region": "channel", "flow rate": 0.04} } dat.config.update(cfg) dat.export.hdf5(path=edest / "{}.rtdc".format(ii), features=features) data = meta_tool.collect_data_tree([edest])[0] assert len(data) == 1, "only one data folder" assert len(data[0]) == 4, "three measurements" # check name assert data[0][0][0] == "test sample" # check order assert data[0][1][1].endswith("1.rtdc") assert data[0][2][1].endswith("2.rtdc") assert data[0][3][1].endswith("3.rtdc") shutil.rmtree(str(edest), ignore_errors=True)
def test_config(): ds = new_dataset(retrieve_data("rtdc_data_hdf5_contour_image_trace.zip")) assert ds.config["setup"]["channel width"] == 30 assert ds.config["setup"]["chip region"].lower() == "channel" assert ds.config["setup"]["flow rate"] == 0.16 assert ds.config["imaging"]["pixel size"] == 0.34 cleanup()
def test_trace(): ds = new_dataset(retrieve_data("rtdc_data_hdf5_contour_image_trace.zip")) assert len(ds["trace"]) == 2 assert ds["trace"]["fl1_raw"].shape == (5, 100) assert np.allclose(np.average( ds["trace"]["fl1_median"][0]), 0.027744706519425219) cleanup()
def test_0basic(): ds = dclab.new_dataset(retrieve_data(example_data_sets[1])) for cc in ['fl1_pos', 'frame', 'size_x', 'size_y', 'contour', 'area_cvx', 'circ', 'image', 'trace', 'fl1_width', 'nevents', 'pos_x', 'pos_y', 'fl1_area', 'fl1_max', ]: assert cc in ds # ancillaries for cc in ["deform", "area_um", "aspect", "frame", "index", "time", ]: assert cc in ds cleanup()
def test_invert_copy(): dclab.PolygonFilter.clear_all_filters() ddict = example_data_dict(size=1234, keys=["area_um", "deform"]) ds = dclab.new_dataset(ddict) # points of polygon filter points = [[np.min(ddict["area_um"]), np.min(ddict["deform"])], [np.min(ddict["area_um"]), np.max(ddict["deform"])], [np.average(ddict["area_um"]), np.max(ddict["deform"])], [np.average(ddict["area_um"]), np.min(ddict["deform"])], ] filt1 = dclab.PolygonFilter(axes=["area_um", "deform"], points=points, inverted=False) ds.polygon_filter_add(filt1) assert [0] == ds.config["filtering"]["polygon filters"] n1 = np.sum(ds._filter) ds.apply_filter() n2 = np.sum(ds._filter) assert n1 != n2 filt2 = filt1.copy(invert=True) ds.polygon_filter_add(filt2) assert [0, 1] == ds.config["filtering"]["polygon filters"] ds.apply_filter() assert np.sum(ds._filter) == 0, "inverted+normal filter filters all" dclab.PolygonFilter.clear_all_filters()
def test_collect_data_tree_order(): features = ["area_um", "deform", "time"] edest = pathlib.Path(tempfile.mkdtemp(prefix="shapeout_test")) for ii in range(1, 13): dat = new_dataset(data=example_data_dict(ii + 10, keys=features)) cfg = {"experiment": {"sample": "test sample", "run index": ii}, "imaging": {"pixel size": 0.34}, "setup": {"channel width": 20, "chip region": "channel", "flow rate": 0.04} } dat.config.update(cfg) dat.export.hdf5(path=edest / "M{}_data.rtdc".format(ii), features=features) data = meta_tool.collect_data_tree([edest])[0] assert pathlib.Path(data[0][1][1]).name == "M1_data.rtdc" assert pathlib.Path(data[0][2][1]).name == "M2_data.rtdc" assert pathlib.Path(data[0][3][1]).name == "M3_data.rtdc" assert pathlib.Path(data[0][4][1]).name == "M4_data.rtdc" assert pathlib.Path(data[0][5][1]).name == "M5_data.rtdc" assert pathlib.Path(data[0][6][1]).name == "M6_data.rtdc" assert pathlib.Path(data[0][7][1]).name == "M7_data.rtdc" assert pathlib.Path(data[0][8][1]).name == "M8_data.rtdc" assert pathlib.Path(data[0][9][1]).name == "M9_data.rtdc" assert pathlib.Path(data[0][10][1]).name == "M10_data.rtdc" assert pathlib.Path(data[0][11][1]).name == "M11_data.rtdc" assert pathlib.Path(data[0][12][1]).name == "M12_data.rtdc" shutil.rmtree(str(edest), ignore_errors=True)
def test_inert_ratio_raw(): ds = new_dataset(retrieve_data("rtdc_data_hdf5_mask_contour.zip")) raw = ir.get_inert_ratio_raw(cont=ds["contour"]) ref = np.array([4.25854232, 1.22342663, 4.64971179, 1.70914857, 3.62797492, 1.51502192, 2.74757573, 1.79841136]) assert np.allclose(ref, raw, rtol=0, atol=5e-9) cleanup()
def test_trace_methods(): ds = new_dataset(retrieve_data(example_data_sets[1])) for k in list(ds["trace"].keys()): assert k in dclab.definitions.FLUOR_TRACES for k in ds["trace"]: assert k in dclab.definitions.FLUOR_TRACES assert ds["trace"].__repr__().count("<loaded into memory>") cleanup()
def test_kde_log_scatter_points(): ddict = example_data_dict(size=300, keys=["area_um", "deform"]) ds = dclab.new_dataset(ddict) a = ds.get_kde_scatter(yscale="log") b = ds.get_kde_scatter(yscale="log", positions=[ds["area_um"], ds["deform"]]) assert np.all(a == b)
def test_wrong_things(): ddict = example_data_dict(size=67, keys=["area_um", "deform"]) ds = new_dataset(ddict) # Check unknown variable (warning will be displayed try: ds.apply_filter(force=["on_purpose_unknown"]) except ValueError: pass
def test_trace_basic(): ds = new_dataset(retrieve_data(example_data_sets[1])) msg = "traces should not be loaded into memory before first access" assert ds["trace"].__repr__().count("<not loaded into memory>"), msg assert len(ds["trace"]) == 2 assert np.allclose(np.average( ds["trace"]["fl1_median"][0]), 287.08999999999997) cleanup()
def test_downsample_none(): keys = ["area_um", "deform", "time", "frame", "fl3_width"] ddict = example_data_dict(size=8472, keys=keys) ds = dclab.new_dataset(ddict) assert np.sum(ds._plot_filter) == 8472 ds.apply_filter() ds.get_downsampled_scatter(downsample=0) assert np.sum(ds._plot_filter) == 8472
def test_image_out_of_bounds(): ds = new_dataset(retrieve_data(example_data_sets[1])) try: ds["image"][5] except IndexError: pass else: raise ValueError("IndexError should have been raised!") cleanup()
def test_avi_export(): ds = new_dataset(retrieve_data(example_data_sets[1])) edest = tempfile.mkdtemp() f1 = join(edest, "test.avi") ds.export.avi(path=f1) assert os.stat( f1)[6] > 1e4, "Resulting file to small, Something went wrong!" shutil.rmtree(edest, ignore_errors=True) cleanup()
def test_kde_positions(): ddict = example_data_dict() ds = dclab.new_dataset(ddict) ds.config["filtering"]["enable filters"] = False sc = ds.get_kde_scatter(xax="area_um", yax="deform") sc2 = ds.get_kde_scatter(xax="area_um", yax="deform", positions=(ds["area_um"], ds["deform"])) assert np.all(sc == sc2)
def test_mask_basic(): ds = new_dataset(retrieve_data(example_data_sets[1])) assert len(ds["mask"]) == 12 # Test mask computation by averaging brightness and comparing to # the ancillary feature "bright_avg". bavg1 = ds["bright_avg"][1] bavg2 = np.mean(ds["image"][1][ds["mask"][1]]) assert np.allclose(bavg1, bavg2), "mask is correctly computed from contour" cleanup()
def test_fl_crosstalk_missing(): data = {"fl1_max": np.linspace(1, 1.1, 10), "fl2_max": np.linspace(0, 4.1, 10), } ds = dclab.new_dataset(data) analysis = {"calculation": {"crosstalk fl12": .4, }} ds.config.update(analysis) assert "fl2_max_ctc" not in ds
def test_kde_log_contour(): ddict = example_data_dict(size=300, keys=["area_um", "deform"]) ddict["deform"][:20] = .1 ddict["area_um"][:20] = .5 ds = dclab.new_dataset(ddict) xm, ym, _ = ds.get_kde_contour(yscale="log") dx = np.diff(xm[0]) dy = np.diff(np.log(ym[:, 0])) assert np.allclose(dx, dx[0]) assert np.allclose(dy, dy[0])
def test_kde_nofilt(): ddict = example_data_dict() ds = dclab.new_dataset(ddict) ds.config["filtering"]["enable filters"] = False sc = ds.get_kde_scatter() cc = ds.get_kde_contour() assert sc.shape[0] == 100 # This will fail if the default contour accuracy is changed # in `get_kde_contour`. assert cc[0].shape == (43, 41)