def test_ic_fl_max_ctc(): # Testing dataset with negative fl_max_ctc values ddict = example_data_dict(size=8472, keys=["fl1_max_ctc"]) ddict["fl1_max_ctc"] -= min(ddict["fl1_max_ctc"]) + 1 ds = new_dataset(ddict) with check.IntegrityChecker(ds) as ic: cues = ic.check_fl_max_ctc_positive() assert cues[0].level == "alert" assert cues[0].category == "feature data" # Testing dataset with fl_max_ctc values of 0.1 ddict = example_data_dict(size=8472, keys=["fl1_max_ctc"]) ddict["fl1_max_ctc"] -= min(ddict["fl1_max_ctc"]) - 0.1 ds = new_dataset(ddict) with check.IntegrityChecker(ds) as ic: cues = ic.check_fl_max_ctc_positive() assert cues[0].level == "alert" assert cues[0].category == "feature data" # Testing dataset with fl_max_ctc values > 0.1 ddict = example_data_dict(size=8472, keys=["fl1_max_ctc"]) ddict["fl1_max_ctc"] -= min(ddict["fl1_max_ctc"]) - 1 ds = new_dataset(ddict) with check.IntegrityChecker(ds) as ic: cues = ic.check_fl_max_ctc_positive() assert not cues
def test_kde_log_scatter(): ddict = example_data_dict(size=300, keys=["area_um", "deform"]) ddict["deform"][:20] = .1 ddict["area_um"][:20] = .5 ds = dclab.new_dataset(ddict) a = ds.get_kde_scatter(yscale="log") assert np.all(a[:20] == a[0])
def test_invert_copy(): dclab.PolygonFilter.clear_all_filters() ddict = example_data_dict(size=1234, keys=["area_um", "deform"]) ds = dclab.new_dataset(ddict) # points of polygon filter points = [[np.min(ddict["area_um"]), np.min(ddict["deform"])], [np.min(ddict["area_um"]), np.max(ddict["deform"])], [np.average(ddict["area_um"]), np.max(ddict["deform"])], [np.average(ddict["area_um"]), np.min(ddict["deform"])], ] filt1 = dclab.PolygonFilter(axes=["area_um", "deform"], points=points, inverted=False) ds.polygon_filter_add(filt1) assert [0] == ds.config["filtering"]["polygon filters"] n1 = np.sum(ds._filter) ds.apply_filter() n2 = np.sum(ds._filter) assert n1 != n2 filt2 = filt1.copy(invert=True) ds.polygon_filter_add(filt2) assert [0, 1] == ds.config["filtering"]["polygon filters"] ds.apply_filter() assert np.sum(ds._filter) == 0, "inverted+normal filter filters all" dclab.PolygonFilter.clear_all_filters()
def test_shape_dict_trace(): ddict = example_data_dict(size=67, keys=["area_um", "deform", "trace"]) ds = new_dataset(ddict) assert ds["trace"].shape == (6, 67, 100) assert len(ds["trace"]) == 6 assert len(ds["trace"]["fl1_raw"]) == 67 assert len(ds["trace"]["fl1_raw"][0]) == 100
def test_af_emodulus_reservoir(): """Reservoir measurements should not have emodulus""" keys = ["area_um", "deform"] ddict = example_data_dict(size=8472, keys=keys) # legacy ds = dclab.new_dataset(ddict) ds.config["setup"]["flow rate"] = 0.16 ds.config["setup"]["channel width"] = 30 ds.config["imaging"]["pixel size"] = .34 ds.config["calculation"] = {"emodulus lut": "LE-2D-FEM-19", "emodulus medium": "CellCarrier", "emodulus temperature": 23.0, "emodulus viscosity": 0.5 } assert "emodulus" in ds ds2 = dclab.new_dataset(ddict) ds2.config["setup"]["flow rate"] = 0.16 ds2.config["setup"]["channel width"] = 30 ds2.config["imaging"]["pixel size"] = .34 ds2.config["calculation"] = {"emodulus lut": "LE-2D-FEM-19", "emodulus medium": "CellCarrier", "emodulus temperature": 23.0, "emodulus viscosity": 0.5 } ds2.config["setup"]["chip region"] = "reservoir" assert "emodulus" not in ds2
def test_hdf5_filtered(): N = 10 keys = ["area_um", "image"] ddict = example_data_dict(size=N, keys=keys) ddict["image"][3] = np.arange(10 * 20, dtype=np.uint8).reshape(10, 20) + 22 ds1 = dclab.new_dataset(ddict) ds1.config["experiment"]["sample"] = "test" ds1.config["experiment"]["run index"] = 1 ds1.filter.manual[2] = False ds1.apply_filter() fta = ds1.filter.manual.copy() edest = tempfile.mkdtemp() f1 = join(edest, "dclab_test_export_hdf5_filtered.rtdc") ds1.export.hdf5(f1, keys) ds2 = dclab.new_dataset(f1) assert ds1 != ds2 assert np.allclose(ds2["area_um"], ds1["area_um"][fta]) assert np.allclose(ds2["image"][2], ds1["image"][3]) assert np.all(ds2["image"][2] != ds1["image"][2]) # cleanup shutil.rmtree(edest, ignore_errors=True)
def test_collect_data_tree(): features = ["area_um", "deform", "time"] edest = pathlib.Path(tempfile.mkdtemp(prefix="shapeout_test")) for ii in range(1, 4): dat = new_dataset(data=example_data_dict(ii + 10, keys=features)) cfg = {"experiment": {"sample": "test sample", "run index": ii}, "imaging": {"pixel size": 0.34}, "setup": {"channel width": 20, "chip region": "channel", "flow rate": 0.04} } dat.config.update(cfg) dat.export.hdf5(path=edest / "{}.rtdc".format(ii), features=features) data = meta_tool.collect_data_tree([edest])[0] assert len(data) == 1, "only one data folder" assert len(data[0]) == 4, "three measurements" # check name assert data[0][0][0] == "test sample" # check order assert data[0][1][1].endswith("1.rtdc") assert data[0][2][1].endswith("2.rtdc") assert data[0][3][1].endswith("3.rtdc") shutil.rmtree(str(edest), ignore_errors=True)
def test_collect_data_tree_order(): features = ["area_um", "deform", "time"] edest = pathlib.Path(tempfile.mkdtemp(prefix="shapeout_test")) for ii in range(1, 13): dat = new_dataset(data=example_data_dict(ii + 10, keys=features)) cfg = {"experiment": {"sample": "test sample", "run index": ii}, "imaging": {"pixel size": 0.34}, "setup": {"channel width": 20, "chip region": "channel", "flow rate": 0.04} } dat.config.update(cfg) dat.export.hdf5(path=edest / "M{}_data.rtdc".format(ii), features=features) data = meta_tool.collect_data_tree([edest])[0] assert pathlib.Path(data[0][1][1]).name == "M1_data.rtdc" assert pathlib.Path(data[0][2][1]).name == "M2_data.rtdc" assert pathlib.Path(data[0][3][1]).name == "M3_data.rtdc" assert pathlib.Path(data[0][4][1]).name == "M4_data.rtdc" assert pathlib.Path(data[0][5][1]).name == "M5_data.rtdc" assert pathlib.Path(data[0][6][1]).name == "M6_data.rtdc" assert pathlib.Path(data[0][7][1]).name == "M7_data.rtdc" assert pathlib.Path(data[0][8][1]).name == "M8_data.rtdc" assert pathlib.Path(data[0][9][1]).name == "M9_data.rtdc" assert pathlib.Path(data[0][10][1]).name == "M10_data.rtdc" assert pathlib.Path(data[0][11][1]).name == "M11_data.rtdc" assert pathlib.Path(data[0][12][1]).name == "M12_data.rtdc" shutil.rmtree(str(edest), ignore_errors=True)
def test_with_rtdc_warning(): keys = ["area_um", "deform"] ddict = example_data_dict(size=8472, keys=keys) # legacy ds = dclab.new_dataset(ddict) ds.config["setup"]["flow rate"] = 0.16 ds.config["setup"]["channel width"] = 30 ds.config["setup"]["medium"] = "CellCarrier" ds.config["imaging"]["pixel size"] = .34 i1 = iso.get_default() with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") # Trigger a warning (temperature missing). i1.get_with_rtdcbase(col1="area_um", col2="deform", lut_identifier="LE-2D-FEM-19", dataset=ds, viscosity=None, add_px_err=False) # Verify some things assert len(w) == 1 assert issubclass(w[-1].category, iso.IsoelasticsEmodulusMeaninglessWarning) assert "plotting" in str(w[-1].message)
def test_manual_exclude(): data = example_data_dict(42, keys=["area_um", "deform"]) p = new_dataset(data) c1 = new_dataset(p) c2 = new_dataset(c1) c3 = new_dataset(c2) c1.filter.manual[0] = False c2.apply_filter() c2.filter.manual[1] = False c3.apply_filter() # simple exclusion of few events assert len(c3) == len(p) - 2 # removing same event in parent removes the event from the # child altogether, including the manual filter c3.filter.manual[0] = False c2.filter.manual[0] = False c3.apply_filter() assert np.alltrue(c3.filter.manual) # reinserting the event in the parent, retrieves back # the manual filter in the child c2.filter.manual[0] = True c3.apply_filter() assert not c3.filter.manual[0]
def test_invert_saveload(): dclab.PolygonFilter.clear_all_filters() ddict = example_data_dict(size=1234, keys=["area_um", "deform"]) # points of polygon filter points = [[np.min(ddict["area_um"]), np.min(ddict["deform"])], [np.min(ddict["area_um"]), np.max(ddict["deform"])], [np.average(ddict["area_um"]), np.max(ddict["deform"])], [np.average(ddict["area_um"]), np.min(ddict["deform"])], ] filt1 = dclab.PolygonFilter(axes=["area_um", "deform"], points=points, inverted=True) name = tempfile.mktemp(prefix="test_dclab_polygon_") filt1.save(name) filt2 = dclab.PolygonFilter(filename=name) assert filt2 == filt1 filt3 = dclab.PolygonFilter(axes=["area_um", "deform"], points=points, inverted=False) try: os.remove(name) except OSError: pass name = tempfile.mktemp(prefix="test_dclab_polygon_") filt3.save(name) filt4 = dclab.PolygonFilter(filename=name) assert filt4 == filt3 try: os.remove(name) except OSError: pass
def test_register_external_lut_and_get_emodulus(): keys = ["area_um", "deform"] ddict = example_data_dict(size=8472, keys=keys) # from internal LUT ds = dclab.new_dataset(ddict) ds.config["setup"]["flow rate"] = 0.16 ds.config["setup"]["channel width"] = 30 ds.config["imaging"]["pixel size"] = .34 ds.config["calculation"] = {"emodulus lut": "LE-2D-FEM-19", "emodulus medium": "CellCarrier", "emodulus temperature": 23.0 } assert np.sum(~np.isnan(ds["emodulus"])) > 0 # from external LUT identifier = "test-test_register_external_lut" path = resource_filename("dclab.features.emodulus", emodulus.load.INTERNAL_LUTS["LE-2D-FEM-19"]) emodulus.register_lut(path, identifier=identifier) ds2 = dclab.new_dataset(ddict) ds2.config["setup"]["flow rate"] = 0.16 ds2.config["setup"]["channel width"] = 30 ds2.config["imaging"]["pixel size"] = .34 ds2.config["calculation"] = {"emodulus lut": identifier, "emodulus medium": "CellCarrier", "emodulus temperature": 23.0 } assert np.sum(~np.isnan(ds2["emodulus"])) > 0 assert np.allclose(ds["emodulus"], ds2["emodulus"], equal_nan=True, rtol=0, atol=1e-15)
def test_af_emodulus_visc_only_2(): keys = ["area_um", "deform"] ddict = example_data_dict(size=8472, keys=keys) visc = dclab.features.emodulus.viscosity.get_viscosity( medium="CellCarrier", channel_width=30, flow_rate=0.16, temperature=23.0) # legacy ds = dclab.new_dataset(ddict) ds.config["setup"]["flow rate"] = 0.16 ds.config["setup"]["channel width"] = 30 ds.config["imaging"]["pixel size"] = .34 ds.config["calculation"] = {"emodulus lut": "LE-2D-FEM-19", "emodulus medium": "other", "emodulus temperature": 47.0, # irrelevant "emodulus viscosity": visc } # visc-only ds2 = dclab.new_dataset(ddict) ds2.config["setup"]["flow rate"] = 0.16 ds2.config["setup"]["channel width"] = 30 ds2.config["imaging"]["pixel size"] = .34 ds2.config["calculation"] = {"emodulus lut": "LE-2D-FEM-19", "emodulus viscosity": visc } assert np.sum(~np.isnan(ds["emodulus"])) > 0 assert np.allclose(ds["emodulus"], ds2["emodulus"], equal_nan=True, rtol=0, atol=1e-15)
def test_collect_data_tree(): features = ["area_um", "deform", "time"] edest = pathlib.Path(tempfile.mkdtemp(prefix="shapeout_test")) for ii in range(1, 4): dat = new_dataset(data=example_data_dict(ii + 10, keys=features)) cfg = {"experiment": {"sample": "test sample", "run index": ii}, "imaging": {"pixel size": 0.34}, "setup": {"channel width": 20, "chip region": "channel", "flow rate": 0.04} } dat.config.update(cfg) dat.export.hdf5(path=edest / "{}.rtdc".format(ii), features=features) data = meta_tool.collect_data_tree([edest])[0] assert len(data) == 1, "only one data folder" assert len(data[0]) == 4, "three measurements" # check name assert data[0][0][0] == "test sample" # check order assert data[0][1][1].endswith("1.rtdc") assert data[0][2][1].endswith("2.rtdc") assert data[0][3][1].endswith("3.rtdc") shutil.rmtree(str(edest), ignore_errors=True)
def test_with_rtdc(): keys = ["area_um", "deform"] ddict = example_data_dict(size=8472, keys=keys) # legacy ds = dclab.new_dataset(ddict) ds.config["setup"]["flow rate"] = 0.16 ds.config["setup"]["channel width"] = 30 ds.config["setup"]["temperature"] = 23.0 ds.config["setup"]["medium"] = "CellCarrier" ds.config["imaging"]["pixel size"] = .34 i1 = iso.get_default() data1 = i1.get_with_rtdcbase(col1="area_um", col2="deform", lut_identifier="LE-2D-FEM-19", dataset=ds, viscosity=None, add_px_err=False) viscosity = emodulus.viscosity.get_viscosity( medium="CellCarrier", channel_width=ds.config["setup"]["channel width"], flow_rate=ds.config["setup"]["flow rate"], temperature=ds.config["setup"]["temperature"]) data2 = i1.get(col1="area_um", col2="deform", lut_identifier="LE-2D-FEM-19", channel_width=ds.config["setup"]["channel width"], flow_rate=ds.config["setup"]["flow rate"], viscosity=viscosity, add_px_err=False, px_um=ds.config["imaging"]["pixel size"]) for d1, d2 in zip(data1, data2): assert np.allclose(d1, d2, atol=0, rtol=1e-14)
def test_kde_log_scatter(): ddict = example_data_dict(size=300, keys=["area_um", "deform"]) ddict["deform"][:20] = .1 ddict["area_um"][:20] = .5 ds = dclab.new_dataset(ddict) a = ds.get_kde_scatter(yscale="log") assert np.all(a[:20] == a[0])
def test_manual_exclude_parent_changed(): data = example_data_dict(42, keys=["area_um", "tilt"]) p = new_dataset(data) p.filter.manual[4] = False c = new_dataset(p) c.filter.manual[5] = False c.apply_filter() p.config["filtering"]["tilt min"] = 0 p.config["filtering"]["tilt max"] = .5 p.apply_filter() assert np.sum(p.filter.all) == 21 # size of child is directly determined from parent assert len(c) == 21 # filters have not yet been updated assert len(c.filter.all) == 41 assert c.filter.parent_changed # the initially excluded event assert c.filter.retrieve_manual_indices(c) == [6] # try to change the excluded events try: c.filter.apply_manual_indices(c, [1, 2]) except fmt_hierarchy.HierarchyFilterError: pass else: assert False, "expected HierarchyFilterError" # this can be resolved by applying the filter c.apply_filter() c.filter.apply_manual_indices(c, [1, 2]) assert c.filter.retrieve_manual_indices(c) == [1, 2]
def test_invert_copy(): dclab.PolygonFilter.clear_all_filters() ddict = example_data_dict(size=1234, keys=["aspect", "tilt"]) ds = dclab.new_dataset(ddict) # points of polygon filter points = [[np.min(ddict["aspect"]), np.min(ddict["tilt"])], [np.min(ddict["aspect"]), np.max(ddict["tilt"])], [np.average(ddict["aspect"]), np.max(ddict["tilt"])], [np.average(ddict["aspect"]), np.min(ddict["tilt"])], ] filt1 = dclab.PolygonFilter(axes=["aspect", "tilt"], points=points, inverted=False) ds.polygon_filter_add(filt1) assert [0] == ds.config["filtering"]["polygon filters"] n1 = np.sum(ds.filter.all) ds.apply_filter() n2 = np.sum(ds.filter.all) assert n1 != n2 filt2 = filt1.copy(invert=True) ds.polygon_filter_add(filt2) assert [0, 1] == ds.config["filtering"]["polygon filters"] ds.apply_filter() assert np.sum(ds.filter.all) == 0, "inverted+normal filter filters all" dclab.PolygonFilter.clear_all_filters()
def test_invert_saveload(): dclab.PolygonFilter.clear_all_filters() ddict = example_data_dict(size=1234, keys=["aspect", "tilt"]) # points of polygon filter points = [[np.min(ddict["aspect"]), np.min(ddict["tilt"])], [np.min(ddict["aspect"]), np.max(ddict["tilt"])], [np.average(ddict["aspect"]), np.max(ddict["tilt"])], [np.average(ddict["aspect"]), np.min(ddict["tilt"])], ] filt1 = dclab.PolygonFilter(axes=["aspect", "tilt"], points=points, inverted=True) name = tempfile.mktemp(prefix="test_dclab_polygon_") filt1.save(name) filt2 = dclab.PolygonFilter(filename=name) assert filt2 == filt1 filt3 = dclab.PolygonFilter(axes=["aspect", "tilt"], points=points, inverted=False) try: os.remove(name) except OSError: pass name = tempfile.mktemp(prefix="test_dclab_polygon_") filt3.save(name) filt4 = dclab.PolygonFilter(filename=name) assert filt4 == filt3 try: os.remove(name) except OSError: pass
def test_get_feat_range_opt(): keys = ["area_um", "deform", "fl1_max"] dicts = [example_data_dict(s, keys) for s in [10, 100, 12, 382]] # add a fl2_max column with negative numbers for dd in dicts: lend = len(dd["deform"]) dd["fl2_max"] = np.linspace(-1, 5000, lend) dd["userdef1"] = np.linspace(-2, -3, lend) dd["userdef2"] = np.linspace(-1, 1, lend) #dicts.append(otherdict) anal = analysis.Analysis([dclab.new_dataset(d) for d in dicts]) assert anal.get_feat_range_opt(feature="deform") == (0, .2) assert np.allclose(anal.get_feat_range_opt(feature="area_um"), (0.000507009623987198, 0.9989715591819257)) assert np.allclose(anal.get_feat_range_opt(feature="fl1_max"), (0.0036766675498647317, 0.9970937546290722)) assert np.allclose(anal.get_feat_range_opt(feature="fl1_max", scale="log"), (0.0036766675498647317, 0.9970937546290722)) assert np.allclose(anal.get_feat_range_opt(feature="fl2_max"), (-1, 5000)) assert np.allclose(anal.get_feat_range_opt(feature="fl2_max", scale="log"), (1, 5000)) assert np.allclose(anal.get_feat_range_opt(feature="userdef1"), (-3, -2)) assert np.allclose( anal.get_feat_range_opt(feature="userdef1", scale="log"), (.1, 1)) assert np.allclose(anal.get_feat_range_opt(feature="userdef2"), (-1, 1)) assert np.allclose( anal.get_feat_range_opt(feature="userdef2", scale="log"), (0.051197602631569354, 1.0))
def test_kde_log_scatter_points(): ddict = example_data_dict(size=300, keys=["area_um", "deform"]) ds = dclab.new_dataset(ddict) a = ds.get_kde_scatter(yscale="log") b = ds.get_kde_scatter(yscale="log", positions=[ds["area_um"], ds["deform"]]) assert np.all(a == b)
def test_wrong_things(): ddict = example_data_dict(size=67, keys=["area_um", "deform"]) ds = new_dataset(ddict) # Check unknown variable (warning will be displayed try: ds.apply_filter(force=["on_purpose_unknown"]) except ValueError: pass
def test_downsample_none(): keys = ["area_um", "deform", "time", "frame", "fl3_width"] ddict = example_data_dict(size=8472, keys=keys) ds = dclab.new_dataset(ddict) ds.apply_filter() _, _, idx = ds.get_downsampled_scatter(downsample=0, ret_mask=True) assert np.sum(idx) == 8472
def test_data_size(): dicts = [example_data_dict(s) for s in [10, 100, 12, 382]] anal = analysis.Analysis([dclab.new_dataset(d) for d in dicts]) minsize = anal.ForceSameDataSize() assert minsize == 10 for mm in anal.measurements: assert np.sum(mm._filter) == minsize
def test_tsv_not_filtered(): keys = ["area_um", "deform", "time", "frame", "fl3_width"] ddict = example_data_dict(size=127, keys=keys) ds = dclab.new_dataset(ddict) edest = tempfile.mkdtemp() f1 = join(edest, "test.tsv") ds.export.tsv(f1, keys, filtered=False)
def test_axes_usable(): keys = ["area_um", "circ"] dicts = [example_data_dict(s, keys=keys) for s in [10, 100, 12, 382]] anal = analysis.Analysis([dclab.new_dataset(d) for d in dicts]) axes = anal.GetUsableAxes() for ax in keys: assert ax in axes
def test_filter_manual(): # make sure min/max values are filtered ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) ds.filter.manual[[0, 8471]] = False ds.apply_filter() assert len(ds["deform"][ds.filter.all]) == 8470 assert ds["deform"][1] == ds["deform"][ds.filter.all][0]
def test_wrong_things(): ddict = example_data_dict(size=67, keys=["area_um", "deform"]) ds = new_dataset(ddict) # Check unknown variable (warning will be displayed try: ds.apply_filter(force=["on_purpose_unknown"]) except ValueError: pass
def test_index_deep_contour(): data = example_data_dict(42, keys=["area_um", "contour", "deform"]) ds = new_dataset(data) ds.filter.manual[3] = False c1 = new_dataset(ds) c1.filter.manual[1] = False c2 = new_dataset(c1) assert np.all(c2["contour"][3] == ds["contour"][5])
def test_ic_flow_rate_not_zero(): ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) ds.config["setup"]["flow rate"] = 0 with check.IntegrityChecker(ds) as ic: cues = ic.check_metadata_bad_greater_zero() assert cues[0].category == "metadata wrong" assert cues[0].cfg_section == "setup" assert cues[0].cfg_key == "flow rate"
def test_ic_expand_section(): ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds1 = new_dataset(ddict) ds2 = new_dataset(ddict) with check.IntegrityChecker(ds1) as ic: cues1 = ic.check_metadata_missing(expand_section=True) with check.IntegrityChecker(ds2) as ic: cues2 = ic.check_metadata_missing(expand_section=False) assert len(cues1) > len(cues2)
def test_ic_fl_metadata_channel_names_2(): ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) ds.config["fluorescence"]["channel 1 name"] = "peter" with check.IntegrityChecker(ds) as ic: cues = ic.check_fl_metadata_channel_names() assert cues[0].category == "metadata invalid" assert cues[0].cfg_section == "fluorescence" assert cues[0].cfg_key == "channel 1 name"
def test_temperature(): # there are probably a million things wrong with this dataset, but # we are only looking for the temperature thing ddict = example_data_dict(size=8472, keys=["area_um", "deform", "temp"]) ds = new_dataset(ddict) sstr = "Metadata: Missing key [setup] 'temperature', " \ + "because the 'temp' feature is given" _, aler, _ = check_dataset(ds) assert sstr in aler
def test_downsample_none(): keys = ["area_um", "deform", "time", "frame", "fl3_width"] ddict = example_data_dict(size=8472, keys=keys) ds = dclab.new_dataset(ddict) assert np.sum(ds._plot_filter) == 8472 ds.apply_filter() ds.get_downsampled_scatter(downsample=0) assert np.sum(ds._plot_filter) == 8472
def test_ic_metadata_choices_medium(): ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) ds.config["setup"]["medium"] = "honey" with check.IntegrityChecker(ds) as ic: cues = ic.check_metadata_choices() # changed in 0.29.1: medium can now be an arbitrary string # except for an empty string. assert len(cues) == 0
def test_disable_filters(): """Disabling the filters should only affect RTDCBase.filter.all""" ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) ds.filter.manual[[0, 8471]] = False ds.apply_filter() ds.config["filtering"]["enable filters"] = False ds.apply_filter() assert np.alltrue(ds.filter.all)
def test_kde_positions(): ddict = example_data_dict() ds = dclab.new_dataset(ddict) ds.config["filtering"]["enable filters"] = False sc = ds.get_kde_scatter(xax="area_um", yax="deform") sc2 = ds.get_kde_scatter(xax="area_um", yax="deform", positions=(ds["area_um"], ds["deform"])) assert np.all(sc == sc2)
def test_ic_fl_metadata_channel_names(): ddict = example_data_dict(size=8472, keys=["area_um", "deform", "fl1_max"]) ddict["trace"] = {"fl1_raw": [range(10)] * 1000} ds = new_dataset(ddict) with check.IntegrityChecker(ds) as ic: cues = ic.check_fl_metadata_channel_names() assert cues[0].category == "metadata missing" assert cues[0].cfg_section == "fluorescence" assert cues[0].cfg_key == "channel 1 name"
def test_kde_log_contour(): ddict = example_data_dict(size=300, keys=["area_um", "deform"]) ddict["deform"][:20] = .1 ddict["area_um"][:20] = .5 ds = dclab.new_dataset(ddict) xm, ym, _ = ds.get_kde_contour(yscale="log") dx = np.diff(xm[0]) dy = np.diff(np.log(ym[:, 0])) assert np.allclose(dx, dx[0]) assert np.allclose(dy, dy[0])
def test_af_emodulus_legacy_none2(): keys = ["area_um", "deform"] ddict = example_data_dict(size=8472, keys=keys) ds = dclab.new_dataset(ddict) assert "emodulus" not in ds, "not config for emodulus" ds.config["calculation"] = {"emodulus medium": "CellCarrier", "emodulus temperature": 23.0, "emodulus viscosity": 0.5 } assert "emodulus" not in ds, "emodulus lut should be missing"
def test_kde_nofilt(): ddict = example_data_dict() ds = dclab.new_dataset(ddict) ds.config["filtering"]["enable filters"] = False sc = ds.get_kde_scatter() cc = ds.get_kde_contour() assert sc.shape[0] == 100 # This will fail if the default contour accuracy is changed # in `get_kde_contour`. assert cc[0].shape == (43, 41)
def test_0error(): keys = ["circ"] ddict = example_data_dict(size=8472, keys=keys) ds = dclab.new_dataset(ddict) try: ds["unknown_column"] except KeyError: pass else: raise ValueError("Should have raised KeyError!")
def test_af_0error(): keys = ["circ"] ddict = example_data_dict(size=8472, keys=keys) ds = dclab.new_dataset(ddict) try: ds["unknown_column"] except KeyError: pass else: raise ValueError("Should have raised KeyError!")
def test_emodulus_none2(): keys = ["area_um", "deform"] ddict = example_data_dict(size=8472, keys=keys) ds = dclab.new_dataset(ddict) assert "emodulus" not in ds, "not config for emodulus" ds.config["calculation"] = {"emodulus medium": "CellCarrier", "emodulus temperature": 23.0, "emodulus viscosity": 0.5 } assert "emodulus" not in ds, "emodulus model should be missing"
def test_tsv_not_filtered(): keys = ["area_um", "deform", "time", "frame", "fl3_width"] ddict = example_data_dict(size=127, keys=keys) ds = dclab.new_dataset(ddict) edest = tempfile.mkdtemp() f1 = join(edest, "test.tsv") ds.export.tsv(f1, keys, filtered=False) # cleanup shutil.rmtree(edest, ignore_errors=True)
def test_min_max_update(): ddict = example_data_dict(size=67, keys=["area_um", "deform"]) ds = new_dataset(ddict) cfg = ds.config.copy() # Force updating circularity cfg["filtering"]["defo min"] = .4 cfg["filtering"]["defo max"] = .8 ds.config.update(cfg) ds.apply_filter()
def test_limit_above(): keys = ["area_um", "deform", "time", "frame"] ddict = example_data_dict(size=9999, keys=keys) ds = dclab.new_dataset(ddict) assert np.sum(ds._filter) == 9999 filtflt = {"limit events": 10000} cfg = {"filtering": filtflt} ds.config.update(cfg) ds.apply_filter() assert np.sum(ds._filter) == 9999
def test_kde_log_scatter_invalid(): ddict = example_data_dict(size=300, keys=["area_um", "deform"]) ddict["deform"][:20] = .1 ddict["area_um"][:20] = .5 ddict["deform"][21] = np.nan ddict["deform"][22] = np.inf ddict["deform"][23] = -.1 ds = dclab.new_dataset(ddict) a = ds.get_kde_scatter(yscale="log") assert np.all(a[:20] == a[0]) assert np.isnan(a[21]) assert np.isnan(a[22]) assert np.isnan(a[23])
def test_kde_general(): # Download and extract data ddict = example_data_dict() ds = dclab.new_dataset(ddict) dcont = [] dscat = [] for kde_type in dclab.kde_methods.methods: dcont.append(ds.get_kde_contour(kde_type=kde_type)) dscat.append(ds.get_kde_scatter(kde_type=kde_type)) for ii in range(1, len(dcont) - 1): assert not np.allclose(dcont[ii], dcont[0]) assert not np.allclose(dscat[ii], dscat[0])
def test_avi_no_images(): keys = ["area_um", "deform", "time", "frame", "fl3_width"] ddict = example_data_dict(size=127, keys=keys) ds = dclab.new_dataset(ddict) edest = tempfile.mkdtemp() f1 = join(edest, "test.avi") try: ds.export.avi(f1) except OSError: pass else: raise ValueError("There should be no image data to write!") shutil.rmtree(edest, ignore_errors=True)
def test_downsample_up(): """ Likely causes removal of too many points and requires re-inserting them. """ keys = ["area_um", "deform", "time", "frame", "fl3_width"] ddict = example_data_dict(size=10000, keys=keys) ds = dclab.new_dataset(ddict) assert np.sum(ds._plot_filter) == 10000 ds.apply_filter() ds.get_downsampled_scatter(downsample=9999) assert np.sum(ds._plot_filter) == 9999 ds.get_downsampled_scatter(downsample=9999)
def test_filter_min_max(): # make sure min/max values are filtered ddict = example_data_dict(size=8472, keys=["area_um", "deform"]) ds = new_dataset(ddict) amin, amax = ds["area_um"].min(), ds["area_um"].max() ds.config["filtering"]["area_um min"] = (amax + amin) / 2 ds.config["filtering"]["area_um max"] = amax ds.apply_filter() assert np.sum(ds.filter.all) == 4256 # make sure data is not filtered before calling ds.apply_filter dmin, dmax = ds["deform"].min(), ds["deform"].max() ds.config["filtering"]["deform min"] = (dmin + dmax) / 2 ds.config["filtering"]["deform max"] = dmax assert np.sum(ds.filter.all) == 4256
def test_fcs_override(): keys = ["area_um", "deform", "time", "frame", "fl3_width"] ddict = example_data_dict(size=212, keys=keys) ds = dclab.new_dataset(ddict) edest = tempfile.mkdtemp() f1 = join(edest, "test.fcs") ds.export.fcs(f1, keys, override=True) try: ds.export.fcs(f1[:-4], keys, override=False) except OSError: pass else: raise ValueError("Should append .fcs and not override!") # cleanup shutil.rmtree(edest, ignore_errors=True)
def test_inverted_wrong(): dclab.PolygonFilter.clear_all_filters() ddict = example_data_dict(size=1234, keys=["area_um", "deform"]) # points of polygon filter points = [[np.min(ddict["area_um"]), np.min(ddict["deform"])], [np.min(ddict["area_um"]), np.max(ddict["deform"])], [np.average(ddict["area_um"]), np.max(ddict["deform"])], [np.average(ddict["area_um"]), np.min(ddict["deform"])], ] try: dclab.PolygonFilter(axes=["area_um", "deform"], points=points, inverted=0) except dclab.polygon_filter.PolygonFilterError: pass else: raise ValueError("inverted should only be allowed to be bool")
def test_downsample_nan(): """Deal with nans""" keys = ["area_um", "deform", "time", "frame", "fl3_width"] ddict = example_data_dict(size=8472, keys=keys) ddict["area_um"][:4700] = np.nan ds = dclab.new_dataset(ddict) ds.config["filtering"]["limit events"] = 2000 assert np.sum(ds.filter.all) == 8472 ds.apply_filter() assert np.sum(ds.filter.all) == 2000 # this is the general test; nans are included assert np.sum(np.isnan(ds["area_um"][ds.filter.all])) != 0 # this is the exact test (e.g. for the random state used) assert np.sum(np.isnan(ds["area_um"][ds.filter.all])) == 1150
def test_emodulus_area(): # computes "area_um" from "area_cvx" keys = ["area_cvx", "deform"] ddict = example_data_dict(size=8472, keys=keys) ds = dclab.new_dataset(ddict) # area can be computed from areapix ds.config["imaging"]["pixel size"] = .34 assert "area_um" in ds assert "emodulus" not in ds, "not config for emodulus" ds.config["setup"]["flow rate"] = 0.16 ds.config["setup"]["channel width"] = 30 ds.config["calculation"] = {"emodulus medium": "CellCarrier", "emodulus model": "elastic sphere", "emodulus temperature": 23.0, "emodulus viscosity": 0.5 } assert "emodulus" in ds
def test_downsample_log(): keys = ["area_um", "deform", "time", "frame", "fl3_width"] ddict = example_data_dict(size=8472, keys=keys) ds = dclab.new_dataset(ddict) ds.apply_filter() xlin, ylin = ds.get_downsampled_scatter(downsample=100, xscale="linear", yscale="linear") xlog, ylog = ds.get_downsampled_scatter(downsample=100, xscale="log", yscale="log") assert not np.all(xlin == xlog) assert not np.all(ylin == ylog) for x in xlog: assert x in ds["area_um"] for y in ylog: assert y in ds["deform"]
def test_collect_data_tree_unicode(): features = ["area_um", "deform", "time"] edest = pathlib.Path(tempfile.mkdtemp(prefix="shapeout_test")) from shapeout.util import safe_path for ii in range(1, 4): dat = new_dataset(data=example_data_dict(ii + 10, keys=features)) cfg = {"experiment": {"sample": "test sample", "run index": ii}, "imaging": {"pixel size": 0.34}, "setup": {"channel width": 20, "chip region": "channel", "flow rate": 0.04} } dat.config.update(cfg) dat.export.hdf5(path=safe_path(edest / "únícòdè_{}.rtdc".format(ii)), features=features) meta_tool.collect_data_tree([edest])[0] shutil.rmtree(str(edest), ignore_errors=True)
def test_emodulus(): keys = ["area_um", "deform"] ddict = example_data_dict(size=8472, keys=keys) ds = dclab.new_dataset(ddict) ds.config["setup"]["flow rate"] = 0.16 ds.config["setup"]["channel width"] = 30 ds.config["imaging"]["pixel size"] = .34 ds.config["calculation"] = {"emodulus medium": "CellCarrier", "emodulus model": "elastic sphere", "emodulus temperature": 23.0, "emodulus viscosity": 0.5 } t1 = time.time() assert "emodulus" in ds t2 = time.time() t3 = time.time() ds["emodulus"] t4 = time.time() assert t4 - t3 > t2 - t1
def test_false_method(): def bad_method(x): return x + 1 dclab.statistics.Statistics(name="bad", req_feature=False, method=bad_method) ddict = example_data_dict(size=77, keys=["area_um", "deform"]) ds = dclab.new_dataset(ddict) head1, vals1 = dclab.statistics.get_statistics(ds, features=["deform"]) out = {} for h, v in zip(head1, vals1): out[h] = v assert np.isnan(out["bad"]) # clean up mth = dclab.statistics.Statistics.available_methods for k in mth: if k == "bad": mth.pop(k) break