def test_ic_fl_max_ctc():
# Testing dataset with negative fl_max_ctc values
ddict = example_data_dict(size=8472, keys=["fl1_max_ctc"])
ddict["fl1_max_ctc"] -= min(ddict["fl1_max_ctc"]) + 1
ds = new_dataset(ddict)
with check.IntegrityChecker(ds) as ic:
cues = ic.check_fl_max_ctc_positive()
assert cues[0].level == "alert"
assert cues[0].category == "feature data"
# Testing dataset with fl_max_ctc values of 0.1
ddict = example_data_dict(size=8472, keys=["fl1_max_ctc"])
ddict["fl1_max_ctc"] -= min(ddict["fl1_max_ctc"]) - 0.1
ds = new_dataset(ddict)
with check.IntegrityChecker(ds) as ic:
cues = ic.check_fl_max_ctc_positive()
assert cues[0].level == "alert"
assert cues[0].category == "feature data"
# Testing dataset with fl_max_ctc values > 0.1
ddict = example_data_dict(size=8472, keys=["fl1_max_ctc"])
ddict["fl1_max_ctc"] -= min(ddict["fl1_max_ctc"]) - 1
ds = new_dataset(ddict)
with check.IntegrityChecker(ds) as ic:
cues = ic.check_fl_max_ctc_positive()
assert not cues
def test_kde_log_scatter():
ddict = example_data_dict(size=300, keys=["area_um", "deform"])
ddict["deform"][:20] = .1
ddict["area_um"][:20] = .5
ds = dclab.new_dataset(ddict)
a = ds.get_kde_scatter(yscale="log")
assert np.all(a[:20] == a[0])
def test_invert_copy():
dclab.PolygonFilter.clear_all_filters()
ddict = example_data_dict(size=1234, keys=["area_um", "deform"])
ds = dclab.new_dataset(ddict)
# points of polygon filter
points = [[np.min(ddict["area_um"]), np.min(ddict["deform"])],
[np.min(ddict["area_um"]), np.max(ddict["deform"])],
[np.average(ddict["area_um"]), np.max(ddict["deform"])],
[np.average(ddict["area_um"]), np.min(ddict["deform"])],
]
filt1 = dclab.PolygonFilter(axes=["area_um", "deform"],
points=points,
inverted=False)
ds.polygon_filter_add(filt1)
assert [0] == ds.config["filtering"]["polygon filters"]
n1 = np.sum(ds._filter)
ds.apply_filter()
n2 = np.sum(ds._filter)
assert n1 != n2
filt2 = filt1.copy(invert=True)
ds.polygon_filter_add(filt2)
assert [0, 1] == ds.config["filtering"]["polygon filters"]
ds.apply_filter()
assert np.sum(ds._filter) == 0, "inverted+normal filter filters all"
dclab.PolygonFilter.clear_all_filters()
def test_shape_dict_trace():
ddict = example_data_dict(size=67, keys=["area_um", "deform", "trace"])
ds = new_dataset(ddict)
assert ds["trace"].shape == (6, 67, 100)
assert len(ds["trace"]) == 6
assert len(ds["trace"]["fl1_raw"]) == 67
assert len(ds["trace"]["fl1_raw"][0]) == 100
def test_af_emodulus_reservoir():
"""Reservoir measurements should not have emodulus"""
keys = ["area_um", "deform"]
ddict = example_data_dict(size=8472, keys=keys)
# legacy
ds = dclab.new_dataset(ddict)
ds.config["setup"]["flow rate"] = 0.16
ds.config["setup"]["channel width"] = 30
ds.config["imaging"]["pixel size"] = .34
ds.config["calculation"] = {"emodulus lut": "LE-2D-FEM-19",
"emodulus medium": "CellCarrier",
"emodulus temperature": 23.0,
"emodulus viscosity": 0.5
}
assert "emodulus" in ds
ds2 = dclab.new_dataset(ddict)
ds2.config["setup"]["flow rate"] = 0.16
ds2.config["setup"]["channel width"] = 30
ds2.config["imaging"]["pixel size"] = .34
ds2.config["calculation"] = {"emodulus lut": "LE-2D-FEM-19",
"emodulus medium": "CellCarrier",
"emodulus temperature": 23.0,
"emodulus viscosity": 0.5
}
ds2.config["setup"]["chip region"] = "reservoir"
assert "emodulus" not in ds2
def test_hdf5_filtered():
N = 10
keys = ["area_um", "image"]
ddict = example_data_dict(size=N, keys=keys)
ddict["image"][3] = np.arange(10 * 20, dtype=np.uint8).reshape(10, 20) + 22
ds1 = dclab.new_dataset(ddict)
ds1.config["experiment"]["sample"] = "test"
ds1.config["experiment"]["run index"] = 1
ds1.filter.manual[2] = False
ds1.apply_filter()
fta = ds1.filter.manual.copy()
edest = tempfile.mkdtemp()
f1 = join(edest, "dclab_test_export_hdf5_filtered.rtdc")
ds1.export.hdf5(f1, keys)
ds2 = dclab.new_dataset(f1)
assert ds1 != ds2
assert np.allclose(ds2["area_um"], ds1["area_um"][fta])
assert np.allclose(ds2["image"][2], ds1["image"][3])
assert np.all(ds2["image"][2] != ds1["image"][2])
# cleanup
shutil.rmtree(edest, ignore_errors=True)
def test_collect_data_tree():
features = ["area_um", "deform", "time"]
edest = pathlib.Path(tempfile.mkdtemp(prefix="shapeout_test"))
for ii in range(1, 4):
dat = new_dataset(data=example_data_dict(ii + 10, keys=features))
cfg = {"experiment": {"sample": "test sample",
"run index": ii},
"imaging": {"pixel size": 0.34},
"setup": {"channel width": 20,
"chip region": "channel",
"flow rate": 0.04}
}
dat.config.update(cfg)
dat.export.hdf5(path=edest / "{}.rtdc".format(ii),
features=features)
data = meta_tool.collect_data_tree([edest])[0]
assert len(data) == 1, "only one data folder"
assert len(data[0]) == 4, "three measurements"
# check name
assert data[0][0][0] == "test sample"
# check order
assert data[0][1][1].endswith("1.rtdc")
assert data[0][2][1].endswith("2.rtdc")
assert data[0][3][1].endswith("3.rtdc")
shutil.rmtree(str(edest), ignore_errors=True)
def test_collect_data_tree_order():
features = ["area_um", "deform", "time"]
edest = pathlib.Path(tempfile.mkdtemp(prefix="shapeout_test"))
for ii in range(1, 13):
dat = new_dataset(data=example_data_dict(ii + 10, keys=features))
cfg = {"experiment": {"sample": "test sample",
"run index": ii},
"imaging": {"pixel size": 0.34},
"setup": {"channel width": 20,
"chip region": "channel",
"flow rate": 0.04}
}
dat.config.update(cfg)
dat.export.hdf5(path=edest / "M{}_data.rtdc".format(ii),
features=features)
data = meta_tool.collect_data_tree([edest])[0]
assert pathlib.Path(data[0][1][1]).name == "M1_data.rtdc"
assert pathlib.Path(data[0][2][1]).name == "M2_data.rtdc"
assert pathlib.Path(data[0][3][1]).name == "M3_data.rtdc"
assert pathlib.Path(data[0][4][1]).name == "M4_data.rtdc"
assert pathlib.Path(data[0][5][1]).name == "M5_data.rtdc"
assert pathlib.Path(data[0][6][1]).name == "M6_data.rtdc"
assert pathlib.Path(data[0][7][1]).name == "M7_data.rtdc"
assert pathlib.Path(data[0][8][1]).name == "M8_data.rtdc"
assert pathlib.Path(data[0][9][1]).name == "M9_data.rtdc"
assert pathlib.Path(data[0][10][1]).name == "M10_data.rtdc"
assert pathlib.Path(data[0][11][1]).name == "M11_data.rtdc"
assert pathlib.Path(data[0][12][1]).name == "M12_data.rtdc"
shutil.rmtree(str(edest), ignore_errors=True)
def test_with_rtdc_warning():
keys = ["area_um", "deform"]
ddict = example_data_dict(size=8472, keys=keys)
# legacy
ds = dclab.new_dataset(ddict)
ds.config["setup"]["flow rate"] = 0.16
ds.config["setup"]["channel width"] = 30
ds.config["setup"]["medium"] = "CellCarrier"
ds.config["imaging"]["pixel size"] = .34
i1 = iso.get_default()
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
# Trigger a warning (temperature missing).
i1.get_with_rtdcbase(col1="area_um",
col2="deform",
lut_identifier="LE-2D-FEM-19",
dataset=ds,
viscosity=None,
add_px_err=False)
# Verify some things
assert len(w) == 1
assert issubclass(w[-1].category,
iso.IsoelasticsEmodulusMeaninglessWarning)
assert "plotting" in str(w[-1].message)
def test_manual_exclude():
data = example_data_dict(42, keys=["area_um", "deform"])
p = new_dataset(data)
c1 = new_dataset(p)
c2 = new_dataset(c1)
c3 = new_dataset(c2)
c1.filter.manual[0] = False
c2.apply_filter()
c2.filter.manual[1] = False
c3.apply_filter()
# simple exclusion of few events
assert len(c3) == len(p) - 2
# removing same event in parent removes the event from the
# child altogether, including the manual filter
c3.filter.manual[0] = False
c2.filter.manual[0] = False
c3.apply_filter()
assert np.alltrue(c3.filter.manual)
# reinserting the event in the parent, retrieves back
# the manual filter in the child
c2.filter.manual[0] = True
c3.apply_filter()
assert not c3.filter.manual[0]
def test_invert_saveload():
dclab.PolygonFilter.clear_all_filters()
ddict = example_data_dict(size=1234, keys=["area_um", "deform"])
# points of polygon filter
points = [[np.min(ddict["area_um"]), np.min(ddict["deform"])],
[np.min(ddict["area_um"]), np.max(ddict["deform"])],
[np.average(ddict["area_um"]), np.max(ddict["deform"])],
[np.average(ddict["area_um"]), np.min(ddict["deform"])],
]
filt1 = dclab.PolygonFilter(axes=["area_um", "deform"],
points=points,
inverted=True)
name = tempfile.mktemp(prefix="test_dclab_polygon_")
filt1.save(name)
filt2 = dclab.PolygonFilter(filename=name)
assert filt2 == filt1
filt3 = dclab.PolygonFilter(axes=["area_um", "deform"],
points=points,
inverted=False)
try:
os.remove(name)
except OSError:
pass
name = tempfile.mktemp(prefix="test_dclab_polygon_")
filt3.save(name)
filt4 = dclab.PolygonFilter(filename=name)
assert filt4 == filt3
try:
os.remove(name)
except OSError:
pass
def test_register_external_lut_and_get_emodulus():
keys = ["area_um", "deform"]
ddict = example_data_dict(size=8472, keys=keys)
# from internal LUT
ds = dclab.new_dataset(ddict)
ds.config["setup"]["flow rate"] = 0.16
ds.config["setup"]["channel width"] = 30
ds.config["imaging"]["pixel size"] = .34
ds.config["calculation"] = {"emodulus lut": "LE-2D-FEM-19",
"emodulus medium": "CellCarrier",
"emodulus temperature": 23.0
}
assert np.sum(~np.isnan(ds["emodulus"])) > 0
# from external LUT
identifier = "test-test_register_external_lut"
path = resource_filename("dclab.features.emodulus",
emodulus.load.INTERNAL_LUTS["LE-2D-FEM-19"])
emodulus.register_lut(path, identifier=identifier)
ds2 = dclab.new_dataset(ddict)
ds2.config["setup"]["flow rate"] = 0.16
ds2.config["setup"]["channel width"] = 30
ds2.config["imaging"]["pixel size"] = .34
ds2.config["calculation"] = {"emodulus lut": identifier,
"emodulus medium": "CellCarrier",
"emodulus temperature": 23.0
}
assert np.sum(~np.isnan(ds2["emodulus"])) > 0
assert np.allclose(ds["emodulus"], ds2["emodulus"], equal_nan=True,
rtol=0, atol=1e-15)
def test_af_emodulus_visc_only_2():
keys = ["area_um", "deform"]
ddict = example_data_dict(size=8472, keys=keys)
visc = dclab.features.emodulus.viscosity.get_viscosity(
medium="CellCarrier",
channel_width=30,
flow_rate=0.16,
temperature=23.0)
# legacy
ds = dclab.new_dataset(ddict)
ds.config["setup"]["flow rate"] = 0.16
ds.config["setup"]["channel width"] = 30
ds.config["imaging"]["pixel size"] = .34
ds.config["calculation"] = {"emodulus lut": "LE-2D-FEM-19",
"emodulus medium": "other",
"emodulus temperature": 47.0, # irrelevant
"emodulus viscosity": visc
}
# visc-only
ds2 = dclab.new_dataset(ddict)
ds2.config["setup"]["flow rate"] = 0.16
ds2.config["setup"]["channel width"] = 30
ds2.config["imaging"]["pixel size"] = .34
ds2.config["calculation"] = {"emodulus lut": "LE-2D-FEM-19",
"emodulus viscosity": visc
}
assert np.sum(~np.isnan(ds["emodulus"])) > 0
assert np.allclose(ds["emodulus"], ds2["emodulus"], equal_nan=True,
rtol=0, atol=1e-15)
def test_collect_data_tree():
features = ["area_um", "deform", "time"]
edest = pathlib.Path(tempfile.mkdtemp(prefix="shapeout_test"))
for ii in range(1, 4):
dat = new_dataset(data=example_data_dict(ii + 10, keys=features))
cfg = {"experiment": {"sample": "test sample",
"run index": ii},
"imaging": {"pixel size": 0.34},
"setup": {"channel width": 20,
"chip region": "channel",
"flow rate": 0.04}
}
dat.config.update(cfg)
dat.export.hdf5(path=edest / "{}.rtdc".format(ii),
features=features)
data = meta_tool.collect_data_tree([edest])[0]
assert len(data) == 1, "only one data folder"
assert len(data[0]) == 4, "three measurements"
# check name
assert data[0][0][0] == "test sample"
# check order
assert data[0][1][1].endswith("1.rtdc")
assert data[0][2][1].endswith("2.rtdc")
assert data[0][3][1].endswith("3.rtdc")
shutil.rmtree(str(edest), ignore_errors=True)
def test_with_rtdc():
keys = ["area_um", "deform"]
ddict = example_data_dict(size=8472, keys=keys)
# legacy
ds = dclab.new_dataset(ddict)
ds.config["setup"]["flow rate"] = 0.16
ds.config["setup"]["channel width"] = 30
ds.config["setup"]["temperature"] = 23.0
ds.config["setup"]["medium"] = "CellCarrier"
ds.config["imaging"]["pixel size"] = .34
i1 = iso.get_default()
data1 = i1.get_with_rtdcbase(col1="area_um",
col2="deform",
lut_identifier="LE-2D-FEM-19",
dataset=ds,
viscosity=None,
add_px_err=False)
viscosity = emodulus.viscosity.get_viscosity(
medium="CellCarrier",
channel_width=ds.config["setup"]["channel width"],
flow_rate=ds.config["setup"]["flow rate"],
temperature=ds.config["setup"]["temperature"])
data2 = i1.get(col1="area_um",
col2="deform",
lut_identifier="LE-2D-FEM-19",
channel_width=ds.config["setup"]["channel width"],
flow_rate=ds.config["setup"]["flow rate"],
viscosity=viscosity,
add_px_err=False,
px_um=ds.config["imaging"]["pixel size"])
for d1, d2 in zip(data1, data2):
assert np.allclose(d1, d2, atol=0, rtol=1e-14)
def test_kde_log_scatter():
ddict = example_data_dict(size=300, keys=["area_um", "deform"])
ddict["deform"][:20] = .1
ddict["area_um"][:20] = .5
ds = dclab.new_dataset(ddict)
a = ds.get_kde_scatter(yscale="log")
assert np.all(a[:20] == a[0])
def test_manual_exclude_parent_changed():
data = example_data_dict(42, keys=["area_um", "tilt"])
p = new_dataset(data)
p.filter.manual[4] = False
c = new_dataset(p)
c.filter.manual[5] = False
c.apply_filter()
p.config["filtering"]["tilt min"] = 0
p.config["filtering"]["tilt max"] = .5
p.apply_filter()
assert np.sum(p.filter.all) == 21
# size of child is directly determined from parent
assert len(c) == 21
# filters have not yet been updated
assert len(c.filter.all) == 41
assert c.filter.parent_changed
# the initially excluded event
assert c.filter.retrieve_manual_indices(c) == [6]
# try to change the excluded events
try:
c.filter.apply_manual_indices(c, [1, 2])
except fmt_hierarchy.HierarchyFilterError:
pass
else:
assert False, "expected HierarchyFilterError"
# this can be resolved by applying the filter
c.apply_filter()
c.filter.apply_manual_indices(c, [1, 2])
assert c.filter.retrieve_manual_indices(c) == [1, 2]
def test_invert_copy():
dclab.PolygonFilter.clear_all_filters()
ddict = example_data_dict(size=1234, keys=["aspect", "tilt"])
ds = dclab.new_dataset(ddict)
# points of polygon filter
points = [[np.min(ddict["aspect"]), np.min(ddict["tilt"])],
[np.min(ddict["aspect"]), np.max(ddict["tilt"])],
[np.average(ddict["aspect"]), np.max(ddict["tilt"])],
[np.average(ddict["aspect"]), np.min(ddict["tilt"])],
]
filt1 = dclab.PolygonFilter(axes=["aspect", "tilt"],
points=points,
inverted=False)
ds.polygon_filter_add(filt1)
assert [0] == ds.config["filtering"]["polygon filters"]
n1 = np.sum(ds.filter.all)
ds.apply_filter()
n2 = np.sum(ds.filter.all)
assert n1 != n2
filt2 = filt1.copy(invert=True)
ds.polygon_filter_add(filt2)
assert [0, 1] == ds.config["filtering"]["polygon filters"]
ds.apply_filter()
assert np.sum(ds.filter.all) == 0, "inverted+normal filter filters all"
dclab.PolygonFilter.clear_all_filters()
def test_invert_saveload():
dclab.PolygonFilter.clear_all_filters()
ddict = example_data_dict(size=1234, keys=["aspect", "tilt"])
# points of polygon filter
points = [[np.min(ddict["aspect"]), np.min(ddict["tilt"])],
[np.min(ddict["aspect"]), np.max(ddict["tilt"])],
[np.average(ddict["aspect"]), np.max(ddict["tilt"])],
[np.average(ddict["aspect"]), np.min(ddict["tilt"])],
]
filt1 = dclab.PolygonFilter(axes=["aspect", "tilt"],
points=points,
inverted=True)
name = tempfile.mktemp(prefix="test_dclab_polygon_")
filt1.save(name)
filt2 = dclab.PolygonFilter(filename=name)
assert filt2 == filt1
filt3 = dclab.PolygonFilter(axes=["aspect", "tilt"],
points=points,
inverted=False)
try:
os.remove(name)
except OSError:
pass
name = tempfile.mktemp(prefix="test_dclab_polygon_")
filt3.save(name)
filt4 = dclab.PolygonFilter(filename=name)
assert filt4 == filt3
try:
os.remove(name)
except OSError:
pass
def test_get_feat_range_opt():
keys = ["area_um", "deform", "fl1_max"]
dicts = [example_data_dict(s, keys) for s in [10, 100, 12, 382]]
# add a fl2_max column with negative numbers
for dd in dicts:
lend = len(dd["deform"])
dd["fl2_max"] = np.linspace(-1, 5000, lend)
dd["userdef1"] = np.linspace(-2, -3, lend)
dd["userdef2"] = np.linspace(-1, 1, lend)
#dicts.append(otherdict)
anal = analysis.Analysis([dclab.new_dataset(d) for d in dicts])
assert anal.get_feat_range_opt(feature="deform") == (0, .2)
assert np.allclose(anal.get_feat_range_opt(feature="area_um"),
(0.000507009623987198, 0.9989715591819257))
assert np.allclose(anal.get_feat_range_opt(feature="fl1_max"),
(0.0036766675498647317, 0.9970937546290722))
assert np.allclose(anal.get_feat_range_opt(feature="fl1_max", scale="log"),
(0.0036766675498647317, 0.9970937546290722))
assert np.allclose(anal.get_feat_range_opt(feature="fl2_max"), (-1, 5000))
assert np.allclose(anal.get_feat_range_opt(feature="fl2_max", scale="log"),
(1, 5000))
assert np.allclose(anal.get_feat_range_opt(feature="userdef1"), (-3, -2))
assert np.allclose(
anal.get_feat_range_opt(feature="userdef1", scale="log"), (.1, 1))
assert np.allclose(anal.get_feat_range_opt(feature="userdef2"), (-1, 1))
assert np.allclose(
anal.get_feat_range_opt(feature="userdef2", scale="log"),
(0.051197602631569354, 1.0))
def test_kde_log_scatter_points():
ddict = example_data_dict(size=300, keys=["area_um", "deform"])
ds = dclab.new_dataset(ddict)
a = ds.get_kde_scatter(yscale="log")
b = ds.get_kde_scatter(yscale="log",
positions=[ds["area_um"], ds["deform"]])
assert np.all(a == b)
def test_wrong_things():
ddict = example_data_dict(size=67, keys=["area_um", "deform"])
ds = new_dataset(ddict)
# Check unknown variable (warning will be displayed
try:
ds.apply_filter(force=["on_purpose_unknown"])
except ValueError:
pass
def test_downsample_none():
keys = ["area_um", "deform", "time", "frame", "fl3_width"]
ddict = example_data_dict(size=8472, keys=keys)
ds = dclab.new_dataset(ddict)
ds.apply_filter()
_, _, idx = ds.get_downsampled_scatter(downsample=0, ret_mask=True)
assert np.sum(idx) == 8472
def test_data_size():
dicts = [example_data_dict(s) for s in [10, 100, 12, 382]]
anal = analysis.Analysis([dclab.new_dataset(d) for d in dicts])
minsize = anal.ForceSameDataSize()
assert minsize == 10
for mm in anal.measurements:
assert np.sum(mm._filter) == minsize
def test_tsv_not_filtered():
keys = ["area_um", "deform", "time", "frame", "fl3_width"]
ddict = example_data_dict(size=127, keys=keys)
ds = dclab.new_dataset(ddict)
edest = tempfile.mkdtemp()
f1 = join(edest, "test.tsv")
ds.export.tsv(f1, keys, filtered=False)
def test_axes_usable():
keys = ["area_um", "circ"]
dicts = [example_data_dict(s, keys=keys) for s in [10, 100, 12, 382]]
anal = analysis.Analysis([dclab.new_dataset(d) for d in dicts])
axes = anal.GetUsableAxes()
for ax in keys:
assert ax in axes
def test_filter_manual():
# make sure min/max values are filtered
ddict = example_data_dict(size=8472, keys=["area_um", "deform"])
ds = new_dataset(ddict)
ds.filter.manual[[0, 8471]] = False
ds.apply_filter()
assert len(ds["deform"][ds.filter.all]) == 8470
assert ds["deform"][1] == ds["deform"][ds.filter.all][0]
def test_wrong_things():
ddict = example_data_dict(size=67, keys=["area_um", "deform"])
ds = new_dataset(ddict)
# Check unknown variable (warning will be displayed
try:
ds.apply_filter(force=["on_purpose_unknown"])
except ValueError:
pass
def test_index_deep_contour():
data = example_data_dict(42, keys=["area_um", "contour", "deform"])
ds = new_dataset(data)
ds.filter.manual[3] = False
c1 = new_dataset(ds)
c1.filter.manual[1] = False
c2 = new_dataset(c1)
assert np.all(c2["contour"][3] == ds["contour"][5])
def test_ic_flow_rate_not_zero():
ddict = example_data_dict(size=8472, keys=["area_um", "deform"])
ds = new_dataset(ddict)
ds.config["setup"]["flow rate"] = 0
with check.IntegrityChecker(ds) as ic:
cues = ic.check_metadata_bad_greater_zero()
assert cues[0].category == "metadata wrong"
assert cues[0].cfg_section == "setup"
assert cues[0].cfg_key == "flow rate"
def test_ic_expand_section():
ddict = example_data_dict(size=8472, keys=["area_um", "deform"])
ds1 = new_dataset(ddict)
ds2 = new_dataset(ddict)
with check.IntegrityChecker(ds1) as ic:
cues1 = ic.check_metadata_missing(expand_section=True)
with check.IntegrityChecker(ds2) as ic:
cues2 = ic.check_metadata_missing(expand_section=False)
assert len(cues1) > len(cues2)
def test_ic_fl_metadata_channel_names_2():
ddict = example_data_dict(size=8472, keys=["area_um", "deform"])
ds = new_dataset(ddict)
ds.config["fluorescence"]["channel 1 name"] = "peter"
with check.IntegrityChecker(ds) as ic:
cues = ic.check_fl_metadata_channel_names()
assert cues[0].category == "metadata invalid"
assert cues[0].cfg_section == "fluorescence"
assert cues[0].cfg_key == "channel 1 name"
def test_temperature():
# there are probably a million things wrong with this dataset, but
# we are only looking for the temperature thing
ddict = example_data_dict(size=8472, keys=["area_um", "deform", "temp"])
ds = new_dataset(ddict)
sstr = "Metadata: Missing key [setup] 'temperature', " \
+ "because the 'temp' feature is given"
_, aler, _ = check_dataset(ds)
assert sstr in aler
def test_downsample_none():
keys = ["area_um", "deform", "time", "frame", "fl3_width"]
ddict = example_data_dict(size=8472, keys=keys)
ds = dclab.new_dataset(ddict)
assert np.sum(ds._plot_filter) == 8472
ds.apply_filter()
ds.get_downsampled_scatter(downsample=0)
assert np.sum(ds._plot_filter) == 8472
def test_ic_metadata_choices_medium():
ddict = example_data_dict(size=8472, keys=["area_um", "deform"])
ds = new_dataset(ddict)
ds.config["setup"]["medium"] = "honey"
with check.IntegrityChecker(ds) as ic:
cues = ic.check_metadata_choices()
# changed in 0.29.1: medium can now be an arbitrary string
# except for an empty string.
assert len(cues) == 0
def test_disable_filters():
"""Disabling the filters should only affect RTDCBase.filter.all"""
ddict = example_data_dict(size=8472, keys=["area_um", "deform"])
ds = new_dataset(ddict)
ds.filter.manual[[0, 8471]] = False
ds.apply_filter()
ds.config["filtering"]["enable filters"] = False
ds.apply_filter()
assert np.alltrue(ds.filter.all)
def test_kde_positions():
ddict = example_data_dict()
ds = dclab.new_dataset(ddict)
ds.config["filtering"]["enable filters"] = False
sc = ds.get_kde_scatter(xax="area_um", yax="deform")
sc2 = ds.get_kde_scatter(xax="area_um", yax="deform",
positions=(ds["area_um"], ds["deform"]))
assert np.all(sc == sc2)
def test_ic_fl_metadata_channel_names():
ddict = example_data_dict(size=8472, keys=["area_um", "deform", "fl1_max"])
ddict["trace"] = {"fl1_raw": [range(10)] * 1000}
ds = new_dataset(ddict)
with check.IntegrityChecker(ds) as ic:
cues = ic.check_fl_metadata_channel_names()
assert cues[0].category == "metadata missing"
assert cues[0].cfg_section == "fluorescence"
assert cues[0].cfg_key == "channel 1 name"
def test_kde_log_contour():
ddict = example_data_dict(size=300, keys=["area_um", "deform"])
ddict["deform"][:20] = .1
ddict["area_um"][:20] = .5
ds = dclab.new_dataset(ddict)
xm, ym, _ = ds.get_kde_contour(yscale="log")
dx = np.diff(xm[0])
dy = np.diff(np.log(ym[:, 0]))
assert np.allclose(dx, dx[0])
assert np.allclose(dy, dy[0])
def test_af_emodulus_legacy_none2():
keys = ["area_um", "deform"]
ddict = example_data_dict(size=8472, keys=keys)
ds = dclab.new_dataset(ddict)
assert "emodulus" not in ds, "not config for emodulus"
ds.config["calculation"] = {"emodulus medium": "CellCarrier",
"emodulus temperature": 23.0,
"emodulus viscosity": 0.5
}
assert "emodulus" not in ds, "emodulus lut should be missing"
def test_kde_nofilt():
ddict = example_data_dict()
ds = dclab.new_dataset(ddict)
ds.config["filtering"]["enable filters"] = False
sc = ds.get_kde_scatter()
cc = ds.get_kde_contour()
assert sc.shape[0] == 100
# This will fail if the default contour accuracy is changed
# in `get_kde_contour`.
assert cc[0].shape == (43, 41)
def test_0error():
keys = ["circ"]
ddict = example_data_dict(size=8472, keys=keys)
ds = dclab.new_dataset(ddict)
try:
ds["unknown_column"]
except KeyError:
pass
else:
raise ValueError("Should have raised KeyError!")
def test_af_0error():
keys = ["circ"]
ddict = example_data_dict(size=8472, keys=keys)
ds = dclab.new_dataset(ddict)
try:
ds["unknown_column"]
except KeyError:
pass
else:
raise ValueError("Should have raised KeyError!")
def test_emodulus_none2():
keys = ["area_um", "deform"]
ddict = example_data_dict(size=8472, keys=keys)
ds = dclab.new_dataset(ddict)
assert "emodulus" not in ds, "not config for emodulus"
ds.config["calculation"] = {"emodulus medium": "CellCarrier",
"emodulus temperature": 23.0,
"emodulus viscosity": 0.5
}
assert "emodulus" not in ds, "emodulus model should be missing"
def test_tsv_not_filtered():
keys = ["area_um", "deform", "time", "frame", "fl3_width"]
ddict = example_data_dict(size=127, keys=keys)
ds = dclab.new_dataset(ddict)
edest = tempfile.mkdtemp()
f1 = join(edest, "test.tsv")
ds.export.tsv(f1, keys, filtered=False)
# cleanup
shutil.rmtree(edest, ignore_errors=True)
def test_min_max_update():
ddict = example_data_dict(size=67, keys=["area_um", "deform"])
ds = new_dataset(ddict)
cfg = ds.config.copy()
# Force updating circularity
cfg["filtering"]["defo min"] = .4
cfg["filtering"]["defo max"] = .8
ds.config.update(cfg)
ds.apply_filter()
def test_limit_above():
keys = ["area_um", "deform", "time", "frame"]
ddict = example_data_dict(size=9999, keys=keys)
ds = dclab.new_dataset(ddict)
assert np.sum(ds._filter) == 9999
filtflt = {"limit events": 10000}
cfg = {"filtering": filtflt}
ds.config.update(cfg)
ds.apply_filter()
assert np.sum(ds._filter) == 9999
def test_kde_log_scatter_invalid():
ddict = example_data_dict(size=300, keys=["area_um", "deform"])
ddict["deform"][:20] = .1
ddict["area_um"][:20] = .5
ddict["deform"][21] = np.nan
ddict["deform"][22] = np.inf
ddict["deform"][23] = -.1
ds = dclab.new_dataset(ddict)
a = ds.get_kde_scatter(yscale="log")
assert np.all(a[:20] == a[0])
assert np.isnan(a[21])
assert np.isnan(a[22])
assert np.isnan(a[23])
def test_kde_general():
# Download and extract data
ddict = example_data_dict()
ds = dclab.new_dataset(ddict)
dcont = []
dscat = []
for kde_type in dclab.kde_methods.methods:
dcont.append(ds.get_kde_contour(kde_type=kde_type))
dscat.append(ds.get_kde_scatter(kde_type=kde_type))
for ii in range(1, len(dcont) - 1):
assert not np.allclose(dcont[ii], dcont[0])
assert not np.allclose(dscat[ii], dscat[0])
def test_avi_no_images():
keys = ["area_um", "deform", "time", "frame", "fl3_width"]
ddict = example_data_dict(size=127, keys=keys)
ds = dclab.new_dataset(ddict)
edest = tempfile.mkdtemp()
f1 = join(edest, "test.avi")
try:
ds.export.avi(f1)
except OSError:
pass
else:
raise ValueError("There should be no image data to write!")
shutil.rmtree(edest, ignore_errors=True)
def test_downsample_up():
"""
Likely causes removal of too many points and requires
re-inserting them.
"""
keys = ["area_um", "deform", "time", "frame", "fl3_width"]
ddict = example_data_dict(size=10000, keys=keys)
ds = dclab.new_dataset(ddict)
assert np.sum(ds._plot_filter) == 10000
ds.apply_filter()
ds.get_downsampled_scatter(downsample=9999)
assert np.sum(ds._plot_filter) == 9999
ds.get_downsampled_scatter(downsample=9999)
def test_filter_min_max():
# make sure min/max values are filtered
ddict = example_data_dict(size=8472, keys=["area_um", "deform"])
ds = new_dataset(ddict)
amin, amax = ds["area_um"].min(), ds["area_um"].max()
ds.config["filtering"]["area_um min"] = (amax + amin) / 2
ds.config["filtering"]["area_um max"] = amax
ds.apply_filter()
assert np.sum(ds.filter.all) == 4256
# make sure data is not filtered before calling ds.apply_filter
dmin, dmax = ds["deform"].min(), ds["deform"].max()
ds.config["filtering"]["deform min"] = (dmin + dmax) / 2
ds.config["filtering"]["deform max"] = dmax
assert np.sum(ds.filter.all) == 4256
def test_fcs_override():
keys = ["area_um", "deform", "time", "frame", "fl3_width"]
ddict = example_data_dict(size=212, keys=keys)
ds = dclab.new_dataset(ddict)
edest = tempfile.mkdtemp()
f1 = join(edest, "test.fcs")
ds.export.fcs(f1, keys, override=True)
try:
ds.export.fcs(f1[:-4], keys, override=False)
except OSError:
pass
else:
raise ValueError("Should append .fcs and not override!")
# cleanup
shutil.rmtree(edest, ignore_errors=True)
def test_inverted_wrong():
dclab.PolygonFilter.clear_all_filters()
ddict = example_data_dict(size=1234, keys=["area_um", "deform"])
# points of polygon filter
points = [[np.min(ddict["area_um"]), np.min(ddict["deform"])],
[np.min(ddict["area_um"]), np.max(ddict["deform"])],
[np.average(ddict["area_um"]), np.max(ddict["deform"])],
[np.average(ddict["area_um"]), np.min(ddict["deform"])],
]
try:
dclab.PolygonFilter(axes=["area_um", "deform"],
points=points,
inverted=0)
except dclab.polygon_filter.PolygonFilterError:
pass
else:
raise ValueError("inverted should only be allowed to be bool")
def test_downsample_nan():
"""Deal with nans"""
keys = ["area_um", "deform", "time", "frame", "fl3_width"]
ddict = example_data_dict(size=8472, keys=keys)
ddict["area_um"][:4700] = np.nan
ds = dclab.new_dataset(ddict)
ds.config["filtering"]["limit events"] = 2000
assert np.sum(ds.filter.all) == 8472
ds.apply_filter()
assert np.sum(ds.filter.all) == 2000
# this is the general test; nans are included
assert np.sum(np.isnan(ds["area_um"][ds.filter.all])) != 0
# this is the exact test (e.g. for the random state used)
assert np.sum(np.isnan(ds["area_um"][ds.filter.all])) == 1150
def test_emodulus_area():
# computes "area_um" from "area_cvx"
keys = ["area_cvx", "deform"]
ddict = example_data_dict(size=8472, keys=keys)
ds = dclab.new_dataset(ddict)
# area can be computed from areapix
ds.config["imaging"]["pixel size"] = .34
assert "area_um" in ds
assert "emodulus" not in ds, "not config for emodulus"
ds.config["setup"]["flow rate"] = 0.16
ds.config["setup"]["channel width"] = 30
ds.config["calculation"] = {"emodulus medium": "CellCarrier",
"emodulus model": "elastic sphere",
"emodulus temperature": 23.0,
"emodulus viscosity": 0.5
}
assert "emodulus" in ds
def test_downsample_log():
keys = ["area_um", "deform", "time", "frame", "fl3_width"]
ddict = example_data_dict(size=8472, keys=keys)
ds = dclab.new_dataset(ddict)
ds.apply_filter()
xlin, ylin = ds.get_downsampled_scatter(downsample=100,
xscale="linear",
yscale="linear")
xlog, ylog = ds.get_downsampled_scatter(downsample=100,
xscale="log",
yscale="log")
assert not np.all(xlin == xlog)
assert not np.all(ylin == ylog)
for x in xlog:
assert x in ds["area_um"]
for y in ylog:
assert y in ds["deform"]
def test_collect_data_tree_unicode():
features = ["area_um", "deform", "time"]
edest = pathlib.Path(tempfile.mkdtemp(prefix="shapeout_test"))
from shapeout.util import safe_path
for ii in range(1, 4):
dat = new_dataset(data=example_data_dict(ii + 10, keys=features))
cfg = {"experiment": {"sample": "test sample",
"run index": ii},
"imaging": {"pixel size": 0.34},
"setup": {"channel width": 20,
"chip region": "channel",
"flow rate": 0.04}
}
dat.config.update(cfg)
dat.export.hdf5(path=safe_path(edest / "únícòdè_{}.rtdc".format(ii)),
features=features)
meta_tool.collect_data_tree([edest])[0]
shutil.rmtree(str(edest), ignore_errors=True)
def test_emodulus():
keys = ["area_um", "deform"]
ddict = example_data_dict(size=8472, keys=keys)
ds = dclab.new_dataset(ddict)
ds.config["setup"]["flow rate"] = 0.16
ds.config["setup"]["channel width"] = 30
ds.config["imaging"]["pixel size"] = .34
ds.config["calculation"] = {"emodulus medium": "CellCarrier",
"emodulus model": "elastic sphere",
"emodulus temperature": 23.0,
"emodulus viscosity": 0.5
}
t1 = time.time()
assert "emodulus" in ds
t2 = time.time()
t3 = time.time()
ds["emodulus"]
t4 = time.time()
assert t4 - t3 > t2 - t1
def test_false_method():
def bad_method(x):
return x + 1
dclab.statistics.Statistics(name="bad",
req_feature=False,
method=bad_method)
ddict = example_data_dict(size=77, keys=["area_um", "deform"])
ds = dclab.new_dataset(ddict)
head1, vals1 = dclab.statistics.get_statistics(ds, features=["deform"])
out = {}
for h, v in zip(head1, vals1):
out[h] = v
assert np.isnan(out["bad"])
# clean up
mth = dclab.statistics.Statistics.available_methods
for k in mth:
if k == "bad":
mth.pop(k)
break