def test_expr_model():
"""Make sure that a model with an expression is fitted correctly"""
# Reference fit
rgrp = IndentationGroup(jpkfile)
ridnt = rgrp[0]
ridnt.apply_preprocessing(["compute_tip_position",
"correct_force_offset"])
ridnt.fit_model(model_key="hertz_para",
params_initial=None,
x_axis="tip position",
y_axis="force",
weight_cp=False,
segment="retract")
rparms = ridnt.fit_properties["params_fitted"]
remod = rparms["E"].value
with MockModelExpr() as mod:
grp = IndentationGroup(jpkfile)
idnt = grp[0]
idnt.apply_preprocessing(["compute_tip_position",
"correct_force_offset"])
idnt.fit_model(model_key=mod.model_key,
params_initial=None,
x_axis="tip position",
y_axis="force",
weight_cp=False,
segment="retract")
parms = idnt.fit_properties["params_fitted"]
# make sure the expression survives
assert parms["E1"].expr == "virtual_parameter+E"
emod = parms["E1"].value
# There are some difference due to heuristics
assert np.allclose(emod, remod, atol=0, rtol=3e-3)
def test_unknown_method():
idnt = IndentationGroup(datadir / "spot3-0192.jpk-force")[0]
try:
idnt.apply_preprocessing(["compute_tip_position", "unknown_method"])
except KeyError:
pass
else:
assert False, "Preprocessing with an unknown method must not work."
def test_poc_estimation_via_indent(method, contact_point):
fd = IndentationGroup(data_path / "fmt-jpk-fd_spot3-0192.jpk-force")[0]
fd.apply_preprocessing(
["compute_tip_position", "correct_force_offset", "correct_tip_offset"],
options={"correct_tip_offset": {
"method": method
}})
assert np.argmin(np.abs(fd["tip position"])) == contact_point
def setuph5(ret_idnt=False, path=jpkfile):
tdir = tempfile.mkdtemp(prefix="test_nanite_rate_io_")
tdir = pathlib.Path(tdir)
h5path = tdir / "simple.h5"
grp = IndentationGroup(path)
for idnt in grp:
idnt.apply_preprocessing(["compute_tip_position"])
inparams = model.model_hertz_paraboloidal.get_parameter_defaults()
inparams["baseline"].vary = True
inparams["contact_point"].set(1.8e-5)
# Fit with absolute full range
idnt.fit_model(model_key="hertz_para",
params_initial=inparams,
range_x=(0, 0),
range_type="absolute",
x_axis="tip position",
y_axis="force",
segment="approach",
weight_cp=False)
save_hdf5(h5path=h5path,
indent=idnt,
user_rate=5,
user_name="hans",
user_comment="this is a comment",
h5mode="a")
if ret_idnt:
return tdir, h5path, idnt
else:
return tdir, h5path
def test_gcf_k_no_change_in_fitted_curve(gcf_k):
"""Fit result for contact point does not change with gcf_k"""
ds1 = IndentationGroup(jpkfile)
apret = ds1[0]
apret.apply_preprocessing(["compute_tip_position"])
inparams = nanite.model.model_hertz_paraboloidal.get_parameter_defaults()
inparams["baseline"].vary = True
inparams["contact_point"].set(1.8321e-5)
# sane initial fit parameters with gcf_k in mind
inparams["E"].set(inparams["E"].value * (1/gcf_k)**(3/2))
# Fit with absolute full range
kwargs = dict(model_key="hertz_para",
params_initial=inparams,
range_x=(0, 0),
range_type="absolute",
x_axis="tip position",
y_axis="force",
segment="approach",
weight_cp=False,
gcf_k=gcf_k)
apret.fit_model(**kwargs)
assert np.allclose(np.nanmax(apret["fit"]),
3.496319691452543e-09,
atol=0.000001e-9,
rtol=0)
assert np.allclose(np.nanmax(apret["fit residuals"]),
2.233069676202635e-10,
atol=0.000001e-10,
rtol=0)
def test_hash_time():
ds1 = IndentationGroup(jpkfile)
apret = ds1[0]
apret.apply_preprocessing(["compute_tip_position"])
inparams = nanite.model.model_hertz_paraboloidal.get_parameter_defaults()
inparams["baseline"].vary = True
inparams["contact_point"].set(1.8321e-5)
# Fit with absolute full range
kwargs = dict(model_key="hertz_para",
params_initial=inparams,
range_x=(0, 0),
range_type="absolute",
x_axis="tip position",
y_axis="force",
segment="approach",
weight_cp=False)
t0 = time.perf_counter()
apret.fit_model(**kwargs)
t1 = time.perf_counter()
apret.fit_model()
t2 = time.perf_counter()
kwargs["weight_cp"] = 1e-5
apret.fit_model(**kwargs)
t3 = time.perf_counter()
apret.fit_model()
t4 = time.perf_counter()
assert t1-t0 >= 100 * \
(t2-t1), "Consecutive fits with same parameters should be instant"
assert t3-t2 >= 100 * \
(t2-t1), "Changing parameters again should cause a new fit"
assert t3 - t2 >= 100 * (t4 -
t3), "And computing the same should be faster"
def test_gcf_k_scaling_of_youngs_modulus(gcf_k):
"""Fit result for Young's modulus should scale with gcf_k"""
ds1 = IndentationGroup(jpkfile)
apret = ds1[0]
apret.apply_preprocessing(["compute_tip_position", "correct_tip_offset"])
inparams = nanite.model.model_hertz_paraboloidal.get_parameter_defaults()
inparams["baseline"].vary = True
inparams["contact_point"].set(0)
# Fit with absolute full range
kwargs = dict(model_key="hertz_para",
params_initial=inparams,
range_x=(0, 0),
range_type="absolute",
x_axis="tip position",
y_axis="force",
segment="approach",
weight_cp=False,
gcf_k=gcf_k)
apret.fit_model(**kwargs)
params1 = apret.fit_properties["params_fitted"]
# proportionality between E and gcf_k
corrval = 14741.950622347102 * (1/gcf_k)**(3/2)
assert np.allclose(params1["E"].value,
corrval,
rtol=0.0001,
atol=0,
)
def test_gcf_k_no_change_in_contact_point(gcf_k):
"""Fit result for contact point does not change with gcf_k"""
ds1 = IndentationGroup(jpkfile)
apret = ds1[0]
apret.apply_preprocessing(["compute_tip_position"])
inparams = nanite.model.model_hertz_paraboloidal.get_parameter_defaults()
inparams["baseline"].vary = True
inparams["contact_point"].set(1.8321e-5)
# sane initial fit parameters with gcf_k in mind
inparams["E"].set(inparams["E"].value * (1/gcf_k)**(3/2))
# Fit with absolute full range
kwargs = dict(model_key="hertz_para",
params_initial=inparams,
range_x=(0, 0),
range_type="absolute",
x_axis="tip position",
y_axis="force",
segment="approach",
weight_cp=False,
gcf_k=gcf_k)
apret.fit_model(**kwargs)
params1 = apret.fit_properties["params_fitted"]
assert np.allclose(params1["contact_point"].value,
1.802931023582261e-05,
rtol=0,
atol=0.000000000005,
)
def test_app_ret():
grp = IndentationGroup(jpkfile)
idnt = grp[0]
idnt.apply_preprocessing(["compute_tip_position", "correct_force_offset"])
idp = idnt.estimate_contact_point_index()
aprid = ~idnt["segment"]
x = idnt["tip position"][aprid]
y = idnt["force"][aprid]
contact_point = x[idp]
# crop x and y around contact_point
distcp = x.shape[0] - idp
x = x[-3 * distcp:]
y = y[-3 * distcp:]
params = lmfit.Parameters()
params.add("contact_point", value=contact_point)
params.add("baseline", value=0, vary=False)
params.add("E", value=300e3, min=0)
params.add("nu", value=.5, vary=False)
params.add("R", value=40e-9, vary=False)
fit_w = lmfit.minimize(hertz.residual, params, args=(x, y, True))
fit_n = lmfit.minimize(hertz.residual, params, args=(x, y, False))
# Correctly reproduces fit results in the JPK analysis software
# with "Vertical Tip Position", "Switchable Baseline Operation",
# and a parabolic indenter with the "Hertz/Sneddon" "Model type".
# Set tip radius to 40nm.
E_jpk = 233.1e3
cp_jpk = 18.03e-6
assert np.allclose(fit_n.params["E"].value, E_jpk, rtol=2e-3, atol=0)
assert np.allclose(fit_n.params["contact_point"].value,
cp_jpk,
rtol=4e-5,
atol=0)
if __name__ == "__main__" and False:
import matplotlib.pylab as plt
_fig, axes = plt.subplots(2, 1)
xf = np.linspace(x[0], x[-1], 100)
axes[0].plot(xf,
hertz.model(fit_w.params, xf),
label="fit with weights")
axes[0].plot(xf, hertz.model(fit_n.params, xf), label="fit no weights")
axes[0].plot(x, y, label="data")
axes[0].legend()
axes[0].grid()
axes[1].plot(x,
hertz.residual(fit_w.params, x, y, weight_cp=True),
label="residuals with weight")
axes[1].plot(x,
hertz.residual(fit_n.params, x, y, weight_cp=False),
label="residuals no weight")
axes[1].legend()
axes[1].grid()
plt.show()
def test_correct_force_offset():
grp = IndentationGroup(jpkfile)
idnt = grp[0]
idnt.apply_preprocessing(["compute_tip_position",
"correct_force_offset"])
idp = idnt.estimate_contact_point_index()
assert np.allclose(np.average(idnt["force"][:idp]), 0)
def test_cache_emodulus():
# Check that the fitting procedure does not perform unneccessary
# double fits and uses the cached variables for emoduli and
# minimal indentations.
ds = IndentationGroup(jpkfile)
ar = ds[0]
ar.apply_preprocessing(
["compute_tip_position", "correct_force_offset", "correct_tip_offset"])
t01 = time.perf_counter()
ar.fit_model(
model_key="hertz_cone",
params_initial=None,
x_axis="tip position",
y_axis="force",
weight_cp=False,
segment="approach",
optimal_fit_edelta=True,
)
t02 = time.perf_counter()
t11 = time.perf_counter()
ar.fit_model()
t12 = time.perf_counter()
assert (t02-t01)*1e-4 > t12 - \
t11, "Second computation should yield cached value (faster)!"
# Make sure that changing the fit model is longer again
t21 = time.perf_counter()
ar.fit_model(model_key="hertz_para")
t22 = time.perf_counter()
assert ((t22 - t21) * 1e-4 >
(t12 - t11)), "Changing model_key should slow down computation!"
def test_correct_app_ret():
grp = IndentationGroup(jpkfile)
idnt = grp[0]
idnt.apply_preprocessing(["compute_tip_position",
"correct_split_approach_retract"])
a = idnt["segment"][idnt["segment"] == 0]
assert len(a) == 2006
def test_base():
ds1 = IndentationGroup(jpkfile)
ds2 = IndentationGroup(jpkfile)
ds3 = ds1 + ds2
assert len(ds3) == 2
assert len(ds2) == 1
assert len(ds1) == 1
ds2 += ds3
assert len(ds3) == 2
assert len(ds2) == 3
for apret in ds3:
assert isinstance(apret, Indentation)
# test repr
print(ds3)
def test_tip_sample_separation():
grp = IndentationGroup(jpkfile)
idnt = grp[0]
# This computation correctly reproduces the column
# "Vertical Tip Position" as it is exported by the
# JPK analysis software with the checked option
# "Use Unsmoothed Height".
idnt.apply_preprocessing(["compute_tip_position"])
tip = np.array(idnt["tip position"])
assert tip[0] == 2.2803841798545836e-05
def test_poc_details_fit_line_polynomial():
fd = IndentationGroup(
data_path /
"fmt-jpk-fd_single_tilted-baseline-mitotic_2021-01-29.jpk-force")[0]
details = fd.apply_preprocessing(
["compute_tip_position", "correct_tip_offset"],
options={"correct_tip_offset": {
"method": "fit_line_polynomial"
}},
ret_details=True)
pocd = details["correct_tip_offset"]
for key in ["plot force", "plot fit", "plot poc"]:
assert key in pocd
assert np.allclose(pocd["plot poc"][0][0], 15602, atol=0)
assert np.allclose(
fd["force"][15602],
1.7313584441928315e-09,
atol=0,
)
def test_app_ret():
grp = IndentationGroup(jpkfile)
idnt = grp[0]
idnt.apply_preprocessing(["smooth_height"])
hms = np.array(idnt.data["height (measured, smoothed)"])
idnt.apply_preprocessing(["compute_tip_position",
"smooth_height"])
hms2 = np.array(idnt.data["height (measured, smoothed)"])
assert np.all(hms == hms2)
np.array(idnt.data["tip position (smoothed)"])
def test_poc_details_deviation_from_baseline():
fd = IndentationGroup(data_path / "fmt-jpk-fd_spot3-0192.jpk-force")[0]
details = fd.apply_preprocessing(
["compute_tip_position", "correct_force_offset", "correct_tip_offset"],
options={"correct_tip_offset": {
"method": "deviation_from_baseline"
}},
ret_details=True)
pocd = details["correct_tip_offset"]
for key in [
"plot force", "plot baseline mean", "plot baseline threshold",
"plot poc"
]:
assert key in pocd
assert np.allclose(np.mean(pocd["plot baseline threshold"][1]),
8.431890003513514e-11,
atol=0)
assert np.allclose(np.mean(pocd["plot baseline mean"][1]),
-7.573822405258677e-12,
atol=0)
def test_process_bad():
for bf in bad_files:
ds = IndentationGroup(bf)
for idnt in ds:
# Walk through the standard analysis pipeline without
# throwing any exceptions.
idnt.apply_preprocessing([
"compute_tip_position", "correct_force_offset",
"correct_tip_offset", "correct_split_approach_retract"
])
idnt.fit_model()
def test_changed_fit_properties():
ar = IndentationGroup(jpkfile)[0]
# Initially, fit properties are not set
assert not ar.fit_properties
assert isinstance(ar.fit_properties, dict)
# Prepprocessing
ar.apply_preprocessing(
["compute_tip_position", "correct_tip_offset", "correct_force_offset"])
ar.fit_model()
hash1 = ar.fit_properties["hash"]
pinit = copy.deepcopy(ar.fit_properties["params_initial"])
pinit["E"].vary = False
assert "hash" in ar.fit_properties, "make sure we didn't change anything"
ar.fit_properties["params_initial"] = pinit
assert "hash" not in ar.fit_properties
ar.fit_model()
assert hash1 != ar.fit_properties["hash"]
def test_basic_nofit():
idnt = IndentationGroup(jpkfile)[0]
feat = features.IndentationFeatures(idnt)
assert not feat.is_fitted
assert not feat.is_valid
assert not feat.has_contact_point
try:
feat.contact_point
except ValueError:
pass
else:
assert False, "without fit - should not have a CP"
def test_fit_data_error():
# a FitDataError is raised when it is not possible to compute an
# E(delta) curve:
ds = IndentationGroup(badjpk)
ar = ds[0]
ar.apply_preprocessing(
["compute_tip_position", "correct_force_offset", "correct_tip_offset"])
try:
ar.compute_emodulus_mindelta()
except FitDataError:
pass
else:
assert False, "Invalid input data should not allow E(delt) computation"
def test_expr_model_sign():
"""Fit with negative limit"""
# Reference fit
rgrp = IndentationGroup(jpkfile)
ridnt = rgrp[0]
ridnt.apply_preprocessing(["compute_tip_position",
"correct_force_offset"])
ridnt.fit_model(model_key="hertz_para",
params_initial=None,
x_axis="tip position",
y_axis="force",
weight_cp=False,
segment="retract")
rparms = ridnt.fit_properties["params_fitted"]
rparmsi = ridnt.fit_properties["params_initial"]
remod = rparms["E"].value
with MockModelExpr() as mod:
grp = IndentationGroup(jpkfile)
idnt = grp[0]
idnt.apply_preprocessing(["compute_tip_position",
"correct_force_offset"])
params_initial = mod.get_parameter_defaults()
params_initial["E"].set(value=20000)
params_initial["virtual_parameter"].set(value=-1, min=-np.inf, max=0)
params_initial["contact_point"].set(
value=rparmsi["contact_point"].value)
idnt.fit_model(model_key=mod.model_key,
params_initial=params_initial,
x_axis="tip position",
y_axis="force",
weight_cp=False,
segment="retract")
parms = idnt.fit_properties["params_fitted"]
# make sure the expression survives
assert parms["E1"].expr == "virtual_parameter+E"
emod = parms["E1"].value
# There are some difference due to heuristics
assert np.allclose(emod, remod, atol=0, rtol=3e-3)
def test_app_ret():
grp = IndentationGroup(jpkfile)
idnt = grp[0]
idnt.apply_preprocessing(["compute_tip_position"])
height = np.array(idnt["height (measured)"], copy=True)
tip_position = np.array(idnt["tip position"], copy=True)
idnt.apply_preprocessing(["smooth_height"])
hms = np.array(idnt["height (measured)"])
assert not np.all(height == hms)
idnt.apply_preprocessing(["compute_tip_position",
"smooth_height"])
hms2 = np.array(idnt["height (measured)"])
assert np.all(hms == hms2)
assert not np.all(idnt["tip position"] == tip_position)
def test_app_ret():
ds = IndentationGroup(jpkfile)
ar = ds[0]
ar.apply_preprocessing(["compute_tip_position", "correct_force_offset"])
idp = ar.estimate_contact_point_index()
aprid = ar["segment"] == 0
x = ar["tip position"][aprid]
y = ar["force"][aprid]
contact_point = x[idp]
# crop x and y around contact_point
distcp = x.shape[0] - idp
x = x[-3 * distcp:]
y = y[-3 * distcp:]
params = lmfit.Parameters()
params.add("contact_point", value=contact_point)
params.add("baseline", value=0, vary=False)
params.add("E", value=300e3, min=0)
params.add("nu", value=.5, vary=False)
params.add("R", value=40e-9, vary=False)
fit_w = lmfit.minimize(mod_ssa.residual, params, args=(x, y, True))
fit_n = lmfit.minimize(mod_ssa.residual, params, args=(x, y, False))
if __name__ == "__main__" and False:
import matplotlib.pylab as plt
_fig, axes = plt.subplots(2, 1)
xf = np.linspace(x[0], x[-1], 100)
axes[0].plot(xf,
mod_ssa.model(fit_w.params, xf),
label="fit with weights")
axes[0].plot(xf,
mod_ssa.model(fit_n.params, xf),
label="fit no weights")
axes[0].plot(x, y, label="data")
axes[0].legend()
axes[0].grid()
axes[1].plot(x,
mod_ssa.residual(fit_w.params, x, y, weight_cp=True),
label="residuals with weight")
axes[1].plot(x,
mod_ssa.residual(fit_n.params, x, y, weight_cp=False),
label="residuals no weight")
axes[1].legend()
axes[1].grid()
plt.show()
def test_expr_model_limit():
"""Fit with a limit towards the correct solution"""
# Reference fit
rgrp = IndentationGroup(jpkfile)
ridnt = rgrp[0]
ridnt.apply_preprocessing(["compute_tip_position",
"correct_force_offset"])
ridnt.fit_model(model_key="hertz_para",
params_initial=None,
x_axis="tip position",
y_axis="force",
weight_cp=False,
segment="retract")
rparmsi = ridnt.fit_properties["params_initial"]
with MockModelExpr() as mod:
grp = IndentationGroup(jpkfile)
idnt = grp[0]
idnt.apply_preprocessing(["compute_tip_position",
"correct_force_offset"])
params_initial = mod.get_parameter_defaults()
params_initial["E"].set(value=19000)
params_initial["virtual_parameter"].set(value=10, min=0, max=200)
params_initial["contact_point"].set(
value=rparmsi["contact_point"].value)
idnt.fit_model(model_key=mod.model_key,
params_initial=params_initial,
x_axis="tip position",
y_axis="force",
weight_cp=False,
segment="retract")
parms = idnt.fit_properties["params_fitted"]
# make sure the expression survives
assert parms["E1"].expr == "virtual_parameter+E"
emod = parms["E1"].value
# It should have gone to the boundary
assert np.allclose(emod, 19200)
def test_change_model_key():
ar = IndentationGroup(jpkfile)[0]
# Prepprocessing
ar.apply_preprocessing(
["compute_tip_position", "correct_tip_offset", "correct_force_offset"])
# Fitting the data set will populate the dict
ar.fit_model(model_key="hertz_para")
# Change the model
assert ar.fit_properties["params_initial"] is not None
ar.fit_properties["model_key"] = "hertz_cone"
# Changing the model key should reset the initial parameters
assert ar.fit_properties["params_initial"] is None
def test_emodulus_search():
ds = IndentationGroup(jpkfile)
ar = ds[0]
ar.apply_preprocessing(
["compute_tip_position", "correct_force_offset", "correct_tip_offset"])
ar.fit_model(
model_key="hertz_cone",
params_initial=None,
x_axis="tip position",
y_axis="force",
weight_cp=False,
segment="approach",
optimal_fit_edelta=True,
)
assert "optimal_fit_delta_array" in ar.fit_properties
assert "optimal_fit_E_array" in ar.fit_properties
assert "optimal_fit_delta" in ar.fit_properties
# This assertion might fail when the preprocessing changes
# or when the search algorithm for the optimal fit changes.
dopt = -2.07633035802137e-07
assert np.allclose(ar.fit_properties["optimal_fit_delta"], dopt)
if __name__ == "__main__":
import matplotlib.pylab as plt
_fig, axes = plt.subplots(2, 1)
axes[0].plot(ar["tip position"] * 1e6, ar["force"] * 1e9, label="data")
axes[0].plot(ar["tip position"] * 1e6, ar["fit"] * 1e9, label="fit")
axes[0].legend()
axes[0].grid()
axes[0].set_xlabel("indentation [µm]")
axes[0].set_ylabel("force [nN]")
deltas = ar.fit_properties["optimal_fit_delta_array"] * 1e6
emod = ar.fit_properties["optimal_fit_E_array"]
axes[1].plot(deltas, emod, label="emodulus/minimal-indentation-curve")
axes[1].set_xlabel("minimal indentation [µm]")
axes[1].set_ylabel("emodulus [Pa]")
axes[1].vlines(ar.fit_properties["optimal_fit_delta"] * 1e6,
emod.min(),
emod.max(),
label="optimal minimal indentation at plateau",
color="r")
axes[1].grid()
axes[1].legend()
plt.tight_layout()
plt.show()
def test_correct_split_approach_retract():
fd = IndentationGroup(data_path / "fmt-jpk-fd_spot3-0192.jpk-force")[0]
fd.apply_preprocessing(
["compute_tip_position", "correct_force_offset", "correct_tip_offset"])
assert fd.appr["segment"].size == 2000
fd.apply_preprocessing([
"compute_tip_position", "correct_force_offset", "correct_tip_offset",
"correct_split_approach_retract"
])
assert fd.appr["segment"].size == 2006
def test_wrong_params_initial():
ar = IndentationGroup(jpkfile)[0]
# Prepprocessing
ar.apply_preprocessing(
["compute_tip_position", "correct_tip_offset", "correct_force_offset"])
md = model.models_available["hertz_para"]
params = md.get_parameter_defaults()
ar.fit_properties["model_key"] = "hertz_cone"
try:
ar.fit_model(params_initial=params)
except FitKeyError:
# We forced the wrong fitting parameters.
pass
else:
raise ValueError("Should not be able to use wrong fit parameters!")
def test_user_training_set():
tdir = setup_training_set()
# load a curve
idnt = IndentationGroup(jpkfile)[0]
# fit it
idnt.fit_model(
model_key="sneddon_spher_approx",
preprocessing=["compute_tip_position", "correct_force_offset"])
r1 = idnt.rate_quality(regressor="Extra Trees", training_set="zef18")
assert r1 > 9, "sanity check"
r2 = idnt.rate_quality(regressor="Extra Trees", training_set=tdir)
assert 4 < r2 < 5, "with the given random state we end up at 4.55"
