def test_errormodel(large_reflection_table, test_sg):
"""Test the initialisation and methods of the error model."""
Ih_table = IhTable([large_reflection_table], test_sg, nblocks=1)
block = Ih_table.blocked_data_list[0]
params = generated_param()
params.weighting.error_model.basic.n_bins = 2
params.weighting.error_model.basic.min_Ih = 1.0
em = BasicErrorModel
em.min_reflections_required = 1
error_model = em(basic_params=params.weighting.error_model.basic)
error_model.min_reflections_required = 1
error_model.configure_for_refinement(block)
assert error_model.binner.summation_matrix[0, 1] == 1
assert error_model.binner.summation_matrix[1, 1] == 1
assert error_model.binner.summation_matrix[2, 0] == 1
assert error_model.binner.summation_matrix[3, 0] == 1
assert error_model.binner.summation_matrix[4, 0] == 1
assert error_model.binner.summation_matrix.non_zeroes == 5
assert list(error_model.binner.binning_info["refl_per_bin"]) == [3, 2]
# Test calc sigmaprime
x0 = 1.0
x1 = 0.1
sigmaprime = calc_sigmaprime([x0, x1], error_model.filtered_Ih_table)
cal_sigpr = list(
x0 * flex.sqrt(block.variances + flex.pow2(x1 * block.intensities)) /
block.inverse_scale_factors)
assert list(sigmaprime) == pytest.approx(cal_sigpr[4:7] + cal_sigpr[-2:])
# Test calc delta_hl
sigmaprime = calc_sigmaprime([1.0, 0.0],
error_model.filtered_Ih_table) # Reset
# Calculate example for three elements, with intensities 1, 5 and 10 and
# variances 1, 5 and 10 using he formula
# delta_hl = math.sqrt(n_h - 1 / n_h) * (Ihl/ghl - Ih) / sigmaprime
delta_hl = calc_deltahl(
error_model.filtered_Ih_table,
error_model.filtered_Ih_table.calc_nh(),
sigmaprime,
)
expected_deltas = [
(-3.0 / 2.0) * math.sqrt(2.0 / 3.0),
(5.0 / 2.0) * math.sqrt(2.0 / 15.0),
5.0 * math.sqrt(2.0 / 30.0),
-0.117647058824,
0.124783549621,
]
assert list(delta_hl) == pytest.approx(expected_deltas)
def make_error_model_plots(params, experiments):
"""Generate normal probability plot data."""
data = {}
if experiments[0].scaling_model.error_model:
em = experiments[0].scaling_model.error_model
if em.filtered_Ih_table:
table = em.filtered_Ih_table
data["intensity"] = table.intensities
sigmaprime = calc_sigmaprime(em.parameters, table)
data["delta_hl"] = calc_deltahl(table, table.calc_nh(), sigmaprime)
data["inv_scale"] = table.inverse_scale_factors
data["sigma"] = sigmaprime * data["inv_scale"]
data["binning_info"] = em.binner.binning_info
em.clear_Ih_table()
if params.weighting.error_model.basic.minimisation == "regression":
x, y = calculate_regression_x_y(em.filtered_Ih_table)
data["regression_x"] = x
data["regression_y"] = y
data["model_a"] = em.parameters[0]
data["model_b"] = em.parameters[1]
data["summary"] = str(em)
d = {"error_model_plots": {}, "error_model_summary": "No error model applied"}
if "delta_hl" in data:
d["error_model_plots"].update(normal_probability_plot(data))
d["error_model_plots"].update(
i_over_sig_i_vs_i_plot(data["intensity"], data["sigma"])
)
d["error_model_plots"].update(error_model_variance_plot(data))
if "regression_x" in data:
d["error_model_plots"].update(error_regression_plot(data))
if "summary" in data:
d["error_model_summary"] = data["summary"]
return d
def make_error_model_plots(params, experiments):
"""Generate normal probability plot data."""
d = {
"error_model_plots": {},
"error_model_summary": "No error model applied"
}
error_model_data = [] # a list of dicts of error model data
if experiments[0].scaling_model.error_model:
error_models = [e.scaling_model.error_model for e in experiments]
unique_error_models = OrderedSet(error_models)
if len(unique_error_models) == 1:
d["error_model_summary"] = str(error_models[0])
else:
d["error_model_summary"] = ""
for i, e in enumerate(unique_error_models):
indices = [
str(j + 1) for j, x in enumerate(error_models) if e is x
]
d["error_model_summary"] += (
f"\nError model {i+1}, applied to sweeps {', '.join(indices)}:"
+ str(e))
for em in unique_error_models:
if em.filtered_Ih_table:
data_i = {}
table = em.filtered_Ih_table
data_i["intensity"] = table.intensities
sigmaprime = calc_sigmaprime(em.parameters, table)
data_i["delta_hl"] = calc_deltahl(table, table.calc_nh(),
sigmaprime)
data_i["inv_scale"] = table.inverse_scale_factors
data_i["sigma"] = sigmaprime * data_i["inv_scale"]
data_i["binning_info"] = em.binner.binning_info
em.clear_Ih_table()
if params.weighting.error_model.basic.minimisation == "regression":
x, y = calculate_regression_x_y(em.filtered_Ih_table)
data_i["regression_x"] = x
data_i["regression_y"] = y
data_i["model_a"] = em.parameters[0]
data_i["model_b"] = em.parameters[1]
error_model_data.append(data_i)
if error_model_data:
for i, emd in enumerate(error_model_data):
d["error_model_plots"].update(
normal_probability_plot(emd, label=i + 1))
d["error_model_plots"].update(
i_over_sig_i_vs_i_plot(
flumpy.from_numpy(emd["intensity"]),
flumpy.from_numpy(emd["sigma"]),
label=i + 1,
))
d["error_model_plots"].update(
error_model_variance_plot(emd, label=i + 1))
if "regression_x" in emd:
d["error_model_plots"].update(
error_regression_plot(emd, label=i + 1))
return d
def make_output(model, params):
"""Get relevant data from the model and make html report."""
if not (params.output.html or params.output.json):
return
data = {}
table = model.filtered_Ih_table
data["intensity"] = table.intensities
sigmaprime = calc_sigmaprime(model.parameters, table)
data["delta_hl"] = calc_deltahl(table, table.calc_nh(), sigmaprime)
data["inv_scale"] = table.inverse_scale_factors
data["sigma"] = sigmaprime * data["inv_scale"]
data["binning_info"] = model.binner.binning_info
d = {"error_model_plots": {}}
d["error_model_plots"].update(normal_probability_plot(data))
d["error_model_plots"].update(
i_over_sig_i_vs_i_plot(data["intensity"], data["sigma"])
)
d["error_model_plots"].update(error_model_variance_plot(data))
if params.basic.minimisation == "regression":
x, y = calculate_regression_x_y(model.filtered_Ih_table)
data["regression_x"] = x
data["regression_y"] = y
data["model_a"] = model.parameters[0]
data["model_b"] = model.parameters[1]
d["error_model_plots"].update(error_regression_plot(data))
if params.output.html:
logger.info("Writing html report to: %s", params.output.html)
loader = ChoiceLoader(
[
PackageLoader("dials", "templates"),
PackageLoader("dials", "static", encoding="utf-8"),
]
)
env = Environment(loader=loader)
template = env.get_template("simple_report.html")
html = template.render(
page_title="DIALS error model refinement report",
panel_title="Error distribution plots",
panel_id="error_model_plots",
graphs=d["error_model_plots"],
)
with open(params.output.html, "wb") as f:
f.write(html.encode("utf-8", "xmlcharrefreplace"))
if params.output.json:
logger.info("Writing html report data to: %s", params.output.json)
with open(params.output.json, "w") as outfile:
json.dump(d, outfile)
def update(self, scaler):
if scaler.error_model:
if scaler.error_model.filtered_Ih_table:
table = scaler.error_model.filtered_Ih_table
self.data["intensity"] = table.intensities
sigmaprime = calc_sigmaprime(scaler.error_model.parameters, table)
self.data["delta_hl"] = calc_deltahl(table, table.calc_nh(), sigmaprime)
self.data["inv_scale"] = table.inverse_scale_factors
self.data["sigma"] = sigmaprime * self.data["inv_scale"]
self.data["binning_info"] = scaler.error_model.binner.binning_info
scaler.error_model.clear_Ih_table()
if scaler.params.weighting.error_model.basic.minimisation == "regression":
x, y = calculate_regression_x_y(scaler.error_model.filtered_Ih_table)
self.data["regression_x"] = x
self.data["regression_y"] = y
self.data["model_a"] = scaler.error_model.parameters[0]
self.data["model_b"] = scaler.error_model.parameters[1]
self.data["summary"] = str(scaler.error_model)