def calculate_sorted_deviations(self, parameters):
"""Sort the x,y data."""
sigmaprime = calc_sigmaprime(parameters, self.filtered_Ih_table)
delta_hl = calc_deltahl(self.filtered_Ih_table,
self.filtered_Ih_table.calc_nh(), sigmaprime)
central_cutoff = 1.5
n = delta_hl.size
self.sortedy = np.sort(delta_hl)
v1 = norm.cdf(-central_cutoff)
v2 = norm.cdf(central_cutoff)
idx_cutoff_min = ceil(
(v1 * n) - 0.5) # first one within the central cutoff range
idx_cutoff_max = ceil(
(v2 * n) - 0.5) # first one above the central cutoff range
central_n = idx_cutoff_max - idx_cutoff_min
v = np.linspace(
start=(idx_cutoff_min + 0.5) / n,
stop=(idx_cutoff_max + 0.5) / n,
endpoint=False,
num=central_n,
)
self.sortedx = flumpy.from_numpy(norm.ppf(v))
self.sortedy = flumpy.from_numpy(
self.sortedy[idx_cutoff_min:idx_cutoff_max])
def calculate_gradients(self, apm):
"calculate the gradient vector"
a = self.error_model.components["a"].parameters[0]
b = apm.x[0]
Ih_table = self.error_model.binner.Ih_table
I_hl = Ih_table.intensities
g_hl = Ih_table.inverse_scale_factors
weights = self.error_model.binner.weights
bin_vars = self.error_model.binner.bin_variances
sum_matrix = self.error_model.binner.summation_matrix
bin_counts = self.error_model.binner.binning_info["refl_per_bin"]
dsig_dc = (b * np.square(I_hl) * (a**2) /
(self.error_model.binner.sigmaprime * np.square(g_hl)))
ddelta_dsigma = (-1.0 * self.error_model.binner.delta_hl /
self.error_model.binner.sigmaprime)
deriv = ddelta_dsigma * dsig_dc
dphi_by_dvar = -2.0 * (np.full(bin_vars.size, 0.5) - bin_vars +
(1.0 / (2.0 * np.square(bin_vars))))
term1 = flumpy.to_numpy(
flumpy.from_numpy(2.0 * self.error_model.binner.delta_hl * deriv) *
sum_matrix)
term2a = flumpy.to_numpy(
flumpy.from_numpy(self.error_model.binner.delta_hl) * sum_matrix)
term2b = flumpy.to_numpy(flumpy.from_numpy(deriv) * sum_matrix)
grad = dphi_by_dvar * ((term1 / bin_counts) -
(2.0 * term2a * term2b / np.square(bin_counts)))
gradients = flex.double([np.sum(grad * weights) / np.sum(weights)])
return gradients
def rmsd(self):
"""
The RMSD in pixels
"""
mse = np.array([0.0, 0.0], dtype=np.float64)
for i in range(len(self.data)):
R = self.data[i].R
mbar = self.data[i].conditional.mean()
xobs = self.data[i].mobs
norm_s0 = self.data[i].norm_s0
s1 = np.matmul(
R.T,
np.array([mbar[0, 0], mbar[1, 0], norm_s0],
dtype=np.float64).reshape(3, 1),
)
s3 = np.matmul(
R.T,
np.array([xobs[0, 0], xobs[1, 0], norm_s0],
dtype=np.float64).reshape(3, 1),
)
s1 = matrix.col(flumpy.from_numpy(s1[:, 0]))
s3 = matrix.col(flumpy.from_numpy(s3[:, 0]))
xyzcal = self.model.experiment.detector[0].get_ray_intersection_px(
s1)
xyzobs = self.model.experiment.detector[0].get_ray_intersection_px(
s3)
r_x = xyzcal[0] - xyzobs[0]
r_y = xyzcal[1] - xyzobs[1]
mse += np.array([r_x**2, r_y**2])
mse /= len(self.data)
return np.sqrt(mse)
def compute_residuals_and_gradients(cls, Ih_table):
"""Return the residuals array, jacobian matrix and weights."""
residuals = cls.calculate_residuals(Ih_table)
jacobian = cls.calculate_jacobian(Ih_table)
weights = Ih_table.weights
Ih_table.derivatives = None
return flumpy.from_numpy(residuals), jacobian, flumpy.from_numpy(
weights)
def make_error_model_plots(params, experiments):
"""Generate normal probability plot data."""
d = {
"error_model_plots": {},
"error_model_summary": "No error model applied"
}
error_model_data = [] # a list of dicts of error model data
if experiments[0].scaling_model.error_model:
error_models = [e.scaling_model.error_model for e in experiments]
unique_error_models = OrderedSet(error_models)
if len(unique_error_models) == 1:
d["error_model_summary"] = str(error_models[0])
else:
d["error_model_summary"] = ""
for i, e in enumerate(unique_error_models):
indices = [
str(j + 1) for j, x in enumerate(error_models) if e is x
]
d["error_model_summary"] += (
f"\nError model {i+1}, applied to sweeps {', '.join(indices)}:"
+ str(e))
for em in unique_error_models:
if em.filtered_Ih_table:
data_i = {}
table = em.filtered_Ih_table
data_i["intensity"] = table.intensities
sigmaprime = calc_sigmaprime(em.parameters, table)
data_i["delta_hl"] = calc_deltahl(table, table.calc_nh(),
sigmaprime)
data_i["inv_scale"] = table.inverse_scale_factors
data_i["sigma"] = sigmaprime * data_i["inv_scale"]
data_i["binning_info"] = em.binner.binning_info
em.clear_Ih_table()
if params.weighting.error_model.basic.minimisation == "regression":
x, y = calculate_regression_x_y(em.filtered_Ih_table)
data_i["regression_x"] = x
data_i["regression_y"] = y
data_i["model_a"] = em.parameters[0]
data_i["model_b"] = em.parameters[1]
error_model_data.append(data_i)
if error_model_data:
for i, emd in enumerate(error_model_data):
d["error_model_plots"].update(
normal_probability_plot(emd, label=i + 1))
d["error_model_plots"].update(
i_over_sig_i_vs_i_plot(
flumpy.from_numpy(emd["intensity"]),
flumpy.from_numpy(emd["sigma"]),
label=i + 1,
))
d["error_model_plots"].update(
error_model_variance_plot(emd, label=i + 1))
if "regression_x" in emd:
d["error_model_plots"].update(
error_regression_plot(emd, label=i + 1))
return d
def test_flex_loop_nesting():
fo = flex.int(10)
npo = flumpy.to_numpy(fo)
assert fo is flumpy.from_numpy(npo)
# Now try vec
fo = flex.complex_double(5)
npo = flumpy.to_numpy(fo)
assert flumpy.from_numpy(npo) is fo
def calculate_bin_variances(self) -> np.array:
"""Calculate the variance of each bin."""
sum_deltasq = flumpy.to_numpy(
flumpy.from_numpy(np.square(self.delta_hl)) *
self.summation_matrix)
sum_delta_sq = np.square(
flumpy.to_numpy(
flumpy.from_numpy(self.delta_hl) * self.summation_matrix))
bin_vars = (sum_deltasq / self.binning_info["refl_per_bin"]) - (
sum_delta_sq / np.square(self.binning_info["refl_per_bin"]))
self.binning_info["bin_variances"] = bin_vars
return bin_vars
def make_output(model, params):
"""Get relevant data from the model and make html report."""
if not (params.output.html or params.output.json):
return
data = {}
table = model.filtered_Ih_table
data["intensity"] = flumpy.from_numpy(table.intensities)
sigmaprime = calc_sigmaprime(model.parameters, table)
data["delta_hl"] = calc_deltahl(table, table.calc_nh(), sigmaprime)
data["inv_scale"] = table.inverse_scale_factors
data["sigma"] = flumpy.from_numpy(sigmaprime * table.inverse_scale_factors)
data["binning_info"] = model.binner.binning_info
d = {"error_model_plots": {}}
d["error_model_plots"].update(normal_probability_plot(data))
d["error_model_plots"].update(
i_over_sig_i_vs_i_plot(data["intensity"], data["sigma"]))
d["error_model_plots"].update(error_model_variance_plot(data))
if params.basic.minimisation == "regression":
x, y = calculate_regression_x_y(model.filtered_Ih_table)
data["regression_x"] = x
data["regression_y"] = y
data["model_a"] = model.parameters[0]
data["model_b"] = model.parameters[1]
d["error_model_plots"].update(error_regression_plot(data))
if params.output.html:
logger.info("Writing html report to: %s", params.output.html)
loader = ChoiceLoader([
PackageLoader("dials", "templates"),
PackageLoader("dials", "static", encoding="utf-8"),
])
env = Environment(loader=loader)
template = env.get_template("simple_report.html")
html = template.render(
page_title="DIALS error model refinement report",
panel_title="Error distribution plots",
panel_id="error_model_plots",
graphs=d["error_model_plots"],
)
with open(params.output.html, "wb") as f:
f.write(html.encode("utf-8", "xmlcharrefreplace"))
if params.output.json:
logger.info("Writing html report data to: %s", params.output.json)
with open(params.output.json, "w") as outfile:
json.dump(d, outfile)
def test_Simple1ProfileModel_predict_reflections(
simple1_profile_model,
test_experiment,
):
# Create the index generator
index_generator = IndexGenerator(
test_experiment.crystal.get_unit_cell(),
test_experiment.crystal.get_space_group().type(),
2.0,
)
# Get an array of miller indices
miller_indices = index_generator.to_array()
reflections = simple1_profile_model.predict_reflections([test_experiment],
miller_indices,
probability=0.9973)
s0 = matrix.col(test_experiment.beam.get_s0())
quantile = chisq_quantile(3, 0.9973)
sigma_inv = matrix.sqr(flumpy.from_numpy(
simple1_profile_model.sigma())).inverse()
for s2 in reflections["s2"]:
s2_ = matrix.col(s2)
x = s2_.normalize() * s0.length() - s2_
d = (x.transpose() * sigma_inv * x)[0]
assert d < quantile
def mosaicity(self) -> Dict:
"""One value for mosaicity for Simple1"""
decomp = linalg.eigensystem.real_symmetric(
matrix.sqr(flumpy.from_numpy(
self.sigma())).as_flex_double_matrix())
v = list(mosaicity_from_eigen_decomposition(decomp.values()))
return {"spherical": v[0]}
def test_reverse_bool():
sums = np.sum(np.indices((5, 5, 5, 5, 5)), axis=0)
npo = np.logical_or(sums % 3 == 3, sums % 5 == 0)
fo = flumpy.from_numpy(npo)
for idx in itertools.product(*[range(5)] * 5):
assert fo[idx] == npo[idx]
assert fo.count(True) == npo.sum()
def test_Simple6ProfileModel_compute_mask(simple6_profile_model,
test_experiment):
experiments = [test_experiment]
# Create the index generator
index_generator = IndexGenerator(
experiments[0].crystal.get_unit_cell(),
experiments[0].crystal.get_space_group().type(),
2.0,
)
# Get an array of miller indices
miller_indices = index_generator.to_array()
reflections = simple6_profile_model.predict_reflections(experiments,
miller_indices,
probability=0.9973)
s2 = reflections["s2"]
s0 = matrix.col(experiments[0].beam.get_s0())
quantile = chisq_quantile(3, 0.9973)
sigma_inv = matrix.sqr(flumpy.from_numpy(
simple6_profile_model.sigma())).inverse()
for s2 in map(matrix.col, reflections["s2"]):
x = s2.normalize() * s0.length() - s2
d = (x.transpose() * sigma_inv * x)[0]
assert d < quantile
simple6_profile_model.compute_bbox(experiments, reflections)
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"],
allocate=True)
simple6_profile_model.compute_mask(experiments, reflections)
def test_reverse_complex():
npo = np.array([3j, 4j, 3 + 5j])
fo = flumpy.from_numpy(npo)
assert fo[2] == 3 + 5j
assert (npo == fo).all()
fo[0] = 1j
assert npo[0] == 1j
def as_reflection_table(self) -> flex.reflection_table:
"""Return the data in flex reflection table format"""
table = flex.reflection_table()
table["asu_miller_index"] = self.asu_miller_index
for k, v in self.Ih_table.iteritems():
table[k] = flumpy.from_numpy(v.to_numpy())
return table
def calculate_jacobian(Ih_table):
"""Calculate the jacobian matrix, size Ih_table.size by len(self.apm.x)."""
gsq = np.square(Ih_table.inverse_scale_factors) * Ih_table.weights
sumgsq = Ih_table.sum_in_groups(gsq)
dIh = (Ih_table.intensities -
(Ih_table.Ih_values * 2.0 *
Ih_table.inverse_scale_factors)) * Ih_table.weights
jacobian = calc_jacobian(
Ih_table.derivatives.transpose(),
Ih_table.h_index_matrix,
flumpy.from_numpy(Ih_table.Ih_values),
flumpy.from_numpy(Ih_table.inverse_scale_factors),
flumpy.from_numpy(dIh),
flumpy.from_numpy(sumgsq),
)
return jacobian
def _index(reflection_table, experiment, fail_on_bad_index=False):
"""Index the strong spots"""
# Get some stuff from experiment
A = np.array(experiment.crystal.get_A(), dtype=np.float64).reshape(3, 3)
s0 = np.array([experiment.beam.get_s0()], dtype=np.float64).reshape(3, 1)
s0_length = norm(s0)
detector = experiment.detector
# Create array if necessary
if "miller_index" not in reflection_table:
reflection_table["miller_index"] = flex.miller_index(
len(reflection_table))
# Index all the reflections
miller_index = reflection_table["miller_index"]
selection = flex.size_t()
num_reindexed = 0
for i, xyz in enumerate(reflection_table["xyzobs.px.value"]):
# Get the observed pixel coordinate
x, y, _ = xyz
# Get the lab coord
s1 = np.array(detector[0].get_pixel_lab_coord((x, y)),
dtype=np.float64).reshape(3, 1)
s1_norm = norm(s1)
s1 *= s0_length / s1_norm
# Get the reciprocal lattice vector
r = s1 - s0
# Compute the fractional miller index
hf = np.matmul(inv(A), r)
# Compute the integer miller index
h = np.array([int(floor(j + 0.5)) for j in hf[:, 0]],
dtype=int).reshape(3, 1)
# Print warning if reindexing
if tuple(h) != miller_index[i]:
logger.warn("Reindexing (% 3d, % 3d, % 3d) -> (% 3d, % 3d, % 3d)" %
(miller_index[i] + tuple(h)))
num_reindexed += 1
miller_index[i] = matrix.col(flumpy.from_numpy(h))
if fail_on_bad_index:
raise RuntimeError("Bad index")
# If its not indexed as 0, 0, 0 then append
if h.any() and norm(h - hf) < 0.3:
selection.append(i)
# Print some info
logger.info("Reindexed %d/%d input reflections" %
(num_reindexed, len(reflection_table)))
logger.info("Selected %d/%d input reflections" %
(len(selection), len(reflection_table)))
# Select all the indexed reflections
reflection_table.set_flags(selection, reflection_table.flags.indexed)
reflection_table = reflection_table.select(selection)
return reflection_table
def refine_fisher_scoring(self):
"""
Perform the profile refinement
"""
# Print information
logger.info("\nComponents to refine:")
logger.info(" Orientation: %s" %
(not self.state.is_orientation_fixed))
logger.info(" Unit cell: %s" %
(not self.state.is_unit_cell_fixed))
logger.info(" RLP mosaicity: %s" %
(not self.state.is_mosaic_spread_fixed))
logger.info(" Wavelength spread: %s\n" %
(not self.state.is_wavelength_spread_fixed))
# Initialise the algorithm
self.ml = FisherScoringMaximumLikelihood(
self.state,
self.s0,
self.sp_list,
self.h_list,
self.ctot_list,
self.mobs_list,
self.sobs_list,
)
# Solve the maximum likelihood equations
self.ml.solve()
# Get the parameters
self.parameters = flex.double(self.ml.parameters)
# set the parameters
self.state.active_parameters = self.parameters
# Print summary table of refinement.
rows = []
headers = ["Iteration", "likelihood", "RMSD (pixel) X,Y"]
for i, h in enumerate(self.ml.history):
l = h["likelihood"]
rmsd = h["rmsd"]
rows.append([str(i), f"{l:.4f}", f"{rmsd[0]:.3f}, {rmsd[1]:.3f}"])
logger.info("\nRefinement steps:\n\n" +
textwrap.indent(tabulate(rows, headers), " "))
# Print the eigen values and vectors of sigma_m
if not self.state.is_mosaic_spread_fixed:
logger.info("\nDecomposition of Sigma_M:")
print_eigen_values_and_vectors(
matrix.sqr(
flumpy.from_numpy(self.state.mosaicity_covariance_matrix)))
# Save the history
self.history = self.ml.history
# Return the optimizer
return self.ml
def calculate_residuals(self, _):
"""Return the residual vector"""
bin_vars = self.error_model.binner.bin_variances
R = ((np.square(np.full(bin_vars.size, 0.5) - bin_vars) +
(1.0 / bin_vars) - np.full(bin_vars.size, 1.25)) *
self.error_model.binner.weights /
np.sum(self.error_model.binner.weights))
return flumpy.from_numpy(R)
def score(self, x):
"""
:param x: The parameter estimate
:return: The score at x
"""
self.model.active_parameters = x
self._ml_target.update()
return flumpy.from_numpy(self._ml_target.first_derivatives())
def compute_bbox(self, experiments, reflections, probability=0.9973):
"""
Compute the bounding box
"""
calculator = BBoxCalculatorAngular(
experiments[0], matrix.sqr(flumpy.from_numpy(self.sigma())),
probability, 4)
calculator.compute(reflections)
def test_reverse_numeric_4d(flex_numeric):
dtype = lookup_flex_type_to_numpy[flex_numeric.__name__]
npo = np.zeros((1, 9, 8, 5), dtype=dtype)
fo = flumpy.from_numpy(npo)
assert isinstance(fo, flex_numeric)
assert fo.nd() == 4
for indices in itertools.product(range(1), range(9), range(8), range(5)):
fo[indices] = sum(indices)
assert fo[indices] == npo[indices]
def compute_mask(self, experiments, reflections, probability=0.9973):
"""
Compute the mask
"""
calculator = MaskCalculatorAngular(
experiments[0], matrix.sqr(flumpy.from_numpy(self.sigma())),
probability)
calculator.compute(reflections)
def test_nonowning():
f_a = flex.double([0, 1, 2, 3, 4])
n_b = flumpy.to_numpy(f_a)
f_c = flumpy.from_numpy(n_b[1:])
assert f_c[0] == 1
f_c[1] = 9
assert f_a[2] == 9
assert n_b[2] == 9
assert f_c[1] == 9
def test_reverse_numeric_2d(flex_numeric):
dtype = lookup_flex_type_to_numpy[flex_numeric.__name__]
npo = np.array(
[[240, 259, 144, 187], [240, 259, 144, 187], [240, 259, 144, 187]],
dtype=dtype)
fo = flumpy.from_numpy(npo)
assert isinstance(fo, flex_numeric)
assert fo.all() == npo.shape
assert all(fo[x] == npo[x] for x in itertools.product(range(3), range(4)))
npo[0] = 42
assert all(fo[x] == npo[x] for x in itertools.product(range(3), range(4)))
assert fo[0] == 42
fo[0, 1] = 2
assert npo[0, 1] == 2
# Test zero-dimensional arrays
npo_zero = np.zeros((0, 3))
assert list(flumpy.from_numpy(npo_zero).all()) == [0, 3]
assert flumpy.from_numpy(npo_zero).size() == 0
def test_reverse_numeric_1d(flex_numeric):
dtype = lookup_flex_type_to_numpy[flex_numeric.__name__]
npo = np.array([240, 259, 144, 187], dtype=dtype)
fo = flumpy.from_numpy(npo)
assert isinstance(fo, flex_numeric)
assert fo.all() == npo.shape
assert all(fo[x] == npo[x] for x in range(4))
npo[0] = 42
assert all(fo[x] == npo[x] for x in range(4))
assert fo[0] == 42
def score_and_fisher_information(self, x):
"""
:param x: The parameter estimate
:return: The score and fisher information at x
"""
self.model.active_parameters = x
self._ml_target.update()
S = flumpy.from_numpy(self._ml_target.first_derivatives())
I = self._ml_target.fisher_information()
return S, I
def calculate_gradients(Ih_table):
"""Return a gradient vector on length len(self.apm.x)."""
gsq = np.square(Ih_table.inverse_scale_factors) * Ih_table.weights
sumgsq = flumpy.from_numpy(Ih_table.sum_in_groups(gsq))
prefactor = (-2.0 * Ih_table.weights *
(Ih_table.intensities -
(Ih_table.Ih_values * Ih_table.inverse_scale_factors)))
dIh = flumpy.from_numpy(
(Ih_table.intensities -
(Ih_table.Ih_values * 2.0 * Ih_table.inverse_scale_factors)) *
Ih_table.weights)
dIh_by_dpi = calc_dIh_by_dpi(dIh, sumgsq, Ih_table.h_index_matrix,
Ih_table.derivatives.transpose())
term_1 = (flumpy.from_numpy(prefactor * Ih_table.Ih_values) *
Ih_table.derivatives)
term_2 = (flumpy.from_numpy(
Ih_table.sum_in_groups(prefactor * Ih_table.inverse_scale_factors))
* dIh_by_dpi)
gradient = term_1 + term_2
return gradient
def mosaicity(self) -> Dict:
"""Three components for mosaicity for Simple6"""
decomp = linalg.eigensystem.real_symmetric(
matrix.sqr(flumpy.from_numpy(
self.sigma())).as_flex_double_matrix())
vals = list(mosaicity_from_eigen_decomposition(decomp.values()))
min_m = min(vals)
max_m = max(vals)
vals.remove(min_m)
vals.remove(max_m)
return {"min": min_m, "mid": vals[0], "max": max_m}
def predict_reflections(self,
experiments,
miller_indices,
probability=0.9973):
"""
Predict the reflections
"""
predictor = PredictorAngular(
experiments[0], matrix.sqr(flumpy.from_numpy(self.sigma())),
probability)
return predictor.predict(miller_indices)
def print_eigen_values_and_vectors_of_observed_covariance(A, s0):
"""
Print the eigen values and vectors of a matrix
"""
# Compute the eigen decomposition of the covariance matrix
A = matrix.sqr(flumpy.from_numpy(A))
s0 = matrix.col(flumpy.from_numpy(s0))
eigen_decomposition = linalg.eigensystem.real_symmetric(
A.as_flex_double_matrix())
Q = matrix.sqr(eigen_decomposition.vectors())
L = matrix.diag(eigen_decomposition.values())
# Print the matrix eigen values
logger.info(f"\nEigen Values:\n{print_matrix(L, indent=2)}\n")
logger.info(f"\nEigen Vectors:\n{print_matrix(Q, indent=2)}\n")
logger.info("Observed covariance in degrees equivalent units")
logger.info("C1: %.5f degrees" % (sqrt(L[0]) * (180.0 / pi) / s0.length()))
logger.info("C2: %.5f degrees" % (sqrt(L[3]) * (180.0 / pi) / s0.length()))
