def test_calc_lookup_index():
pi = 3.141
theta_phi = flex.vec2_double([(0.001, -pi), (pi, pi), (0.001, pi), (pi, -pi)])
indices = calc_lookup_index(theta_phi, 1)
assert list(indices) == [0, 64799, 359, 64440]
indices = calc_lookup_index(theta_phi, 2)
assert list(indices) == [0, 259199, 719, 258480]
def configure_components(self, reflection_table, experiment, params):
"""Add the required reflection table data to the model components."""
if "scale" in self.components:
norm = (reflection_table["xyzobs.px.value"].parts()[2] *
self._configdict["s_norm_fac"])
self.components["scale"].data = {"x": norm}
if "decay" in self.components:
norm = (reflection_table["xyzobs.px.value"].parts()[2] *
self._configdict["d_norm_fac"])
self.components["decay"].parameter_restraints = flex.double(
self.components["decay"].parameters.size(),
params.physical.decay_restraint,
)
self.components["decay"].data = {
"x": norm,
"d": reflection_table["d"]
}
if "absorption" in self.components:
lmax = self._configdict["lmax"]
if reflection_table.size() > 100000:
assert "s0c" in reflection_table
assert "s1c" in reflection_table
theta_phi_0 = calc_theta_phi(
reflection_table["s0c"]) # array of tuples in radians
theta_phi_1 = calc_theta_phi(reflection_table["s1c"])
s0_lookup_index = calc_lookup_index(theta_phi_0,
points_per_degree=2)
s1_lookup_index = calc_lookup_index(theta_phi_1,
points_per_degree=2)
if SHScaleComponent.coefficients_list is None:
SHScaleComponent.coefficients_list = create_sph_harm_lookup_table(
lmax, points_per_degree=2
) # set the class variable and share
elif len(SHScaleComponent.coefficients_list) < (lmax *
(2.0 + lmax)):
# this (rare) case can happen if adding a new dataset with a larger lmax!
SHScaleComponent.coefficients_list = create_sph_harm_lookup_table(
lmax, points_per_degree=2
) # set the class variable and share
self.components["absorption"].data = {
"s0_lookup": s0_lookup_index,
"s1_lookup": s1_lookup_index,
}
# here just pass in good reflections
else:
self.components["absorption"].data = {
"sph_harm_table": sph_harm_table(reflection_table, lmax)
}
surface_weight = self._configdict["abs_surface_weight"]
parameter_restraints = flex.double([])
for i in range(1, lmax + 1):
parameter_restraints.extend(flex.double([1.0] * ((2 * i) + 1)))
parameter_restraints *= surface_weight
self.components[
"absorption"].parameter_restraints = parameter_restraints
def test_equality_of_two_harmonic_table_methods(dials_regression, run_in_tmpdir):
from dials_scaling_ext import calc_theta_phi, calc_lookup_index
from dxtbx.serialize import load
from dials.util.options import OptionParser
from libtbx import phil
from dials.algorithms.scaling.scaling_library import create_scaling_model
data_dir = os.path.join(dials_regression, "xia2-28")
pickle_path = os.path.join(data_dir, "20_integrated.pickle")
sequence_path = os.path.join(data_dir, "20_integrated_experiments.json")
phil_scope = phil.parse(
"""
include scope dials.command_line.scale.phil_scope
""",
process_includes=True,
)
optionparser = OptionParser(phil=phil_scope, check_format=False)
params, _ = optionparser.parse_args(args=[], quick_parse=True)
params.model = "physical"
lmax = 2
params.physical.lmax = lmax
reflection_table = flex.reflection_table.from_file(pickle_path)
experiments = load.experiment_list(sequence_path, check_format=False)
experiments = create_scaling_model(params, experiments, [reflection_table])
experiment = experiments[0]
# New method
reflection_table["phi"] = (
reflection_table["xyzobs.px.value"].parts()[2]
* experiment.scan.get_oscillation()[1]
)
reflection_table = calc_crystal_frame_vectors(reflection_table, experiment)
theta_phi_0 = calc_theta_phi(reflection_table["s0c"]) # array of tuples in radians
theta_phi_1 = calc_theta_phi(reflection_table["s1c"])
points_per_degree = 2
s0_lookup_index = calc_lookup_index(theta_phi_0, points_per_degree)
s1_lookup_index = calc_lookup_index(theta_phi_1, points_per_degree)
print(list(s0_lookup_index[0:20]))
print(list(s1_lookup_index[0:20]))
coefficients_list = create_sph_harm_lookup_table(lmax, points_per_degree)
experiment.scaling_model.components["absorption"].data = {
"s0_lookup": s0_lookup_index,
"s1_lookup": s1_lookup_index,
}
experiment.scaling_model.components[
"absorption"
].coefficients_list = coefficients_list
assert experiment.scaling_model.components["absorption"]._mode == "memory"
experiment.scaling_model.components["absorption"].update_reflection_data()
absorption = experiment.scaling_model.components["absorption"]
harmonic_values_list = absorption.harmonic_values[0]
experiment.scaling_model.components["absorption"].parameters = flex.double(
[0.1, -0.1, 0.05, 0.02, 0.01, -0.05, 0.12, -0.035]
)
scales, derivatives = experiment.scaling_model.components[
"absorption"
].calculate_scales_and_derivatives()
# Old method:
old_data = {"sph_harm_table": create_sph_harm_table(theta_phi_0, theta_phi_1, lmax)}
experiment.scaling_model.components["absorption"].data = old_data
assert experiment.scaling_model.components["absorption"]._mode == "speed"
experiment.scaling_model.components["absorption"].update_reflection_data()
old_harmonic_values = absorption.harmonic_values[0]
for i in range(0, 8):
print(i)
assert list(harmonic_values_list[i]) == pytest.approx(
list(old_harmonic_values.col(i).as_dense_vector()), abs=0.01
)
experiment.scaling_model.components["absorption"].parameters = flex.double(
[0.1, -0.1, 0.05, 0.02, 0.01, -0.05, 0.12, -0.035]
)
scales_1, derivatives_1 = experiment.scaling_model.components[
"absorption"
].calculate_scales_and_derivatives()
assert list(scales_1) == pytest.approx(list(scales), abs=0.001)
assert list(scales_1) != [1.0] * len(scales_1)
def plot_absorption_plots(physical_model, reflection_table=None):
"""Make a number of plots to help with the interpretation of the
absorption correction."""
# First plot the absorption surface
d = {
"absorption_surface": {
"data": [],
"layout": {
"title": "Absorption correction surface",
"xaxis": {
"domain": [0, 1],
"anchor": "y",
"title": "azimuthal angle (degrees)",
},
"yaxis": {
"domain": [0, 1],
"anchor": "x",
"title": "polar angle (degrees)",
},
},
"help": absorption_help_msg,
}
}
params = physical_model.components["absorption"].parameters
order = int(-1.0 + ((1.0 + len(params)) ** 0.5))
lfg = scitbxmath.log_factorial_generator(2 * order + 1)
STEPS = 50
azimuth_ = np.linspace(0, 2 * np.pi, 2 * STEPS)
polar_ = np.linspace(0, np.pi, STEPS)
THETA, _ = np.meshgrid(azimuth_, polar_, indexing="ij")
lmax = int(-1.0 + ((1.0 + len(params)) ** 0.5))
Intensity = np.ones(THETA.shape)
undiffracted_intensity = np.ones(THETA.shape)
counter = 0
sqrt2 = math.sqrt(2)
nsssphe = scitbxmath.nss_spherical_harmonics(order, 50000, lfg)
for l in range(1, lmax + 1):
for m in range(-l, l + 1):
for it, t in enumerate(polar_):
for ip, p in enumerate(azimuth_):
Ylm = nsssphe.spherical_harmonic(l, abs(m), t, p)
if m < 0:
r = sqrt2 * ((-1) ** m) * Ylm.imag
elif m == 0:
assert Ylm.imag == 0.0
r = Ylm.real
else:
r = sqrt2 * ((-1) ** m) * Ylm.real
Intensity[ip, it] += params[counter] * r
# for the undiffracted intensity, we want to add the correction
# at each point to the parity conjugate. We can use the fact
# that the odd l terms are parity odd, and even are even, to
# just calculate the even terms as follows
if l % 2 == 0:
undiffracted_intensity[ip, it] += params[counter] * r
counter += 1
d["absorption_surface"]["data"].append(
{
"x": list(azimuth_ * 180.0 / np.pi),
"y": list(polar_ * 180.0 / np.pi),
"z": list(Intensity.T.tolist()),
"type": "heatmap",
"colorscale": "Viridis",
"colorbar": {"title": "inverse <br>scale factor"},
"name": "absorption surface",
"xaxis": "x",
"yaxis": "y",
}
)
d["undiffracted_absorption_surface"] = {
"data": [],
"layout": {
"title": "Undiffracted absorption correction",
"xaxis": {
"domain": [0, 1],
"anchor": "y",
"title": "azimuthal angle (degrees)",
},
"yaxis": {
"domain": [0, 1],
"anchor": "x",
"title": "polar angle (degrees)",
},
},
"help": """
This plot shows the calculated relative absorption for a paths travelling
straight through the crystal at a given direction in a crystal-fixed frame of
reference (in spherical coordinates). This gives an indication of the effective
shape of the crystal for absorbing x-rays. In this plot, the pole (polar angle 0)
corresponds to the laboratory x-axis.
""",
}
d["undiffracted_absorption_surface"]["data"].append(
{
"x": list(azimuth_ * 180.0 / np.pi),
"y": list(polar_ * 180.0 / np.pi),
"z": list(undiffracted_intensity.T.tolist()),
"type": "heatmap",
"colorscale": "Viridis",
"colorbar": {"title": "inverse <br>scale factor"},
"name": "Undiffracted absorption correction",
"xaxis": "x",
"yaxis": "y",
}
)
if not reflection_table:
return d
# now plot the directions of the scattering vectors
d["vector_directions"] = {
"data": [],
"layout": {
"title": "Scattering vectors in crystal frame",
"xaxis": {
"domain": [0, 1],
"anchor": "y",
"title": "azimuthal angle (degrees)",
"range": [0, 360],
},
"yaxis": {
"domain": [0, 1],
"anchor": "x",
"title": "polar angle (degrees)",
"range": [0, 180],
},
"coloraxis": {
"showscale": False,
},
},
"help": """
This plot shows the scattering vector directions in the crystal reference frame
used to determine the absorption correction. The s0 vectors are plotted in yellow,
the s1 vectors are plotted in teal. This gives an indication of which parts of
the absorption correction surface are sampled when determining the absorption
correction. In this plot, the pole (polar angle 0) corresponds to the laboratory
x-axis.""",
}
STEPS = 180 # do one point per degree
azimuth_ = np.linspace(0, 2 * np.pi, 2 * STEPS)
polar_ = np.linspace(0, np.pi, STEPS)
THETA, _ = np.meshgrid(azimuth_, polar_, indexing="ij")
Intensity = np.full(THETA.shape, np.NAN)
# note, the s1_lookup, s0_lookup is only calculated for large datasets, so
# for small datasets we need to calculate again.
if "s1_lookup" not in physical_model.components["absorption"].data:
s1_lookup = calc_lookup_index(
calc_theta_phi(reflection_table["s1c"]), points_per_degree=1
)
idx_polar, idx_azimuth = np.divmod(np.unique(s1_lookup), 360)
Intensity[idx_azimuth, idx_polar] = 1
else:
s1_lookup = np.unique(physical_model.components["absorption"].data["s1_lookup"])
# x is phi, y is theta
idx_polar, idx_azimuth = np.divmod(s1_lookup, 720)
idx_polar = idx_polar // 2 # convert from two points per degree to one
idx_azimuth = idx_azimuth // 2
Intensity[idx_azimuth, idx_polar] = 1
d["vector_directions"]["data"].append(
{
"x": list(azimuth_ * 180.0 / np.pi),
"y": list(polar_ * 180.0 / np.pi),
"z": list(Intensity.T.tolist()),
"type": "heatmap",
"colorscale": "Viridis",
"showscale": False,
"xaxis": "x",
"yaxis": "y",
"zmin": 0,
"zmax": 2,
}
)
Intensity = np.full(THETA.shape, np.NAN)
if "s0_lookup" not in physical_model.components["absorption"].data:
s0_lookup = calc_lookup_index(
calc_theta_phi(reflection_table["s0c"]), points_per_degree=1
)
idx_polar, idx_azimuth = np.divmod(np.unique(s0_lookup), 360)
Intensity[idx_azimuth, idx_polar] = 2
else:
s0_lookup = np.unique(physical_model.components["absorption"].data["s0_lookup"])
# x is phi, y is theta
idx_polar, idx_azimuth = np.divmod(s0_lookup, 720)
idx_polar = idx_polar // 2 # convert from two points per degree to one
idx_azimuth = idx_azimuth // 2
Intensity[idx_azimuth, idx_polar] = 2
d["vector_directions"]["data"].append(
{
"x": list(azimuth_ * 180.0 / np.pi),
"y": list(polar_ * 180.0 / np.pi),
"z": list(Intensity.T.tolist()),
"type": "heatmap",
"colorscale": "Viridis",
"showscale": False,
"xaxis": "x",
"yaxis": "y",
"zmin": 0,
"zmax": 2,
}
)
scales = physical_model.components["absorption"].calculate_scales()
hist = flex.histogram(scales, n_slots=min(100, int(scales.size() * 10)))
d["absorption_corrections"] = {
"data": [
{
"x": list(hist.slot_centers()),
"y": list(hist.slots()),
"type": "bar",
"name": "Applied absorption corrections",
},
],
"layout": {
"title": "Applied absorption corrections",
"xaxis": {"anchor": "y", "title": "Inverse scale factor"},
"yaxis": {"anchor": "x", "title": "Number of reflections"},
},
}
return d
def test_equality_of_two_harmonic_table_methods(dials_data):
location = dials_data("l_cysteine_dials_output", pathlib=True)
refl = location / "20_integrated.pickle"
expt = location / "20_integrated_experiments.json"
phil_scope = phil.parse(
"""
include scope dials.command_line.scale.phil_scope
""",
process_includes=True,
)
parser = ArgumentParser(phil=phil_scope, check_format=False)
params, _ = parser.parse_args(args=[], quick_parse=True)
params.model = "physical"
lmax = 2
params.physical.lmax = lmax
reflection_table = flex.reflection_table.from_file(refl)
experiments = load.experiment_list(expt, check_format=False)
experiments = create_scaling_model(params, experiments, [reflection_table])
experiment = experiments[0]
# New method
reflection_table["phi"] = (reflection_table["xyzobs.px.value"].parts()[2] *
experiment.scan.get_oscillation()[1])
reflection_table = calc_crystal_frame_vectors(reflection_table, experiment)
reflection_table["s1c"] = align_axis_along_z((1.0, 0.0, 0.0),
reflection_table["s1c"])
reflection_table["s0c"] = align_axis_along_z((1.0, 0.0, 0.0),
reflection_table["s0c"])
theta_phi_0 = calc_theta_phi(
reflection_table["s0c"]) # array of tuples in radians
theta_phi_1 = calc_theta_phi(reflection_table["s1c"])
points_per_degree = 4
s0_lookup_index = calc_lookup_index(theta_phi_0, points_per_degree)
s1_lookup_index = calc_lookup_index(theta_phi_1, points_per_degree)
print(list(s0_lookup_index[0:20]))
print(list(s1_lookup_index[0:20]))
coefficients_list = create_sph_harm_lookup_table(lmax, points_per_degree)
experiment.scaling_model.components["absorption"].data = {
"s0_lookup": s0_lookup_index,
"s1_lookup": s1_lookup_index,
}
experiment.scaling_model.components[
"absorption"].coefficients_list = coefficients_list
assert experiment.scaling_model.components["absorption"]._mode == "memory"
experiment.scaling_model.components["absorption"].update_reflection_data()
absorption = experiment.scaling_model.components["absorption"]
harmonic_values_list = absorption.harmonic_values[0]
experiment.scaling_model.components["absorption"].parameters = flex.double(
[0.1, -0.1, 0.05, 0.02, 0.01, -0.05, 0.12, -0.035])
scales, derivatives = experiment.scaling_model.components[
"absorption"].calculate_scales_and_derivatives()
# Old method:
old_data = {
"sph_harm_table": create_sph_harm_table(theta_phi_0, theta_phi_1, lmax)
}
experiment.scaling_model.components["absorption"].data = old_data
assert experiment.scaling_model.components["absorption"]._mode == "speed"
experiment.scaling_model.components["absorption"].update_reflection_data()
old_harmonic_values = absorption.harmonic_values[0]
for i in range(0, 8):
print(i)
assert list(harmonic_values_list[i]) == pytest.approx(list(
old_harmonic_values.col(i).as_dense_vector()),
abs=0.01)
experiment.scaling_model.components["absorption"].parameters = flex.double(
[0.1, -0.1, 0.05, 0.02, 0.01, -0.05, 0.12, -0.035])
scales_1, derivatives_1 = experiment.scaling_model.components[
"absorption"].calculate_scales_and_derivatives()
assert list(scales_1) == pytest.approx(list(scales), abs=0.001)
assert list(scales_1) != [1.0] * len(scales_1)