def test_create_sph_harm_table(test_reflection_table, mock_exp):
"""Simple test for the spherical harmonic table, constructing the table step
by step, and verifying the values of a few easy-to-calculate entries.
This also acts as a test for the calc_theta_phi function as well."""
rt, exp = test_reflection_table, mock_exp
reflection_table = calc_crystal_frame_vectors(rt, exp)
theta_phi = calc_theta_phi(reflection_table["s0c"])
expected = [
(pi / 2.0, 0.0),
(pi / 2.0, -1.0 * pi / 4.0),
(pi / 2.0, -1.0 * pi / 2.0),
]
for v1, v2 in zip(theta_phi, expected):
assert v1 == pytest.approx(v2)
theta_phi_2 = calc_theta_phi(reflection_table["s1c"])
expected = [
(pi / 4.0, 0.0),
(pi / 4.0, -1.0 * pi / 4.0),
(pi / 4.0, -1.0 * pi / 2.0),
]
for v1, v2 in zip(theta_phi_2, expected):
assert v1 == pytest.approx(v2)
sph_h_t = create_sph_harm_table(theta_phi, theta_phi_2, 2)
Y10 = ((3.0 / (8.0 * pi)) ** 0.5) / 2.0
Y20 = -1.0 * ((5.0 / (256.0 * pi)) ** 0.5)
assert sph_h_t[1, 0] == pytest.approx(Y10)
assert sph_h_t[1, 1] == pytest.approx(Y10)
assert sph_h_t[1, 2] == pytest.approx(Y10)
assert sph_h_t[5, 0] == pytest.approx(Y20)
assert sph_h_t[5, 1] == pytest.approx(Y20)
assert sph_h_t[5, 2] == pytest.approx(Y20)
def sph_harm_table(reflection_table, lmax):
"""Calculate the spherical harmonic table for a spherical
harmonic absorption correction."""
theta_phi = calc_theta_phi(reflection_table["s0c"])
theta_phi_2 = calc_theta_phi(reflection_table["s1c"])
sph_h_t = create_sph_harm_table(theta_phi, theta_phi_2, lmax)
return sph_h_t
def configure_components(self, reflection_table, experiment, params):
"""Add the required reflection table data to the model components."""
if "scale" in self.components:
norm = (reflection_table["xyzobs.px.value"].parts()[2] *
self._configdict["s_norm_fac"])
self.components["scale"].data = {"x": norm}
if "decay" in self.components:
norm = (reflection_table["xyzobs.px.value"].parts()[2] *
self._configdict["d_norm_fac"])
self.components["decay"].parameter_restraints = flex.double(
self.components["decay"].parameters.size(),
params.physical.decay_restraint,
)
self.components["decay"].data = {
"x": norm,
"d": reflection_table["d"]
}
if "absorption" in self.components:
lmax = self._configdict["lmax"]
if reflection_table.size() > 100000:
assert "s0c" in reflection_table
assert "s1c" in reflection_table
theta_phi_0 = calc_theta_phi(
reflection_table["s0c"]) # array of tuples in radians
theta_phi_1 = calc_theta_phi(reflection_table["s1c"])
s0_lookup_index = calc_lookup_index(theta_phi_0,
points_per_degree=2)
s1_lookup_index = calc_lookup_index(theta_phi_1,
points_per_degree=2)
if SHScaleComponent.coefficients_list is None:
SHScaleComponent.coefficients_list = create_sph_harm_lookup_table(
lmax, points_per_degree=2
) # set the class variable and share
elif len(SHScaleComponent.coefficients_list) < (lmax *
(2.0 + lmax)):
# this (rare) case can happen if adding a new dataset with a larger lmax!
SHScaleComponent.coefficients_list = create_sph_harm_lookup_table(
lmax, points_per_degree=2
) # set the class variable and share
self.components["absorption"].data = {
"s0_lookup": s0_lookup_index,
"s1_lookup": s1_lookup_index,
}
# here just pass in good reflections
else:
self.components["absorption"].data = {
"sph_harm_table": sph_harm_table(reflection_table, lmax)
}
surface_weight = self._configdict["abs_surface_weight"]
parameter_restraints = flex.double([])
for i in range(1, lmax + 1):
parameter_restraints.extend(flex.double([1.0] * ((2 * i) + 1)))
parameter_restraints *= surface_weight
self.components[
"absorption"].parameter_restraints = parameter_restraints
def select_highly_connected_reflections(Ih_table_block,
experiment,
min_per_area,
n_resolution_bins,
print_summary=False):
"""Select highly connected reflections within a dataset, across resolutions."""
min_per_bin = min_per_area * 12 * 1.5
max_per_bin = min_per_area * 12 * 3.0
assert "s1c" in Ih_table_block.Ih_table
theta_phi_1 = calc_theta_phi(Ih_table_block.Ih_table["s1c"])
theta = theta_phi_1.parts()[0]
phi = theta_phi_1.parts()[1]
Ih_table_block.Ih_table["phi"] = (phi * 180 / pi) + 180.0
Ih_table_block.Ih_table["theta"] = theta * 180 / pi
Ih_table_block.Ih_table = assign_segment_index(Ih_table_block.Ih_table)
Ih_table_block.setup_binner(
experiment.crystal.get_unit_cell(),
experiment.crystal.get_space_group(),
n_resolution_bins,
)
binner = Ih_table_block.binner
overall_indices = flex.size_t()
header = ["d-range", "n_refl"] + [str(i) for i in range(0, 12)]
rows = []
for ibin in binner.range_all():
sel = binner.selection(ibin)
sel_Ih_table_block = Ih_table_block.select(sel)
indices_wrt_original = Ih_table_block.Ih_table["loc_indices"].select(
sel)
indices, total_in_classes = select_highly_connected_reflections_in_bin(
sel_Ih_table_block, min_per_area, min_per_bin, max_per_bin)
if indices:
overall_indices.extend(indices_wrt_original.select(indices))
d0, d1 = binner.bin_d_range(ibin)
rows.append([
str(round(d0, 3)) + " - " + str(round(d1, 3)),
str(int(flex.sum(total_in_classes))),
] + [str(int(i)) for i in total_in_classes])
st = simple_table(rows, header)
msg = """\nSummary of reflection selection algorithm for this dataset:
%s resolution bins, target: at least %s reflections per area,
between %s and %s reflections per resolution bin""" % (
n_resolution_bins,
min_per_area,
18 * min_per_area,
36 * min_per_area,
)
if print_summary:
logger.info(msg)
logger.info(st.format())
else:
logger.debug(msg)
logger.debug(st.format())
return overall_indices
def test_create_sph_harm_table(test_reflection_table, test_experiment_singleaxisgonio):
"""Simple test for the spherical harmonic table, constructing the table step
by step, and verifying the values of a few easy-to-calculate entries.
This also acts as a test for the calc_theta_phi function as well."""
rt, exp = test_reflection_table, test_experiment_singleaxisgonio
reflection_table = calc_crystal_frame_vectors(rt, exp)
reflection_table["s0c"] = align_axis_along_z(
(1.0, 0.0, 0.0), reflection_table["s0c"]
)
reflection_table["s1c"] = align_axis_along_z(
(1.0, 0.0, 0.0), reflection_table["s1c"]
)
theta_phi = calc_theta_phi(reflection_table["s0c"])
# so s0c vectors realigned in xyz is
# (1.0, 0.0, 0.0),
# (1.0 / sqrt(2.0), -1.0 / sqrt(2.0), 0.0),
# (0.0, -1.0, 0.0),
# physics conventions, theta from 0 to pi, phi from 0 to 2pi
expected = [
(pi / 2.0, 0.0),
(pi / 2.0, 7.0 * pi / 4.0),
(pi / 2.0, 3.0 * pi / 2.0),
]
for v1, v2 in zip(theta_phi, expected):
assert v1 == pytest.approx(v2)
theta_phi_2 = calc_theta_phi(reflection_table["s1c"])
expected = [
(pi / 4.0, pi),
(pi / 4.0, 3 * pi / 4.0),
(pi / 4.0, 1.0 * pi / 2.0),
]
for v1, v2 in zip(theta_phi_2, expected):
assert v1 == pytest.approx(v2)
sph_h_t = create_sph_harm_table(theta_phi, theta_phi_2, 2)
Y10 = ((3.0 / (8.0 * pi)) ** 0.5) / 2.0
Y20 = -1.0 * ((5.0 / (256.0 * pi)) ** 0.5)
assert sph_h_t[1, 0] == pytest.approx(Y10)
assert sph_h_t[1, 1] == pytest.approx(Y10)
assert sph_h_t[1, 2] == pytest.approx(Y10)
assert sph_h_t[5, 0] == pytest.approx(Y20)
assert sph_h_t[5, 1] == pytest.approx(Y20)
assert sph_h_t[5, 2] == pytest.approx(Y20)
def test_equality_of_two_harmonic_table_methods(dials_data):
data_dir = dials_data("l_cysteine_dials_output")
pickle_path = data_dir / "20_integrated.pickle"
sequence_path = data_dir / "20_integrated_experiments.json"
phil_scope = phil.parse(
"""
include scope dials.command_line.scale.phil_scope
""",
process_includes=True,
)
optionparser = OptionParser(phil=phil_scope, check_format=False)
params, _ = optionparser.parse_args(args=[], quick_parse=True)
params.model = "physical"
lmax = 2
params.physical.lmax = lmax
reflection_table = flex.reflection_table.from_file(pickle_path)
experiments = load.experiment_list(sequence_path, check_format=False)
experiments = create_scaling_model(params, experiments, [reflection_table])
experiment = experiments[0]
# New method
reflection_table["phi"] = (
reflection_table["xyzobs.px.value"].parts()[2]
* experiment.scan.get_oscillation()[1]
)
reflection_table = calc_crystal_frame_vectors(reflection_table, experiment)
theta_phi_0 = calc_theta_phi(reflection_table["s0c"]) # array of tuples in radians
theta_phi_1 = calc_theta_phi(reflection_table["s1c"])
points_per_degree = 2
s0_lookup_index = calc_lookup_index(theta_phi_0, points_per_degree)
s1_lookup_index = calc_lookup_index(theta_phi_1, points_per_degree)
print(list(s0_lookup_index[0:20]))
print(list(s1_lookup_index[0:20]))
coefficients_list = create_sph_harm_lookup_table(lmax, points_per_degree)
experiment.scaling_model.components["absorption"].data = {
"s0_lookup": s0_lookup_index,
"s1_lookup": s1_lookup_index,
}
experiment.scaling_model.components[
"absorption"
].coefficients_list = coefficients_list
assert experiment.scaling_model.components["absorption"]._mode == "memory"
experiment.scaling_model.components["absorption"].update_reflection_data()
absorption = experiment.scaling_model.components["absorption"]
harmonic_values_list = absorption.harmonic_values[0]
experiment.scaling_model.components["absorption"].parameters = flex.double(
[0.1, -0.1, 0.05, 0.02, 0.01, -0.05, 0.12, -0.035]
)
scales, derivatives = experiment.scaling_model.components[
"absorption"
].calculate_scales_and_derivatives()
# Old method:
old_data = {"sph_harm_table": create_sph_harm_table(theta_phi_0, theta_phi_1, lmax)}
experiment.scaling_model.components["absorption"].data = old_data
assert experiment.scaling_model.components["absorption"]._mode == "speed"
experiment.scaling_model.components["absorption"].update_reflection_data()
old_harmonic_values = absorption.harmonic_values[0]
for i in range(0, 8):
print(i)
assert list(harmonic_values_list[i]) == pytest.approx(
list(old_harmonic_values.col(i).as_dense_vector()), abs=0.01
)
experiment.scaling_model.components["absorption"].parameters = flex.double(
[0.1, -0.1, 0.05, 0.02, 0.01, -0.05, 0.12, -0.035]
)
scales_1, derivatives_1 = experiment.scaling_model.components[
"absorption"
].calculate_scales_and_derivatives()
assert list(scales_1) == pytest.approx(list(scales), abs=0.001)
assert list(scales_1) != [1.0] * len(scales_1)
def plot_absorption_plots(physical_model, reflection_table=None):
"""Make a number of plots to help with the interpretation of the
absorption correction."""
# First plot the absorption surface
d = {
"absorption_surface": {
"data": [],
"layout": {
"title": "Absorption correction surface",
"xaxis": {
"domain": [0, 1],
"anchor": "y",
"title": "azimuthal angle (degrees)",
},
"yaxis": {
"domain": [0, 1],
"anchor": "x",
"title": "polar angle (degrees)",
},
},
"help": absorption_help_msg,
}
}
params = physical_model.components["absorption"].parameters
order = int(-1.0 + ((1.0 + len(params)) ** 0.5))
lfg = scitbxmath.log_factorial_generator(2 * order + 1)
STEPS = 50
azimuth_ = np.linspace(0, 2 * np.pi, 2 * STEPS)
polar_ = np.linspace(0, np.pi, STEPS)
THETA, _ = np.meshgrid(azimuth_, polar_, indexing="ij")
lmax = int(-1.0 + ((1.0 + len(params)) ** 0.5))
Intensity = np.ones(THETA.shape)
undiffracted_intensity = np.ones(THETA.shape)
counter = 0
sqrt2 = math.sqrt(2)
nsssphe = scitbxmath.nss_spherical_harmonics(order, 50000, lfg)
for l in range(1, lmax + 1):
for m in range(-l, l + 1):
for it, t in enumerate(polar_):
for ip, p in enumerate(azimuth_):
Ylm = nsssphe.spherical_harmonic(l, abs(m), t, p)
if m < 0:
r = sqrt2 * ((-1) ** m) * Ylm.imag
elif m == 0:
assert Ylm.imag == 0.0
r = Ylm.real
else:
r = sqrt2 * ((-1) ** m) * Ylm.real
Intensity[ip, it] += params[counter] * r
# for the undiffracted intensity, we want to add the correction
# at each point to the parity conjugate. We can use the fact
# that the odd l terms are parity odd, and even are even, to
# just calculate the even terms as follows
if l % 2 == 0:
undiffracted_intensity[ip, it] += params[counter] * r
counter += 1
d["absorption_surface"]["data"].append(
{
"x": list(azimuth_ * 180.0 / np.pi),
"y": list(polar_ * 180.0 / np.pi),
"z": list(Intensity.T.tolist()),
"type": "heatmap",
"colorscale": "Viridis",
"colorbar": {"title": "inverse <br>scale factor"},
"name": "absorption surface",
"xaxis": "x",
"yaxis": "y",
}
)
d["undiffracted_absorption_surface"] = {
"data": [],
"layout": {
"title": "Undiffracted absorption correction",
"xaxis": {
"domain": [0, 1],
"anchor": "y",
"title": "azimuthal angle (degrees)",
},
"yaxis": {
"domain": [0, 1],
"anchor": "x",
"title": "polar angle (degrees)",
},
},
"help": """
This plot shows the calculated relative absorption for a paths travelling
straight through the crystal at a given direction in a crystal-fixed frame of
reference (in spherical coordinates). This gives an indication of the effective
shape of the crystal for absorbing x-rays. In this plot, the pole (polar angle 0)
corresponds to the laboratory x-axis.
""",
}
d["undiffracted_absorption_surface"]["data"].append(
{
"x": list(azimuth_ * 180.0 / np.pi),
"y": list(polar_ * 180.0 / np.pi),
"z": list(undiffracted_intensity.T.tolist()),
"type": "heatmap",
"colorscale": "Viridis",
"colorbar": {"title": "inverse <br>scale factor"},
"name": "Undiffracted absorption correction",
"xaxis": "x",
"yaxis": "y",
}
)
if not reflection_table:
return d
# now plot the directions of the scattering vectors
d["vector_directions"] = {
"data": [],
"layout": {
"title": "Scattering vectors in crystal frame",
"xaxis": {
"domain": [0, 1],
"anchor": "y",
"title": "azimuthal angle (degrees)",
"range": [0, 360],
},
"yaxis": {
"domain": [0, 1],
"anchor": "x",
"title": "polar angle (degrees)",
"range": [0, 180],
},
"coloraxis": {
"showscale": False,
},
},
"help": """
This plot shows the scattering vector directions in the crystal reference frame
used to determine the absorption correction. The s0 vectors are plotted in yellow,
the s1 vectors are plotted in teal. This gives an indication of which parts of
the absorption correction surface are sampled when determining the absorption
correction. In this plot, the pole (polar angle 0) corresponds to the laboratory
x-axis.""",
}
STEPS = 180 # do one point per degree
azimuth_ = np.linspace(0, 2 * np.pi, 2 * STEPS)
polar_ = np.linspace(0, np.pi, STEPS)
THETA, _ = np.meshgrid(azimuth_, polar_, indexing="ij")
Intensity = np.full(THETA.shape, np.NAN)
# note, the s1_lookup, s0_lookup is only calculated for large datasets, so
# for small datasets we need to calculate again.
if "s1_lookup" not in physical_model.components["absorption"].data:
s1_lookup = calc_lookup_index(
calc_theta_phi(reflection_table["s1c"]), points_per_degree=1
)
idx_polar, idx_azimuth = np.divmod(np.unique(s1_lookup), 360)
Intensity[idx_azimuth, idx_polar] = 1
else:
s1_lookup = np.unique(physical_model.components["absorption"].data["s1_lookup"])
# x is phi, y is theta
idx_polar, idx_azimuth = np.divmod(s1_lookup, 720)
idx_polar = idx_polar // 2 # convert from two points per degree to one
idx_azimuth = idx_azimuth // 2
Intensity[idx_azimuth, idx_polar] = 1
d["vector_directions"]["data"].append(
{
"x": list(azimuth_ * 180.0 / np.pi),
"y": list(polar_ * 180.0 / np.pi),
"z": list(Intensity.T.tolist()),
"type": "heatmap",
"colorscale": "Viridis",
"showscale": False,
"xaxis": "x",
"yaxis": "y",
"zmin": 0,
"zmax": 2,
}
)
Intensity = np.full(THETA.shape, np.NAN)
if "s0_lookup" not in physical_model.components["absorption"].data:
s0_lookup = calc_lookup_index(
calc_theta_phi(reflection_table["s0c"]), points_per_degree=1
)
idx_polar, idx_azimuth = np.divmod(np.unique(s0_lookup), 360)
Intensity[idx_azimuth, idx_polar] = 2
else:
s0_lookup = np.unique(physical_model.components["absorption"].data["s0_lookup"])
# x is phi, y is theta
idx_polar, idx_azimuth = np.divmod(s0_lookup, 720)
idx_polar = idx_polar // 2 # convert from two points per degree to one
idx_azimuth = idx_azimuth // 2
Intensity[idx_azimuth, idx_polar] = 2
d["vector_directions"]["data"].append(
{
"x": list(azimuth_ * 180.0 / np.pi),
"y": list(polar_ * 180.0 / np.pi),
"z": list(Intensity.T.tolist()),
"type": "heatmap",
"colorscale": "Viridis",
"showscale": False,
"xaxis": "x",
"yaxis": "y",
"zmin": 0,
"zmax": 2,
}
)
scales = physical_model.components["absorption"].calculate_scales()
hist = flex.histogram(scales, n_slots=min(100, int(scales.size() * 10)))
d["absorption_corrections"] = {
"data": [
{
"x": list(hist.slot_centers()),
"y": list(hist.slots()),
"type": "bar",
"name": "Applied absorption corrections",
},
],
"layout": {
"title": "Applied absorption corrections",
"xaxis": {"anchor": "y", "title": "Inverse scale factor"},
"yaxis": {"anchor": "x", "title": "Number of reflections"},
},
}
return d
