def refine_error_model(params, experiments, reflection_tables):
"""Do error model refinement."""
# prepare relevant data for datastructures
for i, table in enumerate(reflection_tables):
# First get the good data
table = table.select(~table.get_flags(table.flags.bad_for_scaling, all=False))
# Now chose intensities, ideally these two options could be combined
# with a smart refactor
if params.intensity_choice == "combine":
if not params.combine.Imid:
sys.exit("Imid value must be provided if intensity_choice=combine")
table = calculate_prescaling_correction(table) # needed for below.
I, V = combine_intensities(table, params.combine.Imid)
table["intensity"] = I
table["variance"] = V
else:
table = choose_initial_scaling_intensities(
table, intensity_choice=params.intensity_choice
)
reflection_tables[i] = table
space_group = experiments[0].crystal.get_space_group()
Ih_table = IhTable(
reflection_tables, space_group, additional_cols=["partiality"], anomalous=True
)
# now do the error model refinement
model = BasicErrorModel(basic_params=params.basic)
try:
model = run_error_model_refinement(model, Ih_table)
except (ValueError, RuntimeError) as e:
logger.info(e)
else:
return model
def choose_scaling_intensities(reflection_table, intensity_choice="profile"):
"""Choose which intensities to use for scaling. The LP, QE and
partiality corrections are also applied. Two new columns are
added to the reflection table 'intensity' and 'variance', which have
all corrections applied except an inverse scale factor."""
if intensity_choice == "profile":
intensity_choice = "prf" # rename to allow string matching with refl table
if intensity_choice == "prf":
if (reflection_table.get_flags(
reflection_table.flags.integrated_prf).count(True) == 0):
logger.warning(
"No profile fitted reflections in this dataset, using summation intensities"
)
intensity_choice = "sum"
reflection_table = calculate_prescaling_correction(reflection_table)
conv = reflection_table["prescaling_correction"]
intstr = "intensity." + intensity_choice + ".value"
if intstr not in reflection_table:
# Can't find selection, try to choose prf, if not then sum (also catches combine
# which should not be used at this point)
if "intensity.prf.value" in reflection_table:
intstr = "intensity.prf.value"
else:
assert ("intensity.sum.value"
in reflection_table), """No recognised
intensity values found."""
intstr = "intensity.sum.value"
varstr = intstr.rstrip("value") + "variance"
else:
varstr = intstr.rstrip("value") + "variance"
logger.info(
"""%s intensities will be used for scaling (and mtz output if applicable). \n""",
intstr,
)
# prf partial intensities are the 'full' intensity values but sum are not
if "partiality" in reflection_table and intstr == "intensity.sum.value":
inverse_partiality = flex.double(reflection_table.size(), 1.0)
nonzero_partiality_sel = reflection_table["partiality"] > 0.0
good_refl = reflection_table.select(
reflection_table["partiality"] > 0.0)
inverse_partiality.set_selected(nonzero_partiality_sel.iselection(),
1.0 / good_refl["partiality"])
conv *= inverse_partiality
reflection_table["intensity"] = reflection_table[intstr] * conv
reflection_table["variance"] = reflection_table[varstr] * conv * conv
if ("partiality.inv.variance" in reflection_table
and intstr == "intensity.sum.value"):
reflection_table["variance"] += (
reflection_table[intstr] *
reflection_table["partiality.inv.variance"])
variance_mask = reflection_table["variance"] <= 0.0
reflection_table.set_flags(variance_mask,
reflection_table.flags.excluded_for_scaling)
return reflection_table
def generated_refl_for_comb():
"""Create a reflection table suitable for splitting into blocks."""
reflections = flex.reflection_table()
reflections["intensity"] = flex.double(
[1.0, 2.0, 3.0, 4.0, 500.0, 6.0, 2.0, 2.0])
reflections["variance"] = flex.double(8, 1.0)
reflections["intensity.prf.value"] = flex.double(
[1.0, 3.0, 3.0, 4.0, 50.0, 6.0, 3.0, 2.0])
reflections["intensity.prf.variance"] = flex.double(
[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0])
reflections["intensity.sum.value"] = flex.double(
[1.0, 4.0, 3.0, 4.0, 500.0, 6.0, 6.0, 2.0])
reflections["intensity.sum.variance"] = flex.double(8, 1.0)
reflections["miller_index"] = flex.miller_index([
(1, 0, 0),
(2, 0, 0),
(0, 0, 1),
(2, 2, 2),
(1, 0, 0),
(2, 0, 0),
(1, 0, 0),
(1, 0, 0),
])
reflections["d"] = flex.double([0.8, 2.1, 2.0, 1.4, 1.6, 2.5, 2.5, 2.5])
reflections["partiality"] = flex.double(8, 1.0)
reflections["Esq"] = flex.double(8, 1.0)
reflections["inverse_scale_factor"] = flex.double(8, 1.0)
reflections["xyzobs.px.value"] = flex.vec3_double([
(0.0, 0.0, 0.0),
(0.0, 0.0, 5.0),
(0.0, 0.0, 8.0),
(0.0, 0.0, 10.0),
(0.0, 0.0, 12.0),
(0.0, 0.0, 15.0),
(0.0, 0.0, 15.0),
(0.0, 0.0, 15.0),
])
reflections["s1"] = flex.vec3_double([
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
(0.0, 0.1, 1.0),
])
reflections.set_flags(flex.bool(8, True), reflections.flags.integrated)
reflections.set_flags(flex.bool([False] * 5 + [True] + [False] * 2),
reflections.flags.bad_for_scaling)
reflections["id"] = flex.int(8, 0)
reflections.experiment_identifiers()[0] = "0"
reflections = calculate_prescaling_correction(reflections)
return reflections
def test_calculate_prescaling_correction():
"""Test the helper function that applies the lp, dqe and partiality corr."""
reflection_table = flex.reflection_table()
reflection_table["lp"] = flex.double([1.0, 0.9, 0.8])
reflection_table["qe"] = flex.double([0.6, 0.5, 0.4])
reflection_table = calculate_prescaling_correction(reflection_table)
assert list(reflection_table["prescaling_correction"]) == [
1.0 / 0.6,
0.9 / 0.5,
0.8 / 0.4,
]
# Test compatibilty for old datasets
del reflection_table["qe"]
reflection_table["dqe"] = flex.double([0.6, 0.5, 0.4])
reflection_table = calculate_prescaling_correction(reflection_table)
assert list(reflection_table["prescaling_correction"]) == [
1.0 / 0.6,
0.9 / 0.5,
0.8 / 0.4,
]
def generate_simple_table(prf=True):
"""Generate a reflection table for testing intensity combination.
The numbers are contrived to make sum intensities agree well at high
intensity but terribly at low and vice versa for profile intensities."""
reflections = flex.reflection_table()
reflections["miller_index"] = flex.miller_index(
[
(0, 0, 1),
(0, 0, 1),
(0, 0, 1),
(0, 0, 1),
(0, 0, 1),
(0, 0, 2),
(0, 0, 2),
(0, 0, 2),
(0, 0, 2),
(0, 0, 2),
(0, 0, 3),
(0, 0, 3),
(0, 0, 3),
(0, 0, 3),
(0, 0, 3),
(0, 0, 4),
(0, 0, 4),
(0, 0, 4),
(0, 0, 4),
(0, 0, 4),
(0, 0, 5),
(0, 0, 5),
(0, 0, 5),
(0, 0, 5),
(0, 0, 5),
]
)
reflections["inverse_scale_factor"] = flex.double(25, 1.0)
# Contrive an example that should give the best cc12 when combined.
# make sum intensities agree well at high intensity but terribly at low
# and vice versa for profile intensities.
# profile less consistent at high intensity here
# sumless consistent at low intensity here
reflections["intensity.sum.value"] = flex.double(
[
10000.0,
11000.0,
9000.0,
8000.0,
12000.0,
500.0,
5600.0,
5500.0,
2000.0,
6000.0,
100.0,
50.0,
150.0,
75.0,
125.0,
30.0,
10.0,
2.0,
35.0,
79.0,
1.0,
10.0,
20.0,
10.0,
5.0,
]
)
reflections["intensity.sum.variance"] = flex.double(
[10000] * 5 + [5000] * 5 + [100] * 5 + [30] * 5 + [10] * 5
)
reflections.set_flags(flex.bool(25, False), reflections.flags.outlier_in_scaling)
reflections.set_flags(flex.bool(25, True), reflections.flags.integrated)
reflections["lp"] = flex.double(25, 0.5)
if prf:
reflections["intensity.prf.value"] = flex.double(
[
10000.0,
16000.0,
12000.0,
6000.0,
9000.0,
5000.0,
2000.0,
1500.0,
1300.0,
9000.0,
100.0,
80.0,
120.0,
90.0,
100.0,
30.0,
40.0,
50.0,
30.0,
30.0,
10.0,
12.0,
9.0,
8.0,
10.0,
]
)
reflections["intensity.prf.variance"] = flex.double(
[1000] * 5 + [500] * 5 + [10] * 5 + [3] * 5 + [1] * 5
)
reflections = calculate_prescaling_correction(reflections)
return reflections
def choose_initial_scaling_intensities(reflection_table, intensity_choice="profile"):
"""Choose which intensities to initially use for scaling. The LP, QE and
partiality corrections are also applied. Two new columns are
added to the reflection table 'intensity' and 'variance', which have
all corrections applied except an inverse scale factor."""
if intensity_choice == "profile":
intensity_choice = "prf" # rename to allow string matching with refl table
if "intensity.prf.value" not in reflection_table:
intensity_choice = "sum"
elif intensity_choice == "prf":
if (
reflection_table.get_flags(reflection_table.flags.integrated_prf).count(
True
)
== 0
):
logger.warning(
"No profile fitted reflections in this dataset, using summation intensities"
)
intensity_choice = "sum"
reflection_table = calculate_prescaling_correction(reflection_table)
conv = reflection_table["prescaling_correction"]
# if prf/sum, use those. If combine, use prf else sum for each refl.
if intensity_choice == "prf":
reflection_table["intensity"] = reflection_table["intensity.prf.value"] * conv
reflection_table["variance"] = (
reflection_table["intensity.prf.variance"] * conv * conv
)
else:
# first fill in summation intensities.
if "partiality" in reflection_table:
inverse_partiality = flex.double(reflection_table.size(), 1.0)
nonzero_partiality_sel = reflection_table["partiality"] > 0.0
good_refl = reflection_table.select(reflection_table["partiality"] > 0.0)
inverse_partiality.set_selected(
nonzero_partiality_sel.iselection(), 1.0 / good_refl["partiality"]
)
reflection_table["intensity"] = (
reflection_table["intensity.sum.value"] * conv * inverse_partiality
)
reflection_table["variance"] = reflection_table[
"intensity.sum.variance"
] * flex.pow2(conv * inverse_partiality)
if "partiality.inv.variance" in reflection_table:
reflection_table["variance"] += (
reflection_table["intensity.sum.value"]
* reflection_table["partiality.inv.variance"]
)
else:
reflection_table["intensity"] = (
reflection_table["intensity.sum.value"] * conv
)
reflection_table["variance"] = (
reflection_table["intensity.sum.variance"] * conv * conv
)
if intensity_choice == "combine":
# now overwrite prf if we have it.
sel = reflection_table.get_flags(reflection_table.flags.integrated_prf)
isel = sel.iselection()
Iprf = (reflection_table["intensity.prf.value"] * conv).select(sel)
Vprf = (reflection_table["intensity.prf.variance"] * conv * conv).select(
sel
)
reflection_table["intensity"].set_selected(isel, Iprf)
reflection_table["variance"].set_selected(isel, Vprf)
variance_mask = reflection_table["variance"] <= 0.0
reflection_table.set_flags(
variance_mask, reflection_table.flags.excluded_for_scaling
)
return reflection_table
