def test_reject_outliers(mock_exp_with_sg):
"""Test the reject outliers function"""
refls = generate_outlier_table_2()
exp = mock_exp_with_sg
refls = reject_outliers(refls, exp)
assert (list(refls.get_flags(
refls.flags.outlier_in_scaling)) == expected_standard_output[1:])
# Try the case for two tables joined together
refls = generate_outlier_table_2()
refls2 = generate_outlier_table_2()
refls.extend(refls2)
refls = reject_outliers(refls, exp)
assert (list(refls.get_flags(
refls.flags.outlier_in_scaling)) == expected_standard_output[1:] * 2)
# Check that any existing outlier flags are overwritten
refls = generate_outlier_table_2()
refls.set_flags(flex.bool([True] + [False] * 10),
refls.flags.outlier_in_scaling)
refls = reject_outliers(refls, exp)
assert (list(refls.get_flags(
refls.flags.outlier_in_scaling)) == expected_standard_output[1:])
# Test for suitable error raising if intensity column not present
del refls["intensity"]
with pytest.raises(AssertionError):
refls = reject_outliers(refls, exp)
def filtered_arrays_from_experiments_reflections(
experiments,
reflections,
outlier_rejection_after_filter=False,
partiality_threshold=0.99,
):
"""Create a list of filtered arrays from experiments and reflections.
A partiality threshold can be set, and if outlier_rejection_after_filter
is True, and intensity.scale values are not present, then a round of
outlier rejection will take place.
Raises:
ValueError: if no datasets remain after filtering.
"""
miller_arrays = []
ids_to_del = []
for idx, (expt, refl) in enumerate(zip(experiments, reflections)):
crystal_symmetry = crystal.symmetry(
unit_cell=expt.crystal.get_unit_cell(),
space_group=expt.crystal.get_space_group(),
)
# want to use scale intensities if present, else sum + prf (if available)
if "intensity.scale.value" in refl:
intensity_choice = ["scale"]
intensity_to_use = "intensity.scale"
else:
assert "intensity.sum.value" in refl
intensity_to_use = "intensity.sum"
intensity_choice = ["sum"]
if "intensity.prf.value" in refl:
intensity_choice.append("profile")
intensity_to_use = "intensity.prf"
try:
logger.info(
"Filtering reflections for dataset %s"
% (expt.identifier if expt.identifier else idx)
)
refl = filter_reflection_table(
refl,
intensity_choice,
min_isigi=-5,
filter_ice_rings=False,
combine_partials=True,
partiality_threshold=partiality_threshold,
)
except ValueError:
logger.info(
"Dataset %s removed as no reflections left after filtering", idx
)
ids_to_del.append(idx)
else:
# If scale was chosen - will return scale or have raised ValueError
# If prf or sum, possible was no prf but want to continue.
try:
refl["intensity"] = refl[intensity_to_use + ".value"]
refl["variance"] = refl[intensity_to_use + ".variance"]
except RuntimeError: # catch case where prf were removed.
refl["intensity"] = refl["intensity.sum.value"]
refl["variance"] = refl["intensity.sum.variance"]
if outlier_rejection_after_filter and intensity_to_use != "intensity.scale":
refl = reject_outliers(refl, expt, method="simple", zmax=12.0)
refl = refl.select(~refl.get_flags(refl.flags.outlier_in_scaling))
miller_set = miller.set(
crystal_symmetry, refl["miller_index"], anomalous_flag=False
)
intensities = miller.array(
miller_set, data=refl["intensity"], sigmas=flex.sqrt(refl["variance"])
)
intensities.set_observation_type_xray_intensity()
intensities.set_info(
miller.array_info(source="DIALS", source_type="pickle")
)
miller_arrays.append(intensities)
if not miller_arrays:
raise ValueError(
"""No datasets remain after pre-filtering. Please check input data.
The datasets may not contain any full reflections; the command line
option partiality_threshold can be lowered to include partials."""
)
for id_ in ids_to_del[::-1]:
del experiments[id_]
del reflections[id_]
return miller_arrays