def test_reject_outliers(mock_exp_with_sg): """Test the reject outliers function""" refls = generate_outlier_table_2() exp = mock_exp_with_sg refls = reject_outliers(refls, exp) assert (list(refls.get_flags( refls.flags.outlier_in_scaling)) == expected_standard_output[1:]) # Try the case for two tables joined together refls = generate_outlier_table_2() refls2 = generate_outlier_table_2() refls.extend(refls2) refls = reject_outliers(refls, exp) assert (list(refls.get_flags( refls.flags.outlier_in_scaling)) == expected_standard_output[1:] * 2) # Check that any existing outlier flags are overwritten refls = generate_outlier_table_2() refls.set_flags(flex.bool([True] + [False] * 10), refls.flags.outlier_in_scaling) refls = reject_outliers(refls, exp) assert (list(refls.get_flags( refls.flags.outlier_in_scaling)) == expected_standard_output[1:]) # Test for suitable error raising if intensity column not present del refls["intensity"] with pytest.raises(AssertionError): refls = reject_outliers(refls, exp)
def filtered_arrays_from_experiments_reflections( experiments, reflections, outlier_rejection_after_filter=False, partiality_threshold=0.99, ): """Create a list of filtered arrays from experiments and reflections. A partiality threshold can be set, and if outlier_rejection_after_filter is True, and intensity.scale values are not present, then a round of outlier rejection will take place. Raises: ValueError: if no datasets remain after filtering. """ miller_arrays = [] ids_to_del = [] for idx, (expt, refl) in enumerate(zip(experiments, reflections)): crystal_symmetry = crystal.symmetry( unit_cell=expt.crystal.get_unit_cell(), space_group=expt.crystal.get_space_group(), ) # want to use scale intensities if present, else sum + prf (if available) if "intensity.scale.value" in refl: intensity_choice = ["scale"] intensity_to_use = "intensity.scale" else: assert "intensity.sum.value" in refl intensity_to_use = "intensity.sum" intensity_choice = ["sum"] if "intensity.prf.value" in refl: intensity_choice.append("profile") intensity_to_use = "intensity.prf" try: logger.info( "Filtering reflections for dataset %s" % (expt.identifier if expt.identifier else idx) ) refl = filter_reflection_table( refl, intensity_choice, min_isigi=-5, filter_ice_rings=False, combine_partials=True, partiality_threshold=partiality_threshold, ) except ValueError: logger.info( "Dataset %s removed as no reflections left after filtering", idx ) ids_to_del.append(idx) else: # If scale was chosen - will return scale or have raised ValueError # If prf or sum, possible was no prf but want to continue. try: refl["intensity"] = refl[intensity_to_use + ".value"] refl["variance"] = refl[intensity_to_use + ".variance"] except RuntimeError: # catch case where prf were removed. refl["intensity"] = refl["intensity.sum.value"] refl["variance"] = refl["intensity.sum.variance"] if outlier_rejection_after_filter and intensity_to_use != "intensity.scale": refl = reject_outliers(refl, expt, method="simple", zmax=12.0) refl = refl.select(~refl.get_flags(refl.flags.outlier_in_scaling)) miller_set = miller.set( crystal_symmetry, refl["miller_index"], anomalous_flag=False ) intensities = miller.array( miller_set, data=refl["intensity"], sigmas=flex.sqrt(refl["variance"]) ) intensities.set_observation_type_xray_intensity() intensities.set_info( miller.array_info(source="DIALS", source_type="pickle") ) miller_arrays.append(intensities) if not miller_arrays: raise ValueError( """No datasets remain after pre-filtering. Please check input data. The datasets may not contain any full reflections; the command line option partiality_threshold can be lowered to include partials.""" ) for id_ in ids_to_del[::-1]: del experiments[id_] del reflections[id_] return miller_arrays