def test_set_Ih_values_to_target(test_sg): """Test the setting of Ih values for targeted scaling.""" """Generate input for testing joint_Ih_table.""" Ih_table = IhTable( [generated_refl_for_splitting_1(), generated_refl_for_splitting_2()], test_sg, nblocks=2, ) Ih_table.calc_Ih() # First check that values are set up correctly. block_list = Ih_table.blocked_data_list assert list(block_list[0].Ih_values) == pytest.approx( [6.0, 17.0 / 3.0, 17.0 / 3.0, 6.0, 17.0 / 3.0]) assert list(block_list[1].Ih_values) == pytest.approx( [16.0 / 3.0, 16.0 / 3.0, 7.0, 16.0 / 3.0, 7.0, 7.0]) # set some values in the target # change the (2, 0, 0) reflections to (4, 0, 0) to test if they are removed # from the blocks t1 = generated_refl_for_splitting_1() t2 = generated_refl_for_splitting_2() t1["miller_index"][1] = (4, 0, 0) t1["miller_index"][5] = (4, 0, 0) t2["miller_index"][1] = (4, 0, 0) target = IhTable([t1, t2], test_sg, nblocks=1) target.blocked_data_list[0].Ih_table["Ih_values"] = np.array( [0.1, 0.2, 0.2, 0.3, 0.3, 0.4, 0.1, 0.2, 0.3, 0.4, 0.4]) for block in block_list: block.match_Ih_values_to_target(target) assert list(block_list[0].Ih_values) == [0.1, 0.2, 0.2, 0.1, 0.2] assert list(block_list[1].Ih_values) == [0.4, 0.4, 0.4]
def test_set_Ih_values_to_target(test_sg): """Test the setting of Ih values for targeted scaling.""" """Generate input for testing joint_Ih_table.""" Ih_table = IhTable( [generated_refl_for_splitting_1(), generated_refl_for_splitting_2()], test_sg, nblocks=2, ) Ih_table.calc_Ih() # First check that values are set up correctly. block_list = Ih_table.blocked_data_list assert list(block_list[0].Ih_values) == [ 6.0, 17.0 / 3.0, 17.0 / 3.0, 6.0, 17.0 / 3.0, ] assert list(block_list[1].Ih_values) == [ 16.0 / 3.0, 16.0 / 3.0, 7.0, 16.0 / 3.0, 7.0, 7.0, ] target = IhTable( [generated_refl_for_splitting_1(), generated_refl_for_splitting_2()], test_sg, nblocks=1, ) # set some values in the target # change the (2, 0, 0) reflections to (4, 0, 0) to test if they are removed # from the blocks vals = target.blocked_data_list[0].asu_miller_index vals[3] = (4, 0, 0) vals[4] = (4, 0, 0) vals[8] = (4, 0, 0) target.blocked_data_list[0].Ih_table["Ih_values"] = flex.double( [0.1, 0.2, 0.2, 0.3, 0.3, 0.4, 0.1, 0.2, 0.3, 0.4, 0.4]) for block in block_list: block.match_Ih_values_to_target(target) assert list(block_list[0].Ih_values) == [0.1, 0.2, 0.2, 0.1, 0.2] assert list(block_list[1].Ih_values) == [0.4, 0.4, 0.4]
def test_error_model_target(large_reflection_table, test_sg): """Test the error model target.""" Ih_table = IhTable([large_reflection_table], test_sg, nblocks=1) block = Ih_table.blocked_data_list[0] em = get_error_model("basic") em.min_reflections_required = 1 error_model = em(block, n_bins=2, min_Ih=2.0) error_model.update_for_minimisation([1.0, 0.05]) target = ErrorModelTarget(error_model, starting_values=[1.0, 0.05]) # Test residual calculation residuals = target.calculate_residuals() assert residuals == (flex.double(2, 1.0) - error_model.bin_variances)**2 # Test gradient calculation against finite differences. gradients = target.calculate_gradients() gradient_fd = calculate_gradient_fd(target) assert approx_equal(list(gradients), list(gradient_fd)) # Test the method calls r, g = target.compute_functional_gradients() assert r == residuals assert list(gradients) == list(g) r, g, c = target.compute_functional_gradients_and_curvatures() assert r == residuals assert list(gradients) == list(g) assert c is None
def refine_error_model(params, experiments, reflection_tables): """Do error model refinement.""" # prepare relevant data for datastructures for i, table in enumerate(reflection_tables): # First get the good data table = table.select(~table.get_flags(table.flags.bad_for_scaling, all=False)) # Now chose intensities, ideally these two options could be combined # with a smart refactor if params.intensity_choice == "combine": if not params.combine.Imid: sys.exit("Imid value must be provided if intensity_choice=combine") table = calculate_prescaling_correction(table) # needed for below. I, V = combine_intensities(table, params.combine.Imid) table["intensity"] = I table["variance"] = V else: table = choose_initial_scaling_intensities( table, intensity_choice=params.intensity_choice ) reflection_tables[i] = table space_group = experiments[0].crystal.get_space_group() Ih_table = IhTable( reflection_tables, space_group, additional_cols=["partiality"], anomalous=True ) # now do the error model refinement model = BasicErrorModel(basic_params=params.basic) try: model = run_error_model_refinement(model, Ih_table) except (ValueError, RuntimeError) as e: logger.info(e) else: return model
def test_error_model_target(large_reflection_table, test_sg): """Test the error model target.""" Ih_table = IhTable([large_reflection_table], test_sg, nblocks=1) block = Ih_table.blocked_data_list[0] em = BasicErrorModel em.min_reflections_required = 1 params = generated_param() params.weighting.error_model.basic.n_bins = 2 params.weighting.error_model.basic.min_Ih = 1.0 error_model = em(basic_params=params.weighting.error_model.basic) error_model.configure_for_refinement(block) error_model.parameters = [1.0, 0.05] parameterisation = ErrorModelB_APM(error_model) target = ErrorModelTargetB(error_model) target.predict(parameterisation) # Test residual calculation residuals = target.calculate_residuals(parameterisation) assert residuals == ( flex.double(2, 1.0) - flex.pow2(error_model.binner.binning_info["bin_variances"])) # Test gradient calculation against finite differences. gradients = target.calculate_gradients(parameterisation) gradient_fd = calculate_gradient_fd(target, parameterisation) assert list(gradients) == pytest.approx(list(gradient_fd)) # Test the method calls r, g = target.compute_functional_gradients(parameterisation) assert r == residuals assert list(gradients) == pytest.approx(list(g)) r, g = target.compute_functional_gradients(parameterisation) assert r == residuals assert list(gradients) == pytest.approx(list(g))
def create_Ih_table(experiments, reflections, selections=None, n_blocks=1): """Create an Ih table from a list of experiments and reflections. Optionally, a selection list can also be given, to select data from each reflection table. Allow an unequal number of experiments and reflections, as only need to extract one space group value (can optionally check all same if many).""" if selections: assert len(selections) == len( reflections), """Must have an equal number of reflection tables and selections in the input lists.""" space_group_0 = experiments[0].crystal.get_space_group() for experiment in experiments: assert (experiment.crystal.get_space_group() == space_group_0 ), """The space groups of all experiments must be equal.""" input_tables = [] indices_lists = [] for i, reflection in enumerate(reflections): if "inverse_scale_factor" not in reflection: reflection["inverse_scale_factor"] = flex.double( reflection.size(), 1.0) if selections: input_tables.append(reflection.select(selections[i])) indices_lists.append(selections[i].iselection()) else: input_tables.append(reflection) indices_lists = None Ih_table = IhTable(input_tables, space_group_0, indices_lists, nblocks=n_blocks) return Ih_table
def test_select_connected_reflections_across_datasets(): """Test the basic cross-dataset reflection selection algorithm. Make three reflection tables with the following reflections: symmetry groups 0 1 2 3 4 5 6 0 3 3 2 0 1 1 1 classes 1 0 2 0 0 3 2 1 2 2 1 1 5 0 4 0 With target=5, expect: number of chosen reflections per class: [8, 7, 7] symmetry groups used: [1, 5, 0, 4] """ n1 = [3, 3, 2, 0, 1, 1, 1] n2 = [0, 2, 0, 0, 3, 2, 1] n3 = [2, 1, 1, 5, 0, 4, 0] def make_refl_table(n_list, class_idx=0): """Make a reflection table with groups based on n_list.""" r1 = flex.reflection_table() miller_indices = [[(0, 0, i + 1)] * n for i, n in enumerate(n_list)] r1["miller_index"] = flex.miller_index( list(itertools.chain.from_iterable(miller_indices))) r1["intensity"] = flex.double(sum(n_list), 1) r1["variance"] = flex.double(sum(n_list), 1) r1["inverse_scale_factor"] = flex.double(sum(n_list), 1) r1["class_index"] = flex.int(sum(n_list), class_idx) return r1 reflections = [ make_refl_table(n1, 0), make_refl_table(n2, 1), make_refl_table(n3, 2), ] space_group = sgtbx.space_group("P1") table = IhTable(reflections, space_group) indices, datset_ids, total_in_classes = select_connected_reflections_across_datasets( table, min_per_class=5, Isigma_cutoff=0.0) assert list(total_in_classes) == [8, 7, 7] assert list(indices) == [0, 1, 2, 3, 4, 5, 8, 9] + [0, 1, 2, 3, 4, 5, 6 ] + [ 0, 1, 2, 9, 10, 11, 12, ] assert list(datset_ids) == [0] * 8 + [1] * 7 + [2] * 7
def reject_outliers(reflection_table, experiment, method="standard", zmax=6.0): """ Run an outlier algorithm on symmetry-equivalent intensities. This method runs an intensity-based outlier rejection algorithm, comparing the deviations from the weighted mean in groups of symmetry equivalent reflections. The outliers are determined and the outlier_in_scaling flag is set in the reflection table. The values intensity and variance must be set in the reflection table; these should be corrected but unscaled values, as an inverse_scale_factor will be applied during outlier rejection if this is present in the reflection table. The reflection table should also be prefiltered (e.g. not-integrated reflections should not be present) as no further filtering is done on the input table. Args: reflection_table: A reflection table. experiment: A single experiment object. method (str): Name (alias) of outlier rejection algorithm to use. zmax (float): Normalised deviation threshold for classifying an outlier. Returns: reflection_table: The input table with the outlier_in_scaling flag set. """ assert "intensity" in reflection_table, "reflection table has no 'intensity' column" assert "variance" in reflection_table, "reflection table has no 'variance' column" if not "inverse_scale_factor" in reflection_table: reflection_table["inverse_scale_factor"] = flex.double( reflection_table.size(), 1.0) Ih_table = IhTable([reflection_table], experiment.crystal.get_space_group(), nblocks=1) outlier_indices = determine_outlier_index_arrays(Ih_table, method=method, zmax=zmax)[0] # Unset any existing outlier flags before setting the new ones reflection_table.unset_flags( reflection_table.get_flags(reflection_table.flags.outlier_in_scaling), reflection_table.flags.outlier_in_scaling, ) reflection_table.set_flags(outlier_indices, reflection_table.flags.outlier_in_scaling) return reflection_table
def test_errormodel(large_reflection_table, test_sg): """Test the initialisation and methods of the error model.""" Ih_table = IhTable([large_reflection_table], test_sg, nblocks=1) block = Ih_table.blocked_data_list[0] params = generated_param() params.weighting.error_model.basic.n_bins = 2 params.weighting.error_model.basic.min_Ih = 1.0 em = BasicErrorModel em.min_reflections_required = 1 error_model = em(basic_params=params.weighting.error_model.basic) error_model.min_reflections_required = 1 error_model.configure_for_refinement(block) assert error_model.binner.summation_matrix[0, 1] == 1 assert error_model.binner.summation_matrix[1, 1] == 1 assert error_model.binner.summation_matrix[2, 0] == 1 assert error_model.binner.summation_matrix[3, 0] == 1 assert error_model.binner.summation_matrix[4, 0] == 1 assert error_model.binner.summation_matrix.non_zeroes == 5 assert list(error_model.binner.binning_info["refl_per_bin"]) == [3, 2] # Test calc sigmaprime x0 = 1.0 x1 = 0.1 sigmaprime = calc_sigmaprime([x0, x1], error_model.filtered_Ih_table) cal_sigpr = list( x0 * flex.sqrt(block.variances + flex.pow2(x1 * block.intensities)) / block.inverse_scale_factors) assert list(sigmaprime) == pytest.approx(cal_sigpr[4:7] + cal_sigpr[-2:]) # Test calc delta_hl sigmaprime = calc_sigmaprime([1.0, 0.0], error_model.filtered_Ih_table) # Reset # Calculate example for three elements, with intensities 1, 5 and 10 and # variances 1, 5 and 10 using he formula # delta_hl = math.sqrt(n_h - 1 / n_h) * (Ihl/ghl - Ih) / sigmaprime delta_hl = calc_deltahl( error_model.filtered_Ih_table, error_model.filtered_Ih_table.calc_nh(), sigmaprime, ) expected_deltas = [ (-3.0 / 2.0) * math.sqrt(2.0 / 3.0), (5.0 / 2.0) * math.sqrt(2.0 / 15.0), 5.0 * math.sqrt(2.0 / 30.0), -0.117647058824, 0.124783549621, ] assert list(delta_hl) == pytest.approx(expected_deltas)
def test_target_gradient_calculation_finite_difference(small_reflection_table, single_exp, physical_param): """Test the calculated gradients against a finite difference calculation.""" model = PhysicalScalingModel.from_data(physical_param, single_exp, small_reflection_table) # need to 'add_data' model.configure_components(small_reflection_table, single_exp, physical_param) model.components["scale"].update_reflection_data() model.components["decay"].update_reflection_data() apm = multi_active_parameter_manager( ScalingTarget(), [model.components], [["scale", "decay"]], scaling_active_parameter_manager, ) model.components["scale"].inverse_scales = flex.double([2.0, 1.0, 2.0]) model.components["decay"].inverse_scales = flex.double([1.0, 1.0, 0.4]) Ih_table = IhTable([small_reflection_table], single_exp.crystal.get_space_group()) with patch.object(SingleScaler, "__init__", lambda x, y, z, k: None): scaler = SingleScaler(None, None, None) scaler._Ih_table = Ih_table # Now do finite difference check. target = ScalingTarget() scaler.update_for_minimisation(apm, 0) grad = target.calculate_gradients(scaler.Ih_table.blocked_data_list[0]) res = target.calculate_residuals(scaler.Ih_table.blocked_data_list[0]) assert (res > 1e-8), """residual should not be zero, or the gradient test below will not really be working!""" # Now compare to finite difference f_d_grad = calculate_gradient_fd(target, scaler, apm) print(list(f_d_grad)) print(list(grad)) assert list(grad) == pytest.approx(list(f_d_grad)) sel = f_d_grad > 1e-8 assert sel, """assert sel has some elements, as finite difference grad should
def test_multi_dataset_outlier_rejection(test_sg): """Test outlier rejection with two datasets.""" rt1 = flex.reflection_table() rt1["intensity"] = flex.double( [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 20.0, 400.0, 10.0]) rt1["variance"] = flex.double(9, 1.0) rt1["inverse_scale_factor"] = flex.double(9, 1.0) rt1["miller_index"] = flex.miller_index([ (0, 0, 1), (0, 0, 1), (0, 0, 1), (0, 0, 1), (0, 0, 2), (0, 0, 2), (0, 0, 2), (0, 0, 2), (0, 0, 3), ]) rt1.set_flags( flex.bool( [True, False, False, False, False, False, False, False, False]), rt1.flags.excluded_for_scaling, ) rt1.set_flags(flex.bool(9, False), rt1.flags.user_excluded_in_scaling) rt2 = flex.reflection_table() rt2["intensity"] = flex.double([10.0, 20.0, 500.0]) rt2["variance"] = flex.double(3, 1.0) rt2["inverse_scale_factor"] = flex.double(3, 1.0) rt2["miller_index"] = flex.miller_index([(0, 0, 23), (0, 0, 1), (0, 0, 2)]) rt2.set_flags(flex.bool([False, False, False]), rt1.flags.excluded_for_scaling) rt2.set_flags(flex.bool(3, False), rt2.flags.user_excluded_in_scaling) Ih_table = IhTable([rt1, rt2], test_sg, nblocks=1) zmax = 6.0 outlier_rej = SimpleNormDevOutlierRejection(Ih_table, zmax) outlier_rej.run() outliers = outlier_rej.final_outlier_arrays assert len(outliers) == 2 assert list(outliers[0]) == [4, 5, 6, 7] assert list(outliers[1]) == [1, 2] outlier_rej = NormDevOutlierRejection(Ih_table, zmax) outlier_rej.run() outliers = outlier_rej.final_outlier_arrays assert len(outliers) == 2 assert list(outliers[0]) == [7, 6] assert list(outliers[1]) == [1, 2]
def test_error_model_on_simulated_data(background_variance, multiplicity, abs_tolerances, model_b): """Test the refinement of the error model using simulated data. The simulated data consists of 2150 unique reflections, with I = 5.0 * d^4, giving an intensity range from 122070.3 to 5.0, with a mean of 285.21 and a median of 13.76. Each reflection is sampled 'multiplicity' times from a poisson distribution to give the intensities, which is scaled away from the mean I using the model_bm factor (scaled such that when the model is refined, the correct b should be returned). The test uses three different representative levels of background variance (1=None, 5=low, 50=high), which is added to the variance from poisson counting statistics. A tolerance of 10 % is used for the model b parameter for good cases, which is increased to 20 % for tough cases. The purpose of the test is to show that the error model is finding the correct solution for a variety of background levels, with varying levels of systematic error (model b parameter, which is the one that has the highest effect on the errors output by scaling). Included are some more realistic cases, with lower multiplicity of measurement and high background variances. These tests are also designed to help validate the cutoff choices in the error model code: - the need for a min_Ih to ensure that poisson approx normal distribution - the need for an avg_I_over_var cutoff to remove background effects - cutting off extreme 'outlier' deviations as not to mislead the model.""" data = data_for_error_model_test(int(background_variance), int(multiplicity), b=model_b) Ih_table = IhTable([data], space_group("P 2ac 2ab")) em = get_error_model("basic") block = Ih_table.blocked_data_list[0] em.min_reflections_required = 250 error_model = em(block, n_bins=10) assert error_model.summation_matrix.n_rows > 400 refinery = error_model_refinery(engine="SimpleLBFGS", target=ErrorModelTarget(error_model), max_iterations=100) refinery.run() error_model = refinery.return_error_model() assert error_model.refined_parameters[0] == pytest.approx( 1.00, abs=abs_tolerances[0]) assert abs(error_model.refined_parameters[1]) == pytest.approx( model_b, abs=abs_tolerances[1])
def test_determine_Esq_outlier_index_arrays( generated_Ih_table, mock_exp_with_sg, test_sg ): # Set the emax lower to check that two reflections are identified as outliers outliers = determine_Esq_outlier_index_arrays( generated_Ih_table, mock_exp_with_sg, emax=1.5 ) assert list(outliers[0]) == [8, 9] # now split the dataset into two, to check the output is correctly formed rt = generate_outlier_table() rt1 = rt[0:9] rt2 = rt[9:] Ih_table = IhTable([rt1, rt2], test_sg) outliers = determine_Esq_outlier_index_arrays(Ih_table, mock_exp_with_sg, emax=1.5) assert list(outliers[0]) == [8] assert list(outliers[1]) == [0] assert len(outliers) == 2
def test_target_jacobian_calculation_finite_difference(physical_param, single_exp, large_reflection_table): """Test the calculated jacobian against a finite difference calculation.""" physical_param.physical.decay_correction = False model = PhysicalScalingModel.from_data(physical_param, single_exp, large_reflection_table) # need to 'add_data' model.configure_components(large_reflection_table, single_exp, physical_param) model.components["scale"].update_reflection_data() apm = multi_active_parameter_manager( ScalingTarget(), [model.components], [["scale"]], scaling_active_parameter_manager, ) Ih_table = IhTable([large_reflection_table], single_exp.crystal.get_space_group()) with patch.object(SingleScaler, "__init__", lambda x, y, z, k: None): scaler = SingleScaler(None, None, None) scaler._Ih_table = Ih_table target = ScalingTarget() scaler.update_for_minimisation(apm, 0) fd_jacobian = calculate_jacobian_fd(target, scaler, apm) r, jacobian, w = target.compute_residuals_and_gradients( scaler.Ih_table.blocked_data_list[0]) assert r == pytest.approx( [-50.0 / 3.0, 70.0 / 3.0, -20.0 / 3.0, 12.5, -2.5] + [-25.0, 0.0, -75.0, 0.0, 200.0]) assert w == pytest.approx( [0.1, 0.1, 0.1, 0.02, 0.1, 0.02, 0.01, 0.02, 0.01, 0.01]) n_rows = jacobian.n_rows n_cols = jacobian.n_cols print(jacobian) print(fd_jacobian) for i in range(0, n_rows): for j in range(0, n_cols): assert jacobian[i, j] == pytest.approx(fd_jacobian[i, j], abs=1e-4)
def test_errormodel(large_reflection_table, test_sg): """Test the initialisation and methods of the error model.""" # first test get_error_model helper function. with pytest.raises(ValueError): em = get_error_model("bad") em = get_error_model("basic") em.min_reflections_required = 1 Ih_table = IhTable([large_reflection_table], test_sg, nblocks=1) block = Ih_table.blocked_data_list[0] error_model = em(block, n_bins=2, min_Ih=1.0) assert error_model.summation_matrix[0, 1] == 1 assert error_model.summation_matrix[1, 1] == 1 assert error_model.summation_matrix[2, 0] == 1 assert error_model.summation_matrix[3, 0] == 1 assert error_model.summation_matrix[4, 0] == 1 assert error_model.summation_matrix.non_zeroes == 5 assert list(error_model.bin_counts) == [3, 2] # Test calc sigmaprime x0 = 1.0 x1 = 0.1 error_model.sigmaprime = error_model.calc_sigmaprime([x0, x1]) cal_sigpr = list(x0 * ((block.variances + ((x1 * block.intensities)**2))**0.5) / block.inverse_scale_factors) assert list(error_model.sigmaprime) == pytest.approx(cal_sigpr[4:7] + cal_sigpr[-2:]) # Test calc delta_hl error_model.sigmaprime = error_model.calc_sigmaprime([1.0, 0.0]) # Reset # Calculate example for three elements, with intensities 1, 5 and 10 and # variances 1, 5 and 10 using he formula # delta_hl = math.sqrt(n_h - 1 / n_h) * (Ihl/ghl - Ih) / sigmaprime error_model.delta_hl = error_model.calc_deltahl() expected_deltas = [ (-3.0 / 2.0) * math.sqrt(2.0 / 3.0), (5.0 / 2.0) * math.sqrt(2.0 / 15.0), 5.0 * math.sqrt(2.0 / 30.0), -0.117647058824, 0.124783549621, ] assert list(error_model.delta_hl) == pytest.approx(expected_deltas)
def test_reflection_selection(dials_regression): """Use a real dataset to test the selection algorithm.""" data_dir = os.path.join(dials_regression, "xia2-28") pickle_path = os.path.join(data_dir, "20_integrated.pickle") sequence_path = os.path.join(data_dir, "20_integrated_experiments.json") reflection_table = flex.reflection_table.from_file(pickle_path) experiment = load.experiment_list(sequence_path, check_format=False)[0] reflection_table["intensity"] = reflection_table["intensity.sum.value"] reflection_table["variance"] = reflection_table["intensity.sum.variance"] reflection_table["inverse_scale_factor"] = flex.double( reflection_table.size(), 1.0) reflection_table = reflection_table.select( reflection_table["variance"] > 0) reflection_table = reflection_table.select( reflection_table.get_flags(reflection_table.flags.integrated, all=True)) Ih_table_block = IhTable( [reflection_table], experiment.crystal.get_space_group()).Ih_table_blocks[0] reflection_table["phi"] = (reflection_table["xyzobs.px.value"].parts()[2] * experiment.scan.get_oscillation()[1]) reflection_table = calc_crystal_frame_vectors(reflection_table, experiment) Ih_table_block.Ih_table["s1c"] = reflection_table["s1c"].select( Ih_table_block.Ih_table["loc_indices"]) indices = select_highly_connected_reflections(Ih_table_block, experiment, min_per_area=10, n_resolution_bins=10) assert len(indices) > 1710 and len(indices) < 1800 # Give a high min_per_area to check that all reflections with multiplciity > 1 # are selected. indices = select_highly_connected_reflections(Ih_table_block, experiment, min_per_area=50, n_resolution_bins=10) # this dataset has 48 reflections with multiplicity = 1 assert len(indices) == reflection_table.size() - 48
def test_select_highly_connected_reflections_in_bin(): """Test the single-bin selection algorithm.""" r1 = flex.reflection_table() n_list = [3, 3, 2, 1, 1, 2, 2] miller_indices = [[(0, 0, i + 1)] * n for i, n in enumerate(n_list)] r1["miller_index"] = flex.miller_index( list(itertools.chain.from_iterable(miller_indices))) r1["class_index"] = flex.int([0, 1, 1, 0, 1, 2, 0, 0, 2, 1, 1, 2, 0, 1]) r1["intensity"] = flex.double(sum(n_list), 1) r1["variance"] = flex.double(sum(n_list), 1) r1["inverse_scale_factor"] = flex.double(sum(n_list), 1) sg = sgtbx.space_group("P1") Ih_table_block = IhTable([r1], sg).Ih_table_blocks[0] Ih_table_block.Ih_table["class_index"] = r1["class_index"].select( Ih_table_block.Ih_table["loc_indices"]) indices, total_in_classes = select_highly_connected_reflections_in_bin( Ih_table_block, min_per_class=2, min_total=6, max_total=100) assert list(total_in_classes) == [2, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] assert list(indices) == [0, 1, 2, 3, 4, 5, 10, 11]
def test_outlier_rejection_with_small_outliers(): rt = flex.reflection_table() rt["intensity"] = flex.double( [3560.84231, 3433.66407, 3830.64235, 0.20552, 3786.59537] + [4009.98652, 0.00000, 3578.91470, 3549.19151, 3379.58616] + [3686.38610, 3913.42869, 0.00000, 3608.84869, 3681.11110]) rt["variance"] = flex.double( [10163.98104, 9577.90389, 9702.84868, 3.77427, 8244.70685] + [9142.38221, 1.51118, 9634.53782, 9870.73103, 9078.23488] + [8977.26984, 8712.91360, 1.78802, 7473.26521, 10075.49862]) rt["inverse_scale_factor"] = flex.double(rt.size(), 1.0) rt["miller_index"] = flex.miller_index([(0, 0, 1)] * rt.size()) expected_outliers = [3, 6, 12] OutlierRej = NormDevOutlierRejection(IhTable([rt], space_group("P 1")), zmax=6.0) OutlierRej.run() outliers = OutlierRej.final_outlier_arrays assert len(outliers) == 1 assert set(outliers[0]) == set(expected_outliers)
def test_limit_outlier_weights(): rt = flex.reflection_table() rt["intensity"] = flex.double([100.0, 101.0, 109.0, 105.0, 1.0]) rt["variance"] = flex.double([100.0, 101.0, 109.0, 105.0, 1.0]) rt["inverse_scale_factor"] = flex.double(rt.size(), 1.0) rt["miller_index"] = flex.miller_index([(0, 0, 1)] * rt.size()) rt2 = flex.reflection_table() rt2["intensity"] = flex.double([100.0, 101.0, 102.0, 105.0, 1.0]) rt2["variance"] = flex.double([100.0, 101.0, 102.0, 105.0, 1.0]) rt2["inverse_scale_factor"] = flex.double(rt.size(), 1.0) rt2["miller_index"] = flex.miller_index([(0, 0, 1)] * rt.size()) table = IhTable([rt, rt2], space_group("P 1")) import copy new_weights = limit_outlier_weights( copy.deepcopy(table.Ih_table_blocks[0].weights), table.Ih_table_blocks[0].h_index_matrix, ) assert all(i <= 0.1 for i in new_weights)
def test_error_model_on_simulated_data(background_variance, multiplicity, abs_tolerances, model_a, model_b): """Test the refinement of the error model using simulated data. The simulated data consists of 2150 unique reflections, with I = 5.0 * d^4, giving an intensity range from 122070.3 to 5.0, with a mean of 285.21 and a median of 13.76. Each reflection is sampled 'multiplicity' times from a poisson distribution to give the intensities, which is scaled away from the mean I using the model_bm factor (scaled such that when the model is refined, the correct b should be returned). The test uses three different representative levels of background variance (1=None, 5=low, 50=high), which is added to the variance from poisson counting statistics. A tolerance of 10 % is used for the model b parameter for good cases, which is increased to 20 % for tough cases. """ data = data_for_error_model_test(int(background_variance), int(multiplicity), b=model_b, a=model_a) Ih_table = IhTable([data], space_group("P 2ac 2ab")) block = Ih_table.blocked_data_list[0] BasicErrorModel.min_reflections_required = 250 error_model = BasicErrorModel() error_model.configure_for_refinement(block) assert error_model.binner.summation_matrix.n_rows > 400 refinery = ErrorModelRefinery(error_model, parameters_to_refine=["a", "b"]) refinery.run() assert refinery.model.parameters[0] == pytest.approx(model_a, abs=abs_tolerances[0]) assert abs(refinery.model.parameters[1]) == pytest.approx( model_b, abs=abs_tolerances[1])
def outlier_target_table(test_sg): """Generate an Ih_table for targeted outlier rejection""" target = generate_target_table() target_Ih = IhTable([target], test_sg, nblocks=1) return target_Ih
def generated_Ih_table(test_sg): """Generate an Ih_table""" rt = generate_outlier_table() Ih_table = IhTable([rt], test_sg, nblocks=1) return Ih_table
def test_IhTable_freework(large_reflection_table, small_reflection_table, test_sg): sel1 = flex.bool(7, True) sel1[6] = False sel2 = flex.bool(4, True) sel2[1] = False Ih_table = IhTable( reflection_tables=[ large_reflection_table.select(sel1), small_reflection_table.select(sel2), ], indices_lists=[sel1.iselection(), sel2.iselection()], space_group=test_sg, nblocks=2, free_set_percentage=50.0, ) assert len(Ih_table.blocked_data_list) == 3 assert Ih_table.n_datasets == 2 assert Ih_table.n_work_blocks == 2 block_list = Ih_table.Ih_table_blocks # two standard blocks assert block_list[0].h_index_matrix[0, 0] == 1 assert block_list[0].h_index_matrix.non_zeroes == 1 assert block_list[1].h_index_matrix[0, 0] == 1 assert block_list[1].h_index_matrix[1, 0] == 1 assert block_list[1].h_index_matrix[2, 0] == 1 assert block_list[1].h_index_matrix.non_zeroes == 3 # free set block assert block_list[2].h_index_matrix[0, 0] == 1 assert block_list[2].h_index_matrix[1, 1] == 1 assert block_list[2].h_index_matrix[2, 2] == 1 assert block_list[2].h_index_matrix[3, 2] == 1 assert block_list[2].h_index_matrix[4, 3] == 1 assert block_list[2].h_index_matrix.non_zeroes == 5 assert list(block_list[0].block_selections[0]) == [5] assert list(block_list[0].block_selections[1]) == [] assert list(block_list[1].block_selections[0]) == [0, 2] assert list(block_list[1].block_selections[1]) == [0] assert list(block_list[2].block_selections[0]) == [1, 3, 4] assert list(block_list[2].block_selections[1]) == [3, 2] # test get_block_selections_for_dataset block_sels_0 = Ih_table.get_block_selections_for_dataset(0) assert len(block_sels_0) == 3 assert list(block_sels_0[0]) == [5] assert list(block_sels_0[1]) == [0, 2] assert list(block_sels_0[2]) == [1, 3, 4] block_sels_1 = Ih_table.get_block_selections_for_dataset(1) assert len(block_sels_1) == 3 assert list(block_sels_1[0]) == [] assert list(block_sels_1[1]) == [0] assert list(block_sels_1[2]) == [3, 2] with pytest.raises(AssertionError): _ = Ih_table.get_block_selections_for_dataset(2) Ih_table.calc_Ih() # test setting data # set scale factors new_s_block_2 = flex.double(range(1, 6)) Ih_table.set_inverse_scale_factors(new_s_block_2, 2) assert list(Ih_table.Ih_table_blocks[2].inverse_scale_factors) == list( new_s_block_2 ) # set derivatives derivs = Mock() Ih_table.set_derivatives(derivs, 0) assert Ih_table.Ih_table_blocks[0].derivatives is derivs def update_vars_side_effect(*args): return flex.double([0.5] * len(args[0])) # test setting an error model em = Mock() em.update_variances.side_effect = update_vars_side_effect Ih_table.update_weights(em) for block in Ih_table.Ih_table_blocks: assert list(block.weights) == pytest.approx([2.0] * block.size) Ih_table.calc_Ih(1) # now test free set with offset Ih_table = IhTable( reflection_tables=[ large_reflection_table.select(sel1), small_reflection_table.select(sel2), ], indices_lists=[sel1.iselection(), sel2.iselection()], space_group=test_sg, nblocks=2, free_set_percentage=50.0, free_set_offset=1, ) assert len(Ih_table.blocked_data_list) == 3 assert Ih_table.n_datasets == 2 assert Ih_table.n_work_blocks == 2 block_list = Ih_table.Ih_table_blocks # two standard blocks assert block_list[0].h_index_matrix[0, 0] == 1 assert block_list[0].h_index_matrix[1, 1] == 1 assert block_list[0].h_index_matrix.non_zeroes == 2 assert block_list[1].h_index_matrix[0, 0] == 1 assert block_list[1].h_index_matrix[1, 0] == 1 assert block_list[1].h_index_matrix[2, 1] == 1 assert block_list[1].h_index_matrix.non_zeroes == 3 # free set block assert block_list[2].h_index_matrix[0, 0] == 1 assert block_list[2].h_index_matrix[1, 1] == 1 assert block_list[2].h_index_matrix[2, 1] == 1 assert block_list[2].h_index_matrix[3, 1] == 1 assert block_list[2].h_index_matrix.non_zeroes == 4 assert list(block_list[0].block_selections[0]) == [1, 3] assert list(block_list[0].block_selections[1]) == [] assert list(block_list[1].block_selections[0]) == [4] assert list(block_list[1].block_selections[1]) == [3, 2] assert list(block_list[2].block_selections[0]) == [5, 0, 2] assert list(block_list[2].block_selections[1]) == [0] Ih_table.calc_Ih() # Test the 'as_miller_array' method. unit_cell = uctbx.unit_cell((1.0, 1.0, 1.0, 90.0, 90.0, 90.0)) arr = Ih_table.as_miller_array(unit_cell) assert arr.size() == 5 assert list(arr.indices()) == [ (0, 0, 1), (0, 2, 0), (0, 4, 0), (0, 4, 0), (10, 0, 0), ] assert list(arr.data()) == pytest.approx( [x / 2.0 for x in [100.0, 60.0, 30.0, 30.0, 10.0]] ) assert list(arr.sigmas()) == pytest.approx( [(x / 4.0) ** 0.5 for x in [100.0, 60.0, 30.0, 30.0, 10.0]] ) arr = Ih_table.as_miller_array(unit_cell, return_free_set_data=True) assert arr.size() == 4 assert list(arr.indices()) == [(0, 0, 2), (1, 0, 0), (1, 0, 0), (1, 0, 0)] assert list(arr.data()) == pytest.approx( [x / 2.0 for x in [40.0, 100.0, 80.0, 60.0]] ) assert list(arr.sigmas()) == pytest.approx( [(x / 4.0) ** 0.5 for x in [50.0, 90.0, 90.0, 60.0]] )
def test_IhTable_split_into_blocks( large_reflection_table, small_reflection_table, test_sg ): """Test that the Ih_table datastructure correctly organises the data from two reflection tables into two IhTableBlocks.""" sel1 = flex.bool(7, True) sel1[6] = False sel2 = flex.bool(4, True) sel2[1] = False Ih_table = IhTable( reflection_tables=[ large_reflection_table.select(sel1), small_reflection_table.select(sel2), ], indices_lists=[sel1.iselection(), sel2.iselection()], space_group=test_sg, nblocks=2, ) assert Ih_table.n_datasets == 2 assert Ih_table.n_work_blocks == 2 block_list = Ih_table.Ih_table_blocks assert list(block_list[0].Ih_table["asu_miller_index"]) == [ (0, 0, 1), (0, 0, 2), (0, 2, 0), ] assert list(block_list[1].Ih_table["asu_miller_index"]) == [ (0, 4, 0), (1, 0, 0), (1, 0, 0), (0, 4, 0), (1, 0, 0), (10, 0, 0), ] assert list(block_list[0].block_selections[0]) == [1, 5, 3] assert list(block_list[0].block_selections[1]) == [] assert list(block_list[1].block_selections[0]) == [4, 0, 2] assert list(block_list[1].block_selections[1]) == [3, 0, 2] # test the 'get_block_selections_for_dataset' method block_sels_0 = Ih_table.get_block_selections_for_dataset(dataset=0) assert len(block_sels_0) == 2 assert list(block_sels_0[0]) == [1, 5, 3] assert list(block_sels_0[1]) == [4, 0, 2] block_sels_1 = Ih_table.get_block_selections_for_dataset(dataset=1) assert len(block_sels_1) == 2 assert list(block_sels_1[0]) == [] assert list(block_sels_1[1]) == [3, 0, 2] # test the size method assert Ih_table.size == 9 expected_h_idx_matrix = sparse.matrix(4, 3) expected_h_idx_matrix[0, 0] = 1 expected_h_idx_matrix[1, 1] = 1 expected_h_idx_matrix[2, 2] = 1 assert block_list[0].h_index_matrix == expected_h_idx_matrix expected_h_idx_matrix = sparse.matrix(7, 3) expected_h_idx_matrix[0, 0] = 1 expected_h_idx_matrix[1, 1] = 1 expected_h_idx_matrix[2, 1] = 1 expected_h_idx_matrix[3, 0] = 1 expected_h_idx_matrix[4, 1] = 1 expected_h_idx_matrix[5, 2] = 1 assert block_list[1].h_index_matrix == expected_h_idx_matrix # check that only dataset 1 intensities are updated new_intensities = flex.double([60.0, 50.0, 40.0, 30.0, 20.0, 10.0]) Ih_table.update_data_in_blocks(data=new_intensities, dataset_id=0) assert list(Ih_table.blocked_data_list[0].intensities) == [50.0, 10.0, 30.0] assert list(Ih_table.blocked_data_list[1].intensities) == [ 20.0, 60.0, 40.0, 30.0, 60.0, 10.0, ] # try updating variances new_vars = flex.double([100.0, 200.0, 300.0, 400.0]) Ih_table.update_data_in_blocks(data=new_vars, dataset_id=1, column="variance") assert list(Ih_table.blocked_data_list[0].variances) == [100.0, 50.0, 60.0] assert list(Ih_table.blocked_data_list[1].variances) == [ 30.0, 90.0, 90.0, 400.0, 100.0, 300.0, ]
def run(): parser = OptionParser( read_experiments=True, read_reflections=True, check_format=False, epilog=__doc__, ) params, _, args = parser.parse_args(show_diff_phil=False, return_unhandled=True) log.config(verbosity=1, logfile="dials.cluster_filter.log") logger.info(dials_version()) diff_phil = parser.diff_phil.as_str() if diff_phil: logger.info("The following parameters have been modified:\n%s", diff_phil) reflections, expts = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) refls = reflections[0] refls["intensity"] = refls["intensity.scale.value"] refls["variance"] = refls["intensity.scale.variance"] refls["initial_index"] = flex.size_t_range(refls.size()) good_refls = refls.select(refls.get_flags(refls.flags.scaled)) Ih_table = IhTable( [good_refls], space_group=expts[0].crystal.get_space_group(), indices_lists=[good_refls["initial_index"]], ) block = Ih_table.blocked_data_list[0] to_exclude = flex.size_t([]) for group_idx in range(0, block.n_groups): sel = flex.bool(block.n_groups, False) sel[group_idx] = True sel_block = block.select_on_groups(sel) sel = sel_block.intensities / (sel_block.variances**0.5) > -1.0 sel_block = sel_block.select(sel) if sel_block.size: I = sel_block.intensities / sel_block.inverse_scale_factors V = sel_block.variances / (sel_block.inverse_scale_factors**2) result = test_group(I, V, sel_block.asu_miller_index[0]) if result: in_real = result[0] to_exclude.extend( flumpy.from_numpy( sel_block.Ih_table["loc_indices"].to_numpy()).select( ~in_real)) logger.info(to_exclude.size()) logger.info(refls.size()) bad = flex.bool(refls.size(), False) bad.set_selected(to_exclude, True) refls = refls.select(~bad) refls.as_file("filtered.refl") logger.info("Done")