コード例 #1
0
def test_check_and_fix(tc):

    n_det = tc.det_param.num_free()
    n_beam = tc.s0_param.num_free()
    n_xlo = tc.xlo_param.num_free()
    n_xluc = tc.xluc_param.num_free()

    # Similar to test_check_and_fail, setting 137 reflections as the minimum
    # should leave all parameters free
    options = ar_phil_scope.extract()
    options.min_nref_per_parameter = 137
    ar = AutoReduce(
        options,
        [tc.det_param],
        [tc.s0_param],
        [tc.xlo_param],
        [tc.xluc_param],
        gon_params=[],
        reflection_manager=tc.refman,
    )
    ar.check_and_fix()

    assert ar.det_params[0].num_free() == n_det == 6
    assert ar.beam_params[0].num_free() == n_beam == 3
    assert ar.xl_ori_params[0].num_free() == n_xlo == 3
    assert ar.xl_uc_params[0].num_free() == n_xluc == 6

    # Setting 138 reflections as the minimum should fix all the detector
    # parameters and remove that parameterisation. The crystal unit cell also
    # has 6 parameters, but each parameter is considered separately, so the
    # critical minimum number of reflections is 138*1 not 138*6 in that case
    options = ar_phil_scope.extract()
    options.min_nref_per_parameter = 138
    ar = AutoReduce(
        options,
        [tc.det_param],
        [tc.s0_param],
        [tc.xlo_param],
        [tc.xluc_param],
        gon_params=[],
        reflection_manager=tc.refman,
    )
    ar.check_and_fix()

    assert not ar.det_params
    assert ar.xl_uc_params[0].num_free() == n_xluc
    assert ar.beam_params[0].num_free() == n_beam
    assert ar.xl_ori_params[0].num_free() == n_xlo
コード例 #2
0
def test_check_and_fix(tc):

    n_det = tc.det_param.num_free()
    n_beam = tc.s0_param.num_free()
    n_xlo = tc.xlo_param.num_free()
    n_xluc = tc.xluc_param.num_free()

    # Similar to test_check_and_fail, setting 792 reflections as the minimum
    # should leave all parameters free
    options = ar_phil_scope.extract()
    options.min_nref_per_parameter = 792
    ar = AutoReduce(options,
                    pred_param=tc.pred_param,
                    reflection_manager=tc.refman)
    ar.check_and_fix()

    det_params = tc.pred_param.get_detector_parameterisations()
    beam_params = tc.pred_param.get_beam_parameterisations()
    xl_ori_params = tc.pred_param.get_crystal_orientation_parameterisations()
    xl_uc_params = tc.pred_param.get_crystal_unit_cell_parameterisations()
    assert det_params[0].num_free() == n_det == 6
    assert beam_params[0].num_free() == n_beam == 3
    assert xl_ori_params[0].num_free() == n_xlo == 3
    assert xl_uc_params[0].num_free() == n_xluc == 6

    # Setting 793 reflections as the minimum should fix crystal unit cell
    # parameters g_param_0, g_param_3 and g_param_4
    options = ar_phil_scope.extract()
    options.min_nref_per_parameter = 793
    ar = AutoReduce(options,
                    pred_param=tc.pred_param,
                    reflection_manager=tc.refman)
    ar.check_and_fix()

    det_params = tc.pred_param.get_detector_parameterisations()
    beam_params = tc.pred_param.get_beam_parameterisations()
    xl_ori_params = tc.pred_param.get_crystal_orientation_parameterisations()
    xl_uc_params = tc.pred_param.get_crystal_unit_cell_parameterisations()
    assert det_params[0].num_free() == n_det
    assert xl_uc_params[0].num_free() == 3
    assert xl_uc_params[0].get_param_names() == [
        "g_param_1", "g_param_2", "g_param_5"
    ]
    assert beam_params[0].num_free() == n_beam
    assert xl_ori_params[0].num_free() == n_xlo
コード例 #3
0
def test_check_and_fail(tc):

    # There are 823 reflections
    assert len(tc.refman.get_matches()) == 823

    # The parameters affecting the smallest number of reflections are
    # g_param_0, g_param_3 and g_param_4, all of which have gradients for
    # 792 reflections. Setting 792 reflections as the minimum should pass.
    options = ar_phil_scope.extract()
    options.min_nref_per_parameter = 792
    ar = AutoReduce(options, pred_param=tc.pred_param, reflection_manager=tc.refman)

    ar.check_and_fail()

    # Setting 793 reflections as the minimum should fail
    options.min_nref_per_parameter = 793
    ar = AutoReduce(options, pred_param=tc.pred_param, reflection_manager=tc.refman)

    with pytest.raises(DialsRefineConfigError):
        ar.check_and_fail()
コード例 #4
0
def test_check_and_fail(tc):

    # There are 823 reflections and the detector parameterisation has 6 free
    # parameters
    assert len(tc.refman.get_matches()) == 823
    assert tc.det_param.num_free() == 6

    # Setting 137 reflections as the minimum should pass (137*6<823)
    options = ar_phil_scope.extract()
    options.min_nref_per_parameter = 137
    ar = AutoReduce(
        options,
        [tc.det_param],
        [tc.s0_param],
        [tc.xlo_param],
        [tc.xluc_param],
        gon_params=[],
        reflection_manager=tc.refman,
    )

    ar.check_and_fail()

    # Setting 138 reflections as the minimum should fail (138*6>823)
    options.min_nref_per_parameter = 138
    ar = AutoReduce(
        options,
        [tc.det_param],
        [tc.s0_param],
        [tc.xlo_param],
        [tc.xluc_param],
        gon_params=[],
        reflection_manager=tc.refman,
    )

    with pytest.raises(DialsRefineConfigError):
        ar.check_and_fail()
コード例 #5
0
def test_check_and_remove():

    test = _Test()

    # Override the single panel model and parameterisation. This test function
    # exercises the code for non-hierarchical multi-panel detectors. The
    # hierarchical detector version is tested via test_cspad_refinement.py
    multi_panel_detector = Detector()
    for x in range(3):
        for y in range(3):
            new_panel = make_panel_in_array((x, y), test.detector[0])
            multi_panel_detector.add_panel(new_panel)
    test.detector = multi_panel_detector
    test.stills_experiments[0].detector = multi_panel_detector
    test.det_param = DetectorParameterisationMultiPanel(multi_panel_detector, test.beam)

    # update the generated reflections
    test.generate_reflections()

    # Predict the reflections in place and put in a reflection manager
    ref_predictor = StillsExperimentsPredictor(test.stills_experiments)
    ref_predictor(test.reflections)
    test.refman = ReflectionManagerFactory.from_parameters_reflections_experiments(
        refman_phil_scope.extract(),
        test.reflections,
        test.stills_experiments,
        do_stills=True,
    )
    test.refman.finalise()

    # Build a prediction parameterisation for the stills experiment
    test.pred_param = StillsPredictionParameterisation(
        test.stills_experiments,
        detector_parameterisations=[test.det_param],
        beam_parameterisations=[test.s0_param],
        xl_orientation_parameterisations=[test.xlo_param],
        xl_unit_cell_parameterisations=[test.xluc_param],
    )

    # A non-hierarchical detector does not have panel groups, thus panels are
    # not treated independently wrt which reflections affect their parameters.
    # As before, setting 792 reflections as the minimum should leave all
    # parameters free, and should not remove any reflections
    options = ar_phil_scope.extract()
    options.min_nref_per_parameter = 792
    ar = AutoReduce(options, pred_param=test.pred_param, reflection_manager=test.refman)
    ar.check_and_remove()

    det_params = test.pred_param.get_detector_parameterisations()
    beam_params = test.pred_param.get_beam_parameterisations()
    xl_ori_params = test.pred_param.get_crystal_orientation_parameterisations()
    xl_uc_params = test.pred_param.get_crystal_unit_cell_parameterisations()
    assert det_params[0].num_free() == 6
    assert beam_params[0].num_free() == 3
    assert xl_ori_params[0].num_free() == 3
    assert xl_uc_params[0].num_free() == 6
    assert len(test.refman.get_obs()) == 823

    # Setting 793 reflections as the minimum fixes 3 unit cell parameters,
    # and removes all those reflections. There are then too few reflections
    # for any parameterisation and all will be fixed, leaving no free
    # parameters for refinement. This fails within PredictionParameterisation,
    # during update so the final 31 reflections are not removed.
    options = ar_phil_scope.extract()
    options.min_nref_per_parameter = 793
    ar = AutoReduce(options, pred_param=test.pred_param, reflection_manager=test.refman)
    with pytest.raises(
        DialsRefineConfigError, match="There are no free parameters for refinement"
    ):
        ar.check_and_remove()

    det_params = test.pred_param.get_detector_parameterisations()
    beam_params = test.pred_param.get_beam_parameterisations()
    xl_ori_params = test.pred_param.get_crystal_orientation_parameterisations()
    xl_uc_params = test.pred_param.get_crystal_unit_cell_parameterisations()
    assert det_params[0].num_free() == 0
    assert beam_params[0].num_free() == 0
    assert xl_ori_params[0].num_free() == 0
    assert xl_uc_params[0].num_free() == 0
    assert len(test.refman.get_obs()) == 823 - 792
コード例 #6
0
ファイル: refiner.py プロジェクト: huwjenkins/dials
    def _build_components(cls, params, reflections, experiments):
        """low level build"""

        # Currently a refinement job can only have one parameterisation of the
        # prediction equation. This can either be of the XYDelPsi (stills) type, the
        # XYPhi (scans) type or the scan-varying XYPhi type with a varying crystal
        # model
        single_as_still = params.refinement.parameterisation.treat_single_image_as_still
        exps_are_stills = []
        for exp in experiments:
            if exp.scan is None:
                exps_are_stills.append(True)
            elif exp.scan.get_num_images() == 1:
                if single_as_still:
                    exps_are_stills.append(True)
                elif exp.scan.is_still():
                    exps_are_stills.append(True)
                else:
                    exps_are_stills.append(False)
            else:
                if exp.scan.get_oscillation()[1] <= 0.0:
                    raise DialsRefineConfigError(
                        "Cannot refine a zero-width scan")
                exps_are_stills.append(False)

        # check experiment types are consistent
        if not all(exps_are_stills[0] == e for e in exps_are_stills):
            raise DialsRefineConfigError(
                "Cannot refine a mixture of stills and scans")
        do_stills = exps_are_stills[0]

        # If experiments are stills, ensure scan-varying refinement won't be attempted
        if do_stills:
            params.refinement.parameterisation.scan_varying = False

        # Refiner does not accept scan_varying=Auto. This is a special case for
        # doing macrocycles of refinement in dials.refine.
        if params.refinement.parameterisation.scan_varying is libtbx.Auto:
            params.refinement.parameterisation.scan_varying = False

        # Trim scans and calculate reflection block_width if required for scan-varying refinement
        if (params.refinement.parameterisation.scan_varying and
                params.refinement.parameterisation.trim_scan_to_observations):

            experiments = _trim_scans_to_observations(experiments, reflections)

            from dials.algorithms.refinement.reflection_manager import BlockCalculator

            block_calculator = BlockCalculator(experiments, reflections)
            if params.refinement.parameterisation.compose_model_per == "block":
                reflections = block_calculator.per_width(
                    params.refinement.parameterisation.block_width, deg=True)
            elif params.refinement.parameterisation.compose_model_per == "image":
                reflections = block_calculator.per_image()

        logger.debug("\nBuilding reflection manager")
        logger.debug("Input reflection list size = %d observations",
                     len(reflections))

        # create reflection manager
        refman = ReflectionManagerFactory.from_parameters_reflections_experiments(
            params.refinement.reflections, reflections, experiments, do_stills)

        logger.debug(
            "Number of observations that pass initial inclusion criteria = %d",
            refman.get_accepted_refs_size(),
        )
        sample_size = refman.get_sample_size()
        if sample_size > 0:
            logger.debug("Working set size = %d observations", sample_size)
        logger.debug("Reflection manager built\n")

        # configure use of sparse data types
        params = cls.config_sparse(params, experiments)
        do_sparse = params.refinement.parameterisation.sparse

        # create managed reflection predictor
        ref_predictor = ExperimentsPredictorFactory.from_experiments(
            experiments,
            force_stills=do_stills,
            spherical_relp=params.refinement.parameterisation.
            spherical_relp_model,
        )

        # Predict for the managed observations, set columns for residuals and set
        # the used_in_refinement flag to the predictions
        obs = refman.get_obs()
        ref_predictor(obs)
        x_obs, y_obs, phi_obs = obs["xyzobs.mm.value"].parts()
        x_calc, y_calc, phi_calc = obs["xyzcal.mm"].parts()
        obs["x_resid"] = x_calc - x_obs
        obs["y_resid"] = y_calc - y_obs
        obs["phi_resid"] = phi_calc - phi_obs

        # determine whether to do basic centroid analysis to automatically
        # determine outlier rejection block
        if params.refinement.reflections.outlier.block_width is libtbx.Auto:
            ca = refman.get_centroid_analyser()
            analysis = ca(calc_average_residuals=False,
                          calc_periodograms=False)
        else:
            analysis = None

        # Now predictions and centroid analysis are available, so we can finalise
        # the reflection manager
        refman.finalise(analysis)

        # Create model parameterisations
        logger.debug("Building prediction equation parameterisation")
        pred_param = build_prediction_parameterisation(
            params.refinement.parameterisation, experiments, refman, do_stills)

        # Build a constraints manager, if requested
        cmf = ConstraintManagerFactory(params, pred_param)
        constraints_manager = cmf()

        # Test for parameters that have too little data to refine and act accordingly
        autoreduce = AutoReduce(
            params.refinement.parameterisation.auto_reduction,
            pred_param,
            refman,
            constraints_manager,
            cmf,
        )
        autoreduce()

        # if reduction was done, constraints_manager will have changed
        constraints_manager = autoreduce.constraints_manager

        # Build a restraints parameterisation (if requested).
        # Only unit cell restraints are supported at the moment.
        restraints_parameterisation = cls.config_restraints(
            params.refinement.parameterisation, pred_param)

        # Parameter reporting
        logger.debug("Prediction equation parameterisation built")
        logger.debug("Parameter order : name mapping")
        for i, e in enumerate(pred_param.get_param_names()):
            logger.debug("Parameter %03d : %s", i + 1, e)

        param_reporter = ParameterReporter(
            pred_param.get_detector_parameterisations(),
            pred_param.get_beam_parameterisations(),
            pred_param.get_crystal_orientation_parameterisations(),
            pred_param.get_crystal_unit_cell_parameterisations(),
            pred_param.get_goniometer_parameterisations(),
        )

        # Create target function
        logger.debug("Building target function")
        target = cls.config_target(
            params.refinement.target,
            experiments,
            refman,
            ref_predictor,
            pred_param,
            restraints_parameterisation,
            do_stills,
            do_sparse,
        )
        logger.debug("Target function built")

        # create refinery
        logger.debug("Building refinement engine")
        refinery = cls.config_refinery(params, target, pred_param,
                                       constraints_manager)
        logger.debug("Refinement engine built")

        nparam = len(pred_param)
        ndim = target.dim
        nref = len(refman.get_matches())
        logger.info(
            "There are %s parameters to refine against %s reflections in %s dimensions",
            nparam,
            nref,
            ndim,
        )

        if not params.refinement.parameterisation.sparse and isinstance(
                refinery, AdaptLstbx):
            dense_jacobian_gigabytes = (nparam * nref * ndim *
                                        flex.double.element_size()) / 1e9
            avail_memory_gigabytes = psutil.virtual_memory().available / 1e9
            # Report if the Jacobian requires a large amount of storage
            if (dense_jacobian_gigabytes > 0.2 * avail_memory_gigabytes
                    or dense_jacobian_gigabytes > 0.5):
                logger.info(
                    "Storage of the Jacobian matrix requires %.1f GB",
                    dense_jacobian_gigabytes,
                )

        # build refiner interface and return
        if params.refinement.parameterisation.scan_varying:
            refiner = ScanVaryingRefiner
        else:
            refiner = Refiner
        return refiner(experiments, pred_param, param_reporter, refman, target,
                       refinery)
コード例 #7
0
def test_check_and_remove():

    test = _Test()

    # Override the single panel model and parameterisation. This test function
    # exercises the code for non-hierarchical multi-panel detectors. The
    # hierarchical detector version is tested via test_cspad_refinement.py
    from dxtbx.model import Detector
    from dials.algorithms.refinement.parameterisation.detector_parameters import (
        DetectorParameterisationMultiPanel, )
    from dials.test.algorithms.refinement.test_multi_panel_detector_parameterisation import (
        make_panel_in_array, )

    multi_panel_detector = Detector()
    for x in range(3):
        for y in range(3):
            new_panel = make_panel_in_array((x, y), test.detector[0])
            multi_panel_detector.add_panel(new_panel)
    test.detector = multi_panel_detector
    test.stills_experiments[0].detector = multi_panel_detector
    test.det_param = DetectorParameterisationMultiPanel(
        multi_panel_detector, test.beam)

    # update the generated reflections
    test.generate_reflections()

    # Predict the reflections in place and put in a reflection manager
    ref_predictor = StillsExperimentsPredictor(test.stills_experiments)
    ref_predictor(test.reflections)
    test.refman = ReflectionManagerFactory.from_parameters_reflections_experiments(
        refman_phil_scope.extract(),
        test.reflections,
        test.stills_experiments,
        do_stills=True,
    )
    test.refman.finalise()

    # A non-hierarchical detector does not have panel groups, thus panels are
    # not treated independently wrt which reflections affect their parameters.
    # As before, setting 137 reflections as the minimum should leave all
    # parameters free, and should not remove any reflections
    options = ar_phil_scope.extract()
    options.min_nref_per_parameter = 137
    ar = AutoReduce(
        options,
        [test.det_param],
        [test.s0_param],
        [test.xlo_param],
        [test.xluc_param],
        gon_params=[],
        reflection_manager=test.refman,
    )
    ar.check_and_remove()

    assert ar.det_params[0].num_free() == 6
    assert ar.beam_params[0].num_free() == 3
    assert ar.xl_ori_params[0].num_free() == 3
    assert ar.xl_uc_params[0].num_free() == 6
    assert len(ar.reflection_manager.get_obs()) == 823

    # Setting reflections as the minimum should fix the detector parameters,
    # which removes that parameterisation. Because all reflections are recorded
    # on that detector, they will all be removed as well. This then affects all
    # other parameterisations, which will be removed.
    options = ar_phil_scope.extract()
    options.min_nref_per_parameter = 138
    ar = AutoReduce(
        options,
        [test.det_param],
        [test.s0_param],
        [test.xlo_param],
        [test.xluc_param],
        gon_params=[],
        reflection_manager=test.refman,
    )
    ar.check_and_remove()

    assert not ar.det_params
    assert not ar.beam_params
    assert not ar.xl_ori_params
    assert not ar.xl_uc_params
    assert len(ar.reflection_manager.get_obs()) == 0