Example #1
0
    def __call__(self):
        """Determine optimal mosaicity and domain size model (monochromatic)"""
        if self.refinery is None:
            RR = self.reflections
        else:
            RR = self.refinery.predict_for_reflection_table(self.reflections)

        all_crystals = []
        self.nv_acceptance_flags = flex.bool(len(self.reflections["id"]))
        from dxtbx.model import MosaicCrystalSauter2014

        for iid, experiment in enumerate(self.experiments):
            excursion_rad = RR["delpsical.rad"].select(RR["id"] == iid)
            delta_psi_deg = excursion_rad * 180.0 / math.pi
            logger.info("")
            logger.info("%s %s", flex.max(delta_psi_deg),
                        flex.min(delta_psi_deg))
            mean_excursion = flex.mean(delta_psi_deg)
            logger.info(
                "The mean excursion is %7.3f degrees, r.m.s.d %7.3f",
                mean_excursion,
                math.sqrt(flex.mean(RR["delpsical2"].select(RR["id"] == iid))),
            )

            crystal = MosaicCrystalSauter2014(self.experiments[iid].crystal)
            self.experiments[iid].crystal = crystal
            beam = self.experiments[iid].beam
            miller_indices = self.reflections["miller_index"].select(
                self.reflections["id"] == iid)

            # FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection
            two_thetas = crystal.get_unit_cell().two_theta(
                miller_indices, beam.get_wavelength(), deg=True)
            dspacings = crystal.get_unit_cell().d(miller_indices)

            # First -- try to get a reasonable envelope for the observed excursions.
            # minimum of three regions; maximum of 50 measurements in each bin
            logger.info("fitting parameters on %d spots", len(excursion_rad))
            n_bins = min(max(3, len(excursion_rad) // 25), 50)
            bin_sz = len(excursion_rad) // n_bins
            logger.info("nbins %s bin_sz %s", n_bins, bin_sz)
            order = flex.sort_permutation(two_thetas)
            two_thetas_env = flex.double()
            dspacings_env = flex.double()
            excursion_rads_env = flex.double()
            for x in range(0, n_bins):
                subset = order[x * bin_sz:(x + 1) * bin_sz]
                two_thetas_env.append(flex.mean(two_thetas.select(subset)))
                dspacings_env.append(flex.mean(dspacings.select(subset)))
                excursion_rads_env.append(
                    flex.max(flex.abs(excursion_rad.select(subset))))

            # Second -- parameter fit
            # solve the normal equations
            sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
            sum_inv_u = flex.sum(dspacings_env)
            sum_te_u = flex.sum(dspacings_env * excursion_rads_env)
            sum_te = flex.sum(excursion_rads_env)
            Normal_Mat = sqr(
                (sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
            Vector = col((sum_te_u, sum_te))
            solution = Normal_Mat.inverse() * Vector
            s_ang = 1.0 / (2 * solution[0])
            logger.info("Best LSQ fit Scheerer domain size is %9.2f ang",
                        s_ang)

            k_degrees = solution[1] * 180.0 / math.pi
            logger.info(
                "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f",
                2 * k_degrees,
                k_degrees,
            )

            from xfel.mono_simulation.max_like import minimizer

            # coerce the estimates to be positive for max-likelihood
            lower_limit_domain_size = (
                math.pow(crystal.get_unit_cell().volume(), 1.0 / 3.0) * 3
            )  # params.refinement.domain_size_lower_limit

            d_estimate = max(s_ang, lower_limit_domain_size)
            M = minimizer(
                d_i=dspacings,
                psi_i=excursion_rad,
                eta_rad=abs(2.0 * solution[1]),
                Deff=d_estimate,
            )
            logger.info(
                "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots",
                M.x[1] * 180.0 / math.pi,
                2.0 / M.x[0],
                len(two_thetas),
            )
            tan_phi_rad_ML = dspacings / (2.0 / M.x[0])
            tan_phi_deg_ML = tan_phi_rad_ML * 180.0 / math.pi
            tan_outer_deg_ML = tan_phi_deg_ML + 0.5 * M.x[1] * 180.0 / math.pi

            # Only set the flags for those reflections that were indexed for this lattice
            self.nv_acceptance_flags.set_selected(
                self.reflections["id"] == iid,
                flex.abs(delta_psi_deg) < tan_outer_deg_ML,
            )

            if (
                    self.graph_verbose
            ):  # params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit
                AD1TF7B_MAX2T = 30.0
                AD1TF7B_MAXDP = 1.0
                from matplotlib import pyplot as plt

                plt.plot(two_thetas, delta_psi_deg, "bo")
                minplot = flex.min(two_thetas)
                plt.plot([0, minplot], [mean_excursion, mean_excursion], "k-")
                LR = flex.linear_regression(two_thetas, delta_psi_deg)
                model_y = LR.slope() * two_thetas + LR.y_intercept()
                plt.plot(two_thetas, model_y, "k-")

                plt.title(
                    "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" %
                    (M.x[1] * 180.0 / math.pi, 2.0 / M.x[0], len(two_thetas)))
                plt.plot(two_thetas, tan_phi_deg_ML, "r.")
                plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
                plt.plot(two_thetas, tan_outer_deg_ML, "g.")
                plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
                plt.xlim([0, AD1TF7B_MAX2T])
                plt.ylim([-AD1TF7B_MAXDP, AD1TF7B_MAXDP])
                plt.show()
                plt.close()

            from xfel.mono_simulation.util import green_curve_area

            self.green_curve_area = green_curve_area(two_thetas,
                                                     tan_outer_deg_ML)
            logger.info("The green curve area is %s", self.green_curve_area)

            crystal.set_half_mosaicity_deg(M.x[1] * 180.0 / (2.0 * math.pi))
            crystal.set_domain_size_ang(2.0 / M.x[0])
            self._ML_full_mosaicity_rad = M.x[1]
            self._ML_domain_size_ang = 2.0 / M.x[0]

            # params.refinement.mosaic.model_expansion_factor
            """The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes
        as close as possible to # of observed reflections input, in the last integration call.  Determine this by
        inspecting the output log file interactively.  Do not exceed the bare minimum threshold needed.
        The intention is to find an optimal value, global for a given dataset."""
            model_expansion_factor = 1.4
            crystal.set_half_mosaicity_deg(crystal.get_half_mosaicity_deg() *
                                           model_expansion_factor)
            crystal.set_domain_size_ang(crystal.get_domain_size_ang() /
                                        model_expansion_factor)

            if (self.ewald_proximal_volume(iid) >
                    self.params.indexing.stills.ewald_proximal_volume_max):
                raise DialsIndexError("Ewald proximity volume too high, %f" %
                                      self.ewald_proximal_volume(iid))

            all_crystals.append(crystal)
        return all_crystals
Example #2
0
    def __call__(self):
        """Determine optimal mosaicity and domain size model (monochromatic)"""
        RR = self.refinery.predict_for_reflection_table(self.reflections)
        excursion_rad = RR["delpsical.rad"]
        delta_psi_deg = excursion_rad * 180. / math.pi
        print
        print flex.max(delta_psi_deg), flex.min(delta_psi_deg)
        mean_excursion = flex.mean(delta_psi_deg)
        print "The mean excursion is %7.3f degrees, r.m.s.d %7.3f" % (
            mean_excursion, math.sqrt(flex.mean(RR["delpsical2"])))

        crystal = self.experiments[0].crystal
        beam = self.experiments[0].beam
        miller_indices = self.reflections["miller_index"]

        # FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection
        two_thetas = crystal.get_unit_cell().two_theta(miller_indices,
                                                       beam.get_wavelength(),
                                                       deg=True)
        dspacings = crystal.get_unit_cell().d(miller_indices)
        dspace_sq = dspacings * dspacings

        #  First -- try to get a reasonable envelope for the observed excursions.
        ## minimum of three regions; maximum of 50 measurements in each bin
        print "fitting parameters on %d spots" % len(excursion_rad)
        n_bins = min(max(3, len(excursion_rad) // 25), 50)
        bin_sz = len(excursion_rad) // n_bins
        print "nbins", n_bins, "bin_sz", bin_sz
        order = flex.sort_permutation(two_thetas)
        two_thetas_env = flex.double()
        dspacings_env = flex.double()
        excursion_rads_env = flex.double()
        for x in xrange(0, n_bins):
            subset = order[x * bin_sz:(x + 1) * bin_sz]
            two_thetas_env.append(flex.mean(two_thetas.select(subset)))
            dspacings_env.append(flex.mean(dspacings.select(subset)))
            excursion_rads_env.append(
                flex.max(flex.abs(excursion_rad.select(subset))))

        #  Second -- parameter fit
        ## solve the normal equations
        sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
        sum_inv_u = flex.sum(dspacings_env)
        sum_te_u = flex.sum(dspacings_env * excursion_rads_env)
        sum_te = flex.sum(excursion_rads_env)
        Normal_Mat = sqr(
            (sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
        Vector = col((sum_te_u, sum_te))
        solution = Normal_Mat.inverse() * Vector
        s_ang = 1. / (2 * solution[0])
        print "Best LSQ fit Scheerer domain size is %9.2f ang" % (s_ang)

        tan_phi_rad = dspacings / (2. * s_ang)
        tan_phi_deg = tan_phi_rad * 180. / math.pi
        k_degrees = solution[1] * 180. / math.pi
        print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f" % (
            2 * k_degrees, k_degrees)
        tan_outer_deg = tan_phi_deg + k_degrees

        from xfel.mono_simulation.max_like import minimizer
        # coerce the estimates to be positive for max-likelihood
        lower_limit_domain_size = math.pow(
            crystal.get_unit_cell().volume(),
            1. / 3.) * 3  # params.refinement.domain_size_lower_limit

        d_estimate = max(s_ang, lower_limit_domain_size)
        M = minimizer(d_i=dspacings,
                      psi_i=excursion_rad,
                      eta_rad=abs(2. * solution[1]),
                      Deff=d_estimate)
        print "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" % (
            M.x[1] * 180. / math.pi, 2. / M.x[0], len(two_thetas))
        tan_phi_rad_ML = dspacings / (2. / M.x[0])
        tan_phi_deg_ML = tan_phi_rad_ML * 180. / math.pi
        tan_outer_deg_ML = tan_phi_deg_ML + 0.5 * M.x[1] * 180. / math.pi

        self.nv_acceptance_flags = flex.abs(delta_psi_deg) < tan_outer_deg_ML

        if self.graph_verbose:  #params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit
            AD1TF7B_MAX2T = 30.
            AD1TF7B_MAXDP = 1.
            from matplotlib import pyplot as plt
            plt.plot(two_thetas, delta_psi_deg, "bo")
            minplot = flex.min(two_thetas)
            plt.plot([0, minplot], [mean_excursion, mean_excursion], "k-")
            LR = flex.linear_regression(two_thetas, delta_psi_deg)
            model_y = LR.slope() * two_thetas + LR.y_intercept()
            plt.plot(two_thetas, model_y, "k-")

            plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" %
                      (M.x[1] * 180. / math.pi, 2. / M.x[0], len(two_thetas)))
            plt.plot(two_thetas, tan_phi_deg_ML, "r.")
            plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
            plt.plot(two_thetas, tan_outer_deg_ML, "g.")
            plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
            plt.xlim([0, AD1TF7B_MAX2T])
            plt.ylim([-AD1TF7B_MAXDP, AD1TF7B_MAXDP])
            plt.show()
            plt.close()

        from xfel.mono_simulation.util import green_curve_area
        self.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML)
        print "The green curve area is ", self.green_curve_area

        crystal._ML_half_mosaicity_deg = M.x[1] * 180. / (2. * math.pi)
        crystal._ML_domain_size_ang = 2. / M.x[0]
        self._ML_full_mosaicity_rad = M.x[1]
        self._ML_domain_size_ang = 2. / M.x[0]

        #params.refinement.mosaic.model_expansion_factor
        """The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes
    as close as possible to # of observed reflections input, in the last integration call.  Determine this by
    inspecting the output log file interactively.  Do not exceed the bare minimum threshold needed.
    The intention is to find an optimal value, global for a given dataset."""
        model_expansion_factor = 1.4
        crystal._ML_half_mosaicity_deg *= model_expansion_factor
        crystal._ML_domain_size_ang /= model_expansion_factor

        return crystal
    def refine_rotx_roty2(OO, enable_rotational_target=True):

        helper = OO.per_frame_helper_factory()
        helper.restart()

        if enable_rotational_target:
            print "Trying least squares minimization of excursions",
            from scitbx.lstbx import normal_eqns_solving
            iterations = normal_eqns_solving.naive_iterations(
                non_linear_ls=helper, gradient_threshold=1.E-10)

        results = helper.x

        print "with %d reflections" % len(OO.parent.indexed_pairs),
        print "result %6.2f degrees" % (results[1] * 180. / math.pi),
        print "result %6.2f degrees" % (results[0] * 180. / math.pi)

        if False:  # Excursion histogram
            print "The input mosaicity is %7.3f deg full width" % OO.parent.inputai.getMosaicity(
            )
            # final histogram
            if OO.pvr_fix:
                final = 360. * helper.fvec_callable_pvr(results)
            else:
                final = 360. * helper.fvec_callable_NOT_USED_AFTER_BUGFIX(
                    results)

            rmsdexc = math.sqrt(flex.mean(final * final))
            from matplotlib import pyplot as plt
            nbins = len(final) // 20
            n, bins, patches = plt.hist(final,
                                        nbins,
                                        normed=0,
                                        facecolor="orange",
                                        alpha=0.75)
            plt.xlabel("Rotation on e1 axis, rmsd %7.3f deg" % rmsdexc)
            plt.title("Histogram of cctbx.xfel misorientation")
            plt.axis([-0.5, 0.5, 0, 100])
            plt.plot([rmsdexc], [18], "b|")
            plt.show()

        # Determine optimal mosaicity and domain size model (monochromatic)
        if OO.pvr_fix:
            final = 360. * helper.fvec_callable_pvr(results)
        else:
            final = 360. * helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results)
        #Guard against misindexing -- seen in simulated data, with zone nearly perfectly aligned
        guard_stats = flex.max(final), flex.min(final)
        if False and REMOVETEST_KILLING_LEGITIMATE_EXCURSIONS(
                guard_stats[0] > 2.0 or guard_stats[1] < -2.0):
            raise Exception(
                "Misindexing diagnosed by meaningless excursion angle (bandpass_gaussian model)"
            )
        print "The mean excursion is %7.3f degrees" % (flex.mean(final))

        two_thetas = helper.last_set_orientation.unit_cell().two_theta(
            OO.reserve_indices, OO.central_wavelength_ang, deg=True)
        dspacings = helper.last_set_orientation.unit_cell().d(
            OO.reserve_indices)
        dspace_sq = dspacings * dspacings
        excursion_rad = final * math.pi / 180.

        #  First -- try to get a reasonable envelope for the observed excursions.
        ## minimum of three regions; maximum of 50 measurements in each bin
        print "fitting parameters on %d spots" % len(excursion_rad)
        n_bins = min(max(3, len(excursion_rad) // 25), 50)
        bin_sz = len(excursion_rad) // n_bins
        print "nbins", n_bins, "bin_sz", bin_sz
        order = flex.sort_permutation(two_thetas)
        two_thetas_env = flex.double()
        dspacings_env = flex.double()
        excursion_rads_env = flex.double()
        for x in range(0, n_bins):
            subset = order[x * bin_sz:(x + 1) * bin_sz]
            two_thetas_env.append(flex.mean(two_thetas.select(subset)))
            dspacings_env.append(flex.mean(dspacings.select(subset)))
            excursion_rads_env.append(
                flex.max(flex.abs(excursion_rad.select(subset))))

        #  Second -- parameter fit
        ## solve the normal equations
        sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
        sum_inv_u = flex.sum(dspacings_env)
        sum_te_u = flex.sum(dspacings_env * excursion_rads_env)
        sum_te = flex.sum(excursion_rads_env)
        Normal_Mat = sqr(
            (sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
        Vector = col((sum_te_u, sum_te))
        solution = Normal_Mat.inverse() * Vector
        s_ang = 1. / (2 * solution[0])
        print "Best LSQ fit Scheerer domain size is %9.2f ang" % (s_ang)
        tan_phi_rad = helper.last_set_orientation.unit_cell().d(
            OO.reserve_indices) / (2. * s_ang)
        tan_phi_deg = tan_phi_rad * 180. / math.pi
        k_degrees = solution[1] * 180. / math.pi
        print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f" % (
            2 * k_degrees, k_degrees)
        tan_outer_deg = tan_phi_deg + k_degrees

        if OO.mosaic_refinement_target == "ML":
            from xfel.mono_simulation.max_like import minimizer
            print "input", s_ang, 2. * solution[1] * 180 / math.pi
            # coerce the estimates to be positive for max-likelihood
            lower_limit_domain_size = math.pow(
                helper.last_set_orientation.unit_cell().volume(), 1. /
                3.) * 20  # 10-unit cell block size minimum reasonable domain

            d_estimate = max(s_ang, lower_limit_domain_size)
            M = minimizer(d_i=dspacings,
                          psi_i=excursion_rad,
                          eta_rad=abs(2. * solution[1]),
                          Deff=d_estimate)
            print "output", 1. / M.x[0], M.x[1] * 180. / math.pi
            tan_phi_rad_ML = helper.last_set_orientation.unit_cell().d(
                OO.reserve_indices) / (2. / M.x[0])
            tan_phi_deg_ML = tan_phi_rad_ML * 180. / math.pi
            # bugfix: Need factor of 0.5 because the plot shows half mosaicity (displacement from the center point defined as zero)
            tan_outer_deg_ML = tan_phi_deg_ML + 0.5 * M.x[1] * 180. / math.pi

        if OO.parent.horizons_phil.integration.mosaic.enable_polychromatic:
            # add code here to perform polychromatic modeling.
            """
        get miller indices DONE
        get model-predicted mono-wavelength centroid S1 vectors
        back-convert S1vec, with mono-wavelength, to detector-plane position, factoring in subpixel correction
        compare with spot centroid measured position
        compare with locus of bodypixels
        """
            print list(OO.reserve_indices)
            print len(OO.reserve_indices), len(two_thetas)
            positions = [
                OO.ucbp3.simple_forward_calculation_spot_position(
                    wavelength=OO.central_wavelength_ang,
                    observation_no=obsno).position
                for obsno in range(len(OO.parent.indexed_pairs))
            ]
            print len(positions)
            print positions  # model-predicted positions
            print len(OO.parent.spots)
            print OO.parent.indexed_pairs
            print OO.parent.spots
            print len(OO.parent.spots)
            meas_spots = [
                OO.parent.spots[pair["spot"]]
                for pair in OO.parent.indexed_pairs
            ]
            #      for xspot in meas_spots:
            #        xspot.ctr_mass_x(),xspot.ctr_mass_y()
            #        xspot.max_pxl_x()
            #        xspot.bodypixels
            #        xspot.ctr_mass_x()

            # Do some work to calculate an rmsd
            diff_vecs = flex.vec3_double()
            for p, xspot in zip(positions, meas_spots):
                diff_vecs.append((p[0] - xspot.ctr_mass_y(),
                                  p[1] - xspot.ctr_mass_x(), 0.0))
            # could use diff_vecs.rms_length()
            diff_vecs_sq = diff_vecs.dot(diff_vecs)
            mean_diff_vec_sq = flex.mean(diff_vecs_sq)
            rmsd = math.sqrt(mean_diff_vec_sq)
            print "mean obs-pred diff vec on %d spots is %6.2f pixels" % (
                len(positions), rmsd)

            positions_to_fictitious = [
                OO.ucbp3.simple_forward_calculation_spot_position(
                    wavelength=OO.central_wavelength_ang,
                    observation_no=obsno).position_to_fictitious
                for obsno in range(len(OO.parent.indexed_pairs))
            ]
            # Do some work to calculate an rmsd
            diff_vecs = flex.vec3_double()
            for p, xspot in zip(positions_to_fictitious, meas_spots):
                diff_vecs.append((p[0] - xspot.ctr_mass_y(),
                                  p[1] - xspot.ctr_mass_x(), 0.0))
            rmsd = diff_vecs.rms_length()
            print "mean obs-pred_to_fictitious diff vec on %d spots is %6.2f pixels" % (
                len(positions), rmsd)
            """
        actually, it might be better if the entire set of experimental observations
        is transformed into the ideal detector plane, for the purposes of poly_treatment.


        start here.  Now it would be good to actually implement probability of observing a body pixel given the model.
        We have everything needed right here.
        """
            if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B:
                # Image plot: obs and predicted positions + bodypixels
                from matplotlib import pyplot as plt
                plt.plot([p[0] for p in positions_to_fictitious],
                         [p[1] for p in positions_to_fictitious], "r.")
                plt.plot([xspot.ctr_mass_y() for xspot in meas_spots],
                         [xspot.ctr_mass_x() for xspot in meas_spots], "g.")
                bodypx = []
                for xspot in meas_spots:
                    for body in xspot.bodypixels:
                        bodypx.append(body)
                plt.plot([b.y for b in bodypx], [b.x for b in bodypx], "b.")
                plt.axes().set_aspect("equal")
                plt.show()

        print "MEAN excursion", flex.mean(final),
        if OO.mosaic_refinement_target == "ML":
            print "mosaicity deg FW=", M.x[1] * 180. / math.pi
        else:
            print
        if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B:  # Excursion vs resolution fit
            AD1TF7B_MAX2T = 30.
            AD1TF7B_MAXDP = 1.
            from matplotlib import pyplot as plt
            fig = plt.figure()
            plt.plot(two_thetas, final, "bo")
            mean = flex.mean(final)
            minplot = flex.min(two_thetas)
            plt.plot([0, minplot], [mean, mean], "k-")
            LR = flex.linear_regression(two_thetas, final)
            #LR.show_summary()
            model_y = LR.slope() * two_thetas + LR.y_intercept()
            plt.plot(two_thetas, model_y, "k-")
            print helper.last_set_orientation.unit_cell()
            #for sdp,tw in zip (dspacings,two_thetas):
            #print sdp,tw
            if OO.mosaic_refinement_target == "ML":
                plt.title(
                    "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" %
                    (M.x[1] * 180. / math.pi, 2. / M.x[0], len(two_thetas)))
                plt.plot(two_thetas, tan_phi_deg_ML, "r.")
                plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
                plt.plot(two_thetas, tan_outer_deg_ML, "g.")
                plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
            else:
                plt.plot(two_thetas_env, excursion_rads_env * 180. / math.pi,
                         "r|")
                plt.plot(two_thetas_env, -excursion_rads_env * 180. / math.pi,
                         "r|")
                plt.plot(two_thetas_env, excursion_rads_env * 180. / math.pi,
                         "r-")
                plt.plot(two_thetas_env, -excursion_rads_env * 180. / math.pi,
                         "r-")
                plt.plot(two_thetas, tan_phi_deg, "r.")
                plt.plot(two_thetas, -tan_phi_deg, "r.")
                plt.plot(two_thetas, tan_outer_deg, "g.")
                plt.plot(two_thetas, -tan_outer_deg, "g.")
            plt.xlim([0, AD1TF7B_MAX2T])
            plt.ylim([-AD1TF7B_MAXDP, AD1TF7B_MAXDP])
            OO.parent.show_figure(plt, fig, "psi")
            plt.close()

        from xfel.mono_simulation.util import green_curve_area, ewald_proximal_volume
        if OO.mosaic_refinement_target == "ML":
            OO.parent.green_curve_area = green_curve_area(
                two_thetas, tan_outer_deg_ML)
            OO.parent.inputai.setMosaicity(M.x[1] * 180. /
                                           math.pi)  # full width, degrees
            OO.parent.ML_half_mosaicity_deg = M.x[1] * 180. / (2. * math.pi)
            OO.parent.ML_domain_size_ang = 1. / M.x[0]
            OO.parent.ewald_proximal_volume = ewald_proximal_volume(
                wavelength_ang=OO.central_wavelength_ang,
                resolution_cutoff_ang=OO.parent.horizons_phil.integration.
                mosaic.ewald_proximal_volume_resolution_cutoff,
                domain_size_ang=1. / M.x[0],
                full_mosaicity_rad=M.x[1])
            return results, helper.last_set_orientation, 1. / M.x[
                0]  # full width domain size, angstroms
        else:
            assert OO.mosaic_refinement_target == "LSQ"
            OO.parent.green_curve_area = green_curve_area(
                two_thetas, tan_outer_deg)
            OO.parent.inputai.setMosaicity(2 * k_degrees)  # full width
            OO.parent.ML_half_mosaicity_deg = k_degrees
            OO.parent.ML_domain_size_ang = s_ang
            OO.parent.ewald_proximal_volume = ewald_proximal_volume(
                wavelength_ang=OO.central_wavelength_ang,
                resolution_cutoff_ang=OO.parent.horizons_phil.integration.
                mosaic.ewald_proximal_volume_resolution_cutoff,
                domain_size_ang=s_ang,
                full_mosaicity_rad=2 * k_degrees * math.pi / 180.)
            return results, helper.last_set_orientation, s_ang  # full width domain size, angstroms
Example #4
0
  def refine_rotx_roty2(OO,enable_rotational_target=True):

      helper = OO.per_frame_helper_factory()
      helper.restart()

      if enable_rotational_target:
        print "Trying least squares minimization of excursions",
        from scitbx.lstbx import normal_eqns_solving
        iterations = normal_eqns_solving.naive_iterations(
          non_linear_ls = helper,
          gradient_threshold = 1.E-10)

      results =  helper.x

      print "with %d reflections"%len(OO.parent.indexed_pairs),
      print "result %6.2f degrees"%(results[1]*180./math.pi),
      print "result %6.2f degrees"%(results[0]*180./math.pi)

      if False: # Excursion histogram
        print "The input mosaicity is %7.3f deg full width"%OO.parent.inputai.getMosaicity()
        # final histogram
        if OO.pvr_fix:
          final = 360.* helper.fvec_callable_pvr(results)
        else:
          final = 360.* helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results)

        rmsdexc = math.sqrt(flex.mean(final*final))
        from matplotlib import pyplot as plt
        nbins = len(final)//20
        n,bins,patches = plt.hist(final,
          nbins, normed=0, facecolor="orange", alpha=0.75)
        plt.xlabel("Rotation on e1 axis, rmsd %7.3f deg"%rmsdexc)
        plt.title("Histogram of cctbx.xfel misorientation")
        plt.axis([-0.5,0.5,0,100])
        plt.plot([rmsdexc],[18],"b|")
        plt.show()

      # Determine optimal mosaicity and domain size model (monochromatic)
      if OO.pvr_fix:
        final = 360.* helper.fvec_callable_pvr(results)
      else:
        final = 360.* helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results)
      #Guard against misindexing -- seen in simulated data, with zone nearly perfectly aligned
      guard_stats = flex.max(final), flex.min(final)
      if False and REMOVETEST_KILLING_LEGITIMATE_EXCURSIONS (guard_stats[0] > 2.0 or guard_stats[1] < -2.0):
        raise Exception("Misindexing diagnosed by meaningless excursion angle (bandpass_gaussian model)");
      print "The mean excursion is %7.3f degrees"%(flex.mean(final))

      two_thetas = helper.last_set_orientation.unit_cell().two_theta(OO.reserve_indices,OO.central_wavelength_ang,deg=True)
      dspacings = helper.last_set_orientation.unit_cell().d(OO.reserve_indices)
      dspace_sq = dspacings * dspacings
      excursion_rad = final * math.pi/ 180.

      #  First -- try to get a reasonable envelope for the observed excursions.
          ## minimum of three regions; maximum of 50 measurements in each bin
      print "fitting parameters on %d spots"%len(excursion_rad)
      n_bins = min(max(3, len(excursion_rad)//25),50)
      bin_sz = len(excursion_rad)//n_bins
      print "nbins",n_bins,"bin_sz",bin_sz
      order = flex.sort_permutation(two_thetas)
      two_thetas_env = flex.double()
      dspacings_env = flex.double()
      excursion_rads_env = flex.double()
      for x in xrange(0,n_bins):
        subset = order[x*bin_sz:(x+1)*bin_sz]
        two_thetas_env.append( flex.mean(two_thetas.select(subset)) )
        dspacings_env.append( flex.mean(dspacings.select(subset)))
        excursion_rads_env.append( flex.max( flex.abs( excursion_rad.select(subset))))

      #  Second -- parameter fit
          ## solve the normal equations
      sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
      sum_inv_u    = flex.sum(dspacings_env)
      sum_te_u     = flex.sum(dspacings_env * excursion_rads_env)
      sum_te       = flex.sum(excursion_rads_env)
      Normal_Mat   = sqr((sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
      Vector       = col((sum_te_u, sum_te))
      solution     = Normal_Mat.inverse() * Vector
      s_ang = 1./(2*solution[0])
      print "Best LSQ fit Scheerer domain size is %9.2f ang"%(
        s_ang)
      tan_phi_rad = helper.last_set_orientation.unit_cell().d(OO.reserve_indices) / (2. * s_ang)
      tan_phi_deg = tan_phi_rad * 180./math.pi
      k_degrees = solution[1]* 180./math.pi
      print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f"%(2*k_degrees, k_degrees)
      tan_outer_deg = tan_phi_deg + k_degrees

      if OO.mosaic_refinement_target=="ML":
        from xfel.mono_simulation.max_like import minimizer
        print "input", s_ang,2. * solution[1]*180/math.pi
        # coerce the estimates to be positive for max-likelihood
        lower_limit_domain_size = math.pow(
         helper.last_set_orientation.unit_cell().volume(),
         1./3.)*20 # 10-unit cell block size minimum reasonable domain

        d_estimate = max(s_ang, lower_limit_domain_size)
        M = minimizer(d_i = dspacings, psi_i = excursion_rad, eta_rad = abs(2. * solution[1]),
                      Deff = d_estimate)
        print "output",1./M.x[0], M.x[1]*180./math.pi
        tan_phi_rad_ML = helper.last_set_orientation.unit_cell().d(OO.reserve_indices) / (2. / M.x[0])
        tan_phi_deg_ML = tan_phi_rad_ML * 180./math.pi
        # bugfix: Need factor of 0.5 because the plot shows half mosaicity (displacement from the center point defined as zero)
        tan_outer_deg_ML = tan_phi_deg_ML + 0.5*M.x[1]*180./math.pi

      if OO.parent.horizons_phil.integration.mosaic.enable_polychromatic:
        # add code here to perform polychromatic modeling.
        """
        get miller indices DONE
        get model-predicted mono-wavelength centroid S1 vectors
        back-convert S1vec, with mono-wavelength, to detector-plane position, factoring in subpixel correction
        compare with spot centroid measured position
        compare with locus of bodypixels
        """
        print list(OO.reserve_indices)
        print len(OO.reserve_indices), len(two_thetas)
        positions = [
              OO.ucbp3.simple_forward_calculation_spot_position(
              wavelength = OO.central_wavelength_ang,
              observation_no = obsno).position
              for obsno in xrange(len(OO.parent.indexed_pairs))]
        print len(positions)
        print positions # model-predicted positions
        print len(OO.parent.spots)
        print OO.parent.indexed_pairs
        print OO.parent.spots
        print len(OO.parent.spots)
        meas_spots = [OO.parent.spots[pair["spot"]] for pair in OO.parent.indexed_pairs]
  #      for xspot in meas_spots:
  #        xspot.ctr_mass_x(),xspot.ctr_mass_y()
  #        xspot.max_pxl_x()
  #        xspot.bodypixels
  #        xspot.ctr_mass_x()

        # Do some work to calculate an rmsd
        diff_vecs = flex.vec3_double()
        for p,xspot in zip(positions, meas_spots):
          diff_vecs.append((p[0]-xspot.ctr_mass_y(), p[1]-xspot.ctr_mass_x(), 0.0))
        # could use diff_vecs.rms_length()
        diff_vecs_sq = diff_vecs.dot(diff_vecs)
        mean_diff_vec_sq = flex.mean(diff_vecs_sq)
        rmsd = math.sqrt(mean_diff_vec_sq)
        print "mean obs-pred diff vec on %d spots is %6.2f pixels"%(len(positions),rmsd)

        positions_to_fictitious = [
              OO.ucbp3.simple_forward_calculation_spot_position(
              wavelength = OO.central_wavelength_ang,
              observation_no = obsno).position_to_fictitious
              for obsno in xrange(len(OO.parent.indexed_pairs))]
        # Do some work to calculate an rmsd
        diff_vecs = flex.vec3_double()
        for p,xspot in zip(positions_to_fictitious, meas_spots):
          diff_vecs.append((p[0]-xspot.ctr_mass_y(), p[1]-xspot.ctr_mass_x(), 0.0))
        rmsd = diff_vecs.rms_length()
        print "mean obs-pred_to_fictitious diff vec on %d spots is %6.2f pixels"%(len(positions),rmsd)

        """
        actually, it might be better if the entire set of experimental observations
        is transformed into the ideal detector plane, for the purposes of poly_treatment.


        start here.  Now it would be good to actually implement probability of observing a body pixel given the model.
        We have everything needed right here.
        """
        if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B:
          # Image plot: obs and predicted positions + bodypixels
          from matplotlib import pyplot as plt
          plt.plot( [p[0] for p in positions_to_fictitious], [p[1] for p in positions_to_fictitious], "r.")
          plt.plot( [xspot.ctr_mass_y() for xspot in meas_spots],
                    [xspot.ctr_mass_x() for xspot in meas_spots], "g.")
          bodypx = []
          for xspot in meas_spots:
            for body in xspot.bodypixels:
              bodypx.append(body)
          plt.plot( [b.y for b in bodypx], [b.x for b in bodypx], "b.")
          plt.axes().set_aspect("equal")
          plt.show()

      print "MEAN excursion",flex.mean(final),
      if OO.mosaic_refinement_target=="ML":
        print "mosaicity deg FW=",M.x[1]*180./math.pi
      else:
        print
      if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B: # Excursion vs resolution fit
        AD1TF7B_MAX2T = 30.
        AD1TF7B_MAXDP = 1.
        from matplotlib import pyplot as plt
        fig = plt.figure()
        plt.plot(two_thetas, final, "bo")
        mean = flex.mean(final)
        minplot = flex.min(two_thetas)
        plt.plot([0,minplot],[mean,mean],"k-")
        LR = flex.linear_regression(two_thetas, final)
        #LR.show_summary()
        model_y = LR.slope()*two_thetas + LR.y_intercept()
        plt.plot(two_thetas, model_y, "k-")
        print helper.last_set_orientation.unit_cell()
        #for sdp,tw in zip (dspacings,two_thetas):
          #print sdp,tw
        if OO.mosaic_refinement_target=="ML":
          plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas)))
          plt.plot(two_thetas, tan_phi_deg_ML, "r.")
          plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
          plt.plot(two_thetas, tan_outer_deg_ML, "g.")
          plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
        else:
          plt.plot(two_thetas_env, excursion_rads_env *180./math.pi, "r|")
          plt.plot(two_thetas_env, -excursion_rads_env *180./math.pi, "r|")
          plt.plot(two_thetas_env, excursion_rads_env *180./math.pi, "r-")
          plt.plot(two_thetas_env, -excursion_rads_env *180./math.pi, "r-")
          plt.plot(two_thetas, tan_phi_deg, "r.")
          plt.plot(two_thetas, -tan_phi_deg, "r.")
          plt.plot(two_thetas, tan_outer_deg, "g.")
          plt.plot(two_thetas, -tan_outer_deg, "g.")
        plt.xlim([0,AD1TF7B_MAX2T])
        plt.ylim([-AD1TF7B_MAXDP,AD1TF7B_MAXDP])
        OO.parent.show_figure(plt,fig,"psi")
        plt.close()

      from xfel.mono_simulation.util import green_curve_area,ewald_proximal_volume
      if OO.mosaic_refinement_target=="ML":
        OO.parent.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML)
        OO.parent.inputai.setMosaicity(M.x[1]*180./math.pi) # full width, degrees
        OO.parent.ML_half_mosaicity_deg = M.x[1]*180./(2.*math.pi)
        OO.parent.ML_domain_size_ang = 1./M.x[0]
        OO.parent.ewald_proximal_volume = ewald_proximal_volume(
            wavelength_ang = OO.central_wavelength_ang,
            resolution_cutoff_ang = OO.parent.horizons_phil.integration.mosaic.ewald_proximal_volume_resolution_cutoff,
            domain_size_ang = 1./M.x[0],
            full_mosaicity_rad = M.x[1])
        return results, helper.last_set_orientation,1./M.x[0] # full width domain size, angstroms
      else:
        assert OO.mosaic_refinement_target=="LSQ"
        OO.parent.green_curve_area = green_curve_area(two_thetas, tan_outer_deg)
        OO.parent.inputai.setMosaicity(2*k_degrees) # full width
        OO.parent.ML_half_mosaicity_deg = k_degrees
        OO.parent.ML_domain_size_ang = s_ang
        OO.parent.ewald_proximal_volume = ewald_proximal_volume(
            wavelength_ang = OO.central_wavelength_ang,
            resolution_cutoff_ang = OO.parent.horizons_phil.integration.mosaic.ewald_proximal_volume_resolution_cutoff,
            domain_size_ang = s_ang,
            full_mosaicity_rad = 2*k_degrees*math.pi/180.)
        return results, helper.last_set_orientation,s_ang # full width domain size, angstroms
Example #5
0
  def __call__(self):
    """Determine optimal mosaicity and domain size model (monochromatic)"""
    RR = self.refinery.predict_for_reflection_table(self.reflections)
    excursion_rad = RR["delpsical.rad"]
    delta_psi_deg = excursion_rad * 180./math.pi
    print
    print flex.max(delta_psi_deg), flex.min(delta_psi_deg)
    mean_excursion = flex.mean(delta_psi_deg)
    print "The mean excursion is %7.3f degrees, r.m.s.d %7.3f"%(mean_excursion, math.sqrt(flex.mean(RR["delpsical2"])))

    crystal = self.experiments[0].crystal
    beam = self.experiments[0].beam
    miller_indices = self.reflections["miller_index"]

    # FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection
    two_thetas = crystal.get_unit_cell().two_theta(miller_indices,beam.get_wavelength(),deg=True)
    dspacings = crystal.get_unit_cell().d(miller_indices)
    dspace_sq = dspacings * dspacings

    #  First -- try to get a reasonable envelope for the observed excursions.
        ## minimum of three regions; maximum of 50 measurements in each bin
    print "fitting parameters on %d spots"%len(excursion_rad)
    n_bins = min(max(3, len(excursion_rad)//25),50)
    bin_sz = len(excursion_rad)//n_bins
    print "nbins",n_bins,"bin_sz",bin_sz
    order = flex.sort_permutation(two_thetas)
    two_thetas_env = flex.double()
    dspacings_env = flex.double()
    excursion_rads_env = flex.double()
    for x in xrange(0,n_bins):
      subset = order[x*bin_sz:(x+1)*bin_sz]
      two_thetas_env.append(flex.mean(two_thetas.select(subset)))
      dspacings_env.append(flex.mean(dspacings.select(subset)))
      excursion_rads_env.append(flex.max(flex.abs(excursion_rad.select(subset))))

    #  Second -- parameter fit
        ## solve the normal equations
    sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
    sum_inv_u    = flex.sum(dspacings_env)
    sum_te_u     = flex.sum(dspacings_env * excursion_rads_env)
    sum_te       = flex.sum(excursion_rads_env)
    Normal_Mat   = sqr((sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
    Vector       = col((sum_te_u, sum_te))
    solution     = Normal_Mat.inverse() * Vector
    s_ang = 1./(2*solution[0])
    print "Best LSQ fit Scheerer domain size is %9.2f ang"%(
      s_ang)

    tan_phi_rad = dspacings / (2. * s_ang)
    tan_phi_deg = tan_phi_rad * 180./math.pi
    k_degrees = solution[1]* 180./math.pi
    print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f"%(2*k_degrees, k_degrees)
    tan_outer_deg = tan_phi_deg + k_degrees

    from xfel.mono_simulation.max_like import minimizer
    # coerce the estimates to be positive for max-likelihood
    lower_limit_domain_size = math.pow(crystal.get_unit_cell().volume(),
     1./3.)*3 # params.refinement.domain_size_lower_limit

    d_estimate = max(s_ang, lower_limit_domain_size)
    M = minimizer(d_i = dspacings, psi_i = excursion_rad, eta_rad = abs(2. * solution[1]),
                  Deff = d_estimate)
    print "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas))
    tan_phi_rad_ML = dspacings / (2. / M.x[0])
    tan_phi_deg_ML = tan_phi_rad_ML * 180./math.pi
    tan_outer_deg_ML = tan_phi_deg_ML + 0.5*M.x[1]*180./math.pi

    self.nv_acceptance_flags = flex.abs(delta_psi_deg) < tan_outer_deg_ML

    if self.graph_verbose: #params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit
      AD1TF7B_MAX2T = 30.
      AD1TF7B_MAXDP = 1.
      from matplotlib import pyplot as plt
      plt.plot(two_thetas, delta_psi_deg, "bo")
      minplot = flex.min(two_thetas)
      plt.plot([0,minplot],[mean_excursion,mean_excursion],"k-")
      LR = flex.linear_regression(two_thetas, delta_psi_deg)
      model_y = LR.slope()*two_thetas + LR.y_intercept()
      plt.plot(two_thetas, model_y, "k-")

      plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas)))
      plt.plot(two_thetas, tan_phi_deg_ML, "r.")
      plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
      plt.plot(two_thetas, tan_outer_deg_ML, "g.")
      plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
      plt.xlim([0,AD1TF7B_MAX2T])
      plt.ylim([-AD1TF7B_MAXDP,AD1TF7B_MAXDP])
      plt.show()
      plt.close()

    from xfel.mono_simulation.util import green_curve_area
    self.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML)
    print "The green curve area is ", self.green_curve_area

    crystal._ML_half_mosaicity_deg = M.x[1]*180./(2.*math.pi)
    crystal._ML_domain_size_ang = 2./M.x[0]
    self._ML_full_mosaicity_rad = M.x[1]
    self._ML_domain_size_ang = 2./M.x[0]

    #params.refinement.mosaic.model_expansion_factor
    """The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes
    as close as possible to # of observed reflections input, in the last integration call.  Determine this by
    inspecting the output log file interactively.  Do not exceed the bare minimum threshold needed.
    The intention is to find an optimal value, global for a given dataset."""
    model_expansion_factor = 1.4
    crystal._ML_half_mosaicity_deg *= model_expansion_factor
    crystal._ML_domain_size_ang /= model_expansion_factor

    return crystal