Exemple #1
0
    def setup_binner(
        self,
        unit_cell: uctbx.unit_cell,
        space_group: sgtbx.space_group,
        n_resolution_bins: int,
    ) -> None:
        """Create a binner for the reflections contained in the table."""
        ma = _reflection_table_to_iobs(self.as_reflection_table(), unit_cell,
                                       space_group)
        # need d star sq step
        d_star_sq = ma.d_star_sq().data()
        d_star_sq_min = flex.min(d_star_sq)
        d_star_sq_max = flex.max(d_star_sq)
        span = d_star_sq_max - d_star_sq_min
        relative_tolerance = 1e-6
        d_star_sq_max += span * relative_tolerance
        d_star_sq_min -= span * relative_tolerance
        # Avoid a zero-size step that would otherwise anger the d_star_sq_step binner.
        step = max((d_star_sq_max - d_star_sq_min) / n_resolution_bins, 0.004)

        self.binner = ma.setup_binner_d_star_sq_step(
            auto_binning=False,
            d_max=uctbx.d_star_sq_as_d(d_star_sq_max),
            d_min=uctbx.d_star_sq_as_d(d_star_sq_min),
            d_star_sq_step=step,
        )
Exemple #2
0
def d_star_sq_to_d_ticks(d_star_sq, nticks):
    from cctbx import uctbx
    d_spacings = uctbx.d_star_sq_as_d(flex.double(d_star_sq))
    min_d_star_sq = min(d_star_sq)
    dstep = (max(d_star_sq) - min_d_star_sq) / nticks
    tickvals = list(min_d_star_sq + (i * dstep) for i in range(nticks))
    ticktext = ['%.2f' % (uctbx.d_star_sq_as_d(dsq)) for dsq in tickvals]
    return tickvals, ticktext
Exemple #3
0
 def d_star_sq_to_d_ticks(d_star_sq, nticks):
   from cctbx import uctbx
   d_spacings = uctbx.d_star_sq_as_d(flex.double(d_star_sq))
   min_d_star_sq = min(d_star_sq)
   dstep = (max(d_star_sq) - min_d_star_sq)/nticks
   tickvals = list(min_d_star_sq + (i*dstep) for i in range(nticks))
   ticktext = ['%.2f' %(uctbx.d_star_sq_as_d(dsq)) for dsq in tickvals]
   return tickvals, ticktext
Exemple #4
0
    def count_ice_rings(self, width=0.002, verbose=False):
        """ A function to find and count ice rings (modeled after
        dials.algorithms.integration.filtering.PowderRingFilter, with some alterations:
            1. Hard-coded with ice unit cell / space group
            2. Returns spot counts vs. water diffraction resolution "bin"

        Rather than finding ice rings themselves (which may be laborious and time
        consuming), this method relies on counting the number of found spots that land
        in regions of water diffraction. A weakness of this approach is that if any spot
        filtering or spot-finding parameters are applied by prior methods, not all ice
        rings may be found. This is acceptable, since the purpose of this method is to
        determine if water and protein diffraction occupy the same resolutions.
        """
        ice_start = time.time()

        unit_cell = uctbx.unit_cell((4.498, 4.498, 7.338, 90, 90, 120))
        space_group = sgtbx.space_group_info(number=194).group()

        # Correct unit cell
        unit_cell = space_group.average_unit_cell(unit_cell)

        half_width = width / 2
        d_min = uctbx.d_star_sq_as_d(uctbx.d_as_d_star_sq(self.d_min) + half_width)

        # Generate a load of indices
        generator = index_generator(unit_cell, space_group.type(), False, d_min)
        indices = generator.to_array()

        # Compute d spacings and sort by resolution
        d_star_sq = flex.sorted(unit_cell.d_star_sq(indices))
        d = uctbx.d_star_sq_as_d(d_star_sq)
        dd = list(zip(d_star_sq, d))

        # identify if spots fall within ice ring areas
        results = []
        for ds2, d_res in dd:
            result = [i for i in (flex.abs(self.ref_d_star_sq - ds2) < half_width) if i]
            results.append((d_res, len(result)))

        possible_ice = [r for r in results if r[1] / len(self.observed) * 100 >= 5]

        if verbose:
            print(
                "SCORER: ice ring time = {:.5f} seconds".format(time.time() - ice_start)
            )

        self.n_ice_rings = len(possible_ice)  # output in info

        return self.n_ice_rings
Exemple #5
0
def resolution_histogram(reflections, imageset, plot_filename=None):
    d_star_sq = flex.pow2(reflections['rlp'].norms())
    hist = get_histogram(d_star_sq)

    if plot_filename is not None:
        if pyplot is None:
            raise Sorry("matplotlib must be installed to generate a plot.")
        fig = pyplot.figure()
        ax = fig.add_subplot(1, 1, 1)
        ax.bar(hist.slot_centers() - 0.5 * hist.slot_width(),
               hist.slots(),
               width=hist.slot_width())
        ax.set_xlabel("d_star_sq")
        ax.set_ylabel("Frequency")

        ax_ = ax.twiny()  # ax2 is responsible for "top" axis and "right" axis
        xticks = ax.get_xticks()
        xlim = ax.get_xlim()
        xticks_d = [
            uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
        ]
        xticks_ = [ds2 / (xlim[1] - xlim[0]) for ds2 in xticks]
        ax_.set_xticks(xticks)
        ax_.set_xlim(ax.get_xlim())
        ax_.set_xlabel(r"Resolution ($\AA$)")
        ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
        #pyplot.show()
        pyplot.savefig(plot_filename)
        pyplot.close()
def resolution_histogram(reflections, imageset, plot_filename=None):
    d_star_sq = flex.pow2(reflections["rlp"].norms())
    hist = get_histogram(d_star_sq)

    if plot_filename is not None:
        from matplotlib import pyplot

        fig = pyplot.figure()
        ax = fig.add_subplot(1, 1, 1)
        ax.bar(
            hist.slot_centers() - 0.5 * hist.slot_width(),
            hist.slots(),
            width=hist.slot_width(),
        )
        ax.set_xlabel("d_star_sq")
        ax.set_ylabel("Frequency")

        ax_ = ax.twiny()  # ax2 is responsible for "top" axis and "right" axis
        xticks = ax.get_xticks()
        xticks_d = [
            uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
        ]
        ax_.set_xticks(xticks)
        ax_.set_xlim(ax.get_xlim())
        ax_.set_xlabel(r"Resolution ($\AA$)")
        ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
        pyplot.savefig(plot_filename)
        pyplot.close()
def stats_single_image(imageset, reflections, i=None, resolution_analysis=True,
                       plot=False):
  reflections = map_to_reciprocal_space(reflections, imageset)
  if plot and i is not None:
    filename = "i_over_sigi_vs_resolution_%d.png" %(i+1)
    hist_filename = "spot_count_vs_resolution_%d.png" %(i+1)
    extra_filename = "log_sum_i_sigi_vs_resolution_%d.png" %(i+1)
    distl_method_1_filename = "distl_method_1_%d.png" %(i+1)
    distl_method_2_filename = "distl_method_2_%d.png" %(i+1)
  else:
    filename = None
    hist_filename = None
    extra_filename = None
    distl_method_1_filename = None
    distl_method_2_filename = None

  d_star_sq = flex.pow2(reflections['rlp'].norms())
  d_spacings = uctbx.d_star_sq_as_d(d_star_sq)

  #plot_ordered_d_star_sq(reflections, imageset)
  reflections_all = reflections
  ice_sel = ice_rings_selection(reflections_all)
  reflections_no_ice = reflections_all.select(~ice_sel)
  n_spots_total = len(reflections_all)
  n_spots_no_ice = len(reflections_no_ice)
  n_spot_4A = (d_spacings > 4).count(True)
  intensities = reflections_no_ice['intensity.sum.value']
  total_intensity = flex.sum(intensities)
  #print i
  if hist_filename is not None:
    resolution_histogram(
      reflections, imageset, plot_filename=hist_filename)
  if extra_filename is not None:
    log_sum_i_sigi_vs_resolution(
      reflections, imageset, plot_filename=extra_filename)
  if resolution_analysis and n_spots_no_ice > 10:
    estimated_d_min = estimate_resolution_limit(
      reflections_all, imageset, ice_sel=ice_sel, plot_filename=filename)
    d_min_distl_method_1, noisiness_method_1 \
      = estimate_resolution_limit_distl_method1(
        reflections_all, imageset, ice_sel, plot_filename=distl_method_1_filename)
    d_min_distl_method_2, noisiness_method_2 = \
      estimate_resolution_limit_distl_method2(
        reflections_all, imageset, ice_sel, plot_filename=distl_method_2_filename)
  else:
    estimated_d_min = -1.0
    d_min_distl_method_1 = -1.0
    noisiness_method_1 = -1.0
    d_min_distl_method_2 = -1.0
    noisiness_method_2 = -1.0

  return group_args(n_spots_total=n_spots_total,
                    n_spots_no_ice=n_spots_no_ice,
                    n_spots_4A=n_spot_4A,
                    total_intensity=total_intensity,
                    estimated_d_min=estimated_d_min,
                    d_min_distl_method_1=d_min_distl_method_1,
                    noisiness_method_1=noisiness_method_1,
                    d_min_distl_method_2=d_min_distl_method_2,
                    noisiness_method_2=noisiness_method_2)
def resolution_histogram(reflections, imageset, plot_filename=None):
  d_star_sq = flex.pow2(reflections['rlp'].norms())
  hist = get_histogram(d_star_sq)

  if plot_filename is not None:
    if pyplot is None:
      raise Sorry("matplotlib must be installed to generate a plot.")
    fig = pyplot.figure()
    ax = fig.add_subplot(1,1,1)
    ax.bar(hist.slot_centers()-0.5*hist.slot_width(), hist.slots(),
               width=hist.slot_width())
    ax.set_xlabel("d_star_sq")
    ax.set_ylabel("Frequency")

    ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
    xticks = ax.get_xticks()
    xlim = ax.get_xlim()
    xticks_d = [
      uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
    xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
    ax_.set_xticks(xticks)
    ax_.set_xlim(ax.get_xlim())
    ax_.set_xlabel(r"Resolution ($\AA$)")
    ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
    #pyplot.show()
    pyplot.savefig(plot_filename)
    pyplot.close()
Exemple #9
0
    def __init__(self, unit_cell, space_group, d_min, width):
        """
        Initialise the filter.

        :param unit_cell: The unit_cell of the powder rings
        :param space_group: The space group of the powder rings
        :param d_min: The maximum resolution to filter to
        :param width: The resolution width to filter around
        """
        assert d_min > 0
        assert width > 0

        # Correct unit cell
        unit_cell = space_group.average_unit_cell(unit_cell)

        self.half_width = width / 2.0
        d_min = uctbx.d_star_sq_as_d(
            uctbx.d_as_d_star_sq(d_min) + self.half_width)

        # Generate a load of indices
        generator = index_generator(unit_cell, space_group.type(), False,
                                    d_min)
        indices = generator.to_array()

        # Compute d spacings and sort by resolution
        self.d_star_sq = flex.sorted(unit_cell.d_star_sq(indices))
Exemple #10
0
def make_merging_stats_plots(script):
    """Make merging stats plots for HTML report"""
    d = {
        "scaling_tables": ([], []),
        "resolution_plots": {},
        "batch_plots": {},
        "misc_plots": {},
        "anom_plots": {},
        "image_range_tables": [],
    }
    (
        batches,
        rvb,
        isigivb,
        svb,
        batch_data,
    ) = reflection_tables_to_batch_dependent_properties(  # pylint: disable=unbalanced-tuple-unpacking
        script.reflections,
        script.experiments,
        script.scaled_miller_array,
    )
    bm = batch_manager(batches, batch_data)
    image_range_tables = make_image_range_table(script.experiments, bm)

    if script.merging_statistics_result:
        stats = script.merging_statistics_result
        anom_stats = script.anom_merging_statistics_result
        is_centric = script.scaled_miller_array.space_group().is_centric()
        # Now calculate batch data
        plotter = ResolutionPlotsAndStats(stats, anom_stats, is_centric)
        d["resolution_plots"].update(plotter.make_all_plots())
        d["scaling_tables"] = plotter.statistics_tables()
        d["batch_plots"].update(scale_rmerge_vs_batch_plot(bm, rvb, svb))
        d["batch_plots"].update(i_over_sig_i_vs_batch_plot(bm, isigivb))
        plotter = IntensityStatisticsPlots(
            script.scaled_miller_array, run_xtriage_analysis=False
        )
        d["resolution_plots"].update(plotter.generate_resolution_dependent_plots())
        if d["resolution_plots"]["cc_one_half"]["data"][2]:
            cc_anom = d["resolution_plots"]["cc_one_half"]["data"][2]["y"]
            significance = d["resolution_plots"]["cc_one_half"]["data"][3]["y"]
            sig = flex.double(cc_anom) > flex.double(significance)
            max_anom = 0
            for i, v in enumerate(sig):
                if v:
                    max_anom = i
                else:
                    break
            d_min = uctbx.d_star_sq_as_d(plotter.binner.limits())[max_anom + 1]
        else:
            d_min = 0.0
        d["misc_plots"].update(plotter.generate_miscellanous_plots())
        intensities_anom = script.scaled_miller_array.as_anomalous_array()
        intensities_anom = intensities_anom.map_to_asu().customized_copy(
            info=script.scaled_miller_array.info()
        )
        anom_plotter = AnomalousPlotter(intensities_anom, strong_cutoff=d_min)
        d["anom_plots"].update(anom_plotter.make_plots())
    d["image_range_tables"] = [image_range_tables]
    return d
def estimate_resolution_limit_distl_method2(reflections, plot_filename=None):
    # Implementation of Method 2 (section 2.4.4) of:
    # Z. Zhang, N. K. Sauter, H. van den Bedem, G. Snell and A. M. Deacon
    # J. Appl. Cryst. (2006). 39, 112-119
    # https://doi.org/10.1107/S0021889805040677

    variances = reflections["intensity.sum.variance"]

    sel = variances > 0
    reflections = reflections.select(sel)
    d_star_sq = flex.pow2(reflections["rlp"].norms())
    d_spacings = uctbx.d_star_sq_as_d(d_star_sq)

    binner = binner_d_star_cubed(d_spacings)

    bin_counts = flex.size_t()

    for i_slot, slot in enumerate(binner.bins):
        sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
        sel = sel_all

        bin_counts.append(sel.count(True))

    # print list(bin_counts)
    t0 = (bin_counts[0] + bin_counts[1]) / 2

    mu = 0.15

    for i in range(len(bin_counts) - 1):
        tj = bin_counts[i]
        tj1 = bin_counts[i + 1]
        if (tj < (mu * t0)) and (tj1 < (mu * t0)):
            break

    d_min = binner.bins[i].d_min
    noisiness = 0
    m = len(bin_counts)
    for i in range(m):
        for j in range(i + 1, m):
            if bin_counts[i] <= bin_counts[j]:
                noisiness += 1
    noisiness /= 0.5 * m * (m - 1)

    if plot_filename is not None:
        from matplotlib import pyplot

        fig = pyplot.figure()
        ax = fig.add_subplot(1, 1, 1)
        ax.scatter(range(len(bin_counts)), bin_counts)
        ax.set_ylabel("number of spots in shell")
        xlim = pyplot.xlim()
        ylim = pyplot.ylim()
        ax.vlines(i, ylim[0], ylim[1], colors="red")
        pyplot.xlim(0, xlim[1])
        pyplot.ylim(0, ylim[1])
        pyplot.savefig(plot_filename)
        pyplot.close()

    return d_min, noisiness
Exemple #12
0
def test_powder_ring_filter():
    unit_cell = uctbx.unit_cell((4.498, 4.498, 7.338, 90, 90, 120))
    space_group = sgtbx.space_group_info(number=194).group()
    d_spacings = flex.double([1.9220704466392748])
    d_min = flex.min(d_spacings)
    ice_filter = filtering.PowderRingFilter(unit_cell, space_group, d_min, width=0.004)
    assert min(uctbx.d_star_sq_as_d(ice_filter.d_star_sq)) < d_min
    assert all(ice_filter(d_spacings))
Exemple #13
0
    def filter_by_resolution(self, refl, d_min, d_max):
        d_min = float(d_min) if d_min is not None else 0.1
        d_max = float(d_max) if d_max is not None else 99

        d_star_sq = flex.pow2(refl["rlp"].norms())
        d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
        filter = flex.bool(len(d_spacings), False)
        filter = filter | (d_spacings >= d_min) & (d_spacings <= d_max)
        return filter
Exemple #14
0
    def setup_binner(self, unit_cell, space_group, n_resolution_bins):
        ma = _reflection_table_to_iobs(self.Ih_table, unit_cell, space_group)
        # need d star sq step
        d_star_sq = ma.d_star_sq().data()
        d_star_sq_min = flex.min(d_star_sq)
        d_star_sq_max = flex.max(d_star_sq)
        span = d_star_sq_max - d_star_sq_min
        relative_tolerance = 1e-6
        d_star_sq_max += span * relative_tolerance
        d_star_sq_min -= span * relative_tolerance
        step = (d_star_sq_max - d_star_sq_min) / n_resolution_bins

        self.binner = ma.setup_binner_d_star_sq_step(
            auto_binning=False,
            d_max=uctbx.d_star_sq_as_d(d_star_sq_max),
            d_min=uctbx.d_star_sq_as_d(d_star_sq_min),
            d_star_sq_step=step,
        )
Exemple #15
0
 def make_plots(self):
     """Generate tables of overall and resolution-binned merging statistics."""
     d = {
         "scaling_tables": ([], []),
         "resolution_plots": OrderedDict(),
         "batch_plots": OrderedDict(),
         "misc_plots": OrderedDict(),
         "anom_plots": OrderedDict(),
         "image_range_tables": [],
     }
     if "statistics" in self.data:
         plotter = ResolutionPlotsAndStats(
             self.data["statistics"],
             self.data["anomalous_statistics"],
             is_centric=self.data["is_centric"],
         )
         d["resolution_plots"].update(plotter.make_all_plots())
         d["scaling_tables"] = plotter.statistics_tables()
         d["batch_plots"].update(
             scale_rmerge_vs_batch_plot(
                 self.data["bm"],
                 self.data["r_merge_vs_batch"],
                 self.data["scale_vs_batch"],
             ))
         d["batch_plots"].update(
             i_over_sig_i_vs_batch_plot(self.data["bm"],
                                        self.data["isigivsbatch"]))
         plotter = IntensityStatisticsPlots(
             self.data["scaled_miller_array"], run_xtriage_analysis=False)
         d["resolution_plots"].update(
             plotter.generate_resolution_dependent_plots())
         if d["resolution_plots"]["cc_one_half"]["data"][2]:
             cc_anom = d["resolution_plots"]["cc_one_half"]["data"][2]["y"]
             significance = d["resolution_plots"]["cc_one_half"]["data"][3][
                 "y"]
             sig = flex.double(cc_anom) > flex.double(significance)
             max_anom = 0
             for i, v in enumerate(sig):
                 if v:
                     max_anom = i
                 else:
                     break
             d_min = uctbx.d_star_sq_as_d(
                 plotter.binner.limits())[max_anom + 1]
         else:
             d_min = 0.0
         d["misc_plots"].update(plotter.generate_miscellanous_plots())
         intensities_anom = self.data[
             "scaled_miller_array"].as_anomalous_array()
         intensities_anom = intensities_anom.map_to_asu().customized_copy(
             info=self.data["scaled_miller_array"].info())
         anom_plotter = AnomalousPlotter(intensities_anom,
                                         strong_cutoff=d_min)
         d["anom_plots"].update(anom_plotter.make_plots())
         d["image_range_tables"] = self.data["image_range_tables"]
     return d
Exemple #16
0
 def savefig(self, filename):
   from cctbx import uctbx
   xticks = self.ax.get_xticks()
   xticks_d = [
     '%.2f' %uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks]
   self.ax.set_xticklabels(xticks_d)
   self.ax.set_xlabel('Resolution (A)')
   self.ax.set_ylabel(self.ylabel)
   self.ax.legend(loc='best')
   self.fig.savefig(filename)
Exemple #17
0
 def savefig(self, filename):
   from cctbx import uctbx
   xticks = self.ax.get_xticks()
   xticks_d = [
     '%.2f' %uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks]
   self.ax.set_xticklabels(xticks_d)
   self.ax.set_xlabel('Resolution (A)')
   self.ax.set_ylabel(self.ylabel)
   self.ax.legend(loc='best')
   self.fig.savefig(filename)
def resolution_fit(d_star_sq, y_obs, model, limit, sel=None):
    """Estimate a resolution limit based on the input merging statistics

    The function defined by `model` will be fit to the input `d_star_sq` and `y_obs`.
    The estimated resolution limit is chosen as the `d_star_sq` value at which the
    fitted function equals `limit`.

    Args:
        d_star_sq (scitbx.array_family.flex.double): The high resolution limits of the
            resolution bins in units 1/d*2
        y_obs (scitbx.array_family.flex.double): The statistic against which to fit the
            function `model`
        model: The function to fit against `y_obs`. Must be callable, taking as input x
            (d_star_sq) and y (the metric to be fitted) values, returning the fitted
            y(x) values.
        limit (float): The resolution limit criterion.
        sel (scitbx.array_family.flex.bool): An optional selection to apply to the
            `d_star_sq` and `y_obs` values.

    Returns: The estimated resolution limit in units of Å^-1

    Raises:
        RuntimeError: Raised if no `y_obs` values remain after application of the
        selection `sel`
    """
    if not sel:
        sel = flex.bool(len(d_star_sq), True)
    sel &= y_obs > 0
    y_obs = y_obs.select(sel)
    d_star_sq = d_star_sq.select(sel)

    if not len(y_obs):
        raise RuntimeError("No reflections left for fitting")
    y_fit = model(d_star_sq, y_obs, 6)
    logger.debug(
        tabulate(
            [("d*2", "d", "obs", "fit")]
            + [
                (ds2, uctbx.d_star_sq_as_d(ds2), yo, yf)
                for ds2, yo, yf in zip(d_star_sq, y_obs, y_fit)
            ],
            headers="firstrow",
        )
    )

    if flex.min(y_obs) > limit:
        d_min = 1.0 / math.sqrt(flex.max(d_star_sq))
    else:
        try:
            d_min = 1.0 / math.sqrt(interpolate_value(d_star_sq, y_fit, limit))
        except RuntimeError as e:
            logger.debug(f"Error interpolating value: {e}")
            d_min = None

    return ResolutionResult(d_star_sq, y_obs, y_fit, d_min)
Exemple #19
0
    def quasi_normalisation(intensities):
        """Quasi-normalisation of the input intensities.

        Args:
          intensities (cctbx.miller.array): The intensities to be normalised.

        Returns:
          cctbx.miller.array: The normalised intensities.
        """
        # handle negative reflections to minimise effect on mean I values.
        work = intensities.deep_copy()
        work.data().set_selected(work.data() < 0.0, 0.0)

        # set up binning objects
        if work.size() > 20000:
            n_refl_shells = 20
        elif work.size() > 15000:
            n_refl_shells = 15
        else:
            n_refl_shells = 10
        d_star_sq = work.d_star_sq().data()
        d_star_sq_max = flex.max(d_star_sq)
        d_star_sq_min = flex.min(d_star_sq)
        span = d_star_sq_max - d_star_sq_min
        d_star_sq_max += span * 1e-6
        d_star_sq_min -= span * 1e-6
        d_star_sq_step = (d_star_sq_max - d_star_sq_min) / n_refl_shells
        work.setup_binner_d_star_sq_step(
            d_min=uctbx.d_star_sq_as_d(
                d_star_sq_min),  # cctbx/cctbx_project#588
            d_max=uctbx.d_star_sq_as_d(
                d_star_sq_max),  # cctbx/cctbx_project#588
            d_star_sq_step=d_star_sq_step,
            auto_binning=False,
        )
        normalisations = work.intensity_quasi_normalisations()
        return intensities.customized_copy(
            data=(intensities.data() / normalisations.data()),
            sigmas=(intensities.sigmas() / normalisations.data()),
        )
def stats_for_reflection_table(reflections,
                               resolution_analysis=True,
                               filter_ice=True,
                               ice_rings_width=0.004):
    assert "rlp" in reflections, "Reflections must have been mapped to reciprocal space"
    reflections = reflections.select(reflections["rlp"].norms() > 0)

    d_star_sq = flex.pow2(reflections["rlp"].norms())
    d_spacings = uctbx.d_star_sq_as_d(d_star_sq)

    reflections_all = reflections
    reflections_no_ice = reflections_all
    ice_sel = None
    if reflections.size() and filter_ice:
        ice_sel = ice_rings_selection(reflections_all, width=ice_rings_width)
        if ice_sel is not None:
            reflections_no_ice = reflections_all.select(~ice_sel)
    n_spots_total = len(reflections_all)
    n_spots_no_ice = len(reflections_no_ice)
    n_spot_4A = (d_spacings > 4).count(True)
    intensities = reflections_no_ice["intensity.sum.value"]
    total_intensity = flex.sum(intensities)
    if resolution_analysis and n_spots_no_ice > 10:
        estimated_d_min = estimate_resolution_limit(reflections_all,
                                                    ice_sel=ice_sel)
        (
            d_min_distl_method_1,
            noisiness_method_1,
        ) = estimate_resolution_limit_distl_method1(reflections_all)
        (
            d_min_distl_method_2,
            noisiness_method_2,
        ) = estimate_resolution_limit_distl_method2(reflections_all)
    else:
        estimated_d_min = -1.0
        d_min_distl_method_1 = -1.0
        noisiness_method_1 = -1.0
        d_min_distl_method_2 = -1.0
        noisiness_method_2 = -1.0

    return StatsSingleImage(
        n_spots_total=n_spots_total,
        n_spots_no_ice=n_spots_no_ice,
        n_spots_4A=n_spot_4A,
        total_intensity=total_intensity,
        estimated_d_min=estimated_d_min,
        d_min_distl_method_1=d_min_distl_method_1,
        noisiness_method_1=noisiness_method_1,
        d_min_distl_method_2=d_min_distl_method_2,
        noisiness_method_2=noisiness_method_2,
    )
Exemple #21
0
def log_sum_i_sigi_vs_resolution(reflections, imageset, plot_filename=None):
    d_star_sq = flex.pow2(reflections['rlp'].norms())
    hist = get_histogram(d_star_sq)

    intensities = reflections['intensity.sum.value']
    variances = reflections['intensity.sum.variance']

    sel = variances > 0
    intensities = intensities.select(sel)
    variances = intensities.select(sel)

    i_over_sigi = intensities / flex.sqrt(variances)
    #log_i_over_sigi = flex.log(i_over_sigi)

    slots = []
    for slot in hist.slot_infos():
        sel = (d_star_sq > slot.low_cutoff) & (d_star_sq < slot.high_cutoff)
        if sel.count(True) > 0:
            slots.append(math.log(flex.sum(i_over_sigi.select(sel))))
        else:
            slots.append(0)

    if plot_filename is not None:
        if pyplot is None:
            raise Sorry("matplotlib must be installed to generate a plot.")
        fig = pyplot.figure()
        ax = fig.add_subplot(1, 1, 1)
        #ax.bar(hist.slot_centers()-0.5*hist.slot_width(), hist.slots(),
        ax.scatter(hist.slot_centers() - 0.5 * hist.slot_width(),
                   slots,
                   s=20,
                   color='blue',
                   marker='o',
                   alpha=0.5)
        ax.set_xlabel("d_star_sq")
        ax.set_ylabel("ln(sum(I/sigI))")

        ax_ = ax.twiny()  # ax2 is responsible for "top" axis and "right" axis
        xticks = ax.get_xticks()
        xlim = ax.get_xlim()
        xticks_d = [
            uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
        ]
        xticks_ = [ds2 / (xlim[1] - xlim[0]) for ds2 in xticks]
        ax_.set_xticks(xticks)
        ax_.set_xlim(ax.get_xlim())
        ax_.set_xlabel(r"Resolution ($\AA$)")
        ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
        #pyplot.show()
        pyplot.savefig(plot_filename)
        pyplot.close()
def ice_rings_selection(reflections):
  d_star_sq = flex.pow2(reflections['rlp'].norms())
  d_spacings = uctbx.d_star_sq_as_d(d_star_sq)

  from dials.algorithms.integration import filtering

  unit_cell = uctbx.unit_cell((4.498,4.498,7.338,90,90,120))
  space_group = sgtbx.space_group_info(number=194).group()
  width = 0.06

  ice_filter = filtering.PowderRingFilter(
    unit_cell, space_group, flex.min(d_spacings)-width, width)

  ice_sel = ice_filter(d_spacings)
  return ice_sel
Exemple #23
0
 def __init__(self, d_star_sq, target_n_per_bin=20, max_slots=20, min_slots=5):
     n_slots = len(d_star_sq) // target_n_per_bin
     if max_slots is not None:
         n_slots = min(n_slots, max_slots)
     if min_slots is not None:
         n_slots = max(n_slots, min_slots)
     self.bins = []
     n_per_bin = len(d_star_sq) / n_slots
     d_star_sq_sorted = flex.sorted(d_star_sq)
     d_sorted = uctbx.d_star_sq_as_d(d_star_sq_sorted)
     d_max = d_sorted[0]
     for i in range(n_slots):
         d_min = d_sorted[nint((i + 1) * n_per_bin) - 1]
         self.bins.append(Slot(d_min, d_max))
         d_max = d_min
Exemple #24
0
 def update_dvals(self):
   refls = self.refls_pruned
   if not self.centroid_px_mm_done:
     refls.centroid_px_to_mm(self.experiments)
     self.centroid_px_mm_done = True
   refls.map_centroids_to_reciprocal_space(self.experiments)
   d_star_sq = flex.pow2(refls['rlp'].norms())
   refls['d'] = uctbx.d_star_sq_as_d(d_star_sq)
   sel_lt = refls['d'] < self.params.center_scan.d_max
   sel_gt = refls['d'] > self.params.center_scan.d_min
   sel = sel_lt & sel_gt
   count = sel.count(True)
   if count > self.target_refl_count:
     self.target_refl_count = count
   self.dvals = refls.select(sel)['d']
def ice_rings_selection(reflections, width=0.004):
    d_star_sq = flex.pow2(reflections["rlp"].norms())
    d_spacings = uctbx.d_star_sq_as_d(d_star_sq)

    unit_cell = uctbx.unit_cell((4.498, 4.498, 7.338, 90, 90, 120))
    space_group = sgtbx.space_group_info(number=194).group()

    if d_spacings:
        ice_filter = filtering.PowderRingFilter(unit_cell, space_group,
                                                flex.min(d_spacings), width)

        ice_sel = ice_filter(d_spacings)

        return ice_sel
    else:
        return None
 def __init__(self, d_star_sq, target_n_per_bin=20, max_slots=20, min_slots=5):
   from libtbx.math_utils import nearest_integer as nint
   n_slots = len(d_star_sq)//target_n_per_bin
   if max_slots is not None:
     n_slots = min(n_slots, max_slots)
   if min_slots is not None:
     n_slots = max(n_slots, min_slots)
   self.bins = []
   n_per_bin = len(d_star_sq)/n_slots
   d_star_sq_sorted = flex.sorted(d_star_sq)
   d_sorted = uctbx.d_star_sq_as_d(d_star_sq_sorted)
   d_max = d_sorted[0]
   for i in range(n_slots):
     d_min = d_sorted[nint((i+1)*n_per_bin)-1]
     self.bins.append(slot(d_min, d_max))
     d_max = d_min
Exemple #27
0
def _filter_by_resolution(experiments, reflections, d_min=None, d_max=None):
    reflections.centroid_px_to_mm(experiments)
    reflections.map_centroids_to_reciprocal_space(experiments)
    d_star_sq = flex.pow2(reflections["rlp"].norms())
    reflections["d"] = uctbx.d_star_sq_as_d(d_star_sq)
    # Filter based on resolution
    if d_min is not None:
        selection = reflections["d"] >= d_min
        reflections = reflections.select(selection)
        logger.debug(f"Selected {len(reflections)} reflections with d >= {d_min:f}")

    # Filter based on resolution
    if d_max is not None:
        selection = reflections["d"] <= d_max
        reflections = reflections.select(selection)
        logger.debug(f"Selected {len(reflections)} reflections with d <= {d_max:f}")
    return reflections
def log_sum_i_sigi_vs_resolution(reflections, imageset, plot_filename=None):
  d_star_sq = flex.pow2(reflections['rlp'].norms())
  hist = get_histogram(d_star_sq)

  intensities = reflections['intensity.sum.value']
  variances = reflections['intensity.sum.variance']

  sel = variances > 0
  intensities = intensities.select(sel)
  variances = intensities.select(sel)

  i_over_sigi = intensities/flex.sqrt(variances)
  #log_i_over_sigi = flex.log(i_over_sigi)

  slots = []
  for slot in hist.slot_infos():
    sel = (d_star_sq > slot.low_cutoff) & (d_star_sq < slot.high_cutoff)
    if sel.count(True) > 0:
      slots.append(math.log(flex.sum(i_over_sigi.select(sel))))
    else:
      slots.append(0)

  if plot_filename is not None:
    if pyplot is None:
      raise Sorry("matplotlib must be installed to generate a plot.")
    fig = pyplot.figure()
    ax = fig.add_subplot(1,1,1)
    #ax.bar(hist.slot_centers()-0.5*hist.slot_width(), hist.slots(),
    ax.scatter(hist.slot_centers()-0.5*hist.slot_width(), slots, s=20, color='blue', marker='o', alpha=0.5)
    ax.set_xlabel("d_star_sq")
    ax.set_ylabel("ln(sum(I/sigI))")

    ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
    xticks = ax.get_xticks()
    xlim = ax.get_xlim()
    xticks_d = [
      uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
    xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
    ax_.set_xticks(xticks)
    ax_.set_xlim(ax.get_xlim())
    ax_.set_xlabel(r"Resolution ($\AA$)")
    ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
    #pyplot.show()
    pyplot.savefig(plot_filename)
    pyplot.close()
def plot_merging_stats(results, labels=None, plots=None, prefix=None,
                       size_inches=None, image_dir=None):
  import matplotlib
  matplotlib.use('Agg')
  from matplotlib import pyplot
  pyplot.style.use('ggplot')

  if plots is None:
    plots = ('r_merge', 'r_meas', 'r_pim', 'cc_one_half', 'i_over_sigma_mean')
  if prefix is None:
    prefix = ''
  if labels is not None:
    assert len(results) == len(labels)
  if image_dir is None:
    image_dir = '.'
  for k in plots:
    for i, result in enumerate(results):
      if labels is not None:
        label = labels[i]
      else:
        label = None
      bins = result.bins
      x = [bins[i].d_min for i in range(len(bins))]
      x = [uctbx.d_as_d_star_sq(d) for d in x]
      y = [getattr(bins[i], k) for i in range(len(bins))]
      pyplot.plot(x, y, label=label)
    pyplot.xlabel('d spacing')
    pyplot.ylabel(k)
    if k == 'cc_one_half':
      pyplot.ylim(0, 1.05)
    else:
      pyplot.ylim(0, pyplot.ylim()[1])
    ax = pyplot.gca()
    xticks = ax.get_xticks()
    xticks_d = [
      '%.2f' %uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks]
    ax.set_xticklabels(xticks_d)
    if size_inches is not None:
      fig = pyplot.gcf()
      fig.set_size_inches(size_inches)
    if labels is not None:
      pyplot.legend(loc='best')
    pyplot.tight_layout()
    pyplot.savefig(os.path.join(image_dir, prefix+ k + '.png'))
    pyplot.clf()
def ice_rings_selection(reflections):
  d_star_sq = flex.pow2(reflections['rlp'].norms())
  d_spacings = uctbx.d_star_sq_as_d(d_star_sq)

  from dials.algorithms.integration import filtering

  unit_cell = uctbx.unit_cell((4.498,4.498,7.338,90,90,120))
  space_group = sgtbx.space_group_info(number=194).group()
  width = 0.06

  if d_spacings:
    ice_filter = filtering.PowderRingFilter(
      unit_cell, space_group, flex.min(d_spacings)-width, width)

    ice_sel = ice_filter(d_spacings)

    return ice_sel
  else:
    return None
Exemple #31
0
    def __init__(self, experiments, observed, config):
        self.experiments = experiments
        self.observed = observed
        self.cfg = config

        # extract reflections and map to reciprocal space
        self.refl = observed.select(observed["id"] == 0)
        self.refl.centroid_px_to_mm([experiments[0]])
        self.refl.map_centroids_to_reciprocal_space([experiments[0]])

        # calculate d-spacings
        self.ref_d_star_sq = flex.pow2(self.refl["rlp"].norms())
        self.d_spacings = uctbx.d_star_sq_as_d(self.ref_d_star_sq)
        self.d_min = flex.min(self.d_spacings)

        # initialize desired parameters (so that can back out without having to
        # re-run functions)
        self.n_ice_rings = 0
        self.mean_spot_shape_ratio = 1
        self.n_overloads = self.count_overloads()
        self.hres = 99.9
        self.n_spots = 0
def estimate_resolution_limit(reflections, ice_sel=None, plot_filename=None):
    if ice_sel is None:
        ice_sel = flex.bool(len(reflections), False)

    d_star_sq = flex.pow2(reflections["rlp"].norms())
    d_spacings = uctbx.d_star_sq_as_d(d_star_sq)

    intensities = reflections["intensity.sum.value"]
    variances = reflections["intensity.sum.variance"]

    sel = variances > 0
    intensities = intensities.select(sel)
    variances = variances.select(sel)
    ice_sel = ice_sel.select(sel)

    i_over_sigi = intensities / flex.sqrt(variances)
    log_i_over_sigi = flex.log(i_over_sigi)

    fit = flex.linear_regression(d_star_sq.select(~ice_sel),
                                 log_i_over_sigi.select(~ice_sel))
    m = fit.slope()
    c = fit.y_intercept()

    log_i_sigi_lower = flex.double()
    d_star_sq_lower = flex.double()
    log_i_sigi_upper = flex.double()
    d_star_sq_upper = flex.double()

    binner = binner_equal_population(d_star_sq,
                                     target_n_per_bin=20,
                                     max_slots=20,
                                     min_slots=5)

    outliers_all = flex.bool(len(reflections), False)

    low_percentile_limit = 0.1
    upper_percentile_limit = 1 - low_percentile_limit
    for i_slot, slot in enumerate(binner.bins):
        sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
        sel = ~(ice_sel) & sel_all

        if sel.count(True) == 0:
            continue

        outliers = wilson_outliers(reflections.select(sel_all),
                                   ice_sel=ice_sel.select(sel_all))
        outliers_all.set_selected(sel_all, outliers)

        isel = sel_all.iselection().select(~(outliers)
                                           & ~(ice_sel).select(sel_all))
        log_i_over_sigi_sel = log_i_over_sigi.select(isel)
        d_star_sq_sel = d_star_sq.select(isel)

        perm = flex.sort_permutation(log_i_over_sigi_sel)
        i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
        i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
        log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
        log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
        d_star_sq_upper.append(d_star_sq_sel[i_lower])
        d_star_sq_lower.append(d_star_sq_sel[i_upper])

    fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
    m_upper = fit_upper.slope()
    c_upper = fit_upper.y_intercept()
    fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
    m_lower = fit_lower.slope()
    c_lower = fit_lower.y_intercept()

    if m_upper == m_lower:
        intersection = (-1, -1)
        resolution_estimate = -1
        inside = flex.bool(len(d_star_sq), False)

    else:
        # http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
        # with:
        # a_ = m_upper
        # b_ = m_lower
        # c_ = c_upper
        # d_ = c_lower
        # intersection == ((d_ - c_) / (a_ - b_), (a_ * d_ - b_ * c_) / (a_ - b_))
        intersection = (
            (c_lower - c_upper) / (m_upper - m_lower),
            (m_upper * c_lower - m_lower * c_upper) / (m_upper - m_lower),
        )

        inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper,
                                   c_upper)
        inside = inside & ~outliers_all & ~ice_sel

        if inside.count(True) > 0:
            d_star_sq_estimate = flex.max(d_star_sq.select(inside))
            resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
        else:
            resolution_estimate = -1

    if plot_filename is not None:
        from matplotlib import pyplot

        fig = pyplot.figure()
        ax = fig.add_subplot(1, 1, 1)
        ax.scatter(d_star_sq, log_i_over_sigi, marker="+")
        ax.scatter(
            d_star_sq.select(inside),
            log_i_over_sigi.select(inside),
            marker="+",
            color="green",
        )
        ax.scatter(
            d_star_sq.select(ice_sel),
            log_i_over_sigi.select(ice_sel),
            marker="+",
            color="black",
        )
        ax.scatter(
            d_star_sq.select(outliers_all),
            log_i_over_sigi.select(outliers_all),
            marker="+",
            color="grey",
        )
        ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker="+", color="red")
        ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker="+", color="red")

        if intersection[0] <= ax.get_xlim(
        )[1] and intersection[1] <= ax.get_ylim()[1]:
            ax.scatter([intersection[0]], [intersection[1]],
                       marker="x",
                       s=50,
                       color="b")
        xlim = pyplot.xlim()
        ax.plot(xlim, [(m * x + c) for x in xlim])
        ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color="red")
        ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color="red")
        ax.set_xlabel("d_star_sq")
        ax.set_ylabel("ln(I/sigI)")
        ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
        ax.set_ylim((0, ax.get_ylim()[1]))

        for i_slot, slot in enumerate(binner.bins):
            if i_slot == 0:
                ax.vlines(
                    uctbx.d_as_d_star_sq(slot.d_max),
                    0,
                    ax.get_ylim()[1],
                    linestyle="dotted",
                    color="grey",
                )
            ax.vlines(
                uctbx.d_as_d_star_sq(slot.d_min),
                0,
                ax.get_ylim()[1],
                linestyle="dotted",
                color="grey",
            )

        ax_ = ax.twiny()  # ax2 is responsible for "top" axis and "right" axis
        xticks = ax.get_xticks()
        xticks_d = [
            uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
        ]
        ax_.set_xticks(xticks)
        ax_.set_xlim(ax.get_xlim())
        ax_.set_xlabel(r"Resolution ($\AA$)")
        ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
        pyplot.savefig(plot_filename)
        pyplot.close()

    return resolution_estimate
def quasi_normalisation(reflection_table, experiment):
    """Calculate normalised intensity (Esq) values for reflections, for the purpose
    of selecting subsets based on Esq for scaling. If more involved analyses of
    normalised intensities are needed, then it may be necessary to split this
    procedure to handle acentric and centric reflections separately."""

    logger.info(
        "Calculating normalised intensity values to select a reflection \n"
        "subset for scaling. \n")
    logger.debug(
        "Negative intensities are set to zero for the purpose of \n"
        "calculating mean intensity values for resolution bins. This is to avoid \n"
        "spuriously high E^2 values due to a mean close to zero and should only \n"
        "affect the E^2 values of the highest resolution bins. \n")

    good_refl_sel = ~reflection_table.get_flags(
        reflection_table.flags.bad_for_scaling, all=False)
    rt_subset = reflection_table.select(good_refl_sel)

    # Scaling subset is data that has not been flagged as bad or excluded
    miller_set = miller.set(
        crystal_symmetry=experiment.crystal.get_crystal_symmetry(),
        indices=rt_subset["miller_index"],
    )

    # handle negative reflections to minimise effect on mean I values.
    rt_subset["intensity_for_norm"] = copy.deepcopy(rt_subset["intensity"])
    rt_subset["intensity_for_norm"].set_selected(rt_subset["intensity"] < 0.0,
                                                 0.0)
    miller_array = miller.array(miller_set,
                                data=rt_subset["intensity_for_norm"])
    n_refl = rt_subset.size()

    # set up binning objects
    if n_refl > 20000:
        n_refl_shells = 20
    elif n_refl > 15000:
        n_refl_shells = 15
    elif n_refl > 10000:
        n_refl_shells = 10
    else:
        logger.info(
            "No normalised intensity values were calculated, as an insufficient\n"
            "number of reflections were detected. All normalised intensity \n"
            "values will be set to 1 to allow use in scaling model determination. \n"
        )
        reflection_table["Esq"] = flex.double(reflection_table.size(), 1.0)
        return reflection_table

    d_star_sq = miller_array.d_star_sq().data()
    d_star_sq_min = flex.min(d_star_sq)
    d_star_sq_max = flex.max(d_star_sq)
    span = d_star_sq_max - d_star_sq_min

    relative_tolerance = 1e-6
    d_star_sq_max += span * relative_tolerance
    d_star_sq_min -= span * relative_tolerance
    step = (d_star_sq_max - d_star_sq_min) / n_refl_shells
    miller_array.setup_binner_d_star_sq_step(
        auto_binning=False,
        d_max=uctbx.d_star_sq_as_d(d_star_sq_max),
        d_min=uctbx.d_star_sq_as_d(d_star_sq_min),
        d_star_sq_step=step,
    )

    normalisations = miller_array.intensity_quasi_normalisations()
    normalised_intensities = miller_array.customized_copy(
        data=(miller_array.data() / normalisations.data()))
    reflection_table["Esq"] = flex.double(reflection_table.size(), 0.0)
    reflection_table["Esq"].set_selected(good_refl_sel,
                                         normalised_intensities.data())
    return reflection_table
def estimate_resolution_limit(reflections, imageset, ice_sel=None,
                              plot_filename=None):

  if ice_sel is None:
    ice_sel = flex.bool(len(reflections), False)

  d_star_sq = flex.pow2(reflections['rlp'].norms())
  d_spacings = uctbx.d_star_sq_as_d(d_star_sq)

  intensities = reflections['intensity.sum.value']
  variances = reflections['intensity.sum.variance']

  sel = variances > 0
  intensities = intensities.select(sel)
  variances = variances.select(sel)
  ice_sel = ice_sel.select(sel)

  i_over_sigi = intensities/flex.sqrt(variances)
  log_i_over_sigi = flex.log(i_over_sigi)

  fit = flex.linear_regression(
    d_star_sq.select(~ice_sel), log_i_over_sigi.select(~ice_sel))
  m = fit.slope()
  c = fit.y_intercept()

  log_i_sigi_lower = flex.double()
  d_star_sq_lower = flex.double()
  log_i_sigi_upper = flex.double()
  d_star_sq_upper = flex.double()

  binner = binner_equal_population(
    d_star_sq, target_n_per_bin=20, max_slots=20, min_slots=5)

  outliers_all = flex.bool(len(reflections), False)

  low_percentile_limit = 0.1
  upper_percentile_limit = 1-low_percentile_limit
  d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
  for i_slot, slot in enumerate(binner.bins):
    sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
    sel = ~(ice_sel) & sel_all
    #sel = ~(ice_sel) & (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)

    #print "%.2f" %(sel.count(True)/sel_all.count(True))

    if sel.count(True) == 0:
      #outliers_all.set_selected(sel_all & ice_sel, True)
      continue
      #if i_slot > i_slot_max:
        #break
      #else:
        #continue

    outliers = wilson_outliers(
      reflections.select(sel_all), ice_sel=ice_sel.select(sel_all))
    #print "rejecting %d wilson outliers" %outliers.count(True)
    outliers_all.set_selected(sel_all, outliers)

    #if sel.count(True)/sel_all.count(True) < 0.25:
      #outliers_all.set_selected(sel_all & ice_sel, True)

    #from scitbx.math import median_statistics
    #intensities_sel = intensities.select(sel)
    #stats = median_statistics(intensities_sel)
    #z_score = 0.6745 * (intensities_sel - stats.median)/stats.median_absolute_deviation
    #outliers = z_score > 3.5
    #perm = flex.sort_permutation(intensities_sel)
    ##print ' '.join('%.2f' %v for v in intensities_sel.select(perm))
    ##print ' '.join('%.2f' %v for v in z_score.select(perm))
    ##print

    isel = sel_all.iselection().select(~(outliers) & ~(ice_sel).select(sel_all))
    log_i_over_sigi_sel = log_i_over_sigi.select(isel)
    d_star_sq_sel = d_star_sq.select(isel)

    perm = flex.sort_permutation(log_i_over_sigi_sel)
    i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
    i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
    log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
    log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
    d_star_sq_upper.append(d_star_sq_sel[i_lower])
    d_star_sq_lower.append(d_star_sq_sel[i_upper])

  fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
  m_upper = fit_upper.slope()
  c_upper = fit_upper.y_intercept()
  fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
  m_lower = fit_lower.slope()
  c_lower = fit_lower.y_intercept()

  #fit_upper.show_summary()
  #fit_lower.show_summary()

  if m_upper == m_lower:
    intersection = (-1,-1)
    resolution_estimate = -1
    inside = flex.bool(len(d_star_sq), False)

  else:
    # http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
    intersection = (
      (c_lower-c_upper)/(m_upper-m_lower),
      (m_upper*c_lower-m_lower*c_upper)/(m_upper-m_lower))

    a = m_upper
    c_ = c_upper
    b = m_lower
    d = c_lower
    assert intersection == ((d-c_)/(a-b), (a*d-b*c_)/(a-b))

    #inside = points_inside_envelope(
      #d_star_sq, log_i_over_sigi, m_upper, c_upper, m_lower, c_lower)

    inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper, c_upper)
    inside = inside & ~outliers_all

    if inside.count(True) > 0:
      d_star_sq_estimate = flex.max(d_star_sq.select(inside))
      #d_star_sq_estimate = intersection[0]
      resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
    else:
      resolution_estimate = -1

  #resolution_estimate = max(resolution_estimate, flex.min(d_spacings))

  if plot_filename is not None:
    if pyplot is None:
      raise Sorry("matplotlib must be installed to generate a plot.")
    fig = pyplot.figure()
    ax = fig.add_subplot(1,1,1)
    ax.scatter(d_star_sq, log_i_over_sigi, marker='+')
    ax.scatter(d_star_sq.select(inside), log_i_over_sigi.select(inside),
               marker='+', color='green')
    ax.scatter(d_star_sq.select(ice_sel),
               log_i_over_sigi.select(ice_sel),
               marker='+', color='black')
    ax.scatter(d_star_sq.select(outliers_all),
               log_i_over_sigi.select(outliers_all),
               marker='+', color='grey')
    ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker='+', color='red')
    ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker='+', color='red')

    if (intersection[0] <= ax.get_xlim()[1] and
        intersection[1] <= ax.get_ylim()[1]):
      ax.scatter([intersection[0]], [intersection[1]], marker='x', s=50, color='b')
    #ax.hexbin(d_star_sq, log_i_over_sigi, gridsize=30)
    xlim = pyplot.xlim()
    ax.plot(xlim, [(m * x + c) for x in xlim])
    ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color='red')
    ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color='red')
    ax.set_xlabel('d_star_sq')
    ax.set_ylabel('ln(I/sigI)')
    ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
    ax.set_ylim((0, ax.get_ylim()[1]))

    for i_slot, slot in enumerate(binner.bins):
      if i_slot == 0:
        ax.vlines(uctbx.d_as_d_star_sq(slot.d_max), 0, ax.get_ylim()[1],
                  linestyle='dotted', color='grey')
      ax.vlines(uctbx.d_as_d_star_sq(slot.d_min), 0, ax.get_ylim()[1],
                linestyle='dotted', color='grey')

    ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
    xticks = ax.get_xticks()
    xlim = ax.get_xlim()
    xticks_d = [
      uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
    xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
    ax_.set_xticks(xticks)
    ax_.set_xlim(ax.get_xlim())
    ax_.set_xlabel(r"Resolution ($\AA$)")
    ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
    #pyplot.show()
    pyplot.savefig(plot_filename)
    pyplot.close()

  return resolution_estimate
Exemple #35
0
def d_star_sq_to_d_ticks(d_star_sq, nticks):
    min_d_star_sq = min(d_star_sq)
    dstep = (max(d_star_sq) - min_d_star_sq) / nticks
    tickvals = list(min_d_star_sq + (i * dstep) for i in range(nticks))
    ticktext = ["%.2f" % (uctbx.d_star_sq_as_d(dsq)) for dsq in tickvals]
    return tickvals, ticktext
def run(args):
  import libtbx.load_env
  usage = "%s experiments.json indexed.pickle [options]" %libtbx.env.dispatcher_name

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_experiments=True,
    read_reflections=True,
    check_format=False,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=True)
  experiments = flatten_experiments(params.input.experiments)
  reflections = flatten_reflections(params.input.reflections)
  if len(experiments) == 0:
    parser.print_help()
    return
  elif len(experiments) > 1:
    raise Sorry("More than one experiment present")

  experiment = experiments[0]
  assert(len(reflections) == 1)
  reflections = reflections[0]

  intensities = reflections['intensity.sum.value']
  variances = reflections['intensity.sum.variance']
  if 'intensity.prf.value' in reflections:
    intensities = reflections['intensity.prf.value']
    variances = reflections['intensity.prf.variance']
  sel = (variances > 0)
  intensities = intensities.select(sel)
  variances = variances.select(sel)
  sigmas = flex.sqrt(variances)
  indices = reflections['miller_index'].select(sel)

  from cctbx import crystal, miller
  crystal_symmetry = crystal.symmetry(
    space_group=experiment.crystal.get_space_group(),
    unit_cell=experiment.crystal.get_unit_cell())

  miller_set = miller.set(
    crystal_symmetry=crystal_symmetry,
    anomalous_flag=True,
    indices=indices)
  miller_array = miller.array(
    miller_set=miller_set,
    data=intensities,
    sigmas=sigmas).set_observation_type_xray_intensity()

  #miller_array.setup_binner(n_bins=50, reflections_per_bin=100)
  miller_array.setup_binner(auto_binning=True, n_bins=20)
  result = miller_array.i_over_sig_i(use_binning=True)
  result.show()

  from cctbx import uctbx
  d_star_sq_centre = result.binner.bin_centers(2)
  i_over_sig_i = flex.double(
    [d if d is not None else 0 for d in result.data[1:-1]])
  sel = (i_over_sig_i > 0)
  d_star_sq_centre = d_star_sq_centre.select(sel)
  i_over_sig_i = i_over_sig_i.select(sel)
  log_i_over_sig_i = flex.log(i_over_sig_i)
  weights = result.binner.counts()[1:-1].as_double().select(sel)
  fit = flex.linear_regression(
    d_star_sq_centre, log_i_over_sig_i, weights=weights)

  m = fit.slope()
  c = fit.y_intercept()

  import math
  y_cutoff = math.log(params.i_sigi_cutoff)
  x_cutoff = (y_cutoff - c)/m

  estimated_d_min = uctbx.d_star_sq_as_d(x_cutoff)
  print "estimated d_min: %.2f" %estimated_d_min

  if params.plot:
    from matplotlib import pyplot
    fig = pyplot.figure()
    ax = fig.add_subplot(1,1,1)

    ax.plot(
      list(d_star_sq_centre),
      list(log_i_over_sig_i),
      label=r"ln(I/sigI)")
    ax.plot(pyplot.xlim(), [(m * x + c) for x in pyplot.xlim()], color='red')
    ax.plot([x_cutoff, x_cutoff], pyplot.ylim(), color='grey', linestyle='dashed')
    ax.plot(pyplot.xlim(), [y_cutoff, y_cutoff], color='grey', linestyle='dashed')
    ax.set_xlabel("d_star_sq")
    ax.set_ylabel("ln(I/sigI)")

    ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
    xticks = ax.get_xticks()
    xlim = ax.get_xlim()
    xticks_d = [
      uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
    xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
    ax_.set_xticks(xticks)
    ax_.set_xlim(ax.get_xlim())
    ax_.set_xlabel(r"Resolution ($\AA$)")
    ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
    pyplot.savefig("estimate_resolution_limit.png")
    pyplot.clf()
Exemple #37
0
def run(args):

  from dials.util.options import OptionParser
  from dials.util.options import flatten_experiments
  from dials.util.options import flatten_reflections
  import libtbx.load_env

  usage = "%s [options] datablock.json" %(
    libtbx.env.dispatcher_name)

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_experiments=True,
    read_reflections=True,
    check_format=True,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=True)
  experiments = flatten_experiments(params.input.experiments)
  reflections = flatten_reflections(params.input.reflections)

  if len(experiments) == 0 or len(reflections) == 0:
    parser.print_help()
    exit(0)

  imagesets = experiments.imagesets()
  reflections = reflections[0]
  shadowed = filter_shadowed_reflections(experiments, reflections)

  print "# shadowed reflections: %i/%i (%.2f%%)" %(
    shadowed.count(True), shadowed.size(),
    shadowed.count(True)/shadowed.size() * 100)

  expt = experiments[0]
  x,y,z = reflections['xyzcal.px'].parts()
  z_ = z * expt.scan.get_oscillation()[1]
  zmin, zmax = expt.scan.get_oscillation_range()

  hist_scan_angle = flex.histogram(z_.select(shadowed), n_slots=int(zmax-zmin))
  #hist_scan_angle.show()

  uc = experiments[0].crystal.get_unit_cell()
  d_spacings = uc.d(reflections['miller_index'])
  ds2 = uc.d_star_sq(reflections['miller_index'])

  hist_res = flex.histogram(ds2.select(shadowed), flex.min(ds2), flex.max(ds2), n_slots=20, )
  #hist_res.show()

  import matplotlib
  matplotlib.use('Agg')
  from matplotlib import pyplot as plt

  plt.hist2d(z_.select(shadowed).as_numpy_array(),
             ds2.select(shadowed).as_numpy_array(), bins=(40,40),
             range=((flex.min(z_), flex.max(z_)),(flex.min(ds2), flex.max(ds2))))
  yticks_dsq = flex.double(plt.yticks()[0])
  from cctbx import uctbx
  yticks_d = uctbx.d_star_sq_as_d(yticks_dsq)
  plt.axes().set_yticklabels(['%.2f' %y for y in yticks_d])
  plt.xlabel('Scan angle (degrees)')
  plt.ylabel('Resolution (A^-1)')
  cbar = plt.colorbar()
  cbar.set_label('# shadowed reflections')
  plt.savefig('n_shadowed_hist2d.png')
  plt.clf()

  plt.scatter(hist_scan_angle.slot_centers().as_numpy_array(), hist_scan_angle.slots().as_numpy_array())
  plt.xlabel('Scan angle (degrees)')
  plt.ylabel('# shadowed reflections')
  plt.savefig("n_shadowed_vs_scan_angle.png")
  plt.clf()

  plt.scatter(hist_res.slot_centers().as_numpy_array(), hist_res.slots().as_numpy_array())
  plt.xlabel('d_star_sq')
  plt.savefig("n_shadowed_vs_resolution.png")
  plt.clf()
def estimate_resolution_limit_distl_method2(
  reflections, imageset, ice_sel=None, plot_filename=None):

  # Implementation of Method 2 (section 2.4.4) of:
  # Z. Zhang, N. K. Sauter, H. van den Bedem, G. Snell and A. M. Deacon
  # J. Appl. Cryst. (2006). 39, 112-119
  # http://dx.doi.org/10.1107/S0021889805040677

  if ice_sel is None:
    ice_sel = flex.bool(len(reflections), False)

  variances = reflections['intensity.sum.variance']

  sel = variances > 0
  intensities = reflections['intensity.sum.value']
  variances = variances.select(sel)
  ice_sel = ice_sel.select(sel)
  reflections = reflections.select(sel)
  intensities = reflections['intensity.sum.value']
  d_star_sq = flex.pow2(reflections['rlp'].norms())
  d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
  d_star_cubed = flex.pow(reflections['rlp'].norms(), 3)

  binner = binner_d_star_cubed(d_spacings)

  bin_counts = flex.size_t()

  for i_slot, slot in enumerate(binner.bins):
    sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
    #sel = ~(ice_sel) & sel_all
    sel = sel_all

    bin_counts.append(sel.count(True))

  #print list(bin_counts)
  t0 = (bin_counts[0] + bin_counts[1])/2

  mu = 0.15

  for i in range(len(bin_counts)-1):
    tj = bin_counts[i]
    tj1 = bin_counts[i+1]
    if (tj < (mu * t0)) and (tj1 < (mu * t0)):
      break

  d_min = binner.bins[i].d_min
  noisiness = 0
  m = len(bin_counts)
  for i in range(m):
    for j in range(i+1, m):
      if bin_counts[i] <= bin_counts[j]:
        noisiness += 1
  noisiness /= (0.5 * m * (m-1))

  if plot_filename is not None:
    if pyplot is None:
      raise Sorry("matplotlib must be installed to generate a plot.")
    fig = pyplot.figure()
    ax = fig.add_subplot(1,1,1)
    ax.scatter(range(len(bin_counts)), bin_counts)
    #ax.set_xlabel('')
    ax.set_ylabel('number of spots in shell')
    xlim = pyplot.xlim()
    ylim = pyplot.ylim()
    ax.vlines(i, ylim[0], ylim[1], colors='red')
    pyplot.xlim(0, xlim[1])
    pyplot.ylim(0, ylim[1])
    pyplot.savefig(plot_filename)
    pyplot.close()

  return d_min, noisiness
def estimate_resolution_limit_distl_method1(
  reflections, imageset, ice_sel=None, plot_filename=None):

  # Implementation of Method 1 (section 2.4.4) of:
  # Z. Zhang, N. K. Sauter, H. van den Bedem, G. Snell and A. M. Deacon
  # J. Appl. Cryst. (2006). 39, 112-119
  # http://dx.doi.org/10.1107/S0021889805040677

  if ice_sel is None:
    ice_sel = flex.bool(len(reflections), False)

  variances = reflections['intensity.sum.variance']

  sel = variances > 0
  intensities = reflections['intensity.sum.value']
  variances = variances.select(sel)
  ice_sel = ice_sel.select(sel)
  reflections = reflections.select(sel)
  intensities = reflections['intensity.sum.value']
  d_star_sq = flex.pow2(reflections['rlp'].norms())
  d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
  d_star_cubed = flex.pow(reflections['rlp'].norms(), 3)

  step = 2
  while len(reflections)/step > 40:
    step += 1

  order = flex.sort_permutation(d_spacings, reverse=True)

  ds3_subset = flex.double()
  d_subset = flex.double()
  for i in range(len(reflections)//step):
    ds3_subset.append(d_star_cubed[order[i*step]])
    d_subset.append(d_spacings[order[i*step]])

  x = flex.double(range(len(ds3_subset)))

  # (i)
  # Usually, Pm is the last point, that is, m = n. But m could be smaller than
  # n if an unusually high number of spots are detected around a certain
  # intermediate resolution. In that case, our search for the image resolution
  # does not go outside the spot 'bump;. This is particularly useful when
  # ice-rings are present.

  slopes = (ds3_subset[1:] - ds3_subset[0])/(x[1:]-x[0])
  skip_first = 3
  p_m = flex.max_index(slopes[skip_first:]) + 1 + skip_first

  # (ii)

  from scitbx import matrix
  x1 = matrix.col((0, ds3_subset[0]))
  x2 = matrix.col((p_m, ds3_subset[p_m]))

  gaps = flex.double([0])
  v = matrix.col(((x2[1] - x1[1]), -(x2[0] - x1[0]))).normalize()

  for i in range(1, p_m):
    x0 = matrix.col((i, ds3_subset[i]))
    r = x1 - x0
    g = abs(v.dot(r))
    gaps.append(g)

  mv = flex.mean_and_variance(gaps)
  s = mv.unweighted_sample_standard_deviation()

  # (iii)

  p_k = flex.max_index(gaps)
  g_k = gaps[p_k]
  p_g = p_k
  for i in range(p_k+1, len(gaps)):
    g_i = gaps[i]
    if g_i > (g_k - 0.5 * s):
      p_g = i

  ds3_g = ds3_subset[p_g]
  d_g = d_subset[p_g]

  noisiness = 0
  n = len(ds3_subset)
  for i in range(n-1):
    for j in range(i+1, n-1):
      if slopes[i] >= slopes[j]:
        noisiness += 1
  noisiness /= ((n-1)*(n-2)/2)

  if plot_filename is not None:
    if pyplot is None:
      raise Sorry("matplotlib must be installed to generate a plot.")
    fig = pyplot.figure()
    ax = fig.add_subplot(1,1,1)
    ax.scatter(range(len(ds3_subset)), ds3_subset)
    #ax.set_xlabel('')
    ax.set_ylabel('D^-3')
    xlim = pyplot.xlim()
    ylim = pyplot.ylim()
    ax.vlines(p_g, ylim[0], ylim[1], colors='red')
    pyplot.xlim(0, xlim[1])
    pyplot.ylim(0, ylim[1])
    pyplot.savefig(plot_filename)
    pyplot.close()

  return d_g, noisiness
Exemple #40
0
def exercise_functions():
  d_star_sq_s = 1.2345
  d_star_sq_a = flex.double((1.2345, 0.1234))
  two_theta_s = 0.61725
  two_theta_a = flex.double((0.61725,0.65432))
  # forward conversions
  assert approx_equal(
    uctbx.d_star_sq_as_stol_sq(d_star_sq_s), d_star_sq_s / 4)
  assert approx_equal(
    uctbx.d_star_sq_as_stol_sq(d_star_sq_a),
    [uctbx.d_star_sq_as_stol_sq(i) for i in d_star_sq_a])
  assert approx_equal(
    uctbx.d_star_sq_as_two_stol(d_star_sq_s)**2, d_star_sq_s)
  assert approx_equal(
    uctbx.d_star_sq_as_two_stol(d_star_sq_a),
    [uctbx.d_star_sq_as_two_stol(i) for i in d_star_sq_a])
  assert approx_equal(
    uctbx.d_star_sq_as_stol(d_star_sq_s)**2, d_star_sq_s / 4)
  assert approx_equal(
    uctbx.d_star_sq_as_stol(d_star_sq_a),
    [uctbx.d_star_sq_as_stol(i) for i in d_star_sq_a])
  assert approx_equal(
    1/(uctbx.d_star_sq_as_d(d_star_sq_s)**2), d_star_sq_s)
  assert approx_equal(
    uctbx.d_star_sq_as_d(d_star_sq_a),
    [uctbx.d_star_sq_as_d(i) for i in d_star_sq_a])
  assert approx_equal(
    uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5),
    2 * asin(1.5/2*sqrt(d_star_sq_s)))
  assert approx_equal(
    uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5),
    uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5, False))
  assert approx_equal(
    uctbx.d_star_sq_as_two_theta(d_star_sq_a, 1.5),
    [uctbx.d_star_sq_as_two_theta(i, 1.5) for i in d_star_sq_a])
  assert approx_equal(
    uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5)*180/pi,
    uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5, True))
  # reverse conversions
  for d_star_sq, two_theta in zip(
    (d_star_sq_s, d_star_sq_a),(two_theta_s, two_theta_a)):
    assert approx_equal(
      uctbx.stol_sq_as_d_star_sq(
        uctbx.d_star_sq_as_stol_sq(d_star_sq)), d_star_sq)
    assert approx_equal(
      uctbx.two_stol_as_d_star_sq(
        uctbx.d_star_sq_as_two_stol(d_star_sq)), d_star_sq)
    assert approx_equal(
      uctbx.stol_as_d_star_sq(
        uctbx.d_star_sq_as_stol(d_star_sq)), d_star_sq)
    assert approx_equal(
      uctbx.d_as_d_star_sq(
        uctbx.d_star_sq_as_d(d_star_sq)), d_star_sq)
    assert approx_equal(
      uctbx.two_theta_as_d_star_sq(
        uctbx.d_star_sq_as_two_theta(d_star_sq, 1.5), 1.5), d_star_sq)
    assert approx_equal(
      uctbx.two_theta_as_d_star_sq(two_theta, 1.5),
      uctbx.two_theta_as_d_star_sq(two_theta, 1.5, False))
    assert approx_equal(
      uctbx.two_theta_as_d_star_sq(two_theta, 1.5),
      uctbx.two_theta_as_d_star_sq(two_theta*180/pi, 1.5, True))
    assert approx_equal(
      uctbx.two_theta_as_d(two_theta, 1.5),
      uctbx.d_star_sq_as_d(uctbx.two_theta_as_d_star_sq(two_theta, 1.5)))
    assert approx_equal(
      uctbx.two_theta_as_d(two_theta, 1.5, True),
      uctbx.d_star_sq_as_d(uctbx.two_theta_as_d_star_sq(two_theta, 1.5, True)))
  #
  assert uctbx.fractional_unit_shifts(distance_frac=[0,0,0]) == (0,0,0)
  assert uctbx.fractional_unit_shifts([0.6,7.4,-0.4]) == (1,7,0)
  assert uctbx.fractional_unit_shifts([-6,3,-0.6]) == (-6,3,-1)
  site_frac_1 = [0.3,-8.6,2.1]
  for eps,expected_u2 in [(-1.e-5, 1), (1.e-5, 2)]:
    site_frac_2 = [-3,5.6,0.6-eps]
    assert uctbx.fractional_unit_shifts(
      site_frac_1=site_frac_1,
      site_frac_2=site_frac_2) == (3, -14, expected_u2)
Exemple #41
0
def run(args=None,
        phil=phil_scope):  # type: (List[str], libtbx.phil.scope) -> None
    usage = "dials.missing_reflections [options] scaled.expt scaled.refl"

    parser = OptionParser(
        usage=usage,
        phil=phil,
        read_reflections=True,
        read_experiments=True,
        check_format=False,
        epilog=__doc__,
    )

    params, options = parser.parse_args(args=args, show_diff_phil=False)

    # Configure the logging.
    dials.util.log.config(options.verbose)
    logger.info(dials_version())

    # Log the difference between the PHIL scope definition and the active PHIL scope,
    # which will include the parsed user inputs.
    diff_phil = parser.diff_phil.as_str()
    if diff_phil:
        logger.info("The following parameters have been modified:\n%s",
                    diff_phil)

    experiments = flatten_experiments(params.input.experiments)
    reflections = flatten_reflections(params.input.reflections)

    if not experiments or not reflections:
        parser.print_help()
        return
    if len(reflections) != 1 and len(experiments) != len(reflections):
        sys.exit(
            "Number of experiments must equal the number of reflection tables")

    from dials.util.multi_dataset_handling import (
        assign_unique_identifiers,
        parse_multiple_datasets,
    )

    reflections = parse_multiple_datasets(reflections)
    experiments, reflections = assign_unique_identifiers(
        experiments, reflections)

    if all("inverse_scale_factor" in refl for refl in reflections):
        # Assume all arrays have been scaled
        miller_array = scaled_data_as_miller_array(reflections,
                                                   experiments,
                                                   anomalous_flag=False)
    else:
        # Else get the integrated intensities
        miller_arrays = filtered_arrays_from_experiments_reflections(
            experiments,
            reflections,
        )
        miller_array = miller_arrays[0]
        for ma in miller_arrays[1:]:
            miller_array = miller_array.concatenate(ma)

    if params.d_min or params.d_max:
        miller_array = miller_array.resolution_filter(d_min=params.d_min,
                                                      d_max=params.d_max)

    # Print overall summary of input miller array
    s = io.StringIO()
    ma_unique = miller_array.unique_under_symmetry()
    ma_unique.show_comprehensive_summary(f=s)
    logger.info(f"\n{s.getvalue()}")

    # Get the regions of missing reflections
    complete_set, unique_ms = missing_reflections.connected_components(
        miller_array)
    unique_ms = [
        ms for ms in unique_ms if ms.size() >= params.min_component_size
    ]

    # Print some output for user
    if len(unique_ms):
        logger.info(
            "The following connected regions of missing reflections have been identified:"
        )
        n_expected = complete_set.size()
        rows = []
        for ms in unique_ms:
            d_max, d_min = (uctbx.d_star_sq_as_d(ds2)
                            for ds2 in ms.min_max_d_star_sq())
            rows.append([
                ms.size(),
                f"{100 * ms.size() / n_expected:.1f}",
                f"{d_max:.2f}-{d_min:.2f}",
            ])
        logger.info(
            tabulate(
                rows,
                headers=["# reflections", "% missing",
                         "Resolution range (Å)"]))
    else:
        logger.info("No connected regions of missing reflections identified")
def run(args):
    import matplotlib

    matplotlib.use("Agg")
    import libtbx.load_env

    usage = "%s [options]" % libtbx.env.dispatcher_name

    parser = OptionParser(
        usage=usage, phil=phil_scope, check_format=False, epilog=help_message
    )

    params, options, args = parser.parse_args(
        show_diff_phil=True, return_unhandled=True
    )

    for mtz in args:
        print(mtz)
        assert os.path.isfile(mtz), mtz
        import iotbx.merging_statistics

        i_obs = iotbx.merging_statistics.select_data(mtz, data_labels=params.labels)
        if params.space_group is not None:
            i_obs = i_obs.customized_copy(space_group_info=params.space_group)

        from scitbx.array_family import flex

        # set the sigmas to 1, and calculate the mean intensities and internal variances
        intensities_copy = i_obs.customized_copy(sigmas=flex.double(i_obs.size(), 1))
        merging_internal = intensities_copy.merge_equivalents(
            use_internal_variance=True
        )
        merged = merging_internal.array()

        merging_external = i_obs.merge_equivalents(use_internal_variance=False)

        sigmas_internal = merging_internal.array().sigmas()
        sigmas_external = merging_external.array().sigmas()

        variances_internal = flex.pow2(sigmas_internal)
        variances_external = flex.pow2(sigmas_external)

        n_bins = 100
        i_obs.setup_binner_counting_sorted(n_bins=n_bins)
        sigmas_ratio = sigmas_external / sigmas_internal
        variance_ratio = variances_external / variances_internal

        array_sr = merging_external.array().customized_copy(
            data=sigmas_ratio, sigmas=None
        )
        array_sr.use_binning_of(i_obs)
        mean_sr = array_sr.mean(use_binning=True)

        ds2 = mean_sr.binner.bin_centers(2)
        sr = mean_sr.data[1:-1]

        array_vr = merging_external.array().customized_copy(
            data=variance_ratio, sigmas=None
        )
        array_vr.use_binning_of(i_obs)
        mean_vr = array_vr.mean(use_binning=True)

        d_star_sq = mean_vr.binner.bin_centers(2)
        vr = mean_vr.data[1:-1]

        prefix = params.prefix
        if prefix is None:
            prefix = ""

        from matplotlib import pyplot

        pyplot.style.use("ggplot")

        pyplot.plot(d_star_sq, sr)
        ax = pyplot.gca()
        xticks = ax.get_xticks()
        xticks_d = [
            "%.2f" % uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
        ]
        ax.set_xticklabels(xticks_d)
        pyplot.xlabel("d spacing (A)")
        pyplot.ylabel("<sigI_ext/sigI_int>")
        pyplot.savefig("%ssigmas_ratio.png" % prefix)
        pyplot.clf()

        pyplot.plot(d_star_sq, vr)
        ax = pyplot.gca()
        xticks = ax.get_xticks()
        xticks_d = [
            "%.2f" % uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
        ]
        ax.set_xticklabels(xticks_d)
        pyplot.xlabel("d spacing (A)")
        pyplot.ylabel("<varI_ext/varI_int>")
        pyplot.savefig("%svariances_ratio.png" % prefix)
        pyplot.clf()
def estimate_resolution_limit_distl_method1(reflections, plot_filename=None):
    # Implementation of Method 1 (section 2.4.4) of:
    # Z. Zhang, N. K. Sauter, H. van den Bedem, G. Snell and A. M. Deacon
    # J. Appl. Cryst. (2006). 39, 112-119
    # https://doi.org/10.1107/S0021889805040677

    variances = reflections["intensity.sum.variance"]

    sel = variances > 0
    reflections = reflections.select(sel)
    d_star_sq = flex.pow2(reflections["rlp"].norms())
    d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
    d_star_cubed = flex.pow(reflections["rlp"].norms(), 3)

    step = 2
    while len(reflections) / step > 40:
        step += 1

    order = flex.sort_permutation(d_spacings, reverse=True)

    ds3_subset = flex.double()
    d_subset = flex.double()
    for i in range(len(reflections) // step):
        ds3_subset.append(d_star_cubed[order[i * step]])
        d_subset.append(d_spacings[order[i * step]])

    x = flex.double(range(len(ds3_subset)))

    # (i)
    # Usually, Pm is the last point, that is, m = n. But m could be smaller than
    # n if an unusually high number of spots are detected around a certain
    # intermediate resolution. In that case, our search for the image resolution
    # does not go outside the spot 'bump;. This is particularly useful when
    # ice-rings are present.

    slopes = (ds3_subset[1:] - ds3_subset[0]) / (x[1:] - x[0])
    skip_first = 3
    p_m = flex.max_index(slopes[skip_first:]) + 1 + skip_first

    # (ii)

    x1 = matrix.col((0, ds3_subset[0]))
    x2 = matrix.col((p_m, ds3_subset[p_m]))

    gaps = flex.double([0])
    v = matrix.col(((x2[1] - x1[1]), -(x2[0] - x1[0]))).normalize()

    for i in range(1, p_m):
        x0 = matrix.col((i, ds3_subset[i]))
        r = x1 - x0
        g = abs(v.dot(r))
        gaps.append(g)

    mv = flex.mean_and_variance(gaps)
    s = mv.unweighted_sample_standard_deviation()

    # (iii)

    p_k = flex.max_index(gaps)
    g_k = gaps[p_k]
    p_g = p_k
    for i in range(p_k + 1, len(gaps)):
        g_i = gaps[i]
        if g_i > (g_k - 0.5 * s):
            p_g = i

    d_g = d_subset[p_g]

    noisiness = 0
    n = len(ds3_subset)
    for i in range(n - 1):
        for j in range(i + 1, n - 1):
            if slopes[i] >= slopes[j]:
                noisiness += 1
    noisiness /= (n - 1) * (n - 2) / 2

    if plot_filename is not None:
        from matplotlib import pyplot

        fig = pyplot.figure()
        ax = fig.add_subplot(1, 1, 1)
        ax.scatter(range(len(ds3_subset)), ds3_subset)
        ax.set_ylabel("D^-3")
        xlim = pyplot.xlim()
        ylim = pyplot.ylim()
        ax.vlines(p_g, ylim[0], ylim[1], colors="red")
        pyplot.xlim(0, xlim[1])
        pyplot.ylim(0, ylim[1])
        pyplot.savefig(plot_filename)
        pyplot.close()

    return d_g, noisiness
Exemple #44
0
def run(args):
    import libtbx.load_env

    usage = "%s [options]" % libtbx.env.dispatcher_name

    parser = OptionParser(
        usage=usage, phil=phil_scope, check_format=False, epilog=help_message
    )

    params, options, args = parser.parse_args(
        show_diff_phil=True, return_unhandled=True
    )

    space_group = params.space_group
    if space_group is None:
        space_group = sgtbx.space_group()
    else:
        space_group = space_group.group()

    unit_cell = params.unit_cell
    if unit_cell is None:
        unit_cell = space_group.info().any_compatible_unit_cell(volume=100000)
        print(unit_cell)

    assert len(args) == 2
    from cctbx import crystal, miller

    cs = crystal.symmetry(space_group=space_group, unit_cell=unit_cell)
    intensities = []
    for filename in args:
        hkl, i, sigi = parse_best_hkl(filename)
        ms = miller.set(cs, hkl)
        ma = miller.array(ms, data=i, sigmas=sigi)
        ma.set_observation_type_xray_intensity()
        intensities.append(ma)
        # ma.show_summary()

    # Two subplots, the axes array is 1-d
    from matplotlib import pyplot

    ma1, ma2 = intensities
    hist1 = flex.histogram(ma1.data(), n_slots=100)
    hist2 = flex.histogram(ma2.data(), n_slots=100)
    f, axarr = pyplot.subplots(2, sharex=True, figsize=(16, 12))
    axarr[0].bar(
        hist1.slot_centers() - 0.5 * hist1.slot_width(),
        hist1.slots(),
        align="center",
        width=hist1.slot_width(),
        color="black",
        edgecolor=None,
    )
    axarr[1].bar(
        hist2.slot_centers() - 0.5 * hist2.slot_width(),
        hist2.slots(),
        align="center",
        width=hist2.slot_width(),
        color="black",
        edgecolor=None,
    )
    pyplot.savefig("hist_intensities.png")
    pyplot.clf()

    hist1 = flex.histogram(ma1.data() / ma1.sigmas(), n_slots=100)
    hist2 = flex.histogram(ma2.data() / ma2.sigmas(), n_slots=100)
    f, axarr = pyplot.subplots(2, sharex=True, figsize=(16, 12))
    axarr[0].bar(
        hist1.slot_centers() - 0.5 * hist1.slot_width(),
        hist1.slots(),
        align="center",
        width=hist1.slot_width(),
        color="black",
        edgecolor=None,
    )
    axarr[1].bar(
        hist2.slot_centers() - 0.5 * hist2.slot_width(),
        hist2.slots(),
        align="center",
        width=hist2.slot_width(),
        color="black",
        edgecolor=None,
    )
    pyplot.savefig("hist_isigi.png")
    pyplot.clf()

    print(ma1.d_max_min())
    print(ma2.d_max_min())
    ma1.setup_binner(n_bins=20)
    ma2.setup_binner(n_bins=20)

    imean1 = ma1.mean(use_binning=True)
    imean2 = ma2.mean(use_binning=True)
    f, axarr = pyplot.subplots(2, sharex=True, figsize=(16, 12))
    axarr[0].plot(imean1.binner.bin_centers(2), imean1.data[1:-1])
    axarr[1].plot(imean2.binner.bin_centers(2), imean2.data[1:-1])
    ax = pyplot.gca()
    xticks = ax.get_xticks()
    xticks_d = ["%.2f" % uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks]
    ax.set_xticklabels(xticks_d)
    pyplot.xlabel("d spacing (A)")
    pyplot.savefig("imean_vs_resolution.png")
    pyplot.clf()

    isigi1 = ma1.i_over_sig_i(use_binning=True)
    isigi2 = ma2.i_over_sig_i(use_binning=True)
    f, axarr = pyplot.subplots(2, sharex=True, figsize=(16, 12))
    axarr[0].plot(isigi1.binner.bin_centers(2), isigi1.data[1:-1])
    axarr[1].plot(isigi2.binner.bin_centers(2), isigi2.data[1:-1])
    ax = pyplot.gca()
    xticks = ax.get_xticks()
    xticks_d = ["%.2f" % uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks]
    ax.set_xticklabels(xticks_d)
    pyplot.xlabel("d spacing (A)")
    pyplot.savefig("isigi_vs_resolution.png")
    pyplot.clf()

    best_cb_op = None
    best_count = 0
    for i_op, op in enumerate(
        space_group.build_derived_reflection_intensity_group(False).all_ops()
    ):
        if not op.t().is_zero():
            continue
        cb_op = sgtbx.change_of_basis_op(op)  # .inverse())

        ma1, ma2 = intensities
        ma1, ma2 = ma1.common_sets(ma2.change_basis(cb_op))
        # print cb_op
        # print ma1.size(), ma2.size()
        if ma1.size() > best_count:
            best_cb_op = cb_op
            best_count = ma1.size()

    print("Best cb_op: %s (%i matches)" % (best_cb_op, best_count))
    ma1, ma2 = intensities
    ma1, ma2 = ma1.common_sets(ma2.change_basis(best_cb_op))

    from matplotlib import pyplot

    pyplot.scatter(ma1.data(), ma2.data(), marker="+", alpha=0.5)
    m = max(pyplot.xlim()[1], pyplot.ylim()[1])
    pyplot.plot((0, m), (0, m), c="black")
    pyplot.xlabel(args[0])
    pyplot.ylabel(args[1])
    pyplot.savefig("scatter_intensities.png")
    pyplot.clf()

    pyplot.scatter(ma1.sigmas(), ma2.sigmas(), marker="+", alpha=0.5)
    m = max(pyplot.xlim()[1], pyplot.ylim()[1])
    pyplot.plot((0, m), (0, m), c="black")
    pyplot.savefig("scatter_sigmas.png")
    pyplot.clf()

    pyplot.scatter(
        flex.pow2(ma1.sigmas()), flex.pow2(ma2.sigmas()), marker="+", alpha=0.5
    )
    m = max(pyplot.xlim()[1], pyplot.ylim()[1])
    pyplot.plot((0, m), (0, m), c="black")
    pyplot.savefig("scatter_variances.png")
    pyplot.clf()

    isigi1 = ma1.data() / ma1.sigmas()
    isigi2 = ma2.data() / ma2.sigmas()
    pyplot.scatter(isigi1, isigi2, marker="+", alpha=0.5)
    m = max(pyplot.xlim()[1], pyplot.ylim()[1])
    pyplot.plot((0, m), (0, m), c="black")
    pyplot.savefig("scatter_i_sig_i.png")
    pyplot.clf()

    return