Exemplo n.º 1
0
def filter_ice(reflections, steps):

    from cctbx import miller, sgtbx, uctbx
    from matplotlib import pyplot as plt

    d_spacings = 1 / reflections["rlp"].norms()
    d_star_sq = uctbx.d_as_d_star_sq(d_spacings)

    from dials.algorithms.spot_finding.per_image_analysis import ice_rings_selection

    from dials.algorithms.integration import filtering

    ice_uc = uctbx.unit_cell((4.498, 4.498, 7.338, 90, 90, 120))
    ice_sg = sgtbx.space_group_info(number=194).group()
    ice_generator = miller.index_generator(ice_uc, ice_sg.type(), False,
                                           flex.min(d_spacings))
    ice_indices = ice_generator.to_array()
    ice_d_spacings = flex.sorted(ice_uc.d(ice_indices))
    ice_d_star_sq = uctbx.d_as_d_star_sq(ice_d_spacings)

    cubic_ice_uc = uctbx.unit_cell((6.358, 6.358, 6.358, 90, 90, 90))
    cubic_ice_sg = sgtbx.space_group_info(number=227).group()
    cubic_ice_generator = miller.index_generator(cubic_ice_uc,
                                                 cubic_ice_sg.type(), False,
                                                 flex.min(d_spacings))
    cubic_ice_indices = cubic_ice_generator.to_array()
    cubic_ice_d_spacings = flex.sorted(cubic_ice_uc.d(cubic_ice_indices))
    cubic_ice_d_star_sq = uctbx.d_as_d_star_sq(cubic_ice_d_spacings)

    import numpy

    widths = flex.double(numpy.geomspace(start=0.0001, stop=0.01, num=steps))
    n_spots = flex.double()
    total_intensity = flex.double()
    for width in widths:
        d_min = flex.min(d_spacings)

        ice_filter = filtering.PowderRingFilter(ice_uc, ice_sg, d_min, width)
        ice_sel = ice_filter(d_spacings)

        n_spots.append(ice_sel.count(False))
        if "intensity.sum.value" in reflections:
            total_intensity.append(
                flex.sum(reflections["intensity.sum.value"].select(~ice_sel)))

    fig, axes = plt.subplots(nrows=2, figsize=(12, 8), sharex=True)
    axes[0].plot(widths, n_spots, label="#spots", marker="+")
    if total_intensity.size():
        axes[1].plot(widths,
                     total_intensity,
                     label="total intensity",
                     marker="+")
    axes[0].set_ylabel("# spots remaining")
    axes[1].set_xlabel("Ice ring width (1/d^2)")
    axes[1].set_ylabel("Total intensity")
    for ax in axes:
        ax.set_xlim(0, flex.max(widths))
    plt.savefig("ice_ring_filtering.png")
    plt.clf()
    return
Exemplo n.º 2
0
 def __init__(
     self, dataset_statistics, anomalous_dataset_statistics, is_centric=False
 ):
     self.dataset_statistics = dataset_statistics
     self.anomalous_dataset_statistics = anomalous_dataset_statistics
     self.d_star_sq_bins = [
         0.5 * (uctbx.d_as_d_star_sq(b.d_max) + uctbx.d_as_d_star_sq(b.d_min))
         for b in self.dataset_statistics.bins
     ]
     self.d_star_sq_tickvals, self.d_star_sq_ticktext = d_star_sq_to_d_ticks(
         self.d_star_sq_bins, nticks=5
     )
     self.is_centric = is_centric
Exemplo n.º 3
0
    def plot_resolution_limit(self, d):
        from cctbx import uctbx

        d_star_sq = uctbx.d_as_d_star_sq(d)
        self.ax.plot([d_star_sq, d_star_sq],
                     self.ax.get_ylim(),
                     linestyle="--")
Exemplo n.º 4
0
def resolution_fit_from_merging_stats(merging_stats,
                                      metric,
                                      model,
                                      limit,
                                      sel=None):
    """Estimate a resolution limit based on the input `metric`

    The function defined by `model` will be fit to the selected `metric` which has been
    pre-calculated by the `merging_stats` object. The estimated resolution limit is
    chosen as the `d_star_sq` value at which the fitted function equals `limit`.

    Args:
        merging_stats (iotbx.merging_statistics.dataset_statistics): Pre-calculated
            merging statistics object
        metric (str): The metric to use for estimating a resolution limit. Must be a
            metric calculated by `iotbx.merging_statistics.merging_stats` and
            available as an attribute on the `bins` attribute of the input
            `merging_stats` object.
        model: The function to fit to the selected metric. Must be callable, taking as
            input x (d_star_sq) and y (the metric to be fitted) values, returning the
            fitted y(x) values.
        limit (float): The resolution limit criterion.
        sel (scitbx.array_family.flex.bool): An optional selection to apply to the
            `merging_stats` bins.

    Returns: The estimated resolution limit in units of Å^-1
    """

    y_obs = flex.double(getattr(b, metric)
                        for b in merging_stats.bins).reversed()
    d_star_sq = flex.double(
        uctbx.d_as_d_star_sq(b.d_min) for b in merging_stats.bins).reversed()
    return resolution_fit(d_star_sq, y_obs, model, limit, sel=sel)
Exemplo n.º 5
0
    def __init__(self, unit_cell, space_group, d_min, width):
        """
        Initialise the filter.

        :param unit_cell: The unit_cell of the powder rings
        :param space_group: The space group of the powder rings
        :param d_min: The maximum resolution to filter to
        :param width: The resolution width to filter around
        """
        assert d_min > 0
        assert width > 0

        # Correct unit cell
        unit_cell = space_group.average_unit_cell(unit_cell)

        self.half_width = width / 2.0
        d_min = uctbx.d_star_sq_as_d(
            uctbx.d_as_d_star_sq(d_min) + self.half_width)

        # Generate a load of indices
        generator = index_generator(unit_cell, space_group.type(), False,
                                    d_min)
        indices = generator.to_array()

        # Compute d spacings and sort by resolution
        self.d_star_sq = flex.sorted(unit_cell.d_star_sq(indices))
Exemplo n.º 6
0
def test_resolution_fit(merging_stats):
    d_star_sq = flex.double(
        uctbx.d_as_d_star_sq(b.d_min) for b in merging_stats.bins)
    y_obs = flex.double(b.r_merge for b in merging_stats.bins)
    result = resolution_analysis.resolution_fit(
        d_star_sq, y_obs, resolution_analysis.log_inv_fit, 0.6)
    assert result.d_min == pytest.approx(1.278, abs=1e-3)
    assert flex.max(flex.abs(result.y_obs - result.y_fit)) < 0.05
Exemplo n.º 7
0
 def plot_data(results, k, labels, linestyle=None):
     for i, result in enumerate(results):
         if labels is not None:
             label = labels[i]
         else:
             label = None
         bins = result.bins
         x = [bins[i].d_min for i in range(len(bins))]
         x = [uctbx.d_as_d_star_sq(d) for d in x]
         y = [getattr(bins[i], k) for i in range(len(bins))]
         pyplot.plot(x, y, label=label, linestyle=linestyle)
Exemplo n.º 8
0
    def __call__(self, d):
        """
        True if within powder ring.

        :param d: The resolution
        :return: True/False in powder ring
        """
        result = flex.bool(len(d), False)
        d_star_sq = uctbx.d_as_d_star_sq(d)
        for ds2 in self.d_star_sq:
            result = result | (flex.abs(d_star_sq - ds2) < self.half_width)
        return result
Exemplo n.º 9
0
    def count_ice_rings(self, width=0.002, verbose=False):
        """ A function to find and count ice rings (modeled after
        dials.algorithms.integration.filtering.PowderRingFilter, with some alterations:
            1. Hard-coded with ice unit cell / space group
            2. Returns spot counts vs. water diffraction resolution "bin"

        Rather than finding ice rings themselves (which may be laborious and time
        consuming), this method relies on counting the number of found spots that land
        in regions of water diffraction. A weakness of this approach is that if any spot
        filtering or spot-finding parameters are applied by prior methods, not all ice
        rings may be found. This is acceptable, since the purpose of this method is to
        determine if water and protein diffraction occupy the same resolutions.
        """
        ice_start = time.time()

        unit_cell = uctbx.unit_cell((4.498, 4.498, 7.338, 90, 90, 120))
        space_group = sgtbx.space_group_info(number=194).group()

        # Correct unit cell
        unit_cell = space_group.average_unit_cell(unit_cell)

        half_width = width / 2
        d_min = uctbx.d_star_sq_as_d(uctbx.d_as_d_star_sq(self.d_min) + half_width)

        # Generate a load of indices
        generator = index_generator(unit_cell, space_group.type(), False, d_min)
        indices = generator.to_array()

        # Compute d spacings and sort by resolution
        d_star_sq = flex.sorted(unit_cell.d_star_sq(indices))
        d = uctbx.d_star_sq_as_d(d_star_sq)
        dd = list(zip(d_star_sq, d))

        # identify if spots fall within ice ring areas
        results = []
        for ds2, d_res in dd:
            result = [i for i in (flex.abs(self.ref_d_star_sq - ds2) < half_width) if i]
            results.append((d_res, len(result)))

        possible_ice = [r for r in results if r[1] / len(self.observed) * 100 >= 5]

        if verbose:
            print(
                "SCORER: ice ring time = {:.5f} seconds".format(time.time() - ice_start)
            )

        self.n_ice_rings = len(possible_ice)  # output in info

        return self.n_ice_rings
Exemplo n.º 10
0
    def __call__(self, d):
        '''
    True if within powder ring.

    :param d: The resolution
    :return: True/False in powder ring

    '''
        from dials.array_family import flex
        from cctbx import uctbx
        result = flex.bool(len(d), False)
        d_star_sq = uctbx.d_as_d_star_sq(d)
        for ds2 in self.d_star_sq:
            result = result | (flex.abs(d_star_sq - ds2) < self.half_width)
        return result
Exemplo n.º 11
0
def plot_merging_stats(results, labels=None, plots=None, prefix=None,
                       size_inches=None, image_dir=None):
  import matplotlib
  matplotlib.use('Agg')
  from matplotlib import pyplot
  pyplot.style.use('ggplot')

  if plots is None:
    plots = ('r_merge', 'r_meas', 'r_pim', 'cc_one_half', 'i_over_sigma_mean')
  if prefix is None:
    prefix = ''
  if labels is not None:
    assert len(results) == len(labels)
  if image_dir is None:
    image_dir = '.'
  for k in plots:
    for i, result in enumerate(results):
      if labels is not None:
        label = labels[i]
      else:
        label = None
      bins = result.bins
      x = [bins[i].d_min for i in range(len(bins))]
      x = [uctbx.d_as_d_star_sq(d) for d in x]
      y = [getattr(bins[i], k) for i in range(len(bins))]
      pyplot.plot(x, y, label=label)
    pyplot.xlabel('d spacing')
    pyplot.ylabel(k)
    if k == 'cc_one_half':
      pyplot.ylim(0, 1.05)
    else:
      pyplot.ylim(0, pyplot.ylim()[1])
    ax = pyplot.gca()
    xticks = ax.get_xticks()
    xticks_d = [
      '%.2f' %uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks]
    ax.set_xticklabels(xticks_d)
    if size_inches is not None:
      fig = pyplot.gcf()
      fig.set_size_inches(size_inches)
    if labels is not None:
      pyplot.legend(loc='best')
    pyplot.tight_layout()
    pyplot.savefig(os.path.join(image_dir, prefix+ k + '.png'))
    pyplot.clf()
Exemplo n.º 12
0
def exercise_functions():
    d_star_sq_s = 1.2345
    d_star_sq_a = flex.double((1.2345, 0.1234))
    two_theta_s = 0.61725
    two_theta_a = flex.double((0.61725, 0.65432))
    # forward conversions
    assert approx_equal(uctbx.d_star_sq_as_stol_sq(d_star_sq_s),
                        d_star_sq_s / 4)
    assert approx_equal(uctbx.d_star_sq_as_stol_sq(d_star_sq_a),
                        [uctbx.d_star_sq_as_stol_sq(i) for i in d_star_sq_a])
    assert approx_equal(
        uctbx.d_star_sq_as_two_stol(d_star_sq_s)**2, d_star_sq_s)
    assert approx_equal(uctbx.d_star_sq_as_two_stol(d_star_sq_a),
                        [uctbx.d_star_sq_as_two_stol(i) for i in d_star_sq_a])
    assert approx_equal(
        uctbx.d_star_sq_as_stol(d_star_sq_s)**2, d_star_sq_s / 4)
    assert approx_equal(uctbx.d_star_sq_as_stol(d_star_sq_a),
                        [uctbx.d_star_sq_as_stol(i) for i in d_star_sq_a])
    assert approx_equal(1 / (uctbx.d_star_sq_as_d(d_star_sq_s)**2),
                        d_star_sq_s)
    assert approx_equal(uctbx.d_star_sq_as_d(d_star_sq_a),
                        [uctbx.d_star_sq_as_d(i) for i in d_star_sq_a])
    assert approx_equal(uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5),
                        2 * asin(1.5 / 2 * sqrt(d_star_sq_s)))
    assert approx_equal(uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5),
                        uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5, False))
    assert approx_equal(
        uctbx.d_star_sq_as_two_theta(d_star_sq_a, 1.5),
        [uctbx.d_star_sq_as_two_theta(i, 1.5) for i in d_star_sq_a])
    assert approx_equal(
        uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5) * 180 / pi,
        uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5, True))
    # reverse conversions
    for d_star_sq, two_theta in zip((d_star_sq_s, d_star_sq_a),
                                    (two_theta_s, two_theta_a)):
        assert approx_equal(
            uctbx.stol_sq_as_d_star_sq(uctbx.d_star_sq_as_stol_sq(d_star_sq)),
            d_star_sq)
        assert approx_equal(
            uctbx.two_stol_as_d_star_sq(
                uctbx.d_star_sq_as_two_stol(d_star_sq)), d_star_sq)
        assert approx_equal(
            uctbx.stol_as_d_star_sq(uctbx.d_star_sq_as_stol(d_star_sq)),
            d_star_sq)
        assert approx_equal(
            uctbx.d_as_d_star_sq(uctbx.d_star_sq_as_d(d_star_sq)), d_star_sq)
        assert approx_equal(
            uctbx.two_theta_as_d_star_sq(
                uctbx.d_star_sq_as_two_theta(d_star_sq, 1.5), 1.5), d_star_sq)
        assert approx_equal(
            uctbx.two_theta_as_d_star_sq(two_theta, 1.5),
            uctbx.two_theta_as_d_star_sq(two_theta, 1.5, False))
        assert approx_equal(
            uctbx.two_theta_as_d_star_sq(two_theta, 1.5),
            uctbx.two_theta_as_d_star_sq(two_theta * 180 / pi, 1.5, True))
        assert approx_equal(
            uctbx.two_theta_as_d(two_theta, 1.5),
            uctbx.d_star_sq_as_d(uctbx.two_theta_as_d_star_sq(two_theta, 1.5)))
        assert approx_equal(
            uctbx.two_theta_as_d(two_theta, 1.5, True),
            uctbx.d_star_sq_as_d(
                uctbx.two_theta_as_d_star_sq(two_theta, 1.5, True)))
    #
    assert uctbx.fractional_unit_shifts(distance_frac=[0, 0, 0]) == (0, 0, 0)
    assert uctbx.fractional_unit_shifts([0.6, 7.4, -0.4]) == (1, 7, 0)
    assert uctbx.fractional_unit_shifts([-6, 3, -0.6]) == (-6, 3, -1)
    site_frac_1 = [0.3, -8.6, 2.1]
    for eps, expected_u2 in [(-1.e-5, 1), (1.e-5, 2)]:
        site_frac_2 = [-3, 5.6, 0.6 - eps]
        assert uctbx.fractional_unit_shifts(
            site_frac_1=site_frac_1,
            site_frac_2=site_frac_2) == (3, -14, expected_u2)
Exemplo n.º 13
0
def run(args):
    from dials.util.options import OptionParser
    import libtbx.load_env

    usage = "%s [options] find_spots.json" % (libtbx.env.dispatcher_name)

    parser = OptionParser(usage=usage, phil=phil_scope, epilog=help_message)

    params, options, args = parser.parse_args(show_diff_phil=True,
                                              return_unhandled=True)

    positions = None
    if params.positions is not None:
        with open(params.positions, "rb") as f:
            positions = flex.vec2_double()
            for line in f.readlines():
                line = (line.replace("(", " ").replace(")", "").replace(
                    ",", " ").strip().split())
                assert len(line) == 3
                i, x, y = [float(l) for l in line]
                positions.append((x, y))

    assert len(args) == 1
    json_file = args[0]
    import json

    with open(json_file, "rb") as f:
        results = json.load(f)

    n_indexed = flex.double()
    fraction_indexed = flex.double()
    n_spots = flex.double()
    n_lattices = flex.double()
    crystals = []
    image_names = flex.std_string()

    for r in results:
        n_spots.append(r["n_spots_total"])
        image_names.append(str(r["image"]))
        if "n_indexed" in r:
            n_indexed.append(r["n_indexed"])
            fraction_indexed.append(r["fraction_indexed"])
            n_lattices.append(len(r["lattices"]))
            for d in r["lattices"]:
                from dxtbx.serialize.crystal import from_dict

                crystals.append(from_dict(d["crystal"]))
        else:
            n_indexed.append(0)
            fraction_indexed.append(0)
            n_lattices.append(0)

    import matplotlib

    matplotlib.use("Agg")
    from matplotlib import pyplot

    blue = "#3498db"
    red = "#e74c3c"

    marker = "o"
    alpha = 0.5
    lw = 0

    plot = True
    table = True
    grid = params.grid

    from libtbx import group_args
    from dials.algorithms.peak_finding.per_image_analysis import plot_stats, print_table

    estimated_d_min = flex.double()
    d_min_distl_method_1 = flex.double()
    d_min_distl_method_2 = flex.double()
    n_spots_total = flex.int()
    n_spots_no_ice = flex.int()
    total_intensity = flex.double()

    for d in results:
        estimated_d_min.append(d["estimated_d_min"])
        d_min_distl_method_1.append(d["d_min_distl_method_1"])
        d_min_distl_method_2.append(d["d_min_distl_method_2"])
        n_spots_total.append(d["n_spots_total"])
        n_spots_no_ice.append(d["n_spots_no_ice"])
        total_intensity.append(d["total_intensity"])

    stats = group_args(
        image=image_names,
        n_spots_total=n_spots_total,
        n_spots_no_ice=n_spots_no_ice,
        n_spots_4A=None,
        total_intensity=total_intensity,
        estimated_d_min=estimated_d_min,
        d_min_distl_method_1=d_min_distl_method_1,
        d_min_distl_method_2=d_min_distl_method_2,
        noisiness_method_1=None,
        noisiness_method_2=None,
    )

    if plot:
        plot_stats(stats)
        pyplot.clf()
    if table:
        print_table(stats)

    print("Number of indexed lattices: ", (n_indexed > 0).count(True))

    print(
        "Number with valid d_min but failed indexing: ",
        ((d_min_distl_method_1 > 0)
         & (d_min_distl_method_2 > 0)
         & (estimated_d_min > 0)
         & (n_indexed == 0)).count(True),
    )

    n_rows = 10
    n_rows = min(n_rows, len(n_spots_total))
    perm_n_spots_total = flex.sort_permutation(n_spots_total, reverse=True)
    print("Top %i images sorted by number of spots:" % n_rows)
    print_table(stats, perm=perm_n_spots_total, n_rows=n_rows)

    n_bins = 20
    spot_count_histogram(n_spots_total,
                         n_bins=n_bins,
                         filename="hist_n_spots_total.png",
                         log=True)
    spot_count_histogram(n_spots_no_ice,
                         n_bins=n_bins,
                         filename="hist_n_spots_no_ice.png",
                         log=True)
    spot_count_histogram(
        n_indexed.select(n_indexed > 0),
        n_bins=n_bins,
        filename="hist_n_indexed.png",
        log=False,
    )

    if len(crystals):
        plot_unit_cell_histograms(crystals)

    if params.stereographic_projections and len(crystals):
        from dxtbx.datablock import DataBlockFactory

        datablocks = DataBlockFactory.from_filenames([image_names[0]],
                                                     verbose=False)
        assert len(datablocks) == 1
        imageset = datablocks[0].extract_imagesets()[0]
        s0 = imageset.get_beam().get_s0()
        # XXX what if no goniometer?
        rotation_axis = imageset.get_goniometer().get_rotation_axis()

        indices = ((1, 0, 0), (0, 1, 0), (0, 0, 1))
        for i, index in enumerate(indices):

            from cctbx import crystal, miller
            from scitbx import matrix

            miller_indices = flex.miller_index([index])
            symmetry = crystal.symmetry(
                unit_cell=crystals[0].get_unit_cell(),
                space_group=crystals[0].get_space_group(),
            )
            miller_set = miller.set(symmetry, miller_indices)
            d_spacings = miller_set.d_spacings()
            d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1()
            d_spacings = d_spacings.generate_bijvoet_mates()
            miller_indices = d_spacings.indices()

            # plane normal
            d0 = matrix.col(s0).normalize()
            d1 = d0.cross(matrix.col(rotation_axis)).normalize()
            d2 = d1.cross(d0).normalize()
            reference_poles = (d0, d1, d2)

            from dials.command_line.stereographic_projection import (
                stereographic_projection, )

            projections = []

            for cryst in crystals:
                reciprocal_space_points = (
                    list(cryst.get_U() * cryst.get_B()) *
                    miller_indices.as_vec3_double())
                projections.append(
                    stereographic_projection(reciprocal_space_points,
                                             reference_poles))

                # from dials.algorithms.indexing.compare_orientation_matrices import \
                #  difference_rotation_matrix_and_euler_angles
                # R_ij, euler_angles, cb_op = difference_rotation_matrix_and_euler_angles(
                #  crystals[0], cryst)
                # print max(euler_angles)

            from dials.command_line.stereographic_projection import plot_projections

            plot_projections(projections,
                             filename="projections_%s.png" % ("hkl"[i]))
            pyplot.clf()

    def plot_grid(
        values,
        grid,
        file_name,
        cmap=pyplot.cm.Reds,
        vmin=None,
        vmax=None,
        invalid="white",
    ):
        values = values.as_double()
        # At DLS, fast direction appears to be largest direction
        if grid[0] > grid[1]:
            values.reshape(flex.grid(reversed(grid)))
            values = values.matrix_transpose()
        else:
            values.reshape(flex.grid(grid))

        Z = values.as_numpy_array()

        # f, (ax1, ax2) = pyplot.subplots(2)
        f, ax1 = pyplot.subplots(1)

        mesh1 = ax1.pcolormesh(values.as_numpy_array(),
                               cmap=cmap,
                               vmin=vmin,
                               vmax=vmax)
        mesh1.cmap.set_under(color=invalid, alpha=None)
        mesh1.cmap.set_over(color=invalid, alpha=None)
        # mesh2 = ax2.contour(Z, cmap=cmap, vmin=vmin, vmax=vmax)
        # mesh2 = ax2.contourf(Z, cmap=cmap, vmin=vmin, vmax=vmax)
        ax1.set_aspect("equal")
        ax1.invert_yaxis()
        # ax2.set_aspect('equal')
        # ax2.invert_yaxis()
        pyplot.colorbar(mesh1, ax=ax1)
        # pyplot.colorbar(mesh2, ax=ax2)
        pyplot.savefig(file_name, dpi=600)
        pyplot.clf()

    def plot_positions(
        values,
        positions,
        file_name,
        cmap=pyplot.cm.Reds,
        vmin=None,
        vmax=None,
        invalid="white",
    ):
        values = values.as_double()
        assert positions.size() >= values.size()
        positions = positions[:values.size()]

        if vmin is None:
            vmin = flex.min(values)
        if vmax is None:
            vmax = flex.max(values)

        x, y = positions.parts()
        dx = flex.abs(x[1:] - x[:-1])
        dy = flex.abs(y[1:] - y[:-1])
        dx = dx.select(dx > 0)
        dy = dy.select(dy > 0)

        scale = 1 / flex.min(dx)
        # print scale
        x = (x * scale).iround()
        y = (y * scale).iround()

        from libtbx.math_utils import iceil

        z = flex.double(
            flex.grid(iceil(flex.max(y)) + 1,
                      iceil(flex.max(x)) + 1), -2)
        # print z.all()
        for x_, y_, z_ in zip(x, y, values):
            z[y_, x_] = z_

        plot_grid(
            z.as_1d(),
            z.all(),
            file_name,
            cmap=cmap,
            vmin=vmin,
            vmax=vmax,
            invalid=invalid,
        )
        return

    if grid is not None or positions is not None:
        if grid is not None:
            positions = tuple(reversed(grid))
            plotter = plot_grid
        else:
            plotter = plot_positions

        cmap = pyplot.get_cmap(params.cmap)
        plotter(
            n_spots_total,
            positions,
            "grid_spot_count_total.png",
            cmap=cmap,
            invalid=params.invalid,
        )
        plotter(
            n_spots_no_ice,
            positions,
            "grid_spot_count_no_ice.png",
            cmap=cmap,
            invalid=params.invalid,
        )
        plotter(
            total_intensity,
            positions,
            "grid_total_intensity.png",
            cmap=cmap,
            invalid=params.invalid,
        )
        if flex.max(n_indexed) > 0:
            plotter(
                n_indexed,
                positions,
                "grid_n_indexed.png",
                cmap=cmap,
                invalid=params.invalid,
            )
            plotter(
                fraction_indexed,
                positions,
                "grid_fraction_indexed.png",
                cmap=cmap,
                vmin=0,
                vmax=1,
                invalid=params.invalid,
            )

        for i, d_min in enumerate(
            (estimated_d_min, d_min_distl_method_1, d_min_distl_method_2)):
            from cctbx import uctbx

            d_star_sq = uctbx.d_as_d_star_sq(d_min)
            d_star_sq.set_selected(d_star_sq == 1, 0)
            vmin = flex.min(d_star_sq.select(d_star_sq > 0))
            vmax = flex.max(d_star_sq)

            vmin = flex.min(d_min.select(d_min > 0))
            vmax = flex.max(d_min)
            cmap = pyplot.get_cmap("%s_r" % params.cmap)
            d_min.set_selected(d_min <= 0, vmax)

            if i == 0:
                plotter(
                    d_min,
                    positions,
                    "grid_d_min.png",
                    cmap=cmap,
                    vmin=vmin,
                    vmax=vmax,
                    invalid=params.invalid,
                )
            else:
                plotter(
                    d_min,
                    positions,
                    "grid_d_min_method_%i.png" % i,
                    cmap=cmap,
                    vmin=vmin,
                    vmax=vmax,
                    invalid=params.invalid,
                )

    if flex.max(n_indexed) > 0:
        pyplot.hexbin(n_spots,
                      n_indexed,
                      bins="log",
                      cmap=pyplot.cm.jet,
                      gridsize=50)
        pyplot.colorbar()
        # pyplot.scatter(n_spots, n_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
        xlim = pyplot.xlim()
        ylim = pyplot.ylim()
        pyplot.plot([0, max(n_spots)], [0, max(n_spots)], c=red)
        pyplot.xlim(0, xlim[1])
        pyplot.ylim(0, ylim[1])
        pyplot.xlabel("# spots")
        pyplot.ylabel("# indexed")
        pyplot.savefig("n_spots_vs_n_indexed.png")
        pyplot.clf()

        pyplot.hexbin(n_spots,
                      fraction_indexed,
                      bins="log",
                      cmap=pyplot.cm.jet,
                      gridsize=50)
        pyplot.colorbar()
        # pyplot.scatter(
        # n_spots, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
        pyplot.xlim(0, pyplot.xlim()[1])
        pyplot.ylim(0, pyplot.ylim()[1])
        pyplot.xlabel("# spots")
        pyplot.ylabel("Fraction indexed")
        pyplot.savefig("n_spots_vs_fraction_indexed.png")
        pyplot.clf()

        pyplot.hexbin(n_indexed,
                      fraction_indexed,
                      bins="log",
                      cmap=pyplot.cm.jet,
                      gridsize=50)
        pyplot.colorbar()
        # pyplot.scatter(
        # n_indexed, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
        pyplot.xlim(0, pyplot.xlim()[1])
        pyplot.ylim(0, pyplot.ylim()[1])
        pyplot.xlabel("# indexed")
        pyplot.ylabel("Fraction indexed")
        pyplot.savefig("n_indexed_vs_fraction_indexed.png")
        pyplot.clf()

        pyplot.hexbin(n_spots,
                      n_lattices,
                      bins="log",
                      cmap=pyplot.cm.jet,
                      gridsize=50)
        pyplot.colorbar()
        # pyplot.scatter(
        # n_spots, n_lattices, marker=marker, alpha=alpha, c=blue, lw=lw)
        pyplot.xlim(0, pyplot.xlim()[1])
        pyplot.ylim(0, pyplot.ylim()[1])
        pyplot.xlabel("# spots")
        pyplot.ylabel("# lattices")
        pyplot.savefig("n_spots_vs_n_lattices.png")
        pyplot.clf()

    # pyplot.scatter(
    #  estimated_d_min, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw)
    pyplot.hexbin(
        estimated_d_min,
        d_min_distl_method_1,
        bins="log",
        cmap=pyplot.cm.jet,
        gridsize=50,
    )
    pyplot.colorbar()
    # pyplot.gca().set_aspect('equal')
    xlim = pyplot.xlim()
    ylim = pyplot.ylim()
    m = max(max(estimated_d_min), max(d_min_distl_method_1))
    pyplot.plot([0, m], [0, m], c=red)
    pyplot.xlim(0, xlim[1])
    pyplot.ylim(0, ylim[1])
    pyplot.xlabel("estimated_d_min")
    pyplot.ylabel("d_min_distl_method_1")
    pyplot.savefig("d_min_vs_distl_method_1.png")
    pyplot.clf()

    # pyplot.scatter(
    #  estimated_d_min, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
    pyplot.hexbin(
        estimated_d_min,
        d_min_distl_method_2,
        bins="log",
        cmap=pyplot.cm.jet,
        gridsize=50,
    )
    pyplot.colorbar()
    # pyplot.gca().set_aspect('equal')
    xlim = pyplot.xlim()
    ylim = pyplot.ylim()
    m = max(max(estimated_d_min), max(d_min_distl_method_2))
    pyplot.plot([0, m], [0, m], c=red)
    pyplot.xlim(0, xlim[1])
    pyplot.ylim(0, ylim[1])
    pyplot.xlabel("estimated_d_min")
    pyplot.ylabel("d_min_distl_method_2")
    pyplot.savefig("d_min_vs_distl_method_2.png")
    pyplot.clf()

    # pyplot.scatter(
    #  d_min_distl_method_1, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
    pyplot.hexbin(
        d_min_distl_method_1,
        d_min_distl_method_2,
        bins="log",
        cmap=pyplot.cm.jet,
        gridsize=50,
    )
    pyplot.colorbar()
    # pyplot.gca().set_aspect('equal')
    xlim = pyplot.xlim()
    ylim = pyplot.ylim()
    m = max(max(d_min_distl_method_1), max(d_min_distl_method_2))
    pyplot.plot([0, m], [0, m], c=red)
    pyplot.xlim(0, xlim[1])
    pyplot.ylim(0, ylim[1])
    pyplot.xlabel("d_min_distl_method_1")
    pyplot.ylabel("d_min_distl_method_2")
    pyplot.savefig("distl_method_1_vs_distl_method_2.png")
    pyplot.clf()

    pyplot.hexbin(n_spots,
                  estimated_d_min,
                  bins="log",
                  cmap=pyplot.cm.jet,
                  gridsize=50)
    pyplot.colorbar()
    # pyplot.scatter(
    # n_spots, estimated_d_min, marker=marker, alpha=alpha, c=blue, lw=lw)
    pyplot.xlim(0, pyplot.xlim()[1])
    pyplot.ylim(0, pyplot.ylim()[1])
    pyplot.xlabel("# spots")
    pyplot.ylabel("estimated_d_min")
    pyplot.savefig("n_spots_vs_d_min.png")
    pyplot.clf()

    pyplot.hexbin(n_spots,
                  d_min_distl_method_1,
                  bins="log",
                  cmap=pyplot.cm.jet,
                  gridsize=50)
    pyplot.colorbar()
    # pyplot.scatter(
    # n_spots, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw)
    pyplot.xlim(0, pyplot.xlim()[1])
    pyplot.ylim(0, pyplot.ylim()[1])
    pyplot.xlabel("# spots")
    pyplot.ylabel("d_min_distl_method_1")
    pyplot.savefig("n_spots_vs_distl_method_1.png")
    pyplot.clf()

    pyplot.hexbin(n_spots,
                  d_min_distl_method_2,
                  bins="log",
                  cmap=pyplot.cm.jet,
                  gridsize=50)
    pyplot.colorbar()
    # pyplot.scatter(
    # n_spots, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
    pyplot.xlim(0, pyplot.xlim()[1])
    pyplot.ylim(0, pyplot.ylim()[1])
    pyplot.xlabel("# spots")
    pyplot.ylabel("d_min_distl_method_2")
    pyplot.savefig("n_spots_vs_distl_method_2.png")
    pyplot.clf()
Exemplo n.º 14
0
def test_ResolutionPlotsAndStats(iobs):
    i_obs_anom = iobs.as_anomalous_array()
    iobs_anom = i_obs_anom.map_to_asu().customized_copy(info=iobs.info())
    n_bins = 2
    result = dataset_statistics(
        iobs, assert_is_not_unique_set_under_symmetry=False, n_bins=n_bins
    )
    anom_result = dataset_statistics(
        iobs_anom,
        assert_is_not_unique_set_under_symmetry=False,
        anomalous=True,
        n_bins=n_bins,
    )
    plotter = ResolutionPlotsAndStats(result, anom_result)

    assert plotter.d_star_sq_ticktext == ["1.74", "1.53", "1.38", "1.27", "1.18"]

    assert plotter.d_star_sq_tickvals == pytest.approx(
        [
            0.32984033277048164,
            0.42706274943714834,
            0.524285166103815,
            0.6215075827704818,
            0.7187299994371485,
        ],
        1e-4,
    )

    tables = plotter.statistics_tables()
    assert len(tables) == 2  # overall and per resolution

    # test plots individually
    d = plotter.cc_one_half_plot()
    assert len(d["cc_one_half"]["data"]) == 6
    assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"][:4])
    d["cc_one_half"]["data"][0]["x"] == [
        0.5 * (uctbx.d_as_d_star_sq(b.d_max) + uctbx.d_as_d_star_sq(b.d_min))
        for b in result.bins
    ]

    d = plotter.i_over_sig_i_plot()
    assert len(d["i_over_sig_i"]["data"]) == 1
    assert len(d["i_over_sig_i"]["data"][0]["y"]) == n_bins

    d = plotter.r_pim_plot()
    assert len(d["r_pim"]["data"]) == 1
    assert len(d["r_pim"]["data"][0]["y"]) == n_bins

    d = plotter.completeness_plot()
    assert len(d["completeness"]["data"]) == 2
    assert len(d["completeness"]["data"][0]["y"]) == n_bins

    d = plotter.multiplicity_vs_resolution_plot()
    assert len(d["multiplicity_vs_resolution"]["data"]) == 2
    assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins

    # now try centric options and sigma tau for cc_one_half
    plotter = ResolutionPlotsAndStats(result, anom_result, is_centric=True)
    d = plotter.cc_one_half_plot(method="sigma_tau")
    assert len(d["cc_one_half"]["data"]) == 6
    assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"][:2])
    assert d["cc_one_half"]["data"][2] == {}  # no anomalous plots
    assert d["cc_one_half"]["data"][3] == {}  # no anomalous plots
    assert d["cc_one_half"]["data"][4] == {}  # no cc_fit
    assert d["cc_one_half"]["data"][5] == {}  # no d_min
    d = plotter.completeness_plot()
    assert len(d["completeness"]["data"]) == 2
    assert len(d["completeness"]["data"][0]["y"]) == n_bins
    assert d["completeness"]["data"][1] == {}
    d = plotter.multiplicity_vs_resolution_plot()
    assert len(d["multiplicity_vs_resolution"]["data"]) == 2
    assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins
    assert d["multiplicity_vs_resolution"]["data"][1] == {}

    plots = plotter.make_all_plots()
    assert list(plots.keys()) == [
        "cc_one_half",
        "i_over_sig_i",
        "completeness",
        "multiplicity_vs_resolution",
        "r_pim",
    ]
    for plot in plots.values():
        assert plot["layout"]["xaxis"]["ticktext"] == plotter.d_star_sq_ticktext
        assert plot["layout"]["xaxis"]["tickvals"] == plotter.d_star_sq_tickvals
Exemplo n.º 15
0
def plot_data(
    results,
    k,
    ylabel,
    labels,
    linestyle=None,
    n_rows=None,
    n_cols=None,
    global_labels=None,
    alpha=0.3,
):
    from matplotlib import pyplot as plt

    colors = plt.rcParams["axes.prop_cycle"].by_key()["color"]

    ref_ax = None
    for i, result in enumerate(results):
        if not isinstance(result, (list, tuple)):
            result = (result, )
        if labels is not None:
            label = labels[i].replace("\\$", "$")
        else:
            label = None
        if n_cols > 1:
            ax = plt.subplot(n_rows,
                             n_cols,
                             i + 1,
                             sharex=ref_ax,
                             sharey=ref_ax)
            if label:
                ax.set_title(label, loc="left")
            if ref_ax is None:
                ref_ax = ax
            for other in results:
                if isinstance(other,
                              iotbx.merging_statistics.dataset_statistics):
                    other = (other, )
                for res in other:
                    if res is not None:
                        x = [
                            0.5 * (uctbx.d_as_d_star_sq(b.d_max) +
                                   uctbx.d_as_d_star_sq(b.d_min))
                            for b in res.bins
                        ]
                        y = [getattr(b, k) for b in res.bins]
                        ax.plot(
                            x,
                            y,
                            linestyle="-",
                            color="grey",
                            linewidth=1,
                            alpha=alpha,
                        )
        else:
            ax = plt.gca()

        for i_res, res in enumerate(result):
            if res is not None:
                if global_labels is not None:
                    l = global_labels[i_res]
                else:
                    l = label
                x = [
                    0.5 * (uctbx.d_as_d_star_sq(b.d_max) +
                           uctbx.d_as_d_star_sq(b.d_min)) for b in res.bins
                ]
                y = [getattr(b, k) for b in res.bins]
                color = colors[i_res] if n_cols > 1 else colors[i]
                ax.plot(x, y, label=l, linestyle=linestyle, color=color)

        if n_cols > 1 or i == len(results) - 1:
            ax.set_xlabel(r"Resolution ($\AA$)")
            ax.set_ylabel(ylabel)
            ax.label_outer()
            if k in ("cc_one_half", "cc_one_half_sigma_tau", "completeness"):
                ax.set_ylim(0, 1.05)
            elif k in ("cc_anom", ):
                ax.set_ylim(min(0, ax.get_ylim()[0]), 1.05)
            elif k in ("r_merge", ):
                ax.set_ylim(0, min(4, ax.get_ylim()[1]))
            elif k in ("r_meas", "r_pim"):
                ax.set_ylim(0, min(2, ax.get_ylim()[1]))
            else:
                ax.set_ylim(0)
            xticks = ax.get_xticks()
            xticks_d = [
                "%.2f" % uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0
                for ds2 in xticks
            ]
            ax.set_xticklabels(xticks_d)
Exemplo n.º 16
0
def estimate_resolution_limit(reflections, ice_sel=None, plot_filename=None):
    if ice_sel is None:
        ice_sel = flex.bool(len(reflections), False)

    d_star_sq = flex.pow2(reflections["rlp"].norms())
    d_spacings = uctbx.d_star_sq_as_d(d_star_sq)

    intensities = reflections["intensity.sum.value"]
    variances = reflections["intensity.sum.variance"]

    sel = variances > 0
    intensities = intensities.select(sel)
    variances = variances.select(sel)
    ice_sel = ice_sel.select(sel)

    i_over_sigi = intensities / flex.sqrt(variances)
    log_i_over_sigi = flex.log(i_over_sigi)

    fit = flex.linear_regression(d_star_sq.select(~ice_sel),
                                 log_i_over_sigi.select(~ice_sel))
    m = fit.slope()
    c = fit.y_intercept()

    log_i_sigi_lower = flex.double()
    d_star_sq_lower = flex.double()
    log_i_sigi_upper = flex.double()
    d_star_sq_upper = flex.double()

    binner = binner_equal_population(d_star_sq,
                                     target_n_per_bin=20,
                                     max_slots=20,
                                     min_slots=5)

    outliers_all = flex.bool(len(reflections), False)

    low_percentile_limit = 0.1
    upper_percentile_limit = 1 - low_percentile_limit
    for i_slot, slot in enumerate(binner.bins):
        sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
        sel = ~(ice_sel) & sel_all

        if sel.count(True) == 0:
            continue

        outliers = wilson_outliers(reflections.select(sel_all),
                                   ice_sel=ice_sel.select(sel_all))
        outliers_all.set_selected(sel_all, outliers)

        isel = sel_all.iselection().select(~(outliers)
                                           & ~(ice_sel).select(sel_all))
        log_i_over_sigi_sel = log_i_over_sigi.select(isel)
        d_star_sq_sel = d_star_sq.select(isel)

        perm = flex.sort_permutation(log_i_over_sigi_sel)
        i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
        i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
        log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
        log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
        d_star_sq_upper.append(d_star_sq_sel[i_lower])
        d_star_sq_lower.append(d_star_sq_sel[i_upper])

    fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
    m_upper = fit_upper.slope()
    c_upper = fit_upper.y_intercept()
    fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
    m_lower = fit_lower.slope()
    c_lower = fit_lower.y_intercept()

    if m_upper == m_lower:
        intersection = (-1, -1)
        resolution_estimate = -1
        inside = flex.bool(len(d_star_sq), False)

    else:
        # http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
        # with:
        # a_ = m_upper
        # b_ = m_lower
        # c_ = c_upper
        # d_ = c_lower
        # intersection == ((d_ - c_) / (a_ - b_), (a_ * d_ - b_ * c_) / (a_ - b_))
        intersection = (
            (c_lower - c_upper) / (m_upper - m_lower),
            (m_upper * c_lower - m_lower * c_upper) / (m_upper - m_lower),
        )

        inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper,
                                   c_upper)
        inside = inside & ~outliers_all & ~ice_sel

        if inside.count(True) > 0:
            d_star_sq_estimate = flex.max(d_star_sq.select(inside))
            resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
        else:
            resolution_estimate = -1

    if plot_filename is not None:
        from matplotlib import pyplot

        fig = pyplot.figure()
        ax = fig.add_subplot(1, 1, 1)
        ax.scatter(d_star_sq, log_i_over_sigi, marker="+")
        ax.scatter(
            d_star_sq.select(inside),
            log_i_over_sigi.select(inside),
            marker="+",
            color="green",
        )
        ax.scatter(
            d_star_sq.select(ice_sel),
            log_i_over_sigi.select(ice_sel),
            marker="+",
            color="black",
        )
        ax.scatter(
            d_star_sq.select(outliers_all),
            log_i_over_sigi.select(outliers_all),
            marker="+",
            color="grey",
        )
        ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker="+", color="red")
        ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker="+", color="red")

        if intersection[0] <= ax.get_xlim(
        )[1] and intersection[1] <= ax.get_ylim()[1]:
            ax.scatter([intersection[0]], [intersection[1]],
                       marker="x",
                       s=50,
                       color="b")
        xlim = pyplot.xlim()
        ax.plot(xlim, [(m * x + c) for x in xlim])
        ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color="red")
        ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color="red")
        ax.set_xlabel("d_star_sq")
        ax.set_ylabel("ln(I/sigI)")
        ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
        ax.set_ylim((0, ax.get_ylim()[1]))

        for i_slot, slot in enumerate(binner.bins):
            if i_slot == 0:
                ax.vlines(
                    uctbx.d_as_d_star_sq(slot.d_max),
                    0,
                    ax.get_ylim()[1],
                    linestyle="dotted",
                    color="grey",
                )
            ax.vlines(
                uctbx.d_as_d_star_sq(slot.d_min),
                0,
                ax.get_ylim()[1],
                linestyle="dotted",
                color="grey",
            )

        ax_ = ax.twiny()  # ax2 is responsible for "top" axis and "right" axis
        xticks = ax.get_xticks()
        xticks_d = [
            uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
        ]
        ax_.set_xticks(xticks)
        ax_.set_xlim(ax.get_xlim())
        ax_.set_xlabel(r"Resolution ($\AA$)")
        ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
        pyplot.savefig(plot_filename)
        pyplot.close()

    return resolution_estimate
Exemplo n.º 17
0
def estimate_resolution_limit(reflections,
                              imageset,
                              ice_sel=None,
                              plot_filename=None):

    if ice_sel is None:
        ice_sel = flex.bool(len(reflections), False)

    d_star_sq = flex.pow2(reflections['rlp'].norms())
    d_spacings = uctbx.d_star_sq_as_d(d_star_sq)

    intensities = reflections['intensity.sum.value']
    variances = reflections['intensity.sum.variance']

    sel = variances > 0
    intensities = intensities.select(sel)
    variances = variances.select(sel)
    ice_sel = ice_sel.select(sel)

    i_over_sigi = intensities / flex.sqrt(variances)
    log_i_over_sigi = flex.log(i_over_sigi)

    fit = flex.linear_regression(d_star_sq.select(~ice_sel),
                                 log_i_over_sigi.select(~ice_sel))
    m = fit.slope()
    c = fit.y_intercept()

    log_i_sigi_lower = flex.double()
    d_star_sq_lower = flex.double()
    log_i_sigi_upper = flex.double()
    d_star_sq_upper = flex.double()

    binner = binner_equal_population(d_star_sq,
                                     target_n_per_bin=20,
                                     max_slots=20,
                                     min_slots=5)

    outliers_all = flex.bool(len(reflections), False)

    low_percentile_limit = 0.1
    upper_percentile_limit = 1 - low_percentile_limit
    d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
    for i_slot, slot in enumerate(binner.bins):
        sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
        sel = ~(ice_sel) & sel_all
        #sel = ~(ice_sel) & (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)

        #print "%.2f" %(sel.count(True)/sel_all.count(True))

        if sel.count(True) == 0:
            #outliers_all.set_selected(sel_all & ice_sel, True)
            continue
            #if i_slot > i_slot_max:
            #break
            #else:
            #continue

        outliers = wilson_outliers(reflections.select(sel_all),
                                   ice_sel=ice_sel.select(sel_all))
        #print "rejecting %d wilson outliers" %outliers.count(True)
        outliers_all.set_selected(sel_all, outliers)

        #if sel.count(True)/sel_all.count(True) < 0.25:
        #outliers_all.set_selected(sel_all & ice_sel, True)

        #from scitbx.math import median_statistics
        #intensities_sel = intensities.select(sel)
        #stats = median_statistics(intensities_sel)
        #z_score = 0.6745 * (intensities_sel - stats.median)/stats.median_absolute_deviation
        #outliers = z_score > 3.5
        #perm = flex.sort_permutation(intensities_sel)
        ##print ' '.join('%.2f' %v for v in intensities_sel.select(perm))
        ##print ' '.join('%.2f' %v for v in z_score.select(perm))
        ##print

        isel = sel_all.iselection().select(~(outliers)
                                           & ~(ice_sel).select(sel_all))
        log_i_over_sigi_sel = log_i_over_sigi.select(isel)
        d_star_sq_sel = d_star_sq.select(isel)

        perm = flex.sort_permutation(log_i_over_sigi_sel)
        i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
        i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
        log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
        log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
        d_star_sq_upper.append(d_star_sq_sel[i_lower])
        d_star_sq_lower.append(d_star_sq_sel[i_upper])

    fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
    m_upper = fit_upper.slope()
    c_upper = fit_upper.y_intercept()
    fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
    m_lower = fit_lower.slope()
    c_lower = fit_lower.y_intercept()

    #fit_upper.show_summary()
    #fit_lower.show_summary()

    if m_upper == m_lower:
        intersection = (-1, -1)
        resolution_estimate = -1
        inside = flex.bool(len(d_star_sq), False)

    else:
        # http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
        intersection = ((c_lower - c_upper) / (m_upper - m_lower),
                        (m_upper * c_lower - m_lower * c_upper) /
                        (m_upper - m_lower))

        a = m_upper
        c_ = c_upper
        b = m_lower
        d = c_lower
        assert intersection == ((d - c_) / (a - b), (a * d - b * c_) / (a - b))

        #inside = points_inside_envelope(
        #d_star_sq, log_i_over_sigi, m_upper, c_upper, m_lower, c_lower)

        inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper,
                                   c_upper)
        inside = inside & ~outliers_all

        if inside.count(True) > 0:
            d_star_sq_estimate = flex.max(d_star_sq.select(inside))
            #d_star_sq_estimate = intersection[0]
            resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
        else:
            resolution_estimate = -1

    #resolution_estimate = max(resolution_estimate, flex.min(d_spacings))

    if plot_filename is not None:
        if pyplot is None:
            raise Sorry("matplotlib must be installed to generate a plot.")
        fig = pyplot.figure()
        ax = fig.add_subplot(1, 1, 1)
        ax.scatter(d_star_sq, log_i_over_sigi, marker='+')
        ax.scatter(d_star_sq.select(inside),
                   log_i_over_sigi.select(inside),
                   marker='+',
                   color='green')
        ax.scatter(d_star_sq.select(ice_sel),
                   log_i_over_sigi.select(ice_sel),
                   marker='+',
                   color='black')
        ax.scatter(d_star_sq.select(outliers_all),
                   log_i_over_sigi.select(outliers_all),
                   marker='+',
                   color='grey')
        ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker='+', color='red')
        ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker='+', color='red')

        if (intersection[0] <= ax.get_xlim()[1]
                and intersection[1] <= ax.get_ylim()[1]):
            ax.scatter([intersection[0]], [intersection[1]],
                       marker='x',
                       s=50,
                       color='b')
        #ax.hexbin(d_star_sq, log_i_over_sigi, gridsize=30)
        xlim = pyplot.xlim()
        ax.plot(xlim, [(m * x + c) for x in xlim])
        ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color='red')
        ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color='red')
        ax.set_xlabel('d_star_sq')
        ax.set_ylabel('ln(I/sigI)')
        ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
        ax.set_ylim((0, ax.get_ylim()[1]))

        for i_slot, slot in enumerate(binner.bins):
            if i_slot == 0:
                ax.vlines(uctbx.d_as_d_star_sq(slot.d_max),
                          0,
                          ax.get_ylim()[1],
                          linestyle='dotted',
                          color='grey')
            ax.vlines(uctbx.d_as_d_star_sq(slot.d_min),
                      0,
                      ax.get_ylim()[1],
                      linestyle='dotted',
                      color='grey')

        ax_ = ax.twiny()  # ax2 is responsible for "top" axis and "right" axis
        xticks = ax.get_xticks()
        xlim = ax.get_xlim()
        xticks_d = [
            uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
        ]
        xticks_ = [ds2 / (xlim[1] - xlim[0]) for ds2 in xticks]
        ax_.set_xticks(xticks)
        ax_.set_xlim(ax.get_xlim())
        ax_.set_xlabel(r"Resolution ($\AA$)")
        ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
        #pyplot.show()
        pyplot.savefig(plot_filename)
        pyplot.close()

    return resolution_estimate
Exemplo n.º 18
0
def plot_result(metric, result):
    if metric == metrics.CC_HALF:
        return plots.cc_half_plot(
            result.d_star_sq,
            result.y_obs,
            cc_half_critical_values=result.critical_values,
            cc_half_fit=result.y_fit,
            d_min=result.d_min,
        )
    else:
        d = {
            metrics.MISIGMA: "Merged <I/σ(I)>",
            metrics.ISIGMA: "Unmerged <I/σ(I)>",
            metrics.I_MEAN_OVER_SIGMA_MEAN: "&lt;I&gt;/<σ(I)>",
            metrics.RMERGE: "R<sub>merge</sub> ",
            metrics.COMPLETENESS: "Completeness",
        }
        d_star_sq_tickvals, d_star_sq_ticktext = plots.d_star_sq_to_d_ticks(
            result.d_star_sq, 5)
        return {
            "data": [
                {
                    "x": list(result.d_star_sq),  # d_star_sq
                    "y": list(result.y_obs),
                    "type": "scatter",
                    "name": "y_obs",
                },
                ({
                    "x": list(result.d_star_sq),
                    "y": list(result.y_fit),
                    "type": "scatter",
                    "name": "y_fit",
                    "line": {
                        "color": "rgb(47, 79, 79)"
                    },
                } if result.y_fit else {}),
                ({
                    "x": [uctbx.d_as_d_star_sq(result.d_min)] * 2,
                    "y": [
                        0,
                        max(
                            1,
                            flex.max(result.y_obs),
                            flex.max(result.y_fit) if result.y_fit else 0,
                        ),
                    ],
                    "type":
                    "scatter",
                    "name":
                    "d_min = %.2f Å" % result.d_min,
                    "mode":
                    "lines",
                    "line": {
                        "color": "rgb(169, 169, 169)",
                        "dash": "dot"
                    },
                } if result.d_min else {}),
            ],
            "layout": {
                "title": f"{d.get(metric)} vs. resolution",
                "xaxis": {
                    "title": "Resolution (Å)",
                    "tickvals": d_star_sq_tickvals,
                    "ticktext": d_star_sq_ticktext,
                },
                "yaxis": {
                    "title": d.get(metric),
                    "rangemode": "tozero"
                },
            },
        }
Exemplo n.º 19
0
    def _record_individual_report(self, data_manager, report, cluster_name):
        d = self._report_as_dict(report)

        self._individual_report_dicts[
            cluster_name] = self._individual_report_dict(d, cluster_name)

        self._comparison_graphs.setdefault(
            "radar",
            {
                "data": [],
                "layout": {
                    "polar": {
                        "radialaxis": {
                            "visible": True,
                            "showticklabels": False,
                            "range": [0, 1],
                        }
                    }
                },
            },
        )

        self._comparison_graphs["radar"]["data"].append({
            "type": "scatterpolar",
            "r": [],
            "theta": [],
            "fill": "toself",
            "name": cluster_name,
        })

        for k, text in (
            ("cc_one_half", "CC½"),
            ("mean_redundancy", "Multiplicity"),
            ("completeness", "Completeness"),
            ("i_over_sigma_mean", "I/σ(I)"),
        ):
            self._comparison_graphs["radar"]["data"][-1]["r"].append(
                getattr(report.merging_stats.overall, k))
            self._comparison_graphs["radar"]["data"][-1]["theta"].append(text)

        self._comparison_graphs["radar"]["data"][-1]["r"].append(
            uctbx.d_as_d_star_sq(report.merging_stats.overall.d_min))
        self._comparison_graphs["radar"]["data"][-1]["theta"].append(
            "Resolution")

        for graph in (
                "cc_one_half",
                "i_over_sig_i",
                "completeness",
                "multiplicity_vs_resolution",
                "r_pim",
        ):
            self._comparison_graphs.setdefault(graph, {
                "layout": d[graph]["layout"],
                "data": []
            })
            data = copy.deepcopy(d[graph]["data"][0])
            data["name"] = cluster_name
            data.pop("line", None)  # remove default color override
            self._comparison_graphs[graph]["data"].append(data)

        def remove_html_tags(table):
            return [[
                s.replace("<strong>", "").replace("</strong>", "").replace(
                    "<sub>", "").replace("</sub>", "")
                if isinstance(s, str) else s for s in row
            ] for row in table]

        logger.info(
            "\nOverall merging statistics:\n%s",
            tabulate(remove_html_tags(d["overall_statistics_table"]),
                     headers="firstrow"),
        )
        logger.info(
            "\nResolution shells:\n%s",
            tabulate(remove_html_tags(d["merging_statistics_table"]),
                     headers="firstrow"),
        )
Exemplo n.º 20
0
 def plot_resolution_limit(self, d):
   from cctbx import uctbx
   d_star_sq = uctbx.d_as_d_star_sq(d)
   self.ax.plot([d_star_sq, d_star_sq], self.ax.get_ylim(), linestyle='--')
Exemplo n.º 21
0
def gpeak_from_d_spacing(d, wavl):
    """take a d-spacing and return a peak in GSASII format"""
    twoth = d_star_sq_as_two_theta(d_as_d_star_sq(d), wavl, deg=True)
    return [twoth, 1000, True, False, 0, 0, 0, d, 0]
Exemplo n.º 22
0
def make_dano_plots(anomalous_data):
    """
    Make dicts of data for plotting e.g. for plotly.

    Args:
        anomalous_data (dict) : A dict of (wavelength, anomalous array) data.

    Returns:
        dict: A dictionary containing the plotting data.
    """

    data = {
        "dF": {
            "dano": {
                "data": [],
                "help": """\
This plot shows the size of the anomalous differences of F relative to the uncertainties,
(<|F(+)-F(-)|/σ(F(+)-F(-))>). A value of 0.8 is indicative of pure noise, and
a suggested cutoff is when the value falls below 1.2, although these guides require
reliable sigma estimates. For further information see
https://strucbio.biologie.uni-konstanz.de/ccp4wiki/index.php?title=SHELX_C/D/E
""",
            },
        },
    }

    for i, (wave, anom) in enumerate(anomalous_data.items()):
        dFsdF, resolution_bin_edges = dano_over_sigdano_stats(anom)
        d_star_sq_bins = [
            0.5
            * (
                uctbx.d_as_d_star_sq(resolution_bin_edges[i])
                + uctbx.d_as_d_star_sq(resolution_bin_edges[i + 1])
            )
            for i in range(0, len(resolution_bin_edges[:-1]))
        ]
        d_star_sq_tickvals, d_star_sq_ticktext = d_star_sq_to_d_ticks(
            d_star_sq_bins, nticks=5
        )
        data["dF"]["dano"]["data"].append(
            {
                "x": d_star_sq_bins,
                "y": list(dFsdF),
                "type": "scatter",
                "name": "\u03BB" + f"={wave:.4f}",
            }
        )

    data["dF"]["dano"]["data"].append(
        {
            "x": d_star_sq_bins,
            "y": [0.8] * len(d_star_sq_bins),
            "type": "scatter",
            "mode": "lines",
            "name": "random noise level",
        }
    )
    data["dF"]["dano"]["data"].append(
        {
            "x": d_star_sq_bins,
            "y": [1.2] * len(d_star_sq_bins),
            "type": "scatter",
            "mode": "lines",
            "name": "an approximate <br>threshold for a<br>resolution cutoff",
        }
    )

    data["dF"]["dano"]["layout"] = {
        "title": "<|ΔF|/σ(ΔF)> vs resolution",
        "xaxis": {
            "title": "Resolution (Å)",
            "tickvals": d_star_sq_tickvals,
            "ticktext": d_star_sq_ticktext,
        },
        "yaxis": {"title": "<|ΔF|/σ(ΔF)>", "rangemode": "tozero"},
    }
    return data
Exemplo n.º 23
0
def estimate_resolution_limit(reflections, imageset, ice_sel=None,
                              plot_filename=None):

  if ice_sel is None:
    ice_sel = flex.bool(len(reflections), False)

  d_star_sq = flex.pow2(reflections['rlp'].norms())
  d_spacings = uctbx.d_star_sq_as_d(d_star_sq)

  intensities = reflections['intensity.sum.value']
  variances = reflections['intensity.sum.variance']

  sel = variances > 0
  intensities = intensities.select(sel)
  variances = variances.select(sel)
  ice_sel = ice_sel.select(sel)

  i_over_sigi = intensities/flex.sqrt(variances)
  log_i_over_sigi = flex.log(i_over_sigi)

  fit = flex.linear_regression(
    d_star_sq.select(~ice_sel), log_i_over_sigi.select(~ice_sel))
  m = fit.slope()
  c = fit.y_intercept()

  log_i_sigi_lower = flex.double()
  d_star_sq_lower = flex.double()
  log_i_sigi_upper = flex.double()
  d_star_sq_upper = flex.double()

  binner = binner_equal_population(
    d_star_sq, target_n_per_bin=20, max_slots=20, min_slots=5)

  outliers_all = flex.bool(len(reflections), False)

  low_percentile_limit = 0.1
  upper_percentile_limit = 1-low_percentile_limit
  d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
  for i_slot, slot in enumerate(binner.bins):
    sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
    sel = ~(ice_sel) & sel_all
    #sel = ~(ice_sel) & (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)

    #print "%.2f" %(sel.count(True)/sel_all.count(True))

    if sel.count(True) == 0:
      #outliers_all.set_selected(sel_all & ice_sel, True)
      continue
      #if i_slot > i_slot_max:
        #break
      #else:
        #continue

    outliers = wilson_outliers(
      reflections.select(sel_all), ice_sel=ice_sel.select(sel_all))
    #print "rejecting %d wilson outliers" %outliers.count(True)
    outliers_all.set_selected(sel_all, outliers)

    #if sel.count(True)/sel_all.count(True) < 0.25:
      #outliers_all.set_selected(sel_all & ice_sel, True)

    #from scitbx.math import median_statistics
    #intensities_sel = intensities.select(sel)
    #stats = median_statistics(intensities_sel)
    #z_score = 0.6745 * (intensities_sel - stats.median)/stats.median_absolute_deviation
    #outliers = z_score > 3.5
    #perm = flex.sort_permutation(intensities_sel)
    ##print ' '.join('%.2f' %v for v in intensities_sel.select(perm))
    ##print ' '.join('%.2f' %v for v in z_score.select(perm))
    ##print

    isel = sel_all.iselection().select(~(outliers) & ~(ice_sel).select(sel_all))
    log_i_over_sigi_sel = log_i_over_sigi.select(isel)
    d_star_sq_sel = d_star_sq.select(isel)

    perm = flex.sort_permutation(log_i_over_sigi_sel)
    i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
    i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
    log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
    log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
    d_star_sq_upper.append(d_star_sq_sel[i_lower])
    d_star_sq_lower.append(d_star_sq_sel[i_upper])

  fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
  m_upper = fit_upper.slope()
  c_upper = fit_upper.y_intercept()
  fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
  m_lower = fit_lower.slope()
  c_lower = fit_lower.y_intercept()

  #fit_upper.show_summary()
  #fit_lower.show_summary()

  if m_upper == m_lower:
    intersection = (-1,-1)
    resolution_estimate = -1
    inside = flex.bool(len(d_star_sq), False)

  else:
    # http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
    intersection = (
      (c_lower-c_upper)/(m_upper-m_lower),
      (m_upper*c_lower-m_lower*c_upper)/(m_upper-m_lower))

    a = m_upper
    c_ = c_upper
    b = m_lower
    d = c_lower
    assert intersection == ((d-c_)/(a-b), (a*d-b*c_)/(a-b))

    #inside = points_inside_envelope(
      #d_star_sq, log_i_over_sigi, m_upper, c_upper, m_lower, c_lower)

    inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper, c_upper)
    inside = inside & ~outliers_all

    if inside.count(True) > 0:
      d_star_sq_estimate = flex.max(d_star_sq.select(inside))
      #d_star_sq_estimate = intersection[0]
      resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
    else:
      resolution_estimate = -1

  #resolution_estimate = max(resolution_estimate, flex.min(d_spacings))

  if plot_filename is not None:
    if pyplot is None:
      raise Sorry("matplotlib must be installed to generate a plot.")
    fig = pyplot.figure()
    ax = fig.add_subplot(1,1,1)
    ax.scatter(d_star_sq, log_i_over_sigi, marker='+')
    ax.scatter(d_star_sq.select(inside), log_i_over_sigi.select(inside),
               marker='+', color='green')
    ax.scatter(d_star_sq.select(ice_sel),
               log_i_over_sigi.select(ice_sel),
               marker='+', color='black')
    ax.scatter(d_star_sq.select(outliers_all),
               log_i_over_sigi.select(outliers_all),
               marker='+', color='grey')
    ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker='+', color='red')
    ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker='+', color='red')

    if (intersection[0] <= ax.get_xlim()[1] and
        intersection[1] <= ax.get_ylim()[1]):
      ax.scatter([intersection[0]], [intersection[1]], marker='x', s=50, color='b')
    #ax.hexbin(d_star_sq, log_i_over_sigi, gridsize=30)
    xlim = pyplot.xlim()
    ax.plot(xlim, [(m * x + c) for x in xlim])
    ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color='red')
    ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color='red')
    ax.set_xlabel('d_star_sq')
    ax.set_ylabel('ln(I/sigI)')
    ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
    ax.set_ylim((0, ax.get_ylim()[1]))

    for i_slot, slot in enumerate(binner.bins):
      if i_slot == 0:
        ax.vlines(uctbx.d_as_d_star_sq(slot.d_max), 0, ax.get_ylim()[1],
                  linestyle='dotted', color='grey')
      ax.vlines(uctbx.d_as_d_star_sq(slot.d_min), 0, ax.get_ylim()[1],
                linestyle='dotted', color='grey')

    ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
    xticks = ax.get_xticks()
    xlim = ax.get_xlim()
    xticks_d = [
      uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
    xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
    ax_.set_xticks(xticks)
    ax_.set_xlim(ax.get_xlim())
    ax_.set_xlabel(r"Resolution ($\AA$)")
    ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
    #pyplot.show()
    pyplot.savefig(plot_filename)
    pyplot.close()

  return resolution_estimate
Exemplo n.º 24
0
def cc_half_plot(
    d_star_sq,
    cc_half,
    cc_anom=None,
    cc_half_critical_values=None,
    cc_anom_critical_values=None,
    cc_half_fit=None,
    d_min=None,
):
    d_star_sq_tickvals, d_star_sq_ticktext = d_star_sq_to_d_ticks(d_star_sq, nticks=5)
    return {
        "data": [
            {
                "x": list(d_star_sq),
                "y": list(cc_half),
                "type": "scatter",
                "name": "CC<sub>½</sub>",
                "mode": "lines",
                "line": {"color": "rgb(31, 119, 180)"},
            },
            (
                {
                    "x": list(d_star_sq),
                    "y": list(cc_half_critical_values),
                    "type": "scatter",
                    "name": "CC<sub>½</sub> critical value (p=0.01)",
                    "line": {"color": "rgb(31, 119, 180)", "dash": "dot"},
                }
                if cc_half_critical_values
                else {}
            ),
            (
                {
                    "x": list(d_star_sq),
                    "y": list(cc_anom),
                    "type": "scatter",
                    "name": "CC-anom",
                    "mode": "lines",
                    "line": {"color": "rgb(255, 127, 14)"},
                }
                if cc_anom
                else {}
            ),
            (
                {
                    "x": list(d_star_sq),
                    "y": list(cc_anom_critical_values),
                    "type": "scatter",
                    "name": "CC-anom critical value (p=0.01)",
                    "mode": "lines",
                    "line": {"color": "rgb(255, 127, 14)", "dash": "dot"},
                }
                if cc_anom_critical_values
                else {}
            ),
            (
                {
                    "x": list(d_star_sq),
                    "y": list(cc_half_fit),
                    "type": "scatter",
                    "name": "CC<sub>½</sub> fit",
                    "line": {"color": "rgb(47, 79, 79)"},
                }
                if cc_half_fit
                else {}
            ),
            (
                {
                    "x": [uctbx.d_as_d_star_sq(d_min)] * 2,
                    "y": [0, 1],
                    "type": "scatter",
                    "name": f"d_min = {d_min:.2f} Å",
                    "mode": "lines",
                    "line": {"color": "rgb(169, 169, 169)", "dash": "dot"},
                }
                if d_min
                else {}
            ),
        ],
        "layout": {
            "title": "CC<sub>½</sub> vs resolution",
            "xaxis": {
                "title": "Resolution (Å)",
                "tickvals": d_star_sq_tickvals,
                "ticktext": d_star_sq_ticktext,
            },
            "yaxis": {
                "title": "CC<sub>½</sub>",
                "range": [min(cc_half + cc_anom if cc_anom else [] + [0]), 1],
            },
        },
        "help": """\
The correlation coefficients, CC<sub>½</sub>, between random half-datasets. A correlation
coefficient of +1 indicates good correlation, and 0 indicates no correlation.
CC<sub>½</sub> is typically close to 1 at low resolution, falling off to close to zero at
higher resolution. A typical resolution cutoff based on CC<sub>½</sub> is around 0.3-0.5.

[1] Karplus, P. A., & Diederichs, K. (2012). Science, 336(6084), 1030-1033.
    https://doi.org/10.1126/science.1218231
[2] Diederichs, K., & Karplus, P. A. (2013). Acta Cryst D, 69(7), 1215-1222.
    https://doi.org/10.1107/S0907444913001121
[3] Evans, P. R., & Murshudov, G. N. (2013). Acta Cryst D, 69(7), 1204-1214.
    https://doi.org/10.1107/S0907444913000061
""",
    }
Exemplo n.º 25
0
def gpeak_from_d_spacing(d, wavl):
    from cctbx.uctbx import d_as_d_star_sq, d_star_sq_as_two_theta
    twoth = d_star_sq_as_two_theta(d_as_d_star_sq(d), wavl, deg=True)
    return [twoth, 1000, True, False, 0, 0, 0, d, 0]
Exemplo n.º 26
0
def exercise_functions():
  d_star_sq_s = 1.2345
  d_star_sq_a = flex.double((1.2345, 0.1234))
  two_theta_s = 0.61725
  two_theta_a = flex.double((0.61725,0.65432))
  # forward conversions
  assert approx_equal(
    uctbx.d_star_sq_as_stol_sq(d_star_sq_s), d_star_sq_s / 4)
  assert approx_equal(
    uctbx.d_star_sq_as_stol_sq(d_star_sq_a),
    [uctbx.d_star_sq_as_stol_sq(i) for i in d_star_sq_a])
  assert approx_equal(
    uctbx.d_star_sq_as_two_stol(d_star_sq_s)**2, d_star_sq_s)
  assert approx_equal(
    uctbx.d_star_sq_as_two_stol(d_star_sq_a),
    [uctbx.d_star_sq_as_two_stol(i) for i in d_star_sq_a])
  assert approx_equal(
    uctbx.d_star_sq_as_stol(d_star_sq_s)**2, d_star_sq_s / 4)
  assert approx_equal(
    uctbx.d_star_sq_as_stol(d_star_sq_a),
    [uctbx.d_star_sq_as_stol(i) for i in d_star_sq_a])
  assert approx_equal(
    1/(uctbx.d_star_sq_as_d(d_star_sq_s)**2), d_star_sq_s)
  assert approx_equal(
    uctbx.d_star_sq_as_d(d_star_sq_a),
    [uctbx.d_star_sq_as_d(i) for i in d_star_sq_a])
  assert approx_equal(
    uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5),
    2 * asin(1.5/2*sqrt(d_star_sq_s)))
  assert approx_equal(
    uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5),
    uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5, False))
  assert approx_equal(
    uctbx.d_star_sq_as_two_theta(d_star_sq_a, 1.5),
    [uctbx.d_star_sq_as_two_theta(i, 1.5) for i in d_star_sq_a])
  assert approx_equal(
    uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5)*180/pi,
    uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5, True))
  # reverse conversions
  for d_star_sq, two_theta in zip(
    (d_star_sq_s, d_star_sq_a),(two_theta_s, two_theta_a)):
    assert approx_equal(
      uctbx.stol_sq_as_d_star_sq(
        uctbx.d_star_sq_as_stol_sq(d_star_sq)), d_star_sq)
    assert approx_equal(
      uctbx.two_stol_as_d_star_sq(
        uctbx.d_star_sq_as_two_stol(d_star_sq)), d_star_sq)
    assert approx_equal(
      uctbx.stol_as_d_star_sq(
        uctbx.d_star_sq_as_stol(d_star_sq)), d_star_sq)
    assert approx_equal(
      uctbx.d_as_d_star_sq(
        uctbx.d_star_sq_as_d(d_star_sq)), d_star_sq)
    assert approx_equal(
      uctbx.two_theta_as_d_star_sq(
        uctbx.d_star_sq_as_two_theta(d_star_sq, 1.5), 1.5), d_star_sq)
    assert approx_equal(
      uctbx.two_theta_as_d_star_sq(two_theta, 1.5),
      uctbx.two_theta_as_d_star_sq(two_theta, 1.5, False))
    assert approx_equal(
      uctbx.two_theta_as_d_star_sq(two_theta, 1.5),
      uctbx.two_theta_as_d_star_sq(two_theta*180/pi, 1.5, True))
    assert approx_equal(
      uctbx.two_theta_as_d(two_theta, 1.5),
      uctbx.d_star_sq_as_d(uctbx.two_theta_as_d_star_sq(two_theta, 1.5)))
    assert approx_equal(
      uctbx.two_theta_as_d(two_theta, 1.5, True),
      uctbx.d_star_sq_as_d(uctbx.two_theta_as_d_star_sq(two_theta, 1.5, True)))
  #
  assert uctbx.fractional_unit_shifts(distance_frac=[0,0,0]) == (0,0,0)
  assert uctbx.fractional_unit_shifts([0.6,7.4,-0.4]) == (1,7,0)
  assert uctbx.fractional_unit_shifts([-6,3,-0.6]) == (-6,3,-1)
  site_frac_1 = [0.3,-8.6,2.1]
  for eps,expected_u2 in [(-1.e-5, 1), (1.e-5, 2)]:
    site_frac_2 = [-3,5.6,0.6-eps]
    assert uctbx.fractional_unit_shifts(
      site_frac_1=site_frac_1,
      site_frac_2=site_frac_2) == (3, -14, expected_u2)
Exemplo n.º 27
0
def run(args):
  from dials.util.options import OptionParser
  import libtbx.load_env

  usage = "%s [options] find_spots.json" %(
    libtbx.env.dispatcher_name)

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    epilog=help_message)

  params, options, args = parser.parse_args(
    show_diff_phil=True, return_unhandled=True)

  positions = None
  if params.positions is not None:
    with open(params.positions, 'rb') as f:
      positions = flex.vec2_double()
      for line in f.readlines():
        line = line.replace('(', ' ').replace(')', '').replace(',', ' ').strip().split()
        assert len(line) == 3
        i, x, y = [float(l) for l in line]
        positions.append((x, y))

  assert len(args) == 1
  json_file = args[0]
  import json

  with open(json_file, 'rb') as f:
    results = json.load(f)

  n_indexed = flex.double()
  fraction_indexed = flex.double()
  n_spots = flex.double()
  n_lattices = flex.double()
  crystals = []
  image_names = flex.std_string()

  for r in results:
    n_spots.append(r['n_spots_total'])
    image_names.append(str(r['image']))
    if 'n_indexed' in r:
      n_indexed.append(r['n_indexed'])
      fraction_indexed.append(r['fraction_indexed'])
      n_lattices.append(len(r['lattices']))
      for d in r['lattices']:
        from dxtbx.serialize.crystal import from_dict
        crystals.append(from_dict(d['crystal']))
    else:
      n_indexed.append(0)
      fraction_indexed.append(0)
      n_lattices.append(0)

  import matplotlib
  matplotlib.use('Agg')
  from matplotlib import pyplot

  blue = '#3498db'
  red = '#e74c3c'

  marker = 'o'
  alpha = 0.5
  lw = 0

  plot = True
  table = True
  grid = params.grid

  from libtbx import group_args
  from dials.algorithms.peak_finding.per_image_analysis \
       import plot_stats, print_table

  estimated_d_min = flex.double()
  d_min_distl_method_1 = flex.double()
  d_min_distl_method_2 = flex.double()
  n_spots_total = flex.int()
  n_spots_no_ice = flex.int()
  total_intensity = flex.double()

  for d in results:
    estimated_d_min.append(d['estimated_d_min'])
    d_min_distl_method_1.append(d['d_min_distl_method_1'])
    d_min_distl_method_2.append(d['d_min_distl_method_2'])
    n_spots_total.append(d['n_spots_total'])
    n_spots_no_ice.append(d['n_spots_no_ice'])
    total_intensity.append(d['total_intensity'])

  stats = group_args(image=image_names,
                     n_spots_total=n_spots_total,
                     n_spots_no_ice=n_spots_no_ice,
                     n_spots_4A=None,
                     total_intensity=total_intensity,
                     estimated_d_min=estimated_d_min,
                     d_min_distl_method_1=d_min_distl_method_1,
                     d_min_distl_method_2=d_min_distl_method_2,
                     noisiness_method_1=None,
                     noisiness_method_2=None)

  if plot:
    plot_stats(stats)
    pyplot.clf()
  if table:
    print_table(stats)

  print "Number of indexed lattices: ", (n_indexed > 0).count(True)

  print "Number with valid d_min but failed indexing: ", (
    (d_min_distl_method_1 > 0) &
    (d_min_distl_method_2 > 0) &
    (estimated_d_min > 0) &
    (n_indexed == 0)).count(True)

  n_rows = 10
  n_rows = min(n_rows, len(n_spots_total))
  perm_n_spots_total = flex.sort_permutation(n_spots_total, reverse=True)
  print 'Top %i images sorted by number of spots:' %n_rows
  print_table(stats, perm=perm_n_spots_total, n_rows=n_rows)

  n_bins = 20
  spot_count_histogram(
    n_spots_total, n_bins=n_bins, filename='hist_n_spots_total.png', log=True)
  spot_count_histogram(
    n_spots_no_ice, n_bins=n_bins, filename='hist_n_spots_no_ice.png', log=True)
  spot_count_histogram(
    n_indexed.select(n_indexed > 0), n_bins=n_bins, filename='hist_n_indexed.png', log=False)

  if len(crystals):
    plot_unit_cell_histograms(crystals)

  if params.stereographic_projections and len(crystals):
    from dxtbx.datablock import DataBlockFactory
    datablocks = DataBlockFactory.from_filenames(
      [image_names[0]], verbose=False)
    assert len(datablocks) == 1
    imageset = datablocks[0].extract_imagesets()[0]
    s0 = imageset.get_beam().get_s0()
    # XXX what if no goniometer?
    rotation_axis = imageset.get_goniometer().get_rotation_axis()

    indices = ((1,0,0), (0,1,0), (0,0,1))
    for i, index in enumerate(indices):

      from cctbx import crystal, miller
      from scitbx import matrix
      miller_indices = flex.miller_index([index])
      symmetry = crystal.symmetry(
        unit_cell=crystals[0].get_unit_cell(),
        space_group=crystals[0].get_space_group())
      miller_set = miller.set(symmetry, miller_indices)
      d_spacings = miller_set.d_spacings()
      d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1()
      d_spacings = d_spacings.generate_bijvoet_mates()
      miller_indices = d_spacings.indices()

      # plane normal
      d0 = matrix.col(s0).normalize()
      d1 = d0.cross(matrix.col(rotation_axis)).normalize()
      d2 = d1.cross(d0).normalize()
      reference_poles = (d0, d1, d2)

      from dials.command_line.stereographic_projection import stereographic_projection
      projections = []

      for cryst in crystals:
        reciprocal_space_points = list(cryst.get_U() * cryst.get_B()) * miller_indices.as_vec3_double()
        projections.append(stereographic_projection(
          reciprocal_space_points, reference_poles))

        #from dials.algorithms.indexing.compare_orientation_matrices import \
        #  difference_rotation_matrix_and_euler_angles
        #R_ij, euler_angles, cb_op = difference_rotation_matrix_and_euler_angles(
        #  crystals[0], cryst)
        #print max(euler_angles)

      from dials.command_line.stereographic_projection import plot_projections
      plot_projections(projections, filename='projections_%s.png' %('hkl'[i]))
      pyplot.clf()

  def plot_grid(values, grid, file_name, cmap=pyplot.cm.Reds,
                vmin=None, vmax=None, invalid='white'):
    values = values.as_double()
    # At DLS, fast direction appears to be largest direction
    if grid[0] > grid[1]:
      values.reshape(flex.grid(reversed(grid)))
      values = values.matrix_transpose()
    else:
      values.reshape(flex.grid(grid))

    Z = values.as_numpy_array()

    #f, (ax1, ax2) = pyplot.subplots(2)
    f, ax1 = pyplot.subplots(1)

    mesh1 = ax1.pcolormesh(
      values.as_numpy_array(), cmap=cmap, vmin=vmin, vmax=vmax)
    mesh1.cmap.set_under(color=invalid, alpha=None)
    mesh1.cmap.set_over(color=invalid, alpha=None)
    #mesh2 = ax2.contour(Z, cmap=cmap, vmin=vmin, vmax=vmax)
    #mesh2 = ax2.contourf(Z, cmap=cmap, vmin=vmin, vmax=vmax)
    ax1.set_aspect('equal')
    ax1.invert_yaxis()
    #ax2.set_aspect('equal')
    #ax2.invert_yaxis()
    pyplot.colorbar(mesh1, ax=ax1)
    #pyplot.colorbar(mesh2, ax=ax2)
    pyplot.savefig(file_name, dpi=600)
    pyplot.clf()

  def plot_positions(values, positions, file_name, cmap=pyplot.cm.Reds,
                     vmin=None, vmax=None, invalid='white'):
    values = values.as_double()
    assert positions.size() >= values.size()
    positions = positions[:values.size()]

    if vmin is None:
      vmin = flex.min(values)
    if vmax is None:
      vmax = flex.max(values)

    x, y = positions.parts()
    dx = flex.abs(x[1:] - x[:-1])
    dy = flex.abs(y[1:] - y[:-1])
    dx = dx.select(dx > 0)
    dy = dy.select(dy > 0)

    scale = 1/flex.min(dx)
    #print scale
    x = (x * scale).iround()
    y = (y * scale).iround()

    from libtbx.math_utils import iceil
    z = flex.double(flex.grid(iceil(flex.max(y))+1, iceil(flex.max(x))+1), -2)
    #print z.all()
    for x_, y_, z_ in zip(x, y, values):
      z[y_, x_] = z_

    plot_grid(z.as_1d(), z.all(), file_name, cmap=cmap, vmin=vmin, vmax=vmax,
              invalid=invalid)
    return

  if grid is not None or positions is not None:
    if grid is not None:
      positions = tuple(reversed(grid))
      plotter = plot_grid
    else:
      plotter = plot_positions

    cmap = pyplot.get_cmap(params.cmap)
    plotter(n_spots_total, positions, 'grid_spot_count_total.png', cmap=cmap,
            invalid=params.invalid)
    plotter(n_spots_no_ice, positions, 'grid_spot_count_no_ice.png', cmap=cmap,
            invalid=params.invalid)
    plotter(total_intensity, positions, 'grid_total_intensity.png', cmap=cmap,
            invalid=params.invalid)
    if flex.max(n_indexed) > 0:
      plotter(n_indexed, positions, 'grid_n_indexed.png', cmap=cmap,
              invalid=params.invalid)
      plotter(fraction_indexed, positions, 'grid_fraction_indexed.png',
              cmap=cmap, vmin=0, vmax=1, invalid=params.invalid)

    for i, d_min in enumerate((estimated_d_min, d_min_distl_method_1, d_min_distl_method_2)):
      from cctbx import uctbx
      d_star_sq = uctbx.d_as_d_star_sq(d_min)
      d_star_sq.set_selected(d_star_sq == 1, 0)
      vmin = flex.min(d_star_sq.select(d_star_sq > 0))
      vmax = flex.max(d_star_sq)

      vmin = flex.min(d_min.select(d_min > 0))
      vmax = flex.max(d_min)
      cmap = pyplot.get_cmap('%s_r' %params.cmap)
      d_min.set_selected(d_min <= 0, vmax)

      if i == 0:
        plotter(d_min, positions, 'grid_d_min.png', cmap=cmap, vmin=vmin,
                vmax=vmax, invalid=params.invalid)
      else:
        plotter(
          d_min, positions, 'grid_d_min_method_%i.png' %i, cmap=cmap,
          vmin=vmin, vmax=vmax, invalid=params.invalid)

  if flex.max(n_indexed) > 0:
    pyplot.hexbin(
      n_spots, n_indexed, bins='log', cmap=pyplot.cm.jet, gridsize=50)
    pyplot.colorbar()
    #pyplot.scatter(n_spots, n_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
    xlim = pyplot.xlim()
    ylim = pyplot.ylim()
    pyplot.plot([0, max(n_spots)], [0, max(n_spots)], c=red)
    pyplot.xlim(0, xlim[1])
    pyplot.ylim(0, ylim[1])
    pyplot.xlabel('# spots')
    pyplot.ylabel('# indexed')
    pyplot.savefig('n_spots_vs_n_indexed.png')
    pyplot.clf()

    pyplot.hexbin(
      n_spots, fraction_indexed, bins='log', cmap=pyplot.cm.jet, gridsize=50)
    pyplot.colorbar()
    #pyplot.scatter(
      #n_spots, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
    pyplot.xlim(0, pyplot.xlim()[1])
    pyplot.ylim(0, pyplot.ylim()[1])
    pyplot.xlabel('# spots')
    pyplot.ylabel('Fraction indexed')
    pyplot.savefig('n_spots_vs_fraction_indexed.png')
    pyplot.clf()

    pyplot.hexbin(
      n_indexed, fraction_indexed, bins='log', cmap=pyplot.cm.jet, gridsize=50)
    pyplot.colorbar()
    #pyplot.scatter(
      #n_indexed, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
    pyplot.xlim(0, pyplot.xlim()[1])
    pyplot.ylim(0, pyplot.ylim()[1])
    pyplot.xlabel('# indexed')
    pyplot.ylabel('Fraction indexed')
    pyplot.savefig('n_indexed_vs_fraction_indexed.png')
    pyplot.clf()

    pyplot.hexbin(
      n_spots, n_lattices, bins='log', cmap=pyplot.cm.jet, gridsize=50)
    pyplot.colorbar()
    #pyplot.scatter(
      #n_spots, n_lattices, marker=marker, alpha=alpha, c=blue, lw=lw)
    pyplot.xlim(0, pyplot.xlim()[1])
    pyplot.ylim(0, pyplot.ylim()[1])
    pyplot.xlabel('# spots')
    pyplot.ylabel('# lattices')
    pyplot.savefig('n_spots_vs_n_lattices.png')
    pyplot.clf()

  #pyplot.scatter(
  #  estimated_d_min, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw)
  pyplot.hexbin(estimated_d_min, d_min_distl_method_1, bins='log',
                cmap=pyplot.cm.jet, gridsize=50)
  pyplot.colorbar()
  #pyplot.gca().set_aspect('equal')
  xlim = pyplot.xlim()
  ylim = pyplot.ylim()
  m = max(max(estimated_d_min), max(d_min_distl_method_1))
  pyplot.plot([0, m], [0, m], c=red)
  pyplot.xlim(0, xlim[1])
  pyplot.ylim(0, ylim[1])
  pyplot.xlabel('estimated_d_min')
  pyplot.ylabel('d_min_distl_method_1')
  pyplot.savefig('d_min_vs_distl_method_1.png')
  pyplot.clf()

  #pyplot.scatter(
  #  estimated_d_min, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
  pyplot.hexbin(estimated_d_min, d_min_distl_method_2, bins='log',
                cmap=pyplot.cm.jet, gridsize=50)
  pyplot.colorbar()
  #pyplot.gca().set_aspect('equal')
  xlim = pyplot.xlim()
  ylim = pyplot.ylim()
  m = max(max(estimated_d_min), max(d_min_distl_method_2))
  pyplot.plot([0, m], [0, m], c=red)
  pyplot.xlim(0, xlim[1])
  pyplot.ylim(0, ylim[1])
  pyplot.xlabel('estimated_d_min')
  pyplot.ylabel('d_min_distl_method_2')
  pyplot.savefig('d_min_vs_distl_method_2.png')
  pyplot.clf()

  #pyplot.scatter(
  #  d_min_distl_method_1, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
  pyplot.hexbin(d_min_distl_method_1, d_min_distl_method_2, bins='log',
                cmap=pyplot.cm.jet, gridsize=50)
  pyplot.colorbar()
  #pyplot.gca().set_aspect('equal')
  xlim = pyplot.xlim()
  ylim = pyplot.ylim()
  m = max(max(d_min_distl_method_1), max(d_min_distl_method_2))
  pyplot.plot([0, m], [0, m], c=red)
  pyplot.xlim(0, xlim[1])
  pyplot.ylim(0, ylim[1])
  pyplot.xlabel('d_min_distl_method_1')
  pyplot.ylabel('d_min_distl_method_2')
  pyplot.savefig('distl_method_1_vs_distl_method_2.png')
  pyplot.clf()

  pyplot.hexbin(
    n_spots, estimated_d_min, bins='log', cmap=pyplot.cm.jet, gridsize=50)
  pyplot.colorbar()
  #pyplot.scatter(
    #n_spots, estimated_d_min, marker=marker, alpha=alpha, c=blue, lw=lw)
  pyplot.xlim(0, pyplot.xlim()[1])
  pyplot.ylim(0, pyplot.ylim()[1])
  pyplot.xlabel('# spots')
  pyplot.ylabel('estimated_d_min')
  pyplot.savefig('n_spots_vs_d_min.png')
  pyplot.clf()

  pyplot.hexbin(
    n_spots, d_min_distl_method_1, bins='log', cmap=pyplot.cm.jet, gridsize=50)
  pyplot.colorbar()
  #pyplot.scatter(
    #n_spots, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw)
  pyplot.xlim(0, pyplot.xlim()[1])
  pyplot.ylim(0, pyplot.ylim()[1])
  pyplot.xlabel('# spots')
  pyplot.ylabel('d_min_distl_method_1')
  pyplot.savefig('n_spots_vs_distl_method_1.png')
  pyplot.clf()

  pyplot.hexbin(
    n_spots, d_min_distl_method_2, bins='log', cmap=pyplot.cm.jet, gridsize=50)
  pyplot.colorbar()
  #pyplot.scatter(
    #n_spots, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
  pyplot.xlim(0, pyplot.xlim()[1])
  pyplot.ylim(0, pyplot.ylim()[1])
  pyplot.xlabel('# spots')
  pyplot.ylabel('d_min_distl_method_2')
  pyplot.savefig('n_spots_vs_distl_method_2.png')
  pyplot.clf()