Exemplo n.º 1
0
  def get_bounded_data(self, data, bounds):
    assert len(bounds) == 4
    x = [b[0] for b in bounds]
    y = [b[1] for b in bounds]
    left = sorted(x)[1]
    right = sorted(x)[2]
    top = sorted(y)[2]
    bottom = sorted(y)[1]
    origin = col((left, bottom))
    scale_x = right-left
    scale_y = top-bottom
    scale = min(scale_x, scale_y)

    data_max_x = flex.max(data.parts()[0])
    data_min_x = flex.min(data.parts()[0])
    data_max_y = flex.max(data.parts()[1])
    data_min_y = flex.min(data.parts()[1])
    data_scale_x = data_max_x - data_min_x
    data_scale_y = data_max_y - data_min_y

    if data_scale_x == 0 or data_scale_y == 0:
      print "WARNING bad scale"
      return data

    return flex.vec2_double(data.parts()[0] * (scale/abs(data_scale_x)),
                            data.parts()[1] * (scale/abs(data_scale_y))) + origin
Exemplo n.º 2
0
  def __init__(self, strategies, n_bins=8, degrees_per_bin=5):
    from cctbx import crystal, miller
    import copy

    sg = strategies[0].experiment.crystal.get_space_group() \
      .build_derived_reflection_intensity_group(anomalous_flag=True)
    cs = crystal.symmetry(
      unit_cell=strategies[0].experiment.crystal.get_unit_cell(), space_group=sg)

    for i, strategy in enumerate(strategies):
      if i == 0:
        predicted = copy.deepcopy(strategy.predicted)
      else:
        predicted_ = copy.deepcopy(strategy.predicted)
        predicted_['dose'] += (flex.max(predicted['dose']) + 1)
        predicted.extend(predicted_)
    ms = miller.set(cs, indices=predicted['miller_index'], anomalous_flag=True)
    ma = miller.array(ms, data=flex.double(ms.size(),1),
                      sigmas=flex.double(ms.size(), 1))
    if 1:
      merging = ma.merge_equivalents()
      o = merging.array().customized_copy(
        data=merging.redundancies().data().as_double()).as_mtz_dataset('I').mtz_object()
      o.write('predicted.mtz')

    d_star_sq = ma.d_star_sq().data()

    binner = ma.setup_binner_d_star_sq_step(
      d_star_sq_step=(flex.max(d_star_sq)-flex.min(d_star_sq)+1e-8)/n_bins)

    dose = predicted['dose']
    range_width = 1
    range_min = flex.min(dose) - range_width
    range_max = flex.max(dose)
    n_steps = 2 + int((range_max - range_min) - range_width)

    binner_non_anom = ma.as_non_anomalous_array().use_binning(
      binner)
    self.n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])

    from xia2.Modules.PyChef2 import ChefStatistics
    chef_stats = ChefStatistics(
      ma.indices(), ma.data(), ma.sigmas(),
      ma.d_star_sq().data(), dose, self.n_complete, binner,
      ma.space_group(), ma.anomalous_flag(), n_steps)

    def fraction_new(completeness):
      # Completeness so far at end of image
      completeness_end = completeness[1:]
      # Completeness so far at start of image
      completeness_start = completeness[:-1]
      # Fraction of unique reflections observed for the first time on each image
      return completeness_end - completeness_start

    self.dose = dose
    self.ieither_completeness = chef_stats.ieither_completeness()
    self.iboth_completeness = chef_stats.iboth_completeness()
    self.frac_new_ref = fraction_new(self.ieither_completeness) / degrees_per_bin
    self.frac_new_pairs = fraction_new(self.iboth_completeness) / degrees_per_bin
Exemplo n.º 3
0
  def get_min_max_xy(self, rlist):
    xc, yc, zc = rlist['xyzcal.px'].parts()
    xo, yo, zo = rlist['xyzobs.px.value'].parts()

    min_x = math.floor(min(flex.min(xc), flex.min(xo)))
    min_y = math.floor(min(flex.min(yc), flex.min(yo)))
    max_x = math.ceil(max(flex.max(xc), flex.max(xo)))
    max_y = math.ceil(max(flex.max(yc), flex.max(yo)))
    return min_x, max_x, min_y, max_y
Exemplo n.º 4
0
      def get_min_max_xy(self, rlist):
        xc, yc, zc = rlist['xyzcal.px'].parts()
        xo, yo, zo = rlist['xyzobs.px.value'].parts()
        dx = xc - xo
        dz = zc - zo

        min_x = math.floor(flex.min(dx))
        min_y = math.floor(flex.min(dz))
        max_x = math.ceil(flex.max(dx))
        max_y = math.ceil(flex.max(dz))
        return min_x, max_x, min_y, max_y
Exemplo n.º 5
0
  def per_image(self):
    """Set one block per image for all experiments"""

    self._create_block_columns()

    # get observed phi in radians
    phi_obs = self._reflections['xyzobs.mm.value'].parts()[2]

    for iexp, exp in enumerate(self._experiments):

      sel = self._reflections['id'] == iexp
      isel = sel.iselection()
      exp_phi = phi_obs.select(isel)

      # convert phi to integer frames
      frames = exp.scan.get_array_index_from_angle(exp_phi, deg=False)
      frames = flex.floor(frames).iround()

      start, stop = flex.min(frames), flex.max(frames)
      frame_range = range(start, stop + 1)

      for f_num, f in enumerate(frame_range):
        sub_isel = isel.select(frames == f)
        self._reflections['block'].set_selected(sub_isel, f_num)
        self._reflections['block_centre'].set_selected(sub_isel, f_num)

    return self._reflections
Exemplo n.º 6
0
def profile2d(p, vmin=None, vmax=None):
  from dials.array_family import flex
  import string
  if vmin is None:
    vmin = flex.min(p)
  if vmax is None:
    vmax = flex.max(p)
  assert(vmax >= vmin)
  dv = vmax - vmin
  if dv == 0:
    c = 0
    m = 0
  else:
    m = 35.0 / dv
    c = -m * vmin
  lookup = string.digits + string.ascii_uppercase
  ny, nx = p.all()
  text = ''
  for j in range(ny):
    for i in range(nx):
      v = int(m * p[j,i] + c)
      if v < 0:
        v = 0
      elif v > 35:
        v = 35
      t = lookup[v]
      text += t + ' '
    text += '\n'
  return text
  def compose(self, reflections):
    """Compose scan-varying crystal parameterisations at the specified image
    number, for the specified experiment, for each image. Put the U, B and
    UB matrices in the reflection table, and cache the derivatives."""

    self._prepare_for_compose(reflections)

    for iexp, exp in enumerate(self._experiments):

      # select the reflections of interest
      sel = reflections['id'] == iexp
      isel = sel.iselection()

      blocks = reflections['block'].select(isel)

      # identify which crystal parameterisations to use for this experiment
      xl_op = self._get_xl_orientation_parameterisation(iexp)
      xl_ucp = self._get_xl_unit_cell_parameterisation(iexp)

      # get state and derivatives for each block
      for block in xrange(flex.min(blocks),
                          flex.max(blocks) + 1):

        # determine the subset of reflections this affects
        subsel = isel.select(blocks == block)
        if len(subsel) == 0: continue

        # get the integer frame number nearest the centre of that block
        frames = reflections['block_centre'].select(subsel)

        # can only be false if original block assignment has gone wrong
        assert frames.all_eq(frames[0]), \
            "Failing: a block contains reflections that shouldn't be there"
        frame = int(floor(frames[0]))

        # model states at current frame
        U = self._get_state_from_parameterisation(xl_op, frame)
        if U is None: U = exp.crystal.get_U()

        B = self._get_state_from_parameterisation(xl_ucp, frame)
        if B is None: B = exp.crystal.get_B()

        # set states
        reflections['u_matrix'].set_selected(subsel, U.elems)
        reflections['b_matrix'].set_selected(subsel, B.elems)

        # set derivatives of the states
        if xl_op is not None:
          for j, dU in enumerate(xl_op.get_ds_dp()):
            colname = "dU_dp{0}".format(j)
            reflections[colname].set_selected(subsel, dU)
        if xl_ucp is not None:
          for j, dB in enumerate(xl_ucp.get_ds_dp()):
            colname = "dB_dp{0}".format(j)
            reflections[colname].set_selected(subsel, dB)

    # set the UB matrices for prediction
    reflections['ub_matrix'] = reflections['u_matrix'] * reflections['b_matrix']

    return
Exemplo n.º 8
0
  def _local_setup(self, reflections):
    """Setup additional attributes used in gradients calculation. These are
    specific to scans-type prediction parameterisations"""

    # Spindle rotation matrices for every reflection
    #R = self._axis.axis_and_angle_as_r3_rotation_matrix(phi)
    #R = flex.mat3_double(len(reflections))
    # NB for now use flex.vec3_double.rotate_around_origin each time I need the
    # rotation matrix R.

    # r is the reciprocal lattice vector, in the lab frame
    self._phi_calc = reflections['xyzcal.mm'].parts()[2]
    q = self._fixed_rotation * (self._UB * self._h)
    self._r = self._setting_rotation * q.rotate_around_origin(self._axis, self._phi_calc)

    # All of the derivatives of phi have a common denominator, given by
    # (e X r).s0, where e is the rotation axis. Calculate this once, here.
    self._e_X_r = (self._setting_rotation * self._axis).cross(self._r)
    self._e_r_s0 = (self._e_X_r).dot(self._s0)

    # Note that e_r_s0 -> 0 when the rotation axis, beam vector and
    # relp are coplanar. This occurs when a reflection just touches
    # the Ewald sphere.
    #
    # There is a relationship between e_r_s0 and zeta_factor.
    # Uncommenting the code below shows that
    # s0.(e X r) = zeta * |s X s0|

    #from dials.algorithms.profile_model.gaussian_rs import zeta_factor
    #from libtbx.test_utils import approx_equal
    #s = matrix.col(reflections['s1'][0])
    #z = zeta_factor(axis[0], s0[0], s)
    #ss0 = (s.cross(matrix.col(s0[0]))).length()
    #assert approx_equal(e_r_s0[0], z * ss0)

    # catch small values of e_r_s0
    e_r_s0_mag = flex.abs(self._e_r_s0)
    try:
      assert flex.min(e_r_s0_mag) > 1.e-6
    except AssertionError as e:
      imin = flex.min_index(e_r_s0_mag)
      print "(e X r).s0 too small:"
      print "for", (e_r_s0_mag <= 1.e-6).count(True), "reflections"
      print "out of", len(e_r_s0_mag), "total"
      print "such as", reflections['miller_index'][imin]
      print "with scattering vector", reflections['s1'][imin]
      print "where r =", self._r[imin]
      print "e =", self._axis[imin]
      print "s0 =", self._s0[imin]
      print ("this reflection forms angle with the equatorial plane "
             "normal:")
      vecn = matrix.col(self._s0[imin]).cross(matrix.col(self._axis[imin])).normalize()
      print matrix.col(reflections['s1'][imin]).accute_angle(vecn)
      raise e
    return
Exemplo n.º 9
0
  def generate_mmcif(crystal, refiner, file):
    logger.info('Saving mmCIF information to %s' % file)
    from cctbx import miller
    import datetime
    import iotbx.cif.model
    import math

    block = iotbx.cif.model.block()
    block["_audit.creation_method"] = dials_version()
    block["_audit.creation_date"] = datetime.date.today().isoformat()
#   block["_publ.section_references"] = '' # once there is a reference...

    for cell, esd, cifname in zip(crystal.get_unit_cell().parameters(),
                                  crystal.get_cell_parameter_sd(),
                                  ['length_a', 'length_b', 'length_c', 'angle_alpha', 'angle_beta', 'angle_gamma']):
      block['_cell.%s' % cifname] = "%.8f" % cell
      block['_cell.%s_esd' % cifname] = "%.8f" % esd
    block['_cell.volume'] = "%f" % crystal.get_unit_cell().volume()
    block['_cell.volume_esd'] = "%f" % crystal.get_cell_volume_sd()

    used_reflections = refiner.get_matches()
    block['_cell_measurement.reflns_used'] = len(used_reflections)
    block['_cell_measurement.theta_min'] = flex.min(used_reflections['2theta_obs.rad']) * 180 / math.pi / 2
    block['_cell_measurement.theta_max'] = flex.max(used_reflections['2theta_obs.rad']) * 180 / math.pi / 2
    block['_diffrn_reflns.number'] = len(used_reflections)
    miller_span = miller.index_span(used_reflections['miller_index'])
    min_h, min_k, min_l = miller_span.min()
    max_h, max_k, max_l = miller_span.max()
    block['_diffrn_reflns.limit_h_min'] = min_h
    block['_diffrn_reflns.limit_h_max'] = max_h
    block['_diffrn_reflns.limit_k_min'] = min_k
    block['_diffrn_reflns.limit_k_max'] = max_k
    block['_diffrn_reflns.limit_l_min'] = min_l
    block['_diffrn_reflns.limit_l_max'] = max_l
    block['_diffrn_reflns.theta_min'] = flex.min(used_reflections['2theta_obs.rad']) * 180 / math.pi / 2
    block['_diffrn_reflns.theta_max'] = flex.max(used_reflections['2theta_obs.rad']) * 180 / math.pi / 2

    cif = iotbx.cif.model.cif()
    cif['two_theta_refine'] = block
    with open(file, 'w') as fh:
      cif.show(out=fh)
Exemplo n.º 10
0
def profile3d(p, vmin=None, vmax=None):
  ''' Print a 3D profile. '''
  from dials.array_family import flex
  if vmin is None:
    vmin = flex.min(p)
  if vmax is None:
    vmax = flex.max(p)
  nz, ny, nx = p.all()
  text = []
  for k in range(nz):
    p2 = p[k:k+1,:,:]
    p2.reshape(flex.grid(ny, nx))
    text.append(profile2d(p2, vmin=vmin, vmax=vmax))
  return '\n'.join(text)
Exemplo n.º 11
0
def ice_rings_selection(reflections):
  d_star_sq = flex.pow2(reflections['rlp'].norms())
  d_spacings = uctbx.d_star_sq_as_d(d_star_sq)

  from dials.algorithms.integration import filtering

  unit_cell = uctbx.unit_cell((4.498,4.498,7.338,90,90,120))
  space_group = sgtbx.space_group_info(number=194).group()
  width = 0.06

  ice_filter = filtering.PowderRingFilter(
    unit_cell, space_group, flex.min(d_spacings)-width, width)

  ice_sel = ice_filter(d_spacings)
  return ice_sel
Exemplo n.º 12
0
  def get_normalized_colors(self, data, vmin=None, vmax=None):
    if vmax is None:
      vmax = self.params.residuals.plot_max
    if vmax is None:
      vmax = flex.max(data)
    if vmin is None:
      vmin = flex.min(data)

    # initialize the color map
    norm = Normalize(vmin=vmin, vmax=vmax)
    cmap = plt.cm.get_cmap(self.params.colormap)
    sm = cm.ScalarMappable(norm=norm, cmap=cmap)
    color_vals = np.linspace(vmin, vmax, 11)
    sm.set_array(color_vals) # needed for colorbar

    return norm, cmap, color_vals, sm
Exemplo n.º 13
0
def plot_statistics(statistics, prefix='', degrees_per_bin=5,
                    cutoff_anom=None, cutoff_non_anom=None):

  range_width = 1
  range_min = flex.min(statistics.dose) - range_width
  range_max = flex.max(statistics.dose)
  n_steps = 2 + int((range_max - range_min) - range_width)
  x = flex.double_range(n_steps) * range_width + range_min
  x *= degrees_per_bin

  dpi = 300
  import matplotlib
  matplotlib.use('Agg')
  from matplotlib import pyplot as plt
  try: plt.style.use('ggplot')
  except AttributeError: pass
  line1,  = plt.plot(x, statistics.ieither_completeness, label='Unique reflections')
  line2,  = plt.plot(x, statistics.iboth_completeness, label='Bijvoet pairs')
  if cutoff_non_anom is not None:
    plt.plot([cutoff_non_anom, cutoff_non_anom], plt.ylim(), c=line1.get_color(), linestyle='dashed')
  if cutoff_anom is not None:
    plt.plot([cutoff_anom, cutoff_anom], plt.ylim(), c=line2.get_color(), linestyle='dotted')
  plt.xlim(0, plt.xlim()[1])
  plt.xlabel('Scan angle (degrees)')
  plt.ylabel('Completeness (%)')
  plt.ylim(0, 1)
  plt.legend(loc='lower right', fontsize='small')
  plt.savefig('%scompleteness_vs_scan_angle.png' %prefix, dpi=dpi)
  plt.clf()

  line1, = plt.plot(x[1:], 100 * statistics.frac_new_ref, label='Unique reflections')
  line2, = plt.plot(x[1:], 100 * statistics.frac_new_pairs, label='Bijvoet pairs')
  ylim = plt.ylim()
  if cutoff_non_anom is not None:
    plt.plot([cutoff_non_anom, cutoff_non_anom], ylim, c=line1.get_color(), linestyle='dashed')
  if cutoff_anom is not None:
    plt.plot([cutoff_anom, cutoff_anom], ylim, c=line2.get_color(), linestyle='dotted')
  plt.ylim(ylim)
  plt.xlim(0, plt.xlim()[1])
  plt.xlabel('Scan angle (degrees)')
  plt.ylabel('% new reflections per degree')
  plt.legend(loc='upper right', fontsize='small')
  plt.savefig('%spercent_new_reflections_vs_scan_angle.png' %prefix, dpi=dpi)
  plt.clf()
Exemplo n.º 14
0
  def plot_one_model(self,nrow,out):
    fig = plt.subplot(self.gs[nrow*self.ncols])
    two_thetas = self.reduction.get_two_theta_deg()
    degrees = self.reduction.get_delta_psi_deg()

    if self.color_encoding=="conventional":
          positive = (self.reduction.i_sigi>=0.)
          fig.plot(two_thetas.select(positive), degrees.select(positive), "bo")
          fig.plot(two_thetas.select(~positive), degrees.select(~positive), "r+")
    elif self.color_encoding=="I/sigma":
          positive = (self.reduction.i_sigi>=0.)
          tt_selected = two_thetas.select(positive)
          dp_selected = degrees.select(positive)
          i_sigi_select = self.reduction.i_sigi.select(positive)
          order = flex.sort_permutation(i_sigi_select)
          tt_selected = tt_selected.select(order)
          dp_selected = dp_selected.select(order)
          i_sigi_selected = i_sigi_select.select(order)
          from matplotlib.colors import Normalize
          dnorm = Normalize()
          dcolors = i_sigi_selected.as_numpy_array()
          dnorm.autoscale(dcolors)
          N = len(dcolors)
          CMAP = plt.get_cmap("rainbow")
          if self.refined.get("partiality_array",None) is None:
            for n in xrange(N):
              fig.plot([tt_selected[n]],[dp_selected[n]],
              color=CMAP(dnorm(dcolors[n])),marker=".", markersize=10)
          else:
            partials = self.refined.get("partiality_array")
            partials_select = partials.select(positive)
            partials_selected = partials_select.select(order)
            assert len(partials)==len(positive)
            for n in xrange(N):
              fig.plot([tt_selected[n]],[dp_selected[n]],
              color=CMAP(dnorm(dcolors[n])),marker=".", markersize=20*partials_selected[n])
              # change the markersize to indicate partiality.
          negative = (self.reduction.i_sigi<0.)
          fig.plot(two_thetas.select(negative), degrees.select(negative), "r+", linewidth=1)
    else:
          strong = (self.reduction.i_sigi>=10.)
          positive = ((~strong) & (self.reduction.i_sigi>=0.))
          negative = (self.reduction.i_sigi<0.)
          assert (strong.count(True)+positive.count(True)+negative.count(True) ==
                  len(self.reduction.i_sigi))
          fig.plot(two_thetas.select(positive), degrees.select(positive), "bo")
          fig.plot(two_thetas.select(strong), degrees.select(strong), marker='.',linestyle='None',
           markerfacecolor='#00ee00', markersize=10)
          fig.plot(two_thetas.select(negative), degrees.select(negative), "r+")

    # indicate the imposed resolution filter
    wavelength = self.reduction.experiment.beam.get_wavelength()
    imposed_res_filter = self.reduction.get_imposed_res_filter(out)
    resolution_markers = [
      a for a in [imposed_res_filter,self.reduction.measurements.d_min()] if a is not None]
    for RM in resolution_markers:
          two_th = (180./math.pi)*2.*math.asin(wavelength/(2.*RM))
          plt.plot([two_th, two_th],[self.AD1TF7B_MAXDP*-0.8,self.AD1TF7B_MAXDP*0.8],'k-')
          plt.text(two_th,self.AD1TF7B_MAXDP*-0.9,"%4.2f"%RM)

    #indicate the linefit
    mean = flex.mean(degrees)
    minplot = flex.min(two_thetas)
    plt.plot([0,minplot],[mean,mean],"k-")
    LR = flex.linear_regression(two_thetas, degrees)
    model_y = LR.slope()*two_thetas + LR.y_intercept()
    plt.plot(two_thetas, model_y, "k-")

    #Now let's take care of the red and green lines.
    half_mosaic_rotation_deg = self.refined["half_mosaic_rotation_deg"]
    mosaic_domain_size_ang = self.refined["mosaic_domain_size_ang"]
    red_curve_domain_size_ang = self.refined.get("red_curve_domain_size_ang",mosaic_domain_size_ang)
    a_step = self.AD1TF7B_MAX2T / 50.
    a_range = flex.double([a_step*x for x in xrange(1,50)]) # domain two-theta array
    #Bragg law [d=L/2sinTH]
    d_spacing = (wavelength/(2.*flex.sin(math.pi*a_range/360.)))
    # convert two_theta to a delta-psi.  Formula for Deffective [Dpsi=d/2Deff]
    inner_phi_deg = flex.asin((d_spacing / (2.*red_curve_domain_size_ang)) )*(180./math.pi)
    outer_phi_deg = flex.asin((d_spacing / (2.*mosaic_domain_size_ang)) + \
      half_mosaic_rotation_deg*math.pi/180. )*(180./math.pi)
    plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots\n%s"%(
          2.*half_mosaic_rotation_deg, mosaic_domain_size_ang, len(two_thetas),
          os.path.basename(self.reduction.filename)))
    plt.plot(a_range, inner_phi_deg, "r-")
    plt.plot(a_range,-inner_phi_deg, "r-")
    plt.plot(a_range, outer_phi_deg, "g-")
    plt.plot(a_range, -outer_phi_deg, "g-")
    plt.xlim([0,self.AD1TF7B_MAX2T])
    plt.ylim([-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP])

    #second plot shows histogram
    fig = plt.subplot(self.gs[1+nrow*self.ncols])
    plt.xlim([-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP])
    nbins = 50
    n,bins,patches = plt.hist(dp_selected, nbins,
           range=(-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP),
           weights=self.reduction.i_sigi.select(positive),
           normed=0, facecolor="orange", alpha=0.75)
    #ersatz determine the median i_sigi point:
    isi_positive = self.reduction.i_sigi.select(positive)
    isi_order = flex.sort_permutation(isi_positive)
    reordered = isi_positive.select(isi_order)
    isi_median = reordered[int(len(isi_positive)*0.9)]
    isi_top_half_selection = (isi_positive>isi_median)
    n,bins,patches = plt.hist(dp_selected.select(isi_top_half_selection), nbins,
           range=(-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP),
           weights=isi_positive.select(isi_top_half_selection),
           normed=0, facecolor="#ff0000", alpha=0.75)
    plt.xlabel("(degrees)")
    plt.title("Weighted histogram of Delta-psi")
Exemplo n.º 15
0
      patch[(y, x)] += 1.0 * iw / scale

  cc = profile_correlation(data, patch)
  if params.show:
    print 'Simulated reflection (flattened in Z):'
    print
    for j in range(dy):
      for i in range(dx):
        print '%5d' % int(patch[(j, i)]),
      print
    print 'Correlation coefficient: %.3f isum: %.1f ' % (cc, i0)

  import numpy as np
  maxx = flex.max(all_pix.parts()[0])
  maxy = flex.max(all_pix.parts()[1])
  minx = flex.min(all_pix.parts()[0])
  miny = flex.min(all_pix.parts()[1])
  dx = (maxx-minx)/2
  dy = (maxy-miny)/2
  medx = minx + (dx)
  medy = miny + (dy)
  if maxx-minx > maxy-miny:
    miny = medy-dx
    maxy = medy+dx
  else:
    minx = medx-dy
    maxx = medx+dy
  limits = [[minx, maxx], [miny, maxy]]

  reflection['xyzsim.mm'] = (flex.mean_weighted(all_pix.parts()[0], all_iw),
                             flex.mean_weighted(all_pix.parts()[1], all_iw), 0.0)
Exemplo n.º 16
0
def spot_counts_per_image_plot(reflections, char='*', width=60, height=10):
  from dials.array_family import flex

  if len(reflections) == 0:
    return '\n'

  assert isinstance(char, basestring)
  assert len(char) == 1

  x,y,z = reflections['xyzobs.px.value'].parts()

  min_z = flex.min(z)
  max_z = flex.max(z)

  # image numbers to display on x-axis label
  xlab = (int(round(min_z + 0.5)), int(round(max_z + 0.5)))
  # estimate the total number of images
  image_count = xlab[1] - xlab[0] + 1

  z_range = max_z - min_z + 1
  if z_range <= 1:
    return '%i spots found on 1 image' %len(reflections)

  width = int(min(z_range, width))
  z_step = z_range / width
  z_bound = min_z + z_step - 0.5
# print [round(i * 10) / 10 for i in sorted(z)]

  counts = flex.double()

  sel = (z < z_bound)
  counts.append(sel.count(True))
# print 0, ('-', z_bound), sel.count(True)
  for i in range(1, width-1):
    sel = ((z >= z_bound) & (z < (z_bound + z_step)))
    counts.append(sel.count(True))
#   print i, (z_bound, z_bound + z_step), sel.count(True)
    z_bound += z_step
  sel = (z >= z_bound)
# print i + 1, (z_bound, '-'), sel.count(True)
  counts.append(sel.count(True))

  max_count = flex.max(counts)
  total_counts = flex.sum(counts)
  assert total_counts == len(z)
  counts *= (height/max_count)
  counts = counts.iround()

  rows = []
  rows.append('%i spots found on %i images (max %i / bin)' %(
    total_counts, image_count, max_count))

  for i in range(height, 0, -1):
    row = []
    for j, c in enumerate(counts):
      if c > (i - 1):
        row.append(char)
      else:
        row.append(' ')
    rows.append(''.join(row))

  padding = width - len(str(xlab[0])) - len(str(xlab[1]))
  rows.append('%i%s%i' % (xlab[0],
    (' ' if padding < 7 else 'image').center(padding),
    xlab[1]))
  return '\n'.join(rows)
Exemplo n.º 17
0
    def __call__(self, reflections):
        """Identify outliers in the input and set the centroid_outlier flag.
    Return True if any outliers were detected, otherwise False"""

        if self._verbosity > 0:
            logger.info("Detecting centroid outliers using the {0} algorithm".format(type(self).__name__))

        # check the columns are present
        for col in self._cols:
            assert col in reflections

        sel = reflections.get_flags(reflections.flags.used_in_refinement)
        all_data = reflections.select(sel)
        all_data_indices = sel.iselection()
        nexp = flex.max(all_data["id"]) + 1

        jobs = []
        if self._separate_experiments:
            # split the data set by experiment id
            for iexp in xrange(nexp):
                sel = all_data["id"] == iexp
                job = {
                    "id": iexp,
                    "panel": "all",
                    "data": all_data.select(sel),
                    "indices": all_data_indices.select(sel),
                }
                jobs.append(job)
        else:
            # keep the whole dataset across all experiment ids
            job = {"id": "all", "panel": "all", "data": all_data, "indices": all_data_indices}
            jobs.append(job)

        jobs2 = []
        if self._separate_panels:
            # split further by panel id
            for job in jobs:
                data = job["data"]
                iexp = job["id"]
                indices = job["indices"]
                for ipanel in xrange(flex.max(data["panel"]) + 1):
                    sel = data["panel"] == ipanel
                    job = {"id": iexp, "panel": ipanel, "data": data.select(sel), "indices": indices.select(sel)}
                    jobs2.append(job)
        else:
            # keep the splits as they are
            jobs2 = jobs

        jobs3 = []
        if self.get_block_width() is not None:
            # split into equal-sized phi ranges
            for job in jobs2:
                data = job["data"]
                iexp = job["id"]
                ipanel = job["panel"]
                indices = job["indices"]
                phi = data["xyzobs.mm.value"].parts()[2]
                if len(phi) == 0:  # detect no data in the job
                    jobs3.append(job)
                    continue
                phi_low = flex.min(phi)
                phi_range = flex.max(phi) - phi_low
                if phi_range == 0.0:  # detect stills and do not split
                    jobs3.append(job)
                    continue
                bw = self.get_block_width(iexp)
                if bw is None:  # detect no split for this experiment
                    jobs3.append(job)
                    continue
                nblocks = int(round(RAD2DEG * phi_range / bw))
                nblocks = max(1, nblocks)
                real_width = phi_range / nblocks
                block_end = 0.0
                for iblock in xrange(nblocks - 1):  # all except the last block
                    block_start = iblock * real_width
                    block_end = (iblock + 1) * real_width
                    sel = (phi >= (phi_low + block_start)) & (phi < (phi_low + block_end))
                    job = {
                        "id": iexp,
                        "panel": ipanel,
                        "data": data.select(sel),
                        "indices": indices.select(sel),
                        "phi_start": RAD2DEG * (phi_low + block_start),
                        "phi_end": RAD2DEG * (phi_low + block_end),
                    }
                    jobs3.append(job)
                # now last block
                sel = phi >= (phi_low + block_end)
                job = {
                    "id": iexp,
                    "panel": ipanel,
                    "data": data.select(sel),
                    "indices": indices.select(sel),
                    "phi_start": RAD2DEG * (phi_low + block_end),
                    "phi_end": RAD2DEG * (phi_low + phi_range),
                }
                jobs3.append(job)
        else:
            # keep the splits as they are
            jobs3 = jobs2

        # Work out the format of the jobs table
        if self._verbosity > 0:
            header = ["Job"]
            if self._separate_experiments:
                header.append("Exp\nid")
            if self._separate_panels:
                header.append("Panel\nid")
            if self.get_block_width() is not None:
                header.append("Block range\n(deg)")
            header.extend(["Nref", "Nout", "%out"])
            rows = []

        # now loop over the lowest level of splits
        for i, job in enumerate(jobs3):

            data = job["data"]
            indices = job["indices"]
            iexp = job["id"]
            ipanel = job["panel"]
            nref = len(indices)

            if nref >= self._min_num_obs:

                # get the subset of data as a list of columns
                cols = [data[col] for col in self._cols]

                # determine the position of outliers on this sub-dataset
                outliers = self._detect_outliers(cols)

                # get positions of outliers from the original matches
                ioutliers = indices.select(outliers)

            elif nref > 0:
                # too few reflections in the job
                msg = "For job {0}, fewer than {1} reflections are present.".format(i + 1, self._min_num_obs)
                msg += " All reflections flagged as possible outliers."
                if self._verbosity > 0:
                    logger.debug(msg)
                ioutliers = indices

            else:
                # no reflections in the job
                ioutliers = indices

            # set the centroid_outlier flag in the original reflection table
            nout = len(ioutliers)
            if nout > 0:
                reflections.set_flags(ioutliers, reflections.flags.centroid_outlier)
                self.nreject += nout

            # Add job data to the table
            if self._verbosity > 0:
                row = [str(i + 1)]
                if self._separate_experiments:
                    row.append(str(iexp))
                if self._separate_panels:
                    row.append(str(ipanel))
                if self.get_block_width() is not None:
                    try:
                        row.append("{phi_start:.2f} - {phi_end:.2f}".format(**job))
                    except KeyError:
                        row.append("{0:.2f} - {1:.2f}".format(0.0, 0.0))
                if nref == 0:
                    p100 = 0
                else:
                    p100 = nout / nref * 100.0
                    if p100 > 30.0:
                        msg = ("{0:3.1f}% of reflections were flagged as outliers from job" " {1}").format(p100, i + 1)
                row.extend([str(nref), str(nout), "%3.1f" % p100])
                rows.append(row)

        if self.nreject == 0:
            return False
        if self._verbosity > 0:
            logger.info("{0} reflections have been flagged as outliers".format(self.nreject))
            logger.debug("Outlier rejections per job:")
            st = simple_table(rows, header)
            logger.debug(st.format())

        return True
    def detector_plot_dict(self,
                           detector,
                           data,
                           title,
                           units_str,
                           show=True,
                           reverse_colormap=False):
        """
    Use matplotlib to plot a detector, color coding panels according to data
    @param detector detector reference detector object
    @param data python dictionary of panel names as keys and numbers as values
    @param title title string for plot
    @param units_str string with a formatting statment for units on each panel
    """
        # initialize the color map
        values = flex.double(data.values())
        norm = Normalize(vmin=flex.min(values), vmax=flex.max(values))
        if reverse_colormap:
            cmap = plt.cm.get_cmap(self.params.colormap + "_r")
        else:
            cmap = plt.cm.get_cmap(self.params.colormap)
        sm = cm.ScalarMappable(norm=norm, cmap=cmap)
        if len(values) == 0:
            print "no values"
            return
        elif len(values) == 1:
            sm.set_array(np.arange(values[0], values[0],
                                   1))  # needed for colorbar
        else:
            sm.set_array(
                np.arange(flex.min(values), flex.max(values),
                          (flex.max(values) - flex.min(values)) /
                          20))  # needed for colorbar

        fig = plt.figure()
        ax = fig.add_subplot(111, aspect='equal')
        max_dim = 0
        root = detector.hierarchy()
        rf = col(root.get_fast_axis())
        rs = col(root.get_slow_axis())
        for pg_id, pg in enumerate(
                iterate_detector_at_level(root, 0,
                                          self.params.hierarchy_level)):
            if pg.get_name() not in data:
                continue
            # get panel coordinates
            p0, p1, p2, p3 = get_bounds(root, pg)

            v1 = p1 - p0
            v2 = p3 - p0
            vcen = ((v2 / 2) + (v1 / 2)) + p0

            # add the panel to the plot
            ax.add_patch(
                Polygon((p0[0:2], p1[0:2], p2[0:2], p3[0:2]),
                        closed=True,
                        color=sm.to_rgba(data[pg.get_name()]),
                        fill=True))
            ax.annotate("%d %s" % (pg_id, units_str % data[pg.get_name()]),
                        vcen[0:2],
                        ha='center')

            if self.params.draw_normal_arrows:
                pgn = col(pg.get_normal())
                v = col((rf.dot(pgn), rs.dot(pgn), 0))
                v *= 10000
                ax.arrow(vcen[0],
                         vcen[1],
                         v[0],
                         v[1],
                         head_width=5.0,
                         head_length=10.0,
                         fc='k',
                         ec='k')

            # find the plot maximum dimensions
            for p in [p0, p1, p2, p3]:
                for c in p[0:2]:
                    if abs(c) > max_dim:
                        max_dim = abs(c)

        # plot the results
        ax.set_xlim((-max_dim, max_dim))
        ax.set_ylim((-max_dim, max_dim))
        ax.set_xlabel("mm")
        ax.set_ylabel("mm")
        fig.colorbar(sm)
        plt.title(title)
        if show:
            plt.show()
Exemplo n.º 19
0
    def read_mtzfile(filename, batch_offset=None):
        """
        Read the mtz file
        """
        miller_arrays = mtz.object(file_name=filename).as_miller_arrays(
            merge_equivalents=False)

        # Select the desired columns
        intensities = None
        batches = None
        for array in miller_arrays:
            if array.info().labels == ["I", "SIGI"]:
                intensities = array
            if array.info().labels == ["BATCH"]:
                batches = array
        if not intensities:
            raise KeyError(
                "Intensities not found in mtz file, expected labels I, SIGI")
        if not batches:
            raise KeyError("Batch values not found")
        if batches.data().size() != intensities.data().size():
            raise ValueError("Batch and intensity array sizes do not match")

        # Get the unit cell and space group
        unit_cell = intensities.unit_cell()
        space_group = intensities.crystal_symmetry().space_group()

        # The reflection data
        table = flex.reflection_table()
        table["miller_index"] = intensities.indices()
        table["intensity"] = intensities.data()
        table["variance"] = flex.pow2(intensities.sigmas())

        # Create unit cell list
        zeroed_batches = batches.data() - flex.min(batches.data())
        dataset = flex.int(table.size(), 0)
        sorted_batches = flex.sorted(zeroed_batches)
        sel_perm = flex.sort_permutation(zeroed_batches)

        if not batch_offset:
            previous = 0
            potential_batch_offsets = flex.double()
            for i, b in enumerate(sorted_batches):
                if b - previous > 1:
                    potential_batch_offsets.append(b - previous)
                previous = b
            potential = flex.sorted(potential_batch_offsets)
            # potential is a list of low numbers (where images may not have any spots)
            # and larger numbers between batches.
            if len(potential) == 1:
                batch_offset = potential[0]
                logger.info(
                    """
Using a batch offset of %s to split datasets.
Batch offset can be specified with mtz.batch_offset=
""",
                    batch_offset,
                )
            elif len(potential) > 1:
                diffs = flex.double([
                    potential[i + 1] - p for i, p in enumerate(potential[:-1])
                ])
                i = flex.sort_permutation(diffs)[-1]
                batch_offset = int(potential[i + 1] - (0.2 * diffs[i]))
                logger.info(
                    """
Using an approximate batch offset of %s to split datasets.
Batch offset can be specified with mtz.batch_offset=
""",
                    batch_offset,
                )
            else:
                batch_offset = 1

        previous = 0
        dataset_no = 0
        for i, b in enumerate(sorted_batches):
            if b - previous > batch_offset - 1:
                dataset_no += 1
            dataset[i] = dataset_no
            previous = b

        table["dataset"] = flex.int(table.size(), 0)
        table["dataset"].set_selected(sel_perm, dataset)

        return table, unit_cell, space_group
Exemplo n.º 20
0
  avr2.reshape(avr.accessor())
  avr = avr2
  std2 = stddev.as_1d()
  std2.set_selected(indices, flex.double(len(indices), 0))
  std2.reshape(stddev.accessor())
  stddev = std2
  iod = flex.double(len(avr))
  mask2 = avr2.as_1d() > 0
  indices2 = flex.size_t(range(len(mask))).select(mask2)
  avr2 = avr2.as_1d().select(mask2)
  var2 = std2.as_1d().select(mask2)**2
  iod.set_selected(indices2, var2 / avr2)
  iod.reshape(avr.accessor())
  mask.reshape(avr.accessor())

  print flex.max(avr), flex.min(avr)
  from matplotlib import pylab
  pylab.imshow(iod.as_numpy_array(), vmax=3, interpolation='none')
  pylab.colorbar()
  pylab.show()
  exit(0)
  from matplotlib import pylab
  pylab.imshow(avr.as_numpy_array(), interpolation='none')
  pylab.colorbar()
  pylab.show()
  pylab.hist(avr, bins=100)
  pylab.show()

  #avr = mean_filter(avr, mask, (1,1), 2)
  #print flex.max(avr), flex.min(avr)
Exemplo n.º 21
0
 def onclick(event):
   import math
   ts = event.xdata
   diffs = flex.abs(t - ts)
   ts = t[flex.first_index(diffs, flex.min(diffs))]
   print get_paths_from_timestamps([ts], tag="shot")[0]
Exemplo n.º 22
0
    def index(self):
        experiments = ExperimentList()

        had_refinement_error = False
        have_similar_crystal_models = False

        while True:
            if had_refinement_error or have_similar_crystal_models:
                break
            max_lattices = self.params.multiple_lattice_search.max_lattices
            if max_lattices is not None and len(experiments) >= max_lattices:
                break
            if len(experiments) > 0:
                cutoff_fraction = (self.params.multiple_lattice_search.
                                   recycle_unindexed_reflections_cutoff)
                d_spacings = 1 / self.reflections["rlp"].norms()
                d_min_indexed = flex.min(
                    d_spacings.select(self.indexed_reflections))
                min_reflections_for_indexing = cutoff_fraction * len(
                    self.reflections.select(d_spacings > d_min_indexed))
                crystal_ids = self.reflections.select(
                    d_spacings > d_min_indexed)["id"]
                if (crystal_ids
                        == -1).count(True) < min_reflections_for_indexing:
                    logger.info(
                        "Finish searching for more lattices: %i unindexed reflections remaining."
                        % ((crystal_ids == -1).count(True)))
                    break

            n_lattices_previous_cycle = len(experiments)

            if self.d_min is None:
                self.d_min = self.params.refinement_protocol.d_min_start

            if len(experiments) == 0:
                new_expts = self.find_lattices()
                generate_experiment_identifiers(new_expts)
                experiments.extend(new_expts)
            else:
                try:
                    new = self.find_lattices()
                    generate_experiment_identifiers(new)
                    experiments.extend(new)
                except DialsIndexError:
                    logger.info("Indexing remaining reflections failed")

            if self.params.refinement_protocol.d_min_step is libtbx.Auto:
                n_cycles = self.params.refinement_protocol.n_macro_cycles
                if self.d_min is None or n_cycles == 1:
                    self.params.refinement_protocol.d_min_step = 0
                else:
                    d_spacings = 1 / self.reflections["rlp"].norms()
                    d_min_all = flex.min(d_spacings)
                    self.params.refinement_protocol.d_min_step = (
                        self.d_min - d_min_all) / (n_cycles - 1)
                    logger.info("Using d_min_step %.1f" %
                                self.params.refinement_protocol.d_min_step)

            if len(experiments) == 0:
                raise DialsIndexError("No suitable lattice could be found.")
            elif len(experiments) == n_lattices_previous_cycle:
                # no more lattices found
                break

            for i_cycle in range(
                    self.params.refinement_protocol.n_macro_cycles):
                if (i_cycle > 0 and self.d_min is not None
                        and self.params.refinement_protocol.d_min_step > 0):
                    d_min = self.d_min - self.params.refinement_protocol.d_min_step
                    d_min = max(d_min, 0)
                    if self.params.refinement_protocol.d_min_final is not None:
                        d_min = max(
                            d_min, self.params.refinement_protocol.d_min_final)
                    if d_min >= 0:
                        self.d_min = d_min
                        logger.info("Increasing resolution to %.2f Angstrom" %
                                    d_min)

                # reset reflection lattice flags
                # the lattice a given reflection belongs to: a value of -1 indicates
                # that a reflection doesn't belong to any lattice so far
                self.reflections["id"] = flex.int(len(self.reflections), -1)

                self.index_reflections(experiments, self.reflections)

                if i_cycle == 0 and self.params.known_symmetry.space_group is not None:
                    self._apply_symmetry_post_indexing(
                        experiments, self.reflections,
                        n_lattices_previous_cycle)

                logger.info("\nIndexed crystal models:")
                self.show_experiments(experiments,
                                      self.reflections,
                                      d_min=self.d_min)

                if self._check_have_similar_crystal_models(experiments):
                    have_similar_crystal_models = True
                    break

                logger.info("")
                logger.info("#" * 80)
                logger.info("Starting refinement (macro-cycle %i)" %
                            (i_cycle + 1))
                logger.info("#" * 80)
                logger.info("")
                self.indexed_reflections = self.reflections["id"] > -1

                sel = flex.bool(len(self.reflections), False)
                lengths = 1 / self.reflections["rlp"].norms()
                if self.d_min is not None:
                    isel = (lengths <= self.d_min).iselection()
                    sel.set_selected(isel, True)
                sel.set_selected(self.reflections["id"] == -1, True)
                self.reflections.unset_flags(sel,
                                             self.reflections.flags.indexed)
                self.unindexed_reflections = self.reflections.select(sel)

                reflections_for_refinement = self.reflections.select(
                    self.indexed_reflections)
                if self.params.refinement_protocol.mode == "repredict_only":
                    refined_experiments, refined_reflections = (
                        experiments,
                        reflections_for_refinement,
                    )
                    from dials.algorithms.refinement.prediction.managed_predictors import (
                        ExperimentsPredictorFactory, )

                    ref_predictor = ExperimentsPredictorFactory.from_experiments(
                        experiments,
                        spherical_relp=self.all_params.refinement.
                        parameterisation.spherical_relp_model,
                    )
                    ref_predictor(refined_reflections)
                else:
                    try:
                        refined_experiments, refined_reflections = self.refine(
                            experiments, reflections_for_refinement)
                    except (DialsRefineConfigError,
                            DialsRefineRuntimeError) as e:
                        if len(experiments) == 1:
                            raise DialsIndexRefineError(str(e))
                        had_refinement_error = True
                        logger.info("Refinement failed:")
                        logger.info(e)
                        del experiments[-1]
                        break

                self._unit_cell_volume_sanity_check(experiments,
                                                    refined_experiments)

                self.refined_reflections = refined_reflections
                self.refined_reflections.unset_flags(
                    self.refined_reflections["id"] < 0,
                    self.refined_reflections.flags.indexed,
                )

                for i, expt in enumerate(self.experiments):
                    ref_sel = self.refined_reflections.select(
                        self.refined_reflections["imageset_id"] == i)
                    ref_sel = ref_sel.select(ref_sel["id"] >= 0)
                    for i_expt in set(ref_sel["id"]):
                        refined_expt = refined_experiments[i_expt]
                        expt.detector = refined_expt.detector
                        expt.beam = refined_expt.beam
                        expt.goniometer = refined_expt.goniometer
                        expt.scan = refined_expt.scan
                        refined_expt.imageset = expt.imageset

                if not (self.all_params.refinement.parameterisation.beam.fix
                        == "all" and self.all_params.refinement.
                        parameterisation.detector.fix == "all"):
                    # Experimental geometry may have changed - re-map centroids to
                    # reciprocal space
                    self.reflections.map_centroids_to_reciprocal_space(
                        self.experiments)

                # update for next cycle
                experiments = refined_experiments
                self.refined_experiments = refined_experiments

                logger.info("\nRefined crystal models:")
                self.show_experiments(self.refined_experiments,
                                      self.reflections,
                                      d_min=self.d_min)

                if (i_cycle >= 2 and self.d_min
                        == self.params.refinement_protocol.d_min_final):
                    logger.info(
                        "Target d_min_final reached: finished with refinement")
                    break

        if self.refined_experiments is None:
            raise DialsIndexRefineError(
                "None of the experiments could refine.")

        if len(self.refined_experiments) > 1:
            from dials.algorithms.indexing.compare_orientation_matrices import (
                rotation_matrix_differences, )

            logger.info(
                rotation_matrix_differences(
                    self.refined_experiments.crystals()))

        self._xyzcal_mm_to_px(self.experiments, self.refined_reflections)
Exemplo n.º 23
0
  def __call__(self):
    """Determine optimal mosaicity and domain size model (monochromatic)"""
    RR = self.refinery.predict_for_reflection_table(self.reflections)
    excursion_rad = RR["delpsical.rad"]
    delta_psi_deg = excursion_rad * 180./math.pi
    print
    print flex.max(delta_psi_deg), flex.min(delta_psi_deg)
    mean_excursion = flex.mean(delta_psi_deg)
    print "The mean excursion is %7.3f degrees, r.m.s.d %7.3f"%(mean_excursion, math.sqrt(flex.mean(RR["delpsical2"])))

    crystal = self.experiments[0].crystal
    beam = self.experiments[0].beam
    miller_indices = self.reflections["miller_index"]

    # FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection
    two_thetas = crystal.get_unit_cell().two_theta(miller_indices,beam.get_wavelength(),deg=True)
    dspacings = crystal.get_unit_cell().d(miller_indices)
    dspace_sq = dspacings * dspacings

    #  First -- try to get a reasonable envelope for the observed excursions.
        ## minimum of three regions; maximum of 50 measurements in each bin
    print "fitting parameters on %d spots"%len(excursion_rad)
    n_bins = min(max(3, len(excursion_rad)//25),50)
    bin_sz = len(excursion_rad)//n_bins
    print "nbins",n_bins,"bin_sz",bin_sz
    order = flex.sort_permutation(two_thetas)
    two_thetas_env = flex.double()
    dspacings_env = flex.double()
    excursion_rads_env = flex.double()
    for x in xrange(0,n_bins):
      subset = order[x*bin_sz:(x+1)*bin_sz]
      two_thetas_env.append(flex.mean(two_thetas.select(subset)))
      dspacings_env.append(flex.mean(dspacings.select(subset)))
      excursion_rads_env.append(flex.max(flex.abs(excursion_rad.select(subset))))

    #  Second -- parameter fit
        ## solve the normal equations
    sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
    sum_inv_u    = flex.sum(dspacings_env)
    sum_te_u     = flex.sum(dspacings_env * excursion_rads_env)
    sum_te       = flex.sum(excursion_rads_env)
    Normal_Mat   = sqr((sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
    Vector       = col((sum_te_u, sum_te))
    solution     = Normal_Mat.inverse() * Vector
    s_ang = 1./(2*solution[0])
    print "Best LSQ fit Scheerer domain size is %9.2f ang"%(
      s_ang)

    tan_phi_rad = dspacings / (2. * s_ang)
    tan_phi_deg = tan_phi_rad * 180./math.pi
    k_degrees = solution[1]* 180./math.pi
    print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f"%(2*k_degrees, k_degrees)
    tan_outer_deg = tan_phi_deg + k_degrees

    from xfel.mono_simulation.max_like import minimizer
    # coerce the estimates to be positive for max-likelihood
    lower_limit_domain_size = math.pow(crystal.get_unit_cell().volume(),
     1./3.)*3 # params.refinement.domain_size_lower_limit

    d_estimate = max(s_ang, lower_limit_domain_size)
    M = minimizer(d_i = dspacings, psi_i = excursion_rad, eta_rad = abs(2. * solution[1]),
                  Deff = d_estimate)
    print "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas))
    tan_phi_rad_ML = dspacings / (2. / M.x[0])
    tan_phi_deg_ML = tan_phi_rad_ML * 180./math.pi
    tan_outer_deg_ML = tan_phi_deg_ML + 0.5*M.x[1]*180./math.pi

    self.nv_acceptance_flags = flex.abs(delta_psi_deg) < tan_outer_deg_ML

    if self.graph_verbose: #params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit
      AD1TF7B_MAX2T = 30.
      AD1TF7B_MAXDP = 1.
      from matplotlib import pyplot as plt
      plt.plot(two_thetas, delta_psi_deg, "bo")
      minplot = flex.min(two_thetas)
      plt.plot([0,minplot],[mean_excursion,mean_excursion],"k-")
      LR = flex.linear_regression(two_thetas, delta_psi_deg)
      model_y = LR.slope()*two_thetas + LR.y_intercept()
      plt.plot(two_thetas, model_y, "k-")

      plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas)))
      plt.plot(two_thetas, tan_phi_deg_ML, "r.")
      plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
      plt.plot(two_thetas, tan_outer_deg_ML, "g.")
      plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
      plt.xlim([0,AD1TF7B_MAX2T])
      plt.ylim([-AD1TF7B_MAXDP,AD1TF7B_MAXDP])
      plt.show()
      plt.close()

    from xfel.mono_simulation.util import green_curve_area
    self.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML)
    print "The green curve area is ", self.green_curve_area

    crystal._ML_half_mosaicity_deg = M.x[1]*180./(2.*math.pi)
    crystal._ML_domain_size_ang = 2./M.x[0]
    self._ML_full_mosaicity_rad = M.x[1]
    self._ML_domain_size_ang = 2./M.x[0]

    #params.refinement.mosaic.model_expansion_factor
    """The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes
    as close as possible to # of observed reflections input, in the last integration call.  Determine this by
    inspecting the output log file interactively.  Do not exceed the bare minimum threshold needed.
    The intention is to find an optimal value, global for a given dataset."""
    model_expansion_factor = 1.4
    crystal._ML_half_mosaicity_deg *= model_expansion_factor
    crystal._ML_domain_size_ang /= model_expansion_factor

    return crystal
Exemplo n.º 24
0
  def finalize(self, data, mask):
    '''
    Finalize the model

    :param data: The data array
    :param mask: The mask array

    '''
    from dials.algorithms.image.filter import median_filter, mean_filter
    from dials.algorithms.image.fill_holes import diffusion_fill
    from dials.algorithms.image.fill_holes import simple_fill
    from dials.array_family import flex

    # Print some image properties
    sub_data = data.as_1d().select(mask.as_1d())
    logger.info('Raw image statistics:')
    logger.info('  min:  %d' % int(flex.min(sub_data)))
    logger.info('  max:  %d' % int(flex.max(sub_data)))
    logger.info('  mean: %d' % int(flex.mean(sub_data)))
    logger.info('')

    # Transform to polar
    logger.info('Transforming image data to polar grid')
    result = self.transform.to_polar(data, mask)
    data = result.data()
    mask = result.mask()
    sub_data = data.as_1d().select(mask.as_1d())
    logger.info('Polar image statistics:')
    logger.info('  min:  %d' % int(flex.min(sub_data)))
    logger.info('  max:  %d' % int(flex.max(sub_data)))
    logger.info('  mean: %d' % int(flex.mean(sub_data)))
    logger.info('')

    # Filter the image to remove noise
    if self.kernel_size > 0:
      if self.filter_type == 'median':
        logger.info('Applying median filter')
        data = median_filter(data, mask, (self.kernel_size, 0))
        sub_data = data.as_1d().select(mask.as_1d())
        logger.info('Median polar image statistics:')
        logger.info('  min:  %d' % int(flex.min(sub_data)))
        logger.info('  max:  %d' % int(flex.max(sub_data)))
        logger.info('  mean: %d' % int(flex.mean(sub_data)))
        logger.info('')
      elif self.filter_type == 'mean':
        logger.info('Applying mean filter')
        mask_as_int = mask.as_1d().as_int()
        mask_as_int.reshape(mask.accessor())
        data = mean_filter(data, mask_as_int, (self.kernel_size, 0), 1)
        sub_data = data.as_1d().select(mask.as_1d())
        logger.info('Mean polar image statistics:')
        logger.info('  min:  %d' % int(flex.min(sub_data)))
        logger.info('  max:  %d' % int(flex.max(sub_data)))
        logger.info('  mean: %d' % int(flex.mean(sub_data)))
        logger.info('')
      else:
        raise RuntimeError('Unknown filter_type: %s' % self.filter_type)

    # Fill any remaining holes
    logger.info("Filling holes")
    data = simple_fill(data, mask)
    data = diffusion_fill(data, mask, self.niter)
    mask = flex.bool(data.accessor(), True)
    sub_data = data.as_1d().select(mask.as_1d())
    logger.info('Filled polar image statistics:')
    logger.info('  min:  %d' % int(flex.min(sub_data)))
    logger.info('  max:  %d' % int(flex.max(sub_data)))
    logger.info('  mean: %d' % int(flex.mean(sub_data)))
    logger.info('')

    # Transform back
    logger.info('Transforming image data from polar grid')
    result = self.transform.from_polar(data, mask)
    data = result.data()
    mask = result.mask()
    sub_data = data.as_1d().select(mask.as_1d())
    logger.info('Final image statistics:')
    logger.info('  min:  %d' % int(flex.min(sub_data)))
    logger.info('  max:  %d' % int(flex.max(sub_data)))
    logger.info('  mean: %d' % int(flex.mean(sub_data)))
    logger.info('')

    # Fill in any discontinuities
    # FIXME NEED TO HANDLE DISCONTINUITY
    # mask = ~self.transform.discontinuity()[:-1,:-1]
    # data = diffusion_fill(data, mask, self.niter)

    # Get and apply the mask
    mask = self.experiment.imageset.get_mask(0)[0]
    mask = mask.as_1d().as_int().as_double()
    mask.reshape(data.accessor())
    data *= mask

    # Return the result
    return data
  def _get_gradients_core(self, reflections, D, s0, U, B, axis, fixed_rotation, callback=None):
    """Calculate gradients of the prediction formula with respect to
    each of the parameters of the contained models, for reflection h
    that reflects at rotation angle phi with scattering vector s that
    intersects panel panel_id. That is, calculate dX/dp, dY/dp and
    dphi/dp"""

    # Spindle rotation matrices for every reflection
    #R = self._axis.axis_and_angle_as_r3_rotation_matrix(phi)
    #R = flex.mat3_double(len(reflections))
    # NB for now use flex.vec3_double.rotate_around_origin each time I need the
    # rotation matrix R.

    self._axis = axis
    self._fixed_rotation = fixed_rotation
    self._s0 = s0

    # pv is the 'projection vector' for the ray along s1.
    self._D = D
    self._s1 = reflections['s1']
    self._pv = D * self._s1

    # also need quantities derived from pv, precalculated for efficiency
    u, v, w = self._pv.parts()
    self._w_inv = 1/w
    self._u_w_inv = u * self._w_inv
    self._v_w_inv = v * self._w_inv

    self._UB = U * B
    self._U = U
    self._B = B

    # r is the reciprocal lattice vector, in the lab frame
    self._h = reflections['miller_index'].as_vec3_double()
    self._phi_calc = reflections['xyzcal.mm'].parts()[2]
    self._r = (self._fixed_rotation * (self._UB * self._h)).rotate_around_origin(self._axis, self._phi_calc)

    # All of the derivatives of phi have a common denominator, given by
    # (e X r).s0, where e is the rotation axis. Calculate this once, here.
    self._e_X_r = self._axis.cross(self._r)
    self._e_r_s0 = (self._e_X_r).dot(self._s0)

    # Note that e_r_s0 -> 0 when the rotation axis, beam vector and
    # relp are coplanar. This occurs when a reflection just touches
    # the Ewald sphere.
    #
    # There is a relationship between e_r_s0 and zeta_factor.
    # Uncommenting the code below shows that
    # s0.(e X r) = zeta * |s X s0|

    #from dials.algorithms.profile_model.gaussian_rs import zeta_factor
    #from libtbx.test_utils import approx_equal
    #s = matrix.col(reflections['s1'][0])
    #z = zeta_factor(axis[0], s0[0], s)
    #ss0 = (s.cross(matrix.col(s0[0]))).length()
    #assert approx_equal(e_r_s0[0], z * ss0)

    # catch small values of e_r_s0
    e_r_s0_mag = flex.abs(self._e_r_s0)
    try:
      assert flex.min(e_r_s0_mag) > 1.e-6
    except AssertionError as e:
      imin = flex.min_index(e_r_s0_mag)
      print "(e X r).s0 too small:"
      print "for", (e_r_s0_mag <= 1.e-6).count(True), "reflections"
      print "out of", len(e_r_s0_mag), "total"
      print "such as", reflections['miller_index'][imin]
      print "with scattering vector", reflections['s1'][imin]
      print "where r =", self._r[imin]
      print "e =", self._axis[imin]
      print "s0 =", self._s0[imin]
      print ("this reflection forms angle with the equatorial plane "
             "normal:")
      vecn = matrix.col(self._s0[imin]).cross(matrix.col(self._axis[imin])).normalize()
      print matrix.col(reflections['s1'][imin]).accute_angle(vecn)
      raise e

    # Set up empty list in which to store gradients
    m = len(reflections)
    results = []

    # determine experiment to indices mappings once, here
    experiment_to_idx = []
    for iexp, exp in enumerate(self._experiments):

      sel = reflections['id'] == iexp
      isel = sel.iselection()
      experiment_to_idx.append(isel)

    # reset a pointer to the parameter number
    self._iparam = 0

    ### Work through the parameterisations, calculating their contributions
    ### to derivatives d[pv]/dp and d[phi]/dp

    # loop over the detector parameterisations
    for dp in self._detector_parameterisations:

      # Determine (sub)set of reflections affected by this parameterisation
      isel = flex.size_t()
      for exp_id in dp.get_experiment_ids():
        isel.extend(experiment_to_idx[exp_id])

      # Access the detector model being parameterised
      detector = dp.get_model()

      # Get panel numbers of the affected reflections
      panel = reflections['panel'].select(isel)

      # Extend derivative vectors for this detector parameterisation
      results = self._extend_gradient_vectors(results, m, dp.num_free(),
        keys=self._grad_names)

      # loop through the panels in this detector
      for panel_id, _ in enumerate(exp.detector):

        # get the right subset of array indices to set for this panel
        sub_isel = isel.select(panel == panel_id)
        if len(sub_isel) == 0:
          # if no reflections intersect this panel, skip calculation
          continue
        sub_pv = self._pv.select(sub_isel)
        sub_D = self._D.select(sub_isel)
        dpv_ddet_p = self._detector_derivatives(dp, sub_pv, sub_D, panel_id)

        # convert to dX/dp, dY/dp and assign the elements of the vectors
        # corresponding to this experiment and panel
        sub_w_inv = self._w_inv.select(sub_isel)
        sub_u_w_inv = self._u_w_inv.select(sub_isel)
        sub_v_w_inv = self._v_w_inv.select(sub_isel)
        dX_ddet_p, dY_ddet_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp(
          sub_w_inv, sub_u_w_inv, sub_v_w_inv, dpv_ddet_p)

        # use a local parameter index pointer because we set all derivatives
        # for this panel before moving on to the next
        iparam = self._iparam
        for dX, dY in zip(dX_ddet_p, dY_ddet_p):
          if dX is not None:
            results[iparam]['dX_dp'].set_selected(sub_isel, dX)
          if dY is not None:
            results[iparam]['dY_dp'].set_selected(sub_isel, dY)
          # increment the local parameter index pointer
          iparam += 1

      if callback is not None:
        iparam = self._iparam
        for i in range(dp.num_free()):
          results[iparam] = callback(results[iparam])
          iparam += 1

      # increment the parameter index pointer to the last detector parameter
      self._iparam += dp.num_free()

    # loop over the beam parameterisations
    for bp in self._beam_parameterisations:

      # Determine (sub)set of reflections affected by this parameterisation
      isel = flex.size_t()
      for exp_id in bp.get_experiment_ids():
        isel.extend(experiment_to_idx[exp_id])

      # Extend derivative vectors for this beam parameterisation
      results = self._extend_gradient_vectors(results, m, bp.num_free(),
        keys=self._grad_names)

      if len(isel) == 0:
        # if no reflections are in this experiment, skip calculation
        self._iparam += bp.num_free()
        continue

      # Get required data from those reflections
      r = self._r.select(isel)
      e_X_r = self._e_X_r.select(isel)
      e_r_s0 = self._e_r_s0.select(isel)
      D = self._D.select(isel)

      w_inv = self._w_inv.select(isel)
      u_w_inv = self._u_w_inv.select(isel)
      v_w_inv = self._v_w_inv.select(isel)

      dpv_dbeam_p, dphi_dbeam_p = self._beam_derivatives(bp, r, e_X_r, e_r_s0, D)

      # convert to dX/dp, dY/dp and assign the elements of the vectors
      # corresponding to this experiment
      dX_dbeam_p, dY_dbeam_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp(
        w_inv, u_w_inv, v_w_inv, dpv_dbeam_p)
      for dX, dY, dphi in zip(dX_dbeam_p, dY_dbeam_p, dphi_dbeam_p):
        results[self._iparam][self._grad_names[0]].set_selected(isel, dX)
        results[self._iparam][self._grad_names[1]].set_selected(isel, dY)
        results[self._iparam][self._grad_names[2]].set_selected(isel, dphi)

        if callback is not None:
          results[self._iparam] = callback(results[self._iparam])
        # increment the parameter index pointer
        self._iparam += 1

    # loop over the crystal orientation parameterisations
    for xlop in self._xl_orientation_parameterisations:

      # Determine (sub)set of reflections affected by this parameterisation
      isel = flex.size_t()
      for exp_id in xlop.get_experiment_ids():
        isel.extend(experiment_to_idx[exp_id])

      # Extend derivative vectors for this crystal orientation parameterisation
      results = self._extend_gradient_vectors(results, m, xlop.num_free(),
        keys=self._grad_names)

      if len(isel) == 0:
        # if no reflections are in this experiment, skip calculation
        self._iparam += xlop.num_free()
        continue

      # Get required data from those reflections
      axis = self._axis.select(isel)
      fixed_rotation = self._fixed_rotation.select(isel)
      phi_calc = self._phi_calc.select(isel)
      h = self._h.select(isel)
      s1 = self._s1.select(isel)
      e_X_r = self._e_X_r.select(isel)
      e_r_s0 = self._e_r_s0.select(isel)
      B = self._B.select(isel)
      D = self._D.select(isel)

      w_inv = self._w_inv.select(isel)
      u_w_inv = self._u_w_inv.select(isel)
      v_w_inv = self._v_w_inv.select(isel)

      # get derivatives of the U matrix wrt the parameters
      dU_dxlo_p = [reflections["dU_dp{0}".format(i)].select(isel) \
                   for i in range(xlop.num_free())]
      dpv_dxlo_p, dphi_dxlo_p = self._xl_orientation_derivatives(
        dU_dxlo_p, axis, fixed_rotation, phi_calc, h, s1, e_X_r, e_r_s0, B, D)

      # convert to dX/dp, dY/dp and assign the elements of the vectors
      # corresponding to this experiment
      dX_dxlo_p, dY_dxlo_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp(
        w_inv, u_w_inv, v_w_inv, dpv_dxlo_p)
      for dX, dY, dphi in zip(dX_dxlo_p, dY_dxlo_p, dphi_dxlo_p):
        results[self._iparam][self._grad_names[0]].set_selected(isel, dX)
        results[self._iparam][self._grad_names[1]].set_selected(isel, dY)
        results[self._iparam][self._grad_names[2]].set_selected(isel, dphi)
        if callback is not None:
          results[self._iparam] = callback(results[self._iparam])
        # increment the parameter index pointer
        self._iparam += 1

    # loop over the crystal unit cell parameterisations
    for xlucp in self._xl_unit_cell_parameterisations:

      # Determine (sub)set of reflections affected by this parameterisation
      isel = flex.size_t()
      for exp_id in xlucp.get_experiment_ids():
        isel.extend(experiment_to_idx[exp_id])

      # Extend derivative vectors for this crystal unit cell parameterisation
      results = self._extend_gradient_vectors(results, m, xlucp.num_free(),
        keys=self._grad_names)

      if len(isel) == 0:
        # if no reflections are in this experiment, skip calculation
        self._iparam += xlucp.num_free()
        continue

      # Get required data from those reflections
      axis = self._axis.select(isel)
      fixed_rotation = self._fixed_rotation.select(isel)
      phi_calc = self._phi_calc.select(isel)
      h = self._h.select(isel)
      s1 = self._s1.select(isel)
      e_X_r = self._e_X_r.select(isel)
      e_r_s0 = self._e_r_s0.select(isel)
      U = self._U.select(isel)
      D = self._D.select(isel)

      w_inv = self._w_inv.select(isel)
      u_w_inv = self._u_w_inv.select(isel)
      v_w_inv = self._v_w_inv.select(isel)

      dB_dxluc_p = [reflections["dB_dp{0}".format(i)].select(isel) \
                   for i in range(xlucp.num_free())]
      dpv_dxluc_p, dphi_dxluc_p =  self._xl_unit_cell_derivatives(
        dB_dxluc_p, axis, fixed_rotation, phi_calc, h, s1, e_X_r, e_r_s0, U, D)

      # convert to dX/dp, dY/dp and assign the elements of the vectors
      # corresponding to this experiment
      dX_dxluc_p, dY_dxluc_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp(
        w_inv, u_w_inv, v_w_inv, dpv_dxluc_p)
      for dX, dY, dphi in zip(dX_dxluc_p, dY_dxluc_p, dphi_dxluc_p):
        results[self._iparam][self._grad_names[0]].set_selected(isel, dX)
        results[self._iparam][self._grad_names[1]].set_selected(isel, dY)
        results[self._iparam][self._grad_names[2]].set_selected(isel, dphi)
        if callback is not None:
          results[self._iparam] = callback(results[self._iparam])
        # increment the parameter index pointer
        self._iparam += 1

    return results
Exemplo n.º 26
0
  integrated_data['hash'] = hash_value

  return integrated_data

if __name__ == '__main__':
  import sys
  if len(sys.argv) != 2:
    raise RuntimeError, '%s strong.pickle' % sys.argv[0]

  import cPickle as pickle
  from dials.array_family import flex

  integrated_data = pickle.load(open(sys.argv[1], 'rb'))

  # keep only flagged as integrated reflections
  sel = integrated_data.get_flags(integrated_data.flags.integrated)
  integrated_data = integrated_data.select(sel)

  # only fully recorded reflections
  sel = integrated_data['partiality'] > 0.99
  integrated_data = integrated_data.select(sel)

  integrated_data = add_hash(integrated_data)
  hash_value = integrated_data['hash']

  for h in hash_value:
    sel = hash_value == h
    assert(sel.count(True) == 1)

  print flex.min(hash_value), flex.max(hash_value)
Exemplo n.º 27
0
def paper_test(B, S):

  from numpy.random import poisson
  from math import exp

  background_shape = [1 for i in range(20)]
  signal_shape = [1 if i >= 6 and i < 15 else 0 for i in range(20)]

  background = [poisson(bb * B,1)[0] for bb in background_shape]
  signal = [poisson(ss * S, 1)[0] for ss in signal_shape]
  # background = [bb * B for bb in background_shape]
  # signal = [ss * S for ss in signal_shape]
  total = [b + s for b, s in zip(background, signal)]

  # from matplotlib import pylab
  # pylab.plot(total)
  # pylab.plot(signal)
  # pylab.plot(background)
  # pylab.show()

  total = [0, 1, 0, 0, 0, 0, 3, 1, 3, 3, 6, 6, 4, 1, 4, 0, 2, 0, 1, 1]
  total = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]

  # plot_prob_for_zero(total, background_shape, signal_shape)
  # signal_shape = [exp(-(x - 10.0)**2 / (2*3.0**2)) for x in range(20)]
  # signal_shape = [ss / sum(signal_shape) for ss in signal_shape]
  # print signal_shape
  #plot_valid(background_shape, signal_shape)

  B = 155.0 / 296.0
  from dials.array_family import flex
  from math import log, factorial
  V = flex.double(flex.grid(100, 100))
  L = flex.double(flex.grid(100, 100))
  DB = flex.double(flex.grid(100, 100))
  DS = flex.double(flex.grid(100, 100))
  P = flex.double(flex.grid(100, 100))
  Fb = sum(background_shape)
  Fs = sum(signal_shape)
  SV = []
  MASK = flex.bool(flex.grid(100, 100), False)
  for BB in range(100):
    for SS in range(100):
      B = -5.0 + (BB) / 10.0
      S = -5.0 + (SS) / 10.0
      # SV.append(S)
      VV = 0
      LL = 0
      DDB = 0
      DDS = 0
      for i in range(20):
        s = signal_shape[i]
        b = background_shape[i]
        c = total[i]
        if B*b + S*s <= 0:
          # MASK[BB, SS] = True
          LL = 0
          if b == 0:
            DDB += 0
          else:
            DDB += 1e7
          if s == 0:
            DDS += 0
          else:
            DDS += 1e7
          # break
        else:
        # VV += (b + s)*c / (B*b + S*s)
        # LL += c*log(B*b+S*s) - log(factorial(c)) - B*b - S*s
          DDB += c*b/(B*b+S*s) - b
          DDS += c*s/(B*b+S*s) - s
      VV -= (Fb + Fs)
      # print B, S, VV
      # V[BB, SS] = abs(VV)
      L[BB, SS] = LL
      DB[BB,SS] = DDB
      DS[BB,SS] = DDS

  max_ind = flex.max_index(L)
  j = max_ind // 100
  i = max_ind % 100
  print "Approx: ", (j+1) / 20.0, (i+1) / 20.0
  print "Min/Max DB: ", flex.min(DB), flex.max(DB)
  print "Min/Max DS: ", flex.min(DS), flex.max(DS)
  from matplotlib import pylab
  # pylab.imshow(flex.log(V).as_numpy_array(), extent=[0.05, 5.05, 5.05, 0.05])
  # pylab.plot(SV, V)
  # pylab.plot(SV, [0] * 100)
  # pylab.show()
  im = flex.exp(L).as_numpy_array()
  import numpy
  # im = numpy.ma.masked_array(im, mask=MASK.as_numpy_array())
  # pylab.imshow(im)#, extent=[-5.0, 5.0, 5.0, -5.0], origin='lower')
  pylab.imshow(DB.as_numpy_array(), vmin=-100, vmax=100)#, extent=[-5.0, 5.0, 5.0, -5.0], origin='lower')
  pylab.contour(DB.as_numpy_array(), levels=[0], colors=['red'])
  pylab.contour(DS.as_numpy_array(), levels=[0], colors=['black'])
  pylab.show()
  # im = numpy.ma.masked_array(DB.as_numpy_array(), mask=MASK.as_numpy_array())
  # pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100)
  # pylab.show()
  # im = numpy.ma.masked_array(DS.as_numpy_array(), mask=MASK.as_numpy_array())
  # pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100)
  # pylab.show()
  # exit(0)

  S1, B1 = value(total, background_shape, signal_shape)
  exit(0)
  try:
    S1, B1 = value(total, background_shape, signal_shape)
    print "Result:"
    print S1, B1
    exit(0)
  except Exception, e:
    raise e
    import sys
    import traceback
    traceback.print_exc()
    from dials.array_family import flex
    Fs = sum(signal_shape)
    Fb = sum(background_shape)
    Rs = flex.double(flex.grid(100, 100))
    print "-----"
    print B, S
    # from matplotlib import pylab
    # pylab.plot(total)
    # pylab.show()
    print background_shape
    print signal_shape
    print total
    from math import exp, factorial, log
    minx = -1
    miny = -1
    minr = 9999
    for BB in range(0, 100):
      for SS in range(0, 100):
        B = -10 + (BB) / 5.0
        S = -10 + (SS) / 5.0
        L = 0
        Fb2 = 0
        Fs2 = 0
        for i in range(len(total)):
          c = total[i]
          b = background_shape[i]
          s = signal_shape[i]
          # P = exp(-(B*b + S*s)) * (B*b+S*s)**c / factorial(c)
          # print P
          # if P > 0:
          #   L += log(P)
          den = B*b + S*s
          num1 = b*c
          num2 = s*c
          if den != 0:
            Fb2 += num1 / den
            Fs2 += num2 / den
        R = (Fb2 - Fb)**2 + (Fs2 - Fs)**2
        if R > 1000:
          R = 0
        # Rs[BB,SS] = L#R#Fs2 - Fs
        Rs[BB,SS] = R#Fs2 - Fs
    from matplotlib import pylab
    pylab.imshow(flex.log(Rs).as_numpy_array(), extent=[-5,5,5,-5])
    pylab.show()
    exit(0)
Exemplo n.º 28
0
def blank_integrated_analysis(reflections, scan, phi_step, fractional_loss):
    prf_sel = reflections.get_flags(reflections.flags.integrated_prf)
    if prf_sel.count(True) > 0:
        reflections = reflections.select(prf_sel)
        intensities = reflections["intensity.prf.value"]
        variances = reflections["intensity.prf.variance"]
    else:
        sum_sel = reflections.get_flags(reflections.flags.integrated_sum)
        reflections = reflections.select(sum_sel)
        intensities = reflections["intensity.sum.value"]
        variances = reflections["intensity.sum.variance"]

    i_sigi = intensities / flex.sqrt(variances)

    xyz_px = reflections["xyzobs.px.value"]
    x_px, y_px, z_px = xyz_px.parts()
    phi = scan.get_angle_from_array_index(z_px)

    osc = scan.get_oscillation()[1]
    n_images_per_step = iceil(phi_step / osc)
    phi_step = n_images_per_step * osc

    array_range = scan.get_array_range()
    phi_min = flex.min(phi)
    phi_max = flex.max(phi)
    n_steps = int(round((phi_max - phi_min) / phi_step))
    hist = flex.histogram(z_px,
                          data_min=array_range[0],
                          data_max=array_range[1],
                          n_slots=n_steps)
    logger.debug("Histogram:")
    logger.debug(hist.as_str())

    mean_i_sigi = flex.double()
    for i, slot_info in enumerate(hist.slot_infos()):
        sel = (z_px >= slot_info.low_cutoff) & (z_px < slot_info.high_cutoff)
        if sel.count(True) == 0:
            mean_i_sigi.append(0)
        else:
            mean_i_sigi.append(flex.mean(i_sigi.select(sel)))

    potential_blank_sel = mean_i_sigi <= (fractional_loss *
                                          flex.max(mean_i_sigi))

    xmin, xmax = zip(*[(slot_info.low_cutoff, slot_info.high_cutoff)
                       for slot_info in hist.slot_infos()])

    d = {
        "data": [{
            "x": list(hist.slot_centers()),
            "y": list(mean_i_sigi),
            "xlow": xmin,
            "xhigh": xmax,
            "blank": list(potential_blank_sel),
            "type": "bar",
            "name": "blank_counts_analysis",
        }],
        "layout": {
            "xaxis": {
                "title": "z observed (images)"
            },
            "yaxis": {
                "title": "Number of reflections"
            },
            "bargap": 0,
        },
    }

    blank_regions = blank_regions_from_sel(d["data"][0])
    d["blank_regions"] = blank_regions

    return d
  def compose(self, reflections, skip_derivatives=False):
    """Compose scan-varying crystal parameterisations at the specified image
    number, for the specified experiment, for each image. Put the U, B and
    UB matrices in the reflection table, and cache the derivatives."""

    self._prepare_for_compose(reflections, skip_derivatives)

    for iexp, exp in enumerate(self._experiments):

      # select the reflections of interest
      sel = reflections['id'] == iexp
      isel = sel.iselection()

      blocks = reflections['block'].select(isel)

      # identify which parameterisations to use for this experiment
      xl_op = self._get_xl_orientation_parameterisation(iexp)
      xl_ucp = self._get_xl_unit_cell_parameterisation(iexp)
      bp = self._get_beam_parameterisation(iexp)
      dp = self._get_detector_parameterisation(iexp)

      # reset current frame cache for scan-varying parameterisations
      self._current_frame = {}

      # get state and derivatives for each block
      for block in xrange(flex.min(blocks),
                          flex.max(blocks) + 1):

        # determine the subset of reflections this affects
        subsel = isel.select(blocks == block)
        if len(subsel) == 0: continue

        # get the panels hit by these reflections
        panels = reflections['panel'].select(subsel)

        # get the integer frame number nearest the centre of that block
        frames = reflections['block_centre'].select(subsel)

        # can only be false if original block assignment has gone wrong
        assert frames.all_eq(frames[0]), \
            "Failing: a block contains reflections that shouldn't be there"
        frame = int(floor(frames[0]))

        # model states at current frame
        U = self._get_state_from_parameterisation(xl_op, frame)
        if U is None: U = exp.crystal.get_U()

        B = self._get_state_from_parameterisation(xl_ucp, frame)
        if B is None: B = exp.crystal.get_B()

        s0 = self._get_state_from_parameterisation(bp, frame)
        if s0 is None: s0 = exp.beam.get_s0()

        # set states for crystal and beam
        reflections['u_matrix'].set_selected(subsel, U.elems)
        reflections['b_matrix'].set_selected(subsel, B.elems)
        reflections['s0_vector'].set_selected(subsel, s0.elems)

        # set states and derivatives for multi-panel detector
        if dp is not None and dp.is_multi_state():

          # loop through the panels in this detector
          for panel_id, _ in enumerate(exp.detector):

            # get the right subset of array indices to set for this panel
            subsel2 = subsel.select(panels == panel_id)
            if len(subsel2) == 0:
              # if no reflections intersect this panel, skip calculation
              continue

            dmat  = self._get_state_from_parameterisation(dp,
              frame, multi_state_elt=panel_id)
            if dmat is None: dmat = exp.detector[panel_id].get_d_matrix()
            Dmat = exp.detector[panel_id].get_D_matrix()
            reflections['d_matrix'].set_selected(subsel2, dmat)
            reflections['D_matrix'].set_selected(subsel2, Dmat)

            if dp is not None and self._varying_detectors and not skip_derivatives:
              for j, dd in enumerate(dp.get_ds_dp(multi_state_elt=panel_id,
                                                  use_none_as_null=True)):
                if dd is None: continue
                colname = "dd_dp{0}".format(j)
                reflections[colname].set_selected(subsel, dd)

        else: # set states and derivatives for single panel detector

          dmat  = self._get_state_from_parameterisation(dp, frame)
          if dmat is None: dmat = exp.detector[0].get_d_matrix()
          Dmat = exp.detector[0].get_D_matrix()
          reflections['d_matrix'].set_selected(subsel, dmat)
          reflections['D_matrix'].set_selected(subsel, Dmat)

          if dp is not None and self._varying_detectors and not skip_derivatives:
            for j, dd in enumerate(dp.get_ds_dp(use_none_as_null=True)):
              if dd is None: continue
              colname = "dd_dp{0}".format(j)
              reflections[colname].set_selected(subsel, dd)

        # set derivatives of the states for crystal and beam
        if not skip_derivatives:
          if xl_op is not None and self._varying_xl_orientations:
            for j, dU in enumerate(xl_op.get_ds_dp(use_none_as_null=True)):
              if dU is None: continue
              colname = "dU_dp{0}".format(j)
              reflections[colname].set_selected(subsel, dU)
          if xl_ucp is not None and self._varying_xl_unit_cells:
            for j, dB in enumerate(xl_ucp.get_ds_dp(use_none_as_null=True)):
              if dB is None: continue
              colname = "dB_dp{0}".format(j)
              reflections[colname].set_selected(subsel, dB)
          if bp is not None and self._varying_beams:
            for j, ds0 in enumerate(bp.get_ds_dp(use_none_as_null=True)):
              if ds0 is None: continue
              colname = "ds0_dp{0}".format(j)
              reflections[colname].set_selected(subsel, ds0)

    # set the UB matrices for prediction
    reflections['ub_matrix'] = reflections['u_matrix'] * reflections['b_matrix']

    return
Exemplo n.º 30
0
  def centroid_mean_diff_vs_phi(self, rlist, threshold):
    from os.path import join
    import math
    I = rlist['intensity.sum.value']
    I_sig = flex.sqrt(rlist['intensity.sum.variance'])
    I_over_S = I / I_sig
    mask = I_over_S > threshold
    rlist = rlist.select(mask)
    assert(len(rlist) > 0)

    xc, yc, zc = rlist['xyzcal.mm'].parts()
    xo, yo, zo = rlist['xyzobs.mm.value'].parts()

    dx = xc - xo
    dy = yc - yo
    dphi = (zc - zo) * RAD2DEG

    mean_residuals_x = flex.double()
    mean_residuals_y = flex.double()
    mean_residuals_phi = flex.double()
    rmsd_x = flex.double()
    rmsd_y = flex.double()
    rmsd_phi = flex.double()
    frame = []
    phi_obs_deg = RAD2DEG * zo
    phi = []

    for i_phi in range(int(math.floor(flex.min(phi_obs_deg))),
                   int(math.ceil(flex.max(phi_obs_deg)))):
      sel = (phi_obs_deg >= i_phi) & (phi_obs_deg < (i_phi+1))
      if sel.count(True) == 0:
        continue
      mean_residuals_x.append(flex.mean(dx.select(sel)))
      mean_residuals_y.append(flex.mean(dy.select(sel)))
      mean_residuals_phi.append(flex.mean(dphi.select(sel)))
      rmsd_x.append(math.sqrt(flex.mean_sq(dx.select(sel))))
      rmsd_y.append(math.sqrt(flex.mean_sq(dy.select(sel))))
      rmsd_phi.append(math.sqrt(flex.mean_sq(dphi.select(sel))))
      phi.append(i_phi)

    from matplotlib import pyplot
    fig = pyplot.figure()
    ax = fig.add_subplot(311)
    #fig.subplots_adjust(hspace=0.5)
    pyplot.axhline(0, color='grey')
    ax.scatter(phi, mean_residuals_x)
    ax.set_xlabel('phi (deg)')
    ax.set_ylabel('mean $\Delta$ x (mm)')
    ax = fig.add_subplot(312)
    pyplot.axhline(0, color='grey')
    ax.scatter(phi, mean_residuals_y)
    ax.set_xlabel('phi (deg)')
    ax.set_ylabel('mean $\Delta$ y (mm)')
    ax = fig.add_subplot(313)
    pyplot.axhline(0, color='grey')
    ax.scatter(phi, mean_residuals_phi)
    ax.set_xlabel('phi (deg)')
    ax.set_ylabel('mean $\Delta$ phi (deg)')
    pyplot.savefig(join(self.directory, "centroid_mean_diff_vs_phi.png"))
    pyplot.close()

    fig = pyplot.figure()
    ax = fig.add_subplot(311)
    #fig.subplots_adjust(hspace=0.5)
    pyplot.axhline(flex.mean(rmsd_x), color='grey')
    ax.scatter(phi, rmsd_x)
    ax.set_xlabel('phi (deg)')
    ax.set_ylabel('rmsd x (mm)')
    ax = fig.add_subplot(312)
    pyplot.axhline(flex.mean(rmsd_y), color='grey')
    ax.scatter(phi, rmsd_y)
    ax.set_xlabel('phi (deg)')
    ax.set_ylabel('rmsd y (mm)')
    ax = fig.add_subplot(313)
    pyplot.axhline(flex.mean(rmsd_phi), color='grey')
    ax.scatter(phi, rmsd_phi)
    ax.set_xlabel('phi (deg)')
    ax.set_ylabel('rmsd phi (deg)')
    pyplot.savefig(join(self.directory, "centroid_rmsd_vs_phi.png"))
    pyplot.close()
Exemplo n.º 31
0
exit(0)

from dials.array_family import flex
im1 = flex.double(flex.grid(100, 100))
im2 = flex.double(flex.grid(100, 100))

for j in range(100):
  for i in range(100):
    t1 = -90.0 + i
    t2 = -90.0 + j
    d1 = dt1(c, b, s, t1, t2, mu0, mu1, mu2)
    d2 = dt2(c, b, s, t1, t2, mu0, mu1, mu2)
    im1[j,i] = d1
    im2[j,i] = d2

print flex.min(im1), flex.max(im1)
print flex.min(im2), flex.max(im2)

from matplotlib import pylab
pylab.imshow(im1.as_numpy_array())
pylab.contour(im1.as_numpy_array(), levels=[0])
pylab.contour(im2.as_numpy_array(), levels=[0])
pylab.show()

# print mu0, mu1, mu2

# b = [bb / sum(b) for bb in b]
# s = [ss / sum(s) for ss in s]

# K1 = 0
# K2 = 9.0 / 20.0
Exemplo n.º 32
0
  def __init__(self, reflections, av_callback=flex.mean):

    # flags to indicate at what level the analysis has been performed
    self._average_residuals = False
    self._spectral_analysis = False

    self._av_callback = av_callback

    # Remove invalid reflections
    reflections = reflections.select(~(reflections['miller_index'] == (0,0,0)))

    # FIXME - better way to recognise non-predictions. Can't rely on flags
    # in e.g. indexed.pickle I think.
    x, y, z = reflections['xyzcal.mm'].parts()
    sel = (x == 0) & (y == 0)
    reflections = reflections.select(~sel)
    self._nexp = flex.max(reflections['id']) + 1

    # Ensure required keys are present
    if not all([k in reflections for k in ['x_resid', 'y_resid', 'phi_resid']]):
      x_obs, y_obs, phi_obs = reflections['xyzobs.mm.value'].parts()
      x_cal, y_cal, phi_cal = reflections['xyzcal.mm'].parts()

      # do not wrap around multiples of 2*pi; keep the full rotation
      # from zero to differentiate repeat observations.
      from math import pi
      TWO_PI = 2.0 * pi
      resid = phi_cal - (flex.fmod_positive(phi_obs, TWO_PI))
      # ensure this is the smaller of two possibilities
      resid = flex.fmod_positive((resid + pi), TWO_PI) - pi
      phi_cal = phi_obs + resid
      reflections['x_resid'] = x_cal - x_obs
      reflections['y_resid'] = y_cal - y_obs
      reflections['phi_resid'] = phi_cal - phi_obs

    # create empty results list
    self._results = []

    # first, just determine a suitable block size for analysis
    for iexp in range(self._nexp):
      ref_this_exp = reflections.select(reflections['id'] == iexp)
      if len(ref_this_exp) == 0:
        # can't do anything, just keep an empty dictionary
        self._results.append({})
        continue
      phi_obs_deg = ref_this_exp['xyzobs.mm.value'].parts()[2] * RAD2DEG
      phi_range = flex.min(phi_obs_deg), flex.max(phi_obs_deg)
      phi_width = phi_range[1] - phi_range[0]
      ideal_block_size = 1.0
      old_nblocks = 0
      while True:
        nblocks = int(phi_width // ideal_block_size)
        if nblocks == old_nblocks: nblocks -= 1
        nblocks = max(nblocks, 1)
        block_size = phi_width / nblocks
        nr = flex.int()
        for i in range(nblocks - 1):
          blk_start = phi_range[0] + i * block_size
          blk_end = blk_start + block_size
          sel = (phi_obs_deg >= blk_start) & (phi_obs_deg < blk_end)
          nref_in_block = sel.count(True)
          nr.append(nref_in_block)
        # include max phi in the final block
        blk_start = phi_range[0] + (nblocks - 1) * block_size
        blk_end = phi_range[1]
        sel = (phi_obs_deg >= blk_start) & (phi_obs_deg <= blk_end)
        nref_in_block = sel.count(True)
        nr.append(nref_in_block)
        # Break if there are enough reflections, otherwise increase block size,
        # unless only one block remains
        if nblocks == 1: break
        min_nr = flex.min(nr)
        if min_nr >= 50: break
        if min_nr < 5:
          fac = 2
        else:
          fac = 50 / min_nr
        ideal_block_size *= fac
        old_nblocks = nblocks

      # collect the basic data for this experiment
      self._results.append({'block_size':block_size,
                            'nref_per_block':nr,
                            'nblocks':nblocks,
                            'phi_range':phi_range})

    # keep reflections for analysis
    self._reflections = reflections
Exemplo n.º 33
0
    #from matplotlib import pylab
    #fig = pylab.figure(dpi=300)
    #pylab.imshow(scale_data.as_numpy_array(), vmin=0, vmax=2, interpolation='none')
    #pylab.colorbar()
    #pylab.savefig("scale_%d.png" % frame)
    #pylab.clf()
    ##pylab.show()
    #exit(0)

    #pylab.hist(scale_data.as_1d().select(scale_mask.as_1d()).as_numpy_array(),
    #           bins=100)
    #pylab.show()

    sd1 = scale_data.as_1d()
    sm1 = scale_mask.as_1d()
    scale_min = flex.min(sd1.select(sm1))
    scale_max = flex.max(sd1.select(sm1))
    scale_avr = flex.sum(sd1.select(sm1)) / sm1.count(True)

    background = model_data * scale_data

    reflections['shoebox'].select(indices).apply_pixel_data(
      data.as_double(),
      background,
      raw_mask,
      frame,
      1)

    subset = reflections.select(indices3)
    if len(subset) > 0:
      subset.compute_summed_intensity()
Exemplo n.º 34
0
def show_reflections(
    reflections,
    show_intensities=False,
    show_profile_fit=False,
    show_centroids=False,
    show_all_reflection_data=False,
    show_flags=False,
    max_reflections=None,
    show_identifiers=False,
):

    text = []

    from orderedset import OrderedSet

    formats = collections.OrderedDict((
        ("miller_index", "%i, %i, %i"),
        ("d", "%.2f"),
        ("qe", "%.3f"),
        ("dqe", "%.3f"),
        ("id", "%i"),
        ("imageset_id", "%i"),
        ("panel", "%i"),
        ("flags", "%i"),
        ("background.mean", "%.1f"),
        ("background.dispersion", "%.1f"),
        ("background.mse", "%.1f"),
        ("background.sum.value", "%.1f"),
        ("background.sum.variance", "%.1f"),
        ("intensity.prf.value", "%.1f"),
        ("intensity.prf.variance", "%.1f"),
        ("intensity.sum.value", "%.1f"),
        ("intensity.sum.variance", "%.1f"),
        ("intensity.cor.value", "%.1f"),
        ("intensity.cor.variance", "%.1f"),
        ("intensity.scale.value", "%.1f"),
        ("intensity.scale.variance", "%.1f"),
        ("Ih_values", "%.1f"),
        ("lp", "%.3f"),
        ("num_pixels.background", "%i"),
        ("num_pixels.background_used", "%i"),
        ("num_pixels.foreground", "%i"),
        ("num_pixels.valid", "%i"),
        ("partial_id", "%i"),
        ("partiality", "%.4f"),
        ("profile.correlation", "%.3f"),
        ("profile.rmsd", "%.3f"),
        ("xyzcal.mm", "%.2f, %.2f, %.2f"),
        ("xyzcal.px", "%.2f, %.2f, %.2f"),
        ("delpsical.rad", "%.3f"),
        ("delpsical2", "%.3f"),
        ("delpsical.weights", "%.3f"),
        ("xyzobs.mm.value", "%.2f, %.2f, %.2f"),
        ("xyzobs.mm.variance", "%.4e, %.4e, %.4e"),
        ("xyzobs.px.value", "%.2f, %.2f, %.2f"),
        ("xyzobs.px.variance", "%.4f, %.4f, %.4f"),
        ("s1", "%.4f, %.4f, %.4f"),
        ("s2", "%.4f, %.4f, %.4f"),
        ("shoebox", "%.1f"),
        ("rlp", "%.4f, %.4f, %.4f"),
        ("zeta", "%.3f"),
        ("x_resid", "%.3f"),
        ("x_resid2", "%.3f"),
        ("y_resid", "%.3f"),
        ("y_resid2", "%.3f"),
        ("kapton_absorption_correction", "%.3f"),
        ("kapton_absorption_correction_sigmas", "%.3f"),
        ("inverse_scale_factor", "%.3f"),
        ("inverse_scale_factor_variance", "%.3f"),
    ))

    for rlist in reflections:
        from dials.algorithms.shoebox import MaskCode

        foreground_valid = MaskCode.Valid | MaskCode.Foreground
        text.append("")
        text.append("Reflection list contains %i reflections" % (len(rlist)))

        if len(rlist) == 0:
            continue

        rows = [["Column", "min", "max", "mean"]]
        for k, col in rlist.cols():
            if k in formats and "%" not in formats.get(k, "%s"):
                # Allow blanking out of entries that wouldn't make sense
                rows.append([
                    k,
                    formats.get(k, "%s"),
                    formats.get(k, "%s"),
                    formats.get(k, "%s"),
                ])
            elif type(col) in (flex.double, flex.int, flex.size_t):
                if type(col) in (flex.int, flex.size_t):
                    col = col.as_double()
                rows.append([
                    k,
                    formats.get(k, "%s") % flex.min(col),
                    formats.get(k, "%s") % flex.max(col),
                    formats.get(k, "%s") % flex.mean(col),
                ])
            elif type(col) in (flex.vec3_double, flex.miller_index):
                if isinstance(col, flex.miller_index):
                    col = col.as_vec3_double()
                rows.append([
                    k,
                    formats.get(k, "%s") % col.min(),
                    formats.get(k, "%s") % col.max(),
                    formats.get(k, "%s") % col.mean(),
                ])
            elif isinstance(col, flex.shoebox):
                rows.append([k, "", "", ""])
                si = col.summed_intensity().observed_value()
                rows.append([
                    "  summed I",
                    formats.get(k, "%s") % flex.min(si),
                    formats.get(k, "%s") % flex.max(si),
                    formats.get(k, "%s") % flex.mean(si),
                ])
                x1, x2, y1, y2, z1, z2 = col.bounding_boxes().parts()
                bbox_sizes = ((z2 - z1) * (y2 - y1) * (x2 - x1)).as_double()
                rows.append([
                    "  N pix",
                    formats.get(k, "%s") % flex.min(bbox_sizes),
                    formats.get(k, "%s") % flex.max(bbox_sizes),
                    formats.get(k, "%s") % flex.mean(bbox_sizes),
                ])
                fore_valid = col.count_mask_values(
                    foreground_valid).as_double()
                rows.append([
                    "  N valid foreground pix",
                    formats.get(k, "%s") % flex.min(fore_valid),
                    formats.get(k, "%s") % flex.max(fore_valid),
                    formats.get(k, "%s") % flex.mean(fore_valid),
                ])

        text.append(tabulate(rows, headers="firstrow"))

        if show_flags:
            text.append(_create_flag_count_table(rlist))

        if show_identifiers:
            if rlist.experiment_identifiers():
                text.append(
                    """Experiment identifiers id-map values:\n%s""" %
                    ("\n".join(
                        "id:" + str(k) + " -> experiment identifier:" +
                        str(rlist.experiment_identifiers()[k])
                        for k in rlist.experiment_identifiers().keys())))

    intensity_keys = (
        "miller_index",
        "d",
        "intensity.prf.value",
        "intensity.prf.variance",
        "intensity.sum.value",
        "intensity.sum.variance",
        "background.mean",
        "profile.correlation",
        "profile.rmsd",
    )

    profile_fit_keys = ("miller_index", "d")

    centroid_keys = (
        "miller_index",
        "d",
        "xyzcal.mm",
        "xyzcal.px",
        "xyzobs.mm.value",
        "xyzobs.mm.variance",
        "xyzobs.px.value",
        "xyzobs.px.variance",
    )

    keys_to_print = OrderedSet()

    if show_intensities:
        for k in intensity_keys:
            keys_to_print.add(k)
    if show_profile_fit:
        for k in profile_fit_keys:
            keys_to_print.add(k)
    if show_centroids:
        for k in centroid_keys:
            keys_to_print.add(k)
    if show_all_reflection_data:
        for k in formats:
            keys_to_print.add(k)

    def format_column(key, data, format_strings=None):
        if isinstance(data, flex.vec3_double):
            c_strings = [
                c.as_string(format_strings[i].strip())
                for i, c in enumerate(data.parts())
            ]
        elif isinstance(data, flex.miller_index):
            c_strings = [
                c.as_string(format_strings[i].strip())
                for i, c in enumerate(data.as_vec3_double().parts())
            ]
        elif isinstance(data, flex.size_t):
            c_strings = [data.as_int().as_string(format_strings[0].strip())]
        elif isinstance(data, flex.shoebox):
            x1, x2, y1, y2, z1, z2 = data.bounding_boxes().parts()
            bbox_sizes = ((z2 - z1) * (y2 - y1) * (x2 - x1)).as_double()
            c_strings = [bbox_sizes.as_string(format_strings[0].strip())]
            key += " (N pix)"
        else:
            c_strings = [data.as_string(format_strings[0].strip())]

        column = flex.std_string()
        max_element_lengths = [c.max_element_length() for c in c_strings]
        for i in range(len(c_strings[0])):

            column.append(("%%%is" % len(key)) % ", ".join(
                ("%%%is" % max_element_lengths[j]) % c_strings[j][i]
                for j in range(len(c_strings))))
        return column

    if keys_to_print:
        keys = [k for k in keys_to_print if k in rlist]
        if max_reflections is not None:
            max_reflections = min(len(rlist), max_reflections)
        else:
            max_reflections = len(rlist)

        columns = []

        for k in keys:
            columns.append(
                format_column(k,
                              rlist[k],
                              format_strings=formats[k].split(",")))

        text.append("")
        text.append("Printing %i of %i reflections:" %
                    (max_reflections, len(rlist)))
        line = []
        for j in range(len(columns)):
            key = keys[j]
            if key == "shoebox":
                key += " (N pix)"
            width = max(len(key), columns[j].max_element_length())
            line.append("%%%is" % width % key)
        text.append(" ".join(line))
        for i in range(max_reflections):
            line = (c[i] for c in columns)
            text.append(" ".join(line))

    return "\n".join(text)