Esempio n. 1
0
  def __init__(self, strategies, n_bins=8, degrees_per_bin=5):
    from cctbx import crystal, miller
    import copy

    sg = strategies[0].experiment.crystal.get_space_group() \
      .build_derived_reflection_intensity_group(anomalous_flag=True)
    cs = crystal.symmetry(
      unit_cell=strategies[0].experiment.crystal.get_unit_cell(), space_group=sg)

    for i, strategy in enumerate(strategies):
      if i == 0:
        predicted = copy.deepcopy(strategy.predicted)
      else:
        predicted_ = copy.deepcopy(strategy.predicted)
        predicted_['dose'] += (flex.max(predicted['dose']) + 1)
        predicted.extend(predicted_)
    ms = miller.set(cs, indices=predicted['miller_index'], anomalous_flag=True)
    ma = miller.array(ms, data=flex.double(ms.size(),1),
                      sigmas=flex.double(ms.size(), 1))
    if 1:
      merging = ma.merge_equivalents()
      o = merging.array().customized_copy(
        data=merging.redundancies().data().as_double()).as_mtz_dataset('I').mtz_object()
      o.write('predicted.mtz')

    d_star_sq = ma.d_star_sq().data()

    binner = ma.setup_binner_d_star_sq_step(
      d_star_sq_step=(flex.max(d_star_sq)-flex.min(d_star_sq)+1e-8)/n_bins)

    dose = predicted['dose']
    range_width = 1
    range_min = flex.min(dose) - range_width
    range_max = flex.max(dose)
    n_steps = 2 + int((range_max - range_min) - range_width)

    binner_non_anom = ma.as_non_anomalous_array().use_binning(
      binner)
    self.n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])

    from xia2.Modules.PyChef2 import ChefStatistics
    chef_stats = ChefStatistics(
      ma.indices(), ma.data(), ma.sigmas(),
      ma.d_star_sq().data(), dose, self.n_complete, binner,
      ma.space_group(), ma.anomalous_flag(), n_steps)

    def fraction_new(completeness):
      # Completeness so far at end of image
      completeness_end = completeness[1:]
      # Completeness so far at start of image
      completeness_start = completeness[:-1]
      # Fraction of unique reflections observed for the first time on each image
      return completeness_end - completeness_start

    self.dose = dose
    self.ieither_completeness = chef_stats.ieither_completeness()
    self.iboth_completeness = chef_stats.iboth_completeness()
    self.frac_new_ref = fraction_new(self.ieither_completeness) / degrees_per_bin
    self.frac_new_pairs = fraction_new(self.iboth_completeness) / degrees_per_bin
Esempio n. 2
0
  def __init__(self, strategies, n_bins=8, degrees_per_bin=5):
    from cctbx import crystal, miller
    import copy

    sg = strategies[0].experiment.crystal.get_space_group() \
      .build_derived_reflection_intensity_group(anomalous_flag=True)
    cs = crystal.symmetry(
      unit_cell=strategies[0].experiment.crystal.get_unit_cell(), space_group=sg)

    for i, strategy in enumerate(strategies):
      if i == 0:
        predicted = copy.deepcopy(strategy.predicted)
      else:
        predicted_ = copy.deepcopy(strategy.predicted)
        predicted_['dose'] += (flex.max(predicted['dose']) + 1)
        predicted.extend(predicted_)
    ms = miller.set(cs, indices=predicted['miller_index'], anomalous_flag=True)
    ma = miller.array(ms, data=flex.double(ms.size(),1),
                      sigmas=flex.double(ms.size(), 1))
    if 1:
      merging = ma.merge_equivalents()
      o = merging.array().customized_copy(
        data=merging.redundancies().data().as_double()).as_mtz_dataset('I').mtz_object()
      o.write('predicted.mtz')

    d_star_sq = ma.d_star_sq().data()

    binner = ma.setup_binner_d_star_sq_step(
      d_star_sq_step=(flex.max(d_star_sq)-flex.min(d_star_sq)+1e-8)/n_bins)

    dose = predicted['dose']
    range_width = 1
    range_min = flex.min(dose) - range_width
    range_max = flex.max(dose)
    n_steps = 2 + int((range_max - range_min) - range_width)

    binner_non_anom = ma.as_non_anomalous_array().use_binning(
      binner)
    self.n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])

    from xia2.Modules.PyChef2 import ChefStatistics
    chef_stats = ChefStatistics(
      ma.indices(), ma.data(), ma.sigmas(),
      ma.d_star_sq().data(), dose, self.n_complete, binner,
      ma.space_group(), ma.anomalous_flag(), n_steps)

    def fraction_new(completeness):
      # Completeness so far at end of image
      completeness_end = completeness[1:]
      # Completeness so far at start of image
      completeness_start = completeness[:-1]
      # Fraction of unique reflections observed for the first time on each image
      return completeness_end - completeness_start

    self.dose = dose
    self.ieither_completeness = chef_stats.ieither_completeness()
    self.iboth_completeness = chef_stats.iboth_completeness()
    self.frac_new_ref = fraction_new(self.ieither_completeness) / degrees_per_bin
    self.frac_new_pairs = fraction_new(self.iboth_completeness) / degrees_per_bin
Esempio n. 3
0
  def __init__(self, intensities, dose, n_bins=8,
               range_min=None, range_max=None, range_width=1):

    if isinstance(dose, flex.double):
      sorted_dose = flex.sorted(dose)
      dd = sorted_dose[1:] - sorted_dose[:-1]
      dd = dd.select(dd > 0)
      if len(dd):
        step_size = flex.min(dd)
        dose /= step_size
      dose = dose.iround()
      if flex.min(dose) == 0:
        dose += 1

    # fix for completeness > 1 if screw axes present
    intensities = intensities.customized_copy(
      space_group_info=intensities.space_group().build_derived_reflection_intensity_group(
        anomalous_flag=intensities.anomalous_flag()).info(),
      info=intensities.info())

    self.intensities = intensities
    self.dose = dose
    self.n_bins = n_bins
    self.range_min = range_min
    self.range_max = range_max
    self.range_width = range_width
    assert self.range_width > 0

    if self.range_min is None:
      self.range_min = flex.min(self.dose) - self.range_width
    if self.range_max is None:
      self.range_max = flex.max(self.dose)
    self.n_steps = 2 + int((self.range_max - self.range_min) - self.range_width)

    sel = (self.dose.as_double() <= self.range_max) & (self.dose.as_double() >= self.range_min)
    self.dose = self.dose.select(sel)

    self.intensities = self.intensities.select(sel)
    self.d_star_sq = self.intensities.d_star_sq().data()

    self.binner = self.intensities.setup_binner_d_star_sq_step(
      d_star_sq_step=(flex.max(self.d_star_sq)-flex.min(self.d_star_sq)+1e-8)/self.n_bins)

    #self.dose /= range_width
    self.dose -= int(self.range_min)

    self.dose = flex.size_t(list(self.dose))

    binner_non_anom = intensities.as_non_anomalous_array().use_binning(
      self.binner)
    n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])

    from xia2.Modules.PyChef2 import ChefStatistics
    chef_stats = ChefStatistics(
      intensities.indices(), intensities.data(), intensities.sigmas(),
      intensities.d_star_sq().data(), self.dose, n_complete, self.binner,
      intensities.space_group(), intensities.anomalous_flag(), self.n_steps)

    self.iplus_comp_bins = chef_stats.iplus_completeness_bins()
    self.iminus_comp_bins = chef_stats.iminus_completeness_bins()
    self.ieither_comp_bins = chef_stats.ieither_completeness_bins()
    self.iboth_comp_bins = chef_stats.iboth_completeness_bins()
    self.iplus_comp_overall = chef_stats.iplus_completeness()
    self.iminus_comp_overall = chef_stats.iminus_completeness()
    self.ieither_comp_overall = chef_stats.ieither_completeness()
    self.iboth_comp_overall = chef_stats.iboth_completeness()
    self.rcp_bins = chef_stats.rcp_bins()
    self.rcp = chef_stats.rcp()
    self.scp_bins = chef_stats.scp_bins()
    self.scp = chef_stats.scp()
    self.rd = chef_stats.rd()
Esempio n. 4
0
File: PyChef.py Progetto: xia2/xia2
  def __init__(self, intensities, dose, n_bins=8,
               range_min=None, range_max=None, range_width=1):

    if isinstance(dose, flex.double):
      sorted_dose = flex.sorted(dose)
      dd = sorted_dose[1:] - sorted_dose[:-1]
      step_size = flex.min(dd.select(dd > 0))
      dose /= step_size
      dose = dose.iround()
      if flex.min(dose) == 0:
        dose += 1

    # fix for completeness > 1 if screw axes present
    intensities = intensities.customized_copy(
      space_group_info=intensities.space_group().build_derived_reflection_intensity_group(
        anomalous_flag=intensities.anomalous_flag()).info(),
      info=intensities.info())

    self.intensities = intensities
    self.dose = dose
    self.n_bins = n_bins
    self.range_min = range_min
    self.range_max = range_max
    self.range_width = range_width
    assert self.range_width > 0

    if self.range_min is None:
      self.range_min = flex.min(self.dose) - self.range_width
    if self.range_max is None:
      self.range_max = flex.max(self.dose)
    self.n_steps = 2 + int((self.range_max - self.range_min) - self.range_width)

    sel = (self.dose.as_double() <= self.range_max) & (self.dose.as_double() >= self.range_min)
    self.dose = self.dose.select(sel)

    self.intensities = self.intensities.select(sel)
    self.d_star_sq = self.intensities.d_star_sq().data()

    self.binner = self.intensities.setup_binner_d_star_sq_step(
      d_star_sq_step=(flex.max(self.d_star_sq)-flex.min(self.d_star_sq)+1e-8)/self.n_bins)

    #self.dose /= range_width
    self.dose -= int(self.range_min)

    self.dose = flex.size_t(list(self.dose))

    binner_non_anom = intensities.as_non_anomalous_array().use_binning(
      self.binner)
    n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])

    from xia2.Modules.PyChef2 import ChefStatistics
    chef_stats = ChefStatistics(
      intensities.indices(), intensities.data(), intensities.sigmas(),
      intensities.d_star_sq().data(), self.dose, n_complete, self.binner,
      intensities.space_group(), intensities.anomalous_flag(), self.n_steps)

    self.iplus_comp_bins = chef_stats.iplus_completeness_bins()
    self.iminus_comp_bins = chef_stats.iminus_completeness_bins()
    self.ieither_comp_bins = chef_stats.ieither_completeness_bins()
    self.iboth_comp_bins = chef_stats.iboth_completeness_bins()
    self.iplus_comp_overall = chef_stats.iplus_completeness()
    self.iminus_comp_overall = chef_stats.iminus_completeness()
    self.ieither_comp_overall = chef_stats.ieither_completeness()
    self.iboth_comp_overall = chef_stats.iboth_completeness()
    self.rcp_bins = chef_stats.rcp_bins()
    self.rcp = chef_stats.rcp()
    self.scp_bins = chef_stats.scp_bins()
    self.scp = chef_stats.scp()
    self.rd = chef_stats.rd()
Esempio n. 5
0
def run(args):

    from dials.util.options import OptionParser
    from dials.util.options import flatten_experiments
    from dials.util.options import flatten_reflections
    import libtbx.load_env

    usage = "%s [options] experiments.json | integrated.pickle" % (
        libtbx.env.dispatcher_name)

    parser = OptionParser(usage=usage,
                          phil=phil_scope,
                          read_experiments=True,
                          read_reflections=True,
                          check_format=False,
                          epilog=help_message)

    params, options = parser.parse_args(show_diff_phil=True)
    experiments = flatten_experiments(params.input.experiments)
    reflections = flatten_reflections(params.input.reflections)

    if len(experiments) == 0 or len(reflections) == 0:
        parser.print_help()
        exit()
    reflections = reflections[0]
    cryst = experiments.crystals()[0]
    unit_cell = cryst.get_unit_cell()
    if params.space_group is not None:
        space_group = params.space_group.group()
        assert space_group.is_compatible_unit_cell(unit_cell), unit_cell
    else:
        space_group = cryst.get_space_group()
    print space_group.info()
    print unit_cell

    expt = experiments[0]
    from cctbx import miller, crystal
    from mmtbx.scaling.data_statistics import wilson_scaling
    sel = reflections.get_flags(reflections.flags.integrated_sum)
    reflections = reflections.select(sel)
    cs = crystal.symmetry(unit_cell=unit_cell, space_group=space_group)
    ms = miller.set(cs,
                    indices=reflections['miller_index'],
                    anomalous_flag=True)
    intensities = miller.array(ms,
                               data=reflections['intensity.sum.value'],
                               sigmas=flex.sqrt(
                                   reflections['intensity.sum.variance']))
    intensities.set_observation_type_xray_intensity()
    d_star_sq = intensities.d_star_sq().data()
    n_bins = 20
    #  binner = intensities.setup_binner_d_star_sq_step(
    #    d_star_sq_step=(flex.max(d_star_sq)-flex.min(d_star_sq)+1e-8)/n_bins)
    binner = intensities.setup_binner_counting_sorted(n_bins=n_bins)
    # wilson = intensities.wilson_plot(use_binning=True)
    # wilson.show()
    # from matplotlib import pyplot
    # pyplot.figure()
    # pyplot.scatter(wilson.binner.bin_centers(2), wilson.data[1:-1])
    # pyplot.show()
    intensities = intensities.merge_equivalents().array()
    wilson = wilson_scaling(intensities, n_residues=200)
    wilson.iso_scale_and_b.show()

    from matplotlib import pyplot
    pyplot.figure()
    pyplot.scatter(wilson.d_star_sq, wilson.mean_I_obs_data, label='Data')
    pyplot.plot(wilson.d_star_sq, wilson.mean_I_obs_theory, label='theory')
    pyplot.plot(wilson.d_star_sq,
                wilson.mean_I_normalisation,
                label='smoothed')
    pyplot.yscale('log')
    pyplot.legend()
    pyplot.show()

    import copy
    import math
    # Hack to make the predicter predict reflections outside of the range
    # of the scan
    expt_input = copy.deepcopy(expt)
    scan = expt.scan
    image_range = scan.get_image_range()
    oscillation = scan.get_oscillation()
    scan.set_image_range((1, int(math.ceil(360 / oscillation[1]))))
    scan.set_oscillation((0, oscillation[1]))
    print scan

    # Populate the reflection table with predictions
    predicted = flex.reflection_table.from_predictions(
        expt, force_static=params.force_static, dmin=params.d_min)
    predicted['id'] = flex.int(len(predicted), 0)

    print len(predicted)

    space_group = space_group.build_derived_reflection_intensity_group(
        anomalous_flag=True)
    cs = crystal.symmetry(unit_cell=unit_cell, space_group=space_group)

    ms = miller.set(cs, indices=predicted['miller_index'], anomalous_flag=True)
    ma = miller.array(ms,
                      data=flex.double(ms.size(), 1),
                      sigmas=flex.double(ms.size(), 1))

    d_star_sq = ma.d_star_sq().data()
    n_bins = 1
    binner = ma.setup_binner_d_star_sq_step(
        d_star_sq_step=(flex.max(d_star_sq) - flex.min(d_star_sq) + 1e-8) /
        n_bins)
    image_number = predicted['xyzcal.px'].parts()[2]
    print flex.min(image_number)
    print flex.max(image_number)
    #dose = flex.size_t(list(flex.floor(image_number).iround()))
    angle_deg = predicted['xyzcal.mm'].parts()[2] * 180 / math.pi
    dose = flex.size_t(list(flex.floor(angle_deg).iround()))
    range_width = 1
    range_min = flex.min(dose) - range_width
    range_max = flex.max(dose)
    n_steps = 2 + int((range_max - range_min) - range_width)

    binner_non_anom = ma.as_non_anomalous_array().use_binning(binner)
    n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])

    ranges_dict = {}
    completeness_levels = [10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 99]
    for c in completeness_levels:
        ranges_dict[c] = []
    from xia2.Modules.PyChef2 import ChefStatistics
    step = 1
    for i in range(0, 360, step):
        sel = dose < step
        dose.set_selected(sel, dose.select(sel) + 360)
        dose -= flex.min(dose)
        chef_stats = ChefStatistics(ma.indices(), ma.data(), ma.sigmas(),
                                    ma.d_star_sq().data(), dose, n_complete,
                                    binner, ma.space_group(),
                                    ma.anomalous_flag(), n_steps)

        ieither_completeness = chef_stats.ieither_completeness()
        iboth_completeness = chef_stats.iboth_completeness()

        for c in completeness_levels:
            ranges_dict[c].append(
                min((ieither_completeness > (c / 100)).iselection()))

    from matplotlib import pyplot
    pyplot.figure()
    for c in completeness_levels:
        pyplot.plot(ranges_dict[c], label=str(c))


#  pyplot.plot(range_for_50)
#  pyplot.plot(range_for_99)

#  pyplot.scatter(range(iboth_completeness.size()), iboth_completeness)
    pyplot.legend()
    pyplot.show()
Esempio n. 6
0
def test_exercise_accumulators(xia2_regression):
    from xia2.Modules.PyChef2 import PyChef
    from xia2.Modules.PyChef2 import ChefStatistics
    from iotbx.reflection_file_reader import any_reflection_file
    from cctbx.array_family import flex
    from libtbx.test_utils import approx_equal
    import os

    f = os.path.join(xia2_regression, "test/insulin_dials_scaled_unmerged.mtz")
    reader = any_reflection_file(f)
    assert reader.file_type() == 'ccp4_mtz'
    arrays = reader.as_miller_arrays(merge_equivalents=False)
    for ma in arrays:
        if ma.info().labels == ['BATCH']:
            batches = ma
        elif ma.info().labels == ['I', 'SIGI']:
            intensities = ma
        elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
            intensities = ma
    assert intensities is not None
    assert batches is not None

    anomalous_flag = True
    if anomalous_flag:
        intensities = intensities.as_anomalous_array()

    pystats = PyChef.PyStatistics(intensities, batches.data())

    miller_indices = batches.indices()
    sg = batches.space_group()

    n_steps = pystats.n_steps
    dose = batches.data()
    range_width = 1
    range_max = flex.max(dose)
    range_min = flex.min(dose) - range_width
    dose /= range_width
    dose -= range_min

    binner_non_anom = intensities.as_non_anomalous_array().use_binning(
        pystats.binner)
    n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])

    dose = flex.size_t(list(dose))

    chef_stats = ChefStatistics(miller_indices, intensities.data(),
                                intensities.sigmas(),
                                intensities.d_star_sq().data(), dose,
                                n_complete, pystats.binner, sg, anomalous_flag,
                                n_steps)

    # test completeness

    assert approx_equal(chef_stats.iplus_completeness(),
                        pystats.iplus_comp_overall)
    assert approx_equal(chef_stats.iminus_completeness(),
                        pystats.iminus_comp_overall)
    assert approx_equal(chef_stats.ieither_completeness(),
                        pystats.ieither_comp_overall)
    assert approx_equal(chef_stats.iboth_completeness(),
                        pystats.iboth_comp_overall)

    # test rcp,scp

    assert approx_equal(chef_stats.rcp(), pystats.rcp)
    assert approx_equal(chef_stats.scp(), pystats.scp)

    # test Rd

    assert approx_equal(chef_stats.rd(), pystats.rd)
Esempio n. 7
0
def exercise_accumulators():
  from xia2.Modules.PyChef2 import PyChef
  from xia2.Modules.PyChef2 import ChefStatistics
  from iotbx.reflection_file_reader import any_reflection_file
  from cctbx.array_family import flex
  import libtbx.load_env
  import os

  xia2_regression = libtbx.env.find_in_repositories("xia2_regression")
  if xia2_regression is None:
    print "Skipping exercise_accumulators(): xia2_regression not available"
    return

  f = os.path.join(xia2_regression, "test/insulin_dials_scaled_unmerged.mtz")
  reader = any_reflection_file(f)
  assert reader.file_type() == 'ccp4_mtz'
  arrays = reader.as_miller_arrays(merge_equivalents=False)
  for ma in arrays:
    if ma.info().labels == ['BATCH']:
      batches = ma
    elif ma.info().labels == ['I', 'SIGI']:
      intensities = ma
    elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
      intensities = ma
  assert intensities is not None
  assert batches is not None

  anomalous_flag = True
  if anomalous_flag:
    intensities = intensities.as_anomalous_array()

  pystats = PyChef.PyStatistics(intensities, batches.data())

  miller_indices = batches.indices()
  sg = batches.space_group()

  n_steps = pystats.n_steps
  dose = batches.data()
  range_width  = 1
  range_max = flex.max(dose)
  range_min = flex.min(dose) - range_width
  dose /= range_width
  dose -= range_min

  binner_non_anom = intensities.as_non_anomalous_array().use_binning(
    pystats.binner)
  n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])

  dose = flex.size_t(list(dose))

  chef_stats = ChefStatistics(
    miller_indices, intensities.data(), intensities.sigmas(),
    intensities.d_star_sq().data(), dose, n_complete, pystats.binner,
    sg, anomalous_flag, n_steps)

  # test completeness

  assert approx_equal(chef_stats.iplus_completeness(), pystats.iplus_comp_overall)
  assert approx_equal(chef_stats.iminus_completeness(), pystats.iminus_comp_overall)
  assert approx_equal(chef_stats.ieither_completeness(), pystats.ieither_comp_overall)
  assert approx_equal(chef_stats.iboth_completeness(), pystats.iboth_comp_overall)

  # test rcp,scp

  assert approx_equal(chef_stats.rcp(), pystats.rcp)
  assert approx_equal(chef_stats.scp(), pystats.scp)

  # test Rd

  assert approx_equal(chef_stats.rd(), pystats.rd)

  print "OK"