Пример #1
0
    def integrate(self):

        # Process reference reflections
        self.indexed, _ = self.process_reference(self.indexed)

        # Get integrator from input params
        from dials.algorithms.profile_model.factory import ProfileModelFactory
        from dials.algorithms.integration.integrator import IntegratorFactory

        # Compute the profile model
        self.experiments = ProfileModelFactory.create(self.phil,
                                                      self.experiments,
                                                      self.indexed)

        # Predict the reflections
        predicted = flex.reflection_table.from_predictions_multi(
            self.experiments,
            dmin=self.phil.prediction.d_min,
            dmax=self.phil.prediction.d_max,
            margin=self.phil.prediction.margin,
            force_static=self.phil.prediction.force_static)

        # Match the predictions with the reference
        predicted.match_with_reference(self.indexed)

        # Create the integrator
        integrator = IntegratorFactory.create(self.phil, self.experiments,
                                              predicted)

        # Integrate the reflections
        self.integrated = integrator.integrate()

        if self.integrated.has_key('intensity.prf.value'):
            method = 'prf'  # integration by profile fitting
        elif self.integrated.has_key('intensity.sum.value'):
            method = 'sum'  # integration by simple summation
        self.integrated = self.integrated.select(
            self.integrated['intensity.' + method + '.variance'] >
            0)  # keep only spots with sigmas above zero

        # Save the reflections if selected
        if self.phil.output.integrated_filename:
            self.save_reflections(self.integrated,
                                  self.phil.output.integrated_filename)

        self.write_integration_pickles()
        from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
        rmsd_indexed, _ = calc_2D_rmsd_and_displacements(self.indexed)
        rmsd_integrated, _ = calc_2D_rmsd_and_displacements(self.integrated)
        crystal_model = self.experiments.crystals()[0]
Пример #2
0
 def predict(experiment, table, params):
     elist = ExperimentList([experiment])
     predicted = flex.reflection_table.from_predictions_multi(
         elist,
         dmin=params.prediction.d_min,
         dmax=params.prediction.d_max,
         margin=params.prediction.margin,
         force_static=params.prediction.force_static,
         padding=params.prediction.padding,
     )
     # need to set imageset id?
     elist = ProfileModelFactory.create(params, elist, table)
     predicted.compute_bbox(elist)
     return predicted, elist
Пример #3
0
  def integrate(self):

    # Process reference reflections
    self.indexed,_ = self.process_reference(self.indexed)

    # Get integrator from input params
    from dials.algorithms.profile_model.factory import ProfileModelFactory
    from dials.algorithms.integration.integrator import IntegratorFactory

    # Compute the profile model
    self.experiments = ProfileModelFactory.create(self.phil, self.experiments, self.indexed)

    # Predict the reflections
    predicted = flex.reflection_table.from_predictions_multi(
      self.experiments,
      dmin=self.phil.prediction.d_min,
      dmax=self.phil.prediction.d_max,
      margin=self.phil.prediction.margin,
      force_static=self.phil.prediction.force_static)

    # Match the predictions with the reference
    predicted.match_with_reference(self.indexed)

    # Create the integrator
    integrator = IntegratorFactory.create(self.phil, self.experiments, predicted)

    # Integrate the reflections
    self.integrated = integrator.integrate()

    if self.integrated.has_key('intensity.prf.value'):
      method = 'prf' # integration by profile fitting
    elif self.integrated.has_key('intensity.sum.value'):
      method = 'sum' # integration by simple summation
    self.integrated = self.integrated.select(self.integrated['intensity.' + method + '.variance'] > 0) # keep only spots with sigmas above zero

    # Save the reflections if selected
    if self.phil.output.integrated_filename:
      self.save_reflections(self.integrated, self.phil.output.integrated_filename)

    self.write_integration_pickles()
    from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
    rmsd_indexed, _ = calc_2D_rmsd_and_displacements(self.indexed)
    rmsd_integrated, _ = calc_2D_rmsd_and_displacements(self.integrated)
    crystal_model = self.experiments.crystals()[0]
Пример #4
0
  def integrate(self, experiments, indexed):
    from time import time

    st = time()

    logger.info('*' * 80)
    logger.info('Integrating Reflections')
    logger.info('*' * 80)


    indexed,_ = self.process_reference(indexed)

    # Get the integrator from the input parameters
    logger.info('Configuring integrator from input parameters')
    from dials.algorithms.profile_model.factory import ProfileModelFactory
    from dials.algorithms.integration.integrator import IntegratorFactory
    from dials.array_family import flex

    # Compute the profile model
    # Predict the reflections
    # Match the predictions with the reference
    # Create the integrator
    experiments = ProfileModelFactory.create(self.params, experiments, indexed)
    logger.info("")
    logger.info("=" * 80)
    logger.info("")
    logger.info("Predicting reflections")
    logger.info("")
    predicted = flex.reflection_table.from_predictions_multi(
      experiments,
      dmin=self.params.prediction.d_min,
      dmax=self.params.prediction.d_max,
      margin=self.params.prediction.margin,
      force_static=self.params.prediction.force_static)
    predicted.match_with_reference(indexed)
    logger.info("")
    integrator = IntegratorFactory.create(self.params, experiments, predicted)

    # Integrate the reflections
    integrated = integrator.integrate()

    # Select only those reflections which were integrated
    if 'intensity.prf.variance' in integrated:
      selection = integrated.get_flags(
        integrated.flags.integrated,
        all=True)
    else:
      selection = integrated.get_flags(
        integrated.flags.integrated_sum)
    integrated = integrated.select(selection)

    len_all = len(integrated)
    integrated = integrated.select(~integrated.get_flags(integrated.flags.foreground_includes_bad_pixels))
    print "Filtering %d reflections with at least one bad foreground pixel out of %d"%(len_all-len(integrated), len_all)

    # verify sigmas are sensible
    if 'intensity.prf.value' in integrated:
      if (integrated['intensity.prf.variance'] <= 0).count(True) > 0:
        raise Sorry("Found negative variances")
    if 'intensity.sum.value' in integrated:
      if (integrated['intensity.sum.variance'] <= 0).count(True) > 0:
        raise Sorry("Found negative variances")
      # apply detector gain to summation variances
      integrated['intensity.sum.variance'] *= self.params.integration.summation.detector_gain
    if 'background.sum.value' in integrated:
      if (integrated['background.sum.variance'] < 0).count(True) > 0:
        raise Sorry("Found negative variances")
      if (integrated['background.sum.variance'] == 0).count(True) > 0:
        print "Filtering %d reflections with zero background variance" % ((integrated['background.sum.variance'] == 0).count(True))
        integrated = integrated.select(integrated['background.sum.variance'] > 0)
      # apply detector gain to background summation variances
      integrated['background.sum.variance'] *= self.params.integration.summation.detector_gain

    if self.params.output.integrated_filename:
      # Save the reflections
      self.save_reflections(integrated, self.params.output.integrated_filename)

    self.write_integration_pickles(integrated, experiments)
    from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements

    rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
    log_str = "RMSD indexed (px): %f\n"%(rmsd_indexed)
    for i in xrange(6):
      bright_integrated = integrated.select((integrated['intensity.sum.value']/flex.sqrt(integrated['intensity.sum.variance']))>=i)
      if len(bright_integrated) > 0:
        rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
      else:
        rmsd_integrated = 0
      log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"%(i, len(bright_integrated), rmsd_integrated)

    crystal_model = experiments.crystals()[0]

    if hasattr(crystal_model, '._ML_domain_size_ang'):
      log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg)
    logger.info(log_str)

    logger.info('')
    logger.info('Time Taken = %f seconds' % (time() - st))
    return integrated
Пример #5
0
  def run(self):
    ''' Run the script. '''
    from dials.algorithms.profile_model.factory import ProfileModelFactory
    from dials.util.command_line import Command
    from dials.array_family import flex
    from dials.util.options import flatten_reflections, flatten_experiments
    from dxtbx.model.experiment.experiment_list import ExperimentListDumper
    from libtbx.utils import Sorry
    from dials.util import log

    log.config()

    # Parse the command line
    params, options = self.parser.parse_args(show_diff_phil=True)
    reflections = flatten_reflections(params.input.reflections)
    experiments = flatten_experiments(params.input.experiments)
    if len(reflections) == 0 and len(experiments) == 0:
      self.parser.print_help()
      return
    if len(reflections) != 1:
      raise Sorry('exactly 1 reflection table must be specified')
    if len(experiments) == 0:
      raise Sorry('no experiments were specified')
    if (not 'background.mean' in reflections[0]) and params.subtract_background:
      raise Sorry('for subtract_background need background.mean in reflections')

    reflections, _ = self.process_reference(reflections[0], params)

    # Predict the reflections
    logger.info("")
    logger.info("=" * 80)
    logger.info("")
    logger.info("Predicting reflections")
    logger.info("")
    predicted = flex.reflection_table.from_predictions_multi(
      experiments,
      dmin=params.prediction.d_min,
      dmax=params.prediction.d_max,
      margin=params.prediction.margin,
      force_static=params.prediction.force_static)

    # Match with predicted
    matched, reflections, unmatched = predicted.match_with_reference(reflections)
    assert(len(matched) == len(predicted))
    assert(matched.count(True) <= len(reflections))
    if matched.count(True) == 0:
      raise Sorry('''
        Invalid input for reference reflections.
        Zero reference spots were matched to predictions
      ''')
    elif len(unmatched) != 0:
      logger.info('')
      logger.info('*' * 80)
      logger.info('Warning: %d reference spots were not matched to predictions' % (
        len(unmatched)))
      logger.info('*' * 80)
      logger.info('')

    # Create the profile model
    experiments = ProfileModelFactory.create(params, experiments, reflections)
    for model in experiments:
      sigma_b = model.profile.sigma_b(deg=True)
      sigma_m = model.profile.sigma_m(deg=True)
      if type(sigma_b) == type(1.0):
        logger.info('Sigma B: %f' % sigma_b)
        logger.info('Sigma M: %f' % sigma_m)
      else: # scan varying
        mean_sigma_b = sum(sigma_b) / len(sigma_b)
        mean_sigma_m = sum(sigma_m) / len(sigma_m)
        logger.info('Sigma B: %f' % mean_sigma_b)
        logger.info('Sigma M: %f' % mean_sigma_m)

    # Wrtie the parameters
    Command.start("Writing experiments to %s" % params.output)
    dump = ExperimentListDumper(experiments)
    with open(params.output, "w") as outfile:
      outfile.write(dump.as_json())
    Command.end("Wrote experiments to %s" % params.output)
Пример #6
0
  def integrate(self, experiments, indexed):
    from time import time
    from logging import info

    st = time()

    info('*' * 80)
    info('Integrating Reflections')
    info('*' * 80)


    indexed,_ = self.process_reference(indexed)

    # Get the integrator from the input parameters
    info('Configuring integrator from input parameters')
    from dials.algorithms.profile_model.factory import ProfileModelFactory
    from dials.algorithms.integration.integrator import IntegratorFactory
    from dials.array_family import flex

    # Compute the profile model
    # Predict the reflections
    # Match the predictions with the reference
    # Create the integrator
    experiments = ProfileModelFactory.create(self.params, experiments, indexed)
    info("")
    info("=" * 80)
    info("")
    info("Predicting reflections")
    info("")
    predicted = flex.reflection_table.from_predictions_multi(
      experiments,
      dmin=self.params.prediction.d_min,
      dmax=self.params.prediction.d_max,
      margin=self.params.prediction.margin,
      force_static=self.params.prediction.force_static)
    predicted.match_with_reference(indexed)
    info("")
    integrator = IntegratorFactory.create(self.params, experiments, predicted)

    # Integrate the reflections
    integrated = integrator.integrate()

    if integrated.has_key('intensity.prf.value'):
      method = 'prf' # integration by profile fitting
    elif integrated.has_key('intensity.sum.value'):
      method = 'sum' # integration by simple summation
    integrated = integrated.select(integrated['intensity.' + method + '.variance'] > 0) # keep only spots with sigmas above zero

    if self.params.output.integrated_filename:
      # Save the reflections
      self.save_reflections(integrated, self.params.output.integrated_filename)

    self.write_integration_pickles(integrated, experiments)
    from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
    rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
    rmsd_integrated, _ = calc_2D_rmsd_and_displacements(integrated)
    crystal_model = experiments.crystals()[0]
    print "Integrated. RMSD indexed,", rmsd_indexed, "RMSD integrated", rmsd_integrated, \
      "Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg)

    info('')
    info('Time Taken = %f seconds' % (time() - st))
    return integrated
Пример #7
0
def run_integration(params, experiments, reference=None):
    """Perform the integration.

    Returns:
        experiments: The integrated experiments
        reflections: The integrated reflections
        report(optional): An integration report.

    Raises:
        ValueError: For a number of bad inputs
        RuntimeError: If the profile model creation fails
    """
    predicted = None
    rubbish = None

    for abs_params in params.absorption_correction:
        if abs_params.apply:
            if not (params.integration.debug.output
                    and not params.integration.debug.separate_files):
                raise ValueError(
                    "Shoeboxes must be saved to integration intermediates to apply an absorption correction. "
                    +
                    "Set integration.debug.output=True, integration.debug.separate_files=False and "
                    +
                    "integration.debug.delete_shoeboxes=True to temporarily store shoeboxes."
                )

    # Print if we're using a mask
    for i, exp in enumerate(experiments):
        mask = exp.imageset.external_lookup.mask
        if mask.filename is not None:
            if mask.data:
                logger.info("Using external mask: %s", mask.filename)
                for tile in mask.data:
                    logger.info(" Mask has %d pixels masked",
                                tile.data().count(False))

    # Print the experimental models
    for i, exp in enumerate(experiments):
        summary = "\n".join((
            "",
            "=" * 80,
            "",
            "Experiments",
            "",
            "Models for experiment %d" % i,
            "",
            str(exp.beam),
            str(exp.detector),
        ))
        if exp.goniometer:
            summary += str(exp.goniometer) + "\n"
        if exp.scan:
            summary += str(exp.scan) + "\n"
        summary += str(exp.crystal)
        logger.info(summary)

    logger.info("\n".join(("", "=" * 80, "")))
    logger.info(heading("Initialising"))

    # Load the data
    if reference:
        reference, rubbish = process_reference(reference)

        # Check pixels don't belong to neighbours
        if exp.goniometer is not None and exp.scan is not None:
            reference = filter_reference_pixels(reference, experiments)

        # Modify experiment list if scan range is set.
        experiments, reference = split_for_scan_range(experiments, reference,
                                                      params.scan_range)

    # Modify experiment list if exclude images is set
    if params.exclude_images:
        for experiment in experiments:
            for index in params.exclude_images:
                experiment.imageset.mark_for_rejection(index, True)

    # Predict the reflections
    logger.info("\n".join(("", "=" * 80, "")))
    logger.info(heading("Predicting reflections"))
    predicted = flex.reflection_table.from_predictions_multi(
        experiments,
        dmin=params.prediction.d_min,
        dmax=params.prediction.d_max,
        margin=params.prediction.margin,
        force_static=params.prediction.force_static,
        padding=params.prediction.padding,
    )

    # Match reference with predicted
    if reference:
        matched, reference, unmatched = predicted.match_with_reference(
            reference)
        assert len(matched) == len(predicted)
        assert matched.count(True) <= len(reference)
        if matched.count(True) == 0:
            raise ValueError("""
        Invalid input for reference reflections.
        Zero reference spots were matched to predictions
    """)
        elif unmatched:
            msg = (
                "Warning: %d reference spots were not matched to predictions" %
                unmatched.size())
            border = "\n".join(("", "*" * 80, ""))
            logger.info("".join((border, msg, border)))
            rubbish.extend(unmatched)

        if len(experiments) > 1:
            # filter out any experiments without matched reference reflections
            # f_: filtered

            f_reference = flex.reflection_table()
            f_predicted = flex.reflection_table()
            f_rubbish = flex.reflection_table()
            f_experiments = ExperimentList()
            good_expt_count = 0

            def refl_extend(src, dest, eid):
                old_id = eid
                new_id = good_expt_count
                tmp = src.select(src["id"] == old_id)
                tmp["id"] = flex.int(len(tmp), good_expt_count)
                if old_id in tmp.experiment_identifiers():
                    identifier = tmp.experiment_identifiers()[old_id]
                    del tmp.experiment_identifiers()[old_id]
                    tmp.experiment_identifiers()[new_id] = identifier
                dest.extend(tmp)

            for expt_id, experiment in enumerate(experiments):
                if len(reference.select(reference["id"] == expt_id)) != 0:
                    refl_extend(reference, f_reference, expt_id)
                    refl_extend(predicted, f_predicted, expt_id)
                    refl_extend(rubbish, f_rubbish, expt_id)
                    f_experiments.append(experiment)
                    good_expt_count += 1
                else:
                    logger.info(
                        "Removing experiment %d: no reference reflections matched to predictions",
                        expt_id,
                    )

            reference = f_reference
            predicted = f_predicted
            experiments = f_experiments
            rubbish = f_rubbish

    # Select a random sample of the predicted reflections
    if not params.sampling.integrate_all_reflections:
        predicted = sample_predictions(experiments, predicted, params)

    # Compute the profile model - either load existing or compute
    # can raise RuntimeError
    experiments = ProfileModelFactory.create(params, experiments, reference)
    for expr in experiments:
        if expr.profile is None:
            raise ValueError("No profile information in experiment list")
    del reference

    # Compute the bounding box
    predicted.compute_bbox(experiments)

    # Create the integrator
    integrator = create_integrator(params, experiments, predicted)

    # Integrate the reflections
    reflections = integrator.integrate()

    # Remove unintegrated reflections
    if not params.output.output_unintegrated_reflections:
        keep = reflections.get_flags(reflections.flags.integrated, all=False)
        logger.info(
            "Removing %d unintegrated reflections of %d total",
            keep.count(False),
            keep.size(),
        )

        reflections = reflections.select(keep)

    # Append rubbish data onto the end
    if rubbish is not None and params.output.include_bad_reference:
        mask = flex.bool(len(rubbish), True)
        rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
        rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
        rubbish.set_flags(mask, rubbish.flags.bad_reference)
        reflections.extend(rubbish)

    # Correct integrated intensities for absorption correction, if necessary
    for abs_params in params.absorption_correction:
        if abs_params.apply and abs_params.algorithm == "fuller_kapton":
            from dials.algorithms.integration.kapton_correction import (
                multi_kapton_correction, )

            experiments, reflections = multi_kapton_correction(
                experiments,
                reflections,
                abs_params.fuller_kapton,
                logger=logger)()

    if params.significance_filter.enable:
        from dials.algorithms.integration.stills_significance_filter import (
            SignificanceFilter, )

        sig_filter = SignificanceFilter(params)
        filtered_refls = sig_filter(experiments, reflections)
        accepted_expts = ExperimentList()
        accepted_refls = flex.reflection_table()
        logger.info(
            "Removed %d reflections out of %d when applying significance filter",
            (reflections.size() - filtered_refls.size()),
            reflections.size(),
        )
        for expt_id, expt in enumerate(experiments):
            refls = filtered_refls.select(filtered_refls["id"] == expt_id)
            if refls:
                accepted_expts.append(expt)
                current_id = expt_id
                new_id = len(accepted_expts) - 1
                refls["id"] = flex.int(len(refls), new_id)
                if expt.identifier:
                    del refls.experiment_identifiers()[current_id]
                    refls.experiment_identifiers()[new_id] = expt.identifier
                accepted_refls.extend(refls)
            else:
                logger.info(
                    "Removed experiment %d which has no reflections left after applying significance filter",
                    expt_id,
                )

        if not accepted_refls:
            raise ValueError(
                "No reflections left after applying significance filter")
        experiments = accepted_expts
        reflections = accepted_refls

    # Write a report if requested
    report = None
    if params.output.report is not None:
        report = integrator.report()

    return experiments, reflections, report
Пример #8
0
        assert(matched.count(True) <= len(reference))
        if matched.count(True) == 0:
          raise Sorry('''
            Invalid input for reference reflections.
            Zero reference spots were matched to predictions
          ''')
        elif matched.count(True) != len(reference):
          logger.info('')
          logger.info('*' * 80)
          logger.info('Warning: %d reference spots were not matched to predictions' % (
            len(reference) - matched.count(True)))
          logger.info('*' * 80)
          logger.info('')

        # Compute the profile model
        experiments = ProfileModelFactory.create(params, experiments, reference)

        # Compute the bounding box
        predicted.compute_bbox(experiments)

        # Create the integrator
        integrator = IntegratorFactory.create(params, experiments, predicted)

        # Integrate the reflections
        reflections = integrator.integrate()

        #print len(reflections)

        stats['integrated_intensity'] = flex.sum(reflections['intensity.sum.value'])
      except Exception, e:
        logger.error(e)
Пример #9
0
def integrate_coset(self, experiments, indexed):
        TRANS = self.params.integration.coset.transformation

        # here get a deepcopy that we are not afraid to modify:
        experiments_local = copy.deepcopy(experiments)

        print("*" * 80)
        print("Coset Reflections for modeling or validating the background")
        print("*" * 80)
        from dials.algorithms.profile_model.factory import ProfileModelFactory
        from dials.algorithms.integration.integrator import create_integrator

        # XXX Fixme later implement support for non-primitive lattices NKS
        base_set = miller_set( crystal_symmetry = symmetry(
            unit_cell = experiments_local[0].crystal.get_unit_cell(),
            space_group = experiments_local[0].crystal.get_space_group()),
            indices = indexed["miller_index"]
          )
        triclinic = base_set.customized_copy(
          crystal_symmetry=symmetry(unit_cell = experiments_local[0].crystal.get_unit_cell(),space_group="P1"))

        # ================
        # Compute the profile model
        # Predict the reflections
        # Create the integrator
        # This creates a reference to the experiment, not a copy:
        experiments_local = ProfileModelFactory.create(self.params, experiments_local, indexed)
        # for debug SLT[TRANS].show_summary()

        for e in experiments_local:
          e.crystal.set_space_group(triclinic.space_group())
          Astar = e.crystal.get_A()
          # debug OriAstar = crystal_orientation(Astar,True)
          # debug OriAstar.show(legend="old ")
          Astarprime = sqr(Astar)* ( sqr(SLT[TRANS]._reindex_N).transpose().inverse() )
          e.crystal.set_A(Astarprime)
          # debug OriAstarprime = crystal_orientation(Astarprime,True)
          # debug OriAstarprime.show(legend="new ")

        print("Predicting coset reflections")
        print("")
        predicted = flex.reflection_table.from_predictions_multi(
            experiments_local,
            dmin=self.params.prediction.d_min,
            dmax=self.params.prediction.d_max,
            margin=self.params.prediction.margin,
            force_static=self.params.prediction.force_static,
        )
        print("sublattice total predictions %d"%len(predicted))

        # filter the sublattice, keep only the coset indices
        miller = predicted["miller_index"]
        # coset of modulus 2, wherein there is a binary choice
        # see Sauter & Zwart, Acta D (2009) 65:553, Table 1; select the valid coset using eqn(5).
        coset_select_algorithm_2 = flex.bool()
        M_mat = SLT[TRANS].matS() # the transformation
        M_p = M_mat.inverse()
        for idx in miller:
          H_row = row(idx)
          h_orig_setting = H_row * M_p
          on_coset=False
          for icom in h_orig_setting.elems:
            if icom.denominator() > 1: on_coset=True; break
          coset_select_algorithm_2.append(on_coset)
        predicted = predicted.select(coset_select_algorithm_2)
        print("of which %d are in coset %d"%(len(predicted), TRANS))

        print("")
        integrator = create_integrator(self.params, experiments_local, predicted)

        # Integrate the reflections
        integrated = integrator.integrate()

        # Delete the shoeboxes used for intermediate calculations, if requested
        if self.params.integration.debug.delete_shoeboxes and "shoebox" in integrated:
            del integrated["shoebox"]

        if self.params.output.composite_output:
            if (
                self.params.output.coset_experiments_filename
                or self.params.output.coset_filename
            ):
                assert (
                    self.params.output.coset_experiments_filename is not None
                    and self.params.output.coset_filename is not None
                )
                n = len(self.all_coset_experiments)
                self.all_coset_experiments.extend(experiments_local)
                for i, experiment in enumerate(experiments_local):
                    refls = integrated.select(integrated["id"] == i)
                    refls["id"] = flex.int(len(refls), n)
                    del refls.experiment_identifiers()[i]
                    refls.experiment_identifiers()[n] = experiment.identifier
                    self.all_coset_reflections.extend(refls)
                    n += 1
        else:
            # Dump experiments to disk
            if self.params.output.coset_experiments_filename:

                experiments_local.as_json(self.params.output.coset_experiments_filename)

            if self.params.output.coset_filename:
                # Save the reflections
                self.save_reflections(
                    integrated, self.params.output.coset_filename
                )

        rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
        log_str = "coset RMSD indexed (px): %f\n" % (rmsd_indexed)
        log_str += "integrated %d\n"%len(integrated)
        for i in range(6):
            bright_integrated = integrated.select(
                (
                    integrated["intensity.sum.value"]
                    / flex.sqrt(integrated["intensity.sum.variance"])
                )
                >= i
            )
            if len(bright_integrated) > 0:
                rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
            else:
                rmsd_integrated = 0
            log_str += (
                "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"
                % (i, len(bright_integrated), rmsd_integrated)
            )

        for crystal_model in experiments_local.crystals():
            if hasattr(crystal_model, "get_domain_size_ang"):
                log_str += (
                    ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"
                    % (
                        crystal_model.get_domain_size_ang(),
                        crystal_model.get_half_mosaicity_deg(),
                    )
                )

        print(log_str)
        print("")
Пример #10
0
def run():
    parser = OptionParser(phil=phil_scope)

    params, options = parser.parse_args(show_diff_phil=True)
    assert params.input.single_img is not None
    assert params.output_dir is not None

    # load the image
    img = dxtbx.load(params.input.single_img)
    imgset = MemImageSet([img])
    datablock = DataBlockFactory.from_imageset(imgset)[0]

    spotfinder = SpotFinderFactory.from_parameters(params)
    reflections = spotfinder(datablock)

    base_name = os.path.splitext(params.input.single_img)[0]
    reflections.as_pickle(
        os.path.join(params.output_dir, base_name + "_strong.pickle"))

    # DGW commented out as reflections.minimum_number_of_reflections no longer exists
    # if len(reflections) < params.refinement.reflections.minimum_number_of_reflections:
    #  print "Not enough spots to index"
    #  return

    # create the spot finder

    print("Spotfinder spots found:", len(reflections))

    if params.indexing.method == "fft3d":
        from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer
    elif params.indexing.method == "fft1d":
        from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer
    elif params.method == "real_space_grid_search":
        from dials.algorithms.indexing.real_space_grid_search import (
            indexer_real_space_grid_search as indexer, )
    try:
        idxr = indexer(reflections, [imgset], params=params.indexing)
    except (RuntimeError, Sorry) as e:
        print(str(e))
        return

    indexed = idxr.refined_reflections
    experiments = idxr.refined_experiments
    # from dxtbx.model.experiment.experiment_list import ExperimentListDumper
    # dump = ExperimentListDumper(experiments)
    # dump.as_json(os.path.join(params.output_dir, base_name + "_experiments.json"))
    indexed.as_pickle(
        os.path.join(params.output_dir, base_name + "_indexed.pickle"))

    refiner = RefinerFactory.from_parameters_data_experiments(
        params, indexed, experiments)

    refiner.run()
    refined_experiments = refiner.get_experiments()
    # dump = ExperimentListDumper(refined_experiments)
    # dump.as_json(os.path.join(params.output_dir, base_name + "_refined.json"))

    # Compute the profile model
    # Predict the reflections
    # Match the predictions with the reference
    # Create the integrator
    reference = indexed

    reference = process_reference(reference)
    profile_model = ProfileModelFactory.create(params, refined_experiments,
                                               reference)
    predicted = flex.reflection_table.from_predictions_multi(
        refined_experiments,
        dmin=params.prediction.dmin,
        dmax=params.prediction.dmax,
        margin=params.prediction.margin,
        force_static=params.prediction.force_static,
    )
    predicted.match_with_reference(reference)
    integrator = IntegratorFactory.create(params, experiments, profile_model,
                                          predicted)

    # Integrate the reflections
    integrated = integrator.integrate()
    integrated.as_pickle(
        os.path.join(params.output_dir, base_name + "_integrated.pickle"))
Пример #11
0
        assert(matched.count(True) <= len(reference))
        if matched.count(True) == 0:
          raise Sorry('''
            Invalid input for reference reflections.
            Zero reference spots were matched to predictions
          ''')
        elif matched.count(True) != len(reference):
          logger.info('')
          logger.info('*' * 80)
          logger.info('Warning: %d reference spots were not matched to predictions' % (
            len(reference) - matched.count(True)))
          logger.info('*' * 80)
          logger.info('')

        # Compute the profile model
        experiments = ProfileModelFactory.create(params, experiments, reference)

        # Compute the bounding box
        predicted.compute_bbox(experiments)

        # Create the integrator
        integrator = IntegratorFactory.create(params, experiments, predicted)

        # Integrate the reflections
        reflections = integrator.integrate()

        #print len(reflections)

        stats['integrated_intensity'] = flex.sum(reflections['intensity.sum.value'])
      except Exception, e:
        logger.error(e)
Пример #12
0
    def run(self):
        """ Perform the integration. """
        from dials.util.command_line import heading
        from dials.util.options import flatten_reflections, flatten_experiments
        from dials.util import log
        from logging import info, debug
        from time import time
        from libtbx.utils import Sorry

        # Check the number of arguments is correct
        start_time = time()

        # Parse the command line
        params, options = self.parser.parse_args(show_diff_phil=False)
        reference = flatten_reflections(params.input.reflections)
        experiments = flatten_experiments(params.input.experiments)
        if len(reference) == 0 and len(experiments) == 0:
            self.parser.print_help()
            return
        if len(reference) == 0:
            reference = None
        elif len(reference) != 1:
            raise Sorry("more than 1 reflection file was given")
        else:
            reference = reference[0]
        if len(experiments) == 0:
            raise Sorry("no experiment list was specified")

        # Save phil parameters
        if params.output.phil is not None:
            with open(params.output.phil, "w") as outfile:
                outfile.write(self.parser.diff_phil.as_str())

        # Configure logging
        log.config(params.verbosity, info=params.output.log, debug=params.output.debug_log)

        from dials.util.version import dials_version

        info(dials_version())

        # Log the diff phil
        diff_phil = self.parser.diff_phil.as_str()
        if diff_phil is not "":
            info("The following parameters have been modified:\n")
            info(diff_phil)

        # Print if we're using a mask
        for i, exp in enumerate(experiments):
            mask = exp.imageset.external_lookup.mask
            if mask.filename is not None:
                info("Using external mask: %s" % mask.filename)
                info(" Mask has %d pixels masked" % mask.data.count(False))

        # Print the experimental models
        for i, exp in enumerate(experiments):
            debug("Models for experiment %d" % i)
            debug("")
            debug(str(exp.beam))
            debug(str(exp.detector))
            if exp.goniometer:
                debug(str(exp.goniometer))
            if exp.scan:
                debug(str(exp.scan))
            debug(str(exp.crystal))

        info("=" * 80)
        info("")
        info(heading("Initialising"))
        info("")

        # Load the data
        reference, rubbish = self.process_reference(reference)
        info("")

        # Initialise the integrator
        from dials.algorithms.profile_model.factory import ProfileModelFactory
        from dials.algorithms.integration.integrator import IntegratorFactory
        from dials.array_family import flex

        # Modify experiment list if scan range is set.
        experiments, reference = self.split_for_scan_range(experiments, reference, params.scan_range)

        # Predict the reflections
        info("")
        info("=" * 80)
        info("")
        info(heading("Predicting reflections"))
        info("")
        predicted = flex.reflection_table.from_predictions_multi(
            experiments,
            dmin=params.prediction.d_min,
            dmax=params.prediction.d_max,
            margin=params.prediction.margin,
            force_static=params.prediction.force_static,
        )

        # Match reference with predicted
        if reference:
            matched, reference, unmatched = predicted.match_with_reference(reference)
            assert len(matched) == len(predicted)
            assert matched.count(True) <= len(reference)
            if matched.count(True) == 0:
                raise Sorry(
                    """
          Invalid input for reference reflections.
          Zero reference spots were matched to predictions
        """
                )
            elif len(unmatched) != 0:
                info("")
                info("*" * 80)
                info("Warning: %d reference spots were not matched to predictions" % (len(unmatched)))
                info("*" * 80)
                info("")
            rubbish.extend(unmatched)

        # Select a random sample of the predicted reflections
        if not params.sampling.integrate_all_reflections:
            predicted = self.sample_predictions(experiments, predicted, params)

        # Compute the profile model
        if reference is not None and params.create_profile_model:
            experiments = ProfileModelFactory.create(params, experiments, reference)
        else:
            for expr in experiments:
                expr.profile.params = params.profile
        del reference

        # Compute the bounding box
        predicted.compute_bbox(experiments)

        # Create the integrator
        info("")
        integrator = IntegratorFactory.create(params, experiments, predicted)

        # Integrate the reflections
        reflections = integrator.integrate()

        # Append rubbish data onto the end
        if rubbish is not None and params.output.include_bad_reference:
            mask = flex.bool(len(rubbish), True)
            rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
            rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
            rubbish.set_flags(mask, rubbish.flags.bad_reference)
            reflections.extend(rubbish)

        # Save the reflections
        self.save_reflections(reflections, params.output.reflections)
        self.save_experiments(experiments, params.output.experiments)

        # Write a report if requested
        if params.output.report is not None:
            integrator.report().as_file(params.output.report)

        # Print the total time taken
        info("\nTotal time taken: %f" % (time() - start_time))
Пример #13
0
  def run(self):
    ''' Perform the integration. '''
    from dials.util.command_line import heading
    from dials.util.options import flatten_reflections, flatten_experiments
    from dials.util import log
    from time import time
    from libtbx.utils import Sorry

    # Check the number of arguments is correct
    start_time = time()

    # Parse the command line
    params, options = self.parser.parse_args(show_diff_phil=False)
    reference = flatten_reflections(params.input.reflections)
    experiments = flatten_experiments(params.input.experiments)
    if len(reference) == 0 and len(experiments) == 0:
      self.parser.print_help()
      return
    if len(reference) == 0:
      reference = None
    elif len(reference) != 1:
      raise Sorry('more than 1 reflection file was given')
    else:
      reference = reference[0]
    if len(experiments) == 0:
      raise Sorry('no experiment list was specified')

    # Save phil parameters
    if params.output.phil is not None:
      with open(params.output.phil, "w") as outfile:
        outfile.write(self.parser.diff_phil.as_str())

    # Configure logging
    log.config(
      params.verbosity,
      info=params.output.log,
      debug=params.output.debug_log)

    from dials.util.version import dials_version
    logger.info(dials_version())

    # Log the diff phil
    diff_phil = self.parser.diff_phil.as_str()
    if diff_phil is not '':
      logger.info('The following parameters have been modified:\n')
      logger.info(diff_phil)

    # Print if we're using a mask
    for i, exp in enumerate(experiments):
      mask = exp.imageset.external_lookup.mask
      if mask.filename is not None:
        if mask.data:
          logger.info('Using external mask: %s' % mask.filename)
          logger.info(' Mask has %d pixels masked' % mask.data.count(False))

    # Print the experimental models
    for i, exp in enumerate(experiments):
      logger.debug("Models for experiment %d" % i)
      logger.debug("")
      logger.debug(str(exp.beam))
      logger.debug(str(exp.detector))
      if exp.goniometer:
        logger.debug(str(exp.goniometer))
      if exp.scan:
        logger.debug(str(exp.scan))
      logger.debug(str(exp.crystal))

    logger.info("=" * 80)
    logger.info("")
    logger.info(heading("Initialising"))
    logger.info("")

    # Load the data
    reference, rubbish = self.process_reference(reference)
    logger.info("")

    # Initialise the integrator
    from dials.algorithms.profile_model.factory import ProfileModelFactory
    from dials.algorithms.integration.integrator import IntegratorFactory
    from dials.array_family import flex

    # Modify experiment list if scan range is set.
    experiments, reference = self.split_for_scan_range(
      experiments,
      reference,
      params.scan_range)

    # Predict the reflections
    logger.info("")
    logger.info("=" * 80)
    logger.info("")
    logger.info(heading("Predicting reflections"))
    logger.info("")
    predicted = flex.reflection_table.from_predictions_multi(
      experiments,
      dmin=params.prediction.d_min,
      dmax=params.prediction.d_max,
      margin=params.prediction.margin,
      force_static=params.prediction.force_static)

    # Match reference with predicted
    if reference:
      matched, reference, unmatched = predicted.match_with_reference(reference)
      assert(len(matched) == len(predicted))
      assert(matched.count(True) <= len(reference))
      if matched.count(True) == 0:
        raise Sorry('''
          Invalid input for reference reflections.
          Zero reference spots were matched to predictions
        ''')
      elif len(unmatched) != 0:
        logger.info('')
        logger.info('*' * 80)
        logger.info('Warning: %d reference spots were not matched to predictions' % (
          len(unmatched)))
        logger.info('*' * 80)
        logger.info('')
      rubbish.extend(unmatched)

      if len(experiments) > 1:
        # filter out any experiments without matched reference reflections
        # f_: filtered
        from dxtbx.model.experiment.experiment_list import ExperimentList
        f_reference = flex.reflection_table()
        f_predicted = flex.reflection_table()
        f_rubbish = flex.reflection_table()
        f_experiments = ExperimentList()
        good_expt_count = 0
        def refl_extend(src, dest, eid):
          tmp = src.select(src['id'] == eid)
          tmp['id'] = flex.int(len(tmp), good_expt_count)
          dest.extend(tmp)

        for expt_id, experiment in enumerate(experiments):
          if len(reference.select(reference['id'] == expt_id)) != 0:
            refl_extend(reference, f_reference, expt_id)
            refl_extend(predicted, f_predicted, expt_id)
            refl_extend(rubbish, f_rubbish, expt_id)
            f_experiments.append(experiment)
            good_expt_count += 1
          else:
            logger.info("Removing experiment %d: no reference reflections matched to predictions"%expt_id)

        reference = f_reference
        predicted = f_predicted
        experiments = f_experiments
        rubbish = f_rubbish

    # Select a random sample of the predicted reflections
    if not params.sampling.integrate_all_reflections:
      predicted = self.sample_predictions(experiments, predicted, params)

    # Compute the profile model
    if (params.create_profile_model and
        reference is not None and
        "shoebox" in reference):
      experiments = ProfileModelFactory.create(params, experiments, reference)
    else:
      for expr in experiments:
        if expr.profile is None:
          raise Sorry('No profile information in experiment list')
        expr.profile.params = params.profile
    del reference

    # Compute the bounding box
    predicted.compute_bbox(experiments)

    # Create the integrator
    logger.info("")
    integrator = IntegratorFactory.create(params, experiments, predicted)

    # Integrate the reflections
    reflections = integrator.integrate()

    # Append rubbish data onto the end
    if rubbish is not None and params.output.include_bad_reference:
      mask = flex.bool(len(rubbish), True)
      rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
      rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
      rubbish.set_flags(mask, rubbish.flags.bad_reference)
      reflections.extend(rubbish)

    # Save the reflections
    self.save_reflections(reflections, params.output.reflections)
    self.save_experiments(experiments, params.output.experiments)

    # Write a report if requested
    if params.output.report is not None:
      integrator.report().as_file(params.output.report)

    # Print the total time taken
    logger.info("\nTotal time taken: %f" % (time() - start_time))
Пример #14
0
def work(filename, cl=None):
    if cl is None:
        cl = []

    phil_scope = libtbx.phil.parse("""\
ice_rings {
  filter = True
    .type = bool
  width = 0.004
    .type = float(value_min=0.0)
}
index = False
  .type = bool
integrate = False
  .type = bool
indexing_min_spots = 10
  .type = int(value_min=1)
""")
    interp = phil_scope.command_line_argument_interpreter()
    params, unhandled = interp.process_and_fetch(
        cl, custom_processor="collect_remaining")
    filter_ice = params.extract().ice_rings.filter
    ice_rings_width = params.extract().ice_rings.width
    index = params.extract().index
    integrate = params.extract().integrate
    indexing_min_spots = params.extract().indexing_min_spots

    from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
    from dxtbx.model.experiment_list import ExperimentListFactory
    from dials.array_family import flex

    interp = find_spots_phil_scope.command_line_argument_interpreter()
    phil_scope, unhandled = interp.process_and_fetch(
        unhandled, custom_processor="collect_remaining")
    logger.info("The following spotfinding parameters have been modified:")
    logger.info(find_spots_phil_scope.fetch_diff(source=phil_scope).as_str())
    params = phil_scope.extract()
    # no need to write the hot mask in the server/client
    params.spotfinder.write_hot_mask = False
    experiments = ExperimentListFactory.from_filenames([filename])
    t0 = time.time()
    reflections = flex.reflection_table.from_observations(experiments, params)
    t1 = time.time()
    logger.info("Spotfinding took %.2f seconds" % (t1 - t0))
    from dials.algorithms.spot_finding import per_image_analysis

    imageset = experiments.imagesets()[0]
    reflections.centroid_px_to_mm(experiments)
    reflections.map_centroids_to_reciprocal_space(experiments)
    stats = per_image_analysis.stats_for_reflection_table(
        reflections, filter_ice=filter_ice,
        ice_rings_width=ice_rings_width)._asdict()
    t2 = time.time()
    logger.info("Resolution analysis took %.2f seconds" % (t2 - t1))

    if index and stats["n_spots_no_ice"] > indexing_min_spots:
        logging.basicConfig(stream=sys.stdout, level=logging.INFO)
        from dials.algorithms.indexing import indexer
        from dials.command_line.index import phil_scope as index_phil_scope

        interp = index_phil_scope.command_line_argument_interpreter()
        phil_scope, unhandled = interp.process_and_fetch(
            unhandled, custom_processor="collect_remaining")
        logger.info("The following indexing parameters have been modified:")
        index_phil_scope.fetch_diff(source=phil_scope).show()
        params = phil_scope.extract()

        if (imageset.get_goniometer() is not None
                and imageset.get_scan() is not None
                and imageset.get_scan().is_still()):
            imageset.set_goniometer(None)
            imageset.set_scan(None)

        try:
            idxr = indexer.Indexer.from_parameters(reflections,
                                                   experiments,
                                                   params=params)
            indexing_results = []
            idxr.index()
            indexed_sel = idxr.refined_reflections.get_flags(
                idxr.refined_reflections.flags.indexed)
            indexed_sel &= ~(idxr.refined_reflections.get_flags(
                idxr.refined_reflections.flags.centroid_outlier))
            for i_expt, expt in enumerate(idxr.refined_experiments):
                sel = idxr.refined_reflections["id"] == i_expt
                sel &= indexed_sel
                indexing_results.append({
                    "crystal":
                    expt.crystal.to_dict(),
                    "n_indexed":
                    sel.count(True),
                    "fraction_indexed":
                    sel.count(True) / sel.size(),
                })
            stats["lattices"] = indexing_results
            stats["n_indexed"] = indexed_sel.count(True)
            stats["fraction_indexed"] = indexed_sel.count(True) / len(
                reflections)
        except Exception as e:
            logger.error(e)
            stats["error"] = str(e)
        finally:
            t3 = time.time()
            logger.info("Indexing took %.2f seconds" % (t3 - t2))

        if integrate and "lattices" in stats:

            from dials.algorithms.profile_model.factory import ProfileModelFactory
            from dials.algorithms.integration.integrator import IntegratorFactory
            from dials.command_line.integrate import phil_scope as integrate_phil_scope

            interp = integrate_phil_scope.command_line_argument_interpreter()
            phil_scope, unhandled = interp.process_and_fetch(
                unhandled, custom_processor="collect_remaining")
            logger.error(
                "The following integration parameters have been modified:")
            integrate_phil_scope.fetch_diff(source=phil_scope).show()
            params = phil_scope.extract()

            try:
                params.profile.gaussian_rs.min_spots = 0

                experiments = idxr.refined_experiments
                reference = idxr.refined_reflections

                predicted = flex.reflection_table.from_predictions_multi(
                    experiments,
                    dmin=params.prediction.d_min,
                    dmax=params.prediction.d_max,
                    margin=params.prediction.margin,
                    force_static=params.prediction.force_static,
                )

                matched, reference, unmatched = predicted.match_with_reference(
                    reference)
                assert len(matched) == len(predicted)
                assert matched.count(True) <= len(reference)
                if matched.count(True) == 0:
                    raise Sorry("""
            Invalid input for reference reflections.
            Zero reference spots were matched to predictions
          """)
                elif matched.count(True) != len(reference):
                    logger.info("")
                    logger.info("*" * 80)
                    logger.info(
                        "Warning: %d reference spots were not matched to predictions"
                        % (len(reference) - matched.count(True)))
                    logger.info("*" * 80)
                    logger.info("")

                # Compute the profile model
                experiments = ProfileModelFactory.create(
                    params, experiments, reference)

                # Compute the bounding box
                predicted.compute_bbox(experiments)

                # Create the integrator
                integrator = IntegratorFactory.create(params, experiments,
                                                      predicted)

                # Integrate the reflections
                reflections = integrator.integrate()

                # print len(reflections)

                stats["integrated_intensity"] = flex.sum(
                    reflections["intensity.sum.value"])
            except Exception as e:
                logger.error(e)
                stats["error"] = str(e)
            finally:
                t4 = time.time()
                logger.info("Integration took %.2f seconds" % (t4 - t3))

    return stats
Пример #15
0
    def integrate(self, experiments, indexed):

        # TODO: Figure out if this is necessary and/or how to do this better
        indexed, _ = self.process_reference(indexed)

        if self.params.integration.integration_only_overrides.trusted_range:
            for detector in experiments.detectors():
                for panel in detector:
                    panel.set_trusted_range(
                        self.params.integration.integration_only_overrides.trusted_range
                    )

        # Get the integrator from the input parameters
        from dials.algorithms.integration.integrator import create_integrator
        from dials.algorithms.profile_model.factory import ProfileModelFactory

        # Compute the profile model
        # Predict the reflections
        # Match the predictions with the reference
        # Create the integrator
        experiments = ProfileModelFactory.create(self.params, experiments, indexed)
        new_experiments = ExperimentList()
        new_reflections = flex.reflection_table()
        for expt_id, expt in enumerate(experiments):
            if (
                    self.params.profile.gaussian_rs.parameters.sigma_b_cutoff is None
                    or expt.profile.sigma_b()
                    < self.params.profile.gaussian_rs.parameters.sigma_b_cutoff
            ):
                refls = indexed.select(indexed["id"] == expt_id)
                refls["id"] = flex.int(len(refls), len(new_experiments))
                # refls.reset_ids()
                del refls.experiment_identifiers()[expt_id]
                refls.experiment_identifiers()[len(new_experiments)] = expt.identifier
                new_reflections.extend(refls)
                new_experiments.append(expt)
            else:
                # TODO: this can be done better, also
                print(
                    "Rejected expt %d with sigma_b %f"
                    % (expt_id, expt.profile.sigma_b())
                )
        experiments = new_experiments
        indexed = new_reflections
        if len(experiments) == 0:
            raise RuntimeError("No experiments after filtering by sigma_b")
        predicted = flex.reflection_table.from_predictions_multi(
            experiments,
            dmin=self.params.prediction.d_min,
            dmax=self.params.prediction.d_max,
            margin=self.params.prediction.margin,
            force_static=self.params.prediction.force_static,
        )
        predicted.match_with_reference(indexed)
        integrator = create_integrator(self.params, experiments, predicted)

        # Integrate the reflections
        integrated = integrator.integrate()

        # correct integrated intensities for absorption correction, if necessary
        for abs_params in self.params.integration.absorption_correction:
            if abs_params.apply:
                if abs_params.algorithm == "fuller_kapton":
                    from dials.algorithms.integration.kapton_correction import (
                        multi_kapton_correction,
                    )
                elif abs_params.algorithm == "kapton_2019":
                    from dials.algorithms.integration.kapton_2019_correction import (
                        multi_kapton_correction,
                    )

                experiments, integrated = multi_kapton_correction(
                    experiments, integrated, abs_params.fuller_kapton, logger=logger
                )()

        if self.params.significance_filter.enable:
            from dials.algorithms.integration.stills_significance_filter import (
                SignificanceFilter,
            )

            sig_filter = SignificanceFilter(self.params)
            filtered_refls = sig_filter(experiments, integrated)
            accepted_expts = ExperimentList()
            accepted_refls = flex.reflection_table()

            for expt_id, expt in enumerate(experiments):
                refls = filtered_refls.select(filtered_refls["id"] == expt_id)
                if len(refls) > 0:
                    accepted_expts.append(expt)
                    refls["id"] = flex.int(len(refls), len(accepted_expts) - 1)
                    accepted_refls.extend(refls)
                else:
                    print(
                        "Removed experiment %d which has no reflections left after applying significance filter",
                        expt_id,
                    )

            if len(accepted_refls) == 0:
                raise Sorry("No reflections left after applying significance filter")
            experiments = accepted_expts
            integrated = accepted_refls

        # Delete the shoeboxes used for intermediate calculations, if requested
        if self.params.integration.debug.delete_shoeboxes and "shoebox" in integrated:
            del integrated["shoebox"]

        # Dump experiments to disk
        if self.params.output.integrated_experiments_filename:
            experiments.as_json(self.params.output.integrated_experiments_filename)

        if self.params.output.integrated_filename:
            # Save the reflections
            self.save_reflections(
                integrated, self.params.output.integrated_filename
            )

        self.write_integration_pickles(integrated, experiments)

        # TODO: Figure out what this is
        from dials.algorithms.indexing.stills_indexer import (
            calc_2D_rmsd_and_displacements,
        )

        rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
        log_str = f"RMSD indexed (px): {rmsd_indexed:f}\n"
        for i in range(6):
            bright_integrated = integrated.select(
                (
                        integrated["intensity.sum.value"]
                        / flex.sqrt(integrated["intensity.sum.variance"])
                )
                >= i
            )
            if len(bright_integrated) > 0:
                rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
            else:
                rmsd_integrated = 0
            log_str += (
                    "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"
                    % (i, len(bright_integrated), rmsd_integrated)
            )

        for crystal_model in experiments.crystals():
            if hasattr(crystal_model, "get_domain_size_ang"):
                log_str += ". Final ML model: domain size angstroms: {:f}, half mosaicity degrees: {:f}".format(
                    crystal_model.get_domain_size_ang(),
                    crystal_model.get_half_mosaicity_deg(),
                )

        print(log_str)
        return integrated
Пример #16
0
    def run(self, args=None):
        """Run the script."""
        from dials.algorithms.profile_model.factory import ProfileModelFactory
        from dials.array_family import flex
        from dials.util import Sorry, log
        from dials.util.command_line import Command
        from dials.util.options import reflections_and_experiments_from_files

        log.config()

        # Parse the command line
        params, options = self.parser.parse_args(args, show_diff_phil=True)
        reflections, experiments = reflections_and_experiments_from_files(
            params.input.reflections, params.input.experiments)
        if len(reflections) == 0 and len(experiments) == 0:
            self.parser.print_help()
            return
        if len(reflections) != 1:
            raise Sorry("exactly 1 reflection table must be specified")
        if len(experiments) == 0:
            raise Sorry("no experiments were specified")
        if ("background.mean"
                not in reflections[0]) and params.subtract_background:
            raise Sorry(
                "for subtract_background need background.mean in reflections")

        reflections, _ = self.process_reference(reflections[0], params)

        # Check pixels don't belong to neighbours
        self.filter_reference_pixels(reflections, experiments)

        # Predict the reflections
        logger.info("")
        logger.info("=" * 80)
        logger.info("")
        logger.info("Predicting reflections")
        logger.info("")
        predicted = flex.reflection_table.from_predictions_multi(
            experiments,
            dmin=params.prediction.d_min,
            dmax=params.prediction.d_max,
            margin=params.prediction.margin,
            force_static=params.prediction.force_static,
            padding=params.prediction.padding,
        )

        # Match with predicted
        matched, reflections, unmatched = predicted.match_with_reference(
            reflections)
        assert len(matched) == len(predicted)
        assert matched.count(True) <= len(reflections)
        if matched.count(True) == 0:
            raise Sorry("""
        Invalid input for reference reflections.
        Zero reference spots were matched to predictions
      """)
        elif len(unmatched) != 0:
            logger.info("")
            logger.info("*" * 80)
            logger.info(
                "Warning: %d reference spots were not matched to predictions",
                len(unmatched),
            )
            logger.info("*" * 80)
            logger.info("")

        # Create the profile model
        experiments = ProfileModelFactory.create(params, experiments,
                                                 reflections)
        for model in experiments:
            sigma_b = model.profile.sigma_b(deg=True)
            sigma_m = model.profile.sigma_m(deg=True)
            if model.profile.is_scan_varying():  # scan varying
                mean_sigma_b = sum(sigma_b) / len(sigma_b)
                mean_sigma_m = sum(sigma_m) / len(sigma_m)
                logger.info("Sigma B: %f", mean_sigma_b)
                logger.info("Sigma M: %f", mean_sigma_m)
            else:
                logger.info("Sigma B: %f", sigma_b)
                logger.info("Sigma M: %f", sigma_m)

        # Write the parameters
        Command.start(f"Writing experiments to {params.output}")
        experiments.as_file(params.output)
        Command.end(f"Wrote experiments to {params.output}")
Пример #17
0
    def integrate(self, experiments, indexed):
        from time import time

        st = time()

        logger.info('*' * 80)
        logger.info('Integrating Reflections')
        logger.info('*' * 80)

        indexed, _ = self.process_reference(indexed)

        # Get the integrator from the input parameters
        logger.info('Configuring integrator from input parameters')
        from dials.algorithms.profile_model.factory import ProfileModelFactory
        from dials.algorithms.integration.integrator import IntegratorFactory
        from dials.array_family import flex

        # Compute the profile model
        # Predict the reflections
        # Match the predictions with the reference
        # Create the integrator
        experiments = ProfileModelFactory.create(self.params, experiments,
                                                 indexed)
        logger.info("")
        logger.info("=" * 80)
        logger.info("")
        logger.info("Predicting reflections")
        logger.info("")
        predicted = flex.reflection_table.from_predictions_multi(
            experiments,
            dmin=self.params.prediction.d_min,
            dmax=self.params.prediction.d_max,
            margin=self.params.prediction.margin,
            force_static=self.params.prediction.force_static)
        predicted.match_with_reference(indexed)
        logger.info("")
        integrator = IntegratorFactory.create(self.params, experiments,
                                              predicted)

        # Integrate the reflections
        integrated = integrator.integrate()

        # correct integrated intensities for absorption correction, if necessary
        for abs_params in self.params.integration.absorption_correction:
            if abs_params.apply and abs_params.algorithm == "fuller_kapton":
                from dials.algorithms.integration.kapton_correction import multi_kapton_correction
                experiments, integrated = multi_kapton_correction(
                    experiments,
                    integrated,
                    abs_params.fuller_kapton,
                    logger=logger)()

        if self.params.significance_filter.enable:
            from dials.algorithms.integration.stills_significance_filter import SignificanceFilter
            sig_filter = SignificanceFilter(self.params)
            refls = sig_filter(experiments, integrated)
            logger.info(
                "Removed %d reflections out of %d when applying significance filter"
                % (len(integrated) - len(refls), len(integrated)))
            if len(refls) == 0:
                raise Sorry(
                    "No reflections left after applying significance filter")
            integrated = refls

        # Delete the shoeboxes used for intermediate calculations, if requested
        if self.params.integration.debug.delete_shoeboxes and 'shoebox' in integrated:
            del integrated['shoebox']

        if self.params.output.composite_output:
            if self.params.output.integrated_experiments_filename or self.params.output.integrated_filename:
                assert self.params.output.integrated_experiments_filename is not None and self.params.output.integrated_filename is not None
                from dials.array_family import flex
                n = len(self.all_integrated_experiments)
                self.all_integrated_experiments.extend(experiments)
                for i, experiment in enumerate(experiments):
                    refls = integrated.select(integrated['id'] == i)
                    refls['id'] = flex.int(len(refls), n)
                    self.all_integrated_reflections.extend(refls)
                    n += 1
        else:
            # Dump experiments to disk
            if self.params.output.integrated_experiments_filename:
                from dxtbx.model.experiment_list import ExperimentListDumper
                dump = ExperimentListDumper(experiments)
                dump.as_json(
                    self.params.output.integrated_experiments_filename)

            if self.params.output.integrated_filename:
                # Save the reflections
                self.save_reflections(integrated,
                                      self.params.output.integrated_filename)

        self.write_integration_pickles(integrated, experiments)
        from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements

        rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
        log_str = "RMSD indexed (px): %f\n" % (rmsd_indexed)
        for i in xrange(6):
            bright_integrated = integrated.select(
                (integrated['intensity.sum.value'] /
                 flex.sqrt(integrated['intensity.sum.variance'])) >= i)
            if len(bright_integrated) > 0:
                rmsd_integrated, _ = calc_2D_rmsd_and_displacements(
                    bright_integrated)
            else:
                rmsd_integrated = 0
            log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n" % (
                i, len(bright_integrated), rmsd_integrated)

        for crystal_model in experiments.crystals():
            if hasattr(crystal_model, 'get_domain_size_ang'):
                log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f" % (
                    crystal_model.get_domain_size_ang(),
                    crystal_model.get_half_mosaicity_deg())

        logger.info(log_str)

        logger.info('')
        logger.info('Time Taken = %f seconds' % (time() - st))
        return integrated
Пример #18
0
def work(filename, cl=None):
    if cl is None:
        cl = []
    import libtbx.phil
    phil_scope = libtbx.phil.parse('''\
ice_rings {
  filter = True
    .type = bool
  width = 0.004
    .type = float(value_min=0.0)
}
index = False
  .type = bool
integrate = False
  .type = bool
indexing_min_spots = 10
  .type = int(value_min=1)
''')
    if not os.access(filename, os.R_OK):
        raise RuntimeError("Server does not have read access to file %s" %
                           filename)
    interp = phil_scope.command_line_argument_interpreter()
    params, unhandled = interp.process_and_fetch(
        cl, custom_processor='collect_remaining')
    filter_ice = params.extract().ice_rings.filter
    ice_rings_width = params.extract().ice_rings.width
    index = params.extract().index
    integrate = params.extract().integrate
    indexing_min_spots = params.extract().indexing_min_spots

    from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
    from dxtbx.datablock import DataBlockFactory
    from dials.array_family import flex
    interp = find_spots_phil_scope.command_line_argument_interpreter()
    phil_scope, unhandled = interp.process_and_fetch(
        unhandled, custom_processor='collect_remaining')
    logger.info('The following spotfinding parameters have been modified:')
    logger.info(find_spots_phil_scope.fetch_diff(source=phil_scope).as_str())
    params = phil_scope.extract()
    # no need to write the hot mask in the server/client
    params.spotfinder.write_hot_mask = False
    datablock = DataBlockFactory.from_filenames([filename])[0]
    t0 = time.time()
    reflections = flex.reflection_table.from_observations(datablock, params)
    t1 = time.time()
    logger.info('Spotfinding took %.2f seconds' % (t1 - t0))
    from dials.algorithms.spot_finding import per_image_analysis
    imageset = datablock.extract_imagesets()[0]
    scan = imageset.get_scan()
    if scan is not None:
        i = scan.get_array_range()[0]
    else:
        i = 0
    stats = per_image_analysis.stats_single_image(
        imageset,
        reflections,
        i=i,
        plot=False,
        filter_ice=filter_ice,
        ice_rings_width=ice_rings_width)
    stats = stats.__dict__
    t2 = time.time()
    logger.info('Resolution analysis took %.2f seconds' % (t2 - t1))

    if index and stats['n_spots_no_ice'] > indexing_min_spots:
        import logging
        logging.basicConfig(stream=sys.stdout, level=logging.INFO)
        from dials.algorithms.indexing import indexer
        interp = indexer.master_phil_scope.command_line_argument_interpreter()
        phil_scope, unhandled = interp.process_and_fetch(
            unhandled, custom_processor='collect_remaining')
        imagesets = [imageset]
        logger.info('The following indexing parameters have been modified:')
        indexer.master_phil_scope.fetch_diff(source=phil_scope).show()
        params = phil_scope.extract()
        params.indexing.scan_range = []

        if (imageset.get_goniometer() is not None
                and imageset.get_scan() is not None
                and imageset.get_scan().get_oscillation()[1] == 0):
            imageset.set_goniometer(None)
            imageset.set_scan(None)

        try:
            idxr = indexer.indexer_base.from_parameters(reflections,
                                                        imagesets,
                                                        params=params)
            indexing_results = []
            idxr.index()
            indexed_sel = idxr.refined_reflections.get_flags(
                idxr.refined_reflections.flags.indexed)
            indexed_sel &= ~(idxr.refined_reflections.get_flags(
                idxr.refined_reflections.flags.centroid_outlier))
            for i_expt, expt in enumerate(idxr.refined_experiments):
                sel = idxr.refined_reflections['id'] == i_expt
                sel &= indexed_sel
                indexing_results.append({
                    'crystal':
                    expt.crystal.to_dict(),
                    'n_indexed':
                    sel.count(True),
                    'fraction_indexed':
                    sel.count(True) / sel.size()
                })
            stats['lattices'] = indexing_results
            stats['n_indexed'] = indexed_sel.count(True)
            stats['fraction_indexed'] = indexed_sel.count(True) / len(
                reflections)
        except Exception as e:
            logger.error(e)
            stats['error'] = str(e)
            #stats.crystal = None
            #stats.n_indexed = None
            #stats.fraction_indexed = None
        finally:
            t3 = time.time()
            logger.info('Indexing took %.2f seconds' % (t3 - t2))

        if integrate and 'lattices' in stats:

            from dials.algorithms.profile_model.factory import ProfileModelFactory
            from dials.algorithms.integration.integrator import IntegratorFactory
            from dials.command_line.integrate import phil_scope as integrate_phil_scope
            interp = integrate_phil_scope.command_line_argument_interpreter()
            phil_scope, unhandled = interp.process_and_fetch(
                unhandled, custom_processor='collect_remaining')
            imagesets = [imageset]
            logger.error(
                'The following integration parameters have been modified:')
            integrate_phil_scope.fetch_diff(source=phil_scope).show()
            params = phil_scope.extract()

            try:
                params.profile.gaussian_rs.min_spots = 0

                experiments = idxr.refined_experiments
                reference = idxr.refined_reflections

                predicted = flex.reflection_table.from_predictions_multi(
                    experiments,
                    dmin=params.prediction.d_min,
                    dmax=params.prediction.d_max,
                    margin=params.prediction.margin,
                    force_static=params.prediction.force_static)

                matched, reference, unmatched = predicted.match_with_reference(
                    reference)
                assert (len(matched) == len(predicted))
                assert (matched.count(True) <= len(reference))
                if matched.count(True) == 0:
                    raise Sorry('''
            Invalid input for reference reflections.
            Zero reference spots were matched to predictions
          ''')
                elif matched.count(True) != len(reference):
                    logger.info('')
                    logger.info('*' * 80)
                    logger.info(
                        'Warning: %d reference spots were not matched to predictions'
                        % (len(reference) - matched.count(True)))
                    logger.info('*' * 80)
                    logger.info('')

                # Compute the profile model
                experiments = ProfileModelFactory.create(
                    params, experiments, reference)

                # Compute the bounding box
                predicted.compute_bbox(experiments)

                # Create the integrator
                integrator = IntegratorFactory.create(params, experiments,
                                                      predicted)

                # Integrate the reflections
                reflections = integrator.integrate()

                #print len(reflections)

                stats['integrated_intensity'] = flex.sum(
                    reflections['intensity.sum.value'])
            except Exception as e:
                logger.error(e)
                stats['error'] = str(e)
            finally:
                t4 = time.time()
                logger.info('Integration took %.2f seconds' % (t4 - t3))

    return stats
Пример #19
0
    def integrate(self, experiments, indexed):
        if self.params.skip_hopper:
            return super(Hopper_Processor,
                         self).integrate(experiments, indexed)
        st = time.time()

        logger.info("*" * 80)
        logger.info("Integrating Reflections")
        logger.info("*" * 80)

        indexed, _ = self.process_reference(indexed)

        if self.params.integration.integration_only_overrides.trusted_range:
            for detector in experiments.detectors():
                for panel in detector:
                    panel.set_trusted_range(
                        self.params.integration.integration_only_overrides.
                        trusted_range)

        if self.params.dispatch.coset:
            from xfel.util.sublattice_helper import integrate_coset

            integrate_coset(self, experiments, indexed)

        # Get the integrator from the input parameters
        logger.info("Configuring integrator from input parameters")
        from dials.algorithms.integration.integrator import create_integrator
        from dials.algorithms.profile_model.factory import ProfileModelFactory

        # Compute the profile model
        # Predict the reflections
        # Match the predictions with the reference
        # Create the integrator
        experiments = ProfileModelFactory.create(self.params, experiments,
                                                 indexed)
        new_experiments = ExperimentList()
        new_reflections = flex.reflection_table()
        for expt_id, expt in enumerate(experiments):
            if (self.params.profile.gaussian_rs.parameters.sigma_b_cutoff is
                    None or expt.profile.sigma_b() <
                    self.params.profile.gaussian_rs.parameters.sigma_b_cutoff):
                refls = indexed.select(indexed["id"] == expt_id)
                refls["id"] = flex.int(len(refls), len(new_experiments))
                # refls.reset_ids()
                del refls.experiment_identifiers()[expt_id]
                refls.experiment_identifiers()[len(
                    new_experiments)] = expt.identifier
                new_reflections.extend(refls)
                new_experiments.append(expt)
            else:
                logger.info("Rejected expt %d with sigma_b %f" %
                            (expt_id, expt.profile.sigma_b()))
        experiments = new_experiments
        indexed = new_reflections
        if len(experiments) == 0:
            raise RuntimeError("No experiments after filtering by sigma_b")
        logger.info("")
        logger.info("=" * 80)
        logger.info("")
        logger.info("Predicting reflections")
        logger.info("")
        # NOTE: this is the only changed needed to dials.stills_process
        # TODO: multi xtal
        # TODO: add in normal dials predictions as an option
        predicted, model = predictions.get_predicted_from_pandas(
            self.stage1_df,
            self.params.diffBragg,
            self.observed,
            experiments[0].identifier,
            self.device_id,
            spectrum_override=self.stage1_modeler.SIM.beam.spectrum)
        if self.params.refine_predictions:
            experiments, rnd2_refls = self.refine(experiments,
                                                  predicted,
                                                  refining_predictions=True,
                                                  best=self.stage1_df)
            # TODO: match rnd2_refls with indexed.refl and re-save indexed.refl
            predicted, model = predictions.get_predicted_from_pandas(
                self.stage1_df,
                self.params.diffBragg,
                self.observed,
                experiments[0].identifier,
                self.device_id,
                spectrum_override=self.stage1_modeler.SIM.beam.spectrum)

        predicted.match_with_reference(indexed)
        integrator = create_integrator(self.params, experiments, predicted)

        # Integrate the reflections
        integrated = integrator.integrate()

        if self.params.partial_correct:
            integrated = predictions.normalize_by_partiality(
                integrated,
                model,
                default_F=self.params.diffBragg.predictions.default_Famplitude,
                gain=self.params.diffBragg.refiner.adu_per_photon)

        # correct integrated intensities for absorption correction, if necessary
        for abs_params in self.params.integration.absorption_correction:
            if abs_params.apply:
                if abs_params.algorithm == "fuller_kapton":
                    from dials.algorithms.integration.kapton_correction import (
                        multi_kapton_correction, )
                elif abs_params.algorithm == "kapton_2019":
                    from dials.algorithms.integration.kapton_2019_correction import (
                        multi_kapton_correction, )

                experiments, integrated = multi_kapton_correction(
                    experiments,
                    integrated,
                    abs_params.fuller_kapton,
                    logger=logger)()

        if self.params.significance_filter.enable:
            from dials.algorithms.integration.stills_significance_filter import (
                SignificanceFilter, )

            sig_filter = SignificanceFilter(self.params)
            filtered_refls = sig_filter(experiments, integrated)
            accepted_expts = ExperimentList()
            accepted_refls = flex.reflection_table()
            logger.info(
                "Removed %d reflections out of %d when applying significance filter",
                len(integrated) - len(filtered_refls),
                len(integrated),
            )
            for expt_id, expt in enumerate(experiments):
                refls = filtered_refls.select(filtered_refls["id"] == expt_id)
                if len(refls) > 0:
                    accepted_expts.append(expt)
                    refls["id"] = flex.int(len(refls), len(accepted_expts) - 1)
                    accepted_refls.extend(refls)
                else:
                    logger.info(
                        "Removed experiment %d which has no reflections left after applying significance filter",
                        expt_id,
                    )

            if len(accepted_refls) == 0:
                raise Sorry(
                    "No reflections left after applying significance filter")
            experiments = accepted_expts
            integrated = accepted_refls

        # Delete the shoeboxes used for intermediate calculations, if requested
        if self.params.integration.debug.delete_shoeboxes and "shoebox" in integrated:
            del integrated["shoebox"]

        if self.params.output.composite_output:
            if (self.params.output.integrated_experiments_filename
                    or self.params.output.integrated_filename):
                assert (self.params.output.integrated_experiments_filename
                        is not None
                        and self.params.output.integrated_filename is not None)

                n = len(self.all_integrated_experiments)
                self.all_integrated_experiments.extend(experiments)
                for i, experiment in enumerate(experiments):
                    refls = integrated.select(integrated["id"] == i)
                    refls["id"] = flex.int(len(refls), n)
                    del refls.experiment_identifiers()[i]
                    refls.experiment_identifiers()[n] = experiment.identifier
                    self.all_integrated_reflections.extend(refls)
                    n += 1
        else:
            # Dump experiments to disk
            if self.params.output.integrated_experiments_filename:

                experiments.as_json(
                    self.params.output.integrated_experiments_filename)

            if self.params.output.integrated_filename:
                # Save the reflections
                self.save_reflections(integrated,
                                      self.params.output.integrated_filename)

        self.write_integration_pickles(integrated, experiments)
        from dials.algorithms.indexing.stills_indexer import (
            calc_2D_rmsd_and_displacements, )

        rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
        log_str = "RMSD indexed (px): %f\n" % rmsd_indexed
        for i in range(6):
            bright_integrated = integrated.select(
                (integrated["intensity.sum.value"] /
                 flex.sqrt(integrated["intensity.sum.variance"])) >= i)
            if len(bright_integrated) > 0:
                rmsd_integrated, _ = calc_2D_rmsd_and_displacements(
                    bright_integrated)
            else:
                rmsd_integrated = 0
            log_str += (
                "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"
                % (i, len(bright_integrated), rmsd_integrated))

        for crystal_model in experiments.crystals():
            if hasattr(crystal_model, "get_domain_size_ang"):
                log_str += ". Final ML model: domain size angstroms: {:f}, half mosaicity degrees: {:f}".format(
                    crystal_model.get_domain_size_ang(),
                    crystal_model.get_half_mosaicity_deg(),
                )

        logger.info(log_str)

        logger.info("")
        logger.info("Time Taken = %f seconds", time.time() - st)
        return integrated
Пример #20
0
  def integrate(self, experiments, indexed):
    from time import time

    st = time()

    logger.info('*' * 80)
    logger.info('Integrating Reflections')
    logger.info('*' * 80)


    indexed,_ = self.process_reference(indexed)

    # Get the integrator from the input parameters
    logger.info('Configuring integrator from input parameters')
    from dials.algorithms.profile_model.factory import ProfileModelFactory
    from dials.algorithms.integration.integrator import IntegratorFactory
    from dials.array_family import flex

    # Compute the profile model
    # Predict the reflections
    # Match the predictions with the reference
    # Create the integrator
    experiments = ProfileModelFactory.create(self.params, experiments, indexed)
    logger.info("")
    logger.info("=" * 80)
    logger.info("")
    logger.info("Predicting reflections")
    logger.info("")
    predicted = flex.reflection_table.from_predictions_multi(
      experiments,
      dmin=self.params.prediction.d_min,
      dmax=self.params.prediction.d_max,
      margin=self.params.prediction.margin,
      force_static=self.params.prediction.force_static)
    predicted.match_with_reference(indexed)
    logger.info("")
    integrator = IntegratorFactory.create(self.params, experiments, predicted)

    # Integrate the reflections
    integrated = integrator.integrate()

    # Select only those reflections which were integrated
    if 'intensity.prf.variance' in integrated:
      selection = integrated.get_flags(
        integrated.flags.integrated,
        all=True)
    else:
      selection = integrated.get_flags(
        integrated.flags.integrated_sum)
    integrated = integrated.select(selection)

    len_all = len(integrated)
    integrated = integrated.select(~integrated.get_flags(integrated.flags.foreground_includes_bad_pixels))
    print "Filtering %d reflections with at least one bad foreground pixel out of %d"%(len_all-len(integrated), len_all)

    # verify sigmas are sensible
    if 'intensity.prf.value' in integrated:
      if (integrated['intensity.prf.variance'] <= 0).count(True) > 0:
        raise Sorry("Found negative variances")
    if 'intensity.sum.value' in integrated:
      if (integrated['intensity.sum.variance'] <= 0).count(True) > 0:
        raise Sorry("Found negative variances")
      # apply detector gain to summation variances
      integrated['intensity.sum.variance'] *= self.params.integration.summation.detector_gain
    if 'background.sum.value' in integrated:
      if (integrated['background.sum.variance'] < 0).count(True) > 0:
        raise Sorry("Found negative variances")
      if (integrated['background.sum.variance'] == 0).count(True) > 0:
        print "Filtering %d reflections with zero background variance" % ((integrated['background.sum.variance'] == 0).count(True))
        integrated = integrated.select(integrated['background.sum.variance'] > 0)
      # apply detector gain to background summation variances
      integrated['background.sum.variance'] *= self.params.integration.summation.detector_gain

    # correct integrated intensities for absorption correction, if necessary
    for abs_params in self.params.integration.absorption_correction:
      if abs_params.apply and abs_params.algorithm == "fuller_kapton":
        from dials.algorithms.integration.kapton_correction import multi_kapton_correction
        experiments, integrated = multi_kapton_correction(experiments, integrated,
          abs_params.fuller_kapton, logger=logger)()

    if self.params.significance_filter.enable:
      from dials.algorithms.integration.stills_significance_filter import SignificanceFilter
      sig_filter = SignificanceFilter(self.params)
      refls = sig_filter(experiments, integrated)
      logger.info("Removed %d reflections out of %d when applying significance filter"%(len(integrated)-len(refls), len(integrated)))
      if len(refls) == 0:
        raise Sorry("No reflections left after applying significance filter")
      integrated = refls

    if self.params.output.integrated_filename:
      # Save the reflections
      self.save_reflections(integrated, self.params.output.integrated_filename)

    self.write_integration_pickles(integrated, experiments)
    from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements

    rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
    log_str = "RMSD indexed (px): %f\n"%(rmsd_indexed)
    for i in xrange(6):
      bright_integrated = integrated.select((integrated['intensity.sum.value']/flex.sqrt(integrated['intensity.sum.variance']))>=i)
      if len(bright_integrated) > 0:
        rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
      else:
        rmsd_integrated = 0
      log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"%(i, len(bright_integrated), rmsd_integrated)

    for crystal_model in experiments.crystals():
      if hasattr(crystal_model, '_ML_domain_size_ang'):
        log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg)

    logger.info(log_str)

    logger.info('')
    logger.info('Time Taken = %f seconds' % (time() - st))
    return integrated
Пример #21
0
def run():
  parser = OptionParser(
    phil = phil_scope)

  params, options = parser.parse_args(show_diff_phil=True)
  assert params.input.single_img is not None
  assert params.output_dir is not None

  # load the image
  img = dxtbx.load(params.input.single_img)
  imgset = MemImageSet([img])
  datablock = DataBlockFactory.from_imageset(imgset)[0]

  spotfinder = SpotFinderFactory.from_parameters(params)
  reflections = spotfinder(datablock)

  base_name = os.path.splitext(params.input.single_img)[0]
  reflections.as_pickle(os.path.join(params.output_dir, base_name + "_strong.pickle"))

  # DGW commented out as reflections.minimum_number_of_reflections no longer exists
  #if len(reflections) < params.refinement.reflections.minimum_number_of_reflections:
  #  print "Not enough spots to index"
  #  return

  # create the spot finder

  print "Spotfinder spots found:", len(reflections)

  if params.indexing.method == "fft3d":
    from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer
  elif params.indexing.method == "fft1d":
    from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer
  elif params.method == "real_space_grid_search":
    from dials.algorithms.indexing.real_space_grid_search \
         import indexer_real_space_grid_search as indexer
  try:
    idxr = indexer(reflections, [imgset], params=params.indexing)
  except (RuntimeError, Sorry) as e:
    print str(e)
    return

  indexed = idxr.refined_reflections
  experiments = idxr.refined_experiments
  #from dxtbx.model.experiment.experiment_list import ExperimentListDumper
  #dump = ExperimentListDumper(experiments)
  #dump.as_json(os.path.join(params.output_dir, base_name + "_experiments.json"))
  indexed.as_pickle(os.path.join(params.output_dir, base_name + "_indexed.pickle"))

  refiner = RefinerFactory.from_parameters_data_experiments(
    params, indexed, experiments)

  refiner.run()
  refined_experiments = refiner.get_experiments()
  #dump = ExperimentListDumper(refined_experiments)
  #dump.as_json(os.path.join(params.output_dir, base_name + "_refined.json"))

  # Compute the profile model
  # Predict the reflections
  # Match the predictions with the reference
  # Create the integrator
  reference = indexed

  reference = process_reference(reference)
  profile_model = ProfileModelFactory.create(params, refined_experiments, reference)
  predicted = flex.reflection_table.from_predictions_multi(
    refined_experiments,
    dmin=params.prediction.dmin,
    dmax=params.prediction.dmax,
    margin=params.prediction.margin,
    force_static=params.prediction.force_static)
  predicted.match_with_reference(reference)
  integrator = IntegratorFactory.create(params, experiments, profile_model, predicted)

  # Integrate the reflections
  integrated = integrator.integrate()
  integrated.as_pickle(os.path.join(params.output_dir, base_name + "_integrated.pickle"))
Пример #22
0
    def run(self, args=None):
        """ Perform the integration. """
        from dials.util.command_line import heading
        from dials.util.options import flatten_reflections, flatten_experiments
        from dials.util import log
        from time import time
        from dials.util import Sorry

        # Check the number of arguments is correct
        start_time = time()

        # Parse the command line
        params, options = self.parser.parse_args(args=args,
                                                 show_diff_phil=False)
        reference = flatten_reflections(params.input.reflections)
        experiments = flatten_experiments(params.input.experiments)
        if len(reference) == 0 and len(experiments) == 0:
            self.parser.print_help()
            return
        if len(reference) == 0:
            reference = None
        elif len(reference) != 1:
            raise Sorry("more than 1 reflection file was given")
        else:
            reference = reference[0]
        if len(experiments) == 0:
            raise Sorry("no experiment list was specified")

        # Save phil parameters
        if params.output.phil is not None:
            with open(params.output.phil, "w") as outfile:
                outfile.write(self.parser.diff_phil.as_str())

        if __name__ == "__main__":
            # Configure logging
            log.config(params.verbosity,
                       info=params.output.log,
                       debug=params.output.debug_log)

        from dials.util.version import dials_version

        logger.info(dials_version())

        # Log the diff phil
        diff_phil = self.parser.diff_phil.as_str()
        if diff_phil != "":
            logger.info("The following parameters have been modified:\n")
            logger.info(diff_phil)

        for abs_params in params.absorption_correction:
            if abs_params.apply:
                if not (params.integration.debug.output
                        and not params.integration.debug.separate_files):
                    raise Sorry(
                        "Shoeboxes must be saved to integration intermediates to apply an absorption correction. "
                        +
                        "Set integration.debug.output=True, integration.debug.separate_files=False and "
                        +
                        "integration.debug.delete_shoeboxes=True to temporarily store shoeboxes."
                    )

        # Print if we're using a mask
        for i, exp in enumerate(experiments):
            mask = exp.imageset.external_lookup.mask
            if mask.filename is not None:
                if mask.data:
                    logger.info("Using external mask: %s" % mask.filename)
                    for tile in mask.data:
                        logger.info(" Mask has %d pixels masked" %
                                    tile.data().count(False))

        # Print the experimental models
        for i, exp in enumerate(experiments):
            logger.info("=" * 80)
            logger.info("")
            logger.info("Experiments")
            logger.info("")
            logger.info("Models for experiment %d" % i)
            logger.info("")
            logger.info(str(exp.beam))
            logger.info(str(exp.detector))
            if exp.goniometer:
                logger.info(str(exp.goniometer))
            if exp.scan:
                logger.info(str(exp.scan))
            logger.info(str(exp.crystal))

        logger.info("=" * 80)
        logger.info("")
        logger.info(heading("Initialising"))
        logger.info("")

        # Load the data
        reference, rubbish = self.process_reference(reference)

        # Check pixels don't belong to neighbours
        if reference is not None:
            if exp.goniometer is not None and exp.scan is not None:
                self.filter_reference_pixels(reference, experiments)
        logger.info("")

        # Initialise the integrator
        from dials.algorithms.profile_model.factory import ProfileModelFactory
        from dials.algorithms.integration.integrator import IntegratorFactory

        # Modify experiment list if scan range is set.
        experiments, reference = self.split_for_scan_range(
            experiments, reference, params.scan_range)

        # Modify experiment list if exclude images is set
        experiments = self.exclude_images(experiments, params.exclude_images)

        # Predict the reflections
        logger.info("")
        logger.info("=" * 80)
        logger.info("")
        logger.info(heading("Predicting reflections"))
        logger.info("")
        predicted = flex.reflection_table.from_predictions_multi(
            experiments,
            dmin=params.prediction.d_min,
            dmax=params.prediction.d_max,
            margin=params.prediction.margin,
            force_static=params.prediction.force_static,
            padding=params.prediction.padding,
        )

        # Match reference with predicted
        if reference:
            matched, reference, unmatched = predicted.match_with_reference(
                reference)
            assert len(matched) == len(predicted)
            assert matched.count(True) <= len(reference)
            if matched.count(True) == 0:
                raise Sorry("""
          Invalid input for reference reflections.
          Zero reference spots were matched to predictions
        """)
            elif len(unmatched) != 0:
                logger.info("")
                logger.info("*" * 80)
                logger.info(
                    "Warning: %d reference spots were not matched to predictions"
                    % (len(unmatched)))
                logger.info("*" * 80)
                logger.info("")
            rubbish.extend(unmatched)

            if len(experiments) > 1:
                # filter out any experiments without matched reference reflections
                # f_: filtered
                from dxtbx.model.experiment_list import ExperimentList

                f_reference = flex.reflection_table()
                f_predicted = flex.reflection_table()
                f_rubbish = flex.reflection_table()
                f_experiments = ExperimentList()
                good_expt_count = 0

                def refl_extend(src, dest, eid):
                    tmp = src.select(src["id"] == eid)
                    tmp["id"] = flex.int(len(tmp), good_expt_count)
                    dest.extend(tmp)

                for expt_id, experiment in enumerate(experiments):
                    if len(reference.select(reference["id"] == expt_id)) != 0:
                        refl_extend(reference, f_reference, expt_id)
                        refl_extend(predicted, f_predicted, expt_id)
                        refl_extend(rubbish, f_rubbish, expt_id)
                        f_experiments.append(experiment)
                        good_expt_count += 1
                    else:
                        logger.info(
                            "Removing experiment %d: no reference reflections matched to predictions"
                            % expt_id)

                reference = f_reference
                predicted = f_predicted
                experiments = f_experiments
                rubbish = f_rubbish

        # Select a random sample of the predicted reflections
        if not params.sampling.integrate_all_reflections:
            predicted = self.sample_predictions(experiments, predicted, params)

        # Compute the profile model
        if (params.create_profile_model and reference is not None
                and "shoebox" in reference):
            experiments = ProfileModelFactory.create(params, experiments,
                                                     reference)
        else:
            experiments = ProfileModelFactory.create(params, experiments)
            for expr in experiments:
                if expr.profile is None:
                    raise Sorry("No profile information in experiment list")
        del reference

        # Compute the bounding box
        predicted.compute_bbox(experiments)

        # Create the integrator
        logger.info("")
        integrator = IntegratorFactory.create(params, experiments, predicted)

        # Integrate the reflections
        reflections = integrator.integrate()

        # Append rubbish data onto the end
        if rubbish is not None and params.output.include_bad_reference:
            mask = flex.bool(len(rubbish), True)
            rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
            rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
            rubbish.set_flags(mask, rubbish.flags.bad_reference)
            reflections.extend(rubbish)

        # Correct integrated intensities for absorption correction, if necessary
        for abs_params in params.absorption_correction:
            if abs_params.apply and abs_params.algorithm == "fuller_kapton":
                from dials.algorithms.integration.kapton_correction import (
                    multi_kapton_correction, )

                experiments, reflections = multi_kapton_correction(
                    experiments,
                    reflections,
                    abs_params.fuller_kapton,
                    logger=logger)()

        if params.significance_filter.enable:
            from dials.algorithms.integration.stills_significance_filter import (
                SignificanceFilter, )
            from dxtbx.model.experiment_list import ExperimentList

            sig_filter = SignificanceFilter(params)
            filtered_refls = sig_filter(experiments, reflections)
            accepted_expts = ExperimentList()
            accepted_refls = flex.reflection_table()
            logger.info(
                "Removed %d reflections out of %d when applying significance filter"
                % (len(reflections) - len(filtered_refls), len(reflections)))
            for expt_id, expt in enumerate(experiments):
                refls = filtered_refls.select(filtered_refls["id"] == expt_id)
                if len(refls) > 0:
                    accepted_expts.append(expt)
                    refls["id"] = flex.int(len(refls), len(accepted_expts) - 1)
                    accepted_refls.extend(refls)
                else:
                    logger.info(
                        "Removed experiment %d which has no reflections left after applying significance filter"
                        % expt_id)

            if len(accepted_refls) == 0:
                raise Sorry(
                    "No reflections left after applying significance filter")
            experiments = accepted_expts
            reflections = accepted_refls

        # Delete the shoeboxes used for intermediate calculations, if requested
        if params.integration.debug.delete_shoeboxes and "shoebox" in reflections:
            del reflections["shoebox"]

        # Save the reflections
        self.save_reflections(reflections, params.output.reflections)
        self.save_experiments(experiments, params.output.experiments)

        # Write a report if requested
        if params.output.report is not None:
            integrator.report().as_file(params.output.report)

        # Print the total time taken
        logger.info("\nTotal time taken: %f" % (time() - start_time))

        return experiments, reflections
Пример #23
0
    def run(self):
        ''' Perform the integration. '''
        from dials.util.command_line import heading
        from dials.util.options import flatten_reflections, flatten_experiments
        from dials.util import log
        from time import time
        from libtbx.utils import Sorry

        # Check the number of arguments is correct
        start_time = time()

        # Parse the command line
        params, options = self.parser.parse_args(show_diff_phil=False)
        reference = flatten_reflections(params.input.reflections)
        experiments = flatten_experiments(params.input.experiments)
        if len(reference) == 0 and len(experiments) == 0:
            self.parser.print_help()
            return
        if len(reference) == 0:
            reference = None
        elif len(reference) != 1:
            raise Sorry('more than 1 reflection file was given')
        else:
            reference = reference[0]
        if len(experiments) == 0:
            raise Sorry('no experiment list was specified')

        # Save phil parameters
        if params.output.phil is not None:
            with open(params.output.phil, "w") as outfile:
                outfile.write(self.parser.diff_phil.as_str())

        # Configure logging
        log.config(params.verbosity,
                   info=params.output.log,
                   debug=params.output.debug_log)

        from dials.util.version import dials_version
        logger.info(dials_version())

        # Log the diff phil
        diff_phil = self.parser.diff_phil.as_str()
        if diff_phil is not '':
            logger.info('The following parameters have been modified:\n')
            logger.info(diff_phil)

        # Print if we're using a mask
        for i, exp in enumerate(experiments):
            mask = exp.imageset.external_lookup.mask
            if mask.filename is not None:
                if mask.data:
                    logger.info('Using external mask: %s' % mask.filename)
                    logger.info(' Mask has %d pixels masked' %
                                mask.data.count(False))

        # Print the experimental models
        for i, exp in enumerate(experiments):
            logger.debug("Models for experiment %d" % i)
            logger.debug("")
            logger.debug(str(exp.beam))
            logger.debug(str(exp.detector))
            if exp.goniometer:
                logger.debug(str(exp.goniometer))
            if exp.scan:
                logger.debug(str(exp.scan))
            logger.debug(str(exp.crystal))

        logger.info("=" * 80)
        logger.info("")
        logger.info(heading("Initialising"))
        logger.info("")

        # Load the data
        reference, rubbish = self.process_reference(reference)
        logger.info("")

        # Initialise the integrator
        from dials.algorithms.profile_model.factory import ProfileModelFactory
        from dials.algorithms.integration.integrator import IntegratorFactory
        from dials.array_family import flex

        # Modify experiment list if scan range is set.
        experiments, reference = self.split_for_scan_range(
            experiments, reference, params.scan_range)

        # Predict the reflections
        logger.info("")
        logger.info("=" * 80)
        logger.info("")
        logger.info(heading("Predicting reflections"))
        logger.info("")
        predicted = flex.reflection_table.from_predictions_multi(
            experiments,
            dmin=params.prediction.d_min,
            dmax=params.prediction.d_max,
            margin=params.prediction.margin,
            force_static=params.prediction.force_static)

        # Match reference with predicted
        if reference:
            matched, reference, unmatched = predicted.match_with_reference(
                reference)
            assert (len(matched) == len(predicted))
            assert (matched.count(True) <= len(reference))
            if matched.count(True) == 0:
                raise Sorry('''
          Invalid input for reference reflections.
          Zero reference spots were matched to predictions
        ''')
            elif len(unmatched) != 0:
                logger.info('')
                logger.info('*' * 80)
                logger.info(
                    'Warning: %d reference spots were not matched to predictions'
                    % (len(unmatched)))
                logger.info('*' * 80)
                logger.info('')
            rubbish.extend(unmatched)

            if len(experiments) > 1:
                # filter out any experiments without matched reference reflections
                # f_: filtered
                from dxtbx.model.experiment.experiment_list import ExperimentList
                f_reference = flex.reflection_table()
                f_predicted = flex.reflection_table()
                f_rubbish = flex.reflection_table()
                f_experiments = ExperimentList()
                good_expt_count = 0

                def refl_extend(src, dest, eid):
                    tmp = src.select(src['id'] == eid)
                    tmp['id'] = flex.int(len(tmp), good_expt_count)
                    dest.extend(tmp)

                for expt_id, experiment in enumerate(experiments):
                    if len(reference.select(reference['id'] == expt_id)) != 0:
                        refl_extend(reference, f_reference, expt_id)
                        refl_extend(predicted, f_predicted, expt_id)
                        refl_extend(rubbish, f_rubbish, expt_id)
                        f_experiments.append(experiment)
                        good_expt_count += 1
                    else:
                        logger.info(
                            "Removing experiment %d: no reference reflections matched to predictions"
                            % expt_id)

                reference = f_reference
                predicted = f_predicted
                experiments = f_experiments
                rubbish = f_rubbish

        # Select a random sample of the predicted reflections
        if not params.sampling.integrate_all_reflections:
            predicted = self.sample_predictions(experiments, predicted, params)

        # Compute the profile model
        if (params.create_profile_model and reference is not None
                and "shoebox" in reference):
            experiments = ProfileModelFactory.create(params, experiments,
                                                     reference)
        else:
            for expr in experiments:
                if expr.profile is None:
                    raise Sorry('No profile information in experiment list')
                expr.profile.params = params.profile
        del reference

        # Compute the bounding box
        predicted.compute_bbox(experiments)

        # Create the integrator
        logger.info("")
        integrator = IntegratorFactory.create(params, experiments, predicted)

        # Integrate the reflections
        reflections = integrator.integrate()

        # Append rubbish data onto the end
        if rubbish is not None and params.output.include_bad_reference:
            mask = flex.bool(len(rubbish), True)
            rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
            rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
            rubbish.set_flags(mask, rubbish.flags.bad_reference)
            reflections.extend(rubbish)

        # Save the reflections
        self.save_reflections(reflections, params.output.reflections)
        self.save_experiments(experiments, params.output.experiments)

        # Write a report if requested
        if params.output.report is not None:
            integrator.report().as_file(params.output.report)

        # Print the total time taken
        logger.info("\nTotal time taken: %f" % (time() - start_time))