Beispiel #1
0
  def integrate(self):
    '''
    Integrate the data

    '''
    from dials.algorithms.integration.report import IntegrationReport
    from dials.algorithms.integration.report import ProfileModelReport
    from dials.algorithms.integration.report import ProfileValidationReport
    from dials.util.command_line import heading
    from logging import info, debug
    from dials.util import pprint
    from random import shuffle, seed
    from math import floor, ceil
    from dials.array_family import flex
    from dials.algorithms.profile_model.modeller import MultiExpProfileModeller
    from dials.algorithms.integration.validation import ValidatedMultiExpProfileModeller

    # Ensure we get the same random sample each time
    seed(0)

    # Init the report
    self.profile_model_report = None
    self.integration_report = None

    # Heading
    info("=" * 80)
    info("")
    info(heading("Processing reflections"))
    info("")

    # Create summary format
    fmt = (
      ' Processing the following experiments:\n'
      '\n'
      ' Experiments: %d\n'
      ' Beams:       %d\n'
      ' Detectors:   %d\n'
      ' Goniometers: %d\n'
      ' Scans:       %d\n'
      ' Crystals:    %d\n'
      ' Imagesets:   %d\n'
    )

    # Print the summary
    info(fmt % (
      len(self.experiments),
      len(self.experiments.beams()),
      len(self.experiments.detectors()),
      len(self.experiments.goniometers()),
      len(self.experiments.scans()),
      len(self.experiments.crystals()),
      len(self.experiments.imagesets())))

    # Initialize the reflections
    initialize = self.InitializerClass(
      self.experiments,
      self.params)
    initialize(self.reflections)

    # Check if we want to do some profile fitting
    fitting_class = [e.profile.fitting_class() for e in self.experiments]
    fitting_avail = all([c is not None for c in fitting_class])
    if self.params.profile.fitting and fitting_avail:
      profile_fitting = True
      profile_fitter = None
    else:
      profile_fitting = False
      profile_fitter = None

    # Do profile modelling
    if profile_fitting:

      info("=" * 80)
      info("")
      info(heading("Modelling reflection profiles"))
      info("")

      # Get the selection
      selection = self.reflections.get_flags(
        self.reflections.flags.reference_spot)

      # Get the reference spots
      reference = self.reflections.select(selection)

      # Check if we need to skip
      if len(reference) == 0:
        info("** Skipping profile modelling - no reference profiles given **")
      else:

        # Try to set up the validation
        if self.params.profile.validation.number_of_partitions > 1:
          n = len(reference)
          k_max = int(floor(n / self.params.profile.validation.min_partition_size))
          if k_max < self.params.profile.validation.number_of_partitions:
            num_folds = k_max
          else:
            num_folds = self.params.profile.validation.number_of_partitions
          if num_folds > 1:
            indices = (list(range(num_folds)) * int(ceil(n/num_folds)))[0:n]
            shuffle(indices)
            reference['profile.index'] = flex.size_t(indices)
          if num_folds < 1:
            num_folds = 1
        else:
          num_folds = 1

        # Create the profile fitter
        profile_fitter = ValidatedMultiExpProfileModeller()
        for i in range(num_folds):
          profile_fitter_single = MultiExpProfileModeller()#(num_folds)
          for expr in self.experiments:
            profile_fitter_single.add(expr.profile.fitting_class()(expr))
          profile_fitter.add(profile_fitter_single)

        # Create the data processor
        executor = ProfileModellerExecutor(
          self.experiments,
          profile_fitter)
        processor = ProcessorBuilder(
          self.ProcessorClass,
          self.experiments,
          reference,
          self.params.modelling).build()
        processor.executor = executor

        # Process the reference profiles
        reference, profile_fitter_list, time_info = processor.process()

        # Set the reference spots info
        #self.reflections.set_selected(selection, reference)

        # Finalize the profile models for validation
        assert len(profile_fitter_list) > 0, "No profile fitters"
        profile_fitter = None
        for index, pf in profile_fitter_list.iteritems():
          if pf is None:
            continue
          if profile_fitter is None:
            profile_fitter = pf
          else:
            profile_fitter.accumulate(pf)
        profile_fitter.finalize()

        # Get the finalized modeller
        finalized_profile_fitter = profile_fitter.finalized_model()

        # Print profiles
        if self.params.debug_reference_output:
          reference_debug = []
          for i in range(len(finalized_profile_fitter)):
            m = finalized_profile_fitter[i]
            p = []
            for j in range(len(m)):
              try:
                p.append(m.data(j))
              except Exception:
                p.append(None)
          reference_debug.append(p)
          with open("reference_profiles.pickle", "wb") as outfile:
            import cPickle as pickle
            pickle.dump(reference_debug, outfile)

        for i in range(len(finalized_profile_fitter)):
          m = finalized_profile_fitter[i]
          debug("")
          debug("Profiles for experiment %d" % i)
          for j in range(len(m)):
            debug("Profile %d" % j)
            try:
              debug(pprint.profile3d(m.data(j)))
            except Exception:
              debug("** NO PROFILE **")

        # Print the modeller report
        self.profile_model_report = ProfileModelReport(
          self.experiments,
          finalized_profile_fitter,
          reference)
        info("")
        info(self.profile_model_report.as_str(prefix=' '))

        # Print the time info
        info("")
        info(str(time_info))
        info("")

        # If we have more than 1 fold then do the validation
        if num_folds > 1:

          # Create the data processor
          executor = ProfileValidatorExecutor(
            self.experiments,
            profile_fitter)
          processor = ProcessorBuilder(
            self.ProcessorClass,
            self.experiments,
            reference,
            self.params.modelling).build()
          processor.executor = executor

          # Process the reference profiles
          reference, validation, time_info = processor.process()

          # Print the modeller report
          self.profile_validation_report = ProfileValidationReport(
            self.experiments,
            profile_fitter,
            reference,
            num_folds)
          info("")
          info(self.profile_validation_report.as_str(prefix=' '))

          # Print the time info
          info("")
          info(str(time_info))
          info("")

        # Set to the finalized fitter
        profile_fitter = finalized_profile_fitter

    info("=" * 80)
    info("")
    info(heading("Integrating reflections"))
    info("")

    # Create the data processor
    executor = IntegratorExecutor(
      self.experiments,
      profile_fitter)
    processor = ProcessorBuilder(
      self.ProcessorClass,
      self.experiments,
      self.reflections,
      self.params.integration).build()
    processor.executor = executor

    # Process the reflections
    self.reflections, _, time_info = processor.process()

    # Finalize the reflections
    finalize = self.FinalizerClass(
      self.experiments,
      self.params)
    finalize(self.reflections)

    # Create the integration report
    self.integration_report = IntegrationReport(
      self.experiments,
      self.reflections)
    info("")
    info(self.integration_report.as_str(prefix=' '))

    # Print the time info
    info(str(time_info))
    info("")

    # Return the reflections
    return self.reflections
Beispiel #2
0
    def integrate(self):
        '''
    Integrate the data

    '''
        from dials.algorithms.integration.report import IntegrationReport
        from dials.algorithms.integration.report import ProfileModelReport
        from dials.algorithms.integration.report import ProfileValidationReport
        from dials.util.command_line import heading
        from dials.util import pprint
        from random import shuffle, seed
        from math import floor, ceil
        from dials.array_family import flex
        from dials.algorithms.profile_model.modeller import MultiExpProfileModeller
        from dials.algorithms.integration.validation import ValidatedMultiExpProfileModeller

        # Ensure we get the same random sample each time
        seed(0)

        # Init the report
        self.profile_model_report = None
        self.integration_report = None

        # Heading
        logger.info("=" * 80)
        logger.info("")
        logger.info(heading("Processing reflections"))
        logger.info("")

        # Create summary format
        fmt = (' Processing the following experiments:\n'
               '\n'
               ' Experiments: %d\n'
               ' Beams:       %d\n'
               ' Detectors:   %d\n'
               ' Goniometers: %d\n'
               ' Scans:       %d\n'
               ' Crystals:    %d\n'
               ' Imagesets:   %d\n')

        # Print the summary
        logger.info(
            fmt %
            (len(self.experiments), len(
                self.experiments.beams()), len(self.experiments.detectors()),
             len(self.experiments.goniometers()), len(
                 self.experiments.scans()), len(self.experiments.crystals()),
             len(self.experiments.imagesets())))

        # Initialize the reflections
        initialize = self.InitializerClass(self.experiments, self.params)
        initialize(self.reflections)

        # Check if we want to do some profile fitting
        fitting_class = [e.profile.fitting_class() for e in self.experiments]
        fitting_avail = all(c is not None for c in fitting_class)
        if self.params.profile.fitting and fitting_avail:
            profile_fitting = True
            profile_fitter = None
        else:
            profile_fitting = False
            profile_fitter = None

        # Do profile modelling
        if profile_fitting:

            logger.info("=" * 80)
            logger.info("")
            logger.info(heading("Modelling reflection profiles"))
            logger.info("")

            # Get the selection
            selection = self.reflections.get_flags(
                self.reflections.flags.reference_spot)

            # Get the reference spots
            reference = self.reflections.select(selection)

            # Check if we need to skip
            if len(reference) == 0:
                logger.info(
                    "** Skipping profile modelling - no reference profiles given **"
                )
            else:

                # Try to set up the validation
                if self.params.profile.validation.number_of_partitions > 1:
                    n = len(reference)
                    k_max = int(
                        floor(
                            n /
                            self.params.profile.validation.min_partition_size))
                    if k_max < self.params.profile.validation.number_of_partitions:
                        num_folds = k_max
                    else:
                        num_folds = self.params.profile.validation.number_of_partitions
                    if num_folds > 1:
                        indices = (list(range(num_folds)) *
                                   int(ceil(n / num_folds)))[0:n]
                        shuffle(indices)
                        reference['profile.index'] = flex.size_t(indices)
                    if num_folds < 1:
                        num_folds = 1
                else:
                    num_folds = 1

                # Create the profile fitter
                profile_fitter = ValidatedMultiExpProfileModeller()
                for i in range(num_folds):
                    profile_fitter_single = MultiExpProfileModeller(
                    )  #(num_folds)
                    for expr in self.experiments:
                        profile_fitter_single.add(
                            expr.profile.fitting_class()(expr))
                    profile_fitter.add(profile_fitter_single)

                # Create the data processor
                executor = ProfileModellerExecutor(self.experiments,
                                                   profile_fitter)
                processor = ProcessorBuilder(self.ProcessorClass,
                                             self.experiments, reference,
                                             self.params.modelling).build()
                processor.executor = executor

                # Process the reference profiles
                reference, profile_fitter_list, time_info = processor.process()

                # Set the reference spots info
                #self.reflections.set_selected(selection, reference)

                # Finalize the profile models for validation
                assert len(profile_fitter_list) > 0, "No profile fitters"
                profile_fitter = None
                for index, pf in profile_fitter_list.iteritems():
                    if pf is None:
                        continue
                    if profile_fitter is None:
                        profile_fitter = pf
                    else:
                        profile_fitter.accumulate(pf)
                profile_fitter.finalize()

                # Get the finalized modeller
                finalized_profile_fitter = profile_fitter.finalized_model()

                # Print profiles
                if self.params.debug_reference_output:
                    reference_debug = []
                    for i in range(len(finalized_profile_fitter)):
                        m = finalized_profile_fitter[i]
                        p = []
                        for j in range(len(m)):
                            try:
                                p.append(m.data(j))
                            except Exception:
                                p.append(None)
                    reference_debug.append(p)
                    with open("reference_profiles.pickle", "wb") as outfile:
                        import cPickle as pickle
                        pickle.dump(reference_debug, outfile)

                for i in range(len(finalized_profile_fitter)):
                    m = finalized_profile_fitter[i]
                    logger.debug("")
                    logger.debug("Profiles for experiment %d" % i)
                    for j in range(len(m)):
                        logger.debug("Profile %d" % j)
                        try:
                            logger.debug(pprint.profile3d(m.data(j)))
                        except Exception:
                            logger.debug("** NO PROFILE **")

                # Print the modeller report
                self.profile_model_report = ProfileModelReport(
                    self.experiments, finalized_profile_fitter, reference)
                logger.info("")
                logger.info(self.profile_model_report.as_str(prefix=' '))

                # Print the time info
                logger.info("")
                logger.info(str(time_info))
                logger.info("")

                # If we have more than 1 fold then do the validation
                if num_folds > 1:

                    # Create the data processor
                    executor = ProfileValidatorExecutor(
                        self.experiments, profile_fitter)
                    processor = ProcessorBuilder(
                        self.ProcessorClass, self.experiments, reference,
                        self.params.modelling).build()
                    processor.executor = executor

                    # Process the reference profiles
                    reference, validation, time_info = processor.process()

                    # Print the modeller report
                    self.profile_validation_report = ProfileValidationReport(
                        self.experiments, profile_fitter, reference, num_folds)
                    logger.info("")
                    logger.info(
                        self.profile_validation_report.as_str(prefix=' '))

                    # Print the time info
                    logger.info("")
                    logger.info(str(time_info))
                    logger.info("")

                # Set to the finalized fitter
                profile_fitter = finalized_profile_fitter

        logger.info("=" * 80)
        logger.info("")
        logger.info(heading("Integrating reflections"))
        logger.info("")

        # Create the data processor
        executor = IntegratorExecutor(self.experiments, profile_fitter)
        processor = ProcessorBuilder(self.ProcessorClass, self.experiments,
                                     self.reflections,
                                     self.params.integration).build()
        processor.executor = executor

        # Process the reflections
        self.reflections, _, time_info = processor.process()

        # Finalize the reflections
        finalize = self.FinalizerClass(self.experiments, self.params)
        finalize(self.reflections)

        # Create the integration report
        self.integration_report = IntegrationReport(self.experiments,
                                                    self.reflections)
        logger.info("")
        logger.info(self.integration_report.as_str(prefix=' '))

        # Print the time info
        logger.info(str(time_info))
        logger.info("")

        # Return the reflections
        return self.reflections
Beispiel #3
0
class Integrator(object):
  '''
  The integrator class

  '''

  def __init__(self,
               experiments,
               reflections,
               params):
    '''
    Initialize the integrator

    :param experiments: The experiment list
    :param reflections: The reflections to process
    :param params: The parameters to use

    '''

    # Save some stuff
    self.experiments = experiments
    self.reflections = reflections
    self.params = params
    self.profile_model_report = None
    self.integration_report = None

  def integrate(self):
    '''
    Integrate the data

    '''
    from dials.algorithms.integration.report import IntegrationReport
    from dials.algorithms.integration.report import ProfileModelReport
    from dials.algorithms.integration.report import ProfileValidationReport
    from dials.util.command_line import heading
    from logging import info, debug
    from dials.util import pprint
    from random import shuffle, seed
    from math import floor, ceil
    from dials.array_family import flex
    from dials.algorithms.profile_model.modeller import MultiExpProfileModeller
    from dials.algorithms.integration.validation import ValidatedMultiExpProfileModeller

    # Ensure we get the same random sample each time
    seed(0)

    # Init the report
    self.profile_model_report = None
    self.integration_report = None

    # Heading
    info("=" * 80)
    info("")
    info(heading("Processing reflections"))
    info("")

    # Create summary format
    fmt = (
      ' Processing the following experiments:\n'
      '\n'
      ' Experiments: %d\n'
      ' Beams:       %d\n'
      ' Detectors:   %d\n'
      ' Goniometers: %d\n'
      ' Scans:       %d\n'
      ' Crystals:    %d\n'
      ' Imagesets:   %d\n'
    )

    # Print the summary
    info(fmt % (
      len(self.experiments),
      len(self.experiments.beams()),
      len(self.experiments.detectors()),
      len(self.experiments.goniometers()),
      len(self.experiments.scans()),
      len(self.experiments.crystals()),
      len(self.experiments.imagesets())))

    # Initialize the reflections
    initialize = self.InitializerClass(
      self.experiments,
      self.params)
    initialize(self.reflections)

    # Check if we want to do some profile fitting
    fitting_class = [e.profile.fitting_class() for e in self.experiments]
    fitting_avail = all([c is not None for c in fitting_class])
    if self.params.profile.fitting and fitting_avail:
      profile_fitting = True
      profile_fitter = None
    else:
      profile_fitting = False
      profile_fitter = None

    # Do profile modelling
    if profile_fitting:

      info("=" * 80)
      info("")
      info(heading("Modelling reflection profiles"))
      info("")

      # Get the selection
      selection = self.reflections.get_flags(
        self.reflections.flags.reference_spot)

      # Get the reference spots
      reference = self.reflections.select(selection)

      # Check if we need to skip
      if len(reference) == 0:
        info("** Skipping profile modelling - no reference profiles given **")
      else:

        # Try to set up the validation
        if self.params.profile.validation.number_of_partitions > 1:
          n = len(reference)
          k_max = int(floor(n / self.params.profile.validation.min_partition_size))
          if k_max < self.params.profile.validation.number_of_partitions:
            num_folds = k_max
          else:
            num_folds = self.params.profile.validation.number_of_partitions
          if num_folds > 1:
            indices = (list(range(num_folds)) * int(ceil(n/num_folds)))[0:n]
            shuffle(indices)
            reference['profile.index'] = flex.size_t(indices)
          if num_folds < 1:
            num_folds = 1
        else:
          num_folds = 1

        # Create the profile fitter
        profile_fitter = ValidatedMultiExpProfileModeller()
        for i in range(num_folds):
          profile_fitter_single = MultiExpProfileModeller()#(num_folds)
          for expr in self.experiments:
            profile_fitter_single.add(expr.profile.fitting_class()(expr))
          profile_fitter.add(profile_fitter_single)

        # Create the data processor
        executor = ProfileModellerExecutor(
          self.experiments,
          profile_fitter)
        processor = ProcessorBuilder(
          self.ProcessorClass,
          self.experiments,
          reference,
          self.params.modelling).build()
        processor.executor = executor

        # Process the reference profiles
        reference, profile_fitter_list, time_info = processor.process()

        # Set the reference spots info
        #self.reflections.set_selected(selection, reference)

        # Finalize the profile models for validation
        assert len(profile_fitter_list) > 0, "No profile fitters"
        profile_fitter = None
        for index, pf in profile_fitter_list.iteritems():
          if pf is None:
            continue
          if profile_fitter is None:
            profile_fitter = pf
          else:
            profile_fitter.accumulate(pf)
        profile_fitter.finalize()

        # Get the finalized modeller
        finalized_profile_fitter = profile_fitter.finalized_model()

        # Print profiles
        if self.params.debug_reference_output:
          reference_debug = []
          for i in range(len(finalized_profile_fitter)):
            m = finalized_profile_fitter[i]
            p = []
            for j in range(len(m)):
              try:
                p.append(m.data(j))
              except Exception:
                p.append(None)
          reference_debug.append(p)
          with open("reference_profiles.pickle", "wb") as outfile:
            import cPickle as pickle
            pickle.dump(reference_debug, outfile)

        for i in range(len(finalized_profile_fitter)):
          m = finalized_profile_fitter[i]
          debug("")
          debug("Profiles for experiment %d" % i)
          for j in range(len(m)):
            debug("Profile %d" % j)
            try:
              debug(pprint.profile3d(m.data(j)))
            except Exception:
              debug("** NO PROFILE **")

        # Print the modeller report
        self.profile_model_report = ProfileModelReport(
          self.experiments,
          finalized_profile_fitter,
          reference)
        info("")
        info(self.profile_model_report.as_str(prefix=' '))

        # Print the time info
        info("")
        info(str(time_info))
        info("")

        # If we have more than 1 fold then do the validation
        if num_folds > 1:

          # Create the data processor
          executor = ProfileValidatorExecutor(
            self.experiments,
            profile_fitter)
          processor = ProcessorBuilder(
            self.ProcessorClass,
            self.experiments,
            reference,
            self.params.modelling).build()
          processor.executor = executor

          # Process the reference profiles
          reference, validation, time_info = processor.process()

          # Print the modeller report
          self.profile_validation_report = ProfileValidationReport(
            self.experiments,
            profile_fitter,
            reference,
            num_folds)
          info("")
          info(self.profile_validation_report.as_str(prefix=' '))

          # Print the time info
          info("")
          info(str(time_info))
          info("")

        # Set to the finalized fitter
        profile_fitter = finalized_profile_fitter

    info("=" * 80)
    info("")
    info(heading("Integrating reflections"))
    info("")

    # Create the data processor
    executor = IntegratorExecutor(
      self.experiments,
      profile_fitter)
    processor = ProcessorBuilder(
      self.ProcessorClass,
      self.experiments,
      self.reflections,
      self.params.integration).build()
    processor.executor = executor

    # Process the reflections
    self.reflections, _, time_info = processor.process()

    # Finalize the reflections
    finalize = self.FinalizerClass(
      self.experiments,
      self.params)
    finalize(self.reflections)

    # Create the integration report
    self.integration_report = IntegrationReport(
      self.experiments,
      self.reflections)
    info("")
    info(self.integration_report.as_str(prefix=' '))

    # Print the time info
    info(str(time_info))
    info("")

    # Return the reflections
    return self.reflections

  def report(self):
    '''
    Return the report of the processing

    '''
    from dials.util.report import Report
    result = Report()
    if self.profile_model_report is not None:
      result.combine(self.profile_model_report)
    result.combine(self.integration_report)
    return result

  def summary(self, block_size, block_size_units):
    ''' Print a summary of the integration stuff. '''
    from libtbx.table_utils import format as table
    from dials.util.command_line import heading
    from logging import info

    # Compute the task table
    if self._experiments.all_stills():
      rows = [["#", "Group", "Frame From", "Frame To"]]
      for i in range(len(self)):
        job = self._manager.job(i)
        group = job.index()
        f0, f1 = job.frames()
        rows.append([str(i), str(group), str(f0), str(f1)])
    elif self._experiments.all_sweeps():
      rows = [["#", "Group", "Frame From", "Frame To", "Angle From", "Angle To"]]
      for i in range(len(self)):
        job = self._manager.job(i)
        group = job.index()
        expr = job.expr()
        f0, f1 = job.frames()
        scan = self._experiments[expr[0]].scan
        p0 = scan.get_angle_from_array_index(f0)
        p1 = scan.get_angle_from_array_index(f1)
        rows.append([str(i), str(group), str(f0), str(f1), str(p0), str(p1)])
    else:
      raise RuntimeError('Experiments must be all sweeps or all stills')
    task_table = table(rows, has_header=True, justify="right", prefix=" ")
Beispiel #4
0
class Integrator(object):
    '''
  The integrator class

  '''
    def __init__(self, experiments, reflections, params):
        '''
    Initialize the integrator

    :param experiments: The experiment list
    :param reflections: The reflections to process
    :param params: The parameters to use

    '''

        # Save some stuff
        self.experiments = experiments
        self.reflections = reflections
        self.params = params
        self.profile_model_report = None
        self.integration_report = None

    def integrate(self):
        '''
    Integrate the data

    '''
        from dials.algorithms.integration.report import IntegrationReport
        from dials.algorithms.integration.report import ProfileModelReport
        from dials.algorithms.integration.report import ProfileValidationReport
        from dials.util.command_line import heading
        from dials.util import pprint
        from random import shuffle, seed
        from math import floor, ceil
        from dials.array_family import flex
        from dials.algorithms.profile_model.modeller import MultiExpProfileModeller
        from dials.algorithms.integration.validation import ValidatedMultiExpProfileModeller

        # Ensure we get the same random sample each time
        seed(0)

        # Init the report
        self.profile_model_report = None
        self.integration_report = None

        # Heading
        logger.info("=" * 80)
        logger.info("")
        logger.info(heading("Processing reflections"))
        logger.info("")

        # Create summary format
        fmt = (' Processing the following experiments:\n'
               '\n'
               ' Experiments: %d\n'
               ' Beams:       %d\n'
               ' Detectors:   %d\n'
               ' Goniometers: %d\n'
               ' Scans:       %d\n'
               ' Crystals:    %d\n'
               ' Imagesets:   %d\n')

        # Print the summary
        logger.info(
            fmt %
            (len(self.experiments), len(
                self.experiments.beams()), len(self.experiments.detectors()),
             len(self.experiments.goniometers()), len(
                 self.experiments.scans()), len(self.experiments.crystals()),
             len(self.experiments.imagesets())))

        # Initialize the reflections
        initialize = self.InitializerClass(self.experiments, self.params)
        initialize(self.reflections)

        # Check if we want to do some profile fitting
        fitting_class = [e.profile.fitting_class() for e in self.experiments]
        fitting_avail = all(c is not None for c in fitting_class)
        if self.params.profile.fitting and fitting_avail:
            profile_fitting = True
            profile_fitter = None
        else:
            profile_fitting = False
            profile_fitter = None

        # Do profile modelling
        if profile_fitting:

            logger.info("=" * 80)
            logger.info("")
            logger.info(heading("Modelling reflection profiles"))
            logger.info("")

            # Get the selection
            selection = self.reflections.get_flags(
                self.reflections.flags.reference_spot)

            # Get the reference spots
            reference = self.reflections.select(selection)

            # Check if we need to skip
            if len(reference) == 0:
                logger.info(
                    "** Skipping profile modelling - no reference profiles given **"
                )
            else:

                # Try to set up the validation
                if self.params.profile.validation.number_of_partitions > 1:
                    n = len(reference)
                    k_max = int(
                        floor(
                            n /
                            self.params.profile.validation.min_partition_size))
                    if k_max < self.params.profile.validation.number_of_partitions:
                        num_folds = k_max
                    else:
                        num_folds = self.params.profile.validation.number_of_partitions
                    if num_folds > 1:
                        indices = (list(range(num_folds)) *
                                   int(ceil(n / num_folds)))[0:n]
                        shuffle(indices)
                        reference['profile.index'] = flex.size_t(indices)
                    if num_folds < 1:
                        num_folds = 1
                else:
                    num_folds = 1

                # Create the profile fitter
                profile_fitter = ValidatedMultiExpProfileModeller()
                for i in range(num_folds):
                    profile_fitter_single = MultiExpProfileModeller(
                    )  #(num_folds)
                    for expr in self.experiments:
                        profile_fitter_single.add(
                            expr.profile.fitting_class()(expr))
                    profile_fitter.add(profile_fitter_single)

                # Create the data processor
                executor = ProfileModellerExecutor(self.experiments,
                                                   profile_fitter)
                processor = ProcessorBuilder(self.ProcessorClass,
                                             self.experiments, reference,
                                             self.params.modelling).build()
                processor.executor = executor

                # Process the reference profiles
                reference, profile_fitter_list, time_info = processor.process()

                # Set the reference spots info
                #self.reflections.set_selected(selection, reference)

                # Finalize the profile models for validation
                assert len(profile_fitter_list) > 0, "No profile fitters"
                profile_fitter = None
                for index, pf in profile_fitter_list.iteritems():
                    if pf is None:
                        continue
                    if profile_fitter is None:
                        profile_fitter = pf
                    else:
                        profile_fitter.accumulate(pf)
                profile_fitter.finalize()

                # Get the finalized modeller
                finalized_profile_fitter = profile_fitter.finalized_model()

                # Print profiles
                if self.params.debug_reference_output:
                    reference_debug = []
                    for i in range(len(finalized_profile_fitter)):
                        m = finalized_profile_fitter[i]
                        p = []
                        for j in range(len(m)):
                            try:
                                p.append(m.data(j))
                            except Exception:
                                p.append(None)
                    reference_debug.append(p)
                    with open("reference_profiles.pickle", "wb") as outfile:
                        import cPickle as pickle
                        pickle.dump(reference_debug, outfile)

                for i in range(len(finalized_profile_fitter)):
                    m = finalized_profile_fitter[i]
                    logger.debug("")
                    logger.debug("Profiles for experiment %d" % i)
                    for j in range(len(m)):
                        logger.debug("Profile %d" % j)
                        try:
                            logger.debug(pprint.profile3d(m.data(j)))
                        except Exception:
                            logger.debug("** NO PROFILE **")

                # Print the modeller report
                self.profile_model_report = ProfileModelReport(
                    self.experiments, finalized_profile_fitter, reference)
                logger.info("")
                logger.info(self.profile_model_report.as_str(prefix=' '))

                # Print the time info
                logger.info("")
                logger.info(str(time_info))
                logger.info("")

                # If we have more than 1 fold then do the validation
                if num_folds > 1:

                    # Create the data processor
                    executor = ProfileValidatorExecutor(
                        self.experiments, profile_fitter)
                    processor = ProcessorBuilder(
                        self.ProcessorClass, self.experiments, reference,
                        self.params.modelling).build()
                    processor.executor = executor

                    # Process the reference profiles
                    reference, validation, time_info = processor.process()

                    # Print the modeller report
                    self.profile_validation_report = ProfileValidationReport(
                        self.experiments, profile_fitter, reference, num_folds)
                    logger.info("")
                    logger.info(
                        self.profile_validation_report.as_str(prefix=' '))

                    # Print the time info
                    logger.info("")
                    logger.info(str(time_info))
                    logger.info("")

                # Set to the finalized fitter
                profile_fitter = finalized_profile_fitter

        logger.info("=" * 80)
        logger.info("")
        logger.info(heading("Integrating reflections"))
        logger.info("")

        # Create the data processor
        executor = IntegratorExecutor(self.experiments, profile_fitter)
        processor = ProcessorBuilder(self.ProcessorClass, self.experiments,
                                     self.reflections,
                                     self.params.integration).build()
        processor.executor = executor

        # Process the reflections
        self.reflections, _, time_info = processor.process()

        # Finalize the reflections
        finalize = self.FinalizerClass(self.experiments, self.params)
        finalize(self.reflections)

        # Create the integration report
        self.integration_report = IntegrationReport(self.experiments,
                                                    self.reflections)
        logger.info("")
        logger.info(self.integration_report.as_str(prefix=' '))

        # Print the time info
        logger.info(str(time_info))
        logger.info("")

        # Return the reflections
        return self.reflections

    def report(self):
        '''
    Return the report of the processing

    '''
        from dials.util.report import Report
        result = Report()
        if self.profile_model_report is not None:
            result.combine(self.profile_model_report)
        result.combine(self.integration_report)
        return result

    def summary(self, block_size, block_size_units):
        ''' Print a summary of the integration stuff. '''
        from libtbx.table_utils import format as table
        from dials.util.command_line import heading

        # Compute the task table
        if self._experiments.all_stills():
            rows = [["#", "Group", "Frame From", "Frame To"]]
            for i in range(len(self)):
                job = self._manager.job(i)
                group = job.index()
                f0, f1 = job.frames()
                rows.append([str(i), str(group), str(f0), str(f1)])
        elif self._experiments.all_sweeps():
            rows = [[
                "#", "Group", "Frame From", "Frame To", "Angle From",
                "Angle To"
            ]]
            for i in range(len(self)):
                job = self._manager.job(i)
                group = job.index()
                expr = job.expr()
                f0, f1 = job.frames()
                scan = self._experiments[expr[0]].scan
                p0 = scan.get_angle_from_array_index(f0)
                p1 = scan.get_angle_from_array_index(f1)
                rows.append(
                    [str(i),
                     str(group),
                     str(f0),
                     str(f1),
                     str(p0),
                     str(p1)])
        else:
            raise RuntimeError('Experiments must be all sweeps or all stills')
        task_table = table(rows, has_header=True, justify="right", prefix=" ")