Beispiel #1
0
  def create(params, experiments, reflections):
    ''' Create the integrator from the input configuration. '''
    from dials.interfaces import IntensityIface
    from dials.interfaces import BackgroundIface
    from dials.interfaces import CentroidIface
    from dials.array_family import flex

    # Initialise the strategy classes
    BackgroundAlgorithm = BackgroundIface.extension(
      params.integration.background.algorithm)
    IntensityAlgorithm = IntensityIface.extension(
      params.integration.intensity.algorithm)
    CentroidAlgorithm = CentroidIface.extension(
      params.integration.centroid.algorithm)

    # Set the algorithms in the reflection table
    flex.reflection_table._background_algorithm = \
      flex.strategy(BackgroundAlgorithm, params)
    flex.reflection_table._intensity_algorithm = \
      flex.strategy(IntensityAlgorithm, params)
    flex.reflection_table._centroid_algorithm = \
      flex.strategy(CentroidAlgorithm, params)

    # Get the integrator class
    IntegratorClass = IntegratorFactory.select_integrator(IntensityAlgorithm)

    # Return an instantiation of the class
    return IntegratorClass(
      experiments=experiments,
      reflections=reflections,
      block_size=params.integration.block.size,
      max_procs=params.integration.mp.max_procs,
      mp_method=params.integration.mp.method)
Beispiel #2
0
    def tst_after_import_extensions(self):
        import dials.extensions  # import dependency
        from dials.interfaces import ProfileModelIface
        from dials.interfaces import SpotFinderThresholdIface
        from dials.interfaces import CentroidIface
        from dials.interfaces import BackgroundIface

        # Should have four interfaces
        interfaces = list(Interface.interfaces())
        assert (len(interfaces) == 4)

        # Check we have the expected number of extensions for each interface
        extensions = list(ProfileModelIface.extensions())
        assert (len(extensions) > 0)
        extensions = list(SpotFinderThresholdIface.extensions())
        assert (len(extensions) > 0)
        extensions = list(CentroidIface.extensions())
        assert (len(extensions) > 0)
        extensions = list(BackgroundIface.extensions())
        assert (len(extensions) > 0)

        # Check the interface contain the expected extensions
        from dials.extensions import GaussianRSProfileModelExt
        from dials.extensions import KabschSpotFinderThresholdExt
        from dials.extensions import SimpleCentroidExt
        from dials.extensions import NullBackgroundExt
        from dials.extensions import SimpleBackgroundExt

        extensions = list(ProfileModelIface.extensions())
        assert (GaussianRSProfileModelExt in extensions)
        extensions = list(SpotFinderThresholdIface.extensions())
        assert (KabschSpotFinderThresholdExt in extensions)
        extensions = list(CentroidIface.extensions())
        assert (SimpleCentroidExt in extensions)
        extensions = list(BackgroundIface.extensions())
        assert (NullBackgroundExt in extensions)
        assert (SimpleBackgroundExt in extensions)

        # Test passed
        print 'OK'
Beispiel #3
0
    def create(params, experiments, reflections):
        """Create the integrator from the input configuration."""
        from dials.interfaces import IntensityIface
        from dials.interfaces import BackgroundIface
        from dials.interfaces import CentroidIface
        from dials.array_family import flex

        # Initialise the strategy classes
        BackgroundAlgorithm = BackgroundIface.extension(
            params.integration.background.algorithm
        )
        IntensityAlgorithm = IntensityIface.extension(
            params.integration.intensity.algorithm
        )
        CentroidAlgorithm = CentroidIface.extension(
            params.integration.centroid.algorithm
        )

        # Set the algorithms in the reflection table
        flex.reflection_table._background_algorithm = flex.strategy(
            BackgroundAlgorithm, params
        )
        flex.reflection_table._intensity_algorithm = flex.strategy(
            IntensityAlgorithm, params
        )
        flex.reflection_table._centroid_algorithm = flex.strategy(
            CentroidAlgorithm, params
        )

        # Get the integrator class
        IntegratorClass = IntegratorFactory.select_integrator(IntensityAlgorithm)

        # Return an instantiation of the class
        return IntegratorClass(
            experiments=experiments,
            reflections=reflections,
            block_size=params.integration.block.size,
            max_procs=params.integration.mp.max_procs,
            mp_method=params.integration.mp.method,
        )
Beispiel #4
0
def generate_phil_scope():
  ''' Generate the phil scope. '''
  from dials.interfaces import BackgroundIface
  from dials.interfaces import IntensityIface
  from dials.interfaces import CentroidIface

  phil_scope = phil.parse('''

    integration {

      include scope dials.data.lookup.phil_scope

      mp {
        method = *multiprocessing sge lsf pbs
          .type = choice
          .help = "The multiprocessing method to use"

        max_procs = 1
          .type = int(value_min=1)
          .help = "The number of processes to use."
      }

      block {
        size = 10
          .type = float
          .help = "The block size in rotation angle (degrees)."
      }

      shoebox {
        n_sigma = 3
          .help = "The number of standard deviations of the beam divergence and the"
                  "mosaicity to use for the bounding box size."
          .type = float

        sigma_b = None
          .help = "The E.S.D. of the beam divergence"
          .type = float

        sigma_m = None
          .help = "The E.S.D. of the reflecting range"
          .type = float
      }

      filter {

        min_zeta = 0.05
          .help = "Filter the reflections by the value of zeta. A value of less"
                  "than or equal to zero indicates that this will not be used. A"
                  "positive value is used as the minimum permissable value."
          .type = float

        ice_rings {
          filter = False
            .type = bool
          unit_cell = 4.498,4.498,7.338,90,90,120
            .type = unit_cell
            .help = "The unit cell to generate d_spacings for ice rings."
          space_group = 194
            .type = space_group
            .help = "The space group used to generate d_spacings for ice rings."
          d_min = 0
            .type = int(value_min=0)
            .help = "The minimum resolution to filter ice rings"
          width = 0.06
            .type = float(value_min=0.0)
            .help = "The width of an ice ring (in d-spacing)."
        }
      }
    }
  ''', process_includes=True)
  main_scope = phil_scope.get_without_substitution("integration")
  assert(len(main_scope) == 1)
  main_scope = main_scope[0]
  main_scope.adopt_scope(BackgroundIface.phil_scope())
  main_scope.adopt_scope(IntensityIface.phil_scope())
  main_scope.adopt_scope(CentroidIface.phil_scope())
  return phil_scope
Beispiel #5
0
def generate_phil_scope():
  '''
  Generate the integration phil scope.

  :return: The phil scope

  '''
  import dials.extensions
  from dials.interfaces import BackgroundIface
  from dials.interfaces import CentroidIface

  phil_scope = phil.parse('''

    integration {

      include scope dials.data.multiprocessing.phil_scope
      include scope dials.data.lookup.phil_scope

      block {

        size = auto
          .type = float
          .help = "The block size in rotation angle (degrees)."

        units = *degrees radians frames
          .type = choice
          .help = "The units of the block size"

        threshold = 0.99
          .type = float(value_min=0.0, value_max=1.0)
          .help = "For block size auto the block size is calculated by sorting"
                  "reflections by the number of frames they cover and then"
                  "selecting the block size to be 2*nframes[threshold] such"
                  "that 100*threshold % of reflections are guarenteed to be"
                  "fully contained in 1 block"

        force = False
          .type = bool
          .help = "If the number of processors is 1 and force is False, then the"
                  "number of blocks may be set to 1. If force is True then the"
                  "block size is always calculated."

        max_memory_usage = 0.75
          .type = float(value_min=0.0,value_max=1.0)
          .help = "The maximum percentage of total physical memory to use for"
                  "allocating shoebox arrays."
      }

      debug {

        reference {
          output = False
            .type = bool
            .help = "Save the reference profiles"
        }

        during = modelling *integration
          .type = choice
          .help = "Do debugging during modelling or integration"

        output = False
          .type = bool
          .help = "Save shoeboxes after each processing task."

        separate_files = True
          .type = bool
          .help = "If this is true, the shoeboxes are saved in separate files"
                  "from the output integrated.pickle file. This is necessary"
                  "in most cases since the amount of memory used by the"
                  "shoeboxes is typically greater than the available system"
                  "memory. If, however, you know that memory is not an issue,"
                  "you can saved the shoeboxes in the integrated.pickle file"
                  "by setting this option to False. This only works if the debug"
                  "output is during integrated and not modelling."

        select = None
          .type = reflection_table_selector
          .help = "A string specifying the selection. The string should be of the"
                  "form: select=${COLUMN}[<|<=|==|!=|>=|>]${VALUE}. In addition"
                  "to the items in the reflection table, the following implicit"
                  "columns are defined if the necessary data is there:"
                  " intensity.sum.i_over_sigma"
                  " intensity.prf.i_over_sigma"

        split_experiments = True
          .type = bool
          .help = "Split shoeboxes into different files"

      }

      integrator = *auto 3d flat3d 2d single2d stills volume
        .type = choice
        .help = "The integrator to use."
        .expert_level=3

      profile {

        fitting = True
          .type = bool
          .help = "Use profile fitting if available"

        validation {

          number_of_partitions = 1
            .type = int(value_min=1)
            .help = "The number of subsamples to take from the reference spots."
                    "If the value is 1, then no validation is performed."

          min_partition_size = 100
            .type = int(value_min=1)
            .help = "The minimum number of spots to use in each subsample."

        }
      }

      filter
        .expert_level = 1
      {
        min_zeta = 0.05
          .help = "Filter the reflections by the value of zeta. A value of less"
                  "than or equal to zero indicates that this will not be used. A"
                  "positive value is used as the minimum permissable value."
          .type = float(value_min=0.0, value_max=1.0)

        max_shoebox_overlap = 1.0
          .type = float(value_min=0.0, value_max=1.0)
          .help = "Filter reflections whose shoeboxes are overlapped by greater"
                  "than the requested amount. Note that this is not the"
                  "percentage of the peak that is overlapped but rather the"
                  "percentage of the shoebox (background and foreground). This"
                  "can be useful when the detector is too close and many"
                  "overlapping reflections are predicted at high resolution"
                  "causing memory issues."

        include scope dials.algorithms.integration.filtering.phil_scope
      }
    }
  ''', process_includes=True)
  main_scope = phil_scope.get_without_substitution("integration")
  assert len(main_scope) == 1
  main_scope = main_scope[0]
  main_scope.adopt_scope(BackgroundIface.phil_scope())
  main_scope.adopt_scope(CentroidIface.phil_scope())
  return phil_scope
Beispiel #6
0
  def create(params, experiments, reflections):
    '''
    Create the integrator from the input configuration.

    :param params: The input phil parameters
    :param experiments: The list of experiments
    :param reflections: The reflections to integrate
    :return: The integrator class

    '''
    from dials.algorithms.integration.filtering import MultiPowderRingFilter
    from dials.interfaces import BackgroundIface
    from dials.interfaces import CentroidIface
    from dials.array_family import flex
    from libtbx.utils import Sorry
    import cPickle as pickle

    # Check each experiment has an imageset
    for exp in experiments:
      if exp.imageset is None:
        raise Sorry('''
          One or more experiment does not contain an imageset. Access to the
          image data is crucial for integration.
        ''')

    # Read the mask in if necessary
    if params.integration.lookup.mask is not None:
      if type(params.integration.lookup.mask) == str:
        with open(params.integration.lookup.mask) as infile:
          params.integration.lookup.mask = pickle.load(infile)

    # Initialise the strategy classes
    BackgroundAlgorithm = BackgroundIface.extension(
      params.integration.background.algorithm)
    CentroidAlgorithm = CentroidIface.extension(
      params.integration.centroid.algorithm)

    # Set the algorithms in the reflection table
    flex.reflection_table._background_algorithm = \
      flex.strategy(BackgroundAlgorithm, params)
    flex.reflection_table._centroid_algorithm = \
      flex.strategy(CentroidAlgorithm, params)

    # Get the classes we need
    if params.integration.integrator == 'auto':
      if experiments.all_stills():
        params.integration.integrator = 'stills'
      else:
        params.integration.integrator = '3d'
    if params.integration.integrator == '3d':
      IntegratorClass = Integrator3D
    elif params.integration.integrator == 'flat3d':
      IntegratorClass = IntegratorFlat3D
    elif params.integration.integrator == '2d':
      IntegratorClass = Integrator2D
    elif params.integration.integrator == 'single2d':
      IntegratorClass = IntegratorSingle2D
    elif params.integration.integrator == 'stills':
      IntegratorClass = IntegratorStills
    elif params.integration.integrator == 'volume':
      IntegratorClass = IntegratorVolume
    else:
      raise RuntimeError("Unknown integration type")

    # Remove scan if stills
    if experiments.all_stills():
      for experiment in experiments:
        experiment.scan = None

    # Return an instantiation of the class
    return IntegratorClass(
      experiments,
      reflections,
      Parameters.from_phil(params.integration))
Beispiel #7
0
def generate_phil_scope():
    '''
  Generate the integration phil scope.

  :return: The phil scope

  '''
    import dials.extensions
    from dials.interfaces import BackgroundIface
    from dials.interfaces import CentroidIface

    phil_scope = phil.parse('''

    integration {

      include scope dials.data.multiprocessing.phil_scope
      include scope dials.data.lookup.phil_scope

      block {

        size = auto
          .type = float
          .help = "The block size in rotation angle (degrees)."

        units = *degrees radians frames
          .type = choice
          .help = "The units of the block size"

        threshold = 0.99
          .type = float(value_min=0.0, value_max=1.0)
          .help = "For block size auto the block size is calculated by sorting"
                  "reflections by the number of frames they cover and then"
                  "selecting the block size to be 2*nframes[threshold] such"
                  "that 100*threshold % of reflections are guarenteed to be"
                  "fully contained in 1 block"

        force = False
          .type = bool
          .help = "If the number of processors is 1 and force is False, then the"
                  "number of blocks may be set to 1. If force is True then the"
                  "block size is always calculated."

        max_memory_usage = 0.75
          .type = float(value_min=0.0,value_max=1.0)
          .help = "The maximum percentage of total physical memory to use for"
                  "allocating shoebox arrays."
      }

      debug {

        reference {
          output = False
            .type = bool
            .help = "Save the reference profiles"
        }

        during = modelling *integration
          .type = choice
          .help = "Do debugging during modelling or integration"

        output = False
          .type = bool
          .help = "Save shoeboxes after each processing task."

        separate_files = True
          .type = bool
          .help = "If this is true, the shoeboxes are saved in separate files"
                  "from the output integrated.pickle file. This is necessary"
                  "in most cases since the amount of memory used by the"
                  "shoeboxes is typically greater than the available system"
                  "memory. If, however, you know that memory is not an issue,"
                  "you can saved the shoeboxes in the integrated.pickle file"
                  "by setting this option to False. This only works if the debug"
                  "output is during integrated and not modelling."

        select = None
          .type = reflection_table_selector
          .help = "A string specifying the selection. The string should be of the"
                  "form: select=${COLUMN}[<|<=|==|!=|>=|>]${VALUE}. In addition"
                  "to the items in the reflection table, the following implicit"
                  "columns are defined if the necessary data is there:"
                  " intensity.sum.i_over_sigma"
                  " intensity.prf.i_over_sigma"

        split_experiments = True
          .type = bool
          .help = "Split shoeboxes into different files"

      }

      integrator = *auto 3d flat3d 2d single2d stills volume
        .type = choice
        .help = "The integrator to use."
        .expert_level=3

      profile {

        fitting = True
          .type = bool
          .help = "Use profile fitting if available"

        validation {

          number_of_partitions = 1
            .type = int(value_min=1)
            .help = "The number of subsamples to take from the reference spots."
                    "If the value is 1, then no validation is performed."

          min_partition_size = 100
            .type = int(value_min=1)
            .help = "The minimum number of spots to use in each subsample."

        }
      }

      filter
        .expert_level = 1
      {
        min_zeta = 0.05
          .help = "Filter the reflections by the value of zeta. A value of less"
                  "than or equal to zero indicates that this will not be used. A"
                  "positive value is used as the minimum permissable value."
          .type = float(value_min=0.0, value_max=1.0)

        max_shoebox_overlap = 1.0
          .type = float(value_min=0.0, value_max=1.0)
          .help = "Filter reflections whose shoeboxes are overlapped by greater"
                  "than the requested amount. Note that this is not the"
                  "percentage of the peak that is overlapped but rather the"
                  "percentage of the shoebox (background and foreground). This"
                  "can be useful when the detector is too close and many"
                  "overlapping reflections are predicted at high resolution"
                  "causing memory issues."

        include scope dials.algorithms.integration.filtering.phil_scope
      }
    }
  ''',
                            process_includes=True)
    main_scope = phil_scope.get_without_substitution("integration")
    assert len(main_scope) == 1
    main_scope = main_scope[0]
    main_scope.adopt_scope(BackgroundIface.phil_scope())
    main_scope.adopt_scope(CentroidIface.phil_scope())
    return phil_scope
Beispiel #8
0
    def create(params, experiments, reflections):
        '''
    Create the integrator from the input configuration.

    :param params: The input phil parameters
    :param experiments: The list of experiments
    :param reflections: The reflections to integrate
    :return: The integrator class

    '''
        from dials.algorithms.integration.filtering import MultiPowderRingFilter
        from dials.interfaces import BackgroundIface
        from dials.interfaces import CentroidIface
        from dials.array_family import flex
        from libtbx.utils import Sorry
        import cPickle as pickle

        # Check each experiment has an imageset
        for exp in experiments:
            if exp.imageset is None:
                raise Sorry('''
          One or more experiment does not contain an imageset. Access to the
          image data is crucial for integration.
        ''')

        # Read the mask in if necessary
        if params.integration.lookup.mask is not None:
            if type(params.integration.lookup.mask) == str:
                with open(params.integration.lookup.mask) as infile:
                    params.integration.lookup.mask = pickle.load(infile)

        # Initialise the strategy classes
        BackgroundAlgorithm = BackgroundIface.extension(
            params.integration.background.algorithm)
        CentroidAlgorithm = CentroidIface.extension(
            params.integration.centroid.algorithm)

        # Set the algorithms in the reflection table
        flex.reflection_table._background_algorithm = \
          flex.strategy(BackgroundAlgorithm, params)
        flex.reflection_table._centroid_algorithm = \
          flex.strategy(CentroidAlgorithm, params)

        # Get the classes we need
        if params.integration.integrator == 'auto':
            if experiments.all_stills():
                params.integration.integrator = 'stills'
            else:
                params.integration.integrator = '3d'
        if params.integration.integrator == '3d':
            IntegratorClass = Integrator3D
        elif params.integration.integrator == 'flat3d':
            IntegratorClass = IntegratorFlat3D
        elif params.integration.integrator == '2d':
            IntegratorClass = Integrator2D
        elif params.integration.integrator == 'single2d':
            IntegratorClass = IntegratorSingle2D
        elif params.integration.integrator == 'stills':
            IntegratorClass = IntegratorStills
        elif params.integration.integrator == 'volume':
            IntegratorClass = IntegratorVolume
        else:
            raise RuntimeError("Unknown integration type")

        # Remove scan if stills
        if experiments.all_stills():
            for experiment in experiments:
                experiment.scan = None

        # Return an instantiation of the class
        return IntegratorClass(experiments, reflections,
                               Parameters.from_phil(params.integration))
Beispiel #9
0
def generate_phil_scope():
    """Generate the phil scope."""
    from dials.interfaces import BackgroundIface
    from dials.interfaces import IntensityIface
    from dials.interfaces import CentroidIface

    phil_scope = phil.parse(
        """

    integration {

      include scope dials.data.lookup.phil_scope

      mp {
        method = *multiprocessing sge lsf pbs
          .type = choice
          .help = "The multiprocessing method to use"

        max_procs = 1
          .type = int(value_min=1)
          .help = "The number of processes to use."
      }

      block {
        size = 10
          .type = float
          .help = "The block size in rotation angle (degrees)."
      }

      shoebox {
        n_sigma = 3
          .help = "The number of standard deviations of the beam divergence and the"
                  "mosaicity to use for the bounding box size."
          .type = float

        sigma_b = None
          .help = "The E.S.D. of the beam divergence"
          .type = float

        sigma_m = None
          .help = "The E.S.D. of the reflecting range"
          .type = float
      }

      filter {

        min_zeta = 0.05
          .help = "Filter the reflections by the value of zeta. A value of less"
                  "than or equal to zero indicates that this will not be used. A"
                  "positive value is used as the minimum permissable value."
          .type = float

        ice_rings {
          filter = False
            .type = bool
          unit_cell = 4.498,4.498,7.338,90,90,120
            .type = unit_cell
            .help = "The unit cell to generate d_spacings for ice rings."
          space_group = 194
            .type = space_group
            .help = "The space group used to generate d_spacings for ice rings."
          d_min = 0
            .type = int(value_min=0)
            .help = "The minimum resolution to filter ice rings"
          width = 0.06
            .type = float(value_min=0.0)
            .help = "The width of an ice ring (in d-spacing)."
        }
      }
    }
  """,
        process_includes=True,
    )
    main_scope = phil_scope.get_without_substitution("integration")
    assert len(main_scope) == 1
    main_scope = main_scope[0]
    main_scope.adopt_scope(BackgroundIface.phil_scope())
    main_scope.adopt_scope(IntensityIface.phil_scope())
    main_scope.adopt_scope(CentroidIface.phil_scope())
    return phil_scope