def create(params, experiments, reflections):
''' Create the integrator from the input configuration. '''
from dials.interfaces import IntensityIface
from dials.interfaces import BackgroundIface
from dials.interfaces import CentroidIface
from dials.array_family import flex
# Initialise the strategy classes
BackgroundAlgorithm = BackgroundIface.extension(
params.integration.background.algorithm)
IntensityAlgorithm = IntensityIface.extension(
params.integration.intensity.algorithm)
CentroidAlgorithm = CentroidIface.extension(
params.integration.centroid.algorithm)
# Set the algorithms in the reflection table
flex.reflection_table._background_algorithm = \
flex.strategy(BackgroundAlgorithm, params)
flex.reflection_table._intensity_algorithm = \
flex.strategy(IntensityAlgorithm, params)
flex.reflection_table._centroid_algorithm = \
flex.strategy(CentroidAlgorithm, params)
# Get the integrator class
IntegratorClass = IntegratorFactory.select_integrator(IntensityAlgorithm)
# Return an instantiation of the class
return IntegratorClass(
experiments=experiments,
reflections=reflections,
block_size=params.integration.block.size,
max_procs=params.integration.mp.max_procs,
mp_method=params.integration.mp.method)
def tst_after_import_extensions(self):
import dials.extensions # import dependency
from dials.interfaces import ProfileModelIface
from dials.interfaces import SpotFinderThresholdIface
from dials.interfaces import CentroidIface
from dials.interfaces import BackgroundIface
# Should have four interfaces
interfaces = list(Interface.interfaces())
assert (len(interfaces) == 4)
# Check we have the expected number of extensions for each interface
extensions = list(ProfileModelIface.extensions())
assert (len(extensions) > 0)
extensions = list(SpotFinderThresholdIface.extensions())
assert (len(extensions) > 0)
extensions = list(CentroidIface.extensions())
assert (len(extensions) > 0)
extensions = list(BackgroundIface.extensions())
assert (len(extensions) > 0)
# Check the interface contain the expected extensions
from dials.extensions import GaussianRSProfileModelExt
from dials.extensions import KabschSpotFinderThresholdExt
from dials.extensions import SimpleCentroidExt
from dials.extensions import NullBackgroundExt
from dials.extensions import SimpleBackgroundExt
extensions = list(ProfileModelIface.extensions())
assert (GaussianRSProfileModelExt in extensions)
extensions = list(SpotFinderThresholdIface.extensions())
assert (KabschSpotFinderThresholdExt in extensions)
extensions = list(CentroidIface.extensions())
assert (SimpleCentroidExt in extensions)
extensions = list(BackgroundIface.extensions())
assert (NullBackgroundExt in extensions)
assert (SimpleBackgroundExt in extensions)
# Test passed
print 'OK'
def create(params, experiments, reflections):
"""Create the integrator from the input configuration."""
from dials.interfaces import IntensityIface
from dials.interfaces import BackgroundIface
from dials.interfaces import CentroidIface
from dials.array_family import flex
# Initialise the strategy classes
BackgroundAlgorithm = BackgroundIface.extension(
params.integration.background.algorithm
)
IntensityAlgorithm = IntensityIface.extension(
params.integration.intensity.algorithm
)
CentroidAlgorithm = CentroidIface.extension(
params.integration.centroid.algorithm
)
# Set the algorithms in the reflection table
flex.reflection_table._background_algorithm = flex.strategy(
BackgroundAlgorithm, params
)
flex.reflection_table._intensity_algorithm = flex.strategy(
IntensityAlgorithm, params
)
flex.reflection_table._centroid_algorithm = flex.strategy(
CentroidAlgorithm, params
)
# Get the integrator class
IntegratorClass = IntegratorFactory.select_integrator(IntensityAlgorithm)
# Return an instantiation of the class
return IntegratorClass(
experiments=experiments,
reflections=reflections,
block_size=params.integration.block.size,
max_procs=params.integration.mp.max_procs,
mp_method=params.integration.mp.method,
)
def generate_phil_scope():
''' Generate the phil scope. '''
from dials.interfaces import BackgroundIface
from dials.interfaces import IntensityIface
from dials.interfaces import CentroidIface
phil_scope = phil.parse('''
integration {
include scope dials.data.lookup.phil_scope
mp {
method = *multiprocessing sge lsf pbs
.type = choice
.help = "The multiprocessing method to use"
max_procs = 1
.type = int(value_min=1)
.help = "The number of processes to use."
}
block {
size = 10
.type = float
.help = "The block size in rotation angle (degrees)."
}
shoebox {
n_sigma = 3
.help = "The number of standard deviations of the beam divergence and the"
"mosaicity to use for the bounding box size."
.type = float
sigma_b = None
.help = "The E.S.D. of the beam divergence"
.type = float
sigma_m = None
.help = "The E.S.D. of the reflecting range"
.type = float
}
filter {
min_zeta = 0.05
.help = "Filter the reflections by the value of zeta. A value of less"
"than or equal to zero indicates that this will not be used. A"
"positive value is used as the minimum permissable value."
.type = float
ice_rings {
filter = False
.type = bool
unit_cell = 4.498,4.498,7.338,90,90,120
.type = unit_cell
.help = "The unit cell to generate d_spacings for ice rings."
space_group = 194
.type = space_group
.help = "The space group used to generate d_spacings for ice rings."
d_min = 0
.type = int(value_min=0)
.help = "The minimum resolution to filter ice rings"
width = 0.06
.type = float(value_min=0.0)
.help = "The width of an ice ring (in d-spacing)."
}
}
}
''', process_includes=True)
main_scope = phil_scope.get_without_substitution("integration")
assert(len(main_scope) == 1)
main_scope = main_scope[0]
main_scope.adopt_scope(BackgroundIface.phil_scope())
main_scope.adopt_scope(IntensityIface.phil_scope())
main_scope.adopt_scope(CentroidIface.phil_scope())
return phil_scope
def generate_phil_scope():
'''
Generate the integration phil scope.
:return: The phil scope
'''
import dials.extensions
from dials.interfaces import BackgroundIface
from dials.interfaces import CentroidIface
phil_scope = phil.parse('''
integration {
include scope dials.data.multiprocessing.phil_scope
include scope dials.data.lookup.phil_scope
block {
size = auto
.type = float
.help = "The block size in rotation angle (degrees)."
units = *degrees radians frames
.type = choice
.help = "The units of the block size"
threshold = 0.99
.type = float(value_min=0.0, value_max=1.0)
.help = "For block size auto the block size is calculated by sorting"
"reflections by the number of frames they cover and then"
"selecting the block size to be 2*nframes[threshold] such"
"that 100*threshold % of reflections are guarenteed to be"
"fully contained in 1 block"
force = False
.type = bool
.help = "If the number of processors is 1 and force is False, then the"
"number of blocks may be set to 1. If force is True then the"
"block size is always calculated."
max_memory_usage = 0.75
.type = float(value_min=0.0,value_max=1.0)
.help = "The maximum percentage of total physical memory to use for"
"allocating shoebox arrays."
}
debug {
reference {
output = False
.type = bool
.help = "Save the reference profiles"
}
during = modelling *integration
.type = choice
.help = "Do debugging during modelling or integration"
output = False
.type = bool
.help = "Save shoeboxes after each processing task."
separate_files = True
.type = bool
.help = "If this is true, the shoeboxes are saved in separate files"
"from the output integrated.pickle file. This is necessary"
"in most cases since the amount of memory used by the"
"shoeboxes is typically greater than the available system"
"memory. If, however, you know that memory is not an issue,"
"you can saved the shoeboxes in the integrated.pickle file"
"by setting this option to False. This only works if the debug"
"output is during integrated and not modelling."
select = None
.type = reflection_table_selector
.help = "A string specifying the selection. The string should be of the"
"form: select=${COLUMN}[<|<=|==|!=|>=|>]${VALUE}. In addition"
"to the items in the reflection table, the following implicit"
"columns are defined if the necessary data is there:"
" intensity.sum.i_over_sigma"
" intensity.prf.i_over_sigma"
split_experiments = True
.type = bool
.help = "Split shoeboxes into different files"
}
integrator = *auto 3d flat3d 2d single2d stills volume
.type = choice
.help = "The integrator to use."
.expert_level=3
profile {
fitting = True
.type = bool
.help = "Use profile fitting if available"
validation {
number_of_partitions = 1
.type = int(value_min=1)
.help = "The number of subsamples to take from the reference spots."
"If the value is 1, then no validation is performed."
min_partition_size = 100
.type = int(value_min=1)
.help = "The minimum number of spots to use in each subsample."
}
}
filter
.expert_level = 1
{
min_zeta = 0.05
.help = "Filter the reflections by the value of zeta. A value of less"
"than or equal to zero indicates that this will not be used. A"
"positive value is used as the minimum permissable value."
.type = float(value_min=0.0, value_max=1.0)
max_shoebox_overlap = 1.0
.type = float(value_min=0.0, value_max=1.0)
.help = "Filter reflections whose shoeboxes are overlapped by greater"
"than the requested amount. Note that this is not the"
"percentage of the peak that is overlapped but rather the"
"percentage of the shoebox (background and foreground). This"
"can be useful when the detector is too close and many"
"overlapping reflections are predicted at high resolution"
"causing memory issues."
include scope dials.algorithms.integration.filtering.phil_scope
}
}
''', process_includes=True)
main_scope = phil_scope.get_without_substitution("integration")
assert len(main_scope) == 1
main_scope = main_scope[0]
main_scope.adopt_scope(BackgroundIface.phil_scope())
main_scope.adopt_scope(CentroidIface.phil_scope())
return phil_scope
def create(params, experiments, reflections):
'''
Create the integrator from the input configuration.
:param params: The input phil parameters
:param experiments: The list of experiments
:param reflections: The reflections to integrate
:return: The integrator class
'''
from dials.algorithms.integration.filtering import MultiPowderRingFilter
from dials.interfaces import BackgroundIface
from dials.interfaces import CentroidIface
from dials.array_family import flex
from libtbx.utils import Sorry
import cPickle as pickle
# Check each experiment has an imageset
for exp in experiments:
if exp.imageset is None:
raise Sorry('''
One or more experiment does not contain an imageset. Access to the
image data is crucial for integration.
''')
# Read the mask in if necessary
if params.integration.lookup.mask is not None:
if type(params.integration.lookup.mask) == str:
with open(params.integration.lookup.mask) as infile:
params.integration.lookup.mask = pickle.load(infile)
# Initialise the strategy classes
BackgroundAlgorithm = BackgroundIface.extension(
params.integration.background.algorithm)
CentroidAlgorithm = CentroidIface.extension(
params.integration.centroid.algorithm)
# Set the algorithms in the reflection table
flex.reflection_table._background_algorithm = \
flex.strategy(BackgroundAlgorithm, params)
flex.reflection_table._centroid_algorithm = \
flex.strategy(CentroidAlgorithm, params)
# Get the classes we need
if params.integration.integrator == 'auto':
if experiments.all_stills():
params.integration.integrator = 'stills'
else:
params.integration.integrator = '3d'
if params.integration.integrator == '3d':
IntegratorClass = Integrator3D
elif params.integration.integrator == 'flat3d':
IntegratorClass = IntegratorFlat3D
elif params.integration.integrator == '2d':
IntegratorClass = Integrator2D
elif params.integration.integrator == 'single2d':
IntegratorClass = IntegratorSingle2D
elif params.integration.integrator == 'stills':
IntegratorClass = IntegratorStills
elif params.integration.integrator == 'volume':
IntegratorClass = IntegratorVolume
else:
raise RuntimeError("Unknown integration type")
# Remove scan if stills
if experiments.all_stills():
for experiment in experiments:
experiment.scan = None
# Return an instantiation of the class
return IntegratorClass(
experiments,
reflections,
Parameters.from_phil(params.integration))
def generate_phil_scope():
'''
Generate the integration phil scope.
:return: The phil scope
'''
import dials.extensions
from dials.interfaces import BackgroundIface
from dials.interfaces import CentroidIface
phil_scope = phil.parse('''
integration {
include scope dials.data.multiprocessing.phil_scope
include scope dials.data.lookup.phil_scope
block {
size = auto
.type = float
.help = "The block size in rotation angle (degrees)."
units = *degrees radians frames
.type = choice
.help = "The units of the block size"
threshold = 0.99
.type = float(value_min=0.0, value_max=1.0)
.help = "For block size auto the block size is calculated by sorting"
"reflections by the number of frames they cover and then"
"selecting the block size to be 2*nframes[threshold] such"
"that 100*threshold % of reflections are guarenteed to be"
"fully contained in 1 block"
force = False
.type = bool
.help = "If the number of processors is 1 and force is False, then the"
"number of blocks may be set to 1. If force is True then the"
"block size is always calculated."
max_memory_usage = 0.75
.type = float(value_min=0.0,value_max=1.0)
.help = "The maximum percentage of total physical memory to use for"
"allocating shoebox arrays."
}
debug {
reference {
output = False
.type = bool
.help = "Save the reference profiles"
}
during = modelling *integration
.type = choice
.help = "Do debugging during modelling or integration"
output = False
.type = bool
.help = "Save shoeboxes after each processing task."
separate_files = True
.type = bool
.help = "If this is true, the shoeboxes are saved in separate files"
"from the output integrated.pickle file. This is necessary"
"in most cases since the amount of memory used by the"
"shoeboxes is typically greater than the available system"
"memory. If, however, you know that memory is not an issue,"
"you can saved the shoeboxes in the integrated.pickle file"
"by setting this option to False. This only works if the debug"
"output is during integrated and not modelling."
select = None
.type = reflection_table_selector
.help = "A string specifying the selection. The string should be of the"
"form: select=${COLUMN}[<|<=|==|!=|>=|>]${VALUE}. In addition"
"to the items in the reflection table, the following implicit"
"columns are defined if the necessary data is there:"
" intensity.sum.i_over_sigma"
" intensity.prf.i_over_sigma"
split_experiments = True
.type = bool
.help = "Split shoeboxes into different files"
}
integrator = *auto 3d flat3d 2d single2d stills volume
.type = choice
.help = "The integrator to use."
.expert_level=3
profile {
fitting = True
.type = bool
.help = "Use profile fitting if available"
validation {
number_of_partitions = 1
.type = int(value_min=1)
.help = "The number of subsamples to take from the reference spots."
"If the value is 1, then no validation is performed."
min_partition_size = 100
.type = int(value_min=1)
.help = "The minimum number of spots to use in each subsample."
}
}
filter
.expert_level = 1
{
min_zeta = 0.05
.help = "Filter the reflections by the value of zeta. A value of less"
"than or equal to zero indicates that this will not be used. A"
"positive value is used as the minimum permissable value."
.type = float(value_min=0.0, value_max=1.0)
max_shoebox_overlap = 1.0
.type = float(value_min=0.0, value_max=1.0)
.help = "Filter reflections whose shoeboxes are overlapped by greater"
"than the requested amount. Note that this is not the"
"percentage of the peak that is overlapped but rather the"
"percentage of the shoebox (background and foreground). This"
"can be useful when the detector is too close and many"
"overlapping reflections are predicted at high resolution"
"causing memory issues."
include scope dials.algorithms.integration.filtering.phil_scope
}
}
''',
process_includes=True)
main_scope = phil_scope.get_without_substitution("integration")
assert len(main_scope) == 1
main_scope = main_scope[0]
main_scope.adopt_scope(BackgroundIface.phil_scope())
main_scope.adopt_scope(CentroidIface.phil_scope())
return phil_scope
def create(params, experiments, reflections):
'''
Create the integrator from the input configuration.
:param params: The input phil parameters
:param experiments: The list of experiments
:param reflections: The reflections to integrate
:return: The integrator class
'''
from dials.algorithms.integration.filtering import MultiPowderRingFilter
from dials.interfaces import BackgroundIface
from dials.interfaces import CentroidIface
from dials.array_family import flex
from libtbx.utils import Sorry
import cPickle as pickle
# Check each experiment has an imageset
for exp in experiments:
if exp.imageset is None:
raise Sorry('''
One or more experiment does not contain an imageset. Access to the
image data is crucial for integration.
''')
# Read the mask in if necessary
if params.integration.lookup.mask is not None:
if type(params.integration.lookup.mask) == str:
with open(params.integration.lookup.mask) as infile:
params.integration.lookup.mask = pickle.load(infile)
# Initialise the strategy classes
BackgroundAlgorithm = BackgroundIface.extension(
params.integration.background.algorithm)
CentroidAlgorithm = CentroidIface.extension(
params.integration.centroid.algorithm)
# Set the algorithms in the reflection table
flex.reflection_table._background_algorithm = \
flex.strategy(BackgroundAlgorithm, params)
flex.reflection_table._centroid_algorithm = \
flex.strategy(CentroidAlgorithm, params)
# Get the classes we need
if params.integration.integrator == 'auto':
if experiments.all_stills():
params.integration.integrator = 'stills'
else:
params.integration.integrator = '3d'
if params.integration.integrator == '3d':
IntegratorClass = Integrator3D
elif params.integration.integrator == 'flat3d':
IntegratorClass = IntegratorFlat3D
elif params.integration.integrator == '2d':
IntegratorClass = Integrator2D
elif params.integration.integrator == 'single2d':
IntegratorClass = IntegratorSingle2D
elif params.integration.integrator == 'stills':
IntegratorClass = IntegratorStills
elif params.integration.integrator == 'volume':
IntegratorClass = IntegratorVolume
else:
raise RuntimeError("Unknown integration type")
# Remove scan if stills
if experiments.all_stills():
for experiment in experiments:
experiment.scan = None
# Return an instantiation of the class
return IntegratorClass(experiments, reflections,
Parameters.from_phil(params.integration))
def generate_phil_scope():
"""Generate the phil scope."""
from dials.interfaces import BackgroundIface
from dials.interfaces import IntensityIface
from dials.interfaces import CentroidIface
phil_scope = phil.parse(
"""
integration {
include scope dials.data.lookup.phil_scope
mp {
method = *multiprocessing sge lsf pbs
.type = choice
.help = "The multiprocessing method to use"
max_procs = 1
.type = int(value_min=1)
.help = "The number of processes to use."
}
block {
size = 10
.type = float
.help = "The block size in rotation angle (degrees)."
}
shoebox {
n_sigma = 3
.help = "The number of standard deviations of the beam divergence and the"
"mosaicity to use for the bounding box size."
.type = float
sigma_b = None
.help = "The E.S.D. of the beam divergence"
.type = float
sigma_m = None
.help = "The E.S.D. of the reflecting range"
.type = float
}
filter {
min_zeta = 0.05
.help = "Filter the reflections by the value of zeta. A value of less"
"than or equal to zero indicates that this will not be used. A"
"positive value is used as the minimum permissable value."
.type = float
ice_rings {
filter = False
.type = bool
unit_cell = 4.498,4.498,7.338,90,90,120
.type = unit_cell
.help = "The unit cell to generate d_spacings for ice rings."
space_group = 194
.type = space_group
.help = "The space group used to generate d_spacings for ice rings."
d_min = 0
.type = int(value_min=0)
.help = "The minimum resolution to filter ice rings"
width = 0.06
.type = float(value_min=0.0)
.help = "The width of an ice ring (in d-spacing)."
}
}
}
""",
process_includes=True,
)
main_scope = phil_scope.get_without_substitution("integration")
assert len(main_scope) == 1
main_scope = main_scope[0]
main_scope.adopt_scope(BackgroundIface.phil_scope())
main_scope.adopt_scope(IntensityIface.phil_scope())
main_scope.adopt_scope(CentroidIface.phil_scope())
return phil_scope