def _generate_input_scope(self):
'''
Generate the required input scope.
:return: The input phil scope
'''
from dials.util.phil import parse
# Create the input scope
require_input_scope = (
self._read_datablocks or
self._read_experiments or
self._read_reflections or
self._read_datablocks_from_images)
if not require_input_scope:
return None
input_phil_scope = parse('input {}')
main_scope = input_phil_scope.get_without_substitution("input")
assert(len(main_scope) == 1)
main_scope = main_scope[0]
# Add the datablock phil scope
if self._read_datablocks or self._read_datablocks_from_images:
phil_scope = parse('''
datablock = None
.type = datablock(check_format=%r)
.multiple = True
.help = "The datablock file path"
''' % self._check_format)
main_scope.adopt_scope(phil_scope)
# If reading images, add some more parameters
if self._read_datablocks_from_images:
main_scope.adopt_scope(tolerance_phil_scope)
# Add the experiments phil scope
if self._read_experiments:
phil_scope = parse('''
experiments = None
.type = experiment_list(check_format=%r)
.multiple = True
.help = "The experiment list file path"
''' % self._check_format)
main_scope.adopt_scope(phil_scope)
# Add the reflections scope
if self._read_reflections:
phil_scope = parse('''
reflections = None
.type = reflection_table
.multiple = True
.help = "The reflection table file path"
''')
main_scope.adopt_scope(phil_scope)
# Return the input scope
return input_phil_scope
def _generate_input_scope(self):
"""
Generate the required input scope.
:return: The input phil scope
"""
from dials.util.phil import parse
# Create the input scope
require_input_scope = (
self._read_experiments
or self._read_reflections
or self._read_experiments_from_images
)
if not require_input_scope:
return None
input_phil_scope = parse("input {}")
main_scope = input_phil_scope.get_without_substitution("input")
assert len(main_scope) == 1
main_scope = main_scope[0]
# Add the experiments phil scope
if self._read_experiments or self._read_experiments_from_images:
phil_scope = parse(
f"""
experiments = None
.type = experiment_list(check_format={self._check_format!r})
.multiple = True
.help = "The experiment list file path"
"""
)
main_scope.adopt_scope(phil_scope)
# If reading images, add some more parameters
if self._read_experiments_from_images:
main_scope.adopt_scope(tolerance_phil_scope)
# Add the reflections scope
if self._read_reflections:
phil_scope = parse(
"""
reflections = None
.type = reflection_table
.multiple = True
.help = "The reflection table file path"
"""
)
main_scope.adopt_scope(phil_scope)
# Return the input scope
return input_phil_scope
def from_observations(datablock, params=None):
'''
Construct a reflection table from observations.
:param datablock: The datablock
:param params: The input parameters
:return: The reflection table of observations
'''
from dials.algorithms.spot_finding.factory \
import SpotFinderFactory
from libtbx import Auto
if params is None:
from dials.command_line.find_spots import phil_scope
from dials.util.phil import parse
params = phil_scope.fetch(source=parse("")).extract()
if params.spotfinder.filter.min_spot_size is Auto:
detector = datablock.extract_imagesets()[0].get_detector()
if detector[0].get_type() == 'SENSOR_PAD':
# smaller default value for pixel array detectors
params.spotfinder.filter.min_spot_size = 3
else:
params.spotfinder.filter.min_spot_size = 6
logger.info('Setting spotfinder.filter.min_spot_size=%i' %
(params.spotfinder.filter.min_spot_size))
# Get the integrator from the input parameters
logger.info('Configuring spot finder from input parameters')
find_spots = SpotFinderFactory.from_parameters(datablock=datablock,
params=params)
# Find the spots
return find_spots(datablock)
def run(self, DataBlockFactory, ExperimentListFactory, flex, *args):
phil_scope = parse('''
input {
reflections = %s
.type = reflection_table
datablock = %s
.type = datablock
experiments = %s
.type = experiment_list
}
''' % (self.reflections_path, self.datablock_path, self.experiments_path))
params = phil_scope.extract()
# Check the right filenames were parsed
assert (params.input.reflections.filename == self.reflections_path)
assert (params.input.datablock.filename == self.datablock_path)
assert (params.input.experiments.filename == self.experiments_path)
# Check that we got the expected objects back
assert isinstance(params.input.reflections.data, Mock)
assert isinstance(params.input.datablock.data, Mock)
assert isinstance(params.input.experiments.data, Mock)
# Check we had the correct calls made
flex.reflection_table.from_pickle.assert_called_once_with(
self.reflections_path)
assert DataBlockFactory.from_json_file.call_args[0] == (
self.datablock_path, )
assert ExperimentListFactory.from_json_file.call_args[0] == (
self.experiments_path, )
print 'OK'
def __init__(self,
phil=None,
read_datablocks=False,
read_experiments=False,
read_reflections=False,
read_datablocks_from_images=False,
check_format=True):
'''
Initialise the parser.
:param phil: The phil scope
:param read_datablocks: Try to read the datablocks
:param read_experiments: Try to read the experiments
:param read_reflections: Try to read the reflections
:param read_datablocks_from_images: Try to read the datablocks from images
:param check_format: Check the format when reading images
'''
from dials.util.phil import parse
# Set the system phil scope
if phil is None:
self._system_phil = parse("")
else:
self._system_phil = phil
# Set the flags
self._read_datablocks = read_datablocks
self._read_experiments = read_experiments
self._read_reflections = read_reflections
self._read_datablocks_from_images = read_datablocks_from_images
self._check_format = check_format
# Adopt the input scope
input_phil_scope = self._generate_input_scope()
if input_phil_scope is not None:
self.system_phil.adopt_scope(input_phil_scope)
# Set the working phil scope
self._phil = self.system_phil.fetch(source=parse(""))
def test_few_reflections(dials_data, run_in_tmpdir):
u"""
Test that dials.symmetry does something sensible if given few reflections.
Use some example integrated data generated from a ten-image 1° sweep. These
contain a few dozen integrated reflections.
Args:
dials_data: DIALS custom Pytest fixture for access to test data.
run_in_tmpdir: DIALS custom Pytest fixture to run this test in a temporary
directory.
"""
# Get and use the default parameters for dials.symmetry.
params = symmetry.phil_scope.fetch(source=parse("")).extract()
# Use the integrated data from the first ten images of the first sweep.
data_dir = dials_data("l_cysteine_dials_output")
experiments = ExperimentList.from_file(data_dir / "11_integrated.expt")
reflections = [flex.reflection_table.from_file(data_dir / "11_integrated.refl")]
# Run dials.symmetry on the above data files.
symmetry.symmetry(experiments, reflections, params)
def from_observations(experiments, params=None):
"""
Construct a reflection table from observations.
:param experiments: The experiments
:param params: The input parameters
:return: The reflection table of observations
"""
from dials.algorithms.spot_finding.factory import SpotFinderFactory
if params is None:
from dials.command_line.find_spots import phil_scope
from dials.util.phil import parse
params = phil_scope.fetch(source=parse("")).extract()
if params.spotfinder.filter.min_spot_size is libtbx.Auto:
detector = experiments[0].imageset.get_detector()
if detector[0].get_type() == "SENSOR_PAD":
# smaller default value for pixel array detectors
params.spotfinder.filter.min_spot_size = 3
else:
params.spotfinder.filter.min_spot_size = 6
logger.info(
"Setting spotfinder.filter.min_spot_size=%i",
params.spotfinder.filter.min_spot_size,
)
# Get the integrator from the input parameters
logger.info("Configuring spot finder from input parameters")
find_spots = SpotFinderFactory.from_parameters(
experiments=experiments, params=params
)
# Find the spots
return find_spots(experiments)
def parse_args(
self, args, verbose=False, return_unhandled=False, quick_parse=False
):
"""
Parse the command line arguments.
:param args: The input arguments
:param verbose: Print verbose output
:param return_unhandled: True/False also return unhandled arguments
:param quick_parse: Return as fast as possible and without reading any data,
ignoring class constructor options.
:return: The options and parameters and (optionally) unhandled arguments
"""
from dxtbx.model.experiment_list import (
BeamComparison,
DetectorComparison,
GoniometerComparison,
)
from dials.util.phil import parse
# Parse the command line phil parameters
user_phils = []
unhandled = []
interpretor = self.system_phil.command_line_argument_interpreter()
def _is_a_phil_file(filename):
return any(
filename.endswith(phil_ext)
for phil_ext in (".phil", ".param", ".params", ".eff", ".def")
)
for arg in args:
if (
_is_a_phil_file(arg)
and os.path.isfile(arg)
and os.path.getsize(arg) > 0
):
try:
user_phils.append(parse(file_name=arg))
except Exception:
if return_unhandled:
unhandled.append(arg)
else:
raise
# Treat "has a schema" as "looks like a URL (not phil)
elif "=" in arg and not get_url_scheme(arg):
try:
user_phils.append(interpretor.process_arg(arg=arg))
except Exception:
if return_unhandled:
unhandled.append(arg)
else:
raise
else:
unhandled.append(arg)
# Fetch the phil parameters
self._phil, unused = self.system_phil.fetch(
sources=user_phils, track_unused_definitions=True
)
# Print if bad definitions
if len(unused) > 0:
msg = [item.object.as_str().strip() for item in unused]
msg = "\n".join([" %s" % line for line in msg])
raise RuntimeError(f"The following definitions were not recognised\n{msg}")
# Extract the parameters
params = self._phil.extract()
# Stop at this point if quick_parse is set. A second pass may be needed.
if quick_parse:
return params, unhandled
# Create some comparison functions
if self._read_experiments_from_images:
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction,
)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin,
)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation,
)
scan_tolerance = params.input.tolerance.scan.oscillation
# FIXME Should probably make this smarter since it requires editing here
# and in dials.import phil scope
try:
format_kwargs = {
"dynamic_shadowing": params.format.dynamic_shadowing,
"multi_panel": params.format.multi_panel,
}
except AttributeError:
format_kwargs = None
else:
compare_beam = None
compare_detector = None
compare_goniometer = None
scan_tolerance = None
format_kwargs = None
try:
load_models = params.load_models
except AttributeError:
load_models = True
# Try to import everything
importer = Importer(
unhandled,
read_experiments=self._read_experiments,
read_reflections=self._read_reflections,
read_experiments_from_images=self._read_experiments_from_images,
check_format=self._check_format,
verbose=verbose,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs,
load_models=load_models,
)
# Grab a copy of the errors that occurred in case the caller wants them
self.handling_errors = importer.handling_errors
# Add the cached arguments
for obj in importer.experiments:
params.input.experiments.append(obj)
for obj in importer.reflections:
params.input.reflections.append(obj)
# Convert to phil
self._phil = self.system_phil.format(python_object=params)
return params, importer.unhandled
import logging
import pickle
from dials.util.phil import parse
logger = logging.getLogger(__name__)
phil_scope = parse("""
min_count = 5
.type = int(value_min=1)
nsigma = 6
.type = float(value_min=0)
sigma = 0.5
.type = float(value_min=0)
kernel_size = 9
.type = int
niter = 10
.type = int
""")
class Modeller:
def __init__(self,
beam,
detector,
min_count=5,
def parse_args(self,
args,
verbose=False,
return_unhandled=False,
quick_parse=False):
'''
Parse the command line arguments.
:param args: The input arguments
:param verbose: Print verbose output
:param return_unhandled: True/False also return unhandled arguments
:param quick_parse: Return as fast as possible and without reading any data,
ignoring class constructor options.
:return: The options and parameters and (optionally) unhandled arguments
'''
import os.path
from dxtbx.datablock import BeamComparison
from dxtbx.datablock import DetectorComparison
from dxtbx.datablock import GoniometerComparison
from dials.util.phil import parse
# Parse the command line phil parameters
user_phils = []
unhandled = []
interpretor = self.system_phil.command_line_argument_interpreter()
for arg in args:
if os.path.isfile(arg) and os.path.getsize(arg) > 0:
name, ext = os.path.splitext(arg)
if ext in ['.phil', '.param', '.params', '.eff', '.def']:
try:
user_phils.append(parse(file_name=arg))
except Exception:
if return_unhandled:
unhandled.append(arg)
else:
raise
else:
unhandled.append(arg)
elif arg.find("=") >= 0:
try:
user_phils.append(interpretor.process_arg(arg=arg))
except Exception:
if return_unhandled:
unhandled.append(arg)
else:
raise
else:
unhandled.append(arg)
# Fetch the phil parameters
self._phil, unused = self.system_phil.fetch(
sources=user_phils, track_unused_definitions=True)
# Print if bad definitions
if len(unused) > 0:
msg = [item.object.as_str().strip() for item in unused]
msg = '\n'.join([' %s' % line for line in msg])
raise RuntimeError(
'The following definitions were not recognised\n%s' % msg)
# Extract the parameters
params = self._phil.extract()
# Stop at this point if quick_parse is set. A second pass may be needed.
if quick_parse:
return params, unhandled
# Create some comparison functions
if self._read_datablocks_from_images:
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.
polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.
polarization_fraction)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.
rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.
fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.
setting_rotation)
scan_tolerance = params.input.tolerance.scan.oscillation
# FIXME Should probably make this smarter since it requires editing here
# and in dials.import phil scope
try:
format_kwargs = {
'dynamic_shadowing': params.format.dynamic_shadowing,
'multi_panel': params.format.multi_panel,
}
except Exception:
format_kwargs = None
else:
compare_beam = None
compare_detector = None
compare_goniometer = None
scan_tolerance = None
format_kwargs = None
# Try to import everything
importer = Importer(
unhandled,
read_datablocks=self._read_datablocks,
read_experiments=self._read_experiments,
read_reflections=self._read_reflections,
read_datablocks_from_images=self._read_datablocks_from_images,
check_format=self._check_format,
verbose=verbose,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs)
# Grab a copy of the errors that occured in case the caller wants them
self.handling_errors = importer.handling_errors
# Add the cached arguments
for obj in importer.datablocks:
params.input.datablock.append(obj)
for obj in importer.experiments:
params.input.experiments.append(obj)
for obj in importer.reflections:
params.input.reflections.append(obj)
# Convert to phil
self._phil = self.system_phil.format(python_object=params)
return params, importer.unhandled
def generate_phil_scope():
'''
Generate the integration phil scope.
:return: The phil scope
'''
import dials.extensions
from dials.interfaces import BackgroundIface
from dials.interfaces import CentroidIface
phil_scope = phil.parse('''
integration {
include scope dials.data.lookup.phil_scope
block {
size = auto
.type = float
.help = "The block size in rotation angle (degrees)."
units = *degrees radians frames
.type = choice
.help = "The units of the block size"
threshold = 0.99
.type = float(value_min=0.0, value_max=1.0)
.help = "For block size auto the block size is calculated by sorting"
"reflections by the number of frames they cover and then"
"selecting the block size to be 2*nframes[threshold] such"
"that 100*threshold % of reflections are guarenteed to be"
"fully contained in 1 block"
force = False
.type = bool
.help = "If the number of processors is 1 and force is False, then the"
"number of blocks may be set to 1. If force is True then the"
"block size is always calculated."
max_memory_usage = 0.75
.type = float(value_min=0.0,value_max=1.0)
.help = "The maximum percentage of total physical memory to use for"
"allocating shoebox arrays."
}
debug {
reference {
filename = "reference_profiles.pickle"
.type = str
.help = "The filename for the reference profiles"
output = False
.type = bool
.help = "Save the reference profiles"
}
during = modelling *integration
.type = choice
.help = "Do debugging during modelling or integration"
output = False
.type = bool
.help = "Save shoeboxes after each processing task."
separate_files = True
.type = bool
.help = "If this is true, the shoeboxes are saved in separate files"
"from the output integrated.pickle file. This is necessary"
"in most cases since the amount of memory used by the"
"shoeboxes is typically greater than the available system"
"memory. If, however, you know that memory is not an issue,"
"you can saved the shoeboxes in the integrated.pickle file"
"by setting this option to False. This only works if the debug"
"output is during integrated and not modelling."
select = None
.type = reflection_table_selector
.help = "A string specifying the selection. The string should be of the"
"form: select=${COLUMN}[<|<=|==|!=|>=|>]${VALUE}. In addition"
"to the items in the reflection table, the following implicit"
"columns are defined if the necessary data is there:"
" intensity.sum.i_over_sigma"
" intensity.prf.i_over_sigma"
split_experiments = True
.type = bool
.help = "Split shoeboxes into different files"
}
integrator = *auto 3d flat3d 2d single2d stills volume
.type = choice
.help = "The integrator to use."
.expert_level=3
profile {
fitting = True
.type = bool
.help = "Use profile fitting if available"
validation {
number_of_partitions = 1
.type = int(value_min=1)
.help = "The number of subsamples to take from the reference spots."
"If the value is 1, then no validation is performed."
min_partition_size = 100
.type = int(value_min=1)
.help = "The minimum number of spots to use in each subsample."
}
}
filter
.expert_level = 1
{
min_zeta = 0.05
.help = "Filter the reflections by the value of zeta. A value of less"
"than or equal to zero indicates that this will not be used. A"
"positive value is used as the minimum permissable value."
.type = float(value_min=0.0, value_max=1.0)
max_shoebox_overlap = 1.0
.type = float(value_min=0.0, value_max=1.0)
.help = "Filter reflections whose shoeboxes are overlapped by greater"
"than the requested amount. Note that this is not the"
"percentage of the peak that is overlapped but rather the"
"percentage of the shoebox (background and foreground). This"
"can be useful when the detector is too close and many"
"overlapping reflections are predicted at high resolution"
"causing memory issues."
ice_rings = False
.help = "Set the ice ring flags"
.type = bool
}
include scope dials.algorithms.integration.overlaps_filter.phil_scope
mp {
method = *none drmaa sge lsf pbs
.type = choice
.help = "The cluster method to use"
njobs = 1
.type = int(value_min=1)
.help = "The number of cluster jobs to use"
nproc = 1
.type = int(value_min=1)
.help = "The number of processes to use per cluster job"
}
}
''',
process_includes=True)
main_scope = phil_scope.get_without_substitution("integration")
assert len(main_scope) == 1
main_scope = main_scope[0]
main_scope.adopt_scope(BackgroundIface.phil_scope())
main_scope.adopt_scope(CentroidIface.phil_scope())
return phil_scope
