def _generate_input_scope(self): ''' Generate the required input scope. :return: The input phil scope ''' from dials.util.phil import parse # Create the input scope require_input_scope = ( self._read_datablocks or self._read_experiments or self._read_reflections or self._read_datablocks_from_images) if not require_input_scope: return None input_phil_scope = parse('input {}') main_scope = input_phil_scope.get_without_substitution("input") assert(len(main_scope) == 1) main_scope = main_scope[0] # Add the datablock phil scope if self._read_datablocks or self._read_datablocks_from_images: phil_scope = parse(''' datablock = None .type = datablock(check_format=%r) .multiple = True .help = "The datablock file path" ''' % self._check_format) main_scope.adopt_scope(phil_scope) # If reading images, add some more parameters if self._read_datablocks_from_images: main_scope.adopt_scope(tolerance_phil_scope) # Add the experiments phil scope if self._read_experiments: phil_scope = parse(''' experiments = None .type = experiment_list(check_format=%r) .multiple = True .help = "The experiment list file path" ''' % self._check_format) main_scope.adopt_scope(phil_scope) # Add the reflections scope if self._read_reflections: phil_scope = parse(''' reflections = None .type = reflection_table .multiple = True .help = "The reflection table file path" ''') main_scope.adopt_scope(phil_scope) # Return the input scope return input_phil_scope
def _generate_input_scope(self): """ Generate the required input scope. :return: The input phil scope """ from dials.util.phil import parse # Create the input scope require_input_scope = ( self._read_experiments or self._read_reflections or self._read_experiments_from_images ) if not require_input_scope: return None input_phil_scope = parse("input {}") main_scope = input_phil_scope.get_without_substitution("input") assert len(main_scope) == 1 main_scope = main_scope[0] # Add the experiments phil scope if self._read_experiments or self._read_experiments_from_images: phil_scope = parse( f""" experiments = None .type = experiment_list(check_format={self._check_format!r}) .multiple = True .help = "The experiment list file path" """ ) main_scope.adopt_scope(phil_scope) # If reading images, add some more parameters if self._read_experiments_from_images: main_scope.adopt_scope(tolerance_phil_scope) # Add the reflections scope if self._read_reflections: phil_scope = parse( """ reflections = None .type = reflection_table .multiple = True .help = "The reflection table file path" """ ) main_scope.adopt_scope(phil_scope) # Return the input scope return input_phil_scope
def from_observations(datablock, params=None): ''' Construct a reflection table from observations. :param datablock: The datablock :param params: The input parameters :return: The reflection table of observations ''' from dials.algorithms.spot_finding.factory \ import SpotFinderFactory from libtbx import Auto if params is None: from dials.command_line.find_spots import phil_scope from dials.util.phil import parse params = phil_scope.fetch(source=parse("")).extract() if params.spotfinder.filter.min_spot_size is Auto: detector = datablock.extract_imagesets()[0].get_detector() if detector[0].get_type() == 'SENSOR_PAD': # smaller default value for pixel array detectors params.spotfinder.filter.min_spot_size = 3 else: params.spotfinder.filter.min_spot_size = 6 logger.info('Setting spotfinder.filter.min_spot_size=%i' % (params.spotfinder.filter.min_spot_size)) # Get the integrator from the input parameters logger.info('Configuring spot finder from input parameters') find_spots = SpotFinderFactory.from_parameters(datablock=datablock, params=params) # Find the spots return find_spots(datablock)
def run(self, DataBlockFactory, ExperimentListFactory, flex, *args): phil_scope = parse(''' input { reflections = %s .type = reflection_table datablock = %s .type = datablock experiments = %s .type = experiment_list } ''' % (self.reflections_path, self.datablock_path, self.experiments_path)) params = phil_scope.extract() # Check the right filenames were parsed assert (params.input.reflections.filename == self.reflections_path) assert (params.input.datablock.filename == self.datablock_path) assert (params.input.experiments.filename == self.experiments_path) # Check that we got the expected objects back assert isinstance(params.input.reflections.data, Mock) assert isinstance(params.input.datablock.data, Mock) assert isinstance(params.input.experiments.data, Mock) # Check we had the correct calls made flex.reflection_table.from_pickle.assert_called_once_with( self.reflections_path) assert DataBlockFactory.from_json_file.call_args[0] == ( self.datablock_path, ) assert ExperimentListFactory.from_json_file.call_args[0] == ( self.experiments_path, ) print 'OK'
def __init__(self, phil=None, read_datablocks=False, read_experiments=False, read_reflections=False, read_datablocks_from_images=False, check_format=True): ''' Initialise the parser. :param phil: The phil scope :param read_datablocks: Try to read the datablocks :param read_experiments: Try to read the experiments :param read_reflections: Try to read the reflections :param read_datablocks_from_images: Try to read the datablocks from images :param check_format: Check the format when reading images ''' from dials.util.phil import parse # Set the system phil scope if phil is None: self._system_phil = parse("") else: self._system_phil = phil # Set the flags self._read_datablocks = read_datablocks self._read_experiments = read_experiments self._read_reflections = read_reflections self._read_datablocks_from_images = read_datablocks_from_images self._check_format = check_format # Adopt the input scope input_phil_scope = self._generate_input_scope() if input_phil_scope is not None: self.system_phil.adopt_scope(input_phil_scope) # Set the working phil scope self._phil = self.system_phil.fetch(source=parse(""))
def test_few_reflections(dials_data, run_in_tmpdir): u""" Test that dials.symmetry does something sensible if given few reflections. Use some example integrated data generated from a ten-image 1° sweep. These contain a few dozen integrated reflections. Args: dials_data: DIALS custom Pytest fixture for access to test data. run_in_tmpdir: DIALS custom Pytest fixture to run this test in a temporary directory. """ # Get and use the default parameters for dials.symmetry. params = symmetry.phil_scope.fetch(source=parse("")).extract() # Use the integrated data from the first ten images of the first sweep. data_dir = dials_data("l_cysteine_dials_output") experiments = ExperimentList.from_file(data_dir / "11_integrated.expt") reflections = [flex.reflection_table.from_file(data_dir / "11_integrated.refl")] # Run dials.symmetry on the above data files. symmetry.symmetry(experiments, reflections, params)
def from_observations(experiments, params=None): """ Construct a reflection table from observations. :param experiments: The experiments :param params: The input parameters :return: The reflection table of observations """ from dials.algorithms.spot_finding.factory import SpotFinderFactory if params is None: from dials.command_line.find_spots import phil_scope from dials.util.phil import parse params = phil_scope.fetch(source=parse("")).extract() if params.spotfinder.filter.min_spot_size is libtbx.Auto: detector = experiments[0].imageset.get_detector() if detector[0].get_type() == "SENSOR_PAD": # smaller default value for pixel array detectors params.spotfinder.filter.min_spot_size = 3 else: params.spotfinder.filter.min_spot_size = 6 logger.info( "Setting spotfinder.filter.min_spot_size=%i", params.spotfinder.filter.min_spot_size, ) # Get the integrator from the input parameters logger.info("Configuring spot finder from input parameters") find_spots = SpotFinderFactory.from_parameters( experiments=experiments, params=params ) # Find the spots return find_spots(experiments)
def parse_args( self, args, verbose=False, return_unhandled=False, quick_parse=False ): """ Parse the command line arguments. :param args: The input arguments :param verbose: Print verbose output :param return_unhandled: True/False also return unhandled arguments :param quick_parse: Return as fast as possible and without reading any data, ignoring class constructor options. :return: The options and parameters and (optionally) unhandled arguments """ from dxtbx.model.experiment_list import ( BeamComparison, DetectorComparison, GoniometerComparison, ) from dials.util.phil import parse # Parse the command line phil parameters user_phils = [] unhandled = [] interpretor = self.system_phil.command_line_argument_interpreter() def _is_a_phil_file(filename): return any( filename.endswith(phil_ext) for phil_ext in (".phil", ".param", ".params", ".eff", ".def") ) for arg in args: if ( _is_a_phil_file(arg) and os.path.isfile(arg) and os.path.getsize(arg) > 0 ): try: user_phils.append(parse(file_name=arg)) except Exception: if return_unhandled: unhandled.append(arg) else: raise # Treat "has a schema" as "looks like a URL (not phil) elif "=" in arg and not get_url_scheme(arg): try: user_phils.append(interpretor.process_arg(arg=arg)) except Exception: if return_unhandled: unhandled.append(arg) else: raise else: unhandled.append(arg) # Fetch the phil parameters self._phil, unused = self.system_phil.fetch( sources=user_phils, track_unused_definitions=True ) # Print if bad definitions if len(unused) > 0: msg = [item.object.as_str().strip() for item in unused] msg = "\n".join([" %s" % line for line in msg]) raise RuntimeError(f"The following definitions were not recognised\n{msg}") # Extract the parameters params = self._phil.extract() # Stop at this point if quick_parse is set. A second pass may be needed. if quick_parse: return params, unhandled # Create some comparison functions if self._read_experiments_from_images: compare_beam = BeamComparison( wavelength_tolerance=params.input.tolerance.beam.wavelength, direction_tolerance=params.input.tolerance.beam.direction, polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal, polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction, ) compare_detector = DetectorComparison( fast_axis_tolerance=params.input.tolerance.detector.fast_axis, slow_axis_tolerance=params.input.tolerance.detector.slow_axis, origin_tolerance=params.input.tolerance.detector.origin, ) compare_goniometer = GoniometerComparison( rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis, fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation, setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation, ) scan_tolerance = params.input.tolerance.scan.oscillation # FIXME Should probably make this smarter since it requires editing here # and in dials.import phil scope try: format_kwargs = { "dynamic_shadowing": params.format.dynamic_shadowing, "multi_panel": params.format.multi_panel, } except AttributeError: format_kwargs = None else: compare_beam = None compare_detector = None compare_goniometer = None scan_tolerance = None format_kwargs = None try: load_models = params.load_models except AttributeError: load_models = True # Try to import everything importer = Importer( unhandled, read_experiments=self._read_experiments, read_reflections=self._read_reflections, read_experiments_from_images=self._read_experiments_from_images, check_format=self._check_format, verbose=verbose, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, scan_tolerance=scan_tolerance, format_kwargs=format_kwargs, load_models=load_models, ) # Grab a copy of the errors that occurred in case the caller wants them self.handling_errors = importer.handling_errors # Add the cached arguments for obj in importer.experiments: params.input.experiments.append(obj) for obj in importer.reflections: params.input.reflections.append(obj) # Convert to phil self._phil = self.system_phil.format(python_object=params) return params, importer.unhandled
import logging import pickle from dials.util.phil import parse logger = logging.getLogger(__name__) phil_scope = parse(""" min_count = 5 .type = int(value_min=1) nsigma = 6 .type = float(value_min=0) sigma = 0.5 .type = float(value_min=0) kernel_size = 9 .type = int niter = 10 .type = int """) class Modeller: def __init__(self, beam, detector, min_count=5,
def parse_args(self, args, verbose=False, return_unhandled=False, quick_parse=False): ''' Parse the command line arguments. :param args: The input arguments :param verbose: Print verbose output :param return_unhandled: True/False also return unhandled arguments :param quick_parse: Return as fast as possible and without reading any data, ignoring class constructor options. :return: The options and parameters and (optionally) unhandled arguments ''' import os.path from dxtbx.datablock import BeamComparison from dxtbx.datablock import DetectorComparison from dxtbx.datablock import GoniometerComparison from dials.util.phil import parse # Parse the command line phil parameters user_phils = [] unhandled = [] interpretor = self.system_phil.command_line_argument_interpreter() for arg in args: if os.path.isfile(arg) and os.path.getsize(arg) > 0: name, ext = os.path.splitext(arg) if ext in ['.phil', '.param', '.params', '.eff', '.def']: try: user_phils.append(parse(file_name=arg)) except Exception: if return_unhandled: unhandled.append(arg) else: raise else: unhandled.append(arg) elif arg.find("=") >= 0: try: user_phils.append(interpretor.process_arg(arg=arg)) except Exception: if return_unhandled: unhandled.append(arg) else: raise else: unhandled.append(arg) # Fetch the phil parameters self._phil, unused = self.system_phil.fetch( sources=user_phils, track_unused_definitions=True) # Print if bad definitions if len(unused) > 0: msg = [item.object.as_str().strip() for item in unused] msg = '\n'.join([' %s' % line for line in msg]) raise RuntimeError( 'The following definitions were not recognised\n%s' % msg) # Extract the parameters params = self._phil.extract() # Stop at this point if quick_parse is set. A second pass may be needed. if quick_parse: return params, unhandled # Create some comparison functions if self._read_datablocks_from_images: compare_beam = BeamComparison( wavelength_tolerance=params.input.tolerance.beam.wavelength, direction_tolerance=params.input.tolerance.beam.direction, polarization_normal_tolerance=params.input.tolerance.beam. polarization_normal, polarization_fraction_tolerance=params.input.tolerance.beam. polarization_fraction) compare_detector = DetectorComparison( fast_axis_tolerance=params.input.tolerance.detector.fast_axis, slow_axis_tolerance=params.input.tolerance.detector.slow_axis, origin_tolerance=params.input.tolerance.detector.origin) compare_goniometer = GoniometerComparison( rotation_axis_tolerance=params.input.tolerance.goniometer. rotation_axis, fixed_rotation_tolerance=params.input.tolerance.goniometer. fixed_rotation, setting_rotation_tolerance=params.input.tolerance.goniometer. setting_rotation) scan_tolerance = params.input.tolerance.scan.oscillation # FIXME Should probably make this smarter since it requires editing here # and in dials.import phil scope try: format_kwargs = { 'dynamic_shadowing': params.format.dynamic_shadowing, 'multi_panel': params.format.multi_panel, } except Exception: format_kwargs = None else: compare_beam = None compare_detector = None compare_goniometer = None scan_tolerance = None format_kwargs = None # Try to import everything importer = Importer( unhandled, read_datablocks=self._read_datablocks, read_experiments=self._read_experiments, read_reflections=self._read_reflections, read_datablocks_from_images=self._read_datablocks_from_images, check_format=self._check_format, verbose=verbose, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, scan_tolerance=scan_tolerance, format_kwargs=format_kwargs) # Grab a copy of the errors that occured in case the caller wants them self.handling_errors = importer.handling_errors # Add the cached arguments for obj in importer.datablocks: params.input.datablock.append(obj) for obj in importer.experiments: params.input.experiments.append(obj) for obj in importer.reflections: params.input.reflections.append(obj) # Convert to phil self._phil = self.system_phil.format(python_object=params) return params, importer.unhandled
def generate_phil_scope(): ''' Generate the integration phil scope. :return: The phil scope ''' import dials.extensions from dials.interfaces import BackgroundIface from dials.interfaces import CentroidIface phil_scope = phil.parse(''' integration { include scope dials.data.lookup.phil_scope block { size = auto .type = float .help = "The block size in rotation angle (degrees)." units = *degrees radians frames .type = choice .help = "The units of the block size" threshold = 0.99 .type = float(value_min=0.0, value_max=1.0) .help = "For block size auto the block size is calculated by sorting" "reflections by the number of frames they cover and then" "selecting the block size to be 2*nframes[threshold] such" "that 100*threshold % of reflections are guarenteed to be" "fully contained in 1 block" force = False .type = bool .help = "If the number of processors is 1 and force is False, then the" "number of blocks may be set to 1. If force is True then the" "block size is always calculated." max_memory_usage = 0.75 .type = float(value_min=0.0,value_max=1.0) .help = "The maximum percentage of total physical memory to use for" "allocating shoebox arrays." } debug { reference { filename = "reference_profiles.pickle" .type = str .help = "The filename for the reference profiles" output = False .type = bool .help = "Save the reference profiles" } during = modelling *integration .type = choice .help = "Do debugging during modelling or integration" output = False .type = bool .help = "Save shoeboxes after each processing task." separate_files = True .type = bool .help = "If this is true, the shoeboxes are saved in separate files" "from the output integrated.pickle file. This is necessary" "in most cases since the amount of memory used by the" "shoeboxes is typically greater than the available system" "memory. If, however, you know that memory is not an issue," "you can saved the shoeboxes in the integrated.pickle file" "by setting this option to False. This only works if the debug" "output is during integrated and not modelling." select = None .type = reflection_table_selector .help = "A string specifying the selection. The string should be of the" "form: select=${COLUMN}[<|<=|==|!=|>=|>]${VALUE}. In addition" "to the items in the reflection table, the following implicit" "columns are defined if the necessary data is there:" " intensity.sum.i_over_sigma" " intensity.prf.i_over_sigma" split_experiments = True .type = bool .help = "Split shoeboxes into different files" } integrator = *auto 3d flat3d 2d single2d stills volume .type = choice .help = "The integrator to use." .expert_level=3 profile { fitting = True .type = bool .help = "Use profile fitting if available" validation { number_of_partitions = 1 .type = int(value_min=1) .help = "The number of subsamples to take from the reference spots." "If the value is 1, then no validation is performed." min_partition_size = 100 .type = int(value_min=1) .help = "The minimum number of spots to use in each subsample." } } filter .expert_level = 1 { min_zeta = 0.05 .help = "Filter the reflections by the value of zeta. A value of less" "than or equal to zero indicates that this will not be used. A" "positive value is used as the minimum permissable value." .type = float(value_min=0.0, value_max=1.0) max_shoebox_overlap = 1.0 .type = float(value_min=0.0, value_max=1.0) .help = "Filter reflections whose shoeboxes are overlapped by greater" "than the requested amount. Note that this is not the" "percentage of the peak that is overlapped but rather the" "percentage of the shoebox (background and foreground). This" "can be useful when the detector is too close and many" "overlapping reflections are predicted at high resolution" "causing memory issues." ice_rings = False .help = "Set the ice ring flags" .type = bool } include scope dials.algorithms.integration.overlaps_filter.phil_scope mp { method = *none drmaa sge lsf pbs .type = choice .help = "The cluster method to use" njobs = 1 .type = int(value_min=1) .help = "The number of cluster jobs to use" nproc = 1 .type = int(value_min=1) .help = "The number of processes to use per cluster job" } } ''', process_includes=True) main_scope = phil_scope.get_without_substitution("integration") assert len(main_scope) == 1 main_scope = main_scope[0] main_scope.adopt_scope(BackgroundIface.phil_scope()) main_scope.adopt_scope(CentroidIface.phil_scope()) return phil_scope