def test_experimentlist_change_basis(dials_data): experiments = ExperimentList() for i in range(4): experiments.extend( ExperimentList.from_file( dials_data("vmxi_proteinase_k_sweeps") / ("experiments_%i.expt" % i), check_format=False, )) reindexed_uc = (68.368, 103.968, 68.368, 90.000, 90.000, 90.000) reindexed_sg = sgtbx.space_group_info("P 4 2 2 (b,c,a)").group() cb_op = sgtbx.change_of_basis_op("-a,-c,-b") for cb_op in (cb_op, [cb_op] * len(experiments)): expts_rdx = experiments.change_basis(cb_op) for expt in expts_rdx: assert expt.crystal.get_unit_cell().parameters() == pytest.approx( reindexed_uc, abs=0.1) assert expt.crystal.get_space_group() == reindexed_sg experiments.change_basis(cb_op, in_place=True) for expt in experiments: assert expt.crystal.get_unit_cell().parameters() == pytest.approx( reindexed_uc, abs=0.1) assert expt.crystal.get_space_group() == reindexed_sg with pytest.raises(AssertionError): experiments.change_basis([cb_op, cb_op])
def protk_experiments_and_reflections(dials_data): data_dir = dials_data("multi_crystal_proteinase_k") # Load experiments experiments = ExperimentList() for expt_file in sorted(f.strpath for f in data_dir.listdir("experiments*.json")): experiments.extend(load.experiment_list(expt_file, check_format=False)) # Load reflection tables reflections = [ flex.reflection_table.from_file(refl_file) for refl_file in sorted( f.strpath for f in data_dir.listdir("reflections*.pickle")) ] # Setup experiment identifiers reflections = parse_multiple_datasets(reflections) experiments, reflections = assign_unique_identifiers( experiments, reflections) # Combine into single ExperimentList reflections_all = flex.reflection_table() for i, (expt, refl) in enumerate(zip(experiments, reflections)): reflections_all.extend(refl) reflections_all.assert_experiment_identifiers_are_consistent(experiments) return experiments, reflections_all
def test_update_imageset_ids(dials_data): expts = ExperimentList() refls = [] for i in [1, 2, 3, 4, 5, 7, 8, 10]: refls.append( flex.reflection_table.from_file( dials_data("multi_crystal_proteinase_k", pathlib=True) / f"reflections_{i}.pickle")) expts.extend( load.experiment_list( dials_data("multi_crystal_proteinase_k", pathlib=True) / f"experiments_{i}.json", check_format=False, )) # first make sure ids are set up correctly. experiments, reflections = assign_unique_identifiers(expts, refls) reflections = update_imageset_ids(experiments, reflections) joint_reflections = flex.reflection_table() for refls in reflections: joint_reflections.extend(refls) # check that there are 8 unique id and imageset_ids, and that these # correctly correspond to each experiment assert len(set(joint_reflections["id"])) == 8 assert len(set(joint_reflections["imageset_id"])) == 8 for id_ in range(8): sel = joint_reflections["id"] == id_ assert set(joint_reflections["imageset_id"].select(sel)) == {id_}
def from_filenames( filenames, verbose=False, unhandled=None, compare_beam=None, compare_detector=None, compare_goniometer=None, scan_tolerance=None, format_kwargs=None, load_models=True, ): """Create a list of data blocks from a list of directory or file names.""" experiments = ExperimentList() for db in DataBlockFactory.from_filenames( filenames, verbose=verbose, unhandled=unhandled, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, scan_tolerance=scan_tolerance, format_kwargs=format_kwargs, ): experiments.extend( ExperimentListFactory.from_datablock_and_crystal(db, None, load_models) ) return experiments
def from_datablock_and_crystal(datablock, crystal, load_models=True): """Load an experiment list from a datablock.""" # Initialise the experiment list experiments = ExperimentList() # If we have a list, loop through if isinstance(datablock, list): for db in datablock: experiments.extend( ExperimentListFactory.from_datablock_and_crystal( db, crystal, load_models ) ) return experiments # Add all the imagesets for imageset in datablock.extract_imagesets(): experiments.extend( ExperimentListFactory.from_imageset_and_crystal( imageset, crystal, load_models ) ) # Check the list is consistent assert experiments.is_consistent() return experiments
def test_multi_sweep_fixed_rotation(dials_regression, run_in_tmpdir): data_dir = os.path.join(dials_regression, "indexing_test_data", "multi_sweep") reflection_files = sorted( glob.glob( os.path.join(data_dir, "SWEEP[1,2]", "index", "*_strong.pickle"))) experiment_files = sorted( glob.glob( os.path.join(data_dir, "SWEEP[1,2]", "index", "*_datablock_import.json"))) search_beam_position.run(reflection_files + experiment_files) assert os.path.exists("optimised.expt") experiments = ExperimentList() for path in experiment_files: experiments.extend(load.experiment_list(path, check_format=False)) optimised_experiments = load.experiment_list("optimised.expt", check_format=False) for orig_expt, new_expt in zip(experiments, optimised_experiments): shift = scitbx.matrix.col( orig_expt.detector[0].get_origin()) - scitbx.matrix.col( new_expt.detector[0].get_origin()) print(shift) assert shift.elems == pytest.approx((2.293, -0.399, 0), abs=1e-2)
def from_args(args, verbose=False, unhandled=None): ''' Try to load experiment from any recognised format. ''' from dxtbx.datablock import DataBlockFactory # Create a list for unhandled arguments if unhandled is None: unhandled = [] experiments = ExperimentList() ## First try as image files #experiments = ExperimentListFactory.from_datablock( #DataBlockFactory.from_args(args, verbose, unhandled1)) # Try to load from serialized formats for filename in args: try: experiments.extend( ExperimentListFactory.from_serialized_format(filename)) if verbose: print('Loaded experiments from %s' % filename) except Exception as e: if verbose: print("Could not load experiments from %s: %s" % (filename, str(e))) unhandled.append(filename) # Return the experiments return experiments
class ExperimentListTemplateImporter(object): """A class to import an experiment list from a template.""" def __init__(self, templates, **kwargs): importer = DataBlockTemplateImporter(templates, **kwargs) self.experiments = ExperimentList() for db in importer.datablocks: self.experiments.extend( ExperimentListFactory.from_datablock_and_crystal(db, None))
def flatten_experiments(filename_object_list): """ Flatten a list of experiment lists :param filename_object_list: The parameter item :return: The flattened experiment lists """ result = ExperimentList() for o in filename_object_list: result.extend(o.data) return result
def from_args(args, unhandled=None): """Try to load serialised experiments from any recognised format.""" # Create a list for unhandled arguments if unhandled is None: unhandled = [] experiments = ExperimentList() # Try to load from serialized formats for filename in args: try: experiments.extend( ExperimentListFactory.from_serialized_format(filename)) logger.debug(f"Loaded experiments from {filename}") except Exception as e: logger.debug( f"Could not load experiments from {filename}: {e}") unhandled.append(filename) return experiments
def from_args(args, verbose=False, unhandled=None): """Try to load serialised experiments from any recognised format.""" # Create a list for unhandled arguments if unhandled is None: unhandled = [] experiments = ExperimentList() # Try to load from serialized formats for filename in args: try: experiments.extend( ExperimentListFactory.from_serialized_format(filename) ) if verbose: print("Loaded experiments from %s" % filename) except Exception as e: if verbose: print("Could not load experiments from %s: %s" % (filename, str(e))) unhandled.append(filename) return experiments
def from_dict(obj, check_format=True, directory=None): """ Load an experiment list from a dictionary. Args: obj (dict): Dictionary containing either ExperimentList or DataBlock structure. check_format (bool): If True, the file will be read to verify metadata. directory (str): Returns: ExperimentList: The dictionary converted """ try: experiments = ExperimentList() for db in DataBlockFactory.from_dict( obj, check_format=check_format, directory=directory ): experiments.extend( ExperimentListFactory.from_datablock_and_crystal(db, None) ) except Exception: experiments = None # Decode the experiments from the dictionary if experiments is None: experiments = ExperimentListDict( obj, check_format=check_format, directory=directory ).decode() # Check the list is consistent assert experiments.is_consistent() return experiments
def load_imagesets( template, directory, id_image=None, image_range=None, use_cache=True, reversephi=False, ): global imageset_cache from dxtbx.model.experiment_list import ExperimentListFactory from xia2.Applications.xia2setup import known_hdf5_extensions from dxtbx.imageset import ImageSequence full_template_path = os.path.join(directory, template) if full_template_path not in imageset_cache or not use_cache: from dxtbx.model.experiment_list import BeamComparison from dxtbx.model.experiment_list import DetectorComparison from dxtbx.model.experiment_list import GoniometerComparison params = PhilIndex.params.xia2.settings compare_beam = BeamComparison( wavelength_tolerance=params.input.tolerance.beam.wavelength, direction_tolerance=params.input.tolerance.beam.direction, polarization_normal_tolerance=params.input.tolerance.beam. polarization_normal, polarization_fraction_tolerance=params.input.tolerance.beam. polarization_fraction, ) compare_detector = DetectorComparison( fast_axis_tolerance=params.input.tolerance.detector.fast_axis, slow_axis_tolerance=params.input.tolerance.detector.slow_axis, origin_tolerance=params.input.tolerance.detector.origin, ) compare_goniometer = GoniometerComparison( rotation_axis_tolerance=params.input.tolerance.goniometer. rotation_axis, fixed_rotation_tolerance=params.input.tolerance.goniometer. fixed_rotation, setting_rotation_tolerance=params.input.tolerance.goniometer. setting_rotation, ) scan_tolerance = params.input.tolerance.scan.oscillation # If diamond anvil cell data, always use dynamic shadowing high_pressure = PhilIndex.params.dials.high_pressure.correction format_kwargs = { "dynamic_shadowing": params.input.format.dynamic_shadowing or high_pressure, "multi_panel": params.input.format.multi_panel, } if os.path.splitext(full_template_path)[-1] in known_hdf5_extensions: # if we are passed the correct file, use this, else look for a master # file (i.e. something_master.h5) if os.path.exists(full_template_path) and os.path.isfile( full_template_path): master_file = full_template_path else: g = glob.glob(os.path.join(directory, "*_master.h5")) master_file = None for p in g: substr = longest_common_substring(template, p) if substr: if master_file is None or (len(substr) > len( longest_common_substring( template, master_file))): master_file = p if master_file is None: raise RuntimeError("Can't find master file for %s" % full_template_path) unhandled = [] experiments = ExperimentListFactory.from_filenames( [master_file], verbose=False, unhandled=unhandled, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, scan_tolerance=scan_tolerance, format_kwargs=format_kwargs, ) assert len(unhandled) == 0, ( "unhandled image files identified: %s" % unhandled) else: from dxtbx.sequence_filenames import locate_files_matching_template_string params = PhilIndex.get_python_object() read_all_image_headers = params.xia2.settings.read_all_image_headers if read_all_image_headers: paths = sorted( locate_files_matching_template_string(full_template_path)) unhandled = [] experiments = ExperimentListFactory.from_filenames( paths, verbose=False, unhandled=unhandled, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, scan_tolerance=scan_tolerance, format_kwargs=format_kwargs, ) assert len(unhandled) == 0, ( "unhandled image files identified: %s" % unhandled) else: from xia2.Handlers.CommandLine import CommandLine experiments = ExperimentList() start_ends = CommandLine.get_start_ends(full_template_path) if not start_ends: start_ends.append(None) for start_end in start_ends: importer = ExperimentListTemplateImporter( [full_template_path], format_kwargs=format_kwargs, image_range=start_end, ) experiments.extend(importer.experiments) imagesets = [ iset for iset in experiments.imagesets() if isinstance(iset, ImageSequence) ] assert len(imagesets) > 0, "no imageset found" imageset_cache[full_template_path] = collections.OrderedDict() if reversephi: for imageset in imagesets: goniometer = imageset.get_goniometer() goniometer.set_rotation_axis( tuple(-g for g in goniometer.get_rotation_axis())) reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry if reference_geometry is not None and len(reference_geometry) > 0: update_with_reference_geometry(imagesets, reference_geometry) # Update the geometry params = PhilIndex.params.xia2.settings update_geometry = [] from dials.command_line.dials_import import ManualGeometryUpdater from dials.util.options import geometry_phil_scope # Then add manual geometry work_phil = geometry_phil_scope.format(params.input) diff_phil = geometry_phil_scope.fetch_diff(source=work_phil) if diff_phil.as_str() != "": update_geometry.append(ManualGeometryUpdater(params.input)) imageset_list = [] for imageset in imagesets: for updater in update_geometry: imageset = updater(imageset) imageset_list.append(imageset) imagesets = imageset_list for imageset in imagesets: scan = imageset.get_scan() exposure_times = scan.get_exposure_times() epochs = scan.get_epochs() if exposure_times.all_eq(0) or exposure_times[0] == 0: exposure_times = flex.double(exposure_times.size(), 1) scan.set_exposure_times(exposure_times) elif not exposure_times.all_gt(0): exposure_times = flex.double(exposure_times.size(), exposure_times[0]) scan.set_exposure_times(exposure_times) if epochs.size() > 1 and not epochs.all_gt(0): if epochs[0] == 0: epochs[0] = 1 for i in range(1, epochs.size()): epochs[i] = epochs[i - 1] + exposure_times[i - 1] scan.set_epochs(epochs) _id_image = scan.get_image_range()[0] imageset_cache[full_template_path][_id_image] = imageset if id_image is not None: return [imageset_cache[full_template_path][id_image]] elif image_range is not None: for imageset in imageset_cache[full_template_path].values(): scan = imageset.get_scan() scan_image_range = scan.get_image_range() if (image_range[0] >= scan_image_range[0] and image_range[1] <= scan_image_range[1]): b0 = scan.get_batch_offset() i0 = image_range[0] - scan_image_range[0] + b0 i1 = image_range[1] - scan_image_range[0] + b0 imagesets = [imageset[i0:i1 + 1]] assert len( imagesets[0]) == image_range[1] - image_range[0] + 1, len( imagesets[0]) return imagesets return list(imageset_cache[full_template_path].values())
def test_experiment_list_extend(): """Check that the extend method of ExperimentList works.""" # Create a minimal ExperimentList instance. expts = ExperimentList([Experiment()]) # Try to extend it. expts.extend(expts)
class Screen19(object): """Encapsulates the screening script.""" def __init__(self): # Throughout the pipeline, retain the state of the processing. self.expts = ExperimentList([]) self.refls = flex.reflection_table() # Get some default parameters. These must be extracted from the 'fetched' # PHIL scope, rather than the 'definition' phil scope returned by # iotbx.phil.parse. Confused? Blame PHIL. self.params = phil_scope.fetch(iotbx.phil.parse("")).extract() def _quick_import(self, files): # type: (List[str]) -> bool """ Generate xia2-style templates from file names and attempt a quick import. From each given filename, generate a filename template by substituting a hash character (#) for each numeral in the last contiguous group of numerals before the file extension. For example, the filename `example_01_0001.cbf` becomes `example_01_####.cbf`. Contiguous image ranges are recorded by associating the start and end image number of the range with the relevant filename template. dials.import is then run with options to extrapolate header information from the first image file, thereby running more quickly than reading each image header individually. Args: files: List of image filenames. Returns: Boolean flag indicating whether the quick import has succeeded. """ if len(files) == 1: # No point in quick-importing a single file return False debug("Attempting quick import...") files.sort() templates = {} # type: Dict[str, List[Optional[List[int]]]] for f in files: template, image = screen19.make_template(f) if template not in templates: image_range = [image, image] if image else [] templates.update({template: [image_range]}) elif image == templates[template][-1][-1] + 1: templates[template][-1][-1] = image elif image == templates[template][-1][-1]: # We have a duplicate input file name. Do nothing. pass else: templates[template].append([image, image]) # Return tuple of template and image range for each unique image range templates = [(t, tuple(r)) for t, ranges in templates.items() for r in ranges] # type: Templates return self._quick_import_templates(templates) def _quick_import_templates(self, templates): # type: (Templates) -> bool """ Take image file templates and frame number ranges and try to run dials.import. dials.import is run with options to extrapolate header information from the first image file, thereby running more quickly than reading each image header individually. Args: templates: A list of tuples, each tuple containing a xia2-style filename template and the start and end image numbers of the associated sweep. Returns: Boolean flag indicating whether the quick import has succeeded. """ debug("Quick import template summary:\n\t%s", templates) if len(templates) > 1: debug("Cannot currently run quick import on multiple templates.") return False try: scan_range = templates[0][1] # type: Tuple[int, int] if not scan_range: raise IndexError except IndexError: debug( "Cannot run quick import: could not determine image naming template." ) return False info("Running quick import.") self.params.dials_import.input.template = [templates[0][0]] self.params.dials_import.geometry.scan.image_range = scan_range self.params.dials_import.geometry.scan.extrapolate_scan = True self._run_dials_import() return True def _import(self, files): # type: (List[str]) -> None """ Try to run a quick call of dials.import. Failing that, run a slow call. Try initially to construct file name templates contiguous groups of files. Failing that, pass a full list of the files to the importer (slower). Args: files: List of image filenames. """ info("\nImporting data...") if len(files) == 1: if os.path.isdir(files[0]): debug("You specified a directory. Importing all CBF files in " "that directory.") # TODO Support HDF5. files = [ os.path.join(files[0], f) for f in os.listdir(files[0]) if f.endswith(".cbf") or f.endswith(".cbf.gz") or f.endswith(".cbf.bz2") ] elif len(files[0].split(":")) == 3: debug("You specified an image range in the xia2 format. " "Importing all specified files.") template, start, end = files[0].split(":") template = screen19.make_template(template)[0] start, end = int(start), int(end) if not self._quick_import_templates([(template, (start, end))]): warning("Could not import specified image range.") sys.exit(1) info("Quick import successful.") return elif files[0].endswith(".expt"): debug("You specified an existing experiment list file. " "No import necessary.") try: self.expts = ExperimentList.from_file(files[0]) except (IOError, PickleError, ValueError): pass else: self.params.dials_import.output.experiments = files[0] if self.expts: return if not files: warning("No images found matching input.") sys.exit(1) # Can the files be quick-imported? if self._quick_import(files): info("Quick import successful.") return self.params.dials_import.input.experiments = files self._run_dials_import() def _run_dials_import(self): """ Perform a minimal version of dials.import to get an experiment list. Use some filleted bits of dials.import and dials.util.options.Importer. """ # Get some key data format arguments. try: format_kwargs = { "dynamic_shadowing": self.params.dials_import.format.dynamic_shadowing, "multi_panel": self.params.dials_import.format.multi_panel, } except AttributeError: format_kwargs = {} # If filenames contain wildcards, expand args = [] for arg in self.params.dials_import.input.experiments: if "*" in arg: args.extend(glob(arg)) else: args.append(arg) if args: # Are compare{beam,detector,goniometer} and scan_tolerance necessary? # They are cargo-culted from the DIALS option parser. tol_params = self.params.dials_import.input.tolerance compare_beam = BeamComparison( wavelength_tolerance=tol_params.beam.wavelength, direction_tolerance=tol_params.beam.direction, polarization_normal_tolerance=tol_params.beam. polarization_normal, polarization_fraction_tolerance=tol_params.beam. polarization_fraction, ) compare_detector = DetectorComparison( fast_axis_tolerance=tol_params.detector.fast_axis, slow_axis_tolerance=tol_params.detector.slow_axis, origin_tolerance=tol_params.detector.origin, ) compare_goniometer = GoniometerComparison( rotation_axis_tolerance=tol_params.goniometer.rotation_axis, fixed_rotation_tolerance=tol_params.goniometer.fixed_rotation, setting_rotation_tolerance=tol_params.goniometer. setting_rotation, ) scan_tolerance = tol_params.scan.oscillation # Import an experiment list from image data. try: experiments = ExperimentListFactory.from_filenames( args, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, scan_tolerance=scan_tolerance, format_kwargs=format_kwargs, ) except IOError as e: warning("%s '%s'", e.strerror, e.filename) sys.exit(1) # Record the imported experiments for use elsewhere. # Quit if there aren't any. self.expts.extend(experiments) if not self.expts: warning("No images found.") sys.exit(1) else: # Use the template importer. if len(self.params.dials_import.input.template) > 0: importer = ExperimentListTemplateImporter( self.params.dials_import.input.template, format_kwargs=format_kwargs) # Record the imported experiments for use elsewhere. # Quit if there aren't any. self.expts.extend(importer.experiments) if not self.expts: warning("No images found matching template %s" % self.params.dials_import.input.template[0]) sys.exit(1) # Setup the metadata updater metadata_updater = MetaDataUpdater(self.params.dials_import) # Extract the experiments and loop through self.expts = metadata_updater(self.expts.imagesets()) def _count_processors(self, nproc=None): # type: (Optional[int]) -> None """ Determine the number of processors and save it as an instance variable. The user may specify the number of processors to use. If no value is given, the number of available processors is returned. Args: nproc (optional): Number of processors. """ if nproc and nproc is not Auto: self.nproc = nproc return # if environmental variable NSLOTS is set to a number then use that try: self.nproc = int(os.environ.get("NSLOTS")) return except (ValueError, TypeError): pass self.nproc = number_of_processors(return_value_if_unknown=-1) if self.nproc <= 0: warning( "Could not determine number of available processors. Error code %d", self.nproc, ) sys.exit(1) def _count_images(self): # type: () -> int """ Attempt to determine the number of diffraction images. The number of diffraction images is determined from the imported_experiments JSON file. Returns: Number of images. """ # FIXME: This exception handling should be redundant. Empty experiment # lists should get caught at the import stage. Is this so? try: return self.expts[0].imageset.size() except IndexError: warning("Could not determine number of images in dataset.") sys.exit(1) def _check_intensities(self, mosaicity_correction=True): # type: (bool) -> None """ Run xia2.overload and plot a histogram of pixel intensities. If `mosaicity_correction` is true, the pixel intensities are approximately adjusted to take account of a systematic defect in the detector count rate correction. See https://github.com/xia2/screen19/wiki#mosaicity-correction Args: mosaicity_correction (optional): default is `True`. """ info("\nTesting pixel intensities...") command = ["xia2.overload", "nproc=%s" % self.nproc, "indexed.expt"] debug("running %s", command) result = procrunner.run(command, print_stdout=False, debug=procrunner_debug) debug("result = %s", screen19.prettyprint_dictionary(result)) info("Successfully completed (%.1f sec)", result["runtime"]) if result["exitcode"] != 0: warning("Failed with exit code %d", result["exitcode"]) sys.exit(1) with open("overload.json") as fh: overload_data = json.load(fh) info("Pixel intensity distribution:") count_sum = 0 hist = {} if "bins" in overload_data: for b in range(overload_data["bin_count"]): if overload_data["bins"][b] > 0: hist[b] = overload_data["bins"][b] count_sum += b * overload_data["bins"][b] else: hist = { int(k): v for k, v in overload_data["counts"].items() if int(k) > 0 } count_sum = sum([k * v for k, v in hist.items()]) average_to_peak = 1 if mosaicity_correction: # Adjust for the detector count rate correction if self._sigma_m: delta_z = self._oscillation / self._sigma_m / math.sqrt(2) average_to_peak = ( math.sqrt(math.pi) * delta_z * math.erf(delta_z) + math.exp(-(delta_z**2)) - 1) / delta_z**2 info("Average-to-peak intensity ratio: %f", average_to_peak) scale = 100 * overload_data["scale_factor"] / average_to_peak info("Determined scale factor for intensities as %f", scale) debug( "intensity histogram: { %s }", ", ".join(["%d:%d" % (k, hist[k]) for k in sorted(hist)]), ) max_count = max(hist.keys()) hist_max = max_count * scale hist_granularity, hist_format = 1, "%.0f" if hist_max < 50: hist_granularity, hist_format = 2, "%.1f" if hist_max < 15: hist_granularity, hist_format = 10, "%.1f" rescaled_hist = {} for x in hist.keys(): rescaled = round(x * scale * hist_granularity) if rescaled > 0: rescaled_hist[rescaled] = hist[x] + rescaled_hist.get( rescaled, 0) hist = rescaled_hist debug( "rescaled histogram: { %s }", ", ".join([(hist_format + ":%d") % (k / hist_granularity, hist[k]) for k in sorted(hist)]), ) screen19.plot_intensities(hist, 1 / hist_granularity, procrunner_debug=procrunner_debug) linear_response_limit = 100 * self.params.maximum_flux.trusted_range_correction marginal_limit = max(70, linear_response_limit) text = "".join(( "Strongest pixel (%d counts) " % max_count, "reaches %.1f%% " % hist_max, "of the detector count rate limit", )) if hist_max > 100: warning("Warning: %s!", text) else: info(text) if ("overload_limit" in overload_data and max_count >= overload_data["overload_limit"]): warning( "Warning: THE DATA CONTAIN REGULAR OVERLOADS!\n" " The photon incidence rate is outside the specified " "limits of the detector.\n" " The built-in detector count rate correction cannot " "adjust for this.\n" " You should aim for count rates below {:.0%} of the " "detector limit.".format( self.params.maximum_flux.trusted_range_correction)) elif hist_max > marginal_limit: warning( "Warning: The photon incidence rate is well outside the " "linear response region of the detector (<{:.0%}).\n" " The built-in detector count rate correction may not be " "able to adjust for this.".format( self.params.maximum_flux.trusted_range_correction)) elif hist_max > linear_response_limit: info("The photon incidence rate is outside the linear response " "region of the detector (<{:.0%}).\n" " The built-in detector count rate correction may be able " "to adjust for this.".format( self.params.maximum_flux.trusted_range_correction)) if not mosaicity_correction: warning( "Warning: Not enough data for proper profile estimation." " The spot intensities are not corrected for mosaicity.\n" " The true photon incidence rate will be higher than the " "given estimate.") info("Total sum of counts in dataset: %d", count_sum) def _find_spots(self, args=None): # type: (Optional[List[str]]) -> None """ Call `dials.find_spots` on the imported experiment list. Args: args (optional): List of any additional PHIL parameters to be used by dials.import. """ info("\nFinding spots...") dials_start = timeit.default_timer() # Use some choice fillets from dials.find_spots # Ignore `args`, use `self.params` # Loop through all the imagesets and find the strong spots self.refls = flex.reflection_table.from_observations( self.expts, self.params.dials_find_spots) # Add n_signal column - before deleting shoeboxes good = MaskCode.Foreground | MaskCode.Valid self.refls["n_signal"] = self.refls["shoebox"].count_mask_values(good) # Delete the shoeboxes if not self.params.dials_find_spots.output.shoeboxes: del self.refls["shoebox"] info( 60 * "-" + "\n%s\n" + 60 * "-" + "\nSuccessfully completed (%.1f sec)", spot_counts_per_image_plot(self.refls), timeit.default_timer() - dials_start, ) def _index(self): # type: () -> bool """ Call `dials.index` on the output of spot finding. Returns: Boolean value indicating whether indexing was successful. """ dials_start = timeit.default_timer() # Prepare max_cell constraint strategies. max_cell = self.params.dials_index.indexing.max_cell # By default, try unconstrained max_cell followed by max_cell=20. # If the user has already specified a max_cell < 20, do not relax to 20Å. cell_constraints = [([], max_cell)] if not max_cell or max_cell is Auto or max_cell > 20: cell_constraints += [(["max_cell constraint"], 20)] # Prepare indexing methods, preferring the real_space_grid_search if a # known unit cell has been specified, otherwise using 3D FFT, then 1D FFT. methods = ([ (["real space grid search"], "real_space_grid_search") ] if self.params.dials_index.indexing.known_symmetry.unit_cell else []) methods += [(["3D FFT"], "fft3d"), (["1D FFT"], "fft1d")] # Cycle through the indexing methods for each of the max_cell constraint # strategies until an indexing solution is found. for i, (max_cell_msg, max_cell) in enumerate(cell_constraints): # Set the max_cell constraint strategy. self.params.dials_index.indexing.max_cell = max_cell for j, (method_msg, method) in enumerate(methods): # Set the indexing method. self.params.dials_index.indexing.method = method # Log a handy message to the user. msg = ("Retrying with " + " and ".join(method_msg + max_cell_msg) if i + j else "Indexing") info("\n%s...", msg) try: # If indexing is successful, break out of the inner loop. self.expts, self.refls = index(self.expts, [self.refls], self.params.dials_index) break except (DialsIndexError, ValueError) as e: # If indexing is unsuccessful, try again with the next # strategy. warning("Failed: %s", str(e)) continue else: # When all the indexing methods are unsuccessful, move onto # the next max_cell constraint strategy and try again. continue # We should only get here if successfully indexed. Break out of the loop break else: # Indexing completely unsuccessful. return False sg_type = self.expts[0].crystal.get_crystal_symmetry().space_group( ).type() symb = sg_type.universal_hermann_mauguin_symbol() unit_cell = self.expts[0].crystal.get_unit_cell() self.refls.as_file(self.params.dials_index.output.reflections) self.expts.as_file(self.params.dials_index.output.experiments) self.refls.as_file(self.params.dials_index.output.reflections) info( "Found primitive solution: %s %s using %s reflections\n" "Indexed experiments and reflections saved as %s, %s\n" "Successfully completed (%.1f sec)", symb, unit_cell, self.refls["id"].count(0), self.params.dials_index.output.experiments, self.params.dials_index.output.reflections, timeit.default_timer() - dials_start, ) # Report the indexing successful. return True def _wilson_calculation(self): # type: () -> None """ Run `screen19.minimum_exposure` on an experiment list and reflection table. For best results, the reflections and experiment list should contain the results of integration or scaling. If only strong spots are used, the Wilson plot fit may be poor. """ dials_start = timeit.default_timer() info("\nEstimating lower exposure bound...") suggest_minimum_exposure(self.expts, self.refls, self.params.minimum_exposure) info("Successfully completed (%.1f sec)", timeit.default_timer() - dials_start) def _refine(self): # type: () -> None """ Run `dials.refine` on the results of indexing. """ dials_start = timeit.default_timer() info("\nRefining...") try: self.expts, self.refls, _, _ = run_dials_refine( self.expts, self.refls, self.params.dials_refine) except Sorry as e: warning("dials.refine failed: %d\nGiving up.\n", e) sys.exit(1) info("Successfully refined (%.1f sec)", timeit.default_timer() - dials_start) def _create_profile_model(self): # type: () -> bool """ Run `dials.create_profile_model` on indexed reflections. The indexed experiment list will be overwritten with a copy that includes the profile model but is otherwise identical. Returns: Boolean value indicating whether it was possible to determine a profile model from the data. """ info("\nCreating profile model...") command = [ "dials.create_profile_model", self.params.dials_index.output.experiments, self.params.dials_index.output.reflections, "output = %s" % self.params.dials_index.output.experiments, ] result = procrunner.run(command, print_stdout=False, debug=procrunner_debug) debug("result = %s", screen19.prettyprint_dictionary(result)) self._sigma_m = None if result["exitcode"] == 0: db = ExperimentList.from_file( self.params.dials_index.output.experiments)[0] self._oscillation = db.imageset.get_scan().get_oscillation()[1] self._sigma_m = db.profile.sigma_m() info( u"%d images, %s° oscillation, σ_m=%.3f°", db.imageset.get_scan().get_num_images(), str(self._oscillation), self._sigma_m, ) info("Successfully completed (%.1f sec)", result["runtime"]) return True warning("Failed with exit code %d", result["exitcode"]) return False def _integrate(self): # type: () -> None """Run `dials.integrate` to integrate reflection intensities.""" dials_start = timeit.default_timer() info("\nIntegrating...") # Don't waste time recreating the profile model self.params.dials_integrate.create_profile_model = False # Get the dials.integrate PHIL scope, populated with parsed input parameters integrate_scope = phil_scope.get("dials_integrate").objects[0] integrate_scope.name = "" integrate_scope = integrate_scope.format(self.params.dials_integrate) try: integrated_experiments, integrated_reflections = _run_integration( integrate_scope, self.params.dials_index.output.experiments, self.params.dials_index.output.reflections, ) # Save the output to files integrated_reflections.as_file( self.params.dials_integrate.output.reflections) integrated_experiments.as_file( self.params.dials_integrate.output.experiments) # ... and also store the output internally self.expts, self.refls = integrated_experiments, integrated_reflections info( "Successfully completed (%.1f sec)", timeit.default_timer() - dials_start, ) except SystemExit as e: if e.code: warning("dials.integrate failed with exit code %d\nGiving up.", e.code) sys.exit(1) # This is a hacky check but should work for as long as DIALS 2.0 is supported. if version.dials_version() < "DIALS 2.1": def _refine_bravais(self, experiments, reflections): # type: (ExperimentList, flex.reflection_table) -> None """ Run `dials.refine_bravais_settings` on an experiments and reflections. Args: experiments: An experiment list.. reflections: The corresponding reflection table. """ info("\nRefining Bravais settings...") command = [ "dials.refine_bravais_settings", experiments, reflections ] result = procrunner.run(command, print_stdout=False, debug=procrunner_debug) debug("result = %s", screen19.prettyprint_dictionary(result)) if result["exitcode"] == 0: m = re.search( r"[-+]{3,}\n[^\n]*\n[-+|]{3,}\n(.*\n)*[-+]{3,}", result["stdout"].decode("utf-8"), ) if m: info(m.group(0)) else: info( "Could not interpret dials.refine_bravais_settings output, " "please check dials.refine_bravais_settings.log") info("Successfully completed (%.1f sec)", result["runtime"]) else: warning("Failed with exit code %d", result["exitcode"]) sys.exit(1) else: def _refine_bravais(self): # type: () -> None """Run `dials.refine_bravais_settings` to determine the space group.""" dials_start = timeit.default_timer() info("\nRefining Bravais settings...") self.refls = eliminate_sys_absent(self.expts, self.refls) map_to_primitive(self.expts, self.refls) try: refined_settings = refined_settings_from_refined_triclinic( self.expts, self.refls, self.params.dials_refine_bravais) except RuntimeError as e: warning("dials.refine_bravais_settings failed.\nGiving up.") sys.exit(e) possible_bravais_settings = { solution["bravais"] for solution in refined_settings } bravais_lattice_to_space_group_table(possible_bravais_settings) try: # Old version of dials with as_str() method logger.info(refined_settings.as_str()) except AttributeError: # Newer versions of dials (>= 2.2.2) has proper __str__ method logger.info(refined_settings) info( "Successfully completed (%.1f sec)", timeit.default_timer() - dials_start, ) def _report(self, experiments, reflections): # type: (ExperimentList, flex.reflection_table) -> None """ Run `dials.report` on an experiment list and reflection table. Args: experiments: An experiment list. reflections: The corresponding reflection table. """ info("\nCreating report...") command = ["dials.report", experiments, reflections] result = procrunner.run(command, print_stdout=False, debug=procrunner_debug) debug("result = %s", screen19.prettyprint_dictionary(result)) if result["exitcode"] == 0: info("Successfully completed (%.1f sec)", result["runtime"]) # if sys.stdout.isatty(): # info("Trying to start browser") # try: # import subprocess # d = dict(os.environ) # d["LD_LIBRARY_PATH"] = "" # subprocess.Popen(["xdg-open", "dials-report.html"], env=d) # except Exception as e: # debug("Could not open browser\n%s", str(e)) else: warning("Failed with exit code %d", result["exitcode"]) sys.exit(1) def run(self, args=None, phil=phil_scope, set_up_logging=False): # type: (Optional[List[str]], scope, bool) -> None """ TODO: Docstring. Args: args: phil: set_up_logging: Returns: """ usage = "%prog [options] image_directory | image_files.cbf | imported.expt" parser = OptionParser(usage=usage, epilog=__doc__, phil=phil, check_format=False) self.params, options, unhandled = parser.parse_args( args=args, show_diff_phil=True, return_unhandled=True, quick_parse=True) version_information = "screen19 v%s using %s (%s)" % ( screen19.__version__, dials.util.version.dials_version(), time.strftime("%Y-%m-%d %H:%M:%S"), ) start = timeit.default_timer() if len(unhandled) == 0: print(__doc__) print(version_information) return if set_up_logging: # Configure the logging log.config(verbosity=self.params.verbosity, logfile=self.params.output.log) # Unless verbose output has been requested, suppress generation of # debug and info log records from any child DIALS command, retaining # those from screen19 itself. if not self.params.verbosity: logging.getLogger("dials").setLevel(logging.WARNING) logging.getLogger("dials.screen19").setLevel(logging.INFO) info(version_information) debug("Run with:\n%s\n%s", " ".join(unhandled), parser.diff_phil.as_str()) self._count_processors(nproc=self.params.nproc) debug("Using %s processors", self.nproc) # Set multiprocessing settings for spot-finding, indexing and # integration to match the top-level specified number of processors self.params.dials_find_spots.spotfinder.mp.nproc = self.nproc self.params.dials_index.indexing.nproc = self.nproc # Setting self.params.dials_refine.refinement.mp.nproc is not helpful self.params.dials_integrate.integration.mp.nproc = self.nproc # Set the input and output parameters for the DIALS components # TODO: Compare to diff_phil and start from later in the pipeline if # appropriate self._import(unhandled) imported_name = self.params.dials_import.output.experiments self._find_spots() if not self._index(): info("\nRetrying for stronger spots only...") strong_refls = self.refls self.params.dials_find_spots.spotfinder.threshold.dispersion.sigma_strong = ( 15) self._find_spots() if not self._index(): warning("Giving up.") self.expts.as_file(imported_name) strong_refls.as_file("strong.refl") self.refls.as_file("stronger.refl") info( "Could not find an indexing solution. You may want to " "have a look at the reciprocal space by running:\n\n" " dials.reciprocal_lattice_viewer %s %s\n\n" "or, to only include stronger spots:\n\n" " dials.reciprocal_lattice_viewer %s %s\n", imported_name, "strong.refl", imported_name, "stronger.refl", ) sys.exit(1) if not self._create_profile_model(): info( "\nRefining model to attempt to increase number of valid spots..." ) self._refine() if not self._create_profile_model(): warning("Giving up.") info( "The identified indexing solution may not be correct. " "You may want to have a look at the reciprocal space by " "running:\n\n" " dials.reciprocal_lattice_viewer indexed.expt indexed.refl\n" ) sys.exit(1) self._check_intensities() if self.params.minimum_exposure.data == "integrated": self._integrate() self._wilson_calculation() experiments = self.params.dials_integrate.output.experiments reflections = self.params.dials_integrate.output.reflections else: self._wilson_calculation() experiments = self.params.dials_create_profile.output reflections = self.params.dials_index.output.reflections # This is a hacky check but should work for as long as DIALS 2.0 is supported. if version.dials_version() < "DIALS 2.1": self._refine_bravais(experiments, reflections) else: self._refine_bravais() self._report(experiments, reflections) runtime = timeit.default_timer() - start debug( "Finished at %s, total runtime: %.1f", time.strftime("%Y-%m-%d %H:%M:%S"), runtime, ) info("screen19 successfully completed (%.1f sec).", runtime)
class XDSRefiner(Refiner): def __init__(self): super(XDSRefiner, self).__init__() # factory functions def ExportXDS(self): from xia2.Wrappers.Dials.ExportXDS import ExportXDS as _ExportXDS export_xds = _ExportXDS() export_xds.set_working_directory(self.get_working_directory()) auto_logfiler(export_xds) return export_xds def _refine_prepare(self): for epoch, idxr in self._refinr_indexers.iteritems(): experiments = idxr.get_indexer_experiment_list() assert len( experiments) == 1 # currently only handle one lattice/sweep experiment = experiments[0] crystal_model = experiment.crystal lattice = idxr.get_indexer_lattice() # check if the lattice was user assigned... user_assigned = idxr.get_indexer_user_input_lattice() # hack to figure out if we did indexing with Dials - if so then need to # run XYCORR, INIT, and then dials.export_xds before we are ready to # integrate with XDS from xia2.Modules.Indexer.DialsIndexer import DialsIndexer if isinstance(idxr, DialsIndexer): sweep = idxr.get_indexer_sweep() imageset = idxr._indxr_imagesets[0] scan = imageset.get_scan() first, last = scan.get_image_range() phi_width = scan.get_oscillation()[1] last_background = int(round(5.0 / phi_width)) - 1 + first last_background = min(last, last_background) from xia2.Modules.Indexer.XDSIndexer import XDSIndexer xds_idxr = XDSIndexer() xds_idxr.set_working_directory(self.get_working_directory()) xds_idxr.set_indexer_sweep(sweep) xds_idxr.add_indexer_imageset(imageset) # next start to process these - first xycorr # FIXME run these *afterwards* as then we have a refined detector geometry # so the parallax correction etc. should be slightly better. #self._indxr_images = [(first, last)] xycorr = xds_idxr.Xycorr() xycorr.set_data_range(first, last) xycorr.set_background_range(first, last_background) xycorr.set_working_directory(self.get_working_directory()) xycorr.run() xds_data_files = {} for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']: xds_data_files[file] = xycorr.get_output_data_file(file) # next start to process these - then init init = xds_idxr.Init() for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']: init.set_input_data_file(file, xds_data_files[file]) init.set_data_range(first, last) init.set_background_range(first, last_background) init.set_working_directory(self.get_working_directory()) init.run() for file in ['BLANK.cbf', 'BKGINIT.cbf', 'GAIN.cbf']: xds_data_files[file] = init.get_output_data_file(file) exporter = self.ExportXDS() exporter.set_experiments_filename( idxr.get_solution()['experiments_file']) exporter.run() for file in ['XPARM.XDS']: xds_data_files[file] = os.path.join( self.get_working_directory(), 'xds', file) for k, v in xds_data_files.items(): idxr.set_indexer_payload(k, v) # check that the indexer is an XDS indexer - if not then # create one... elif not idxr.get_indexer_payload('XPARM.XDS'): Debug.write('Generating an XDS indexer') idxr_old = idxr from xia2.Modules.Indexer.XDSIndexer import XDSIndexer idxr = XDSIndexer() idxr.set_indexer_sweep(idxr_old.get_indexer_sweep()) self._refinr_indexers[epoch] = idxr self.set_refiner_prepare_done(False) # note to self for the future - this set will reset the # integrater prepare done flag - this means that we will # go through this routine all over again. However this # is not a problem as all that will happen is that the # results will be re-got, no additional processing will # be performed... # set the indexer up as per the frameprocessor interface... # this would usually happen within the IndexerFactory. idxr.set_indexer_sweep_name(idxr_old.get_indexer_sweep_name()) idxr.add_indexer_imageset(idxr_old.get_imageset()) idxr.set_working_directory(idxr_old.get_working_directory()) # now copy information from the old indexer to the new # one - lattice, cell, distance etc. # bug # 2434 - providing the correct target cell # may be screwing things up - perhaps it would # be best to allow XDS just to index with a free # cell but target lattice?? cell = crystal_model.get_unit_cell().parameters() check = PhilIndex.params.xia2.settings.xds_check_cell_deviation # FIXME this was changed in #42 but not sure logic is right if not check: Debug.write( 'Inputting target cell: %.2f %.2f %.2f %.2f %.2f %.2f' % \ cell) idxr.set_indexer_input_cell(cell) input_cell = cell from cctbx.sgtbx import bravais_types lattice = str( bravais_types.bravais_lattice( group=crystal_model.get_space_group())) idxr.set_indexer_input_lattice(lattice) if user_assigned: Debug.write('Assigning the user given lattice: %s' % \ lattice) idxr.set_indexer_user_input_lattice(True) idxr.set_detector(experiment.detector) idxr.set_beam(experiment.beam) idxr.set_goniometer(experiment.goniometer) # re-get the unit cell &c. and check that the indexing # worked correctly Debug.write('Rerunning indexing with XDS') experiments = idxr.get_indexer_experiment_list() assert len(experiments ) == 1 # currently only handle one lattice/sweep experiment = experiments[0] crystal_model = experiment.crystal # then in here check that the target unit cell corresponds # to the unit cell I wanted as input...? now for this I # should probably compute the unit cell volume rather # than comparing the cell axes as they may have been # switched around... # FIXME comparison needed def _refine(self): import copy from dxtbx.model import ExperimentList self._refinr_refined_experiment_list = ExperimentList() for epoch, idxr in self._refinr_indexers.iteritems(): self._refinr_payload[epoch] = copy.deepcopy(idxr._indxr_payload) self._refinr_refined_experiment_list.extend( idxr.get_indexer_experiment_list()) def _refine_finish(self): pass
def index_all_concurrent(experiments, reflections, params, method_list): # first determine n_strong per image: n_strong_per_image = {} for i, (expt, table) in enumerate(zip(experiments, reflections)): img = expt.imageset.get_path(i).split("/")[-1] n_strong = table.get_flags(table.flags.strong).count(True) n_strong_per_image[img] = n_strong with concurrent.futures.ProcessPoolExecutor( max_workers=params.indexing.nproc) as pool: sys.stdout = open(os.devnull, "w") # block printing from rstbx futures = { pool.submit(_index_one, expt, table, params, method_list, i): i for i, (table, expt) in enumerate(zip(reflections, experiments)) } tables_list = [None] * len(reflections) expts_list = [None] * len(reflections) for future in concurrent.futures.as_completed(futures): try: expts, refls = future.result() j = futures[future] except Exception as e: logger.info(e) else: if refls and expts: tables_list[j] = refls elist = ExperimentList() for jexpt in expts: elist.append( Experiment( identifier=jexpt.identifier, beam=jexpt.beam, detector=jexpt.detector, scan=jexpt.scan, goniometer=jexpt.goniometer, crystal=jexpt.crystal, imageset=jexpt.imageset[j:j + 1], )) expts_list[j] = elist sys.stdout = sys.__stdout__ # restore printing # now postprocess - record generic information results_summary = defaultdict(list) indexed_experiments = ExperimentList() indexed_reflections = flex.reflection_table() n_tot = 0 for idx, (elist, table) in enumerate(zip(expts_list, tables_list)): if not (elist and table): img = experiments[idx].imageset.get_path(idx).split("/")[-1] n_strong = n_strong_per_image[img] results_summary[idx].append({ "Image": img, "n_indexed": 0, "n_strong": n_strong, }) continue path = elist[0].imageset.get_path(0) img = path.split("/")[-1] n_strong = n_strong_per_image[img] indexed_experiments.extend(elist) ids_map = dict(table.experiment_identifiers()) for k in table.experiment_identifiers().keys(): del table.experiment_identifiers()[k] table["id"] += n_tot for k, v in ids_map.items(): table.experiment_identifiers()[k + n_tot] = v n_tot += len(ids_map.keys()) indexed_reflections.extend(table) # record some things for printing to output log/html for id_ in table.experiment_identifiers().keys(): selr = table.select(table["id"] == id_) calx, caly, _ = selr["xyzcal.px"].parts() obsx, obsy, _ = selr["xyzobs.px.value"].parts() delpsi = selr["delpsical.rad"] rmsd_x = flex.mean((calx - obsx)**2)**0.5 rmsd_y = flex.mean((caly - obsy)**2)**0.5 rmsd_z = flex.mean(((delpsi) * RAD2DEG)**2)**0.5 n_id_ = calx.size() results_summary[idx].append({ "Image": img, "n_indexed": n_id_, "n_strong": n_strong, "RMSD_X": rmsd_x, "RMSD_Y": rmsd_y, "RMSD_dPsi": rmsd_z, }) indexed_reflections.assert_experiment_identifiers_are_consistent( indexed_experiments) return indexed_experiments, indexed_reflections, results_summary
def index(self): experiments = ExperimentList() had_refinement_error = False have_similar_crystal_models = False while True: if had_refinement_error or have_similar_crystal_models: break max_lattices = self.params.multiple_lattice_search.max_lattices if max_lattices is not None and len(experiments) >= max_lattices: break if len(experiments) > 0: cutoff_fraction = (self.params.multiple_lattice_search. recycle_unindexed_reflections_cutoff) d_spacings = 1 / self.reflections["rlp"].norms() d_min_indexed = flex.min( d_spacings.select(self.indexed_reflections)) min_reflections_for_indexing = cutoff_fraction * len( self.reflections.select(d_spacings > d_min_indexed)) crystal_ids = self.reflections.select( d_spacings > d_min_indexed)["id"] if (crystal_ids == -1).count(True) < min_reflections_for_indexing: logger.info( "Finish searching for more lattices: %i unindexed reflections remaining." % ((crystal_ids == -1).count(True))) break n_lattices_previous_cycle = len(experiments) if self.d_min is None: self.d_min = self.params.refinement_protocol.d_min_start if len(experiments) == 0: new_expts = self.find_lattices() generate_experiment_identifiers(new_expts) experiments.extend(new_expts) else: try: new = self.find_lattices() generate_experiment_identifiers(new) experiments.extend(new) except DialsIndexError: logger.info("Indexing remaining reflections failed") if self.params.refinement_protocol.d_min_step is libtbx.Auto: n_cycles = self.params.refinement_protocol.n_macro_cycles if self.d_min is None or n_cycles == 1: self.params.refinement_protocol.d_min_step = 0 else: d_spacings = 1 / self.reflections["rlp"].norms() d_min_all = flex.min(d_spacings) self.params.refinement_protocol.d_min_step = ( self.d_min - d_min_all) / (n_cycles - 1) logger.info("Using d_min_step %.1f" % self.params.refinement_protocol.d_min_step) if len(experiments) == 0: raise DialsIndexError("No suitable lattice could be found.") elif len(experiments) == n_lattices_previous_cycle: # no more lattices found break for i_cycle in range( self.params.refinement_protocol.n_macro_cycles): if (i_cycle > 0 and self.d_min is not None and self.params.refinement_protocol.d_min_step > 0): d_min = self.d_min - self.params.refinement_protocol.d_min_step d_min = max(d_min, 0) if self.params.refinement_protocol.d_min_final is not None: d_min = max( d_min, self.params.refinement_protocol.d_min_final) if d_min >= 0: self.d_min = d_min logger.info("Increasing resolution to %.2f Angstrom" % d_min) # reset reflection lattice flags # the lattice a given reflection belongs to: a value of -1 indicates # that a reflection doesn't belong to any lattice so far self.reflections["id"] = flex.int(len(self.reflections), -1) self.index_reflections(experiments, self.reflections) if i_cycle == 0 and self.params.known_symmetry.space_group is not None: self._apply_symmetry_post_indexing( experiments, self.reflections, n_lattices_previous_cycle) logger.info("\nIndexed crystal models:") self.show_experiments(experiments, self.reflections, d_min=self.d_min) if self._check_have_similar_crystal_models(experiments): have_similar_crystal_models = True break logger.info("") logger.info("#" * 80) logger.info("Starting refinement (macro-cycle %i)" % (i_cycle + 1)) logger.info("#" * 80) logger.info("") self.indexed_reflections = self.reflections["id"] > -1 sel = flex.bool(len(self.reflections), False) lengths = 1 / self.reflections["rlp"].norms() if self.d_min is not None: isel = (lengths <= self.d_min).iselection() sel.set_selected(isel, True) sel.set_selected(self.reflections["id"] == -1, True) self.reflections.unset_flags(sel, self.reflections.flags.indexed) self.unindexed_reflections = self.reflections.select(sel) reflections_for_refinement = self.reflections.select( self.indexed_reflections) if self.params.refinement_protocol.mode == "repredict_only": refined_experiments, refined_reflections = ( experiments, reflections_for_refinement, ) from dials.algorithms.refinement.prediction.managed_predictors import ( ExperimentsPredictorFactory, ) ref_predictor = ExperimentsPredictorFactory.from_experiments( experiments, spherical_relp=self.all_params.refinement. parameterisation.spherical_relp_model, ) ref_predictor(refined_reflections) else: try: refined_experiments, refined_reflections = self.refine( experiments, reflections_for_refinement) except (DialsRefineConfigError, DialsRefineRuntimeError) as e: if len(experiments) == 1: raise DialsIndexRefineError(str(e)) had_refinement_error = True logger.info("Refinement failed:") logger.info(e) del experiments[-1] # remove experiment id from the reflections associated # with this deleted experiment - indexed flag removed # below last = len(experiments) sel = refined_reflections["id"] == last logger.info("Removing %d reflections with id %d" % (sel.count(True), last)) refined_reflections["id"].set_selected(sel, -1) break self._unit_cell_volume_sanity_check(experiments, refined_experiments) self.refined_reflections = refined_reflections self.refined_reflections.unset_flags( self.refined_reflections["id"] < 0, self.refined_reflections.flags.indexed, ) for i, expt in enumerate(self.experiments): ref_sel = self.refined_reflections.select( self.refined_reflections["imageset_id"] == i) ref_sel = ref_sel.select(ref_sel["id"] >= 0) for i_expt in set(ref_sel["id"]): refined_expt = refined_experiments[i_expt] expt.detector = refined_expt.detector expt.beam = refined_expt.beam expt.goniometer = refined_expt.goniometer expt.scan = refined_expt.scan refined_expt.imageset = expt.imageset if not (self.all_params.refinement.parameterisation.beam.fix == "all" and self.all_params.refinement. parameterisation.detector.fix == "all"): # Experimental geometry may have changed - re-map centroids to # reciprocal space self.reflections.map_centroids_to_reciprocal_space( self.experiments) # update for next cycle experiments = refined_experiments self.refined_experiments = refined_experiments logger.info("\nRefined crystal models:") self.show_experiments(self.refined_experiments, self.reflections, d_min=self.d_min) if (i_cycle >= 2 and self.d_min == self.params.refinement_protocol.d_min_final): logger.info( "Target d_min_final reached: finished with refinement") break if self.refined_experiments is None: raise DialsIndexRefineError( "None of the experiments could refine.") if len(self.refined_experiments) > 1: from dials.algorithms.indexing.compare_orientation_matrices import ( rotation_matrix_differences, ) logger.info( rotation_matrix_differences( self.refined_experiments.crystals())) self._xyzcal_mm_to_px(self.refined_experiments, self.refined_reflections)
def from_filenames( filenames, unhandled=None, compare_beam=None, compare_detector=None, compare_goniometer=None, scan_tolerance=None, format_kwargs=None, load_models=True, ): """Create a list of data blocks from a list of directory or file names.""" experiments = ExperimentList() # Process each file given by this path list to_process = _openingpathiterator(filenames) find_format = dxtbx.datablock.FormatChecker() format_groups = collections.OrderedDict() if format_kwargs is None: format_kwargs = {} for filename in to_process: # We now have a file, pre-opened by Format.open_file (therefore # cached). Determine its type, and prepare to put into a group format_class = find_format.find_format(filename) # Verify this makes sense if not format_class: # No format class found? logger.debug("Could not determine format for %s", filename) if unhandled is not None: unhandled.append(filename) elif format_class.is_abstract(): logger.debug( f"Image file {filename} appears to be a '{format_class.__name__}', but this is an abstract Format" ) # Invalid format class found? if unhandled is not None: unhandled.append(filename) elif issubclass(format_class, FormatMultiImage): imageset = format_class.get_imageset( os.path.abspath(filename), format_kwargs=format_kwargs) format_groups.setdefault(format_class, []).append(imageset) logger.debug("Loaded file: %s", filename) else: format_object = format_class(filename, **format_kwargs) meta = ImageMetadataRecord.from_format(format_object) assert meta.filename == filename # Add this entry to our table of formats format_groups.setdefault(format_class, []).append(meta) logger.debug("Loaded metadata of file: %s", filename) # Now, build experiments from these files. Duplicating the logic of # the previous implementation: # - FormatMultiImage files each have their own ImageSet # - Every set of images forming a scan goes into its own ImageSequence # - Any consecutive still frames that share any metadata with the # previous still fram get collected into one ImageSet # Treat each format as a separate datablock for format_class, records in format_groups.items(): if issubclass(format_class, FormatMultiImage): for imageset in records: experiments.extend( ExperimentListFactory.from_imageset_and_crystal( imageset, crystal=None, load_models=load_models)) continue # Merge any consecutive and identical metadata together _merge_model_metadata( records, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, ) records = _merge_scans(records, scan_tolerance=scan_tolerance) imagesets = _convert_to_imagesets(records, format_class, format_kwargs) imagesets = list(imagesets) # Validate this datablock and store it assert imagesets, "Datablock got no imagesets?" for imageset in imagesets: experiments.extend( ExperimentListFactory.from_imageset_and_crystal( imageset, crystal=None, load_models=load_models)) return experiments
def from_templates(templates, **kwargs): """Import an experiment list from templates""" assert "verbose" not in kwargs, "The verbose parameter has been removed" assert len(templates) > 0 experiments = ExperimentList() find_format = dxtbx.datablock.FormatChecker() # For each template do an import for template in templates: template = os.path.normpath(template) filenames = sorted(locate_files_matching_template_string(template)) if len(filenames): logger.debug( "The following files matched the template string:\n%s", "\n".join(f" {p}" for p in filenames), ) # Check if we've matched any filenames if len(filenames) == 0: raise ValueError( f"Template '{template}' does not match any files") # Get the format from the first image format_class = find_format.find_format(filenames[0]) # Verify this makes sense if format_class is None: raise ValueError( f"Image file {filenames[0]} format is unknown") elif format_class.is_abstract(): raise ValueError( f"Image file {filenames[0]} appears to be a '{type(format_class).__name__}', but this is an abstract Format" ) else: index = slice(*template_string_number_index(template)) image_range = kwargs.get("image_range") if image_range: first, last = image_range else: first = int(filenames[0][index]) last = int(filenames[-1][index]) # Check all images in range are present - if allowed if not kwargs.get("allow_incomplete_sequences", False): all_numbers = {int(f[index]) for f in filenames} missing = set(range(first, last + 1)) - all_numbers if missing: raise ValueError( "Missing image{} {} from imageset ({}-{})".format( "s" if len(missing) > 1 else "", ", ".join(str(x) for x in sorted(missing)), first, last, )) # Read the image fmt = format_class(filenames[0], **(kwargs.get("format_kwargs", {}))) # Update the image range image_range = (first, last) scan = fmt.get_scan() scan.set_image_range(image_range) # Create the sequence and experiment imageset = dxtbx.imageset.ImageSetFactory.make_sequence( template, list(range(first, last + 1)), format_class, fmt.get_beam(), fmt.get_detector(), fmt.get_goniometer(), scan, format_kwargs=kwargs.get("format_kwargs"), ) experiments.extend( ExperimentListFactory.from_imageset_and_crystal( imageset, crystal=None, load_models=True, )) return experiments