def __init__(self, templates, verbose=False): ''' Import the datablocks from the given templates. ''' from dxtbx.sweep_filenames import locate_files_matching_template_string assert(len(templates) > 0) self.datablocks = [] # A function to append or create a new datablock def append_to_datablocks(iset): try: self.datablocks[-1].append(iset) except Exception: self.datablocks.append(DataBlock([iset])) if verbose: print 'Added imageset to datablock %d' % (len(self.datablocks) - 1) # For each template do an import for template in templates: paths = sorted(locate_files_matching_template_string(template)) if verbose: print 'The following files matched the template string:' if len(paths) > 0: for p in paths: print ' %s' % p else: print ' No files found' # Get the format from the first image find_format = FormatChecker(verbose=verbose) fmt = find_format(paths[0]) if fmt is None: raise RuntimeError('Image file %s format is unknown' % paths[0]) else: imageset = self._create_imageset(fmt, template, paths) append_to_datablocks(imageset)
def __init__(self, templates, verbose=False, **kwargs): ''' Import the datablocks from the given templates. ''' from dxtbx.sweep_filenames import locate_files_matching_template_string from libtbx.utils import Sorry from os.path import normpath assert(len(templates) > 0) self.datablocks = [] # A function to append or create a new datablock def append_to_datablocks(iset): try: self.datablocks[-1].append(iset) except Exception: self.datablocks.append(DataBlock([iset])) if verbose: print('Added imageset to datablock %d' % (len(self.datablocks) - 1)) # For each template do an import for template in templates: template = normpath(template) paths = sorted(locate_files_matching_template_string(template)) if verbose: print('The following files matched the template string:') if len(paths) > 0: for p in paths: print(' %s' % p) else: print(' No files found') # Check if we've matched any filenames if len(paths) == 0: raise Sorry('Template "%s" does not match any files' % template) # Get the format from the first image find_format = FormatChecker(verbose=verbose) fmt = find_format(paths[0]) if fmt is None: raise Sorry('Image file %s format is unknown' % paths[0]) elif fmt.ignore(): raise Sorry('Image file %s format will be ignored' % paths[0]) else: imageset = self._create_imageset(fmt, template, paths, **kwargs) append_to_datablocks(imageset)
def __init__(self, templates, verbose=False, **kwargs): ''' Import the datablocks from the given templates. ''' from dxtbx.sweep_filenames import locate_files_matching_template_string from libtbx.utils import Sorry assert(len(templates) > 0) self.datablocks = [] # A function to append or create a new datablock def append_to_datablocks(iset): try: self.datablocks[-1].append(iset) except Exception: self.datablocks.append(DataBlock([iset])) if verbose: print 'Added imageset to datablock %d' % (len(self.datablocks) - 1) # For each template do an import for template in templates: paths = sorted(locate_files_matching_template_string(template)) if verbose: print 'The following files matched the template string:' if len(paths) > 0: for p in paths: print ' %s' % p else: print ' No files found' # Check if we've matched any filenames if len(paths) == 0: raise Sorry('Template "%s" does not match any files' % template) # Get the format from the first image find_format = FormatChecker(verbose=verbose) fmt = find_format(paths[0]) if fmt is None: raise Sorry('Image file %s format is unknown' % paths[0]) elif fmt.ignore(): raise Sorry('Image file %s format will be ignored' % paths[0]) else: imageset = self._create_imageset(fmt, template, paths, **kwargs) append_to_datablocks(imageset)
def load_imagesets( template, directory, id_image=None, image_range=None, use_cache=True, reversephi=False, ): global imageset_cache from dxtbx.model.experiment_list import ExperimentListFactory from xia2.Applications.xia2setup import known_hdf5_extensions from dxtbx.imageset import ImageSweep full_template_path = os.path.join(directory, template) if full_template_path not in imageset_cache or not use_cache: from dxtbx.model.experiment_list import BeamComparison from dxtbx.model.experiment_list import DetectorComparison from dxtbx.model.experiment_list import GoniometerComparison params = PhilIndex.params.xia2.settings compare_beam = BeamComparison( wavelength_tolerance=params.input.tolerance.beam.wavelength, direction_tolerance=params.input.tolerance.beam.direction, polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal, polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction, ) compare_detector = DetectorComparison( fast_axis_tolerance=params.input.tolerance.detector.fast_axis, slow_axis_tolerance=params.input.tolerance.detector.slow_axis, origin_tolerance=params.input.tolerance.detector.origin, ) compare_goniometer = GoniometerComparison( rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis, fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation, setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation, ) scan_tolerance = params.input.tolerance.scan.oscillation format_kwargs = { "dynamic_shadowing": params.input.format.dynamic_shadowing, "multi_panel": params.input.format.multi_panel, } if os.path.splitext(full_template_path)[-1] in known_hdf5_extensions: # if we are passed the correct file, use this, else look for a master # file (i.e. something_master.h5) if os.path.exists(full_template_path) and os.path.isfile( full_template_path ): master_file = full_template_path else: import glob g = glob.glob(os.path.join(directory, "*_master.h5")) master_file = None for p in g: substr = longest_common_substring(template, p) if substr: if master_file is None or ( len(substr) > len(longest_common_substring(template, master_file)) ): master_file = p if master_file is None: raise RuntimeError("Can't find master file for %s" % full_template_path) unhandled = [] experiments = ExperimentListFactory.from_filenames( [master_file], verbose=False, unhandled=unhandled, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, scan_tolerance=scan_tolerance, format_kwargs=format_kwargs, ) assert len(unhandled) == 0, ( "unhandled image files identified: %s" % unhandled ) else: from dxtbx.sweep_filenames import locate_files_matching_template_string params = PhilIndex.get_python_object() read_all_image_headers = params.xia2.settings.read_all_image_headers if read_all_image_headers: paths = sorted( locate_files_matching_template_string(full_template_path) ) unhandled = [] experiments = ExperimentListFactory.from_filenames( paths, verbose=False, unhandled=unhandled, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, scan_tolerance=scan_tolerance, format_kwargs=format_kwargs, ) assert len(unhandled) == 0, ( "unhandled image files identified: %s" % unhandled ) else: from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter( [full_template_path], format_kwargs=format_kwargs ) experiments = importer.experiments imagesets = [ iset for iset in experiments.imagesets() if isinstance(iset, ImageSweep) ] assert len(imagesets) > 0, "no imageset found" imageset_cache[full_template_path] = collections.OrderedDict() if reversephi: for imageset in imagesets: goniometer = imageset.get_goniometer() goniometer.set_rotation_axis( tuple(-g for g in goniometer.get_rotation_axis()) ) reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry if reference_geometry is not None and len(reference_geometry) > 0: update_with_reference_geometry(imagesets, reference_geometry) # Update the geometry params = PhilIndex.params.xia2.settings update_geometry = [] from dials.command_line.dials_import import ManualGeometryUpdater from dials.util.options import geometry_phil_scope # Then add manual geometry work_phil = geometry_phil_scope.format(params.input) diff_phil = geometry_phil_scope.fetch_diff(source=work_phil) if diff_phil.as_str() != "": update_geometry.append(ManualGeometryUpdater(params.input)) imageset_list = [] for imageset in imagesets: for updater in update_geometry: imageset = updater(imageset) imageset_list.append(imageset) imagesets = imageset_list from scitbx.array_family import flex for imageset in imagesets: scan = imageset.get_scan() exposure_times = scan.get_exposure_times() epochs = scan.get_epochs() if exposure_times.all_eq(0) or exposure_times[0] == 0: exposure_times = flex.double(exposure_times.size(), 1) scan.set_exposure_times(exposure_times) elif not exposure_times.all_gt(0): exposure_times = flex.double(exposure_times.size(), exposure_times[0]) scan.set_exposure_times(exposure_times) if epochs.size() > 1 and not epochs.all_gt(0): if epochs[0] == 0: epochs[0] = 1 for i in range(1, epochs.size()): epochs[i] = epochs[i - 1] + exposure_times[i - 1] scan.set_epochs(epochs) _id_image = scan.get_image_range()[0] imageset_cache[full_template_path][_id_image] = imageset if id_image is not None: return [imageset_cache[full_template_path][id_image]] elif image_range is not None: for imageset in imageset_cache[full_template_path].values(): scan = imageset.get_scan() scan_image_range = scan.get_image_range() if ( image_range[0] >= scan_image_range[0] and image_range[1] <= scan_image_range[1] ): imagesets = [ imageset[ image_range[0] - scan_image_range[0] : image_range[1] + 1 - scan_image_range[0] ] ] assert len(imagesets[0]) == image_range[1] - image_range[0] + 1, len( imagesets[0] ) return imagesets return imageset_cache[full_template_path].values()
def load_imagesets(template, directory, id_image=None, image_range=None, use_cache=True, reversephi=False): global imageset_cache from dxtbx.datablock import DataBlockFactory from xia2.Applications.xia2setup import known_hdf5_extensions full_template_path = os.path.join(directory, template) if full_template_path not in imageset_cache or not use_cache: from dxtbx.datablock import BeamComparison from dxtbx.datablock import DetectorComparison from dxtbx.datablock import GoniometerComparison params = PhilIndex.params.xia2.settings compare_beam = BeamComparison( wavelength_tolerance=params.input.tolerance.beam.wavelength, direction_tolerance=params.input.tolerance.beam.direction, polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal, polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction) compare_detector = DetectorComparison( fast_axis_tolerance=params.input.tolerance.detector.fast_axis, slow_axis_tolerance=params.input.tolerance.detector.slow_axis, origin_tolerance=params.input.tolerance.detector.origin) compare_goniometer = GoniometerComparison( rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis, fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation, setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation) scan_tolerance = params.input.tolerance.scan.oscillation format_kwargs = { 'dynamic_shadowing' : params.input.format.dynamic_shadowing, 'multi_panel' : params.input.format.multi_panel, } if os.path.splitext(full_template_path)[-1] in known_hdf5_extensions: import glob g = glob.glob(os.path.join(directory, '*_master.h5')) master_file = None for p in g: substr = longest_common_substring(template, p) if substr: if (master_file is None or (len(substr) > len(longest_common_substring(template, master_file)))): master_file = p if master_file is None: raise RuntimeError("Can't find master file for %s" %full_template_path) unhandled = [] datablocks = DataBlockFactory.from_filenames( [master_file], verbose=False, unhandled=unhandled, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, scan_tolerance=scan_tolerance, format_kwargs=format_kwargs) assert len(unhandled) == 0, "unhandled image files identified: %s" % \ unhandled assert len(datablocks) == 1, "1 datablock expected, %d found" % \ len(datablocks) else: from dxtbx.sweep_filenames import locate_files_matching_template_string params = PhilIndex.get_python_object() read_all_image_headers = params.xia2.settings.read_all_image_headers if read_all_image_headers: paths = sorted(locate_files_matching_template_string(full_template_path)) unhandled = [] datablocks = DataBlockFactory.from_filenames( paths, verbose=False, unhandled=unhandled, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, scan_tolerance=scan_tolerance, format_kwargs=format_kwargs) assert len(unhandled) == 0, "unhandled image files identified: %s" % \ unhandled assert len(datablocks) == 1, "1 datablock expected, %d found" % \ len(datablocks) else: from dxtbx.datablock import DataBlockTemplateImporter importer = DataBlockTemplateImporter( [full_template_path], kwargs=format_kwargs) datablocks = importer.datablocks imagesets = datablocks[0].extract_sweeps() assert len(imagesets) > 0, "no imageset found" imageset_cache[full_template_path] = OrderedDict() if reversephi: for imageset in imagesets: goniometer = imageset.get_goniometer() goniometer.set_rotation_axis( tuple((-g for g in goniometer.get_rotation_axis()))) reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry if reference_geometry is not None and len(reference_geometry) > 0: update_with_reference_geometry(imagesets, reference_geometry) from dials.command_line.dials_import import ManualGeometryUpdater update_geometry = ManualGeometryUpdater( PhilIndex.params.xia2.settings.input) imagesets = [update_geometry(imageset) for imageset in imagesets] from scitbx.array_family import flex for imageset in imagesets: scan = imageset.get_scan() exposure_times = scan.get_exposure_times() epochs = scan.get_epochs() if exposure_times.all_eq(0): exposure_times = flex.double(exposure_times.size(), 1) scan.set_exposure_times(exposure_times) elif not exposure_times.all_gt(0): exposure_times = flex.double(exposure_times.size(), exposure_times[0]) scan.set_exposure_times(exposure_times) if epochs.size() > 1 and not epochs.all_gt(0): for i in range(1, epochs.size()): epochs[i] = epochs[i-1] + exposure_times[i-1] scan.set_epochs(epochs) _id_image = scan.get_image_range()[0] imageset_cache[full_template_path][_id_image] = imageset if id_image is not None: return [imageset_cache[full_template_path][id_image]] elif image_range is not None: for imageset in imageset_cache[full_template_path].values(): scan = imageset.get_scan() scan_image_range = scan.get_image_range() if (image_range[0] >= scan_image_range[0] and image_range[1] <= scan_image_range[1]): imagesets = [imageset[ image_range[0] - scan_image_range[0]: image_range[1] + 1 - scan_image_range[0]]] assert len(imagesets[0]) == image_range[1] - image_range[0] + 1, \ len(imagesets[0]) return imagesets return imageset_cache[full_template_path].values()
def load_imagesets(template, directory, id_image=None, image_range=None, use_cache=True, reversephi=False): global imageset_cache full_template_path = os.path.join(directory, template) if full_template_path not in imageset_cache or not use_cache: from dxtbx.datablock import DataBlockFactory from dxtbx.sweep_filenames import locate_files_matching_template_string params = PhilIndex.get_python_object() read_all_image_headers = params.xia2.settings.read_all_image_headers if read_all_image_headers: paths = sorted(locate_files_matching_template_string(full_template_path)) unhandled = [] datablocks = DataBlockFactory.from_filenames( paths, verbose=False, unhandled=unhandled) assert len(unhandled) == 0, "unhandled image files identified: %s" % \ unhandled assert len(datablocks) == 1, "1 datablock expected, %d found" % \ len(datablocks) else: from dxtbx.datablock import DataBlockTemplateImporter importer = DataBlockTemplateImporter([full_template_path]) datablocks = importer.datablocks imagesets = datablocks[0].extract_sweeps() assert len(imagesets) > 0, "no imageset found" imageset_cache[full_template_path] = OrderedDict() if reversephi: for imageset in imagesets: goniometer = imageset.get_goniometer() goniometer.set_rotation_axis( tuple((-g for g in goniometer.get_rotation_axis()))) reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry if reference_geometry is not None and len(reference_geometry) > 0: update_with_reference_geometry(imagesets, reference_geometry) for imageset in imagesets: scan = imageset.get_scan() _id_image = scan.get_image_range()[0] imageset_cache[full_template_path][_id_image] = imageset if id_image is not None: return [imageset_cache[full_template_path][id_image]] elif image_range is not None: for imageset in imageset_cache[full_template_path].values(): scan = imageset.get_scan() scan_image_range = scan.get_image_range() if (image_range[0] >= scan_image_range[0] and image_range[1] <= scan_image_range[1]): imagesets = [imageset[ image_range[0] - scan_image_range[0]: image_range[1] + 1 - scan_image_range[0]]] assert len(imagesets[0]) == image_range[1] - image_range[0] + 1, \ len(imagesets[0]) return imagesets return imageset_cache[full_template_path].values()