def tst_from_imageset(self): from dxtbx.imageset import ImageSet, NullReader from dxtbx.model import Beam, Detector, Goniometer, Scan from dxtbx.model.crystal import crystal_model imageset = ImageSet(NullReader(["filename.cbf"])) imageset.set_beam(Beam(), 0) imageset.set_detector(Detector(), 0) crystal = crystal_model( (1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol=0) experiments = ExperimentListFactory.from_imageset_and_crystal( imageset, crystal) assert(len(experiments) == 1) assert(experiments[0].imageset is not None) assert(experiments[0].beam is not None) assert(experiments[0].detector is not None) assert(experiments[0].crystal is not None) print 'OK'
def test_cspad_cbf_in_memory(dials_regression, tmpdir): tmpdir.chdir() # Check the data files for this test exist image_path = os.path.join(dials_regression, "image_examples", "LCLS_cspad_nexus", 'idx-20130301060858801.cbf') assert os.path.isfile(image_path) with open("process_lcls.phil", 'w') as f: f.write(""" dispatch.squash_errors = False spotfinder { filter.min_spot_size=2 threshold.dispersion.gain=25 threshold.dispersion.global_threshold=100 } indexing { known_symmetry { space_group = P6122 unit_cell = 92.9 92.9 130.4 90 90 120 } refinement_protocol.d_min_start=1.7 stills.refine_candidates_with_known_symmetry=True } """) params = phil_scope.fetch(parse(file_name="process_lcls.phil")).extract() params.output.datablock_filename = None processor = Processor(params) mem_img = dxtbx.load(image_path) raw_data = mem_img.get_raw_data( ) # cache the raw data to prevent swig errors mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle) mem_img._raw_data = raw_data mem_img._cbf_handle = None # drop the file handle to prevent swig errors imgset = ImageSet(ImageSetData(MemReader([mem_img]), MemMasker([mem_img]))) imgset.set_beam(mem_img.get_beam()) imgset.set_detector(mem_img.get_detector()) datablock = DataBlockFactory.from_imageset(imgset)[0] processor.process_datablock("20130301060858801", datablock) # index/integrate the image result = "idx-20130301060858801_integrated.pickle" n_refls = range( 140, 152) # large ranges to handle platform-specific differences with open(result, 'rb') as f: table = pickle.load(f) assert len(table) in n_refls, len(table) assert 'id' in table assert (table['id'] == 0).count(False) == 0
def do_import(filename): logger.info("Loading %s"%os.path.basename(filename)) try: datablocks = DataBlockFactory.from_json_file(filename) except ValueError: datablocks = DataBlockFactory.from_filenames([filename]) if len(datablocks) == 0: raise Abort("Could not load %s"%filename) if len(datablocks) > 1: raise Abort("Got multiple datablocks from file %s"%filename) # Ensure the indexer and downstream applications treat this as set of stills from dxtbx.imageset import ImageSet reset_sets = [] for imageset in datablocks[0].extract_imagesets(): imageset = ImageSet(imageset.reader(), imageset.indices()) imageset._models = imageset._models imageset.set_scan(None) imageset.set_goniometer(None) reset_sets.append(imageset) return DataBlockFactory.from_imageset(reset_sets)[0]
def test_cspad_cbf_in_memory(dials_regression, run_in_tmpdir, composite_output): # Check the data files for this test exist image_path = os.path.join( dials_regression, "image_examples", "LCLS_cspad_nexus", "idx-20130301060858801.cbf", ) assert os.path.isfile(image_path) with open("process_lcls.phil", "w") as f: f.write(cspad_cbf_in_memory_phil) params = phil_scope.fetch(parse(file_name="process_lcls.phil")).extract() params.output.experiments_filename = None params.output.composite_output = composite_output if composite_output: processor = Processor(params, composite_tag="memtest") else: processor = Processor(params) mem_img = dxtbx.load(image_path) raw_data = mem_img.get_raw_data( ) # cache the raw data to prevent swig errors mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle) mem_img._raw_data = raw_data mem_img._cbf_handle = None # drop the file handle to prevent swig errors imgset = ImageSet(ImageSetData(MemReader([mem_img]), None)) imgset.set_beam(mem_img.get_beam()) imgset.set_detector(mem_img.get_detector()) experiments = ExperimentListFactory.from_imageset_and_crystal(imgset, None) processor.process_experiments("20130301060858801", experiments) # index/integrate the image if composite_output: processor.finalize() result = "idx-memtest_integrated.refl" else: result = "idx-20130301060858801_integrated.refl" n_refls = list(range( 140, 152)) # large ranges to handle platform-specific differences table = flex.reflection_table.from_file(result) assert len(table) in n_refls, len(table) assert "id" in table assert (table["id"] == 0).count(False) == 0
def datablock_from_numpyarrays(image, detector, beam, mask=None): """ So that one can do e.g. >> dblock = datablock_from_numpyarrays( image, detector, beam) >> refl = flex.reflection_table.from_observations(dblock, spot_finder_params) without having to utilize the harddisk :param image: numpy array image :param mask: numpy mask :param detector: dxtbx detector model :param beam: dxtbx beam model :return: datablock for the image """ I = FormatInMemory(image=image, mask=mask) reader = MemReader([I]) masker = MemMasker([I]) iset_Data = ImageSetData(reader, masker) iset = ImageSet(iset_Data) iset.set_beam(beam) iset.set_detector(detector) dblock = DataBlockFactory.from_imageset([iset])[0] return dblock
def tst_from_imageset(self): from dxtbx.imageset import ImageSet, NullReader from dxtbx.model import Beam, Detector, Goniometer, Scan from dxtbx.model import Crystal imageset = ImageSet(NullReader(["filename.cbf"])) imageset.set_beam(Beam(), 0) imageset.set_detector(Detector(), 0) crystal = Crystal((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol="P1") experiments = ExperimentListFactory.from_imageset_and_crystal( imageset, crystal) assert (len(experiments) == 1) assert (experiments[0].imageset is not None) assert (experiments[0].beam is not None) assert (experiments[0].detector is not None) assert (experiments[0].crystal is not None) print 'OK'
def datablock_from_numpyarrays(image, detector, beam, mask=None): """ put the numpy array image(s) into a dials datablock :param image: numpy array of images or an image :param detector: dxtbx det :param beam: dxtbx beam :param mask: mask same shape as image , 1 is not masked, boolean :return: """ if isinstance( image, list): image = np.array( image) if mask is not None: if isinstance( mask, list): mask = np.array(mask).astype(bool) I = FormatInMemory(image=image, mask=mask) reader = MemReader([I]) masker = MemMasker([I]) iset_Data = ImageSetData(reader, masker) iset = ImageSet(iset_Data) iset.set_beam(beam) iset.set_detector(detector) dblock = DataBlockFactory.from_imageset([iset])[0] return dblock
def get_imageset( Class, input_filenames, beam=None, detector=None, goniometer=None, scan=None, as_imageset=False, as_sequence=False, single_file_indices=None, format_kwargs=None, template=None, check_format=True, ): """ Factory method to create an imageset """ # Import here to avoid cyclic imports from dxtbx.imageset import ImageSequence, ImageSet, ImageSetData # Get filename absolute paths, for entries that are filenames filenames = [ os.path.abspath(x) if not get_url_scheme(x) else x for x in input_filenames ] # Make it a dict if format_kwargs is None: format_kwargs = {} # Get some information from the format class reader = Class.get_reader()(filenames, **format_kwargs) # Get the format instance if check_format is True: Class._current_filename_ = None Class._current_instance_ = None format_instance = Class.get_instance(filenames[0], **format_kwargs) else: format_instance = None # Read the vendor type if check_format is True: vendor = format_instance.get_vendortype() else: vendor = "" # Get the format kwargs params = format_kwargs # Make sure only 1 or none is set assert [as_imageset, as_sequence].count(True) < 2 if as_imageset: is_sequence = False elif as_sequence: is_sequence = True else: if scan is None and format_instance is None: raise RuntimeError(""" One of the following needs to be set - as_imageset=True - as_sequence=True - scan - check_format=True """) if scan is None: test_scan = format_instance.get_scan() else: test_scan = scan if test_scan is not None and test_scan.get_oscillation()[1] != 0: is_sequence = True else: is_sequence = False # Create an imageset or sequence if not is_sequence: # Create the imageset iset = ImageSet( ImageSetData( reader=reader, masker=None, vendor=vendor, params=params, format=Class, )) # If any are None then read from format if [beam, detector, goniometer, scan].count(None) != 0: # Get list of models beam = [] detector = [] goniometer = [] scan = [] for f in filenames: format_instance = Class(f, **format_kwargs) beam.append(format_instance.get_beam()) detector.append(format_instance.get_detector()) goniometer.append(format_instance.get_goniometer()) scan.append(format_instance.get_scan()) # Set the list of models for i in range(len(filenames)): iset.set_beam(beam[i], i) iset.set_detector(detector[i], i) iset.set_goniometer(goniometer[i], i) iset.set_scan(scan[i], i) else: # Get the template if template is None: template = template_regex(filenames[0])[0] else: template = str(template) # Check scan makes sense if scan: if check_format is True: assert scan.get_num_images() == len(filenames) # If any are None then read from format if beam is None and format_instance is not None: beam = format_instance.get_beam() if detector is None and format_instance is not None: detector = format_instance.get_detector() if goniometer is None and format_instance is not None: goniometer = format_instance.get_goniometer() if scan is None and format_instance is not None: scan = format_instance.get_scan() if scan is not None: for f in filenames[1:]: format_instance = Class(f, **format_kwargs) scan += format_instance.get_scan() assert beam is not None, "Can't create Sequence without beam" assert detector is not None, "Can't create Sequence without detector" assert goniometer is not None, "Can't create Sequence without goniometer" assert scan is not None, "Can't create Sequence without scan" # Create the masker if format_instance is not None: masker = format_instance.get_masker(goniometer=goniometer) else: masker = None # Create the sequence iset = ImageSequence( ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=Class, template=template, ), beam=beam, detector=detector, goniometer=goniometer, scan=scan, ) if format_instance is not None: static_mask = format_instance.get_static_mask() if static_mask is not None: if not iset.external_lookup.mask.data.empty(): for m1, m2 in zip(static_mask, iset.external_lookup.mask.data): m1 &= m2.data() iset.external_lookup.mask.data = ImageBool(static_mask) else: iset.external_lookup.mask.data = ImageBool(static_mask) return iset
def get_imageset( Class, filenames, beam=None, detector=None, goniometer=None, scan=None, as_imageset=False, as_sweep=False, single_file_indices=None, format_kwargs=None, template=None, check_format=True, ): """ Factory method to create an imageset """ from dxtbx.imageset import ImageSetData from dxtbx.imageset import ImageSet from dxtbx.imageset import ImageSweep from dxtbx.sweep_filenames import template_regex from os.path import abspath # Get filename absolute paths filenames = map(abspath, filenames) # Make it a dict if format_kwargs is None: format_kwargs = {} # Get some information from the format class reader = Class.get_reader()(filenames, **format_kwargs) masker = Class.get_masker()(filenames, **format_kwargs) # Get the format instance if check_format is True: format_instance = Class(filenames[0], **format_kwargs) else: format_instance = None # Read the vendor type if check_format is True: vendor = format_instance.get_vendortype() else: vendor = "" # Get the format kwargs params = format_kwargs # Make sure only 1 or none is set assert [as_imageset, as_sweep].count(True) < 2 if as_imageset: is_sweep = False elif as_sweep: is_sweep = True else: if scan is None and format_instance is None: raise RuntimeError(""" One of the following needs to be set - as_imageset=True - as_sweep=True - scan - check_format=True """) if scan is None: test_scan = format_instance.get_scan() else: test_scan = scan if test_scan is not None and test_scan.get_oscillation()[1] != 0: is_sweep = True else: is_sweep = False # Create an imageset or sweep if not is_sweep: # Create the imageset iset = ImageSet( ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=Class, )) # If any are None then read from format if [beam, detector, goniometer, scan].count(None) != 0: # Get list of models beam = [] detector = [] goniometer = [] scan = [] for f in filenames: format_instance = Class(f, **format_kwargs) beam.append(format_instance.get_beam()) detector.append(format_instance.get_detector()) goniometer.append(format_instance.get_goniometer()) scan.append(format_instance.get_scan()) # Set the list of models for i in range(len(filenames)): iset.set_beam(beam[i], i) iset.set_detector(detector[i], i) iset.set_goniometer(goniometer[i], i) iset.set_scan(scan[i], i) else: # Get the template if template is None: template = template_regex(filenames[0])[0] else: template = str(template) # Check scan makes sense if scan: if check_format is True: assert scan.get_num_images() == len(filenames) # If any are None then read from format if beam is None and format_instance is not None: beam = format_instance.get_beam() if detector is None and format_instance is not None: detector = format_instance.get_detector() if goniometer is None and format_instance is not None: goniometer = format_instance.get_goniometer() if scan is None and format_instance is not None: scan = format_instance.get_scan() if scan is not None: for f in filenames[1:]: format_instance = Class(f, **format_kwargs) scan += format_instance.get_scan() assert beam is not None, "Can't create Sweep without beam" assert detector is not None, "Can't create Sweep without detector" assert goniometer is not None, "Can't create Sweep without goniometer" assert scan is not None, "Can't create Sweep without scan" # Create the sweep iset = ImageSweep( ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=Class, template=template, ), beam=beam, detector=detector, goniometer=goniometer, scan=scan, ) # Return the imageset return iset
def get_imageset( Class, filenames, beam=None, detector=None, goniometer=None, scan=None, as_sweep=False, as_imageset=False, single_file_indices=None, format_kwargs=None, template=None, check_format=True, lazy=False, ): """ Factory method to create an imageset """ from dxtbx.imageset import ImageSetData from dxtbx.imageset import ImageSweep from os.path import abspath from scitbx.array_family import flex if isinstance(filenames, str): filenames = [filenames] elif len(filenames) > 1: assert len(set(filenames)) == 1 filenames = filenames[0:1] # Make filenames absolute filenames = map(abspath, filenames) # Make it a dictionary if format_kwargs is None: format_kwargs = {} # If get_num_images hasn't been implemented, we need indices for number of images if Class.get_num_images == FormatMultiImage.get_num_images: assert single_file_indices is not None assert min(single_file_indices) >= 0 num_images = max(single_file_indices) + 1 else: num_images = None # Get some information from the format class reader = Class.get_reader()(filenames, num_images=num_images, **format_kwargs) masker = Class.get_masker()(filenames, num_images=num_images, **format_kwargs) # Get the format instance assert len(filenames) == 1 if check_format is True: format_instance = Class(filenames[0], **format_kwargs) else: format_instance = None if not as_sweep: lazy = True # Read the vendor type if check_format is True: vendor = format_instance.get_vendortype() else: vendor = "" # Get the format kwargs params = format_kwargs # Check if we have a sweep # Make sure only 1 or none is set assert [as_imageset, as_sweep].count(True) < 2 if as_imageset: is_sweep = False elif as_sweep: is_sweep = True else: if scan is None and format_instance is None: raise RuntimeError( """ One of the following needs to be set - as_imageset=True - as_sweep=True - scan - check_format=True """ ) if scan is None: test_scan = format_instance.get_scan() else: test_scan = scan if test_scan is not None and test_scan.get_oscillation()[1] != 0: is_sweep = True else: is_sweep = False assert not (as_sweep and lazy), "No lazy support for sweeps" if single_file_indices is not None: single_file_indices = flex.size_t(single_file_indices) # Create an imageset or sweep if not is_sweep: # Use imagesetlazy # Setup ImageSetLazy and just return it. No models are set. if lazy: from dxtbx.imageset import ImageSetLazy iset = ImageSetLazy( ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=Class, ), indices=single_file_indices, ) return iset # Create the imageset from dxtbx.imageset import ImageSet iset = ImageSet( ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=Class, ), indices=single_file_indices, ) # If any are None then read from format if [beam, detector, goniometer, scan].count(None) != 0: # Get list of models beam = [] detector = [] goniometer = [] scan = [] for i in range(format_instance.get_num_images()): beam.append(format_instance.get_beam(i)) detector.append(format_instance.get_detector(i)) goniometer.append(format_instance.get_goniometer(i)) scan.append(format_instance.get_scan(i)) if single_file_indices is None: single_file_indices = list(range(format_instance.get_num_images())) # Set the list of models for i in range(len(single_file_indices)): iset.set_beam(beam[single_file_indices[i]], i) iset.set_detector(detector[single_file_indices[i]], i) iset.set_goniometer(goniometer[single_file_indices[i]], i) iset.set_scan(scan[single_file_indices[i]], i) else: # Get the template template = filenames[0] # Check indices are sequential if single_file_indices is not None: assert all( i + 1 == j for i, j in zip(single_file_indices[:-1], single_file_indices[1:]) ) num_images = len(single_file_indices) else: num_images = format_instance.get_num_images() # Check the scan makes sense - we must want to use <= total images if scan is not None: assert scan.get_num_images() <= num_images # If any are None then read from format if beam is None: beam = format_instance.get_beam() if detector is None: detector = format_instance.get_detector() if goniometer is None: goniometer = format_instance.get_goniometer() if scan is None: scan = format_instance.get_scan() if scan is not None: for f in filenames[1:]: format_instance = Class(f, **format_kwargs) scan += format_instance.get_scan() isetdata = ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=Class, template=template, ) # Create the sweep iset = ImageSweep( isetdata, beam=beam, detector=detector, goniometer=goniometer, scan=scan, indices=single_file_indices, ) # Return the imageset return iset
def from_parameters(reflections, experiments, known_crystal_models=None, params=None): '''Sets up indexer object that will be used for indexing ''' # if params is None: # params = master_params if known_crystal_models is not None: #from dials.algorithms.indexing.known_orientation \ # import indexer_known_orientation #idxr = iota_indexer_known_orientation( # reflections, experiments, params, known_crystal_models) idxr = IOTA_StillsIndexerKnownOrientation(reflections, experiments, params, known_crystal_models) #idxr = indexer_known_orientation( # reflections, imagesets, params, known_crystal_models) else: has_stills = False has_sweeps = False for expt in experiments: if expt.goniometer is None or expt.scan is None or \ expt.scan.get_oscillation()[1] == 0: if has_sweeps: raise Sorry( "Please provide only stills or only sweeps, not both" ) has_stills = True else: if has_stills: raise Sorry( "Please provide only stills or only sweeps, not both" ) has_sweeps = True assert not (has_stills and has_sweeps) use_stills_indexer = has_stills if not (params.indexing.stills.indexer is libtbx.Auto or params.indexing.stills.indexer.lower() == 'auto'): if params.indexing.stills.indexer == 'stills': use_stills_indexer = True elif params.indexing.stills.indexer == 'sweeps': use_stills_indexer = False else: assert False if params.indexing.basis_vector_combinations.max_refine is libtbx.Auto: params.indexing.basis_vector_combinations.max_refine = 5 # Ensure the indexer and downstream applications treat this as set of stills from dxtbx.imageset import ImageSet # DIALS 2.0 stuff here for experiment in experiments: experiment.imageset = ImageSet(experiment.imageset.data(), experiment.imageset.indices()) experiment.imageset.set_scan(None) experiment.imageset.set_goniometer(None) experiment.scan = None experiment.goniometer = None # Old code prior to DIALS 2.0 #reset_sets = [] #for i in range(len(imagesets)): # imagesweep = imagesets.pop(0) # imageset = ImageSet(imagesweep.data(), imagesweep.indices()) # imageset.set_scan(None) # imageset.set_goniometer(None) # reset_sets.append(imageset) #imagesets.extend(reset_sets) if known_crystal_models is not None: print('Known Orientation Indexing') return idxr #from dials.algorithms.indexing.known_orientation \ # import indexer_known_orientation #idxr = indexer_known_orientation( # reflections, experiments, params, known_crystal_models) for entry_point in pkg_resources.iter_entry_points( "dials.index.basis_vector_search"): if params.indexing.method == entry_point.name: idxr = IOTA_StillsIndexerBasisVectorSearch(reflections, experiments, params=params) return idxr
def from_parameters(reflections, experiments, known_crystal_models=None, params=None): if known_crystal_models is not None: from dials.algorithms.indexing.known_orientation import ( IndexerKnownOrientation, ) if params.indexing.known_symmetry.space_group is None: params.indexing.known_symmetry.space_group = ( known_crystal_models[0].get_space_group().info()) idxr = IndexerKnownOrientation(reflections, experiments, params, known_crystal_models) else: has_stills = False has_sequences = False for expt in experiments: if isinstance(expt.imageset, ImageSequence): has_sequences = True else: has_stills = True if has_stills and has_sequences: raise ValueError( "Please provide only stills or only sequences, not both") use_stills_indexer = has_stills if not (params.indexing.stills.indexer is libtbx.Auto or params.indexing.stills.indexer.lower() == "auto"): if params.indexing.stills.indexer == "stills": use_stills_indexer = True elif params.indexing.stills.indexer == "sequences": use_stills_indexer = False else: assert False if params.indexing.basis_vector_combinations.max_refine is libtbx.Auto: if use_stills_indexer: params.indexing.basis_vector_combinations.max_refine = 5 else: params.indexing.basis_vector_combinations.max_refine = 50 if use_stills_indexer: # Ensure the indexer and downstream applications treat this as set of stills from dxtbx.imageset import ImageSet # , MemImageSet for experiment in experiments: experiment.imageset = ImageSet( experiment.imageset.data(), experiment.imageset.indices()) # if isinstance(imageset, MemImageSet): # imageset = MemImageSet(imagesequence._images, imagesequence.indices()) # else: # imageset = ImageSet(imagesequence.reader(), imagesequence.indices()) # imageset._models = imagesequence._models experiment.imageset.set_scan(None) experiment.imageset.set_goniometer(None) experiment.scan = None experiment.goniometer = None IndexerType = None for entry_point in pkg_resources.iter_entry_points( "dials.index.basis_vector_search"): if params.indexing.method == entry_point.name: if use_stills_indexer: # do something from dials.algorithms.indexing.stills_indexer import ( StillsIndexerBasisVectorSearch as IndexerType, ) else: from dials.algorithms.indexing.lattice_search import ( BasisVectorSearch as IndexerType, ) if IndexerType is None: for entry_point in pkg_resources.iter_entry_points( "dials.index.lattice_search"): if params.indexing.method == entry_point.name: if use_stills_indexer: from dials.algorithms.indexing.stills_indexer import ( StillsIndexerLatticeSearch as IndexerType, ) else: from dials.algorithms.indexing.lattice_search import ( LatticeSearch as IndexerType, ) assert IndexerType is not None idxr = IndexerType(reflections, experiments, params=params) return idxr
def from_parameters(reflections, experiments, known_crystal_models=None, params=None): if known_crystal_models is not None: from dials.algorithms.indexing.known_orientation import ( IndexerKnownOrientation, ) if params.indexing.known_symmetry.space_group is None: params.indexing.known_symmetry.space_group = ( known_crystal_models[0].get_space_group().info()) idxr = IndexerKnownOrientation(reflections, experiments, params, known_crystal_models) else: has_stills = False has_sweeps = False for expt in experiments: if (expt.goniometer is None or expt.scan is None or expt.scan.get_oscillation()[1] == 0): if has_sweeps: raise Sorry( "Please provide only stills or only sweeps, not both" ) has_stills = True else: if has_stills: raise Sorry( "Please provide only stills or only sweeps, not both" ) has_sweeps = True assert not (has_stills and has_sweeps) use_stills_indexer = has_stills if not (params.indexing.stills.indexer is libtbx.Auto or params.indexing.stills.indexer.lower() == "auto"): if params.indexing.stills.indexer == "stills": use_stills_indexer = True elif params.indexing.stills.indexer == "sweeps": use_stills_indexer = False else: assert False if params.indexing.basis_vector_combinations.max_refine is libtbx.Auto: if use_stills_indexer: params.indexing.basis_vector_combinations.max_refine = 5 else: params.indexing.basis_vector_combinations.max_refine = 50 if use_stills_indexer: # Ensure the indexer and downstream applications treat this as set of stills from dxtbx.imageset import ImageSet # , MemImageSet for experiment in experiments: experiment.imageset = ImageSet( experiment.imageset.data(), experiment.imageset.indices()) # if isinstance(imageset, MemImageSet): # imageset = MemImageSet(imagesweep._images, imagesweep.indices()) # else: # imageset = ImageSet(imagesweep.reader(), imagesweep.indices()) # imageset._models = imagesweep._models experiment.imageset.set_scan(None) experiment.imageset.set_goniometer(None) experiment.scan = None experiment.goniometer = None if params.indexing.method in ("fft1d", "fft3d", "real_space_grid_search"): if use_stills_indexer: # do something from dials.algorithms.indexing.stills_indexer import ( StillsIndexerBasisVectorSearch as BasisVectorSearch, ) else: from dials.algorithms.indexing.lattice_search import ( BasisVectorSearch, ) idxr = BasisVectorSearch(reflections, experiments, params=params) return idxr
def test_elliptical_distortion(run_in_tmpdir): """Create distortion maps for elliptical distortion using a dummy experiments with a small detector, for speed. Check those maps seem sensible""" # Make a detector model d = make_detector() # The beam is also essential for a experiments to be serialisable b = Beam((0, 0, 1), 1.0) # Create and write out a experiments imageset = ImageSet(ImageSetData(Reader(None, ["non-existent.cbf"]), None)) imageset.set_detector(d) imageset.set_beam(b) experiments = ExperimentListFactory.from_imageset_and_crystal( imageset, None) experiments.as_json("dummy.expt") # Centre of distortion will be the far corner from the origin of the first # panel centre_xy = d[0].get_image_size_mm() # Generate distortion maps cmd = ("dials.generate_distortion_maps dummy.expt " "mode=ellipse centre_xy={},{} " "phi=0 l1=1.0 l2=0.95").format(*centre_xy) easy_run.fully_buffered(command=cmd).raise_if_errors() # Load the maps with open("dx.pickle", "rb") as f: dx = pickle.load(f) with open("dy.pickle", "rb") as f: dy = pickle.load(f) # Check there are 4 maps each assert len(dx) == len(dy) == 4 # Ellipse has phi=0, so all correction is in the dy map for arr in dx: assert min(arr) == max(arr) == 0.0 # The ellipse correction is centred at the middle of the detector and all in # the Y direction. Therefore we expect a few things from the dy maps: # # (1) Within each panel the columns of the array are identical. # (2) The two upper panels should be the same # (3) The two lower panels should be the same. # (4) One column from an upper panel is a negated, reversed column from a # lower panel. # # All together expect the 4 dy maps to look something like this: # # /-----------\ /-----------\ # |-3 -3 -3 -3| |-3 -3 -3 -3| # |-2 -2 -2 -2| |-2 -2 -2 -2| # |-1 -1 -1 -1| |-1 -1 -1 -1| # | 0 0 0 0| | 0 0 0 0| # \-----------/ \-----------/ # /-----------\ /-----------\ # | 0 0 0 0| | 0 0 0 0| # | 1 1 1 1| | 1 1 1 1| # | 2 2 2 2| | 2 2 2 2| # | 3 3 3 3| | 3 3 3 3| # \-----------/ \-----------/ # So the fundamental data is all in the first column of first panel's map col0 = dy[0].matrix_copy_column(0) # The correction should be 5% of the distance from the ellipse centre to a # corrected pixel (l2 = 0.95 above) along the slow axis. Check that is the # case (for the first pixel at least) vec_centre_to_first_px = matrix.col(d[0].get_pixel_lab_coord( (0.5, 0.5))) - matrix.col(d[0].get_lab_coord(centre_xy)) dist_centre_to_first_px = vec_centre_to_first_px.dot( matrix.col(d[0].get_slow_axis())) corr_mm = dist_centre_to_first_px * 0.05 corr_px = corr_mm / d[0].get_pixel_size()[1] assert col0[0] == pytest.approx(corr_px) # Test (1) from above list for panel 0 for i in range(1, 50): assert (col0 == dy[0].matrix_copy_column(i)).all_eq(True) # Test (2) assert (dy[0] == dy[1]).all_eq(True) # Test (3) assert (dy[2] == dy[3]).all_eq(True) # Test (4) assert col0 == pytest.approx(-1.0 * dy[2].matrix_copy_column(0).reversed()) # Test (1) for panel 2 as well, which then covers everything needed col0 = dy[2].matrix_copy_column(0) for i in range(1, 50): assert (col0 == dy[2].matrix_copy_column(i)).all_eq(True)
def test_cspad_cbf_in_memory(self): from os.path import join, exists import os, dxtbx from uuid import uuid4 from dials.command_line.stills_process import phil_scope, Processor from libtbx.phil import parse from dxtbx.imageset import ImageSet, ImageSetData, MemReader, MemMasker from dxtbx.datablock import DataBlockFactory from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory import cPickle as pickle dirname = 'tmp_%s' % uuid4().hex os.mkdir(dirname) os.chdir(dirname) assert exists(join(self.lcls_path, 'idx-20130301060858801.cbf')) f = open("process_lcls.phil", 'w') f.write(""" dispatch.squash_errors = False spotfinder { filter.min_spot_size=2 threshold.dispersion.gain=25 threshold.dispersion.global_threshold=100 } indexing { known_symmetry { space_group = P6122 unit_cell = 92.9 92.9 130.4 90 90 120 } refinement_protocol.d_min_start=1.7 stills.refine_candidates_with_known_symmetry=True } """) f.close() params = phil_scope.fetch( parse(file_name="process_lcls.phil")).extract() params.output.datablock_filename = None processor = Processor(params) mem_img = dxtbx.load(join(self.lcls_path, 'idx-20130301060858801.cbf')) raw_data = mem_img.get_raw_data( ) # cache the raw data to prevent swig errors mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle) mem_img._raw_data = raw_data mem_img._cbf_handle = None # drop the file handle to prevent swig errors imgset = ImageSet( ImageSetData(MemReader([mem_img]), MemMasker([mem_img]))) imgset.set_beam(mem_img.get_beam()) imgset.set_detector(mem_img.get_detector()) datablock = DataBlockFactory.from_imageset(imgset)[0] processor.process_datablock("20130301060858801", datablock) # index/integrate the image result = "idx-20130301060858801_integrated.pickle" #n_refls = range(140,152) # large ranges to handle platform-specific differences # 09/20/17 Changes to still indexer: refine candidate basis vectors in target symmetry if supplied #n_refls = range(128,140) # large ranges to handle platform-specific differences # 09/27/17 Bugfix for refine_candidates_with_known_symmetry n_refls = range( 140, 152) # large ranges to handle platform-specific differences table = pickle.load(open(result, 'rb')) assert len(table) in n_refls, len(table) assert 'id' in table assert (table['id'] == 0).count(False) == 0 print 'OK'
def __call__(self, imageset): ''' Override the parameters ''' from dxtbx.imageset import ImageSet from dxtbx.imageset import ImageSweep from dxtbx.model import BeamFactory from dxtbx.model import DetectorFactory from dxtbx.model import GoniometerFactory from dxtbx.model import ScanFactory from copy import deepcopy if self.params.geometry.convert_sweeps_to_stills: imageset = ImageSet(data=imageset.data()) if not isinstance(imageset, ImageSweep): if self.params.geometry.convert_stills_to_sweeps: imageset = self.convert_stills_to_sweep(imageset) if isinstance(imageset, ImageSweep): beam = BeamFactory.from_phil( self.params.geometry, imageset.get_beam()) detector = DetectorFactory.from_phil( self.params.geometry, imageset.get_detector(), beam) goniometer = GoniometerFactory.from_phil( self.params.geometry, imageset.get_goniometer()) scan = ScanFactory.from_phil( self.params.geometry, deepcopy(imageset.get_scan())) i0, i1 = scan.get_array_range() j0, j1 = imageset.get_scan().get_array_range() if i0 < j0 or i1 > j1: imageset = self.extrapolate_imageset( imageset = imageset, beam = beam, detector = detector, goniometer = goniometer, scan = scan) else: imageset.set_beam(beam) imageset.set_detector(detector) imageset.set_goniometer(goniometer) imageset.set_scan(scan) else: for i in range(len(imageset)): beam = BeamFactory.from_phil( self.params.geometry, imageset.get_beam(i)) detector = DetectorFactory.from_phil( self.params.geometry, imageset.get_detector(i), beam) goniometer = GoniometerFactory.from_phil( self.params.geometry, imageset.get_goniometer(i)) scan = ScanFactory.from_phil( self.params.geometry, imageset.get_scan(i)) imageset.set_beam(beam, i) imageset.set_detector(detector, i) imageset.set_goniometer(goniometer, i) imageset.set_scan(scan, i) return imageset
class CctbxPsanaEventProcessor(Processor): """ Processor class for psana events """ def __init__(self, params_filename, output_tag, logfile=None): """ @param params_filename cctbx.xfel/DIALS parameter file for processing @output_tag String that will prefix output files @logfile File name for logging """ self.parsed_params = parse(file_name=params_filename) dials_params = phil_scope.fetch(self.parsed_params).extract() super(CctbxPsanaEventProcessor, self).__init__(dials_params, output_tag) self.update_geometry = ManualGeometryUpdater(dials_params) simple_script = SimpleScript(dials_params) simple_script.load_reference_geometry() self.reference_detector = getattr(simple_script, 'reference_detector', None) self.output_tag = output_tag self.detector_params = None if logfile is not None: log.config(logfile=logfile) def setup_run(self, run, psana_detector): """ Initialize processing for a given run @param run psana Run object @param psana_detector psana Detector object """ if psana_detector.is_cspad(): format_class = FormatXTCCspadSingleEvent detector_scope = cspad_locator_scope elif psana_detector.is_epix10ka2m(): format_class = FormatXTCEpixSingleEvent detector_scope = epix_locator_scope elif psana_detector.is_jungfrau(): format_class = FormatXTCJungfrauSingleEvent detector_scope = jungfrau_locator_scope elif 'rayonix' in psana_detector.name.dev.lower(): format_class = FormatXTCRayonixSingleEvent detector_scope = rayonix_locator_scope else: raise RuntimeError('Unrecognized detector %s' % psana_detector.name) detector_params = detector_scope.fetch(self.parsed_params).extract() self.dxtbx_img = format_class(detector_params, run, psana_detector) self.imageset = ImageSet( ImageSetData(MemReader([self.dxtbx_img]), None)) def process_event(self, event, event_tag): """ Process a single psana event @param event psana Event object @param event_tag string identifying the event """ experiments = self.experiments_from_event(event) self.process_experiments('%s_%s' % (self.output_tag, event_tag), experiments) def experiments_from_event(self, event): """ Create an ExperimentList from a psana Event @param event psana Event object """ self.dxtbx_img.event = event self.imageset.set_beam(self.dxtbx_img.get_beam()) self.imageset.set_detector(self.dxtbx_img.get_detector()) self.update_geometry(self.imageset) experiments = ExperimentListFactory.from_imageset_and_crystal( self.imageset, None) if self.reference_detector is not None: experiment = experiments[0] sync_geometry(self.reference_detector.hierarchy(), self.imageset.get_detector().hierarchy()) experiment.detector = self.imageset.get_detector() return experiments
def get_imageset( cls, filenames, beam=None, detector=None, goniometer=None, scan=None, as_sequence=False, as_imageset=False, single_file_indices=None, format_kwargs=None, template=None, check_format=True, lazy=False, ): """ Factory method to create an imageset """ if isinstance(filenames, str): filenames = [filenames] elif len(filenames) > 1: assert len(set(filenames)) == 1 filenames = filenames[0:1] # Make filenames absolute filenames = [os.path.abspath(x) for x in filenames] # Make it a dictionary if format_kwargs is None: format_kwargs = {} # If get_num_images hasn't been implemented, we need indices for number of images if cls.get_num_images == FormatMultiImage.get_num_images: assert single_file_indices is not None assert min(single_file_indices) >= 0 num_images = max(single_file_indices) + 1 else: num_images = None # Get the format instance assert len(filenames) == 1 if check_format is True: format_instance = cls(filenames[0], **format_kwargs) if num_images is None and not lazy: # As we now have the actual format class we can get the number # of images from here. This saves having to create another # format class instance in the Reader() constructor # NOTE: Having this information breaks internal assumptions in # *Lazy classes, so they have to figure this out in # their own time. num_images = format_instance.get_num_images() else: format_instance = None if not as_sequence: lazy = True # Get some information from the format class reader = cls.get_reader()(filenames, num_images=num_images, **format_kwargs) # Read the vendor type if check_format is True: vendor = format_instance.get_vendortype() else: vendor = "" # Get the format kwargs params = format_kwargs # Check if we have a sequence # Make sure only 1 or none is set assert [as_imageset, as_sequence].count(True) < 2 if as_imageset: is_sequence = False elif as_sequence: is_sequence = True else: if scan is None and format_instance is None: raise RuntimeError( """ One of the following needs to be set - as_imageset=True - as_sequence=True - scan - check_format=True """ ) if scan is None: test_scan = format_instance.get_scan() else: test_scan = scan if test_scan is not None: is_sequence = True else: is_sequence = False assert not (as_sequence and lazy), "No lazy support for sequences" if single_file_indices is not None: single_file_indices = flex.size_t(single_file_indices) # Create an imageset or sequence if not is_sequence: # Use imagesetlazy # Setup ImageSetLazy and just return it. No models are set. if lazy: iset = ImageSetLazy( ImageSetData( reader=reader, masker=None, vendor=vendor, params=params, format=cls, ), indices=single_file_indices, ) return iset # Create the imageset iset = ImageSet( ImageSetData( reader=reader, masker=None, vendor=vendor, params=params, format=cls ), indices=single_file_indices, ) # If any are None then read from format if [beam, detector, goniometer, scan].count(None) != 0: # Get list of models beam = [] detector = [] goniometer = [] scan = [] for i in range(format_instance.get_num_images()): beam.append(format_instance.get_beam(i)) detector.append(format_instance.get_detector(i)) goniometer.append(format_instance.get_goniometer(i)) scan.append(format_instance.get_scan(i)) if single_file_indices is None: single_file_indices = list(range(format_instance.get_num_images())) # Set the list of models for i in range(len(single_file_indices)): iset.set_beam(beam[single_file_indices[i]], i) iset.set_detector(detector[single_file_indices[i]], i) iset.set_goniometer(goniometer[single_file_indices[i]], i) iset.set_scan(scan[single_file_indices[i]], i) else: # Get the template template = filenames[0] # Check indices are sequential if single_file_indices is not None: assert all( i + 1 == j for i, j in zip(single_file_indices[:-1], single_file_indices[1:]) ) num_images = len(single_file_indices) else: num_images = format_instance.get_num_images() # Check the scan makes sense - we must want to use <= total images if scan is not None: assert scan.get_num_images() <= num_images # If any are None then read from format if beam is None: beam = format_instance.get_beam() if detector is None: detector = format_instance.get_detector() if goniometer is None: goniometer = format_instance.get_goniometer() if scan is None: scan = format_instance.get_scan() if scan is not None: for f in filenames[1:]: format_instance = cls(f, **format_kwargs) scan += format_instance.get_scan() # Create the masker if format_instance is not None: masker = format_instance.get_masker(goniometer=goniometer) else: masker = None isetdata = ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=cls, template=template, ) # Create the sequence iset = ImageSequence( isetdata, beam=beam, detector=detector, goniometer=goniometer, scan=scan, indices=single_file_indices, ) if format_instance is not None: static_mask = format_instance.get_static_mask() if static_mask is not None: if not iset.external_lookup.mask.data.empty(): for m1, m2 in zip(static_mask, iset.external_lookup.mask.data): m1 &= m2.data() iset.external_lookup.mask.data = ImageBool(static_mask) else: iset.external_lookup.mask.data = ImageBool(static_mask) return iset
def tst_null_reader_imageset(self): from dxtbx.imageset import NullReader, ImageSet from dxtbx.model import Beam, Detector paths = ['hello_world.cbf'] # Create the null reader reader = NullReader(paths) # Create the imageset imageset = ImageSet(reader) # Try to get an item try: imageset[0] assert(False) except Exception: print 'OK' # Try to slice the imageset imageset2 = imageset[0:1] print 'OK' # Try some functions which should work assert(len(imageset) == 1) assert(imageset == imageset) assert(imageset.indices() == [0]) assert(imageset.is_valid()) print 'OK' # Try to get models (expect failure) try: imageset.get_image_models(0) assert(False) except Exception: print 'OK' # Get the image paths assert(imageset.paths() == paths) assert(imageset.get_path(0) == paths[0]) print 'OK' imageset.set_beam(Beam(), 0) imageset.set_detector(Detector(), 0) assert(isinstance(imageset.get_beam(0), Beam)) assert(isinstance(imageset.get_detector(0), Detector)) print 'OK'