def experiment(): beam = BeamFactory.make_beam(wavelength=0.97625, sample_to_source=(0, 0, 1)) detector = DetectorFactory.simple( sensor="PAD", distance=265.27, beam_centre=(210.7602, 205.27684), fast_direction="+x", slow_direction="-y", pixel_size=(0.172, 0.172), image_size=(2463, 2527), trusted_range=(-1, 1e8), ) goniometer = GoniometerFactory.single_axis() scan = ScanFactory.make_scan( image_range=(1, 20), exposure_times=0.067, oscillation=(82, 0.15), epochs=[0] * 20, ) isetdata = ImageSetData( reader=Format.Reader(None, ["path"] * len(scan)), masker=None ) iset = ImageSequence( isetdata, beam=beam, detector=detector, goniometer=goniometer, scan=scan ) return Experiment( imageset=iset, beam=beam, detector=detector, goniometer=goniometer, scan=scan )
def datablock_from_numpyarrays(image, detector, beam, mask=None): """ So that one can do e.g. >> dblock = datablock_from_numpyarrays( image, detector, beam) >> refl = flex.reflection_table.from_observations(dblock, spot_finder_params) without having to utilize the harddisk :param image: numpy array image, or list of numpy arrays :param mask: numpy mask, should be same shape format as numpy array :param detector: dxtbx detector model :param beam: dxtbx beam model :return: datablock for the image """ if isinstance(image, list): image = np.array(image) if mask is not None: if isinstance(mask, list): mask = np.array(mask).astype(bool) I = FormatInMemory(image=image, mask=mask) reader = MemReader([I]) #masker = MemMasker([I]) iset_Data = ImageSetData(reader, None) #, masker) iset = ImageSet(iset_Data) iset.set_beam(beam) iset.set_detector(detector) dblock = DataBlockFactory.from_imageset([iset])[0] return dblock
def index(reflections, detector, known_symmetry, beams): params = master_phil_scope.extract() params.refinement.parameterisation.beam.fix = "all" params.refinement.parameterisation.detector.fix = "all" params.indexing.known_symmetry.space_group = known_symmetry.space_group_info( ) params.refinement.verbosity = 3 params.indexing.refinement_protocol.d_min_start params.indexing.refinement_protocol.n_macro_cycles = 1 params.indexing.known_symmetry.unit_cell = known_symmetry.unit_cell() params.indexing.debug = True params.indexing.known_symmetry.absolute_angle_tolerance = 5.0 params.indexing.known_symmetry.relative_length_tolerance = 0.3 filenames = [""] reader = Reader(filenames) masker = Masker(filenames) iset_data = ImageSetData(reader, masker) imgset = ImageSet(iset_data) imgset.set_beam(beams[0]) imgset.set_detector(detector) imagesets = [imgset] # dermen: # modified rmsd_min_px this so test will pass, was originally set # to 2 and the stills indexer's best.rmsd was of order ~2.8ish # I dont know enough about the code itself to make the call on # whether this is legit or not or whether 2.8 best.rmsd is # something we should focus on improving... params.indexing.stills.rmsd_min_px = 3.5 orient = indexer_two_color(reflections, imagesets, params) orient.index() return orient
def index(reflections, detector, known_symmetry, beams): '''calls the two color indexer class after setting up the phil parameters and returns a class object''' # cmd_line = command_line.argument_interpreter(master_params=master_phil_scope) # working_phil = cmd_line.process_and_fetch(args=[]) params = master_phil_scope.extract() params.refinement.parameterisation.beam.fix = "all" params.refinement.parameterisation.detector.fix = "all" params.indexing.known_symmetry.space_group = known_symmetry.space_group_info( ) params.refinement.verbosity = 3 params.indexing.refinement_protocol.d_min_start params.indexing.refinement_protocol.n_macro_cycles = 1 params.indexing.known_symmetry.unit_cell = known_symmetry.unit_cell() params.indexing.debug = True params.indexing.known_symmetry.absolute_angle_tolerance = 5.0 params.indexing.known_symmetry.relative_length_tolerance = 0.3 params.indexing.stills.rmsd_min_px = 3.5 filenames = [""] reader = Reader(filenames) masker = Masker(filenames) imgsetdata = ImageSetData(reader, masker) imgset = ImageSet(imgsetdata) imgset.set_beam(beams[0]) imgset.set_detector(detector) imagesets = [imgset] orient = indexer_two_color(reflections, imagesets, params) orient.index() return orient
def index(reflections, detector, known_symmetry, beams): '''sets up two color indexer parameters and calls the indexer; returns the candidate basis vectors from the 2-D grid search in two color indexer''' # cmd_line = command_line.argument_interpreter(master_params=master_phil_scope) # working_phil = cmd_line.process_and_fetch(args=[]) params = master_phil_scope.extract() params.refinement.parameterisation.beam.fix = "all" params.refinement.parameterisation.detector.fix = "all" params.indexing.known_symmetry.space_group = known_symmetry.space_group_info( ) params.refinement.verbosity = 3 params.indexing.refinement_protocol.d_min_start params.indexing.refinement_protocol.n_macro_cycles = 1 # 5 params.indexing.known_symmetry.unit_cell = known_symmetry.unit_cell() params.indexing.debug = True params.indexing.known_symmetry.absolute_angle_tolerance = 5.0 params.indexing.known_symmetry.relative_length_tolerance = 0.3 params.indexing.stills.rmsd_min_px = 3.5 filenames = [""] reader = Reader(filenames) masker = Masker(filenames) iset_data = ImageSetData(reader, masker) imgset = ImageSet(iset_data) imgset.set_beam(beams[0]) imgset.set_detector(detector) imagesets = [imgset] orient = indexer_two_color(reflections, imagesets, params) #, beams=beams) orient.index() cvecs = orient.candidate_basis_vectors return cvecs
def setup_run(self, run, psana_detector): """ Initialize processing for a given run @param run psana Run object @param psana_detector psana Detector object """ if psana_detector.is_cspad(): format_class = FormatXTCCspadSingleEvent detector_scope = cspad_locator_scope elif psana_detector.is_epix10ka2m(): format_class = FormatXTCEpixSingleEvent detector_scope = epix_locator_scope elif psana_detector.is_jungfrau(): format_class = FormatXTCJungfrauSingleEvent detector_scope = jungfrau_locator_scope elif 'rayonix' in psana_detector.name.dev.lower(): format_class = FormatXTCRayonixSingleEvent detector_scope = rayonix_locator_scope else: raise RuntimeError('Unrecognized detector %s' % psana_detector.name) detector_params = detector_scope.fetch(self.parsed_params).extract() self.dxtbx_img = format_class(detector_params, run, psana_detector) self.imageset = ImageSet( ImageSetData(MemReader([self.dxtbx_img]), None))
def generate_spots(crystal_model, detector, beam, goniometer=None, scan=None, sel_fraction=1.0): experiment = Experiment(beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal_model) # if we don't set the imageset then from_predictions uses the StillsReflectionPredictor :-( filenames = [""] * len(scan) reader = Reader(filenames) masker = Masker(filenames) data = ImageSetData(reader, masker) imageset = ImageSweep(data, beam, detector, goniometer, scan) experiment.imageset = imageset predicted = flex.reflection_table.from_predictions(experiment) sel = flex.random_selection(len(predicted), int(math.floor(sel_fraction * len(predicted)))) predicted = predicted.select(sel) predicted['imageset_id'] = flex.size_t(len(predicted), 0) predicted['xyzobs.px.value'] = predicted['xyzcal.px'] predicted['xyzobs.px.variance'] = flex.vec3_double(len(predicted), (0.5, 0.5, 0.5)) return predicted
def make_imageset(data, beam, detector): format_class = FormatBraggInMemoryMultiPanel(data) reader = MemReaderNamedPath("virtual_Bragg_path", [format_class]) reader.format_class = FormatBraggInMemory imageset_data = ImageSetData(reader, None) imageset = ImageSet(imageset_data) imageset.set_beam(beam) imageset.set_detector(detector) return imageset
def imageset(self): format_class = FormatBraggInMemory(self.raw_pixels) reader = MemReaderNamedPath("virtual_Bragg_path", [format_class]) reader.format_class = FormatBraggInMemory imageset_data = ImageSetData(reader, None) imageset = ImageSet(imageset_data) imageset.set_beam(self.beam) imageset.set_detector(self.detector) return imageset
def test_cspad_cbf_in_memory(dials_regression, run_in_tmpdir): # Check the data files for this test exist image_path = os.path.join( dials_regression, "image_examples", "LCLS_cspad_nexus", "idx-20130301060858801.cbf", ) assert os.path.isfile(image_path) with open("process_lcls.phil", "w") as f: f.write( """ dispatch.squash_errors = False spotfinder { filter.min_spot_size=2 threshold.dispersion.gain=25 threshold.dispersion.global_threshold=100 } indexing { known_symmetry { space_group = P6122 unit_cell = 92.9 92.9 130.4 90 90 120 } refinement_protocol.d_min_start=1.7 stills.refine_candidates_with_known_symmetry=True } """ ) params = phil_scope.fetch(parse(file_name="process_lcls.phil")).extract() params.output.experiments_filename = None processor = Processor(params) mem_img = dxtbx.load(image_path) raw_data = mem_img.get_raw_data() # cache the raw data to prevent swig errors mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle) mem_img._raw_data = raw_data mem_img._cbf_handle = None # drop the file handle to prevent swig errors imgset = ImageSet(ImageSetData(MemReader([mem_img]), None)) imgset.set_beam(mem_img.get_beam()) imgset.set_detector(mem_img.get_detector()) experiments = ExperimentListFactory.from_imageset_and_crystal(imgset, None) processor.process_experiments( "20130301060858801", experiments ) # index/integrate the image result = "idx-20130301060858801_integrated.refl" n_refls = list( range(140, 152) ) # large ranges to handle platform-specific differences table = flex.reflection_table.from_file(result) assert len(table) in n_refls, len(table) assert "id" in table assert (table["id"] == 0).count(False) == 0
def test_FormatPYunspecifiedStillInMemory(dials_regression): filename = os.path.join( dials_regression, "image_examples/LCLS_CXI/shot-s00-2011-12-02T21_07Z29.723_00569.pickle", ) assert not FormatPYunspecifiedStillInMemory.understand(filename) with open(filename, "rb") as f: d = pickle.load(f) assert FormatPYunspecifiedStillInMemory.understand(d) img = FormatPYunspecifiedStillInMemory(d) imageset = ImageSet(ImageSetData(MemReader([img]), None)) mask = imageset.get_mask(0) assert len(mask) == 1 assert mask[0].count(False) == 867109
def test_cspad_cbf_in_memory(dials_regression, run_in_tmpdir, composite_output): # Check the data files for this test exist image_path = os.path.join( dials_regression, "image_examples", "LCLS_cspad_nexus", "idx-20130301060858801.cbf", ) assert os.path.isfile(image_path) with open("process_lcls.phil", "w") as f: f.write(cspad_cbf_in_memory_phil) params = phil_scope.fetch(parse(file_name="process_lcls.phil")).extract() params.output.experiments_filename = None params.output.composite_output = composite_output if composite_output: processor = Processor(params, composite_tag="memtest") else: processor = Processor(params) mem_img = dxtbx.load(image_path) raw_data = mem_img.get_raw_data( ) # cache the raw data to prevent swig errors mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle) mem_img._raw_data = raw_data mem_img._cbf_handle = None # drop the file handle to prevent swig errors imgset = ImageSet(ImageSetData(MemReader([mem_img]), None)) imgset.set_beam(mem_img.get_beam()) imgset.set_detector(mem_img.get_detector()) experiments = ExperimentListFactory.from_imageset_and_crystal(imgset, None) processor.process_experiments("20130301060858801", experiments) # index/integrate the image if composite_output: processor.finalize() result = "idx-memtest_integrated.refl" else: result = "idx-20130301060858801_integrated.refl" n_refls = list(range( 140, 152)) # large ranges to handle platform-specific differences table = flex.reflection_table.from_file(result) assert len(table) in n_refls, len(table) assert "id" in table assert (table["id"] == 0).count(False) == 0
def datablock_from_numpyarrays(image, detector, beam, mask=None): """ So that one can do e.g. >> dblock = datablock_from_numpyarrays( image, detector, beam) >> refl = flex.reflection_table.from_observations(dblock, spot_finder_params) without having to utilize the harddisk :param image: numpy array image :param mask: numpy mask :param detector: dxtbx detector model :param beam: dxtbx beam model :return: datablock for the image """ I = FormatInMemory(image=image, mask=mask) reader = MemReader([I]) masker = MemMasker([I]) iset_Data = ImageSetData(reader, masker) iset = ImageSet(iset_Data) iset.set_beam(beam) iset.set_detector(detector) dblock = DataBlockFactory.from_imageset([iset])[0] return dblock
def datablock_from_numpyarrays(image, detector, beam, mask=None): """ put the numpy array image(s) into a dials datablock :param image: numpy array of images or an image :param detector: dxtbx det :param beam: dxtbx beam :param mask: mask same shape as image , 1 is not masked, boolean :return: """ if isinstance( image, list): image = np.array( image) if mask is not None: if isinstance( mask, list): mask = np.array(mask).astype(bool) I = FormatInMemory(image=image, mask=mask) reader = MemReader([I]) masker = MemMasker([I]) iset_Data = ImageSetData(reader, masker) iset = ImageSet(iset_Data) iset.set_beam(beam) iset.set_detector(detector) dblock = DataBlockFactory.from_imageset([iset])[0] return dblock
def test_imagesetdata(centroid_files): ReaderClass = FormatClass.get_reader() reader = ReaderClass(centroid_files) masker = FormatClass(centroid_files[0]).get_masker() handle = ImageSetData(reader, masker) assert handle.get_data(0).as_int().tile(0).data() assert handle.has_single_file_reader() is False path = handle.get_path(0) assert path == centroid_files[0] master_path = handle.get_master_path() assert master_path == "" identifier = handle.get_image_identifier(0) assert identifier == centroid_files[0] beam = FormatClass(centroid_files[0]).get_beam() detector = FormatClass(centroid_files[0]).get_detector() goniometer = FormatClass(centroid_files[0]).get_goniometer() scan = FormatClass(centroid_files[0]).get_scan() handle.set_beam(beam, 0) handle.set_detector(detector, 0) handle.set_goniometer(goniometer, 0) handle.set_scan(scan, 0) beam2 = handle.get_beam(0) detector2 = handle.get_detector(0) goniometer2 = handle.get_goniometer(0) scan2 = handle.get_scan(0) assert beam2 == beam assert detector2 == detector assert goniometer2 == goniometer assert scan2 == scan mask = flex.bool(flex.grid(10, 10), True) gain = flex.double(flex.grid(10, 10), 1) pedestal = flex.double(flex.grid(10, 10), 2) handle.external_lookup.mask.data = dxtbx.format.image.ImageBool( dxtbx.format.image.ImageTileBool(mask)) handle.external_lookup.gain.data = dxtbx.format.image.ImageDouble( dxtbx.format.image.ImageTileDouble(gain)) handle.external_lookup.pedestal.data = dxtbx.format.image.ImageDouble( dxtbx.format.image.ImageTileDouble(pedestal)) mask2 = handle.external_lookup.mask.data.tile(0).data() gain2 = handle.external_lookup.gain.data.tile(0).data() pedestal2 = handle.external_lookup.pedestal.data.tile(0).data() assert mask2.all_eq(mask) assert gain2.all_eq(gain) assert pedestal2.all_eq(pedestal)
def get_imageset( Class, filenames, beam=None, detector=None, goniometer=None, scan=None, as_sweep=False, as_imageset=False, single_file_indices=None, format_kwargs=None, template=None, check_format=True, lazy=False, ): """ Factory method to create an imageset """ from dxtbx.imageset import ImageSetData from dxtbx.imageset import ImageSweep from os.path import abspath from scitbx.array_family import flex if isinstance(filenames, str): filenames = [filenames] elif len(filenames) > 1: assert len(set(filenames)) == 1 filenames = filenames[0:1] # Make filenames absolute filenames = map(abspath, filenames) # Make it a dictionary if format_kwargs is None: format_kwargs = {} # If get_num_images hasn't been implemented, we need indices for number of images if Class.get_num_images == FormatMultiImage.get_num_images: assert single_file_indices is not None assert min(single_file_indices) >= 0 num_images = max(single_file_indices) + 1 else: num_images = None # Get some information from the format class reader = Class.get_reader()(filenames, num_images=num_images, **format_kwargs) masker = Class.get_masker()(filenames, num_images=num_images, **format_kwargs) # Get the format instance assert len(filenames) == 1 if check_format is True: format_instance = Class(filenames[0], **format_kwargs) else: format_instance = None if not as_sweep: lazy = True # Read the vendor type if check_format is True: vendor = format_instance.get_vendortype() else: vendor = "" # Get the format kwargs params = format_kwargs # Check if we have a sweep # Make sure only 1 or none is set assert [as_imageset, as_sweep].count(True) < 2 if as_imageset: is_sweep = False elif as_sweep: is_sweep = True else: if scan is None and format_instance is None: raise RuntimeError( """ One of the following needs to be set - as_imageset=True - as_sweep=True - scan - check_format=True """ ) if scan is None: test_scan = format_instance.get_scan() else: test_scan = scan if test_scan is not None and test_scan.get_oscillation()[1] != 0: is_sweep = True else: is_sweep = False assert not (as_sweep and lazy), "No lazy support for sweeps" if single_file_indices is not None: single_file_indices = flex.size_t(single_file_indices) # Create an imageset or sweep if not is_sweep: # Use imagesetlazy # Setup ImageSetLazy and just return it. No models are set. if lazy: from dxtbx.imageset import ImageSetLazy iset = ImageSetLazy( ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=Class, ), indices=single_file_indices, ) return iset # Create the imageset from dxtbx.imageset import ImageSet iset = ImageSet( ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=Class, ), indices=single_file_indices, ) # If any are None then read from format if [beam, detector, goniometer, scan].count(None) != 0: # Get list of models beam = [] detector = [] goniometer = [] scan = [] for i in range(format_instance.get_num_images()): beam.append(format_instance.get_beam(i)) detector.append(format_instance.get_detector(i)) goniometer.append(format_instance.get_goniometer(i)) scan.append(format_instance.get_scan(i)) if single_file_indices is None: single_file_indices = list(range(format_instance.get_num_images())) # Set the list of models for i in range(len(single_file_indices)): iset.set_beam(beam[single_file_indices[i]], i) iset.set_detector(detector[single_file_indices[i]], i) iset.set_goniometer(goniometer[single_file_indices[i]], i) iset.set_scan(scan[single_file_indices[i]], i) else: # Get the template template = filenames[0] # Check indices are sequential if single_file_indices is not None: assert all( i + 1 == j for i, j in zip(single_file_indices[:-1], single_file_indices[1:]) ) num_images = len(single_file_indices) else: num_images = format_instance.get_num_images() # Check the scan makes sense - we must want to use <= total images if scan is not None: assert scan.get_num_images() <= num_images # If any are None then read from format if beam is None: beam = format_instance.get_beam() if detector is None: detector = format_instance.get_detector() if goniometer is None: goniometer = format_instance.get_goniometer() if scan is None: scan = format_instance.get_scan() if scan is not None: for f in filenames[1:]: format_instance = Class(f, **format_kwargs) scan += format_instance.get_scan() isetdata = ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=Class, template=template, ) # Create the sweep iset = ImageSweep( isetdata, beam=beam, detector=detector, goniometer=goniometer, scan=scan, indices=single_file_indices, ) # Return the imageset return iset
def get_imageset( Class, input_filenames, beam=None, detector=None, goniometer=None, scan=None, as_imageset=False, as_sequence=False, single_file_indices=None, format_kwargs=None, template=None, check_format=True, ): """ Factory method to create an imageset """ # Import here to avoid cyclic imports from dxtbx.imageset import ImageSequence, ImageSet, ImageSetData # Get filename absolute paths, for entries that are filenames filenames = [ os.path.abspath(x) if not get_url_scheme(x) else x for x in input_filenames ] # Make it a dict if format_kwargs is None: format_kwargs = {} # Get some information from the format class reader = Class.get_reader()(filenames, **format_kwargs) # Get the format instance if check_format is True: Class._current_filename_ = None Class._current_instance_ = None format_instance = Class.get_instance(filenames[0], **format_kwargs) else: format_instance = None # Read the vendor type if check_format is True: vendor = format_instance.get_vendortype() else: vendor = "" # Get the format kwargs params = format_kwargs # Make sure only 1 or none is set assert [as_imageset, as_sequence].count(True) < 2 if as_imageset: is_sequence = False elif as_sequence: is_sequence = True else: if scan is None and format_instance is None: raise RuntimeError(""" One of the following needs to be set - as_imageset=True - as_sequence=True - scan - check_format=True """) if scan is None: test_scan = format_instance.get_scan() else: test_scan = scan if test_scan is not None and test_scan.get_oscillation()[1] != 0: is_sequence = True else: is_sequence = False # Create an imageset or sequence if not is_sequence: # Create the imageset iset = ImageSet( ImageSetData( reader=reader, masker=None, vendor=vendor, params=params, format=Class, )) # If any are None then read from format if [beam, detector, goniometer, scan].count(None) != 0: # Get list of models beam = [] detector = [] goniometer = [] scan = [] for f in filenames: format_instance = Class(f, **format_kwargs) beam.append(format_instance.get_beam()) detector.append(format_instance.get_detector()) goniometer.append(format_instance.get_goniometer()) scan.append(format_instance.get_scan()) # Set the list of models for i in range(len(filenames)): iset.set_beam(beam[i], i) iset.set_detector(detector[i], i) iset.set_goniometer(goniometer[i], i) iset.set_scan(scan[i], i) else: # Get the template if template is None: template = template_regex(filenames[0])[0] else: template = str(template) # Check scan makes sense if scan: if check_format is True: assert scan.get_num_images() == len(filenames) # If any are None then read from format if beam is None and format_instance is not None: beam = format_instance.get_beam() if detector is None and format_instance is not None: detector = format_instance.get_detector() if goniometer is None and format_instance is not None: goniometer = format_instance.get_goniometer() if scan is None and format_instance is not None: scan = format_instance.get_scan() if scan is not None: for f in filenames[1:]: format_instance = Class(f, **format_kwargs) scan += format_instance.get_scan() assert beam is not None, "Can't create Sequence without beam" assert detector is not None, "Can't create Sequence without detector" assert goniometer is not None, "Can't create Sequence without goniometer" assert scan is not None, "Can't create Sequence without scan" # Create the masker if format_instance is not None: masker = format_instance.get_masker(goniometer=goniometer) else: masker = None # Create the sequence iset = ImageSequence( ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=Class, template=template, ), beam=beam, detector=detector, goniometer=goniometer, scan=scan, ) if format_instance is not None: static_mask = format_instance.get_static_mask() if static_mask is not None: if not iset.external_lookup.mask.data.empty(): for m1, m2 in zip(static_mask, iset.external_lookup.mask.data): m1 &= m2.data() iset.external_lookup.mask.data = ImageBool(static_mask) else: iset.external_lookup.mask.data = ImageBool(static_mask) return iset
def get_imageset( Class, filenames, beam=None, detector=None, goniometer=None, scan=None, as_imageset=False, as_sweep=False, single_file_indices=None, format_kwargs=None, template=None, check_format=True, ): """ Factory method to create an imageset """ from dxtbx.imageset import ImageSetData from dxtbx.imageset import ImageSet from dxtbx.imageset import ImageSweep from dxtbx.sweep_filenames import template_regex from os.path import abspath # Get filename absolute paths filenames = map(abspath, filenames) # Make it a dict if format_kwargs is None: format_kwargs = {} # Get some information from the format class reader = Class.get_reader()(filenames, **format_kwargs) masker = Class.get_masker()(filenames, **format_kwargs) # Get the format instance if check_format is True: format_instance = Class(filenames[0], **format_kwargs) else: format_instance = None # Read the vendor type if check_format is True: vendor = format_instance.get_vendortype() else: vendor = "" # Get the format kwargs params = format_kwargs # Make sure only 1 or none is set assert [as_imageset, as_sweep].count(True) < 2 if as_imageset: is_sweep = False elif as_sweep: is_sweep = True else: if scan is None and format_instance is None: raise RuntimeError(""" One of the following needs to be set - as_imageset=True - as_sweep=True - scan - check_format=True """) if scan is None: test_scan = format_instance.get_scan() else: test_scan = scan if test_scan is not None and test_scan.get_oscillation()[1] != 0: is_sweep = True else: is_sweep = False # Create an imageset or sweep if not is_sweep: # Create the imageset iset = ImageSet( ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=Class, )) # If any are None then read from format if [beam, detector, goniometer, scan].count(None) != 0: # Get list of models beam = [] detector = [] goniometer = [] scan = [] for f in filenames: format_instance = Class(f, **format_kwargs) beam.append(format_instance.get_beam()) detector.append(format_instance.get_detector()) goniometer.append(format_instance.get_goniometer()) scan.append(format_instance.get_scan()) # Set the list of models for i in range(len(filenames)): iset.set_beam(beam[i], i) iset.set_detector(detector[i], i) iset.set_goniometer(goniometer[i], i) iset.set_scan(scan[i], i) else: # Get the template if template is None: template = template_regex(filenames[0])[0] else: template = str(template) # Check scan makes sense if scan: if check_format is True: assert scan.get_num_images() == len(filenames) # If any are None then read from format if beam is None and format_instance is not None: beam = format_instance.get_beam() if detector is None and format_instance is not None: detector = format_instance.get_detector() if goniometer is None and format_instance is not None: goniometer = format_instance.get_goniometer() if scan is None and format_instance is not None: scan = format_instance.get_scan() if scan is not None: for f in filenames[1:]: format_instance = Class(f, **format_kwargs) scan += format_instance.get_scan() assert beam is not None, "Can't create Sweep without beam" assert detector is not None, "Can't create Sweep without detector" assert goniometer is not None, "Can't create Sweep without goniometer" assert scan is not None, "Can't create Sweep without scan" # Create the sweep iset = ImageSweep( ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=Class, template=template, ), beam=beam, detector=detector, goniometer=goniometer, scan=scan, ) # Return the imageset return iset
def test_cspad_cbf_in_memory(self): from os.path import join, exists import os, dxtbx from uuid import uuid4 from dials.command_line.stills_process import phil_scope, Processor from libtbx.phil import parse from dxtbx.imageset import ImageSet, ImageSetData, MemReader, MemMasker from dxtbx.datablock import DataBlockFactory from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory import cPickle as pickle dirname = 'tmp_%s' % uuid4().hex os.mkdir(dirname) os.chdir(dirname) assert exists(join(self.lcls_path, 'idx-20130301060858801.cbf')) f = open("process_lcls.phil", 'w') f.write(""" dispatch.squash_errors = False spotfinder { filter.min_spot_size=2 threshold.dispersion.gain=25 threshold.dispersion.global_threshold=100 } indexing { known_symmetry { space_group = P6122 unit_cell = 92.9 92.9 130.4 90 90 120 } refinement_protocol.d_min_start=1.7 stills.refine_candidates_with_known_symmetry=True } """) f.close() params = phil_scope.fetch( parse(file_name="process_lcls.phil")).extract() params.output.datablock_filename = None processor = Processor(params) mem_img = dxtbx.load(join(self.lcls_path, 'idx-20130301060858801.cbf')) raw_data = mem_img.get_raw_data( ) # cache the raw data to prevent swig errors mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle) mem_img._raw_data = raw_data mem_img._cbf_handle = None # drop the file handle to prevent swig errors imgset = ImageSet( ImageSetData(MemReader([mem_img]), MemMasker([mem_img]))) imgset.set_beam(mem_img.get_beam()) imgset.set_detector(mem_img.get_detector()) datablock = DataBlockFactory.from_imageset(imgset)[0] processor.process_datablock("20130301060858801", datablock) # index/integrate the image result = "idx-20130301060858801_integrated.pickle" #n_refls = range(140,152) # large ranges to handle platform-specific differences # 09/20/17 Changes to still indexer: refine candidate basis vectors in target symmetry if supplied #n_refls = range(128,140) # large ranges to handle platform-specific differences # 09/27/17 Bugfix for refine_candidates_with_known_symmetry n_refls = range( 140, 152) # large ranges to handle platform-specific differences table = pickle.load(open(result, 'rb')) assert len(table) in n_refls, len(table) assert 'id' in table assert (table['id'] == 0).count(False) == 0 print 'OK'
def test_elliptical_distortion(run_in_tmpdir): """Create distortion maps for elliptical distortion using a dummy experiments with a small detector, for speed. Check those maps seem sensible""" # Make a detector model d = make_detector() # The beam is also essential for a experiments to be serialisable b = Beam((0, 0, 1), 1.0) # Create and write out a experiments imageset = ImageSet(ImageSetData(Reader(None, ["non-existent.cbf"]), None)) imageset.set_detector(d) imageset.set_beam(b) experiments = ExperimentListFactory.from_imageset_and_crystal( imageset, None) experiments.as_json("dummy.expt") # Centre of distortion will be the far corner from the origin of the first # panel centre_xy = d[0].get_image_size_mm() # Generate distortion maps cmd = ("dials.generate_distortion_maps dummy.expt " "mode=ellipse centre_xy={},{} " "phi=0 l1=1.0 l2=0.95").format(*centre_xy) easy_run.fully_buffered(command=cmd).raise_if_errors() # Load the maps with open("dx.pickle", "rb") as f: dx = pickle.load(f) with open("dy.pickle", "rb") as f: dy = pickle.load(f) # Check there are 4 maps each assert len(dx) == len(dy) == 4 # Ellipse has phi=0, so all correction is in the dy map for arr in dx: assert min(arr) == max(arr) == 0.0 # The ellipse correction is centred at the middle of the detector and all in # the Y direction. Therefore we expect a few things from the dy maps: # # (1) Within each panel the columns of the array are identical. # (2) The two upper panels should be the same # (3) The two lower panels should be the same. # (4) One column from an upper panel is a negated, reversed column from a # lower panel. # # All together expect the 4 dy maps to look something like this: # # /-----------\ /-----------\ # |-3 -3 -3 -3| |-3 -3 -3 -3| # |-2 -2 -2 -2| |-2 -2 -2 -2| # |-1 -1 -1 -1| |-1 -1 -1 -1| # | 0 0 0 0| | 0 0 0 0| # \-----------/ \-----------/ # /-----------\ /-----------\ # | 0 0 0 0| | 0 0 0 0| # | 1 1 1 1| | 1 1 1 1| # | 2 2 2 2| | 2 2 2 2| # | 3 3 3 3| | 3 3 3 3| # \-----------/ \-----------/ # So the fundamental data is all in the first column of first panel's map col0 = dy[0].matrix_copy_column(0) # The correction should be 5% of the distance from the ellipse centre to a # corrected pixel (l2 = 0.95 above) along the slow axis. Check that is the # case (for the first pixel at least) vec_centre_to_first_px = matrix.col(d[0].get_pixel_lab_coord( (0.5, 0.5))) - matrix.col(d[0].get_lab_coord(centre_xy)) dist_centre_to_first_px = vec_centre_to_first_px.dot( matrix.col(d[0].get_slow_axis())) corr_mm = dist_centre_to_first_px * 0.05 corr_px = corr_mm / d[0].get_pixel_size()[1] assert col0[0] == pytest.approx(corr_px) # Test (1) from above list for panel 0 for i in range(1, 50): assert (col0 == dy[0].matrix_copy_column(i)).all_eq(True) # Test (2) assert (dy[0] == dy[1]).all_eq(True) # Test (3) assert (dy[2] == dy[3]).all_eq(True) # Test (4) assert col0 == pytest.approx(-1.0 * dy[2].matrix_copy_column(0).reversed()) # Test (1) for panel 2 as well, which then covers everything needed col0 = dy[2].matrix_copy_column(0) for i in range(1, 50): assert (col0 == dy[2].matrix_copy_column(i)).all_eq(True)
def run(self): from dxtbx.imageset import ImageSetData from dxtbx.format.image import ImageTileBool from dxtbx.format.image import ImageTileDouble from dxtbx.format.image import ImageBool from dxtbx.format.image import ImageDouble from scitbx.array_family import flex import os.path from glob import glob from dxtbx.format.FormatCBFMiniPilatus import FormatCBFMiniPilatus as FormatClass from os.path import join dials_regression = libtbx.env.dist_path('dials_regression') filenames = sorted( glob(join(dials_regression, "centroid_test_data", "*.cbf"))) ReaderClass = FormatClass.get_reader() MaskerClass = FormatClass.get_masker() reader = ReaderClass(filenames) masker = MaskerClass(filenames) handle = ImageSetData(reader, masker) data = handle.get_data(0).as_int().tile(0).data() mask = handle.get_mask(0).tile(0).data() assert handle.has_single_file_reader() == False path = handle.get_path(0) assert path == filenames[0] master_path = handle.get_master_path() assert master_path == "" identifier = handle.get_image_identifier(0) assert identifier == filenames[0] beam = FormatClass(filenames[0]).get_beam() detector = FormatClass(filenames[0]).get_detector() goniometer = FormatClass(filenames[0]).get_goniometer() scan = FormatClass(filenames[0]).get_scan() handle.set_beam(beam, 0) handle.set_detector(detector, 0) handle.set_goniometer(goniometer, 0) handle.set_scan(scan, 0) beam2 = handle.get_beam(0) detector2 = handle.get_detector(0) goniometer2 = handle.get_goniometer(0) scan2 = handle.get_scan(0) assert beam2 == beam assert detector2 == detector assert goniometer2 == goniometer assert scan2 == scan mask = flex.bool(flex.grid(10, 10), True) gain = flex.double(flex.grid(10, 10), 1) pedestal = flex.double(flex.grid(10, 10), 2) handle.external_lookup.mask.data = ImageBool(ImageTileBool(mask)) handle.external_lookup.gain.data = ImageDouble(ImageTileDouble(gain)) handle.external_lookup.pedestal.data = ImageDouble( ImageTileDouble(pedestal)) mask2 = handle.external_lookup.mask.data.tile(0).data() gain2 = handle.external_lookup.gain.data.tile(0).data() pedestal2 = handle.external_lookup.pedestal.data.tile(0).data() assert mask2.all_eq(mask) assert gain2.all_eq(gain) assert pedestal2.all_eq(pedestal) print 'OK'
def get_imageset( cls, filenames, beam=None, detector=None, goniometer=None, scan=None, as_sequence=False, as_imageset=False, single_file_indices=None, format_kwargs=None, template=None, check_format=True, lazy=False, ): """ Factory method to create an imageset """ if isinstance(filenames, str): filenames = [filenames] elif len(filenames) > 1: assert len(set(filenames)) == 1 filenames = filenames[0:1] # Make filenames absolute filenames = [os.path.abspath(x) for x in filenames] # Make it a dictionary if format_kwargs is None: format_kwargs = {} # If get_num_images hasn't been implemented, we need indices for number of images if cls.get_num_images == FormatMultiImage.get_num_images: assert single_file_indices is not None assert min(single_file_indices) >= 0 num_images = max(single_file_indices) + 1 else: num_images = None # Get the format instance assert len(filenames) == 1 if check_format is True: format_instance = cls(filenames[0], **format_kwargs) if num_images is None and not lazy: # As we now have the actual format class we can get the number # of images from here. This saves having to create another # format class instance in the Reader() constructor # NOTE: Having this information breaks internal assumptions in # *Lazy classes, so they have to figure this out in # their own time. num_images = format_instance.get_num_images() else: format_instance = None if not as_sequence: lazy = True # Get some information from the format class reader = cls.get_reader()(filenames, num_images=num_images, **format_kwargs) # Read the vendor type if check_format is True: vendor = format_instance.get_vendortype() else: vendor = "" # Get the format kwargs params = format_kwargs # Check if we have a sequence # Make sure only 1 or none is set assert [as_imageset, as_sequence].count(True) < 2 if as_imageset: is_sequence = False elif as_sequence: is_sequence = True else: if scan is None and format_instance is None: raise RuntimeError( """ One of the following needs to be set - as_imageset=True - as_sequence=True - scan - check_format=True """ ) if scan is None: test_scan = format_instance.get_scan() else: test_scan = scan if test_scan is not None: is_sequence = True else: is_sequence = False assert not (as_sequence and lazy), "No lazy support for sequences" if single_file_indices is not None: single_file_indices = flex.size_t(single_file_indices) # Create an imageset or sequence if not is_sequence: # Use imagesetlazy # Setup ImageSetLazy and just return it. No models are set. if lazy: iset = ImageSetLazy( ImageSetData( reader=reader, masker=None, vendor=vendor, params=params, format=cls, ), indices=single_file_indices, ) return iset # Create the imageset iset = ImageSet( ImageSetData( reader=reader, masker=None, vendor=vendor, params=params, format=cls ), indices=single_file_indices, ) # If any are None then read from format if [beam, detector, goniometer, scan].count(None) != 0: # Get list of models beam = [] detector = [] goniometer = [] scan = [] for i in range(format_instance.get_num_images()): beam.append(format_instance.get_beam(i)) detector.append(format_instance.get_detector(i)) goniometer.append(format_instance.get_goniometer(i)) scan.append(format_instance.get_scan(i)) if single_file_indices is None: single_file_indices = list(range(format_instance.get_num_images())) # Set the list of models for i in range(len(single_file_indices)): iset.set_beam(beam[single_file_indices[i]], i) iset.set_detector(detector[single_file_indices[i]], i) iset.set_goniometer(goniometer[single_file_indices[i]], i) iset.set_scan(scan[single_file_indices[i]], i) else: # Get the template template = filenames[0] # Check indices are sequential if single_file_indices is not None: assert all( i + 1 == j for i, j in zip(single_file_indices[:-1], single_file_indices[1:]) ) num_images = len(single_file_indices) else: num_images = format_instance.get_num_images() # Check the scan makes sense - we must want to use <= total images if scan is not None: assert scan.get_num_images() <= num_images # If any are None then read from format if beam is None: beam = format_instance.get_beam() if detector is None: detector = format_instance.get_detector() if goniometer is None: goniometer = format_instance.get_goniometer() if scan is None: scan = format_instance.get_scan() if scan is not None: for f in filenames[1:]: format_instance = cls(f, **format_kwargs) scan += format_instance.get_scan() # Create the masker if format_instance is not None: masker = format_instance.get_masker(goniometer=goniometer) else: masker = None isetdata = ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=cls, template=template, ) # Create the sequence iset = ImageSequence( isetdata, beam=beam, detector=detector, goniometer=goniometer, scan=scan, indices=single_file_indices, ) if format_instance is not None: static_mask = format_instance.get_static_mask() if static_mask is not None: if not iset.external_lookup.mask.data.empty(): for m1, m2 in zip(static_mask, iset.external_lookup.mask.data): m1 &= m2.data() iset.external_lookup.mask.data = ImageBool(static_mask) else: iset.external_lookup.mask.data = ImageBool(static_mask) return iset