def format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address, round_to_int=True): """ Given a preloaded dxtbx format object and raw data, assemble the tiles and set the distance. @param base_dxtbx A header only dxtbx format object @param data 32x185x388 CSPAD byte array from XTC stream @param distance Detector distance (mm) @param wavelength Shot wavelength (angstroms) @param timestamp Human readable timestamp @param address Detector address, put in CBF header """ from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory import copy import numpy as np cbf = copy_cbf_header(base_dxtbx._cbf_handle) cspad_img = FormatCBFCspadInMemory(cbf) cbf.set_datablockname(address + "_" + timestamp) if round_to_int: data = flex.double(data.astype(np.float64)).iround() else: data = flex.double(data.astype(np.float64)) data.reshape(flex.grid((4,8,185,388))) n_asics = data.focus()[0] * data.focus()[1] add_frame_specific_cbf_tables(cbf, wavelength,timestamp, [(cspad_min_trusted_value,cspad_saturated_value)]*n_asics) # Set the distance, I.E., the length translated along the Z axis cbf.find_category("diffrn_scan_frame_axis") cbf.find_column("axis_id") cbf.find_row("AXIS_D0_Z") # XXX discover the Z axis somehow, don't use D0 here cbf.find_column("displacement") cbf.set_value(str(-distance)) # Explicitly reset the detector object now that the distance is set correctly cspad_img._detector_instance = cspad_img._detector() # Explicitly set up the beam object now that the tables are all loaded correctly cspad_img._beam_instance = cspad_img._beam() # Get the data and add it to the cbf handle. Split it out by quads. tiles = {} for i in xrange(4): tiles[(0,i)] = data[i:i+1,:,:,:] tiles[(0,i)].reshape(flex.grid((8,185,388))) add_tiles_to_cbf(cbf,tiles) return cspad_img
def test_cspad_cbf_in_memory(dials_regression, run_in_tmpdir): # Check the data files for this test exist image_path = os.path.join( dials_regression, "image_examples", "LCLS_cspad_nexus", "idx-20130301060858801.cbf", ) assert os.path.isfile(image_path) with open("process_lcls.phil", "w") as f: f.write( """ dispatch.squash_errors = False spotfinder { filter.min_spot_size=2 threshold.dispersion.gain=25 threshold.dispersion.global_threshold=100 } indexing { known_symmetry { space_group = P6122 unit_cell = 92.9 92.9 130.4 90 90 120 } refinement_protocol.d_min_start=1.7 stills.refine_candidates_with_known_symmetry=True } """ ) params = phil_scope.fetch(parse(file_name="process_lcls.phil")).extract() params.output.experiments_filename = None processor = Processor(params) mem_img = dxtbx.load(image_path) raw_data = mem_img.get_raw_data() # cache the raw data to prevent swig errors mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle) mem_img._raw_data = raw_data mem_img._cbf_handle = None # drop the file handle to prevent swig errors imgset = ImageSet(ImageSetData(MemReader([mem_img]), None)) imgset.set_beam(mem_img.get_beam()) imgset.set_detector(mem_img.get_detector()) experiments = ExperimentListFactory.from_imageset_and_crystal(imgset, None) processor.process_experiments( "20130301060858801", experiments ) # index/integrate the image result = "idx-20130301060858801_integrated.refl" n_refls = list( range(140, 152) ) # large ranges to handle platform-specific differences table = flex.reflection_table.from_file(result) assert len(table) in n_refls, len(table) assert "id" in table assert (table["id"] == 0).count(False) == 0
def format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address, round_to_int=True): """ Given a preloaded dxtbx format object and raw data, assemble the tiles and set the distance. @param base_dxtbx A header only dxtbx format object @param data 32x185x388 CSPAD byte array from XTC stream @param distance Detector distance (mm) @param wavelength Shot wavelength (angstroms) @param timestamp Human readable timestamp @param address Detector address, put in CBF header """ from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory import copy import numpy as np cbf = copy_cbf_header(base_dxtbx._cbf_handle) cspad_img = FormatCBFCspadInMemory(cbf) cbf.set_datablockname(address + "_" + timestamp) if round_to_int: data = flex.double(data.astype(np.float64)).iround() else: data = flex.double(data.astype(np.float64)) data.reshape(flex.grid((4,8,185,388))) n_asics = data.focus()[0] * data.focus()[1] add_frame_specific_cbf_tables(cbf, wavelength,timestamp, [(cspad_min_trusted_value,cspad_saturated_value)]*n_asics) # Set the distance, I.E., the length translated along the Z axis cbf.find_category("diffrn_scan_frame_axis") cbf.find_column("axis_id") cbf.find_row("AXIS_D0_Z") # XXX discover the Z axis somehow, don't use D0 here cbf.find_column("displacement") cbf.set_value(str(-distance)) # Explicitly reset the detector object now that the distance is set correctly cspad_img._detector_instance = cspad_img._detector() # Explicitly set up the beam object now that the tables are all loaded correctly cspad_img._beam_instance = cspad_img._beam() # Get the data and add it to the cbf handle. Split it out by quads. tiles = {} for i in xrange(4): tiles[(0,i)] = data[i:i+1,:,:,:] tiles[(0,i)].reshape(flex.grid((8,185,388))) add_tiles_to_cbf(cbf,tiles) return cspad_img
def env_dxtbx_from_slac_metrology(run, address, metro=None): """ Loads a dxtbx cspad cbf header only object from the metrology path stored in a psana run object's calibration store @param env psana run object @param address address string for a detector """ if metro is not None: # FIXME: get metrology from a pickle file until det interface is ready cbf = get_cspad_cbf_handle(None, metro, 'cbf', None, "test", None, 100, verbose = True, header_only = True) from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory return FormatCBFCspadInMemory(cbf) from psana import Detector try: # try to load the geometry from the detector interface psana_det = Detector(address, run.env()) geometry = psana_det.pyda.geoaccess(run) except Exception, e: geometry = None
def env_dxtbx_from_slac_metrology(run, address): """ Loads a dxtbx cspad cbf header only object from the metrology path stored in a psana run object's calibration store @param env psana run object @param address address string for a detector """ from xfel.command_line.xtc_process import PSANA2_VERSION if PSANA2_VERSION: det = run.ds.Detector(address) geometry = det.geometry(run) else: from psana import Detector try: # try to load the geometry from the detector interface psana_det = Detector(address, run.env()) geometry = psana_det.pyda.geoaccess(run.run()) except Exception as e: geometry = None if geometry is None: metro_path = get_calib_file_path(run.env(), address, run) elif geometry.valid: metro_path = None else: from libtbx.utils import Sorry import socket, os raise Sorry("Could not read geometry, hostname: %s"%socket.gethostname()) if metro_path is None and geometry is None: return None metro = read_slac_metrology(metro_path, geometry) cbf = get_cspad_cbf_handle(None, metro, 'cbf', None, "test", None, 100, verbose = True, header_only = True) from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory return FormatCBFCspadInMemory(cbf)
def test_cspad_cbf_in_memory(dials_regression, run_in_tmpdir, composite_output): # Check the data files for this test exist image_path = os.path.join( dials_regression, "image_examples", "LCLS_cspad_nexus", "idx-20130301060858801.cbf", ) assert os.path.isfile(image_path) with open("process_lcls.phil", "w") as f: f.write(cspad_cbf_in_memory_phil) params = phil_scope.fetch(parse(file_name="process_lcls.phil")).extract() params.output.experiments_filename = None params.output.composite_output = composite_output if composite_output: processor = Processor(params, composite_tag="memtest") else: processor = Processor(params) mem_img = dxtbx.load(image_path) raw_data = mem_img.get_raw_data( ) # cache the raw data to prevent swig errors mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle) mem_img._raw_data = raw_data mem_img._cbf_handle = None # drop the file handle to prevent swig errors imgset = ImageSet(ImageSetData(MemReader([mem_img]), None)) imgset.set_beam(mem_img.get_beam()) imgset.set_detector(mem_img.get_detector()) experiments = ExperimentListFactory.from_imageset_and_crystal(imgset, None) processor.process_experiments("20130301060858801", experiments) # index/integrate the image if composite_output: processor.finalize() result = "idx-memtest_integrated.refl" else: result = "idx-20130301060858801_integrated.refl" n_refls = list(range( 140, 152)) # large ranges to handle platform-specific differences table = flex.reflection_table.from_file(result) assert len(table) in n_refls, len(table) assert "id" in table assert (table["id"] == 0).count(False) == 0
if params.resolution is None: print "Generating annular gain mask between %f and %f angstroms, assuming a distance %s mm and wavelength %s angstroms" % \ (params.annulus_inner, params.annulus_outer, params.distance, params.wavelength) else: print "Generating annular gain mask between %f and %f angstroms, assuming a distance %s mm and wavelength %s angstroms. Also, pixels higher than %f angstroms will be set to low gain." % \ (params.annulus_inner, params.annulus_outer, params.distance, params.wavelength, params.resolution) elif params.resolution is not None: print "Generating circular gain mask %s angstroms, assuming a distance %s mm and wavelength %s angstroms" % \ (params.resolution, params.distance, params.wavelength) from xfel.cftbx.detector.cspad_cbf_tbx import read_slac_metrology, get_cspad_cbf_handle from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory from dxtbx.model import BeamFactory metro = read_slac_metrology(path = params.optical_metrology_path) cbf = get_cspad_cbf_handle(None, metro, 'cbf', None, "test", None, params.distance, verbose = True, header_only = True) img = FormatCBFCspadInMemory(cbf) beam = BeamFactory.simple(params.wavelength).get_s0() beam_center = (0,0) data = numpy.zeros((11840,194)) if annulus: inner = params.distance * math.tan(2*math.sinh(params.wavelength/(2*params.annulus_inner))) outer = params.distance * math.tan(2*math.sinh(params.wavelength/(2*params.annulus_outer))) print "Inner (mm):", inner print "Outer (mm):", outer if params.resolution is not None: radius = params.distance * math.tan(2*math.sinh(params.wavelength/(2*params.resolution))) print "Radius (mm):", radius print "Panel:",; sys.stdout.flush()
elif geometry.valid: metro_path = None else: from libtbx.utils import Sorry import socket, os raise Sorry("Could not read geometry, hostname: %s"%socket.gethostname()) if metro_path is None and geometry is None: return None metro = read_slac_metrology(metro_path, geometry) cbf = get_cspad_cbf_handle(None, metro, 'cbf', None, "test", None, 100, verbose = True, header_only = True) from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory return FormatCBFCspadInMemory(cbf) def format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address, round_to_int=True): """ Given a preloaded dxtbx format object and raw data, assemble the tiles and set the distance. @param base_dxtbx A header only dxtbx format object @param data 32x185x388 CSPAD byte array from XTC stream @param distance Detector distance (mm) @param wavelength Shot wavelength (angstroms) @param timestamp Human readable timestamp @param address Detector address, put in CBF header """ from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory import copy import numpy as np
if params.resolution is None: print "Generating annular gain mask between %f and %f angstroms, assuming a distance %s mm and wavelength %s angstroms" % \ (params.annulus_inner, params.annulus_outer, params.distance, params.wavelength) else: print "Generating annular gain mask between %f and %f angstroms, assuming a distance %s mm and wavelength %s angstroms. Also, pixels higher than %f angstroms will be set to low gain." % \ (params.annulus_inner, params.annulus_outer, params.distance, params.wavelength, params.resolution) elif params.resolution is not None: print "Generating circular gain mask %s angstroms, assuming a distance %s mm and wavelength %s angstroms" % \ (params.resolution, params.distance, params.wavelength) from xfel.cftbx.detector.cspad_cbf_tbx import read_slac_metrology, get_cspad_cbf_handle from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory from dxtbx.model.beam import beam_factory metro = read_slac_metrology(path = params.optical_metrology_path) cbf = get_cspad_cbf_handle(None, metro, 'cbf', None, "test", None, params.distance, verbose = True, header_only = True) img = FormatCBFCspadInMemory(cbf) beam = beam_factory.simple(params.wavelength).get_s0() beam_center = (0,0) data = numpy.zeros((11840,194)) if annulus: inner = params.distance * math.tan(2*math.sinh(params.wavelength/(2*params.annulus_inner))) outer = params.distance * math.tan(2*math.sinh(params.wavelength/(2*params.annulus_outer))) print "Inner (mm):", inner print "Outer (mm):", outer if params.resolution is not None: radius = params.distance * math.tan(2*math.sinh(params.wavelength/(2*params.resolution))) print "Radius (mm):", radius print "Panel:",; sys.stdout.flush()
def test_cspad_cbf_in_memory(self): from os.path import join, exists import os, dxtbx from uuid import uuid4 from dials.command_line.stills_process import phil_scope, Processor from libtbx.phil import parse from dxtbx.imageset import ImageSet, ImageSetData, MemReader, MemMasker from dxtbx.datablock import DataBlockFactory from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory import cPickle as pickle dirname = 'tmp_%s' % uuid4().hex os.mkdir(dirname) os.chdir(dirname) assert exists(join(self.lcls_path, 'idx-20130301060858801.cbf')) f = open("process_lcls.phil", 'w') f.write(""" dispatch.squash_errors = False spotfinder { filter.min_spot_size=2 threshold.dispersion.gain=25 threshold.dispersion.global_threshold=100 } indexing { known_symmetry { space_group = P6122 unit_cell = 92.9 92.9 130.4 90 90 120 } refinement_protocol.d_min_start=1.7 stills.refine_candidates_with_known_symmetry=True } """) f.close() params = phil_scope.fetch( parse(file_name="process_lcls.phil")).extract() params.output.datablock_filename = None processor = Processor(params) mem_img = dxtbx.load(join(self.lcls_path, 'idx-20130301060858801.cbf')) raw_data = mem_img.get_raw_data( ) # cache the raw data to prevent swig errors mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle) mem_img._raw_data = raw_data mem_img._cbf_handle = None # drop the file handle to prevent swig errors imgset = ImageSet( ImageSetData(MemReader([mem_img]), MemMasker([mem_img]))) imgset.set_beam(mem_img.get_beam()) imgset.set_detector(mem_img.get_detector()) datablock = DataBlockFactory.from_imageset(imgset)[0] processor.process_datablock("20130301060858801", datablock) # index/integrate the image result = "idx-20130301060858801_integrated.pickle" #n_refls = range(140,152) # large ranges to handle platform-specific differences # 09/20/17 Changes to still indexer: refine candidate basis vectors in target symmetry if supplied #n_refls = range(128,140) # large ranges to handle platform-specific differences # 09/27/17 Bugfix for refine_candidates_with_known_symmetry n_refls = range( 140, 152) # large ranges to handle platform-specific differences table = pickle.load(open(result, 'rb')) assert len(table) in n_refls, len(table) assert 'id' in table assert (table['id'] == 0).count(False) == 0 print 'OK'