def format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address, round_to_int=True):
"""
Given a preloaded dxtbx format object and raw data, assemble the tiles
and set the distance.
@param base_dxtbx A header only dxtbx format object
@param data 32x185x388 CSPAD byte array from XTC stream
@param distance Detector distance (mm)
@param wavelength Shot wavelength (angstroms)
@param timestamp Human readable timestamp
@param address Detector address, put in CBF header
"""
from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory
import copy
import numpy as np
cbf = copy_cbf_header(base_dxtbx._cbf_handle)
cspad_img = FormatCBFCspadInMemory(cbf)
cbf.set_datablockname(address + "_" + timestamp)
if round_to_int:
data = flex.double(data.astype(np.float64)).iround()
else:
data = flex.double(data.astype(np.float64))
data.reshape(flex.grid((4,8,185,388)))
n_asics = data.focus()[0] * data.focus()[1]
add_frame_specific_cbf_tables(cbf, wavelength,timestamp,
[(cspad_min_trusted_value,cspad_saturated_value)]*n_asics)
# Set the distance, I.E., the length translated along the Z axis
cbf.find_category("diffrn_scan_frame_axis")
cbf.find_column("axis_id")
cbf.find_row("AXIS_D0_Z") # XXX discover the Z axis somehow, don't use D0 here
cbf.find_column("displacement")
cbf.set_value(str(-distance))
# Explicitly reset the detector object now that the distance is set correctly
cspad_img._detector_instance = cspad_img._detector()
# Explicitly set up the beam object now that the tables are all loaded correctly
cspad_img._beam_instance = cspad_img._beam()
# Get the data and add it to the cbf handle. Split it out by quads.
tiles = {}
for i in xrange(4):
tiles[(0,i)] = data[i:i+1,:,:,:]
tiles[(0,i)].reshape(flex.grid((8,185,388)))
add_tiles_to_cbf(cbf,tiles)
return cspad_img
def test_cspad_cbf_in_memory(dials_regression, run_in_tmpdir):
# Check the data files for this test exist
image_path = os.path.join(
dials_regression,
"image_examples",
"LCLS_cspad_nexus",
"idx-20130301060858801.cbf",
)
assert os.path.isfile(image_path)
with open("process_lcls.phil", "w") as f:
f.write(
"""
dispatch.squash_errors = False
spotfinder {
filter.min_spot_size=2
threshold.dispersion.gain=25
threshold.dispersion.global_threshold=100
}
indexing {
known_symmetry {
space_group = P6122
unit_cell = 92.9 92.9 130.4 90 90 120
}
refinement_protocol.d_min_start=1.7
stills.refine_candidates_with_known_symmetry=True
}
"""
)
params = phil_scope.fetch(parse(file_name="process_lcls.phil")).extract()
params.output.experiments_filename = None
processor = Processor(params)
mem_img = dxtbx.load(image_path)
raw_data = mem_img.get_raw_data() # cache the raw data to prevent swig errors
mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle)
mem_img._raw_data = raw_data
mem_img._cbf_handle = None # drop the file handle to prevent swig errors
imgset = ImageSet(ImageSetData(MemReader([mem_img]), None))
imgset.set_beam(mem_img.get_beam())
imgset.set_detector(mem_img.get_detector())
experiments = ExperimentListFactory.from_imageset_and_crystal(imgset, None)
processor.process_experiments(
"20130301060858801", experiments
) # index/integrate the image
result = "idx-20130301060858801_integrated.refl"
n_refls = list(
range(140, 152)
) # large ranges to handle platform-specific differences
table = flex.reflection_table.from_file(result)
assert len(table) in n_refls, len(table)
assert "id" in table
assert (table["id"] == 0).count(False) == 0
def format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address, round_to_int=True):
"""
Given a preloaded dxtbx format object and raw data, assemble the tiles
and set the distance.
@param base_dxtbx A header only dxtbx format object
@param data 32x185x388 CSPAD byte array from XTC stream
@param distance Detector distance (mm)
@param wavelength Shot wavelength (angstroms)
@param timestamp Human readable timestamp
@param address Detector address, put in CBF header
"""
from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory
import copy
import numpy as np
cbf = copy_cbf_header(base_dxtbx._cbf_handle)
cspad_img = FormatCBFCspadInMemory(cbf)
cbf.set_datablockname(address + "_" + timestamp)
if round_to_int:
data = flex.double(data.astype(np.float64)).iround()
else:
data = flex.double(data.astype(np.float64))
data.reshape(flex.grid((4,8,185,388)))
n_asics = data.focus()[0] * data.focus()[1]
add_frame_specific_cbf_tables(cbf, wavelength,timestamp,
[(cspad_min_trusted_value,cspad_saturated_value)]*n_asics)
# Set the distance, I.E., the length translated along the Z axis
cbf.find_category("diffrn_scan_frame_axis")
cbf.find_column("axis_id")
cbf.find_row("AXIS_D0_Z") # XXX discover the Z axis somehow, don't use D0 here
cbf.find_column("displacement")
cbf.set_value(str(-distance))
# Explicitly reset the detector object now that the distance is set correctly
cspad_img._detector_instance = cspad_img._detector()
# Explicitly set up the beam object now that the tables are all loaded correctly
cspad_img._beam_instance = cspad_img._beam()
# Get the data and add it to the cbf handle. Split it out by quads.
tiles = {}
for i in xrange(4):
tiles[(0,i)] = data[i:i+1,:,:,:]
tiles[(0,i)].reshape(flex.grid((8,185,388)))
add_tiles_to_cbf(cbf,tiles)
return cspad_img
def env_dxtbx_from_slac_metrology(run, address, metro=None):
""" Loads a dxtbx cspad cbf header only object from the metrology path stored
in a psana run object's calibration store
@param env psana run object
@param address address string for a detector
"""
if metro is not None:
# FIXME: get metrology from a pickle file until det interface is ready
cbf = get_cspad_cbf_handle(None, metro, 'cbf', None, "test", None, 100, verbose = True, header_only = True)
from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory
return FormatCBFCspadInMemory(cbf)
from psana import Detector
try:
# try to load the geometry from the detector interface
psana_det = Detector(address, run.env())
geometry = psana_det.pyda.geoaccess(run)
except Exception, e:
geometry = None
def env_dxtbx_from_slac_metrology(run, address):
""" Loads a dxtbx cspad cbf header only object from the metrology path stored
in a psana run object's calibration store
@param env psana run object
@param address address string for a detector
"""
from xfel.command_line.xtc_process import PSANA2_VERSION
if PSANA2_VERSION:
det = run.ds.Detector(address)
geometry = det.geometry(run)
else:
from psana import Detector
try:
# try to load the geometry from the detector interface
psana_det = Detector(address, run.env())
geometry = psana_det.pyda.geoaccess(run.run())
except Exception as e:
geometry = None
if geometry is None:
metro_path = get_calib_file_path(run.env(), address, run)
elif geometry.valid:
metro_path = None
else:
from libtbx.utils import Sorry
import socket, os
raise Sorry("Could not read geometry, hostname: %s"%socket.gethostname())
if metro_path is None and geometry is None:
return None
metro = read_slac_metrology(metro_path, geometry)
cbf = get_cspad_cbf_handle(None, metro, 'cbf', None, "test", None, 100, verbose = True, header_only = True)
from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory
return FormatCBFCspadInMemory(cbf)
def test_cspad_cbf_in_memory(dials_regression, run_in_tmpdir,
composite_output):
# Check the data files for this test exist
image_path = os.path.join(
dials_regression,
"image_examples",
"LCLS_cspad_nexus",
"idx-20130301060858801.cbf",
)
assert os.path.isfile(image_path)
with open("process_lcls.phil", "w") as f:
f.write(cspad_cbf_in_memory_phil)
params = phil_scope.fetch(parse(file_name="process_lcls.phil")).extract()
params.output.experiments_filename = None
params.output.composite_output = composite_output
if composite_output:
processor = Processor(params, composite_tag="memtest")
else:
processor = Processor(params)
mem_img = dxtbx.load(image_path)
raw_data = mem_img.get_raw_data(
) # cache the raw data to prevent swig errors
mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle)
mem_img._raw_data = raw_data
mem_img._cbf_handle = None # drop the file handle to prevent swig errors
imgset = ImageSet(ImageSetData(MemReader([mem_img]), None))
imgset.set_beam(mem_img.get_beam())
imgset.set_detector(mem_img.get_detector())
experiments = ExperimentListFactory.from_imageset_and_crystal(imgset, None)
processor.process_experiments("20130301060858801",
experiments) # index/integrate the image
if composite_output:
processor.finalize()
result = "idx-memtest_integrated.refl"
else:
result = "idx-20130301060858801_integrated.refl"
n_refls = list(range(
140, 152)) # large ranges to handle platform-specific differences
table = flex.reflection_table.from_file(result)
assert len(table) in n_refls, len(table)
assert "id" in table
assert (table["id"] == 0).count(False) == 0
if params.resolution is None:
print "Generating annular gain mask between %f and %f angstroms, assuming a distance %s mm and wavelength %s angstroms" % \
(params.annulus_inner, params.annulus_outer, params.distance, params.wavelength)
else:
print "Generating annular gain mask between %f and %f angstroms, assuming a distance %s mm and wavelength %s angstroms. Also, pixels higher than %f angstroms will be set to low gain." % \
(params.annulus_inner, params.annulus_outer, params.distance, params.wavelength, params.resolution)
elif params.resolution is not None:
print "Generating circular gain mask %s angstroms, assuming a distance %s mm and wavelength %s angstroms" % \
(params.resolution, params.distance, params.wavelength)
from xfel.cftbx.detector.cspad_cbf_tbx import read_slac_metrology, get_cspad_cbf_handle
from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory
from dxtbx.model import BeamFactory
metro = read_slac_metrology(path = params.optical_metrology_path)
cbf = get_cspad_cbf_handle(None, metro, 'cbf', None, "test", None, params.distance, verbose = True, header_only = True)
img = FormatCBFCspadInMemory(cbf)
beam = BeamFactory.simple(params.wavelength).get_s0()
beam_center = (0,0)
data = numpy.zeros((11840,194))
if annulus:
inner = params.distance * math.tan(2*math.sinh(params.wavelength/(2*params.annulus_inner)))
outer = params.distance * math.tan(2*math.sinh(params.wavelength/(2*params.annulus_outer)))
print "Inner (mm):", inner
print "Outer (mm):", outer
if params.resolution is not None:
radius = params.distance * math.tan(2*math.sinh(params.wavelength/(2*params.resolution)))
print "Radius (mm):", radius
print "Panel:",; sys.stdout.flush()
elif geometry.valid:
metro_path = None
else:
from libtbx.utils import Sorry
import socket, os
raise Sorry("Could not read geometry, hostname: %s"%socket.gethostname())
if metro_path is None and geometry is None:
return None
metro = read_slac_metrology(metro_path, geometry)
cbf = get_cspad_cbf_handle(None, metro, 'cbf', None, "test", None, 100, verbose = True, header_only = True)
from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory
return FormatCBFCspadInMemory(cbf)
def format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address, round_to_int=True):
"""
Given a preloaded dxtbx format object and raw data, assemble the tiles
and set the distance.
@param base_dxtbx A header only dxtbx format object
@param data 32x185x388 CSPAD byte array from XTC stream
@param distance Detector distance (mm)
@param wavelength Shot wavelength (angstroms)
@param timestamp Human readable timestamp
@param address Detector address, put in CBF header
"""
from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory
import copy
import numpy as np
if params.resolution is None:
print "Generating annular gain mask between %f and %f angstroms, assuming a distance %s mm and wavelength %s angstroms" % \
(params.annulus_inner, params.annulus_outer, params.distance, params.wavelength)
else:
print "Generating annular gain mask between %f and %f angstroms, assuming a distance %s mm and wavelength %s angstroms. Also, pixels higher than %f angstroms will be set to low gain." % \
(params.annulus_inner, params.annulus_outer, params.distance, params.wavelength, params.resolution)
elif params.resolution is not None:
print "Generating circular gain mask %s angstroms, assuming a distance %s mm and wavelength %s angstroms" % \
(params.resolution, params.distance, params.wavelength)
from xfel.cftbx.detector.cspad_cbf_tbx import read_slac_metrology, get_cspad_cbf_handle
from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory
from dxtbx.model.beam import beam_factory
metro = read_slac_metrology(path = params.optical_metrology_path)
cbf = get_cspad_cbf_handle(None, metro, 'cbf', None, "test", None, params.distance, verbose = True, header_only = True)
img = FormatCBFCspadInMemory(cbf)
beam = beam_factory.simple(params.wavelength).get_s0()
beam_center = (0,0)
data = numpy.zeros((11840,194))
if annulus:
inner = params.distance * math.tan(2*math.sinh(params.wavelength/(2*params.annulus_inner)))
outer = params.distance * math.tan(2*math.sinh(params.wavelength/(2*params.annulus_outer)))
print "Inner (mm):", inner
print "Outer (mm):", outer
if params.resolution is not None:
radius = params.distance * math.tan(2*math.sinh(params.wavelength/(2*params.resolution)))
print "Radius (mm):", radius
print "Panel:",; sys.stdout.flush()
def test_cspad_cbf_in_memory(self):
from os.path import join, exists
import os, dxtbx
from uuid import uuid4
from dials.command_line.stills_process import phil_scope, Processor
from libtbx.phil import parse
from dxtbx.imageset import ImageSet, ImageSetData, MemReader, MemMasker
from dxtbx.datablock import DataBlockFactory
from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory
import cPickle as pickle
dirname = 'tmp_%s' % uuid4().hex
os.mkdir(dirname)
os.chdir(dirname)
assert exists(join(self.lcls_path, 'idx-20130301060858801.cbf'))
f = open("process_lcls.phil", 'w')
f.write("""
dispatch.squash_errors = False
spotfinder {
filter.min_spot_size=2
threshold.dispersion.gain=25
threshold.dispersion.global_threshold=100
}
indexing {
known_symmetry {
space_group = P6122
unit_cell = 92.9 92.9 130.4 90 90 120
}
refinement_protocol.d_min_start=1.7
stills.refine_candidates_with_known_symmetry=True
}
""")
f.close()
params = phil_scope.fetch(
parse(file_name="process_lcls.phil")).extract()
params.output.datablock_filename = None
processor = Processor(params)
mem_img = dxtbx.load(join(self.lcls_path, 'idx-20130301060858801.cbf'))
raw_data = mem_img.get_raw_data(
) # cache the raw data to prevent swig errors
mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle)
mem_img._raw_data = raw_data
mem_img._cbf_handle = None # drop the file handle to prevent swig errors
imgset = ImageSet(
ImageSetData(MemReader([mem_img]), MemMasker([mem_img])))
imgset.set_beam(mem_img.get_beam())
imgset.set_detector(mem_img.get_detector())
datablock = DataBlockFactory.from_imageset(imgset)[0]
processor.process_datablock("20130301060858801",
datablock) # index/integrate the image
result = "idx-20130301060858801_integrated.pickle"
#n_refls = range(140,152) # large ranges to handle platform-specific differences
# 09/20/17 Changes to still indexer: refine candidate basis vectors in target symmetry if supplied
#n_refls = range(128,140) # large ranges to handle platform-specific differences
# 09/27/17 Bugfix for refine_candidates_with_known_symmetry
n_refls = range(
140, 152) # large ranges to handle platform-specific differences
table = pickle.load(open(result, 'rb'))
assert len(table) in n_refls, len(table)
assert 'id' in table
assert (table['id'] == 0).count(False) == 0
print 'OK'
