class FormatCBFMiniPilatus(FormatCBFMini):
'''A class for reading mini CBF format Pilatus images, and correctly
constructing a model for the experiment from this.'''
@staticmethod
def understand(image_file):
'''Check to see if this looks like an Pilatus mini CBF format image,
i.e. we can make sense of it.'''
if 'ENABLE_PHOTON_FACTORY_TWO_EIGER' in os.environ:
return False
header = FormatCBFMini.get_cbf_header(image_file)
for record in header.split('\n'):
if '# Detector' in record and \
'EIGER' in record:
return False
for record in header.split('\n'):
if '_array_data.header_convention' in record and \
'PILATUS' in record:
return True
if '_array_data.header_convention' in record and \
'SLS' in record:
return True
if '_array_data.header_convention' in record and \
'?' in record:
return True
if '# Detector' in record and \
'PILATUS' in record: #CBFlib v0.8.0 allowed
return True
return False
def __init__(self, image_file):
'''Initialise the image structure from the given file, including a
proper model of the experiment.'''
assert(self.understand(image_file))
FormatCBFMini.__init__(self, image_file)
return
def _start(self):
FormatCBFMini._start(self)
try:
from iotbx.detectors.pilatus_minicbf import PilatusImage
self.detectorbase = PilatusImage(self._image_file)
self.detectorbase.readHeader()
except KeyError, e:
pass
def pilatus_slice_from_file_url(url):
from six.moves import urllib
parsed = urllib.parse.urlparse(url)
assert parsed.scheme == "file"
file = parsed.path.split("?")[0]
if file == parsed.path:
return PilatusImage(file)
qs = urllib.parse.parse_qs(parsed.path.split("?")[1])
sliceindex = int(qs["slice"][0])
object = PilatusImage(file)
object.readHeader()
return pilatus_slice_from_object_and_slicenumber(object, sliceindex)
def pilatus_slice_from_file_url(url):
#backward compatibility with Python 2.5
try: from urlparse import parse_qs
except Exception: from cgi import parse_qs
from urlparse import urlparse
parsed = urlparse(url)
assert parsed.scheme == "file"
file = parsed.path.split("?")[0]
if file == parsed.path:
return PilatusImage(file)
qs = parse_qs(parsed.path.split("?")[1])
sliceindex = int(qs["slice"][0])
object = PilatusImage(file)
object.readHeader()
return pilatus_slice_from_object_and_slicenumber(object,sliceindex)
class FormatCBFMiniEigerPhotonFactory(FormatCBFMini):
'''A class for reading mini CBF format Eiger images, and correctly
constructing a model for the experiment from this.
Specific for 2-Eiger setup at Photon Factory based on images sent by
Yusuke Yamada. Geometry currently hard-coded as no geometry information in
image headers. Assumes image filenames contain the string '_upper_' or
'_lower_' to distinguish the two detectors.
Only works if environment variable ENABLE_PHOTON_FACTORY_TWO_EIGER is set.
'''
@staticmethod
def understand(image_file):
# no way to uniquely identify given example images
if 'ENABLE_PHOTON_FACTORY_TWO_EIGER' in os.environ:
return True
return False
def __init__(self, image_file):
'''Initialise the image structure from the given file, including a
proper model of the experiment.'''
assert(self.understand(image_file))
FormatCBFMini.__init__(self, image_file)
return
def _start(self):
FormatCBFMini._start(self)
try:
from iotbx.detectors.pilatus_minicbf import PilatusImage
self.detectorbase = PilatusImage(self._image_file)
self.detectorbase.readHeader()
except KeyError, e:
pass
class FormatCBFMiniEiger(FormatCBFMini):
'''A class for reading mini CBF format Eiger images, and correctly
constructing a model for the experiment from this.'''
@staticmethod
def understand(image_file):
'''Check to see if this looks like an Eiger mini CBF format image,
i.e. we can make sense of it.'''
header = FormatCBFMini.get_cbf_header(image_file)
for record in header.split('\n'):
if '# Detector' in record and \
'EIGER' in record:
return True
return False
def __init__(self, image_file):
'''Initialise the image structure from the given file, including a
proper model of the experiment.'''
assert(self.understand(image_file))
FormatCBFMini.__init__(self, image_file)
return
def _start(self):
FormatCBFMini._start(self)
try:
from iotbx.detectors.pilatus_minicbf import PilatusImage
self.detectorbase = PilatusImage(self._image_file)
self.detectorbase.readHeader()
except KeyError, e:
pass
class FormatCBFMiniEigerPhotonFactory(FormatCBFMini):
'''A class for reading mini CBF format Eiger images, and correctly
constructing a model for the experiment from this.
Specific for 2-Eiger setup at Photon Factory based on images sent by
Yusuke Yamada. Geometry currently hard-coded as no geometry information in
image headers. Assumes image filenames contain the string '_upper_' or
'_lower_' to distinguish the two detectors.
Only works if environment variable ENABLE_PHOTON_FACTORY_TWO_EIGER is set.
'''
@staticmethod
def understand(image_file):
# no way to uniquely identify given example images
if 'ENABLE_PHOTON_FACTORY_TWO_EIGER' in os.environ:
return True
return False
def __init__(self, image_file):
'''Initialise the image structure from the given file, including a
proper model of the experiment.'''
assert(self.understand(image_file))
FormatCBFMini.__init__(self, image_file)
return
def _start(self):
FormatCBFMini._start(self)
def _goniometer(self):
return self._goniometer_factory.make_goniometer(
(1,0,0),(1,0,0,0,1,0,0,0,1))
def _detector(self):
if '_lower_' in self._image_file:
#Detector:
#Panel:
#pixel_size:{0.075,0.075}
#image_size: {2070,2167}
#trusted_range: {0,1e+07}
#thickness: 0
#material:
#mu: 0
#fast_axis: {0.999997,-0.00228845,0.00127629}
#slow_axis: {-0.0026197,-0.862841,0.505469}
#origin: {-79.3767,2.47119,-169.509}
detector = self._detector_factory.complex(
'PAD',
origin=(-79.3767,2.47119,-169.509),
fast=(0.999997,-0.00228845,0.00127629),
slow=(-0.0026197,-0.862841,0.505469),
pixel=(0.075,0.075),
size=(2070,2167),
trusted_range=(-1,1e7), # XXX FIXME
)
elif '_upper_' in self._image_file:
#Detector:
#Panel:
#pixel_size:{0.075,0.075}
#image_size: {2070,2167}
#trusted_range: {0,1e+07}
#thickness: 0
#material:
#mu: 0
#fast_axis: {-0.999993,0.00338011,-0.00156153}
#slow_axis: {0.00213163,0.863573,0.504219}
#origin: {75.1912,14.1117,-124.34}
detector = self._detector_factory.complex(
'PAD',
origin=(75.1912,14.1117,-124.34),
fast=(-0.999993,0.00338011,-0.00156153),
slow=(0.00213163,0.863573,0.504219),
pixel=(0.075,0.075),
size=(2070,2167),
trusted_range=(-1,1e7), # XXX FIXME
)
else:
raise RuntimeError("Don't understand image")
return detector
def _beam(self):
#Beam:
#wavelength: 1.1
#sample to source direction : {-0.00573979,-0,0.999984}
#divergence: 0
#sigma divergence: 0
#polarization normal: {0,1,0}
#polarization fraction: 0.999
return self._beam_factory.complex(
sample_to_source=(-0.00573979,-0,0.999984),
polarization_fraction=0.999,
polarization_plane_normal=(0,1,0),
wavelength=1.1)
def _scan(self):
format = self._scan_factory.format('CBF')
exposure_time = 1 #XXX
osc_start = float(
self._cif_header_dictionary['Start_angle'].split()[0])
osc_range = 0.1
timestamp = 1 #XXX
return self._scan_factory.single(
self._image_file, format, exposure_time,
osc_start, osc_range, timestamp)
def detectorbase_start(self):
from iotbx.detectors.pilatus_minicbf import PilatusImage
self.detectorbase = PilatusImage(self._image_file)
self.detectorbase.readHeader()
self.detectorbase.parameters['SIZE1'] = 2167
self.detectorbase.parameters['SIZE2'] = 2070
class FormatCBFMiniEiger(FormatCBFMini):
'''A class for reading mini CBF format Eiger images, and correctly
constructing a model for the experiment from this.'''
@staticmethod
def understand(image_file):
'''Check to see if this looks like an Eiger mini CBF format image,
i.e. we can make sense of it.'''
header = FormatCBFMini.get_cbf_header(image_file)
for record in header.split('\n'):
if '# Detector' in record and \
'EIGER' in record:
return True
return False
def __init__(self, image_file):
'''Initialise the image structure from the given file, including a
proper model of the experiment.'''
assert(self.understand(image_file))
FormatCBFMini.__init__(self, image_file)
return
def _start(self):
FormatCBFMini._start(self)
def _goniometer(self):
return self._goniometer_factory.single_axis_reverse()
def _detector(self):
distance = float(
self._cif_header_dictionary['Detector_distance'].split()[0])
beam_xy = self._cif_header_dictionary['Beam_xy'].replace(
'(', '').replace(')', '').replace(',', '').split()[:2]
wavelength = float(
self._cif_header_dictionary['Wavelength'].split()[0])
beam_x, beam_y = map(float, beam_xy)
pixel_xy = self._cif_header_dictionary['Pixel_size'].replace(
'm', '').replace('x', '').split()
pixel_x, pixel_y = map(float, pixel_xy)
thickness = float(
self._cif_header_dictionary['Silicon'].split()[-2]) * 1000.0
nx = int(
self._cif_header_dictionary['X-Binary-Size-Fastest-Dimension'])
ny = int(
self._cif_header_dictionary['X-Binary-Size-Second-Dimension'])
overload = int(
self._cif_header_dictionary['Count_cutoff'].split()[0])
underload = -1
from cctbx.eltbx import attenuation_coefficient
table = attenuation_coefficient.get_table("Si")
mu = table.mu_at_angstrom(wavelength) / 10.0
t0 = thickness
detector = self._detector_factory.simple(
'PAD', distance * 1000.0, (beam_x * pixel_x * 1000.0,
beam_y * pixel_y * 1000.0), '+x', '-y',
(1000 * pixel_x, 1000 * pixel_y),
(nx, ny), (underload, overload), [],
ParallaxCorrectedPxMmStrategy(mu, t0))
for f0, s0, f1, s1 in [(0, 513, 1030, 550)]:
detector[0].add_mask(f0, s0, f1, s1)
return detector
def _beam(self):
wavelength = float(
self._cif_header_dictionary['Wavelength'].split()[0])
return self._beam_factory.simple(wavelength)
def _scan(self):
format = self._scan_factory.format('CBF')
exposure_time = float(
self._cif_header_dictionary['Exposure_period'].split()[0])
osc_start = float(
self._cif_header_dictionary['Start_angle'].split()[0])
osc_range = float(
self._cif_header_dictionary['Angle_increment'].split()[0])
timestamp = get_pilatus_timestamp(
self._cif_header_dictionary['timestamp'])
return self._scan_factory.single(
self._image_file, format, exposure_time,
osc_start, osc_range, timestamp)
def detectorbase_start(self):
from iotbx.detectors.pilatus_minicbf import PilatusImage
self.detectorbase = PilatusImage(self._image_file)
self.detectorbase.readHeader()
class FormatCBFMini(FormatCBF):
'''An image reading class for mini CBF format images i.e. those from
Dectris, which will read the header into a dictionary.'''
@staticmethod
def understand(image_file):
'''Check to see if this looks like an CBF format image, i.e. we can
make sense of it.'''
header = FormatCBF.get_cbf_header(image_file)
if '_diffrn.id' in header and '_diffrn_source' in header:
return False
def one_of_these_in(record):
these = [
'PILATUS',
'SLS',
'SSRL',
'?',
'XDS special',
'GENERIC_MINI', # intended for simulated PAD data, non-Pilatus array size
]
for convention in these:
if convention in record: return True
return False
for record in header.split('\n'):
if '_array_data.header_convention' in record and one_of_these_in(
record):
return True
if '# Detector' in record and \
'PILATUS' in record: #CBFlib v0.8.0 allowed
return True
if '# Detector' in record and \
'ADSC' in record and 'HF-4M' in header:
return True
return False
def __init__(self, image_file, **kwargs):
'''Initialise the image structure from the given file.'''
from dxtbx import IncorrectFormatError
if not self.understand(image_file):
raise IncorrectFormatError(self, image_file)
FormatCBF.__init__(self, image_file, **kwargs)
self._raw_data = None
return
def _start(self):
'''Open the image file, read the image header, copy it into a
dictionary for future reference.'''
FormatCBF._start(self)
cif_header = FormatCBF.get_cbf_header(self._image_file)
self._cif_header_dictionary = {}
for record in cif_header.split('\n'):
if not '#' in record[:1]:
continue
if len(record[1:].split()) <= 2 and record.count(':') == 2:
self._cif_header_dictionary['timestamp'] = record[1:].strip()
continue
tokens = record.replace('=', '').replace(':', '').split()[1:]
self._cif_header_dictionary[tokens[0]] = ' '.join(tokens[1:])
for record in self._mime_header.split('\n'):
if not record.strip():
continue
token, value = record.split(':')
self._cif_header_dictionary[token.strip()] = value.strip()
return
def _detector(self):
'''Return a model for a simple detector, presuming no one has
one of these on a two-theta stage. Assert that the beam centre is
provided in the Mosflm coordinate frame.'''
distance = float(
self._cif_header_dictionary['Detector_distance'].split()[0])
beam_xy = self._cif_header_dictionary['Beam_xy'].replace(
'(', '').replace(')', '').replace(',', '').split()[:2]
wavelength = float(
self._cif_header_dictionary['Wavelength'].split()[0])
beam_x, beam_y = map(float, beam_xy)
pixel_xy = self._cif_header_dictionary['Pixel_size'].replace(
'm', '').replace('x', '').split()
pixel_x, pixel_y = map(float, pixel_xy)
if 'Silicon' in self._cif_header_dictionary:
thickness = float(
self._cif_header_dictionary['Silicon'].split()[2]) * 1000.0
material = 'Si'
elif 'CdTe' in self._cif_header_dictionary:
thickness = float(
self._cif_header_dictionary['CdTe'].split()[2]) * 1000.0
material = 'CdTe'
else:
thickness = 0.450
material = 'Si'
nx = int(
self._cif_header_dictionary['X-Binary-Size-Fastest-Dimension'])
ny = int(self._cif_header_dictionary['X-Binary-Size-Second-Dimension'])
overload = dxtbx_overload_scale * int(
self._cif_header_dictionary['Count_cutoff'].split()[0])
underload = -1
# take into consideration here the thickness of the sensor also the
# wavelength of the radiation (which we have in the same file...)
from cctbx.eltbx import attenuation_coefficient
table = attenuation_coefficient.get_table(material)
mu = table.mu_at_angstrom(wavelength) / 10.0
t0 = thickness
detector = self._detector_factory.simple(
'PAD', distance * 1000.0,
(beam_x * pixel_x * 1000.0, beam_y * pixel_y * 1000.0), '+x', '-y',
(1000 * pixel_x, 1000 * pixel_y), (nx, ny), (underload, overload),
[], ParallaxCorrectedPxMmStrategy(mu, t0))
detector[0].set_thickness(thickness)
detector[0].set_material(material)
detector[0].set_mu(mu)
return detector
def _goniometer(self):
'''Return a model for a simple single-axis goniometer. This should
probably be checked against the image header, though for miniCBF
there are limited options for this.'''
if 'Phi' in self._cif_header_dictionary:
phi_value = float(self._cif_header_dictionary['Phi'].split()[0])
return self._goniometer_factory.single_axis()
def _beam(self):
'''Return a simple model for the beam.'''
wavelength = float(
self._cif_header_dictionary['Wavelength'].split()[0])
beam = self._beam_factory.simple(wavelength)
try:
flux = float(self._cif_header_dictionary['Flux'].split()[0])
beam.set_flux(flux)
except KeyError:
pass
except IndexError:
pass
try:
transmission = float(
self._cif_header_dictionary['Transmission'].split()[0])
beam.set_transmission(transmission)
except KeyError:
pass
except IndexError:
pass
return beam
def read_cbf_image(self, cbf_image):
from cbflib_adaptbx import uncompress
import binascii
start_tag = binascii.unhexlify('0c1a04d5')
data = self.open_file(cbf_image, 'rb').read()
data_offset = data.find(start_tag) + 4
cbf_header = data[:data_offset - 4]
fast = 0
slow = 0
length = 0
byte_offset = False
no_compression = False
for record in cbf_header.split('\n'):
if 'X-Binary-Size-Fastest-Dimension' in record:
fast = int(record.split()[-1])
elif 'X-Binary-Size-Second-Dimension' in record:
slow = int(record.split()[-1])
elif 'X-Binary-Number-of-Elements' in record:
length = int(record.split()[-1])
elif 'X-Binary-Size:' in record:
size = int(record.split()[-1])
elif 'conversions' in record:
if 'x-CBF_BYTE_OFFSET' in record:
byte_offset = True
elif 'x-CBF_NONE' in record:
no_compression = True
assert (length == fast * slow)
if byte_offset:
pixel_values = uncompress(packed=data[data_offset:data_offset +
size],
fast=fast,
slow=slow)
elif no_compression:
from boost.python import streambuf
from dxtbx import read_int32
from scitbx.array_family import flex
assert (len(self.get_detector()) == 1)
f = self.open_file(self._image_file)
f.read(data_offset)
pixel_values = read_int32(streambuf(f), int(slow * fast))
pixel_values.reshape(flex.grid(slow, fast))
else:
raise ValueError(
"Uncompression of type other than byte_offset or none is not supported (contact authors)"
)
return pixel_values
def get_raw_data(self):
if self._raw_data is None:
data = self.read_cbf_image(self._image_file)
self._raw_data = data
return self._raw_data
def detectorbase_start(self):
from iotbx.detectors.pilatus_minicbf import PilatusImage
self.detectorbase = PilatusImage(self._image_file)
self.detectorbase.readHeader() # necessary for LABELIT
@staticmethod
def as_file(detector,
beam,
gonio,
scan,
data,
path,
header_convention="GENERIC_MINI",
det_type="GENERIC"):
"""Note to developers: first attempt to write a miniCBF given a dxtbx-style experiment,
But fields are not filled rigorously as in cbflib/src/cbf_minicbf_header.c
Present code does not account for:
Pilatus model number and serial number
Data collection date and time
Sensor material
Dead time (exposure time - exposure period)
Highest trusted value
Energy threshold for photon counting
Gain
Bad pixels
Auxiliary files (bad pixel mask, flat field, trim, image path)
Detector not normal to beam
"""
import pycbf
from dxtbx.format.FormatCBFMultiTile import cbf_wrapper
cbf = cbf_wrapper()
cbf_root = os.path.splitext(os.path.basename(path))[0] + ".cbf"
cbf.new_datablock(os.path.splitext(os.path.basename(path))[0])
""" Data items in the ARRAY_DATA category are the containers for the array data
items described in the category ARRAY_STRUCTURE. """
cbf.add_category("array_data",
["header_convention", "header_contents", "data"])
# get pixel info out of detector object
panel = detector[0]
pixel_xy = panel.get_pixel_size()
pixel_x_microns = pixel_xy[0] * 1000
pixel_y_microns = pixel_xy[1] * 1000
# make sure we get the right units
thickness_meters = max(0.000001, panel.get_thickness() / 1000.0)
# maybe someday more people will do this
flux = beam.get_flux()
transmission = beam.get_transmission()
polarization_fraction = beam.get_polarization_fraction()
# get exposure information
if scan is None:
exposure_period = phi_start = osc = 0
else:
exposure_period = scan.get_exposure_times()[0]
phi_start = scan.get_oscillation()[0]
osc = scan.get_oscillation()[1]
# automatically account for read-out time?
# exposure_time = exposure_period-0.00203 # this is apropriate for Pilatus3 S
exposure_time = exposure_period # simulation is a perfect detector
tau = 0 # assume simulation is a perfect detector with no pile-up error
count_cutoff = detector[0].get_trusted_range()[1]
wavelength = beam.get_wavelength(
) # get the wavelength in the conventional way
energy = 12398.4245 / wavelength
threshold = energy / 2 # presume normal data collection convention
bad_pixels = 0 # maybe get this from negative pixel values?
assert len(detector) == 1, "only single-panel detectors supported"
distance_meters = detector[0].get_distance() / 1000
# fixed now, no longer backwards
beam_center = detector[0].get_beam_centre_px(beam.get_s0())
ORGX, ORGY = beam_center
cbf.add_row([
header_convention,
"""
# Detector: %(det_type)s, S/N 60-0000
# 1972-01-01T00:00:00.000
# Pixel_size %(pixel_x_microns).0fe-6 m x %(pixel_x_microns).0fe-6 m
# Silicon sensor, thickness %(thickness_meters)f m
# Exposure_time %(exposure_time).7f s
# Exposure_period %(exposure_period).7f s
# Tau = %(tau)f s
# Count_cutoff %(count_cutoff)d counts
# Threshold_setting: %(threshold)d eV
# Gain_setting: autog (vrf = 1.000)
# N_excluded_pixels = %(bad_pixels)d
# Excluded_pixels: badpix_mask.tif
# Flat_field: (nil)
# Trim_file: (nil)
# Image_path: /ramdisk/
# Wavelength %(wavelength).5f A
# Detector_distance %(distance_meters).5f m
# Beam_xy (%(ORGX).2f, %(ORGY).2f) pixels
# Flux %(flux)g
# Filter_transmission %(transmission).4f
# Start_angle %(phi_start).4f deg.
# Angle_increment %(osc).4f deg.
# Polarization %(polarization_fraction).3f
# Detector_2theta 0.0000 deg.
# Kappa 0.0000 deg.
# Phi %(phi_start).4f deg.
""" % locals()
])
binary_id = 1
focus = data.focus()
data2 = data.copy_to_byte_str()
elements = len(data)
byteorder = "little_endian"
dimfast = focus[1]
dimmid = focus[0]
dimslow = 1
padding = 0
elsize = 4
elsigned = 1
cbf.set_integerarray_wdims_fs(pycbf.CBF_BYTE_OFFSET, binary_id, data2,
elsize, elsigned, elements, byteorder,
dimfast, dimmid, dimslow, padding)
cbf.write_widefile(cbf_root,pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K,0)
class FormatCBFMiniPilatus(FormatCBFMini):
'''A class for reading mini CBF format Pilatus images, and correctly
constructing a model for the experiment from this.'''
@staticmethod
def understand(image_file):
'''Check to see if this looks like an Pilatus mini CBF format image,
i.e. we can make sense of it.'''
if 'ENABLE_PHOTON_FACTORY_TWO_EIGER' in os.environ:
return False
header = FormatCBFMini.get_cbf_header(image_file)
for record in header.split('\n'):
if '# Detector' in record and \
'EIGER' in record.upper():
return False
for record in header.split('\n'):
if '_array_data.header_convention' in record and \
'PILATUS' in record:
return True
if '_array_data.header_convention' in record and \
'SLS' in record:
return True
if '_array_data.header_convention' in record and \
'?' in record:
return True
if '# Detector' in record and \
'PILATUS' in record: #CBFlib v0.8.0 allowed
return True
return False
def __init__(self, image_file, **kwargs):
'''Initialise the image structure from the given file, including a
proper model of the experiment.'''
assert(self.understand(image_file))
FormatCBFMini.__init__(self, image_file, **kwargs)
self._raw_data = None
return
def _start(self):
FormatCBFMini._start(self)
def _goniometer(self):
'''Return a model for a simple single-axis goniometer. This should
probably be checked against the image header, though for miniCBF
there are limited options for this.'''
if 'Phi' in self._cif_header_dictionary:
phi_value = float(self._cif_header_dictionary['Phi'].split()[0])
return self._goniometer_factory.single_axis()
def _detector(self):
'''Return a model for a simple detector, presuming no one has
one of these on a two-theta stage. Assert that the beam centre is
provided in the Mosflm coordinate frame.'''
distance = float(
self._cif_header_dictionary['Detector_distance'].split()[0])
beam_xy = self._cif_header_dictionary['Beam_xy'].replace(
'(', '').replace(')', '').replace(',', '').split()[:2]
wavelength = float(
self._cif_header_dictionary['Wavelength'].split()[0])
beam_x, beam_y = map(float, beam_xy)
pixel_xy = self._cif_header_dictionary['Pixel_size'].replace(
'm', '').replace('x', '').split()
pixel_x, pixel_y = map(float, pixel_xy)
thickness = float(
self._cif_header_dictionary['Silicon'].split()[2]) * 1000.0
nx = int(
self._cif_header_dictionary['X-Binary-Size-Fastest-Dimension'])
ny = int(
self._cif_header_dictionary['X-Binary-Size-Second-Dimension'])
overload = dxtbx_overload_scale * int(
self._cif_header_dictionary['Count_cutoff'].split()[0])
underload = -1
# take into consideration here the thickness of the sensor also the
# wavelength of the radiation (which we have in the same file...)
from cctbx.eltbx import attenuation_coefficient
table = attenuation_coefficient.get_table("Si")
mu = table.mu_at_angstrom(wavelength) / 10.0
t0 = thickness
detector = self._detector_factory.simple(
'PAD', distance * 1000.0, (beam_x * pixel_x * 1000.0,
beam_y * pixel_y * 1000.0), '+x', '-y',
(1000 * pixel_x, 1000 * pixel_y),
(nx, ny), (underload, overload), [],
ParallaxCorrectedPxMmStrategy(mu, t0))
for f0, s0, f1, s1 in determine_pilatus_mask(detector):
detector[0].add_mask(f0, s0, f1, s1)
detector[0].set_thickness(thickness)
detector[0].set_material('Si')
detector[0].set_mu(mu)
return detector
def _beam(self):
'''Return a simple model for the beam.'''
wavelength = float(
self._cif_header_dictionary['Wavelength'].split()[0])
return self._beam_factory.simple(wavelength)
def _scan(self):
'''Return the scan information for this image.'''
format = self._scan_factory.format('CBF')
exposure_time = float(
self._cif_header_dictionary['Exposure_period'].split()[0])
osc_start = float(
self._cif_header_dictionary['Start_angle'].split()[0])
osc_range = float(
self._cif_header_dictionary['Angle_increment'].split()[0])
timestamp = get_pilatus_timestamp(
self._cif_header_dictionary['timestamp'])
return self._scan_factory.single(
self._image_file, format, exposure_time,
osc_start, osc_range, timestamp)
def read_cbf_image(self, cbf_image):
from cbflib_adaptbx import uncompress
import binascii
from scitbx.array_family import flex
start_tag = binascii.unhexlify('0c1a04d5')
data = self.open_file(cbf_image, 'rb').read()
data_offset = data.find(start_tag) + 4
cbf_header = data[:data_offset - 4]
fast = 0
slow = 0
length = 0
for record in cbf_header.split('\n'):
if 'X-Binary-Size-Fastest-Dimension' in record:
fast = int(record.split()[-1])
elif 'X-Binary-Size-Second-Dimension' in record:
slow = int(record.split()[-1])
elif 'X-Binary-Number-of-Elements' in record:
length = int(record.split()[-1])
elif 'X-Binary-Size:' in record:
size = int(record.split()[-1])
assert(length == fast * slow)
pixel_values = uncompress(packed = data[data_offset:data_offset + size],
fast = fast, slow = slow)
return pixel_values
def get_raw_data(self):
if self._raw_data is None:
data = self.read_cbf_image(self._image_file)
self._raw_data = data
return self._raw_data
def get_mask(self, goniometer=None):
from scitbx.array_family import flex
detector = self.get_detector()
mask = [flex.bool(flex.grid(reversed(p.get_image_size())), True)
for p in detector]
for i, p in enumerate(detector):
untrusted_regions = p.get_mask()
for j, (f0, s0, f1, s1) in enumerate(untrusted_regions):
sub_array = flex.bool(flex.grid(s1-s0+1, f1-f0+1), False)
mask[i].matrix_paste_block_in_place(sub_array, s0-1, f0-1)
if len(detector) == 1:
raw_data = [self.get_raw_data()]
else:
raw_data = self.get_raw_data()
assert(len(raw_data) == len(detector))
trusted_mask = [
p.get_trusted_range_mask(im) for im, p in zip(raw_data, detector)]
# returns merged untrusted pixels and active areas using bitwise AND (pixels are accepted
# if they are inside of the active areas AND inside of the trusted range)
return tuple([m & tm for m, tm in zip(mask, trusted_mask)])
def detectorbase_start(self):
from iotbx.detectors.pilatus_minicbf import PilatusImage
self.detectorbase = PilatusImage(self._image_file)
self.detectorbase.readHeader() # necessary for LABELIT
class FormatCBFMiniEigerPhotonFactory(FormatCBFMini):
"""A class for reading mini CBF format Eiger images, and correctly
constructing a model for the experiment from this.
Specific for 2-Eiger setup at Photon Factory based on images sent by
Yusuke Yamada. Geometry currently hard-coded as no geometry information in
image headers. Assumes image filenames contain the string '_upper_' or
'_lower_' to distinguish the two detectors.
Only works if environment variable ENABLE_PHOTON_FACTORY_TWO_EIGER is set.
"""
@staticmethod
def understand(image_file):
# no way to uniquely identify given example images
if "ENABLE_PHOTON_FACTORY_TWO_EIGER" in os.environ:
return True
return False
def _goniometer(self):
return self._goniometer_factory.make_goniometer(
(1, 0, 0), (1, 0, 0, 0, 1, 0, 0, 0, 1))
def _detector(self):
if "_lower_" in self._image_file:
# Detector:
# Panel:
# pixel_size:{0.075,0.075}
# image_size: {2070,2167}
# trusted_range: {0,1e+07}
# thickness: 0
# material:
# mu: 0
# fast_axis: {0.999997,-0.00228845,0.00127629}
# slow_axis: {-0.0026197,-0.862841,0.505469}
# origin: {-79.3767,2.47119,-169.509}
detector = self._detector_factory.complex(
"PAD",
origin=(-79.3767, 2.47119, -169.509),
fast=(0.999997, -0.00228845, 0.00127629),
slow=(-0.0026197, -0.862841, 0.505469),
pixel=(0.075, 0.075),
size=(2070, 2167),
trusted_range=(-1, 1e7), # XXX FIXME
)
elif "_upper_" in self._image_file:
# Detector:
# Panel:
# pixel_size:{0.075,0.075}
# image_size: {2070,2167}
# trusted_range: {0,1e+07}
# thickness: 0
# material:
# mu: 0
# fast_axis: {-0.999993,0.00338011,-0.00156153}
# slow_axis: {0.00213163,0.863573,0.504219}
# origin: {75.1912,14.1117,-124.34}
detector = self._detector_factory.complex(
"PAD",
origin=(75.1912, 14.1117, -124.34),
fast=(-0.999993, 0.00338011, -0.00156153),
slow=(0.00213163, 0.863573, 0.504219),
pixel=(0.075, 0.075),
size=(2070, 2167),
trusted_range=(-1, 1e7), # XXX FIXME
)
else:
raise RuntimeError("Don't understand image")
return detector
def _beam(self):
# Beam:
# wavelength: 1.1
# sample to source direction : {-0.00573979,-0,0.999984}
# divergence: 0
# sigma divergence: 0
# polarization normal: {0,1,0}
# polarization fraction: 0.999
return self._beam_factory.complex(
sample_to_source=(-0.00573979, -0, 0.999984),
polarization_fraction=0.999,
polarization_plane_normal=(0, 1, 0),
wavelength=1.1,
)
def _scan(self):
format = self._scan_factory.format("CBF")
exposure_time = 1 # XXX
osc_start = float(
self._cif_header_dictionary["Start_angle"].split()[0])
osc_range = 0.1
timestamp = 1 # XXX
return self._scan_factory.single(self._image_file, format,
exposure_time, osc_start, osc_range,
timestamp)
def detectorbase_start(self):
self.detectorbase = PilatusImage(self._image_file)
self.detectorbase.readHeader()
self.detectorbase.parameters["SIZE1"] = 2167
self.detectorbase.parameters["SIZE2"] = 2070
def get_vendortype(self):
return gv(self.get_detector())
class FormatCBFMini(FormatCBF):
"""An image reading class for mini CBF format images i.e. those from
Dectris, which will read the header into a dictionary."""
@staticmethod
def understand(image_file):
"""Check to see if this looks like an CBF format image, i.e. we can
make sense of it."""
header = FormatCBF.get_cbf_header(image_file)
if "_diffrn.id" in header and "_diffrn_source" in header:
return False
def one_of_these_in(record):
these = [
"PILATUS",
"SLS",
"SSRL",
"?",
"XDS special",
"GENERIC_MINI", # intended for simulated PAD data, non-Pilatus array size
]
for convention in these:
if convention in record:
return True
return False
for record in header.split("\n"):
if "_array_data.header_convention" in record and one_of_these_in(record):
return True
if "# Detector" in record and "PILATUS" in record: # CBFlib v0.8.0 allowed
return True
if "# Detector" in record and "ADSC" in record and "HF-4M" in header:
return True
return False
def __init__(self, image_file, **kwargs):
"""Initialise the image structure from the given file."""
from dxtbx import IncorrectFormatError
if not self.understand(image_file):
raise IncorrectFormatError(self, image_file)
FormatCBF.__init__(self, image_file, **kwargs)
self._raw_data = None
def _start(self):
"""Open the image file, read the image header, copy it into a
dictionary for future reference."""
FormatCBF._start(self)
cif_header = FormatCBF.get_cbf_header(self._image_file)
self._cif_header_dictionary = {}
for record in cif_header.split("\n"):
if record[:1] != "#":
continue
if len(record[1:].split()) <= 2 and record.count(":") == 2:
self._cif_header_dictionary["timestamp"] = record[1:].strip()
continue
tokens = record.replace("=", "").replace(":", "").split()
self._cif_header_dictionary[tokens[1]] = " ".join(tokens[2:])
for record in self._mime_header.split("\n"):
if not record.strip():
continue
token, value = record.split(":")
self._cif_header_dictionary[token.strip()] = value.strip()
def _detector(self):
"""Return a model for a simple detector, presuming no one has
one of these on a two-theta stage. Assert that the beam centre is
provided in the Mosflm coordinate frame."""
distance = float(self._cif_header_dictionary["Detector_distance"].split()[0])
beam_xy = (
self._cif_header_dictionary["Beam_xy"]
.replace("(", "")
.replace(")", "")
.replace(",", "")
.split()[:2]
)
wavelength = float(self._cif_header_dictionary["Wavelength"].split()[0])
beam_x, beam_y = map(float, beam_xy)
pixel_xy = (
self._cif_header_dictionary["Pixel_size"]
.replace("m", "")
.replace("x", "")
.split()
)
pixel_x, pixel_y = map(float, pixel_xy)
if "Silicon" in self._cif_header_dictionary:
thickness = (
float(self._cif_header_dictionary["Silicon"].split()[2]) * 1000.0
)
material = "Si"
sensor = "PAD"
elif "CdTe" in self._cif_header_dictionary:
thickness = float(self._cif_header_dictionary["CdTe"].split()[2]) * 1000.0
material = "CdTe"
sensor = "PAD"
elif "CCD" in self._cif_header_dictionary:
thickness = 0
material = None
sensor = "CCD"
else:
thickness = 0
material = None
sensor = None
nx = int(self._cif_header_dictionary["X-Binary-Size-Fastest-Dimension"])
ny = int(self._cif_header_dictionary["X-Binary-Size-Second-Dimension"])
overload = dxtbx_overload_scale * int(
self._cif_header_dictionary["Count_cutoff"].split()[0]
)
underload = -1
if material is not None:
# take into consideration here the thickness of the sensor also the
# wavelength of the radiation (which we have in the same file...)
from cctbx.eltbx import attenuation_coefficient
table = attenuation_coefficient.get_table(material)
mu = table.mu_at_angstrom(wavelength) / 10.0
t0 = thickness
px_mm = ParallaxCorrectedPxMmStrategy(mu, t0)
else:
px_mm = SimplePxMmStrategy()
detector = self._detector_factory.simple(
sensor,
distance * 1000.0,
(beam_x * pixel_x * 1000.0, beam_y * pixel_y * 1000.0),
"+x",
"-y",
(1000 * pixel_x, 1000 * pixel_y),
(nx, ny),
(underload, overload),
[],
px_mm=px_mm,
)
if material is not None:
detector[0].set_thickness(thickness)
detector[0].set_material(material)
detector[0].set_mu(mu)
return detector
def _goniometer(self):
"""Return a model for a simple single-axis goniometer. This should
probably be checked against the image header, though for miniCBF
there are limited options for this."""
# if "Phi" in self._cif_header_dictionary:
# phi_value = float(self._cif_header_dictionary["Phi"].split()[0])
return self._goniometer_factory.single_axis()
def _beam(self):
"""Return a simple model for the beam."""
wavelength = float(self._cif_header_dictionary["Wavelength"].split()[0])
beam = self._beam_factory.simple(wavelength)
try:
flux = float(self._cif_header_dictionary["Flux"].split()[0])
beam.set_flux(flux)
except (IndexError, KeyError):
pass
try:
transmission = float(self._cif_header_dictionary["Transmission"].split()[0])
beam.set_transmission(transmission)
except (IndexError, KeyError):
pass
return beam
def _scan(self):
format = self._scan_factory.format("CBF")
exposure_time = float(self._cif_header_dictionary["Exposure_period"].split()[0])
osc_start = float(self._cif_header_dictionary["Start_angle"].split()[0])
osc_range = float(self._cif_header_dictionary["Angle_increment"].split()[0])
if "timestamp" in self._cif_header_dictionary:
timestamp = get_pilatus_timestamp(self._cif_header_dictionary["timestamp"])
else:
timestamp = 0.0
return self._scan_factory.single(
self._image_file, format, exposure_time, osc_start, osc_range, timestamp
)
def _read_cbf_image(self):
from cbflib_adaptbx import uncompress
start_tag = binascii.unhexlify("0c1a04d5")
with self.open_file(self._image_file, "rb") as fh:
data = fh.read()
data_offset = data.find(start_tag) + 4
cbf_header = self._parse_cbf_header(
data[: data_offset - 4].decode("ascii", "ignore")
)
if cbf_header["byte_offset"]:
pixel_values = uncompress(
packed=data[data_offset : data_offset + cbf_header["size"]],
fast=cbf_header["fast"],
slow=cbf_header["slow"],
)
elif cbf_header["no_compression"]:
from boost.python import streambuf
from dxtbx import read_int32
from scitbx.array_family import flex
assert len(self.get_detector()) == 1
with self.open_file(self._image_file) as f:
f.read(data_offset)
pixel_values = read_int32(streambuf(f), cbf_header["length"])
pixel_values.reshape(flex.grid(cbf_header["slow"], cbf_header["fast"]))
else:
raise ValueError(
"Compression of type other than byte_offset or none is not supported (contact authors)"
)
return pixel_values
def get_raw_data(self):
if self._raw_data is None:
data = self._read_cbf_image()
self._raw_data = data
return self._raw_data
def detectorbase_start(self):
from iotbx.detectors.pilatus_minicbf import PilatusImage
self.detectorbase = PilatusImage(self._image_file)
self.detectorbase.readHeader() # necessary for LABELIT
@staticmethod
def as_file(
detector,
beam,
gonio,
scan,
data,
path,
header_convention="GENERIC_MINI",
det_type="GENERIC",
):
"""Note to developers: first attempt to write a miniCBF given a dxtbx-style experiment,
But fields are not filled rigorously as in cbflib/src/cbf_minicbf_header.c
Present code does not account for:
Pilatus model number and serial number
Data collection date and time
Sensor material
Dead time (exposure time - exposure period)
Highest trusted value
Energy threshold for photon counting
Gain
Bad pixels
Auxiliary files (bad pixel mask, flat field, trim, image path)
Detector not normal to beam
"""
import pycbf
from dxtbx.format.FormatCBFMultiTile import cbf_wrapper
cbf = cbf_wrapper()
cbf_root = os.path.splitext(os.path.basename(path))[0] + ".cbf"
cbf.new_datablock(os.path.splitext(os.path.basename(path))[0].encode())
"""Data items in the ARRAY_DATA category are the containers for the array data
items described in the category ARRAY_STRUCTURE."""
cbf.add_category("array_data", ["header_convention", "header_contents", "data"])
# get pixel info out of detector object
panel = detector[0]
pixel_xy = panel.get_pixel_size()
pixel_x_microns = pixel_xy[0] * 1000
pixel_y_microns = pixel_xy[1] * 1000
# make sure we get the right units
thickness_meters = panel.get_thickness() / 1000.0
material = panel.get_material()
if material == "Si":
sensor = "Silicon"
elif material == "CdTe":
sensor = "CdTe"
elif panel.get_type() == "SENSOR_CCD":
sensor = "CCD"
# maybe someday more people will do this
flux = beam.get_flux()
transmission = beam.get_transmission()
polarization_fraction = beam.get_polarization_fraction()
# get exposure information
if scan is None:
exposure_period = phi_start = osc = 0
else:
exposure_period = scan.get_exposure_times()[0]
phi_start = scan.get_oscillation()[0]
osc = scan.get_oscillation()[1]
# automatically account for read-out time?
# exposure_time = exposure_period-0.00203 # this is apropriate for Pilatus3 S
exposure_time = exposure_period # simulation is a perfect detector
tau = 0 # assume simulation is a perfect detector with no pile-up error
trusted_range = detector[0].get_trusted_range()
count_cutoff = trusted_range[1]
wavelength = beam.get_wavelength() # get the wavelength in the conventional way
energy = 12398.4245 / wavelength
threshold = energy / 2 # presume normal data collection convention
bad_pixels = 0 # maybe get this from negative pixel values?
assert len(detector) == 1, "only single-panel detectors supported"
distance_meters = detector[0].get_distance() / 1000
# fixed now, no longer backwards
beam_center = detector[0].get_beam_centre_px(beam.get_s0())
ORGX, ORGY = beam_center
cbf.add_row(
[
header_convention,
"""
# Detector: %(det_type)s, S/N 60-0000
# 1972-01-01T00:00:00.000
# Pixel_size %(pixel_x_microns)ge-6 m x %(pixel_x_microns)ge-6 m
# %(sensor)s sensor, thickness %(thickness_meters)f m
# Exposure_time %(exposure_time).7f s
# Exposure_period %(exposure_period).7f s
# Tau = %(tau)f s
# Count_cutoff %(count_cutoff)d counts
# Threshold_setting: %(threshold)d eV
# Gain_setting: autog (vrf = 1.000)
# N_excluded_pixels = %(bad_pixels)d
# Excluded_pixels: badpix_mask.tif
# Flat_field: (nil)
# Trim_file: (nil)
# Image_path: /ramdisk/
# Wavelength %(wavelength)g A
# Detector_distance %(distance_meters)g m
# Beam_xy (%(ORGX)g, %(ORGY)g) pixels
# Flux %(flux)g
# Filter_transmission %(transmission).4f
# Start_angle %(phi_start).4f deg.
# Angle_increment %(osc).4f deg.
# Polarization %(polarization_fraction).3f
# Detector_2theta 0.0000 deg.
# Kappa 0.0000 deg.
# Phi %(phi_start).4f deg.
"""
% locals(),
]
)
binary_id = 1
focus = data.focus()
data2 = data.copy_to_byte_str()
elements = len(data)
byteorder = b"little_endian"
dimfast = focus[1]
dimmid = focus[0]
dimslow = 1
padding = 0
elsize = 4
elsigned = 1
cbf.set_integerarray_wdims_fs(
pycbf.CBF_BYTE_OFFSET,
binary_id,
data2,
elsize,
elsigned,
elements,
byteorder,
dimfast,
dimmid,
dimslow,
padding,
)
cbf.write_widefile(
cbf_root.encode(),
pycbf.CBF,
pycbf.MIME_HEADERS | pycbf.MSG_DIGEST | pycbf.PAD_4K,
0,
)
class FormatCBFMiniEiger(FormatCBFMini):
'''A class for reading mini CBF format Eiger images, and correctly
constructing a model for the experiment from this.'''
@staticmethod
def understand(image_file):
'''Check to see if this looks like an Eiger mini CBF format image,
i.e. we can make sense of it.'''
header = FormatCBFMini.get_cbf_header(image_file)
for record in header.split('\n'):
if '# Detector' in record and \
'EIGER' in record:
return True
return False
def __init__(self, image_file, **kwargs):
'''Initialise the image structure from the given file, including a
proper model of the experiment.'''
assert(self.understand(image_file))
FormatCBFMini.__init__(self, image_file, **kwargs)
return
def _start(self):
FormatCBFMini._start(self)
def _goniometer(self):
return self._goniometer_factory.single_axis_reverse()
def _detector(self):
distance = float(
self._cif_header_dictionary['Detector_distance'].split()[0])
beam_xy = self._cif_header_dictionary['Beam_xy'].replace(
'(', '').replace(')', '').replace(',', '').split()[:2]
wavelength = float(
self._cif_header_dictionary['Wavelength'].split()[0])
beam_x, beam_y = map(float, beam_xy)
pixel_xy = self._cif_header_dictionary['Pixel_size'].replace(
'm', '').replace('x', '').split()
pixel_x, pixel_y = map(float, pixel_xy)
thickness = float(
self._cif_header_dictionary['Silicon'].split()[-2]) * 1000.0
nx = int(
self._cif_header_dictionary['X-Binary-Size-Fastest-Dimension'])
ny = int(
self._cif_header_dictionary['X-Binary-Size-Second-Dimension'])
overload = int(
self._cif_header_dictionary['Count_cutoff'].split()[0])
underload = -1
from cctbx.eltbx import attenuation_coefficient
table = attenuation_coefficient.get_table("Si")
mu = table.mu_at_angstrom(wavelength) / 10.0
t0 = thickness
detector = self._detector_factory.simple(
'PAD', distance * 1000.0, (beam_x * pixel_x * 1000.0,
beam_y * pixel_y * 1000.0), '+x', '-y',
(1000 * pixel_x, 1000 * pixel_y),
(nx, ny), (underload, overload), [],
ParallaxCorrectedPxMmStrategy(mu, t0))
for f0, s0, f1, s1 in [(0, 513, 1030, 550)]:
detector[0].add_mask(f0, s0, f1, s1)
return detector
def _beam(self):
wavelength = float(
self._cif_header_dictionary['Wavelength'].split()[0])
return self._beam_factory.simple(wavelength)
def _scan(self):
format = self._scan_factory.format('CBF')
exposure_time = float(
self._cif_header_dictionary['Exposure_period'].split()[0])
osc_start = float(
self._cif_header_dictionary['Start_angle'].split()[0])
osc_range = float(
self._cif_header_dictionary['Angle_increment'].split()[0])
timestamp = get_pilatus_timestamp(
self._cif_header_dictionary['timestamp'])
return self._scan_factory.single(
self._image_file, format, exposure_time,
osc_start, osc_range, timestamp)
def detectorbase_start(self):
from iotbx.detectors.pilatus_minicbf import PilatusImage
self.detectorbase = PilatusImage(self._image_file)
self.detectorbase.readHeader()