def pydiffdump(files):
'''Print the class which claims to work with each file.'''
s = time.time()
for f in files:
print f
format = Registry.find(f)
print format.__name__
if format.understand(f):
i = format(f)
print 'Beam:'
print i.get_beam()
print 'Goniometer:'
print i.get_goniometer()
print 'Detector:'
print i.get_detector()
print 'Scan:'
print i.get_scan()
return time.time() - s
def pydiffdump_fast(files):
'''First find the class, then read every frame with it.'''
s = time.time()
format = Registry.find(files[0])
scan = None
for f in files:
i = format(f)
print 'Beam:'
print i.get_xbeam()
print 'Goniometer:'
print i.get_xgoniometer()
print 'Detector:'
print i.get_xdetector()
print 'Scan:'
print i.get_xscan()
if scan is None:
scan = i.get_xscan()
else:
scan += i.get_xscan()
print scan
return time.time() - s
def run(self):
from dxtbx.imageset import MultiFileReader, ImageSweep
from dxtbx.format.Registry import Registry
# Get the filenames
filenames = self.get_file_list()
# Create the format class
format_class = Registry.find(filenames[0])
# Create the reader
reader = MultiFileReader(format_class, filenames)
# Create the sweep
sweep = ImageSweep(reader)
# Run a load of tests
self.tst_get_item(sweep)
self.tst_len(sweep, len(filenames))
self.tst_iter(sweep)
self.tst_indices(sweep, range(0, 9))
self.tst_paths(sweep, filenames)
self.tst_is_valid(sweep)
self.tst_get_detectorbase(sweep, range(len(filenames)), 9)
self.tst_get_models(sweep, range(len(filenames)), 9)
self.tst_get_array_range(sweep, (0, 9))
self.tst_to_array(sweep, (3, 7), (3, 7, 50, 100, 100, 200))
self.tst_set_models(sweep)
def run(self):
from dxtbx.imageset import MultiFileReader, ImageSet
from dxtbx.format.Registry import Registry
# Get the filenames
filenames = self.get_file_list()
# Create the format class
format_class = Registry.find(filenames[0])
# Create the reader
reader = MultiFileReader(format_class, filenames)
# Create the imageset
imageset = ImageSet(reader)
# Run a load of tests
self.tst_get_item(imageset)
self.tst_len(imageset, len(filenames))
self.tst_iter(imageset)
self.tst_indices(imageset, range(0, 9))
self.tst_paths(imageset, filenames)
self.tst_is_valid(imageset)
self.tst_get_detectorbase(imageset, range(len(filenames)), 9)
self.tst_get_models(imageset, range(len(filenames)), 9)
def read_single_image(path):
if command_line.options.verbose:
sys.stdout.write("Processing %s...\n" % path)
from dxtbx.format.Registry import Registry
format_class = Registry.find(path)
i = format_class(path)
beam = i.get_beam()
assert len(i.get_detector()) == 1
detector = i.get_detector()[0]
beam_center = detector.get_beam_centre(beam.get_s0())
detector_address = format_class.__name__
distance = detector.get_distance()
img = i.get_raw_data().as_1d().as_double()
pixel_size = 0.5 * sum(detector.get_pixel_size())
saturated_value = int(round(detector.get_trusted_range()[1]))
size = detector.get_image_size()
scan = i.get_scan()
if scan is None:
time_tuple = (0, 0)
else:
time_tuple = (scan.get_epochs()[0], 0)
wavelength = beam.get_wavelength()
active_areas = flex.int((0, 0, size[0], size[1]))
return beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas
def test_nexus_file(dials_regression):
filename = os.path.join(dials_regression, "image_examples",
"LCLS_cspad_nexus",
"cxi78513_bslz4_r0014_subset4_master.h5")
from dxtbx.format.Registry import Registry
format_class = Registry.find(filename)
iset = format_class.get_imageset([filename])
assert len(iset) == 2
for i in range(len(iset)):
data = iset.get_raw_data(i)
mask = iset.get_mask(i)
b = iset.get_beam(i)
d = iset.get_detector(i)
g = iset.get_goniometer(i)
s = iset.get_scan(i)
iset = format_class.get_imageset([filename], single_file_indices=[1])
assert len(iset) == 1
for i in range(len(iset)):
data = iset.get_raw_data(i)
mask = iset.get_mask(i)
b = iset.get_beam(i)
d = iset.get_detector(i)
g = iset.get_goniometer(i)
s = iset.get_scan(i)
def TestRegistry2(files):
'''First find the class, then read every frame with it.'''
s = time.time()
format = Registry.find(files[0])
b0 = format(files[0]).get_beam()
g0 = format(files[0]).get_goniometer()
d0 = format(files[0]).get_detector()
for f in files:
print f
i = format(f)
print i.get_beam()
print i.get_goniometer()
print i.get_detector()
print i.get_scan()
print i.get_cube()
print i.get_beam() == b0, i.get_goniometer() == g0, \
i.get_detector() == d0
return time.time() - s
def dxtbx_spotfinder_factory(phil_params): from dxtbx.format.Registry import Registry reader = Registry.find(phil_params.distl.image[0]) from spotfinder.dxtbx_toolbox.practical_heuristics import heuristics_base Spotfinder = heuristics_base(phil_params) return Spotfinder
def ImageFactory(filename, optional_index=None):
from iotbx.detectors import url_support
from libtbx.utils import Sorry
if os.path.isfile(filename):
if not os.access(filename, os.R_OK):
raise Sorry("No read access to file %s" % filename)
from dxtbx.format.Registry import Registry
format_instance = Registry.find(filename)
instance = format_instance(filename)
if optional_index is not None:
return instance.get_detectorbase(optional_index)
return instance.get_detectorbase()
A = url_support.potential_url_request(filename)
if A.is_url_request():
for utype in all_url_types:
try:
I = utype(filename)
I.readHeader()
return I
except Exception:
pass
raise ImageException(filename +
" does not work as a functioning image URL")
raise ImageException(filename +
" not recognized as any known detector image type")
def read(self, path):
if self.command_line.options.verbose:
print "Processing %s" % path
from dxtbx.format.Registry import Registry
from dxtbx.format.FormatMultiImage import FormatMultiImage
format_class = Registry.find(path)
assert not issubclass(
format_class,
FormatMultiImage), "Average container files seperately"
img_instance = format_class(path)
beam = img_instance.get_beam()
assert len(img_instance.get_detector()) == 1
detector = img_instance.get_detector()[0]
beam_center = detector.get_beam_centre(beam.get_s0())
detector_address = format_class.__name__
distance = detector.get_distance()
img = img_instance.get_raw_data().as_1d().as_double()
pixel_size = 0.5 * sum(detector.get_pixel_size())
saturated_value = int(round(detector.get_trusted_range()[1]))
size = detector.get_image_size()
wavelength = beam.get_wavelength()
active_areas = flex.int((0, 0, size[0], size[1]))
return beam_center, detector_address, distance, img, pixel_size, saturated_value, size, wavelength, active_areas
def make_imageset(filenames, format_class=None, check_format=True,
single_file_indices=None, format_kwargs=None):
'''Create an image set'''
from dxtbx.format.Registry import Registry
from format.FormatMultiImage import FormatMultiImage
# Get the format object
if format_class == None and check_format:
format_class = Registry.find(filenames[0])
if format_class is None:
reader = NullReader(filenames, single_file_indices is not None)
else:
if issubclass(format_class, FormatMultiImage):
assert len(set(filenames)) == 1
if format_kwargs is None:
format_kwargs = {}
format_instance = format_class(filenames[0], **format_kwargs)
reader = SingleFileReader(format_instance)
else:
reader = MultiFileReader(format_class, filenames,
format_kwargs=format_kwargs)
# Return the imageset
return ImageSet(reader, indices=single_file_indices,
format_kwargs=format_kwargs)
def test_SACLA_MPCCD_Cheetah_File(dials_regression, lazy):
filename = os.path.join(dials_regression, "image_examples",
"SACLA_MPCCD_Cheetah", "run266702-0-subset.h5")
from dxtbx.format.Registry import Registry
format_class = Registry.find(filename)
iset = format_class.get_imageset([filename], lazy=lazy)
assert len(iset) == 4
for i in range(len(iset)):
assert iset.get_raw_data(i)
# assert iset.get_mask(i)
assert iset.get_beam(i)
assert iset.get_detector(i)
assert iset.get_goniometer(i) is None
assert iset.get_scan(i) is None
iset = format_class.get_imageset([filename],
single_file_indices=[1],
lazy=lazy)
assert len(iset) == 1
for i in range(len(iset)):
assert iset.get_raw_data(i)
# assert iset.get_mask(i)
assert iset.get_beam(i)
assert iset.get_detector(i)
assert iset.get_goniometer(i) is None
assert iset.get_scan(i) is None
def run(self):
from dxtbx.format.Registry import Registry
format_class = Registry.find(self.filename)
iset = format_class.get_imageset([self.filename])
assert len(iset) == 2
for i in range(len(iset)):
data = iset.get_raw_data(i)
mask = iset.get_mask(i)
b = iset.get_beam(i)
d = iset.get_detector(i)
g = iset.get_goniometer(i)
s = iset.get_scan(i)
print 'OK'
iset = format_class.get_imageset([self.filename],
single_file_indices=[1])
assert len(iset) == 1
for i in range(len(iset)):
data = iset.get_raw_data(i)
mask = iset.get_mask(i)
b = iset.get_beam(i)
d = iset.get_detector(i)
g = iset.get_goniometer(i)
s = iset.get_scan(i)
print 'OK'
def make_imageset(filenames,
format_class=None,
check_format=True,
single_file_indices=None,
format_kwargs=None):
'''Create an image set'''
from dxtbx.format.Registry import Registry
from dxtbx.format.Format import Format
from dxtbx.format.FormatMultiImage import FormatMultiImage
# Get the format object
if format_class == None:
if check_format:
format_class = Registry.find(filenames[0])
else:
if single_file_indices is None or len(single_file_indices) == 0:
format_class = Format
else:
format_class = FormatMultiImage
else:
format_class = format_class
imageset = format_class.get_imageset(
filenames,
single_file_indices = single_file_indices,
as_imageset = True,
format_kwargs = format_kwargs,
check_format = check_format)
# Return the imageset
return imageset
def _create_imageset(filelist, check_headers):
'''Create an image set'''
from dxtbx.format.Registry import Registry
# Extract info from filelist
template, indices, is_sweep = filelist
# Get the template format
count = template.count('#')
if count > 0:
pfx = template.split('#')[0]
sfx = template.split('#')[-1]
template_format = '%s%%0%dd%s' % (pfx, template.count('#'), sfx)
filenames = [template_format % index for index in indices]
else:
filenames = [template]
# Sort the filenames
filenames = sorted(filenames)
# Get the format object
format_class = Registry.find(filenames[0])
# Create the imageset
imageset = format_class.get_imageset(filenames, as_imageset=True)
# Return the image set
return imageset
def from_template(template, image_range=None, check_headers=False,
check_format=True):
'''Create a new sweep from a template.
Params:
template The template argument
image_range The image range
check_headers Check the headers to ensure all images are valid
Returns:
A list of sweeps
'''
import os
from dxtbx.format.Registry import Registry
from dxtbx.sweep_filenames import template_image_range
if not check_format: assert not check_headers
# Check the template is valid
if template.count('#') < 1:
raise ValueError("Invalid template")
# Get the template format
pfx = template.split('#')[0]
sfx = template.split('#')[-1]
template_format = '%s%%0%dd%s' % (pfx, template.count('#'), sfx)
# Get the template image range
if image_range is None:
image_range = template_image_range(template)
# Set the image range
array_range = (image_range[0] - 1, image_range[1])
# Create the sweep file list
filenames = SweepFileList(template_format, array_range)
# Get the format class
if check_format:
format_class = Registry.find(filenames[0])
from format.FormatMultiImage import FormatMultiImage
if issubclass(format_class, FormatMultiImage):
assert len(filenames) == 1
format_instance = format_class(filenames[0])
reader = SingleFileReader(format_instance)
else:
reader = MultiFileReader(format_class, filenames)
else:
reader = NullReader(filenames)
# Create the sweep object
sweep = ImageSweep(reader)
# Check the sweep is valid
if check_headers and not sweep.is_valid():
raise RuntimeError('Invalid sweep of images')
# Return the sweep
return [sweep]
def cbf_file_to_basis_dict(path): """ Maps a cbf file to a dictionary of tuples and basis objects, in the same form as the above from read_optical_metrology_from_flat_file @param path cbf file path """ from dxtbx.format.Registry import Registry reader = Registry.find(path) instance = reader(path) return map_detector_to_basis_dict(instance.get_detector())
def cbf_file_to_basis_dict(path): """ Maps a cbf file to a dictionary of tuples and basis objects, in the same form as the above from read_optical_metrology_from_flat_file @param path cbf file path """ from dxtbx.format.Registry import Registry reader = Registry.find(path) instance = reader(path) return map_detector_to_basis_dict(instance.get_detector())
def find_format(self, filename):
''' Check the current and child formats, otherwise search the registry. '''
from dxtbx.format.Registry import Registry
try:
if self._format_class == None or not self.understand(filename):
self._format_class = Registry.find(filename)
self._format_class = self.check_child_formats(filename)
except Exception:
return None
return self._format_class
def find_format(self, filename):
''' Check the current and child formats, otherwise search the registry. '''
from dxtbx.format.Registry import Registry
try:
if self._format_class == None or not self.understand(filename):
self._format_class = Registry.find(filename)
self._format_class = self.check_child_formats(filename)
except Exception:
return None
return self._format_class
def test_no_multiple_format_understanding(test_image):
""" for a given image file, walks the whole tree of Format objects.
If the file can be understood by multiple Format objects at the same
inheiritance level, it will return False, otherwise True."""
Registry.setup()
global any_understood, highest_level, found_a_repeat
any_understood = False
def recurse(format, image_file, level):
global any_understood, highest_level, found_a_repeat
for child in format._children:
try:
understood = child.understand(image_file)
except Exception as e:
understood = False
if understood:
any_understood = True
print ("level: %d" % (level), child.__name__)
found_a_repeat = level == highest_level
highest_level = level
recurse(child, image_file, level + 1)
level = 1
highest_level = 0
found_a_repeat = False
for format in Registry._formats:
try:
understood = format.understand(test_image)
except Exception as e:
understood = False
if understood:
any_understood = True
print ("level: %d" % (level), format.__name__)
found_a_repeat = level == highest_level
highest_level = 1
recurse(format, test_image, level + 1)
assert not found_a_repeat, "image file understood by multiple Format objects"
# It's a failure if nothing could understand this file
assert any_understood, "No formatter could be found"
def centroid_files_and_imageset(centroid_files):
from dxtbx.format.Registry import Registry
# Create the format class
format_class = Registry.find(centroid_files[0])
# Create the reader
imageset = format_class.get_imageset(centroid_files, as_imageset=True)
return centroid_files, imageset
def __init__(self, filename=None, imageset=None):
""" Provide a file name or imageset as input """
assert [filename, imageset].count(
None
) == 1, "Supply either filename or imageset"
if filename is not None:
format_class = Registry.find(filename)
imageset = format_class.get_imageset([filename])
self.imageset = imageset
def tst_dxtbx_compressed():
import libtbx.load_env
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError:
print 'FAIL: dials_regression not configured'
return
import os
from dxtbx.format.Registry import Registry
from dials_regression.image_examples.get_all_working_images import \
get_all_working_images
# test that reading gz or bz2 compressed files works: it doesn't!
from libtbx import smart_open
from libtbx.test_utils import open_tmp_directory
import shutil
tmp_dir = open_tmp_directory()
print tmp_dir
for directory, image in get_all_working_images():
file_path = os.path.join(dials_regression, 'image_examples', directory,
image)
for ext in ('.gz', '.bz2')[:]:
compressed_path = os.path.join(tmp_dir,
os.path.basename(file_path)) + ext
with open(file_path, 'rb') as f_in, smart_open.for_writing(
compressed_path) as f_out:
shutil.copyfileobj(f_in, f_out)
print file_path, compressed_path
format = Registry.find(compressed_path)
try:
i = format(compressed_path)
except Exception:
print 'Error reading compressed file: %s' % compressed_path
import traceback
traceback.print_exc()
else:
print 'Successfully read compressed file: %s' % compressed_path
det = i.get_detector()
if det is not None:
size = det[0].get_image_size()
b = i.get_beam()
g = i.get_goniometer()
s = i.get_scan()
try:
d = i.get_raw_data()
except IOError:
pass
print 'OK'
def __call__(self, filename):
''' Check the current and child formats, otherwise search the registry. '''
from dxtbx.format.Registry import Registry
try:
if self._format_class == None or not self.understand(filename):
self._format_class = Registry.find(filename)
self._format_class = self.check_child_formats(filename)
if self._verbose:
print 'Using %s for %s' % (self._format_class.__name__, filename)
except Exception:
return None
return self._format_class
def __call__(self, filename):
''' Check the current and child formats, otherwise search the registry. '''
from dxtbx.format.Registry import Registry
try:
if self._format_class == None or not self.understand(filename):
self._format_class = Registry.find(filename)
self._format_class = self.check_child_formats(filename)
if self._verbose:
print('Using %s for %s' % (self._format_class.__name__, filename))
except Exception:
return None
return self._format_class
def show_matching_formats():
import sys
from dxtbx.format.Registry import Registry
for arg in sys.argv[1:]:
print '=== %s ===' % arg
for fmt in Registry.get():
if fmt.understand(arg):
print fmt.__name__, fmt.understand(arg)
recurse(fmt, arg)
else:
print fmt.__name__, fmt.understand(arg)
def oneImage(self,framenumber):
self.reporters[framenumber] = []
from dxtbx.format.Registry import Registry
filename = self.phil_params.distl.image[framenumber]
reader = Registry.find(filename)
img = reader(filename)
detector = img.get_detector()
beam = img.get_beam()
S0 = beam.get_s0()
data = img.get_raw_data()
scan = img.get_scan()
print scan
if scan is None:
print "No scan"
RR = (0,1)
else:
print scan.get_oscillation()
RR = scan.get_oscillation_range()
from spotfinder.dxtbx_toolbox import Distl
sfall = Distl(params = self.phil_params, detector = detector, beam = beam, data = data)
resolutions = flex.double()
spotlist = []
from dials.model.data import ReflectionList,Reflection
reflections = ReflectionList()
for ip,panel in enumerate(detector):
for spot in sfall.finderlist[ip].spots:
resolutions.append( panel.get_resolution_at_pixel(S0, (spot.ctr_mass_x(), spot.ctr_mass_y())) )
spotlist.append(spot)
refl = Reflection()
refl.panel_number = ip
refl.centroid_position = (spot.ctr_mass_x(), spot.ctr_mass_y(),0.0)
refl.centroid_variance = (0.5,0.5,0.0)
reflections.append(refl)
selection = (resolutions>0.0)
if self.phil_params.distl.res.outer is not None:
selection = (selection and (resolutions>self.phil_params.distl.res.outer))
if self.phil_params.distl.res.inner is not None:
selection = (selection and (resolutions<self.phil_params.distl.res.inner))
reflections = reflections.select(selection)
return dict(detector=detector, beam=beam, reflections=reflections, scan = scan,
gonio = img.get_goniometer())
def __init__(self, name, bases, attributes):
super(_MetaFormat, self).__init__(name, bases, attributes)
# Do-nothing until the Format module, defining the base class,
# has been loaded.
try:
sys.modules[Format.__module__]
except NameError:
return
# Add the class to the registry if it is directly derived from
# Format.
self._children = []
if Format in bases:
from dxtbx.format.Registry import Registry
Registry.add(self)
return
# Add the class to the list of children of its superclasses.
for base in bases:
base._children.append(self)
return
def load(filename):
"""Use DXTBX to get the files from the input filename.
Params:
filename The input filename
Returns:
The dxtbx format instance
"""
from dxtbx.format.Registry import Registry
format_instance = Registry.find(filename)
return format_instance(filename)
def load(filename):
"""Use DXTBX to get the files from the input filename.
Params:
filename The input filename
Returns:
The dxtbx format instance
"""
from dxtbx.format.Registry import Registry
format_instance = Registry.find(filename)
return format_instance(filename)
def show_matching_formats(files):
from dxtbx.format.Registry import Registry
for filename in files:
print('\n=== %s ===' % filename)
if os.path.exists(filename):
for fmt in Registry.get():
understood = fmt.understand(filename)
print("%s: %s" % (fmt.__name__, understood))
if understood:
recurse(fmt, filename)
else:
print("File not found.")
def __init__(self, name, bases, attributes):
super(_MetaFormat, self).__init__(name, bases, attributes)
# Do-nothing until the Format module, defining the base class,
# has been loaded.
try:
sys.modules[Format.__module__]
except NameError:
return
# Add the class to the registry if it is directly derived from
# Format.
self._children = []
if Format in bases:
from dxtbx.format.Registry import Registry
Registry.add(self)
return
# Add the class to the list of children of its superclasses.
for base in bases:
base._children.append(self)
return
def _create_sweep(filelist, check_headers):
"""Create a sweep"""
import os
from dxtbx.format.Registry import Registry
# Extract info from filelist
template, indices, is_sweep = filelist
# Get the template format
count = template.count("#")
if count > 0:
pfx = template.split("#")[0]
sfx = template.split("#")[-1]
template_format = "%s%%0%dd%s" % (pfx, template.count("#"), sfx)
filenames = [template_format % index for index in indices]
else:
filenames = [template]
# Sort the filenames
filenames = sorted(filenames)
# Get the format object
format_class = Registry.find(filenames[0])
# Get the first image and our understanding
first_image = filenames[0]
# Get the directory and first filename and set the template format
directory, first_image_name = os.path.split(first_image)
first_image_number = indices[0]
# Get the template format
pfx = template.split("#")[0]
sfx = template.split("#")[-1]
template_format = "%s%%0%dd%s" % (pfx, template.count("#"), sfx)
# Set the image range
array_range = (min(indices) - 1, max(indices))
# Create the sweep file list
filenames = SweepFileList(template_format, array_range)
# Create the sweep object
sweep = ImageSweep(MultiFileReader(format_class, filenames))
# Check the sweep is valid
if check_headers and not sweep.is_valid():
raise RuntimeError("Invalid sweep of images")
# Return the sweep
return sweep
def _create_sweep(filelist, check_headers):
'''Create a sweep'''
import os
from dxtbx.format.Registry import Registry
# Extract info from filelist
template, indices, is_sweep = filelist
# Get the template format
count = template.count('#')
if count > 0:
pfx = template.split('#')[0]
sfx = template.split('#')[-1]
template_format = '%s%%0%dd%s' % (pfx, template.count('#'), sfx)
filenames = [template_format % index for index in indices]
else:
filenames = [template]
# Sort the filenames
filenames = sorted(filenames)
# Get the format object
format_class = Registry.find(filenames[0])
# Get the first image and our understanding
first_image = filenames[0]
# Get the directory and first filename and set the template format
directory, first_image_name = os.path.split(first_image)
first_image_number = indices[0]
# Get the template format
pfx = template.split('#')[0]
sfx = template.split('#')[-1]
template_format = '%s%%0%dd%s' % (pfx, template.count('#'), sfx)
# Set the image range
array_range = (min(indices) - 1, max(indices))
# Create the sweep file list
filenames = SweepFileList(template_format, array_range)
# Create the sweep object
sweep = ImageSweep(MultiFileReader(format_class, filenames))
# Check the sweep is valid
if check_headers and not sweep.is_valid():
raise RuntimeError('Invalid sweep of images')
# Return the sweep
return sweep
def print_detector_info2(image):
"""
Print out information on the detector given an image
"""
format_instance = Registry.find(image)
instance = format_instance(image)
# adds parameters (iotbx)
temp = instance.get_detectorbase()
print "\nInformation from dxtbx Registry"
print "================================="
for key, val in temp.parameters.iteritems():
print "%20s::%s" % (key, val)
def load(filename):
"""Use DXTBX to load the input filename.
:param filename: The input filename
:type filename: os.PathLike or str or bytes
:returns: A dxtbx Format-subclass instance for the file type
:raises IOError: if the file format could not be determined
"""
from dxtbx.format.Registry import Registry
# Unwrap PEP-519-style objects. This covers py.path, pathlib, ...
if hasattr(filename, "__fspath__"):
filename = filename.__fspath__()
format_instance = Registry.find(filename)
return format_instance(filename)
def load(filename):
"""Use DXTBX to load the input filename.
:param filename: The input filename
:type filename: str or py.path
:returns: A dxtbx Format-subclass instance for the file type
:raises IOError: if the file format could not be determined
"""
from dxtbx.format.Registry import Registry
# Unwrap py.path objects into strings
if hasattr(filename, "strpath"):
filename = filename.strpath
format_instance = Registry.find(filename)
return format_instance(filename)
def print_detector_info2(image):
"""
Print out information on the detector given an image
"""
format_instance = Registry.find(image)
instance = format_instance(image)
# adds parameters (iotbx)
temp = instance.get_detectorbase()
print "\nInformation from dxtbx Registry"
print "================================="
for key, val in temp.parameters.iteritems():
print "%20s::%s" % (key, val)
def _create_sweep(filelist, check_headers):
'''Create a sweep'''
import os
from dxtbx.format.Registry import Registry
# Extract info from filelist
template, indices, is_sweep = filelist
# Get the template format
count = template.count('#')
if count > 0:
pfx = template.split('#')[0]
sfx = template.split('#')[-1]
template_format = '%s%%0%dd%s' % (pfx, template.count('#'), sfx)
filenames = [template_format % index for index in indices]
else:
filenames = [template]
# Sort the filenames
filenames = sorted(filenames)
# Get the format object
format_class = Registry.find(filenames[0])
# Get the first image and our understanding
first_image = filenames[0]
# Get the directory and first filename and set the template format
directory, first_image_name = os.path.split(first_image)
first_image_number = indices[0]
# Get the template format
pfx = template.split('#')[0]
sfx = template.split('#')[-1]
template_format = '%s%%0%dd%s' % (pfx, template.count('#'), sfx)
# Set the image range
array_range = (min(indices) - 1, max(indices))
# Create the sweep file list
filenames = [template_format % (i + 1) for i in range(*array_range)]
sweep = format_class.get_imageset(filenames,
template=template,
as_sweep=True)
# Return the sweep
return sweep
def run(self):
from dxtbx.imageset import MultiFileState
from dxtbx.format.Registry import Registry
# Get the filenames
filenames = self.get_filenames()
# Get the parameters we need
format_class = Registry.find(filenames[0])
# Create the state object
state = MultiFileState(format_class)
# Run a load of tests
self.tst_format_class(state, format_class)
self.tst_load_file(state, filenames)
def cbf2img(file):
from dxtbx.format.Registry import Registry
import sys, os
f = None
if file.split(".")[-1].lower() == "cbf" and os.path.exists(file):
if f is None:
f = Registry.find(file)
img = f(file)
db = img.get_detectorbase()
db.readHeader()
db.read()
db.show_header()
destpath = file.rstrip(".cbf") + ".img"
print "Writing %s as %s"%(file,destpath)
db.debug_write(destpath)
def print_header():
import sys
from dxtbx.format.Registry import Registry
from scitbx.array_family import flex
# this will do the lookup for every frame - this is strictly not needed
# if all frames are from the same instrument
for arg in sys.argv[1:]:
print '=== %s ===' % arg
format_class = Registry.find(arg)
print 'Using header reader: %s' % format_class.__name__
i = format_class(arg)
beam = i.get_beam()
goniometer = i.get_goniometer()
detector = i.get_detector()
scan = i.get_scan()
if beam is None:
print 'No beam model found'
else:
print beam
if detector is None:
print 'No detector model found'
else:
print detector
if goniometer is None:
print 'No goniometer model found'
else:
print goniometer
if scan is None:
print 'No scan model found'
else:
print scan
from dxtbx.format.FormatMultiImage import FormatMultiImage
if not issubclass(format_class, FormatMultiImage):
try:
raw_data = i.get_raw_data()
if not isinstance(raw_data, tuple):
raw_data = (raw_data,)
d = [p.as_1d() for p in raw_data]
print 'Total Counts: %d' % sum([flex.sum(p.select(p >= 0)) for p in d])
except AttributeError, e:
print "Could not read image data"
def make_imageset(filenames, format_class=None, check_format=True):
"""Create an image set"""
from dxtbx.format.Registry import Registry
from format.FormatMultiImage import FormatMultiImage
# Get the format object
if format_class == None and check_format:
format_class = Registry.find(filenames[0])
if format_class is None:
reader = NullReader(filenames)
else:
if issubclass(format_class, FormatMultiImage):
assert len(filenames) == 1
format_instance = format_class(filenames[0])
reader = SingleFileReader(format_instance)
else:
reader = MultiFileReader(format_class, filenames)
# Return the imageset
return ImageSet(reader)
def TestRegistry(files):
'''Print the class which claims to work with each file.'''
s = time.time()
for f in files:
print f
format = Registry.find(f)
print format.__name__
if format.understand(f):
i = format(f)
print i.get_beam()
print i.get_goniometer()
print i.get_detector()
return time.time() - s
def print_total():
import sys
from dxtbx.format.Registry import Registry
# this will do the lookup for every frame - this is strictly not needed
# if all frames are from the same instrument
for arg in sys.argv[1:]:
print '=== %s ===' % arg
format_class = Registry.find(arg)
print 'Using header reader: %s' % format_class.__name__
i = format_class(arg)
image_size = i.get_detector()[0].get_image_size()
d = i.get_raw_data()
if not isinstance(d, tuple):
d = (d,)
d = [p.as_1d() for p in d]
total = sum([sum(p.select(p >= 0)) for p in d])
print 'Total Counts: %d' % total
print 'Average Counts: %.2f' % (total / (image_size[0] * image_size[1]))
def run_tests(self, filenames):
from dxtbx.imageset import MultiFileReader
from dxtbx.format.Registry import Registry
# Get the parameters we need
format_class = Registry.find(filenames[0])
# Create the reader
reader = MultiFileReader(format_class, filenames)
# Run a load of tests
self.tst_get_image_paths(reader, filenames)
self.tst_get_format_class(reader, format_class)
self.tst_get_image_size(reader)
self.tst_get_path(reader, filenames)
self.tst_get_detectorbase(reader)
self.tst_get_models(reader)
self.tst_read(reader)
self.tst_get_format(reader)
self.tst_is_valid(reader)
def ImageFactory(filename):
from iotbx.detectors import url_support
from libtbx.utils import Sorry
if os.path.isfile(filename):
if not os.access(filename, os.R_OK):
raise Sorry("No read access to file %s" % filename)
from dxtbx.format.Registry import Registry
format_instance = Registry.find(filename)
instance = format_instance(filename)
return instance.get_detectorbase()
A = url_support.potential_url_request(filename)
if A.is_url_request():
for utype in all_url_types:
try:
I = utype(filename)
I.readHeader()
return I
except Exception:
pass
raise ImageException(filename+" does not work as a functioning image URL")
raise ImageException(filename+" not recognized as any known detector image type")
def _create_imageset(filelist, check_headers):
"""Create an image set"""
from dxtbx.format.Registry import Registry
# Extract info from filelist
template, indices, is_sweep = filelist
# Get the template format
count = template.count("#")
if count > 0:
pfx = template.split("#")[0]
sfx = template.split("#")[-1]
template_format = "%s%%0%dd%s" % (pfx, template.count("#"), sfx)
filenames = [template_format % index for index in indices]
else:
filenames = [template]
# Sort the filenames
filenames = sorted(filenames)
# Get the format object
format_class = Registry.find(filenames[0])
# Create the image set object
from format.FormatMultiImage import FormatMultiImage
if issubclass(format_class, FormatMultiImage):
assert len(filenames) == 1
format_instance = format_class(filenames[0])
image_set = ImageSet(SingleFileReader(format_instance))
else:
image_set = ImageSet(MultiFileReader(format_class, filenames))
# Check the image set is valid
if check_headers and not image_set.is_valid():
raise RuntimeError("Invalid ImageSet")
# Return the image set
return image_set
def TestRegistry3(files):
'''First find the class, then read every frame with it, then add the scans
together to make sure that they all make sense.'''
s = time.time()
format = Registry.find(files[0])
scan = format(files[0]).get_scan()
for f in files[1:]:
i = format(f)
scan += i.get_scan()
print scan
print scan[:len(scan) // 2]
print scan[:]
print scan[:len(scan)]
print scan[1 + len(scan) // 2:]
return time.time() - s
def failover_dxtbx(image_file):
'''Failover to use the dxtbx to read the image headers...'''
# replacement dxtbx for rigaku saturns sometimes
from dxtbx.format.Registry import Registry
from dxtbx.model.detector_helpers_types import detector_helpers_types
global last_format
if last_format:
iformat = last_format
else:
iformat = Registry.find(image_file)
from xia2.Handlers.Streams import Debug
Debug.write('Using dxtbx format instance: %s' % iformat.__name__)
if not iformat.understand(image_file):
raise RuntimeError, 'image file %s not understood by dxtbx' % \
image_file
last_format = iformat
i = iformat(image_file)
b = i.get_beam()
g = i.get_goniometer()
d = i.get_detector()
s = i.get_scan()
header = { }
if not hasattr(d, 'get_image_size'):
# cope with new detector as array of panels dxtbx api
fast, slow = map(int, d[0].get_image_size())
_f, _s = d[0].get_pixel_size()
F = matrix.col(d[0].get_fast_axis())
S = matrix.col(d[0].get_slow_axis())
N = F.cross(S)
origin = matrix.col(d[0].get_origin())
else:
fast, slow = map(int, d.get_image_size())
_f, _s = d.get_pixel_size()
F = matrix.col(d.get_fast_axis())
S = matrix.col(d.get_slow_axis())
N = F.cross(S)
origin = matrix.col(d.get_origin())
beam = matrix.col(b.get_direction())
# FIXME detector has methods to compute the beam centre now...
centre = - (origin - origin.dot(N) * N)
x = centre.dot(F)
y = centre.dot(S)
header['fast_direction'] = F.elems
header['slow_direction'] = S.elems
header['rotation_axis'] = g.get_rotation_axis()
if hasattr(s, 'get_exposure_time'):
header['exposure_time'] = s.get_exposure_time()
else:
header['exposure_time'] = s.get_exposure_times()[0]
header['distance'] = math.fabs(origin.dot(N))
if math.fabs(beam.angle(N, deg = True) - 180) < 0.1:
header['two_theta'] = 180 - beam.angle(N, deg = True)
else:
header['two_theta'] = - beam.angle(N, deg = True)
header['raw_beam'] = x, y
header['phi_start'] = s.get_oscillation()[0]
header['phi_width'] = s.get_oscillation()[1]
header['phi_end'] = sum(s.get_oscillation())
header['pixel'] = _f, _s
# FIXME this is very bad as it relates to teh legacy backwards Mosflm
# beam centre standard still... FIXME-SCI-948
header['beam'] = y, x
header['epoch'] = s.get_image_epoch(s.get_image_range()[0])
header['date'] = time.ctime(header['epoch'])
header['wavelength'] = b.get_wavelength()
header['size'] = fast, slow
if hasattr(i, 'detector_class'):
header['detector_class'] = i.detector_class
header['detector'] = i.detector
else:
if hasattr(d, 'get_type'):
# cope with new detector as array of panels API
dtype = d.get_type()
else:
dtype = d[0].get_type()
detector_type = detector_helpers_types.get(
dtype, fast, slow, int(1000 * _f), int(1000 * _s))
header['detector_class'] = detector_type.replace('-', ' ')
header['detector'] = detector_type.split('-')[0]
return header
def run(argv=None):
"""Compute mean, standard deviation, and maximum projection images
from a set of CSPAD cbf images given on the command line.
@param argv Command line argument list
@return @c 0 on successful termination, @c 1 on error, and @c 2
for command line syntax errors
"""
import libtbx.load_env
from libtbx import option_parser
from scitbx.array_family import flex
from dxtbx.format.Registry import Registry
from xfel.cftbx.detector.cspad_cbf_tbx import cbf_file_to_basis_dict, write_cspad_cbf
# from xfel.cxi.cspad_ana import cspad_tbx
# from iotbx.detectors.cspad_detector_formats import reverse_timestamp
if argv is None:
argv = sys.argv
command_line = (option_parser.option_parser(
usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \
"image1 image2 [image3 ...]" % libtbx.env.dispatcher_name)
.option(None, "--average-path", "-a",
type="string",
default=None,
dest="avg_path",
metavar="PATH",
help="Write average image to PATH")
.option(None, "--maximum-path", "-m",
type="string",
default=None,
dest="max_path",
metavar="PATH",
help="Write maximum projection image to PATH")
.option(None, "--stddev-path", "-s",
type="string",
default=None,
dest="stddev_path",
metavar="PATH",
help="Write standard deviation image to PATH")
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
).process(args=argv[1:])
# Note that it is not an error to omit the output paths, because
# certain statistics could still be printed, e.g. with the verbose
# option.
paths = command_line.args
if len(paths) == 0:
command_line.parser.print_usage(file=sys.stderr)
return 2
# Loop over all images and accumulate statistics.
nfail = 0
nmemb = 0
for path in paths:
if command_line.options.verbose:
sys.stdout.write("Processing %s...\n" % path)
try:
# Promote the image to double-precision floating point type.
# All real-valued flex arrays have the as_double() function.
# Warn if the header items across the set of images do not match
# up. Note that discrepancies regarding the image size are
# fatal.
if not 'reader' in locals():
reader = Registry.find(path)
img = reader(path)
if 'detector' in locals():
test_detector = img.get_detector()
if len(test_detector) != len(detector):
sys.stderr.write("Detectors do not have the same number of panels\n")
return 1
for t, d in zip(test_detector, detector):
if t.get_image_size() != d.get_image_size():
sys.stderr.write("Panel sizes do not match\n")
return 1
if t.get_pixel_size() != d.get_pixel_size():
sys.stderr.write("Pixel sizes do not match\n")
return 1
if t.get_d_matrix() != d.get_d_matrix():
sys.stderr.write("Detector panels are not all in the same location. The average will use the positions of the first image.\n")
detector = test_detector
else:
detector = img.get_detector()
data = [img.get_raw_data()[i].as_1d().as_double() for i in xrange(len(detector))]
wavelength = img.get_beam().get_wavelength()
distance = flex.mean(flex.double([d.get_directed_distance() for d in detector]))
except Exception:
nfail += 1
continue
# The sum-of-squares image is accumulated using long integers, as
# this delays the point where overflow occurs. But really, this
# is just a band-aid...
if nmemb == 0:
max_img = copy.deepcopy(data)
sum_distance = distance
sum_img = copy.deepcopy(data)
ssq_img = [flex.pow2(d) for d in data]
sum_wavelength = wavelength
metro = cbf_file_to_basis_dict(path)
else:
sel = [(d > max_d).as_1d() for d, max_d in zip(data, max_img)]
for d, max_d, s in zip(data, max_img, sel): max_d.set_selected(s, d.select(s))
sum_distance += distance
for d, sum_d in zip(data, sum_img): sum_d += d
for d, ssq_d in zip(data, ssq_img): ssq_d += flex.pow2(d)
sum_wavelength += wavelength
nmemb += 1
# Early exit if no statistics were accumulated.
if command_line.options.verbose:
sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail))
if nmemb == 0:
return 0
# Calculate averages for measures where other statistics do not make
# sense. Note that avg_img is required for stddev_img.
avg_img = [sum_d.as_double() / nmemb for sum_d in sum_img]
avg_distance = sum_distance / nmemb
avg_wavelength = sum_wavelength / nmemb
def make_tiles(data, detector):
"""
Assemble a tiles dictionary as required by write_cspad_cbf, consisting of 4 arrays of shape 8x185x388.
Assumes the order in the data array matches the order of the enumerated detector panels.
"""
assert len(data) == 64
tiles = {}
s, f = 185, 194
for q_id in xrange(4):
tiles[0,q_id] = flex.double((flex.grid(s*8, f*2)))
for s_id in xrange(8):
for a_id in xrange(2):
asic_idx = (q_id*16) + (s_id*2) + a_id
asic = data[asic_idx]
asic.reshape(flex.grid((s, f)))
tiles[0, q_id].matrix_paste_block_in_place(asic, s_id*s, a_id*f)
tiles[0, q_id].reshape(flex.grid((8, s, f*2)))
return tiles
# Output the average image, maximum projection image, and standard
# deviation image, if requested.
if command_line.options.avg_path is not None:
tiles = make_tiles(avg_img, detector)
write_cspad_cbf(tiles, metro, 'cbf', None, command_line.options.avg_path, avg_wavelength, avg_distance)
if command_line.options.max_path is not None:
tiles = make_tiles(max_img, detector)
write_cspad_cbf(tiles, metro, 'cbf', None, command_line.options.max_path, avg_wavelength, avg_distance)
if command_line.options.stddev_path is not None:
stddev_img = [ssq_d.as_double() - sum_d.as_double() * avg_d for ssq_d, sum_d, avg_d in zip(ssq_img, sum_img, avg_img)]
# Accumulating floating-point numbers introduces errors, which may
# cause negative variances. Since a two-pass approach is
# unacceptable, the standard deviation is clamped at zero.
for stddev_d in stddev_img:
stddev_d.set_selected(stddev_d < 0, 0)
if nmemb == 1:
stddev_img = [flex.sqrt(stddev_d) for stddev_d in stddev_img]
else:
stddev_img = [flex.sqrt(stddev_d / (nmemb - 1)) for stddev_d in stddev_img]
tiles = make_tiles(stddev_img, detector)
write_cspad_cbf(tiles, metro, 'cbf', None, command_line.options.stddev_path, avg_wavelength, avg_distance)
return 0
def event(self, evt, env):
from dxtbx.format.Registry import Registry
from os.path import exists
from time import sleep
# Nop if there is no image. For experiments configured to have
# exactly one event per calibration cycle, this should never
# happen.
if self._path is None:
evt.put(skip_event_flag(), "skip_event")
return
# Skip this event if the template isn't in the path
if self._template is not None and not True in [t in self._path for t in self._template.split(',')]:
evt.put(skip_event_flag(), "skip_event")
return
if "phi" in self._path:
evt.put(skip_event_flag(), "skip_event")
return
# Wait for the image to appear in the file system, probing for it
# at exponentially increasing delays.
t = 1
t_tot = 0
if not exists(self._path):
self._logger.info("Waiting for path %s"%self._path)
while not exists(self._path):
if t_tot > 1:
self._logger.info("Timeout waiting for path %s"%self._path)
evt.put(skip_event_flag(), "skip_event")
self._logger.info("Image not found: %s"%self._path)
return
sleep(t)
t_tot += t
t *= 2
# Find a matching Format object and instantiate it using the
# given path. If the Format object does not understand the image,
# try determining a new format. XXX Emits "Couldn't create a
# detector model for this image".
if self._fmt is None:
self._fmt = Registry.find(self._path)
if self._fmt is None:
evt.put(skip_event_flag(), "skip_event")
return
img = self._fmt(self._path)
if img is None:
self._fmt = Registry.find(self._path)
if self._fmt is None:
evt.put(skip_event_flag(), "skip_event")
return
img = self._fmt(self._path)
if img is None:
evt.put(skip_event_flag(), "skip_event")
return
self._logger.info(
"Reading %s using %s" % (self._path, self._fmt.__name__))
# Get the raw image data and convert to double precision floating
# point array. XXX Why will img.get_raw_data() not work, like it
# does in print_header?
db = img.get_detectorbase()
db.readHeader()
db.read()
data = db.get_raw_data().as_double()
# Get the pixel size and store it for common_mode.py
detector = img.get_detector()[0]
ps = detector.get_pixel_size()
assert ps[0] == ps[1]
pixel_size = ps[0]
evt.put(ps[0],"marccd_pixel_size")
evt.put(detector.get_trusted_range()[1],"marccd_saturated_value")
evt.put(detector.get_distance(),"marccd_distance")
# If the beam center isn't provided in the config file, get it from the
# image. It will probably be wrong.
if self._beam_x is None or self._beam_y is None:
self._beam_x, self._beam_y = detector.get_beam_centre_px(img.get_beam().get_s0())
self._beam_x = int(round(self._beam_x))
self._beam_y = int(round(self._beam_y))
# Crop the data so that the beam center is in the center of the image
maxy, maxx = data.focus()
minsize = min([self._beam_x,self._beam_y,maxx-self._beam_x,maxy-self._beam_y])
data = data[self._beam_y-minsize:self._beam_y+minsize,self._beam_x-minsize:self._beam_x+minsize]
evt.put((minsize,minsize),"marccd_beam_center")
evt.put(flex.int([0,0,minsize*2,minsize*2]),"marccd_active_areas")
# Store the image in the event.
evt.put(data, self._address)
# Store the .mmcd file name in the event
evt.put(self._mccd_name, "mccd_name")
evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
self._logger.info("converted %s to pickle with timestamp %s" %(self._path, timestamp))
# This should not be necessary as the machine is configured with
# one event per calibration cycle.
self._path = None
def make_sweep(
template, indices, format_class=None, beam=None, detector=None, goniometer=None, scan=None, check_format=True
):
"""Create a sweep"""
import os
from dxtbx.format.Registry import Registry
from format.FormatMultiImage import FormatMultiImage
indices = sorted(indices)
# Get the template format
count = template.count("#")
if count > 0:
pfx = template.split("#")[0]
sfx = template.split("#")[-1]
template_format = "%s%%0%dd%s" % (pfx, template.count("#"), sfx)
filenames = [template_format % index for index in indices]
else:
template_format = None
filenames = [template]
# Sort the filenames
filenames = sorted(filenames)
# Get the first image and our understanding
first_image = filenames[0]
# Get the directory and first filename and set the template format
directory, first_image_name = os.path.split(first_image)
first_image_number = indices[0]
# Set the image range
array_range = (min(indices) - 1, max(indices))
if scan is not None:
assert array_range == scan.get_array_range()
# Get the format object and reader
if format_class is None and check_format:
format_class = Registry.find(filenames[0])
# Create the reader
indices = None
if format_class is None:
if template_format is not None:
filenames = SweepFileList(template_format, array_range)
reader = NullReader(filenames)
else:
if issubclass(format_class, FormatMultiImage):
assert len(filenames) == 1
format_instance = format_class(filenames[0])
if scan is not None:
image0 = scan.get_array_range()[0]
indices = list(range(scan.get_num_images()))
reader = SingleFileReader(format_instance)
else:
assert template_format is not None
filenames = SweepFileList(template_format, array_range)
reader = MultiFileReader(format_class, filenames)
# Create the sweep object
sweep = ImageSweep(reader, indices=indices, beam=beam, detector=detector, goniometer=goniometer, scan=scan)
# Return the sweep
return sweep
