def __init__(self, image_file, **kwargs):
if not self.understand(image_file):
raise IncorrectFormatError(self, image_file)
FormatMultiImageLazy.__init__(self, **kwargs)
FormatStill.__init__(self, image_file, **kwargs)
Format.__init__(self, image_file, **kwargs)
self.current_index = None
self.current_event = None
self._psana_runs = {
} # empty container, to prevent breaking other formats
if "locator_scope" in kwargs:
self.params = FormatXTC.params_from_phil(
master_phil=kwargs["locator_scope"],
user_phil=image_file,
strict=True)
else:
self.params = FormatXTC.params_from_phil(master_phil=locator_scope,
user_phil=image_file,
strict=True)
assert self.params.mode == "idx", "idx mode should be used for analysis"
self._ds = FormatXTC._get_datasource(image_file, self.params)
self.populate_events()
self.n_images = len(self.times)
self._cached_psana_detectors = {}
self._beam_index = None
self._beam_cache = None
self._initialized = True
self._fee = None
def __init__(self, image_file, **kwargs):
'''Initialise the image structure from the given file.'''
Format.__init__(self, image_file, **kwargs)
assert(self.understand(image_file))
return
def __init__(self, image_file, **kwargs):
"""Initialise the image structure from the given file."""
if not self.understand(image_file):
raise IncorrectFormatError(self, image_file)
Format.__init__(self, image_file, **kwargs)
def __init__(self, image_file):
'''Initialise the image structure from the given file.'''
Format.__init__(self, image_file)
assert(self.understand(image_file))
return
def __init__(self, image_file, **kwargs):
from dxtbx import IncorrectFormatError
if not self.understand(image_file):
raise IncorrectFormatError(self, image_file)
FormatMultiImage.__init__(self, **kwargs)
Format.__init__(self, image_file, **kwargs)
def __init__(self, image_file, **kwargs):
'''Initialise the image structure from the given file.'''
from dxtbx import IncorrectFormatError
if not self.understand(image_file):
raise IncorrectFormatError(self, image_file)
Format.__init__(self, image_file, **kwargs)
def __init__(self, params, run, detector, **kwargs):
FormatMultiImageLazy.__init__(self, **kwargs)
FormatStill.__init__(self, None, **kwargs)
Format.__init__(self, None, **kwargs)
self.event = None
self.params = params
self.params.detector_address = [str(detector.name)]
self._cached_psana_detectors = {run.run(): detector}
self._beam_index = None
self._beam_cache = None
self._run = run
def _openingpathiterator(pathnames: Iterable[str]):
"""Utility function to efficiently open all paths.
A path is a potential file or directory.
Each path will be opened with :meth:`dxtbx.format.Format.open_file`,
but in order to do so each file will only be opened once, and extraneous
use of :func:`os.stat` will be avoided.
Any path entries that are a directory will be recursed into, once -
any further directories found will be ignored. Any path that is not
a file or directory, or on which IO fails for any reason, will still
be returned.
Args:
pathnames: Paths to attempt to open
"""
# Store a tuple of (recurse, pathname) to track what was root level
paths = collections.deque((True, x) for x in sorted(pathnames))
while paths:
# Get the next path from the queue
(do_recurse, pathname) = paths.popleft()
pathname = os.fspath(pathname)
try:
# Attempt to open this 'path'
Format.open_file(pathname)
except OSError as e:
if e.errno == errno.EISDIR:
if do_recurse:
# We've tried to open a directory. Get all the entries...
subdir_paths = sorted(
os.path.join(pathname, x)
for x in os.listdir(pathname))
# ... and add them to our queue. Make sure not to mark for recursion
paths.extendleft(
(False, x) for x in reversed(subdir_paths))
logger.debug("Adding %d files from %s", len(subdir_paths),
pathname)
else:
logger.debug("Not adding sub-level directory entry %s",
pathname)
# Don't return directory instances
continue
else:
# A non-directory-related IO error
logger.debug("Could not import %s: %s", pathname,
os.strerror(e.errno))
yield pathname
def __init__(self, image_file, **kwargs):
from dxtbx import IncorrectFormatError
if not self.understand(image_file):
raise IncorrectFormatError(self, image_file)
FormatMultiImageLazy.__init__(self, **kwargs)
FormatStill.__init__(self, image_file, **kwargs)
Format.__init__(self, image_file, **kwargs)
self.current_index = None
self.current_event = None
if 'locator_scope' in kwargs:
self.params = FormatXTC.params_from_phil(kwargs['locator_scope'],image_file,strict=True)
else:
self.params = FormatXTC.params_from_phil(locator_scope,image_file,strict=True)
assert self.params.mode == 'idx', 'idx mode should be used for analysis'
self._initialized = True
def tst_from_sweep(self):
from dxtbx.model import Beam, Detector, Goniometer, Scan
from dxtbx.model import Crystal
from dxtbx.format.Format import Format
filenames = ["filename_%01d.cbf" % (i + 1) for i in range(0, 2)]
imageset = Format.get_imageset(filenames,
beam=Beam(),
detector=Detector(),
goniometer=Goniometer(),
scan=Scan((1, 2), (0, 1)),
as_sweep=True)
crystal = Crystal((1, 0, 0), (0, 1, 0), (0, 0, 1),
space_group_symbol="P1")
experiments = ExperimentListFactory.from_imageset_and_crystal(
imageset, crystal)
assert (len(experiments) == 1)
assert (experiments[0].imageset is not None)
assert (experiments[0].beam is not None)
assert (experiments[0].detector is not None)
assert (experiments[0].goniometer is not None)
assert (experiments[0].scan is not None)
assert (experiments[0].crystal is not None)
print 'OK'
def test_experimentlist_dumper_dump_empty_sweep(tmpdir):
tmpdir.chdir()
from dxtbx.model import Beam, Detector, Goniometer, Scan
from dxtbx.model import Crystal
from dxtbx.format.Format import Format
filenames = ["filename_%01d.cbf" % (i+1) for i in range(0, 2)]
imageset = Format.get_imageset(
filenames,
beam = Beam((1, 0, 0)),
detector = Detector(),
goniometer = Goniometer(),
scan = Scan((1,2), (0.0, 1.0)),
as_sweep=True)
crystal = Crystal((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol="P1")
experiments = ExperimentListFactory.from_imageset_and_crystal(
imageset, crystal)
dump = ExperimentListDumper(experiments)
filename = 'temp.json'
dump.as_json(filename)
experiments2 = ExperimentListFactory.from_json_file(filename,
check_format=False)
check(experiments, experiments2)
def test_experimentlist_factory_from_datablock():
from dxtbx.model import Beam, Detector, Goniometer, Scan
from dxtbx.datablock import DataBlockFactory
from dxtbx.model import Crystal
from dxtbx.format.Format import Format
filenames = ["filename_%01d.cbf" % (i+1) for i in range(0, 2)]
imageset = Format.get_imageset(
filenames,
beam = Beam(),
detector = Detector(),
goniometer = Goniometer(),
scan = Scan((1,2), (0,1)),
as_sweep=True)
crystal = Crystal((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol="P1")
datablock = DataBlockFactory.from_imageset(imageset)
experiments = ExperimentListFactory.from_datablock_and_crystal(
datablock, crystal)
assert len(experiments) == 1
assert experiments[0].imageset is not None
assert experiments[0].beam is not None
assert experiments[0].detector is not None
assert experiments[0].goniometer is not None
assert experiments[0].scan is not None
assert experiments[0].crystal is not None
def tst_from_null_sweep(self):
from dxtbx.format.Format import Format
from dxtbx.imageset import ImageSweep
from dxtbx.model import Beam, Detector, Goniometer, Scan
filenames = ["template_%2d.cbf" % (i+1) for i in range(0, 10)]
sweep = Format.get_imageset(
filenames,
beam = Beam((0, 0, 1)),
detector = Detector(),
goniometer = Goniometer((1, 0, 0)),
scan = Scan((1, 10), (0, 0.1)))
# Create the datablock
datablock = DataBlockFactory.from_imageset(sweep)
assert(len(datablock) == 1)
datablock = datablock[0]
sweeps = datablock.extract_sweeps()
assert(len(sweeps) == 1)
assert(sweeps[0].get_beam() == sweep.get_beam())
assert(sweeps[0].get_detector() == sweep.get_detector())
assert(sweeps[0].get_goniometer() == sweep.get_goniometer())
assert(sweeps[0].get_scan() == sweep.get_scan())
print 'OK'
def find(self, image_file):
'''More useful - find the best format handler in the registry for your
image file. N.B. this is in principle a factory function.'''
self.setup()
# Recursively check whether any of the children understand
# image_file, in which case they are preferred over the parent
# format.
def recurse(format, image_file):
for child in sorted(format._children, key=lambda x: x.__name__):
try:
if child.understand(image_file):
return recurse(child, image_file)
except Exception:
pass
return format
for format in sorted(self._formats, key=lambda x: x.__name__):
try:
if format.understand(image_file):
return recurse(format, image_file)
except Exception:
pass
# Try opening the file; this could be an easy reason for failure
if not Format.is_url(image_file):
try:
with open(image_file) as f:
pass
except IOError as e:
# Assume that the OS file error makes sense to the user
raise SorryIOError(e)
raise IOError('no format support found for %s' % image_file)
def test_experimentlist_factory_from_datablock():
filenames = ["filename_%01d.cbf" % (i + 1) for i in range(0, 2)]
imageset = Format.get_imageset(
filenames,
beam=Beam(),
detector=Detector(),
goniometer=Goniometer(),
scan=Scan((1, 2), (0, 1)),
as_sequence=True,
)
crystal = Crystal((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol="P1")
datablock = DataBlockFactory.from_imageset(imageset)
assert datablock[0].format_class()
experiments = ExperimentListFactory.from_datablock_and_crystal(
datablock, crystal)
assert len(experiments) == 1
assert experiments[0].imageset
assert experiments[0].beam
assert experiments[0].detector is not None
assert experiments[0].goniometer
assert experiments[0].scan
assert experiments[0].crystal
def understand(image_file):
'''See if this looks like an RAXIS format image - clue is first
5 letters of file should be RAXIS.'''
if Format.open_file(image_file).read(5) == 'RAXIS':
return True
return False
def understand(image_file):
'''See if this looks like an RAXIS format image - clue is first
5 letters of file should be RAXIS.'''
if Format.open_file(image_file).read(5) == 'RAXIS':
return True
return False
def _start(self):
self._header_bytes = Format.open_file(self._image_file).read(1024)
if self._header_bytes[812:822].strip() in ["SGI", "IRIS"]:
self._f = ">f"
self._i = ">i"
else:
self._f = "<f"
self._i = "<i"
def __init__(self, image_file, **kwargs):
if not injected_data:
raise IncorrectFormatError(self, image_file)
self.header = {
"configuration": json.loads(injected_data.get("header2", "")),
"info": json.loads(injected_data.get("streamfile_2", "")),
}
self._goniometer_instance = None
self._detector_instance = None
self._beam_instance = None
self._scan_instance = None
FormatMultiImage.__init__(self, **kwargs)
Format.__init__(self, image_file, **kwargs)
self.setup()
def _start(self):
self._header_bytes = Format.open_file(self._image_file).read(1024)
if self._header_bytes[812:822].strip() in ['SGI', 'IRIS']:
self._f = '>f'
self._i = '>i'
else:
self._f = '<f'
self._i = '<i'
def _start(self):
self._header_bytes = Format.open_file(self._image_file).read(1024)
if self._header_bytes[812:822].strip() in ['SGI', 'IRIS']:
self._f = '>f'
self._i = '>i'
else:
self._f = '<f'
self._i = '<i'
def _start(self):
with Format.open_file(self._image_file) as fh:
self._header_bytes = fh.read(1024)
if self._header_bytes[812:822].strip() in (b"SGI", b"IRIS"):
self._f = ">f"
self._i = ">i"
else:
self._f = "<f"
self._i = "<i"
def __init__(self, image_file, **kwargs):
'''
Initialise the class
'''
assert (self.understand(image_file))
import json
self.header = json.load(open(image_file))
self._goniometer_instance = None
self._detector_instance = None
self._beam_instance = None
self._scan_instance = None
FormatMultiImage.__init__(self, **kwargs)
Format.__init__(self, image_file, **kwargs)
self.setup()
return
def read_basic_tiff_header(filename):
"""Read the TIFF header (assuming for the moment a 4k header...) and
return ... something."""
# things we hope to learn from the vanilla TIFF header
image_width = None
image_height = None
image_depth = None
header_size = None
with Format.open_file(filename, "rb") as fh:
tiff_header = fh.read(1024)
byte_order = _tiff_byte_order(tiff_header[:4])
if not byte_order:
return False
if byte_order == LITTLE_ENDIAN:
_I = "<I"
_H = "<H"
else:
_I = ">I"
_H = ">H"
offset = struct.unpack(_I, tiff_header[4:8])[0]
ntags = struct.unpack(_H, tiff_header[offset:offset + 2])[0]
start = offset + 2
for j in range(ntags):
type_desc = struct.unpack(_H, tiff_header[start:start + 2])[0]
start += 2
type_type = struct.unpack(_H, tiff_header[start:start + 2])[0]
start += 2
# type_size = struct.unpack(_I, tiff_header[start : start + 4])[0]
start += 4
if type_type == 4:
type_offset_or_value = struct.unpack(_I, tiff_header[start:start +
4])[0]
start += 4
elif type_type == 3:
type_offset_or_value = struct.unpack(_H, tiff_header[start:start +
2])[0]
start += 4
if type_desc == 256:
image_width = type_offset_or_value
elif type_desc == 257:
image_height = type_offset_or_value
elif type_desc == 258:
image_depth = type_offset_or_value
elif type_desc == 273:
header_size = type_offset_or_value
return image_width, image_height, image_depth, header_size, byte_order
def read_basic_tiff_header(filename):
'''Read the TIFF header (assuming for the moment a 4k header...) and
return ... something.'''
# things we hope to learn from the vanilla TIFF header
image_width = None
image_height = None
image_depth = None
header_size = None
byte_order = None
# OK then let's get started - and let's assume that the size is > 1 kb
byte_order = tiff_byte_order(filename)
tiff_header = Format.open_file(filename, 'rb').read(1024)
if byte_order == LITTLE_ENDIAN:
_I = '<I'
_H = '<H'
else:
_I = '>I'
_H = '>H'
offset = struct.unpack(_I, tiff_header[4:8])[0]
ntags = struct.unpack(_H, tiff_header[offset:offset + 2])[0]
start = offset + 2
for j in range(ntags):
type_desc = struct.unpack(_H, tiff_header[start:start + 2])[0]
start += 2
type_type = struct.unpack(_H, tiff_header[start:start + 2])[0]
start += 2
type_size = struct.unpack(_I, tiff_header[start:start + 4])[0]
start += 4
if type_type == 4:
type_offset_or_value = struct.unpack(
_I, tiff_header[start:start + 4])[0]
start += 4
elif type_type == 3:
type_offset_or_value = struct.unpack(
_H, tiff_header[start:start + 2])[0]
start += 4
if type_desc == 256:
image_width = type_offset_or_value
elif type_desc == 257:
image_height = type_offset_or_value
elif type_desc == 258:
image_depth = type_offset_or_value
elif type_desc == 273:
header_size = type_offset_or_value
return image_width, image_height, image_depth, header_size, byte_order
def __init__(self, image_file, **kwargs):
'''
Initialise the class
'''
assert(self.understand(image_file))
import json
self.header = json.load(open(image_file))
self._goniometer_instance = None
self._detector_instance = None
self._beam_instance = None
self._scan_instance = None
FormatMultiImage.__init__(self, **kwargs)
Format.__init__(self, image_file, **kwargs)
self.setup()
return
def _add_static_mask_to_iset(format_instance: Format, iset: ImageSet) -> None:
"""Combine any static mask from a Format with the ImageSet's mask"""
if format_instance is not None:
static_mask = format_instance.get_static_mask()
if static_mask is not None:
if not iset.external_lookup.mask.data.empty():
for m1, m2 in zip(static_mask, iset.external_lookup.mask.data):
m1 &= m2.data()
iset.external_lookup.mask.data = ImageBool(static_mask)
else:
iset.external_lookup.mask.data = ImageBool(static_mask)
def test_experimentlist_to_datablock_imageset(dials_data):
filenames = [
str(f) for f in dials_data("thaumatin_grid_scan").listdir(
"thau_3_2_*.cbf.bz2")
]
imageset = Format.get_imageset(filenames, as_imageset=True)
expts = ExperimentListFactory.from_imageset_and_crystal(imageset,
crystal=None)
datablocks = expts.to_datablocks()
assert len(datablocks) == 1
assert datablocks[0].num_images() == len(expts) == len(imageset)
assert len(datablocks[0].extract_imagesets()) == len(expts)
def __init__(self, image_file, **kwargs):
'''
Initialise the class
'''
from dxtbx import IncorrectFormatError
if not self.understand(image_file):
raise IncorrectFormatError(self, image_file)
import json
self.header = json.load(open(image_file))
self._goniometer_instance = None
self._detector_instance = None
self._beam_instance = None
self._scan_instance = None
FormatMultiImage.__init__(self, **kwargs)
Format.__init__(self, image_file, **kwargs)
self.setup()
return
def _start(self):
self._header_bytes = Format.open_file(self._image_file).read(1024)
if self._header_bytes[812:822].strip() in ['SGI', 'IRIS']:
self._f = '>f'
self._i = '>i'
else:
self._f = '<f'
self._i = '<i'
from iotbx.detectors.raxis import RAXISImage
self.detectorbase = RAXISImage(self._image_file)
self.detectorbase.readHeader()
def tiff_byte_order(filename):
'''Determine the byte order for the file from the magic numbers at the
very start of the file.'''
four_bytes = Format.open_file(filename, 'rb').read(4)
if 'II' in four_bytes[:2]:
assert(struct.unpack('<H', four_bytes[2:])[0] == 42)
return LITTLE_ENDIAN
elif 'MM' in four_bytes[:2]:
assert(struct.unpack('>H', four_bytes[2:])[0] == 42)
return BIG_ENDIAN
raise RuntimeError, '%s not recognised as TIFF' % filename
def __init__(self, image_file, **kwargs):
from dxtbx import IncorrectFormatError
if not self.understand(image_file):
raise IncorrectFormatError(self, image_file)
FormatMultiImageLazy.__init__(self, **kwargs)
FormatStill.__init__(self, image_file, **kwargs)
Format.__init__(self, image_file, **kwargs)
self.current_index = None
self.current_event = None
self._psana_runs = {
} ## empty container, to prevent breaking other formats
if "locator_scope" in kwargs:
self.params = FormatXTC.params_from_phil(
master_phil=kwargs["locator_scope"],
user_phil=image_file,
strict=True)
else:
self.params = FormatXTC.params_from_phil(master_phil=locator_scope,
user_phil=image_file,
strict=True)
assert self.params.mode == "idx", "idx mode should be used for analysis"
self._initialized = True
def tiff_byte_order(filename):
"""Determine the byte order for the file from the magic numbers at the
very start of the file."""
four_bytes = Format.open_file(filename, "rb").read(4)
if "II" in four_bytes[:2]:
assert struct.unpack("<H", four_bytes[2:])[0] == 42
return LITTLE_ENDIAN
elif "MM" in four_bytes[:2]:
assert struct.unpack(">H", four_bytes[2:])[0] == 42
return BIG_ENDIAN
raise RuntimeError("%s not recognised as TIFF" % filename)
def _start(self):
'''Open the image file, read the image header, copy it into memory
for future inspection.'''
Format._start(self)
self._cif_header = FormatCBF.get_cbf_header(self._image_file)
self._mime_header = ''
in_binary_format_section = False
for record in FormatCBF.open_file(self._image_file, 'rb'):
if '--CIF-BINARY-FORMAT-SECTION--' in record:
in_binary_format_section = True
elif in_binary_format_section and record[0] == 'X':
self._mime_header += record
if in_binary_format_section and len(record.strip()) == 0:
# http://sourceforge.net/apps/trac/cbflib/wiki/ARRAY_DATA%20Category
# In an imgCIF file, the encoded binary data begins after
# the empty line terminating the header.
break
return
def _start(self):
'''Open the image file, read the image header, copy it into memory
for future inspection.'''
Format._start(self)
self._cif_header = FormatCBF.get_cbf_header(self._image_file)
self._mime_header = ''
in_binary_format_section = False
for record in FormatCBF.open_file(self._image_file, 'rb'):
if '--CIF-BINARY-FORMAT-SECTION--' in record:
in_binary_format_section = True
elif in_binary_format_section and record[0] == 'X':
self._mime_header += record
if in_binary_format_section and len(record.strip()) == 0:
# http://sourceforge.net/apps/trac/cbflib/wiki/ARRAY_DATA%20Category
# In an imgCIF file, the encoded binary data begins after
# the empty line terminating the header.
break
return
def test_experimentlist_factory_from_imageset():
imageset = Format.get_imageset(["filename.cbf"], as_imageset=True)
imageset.set_beam(Beam(), 0)
imageset.set_detector(Detector(), 0)
crystal = Crystal((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol="P1")
experiments = ExperimentListFactory.from_imageset_and_crystal(
imageset, crystal)
assert len(experiments) == 1
assert experiments[0].imageset
assert experiments[0].beam
assert experiments[0].detector is not None
assert experiments[0].crystal
def get_image_file(self, index=None): return Format.get_image_file(self)
def get_scan(self, index=None):
if index == None:
return Format.get_scan(self)
else:
scan = Format.get_scan(self)
return scan[index]
def get_beam(self, index=None): return Format.get_beam(self)
def get_detector(self, index=None): return Format.get_detector(self)
def get_goniometer(self, index=None): return Format.get_goniometer(self)
def get_detectorbase(self, index=None): #self.detectorbase.img_number = index return Format.get_detectorbase(self)
def __init__(self, image_file, **kwargs):
'''Initialise the image structure from the given file.'''
assert(self.understand(image_file))
Format.__init__(self, image_file, **kwargs)
elif type_type == 2:
type_offset_or_value = struct.unpack(
_I, tiff_header[start:start + 4])[0]
start += 4
if type_desc == 270:
start = type_offset_or_value
end = type_offset_or_value + type_size
header_text = tiff_header[start:end].strip()
return header_text
if __name__ == '__main__':
import sys
for arg in sys.argv[1:]:
width, height, depth, header, order = read_basic_tiff_header(arg)
print '(%d x %d) @ %d + %d' % (width, height, depth, header)
tiff_header = Format.open_file(arg, 'rb').read(header)
text = read_tiff_image_description(tiff_header, order)
if text:
print text
else:
print 'No text found'
def __init__(self, image_file):
assert(self.understand(image_file))
Format.__init__(self, image_file)
return
def __init__(self, image_file): assert(self.understand(image_file)) FormatMultiImage.__init__(self) Format.__init__(self, image_file)
