def tst_from_imageset(self):
from dxtbx.imageset import ImageSet, NullReader
from dxtbx.model import Beam, Detector, Goniometer, Scan
from dxtbx.model.crystal import crystal_model
imageset = ImageSet(NullReader(["filename.cbf"]))
imageset.set_beam(Beam(), 0)
imageset.set_detector(Detector(), 0)
crystal = crystal_model(
(1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol=0)
experiments = ExperimentListFactory.from_imageset_and_crystal(
imageset, crystal)
assert(len(experiments) == 1)
assert(experiments[0].imageset is not None)
assert(experiments[0].beam is not None)
assert(experiments[0].detector is not None)
assert(experiments[0].crystal is not None)
print 'OK'
def test_cspad_cbf_in_memory(dials_regression, tmpdir):
tmpdir.chdir()
# Check the data files for this test exist
image_path = os.path.join(dials_regression, "image_examples",
"LCLS_cspad_nexus", 'idx-20130301060858801.cbf')
assert os.path.isfile(image_path)
with open("process_lcls.phil", 'w') as f:
f.write("""
dispatch.squash_errors = False
spotfinder {
filter.min_spot_size=2
threshold.dispersion.gain=25
threshold.dispersion.global_threshold=100
}
indexing {
known_symmetry {
space_group = P6122
unit_cell = 92.9 92.9 130.4 90 90 120
}
refinement_protocol.d_min_start=1.7
stills.refine_candidates_with_known_symmetry=True
}
""")
params = phil_scope.fetch(parse(file_name="process_lcls.phil")).extract()
params.output.datablock_filename = None
processor = Processor(params)
mem_img = dxtbx.load(image_path)
raw_data = mem_img.get_raw_data(
) # cache the raw data to prevent swig errors
mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle)
mem_img._raw_data = raw_data
mem_img._cbf_handle = None # drop the file handle to prevent swig errors
imgset = ImageSet(ImageSetData(MemReader([mem_img]), MemMasker([mem_img])))
imgset.set_beam(mem_img.get_beam())
imgset.set_detector(mem_img.get_detector())
datablock = DataBlockFactory.from_imageset(imgset)[0]
processor.process_datablock("20130301060858801",
datablock) # index/integrate the image
result = "idx-20130301060858801_integrated.pickle"
n_refls = range(
140, 152) # large ranges to handle platform-specific differences
with open(result, 'rb') as f:
table = pickle.load(f)
assert len(table) in n_refls, len(table)
assert 'id' in table
assert (table['id'] == 0).count(False) == 0
def do_import(filename):
logger.info("Loading %s"%os.path.basename(filename))
try:
datablocks = DataBlockFactory.from_json_file(filename)
except ValueError:
datablocks = DataBlockFactory.from_filenames([filename])
if len(datablocks) == 0:
raise Abort("Could not load %s"%filename)
if len(datablocks) > 1:
raise Abort("Got multiple datablocks from file %s"%filename)
# Ensure the indexer and downstream applications treat this as set of stills
from dxtbx.imageset import ImageSet
reset_sets = []
for imageset in datablocks[0].extract_imagesets():
imageset = ImageSet(imageset.reader(), imageset.indices())
imageset._models = imageset._models
imageset.set_scan(None)
imageset.set_goniometer(None)
reset_sets.append(imageset)
return DataBlockFactory.from_imageset(reset_sets)[0]
def test_cspad_cbf_in_memory(dials_regression, run_in_tmpdir,
composite_output):
# Check the data files for this test exist
image_path = os.path.join(
dials_regression,
"image_examples",
"LCLS_cspad_nexus",
"idx-20130301060858801.cbf",
)
assert os.path.isfile(image_path)
with open("process_lcls.phil", "w") as f:
f.write(cspad_cbf_in_memory_phil)
params = phil_scope.fetch(parse(file_name="process_lcls.phil")).extract()
params.output.experiments_filename = None
params.output.composite_output = composite_output
if composite_output:
processor = Processor(params, composite_tag="memtest")
else:
processor = Processor(params)
mem_img = dxtbx.load(image_path)
raw_data = mem_img.get_raw_data(
) # cache the raw data to prevent swig errors
mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle)
mem_img._raw_data = raw_data
mem_img._cbf_handle = None # drop the file handle to prevent swig errors
imgset = ImageSet(ImageSetData(MemReader([mem_img]), None))
imgset.set_beam(mem_img.get_beam())
imgset.set_detector(mem_img.get_detector())
experiments = ExperimentListFactory.from_imageset_and_crystal(imgset, None)
processor.process_experiments("20130301060858801",
experiments) # index/integrate the image
if composite_output:
processor.finalize()
result = "idx-memtest_integrated.refl"
else:
result = "idx-20130301060858801_integrated.refl"
n_refls = list(range(
140, 152)) # large ranges to handle platform-specific differences
table = flex.reflection_table.from_file(result)
assert len(table) in n_refls, len(table)
assert "id" in table
assert (table["id"] == 0).count(False) == 0
def datablock_from_numpyarrays(image, detector, beam, mask=None):
"""
So that one can do e.g.
>> dblock = datablock_from_numpyarrays( image, detector, beam)
>> refl = flex.reflection_table.from_observations(dblock, spot_finder_params)
without having to utilize the harddisk
:param image: numpy array image
:param mask: numpy mask
:param detector: dxtbx detector model
:param beam: dxtbx beam model
:return: datablock for the image
"""
I = FormatInMemory(image=image, mask=mask)
reader = MemReader([I])
masker = MemMasker([I])
iset_Data = ImageSetData(reader, masker)
iset = ImageSet(iset_Data)
iset.set_beam(beam)
iset.set_detector(detector)
dblock = DataBlockFactory.from_imageset([iset])[0]
return dblock
def tst_from_imageset(self):
from dxtbx.imageset import ImageSet, NullReader
from dxtbx.model import Beam, Detector, Goniometer, Scan
from dxtbx.model import Crystal
imageset = ImageSet(NullReader(["filename.cbf"]))
imageset.set_beam(Beam(), 0)
imageset.set_detector(Detector(), 0)
crystal = Crystal((1, 0, 0), (0, 1, 0), (0, 0, 1),
space_group_symbol="P1")
experiments = ExperimentListFactory.from_imageset_and_crystal(
imageset, crystal)
assert (len(experiments) == 1)
assert (experiments[0].imageset is not None)
assert (experiments[0].beam is not None)
assert (experiments[0].detector is not None)
assert (experiments[0].crystal is not None)
print 'OK'
def datablock_from_numpyarrays(image, detector, beam, mask=None):
"""
put the numpy array image(s) into a dials datablock
:param image: numpy array of images or an image
:param detector: dxtbx det
:param beam: dxtbx beam
:param mask: mask same shape as image , 1 is not masked, boolean
:return:
"""
if isinstance( image, list):
image = np.array( image)
if mask is not None:
if isinstance( mask, list):
mask = np.array(mask).astype(bool)
I = FormatInMemory(image=image, mask=mask)
reader = MemReader([I])
masker = MemMasker([I])
iset_Data = ImageSetData(reader, masker)
iset = ImageSet(iset_Data)
iset.set_beam(beam)
iset.set_detector(detector)
dblock = DataBlockFactory.from_imageset([iset])[0]
return dblock
def get_imageset(
Class,
input_filenames,
beam=None,
detector=None,
goniometer=None,
scan=None,
as_imageset=False,
as_sequence=False,
single_file_indices=None,
format_kwargs=None,
template=None,
check_format=True,
):
"""
Factory method to create an imageset
"""
# Import here to avoid cyclic imports
from dxtbx.imageset import ImageSequence, ImageSet, ImageSetData
# Get filename absolute paths, for entries that are filenames
filenames = [
os.path.abspath(x) if not get_url_scheme(x) else x
for x in input_filenames
]
# Make it a dict
if format_kwargs is None:
format_kwargs = {}
# Get some information from the format class
reader = Class.get_reader()(filenames, **format_kwargs)
# Get the format instance
if check_format is True:
Class._current_filename_ = None
Class._current_instance_ = None
format_instance = Class.get_instance(filenames[0], **format_kwargs)
else:
format_instance = None
# Read the vendor type
if check_format is True:
vendor = format_instance.get_vendortype()
else:
vendor = ""
# Get the format kwargs
params = format_kwargs
# Make sure only 1 or none is set
assert [as_imageset, as_sequence].count(True) < 2
if as_imageset:
is_sequence = False
elif as_sequence:
is_sequence = True
else:
if scan is None and format_instance is None:
raise RuntimeError("""
One of the following needs to be set
- as_imageset=True
- as_sequence=True
- scan
- check_format=True
""")
if scan is None:
test_scan = format_instance.get_scan()
else:
test_scan = scan
if test_scan is not None and test_scan.get_oscillation()[1] != 0:
is_sequence = True
else:
is_sequence = False
# Create an imageset or sequence
if not is_sequence:
# Create the imageset
iset = ImageSet(
ImageSetData(
reader=reader,
masker=None,
vendor=vendor,
params=params,
format=Class,
))
# If any are None then read from format
if [beam, detector, goniometer, scan].count(None) != 0:
# Get list of models
beam = []
detector = []
goniometer = []
scan = []
for f in filenames:
format_instance = Class(f, **format_kwargs)
beam.append(format_instance.get_beam())
detector.append(format_instance.get_detector())
goniometer.append(format_instance.get_goniometer())
scan.append(format_instance.get_scan())
# Set the list of models
for i in range(len(filenames)):
iset.set_beam(beam[i], i)
iset.set_detector(detector[i], i)
iset.set_goniometer(goniometer[i], i)
iset.set_scan(scan[i], i)
else:
# Get the template
if template is None:
template = template_regex(filenames[0])[0]
else:
template = str(template)
# Check scan makes sense
if scan:
if check_format is True:
assert scan.get_num_images() == len(filenames)
# If any are None then read from format
if beam is None and format_instance is not None:
beam = format_instance.get_beam()
if detector is None and format_instance is not None:
detector = format_instance.get_detector()
if goniometer is None and format_instance is not None:
goniometer = format_instance.get_goniometer()
if scan is None and format_instance is not None:
scan = format_instance.get_scan()
if scan is not None:
for f in filenames[1:]:
format_instance = Class(f, **format_kwargs)
scan += format_instance.get_scan()
assert beam is not None, "Can't create Sequence without beam"
assert detector is not None, "Can't create Sequence without detector"
assert goniometer is not None, "Can't create Sequence without goniometer"
assert scan is not None, "Can't create Sequence without scan"
# Create the masker
if format_instance is not None:
masker = format_instance.get_masker(goniometer=goniometer)
else:
masker = None
# Create the sequence
iset = ImageSequence(
ImageSetData(
reader=reader,
masker=masker,
vendor=vendor,
params=params,
format=Class,
template=template,
),
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
)
if format_instance is not None:
static_mask = format_instance.get_static_mask()
if static_mask is not None:
if not iset.external_lookup.mask.data.empty():
for m1, m2 in zip(static_mask,
iset.external_lookup.mask.data):
m1 &= m2.data()
iset.external_lookup.mask.data = ImageBool(static_mask)
else:
iset.external_lookup.mask.data = ImageBool(static_mask)
return iset
def get_imageset(
Class,
filenames,
beam=None,
detector=None,
goniometer=None,
scan=None,
as_imageset=False,
as_sweep=False,
single_file_indices=None,
format_kwargs=None,
template=None,
check_format=True,
):
"""
Factory method to create an imageset
"""
from dxtbx.imageset import ImageSetData
from dxtbx.imageset import ImageSet
from dxtbx.imageset import ImageSweep
from dxtbx.sweep_filenames import template_regex
from os.path import abspath
# Get filename absolute paths
filenames = map(abspath, filenames)
# Make it a dict
if format_kwargs is None:
format_kwargs = {}
# Get some information from the format class
reader = Class.get_reader()(filenames, **format_kwargs)
masker = Class.get_masker()(filenames, **format_kwargs)
# Get the format instance
if check_format is True:
format_instance = Class(filenames[0], **format_kwargs)
else:
format_instance = None
# Read the vendor type
if check_format is True:
vendor = format_instance.get_vendortype()
else:
vendor = ""
# Get the format kwargs
params = format_kwargs
# Make sure only 1 or none is set
assert [as_imageset, as_sweep].count(True) < 2
if as_imageset:
is_sweep = False
elif as_sweep:
is_sweep = True
else:
if scan is None and format_instance is None:
raise RuntimeError("""
One of the following needs to be set
- as_imageset=True
- as_sweep=True
- scan
- check_format=True
""")
if scan is None:
test_scan = format_instance.get_scan()
else:
test_scan = scan
if test_scan is not None and test_scan.get_oscillation()[1] != 0:
is_sweep = True
else:
is_sweep = False
# Create an imageset or sweep
if not is_sweep:
# Create the imageset
iset = ImageSet(
ImageSetData(
reader=reader,
masker=masker,
vendor=vendor,
params=params,
format=Class,
))
# If any are None then read from format
if [beam, detector, goniometer, scan].count(None) != 0:
# Get list of models
beam = []
detector = []
goniometer = []
scan = []
for f in filenames:
format_instance = Class(f, **format_kwargs)
beam.append(format_instance.get_beam())
detector.append(format_instance.get_detector())
goniometer.append(format_instance.get_goniometer())
scan.append(format_instance.get_scan())
# Set the list of models
for i in range(len(filenames)):
iset.set_beam(beam[i], i)
iset.set_detector(detector[i], i)
iset.set_goniometer(goniometer[i], i)
iset.set_scan(scan[i], i)
else:
# Get the template
if template is None:
template = template_regex(filenames[0])[0]
else:
template = str(template)
# Check scan makes sense
if scan:
if check_format is True:
assert scan.get_num_images() == len(filenames)
# If any are None then read from format
if beam is None and format_instance is not None:
beam = format_instance.get_beam()
if detector is None and format_instance is not None:
detector = format_instance.get_detector()
if goniometer is None and format_instance is not None:
goniometer = format_instance.get_goniometer()
if scan is None and format_instance is not None:
scan = format_instance.get_scan()
if scan is not None:
for f in filenames[1:]:
format_instance = Class(f, **format_kwargs)
scan += format_instance.get_scan()
assert beam is not None, "Can't create Sweep without beam"
assert detector is not None, "Can't create Sweep without detector"
assert goniometer is not None, "Can't create Sweep without goniometer"
assert scan is not None, "Can't create Sweep without scan"
# Create the sweep
iset = ImageSweep(
ImageSetData(
reader=reader,
masker=masker,
vendor=vendor,
params=params,
format=Class,
template=template,
),
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
)
# Return the imageset
return iset
def get_imageset(
Class,
filenames,
beam=None,
detector=None,
goniometer=None,
scan=None,
as_sweep=False,
as_imageset=False,
single_file_indices=None,
format_kwargs=None,
template=None,
check_format=True,
lazy=False,
):
"""
Factory method to create an imageset
"""
from dxtbx.imageset import ImageSetData
from dxtbx.imageset import ImageSweep
from os.path import abspath
from scitbx.array_family import flex
if isinstance(filenames, str):
filenames = [filenames]
elif len(filenames) > 1:
assert len(set(filenames)) == 1
filenames = filenames[0:1]
# Make filenames absolute
filenames = map(abspath, filenames)
# Make it a dictionary
if format_kwargs is None:
format_kwargs = {}
# If get_num_images hasn't been implemented, we need indices for number of images
if Class.get_num_images == FormatMultiImage.get_num_images:
assert single_file_indices is not None
assert min(single_file_indices) >= 0
num_images = max(single_file_indices) + 1
else:
num_images = None
# Get some information from the format class
reader = Class.get_reader()(filenames, num_images=num_images, **format_kwargs)
masker = Class.get_masker()(filenames, num_images=num_images, **format_kwargs)
# Get the format instance
assert len(filenames) == 1
if check_format is True:
format_instance = Class(filenames[0], **format_kwargs)
else:
format_instance = None
if not as_sweep:
lazy = True
# Read the vendor type
if check_format is True:
vendor = format_instance.get_vendortype()
else:
vendor = ""
# Get the format kwargs
params = format_kwargs
# Check if we have a sweep
# Make sure only 1 or none is set
assert [as_imageset, as_sweep].count(True) < 2
if as_imageset:
is_sweep = False
elif as_sweep:
is_sweep = True
else:
if scan is None and format_instance is None:
raise RuntimeError(
"""
One of the following needs to be set
- as_imageset=True
- as_sweep=True
- scan
- check_format=True
"""
)
if scan is None:
test_scan = format_instance.get_scan()
else:
test_scan = scan
if test_scan is not None and test_scan.get_oscillation()[1] != 0:
is_sweep = True
else:
is_sweep = False
assert not (as_sweep and lazy), "No lazy support for sweeps"
if single_file_indices is not None:
single_file_indices = flex.size_t(single_file_indices)
# Create an imageset or sweep
if not is_sweep:
# Use imagesetlazy
# Setup ImageSetLazy and just return it. No models are set.
if lazy:
from dxtbx.imageset import ImageSetLazy
iset = ImageSetLazy(
ImageSetData(
reader=reader,
masker=masker,
vendor=vendor,
params=params,
format=Class,
),
indices=single_file_indices,
)
return iset
# Create the imageset
from dxtbx.imageset import ImageSet
iset = ImageSet(
ImageSetData(
reader=reader,
masker=masker,
vendor=vendor,
params=params,
format=Class,
),
indices=single_file_indices,
)
# If any are None then read from format
if [beam, detector, goniometer, scan].count(None) != 0:
# Get list of models
beam = []
detector = []
goniometer = []
scan = []
for i in range(format_instance.get_num_images()):
beam.append(format_instance.get_beam(i))
detector.append(format_instance.get_detector(i))
goniometer.append(format_instance.get_goniometer(i))
scan.append(format_instance.get_scan(i))
if single_file_indices is None:
single_file_indices = list(range(format_instance.get_num_images()))
# Set the list of models
for i in range(len(single_file_indices)):
iset.set_beam(beam[single_file_indices[i]], i)
iset.set_detector(detector[single_file_indices[i]], i)
iset.set_goniometer(goniometer[single_file_indices[i]], i)
iset.set_scan(scan[single_file_indices[i]], i)
else:
# Get the template
template = filenames[0]
# Check indices are sequential
if single_file_indices is not None:
assert all(
i + 1 == j
for i, j in zip(single_file_indices[:-1], single_file_indices[1:])
)
num_images = len(single_file_indices)
else:
num_images = format_instance.get_num_images()
# Check the scan makes sense - we must want to use <= total images
if scan is not None:
assert scan.get_num_images() <= num_images
# If any are None then read from format
if beam is None:
beam = format_instance.get_beam()
if detector is None:
detector = format_instance.get_detector()
if goniometer is None:
goniometer = format_instance.get_goniometer()
if scan is None:
scan = format_instance.get_scan()
if scan is not None:
for f in filenames[1:]:
format_instance = Class(f, **format_kwargs)
scan += format_instance.get_scan()
isetdata = ImageSetData(
reader=reader,
masker=masker,
vendor=vendor,
params=params,
format=Class,
template=template,
)
# Create the sweep
iset = ImageSweep(
isetdata,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
indices=single_file_indices,
)
# Return the imageset
return iset
def from_parameters(reflections,
experiments,
known_crystal_models=None,
params=None):
'''Sets up indexer object that will be used for indexing '''
# if params is None:
# params = master_params
if known_crystal_models is not None:
#from dials.algorithms.indexing.known_orientation \
# import indexer_known_orientation
#idxr = iota_indexer_known_orientation(
# reflections, experiments, params, known_crystal_models)
idxr = IOTA_StillsIndexerKnownOrientation(reflections, experiments,
params,
known_crystal_models)
#idxr = indexer_known_orientation(
# reflections, imagesets, params, known_crystal_models)
else:
has_stills = False
has_sweeps = False
for expt in experiments:
if expt.goniometer is None or expt.scan is None or \
expt.scan.get_oscillation()[1] == 0:
if has_sweeps:
raise Sorry(
"Please provide only stills or only sweeps, not both"
)
has_stills = True
else:
if has_stills:
raise Sorry(
"Please provide only stills or only sweeps, not both"
)
has_sweeps = True
assert not (has_stills and has_sweeps)
use_stills_indexer = has_stills
if not (params.indexing.stills.indexer is libtbx.Auto
or params.indexing.stills.indexer.lower() == 'auto'):
if params.indexing.stills.indexer == 'stills':
use_stills_indexer = True
elif params.indexing.stills.indexer == 'sweeps':
use_stills_indexer = False
else:
assert False
if params.indexing.basis_vector_combinations.max_refine is libtbx.Auto:
params.indexing.basis_vector_combinations.max_refine = 5
# Ensure the indexer and downstream applications treat this as set of stills
from dxtbx.imageset import ImageSet
# DIALS 2.0 stuff here
for experiment in experiments:
experiment.imageset = ImageSet(experiment.imageset.data(),
experiment.imageset.indices())
experiment.imageset.set_scan(None)
experiment.imageset.set_goniometer(None)
experiment.scan = None
experiment.goniometer = None
# Old code prior to DIALS 2.0
#reset_sets = []
#for i in range(len(imagesets)):
# imagesweep = imagesets.pop(0)
# imageset = ImageSet(imagesweep.data(), imagesweep.indices())
# imageset.set_scan(None)
# imageset.set_goniometer(None)
# reset_sets.append(imageset)
#imagesets.extend(reset_sets)
if known_crystal_models is not None:
print('Known Orientation Indexing')
return idxr
#from dials.algorithms.indexing.known_orientation \
# import indexer_known_orientation
#idxr = indexer_known_orientation(
# reflections, experiments, params, known_crystal_models)
for entry_point in pkg_resources.iter_entry_points(
"dials.index.basis_vector_search"):
if params.indexing.method == entry_point.name:
idxr = IOTA_StillsIndexerBasisVectorSearch(reflections,
experiments,
params=params)
return idxr
def from_parameters(reflections,
experiments,
known_crystal_models=None,
params=None):
if known_crystal_models is not None:
from dials.algorithms.indexing.known_orientation import (
IndexerKnownOrientation, )
if params.indexing.known_symmetry.space_group is None:
params.indexing.known_symmetry.space_group = (
known_crystal_models[0].get_space_group().info())
idxr = IndexerKnownOrientation(reflections, experiments, params,
known_crystal_models)
else:
has_stills = False
has_sequences = False
for expt in experiments:
if isinstance(expt.imageset, ImageSequence):
has_sequences = True
else:
has_stills = True
if has_stills and has_sequences:
raise ValueError(
"Please provide only stills or only sequences, not both")
use_stills_indexer = has_stills
if not (params.indexing.stills.indexer is libtbx.Auto
or params.indexing.stills.indexer.lower() == "auto"):
if params.indexing.stills.indexer == "stills":
use_stills_indexer = True
elif params.indexing.stills.indexer == "sequences":
use_stills_indexer = False
else:
assert False
if params.indexing.basis_vector_combinations.max_refine is libtbx.Auto:
if use_stills_indexer:
params.indexing.basis_vector_combinations.max_refine = 5
else:
params.indexing.basis_vector_combinations.max_refine = 50
if use_stills_indexer:
# Ensure the indexer and downstream applications treat this as set of stills
from dxtbx.imageset import ImageSet # , MemImageSet
for experiment in experiments:
experiment.imageset = ImageSet(
experiment.imageset.data(),
experiment.imageset.indices())
# if isinstance(imageset, MemImageSet):
# imageset = MemImageSet(imagesequence._images, imagesequence.indices())
# else:
# imageset = ImageSet(imagesequence.reader(), imagesequence.indices())
# imageset._models = imagesequence._models
experiment.imageset.set_scan(None)
experiment.imageset.set_goniometer(None)
experiment.scan = None
experiment.goniometer = None
IndexerType = None
for entry_point in pkg_resources.iter_entry_points(
"dials.index.basis_vector_search"):
if params.indexing.method == entry_point.name:
if use_stills_indexer:
# do something
from dials.algorithms.indexing.stills_indexer import (
StillsIndexerBasisVectorSearch as IndexerType, )
else:
from dials.algorithms.indexing.lattice_search import (
BasisVectorSearch as IndexerType, )
if IndexerType is None:
for entry_point in pkg_resources.iter_entry_points(
"dials.index.lattice_search"):
if params.indexing.method == entry_point.name:
if use_stills_indexer:
from dials.algorithms.indexing.stills_indexer import (
StillsIndexerLatticeSearch as IndexerType, )
else:
from dials.algorithms.indexing.lattice_search import (
LatticeSearch as IndexerType, )
assert IndexerType is not None
idxr = IndexerType(reflections, experiments, params=params)
return idxr
def from_parameters(reflections,
experiments,
known_crystal_models=None,
params=None):
if known_crystal_models is not None:
from dials.algorithms.indexing.known_orientation import (
IndexerKnownOrientation, )
if params.indexing.known_symmetry.space_group is None:
params.indexing.known_symmetry.space_group = (
known_crystal_models[0].get_space_group().info())
idxr = IndexerKnownOrientation(reflections, experiments, params,
known_crystal_models)
else:
has_stills = False
has_sweeps = False
for expt in experiments:
if (expt.goniometer is None or expt.scan is None
or expt.scan.get_oscillation()[1] == 0):
if has_sweeps:
raise Sorry(
"Please provide only stills or only sweeps, not both"
)
has_stills = True
else:
if has_stills:
raise Sorry(
"Please provide only stills or only sweeps, not both"
)
has_sweeps = True
assert not (has_stills and has_sweeps)
use_stills_indexer = has_stills
if not (params.indexing.stills.indexer is libtbx.Auto
or params.indexing.stills.indexer.lower() == "auto"):
if params.indexing.stills.indexer == "stills":
use_stills_indexer = True
elif params.indexing.stills.indexer == "sweeps":
use_stills_indexer = False
else:
assert False
if params.indexing.basis_vector_combinations.max_refine is libtbx.Auto:
if use_stills_indexer:
params.indexing.basis_vector_combinations.max_refine = 5
else:
params.indexing.basis_vector_combinations.max_refine = 50
if use_stills_indexer:
# Ensure the indexer and downstream applications treat this as set of stills
from dxtbx.imageset import ImageSet # , MemImageSet
for experiment in experiments:
experiment.imageset = ImageSet(
experiment.imageset.data(),
experiment.imageset.indices())
# if isinstance(imageset, MemImageSet):
# imageset = MemImageSet(imagesweep._images, imagesweep.indices())
# else:
# imageset = ImageSet(imagesweep.reader(), imagesweep.indices())
# imageset._models = imagesweep._models
experiment.imageset.set_scan(None)
experiment.imageset.set_goniometer(None)
experiment.scan = None
experiment.goniometer = None
if params.indexing.method in ("fft1d", "fft3d",
"real_space_grid_search"):
if use_stills_indexer:
# do something
from dials.algorithms.indexing.stills_indexer import (
StillsIndexerBasisVectorSearch as BasisVectorSearch, )
else:
from dials.algorithms.indexing.lattice_search import (
BasisVectorSearch, )
idxr = BasisVectorSearch(reflections,
experiments,
params=params)
return idxr
def test_elliptical_distortion(run_in_tmpdir):
"""Create distortion maps for elliptical distortion using a dummy experiments
with a small detector, for speed. Check those maps seem sensible"""
# Make a detector model
d = make_detector()
# The beam is also essential for a experiments to be serialisable
b = Beam((0, 0, 1), 1.0)
# Create and write out a experiments
imageset = ImageSet(ImageSetData(Reader(None, ["non-existent.cbf"]), None))
imageset.set_detector(d)
imageset.set_beam(b)
experiments = ExperimentListFactory.from_imageset_and_crystal(
imageset, None)
experiments.as_json("dummy.expt")
# Centre of distortion will be the far corner from the origin of the first
# panel
centre_xy = d[0].get_image_size_mm()
# Generate distortion maps
cmd = ("dials.generate_distortion_maps dummy.expt "
"mode=ellipse centre_xy={},{} "
"phi=0 l1=1.0 l2=0.95").format(*centre_xy)
easy_run.fully_buffered(command=cmd).raise_if_errors()
# Load the maps
with open("dx.pickle", "rb") as f:
dx = pickle.load(f)
with open("dy.pickle", "rb") as f:
dy = pickle.load(f)
# Check there are 4 maps each
assert len(dx) == len(dy) == 4
# Ellipse has phi=0, so all correction is in the dy map
for arr in dx:
assert min(arr) == max(arr) == 0.0
# The ellipse correction is centred at the middle of the detector and all in
# the Y direction. Therefore we expect a few things from the dy maps:
#
# (1) Within each panel the columns of the array are identical.
# (2) The two upper panels should be the same
# (3) The two lower panels should be the same.
# (4) One column from an upper panel is a negated, reversed column from a
# lower panel.
#
# All together expect the 4 dy maps to look something like this:
#
# /-----------\ /-----------\
# |-3 -3 -3 -3| |-3 -3 -3 -3|
# |-2 -2 -2 -2| |-2 -2 -2 -2|
# |-1 -1 -1 -1| |-1 -1 -1 -1|
# | 0 0 0 0| | 0 0 0 0|
# \-----------/ \-----------/
# /-----------\ /-----------\
# | 0 0 0 0| | 0 0 0 0|
# | 1 1 1 1| | 1 1 1 1|
# | 2 2 2 2| | 2 2 2 2|
# | 3 3 3 3| | 3 3 3 3|
# \-----------/ \-----------/
# So the fundamental data is all in the first column of first panel's map
col0 = dy[0].matrix_copy_column(0)
# The correction should be 5% of the distance from the ellipse centre to a
# corrected pixel (l2 = 0.95 above) along the slow axis. Check that is the
# case (for the first pixel at least)
vec_centre_to_first_px = matrix.col(d[0].get_pixel_lab_coord(
(0.5, 0.5))) - matrix.col(d[0].get_lab_coord(centre_xy))
dist_centre_to_first_px = vec_centre_to_first_px.dot(
matrix.col(d[0].get_slow_axis()))
corr_mm = dist_centre_to_first_px * 0.05
corr_px = corr_mm / d[0].get_pixel_size()[1]
assert col0[0] == pytest.approx(corr_px)
# Test (1) from above list for panel 0
for i in range(1, 50):
assert (col0 == dy[0].matrix_copy_column(i)).all_eq(True)
# Test (2)
assert (dy[0] == dy[1]).all_eq(True)
# Test (3)
assert (dy[2] == dy[3]).all_eq(True)
# Test (4)
assert col0 == pytest.approx(-1.0 * dy[2].matrix_copy_column(0).reversed())
# Test (1) for panel 2 as well, which then covers everything needed
col0 = dy[2].matrix_copy_column(0)
for i in range(1, 50):
assert (col0 == dy[2].matrix_copy_column(i)).all_eq(True)
def test_cspad_cbf_in_memory(self):
from os.path import join, exists
import os, dxtbx
from uuid import uuid4
from dials.command_line.stills_process import phil_scope, Processor
from libtbx.phil import parse
from dxtbx.imageset import ImageSet, ImageSetData, MemReader, MemMasker
from dxtbx.datablock import DataBlockFactory
from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory
import cPickle as pickle
dirname = 'tmp_%s' % uuid4().hex
os.mkdir(dirname)
os.chdir(dirname)
assert exists(join(self.lcls_path, 'idx-20130301060858801.cbf'))
f = open("process_lcls.phil", 'w')
f.write("""
dispatch.squash_errors = False
spotfinder {
filter.min_spot_size=2
threshold.dispersion.gain=25
threshold.dispersion.global_threshold=100
}
indexing {
known_symmetry {
space_group = P6122
unit_cell = 92.9 92.9 130.4 90 90 120
}
refinement_protocol.d_min_start=1.7
stills.refine_candidates_with_known_symmetry=True
}
""")
f.close()
params = phil_scope.fetch(
parse(file_name="process_lcls.phil")).extract()
params.output.datablock_filename = None
processor = Processor(params)
mem_img = dxtbx.load(join(self.lcls_path, 'idx-20130301060858801.cbf'))
raw_data = mem_img.get_raw_data(
) # cache the raw data to prevent swig errors
mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle)
mem_img._raw_data = raw_data
mem_img._cbf_handle = None # drop the file handle to prevent swig errors
imgset = ImageSet(
ImageSetData(MemReader([mem_img]), MemMasker([mem_img])))
imgset.set_beam(mem_img.get_beam())
imgset.set_detector(mem_img.get_detector())
datablock = DataBlockFactory.from_imageset(imgset)[0]
processor.process_datablock("20130301060858801",
datablock) # index/integrate the image
result = "idx-20130301060858801_integrated.pickle"
#n_refls = range(140,152) # large ranges to handle platform-specific differences
# 09/20/17 Changes to still indexer: refine candidate basis vectors in target symmetry if supplied
#n_refls = range(128,140) # large ranges to handle platform-specific differences
# 09/27/17 Bugfix for refine_candidates_with_known_symmetry
n_refls = range(
140, 152) # large ranges to handle platform-specific differences
table = pickle.load(open(result, 'rb'))
assert len(table) in n_refls, len(table)
assert 'id' in table
assert (table['id'] == 0).count(False) == 0
print 'OK'
def __call__(self, imageset):
'''
Override the parameters
'''
from dxtbx.imageset import ImageSet
from dxtbx.imageset import ImageSweep
from dxtbx.model import BeamFactory
from dxtbx.model import DetectorFactory
from dxtbx.model import GoniometerFactory
from dxtbx.model import ScanFactory
from copy import deepcopy
if self.params.geometry.convert_sweeps_to_stills:
imageset = ImageSet(data=imageset.data())
if not isinstance(imageset, ImageSweep):
if self.params.geometry.convert_stills_to_sweeps:
imageset = self.convert_stills_to_sweep(imageset)
if isinstance(imageset, ImageSweep):
beam = BeamFactory.from_phil(
self.params.geometry,
imageset.get_beam())
detector = DetectorFactory.from_phil(
self.params.geometry,
imageset.get_detector(),
beam)
goniometer = GoniometerFactory.from_phil(
self.params.geometry,
imageset.get_goniometer())
scan = ScanFactory.from_phil(
self.params.geometry,
deepcopy(imageset.get_scan()))
i0, i1 = scan.get_array_range()
j0, j1 = imageset.get_scan().get_array_range()
if i0 < j0 or i1 > j1:
imageset = self.extrapolate_imageset(
imageset = imageset,
beam = beam,
detector = detector,
goniometer = goniometer,
scan = scan)
else:
imageset.set_beam(beam)
imageset.set_detector(detector)
imageset.set_goniometer(goniometer)
imageset.set_scan(scan)
else:
for i in range(len(imageset)):
beam = BeamFactory.from_phil(
self.params.geometry,
imageset.get_beam(i))
detector = DetectorFactory.from_phil(
self.params.geometry,
imageset.get_detector(i),
beam)
goniometer = GoniometerFactory.from_phil(
self.params.geometry,
imageset.get_goniometer(i))
scan = ScanFactory.from_phil(
self.params.geometry,
imageset.get_scan(i))
imageset.set_beam(beam, i)
imageset.set_detector(detector, i)
imageset.set_goniometer(goniometer, i)
imageset.set_scan(scan, i)
return imageset
class CctbxPsanaEventProcessor(Processor):
""" Processor class for psana events """
def __init__(self, params_filename, output_tag, logfile=None):
"""
@param params_filename cctbx.xfel/DIALS parameter file for processing
@output_tag String that will prefix output files
@logfile File name for logging
"""
self.parsed_params = parse(file_name=params_filename)
dials_params = phil_scope.fetch(self.parsed_params).extract()
super(CctbxPsanaEventProcessor, self).__init__(dials_params,
output_tag)
self.update_geometry = ManualGeometryUpdater(dials_params)
simple_script = SimpleScript(dials_params)
simple_script.load_reference_geometry()
self.reference_detector = getattr(simple_script, 'reference_detector',
None)
self.output_tag = output_tag
self.detector_params = None
if logfile is not None:
log.config(logfile=logfile)
def setup_run(self, run, psana_detector):
""" Initialize processing for a given run
@param run psana Run object
@param psana_detector psana Detector object
"""
if psana_detector.is_cspad():
format_class = FormatXTCCspadSingleEvent
detector_scope = cspad_locator_scope
elif psana_detector.is_epix10ka2m():
format_class = FormatXTCEpixSingleEvent
detector_scope = epix_locator_scope
elif psana_detector.is_jungfrau():
format_class = FormatXTCJungfrauSingleEvent
detector_scope = jungfrau_locator_scope
elif 'rayonix' in psana_detector.name.dev.lower():
format_class = FormatXTCRayonixSingleEvent
detector_scope = rayonix_locator_scope
else:
raise RuntimeError('Unrecognized detector %s' %
psana_detector.name)
detector_params = detector_scope.fetch(self.parsed_params).extract()
self.dxtbx_img = format_class(detector_params, run, psana_detector)
self.imageset = ImageSet(
ImageSetData(MemReader([self.dxtbx_img]), None))
def process_event(self, event, event_tag):
""" Process a single psana event
@param event psana Event object
@param event_tag string identifying the event
"""
experiments = self.experiments_from_event(event)
self.process_experiments('%s_%s' % (self.output_tag, event_tag),
experiments)
def experiments_from_event(self, event):
""" Create an ExperimentList from a psana Event
@param event psana Event object
"""
self.dxtbx_img.event = event
self.imageset.set_beam(self.dxtbx_img.get_beam())
self.imageset.set_detector(self.dxtbx_img.get_detector())
self.update_geometry(self.imageset)
experiments = ExperimentListFactory.from_imageset_and_crystal(
self.imageset, None)
if self.reference_detector is not None:
experiment = experiments[0]
sync_geometry(self.reference_detector.hierarchy(),
self.imageset.get_detector().hierarchy())
experiment.detector = self.imageset.get_detector()
return experiments
def get_imageset(
cls,
filenames,
beam=None,
detector=None,
goniometer=None,
scan=None,
as_sequence=False,
as_imageset=False,
single_file_indices=None,
format_kwargs=None,
template=None,
check_format=True,
lazy=False,
):
"""
Factory method to create an imageset
"""
if isinstance(filenames, str):
filenames = [filenames]
elif len(filenames) > 1:
assert len(set(filenames)) == 1
filenames = filenames[0:1]
# Make filenames absolute
filenames = [os.path.abspath(x) for x in filenames]
# Make it a dictionary
if format_kwargs is None:
format_kwargs = {}
# If get_num_images hasn't been implemented, we need indices for number of images
if cls.get_num_images == FormatMultiImage.get_num_images:
assert single_file_indices is not None
assert min(single_file_indices) >= 0
num_images = max(single_file_indices) + 1
else:
num_images = None
# Get the format instance
assert len(filenames) == 1
if check_format is True:
format_instance = cls(filenames[0], **format_kwargs)
if num_images is None and not lazy:
# As we now have the actual format class we can get the number
# of images from here. This saves having to create another
# format class instance in the Reader() constructor
# NOTE: Having this information breaks internal assumptions in
# *Lazy classes, so they have to figure this out in
# their own time.
num_images = format_instance.get_num_images()
else:
format_instance = None
if not as_sequence:
lazy = True
# Get some information from the format class
reader = cls.get_reader()(filenames, num_images=num_images, **format_kwargs)
# Read the vendor type
if check_format is True:
vendor = format_instance.get_vendortype()
else:
vendor = ""
# Get the format kwargs
params = format_kwargs
# Check if we have a sequence
# Make sure only 1 or none is set
assert [as_imageset, as_sequence].count(True) < 2
if as_imageset:
is_sequence = False
elif as_sequence:
is_sequence = True
else:
if scan is None and format_instance is None:
raise RuntimeError(
"""
One of the following needs to be set
- as_imageset=True
- as_sequence=True
- scan
- check_format=True
"""
)
if scan is None:
test_scan = format_instance.get_scan()
else:
test_scan = scan
if test_scan is not None:
is_sequence = True
else:
is_sequence = False
assert not (as_sequence and lazy), "No lazy support for sequences"
if single_file_indices is not None:
single_file_indices = flex.size_t(single_file_indices)
# Create an imageset or sequence
if not is_sequence:
# Use imagesetlazy
# Setup ImageSetLazy and just return it. No models are set.
if lazy:
iset = ImageSetLazy(
ImageSetData(
reader=reader,
masker=None,
vendor=vendor,
params=params,
format=cls,
),
indices=single_file_indices,
)
return iset
# Create the imageset
iset = ImageSet(
ImageSetData(
reader=reader, masker=None, vendor=vendor, params=params, format=cls
),
indices=single_file_indices,
)
# If any are None then read from format
if [beam, detector, goniometer, scan].count(None) != 0:
# Get list of models
beam = []
detector = []
goniometer = []
scan = []
for i in range(format_instance.get_num_images()):
beam.append(format_instance.get_beam(i))
detector.append(format_instance.get_detector(i))
goniometer.append(format_instance.get_goniometer(i))
scan.append(format_instance.get_scan(i))
if single_file_indices is None:
single_file_indices = list(range(format_instance.get_num_images()))
# Set the list of models
for i in range(len(single_file_indices)):
iset.set_beam(beam[single_file_indices[i]], i)
iset.set_detector(detector[single_file_indices[i]], i)
iset.set_goniometer(goniometer[single_file_indices[i]], i)
iset.set_scan(scan[single_file_indices[i]], i)
else:
# Get the template
template = filenames[0]
# Check indices are sequential
if single_file_indices is not None:
assert all(
i + 1 == j
for i, j in zip(single_file_indices[:-1], single_file_indices[1:])
)
num_images = len(single_file_indices)
else:
num_images = format_instance.get_num_images()
# Check the scan makes sense - we must want to use <= total images
if scan is not None:
assert scan.get_num_images() <= num_images
# If any are None then read from format
if beam is None:
beam = format_instance.get_beam()
if detector is None:
detector = format_instance.get_detector()
if goniometer is None:
goniometer = format_instance.get_goniometer()
if scan is None:
scan = format_instance.get_scan()
if scan is not None:
for f in filenames[1:]:
format_instance = cls(f, **format_kwargs)
scan += format_instance.get_scan()
# Create the masker
if format_instance is not None:
masker = format_instance.get_masker(goniometer=goniometer)
else:
masker = None
isetdata = ImageSetData(
reader=reader,
masker=masker,
vendor=vendor,
params=params,
format=cls,
template=template,
)
# Create the sequence
iset = ImageSequence(
isetdata,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
indices=single_file_indices,
)
if format_instance is not None:
static_mask = format_instance.get_static_mask()
if static_mask is not None:
if not iset.external_lookup.mask.data.empty():
for m1, m2 in zip(static_mask, iset.external_lookup.mask.data):
m1 &= m2.data()
iset.external_lookup.mask.data = ImageBool(static_mask)
else:
iset.external_lookup.mask.data = ImageBool(static_mask)
return iset
def tst_null_reader_imageset(self):
from dxtbx.imageset import NullReader, ImageSet
from dxtbx.model import Beam, Detector
paths = ['hello_world.cbf']
# Create the null reader
reader = NullReader(paths)
# Create the imageset
imageset = ImageSet(reader)
# Try to get an item
try:
imageset[0]
assert(False)
except Exception:
print 'OK'
# Try to slice the imageset
imageset2 = imageset[0:1]
print 'OK'
# Try some functions which should work
assert(len(imageset) == 1)
assert(imageset == imageset)
assert(imageset.indices() == [0])
assert(imageset.is_valid())
print 'OK'
# Try to get models (expect failure)
try:
imageset.get_image_models(0)
assert(False)
except Exception:
print 'OK'
# Get the image paths
assert(imageset.paths() == paths)
assert(imageset.get_path(0) == paths[0])
print 'OK'
imageset.set_beam(Beam(), 0)
imageset.set_detector(Detector(), 0)
assert(isinstance(imageset.get_beam(0), Beam))
assert(isinstance(imageset.get_detector(0), Detector))
print 'OK'
