def process_datablock(self, tag, datablock):
import os
s = tag
# before processing, set output paths according to the templates
if self.params.output.datablock_filename is not None and "%s" in self.params.output.datablock_filename:
self.params.output.datablock_filename = os.path.join(self.params.output.output_dir, self.params.output.datablock_filename%("idx-" + s))
if self.params.output.strong_filename is not None and "%s" in self.params.output.strong_filename:
self.params.output.strong_filename = os.path.join(self.params.output.output_dir, self.params.output.strong_filename%("idx-" + s))
if self.params.output.indexed_filename is not None and "%s" in self.params.output.indexed_filename:
self.params.output.indexed_filename = os.path.join(self.params.output.output_dir, self.params.output.indexed_filename%("idx-" + s))
if "%s" in self.params.output.refined_experiments_filename:
self.params.output.refined_experiments_filename = os.path.join(self.params.output.output_dir, self.params.output.refined_experiments_filename%("idx-" + s))
if "%s" in self.params.output.integrated_filename:
self.params.output.integrated_filename = os.path.join(self.params.output.output_dir, self.params.output.integrated_filename%("idx-" + s))
self.tag = tag
if self.params.output.datablock_filename:
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(datablock)
dump.as_json(self.params.output.datablock_filename)
# Do the processing
try:
observed = self.find_spots(datablock)
except Exception, e:
print "Error spotfinding", tag, str(e)
return
def run(self):
'''Execute the script.'''
from dials.array_family import flex
from dials.util.options import flatten_datablocks
from dials.util.options import flatten_reflections
from time import time
from dials.util import log
from logging import info, debug
from libtbx.utils import Sorry
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# Configure the logging
log.config(
params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
info('The following parameters have been modified:\n')
info(diff_phil)
# Ensure we have a data block
datablocks = flatten_datablocks(params.input.datablock)
reflections = flatten_reflections(params.input.reflections)
if len(datablocks) == 0 and len(reflections) == 0:
self.parser.print_help()
return
elif len(datablocks) != len(reflections):
raise Sorry("Must have same number of datablocks and reflection tables")
# Combine the datablocks and reflections
datablock, reflections = combine(
datablocks,
reflections,
params)
# Save the reflections to file
info('\n' + '-' * 80)
reflections.as_pickle(params.output.reflections)
info('Saved {0} reflections to {1}'.format(
len(reflections), params.output.reflections))
# Save the datablock
from dxtbx.datablock import DataBlockDumper
info('Saving datablocks to {0}'.format(
params.output.datablock))
dump = DataBlockDumper(datablocks)
dump.as_file(params.output.datablock)
# Print the time
info("Time Taken: %f" % (time() - start_time))
def process_datablock(self, tag, datablock):
import os
s = tag
# before processing, set output paths according to the templates
if self.params.output.datablock_filename is not None and "%s" in self.params.output.datablock_filename:
self.params.output.datablock_filename = os.path.join(self.params.output.output_dir, self.params.output.datablock_filename%("idx-" + s))
if self.params.output.strong_filename is not None and "%s" in self.params.output.strong_filename:
self.params.output.strong_filename = os.path.join(self.params.output.output_dir, self.params.output.strong_filename%("idx-" + s))
if self.params.output.indexed_filename is not None and "%s" in self.params.output.indexed_filename:
self.params.output.indexed_filename = os.path.join(self.params.output.output_dir, self.params.output.indexed_filename%("idx-" + s))
if "%s" in self.params.output.refined_experiments_filename:
self.params.output.refined_experiments_filename = os.path.join(self.params.output.output_dir, self.params.output.refined_experiments_filename%("idx-" + s))
if "%s" in self.params.output.integrated_filename:
self.params.output.integrated_filename = os.path.join(self.params.output.output_dir, self.params.output.integrated_filename%("idx-" + s))
self.tag = tag
if self.params.output.datablock_filename:
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(datablock)
dump.as_json(self.params.output.datablock_filename)
# Do the processing
try:
observed = self.find_spots(datablock)
except Exception, e:
print "Error spotfinding", tag, str(e)
return
def test_split_single_image_datablock(dials_regression, tmpdir):
tmpdir.chdir()
pytest.importorskip("h5py")
sacla_file = os.path.join(
dials_regression,
"image_examples",
"SACLA_MPCCD_Cheetah",
"run266702-0-subset.h5",
)
db = DataBlockFactory.from_filenames([sacla_file])[0]
assert db.num_images() == 4
imageset = db.extract_imagesets()[0]
subset = imageset[2:3]
subblock = DataBlockFactory.from_imageset(subset)[0]
assert subblock.num_images() == 1
assert get_indices(subblock) == [2]
dumped_filename = "split_datablock.json"
dump = DataBlockDumper(subblock)
dump.as_json(dumped_filename)
db = DataBlockFactory.from_json_file(dumped_filename, check_format=True)[0]
assert db.num_images() == 1
assert get_indices(db) == [2]
db = DataBlockFactory.from_json_file(dumped_filename,
check_format=False)[0]
assert db.num_images() == 1
assert get_indices(db) == [2]
def run(self):
'''Execute the script.'''
from dials.array_family import flex
from dials.util.options import flatten_datablocks
from dials.util.options import flatten_reflections
from time import time
from dials.util import log
from libtbx.utils import Sorry
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# Configure the logging
log.config(
params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Ensure we have a data block
datablocks = flatten_datablocks(params.input.datablock)
reflections = flatten_reflections(params.input.reflections)
if len(datablocks) == 0 and len(reflections) == 0:
self.parser.print_help()
return
elif len(datablocks) != len(reflections):
raise Sorry("Must have same number of datablocks and reflection tables")
# Combine the datablocks and reflections
datablock, reflections = combine(
datablocks,
reflections,
params)
# Save the reflections to file
logger.info('\n' + '-' * 80)
reflections.as_pickle(params.output.reflections)
logger.info('Saved {0} reflections to {1}'.format(
len(reflections), params.output.reflections))
# Save the datablock
from dxtbx.datablock import DataBlockDumper
logger.info('Saving datablocks to {0}'.format(
params.output.datablock))
dump = DataBlockDumper(datablocks)
dump.as_file(params.output.datablock)
# Print the time
logger.info("Time Taken: %f" % (time() - start_time))
def write_datablocks(self, datablocks, params):
'''
Output the datablock to file.
'''
if params.output.datablock:
logger.info("-" * 80)
logger.info('Writing datablocks to %s' % params.output.datablock)
dump = DataBlockDumper(datablocks)
dump.as_file(params.output.datablock, compact=params.output.compact)
def write_datablocks(self, datablocks, params):
'''
Output the datablock to file.
'''
if params.output.datablock:
logger.info("-" * 80)
logger.info('Writing datablocks to %s' % params.output.datablock)
dump = DataBlockDumper(datablocks)
dump.as_file(params.output.datablock, compact=params.output.compact)
def process_datablock(self, tag, datablock):
import os
if not self.params.output.composite_output:
self.setup_filenames(tag)
self.tag = tag
if self.params.output.datablock_filename:
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(datablock)
dump.as_json(self.params.output.datablock_filename)
# Do the processing
try:
self.pre_process(datablock)
except Exception as e:
print("Error in pre-process", tag, str(e))
if not self.params.dispatch.squash_errors: raise
return
try:
if self.params.dispatch.find_spots:
observed = self.find_spots(datablock)
else:
print("Spot Finding turned off. Exiting")
return
except Exception as e:
print("Error spotfinding", tag, str(e))
if not self.params.dispatch.squash_errors: raise
return
try:
if self.params.dispatch.index:
experiments, indexed = self.index(datablock, observed)
else:
print("Indexing turned off. Exiting")
return
except Exception as e:
print("Couldn't index", tag, str(e))
if not self.params.dispatch.squash_errors: raise
return
try:
experiments, indexed = self.refine(experiments, indexed)
except Exception as e:
print("Error refining", tag, str(e))
if not self.params.dispatch.squash_errors: raise
return
try:
if self.params.dispatch.integrate:
integrated = self.integrate(experiments, indexed)
else:
print("Integration turned off. Exiting")
return
except Exception as e:
print("Error integrating", tag, str(e))
if not self.params.dispatch.squash_errors: raise
return
def run(self):
''' Run the script. '''
from dials.util.masking import MaskGenerator
from dials.util.options import flatten_datablocks
from libtbx.utils import Sorry
import six.moves.cPickle as pickle
from dials.util import log
from dxtbx.format.image import ImageBool
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
# Configure logging
log.config()
# Check number of args
if len(datablocks) == 0:
self.parser.print_help()
return
if len(datablocks) != 1:
raise Sorry('exactly 1 datablock must be specified')
datablock = datablocks[0]
imagesets = datablock.extract_imagesets()
if len(imagesets) != 1:
raise Sorry('datablock must contain exactly 1 imageset')
imageset = imagesets[0]
# Generate the mask
generator = MaskGenerator(params)
mask = generator.generate(imageset)
# Save the mask to file
print("Writing mask to %s" % params.output.mask)
with open(params.output.mask, "wb") as fh:
pickle.dump(mask, fh)
# Save the datablock
if params.output.datablock is not None:
imageset.external_lookup.mask.data = ImageBool(mask)
imageset.external_lookup.mask.filename = params.output.mask
from dxtbx.datablock import DataBlockDumper
print('Saving datablocks to {0}'.format(params.output.datablock))
dump = DataBlockDumper(datablocks)
dump.as_file(params.output.datablock)
def run(all_paths, hits):
print 'importing {}'.format(all_paths)
datablocks = [do_import(path) for path in all_paths]
split_datablocks = []
print 'processing datablocks'
counter = 0
for datablock in datablocks:
for imageset in datablock.extract_imagesets():
paths = imageset.paths()
for i in xrange(len(imageset)):
print i
subset = imageset[i:i + 1]
split_datablocks.append(
DataBlockFactory.from_imageset(subset)[0])
if i in hits:
counter += 1
print(paths[i])
dump = DataBlockDumper(split_datablocks[i])
dump.as_json('datablock_%i.json' % i)
return counter
def run(self):
''' Run the script. '''
from dials.util.options import flatten_datablocks
from dxtbx.datablock import DataBlockDumper
from libtbx.utils import Sorry
import cPickle as pickle
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
# Check number of args
if len(datablocks) == 0:
self.parser.print_help()
return
# Check the mask file is given
if params.input.mask is None:
self.parser.print_help()
return
# Check nbumber of datablocks
if len(datablocks) != 1:
raise Sorry('exactly 1 datablock must be specified')
# Get the imageset
datablock = datablocks[0]
imagesets = datablock.extract_imagesets()
if len(imagesets) != 1:
raise Sorry('datablock must contain exactly 1 imageset')
imageset = imagesets[0]
# Set the lookup
imageset.external_lookup.mask.filename = params.input.mask
# Dump the datablock
print "Writing datablock to %s" % params.output.datablock
dump = DataBlockDumper(datablock)
dump.as_json(filename=params.output.datablock)
def run(self):
''' Run the script. '''
from dials.util.options import flatten_datablocks
from dxtbx.datablock import DataBlockDumper
from libtbx.utils import Sorry
import cPickle as pickle
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
# Check number of args
if len(datablocks) == 0:
self.parser.print_help()
return
# Check the mask file is given
if params.input.mask is None:
self.parser.print_help()
return
# Check nbumber of datablocks
if len(datablocks) != 1:
raise Sorry('exactly 1 datablock must be specified')
# Get the imageset
datablock = datablocks[0]
imagesets = datablock.extract_imagesets()
if len(imagesets) != 1:
raise Sorry('datablock must contain exactly 1 imageset')
imageset = imagesets[0]
# Set the lookup
imageset.external_lookup.mask.filename = params.input.mask
# Dump the datablock
print "Writing datablock to %s" % params.output.datablock
dump = DataBlockDumper(datablock)
dump.as_json(filename=params.output.datablock)
def run(self):
'''Execute the script.'''
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
try:
assert len(params.input.reflections) == len(params.input.datablock)
except AssertionError:
raise Sorry(
"The number of input reflections files does not match the "
"number of input datablocks")
datablocks = flatten_datablocks(params.input.datablock)
reflections = flatten_reflections(params.input.reflections)
if len(reflections):
r = self.combine_reflections(reflections)
# print number of reflections per imageset
from libtbx.table_utils import simple_table
max_id = max(r['id'])
header = ["Imageset", "Nref"]
nrefs_per_imset = [(r['id'] == i).count(True)
for i in range(max_id + 1)]
rows = [(str(i), str(n)) for (i, n) in enumerate(nrefs_per_imset)]
st = simple_table(rows, header)
print(st.format())
rf = params.output.reflections_filename
print('Saving combined reflections to {0}'.format(rf))
r.as_pickle(rf)
if len(datablocks):
db = self.combine_datablocks(datablocks)
dbf = params.output.datablocks_filename
print('Saving combined datablocks to {0}'.format(dbf))
dump = DataBlockDumper(db)
dump.as_file(dbf, compact=params.output.compact)
return
def process_datablock(self, tag, datablock):
if not self.params.output.composite_output:
self.setup_filenames(tag)
self.tag = tag
if self.params.output.datablock_filename:
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(datablock)
dump.as_json(self.params.output.datablock_filename)
# Do the processing
try:
observed = self.find_spots(datablock)
except Exception as e:
print "Error spotfinding", tag, str(e)
if not self.params.dispatch.squash_errors: raise
return
try:
experiments, indexed = self.index(datablock, observed)
except Exception as e:
print "Couldn't index", tag, str(e)
if not self.params.dispatch.squash_errors: raise
return
try:
experiments, indexed = self.refine(experiments, indexed)
except Exception as e:
print "Error refining", tag, str(e)
if not self.params.dispatch.squash_errors: raise
return
from diffuse_scattering.sematura import DiffuseExperiment, DiffuseImage
def from_experiments(self, experiments):
exp_xtal = experiments.crystals()[0]
### define useful attributes
self.crystal = exp_xtal
uc = self.crystal.get_unit_cell()
uc_nospace = str(uc).replace(" ", "")
uc_nospace_noparen = uc_nospace[1:-1]
self.unit_cell = uc_nospace_noparen
self.space_group = self.crystal.get_space_group()
self.laue_group = self.space_group.laue_group_type()
# self.a_matrix = crystal.get_A()
self.experiments = experiments
DiffuseExperiment.from_experiments = from_experiments
if (self.params.mp.method == 'mpi'):
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank(
) # each process in MPI has a unique id, 0-indexed
else:
rank = 0
test_exp = DiffuseExperiment()
test_exp.from_experiments(experiments)
img_set = test_exp.experiments.imagesets()
imgs = img_set[0]
file_list = imgs.paths()
img_file = file_list[0]
test_img = DiffuseImage(img_file)
test_img.set_general_variables()
test_img.remove_bragg_peaks(radial=True)
if self.ncalls == 0:
if rank == 0:
self.ref_data = deepcopy(test_img.lunus_data_scitbx)
# print "ref_data is",self.ref_data
# else:
# self.ref_data = None
if (self.params.mp.method == 'mpi'):
# from mpi4py import MPI
# comm = MPI.COMM_WORLD
# print "Barrier, rank = ",rank
# print "Broadcast, rank = ",rank
self.ref_data = comm.bcast(self.ref_data, root=0)
comm.barrier()
# print "Broadcast done, rank = ",rank
if self.ref_data == None:
print "ref_data = None for Rank = ", rank
test_img.scale_factor_from_images(self.ref_data)
# test_img.scale_factor()
test_img.crystal_geometry(test_exp.crystal)
test_img.setup_diffuse_lattice(self.params.lunus.d_min)
test_img.integrate_diffuse()
self.ncalls = self.ncalls + 1
from __future__ import absolute_import, division, print_function
from dxtbx.datablock import DataBlockFactory, DataBlockDumper
import sys
from six.moves import range
fin = sys.argv[1]
px_size = 0.05 # in mm
dpx = [0, 0, 1, 1, 0, 1, 1, 0] # in px
dpy = [0, 0, 1, 1, 0, 1, 1, 0] # in px
db = DataBlockFactory.from_json_file(fin)
db0 = db[0]
dd = db0.to_dict()
for i in range(8):
x = dd['detector'][0]['panels'][i]['origin'][0] + dpx[i] * px_size
y = dd['detector'][0]['panels'][i]['origin'][1] + dpy[i] * px_size
dd['detector'][0]['panels'][i]['origin'] = (x, y, 0)
xx = DataBlockFactory.from_dict(dd)
yy = DataBlockDumper(xx)
yy.as_file('new_detector_geom.json')
if not stills:
num_stills = 0
else:
num_stills = len(stills)
# Print some data block info
print "-" * 80
print "DataBlock %d" % i
print " format: %s" % str(datablock.format_class())
print " num images: %d" % datablock.num_images()
print " num sweeps: %d" % len(sweeps)
print " num stills: %d" % num_stills
# Loop through all the sweeps
if options.verbose > 1:
for j, sweep in enumerate(sweeps):
print ""
print "Sweep %d" % j
print " length %d" % len(sweep)
print sweep.get_beam()
print sweep.get_goniometer()
print sweep.get_detector()
print sweep.get_scan()
# Write the datablock to a JSON or pickle file
if options.output:
print "-" * 80
print 'Writing datablocks to %s' % options.output
dump = DataBlockDumper(datablocks)
dump.as_file(options.output, compact=options.compact)
def run(self):
'''Execute the script.'''
from dxtbx.datablock import DataBlockTemplateImporter
from dials.util.options import flatten_datablocks
from dials.util import log
from logging import info
from time import time
from libtbx.utils import Abort
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
datablocks = flatten_datablocks(params.input.datablock)
# Check we have some filenames
if len(datablocks) == 0:
# Check if a template has been set and print help if not, otherwise try to
# import the images based on the template input
if len(params.input.template) == 0:
self.parser.print_help()
exit(0)
else:
importer = DataBlockTemplateImporter(params.input.template,
options.verbose)
datablocks = importer.datablocks
# Save the options
self.options = options
self.params = params
self.load_reference_geometry()
st = time()
# Import stuff
if len(datablocks) == 0:
raise Abort('No datablocks specified')
elif len(datablocks) > 1:
raise Abort('Only 1 datablock can be processed at a time.')
datablock = datablocks[0]
if self.reference_detector is not None:
for imageset in datablock.extract_imagesets():
imageset.set_detector(self.reference_detector)
# Configure logging
log.config(params.verbosity,
info='dials.process.log',
debug='dials.process.debug.log')
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
info('The following parameters have been modified:\n')
info(diff_phil)
if self.params.output.datablock_filename:
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(datablock)
dump.as_json(self.params.output.datablock_filename)
# Do the processing
observed = self.find_spots(datablock)
experiments, indexed = self.index(datablock, observed)
experiments, indexed = self.refine(experiments, indexed)
integrated = self.integrate(experiments, indexed)
# Total Time
info("")
info("Total Time Taken = %f seconds" % (time() - st))
def run(self):
'''Execute the script.'''
from dxtbx.datablock import DataBlockTemplateImporter
from dials.util.options import flatten_datablocks
from dials.util import log
from logging import info
from time import time
from libtbx.utils import Abort
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
datablocks = flatten_datablocks(params.input.datablock)
# Check we have some filenames
if len(datablocks) == 0:
# Check if a template has been set and print help if not, otherwise try to
# import the images based on the template input
if len(params.input.template) == 0:
self.parser.print_help()
exit(0)
else:
importer = DataBlockTemplateImporter(
params.input.template,
options.verbose)
datablocks = importer.datablocks
# Save the options
self.options = options
self.params = params
self.load_reference_geometry()
st = time()
# Import stuff
if len(datablocks) == 0:
raise Abort('No datablocks specified')
elif len(datablocks) > 1:
raise Abort('Only 1 datablock can be processed at a time.')
datablock = datablocks[0]
if self.reference_detector is not None:
for imageset in datablock.extract_imagesets():
imageset.set_detector(self.reference_detector)
# Configure logging
log.config(
params.verbosity,
info='dials.process.log',
debug='dials.process.debug.log')
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
info('The following parameters have been modified:\n')
info(diff_phil)
if self.params.output.datablock_filename:
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(datablock)
dump.as_json(self.params.output.datablock_filename)
# Do the processing
observed = self.find_spots(datablock)
experiments, indexed = self.index(datablock, observed)
experiments = self.refine(experiments, indexed)
integrated = self.integrate(experiments, indexed)
# Total Time
info("")
info("Total Time Taken = %f seconds" % (time() - st))
def run(self):
'''Execute the script.'''
from dials.util.options import flatten_reflections, flatten_experiments, \
flatten_datablocks
import cPickle as pickle
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
datablocks = flatten_datablocks(params.input.datablock)
# Try to load the models and data
slice_exps = len(experiments) > 0
slice_refs = len(reflections) > 0
slice_dbs = len(datablocks) > 0
# Catch case of nothing to do
if not any([slice_exps, slice_refs, slice_dbs]):
print "No suitable input provided"
self.parser.print_help()
return
if reflections:
if len(reflections) > 1:
raise Sorry(
"Only one reflections list can be imported at present")
reflections = reflections[0]
# calculate frame numbers if needed
if experiments:
reflections = calculate_frame_numbers(reflections, experiments)
# if we still don't have the right column give up
if 'xyzobs.px.value' not in reflections:
raise Sorry(
"These reflections do not have frame numbers set, and "
"there are no experiments provided to calculate these.")
# set trivial case where no scan range is provided at all
if not params.scan_range:
params.scan_range = [None]
# check if slicing into blocks
if params.block_size is not None:
# in this case for simplicity, ensure that there is either an
# an experiment list or datablocks, but not both. Ensure there is only
# a single scan contained within.
if [slice_exps, slice_dbs].count(True) != 1:
raise Sorry(
"For slicing into blocks please provide either datablocks"
" or experiments, but not both.")
if slice_exps:
if len(experiments) > 1:
raise Sorry(
"For slicing into blocks please provide a single "
"scan only")
scan = experiments[0].scan
if slice_dbs:
scans = datablocks[0].unique_scans()
if len(scans) > 1 or len(datablocks) > 1:
raise Sorry(
"For slicing into blocks please provide a single "
"scan only")
scan = scans[0]
# Having extracted the scan, calculate the blocks
params.scan_range = calculate_block_ranges(scan, params.block_size)
# Do the slicing then recombine
if slice_exps:
sliced = [slice_experiments(experiments, [sr])[0] \
for sr in params.scan_range]
sliced_experiments = ExperimentList()
for exp in sliced:
sliced_experiments.append(exp)
if slice_dbs:
sliced = [slice_datablocks(datablocks, [sr])[0] \
for sr in params.scan_range]
imagesets = [db.extract_imagesets()[0] for db in sliced]
sliced_datablocks = DataBlock(imagesets)
# slice reflections if present
if slice_refs:
sliced = [slice_reflections(reflections, [sr]) \
for sr in params.scan_range]
sliced_reflections = sliced[0]
for i, rt in enumerate(sliced[1:]):
rt['id'] += (i + 1) # set id
sliced_reflections.extend(rt)
else:
# slice each dataset into the requested subset
if slice_exps:
sliced_experiments = slice_experiments(experiments,
params.scan_range)
if slice_refs:
sliced_reflections = slice_reflections(reflections,
params.scan_range)
if slice_dbs:
sliced_datablocks = slice_datablocks(datablocks,
params.scan_range)
# Save sliced experiments
if slice_exps:
output_experiments_filename = params.output.experiments_filename
if output_experiments_filename is None:
# take first filename as template
bname = basename(params.input.experiments[0].filename)
bname = splitext(bname)[0]
if not bname: bname = "experiments"
if len(params.scan_range
) == 1 and params.scan_range[0] is not None:
ext = "_{0}_{1}.json".format(*params.scan_range[0])
else:
ext = "_sliced.json"
output_experiments_filename = bname + ext
print 'Saving sliced experiments to {0}'.format(
output_experiments_filename)
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(sliced_experiments)
dump.as_json(output_experiments_filename)
# Save sliced reflections
if slice_refs:
output_reflections_filename = params.output.reflections_filename
if output_reflections_filename is None:
# take first filename as template
bname = basename(params.input.reflections[0].filename)
bname = splitext(bname)[0]
if not bname: bname = "reflections"
if len(params.scan_range
) == 1 and params.scan_range[0] is not None:
ext = "_{0}_{1}.pickle".format(*params.scan_range[0])
else:
ext = "_sliced.pickle"
output_reflections_filename = bname + ext
print 'Saving sliced reflections to {0}'.format(
output_reflections_filename)
sliced_reflections.as_pickle(output_reflections_filename)
# Save sliced datablocks
if slice_dbs:
output_datablocks_filename = params.output.datablocks_filename
if output_datablocks_filename is None:
# take first filename as template
bname = basename(params.input.datablock[0].filename)
bname = splitext(bname)[0]
if not bname: bname = "datablock"
if len(params.scan_range
) == 1 and params.scan_range[0] is not None:
ext = "_{0}_{1}.json".format(*params.scan_range[0])
else:
ext = "_sliced.json"
output_datablocks_filename = bname + ext
print 'Saving sliced datablocks to {0}'.format(
output_datablocks_filename)
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(sliced_datablocks)
dump.as_file(output_datablocks_filename)
return
def datablock(obj, outfile, **kwargs): ''' Dump the given object to file. ''' from dxtbx.datablock import DataBlockDumper dump = DataBlockDumper(obj) dump.as_file(outfile, **kwargs)
def test_elliptical_distortion(tmpdir):
"""Create distortion maps for elliptical distortion using a dummy datablock
with a small detector, for speed. Check those maps seem sensible"""
tmpdir.chdir()
# Make a detector model
d = make_detector()
# The beam is also essential for a datablock to be serialisable
b = Beam((0, 0, 1), 1.0)
# Create and write out a datablock
imageset = ImageSet(
ImageSetData(Reader(["non-existent.cbf"]),
Masker(["non-existent.cbf"])))
imageset.set_detector(d)
imageset.set_beam(b)
datablocks = DataBlockFactory.from_imageset(imageset)
dump = DataBlockDumper(datablocks)
dump.as_file("dummy_datablock.json")
# Centre of distortion will be the far corner from the origin of the first
# panel
centre_xy = d[0].get_image_size_mm()
# Generate distortion maps
cmd = ("dials.generate_distortion_maps dummy_datablock.json "
"mode=ellipse centre_xy={},{} "
"phi=0 l1=1.0 l2=0.95").format(*centre_xy)
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
# Load the maps
with open("dx.pickle", "rb") as f:
dx = pickle.load(f)
with open("dy.pickle", "rb") as f:
dy = pickle.load(f)
# Check there are 4 maps each
assert len(dx) == len(dy) == 4
# Ellipse has phi=0, so all correction is in the dy map
for arr in dx:
assert min(arr) == max(arr) == 0.0
# The ellipse correction is centred at the middle of the detector and all in
# the Y direction. Therefore we expect a few things from the dy maps:
#
# (1) Within each panel the columns of the array are identical.
# (2) The two upper panels should be the same
# (3) The two lower panels should be the same.
# (4) One column from an upper panel is a negated, reversed column from a
# lower panel.
#
# All together expect the 4 dy maps to look something like this:
#
# /-----------\ /-----------\
# |-3 -3 -3 -3| |-3 -3 -3 -3|
# |-2 -2 -2 -2| |-2 -2 -2 -2|
# |-1 -1 -1 -1| |-1 -1 -1 -1|
# | 0 0 0 0| | 0 0 0 0|
# \-----------/ \-----------/
# /-----------\ /-----------\
# | 0 0 0 0| | 0 0 0 0|
# | 1 1 1 1| | 1 1 1 1|
# | 2 2 2 2| | 2 2 2 2|
# | 3 3 3 3| | 3 3 3 3|
# \-----------/ \-----------/
# So the fundamental data is all in the first column of first panel's map
col0 = dy[0].matrix_copy_column(0)
# The correction should be 5% of the distance from the ellipse centre to a
# corrected pixel (l2 = 0.95 above) along the slow axis. Check that is the
# case (for the first pixel at least)
vec_centre_to_first_px = (matrix.col(d[0].get_pixel_lab_coord(
(0.5, 0.5))) - matrix.col(d[0].get_lab_coord(centre_xy)))
dist_centre_to_first_px = vec_centre_to_first_px.dot(
matrix.col(d[0].get_slow_axis()))
corr_mm = dist_centre_to_first_px * 0.05
corr_px = corr_mm / d[0].get_pixel_size()[1]
assert col0[0] == pytest.approx(corr_px)
# Test (1) from above list for panel 0
for i in range(1, 50):
assert (col0 == dy[0].matrix_copy_column(i)).all_eq(True)
# Test (2)
assert (dy[0] == dy[1]).all_eq(True)
# Test (3)
assert (dy[2] == dy[3]).all_eq(True)
# Test (4)
assert col0 == pytest.approx(-1.0 * dy[2].matrix_copy_column(0).reversed())
# Test (1) for panel 2 as well, which then covers everything needed
col0 = dy[2].matrix_copy_column(0)
for i in range(1, 50):
assert (col0 == dy[2].matrix_copy_column(i)).all_eq(True)
if not stills:
num_stills = 0
else:
num_stills = len(stills)
# Print some data block info
print "-" * 80
print "DataBlock %d" % i
print " format: %s" % str(datablock.format_class())
print " num images: %d" % datablock.num_images()
print " num sweeps: %d" % len(sweeps)
print " num stills: %d" % num_stills
# Loop through all the sweeps
if options.verbose > 1:
for j, sweep in enumerate(sweeps):
print ""
print "Sweep %d" % j
print " length %d" % len(sweep)
print sweep.get_beam()
print sweep.get_goniometer()
print sweep.get_detector()
print sweep.get_scan()
# Write the datablock to a JSON or pickle file
if options.output:
print "-" * 80
print 'Writing datablocks to %s' % options.output
dump = DataBlockDumper(datablocks)
dump.as_file(options.output, compact=options.compact)
def run(self):
'''Execute the script.'''
from dials.array_family import flex
from dials.util.options import flatten_datablocks
from time import time
from dials.util import log
from libtbx.utils import Sorry
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# Configure the logging
log.config(params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Ensure we have a data block
datablocks = flatten_datablocks(params.input.datablock)
if len(datablocks) == 0:
self.parser.print_help()
return
elif len(datablocks) != 1:
raise Sorry('only 1 datablock can be processed at a time')
# Loop through all the imagesets and find the strong spots
reflections = flex.reflection_table.from_observations(
datablocks[0], params)
# Delete the shoeboxes
if not params.output.shoeboxes:
del reflections['shoebox']
# ascii spot count per image plot
from dials.util.ascii_art import spot_counts_per_image_plot
for i, imageset in enumerate(datablocks[0].extract_imagesets()):
ascii_plot = spot_counts_per_image_plot(
reflections.select(reflections['id'] == i))
if len(ascii_plot):
logger.info(
'\nHistogram of per-image spot count for imageset %i:' % i)
logger.info(ascii_plot)
# Save the reflections to file
logger.info('\n' + '-' * 80)
reflections.as_pickle(params.output.reflections)
logger.info('Saved {0} reflections to {1}'.format(
len(reflections), params.output.reflections))
# Save the datablock
if params.output.datablock:
from dxtbx.datablock import DataBlockDumper
logger.info('Saving datablocks to {0}'.format(
params.output.datablock))
dump = DataBlockDumper(datablocks)
dump.as_file(params.output.datablock)
# Print some per image statistics
if params.per_image_statistics:
from dials.algorithms.spot_finding import per_image_analysis
from cStringIO import StringIO
s = StringIO()
for i, imageset in enumerate(datablocks[0].extract_imagesets()):
print >> s, "Number of centroids per image for imageset %i:" % i
stats = per_image_analysis.stats_imageset(
imageset,
reflections.select(reflections['id'] == i),
resolution_analysis=False)
per_image_analysis.print_table(stats, out=s)
logger.info(s.getvalue())
# Print the time
logger.info("Time Taken: %f" % (time() - start_time))
def datablock(obj, outfile, **kwargs):
""" Dump the given object to file. """
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(obj)
dump.as_file(outfile, **kwargs)
def datablock(obj, outfile, **kwargs):
"""Dump the given object to file."""
dump = DataBlockDumper(obj)
dump.as_file(outfile, **kwargs)
from dxtbx.datablock import DataBlockFactory, DataBlockDumper
from dxtbx.imageset import ImageSetFactory, ImageSweep, ImageSetData
from dxtbx.model.goniometer import GoniometerFactory
from dxtbx.model.scan import ScanFactory
import glob, os, sys
"""
Modification of AB's prepare_sweep.py.
Usage: libtbx.python prepare_sweep.py img_dir start_img, end_img savefile
"""
root = sys.argv[1]
start, end = int(sys.argv[2]), int(sys.argv[3])
g = GoniometerFactory.single_axis()
s = ScanFactory.make_scan((start, end), 0, (0, 1), [0] * (end - start + 1))
sw = ImageSetFactory.from_template(template=os.path.join(
root, "fft_frame_I_mf_####.cbf"),
scan=s,
goniometer=g,
image_range=(start, end))
dump = DataBlockDumper(DataBlockFactory.from_imageset(sw))
dump.as_file(sys.argv[4])
def process_datablock(self, tag, datablock):
if not self.params.output.composite_output:
self.setup_filenames(tag)
self.tag = tag
if self.params.output.datablock_filename:
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(datablock)
dump.as_json(self.params.output.datablock_filename)
# Do the processing
try:
observed = self.find_spots(datablock)
except Exception as e:
print "Error spotfinding", tag, str(e)
if not self.params.dispatch.squash_errors: raise
return
try:
experiments, indexed = self.index(datablock, observed)
except Exception as e:
print "Couldn't index", tag, str(e)
if not self.params.dispatch.squash_errors: raise
return
try:
experiments, indexed = self.refine(experiments, indexed)
except Exception as e:
print "Error refining", tag, str(e)
if not self.params.dispatch.squash_errors: raise
return
from diffuse_scattering.sematura import DiffuseExperiment, DiffuseImage
def from_experiments(self,experiments):
exp_xtal = experiments.crystals()[0]
### define useful attributes
self.crystal = exp_xtal
uc = self.crystal.get_unit_cell()
uc_nospace = str(uc).replace(" ", "")
uc_nospace_noparen = uc_nospace[1:-1]
self.unit_cell = uc_nospace_noparen
self.space_group = self.crystal.get_space_group()
self.laue_group = self.space_group.laue_group_type()
# self.a_matrix = crystal.get_A()
self.experiments = experiments
DiffuseExperiment.from_experiments = from_experiments
if (self.params.mp.method == 'mpi'):
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank() # each process in MPI has a unique id, 0-indexed
else:
rank = 0
test_exp = DiffuseExperiment()
test_exp.from_experiments(experiments)
img_set = test_exp.experiments.imagesets()
imgs = img_set[0]
file_list = imgs.paths()
img_file = file_list[0]
test_img = DiffuseImage(img_file)
test_img.set_general_variables()
test_img.remove_bragg_peaks(radial=True)
if self.ncalls == 0:
if rank == 0:
self.ref_data = deepcopy(test_img.lunus_data_scitbx)
# print "ref_data is",self.ref_data
# else:
# self.ref_data = None
if (self.params.mp.method == 'mpi'):
# from mpi4py import MPI
# comm = MPI.COMM_WORLD
# print "Barrier, rank = ",rank
# print "Broadcast, rank = ",rank
self.ref_data = comm.bcast(self.ref_data,root=0)
comm.barrier()
# print "Broadcast done, rank = ",rank
if self.ref_data == None:
print "ref_data = None for Rank = ",rank
test_img.scale_factor_from_images(self.ref_data)
# test_img.scale_factor()
test_img.crystal_geometry(test_exp.crystal)
test_img.setup_diffuse_lattice(self.params.lunus.d_min)
test_img.integrate_diffuse()
self.ncalls = self.ncalls + 1
def run(self):
'''Execute the script.'''
from dials.array_family import flex
from dials.util.options import flatten_datablocks
from time import time
from dials.util import log
from libtbx.utils import Sorry
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# Configure the logging
log.config(
params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Ensure we have a data block
datablocks = flatten_datablocks(params.input.datablock)
if len(datablocks) == 0:
self.parser.print_help()
return
elif len(datablocks) != 1:
raise Sorry('only 1 datablock can be processed at a time')
# Loop through all the imagesets and find the strong spots
reflections = flex.reflection_table.from_observations(
datablocks[0], params)
# Delete the shoeboxes
if not params.output.shoeboxes:
del reflections['shoebox']
# ascii spot count per image plot
from dials.util.ascii_art import spot_counts_per_image_plot
for i, imageset in enumerate(datablocks[0].extract_imagesets()):
ascii_plot = spot_counts_per_image_plot(
reflections.select(reflections['id'] == i))
if len(ascii_plot):
logger.info('\nHistogram of per-image spot count for imageset %i:' %i)
logger.info(ascii_plot)
# Save the reflections to file
logger.info('\n' + '-' * 80)
reflections.as_pickle(params.output.reflections)
logger.info('Saved {0} reflections to {1}'.format(
len(reflections), params.output.reflections))
# Save the datablock
if params.output.datablock:
from dxtbx.datablock import DataBlockDumper
logger.info('Saving datablocks to {0}'.format(
params.output.datablock))
dump = DataBlockDumper(datablocks)
dump.as_file(params.output.datablock)
# Print some per image statistics
if params.per_image_statistics:
from dials.algorithms.spot_finding import per_image_analysis
from cStringIO import StringIO
s = StringIO()
for i, imageset in enumerate(datablocks[0].extract_imagesets()):
print >> s, "Number of centroids per image for imageset %i:" %i
stats = per_image_analysis.stats_imageset(
imageset, reflections.select(reflections['id'] == i),
resolution_analysis=False)
per_image_analysis.print_table(stats, out=s)
logger.info(s.getvalue())
# Print the time
logger.info("Time Taken: %f" % (time() - start_time))
def write_expt(experiments, filename):
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(experiments)
dump.as_file(filename)
