def process_datablock(self, tag, datablock):
import os
s = tag
# before processing, set output paths according to the templates
if self.params.output.datablock_filename is not None and "%s" in self.params.output.datablock_filename:
self.params.output.datablock_filename = os.path.join(self.params.output.output_dir, self.params.output.datablock_filename%("idx-" + s))
if self.params.output.strong_filename is not None and "%s" in self.params.output.strong_filename:
self.params.output.strong_filename = os.path.join(self.params.output.output_dir, self.params.output.strong_filename%("idx-" + s))
if self.params.output.indexed_filename is not None and "%s" in self.params.output.indexed_filename:
self.params.output.indexed_filename = os.path.join(self.params.output.output_dir, self.params.output.indexed_filename%("idx-" + s))
if "%s" in self.params.output.refined_experiments_filename:
self.params.output.refined_experiments_filename = os.path.join(self.params.output.output_dir, self.params.output.refined_experiments_filename%("idx-" + s))
if "%s" in self.params.output.integrated_filename:
self.params.output.integrated_filename = os.path.join(self.params.output.output_dir, self.params.output.integrated_filename%("idx-" + s))
self.tag = tag
if self.params.output.datablock_filename:
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(datablock)
dump.as_json(self.params.output.datablock_filename)
# Do the processing
try:
observed = self.find_spots(datablock)
except Exception, e:
print "Error spotfinding", tag, str(e)
return
def test_split_single_image_datablock(dials_regression, tmpdir):
tmpdir.chdir()
pytest.importorskip("h5py")
sacla_file = os.path.join(
dials_regression,
"image_examples",
"SACLA_MPCCD_Cheetah",
"run266702-0-subset.h5",
)
db = DataBlockFactory.from_filenames([sacla_file])[0]
assert db.num_images() == 4
imageset = db.extract_imagesets()[0]
subset = imageset[2:3]
subblock = DataBlockFactory.from_imageset(subset)[0]
assert subblock.num_images() == 1
assert get_indices(subblock) == [2]
dumped_filename = "split_datablock.json"
dump = DataBlockDumper(subblock)
dump.as_json(dumped_filename)
db = DataBlockFactory.from_json_file(dumped_filename, check_format=True)[0]
assert db.num_images() == 1
assert get_indices(db) == [2]
db = DataBlockFactory.from_json_file(dumped_filename,
check_format=False)[0]
assert db.num_images() == 1
assert get_indices(db) == [2]
def process_datablock(self, tag, datablock):
import os
s = tag
# before processing, set output paths according to the templates
if self.params.output.datablock_filename is not None and "%s" in self.params.output.datablock_filename:
self.params.output.datablock_filename = os.path.join(self.params.output.output_dir, self.params.output.datablock_filename%("idx-" + s))
if self.params.output.strong_filename is not None and "%s" in self.params.output.strong_filename:
self.params.output.strong_filename = os.path.join(self.params.output.output_dir, self.params.output.strong_filename%("idx-" + s))
if self.params.output.indexed_filename is not None and "%s" in self.params.output.indexed_filename:
self.params.output.indexed_filename = os.path.join(self.params.output.output_dir, self.params.output.indexed_filename%("idx-" + s))
if "%s" in self.params.output.refined_experiments_filename:
self.params.output.refined_experiments_filename = os.path.join(self.params.output.output_dir, self.params.output.refined_experiments_filename%("idx-" + s))
if "%s" in self.params.output.integrated_filename:
self.params.output.integrated_filename = os.path.join(self.params.output.output_dir, self.params.output.integrated_filename%("idx-" + s))
self.tag = tag
if self.params.output.datablock_filename:
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(datablock)
dump.as_json(self.params.output.datablock_filename)
# Do the processing
try:
observed = self.find_spots(datablock)
except Exception, e:
print "Error spotfinding", tag, str(e)
return
def process_datablock(self, tag, datablock):
import os
if not self.params.output.composite_output:
self.setup_filenames(tag)
self.tag = tag
if self.params.output.datablock_filename:
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(datablock)
dump.as_json(self.params.output.datablock_filename)
# Do the processing
try:
self.pre_process(datablock)
except Exception as e:
print("Error in pre-process", tag, str(e))
if not self.params.dispatch.squash_errors: raise
return
try:
if self.params.dispatch.find_spots:
observed = self.find_spots(datablock)
else:
print("Spot Finding turned off. Exiting")
return
except Exception as e:
print("Error spotfinding", tag, str(e))
if not self.params.dispatch.squash_errors: raise
return
try:
if self.params.dispatch.index:
experiments, indexed = self.index(datablock, observed)
else:
print("Indexing turned off. Exiting")
return
except Exception as e:
print("Couldn't index", tag, str(e))
if not self.params.dispatch.squash_errors: raise
return
try:
experiments, indexed = self.refine(experiments, indexed)
except Exception as e:
print("Error refining", tag, str(e))
if not self.params.dispatch.squash_errors: raise
return
try:
if self.params.dispatch.integrate:
integrated = self.integrate(experiments, indexed)
else:
print("Integration turned off. Exiting")
return
except Exception as e:
print("Error integrating", tag, str(e))
if not self.params.dispatch.squash_errors: raise
return
def run(all_paths, hits):
print 'importing {}'.format(all_paths)
datablocks = [do_import(path) for path in all_paths]
split_datablocks = []
print 'processing datablocks'
counter = 0
for datablock in datablocks:
for imageset in datablock.extract_imagesets():
paths = imageset.paths()
for i in xrange(len(imageset)):
print i
subset = imageset[i:i + 1]
split_datablocks.append(
DataBlockFactory.from_imageset(subset)[0])
if i in hits:
counter += 1
print(paths[i])
dump = DataBlockDumper(split_datablocks[i])
dump.as_json('datablock_%i.json' % i)
return counter
def run(self):
''' Run the script. '''
from dials.util.options import flatten_datablocks
from dxtbx.datablock import DataBlockDumper
from libtbx.utils import Sorry
import cPickle as pickle
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
# Check number of args
if len(datablocks) == 0:
self.parser.print_help()
return
# Check the mask file is given
if params.input.mask is None:
self.parser.print_help()
return
# Check nbumber of datablocks
if len(datablocks) != 1:
raise Sorry('exactly 1 datablock must be specified')
# Get the imageset
datablock = datablocks[0]
imagesets = datablock.extract_imagesets()
if len(imagesets) != 1:
raise Sorry('datablock must contain exactly 1 imageset')
imageset = imagesets[0]
# Set the lookup
imageset.external_lookup.mask.filename = params.input.mask
# Dump the datablock
print "Writing datablock to %s" % params.output.datablock
dump = DataBlockDumper(datablock)
dump.as_json(filename=params.output.datablock)
def run(self):
''' Run the script. '''
from dials.util.options import flatten_datablocks
from dxtbx.datablock import DataBlockDumper
from libtbx.utils import Sorry
import cPickle as pickle
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
# Check number of args
if len(datablocks) == 0:
self.parser.print_help()
return
# Check the mask file is given
if params.input.mask is None:
self.parser.print_help()
return
# Check nbumber of datablocks
if len(datablocks) != 1:
raise Sorry('exactly 1 datablock must be specified')
# Get the imageset
datablock = datablocks[0]
imagesets = datablock.extract_imagesets()
if len(imagesets) != 1:
raise Sorry('datablock must contain exactly 1 imageset')
imageset = imagesets[0]
# Set the lookup
imageset.external_lookup.mask.filename = params.input.mask
# Dump the datablock
print "Writing datablock to %s" % params.output.datablock
dump = DataBlockDumper(datablock)
dump.as_json(filename=params.output.datablock)
def run(self):
'''Execute the script.'''
from dxtbx.datablock import DataBlockTemplateImporter
from dials.util.options import flatten_datablocks
from dials.util import log
from logging import info
from time import time
from libtbx.utils import Abort
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
datablocks = flatten_datablocks(params.input.datablock)
# Check we have some filenames
if len(datablocks) == 0:
# Check if a template has been set and print help if not, otherwise try to
# import the images based on the template input
if len(params.input.template) == 0:
self.parser.print_help()
exit(0)
else:
importer = DataBlockTemplateImporter(params.input.template,
options.verbose)
datablocks = importer.datablocks
# Save the options
self.options = options
self.params = params
self.load_reference_geometry()
st = time()
# Import stuff
if len(datablocks) == 0:
raise Abort('No datablocks specified')
elif len(datablocks) > 1:
raise Abort('Only 1 datablock can be processed at a time.')
datablock = datablocks[0]
if self.reference_detector is not None:
for imageset in datablock.extract_imagesets():
imageset.set_detector(self.reference_detector)
# Configure logging
log.config(params.verbosity,
info='dials.process.log',
debug='dials.process.debug.log')
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
info('The following parameters have been modified:\n')
info(diff_phil)
if self.params.output.datablock_filename:
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(datablock)
dump.as_json(self.params.output.datablock_filename)
# Do the processing
observed = self.find_spots(datablock)
experiments, indexed = self.index(datablock, observed)
experiments, indexed = self.refine(experiments, indexed)
integrated = self.integrate(experiments, indexed)
# Total Time
info("")
info("Total Time Taken = %f seconds" % (time() - st))
def run(self):
'''Execute the script.'''
from dxtbx.datablock import DataBlockTemplateImporter
from dials.util.options import flatten_datablocks
from dials.util import log
from logging import info
from time import time
from libtbx.utils import Abort
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
datablocks = flatten_datablocks(params.input.datablock)
# Check we have some filenames
if len(datablocks) == 0:
# Check if a template has been set and print help if not, otherwise try to
# import the images based on the template input
if len(params.input.template) == 0:
self.parser.print_help()
exit(0)
else:
importer = DataBlockTemplateImporter(
params.input.template,
options.verbose)
datablocks = importer.datablocks
# Save the options
self.options = options
self.params = params
self.load_reference_geometry()
st = time()
# Import stuff
if len(datablocks) == 0:
raise Abort('No datablocks specified')
elif len(datablocks) > 1:
raise Abort('Only 1 datablock can be processed at a time.')
datablock = datablocks[0]
if self.reference_detector is not None:
for imageset in datablock.extract_imagesets():
imageset.set_detector(self.reference_detector)
# Configure logging
log.config(
params.verbosity,
info='dials.process.log',
debug='dials.process.debug.log')
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
info('The following parameters have been modified:\n')
info(diff_phil)
if self.params.output.datablock_filename:
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(datablock)
dump.as_json(self.params.output.datablock_filename)
# Do the processing
observed = self.find_spots(datablock)
experiments, indexed = self.index(datablock, observed)
experiments = self.refine(experiments, indexed)
integrated = self.integrate(experiments, indexed)
# Total Time
info("")
info("Total Time Taken = %f seconds" % (time() - st))
def process_datablock(self, tag, datablock):
if not self.params.output.composite_output:
self.setup_filenames(tag)
self.tag = tag
if self.params.output.datablock_filename:
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(datablock)
dump.as_json(self.params.output.datablock_filename)
# Do the processing
try:
observed = self.find_spots(datablock)
except Exception as e:
print "Error spotfinding", tag, str(e)
if not self.params.dispatch.squash_errors: raise
return
try:
experiments, indexed = self.index(datablock, observed)
except Exception as e:
print "Couldn't index", tag, str(e)
if not self.params.dispatch.squash_errors: raise
return
try:
experiments, indexed = self.refine(experiments, indexed)
except Exception as e:
print "Error refining", tag, str(e)
if not self.params.dispatch.squash_errors: raise
return
from diffuse_scattering.sematura import DiffuseExperiment, DiffuseImage
def from_experiments(self,experiments):
exp_xtal = experiments.crystals()[0]
### define useful attributes
self.crystal = exp_xtal
uc = self.crystal.get_unit_cell()
uc_nospace = str(uc).replace(" ", "")
uc_nospace_noparen = uc_nospace[1:-1]
self.unit_cell = uc_nospace_noparen
self.space_group = self.crystal.get_space_group()
self.laue_group = self.space_group.laue_group_type()
# self.a_matrix = crystal.get_A()
self.experiments = experiments
DiffuseExperiment.from_experiments = from_experiments
if (self.params.mp.method == 'mpi'):
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank() # each process in MPI has a unique id, 0-indexed
else:
rank = 0
test_exp = DiffuseExperiment()
test_exp.from_experiments(experiments)
img_set = test_exp.experiments.imagesets()
imgs = img_set[0]
file_list = imgs.paths()
img_file = file_list[0]
test_img = DiffuseImage(img_file)
test_img.set_general_variables()
test_img.remove_bragg_peaks(radial=True)
if self.ncalls == 0:
if rank == 0:
self.ref_data = deepcopy(test_img.lunus_data_scitbx)
# print "ref_data is",self.ref_data
# else:
# self.ref_data = None
if (self.params.mp.method == 'mpi'):
# from mpi4py import MPI
# comm = MPI.COMM_WORLD
# print "Barrier, rank = ",rank
# print "Broadcast, rank = ",rank
self.ref_data = comm.bcast(self.ref_data,root=0)
comm.barrier()
# print "Broadcast done, rank = ",rank
if self.ref_data == None:
print "ref_data = None for Rank = ",rank
test_img.scale_factor_from_images(self.ref_data)
# test_img.scale_factor()
test_img.crystal_geometry(test_exp.crystal)
test_img.setup_diffuse_lattice(self.params.lunus.d_min)
test_img.integrate_diffuse()
self.ncalls = self.ncalls + 1
def process_datablock(self, tag, datablock):
if not self.params.output.composite_output:
self.setup_filenames(tag)
self.tag = tag
if self.params.output.datablock_filename:
from dxtbx.datablock import DataBlockDumper
dump = DataBlockDumper(datablock)
dump.as_json(self.params.output.datablock_filename)
# Do the processing
try:
observed = self.find_spots(datablock)
except Exception as e:
print "Error spotfinding", tag, str(e)
if not self.params.dispatch.squash_errors: raise
return
try:
experiments, indexed = self.index(datablock, observed)
except Exception as e:
print "Couldn't index", tag, str(e)
if not self.params.dispatch.squash_errors: raise
return
try:
experiments, indexed = self.refine(experiments, indexed)
except Exception as e:
print "Error refining", tag, str(e)
if not self.params.dispatch.squash_errors: raise
return
from diffuse_scattering.sematura import DiffuseExperiment, DiffuseImage
def from_experiments(self, experiments):
exp_xtal = experiments.crystals()[0]
### define useful attributes
self.crystal = exp_xtal
uc = self.crystal.get_unit_cell()
uc_nospace = str(uc).replace(" ", "")
uc_nospace_noparen = uc_nospace[1:-1]
self.unit_cell = uc_nospace_noparen
self.space_group = self.crystal.get_space_group()
self.laue_group = self.space_group.laue_group_type()
# self.a_matrix = crystal.get_A()
self.experiments = experiments
DiffuseExperiment.from_experiments = from_experiments
if (self.params.mp.method == 'mpi'):
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank(
) # each process in MPI has a unique id, 0-indexed
else:
rank = 0
test_exp = DiffuseExperiment()
test_exp.from_experiments(experiments)
img_set = test_exp.experiments.imagesets()
imgs = img_set[0]
file_list = imgs.paths()
img_file = file_list[0]
test_img = DiffuseImage(img_file)
test_img.set_general_variables()
test_img.remove_bragg_peaks(radial=True)
if self.ncalls == 0:
if rank == 0:
self.ref_data = deepcopy(test_img.lunus_data_scitbx)
# print "ref_data is",self.ref_data
# else:
# self.ref_data = None
if (self.params.mp.method == 'mpi'):
# from mpi4py import MPI
# comm = MPI.COMM_WORLD
# print "Barrier, rank = ",rank
# print "Broadcast, rank = ",rank
self.ref_data = comm.bcast(self.ref_data, root=0)
comm.barrier()
# print "Broadcast done, rank = ",rank
if self.ref_data == None:
print "ref_data = None for Rank = ", rank
test_img.scale_factor_from_images(self.ref_data)
# test_img.scale_factor()
test_img.crystal_geometry(test_exp.crystal)
test_img.setup_diffuse_lattice(self.params.lunus.d_min)
test_img.integrate_diffuse()
self.ncalls = self.ncalls + 1