def run_indexing(datablock, strong_spots, crystal_model, rmsds):
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory(suffix="test_dials_index"))
os.chdir(tmp_dir)
sweep_path = os.path.join(tmp_dir, "datablock.json")
pickle_path = os.path.join(tmp_dir, "strong.pickle")
dump.datablock(datablock, sweep_path)
easy_pickle.dump(pickle_path, strong_spots)
from dials.test.algorithms.indexing.tst_index import run_one_indexing
space_group_info = crystal_model.get_space_group()
symmetry = crystal.symmetry(unit_cell=crystal_model.get_unit_cell(),
space_group=crystal_model.get_space_group())
expected_rmsds = [1.1*r for r in rmsds]
imageset = datablock[0].extract_imagesets()[0]
pixel_size = imageset.get_detector()[0].get_pixel_size()
phi_width = imageset.get_scan().get_oscillation()[1] * math.pi/180
expected_rmsds = [1.1 * rmsds[0] * pixel_size[0],
1.1 * rmsds[1] * pixel_size[1],
1.1 * rmsds[2] * phi_width]
run_one_indexing(pickle_path=pickle_path, sweep_path=sweep_path,
extra_args=[],
expected_unit_cell=symmetry.minimum_cell().unit_cell(),
expected_rmsds=expected_rmsds,
#expected_hall_symbol=crystal_model.get_space_group().type().hall_symbol(),
expected_hall_symbol=' P 1',
)
def run_indexing(datablock, strong_spots, crystal_model, rmsds):
sweep_path = "datablock.json"
pickle_path = "strong.pickle"
dump.datablock(datablock, sweep_path)
easy_pickle.dump(pickle_path, strong_spots)
space_group_info = crystal_model.get_space_group()
symmetry = crystal.symmetry(unit_cell=crystal_model.get_unit_cell(),
space_group=crystal_model.get_space_group())
expected_rmsds = [1.1 * r for r in rmsds]
imageset = datablock[0].extract_imagesets()[0]
pixel_size = imageset.get_detector()[0].get_pixel_size()
phi_width = imageset.get_scan().get_oscillation()[1] * math.pi / 180
expected_rmsds = [
1.1 * rmsds[0] * pixel_size[0], 1.1 * rmsds[1] * pixel_size[1],
1.1 * rmsds[2] * phi_width
]
run_one_indexing(
pickle_path=pickle_path,
sweep_path=sweep_path,
extra_args=[],
expected_unit_cell=symmetry.minimum_cell().unit_cell(),
expected_rmsds=expected_rmsds,
expected_hall_symbol=' P 1',
)
def run(args):
import libtbx.load_env
from dials.util import log
usage = "%s [options] datablock.json strong.pickle" % libtbx.env.dispatcher_name
parser = OptionParser(usage=usage,
phil=phil_scope,
read_datablocks=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=False)
datablocks = flatten_datablocks(params.input.datablock)
reflections = flatten_reflections(params.input.reflections)
if len(datablocks) == 0 or len(reflections) == 0:
parser.print_help()
exit(0)
# Configure the logging
log.config(info=params.output.log, debug=params.output.debug_log)
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
imagesets = []
for datablock in datablocks:
imagesets.extend(datablock.extract_imagesets())
assert len(imagesets) > 0
assert len(reflections) == len(imagesets)
if params.scan_range is not None and len(params.scan_range) > 0:
reflections = [
filter_reflections_by_scan_range(refl, params.scan_range)
for refl in reflections
]
dps_params = dps_phil_scope.extract()
# for development, we want an exhaustive plot of beam probability map:
dps_params.indexing.plot_search_scope = params.plot_search_scope
dps_params.indexing.mm_search_scope = params.mm_search_scope
new_detector, new_beam = discover_better_experimental_model(
imagesets,
reflections,
params,
dps_params,
nproc=params.nproc,
wide_search_binning=params.wide_search_binning)
for imageset in imagesets:
imageset.set_detector(new_detector)
imageset.set_beam(new_beam)
from dxtbx.serialize import dump
dump.datablock(datablock, params.output.datablock)
def run(args):
import libtbx.load_env
from libtbx.utils import Sorry
from dials.util import log
usage = "%s [options] datablock.json strong.pickle" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_datablocks=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=False)
datablocks = flatten_datablocks(params.input.datablock)
reflections = flatten_reflections(params.input.reflections)
if len(datablocks) == 0 or len(reflections) == 0:
parser.print_help()
exit(0)
# Configure the logging
log.config(
info=params.output.log,
debug=params.output.debug_log)
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil is not '':
info('The following parameters have been modified:\n')
info(diff_phil)
imagesets = []
for datablock in datablocks:
imagesets.extend(datablock.extract_imagesets())
assert len(imagesets) > 0
assert len(reflections) == len(imagesets)
if params.scan_range is not None and len(params.scan_range) > 0:
reflections = [
filter_reflections_by_scan_range(refl, params.scan_range)
for refl in reflections]
dps_params = dps_phil_scope.extract()
# for development, we want an exhaustive plot of beam probability map:
dps_params.indexing.plot_search_scope = params.plot_search_scope
dps_params.indexing.mm_search_scope = params.mm_search_scope
new_detector, new_beam = discover_better_experimental_model(
imagesets, reflections, params, dps_params, nproc=params.nproc,
wide_search_binning=params.wide_search_binning)
for imageset in imagesets:
imageset.set_detector(new_detector)
imageset.set_beam(new_beam)
from dxtbx.serialize import dump
dump.datablock(datablock, params.output.datablock)
def save_datablock(filename):
from xia2.Schema import imageset_cache
from dxtbx.datablock import DataBlock
from dxtbx.serialize import dump
datablock = DataBlock([])
for imagesets in imageset_cache.values():
for imageset in imagesets.values():
datablock.append(imageset)
dump.datablock(datablock, filename, compact=True)
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_datablocks
from dials.util.options import flatten_experiments
import libtbx.load_env
usage = "%s [options] datablock.json | experiments.json" %(
libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_datablocks=True,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
datablocks = flatten_datablocks(params.input.datablock)
if len(experiments) == 0 and len(datablocks) == 0:
parser.print_help()
exit(0)
from dials.command_line.dials_import import ManualGeometryUpdater
update_geometry = ManualGeometryUpdater(params)
if len(experiments):
imagesets = experiments.imagesets()
elif len(datablocks):
assert len(datablocks) == 1
imagesets = datablocks[0].extract_imagesets()
for imageset in imagesets:
imageset_new = update_geometry(imageset)
imageset.set_detector(imageset_new.get_detector())
imageset.set_beam(imageset_new.get_beam())
imageset.set_goniometer(imageset_new.get_goniometer())
imageset.set_scan(imageset_new.get_scan())
from dxtbx.serialize import dump
if len(experiments):
print "Saving modified experiments to %s" %params.output.experiments
dump.experiment_list(experiments, params.output.experiments)
elif len(datablocks):
print "Saving modified datablock to %s" %params.output.datablock
dump.datablock(datablocks, params.output.datablock)
def run_indexing(datablock, strong_spots, crystal_model, rmsds):
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory(suffix="test_dials_index"))
os.chdir(tmp_dir)
sweep_path = os.path.join(tmp_dir, "datablock.json")
pickle_path = os.path.join(tmp_dir, "strong.pickle")
dump.datablock(datablock, sweep_path)
easy_pickle.dump(pickle_path, strong_spots)
from dials.test.algorithms.indexing.tst_index import run_one_indexing
space_group_info = crystal_model.get_space_group()
symmetry = crystal.symmetry(unit_cell=crystal_model.get_unit_cell(),
space_group=crystal_model.get_space_group())
expected_rmsds = [1.1 * r for r in rmsds]
imageset = datablock[0].extract_imagesets()[0]
pixel_size = imageset.get_detector()[0].get_pixel_size()
phi_width = imageset.get_scan().get_oscillation()[1] * math.pi / 180
expected_rmsds = [
1.1 * rmsds[0] * pixel_size[0], 1.1 * rmsds[1] * pixel_size[1],
1.1 * rmsds[2] * phi_width
]
run_one_indexing(
pickle_path=pickle_path,
sweep_path=sweep_path,
extra_args=[],
expected_unit_cell=symmetry.minimum_cell().unit_cell(),
expected_rmsds=expected_rmsds,
#expected_hall_symbol=crystal_model.get_space_group().type().hall_symbol(),
expected_hall_symbol=' P 1',
)
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_datablocks
import libtbx.load_env
usage = "%s [options] datablock.json reference=reference_datablock.json" % (
libtbx.env.dispatcher_name
)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_datablocks=True,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
if len(datablocks) == 0:
parser.print_help()
exit()
# Load reference geometry
reference_detector = None
if params.input.reference is not None:
from dxtbx.serialize import load
try:
reference_experiments = load.experiment_list(
params.input.reference, check_format=False
)
assert len(reference_experiments.detectors()) == 1
reference_detector = reference_experiments.detectors()[0]
except Exception as e:
reference_datablocks = load.datablock(params.input.reference)
assert len(reference_datablocks) == 1
imageset = reference_datablocks[0].extract_imagesets()[0]
reference_detector = imageset.get_detector()
assert len(datablocks) == 1
imageset = datablocks[0].extract_imagesets()[0]
detector = imageset.get_detector()
h = detector.hierarchy()
href = reference_detector.hierarchy()
assert len(h) == len(href)
assert (
abs(h.get_directed_distance() - href.get_directed_distance())
< params.max_delta_distance
)
for panel, panel_ref in zip(h.children(), href.children()):
panel.set_local_frame(
panel_ref.get_local_fast_axis(),
panel_ref.get_local_slow_axis(),
panel_ref.get_local_origin(),
)
print("Writing metrology-corrected datablock to %s" % params.output.datablock)
from dxtbx.serialize import dump
dump.datablock(datablocks, params.output.datablock)
return
def _index_prepare(self):
from xia2.Handlers.Citations import Citations
Citations.cite('dials')
#all_images = self.get_matching_images()
#first = min(all_images)
#last = max(all_images)
spot_lists = []
datablocks = []
for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps):
Chatter.banner('Spotfinding %s' % xsweep.get_name())
first, last = imageset.get_scan().get_image_range()
# at this stage, break out to run the DIALS code: this sets itself up
# now cheat and pass in some information... save re-reading all of the
# image headers
# FIXME need to adjust this to allow (say) three chunks of images
from dxtbx.serialize import dump
from dxtbx.datablock import DataBlock
sweep_filename = os.path.join(
self.get_working_directory(),
'%s_datablock.json' % xsweep.get_name())
dump.datablock(DataBlock([imageset]), sweep_filename)
genmask = self.GenerateMask()
genmask.set_input_datablock(sweep_filename)
genmask.set_output_datablock(
os.path.join(
self.get_working_directory(), '%s_%s_datablock.json' %
(genmask.get_xpid(), xsweep.get_name())))
genmask.set_params(PhilIndex.params.dials.masking)
sweep_filename, mask_pickle = genmask.run()
Debug.write('Generated mask for %s: %s' %
(xsweep.get_name(), mask_pickle))
gain = PhilIndex.params.xia2.settings.input.gain
if gain is libtbx.Auto:
gain_estimater = self.EstimateGain()
gain_estimater.set_sweep_filename(sweep_filename)
gain_estimater.run()
gain = gain_estimater.get_gain()
Chatter.write('Estimated gain: %.2f' % gain)
PhilIndex.params.xia2.settings.input.gain = gain
# FIXME this should really use the assigned spot finding regions
#offset = self.get_frame_offset()
spotfinder = self.Spotfinder()
if last - first > 10:
spotfinder.set_write_hot_mask(True)
spotfinder.set_input_sweep_filename(sweep_filename)
spotfinder.set_output_sweep_filename(
'%s_%s_datablock.json' %
(spotfinder.get_xpid(), xsweep.get_name()))
spotfinder.set_input_spot_filename(
'%s_%s_strong.pickle' %
(spotfinder.get_xpid(), xsweep.get_name()))
if PhilIndex.params.dials.fast_mode:
wedges = self._index_select_images_i(imageset)
spotfinder.set_scan_ranges(wedges)
else:
spotfinder.set_scan_ranges([(first, last)])
if PhilIndex.params.dials.find_spots.phil_file is not None:
spotfinder.set_phil_file(
PhilIndex.params.dials.find_spots.phil_file)
min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size
if min_spot_size is libtbx.Auto:
if imageset.get_detector()[0].get_type() == 'SENSOR_PAD':
min_spot_size = 3
else:
min_spot_size = None
if min_spot_size is not None:
spotfinder.set_min_spot_size(min_spot_size)
min_local = PhilIndex.params.dials.find_spots.min_local
if min_local is not None:
spotfinder.set_min_local(min_local)
sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong
if sigma_strong:
spotfinder.set_sigma_strong(sigma_strong)
gain = PhilIndex.params.xia2.settings.input.gain
if gain:
spotfinder.set_gain(gain)
filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings
if filter_ice_rings:
spotfinder.set_filter_ice_rings(filter_ice_rings)
kernel_size = PhilIndex.params.dials.find_spots.kernel_size
if kernel_size:
spotfinder.set_kernel_size(kernel_size)
global_threshold = PhilIndex.params.dials.find_spots.global_threshold
if global_threshold is not None:
spotfinder.set_global_threshold(global_threshold)
spotfinder.run()
spot_filename = spotfinder.get_spot_filename()
if not os.path.exists(spot_filename):
raise RuntimeError("Spotfinding failed: %s does not exist." %
os.path.basename(spot_filename))
spot_lists.append(spot_filename)
datablocks.append(spotfinder.get_output_sweep_filename())
from libtbx import easy_pickle
from dials.util.ascii_art import spot_counts_per_image_plot
refl = easy_pickle.load(spot_filename)
if not len(refl):
raise RuntimeError('No spots found in sweep %s' %
xsweep.get_name())
Chatter.write(spot_counts_per_image_plot(refl), strip=False)
if not PhilIndex.params.dials.fast_mode:
detectblanks = self.DetectBlanks()
detectblanks.set_sweep_filename(datablocks[-1])
detectblanks.set_reflections_filename(spot_filename)
detectblanks.run()
json = detectblanks.get_results()
offset = imageset.get_scan().get_image_range()[0]
blank_regions = json['strong']['blank_regions']
if len(blank_regions):
blank_regions = [(int(s), int(e))
for s, e in blank_regions]
for blank_start, blank_end in blank_regions:
Chatter.write(
'WARNING: Potential blank images: %i -> %i' %
(blank_start + 1, blank_end))
if PhilIndex.params.xia2.settings.remove_blanks:
non_blanks = []
start, end = imageset.get_array_range()
last_blank_end = start
for blank_start, blank_end in blank_regions:
if blank_start > start:
non_blanks.append(
(last_blank_end, blank_start))
last_blank_end = blank_end
if last_blank_end + 1 < end:
non_blanks.append((last_blank_end, end))
xsweep = self.get_indexer_sweep()
xwav = xsweep.get_wavelength()
xsample = xsweep.get_xsample()
sweep_name = xsweep.get_name()
import string
for i, (nb_start, nb_end) in enumerate(non_blanks):
assert i < 26
if i == 0:
sub_imageset = imageset[nb_start -
start:nb_end - start]
xsweep._frames_to_process = (nb_start + 1,
nb_end + 1)
self.set_indexer_prepare_done(done=False)
self._indxr_imagesets[
self._indxr_imagesets.index(
imageset)] = sub_imageset
xsweep._integrater._setup_from_imageset(
sub_imageset)
else:
new_name = '_'.join(
(sweep_name, string.ascii_lowercase[i]))
new_sweep = xwav.add_sweep(
new_name,
xsample,
directory=os.path.join(
os.path.basename(
xsweep.get_directory()), new_name),
image=imageset.get_path(nb_start - start),
frames_to_process=(nb_start + 1, nb_end),
)
Chatter.write(
"Generating new sweep: %s (%s:%i:%i)" %
(new_sweep.get_name(),
new_sweep.get_image(),
new_sweep.get_frames_to_process()[0],
new_sweep.get_frames_to_process()[1]))
return
if not PhilIndex.params.xia2.settings.trust_beam_centre:
discovery = self.DiscoverBetterExperimentalModel()
discovery.set_sweep_filename(datablocks[-1])
discovery.set_spot_filename(spot_filename)
#wedges = self._index_select_images_i(imageset)
#discovery.set_scan_ranges(wedges)
#discovery.set_scan_ranges([(first + offset, last + offset)])
try:
discovery.run()
except Exception as e:
Debug.write('DIALS beam centre search failed: %s' % str(e))
else:
# overwrite datablock.json in datablocks list
datablocks[
-1] = discovery.get_optimized_datablock_filename()
self.set_indexer_payload("spot_lists", spot_lists)
self.set_indexer_payload("datablocks", datablocks)
return
def _index_prepare(self):
'''Prepare to do autoindexing - in XDS terms this will mean
calling xycorr, init and colspot on the input images.'''
# decide on images to work with
Debug.write('XDS INDEX PREPARE:')
Debug.write('Wavelength: %.6f' % self.get_wavelength())
Debug.write('Distance: %.2f' % self.get_distance())
if self._indxr_images == []:
_select_images_function = getattr(
self, '_index_select_images_%s' % (self._index_select_images))
wedges = _select_images_function()
for wedge in wedges:
self.add_indexer_image_wedge(wedge)
self.set_indexer_prepare_done(True)
all_images = self.get_matching_images()
first = min(all_images)
last = max(all_images)
# next start to process these - first xycorr
xycorr = self.Xycorr()
xycorr.set_data_range(first, last)
xycorr.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
xycorr.set_beam_centre(xds_beam_centre[0],
xds_beam_centre[1])
for block in self._indxr_images:
xycorr.add_spot_range(block[0], block[1])
# FIXME need to set the origin here
xycorr.run()
for file in ['X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf']:
self._indxr_payload[file] = xycorr.get_output_data_file(file)
# next start to process these - then init
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
# find the region of the scan with the least predicted shadow
# to use for background determination in XDS INIT step
from dxtbx.serialize import dump
from dxtbx.datablock import DataBlock
imageset = self._indxr_imagesets[0]
xsweep = self._indxr_sweeps[0]
sweep_filename = os.path.join(
self.get_working_directory(), '%s_datablock.json' %xsweep.get_name())
dump.datablock(DataBlock([imageset]), sweep_filename)
from xia2.Wrappers.Dials.ShadowPlot import ShadowPlot
shadow_plot = ShadowPlot()
shadow_plot.set_working_directory(self.get_working_directory())
auto_logfiler(shadow_plot)
shadow_plot.set_sweep_filename(sweep_filename)
shadow_plot.set_json_filename(
os.path.join(
self.get_working_directory(),
'%s_shadow_plot.json' %shadow_plot.get_xpid()))
shadow_plot.run()
results = shadow_plot.get_results()
from scitbx.array_family import flex
fraction_shadowed = flex.double(results['fraction_shadowed'])
scan_points = flex.double(results['scan_points'])
phi_width = self.get_phi_width()
scan = imageset.get_scan()
oscillation_range = scan.get_oscillation_range()
oscillation = scan.get_oscillation()
bg_images = self._background_images
bg_range_deg = (scan.get_angle_from_image_index(bg_images[0]),
scan.get_angle_from_image_index(bg_images[1]))
bg_range_width = bg_range_deg[1] - bg_range_deg[0]
min_shadow = 100
best_bg_range = bg_range_deg
from libtbx.utils import frange
for bg_range_start in frange(flex.min(scan_points), flex.max(scan_points) - bg_range_width, step=oscillation[1]):
bg_range_deg = (bg_range_start, bg_range_start + bg_range_width)
sel = (scan_points >= bg_range_deg[0]) & (scan_points <= bg_range_deg[1])
mean_shadow = flex.mean(fraction_shadowed.select(sel))
if mean_shadow < min_shadow:
min_shadow = mean_shadow
best_bg_range = bg_range_deg
self._background_images = (
scan.get_image_index_from_angle(best_bg_range[0]),
scan.get_image_index_from_angle(best_bg_range[1]))
Debug.write('Setting background images: %s -> %s' %self._background_images)
init = self.Init()
for file in ['X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf']:
init.set_input_data_file(file, self._indxr_payload[file])
init.set_data_range(first, last)
if self._background_images:
init.set_background_range(self._background_images[0],
self._background_images[1])
else:
init.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images:
init.add_spot_range(block[0], block[1])
init.run()
# at this stage, need to (perhaps) modify the BKGINIT.cbf image
# to mark out the back stop
if PhilIndex.params.general.backstop_mask:
Debug.write('Applying mask to BKGINIT.pck')
# copy the original file
cbf_old = os.path.join(init.get_working_directory(),
'BKGINIT.cbf')
cbf_save = os.path.join(init.get_working_directory(),
'BKGINIT.sav')
shutil.copyfile(cbf_old, cbf_save)
# modify the file to give the new mask
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask.apply_mask_xds(self.get_header(), cbf_save, cbf_old)
init.reload()
for file in ['BLANK.cbf',
'BKGINIT.cbf',
'GAIN.cbf']:
self._indxr_payload[file] = init.get_output_data_file(file)
if PhilIndex.params.xia2.settings.developmental.use_dials_spotfinder:
spotfinder = self.DialsSpotfinder()
for block in self._indxr_images:
spotfinder.add_spot_range(block[0], block[1])
spotfinder.run()
export = self.DialsExportSpotXDS()
export.set_input_data_file(
'reflections.pickle',
spotfinder.get_output_data_file('reflections.pickle'))
export.run()
for file in ['SPOT.XDS']:
self._indxr_payload[file] = export.get_output_data_file(file)
else:
# next start to process these - then colspot
colspot = self.Colspot()
for file in ['X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf',
'BLANK.cbf',
'BKGINIT.cbf',
'GAIN.cbf']:
colspot.set_input_data_file(file, self._indxr_payload[file])
colspot.set_data_range(first, last)
colspot.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images:
colspot.add_spot_range(block[0], block[1])
colspot.run()
for file in ['SPOT.XDS']:
self._indxr_payload[file] = colspot.get_output_data_file(file)
# that should be everything prepared... all of the important
# files should be loaded into memory to be able to cope with
# integration happening somewhere else
return
def _index_prepare(self):
'''Prepare to do autoindexing - in XDS terms this will mean
calling xycorr, init and colspot on the input images.'''
# decide on images to work with
Debug.write('XDS INDEX PREPARE:')
Debug.write('Wavelength: %.6f' % self.get_wavelength())
Debug.write('Distance: %.2f' % self.get_distance())
if self._indxr_images == []:
_select_images_function = getattr(
self, '_index_select_images_%s' % (self._index_select_images))
wedges = _select_images_function()
for wedge in wedges:
self.add_indexer_image_wedge(wedge)
self.set_indexer_prepare_done(True)
all_images = self.get_matching_images()
first = min(all_images)
last = max(all_images)
# next start to process these - first xycorr
xycorr = self.Xycorr()
xycorr.set_data_range(first, last)
xycorr.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
xycorr.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
for block in self._indxr_images:
xycorr.add_spot_range(block[0], block[1])
# FIXME need to set the origin here
xycorr.run()
for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']:
self._indxr_payload[file] = xycorr.get_output_data_file(file)
# next start to process these - then init
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
imageset = self._indxr_imagesets[0]
masker = imageset.masker().format_class(
imageset.paths()[0]).get_goniometer_shadow_masker()
if masker is None:
# disable dynamic_shadowing
PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
# find the region of the scan with the least predicted shadow
# to use for background determination in XDS INIT step
from dxtbx.serialize import dump
from dxtbx.datablock import DataBlock
imageset = self._indxr_imagesets[0]
xsweep = self._indxr_sweeps[0]
sweep_filename = os.path.join(
self.get_working_directory(),
'%s_datablock.json' % xsweep.get_name())
dump.datablock(DataBlock([imageset]), sweep_filename)
from xia2.Wrappers.Dials.ShadowPlot import ShadowPlot
shadow_plot = ShadowPlot()
shadow_plot.set_working_directory(self.get_working_directory())
auto_logfiler(shadow_plot)
shadow_plot.set_sweep_filename(sweep_filename)
shadow_plot.set_json_filename(
os.path.join(self.get_working_directory(),
'%s_shadow_plot.json' % shadow_plot.get_xpid()))
shadow_plot.run()
results = shadow_plot.get_results()
from scitbx.array_family import flex
fraction_shadowed = flex.double(results['fraction_shadowed'])
if flex.max(fraction_shadowed) == 0:
PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False
else:
scan_points = flex.double(results['scan_points'])
phi_width = self.get_phi_width()
scan = imageset.get_scan()
oscillation_range = scan.get_oscillation_range()
oscillation = scan.get_oscillation()
if self._background_images is not None:
bg_images = self._background_images
bg_range_deg = (scan.get_angle_from_image_index(
bg_images[0]),
scan.get_angle_from_image_index(
bg_images[1]))
bg_range_width = bg_range_deg[1] - bg_range_deg[0]
min_shadow = 100
best_bg_range = bg_range_deg
from libtbx.utils import frange
for bg_range_start in frange(flex.min(scan_points),
flex.max(scan_points) -
bg_range_width,
step=oscillation[1]):
bg_range_deg = (bg_range_start,
bg_range_start + bg_range_width)
sel = (scan_points >= bg_range_deg[0]) & (
scan_points <= bg_range_deg[1])
mean_shadow = flex.mean(fraction_shadowed.select(sel))
if mean_shadow < min_shadow:
min_shadow = mean_shadow
best_bg_range = bg_range_deg
self._background_images = (scan.get_image_index_from_angle(
best_bg_range[0]),
scan.get_image_index_from_angle(
best_bg_range[1]))
Debug.write('Setting background images: %s -> %s' %
self._background_images)
init = self.Init()
for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']:
init.set_input_data_file(file, self._indxr_payload[file])
init.set_data_range(first, last)
if self._background_images:
init.set_background_range(self._background_images[0],
self._background_images[1])
else:
init.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images:
init.add_spot_range(block[0], block[1])
init.run()
# at this stage, need to (perhaps) modify the BKGINIT.cbf image
# to mark out the back stop
if PhilIndex.params.general.backstop_mask:
Debug.write('Applying mask to BKGINIT.pck')
# copy the original file
cbf_old = os.path.join(init.get_working_directory(), 'BKGINIT.cbf')
cbf_save = os.path.join(init.get_working_directory(),
'BKGINIT.sav')
shutil.copyfile(cbf_old, cbf_save)
# modify the file to give the new mask
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask.apply_mask_xds(self.get_header(), cbf_save, cbf_old)
init.reload()
for file in ['BLANK.cbf', 'BKGINIT.cbf', 'GAIN.cbf']:
self._indxr_payload[file] = init.get_output_data_file(file)
if PhilIndex.params.xia2.settings.developmental.use_dials_spotfinder:
spotfinder = self.DialsSpotfinder()
for block in self._indxr_images:
spotfinder.add_spot_range(block[0], block[1])
spotfinder.run()
export = self.DialsExportSpotXDS()
export.set_input_data_file(
'reflections.pickle',
spotfinder.get_output_data_file('reflections.pickle'))
export.run()
for file in ['SPOT.XDS']:
self._indxr_payload[file] = export.get_output_data_file(file)
else:
# next start to process these - then colspot
colspot = self.Colspot()
for file in [
'X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf', 'BLANK.cbf',
'BKGINIT.cbf', 'GAIN.cbf'
]:
colspot.set_input_data_file(file, self._indxr_payload[file])
colspot.set_data_range(first, last)
colspot.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images:
colspot.add_spot_range(block[0], block[1])
colspot.run()
for file in ['SPOT.XDS']:
self._indxr_payload[file] = colspot.get_output_data_file(file)
# that should be everything prepared... all of the important
# files should be loaded into memory to be able to cope with
# integration happening somewhere else
return
imageset = reference_datablocks[0].extract_imagesets()[0]
reference_detector = imageset.get_detector()
assert len(datablocks) == 1
imageset = datablocks[0].extract_imagesets()[0]
detector = imageset.get_detector()
h = detector.hierarchy()
href = reference_detector.hierarchy()
assert len(h) == len(href)
assert abs(h.get_directed_distance() - href.get_directed_distance()) < params.max_delta_distance
for panel, panel_ref in zip(h.children(), href.children()):
panel.set_local_frame(
panel_ref.get_local_fast_axis(),
panel_ref.get_local_slow_axis(),
panel_ref.get_local_origin()
)
print 'Writing metrology-corrected datablock to %s' %params.output.datablock
from dxtbx.serialize import dump
dump.datablock(datablocks, params.output.datablock)
return
if __name__ == '__main__':
import sys
run(sys.argv[1:])
def _index_prepare(self):
from xia2.Handlers.Citations import Citations
Citations.cite('dials')
#all_images = self.get_matching_images()
#first = min(all_images)
#last = max(all_images)
spot_lists = []
datablocks = []
for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps):
Chatter.banner('Spotfinding %s' %xsweep.get_name())
first, last = imageset.get_scan().get_image_range()
# at this stage, break out to run the DIALS code: this sets itself up
# now cheat and pass in some information... save re-reading all of the
# image headers
# FIXME need to adjust this to allow (say) three chunks of images
from dxtbx.serialize import dump
from dxtbx.datablock import DataBlock
sweep_filename = os.path.join(
self.get_working_directory(), '%s_datablock.json' %xsweep.get_name())
dump.datablock(DataBlock([imageset]), sweep_filename)
gain = PhilIndex.params.xia2.settings.input.gain
if gain is libtbx.Auto:
gain_estimater = self.EstimateGain()
gain_estimater.set_sweep_filename(sweep_filename)
gain_estimater.run()
gain = gain_estimater.get_gain()
Chatter.write('Estimated gain: %.2f' %gain)
PhilIndex.params.xia2.settings.input.gain = gain
# FIXME this should really use the assigned spot finding regions
#offset = self.get_frame_offset()
spotfinder = self.Spotfinder()
if last - first > 10:
spotfinder.set_write_hot_mask(True)
spotfinder.set_input_sweep_filename(sweep_filename)
spotfinder.set_output_sweep_filename(
'%s_%s_datablock.json' %(spotfinder.get_xpid(), xsweep.get_name()))
spotfinder.set_input_spot_filename(
'%s_%s_strong.pickle' %(spotfinder.get_xpid(), xsweep.get_name()))
if PhilIndex.params.dials.fast_mode:
wedges = self._index_select_images_i(imageset)
spotfinder.set_scan_ranges(wedges)
else:
spotfinder.set_scan_ranges([(first, last)])
if PhilIndex.params.dials.find_spots.phil_file is not None:
spotfinder.set_phil_file(PhilIndex.params.dials.find_spots.phil_file)
min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size
if min_spot_size is libtbx.Auto:
if imageset.get_detector()[0].get_type() == 'SENSOR_PAD':
min_spot_size = 3
else:
min_spot_size = None
if min_spot_size is not None:
spotfinder.set_min_spot_size(min_spot_size)
min_local = PhilIndex.params.dials.find_spots.min_local
if min_local is not None:
spotfinder.set_min_local(min_local)
sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong
if sigma_strong:
spotfinder.set_sigma_strong(sigma_strong)
gain = PhilIndex.params.xia2.settings.input.gain
if gain:
spotfinder.set_gain(gain)
filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings
if filter_ice_rings:
spotfinder.set_filter_ice_rings(filter_ice_rings)
kernel_size = PhilIndex.params.dials.find_spots.kernel_size
if kernel_size:
spotfinder.set_kernel_size(kernel_size)
global_threshold = PhilIndex.params.dials.find_spots.global_threshold
if global_threshold is not None:
spotfinder.set_global_threshold(global_threshold)
spotfinder.run()
spot_filename = spotfinder.get_spot_filename()
if not os.path.exists(spot_filename):
raise RuntimeError("Spotfinding failed: %s does not exist."
%os.path.basename(spot_filename))
spot_lists.append(spot_filename)
datablocks.append(spotfinder.get_output_sweep_filename())
from libtbx import easy_pickle
from dials.util.ascii_art import spot_counts_per_image_plot
refl = easy_pickle.load(spot_filename)
if not len(refl):
raise RuntimeError('No spots found in sweep %s' %xsweep.get_name())
Chatter.write(spot_counts_per_image_plot(refl), strip=False)
if not PhilIndex.params.dials.fast_mode:
detectblanks = self.DetectBlanks()
detectblanks.set_sweep_filename(datablocks[-1])
detectblanks.set_reflections_filename(spot_filename)
detectblanks.run()
json = detectblanks.get_results()
offset = imageset.get_scan().get_image_range()[0]
blank_regions = json['strong']['blank_regions']
if len(blank_regions):
blank_regions = [(int(s), int(e)) for s, e in blank_regions]
for blank_start, blank_end in blank_regions:
Chatter.write('WARNING: Potential blank images: %i -> %i' %(
blank_start+1, blank_end))
if PhilIndex.params.xia2.settings.remove_blanks:
non_blanks = []
start, end = imageset.get_array_range()
last_blank_end = start
for blank_start, blank_end in blank_regions:
if blank_start > start:
non_blanks.append((last_blank_end, blank_start))
last_blank_end = blank_end
if last_blank_end+1 < end:
non_blanks.append((last_blank_end, end))
xsweep = self.get_indexer_sweep()
xwav = xsweep.get_wavelength()
xsample = xsweep.get_xsample()
sweep_name = xsweep.get_name()
import string
for i, (nb_start, nb_end) in enumerate(non_blanks):
assert i < 26
if i == 0:
sub_imageset = imageset[nb_start-start:nb_end-start]
xsweep._frames_to_process = (nb_start+1, nb_end+1)
self.set_indexer_prepare_done(done=False)
self._indxr_imagesets[self._indxr_imagesets.index(imageset)] = sub_imageset
xsweep._integrater._setup_from_imageset(sub_imageset)
else:
new_name = '_'.join((sweep_name, string.ascii_lowercase[i]))
new_sweep = xwav.add_sweep(new_name,
xsample,
directory=os.path.join(
os.path.basename(xsweep.get_directory()), new_name),
image=imageset.get_path(nb_start-start),
frames_to_process=(nb_start+1, nb_end),
)
Chatter.write("Generating new sweep: %s (%s:%i:%i)" %(
new_sweep.get_name(),
new_sweep.get_image(),
new_sweep.get_frames_to_process()[0],
new_sweep.get_frames_to_process()[1]))
return
if not PhilIndex.params.xia2.settings.trust_beam_centre:
discovery = self.DiscoverBetterExperimentalModel()
discovery.set_sweep_filename(datablocks[-1])
discovery.set_spot_filename(spot_filename)
#wedges = self._index_select_images_i(imageset)
#discovery.set_scan_ranges(wedges)
#discovery.set_scan_ranges([(first + offset, last + offset)])
try:
discovery.run()
except Exception, e:
Debug.write('DIALS beam centre search failed: %s' %str(e))
else:
# overwrite datablock.json in datablocks list
datablocks[-1] = discovery.get_optimized_datablock_filename()