def _mosflm_parallel_integrate(self):
'''Perform the integration as before, but this time as a
number of parallel Mosflm jobs (hence, in separate directories)
and including a step of pre-refinement of the mosaic spread and
missets. This will all be kind of explicit and hence probably
messy!'''
refinr = self.get_integrater_refiner()
lattice = refinr.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup(lattice)
mosaic = refinr.get_refiner_payload('mosaic')
beam = refinr.get_refiner_payload('beam')
distance = refinr.get_refiner_payload('distance')
matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')
integration_params = refinr.get_refiner_payload(
'mosflm_integration_parameters')
if integration_params:
if 'separation' in integration_params:
self.set_integrater_parameter(
'mosflm', 'separation',
'%s %s' % tuple(integration_params['separation']))
if 'raster' in integration_params:
self.set_integrater_parameter(
'mosflm', 'raster',
'%d %d %d %d %d' % tuple(integration_params['raster']))
refinr.set_refiner_payload('mosflm_integration_parameters', None)
pname, xname, dname = self.get_integrater_project_info()
# what follows below should (i) be run in separate directories
# and (ii) be repeated N=parallel times.
nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
parallel = nproc
# FIXME this is something of a kludge - if too few frames refinement
# and integration does not work well... ideally want at least 15
# frames / chunk (say)
nframes = self._intgr_wedge[1] - self._intgr_wedge[0] + 1
if parallel > nframes / 15:
parallel = nframes // 15
if not parallel:
raise RuntimeError('parallel not set')
if parallel < 2:
raise RuntimeError('parallel not parallel: %s' % parallel)
jobs = []
hklouts = []
nref = 0
# calculate the chunks to use
offset = self.get_frame_offset()
start = self._intgr_wedge[0] - offset
end = self._intgr_wedge[1] - offset
left_images = 1 + end - start
left_chunks = parallel
chunks = []
while left_images > 0:
size = left_images // left_chunks
chunks.append((start, start + size - 1))
start += size
left_images -= size
left_chunks -= 1
summary_files = []
for j in range(parallel):
# make some working directories, as necessary - chunk-(0:N-1)
wd = os.path.join(self.get_working_directory(), 'chunk-%d' % j)
if not os.path.exists(wd):
os.makedirs(wd)
job = MosflmIntegrate()
job.set_working_directory(wd)
auto_logfiler(job)
l = refinr.get_refiner_lattice()
# create the starting point
f = open(os.path.join(wd, 'xiaintegrate-%s.mat' % l), 'w')
for m in matrix:
f.write(m)
f.close()
spacegroup_number = lattice_to_spacegroup(lattice)
job.set_refine_profiles(self._mosflm_refine_profiles)
# N.B. for harvesting need to append N to dname.
if pname is not None and xname is not None and dname is not None:
Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname))
harvest_dir = self.get_working_directory()
temp_dname = '%s_%s' % \
(dname, self.get_integrater_sweep_name())
job.set_pname_xname_dname(pname, xname, temp_dname)
job.set_template(os.path.basename(self.get_template()))
job.set_directory(self.get_directory())
# check for ice - and if so, exclude (ranges taken from
# XDS documentation)
if self.get_integrater_ice() != 0:
Debug.write('Excluding ice rings')
job.set_exclude_ice(True)
# exclude specified resolution ranges
if len(self.get_integrater_excluded_regions()) != 0:
regions = self.get_integrater_excluded_regions()
Debug.write('Excluding regions: %s' % repr(regions))
job.set_exclude_regions(regions)
mask = standard_mask(self.get_detector())
for m in mask:
job.add_instruction(m)
job.set_input_mat_file('xiaintegrate-%s.mat' % l)
job.set_beam_centre(beam)
job.set_distance(distance)
job.set_space_group_number(spacegroup_number)
job.set_mosaic(mosaic)
if self.get_wavelength_prov() == 'user':
job.set_wavelength(self.get_wavelength())
parameters = self.get_integrater_parameters('mosflm')
job.update_parameters(parameters)
if self._mosflm_gain:
job.set_gain(self._mosflm_gain)
# check for resolution limits
if self._intgr_reso_high > 0.0:
job.set_d_min(self._intgr_reso_high)
if self._intgr_reso_low:
job.set_d_max(self._intgr_reso_low)
if PhilIndex.params.general.backstop_mask:
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask = mask.calculate_mask_mosflm(self.get_header())
job.set_mask(mask)
detector = self.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
job.set_limits(lim_x, lim_y)
job.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
job.set_pre_refinement(True)
job.set_image_range(chunks[j])
# these are now running so ...
jobs.append(job)
continue
# ok, at this stage I need to ...
#
# (i) accumulate the statistics as a function of batch
# (ii) mong them into a single block
#
# This is likely to be a pain in the arse!
first_integrated_batch = 1.0e6
last_integrated_batch = -1.0e6
all_residuals = []
threads = []
for j in range(parallel):
job = jobs[j]
# now wait for them to finish - first wait will really be the
# first one, then all should be finished...
thread = Background(job, 'run')
thread.start()
threads.append(thread)
mosaics = []
postref_result = {}
integrated_images_first = 1.0e6
integrated_images_last = -1.0e6
self._intgr_per_image_statistics = {}
for j in range(parallel):
thread = threads[j]
thread.stop()
job = jobs[j]
# get the log file
output = job.get_all_output()
# record a copy of it, perhaps - though not if parallel
if self.get_integrater_sweep_name() and False:
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file(
'%s %s %s %s mosflm integrate' % \
(self.get_integrater_sweep_name(),
pname, xname, '%s_%d' % (dname, j)),
job.get_log_file())
# look for things that we want to know...
# that is, the output reflection file name, the updated
# value for the gain (if present,) any warnings, errors,
# or just interesting facts.
batches = job.get_batches_out()
integrated_images_first = min(batches[0], integrated_images_first)
integrated_images_last = max(batches[1], integrated_images_last)
mosaics.extend(job.get_mosaic_spreads())
if min(mosaics) < 0:
raise IntegrationError('negative mosaic spread: %s' %
min(mosaic))
if (job.get_detector_gain_error()
and not (self.get_imageset().get_detector()[0].get_type()
== 'SENSOR_PAD')):
gain = job.get_suggested_gain()
if gain is not None:
self.set_integrater_parameter('mosflm', 'gain', gain)
self.set_integrater_export_parameter(
'mosflm', 'gain', gain)
if self._mosflm_gain:
Debug.write('GAIN updated to %f' % gain)
else:
Debug.write('GAIN found to be %f' % gain)
self._mosflm_gain = gain
self._mosflm_rerun_integration = True
hklout = job.get_hklout()
Debug.write('Integration output: %s' % hklout)
hklouts.append(hklout)
nref += job.get_nref()
# if a BGSIG error happened try not refining the
# profile and running again...
if job.get_bgsig_too_large():
if not self._mosflm_refine_profiles:
raise RuntimeError('BGSIG error with profiles fixed')
Debug.write('BGSIG error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
if job.get_getprof_error():
Debug.write('GETPROF error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
# here
# write the report for each image as .*-#$ to Chatter -
# detailed report will be written automagically to science...
self._intgr_per_image_statistics.update(
job.get_per_image_statistics())
postref_result.update(job.get_postref_result())
# inspect the output for e.g. very high weighted residuals
all_residuals.extend(job.get_residuals())
self._intgr_batches_out = (integrated_images_first,
integrated_images_last)
if mosaics and len(mosaics) > 0:
self.set_integrater_mosaic_min_mean_max(
min(mosaics),
sum(mosaics) / len(mosaics), max(mosaics))
else:
m = indxr.get_indexer_mosaic()
self.set_integrater_mosaic_min_mean_max(m, m, m)
Chatter.write(self.show_per_image_statistics())
Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
self.get_integrater_mosaic_min_mean_max())
# gather the statistics from the postrefinement for all sweeps
# now write this to a postrefinement log
postref_log = os.path.join(self.get_working_directory(),
'postrefinement.log')
fout = open(postref_log, 'w')
fout.write('$TABLE: Postrefinement for %s:\n' % \
self._intgr_sweep_name)
fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n')
fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n')
for image in sorted(postref_result):
phix = postref_result[image].get('phix', 0.0)
phiy = postref_result[image].get('phiy', 0.0)
phiz = postref_result[image].get('phiz', 0.0)
fout.write('%d %5.2f %5.2f %5.2f\n' % \
(image, phix, phiy, phiz))
fout.write('$$\n')
fout.close()
if self.get_integrater_sweep_name():
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file('%s %s %s %s postrefinement' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
postref_log)
hklouts.sort()
hklout = os.path.join(self.get_working_directory(),
os.path.split(hklouts[0])[-1])
Debug.write('Sorting data to %s' % hklout)
for hklin in hklouts:
Debug.write('<= %s' % hklin)
sortmtz = Sortmtz()
sortmtz.set_hklout(hklout)
for hklin in hklouts:
sortmtz.add_hklin(hklin)
sortmtz.sort()
self._mosflm_hklout = hklout
return self._mosflm_hklout
def _index_prepare(self):
"""Prepare to do autoindexing - in XDS terms this will mean
calling xycorr, init and colspot on the input images."""
# decide on images to work with
logger.debug("XDS INDEX PREPARE:")
logger.debug("Wavelength: %.6f", self.get_wavelength())
logger.debug("Distance: %.2f", self.get_distance())
if self._indxr_images == []:
_select_images_function = getattr(
self, "_index_select_images_%s" % self._index_select_images
)
wedges = _select_images_function()
for wedge in wedges:
self.add_indexer_image_wedge(wedge)
self.set_indexer_prepare_done(True)
all_images = self.get_matching_images()
first = min(all_images)
last = max(all_images)
# next start to process these - first xycorr
xycorr = self.Xycorr()
xycorr.set_data_range(first, last)
xycorr.set_background_range(self._indxr_images[0][0], self._indxr_images[0][1])
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
xycorr.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
for block in self._indxr_images:
xycorr.add_spot_range(block[0], block[1])
# FIXME need to set the origin here
xycorr.run()
for file in ["X-CORRECTIONS.cbf", "Y-CORRECTIONS.cbf"]:
self._indxr_payload[file] = xycorr.get_output_data_file(file)
# next start to process these - then init
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
imageset = self._indxr_imagesets[0]
masker = (
imageset.get_format_class()
.get_instance(imageset.paths()[0])
.get_masker()
)
if masker is None:
# disable dynamic_shadowing
PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
# find the region of the scan with the least predicted shadow
# to use for background determination in XDS INIT step
from dxtbx.model.experiment_list import ExperimentListFactory
imageset = self._indxr_imagesets[0]
xsweep = self._indxr_sweeps[0]
sweep_filename = os.path.join(
self.get_working_directory(), "%s_indexed.expt" % xsweep.get_name()
)
ExperimentListFactory.from_imageset_and_crystal(imageset, None).as_file(
sweep_filename
)
from xia2.Wrappers.Dials.ShadowPlot import ShadowPlot
shadow_plot = ShadowPlot()
shadow_plot.set_working_directory(self.get_working_directory())
auto_logfiler(shadow_plot)
shadow_plot.set_sweep_filename(sweep_filename)
shadow_plot.set_json_filename(
os.path.join(
self.get_working_directory(),
"%s_shadow_plot.json" % shadow_plot.get_xpid(),
)
)
shadow_plot.run()
results = shadow_plot.get_results()
fraction_shadowed = flex.double(results["fraction_shadowed"])
if flex.max(fraction_shadowed) == 0:
PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False
else:
scan_points = flex.double(results["scan_points"])
scan = imageset.get_scan()
oscillation = scan.get_oscillation()
if self._background_images is not None:
bg_images = self._background_images
bg_range_deg = (
scan.get_angle_from_image_index(bg_images[0]),
scan.get_angle_from_image_index(bg_images[1]),
)
bg_range_width = bg_range_deg[1] - bg_range_deg[0]
min_shadow = 100
best_bg_range = bg_range_deg
from libtbx.utils import frange
for bg_range_start in frange(
flex.min(scan_points),
flex.max(scan_points) - bg_range_width,
step=oscillation[1],
):
bg_range_deg = (bg_range_start, bg_range_start + bg_range_width)
sel = (scan_points >= bg_range_deg[0]) & (
scan_points <= bg_range_deg[1]
)
mean_shadow = flex.mean(fraction_shadowed.select(sel))
if mean_shadow < min_shadow:
min_shadow = mean_shadow
best_bg_range = bg_range_deg
self._background_images = (
scan.get_image_index_from_angle(best_bg_range[0]),
scan.get_image_index_from_angle(best_bg_range[1]),
)
logger.debug(
"Setting background images: %s -> %s" % self._background_images
)
init = self.Init()
for file in ["X-CORRECTIONS.cbf", "Y-CORRECTIONS.cbf"]:
init.set_input_data_file(file, self._indxr_payload[file])
init.set_data_range(first, last)
if self._background_images:
init.set_background_range(
self._background_images[0], self._background_images[1]
)
else:
init.set_background_range(
self._indxr_images[0][0], self._indxr_images[0][1]
)
for block in self._indxr_images:
init.add_spot_range(block[0], block[1])
init.run()
# at this stage, need to (perhaps) modify the BKGINIT.cbf image
# to mark out the back stop
if PhilIndex.params.xds.backstop_mask:
logger.debug("Applying mask to BKGINIT.pck")
# copy the original file
cbf_old = os.path.join(init.get_working_directory(), "BKGINIT.cbf")
cbf_save = os.path.join(init.get_working_directory(), "BKGINIT.sav")
shutil.copyfile(cbf_old, cbf_save)
# modify the file to give the new mask
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.xds.backstop_mask)
mask.apply_mask_xds(self.get_header(), cbf_save, cbf_old)
init.reload()
for file in ["BLANK.cbf", "BKGINIT.cbf", "GAIN.cbf"]:
self._indxr_payload[file] = init.get_output_data_file(file)
if PhilIndex.params.xia2.settings.developmental.use_dials_spotfinder:
spotfinder = self.DialsSpotfinder()
for block in self._indxr_images:
spotfinder.add_spot_range(block[0], block[1])
spotfinder.run()
export = self.DialsExportSpotXDS()
export.set_input_data_file(
"observations.refl",
spotfinder.get_output_data_file("observations.refl"),
)
export.run()
for file in ["SPOT.XDS"]:
self._indxr_payload[file] = export.get_output_data_file(file)
else:
# next start to process these - then colspot
colspot = self.Colspot()
for file in (
"X-CORRECTIONS.cbf",
"Y-CORRECTIONS.cbf",
"BLANK.cbf",
"BKGINIT.cbf",
"GAIN.cbf",
):
colspot.set_input_data_file(file, self._indxr_payload[file])
colspot.set_data_range(first, last)
colspot.set_background_range(
self._indxr_images[0][0], self._indxr_images[0][1]
)
for block in self._indxr_images:
colspot.add_spot_range(block[0], block[1])
colspot.run()
for file in ["SPOT.XDS"]:
self._indxr_payload[file] = colspot.get_output_data_file(file)
def _mosflm_integrate(self):
'''Perform the actual integration, based on the results of the
cell refinement or indexing (they have the equivalent form.)'''
refinr = self.get_integrater_refiner()
if not refinr.get_refiner_payload('mosflm_orientation_matrix'):
raise RuntimeError('unexpected situation in indexing')
lattice = refinr.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup(lattice)
mosaic = refinr.get_refiner_payload('mosaic')
beam = refinr.get_refiner_payload('beam')
distance = refinr.get_refiner_payload('distance')
matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')
integration_params = refinr.get_refiner_payload(
'mosflm_integration_parameters')
if integration_params:
if 'separation' in integration_params:
self.set_integrater_parameter(
'mosflm', 'separation',
'%s %s' % tuple(integration_params['separation']))
if 'raster' in integration_params:
self.set_integrater_parameter(
'mosflm', 'raster',
'%d %d %d %d %d' % tuple(integration_params['raster']))
refinr.set_refiner_payload('mosflm_integration_parameters', None)
f = open(
os.path.join(self.get_working_directory(), 'xiaintegrate.mat'),
'w')
for m in matrix:
f.write(m)
f.close()
# then start the integration
integrater = MosflmIntegrate()
integrater.set_working_directory(self.get_working_directory())
auto_logfiler(integrater)
integrater.set_refine_profiles(self._mosflm_refine_profiles)
pname, xname, dname = self.get_integrater_project_info()
if pname is not None and xname is not None and dname is not None:
Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname))
harvest_dir = self.get_working_directory()
# harvest file name will be %s.mosflm_run_start_end % dname
temp_dname = '%s_%s' % \
(dname, self.get_integrater_sweep_name())
integrater.set_pname_xname_dname(pname, xname, temp_dname)
integrater.set_template(os.path.basename(self.get_template()))
integrater.set_directory(self.get_directory())
# check for ice - and if so, exclude (ranges taken from
# XDS documentation)
if self.get_integrater_ice() != 0:
Debug.write('Excluding ice rings')
integrater.set_exclude_ice(True)
# exclude specified resolution ranges
if len(self.get_integrater_excluded_regions()) != 0:
regions = self.get_integrater_excluded_regions()
Debug.write('Excluding regions: %s' % repr(regions))
integrater.set_exclude_regions(regions)
mask = standard_mask(self.get_detector())
for m in mask:
integrater.add_instruction(m)
integrater.set_input_mat_file('xiaintegrate.mat')
integrater.set_beam_centre(beam)
integrater.set_distance(distance)
integrater.set_space_group_number(spacegroup_number)
integrater.set_mosaic(mosaic)
if self.get_wavelength_prov() == 'user':
integrater.set_wavelength(self.get_wavelength())
parameters = self.get_integrater_parameters('mosflm')
integrater.update_parameters(parameters)
if self._mosflm_gain:
integrater.set_gain(self._mosflm_gain)
# check for resolution limits
if self._intgr_reso_high > 0.0:
integrater.set_d_min(self._intgr_reso_high)
if self._intgr_reso_low:
integrater.set_d_max(self._intgr_reso_low)
if PhilIndex.params.general.backstop_mask:
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask = mask.calculate_mask_mosflm(self.get_header())
integrater.set_mask(mask)
detector = self.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
integrater.set_limits(lim_x, lim_y)
integrater.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
offset = self.get_frame_offset()
integrater.set_image_range(
(self._intgr_wedge[0] - offset, self._intgr_wedge[1] - offset))
try:
integrater.run()
except RuntimeError as e:
if 'integration failed: reason unknown' in str(e):
Chatter.write('Mosflm has failed in integration')
message = 'The input was:\n\n'
for input in integrater.get_all_input():
message += ' %s' % input
Chatter.write(message)
raise
FileHandler.record_log_file(
'%s %s %s %s mosflm integrate' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
integrater.get_log_file())
self._intgr_per_image_statistics = integrater.get_per_image_statistics(
)
self._mosflm_hklout = integrater.get_hklout()
Debug.write('Integration output: %s' % self._mosflm_hklout)
self._intgr_n_ref = integrater.get_nref()
# if a BGSIG error happened try not refining the
# profile and running again...
if integrater.get_bgsig_too_large():
if not self._mosflm_refine_profiles:
raise RuntimeError('BGSIG error with profiles fixed')
Debug.write('BGSIG error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
if integrater.get_getprof_error():
Debug.write('GETPROF error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
if (integrater.get_detector_gain_error()
and not (self.get_imageset().get_detector()[0].get_type()
== 'SENSOR_PAD')):
gain = integrater.get_suggested_gain()
if gain is not None:
self.set_integrater_parameter('mosflm', 'gain', gain)
self.set_integrater_export_parameter('mosflm', 'gain', gain)
if self._mosflm_gain:
Debug.write('GAIN updated to %f' % gain)
else:
Debug.write('GAIN found to be %f' % gain)
self._mosflm_gain = gain
self._mosflm_rerun_integration = True
if not self._mosflm_hklout:
raise RuntimeError('processing abandoned')
self._intgr_batches_out = integrater.get_batches_out()
mosaics = integrater.get_mosaic_spreads()
if mosaics and len(mosaics) > 0:
self.set_integrater_mosaic_min_mean_max(
min(mosaics),
sum(mosaics) / len(mosaics), max(mosaics))
else:
m = indxr.get_indexer_mosaic()
self.set_integrater_mosaic_min_mean_max(m, m, m)
# write the report for each image as .*-#$ to Chatter -
# detailed report will be written automagically to science...
Chatter.write(self.show_per_image_statistics())
Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
self.get_integrater_mosaic_min_mean_max())
# gather the statistics from the postrefinement
postref_result = integrater.get_postref_result()
# now write this to a postrefinement log
postref_log = os.path.join(self.get_working_directory(),
'postrefinement.log')
fout = open(postref_log, 'w')
fout.write('$TABLE: Postrefinement for %s:\n' % \
self._intgr_sweep_name)
fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n')
fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n')
for image in sorted(postref_result):
phix = postref_result[image].get('phix', 0.0)
phiy = postref_result[image].get('phiy', 0.0)
phiz = postref_result[image].get('phiz', 0.0)
fout.write('%d %5.2f %5.2f %5.2f\n' % \
(image, phix, phiy, phiz))
fout.write('$$\n')
fout.close()
if self.get_integrater_sweep_name():
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file('%s %s %s %s postrefinement' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
postref_log)
return self._mosflm_hklout
def _mosflm_parallel_integrate(self):
'''Perform the integration as before, but this time as a
number of parallel Mosflm jobs (hence, in separate directories)
and including a step of pre-refinement of the mosaic spread and
missets. This will all be kind of explicit and hence probably
messy!'''
refinr = self.get_integrater_refiner()
lattice = refinr.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup(lattice)
mosaic = refinr.get_refiner_payload('mosaic')
beam = refinr.get_refiner_payload('beam')
distance = refinr.get_refiner_payload('distance')
matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')
integration_params = refinr.get_refiner_payload(
'mosflm_integration_parameters')
if integration_params:
if 'separation' in integration_params:
self.set_integrater_parameter(
'mosflm', 'separation',
'%s %s' % tuple(integration_params['separation']))
if 'raster' in integration_params:
self.set_integrater_parameter(
'mosflm', 'raster',
'%d %d %d %d %d' % tuple(integration_params['raster']))
refinr.set_refiner_payload('mosflm_integration_parameters', None)
pname, xname, dname = self.get_integrater_project_info()
# what follows below should (i) be run in separate directories
# and (ii) be repeated N=parallel times.
nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
parallel = nproc
# FIXME this is something of a kludge - if too few frames refinement
# and integration does not work well... ideally want at least 15
# frames / chunk (say)
nframes = self._intgr_wedge[1] - self._intgr_wedge[0] + 1
if parallel > nframes / 15:
parallel = nframes // 15
if not parallel:
raise RuntimeError, 'parallel not set'
if parallel < 2:
raise RuntimeError, 'parallel not parallel: %s' % parallel
jobs = []
hklouts = []
nref = 0
# calculate the chunks to use
offset = self.get_frame_offset()
start = self._intgr_wedge[0] - offset
end = self._intgr_wedge[1] - offset
left_images = 1 + end - start
left_chunks = parallel
chunks = []
while left_images > 0:
size = left_images // left_chunks
chunks.append((start, start + size - 1))
start += size
left_images -= size
left_chunks -= 1
summary_files = []
for j in range(parallel):
# make some working directories, as necessary - chunk-(0:N-1)
wd = os.path.join(self.get_working_directory(),
'chunk-%d' % j)
if not os.path.exists(wd):
os.makedirs(wd)
job = MosflmIntegrate()
job.set_working_directory(wd)
auto_logfiler(job)
l = refinr.get_refiner_lattice()
# create the starting point
f = open(os.path.join(wd, 'xiaintegrate-%s.mat' % l), 'w')
for m in matrix:
f.write(m)
f.close()
spacegroup_number = lattice_to_spacegroup(lattice)
job.set_refine_profiles(self._mosflm_refine_profiles)
# N.B. for harvesting need to append N to dname.
if pname is not None and xname is not None and dname is not None:
Debug.write('Harvesting: %s/%s/%s' %
(pname, xname, dname))
harvest_dir = self.get_working_directory()
temp_dname = '%s_%s' % \
(dname, self.get_integrater_sweep_name())
job.set_pname_xname_dname(pname, xname, temp_dname)
job.set_template(os.path.basename(self.get_template()))
job.set_directory(self.get_directory())
# check for ice - and if so, exclude (ranges taken from
# XDS documentation)
if self.get_integrater_ice() != 0:
Debug.write('Excluding ice rings')
job.set_exclude_ice(True)
# exclude specified resolution ranges
if len(self.get_integrater_excluded_regions()) != 0:
regions = self.get_integrater_excluded_regions()
Debug.write('Excluding regions: %s' % `regions`)
job.set_exclude_regions(regions)
mask = standard_mask(self.get_detector())
for m in mask:
job.add_instruction(m)
job.set_input_mat_file('xiaintegrate-%s.mat' % l)
job.set_beam_centre(beam)
job.set_distance(distance)
job.set_space_group_number(spacegroup_number)
job.set_mosaic(mosaic)
if self.get_wavelength_prov() == 'user':
job.set_wavelength(self.get_wavelength())
parameters = self.get_integrater_parameters('mosflm')
job.update_parameters(parameters)
if self._mosflm_gain:
job.set_gain(self._mosflm_gain)
# check for resolution limits
if self._intgr_reso_high > 0.0:
job.set_d_min(self._intgr_reso_high)
if self._intgr_reso_low:
job.set_d_max(self._intgr_reso_low)
if PhilIndex.params.general.backstop_mask:
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask = mask.calculate_mask_mosflm(self.get_header())
job.set_mask(mask)
detector = self.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
job.set_limits(lim_x, lim_y)
job.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
job.set_pre_refinement(True)
job.set_image_range(chunks[j])
# these are now running so ...
jobs.append(job)
continue
# ok, at this stage I need to ...
#
# (i) accumulate the statistics as a function of batch
# (ii) mong them into a single block
#
# This is likely to be a pain in the arse!
first_integrated_batch = 1.0e6
last_integrated_batch = -1.0e6
all_residuals = []
threads = []
for j in range(parallel):
job = jobs[j]
# now wait for them to finish - first wait will really be the
# first one, then all should be finished...
thread = Background(job, 'run')
thread.start()
threads.append(thread)
mosaics = []
postref_result = { }
integrated_images_first = 1.0e6
integrated_images_last = -1.0e6
self._intgr_per_image_statistics = {}
for j in range(parallel):
thread = threads[j]
thread.stop()
job = jobs[j]
# get the log file
output = job.get_all_output()
# record a copy of it, perhaps - though not if parallel
if self.get_integrater_sweep_name() and False:
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file(
'%s %s %s %s mosflm integrate' % \
(self.get_integrater_sweep_name(),
pname, xname, '%s_%d' % (dname, j)),
job.get_log_file())
# look for things that we want to know...
# that is, the output reflection file name, the updated
# value for the gain (if present,) any warnings, errors,
# or just interesting facts.
batches = job.get_batches_out()
integrated_images_first = min(batches[0], integrated_images_first)
integrated_images_last = max(batches[1], integrated_images_last)
mosaics.extend(job.get_mosaic_spreads())
if min(mosaics) < 0:
raise IntegrationError, 'negative mosaic spread: %s' % min(mosaic)
if (job.get_detector_gain_error() and not
(self.get_imageset().get_detector()[0].get_type() == 'SENSOR_PAD')):
gain = job.get_suggested_gain()
if gain is not None:
self.set_integrater_parameter('mosflm', 'gain', gain)
self.set_integrater_export_parameter('mosflm', 'gain', gain)
if self._mosflm_gain:
Debug.write('GAIN updated to %f' % gain)
else:
Debug.write('GAIN found to be %f' % gain)
self._mosflm_gain = gain
self._mosflm_rerun_integration = True
hklout = job.get_hklout()
Debug.write('Integration output: %s' % hklout)
hklouts.append(hklout)
nref += job.get_nref()
# if a BGSIG error happened try not refining the
# profile and running again...
if job.get_bgsig_too_large():
if not self._mosflm_refine_profiles:
raise RuntimeError, 'BGSIG error with profiles fixed'
Debug.write(
'BGSIG error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
if job.get_getprof_error():
Debug.write(
'GETPROF error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
# here
# write the report for each image as .*-#$ to Chatter -
# detailed report will be written automagically to science...
self._intgr_per_image_statistics.update(job.get_per_image_statistics())
postref_result.update(job.get_postref_result())
# inspect the output for e.g. very high weighted residuals
all_residuals.extend(job.get_residuals())
self._intgr_batches_out = (integrated_images_first,
integrated_images_last)
if mosaics and len(mosaics) > 0:
self.set_integrater_mosaic_min_mean_max(
min(mosaics), sum(mosaics) / len(mosaics), max(mosaics))
else:
m = indxr.get_indexer_mosaic()
self.set_integrater_mosaic_min_mean_max(m, m, m)
Chatter.write(self.show_per_image_statistics())
Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
self.get_integrater_mosaic_min_mean_max())
# gather the statistics from the postrefinement for all sweeps
# now write this to a postrefinement log
postref_log = os.path.join(self.get_working_directory(),
'postrefinement.log')
fout = open(postref_log, 'w')
fout.write('$TABLE: Postrefinement for %s:\n' % \
self._intgr_sweep_name)
fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n')
fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n')
for image in sorted(postref_result):
phix = postref_result[image].get('phix', 0.0)
phiy = postref_result[image].get('phiy', 0.0)
phiz = postref_result[image].get('phiz', 0.0)
fout.write('%d %5.2f %5.2f %5.2f\n' % \
(image, phix, phiy, phiz))
fout.write('$$\n')
fout.close()
if self.get_integrater_sweep_name():
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file('%s %s %s %s postrefinement' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
postref_log)
hklouts.sort()
hklout = os.path.join(self.get_working_directory(),
os.path.split(hklouts[0])[-1])
Debug.write('Sorting data to %s' % hklout)
for hklin in hklouts:
Debug.write('<= %s' % hklin)
sortmtz = Sortmtz()
sortmtz.set_hklout(hklout)
for hklin in hklouts:
sortmtz.add_hklin(hklin)
sortmtz.sort()
self._mosflm_hklout = hklout
return self._mosflm_hklout
def _mosflm_integrate(self):
'''Perform the actual integration, based on the results of the
cell refinement or indexing (they have the equivalent form.)'''
refinr = self.get_integrater_refiner()
if not refinr.get_refiner_payload('mosflm_orientation_matrix'):
raise RuntimeError, 'unexpected situation in indexing'
lattice = refinr.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup(lattice)
mosaic = refinr.get_refiner_payload('mosaic')
beam = refinr.get_refiner_payload('beam')
distance = refinr.get_refiner_payload('distance')
matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')
integration_params = refinr.get_refiner_payload(
'mosflm_integration_parameters')
if integration_params:
if 'separation' in integration_params:
self.set_integrater_parameter(
'mosflm', 'separation',
'%s %s' % tuple(integration_params['separation']))
if 'raster' in integration_params:
self.set_integrater_parameter(
'mosflm', 'raster',
'%d %d %d %d %d' % tuple(integration_params['raster']))
refinr.set_refiner_payload('mosflm_integration_parameters', None)
f = open(os.path.join(self.get_working_directory(),
'xiaintegrate.mat'), 'w')
for m in matrix:
f.write(m)
f.close()
# then start the integration
integrater = MosflmIntegrate()
integrater.set_working_directory(self.get_working_directory())
auto_logfiler(integrater)
integrater.set_refine_profiles(self._mosflm_refine_profiles)
pname, xname, dname = self.get_integrater_project_info()
if pname is not None and xname is not None and dname is not None:
Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname))
harvest_dir = self.get_working_directory()
# harvest file name will be %s.mosflm_run_start_end % dname
temp_dname = '%s_%s' % \
(dname, self.get_integrater_sweep_name())
integrater.set_pname_xname_dname(pname, xname, temp_dname)
integrater.set_template(os.path.basename(self.get_template()))
integrater.set_directory(self.get_directory())
# check for ice - and if so, exclude (ranges taken from
# XDS documentation)
if self.get_integrater_ice() != 0:
Debug.write('Excluding ice rings')
integrater.set_exclude_ice(True)
# exclude specified resolution ranges
if len(self.get_integrater_excluded_regions()) != 0:
regions = self.get_integrater_excluded_regions()
Debug.write('Excluding regions: %s' % `regions`)
integrater.set_exclude_regions(regions)
mask = standard_mask(self.get_detector())
for m in mask:
integrater.add_instruction(m)
integrater.set_input_mat_file('xiaintegrate.mat')
integrater.set_beam_centre(beam)
integrater.set_distance(distance)
integrater.set_space_group_number(spacegroup_number)
integrater.set_mosaic(mosaic)
if self.get_wavelength_prov() == 'user':
integrater.set_wavelength(self.get_wavelength())
parameters = self.get_integrater_parameters('mosflm')
integrater.update_parameters(parameters)
if self._mosflm_gain:
integrater.set_gain(self._mosflm_gain)
# check for resolution limits
if self._intgr_reso_high > 0.0:
integrater.set_d_min(self._intgr_reso_high)
if self._intgr_reso_low:
integrater.set_d_max(self._intgr_reso_low)
if PhilIndex.params.general.backstop_mask:
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask = mask.calculate_mask_mosflm(self.get_header())
integrater.set_mask(mask)
detector = self.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
integrater.set_limits(lim_x, lim_y)
integrater.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
offset = self.get_frame_offset()
integrater.set_image_range(
(self._intgr_wedge[0] - offset, self._intgr_wedge[1] - offset))
try:
integrater.run()
except RuntimeError, e:
if 'integration failed: reason unknown' in str(e):
Chatter.write('Mosflm has failed in integration')
message = 'The input was:\n\n'
for input in integrater.get_all_input():
message += ' %s' % input
Chatter.write(message)
raise
def _index_prepare(self):
'''Prepare to do autoindexing - in XDS terms this will mean
calling xycorr, init and colspot on the input images.'''
# decide on images to work with
Debug.write('XDS INDEX PREPARE:')
Debug.write('Wavelength: %.6f' % self.get_wavelength())
Debug.write('Distance: %.2f' % self.get_distance())
if self._indxr_images == []:
_select_images_function = getattr(
self, '_index_select_images_%s' % (self._index_select_images))
wedges = _select_images_function()
for wedge in wedges:
self.add_indexer_image_wedge(wedge)
self.set_indexer_prepare_done(True)
all_images = self.get_matching_images()
first = min(all_images)
last = max(all_images)
# next start to process these - first xycorr
xycorr = self.Xycorr()
xycorr.set_data_range(first, last)
xycorr.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
xycorr.set_beam_centre(xds_beam_centre[0],
xds_beam_centre[1])
for block in self._indxr_images:
xycorr.add_spot_range(block[0], block[1])
# FIXME need to set the origin here
xycorr.run()
for file in ['X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf']:
self._indxr_payload[file] = xycorr.get_output_data_file(file)
# next start to process these - then init
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
# find the region of the scan with the least predicted shadow
# to use for background determination in XDS INIT step
from dxtbx.serialize import dump
from dxtbx.datablock import DataBlock
imageset = self._indxr_imagesets[0]
xsweep = self._indxr_sweeps[0]
sweep_filename = os.path.join(
self.get_working_directory(), '%s_datablock.json' %xsweep.get_name())
dump.datablock(DataBlock([imageset]), sweep_filename)
from xia2.Wrappers.Dials.ShadowPlot import ShadowPlot
shadow_plot = ShadowPlot()
shadow_plot.set_working_directory(self.get_working_directory())
auto_logfiler(shadow_plot)
shadow_plot.set_sweep_filename(sweep_filename)
shadow_plot.set_json_filename(
os.path.join(
self.get_working_directory(),
'%s_shadow_plot.json' %shadow_plot.get_xpid()))
shadow_plot.run()
results = shadow_plot.get_results()
from scitbx.array_family import flex
fraction_shadowed = flex.double(results['fraction_shadowed'])
scan_points = flex.double(results['scan_points'])
phi_width = self.get_phi_width()
scan = imageset.get_scan()
oscillation_range = scan.get_oscillation_range()
oscillation = scan.get_oscillation()
bg_images = self._background_images
bg_range_deg = (scan.get_angle_from_image_index(bg_images[0]),
scan.get_angle_from_image_index(bg_images[1]))
bg_range_width = bg_range_deg[1] - bg_range_deg[0]
min_shadow = 100
best_bg_range = bg_range_deg
from libtbx.utils import frange
for bg_range_start in frange(flex.min(scan_points), flex.max(scan_points) - bg_range_width, step=oscillation[1]):
bg_range_deg = (bg_range_start, bg_range_start + bg_range_width)
sel = (scan_points >= bg_range_deg[0]) & (scan_points <= bg_range_deg[1])
mean_shadow = flex.mean(fraction_shadowed.select(sel))
if mean_shadow < min_shadow:
min_shadow = mean_shadow
best_bg_range = bg_range_deg
self._background_images = (
scan.get_image_index_from_angle(best_bg_range[0]),
scan.get_image_index_from_angle(best_bg_range[1]))
Debug.write('Setting background images: %s -> %s' %self._background_images)
init = self.Init()
for file in ['X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf']:
init.set_input_data_file(file, self._indxr_payload[file])
init.set_data_range(first, last)
if self._background_images:
init.set_background_range(self._background_images[0],
self._background_images[1])
else:
init.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images:
init.add_spot_range(block[0], block[1])
init.run()
# at this stage, need to (perhaps) modify the BKGINIT.cbf image
# to mark out the back stop
if PhilIndex.params.general.backstop_mask:
Debug.write('Applying mask to BKGINIT.pck')
# copy the original file
cbf_old = os.path.join(init.get_working_directory(),
'BKGINIT.cbf')
cbf_save = os.path.join(init.get_working_directory(),
'BKGINIT.sav')
shutil.copyfile(cbf_old, cbf_save)
# modify the file to give the new mask
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask.apply_mask_xds(self.get_header(), cbf_save, cbf_old)
init.reload()
for file in ['BLANK.cbf',
'BKGINIT.cbf',
'GAIN.cbf']:
self._indxr_payload[file] = init.get_output_data_file(file)
if PhilIndex.params.xia2.settings.developmental.use_dials_spotfinder:
spotfinder = self.DialsSpotfinder()
for block in self._indxr_images:
spotfinder.add_spot_range(block[0], block[1])
spotfinder.run()
export = self.DialsExportSpotXDS()
export.set_input_data_file(
'reflections.pickle',
spotfinder.get_output_data_file('reflections.pickle'))
export.run()
for file in ['SPOT.XDS']:
self._indxr_payload[file] = export.get_output_data_file(file)
else:
# next start to process these - then colspot
colspot = self.Colspot()
for file in ['X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf',
'BLANK.cbf',
'BKGINIT.cbf',
'GAIN.cbf']:
colspot.set_input_data_file(file, self._indxr_payload[file])
colspot.set_data_range(first, last)
colspot.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images:
colspot.add_spot_range(block[0], block[1])
colspot.run()
for file in ['SPOT.XDS']:
self._indxr_payload[file] = colspot.get_output_data_file(file)
# that should be everything prepared... all of the important
# files should be loaded into memory to be able to cope with
# integration happening somewhere else
return
def _index_prepare(self):
'''Prepare to do autoindexing - in XDS terms this will mean
calling xycorr, init and colspot on the input images.'''
# decide on images to work with
Debug.write('XDS INDEX PREPARE:')
Debug.write('Wavelength: %.6f' % self.get_wavelength())
Debug.write('Distance: %.2f' % self.get_distance())
if self._indxr_images == []:
_select_images_function = getattr(
self, '_index_select_images_%s' % (self._index_select_images))
wedges = _select_images_function()
for wedge in wedges:
self.add_indexer_image_wedge(wedge)
self.set_indexer_prepare_done(True)
all_images = self.get_matching_images()
first = min(all_images)
last = max(all_images)
# next start to process these - first xycorr
xycorr = self.Xycorr()
xycorr.set_data_range(first, last)
xycorr.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
xycorr.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
for block in self._indxr_images:
xycorr.add_spot_range(block[0], block[1])
# FIXME need to set the origin here
xycorr.run()
for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']:
self._indxr_payload[file] = xycorr.get_output_data_file(file)
# next start to process these - then init
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
imageset = self._indxr_imagesets[0]
masker = imageset.masker().format_class(
imageset.paths()[0]).get_goniometer_shadow_masker()
if masker is None:
# disable dynamic_shadowing
PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
# find the region of the scan with the least predicted shadow
# to use for background determination in XDS INIT step
from dxtbx.serialize import dump
from dxtbx.datablock import DataBlock
imageset = self._indxr_imagesets[0]
xsweep = self._indxr_sweeps[0]
sweep_filename = os.path.join(
self.get_working_directory(),
'%s_datablock.json' % xsweep.get_name())
dump.datablock(DataBlock([imageset]), sweep_filename)
from xia2.Wrappers.Dials.ShadowPlot import ShadowPlot
shadow_plot = ShadowPlot()
shadow_plot.set_working_directory(self.get_working_directory())
auto_logfiler(shadow_plot)
shadow_plot.set_sweep_filename(sweep_filename)
shadow_plot.set_json_filename(
os.path.join(self.get_working_directory(),
'%s_shadow_plot.json' % shadow_plot.get_xpid()))
shadow_plot.run()
results = shadow_plot.get_results()
from scitbx.array_family import flex
fraction_shadowed = flex.double(results['fraction_shadowed'])
if flex.max(fraction_shadowed) == 0:
PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False
else:
scan_points = flex.double(results['scan_points'])
phi_width = self.get_phi_width()
scan = imageset.get_scan()
oscillation_range = scan.get_oscillation_range()
oscillation = scan.get_oscillation()
if self._background_images is not None:
bg_images = self._background_images
bg_range_deg = (scan.get_angle_from_image_index(
bg_images[0]),
scan.get_angle_from_image_index(
bg_images[1]))
bg_range_width = bg_range_deg[1] - bg_range_deg[0]
min_shadow = 100
best_bg_range = bg_range_deg
from libtbx.utils import frange
for bg_range_start in frange(flex.min(scan_points),
flex.max(scan_points) -
bg_range_width,
step=oscillation[1]):
bg_range_deg = (bg_range_start,
bg_range_start + bg_range_width)
sel = (scan_points >= bg_range_deg[0]) & (
scan_points <= bg_range_deg[1])
mean_shadow = flex.mean(fraction_shadowed.select(sel))
if mean_shadow < min_shadow:
min_shadow = mean_shadow
best_bg_range = bg_range_deg
self._background_images = (scan.get_image_index_from_angle(
best_bg_range[0]),
scan.get_image_index_from_angle(
best_bg_range[1]))
Debug.write('Setting background images: %s -> %s' %
self._background_images)
init = self.Init()
for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']:
init.set_input_data_file(file, self._indxr_payload[file])
init.set_data_range(first, last)
if self._background_images:
init.set_background_range(self._background_images[0],
self._background_images[1])
else:
init.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images:
init.add_spot_range(block[0], block[1])
init.run()
# at this stage, need to (perhaps) modify the BKGINIT.cbf image
# to mark out the back stop
if PhilIndex.params.general.backstop_mask:
Debug.write('Applying mask to BKGINIT.pck')
# copy the original file
cbf_old = os.path.join(init.get_working_directory(), 'BKGINIT.cbf')
cbf_save = os.path.join(init.get_working_directory(),
'BKGINIT.sav')
shutil.copyfile(cbf_old, cbf_save)
# modify the file to give the new mask
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask.apply_mask_xds(self.get_header(), cbf_save, cbf_old)
init.reload()
for file in ['BLANK.cbf', 'BKGINIT.cbf', 'GAIN.cbf']:
self._indxr_payload[file] = init.get_output_data_file(file)
if PhilIndex.params.xia2.settings.developmental.use_dials_spotfinder:
spotfinder = self.DialsSpotfinder()
for block in self._indxr_images:
spotfinder.add_spot_range(block[0], block[1])
spotfinder.run()
export = self.DialsExportSpotXDS()
export.set_input_data_file(
'reflections.pickle',
spotfinder.get_output_data_file('reflections.pickle'))
export.run()
for file in ['SPOT.XDS']:
self._indxr_payload[file] = export.get_output_data_file(file)
else:
# next start to process these - then colspot
colspot = self.Colspot()
for file in [
'X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf', 'BLANK.cbf',
'BKGINIT.cbf', 'GAIN.cbf'
]:
colspot.set_input_data_file(file, self._indxr_payload[file])
colspot.set_data_range(first, last)
colspot.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images:
colspot.add_spot_range(block[0], block[1])
colspot.run()
for file in ['SPOT.XDS']:
self._indxr_payload[file] = colspot.get_output_data_file(file)
# that should be everything prepared... all of the important
# files should be loaded into memory to be able to cope with
# integration happening somewhere else
return