def Cellparm(self): '''Create a Cellparm wrapper from _Cellparm - set the working directory and log file stuff as a part of this...''' cellparm = _Cellparm() cellparm.set_working_directory(self.get_working_directory()) auto_logfiler(cellparm) return cellparm
def get_real_space_primitive_matrix(lattice, matrix, wd = None):
'''Get the primitive real space vectors for the unit cell and
lattice type. Note that the resulting matrix will need to be
scaled by a factor equal to the wavelength in Angstroms.'''
# parse the orientation matrix
cell, a, u = parse_matrix(matrix)
# generate other possibilities
o = _Othercell()
if wd:
o.set_working_directory(wd)
auto_logfiler(o)
o.set_cell(cell)
o.set_lattice(lattice)
o.generate()
# transform the possibly centred cell to the primitive setting
new_cell = o.get_cell('aP')
op = symop_to_mat(o.get_reindex_op('aP'))
primitive_a = matmul(invert(op), a)
# then convert to real space
real_a = invert(primitive_a)
return real_a[0:3], real_a[3:6], real_a[6:9]
def get_reciprocal_space_primitive_matrix(lattice, matrix, wd = None):
'''Get the primitive reciprocal space vectors for this matrix.'''
# parse the orientation matrix
cell, a, u = parse_matrix(matrix)
# generate other possibilities
o = _Othercell()
if wd:
o.set_working_directory(wd)
auto_logfiler(o)
o.set_cell(cell)
o.set_lattice(lattice)
o.generate()
# transform the possibly centred cell to the primitive setting
new_cell = o.get_cell('aP')
op = symop_to_mat(o.get_reindex_op('aP'))
primitive_a = matmul(invert(op), a)
return mat2vec(primitive_a)
def Mtzdump(self): '''Create a Mtzdump wrapper from _Mtzdump - set the working directory and log file stuff as a part of this...''' mtzdump = _Mtzdump() mtzdump.set_working_directory(self.get_working_directory()) auto_logfiler(mtzdump) return mtzdump
def Correct(self):
correct = _Correct(params=PhilIndex.params.xds.correct)
correct.set_working_directory(self.get_working_directory())
correct.setup_from_imageset(self.get_imageset())
if self.get_distance():
correct.set_distance(self.get_distance())
if self.get_wavelength():
correct.set_wavelength(self.get_wavelength())
if self.get_integrater_ice():
correct.set_ice(self.get_integrater_ice())
if self.get_integrater_excluded_regions():
correct.set_excluded_regions(self.get_integrater_excluded_regions())
if self.get_integrater_anomalous():
correct.set_anomalous(True)
if self.get_integrater_low_resolution() > 0.0:
Debug.write('Using low resolution limit: %.2f' % \
self.get_integrater_low_resolution())
correct.set_resolution_high(0.0)
correct.set_resolution_low(
self.get_integrater_low_resolution())
auto_logfiler(correct, 'CORRECT')
return correct
def RefineBravaisSettings(self):
rbs = _RefineBravaisSettings()
rbs.set_working_directory(self.get_working_directory())
rbs.set_close_to_spindle_cutoff(
PhilIndex.params.dials.close_to_spindle_cutoff)
auto_logfiler(rbs)
return rbs
def Idxref(self):
from xia2.Handlers.Phil import PhilIndex
idxref = _Idxref(params=PhilIndex.params.xds.index)
idxref.set_working_directory(self.get_working_directory())
idxref.setup_from_imageset(self.get_imageset())
if self.get_distance():
idxref.set_distance(self.get_distance())
if self.get_wavelength():
idxref.set_wavelength(self.get_wavelength())
# if we have a refined set of parameters to apply, apply these
if Flags.get_xparm():
idxref.set_refined_origin(Flags.get_xparm_origin())
idxref.set_refined_beam_vector(Flags.get_xparm_beam_vector())
idxref.set_refined_rotation_axis(Flags.get_xparm_rotation_axis())
idxref.set_refined_distance(Flags.get_xparm_distance())
# hacks for Jira 493
if Flags.get_xparm_a():
idxref.set_a_axis(Flags.get_xparm_a())
if Flags.get_xparm_b():
idxref.set_b_axis(Flags.get_xparm_b())
if Flags.get_xparm_c():
idxref.set_c_axis(Flags.get_xparm_c())
auto_logfiler(idxref, 'IDXREF')
return idxref
def Report(self): report = _Report() report.set_working_directory(self.get_working_directory()) report.set_experiments_filename(self._intgr_experiments_filename) report.set_reflections_filename(self._intgr_integrated_pickle) auto_logfiler(report, 'REPORT') return report
def Truncate(self): '''Create a Truncate wrapper from _Truncate - set the working directory and log file stuff as a part of this...''' truncate = _Truncate() truncate.set_working_directory(self.get_working_directory()) auto_logfiler(truncate) return truncate
def Rebatch(self): '''Create a Rebatch wrapper from _Rebatch - set the working directory and log file stuff as a part of this...''' rebatch = _Rebatch() rebatch.set_working_directory(self.get_working_directory()) auto_logfiler(rebatch) return rebatch
def _integrate_finish(self):
'''Finish the integration - if necessary performing reindexing
based on the pointgroup and the reindexing operator.'''
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()):
return self._mosflm_hklout
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == 0:
return self._mosflm_hklout
Debug.write('Reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
hklin = self._mosflm_hklout
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
reindex.set_operator(self._intgr_reindex_operator)
if self._intgr_spacegroup_number:
reindex.set_spacegroup(self._intgr_spacegroup_number)
hklout = '%s_reindex.mtz' % hklin[:-4]
reindex.set_hklin(hklin)
reindex.set_hklout(hklout)
reindex.reindex()
return hklout
def Pointless(self): '''Create a Pointless wrapper from _Pointless - and set the working directory and log file stuff as a part of this...''' pointless = _Pointless() pointless.set_working_directory(self.get_working_directory()) auto_logfiler(pointless) return pointless
def _index_select_images(self):
'''Select correct images based on image headers. This will in
general use the 20 frames. N.B. only if they have good
spots on them!'''
phi_width = self.get_phi_width()
images = self.get_matching_images()
# N.B. now bodging this to use up to 20 frames which have decent
# spots on, spaced from throughout the data set.
spacing = max(1, int(len(images) // 20))
selected = []
for j in range(0, len(images), spacing):
selected.append(images[j])
for image in selected[:20]:
ld = LabelitDistl()
ld.set_working_directory(self.get_working_directory())
auto_logfiler(ld)
ld.add_image(self.get_image_name(image))
ld.distl()
spots = ld.get_statistics(
self.get_image_name(image))['spots_good']
Debug.write('Image %d good spots %d' % (image, spots))
if spots > 10:
self.add_indexer_image_wedge(image)
return
def Sortmtz(self): '''Create a Sortmtz wrapper from _Sortmtz - set the working directory and log file stuff as a part of this...''' sortmtz = _Sortmtz() sortmtz.set_working_directory(self.get_working_directory()) auto_logfiler(sortmtz) return sortmtz
def Reindex(self): '''Create a Reindex wrapper from _Reindex - set the working directory and log file stuff as a part of this...''' reindex = _Reindex() reindex.set_working_directory(self.get_working_directory()) auto_logfiler(reindex) return reindex
def Chef(self): '''Create a Chef wrapper from _Chef - set the working directory and log file stuff as a part of this...''' chef = _Chef() chef.set_working_directory(self.get_working_directory()) auto_logfiler(chef) return chef
def Scaleit(self): '''Create a Scaleit wrapper from _Scaleit - set the working directory and log file stuff as a part of this...''' scaleit = _Scaleit() scaleit.set_working_directory(self.get_working_directory()) auto_logfiler(scaleit) return scaleit
def Matthews_coef(self): '''Create a Matthews_coef wrapper from _Matthews_coef - set the working directory and log file stuff as a part of this...''' matthews_coef = _Matthews_coef() matthews_coef.set_working_directory(self.get_working_directory()) auto_logfiler(matthews_coef) return matthews_coef
def Freerflag(self): '''Create a Freerflag wrapper from _Freerflag - set the working directory and log file stuff as a part of this...''' freerflag = _Freerflag() freerflag.set_working_directory(self.get_working_directory()) auto_logfiler(freerflag) return freerflag
def Cad(self): '''Create a Cad wrapper from _Cad - set the working directory and log file stuff as a part of this...''' cad = _Cad() cad.set_working_directory(self.get_working_directory()) auto_logfiler(cad) return cad
def Mtz2various(self): '''Create a Mtz2various wrapper from _Mtz2various - set the working directory and log file stuff as a part of this...''' mtz2various = _Mtz2various() mtz2various.set_working_directory(self.get_working_directory()) auto_logfiler(mtz2various) return mtz2various
def spot_xds_to_reflection_pickle(spot_xds, working_directory): from xia2.Wrappers.Dials.ImportXDS import ImportXDS importer = ImportXDS() importer.set_working_directory(working_directory) auto_logfiler(importer) importer.set_spot_xds(spot_xds) importer.run() return importer.get_reflection_filename()
def _prepare_pointless_hklin(working_directory,
hklin,
phi_width):
'''Prepare some data for pointless - this will take only 180 degrees
of data if there is more than this (through a "rebatch" command) else
will simply return hklin.'''
# also remove blank images?
if not Flags.get_microcrystal() and not Flags.get_small_molecule():
Debug.write('Excluding blank images')
hklout = os.path.join(
working_directory,
'%s_noblank.mtz' % (os.path.split(hklin)[-1][:-4]))
FileHandler.record_temporary_file(hklout)
hklin = remove_blank(hklin, hklout)
# find the number of batches
md = Mtzdump()
md.set_working_directory(working_directory)
auto_logfiler(md)
md.set_hklin(hklin)
md.dump()
batches = max(md.get_batches()) - min(md.get_batches())
phi_limit = 180
if batches * phi_width < phi_limit or Flags.get_small_molecule():
return hklin
hklout = os.path.join(
working_directory,
'%s_prepointless.mtz' % (os.path.split(hklin)[-1][:-4]))
rb = Rebatch()
rb.set_working_directory(working_directory)
auto_logfiler(rb)
rb.set_hklin(hklin)
rb.set_hklout(hklout)
first = min(md.get_batches())
last = first + int(phi_limit / phi_width)
Debug.write('Preparing data for pointless - %d batches (%d degrees)' % \
((last - first), phi_limit))
rb.limit_batches(first, last)
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
return hklout
def CombineExperiments(self):
combiner = _CombineExperiments()
combiner.set_working_directory(self.get_working_directory())
auto_logfiler(combiner)
for idxr in self._refinr_indexers.values():
combiner.add_experiments(
idxr.get_indexer_payload("experiments_filename"))
combiner.add_reflections(idxr.get_indexed_filename())
return combiner
def DialsSpotfinder(self):
from xia2.Wrappers.Dials.Spotfinder import Spotfinder
spotfinder = Spotfinder(params=PhilIndex.params.dials.find_spots)
spotfinder.set_working_directory(self.get_working_directory())
spotfinder.setup_from_imageset(self.get_imageset())
auto_logfiler(spotfinder, 'SPOTFINDER')
return spotfinder
def Import(self):
importer = _Import()
importer.set_working_directory(self.get_working_directory())
importer.setup_from_imageset(self.get_imageset())
auto_logfiler(importer)
importer.set_mosflm_beam_centre(self.get_beam_centre())
importer.set_sweep_filename(
os.path.join(self.get_working_directory(),
'%s_datablock_import.json' %importer.get_xpid()))
return importer
def transmogrify_matrix(lattice, matrix, target_lattice,
wavelength, wd = None):
'''Transmogrify a matrix for lattice X into a matrix for lattice
Y. This should work find for Mosflm... Will also return the new
unit cell.'''
cell, a, u = parse_matrix(matrix)
o = _Othercell()
if wd:
o.set_working_directory(wd)
auto_logfiler(o)
o.set_cell(cell)
o.set_lattice(lattice)
o.generate()
new_cell = o.get_cell(target_lattice)
op = symop_to_mat(o.get_reindex_op(target_lattice))
# HACK! again - bug # 3193
if 'lattice_symmetry' in o.get_executable():
op = transpose(op)
# why is only one of the matrices inverted?!
a = matmul(invert(op), a)
u = matmul(op, u)
# in here test that the given unit cell corresponds to the
# one calculated from the A matrix.
anew = matscl(a, 1.0 / wavelength)
reala = transpose(invert(anew))
_a, _b, _c = mat2vec(reala)
la = math.sqrt(dot(_a, _a))
lb = math.sqrt(dot(_b, _b))
lc = math.sqrt(dot(_c, _c))
if math.fabs(la - new_cell[0]) / new_cell[0] > 0.01:
raise RuntimeError, 'cell check failed (wavelength != %f)' % \
wavelength
if math.fabs(lb - new_cell[1]) / new_cell[1] > 0.01:
raise RuntimeError, 'cell check failed (wavelength != %f)' % \
wavelength
if math.fabs(lc - new_cell[2]) / new_cell[2] > 0.01:
raise RuntimeError, 'cell check failed (wavelength != %f)' % \
wavelength
return format_matrix(new_cell, a, u)
def Index(self):
index = _Index()
index.set_working_directory(self.get_working_directory())
params = PhilIndex.params.dials.index
index.set_reflections_per_degree(params.reflections_per_degree)
if params.fft3d.n_points is not None:
index.set_fft3d_n_points(params.fft3d.n_points)
auto_logfiler(index)
index.set_outlier_algorithm(PhilIndex.params.dials.outlier.algorithm)
index.set_histogram_binning(PhilIndex.params.dials.index.histogram_binning)
return index
def ExportMtz(self):
params = PhilIndex.params.dials.integrate
export = _ExportMtz()
export.set_working_directory(self.get_working_directory())
export.set_experiments_filename(self._intgr_experiments_filename)
export.set_include_partials(params.include_partials)
auto_logfiler(export, 'EXPORTMTZ')
return export
def get_integrater_corrected_intensities(self): self.integrate() from xia2.Wrappers.Dials.ExportXDSASCII import ExportXDSASCII exporter = ExportXDSASCII() exporter.set_experiments_filename(self.get_integrated_experiments()) exporter.set_reflections_filename(self.get_integrated_reflections()) auto_logfiler(exporter) self._intgr_corrected_hklout = '%i_DIALS.HKL' %exporter.get_xpid() exporter.set_hkl_filename(self._intgr_corrected_hklout) exporter.run() assert os.path.exists(self._intgr_corrected_hklout) return self._intgr_corrected_hklout
def ExportMtz(self):
params = PhilIndex.params.dials.integrate
export = _ExportMtz()
_, xname, _ = self.get_integrater_project_info()
export.crystal_name = xname
export.set_working_directory(self.get_working_directory())
export.set_experiments_filename(self._intgr_experiments_filename)
export.set_combine_partials(params.combine_partials)
export.set_partiality_threshold(params.partiality_threshold)
if len(self.get_matching_images()) == 1:
export.set_partiality_threshold(0.1)
if (len(self.get_matching_images()) == 1
or PhilIndex.params.dials.fast_mode or
not PhilIndex.params.xia2.settings.integration.profile_fitting
):
# With no profiles available have to rely on summation alone
export.set_intensity_choice("sum")
auto_logfiler(export, "EXPORTMTZ")
return export
def split_experiments(self, experiment, reflection, sweep_handler):
"""Split a multi-experiment dataset into individual datasets and set in the
sweep handler."""
splitter = SplitExperiments()
splitter.add_experiments(experiment)
splitter.add_reflections(reflection)
splitter.set_working_directory(self.get_working_directory())
auto_logfiler(splitter)
splitter.run()
nn = len(sweep_handler.get_epochs())
fmt = "%%0%dd" % (math.log10(nn) + 1)
for i, epoch in enumerate(sweep_handler.get_epochs()):
si = sweep_handler.get_sweep_information(epoch)
nums = fmt % i
si.set_reflections(
os.path.join(self.get_working_directory(), "split_%s.refl" % nums)
)
si.set_experiments(
os.path.join(self.get_working_directory(), "split_%s.expt" % nums)
)
return sweep_handler
def _scale_aimless(self, d_min=None):
logger.debug('Scaling with aimless')
PhilIndex.params.xia2.settings.multiprocessing.nproc = 1
scaler = Aimless()
auto_logfiler(scaler)
self._scaled_mtz = '%i_scaled.mtz' % scaler.get_xpid()
scaler.set_surface_link(False) # multi-crystal
scaler.set_hklin(self._sorted_mtz)
scaler.set_hklout(self._scaled_mtz)
scaler.set_surface_tie(self._params.scaling.surface_tie)
if self._params.scaling.secondary.frame == 'camera':
secondary = 'secondary'
else:
secondary = 'absorption'
lmax = self._params.scaling.secondary.lmax
scaler.set_secondary(mode=secondary, lmax=lmax)
scaler.set_spacing(self._params.scaling.rotation.spacing)
if d_min is not None:
scaler.set_resolution(d_min)
scaler.scale()
self._scaled_unmerged_mtz \
= os.path.splitext(self._scaled_mtz)[0] + '_unmerged.mtz'
return scaler
def Refine(self):
refine = _Refine()
params = PhilIndex.params.dials.refine
refine.set_phil_file(params.phil_file)
refine.set_working_directory(self.get_working_directory())
if PhilIndex.params.dials.fast_mode:
# scan-static refinement in fast mode
refine.set_scan_varying(False)
else:
refine.set_scan_varying(params.scan_varying)
refine.set_reflections_per_degree(params.reflections_per_degree)
refine.set_interval_width_degrees(params.interval_width_degrees)
refine.set_outlier_algorithm(PhilIndex.params.dials.outlier.algorithm)
if PhilIndex.params.dials.fix_geometry:
refine.set_detector_fix("all")
refine.set_beam_fix("all")
elif PhilIndex.params.dials.fix_distance:
refine.set_detector_fix("distance")
refine.set_close_to_spindle_cutoff(
PhilIndex.params.dials.close_to_spindle_cutoff)
auto_logfiler(refine, "REFINE")
return refine
def cosym(self):
logger.debug("Running cosym analysis")
experiments_filename = self._data_manager.export_experiments(
"tmp.expt")
reflections_filename = self._data_manager.export_reflections(
"tmp.refl")
cosym = DialsCosym()
auto_logfiler(cosym)
cosym.add_experiments_json(experiments_filename)
cosym.add_reflections_file(reflections_filename)
if self._params.symmetry.space_group is not None:
cosym.set_space_group(self._params.symmetry.space_group.group())
if self._params.symmetry.laue_group is not None:
cosym.set_space_group(self._params.symmetry.laue_group.group())
cosym.set_best_monoclinic_beta(
self._params.symmetry.cosym.best_monoclinic_beta)
cosym.run()
self._cosym_analysis = cosym.get_cosym_analysis()
self._experiments_filename = cosym.get_reindexed_experiments()
self._reflections_filename = cosym.get_reindexed_reflections()
self._data_manager.experiments = load.experiment_list(
self._experiments_filename, check_format=False)
self._data_manager.reflections = flex.reflection_table.from_file(
self._reflections_filename)
if not any([
self._params.symmetry.space_group,
self._params.symmetry.laue_group
]):
best_solution = cosym.get_best_solution()
best_space_group = sgtbx.space_group(
str(best_solution["patterson_group"])
).build_derived_acentric_group()
self._params.symmetry.laue_group = best_space_group.info()
logger.info("Laue group determined by dials.cosym: %s" %
best_space_group.info())
def multiplicity_plots(self):
from xia2.Wrappers.XIA.PlotMultiplicity import PlotMultiplicity
mult_json_files = {}
mult_img_files = {}
from xia2.lib.bits import auto_logfiler
cwd = os.getcwd()
try:
os.chdir(self.report_dir)
for axis in ('h', 'k', 'l'):
pm = PlotMultiplicity()
pm.set_mtz_filename(self.unmerged_mtz)
pm.set_slice_axis(axis)
pm.set_show_missing(True)
auto_logfiler(pm)
pm.run()
mult_json_files[axis] = pm.get_json_filename()
with open(pm.get_plot_filename(), 'rb') as fh:
mult_img_files[axis] = fh.read().encode('base64').replace(
'\n', '')
return OrderedDict(('multiplicity_%s' % axis, mult_img_files[axis])
for axis in ('h', 'k', 'l'))
finally:
os.chdir(cwd)
def _decide_space_group_dials(self):
logger.debug('Deciding space group with dials.symmetry')
symmetry = DialsSymmetry()
auto_logfiler(symmetry)
self._sorted_mtz = '%i_sorted.mtz' % symmetry.get_xpid()
self._experiments_filename = '%i_experiments_reindexed.json' % symmetry.get_xpid(
)
self._reflections_filename = '%i_reflections_reindexed.pickle' % symmetry.get_xpid(
)
experiments_filename = 'tmp_experiments.json'
reflections_filename = 'tmp_reflections.pickle'
self._data_manager.export_experiments(experiments_filename)
self._data_manager.export_reflections(reflections_filename)
symmetry.set_experiments_filename(experiments_filename)
symmetry.set_reflections_filename(reflections_filename)
symmetry.set_output_experiments_filename(self._experiments_filename)
symmetry.set_output_reflections_filename(self._reflections_filename)
symmetry.decide_pointgroup()
space_group = sgtbx.space_group_info(
symbol=str(symmetry.get_pointgroup())).group()
cb_op = sgtbx.change_of_basis_op(symmetry.get_reindex_operator())
self._data_manager.experiments = load.experiment_list(
self._experiments_filename, check_format=False)
self._data_manager.reflections = flex.reflection_table.from_pickle(
self._reflections_filename)
# export reflections
self._sorted_mtz = self._data_manager.export_mtz(
filename=self._sorted_mtz)
logger.info('Space group determined by dials.symmetry: %s' %
space_group.info())
return space_group, cb_op
def Spotfinder(self):
spotfinder = _Spotfinder()
spotfinder.set_working_directory(self.get_working_directory())
auto_logfiler(spotfinder)
spotfinder.set_hot_mask_prefix('%d_hot_mask' % spotfinder.get_xpid())
return spotfinder
def EstimateGain(self):
estimater = _EstimateGain()
estimater.set_working_directory(self.get_working_directory())
auto_logfiler(estimater)
return estimater
def GenerateMask(self):
genmask = _GenerateMask()
genmask.set_working_directory(self.get_working_directory())
auto_logfiler(genmask)
return genmask
def run(self):
from xia2.Handlers.Streams import Chatter, Debug
if self._reindexing_operators:
Debug.write("Reindexing sweeps for dials.two_theta_refine")
from xia2.lib.bits import auto_logfiler
from xia2.Wrappers.Dials.Reindex import Reindex
self._reindexed_experiments, self._reindexed_reflections = [], []
for e, p, op in zip(
self._experiments,
self._reflection_files,
self._reindexing_operators,
):
reindexer = Reindex()
reindexer.set_cb_op(op)
reindexer.set_experiments_filename(e)
reindexer.set_indexed_filename(p)
reindexer.set_working_directory(
self.get_working_directory())
auto_logfiler(reindexer)
reindexer.run()
self._reindexed_experiments.append(
reindexer.get_reindexed_experiments_filename())
self._reindexed_reflections.append(
reindexer.get_reindexed_reflections_filename())
Debug.write("Running dials.two_theta_refine")
self._output_cif = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.cif" % self.get_xpid(),
)
self._output_mmcif = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.mmcif" % self.get_xpid(),
)
if not self._output_p4p:
self._output_p4p = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.p4p" % self.get_xpid(),
)
self._output_correlation_plot = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.png" % self.get_xpid(),
)
self._output_experiments = os.path.join(
self.get_working_directory(),
"%s_refined_cell.expt" % self.get_xpid())
self.clear_command_line()
if self._reindexing_operators:
for experiment in self._reindexed_experiments:
self.add_command_line(experiment)
for reflection_file in self._reindexed_reflections:
self.add_command_line(reflection_file)
else:
for experiment in self._experiments:
self.add_command_line(experiment)
for reflection_file in self._reflection_files:
self.add_command_line(reflection_file)
self.add_command_line("combine_crystal_models=%s" %
self._combine_crystal_models)
self.add_command_line("output.cif=%s" % self._output_cif)
self.add_command_line("output.mmcif=%s" % self._output_mmcif)
self.add_command_line("output.p4p=%s" % self._output_p4p)
if self._output_correlation_plot is not None:
self.add_command_line("output.correlation_plot.filename=%s" %
self._output_correlation_plot)
if self._output_experiments is not None:
self.add_command_line("output.experiments=%s" %
self._output_experiments)
if self._phil_file is not None:
self.add_command_line("%s" % self._phil_file)
self.start()
self.close_wait()
if not os.path.isfile(self._output_cif):
Chatter.write(
"TwoTheta refinement failed, see log file for more details:\n %s"
% self.get_log_file())
raise RuntimeError("unit cell not refined")
self.check_for_errors()
from dxtbx.model.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(
self.get_output_experiments(), check_format=False)
self._crystal = experiments.crystals()[0]
def _mosflm_parallel_integrate(self):
'''Perform the integration as before, but this time as a
number of parallel Mosflm jobs (hence, in separate directories)
and including a step of pre-refinement of the mosaic spread and
missets. This will all be kind of explicit and hence probably
messy!'''
refinr = self.get_integrater_refiner()
lattice = refinr.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup(lattice)
mosaic = refinr.get_refiner_payload('mosaic')
beam = refinr.get_refiner_payload('beam')
distance = refinr.get_refiner_payload('distance')
matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')
integration_params = refinr.get_refiner_payload(
'mosflm_integration_parameters')
if integration_params:
if 'separation' in integration_params:
self.set_integrater_parameter(
'mosflm', 'separation',
'%s %s' % tuple(integration_params['separation']))
if 'raster' in integration_params:
self.set_integrater_parameter(
'mosflm', 'raster',
'%d %d %d %d %d' % tuple(integration_params['raster']))
refinr.set_refiner_payload('mosflm_integration_parameters', None)
pname, xname, dname = self.get_integrater_project_info()
# what follows below should (i) be run in separate directories
# and (ii) be repeated N=parallel times.
nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
parallel = nproc
# FIXME this is something of a kludge - if too few frames refinement
# and integration does not work well... ideally want at least 15
# frames / chunk (say)
nframes = self._intgr_wedge[1] - self._intgr_wedge[0] + 1
if parallel > nframes / 15:
parallel = nframes // 15
if not parallel:
raise RuntimeError('parallel not set')
if parallel < 2:
raise RuntimeError('parallel not parallel: %s' % parallel)
jobs = []
hklouts = []
nref = 0
# calculate the chunks to use
offset = self.get_frame_offset()
start = self._intgr_wedge[0] - offset
end = self._intgr_wedge[1] - offset
left_images = 1 + end - start
left_chunks = parallel
chunks = []
while left_images > 0:
size = left_images // left_chunks
chunks.append((start, start + size - 1))
start += size
left_images -= size
left_chunks -= 1
summary_files = []
for j in range(parallel):
# make some working directories, as necessary - chunk-(0:N-1)
wd = os.path.join(self.get_working_directory(), 'chunk-%d' % j)
if not os.path.exists(wd):
os.makedirs(wd)
job = MosflmIntegrate()
job.set_working_directory(wd)
auto_logfiler(job)
l = refinr.get_refiner_lattice()
# create the starting point
f = open(os.path.join(wd, 'xiaintegrate-%s.mat' % l), 'w')
for m in matrix:
f.write(m)
f.close()
spacegroup_number = lattice_to_spacegroup(lattice)
job.set_refine_profiles(self._mosflm_refine_profiles)
# N.B. for harvesting need to append N to dname.
if pname is not None and xname is not None and dname is not None:
Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname))
harvest_dir = self.get_working_directory()
temp_dname = '%s_%s' % \
(dname, self.get_integrater_sweep_name())
job.set_pname_xname_dname(pname, xname, temp_dname)
job.set_template(os.path.basename(self.get_template()))
job.set_directory(self.get_directory())
# check for ice - and if so, exclude (ranges taken from
# XDS documentation)
if self.get_integrater_ice() != 0:
Debug.write('Excluding ice rings')
job.set_exclude_ice(True)
# exclude specified resolution ranges
if len(self.get_integrater_excluded_regions()) != 0:
regions = self.get_integrater_excluded_regions()
Debug.write('Excluding regions: %s' % repr(regions))
job.set_exclude_regions(regions)
mask = standard_mask(self.get_detector())
for m in mask:
job.add_instruction(m)
job.set_input_mat_file('xiaintegrate-%s.mat' % l)
job.set_beam_centre(beam)
job.set_distance(distance)
job.set_space_group_number(spacegroup_number)
job.set_mosaic(mosaic)
if self.get_wavelength_prov() == 'user':
job.set_wavelength(self.get_wavelength())
parameters = self.get_integrater_parameters('mosflm')
job.update_parameters(parameters)
if self._mosflm_gain:
job.set_gain(self._mosflm_gain)
# check for resolution limits
if self._intgr_reso_high > 0.0:
job.set_d_min(self._intgr_reso_high)
if self._intgr_reso_low:
job.set_d_max(self._intgr_reso_low)
if PhilIndex.params.general.backstop_mask:
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask = mask.calculate_mask_mosflm(self.get_header())
job.set_mask(mask)
detector = self.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
job.set_limits(lim_x, lim_y)
job.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
job.set_pre_refinement(True)
job.set_image_range(chunks[j])
# these are now running so ...
jobs.append(job)
continue
# ok, at this stage I need to ...
#
# (i) accumulate the statistics as a function of batch
# (ii) mong them into a single block
#
# This is likely to be a pain in the arse!
first_integrated_batch = 1.0e6
last_integrated_batch = -1.0e6
all_residuals = []
threads = []
for j in range(parallel):
job = jobs[j]
# now wait for them to finish - first wait will really be the
# first one, then all should be finished...
thread = Background(job, 'run')
thread.start()
threads.append(thread)
mosaics = []
postref_result = {}
integrated_images_first = 1.0e6
integrated_images_last = -1.0e6
self._intgr_per_image_statistics = {}
for j in range(parallel):
thread = threads[j]
thread.stop()
job = jobs[j]
# get the log file
output = job.get_all_output()
# record a copy of it, perhaps - though not if parallel
if self.get_integrater_sweep_name() and False:
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file(
'%s %s %s %s mosflm integrate' % \
(self.get_integrater_sweep_name(),
pname, xname, '%s_%d' % (dname, j)),
job.get_log_file())
# look for things that we want to know...
# that is, the output reflection file name, the updated
# value for the gain (if present,) any warnings, errors,
# or just interesting facts.
batches = job.get_batches_out()
integrated_images_first = min(batches[0], integrated_images_first)
integrated_images_last = max(batches[1], integrated_images_last)
mosaics.extend(job.get_mosaic_spreads())
if min(mosaics) < 0:
raise IntegrationError('negative mosaic spread: %s' %
min(mosaic))
if (job.get_detector_gain_error()
and not (self.get_imageset().get_detector()[0].get_type()
== 'SENSOR_PAD')):
gain = job.get_suggested_gain()
if gain is not None:
self.set_integrater_parameter('mosflm', 'gain', gain)
self.set_integrater_export_parameter(
'mosflm', 'gain', gain)
if self._mosflm_gain:
Debug.write('GAIN updated to %f' % gain)
else:
Debug.write('GAIN found to be %f' % gain)
self._mosflm_gain = gain
self._mosflm_rerun_integration = True
hklout = job.get_hklout()
Debug.write('Integration output: %s' % hklout)
hklouts.append(hklout)
nref += job.get_nref()
# if a BGSIG error happened try not refining the
# profile and running again...
if job.get_bgsig_too_large():
if not self._mosflm_refine_profiles:
raise RuntimeError('BGSIG error with profiles fixed')
Debug.write('BGSIG error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
if job.get_getprof_error():
Debug.write('GETPROF error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
# here
# write the report for each image as .*-#$ to Chatter -
# detailed report will be written automagically to science...
self._intgr_per_image_statistics.update(
job.get_per_image_statistics())
postref_result.update(job.get_postref_result())
# inspect the output for e.g. very high weighted residuals
all_residuals.extend(job.get_residuals())
self._intgr_batches_out = (integrated_images_first,
integrated_images_last)
if mosaics and len(mosaics) > 0:
self.set_integrater_mosaic_min_mean_max(
min(mosaics),
sum(mosaics) / len(mosaics), max(mosaics))
else:
m = indxr.get_indexer_mosaic()
self.set_integrater_mosaic_min_mean_max(m, m, m)
Chatter.write(self.show_per_image_statistics())
Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
self.get_integrater_mosaic_min_mean_max())
# gather the statistics from the postrefinement for all sweeps
# now write this to a postrefinement log
postref_log = os.path.join(self.get_working_directory(),
'postrefinement.log')
fout = open(postref_log, 'w')
fout.write('$TABLE: Postrefinement for %s:\n' % \
self._intgr_sweep_name)
fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n')
fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n')
for image in sorted(postref_result):
phix = postref_result[image].get('phix', 0.0)
phiy = postref_result[image].get('phiy', 0.0)
phiz = postref_result[image].get('phiz', 0.0)
fout.write('%d %5.2f %5.2f %5.2f\n' % \
(image, phix, phiy, phiz))
fout.write('$$\n')
fout.close()
if self.get_integrater_sweep_name():
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file('%s %s %s %s postrefinement' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
postref_log)
hklouts.sort()
hklout = os.path.join(self.get_working_directory(),
os.path.split(hklouts[0])[-1])
Debug.write('Sorting data to %s' % hklout)
for hklin in hklouts:
Debug.write('<= %s' % hklin)
sortmtz = Sortmtz()
sortmtz.set_hklout(hklout)
for hklin in hklouts:
sortmtz.add_hklin(hklin)
sortmtz.sort()
self._mosflm_hklout = hklout
return self._mosflm_hklout
def run():
try:
check_environment()
except Exception as e:
with open("xia2-error.txt", "w") as fh:
traceback.print_exc(file=fh)
logger.error('Status: error "%s"', str(e))
if len(sys.argv) < 2 or "-help" in sys.argv:
help()
sys.exit()
cwd = os.getcwd()
try:
from .xia2_main import xia2_main
xia2_main(stop_after="integrate")
logger.info("Status: normal termination")
wd = os.path.join(cwd, "strategy")
if not os.path.exists(wd):
os.mkdir(wd)
os.chdir(wd)
CommandLine = get_command_line()
xinfo = CommandLine.get_xinfo()
crystals = xinfo.get_crystals()
assert len(crystals) == 1
crystal = list(crystals.values())[0]
assert len(crystal.get_wavelength_names()) == 1
wavelength = crystal.get_xwavelength(crystal.get_wavelength_names()[0])
sweeps = wavelength.get_sweeps()
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
strategy_params = params.strategy
if not len(strategy_params):
strategy_params = [
PhilIndex.get_scope_by_name("strategy")[0].extract()
]
from dxtbx.model import MultiAxisGoniometer
gonio = sweeps[0].get_imageset().get_goniometer()
if (isinstance(gonio, MultiAxisGoniometer)
and len(gonio.get_axes()) == 3 and gonio.get_scan_axis() == 2):
from xia2.Wrappers.Dials.AlignCrystal import AlignCrystal
align_crystal = AlignCrystal()
align_crystal.set_experiments_filename(
sweeps[0]._get_integrater().get_integrated_experiments())
align_crystal.set_working_directory(wd)
auto_logfiler(align_crystal)
align_crystal.set_json_filename("%i_align_crystal.json" %
align_crystal.get_xpid())
align_crystal.run()
logger.info("".join(align_crystal.get_all_output()))
results_all = {}
def process_one_strategy(args):
assert len(args) == 4
experiments, reflections, strategy, t_ref = args
from xia2.Wrappers.EMBL import Best
best = Best.BestStrategy()
for isweep, (expt, refl) in enumerate(zip(experiments,
reflections)):
from xia2.Wrappers.Dials.ExportBest import ExportBest
export = ExportBest()
export.set_experiments_filename(expt)
export.set_reflections_filename(refl)
export.set_working_directory(wd)
auto_logfiler(export)
prefix = "%i_best" % export.get_xpid()
export.set_prefix(prefix)
export.run()
if isweep == 0:
best.set_t_ref(t_ref)
best.set_mos_dat("%s.dat" % prefix)
best.set_mos_par("%s.par" % prefix)
best.add_mos_hkl("%s.hkl" % prefix)
best.set_i2s(strategy.i_over_sigi)
best.set_T_max(strategy.max_total_exposure)
best.set_t_min(strategy.min_exposure)
# best.set_trans_ref(25.0)
best.set_S_max(strategy.max_rotation_speed)
best.set_w_min(strategy.min_oscillation_width)
best.set_M_min(strategy.multiplicity)
best.set_C_min(strategy.completeness)
best.set_GpS(strategy.dose_rate)
best.set_shape(strategy.shape)
best.set_susceptibility(strategy.susceptibility)
best.set_anomalous(strategy.anomalous)
best.set_detector("pilatus6m")
best.set_working_directory(wd)
auto_logfiler(best)
xmlout = os.path.join(best.get_working_directory(),
"%i_best.xml" % best.get_xpid())
best.set_xmlout(xmlout)
best.strategy()
results = best.get_results_dict()
results["description"] = strategy.description
if "phi_end" not in results:
results["phi_end"] = str(
float(results["phi_start"]) +
float(results["number_of_images"]) *
float(results["phi_width"]))
from dxtbx.serialize import load
expt = load.experiment_list(experiments[0])[0]
results["spacegroup"] = (
expt.crystal.get_space_group().type().lookup_symbol())
return results
args = []
for istrategy, strategy in enumerate(strategy_params):
imageset = sweeps[0].get_imageset()
scan = imageset.get_scan()
experiments = [
sweep._get_integrater().get_integrated_experiments()
for sweep in sweeps
]
reflections = [
sweep._get_integrater().get_integrated_reflections()
for sweep in sweeps
]
t_ref = scan.get_exposure_times()[0]
args.append((experiments, reflections, strategy, t_ref))
nproc = params.xia2.settings.multiprocessing.nproc
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_strategy,
args,
processes=nproc,
method="multiprocessing",
preserve_order=True,
preserve_exception_message=True,
)
for istrategy, (result,
strategy) in enumerate(zip(results, strategy_params)):
name = strategy.name
description = strategy.description
if name is None:
name = "Strategy%i" % (istrategy + 1)
results_all[name] = result
multiplicity = result["redundancy"]
try:
multiplicity = "%.2f" % multiplicity
except TypeError:
pass
logger.info("Strategy %i", istrategy)
if description is not None:
logger.info(description)
logger.info(
"Start / end / width: %.2f/%.2f/%.2f",
float(result["phi_start"]),
float(result["phi_end"]),
float(result["phi_width"]),
)
logger.info(
"Completeness / multiplicity / resolution: %.2f/%s/%.2f",
float(result["completeness"]),
multiplicity,
float(result["resolution"]),
)
logger.info(
"Transmission / exposure %.3f/%.3f",
float(result["transmission"]),
float(result["exposure_time"]),
)
with open("strategies.json", "wb") as f:
json.dump(results_all, f, indent=2)
except Exception as e:
with open(os.path.join(cwd, "xia2-error.txt"), "w") as fh:
traceback.print_exc(file=fh)
logger.error('Status: error "%s"', str(e))
os.chdir(cwd)
def _index(self):
'''Actually index the diffraction pattern. Note well that
this is not going to compute the matrix...'''
# acknowledge this program
Citations.cite('labelit')
Citations.cite('distl')
#self.reset()
_images = []
for i in self._indxr_images:
for j in i:
if not j in _images:
_images.append(j)
_images.sort()
images_str = '%d' % _images[0]
for i in _images[1:]:
images_str += ', %d' % i
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
if self._indxr_sweep_name:
# then this is a proper autoindexing run - describe this
# to the journal entry
#if len(self._fp_directory) <= 50:
#dirname = self._fp_directory
#else:
#dirname = '...%s' % self._fp_directory[-46:]
dirname = os.path.dirname(self.get_imageset().get_template())
Journal.block(
'autoindexing', self._indxr_sweep_name, 'labelit', {
'images': images_str,
'target cell': cell_str,
'target lattice': self._indxr_input_lattice,
'template': self.get_imageset().get_template(),
'directory': dirname
})
if len(_images) > 4:
raise RuntimeError('cannot use more than 4 images')
from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex
index = LabelitIndex()
index.set_working_directory(self.get_working_directory())
auto_logfiler(index)
#task = 'Autoindex from images:'
#for i in _images:
#task += ' %s' % self.get_image_name(i)
#self.set_task(task)
Debug.write('Indexing from images:')
for i in _images:
index.add_image(self.get_image_name(i))
Debug.write('%s' % self.get_image_name(i))
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
index.set_distance(xsweep.get_distance())
#if self.get_wavelength_prov() == 'user':
#index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
index.set_beam_centre(xsweep.get_beam_centre())
if self._refine_beam is False:
index.set_refine_beam(False)
else:
index.set_refine_beam(True)
index.set_beam_search_scope(self._beam_search_scope)
if ((math.fabs(self.get_wavelength() - 1.54) < 0.01)
or (math.fabs(self.get_wavelength() - 2.29) < 0.01)):
index.set_Cu_KA_or_Cr_KA(True)
#sweep = self.get_indexer_sweep_name()
#FileHandler.record_log_file(
#'%s INDEX' % (sweep), self.get_log_file())
try:
index.run()
except RuntimeError as e:
if self._refine_beam is False:
raise e
# can we improve the situation?
if self._beam_search_scope < 4.0:
self._beam_search_scope += 4.0
# try repeating the indexing!
self.set_indexer_done(False)
return 'failed'
# otherwise this is beyond redemption
raise e
self._solutions = index.get_solutions()
# FIXME this needs to check the smilie status e.g.
# ":)" or ";(" or " ".
# FIXME need to check the value of the RMSD and raise an
# exception if the P1 solution has an RMSD > 1.0...
# Change 27/FEB/08 to support user assigned spacegroups
# (euugh!) have to "ignore" solutions with higher symmetry
# otherwise the rest of xia will override us. Bummer.
for i, solution in self._solutions.iteritems():
if self._indxr_user_input_lattice:
if (lattice_to_spacegroup(solution['lattice']) >
lattice_to_spacegroup(self._indxr_input_lattice)):
Debug.write('Ignoring solution: %s' % solution['lattice'])
del self._solutions[i]
# check the RMSD from the triclinic unit cell
if self._solutions[1]['rmsd'] > 1.0 and False:
# don't know when this is useful - but I know when it is not!
raise RuntimeError('high RMSD for triclinic solution')
# configure the "right" solution
self._solution = self.get_solution()
# now store also all of the other solutions... keyed by the
# lattice - however these should only be added if they
# have a smiley in the appropriate record, perhaps?
for solution in self._solutions.keys():
lattice = self._solutions[solution]['lattice']
if lattice in self._indxr_other_lattice_cell:
if self._indxr_other_lattice_cell[lattice]['goodness'] < \
self._solutions[solution]['metric']:
continue
self._indxr_other_lattice_cell[lattice] = {
'goodness': self._solutions[solution]['metric'],
'cell': self._solutions[solution]['cell']
}
self._indxr_lattice = self._solution['lattice']
self._indxr_cell = tuple(self._solution['cell'])
self._indxr_mosaic = self._solution['mosaic']
lms = LabelitMosflmScript()
lms.set_working_directory(self.get_working_directory())
lms.set_solution(self._solution['number'])
self._indxr_payload['mosflm_orientation_matrix'] = lms.calculate()
# get the beam centre from the mosflm script - mosflm
# may have inverted the beam centre and labelit will know
# this!
mosflm_beam_centre = lms.get_mosflm_beam()
if mosflm_beam_centre:
self._indxr_payload['mosflm_beam_centre'] = tuple(
mosflm_beam_centre)
import copy
detector = copy.deepcopy(self.get_detector())
beam = copy.deepcopy(self.get_beam())
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
set_mosflm_beam_centre(detector, beam, mosflm_beam_centre)
from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
from scitbx import matrix
from cctbx import sgtbx, uctbx
from dxtbx.model import CrystalFactory
mosflm_matrix = matrix.sqr([
float(i) for line in lms.calculate()
for i in line.replace("-", " -").split()
][:9])
space_group = sgtbx.space_group_info(
lattice_to_spacegroup_number(self._solution['lattice'])).group()
crystal_model = CrystalFactory.from_mosflm_matrix(
mosflm_matrix,
unit_cell=uctbx.unit_cell(tuple(self._solution['cell'])),
space_group=space_group)
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(
beam=beam,
detector=detector,
goniometer=self.get_goniometer(),
scan=self.get_scan(),
crystal=crystal_model,
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# also get an estimate of the resolution limit from the
# labelit.stats_distl output... FIXME the name is wrong!
lsd = LabelitStats_distl()
lsd.set_working_directory(self.get_working_directory())
lsd.stats_distl()
resolution = 1.0e6
for i in _images:
stats = lsd.get_statistics(self.get_image_name(i))
resol = 0.5 * (stats['resol_one'] + stats['resol_two'])
if resol < resolution:
resolution = resol
self._indxr_resolution_estimate = resolution
return 'ok'
def dials_symmetry(self):
symmetry = _DialsSymmetry()
symmetry.set_working_directory(self.get_working_directory())
auto_logfiler(symmetry)
return symmetry
def _index_prepare(self):
"""Prepare to do autoindexing - in XDS terms this will mean
calling xycorr, init and colspot on the input images."""
# decide on images to work with
logger.debug("XDS INDEX PREPARE:")
logger.debug("Wavelength: %.6f", self.get_wavelength())
logger.debug("Distance: %.2f", self.get_distance())
if self._indxr_images == []:
_select_images_function = getattr(
self, "_index_select_images_%s" % self._index_select_images
)
wedges = _select_images_function()
for wedge in wedges:
self.add_indexer_image_wedge(wedge)
self.set_indexer_prepare_done(True)
all_images = self.get_matching_images()
first = min(all_images)
last = max(all_images)
# next start to process these - first xycorr
xycorr = self.Xycorr()
xycorr.set_data_range(first, last)
xycorr.set_background_range(self._indxr_images[0][0], self._indxr_images[0][1])
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
xycorr.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
for block in self._indxr_images:
xycorr.add_spot_range(block[0], block[1])
# FIXME need to set the origin here
xycorr.run()
for file in ["X-CORRECTIONS.cbf", "Y-CORRECTIONS.cbf"]:
self._indxr_payload[file] = xycorr.get_output_data_file(file)
# next start to process these - then init
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
imageset = self._indxr_imagesets[0]
masker = (
imageset.get_format_class()
.get_instance(imageset.paths()[0])
.get_masker()
)
if masker is None:
# disable dynamic_shadowing
PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
# find the region of the scan with the least predicted shadow
# to use for background determination in XDS INIT step
from dxtbx.model.experiment_list import ExperimentListFactory
imageset = self._indxr_imagesets[0]
xsweep = self._indxr_sweeps[0]
sweep_filename = os.path.join(
self.get_working_directory(), "%s_indexed.expt" % xsweep.get_name()
)
ExperimentListFactory.from_imageset_and_crystal(imageset, None).as_file(
sweep_filename
)
from xia2.Wrappers.Dials.ShadowPlot import ShadowPlot
shadow_plot = ShadowPlot()
shadow_plot.set_working_directory(self.get_working_directory())
auto_logfiler(shadow_plot)
shadow_plot.set_sweep_filename(sweep_filename)
shadow_plot.set_json_filename(
os.path.join(
self.get_working_directory(),
"%s_shadow_plot.json" % shadow_plot.get_xpid(),
)
)
shadow_plot.run()
results = shadow_plot.get_results()
fraction_shadowed = flex.double(results["fraction_shadowed"])
if flex.max(fraction_shadowed) == 0:
PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False
else:
scan_points = flex.double(results["scan_points"])
scan = imageset.get_scan()
oscillation = scan.get_oscillation()
if self._background_images is not None:
bg_images = self._background_images
bg_range_deg = (
scan.get_angle_from_image_index(bg_images[0]),
scan.get_angle_from_image_index(bg_images[1]),
)
bg_range_width = bg_range_deg[1] - bg_range_deg[0]
min_shadow = 100
best_bg_range = bg_range_deg
from libtbx.utils import frange
for bg_range_start in frange(
flex.min(scan_points),
flex.max(scan_points) - bg_range_width,
step=oscillation[1],
):
bg_range_deg = (bg_range_start, bg_range_start + bg_range_width)
sel = (scan_points >= bg_range_deg[0]) & (
scan_points <= bg_range_deg[1]
)
mean_shadow = flex.mean(fraction_shadowed.select(sel))
if mean_shadow < min_shadow:
min_shadow = mean_shadow
best_bg_range = bg_range_deg
self._background_images = (
scan.get_image_index_from_angle(best_bg_range[0]),
scan.get_image_index_from_angle(best_bg_range[1]),
)
logger.debug(
"Setting background images: %s -> %s" % self._background_images
)
init = self.Init()
for file in ["X-CORRECTIONS.cbf", "Y-CORRECTIONS.cbf"]:
init.set_input_data_file(file, self._indxr_payload[file])
init.set_data_range(first, last)
if self._background_images:
init.set_background_range(
self._background_images[0], self._background_images[1]
)
else:
init.set_background_range(
self._indxr_images[0][0], self._indxr_images[0][1]
)
for block in self._indxr_images:
init.add_spot_range(block[0], block[1])
init.run()
# at this stage, need to (perhaps) modify the BKGINIT.cbf image
# to mark out the back stop
if PhilIndex.params.xds.backstop_mask:
logger.debug("Applying mask to BKGINIT.pck")
# copy the original file
cbf_old = os.path.join(init.get_working_directory(), "BKGINIT.cbf")
cbf_save = os.path.join(init.get_working_directory(), "BKGINIT.sav")
shutil.copyfile(cbf_old, cbf_save)
# modify the file to give the new mask
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.xds.backstop_mask)
mask.apply_mask_xds(self.get_header(), cbf_save, cbf_old)
init.reload()
for file in ["BLANK.cbf", "BKGINIT.cbf", "GAIN.cbf"]:
self._indxr_payload[file] = init.get_output_data_file(file)
if PhilIndex.params.xia2.settings.developmental.use_dials_spotfinder:
spotfinder = self.DialsSpotfinder()
for block in self._indxr_images:
spotfinder.add_spot_range(block[0], block[1])
spotfinder.run()
export = self.DialsExportSpotXDS()
export.set_input_data_file(
"observations.refl",
spotfinder.get_output_data_file("observations.refl"),
)
export.run()
for file in ["SPOT.XDS"]:
self._indxr_payload[file] = export.get_output_data_file(file)
else:
# next start to process these - then colspot
colspot = self.Colspot()
for file in (
"X-CORRECTIONS.cbf",
"Y-CORRECTIONS.cbf",
"BLANK.cbf",
"BKGINIT.cbf",
"GAIN.cbf",
):
colspot.set_input_data_file(file, self._indxr_payload[file])
colspot.set_data_range(first, last)
colspot.set_background_range(
self._indxr_images[0][0], self._indxr_images[0][1]
)
for block in self._indxr_images:
colspot.add_spot_range(block[0], block[1])
colspot.run()
for file in ["SPOT.XDS"]:
self._indxr_payload[file] = colspot.get_output_data_file(file)
def _index(self):
"""Actually index the diffraction pattern. Note well that
this is not going to compute the matrix..."""
# acknowledge this program
if not self._indxr_images:
raise RuntimeError("No good spots found on any images")
Citations.cite("labelit")
Citations.cite("distl")
_images = []
for i in self._indxr_images:
for j in i:
if not j in _images:
_images.append(j)
_images.sort()
images_str = "%d" % _images[0]
for i in _images[1:]:
images_str += ", %d" % i
cell_str = None
if self._indxr_input_cell:
cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % self._indxr_input_cell
if self._indxr_sweep_name:
# then this is a proper autoindexing run - describe this
# to the journal entry
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = "...%s" % self._fp_directory[-46:]
Journal.block(
"autoindexing",
self._indxr_sweep_name,
"labelit",
{
"images": images_str,
"target cell": cell_str,
"target lattice": self._indxr_input_lattice,
"template": self._fp_template,
"directory": dirname,
},
)
# auto_logfiler(self)
from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex
index = LabelitIndex()
index.set_working_directory(self.get_working_directory())
auto_logfiler(index)
# task = 'Autoindex from images:'
# for i in _images:
# task += ' %s' % self.get_image_name(i)
# self.set_task(task)
# self.add_command_line('--index_only')
Debug.write("Indexing from images:")
for i in _images:
index.add_image(self.get_image_name(i))
Debug.write("%s" % self.get_image_name(i))
if self._primitive_unit_cell:
index.set_primitive_unit_cell(self._primitive_unit_cell)
if self._indxr_input_cell:
index.set_max_cell(1.25 * max(self._indxr_input_cell[:3]))
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
index.set_distance(xsweep.get_distance())
# if self.get_wavelength_prov() == 'user':
# index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
index.set_beam_centre(xsweep.get_beam_centre())
if self._refine_beam is False:
index.set_refine_beam(False)
else:
index.set_refine_beam(True)
index.set_beam_search_scope(self._beam_search_scope)
if (math.fabs(self.get_wavelength() - 1.54) <
0.01) or (math.fabs(self.get_wavelength() - 2.29) < 0.01):
index.set_Cu_KA_or_Cr_KA(True)
try:
index.run()
except RuntimeError as e:
if self._refine_beam is False:
raise e
# can we improve the situation?
if self._beam_search_scope < 4.0:
self._beam_search_scope += 4.0
# try repeating the indexing!
self.set_indexer_done(False)
return "failed"
# otherwise this is beyond redemption
raise e
self._solutions = index.get_solutions()
# FIXME this needs to check the smilie status e.g.
# ":)" or ";(" or " ".
# FIXME need to check the value of the RMSD and raise an
# exception if the P1 solution has an RMSD > 1.0...
# Change 27/FEB/08 to support user assigned spacegroups
# (euugh!) have to "ignore" solutions with higher symmetry
# otherwise the rest of xia will override us. Bummer.
for i, solution in self._solutions.iteritems():
if self._indxr_user_input_lattice:
if lattice_to_spacegroup(
solution["lattice"]) > lattice_to_spacegroup(
self._indxr_input_lattice):
Debug.write("Ignoring solution: %s" % solution["lattice"])
del self._solutions[i]
# configure the "right" solution
self._solution = self.get_solution()
# now store also all of the other solutions... keyed by the
# lattice - however these should only be added if they
# have a smiley in the appropriate record, perhaps?
for solution in self._solutions.keys():
lattice = self._solutions[solution]["lattice"]
if lattice in self._indxr_other_lattice_cell:
if (self._indxr_other_lattice_cell[lattice]["goodness"] <
self._solutions[solution]["metric"]):
continue
self._indxr_other_lattice_cell[lattice] = {
"goodness": self._solutions[solution]["metric"],
"cell": self._solutions[solution]["cell"],
}
self._indxr_lattice = self._solution["lattice"]
self._indxr_cell = tuple(self._solution["cell"])
self._indxr_mosaic = self._solution["mosaic"]
lms = LabelitMosflmMatrix()
lms.set_working_directory(self.get_working_directory())
lms.set_solution(self._solution["number"])
self._indxr_payload["mosflm_orientation_matrix"] = lms.calculate()
# get the beam centre from the mosflm script - mosflm
# may have inverted the beam centre and labelit will know
# this!
mosflm_beam_centre = lms.get_mosflm_beam()
if mosflm_beam_centre:
self._indxr_payload["mosflm_beam_centre"] = tuple(
mosflm_beam_centre)
detector = copy.deepcopy(self.get_detector())
beam = copy.deepcopy(self.get_beam())
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
set_mosflm_beam_centre(detector, beam, mosflm_beam_centre)
from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
from scitbx import matrix
from cctbx import sgtbx, uctbx
from dxtbx.model import CrystalFactory
mosflm_matrix = matrix.sqr([
float(i) for line in lms.calculate()
for i in line.replace("-", " -").split()
][:9])
space_group = sgtbx.space_group_info(
lattice_to_spacegroup_number(self._solution["lattice"])).group()
crystal_model = CrystalFactory.from_mosflm_matrix(
mosflm_matrix,
unit_cell=uctbx.unit_cell(tuple(self._solution["cell"])),
space_group=space_group,
)
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(
beam=beam,
detector=detector,
goniometer=self.get_goniometer(),
scan=self.get_scan(),
crystal=crystal_model,
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# also get an estimate of the resolution limit from the
# labelit.stats_distl output... FIXME the name is wrong!
lsd = LabelitStats_distl()
lsd.set_working_directory(self.get_working_directory())
lsd.stats_distl()
resolution = 1.0e6
for i in _images:
stats = lsd.get_statistics(self.get_image_name(i))
resol = 0.5 * (stats["resol_one"] + stats["resol_two"])
if resol < resolution:
resolution = resol
self._indxr_resolution_estimate = resolution
return "ok"
def process_one_sweep(args):
assert len(args) == 1
args = args[0]
# stop_after = args.stop_after
command_line_args = args.command_line_args
nproc = args.nproc
crystal_id = args.crystal_id
wavelength_id = args.wavelength_id
sweep_id = args.sweep_id
failover = args.failover
driver_type = args.driver_type
default_driver_type = DriverFactory.get_driver_type()
DriverFactory.set_driver_type(driver_type)
curdir = os.path.abspath(os.curdir)
if "-xinfo" in command_line_args:
idx = command_line_args.index("-xinfo")
del command_line_args[idx + 1]
del command_line_args[idx]
xia2_integrate = XIA2Integrate()
# import tempfile
# tmpdir = tempfile.mkdtemp(dir=curdir)
tmpdir = os.path.join(curdir, str(uuid.uuid4()))
os.makedirs(tmpdir)
xia2_integrate.set_working_directory(tmpdir)
xia2_integrate.add_command_line_args(args.command_line_args)
xia2_integrate.set_phil_file(os.path.join(curdir, "xia2-working.phil"))
xia2_integrate.add_command_line_args(["sweep.id=%s" % sweep_id])
xia2_integrate.set_nproc(nproc)
xia2_integrate.set_njob(1)
xia2_integrate.set_mp_mode("serial")
auto_logfiler(xia2_integrate)
sweep_tmp_dir = os.path.join(tmpdir, crystal_id, wavelength_id, sweep_id)
sweep_target_dir = os.path.join(curdir, crystal_id, wavelength_id,
sweep_id)
output = None
success = False
xsweep_dict = None
try:
xia2_integrate.run()
output = get_sweep_output_only(xia2_integrate.get_all_output())
success = True
except Exception as e:
logger.warning("Processing sweep %s failed: %s", sweep_id, str(e))
if not failover:
raise
finally:
from xia2.Schema.XProject import XProject
xia2_json = os.path.join(tmpdir, "xia2.json")
json_files = glob.glob(os.path.join(sweep_tmp_dir, "*", "*.json"))
json_files.extend(glob.glob(os.path.join(sweep_tmp_dir, "*",
"*.expt")))
if os.path.exists(xia2_json):
json_files.append(xia2_json)
import fileinput
for line in fileinput.FileInput(files=json_files, inplace=1):
line = line.replace(sweep_tmp_dir, sweep_target_dir)
print(line)
if os.path.exists(xia2_json):
new_json = os.path.join(curdir, "xia2-%s.json" % sweep_id)
shutil.copyfile(xia2_json, new_json)
move_output_folder(sweep_tmp_dir, sweep_target_dir)
if success:
xinfo = XProject.from_json(new_json)
xcryst = list(xinfo.get_crystals().values())[0]
xsweep = xcryst.get_xwavelength(wavelength_id).get_sweeps()[0]
xsweep_dict = xsweep.to_dict()
shutil.rmtree(tmpdir, ignore_errors=True)
if os.path.exists(tmpdir):
shutil.rmtree(tmpdir, ignore_errors=True)
DriverFactory.set_driver_type(default_driver_type)
return success, output, xsweep_dict
def ExportXDS(self):
from xia2.Wrappers.Dials.ExportXDS import ExportXDS as _ExportXDS
export_xds = _ExportXDS()
export_xds.set_working_directory(self.get_working_directory())
auto_logfiler(export_xds)
return export_xds
def DetectBlanks(self):
detectblanks = _DetectBlanks()
detectblanks.set_working_directory(self.get_working_directory())
auto_logfiler(detectblanks)
return detectblanks
def Report(self):
report = _Report()
report.set_working_directory(self.get_working_directory())
auto_logfiler(report, 'REPORT')
return report
def _mosflm_test_refine_cell(self, idxr, test_lattice):
'''Test performing cell refinement in with a different
lattice to the one which was selected by the autoindex
procedure. This should not change anything in the class.'''
#idxr = self.get_integrater_indexer()
lattice = idxr.get_indexer_lattice()
mosaic = idxr.get_indexer_mosaic()
beam_centre = idxr.get_indexer_beam_centre()
distance = idxr.get_indexer_distance()
matrix = idxr.get_indexer_payload('mosflm_orientation_matrix')
phi_width = idxr.get_phi_width()
if mosaic < 0.25 * phi_width:
mosaic = 0.25 * phi_width
input_matrix = ''
for m in matrix:
input_matrix += '%s\n' % m
new_matrix = transmogrify_matrix(lattice, input_matrix, test_lattice,
idxr.get_wavelength(),
self.get_working_directory())
spacegroup_number = lattice_to_spacegroup(test_lattice)
if not self._mosflm_cell_ref_images:
raise RuntimeError('wedges must be assigned already')
open(
os.path.join(self.get_working_directory(),
'test-xiaindex-%s.mat' % lattice),
'w').write(new_matrix)
refiner = MosflmRefineCell()
refiner.set_working_directory(self.get_working_directory())
auto_logfiler(refiner)
if self._mosflm_gain:
refiner.set_gain(self._mosflm_gain)
refiner.set_template(os.path.basename(idxr.get_template()))
refiner.set_directory(idxr.get_directory())
refiner.set_input_mat_file('test-xiaindex-%s.mat' % lattice)
refiner.set_output_mat_file('test-xiarefine.mat')
refiner.set_beam_centre(beam_centre)
refiner.set_distance(distance)
refiner.set_space_group_number(spacegroup_number)
# FIXME 18/JUN/08 - it may help to have an overestimate
# of the mosaic spread in here as it *may* refine down
# better than up... - this is not a good idea as it may
# also not refine at all! - 12972 # integration failed
# Bug # 3103
if self._mosflm_cell_ref_double_mosaic:
mosaic *= 2.0
refiner.set_mosaic(mosaic)
# if set, use the resolution for cell refinement - see
# bug # 2078...
if self._mosflm_cell_ref_resolution:
refiner.set_resolution(self._mosflm_cell_ref_resolution)
refiner.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
# note well that the beam centre is coming from indexing so
# should be already properly handled
#if idxr.get_wavelength_prov() == 'user':
#refiner.set_wavelength(idxr.get_wavelength())
# belt + braces mode - only to be used when considering failover,
# will run an additional step of autoindexing prior to cell
# refinement, to be used only after proving that not going it
# will result in cell refinement failure - will use the first
# wedge... N.B. this is only useful if the indexer is Labelit
# not Mosflm...
refiner.set_add_autoindex(self._mosflm_cell_ref_add_autoindex)
# XXX
# get all of the stored parameter values
parameters = self.get_refiner_parameters('mosflm')
refiner.update_parameters(parameters)
detector = idxr.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
refiner.set_limits(lim_x, lim_y)
refiner.set_images(self._mosflm_cell_ref_images)
refiner.run()
rms_values = refiner.get_rms_values()
background_residual = refiner.get_background_residual()
return rms_values, background_residual
def SearchBeamPosition(self):
discovery = _SearchBeamPosition()
discovery.set_working_directory(self.get_working_directory())
# params = PhilIndex.params.dials.index
auto_logfiler(discovery)
return discovery
def DiscoverBetterExperimentalModel(self):
discovery = _DiscoverBetterExperimentalModel()
discovery.set_working_directory(self.get_working_directory())
#params = PhilIndex.params.dials.index
auto_logfiler(discovery)
return discovery
def _mosflm_integrate(self):
'''Perform the actual integration, based on the results of the
cell refinement or indexing (they have the equivalent form.)'''
refinr = self.get_integrater_refiner()
if not refinr.get_refiner_payload('mosflm_orientation_matrix'):
raise RuntimeError('unexpected situation in indexing')
lattice = refinr.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup(lattice)
mosaic = refinr.get_refiner_payload('mosaic')
beam = refinr.get_refiner_payload('beam')
distance = refinr.get_refiner_payload('distance')
matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')
integration_params = refinr.get_refiner_payload(
'mosflm_integration_parameters')
if integration_params:
if 'separation' in integration_params:
self.set_integrater_parameter(
'mosflm', 'separation',
'%s %s' % tuple(integration_params['separation']))
if 'raster' in integration_params:
self.set_integrater_parameter(
'mosflm', 'raster',
'%d %d %d %d %d' % tuple(integration_params['raster']))
refinr.set_refiner_payload('mosflm_integration_parameters', None)
f = open(
os.path.join(self.get_working_directory(), 'xiaintegrate.mat'),
'w')
for m in matrix:
f.write(m)
f.close()
# then start the integration
integrater = MosflmIntegrate()
integrater.set_working_directory(self.get_working_directory())
auto_logfiler(integrater)
integrater.set_refine_profiles(self._mosflm_refine_profiles)
pname, xname, dname = self.get_integrater_project_info()
if pname is not None and xname is not None and dname is not None:
Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname))
harvest_dir = self.get_working_directory()
# harvest file name will be %s.mosflm_run_start_end % dname
temp_dname = '%s_%s' % \
(dname, self.get_integrater_sweep_name())
integrater.set_pname_xname_dname(pname, xname, temp_dname)
integrater.set_template(os.path.basename(self.get_template()))
integrater.set_directory(self.get_directory())
# check for ice - and if so, exclude (ranges taken from
# XDS documentation)
if self.get_integrater_ice() != 0:
Debug.write('Excluding ice rings')
integrater.set_exclude_ice(True)
# exclude specified resolution ranges
if len(self.get_integrater_excluded_regions()) != 0:
regions = self.get_integrater_excluded_regions()
Debug.write('Excluding regions: %s' % repr(regions))
integrater.set_exclude_regions(regions)
mask = standard_mask(self.get_detector())
for m in mask:
integrater.add_instruction(m)
integrater.set_input_mat_file('xiaintegrate.mat')
integrater.set_beam_centre(beam)
integrater.set_distance(distance)
integrater.set_space_group_number(spacegroup_number)
integrater.set_mosaic(mosaic)
if self.get_wavelength_prov() == 'user':
integrater.set_wavelength(self.get_wavelength())
parameters = self.get_integrater_parameters('mosflm')
integrater.update_parameters(parameters)
if self._mosflm_gain:
integrater.set_gain(self._mosflm_gain)
# check for resolution limits
if self._intgr_reso_high > 0.0:
integrater.set_d_min(self._intgr_reso_high)
if self._intgr_reso_low:
integrater.set_d_max(self._intgr_reso_low)
if PhilIndex.params.general.backstop_mask:
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask = mask.calculate_mask_mosflm(self.get_header())
integrater.set_mask(mask)
detector = self.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
integrater.set_limits(lim_x, lim_y)
integrater.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
offset = self.get_frame_offset()
integrater.set_image_range(
(self._intgr_wedge[0] - offset, self._intgr_wedge[1] - offset))
try:
integrater.run()
except RuntimeError as e:
if 'integration failed: reason unknown' in str(e):
Chatter.write('Mosflm has failed in integration')
message = 'The input was:\n\n'
for input in integrater.get_all_input():
message += ' %s' % input
Chatter.write(message)
raise
FileHandler.record_log_file(
'%s %s %s %s mosflm integrate' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
integrater.get_log_file())
self._intgr_per_image_statistics = integrater.get_per_image_statistics(
)
self._mosflm_hklout = integrater.get_hklout()
Debug.write('Integration output: %s' % self._mosflm_hklout)
self._intgr_n_ref = integrater.get_nref()
# if a BGSIG error happened try not refining the
# profile and running again...
if integrater.get_bgsig_too_large():
if not self._mosflm_refine_profiles:
raise RuntimeError('BGSIG error with profiles fixed')
Debug.write('BGSIG error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
if integrater.get_getprof_error():
Debug.write('GETPROF error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
if (integrater.get_detector_gain_error()
and not (self.get_imageset().get_detector()[0].get_type()
== 'SENSOR_PAD')):
gain = integrater.get_suggested_gain()
if gain is not None:
self.set_integrater_parameter('mosflm', 'gain', gain)
self.set_integrater_export_parameter('mosflm', 'gain', gain)
if self._mosflm_gain:
Debug.write('GAIN updated to %f' % gain)
else:
Debug.write('GAIN found to be %f' % gain)
self._mosflm_gain = gain
self._mosflm_rerun_integration = True
if not self._mosflm_hklout:
raise RuntimeError('processing abandoned')
self._intgr_batches_out = integrater.get_batches_out()
mosaics = integrater.get_mosaic_spreads()
if mosaics and len(mosaics) > 0:
self.set_integrater_mosaic_min_mean_max(
min(mosaics),
sum(mosaics) / len(mosaics), max(mosaics))
else:
m = indxr.get_indexer_mosaic()
self.set_integrater_mosaic_min_mean_max(m, m, m)
# write the report for each image as .*-#$ to Chatter -
# detailed report will be written automagically to science...
Chatter.write(self.show_per_image_statistics())
Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
self.get_integrater_mosaic_min_mean_max())
# gather the statistics from the postrefinement
postref_result = integrater.get_postref_result()
# now write this to a postrefinement log
postref_log = os.path.join(self.get_working_directory(),
'postrefinement.log')
fout = open(postref_log, 'w')
fout.write('$TABLE: Postrefinement for %s:\n' % \
self._intgr_sweep_name)
fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n')
fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n')
for image in sorted(postref_result):
phix = postref_result[image].get('phix', 0.0)
phiy = postref_result[image].get('phiy', 0.0)
phiz = postref_result[image].get('phiz', 0.0)
fout.write('%d %5.2f %5.2f %5.2f\n' % \
(image, phix, phiy, phiz))
fout.write('$$\n')
fout.close()
if self.get_integrater_sweep_name():
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file('%s %s %s %s postrefinement' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
postref_log)
return self._mosflm_hklout
def CheckIndexingSymmetry(self):
checksym = _CheckIndexingSymmetry()
checksym.set_working_directory(self.get_working_directory())
auto_logfiler(checksym)
return checksym
def Reindex(self):
reindex = _Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
return reindex
def _mosflm_refine_cell(self, idxr, set_spacegroup=None):
'''Perform the refinement of the unit cell. This will populate
all of the information needed to perform the integration.'''
# FIXME this will die after #1285
#if not self.get_integrater_indexer():
#Debug.write('Replacing indexer of %s with self at %d' % \
#(str(self.get_integrater_indexer()), __line__))
#self.set_integrater_indexer(self)
#idxr = self.get_integrater_indexer()
if not idxr.get_indexer_payload('mosflm_orientation_matrix'):
raise RuntimeError('unexpected situation in indexing')
lattice = idxr.get_indexer_lattice()
mosaic = idxr.get_indexer_mosaic()
cell = idxr.get_indexer_cell()
beam_centre = idxr.get_indexer_beam_centre()
# bug # 3174 - if mosaic is very small (here defined to be
# 0.25 x osc_width) then set to this minimum value.
phi_width = idxr.get_phi_width()
if mosaic < 0.25 * phi_width:
mosaic = 0.25 * phi_width
if idxr.get_indexer_payload('mosflm_beam_centre'):
beam_centre = idxr.get_indexer_payload('mosflm_beam_centre')
distance = idxr.get_indexer_distance()
matrix = idxr.get_indexer_payload('mosflm_orientation_matrix')
integration_params = idxr.get_indexer_payload(
'mosflm_integration_parameters')
if integration_params is None:
integration_params = {}
if integration_params:
if 'separation' in integration_params:
self.set_refiner_parameter(
'mosflm', 'separation',
'%f %f' % tuple(integration_params['separation']))
if 'raster' in integration_params:
self.set_refiner_parameter(
'mosflm', 'raster',
'%d %d %d %d %d' % tuple(integration_params['raster']))
idxr.set_indexer_payload('mosflm_integration_parameters', None)
spacegroup_number = lattice_to_spacegroup(lattice)
# copy these into myself for later reference, if indexer
# is not myself - everything else is copied via the
# cell refinement process...
from cctbx import sgtbx
from dxtbx.model import Crystal
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
experiment = idxr.get_indexer_experiment_list()[0]
set_mosflm_beam_centre(experiment.detector, experiment.beam,
beam_centre)
space_group = sgtbx.space_group_info(number=spacegroup_number).group()
a, b, c = experiment.crystal.get_real_space_vectors()
experiment.crystal = Crystal(a, b, c, space_group=space_group)
# FIXME surely these have been assigned further up?!
if not self._mosflm_cell_ref_images:
self._mosflm_cell_ref_images = self._refine_select_images(mosaic)
f = open(
os.path.join(self.get_working_directory(),
'xiaindex-%s.mat' % lattice), 'w')
for m in matrix:
f.write(m)
f.close()
# then start the cell refinement
refiner = MosflmRefineCell()
refiner.set_working_directory(self.get_working_directory())
auto_logfiler(refiner)
if self._mosflm_gain:
refiner.set_gain(self._mosflm_gain)
refiner.set_template(os.path.basename(idxr.get_template()))
refiner.set_directory(idxr.get_directory())
refiner.set_input_mat_file('xiaindex-%s.mat' % lattice)
refiner.set_output_mat_file('xiarefine.mat')
refiner.set_beam_centre(beam_centre)
refiner.set_unit_cell(cell)
refiner.set_distance(distance)
if set_spacegroup:
refiner.set_space_group_number(set_spacegroup)
else:
refiner.set_space_group_number(spacegroup_number)
# FIXME 18/JUN/08 - it may help to have an overestimate
# of the mosaic spread in here as it *may* refine down
# better than up... - this is not a good idea as it may
# also not refine at all! - 12972 # integration failed
# Bug # 3103
if self._mosflm_cell_ref_double_mosaic:
mosaic *= 2.0
refiner.set_mosaic(mosaic)
# if set, use the resolution for cell refinement - see
# bug # 2078...
if self._mosflm_cell_ref_resolution:
refiner.set_resolution(self._mosflm_cell_ref_resolution)
refiner.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
# note well that the beam centre is coming from indexing so
# should be already properly handled
#if idxr.get_wavelength_prov() == 'user':
#refiner.set_wavelength(idxr.get_wavelength())
# belt + braces mode - only to be used when considering failover,
# will run an additional step of autoindexing prior to cell
# refinement, to be used only after proving that not going it
# will result in cell refinement failure - will use the first
# wedge... N.B. this is only useful if the indexer is Labelit
# not Mosflm...
refiner.set_add_autoindex(self._mosflm_cell_ref_add_autoindex)
# get all of the stored parameter values
parameters = self.get_refiner_parameters('mosflm')
refiner.update_parameters(parameters)
detector = idxr.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
refiner.set_limits(lim_x, lim_y)
refiner.set_images(self._mosflm_cell_ref_images)
failover = PhilIndex.params.xia2.settings.failover
if failover and not self._mosflm_cell_ref_add_autoindex:
refiner.set_ignore_cell_refinement_failure(True)
refiner.run()
# then look to see if the cell refinement worked ok - if it
# didn't then this may indicate that the lattice was wrongly
# selected.
cell_refinement_ok = refiner.cell_refinement_ok()
if not cell_refinement_ok:
Debug.write('Repeating cell refinement...')
self.set_integrater_prepare_done(False)
self._mosflm_cell_ref_add_autoindex = True
return [0.0], [0.0]
rms_values = refiner.get_rms_values()
background_residual = refiner.get_background_residual()
self._refinr_cell = refiner.get_refined_unit_cell()
distance = refiner.get_refined_distance2()
experiment = idxr.get_indexer_experiment_list()[0]
from xia2.Wrappers.Mosflm.AutoindexHelpers import set_distance
set_distance(experiment.detector, distance)
self.set_refiner_parameter('mosflm', 'distortion yscale',
refiner.get_refined_distortion_yscale())
self.set_refiner_parameter('mosflm', 'raster', refiner.get_raster())
#integration_params['distortion yscale'] \
#= refiner.get_refined_distortion_yscale()
#integration_params['raster'] = refiner.get_raster()
separation = refiner.get_separation()
if separation is not None:
self.set_refiner_parameter('mosflm', 'separation',
'%s %s' % refiner.get_separation())
#integration_params['separation'] = refiner.get_separation()
self.set_refiner_parameter('mosflm', 'beam',
'%s %s' % refiner.get_refined_beam_centre())
self.set_refiner_parameter('mosflm', 'distance',
refiner.get_refined_distance())
self.set_refiner_parameter('mosflm', 'distortion tilt',
refiner.get_refined_distortion_tilt())
self.set_refiner_parameter('mosflm', 'distortion twist',
refiner.get_refined_distortion_twist())
integration_params['beam'] = tuple(
float(b) for b in refiner.get_refined_beam_centre())
integration_params['distance'] = refiner.get_refined_distance()
integration_params[
'distortion tilt'] = refiner.get_refined_distortion_tilt()
integration_params[
'distortion twist'] = refiner.get_refined_distortion_twist()
idxr._indxr_mosaic = refiner.get_refined_mosaic()
idxr.set_indexer_payload(
'mosflm_orientation_matrix',
open(os.path.join(self.get_working_directory(), 'xiarefine.mat'),
'r').readlines())
self.set_refiner_payload(
'mosflm_orientation_matrix',
idxr.get_indexer_payload('mosflm_orientation_matrix'))
self.set_refiner_payload('mosaic', refiner.get_refined_mosaic())
self.set_refiner_payload('beam', integration_params['beam'])
self.set_refiner_payload('distance', integration_params['distance'])
from xia2.Wrappers.Mosflm.AutoindexHelpers import crystal_model_from_mosflm_mat
# make a dxtbx crystal_model object from the mosflm matrix
experiment = idxr.get_indexer_experiment_list()[0]
crystal_model = crystal_model_from_mosflm_mat(
idxr._indxr_payload['mosflm_orientation_matrix'],
unit_cell=refiner.get_refined_unit_cell(),
space_group=experiment.crystal.get_space_group())
experiment.crystal = crystal_model
#self.set_refiner_payload(
#'mosflm_integration_parameters', integration_params)
self._refinr_refined_experiment_list = ExperimentList([experiment])
return rms_values, background_residual
def _index(self):
'''Implement the indexer interface.'''
Citations.cite('mosflm')
indexer = MosflmIndex()
indexer.set_working_directory(self.get_working_directory())
auto_logfiler(indexer)
from xia2.lib.bits import unique_elements
_images = unique_elements(self._indxr_images)
indexer.set_images(_images)
images_str = ', '.join(map(str, _images))
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
if self._indxr_sweep_name:
#if len(self._fp_directory) <= 50:
#dirname = self._fp_directory
#else:
#dirname = '...%s' % self._fp_directory[-46:]
dirname = os.path.dirname(self.get_imageset().get_template())
Journal.block(
'autoindexing', self._indxr_sweep_name, 'mosflm', {
'images': images_str,
'target cell': self._indxr_input_cell,
'target lattice': self._indxr_input_lattice,
'template': self.get_imageset().get_template(),
'directory': dirname
})
#task = 'Autoindex from images:'
#for i in _images:
#task += ' %s' % self.get_image_name(i)
#self.set_task(task)
indexer.set_template(os.path.basename(self.get_template()))
indexer.set_directory(self.get_directory())
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
indexer.set_distance(xsweep.get_distance())
#if self.get_wavelength_prov() == 'user':
#index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
indexer.set_beam_centre(xsweep.get_beam_centre())
if self._indxr_input_cell:
indexer.set_unit_cell(self._indxr_input_cell)
if self._indxr_input_lattice is not None:
spacegroup_number = lattice_to_spacegroup(
self._indxr_input_lattice)
indexer.set_space_group_number(spacegroup_number)
if not self._mosflm_autoindex_thresh:
try:
min_peaks = 200
Debug.write('Aiming for at least %d spots...' % min_peaks)
thresholds = []
for i in _images:
p = Printpeaks()
p.set_working_directory(self.get_working_directory())
auto_logfiler(p)
p.set_image(self.get_image_name(i))
thresh = p.threshold(min_peaks)
Debug.write('Autoindex threshold for image %d: %d' % \
(i, thresh))
thresholds.append(thresh)
thresh = min(thresholds)
self._mosflm_autoindex_thresh = thresh
except Exception as e:
print str(e) #XXX this should disappear!
Debug.write('Error computing threshold: %s' % str(e))
Debug.write('Using default of 20.0')
thresh = 20.0
else:
thresh = self._mosflm_autoindex_thresh
Debug.write('Using autoindex threshold: %d' % thresh)
if self._mosflm_autoindex_sol:
indexer.set_solution_number(self._mosflm_autoindex_sol)
indexer.set_threshold(thresh)
# now forget this to prevent weird things happening later on
if self._mosflm_autoindex_sol:
self._mosflm_autoindex_sol = 0
indexer.run()
indxr_cell = indexer.get_refined_unit_cell()
self._indxr_lattice = indexer.get_lattice()
space_group_number = indexer.get_indexed_space_group_number()
detector_distance = indexer.get_refined_distance()
beam_centre = indexer.get_refined_beam_centre()
mosaic_spreads = indexer.get_mosaic_spreads()
if min(list(indxr_cell)) < 10.0 and \
indxr_cell[2] / indxr_cell[0] > 6:
Debug.write('Unrealistic autoindexing solution: ' +
'%.2f %.2f %.2f %.2f %.2f %.2f' % indxr_cell)
# tweak some parameters and try again...
self._mosflm_autoindex_thresh *= 1.5
self.set_indexer_done(False)
return
intgr_params = {}
# look up other possible indexing solutions (not well - in
# standard settings only!) This is moved earlier as it could
# result in returning if Mosflm has selected the wrong
# solution!
try:
self._indxr_other_lattice_cell = indexer.get_solutions()
# Change 27/FEB/08 to support user assigned spacegroups
if self._indxr_user_input_lattice:
lattice_to_spacegroup_dict = {
'aP': 1,
'mP': 3,
'mC': 5,
'oP': 16,
'oC': 20,
'oF': 22,
'oI': 23,
'tP': 75,
'tI': 79,
'hP': 143,
'hR': 146,
'cP': 195,
'cF': 196,
'cI': 197
}
for k in self._indxr_other_lattice_cell.keys():
if lattice_to_spacegroup_dict[k] > \
lattice_to_spacegroup_dict[
self._indxr_input_lattice]:
del (self._indxr_other_lattice_cell[k])
# check that the selected unit cell matches - and if
# not raise a "horrible" exception
if self._indxr_input_cell:
assert indxr_cell is not None
for j in range(6):
if math.fabs(self._indxr_input_cell[j] -
indxr_cell[j]) > 2.0:
Chatter.write('Mosflm autoindexing did not select ' +
'correct (target) unit cell')
raise RuntimeError(
'something horrible happened in indexing')
except RuntimeError as e:
# check if mosflm rejected a solution we have it
if 'horribl' in str(e):
# ok it did - time to break out the big guns...
if not self._indxr_input_cell:
raise RuntimeError(
'error in solution selection when not preset')
# XXX FIXME
self._mosflm_autoindex_sol = _get_indexing_solution_number(
indexer.get_all_output(), self._indxr_input_cell,
self._indxr_input_lattice)
# set the fact that we are not done...
self.set_indexer_done(False)
# and return - hopefully this will restart everything
return
else:
raise e
if len(mosaic_spreads) == 0:
# then consider setting it do a default value...
# equal to the oscillation width (a good guess)
phi_width = self.get_phi_width()
Chatter.write(
'Mosaic estimation failed, so guessing at %4.2f' % \
phi_width)
# only consider this if we have thus far no idea on the
# mosaic spread...
mosaic_spreads.append(phi_width)
intgr_params['raster'] = indexer.get_raster()
intgr_params['separation'] = indexer.get_separation()
self._indxr_resolution_estimate = indexer.get_resolution_estimate()
# compute mosaic as mean(mosaic_spreads)
self._indxr_mosaic = sum(mosaic_spreads) / len(mosaic_spreads)
self._indxr_payload['mosflm_integration_parameters'] = intgr_params
self._indxr_payload['mosflm_orientation_matrix'] = open(
os.path.join(self.get_working_directory(), 'xiaindex.mat'),
'r').readlines()
import copy
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
from xia2.Wrappers.Mosflm.AutoindexHelpers import set_distance
from xia2.Wrappers.Mosflm.AutoindexHelpers import crystal_model_from_mosflm_mat
from cctbx import sgtbx, uctbx
# update the beam centre (i.e. shift the origin of the detector)
detector = copy.deepcopy(self.get_detector())
beam = copy.deepcopy(self.get_beam())
set_mosflm_beam_centre(detector, beam, beam_centre)
if detector_distance is not None:
set_distance(detector, detector_distance)
# make a dxtbx crystal_model object from the mosflm matrix
space_group = sgtbx.space_group_info(number=space_group_number).group()
crystal_model = crystal_model_from_mosflm_mat(
self._indxr_payload['mosflm_orientation_matrix'],
unit_cell=uctbx.unit_cell(tuple(indxr_cell)),
space_group=space_group)
# construct an experiment_list
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(beam=beam,
detector=detector,
goniometer=self.get_goniometer(),
scan=self.get_scan(),
crystal=crystal_model)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
def estimate_resolution_limit(self):
# see also xia2/Modules/Scaler/CommonScaler.py: CommonScaler._estimate_resolution_limit()
params = self._params.resolution
m = EstimateResolution()
auto_logfiler(m)
# use the scaled .refl and .expt file
if self._experiments_filename and self._reflections_filename:
m.set_reflections(self._reflections_filename)
m.set_experiments(self._experiments_filename)
else:
m.set_hklin(self._scaled_unmerged_mtz)
m.set_limit_rmerge(params.rmerge)
m.set_limit_completeness(params.completeness)
m.set_limit_cc_half(params.cc_half)
m.set_cc_half_fit(params.cc_half_fit)
m.set_cc_half_significance_level(params.cc_half_significance_level)
m.set_limit_isigma(params.isigma)
m.set_limit_misigma(params.misigma)
m.set_labels(params.labels)
# if batch_range is not None:
# start, end = batch_range
# m.set_batch_range(start, end)
m.run()
resolution_limits = []
reasoning = []
if params.completeness is not None:
r_comp = m.get_resolution_completeness()
resolution_limits.append(r_comp)
reasoning.append("completeness > %s" % params.completeness)
if params.cc_half is not None:
r_cc_half = m.get_resolution_cc_half()
resolution_limits.append(r_cc_half)
reasoning.append("cc_half > %s" % params.cc_half)
if params.rmerge is not None:
r_rm = m.get_resolution_rmerge()
resolution_limits.append(r_rm)
reasoning.append("rmerge > %s" % params.rmerge)
if params.isigma is not None:
r_uis = m.get_resolution_isigma()
resolution_limits.append(r_uis)
reasoning.append("unmerged <I/sigI> > %s" % params.isigma)
if params.misigma is not None:
r_mis = m.get_resolution_misigma()
resolution_limits.append(r_mis)
reasoning.append("merged <I/sigI> > %s" % params.misigma)
if len(resolution_limits):
resolution = max(resolution_limits)
reasoning = [
reason for limit, reason in zip(resolution_limits, reasoning)
if limit >= resolution
]
reasoning = ", ".join(reasoning)
else:
resolution = 0.0
reasoning = None
return resolution, reasoning
