def _integrate_finish(self):
'''Finish the integration - if necessary performing reindexing
based on the pointgroup and the reindexing operator.'''
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()):
return self._mosflm_hklout
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == 0:
return self._mosflm_hklout
Debug.write('Reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
hklin = self._mosflm_hklout
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
reindex.set_operator(self._intgr_reindex_operator)
if self._intgr_spacegroup_number:
reindex.set_spacegroup(self._intgr_spacegroup_number)
hklout = '%s_reindex.mtz' % hklin[:-4]
reindex.set_hklin(hklin)
reindex.set_hklout(hklout)
reindex.reindex()
return hklout
def _integrate_finish(self):
'''Finish the integration - if necessary performing reindexing
based on the pointgroup and the reindexing operator.'''
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()):
return self._mosflm_hklout
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == 0:
return self._mosflm_hklout
Debug.write('Reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
hklin = self._mosflm_hklout
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
reindex.set_operator(self._intgr_reindex_operator)
if self._intgr_spacegroup_number:
reindex.set_spacegroup(self._intgr_spacegroup_number)
hklout = '%s_reindex.mtz' % hklin[:-4]
reindex.set_hklin(hklin)
reindex.set_hklout(hklout)
reindex.reindex()
return hklout
def _index(self):
'''Implement the indexer interface.'''
Citations.cite('mosflm')
indexer = MosflmIndex()
indexer.set_working_directory(self.get_working_directory())
auto_logfiler(indexer)
from xia2.lib.bits import unique_elements
_images = unique_elements(self._indxr_images)
indexer.set_images(_images)
images_str = ', '.join(map(str, _images))
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
if self._indxr_sweep_name:
#if len(self._fp_directory) <= 50:
#dirname = self._fp_directory
#else:
#dirname = '...%s' % self._fp_directory[-46:]
dirname = os.path.dirname(self.get_imageset().get_template())
Journal.block(
'autoindexing', self._indxr_sweep_name, 'mosflm', {
'images': images_str,
'target cell': self._indxr_input_cell,
'target lattice': self._indxr_input_lattice,
'template': self.get_imageset().get_template(),
'directory': dirname
})
#task = 'Autoindex from images:'
#for i in _images:
#task += ' %s' % self.get_image_name(i)
#self.set_task(task)
indexer.set_template(os.path.basename(self.get_template()))
indexer.set_directory(self.get_directory())
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
indexer.set_distance(xsweep.get_distance())
#if self.get_wavelength_prov() == 'user':
#index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
indexer.set_beam_centre(xsweep.get_beam_centre())
if self._indxr_input_cell:
indexer.set_unit_cell(self._indxr_input_cell)
if self._indxr_input_lattice is not None:
spacegroup_number = lattice_to_spacegroup(
self._indxr_input_lattice)
indexer.set_space_group_number(spacegroup_number)
if not self._mosflm_autoindex_thresh:
try:
min_peaks = 200
Debug.write('Aiming for at least %d spots...' % min_peaks)
thresholds = []
for i in _images:
p = Printpeaks()
p.set_working_directory(self.get_working_directory())
auto_logfiler(p)
p.set_image(self.get_image_name(i))
thresh = p.threshold(min_peaks)
Debug.write('Autoindex threshold for image %d: %d' % \
(i, thresh))
thresholds.append(thresh)
thresh = min(thresholds)
self._mosflm_autoindex_thresh = thresh
except Exception as e:
print str(e) #XXX this should disappear!
Debug.write('Error computing threshold: %s' % str(e))
Debug.write('Using default of 20.0')
thresh = 20.0
else:
thresh = self._mosflm_autoindex_thresh
Debug.write('Using autoindex threshold: %d' % thresh)
if self._mosflm_autoindex_sol:
indexer.set_solution_number(self._mosflm_autoindex_sol)
indexer.set_threshold(thresh)
# now forget this to prevent weird things happening later on
if self._mosflm_autoindex_sol:
self._mosflm_autoindex_sol = 0
indexer.run()
indxr_cell = indexer.get_refined_unit_cell()
self._indxr_lattice = indexer.get_lattice()
space_group_number = indexer.get_indexed_space_group_number()
detector_distance = indexer.get_refined_distance()
beam_centre = indexer.get_refined_beam_centre()
mosaic_spreads = indexer.get_mosaic_spreads()
if min(list(indxr_cell)) < 10.0 and \
indxr_cell[2] / indxr_cell[0] > 6:
Debug.write('Unrealistic autoindexing solution: ' +
'%.2f %.2f %.2f %.2f %.2f %.2f' % indxr_cell)
# tweak some parameters and try again...
self._mosflm_autoindex_thresh *= 1.5
self.set_indexer_done(False)
return
intgr_params = {}
# look up other possible indexing solutions (not well - in
# standard settings only!) This is moved earlier as it could
# result in returning if Mosflm has selected the wrong
# solution!
try:
self._indxr_other_lattice_cell = indexer.get_solutions()
# Change 27/FEB/08 to support user assigned spacegroups
if self._indxr_user_input_lattice:
lattice_to_spacegroup_dict = {
'aP': 1,
'mP': 3,
'mC': 5,
'oP': 16,
'oC': 20,
'oF': 22,
'oI': 23,
'tP': 75,
'tI': 79,
'hP': 143,
'hR': 146,
'cP': 195,
'cF': 196,
'cI': 197
}
for k in self._indxr_other_lattice_cell.keys():
if lattice_to_spacegroup_dict[k] > \
lattice_to_spacegroup_dict[
self._indxr_input_lattice]:
del (self._indxr_other_lattice_cell[k])
# check that the selected unit cell matches - and if
# not raise a "horrible" exception
if self._indxr_input_cell:
assert indxr_cell is not None
for j in range(6):
if math.fabs(self._indxr_input_cell[j] -
indxr_cell[j]) > 2.0:
Chatter.write('Mosflm autoindexing did not select ' +
'correct (target) unit cell')
raise RuntimeError(
'something horrible happened in indexing')
except RuntimeError as e:
# check if mosflm rejected a solution we have it
if 'horribl' in str(e):
# ok it did - time to break out the big guns...
if not self._indxr_input_cell:
raise RuntimeError(
'error in solution selection when not preset')
# XXX FIXME
self._mosflm_autoindex_sol = _get_indexing_solution_number(
indexer.get_all_output(), self._indxr_input_cell,
self._indxr_input_lattice)
# set the fact that we are not done...
self.set_indexer_done(False)
# and return - hopefully this will restart everything
return
else:
raise e
if len(mosaic_spreads) == 0:
# then consider setting it do a default value...
# equal to the oscillation width (a good guess)
phi_width = self.get_phi_width()
Chatter.write(
'Mosaic estimation failed, so guessing at %4.2f' % \
phi_width)
# only consider this if we have thus far no idea on the
# mosaic spread...
mosaic_spreads.append(phi_width)
intgr_params['raster'] = indexer.get_raster()
intgr_params['separation'] = indexer.get_separation()
self._indxr_resolution_estimate = indexer.get_resolution_estimate()
# compute mosaic as mean(mosaic_spreads)
self._indxr_mosaic = sum(mosaic_spreads) / len(mosaic_spreads)
self._indxr_payload['mosflm_integration_parameters'] = intgr_params
self._indxr_payload['mosflm_orientation_matrix'] = open(
os.path.join(self.get_working_directory(), 'xiaindex.mat'),
'r').readlines()
import copy
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
from xia2.Wrappers.Mosflm.AutoindexHelpers import set_distance
from xia2.Wrappers.Mosflm.AutoindexHelpers import crystal_model_from_mosflm_mat
from cctbx import sgtbx, uctbx
# update the beam centre (i.e. shift the origin of the detector)
detector = copy.deepcopy(self.get_detector())
beam = copy.deepcopy(self.get_beam())
set_mosflm_beam_centre(detector, beam, beam_centre)
if detector_distance is not None:
set_distance(detector, detector_distance)
# make a dxtbx crystal_model object from the mosflm matrix
space_group = sgtbx.space_group_info(number=space_group_number).group()
crystal_model = crystal_model_from_mosflm_mat(
self._indxr_payload['mosflm_orientation_matrix'],
unit_cell=uctbx.unit_cell(tuple(indxr_cell)),
space_group=space_group)
# construct an experiment_list
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(beam=beam,
detector=detector,
goniometer=self.get_goniometer(),
scan=self.get_scan(),
crystal=crystal_model)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
def _integrate_finish(self):
'''Finish off the integration by running dials.export.'''
# FIXME - do we want to export every time we call this method
# (the file will not have changed) and also (more important) do
# we want a different exported MTZ file every time (I do not think
# that we do; these can be very large) - was exporter.get_xpid() ->
# now dials
exporter = self.ExportMtz()
exporter.set_reflections_filename(self._intgr_integrated_pickle)
mtz_filename = os.path.join(
self.get_working_directory(), '%s_integrated.mtz' % 'dials')
exporter.set_mtz_filename(mtz_filename)
exporter.run()
self._intgr_integrated_filename = mtz_filename
# record integrated MTZ file for e.g. BLEND.
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(
'%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep), mtz_filename)
if not os.path.isfile(self._intgr_integrated_filename):
raise RuntimeError("dials.export failed: %s does not exist."
% self._intgr_integrated_filename)
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()):
Debug.write('Not reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return mtz_filename
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == 0:
Debug.write('Not reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return mtz_filename
Debug.write('Reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
hklin = mtz_filename
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
reindex.set_operator(self._intgr_reindex_operator)
if self._intgr_spacegroup_number:
reindex.set_spacegroup(self._intgr_spacegroup_number)
else:
reindex.set_spacegroup(lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()))
hklout = '%s_reindex.mtz' % hklin[:-4]
reindex.set_hklin(hklin)
reindex.set_hklout(hklout)
reindex.reindex()
self._intgr_integrated_filename = hklout
self._intgr_cell = reindex.get_cell()
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(
'%s %s %s %s experiments' % (pname, xname, dname, sweep),
self.get_integrated_experiments())
from iotbx.reflection_file_reader import any_reflection_file
miller_arrays = any_reflection_file(hklout).as_miller_arrays()
# look for profile-fitted intensities
intensities = [ma for ma in miller_arrays
if ma.info().labels == ['IPR', 'SIGIPR']]
if len(intensities) == 0:
# look instead for summation-integrated intensities
intensities = [ma for ma in miller_arrays
if ma.info().labels == ['I', 'SIGI']]
assert len(intensities)
self._intgr_n_ref = intensities[0].size()
return hklout
def _index(self):
'''Actually index the diffraction pattern. Note well that
this is not going to compute the matrix...'''
# acknowledge this program
Citations.cite('labelit')
Citations.cite('distl')
#self.reset()
_images = []
for i in self._indxr_images:
for j in i:
if not j in _images:
_images.append(j)
_images.sort()
images_str = '%d' % _images[0]
for i in _images[1:]:
images_str += ', %d' % i
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
if self._indxr_sweep_name:
# then this is a proper autoindexing run - describe this
# to the journal entry
#if len(self._fp_directory) <= 50:
#dirname = self._fp_directory
#else:
#dirname = '...%s' % self._fp_directory[-46:]
dirname = os.path.dirname(self.get_imageset().get_template())
Journal.block(
'autoindexing', self._indxr_sweep_name, 'labelit', {
'images': images_str,
'target cell': cell_str,
'target lattice': self._indxr_input_lattice,
'template': self.get_imageset().get_template(),
'directory': dirname
})
if len(_images) > 4:
raise RuntimeError('cannot use more than 4 images')
from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex
index = LabelitIndex()
index.set_working_directory(self.get_working_directory())
auto_logfiler(index)
#task = 'Autoindex from images:'
#for i in _images:
#task += ' %s' % self.get_image_name(i)
#self.set_task(task)
Debug.write('Indexing from images:')
for i in _images:
index.add_image(self.get_image_name(i))
Debug.write('%s' % self.get_image_name(i))
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
index.set_distance(xsweep.get_distance())
#if self.get_wavelength_prov() == 'user':
#index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
index.set_beam_centre(xsweep.get_beam_centre())
if self._refine_beam is False:
index.set_refine_beam(False)
else:
index.set_refine_beam(True)
index.set_beam_search_scope(self._beam_search_scope)
if ((math.fabs(self.get_wavelength() - 1.54) < 0.01)
or (math.fabs(self.get_wavelength() - 2.29) < 0.01)):
index.set_Cu_KA_or_Cr_KA(True)
#sweep = self.get_indexer_sweep_name()
#FileHandler.record_log_file(
#'%s INDEX' % (sweep), self.get_log_file())
try:
index.run()
except RuntimeError as e:
if self._refine_beam is False:
raise e
# can we improve the situation?
if self._beam_search_scope < 4.0:
self._beam_search_scope += 4.0
# try repeating the indexing!
self.set_indexer_done(False)
return 'failed'
# otherwise this is beyond redemption
raise e
self._solutions = index.get_solutions()
# FIXME this needs to check the smilie status e.g.
# ":)" or ";(" or " ".
# FIXME need to check the value of the RMSD and raise an
# exception if the P1 solution has an RMSD > 1.0...
# Change 27/FEB/08 to support user assigned spacegroups
# (euugh!) have to "ignore" solutions with higher symmetry
# otherwise the rest of xia will override us. Bummer.
for i, solution in self._solutions.iteritems():
if self._indxr_user_input_lattice:
if (lattice_to_spacegroup(solution['lattice']) >
lattice_to_spacegroup(self._indxr_input_lattice)):
Debug.write('Ignoring solution: %s' % solution['lattice'])
del self._solutions[i]
# check the RMSD from the triclinic unit cell
if self._solutions[1]['rmsd'] > 1.0 and False:
# don't know when this is useful - but I know when it is not!
raise RuntimeError('high RMSD for triclinic solution')
# configure the "right" solution
self._solution = self.get_solution()
# now store also all of the other solutions... keyed by the
# lattice - however these should only be added if they
# have a smiley in the appropriate record, perhaps?
for solution in self._solutions.keys():
lattice = self._solutions[solution]['lattice']
if lattice in self._indxr_other_lattice_cell:
if self._indxr_other_lattice_cell[lattice]['goodness'] < \
self._solutions[solution]['metric']:
continue
self._indxr_other_lattice_cell[lattice] = {
'goodness': self._solutions[solution]['metric'],
'cell': self._solutions[solution]['cell']
}
self._indxr_lattice = self._solution['lattice']
self._indxr_cell = tuple(self._solution['cell'])
self._indxr_mosaic = self._solution['mosaic']
lms = LabelitMosflmScript()
lms.set_working_directory(self.get_working_directory())
lms.set_solution(self._solution['number'])
self._indxr_payload['mosflm_orientation_matrix'] = lms.calculate()
# get the beam centre from the mosflm script - mosflm
# may have inverted the beam centre and labelit will know
# this!
mosflm_beam_centre = lms.get_mosflm_beam()
if mosflm_beam_centre:
self._indxr_payload['mosflm_beam_centre'] = tuple(
mosflm_beam_centre)
import copy
detector = copy.deepcopy(self.get_detector())
beam = copy.deepcopy(self.get_beam())
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
set_mosflm_beam_centre(detector, beam, mosflm_beam_centre)
from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
from scitbx import matrix
from cctbx import sgtbx, uctbx
from dxtbx.model import CrystalFactory
mosflm_matrix = matrix.sqr([
float(i) for line in lms.calculate()
for i in line.replace("-", " -").split()
][:9])
space_group = sgtbx.space_group_info(
lattice_to_spacegroup_number(self._solution['lattice'])).group()
crystal_model = CrystalFactory.from_mosflm_matrix(
mosflm_matrix,
unit_cell=uctbx.unit_cell(tuple(self._solution['cell'])),
space_group=space_group)
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(
beam=beam,
detector=detector,
goniometer=self.get_goniometer(),
scan=self.get_scan(),
crystal=crystal_model,
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# also get an estimate of the resolution limit from the
# labelit.stats_distl output... FIXME the name is wrong!
lsd = LabelitStats_distl()
lsd.set_working_directory(self.get_working_directory())
lsd.stats_distl()
resolution = 1.0e6
for i in _images:
stats = lsd.get_statistics(self.get_image_name(i))
resol = 0.5 * (stats['resol_one'] + stats['resol_two'])
if resol < resolution:
resolution = resol
self._indxr_resolution_estimate = resolution
return 'ok'
def _index(self):
"""Actually index the diffraction pattern. Note well that
this is not going to compute the matrix..."""
# acknowledge this program
if not self._indxr_images:
raise RuntimeError("No good spots found on any images")
Citations.cite("labelit")
Citations.cite("distl")
_images = []
for i in self._indxr_images:
for j in i:
if not j in _images:
_images.append(j)
_images.sort()
images_str = "%d" % _images[0]
for i in _images[1:]:
images_str += ", %d" % i
cell_str = None
if self._indxr_input_cell:
cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % self._indxr_input_cell
if self._indxr_sweep_name:
# then this is a proper autoindexing run - describe this
# to the journal entry
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = "...%s" % self._fp_directory[-46:]
Journal.block(
"autoindexing",
self._indxr_sweep_name,
"labelit",
{
"images": images_str,
"target cell": cell_str,
"target lattice": self._indxr_input_lattice,
"template": self._fp_template,
"directory": dirname,
},
)
# auto_logfiler(self)
from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex
index = LabelitIndex()
index.set_working_directory(self.get_working_directory())
auto_logfiler(index)
# task = 'Autoindex from images:'
# for i in _images:
# task += ' %s' % self.get_image_name(i)
# self.set_task(task)
# self.add_command_line('--index_only')
Debug.write("Indexing from images:")
for i in _images:
index.add_image(self.get_image_name(i))
Debug.write("%s" % self.get_image_name(i))
if self._primitive_unit_cell:
index.set_primitive_unit_cell(self._primitive_unit_cell)
if self._indxr_input_cell:
index.set_max_cell(1.25 * max(self._indxr_input_cell[:3]))
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
index.set_distance(xsweep.get_distance())
# if self.get_wavelength_prov() == 'user':
# index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
index.set_beam_centre(xsweep.get_beam_centre())
if self._refine_beam is False:
index.set_refine_beam(False)
else:
index.set_refine_beam(True)
index.set_beam_search_scope(self._beam_search_scope)
if (math.fabs(self.get_wavelength() - 1.54) <
0.01) or (math.fabs(self.get_wavelength() - 2.29) < 0.01):
index.set_Cu_KA_or_Cr_KA(True)
try:
index.run()
except RuntimeError as e:
if self._refine_beam is False:
raise e
# can we improve the situation?
if self._beam_search_scope < 4.0:
self._beam_search_scope += 4.0
# try repeating the indexing!
self.set_indexer_done(False)
return "failed"
# otherwise this is beyond redemption
raise e
self._solutions = index.get_solutions()
# FIXME this needs to check the smilie status e.g.
# ":)" or ";(" or " ".
# FIXME need to check the value of the RMSD and raise an
# exception if the P1 solution has an RMSD > 1.0...
# Change 27/FEB/08 to support user assigned spacegroups
# (euugh!) have to "ignore" solutions with higher symmetry
# otherwise the rest of xia will override us. Bummer.
for i, solution in self._solutions.iteritems():
if self._indxr_user_input_lattice:
if lattice_to_spacegroup(
solution["lattice"]) > lattice_to_spacegroup(
self._indxr_input_lattice):
Debug.write("Ignoring solution: %s" % solution["lattice"])
del self._solutions[i]
# configure the "right" solution
self._solution = self.get_solution()
# now store also all of the other solutions... keyed by the
# lattice - however these should only be added if they
# have a smiley in the appropriate record, perhaps?
for solution in self._solutions.keys():
lattice = self._solutions[solution]["lattice"]
if lattice in self._indxr_other_lattice_cell:
if (self._indxr_other_lattice_cell[lattice]["goodness"] <
self._solutions[solution]["metric"]):
continue
self._indxr_other_lattice_cell[lattice] = {
"goodness": self._solutions[solution]["metric"],
"cell": self._solutions[solution]["cell"],
}
self._indxr_lattice = self._solution["lattice"]
self._indxr_cell = tuple(self._solution["cell"])
self._indxr_mosaic = self._solution["mosaic"]
lms = LabelitMosflmMatrix()
lms.set_working_directory(self.get_working_directory())
lms.set_solution(self._solution["number"])
self._indxr_payload["mosflm_orientation_matrix"] = lms.calculate()
# get the beam centre from the mosflm script - mosflm
# may have inverted the beam centre and labelit will know
# this!
mosflm_beam_centre = lms.get_mosflm_beam()
if mosflm_beam_centre:
self._indxr_payload["mosflm_beam_centre"] = tuple(
mosflm_beam_centre)
detector = copy.deepcopy(self.get_detector())
beam = copy.deepcopy(self.get_beam())
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
set_mosflm_beam_centre(detector, beam, mosflm_beam_centre)
from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
from scitbx import matrix
from cctbx import sgtbx, uctbx
from dxtbx.model import CrystalFactory
mosflm_matrix = matrix.sqr([
float(i) for line in lms.calculate()
for i in line.replace("-", " -").split()
][:9])
space_group = sgtbx.space_group_info(
lattice_to_spacegroup_number(self._solution["lattice"])).group()
crystal_model = CrystalFactory.from_mosflm_matrix(
mosflm_matrix,
unit_cell=uctbx.unit_cell(tuple(self._solution["cell"])),
space_group=space_group,
)
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(
beam=beam,
detector=detector,
goniometer=self.get_goniometer(),
scan=self.get_scan(),
crystal=crystal_model,
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# also get an estimate of the resolution limit from the
# labelit.stats_distl output... FIXME the name is wrong!
lsd = LabelitStats_distl()
lsd.set_working_directory(self.get_working_directory())
lsd.stats_distl()
resolution = 1.0e6
for i in _images:
stats = lsd.get_statistics(self.get_image_name(i))
resol = 0.5 * (stats["resol_one"] + stats["resol_two"])
if resol < resolution:
resolution = resol
self._indxr_resolution_estimate = resolution
return "ok"
def _mosflm_refine_cell(self, idxr, set_spacegroup=None):
'''Perform the refinement of the unit cell. This will populate
all of the information needed to perform the integration.'''
# FIXME this will die after #1285
#if not self.get_integrater_indexer():
#Debug.write('Replacing indexer of %s with self at %d' % \
#(str(self.get_integrater_indexer()), __line__))
#self.set_integrater_indexer(self)
#idxr = self.get_integrater_indexer()
if not idxr.get_indexer_payload('mosflm_orientation_matrix'):
raise RuntimeError('unexpected situation in indexing')
lattice = idxr.get_indexer_lattice()
mosaic = idxr.get_indexer_mosaic()
cell = idxr.get_indexer_cell()
beam_centre = idxr.get_indexer_beam_centre()
# bug # 3174 - if mosaic is very small (here defined to be
# 0.25 x osc_width) then set to this minimum value.
phi_width = idxr.get_phi_width()
if mosaic < 0.25 * phi_width:
mosaic = 0.25 * phi_width
if idxr.get_indexer_payload('mosflm_beam_centre'):
beam_centre = idxr.get_indexer_payload('mosflm_beam_centre')
distance = idxr.get_indexer_distance()
matrix = idxr.get_indexer_payload('mosflm_orientation_matrix')
integration_params = idxr.get_indexer_payload(
'mosflm_integration_parameters')
if integration_params is None:
integration_params = {}
if integration_params:
if 'separation' in integration_params:
self.set_refiner_parameter(
'mosflm', 'separation',
'%f %f' % tuple(integration_params['separation']))
if 'raster' in integration_params:
self.set_refiner_parameter(
'mosflm', 'raster',
'%d %d %d %d %d' % tuple(integration_params['raster']))
idxr.set_indexer_payload('mosflm_integration_parameters', None)
spacegroup_number = lattice_to_spacegroup(lattice)
# copy these into myself for later reference, if indexer
# is not myself - everything else is copied via the
# cell refinement process...
from cctbx import sgtbx
from dxtbx.model import Crystal
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
experiment = idxr.get_indexer_experiment_list()[0]
set_mosflm_beam_centre(experiment.detector, experiment.beam,
beam_centre)
space_group = sgtbx.space_group_info(number=spacegroup_number).group()
a, b, c = experiment.crystal.get_real_space_vectors()
experiment.crystal = Crystal(a, b, c, space_group=space_group)
# FIXME surely these have been assigned further up?!
if not self._mosflm_cell_ref_images:
self._mosflm_cell_ref_images = self._refine_select_images(mosaic)
f = open(
os.path.join(self.get_working_directory(),
'xiaindex-%s.mat' % lattice), 'w')
for m in matrix:
f.write(m)
f.close()
# then start the cell refinement
refiner = MosflmRefineCell()
refiner.set_working_directory(self.get_working_directory())
auto_logfiler(refiner)
if self._mosflm_gain:
refiner.set_gain(self._mosflm_gain)
refiner.set_template(os.path.basename(idxr.get_template()))
refiner.set_directory(idxr.get_directory())
refiner.set_input_mat_file('xiaindex-%s.mat' % lattice)
refiner.set_output_mat_file('xiarefine.mat')
refiner.set_beam_centre(beam_centre)
refiner.set_unit_cell(cell)
refiner.set_distance(distance)
if set_spacegroup:
refiner.set_space_group_number(set_spacegroup)
else:
refiner.set_space_group_number(spacegroup_number)
# FIXME 18/JUN/08 - it may help to have an overestimate
# of the mosaic spread in here as it *may* refine down
# better than up... - this is not a good idea as it may
# also not refine at all! - 12972 # integration failed
# Bug # 3103
if self._mosflm_cell_ref_double_mosaic:
mosaic *= 2.0
refiner.set_mosaic(mosaic)
# if set, use the resolution for cell refinement - see
# bug # 2078...
if self._mosflm_cell_ref_resolution:
refiner.set_resolution(self._mosflm_cell_ref_resolution)
refiner.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
# note well that the beam centre is coming from indexing so
# should be already properly handled
#if idxr.get_wavelength_prov() == 'user':
#refiner.set_wavelength(idxr.get_wavelength())
# belt + braces mode - only to be used when considering failover,
# will run an additional step of autoindexing prior to cell
# refinement, to be used only after proving that not going it
# will result in cell refinement failure - will use the first
# wedge... N.B. this is only useful if the indexer is Labelit
# not Mosflm...
refiner.set_add_autoindex(self._mosflm_cell_ref_add_autoindex)
# get all of the stored parameter values
parameters = self.get_refiner_parameters('mosflm')
refiner.update_parameters(parameters)
detector = idxr.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
refiner.set_limits(lim_x, lim_y)
refiner.set_images(self._mosflm_cell_ref_images)
failover = PhilIndex.params.xia2.settings.failover
if failover and not self._mosflm_cell_ref_add_autoindex:
refiner.set_ignore_cell_refinement_failure(True)
refiner.run()
# then look to see if the cell refinement worked ok - if it
# didn't then this may indicate that the lattice was wrongly
# selected.
cell_refinement_ok = refiner.cell_refinement_ok()
if not cell_refinement_ok:
Debug.write('Repeating cell refinement...')
self.set_integrater_prepare_done(False)
self._mosflm_cell_ref_add_autoindex = True
return [0.0], [0.0]
rms_values = refiner.get_rms_values()
background_residual = refiner.get_background_residual()
self._refinr_cell = refiner.get_refined_unit_cell()
distance = refiner.get_refined_distance2()
experiment = idxr.get_indexer_experiment_list()[0]
from xia2.Wrappers.Mosflm.AutoindexHelpers import set_distance
set_distance(experiment.detector, distance)
self.set_refiner_parameter('mosflm', 'distortion yscale',
refiner.get_refined_distortion_yscale())
self.set_refiner_parameter('mosflm', 'raster', refiner.get_raster())
#integration_params['distortion yscale'] \
#= refiner.get_refined_distortion_yscale()
#integration_params['raster'] = refiner.get_raster()
separation = refiner.get_separation()
if separation is not None:
self.set_refiner_parameter('mosflm', 'separation',
'%s %s' % refiner.get_separation())
#integration_params['separation'] = refiner.get_separation()
self.set_refiner_parameter('mosflm', 'beam',
'%s %s' % refiner.get_refined_beam_centre())
self.set_refiner_parameter('mosflm', 'distance',
refiner.get_refined_distance())
self.set_refiner_parameter('mosflm', 'distortion tilt',
refiner.get_refined_distortion_tilt())
self.set_refiner_parameter('mosflm', 'distortion twist',
refiner.get_refined_distortion_twist())
integration_params['beam'] = tuple(
float(b) for b in refiner.get_refined_beam_centre())
integration_params['distance'] = refiner.get_refined_distance()
integration_params[
'distortion tilt'] = refiner.get_refined_distortion_tilt()
integration_params[
'distortion twist'] = refiner.get_refined_distortion_twist()
idxr._indxr_mosaic = refiner.get_refined_mosaic()
idxr.set_indexer_payload(
'mosflm_orientation_matrix',
open(os.path.join(self.get_working_directory(), 'xiarefine.mat'),
'r').readlines())
self.set_refiner_payload(
'mosflm_orientation_matrix',
idxr.get_indexer_payload('mosflm_orientation_matrix'))
self.set_refiner_payload('mosaic', refiner.get_refined_mosaic())
self.set_refiner_payload('beam', integration_params['beam'])
self.set_refiner_payload('distance', integration_params['distance'])
from xia2.Wrappers.Mosflm.AutoindexHelpers import crystal_model_from_mosflm_mat
# make a dxtbx crystal_model object from the mosflm matrix
experiment = idxr.get_indexer_experiment_list()[0]
crystal_model = crystal_model_from_mosflm_mat(
idxr._indxr_payload['mosflm_orientation_matrix'],
unit_cell=refiner.get_refined_unit_cell(),
space_group=experiment.crystal.get_space_group())
experiment.crystal = crystal_model
#self.set_refiner_payload(
#'mosflm_integration_parameters', integration_params)
self._refinr_refined_experiment_list = ExperimentList([experiment])
return rms_values, background_residual
def _mosflm_test_refine_cell(self, idxr, test_lattice):
'''Test performing cell refinement in with a different
lattice to the one which was selected by the autoindex
procedure. This should not change anything in the class.'''
#idxr = self.get_integrater_indexer()
lattice = idxr.get_indexer_lattice()
mosaic = idxr.get_indexer_mosaic()
beam_centre = idxr.get_indexer_beam_centre()
distance = idxr.get_indexer_distance()
matrix = idxr.get_indexer_payload('mosflm_orientation_matrix')
phi_width = idxr.get_phi_width()
if mosaic < 0.25 * phi_width:
mosaic = 0.25 * phi_width
input_matrix = ''
for m in matrix:
input_matrix += '%s\n' % m
new_matrix = transmogrify_matrix(lattice, input_matrix, test_lattice,
idxr.get_wavelength(),
self.get_working_directory())
spacegroup_number = lattice_to_spacegroup(test_lattice)
if not self._mosflm_cell_ref_images:
raise RuntimeError('wedges must be assigned already')
open(
os.path.join(self.get_working_directory(),
'test-xiaindex-%s.mat' % lattice),
'w').write(new_matrix)
refiner = MosflmRefineCell()
refiner.set_working_directory(self.get_working_directory())
auto_logfiler(refiner)
if self._mosflm_gain:
refiner.set_gain(self._mosflm_gain)
refiner.set_template(os.path.basename(idxr.get_template()))
refiner.set_directory(idxr.get_directory())
refiner.set_input_mat_file('test-xiaindex-%s.mat' % lattice)
refiner.set_output_mat_file('test-xiarefine.mat')
refiner.set_beam_centre(beam_centre)
refiner.set_distance(distance)
refiner.set_space_group_number(spacegroup_number)
# FIXME 18/JUN/08 - it may help to have an overestimate
# of the mosaic spread in here as it *may* refine down
# better than up... - this is not a good idea as it may
# also not refine at all! - 12972 # integration failed
# Bug # 3103
if self._mosflm_cell_ref_double_mosaic:
mosaic *= 2.0
refiner.set_mosaic(mosaic)
# if set, use the resolution for cell refinement - see
# bug # 2078...
if self._mosflm_cell_ref_resolution:
refiner.set_resolution(self._mosflm_cell_ref_resolution)
refiner.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
# note well that the beam centre is coming from indexing so
# should be already properly handled
#if idxr.get_wavelength_prov() == 'user':
#refiner.set_wavelength(idxr.get_wavelength())
# belt + braces mode - only to be used when considering failover,
# will run an additional step of autoindexing prior to cell
# refinement, to be used only after proving that not going it
# will result in cell refinement failure - will use the first
# wedge... N.B. this is only useful if the indexer is Labelit
# not Mosflm...
refiner.set_add_autoindex(self._mosflm_cell_ref_add_autoindex)
# XXX
# get all of the stored parameter values
parameters = self.get_refiner_parameters('mosflm')
refiner.update_parameters(parameters)
detector = idxr.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
refiner.set_limits(lim_x, lim_y)
refiner.set_images(self._mosflm_cell_ref_images)
refiner.run()
rms_values = refiner.get_rms_values()
background_residual = refiner.get_background_residual()
return rms_values, background_residual
def _mosflm_parallel_integrate(self):
'''Perform the integration as before, but this time as a
number of parallel Mosflm jobs (hence, in separate directories)
and including a step of pre-refinement of the mosaic spread and
missets. This will all be kind of explicit and hence probably
messy!'''
refinr = self.get_integrater_refiner()
lattice = refinr.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup(lattice)
mosaic = refinr.get_refiner_payload('mosaic')
beam = refinr.get_refiner_payload('beam')
distance = refinr.get_refiner_payload('distance')
matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')
integration_params = refinr.get_refiner_payload(
'mosflm_integration_parameters')
if integration_params:
if 'separation' in integration_params:
self.set_integrater_parameter(
'mosflm', 'separation',
'%s %s' % tuple(integration_params['separation']))
if 'raster' in integration_params:
self.set_integrater_parameter(
'mosflm', 'raster',
'%d %d %d %d %d' % tuple(integration_params['raster']))
refinr.set_refiner_payload('mosflm_integration_parameters', None)
pname, xname, dname = self.get_integrater_project_info()
# what follows below should (i) be run in separate directories
# and (ii) be repeated N=parallel times.
nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
parallel = nproc
# FIXME this is something of a kludge - if too few frames refinement
# and integration does not work well... ideally want at least 15
# frames / chunk (say)
nframes = self._intgr_wedge[1] - self._intgr_wedge[0] + 1
if parallel > nframes / 15:
parallel = nframes // 15
if not parallel:
raise RuntimeError('parallel not set')
if parallel < 2:
raise RuntimeError('parallel not parallel: %s' % parallel)
jobs = []
hklouts = []
nref = 0
# calculate the chunks to use
offset = self.get_frame_offset()
start = self._intgr_wedge[0] - offset
end = self._intgr_wedge[1] - offset
left_images = 1 + end - start
left_chunks = parallel
chunks = []
while left_images > 0:
size = left_images // left_chunks
chunks.append((start, start + size - 1))
start += size
left_images -= size
left_chunks -= 1
summary_files = []
for j in range(parallel):
# make some working directories, as necessary - chunk-(0:N-1)
wd = os.path.join(self.get_working_directory(), 'chunk-%d' % j)
if not os.path.exists(wd):
os.makedirs(wd)
job = MosflmIntegrate()
job.set_working_directory(wd)
auto_logfiler(job)
l = refinr.get_refiner_lattice()
# create the starting point
f = open(os.path.join(wd, 'xiaintegrate-%s.mat' % l), 'w')
for m in matrix:
f.write(m)
f.close()
spacegroup_number = lattice_to_spacegroup(lattice)
job.set_refine_profiles(self._mosflm_refine_profiles)
# N.B. for harvesting need to append N to dname.
if pname is not None and xname is not None and dname is not None:
Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname))
harvest_dir = self.get_working_directory()
temp_dname = '%s_%s' % \
(dname, self.get_integrater_sweep_name())
job.set_pname_xname_dname(pname, xname, temp_dname)
job.set_template(os.path.basename(self.get_template()))
job.set_directory(self.get_directory())
# check for ice - and if so, exclude (ranges taken from
# XDS documentation)
if self.get_integrater_ice() != 0:
Debug.write('Excluding ice rings')
job.set_exclude_ice(True)
# exclude specified resolution ranges
if len(self.get_integrater_excluded_regions()) != 0:
regions = self.get_integrater_excluded_regions()
Debug.write('Excluding regions: %s' % repr(regions))
job.set_exclude_regions(regions)
mask = standard_mask(self.get_detector())
for m in mask:
job.add_instruction(m)
job.set_input_mat_file('xiaintegrate-%s.mat' % l)
job.set_beam_centre(beam)
job.set_distance(distance)
job.set_space_group_number(spacegroup_number)
job.set_mosaic(mosaic)
if self.get_wavelength_prov() == 'user':
job.set_wavelength(self.get_wavelength())
parameters = self.get_integrater_parameters('mosflm')
job.update_parameters(parameters)
if self._mosflm_gain:
job.set_gain(self._mosflm_gain)
# check for resolution limits
if self._intgr_reso_high > 0.0:
job.set_d_min(self._intgr_reso_high)
if self._intgr_reso_low:
job.set_d_max(self._intgr_reso_low)
if PhilIndex.params.general.backstop_mask:
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask = mask.calculate_mask_mosflm(self.get_header())
job.set_mask(mask)
detector = self.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
job.set_limits(lim_x, lim_y)
job.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
job.set_pre_refinement(True)
job.set_image_range(chunks[j])
# these are now running so ...
jobs.append(job)
continue
# ok, at this stage I need to ...
#
# (i) accumulate the statistics as a function of batch
# (ii) mong them into a single block
#
# This is likely to be a pain in the arse!
first_integrated_batch = 1.0e6
last_integrated_batch = -1.0e6
all_residuals = []
threads = []
for j in range(parallel):
job = jobs[j]
# now wait for them to finish - first wait will really be the
# first one, then all should be finished...
thread = Background(job, 'run')
thread.start()
threads.append(thread)
mosaics = []
postref_result = {}
integrated_images_first = 1.0e6
integrated_images_last = -1.0e6
self._intgr_per_image_statistics = {}
for j in range(parallel):
thread = threads[j]
thread.stop()
job = jobs[j]
# get the log file
output = job.get_all_output()
# record a copy of it, perhaps - though not if parallel
if self.get_integrater_sweep_name() and False:
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file(
'%s %s %s %s mosflm integrate' % \
(self.get_integrater_sweep_name(),
pname, xname, '%s_%d' % (dname, j)),
job.get_log_file())
# look for things that we want to know...
# that is, the output reflection file name, the updated
# value for the gain (if present,) any warnings, errors,
# or just interesting facts.
batches = job.get_batches_out()
integrated_images_first = min(batches[0], integrated_images_first)
integrated_images_last = max(batches[1], integrated_images_last)
mosaics.extend(job.get_mosaic_spreads())
if min(mosaics) < 0:
raise IntegrationError('negative mosaic spread: %s' %
min(mosaic))
if (job.get_detector_gain_error()
and not (self.get_imageset().get_detector()[0].get_type()
== 'SENSOR_PAD')):
gain = job.get_suggested_gain()
if gain is not None:
self.set_integrater_parameter('mosflm', 'gain', gain)
self.set_integrater_export_parameter(
'mosflm', 'gain', gain)
if self._mosflm_gain:
Debug.write('GAIN updated to %f' % gain)
else:
Debug.write('GAIN found to be %f' % gain)
self._mosflm_gain = gain
self._mosflm_rerun_integration = True
hklout = job.get_hklout()
Debug.write('Integration output: %s' % hklout)
hklouts.append(hklout)
nref += job.get_nref()
# if a BGSIG error happened try not refining the
# profile and running again...
if job.get_bgsig_too_large():
if not self._mosflm_refine_profiles:
raise RuntimeError('BGSIG error with profiles fixed')
Debug.write('BGSIG error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
if job.get_getprof_error():
Debug.write('GETPROF error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
# here
# write the report for each image as .*-#$ to Chatter -
# detailed report will be written automagically to science...
self._intgr_per_image_statistics.update(
job.get_per_image_statistics())
postref_result.update(job.get_postref_result())
# inspect the output for e.g. very high weighted residuals
all_residuals.extend(job.get_residuals())
self._intgr_batches_out = (integrated_images_first,
integrated_images_last)
if mosaics and len(mosaics) > 0:
self.set_integrater_mosaic_min_mean_max(
min(mosaics),
sum(mosaics) / len(mosaics), max(mosaics))
else:
m = indxr.get_indexer_mosaic()
self.set_integrater_mosaic_min_mean_max(m, m, m)
Chatter.write(self.show_per_image_statistics())
Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
self.get_integrater_mosaic_min_mean_max())
# gather the statistics from the postrefinement for all sweeps
# now write this to a postrefinement log
postref_log = os.path.join(self.get_working_directory(),
'postrefinement.log')
fout = open(postref_log, 'w')
fout.write('$TABLE: Postrefinement for %s:\n' % \
self._intgr_sweep_name)
fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n')
fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n')
for image in sorted(postref_result):
phix = postref_result[image].get('phix', 0.0)
phiy = postref_result[image].get('phiy', 0.0)
phiz = postref_result[image].get('phiz', 0.0)
fout.write('%d %5.2f %5.2f %5.2f\n' % \
(image, phix, phiy, phiz))
fout.write('$$\n')
fout.close()
if self.get_integrater_sweep_name():
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file('%s %s %s %s postrefinement' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
postref_log)
hklouts.sort()
hklout = os.path.join(self.get_working_directory(),
os.path.split(hklouts[0])[-1])
Debug.write('Sorting data to %s' % hklout)
for hklin in hklouts:
Debug.write('<= %s' % hklin)
sortmtz = Sortmtz()
sortmtz.set_hklout(hklout)
for hklin in hklouts:
sortmtz.add_hklin(hklin)
sortmtz.sort()
self._mosflm_hklout = hklout
return self._mosflm_hklout
def _mosflm_integrate(self):
'''Perform the actual integration, based on the results of the
cell refinement or indexing (they have the equivalent form.)'''
refinr = self.get_integrater_refiner()
if not refinr.get_refiner_payload('mosflm_orientation_matrix'):
raise RuntimeError('unexpected situation in indexing')
lattice = refinr.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup(lattice)
mosaic = refinr.get_refiner_payload('mosaic')
beam = refinr.get_refiner_payload('beam')
distance = refinr.get_refiner_payload('distance')
matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')
integration_params = refinr.get_refiner_payload(
'mosflm_integration_parameters')
if integration_params:
if 'separation' in integration_params:
self.set_integrater_parameter(
'mosflm', 'separation',
'%s %s' % tuple(integration_params['separation']))
if 'raster' in integration_params:
self.set_integrater_parameter(
'mosflm', 'raster',
'%d %d %d %d %d' % tuple(integration_params['raster']))
refinr.set_refiner_payload('mosflm_integration_parameters', None)
f = open(
os.path.join(self.get_working_directory(), 'xiaintegrate.mat'),
'w')
for m in matrix:
f.write(m)
f.close()
# then start the integration
integrater = MosflmIntegrate()
integrater.set_working_directory(self.get_working_directory())
auto_logfiler(integrater)
integrater.set_refine_profiles(self._mosflm_refine_profiles)
pname, xname, dname = self.get_integrater_project_info()
if pname is not None and xname is not None and dname is not None:
Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname))
harvest_dir = self.get_working_directory()
# harvest file name will be %s.mosflm_run_start_end % dname
temp_dname = '%s_%s' % \
(dname, self.get_integrater_sweep_name())
integrater.set_pname_xname_dname(pname, xname, temp_dname)
integrater.set_template(os.path.basename(self.get_template()))
integrater.set_directory(self.get_directory())
# check for ice - and if so, exclude (ranges taken from
# XDS documentation)
if self.get_integrater_ice() != 0:
Debug.write('Excluding ice rings')
integrater.set_exclude_ice(True)
# exclude specified resolution ranges
if len(self.get_integrater_excluded_regions()) != 0:
regions = self.get_integrater_excluded_regions()
Debug.write('Excluding regions: %s' % repr(regions))
integrater.set_exclude_regions(regions)
mask = standard_mask(self.get_detector())
for m in mask:
integrater.add_instruction(m)
integrater.set_input_mat_file('xiaintegrate.mat')
integrater.set_beam_centre(beam)
integrater.set_distance(distance)
integrater.set_space_group_number(spacegroup_number)
integrater.set_mosaic(mosaic)
if self.get_wavelength_prov() == 'user':
integrater.set_wavelength(self.get_wavelength())
parameters = self.get_integrater_parameters('mosflm')
integrater.update_parameters(parameters)
if self._mosflm_gain:
integrater.set_gain(self._mosflm_gain)
# check for resolution limits
if self._intgr_reso_high > 0.0:
integrater.set_d_min(self._intgr_reso_high)
if self._intgr_reso_low:
integrater.set_d_max(self._intgr_reso_low)
if PhilIndex.params.general.backstop_mask:
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask = mask.calculate_mask_mosflm(self.get_header())
integrater.set_mask(mask)
detector = self.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
integrater.set_limits(lim_x, lim_y)
integrater.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
offset = self.get_frame_offset()
integrater.set_image_range(
(self._intgr_wedge[0] - offset, self._intgr_wedge[1] - offset))
try:
integrater.run()
except RuntimeError as e:
if 'integration failed: reason unknown' in str(e):
Chatter.write('Mosflm has failed in integration')
message = 'The input was:\n\n'
for input in integrater.get_all_input():
message += ' %s' % input
Chatter.write(message)
raise
FileHandler.record_log_file(
'%s %s %s %s mosflm integrate' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
integrater.get_log_file())
self._intgr_per_image_statistics = integrater.get_per_image_statistics(
)
self._mosflm_hklout = integrater.get_hklout()
Debug.write('Integration output: %s' % self._mosflm_hklout)
self._intgr_n_ref = integrater.get_nref()
# if a BGSIG error happened try not refining the
# profile and running again...
if integrater.get_bgsig_too_large():
if not self._mosflm_refine_profiles:
raise RuntimeError('BGSIG error with profiles fixed')
Debug.write('BGSIG error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
if integrater.get_getprof_error():
Debug.write('GETPROF error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
if (integrater.get_detector_gain_error()
and not (self.get_imageset().get_detector()[0].get_type()
== 'SENSOR_PAD')):
gain = integrater.get_suggested_gain()
if gain is not None:
self.set_integrater_parameter('mosflm', 'gain', gain)
self.set_integrater_export_parameter('mosflm', 'gain', gain)
if self._mosflm_gain:
Debug.write('GAIN updated to %f' % gain)
else:
Debug.write('GAIN found to be %f' % gain)
self._mosflm_gain = gain
self._mosflm_rerun_integration = True
if not self._mosflm_hklout:
raise RuntimeError('processing abandoned')
self._intgr_batches_out = integrater.get_batches_out()
mosaics = integrater.get_mosaic_spreads()
if mosaics and len(mosaics) > 0:
self.set_integrater_mosaic_min_mean_max(
min(mosaics),
sum(mosaics) / len(mosaics), max(mosaics))
else:
m = indxr.get_indexer_mosaic()
self.set_integrater_mosaic_min_mean_max(m, m, m)
# write the report for each image as .*-#$ to Chatter -
# detailed report will be written automagically to science...
Chatter.write(self.show_per_image_statistics())
Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
self.get_integrater_mosaic_min_mean_max())
# gather the statistics from the postrefinement
postref_result = integrater.get_postref_result()
# now write this to a postrefinement log
postref_log = os.path.join(self.get_working_directory(),
'postrefinement.log')
fout = open(postref_log, 'w')
fout.write('$TABLE: Postrefinement for %s:\n' % \
self._intgr_sweep_name)
fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n')
fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n')
for image in sorted(postref_result):
phix = postref_result[image].get('phix', 0.0)
phiy = postref_result[image].get('phiy', 0.0)
phiz = postref_result[image].get('phiz', 0.0)
fout.write('%d %5.2f %5.2f %5.2f\n' % \
(image, phix, phiy, phiz))
fout.write('$$\n')
fout.close()
if self.get_integrater_sweep_name():
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file('%s %s %s %s postrefinement' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
postref_log)
return self._mosflm_hklout
def _mosflm_parallel_integrate(self):
'''Perform the integration as before, but this time as a
number of parallel Mosflm jobs (hence, in separate directories)
and including a step of pre-refinement of the mosaic spread and
missets. This will all be kind of explicit and hence probably
messy!'''
refinr = self.get_integrater_refiner()
lattice = refinr.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup(lattice)
mosaic = refinr.get_refiner_payload('mosaic')
beam = refinr.get_refiner_payload('beam')
distance = refinr.get_refiner_payload('distance')
matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')
integration_params = refinr.get_refiner_payload(
'mosflm_integration_parameters')
if integration_params:
if 'separation' in integration_params:
self.set_integrater_parameter(
'mosflm', 'separation',
'%s %s' % tuple(integration_params['separation']))
if 'raster' in integration_params:
self.set_integrater_parameter(
'mosflm', 'raster',
'%d %d %d %d %d' % tuple(integration_params['raster']))
refinr.set_refiner_payload('mosflm_integration_parameters', None)
pname, xname, dname = self.get_integrater_project_info()
# what follows below should (i) be run in separate directories
# and (ii) be repeated N=parallel times.
nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
parallel = nproc
# FIXME this is something of a kludge - if too few frames refinement
# and integration does not work well... ideally want at least 15
# frames / chunk (say)
nframes = self._intgr_wedge[1] - self._intgr_wedge[0] + 1
if parallel > nframes / 15:
parallel = nframes // 15
if not parallel:
raise RuntimeError, 'parallel not set'
if parallel < 2:
raise RuntimeError, 'parallel not parallel: %s' % parallel
jobs = []
hklouts = []
nref = 0
# calculate the chunks to use
offset = self.get_frame_offset()
start = self._intgr_wedge[0] - offset
end = self._intgr_wedge[1] - offset
left_images = 1 + end - start
left_chunks = parallel
chunks = []
while left_images > 0:
size = left_images // left_chunks
chunks.append((start, start + size - 1))
start += size
left_images -= size
left_chunks -= 1
summary_files = []
for j in range(parallel):
# make some working directories, as necessary - chunk-(0:N-1)
wd = os.path.join(self.get_working_directory(),
'chunk-%d' % j)
if not os.path.exists(wd):
os.makedirs(wd)
job = MosflmIntegrate()
job.set_working_directory(wd)
auto_logfiler(job)
l = refinr.get_refiner_lattice()
# create the starting point
f = open(os.path.join(wd, 'xiaintegrate-%s.mat' % l), 'w')
for m in matrix:
f.write(m)
f.close()
spacegroup_number = lattice_to_spacegroup(lattice)
job.set_refine_profiles(self._mosflm_refine_profiles)
# N.B. for harvesting need to append N to dname.
if pname is not None and xname is not None and dname is not None:
Debug.write('Harvesting: %s/%s/%s' %
(pname, xname, dname))
harvest_dir = self.get_working_directory()
temp_dname = '%s_%s' % \
(dname, self.get_integrater_sweep_name())
job.set_pname_xname_dname(pname, xname, temp_dname)
job.set_template(os.path.basename(self.get_template()))
job.set_directory(self.get_directory())
# check for ice - and if so, exclude (ranges taken from
# XDS documentation)
if self.get_integrater_ice() != 0:
Debug.write('Excluding ice rings')
job.set_exclude_ice(True)
# exclude specified resolution ranges
if len(self.get_integrater_excluded_regions()) != 0:
regions = self.get_integrater_excluded_regions()
Debug.write('Excluding regions: %s' % `regions`)
job.set_exclude_regions(regions)
mask = standard_mask(self.get_detector())
for m in mask:
job.add_instruction(m)
job.set_input_mat_file('xiaintegrate-%s.mat' % l)
job.set_beam_centre(beam)
job.set_distance(distance)
job.set_space_group_number(spacegroup_number)
job.set_mosaic(mosaic)
if self.get_wavelength_prov() == 'user':
job.set_wavelength(self.get_wavelength())
parameters = self.get_integrater_parameters('mosflm')
job.update_parameters(parameters)
if self._mosflm_gain:
job.set_gain(self._mosflm_gain)
# check for resolution limits
if self._intgr_reso_high > 0.0:
job.set_d_min(self._intgr_reso_high)
if self._intgr_reso_low:
job.set_d_max(self._intgr_reso_low)
if PhilIndex.params.general.backstop_mask:
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask = mask.calculate_mask_mosflm(self.get_header())
job.set_mask(mask)
detector = self.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
job.set_limits(lim_x, lim_y)
job.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
job.set_pre_refinement(True)
job.set_image_range(chunks[j])
# these are now running so ...
jobs.append(job)
continue
# ok, at this stage I need to ...
#
# (i) accumulate the statistics as a function of batch
# (ii) mong them into a single block
#
# This is likely to be a pain in the arse!
first_integrated_batch = 1.0e6
last_integrated_batch = -1.0e6
all_residuals = []
threads = []
for j in range(parallel):
job = jobs[j]
# now wait for them to finish - first wait will really be the
# first one, then all should be finished...
thread = Background(job, 'run')
thread.start()
threads.append(thread)
mosaics = []
postref_result = { }
integrated_images_first = 1.0e6
integrated_images_last = -1.0e6
self._intgr_per_image_statistics = {}
for j in range(parallel):
thread = threads[j]
thread.stop()
job = jobs[j]
# get the log file
output = job.get_all_output()
# record a copy of it, perhaps - though not if parallel
if self.get_integrater_sweep_name() and False:
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file(
'%s %s %s %s mosflm integrate' % \
(self.get_integrater_sweep_name(),
pname, xname, '%s_%d' % (dname, j)),
job.get_log_file())
# look for things that we want to know...
# that is, the output reflection file name, the updated
# value for the gain (if present,) any warnings, errors,
# or just interesting facts.
batches = job.get_batches_out()
integrated_images_first = min(batches[0], integrated_images_first)
integrated_images_last = max(batches[1], integrated_images_last)
mosaics.extend(job.get_mosaic_spreads())
if min(mosaics) < 0:
raise IntegrationError, 'negative mosaic spread: %s' % min(mosaic)
if (job.get_detector_gain_error() and not
(self.get_imageset().get_detector()[0].get_type() == 'SENSOR_PAD')):
gain = job.get_suggested_gain()
if gain is not None:
self.set_integrater_parameter('mosflm', 'gain', gain)
self.set_integrater_export_parameter('mosflm', 'gain', gain)
if self._mosflm_gain:
Debug.write('GAIN updated to %f' % gain)
else:
Debug.write('GAIN found to be %f' % gain)
self._mosflm_gain = gain
self._mosflm_rerun_integration = True
hklout = job.get_hklout()
Debug.write('Integration output: %s' % hklout)
hklouts.append(hklout)
nref += job.get_nref()
# if a BGSIG error happened try not refining the
# profile and running again...
if job.get_bgsig_too_large():
if not self._mosflm_refine_profiles:
raise RuntimeError, 'BGSIG error with profiles fixed'
Debug.write(
'BGSIG error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
if job.get_getprof_error():
Debug.write(
'GETPROF error detected - try fixing profile...')
self._mosflm_refine_profiles = False
self.set_integrater_done(False)
return
# here
# write the report for each image as .*-#$ to Chatter -
# detailed report will be written automagically to science...
self._intgr_per_image_statistics.update(job.get_per_image_statistics())
postref_result.update(job.get_postref_result())
# inspect the output for e.g. very high weighted residuals
all_residuals.extend(job.get_residuals())
self._intgr_batches_out = (integrated_images_first,
integrated_images_last)
if mosaics and len(mosaics) > 0:
self.set_integrater_mosaic_min_mean_max(
min(mosaics), sum(mosaics) / len(mosaics), max(mosaics))
else:
m = indxr.get_indexer_mosaic()
self.set_integrater_mosaic_min_mean_max(m, m, m)
Chatter.write(self.show_per_image_statistics())
Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
self.get_integrater_mosaic_min_mean_max())
# gather the statistics from the postrefinement for all sweeps
# now write this to a postrefinement log
postref_log = os.path.join(self.get_working_directory(),
'postrefinement.log')
fout = open(postref_log, 'w')
fout.write('$TABLE: Postrefinement for %s:\n' % \
self._intgr_sweep_name)
fout.write('$GRAPHS: Missetting angles:A:1, 2, 3, 4: $$\n')
fout.write('Batch PhiX PhiY PhiZ $$ Batch PhiX PhiY PhiZ $$\n')
for image in sorted(postref_result):
phix = postref_result[image].get('phix', 0.0)
phiy = postref_result[image].get('phiy', 0.0)
phiz = postref_result[image].get('phiz', 0.0)
fout.write('%d %5.2f %5.2f %5.2f\n' % \
(image, phix, phiy, phiz))
fout.write('$$\n')
fout.close()
if self.get_integrater_sweep_name():
pname, xname, dname = self.get_integrater_project_info()
FileHandler.record_log_file('%s %s %s %s postrefinement' % \
(self.get_integrater_sweep_name(),
pname, xname, dname),
postref_log)
hklouts.sort()
hklout = os.path.join(self.get_working_directory(),
os.path.split(hklouts[0])[-1])
Debug.write('Sorting data to %s' % hklout)
for hklin in hklouts:
Debug.write('<= %s' % hklin)
sortmtz = Sortmtz()
sortmtz.set_hklout(hklout)
for hklin in hklouts:
sortmtz.add_hklin(hklin)
sortmtz.sort()
self._mosflm_hklout = hklout
return self._mosflm_hklout
def _mosflm_integrate(self):
'''Perform the actual integration, based on the results of the
cell refinement or indexing (they have the equivalent form.)'''
refinr = self.get_integrater_refiner()
if not refinr.get_refiner_payload('mosflm_orientation_matrix'):
raise RuntimeError, 'unexpected situation in indexing'
lattice = refinr.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup(lattice)
mosaic = refinr.get_refiner_payload('mosaic')
beam = refinr.get_refiner_payload('beam')
distance = refinr.get_refiner_payload('distance')
matrix = refinr.get_refiner_payload('mosflm_orientation_matrix')
integration_params = refinr.get_refiner_payload(
'mosflm_integration_parameters')
if integration_params:
if 'separation' in integration_params:
self.set_integrater_parameter(
'mosflm', 'separation',
'%s %s' % tuple(integration_params['separation']))
if 'raster' in integration_params:
self.set_integrater_parameter(
'mosflm', 'raster',
'%d %d %d %d %d' % tuple(integration_params['raster']))
refinr.set_refiner_payload('mosflm_integration_parameters', None)
f = open(os.path.join(self.get_working_directory(),
'xiaintegrate.mat'), 'w')
for m in matrix:
f.write(m)
f.close()
# then start the integration
integrater = MosflmIntegrate()
integrater.set_working_directory(self.get_working_directory())
auto_logfiler(integrater)
integrater.set_refine_profiles(self._mosflm_refine_profiles)
pname, xname, dname = self.get_integrater_project_info()
if pname is not None and xname is not None and dname is not None:
Debug.write('Harvesting: %s/%s/%s' % (pname, xname, dname))
harvest_dir = self.get_working_directory()
# harvest file name will be %s.mosflm_run_start_end % dname
temp_dname = '%s_%s' % \
(dname, self.get_integrater_sweep_name())
integrater.set_pname_xname_dname(pname, xname, temp_dname)
integrater.set_template(os.path.basename(self.get_template()))
integrater.set_directory(self.get_directory())
# check for ice - and if so, exclude (ranges taken from
# XDS documentation)
if self.get_integrater_ice() != 0:
Debug.write('Excluding ice rings')
integrater.set_exclude_ice(True)
# exclude specified resolution ranges
if len(self.get_integrater_excluded_regions()) != 0:
regions = self.get_integrater_excluded_regions()
Debug.write('Excluding regions: %s' % `regions`)
integrater.set_exclude_regions(regions)
mask = standard_mask(self.get_detector())
for m in mask:
integrater.add_instruction(m)
integrater.set_input_mat_file('xiaintegrate.mat')
integrater.set_beam_centre(beam)
integrater.set_distance(distance)
integrater.set_space_group_number(spacegroup_number)
integrater.set_mosaic(mosaic)
if self.get_wavelength_prov() == 'user':
integrater.set_wavelength(self.get_wavelength())
parameters = self.get_integrater_parameters('mosflm')
integrater.update_parameters(parameters)
if self._mosflm_gain:
integrater.set_gain(self._mosflm_gain)
# check for resolution limits
if self._intgr_reso_high > 0.0:
integrater.set_d_min(self._intgr_reso_high)
if self._intgr_reso_low:
integrater.set_d_max(self._intgr_reso_low)
if PhilIndex.params.general.backstop_mask:
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.general.backstop_mask)
mask = mask.calculate_mask_mosflm(self.get_header())
integrater.set_mask(mask)
detector = self.get_detector()
detector_width, detector_height = detector[0].get_image_size_mm()
lim_x = 0.5 * detector_width
lim_y = 0.5 * detector_height
Debug.write('Scanner limits: %.1f %.1f' % (lim_x, lim_y))
integrater.set_limits(lim_x, lim_y)
integrater.set_fix_mosaic(self._mosflm_postref_fix_mosaic)
offset = self.get_frame_offset()
integrater.set_image_range(
(self._intgr_wedge[0] - offset, self._intgr_wedge[1] - offset))
try:
integrater.run()
except RuntimeError, e:
if 'integration failed: reason unknown' in str(e):
Chatter.write('Mosflm has failed in integration')
message = 'The input was:\n\n'
for input in integrater.get_all_input():
message += ' %s' % input
Chatter.write(message)
raise
def _index(self):
'''Actually index the diffraction pattern. Note well that
this is not going to compute the matrix...'''
# acknowledge this program
if not self._indxr_images:
raise RuntimeError, 'No good spots found on any images'
Citations.cite('labelit')
Citations.cite('distl')
_images = []
for i in self._indxr_images:
for j in i:
if not j in _images:
_images.append(j)
_images.sort()
images_str = '%d' % _images[0]
for i in _images[1:]:
images_str += ', %d' % i
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
if self._indxr_sweep_name:
# then this is a proper autoindexing run - describe this
# to the journal entry
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = '...%s' % self._fp_directory[-46:]
Journal.block(
'autoindexing', self._indxr_sweep_name, 'labelit',
{'images':images_str,
'target cell':cell_str,
'target lattice':self._indxr_input_lattice,
'template':self._fp_template,
'directory':dirname})
#auto_logfiler(self)
from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex
index = LabelitIndex()
index.set_working_directory(self.get_working_directory())
auto_logfiler(index)
#task = 'Autoindex from images:'
#for i in _images:
#task += ' %s' % self.get_image_name(i)
#self.set_task(task)
#self.add_command_line('--index_only')
Debug.write('Indexing from images:')
for i in _images:
index.add_image(self.get_image_name(i))
Debug.write('%s' % self.get_image_name(i))
if self._indxr_input_lattice and False:
index.set_space_group_number(
lattice_to_spacegroup(self._indxr_input_lattice))
if self._primitive_unit_cell:
index.set_primitive_unit_cell(self._primitive_unit_cell)
if self._indxr_input_cell:
index.set_max_cell(1.25 * max(self._indxr_input_cell[:3]))
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
index.set_distance(xsweep.get_distance())
#if self.get_wavelength_prov() == 'user':
#index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
index.set_beam_centre(xsweep.get_beam_centre())
if self._refine_beam is False:
index.set_refine_beam(False)
else:
index.set_refine_beam(True)
index.set_beam_search_scope(self._beam_search_scope)
if ((math.fabs(self.get_wavelength() - 1.54) < 0.01) or
(math.fabs(self.get_wavelength() - 2.29) < 0.01)):
index.set_Cu_KA_or_Cr_KA(True)
try:
index.run()
except RuntimeError, e:
if self._refine_beam is False:
raise e
# can we improve the situation?
if self._beam_search_scope < 4.0:
self._beam_search_scope += 4.0
# try repeating the indexing!
self.set_indexer_done(False)
return 'failed'
# otherwise this is beyond redemption
raise e
def _integrate_finish(self):
'''Finish off the integration by running dials.export.'''
# FIXME - do we want to export every time we call this method
# (the file will not have changed) and also (more important) do
# we want a different exported MTZ file every time (I do not think
# that we do; these can be very large) - was exporter.get_xpid() ->
# now dials
exporter = self.ExportMtz()
exporter.set_reflections_filename(self._intgr_integrated_pickle)
mtz_filename = os.path.join(self.get_working_directory(),
'%s_integrated.mtz' % 'dials')
exporter.set_mtz_filename(mtz_filename)
exporter.run()
self._intgr_integrated_filename = mtz_filename
# record integrated MTZ file for e.g. BLEND.
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(
'%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep),
mtz_filename)
from iotbx.reflection_file_reader import any_reflection_file
miller_arrays = any_reflection_file(
self._intgr_integrated_filename).as_miller_arrays()
# look for profile-fitted intensities
intensities = [
ma for ma in miller_arrays
if ma.info().labels == ['IPR', 'SIGIPR']
]
if len(intensities) == 0:
# look instead for summation-integrated intensities
intensities = [
ma for ma in miller_arrays
if ma.info().labels == ['I', 'SIGI']
]
assert len(intensities)
self._intgr_n_ref = intensities[0].size()
if not os.path.isfile(self._intgr_integrated_filename):
raise RuntimeError("dials.export failed: %s does not exist." %
self._intgr_integrated_filename)
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()):
Debug.write('Not reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return mtz_filename
if self._intgr_reindex_operator is None and \
self._intgr_spacegroup_number == 0:
Debug.write('Not reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return mtz_filename
Debug.write('Reindexing to spacegroup %d (%s)' % \
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
hklin = mtz_filename
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
reindex.set_operator(self._intgr_reindex_operator)
if self._intgr_spacegroup_number:
reindex.set_spacegroup(self._intgr_spacegroup_number)
else:
reindex.set_spacegroup(
lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()))
hklout = '%s_reindex.mtz' % hklin[:-4]
reindex.set_hklin(hklin)
reindex.set_hklout(hklout)
reindex.reindex()
self._intgr_integrated_filename = hklout
self._intgr_cell = reindex.get_cell()
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(
'%s %s %s %s experiments' % (pname, xname, dname, sweep),
self.get_integrated_experiments())
return hklout
def _index(self):
'''Implement the indexer interface.'''
Citations.cite('mosflm')
indexer = MosflmIndex()
indexer.set_working_directory(self.get_working_directory())
auto_logfiler(indexer)
from xia2.lib.bits import unique_elements
_images = unique_elements(self._indxr_images)
indexer.set_images(_images)
images_str = ', '.join(map(str, _images))
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
if self._indxr_sweep_name:
#if len(self._fp_directory) <= 50:
#dirname = self._fp_directory
#else:
#dirname = '...%s' % self._fp_directory[-46:]
dirname = os.path.dirname(self.get_imageset().get_template())
Journal.block(
'autoindexing', self._indxr_sweep_name, 'mosflm',
{'images':images_str,
'target cell':self._indxr_input_cell,
'target lattice':self._indxr_input_lattice,
'template':self.get_imageset().get_template(),
'directory':dirname})
#task = 'Autoindex from images:'
#for i in _images:
#task += ' %s' % self.get_image_name(i)
#self.set_task(task)
indexer.set_template(os.path.basename(self.get_template()))
indexer.set_directory(self.get_directory())
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
indexer.set_distance(xsweep.get_distance())
#if self.get_wavelength_prov() == 'user':
#index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
indexer.set_beam_centre(xsweep.get_beam_centre())
if self._indxr_input_cell:
indexer.set_unit_cell(self._indxr_input_cell)
if self._indxr_input_lattice != None:
spacegroup_number = lattice_to_spacegroup(
self._indxr_input_lattice)
indexer.set_space_group_number(spacegroup_number)
if not self._mosflm_autoindex_thresh:
try:
min_peaks = 200
Debug.write('Aiming for at least %d spots...' % min_peaks)
thresholds = []
for i in _images:
p = Printpeaks()
p.set_working_directory(self.get_working_directory())
auto_logfiler(p)
p.set_image(self.get_image_name(i))
thresh = p.threshold(min_peaks)
Debug.write('Autoindex threshold for image %d: %d' % \
(i, thresh))
thresholds.append(thresh)
thresh = min(thresholds)
self._mosflm_autoindex_thresh = thresh
except Exception as e:
print str(e) #XXX this should disappear!
Debug.write('Error computing threshold: %s' % str(e))
Debug.write('Using default of 20.0')
thresh = 20.0
else:
thresh = self._mosflm_autoindex_thresh
Debug.write('Using autoindex threshold: %d' % thresh)
if self._mosflm_autoindex_sol:
indexer.set_solution_number(self._mosflm_autoindex_sol)
indexer.set_threshold(thresh)
# now forget this to prevent weird things happening later on
if self._mosflm_autoindex_sol:
self._mosflm_autoindex_sol = 0
indexer.run()
#sweep = self.get_indexer_sweep_name()
#FileHandler.record_log_file(
#'%s INDEX' % (sweep), self.get_log_file())
indxr_cell = indexer.get_refined_unit_cell()
self._indxr_lattice = indexer.get_lattice()
space_group_number = indexer.get_indexed_space_group_number()
detector_distance = indexer.get_refined_distance()
beam_centre = indexer.get_refined_beam_centre()
mosaic_spreads = indexer.get_mosaic_spreads()
if min(list(indxr_cell)) < 10.0 and \
indxr_cell[2] / indxr_cell[0] > 6:
Debug.write(
'Unrealistic autoindexing solution: ' +
'%.2f %.2f %.2f %.2f %.2f %.2f' % indxr_cell)
# tweak some parameters and try again...
self._mosflm_autoindex_thresh *= 1.5
self.set_indexer_done(False)
return
intgr_params = { }
# look up other possible indexing solutions (not well - in
# standard settings only!) This is moved earlier as it could
# result in returning if Mosflm has selected the wrong
# solution!
try:
self._indxr_other_lattice_cell = indexer.get_solutions()
# Change 27/FEB/08 to support user assigned spacegroups
if self._indxr_user_input_lattice:
lattice_to_spacegroup_dict = {
'aP':1, 'mP':3, 'mC':5, 'oP':16, 'oC':20, 'oF':22,
'oI':23, 'tP':75, 'tI':79, 'hP':143, 'hR':146,
'cP':195, 'cF':196, 'cI':197}
for k in self._indxr_other_lattice_cell.keys():
if lattice_to_spacegroup_dict[k] > \
lattice_to_spacegroup_dict[
self._indxr_input_lattice]:
del(self._indxr_other_lattice_cell[k])
# check that the selected unit cell matches - and if
# not raise a "horrible" exception
if self._indxr_input_cell:
assert indxr_cell is not None
for j in range(6):
if math.fabs(self._indxr_input_cell[j] - indxr_cell[j]) > 2.0:
Chatter.write(
'Mosflm autoindexing did not select ' +
'correct (target) unit cell')
raise RuntimeError, \
'something horrible happened in indexing'
except RuntimeError, e:
# check if mosflm rejected a solution we have it
if 'horribl' in str(e):
# ok it did - time to break out the big guns...
if not self._indxr_input_cell:
raise RuntimeError, \
'error in solution selection when not preset'
# XXX FIXME
self._mosflm_autoindex_sol = _get_indexing_solution_number(
indexer.get_all_output(),
self._indxr_input_cell,
self._indxr_input_lattice)
# set the fact that we are not done...
self.set_indexer_done(False)
# and return - hopefully this will restart everything
return
else:
raise e
def _integrate_finish(self):
"""Finish off the integration by running dials.export."""
# FIXME - do we want to export every time we call this method
# (the file will not have changed) and also (more important) do
# we want a different exported MTZ file every time (I do not think
# that we do; these can be very large) - was exporter.get_xpid() ->
# now dials
if self._output_format == "hkl":
exporter = self.ExportMtz()
exporter.set_reflections_filename(
self._intgr_integrated_reflections)
mtz_filename = os.path.join(self.get_working_directory(),
"%s_integrated.mtz" % "dials")
exporter.set_mtz_filename(mtz_filename)
exporter.run()
self._intgr_integrated_filename = mtz_filename
# record integrated MTZ file for e.g. BLEND.
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(
"%s %s %s %s INTEGRATE" % (pname, xname, dname, sweep),
mtz_filename)
from iotbx.reflection_file_reader import any_reflection_file
miller_arrays = any_reflection_file(
self._intgr_integrated_filename).as_miller_arrays()
# look for profile-fitted intensities
intensities = [
ma for ma in miller_arrays
if ma.info().labels == ["IPR", "SIGIPR"]
]
if len(intensities) == 0:
# look instead for summation-integrated intensities
intensities = [
ma for ma in miller_arrays
if ma.info().labels == ["I", "SIGI"]
]
assert len(intensities)
self._intgr_n_ref = intensities[0].size()
if not os.path.isfile(self._intgr_integrated_filename):
raise RuntimeError("dials.export failed: %s does not exist." %
self._intgr_integrated_filename)
if (self._intgr_reindex_operator is None
and self._intgr_spacegroup_number == lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice())):
Debug.write("Not reindexing to spacegroup %d (%s)" %
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return mtz_filename
if (self._intgr_reindex_operator is None
and self._intgr_spacegroup_number == 0):
Debug.write("Not reindexing to spacegroup %d (%s)" %
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return mtz_filename
Debug.write(
"Reindexing to spacegroup %d (%s)" %
(self._intgr_spacegroup_number, self._intgr_reindex_operator))
hklin = mtz_filename
from xia2.Wrappers.CCP4.Reindex import Reindex
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
reindex.set_operator(self._intgr_reindex_operator)
if self._intgr_spacegroup_number:
reindex.set_spacegroup(self._intgr_spacegroup_number)
else:
reindex.set_spacegroup(
lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()))
hklout = "%s_reindex.mtz" % hklin[:-4]
reindex.set_hklin(hklin)
reindex.set_hklout(hklout)
reindex.reindex()
self._intgr_integrated_filename = hklout
self._intgr_cell = reindex.get_cell()
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(
"%s %s %s %s" % (pname, xname, dname, sweep),
self.get_integrated_experiments(),
)
FileHandler.record_more_data_file(
"%s %s %s %s" % (pname, xname, dname, sweep),
self.get_integrated_reflections(),
)
return hklout
elif self._output_format == "pickle":
if (self._intgr_reindex_operator is None
and self._intgr_spacegroup_number == lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice())):
Debug.write("Not reindexing to spacegroup %d (%s)" %
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return self._intgr_integrated_reflections
if (self._intgr_reindex_operator is None
and self._intgr_spacegroup_number == 0):
Debug.write("Not reindexing to spacegroup %d (%s)" %
(self._intgr_spacegroup_number,
self._intgr_reindex_operator))
return self._intgr_integrated_reflections
Debug.write(
"Reindexing to spacegroup %d (%s)" %
(self._intgr_spacegroup_number, self._intgr_reindex_operator))
from xia2.Wrappers.Dials.Reindex import Reindex
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
reindex.set_cb_op(self._intgr_reindex_operator)
if self._intgr_spacegroup_number:
reindex.set_space_group(self._intgr_spacegroup_number)
else:
reindex.set_space_group(
lattice_to_spacegroup(
self.get_integrater_refiner().get_refiner_lattice()))
reindex.set_experiments_filename(self.get_integrated_experiments())
reindex.set_indexed_filename(self.get_integrated_reflections())
reindex.run()
self._intgr_integrated_reflections = (
reindex.get_reindexed_reflections_filename())
self._intgr_integrated_filename = (
reindex.get_reindexed_reflections_filename())
self._intgr_experiments_filename = (
reindex.get_reindexed_experiments_filename())
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_more_data_file(
"%s %s %s %s" % (pname, xname, dname, sweep),
self.get_integrated_experiments(),
)
FileHandler.record_more_data_file(
"%s %s %s %s" % (pname, xname, dname, sweep),
self.get_integrated_reflections(),
)
return None # this will be set to intgr_hklout - better to cause failure