def reindex(self):
'''Actually perform the reindexing.'''
if PhilIndex.params.ccp4.reindex.program == 'reindex':
return self.reindex_old()
self.check_hklin()
self.check_hklout()
if not self._spacegroup and not self._operator:
raise RuntimeError, 'reindex requires spacegroup or operator'
if self._operator:
self._operator = self._operator.replace('[', '').replace(']', '')
Debug.write('Reindex... %s %s' % (self._spacegroup, self._operator))
if self._spacegroup and Flags.get_small_molecule() and False:
if not self._operator or self._operator.replace(' ', '') == 'h,k,l':
return self.cctbx_reindex()
self.start()
if self._spacegroup:
if type(self._spacegroup) == type(0):
spacegroup = Syminfo.spacegroup_number_to_name(
self._spacegroup)
elif self._spacegroup[0] in '0123456789':
spacegroup = Syminfo.spacegroup_number_to_name(
int(self._spacegroup))
else:
spacegroup = self._spacegroup
self.input('spacegroup \'%s\'' % spacegroup)
if self._operator:
# likewise
self.input('reindex \'%s\'' % self._operator)
else:
self.input('reindex \'h,k,l\'')
self.close_wait()
# check for errors
try:
self.check_for_errors()
except RuntimeError, e:
try:
os.remove(self.get_hklout())
except:
pass
raise e
def set_lattice(self, lattice):
lattice_to_spacegroup = {'aP':1,
'mP':3,
'mC':5,
'oP':16,
'oC':20,
'oF':22,
'oI':23,
'tP':75,
'tI':79,
'hP':143,
'hR':146,
'cP':195,
'cF':196,
'cI':197}
self._spacegroup = Syminfo.spacegroup_number_to_name(
lattice_to_spacegroup[lattice])
# bug 22/JUL/08 latest lattice symmetry no longer recognises
# the spacegroup H3...
if self._spacegroup == 'H3':
self._spacegroup = 'R3:H'
return
def set_lattice(self, lattice):
lattice_to_spacegroup = {'aP':1,
'mP':3,
'mC':5,
'oP':16,
'oC':20,
'oF':22,
'oI':23,
'tP':75,
'tI':79,
'hP':143,
'hR':146,
'cP':195,
'cF':196,
'cI':197}
self._spacegroup = Syminfo.spacegroup_number_to_name(
lattice_to_spacegroup[lattice])
# bug 22/JUL/08 latest lattice symmetry no longer recognises
# the spacegroup H3...
if self._spacegroup == 'H3':
self._spacegroup = 'R3:H'
return
def set_lattice(self, lattice):
lattice_to_spacegroup = {
"aP": 1,
"mP": 3,
"mC": 5,
"oP": 16,
"oC": 20,
"oF": 22,
"oI": 23,
"tP": 75,
"tI": 79,
"hP": 143,
"hR": 146,
"cP": 195,
"cF": 196,
"cI": 197,
}
self._spacegroup = Syminfo.spacegroup_number_to_name(
lattice_to_spacegroup[lattice])
# bug 22/JUL/08 latest lattice symmetry no longer recognises
# the spacegroup H3...
if self._spacegroup == "H3":
self._spacegroup = "R3:H"
return
def dump(self):
'''Actually obtain the contents of the mtz file header.'''
assert self._hklin, self._hklin
assert os.path.exists(self._hklin), self._hklin
mtz_obj = mtz.object(self._hklin)
# work through the file acculumating the necessary information
self._header['datasets'] = []
self._header['dataset_info'] = { }
self._batches = [batch.num() for batch in mtz_obj.batches()]
self._header['column_labels'] = [column.label() for column
in mtz_obj.columns()]
self._header['column_types'] = [column.type() for column
in mtz_obj.columns()]
self._resolution_range = mtz_obj.max_min_resolution()
spacegroup_and_no = mtz_obj.space_group().info().symbol_and_number()
spacegroup_number = int(spacegroup_and_no.replace(')', '').split()[-1])
from xia2.Handlers.Syminfo import Syminfo
spacegroup = Syminfo.spacegroup_number_to_name(spacegroup_number)
self._header['spacegroup'] = spacegroup
self._reflections = mtz_obj.n_reflections()
for crystal in mtz_obj.crystals():
if crystal.name() == 'HKL_base':
continue
pname = crystal.project_name()
xname = crystal.name()
cell = crystal.unit_cell().parameters()
for dataset in crystal.datasets():
dname = dataset.name()
wavelength = dataset.wavelength()
dataset_id = '%s/%s/%s' % (pname, xname, dname)
dataset_number = dataset.i_dataset()
assert(not dataset_id in self._header['datasets'])
self._header['datasets'].append(dataset_id)
self._header['dataset_info'][dataset_id] = { }
self._header['dataset_info'][
dataset_id]['wavelength'] = wavelength
self._header['dataset_info'][
dataset_id]['cell'] = cell
self._header['dataset_info'][
dataset_id]['id'] = dataset_number
return
def _scale_prepare(self):
"""Perform all of the preparation required to deliver the scaled
data. This should sort together the reflection files, ensure that
they are correctly indexed (via pointless) and generally tidy
things up."""
# acknowledge all of the programs we are about to use...
Citations.cite("pointless")
Citations.cite("aimless")
Citations.cite("ccp4")
# ---------- GATHER ----------
self._sweep_handler = SweepInformationHandler(self._scalr_integraters)
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
exclude_sweep = False
for sweep in PhilIndex.params.xia2.settings.sweep:
if sweep.id == sname and sweep.exclude:
exclude_sweep = True
break
if exclude_sweep:
self._sweep_handler.remove_epoch(epoch)
logger.debug("Excluding sweep %s", sname)
else:
logger.debug("%-30s %s/%s/%s", "adding data from:", xname, dname, sname)
# gather data for all images which belonged to the parent
# crystal - allowing for the fact that things could go wrong
# e.g. epoch information not available, exposure times not in
# headers etc...
for e in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(e)
assert is_mtz_file(si.get_reflections()), repr(si.get_reflections())
p, x = self._sweep_handler.get_project_info()
self._scalr_pname = p
self._scalr_xname = x
# verify that the lattices are consistent, calling eliminate if
# they are not N.B. there could be corner cases here
need_to_return = False
multi_sweep_indexing = PhilIndex.params.xia2.settings.multi_sweep_indexing
# START OF if more than one epoch
if len(self._sweep_handler.get_epochs()) > 1:
# if we have multi-sweep-indexing going on then logic says all should
# share common lattice & UB definition => this is not used here?
# START OF if multi_sweep indexing and not input pg
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
batches = MtzUtils.batches_from_mtz(hklin)
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
logger.debug("Biggest sweep has %d batches", max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
refiners = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
refiners.append(refiner)
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()
)
hklout = os.path.join(
self.get_working_directory(),
"%s_%s_%s_%s_prepointless.mtz"
% (pname, xname, dname, si.get_sweep_name()),
)
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
rebatch(
hklin,
hklout,
first_batch=counter * max_batches + 1,
pname=pname,
xname=xname,
dname=dname,
)
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
# SUMMARY - have added all sweeps to pointless_hklins
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(
self.get_working_directory(),
"%s_%s_prepointless_sorted.mtz"
% (self._scalr_pname, self._scalr_xname),
)
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
# FIXME xia2-51 in here look at running constant scaling on the
# pointless hklin to put the runs on the same scale. Ref=[A]
pointless_const = os.path.join(
self.get_working_directory(),
"%s_%s_prepointless_const.mtz"
% (self._scalr_pname, self._scalr_xname),
)
FileHandler.record_temporary_file(pointless_const)
aimless_const = self._factory.Aimless()
aimless_const.set_hklin(pointless_hklin)
aimless_const.set_hklout(pointless_const)
aimless_const.const()
pointless_const = os.path.join(
self.get_working_directory(),
"%s_%s_prepointless_const_unmerged.mtz"
% (self._scalr_pname, self._scalr_xname),
)
FileHandler.record_temporary_file(pointless_const)
pointless_hklin = pointless_const
# FIXME xia2-51 in here need to pass all refiners to ensure that the
# information is passed back to all of them not just the last one...
logger.debug(
"Running multisweep pointless for %d sweeps", len(refiners)
)
pointgroup, reindex_op, ntr, pt = self._pointless_indexer_multisweep(
pointless_hklin, refiners
)
logger.debug("X1698: %s: %s", pointgroup, reindex_op)
lattices = [Syminfo.get_lattice(pointgroup)]
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
# SUMMARY - added all sweeps together into an mtz, ran
# _pointless_indexer_multisweep on this, made a list of one lattice
# and potentially reset reindex op?
# END OF if multi_sweep indexing and not input pg
# START OF if not multi_sweep, or input pg given
else:
lattices = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
pointgroup = self._scalr_input_pointgroup
reindex_op = "h,k,l"
ntr = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()
)
pointgroup, reindex_op, ntr, pt = self._pointless_indexer_jiffy(
pointless_hklin, refiner
)
logger.debug("X1698: %s: %s", pointgroup, reindex_op)
lattice = Syminfo.get_lattice(pointgroup)
if lattice not in lattices:
lattices.append(lattice)
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
# SUMMARY do pointless_indexer on each sweep, get lattices and make a list
# of unique lattices, potentially reset reindex op.
# END OF if not multi_sweep, or input pg given
# SUMMARY - still within if more than one epoch, now have a list of number
# of lattices
# START OF if multiple-lattices
if len(lattices) > 1:
# why not using pointless indexer jiffy??!
correct_lattice = sort_lattices(lattices)[0]
logger.info("Correct lattice asserted to be %s", correct_lattice)
# transfer this information back to the indexers
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
refiner = si.get_integrater().get_integrater_refiner()
sname = si.get_sweep_name()
state = refiner.set_refiner_asserted_lattice(correct_lattice)
if state == refiner.LATTICE_CORRECT:
logger.info(
"Lattice %s ok for sweep %s", correct_lattice, sname
)
elif state == refiner.LATTICE_IMPOSSIBLE:
raise RuntimeError(
f"Lattice {correct_lattice} impossible for {sname}"
)
elif state == refiner.LATTICE_POSSIBLE:
logger.info(
"Lattice %s assigned for sweep %s", correct_lattice, sname
)
need_to_return = True
# END OF if multiple-lattices
# SUMMARY - forced all lattices to be same and hope its okay.
# END OF if more than one epoch
# if one or more of them was not in the lowest lattice,
# need to return here to allow reprocessing
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# ---------- REINDEX ALL DATA TO CORRECT POINTGROUP ----------
# all should share the same pointgroup, unless twinned... in which
# case force them to be...
pointgroups = {}
reindex_ops = {}
probably_twinned = False
need_to_return = False
multi_sweep_indexing = PhilIndex.params.xia2.settings.multi_sweep_indexing
# START OF if multi-sweep and not input pg
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
batches = MtzUtils.batches_from_mtz(hklin)
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
logger.debug("Biggest sweep has %d batches", max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
refiners = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
refiners.append(refiner)
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()
)
hklout = os.path.join(
self.get_working_directory(),
"%s_%s_%s_%s_prepointless.mtz"
% (pname, xname, dname, si.get_sweep_name()),
)
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
rebatch(
hklin,
hklout,
first_batch=counter * max_batches + 1,
pname=pname,
xname=xname,
dname=dname,
)
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
# FIXME related to xia2-51 - this looks very very similar to the logic
# in [A] above - is this duplicated logic?
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(
self.get_working_directory(),
"%s_%s_prepointless_sorted.mtz"
% (self._scalr_pname, self._scalr_xname),
)
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
pointless_const = os.path.join(
self.get_working_directory(),
f"{self._scalr_pname}_{self._scalr_xname}_prepointless_const.mtz",
)
FileHandler.record_temporary_file(pointless_const)
aimless_const = self._factory.Aimless()
aimless_const.set_hklin(pointless_hklin)
aimless_const.set_hklout(pointless_const)
aimless_const.const()
pointless_const = os.path.join(
self.get_working_directory(),
"%s_%s_prepointless_const_unmerged.mtz"
% (self._scalr_pname, self._scalr_xname),
)
FileHandler.record_temporary_file(pointless_const)
pointless_hklin = pointless_const
pointgroup, reindex_op, ntr, pt = self._pointless_indexer_multisweep(
pointless_hklin, refiners
)
for epoch in self._sweep_handler.get_epochs():
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
# SUMMARY ran pointless multisweep on combined mtz and made a dict
# of pointgroups and reindex_ops (all same)
# END OF if multi-sweep and not input pg
# START OF if not mulit-sweep or pg given
else:
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
if self._scalr_input_pointgroup:
logger.debug(
"Using input pointgroup: %s", self._scalr_input_pointgroup
)
pointgroup = self._scalr_input_pointgroup
reindex_op = "h,k,l"
pt = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()
)
pointgroup, reindex_op, ntr, pt = self._pointless_indexer_jiffy(
pointless_hklin, refiner
)
logger.debug("X1698: %s: %s", pointgroup, reindex_op)
if ntr:
integrater.integrater_reset_reindex_operator()
need_to_return = True
if pt and not probably_twinned:
probably_twinned = True
logger.debug("Pointgroup: %s (%s)", pointgroup, reindex_op)
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
# SUMMARY - for each sweep, run indexer jiffy and get reindex operators
# and pointgroups dictionaries (could be different between sweeps)
# END OF if not mulit-sweep or pg given
overall_pointgroup = None
pointgroup_set = {pointgroups[e] for e in pointgroups}
if len(pointgroup_set) > 1 and not probably_twinned:
raise RuntimeError(
"non uniform pointgroups: %s" % str(list(pointgroup_set))
)
if len(pointgroup_set) > 1:
logger.debug(
"Probably twinned, pointgroups: %s",
" ".join(p.replace(" ", "") for p in pointgroup_set),
)
numbers = (Syminfo.spacegroup_name_to_number(ps) for ps in pointgroup_set)
overall_pointgroup = Syminfo.spacegroup_number_to_name(min(numbers))
self._scalr_input_pointgroup = overall_pointgroup
logger.info("Twinning detected, assume pointgroup %s", overall_pointgroup)
need_to_return = True
else:
overall_pointgroup = pointgroup_set.pop()
# SUMMARY - Have handled if different pointgroups & chosen an overall_pointgroup
# which is the lowest symmetry
# Now go through sweeps and do reindexing
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
integrater = si.get_integrater()
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(overall_pointgroup)
)
integrater.set_integrater_reindex_operator(
reindex_ops[epoch], reason="setting point group"
)
# This will give us the reflections in the correct point group
si.set_reflections(integrater.get_integrater_intensities())
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# in here now optionally work through the data files which should be
# indexed with a consistent point group, and transform the orientation
# matrices by the lattice symmetry operations (if possible) to get a
# consistent definition of U matrix modulo fixed rotations
if PhilIndex.params.xia2.settings.unify_setting:
self.unify_setting()
if self.get_scaler_reference_reflection_file():
self._reference = self.get_scaler_reference_reflection_file()
logger.debug("Using HKLREF %s", self._reference)
elif PhilIndex.params.xia2.settings.scale.reference_reflection_file:
self._reference = (
PhilIndex.params.xia2.settings.scale.reference_reflection_file
)
logger.debug("Using HKLREF %s", self._reference)
params = PhilIndex.params
use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs
if len(self._sweep_handler.get_epochs()) > 1 and use_brehm_diederichs:
self.brehm_diederichs_reindexing()
# If not Brehm-deidrichs, set reference as first sweep
elif len(self._sweep_handler.get_epochs()) > 1 and not self._reference:
first = self._sweep_handler.get_epochs()[0]
si = self._sweep_handler.get_sweep_information(first)
self._reference = si.get_reflections()
# Now reindex to be consistent with first dataset - run pointless on each
# dataset with reference
if self._reference:
md = self._factory.Mtzdump()
md.set_hklin(self._reference)
md.dump()
datasets = md.get_datasets()
# then get the unit cell, lattice etc.
reference_lattice = Syminfo.get_lattice(md.get_spacegroup())
reference_cell = md.get_dataset_info(datasets[0])["cell"]
# then compute the pointgroup from this...
# ---------- REINDEX TO CORRECT (REFERENCE) SETTING ----------
for epoch in self._sweep_handler.get_epochs():
# if we are working with unified UB matrix then this should not
# be a problem here (note, *if*; *should*)
# what about e.g. alternative P1 settings?
# see JIRA MXSW-904
if PhilIndex.params.xia2.settings.unify_setting:
continue
pl = self._factory.Pointless()
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
pl.set_hklin(
self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()
)
)
hklout = os.path.join(
self.get_working_directory(),
"%s_rdx2.mtz" % os.path.split(hklin)[-1][:-4],
)
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
# now set the initial reflection set as a reference...
pl.set_hklref(self._reference)
# https://github.com/xia2/xia2/issues/115 - should ideally iteratively
# construct a reference or a tree of correlations to ensure correct
# reference setting - however if small molecule assume has been
# multi-sweep-indexed so can ignore "fatal errors" - temporary hack
pl.decide_pointgroup(
ignore_errors=PhilIndex.params.xia2.settings.small_molecule
)
logger.debug("Reindexing analysis of %s", pl.get_hklin())
pointgroup = pl.get_pointgroup()
reindex_op = pl.get_reindex_operator()
logger.debug("Operator: %s", reindex_op)
# apply this...
integrater = si.get_integrater()
integrater.set_integrater_reindex_operator(
reindex_op, reason="match reference"
)
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup)
)
si.set_reflections(integrater.get_integrater_intensities())
md = self._factory.Mtzdump()
md.set_hklin(si.get_reflections())
md.dump()
datasets = md.get_datasets()
if len(datasets) > 1:
raise RuntimeError(
"more than one dataset in %s" % si.get_reflections()
)
# then get the unit cell, lattice etc.
lattice = Syminfo.get_lattice(md.get_spacegroup())
cell = md.get_dataset_info(datasets[0])["cell"]
if lattice != reference_lattice:
raise RuntimeError(
"lattices differ in %s and %s"
% (self._reference, si.get_reflections())
)
logger.debug("Cell: %.2f %.2f %.2f %.2f %.2f %.2f" % cell)
logger.debug("Ref: %.2f %.2f %.2f %.2f %.2f %.2f" % reference_cell)
for j in range(6):
if (
math.fabs((cell[j] - reference_cell[j]) / reference_cell[j])
> 0.1
):
raise RuntimeError(
"unit cell parameters differ in %s and %s"
% (self._reference, si.get_reflections())
)
# ---------- SORT TOGETHER DATA ----------
self._sort_together_data_ccp4()
self._scalr_resolution_limits = {}
# store central resolution limit estimates
batch_ranges = [
self._sweep_handler.get_sweep_information(epoch).get_batch_range()
for epoch in self._sweep_handler.get_epochs()
]
self._resolution_limit_estimates = ersatz_resolution(
self._prepared_reflections, batch_ranges
)
def _scale_prepare(self):
'''Perform all of the preparation required to deliver the scaled
data. This should sort together the reflection files, ensure that
they are correctly indexed (via pointless) and generally tidy
things up.'''
# acknowledge all of the programs we are about to use...
Citations.cite('pointless')
Citations.cite('aimless')
Citations.cite('ccp4')
# ---------- GATHER ----------
self._sweep_handler = SweepInformationHandler(self._scalr_integraters)
Journal.block(
'gathering', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'working directory':self.get_working_directory()})
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
exclude_sweep = False
for sweep in PhilIndex.params.xia2.settings.sweep:
if sweep.id == sname and sweep.exclude:
exclude_sweep = True
break
if exclude_sweep:
self._sweep_handler.remove_epoch(epoch)
Debug.write('Excluding sweep %s' %sname)
else:
Journal.entry({'adding data from':'%s/%s/%s' % \
(xname, dname, sname)})
# gather data for all images which belonged to the parent
# crystal - allowing for the fact that things could go wrong
# e.g. epoch information not available, exposure times not in
# headers etc...
for e in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(e)
assert is_mtz_file(si.get_reflections())
p, x = self._sweep_handler.get_project_info()
self._scalr_pname = p
self._scalr_xname = x
# verify that the lattices are consistent, calling eliminate if
# they are not N.B. there could be corner cases here
need_to_return = False
multi_sweep_indexing = \
PhilIndex.params.xia2.settings.developmental.multi_sweep_indexing
if len(self._sweep_handler.get_epochs()) > 1:
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
md = self._factory.Mtzdump()
md.set_hklin(hklin)
md.dump()
batches = md.get_batches()
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
datasets = md.get_datasets()
Debug.write('In reflection file %s found:' % hklin)
for d in datasets:
Debug.write('... %s' % d)
dataset_info = md.get_dataset_info(datasets[0])
from xia2.lib.bits import nifty_power_of_ten
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
rb = self._factory.Rebatch()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_prepointless.mtz' % \
(pname, xname, dname, si.get_sweep_name()))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
rb.set_hklin(hklin)
rb.set_first_batch(counter * max_batches + 1)
rb.set_project_info(pname, xname, dname)
rb.set_hklout(hklout)
new_batches = rb.rebatch()
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattices = [Syminfo.get_lattice(pointgroup)]
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
else:
lattices = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
ntr = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattice = Syminfo.get_lattice(pointgroup)
if not lattice in lattices:
lattices.append(lattice)
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
if len(lattices) > 1:
# why not using pointless indexer jiffy??!
correct_lattice = sort_lattices(lattices)[0]
Chatter.write('Correct lattice asserted to be %s' % \
correct_lattice)
# transfer this information back to the indexers
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
refiner = si.get_integrater().get_integrater_refiner()
sname = si.get_sweep_name()
state = refiner.set_refiner_asserted_lattice(
correct_lattice)
if state == refiner.LATTICE_CORRECT:
Chatter.write('Lattice %s ok for sweep %s' % \
(correct_lattice, sname))
elif state == refiner.LATTICE_IMPOSSIBLE:
raise RuntimeError, 'Lattice %s impossible for %s' \
% (correct_lattice, sname)
elif state == refiner.LATTICE_POSSIBLE:
Chatter.write('Lattice %s assigned for sweep %s' % \
(correct_lattice, sname))
need_to_return = True
# if one or more of them was not in the lowest lattice,
# need to return here to allow reprocessing
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# ---------- REINDEX ALL DATA TO CORRECT POINTGROUP ----------
# all should share the same pointgroup, unless twinned... in which
# case force them to be...
pointgroups = { }
reindex_ops = { }
probably_twinned = False
need_to_return = False
multi_sweep_indexing = \
PhilIndex.params.xia2.settings.developmental.multi_sweep_indexing
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
md = self._factory.Mtzdump()
md.set_hklin(hklin)
md.dump()
batches = md.get_batches()
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
datasets = md.get_datasets()
Debug.write('In reflection file %s found:' % hklin)
for d in datasets:
Debug.write('... %s' % d)
dataset_info = md.get_dataset_info(datasets[0])
from xia2.lib.bits import nifty_power_of_ten
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
rb = self._factory.Rebatch()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_prepointless.mtz' % \
(pname, xname, dname, si.get_sweep_name()))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
rb.set_hklin(hklin)
rb.set_first_batch(counter * max_batches + 1)
rb.set_project_info(pname, xname, dname)
rb.set_hklout(hklout)
new_batches = rb.rebatch()
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
for epoch in self._sweep_handler.get_epochs():
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
else:
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
#hklout = os.path.join(
#self.get_working_directory(),
#os.path.split(hklin)[-1].replace('.mtz', '_rdx.mtz'))
#FileHandler.record_temporary_file(hklout)
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
pt = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
if ntr:
integrater.integrater_reset_reindex_operator()
need_to_return = True
if pt and not probably_twinned:
probably_twinned = True
Debug.write('Pointgroup: %s (%s)' % (pointgroup, reindex_op))
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
overall_pointgroup = None
pointgroup_set = set([pointgroups[e] for e in pointgroups])
if len(pointgroup_set) > 1 and \
not probably_twinned:
raise RuntimeError, 'non uniform pointgroups'
if len(pointgroup_set) > 1:
Debug.write('Probably twinned, pointgroups: %s' % \
' '.join([p.replace(' ', '') for p in \
list(pointgroup_set)]))
numbers = [Syminfo.spacegroup_name_to_number(s) for s in \
pointgroup_set]
overall_pointgroup = Syminfo.spacegroup_number_to_name(
min(numbers))
self._scalr_input_pointgroup = overall_pointgroup
Chatter.write('Twinning detected, assume pointgroup %s' % \
overall_pointgroup)
need_to_return = True
else:
overall_pointgroup = pointgroup_set.pop()
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
integrater = si.get_integrater()
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(overall_pointgroup))
integrater.set_integrater_reindex_operator(
reindex_ops[epoch], reason='setting point group')
# This will give us the reflections in the correct point group
si.set_reflections(integrater.get_integrater_intensities())
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# in here now optinally work through the data files which should be
# indexed with a consistent point group, and transform the orientation
# matrices by the lattice symmetry operations (if possible) to get a
# consistent definition of U matrix modulo fixed rotations
if PhilIndex.params.xia2.settings.unify_setting:
from scitbx.matrix import sqr
reference_U = None
i3 = sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
fixed = sqr(intgr.get_goniometer().get_fixed_rotation())
u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections())
U = fixed.inverse() * sqr(u).transpose()
B = sqr(b)
if reference_U is None:
reference_U = U
continue
results = []
for op in s.all_ops():
R = B * sqr(op.r().as_double()).transpose() * B.inverse()
nearly_i3 = (U * R).inverse() * reference_U
score = sum([abs(_n - _i) for (_n, _i) in zip(nearly_i3, i3)])
results.append((score, op.r().as_hkl(), op))
results.sort()
best = results[0]
Debug.write('Best reindex: %s %.3f' % (best[1], best[0]))
intgr.set_integrater_reindex_operator(best[2].r().inverse().as_hkl(),
reason='unifying [U] setting')
si.set_reflections(intgr.get_integrater_intensities())
# recalculate to verify
u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections())
U = fixed.inverse() * sqr(u).transpose()
Debug.write('New reindex: %s' % (U.inverse() * reference_U))
# FIXME I should probably raise an exception at this stage if this
# is not about I3...
if self.get_scaler_reference_reflection_file():
self._reference = self.get_scaler_reference_reflection_file()
Debug.write('Using HKLREF %s' % self._reference)
elif Flags.get_reference_reflection_file():
self._reference = Flags.get_reference_reflection_file()
Debug.write('Using HKLREF %s' % self._reference)
params = PhilIndex.params
use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs
if len(self._sweep_handler.get_epochs()) > 1 and use_brehm_diederichs:
brehm_diederichs_files_in = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
brehm_diederichs_files_in.append(hklin)
# now run cctbx.brehm_diederichs to figure out the indexing hand for
# each sweep
from xia2.Wrappers.Cctbx.BrehmDiederichs import BrehmDiederichs
from xia2.lib.bits import auto_logfiler
brehm_diederichs = BrehmDiederichs()
brehm_diederichs.set_working_directory(self.get_working_directory())
auto_logfiler(brehm_diederichs)
brehm_diederichs.set_input_filenames(brehm_diederichs_files_in)
# 1 or 3? 1 seems to work better?
brehm_diederichs.set_asymmetric(1)
brehm_diederichs.run()
reindexing_dict = brehm_diederichs.get_reindexing_dict()
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
reindex_op = reindexing_dict.get(os.path.abspath(hklin))
assert reindex_op is not None
if 1 or reindex_op != 'h,k,l':
# apply the reindexing operator
intgr.set_integrater_reindex_operator(
reindex_op, reason='match reference')
si.set_reflections(intgr.get_integrater_intensities())
elif len(self._sweep_handler.get_epochs()) > 1 and \
not self._reference:
first = self._sweep_handler.get_epochs()[0]
si = self._sweep_handler.get_sweep_information(first)
self._reference = si.get_reflections()
if self._reference:
md = self._factory.Mtzdump()
md.set_hklin(self._reference)
md.dump()
if md.get_batches() and False:
raise RuntimeError, 'reference reflection file %s unmerged' % \
self._reference
datasets = md.get_datasets()
if len(datasets) > 1 and False:
raise RuntimeError, 'more than one dataset in %s' % \
self._reference
# then get the unit cell, lattice etc.
reference_lattice = Syminfo.get_lattice(md.get_spacegroup())
reference_cell = md.get_dataset_info(datasets[0])['cell']
# then compute the pointgroup from this...
# ---------- REINDEX TO CORRECT (REFERENCE) SETTING ----------
for epoch in self._sweep_handler.get_epochs():
pl = self._factory.Pointless()
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
pl.set_hklin(self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()))
hklout = os.path.join(
self.get_working_directory(),
'%s_rdx2.mtz' % os.path.split(hklin)[-1][:-4])
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
# now set the initial reflection set as a reference...
pl.set_hklref(self._reference)
# write a pointless log file...
pl.decide_pointgroup()
Debug.write('Reindexing analysis of %s' % pl.get_hklin())
pointgroup = pl.get_pointgroup()
reindex_op = pl.get_reindex_operator()
Debug.write('Operator: %s' % reindex_op)
# apply this...
integrater = si.get_integrater()
integrater.set_integrater_reindex_operator(reindex_op,
reason='match reference')
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
si.set_reflections(integrater.get_integrater_intensities())
md = self._factory.Mtzdump()
md.set_hklin(si.get_reflections())
md.dump()
datasets = md.get_datasets()
if len(datasets) > 1:
raise RuntimeError, 'more than one dataset in %s' % \
si.get_reflections()
# then get the unit cell, lattice etc.
lattice = Syminfo.get_lattice(md.get_spacegroup())
cell = md.get_dataset_info(datasets[0])['cell']
if lattice != reference_lattice:
raise RuntimeError, 'lattices differ in %s and %s' % \
(self._reference, si.get_reflections())
for j in range(6):
if math.fabs((cell[j] - reference_cell[j]) /
reference_cell[j]) > 0.1:
raise RuntimeError, \
'unit cell parameters differ in %s and %s' % \
(self._reference, si.get_reflections())
# ---------- SORT TOGETHER DATA ----------
self._sort_together_data_ccp4()
self._scalr_resolution_limits = { }
# store central resolution limit estimates
batch_ranges = [self._sweep_handler.get_sweep_information(
epoch).get_batch_range() for epoch in
self._sweep_handler.get_epochs()]
self._resolution_limit_estimates = erzatz_resolution(
self._prepared_reflections, batch_ranges)
return
def reindex(self):
"""Actually perform the reindexing."""
if PhilIndex.params.ccp4.reindex.program == "reindex":
return self.reindex_old()
elif PhilIndex.params.ccp4.reindex.program == "cctbx":
return self.cctbx_reindex()
self.check_hklin()
self.check_hklout()
if not self._spacegroup and not self._operator:
raise RuntimeError("reindex requires spacegroup or operator")
if self._operator:
self._operator = self._operator.replace("[",
"").replace("]", "")
Debug.write("Reindex... %s %s" %
(self._spacegroup, self._operator))
self.start()
if self._spacegroup:
if isinstance(self._spacegroup, type(0)):
spacegroup = Syminfo.spacegroup_number_to_name(
self._spacegroup)
elif self._spacegroup[0] in "0123456789":
spacegroup = Syminfo.spacegroup_number_to_name(
int(self._spacegroup))
else:
spacegroup = self._spacegroup
self.input("spacegroup '%s'" % spacegroup)
if self._operator:
# likewise
self.input("reindex '%s'" % self._operator)
else:
self.input("reindex 'h,k,l'")
self.close_wait()
# check for errors
try:
self.check_for_errors()
except RuntimeError as e:
try:
os.remove(self.get_hklout())
except Exception:
pass
raise e
output = self.get_all_output()
for j, o in enumerate(output):
if "Cell Dimensions : (obsolete" in o:
self._cell = map(float, output[j + 2].split())
elif "ReindexOp: syntax error in operator" in o:
raise RuntimeError(o)
return "OK"
def _scale_prepare(self):
'''Perform all of the preparation required to deliver the scaled
data. This should sort together the reflection files, ensure that
they are correctly indexed (via pointless) and generally tidy
things up.'''
# acknowledge all of the programs we are about to use...
Citations.cite('pointless')
Citations.cite('aimless')
Citations.cite('ccp4')
# ---------- GATHER ----------
self._sweep_handler = SweepInformationHandler(self._scalr_integraters)
Journal.block(
'gathering', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'working directory':self.get_working_directory()})
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
exclude_sweep = False
for sweep in PhilIndex.params.xia2.settings.sweep:
if sweep.id == sname and sweep.exclude:
exclude_sweep = True
break
if exclude_sweep:
self._sweep_handler.remove_epoch(epoch)
Debug.write('Excluding sweep %s' % sname)
else:
Journal.entry({'adding data from':'%s/%s/%s' % \
(xname, dname, sname)})
# gather data for all images which belonged to the parent
# crystal - allowing for the fact that things could go wrong
# e.g. epoch information not available, exposure times not in
# headers etc...
for e in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(e)
assert is_mtz_file(si.get_reflections())
p, x = self._sweep_handler.get_project_info()
self._scalr_pname = p
self._scalr_xname = x
# verify that the lattices are consistent, calling eliminate if
# they are not N.B. there could be corner cases here
need_to_return = False
multi_sweep_indexing = \
PhilIndex.params.xia2.settings.multi_sweep_indexing == True
if len(self._sweep_handler.get_epochs()) > 1:
# if we have multi-sweep-indexing going on then logic says all should
# share common lattice & UB definition => this is not used here?
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
batches = MtzUtils.batches_from_mtz(hklin)
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
from xia2.lib.bits import nifty_power_of_ten
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
refiners = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
refiners.append(refiner)
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_prepointless.mtz' % \
(pname, xname, dname, si.get_sweep_name()))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
from xia2.Modules.Scaler.rebatch import rebatch
new_batches = rebatch(
hklin, hklout, first_batch=counter * max_batches + 1,
pname=pname, xname=xname, dname=dname)
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
# FIXME xia2-51 in here look at running constant scaling on the
# pointless hklin to put the runs on the same scale. Ref=[A]
pointless_const = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_const.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(pointless_const)
aimless_const = self._factory.Aimless()
aimless_const.set_hklin(pointless_hklin)
aimless_const.set_hklout(pointless_const)
aimless_const.const()
pointless_const = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_const_unmerged.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(pointless_const)
pointless_hklin = pointless_const
# FIXME xia2-51 in here need to pass all refiners to ensure that the
# information is passed back to all of them not just the last one...
Debug.write('Running multisweep pointless for %d sweeps' %
len(refiners))
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_multisweep(pointless_hklin,
refiners)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattices = [Syminfo.get_lattice(pointgroup)]
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
else:
lattices = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
ntr = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattice = Syminfo.get_lattice(pointgroup)
if not lattice in lattices:
lattices.append(lattice)
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
if len(lattices) > 1:
# why not using pointless indexer jiffy??!
correct_lattice = sort_lattices(lattices)[0]
Chatter.write('Correct lattice asserted to be %s' % \
correct_lattice)
# transfer this information back to the indexers
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
refiner = si.get_integrater().get_integrater_refiner()
sname = si.get_sweep_name()
state = refiner.set_refiner_asserted_lattice(
correct_lattice)
if state == refiner.LATTICE_CORRECT:
Chatter.write('Lattice %s ok for sweep %s' % \
(correct_lattice, sname))
elif state == refiner.LATTICE_IMPOSSIBLE:
raise RuntimeError('Lattice %s impossible for %s' \
% (correct_lattice, sname))
elif state == refiner.LATTICE_POSSIBLE:
Chatter.write('Lattice %s assigned for sweep %s' % \
(correct_lattice, sname))
need_to_return = True
# if one or more of them was not in the lowest lattice,
# need to return here to allow reprocessing
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# ---------- REINDEX ALL DATA TO CORRECT POINTGROUP ----------
# all should share the same pointgroup, unless twinned... in which
# case force them to be...
pointgroups = {}
reindex_ops = {}
probably_twinned = False
need_to_return = False
multi_sweep_indexing = \
PhilIndex.params.xia2.settings.multi_sweep_indexing == True
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
batches = MtzUtils.batches_from_mtz(hklin)
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
from xia2.lib.bits import nifty_power_of_ten
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
refiners = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
refiners.append(refiner)
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_prepointless.mtz' % \
(pname, xname, dname, si.get_sweep_name()))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
from xia2.Modules.Scaler.rebatch import rebatch
new_batches = rebatch(
hklin, hklout, first_batch=counter * max_batches + 1,
pname=pname, xname=xname, dname=dname)
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
# FIXME related to xia2-51 - this looks very very similar to the logic
# in [A] above - is this duplicated logic?
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
pointless_const = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_const.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(pointless_const)
aimless_const = self._factory.Aimless()
aimless_const.set_hklin(pointless_hklin)
aimless_const.set_hklout(pointless_const)
aimless_const.const()
pointless_const = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_const_unmerged.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(pointless_const)
pointless_hklin = pointless_const
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_multisweep(
pointless_hklin, refiners)
for epoch in self._sweep_handler.get_epochs():
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
else:
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
pt = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
if ntr:
integrater.integrater_reset_reindex_operator()
need_to_return = True
if pt and not probably_twinned:
probably_twinned = True
Debug.write('Pointgroup: %s (%s)' % (pointgroup, reindex_op))
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
overall_pointgroup = None
pointgroup_set = {pointgroups[e] for e in pointgroups}
if len(pointgroup_set) > 1 and \
not probably_twinned:
raise RuntimeError('non uniform pointgroups')
if len(pointgroup_set) > 1:
Debug.write('Probably twinned, pointgroups: %s' % \
' '.join([p.replace(' ', '') for p in \
list(pointgroup_set)]))
numbers = [Syminfo.spacegroup_name_to_number(s) for s in \
pointgroup_set]
overall_pointgroup = Syminfo.spacegroup_number_to_name(min(numbers))
self._scalr_input_pointgroup = overall_pointgroup
Chatter.write('Twinning detected, assume pointgroup %s' % \
overall_pointgroup)
need_to_return = True
else:
overall_pointgroup = pointgroup_set.pop()
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
integrater = si.get_integrater()
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(overall_pointgroup))
integrater.set_integrater_reindex_operator(
reindex_ops[epoch], reason='setting point group')
# This will give us the reflections in the correct point group
si.set_reflections(integrater.get_integrater_intensities())
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# in here now optionally work through the data files which should be
# indexed with a consistent point group, and transform the orientation
# matrices by the lattice symmetry operations (if possible) to get a
# consistent definition of U matrix modulo fixed rotations
if PhilIndex.params.xia2.settings.unify_setting:
from scitbx.matrix import sqr
reference_U = None
i3 = sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
fixed = sqr(intgr.get_goniometer().get_fixed_rotation())
u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections())
U = fixed.inverse() * sqr(u).transpose()
B = sqr(b)
if reference_U is None:
reference_U = U
continue
results = []
for op in s.all_ops():
R = B * sqr(op.r().as_double()).transpose() * B.inverse()
nearly_i3 = (U * R).inverse() * reference_U
score = sum([abs(_n - _i) for (_n, _i) in zip(nearly_i3, i3)])
results.append((score, op.r().as_hkl(), op))
results.sort()
best = results[0]
Debug.write('Best reindex: %s %.3f' % (best[1], best[0]))
intgr.set_integrater_reindex_operator(best[2].r().inverse().as_hkl(),
reason='unifying [U] setting')
si.set_reflections(intgr.get_integrater_intensities())
# recalculate to verify
u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections())
U = fixed.inverse() * sqr(u).transpose()
Debug.write('New reindex: %s' % (U.inverse() * reference_U))
# FIXME I should probably raise an exception at this stage if this
# is not about I3...
if self.get_scaler_reference_reflection_file():
self._reference = self.get_scaler_reference_reflection_file()
Debug.write('Using HKLREF %s' % self._reference)
elif PhilIndex.params.xia2.settings.scale.reference_reflection_file:
self._reference = PhilIndex.params.xia2.settings.scale.reference_reflection_file
Debug.write('Using HKLREF %s' % self._reference)
params = PhilIndex.params
use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs
if len(self._sweep_handler.get_epochs()) > 1 and use_brehm_diederichs:
brehm_diederichs_files_in = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
brehm_diederichs_files_in.append(hklin)
# now run cctbx.brehm_diederichs to figure out the indexing hand for
# each sweep
from xia2.Wrappers.Cctbx.BrehmDiederichs import BrehmDiederichs
from xia2.lib.bits import auto_logfiler
brehm_diederichs = BrehmDiederichs()
brehm_diederichs.set_working_directory(self.get_working_directory())
auto_logfiler(brehm_diederichs)
brehm_diederichs.set_input_filenames(brehm_diederichs_files_in)
# 1 or 3? 1 seems to work better?
brehm_diederichs.set_asymmetric(1)
brehm_diederichs.run()
reindexing_dict = brehm_diederichs.get_reindexing_dict()
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
reindex_op = reindexing_dict.get(os.path.abspath(hklin))
assert reindex_op is not None
if 1 or reindex_op != 'h,k,l':
# apply the reindexing operator
intgr.set_integrater_reindex_operator(
reindex_op, reason='match reference')
si.set_reflections(intgr.get_integrater_intensities())
elif len(self._sweep_handler.get_epochs()) > 1 and \
not self._reference:
first = self._sweep_handler.get_epochs()[0]
si = self._sweep_handler.get_sweep_information(first)
self._reference = si.get_reflections()
if self._reference:
md = self._factory.Mtzdump()
md.set_hklin(self._reference)
md.dump()
if md.get_batches() and False:
raise RuntimeError('reference reflection file %s unmerged' % \
self._reference)
datasets = md.get_datasets()
if len(datasets) > 1 and False:
raise RuntimeError('more than one dataset in %s' % \
self._reference)
# then get the unit cell, lattice etc.
reference_lattice = Syminfo.get_lattice(md.get_spacegroup())
reference_cell = md.get_dataset_info(datasets[0])['cell']
# then compute the pointgroup from this...
# ---------- REINDEX TO CORRECT (REFERENCE) SETTING ----------
for epoch in self._sweep_handler.get_epochs():
# if we are working with unified UB matrix then this should not
# be a problem here (note, *if*; *should*)
# what about e.g. alternative P1 settings?
# see JIRA MXSW-904
if PhilIndex.params.xia2.settings.unify_setting:
continue
pl = self._factory.Pointless()
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
pl.set_hklin(self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()))
hklout = os.path.join(
self.get_working_directory(),
'%s_rdx2.mtz' % os.path.split(hklin)[-1][:-4])
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
# now set the initial reflection set as a reference...
pl.set_hklref(self._reference)
# https://github.com/xia2/xia2/issues/115 - should ideally iteratively
# construct a reference or a tree of correlations to ensure correct
# reference setting - however if small molecule assume has been
# multi-sweep-indexed so can ignore "fatal errors" - temporary hack
pl.decide_pointgroup(
ignore_errors=PhilIndex.params.xia2.settings.small_molecule)
Debug.write('Reindexing analysis of %s' % pl.get_hklin())
pointgroup = pl.get_pointgroup()
reindex_op = pl.get_reindex_operator()
Debug.write('Operator: %s' % reindex_op)
# apply this...
integrater = si.get_integrater()
integrater.set_integrater_reindex_operator(reindex_op,
reason='match reference')
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
si.set_reflections(integrater.get_integrater_intensities())
md = self._factory.Mtzdump()
md.set_hklin(si.get_reflections())
md.dump()
datasets = md.get_datasets()
if len(datasets) > 1:
raise RuntimeError('more than one dataset in %s' % \
si.get_reflections())
# then get the unit cell, lattice etc.
lattice = Syminfo.get_lattice(md.get_spacegroup())
cell = md.get_dataset_info(datasets[0])['cell']
if lattice != reference_lattice:
raise RuntimeError('lattices differ in %s and %s' % \
(self._reference, si.get_reflections()))
Debug.write('Cell: %.2f %.2f %.2f %.2f %.2f %.2f' % cell)
Debug.write('Ref: %.2f %.2f %.2f %.2f %.2f %.2f' % reference_cell)
for j in range(6):
if math.fabs((cell[j] - reference_cell[j]) /
reference_cell[j]) > 0.1:
raise RuntimeError( \
'unit cell parameters differ in %s and %s' % \
(self._reference, si.get_reflections()))
# ---------- SORT TOGETHER DATA ----------
self._sort_together_data_ccp4()
self._scalr_resolution_limits = {}
# store central resolution limit estimates
batch_ranges = [
self._sweep_handler.get_sweep_information(epoch).get_batch_range()
for epoch in self._sweep_handler.get_epochs()
]
self._resolution_limit_estimates = ersatz_resolution(
self._prepared_reflections, batch_ranges)
def reindex(self):
'''Actually perform the reindexing.'''
if PhilIndex.params.ccp4.reindex.program == 'reindex':
return self.reindex_old()
elif PhilIndex.params.ccp4.reindex.program == 'cctbx':
return self.cctbx_reindex()
self.check_hklin()
self.check_hklout()
if not self._spacegroup and not self._operator:
raise RuntimeError('reindex requires spacegroup or operator')
if self._operator:
self._operator = self._operator.replace('[',
'').replace(']', '')
Debug.write('Reindex... %s %s' %
(self._spacegroup, self._operator))
if False and self._spacegroup and PhilIndex.params.xia2.settings.small_molecule == True: ## FIXME: This still needed?
if not self._operator or self._operator.replace(' ',
'') == 'h,k,l':
return self.cctbx_reindex()
self.start()
if self._spacegroup:
if isinstance(self._spacegroup, type(0)):
spacegroup = Syminfo.spacegroup_number_to_name(
self._spacegroup)
elif self._spacegroup[0] in '0123456789':
spacegroup = Syminfo.spacegroup_number_to_name(
int(self._spacegroup))
else:
spacegroup = self._spacegroup
self.input('spacegroup \'%s\'' % spacegroup)
if self._operator:
# likewise
self.input('reindex \'%s\'' % self._operator)
else:
self.input('reindex \'h,k,l\'')
self.close_wait()
# check for errors
try:
self.check_for_errors()
except RuntimeError as e:
try:
os.remove(self.get_hklout())
except Exception:
pass
raise e
output = self.get_all_output()
for j, o in enumerate(output):
if 'Cell Dimensions : (obsolete' in o:
self._cell = map(float, output[j + 2].split())
elif 'ReindexOp: syntax error in operator' in o:
raise RuntimeError(o)
return 'OK'
def _standard_scale_prepare(self):
pointgroups = {}
reindex_ops = {}
probably_twinned = False
need_to_return = False
lattices = []
# First check for the existence of multiple lattices. If only one
# epoch, then this gives the necessary data for proceeding straight
# to the point group check.
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
experiment = intgr.get_integrated_experiments()
reflections = intgr.get_integrated_reflections()
refiner = intgr.get_integrater_refiner()
pointgroup, reindex_op, ntr, pt, _, __, ___ = self._dials_symmetry_indexer_jiffy(
[experiment], [reflections], [refiner]
)
lattice = Syminfo.get_lattice(pointgroup)
if lattice not in lattices:
lattices.append(lattice)
if ntr:
si.get_integrater().integrater_reset_reindex_operator()
need_to_return = True
if pt:
probably_twinned = True
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
Debug.write("Pointgroup: %s (%s)" % (pointgroup, reindex_op))
if len(lattices) > 1:
# Check consistency of lattices if more than one. If not, then
# can proceed to straight to checking point group consistency
# using the cached results.
correct_lattice = sort_lattices(lattices)[0]
Chatter.write("Correct lattice asserted to be %s" % correct_lattice)
# transfer this information back to the indexers
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
refiner = si.get_integrater().get_integrater_refiner()
_tup = (correct_lattice, si.get_sweep_name())
state = refiner.set_refiner_asserted_lattice(correct_lattice)
if state == refiner.LATTICE_CORRECT:
Chatter.write("Lattice %s ok for sweep %s" % _tup)
elif state == refiner.LATTICE_IMPOSSIBLE:
raise RuntimeError("Lattice %s impossible for %s" % _tup)
elif state == refiner.LATTICE_POSSIBLE:
Chatter.write("Lattice %s assigned for sweep %s" % _tup)
need_to_return = True
if need_to_return:
return need_to_return
need_to_return = False
pointgroup_set = {pointgroups[e] for e in pointgroups}
if len(pointgroup_set) > 1 and not probably_twinned:
raise RuntimeError(
"non uniform pointgroups: %s" % str(list(pointgroup_set))
)
if len(pointgroup_set) > 1:
Debug.write(
"Probably twinned, pointgroups: %s"
% " ".join([p.replace(" ", "") for p in list(pointgroup_set)])
)
numbers = [Syminfo.spacegroup_name_to_number(s) for s in pointgroup_set]
overall_pointgroup = Syminfo.spacegroup_number_to_name(min(numbers))
self._scalr_input_pointgroup = overall_pointgroup
Chatter.write(
"Twinning detected, assume pointgroup %s" % overall_pointgroup
)
need_to_return = True
else:
overall_pointgroup = pointgroup_set.pop()
self._scalr_likely_spacegroups = [overall_pointgroup]
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
self._helper.reindex_jiffy(si, overall_pointgroup, reindex_ops[epoch])
return need_to_return