def _scale_finish_chunk_8_raddam(self):
crd = CCP4InterRadiationDamageDetector()
crd.set_working_directory(self.get_working_directory())
crd.set_hklin(self._scalr_scaled_reflection_files['mtz'])
if self.get_scaler_anomalous():
crd.set_anomalous(True)
hklout = os.path.join(self.get_working_directory(), 'temp.mtz')
FileHandler.record_temporary_file(hklout)
crd.set_hklout(hklout)
status = crd.detect()
if status:
Chatter.write('')
Chatter.banner('Local Scaling %s' % self._scalr_xname)
for s in status:
Chatter.write('%s %s' % s)
Chatter.banner('')
else:
Debug.write('Local scaling failed')
def _prepare_pointless_hklin(working_directory,
hklin,
phi_width):
'''Prepare some data for pointless - this will take only 180 degrees
of data if there is more than this (through a "rebatch" command) else
will simply return hklin.'''
# also remove blank images?
if not Flags.get_microcrystal() and not Flags.get_small_molecule():
Debug.write('Excluding blank images')
hklout = os.path.join(
working_directory,
'%s_noblank.mtz' % (os.path.split(hklin)[-1][:-4]))
FileHandler.record_temporary_file(hklout)
hklin = remove_blank(hklin, hklout)
# find the number of batches
md = Mtzdump()
md.set_working_directory(working_directory)
auto_logfiler(md)
md.set_hklin(hklin)
md.dump()
batches = max(md.get_batches()) - min(md.get_batches())
phi_limit = 180
if batches * phi_width < phi_limit or Flags.get_small_molecule():
return hklin
hklout = os.path.join(
working_directory,
'%s_prepointless.mtz' % (os.path.split(hklin)[-1][:-4]))
rb = Rebatch()
rb.set_working_directory(working_directory)
auto_logfiler(rb)
rb.set_hklin(hklin)
rb.set_hklout(hklout)
first = min(md.get_batches())
last = first + int(phi_limit / phi_width)
Debug.write('Preparing data for pointless - %d batches (%d degrees)' % \
((last - first), phi_limit))
rb.limit_batches(first, last)
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
return hklout
def _prepare_pointless_hklin(working_directory, hklin, phi_width):
"""Prepare some data for pointless - this will take only 180 degrees
of data if there is more than this (through a "pointless" command) else
will simply return hklin."""
# also remove blank images?
if not PhilIndex.params.xia2.settings.small_molecule:
Debug.write("Excluding blank images")
hklout = os.path.join(
working_directory, "%s_noblank.mtz" % (os.path.split(hklin)[-1][:-4])
)
FileHandler.record_temporary_file(hklout)
hklin = remove_blank(hklin, hklout)
# find the number of batches
batches = MtzUtils.batches_from_mtz(hklin)
n_batches = max(batches) - min(batches)
phi_limit = 180
if (
n_batches * phi_width < phi_limit
or PhilIndex.params.xia2.settings.small_molecule
):
return hklin
hklout = os.path.join(
working_directory, "%s_prepointless.mtz" % (os.path.split(hklin)[-1][:-4])
)
pl = xia2.Wrappers.CCP4.Pointless.Pointless()
pl.set_working_directory(working_directory)
auto_logfiler(pl)
pl.set_hklin(hklin)
pl.set_hklout(hklout)
first = min(batches)
last = first + int(phi_limit / phi_width)
Debug.write(
"Preparing data for pointless - %d batches (%d degrees)"
% ((last - first), phi_limit)
)
pl.limit_batches(first, last)
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
return hklout
def _scale_finish_chunk_2_report(self):
from cctbx.array_family import flex
from iotbx.reflection_file_reader import any_reflection_file
from xia2.lib.bits import auto_logfiler
from xia2.Wrappers.XIA.Report import Report
for wavelength in self._scalr_scaled_refl_files.keys():
mtz_unmerged = self._scalr_scaled_reflection_files['mtz_unmerged'][
wavelength]
reader = any_reflection_file(mtz_unmerged)
mtz_object = reader.file_content()
batches = mtz_object.as_miller_arrays_dict()['HKL_base',
'HKL_base', 'BATCH']
dose = flex.double(batches.size(), -1)
batch_to_dose = self.get_batch_to_dose()
for i, b in enumerate(batches.data()):
dose[i] = batch_to_dose[b]
c = mtz_object.crystals()[0]
d = c.datasets()[0]
d.add_column('DOSE', 'R').set_values(dose.as_float())
tmp_mtz = os.path.join(self.get_working_directory(),
'dose_tmp.mtz')
mtz_object.write(tmp_mtz)
hklin = tmp_mtz
FileHandler.record_temporary_file(hklin)
report = Report()
report.set_working_directory(self.get_working_directory())
report.set_mtz_filename(hklin)
htmlout = os.path.join(
self.get_working_directory(), '%s_%s_%s_report.html' %
(self._scalr_pname, self._scalr_xname, wavelength))
report.set_html_filename(htmlout)
report.set_chef_min_completeness(0.95) # sensible?
auto_logfiler(report)
try:
report.run()
FileHandler.record_html_file(
'%s %s %s report' %
(self._scalr_pname, self._scalr_xname, wavelength),
htmlout)
except Exception as e:
Debug.write('xia2.report failed:')
Debug.write(str(e))
def _do_multisweep_symmetry_analysis(self):
refiners = []
experiments = []
reflections = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
integrater = si.get_integrater()
experiments.append(integrater.get_integrated_experiments())
reflections.append(integrater.get_integrated_reflections())
refiners.append(integrater.get_integrater_refiner())
Debug.write("Running multisweep dials.symmetry for %d sweeps" % len(refiners))
pointgroup, reindex_op, ntr, pt, reind_refl, reind_exp, reindex_initial = self._dials_symmetry_indexer_jiffy(
experiments, reflections, refiners, multisweep=True
)
FileHandler.record_temporary_file(reind_refl)
FileHandler.record_temporary_file(reind_exp)
return pointgroup, reindex_op, ntr, pt, reind_refl, reind_exp, reindex_initial
def _scale_finish_chunk_4_mad_mangling(self):
if len(self._scalr_scaled_refl_files.keys()) > 1:
reflection_files = {}
for wavelength in self._scalr_scaled_refl_files.keys():
cad = self._factory.Cad()
cad.add_hklin(self._scalr_scaled_refl_files[wavelength])
cad.set_hklout(
os.path.join(self.get_working_directory(),
'cad-tmp-%s.mtz' % wavelength))
cad.set_new_suffix(wavelength)
cad.update()
reflection_files[wavelength] = cad.get_hklout()
FileHandler.record_temporary_file(cad.get_hklout())
# now merge the reflection files together...
hklout = os.path.join(
self.get_working_directory(),
'%s_%s_merged.mtz' % (self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(hklout)
Debug.write('Merging all data sets to %s' % hklout)
cad = self._factory.Cad()
for wavelength in reflection_files.keys():
cad.add_hklin(reflection_files[wavelength])
cad.set_hklout(hklout)
cad.merge()
self._scalr_scaled_reflection_files['mtz_merged'] = hklout
else:
self._scalr_scaled_reflection_files[
'mtz_merged'] = self._scalr_scaled_refl_files[
self._scalr_scaled_refl_files.keys()[0]]
def _sort_together_data_xds_one_sweep(self):
assert len(self._sweep_information) == 1
epoch = self._sweep_information.keys()[0]
hklin = self._sweep_information[epoch]['scaled_reflections']
if self.get_scaler_reference_reflection_file():
spacegroups = [
MtzUtils.space_group_name_from_mtz(
self.get_scaler_reference_reflection_file())
]
reindex_operator = 'h,k,l'
elif self._scalr_input_spacegroup:
Debug.write('Assigning user input spacegroup: %s' % \
self._scalr_input_spacegroup)
spacegroups = [self._scalr_input_spacegroup]
reindex_operator = 'h,k,l'
else:
pointless = self._factory.Pointless()
pointless.set_hklin(hklin)
pointless.decide_spacegroup()
FileHandler.record_log_file('%s %s pointless' % \
(self._scalr_pname,
self._scalr_xname),
pointless.get_log_file())
spacegroups = pointless.get_likely_spacegroups()
reindex_operator = pointless.get_spacegroup_reindex_operator()
self._scalr_likely_spacegroups = spacegroups
spacegroup = self._scalr_likely_spacegroups[0]
self._scalr_reindex_operator = clean_reindex_operator(reindex_operator)
Chatter.write('Likely spacegroups:')
for spag in self._scalr_likely_spacegroups:
Chatter.write('%s' % spag)
Chatter.write(
'Reindexing to first spacegroup setting: %s (%s)' % \
(spacegroup, clean_reindex_operator(reindex_operator)))
hklout = os.path.join(self.get_working_directory(),
'%s_%s_reindex.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(hklout)
if reindex_operator == '[h,k,l]':
# just assign spacegroup
from cctbx import sgtbx
s = sgtbx.space_group(
sgtbx.space_group_symbols(str(spacegroup)).hall())
m = mtz.object(hklin)
m.set_space_group(s).write(hklout)
self._scalr_cell = m.crystals()[-1].unit_cell().parameters()
Debug.write(
'Updating unit cell to %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(self._scalr_cell))
del m
del s
else:
ri = self._factory.Reindex()
ri.set_hklin(hklin)
ri.set_hklout(hklout)
ri.set_spacegroup(spacegroup)
ri.set_operator(reindex_operator)
ri.reindex()
Debug.write(
'Updating unit cell to %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(ri.get_cell()))
self._scalr_cell = tuple(ri.get_cell())
hklin = hklout
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s = self._factory.Sortmtz()
s.set_hklin(hklin)
s.set_hklout(hklout)
s.sort(vrset=-99999999.0)
self._prepared_reflections = hklout
def _sort_together_data_ccp4(self):
'''Sort together in the right order (rebatching as we go) the sweeps
we want to scale together.'''
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
hklin = si.get_reflections()
# limit the reflections - e.g. if we are re-running the scaling step
# on just a subset of the integrated data
hklin = si.get_reflections()
limit_batch_range = None
for sweep in PhilIndex.params.xia2.settings.sweep:
if sweep.id == sname and sweep.range is not None:
limit_batch_range = sweep.range
break
if limit_batch_range is not None:
Debug.write('Limiting batch range for %s: %s' %
(sname, limit_batch_range))
start, end = limit_batch_range
hklout = os.path.splitext(hklin)[0] + '_tmp.mtz'
FileHandler.record_temporary_file(hklout)
rb = self._factory.Pointless()
rb.set_hklin(hklin)
rb.set_hklout(hklout)
rb.limit_batches(start, end)
si.set_reflections(hklout)
si.set_batches(limit_batch_range)
# keep a count of the maximum number of batches in a block -
# this will be used to make rebatch work below.
hklin = si.get_reflections()
batches = MtzUtils.batches_from_mtz(hklin)
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
# then rebatch the files, to make sure that the batch numbers are
# in the same order as the epochs of data collection.
counter = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_integrated.mtz' % \
(pname, xname, dname, sname))
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
from xia2.Modules.Scaler.rebatch import rebatch
new_batches = rebatch(hklin,
hklout,
first_batch=counter * max_batches + 1,
pname=pname,
xname=xname,
dname=dname)
# update the "input information"
si.set_reflections(hklout)
si.set_batches(new_batches)
# update the counter & recycle
counter += 1
s = self._factory.Sortmtz()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(hklout)
for epoch in self._sweep_handler.get_epochs():
s.add_hklin(
self._sweep_handler.get_sweep_information(
epoch).get_reflections())
s.sort()
# verify that the measurements are in the correct setting
# choice for the spacegroup
hklin = hklout
hklout = hklin.replace('sorted.mtz', 'temp.mtz')
if not self.get_scaler_reference_reflection_file():
if PhilIndex.params.xia2.settings.symmetry.program == 'dials':
p = self._factory.dials_symmetry()
else:
p = self._factory.Pointless()
FileHandler.record_log_file('%s %s pointless' % \
(self._scalr_pname,
self._scalr_xname),
p.get_log_file())
if len(self._sweep_handler.get_epochs()) > 1:
p.set_hklin(hklin)
else:
# permit the use of pointless preparation...
epoch = self._sweep_handler.get_epochs()[0]
p.set_hklin(
self._prepare_pointless_hklin(
hklin,
self._sweep_handler.get_sweep_information(
epoch).get_integrater().get_phi_width()))
if self._scalr_input_spacegroup:
Debug.write('Assigning user input spacegroup: %s' % \
self._scalr_input_spacegroup)
p.decide_spacegroup()
spacegroup = p.get_spacegroup()
reindex_operator = p.get_spacegroup_reindex_operator()
Debug.write('Pointless thought %s (reindex as %s)' % \
(spacegroup, reindex_operator))
spacegroup = self._scalr_input_spacegroup
reindex_operator = 'h,k,l'
self._spacegroup_reindex_operator = reindex_operator
else:
p.decide_spacegroup()
spacegroup = p.get_spacegroup()
reindex_operator = p.get_spacegroup_reindex_operator()
self._spacegroup_reindex_operator = clean_reindex_operator(
reindex_operator)
Debug.write('Pointless thought %s (reindex as %s)' % \
(spacegroup, reindex_operator))
if self._scalr_input_spacegroup:
self._scalr_likely_spacegroups = [self._scalr_input_spacegroup]
else:
self._scalr_likely_spacegroups = p.get_likely_spacegroups()
Chatter.write('Likely spacegroups:')
for spag in self._scalr_likely_spacegroups:
Chatter.write('%s' % spag)
Chatter.write(
'Reindexing to first spacegroup setting: %s (%s)' % \
(spacegroup, clean_reindex_operator(reindex_operator)))
else:
spacegroup = MtzUtils.space_group_name_from_mtz(
self.get_scaler_reference_reflection_file())
reindex_operator = 'h,k,l'
self._scalr_likely_spacegroups = [spacegroup]
Debug.write('Assigning spacegroup %s from reference' % \
spacegroup)
# then run reindex to set the correct spacegroup
ri = self._factory.Reindex()
ri.set_hklin(hklin)
ri.set_hklout(hklout)
ri.set_spacegroup(spacegroup)
ri.set_operator(reindex_operator)
ri.reindex()
FileHandler.record_temporary_file(hklout)
# then resort the reflections (one last time!)
s = self._factory.Sortmtz()
temp = hklin
hklin = hklout
hklout = temp
s.add_hklin(hklin)
s.set_hklout(hklout)
s.sort()
# done preparing!
self._prepared_reflections = s.get_hklout()
def _sort_together_data_ccp4(self):
'''Sort together in the right order (rebatching as we go) the sweeps
we want to scale together.'''
max_batches = 0
for e in self._sweep_handler.get_epochs():
if Flags.get_small_molecule():
continue
si = self._sweep_handler.get_sweep_information(e)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
# limit the reflections - e.g. if we are re-running the scaling step
# on just a subset of the integrated data
hklin = si.get_reflections()
limit_batch_range = None
for sweep in PhilIndex.params.xia2.settings.sweep:
if sweep.id == sname and sweep.range is not None:
limit_batch_range = sweep.range
break
if limit_batch_range is not None:
Debug.write('Limiting batch range for %s: %s' %(sname, limit_batch_range))
start, end = limit_batch_range
hklout = os.path.splitext(hklin)[0] + '_tmp.mtz'
FileHandler.record_temporary_file(hklout)
rb = self._factory.Pointless()
rb.set_hklin(hklin)
rb.set_hklout(hklout)
rb.limit_batches(start, end)
si.set_reflections(hklout)
si.set_batches(limit_batch_range)
# keep a count of the maximum number of batches in a block -
# this will be used to make rebatch work below.
hklin = si.get_reflections()
md = self._factory.Mtzdump()
md.set_hklin(hklin)
md.dump()
batches = md.get_batches()
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
datasets = md.get_datasets()
Debug.write('In reflection file %s found:' % hklin)
for d in datasets:
Debug.write('... %s' % d)
dataset_info = md.get_dataset_info(datasets[0])
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
# then rebatch the files, to make sure that the batch numbers are
# in the same order as the epochs of data collection.
counter = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
rb = self._factory.Rebatch()
hklin = si.get_reflections()
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_integrated.mtz' % \
(pname, xname, dname, sname))
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
rb.set_hklin(hklin)
rb.set_first_batch(counter * max_batches + 1)
rb.set_project_info(pname, xname, dname)
rb.set_hklout(hklout)
new_batches = rb.rebatch()
# update the "input information"
si.set_reflections(hklout)
si.set_batches(new_batches)
# update the counter & recycle
counter += 1
s = self._factory.Sortmtz()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(hklout)
for epoch in self._sweep_handler.get_epochs():
s.add_hklin(self._sweep_handler.get_sweep_information(
epoch).get_reflections())
s.sort()
# verify that the measurements are in the correct setting
# choice for the spacegroup
hklin = hklout
hklout = hklin.replace('sorted.mtz', 'temp.mtz')
if not self.get_scaler_reference_reflection_file():
p = self._factory.Pointless()
FileHandler.record_log_file('%s %s pointless' % \
(self._scalr_pname,
self._scalr_xname),
p.get_log_file())
if len(self._sweep_handler.get_epochs()) > 1:
p.set_hklin(hklin)
else:
# permit the use of pointless preparation...
epoch = self._sweep_handler.get_epochs()[0]
p.set_hklin(self._prepare_pointless_hklin(
hklin, self._sweep_handler.get_sweep_information(
epoch).get_integrater().get_phi_width()))
if self._scalr_input_spacegroup:
Debug.write('Assigning user input spacegroup: %s' % \
self._scalr_input_spacegroup)
p.decide_spacegroup()
spacegroup = p.get_spacegroup()
reindex_operator = p.get_spacegroup_reindex_operator()
Debug.write('Pointless thought %s (reindex as %s)' % \
(spacegroup, reindex_operator))
spacegroup = self._scalr_input_spacegroup
reindex_operator = 'h,k,l'
elif Flags.get_small_molecule() and False:
p.decide_pointgroup()
spacegroup = p.get_pointgroup()
reindex_operator = p.get_reindex_operator()
Debug.write('Pointless thought %s (reindex as %s)' % \
(spacegroup, reindex_operator))
self._scalr_likely_spacegroups = [spacegroup]
else:
p.decide_spacegroup()
spacegroup = p.get_spacegroup()
reindex_operator = p.get_spacegroup_reindex_operator()
Debug.write('Pointless thought %s (reindex as %s)' % \
(spacegroup, reindex_operator))
if self._scalr_input_spacegroup:
self._scalr_likely_spacegroups = [self._scalr_input_spacegroup]
else:
self._scalr_likely_spacegroups = p.get_likely_spacegroups()
Chatter.write('Likely spacegroups:')
for spag in self._scalr_likely_spacegroups:
Chatter.write('%s' % spag)
Chatter.write(
'Reindexing to first spacegroup setting: %s (%s)' % \
(spacegroup, clean_reindex_operator(reindex_operator)))
else:
md = self._factory.Mtzdump()
md.set_hklin(self.get_scaler_reference_reflection_file())
md.dump()
spacegroup = md.get_spacegroup()
reindex_operator = 'h,k,l'
self._scalr_likely_spacegroups = [spacegroup]
Debug.write('Assigning spacegroup %s from reference' % \
spacegroup)
# then run reindex to set the correct spacegroup
ri = self._factory.Reindex()
ri.set_hklin(hklin)
ri.set_hklout(hklout)
ri.set_spacegroup(spacegroup)
ri.set_operator(reindex_operator)
ri.reindex()
FileHandler.record_temporary_file(hklout)
# then resort the reflections (one last time!)
s = self._factory.Sortmtz()
temp = hklin
hklin = hklout
hklout = temp
s.add_hklin(hklin)
s.set_hklout(hklout)
s.sort()
# done preparing!
self._prepared_reflections = s.get_hklout()
return
def _scale_prepare(self):
"""Perform all of the preparation required to deliver the scaled
data. This should sort together the reflection files, ensure that
they are correctly indexed (via pointless) and generally tidy
things up."""
# acknowledge all of the programs we are about to use...
Citations.cite("pointless")
Citations.cite("aimless")
Citations.cite("ccp4")
# ---------- GATHER ----------
self._sweep_handler = SweepInformationHandler(self._scalr_integraters)
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
exclude_sweep = False
for sweep in PhilIndex.params.xia2.settings.sweep:
if sweep.id == sname and sweep.exclude:
exclude_sweep = True
break
if exclude_sweep:
self._sweep_handler.remove_epoch(epoch)
logger.debug("Excluding sweep %s", sname)
else:
logger.debug("%-30s %s/%s/%s", "adding data from:", xname, dname, sname)
# gather data for all images which belonged to the parent
# crystal - allowing for the fact that things could go wrong
# e.g. epoch information not available, exposure times not in
# headers etc...
for e in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(e)
assert is_mtz_file(si.get_reflections()), repr(si.get_reflections())
p, x = self._sweep_handler.get_project_info()
self._scalr_pname = p
self._scalr_xname = x
# verify that the lattices are consistent, calling eliminate if
# they are not N.B. there could be corner cases here
need_to_return = False
multi_sweep_indexing = PhilIndex.params.xia2.settings.multi_sweep_indexing
# START OF if more than one epoch
if len(self._sweep_handler.get_epochs()) > 1:
# if we have multi-sweep-indexing going on then logic says all should
# share common lattice & UB definition => this is not used here?
# START OF if multi_sweep indexing and not input pg
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
batches = MtzUtils.batches_from_mtz(hklin)
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
logger.debug("Biggest sweep has %d batches", max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
refiners = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
refiners.append(refiner)
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()
)
hklout = os.path.join(
self.get_working_directory(),
"%s_%s_%s_%s_prepointless.mtz"
% (pname, xname, dname, si.get_sweep_name()),
)
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
rebatch(
hklin,
hklout,
first_batch=counter * max_batches + 1,
pname=pname,
xname=xname,
dname=dname,
)
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
# SUMMARY - have added all sweeps to pointless_hklins
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(
self.get_working_directory(),
"%s_%s_prepointless_sorted.mtz"
% (self._scalr_pname, self._scalr_xname),
)
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
# FIXME xia2-51 in here look at running constant scaling on the
# pointless hklin to put the runs on the same scale. Ref=[A]
pointless_const = os.path.join(
self.get_working_directory(),
"%s_%s_prepointless_const.mtz"
% (self._scalr_pname, self._scalr_xname),
)
FileHandler.record_temporary_file(pointless_const)
aimless_const = self._factory.Aimless()
aimless_const.set_hklin(pointless_hklin)
aimless_const.set_hklout(pointless_const)
aimless_const.const()
pointless_const = os.path.join(
self.get_working_directory(),
"%s_%s_prepointless_const_unmerged.mtz"
% (self._scalr_pname, self._scalr_xname),
)
FileHandler.record_temporary_file(pointless_const)
pointless_hklin = pointless_const
# FIXME xia2-51 in here need to pass all refiners to ensure that the
# information is passed back to all of them not just the last one...
logger.debug(
"Running multisweep pointless for %d sweeps", len(refiners)
)
pointgroup, reindex_op, ntr, pt = self._pointless_indexer_multisweep(
pointless_hklin, refiners
)
logger.debug("X1698: %s: %s", pointgroup, reindex_op)
lattices = [Syminfo.get_lattice(pointgroup)]
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
# SUMMARY - added all sweeps together into an mtz, ran
# _pointless_indexer_multisweep on this, made a list of one lattice
# and potentially reset reindex op?
# END OF if multi_sweep indexing and not input pg
# START OF if not multi_sweep, or input pg given
else:
lattices = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
pointgroup = self._scalr_input_pointgroup
reindex_op = "h,k,l"
ntr = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()
)
pointgroup, reindex_op, ntr, pt = self._pointless_indexer_jiffy(
pointless_hklin, refiner
)
logger.debug("X1698: %s: %s", pointgroup, reindex_op)
lattice = Syminfo.get_lattice(pointgroup)
if lattice not in lattices:
lattices.append(lattice)
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
# SUMMARY do pointless_indexer on each sweep, get lattices and make a list
# of unique lattices, potentially reset reindex op.
# END OF if not multi_sweep, or input pg given
# SUMMARY - still within if more than one epoch, now have a list of number
# of lattices
# START OF if multiple-lattices
if len(lattices) > 1:
# why not using pointless indexer jiffy??!
correct_lattice = sort_lattices(lattices)[0]
logger.info("Correct lattice asserted to be %s", correct_lattice)
# transfer this information back to the indexers
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
refiner = si.get_integrater().get_integrater_refiner()
sname = si.get_sweep_name()
state = refiner.set_refiner_asserted_lattice(correct_lattice)
if state == refiner.LATTICE_CORRECT:
logger.info(
"Lattice %s ok for sweep %s", correct_lattice, sname
)
elif state == refiner.LATTICE_IMPOSSIBLE:
raise RuntimeError(
f"Lattice {correct_lattice} impossible for {sname}"
)
elif state == refiner.LATTICE_POSSIBLE:
logger.info(
"Lattice %s assigned for sweep %s", correct_lattice, sname
)
need_to_return = True
# END OF if multiple-lattices
# SUMMARY - forced all lattices to be same and hope its okay.
# END OF if more than one epoch
# if one or more of them was not in the lowest lattice,
# need to return here to allow reprocessing
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# ---------- REINDEX ALL DATA TO CORRECT POINTGROUP ----------
# all should share the same pointgroup, unless twinned... in which
# case force them to be...
pointgroups = {}
reindex_ops = {}
probably_twinned = False
need_to_return = False
multi_sweep_indexing = PhilIndex.params.xia2.settings.multi_sweep_indexing
# START OF if multi-sweep and not input pg
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
batches = MtzUtils.batches_from_mtz(hklin)
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
logger.debug("Biggest sweep has %d batches", max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
refiners = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
refiners.append(refiner)
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()
)
hklout = os.path.join(
self.get_working_directory(),
"%s_%s_%s_%s_prepointless.mtz"
% (pname, xname, dname, si.get_sweep_name()),
)
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
rebatch(
hklin,
hklout,
first_batch=counter * max_batches + 1,
pname=pname,
xname=xname,
dname=dname,
)
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
# FIXME related to xia2-51 - this looks very very similar to the logic
# in [A] above - is this duplicated logic?
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(
self.get_working_directory(),
"%s_%s_prepointless_sorted.mtz"
% (self._scalr_pname, self._scalr_xname),
)
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
pointless_const = os.path.join(
self.get_working_directory(),
f"{self._scalr_pname}_{self._scalr_xname}_prepointless_const.mtz",
)
FileHandler.record_temporary_file(pointless_const)
aimless_const = self._factory.Aimless()
aimless_const.set_hklin(pointless_hklin)
aimless_const.set_hklout(pointless_const)
aimless_const.const()
pointless_const = os.path.join(
self.get_working_directory(),
"%s_%s_prepointless_const_unmerged.mtz"
% (self._scalr_pname, self._scalr_xname),
)
FileHandler.record_temporary_file(pointless_const)
pointless_hklin = pointless_const
pointgroup, reindex_op, ntr, pt = self._pointless_indexer_multisweep(
pointless_hklin, refiners
)
for epoch in self._sweep_handler.get_epochs():
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
# SUMMARY ran pointless multisweep on combined mtz and made a dict
# of pointgroups and reindex_ops (all same)
# END OF if multi-sweep and not input pg
# START OF if not mulit-sweep or pg given
else:
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
if self._scalr_input_pointgroup:
logger.debug(
"Using input pointgroup: %s", self._scalr_input_pointgroup
)
pointgroup = self._scalr_input_pointgroup
reindex_op = "h,k,l"
pt = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()
)
pointgroup, reindex_op, ntr, pt = self._pointless_indexer_jiffy(
pointless_hklin, refiner
)
logger.debug("X1698: %s: %s", pointgroup, reindex_op)
if ntr:
integrater.integrater_reset_reindex_operator()
need_to_return = True
if pt and not probably_twinned:
probably_twinned = True
logger.debug("Pointgroup: %s (%s)", pointgroup, reindex_op)
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
# SUMMARY - for each sweep, run indexer jiffy and get reindex operators
# and pointgroups dictionaries (could be different between sweeps)
# END OF if not mulit-sweep or pg given
overall_pointgroup = None
pointgroup_set = {pointgroups[e] for e in pointgroups}
if len(pointgroup_set) > 1 and not probably_twinned:
raise RuntimeError(
"non uniform pointgroups: %s" % str(list(pointgroup_set))
)
if len(pointgroup_set) > 1:
logger.debug(
"Probably twinned, pointgroups: %s",
" ".join(p.replace(" ", "") for p in pointgroup_set),
)
numbers = (Syminfo.spacegroup_name_to_number(ps) for ps in pointgroup_set)
overall_pointgroup = Syminfo.spacegroup_number_to_name(min(numbers))
self._scalr_input_pointgroup = overall_pointgroup
logger.info("Twinning detected, assume pointgroup %s", overall_pointgroup)
need_to_return = True
else:
overall_pointgroup = pointgroup_set.pop()
# SUMMARY - Have handled if different pointgroups & chosen an overall_pointgroup
# which is the lowest symmetry
# Now go through sweeps and do reindexing
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
integrater = si.get_integrater()
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(overall_pointgroup)
)
integrater.set_integrater_reindex_operator(
reindex_ops[epoch], reason="setting point group"
)
# This will give us the reflections in the correct point group
si.set_reflections(integrater.get_integrater_intensities())
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# in here now optionally work through the data files which should be
# indexed with a consistent point group, and transform the orientation
# matrices by the lattice symmetry operations (if possible) to get a
# consistent definition of U matrix modulo fixed rotations
if PhilIndex.params.xia2.settings.unify_setting:
self.unify_setting()
if self.get_scaler_reference_reflection_file():
self._reference = self.get_scaler_reference_reflection_file()
logger.debug("Using HKLREF %s", self._reference)
elif PhilIndex.params.xia2.settings.scale.reference_reflection_file:
self._reference = (
PhilIndex.params.xia2.settings.scale.reference_reflection_file
)
logger.debug("Using HKLREF %s", self._reference)
params = PhilIndex.params
use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs
if len(self._sweep_handler.get_epochs()) > 1 and use_brehm_diederichs:
self.brehm_diederichs_reindexing()
# If not Brehm-deidrichs, set reference as first sweep
elif len(self._sweep_handler.get_epochs()) > 1 and not self._reference:
first = self._sweep_handler.get_epochs()[0]
si = self._sweep_handler.get_sweep_information(first)
self._reference = si.get_reflections()
# Now reindex to be consistent with first dataset - run pointless on each
# dataset with reference
if self._reference:
md = self._factory.Mtzdump()
md.set_hklin(self._reference)
md.dump()
datasets = md.get_datasets()
# then get the unit cell, lattice etc.
reference_lattice = Syminfo.get_lattice(md.get_spacegroup())
reference_cell = md.get_dataset_info(datasets[0])["cell"]
# then compute the pointgroup from this...
# ---------- REINDEX TO CORRECT (REFERENCE) SETTING ----------
for epoch in self._sweep_handler.get_epochs():
# if we are working with unified UB matrix then this should not
# be a problem here (note, *if*; *should*)
# what about e.g. alternative P1 settings?
# see JIRA MXSW-904
if PhilIndex.params.xia2.settings.unify_setting:
continue
pl = self._factory.Pointless()
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
pl.set_hklin(
self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()
)
)
hklout = os.path.join(
self.get_working_directory(),
"%s_rdx2.mtz" % os.path.split(hklin)[-1][:-4],
)
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
# now set the initial reflection set as a reference...
pl.set_hklref(self._reference)
# https://github.com/xia2/xia2/issues/115 - should ideally iteratively
# construct a reference or a tree of correlations to ensure correct
# reference setting - however if small molecule assume has been
# multi-sweep-indexed so can ignore "fatal errors" - temporary hack
pl.decide_pointgroup(
ignore_errors=PhilIndex.params.xia2.settings.small_molecule
)
logger.debug("Reindexing analysis of %s", pl.get_hklin())
pointgroup = pl.get_pointgroup()
reindex_op = pl.get_reindex_operator()
logger.debug("Operator: %s", reindex_op)
# apply this...
integrater = si.get_integrater()
integrater.set_integrater_reindex_operator(
reindex_op, reason="match reference"
)
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup)
)
si.set_reflections(integrater.get_integrater_intensities())
md = self._factory.Mtzdump()
md.set_hklin(si.get_reflections())
md.dump()
datasets = md.get_datasets()
if len(datasets) > 1:
raise RuntimeError(
"more than one dataset in %s" % si.get_reflections()
)
# then get the unit cell, lattice etc.
lattice = Syminfo.get_lattice(md.get_spacegroup())
cell = md.get_dataset_info(datasets[0])["cell"]
if lattice != reference_lattice:
raise RuntimeError(
"lattices differ in %s and %s"
% (self._reference, si.get_reflections())
)
logger.debug("Cell: %.2f %.2f %.2f %.2f %.2f %.2f" % cell)
logger.debug("Ref: %.2f %.2f %.2f %.2f %.2f %.2f" % reference_cell)
for j in range(6):
if (
math.fabs((cell[j] - reference_cell[j]) / reference_cell[j])
> 0.1
):
raise RuntimeError(
"unit cell parameters differ in %s and %s"
% (self._reference, si.get_reflections())
)
# ---------- SORT TOGETHER DATA ----------
self._sort_together_data_ccp4()
self._scalr_resolution_limits = {}
# store central resolution limit estimates
batch_ranges = [
self._sweep_handler.get_sweep_information(epoch).get_batch_range()
for epoch in self._sweep_handler.get_epochs()
]
self._resolution_limit_estimates = ersatz_resolution(
self._prepared_reflections, batch_ranges
)
else:
sc.scale()
# then gather up all of the resulting reflection files
# and convert them into the required formats (.sca, .mtz.)
data = sc.get_summary()
loggraph = sc.parse_ccp4_loggraph()
resolution_info = { }
reflection_files = sc.get_scaled_reflection_files()
for dataset in reflection_files:
FileHandler.record_temporary_file(reflection_files[dataset])
for key in loggraph:
if 'Analysis against resolution' in key:
dataset = key.split(',')[-1].strip()
resolution_info[dataset] = transpose_loggraph(
loggraph[key])
highest_resolution = 100.0
# check in here that there is actually some data to scale..!
if len(resolution_info) == 0:
raise RuntimeError, 'no resolution info'
for epoch in epochs:
def _scale(self):
'''Perform all of the operations required to deliver the scaled
data.'''
epochs = self._sweep_handler.get_epochs()
if self._scalr_corrections:
Journal.block(
'scaling', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'scaling model':'automatic',
'absorption':self._scalr_correct_absorption,
'decay':self._scalr_correct_decay
})
else:
Journal.block(
'scaling', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'scaling model':'default'})
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_chef_unmerged(True)
sc.set_new_scales_file('%s.scales' % self._scalr_xname)
user_resolution_limits = {}
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
intgr = si.get_integrater()
if intgr.get_integrater_user_resolution():
dmin = intgr.get_integrater_high_resolution()
if (dname, sname) not in user_resolution_limits:
user_resolution_limits[(dname, sname)] = dmin
elif dmin < user_resolution_limits[(dname, sname)]:
user_resolution_limits[(dname, sname)] = dmin
start, end = si.get_batch_range()
if (dname, sname) in self._scalr_resolution_limits:
resolution, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude=False, resolution=resolution, name=sname)
else:
sc.add_run(start, end, name=sname)
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_scaled_test.mtz' % \
(self._scalr_pname, self._scalr_xname)))
if self.get_scaler_anomalous():
sc.set_anomalous()
# what follows, sucks
failover = PhilIndex.params.xia2.settings.failover
if failover:
try:
sc.scale()
except RuntimeError as e:
es = str(e)
if 'bad batch' in es or \
'negative scales run' in es or \
'no observations' in es:
# first ID the sweep from the batch no
batch = int(es.split()[-1])
epoch = self._identify_sweep_epoch(batch)
sweep = self._scalr_integraters[epoch].get_integrater_sweep()
# then remove it from my parent xcrystal
self.get_scaler_xcrystal().remove_sweep(sweep)
# then remove it from the scaler list of intergraters
# - this should really be a scaler interface method
del self._scalr_integraters[epoch]
# then tell the user what is happening
Chatter.write(
'Sweep %s gave negative scales - removing' % \
sweep.get_name())
# then reset the prepare, do, finish flags
self.set_scaler_prepare_done(False)
self.set_scaler_done(False)
self.set_scaler_finish_done(False)
# and return
return
else:
raise e
else:
sc.scale()
# then gather up all of the resulting reflection files
# and convert them into the required formats (.sca, .mtz.)
data = sc.get_summary()
loggraph = sc.parse_ccp4_loggraph()
resolution_info = {}
reflection_files = sc.get_scaled_reflection_files()
for dataset in reflection_files:
FileHandler.record_temporary_file(reflection_files[dataset])
for key in loggraph:
if 'Analysis against resolution' in key:
dataset = key.split(',')[-1].strip()
resolution_info[dataset] = transpose_loggraph(loggraph[key])
highest_resolution = 100.0
highest_suggested_resolution = None
# check in here that there is actually some data to scale..!
if len(resolution_info) == 0:
raise RuntimeError('no resolution info')
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
intgr = si.get_integrater()
start, end = si.get_batch_range()
if (dname, sname) in self._scalr_resolution_limits:
continue
elif (dname, sname) in user_resolution_limits:
limit = user_resolution_limits[(dname, sname)]
self._scalr_resolution_limits[(dname, sname)] = (limit, None)
if limit < highest_resolution:
highest_resolution = limit
Chatter.write('Resolution limit for %s: %5.2f (user provided)' % \
(dname, limit))
continue
hklin = sc.get_unmerged_reflection_file()
limit, reasoning = self._estimate_resolution_limit(
hklin, batch_range=(start, end))
if PhilIndex.params.xia2.settings.resolution.keep_all_reflections == True:
suggested = limit
if highest_suggested_resolution is None or limit < highest_suggested_resolution:
highest_suggested_resolution = limit
limit = intgr.get_detector().get_max_resolution(intgr.get_beam_obj().get_s0())
self._scalr_resolution_limits[(dname, sname)] = (limit, suggested)
Debug.write('keep_all_reflections set, using detector limits')
Debug.write('Resolution for sweep %s: %.2f' % \
(sname, limit))
if not (dname, sname) in self._scalr_resolution_limits:
self._scalr_resolution_limits[(dname, sname)] = (limit, None)
self.set_scaler_done(False)
if limit < highest_resolution:
highest_resolution = limit
limit, suggested = self._scalr_resolution_limits[(dname, sname)]
if suggested is None or limit == suggested:
reasoning_str = ''
if reasoning:
reasoning_str = ' (%s)' % reasoning
Chatter.write('Resolution for sweep %s/%s: %.2f%s' % \
(dname, sname, limit, reasoning_str))
else:
Chatter.write('Resolution limit for %s/%s: %5.2f (%5.2f suggested)' % \
(dname, sname, limit, suggested))
if highest_suggested_resolution is not None and \
highest_resolution >= (highest_suggested_resolution - 0.004):
Debug.write('Dropping resolution cut-off suggestion since it is'
' essentially identical to the actual resolution limit.')
highest_suggested_resolution = None
self._scalr_highest_resolution = highest_resolution
self._scalr_highest_suggested_resolution = highest_suggested_resolution
if highest_suggested_resolution is not None:
Debug.write('Suggested highest resolution is %5.2f (%5.2f suggested)' % \
(highest_resolution, highest_suggested_resolution))
else:
Debug.write('Scaler highest resolution set to %5.2f' % \
highest_resolution)
if not self.get_scaler_done():
Debug.write('Returning as scaling not finished...')
return
batch_info = {}
for key in loggraph:
if 'Analysis against Batch' in key:
dataset = key.split(',')[-1].strip()
batch_info[dataset] = transpose_loggraph(loggraph[key])
sc = self._updated_aimless()
FileHandler.record_log_file('%s %s aimless' % (self._scalr_pname,
self._scalr_xname),
sc.get_log_file())
sc.set_hklin(self._prepared_reflections)
sc.set_new_scales_file('%s_final.scales' % self._scalr_xname)
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude=False, resolution=resolution_limit,
name=xname)
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_scaled.mtz' % \
(self._scalr_pname, self._scalr_xname)))
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
FileHandler.record_xml_file('%s %s aimless xml' % (self._scalr_pname,
self._scalr_xname),
sc.get_xmlout())
data = sc.get_summary()
scales_file = sc.get_new_scales_file()
loggraph = sc.parse_ccp4_loggraph()
standard_deviation_info = {}
for key in loggraph:
if 'standard deviation v. Intensity' in key:
dataset = key.split(',')[-1].strip()
standard_deviation_info[dataset] = transpose_loggraph(loggraph[key])
resolution_info = {}
for key in loggraph:
if 'Analysis against resolution' in key:
dataset = key.split(',')[-1].strip()
resolution_info[dataset] = transpose_loggraph(loggraph[key])
batch_info = {}
for key in loggraph:
if 'Analysis against Batch' in key:
dataset = key.split(',')[-1].strip()
batch_info[dataset] = transpose_loggraph(loggraph[key])
# finally put all of the results "somewhere useful"
self._scalr_statistics = data
self._scalr_scaled_refl_files = copy.deepcopy(
sc.get_scaled_reflection_files())
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_scales_file(scales_file)
self._wavelengths_in_order = []
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude=False, resolution=resolution_limit,
name=sname)
if not dname in self._wavelengths_in_order:
self._wavelengths_in_order.append(dname)
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_scaled.mtz' % \
(self._scalr_pname,
self._scalr_xname)))
sc.set_scalepack()
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
self._update_scaled_unit_cell()
self._scalr_scaled_reflection_files = {}
self._scalr_scaled_reflection_files['sca'] = {}
self._scalr_scaled_reflection_files['sca_unmerged'] = {}
self._scalr_scaled_reflection_files['mtz_unmerged'] = {}
for key in self._scalr_scaled_refl_files:
hklout = self._scalr_scaled_refl_files[key]
scaout = '%s.sca' % hklout[:-4]
self._scalr_scaled_reflection_files['sca'][key] = scaout
FileHandler.record_data_file(scaout)
scalepack = os.path.join(os.path.split(hklout)[0],
os.path.split(hklout)[1].replace(
'_scaled', '_scaled_unmerged').replace('.mtz', '.sca'))
self._scalr_scaled_reflection_files['sca_unmerged'][key] = scalepack
FileHandler.record_data_file(scalepack)
mtz_unmerged = os.path.splitext(scalepack)[0] + '.mtz'
self._scalr_scaled_reflection_files['mtz_unmerged'][key] = mtz_unmerged
FileHandler.record_data_file(mtz_unmerged)
if self._scalr_cell_esd is not None:
# patch .mtz and overwrite unit cell information
import xia2.Modules.Scaler.tools as tools
override_cell = self._scalr_cell_dict.get('%s_%s_%s' %
(self._scalr_pname, self._scalr_xname, key))[0]
tools.patch_mtz_unit_cell(mtz_unmerged, override_cell)
tools.patch_mtz_unit_cell(hklout, override_cell)
self._scalr_scaled_reflection_files['mtz_unmerged'][key] = mtz_unmerged
FileHandler.record_data_file(mtz_unmerged)
if PhilIndex.params.xia2.settings.merging_statistics.source == 'cctbx':
for key in self._scalr_scaled_refl_files:
stats = self._compute_scaler_statistics(
self._scalr_scaled_reflection_files['mtz_unmerged'][key],
selected_band=(highest_suggested_resolution, None), wave=key)
self._scalr_statistics[
(self._scalr_pname, self._scalr_xname, key)] = stats
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_scales_file(scales_file)
self._wavelengths_in_order = []
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude=False, resolution=resolution_limit,
name=sname)
if not dname in self._wavelengths_in_order:
self._wavelengths_in_order.append(dname)
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_chef.mtz' % \
(self._scalr_pname,
self._scalr_xname)))
sc.set_chef_unmerged(True)
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
if not PhilIndex.params.dials.fast_mode:
try:
self._generate_absorption_map(sc)
except Exception as e:
# Map generation may fail for number of reasons, eg. matplotlib borken
Debug.write("Could not generate absorption map (%s)" % e)
def _scale_prepare(self):
'''Perform all of the preparation required to deliver the scaled
data. This should sort together the reflection files, ensure that
they are correctly indexed (via pointless) and generally tidy
things up.'''
# acknowledge all of the programs we are about to use...
Citations.cite('pointless')
Citations.cite('aimless')
Citations.cite('ccp4')
# ---------- GATHER ----------
self._sweep_handler = SweepInformationHandler(self._scalr_integraters)
Journal.block(
'gathering', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'working directory':self.get_working_directory()})
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
exclude_sweep = False
for sweep in PhilIndex.params.xia2.settings.sweep:
if sweep.id == sname and sweep.exclude:
exclude_sweep = True
break
if exclude_sweep:
self._sweep_handler.remove_epoch(epoch)
Debug.write('Excluding sweep %s' % sname)
else:
Journal.entry({'adding data from':'%s/%s/%s' % \
(xname, dname, sname)})
# gather data for all images which belonged to the parent
# crystal - allowing for the fact that things could go wrong
# e.g. epoch information not available, exposure times not in
# headers etc...
for e in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(e)
assert is_mtz_file(si.get_reflections())
p, x = self._sweep_handler.get_project_info()
self._scalr_pname = p
self._scalr_xname = x
# verify that the lattices are consistent, calling eliminate if
# they are not N.B. there could be corner cases here
need_to_return = False
multi_sweep_indexing = \
PhilIndex.params.xia2.settings.multi_sweep_indexing == True
if len(self._sweep_handler.get_epochs()) > 1:
# if we have multi-sweep-indexing going on then logic says all should
# share common lattice & UB definition => this is not used here?
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
batches = MtzUtils.batches_from_mtz(hklin)
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
from xia2.lib.bits import nifty_power_of_ten
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
refiners = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
refiners.append(refiner)
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_prepointless.mtz' % \
(pname, xname, dname, si.get_sweep_name()))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
from xia2.Modules.Scaler.rebatch import rebatch
new_batches = rebatch(
hklin, hklout, first_batch=counter * max_batches + 1,
pname=pname, xname=xname, dname=dname)
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
# FIXME xia2-51 in here look at running constant scaling on the
# pointless hklin to put the runs on the same scale. Ref=[A]
pointless_const = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_const.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(pointless_const)
aimless_const = self._factory.Aimless()
aimless_const.set_hklin(pointless_hklin)
aimless_const.set_hklout(pointless_const)
aimless_const.const()
pointless_const = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_const_unmerged.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(pointless_const)
pointless_hklin = pointless_const
# FIXME xia2-51 in here need to pass all refiners to ensure that the
# information is passed back to all of them not just the last one...
Debug.write('Running multisweep pointless for %d sweeps' %
len(refiners))
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_multisweep(pointless_hklin,
refiners)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattices = [Syminfo.get_lattice(pointgroup)]
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
else:
lattices = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
ntr = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattice = Syminfo.get_lattice(pointgroup)
if not lattice in lattices:
lattices.append(lattice)
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
if len(lattices) > 1:
# why not using pointless indexer jiffy??!
correct_lattice = sort_lattices(lattices)[0]
Chatter.write('Correct lattice asserted to be %s' % \
correct_lattice)
# transfer this information back to the indexers
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
refiner = si.get_integrater().get_integrater_refiner()
sname = si.get_sweep_name()
state = refiner.set_refiner_asserted_lattice(
correct_lattice)
if state == refiner.LATTICE_CORRECT:
Chatter.write('Lattice %s ok for sweep %s' % \
(correct_lattice, sname))
elif state == refiner.LATTICE_IMPOSSIBLE:
raise RuntimeError('Lattice %s impossible for %s' \
% (correct_lattice, sname))
elif state == refiner.LATTICE_POSSIBLE:
Chatter.write('Lattice %s assigned for sweep %s' % \
(correct_lattice, sname))
need_to_return = True
# if one or more of them was not in the lowest lattice,
# need to return here to allow reprocessing
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# ---------- REINDEX ALL DATA TO CORRECT POINTGROUP ----------
# all should share the same pointgroup, unless twinned... in which
# case force them to be...
pointgroups = {}
reindex_ops = {}
probably_twinned = False
need_to_return = False
multi_sweep_indexing = \
PhilIndex.params.xia2.settings.multi_sweep_indexing == True
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
batches = MtzUtils.batches_from_mtz(hklin)
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
from xia2.lib.bits import nifty_power_of_ten
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
refiners = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
refiners.append(refiner)
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_prepointless.mtz' % \
(pname, xname, dname, si.get_sweep_name()))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
from xia2.Modules.Scaler.rebatch import rebatch
new_batches = rebatch(
hklin, hklout, first_batch=counter * max_batches + 1,
pname=pname, xname=xname, dname=dname)
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
# FIXME related to xia2-51 - this looks very very similar to the logic
# in [A] above - is this duplicated logic?
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
pointless_const = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_const.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(pointless_const)
aimless_const = self._factory.Aimless()
aimless_const.set_hklin(pointless_hklin)
aimless_const.set_hklout(pointless_const)
aimless_const.const()
pointless_const = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_const_unmerged.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(pointless_const)
pointless_hklin = pointless_const
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_multisweep(
pointless_hklin, refiners)
for epoch in self._sweep_handler.get_epochs():
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
else:
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
pt = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
if ntr:
integrater.integrater_reset_reindex_operator()
need_to_return = True
if pt and not probably_twinned:
probably_twinned = True
Debug.write('Pointgroup: %s (%s)' % (pointgroup, reindex_op))
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
overall_pointgroup = None
pointgroup_set = {pointgroups[e] for e in pointgroups}
if len(pointgroup_set) > 1 and \
not probably_twinned:
raise RuntimeError('non uniform pointgroups')
if len(pointgroup_set) > 1:
Debug.write('Probably twinned, pointgroups: %s' % \
' '.join([p.replace(' ', '') for p in \
list(pointgroup_set)]))
numbers = [Syminfo.spacegroup_name_to_number(s) for s in \
pointgroup_set]
overall_pointgroup = Syminfo.spacegroup_number_to_name(min(numbers))
self._scalr_input_pointgroup = overall_pointgroup
Chatter.write('Twinning detected, assume pointgroup %s' % \
overall_pointgroup)
need_to_return = True
else:
overall_pointgroup = pointgroup_set.pop()
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
integrater = si.get_integrater()
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(overall_pointgroup))
integrater.set_integrater_reindex_operator(
reindex_ops[epoch], reason='setting point group')
# This will give us the reflections in the correct point group
si.set_reflections(integrater.get_integrater_intensities())
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# in here now optionally work through the data files which should be
# indexed with a consistent point group, and transform the orientation
# matrices by the lattice symmetry operations (if possible) to get a
# consistent definition of U matrix modulo fixed rotations
if PhilIndex.params.xia2.settings.unify_setting:
from scitbx.matrix import sqr
reference_U = None
i3 = sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
fixed = sqr(intgr.get_goniometer().get_fixed_rotation())
u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections())
U = fixed.inverse() * sqr(u).transpose()
B = sqr(b)
if reference_U is None:
reference_U = U
continue
results = []
for op in s.all_ops():
R = B * sqr(op.r().as_double()).transpose() * B.inverse()
nearly_i3 = (U * R).inverse() * reference_U
score = sum([abs(_n - _i) for (_n, _i) in zip(nearly_i3, i3)])
results.append((score, op.r().as_hkl(), op))
results.sort()
best = results[0]
Debug.write('Best reindex: %s %.3f' % (best[1], best[0]))
intgr.set_integrater_reindex_operator(best[2].r().inverse().as_hkl(),
reason='unifying [U] setting')
si.set_reflections(intgr.get_integrater_intensities())
# recalculate to verify
u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections())
U = fixed.inverse() * sqr(u).transpose()
Debug.write('New reindex: %s' % (U.inverse() * reference_U))
# FIXME I should probably raise an exception at this stage if this
# is not about I3...
if self.get_scaler_reference_reflection_file():
self._reference = self.get_scaler_reference_reflection_file()
Debug.write('Using HKLREF %s' % self._reference)
elif PhilIndex.params.xia2.settings.scale.reference_reflection_file:
self._reference = PhilIndex.params.xia2.settings.scale.reference_reflection_file
Debug.write('Using HKLREF %s' % self._reference)
params = PhilIndex.params
use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs
if len(self._sweep_handler.get_epochs()) > 1 and use_brehm_diederichs:
brehm_diederichs_files_in = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
brehm_diederichs_files_in.append(hklin)
# now run cctbx.brehm_diederichs to figure out the indexing hand for
# each sweep
from xia2.Wrappers.Cctbx.BrehmDiederichs import BrehmDiederichs
from xia2.lib.bits import auto_logfiler
brehm_diederichs = BrehmDiederichs()
brehm_diederichs.set_working_directory(self.get_working_directory())
auto_logfiler(brehm_diederichs)
brehm_diederichs.set_input_filenames(brehm_diederichs_files_in)
# 1 or 3? 1 seems to work better?
brehm_diederichs.set_asymmetric(1)
brehm_diederichs.run()
reindexing_dict = brehm_diederichs.get_reindexing_dict()
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
reindex_op = reindexing_dict.get(os.path.abspath(hklin))
assert reindex_op is not None
if 1 or reindex_op != 'h,k,l':
# apply the reindexing operator
intgr.set_integrater_reindex_operator(
reindex_op, reason='match reference')
si.set_reflections(intgr.get_integrater_intensities())
elif len(self._sweep_handler.get_epochs()) > 1 and \
not self._reference:
first = self._sweep_handler.get_epochs()[0]
si = self._sweep_handler.get_sweep_information(first)
self._reference = si.get_reflections()
if self._reference:
md = self._factory.Mtzdump()
md.set_hklin(self._reference)
md.dump()
if md.get_batches() and False:
raise RuntimeError('reference reflection file %s unmerged' % \
self._reference)
datasets = md.get_datasets()
if len(datasets) > 1 and False:
raise RuntimeError('more than one dataset in %s' % \
self._reference)
# then get the unit cell, lattice etc.
reference_lattice = Syminfo.get_lattice(md.get_spacegroup())
reference_cell = md.get_dataset_info(datasets[0])['cell']
# then compute the pointgroup from this...
# ---------- REINDEX TO CORRECT (REFERENCE) SETTING ----------
for epoch in self._sweep_handler.get_epochs():
# if we are working with unified UB matrix then this should not
# be a problem here (note, *if*; *should*)
# what about e.g. alternative P1 settings?
# see JIRA MXSW-904
if PhilIndex.params.xia2.settings.unify_setting:
continue
pl = self._factory.Pointless()
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
pl.set_hklin(self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()))
hklout = os.path.join(
self.get_working_directory(),
'%s_rdx2.mtz' % os.path.split(hklin)[-1][:-4])
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
# now set the initial reflection set as a reference...
pl.set_hklref(self._reference)
# https://github.com/xia2/xia2/issues/115 - should ideally iteratively
# construct a reference or a tree of correlations to ensure correct
# reference setting - however if small molecule assume has been
# multi-sweep-indexed so can ignore "fatal errors" - temporary hack
pl.decide_pointgroup(
ignore_errors=PhilIndex.params.xia2.settings.small_molecule)
Debug.write('Reindexing analysis of %s' % pl.get_hklin())
pointgroup = pl.get_pointgroup()
reindex_op = pl.get_reindex_operator()
Debug.write('Operator: %s' % reindex_op)
# apply this...
integrater = si.get_integrater()
integrater.set_integrater_reindex_operator(reindex_op,
reason='match reference')
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
si.set_reflections(integrater.get_integrater_intensities())
md = self._factory.Mtzdump()
md.set_hklin(si.get_reflections())
md.dump()
datasets = md.get_datasets()
if len(datasets) > 1:
raise RuntimeError('more than one dataset in %s' % \
si.get_reflections())
# then get the unit cell, lattice etc.
lattice = Syminfo.get_lattice(md.get_spacegroup())
cell = md.get_dataset_info(datasets[0])['cell']
if lattice != reference_lattice:
raise RuntimeError('lattices differ in %s and %s' % \
(self._reference, si.get_reflections()))
Debug.write('Cell: %.2f %.2f %.2f %.2f %.2f %.2f' % cell)
Debug.write('Ref: %.2f %.2f %.2f %.2f %.2f %.2f' % reference_cell)
for j in range(6):
if math.fabs((cell[j] - reference_cell[j]) /
reference_cell[j]) > 0.1:
raise RuntimeError( \
'unit cell parameters differ in %s and %s' % \
(self._reference, si.get_reflections()))
# ---------- SORT TOGETHER DATA ----------
self._sort_together_data_ccp4()
self._scalr_resolution_limits = {}
# store central resolution limit estimates
batch_ranges = [
self._sweep_handler.get_sweep_information(epoch).get_batch_range()
for epoch in self._sweep_handler.get_epochs()
]
self._resolution_limit_estimates = ersatz_resolution(
self._prepared_reflections, batch_ranges)
def _scale_finish(self):
# compute anomalous signals if anomalous
if self.get_scaler_anomalous():
for key in self._scalr_scaled_refl_files:
f = self._scalr_scaled_refl_files[key]
from iotbx import mtz
m = mtz.object(f)
if m.space_group().is_centric():
Debug.write('Spacegroup is centric: %s' % f)
continue
Debug.write('Running anomalous signal analysis on %s' % f)
a_s = anomalous_signals(f)
self._scalr_statistics[
(self._scalr_pname, self._scalr_xname, key)
]['dF/F'] = [a_s[0]]
self._scalr_statistics[
(self._scalr_pname, self._scalr_xname, key)
]['dI/s(dI)'] = [a_s[1]]
# next transform to F's from I's etc.
if len(self._scalr_scaled_refl_files.keys()) == 0:
raise RuntimeError, 'no reflection files stored'
# run xia2.report on each unmerged mtz file
from iotbx.reflection_file_reader import any_reflection_file
from iotbx import mtz
from cctbx.array_family import flex
for wavelength in self._scalr_scaled_refl_files.keys():
mtz_unmerged = self._scalr_scaled_reflection_files['mtz_unmerged'][wavelength]
reader = any_reflection_file(mtz_unmerged)
mtz_object = reader.file_content()
batches = mtz_object.as_miller_arrays_dict()['HKL_base', 'HKL_base', 'BATCH']
dose = flex.double(batches.size(), -1)
batch_to_dose = self.get_batch_to_dose()
for i, b in enumerate(batches.data()):
dose[i] = batch_to_dose[b]
c = mtz_object.crystals()[0]
d = c.datasets()[0]
d.add_column('DOSE', 'R').set_values(dose.as_float())
tmp_mtz = os.path.join(self.get_working_directory(), 'dose_tmp.mtz')
mtz_object.write(tmp_mtz)
hklin = tmp_mtz
FileHandler.record_temporary_file(hklin)
from xia2.Wrappers.XIA.Report import Report
report = Report()
report.set_working_directory(self.get_working_directory())
report.set_mtz_filename(hklin)
htmlout = os.path.join(
self.get_working_directory(), '%s_%s_%s_report.html' %(
self._scalr_pname, self._scalr_xname, wavelength))
report.set_html_filename(htmlout)
report.set_chef_min_completeness(0.95) # sensible?
from xia2.lib.bits import auto_logfiler
auto_logfiler(report)
try:
report.run()
except Exception, e:
Debug.write('xia2.report failed:')
Debug.write(str(e))
continue
FileHandler.record_html_file(
'%s %s %s report' %(
self._scalr_pname, self._scalr_xname, wavelength), htmlout)
def _sort_together_data_xds_one_sweep(self):
assert(len(self._sweep_information) == 1)
epoch = self._sweep_information.keys()[0]
hklin = self._sweep_information[epoch]['scaled_reflections']
if Flags.get_chef():
self._sweep_information_to_chef()
if self.get_scaler_reference_reflection_file():
md = self._factory.Mtzdump()
md.set_hklin(self.get_scaler_reference_reflection_file())
md.dump()
spacegroups = [md.get_spacegroup()]
reindex_operator = 'h,k,l'
elif self._scalr_input_spacegroup:
Debug.write('Assigning user input spacegroup: %s' % \
self._scalr_input_spacegroup)
spacegroups = [self._scalr_input_spacegroup]
reindex_operator = 'h,k,l'
else:
pointless = self._factory.Pointless()
pointless.set_hklin(hklin)
pointless.decide_spacegroup()
FileHandler.record_log_file('%s %s pointless' % \
(self._scalr_pname,
self._scalr_xname),
pointless.get_log_file())
spacegroups = pointless.get_likely_spacegroups()
reindex_operator = pointless.get_spacegroup_reindex_operator()
self._scalr_likely_spacegroups = spacegroups
spacegroup = self._scalr_likely_spacegroups[0]
self._scalr_reindex_operator = reindex_operator
Chatter.write('Likely spacegroups:')
for spag in self._scalr_likely_spacegroups:
Chatter.write('%s' % spag)
Chatter.write(
'Reindexing to first spacegroup setting: %s (%s)' % \
(spacegroup, clean_reindex_operator(reindex_operator)))
hklout = os.path.join(self.get_working_directory(),
'%s_%s_reindex.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(hklout)
if reindex_operator == '[h,k,l]':
# just assign spacegroup
from iotbx import mtz
from cctbx import sgtbx
s = sgtbx.space_group(sgtbx.space_group_symbols(
str(spacegroup)).hall())
m = mtz.object(hklin)
m.set_space_group(s).write(hklout)
self._scalr_cell = m.crystals()[-1].unit_cell().parameters()
Debug.write(
'Updating unit cell to %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(self._scalr_cell))
del(m)
del(s)
else:
ri = self._factory.Reindex()
ri.set_hklin(hklin)
ri.set_hklout(hklout)
ri.set_spacegroup(spacegroup)
ri.set_operator(reindex_operator)
ri.reindex()
Debug.write(
'Updating unit cell to %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(ri.get_cell()))
self._scalr_cell = tuple(ri.get_cell())
hklin = hklout
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s = self._factory.Sortmtz()
s.set_hklin(hklin)
s.set_hklout(hklout)
s.sort(vrset = -99999999.0)
self._prepared_reflections = hklout
return
def _scale_finish_chunk_6_add_free_r(self):
hklout = os.path.join(
self.get_working_directory(),
'%s_%s_free_temp.mtz' % (self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(hklout)
scale_params = PhilIndex.params.xia2.settings.scale
if self.get_scaler_freer_file():
# e.g. via .xinfo file
freein = self.get_scaler_freer_file()
Debug.write('Copying FreeR_flag from %s' % freein)
c = self._factory.Cad()
c.set_freein(freein)
c.add_hklin(self._scalr_scaled_reflection_files['mtz_merged'])
c.set_hklout(hklout)
c.copyfree()
elif scale_params.freer_file is not None:
# e.g. via -freer_file command line argument
freein = scale_params.freer_file
Debug.write('Copying FreeR_flag from %s' % freein)
c = self._factory.Cad()
c.set_freein(freein)
c.add_hklin(self._scalr_scaled_reflection_files['mtz_merged'])
c.set_hklout(hklout)
c.copyfree()
else:
if scale_params.free_total:
ntot = scale_params.free_total
# need to get a fraction, so...
nref = MtzUtils.nref_from_mtz(hklin)
free_fraction = float(ntot) / float(nref)
else:
free_fraction = scale_params.free_fraction
f = self._factory.Freerflag()
f.set_free_fraction(free_fraction)
f.set_hklin(self._scalr_scaled_reflection_files['mtz_merged'])
f.set_hklout(hklout)
f.add_free_flag()
# then check that this FreeR set is complete
hklin = hklout
hklout = os.path.join(
self.get_working_directory(),
'%s_%s_free.mtz' % (self._scalr_pname, self._scalr_xname))
# default fraction of 0.05
free_fraction = 0.05
if scale_params.free_fraction:
free_fraction = scale_params.free_fraction
elif scale_params.free_total:
ntot = scale_params.free_total()
# need to get a fraction, so...
nref = MtzUtils.nref_from_mtz(hklin)
free_fraction = float(ntot) / float(nref)
f = self._factory.Freerflag()
f.set_free_fraction(free_fraction)
f.set_hklin(hklin)
f.set_hklout(hklout)
f.complete_free_flag()
# remove 'mtz_merged' from the dictionary - this is made
# redundant by the merged free...
del self._scalr_scaled_reflection_files['mtz_merged']
# changed from mtz_merged_free to plain ol' mtz
self._scalr_scaled_reflection_files['mtz'] = hklout
# record this for future reference
FileHandler.record_data_file(hklout)
def _scale_prepare(self):
'''Prepare the data for scaling - this will reindex it the
reflections to the correct pointgroup and setting, for instance,
and move the reflection files to the scale directory.'''
Citations.cite('xds')
Citations.cite('ccp4')
Citations.cite('pointless')
# GATHER phase - get the reflection files together... note that
# it is not necessary in here to keep the batch information as we
# don't wish to rebatch the reflections prior to scaling.
# FIXME need to think about what I will do about the radiation
# damage analysis in here...
self._sweep_information = { }
# FIXME in here I want to record the batch number to
# epoch mapping as per the CCP4 Scaler implementation.
Journal.block(
'gathering', self.get_scaler_xcrystal().get_name(), 'XDS',
{'working directory':self.get_working_directory()})
for epoch in self._scalr_integraters.keys():
intgr = self._scalr_integraters[epoch]
pname, xname, dname = intgr.get_integrater_project_info()
sname = intgr.get_integrater_sweep_name()
self._sweep_information[epoch] = {
'pname':pname,
'xname':xname,
'dname':dname,
'integrater':intgr,
'corrected_intensities':intgr.get_integrater_corrected_intensities(),
'prepared_reflections':None,
'scaled_reflections':None,
'header':intgr.get_header(),
'batches':intgr.get_integrater_batches(),
'image_to_epoch':intgr.get_integrater_sweep(
).get_image_to_epoch(),
'image_to_dose':{},
'batch_offset':0,
'sname':sname
}
Journal.entry({'adding data from':'%s/%s/%s' % \
(xname, dname, sname)})
# what are these used for?
# pname / xname / dname - dataset identifiers
# image to epoch / batch offset / batches - for RD analysis
Debug.write('For EPOCH %s have:' % str(epoch))
Debug.write('ID = %s/%s/%s' % (pname, xname, dname))
Debug.write('SWEEP = %s' % intgr.get_integrater_sweep_name())
# next work through all of the reflection files and make sure that
# they are XDS_ASCII format...
epochs = self._sweep_information.keys()
epochs.sort()
self._first_epoch = min(epochs)
self._scalr_pname = self._sweep_information[epochs[0]]['pname']
self._scalr_xname = self._sweep_information[epochs[0]]['xname']
for epoch in epochs:
intgr = self._scalr_integraters[epoch]
pname = self._sweep_information[epoch]['pname']
xname = self._sweep_information[epoch]['xname']
dname = self._sweep_information[epoch]['dname']
sname = self._sweep_information[epoch]['sname']
if self._scalr_pname != pname:
raise RuntimeError, 'all data must have a common project name'
xname = self._sweep_information[epoch]['xname']
if self._scalr_xname != xname:
raise RuntimeError, \
'all data for scaling must come from one crystal'
xsh = XDSScalerHelper()
xsh.set_working_directory(self.get_working_directory())
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_CORRECTED.HKL' %(
pname, xname, dname, sname))
sweep = intgr.get_integrater_sweep()
if sweep.get_frames_to_process() is not None:
offset = intgr.get_frame_offset()
#print "offset: %d" %offset
start, end = sweep.get_frames_to_process()
start -= offset
end -= offset
#end += 1 ????
#print "limiting batches: %d-%d" %(start, end)
xsh.limit_batches(hklin, hklout, start, end)
self._sweep_information[epoch]['corrected_intensities'] = hklout
# if there is more than one sweep then compare the lattices
# and eliminate all but the lowest symmetry examples if
# there are more than one...
# -------------------------------------------------
# Ensure that the integration lattices are the same
# -------------------------------------------------
need_to_return = False
if len(self._sweep_information.keys()) > 1:
lattices = []
# FIXME run this stuff in parallel as well...
for epoch in self._sweep_information.keys():
intgr = self._sweep_information[epoch]['integrater']
hklin = self._sweep_information[epoch]['corrected_intensities']
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
ntr = False
else:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattice = Syminfo.get_lattice(pointgroup)
if not lattice in lattices:
lattices.append(lattice)
if ntr:
# if we need to return, we should logically reset
# any reindexing operator right? right here all
# we are talking about is the correctness of
# individual pointgroups?? Bug # 3373
reindex_op = 'h,k,l'
# actually, should this not be done "by magic"
# when a new pointgroup is assigned in the
# pointless indexer jiffy above?!
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
need_to_return = True
# bug # 2433 - need to ensure that all of the lattice
# conclusions were the same...
if len(lattices) > 1:
ordered_lattices = []
for l in lattices_in_order():
if l in lattices:
ordered_lattices.append(l)
correct_lattice = ordered_lattices[0]
Debug.write('Correct lattice asserted to be %s' % \
correct_lattice)
# transfer this information back to the indexers
for epoch in self._sweep_information.keys():
integrater = self._sweep_information[
epoch]['integrater']
refiner = integrater.get_integrater_refiner()
sname = integrater.get_integrater_sweep_name()
if not refiner:
continue
state = refiner.set_refiner_asserted_lattice(
correct_lattice)
if state == refiner.LATTICE_CORRECT:
Debug.write('Lattice %s ok for sweep %s' % \
(correct_lattice, sname))
elif state == refiner.LATTICE_IMPOSSIBLE:
raise RuntimeError, 'Lattice %s impossible for %s' % \
(correct_lattice, sname)
elif state == refiner.LATTICE_POSSIBLE:
Debug.write('Lattice %s assigned for sweep %s' % \
(correct_lattice, sname))
need_to_return = True
# if one or more of them was not in the lowest lattice,
# need to return here to allow reprocessing
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# next if there is more than one sweep then generate
# a merged reference reflection file to check that the
# setting for all reflection files is the same...
# if we get to here then all data was processed with the same
# lattice
# ----------------------------------------------------------
# next ensure that all sweeps are set in the correct setting
# ----------------------------------------------------------
if self.get_scaler_reference_reflection_file():
self._reference = self.get_scaler_reference_reflection_file()
Debug.write('Using HKLREF %s' % self._reference)
md = self._factory.Mtzdump()
md.set_hklin(self.get_scaler_reference_reflection_file())
md.dump()
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(
md.get_spacegroup())
Debug.write('Spacegroup %d' % self._xds_spacegroup)
elif PhilIndex.params.xia2.settings.scale.reference_reflection_file:
self._reference = PhilIndex.params.xia2.settings.scale.reference_reflection_file
Debug.write('Using HKLREF %s' % self._reference)
md = self._factory.Mtzdump()
md.set_hklin(PhilIndex.params.xia2.settings.scale.reference_reflection_file)
md.dump()
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(
md.get_spacegroup())
Debug.write('Spacegroup %d' % self._xds_spacegroup)
params = PhilIndex.params
use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs
if len(self._sweep_information.keys()) > 1 and use_brehm_diederichs:
brehm_diederichs_files_in = []
for epoch in self._sweep_information.keys():
intgr = self._sweep_information[epoch]['integrater']
hklin = self._sweep_information[epoch]['corrected_intensities']
refiner = intgr.get_integrater_refiner()
# in here need to consider what to do if the user has
# assigned the pointgroup on the command line ...
if not self._scalr_input_pointgroup:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
if ntr:
# Bug # 3373
Debug.write('Reindex to standard (PIJ): %s' % \
reindex_op)
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
else:
# 27/FEB/08 to support user assignment of pointgroups
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
self._sweep_information[epoch]['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# convert the XDS_ASCII for this sweep to mtz - on the next
# get this should be in the correct setting...
dname = self._sweep_information[epoch]['dname']
sname = intgr.get_integrater_sweep_name()
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.mtz' % (dname, sname))
FileHandler.record_temporary_file(hklout)
# now use pointless to make this conversion
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
brehm_diederichs_files_in.append(hklout)
# now run cctbx.brehm_diederichs to figure out the indexing hand for
# each sweep
from xia2.Wrappers.Cctbx.BrehmDiederichs import BrehmDiederichs
brehm_diederichs = BrehmDiederichs()
brehm_diederichs.set_working_directory(self.get_working_directory())
auto_logfiler(brehm_diederichs)
brehm_diederichs.set_input_filenames(brehm_diederichs_files_in)
# 1 or 3? 1 seems to work better?
brehm_diederichs.set_asymmetric(1)
brehm_diederichs.run()
reindexing_dict = brehm_diederichs.get_reindexing_dict()
for epoch in self._sweep_information.keys():
intgr = self._sweep_information[epoch]['integrater']
dname = self._sweep_information[epoch]['dname']
sname = intgr.get_integrater_sweep_name()
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.mtz' % (dname, sname))
# apply the reindexing operator
intgr.set_integrater_reindex_operator(reindex_op)
# and copy the reflection file to the local directory
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
self._sweep_information[epoch][
'prepared_reflections'] = os.path.split(hklout)[-1]
elif len(self._sweep_information.keys()) > 1 and \
not self._reference:
# need to generate a reference reflection file - generate this
# from the reflections in self._first_epoch
#
# FIXME this should really use the Brehm and Diederichs method
# if you have lots of little sweeps...
intgr = self._sweep_information[self._first_epoch]['integrater']
hklin = self._sweep_information[epoch]['corrected_intensities']
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
ntr = False
reindex_op = 'h,k,l'
else:
pointgroup, reindex_op, ntr = self._pointless_indexer_jiffy(
hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
reference_reindex_op = intgr.get_integrater_reindex_operator()
if ntr:
# Bug # 3373
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(pointgroup)
# next pass this reindexing operator back to the source
# of the reflections
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
self._sweep_information[epoch]['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'xds-pointgroup-reference-unsorted.mtz')
FileHandler.record_temporary_file(hklout)
# now use pointless to handle this conversion
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
self._reference = hklout
if self._reference:
from xia2.Driver.DriverFactory import DriverFactory
def run_one_sweep(args):
sweep_information = args[0]
pointless_indexer_jiffy = args[1]
factory = args[2]
job_type = args[3]
if job_type:
DriverFactory.set_driver_type(job_type)
intgr = sweep_information['integrater']
hklin = sweep_information['corrected_intensities']
refiner = intgr.get_integrater_refiner()
# in here need to consider what to do if the user has
# assigned the pointgroup on the command line ...
if not self._scalr_input_pointgroup:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
if ntr:
# Bug # 3373
Debug.write('Reindex to standard (PIJ): %s' % \
reindex_op)
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
else:
# 27/FEB/08 to support user assignment of pointgroups
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
sweep_information['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# convert the XDS_ASCII for this sweep to mtz - on the next
# get this should be in the correct setting...
hklin = sweep_information['corrected_intensities']
# now use pointless to make this conversion
# try with no conversion?!
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
hklout = os.path.join(
self.get_working_directory(),
'%d_xds-pointgroup-unsorted.mtz' %pointless.get_xpid())
FileHandler.record_temporary_file(hklout)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
pointless = self._factory.Pointless()
pointless.set_hklin(hklout)
pointless.set_hklref(self._reference)
pointless.decide_pointgroup()
pointgroup = pointless.get_pointgroup()
reindex_op = pointless.get_reindex_operator()
# for debugging print out the reindexing operations and
# what have you...
Debug.write('Reindex to standard: %s' % reindex_op)
# this should send back enough information that this
# is in the correct pointgroup (from the call above) and
# also in the correct setting, from the interaction
# with the reference set... - though I guess that the
# spacegroup number should not have changed, right?
# set the reindex operation afterwards... though if the
# spacegroup number is the same this should make no
# difference, right?!
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
intgr.set_integrater_reindex_operator(reindex_op)
sweep_information['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# and copy the reflection file to the local directory
dname = sweep_information['dname']
sname = intgr.get_integrater_sweep_name()
hklin = sweep_information['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
sweep_information['prepared_reflections'] = os.path.split(hklout)[-1]
return sweep_information
from libtbx import easy_mp
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
if njob > 1:
# cache drivertype
drivertype = DriverFactory.get_driver_type()
args = [
(self._sweep_information[epoch], self._pointless_indexer_jiffy,
self._factory, mp_params.type)
for epoch in self._sweep_information.keys()]
results_list = easy_mp.parallel_map(
run_one_sweep, args, params=None,
processes=njob,
method="threading",
asynchronous=True,
callback=None,
preserve_order=True,
preserve_exception_message=True)
# restore drivertype
DriverFactory.set_driver_type(drivertype)
# results should be given back in the same order
for i, epoch in enumerate(self._sweep_information.keys()):
self._sweep_information[epoch] = results_list[i]
else:
for epoch in self._sweep_information.keys():
self._sweep_information[epoch] = run_one_sweep(
(self._sweep_information[epoch], self._pointless_indexer_jiffy,
self._factory, None))
else:
# convert the XDS_ASCII for this sweep to mtz
epoch = self._first_epoch
intgr = self._sweep_information[epoch]['integrater']
refiner = intgr.get_integrater_refiner()
sname = intgr.get_integrater_sweep_name()
hklout = os.path.join(self.get_working_directory(),
'%s-pointless.mtz' % sname)
FileHandler.record_temporary_file(hklout)
pointless = self._factory.Pointless()
pointless.set_xdsin(self._sweep_information[epoch]['corrected_intensities'])
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
# run it through pointless interacting with the
# Indexer which belongs to this sweep
hklin = hklout
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
ntr = False
reindex_op = 'h,k,l'
else:
pointgroup, reindex_op, ntr = self._pointless_indexer_jiffy(
hklin, refiner)
if ntr:
# if we need to return, we should logically reset
# any reindexing operator right? right here all
# we are talking about is the correctness of
# individual pointgroups?? Bug # 3373
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
need_to_return = True
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(pointgroup)
# next pass this reindexing operator back to the source
# of the reflections
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
self._sweep_information[epoch]['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
hklin = self._sweep_information[epoch]['corrected_intensities']
dname = self._sweep_information[epoch]['dname']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
# and copy the reflection file to the local
# directory
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
self._sweep_information[epoch][
'prepared_reflections'] = os.path.split(hklout)[-1]
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
unit_cell_list = []
for epoch in self._sweep_information.keys():
integrater = self._sweep_information[epoch]['integrater']
cell = integrater.get_integrater_cell()
n_ref = integrater.get_integrater_n_ref()
Debug.write('Cell for %s: %.2f %.2f %.2f %.2f %.2f %.2f' % \
(integrater.get_integrater_sweep_name(),
cell[0], cell[1], cell[2],
cell[3], cell[4], cell[5]))
Debug.write('=> %d reflections' % n_ref)
unit_cell_list.append((cell, n_ref))
self._scalr_cell = compute_average_unit_cell(unit_cell_list)
self._scalr_resolution_limits = { }
Debug.write('Determined unit cell: %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(self._scalr_cell))
if os.path.exists(os.path.join(
self.get_working_directory(),
'REMOVE.HKL')):
os.remove(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'))
Debug.write('Deleting REMOVE.HKL at end of scale prepare.')
return
def run_one_sweep(args):
sweep_information = args[0]
pointless_indexer_jiffy = args[1]
factory = args[2]
job_type = args[3]
if job_type:
DriverFactory.set_driver_type(job_type)
intgr = sweep_information['integrater']
hklin = sweep_information['corrected_intensities']
refiner = intgr.get_integrater_refiner()
# in here need to consider what to do if the user has
# assigned the pointgroup on the command line ...
if not self._scalr_input_pointgroup:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
if ntr:
# Bug # 3373
Debug.write('Reindex to standard (PIJ): %s' % \
reindex_op)
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
else:
# 27/FEB/08 to support user assignment of pointgroups
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
sweep_information['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# convert the XDS_ASCII for this sweep to mtz - on the next
# get this should be in the correct setting...
hklin = sweep_information['corrected_intensities']
# now use pointless to make this conversion
# try with no conversion?!
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
hklout = os.path.join(
self.get_working_directory(),
'%d_xds-pointgroup-unsorted.mtz' %pointless.get_xpid())
FileHandler.record_temporary_file(hklout)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
pointless = self._factory.Pointless()
pointless.set_hklin(hklout)
pointless.set_hklref(self._reference)
pointless.decide_pointgroup()
pointgroup = pointless.get_pointgroup()
reindex_op = pointless.get_reindex_operator()
# for debugging print out the reindexing operations and
# what have you...
Debug.write('Reindex to standard: %s' % reindex_op)
# this should send back enough information that this
# is in the correct pointgroup (from the call above) and
# also in the correct setting, from the interaction
# with the reference set... - though I guess that the
# spacegroup number should not have changed, right?
# set the reindex operation afterwards... though if the
# spacegroup number is the same this should make no
# difference, right?!
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
intgr.set_integrater_reindex_operator(reindex_op)
sweep_information['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# and copy the reflection file to the local directory
dname = sweep_information['dname']
sname = intgr.get_integrater_sweep_name()
hklin = sweep_information['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
sweep_information['prepared_reflections'] = os.path.split(hklout)[-1]
return sweep_information
def _scale_prepare(self):
'''Perform all of the preparation required to deliver the scaled
data. This should sort together the reflection files, ensure that
they are correctly indexed (via pointless) and generally tidy
things up.'''
# acknowledge all of the programs we are about to use...
Citations.cite('pointless')
Citations.cite('aimless')
Citations.cite('ccp4')
# ---------- GATHER ----------
self._sweep_handler = SweepInformationHandler(self._scalr_integraters)
Journal.block(
'gathering', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'working directory':self.get_working_directory()})
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
exclude_sweep = False
for sweep in PhilIndex.params.xia2.settings.sweep:
if sweep.id == sname and sweep.exclude:
exclude_sweep = True
break
if exclude_sweep:
self._sweep_handler.remove_epoch(epoch)
Debug.write('Excluding sweep %s' %sname)
else:
Journal.entry({'adding data from':'%s/%s/%s' % \
(xname, dname, sname)})
# gather data for all images which belonged to the parent
# crystal - allowing for the fact that things could go wrong
# e.g. epoch information not available, exposure times not in
# headers etc...
for e in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(e)
assert is_mtz_file(si.get_reflections())
p, x = self._sweep_handler.get_project_info()
self._scalr_pname = p
self._scalr_xname = x
# verify that the lattices are consistent, calling eliminate if
# they are not N.B. there could be corner cases here
need_to_return = False
multi_sweep_indexing = \
PhilIndex.params.xia2.settings.developmental.multi_sweep_indexing
if len(self._sweep_handler.get_epochs()) > 1:
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
md = self._factory.Mtzdump()
md.set_hklin(hklin)
md.dump()
batches = md.get_batches()
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
datasets = md.get_datasets()
Debug.write('In reflection file %s found:' % hklin)
for d in datasets:
Debug.write('... %s' % d)
dataset_info = md.get_dataset_info(datasets[0])
from xia2.lib.bits import nifty_power_of_ten
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
rb = self._factory.Rebatch()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_prepointless.mtz' % \
(pname, xname, dname, si.get_sweep_name()))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
rb.set_hklin(hklin)
rb.set_first_batch(counter * max_batches + 1)
rb.set_project_info(pname, xname, dname)
rb.set_hklout(hklout)
new_batches = rb.rebatch()
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattices = [Syminfo.get_lattice(pointgroup)]
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
else:
lattices = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
ntr = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattice = Syminfo.get_lattice(pointgroup)
if not lattice in lattices:
lattices.append(lattice)
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
if len(lattices) > 1:
# why not using pointless indexer jiffy??!
correct_lattice = sort_lattices(lattices)[0]
Chatter.write('Correct lattice asserted to be %s' % \
correct_lattice)
# transfer this information back to the indexers
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
refiner = si.get_integrater().get_integrater_refiner()
sname = si.get_sweep_name()
state = refiner.set_refiner_asserted_lattice(
correct_lattice)
if state == refiner.LATTICE_CORRECT:
Chatter.write('Lattice %s ok for sweep %s' % \
(correct_lattice, sname))
elif state == refiner.LATTICE_IMPOSSIBLE:
raise RuntimeError, 'Lattice %s impossible for %s' \
% (correct_lattice, sname)
elif state == refiner.LATTICE_POSSIBLE:
Chatter.write('Lattice %s assigned for sweep %s' % \
(correct_lattice, sname))
need_to_return = True
# if one or more of them was not in the lowest lattice,
# need to return here to allow reprocessing
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# ---------- REINDEX ALL DATA TO CORRECT POINTGROUP ----------
# all should share the same pointgroup, unless twinned... in which
# case force them to be...
pointgroups = { }
reindex_ops = { }
probably_twinned = False
need_to_return = False
multi_sweep_indexing = \
PhilIndex.params.xia2.settings.developmental.multi_sweep_indexing
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
md = self._factory.Mtzdump()
md.set_hklin(hklin)
md.dump()
batches = md.get_batches()
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
datasets = md.get_datasets()
Debug.write('In reflection file %s found:' % hklin)
for d in datasets:
Debug.write('... %s' % d)
dataset_info = md.get_dataset_info(datasets[0])
from xia2.lib.bits import nifty_power_of_ten
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
rb = self._factory.Rebatch()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_prepointless.mtz' % \
(pname, xname, dname, si.get_sweep_name()))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
rb.set_hklin(hklin)
rb.set_first_batch(counter * max_batches + 1)
rb.set_project_info(pname, xname, dname)
rb.set_hklout(hklout)
new_batches = rb.rebatch()
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
for epoch in self._sweep_handler.get_epochs():
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
else:
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
#hklout = os.path.join(
#self.get_working_directory(),
#os.path.split(hklin)[-1].replace('.mtz', '_rdx.mtz'))
#FileHandler.record_temporary_file(hklout)
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
pt = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
if ntr:
integrater.integrater_reset_reindex_operator()
need_to_return = True
if pt and not probably_twinned:
probably_twinned = True
Debug.write('Pointgroup: %s (%s)' % (pointgroup, reindex_op))
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
overall_pointgroup = None
pointgroup_set = set([pointgroups[e] for e in pointgroups])
if len(pointgroup_set) > 1 and \
not probably_twinned:
raise RuntimeError, 'non uniform pointgroups'
if len(pointgroup_set) > 1:
Debug.write('Probably twinned, pointgroups: %s' % \
' '.join([p.replace(' ', '') for p in \
list(pointgroup_set)]))
numbers = [Syminfo.spacegroup_name_to_number(s) for s in \
pointgroup_set]
overall_pointgroup = Syminfo.spacegroup_number_to_name(
min(numbers))
self._scalr_input_pointgroup = overall_pointgroup
Chatter.write('Twinning detected, assume pointgroup %s' % \
overall_pointgroup)
need_to_return = True
else:
overall_pointgroup = pointgroup_set.pop()
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
integrater = si.get_integrater()
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(overall_pointgroup))
integrater.set_integrater_reindex_operator(
reindex_ops[epoch], reason='setting point group')
# This will give us the reflections in the correct point group
si.set_reflections(integrater.get_integrater_intensities())
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# in here now optinally work through the data files which should be
# indexed with a consistent point group, and transform the orientation
# matrices by the lattice symmetry operations (if possible) to get a
# consistent definition of U matrix modulo fixed rotations
if PhilIndex.params.xia2.settings.unify_setting:
from scitbx.matrix import sqr
reference_U = None
i3 = sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
fixed = sqr(intgr.get_goniometer().get_fixed_rotation())
u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections())
U = fixed.inverse() * sqr(u).transpose()
B = sqr(b)
if reference_U is None:
reference_U = U
continue
results = []
for op in s.all_ops():
R = B * sqr(op.r().as_double()).transpose() * B.inverse()
nearly_i3 = (U * R).inverse() * reference_U
score = sum([abs(_n - _i) for (_n, _i) in zip(nearly_i3, i3)])
results.append((score, op.r().as_hkl(), op))
results.sort()
best = results[0]
Debug.write('Best reindex: %s %.3f' % (best[1], best[0]))
intgr.set_integrater_reindex_operator(best[2].r().inverse().as_hkl(),
reason='unifying [U] setting')
si.set_reflections(intgr.get_integrater_intensities())
# recalculate to verify
u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections())
U = fixed.inverse() * sqr(u).transpose()
Debug.write('New reindex: %s' % (U.inverse() * reference_U))
# FIXME I should probably raise an exception at this stage if this
# is not about I3...
if self.get_scaler_reference_reflection_file():
self._reference = self.get_scaler_reference_reflection_file()
Debug.write('Using HKLREF %s' % self._reference)
elif Flags.get_reference_reflection_file():
self._reference = Flags.get_reference_reflection_file()
Debug.write('Using HKLREF %s' % self._reference)
params = PhilIndex.params
use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs
if len(self._sweep_handler.get_epochs()) > 1 and use_brehm_diederichs:
brehm_diederichs_files_in = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
brehm_diederichs_files_in.append(hklin)
# now run cctbx.brehm_diederichs to figure out the indexing hand for
# each sweep
from xia2.Wrappers.Cctbx.BrehmDiederichs import BrehmDiederichs
from xia2.lib.bits import auto_logfiler
brehm_diederichs = BrehmDiederichs()
brehm_diederichs.set_working_directory(self.get_working_directory())
auto_logfiler(brehm_diederichs)
brehm_diederichs.set_input_filenames(brehm_diederichs_files_in)
# 1 or 3? 1 seems to work better?
brehm_diederichs.set_asymmetric(1)
brehm_diederichs.run()
reindexing_dict = brehm_diederichs.get_reindexing_dict()
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
reindex_op = reindexing_dict.get(os.path.abspath(hklin))
assert reindex_op is not None
if 1 or reindex_op != 'h,k,l':
# apply the reindexing operator
intgr.set_integrater_reindex_operator(
reindex_op, reason='match reference')
si.set_reflections(intgr.get_integrater_intensities())
elif len(self._sweep_handler.get_epochs()) > 1 and \
not self._reference:
first = self._sweep_handler.get_epochs()[0]
si = self._sweep_handler.get_sweep_information(first)
self._reference = si.get_reflections()
if self._reference:
md = self._factory.Mtzdump()
md.set_hklin(self._reference)
md.dump()
if md.get_batches() and False:
raise RuntimeError, 'reference reflection file %s unmerged' % \
self._reference
datasets = md.get_datasets()
if len(datasets) > 1 and False:
raise RuntimeError, 'more than one dataset in %s' % \
self._reference
# then get the unit cell, lattice etc.
reference_lattice = Syminfo.get_lattice(md.get_spacegroup())
reference_cell = md.get_dataset_info(datasets[0])['cell']
# then compute the pointgroup from this...
# ---------- REINDEX TO CORRECT (REFERENCE) SETTING ----------
for epoch in self._sweep_handler.get_epochs():
pl = self._factory.Pointless()
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
pl.set_hklin(self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()))
hklout = os.path.join(
self.get_working_directory(),
'%s_rdx2.mtz' % os.path.split(hklin)[-1][:-4])
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
# now set the initial reflection set as a reference...
pl.set_hklref(self._reference)
# write a pointless log file...
pl.decide_pointgroup()
Debug.write('Reindexing analysis of %s' % pl.get_hklin())
pointgroup = pl.get_pointgroup()
reindex_op = pl.get_reindex_operator()
Debug.write('Operator: %s' % reindex_op)
# apply this...
integrater = si.get_integrater()
integrater.set_integrater_reindex_operator(reindex_op,
reason='match reference')
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
si.set_reflections(integrater.get_integrater_intensities())
md = self._factory.Mtzdump()
md.set_hklin(si.get_reflections())
md.dump()
datasets = md.get_datasets()
if len(datasets) > 1:
raise RuntimeError, 'more than one dataset in %s' % \
si.get_reflections()
# then get the unit cell, lattice etc.
lattice = Syminfo.get_lattice(md.get_spacegroup())
cell = md.get_dataset_info(datasets[0])['cell']
if lattice != reference_lattice:
raise RuntimeError, 'lattices differ in %s and %s' % \
(self._reference, si.get_reflections())
for j in range(6):
if math.fabs((cell[j] - reference_cell[j]) /
reference_cell[j]) > 0.1:
raise RuntimeError, \
'unit cell parameters differ in %s and %s' % \
(self._reference, si.get_reflections())
# ---------- SORT TOGETHER DATA ----------
self._sort_together_data_ccp4()
self._scalr_resolution_limits = { }
# store central resolution limit estimates
batch_ranges = [self._sweep_handler.get_sweep_information(
epoch).get_batch_range() for epoch in
self._sweep_handler.get_epochs()]
self._resolution_limit_estimates = erzatz_resolution(
self._prepared_reflections, batch_ranges)
return
def _sort_together_data_xds(self):
if len(self._sweep_information) == 1:
return self._sort_together_data_xds_one_sweep()
max_batches = 0
for epoch in self._sweep_information.keys():
hklin = self._sweep_information[epoch]['scaled_reflections']
if self._sweep_information[epoch]['batches'] == [0, 0]:
Chatter.write('Getting batches from %s' % hklin)
batches = MtzUtils.batches_from_mtz(hklin)
self._sweep_information[epoch]['batches'] = [
min(batches), max(batches)
]
Chatter.write('=> %d to %d' % (min(batches), max(batches)))
batches = self._sweep_information[epoch]['batches']
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
epochs = sorted(self._sweep_information.keys())
counter = 0
for epoch in epochs:
hklin = self._sweep_information[epoch]['scaled_reflections']
pname = self._sweep_information[epoch]['pname']
xname = self._sweep_information[epoch]['xname']
dname = self._sweep_information[epoch]['dname']
sname = self._sweep_information[epoch]['sname']
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%d.mtz' % \
(pname, xname, dname, counter))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
# record this for future reference - will be needed in the
# radiation damage analysis...
# hack - reset this as it gets in a muddle...
intgr = self._sweep_information[epoch]['integrater']
self._sweep_information[epoch][
'batches'] = intgr.get_integrater_batches()
first_batch = min(self._sweep_information[epoch]['batches'])
offset = counter * max_batches - first_batch + 1
self._sweep_information[epoch]['batch_offset'] = offset
from xia2.Modules.Scaler.rebatch import rebatch
new_batches = rebatch(hklin,
hklout,
add_batch=offset,
pname=pname,
xname=xname,
dname=dname)
# update the "input information"
self._sweep_information[epoch]['hklin'] = hklout
self._sweep_information[epoch]['batches'] = new_batches
# update the counter & recycle
counter += 1
s = self._factory.Sortmtz()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(hklout)
for epoch in epochs:
s.add_hklin(self._sweep_information[epoch]['hklin'])
s.sort(vrset=-99999999.0)
self._prepared_reflections = hklout
if self.get_scaler_reference_reflection_file():
spacegroups = [
MtzUtils.space_group_name_from_mtz(
self.get_scaler_reference_reflection_file())
]
reindex_operator = 'h,k,l'
else:
pointless = self._factory.Pointless()
pointless.set_hklin(hklout)
pointless.decide_spacegroup()
FileHandler.record_log_file('%s %s pointless' % \
(self._scalr_pname,
self._scalr_xname),
pointless.get_log_file())
spacegroups = pointless.get_likely_spacegroups()
reindex_operator = pointless.get_spacegroup_reindex_operator()
if self._scalr_input_spacegroup:
Debug.write('Assigning user input spacegroup: %s' % \
self._scalr_input_spacegroup)
spacegroups = [self._scalr_input_spacegroup]
reindex_operator = 'h,k,l'
self._scalr_likely_spacegroups = spacegroups
spacegroup = self._scalr_likely_spacegroups[0]
self._scalr_reindex_operator = reindex_operator
Chatter.write('Likely spacegroups:')
for spag in self._scalr_likely_spacegroups:
Chatter.write('%s' % spag)
Chatter.write(
'Reindexing to first spacegroup setting: %s (%s)' % \
(spacegroup, clean_reindex_operator(reindex_operator)))
hklin = self._prepared_reflections
hklout = os.path.join(self.get_working_directory(),
'%s_%s_reindex.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(hklout)
ri = self._factory.Reindex()
ri.set_hklin(hklin)
ri.set_hklout(hklout)
ri.set_spacegroup(spacegroup)
ri.set_operator(reindex_operator)
ri.reindex()
hklin = hklout
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s = self._factory.Sortmtz()
s.set_hklin(hklin)
s.set_hklout(hklout)
s.sort(vrset=-99999999.0)
self._prepared_reflections = hklout
Debug.write(
'Updating unit cell to %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(ri.get_cell()))
self._scalr_cell = tuple(ri.get_cell())
return
def _scale(self):
"Perform all of the operations required to deliver the scaled data."
epochs = self._sweep_handler.get_epochs()
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_chef_unmerged(True)
sc.set_new_scales_file("%s.scales" % self._scalr_xname)
user_resolution_limits = {}
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
intgr = si.get_integrater()
if intgr.get_integrater_user_resolution():
dmin = intgr.get_integrater_high_resolution()
if (dname, sname) not in user_resolution_limits:
user_resolution_limits[(dname, sname)] = dmin
elif dmin < user_resolution_limits[(dname, sname)]:
user_resolution_limits[(dname, sname)] = dmin
start, end = si.get_batch_range()
if (dname, sname) in self._scalr_resolution_limits:
resolution, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude=False, resolution=resolution, name=sname)
else:
sc.add_run(start, end, name=sname)
sc.set_hklout(
os.path.join(
self.get_working_directory(),
f"{self._scalr_pname}_{self._scalr_xname}_scaled_test.mtz",
)
)
if self.get_scaler_anomalous():
sc.set_anomalous()
# what follows, sucks
failover = PhilIndex.params.xia2.settings.failover
if failover:
try:
sc.scale()
except RuntimeError as e:
es = str(e)
if (
"bad batch" in es
or "negative scales run" in es
or "no observations" in es
):
# first ID the sweep from the batch no
batch = int(es.split()[-1])
epoch = self._identify_sweep_epoch(batch)
sweep = self._scalr_integraters[epoch].get_integrater_sweep()
# then remove it from my parent xcrystal
self.get_scaler_xcrystal().remove_sweep(sweep)
# then remove it from the scaler list of intergraters
# - this should really be a scaler interface method
del self._scalr_integraters[epoch]
# then tell the user what is happening
logger.info(
"Sweep %s gave negative scales - removing", sweep.get_name()
)
# then reset the prepare, do, finish flags
self.set_scaler_prepare_done(False)
self.set_scaler_done(False)
self.set_scaler_finish_done(False)
# and return
return
else:
raise e
else:
sc.scale()
# then gather up all of the resulting reflection files
# and convert them into the required formats (.sca, .mtz.)
loggraph = sc.parse_ccp4_loggraph()
resolution_info = {}
reflection_files = sc.get_scaled_reflection_files()
for dataset in reflection_files:
FileHandler.record_temporary_file(reflection_files[dataset])
for key in loggraph:
if "Analysis against resolution" in key:
dataset = key.split(",")[-1].strip()
resolution_info[dataset] = transpose_loggraph(loggraph[key])
# check in here that there is actually some data to scale..!
if not resolution_info:
raise RuntimeError("no resolution info")
highest_suggested_resolution = self.assess_resolution_limits(
sc.get_unmerged_reflection_file(), user_resolution_limits
)
if not self.get_scaler_done():
logger.debug("Returning as scaling not finished...")
return
batch_info = {}
for key in loggraph:
if "Analysis against Batch" in key:
dataset = key.split(",")[-1].strip()
batch_info[dataset] = transpose_loggraph(loggraph[key])
sc = self._updated_aimless()
FileHandler.record_log_file(
f"{self._scalr_pname} {self._scalr_xname} aimless", sc.get_log_file()
)
sc.set_hklin(self._prepared_reflections)
sc.set_new_scales_file("%s_final.scales" % self._scalr_xname)
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(
start, end, exclude=False, resolution=resolution_limit, name=xname
)
sc.set_hklout(
os.path.join(
self.get_working_directory(),
f"{self._scalr_pname}_{self._scalr_xname}_scaled.mtz",
)
)
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
FileHandler.record_xml_file(
f"{self._scalr_pname} {self._scalr_xname} aimless", sc.get_xmlout()
)
data = sc.get_summary()
scales_file = sc.get_new_scales_file()
loggraph = sc.parse_ccp4_loggraph()
standard_deviation_info = {}
for key in loggraph:
if "standard deviation v. Intensity" in key:
dataset = key.split(",")[-1].strip()
standard_deviation_info[dataset] = transpose_loggraph(loggraph[key])
resolution_info = {}
for key in loggraph:
if "Analysis against resolution" in key:
dataset = key.split(",")[-1].strip()
resolution_info[dataset] = transpose_loggraph(loggraph[key])
batch_info = {}
for key in loggraph:
if "Analysis against Batch" in key:
dataset = key.split(",")[-1].strip()
batch_info[dataset] = transpose_loggraph(loggraph[key])
# finally put all of the results "somewhere useful"
self._scalr_statistics = data
self._scalr_scaled_refl_files = copy.deepcopy(sc.get_scaled_reflection_files())
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_scales_file(scales_file)
self._wavelengths_in_order = []
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(
start, end, exclude=False, resolution=resolution_limit, name=sname
)
if dname not in self._wavelengths_in_order:
self._wavelengths_in_order.append(dname)
sc.set_hklout(
os.path.join(
self.get_working_directory(),
f"{self._scalr_pname}_{self._scalr_xname}_scaled.mtz",
)
)
sc.set_scalepack()
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
self._update_scaled_unit_cell()
self._scalr_scaled_reflection_files = {}
self._scalr_scaled_reflection_files["sca"] = {}
self._scalr_scaled_reflection_files["sca_unmerged"] = {}
self._scalr_scaled_reflection_files["mtz_unmerged"] = {}
for key in self._scalr_scaled_refl_files:
hklout = self._scalr_scaled_refl_files[key]
scaout = "%s.sca" % hklout[:-4]
self._scalr_scaled_reflection_files["sca"][key] = scaout
FileHandler.record_data_file(scaout)
scalepack = os.path.join(
os.path.split(hklout)[0],
os.path.split(hklout)[1]
.replace("_scaled", "_scaled_unmerged")
.replace(".mtz", ".sca"),
)
self._scalr_scaled_reflection_files["sca_unmerged"][key] = scalepack
FileHandler.record_data_file(scalepack)
mtz_unmerged = os.path.splitext(scalepack)[0] + ".mtz"
self._scalr_scaled_reflection_files["mtz_unmerged"][key] = mtz_unmerged
FileHandler.record_data_file(mtz_unmerged)
if self._scalr_cell_esd is not None:
# patch .mtz and overwrite unit cell information
import xia2.Modules.Scaler.tools as tools
override_cell = self._scalr_cell_dict.get(
f"{self._scalr_pname}_{self._scalr_xname}_{key}"
)[0]
tools.patch_mtz_unit_cell(mtz_unmerged, override_cell)
tools.patch_mtz_unit_cell(hklout, override_cell)
self._scalr_scaled_reflection_files["mtz_unmerged"][key] = mtz_unmerged
FileHandler.record_data_file(mtz_unmerged)
if PhilIndex.params.xia2.settings.merging_statistics.source == "cctbx":
for key in self._scalr_scaled_refl_files:
stats = self._compute_scaler_statistics(
self._scalr_scaled_reflection_files["mtz_unmerged"][key],
selected_band=(highest_suggested_resolution, None),
wave=key,
)
self._scalr_statistics[
(self._scalr_pname, self._scalr_xname, key)
] = stats
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_scales_file(scales_file)
self._wavelengths_in_order = []
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
start, end = si.get_batch_range()
resolution_limit, _ = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(
start, end, exclude=False, resolution=resolution_limit, name=sname
)
if dname not in self._wavelengths_in_order:
self._wavelengths_in_order.append(dname)
sc.set_hklout(
os.path.join(
self.get_working_directory(),
f"{self._scalr_pname}_{self._scalr_xname}_chef.mtz",
)
)
sc.set_chef_unmerged(True)
if self.get_scaler_anomalous():
sc.set_anomalous()
sc.scale()
if not PhilIndex.params.dials.fast_mode:
try:
self._generate_absorption_map(sc)
except Exception as e:
# Map generation may fail for number of reasons, eg. matplotlib borken
logger.debug("Could not generate absorption map (%s)", e)
def _sort_together_data_xds(self):
if len(self._sweep_information) == 1:
return self._sort_together_data_xds_one_sweep()
max_batches = 0
for epoch in self._sweep_information.keys():
hklin = self._sweep_information[epoch]['scaled_reflections']
md = self._factory.Mtzdump()
md.set_hklin(hklin)
md.dump()
if self._sweep_information[epoch]['batches'] == [0, 0]:
Chatter.write('Getting batches from %s' % hklin)
batches = md.get_batches()
self._sweep_information[epoch]['batches'] = [min(batches),
max(batches)]
Chatter.write('=> %d to %d' % (min(batches),
max(batches)))
batches = self._sweep_information[epoch]['batches']
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
datasets = md.get_datasets()
Debug.write('In reflection file %s found:' % hklin)
for d in datasets:
Debug.write('... %s' % d)
dataset_info = md.get_dataset_info(datasets[0])
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
epochs = self._sweep_information.keys()
epochs.sort()
counter = 0
for epoch in epochs:
rb = self._factory.Rebatch()
hklin = self._sweep_information[epoch]['scaled_reflections']
pname = self._sweep_information[epoch]['pname']
xname = self._sweep_information[epoch]['xname']
dname = self._sweep_information[epoch]['dname']
sname = self._sweep_information[epoch]['sname']
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%d.mtz' % \
(pname, xname, dname, counter))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
# record this for future reference - will be needed in the
# radiation damage analysis...
# hack - reset this as it gets in a muddle...
intgr = self._sweep_information[epoch]['integrater']
self._sweep_information[epoch][
'batches'] = intgr.get_integrater_batches()
first_batch = min(self._sweep_information[epoch]['batches'])
self._sweep_information[epoch][
'batch_offset'] = counter * max_batches - first_batch + 1
rb.set_hklin(hklin)
rb.set_first_batch(counter * max_batches + 1)
rb.set_hklout(hklout)
new_batches = rb.rebatch()
# update the "input information"
self._sweep_information[epoch]['hklin'] = hklout
self._sweep_information[epoch]['batches'] = new_batches
# update the counter & recycle
counter += 1
if Flags.get_chef():
self._sweep_information_to_chef()
s = self._factory.Sortmtz()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(hklout)
for epoch in epochs:
s.add_hklin(self._sweep_information[epoch]['hklin'])
s.sort(vrset = -99999999.0)
self._prepared_reflections = hklout
if self.get_scaler_reference_reflection_file():
md = self._factory.Mtzdump()
md.set_hklin(self.get_scaler_reference_reflection_file())
md.dump()
spacegroups = [md.get_spacegroup()]
reindex_operator = 'h,k,l'
else:
pointless = self._factory.Pointless()
pointless.set_hklin(hklout)
pointless.decide_spacegroup()
FileHandler.record_log_file('%s %s pointless' % \
(self._scalr_pname,
self._scalr_xname),
pointless.get_log_file())
spacegroups = pointless.get_likely_spacegroups()
reindex_operator = pointless.get_spacegroup_reindex_operator()
if self._scalr_input_spacegroup:
Debug.write('Assigning user input spacegroup: %s' % \
self._scalr_input_spacegroup)
spacegroups = [self._scalr_input_spacegroup]
reindex_operator = 'h,k,l'
self._scalr_likely_spacegroups = spacegroups
spacegroup = self._scalr_likely_spacegroups[0]
self._scalr_reindex_operator = reindex_operator
Chatter.write('Likely spacegroups:')
for spag in self._scalr_likely_spacegroups:
Chatter.write('%s' % spag)
Chatter.write(
'Reindexing to first spacegroup setting: %s (%s)' % \
(spacegroup, clean_reindex_operator(reindex_operator)))
hklin = self._prepared_reflections
hklout = os.path.join(self.get_working_directory(),
'%s_%s_reindex.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(hklout)
ri = self._factory.Reindex()
ri.set_hklin(hklin)
ri.set_hklout(hklout)
ri.set_spacegroup(spacegroup)
ri.set_operator(reindex_operator)
ri.reindex()
hklin = hklout
hklout = os.path.join(self.get_working_directory(),
'%s_%s_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s = self._factory.Sortmtz()
s.set_hklin(hklin)
s.set_hklout(hklout)
s.sort(vrset = -99999999.0)
self._prepared_reflections = hklout
Debug.write(
'Updating unit cell to %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(ri.get_cell()))
self._scalr_cell = tuple(ri.get_cell())
return
