def xia2_setup():
"""Actually process something..."""
Citations.cite("xia2")
# print versions of related software
logger.info(dials_version())
ccp4_version = get_ccp4_version()
if ccp4_version:
logger.info("CCP4 %s", ccp4_version)
CommandLine = get_command_line()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
for name, xtal in xtals.items():
if not xtal.get_all_image_names():
logger.info("-----------------------------------" +
"-" * len(name))
logger.info("| No images assigned for crystal %s |", name)
logger.info("-----------------------------------" +
"-" * len(name))
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
xinfo = CommandLine.get_xinfo()
logger.info(f"Project directory: {xinfo.path}")
logger.info(f"xinfo written to: {params.xia2.settings.input.xinfo}")
logger.info(f"Parameters: {PhilIndex.get_diff().as_str()}")
def _integrate_prepare(self):
"""Prepare for integration - in XDS terms this may mean rerunning
IDXREF to get the XPARM etc. DEFPIX is considered part of the full
integration as it is resolution dependent."""
Citations.cite("dials")
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(min(images), max(images))
logger.debug("DIALS INTEGRATE PREPARE:")
logger.debug("Wavelength: %.6f" % self.get_wavelength())
logger.debug("Distance: %.2f" % self.get_distance())
if not self.get_integrater_low_resolution():
dmax = self._intgr_refiner.get_indexer_low_resolution(
self.get_integrater_epoch()
)
self.set_integrater_low_resolution(dmax)
logger.debug(
"Low resolution set to: %s" % self.get_integrater_low_resolution()
)
## copy the data across
refiner = self.get_integrater_refiner()
# For multi-sweep refinement, get the split experiments from after refinement.
if PhilIndex.params.xia2.settings.multi_sweep_refinement:
self._intgr_experiments_filename = refiner.get_refiner_payload(
f"{self._intgr_sweep._name}_models.expt"
)
self._intgr_indexed_filename = refiner.get_refiner_payload(
f"{self._intgr_sweep._name}_observations.refl"
)
# Otherwise, there should only be a single experiment list and reflection table.
else:
self._intgr_experiments_filename = refiner.get_refiner_payload(
"models.expt"
)
self._intgr_indexed_filename = refiner.get_refiner_payload(
"observations.refl"
)
experiments = load.experiment_list(self._intgr_experiments_filename)
experiment = experiments[0]
# this is the result of the cell refinement
self._intgr_cell = experiment.crystal.get_unit_cell().parameters()
logger.debug("Files available at the end of DIALS integrate prepare:")
for f in self._data_files:
logger.debug("%s" % f)
self.set_detector(experiment.detector)
self.set_beam_obj(experiment.beam)
self.set_goniometer(experiment.goniometer)
def run(args):
cmd_line = command_line.argument_interpreter(
master_params=master_phil_scope)
working_phil, args = cmd_line.process_and_fetch(
args=args, custom_processor="collect_remaining")
working_phil.show()
params = working_phil.extract()
if params.unit_cell is not None:
unit_cell = params.unit_cell
crystal_symmetry = crystal.symmetry(unit_cell=unit_cell)
else:
crystal_symmetry = None
from iotbx.reflection_file_reader import any_reflection_file
result = any_reflection_file(args[0])
unmerged_intensities = None
batches_all = None
for ma in result.as_miller_arrays(merge_equivalents=False,
crystal_symmetry=crystal_symmetry):
#print ma.info().labels
if ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
assert ma.anomalous_flag()
unmerged_intensities = ma
elif ma.info().labels == ['I', 'SIGI']:
assert not ma.anomalous_flag()
unmerged_intensities = ma
elif ma.info().labels == ['BATCH']:
batches_all = ma
assert batches_all is not None
assert unmerged_intensities is not None
id_to_batches = None
if len(params.batch) > 0:
id_to_batches = {}
for b in params.batch:
assert b.id is not None
assert b.range is not None
assert b.id not in id_to_batches, "Duplicate batch id: %s" % b.id
id_to_batches[b.id] = b.range
result = delta_cc_half(unmerged_intensities,
batches_all,
n_bins=params.n_bins,
d_min=params.d_min,
cc_one_half_method=params.cc_one_half_method,
id_to_batches=id_to_batches)
hist_filename = 'delta_cc_hist.png'
print 'Saving histogram to %s' % hist_filename
result.plot_histogram(hist_filename)
print result.get_table()
from xia2.Handlers.Citations import Citations
Citations.cite('delta_cc_half')
for citation in Citations.get_citations_acta():
print citation
def __init__(self):
# generic things
CCP4DriverInstance.__class__.__init__(self)
Citations.cite("ccp4")
self.set_executable(os.path.join(os.environ.get("CBIN", ""), "freerflag"))
self._free_fraction = 0.05
def _integrate(self):
'''Implement the integrater interface.'''
# cite the program
Citations.cite('mosflm')
images_str = '%d to %d' % tuple(self._intgr_wedge)
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % tuple(self._intgr_cell)
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = '...%s' % self._fp_directory[-46:]
Journal.block(
'integrating', self._intgr_sweep_name, 'mosflm',
{'images':images_str,
'cell':cell_str,
'lattice':self.get_integrater_refiner().get_refiner_lattice(),
'template':self._fp_template,
'directory':dirname,
'resolution':'%.2f' % self._intgr_reso_high})
self._mosflm_rerun_integration = False
wd = self.get_working_directory()
try:
if self.get_integrater_sweep_name():
pname, xname, dname = self.get_integrater_project_info()
nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
if nproc > 1:
Debug.write('Parallel integration: %d jobs' %nproc)
self._mosflm_hklout = self._mosflm_parallel_integrate()
else:
self._mosflm_hklout = self._mosflm_integrate()
# record integration output for e.g. BLEND.
sweep = self.get_integrater_sweep_name()
if sweep:
FileHandler.record_more_data_file(
'%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep),
self._mosflm_hklout)
except IntegrationError, e:
if 'negative mosaic spread' in str(e):
if self._mosflm_postref_fix_mosaic:
Chatter.write(
'Negative mosaic spread - stopping integration')
raise BadLatticeError, 'negative mosaic spread'
Chatter.write(
'Negative mosaic spread - rerunning integration')
self.set_integrater_done(False)
self._mosflm_postref_fix_mosaic = True
def run():
if os.path.exists("xia2-working.phil"):
sys.argv.append("xia2-working.phil")
try:
check_environment()
except Exception as e:
traceback.print_exc(file=open("xia2.error", "w"))
Chatter.write('Status: error "%s"' % str(e))
# print the version
Chatter.write(Version)
Citations.cite("xia2")
start_time = time.time()
assert os.path.exists("xia2.json")
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename="xia2.json")
crystals = xinfo.get_crystals()
for crystal_id, crystal in crystals.iteritems():
# cwd = os.path.abspath(os.curdir)
from libtbx import Auto
scale_dir = PhilIndex.params.xia2.settings.scale.directory
if scale_dir is Auto:
scale_dir = "scale"
i = 0
while os.path.exists(os.path.join(crystal.get_name(), scale_dir)):
i += 1
scale_dir = "scale%i" % i
PhilIndex.params.xia2.settings.scale.directory = scale_dir
working_directory = Environment.generate_directory(
[crystal.get_name(), scale_dir])
# os.chdir(working_directory)
crystals[crystal_id]._scaler = None # reset scaler
scaler = crystal._get_scaler()
Chatter.write(xinfo.get_output())
crystal.serialize()
duration = time.time() - start_time
# write out the time taken in a human readable way
Chatter.write("Processing took %s" %
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
# delete all of the temporary mtz files...
cleanup()
write_citations()
xinfo.as_json(filename="xia2.json")
Environment.cleanup()
def run():
if os.path.exists("xia2-working.phil"):
sys.argv.append("xia2-working.phil")
try:
check_environment()
except Exception as e:
with open("xia2-error.txt", "w") as fh:
traceback.print_exc(file=fh)
logger.error('Status: error "%s"', str(e))
# print the version
logger.info(Version)
Citations.cite("xia2")
start_time = time.time()
assert os.path.exists("xia2.json")
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename="xia2.json")
with cleanup(xinfo.path):
crystals = xinfo.get_crystals()
for crystal_id, crystal in crystals.items():
scale_dir = PhilIndex.params.xia2.settings.scale.directory
if scale_dir is Auto:
scale_dir = "scale"
i = 0
while os.path.exists(os.path.join(crystal.get_name(), scale_dir)):
i += 1
scale_dir = "scale%i" % i
PhilIndex.params.xia2.settings.scale.directory = scale_dir
# reset scaler
crystals[crystal_id]._scaler = None
crystal._get_scaler()
logger.info(xinfo.get_output())
crystal.serialize()
duration = time.time() - start_time
# write out the time taken in a human readable way
logger.info(
"Processing took %s", time.strftime("%Hh %Mm %Ss", time.gmtime(duration))
)
write_citations()
xinfo.as_json(filename="xia2.json")
def write_citations():
# tell the user which programs were used...
logger.info("XIA2 used... %s" % ", ".join(Citations.get_programs()))
logger.info("Here are the appropriate citations (BIBTeX in xia2-citations.bib.)")
for citation in Citations.get_citations_acta():
logger.info(citation)
# and write the bibtex versions
out = open("xia2-citations.bib", "w")
for citation in Citations.get_citations():
out.write("%s\n" % citation)
out.close()
def write_citations():
# tell the user which programs were used...
Chatter.write('XIA2 used... %s' % \
', '.join(Citations.get_programs()))
Chatter.write(
'Here are the appropriate citations (BIBTeX in xia2-citations.bib.)')
for citation in Citations.get_citations_acta():
Chatter.write(citation)
# and write the bibtex versions
out = open('xia2-citations.bib', 'w')
for citation in Citations.get_citations():
out.write('%s\n' % citation)
out.close()
def __init__(self):
# generic things
DriverInstance.__class__.__init__(self)
Citations.cite("ccp4")
self.set_executable(
os.path.join(os.environ.get("CBIN", ""), "ctruncate"))
self._anomalous = False
self._nres = 0
self._b_factor = 0.0
self._moments = None
# numbers of reflections in and out, and number of absences
# counted
self._nref_in = 0
self._nref_out = 0
self._nabsent = 0
self._xmlout = None
def citations(xproject):
lines = []
lines.append('\n')
lines.append('References')
lines.append('=' * len(lines[-1]))
# Sort citations
citations = {}
for cdict in Citations.get_citations_dicts():
citations[cdict['acta']] = cdict.get('url')
for citation in sorted(citations.iterkeys()):
if citations[citation]:
lines.append("`%s\n<%s>`_\n" % (citation, citations[citation]))
else:
lines.append("%s\n" % citation)
return lines
def citations(xproject):
lines = []
lines.append('\n')
lines.append('References')
lines.append('=' * len(lines[-1]))
# Sort citations
citations = {}
for cdict in Citations.get_citations_dicts():
citations[cdict['acta']] = cdict.get('url')
for citation in sorted(citations.iterkeys()):
if citations[citation]:
lines.append("`%s\n<%s>`_\n" % (citation, citations[citation]))
else:
lines.append("%s\n" % citation)
return lines
def _scale_prepare(self):
'''Perform all of the preparation required to deliver the scaled
data. This should sort together the reflection files, ensure that
they are correctly indexed (via pointless) and generally tidy
things up.'''
# acknowledge all of the programs we are about to use...
Citations.cite('pointless')
Citations.cite('aimless')
Citations.cite('ccp4')
# ---------- GATHER ----------
self._sweep_handler = SweepInformationHandler(self._scalr_integraters)
Journal.block(
'gathering', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'working directory':self.get_working_directory()})
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
exclude_sweep = False
for sweep in PhilIndex.params.xia2.settings.sweep:
if sweep.id == sname and sweep.exclude:
exclude_sweep = True
break
if exclude_sweep:
self._sweep_handler.remove_epoch(epoch)
Debug.write('Excluding sweep %s' % sname)
else:
Journal.entry({'adding data from':'%s/%s/%s' % \
(xname, dname, sname)})
# gather data for all images which belonged to the parent
# crystal - allowing for the fact that things could go wrong
# e.g. epoch information not available, exposure times not in
# headers etc...
for e in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(e)
assert is_mtz_file(si.get_reflections())
p, x = self._sweep_handler.get_project_info()
self._scalr_pname = p
self._scalr_xname = x
# verify that the lattices are consistent, calling eliminate if
# they are not N.B. there could be corner cases here
need_to_return = False
multi_sweep_indexing = \
PhilIndex.params.xia2.settings.multi_sweep_indexing == True
if len(self._sweep_handler.get_epochs()) > 1:
# if we have multi-sweep-indexing going on then logic says all should
# share common lattice & UB definition => this is not used here?
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
batches = MtzUtils.batches_from_mtz(hklin)
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
from xia2.lib.bits import nifty_power_of_ten
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
refiners = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
refiners.append(refiner)
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_prepointless.mtz' % \
(pname, xname, dname, si.get_sweep_name()))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
from xia2.Modules.Scaler.rebatch import rebatch
new_batches = rebatch(
hklin, hklout, first_batch=counter * max_batches + 1,
pname=pname, xname=xname, dname=dname)
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
# FIXME xia2-51 in here look at running constant scaling on the
# pointless hklin to put the runs on the same scale. Ref=[A]
pointless_const = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_const.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(pointless_const)
aimless_const = self._factory.Aimless()
aimless_const.set_hklin(pointless_hklin)
aimless_const.set_hklout(pointless_const)
aimless_const.const()
pointless_const = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_const_unmerged.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(pointless_const)
pointless_hklin = pointless_const
# FIXME xia2-51 in here need to pass all refiners to ensure that the
# information is passed back to all of them not just the last one...
Debug.write('Running multisweep pointless for %d sweeps' %
len(refiners))
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_multisweep(pointless_hklin,
refiners)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattices = [Syminfo.get_lattice(pointgroup)]
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
else:
lattices = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
ntr = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattice = Syminfo.get_lattice(pointgroup)
if not lattice in lattices:
lattices.append(lattice)
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
if len(lattices) > 1:
# why not using pointless indexer jiffy??!
correct_lattice = sort_lattices(lattices)[0]
Chatter.write('Correct lattice asserted to be %s' % \
correct_lattice)
# transfer this information back to the indexers
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
refiner = si.get_integrater().get_integrater_refiner()
sname = si.get_sweep_name()
state = refiner.set_refiner_asserted_lattice(
correct_lattice)
if state == refiner.LATTICE_CORRECT:
Chatter.write('Lattice %s ok for sweep %s' % \
(correct_lattice, sname))
elif state == refiner.LATTICE_IMPOSSIBLE:
raise RuntimeError('Lattice %s impossible for %s' \
% (correct_lattice, sname))
elif state == refiner.LATTICE_POSSIBLE:
Chatter.write('Lattice %s assigned for sweep %s' % \
(correct_lattice, sname))
need_to_return = True
# if one or more of them was not in the lowest lattice,
# need to return here to allow reprocessing
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# ---------- REINDEX ALL DATA TO CORRECT POINTGROUP ----------
# all should share the same pointgroup, unless twinned... in which
# case force them to be...
pointgroups = {}
reindex_ops = {}
probably_twinned = False
need_to_return = False
multi_sweep_indexing = \
PhilIndex.params.xia2.settings.multi_sweep_indexing == True
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
batches = MtzUtils.batches_from_mtz(hklin)
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
from xia2.lib.bits import nifty_power_of_ten
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
refiners = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
refiners.append(refiner)
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_prepointless.mtz' % \
(pname, xname, dname, si.get_sweep_name()))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
from xia2.Modules.Scaler.rebatch import rebatch
new_batches = rebatch(
hklin, hklout, first_batch=counter * max_batches + 1,
pname=pname, xname=xname, dname=dname)
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
# FIXME related to xia2-51 - this looks very very similar to the logic
# in [A] above - is this duplicated logic?
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
pointless_const = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_const.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(pointless_const)
aimless_const = self._factory.Aimless()
aimless_const.set_hklin(pointless_hklin)
aimless_const.set_hklout(pointless_const)
aimless_const.const()
pointless_const = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_const_unmerged.mtz' % \
(self._scalr_pname, self._scalr_xname))
FileHandler.record_temporary_file(pointless_const)
pointless_hklin = pointless_const
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_multisweep(
pointless_hklin, refiners)
for epoch in self._sweep_handler.get_epochs():
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
else:
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
pt = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
if ntr:
integrater.integrater_reset_reindex_operator()
need_to_return = True
if pt and not probably_twinned:
probably_twinned = True
Debug.write('Pointgroup: %s (%s)' % (pointgroup, reindex_op))
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
overall_pointgroup = None
pointgroup_set = {pointgroups[e] for e in pointgroups}
if len(pointgroup_set) > 1 and \
not probably_twinned:
raise RuntimeError('non uniform pointgroups')
if len(pointgroup_set) > 1:
Debug.write('Probably twinned, pointgroups: %s' % \
' '.join([p.replace(' ', '') for p in \
list(pointgroup_set)]))
numbers = [Syminfo.spacegroup_name_to_number(s) for s in \
pointgroup_set]
overall_pointgroup = Syminfo.spacegroup_number_to_name(min(numbers))
self._scalr_input_pointgroup = overall_pointgroup
Chatter.write('Twinning detected, assume pointgroup %s' % \
overall_pointgroup)
need_to_return = True
else:
overall_pointgroup = pointgroup_set.pop()
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
integrater = si.get_integrater()
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(overall_pointgroup))
integrater.set_integrater_reindex_operator(
reindex_ops[epoch], reason='setting point group')
# This will give us the reflections in the correct point group
si.set_reflections(integrater.get_integrater_intensities())
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# in here now optionally work through the data files which should be
# indexed with a consistent point group, and transform the orientation
# matrices by the lattice symmetry operations (if possible) to get a
# consistent definition of U matrix modulo fixed rotations
if PhilIndex.params.xia2.settings.unify_setting:
from scitbx.matrix import sqr
reference_U = None
i3 = sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
fixed = sqr(intgr.get_goniometer().get_fixed_rotation())
u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections())
U = fixed.inverse() * sqr(u).transpose()
B = sqr(b)
if reference_U is None:
reference_U = U
continue
results = []
for op in s.all_ops():
R = B * sqr(op.r().as_double()).transpose() * B.inverse()
nearly_i3 = (U * R).inverse() * reference_U
score = sum([abs(_n - _i) for (_n, _i) in zip(nearly_i3, i3)])
results.append((score, op.r().as_hkl(), op))
results.sort()
best = results[0]
Debug.write('Best reindex: %s %.3f' % (best[1], best[0]))
intgr.set_integrater_reindex_operator(best[2].r().inverse().as_hkl(),
reason='unifying [U] setting')
si.set_reflections(intgr.get_integrater_intensities())
# recalculate to verify
u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections())
U = fixed.inverse() * sqr(u).transpose()
Debug.write('New reindex: %s' % (U.inverse() * reference_U))
# FIXME I should probably raise an exception at this stage if this
# is not about I3...
if self.get_scaler_reference_reflection_file():
self._reference = self.get_scaler_reference_reflection_file()
Debug.write('Using HKLREF %s' % self._reference)
elif PhilIndex.params.xia2.settings.scale.reference_reflection_file:
self._reference = PhilIndex.params.xia2.settings.scale.reference_reflection_file
Debug.write('Using HKLREF %s' % self._reference)
params = PhilIndex.params
use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs
if len(self._sweep_handler.get_epochs()) > 1 and use_brehm_diederichs:
brehm_diederichs_files_in = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
brehm_diederichs_files_in.append(hklin)
# now run cctbx.brehm_diederichs to figure out the indexing hand for
# each sweep
from xia2.Wrappers.Cctbx.BrehmDiederichs import BrehmDiederichs
from xia2.lib.bits import auto_logfiler
brehm_diederichs = BrehmDiederichs()
brehm_diederichs.set_working_directory(self.get_working_directory())
auto_logfiler(brehm_diederichs)
brehm_diederichs.set_input_filenames(brehm_diederichs_files_in)
# 1 or 3? 1 seems to work better?
brehm_diederichs.set_asymmetric(1)
brehm_diederichs.run()
reindexing_dict = brehm_diederichs.get_reindexing_dict()
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
reindex_op = reindexing_dict.get(os.path.abspath(hklin))
assert reindex_op is not None
if 1 or reindex_op != 'h,k,l':
# apply the reindexing operator
intgr.set_integrater_reindex_operator(
reindex_op, reason='match reference')
si.set_reflections(intgr.get_integrater_intensities())
elif len(self._sweep_handler.get_epochs()) > 1 and \
not self._reference:
first = self._sweep_handler.get_epochs()[0]
si = self._sweep_handler.get_sweep_information(first)
self._reference = si.get_reflections()
if self._reference:
md = self._factory.Mtzdump()
md.set_hklin(self._reference)
md.dump()
if md.get_batches() and False:
raise RuntimeError('reference reflection file %s unmerged' % \
self._reference)
datasets = md.get_datasets()
if len(datasets) > 1 and False:
raise RuntimeError('more than one dataset in %s' % \
self._reference)
# then get the unit cell, lattice etc.
reference_lattice = Syminfo.get_lattice(md.get_spacegroup())
reference_cell = md.get_dataset_info(datasets[0])['cell']
# then compute the pointgroup from this...
# ---------- REINDEX TO CORRECT (REFERENCE) SETTING ----------
for epoch in self._sweep_handler.get_epochs():
# if we are working with unified UB matrix then this should not
# be a problem here (note, *if*; *should*)
# what about e.g. alternative P1 settings?
# see JIRA MXSW-904
if PhilIndex.params.xia2.settings.unify_setting:
continue
pl = self._factory.Pointless()
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
pl.set_hklin(self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()))
hklout = os.path.join(
self.get_working_directory(),
'%s_rdx2.mtz' % os.path.split(hklin)[-1][:-4])
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
# now set the initial reflection set as a reference...
pl.set_hklref(self._reference)
# https://github.com/xia2/xia2/issues/115 - should ideally iteratively
# construct a reference or a tree of correlations to ensure correct
# reference setting - however if small molecule assume has been
# multi-sweep-indexed so can ignore "fatal errors" - temporary hack
pl.decide_pointgroup(
ignore_errors=PhilIndex.params.xia2.settings.small_molecule)
Debug.write('Reindexing analysis of %s' % pl.get_hklin())
pointgroup = pl.get_pointgroup()
reindex_op = pl.get_reindex_operator()
Debug.write('Operator: %s' % reindex_op)
# apply this...
integrater = si.get_integrater()
integrater.set_integrater_reindex_operator(reindex_op,
reason='match reference')
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
si.set_reflections(integrater.get_integrater_intensities())
md = self._factory.Mtzdump()
md.set_hklin(si.get_reflections())
md.dump()
datasets = md.get_datasets()
if len(datasets) > 1:
raise RuntimeError('more than one dataset in %s' % \
si.get_reflections())
# then get the unit cell, lattice etc.
lattice = Syminfo.get_lattice(md.get_spacegroup())
cell = md.get_dataset_info(datasets[0])['cell']
if lattice != reference_lattice:
raise RuntimeError('lattices differ in %s and %s' % \
(self._reference, si.get_reflections()))
Debug.write('Cell: %.2f %.2f %.2f %.2f %.2f %.2f' % cell)
Debug.write('Ref: %.2f %.2f %.2f %.2f %.2f %.2f' % reference_cell)
for j in range(6):
if math.fabs((cell[j] - reference_cell[j]) /
reference_cell[j]) > 0.1:
raise RuntimeError( \
'unit cell parameters differ in %s and %s' % \
(self._reference, si.get_reflections()))
# ---------- SORT TOGETHER DATA ----------
self._sort_together_data_ccp4()
self._scalr_resolution_limits = {}
# store central resolution limit estimates
batch_ranges = [
self._sweep_handler.get_sweep_information(epoch).get_batch_range()
for epoch in self._sweep_handler.get_epochs()
]
self._resolution_limit_estimates = ersatz_resolution(
self._prepared_reflections, batch_ranges)
def _scale_prepare(self):
"""Perform all of the preparation required to deliver the scaled
data. This should sort together the reflection files, ensure that
they are correctly indexed (via pointless) and generally tidy
things up."""
# acknowledge all of the programs we are about to use...
Citations.cite("pointless")
Citations.cite("aimless")
Citations.cite("ccp4")
# ---------- GATHER ----------
self._sweep_handler = SweepInformationHandler(self._scalr_integraters)
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
exclude_sweep = False
for sweep in PhilIndex.params.xia2.settings.sweep:
if sweep.id == sname and sweep.exclude:
exclude_sweep = True
break
if exclude_sweep:
self._sweep_handler.remove_epoch(epoch)
logger.debug("Excluding sweep %s", sname)
else:
logger.debug("%-30s %s/%s/%s", "adding data from:", xname, dname, sname)
# gather data for all images which belonged to the parent
# crystal - allowing for the fact that things could go wrong
# e.g. epoch information not available, exposure times not in
# headers etc...
for e in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(e)
assert is_mtz_file(si.get_reflections()), repr(si.get_reflections())
p, x = self._sweep_handler.get_project_info()
self._scalr_pname = p
self._scalr_xname = x
# verify that the lattices are consistent, calling eliminate if
# they are not N.B. there could be corner cases here
need_to_return = False
multi_sweep_indexing = PhilIndex.params.xia2.settings.multi_sweep_indexing
# START OF if more than one epoch
if len(self._sweep_handler.get_epochs()) > 1:
# if we have multi-sweep-indexing going on then logic says all should
# share common lattice & UB definition => this is not used here?
# START OF if multi_sweep indexing and not input pg
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
batches = MtzUtils.batches_from_mtz(hklin)
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
logger.debug("Biggest sweep has %d batches", max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
refiners = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
refiners.append(refiner)
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()
)
hklout = os.path.join(
self.get_working_directory(),
"%s_%s_%s_%s_prepointless.mtz"
% (pname, xname, dname, si.get_sweep_name()),
)
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
rebatch(
hklin,
hklout,
first_batch=counter * max_batches + 1,
pname=pname,
xname=xname,
dname=dname,
)
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
# SUMMARY - have added all sweeps to pointless_hklins
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(
self.get_working_directory(),
"%s_%s_prepointless_sorted.mtz"
% (self._scalr_pname, self._scalr_xname),
)
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
# FIXME xia2-51 in here look at running constant scaling on the
# pointless hklin to put the runs on the same scale. Ref=[A]
pointless_const = os.path.join(
self.get_working_directory(),
"%s_%s_prepointless_const.mtz"
% (self._scalr_pname, self._scalr_xname),
)
FileHandler.record_temporary_file(pointless_const)
aimless_const = self._factory.Aimless()
aimless_const.set_hklin(pointless_hklin)
aimless_const.set_hklout(pointless_const)
aimless_const.const()
pointless_const = os.path.join(
self.get_working_directory(),
"%s_%s_prepointless_const_unmerged.mtz"
% (self._scalr_pname, self._scalr_xname),
)
FileHandler.record_temporary_file(pointless_const)
pointless_hklin = pointless_const
# FIXME xia2-51 in here need to pass all refiners to ensure that the
# information is passed back to all of them not just the last one...
logger.debug(
"Running multisweep pointless for %d sweeps", len(refiners)
)
pointgroup, reindex_op, ntr, pt = self._pointless_indexer_multisweep(
pointless_hklin, refiners
)
logger.debug("X1698: %s: %s", pointgroup, reindex_op)
lattices = [Syminfo.get_lattice(pointgroup)]
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
# SUMMARY - added all sweeps together into an mtz, ran
# _pointless_indexer_multisweep on this, made a list of one lattice
# and potentially reset reindex op?
# END OF if multi_sweep indexing and not input pg
# START OF if not multi_sweep, or input pg given
else:
lattices = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
pointgroup = self._scalr_input_pointgroup
reindex_op = "h,k,l"
ntr = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()
)
pointgroup, reindex_op, ntr, pt = self._pointless_indexer_jiffy(
pointless_hklin, refiner
)
logger.debug("X1698: %s: %s", pointgroup, reindex_op)
lattice = Syminfo.get_lattice(pointgroup)
if lattice not in lattices:
lattices.append(lattice)
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
# SUMMARY do pointless_indexer on each sweep, get lattices and make a list
# of unique lattices, potentially reset reindex op.
# END OF if not multi_sweep, or input pg given
# SUMMARY - still within if more than one epoch, now have a list of number
# of lattices
# START OF if multiple-lattices
if len(lattices) > 1:
# why not using pointless indexer jiffy??!
correct_lattice = sort_lattices(lattices)[0]
logger.info("Correct lattice asserted to be %s", correct_lattice)
# transfer this information back to the indexers
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
refiner = si.get_integrater().get_integrater_refiner()
sname = si.get_sweep_name()
state = refiner.set_refiner_asserted_lattice(correct_lattice)
if state == refiner.LATTICE_CORRECT:
logger.info(
"Lattice %s ok for sweep %s", correct_lattice, sname
)
elif state == refiner.LATTICE_IMPOSSIBLE:
raise RuntimeError(
f"Lattice {correct_lattice} impossible for {sname}"
)
elif state == refiner.LATTICE_POSSIBLE:
logger.info(
"Lattice %s assigned for sweep %s", correct_lattice, sname
)
need_to_return = True
# END OF if multiple-lattices
# SUMMARY - forced all lattices to be same and hope its okay.
# END OF if more than one epoch
# if one or more of them was not in the lowest lattice,
# need to return here to allow reprocessing
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# ---------- REINDEX ALL DATA TO CORRECT POINTGROUP ----------
# all should share the same pointgroup, unless twinned... in which
# case force them to be...
pointgroups = {}
reindex_ops = {}
probably_twinned = False
need_to_return = False
multi_sweep_indexing = PhilIndex.params.xia2.settings.multi_sweep_indexing
# START OF if multi-sweep and not input pg
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
batches = MtzUtils.batches_from_mtz(hklin)
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
logger.debug("Biggest sweep has %d batches", max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
refiners = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
refiners.append(refiner)
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()
)
hklout = os.path.join(
self.get_working_directory(),
"%s_%s_%s_%s_prepointless.mtz"
% (pname, xname, dname, si.get_sweep_name()),
)
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
rebatch(
hklin,
hklout,
first_batch=counter * max_batches + 1,
pname=pname,
xname=xname,
dname=dname,
)
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
# FIXME related to xia2-51 - this looks very very similar to the logic
# in [A] above - is this duplicated logic?
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(
self.get_working_directory(),
"%s_%s_prepointless_sorted.mtz"
% (self._scalr_pname, self._scalr_xname),
)
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
pointless_const = os.path.join(
self.get_working_directory(),
f"{self._scalr_pname}_{self._scalr_xname}_prepointless_const.mtz",
)
FileHandler.record_temporary_file(pointless_const)
aimless_const = self._factory.Aimless()
aimless_const.set_hklin(pointless_hklin)
aimless_const.set_hklout(pointless_const)
aimless_const.const()
pointless_const = os.path.join(
self.get_working_directory(),
"%s_%s_prepointless_const_unmerged.mtz"
% (self._scalr_pname, self._scalr_xname),
)
FileHandler.record_temporary_file(pointless_const)
pointless_hklin = pointless_const
pointgroup, reindex_op, ntr, pt = self._pointless_indexer_multisweep(
pointless_hklin, refiners
)
for epoch in self._sweep_handler.get_epochs():
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
# SUMMARY ran pointless multisweep on combined mtz and made a dict
# of pointgroups and reindex_ops (all same)
# END OF if multi-sweep and not input pg
# START OF if not mulit-sweep or pg given
else:
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
if self._scalr_input_pointgroup:
logger.debug(
"Using input pointgroup: %s", self._scalr_input_pointgroup
)
pointgroup = self._scalr_input_pointgroup
reindex_op = "h,k,l"
pt = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()
)
pointgroup, reindex_op, ntr, pt = self._pointless_indexer_jiffy(
pointless_hklin, refiner
)
logger.debug("X1698: %s: %s", pointgroup, reindex_op)
if ntr:
integrater.integrater_reset_reindex_operator()
need_to_return = True
if pt and not probably_twinned:
probably_twinned = True
logger.debug("Pointgroup: %s (%s)", pointgroup, reindex_op)
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
# SUMMARY - for each sweep, run indexer jiffy and get reindex operators
# and pointgroups dictionaries (could be different between sweeps)
# END OF if not mulit-sweep or pg given
overall_pointgroup = None
pointgroup_set = {pointgroups[e] for e in pointgroups}
if len(pointgroup_set) > 1 and not probably_twinned:
raise RuntimeError(
"non uniform pointgroups: %s" % str(list(pointgroup_set))
)
if len(pointgroup_set) > 1:
logger.debug(
"Probably twinned, pointgroups: %s",
" ".join(p.replace(" ", "") for p in pointgroup_set),
)
numbers = (Syminfo.spacegroup_name_to_number(ps) for ps in pointgroup_set)
overall_pointgroup = Syminfo.spacegroup_number_to_name(min(numbers))
self._scalr_input_pointgroup = overall_pointgroup
logger.info("Twinning detected, assume pointgroup %s", overall_pointgroup)
need_to_return = True
else:
overall_pointgroup = pointgroup_set.pop()
# SUMMARY - Have handled if different pointgroups & chosen an overall_pointgroup
# which is the lowest symmetry
# Now go through sweeps and do reindexing
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
integrater = si.get_integrater()
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(overall_pointgroup)
)
integrater.set_integrater_reindex_operator(
reindex_ops[epoch], reason="setting point group"
)
# This will give us the reflections in the correct point group
si.set_reflections(integrater.get_integrater_intensities())
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# in here now optionally work through the data files which should be
# indexed with a consistent point group, and transform the orientation
# matrices by the lattice symmetry operations (if possible) to get a
# consistent definition of U matrix modulo fixed rotations
if PhilIndex.params.xia2.settings.unify_setting:
self.unify_setting()
if self.get_scaler_reference_reflection_file():
self._reference = self.get_scaler_reference_reflection_file()
logger.debug("Using HKLREF %s", self._reference)
elif PhilIndex.params.xia2.settings.scale.reference_reflection_file:
self._reference = (
PhilIndex.params.xia2.settings.scale.reference_reflection_file
)
logger.debug("Using HKLREF %s", self._reference)
params = PhilIndex.params
use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs
if len(self._sweep_handler.get_epochs()) > 1 and use_brehm_diederichs:
self.brehm_diederichs_reindexing()
# If not Brehm-deidrichs, set reference as first sweep
elif len(self._sweep_handler.get_epochs()) > 1 and not self._reference:
first = self._sweep_handler.get_epochs()[0]
si = self._sweep_handler.get_sweep_information(first)
self._reference = si.get_reflections()
# Now reindex to be consistent with first dataset - run pointless on each
# dataset with reference
if self._reference:
md = self._factory.Mtzdump()
md.set_hklin(self._reference)
md.dump()
datasets = md.get_datasets()
# then get the unit cell, lattice etc.
reference_lattice = Syminfo.get_lattice(md.get_spacegroup())
reference_cell = md.get_dataset_info(datasets[0])["cell"]
# then compute the pointgroup from this...
# ---------- REINDEX TO CORRECT (REFERENCE) SETTING ----------
for epoch in self._sweep_handler.get_epochs():
# if we are working with unified UB matrix then this should not
# be a problem here (note, *if*; *should*)
# what about e.g. alternative P1 settings?
# see JIRA MXSW-904
if PhilIndex.params.xia2.settings.unify_setting:
continue
pl = self._factory.Pointless()
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
pl.set_hklin(
self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()
)
)
hklout = os.path.join(
self.get_working_directory(),
"%s_rdx2.mtz" % os.path.split(hklin)[-1][:-4],
)
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
# now set the initial reflection set as a reference...
pl.set_hklref(self._reference)
# https://github.com/xia2/xia2/issues/115 - should ideally iteratively
# construct a reference or a tree of correlations to ensure correct
# reference setting - however if small molecule assume has been
# multi-sweep-indexed so can ignore "fatal errors" - temporary hack
pl.decide_pointgroup(
ignore_errors=PhilIndex.params.xia2.settings.small_molecule
)
logger.debug("Reindexing analysis of %s", pl.get_hklin())
pointgroup = pl.get_pointgroup()
reindex_op = pl.get_reindex_operator()
logger.debug("Operator: %s", reindex_op)
# apply this...
integrater = si.get_integrater()
integrater.set_integrater_reindex_operator(
reindex_op, reason="match reference"
)
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup)
)
si.set_reflections(integrater.get_integrater_intensities())
md = self._factory.Mtzdump()
md.set_hklin(si.get_reflections())
md.dump()
datasets = md.get_datasets()
if len(datasets) > 1:
raise RuntimeError(
"more than one dataset in %s" % si.get_reflections()
)
# then get the unit cell, lattice etc.
lattice = Syminfo.get_lattice(md.get_spacegroup())
cell = md.get_dataset_info(datasets[0])["cell"]
if lattice != reference_lattice:
raise RuntimeError(
"lattices differ in %s and %s"
% (self._reference, si.get_reflections())
)
logger.debug("Cell: %.2f %.2f %.2f %.2f %.2f %.2f" % cell)
logger.debug("Ref: %.2f %.2f %.2f %.2f %.2f %.2f" % reference_cell)
for j in range(6):
if (
math.fabs((cell[j] - reference_cell[j]) / reference_cell[j])
> 0.1
):
raise RuntimeError(
"unit cell parameters differ in %s and %s"
% (self._reference, si.get_reflections())
)
# ---------- SORT TOGETHER DATA ----------
self._sort_together_data_ccp4()
self._scalr_resolution_limits = {}
# store central resolution limit estimates
batch_ranges = [
self._sweep_handler.get_sweep_information(epoch).get_batch_range()
for epoch in self._sweep_handler.get_epochs()
]
self._resolution_limit_estimates = ersatz_resolution(
self._prepared_reflections, batch_ranges
)
def _index(self):
'''Actually index the diffraction pattern. Note well that
this is not going to compute the matrix...'''
# acknowledge this program
Citations.cite('labelit')
Citations.cite('distl')
#self.reset()
_images = []
for i in self._indxr_images:
for j in i:
if not j in _images:
_images.append(j)
_images.sort()
images_str = '%d' % _images[0]
for i in _images[1:]:
images_str += ', %d' % i
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
if self._indxr_sweep_name:
# then this is a proper autoindexing run - describe this
# to the journal entry
#if len(self._fp_directory) <= 50:
#dirname = self._fp_directory
#else:
#dirname = '...%s' % self._fp_directory[-46:]
dirname = os.path.dirname(self.get_imageset().get_template())
Journal.block(
'autoindexing', self._indxr_sweep_name, 'labelit',
{'images':images_str,
'target cell':cell_str,
'target lattice':self._indxr_input_lattice,
'template':self.get_imageset().get_template(),
'directory':dirname})
if len(_images) > 4:
raise RuntimeError, 'cannot use more than 4 images'
from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex
index = LabelitIndex()
index.set_working_directory(self.get_working_directory())
auto_logfiler(index)
#task = 'Autoindex from images:'
#for i in _images:
#task += ' %s' % self.get_image_name(i)
#self.set_task(task)
Debug.write('Indexing from images:')
for i in _images:
index.add_image(self.get_image_name(i))
Debug.write('%s' % self.get_image_name(i))
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
index.set_distance(xsweep.get_distance())
#if self.get_wavelength_prov() == 'user':
#index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
index.set_beam_centre(xsweep.get_beam_centre())
if self._refine_beam is False:
index.set_refine_beam(False)
else:
index.set_refine_beam(True)
index.set_beam_search_scope(self._beam_search_scope)
if ((math.fabs(self.get_wavelength() - 1.54) < 0.01) or
(math.fabs(self.get_wavelength() - 2.29) < 0.01)):
index.set_Cu_KA_or_Cr_KA(True)
#sweep = self.get_indexer_sweep_name()
#FileHandler.record_log_file(
#'%s INDEX' % (sweep), self.get_log_file())
try:
index.run()
except RuntimeError, e:
if self._refine_beam is False:
raise e
# can we improve the situation?
if self._beam_search_scope < 4.0:
self._beam_search_scope += 4.0
# try repeating the indexing!
self.set_indexer_done(False)
return 'failed'
# otherwise this is beyond redemption
raise e
def _index(self):
'''Actually index the diffraction pattern. Note well that
this is not going to compute the matrix...'''
# acknowledge this program
Citations.cite('labelit')
Citations.cite('distl')
#self.reset()
_images = []
for i in self._indxr_images:
for j in i:
if not j in _images:
_images.append(j)
_images.sort()
images_str = '%d' % _images[0]
for i in _images[1:]:
images_str += ', %d' % i
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
if self._indxr_sweep_name:
# then this is a proper autoindexing run - describe this
# to the journal entry
#if len(self._fp_directory) <= 50:
#dirname = self._fp_directory
#else:
#dirname = '...%s' % self._fp_directory[-46:]
dirname = os.path.dirname(self.get_imageset().get_template())
Journal.block(
'autoindexing', self._indxr_sweep_name, 'labelit', {
'images': images_str,
'target cell': cell_str,
'target lattice': self._indxr_input_lattice,
'template': self.get_imageset().get_template(),
'directory': dirname
})
if len(_images) > 4:
raise RuntimeError('cannot use more than 4 images')
from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex
index = LabelitIndex()
index.set_working_directory(self.get_working_directory())
auto_logfiler(index)
#task = 'Autoindex from images:'
#for i in _images:
#task += ' %s' % self.get_image_name(i)
#self.set_task(task)
Debug.write('Indexing from images:')
for i in _images:
index.add_image(self.get_image_name(i))
Debug.write('%s' % self.get_image_name(i))
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
index.set_distance(xsweep.get_distance())
#if self.get_wavelength_prov() == 'user':
#index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
index.set_beam_centre(xsweep.get_beam_centre())
if self._refine_beam is False:
index.set_refine_beam(False)
else:
index.set_refine_beam(True)
index.set_beam_search_scope(self._beam_search_scope)
if ((math.fabs(self.get_wavelength() - 1.54) < 0.01)
or (math.fabs(self.get_wavelength() - 2.29) < 0.01)):
index.set_Cu_KA_or_Cr_KA(True)
#sweep = self.get_indexer_sweep_name()
#FileHandler.record_log_file(
#'%s INDEX' % (sweep), self.get_log_file())
try:
index.run()
except RuntimeError as e:
if self._refine_beam is False:
raise e
# can we improve the situation?
if self._beam_search_scope < 4.0:
self._beam_search_scope += 4.0
# try repeating the indexing!
self.set_indexer_done(False)
return 'failed'
# otherwise this is beyond redemption
raise e
self._solutions = index.get_solutions()
# FIXME this needs to check the smilie status e.g.
# ":)" or ";(" or " ".
# FIXME need to check the value of the RMSD and raise an
# exception if the P1 solution has an RMSD > 1.0...
# Change 27/FEB/08 to support user assigned spacegroups
# (euugh!) have to "ignore" solutions with higher symmetry
# otherwise the rest of xia will override us. Bummer.
for i, solution in self._solutions.iteritems():
if self._indxr_user_input_lattice:
if (lattice_to_spacegroup(solution['lattice']) >
lattice_to_spacegroup(self._indxr_input_lattice)):
Debug.write('Ignoring solution: %s' % solution['lattice'])
del self._solutions[i]
# check the RMSD from the triclinic unit cell
if self._solutions[1]['rmsd'] > 1.0 and False:
# don't know when this is useful - but I know when it is not!
raise RuntimeError('high RMSD for triclinic solution')
# configure the "right" solution
self._solution = self.get_solution()
# now store also all of the other solutions... keyed by the
# lattice - however these should only be added if they
# have a smiley in the appropriate record, perhaps?
for solution in self._solutions.keys():
lattice = self._solutions[solution]['lattice']
if lattice in self._indxr_other_lattice_cell:
if self._indxr_other_lattice_cell[lattice]['goodness'] < \
self._solutions[solution]['metric']:
continue
self._indxr_other_lattice_cell[lattice] = {
'goodness': self._solutions[solution]['metric'],
'cell': self._solutions[solution]['cell']
}
self._indxr_lattice = self._solution['lattice']
self._indxr_cell = tuple(self._solution['cell'])
self._indxr_mosaic = self._solution['mosaic']
lms = LabelitMosflmScript()
lms.set_working_directory(self.get_working_directory())
lms.set_solution(self._solution['number'])
self._indxr_payload['mosflm_orientation_matrix'] = lms.calculate()
# get the beam centre from the mosflm script - mosflm
# may have inverted the beam centre and labelit will know
# this!
mosflm_beam_centre = lms.get_mosflm_beam()
if mosflm_beam_centre:
self._indxr_payload['mosflm_beam_centre'] = tuple(
mosflm_beam_centre)
import copy
detector = copy.deepcopy(self.get_detector())
beam = copy.deepcopy(self.get_beam())
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
set_mosflm_beam_centre(detector, beam, mosflm_beam_centre)
from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
from scitbx import matrix
from cctbx import sgtbx, uctbx
from dxtbx.model import CrystalFactory
mosflm_matrix = matrix.sqr([
float(i) for line in lms.calculate()
for i in line.replace("-", " -").split()
][:9])
space_group = sgtbx.space_group_info(
lattice_to_spacegroup_number(self._solution['lattice'])).group()
crystal_model = CrystalFactory.from_mosflm_matrix(
mosflm_matrix,
unit_cell=uctbx.unit_cell(tuple(self._solution['cell'])),
space_group=space_group)
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(
beam=beam,
detector=detector,
goniometer=self.get_goniometer(),
scan=self.get_scan(),
crystal=crystal_model,
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# also get an estimate of the resolution limit from the
# labelit.stats_distl output... FIXME the name is wrong!
lsd = LabelitStats_distl()
lsd.set_working_directory(self.get_working_directory())
lsd.stats_distl()
resolution = 1.0e6
for i in _images:
stats = lsd.get_statistics(self.get_image_name(i))
resol = 0.5 * (stats['resol_one'] + stats['resol_two'])
if resol < resolution:
resolution = resol
self._indxr_resolution_estimate = resolution
return 'ok'
from xia2.Applications.xia2 import get_command_line, write_citations, help
def run():
if os.path.exists('xia2-working.phil'):
sys.argv.append('xia2-working.phil')
try:
check_environment()
check()
except exceptions.Exception, e:
traceback.print_exc(file = open('xia2.error', 'w'))
Chatter.write('Status: error "%s"' % str(e))
# print the version
Chatter.write(Version)
Citations.cite('xia2')
start_time = time.time()
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
for crystal_id, crystal in crystals.iteritems():
#cwd = os.path.abspath(os.curdir)
from libtbx import Auto
scale_dir = PhilIndex.params.xia2.settings.scale.directory
if scale_dir is Auto:
scale_dir = 'scale'
i = 0
def _index(self):
"""Actually index the diffraction pattern. Note well that
this is not going to compute the matrix..."""
# acknowledge this program
if not self._indxr_images:
raise RuntimeError("No good spots found on any images")
Citations.cite("labelit")
Citations.cite("distl")
_images = []
for i in self._indxr_images:
for j in i:
if not j in _images:
_images.append(j)
_images.sort()
images_str = "%d" % _images[0]
for i in _images[1:]:
images_str += ", %d" % i
cell_str = None
if self._indxr_input_cell:
cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % self._indxr_input_cell
if self._indxr_sweep_name:
# then this is a proper autoindexing run - describe this
# to the journal entry
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = "...%s" % self._fp_directory[-46:]
Journal.block(
"autoindexing",
self._indxr_sweep_name,
"labelit",
{
"images": images_str,
"target cell": cell_str,
"target lattice": self._indxr_input_lattice,
"template": self._fp_template,
"directory": dirname,
},
)
# auto_logfiler(self)
from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex
index = LabelitIndex()
index.set_working_directory(self.get_working_directory())
auto_logfiler(index)
# task = 'Autoindex from images:'
# for i in _images:
# task += ' %s' % self.get_image_name(i)
# self.set_task(task)
# self.add_command_line('--index_only')
Debug.write("Indexing from images:")
for i in _images:
index.add_image(self.get_image_name(i))
Debug.write("%s" % self.get_image_name(i))
if self._primitive_unit_cell:
index.set_primitive_unit_cell(self._primitive_unit_cell)
if self._indxr_input_cell:
index.set_max_cell(1.25 * max(self._indxr_input_cell[:3]))
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
index.set_distance(xsweep.get_distance())
# if self.get_wavelength_prov() == 'user':
# index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
index.set_beam_centre(xsweep.get_beam_centre())
if self._refine_beam is False:
index.set_refine_beam(False)
else:
index.set_refine_beam(True)
index.set_beam_search_scope(self._beam_search_scope)
if (math.fabs(self.get_wavelength() - 1.54) <
0.01) or (math.fabs(self.get_wavelength() - 2.29) < 0.01):
index.set_Cu_KA_or_Cr_KA(True)
try:
index.run()
except RuntimeError as e:
if self._refine_beam is False:
raise e
# can we improve the situation?
if self._beam_search_scope < 4.0:
self._beam_search_scope += 4.0
# try repeating the indexing!
self.set_indexer_done(False)
return "failed"
# otherwise this is beyond redemption
raise e
self._solutions = index.get_solutions()
# FIXME this needs to check the smilie status e.g.
# ":)" or ";(" or " ".
# FIXME need to check the value of the RMSD and raise an
# exception if the P1 solution has an RMSD > 1.0...
# Change 27/FEB/08 to support user assigned spacegroups
# (euugh!) have to "ignore" solutions with higher symmetry
# otherwise the rest of xia will override us. Bummer.
for i, solution in self._solutions.iteritems():
if self._indxr_user_input_lattice:
if lattice_to_spacegroup(
solution["lattice"]) > lattice_to_spacegroup(
self._indxr_input_lattice):
Debug.write("Ignoring solution: %s" % solution["lattice"])
del self._solutions[i]
# configure the "right" solution
self._solution = self.get_solution()
# now store also all of the other solutions... keyed by the
# lattice - however these should only be added if they
# have a smiley in the appropriate record, perhaps?
for solution in self._solutions.keys():
lattice = self._solutions[solution]["lattice"]
if lattice in self._indxr_other_lattice_cell:
if (self._indxr_other_lattice_cell[lattice]["goodness"] <
self._solutions[solution]["metric"]):
continue
self._indxr_other_lattice_cell[lattice] = {
"goodness": self._solutions[solution]["metric"],
"cell": self._solutions[solution]["cell"],
}
self._indxr_lattice = self._solution["lattice"]
self._indxr_cell = tuple(self._solution["cell"])
self._indxr_mosaic = self._solution["mosaic"]
lms = LabelitMosflmMatrix()
lms.set_working_directory(self.get_working_directory())
lms.set_solution(self._solution["number"])
self._indxr_payload["mosflm_orientation_matrix"] = lms.calculate()
# get the beam centre from the mosflm script - mosflm
# may have inverted the beam centre and labelit will know
# this!
mosflm_beam_centre = lms.get_mosflm_beam()
if mosflm_beam_centre:
self._indxr_payload["mosflm_beam_centre"] = tuple(
mosflm_beam_centre)
detector = copy.deepcopy(self.get_detector())
beam = copy.deepcopy(self.get_beam())
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
set_mosflm_beam_centre(detector, beam, mosflm_beam_centre)
from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
from scitbx import matrix
from cctbx import sgtbx, uctbx
from dxtbx.model import CrystalFactory
mosflm_matrix = matrix.sqr([
float(i) for line in lms.calculate()
for i in line.replace("-", " -").split()
][:9])
space_group = sgtbx.space_group_info(
lattice_to_spacegroup_number(self._solution["lattice"])).group()
crystal_model = CrystalFactory.from_mosflm_matrix(
mosflm_matrix,
unit_cell=uctbx.unit_cell(tuple(self._solution["cell"])),
space_group=space_group,
)
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(
beam=beam,
detector=detector,
goniometer=self.get_goniometer(),
scan=self.get_scan(),
crystal=crystal_model,
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# also get an estimate of the resolution limit from the
# labelit.stats_distl output... FIXME the name is wrong!
lsd = LabelitStats_distl()
lsd.set_working_directory(self.get_working_directory())
lsd.stats_distl()
resolution = 1.0e6
for i in _images:
stats = lsd.get_statistics(self.get_image_name(i))
resol = 0.5 * (stats["resol_one"] + stats["resol_two"])
if resol < resolution:
resolution = resol
self._indxr_resolution_estimate = resolution
return "ok"
def _scale_prepare(self):
'''Prepare the data for scaling - this will reindex it the
reflections to the correct pointgroup and setting, for instance,
and move the reflection files to the scale directory.'''
Citations.cite('xds')
Citations.cite('ccp4')
Citations.cite('pointless')
# GATHER phase - get the reflection files together... note that
# it is not necessary in here to keep the batch information as we
# don't wish to rebatch the reflections prior to scaling.
# FIXME need to think about what I will do about the radiation
# damage analysis in here...
self._sweep_information = { }
# FIXME in here I want to record the batch number to
# epoch mapping as per the CCP4 Scaler implementation.
Journal.block(
'gathering', self.get_scaler_xcrystal().get_name(), 'XDS',
{'working directory':self.get_working_directory()})
for epoch in self._scalr_integraters.keys():
intgr = self._scalr_integraters[epoch]
pname, xname, dname = intgr.get_integrater_project_info()
sname = intgr.get_integrater_sweep_name()
self._sweep_information[epoch] = {
'pname':pname,
'xname':xname,
'dname':dname,
'integrater':intgr,
'corrected_intensities':intgr.get_integrater_corrected_intensities(),
'prepared_reflections':None,
'scaled_reflections':None,
'header':intgr.get_header(),
'batches':intgr.get_integrater_batches(),
'image_to_epoch':intgr.get_integrater_sweep(
).get_image_to_epoch(),
'image_to_dose':{},
'batch_offset':0,
'sname':sname
}
Journal.entry({'adding data from':'%s/%s/%s' % \
(xname, dname, sname)})
# what are these used for?
# pname / xname / dname - dataset identifiers
# image to epoch / batch offset / batches - for RD analysis
Debug.write('For EPOCH %s have:' % str(epoch))
Debug.write('ID = %s/%s/%s' % (pname, xname, dname))
Debug.write('SWEEP = %s' % intgr.get_integrater_sweep_name())
# next work through all of the reflection files and make sure that
# they are XDS_ASCII format...
epochs = self._sweep_information.keys()
epochs.sort()
self._first_epoch = min(epochs)
self._scalr_pname = self._sweep_information[epochs[0]]['pname']
self._scalr_xname = self._sweep_information[epochs[0]]['xname']
for epoch in epochs:
intgr = self._scalr_integraters[epoch]
pname = self._sweep_information[epoch]['pname']
xname = self._sweep_information[epoch]['xname']
dname = self._sweep_information[epoch]['dname']
sname = self._sweep_information[epoch]['sname']
if self._scalr_pname != pname:
raise RuntimeError, 'all data must have a common project name'
xname = self._sweep_information[epoch]['xname']
if self._scalr_xname != xname:
raise RuntimeError, \
'all data for scaling must come from one crystal'
xsh = XDSScalerHelper()
xsh.set_working_directory(self.get_working_directory())
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_CORRECTED.HKL' %(
pname, xname, dname, sname))
sweep = intgr.get_integrater_sweep()
if sweep.get_frames_to_process() is not None:
offset = intgr.get_frame_offset()
#print "offset: %d" %offset
start, end = sweep.get_frames_to_process()
start -= offset
end -= offset
#end += 1 ????
#print "limiting batches: %d-%d" %(start, end)
xsh.limit_batches(hklin, hklout, start, end)
self._sweep_information[epoch]['corrected_intensities'] = hklout
# if there is more than one sweep then compare the lattices
# and eliminate all but the lowest symmetry examples if
# there are more than one...
# -------------------------------------------------
# Ensure that the integration lattices are the same
# -------------------------------------------------
need_to_return = False
if len(self._sweep_information.keys()) > 1:
lattices = []
# FIXME run this stuff in parallel as well...
for epoch in self._sweep_information.keys():
intgr = self._sweep_information[epoch]['integrater']
hklin = self._sweep_information[epoch]['corrected_intensities']
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
ntr = False
else:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattice = Syminfo.get_lattice(pointgroup)
if not lattice in lattices:
lattices.append(lattice)
if ntr:
# if we need to return, we should logically reset
# any reindexing operator right? right here all
# we are talking about is the correctness of
# individual pointgroups?? Bug # 3373
reindex_op = 'h,k,l'
# actually, should this not be done "by magic"
# when a new pointgroup is assigned in the
# pointless indexer jiffy above?!
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
need_to_return = True
# bug # 2433 - need to ensure that all of the lattice
# conclusions were the same...
if len(lattices) > 1:
ordered_lattices = []
for l in lattices_in_order():
if l in lattices:
ordered_lattices.append(l)
correct_lattice = ordered_lattices[0]
Debug.write('Correct lattice asserted to be %s' % \
correct_lattice)
# transfer this information back to the indexers
for epoch in self._sweep_information.keys():
integrater = self._sweep_information[
epoch]['integrater']
refiner = integrater.get_integrater_refiner()
sname = integrater.get_integrater_sweep_name()
if not refiner:
continue
state = refiner.set_refiner_asserted_lattice(
correct_lattice)
if state == refiner.LATTICE_CORRECT:
Debug.write('Lattice %s ok for sweep %s' % \
(correct_lattice, sname))
elif state == refiner.LATTICE_IMPOSSIBLE:
raise RuntimeError, 'Lattice %s impossible for %s' % \
(correct_lattice, sname)
elif state == refiner.LATTICE_POSSIBLE:
Debug.write('Lattice %s assigned for sweep %s' % \
(correct_lattice, sname))
need_to_return = True
# if one or more of them was not in the lowest lattice,
# need to return here to allow reprocessing
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# next if there is more than one sweep then generate
# a merged reference reflection file to check that the
# setting for all reflection files is the same...
# if we get to here then all data was processed with the same
# lattice
# ----------------------------------------------------------
# next ensure that all sweeps are set in the correct setting
# ----------------------------------------------------------
if self.get_scaler_reference_reflection_file():
self._reference = self.get_scaler_reference_reflection_file()
Debug.write('Using HKLREF %s' % self._reference)
md = self._factory.Mtzdump()
md.set_hklin(self.get_scaler_reference_reflection_file())
md.dump()
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(
md.get_spacegroup())
Debug.write('Spacegroup %d' % self._xds_spacegroup)
elif PhilIndex.params.xia2.settings.scale.reference_reflection_file:
self._reference = PhilIndex.params.xia2.settings.scale.reference_reflection_file
Debug.write('Using HKLREF %s' % self._reference)
md = self._factory.Mtzdump()
md.set_hklin(PhilIndex.params.xia2.settings.scale.reference_reflection_file)
md.dump()
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(
md.get_spacegroup())
Debug.write('Spacegroup %d' % self._xds_spacegroup)
params = PhilIndex.params
use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs
if len(self._sweep_information.keys()) > 1 and use_brehm_diederichs:
brehm_diederichs_files_in = []
for epoch in self._sweep_information.keys():
intgr = self._sweep_information[epoch]['integrater']
hklin = self._sweep_information[epoch]['corrected_intensities']
refiner = intgr.get_integrater_refiner()
# in here need to consider what to do if the user has
# assigned the pointgroup on the command line ...
if not self._scalr_input_pointgroup:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
if ntr:
# Bug # 3373
Debug.write('Reindex to standard (PIJ): %s' % \
reindex_op)
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
else:
# 27/FEB/08 to support user assignment of pointgroups
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
self._sweep_information[epoch]['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# convert the XDS_ASCII for this sweep to mtz - on the next
# get this should be in the correct setting...
dname = self._sweep_information[epoch]['dname']
sname = intgr.get_integrater_sweep_name()
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.mtz' % (dname, sname))
FileHandler.record_temporary_file(hklout)
# now use pointless to make this conversion
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
brehm_diederichs_files_in.append(hklout)
# now run cctbx.brehm_diederichs to figure out the indexing hand for
# each sweep
from xia2.Wrappers.Cctbx.BrehmDiederichs import BrehmDiederichs
brehm_diederichs = BrehmDiederichs()
brehm_diederichs.set_working_directory(self.get_working_directory())
auto_logfiler(brehm_diederichs)
brehm_diederichs.set_input_filenames(brehm_diederichs_files_in)
# 1 or 3? 1 seems to work better?
brehm_diederichs.set_asymmetric(1)
brehm_diederichs.run()
reindexing_dict = brehm_diederichs.get_reindexing_dict()
for epoch in self._sweep_information.keys():
intgr = self._sweep_information[epoch]['integrater']
dname = self._sweep_information[epoch]['dname']
sname = intgr.get_integrater_sweep_name()
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.mtz' % (dname, sname))
# apply the reindexing operator
intgr.set_integrater_reindex_operator(reindex_op)
# and copy the reflection file to the local directory
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
self._sweep_information[epoch][
'prepared_reflections'] = os.path.split(hklout)[-1]
elif len(self._sweep_information.keys()) > 1 and \
not self._reference:
# need to generate a reference reflection file - generate this
# from the reflections in self._first_epoch
#
# FIXME this should really use the Brehm and Diederichs method
# if you have lots of little sweeps...
intgr = self._sweep_information[self._first_epoch]['integrater']
hklin = self._sweep_information[epoch]['corrected_intensities']
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
ntr = False
reindex_op = 'h,k,l'
else:
pointgroup, reindex_op, ntr = self._pointless_indexer_jiffy(
hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
reference_reindex_op = intgr.get_integrater_reindex_operator()
if ntr:
# Bug # 3373
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(pointgroup)
# next pass this reindexing operator back to the source
# of the reflections
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
self._sweep_information[epoch]['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'xds-pointgroup-reference-unsorted.mtz')
FileHandler.record_temporary_file(hklout)
# now use pointless to handle this conversion
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
self._reference = hklout
if self._reference:
from xia2.Driver.DriverFactory import DriverFactory
def run_one_sweep(args):
sweep_information = args[0]
pointless_indexer_jiffy = args[1]
factory = args[2]
job_type = args[3]
if job_type:
DriverFactory.set_driver_type(job_type)
intgr = sweep_information['integrater']
hklin = sweep_information['corrected_intensities']
refiner = intgr.get_integrater_refiner()
# in here need to consider what to do if the user has
# assigned the pointgroup on the command line ...
if not self._scalr_input_pointgroup:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
if ntr:
# Bug # 3373
Debug.write('Reindex to standard (PIJ): %s' % \
reindex_op)
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
else:
# 27/FEB/08 to support user assignment of pointgroups
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
sweep_information['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# convert the XDS_ASCII for this sweep to mtz - on the next
# get this should be in the correct setting...
hklin = sweep_information['corrected_intensities']
# now use pointless to make this conversion
# try with no conversion?!
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
hklout = os.path.join(
self.get_working_directory(),
'%d_xds-pointgroup-unsorted.mtz' %pointless.get_xpid())
FileHandler.record_temporary_file(hklout)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
pointless = self._factory.Pointless()
pointless.set_hklin(hklout)
pointless.set_hklref(self._reference)
pointless.decide_pointgroup()
pointgroup = pointless.get_pointgroup()
reindex_op = pointless.get_reindex_operator()
# for debugging print out the reindexing operations and
# what have you...
Debug.write('Reindex to standard: %s' % reindex_op)
# this should send back enough information that this
# is in the correct pointgroup (from the call above) and
# also in the correct setting, from the interaction
# with the reference set... - though I guess that the
# spacegroup number should not have changed, right?
# set the reindex operation afterwards... though if the
# spacegroup number is the same this should make no
# difference, right?!
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
intgr.set_integrater_reindex_operator(reindex_op)
sweep_information['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# and copy the reflection file to the local directory
dname = sweep_information['dname']
sname = intgr.get_integrater_sweep_name()
hklin = sweep_information['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
sweep_information['prepared_reflections'] = os.path.split(hklout)[-1]
return sweep_information
from libtbx import easy_mp
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
if njob > 1:
# cache drivertype
drivertype = DriverFactory.get_driver_type()
args = [
(self._sweep_information[epoch], self._pointless_indexer_jiffy,
self._factory, mp_params.type)
for epoch in self._sweep_information.keys()]
results_list = easy_mp.parallel_map(
run_one_sweep, args, params=None,
processes=njob,
method="threading",
asynchronous=True,
callback=None,
preserve_order=True,
preserve_exception_message=True)
# restore drivertype
DriverFactory.set_driver_type(drivertype)
# results should be given back in the same order
for i, epoch in enumerate(self._sweep_information.keys()):
self._sweep_information[epoch] = results_list[i]
else:
for epoch in self._sweep_information.keys():
self._sweep_information[epoch] = run_one_sweep(
(self._sweep_information[epoch], self._pointless_indexer_jiffy,
self._factory, None))
else:
# convert the XDS_ASCII for this sweep to mtz
epoch = self._first_epoch
intgr = self._sweep_information[epoch]['integrater']
refiner = intgr.get_integrater_refiner()
sname = intgr.get_integrater_sweep_name()
hklout = os.path.join(self.get_working_directory(),
'%s-pointless.mtz' % sname)
FileHandler.record_temporary_file(hklout)
pointless = self._factory.Pointless()
pointless.set_xdsin(self._sweep_information[epoch]['corrected_intensities'])
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
# run it through pointless interacting with the
# Indexer which belongs to this sweep
hklin = hklout
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
ntr = False
reindex_op = 'h,k,l'
else:
pointgroup, reindex_op, ntr = self._pointless_indexer_jiffy(
hklin, refiner)
if ntr:
# if we need to return, we should logically reset
# any reindexing operator right? right here all
# we are talking about is the correctness of
# individual pointgroups?? Bug # 3373
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
need_to_return = True
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(pointgroup)
# next pass this reindexing operator back to the source
# of the reflections
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
self._sweep_information[epoch]['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
hklin = self._sweep_information[epoch]['corrected_intensities']
dname = self._sweep_information[epoch]['dname']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
# and copy the reflection file to the local
# directory
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
self._sweep_information[epoch][
'prepared_reflections'] = os.path.split(hklout)[-1]
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
unit_cell_list = []
for epoch in self._sweep_information.keys():
integrater = self._sweep_information[epoch]['integrater']
cell = integrater.get_integrater_cell()
n_ref = integrater.get_integrater_n_ref()
Debug.write('Cell for %s: %.2f %.2f %.2f %.2f %.2f %.2f' % \
(integrater.get_integrater_sweep_name(),
cell[0], cell[1], cell[2],
cell[3], cell[4], cell[5]))
Debug.write('=> %d reflections' % n_ref)
unit_cell_list.append((cell, n_ref))
self._scalr_cell = compute_average_unit_cell(unit_cell_list)
self._scalr_resolution_limits = { }
Debug.write('Determined unit cell: %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(self._scalr_cell))
if os.path.exists(os.path.join(
self.get_working_directory(),
'REMOVE.HKL')):
os.remove(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'))
Debug.write('Deleting REMOVE.HKL at end of scale prepare.')
return
def _index(self):
'''Implement the indexer interface.'''
Citations.cite('mosflm')
indexer = MosflmIndex()
indexer.set_working_directory(self.get_working_directory())
auto_logfiler(indexer)
from xia2.lib.bits import unique_elements
_images = unique_elements(self._indxr_images)
indexer.set_images(_images)
images_str = ', '.join(map(str, _images))
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
if self._indxr_sweep_name:
#if len(self._fp_directory) <= 50:
#dirname = self._fp_directory
#else:
#dirname = '...%s' % self._fp_directory[-46:]
dirname = os.path.dirname(self.get_imageset().get_template())
Journal.block(
'autoindexing', self._indxr_sweep_name, 'mosflm',
{'images':images_str,
'target cell':self._indxr_input_cell,
'target lattice':self._indxr_input_lattice,
'template':self.get_imageset().get_template(),
'directory':dirname})
#task = 'Autoindex from images:'
#for i in _images:
#task += ' %s' % self.get_image_name(i)
#self.set_task(task)
indexer.set_template(os.path.basename(self.get_template()))
indexer.set_directory(self.get_directory())
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
indexer.set_distance(xsweep.get_distance())
#if self.get_wavelength_prov() == 'user':
#index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
indexer.set_beam_centre(xsweep.get_beam_centre())
if self._indxr_input_cell:
indexer.set_unit_cell(self._indxr_input_cell)
if self._indxr_input_lattice != None:
spacegroup_number = lattice_to_spacegroup(
self._indxr_input_lattice)
indexer.set_space_group_number(spacegroup_number)
if not self._mosflm_autoindex_thresh:
try:
min_peaks = 200
Debug.write('Aiming for at least %d spots...' % min_peaks)
thresholds = []
for i in _images:
p = Printpeaks()
p.set_working_directory(self.get_working_directory())
auto_logfiler(p)
p.set_image(self.get_image_name(i))
thresh = p.threshold(min_peaks)
Debug.write('Autoindex threshold for image %d: %d' % \
(i, thresh))
thresholds.append(thresh)
thresh = min(thresholds)
self._mosflm_autoindex_thresh = thresh
except Exception as e:
print str(e) #XXX this should disappear!
Debug.write('Error computing threshold: %s' % str(e))
Debug.write('Using default of 20.0')
thresh = 20.0
else:
thresh = self._mosflm_autoindex_thresh
Debug.write('Using autoindex threshold: %d' % thresh)
if self._mosflm_autoindex_sol:
indexer.set_solution_number(self._mosflm_autoindex_sol)
indexer.set_threshold(thresh)
# now forget this to prevent weird things happening later on
if self._mosflm_autoindex_sol:
self._mosflm_autoindex_sol = 0
indexer.run()
#sweep = self.get_indexer_sweep_name()
#FileHandler.record_log_file(
#'%s INDEX' % (sweep), self.get_log_file())
indxr_cell = indexer.get_refined_unit_cell()
self._indxr_lattice = indexer.get_lattice()
space_group_number = indexer.get_indexed_space_group_number()
detector_distance = indexer.get_refined_distance()
beam_centre = indexer.get_refined_beam_centre()
mosaic_spreads = indexer.get_mosaic_spreads()
if min(list(indxr_cell)) < 10.0 and \
indxr_cell[2] / indxr_cell[0] > 6:
Debug.write(
'Unrealistic autoindexing solution: ' +
'%.2f %.2f %.2f %.2f %.2f %.2f' % indxr_cell)
# tweak some parameters and try again...
self._mosflm_autoindex_thresh *= 1.5
self.set_indexer_done(False)
return
intgr_params = { }
# look up other possible indexing solutions (not well - in
# standard settings only!) This is moved earlier as it could
# result in returning if Mosflm has selected the wrong
# solution!
try:
self._indxr_other_lattice_cell = indexer.get_solutions()
# Change 27/FEB/08 to support user assigned spacegroups
if self._indxr_user_input_lattice:
lattice_to_spacegroup_dict = {
'aP':1, 'mP':3, 'mC':5, 'oP':16, 'oC':20, 'oF':22,
'oI':23, 'tP':75, 'tI':79, 'hP':143, 'hR':146,
'cP':195, 'cF':196, 'cI':197}
for k in self._indxr_other_lattice_cell.keys():
if lattice_to_spacegroup_dict[k] > \
lattice_to_spacegroup_dict[
self._indxr_input_lattice]:
del(self._indxr_other_lattice_cell[k])
# check that the selected unit cell matches - and if
# not raise a "horrible" exception
if self._indxr_input_cell:
assert indxr_cell is not None
for j in range(6):
if math.fabs(self._indxr_input_cell[j] - indxr_cell[j]) > 2.0:
Chatter.write(
'Mosflm autoindexing did not select ' +
'correct (target) unit cell')
raise RuntimeError, \
'something horrible happened in indexing'
except RuntimeError, e:
# check if mosflm rejected a solution we have it
if 'horribl' in str(e):
# ok it did - time to break out the big guns...
if not self._indxr_input_cell:
raise RuntimeError, \
'error in solution selection when not preset'
# XXX FIXME
self._mosflm_autoindex_sol = _get_indexing_solution_number(
indexer.get_all_output(),
self._indxr_input_cell,
self._indxr_input_lattice)
# set the fact that we are not done...
self.set_indexer_done(False)
# and return - hopefully this will restart everything
return
else:
raise e
def multi_crystal_analysis(stop_after=None):
'''Actually process something...'''
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
for crystal_id, crystal in crystals.iteritems():
cwd = os.path.abspath(os.curdir)
working_directory = Environment.generate_directory(
[crystal.get_name(), 'analysis'])
os.chdir(working_directory)
scaler = crystal._get_scaler()
#epoch_to_si = {}
epoch_to_batches = {}
epoch_to_integrated_intensities = {}
epoch_to_sweep_name = {}
epoch_to_experiments_filename = {}
epoch_to_experiments = {}
sweep_name_to_epoch = {}
epoch_to_first_image = {}
from dxtbx.serialize import load
try:
epochs = scaler._sweep_information.keys()
for epoch in epochs:
si = scaler._sweep_information[epoch]
epoch_to_batches[epoch] = si['batches']
epoch_to_integrated_intensities[epoch] = si[
'corrected_intensities']
epoch_to_sweep_name[epoch] = si['sname']
sweep_name_to_epoch[si['name']] = epoch
intgr = si['integrater']
epoch_to_experiments_filename[
epoch] = intgr.get_integrated_experiments()
epoch_to_experiments[epoch] = load.experiment_list(
intgr.get_integrated_experiments())
except AttributeError:
epochs = scaler._sweep_handler.get_epochs()
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
epoch_to_batches[epoch] = si.get_batches()
epoch_to_integrated_intensities[epoch] = si.get_reflections()
epoch_to_sweep_name[epoch] = si.get_sweep_name()
sweep_name_to_epoch[si.get_sweep_name()] = epoch
intgr = si.get_integrater()
epoch_to_experiments_filename[
epoch] = intgr.get_integrated_experiments()
epoch_to_experiments[epoch] = load.experiment_list(
intgr.get_integrated_experiments())
from xia2.Wrappers.Dials.StereographicProjection import StereographicProjection
sp_json_files = {}
for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)):
sp = StereographicProjection()
auto_logfiler(sp)
sp.set_working_directory(working_directory)
for experiments in epoch_to_experiments_filename.values():
sp.add_experiments(experiments)
sp.set_hkl(hkl)
sp.run()
sp_json_files[hkl] = sp.get_json_filename()
unmerged_mtz = scaler.get_scaled_reflections(
'mtz_unmerged').values()[0]
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(unmerged_mtz)
from xia2.Wrappers.XIA.PlotMultiplicity import PlotMultiplicity
mult_json_files = {}
for axis in ('h', 'k', 'l'):
pm = PlotMultiplicity()
auto_logfiler(pm)
pm.set_working_directory(working_directory)
pm.set_mtz_filename(unmerged_mtz)
pm.set_slice_axis(axis)
pm.set_show_missing(True)
pm.run()
mult_json_files[axis] = pm.get_json_filename()
intensities = None
batches = None
assert reader.file_type() == 'ccp4_mtz'
arrays = reader.as_miller_arrays(merge_equivalents=False)
for ma in arrays:
if ma.info().labels == ['BATCH']:
batches = ma
elif ma.info().labels == ['I', 'SIGI']:
intensities = ma
elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
intensities = ma
from xia2.Wrappers.CCP4.Blend import Blend
hand_blender = Blend()
hand_blender.set_working_directory(working_directory)
auto_logfiler(hand_blender)
Citations.cite('blend')
from xia2.Handlers.Environment import which
Rscript_binary = which('Rscript', debug=False)
if Rscript_binary is None:
Chatter.write('Skipping BLEND analysis: Rscript not available')
else:
for epoch in epochs:
hand_blender.add_hklin(epoch_to_integrated_intensities[epoch],
label=epoch_to_sweep_name[epoch])
hand_blender.analysis()
Chatter.write("Dendrogram saved to: %s" %
hand_blender.get_dendrogram_file())
analysis = hand_blender.get_analysis()
summary = hand_blender.get_summary()
clusters = hand_blender.get_clusters()
ddict = hand_blender.plot_dendrogram()
phil_files_dir = 'phil_files'
if not os.path.exists(phil_files_dir):
os.makedirs(phil_files_dir)
rows = []
headers = [
'Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'LCV',
'aLCV', 'Average unit cell'
]
completeness = flex.double()
average_unit_cell_params = []
for i, cluster in clusters.iteritems():
print i
sel_cluster = flex.bool(batches.size(), False)
cluster_uc_params = [flex.double() for k in range(6)]
for j in cluster['dataset_ids']:
epoch = epochs[j - 1]
batch_start, batch_end = epoch_to_batches[epoch]
sel_cluster |= ((batches.data() >= batch_start) &
(batches.data() <= batch_end))
expts = epoch_to_experiments.get(epoch)
assert expts is not None, (epoch)
assert len(expts) == 1, len(expts)
expt = expts[0]
uc_params = expt.crystal.get_unit_cell().parameters()
for k in range(6):
cluster_uc_params[k].append(uc_params[k])
intensities_cluster = intensities.select(sel_cluster)
merging = intensities_cluster.merge_equivalents()
merged_intensities = merging.array()
multiplicities = merging.redundancies()
completeness.append(merged_intensities.completeness())
average_unit_cell_params.append(
tuple(flex.mean(p) for p in cluster_uc_params))
dataset_ids = cluster['dataset_ids']
assert min(dataset_ids) > 0
with open(
os.path.join(phil_files_dir,
'blend_cluster_%i_images.phil' % i),
'wb') as f:
sweep_names = [
hand_blender._labels[dataset_id - 1]
for dataset_id in dataset_ids
]
for sweep_name in sweep_names:
expts = epoch_to_experiments.get(
sweep_name_to_epoch.get(sweep_name))
assert expts is not None, (
sweep_name, sweep_name_to_epoch.get(sweep_name))
assert len(expts) == 1, len(expts)
expt = expts[0]
print >> f, 'xia2.settings.input.image = %s' % expt.imageset.get_path(
0)
rows.append([
'%i' % i,
' '.join(['%i'] * len(dataset_ids)) % tuple(dataset_ids),
'%.1f' % flex.mean(multiplicities.data().as_double()),
'%.2f' % completeness[-1],
'%.2f' % cluster['lcv'],
'%.2f' % cluster['alcv'],
'%g %g %g %g %g %g' % average_unit_cell_params[-1]
])
# sort table by completeness
perm = flex.sort_permutation(completeness)
rows = [rows[i] for i in perm]
print
print 'Unit cell clustering summary:'
print tabulate(rows, headers, tablefmt='rst')
print
blend_html = tabulate(rows, headers, tablefmt='html').replace(
'<table>',
'<table class="table table-hover table-condensed">').replace(
'<td>', '<td style="text-align: right;">')
# XXX what about multiple wavelengths?
with open('batches.phil', 'wb') as f:
try:
for epoch, si in scaler._sweep_information.iteritems():
print >> f, "batch {"
print >> f, " id=%s" % si['sname']
print >> f, " range=%i,%i" % tuple(si['batches'])
print >> f, "}"
except AttributeError:
for epoch in scaler._sweep_handler.get_epochs():
si = scaler._sweep_handler.get_sweep_information(epoch)
print >> f, "batch {"
print >> f, " id=%s" % si.get_sweep_name()
print >> f, " range=%i,%i" % tuple(si.get_batches())
print >> f, "}"
from xia2.Wrappers.XIA.MultiCrystalAnalysis import MultiCrystalAnalysis
mca = MultiCrystalAnalysis()
auto_logfiler(mca, extra="MultiCrystalAnalysis")
mca.add_command_line_args([
scaler.get_scaled_reflections(format="sca_unmerged").values()[0],
"unit_cell=%s %s %s %s %s %s" % tuple(scaler.get_scaler_cell()),
"batches.phil"
])
mca.set_working_directory(working_directory)
mca.run()
intensity_clusters = mca.get_clusters()
rows = []
headers = [
'Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'Height',
'Average unit cell'
]
completeness = flex.double()
average_unit_cell_params = []
for i, cluster in intensity_clusters.iteritems():
sel_cluster = flex.bool(batches.size(), False)
cluster_uc_params = [flex.double() for k in range(6)]
for j in cluster['datasets']:
epoch = epochs[j - 1]
batch_start, batch_end = epoch_to_batches[epoch]
sel_cluster |= ((batches.data() >= batch_start) &
(batches.data() <= batch_end))
expts = epoch_to_experiments.get(epoch)
assert expts is not None, (epoch)
assert len(expts) == 1, len(expts)
expt = expts[0]
uc_params = expt.crystal.get_unit_cell().parameters()
for k in range(6):
cluster_uc_params[k].append(uc_params[k])
intensities_cluster = intensities.select(sel_cluster)
merging = intensities_cluster.merge_equivalents()
merged_intensities = merging.array()
multiplicities = merging.redundancies()
completeness.append(merged_intensities.completeness())
average_unit_cell_params.append(
tuple(flex.mean(p) for p in cluster_uc_params))
dataset_ids = cluster['datasets']
rows.append([
'%i' % int(i),
' '.join(['%i'] * len(dataset_ids)) % tuple(dataset_ids),
'%.1f' % flex.mean(multiplicities.data().as_double()),
'%.2f' % completeness[-1],
'%.2f' % cluster['height'],
'%g %g %g %g %g %g' % average_unit_cell_params[-1]
])
# sort table by completeness
perm = flex.sort_permutation(completeness)
rows = [rows[i] for i in perm]
print 'Intensity clustering summary:'
print tabulate(rows, headers, tablefmt='rst')
print
intensity_clustering_html = tabulate(
rows, headers, tablefmt='html').replace(
'<table>',
'<table class="table table-hover table-condensed">').replace(
'<td>', '<td style="text-align: right;">')
import json
json_data = {}
if ddict is not None:
from xia2.Modules.MultiCrystalAnalysis import scipy_dendrogram_to_plotly_json
json_data['blend_dendrogram'] = scipy_dendrogram_to_plotly_json(ddict)
else:
json_data['blend_dendrogram'] = {'data': [], 'layout': {}}
json_data['intensity_clustering'] = mca.get_dict()
del json_data['intensity_clustering']['clusters']
for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)):
with open(sp_json_files[hkl], 'rb') as f:
d = json.load(f)
d['layout'][
'title'] = 'Stereographic projection (hkl=%i%i%i)' % hkl
json_data['stereographic_projection_%s%s%s' % hkl] = d
for axis in ('h', 'k', 'l'):
with open(mult_json_files[axis], 'rb') as f:
json_data['multiplicity_%s' % axis] = json.load(f)
json_str = json.dumps(json_data, indent=2)
javascript = ['var graphs = %s' % (json_str)]
javascript.append(
'Plotly.newPlot(blend_dendrogram, graphs.blend_dendrogram.data, graphs.blend_dendrogram.layout);'
)
javascript.append(
'Plotly.newPlot(intensity_clustering, graphs.intensity_clustering.data, graphs.intensity_clustering.layout);'
)
for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)):
javascript.append(
'Plotly.newPlot(stereographic_projection_%(hkl)s, graphs.stereographic_projection_%(hkl)s.data, graphs.stereographic_projection_%(hkl)s.layout);'
% ({
'hkl': "%s%s%s" % hkl
}))
for axis in ('h', 'k', 'l'):
javascript.append(
'Plotly.newPlot(multiplicity_%(axis)s, graphs.multiplicity_%(axis)s.data, graphs.multiplicity_%(axis)s.layout);'
% ({
'axis': axis
}))
html_header = '''
<head>
<!-- Plotly.js -->
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
<meta name="viewport" content="width=device-width, initial-scale=1" charset="UTF-8">
<link rel="stylesheet" href="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css"/>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
<script src="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
<style type="text/css">
body {
/*font-family: Helmet, Freesans, Helvetica, Arial, sans-serif;*/
margin: 8px;
min-width: 240px;
margin-left: 5%;
margin-right: 5%;
}
.plot {
float: left;
width: 1200px;
height: 800px;
margin-bottom: 20px;
}
.square_plot {
float: left;
width: 800px;
height: 800px;
margin-bottom: 20px;
}
</style>
</head>
'''
html_body = '''
<body>
<div class="page-header">
<h1>Multi-crystal analysis report</h1>
</div>
<div class="panel-group">
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_multiplicity">
<h4 class="panel-title">
<a>Multiplicity plots</a>
</h4>
</div>
<div id="collapse_multiplicity" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_h"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_k"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_l"></div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_stereographic_projection">
<h4 class="panel-title">
<a>Stereographic projections</a>
</h4>
</div>
<div id="collapse_stereographic_projection" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_100"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_010"></div>
<div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_001"></div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_cell">
<h4 class="panel-title">
<a>Unit cell clustering</a>
</h4>
</div>
<div id="collapse_cell" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 plot" id="blend_dendrogram"></div>
<div class="table-responsive" style="width: 800px">
%(blend_html)s
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" data-toggle="collapse" href="#collapse_intensity">
<h4 class="panel-title">
<a>Intensity clustering</a>
</h4>
</div>
<div id="collapse_intensity" class="panel-collapse collapse">
<div class="panel-body">
<div class="col-xs-12 col-sm-12 col-md-12 plot" id="intensity_clustering" style="height:1000px"></div>
<div class="table-responsive" style="width: 800px">
%(intensity_clustering_html)s
</div>
</div>
</div>
</div>
</div>
<script>
%(script)s
</script>
</body>
''' % {
'script': '\n'.join(javascript),
'blend_html': blend_html,
'intensity_clustering_html': intensity_clustering_html
}
html = '\n'.join([html_header, html_body])
print "Writing html report to: %s" % 'multi-crystal-report.html'
with open('multi-crystal-report.html', 'wb') as f:
print >> f, html.encode('ascii', 'xmlcharrefreplace')
write_citations()
Environment.cleanup()
return
def _index(self):
'''Actually index the diffraction pattern. Note well that
this is not going to compute the matrix...'''
# acknowledge this program
if not self._indxr_images:
raise RuntimeError, 'No good spots found on any images'
Citations.cite('labelit')
Citations.cite('distl')
_images = []
for i in self._indxr_images:
for j in i:
if not j in _images:
_images.append(j)
_images.sort()
images_str = '%d' % _images[0]
for i in _images[1:]:
images_str += ', %d' % i
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
if self._indxr_sweep_name:
# then this is a proper autoindexing run - describe this
# to the journal entry
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = '...%s' % self._fp_directory[-46:]
Journal.block(
'autoindexing', self._indxr_sweep_name, 'labelit',
{'images':images_str,
'target cell':cell_str,
'target lattice':self._indxr_input_lattice,
'template':self._fp_template,
'directory':dirname})
#auto_logfiler(self)
from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex
index = LabelitIndex()
index.set_working_directory(self.get_working_directory())
auto_logfiler(index)
#task = 'Autoindex from images:'
#for i in _images:
#task += ' %s' % self.get_image_name(i)
#self.set_task(task)
#self.add_command_line('--index_only')
Debug.write('Indexing from images:')
for i in _images:
index.add_image(self.get_image_name(i))
Debug.write('%s' % self.get_image_name(i))
if self._indxr_input_lattice and False:
index.set_space_group_number(
lattice_to_spacegroup(self._indxr_input_lattice))
if self._primitive_unit_cell:
index.set_primitive_unit_cell(self._primitive_unit_cell)
if self._indxr_input_cell:
index.set_max_cell(1.25 * max(self._indxr_input_cell[:3]))
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
index.set_distance(xsweep.get_distance())
#if self.get_wavelength_prov() == 'user':
#index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
index.set_beam_centre(xsweep.get_beam_centre())
if self._refine_beam is False:
index.set_refine_beam(False)
else:
index.set_refine_beam(True)
index.set_beam_search_scope(self._beam_search_scope)
if ((math.fabs(self.get_wavelength() - 1.54) < 0.01) or
(math.fabs(self.get_wavelength() - 2.29) < 0.01)):
index.set_Cu_KA_or_Cr_KA(True)
try:
index.run()
except RuntimeError, e:
if self._refine_beam is False:
raise e
# can we improve the situation?
if self._beam_search_scope < 4.0:
self._beam_search_scope += 4.0
# try repeating the indexing!
self.set_indexer_done(False)
return 'failed'
# otherwise this is beyond redemption
raise e
def _integrate_prepare(self):
'''Prepare for integration - in XDS terms this may mean rerunning
IDXREF to get the XPARM etc. DEFPIX is considered part of the full
integration as it is resolution dependent.'''
from xia2.Handlers.Citations import Citations
Citations.cite('dials')
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(min(images), max(images))
Debug.write('DIALS INTEGRATE PREPARE:')
Debug.write('Wavelength: %.6f' % self.get_wavelength())
Debug.write('Distance: %.2f' % self.get_distance())
#if not self._intgr_indexer:
#self.set_integrater_indexer(DialsIndexer())
#self.get_integrater_indexer().set_indexer_sweep(
#self.get_integrater_sweep())
#self._intgr_indexer.set_working_directory(
#self.get_working_directory())
#self._intgr_indexer.setup_from_imageset(self.get_imageset())
#if self.get_frame_wedge():
#wedge = self.get_frame_wedge()
#Debug.write('Propogating wedge limit: %d %d' % wedge)
#self._intgr_indexer.set_frame_wedge(wedge[0], wedge[1],
#apply_offset = False)
## this needs to be set up from the contents of the
## Integrater frame processer - wavelength &c.
#if self.get_beam_centre():
#self._intgr_indexer.set_beam_centre(self.get_beam_centre())
#if self.get_distance():
#self._intgr_indexer.set_distance(self.get_distance())
#if self.get_wavelength():
#self._intgr_indexer.set_wavelength(
#self.get_wavelength())
## get the unit cell from this indexer to initiate processing
## if it is new... and also copy out all of the information for
## the Dials indexer if not...
#experiments = self._intgr_indexer.get_indexer_experiment_list()
#assert len(experiments) == 1 # currently only handle one lattice/sweep
#experiment = experiments[0]
#crystal_model = experiment.crystal
#lattice = self._intgr_indexer.get_indexer_lattice()
## check if the lattice was user assigned...
#user_assigned = self._intgr_indexer.get_indexer_user_input_lattice()
# XXX check that the indexer is an Dials indexer - if not then
# create one...
# set a low resolution limit (which isn't really used...)
# this should perhaps be done more intelligently from an
# analysis of the spot list or something...?
if not self.get_integrater_low_resolution():
dmax = self._intgr_refiner.get_indexer_low_resolution(
self.get_integrater_epoch())
self.set_integrater_low_resolution(dmax)
Debug.write('Low resolution set to: %s' % \
self.get_integrater_low_resolution())
## copy the data across
from dxtbx.serialize import load
refiner = self.get_integrater_refiner()
self._intgr_experiments_filename = refiner.get_refiner_payload(
"experiments.json")
experiments = load.experiment_list(self._intgr_experiments_filename)
experiment = experiments[0]
self._intgr_indexed_filename = refiner.get_refiner_payload(
"reflections.pickle")
# this is the result of the cell refinement
self._intgr_cell = experiment.crystal.get_unit_cell().parameters()
Debug.write('Files available at the end of DIALS integrate prepare:')
for f in self._data_files.keys():
Debug.write('%s' % f)
self.set_detector(experiment.detector)
self.set_beam_obj(experiment.beam)
self.set_goniometer(experiment.goniometer)
def _index_prepare(self):
Citations.cite("dials")
# all_images = self.get_matching_images()
# first = min(all_images)
# last = max(all_images)
spot_lists = []
experiments_filenames = []
for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps):
logger.notice(banner("Spotfinding %s" % xsweep.get_name()))
first, last = imageset.get_scan().get_image_range()
# at this stage, break out to run the DIALS code: this sets itself up
# now cheat and pass in some information... save re-reading all of the
# image headers
# FIXME need to adjust this to allow (say) three chunks of images
from dxtbx.model.experiment_list import ExperimentListFactory
sweep_filename = os.path.join(self.get_working_directory(),
"%s_import.expt" % xsweep.get_name())
ExperimentListFactory.from_imageset_and_crystal(
imageset, None).as_file(sweep_filename)
genmask = self.GenerateMask()
genmask.set_input_experiments(sweep_filename)
genmask.set_output_experiments(
os.path.join(
self.get_working_directory(),
"%s_%s_masked.expt" %
(genmask.get_xpid(), xsweep.get_name()),
))
genmask.set_params(PhilIndex.params.dials.masking)
sweep_filename, mask_pickle = genmask.run()
logger.debug("Generated mask for %s: %s", xsweep.get_name(),
mask_pickle)
gain = PhilIndex.params.xia2.settings.input.gain
if gain is libtbx.Auto:
gain_estimater = self.EstimateGain()
gain_estimater.set_sweep_filename(sweep_filename)
gain_estimater.run()
gain = gain_estimater.get_gain()
logger.info("Estimated gain: %.2f", gain)
PhilIndex.params.xia2.settings.input.gain = gain
# FIXME this should really use the assigned spot finding regions
# offset = self.get_frame_offset()
dfs_params = PhilIndex.params.dials.find_spots
spotfinder = self.Spotfinder()
if last - first > 10:
spotfinder.set_write_hot_mask(True)
spotfinder.set_input_sweep_filename(sweep_filename)
spotfinder.set_output_sweep_filename(
"%s_%s_strong.expt" %
(spotfinder.get_xpid(), xsweep.get_name()))
spotfinder.set_input_spot_filename(
"%s_%s_strong.refl" %
(spotfinder.get_xpid(), xsweep.get_name()))
if PhilIndex.params.dials.fast_mode:
wedges = self._index_select_images_i(imageset)
spotfinder.set_scan_ranges(wedges)
else:
spotfinder.set_scan_ranges([(first, last)])
if dfs_params.phil_file is not None:
spotfinder.set_phil_file(dfs_params.phil_file)
if dfs_params.min_spot_size is libtbx.Auto:
if imageset.get_detector()[0].get_type() == "SENSOR_PAD":
dfs_params.min_spot_size = 3
else:
dfs_params.min_spot_size = None
if dfs_params.min_spot_size is not None:
spotfinder.set_min_spot_size(dfs_params.min_spot_size)
if dfs_params.min_local is not None:
spotfinder.set_min_local(dfs_params.min_local)
if dfs_params.sigma_strong:
spotfinder.set_sigma_strong(dfs_params.sigma_strong)
gain = PhilIndex.params.xia2.settings.input.gain
if gain:
spotfinder.set_gain(gain)
if dfs_params.filter_ice_rings:
spotfinder.set_filter_ice_rings(dfs_params.filter_ice_rings)
if dfs_params.kernel_size:
spotfinder.set_kernel_size(dfs_params.kernel_size)
if dfs_params.global_threshold is not None:
spotfinder.set_global_threshold(dfs_params.global_threshold)
if dfs_params.threshold.algorithm is not None:
spotfinder.set_threshold_algorithm(
dfs_params.threshold.algorithm)
spotfinder.run()
spot_filename = spotfinder.get_spot_filename()
if not os.path.exists(spot_filename):
raise RuntimeError("Spotfinding failed: %s does not exist." %
os.path.basename(spot_filename))
spot_lists.append(spot_filename)
experiments_filenames.append(
spotfinder.get_output_sweep_filename())
refl = flex.reflection_table.from_file(spot_filename)
if not len(refl):
raise RuntimeError("No spots found in sweep %s" %
xsweep.get_name())
logger.info(spot_counts_per_image_plot(refl))
if not PhilIndex.params.dials.fast_mode:
detectblanks = self.DetectBlanks()
detectblanks.set_sweep_filename(experiments_filenames[-1])
detectblanks.set_reflections_filename(spot_filename)
detectblanks.run()
json = detectblanks.get_results()
blank_regions = json["strong"]["blank_regions"]
if len(blank_regions):
blank_regions = [(int(s), int(e))
for s, e in blank_regions]
for blank_start, blank_end in blank_regions:
logger.info(
"WARNING: Potential blank images: %i -> %i",
blank_start + 1,
blank_end,
)
if PhilIndex.params.xia2.settings.remove_blanks:
non_blanks = []
start, end = imageset.get_array_range()
last_blank_end = start
for blank_start, blank_end in blank_regions:
if blank_start > start:
non_blanks.append(
(last_blank_end, blank_start))
last_blank_end = blank_end
if last_blank_end + 1 < end:
non_blanks.append((last_blank_end, end))
xsweep = self.get_indexer_sweep()
xwav = xsweep.get_wavelength()
xsample = xsweep.sample
sweep_name = xsweep.get_name()
for i, (nb_start, nb_end) in enumerate(non_blanks):
assert i < 26
if i == 0:
sub_imageset = imageset[nb_start -
start:nb_end - start]
xsweep._frames_to_process = (nb_start + 1,
nb_end + 1)
self.set_indexer_prepare_done(done=False)
self._indxr_imagesets[
self._indxr_imagesets.index(
imageset)] = sub_imageset
xsweep._integrater._setup_from_imageset(
sub_imageset)
else:
min_images = (PhilIndex.params.xia2.settings.
input.min_images)
if (nb_end - nb_start) < min_images:
continue
new_name = "_".join(
(sweep_name, string.ascii_lowercase[i]))
new_sweep = xwav.add_sweep(
new_name,
xsample,
directory=os.path.join(
os.path.basename(
xsweep.get_directory()),
new_name,
),
image=imageset.get_path(nb_start - start),
frames_to_process=(nb_start + 1, nb_end),
beam=xsweep.get_beam_centre(),
reversephi=xsweep.get_reversephi(),
distance=xsweep.get_distance(),
gain=xsweep.get_gain(),
dmin=xsweep.get_resolution_high(),
dmax=xsweep.get_resolution_low(),
polarization=xsweep.get_polarization(),
user_lattice=xsweep.get_user_lattice(),
user_cell=xsweep.get_user_cell(),
epoch=xsweep._epoch,
ice=xsweep._ice,
excluded_regions=xsweep._excluded_regions,
)
logger.info(
"Generating new sweep: %s (%s:%i:%i)",
new_sweep.get_name(),
new_sweep.get_image(),
new_sweep.get_frames_to_process()[0],
new_sweep.get_frames_to_process()[1],
)
return
if not PhilIndex.params.xia2.settings.trust_beam_centre:
discovery = self.SearchBeamPosition()
discovery.set_sweep_filename(experiments_filenames[-1])
discovery.set_spot_filename(spot_filename)
# set scan_range to correspond to not more than 180 degrees
# if we have > 20000 reflections
width = imageset.get_scan().get_oscillation()[1]
if (last - first) * width > 180.0 and len(refl) > 20000:
end = first + int(round(180.0 / width)) - 1
logger.debug("Using %d to %d for beam search", first, end)
discovery.set_image_range((first, end))
try:
discovery.run()
result = discovery.get_optimized_experiments_filename()
# overwrite indexed.expt in experiments list
experiments_filenames[-1] = result
except Exception as e:
logger.debug("DIALS beam centre search failed: %s",
str(e),
exc_info=True)
self.set_indexer_payload("spot_lists", spot_lists)
self.set_indexer_payload("experiments", experiments_filenames)
def generate_xia2_html(xinfo, filename="xia2.html", params=None, args=[]):
assert params is None or len(args) == 0
if params is None:
from xia2.Modules.Analysis import phil_scope
interp = phil_scope.command_line_argument_interpreter()
params, unhandled = interp.process_and_fetch(
args, custom_processor="collect_remaining")
params = params.extract()
xia2_txt = os.path.join(os.path.abspath(os.path.curdir), "xia2.txt")
assert os.path.isfile(xia2_txt), xia2_txt
with open(xia2_txt, "r") as f:
xia2_output = html.escape(f.read())
styles = {}
columns = []
columns.append([
"",
"Wavelength (Å)",
"Resolution range (Å)",
"Completeness (%)",
"Multiplicity",
"CC-half",
"I/sigma",
"Rmerge(I)",
# anomalous statistics
"Anomalous completeness (%)",
"Anomalous multiplicity",
])
individual_dataset_reports = {}
for cname, xcryst in xinfo.get_crystals().items():
reflection_files = xcryst.get_scaled_merged_reflections()
for wname, unmerged_mtz in reflection_files["mtz_unmerged"].items():
xwav = xcryst.get_xwavelength(wname)
from xia2.Modules.MultiCrystalAnalysis import batch_phil_scope
scope = phil.parse(batch_phil_scope)
scaler = xcryst._scaler
try:
for si in scaler._sweep_information.values():
batch_params = scope.extract().batch[0]
batch_params.id = si["sname"]
batch_params.range = si["batches"]
params.batch.append(batch_params)
except AttributeError:
for si in scaler._sweep_handler._sweep_information.values():
batch_params = scope.extract().batch[0]
batch_params.id = si.get_sweep_name()
batch_params.range = si.get_batch_range()
params.batch.append(batch_params)
report_path = xinfo.path.joinpath(cname, "report")
report_path.mkdir(parents=True, exist_ok=True)
report = Report.from_unmerged_mtz(unmerged_mtz,
params,
report_dir=str(report_path))
xtriage_success, xtriage_warnings, xtriage_danger = None, None, None
if params.xtriage_analysis:
try:
(
xtriage_success,
xtriage_warnings,
xtriage_danger,
) = report.xtriage_report()
except Exception as e:
params.xtriage_analysis = False
logger.debug("Exception running xtriage:")
logger.debug(e, exc_info=True)
(
overall_stats_table,
merging_stats_table,
stats_plots,
) = report.resolution_plots_and_stats()
d = {}
d["merging_statistics_table"] = merging_stats_table
d["overall_statistics_table"] = overall_stats_table
individual_dataset_reports[wname] = d
json_data = {}
if params.xtriage_analysis:
json_data["xtriage"] = (xtriage_success + xtriage_warnings +
xtriage_danger)
json_data.update(stats_plots)
json_data.update(report.batch_dependent_plots())
json_data.update(report.intensity_stats_plots(run_xtriage=False))
json_data.update(report.pychef_plots())
json_data.update(report.pychef_plots(n_bins=1))
from scitbx.array_family import flex
max_points = 500
for g in (
"scale_rmerge_vs_batch",
"completeness_vs_dose",
"rcp_vs_dose",
"scp_vs_dose",
"rd_vs_batch_difference",
):
for i, data in enumerate(json_data[g]["data"]):
x = data["x"]
n = len(x)
if n > max_points:
step = n // max_points
sel = (flex.int_range(n) % step) == 0
data["x"] = list(flex.int(data["x"]).select(sel))
data["y"] = list(flex.double(data["y"]).select(sel))
resolution_graphs = OrderedDict((k + "_" + wname, json_data[k])
for k in (
"cc_one_half",
"i_over_sig_i",
"second_moments",
"wilson_intensity_plot",
"completeness",
"multiplicity_vs_resolution",
) if k in json_data)
if params.include_radiation_damage:
batch_graphs = OrderedDict((k + "_" + wname, json_data[k])
for k in (
"scale_rmerge_vs_batch",
"i_over_sig_i_vs_batch",
"completeness_vs_dose",
"rcp_vs_dose",
"scp_vs_dose",
"rd_vs_batch_difference",
))
else:
batch_graphs = OrderedDict((k + "_" + wname, json_data[k])
for k in ("scale_rmerge_vs_batch",
"i_over_sig_i_vs_batch"))
misc_graphs = OrderedDict((k, json_data[k]) for k in (
"cumulative_intensity_distribution",
"l_test",
"multiplicities",
) if k in json_data)
for k, v in report.multiplicity_plots().items():
misc_graphs[k + "_" + wname] = {"img": v}
d["resolution_graphs"] = resolution_graphs
d["batch_graphs"] = batch_graphs
d["misc_graphs"] = misc_graphs
d["xtriage"] = {
"success": xtriage_success,
"warnings": xtriage_warnings,
"danger": xtriage_danger,
}
merging_stats = report.merging_stats
merging_stats_anom = report.merging_stats_anom
overall = merging_stats.overall
overall_anom = merging_stats_anom.overall
outer_shell = merging_stats.bins[-1]
outer_shell_anom = merging_stats_anom.bins[-1]
column = [
wname,
str(xwav.get_wavelength()),
"%.2f - %.2f (%.2f - %.2f)" %
(overall.d_max, overall.d_min, outer_shell.d_max,
outer_shell.d_min),
"%.2f (%.2f)" %
(overall.completeness * 100, outer_shell.completeness * 100),
f"{overall.mean_redundancy:.2f} ({outer_shell.mean_redundancy:.2f})",
f"{overall.cc_one_half:.4f} ({outer_shell.cc_one_half:.4f})",
"%.2f (%.2f)" %
(overall.i_over_sigma_mean, outer_shell.i_over_sigma_mean),
f"{overall.r_merge:.4f} ({outer_shell.r_merge:.4f})",
# anomalous statistics
"%.2f (%.2f)" % (
overall_anom.anom_completeness * 100,
outer_shell_anom.anom_completeness * 100,
),
"%.2f (%.2f)" % (overall_anom.mean_redundancy,
outer_shell_anom.mean_redundancy),
]
columns.append(column)
table = [[c[i] for c in columns] for i in range(len(columns[0]))]
from cctbx import sgtbx
space_groups = xcryst.get_likely_spacegroups()
space_groups = [
sgtbx.space_group_info(symbol=str(symbol)) for symbol in space_groups
]
space_group = space_groups[0].symbol_and_number()
alternative_space_groups = [
sg.symbol_and_number() for sg in space_groups[1:]
]
unit_cell = str(report.intensities.unit_cell())
# reflection files
for cname, xcryst in xinfo.get_crystals().items():
# hack to replace path to reflection files with DataFiles directory
data_dir = os.path.join(os.path.abspath(os.path.curdir), "DataFiles")
g = glob.glob(os.path.join(data_dir, "*"))
reflection_files = xcryst.get_scaled_merged_reflections()
for k, rfile in reflection_files.items():
if isinstance(rfile, str):
for datafile in g:
if os.path.basename(datafile) == os.path.basename(rfile):
reflection_files[k] = datafile
break
else:
for kk in rfile:
for datafile in g:
if os.path.basename(datafile) == os.path.basename(
rfile[kk]):
reflection_files[k][kk] = datafile
break
headers = ["Dataset", "File name"]
merged_mtz = reflection_files["mtz"]
mtz_files = [
headers,
[
"All datasets",
'<a href="%s">%s</a>' %
(os.path.relpath(merged_mtz), os.path.basename(merged_mtz)),
],
]
for wname, unmerged_mtz in reflection_files["mtz_unmerged"].items():
mtz_files.append([
wname,
'<a href="%s">%s</a>' % (os.path.relpath(unmerged_mtz),
os.path.basename(unmerged_mtz)),
])
sca_files = [headers]
if "sca" in reflection_files:
for wname, merged_sca in reflection_files["sca"].items():
sca_files.append([
wname,
'<a href="%s">%s</a>' % (os.path.relpath(merged_sca),
os.path.basename(merged_sca)),
])
unmerged_sca_files = [headers]
if "sca_unmerged" in reflection_files:
for wname, unmerged_sca in reflection_files["sca_unmerged"].items(
):
unmerged_sca_files.append([
wname,
'<a href="%s">%s</a>' % (
os.path.relpath(unmerged_sca),
os.path.basename(unmerged_sca),
),
])
# other files
other_files = []
other_files.append(["File name", "Description"])
for other_file, description in sorted([
("xia2.cif", "Crystallographic information file"),
("xia2.mmcif", "Macromolecular crystallographic information file"),
("shelxt.hkl", "merged structure factors for SHELXT"),
("shelxt.ins", "SHELXT instruction file"),
] + [(fn, "XPREP input file")
for fn in os.listdir(os.path.join(data_dir)) if fn.endswith(".p4p")]):
if os.path.exists(os.path.join(data_dir, other_file)):
other_files.append([
'<a href="DataFiles/{filename}">{filename}</a>'.format(
filename=other_file),
description,
])
# log files
log_files_table = []
log_dir = os.path.join(os.path.abspath(os.path.curdir), "LogFiles")
g = glob.glob(os.path.join(log_dir, "*.log"))
for logfile in g:
html_file = make_logfile_html(logfile)
html_file = os.path.splitext(logfile)[0] + ".html"
if os.path.exists(html_file):
log_files_table.append([
os.path.basename(logfile),
'<a href="%s">original</a>' % os.path.relpath(logfile),
'<a href="%s">html</a>' % os.path.relpath(html_file),
])
else:
log_files_table.append([
os.path.basename(logfile),
'<a href="%s">original</a>' % os.path.relpath(logfile),
" ",
])
references = {
cdict["acta"]: cdict.get("url")
for cdict in Citations.get_citations_dicts()
}
from jinja2 import Environment, ChoiceLoader, PackageLoader
loader = ChoiceLoader([
PackageLoader("xia2", "templates"),
PackageLoader("dials", "templates")
])
env = Environment(loader=loader)
template = env.get_template("xia2.html")
html_source = template.render(
page_title="xia2 processing report",
xia2_output=xia2_output,
space_group=space_group,
alternative_space_groups=alternative_space_groups,
unit_cell=unit_cell,
overall_stats_table=table,
cc_half_significance_level=params.cc_half_significance_level,
mtz_files=mtz_files,
sca_files=sca_files,
unmerged_sca_files=unmerged_sca_files,
other_files=other_files,
log_files_table=log_files_table,
individual_dataset_reports=individual_dataset_reports,
references=references,
styles=styles,
)
with open("%s-report.json" % os.path.splitext(filename)[0], "w") as fh:
json.dump(json_data, fh, indent=2)
with open(filename, "wb") as f:
f.write(html_source.encode("utf-8", "xmlcharrefreplace"))
def TwoThetaRefine(DriverType=None):
"""A factory for RefineWrapper classes."""
from xia2.Handlers.Citations import Citations
Citations.cite("dials")
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class RefineWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable("dials.two_theta_refine")
self._reindexing_operators = None
self._reindexed_experiments = None
self._reindexed_reflections = None
self._experiments = []
self._reflection_files = []
self._phil_file = None
self._combine_crystal_models = True
# The following are set during run() call:
self._output_cif = None
self._output_mmcif = None
self._output_p4p = None
self._output_correlation_plot = None
self._output_experiments = None
self._crystal = None
def set_experiments(self, experiments):
self._experiments = experiments
def get_experiments(self):
return self._experiments
def set_reflection_files(self, reflection_files):
self._reflection_files = reflection_files
def get_reflection_files(self):
return self._reflection_files
def set_phil_file(self, phil_file):
self._phil_file = phil_file
def set_combine_crystal_models(self, combine_crystal_models):
self._combine_crystal_models = combine_crystal_models
def get_output_cif(self):
return self._output_cif
def get_output_mmcif(self):
return self._output_mmcif
def set_output_p4p(self, filename):
self._output_p4p = filename
def get_output_p4p(self):
return self._output_p4p
def get_output_correlation_plot(self):
return self._output_correlation_plot
def get_output_experiments(self):
return self._output_experiments
def get_reindexed_experiments(self):
return self._reindexed_experiments
def get_reindexed_reflections(self):
return self._reindexed_reflections
def get_unit_cell(self):
return self._crystal.get_unit_cell().parameters()
def get_unit_cell_esd(self):
return self._crystal.get_cell_parameter_sd()
def set_reindex_operators(self, operators):
assert len(operators) == len(self._experiments)
self._reindexing_operators = operators
def run(self):
from xia2.Handlers.Streams import Chatter, Debug
if self._reindexing_operators:
Debug.write("Reindexing sweeps for dials.two_theta_refine")
from xia2.lib.bits import auto_logfiler
from xia2.Wrappers.Dials.Reindex import Reindex
self._reindexed_experiments, self._reindexed_reflections = [], []
for e, p, op in zip(
self._experiments,
self._reflection_files,
self._reindexing_operators,
):
reindexer = Reindex()
reindexer.set_cb_op(op)
reindexer.set_experiments_filename(e)
reindexer.set_indexed_filename(p)
reindexer.set_working_directory(
self.get_working_directory())
auto_logfiler(reindexer)
reindexer.run()
self._reindexed_experiments.append(
reindexer.get_reindexed_experiments_filename())
self._reindexed_reflections.append(
reindexer.get_reindexed_reflections_filename())
Debug.write("Running dials.two_theta_refine")
self._output_cif = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.cif" % self.get_xpid(),
)
self._output_mmcif = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.mmcif" % self.get_xpid(),
)
if not self._output_p4p:
self._output_p4p = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.p4p" % self.get_xpid(),
)
self._output_correlation_plot = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.png" % self.get_xpid(),
)
self._output_experiments = os.path.join(
self.get_working_directory(),
"%s_refined_cell.expt" % self.get_xpid())
self.clear_command_line()
if self._reindexing_operators:
for experiment in self._reindexed_experiments:
self.add_command_line(experiment)
for reflection_file in self._reindexed_reflections:
self.add_command_line(reflection_file)
else:
for experiment in self._experiments:
self.add_command_line(experiment)
for reflection_file in self._reflection_files:
self.add_command_line(reflection_file)
self.add_command_line("combine_crystal_models=%s" %
self._combine_crystal_models)
self.add_command_line("output.cif=%s" % self._output_cif)
self.add_command_line("output.mmcif=%s" % self._output_mmcif)
self.add_command_line("output.p4p=%s" % self._output_p4p)
if self._output_correlation_plot is not None:
self.add_command_line("output.correlation_plot.filename=%s" %
self._output_correlation_plot)
if self._output_experiments is not None:
self.add_command_line("output.experiments=%s" %
self._output_experiments)
if self._phil_file is not None:
self.add_command_line("%s" % self._phil_file)
self.start()
self.close_wait()
if not os.path.isfile(self._output_cif):
Chatter.write(
"TwoTheta refinement failed, see log file for more details:\n %s"
% self.get_log_file())
raise RuntimeError("unit cell not refined")
self.check_for_errors()
from dxtbx.model.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(
self.get_output_experiments(), check_format=False)
self._crystal = experiments.crystals()[0]
def import_cif(self):
"""Import relevant lines from .cif output"""
import iotbx.cif
cif = iotbx.cif.reader(file_path=self.get_output_cif()).model()
block = cif["two_theta_refine"]
subset = {
k: block[k]
for k in block.keys() if k.startswith(("_cell", "_diffrn"))
}
return subset
def import_mmcif(self):
"""Import relevant lines from .mmcif output"""
if os.path.isfile(self.get_output_mmcif()):
import iotbx.cif
cif = iotbx.cif.reader(
file_path=self.get_output_mmcif()).model()
block = cif["two_theta_refine"]
else:
import iotbx.cif.model
block = iotbx.cif.model.block()
subset = {
k: block[k]
for k in block.keys() if k.startswith(("_cell", "_diffrn"))
}
return subset
return RefineWrapper()
def load_sweeps_with_common_indexing():
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
import dials # required for gaussian_rs warning
from xia2.Wrappers.Dials.Reindex import Reindex
Citations.cite('dials')
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
import cPickle as pickle
crystals = xinfo.get_crystals()
assert len(crystals) == 1
crystal = next(crystals.itervalues())
working_directory = Environment.generate_directory([crystal.get_name(), 'analysis'])
os.chdir(working_directory)
scaler = crystal._get_scaler()
epoch_to_batches = {}
epoch_to_integrated_intensities = {}
epoch_to_sweep_name = {}
# Aimless only
epochs = scaler._sweep_handler.get_epochs()
reference_cell = None
reference_lattice = None
reference_vectors = None
reference_wavelength = None
# Reindex each sweep to same setting
all_miller_indices = flex.miller_index()
all_two_thetas = flex.double()
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
Chatter.smallbanner(si.get_sweep_name(), True)
Debug.smallbanner(si.get_sweep_name(), True)
intgr = si.get_integrater()
experiments_filename = intgr.get_integrated_experiments()
reflections_filename = intgr.get_integrated_reflections()
refiner = intgr.get_integrater_refiner()
Debug.write('experiment: %s' % experiments_filename)
Debug.write('reflection: %s' % reflections_filename)
# Use setting of first sweep as reference
if reference_vectors is None:
reference_vectors = experiments_filename
# Assume that all sweeps have the same lattice system
if reference_lattice is None:
reference_lattice = refiner.get_refiner_lattice()
else:
assert reference_lattice == refiner.get_refiner_lattice()
Debug.write("lattice: %s" % refiner.get_refiner_lattice())
# Read .json file for sweep
db = ExperimentListFactory.from_json_file(experiments_filename)
# Assume that each file only contains a single experiment
assert (len(db) == 1)
db = db[0]
# Get beam vector
s0 = db.beam.get_unit_s0()
# Use the unit cell of the first sweep as reference
if reference_cell is None:
reference_cell = db.crystal.get_unit_cell()
Debug.write("Reference cell: %s" % str(reference_cell))
dials_reindex = Reindex()
dials_reindex.set_working_directory(working_directory)
dials_reindex.set_cb_op("auto")
dials_reindex.set_reference_filename(reference_vectors)
dials_reindex.set_experiments_filename(experiments_filename)
dials_reindex.set_indexed_filename(reflections_filename)
auto_logfiler(dials_reindex)
dials_reindex.run()
# Assume that all data are collected at same wavelength
if reference_wavelength is None:
reference_wavelength = intgr.get_wavelength()
else:
assert abs(reference_wavelength - intgr.get_wavelength()) < 0.01
Debug.write("wavelength: %f A" % intgr.get_wavelength())
Debug.write("distance: %f mm" % intgr.get_distance())
# Get integrated reflection data
import dials
with open(dials_reindex.get_reindexed_reflections_filename(), 'rb') as fh:
reflections = pickle.load(fh)
selection = reflections.get_flags(reflections.flags.used_in_refinement)
Chatter.write("Found %d reflections used in refinement (out of %d entries)" % (selection.count(True), len(reflections['miller_index'])))
reflections = reflections.select(selection)
# Filter bad reflections
selection = reflections['intensity.sum.variance'] <= 0
if selection.count(True) > 0:
reflections.del_selected(selection)
print 'Removing %d reflections with negative variance' % \
selection.count(True)
if 'intensity.prf.variance' in reflections:
selection = reflections['intensity.prf.variance'] <= 0
if selection.count(True) > 0:
reflections.del_selected(selection)
print 'Removing %d profile reflections with negative variance' % \
selection.count(True)
# Find the observed 2theta angles
miller_indices = flex.miller_index()
two_thetas_obs = flex.double()
for pixel, panel, hkl in zip(reflections['xyzobs.px.value'], reflections['panel'], reflections['miller_index']):
assert hkl != (0, 0, 0)
two_thetas_obs.append(db.detector[panel].get_two_theta_at_pixel(s0, pixel[0:2]))
miller_indices.append(hkl)
# Convert observed 2theta angles to degrees
two_thetas_obs = two_thetas_obs * 180 / 3.14159265359
Chatter.write("Remaining %d reflections are in 2theta range %.3f - %.3f deg" % (len(miller_indices), min(two_thetas_obs), max(two_thetas_obs)))
all_miller_indices.extend(miller_indices)
all_two_thetas.extend(two_thetas_obs)
return all_miller_indices, all_two_thetas, reference_cell, reference_lattice, reference_wavelength
def run(args=sys.argv[1:]):
# Create the parser
parser = OptionParser(
# usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
# epilog=help_message,
)
# Parse the command line
params, options, args = parser.parse_args(args=args,
show_diff_phil=False,
return_unhandled=True)
# Configure the logging
xia2.Handlers.Streams.setup_logging(logfile=params.output.log,
verbose=options.verbose)
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
if params.unit_cell is not None:
unit_cell = params.unit_cell
crystal_symmetry = crystal.symmetry(unit_cell=unit_cell)
else:
crystal_symmetry = None
if len(params.input.experiments):
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
reflections = parse_multiple_datasets(reflections)
if len(experiments) != len(reflections):
raise sys.exit(
"Mismatched number of experiments and reflection tables found: %s & %s."
% (len(experiments), len(reflections)))
experiments, reflections = assign_unique_identifiers(
experiments, reflections)
# transform models into miller arrays
intensities, batches = filtered_arrays_from_experiments_reflections(
experiments,
reflections,
outlier_rejection_after_filter=False,
partiality_threshold=0.99,
return_batches=True,
)
if args and os.path.isfile(args[0]):
result = any_reflection_file(args[0])
unmerged_intensities = None
batches_all = None
for ma in result.as_miller_arrays(merge_equivalents=False,
crystal_symmetry=crystal_symmetry):
if ma.info().labels == ["I(+)", "SIGI(+)", "I(-)", "SIGI(-)"]:
assert ma.anomalous_flag()
unmerged_intensities = ma
elif ma.info().labels == ["I", "SIGI"]:
assert not ma.anomalous_flag()
unmerged_intensities = ma
elif ma.info().labels == ["BATCH"]:
batches_all = ma
assert batches_all is not None
assert unmerged_intensities is not None
sel = unmerged_intensities.sigmas() > 0
unmerged_intensities = unmerged_intensities.select(sel).set_info(
unmerged_intensities.info())
batches_all = batches_all.select(sel)
id_to_batches = None
if len(params.batch) > 0:
id_to_batches = {}
for b in params.batch:
assert b.id is not None
assert b.range is not None
assert b.id not in id_to_batches, "Duplicate batch id: %s" % b.id
id_to_batches[b.id] = b.range
separate = separate_unmerged(unmerged_intensities,
batches_all,
id_to_batches=id_to_batches)
batches = list(separate.batches.values())
intensities = list(separate.intensities.values())
result = DeltaCcHalf(
intensities,
batches,
n_bins=params.n_bins,
d_min=params.d_min,
cc_one_half_method=params.cc_one_half_method,
group_size=params.group_size,
)
logger.info(tabulate(result.get_table(), headers="firstrow"))
hist_filename = "delta_cc_hist.png"
logger.info("Saving histogram to %s" % hist_filename)
result.plot_histogram(hist_filename)
normalised_scores_filename = "normalised_scores.png"
logger.info("Saving normalised scores to %s" % normalised_scores_filename)
result.plot_normalised_scores(normalised_scores_filename)
Citations.cite("delta_cc_half")
for citation in Citations.get_citations_acta():
logger.info(citation)
def _integrate_prepare(self):
'''Prepare for integration - in XDS terms this may mean rerunning
IDXREF to get the XPARM etc. DEFPIX is considered part of the full
integration as it is resolution dependent.'''
from xia2.Handlers.Citations import Citations
Citations.cite('dials')
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(min(images),
max(images))
Debug.write('DIALS INTEGRATE PREPARE:')
Debug.write('Wavelength: %.6f' % self.get_wavelength())
Debug.write('Distance: %.2f' % self.get_distance())
#if not self._intgr_indexer:
#self.set_integrater_indexer(DialsIndexer())
#self.get_integrater_indexer().set_indexer_sweep(
#self.get_integrater_sweep())
#self._intgr_indexer.set_working_directory(
#self.get_working_directory())
#self._intgr_indexer.setup_from_imageset(self.get_imageset())
#if self.get_frame_wedge():
#wedge = self.get_frame_wedge()
#Debug.write('Propogating wedge limit: %d %d' % wedge)
#self._intgr_indexer.set_frame_wedge(wedge[0], wedge[1],
#apply_offset = False)
## this needs to be set up from the contents of the
## Integrater frame processer - wavelength &c.
#if self.get_beam_centre():
#self._intgr_indexer.set_beam_centre(self.get_beam_centre())
#if self.get_distance():
#self._intgr_indexer.set_distance(self.get_distance())
#if self.get_wavelength():
#self._intgr_indexer.set_wavelength(
#self.get_wavelength())
## get the unit cell from this indexer to initiate processing
## if it is new... and also copy out all of the information for
## the Dials indexer if not...
#experiments = self._intgr_indexer.get_indexer_experiment_list()
#assert len(experiments) == 1 # currently only handle one lattice/sweep
#experiment = experiments[0]
#crystal_model = experiment.crystal
#lattice = self._intgr_indexer.get_indexer_lattice()
## check if the lattice was user assigned...
#user_assigned = self._intgr_indexer.get_indexer_user_input_lattice()
# XXX check that the indexer is an Dials indexer - if not then
# create one...
# set a low resolution limit (which isn't really used...)
# this should perhaps be done more intelligently from an
# analysis of the spot list or something...?
if not self.get_integrater_low_resolution():
dmax = self._intgr_refiner.get_indexer_low_resolution(
self.get_integrater_epoch())
self.set_integrater_low_resolution(dmax)
Debug.write('Low resolution set to: %s' % \
self.get_integrater_low_resolution())
## copy the data across
from dxtbx.serialize import load
refiner = self.get_integrater_refiner()
self._intgr_experiments_filename = refiner.get_refiner_payload(
"experiments.json")
experiments = load.experiment_list(self._intgr_experiments_filename)
experiment = experiments[0]
self._intgr_indexed_filename = refiner.get_refiner_payload(
"reflections.pickle")
# this is the result of the cell refinement
self._intgr_cell = experiment.crystal.get_unit_cell().parameters()
Debug.write('Files available at the end of DIALS integrate prepare:')
for f in self._data_files.keys():
Debug.write('%s' % f)
self.set_detector(experiment.detector)
self.set_beam_obj(experiment.beam)
self.set_goniometer(experiment.goniometer)
def _scale_prepare(self):
'''Perform all of the preparation required to deliver the scaled
data. This should sort together the reflection files, ensure that
they are correctly indexed (via pointless) and generally tidy
things up.'''
# acknowledge all of the programs we are about to use...
Citations.cite('pointless')
Citations.cite('aimless')
Citations.cite('ccp4')
# ---------- GATHER ----------
self._sweep_handler = SweepInformationHandler(self._scalr_integraters)
Journal.block(
'gathering', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'working directory':self.get_working_directory()})
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
exclude_sweep = False
for sweep in PhilIndex.params.xia2.settings.sweep:
if sweep.id == sname and sweep.exclude:
exclude_sweep = True
break
if exclude_sweep:
self._sweep_handler.remove_epoch(epoch)
Debug.write('Excluding sweep %s' %sname)
else:
Journal.entry({'adding data from':'%s/%s/%s' % \
(xname, dname, sname)})
# gather data for all images which belonged to the parent
# crystal - allowing for the fact that things could go wrong
# e.g. epoch information not available, exposure times not in
# headers etc...
for e in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(e)
assert is_mtz_file(si.get_reflections())
p, x = self._sweep_handler.get_project_info()
self._scalr_pname = p
self._scalr_xname = x
# verify that the lattices are consistent, calling eliminate if
# they are not N.B. there could be corner cases here
need_to_return = False
multi_sweep_indexing = \
PhilIndex.params.xia2.settings.developmental.multi_sweep_indexing
if len(self._sweep_handler.get_epochs()) > 1:
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
md = self._factory.Mtzdump()
md.set_hklin(hklin)
md.dump()
batches = md.get_batches()
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
datasets = md.get_datasets()
Debug.write('In reflection file %s found:' % hklin)
for d in datasets:
Debug.write('... %s' % d)
dataset_info = md.get_dataset_info(datasets[0])
from xia2.lib.bits import nifty_power_of_ten
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
rb = self._factory.Rebatch()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_prepointless.mtz' % \
(pname, xname, dname, si.get_sweep_name()))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
rb.set_hklin(hklin)
rb.set_first_batch(counter * max_batches + 1)
rb.set_project_info(pname, xname, dname)
rb.set_hklout(hklout)
new_batches = rb.rebatch()
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattices = [Syminfo.get_lattice(pointgroup)]
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
else:
lattices = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
ntr = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattice = Syminfo.get_lattice(pointgroup)
if not lattice in lattices:
lattices.append(lattice)
if ntr:
intgr.integrater_reset_reindex_operator()
need_to_return = True
if len(lattices) > 1:
# why not using pointless indexer jiffy??!
correct_lattice = sort_lattices(lattices)[0]
Chatter.write('Correct lattice asserted to be %s' % \
correct_lattice)
# transfer this information back to the indexers
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
refiner = si.get_integrater().get_integrater_refiner()
sname = si.get_sweep_name()
state = refiner.set_refiner_asserted_lattice(
correct_lattice)
if state == refiner.LATTICE_CORRECT:
Chatter.write('Lattice %s ok for sweep %s' % \
(correct_lattice, sname))
elif state == refiner.LATTICE_IMPOSSIBLE:
raise RuntimeError, 'Lattice %s impossible for %s' \
% (correct_lattice, sname)
elif state == refiner.LATTICE_POSSIBLE:
Chatter.write('Lattice %s assigned for sweep %s' % \
(correct_lattice, sname))
need_to_return = True
# if one or more of them was not in the lowest lattice,
# need to return here to allow reprocessing
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# ---------- REINDEX ALL DATA TO CORRECT POINTGROUP ----------
# all should share the same pointgroup, unless twinned... in which
# case force them to be...
pointgroups = { }
reindex_ops = { }
probably_twinned = False
need_to_return = False
multi_sweep_indexing = \
PhilIndex.params.xia2.settings.developmental.multi_sweep_indexing
if multi_sweep_indexing and not self._scalr_input_pointgroup:
pointless_hklins = []
max_batches = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
md = self._factory.Mtzdump()
md.set_hklin(hklin)
md.dump()
batches = md.get_batches()
if 1 + max(batches) - min(batches) > max_batches:
max_batches = max(batches) - min(batches) + 1
datasets = md.get_datasets()
Debug.write('In reflection file %s found:' % hklin)
for d in datasets:
Debug.write('... %s' % d)
dataset_info = md.get_dataset_info(datasets[0])
from xia2.lib.bits import nifty_power_of_ten
Debug.write('Biggest sweep has %d batches' % max_batches)
max_batches = nifty_power_of_ten(max_batches)
counter = 0
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
rb = self._factory.Rebatch()
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_prepointless.mtz' % \
(pname, xname, dname, si.get_sweep_name()))
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
first_batch = min(si.get_batches())
si.set_batch_offset(counter * max_batches - first_batch + 1)
rb.set_hklin(hklin)
rb.set_first_batch(counter * max_batches + 1)
rb.set_project_info(pname, xname, dname)
rb.set_hklout(hklout)
new_batches = rb.rebatch()
pointless_hklins.append(hklout)
# update the counter & recycle
counter += 1
s = self._factory.Sortmtz()
pointless_hklin = os.path.join(self.get_working_directory(),
'%s_%s_prepointless_sorted.mtz' % \
(self._scalr_pname, self._scalr_xname))
s.set_hklout(pointless_hklin)
for hklin in pointless_hklins:
s.add_hklin(hklin)
s.sort()
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
for epoch in self._sweep_handler.get_epochs():
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
else:
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
#hklout = os.path.join(
#self.get_working_directory(),
#os.path.split(hklin)[-1].replace('.mtz', '_rdx.mtz'))
#FileHandler.record_temporary_file(hklout)
integrater = si.get_integrater()
refiner = integrater.get_integrater_refiner()
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
pt = False
else:
pointless_hklin = self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width())
pointgroup, reindex_op, ntr, pt = \
self._pointless_indexer_jiffy(
pointless_hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
if ntr:
integrater.integrater_reset_reindex_operator()
need_to_return = True
if pt and not probably_twinned:
probably_twinned = True
Debug.write('Pointgroup: %s (%s)' % (pointgroup, reindex_op))
pointgroups[epoch] = pointgroup
reindex_ops[epoch] = reindex_op
overall_pointgroup = None
pointgroup_set = set([pointgroups[e] for e in pointgroups])
if len(pointgroup_set) > 1 and \
not probably_twinned:
raise RuntimeError, 'non uniform pointgroups'
if len(pointgroup_set) > 1:
Debug.write('Probably twinned, pointgroups: %s' % \
' '.join([p.replace(' ', '') for p in \
list(pointgroup_set)]))
numbers = [Syminfo.spacegroup_name_to_number(s) for s in \
pointgroup_set]
overall_pointgroup = Syminfo.spacegroup_number_to_name(
min(numbers))
self._scalr_input_pointgroup = overall_pointgroup
Chatter.write('Twinning detected, assume pointgroup %s' % \
overall_pointgroup)
need_to_return = True
else:
overall_pointgroup = pointgroup_set.pop()
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
integrater = si.get_integrater()
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(overall_pointgroup))
integrater.set_integrater_reindex_operator(
reindex_ops[epoch], reason='setting point group')
# This will give us the reflections in the correct point group
si.set_reflections(integrater.get_integrater_intensities())
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# in here now optinally work through the data files which should be
# indexed with a consistent point group, and transform the orientation
# matrices by the lattice symmetry operations (if possible) to get a
# consistent definition of U matrix modulo fixed rotations
if PhilIndex.params.xia2.settings.unify_setting:
from scitbx.matrix import sqr
reference_U = None
i3 = sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
fixed = sqr(intgr.get_goniometer().get_fixed_rotation())
u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections())
U = fixed.inverse() * sqr(u).transpose()
B = sqr(b)
if reference_U is None:
reference_U = U
continue
results = []
for op in s.all_ops():
R = B * sqr(op.r().as_double()).transpose() * B.inverse()
nearly_i3 = (U * R).inverse() * reference_U
score = sum([abs(_n - _i) for (_n, _i) in zip(nearly_i3, i3)])
results.append((score, op.r().as_hkl(), op))
results.sort()
best = results[0]
Debug.write('Best reindex: %s %.3f' % (best[1], best[0]))
intgr.set_integrater_reindex_operator(best[2].r().inverse().as_hkl(),
reason='unifying [U] setting')
si.set_reflections(intgr.get_integrater_intensities())
# recalculate to verify
u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections())
U = fixed.inverse() * sqr(u).transpose()
Debug.write('New reindex: %s' % (U.inverse() * reference_U))
# FIXME I should probably raise an exception at this stage if this
# is not about I3...
if self.get_scaler_reference_reflection_file():
self._reference = self.get_scaler_reference_reflection_file()
Debug.write('Using HKLREF %s' % self._reference)
elif Flags.get_reference_reflection_file():
self._reference = Flags.get_reference_reflection_file()
Debug.write('Using HKLREF %s' % self._reference)
params = PhilIndex.params
use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs
if len(self._sweep_handler.get_epochs()) > 1 and use_brehm_diederichs:
brehm_diederichs_files_in = []
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
brehm_diederichs_files_in.append(hklin)
# now run cctbx.brehm_diederichs to figure out the indexing hand for
# each sweep
from xia2.Wrappers.Cctbx.BrehmDiederichs import BrehmDiederichs
from xia2.lib.bits import auto_logfiler
brehm_diederichs = BrehmDiederichs()
brehm_diederichs.set_working_directory(self.get_working_directory())
auto_logfiler(brehm_diederichs)
brehm_diederichs.set_input_filenames(brehm_diederichs_files_in)
# 1 or 3? 1 seems to work better?
brehm_diederichs.set_asymmetric(1)
brehm_diederichs.run()
reindexing_dict = brehm_diederichs.get_reindexing_dict()
for epoch in self._sweep_handler.get_epochs():
si = self._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
hklin = si.get_reflections()
reindex_op = reindexing_dict.get(os.path.abspath(hklin))
assert reindex_op is not None
if 1 or reindex_op != 'h,k,l':
# apply the reindexing operator
intgr.set_integrater_reindex_operator(
reindex_op, reason='match reference')
si.set_reflections(intgr.get_integrater_intensities())
elif len(self._sweep_handler.get_epochs()) > 1 and \
not self._reference:
first = self._sweep_handler.get_epochs()[0]
si = self._sweep_handler.get_sweep_information(first)
self._reference = si.get_reflections()
if self._reference:
md = self._factory.Mtzdump()
md.set_hklin(self._reference)
md.dump()
if md.get_batches() and False:
raise RuntimeError, 'reference reflection file %s unmerged' % \
self._reference
datasets = md.get_datasets()
if len(datasets) > 1 and False:
raise RuntimeError, 'more than one dataset in %s' % \
self._reference
# then get the unit cell, lattice etc.
reference_lattice = Syminfo.get_lattice(md.get_spacegroup())
reference_cell = md.get_dataset_info(datasets[0])['cell']
# then compute the pointgroup from this...
# ---------- REINDEX TO CORRECT (REFERENCE) SETTING ----------
for epoch in self._sweep_handler.get_epochs():
pl = self._factory.Pointless()
si = self._sweep_handler.get_sweep_information(epoch)
hklin = si.get_reflections()
pl.set_hklin(self._prepare_pointless_hklin(
hklin, si.get_integrater().get_phi_width()))
hklout = os.path.join(
self.get_working_directory(),
'%s_rdx2.mtz' % os.path.split(hklin)[-1][:-4])
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
# now set the initial reflection set as a reference...
pl.set_hklref(self._reference)
# write a pointless log file...
pl.decide_pointgroup()
Debug.write('Reindexing analysis of %s' % pl.get_hklin())
pointgroup = pl.get_pointgroup()
reindex_op = pl.get_reindex_operator()
Debug.write('Operator: %s' % reindex_op)
# apply this...
integrater = si.get_integrater()
integrater.set_integrater_reindex_operator(reindex_op,
reason='match reference')
integrater.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
si.set_reflections(integrater.get_integrater_intensities())
md = self._factory.Mtzdump()
md.set_hklin(si.get_reflections())
md.dump()
datasets = md.get_datasets()
if len(datasets) > 1:
raise RuntimeError, 'more than one dataset in %s' % \
si.get_reflections()
# then get the unit cell, lattice etc.
lattice = Syminfo.get_lattice(md.get_spacegroup())
cell = md.get_dataset_info(datasets[0])['cell']
if lattice != reference_lattice:
raise RuntimeError, 'lattices differ in %s and %s' % \
(self._reference, si.get_reflections())
for j in range(6):
if math.fabs((cell[j] - reference_cell[j]) /
reference_cell[j]) > 0.1:
raise RuntimeError, \
'unit cell parameters differ in %s and %s' % \
(self._reference, si.get_reflections())
# ---------- SORT TOGETHER DATA ----------
self._sort_together_data_ccp4()
self._scalr_resolution_limits = { }
# store central resolution limit estimates
batch_ranges = [self._sweep_handler.get_sweep_information(
epoch).get_batch_range() for epoch in
self._sweep_handler.get_epochs()]
self._resolution_limit_estimates = erzatz_resolution(
self._prepared_reflections, batch_ranges)
return
def multi_crystal_analysis(stop_after=None):
'''Actually process something...'''
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
for crystal_id, crystal in crystals.iteritems():
cwd = os.path.abspath(os.curdir)
working_directory = Environment.generate_directory(
[crystal.get_name(), 'analysis'])
os.chdir(working_directory)
from xia2.Wrappers.CCP4.Blend import Blend
from xia2.lib.bits import auto_logfiler
hand_blender = Blend()
hand_blender.set_working_directory(working_directory)
auto_logfiler(hand_blender)
Citations.cite('blend')
scaler = crystal._get_scaler()
#epoch_to_si = {}
epoch_to_batches = {}
epoch_to_integrated_intensities = {}
epoch_to_sweep_name = {}
try:
epochs = scaler._sweep_information.keys()
for epoch in epochs:
si = scaler._sweep_information[epoch]
epoch_to_batches[epoch] = si['batches']
epoch_to_integrated_intensities[epoch] = si['corrected_intensities']
epoch_to_sweep_name[epoch] = si['sname']
except AttributeError, e:
epochs = scaler._sweep_handler.get_epochs()
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
epoch_to_batches[epoch] = si.get_batches()
epoch_to_integrated_intensities[epoch] = si.get_reflections()
epoch_to_sweep_name[epoch] = si.get_sweep_name()
unmerged_mtz = scaler.get_scaled_reflections('mtz_unmerged').values()[0]
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(unmerged_mtz)
intensities = None
batches = None
assert reader.file_type() == 'ccp4_mtz'
arrays = reader.as_miller_arrays(merge_equivalents=False)
for ma in arrays:
if ma.info().labels == ['BATCH']:
batches = ma
elif ma.info().labels == ['I', 'SIGI']:
intensities = ma
elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
intensities = ma
from xia2.Handlers.Environment import which
Rscript_binary = which('Rscript', debug=False)
if Rscript_binary is None:
Chatter.write('Skipping BLEND analysis: Rscript not available')
else:
for epoch in epochs:
hand_blender.add_hklin(epoch_to_integrated_intensities[epoch],
label=epoch_to_sweep_name[epoch])
hand_blender.analysis()
Chatter.write("Dendrogram saved to: %s" %hand_blender.get_dendrogram_file())
analysis = hand_blender.get_analysis()
summary = hand_blender.get_summary()
clusters = hand_blender.get_clusters()
linkage_matrix = hand_blender.get_linkage_matrix()
ddict = hand_blender.plot_dendrogram()
rows = []
headers = ['Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'LCV', 'aLCV']
completeness = flex.double()
for i, cluster in clusters.iteritems():
sel_cluster = flex.bool(batches.size(), False)
for j in cluster['dataset_ids']:
batch_start, batch_end = epoch_to_batches[epochs[j-1]]
sel_cluster |= (
(batches.data() >= batch_start) & (batches.data() <= batch_end))
intensities_cluster = intensities.select(sel_cluster)
merging = intensities_cluster.merge_equivalents()
merged_intensities = merging.array()
multiplicities = merging.redundancies()
completeness.append(merged_intensities.completeness())
dataset_ids = cluster['dataset_ids']
rows.append(
['%i' %i, ' '.join(['%i'] * len(dataset_ids)) %tuple(dataset_ids),
'%.1f' %flex.mean(multiplicities.data().as_double()),
'%.2f' %completeness[-1],
'%.2f' %cluster['lcv'], '%.2f' %cluster['alcv']])
# sort table by completeness
perm = flex.sort_permutation(completeness)
rows = [rows[i] for i in perm]
print
print 'Unit cell clustering summary:'
print tabulate(rows, headers, tablefmt='rst')
print
blend_html = tabulate(rows, headers, tablefmt='html').replace(
'<table>', '<table class="table table-hover table-condensed">').replace(
'<td>', '<td style="text-align: right;">')
def _integrate_prepare(self):
'''Prepare for integration - in XDS terms this may mean rerunning
IDXREF to get the XPARM etc. DEFPIX is considered part of the full
integration as it is resolution dependent.'''
from xia2.Handlers.Citations import Citations
Citations.cite('xds')
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(min(images),
max(images))
Debug.write('XDS INTEGRATE PREPARE:')
Debug.write('Wavelength: %.6f' % self.get_wavelength())
Debug.write('Distance: %.2f' % self.get_distance())
idxr = self._intgr_refiner.get_refiner_indexer(self.get_integrater_epoch())
if idxr is None:
idxr = XDSIndexer()
self._intgr_refiner.add_refiner_indexer(
self.get_integrater_epoch(), idxr)
self.set_integrater_prepare_done(False)
#self.set_integrater_indexer()
idxr.set_indexer_sweep(self.get_integrater_sweep())
idxr.set_working_directory(self.get_working_directory())
idxr.setup_from_imageset(self.get_imageset())
if self.get_frame_wedge():
wedge = self.get_frame_wedge()
Debug.write('Propogating wedge limit: %d %d' % wedge)
idxr.set_frame_wedge(wedge[0], wedge[1], apply_offset = False)
# this needs to be set up from the contents of the
# Integrater frame processer - wavelength &c.
if self.get_beam_centre():
idxr.set_beam_centre(self.get_beam_centre())
if self.get_distance():
idxr.set_distance(self.get_distance())
if self.get_wavelength():
idxr.set_wavelength(self.get_wavelength())
# get the unit cell from this indexer to initiate processing
# if it is new... and also copy out all of the information for
# the XDS indexer if not...
# copy the data across
self._xds_data_files = copy.deepcopy(
self._intgr_refiner.get_refiner_payload(self.get_integrater_epoch()))
Debug.write('Files available at the end of XDS integrate prepare:')
for f in self._xds_data_files.keys():
Debug.write('%s' % f)
experiment = self._intgr_refiner.get_refined_experiment_list(
self.get_integrater_epoch())[0]
# copy across the trusted_range - it got lost along the way
old_detector = self.get_detector()
self.set_detector(experiment.detector)
for p1, p2 in zip(old_detector, self.get_detector()):
p2.set_trusted_range(p1.get_trusted_range())
self.set_beam_obj(experiment.beam)
self.set_goniometer(experiment.goniometer)
# set a low resolution limit (which isn't really used...)
# this should perhaps be done more intelligently from an
# analysis of the spot list or something...?
if not self.get_integrater_low_resolution():
dmax = self._intgr_refiner.get_indexer_low_resolution(
self.get_integrater_epoch())
self.set_integrater_low_resolution(dmax)
Debug.write('Low resolution set to: %s' % \
self.get_integrater_low_resolution())
# delete things we should not know e.g. the postrefined cell from
# CORRECT - c/f bug # 2695
self._intgr_cell = None
self._intgr_spacegroup_number = None
return
def TwoThetaRefine(DriverType = None):
'''A factory for RefineWrapper classes.'''
from xia2.Handlers.Citations import Citations
Citations.cite('dials')
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class RefineWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable('dials.two_theta_refine')
self._reindexing_operator = None
self._reindexed_experiments = None
self._reindexed_reflections = None
self._experiments = []
self._pickles = []
self._phil_file = None
# The following are set during run() call:
self._output_cif = None
self._output_mmcif = None
self._output_correlation_plot = None
self._output_experiments = None
self._crystal = None
def set_experiments(self, experiments):
self._experiments = experiments
def get_experiments(self):
return self._experiments
def set_pickles(self, pickles):
self._pickles = pickles
def get_pickles(self):
return self._pickles
def set_phil_file(self, phil_file):
self._phil_file = phil_file
def get_output_cif(self):
return self._output_cif
def get_output_mmcif(self):
return self._output_mmcif
def get_output_correlation_plot(self):
return self._output_correlation_plot
def get_output_experiments(self):
return self._output_experiments
def get_unit_cell(self):
return self._crystal.get_unit_cell().parameters()
def get_unit_cell_esd(self):
return self._crystal.get_cell_parameter_sd()
def set_reindex_operator(self, operator):
self._reindexing_operator = operator
def run(self):
from xia2.Handlers.Streams import Chatter, Debug
if self._reindexing_operator:
Debug.write('Reindexing sweeps for dials.two_theta_refine')
from xia2.lib.bits import auto_logfiler
from xia2.Wrappers.Dials.Reindex import Reindex
self._reindexed_experiments, self._reindexed_reflections = [], []
for e, p in zip(self._experiments, self._pickles):
reindexer = Reindex()
reindexer.set_cb_op(self._reindexing_operator)
reindexer.set_experiments_filename(e)
reindexer.set_indexed_filename(p)
reindexer.set_working_directory(self.get_working_directory())
auto_logfiler(reindexer)
reindexer.run()
self._reindexed_experiments.append(reindexer.get_reindexed_experiments_filename())
self._reindexed_reflections.append(reindexer.get_reindexed_reflections_filename())
Debug.write('Running dials.two_theta_refine')
self._output_cif = os.path.join(
self.get_working_directory(),
'%s_dials.two_theta_refine.cif' % self.get_xpid())
self._output_mmcif = os.path.join(
self.get_working_directory(),
'%s_dials.two_theta_refine.mmcif' % self.get_xpid())
self._output_correlation_plot = os.path.join(
self.get_working_directory(),
'%s_dials.two_theta_refine.png' % self.get_xpid())
self._output_experiments = os.path.join(
self.get_working_directory(),
'%s_refined_cell.json' % self.get_xpid())
self.clear_command_line()
if self._reindexing_operator:
for experiment in self._reindexed_experiments:
self.add_command_line(experiment)
for pickle in self._reindexed_reflections:
self.add_command_line(pickle)
else:
for experiment in self._experiments:
self.add_command_line(experiment)
for pickle in self._pickles:
self.add_command_line(pickle)
self.add_command_line('output.cif=%s' % self._output_cif)
self.add_command_line('output.mmcif=%s' % self._output_mmcif)
if self._output_correlation_plot is not None:
self.add_command_line(
'output.correlation_plot.filename=%s' % self._output_correlation_plot)
if self._output_experiments is not None:
self.add_command_line(
'output.experiments=%s' % self._output_experiments)
if self._phil_file is not None:
self.add_command_line('%s' %self._phil_file)
self.start()
self.close_wait()
if not os.path.isfile(self._output_cif):
Chatter.write(
"TwoTheta refinement failed, see log file for more details:\n %s" % self.get_log_file())
raise RuntimeError, 'unit cell not refined'
self.check_for_errors()
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(self.get_output_experiments())
self._crystal = experiments.crystals()[0]
def import_cif(self):
'''Import relevant lines from .cif output'''
import iotbx.cif
cif = iotbx.cif.reader(file_path=self.get_output_cif()).model()
block = cif['two_theta_refine']
subset = { k: block[k] for k in block.keys() if k.startswith(('_cell', '_diffrn')) }
return subset
def import_mmcif(self):
'''Import relevant lines from .mmcif output'''
if os.path.isfile(self.get_output_mmcif()):
import iotbx.cif
cif = iotbx.cif.reader(file_path=self.get_output_mmcif()).model()
block = cif['two_theta_refine']
else:
import iotbx.cif.model
block = iotbx.cif.model.block()
subset = { k: block[k] for k in block.keys() if k.startswith(('_cell', '_diffrn')) }
return subset
return RefineWrapper()
def generate_xia2_html(xinfo, filename='xia2.html', params=None, args=[]):
assert params is None or len(args) == 0
if params is None:
from xia2.Modules.Analysis import phil_scope
interp = phil_scope.command_line_argument_interpreter()
params, unhandled = interp.process_and_fetch(
args, custom_processor='collect_remaining')
params = params.extract()
from xia2.command_line.report import xia2_report
crystal = xinfo.get_crystals().values()[0]
xia2_txt = os.path.join(os.path.abspath(os.path.curdir), 'xia2.txt')
assert os.path.isfile(xia2_txt), xia2_txt
with open(xia2_txt, 'rb') as f:
xia2_output = f.read().encode('ascii', 'xmlcharrefreplace')
xia2_output = cgi.escape(xia2_output)
styles = {}
reports = []
columns = []
columns.append([
'',
u'Wavelength (Å)',
u'Resolution range (Å)',
'Completeness (%)',
'Multiplicity',
'CC-half',
'I/sigma',
'Rmerge(I)',
# anomalous statistics
'Anomalous completeness (%)',
'Anomalous multiplicity'
])
individual_dataset_reports = {}
for cname, xcryst in xinfo.get_crystals().iteritems():
reflection_files = xcryst.get_scaled_merged_reflections()
for wname, unmerged_mtz in reflection_files['mtz_unmerged'].iteritems(
):
xwav = xcryst.get_xwavelength(wname)
report = xia2_report(unmerged_mtz, params)
reports.append(report)
merging_stats = report.merging_stats
merging_stats_anom = report.merging_stats_anom
overall = merging_stats.overall
overall_anom = merging_stats_anom.overall
outer_shell = merging_stats.bins[-1]
outer_shell_anom = merging_stats_anom.bins[-1]
column = [
wname,
str(xwav.get_wavelength()),
'%.2f - %.2f (%.2f - %.2f)' %
(overall.d_max, overall.d_min, outer_shell.d_max,
outer_shell.d_min),
'%.2f (%.2f)' %
(overall.completeness * 100, outer_shell.completeness * 100),
'%.2f (%.2f)' %
(overall.mean_redundancy, outer_shell.mean_redundancy),
'%.4f (%.4f)' % (overall.cc_one_half, outer_shell.cc_one_half),
'%.2f (%.2f)' %
(overall.i_over_sigma_mean, outer_shell.i_over_sigma_mean),
'%.4f (%.4f)' % (overall.r_merge, outer_shell.r_merge),
# anomalous statistics
'%.2f (%.2f)' % (overall_anom.anom_completeness * 100,
outer_shell_anom.anom_completeness * 100),
'%.2f (%.2f)' % (overall_anom.mean_redundancy,
outer_shell_anom.mean_redundancy),
]
columns.append(column)
xtriage_success, xtriage_warnings, xtriage_danger = None, None, None
if params.xtriage_analysis:
try:
xtriage_success, xtriage_warnings, xtriage_danger = report.xtriage_report(
)
except Exception as e:
from xia2.Handlers.Phil import PhilIndex
if PhilIndex.params.xia2.settings.small_molecule == True:
print("Xtriage output not available: %s" % str(e))
else:
raise
d = {}
d['merging_statistics_table'] = report.merging_statistics_table()
d['overall_statistics_table'] = report.overall_statistics_table()
individual_dataset_reports[wname] = d
json_data = {}
json_data.update(report.multiplicity_vs_resolution_plot())
json_data.update(report.multiplicity_histogram())
json_data.update(report.completeness_plot())
json_data.update(report.scale_rmerge_vs_batch_plot())
json_data.update(report.cc_one_half_plot())
json_data.update(report.i_over_sig_i_plot())
json_data.update(report.i_over_sig_i_vs_batch_plot())
json_data.update(report.second_moments_plot())
json_data.update(report.cumulative_intensity_distribution_plot())
json_data.update(report.l_test_plot())
json_data.update(report.wilson_plot())
json_data.update(report.pychef_plots(n_bins=1))
from scitbx.array_family import flex
max_points = 500
for g in ('scale_rmerge_vs_batch', 'completeness_vs_dose',
'rcp_vs_dose', 'scp_vs_dose', 'rd_vs_batch_difference'):
for i, data in enumerate(json_data[g]['data']):
x = data['x']
n = len(x)
if n > max_points:
step = n // max_points
sel = (flex.int_range(n) % step) == 0
data['x'] = list(flex.int(data['x']).select(sel))
data['y'] = list(flex.double(data['y']).select(sel))
resolution_graphs = collections.OrderedDict(
(k + '_' + wname, json_data[k])
for k in ('cc_one_half', 'i_over_sig_i', 'second_moments',
'wilson_intensity_plot', 'completeness',
'multiplicity_vs_resolution') if k in json_data)
if params.include_radiation_damage:
batch_graphs = collections.OrderedDict(
(k + '_' + wname, json_data[k])
for k in ('scale_rmerge_vs_batch', 'i_over_sig_i_vs_batch',
'completeness_vs_dose', 'rcp_vs_dose',
'scp_vs_dose', 'rd_vs_batch_difference'))
else:
batch_graphs = collections.OrderedDict(
(k + '_' + wname, json_data[k])
for k in ('scale_rmerge_vs_batch',
'i_over_sig_i_vs_batch'))
misc_graphs = collections.OrderedDict(
(k + '_' + wname, json_data[k])
for k in ('cumulative_intensity_distribution', 'l_test',
'multiplicities') if k in json_data)
for k, v in report.multiplicity_plots().iteritems():
misc_graphs[k + '_' + wname] = {'img': v}
d['resolution_graphs'] = resolution_graphs
d['batch_graphs'] = batch_graphs
d['misc_graphs'] = misc_graphs
d['xtriage'] = {
'success': xtriage_success,
'warnings': xtriage_warnings,
'danger': xtriage_danger
}
table = [[c[i] for c in columns] for i in range(len(columns[0]))]
cell = xcryst.get_cell()
from cctbx import sgtbx
space_groups = xcryst.get_likely_spacegroups()
space_groups = [
sgtbx.space_group_info(symbol=str(symbol)) for symbol in space_groups
]
space_group = space_groups[0].symbol_and_number()
alternative_space_groups = [
sg.symbol_and_number() for sg in space_groups[1:]
]
unit_cell = str(report.intensities.unit_cell())
#twinning_score = xcryst._get_scaler()._scalr_twinning_score
#twinning_conclusion = xcryst._get_scaler()._scalr_twinning_conclusion
#if twinning_score is not None:
#table.append(['','',''])
#table.append(['Twinning score', '%.2f' %twinning_score, ''])
#if twinning_conclusion is not None:
#table.append(['', twinning_conclusion, ''])
for row in table:
for i in range(len(row)):
row[i] = row[i].encode('ascii', 'xmlcharrefreplace')
#from libtbx import table_utils
#print table_utils.format(rows=table, has_header=True)
# reflection files
for cname, xcryst in xinfo.get_crystals().iteritems():
# hack to replace path to reflection files with DataFiles directory
data_dir = os.path.join(os.path.abspath(os.path.curdir), 'DataFiles')
g = glob.glob(os.path.join(data_dir, '*'))
reflection_files = xcryst.get_scaled_merged_reflections()
for k, rfile in reflection_files.iteritems():
if isinstance(rfile, basestring):
for datafile in g:
if os.path.basename(datafile) == os.path.basename(rfile):
reflection_files[k] = datafile
break
else:
for kk in rfile.keys():
for datafile in g:
if os.path.basename(datafile) == os.path.basename(
rfile[kk]):
reflection_files[k][kk] = datafile
break
headers = ['Dataset', 'File name']
merged_mtz = reflection_files['mtz']
mtz_files = [
headers,
[
'All datasets',
'<a href="%s">%s</a>' %
(os.path.relpath(merged_mtz), os.path.basename(merged_mtz))
]
]
for wname, unmerged_mtz in reflection_files['mtz_unmerged'].iteritems(
):
mtz_files.append([
wname,
'<a href="%s">%s</a>' %
(os.path.relpath(unmerged_mtz), os.path.basename(unmerged_mtz))
])
sca_files = [headers]
for wname, merged_sca in reflection_files['sca'].iteritems():
sca_files.append([
wname,
'<a href="%s">%s</a>' %
(os.path.relpath(merged_sca), os.path.basename(merged_sca))
])
unmerged_sca_files = [headers]
for wname, unmerged_sca in reflection_files['sca_unmerged'].iteritems(
):
unmerged_sca_files.append([
wname,
'<a href="%s">%s</a>' %
(os.path.relpath(unmerged_sca), os.path.basename(unmerged_sca))
])
# other files
other_files = []
other_files.append(['File name', 'Description'])
for other_file, description in sorted([
('xia2.cif', 'Crystallographic information file'),
('xia2.mmcif', 'Macromolecular crystallographic information file'),
('shelxt.hkl', 'merged structure factors for SHELXT'),
('shelxt.ins', 'SHELXT instruction file'),
] + [
(fn, 'XPREP input file') for fn in os.listdir(os.path.join(data_dir)) \
if fn.endswith('.p4p')
]):
if os.path.exists(os.path.join(data_dir, other_file)):
other_files.append([
'<a href="DataFiles/{filename}">{filename}</a>'.format(
filename=other_file), description
])
# log files
log_files_table = []
log_dir = os.path.join(os.path.abspath(os.path.curdir), 'LogFiles')
g = glob.glob(os.path.join(log_dir, '*.log'))
for logfile in g:
html_file = make_logfile_html(logfile)
html_file = os.path.splitext(logfile)[0] + '.html'
if os.path.exists(html_file):
log_files_table.append([
os.path.basename(logfile),
'<a href="%s">original</a>' % os.path.relpath(logfile),
'<a href="%s">html</a>' % os.path.relpath(html_file),
])
else:
log_files_table.append([
os.path.basename(logfile),
'<a href="%s">original</a>' % os.path.relpath(logfile),
' ',
])
# references
references = {}
for cdict in Citations.get_citations_dicts():
references[cdict['acta']] = cdict.get('url')
from jinja2 import Environment, ChoiceLoader, PackageLoader
loader = ChoiceLoader([
PackageLoader('xia2', 'templates'),
PackageLoader('dials', 'templates')
])
env = Environment(loader=loader)
template = env.get_template('xia2.html')
html = template.render(
page_title='xia2 processing report',
xia2_output=xia2_output,
space_group=space_group,
alternative_space_groups=alternative_space_groups,
unit_cell=unit_cell,
xtriage_success=xtriage_success,
xtriage_warnings=xtriage_warnings,
xtriage_danger=xtriage_danger,
overall_stats_table=table,
cc_half_significance_level=params.cc_half_significance_level,
mtz_files=mtz_files,
sca_files=sca_files,
unmerged_sca_files=unmerged_sca_files,
other_files=other_files,
log_files_table=log_files_table,
individual_dataset_reports=individual_dataset_reports,
references=references,
styles=styles)
with open(filename, 'wb') as f:
f.write(html.encode('ascii', 'xmlcharrefreplace'))
def _integrate(self):
"""Implement the integrater interface."""
# cite the program
Citations.cite("mosflm")
images_str = "%d to %d" % tuple(self._intgr_wedge)
cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % tuple(self._intgr_cell)
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = "...%s" % self._fp_directory[-46:]
Journal.block(
"integrating",
self._intgr_sweep_name,
"mosflm",
{
"images": images_str,
"cell": cell_str,
"lattice": self.get_integrater_refiner().get_refiner_lattice(),
"template": self._fp_template,
"directory": dirname,
"resolution": "%.2f" % self._intgr_reso_high,
},
)
self._mosflm_rerun_integration = False
wd = self.get_working_directory()
try:
if self.get_integrater_sweep_name():
pname, xname, dname = self.get_integrater_project_info()
nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
if nproc > 1:
Debug.write("Parallel integration: %d jobs" % nproc)
self._mosflm_hklout = self._mosflm_parallel_integrate()
else:
self._mosflm_hklout = self._mosflm_integrate()
# record integration output for e.g. BLEND.
sweep = self.get_integrater_sweep_name()
if sweep:
FileHandler.record_more_data_file(
"%s %s %s %s INTEGRATE" % (pname, xname, dname, sweep),
self._mosflm_hklout,
)
except IntegrationError as e:
if "negative mosaic spread" in str(e):
if self._mosflm_postref_fix_mosaic:
Chatter.write(
"Negative mosaic spread - stopping integration")
raise BadLatticeError("negative mosaic spread")
Chatter.write("Negative mosaic spread - rerunning integration")
self.set_integrater_done(False)
self._mosflm_postref_fix_mosaic = True
if self._mosflm_rerun_integration and not PhilIndex.params.dials.fast_mode:
# make sure that this is run again...
Chatter.write("Need to rerun the integration...")
self.set_integrater_done(False)
return self._mosflm_hklout
def _index_prepare(self):
from xia2.Handlers.Citations import Citations
Citations.cite('dials')
#all_images = self.get_matching_images()
#first = min(all_images)
#last = max(all_images)
spot_lists = []
datablocks = []
for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps):
Chatter.banner('Spotfinding %s' % xsweep.get_name())
first, last = imageset.get_scan().get_image_range()
# at this stage, break out to run the DIALS code: this sets itself up
# now cheat and pass in some information... save re-reading all of the
# image headers
# FIXME need to adjust this to allow (say) three chunks of images
from dxtbx.serialize import dump
from dxtbx.datablock import DataBlock
sweep_filename = os.path.join(
self.get_working_directory(),
'%s_datablock.json' % xsweep.get_name())
dump.datablock(DataBlock([imageset]), sweep_filename)
genmask = self.GenerateMask()
genmask.set_input_datablock(sweep_filename)
genmask.set_output_datablock(
os.path.join(
self.get_working_directory(), '%s_%s_datablock.json' %
(genmask.get_xpid(), xsweep.get_name())))
genmask.set_params(PhilIndex.params.dials.masking)
sweep_filename, mask_pickle = genmask.run()
Debug.write('Generated mask for %s: %s' %
(xsweep.get_name(), mask_pickle))
gain = PhilIndex.params.xia2.settings.input.gain
if gain is libtbx.Auto:
gain_estimater = self.EstimateGain()
gain_estimater.set_sweep_filename(sweep_filename)
gain_estimater.run()
gain = gain_estimater.get_gain()
Chatter.write('Estimated gain: %.2f' % gain)
PhilIndex.params.xia2.settings.input.gain = gain
# FIXME this should really use the assigned spot finding regions
#offset = self.get_frame_offset()
spotfinder = self.Spotfinder()
if last - first > 10:
spotfinder.set_write_hot_mask(True)
spotfinder.set_input_sweep_filename(sweep_filename)
spotfinder.set_output_sweep_filename(
'%s_%s_datablock.json' %
(spotfinder.get_xpid(), xsweep.get_name()))
spotfinder.set_input_spot_filename(
'%s_%s_strong.pickle' %
(spotfinder.get_xpid(), xsweep.get_name()))
if PhilIndex.params.dials.fast_mode:
wedges = self._index_select_images_i(imageset)
spotfinder.set_scan_ranges(wedges)
else:
spotfinder.set_scan_ranges([(first, last)])
if PhilIndex.params.dials.find_spots.phil_file is not None:
spotfinder.set_phil_file(
PhilIndex.params.dials.find_spots.phil_file)
min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size
if min_spot_size is libtbx.Auto:
if imageset.get_detector()[0].get_type() == 'SENSOR_PAD':
min_spot_size = 3
else:
min_spot_size = None
if min_spot_size is not None:
spotfinder.set_min_spot_size(min_spot_size)
min_local = PhilIndex.params.dials.find_spots.min_local
if min_local is not None:
spotfinder.set_min_local(min_local)
sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong
if sigma_strong:
spotfinder.set_sigma_strong(sigma_strong)
gain = PhilIndex.params.xia2.settings.input.gain
if gain:
spotfinder.set_gain(gain)
filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings
if filter_ice_rings:
spotfinder.set_filter_ice_rings(filter_ice_rings)
kernel_size = PhilIndex.params.dials.find_spots.kernel_size
if kernel_size:
spotfinder.set_kernel_size(kernel_size)
global_threshold = PhilIndex.params.dials.find_spots.global_threshold
if global_threshold is not None:
spotfinder.set_global_threshold(global_threshold)
spotfinder.run()
spot_filename = spotfinder.get_spot_filename()
if not os.path.exists(spot_filename):
raise RuntimeError("Spotfinding failed: %s does not exist." %
os.path.basename(spot_filename))
spot_lists.append(spot_filename)
datablocks.append(spotfinder.get_output_sweep_filename())
from libtbx import easy_pickle
from dials.util.ascii_art import spot_counts_per_image_plot
refl = easy_pickle.load(spot_filename)
if not len(refl):
raise RuntimeError('No spots found in sweep %s' %
xsweep.get_name())
Chatter.write(spot_counts_per_image_plot(refl), strip=False)
if not PhilIndex.params.dials.fast_mode:
detectblanks = self.DetectBlanks()
detectblanks.set_sweep_filename(datablocks[-1])
detectblanks.set_reflections_filename(spot_filename)
detectblanks.run()
json = detectblanks.get_results()
offset = imageset.get_scan().get_image_range()[0]
blank_regions = json['strong']['blank_regions']
if len(blank_regions):
blank_regions = [(int(s), int(e))
for s, e in blank_regions]
for blank_start, blank_end in blank_regions:
Chatter.write(
'WARNING: Potential blank images: %i -> %i' %
(blank_start + 1, blank_end))
if PhilIndex.params.xia2.settings.remove_blanks:
non_blanks = []
start, end = imageset.get_array_range()
last_blank_end = start
for blank_start, blank_end in blank_regions:
if blank_start > start:
non_blanks.append(
(last_blank_end, blank_start))
last_blank_end = blank_end
if last_blank_end + 1 < end:
non_blanks.append((last_blank_end, end))
xsweep = self.get_indexer_sweep()
xwav = xsweep.get_wavelength()
xsample = xsweep.get_xsample()
sweep_name = xsweep.get_name()
import string
for i, (nb_start, nb_end) in enumerate(non_blanks):
assert i < 26
if i == 0:
sub_imageset = imageset[nb_start -
start:nb_end - start]
xsweep._frames_to_process = (nb_start + 1,
nb_end + 1)
self.set_indexer_prepare_done(done=False)
self._indxr_imagesets[
self._indxr_imagesets.index(
imageset)] = sub_imageset
xsweep._integrater._setup_from_imageset(
sub_imageset)
else:
new_name = '_'.join(
(sweep_name, string.ascii_lowercase[i]))
new_sweep = xwav.add_sweep(
new_name,
xsample,
directory=os.path.join(
os.path.basename(
xsweep.get_directory()), new_name),
image=imageset.get_path(nb_start - start),
frames_to_process=(nb_start + 1, nb_end),
)
Chatter.write(
"Generating new sweep: %s (%s:%i:%i)" %
(new_sweep.get_name(),
new_sweep.get_image(),
new_sweep.get_frames_to_process()[0],
new_sweep.get_frames_to_process()[1]))
return
if not PhilIndex.params.xia2.settings.trust_beam_centre:
discovery = self.DiscoverBetterExperimentalModel()
discovery.set_sweep_filename(datablocks[-1])
discovery.set_spot_filename(spot_filename)
#wedges = self._index_select_images_i(imageset)
#discovery.set_scan_ranges(wedges)
#discovery.set_scan_ranges([(first + offset, last + offset)])
try:
discovery.run()
except Exception as e:
Debug.write('DIALS beam centre search failed: %s' % str(e))
else:
# overwrite datablock.json in datablocks list
datablocks[
-1] = discovery.get_optimized_datablock_filename()
self.set_indexer_payload("spot_lists", spot_lists)
self.set_indexer_payload("datablocks", datablocks)
return
def _integrate(self):
'''Implement the integrater interface.'''
# cite the program
Citations.cite('mosflm')
images_str = '%d to %d' % tuple(self._intgr_wedge)
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % tuple(self._intgr_cell)
if len(self._fp_directory) <= 50:
dirname = self._fp_directory
else:
dirname = '...%s' % self._fp_directory[-46:]
Journal.block(
'integrating', self._intgr_sweep_name, 'mosflm', {
'images': images_str,
'cell': cell_str,
'lattice': self.get_integrater_refiner().get_refiner_lattice(),
'template': self._fp_template,
'directory': dirname,
'resolution': '%.2f' % self._intgr_reso_high
})
self._mosflm_rerun_integration = False
wd = self.get_working_directory()
try:
if self.get_integrater_sweep_name():
pname, xname, dname = self.get_integrater_project_info()
nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
if nproc > 1:
Debug.write('Parallel integration: %d jobs' % nproc)
self._mosflm_hklout = self._mosflm_parallel_integrate()
else:
self._mosflm_hklout = self._mosflm_integrate()
# record integration output for e.g. BLEND.
sweep = self.get_integrater_sweep_name()
if sweep:
FileHandler.record_more_data_file(
'%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep),
self._mosflm_hklout)
except IntegrationError as e:
if 'negative mosaic spread' in str(e):
if self._mosflm_postref_fix_mosaic:
Chatter.write(
'Negative mosaic spread - stopping integration')
raise BadLatticeError('negative mosaic spread')
Chatter.write('Negative mosaic spread - rerunning integration')
self.set_integrater_done(False)
self._mosflm_postref_fix_mosaic = True
if self._mosflm_rerun_integration and not PhilIndex.params.dials.fast_mode:
# make sure that this is run again...
Chatter.write('Need to rerun the integration...')
self.set_integrater_done(False)
return self._mosflm_hklout
def _integrate_prepare(self):
"""Prepare for integration - in XDS terms this may mean rerunning
IDXREF to get the XPARM etc. DEFPIX is considered part of the full
integration as it is resolution dependent."""
Citations.cite("xds")
# decide what images we are going to process, if not already
# specified
if not self._intgr_wedge:
images = self.get_matching_images()
self.set_integrater_wedge(
min(images) + self.get_frame_offset(),
max(images) + self.get_frame_offset(),
)
logger.debug("XDS INTEGRATE PREPARE:")
logger.debug("Wavelength: %.6f" % self.get_wavelength())
logger.debug("Distance: %.2f" % self.get_distance())
idxr = self._intgr_refiner.get_refiner_indexer(
self.get_integrater_epoch())
if idxr is None:
idxr = XDSIndexer()
self._intgr_refiner.add_refiner_indexer(
self.get_integrater_epoch(), idxr)
self.set_integrater_prepare_done(False)
# self.set_integrater_indexer()
idxr.set_indexer_sweep(self.get_integrater_sweep())
idxr.set_working_directory(self.get_working_directory())
idxr.setup_from_imageset(self.get_imageset())
if self.get_frame_wedge():
wedge = self.get_frame_wedge()
logger.debug("Propogating wedge limit: %d %d" % wedge)
idxr.set_frame_wedge(wedge[0], wedge[1], apply_offset=False)
# this needs to be set up from the contents of the
# Integrater frame processer - wavelength &c.
if self.get_beam_centre():
idxr.set_beam_centre(self.get_beam_centre())
if self.get_distance():
idxr.set_distance(self.get_distance())
if self.get_wavelength():
idxr.set_wavelength(self.get_wavelength())
# get the unit cell from this indexer to initiate processing
# if it is new... and also copy out all of the information for
# the XDS indexer if not...
# copy the data across
self._xds_data_files = copy.deepcopy(
self._intgr_refiner.get_refiner_payload(
self.get_integrater_epoch()))
if self._xds_data_files is None:
self._xds_data_files = {}
logger.debug("Files available at the end of XDS integrate prepare:")
for f in self._xds_data_files:
logger.debug("%s" % f)
experiment = self._intgr_refiner.get_refined_experiment_list(
self.get_integrater_epoch())[0]
# copy across the trusted_range - it got lost along the way
old_detector = self.get_detector()
self.set_detector(experiment.detector)
for p1, p2 in zip(old_detector, self.get_detector()):
p2.set_trusted_range(p1.get_trusted_range())
self.set_beam_obj(experiment.beam)
self.set_goniometer(experiment.goniometer)
# set a low resolution limit (which isn't really used...)
# this should perhaps be done more intelligently from an
# analysis of the spot list or something...?
if not self.get_integrater_low_resolution():
dmax = self._intgr_refiner.get_indexer_low_resolution(
self.get_integrater_epoch())
self.set_integrater_low_resolution(dmax)
logger.debug("Low resolution set to: %s" %
self.get_integrater_low_resolution())
# delete things we should not know e.g. the postrefined cell from
# CORRECT - c/f bug # 2695
self._intgr_cell = None
self._intgr_spacegroup_number = None
def xia2_main(stop_after=None):
'''Actually process something...'''
Citations.cite('xia2')
# print versions of related software
from dials.util.version import dials_version
Chatter.write(dials_version())
start_time = time.time()
CommandLine = get_command_line()
start_dir = Flags.get_starting_directory()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
no_images = True
for name in xtals.keys():
xtal = xtals[name]
if not xtal.get_all_image_names():
Chatter.write('-----------------------------------' + \
'-' * len(name))
Chatter.write('| No images assigned for crystal %s |' % name)
Chatter.write('-----------------------------------' + '-' \
* len(name))
else:
no_images = False
args = []
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
from libtbx import group_args
xinfo = CommandLine.get_xinfo()
if os.path.exists('xia2.json'):
from xia2.Schema.XProject import XProject
xinfo_new = xinfo
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new.keys():
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names():
wavelength_new = crystals_new[crystal_id].get_xwavelength(wavelength_id)
if wavelength_id not in crystals[crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = [s.get_name() for s in sweeps]
sweep_keys = [
(s.get_directory(), s.get_template(), s.get_image_range())
for s in sweeps]
for sweep in sweeps_new:
if ((sweep.get_directory(), sweep.get_template(),
sweep.get_image_range()) not in sweep_keys):
if sweep.get_name() in sweep_names:
i = 1
while 'SWEEEP%i' %i in sweep_names:
i += 1
sweep._name = 'SWEEP%i' %i
break
wavelength.add_sweep(
name=sweep.get_name(),
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.append(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
if njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
args.append((
group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
),))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(args):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
if mp_params.type == "qsub":
method = "sge"
else:
method = "multiprocessing"
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command
if not qsub_command:
qsub_command = 'qsub'
qsub_command = '%s -V -cwd -pe smp %d' %(qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep, args, processes=njob,
#method=method,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
assert xsweep_dict is not None
if output is not None:
Chatter.write(output)
if not success:
Chatter.write('Sweep failed: removing %s' %sweep.get_name())
remove_sweeps.append(sweep)
else:
Chatter.write('Loading sweep: %s' % sweep.get_name())
from xia2.Schema.XSweep import XSweep
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
else:
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
try:
if stop_after == 'index':
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception, e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(sweep.get_name(), str(e)))
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
def xia2_main(stop_after=None):
'''Actually process something...'''
Citations.cite('xia2')
# print versions of related software
Chatter.write(dials_version())
ccp4_version = get_ccp4_version()
if ccp4_version is not None:
Chatter.write('CCP4 %s' % ccp4_version)
start_time = time.time()
CommandLine = get_command_line()
start_dir = Flags.get_starting_directory()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
no_images = True
for name in xtals.keys():
xtal = xtals[name]
if not xtal.get_all_image_names():
Chatter.write('-----------------------------------' + \
'-' * len(name))
Chatter.write('| No images assigned for crystal %s |' % name)
Chatter.write('-----------------------------------' + '-' \
* len(name))
else:
no_images = False
args = []
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
from libtbx import group_args
xinfo = CommandLine.get_xinfo()
if os.path.exists('xia2.json'):
from xia2.Schema.XProject import XProject
xinfo_new = xinfo
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new.keys():
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names(
):
wavelength_new = crystals_new[crystal_id].get_xwavelength(
wavelength_id)
if wavelength_id not in crystals[
crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(
wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = [s.get_name() for s in sweeps]
sweep_keys = [(s.get_directory(), s.get_template(),
s.get_image_range()) for s in sweeps]
for sweep in sweeps_new:
if ((sweep.get_directory(), sweep.get_template(),
sweep.get_image_range()) not in sweep_keys):
if sweep.get_name() in sweep_names:
i = 1
while 'SWEEEP%i' % i in sweep_names:
i += 1
sweep._name = 'SWEEP%i' % i
break
wavelength.add_sweep(
name=sweep.get_name(),
sample=sweep.get_xsample(),
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.append(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
if mp_params.mode == 'parallel' and njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
args.append((group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
), ))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(args):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
if mp_params.type == "qsub":
method = "sge"
else:
method = "multiprocessing"
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command
if not qsub_command:
qsub_command = 'qsub'
qsub_command = '%s -V -cwd -pe smp %d' % (qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep,
args,
processes=njob,
#method=method,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
if output is not None:
Chatter.write(output)
if not success:
Chatter.write('Sweep failed: removing %s' %
sweep.get_name())
remove_sweeps.append(sweep)
else:
assert xsweep_dict is not None
Chatter.write('Loading sweep: %s' % sweep.get_name())
from xia2.Schema.XSweep import XSweep
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
else:
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
from dials.command_line.show import show_datablocks
from dxtbx.datablock import DataBlock
Debug.write(sweep.get_name())
Debug.write(
show_datablocks([DataBlock([sweep.get_imageset()])]))
try:
if stop_after == 'index':
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception as e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(sweep.get_name(), str(e)))
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
# save intermediate xia2.json file in case scaling step fails
xinfo.as_json(filename='xia2.json')
if stop_after not in ('index', 'integrate'):
Chatter.write(xinfo.get_output(), strip=False)
for crystal in crystals.values():
crystal.serialize()
# save final xia2.json file in case report generation fails
xinfo.as_json(filename='xia2.json')
duration = time.time() - start_time
# write out the time taken in a human readable way
Chatter.write('Processing took %s' % \
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
if stop_after not in ('index', 'integrate'):
# and the summary file
with open('xia2-summary.dat', 'w') as fh:
for record in xinfo.summarise():
fh.write('%s\n' % record)
# looks like this import overwrites the initial command line
# Phil overrides so... for https://github.com/xia2/xia2/issues/150
from xia2.command_line.html import generate_xia2_html
if params.xia2.settings.small_molecule == True:
params.xia2.settings.report.xtriage_analysis = False
params.xia2.settings.report.include_radiation_damage = False
generate_xia2_html(xinfo,
filename='xia2.html',
params=params.xia2.settings.report)
write_citations()
# delete all of the temporary mtz files...
cleanup()
Environment.cleanup()
def xia2_main(stop_after=None):
"""Actually process something..."""
Citations.cite("xia2")
# print versions of related software
logger.info(dials_version())
ccp4_version = get_ccp4_version()
if ccp4_version:
logger.info("CCP4 %s", ccp4_version)
start_time = time.time()
CommandLine = get_command_line()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
for name, xtal in xtals.items():
if not xtal.get_all_image_names():
logger.info("-----------------------------------" +
"-" * len(name))
logger.info("| No images assigned for crystal %s |", name)
logger.info("-----------------------------------" +
"-" * len(name))
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
xinfo = CommandLine.get_xinfo()
logger.info("Project directory: %s", xinfo.path)
if (params.xia2.settings.developmental.continue_from_previous_job
and os.path.exists("xia2.json")):
logger.debug("==== Starting from existing xia2.json ====")
xinfo_new = xinfo
xinfo = XProject.from_json(filename="xia2.json")
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new:
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names(
):
wavelength_new = crystals_new[crystal_id].get_xwavelength(
wavelength_id)
if wavelength_id not in crystals[
crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(
wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = {s.get_name() for s in sweeps}
sweep_keys = {(s.get_directory(), s.get_template(),
s.get_image_range())
for s in sweeps}
for sweep in sweeps_new:
if (
sweep.get_directory(),
sweep.get_template(),
sweep.get_image_range(),
) not in sweep_keys:
if sweep.get_name() in sweep_names:
i = 1
while "SWEEEP%i" % i in sweep_names:
i += 1
sweep._name = "SWEEP%i" % i
break
wavelength.add_sweep(
name=sweep.get_name(),
sample=sweep.sample,
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.add(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
with cleanup(xinfo.path):
if mp_params.mode == "parallel" and njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
jobs = []
for crystal_id in crystals:
for wavelength_id in crystals[crystal_id].get_wavelength_names(
):
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
jobs.append((group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
), ))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(jobs):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command or "qsub"
qsub_command = "%s -V -cwd -pe smp %d" % (qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep,
jobs,
processes=njob,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True,
)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals:
for wavelength_id in crystals[crystal_id].get_wavelength_names(
):
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
if output is not None:
logger.info(output)
if not success:
logger.info("Sweep failed: removing %s",
sweep.get_name())
remove_sweeps.append(sweep)
else:
assert xsweep_dict is not None
logger.info("Loading sweep: %s", sweep.get_name())
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.sample
sample.remove_sweep(sweep)
else:
for crystal_id in list(crystals.keys()):
for wavelength_id in crystals[crystal_id].get_wavelength_names(
):
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
from dials.command_line.show import show_experiments
from dxtbx.model.experiment_list import ExperimentListFactory
logger.debug(sweep.get_name())
logger.debug(
show_experiments(
ExperimentListFactory.
from_imageset_and_crystal(
sweep.get_imageset(), None)))
Citations.cite("dials")
try:
if stop_after == "index":
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception as e:
if failover:
logger.info(
"Processing sweep %s failed: %s",
sweep.get_name(),
str(e),
)
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.sample
sample.remove_sweep(sweep)
# save intermediate xia2.json file in case scaling step fails
xinfo.as_json(filename="xia2.json")
if stop_after not in ("index", "integrate"):
logger.info(xinfo.get_output())
for crystal in list(crystals.values()):
crystal.serialize()
# save final xia2.json file in case report generation fails
xinfo.as_json(filename="xia2.json")
if stop_after not in ("index", "integrate"):
# and the summary file
with open("xia2-summary.dat", "w") as fh:
for record in xinfo.summarise():
fh.write("%s\n" % record)
# looks like this import overwrites the initial command line
# Phil overrides so... for https://github.com/xia2/xia2/issues/150
from xia2.command_line.html import generate_xia2_html
if params.xia2.settings.small_molecule:
params.xia2.settings.report.xtriage_analysis = False
params.xia2.settings.report.include_radiation_damage = False
with xia2.Driver.timing.record_step("xia2.report"):
generate_xia2_html(xinfo,
filename="xia2.html",
params=params.xia2.settings.report)
duration = time.time() - start_time
# write out the time taken in a human readable way
logger.info("Processing took %s",
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
write_citations()
def _index(self):
'''Implement the indexer interface.'''
Citations.cite('mosflm')
indexer = MosflmIndex()
indexer.set_working_directory(self.get_working_directory())
auto_logfiler(indexer)
from xia2.lib.bits import unique_elements
_images = unique_elements(self._indxr_images)
indexer.set_images(_images)
images_str = ', '.join(map(str, _images))
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
if self._indxr_sweep_name:
#if len(self._fp_directory) <= 50:
#dirname = self._fp_directory
#else:
#dirname = '...%s' % self._fp_directory[-46:]
dirname = os.path.dirname(self.get_imageset().get_template())
Journal.block(
'autoindexing', self._indxr_sweep_name, 'mosflm', {
'images': images_str,
'target cell': self._indxr_input_cell,
'target lattice': self._indxr_input_lattice,
'template': self.get_imageset().get_template(),
'directory': dirname
})
#task = 'Autoindex from images:'
#for i in _images:
#task += ' %s' % self.get_image_name(i)
#self.set_task(task)
indexer.set_template(os.path.basename(self.get_template()))
indexer.set_directory(self.get_directory())
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
indexer.set_distance(xsweep.get_distance())
#if self.get_wavelength_prov() == 'user':
#index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
indexer.set_beam_centre(xsweep.get_beam_centre())
if self._indxr_input_cell:
indexer.set_unit_cell(self._indxr_input_cell)
if self._indxr_input_lattice is not None:
spacegroup_number = lattice_to_spacegroup(
self._indxr_input_lattice)
indexer.set_space_group_number(spacegroup_number)
if not self._mosflm_autoindex_thresh:
try:
min_peaks = 200
Debug.write('Aiming for at least %d spots...' % min_peaks)
thresholds = []
for i in _images:
p = Printpeaks()
p.set_working_directory(self.get_working_directory())
auto_logfiler(p)
p.set_image(self.get_image_name(i))
thresh = p.threshold(min_peaks)
Debug.write('Autoindex threshold for image %d: %d' % \
(i, thresh))
thresholds.append(thresh)
thresh = min(thresholds)
self._mosflm_autoindex_thresh = thresh
except Exception as e:
print str(e) #XXX this should disappear!
Debug.write('Error computing threshold: %s' % str(e))
Debug.write('Using default of 20.0')
thresh = 20.0
else:
thresh = self._mosflm_autoindex_thresh
Debug.write('Using autoindex threshold: %d' % thresh)
if self._mosflm_autoindex_sol:
indexer.set_solution_number(self._mosflm_autoindex_sol)
indexer.set_threshold(thresh)
# now forget this to prevent weird things happening later on
if self._mosflm_autoindex_sol:
self._mosflm_autoindex_sol = 0
indexer.run()
indxr_cell = indexer.get_refined_unit_cell()
self._indxr_lattice = indexer.get_lattice()
space_group_number = indexer.get_indexed_space_group_number()
detector_distance = indexer.get_refined_distance()
beam_centre = indexer.get_refined_beam_centre()
mosaic_spreads = indexer.get_mosaic_spreads()
if min(list(indxr_cell)) < 10.0 and \
indxr_cell[2] / indxr_cell[0] > 6:
Debug.write('Unrealistic autoindexing solution: ' +
'%.2f %.2f %.2f %.2f %.2f %.2f' % indxr_cell)
# tweak some parameters and try again...
self._mosflm_autoindex_thresh *= 1.5
self.set_indexer_done(False)
return
intgr_params = {}
# look up other possible indexing solutions (not well - in
# standard settings only!) This is moved earlier as it could
# result in returning if Mosflm has selected the wrong
# solution!
try:
self._indxr_other_lattice_cell = indexer.get_solutions()
# Change 27/FEB/08 to support user assigned spacegroups
if self._indxr_user_input_lattice:
lattice_to_spacegroup_dict = {
'aP': 1,
'mP': 3,
'mC': 5,
'oP': 16,
'oC': 20,
'oF': 22,
'oI': 23,
'tP': 75,
'tI': 79,
'hP': 143,
'hR': 146,
'cP': 195,
'cF': 196,
'cI': 197
}
for k in self._indxr_other_lattice_cell.keys():
if lattice_to_spacegroup_dict[k] > \
lattice_to_spacegroup_dict[
self._indxr_input_lattice]:
del (self._indxr_other_lattice_cell[k])
# check that the selected unit cell matches - and if
# not raise a "horrible" exception
if self._indxr_input_cell:
assert indxr_cell is not None
for j in range(6):
if math.fabs(self._indxr_input_cell[j] -
indxr_cell[j]) > 2.0:
Chatter.write('Mosflm autoindexing did not select ' +
'correct (target) unit cell')
raise RuntimeError(
'something horrible happened in indexing')
except RuntimeError as e:
# check if mosflm rejected a solution we have it
if 'horribl' in str(e):
# ok it did - time to break out the big guns...
if not self._indxr_input_cell:
raise RuntimeError(
'error in solution selection when not preset')
# XXX FIXME
self._mosflm_autoindex_sol = _get_indexing_solution_number(
indexer.get_all_output(), self._indxr_input_cell,
self._indxr_input_lattice)
# set the fact that we are not done...
self.set_indexer_done(False)
# and return - hopefully this will restart everything
return
else:
raise e
if len(mosaic_spreads) == 0:
# then consider setting it do a default value...
# equal to the oscillation width (a good guess)
phi_width = self.get_phi_width()
Chatter.write(
'Mosaic estimation failed, so guessing at %4.2f' % \
phi_width)
# only consider this if we have thus far no idea on the
# mosaic spread...
mosaic_spreads.append(phi_width)
intgr_params['raster'] = indexer.get_raster()
intgr_params['separation'] = indexer.get_separation()
self._indxr_resolution_estimate = indexer.get_resolution_estimate()
# compute mosaic as mean(mosaic_spreads)
self._indxr_mosaic = sum(mosaic_spreads) / len(mosaic_spreads)
self._indxr_payload['mosflm_integration_parameters'] = intgr_params
self._indxr_payload['mosflm_orientation_matrix'] = open(
os.path.join(self.get_working_directory(), 'xiaindex.mat'),
'r').readlines()
import copy
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
from xia2.Wrappers.Mosflm.AutoindexHelpers import set_distance
from xia2.Wrappers.Mosflm.AutoindexHelpers import crystal_model_from_mosflm_mat
from cctbx import sgtbx, uctbx
# update the beam centre (i.e. shift the origin of the detector)
detector = copy.deepcopy(self.get_detector())
beam = copy.deepcopy(self.get_beam())
set_mosflm_beam_centre(detector, beam, beam_centre)
if detector_distance is not None:
set_distance(detector, detector_distance)
# make a dxtbx crystal_model object from the mosflm matrix
space_group = sgtbx.space_group_info(number=space_group_number).group()
crystal_model = crystal_model_from_mosflm_mat(
self._indxr_payload['mosflm_orientation_matrix'],
unit_cell=uctbx.unit_cell(tuple(indxr_cell)),
space_group=space_group)
# construct an experiment_list
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(beam=beam,
detector=detector,
goniometer=self.get_goniometer(),
scan=self.get_scan(),
crystal=crystal_model)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
def _index_prepare(self):
from xia2.Handlers.Citations import Citations
Citations.cite('dials')
#all_images = self.get_matching_images()
#first = min(all_images)
#last = max(all_images)
spot_lists = []
datablocks = []
for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps):
Chatter.banner('Spotfinding %s' %xsweep.get_name())
first, last = imageset.get_scan().get_image_range()
# at this stage, break out to run the DIALS code: this sets itself up
# now cheat and pass in some information... save re-reading all of the
# image headers
# FIXME need to adjust this to allow (say) three chunks of images
from dxtbx.serialize import dump
from dxtbx.datablock import DataBlock
sweep_filename = os.path.join(
self.get_working_directory(), '%s_datablock.json' %xsweep.get_name())
dump.datablock(DataBlock([imageset]), sweep_filename)
gain = PhilIndex.params.xia2.settings.input.gain
if gain is libtbx.Auto:
gain_estimater = self.EstimateGain()
gain_estimater.set_sweep_filename(sweep_filename)
gain_estimater.run()
gain = gain_estimater.get_gain()
Chatter.write('Estimated gain: %.2f' %gain)
PhilIndex.params.xia2.settings.input.gain = gain
# FIXME this should really use the assigned spot finding regions
#offset = self.get_frame_offset()
spotfinder = self.Spotfinder()
if last - first > 10:
spotfinder.set_write_hot_mask(True)
spotfinder.set_input_sweep_filename(sweep_filename)
spotfinder.set_output_sweep_filename(
'%s_%s_datablock.json' %(spotfinder.get_xpid(), xsweep.get_name()))
spotfinder.set_input_spot_filename(
'%s_%s_strong.pickle' %(spotfinder.get_xpid(), xsweep.get_name()))
if PhilIndex.params.dials.fast_mode:
wedges = self._index_select_images_i(imageset)
spotfinder.set_scan_ranges(wedges)
else:
spotfinder.set_scan_ranges([(first, last)])
if PhilIndex.params.dials.find_spots.phil_file is not None:
spotfinder.set_phil_file(PhilIndex.params.dials.find_spots.phil_file)
min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size
if min_spot_size is libtbx.Auto:
if imageset.get_detector()[0].get_type() == 'SENSOR_PAD':
min_spot_size = 3
else:
min_spot_size = None
if min_spot_size is not None:
spotfinder.set_min_spot_size(min_spot_size)
min_local = PhilIndex.params.dials.find_spots.min_local
if min_local is not None:
spotfinder.set_min_local(min_local)
sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong
if sigma_strong:
spotfinder.set_sigma_strong(sigma_strong)
gain = PhilIndex.params.xia2.settings.input.gain
if gain:
spotfinder.set_gain(gain)
filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings
if filter_ice_rings:
spotfinder.set_filter_ice_rings(filter_ice_rings)
kernel_size = PhilIndex.params.dials.find_spots.kernel_size
if kernel_size:
spotfinder.set_kernel_size(kernel_size)
global_threshold = PhilIndex.params.dials.find_spots.global_threshold
if global_threshold is not None:
spotfinder.set_global_threshold(global_threshold)
spotfinder.run()
spot_filename = spotfinder.get_spot_filename()
if not os.path.exists(spot_filename):
raise RuntimeError("Spotfinding failed: %s does not exist."
%os.path.basename(spot_filename))
spot_lists.append(spot_filename)
datablocks.append(spotfinder.get_output_sweep_filename())
from libtbx import easy_pickle
from dials.util.ascii_art import spot_counts_per_image_plot
refl = easy_pickle.load(spot_filename)
if not len(refl):
raise RuntimeError('No spots found in sweep %s' %xsweep.get_name())
Chatter.write(spot_counts_per_image_plot(refl), strip=False)
if not PhilIndex.params.dials.fast_mode:
detectblanks = self.DetectBlanks()
detectblanks.set_sweep_filename(datablocks[-1])
detectblanks.set_reflections_filename(spot_filename)
detectblanks.run()
json = detectblanks.get_results()
offset = imageset.get_scan().get_image_range()[0]
blank_regions = json['strong']['blank_regions']
if len(blank_regions):
blank_regions = [(int(s), int(e)) for s, e in blank_regions]
for blank_start, blank_end in blank_regions:
Chatter.write('WARNING: Potential blank images: %i -> %i' %(
blank_start+1, blank_end))
if PhilIndex.params.xia2.settings.remove_blanks:
non_blanks = []
start, end = imageset.get_array_range()
last_blank_end = start
for blank_start, blank_end in blank_regions:
if blank_start > start:
non_blanks.append((last_blank_end, blank_start))
last_blank_end = blank_end
if last_blank_end+1 < end:
non_blanks.append((last_blank_end, end))
xsweep = self.get_indexer_sweep()
xwav = xsweep.get_wavelength()
xsample = xsweep.get_xsample()
sweep_name = xsweep.get_name()
import string
for i, (nb_start, nb_end) in enumerate(non_blanks):
assert i < 26
if i == 0:
sub_imageset = imageset[nb_start-start:nb_end-start]
xsweep._frames_to_process = (nb_start+1, nb_end+1)
self.set_indexer_prepare_done(done=False)
self._indxr_imagesets[self._indxr_imagesets.index(imageset)] = sub_imageset
xsweep._integrater._setup_from_imageset(sub_imageset)
else:
new_name = '_'.join((sweep_name, string.ascii_lowercase[i]))
new_sweep = xwav.add_sweep(new_name,
xsample,
directory=os.path.join(
os.path.basename(xsweep.get_directory()), new_name),
image=imageset.get_path(nb_start-start),
frames_to_process=(nb_start+1, nb_end),
)
Chatter.write("Generating new sweep: %s (%s:%i:%i)" %(
new_sweep.get_name(),
new_sweep.get_image(),
new_sweep.get_frames_to_process()[0],
new_sweep.get_frames_to_process()[1]))
return
if not PhilIndex.params.xia2.settings.trust_beam_centre:
discovery = self.DiscoverBetterExperimentalModel()
discovery.set_sweep_filename(datablocks[-1])
discovery.set_spot_filename(spot_filename)
#wedges = self._index_select_images_i(imageset)
#discovery.set_scan_ranges(wedges)
#discovery.set_scan_ranges([(first + offset, last + offset)])
try:
discovery.run()
except Exception, e:
Debug.write('DIALS beam centre search failed: %s' %str(e))
else:
# overwrite datablock.json in datablocks list
datablocks[-1] = discovery.get_optimized_datablock_filename()
