def run_one_sweep(args):
from xia2.Handlers.Streams import Debug
assert len(args) == 3
s, failover, job_type = args
if job_type:
DriverFactory.set_driver_type(job_type)
Chatter.cache()
Debug.cache()
try:
s.get_integrater_intensities()
except Exception as e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(s.get_name(), str(e)))
s = None
else:
raise
finally:
Chatter.uncache()
Debug.uncache()
return s
def SplitExperiments(DriverType=None):
"""A factory for CombineExperimentsWrapper classes."""
DriverInstance = DriverFactory.Driver(DriverType)
class SplitExperimentsWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable("dials.split_experiments")
self._experiments_filename = []
self._reflections_filename = []
self._experiments_prefix = None
self._reflections_prefix = None
self._by_wavelength = False
def add_experiments(self, experiments_filename):
self._experiments_filename.append(experiments_filename)
def add_reflections(self, reflections_filename):
self._reflections_filename.append(reflections_filename)
def get_experiments_filename(self):
return self._experiments_filename
def get_reflections_filename(self):
return self._experiments_filename
def set_by_wavelength(self, boolean):
self._by_wavelength = boolean
def run(self):
logger.debug("Running dials.split_experiments")
self.clear_command_line()
assert len(self._experiments_filename) == 1
assert len(self._experiments_filename) == len(
self._reflections_filename)
self.add_command_line(self._experiments_filename)
self.add_command_line(self._reflections_filename)
if not self._experiments_prefix:
self._experiments_prefix = "split"
self.add_command_line("output.experiments_prefix=%s" %
self._experiments_prefix)
if not self._reflections_prefix:
self._reflections_prefix = "split"
self.add_command_line("output.reflections_prefix=%s" %
self._reflections_prefix)
if self._by_wavelength:
self.add_command_line("by_wavelength=True")
self.start()
self.close_wait()
self.check_for_errors()
return SplitExperimentsWrapper()
def ExportXDS(DriverType=None):
"""A factory for ExportXDSWrapper classes."""
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class ExportXDSWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable("dials.export")
self._sweep_filename = None
self._crystal_filename = None
def set_experiments_filename(self, experiments_filename):
self._experiments_filename = experiments_filename
def run(self):
from xia2.Handlers.Streams import Debug
Debug.write("Running dials.export")
self.clear_command_line()
self.add_command_line(self._experiments_filename)
self.add_command_line("format=xds")
self.start()
self.close_wait()
self.check_for_errors()
return ExportXDSWrapper()
def Xtriage(DriverType=None):
"""A factory for the Xtriage wrappers."""
DriverInstance = DriverFactory.Driver("simple")
class XtriageWrapper(DriverInstance.__class__):
"""A wrapper class for phenix.xtriage."""
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable("mmtbx.xtriage")
self._mtz = None
return
def set_mtz(self, mtz):
self._mtz = mtz
return
def run(self):
import os
assert self._mtz is not None
assert os.path.isfile(self._mtz)
self.add_command_line(self._mtz)
self.start()
self.close_wait()
self.check_for_errors()
return XtriageWrapper()
def Header(DriverType=None):
"""A factory for HeaderWrapper(ipmosflm) classes."""
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class HeaderWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
import os
self.set_executable(
os.path.join(os.environ["CCP4"], "bin", "ipmosflm"))
from collections import defaultdict
self._headers = defaultdict(dict)
def __call__(self, fp, images=None):
from xia2.Handlers.Streams import Debug
if images is None:
images = fp.get_matching_images()
Debug.write("Running mosflm to read header from images %d to %d" %
(min(images), max(images)))
self.start()
self.input('template "%s"' % fp.get_template())
self.input('directory "%s"' % fp.get_directory())
self.input("image %d" % images[0])
self.input("head")
self.input("go")
for image in images[1:]:
self.input("image %d" % image)
self.input("head brief")
self.input("go")
self.close_wait()
image = -1
for record in self.get_all_output():
if "===> image" in record:
image = int(record.split()[-1])
continue
if "Start and end phi values" in record:
tokens = record.split()
start, end = float(tokens[-4]), float(tokens[-2])
image = int(tokens[7])
self._headers["phi-start"][image] = start
self._headers["phi-end"][image] = end
if "recognized as:" in record:
self._headers["mosflm-detector"] = record.split()[-1]
if "Wavelength of" in record:
self._headers["wavelength"] = float(
record.split()[2].replace("A", ""))
return HeaderWrapper()
def BrehmDiederichs(DriverType=None):
'''A factory for BrehmDiederichsWrapper classes.'''
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class BrehmDiederichsWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable('cctbx.brehm_diederichs')
self._input_filenames = []
self._asymmetric = None
self._output_filenames = []
self._reindexing_dict = {}
return
def set_input_filenames(self, filenames):
self._input_filenames = filenames
return
def set_asymmetric(self, asymmetric):
self._asymmetric = asymmetric
def get_output_filenames(self):
return self._output_filenames
def get_reindexing_dict(self):
return self._reindexing_dict
def run(self):
from xia2.Handlers.Streams import Debug
Debug.write('Running cctbx.brehm_diederichs')
self.clear_command_line()
if self._asymmetric is not None:
assert isinstance(self._asymmetric, int)
self.add_command_line('asymmetric=%i' % self._asymmetric)
self.add_command_line('show_plot=False')
self.add_command_line('save_plot=True')
for filename in self._input_filenames:
self.add_command_line(filename)
self.start()
self.close_wait()
self.check_for_errors()
import os
results_filename = os.path.join(self.get_working_directory(),
'reindex.txt')
assert os.path.exists(results_filename)
with open(results_filename, 'rb') as f:
for line in f.readlines():
filename, reindex_op = line.strip().rsplit(' ', 1)
self._reindexing_dict[os.path.abspath(
filename)] = reindex_op
return
return BrehmDiederichsWrapper()
def Header(DriverType = None):
'''A factory for HeaderWrapper(ipmosflm) classes.'''
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class HeaderWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
import os
self.set_executable(os.path.join(
os.environ['CCP4'], 'bin', 'ipmosflm'))
from collections import defaultdict
self._headers = defaultdict(dict)
def __call__(self, fp, images = None):
from xia2.Handlers.Streams import Debug
if images is None:
images = fp.get_matching_images()
Debug.write('Running mosflm to read header from images %d to %d' % \
(min(images), max(images)))
self.start()
self.input('template "%s"' % fp.get_template())
self.input('directory "%s"' % fp.get_directory())
self.input('image %d' % images[0])
self.input('head')
self.input('go')
for image in images[1:]:
self.input('image %d' % image)
self.input('head brief')
self.input('go')
self.close_wait()
image = -1
for record in self.get_all_output():
if '===> image' in record:
image = int(record.split()[-1])
continue
if 'Start and end phi values' in record:
tokens = record.split()
start, end = float(tokens[-4]), float(tokens[-2])
image = int(tokens[7])
self._headers['phi-start'][image] = start
self._headers['phi-end'][image] = end
if 'recognized as:' in record:
self._headers['mosflm-detector'] = record.split()[-1]
if 'Wavelength of' in record:
self._headers['wavelength'] = float(
record.split()[2].replace('A', ''))
return HeaderWrapper()
def anvil_correction(driver_type=None):
"""A factory for AnvilCorrectionWrapper classes."""
driver_instance = DriverFactory.Driver(driver_type)
class AnvilCorrectionWrapper(driver_instance.__class__):
"""Wrap dials.anvil_correction."""
def __init__(self):
super(AnvilCorrectionWrapper, self).__init__()
self.set_executable("dials.anvil_correction")
# Input and output files.
# None is a valid value only for the output experiment list filename.
self.experiments_filenames = [] # type: List[str, ...]
self.reflections_filenames = [] # type: List[str, ...]
self.output_experiments_filename = None # type: Optional[str]
self.output_reflections_filename = None # type: Optional[str]
# Parameters to pass to dials.anvil_correction
self.density = None # type: Optional[SupportsFloat]
self.thickness = None # type: Optional[SupportsFloat]
self.normal = None # type: Optional[Tuple[3 * (SupportsFloat,)]]
def run(self):
"""Run dials.anvil_correction if the parameters are valid."""
# We should only start if the properties have been set.
assert self.experiments_filenames
assert self.reflections_filenames
# None is a valid value for the output experiment list filename.
assert self.output_reflections_filename
assert self.density
assert self.thickness
assert self.normal
self.clear_command_line()
self.add_command_line(self.experiments_filenames)
self.add_command_line(self.reflections_filenames)
if self.output_experiments_filename:
self.add_command_line("output.experiments=%s" %
self.output_experiments_filename)
self.add_command_line("output.reflections=%s" %
self.output_reflections_filename)
self.add_command_line("anvil.density=%s" % self.density)
self.add_command_line("anvil.thickness=%s" % self.thickness)
self.add_command_line("anvil.normal=%s,%s,%s" % tuple(self.normal))
self.start()
self.close_wait()
self.check_for_errors()
assert os.path.exists(self.output_reflections_filename)
return AnvilCorrectionWrapper()
def StereographicProjection(DriverType=None):
'''A factory for StereographicProjectionWrapper classes.'''
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class StereographicProjectionWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable('dials.stereographic_projection')
self._experiments_filenames = []
self._hkl = None
self._plot_filename = None
self._json_filename = None
def add_experiments(self, experiments_filename):
self._experiments_filenames.append(experiments_filename)
def set_hkl(self, hkl):
assert len(hkl) == 3
self._hkl = hkl
def get_plot_filename(self):
return self._plot_filename
def get_json_filename(self):
return self._json_filename
def run(self):
from xia2.Handlers.Streams import Debug
Debug.write('Running dials.stereographic_projection')
assert len(self._experiments_filenames) > 0
assert self._hkl is not None
self.clear_command_line()
for expt in self._experiments_filenames:
self.add_command_line(expt)
self.add_command_line('frame=laboratory')
self.add_command_line('plot.show=False')
self.add_command_line('hkl=%i,%i,%i' % self._hkl)
self._plot_filename = '%i_stereographic_projection_%i%i%i.png' % (
self.get_xpid(), self._hkl[0], self._hkl[1], self._hkl[2])
self._json_filename = '%i_stereographic_projection_%i%i%i.json' % (
self.get_xpid(), self._hkl[0], self._hkl[1], self._hkl[2])
self.add_command_line('plot.filename=%s' % self._plot_filename)
self.add_command_line('json.filename=%s' % self._json_filename)
self.start()
self.close_wait()
self.check_for_errors()
return StereographicProjectionWrapper()
def ExportSpotXDS(DriverType=None, params=None):
DriverInstance = DriverFactory.Driver(DriverType)
class ExportSpotXDSWrapper(DriverInstance.__class__, FrameProcessor):
"""A wrapper for wrapping dials.export_spot_xds."""
def __init__(self, params=None):
super(ExportSpotXDSWrapper, self).__init__()
# phil parameters
if not params:
params = master_params.extract()
self._params = params
# now set myself up...
self.set_executable("dials.export_spot_xds")
self._input_data_files = {}
self._output_data_files = {}
self._input_data_files_list = []
self._output_data_files_list = []
# getter and setter for input / output data
def set_input_data_file(self, name, data):
self._input_data_files[name] = data
def get_output_data_file(self, name):
return self._output_data_files[name]
def run(self):
"""Run dials.spotfinder."""
self.add_command_line(self._input_data_files.keys())
self.start()
self.close_wait()
self.check_for_errors()
self._output_data_files.setdefault(
"SPOT.XDS",
open(
os.path.join(self.get_working_directory(), "SPOT.XDS"), "rb"
).read(),
)
output = self.get_all_output()
print("".join(output))
return ExportSpotXDSWrapper(params=params)
def Report(DriverType=None):
'''A factory for ReportWrapper classes.'''
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class ReportWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable('xia2.report')
self._mtz_filename = None
self._html_filename = None
self._chef_min_completeness = None
return
def set_mtz_filename(self, mtz_filename):
self._mtz_filename = mtz_filename
def set_html_filename(self, html_filename):
self._html_filename = html_filename
def get_html_filename(self, html_filename):
return self._html_filename
def set_chef_min_completeness(self, min_completeness):
self._chef_min_completeness = min_completeness
def run(self):
from xia2.Handlers.Streams import Debug
Debug.write('Running xia2.report')
assert self._mtz_filename is not None
self.clear_command_line()
self.add_command_line(self._mtz_filename)
if self._chef_min_completeness is not None:
self.add_command_line('chef_min_completeness=%s' %
self._chef_min_completeness)
self.start()
self.close_wait()
self.check_for_errors()
html_filename = os.path.join(self.get_working_directory(),
'xia2-report.html')
assert os.path.exists(html_filename)
if self._html_filename is None:
self._html_filename = html_filename
else:
import shutil
shutil.move(html_filename, self._html_filename)
return
return ReportWrapper()
def MultiCrystalAnalysis(DriverType=None):
'''A factory for MultiCrystalAnalysisWrapper classes.'''
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class MultiCrystalAnalysisWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable('cctbx.python')
self._argv = []
self._nproc = None
self._njob = None
self._mp_mode = None
self._phil_file = None
self._clusters = None
return
def add_command_line_args(self, args):
self._argv.extend(args)
def run(self):
from xia2.Handlers.Streams import Debug
Debug.write('Running MultiCrystalAnalysis.py')
self.clear_command_line()
from xia2.Modules import MultiCrystalAnalysis as mca_module
self.add_command_line(mca_module.__file__)
for arg in self._argv:
self.add_command_line(arg)
self.start()
self.close_wait()
self.check_for_errors()
self._clusters_json = os.path.join(self.get_working_directory(),
'intensity_clusters.json')
assert os.path.exists(self._clusters_json)
import json
with open(self._clusters_json, 'rb') as f:
self._dict = json.load(f)
self._clusters = self._dict['clusters']
return
def get_clusters(self):
return self._clusters
def get_dict(self):
return self._dict
return MultiCrystalAnalysisWrapper()
def LabelitMosflmMatrix(DriverType=None):
'''Factory for LabelitMosflmMatrix wrapper classes, with the specified
Driver type.'''
DriverInstance = DriverFactory.Driver(DriverType)
class LabelitMosflmMatrixWrapper(DriverInstance.__class__):
'''A wrapper for the program labelit.mosflm_matrix - which will
calculate the matrix for mosflm integration.'''
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable('labelit.mosflm_matrix')
self._solution = None
self._mosflm_beam = None
def set_solution(self, solution):
self._solution = solution
def calculate(self):
'''Compute matrix for solution #.'''
if self._solution is None:
raise RuntimeError('solution not selected')
task = 'Compute matrix for solution %02d' % self._solution
self.add_command_line('%d' % self._solution)
self.start()
self.close_wait()
output = open(
os.path.join(self.get_working_directory(),
'integration%02d.csh' %
self._solution)).readlines()
matrix = output[2:11]
# also check for the beam centre in mosflm land! - ignoring
# SWUNG OUT though I should probably check the two-theta
# value too...
for o in output:
if 'BEAM' in o[:4]:
self._mosflm_beam = map(float, o.split()[-2:])
return matrix
def get_mosflm_beam(self):
return self._mosflm_beam
return LabelitMosflmMatrixWrapper()
def run_one_sweep(args):
from xia2.Handlers.Streams import Debug
assert len(args) == 3
s, failover, job_type = args
if job_type:
DriverFactory.set_driver_type(job_type)
Chatter.cache()
Debug.cache()
try:
s.get_integrater_intensities()
except Exception, e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(s.get_name(), str(e)))
s = None
else:
raise
def BrehmDiederichs(DriverType=None):
"""A factory for BrehmDiederichsWrapper classes."""
DriverInstance = DriverFactory.Driver(DriverType)
class BrehmDiederichsWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable("cctbx.brehm_diederichs")
self._input_filenames = []
self._asymmetric = None
self._output_filenames = []
self._reindexing_dict = {}
def set_input_filenames(self, filenames):
self._input_filenames = filenames
def set_asymmetric(self, asymmetric):
self._asymmetric = asymmetric
def get_reindexing_dict(self):
return self._reindexing_dict
def run(self):
logger.debug("Running cctbx.brehm_diederichs")
self.clear_command_line()
if self._asymmetric is not None:
assert isinstance(self._asymmetric, int)
self.add_command_line("asymmetric=%i" % self._asymmetric)
self.add_command_line("show_plot=False")
self.add_command_line("save_plot=True")
for filename in self._input_filenames:
self.add_command_line(filename)
self.start()
self.close_wait()
self.check_for_errors()
results_filename = os.path.join(self.get_working_directory(),
"reindex.txt")
assert os.path.exists(results_filename)
with open(results_filename, "rb") as f:
for line in f.readlines():
filename, reindex_op = line.strip().rsplit(" ", 1)
self._reindexing_dict[os.path.abspath(
filename)] = reindex_op
return BrehmDiederichsWrapper()
def ShadowPlot(DriverType=None):
"""A factory for ShadowPlotWrapper classes."""
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class ShadowPlotWrapper(DriverInstance.__class__, FrameProcessor):
def __init__(self):
super(ShadowPlotWrapper, self).__init__()
self.set_executable("dials.shadow_plot")
self._sweep_filename = None
self._json_filename = None
def set_sweep_filename(self, sweep_filename):
self._sweep_filename = sweep_filename
def set_json_filename(self, json_filename):
self._json_filename = json_filename
def get_json_filename(self):
return self._json_filename
def get_results(self):
assert self._json_filename is not None and os.path.isfile(
self._json_filename
)
import json
with open(self._json_filename, "rb") as f:
results = json.load(f)
return results
def run(self):
self.clear_command_line()
assert self._sweep_filename is not None
self.add_command_line("%s" % self._sweep_filename)
if self._json_filename is not None:
self.add_command_line("json=%s" % self._json_filename)
self.add_command_line("mode=1d")
self.start()
self.close_wait()
self.check_for_errors()
return ShadowPlotWrapper()
def ExportBest(DriverType=None):
"""A factory for ExportMtzWrapper classes."""
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class ExportBestWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable("dials.export")
self._experiments_filename = None
self._reflections_filename = None
self._prefix = "best"
def set_experiments_filename(self, experiments_filename):
self._experiments_filename = experiments_filename
def get_experiments_filename(self):
return self._experiments_filename
def set_reflections_filename(self, reflections_filename):
self._reflections_filename = reflections_filename
def get_reflections_filename(self):
return self._reflections_filename
def set_prefix(self, prefix):
self._prefix = prefix
def get_prefix(self):
return self._prefix
def run(self):
from xia2.Handlers.Streams import Debug
Debug.write("Running dials.export")
self.clear_command_line()
self.add_command_line("experiments=%s" % self._experiments_filename)
self.add_command_line("reflections=%s" % self._reflections_filename)
self.add_command_line("format=best")
self.add_command_line("best.prefix=%s" % self._prefix)
self.start()
self.close_wait()
self.check_for_errors()
return ExportBestWrapper()
def Report(DriverType=None):
"""A factory for ReportWrapper classes."""
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class ReportWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable("dials.report")
self._experiments_filename = None
self._reflections_filename = None
self._html_filename = None
def set_experiments_filename(self, experiments_filename):
self._experiments_filename = experiments_filename
def set_reflections_filename(self, reflections_filename):
self._reflections_filename = reflections_filename
def set_html_filename(self, html_filename):
self._html_filename = html_filename
def run(self, wait_for_completion=False):
from xia2.Handlers.Streams import Debug
Debug.write("Running dials.report")
self.clear_command_line()
assert (self._experiments_filename is not None
or self._reflections_filename is not None)
if self._experiments_filename is not None:
self.add_command_line(self._experiments_filename)
if self._reflections_filename is not None:
self.add_command_line(self._reflections_filename)
if self._html_filename is not None:
self.add_command_line("output.html=%s" % self._html_filename)
self.start()
if wait_for_completion:
self.close_wait()
else:
self.close()
self.check_for_errors()
return ReportWrapper()
def ExportXDSASCII(DriverType=None):
"""A factory for ExportXDSASCIISWrapper classes."""
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class ExportXDSASCIISWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable("dials.export")
self._experiments_filename = None
self._reflections_filename = None
self._hkl_filename = "DIALS.HKL"
def set_experiments_filename(self, experiments_filename):
self._experiments_filename = experiments_filename
def set_reflections_filename(self, reflections_filename):
self._reflections_filename = reflections_filename
def set_hkl_filename(self, hkl_filename):
self._hkl_filename = hkl_filename
def get_hkl_filename(self):
return self._hkl_filename
def run(self):
from xia2.Handlers.Streams import Debug
Debug.write("Running dials.export")
assert self._experiments_filename is not None
assert self._reflections_filename is not None
self.clear_command_line()
self.add_command_line(self._experiments_filename)
self.add_command_line(self._reflections_filename)
if self._hkl_filename is not None:
self.add_command_line("xds_ascii.hklout=%s" %
self._hkl_filename)
self.add_command_line("format=xds_ascii")
self.start()
self.close_wait()
self.check_for_errors()
return ExportXDSASCIISWrapper()
def GenerateRaster(DriverType=None):
"""A factory for GenerateRasterWrapper(ipmosflm) classes."""
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class GenerateRasterWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
import os
self.set_executable(
os.path.join(os.environ["CCP4"], "bin", "ipmosflm"))
def __call__(self, indxr, images):
from xia2.Handlers.Streams import Debug
Debug.write("Running mosflm to generate RASTER, SEPARATION")
self.start()
self.input('template "%s"' % indxr.get_template())
self.input('directory "%s"' % indxr.get_directory())
self.input("beam %f %f" % indxr.get_indexer_beam_centre())
self.input("distance %f" % indxr.get_indexer_distance())
self.input("wavelength %f" % indxr.get_wavelength())
self.input("findspots file spots.dat")
for i in images:
self.input("findspots find %d" % i)
self.input("go")
self.close_wait()
p = {}
# scrape from the output the values we want...
for o in self.get_all_output():
if "parameters have been set to" in o:
p["raster"] = map(int, o.split()[-5:])
if "(currently SEPARATION" in o:
p["separation"] = map(float,
o.replace(")", "").split()[-2:])
return p
return GenerateRasterWrapper()
def GenerateRaster(DriverType=None):
'''A factory for GenerateRasterWrapper(ipmosflm) classes.'''
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class GenerateRasterWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
import os
self.set_executable(
os.path.join(os.environ['CCP4'], 'bin', 'ipmosflm'))
def __call__(self, indxr, images):
from xia2.Handlers.Streams import Debug
Debug.write('Running mosflm to generate RASTER, SEPARATION')
self.start()
self.input('template "%s"' % indxr.get_template())
self.input('directory "%s"' % indxr.get_directory())
self.input('beam %f %f' % indxr.get_indexer_beam_centre())
self.input('distance %f' % indxr.get_indexer_distance())
self.input('wavelength %f' % indxr.get_wavelength())
self.input('findspots file spots.dat')
for i in images:
self.input('findspots find %d' % i)
self.input('go')
self.close_wait()
p = {}
# scrape from the output the values we want...
for o in self.get_all_output():
if 'parameters have been set to' in o:
p['raster'] = map(int, o.split()[-5:])
if '(currently SEPARATION' in o:
p['separation'] = map(float,
o.replace(')', '').split()[-2:])
return p
return GenerateRasterWrapper()
def Report(DriverType=None):
"""A factory for ReportWrapper classes."""
DriverInstance = DriverFactory.Driver(DriverType)
class ReportWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable("xia2.report")
self._mtz_filename = None
self._html_filename = None
self._chef_min_completeness = None
def set_mtz_filename(self, mtz_filename):
self._mtz_filename = mtz_filename
def set_html_filename(self, html_filename):
self._html_filename = html_filename
def set_chef_min_completeness(self, min_completeness):
self._chef_min_completeness = min_completeness
def run(self):
logger.debug("Running xia2.report")
assert self._mtz_filename is not None
self.clear_command_line()
self.add_command_line(self._mtz_filename)
if self._chef_min_completeness is not None:
self.add_command_line("chef_min_completeness=%s" %
self._chef_min_completeness)
self.start()
self.close_wait()
self.check_for_errors()
html_filename = os.path.join(self.get_working_directory(),
"xia2-report.html")
assert os.path.exists(html_filename)
if self._html_filename is None:
self._html_filename = html_filename
else:
shutil.move(html_filename, self._html_filename)
return ReportWrapper()
def EstimateResolutionLimit(DriverType=None):
"""A factory for EstimateResolutionLimitWrapper classes."""
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class EstimateResolutionLimitWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable("dials.estimate_resolution_limit")
self._experiments_filename = None
self._reflections_filename = None
self._estimated_d_min = None
def set_experiments_filename(self, experiments_filename):
self._experiments_filename = experiments_filename
def set_reflections_filename(self, reflections_filename):
self._reflections_filename = reflections_filename
def get_estimated_d_min(self):
return self._estimated_d_min
def run(self):
from xia2.Handlers.Streams import Debug
Debug.write("Running dials.estimate_resolution_limit")
self.clear_command_line()
self.add_command_line(self._experiments_filename)
self.add_command_line(self._reflections_filename)
self.start()
self.close_wait()
self.check_for_errors()
for line in self.get_all_output():
if line.startswith("estimated d_min:"):
self._estimated_d_min = float(line.split(":")[1])
return self._estimated_d_min
return EstimateResolutionLimitWrapper()
def Freerflag(DriverType=None):
"""A factory for FreerflagWrapper classes."""
DriverInstance = DriverFactory.Driver(DriverType)
CCP4DriverInstance = DecoratorFactory.Decorate(DriverInstance, "ccp4")
class FreerflagWrapper(CCP4DriverInstance.__class__):
"""A wrapper for Freerflag, using the CCP4-ified Driver."""
def __init__(self):
# generic things
CCP4DriverInstance.__class__.__init__(self)
Citations.cite("ccp4")
self.set_executable(
os.path.join(os.environ.get("CBIN", ""), "freerflag"))
self._free_fraction = 0.05
def set_free_fraction(self, free_fraction):
self._free_fraction = free_fraction
def add_free_flag(self):
self.check_hklin()
self.check_hklout()
self.start()
self.input("freerfrac %.3f" % self._free_fraction)
self.close_wait()
self.check_for_errors()
self.check_ccp4_errors()
def complete_free_flag(self):
self.check_hklin()
self.check_hklout()
free_column = FindFreeFlag(self.get_hklin())
self.start()
self.input("freerfrac %.3f" % self._free_fraction)
self.input("complete FREE=%s" % free_column)
self.close_wait()
self.check_for_errors()
self.check_ccp4_errors()
return FreerflagWrapper()
def EstimateGain(DriverType=None):
"""A factory for EstimateGainWrapper classes."""
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class EstimateGainWrapper(DriverInstance.__class__, FrameProcessor):
def __init__(self):
super(EstimateGainWrapper, self).__init__()
self.set_executable("dials.estimate_gain")
self._sweep_filename = None
self._kernel_size = None
self._gain = None
def set_sweep_filename(self, sweep_filename):
self._sweep_filename = sweep_filename
def set_kernel_size(self, kernel_size):
self._kernel_size = kernel_size
def get_gain(self):
return self._gain
def run(self):
self.clear_command_line()
assert self._sweep_filename is not None
self.add_command_line("%s" % self._sweep_filename)
if self._kernel_size is not None:
self.add_command_line("kernel_size=%i,%i" % self._kernel_size)
self.start()
self.close_wait()
self.check_for_errors()
for line in self.get_all_output():
if "Estimated gain:" in line:
self._gain = float(line.split(":")[-1].strip())
return EstimateGainWrapper()
def Freerflag(DriverType=None):
'''A factory for FreerflagWrapper classes.'''
DriverInstance = DriverFactory.Driver(DriverType)
CCP4DriverInstance = DecoratorFactory.Decorate(DriverInstance, 'ccp4')
class FreerflagWrapper(CCP4DriverInstance.__class__):
'''A wrapper for Freerflag, using the CCP4-ified Driver.'''
def __init__(self):
# generic things
CCP4DriverInstance.__class__.__init__(self)
self.set_executable(
os.path.join(os.environ.get('CBIN', ''), 'freerflag'))
self._free_fraction = 0.05
def set_free_fraction(self, free_fraction):
self._free_fraction = free_fraction
def add_free_flag(self):
self.check_hklin()
self.check_hklout()
self.start()
self.input('freerfrac %.3f' % self._free_fraction)
self.close_wait()
self.check_for_errors()
self.check_ccp4_errors()
def complete_free_flag(self):
self.check_hklin()
self.check_hklout()
free_column = FindFreeFlag(self.get_hklin())
self.start()
self.input('freerfrac %.3f' % self._free_fraction)
self.input('complete FREE=%s' % free_column)
self.close_wait()
self.check_for_errors()
self.check_ccp4_errors()
return FreerflagWrapper()
def Findspots(DriverType=None):
"""A factory for FindspotsWrapper(ipmosflm) classes."""
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class FindspotsWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
import os
self.set_executable(
os.path.join(os.environ["CCP4"], "bin", "ipmosflm"))
def __call__(self, fp, images):
from xia2.Handlers.Streams import Debug
Debug.write("Running mosflm to find spots")
self.start()
self.input('template "%s"' % fp.get_template())
self.input('directory "%s"' % fp.get_directory())
self.input("beam %f %f" % fp.get_beam_centre())
self.input("distance %f" % fp.get_distance())
self.input("wavelength %f" % fp.get_wavelength())
self.input("findspots file spots.dat")
for i in images:
self.input("findspots find %d" % i)
self.input("go")
self.close_wait()
# FIXME I should really gather some interesting information about
# the spot finding in here...
import os
return os.path.join(self.get_working_directory(), "spots.dat")
return FindspotsWrapper()
def AlignCrystal(DriverType=None):
"""A factory for AlignCrystalWrapper classes."""
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class AlignCrystalWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable("dials.align_crystal")
self._experiments_filename = None
self._json_filename = "align_crystal.json"
def set_experiments_filename(self, experiments_filename):
self._experiments_filename = experiments_filename
def get_experiments_filename(self):
return self._experiments_filename
def set_json_filename(self, json_filename):
self._json_filename = json_filename
def get_json_filename(self):
return self._json_filename
def run(self):
from xia2.Handlers.Streams import Debug
Debug.write("Running dials.align_crystal")
self.clear_command_line()
self.add_command_line("experiments=%s" %
self._experiments_filename)
self.add_command_line("output.json=%s" % self._json_filename)
self.start()
self.close_wait()
self.check_for_errors()
return AlignCrystalWrapper()
def xia2_main(stop_after=None):
'''Actually process something...'''
Citations.cite('xia2')
# print versions of related software
from dials.util.version import dials_version
Chatter.write(dials_version())
start_time = time.time()
CommandLine = get_command_line()
start_dir = Flags.get_starting_directory()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
no_images = True
for name in xtals.keys():
xtal = xtals[name]
if not xtal.get_all_image_names():
Chatter.write('-----------------------------------' + \
'-' * len(name))
Chatter.write('| No images assigned for crystal %s |' % name)
Chatter.write('-----------------------------------' + '-' \
* len(name))
else:
no_images = False
args = []
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
from libtbx import group_args
xinfo = CommandLine.get_xinfo()
if os.path.exists('xia2.json'):
from xia2.Schema.XProject import XProject
xinfo_new = xinfo
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new.keys():
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names():
wavelength_new = crystals_new[crystal_id].get_xwavelength(wavelength_id)
if wavelength_id not in crystals[crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = [s.get_name() for s in sweeps]
sweep_keys = [
(s.get_directory(), s.get_template(), s.get_image_range())
for s in sweeps]
for sweep in sweeps_new:
if ((sweep.get_directory(), sweep.get_template(),
sweep.get_image_range()) not in sweep_keys):
if sweep.get_name() in sweep_names:
i = 1
while 'SWEEEP%i' %i in sweep_names:
i += 1
sweep._name = 'SWEEP%i' %i
break
wavelength.add_sweep(
name=sweep.get_name(),
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.append(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
if njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
args.append((
group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
),))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(args):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
if mp_params.type == "qsub":
method = "sge"
else:
method = "multiprocessing"
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command
if not qsub_command:
qsub_command = 'qsub'
qsub_command = '%s -V -cwd -pe smp %d' %(qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep, args, processes=njob,
#method=method,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
assert xsweep_dict is not None
if output is not None:
Chatter.write(output)
if not success:
Chatter.write('Sweep failed: removing %s' %sweep.get_name())
remove_sweeps.append(sweep)
else:
Chatter.write('Loading sweep: %s' % sweep.get_name())
from xia2.Schema.XSweep import XSweep
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
else:
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
try:
if stop_after == 'index':
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception, e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(sweep.get_name(), str(e)))
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
def XScaleR(DriverType = None,
correct_decay = True,
correct_absorption = True,
correct_modulation = True):
DriverInstance = DriverFactory.Driver(DriverType)
class XScaleWrapper(DriverInstance.__class__):
'''A wrapper for wrapping XSCALE.'''
def __init__(self):
# set up the object ancestors...
DriverInstance.__class__.__init__(self)
# now set myself up...
self._parallel = PhilIndex.params.xia2.settings.multiprocessing.nproc
if self._parallel <= 1:
self.set_executable('xscale')
else:
self.set_executable('xscale_par')
self._version = 'new'
# overall information
self._resolution_shells = ''
self._cell = None
self._spacegroup_number = None
self._reindex_matrix = None
# corrections to apply - N.B. default values come from the
# factory function default arguments...
self._correct_decay = correct_decay
self._correct_absorption = correct_absorption
self._correct_modulation = correct_modulation
# input reflections information - including grouping information
# in the same way as the .xinfo files - through the wavelength
# names, which will be used for the output files.
self._input_reflection_files = []
self._input_reflection_wavelength_names = []
self._input_resolution_ranges = []
# these are generated at the run time
self._transposed_input = { }
self._transposed_input_keys = []
# output
self._output_reflection_files = { }
self._remove = []
# decisions about the scaling
self._crystal = None
self._zero_dose = PhilIndex.params.xds.xscale.zero_dose
if self._zero_dose:
Debug.write('Switching on zero-dose extrapolation')
self._anomalous = True
self._merge = False
# scale factor output
self._scale_factor = 1.0
# Rmerge values - for the scale model analysis - N.B. get
# one for each data set, obviously...
self._rmerges = { }
def add_reflection_file(self, reflections, wavelength, resolution):
self._input_reflection_files.append(reflections)
self._input_reflection_wavelength_names.append(wavelength)
self._input_resolution_ranges.append(resolution)
def get_remove(self):
return self._remove
def set_crystal(self, crystal):
self._crystal = crystal
def set_anomalous(self, anomalous = True):
self._anomalous = anomalous
def set_correct_decay(self, correct_decay):
self._correct_decay = correct_decay
def set_correct_absorption(self, correct_absorption):
self._correct_absorption = correct_absorption
def set_correct_modulation(self, correct_modulation):
self._correct_modulation = correct_modulation
def get_output_reflection_files(self):
'''Get a dictionary of output reflection files keyed by
wavelength name.'''
return copy.deepcopy(self._output_reflection_files)
def _transform_input_files(self):
'''Transform the input files to an order we can manage.'''
for j in range(len(self._input_reflection_files)):
hkl = self._input_reflection_files[j]
wave = self._input_reflection_wavelength_names[j]
resol = self._input_resolution_ranges[j]
if wave not in self._transposed_input:
self._transposed_input[wave] = {'hkl':[],
'resol':[]}
self._transposed_input_keys.append(wave)
self._transposed_input[wave]['hkl'].append(hkl)
self._transposed_input[wave]['resol'].append(resol)
def set_spacegroup_number(self, spacegroup_number):
self._spacegroup_number = spacegroup_number
def set_cell(self, cell):
self._cell = cell
def set_reindex_matrix(self, reindex_matrix):
if not len(reindex_matrix) == 12:
raise RuntimeError('reindex matrix must be 12 numbers')
self._reindex_matrix = reindex_matrix
def _write_xscale_inp(self):
'''Write xscale.inp.'''
self._transform_input_files()
xscale_inp = open(os.path.join(self.get_working_directory(),
'XSCALE.INP'), 'w')
# header information
xscale_inp.write('MAXIMUM_NUMBER_OF_PROCESSORS=%d\n' % \
self._parallel)
xscale_inp.write('SPACE_GROUP_NUMBER=%d\n' % \
self._spacegroup_number)
xscale_inp.write('UNIT_CELL_CONSTANTS=')
xscale_inp.write('%6.2f %6.2f %6.2f %6.2f %6.2f %6.2f\n' % \
tuple(self._cell))
xscale_inp.write(
'MINIMUM_I/SIGMA=%.1f\n' %PhilIndex.params.xds.xscale.min_isigma)
if self._reindex_matrix:
xscale_inp.write(
'REIDX=%d %d %d %d %d %d %d %d %d %d %d %d\n' % \
tuple(map(int, self._reindex_matrix)))
# now information about the wavelengths
for wave in self._transposed_input_keys:
self._output_reflection_files[wave] = os.path.join(
self.get_working_directory(), '%s.HKL' % wave)
xscale_inp.write(
'OUTPUT_FILE=%s.HKL ' % wave)
if self._version == 'new':
xscale_inp.write('\n')
if self._anomalous:
xscale_inp.write('FRIEDEL\'S_LAW=FALSE MERGE=FALSE\n')
xscale_inp.write('STRICT_ABSORPTION_CORRECTION=TRUE\n')
else:
xscale_inp.write('FRIEDEL\'S_LAW=TRUE MERGE=FALSE\n')
if self._version == 'new':
xscale_inp.write('\n')
for j in range(len(self._transposed_input[wave]['hkl'])):
# FIXME note to self, this should now be a local
# file which has been placed in here by XDSScaler -
# should check that the files exists though...
resolution = self._transposed_input[wave]['resol'][j]
if resolution[0]:
xscale_inp.write(
'INPUT_FILE=%s XDS_ASCII\n' % \
self._transposed_input[wave]['hkl'][j])
xscale_inp.write(
'INCLUDE_RESOLUTION_RANGE= %.2f %.2f\n' % \
(resolution[1], resolution[0]))
else:
xscale_inp.write(
'INPUT_FILE=%s XDS_ASCII\n' % \
self._transposed_input[wave]['hkl'][j])
# FIXME this needs to be removed before being used again
# in anger!
# xscale_inp.write('CORRECTIONS=DECAY ABSORPTION\n')
corrections = 'CORRECTIONS='
if self._correct_decay:
corrections += ' DECAY'
if self._correct_modulation:
corrections += ' MODULATION'
if self._correct_absorption:
corrections += ' ABSORPTION'
corrections += '\n'
xscale_inp.write(corrections)
if self._crystal and self._zero_dose:
xscale_inp.write('CRYSTAL_NAME=%s\n' % self._crystal)
xscale_inp.close()
def run(self):
'''Actually run XSCALE.'''
self._write_xscale_inp()
# copy the input file...
shutil.copyfile(os.path.join(self.get_working_directory(),
'XSCALE.INP'),
os.path.join(self.get_working_directory(),
'%d_XSCALE.INP' % self.get_xpid()))
self.start()
self.close_wait()
# copy the LP file
shutil.copyfile(os.path.join(self.get_working_directory(),
'XSCALE.LP'),
os.path.join(self.get_working_directory(),
'%d_XSCALE.LP' % self.get_xpid()))
# now look at XSCALE.LP
xds_check_error(self.get_all_output())
dname = None
# get the outlier reflections... and the overall scale factor
for line in open(os.path.join(
self.get_working_directory(),
'XSCALE.LP'), 'r').readlines():
if '"alien"' in line:
h, k, l = tuple(map(int, line.split()[:3]))
z = float(line.split()[4])
if not (h, k, l, z) in self._remove:
self._remove.append((h, k, l, z))
if 'FACTOR TO PLACE ALL DATA SETS TO ' in line:
self._scale_factor = float(line.split()[-1])
if 'STATISTICS OF SCALED OUTPUT DATA SET' in line:
dname = line.split()[-1].replace('.HKL', '')
if 'total' in line and not dname in self._rmerges:
if len(line.split()) > 5:
self._rmerges[dname] = float(
line.replace('%', '').split()[5])
# trac #419 - if the data sets are not correctly indexed,
# throw an exception. N.B. this will only work if the
# data sets are moderately complete (i.e. there are more
# than a handful of common reflections) - which may not be
# the case in MULTICRYSTAL mode.
if ' !!! WARNING !!! ' in line and \
'CORRELATION FACTORS ARE DANGEROUSLY SMALL' in line:
groups = get_correlation_coefficients_and_group(
os.path.join(self.get_working_directory(),
'XSCALE.LP'))
Debug.write('Low correlations - check data sets')
for j, name in enumerate(groups):
Debug.write('Group %d' % j)
for file_name in groups[name]:
Debug.write(file_name)
raise RuntimeError('reindexing error: %s' % \
os.path.join(self.get_working_directory(),
'XSCALE.LP'))
def get_scale_factor(self):
return self._scale_factor
def get_rmerges(self):
return self._rmerges
return XScaleWrapper()
def LabelitDistl(DriverType=None):
'''Factory for LabelitDistl wrapper classes, with the specified
Driver type.'''
DriverInstance = DriverFactory.Driver(DriverType)
class LabelitDistlWrapper(DriverInstance.__class__):
'''A wrapper for the program labelit.distl - which will provide
functionality for looking for ice rings and screening diffraction
images.'''
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable('labelit.distl')
self._images = []
self._statistics = {}
def add_image(self, image):
'''Add an image for indexing.'''
if not image in self._images:
self._images.append(image)
return
def distl(self):
'''Actually analyse the images.'''
self._images.sort()
for i in self._images:
self.add_command_line(i)
task = 'Screen images:'
for i in self._images:
task += ' %s' % i
self.set_task(task)
self.start()
self.close_wait()
# check for errors
self.check_for_errors()
# ok now we're done, let's look through for some useful stuff
output = self.get_all_output()
current_image = None
for o in output:
if 'None' in o and 'Resolution' in o:
l = o.replace('None', '0.0').split()
else:
l = o.split()
if l[:1] == ['File']:
current_image = l[2]
self._statistics[current_image] = {}
if l[:2] == ['Spot', 'Total']:
self._statistics[current_image]['spots_total'] = int(l[-1])
if l[:2] == ['In-Resolution', 'Total']:
self._statistics[current_image]['spots'] = int(l[-1])
if l[:3] == ['Good', 'Bragg', 'Candidates']:
self._statistics[current_image]['spots_good'] = int(l[-1])
if l[:2] == ['Ice', 'Rings']:
self._statistics[current_image]['ice_rings'] = int(l[-1])
if l[:3] == ['Method', '1', 'Resolution']:
self._statistics[current_image]['resol_one'] = float(l[-1])
if l[:3] == ['Method', '2', 'Resolution']:
self._statistics[current_image]['resol_two'] = float(l[-1])
if l[:3] == ['%Saturation,', 'Top', '50']:
self._statistics[current_image]['saturation'] = float(
l[-1])
return 'ok'
# things to get results from the indexing
def get_statistics(self, image):
'''Get the screening statistics from image as dictionary.
The keys are spots_total, spots, spots_good, ice_rings,
resol_one, resol_two.'''
return self._statistics[os.path.split(image)[-1]]
return LabelitDistlWrapper()
def _scale_prepare(self):
'''Prepare the data for scaling - this will reindex it the
reflections to the correct pointgroup and setting, for instance,
and move the reflection files to the scale directory.'''
Citations.cite('xds')
Citations.cite('ccp4')
Citations.cite('pointless')
# GATHER phase - get the reflection files together... note that
# it is not necessary in here to keep the batch information as we
# don't wish to rebatch the reflections prior to scaling.
# FIXME need to think about what I will do about the radiation
# damage analysis in here...
self._sweep_information = { }
# FIXME in here I want to record the batch number to
# epoch mapping as per the CCP4 Scaler implementation.
Journal.block(
'gathering', self.get_scaler_xcrystal().get_name(), 'XDS',
{'working directory':self.get_working_directory()})
for epoch in self._scalr_integraters.keys():
intgr = self._scalr_integraters[epoch]
pname, xname, dname = intgr.get_integrater_project_info()
sname = intgr.get_integrater_sweep_name()
self._sweep_information[epoch] = {
'pname':pname,
'xname':xname,
'dname':dname,
'integrater':intgr,
'corrected_intensities':intgr.get_integrater_corrected_intensities(),
'prepared_reflections':None,
'scaled_reflections':None,
'header':intgr.get_header(),
'batches':intgr.get_integrater_batches(),
'image_to_epoch':intgr.get_integrater_sweep(
).get_image_to_epoch(),
'image_to_dose':{},
'batch_offset':0,
'sname':sname
}
Journal.entry({'adding data from':'%s/%s/%s' % \
(xname, dname, sname)})
# what are these used for?
# pname / xname / dname - dataset identifiers
# image to epoch / batch offset / batches - for RD analysis
Debug.write('For EPOCH %s have:' % str(epoch))
Debug.write('ID = %s/%s/%s' % (pname, xname, dname))
Debug.write('SWEEP = %s' % intgr.get_integrater_sweep_name())
# next work through all of the reflection files and make sure that
# they are XDS_ASCII format...
epochs = self._sweep_information.keys()
epochs.sort()
self._first_epoch = min(epochs)
self._scalr_pname = self._sweep_information[epochs[0]]['pname']
self._scalr_xname = self._sweep_information[epochs[0]]['xname']
for epoch in epochs:
intgr = self._scalr_integraters[epoch]
pname = self._sweep_information[epoch]['pname']
xname = self._sweep_information[epoch]['xname']
dname = self._sweep_information[epoch]['dname']
sname = self._sweep_information[epoch]['sname']
if self._scalr_pname != pname:
raise RuntimeError, 'all data must have a common project name'
xname = self._sweep_information[epoch]['xname']
if self._scalr_xname != xname:
raise RuntimeError, \
'all data for scaling must come from one crystal'
xsh = XDSScalerHelper()
xsh.set_working_directory(self.get_working_directory())
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_CORRECTED.HKL' %(
pname, xname, dname, sname))
sweep = intgr.get_integrater_sweep()
if sweep.get_frames_to_process() is not None:
offset = intgr.get_frame_offset()
#print "offset: %d" %offset
start, end = sweep.get_frames_to_process()
start -= offset
end -= offset
#end += 1 ????
#print "limiting batches: %d-%d" %(start, end)
xsh.limit_batches(hklin, hklout, start, end)
self._sweep_information[epoch]['corrected_intensities'] = hklout
# if there is more than one sweep then compare the lattices
# and eliminate all but the lowest symmetry examples if
# there are more than one...
# -------------------------------------------------
# Ensure that the integration lattices are the same
# -------------------------------------------------
need_to_return = False
if len(self._sweep_information.keys()) > 1:
lattices = []
# FIXME run this stuff in parallel as well...
for epoch in self._sweep_information.keys():
intgr = self._sweep_information[epoch]['integrater']
hklin = self._sweep_information[epoch]['corrected_intensities']
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
ntr = False
else:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattice = Syminfo.get_lattice(pointgroup)
if not lattice in lattices:
lattices.append(lattice)
if ntr:
# if we need to return, we should logically reset
# any reindexing operator right? right here all
# we are talking about is the correctness of
# individual pointgroups?? Bug # 3373
reindex_op = 'h,k,l'
# actually, should this not be done "by magic"
# when a new pointgroup is assigned in the
# pointless indexer jiffy above?!
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
need_to_return = True
# bug # 2433 - need to ensure that all of the lattice
# conclusions were the same...
if len(lattices) > 1:
ordered_lattices = []
for l in lattices_in_order():
if l in lattices:
ordered_lattices.append(l)
correct_lattice = ordered_lattices[0]
Debug.write('Correct lattice asserted to be %s' % \
correct_lattice)
# transfer this information back to the indexers
for epoch in self._sweep_information.keys():
integrater = self._sweep_information[
epoch]['integrater']
refiner = integrater.get_integrater_refiner()
sname = integrater.get_integrater_sweep_name()
if not refiner:
continue
state = refiner.set_refiner_asserted_lattice(
correct_lattice)
if state == refiner.LATTICE_CORRECT:
Debug.write('Lattice %s ok for sweep %s' % \
(correct_lattice, sname))
elif state == refiner.LATTICE_IMPOSSIBLE:
raise RuntimeError, 'Lattice %s impossible for %s' % \
(correct_lattice, sname)
elif state == refiner.LATTICE_POSSIBLE:
Debug.write('Lattice %s assigned for sweep %s' % \
(correct_lattice, sname))
need_to_return = True
# if one or more of them was not in the lowest lattice,
# need to return here to allow reprocessing
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# next if there is more than one sweep then generate
# a merged reference reflection file to check that the
# setting for all reflection files is the same...
# if we get to here then all data was processed with the same
# lattice
# ----------------------------------------------------------
# next ensure that all sweeps are set in the correct setting
# ----------------------------------------------------------
if self.get_scaler_reference_reflection_file():
self._reference = self.get_scaler_reference_reflection_file()
Debug.write('Using HKLREF %s' % self._reference)
md = self._factory.Mtzdump()
md.set_hklin(self.get_scaler_reference_reflection_file())
md.dump()
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(
md.get_spacegroup())
Debug.write('Spacegroup %d' % self._xds_spacegroup)
elif PhilIndex.params.xia2.settings.scale.reference_reflection_file:
self._reference = PhilIndex.params.xia2.settings.scale.reference_reflection_file
Debug.write('Using HKLREF %s' % self._reference)
md = self._factory.Mtzdump()
md.set_hklin(PhilIndex.params.xia2.settings.scale.reference_reflection_file)
md.dump()
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(
md.get_spacegroup())
Debug.write('Spacegroup %d' % self._xds_spacegroup)
params = PhilIndex.params
use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs
if len(self._sweep_information.keys()) > 1 and use_brehm_diederichs:
brehm_diederichs_files_in = []
for epoch in self._sweep_information.keys():
intgr = self._sweep_information[epoch]['integrater']
hklin = self._sweep_information[epoch]['corrected_intensities']
refiner = intgr.get_integrater_refiner()
# in here need to consider what to do if the user has
# assigned the pointgroup on the command line ...
if not self._scalr_input_pointgroup:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
if ntr:
# Bug # 3373
Debug.write('Reindex to standard (PIJ): %s' % \
reindex_op)
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
else:
# 27/FEB/08 to support user assignment of pointgroups
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
self._sweep_information[epoch]['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# convert the XDS_ASCII for this sweep to mtz - on the next
# get this should be in the correct setting...
dname = self._sweep_information[epoch]['dname']
sname = intgr.get_integrater_sweep_name()
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.mtz' % (dname, sname))
FileHandler.record_temporary_file(hklout)
# now use pointless to make this conversion
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
brehm_diederichs_files_in.append(hklout)
# now run cctbx.brehm_diederichs to figure out the indexing hand for
# each sweep
from xia2.Wrappers.Cctbx.BrehmDiederichs import BrehmDiederichs
brehm_diederichs = BrehmDiederichs()
brehm_diederichs.set_working_directory(self.get_working_directory())
auto_logfiler(brehm_diederichs)
brehm_diederichs.set_input_filenames(brehm_diederichs_files_in)
# 1 or 3? 1 seems to work better?
brehm_diederichs.set_asymmetric(1)
brehm_diederichs.run()
reindexing_dict = brehm_diederichs.get_reindexing_dict()
for epoch in self._sweep_information.keys():
intgr = self._sweep_information[epoch]['integrater']
dname = self._sweep_information[epoch]['dname']
sname = intgr.get_integrater_sweep_name()
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.mtz' % (dname, sname))
# apply the reindexing operator
intgr.set_integrater_reindex_operator(reindex_op)
# and copy the reflection file to the local directory
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
self._sweep_information[epoch][
'prepared_reflections'] = os.path.split(hklout)[-1]
elif len(self._sweep_information.keys()) > 1 and \
not self._reference:
# need to generate a reference reflection file - generate this
# from the reflections in self._first_epoch
#
# FIXME this should really use the Brehm and Diederichs method
# if you have lots of little sweeps...
intgr = self._sweep_information[self._first_epoch]['integrater']
hklin = self._sweep_information[epoch]['corrected_intensities']
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
ntr = False
reindex_op = 'h,k,l'
else:
pointgroup, reindex_op, ntr = self._pointless_indexer_jiffy(
hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
reference_reindex_op = intgr.get_integrater_reindex_operator()
if ntr:
# Bug # 3373
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(pointgroup)
# next pass this reindexing operator back to the source
# of the reflections
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
self._sweep_information[epoch]['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'xds-pointgroup-reference-unsorted.mtz')
FileHandler.record_temporary_file(hklout)
# now use pointless to handle this conversion
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
self._reference = hklout
if self._reference:
from xia2.Driver.DriverFactory import DriverFactory
def run_one_sweep(args):
sweep_information = args[0]
pointless_indexer_jiffy = args[1]
factory = args[2]
job_type = args[3]
if job_type:
DriverFactory.set_driver_type(job_type)
intgr = sweep_information['integrater']
hklin = sweep_information['corrected_intensities']
refiner = intgr.get_integrater_refiner()
# in here need to consider what to do if the user has
# assigned the pointgroup on the command line ...
if not self._scalr_input_pointgroup:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
if ntr:
# Bug # 3373
Debug.write('Reindex to standard (PIJ): %s' % \
reindex_op)
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
else:
# 27/FEB/08 to support user assignment of pointgroups
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
sweep_information['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# convert the XDS_ASCII for this sweep to mtz - on the next
# get this should be in the correct setting...
hklin = sweep_information['corrected_intensities']
# now use pointless to make this conversion
# try with no conversion?!
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
hklout = os.path.join(
self.get_working_directory(),
'%d_xds-pointgroup-unsorted.mtz' %pointless.get_xpid())
FileHandler.record_temporary_file(hklout)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
pointless = self._factory.Pointless()
pointless.set_hklin(hklout)
pointless.set_hklref(self._reference)
pointless.decide_pointgroup()
pointgroup = pointless.get_pointgroup()
reindex_op = pointless.get_reindex_operator()
# for debugging print out the reindexing operations and
# what have you...
Debug.write('Reindex to standard: %s' % reindex_op)
# this should send back enough information that this
# is in the correct pointgroup (from the call above) and
# also in the correct setting, from the interaction
# with the reference set... - though I guess that the
# spacegroup number should not have changed, right?
# set the reindex operation afterwards... though if the
# spacegroup number is the same this should make no
# difference, right?!
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
intgr.set_integrater_reindex_operator(reindex_op)
sweep_information['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# and copy the reflection file to the local directory
dname = sweep_information['dname']
sname = intgr.get_integrater_sweep_name()
hklin = sweep_information['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
sweep_information['prepared_reflections'] = os.path.split(hklout)[-1]
return sweep_information
from libtbx import easy_mp
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
if njob > 1:
# cache drivertype
drivertype = DriverFactory.get_driver_type()
args = [
(self._sweep_information[epoch], self._pointless_indexer_jiffy,
self._factory, mp_params.type)
for epoch in self._sweep_information.keys()]
results_list = easy_mp.parallel_map(
run_one_sweep, args, params=None,
processes=njob,
method="threading",
asynchronous=True,
callback=None,
preserve_order=True,
preserve_exception_message=True)
# restore drivertype
DriverFactory.set_driver_type(drivertype)
# results should be given back in the same order
for i, epoch in enumerate(self._sweep_information.keys()):
self._sweep_information[epoch] = results_list[i]
else:
for epoch in self._sweep_information.keys():
self._sweep_information[epoch] = run_one_sweep(
(self._sweep_information[epoch], self._pointless_indexer_jiffy,
self._factory, None))
else:
# convert the XDS_ASCII for this sweep to mtz
epoch = self._first_epoch
intgr = self._sweep_information[epoch]['integrater']
refiner = intgr.get_integrater_refiner()
sname = intgr.get_integrater_sweep_name()
hklout = os.path.join(self.get_working_directory(),
'%s-pointless.mtz' % sname)
FileHandler.record_temporary_file(hklout)
pointless = self._factory.Pointless()
pointless.set_xdsin(self._sweep_information[epoch]['corrected_intensities'])
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
# run it through pointless interacting with the
# Indexer which belongs to this sweep
hklin = hklout
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
ntr = False
reindex_op = 'h,k,l'
else:
pointgroup, reindex_op, ntr = self._pointless_indexer_jiffy(
hklin, refiner)
if ntr:
# if we need to return, we should logically reset
# any reindexing operator right? right here all
# we are talking about is the correctness of
# individual pointgroups?? Bug # 3373
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
need_to_return = True
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(pointgroup)
# next pass this reindexing operator back to the source
# of the reflections
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
self._sweep_information[epoch]['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
hklin = self._sweep_information[epoch]['corrected_intensities']
dname = self._sweep_information[epoch]['dname']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
# and copy the reflection file to the local
# directory
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
self._sweep_information[epoch][
'prepared_reflections'] = os.path.split(hklout)[-1]
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
unit_cell_list = []
for epoch in self._sweep_information.keys():
integrater = self._sweep_information[epoch]['integrater']
cell = integrater.get_integrater_cell()
n_ref = integrater.get_integrater_n_ref()
Debug.write('Cell for %s: %.2f %.2f %.2f %.2f %.2f %.2f' % \
(integrater.get_integrater_sweep_name(),
cell[0], cell[1], cell[2],
cell[3], cell[4], cell[5]))
Debug.write('=> %d reflections' % n_ref)
unit_cell_list.append((cell, n_ref))
self._scalr_cell = compute_average_unit_cell(unit_cell_list)
self._scalr_resolution_limits = { }
Debug.write('Determined unit cell: %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(self._scalr_cell))
if os.path.exists(os.path.join(
self.get_working_directory(),
'REMOVE.HKL')):
os.remove(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'))
Debug.write('Deleting REMOVE.HKL at end of scale prepare.')
return
def Index(DriverType=None):
"""A factory for IndexWrapper classes."""
from xia2.Driver.DriverFactory import DriverFactory
DriverInstance = DriverFactory.Driver(DriverType)
class IndexWrapper(DriverInstance.__class__):
def __init__(self):
DriverInstance.__class__.__init__(self)
self.set_executable("dials.index")
self._sweep_filenames = []
self._spot_filenames = []
self._unit_cell = None
self._space_group = None
self._maximum_spot_error = None
self._detector_fix = None
self._beam_fix = None
self._indexing_method = "fft3d"
self._p1_cell = None
self._indxr_input_cell = None
self._indxr_input_lattice = None
self._reflections_per_degree = None
self._fft3d_n_points = None
self._histogram_binning = None
self._nearest_neighbor_percentile = None
self._experiment_filename = None
self._indexed_filename = None
self._nref = None
self._rmsd_x = None
self._rmsd_y = None
self._rmsd_z = None
self._max_cell = None
self._max_cell_max_height_fraction = None
self._min_cell = None
self._d_min_start = None
self._phil_file = None
self._outlier_algorithm = None
self._close_to_spindle_cutoff = None
def add_sweep_filename(self, sweep_filename):
self._sweep_filenames.append(sweep_filename)
def add_spot_filename(self, spot_filename):
self._spot_filenames.append(spot_filename)
def set_indexer_input_lattice(self, lattice):
self._indxr_input_lattice = lattice
def set_indexer_user_input_lattice(self, user):
self._indxr_user_input_lattice = user
def set_indexer_input_cell(self, cell):
if not isinstance(cell, type(())) or len(cell) != 6:
raise RuntimeError("cell must be a 6-tuple de floats")
self._indxr_input_cell = tuple(map(float, cell))
def set_maximum_spot_error(self, maximum_spot_error):
self._maximum_spot_error = maximum_spot_error
def set_detector_fix(self, detector_fix):
self._detector_fix = detector_fix
def set_beam_fix(self, beam_fix):
self._beam_fix = beam_fix
def set_indexing_method(self, method):
self._indexing_method = method
def get_indexing_method(self):
return self._indexing_method
def set_reflections_per_degree(self, reflections_per_degree):
self._reflections_per_degree = int(reflections_per_degree)
def set_fft3d_n_points(self, n_points):
self._fft3d_n_points = n_points
def set_histogram_binning(self, histogram_binning):
self._histogram_binning = histogram_binning
def set_nearest_neighbor_percentile(self, nearest_neighbor_percentile):
self._nearest_neighbor_percentile = nearest_neighbor_percentile
def get_sweep_filenames(self):
return self._sweep_filenames
def get_experiments_filename(self):
return self._experiment_filename
def get_indexed_filename(self):
return self._indexed_filename
def get_p1_cell(self):
return self._p1_cell
def set_phil_file(self, phil_file):
self._phil_file = phil_file
def set_outlier_algorithm(self, outlier_algorithm):
self._outlier_algorithm = outlier_algorithm
def get_nref_rmsds(self):
return self._nref, (self._rmsd_x, self._rmsd_y, self._rmsd_z)
def set_max_cell(self, max_cell=None, max_height_fraction=None):
if max_cell is not None:
self._max_cell = max_cell
if max_height_fraction is not None:
self._max_cell_max_height_fraction = max_height_fraction
def set_min_cell(self, min_cell):
self._min_cell = min_cell
def set_d_min_start(self, d_min_start):
self._d_min_start = d_min_start
def set_close_to_spindle_cutoff(self, close_to_spindle_cutoff):
self._close_to_spindle_cutoff = close_to_spindle_cutoff
def run(self, method):
from xia2.Handlers.Streams import Debug
Debug.write("Running dials.index")
self.clear_command_line()
for f in self._sweep_filenames:
self.add_command_line(f)
for f in self._spot_filenames:
self.add_command_line(f)
if len(self._sweep_filenames) > 1:
self.add_command_line("auto_reduction.action=fix")
self.add_command_line("indexing.method=%s" % method)
nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
self.set_cpu_threads(nproc)
self.add_command_line("indexing.nproc=%i" % nproc)
if PhilIndex.params.xia2.settings.small_molecule:
self.add_command_line("filter_ice=false")
if self._reflections_per_degree is not None:
self.add_command_line("reflections_per_degree=%i" %
self._reflections_per_degree)
if self._fft3d_n_points is not None:
self.add_command_line(
"fft3d.reciprocal_space_grid.n_points=%i" %
self._fft3d_n_points)
if self._close_to_spindle_cutoff is not None:
self.add_command_line("close_to_spindle_cutoff=%f" %
self._close_to_spindle_cutoff)
if self._outlier_algorithm:
self.add_command_line("outlier.algorithm=%s" %
self._outlier_algorithm)
if self._max_cell:
self.add_command_line("max_cell=%g" % self._max_cell)
if self._max_cell_max_height_fraction is not None:
self.add_command_line("max_height_fraction=%g" %
self._max_cell_max_height_fraction)
if self._min_cell:
self.add_command_line("min_cell=%d" % self._min_cell)
if self._histogram_binning is not None:
self.add_command_line(
"max_cell_estimation.histogram_binning=%s" %
self._histogram_binning)
if self._nearest_neighbor_percentile is not None:
self.add_command_line(
"max_cell_estimation.nearest_neighbor_percentile=%s" %
self._nearest_neighbor_percentile)
if self._d_min_start:
self.add_command_line("d_min_start=%f" % self._d_min_start)
if self._indxr_input_lattice is not None:
from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
self._symm = lattice_to_spacegroup_number(
self._indxr_input_lattice)
self.add_command_line("known_symmetry.space_group=%s" %
self._symm)
if self._indxr_input_cell is not None:
self.add_command_line(
'known_symmetry.unit_cell="%s,%s,%s,%s,%s,%s"' %
self._indxr_input_cell)
if self._maximum_spot_error:
self.add_command_line("maximum_spot_error=%.f" %
self._maximum_spot_error)
if self._detector_fix:
self.add_command_line("detector.fix=%s" % self._detector_fix)
if self._beam_fix:
self.add_command_line("beam.fix=%s" % self._beam_fix)
if self._phil_file is not None:
self.add_command_line("%s" % self._phil_file)
self._experiment_filename = os.path.join(
self.get_working_directory(),
"%d_indexed.expt" % self.get_xpid())
self._indexed_filename = os.path.join(
self.get_working_directory(),
"%d_indexed.refl" % self.get_xpid())
self.add_command_line("output.experiments=%s" %
self._experiment_filename)
self.add_command_line("output.reflections=%s" %
self._indexed_filename)
self.start()
self.close_wait()
if not os.path.isfile(
self._experiment_filename) or not os.path.isfile(
self._indexed_filename):
# Indexing failed
with open(self.get_log_file(), "r") as fh:
if "No suitable lattice could be found" in fh.read():
raise libtbx.utils.Sorry(
"No suitable indexing solution could be found.\n\n"
"You can view the reciprocal space with:\n"
"dials.reciprocal_lattice_viewer %s" % " ".join(
os.path.normpath(
os.path.join(self.get_working_directory(),
p))
for p in self._sweep_filenames +
self._spot_filenames))
else:
raise RuntimeError(
"dials.index failed, see log file for more details: %s"
% self.get_log_file())
self.check_for_errors()
for record in self.get_all_output():
if "Too few reflections to parameterise" in record:
Debug.write(record.strip())
from dials.array_family import flex
from dxtbx.serialize import load
self._experiment_list = load.experiment_list(
self._experiment_filename)
self._reflections = flex.reflection_table.from_file(
self._indexed_filename)
crystal = self._experiment_list.crystals()[0]
self._p1_cell = crystal.get_unit_cell().parameters()
refined_sel = self._reflections.get_flags(
self._reflections.flags.used_in_refinement)
refl = self._reflections.select(refined_sel)
xc, yc, zc = refl["xyzcal.px"].parts()
xo, yo, zo = refl["xyzobs.px.value"].parts()
import math
self._nref = refl.size()
self._rmsd_x = math.sqrt(flex.mean(flex.pow2(xc - xo)))
self._rmsd_y = math.sqrt(flex.mean(flex.pow2(yc - yo)))
self._rmsd_z = math.sqrt(flex.mean(flex.pow2(zc - zo)))
return IndexWrapper()
def run_one_sweep(args):
sweep_information = args[0]
pointless_indexer_jiffy = args[1]
factory = args[2]
job_type = args[3]
if job_type:
DriverFactory.set_driver_type(job_type)
intgr = sweep_information['integrater']
hklin = sweep_information['corrected_intensities']
refiner = intgr.get_integrater_refiner()
# in here need to consider what to do if the user has
# assigned the pointgroup on the command line ...
if not self._scalr_input_pointgroup:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
if ntr:
# Bug # 3373
Debug.write('Reindex to standard (PIJ): %s' % \
reindex_op)
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
else:
# 27/FEB/08 to support user assignment of pointgroups
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
sweep_information['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# convert the XDS_ASCII for this sweep to mtz - on the next
# get this should be in the correct setting...
hklin = sweep_information['corrected_intensities']
# now use pointless to make this conversion
# try with no conversion?!
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
hklout = os.path.join(
self.get_working_directory(),
'%d_xds-pointgroup-unsorted.mtz' %pointless.get_xpid())
FileHandler.record_temporary_file(hklout)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
pointless = self._factory.Pointless()
pointless.set_hklin(hklout)
pointless.set_hklref(self._reference)
pointless.decide_pointgroup()
pointgroup = pointless.get_pointgroup()
reindex_op = pointless.get_reindex_operator()
# for debugging print out the reindexing operations and
# what have you...
Debug.write('Reindex to standard: %s' % reindex_op)
# this should send back enough information that this
# is in the correct pointgroup (from the call above) and
# also in the correct setting, from the interaction
# with the reference set... - though I guess that the
# spacegroup number should not have changed, right?
# set the reindex operation afterwards... though if the
# spacegroup number is the same this should make no
# difference, right?!
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
intgr.set_integrater_reindex_operator(reindex_op)
sweep_information['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# and copy the reflection file to the local directory
dname = sweep_information['dname']
sname = intgr.get_integrater_sweep_name()
hklin = sweep_information['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
sweep_information['prepared_reflections'] = os.path.split(hklout)[-1]
return sweep_information
