def _read_pipeline(settings):
settings = PhilIndex.get_python_object().xia2.settings
indexer, refiner, integrater, scaler = None, None, None, None
if settings.pipeline == '2d':
Debug.write('2DA pipeline selected')
indexer, refiner, integrater, scaler = 'mosflm', 'mosflm', 'mosflmr', 'ccp4a'
elif settings.pipeline == '2di':
Debug.write('2DA pipeline; mosflm indexing selected')
indexer, refiner, integrater, scaler = 'mosflm', 'mosflm', 'mosflmr', 'ccp4a'
elif settings.pipeline == '3d':
Debug.write('3DR pipeline selected')
indexer, refiner, integrater, scaler = 'xds', 'xds', 'xdsr', 'xdsa'
elif settings.pipeline == '3di':
Debug.write('3DR pipeline; XDS indexing selected')
indexer, refiner, integrater, scaler = 'xds', 'xds', 'xdsr', 'xdsa'
elif settings.pipeline == '3dii':
Debug.write('3D II R pipeline (XDS IDXREF all images) selected')
indexer, refiner, integrater, scaler = 'xdsii', 'xds', 'xdsr', 'xdsa'
elif settings.pipeline == '3dd':
Debug.write('3DD pipeline (DIALS indexing) selected')
indexer, refiner, integrater, scaler = 'dials', 'xds', 'xdsr', 'xdsa'
elif settings.pipeline == 'dials':
Debug.write('DIALS pipeline selected')
indexer, refiner, integrater, scaler = 'dials', 'dials', 'dials', 'ccp4a'
if indexer is not None and settings.indexer is None:
PhilIndex.update("xia2.settings.indexer=%s" % indexer)
if refiner is not None and settings.refiner is None:
PhilIndex.update("xia2.settings.refiner=%s" % refiner)
if integrater is not None and settings.integrater is None:
PhilIndex.update("xia2.settings.integrater=%s" % integrater)
if scaler is not None and settings.scaler is None:
PhilIndex.update("xia2.settings.scaler=%s" % scaler)
def _read_pipeline():
settings = PhilIndex.get_python_object().xia2.settings
indexer, refiner, integrater, scaler = None, None, None, None
if settings.pipeline == "2d":
Debug.write("2DA pipeline selected")
print(
"***\n\nWarning: Pipeline '%s' is no longer supported and will be removed in a future release.\n\n***"
% settings.pipeline)
indexer, refiner, integrater, scaler = (
"mosflm",
"mosflm",
"mosflmr",
"ccp4a",
)
elif settings.pipeline == "2di":
Debug.write("2DA pipeline; mosflm indexing selected")
print(
"***\n\nWarning: Pipeline '%s' is no longer supported and will be removed in a future release.\n\n***"
% settings.pipeline)
indexer, refiner, integrater, scaler = (
"mosflm",
"mosflm",
"mosflmr",
"ccp4a",
)
elif settings.pipeline == "3d":
Debug.write("3DR pipeline selected")
indexer, refiner, integrater, scaler = "xds", "xds", "xdsr", "xdsa"
elif settings.pipeline == "3di":
Debug.write("3DR pipeline; XDS indexing selected")
indexer, refiner, integrater, scaler = "xds", "xds", "xdsr", "xdsa"
elif settings.pipeline == "3dii":
Debug.write("3D II R pipeline (XDS IDXREF all images) selected")
indexer, refiner, integrater, scaler = "xdsii", "xds", "xdsr", "xdsa"
elif settings.pipeline == "3dd":
Debug.write("3DD pipeline (DIALS indexing) selected")
indexer, refiner, integrater, scaler = "dials", "xds", "xdsr", "xdsa"
elif settings.pipeline == "dials":
Debug.write("DIALS pipeline selected")
indexer, refiner, integrater, scaler = "dials", "dials", "dials", "dials"
elif settings.pipeline == "dials-full":
Debug.write("DIALS pipeline selected")
print(
"***\n\nWarning: Pipeline '%s' has been renamed to 'dials' and will be removed in a future release.\n\n***"
% settings.pipeline)
indexer, refiner, integrater, scaler = "dials", "dials", "dials", "dials"
elif settings.pipeline == "dials-aimless":
Debug.write(
"DIALS-LEGACY pipeline selected (DIALS, scaling with AIMLESS)")
indexer, refiner, integrater, scaler = "dials", "dials", "dials", "ccp4a"
if indexer is not None and settings.indexer is None:
PhilIndex.update("xia2.settings.indexer=%s" % indexer)
if refiner is not None and settings.refiner is None:
PhilIndex.update("xia2.settings.refiner=%s" % refiner)
if integrater is not None and settings.integrater is None:
PhilIndex.update("xia2.settings.integrater=%s" % integrater)
if scaler is not None and settings.scaler is None:
PhilIndex.update("xia2.settings.scaler=%s" % scaler)
def get_output(self):
result = "Wavelength name: %s\n" % self._name
result += "Wavelength %7.5f\n" % self._wavelength
if self._f_pr != 0.0 and self._f_prpr != 0.0:
result += f"F', F'' = ({self._f_pr:5.2f}, {self._f_prpr:5.2f})\n"
result += "Sweeps:\n"
remove = []
params = PhilIndex.get_python_object()
failover = params.xia2.settings.failover
for s in self._sweeps:
# would be nice to put this somewhere else in the hierarchy - not
# sure how to do that though (should be handled in Interfaces?)
try:
result += "%s\n" % s.get_output()
except Exception as e:
if failover:
logger.warning("Processing sweep %s failed: %s",
s.get_name(), str(e))
remove.append(s)
else:
raise
for s in remove:
self._sweeps.remove(s)
return result[:-1]
def get_qsub_command():
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
if mp_params.qsub_command:
return mp_params.qsub_command
return None
def find_hdf5_lib(template=None):
global __hdf5_lib
from xia2.Applications.xia2setup import is_hd5f_name
if template and not is_hd5f_name(template):
return ""
if __hdf5_lib:
return __hdf5_lib
import os
from xia2.Handlers.Phil import PhilIndex
from dials.util import Sorry
plugin_name = PhilIndex.get_python_object().xds.hdf5_plugin
if os.path.isabs(plugin_name):
if not os.path.exists(plugin_name):
raise Sorry("Cannot find plugin %s" % plugin_name)
__hdf5_lib = "LIB=%s\n" % plugin_name
return __hdf5_lib
for d in os.environ["PATH"].split(os.pathsep):
if os.path.exists(os.path.join(d, plugin_name)):
__hdf5_lib = "LIB=%s\n" % os.path.join(d, plugin_name)
return __hdf5_lib
return ""
def get_qsub_command():
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
if mp_params.qsub_command:
return mp_params.qsub_command
return None
def xia2_setup():
"""Actually process something..."""
Citations.cite("xia2")
# print versions of related software
logger.info(dials_version())
ccp4_version = get_ccp4_version()
if ccp4_version:
logger.info("CCP4 %s", ccp4_version)
CommandLine = get_command_line()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
for name, xtal in xtals.items():
if not xtal.get_all_image_names():
logger.info("-----------------------------------" +
"-" * len(name))
logger.info("| No images assigned for crystal %s |", name)
logger.info("-----------------------------------" +
"-" * len(name))
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
xinfo = CommandLine.get_xinfo()
logger.info(f"Project directory: {xinfo.path}")
logger.info(f"xinfo written to: {params.xia2.settings.input.xinfo}")
logger.info(f"Parameters: {PhilIndex.get_diff().as_str()}")
def get_detector_identification(self):
detector_id = PhilIndex.get_python_object().xia2.settings.developmental.detector_id
# eg. 'PILATUS 2M, S/N 24-0107 Diamond'
if not detector_id and self.get_imageset():
detector_id = self.get_imageset().get_detector()[0].get_identifier()
if detector_id:
Debug.write('Detector identified as %s' % detector_id)
return detector_id
else:
Debug.write('Detector could not be identified')
return None
def _read_pipeline():
settings = PhilIndex.get_python_object().xia2.settings
indexer, refiner, integrater, scaler = None, None, None, None
if settings.pipeline == "3d":
logger.debug("3DR pipeline selected")
indexer, refiner, integrater, scaler = "xds", "xds", "xdsr", "xdsa"
elif settings.pipeline == "3di":
logger.debug("3DR pipeline; XDS indexing selected")
indexer, refiner, integrater, scaler = "xds", "xds", "xdsr", "xdsa"
elif settings.pipeline == "3dii":
logger.debug("3D II R pipeline (XDS IDXREF all images) selected")
indexer, refiner, integrater, scaler = "xdsii", "xds", "xdsr", "xdsa"
elif settings.pipeline == "3dd":
logger.debug("3DD pipeline (DIALS indexing) selected")
indexer, refiner, integrater, scaler = "dials", "xds", "xdsr", "xdsa"
elif settings.pipeline == "dials":
logger.debug("DIALS pipeline selected")
indexer, refiner, integrater, scaler = "dials", "dials", "dials", "dials"
elif settings.pipeline == "dials-aimless":
logger.debug(
"DIALS-LEGACY pipeline selected (DIALS, scaling with AIMLESS)")
indexer, refiner, integrater, scaler = "dials", "dials", "dials", "ccp4a"
if indexer is not None and settings.indexer is None:
PhilIndex.update("xia2.settings.indexer=%s" % indexer)
if refiner is not None and settings.refiner is None:
PhilIndex.update("xia2.settings.refiner=%s" % refiner)
if integrater is not None and settings.integrater is None:
PhilIndex.update("xia2.settings.integrater=%s" % integrater)
if scaler is not None and settings.scaler is None:
PhilIndex.update("xia2.settings.scaler=%s" % scaler)
if settings.scaler is not None:
if settings.pipeline.startswith("2d"):
allowed_scalers = ("ccp4a", )
elif settings.pipeline.startswith("3d"):
allowed_scalers = ("xdsa", "ccp4a")
elif settings.pipeline.startswith("dials"):
allowed_scalers = ("dials", "ccp4a")
if settings.scaler not in allowed_scalers:
raise ValueError(
"scaler=%s not compatible with pipeline=%s (compatible scalers are %s)"
% (settings.scaler, settings.pipeline,
" or ".join(allowed_scalers)))
def load_experiments(filename):
experiments = load.experiment_list(filename, check_format=False)
imagesets = experiments.imagesets()
params = PhilIndex.get_python_object()
reference_geometry = params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
update_with_reference_geometry(imagesets, reference_geometry)
for imageset in imagesets:
template = imageset.get_template()
if template not in imageset_cache:
imageset_cache[template] = collections.OrderedDict()
imageset_cache[template][imageset.get_scan().get_image_range()[0]] = imageset
def get_detector_identification(self):
detector_id = (PhilIndex.get_python_object().xia2.settings.
developmental.detector_id)
# eg. 'PILATUS 2M, S/N 24-0107 Diamond'
if not detector_id and self.get_imageset():
detector_id = self.get_imageset().get_detector()[0].get_identifier(
)
if detector_id:
logger.debug("Detector identified as %s" % detector_id)
return detector_id
else:
logger.debug("Detector could not be identified")
return None
def _read_pipeline(settings):
settings = PhilIndex.get_python_object().xia2.settings
indexer, refiner, integrater, scaler = None, None, None, None
if settings.pipeline == '2d':
Debug.write('2DA pipeline selected')
print(
"***\n\nWarning: Pipeline '%s' is no longer supported and will be removed in a future release.\n\n***"
% settings.pipeline)
indexer, refiner, integrater, scaler = 'mosflm', 'mosflm', 'mosflmr', 'ccp4a'
elif settings.pipeline == '2di':
Debug.write('2DA pipeline; mosflm indexing selected')
print(
"***\n\nWarning: Pipeline '%s' is no longer supported and will be removed in a future release.\n\n***"
% settings.pipeline)
indexer, refiner, integrater, scaler = 'mosflm', 'mosflm', 'mosflmr', 'ccp4a'
elif settings.pipeline == '3d':
Debug.write('3DR pipeline selected')
indexer, refiner, integrater, scaler = 'xds', 'xds', 'xdsr', 'xdsa'
elif settings.pipeline == '3di':
Debug.write('3DR pipeline; XDS indexing selected')
indexer, refiner, integrater, scaler = 'xds', 'xds', 'xdsr', 'xdsa'
elif settings.pipeline == '3dii':
Debug.write('3D II R pipeline (XDS IDXREF all images) selected')
indexer, refiner, integrater, scaler = 'xdsii', 'xds', 'xdsr', 'xdsa'
elif settings.pipeline == '3dd':
Debug.write('3DD pipeline (DIALS indexing) selected')
indexer, refiner, integrater, scaler = 'dials', 'xds', 'xdsr', 'xdsa'
elif settings.pipeline == 'dials':
Debug.write('DIALS pipeline selected')
indexer, refiner, integrater, scaler = 'dials', 'dials', 'dials', 'ccp4a'
if indexer is not None and settings.indexer is None:
PhilIndex.update("xia2.settings.indexer=%s" % indexer)
if refiner is not None and settings.refiner is None:
PhilIndex.update("xia2.settings.refiner=%s" % refiner)
if integrater is not None and settings.integrater is None:
PhilIndex.update("xia2.settings.integrater=%s" % integrater)
if scaler is not None and settings.scaler is None:
PhilIndex.update("xia2.settings.scaler=%s" % scaler)
def get_sweeps(templates):
global known_sweeps
from libtbx import easy_mp
from xia2.Applications.xia2setup_helpers import get_sweep
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
nproc = mp_params.nproc
if params.xia2.settings.read_all_image_headers and nproc > 1:
method = "multiprocessing"
# If xia2 was a proper cctbx module, then we wouldn't have to do this
# FIXME xia2 is now a proper cctbx module ;o)
python_path = 'PYTHONPATH="%s"' % ":".join(sys.path)
qsub_command = "qsub -v %s -V" % python_path
args = [(template, ) for template in templates]
results_list = easy_mp.parallel_map(
get_sweep,
args,
processes=nproc,
method=method,
qsub_command=qsub_command,
asynchronous=True,
preserve_order=True,
preserve_exception_message=True,
)
else:
results_list = [get_sweep((template, )) for template in templates]
from xia2.Schema import imageset_cache
for template, sweeplist in zip(templates, results_list):
if sweeplist is not None:
known_sweeps[template] = sweeplist
for sweep in sweeplist:
imageset = sweep.get_imageset()
if template not in imageset_cache:
imageset_cache[template] = collections.OrderedDict()
imageset_cache[template][imageset.get_scan().get_image_range()
[0]] = imageset
def load_datablock(filename):
from xia2.Schema import imageset_cache, update_with_reference_geometry
from dxtbx.serialize import load
datablocks = load.datablock(filename, check_format=False)
for datablock in datablocks:
imagesets = datablock.extract_imagesets()
params = PhilIndex.get_python_object()
reference_geometry = params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
update_with_reference_geometry(imagesets, reference_geometry)
for imageset in imagesets:
template = imageset.get_template()
if template not in imageset_cache:
imageset_cache[template] = collections.OrderedDict()
imageset_cache[template][imageset.get_scan().get_image_range()
[0]] = imageset
def write_xinfo(filename, directories, template=None, hdf5_master_files=None):
global target_template
target_template = template
settings = PhilIndex.get_python_object().xia2.settings
crystal = settings.crystal
if not os.path.isabs(filename):
filename = os.path.abspath(filename)
directory = os.path.join(os.getcwd(), crystal, 'setup')
try:
os.makedirs(directory)
except OSError as e:
if not 'File exists' in str(e):
raise
# FIXME should I have some exception handling in here...?
start = os.getcwd()
os.chdir(directory)
# if we have given a template and directory on the command line, just
# look there (i.e. not in the subdirectories)
import libtbx.load_env
if CommandLine.get_template() and CommandLine.get_directory():
templates = set()
for directory in CommandLine.get_directory():
templates.update(visit(None, directory, os.listdir(directory)))
get_sweeps(templates)
elif hdf5_master_files is not None:
get_sweeps(hdf5_master_files)
else:
rummage(directories)
fout = open(filename, 'w')
print_sweeps(fout)
# change back directory c/f bug # 2693 - important for error files...
os.chdir(start)
def run():
streams_off()
# test to see if sys.argv[-2] + path is a valid path - to work around
# spaced command lines
argv = CommandLine.get_argv()
if not CommandLine.get_directory():
directories = []
for arg in argv:
if os.path.isdir(arg):
directories.append(os.path.abspath(arg))
if not directories:
raise RuntimeError('directory not found in arguments')
else:
directories = [CommandLine.get_directory()]
directories = [os.path.abspath(d) for d in directories]
# perhaps move to a new directory...
settings = PhilIndex.get_python_object().xia2.settings
crystal = settings.crystal
with open(os.path.join(os.getcwd(), 'automatic.xinfo'), 'w') as fout:
start = os.path.abspath(os.getcwd())
directory = os.path.join(os.getcwd(), crystal, 'setup')
try:
os.makedirs(directory)
except OSError as e:
if not 'File exists' in str(e):
raise e
os.chdir(directory)
rummage(directories)
print_sweeps(fout)
def write_xinfo(filename, directories, template=None, hdf5_master_files=None):
global target_template
target_template = template
settings = PhilIndex.get_python_object().xia2.settings
crystal = settings.crystal
if not os.path.isabs(filename):
filename = os.path.abspath(filename)
directory = os.path.join(os.getcwd(), crystal, "setup")
try:
os.makedirs(directory)
except OSError as e:
if "File exists" not in str(e):
raise
# if we have given a template and directory on the command line, just
# look there (i.e. not in the subdirectories)
if CommandLine.get_template() and CommandLine.get_directory():
# xia2 image=$(dials.data get -q x4wide)/X4_wide_M1S4_2_0001.cbf
templates = set()
for directory in CommandLine.get_directory():
templates.update(visit(directory, os.listdir(directory)))
sweeps = _get_sweeps(templates)
elif hdf5_master_files is not None:
# xia2 image=$(dials.data get -q vmxi_thaumatin)/image_15799_master.h5
sweeps = _get_sweeps(hdf5_master_files)
else:
# xia2 $(dials.data get -q x4wide)
sweeps = _rummage(directories)
with open(filename, "w") as fout:
_write_sweeps(sweeps, fout)
def get_output(self):
result = 'Wavelength name: %s\n' % self._name
result += 'Wavelength %7.5f\n' % self._wavelength
if self._f_pr != 0.0 and self._f_prpr != 0.0:
result += 'F\', F\'\' = (%5.2f, %5.2f)\n' % (self._f_pr,
self._f_prpr)
result += 'Sweeps:\n'
from xia2.Driver.DriverFactory import DriverFactory
def run_one_sweep(args):
from xia2.Handlers.Streams import Debug
assert len(args) == 3
s, failover, job_type = args
if job_type:
DriverFactory.set_driver_type(job_type)
Chatter.cache()
Debug.cache()
try:
s.get_integrater_intensities()
except Exception as e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(s.get_name(), str(e)))
s = None
else:
raise
finally:
Chatter.uncache()
Debug.uncache()
return s
remove = []
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
failover = params.xia2.settings.failover
for s in self._sweeps:
# would be nice to put this somewhere else in the hierarchy - not
# sure how to do that though (should be handled in Interfaces?)
try:
result += '%s\n' % s.get_output()
except Exception as e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(s.get_name(), str(e)))
remove.append(s)
else:
raise
for s in remove:
self._sweeps.remove(s)
return result[:-1]
def _write_sweeps(sweeps, out):
global latest_sequence
_known_sweeps = sweeps
sweeplist = sorted(_known_sweeps)
sweeplist = _filter_aliased_hdf5_sweeps(sweeplist)
assert sweeplist, "no sweeps found"
# sort sweeplist based on epoch of first image of each sweep
epochs = [
_known_sweeps[sweep][0].get_imageset().get_scan().get_epochs()[0]
for sweep in sweeplist
]
if len(epochs) != len(set(epochs)):
logger.debug(
"Duplicate epochs found. Trying to correct epoch information.")
cumulativedelta = 0.0
for sweep in sweeplist:
_known_sweeps[sweep][0].get_imageset().get_scan().set_epochs(
_known_sweeps[sweep][0].get_imageset().get_scan().get_epochs()
+ cumulativedelta)
# could change the image epoch information individually, but only
# the information from the first image is used at this time.
cumulativedelta += sum(
_known_sweeps[sweep]
[0].get_imageset().get_scan().get_exposure_times())
epochs = [
_known_sweeps[sweep][0].get_imageset().get_scan().get_epochs()[0]
for sweep in sweeplist
]
if len(epochs) != len(set(epochs)):
logger.debug("Duplicate epoch information remains.")
# This should only happen with incorrect exposure time information.
sweeplist = [s for _, s in sorted(zip(epochs, sweeplist))]
# analysis pass
wavelengths = []
settings = PhilIndex.get_python_object().xia2.settings
wavelength_tolerance = settings.wavelength_tolerance
min_images = settings.input.min_images
min_oscillation_range = settings.input.min_oscillation_range
for sweep in sweeplist:
sweeps = _known_sweeps[sweep]
# sort on exposure epoch followed by first image number
sweeps = sorted(
sweeps,
key=lambda s: (
s.get_imageset().get_scan().get_epochs()[0],
s.get_images()[0],
),
)
for s in sweeps:
if len(s.get_images()) < min_images:
logger.debug("Rejecting sweep %s:" % s.get_template())
logger.debug(
" Not enough images (found %i, require at least %i)" %
(len(s.get_images()), min_images))
continue
oscillation_range = s.get_imageset().get_scan(
).get_oscillation_range()
width = oscillation_range[1] - oscillation_range[0]
if min_oscillation_range is not None and width < min_oscillation_range:
logger.debug("Rejecting sweep %s:" % s.get_template())
logger.debug(
" Too narrow oscillation range (found %i, require at least %i)"
% (width, min_oscillation_range))
continue
wavelength = s.get_wavelength()
if wavelength not in wavelengths:
have_wavelength = False
for w in wavelengths:
if abs(w - wavelength) < wavelength_tolerance:
have_wavelength = True
s.set_wavelength(w)
if not have_wavelength:
wavelengths.append(wavelength)
assert wavelengths, "No sweeps found matching criteria"
wavelength_map = {}
project = settings.project
crystal = settings.crystal
out.write("BEGIN PROJECT %s\n" % project)
out.write("BEGIN CRYSTAL %s\n" % crystal)
out.write("\n")
# check to see if a user spacegroup has been assigned - if it has,
# copy it in...
if settings.space_group is not None:
out.write("USER_SPACEGROUP %s\n" %
settings.space_group.type().lookup_symbol())
out.write("\n")
if settings.unit_cell is not None:
out.write("USER_CELL %.2f %.2f %.2f %.2f %.2f %.2f\n" %
settings.unit_cell.parameters())
out.write("\n")
freer_file = PhilIndex.params.xia2.settings.scale.freer_file
if freer_file is not None:
out.write("FREER_FILE %s\n" %
PhilIndex.params.xia2.settings.scale.freer_file)
out.write("\n")
if latest_sequence:
out.write("BEGIN AA_SEQUENCE\n")
out.write("\n")
for sequence_chunk in [
latest_sequence[i:i + 60]
for i in range(0, len(latest_sequence), 60)
]:
out.write("%s\n" % sequence_chunk)
out.write("\n")
out.write("END AA_SEQUENCE\n")
out.write("\n")
if settings.input.atom:
out.write("BEGIN HA_INFO\n")
out.write("ATOM %s\n" % settings.input.atom.lower())
out.write("END HA_INFO\n")
out.write("\n")
elif settings.input.anomalous:
out.write("BEGIN HA_INFO\n")
out.write("ATOM X\n")
out.write("END HA_INFO\n")
out.write("\n")
for j, wavelength in enumerate(wavelengths):
anomalous = settings.input.anomalous
if settings.input.atom is not None:
anomalous = True
if len(wavelengths) == 1 and anomalous:
name = "SAD"
elif len(wavelengths) == 1:
name = "NATIVE"
else:
name = "WAVE%d" % (j + 1)
wavelength_map[wavelength] = name
out.write("BEGIN WAVELENGTH %s\n" % name)
dmin = PhilIndex.params.xia2.settings.resolution.d_min
dmax = PhilIndex.params.xia2.settings.resolution.d_max
if dmin and dmax:
out.write(f"RESOLUTION {dmin:f} {dmax:f}\n")
elif dmin:
out.write("RESOLUTION %f\n" % dmin)
out.write("WAVELENGTH %f\n" % wavelengths[j])
out.write("END WAVELENGTH %s\n" % name)
out.write("\n")
j = 0
for sweep in sweeplist:
sweeps = _known_sweeps[sweep]
# sort on exposure epoch followed by first image number
sweeps = sorted(
sweeps,
key=lambda s: (
s.get_imageset().get_scan().get_epochs()[0],
s.get_images()[0],
),
)
for s in sweeps:
# require at least n images to represent a sweep...
if len(s.get_images()) < min_images:
logger.debug("Rejecting sweep %s:" % s.get_template())
logger.debug(
" Not enough images (found %i, require at least %i)" %
(len(s.get_images()), min_images))
continue
oscillation_range = s.get_imageset().get_scan(
).get_oscillation_range()
width = oscillation_range[1] - oscillation_range[0]
if min_oscillation_range is not None and width < min_oscillation_range:
logger.debug("Rejecting sweep %s:" % s.get_template())
logger.debug(
" Too narrow oscillation range (found %i, require at least %i)"
% (width, min_oscillation_range))
continue
key = os.path.join(s.get_directory(), s.get_template())
if CommandLine.get_start_ends(key):
start_ends = CommandLine.get_start_ends(key)
start_good = (min(s.get_images()) <= start_ends[0][0] <= max(
s.get_images()))
end_good = (min(s.get_images()) <= start_ends[0][1] <= max(
s.get_images()))
if not all((start_good, end_good)):
logger.debug("Rejecting sweep %s:" % s.get_template())
if not start_good:
logger.debug(
" Your specified start-point image lies outside the bounds of this sweep."
)
if not end_good:
logger.debug(
" Your specified end-point image lies outside the bounds of this sweep."
)
logger.debug(
" Your specified start and end points were %d & %d," %
start_ends[0])
logger.debug(
" this sweep consists of images from %d to %d." %
(min(s.get_images()), max(s.get_images())))
logger.debug(
""" If there are missing images in your sweep, but you have selected valid
start and end points within a contiguous range of images, you will see this
message, even though all is well with your selection, because xia2 treats
each contiguous image range as a separate sweep.""")
continue
else:
start_ends = [(min(s.get_images()), max(s.get_images()))]
for start_end in start_ends:
j += 1
name = "SWEEP%d" % j
out.write("BEGIN SWEEP %s\n" % name)
if PhilIndex.params.xia2.settings.input.reverse_phi:
out.write("REVERSEPHI\n")
out.write("WAVELENGTH %s\n" %
wavelength_map[s.get_wavelength()])
out.write("DIRECTORY %s\n" % s.get_directory())
imgset = s.get_imageset()
out.write("IMAGE %s\n" % os.path.split(imgset.get_path(0))[-1])
out.write("START_END %d %d\n" % start_end)
# really don't need to store the epoch in the xinfo file
# out.write('EPOCH %d\n' % int(s.get_collect()[0]))
if not settings.trust_beam_centre:
PhilIndex.params.xia2.settings.interactive = False
PhilIndex.get_python_object()
if settings.detector_distance is not None:
out.write("DISTANCE %.2f\n" % settings.detector_distance)
out.write("END SWEEP %s\n" % name)
out.write("\n")
out.write("END CRYSTAL %s\n" % crystal)
out.write("END PROJECT %s\n" % project)
def write_xml(self, file, command_line="", working_phil=None):
if working_phil is not None:
PhilIndex.merge_phil(working_phil)
params = PhilIndex.get_python_object()
fout = open(file, "w")
fout.write('<?xml version="1.0"?>')
fout.write("<AutoProcContainer>\n")
for crystal in sorted(self._crystals):
xcrystal = self._crystals[crystal]
cell = xcrystal.get_cell()
spacegroup = xcrystal.get_likely_spacegroups()[0]
fout.write("<AutoProc><spaceGroup>%s</spaceGroup>" % spacegroup)
self.write_refined_cell(fout, cell)
fout.write("</AutoProc>")
fout.write("<AutoProcScalingContainer>")
fout.write("<AutoProcScaling>")
self.write_date(fout)
fout.write("</AutoProcScaling>")
statistics_all = xcrystal.get_statistics()
reflection_files = xcrystal.get_scaled_merged_reflections()
for key in statistics_all:
pname, xname, dname = key
# FIXME should assert that the dname is a
# valid wavelength name
keys = [
"High resolution limit",
"Low resolution limit",
"Completeness",
"Multiplicity",
"I/sigma",
"Rmerge(I+/-)",
"CC half",
"Anomalous completeness",
"Anomalous correlation",
"Anomalous multiplicity",
"Total observations",
"Total unique",
"Rmeas(I)",
"Rmeas(I+/-)",
"Rpim(I)",
"Rpim(I+/-)",
"Partial Bias",
]
stats = [k for k in keys if k in statistics_all[key]]
xwavelength = xcrystal.get_xwavelength(dname)
sweeps = xwavelength.get_sweeps()
for j, name in enumerate(
["overall", "innerShell", "outerShell"]):
statistics_cache = {}
for s in stats:
if isinstance(statistics_all[key][s], type([])):
statistics_cache[s] = statistics_all[key][s][j]
elif isinstance(statistics_all[key][s], type(())):
statistics_cache[s] = statistics_all[key][s][j]
# send these to be written out
self.write_scaling_statistics(fout, name, statistics_cache)
for sweep in sweeps:
fout.write("<AutoProcIntegrationContainer>\n")
if "#" in sweep.get_template():
image_name = sweep.get_image_name(0)
else:
image_name = os.path.join(sweep.get_directory(),
sweep.get_template())
fout.write("<Image><fileName>%s</fileName>" %
os.path.split(image_name)[-1])
fout.write("<fileLocation>%s</fileLocation></Image>" %
sanitize(os.path.split(image_name)[0]))
fout.write("<AutoProcIntegration>\n")
cell = sweep.get_integrater_cell()
self.write_cell(fout, cell)
# FIXME this is naughty
intgr = sweep._get_integrater()
start, end = intgr.get_integrater_wedge()
fout.write("<startImageNumber>%d</startImageNumber>" %
start)
fout.write("<endImageNumber>%d</endImageNumber>" % end)
# FIXME this is naughty
indxr = sweep._get_indexer()
fout.write(
"<refinedDetectorDistance>%f</refinedDetectorDistance>"
% indxr.get_indexer_distance())
beam = indxr.get_indexer_beam_centre_raw_image()
fout.write("<refinedXBeam>%f</refinedXBeam>" % beam[0])
fout.write("<refinedYBeam>%f</refinedYBeam>" % beam[1])
fout.write("</AutoProcIntegration>\n")
fout.write("</AutoProcIntegrationContainer>\n")
fout.write("</AutoProcScalingContainer>")
# file unpacking nonsense
if not command_line:
from xia2.Handlers.CommandLine import CommandLine
command_line = CommandLine.get_command_line()
pipeline = params.xia2.settings.pipeline
fout.write("<AutoProcProgramContainer><AutoProcProgram>")
fout.write("<processingCommandLine>%s</processingCommandLine>" %
sanitize(command_line))
fout.write("<processingPrograms>xia2 %s</processingPrograms>" %
pipeline)
fout.write("</AutoProcProgram>")
data_directory = self._project.path / "DataFiles"
log_directory = self._project.path / "LogFiles"
for k in reflection_files:
reflection_file = reflection_files[k]
if not isinstance(reflection_file, type("")):
continue
reflection_file = FileHandler.get_data_file(
self._project.path, reflection_file)
basename = os.path.basename(reflection_file)
if data_directory.joinpath(basename).exists():
# Use file in DataFiles directory in preference (if it exists)
reflection_file = str(data_directory.joinpath(basename))
fout.write("<AutoProcProgramAttachment><fileType>Result")
fout.write("</fileType><fileName>%s</fileName>" %
os.path.split(reflection_file)[-1])
fout.write("<filePath>%s</filePath>" %
sanitize(os.path.split(reflection_file)[0]))
fout.write("</AutoProcProgramAttachment>\n")
g = log_directory.glob("*merging-statistics.json")
for merging_stats_json in g:
fout.write("<AutoProcProgramAttachment><fileType>Graph")
fout.write("</fileType><fileName>%s</fileName>" %
os.path.split(str(merging_stats_json))[-1])
fout.write("<filePath>%s</filePath>" %
sanitize(str(log_directory)))
fout.write("</AutoProcProgramAttachment>\n")
# add the xia2.txt file...
fout.write("<AutoProcProgramAttachment><fileType>Log")
fout.write("</fileType><fileName>xia2.txt</fileName>")
fout.write("<filePath>%s</filePath>" % sanitize(os.getcwd()))
fout.write("</AutoProcProgramAttachment>\n")
fout.write("</AutoProcProgramContainer>")
fout.write("</AutoProcContainer>\n")
fout.close()
except Exception, e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(s.get_name(), str(e)))
s = None
else:
raise
finally:
Chatter.uncache()
Debug.uncache()
return s
remove = []
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
failover = params.xia2.settings.failover
for s in self._sweeps:
# would be nice to put this somewhere else in the hierarchy - not
# sure how to do that though (should be handled in Interfaces?)
try:
result += '%s\n' % s.get_output()
except Exception, e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(s.get_name(), str(e)))
remove.append(s)
else:
def print_sweeps(out=sys.stdout):
global known_sweeps, latest_sequence
sweeplists = known_sweeps.keys()
assert len(sweeplists) > 0, "no sweeps found"
sweeplists.sort()
# sort sweeplist based on epoch of first image of each sweep
import operator
epochs = [
known_sweeps[sweep][0].get_imageset().get_scan().get_epochs()[0]
for sweep in sweeplists
]
if len(epochs) != len(set(epochs)):
Debug.write(
'Duplicate epochs found. Trying to correct epoch information.')
cumulativedelta = 0.0
for sweep in sweeplists:
known_sweeps[sweep][0].get_imageset().get_scan().set_epochs(
known_sweeps[sweep][0].get_imageset().get_scan().get_epochs() +
cumulativedelta)
# could change the image epoch information individually, but only
# the information from the first image is used at this time.
cumulativedelta += sum(
known_sweeps[sweep]
[0].get_imageset().get_scan().get_exposure_times())
epochs = [
known_sweeps[sweep][0].get_imageset().get_scan().get_epochs()[0]
for sweep in sweeplists
]
if len(epochs) != len(set(epochs)):
Debug.write('Duplicate epoch information remains.')
# This should only happen with incorrect exposure time information.
sweeplists, epochs = zip(
*sorted(zip(sweeplists, epochs), key=operator.itemgetter(1)))
# analysis pass
wavelengths = []
settings = PhilIndex.get_python_object().xia2.settings
wavelength_tolerance = settings.wavelength_tolerance
min_images = settings.input.min_images
min_oscillation_range = settings.input.min_oscillation_range
for sweep in sweeplists:
sweeps = known_sweeps[sweep]
# sort on exposure epoch
epochs = [s.get_imageset().get_scan().get_epochs()[0] for s in sweeps]
sweeps, epochs = zip(
*sorted(zip(sweeps, epochs), key=operator.itemgetter(1)))
for s in sweeps:
if len(s.get_images()) < min_images:
Debug.write('Rejecting sweep %s:' % s.get_template())
Debug.write(
' Not enough images (found %i, require at least %i)' %
(len(s.get_images()), min_images))
continue
oscillation_range = s.get_imageset().get_scan(
).get_oscillation_range()
width = oscillation_range[1] - oscillation_range[0]
if width < min_oscillation_range:
Debug.write('Rejecting sweep %s:' % s.get_template())
Debug.write(
' Too narrow oscillation range (found %i, require at least %i)'
% (width, min_oscillation_range))
continue
wavelength = s.get_wavelength()
if not wavelength in wavelengths:
have_wavelength = False
for w in wavelengths:
if abs(w - wavelength) < wavelength_tolerance:
have_wavelength = True
s.set_wavelength(w)
if not have_wavelength:
wavelengths.append(wavelength)
assert len(wavelengths), "No sweeps found matching criteria"
wavelength_map = {}
project = settings.project
crystal = settings.crystal
out.write('BEGIN PROJECT %s\n' % project)
out.write('BEGIN CRYSTAL %s\n' % crystal)
out.write('\n')
# check to see if a user spacegroup has been assigned - if it has,
# copy it in...
if settings.space_group is not None:
out.write('USER_SPACEGROUP %s\n' %
settings.space_group.type().lookup_symbol())
out.write('\n')
if settings.unit_cell is not None:
out.write('USER_CELL %.2f %.2f %.2f %.2f %.2f %.2f\n' % \
settings.unit_cell.parameters())
out.write('\n')
freer_file = PhilIndex.params.xia2.settings.scale.freer_file
if freer_file is not None:
out.write('FREER_FILE %s\n' %
PhilIndex.params.xia2.settings.scale.freer_file)
out.write('\n')
if latest_sequence:
out.write('BEGIN AA_SEQUENCE\n')
out.write('\n')
for sequence_chunk in [latest_sequence[i:i + 60] \
for i in range(0, len(latest_sequence), 60)]:
out.write('%s\n' % sequence_chunk)
out.write('\n')
out.write('END AA_SEQUENCE\n')
out.write('\n')
if settings.input.atom:
out.write('BEGIN HA_INFO\n')
out.write('ATOM %s\n' % settings.input.atom.lower())
out.write('END HA_INFO\n')
out.write('\n')
elif settings.input.anomalous:
out.write('BEGIN HA_INFO\n')
out.write('ATOM X\n')
out.write('END HA_INFO\n')
out.write('\n')
for j in range(len(wavelengths)):
anomalous = settings.input.anomalous
if settings.input.atom is not None:
anomalous = True
if len(wavelengths) == 1 and anomalous:
name = 'SAD'
elif len(wavelengths) == 1:
name = 'NATIVE'
else:
name = 'WAVE%d' % (j + 1)
wavelength_map[wavelengths[j]] = name
out.write('BEGIN WAVELENGTH %s\n' % name)
dmin = PhilIndex.params.xia2.settings.resolution.d_min
dmax = PhilIndex.params.xia2.settings.resolution.d_max
if dmin and dmax:
out.write('RESOLUTION %f %f\n' % (dmin, dmax))
elif dmin:
out.write('RESOLUTION %f\n' % dmin)
out.write('WAVELENGTH %f\n' % wavelengths[j])
out.write('END WAVELENGTH %s\n' % name)
out.write('\n')
j = 0
for sweep in sweeplists:
sweeps = known_sweeps[sweep]
# sort on exposure epoch
epochs = [s.get_imageset().get_scan().get_epochs()[0] for s in sweeps]
sweeps, epochs = zip(
*sorted(zip(sweeps, epochs), key=operator.itemgetter(1)))
for s in sweeps:
# require at least n images to represent a sweep...
if len(s.get_images()) < min_images:
Debug.write('Rejecting sweep %s:' % s.get_template())
Debug.write(
' Not enough images (found %i, require at least %i)' %
(len(s.get_images()), min_images))
continue
oscillation_range = s.get_imageset().get_scan(
).get_oscillation_range()
width = oscillation_range[1] - oscillation_range[0]
if width < min_oscillation_range:
Debug.write('Rejecting sweep %s:' % s.get_template())
Debug.write(
' Too narrow oscillation range (found %i, require at least %i)'
% (width, min_oscillation_range))
continue
key = os.path.join(s.get_directory(), s.get_template())
if CommandLine.get_start_ends(key):
start_ends = CommandLine.get_start_ends(key)
else:
start_ends = [(min(s.get_images()), max(s.get_images()))]
for start_end in start_ends:
j += 1
name = 'SWEEP%d' % j
out.write('BEGIN SWEEP %s\n' % name)
if PhilIndex.params.xia2.settings.input.reverse_phi:
out.write('REVERSEPHI\n')
out.write('WAVELENGTH %s\n' %
wavelength_map[s.get_wavelength()])
out.write('DIRECTORY %s\n' % s.get_directory())
imgset = s.get_imageset()
out.write('IMAGE %s\n' % os.path.split(imgset.get_path(0))[-1])
out.write('START_END %d %d\n' % start_end)
# really don't need to store the epoch in the xinfo file
# out.write('EPOCH %d\n' % int(s.get_collect()[0]))
if not settings.trust_beam_centre:
interactive = False
if PhilIndex.params.xia2.settings.interactive == True:
interactive = True
PhilIndex.params.xia2.settings.interactive = False
PhilIndex.get_python_object()
beam_centre = compute_beam_centre(s)
if beam_centre:
out.write('BEAM %6.2f %6.2f\n' % tuple(beam_centre))
PhilIndex.params.xia2.settings.interactive = interactive
PhilIndex.get_python_object()
if settings.detector_distance is not None:
out.write('DISTANCE %.2f\n' % settings.detector_distance)
out.write('END SWEEP %s\n' % name)
out.write('\n')
out.write('END CRYSTAL %s\n' % crystal)
out.write('END PROJECT %s\n' % project)
def load_imagesets(
template,
directory,
id_image=None,
image_range=None,
use_cache=True,
reversephi=False,
):
global imageset_cache
from dxtbx.model.experiment_list import ExperimentListFactory
from xia2.Applications.xia2setup import known_hdf5_extensions
from dxtbx.imageset import ImageSweep
full_template_path = os.path.join(directory, template)
if full_template_path not in imageset_cache or not use_cache:
from dxtbx.model.experiment_list import BeamComparison
from dxtbx.model.experiment_list import DetectorComparison
from dxtbx.model.experiment_list import GoniometerComparison
params = PhilIndex.params.xia2.settings
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction,
)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin,
)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation,
)
scan_tolerance = params.input.tolerance.scan.oscillation
format_kwargs = {
"dynamic_shadowing": params.input.format.dynamic_shadowing,
"multi_panel": params.input.format.multi_panel,
}
if os.path.splitext(full_template_path)[-1] in known_hdf5_extensions:
# if we are passed the correct file, use this, else look for a master
# file (i.e. something_master.h5)
if os.path.exists(full_template_path) and os.path.isfile(
full_template_path
):
master_file = full_template_path
else:
import glob
g = glob.glob(os.path.join(directory, "*_master.h5"))
master_file = None
for p in g:
substr = longest_common_substring(template, p)
if substr:
if master_file is None or (
len(substr)
> len(longest_common_substring(template, master_file))
):
master_file = p
if master_file is None:
raise RuntimeError("Can't find master file for %s" % full_template_path)
unhandled = []
experiments = ExperimentListFactory.from_filenames(
[master_file],
verbose=False,
unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs,
)
assert len(unhandled) == 0, (
"unhandled image files identified: %s" % unhandled
)
else:
from dxtbx.sweep_filenames import locate_files_matching_template_string
params = PhilIndex.get_python_object()
read_all_image_headers = params.xia2.settings.read_all_image_headers
if read_all_image_headers:
paths = sorted(
locate_files_matching_template_string(full_template_path)
)
unhandled = []
experiments = ExperimentListFactory.from_filenames(
paths,
verbose=False,
unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs,
)
assert len(unhandled) == 0, (
"unhandled image files identified: %s" % unhandled
)
else:
from dxtbx.model.experiment_list import ExperimentListTemplateImporter
importer = ExperimentListTemplateImporter(
[full_template_path], format_kwargs=format_kwargs
)
experiments = importer.experiments
imagesets = [
iset for iset in experiments.imagesets() if isinstance(iset, ImageSweep)
]
assert len(imagesets) > 0, "no imageset found"
imageset_cache[full_template_path] = collections.OrderedDict()
if reversephi:
for imageset in imagesets:
goniometer = imageset.get_goniometer()
goniometer.set_rotation_axis(
tuple(-g for g in goniometer.get_rotation_axis())
)
reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
update_with_reference_geometry(imagesets, reference_geometry)
# Update the geometry
params = PhilIndex.params.xia2.settings
update_geometry = []
from dials.command_line.dials_import import ManualGeometryUpdater
from dials.util.options import geometry_phil_scope
# Then add manual geometry
work_phil = geometry_phil_scope.format(params.input)
diff_phil = geometry_phil_scope.fetch_diff(source=work_phil)
if diff_phil.as_str() != "":
update_geometry.append(ManualGeometryUpdater(params.input))
imageset_list = []
for imageset in imagesets:
for updater in update_geometry:
imageset = updater(imageset)
imageset_list.append(imageset)
imagesets = imageset_list
from scitbx.array_family import flex
for imageset in imagesets:
scan = imageset.get_scan()
exposure_times = scan.get_exposure_times()
epochs = scan.get_epochs()
if exposure_times.all_eq(0) or exposure_times[0] == 0:
exposure_times = flex.double(exposure_times.size(), 1)
scan.set_exposure_times(exposure_times)
elif not exposure_times.all_gt(0):
exposure_times = flex.double(exposure_times.size(), exposure_times[0])
scan.set_exposure_times(exposure_times)
if epochs.size() > 1 and not epochs.all_gt(0):
if epochs[0] == 0:
epochs[0] = 1
for i in range(1, epochs.size()):
epochs[i] = epochs[i - 1] + exposure_times[i - 1]
scan.set_epochs(epochs)
_id_image = scan.get_image_range()[0]
imageset_cache[full_template_path][_id_image] = imageset
if id_image is not None:
return [imageset_cache[full_template_path][id_image]]
elif image_range is not None:
for imageset in imageset_cache[full_template_path].values():
scan = imageset.get_scan()
scan_image_range = scan.get_image_range()
if (
image_range[0] >= scan_image_range[0]
and image_range[1] <= scan_image_range[1]
):
imagesets = [
imageset[
image_range[0]
- scan_image_range[0] : image_range[1]
+ 1
- scan_image_range[0]
]
]
assert len(imagesets[0]) == image_range[1] - image_range[0] + 1, len(
imagesets[0]
)
return imagesets
return imageset_cache[full_template_path].values()
def xia2_main(stop_after=None):
"""Actually process something..."""
Citations.cite("xia2")
# print versions of related software
logger.info(dials_version())
ccp4_version = get_ccp4_version()
if ccp4_version:
logger.info("CCP4 %s", ccp4_version)
start_time = time.time()
CommandLine = get_command_line()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
for name, xtal in xtals.items():
if not xtal.get_all_image_names():
logger.info("-----------------------------------" +
"-" * len(name))
logger.info("| No images assigned for crystal %s |", name)
logger.info("-----------------------------------" +
"-" * len(name))
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
xinfo = CommandLine.get_xinfo()
logger.info("Project directory: %s", xinfo.path)
if (params.xia2.settings.developmental.continue_from_previous_job
and os.path.exists("xia2.json")):
logger.debug("==== Starting from existing xia2.json ====")
xinfo_new = xinfo
xinfo = XProject.from_json(filename="xia2.json")
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new:
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names(
):
wavelength_new = crystals_new[crystal_id].get_xwavelength(
wavelength_id)
if wavelength_id not in crystals[
crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(
wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = {s.get_name() for s in sweeps}
sweep_keys = {(s.get_directory(), s.get_template(),
s.get_image_range())
for s in sweeps}
for sweep in sweeps_new:
if (
sweep.get_directory(),
sweep.get_template(),
sweep.get_image_range(),
) not in sweep_keys:
if sweep.get_name() in sweep_names:
i = 1
while "SWEEEP%i" % i in sweep_names:
i += 1
sweep._name = "SWEEP%i" % i
break
wavelength.add_sweep(
name=sweep.get_name(),
sample=sweep.sample,
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.add(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
with cleanup(xinfo.path):
if mp_params.mode == "parallel" and njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
jobs = []
for crystal_id in crystals:
for wavelength_id in crystals[crystal_id].get_wavelength_names(
):
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
jobs.append((group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
), ))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(jobs):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command or "qsub"
qsub_command = "%s -V -cwd -pe smp %d" % (qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep,
jobs,
processes=njob,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True,
)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals:
for wavelength_id in crystals[crystal_id].get_wavelength_names(
):
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
if output is not None:
logger.info(output)
if not success:
logger.info("Sweep failed: removing %s",
sweep.get_name())
remove_sweeps.append(sweep)
else:
assert xsweep_dict is not None
logger.info("Loading sweep: %s", sweep.get_name())
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.sample
sample.remove_sweep(sweep)
else:
for crystal_id in list(crystals.keys()):
for wavelength_id in crystals[crystal_id].get_wavelength_names(
):
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
from dials.command_line.show import show_experiments
from dxtbx.model.experiment_list import ExperimentListFactory
logger.debug(sweep.get_name())
logger.debug(
show_experiments(
ExperimentListFactory.
from_imageset_and_crystal(
sweep.get_imageset(), None)))
Citations.cite("dials")
try:
if stop_after == "index":
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception as e:
if failover:
logger.info(
"Processing sweep %s failed: %s",
sweep.get_name(),
str(e),
)
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.sample
sample.remove_sweep(sweep)
# save intermediate xia2.json file in case scaling step fails
xinfo.as_json(filename="xia2.json")
if stop_after not in ("index", "integrate"):
logger.info(xinfo.get_output())
for crystal in list(crystals.values()):
crystal.serialize()
# save final xia2.json file in case report generation fails
xinfo.as_json(filename="xia2.json")
if stop_after not in ("index", "integrate"):
# and the summary file
with open("xia2-summary.dat", "w") as fh:
for record in xinfo.summarise():
fh.write("%s\n" % record)
# looks like this import overwrites the initial command line
# Phil overrides so... for https://github.com/xia2/xia2/issues/150
from xia2.command_line.html import generate_xia2_html
if params.xia2.settings.small_molecule:
params.xia2.settings.report.xtriage_analysis = False
params.xia2.settings.report.include_radiation_damage = False
with xia2.Driver.timing.record_step("xia2.report"):
generate_xia2_html(xinfo,
filename="xia2.html",
params=params.xia2.settings.report)
duration = time.time() - start_time
# write out the time taken in a human readable way
logger.info("Processing took %s",
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
write_citations()
def __init__(
self,
name,
wavelength,
sample,
directory=None,
image=None,
beam=None,
reversephi=False,
distance=None,
gain=0.0,
dmin=0.0,
dmax=0.0,
polarization=0.0,
frames_to_process=None,
user_lattice=None,
user_cell=None,
epoch=0,
ice=False,
excluded_regions=None,
):
"""Create a new sweep named name, belonging to XWavelength object
wavelength, representing the images in directory starting with image,
with beam centre optionally defined."""
if excluded_regions is None:
excluded_regions = []
# FIXME bug 2221 if DIRECTORY starts with ~/ or ~graeme (say) need to
# interpret this properly - e.g. map it to a full PATH.
directory = expand_path(directory)
self._name = name
self._wavelength = wavelength
self.sample = sample
self._directory = directory
self._image = image
self._reversephi = reversephi
self._epoch = epoch
self._user_lattice = user_lattice
self._user_cell = user_cell
self._header = {}
self._resolution_high = dmin
self._resolution_low = dmax
self._ice = ice
self._excluded_regions = excluded_regions
self._imageset = None
# FIXME in here also need to be able to accumulate the total
# dose from all experimental measurements (complex) and provide
# a _epoch_to_dose dictionary or some such... may be fiddly as
# this will need to parse across multiple templates. c/f Bug # 2798
self._epoch_to_image = {}
self._image_to_epoch = {}
# to allow first, last image for processing to be
# set... c/f integrater interface
self._frames_to_process = frames_to_process
# + derive template, list of images
params = PhilIndex.get_python_object()
if directory and image:
self._template, self._directory = image2template_directory(
os.path.join(directory, image))
from xia2.Schema import load_imagesets
imagesets = load_imagesets(
self._template,
self._directory,
image_range=self._frames_to_process,
reversephi=(params.xia2.settings.input.reverse_phi
or self._reversephi),
)
assert len(
imagesets
) == 1, "one imageset expected, %d found" % len(imagesets)
self._imageset = copy.deepcopy(imagesets[0])
start, end = self._imageset.get_array_range()
self._images = list(range(start + 1, end + 1))
# FIXME in here check that (1) the list of images is continuous
# and (2) that all of the images are readable. This should also
# take into account frames_to_process if set.
if self._frames_to_process is None:
self._frames_to_process = min(self._images), max(self._images)
start, end = self._frames_to_process
error = False
if params.general.check_image_files_readable:
for j in range(start, end + 1):
if j not in self._images:
logger.debug("image %i missing for %s" %
(j, self.get_imageset().get_template()))
error = True
continue
image_name = self.get_imageset().get_path(j - start)
if not os.access(image_name, os.R_OK):
logger.debug("image %s unreadable" % image_name)
error = True
continue
if error:
raise RuntimeError("problem with sweep %s" % self._name)
beam_ = self._imageset.get_beam()
scan = self._imageset.get_scan()
if wavelength is not None:
# If the wavelength value is 0.0 then first set it to the header
# value - note that this assumes that the header value is correct
# (a reasonable assumption)
if wavelength.get_wavelength() == 0.0:
wavelength.set_wavelength(beam_.get_wavelength())
# FIXME 08/DEC/06 in here need to allow for the fact
# that the wavelength in the image header could be wrong and
# in fact it should be replaced with the input value -
# through the user will need to be warned of this and
# also everything using the FrameProcessor interface
# will also have to respect this!
if (math.fabs(beam_.get_wavelength() -
wavelength.get_wavelength()) > 0.0001):
logger.info(
"Header wavelength for sweep %s different"
" to assigned value (%4.2f vs. %4.2f)",
name,
beam_.get_wavelength(),
wavelength.get_wavelength(),
)
# also in here look at the image headers to see if we can
# construct a mapping between exposure epoch and image ...
images = []
if self._frames_to_process:
start, end = self._frames_to_process
for j in self._images:
if j >= start and j <= end:
images.append(j)
else:
images = self._images
for j in images:
epoch = scan.get_image_epoch(j)
if epoch == 0.0:
epoch = float(
os.stat(self._imageset.get_path(j -
images[0])).st_mtime)
self._epoch_to_image[epoch] = j
self._image_to_epoch[j] = epoch
epochs = self._epoch_to_image
logger.debug("Exposure epoch for sweep %s: %d %d" %
(self._template, min(epochs), max(epochs)))
self._input_imageset = copy.deepcopy(self._imageset)
# + get the lattice - can this be a pointer, so that when
# this object updates lattice it is globally-for-this-crystal
# updated? The lattice included directly in here includes an
# exact unit cell for data reduction, the crystal lattice
# contains an approximate unit cell which should be
# from the unit cells from all sweeps contained in the
# XCrystal. FIXME should I be using a LatticeInfo object
# in here? See what the Indexer interface produces. ALT:
# just provide an Indexer implementation "hook".
# See Headnote 001 above. See also _get_indexer,
# _get_integrater below.
self._indexer = None
self._refiner = None
self._integrater = None
# I don't need this - it is equivalent to self.getWavelength(
# ).getCrystal().getLattice()
# self._crystal_lattice = None
# this means that this module will have to present largely the
# same interface as Indexer and Integrater so that the calls
# can be appropriately forwarded.
# finally configure the beam if set
if beam is not None:
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
try:
set_mosflm_beam_centre(
self.get_imageset().get_detector(),
self.get_imageset().get_beam(),
beam,
)
except AssertionError as e:
logger.debug("Error setting mosflm beam centre: %s" % e)
self._beam_centre = beam
self._distance = distance
self._gain = gain
self._polarization = polarization
self._add_detector_identification_to_cif()
def xia2_main(stop_after=None):
'''Actually process something...'''
Citations.cite('xia2')
# print versions of related software
Chatter.write(dials_version())
ccp4_version = get_ccp4_version()
if ccp4_version is not None:
Chatter.write('CCP4 %s' % ccp4_version)
start_time = time.time()
CommandLine = get_command_line()
start_dir = Flags.get_starting_directory()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
no_images = True
for name in xtals.keys():
xtal = xtals[name]
if not xtal.get_all_image_names():
Chatter.write('-----------------------------------' + \
'-' * len(name))
Chatter.write('| No images assigned for crystal %s |' % name)
Chatter.write('-----------------------------------' + '-' \
* len(name))
else:
no_images = False
args = []
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
from libtbx import group_args
xinfo = CommandLine.get_xinfo()
if os.path.exists('xia2.json'):
from xia2.Schema.XProject import XProject
xinfo_new = xinfo
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new.keys():
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names(
):
wavelength_new = crystals_new[crystal_id].get_xwavelength(
wavelength_id)
if wavelength_id not in crystals[
crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(
wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = [s.get_name() for s in sweeps]
sweep_keys = [(s.get_directory(), s.get_template(),
s.get_image_range()) for s in sweeps]
for sweep in sweeps_new:
if ((sweep.get_directory(), sweep.get_template(),
sweep.get_image_range()) not in sweep_keys):
if sweep.get_name() in sweep_names:
i = 1
while 'SWEEEP%i' % i in sweep_names:
i += 1
sweep._name = 'SWEEP%i' % i
break
wavelength.add_sweep(
name=sweep.get_name(),
sample=sweep.get_xsample(),
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.append(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
if mp_params.mode == 'parallel' and njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
args.append((group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
), ))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(args):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
if mp_params.type == "qsub":
method = "sge"
else:
method = "multiprocessing"
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command
if not qsub_command:
qsub_command = 'qsub'
qsub_command = '%s -V -cwd -pe smp %d' % (qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep,
args,
processes=njob,
#method=method,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
if output is not None:
Chatter.write(output)
if not success:
Chatter.write('Sweep failed: removing %s' %
sweep.get_name())
remove_sweeps.append(sweep)
else:
assert xsweep_dict is not None
Chatter.write('Loading sweep: %s' % sweep.get_name())
from xia2.Schema.XSweep import XSweep
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
else:
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
from dials.command_line.show import show_datablocks
from dxtbx.datablock import DataBlock
Debug.write(sweep.get_name())
Debug.write(
show_datablocks([DataBlock([sweep.get_imageset()])]))
try:
if stop_after == 'index':
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception as e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(sweep.get_name(), str(e)))
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
# save intermediate xia2.json file in case scaling step fails
xinfo.as_json(filename='xia2.json')
if stop_after not in ('index', 'integrate'):
Chatter.write(xinfo.get_output(), strip=False)
for crystal in crystals.values():
crystal.serialize()
# save final xia2.json file in case report generation fails
xinfo.as_json(filename='xia2.json')
duration = time.time() - start_time
# write out the time taken in a human readable way
Chatter.write('Processing took %s' % \
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
if stop_after not in ('index', 'integrate'):
# and the summary file
with open('xia2-summary.dat', 'w') as fh:
for record in xinfo.summarise():
fh.write('%s\n' % record)
# looks like this import overwrites the initial command line
# Phil overrides so... for https://github.com/xia2/xia2/issues/150
from xia2.command_line.html import generate_xia2_html
if params.xia2.settings.small_molecule == True:
params.xia2.settings.report.xtriage_analysis = False
params.xia2.settings.report.include_radiation_damage = False
generate_xia2_html(xinfo,
filename='xia2.html',
params=params.xia2.settings.report)
write_citations()
# delete all of the temporary mtz files...
cleanup()
Environment.cleanup()
def _scale_prepare(self):
'''Prepare the data for scaling - this will reindex it the
reflections to the correct pointgroup and setting, for instance,
and move the reflection files to the scale directory.'''
Citations.cite('xds')
Citations.cite('ccp4')
Citations.cite('pointless')
# GATHER phase - get the reflection files together... note that
# it is not necessary in here to keep the batch information as we
# don't wish to rebatch the reflections prior to scaling.
# FIXME need to think about what I will do about the radiation
# damage analysis in here...
self._sweep_information = { }
# FIXME in here I want to record the batch number to
# epoch mapping as per the CCP4 Scaler implementation.
Journal.block(
'gathering', self.get_scaler_xcrystal().get_name(), 'XDS',
{'working directory':self.get_working_directory()})
for epoch in self._scalr_integraters.keys():
intgr = self._scalr_integraters[epoch]
pname, xname, dname = intgr.get_integrater_project_info()
sname = intgr.get_integrater_sweep_name()
self._sweep_information[epoch] = {
'pname':pname,
'xname':xname,
'dname':dname,
'integrater':intgr,
'corrected_intensities':intgr.get_integrater_corrected_intensities(),
'prepared_reflections':None,
'scaled_reflections':None,
'header':intgr.get_header(),
'batches':intgr.get_integrater_batches(),
'image_to_epoch':intgr.get_integrater_sweep(
).get_image_to_epoch(),
'image_to_dose':{},
'batch_offset':0,
'sname':sname
}
Journal.entry({'adding data from':'%s/%s/%s' % \
(xname, dname, sname)})
# what are these used for?
# pname / xname / dname - dataset identifiers
# image to epoch / batch offset / batches - for RD analysis
Debug.write('For EPOCH %s have:' % str(epoch))
Debug.write('ID = %s/%s/%s' % (pname, xname, dname))
Debug.write('SWEEP = %s' % intgr.get_integrater_sweep_name())
# next work through all of the reflection files and make sure that
# they are XDS_ASCII format...
epochs = self._sweep_information.keys()
epochs.sort()
self._first_epoch = min(epochs)
self._scalr_pname = self._sweep_information[epochs[0]]['pname']
self._scalr_xname = self._sweep_information[epochs[0]]['xname']
for epoch in epochs:
intgr = self._scalr_integraters[epoch]
pname = self._sweep_information[epoch]['pname']
xname = self._sweep_information[epoch]['xname']
dname = self._sweep_information[epoch]['dname']
sname = self._sweep_information[epoch]['sname']
if self._scalr_pname != pname:
raise RuntimeError, 'all data must have a common project name'
xname = self._sweep_information[epoch]['xname']
if self._scalr_xname != xname:
raise RuntimeError, \
'all data for scaling must come from one crystal'
xsh = XDSScalerHelper()
xsh.set_working_directory(self.get_working_directory())
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_CORRECTED.HKL' %(
pname, xname, dname, sname))
sweep = intgr.get_integrater_sweep()
if sweep.get_frames_to_process() is not None:
offset = intgr.get_frame_offset()
#print "offset: %d" %offset
start, end = sweep.get_frames_to_process()
start -= offset
end -= offset
#end += 1 ????
#print "limiting batches: %d-%d" %(start, end)
xsh.limit_batches(hklin, hklout, start, end)
self._sweep_information[epoch]['corrected_intensities'] = hklout
# if there is more than one sweep then compare the lattices
# and eliminate all but the lowest symmetry examples if
# there are more than one...
# -------------------------------------------------
# Ensure that the integration lattices are the same
# -------------------------------------------------
need_to_return = False
if len(self._sweep_information.keys()) > 1:
lattices = []
# FIXME run this stuff in parallel as well...
for epoch in self._sweep_information.keys():
intgr = self._sweep_information[epoch]['integrater']
hklin = self._sweep_information[epoch]['corrected_intensities']
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
ntr = False
else:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattice = Syminfo.get_lattice(pointgroup)
if not lattice in lattices:
lattices.append(lattice)
if ntr:
# if we need to return, we should logically reset
# any reindexing operator right? right here all
# we are talking about is the correctness of
# individual pointgroups?? Bug # 3373
reindex_op = 'h,k,l'
# actually, should this not be done "by magic"
# when a new pointgroup is assigned in the
# pointless indexer jiffy above?!
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
need_to_return = True
# bug # 2433 - need to ensure that all of the lattice
# conclusions were the same...
if len(lattices) > 1:
ordered_lattices = []
for l in lattices_in_order():
if l in lattices:
ordered_lattices.append(l)
correct_lattice = ordered_lattices[0]
Debug.write('Correct lattice asserted to be %s' % \
correct_lattice)
# transfer this information back to the indexers
for epoch in self._sweep_information.keys():
integrater = self._sweep_information[
epoch]['integrater']
refiner = integrater.get_integrater_refiner()
sname = integrater.get_integrater_sweep_name()
if not refiner:
continue
state = refiner.set_refiner_asserted_lattice(
correct_lattice)
if state == refiner.LATTICE_CORRECT:
Debug.write('Lattice %s ok for sweep %s' % \
(correct_lattice, sname))
elif state == refiner.LATTICE_IMPOSSIBLE:
raise RuntimeError, 'Lattice %s impossible for %s' % \
(correct_lattice, sname)
elif state == refiner.LATTICE_POSSIBLE:
Debug.write('Lattice %s assigned for sweep %s' % \
(correct_lattice, sname))
need_to_return = True
# if one or more of them was not in the lowest lattice,
# need to return here to allow reprocessing
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# next if there is more than one sweep then generate
# a merged reference reflection file to check that the
# setting for all reflection files is the same...
# if we get to here then all data was processed with the same
# lattice
# ----------------------------------------------------------
# next ensure that all sweeps are set in the correct setting
# ----------------------------------------------------------
if self.get_scaler_reference_reflection_file():
self._reference = self.get_scaler_reference_reflection_file()
Debug.write('Using HKLREF %s' % self._reference)
md = self._factory.Mtzdump()
md.set_hklin(self.get_scaler_reference_reflection_file())
md.dump()
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(
md.get_spacegroup())
Debug.write('Spacegroup %d' % self._xds_spacegroup)
elif PhilIndex.params.xia2.settings.scale.reference_reflection_file:
self._reference = PhilIndex.params.xia2.settings.scale.reference_reflection_file
Debug.write('Using HKLREF %s' % self._reference)
md = self._factory.Mtzdump()
md.set_hklin(PhilIndex.params.xia2.settings.scale.reference_reflection_file)
md.dump()
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(
md.get_spacegroup())
Debug.write('Spacegroup %d' % self._xds_spacegroup)
params = PhilIndex.params
use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs
if len(self._sweep_information.keys()) > 1 and use_brehm_diederichs:
brehm_diederichs_files_in = []
for epoch in self._sweep_information.keys():
intgr = self._sweep_information[epoch]['integrater']
hklin = self._sweep_information[epoch]['corrected_intensities']
refiner = intgr.get_integrater_refiner()
# in here need to consider what to do if the user has
# assigned the pointgroup on the command line ...
if not self._scalr_input_pointgroup:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
if ntr:
# Bug # 3373
Debug.write('Reindex to standard (PIJ): %s' % \
reindex_op)
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
else:
# 27/FEB/08 to support user assignment of pointgroups
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
self._sweep_information[epoch]['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# convert the XDS_ASCII for this sweep to mtz - on the next
# get this should be in the correct setting...
dname = self._sweep_information[epoch]['dname']
sname = intgr.get_integrater_sweep_name()
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.mtz' % (dname, sname))
FileHandler.record_temporary_file(hklout)
# now use pointless to make this conversion
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
brehm_diederichs_files_in.append(hklout)
# now run cctbx.brehm_diederichs to figure out the indexing hand for
# each sweep
from xia2.Wrappers.Cctbx.BrehmDiederichs import BrehmDiederichs
brehm_diederichs = BrehmDiederichs()
brehm_diederichs.set_working_directory(self.get_working_directory())
auto_logfiler(brehm_diederichs)
brehm_diederichs.set_input_filenames(brehm_diederichs_files_in)
# 1 or 3? 1 seems to work better?
brehm_diederichs.set_asymmetric(1)
brehm_diederichs.run()
reindexing_dict = brehm_diederichs.get_reindexing_dict()
for epoch in self._sweep_information.keys():
intgr = self._sweep_information[epoch]['integrater']
dname = self._sweep_information[epoch]['dname']
sname = intgr.get_integrater_sweep_name()
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.mtz' % (dname, sname))
# apply the reindexing operator
intgr.set_integrater_reindex_operator(reindex_op)
# and copy the reflection file to the local directory
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
self._sweep_information[epoch][
'prepared_reflections'] = os.path.split(hklout)[-1]
elif len(self._sweep_information.keys()) > 1 and \
not self._reference:
# need to generate a reference reflection file - generate this
# from the reflections in self._first_epoch
#
# FIXME this should really use the Brehm and Diederichs method
# if you have lots of little sweeps...
intgr = self._sweep_information[self._first_epoch]['integrater']
hklin = self._sweep_information[epoch]['corrected_intensities']
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
ntr = False
reindex_op = 'h,k,l'
else:
pointgroup, reindex_op, ntr = self._pointless_indexer_jiffy(
hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
reference_reindex_op = intgr.get_integrater_reindex_operator()
if ntr:
# Bug # 3373
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(pointgroup)
# next pass this reindexing operator back to the source
# of the reflections
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
self._sweep_information[epoch]['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'xds-pointgroup-reference-unsorted.mtz')
FileHandler.record_temporary_file(hklout)
# now use pointless to handle this conversion
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
self._reference = hklout
if self._reference:
from xia2.Driver.DriverFactory import DriverFactory
def run_one_sweep(args):
sweep_information = args[0]
pointless_indexer_jiffy = args[1]
factory = args[2]
job_type = args[3]
if job_type:
DriverFactory.set_driver_type(job_type)
intgr = sweep_information['integrater']
hklin = sweep_information['corrected_intensities']
refiner = intgr.get_integrater_refiner()
# in here need to consider what to do if the user has
# assigned the pointgroup on the command line ...
if not self._scalr_input_pointgroup:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
if ntr:
# Bug # 3373
Debug.write('Reindex to standard (PIJ): %s' % \
reindex_op)
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
else:
# 27/FEB/08 to support user assignment of pointgroups
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
sweep_information['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# convert the XDS_ASCII for this sweep to mtz - on the next
# get this should be in the correct setting...
hklin = sweep_information['corrected_intensities']
# now use pointless to make this conversion
# try with no conversion?!
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
hklout = os.path.join(
self.get_working_directory(),
'%d_xds-pointgroup-unsorted.mtz' %pointless.get_xpid())
FileHandler.record_temporary_file(hklout)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
pointless = self._factory.Pointless()
pointless.set_hklin(hklout)
pointless.set_hklref(self._reference)
pointless.decide_pointgroup()
pointgroup = pointless.get_pointgroup()
reindex_op = pointless.get_reindex_operator()
# for debugging print out the reindexing operations and
# what have you...
Debug.write('Reindex to standard: %s' % reindex_op)
# this should send back enough information that this
# is in the correct pointgroup (from the call above) and
# also in the correct setting, from the interaction
# with the reference set... - though I guess that the
# spacegroup number should not have changed, right?
# set the reindex operation afterwards... though if the
# spacegroup number is the same this should make no
# difference, right?!
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
intgr.set_integrater_reindex_operator(reindex_op)
sweep_information['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# and copy the reflection file to the local directory
dname = sweep_information['dname']
sname = intgr.get_integrater_sweep_name()
hklin = sweep_information['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
sweep_information['prepared_reflections'] = os.path.split(hklout)[-1]
return sweep_information
from libtbx import easy_mp
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
if njob > 1:
# cache drivertype
drivertype = DriverFactory.get_driver_type()
args = [
(self._sweep_information[epoch], self._pointless_indexer_jiffy,
self._factory, mp_params.type)
for epoch in self._sweep_information.keys()]
results_list = easy_mp.parallel_map(
run_one_sweep, args, params=None,
processes=njob,
method="threading",
asynchronous=True,
callback=None,
preserve_order=True,
preserve_exception_message=True)
# restore drivertype
DriverFactory.set_driver_type(drivertype)
# results should be given back in the same order
for i, epoch in enumerate(self._sweep_information.keys()):
self._sweep_information[epoch] = results_list[i]
else:
for epoch in self._sweep_information.keys():
self._sweep_information[epoch] = run_one_sweep(
(self._sweep_information[epoch], self._pointless_indexer_jiffy,
self._factory, None))
else:
# convert the XDS_ASCII for this sweep to mtz
epoch = self._first_epoch
intgr = self._sweep_information[epoch]['integrater']
refiner = intgr.get_integrater_refiner()
sname = intgr.get_integrater_sweep_name()
hklout = os.path.join(self.get_working_directory(),
'%s-pointless.mtz' % sname)
FileHandler.record_temporary_file(hklout)
pointless = self._factory.Pointless()
pointless.set_xdsin(self._sweep_information[epoch]['corrected_intensities'])
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
# run it through pointless interacting with the
# Indexer which belongs to this sweep
hklin = hklout
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
ntr = False
reindex_op = 'h,k,l'
else:
pointgroup, reindex_op, ntr = self._pointless_indexer_jiffy(
hklin, refiner)
if ntr:
# if we need to return, we should logically reset
# any reindexing operator right? right here all
# we are talking about is the correctness of
# individual pointgroups?? Bug # 3373
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
need_to_return = True
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(pointgroup)
# next pass this reindexing operator back to the source
# of the reflections
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
self._sweep_information[epoch]['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
hklin = self._sweep_information[epoch]['corrected_intensities']
dname = self._sweep_information[epoch]['dname']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
# and copy the reflection file to the local
# directory
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
self._sweep_information[epoch][
'prepared_reflections'] = os.path.split(hklout)[-1]
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
unit_cell_list = []
for epoch in self._sweep_information.keys():
integrater = self._sweep_information[epoch]['integrater']
cell = integrater.get_integrater_cell()
n_ref = integrater.get_integrater_n_ref()
Debug.write('Cell for %s: %.2f %.2f %.2f %.2f %.2f %.2f' % \
(integrater.get_integrater_sweep_name(),
cell[0], cell[1], cell[2],
cell[3], cell[4], cell[5]))
Debug.write('=> %d reflections' % n_ref)
unit_cell_list.append((cell, n_ref))
self._scalr_cell = compute_average_unit_cell(unit_cell_list)
self._scalr_resolution_limits = { }
Debug.write('Determined unit cell: %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(self._scalr_cell))
if os.path.exists(os.path.join(
self.get_working_directory(),
'REMOVE.HKL')):
os.remove(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'))
Debug.write('Deleting REMOVE.HKL at end of scale prepare.')
return
def run():
try:
check_environment()
except Exception as e:
traceback.print_exc(file=open('xia2.error', 'w'))
Chatter.write('Status: error "%s"' % str(e))
if len(sys.argv) < 2 or '-help' in sys.argv:
help()
sys.exit()
cwd = os.getcwd()
try:
from xia2.command_line.xia2_main import xia2_main
xia2_main(stop_after='integrate')
Chatter.write('Status: normal termination')
wd = os.path.join(cwd, 'strategy')
if not os.path.exists(wd):
os.mkdir(wd)
os.chdir(wd)
CommandLine = get_command_line()
xinfo = CommandLine.get_xinfo()
crystals = xinfo.get_crystals()
assert len(crystals) == 1
crystal = crystals.values()[0]
assert len(crystal.get_wavelength_names()) == 1
wavelength = crystal.get_xwavelength(crystal.get_wavelength_names()[0])
sweeps = wavelength.get_sweeps()
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
strategy_params = params.strategy
if not len(strategy_params):
strategy_params = [
PhilIndex.get_scope_by_name('strategy')[0].extract()
]
from dxtbx.model import MultiAxisGoniometer
gonio = sweeps[0].get_imageset().get_goniometer()
if (isinstance(gonio, MultiAxisGoniometer)
and len(gonio.get_axes()) == 3 and gonio.get_scan_axis() == 2):
from xia2.Wrappers.Dials.AlignCrystal import AlignCrystal
align_crystal = AlignCrystal()
align_crystal.set_experiments_filename(
sweeps[0]._get_integrater().get_integrated_experiments())
align_crystal.set_working_directory(wd)
auto_logfiler(align_crystal)
align_crystal.set_json_filename('%i_align_crystal.json' %
align_crystal.get_xpid())
align_crystal.run()
Chatter.write("".join(align_crystal.get_all_output()))
results_all = {}
def process_one_strategy(args):
assert len(args) == 4
experiments, reflections, strategy, t_ref = args
from xia2.Wrappers.EMBL import Best
best = Best.BestStrategy()
for isweep, (expt, refl) in enumerate(zip(experiments,
reflections)):
integrater = sweep._get_integrater()
from xia2.Wrappers.Dials.ExportBest import ExportBest
export = ExportBest()
export.set_experiments_filename(expt)
export.set_reflections_filename(refl)
export.set_working_directory(wd)
auto_logfiler(export)
prefix = '%i_best' % export.get_xpid()
export.set_prefix(prefix)
export.run()
if isweep == 0:
imageset = sweep.get_imageset()
scan = imageset.get_scan()
best.set_t_ref(t_ref)
best.set_mos_dat('%s.dat' % prefix)
best.set_mos_par('%s.par' % prefix)
best.add_mos_hkl('%s.hkl' % prefix)
best.set_i2s(strategy.i_over_sigi)
best.set_T_max(strategy.max_total_exposure)
best.set_t_min(strategy.min_exposure)
#best.set_trans_ref(25.0)
best.set_S_max(strategy.max_rotation_speed)
best.set_w_min(strategy.min_oscillation_width)
best.set_M_min(strategy.multiplicity)
best.set_C_min(strategy.completeness)
best.set_GpS(strategy.dose_rate)
best.set_shape(strategy.shape)
best.set_susceptibility(strategy.susceptibility)
best.set_anomalous(strategy.anomalous)
best.set_detector('pilatus6m')
best.set_working_directory(wd)
auto_logfiler(best)
xmlout = '%s/%i_best.xml' % (best.get_working_directory(),
best.get_xpid())
best.set_xmlout(xmlout)
best.strategy()
results = best.get_results_dict()
results['description'] = strategy.description
if 'phi_end' not in results:
results['phi_end'] = str(
float(results['phi_start']) +
float(results['number_of_images']) *
float(results['phi_width']))
from dxtbx.serialize import load
expt = load.experiment_list(experiments[0])[0]
results['spacegroup'] = expt.crystal.get_space_group().type(
).lookup_symbol()
return results
args = []
for istrategy, strategy in enumerate(strategy_params):
imageset = sweeps[0].get_imageset()
scan = imageset.get_scan()
experiments = [
sweep._get_integrater().get_integrated_experiments()
for sweep in sweeps
]
reflections = [
sweep._get_integrater().get_integrated_reflections()
for sweep in sweeps
]
t_ref = scan.get_exposure_times()[0]
args.append((experiments, reflections, strategy, t_ref))
nproc = params.xia2.settings.multiprocessing.nproc
from libtbx import easy_mp
results = easy_mp.parallel_map(process_one_strategy,
args,
processes=nproc,
method='multiprocessing',
preserve_order=True,
preserve_exception_message=True)
for istrategy, (result,
strategy) in enumerate(zip(results, strategy_params)):
name = strategy.name
description = strategy.description
if name is None:
name = 'Strategy%i' % (istrategy + 1)
results_all[name] = result
multiplicity = result['redundancy']
try:
mutiplicity = '%.2f' % multiplicity
except TypeError:
pass
Chatter.write('Strategy %i' % istrategy)
if description is not None:
Chatter.write(description)
Chatter.write('Start / end / width: %.2f/%.2f/%.2f' %
(float(result['phi_start']), float(
result['phi_end']), float(result['phi_width'])))
Chatter.write(
'Completeness / multiplicity / resolution: %.2f/%s/%.2f' %
(float(result['completeness']), multiplicity,
float(result['resolution'])))
Chatter.write('Transmission / exposure %.3f/%.3f' % (float(
result['transmission']), float(result['exposure_time'])))
import json
with open('strategies.json', 'wb') as f:
json.dump(results_all, f, indent=2)
except Exception as e:
traceback.print_exc(file=open(os.path.join(cwd, 'xia2.error'), 'w'))
Chatter.write('Status: error "%s"' % str(e))
os.chdir(cwd)
def imageset_to_xds(
imageset,
synchrotron=None,
refined_beam_vector=None,
refined_rotation_axis=None,
refined_distance=None,
):
"""A function to take an input header dictionary from Diffdump
and generate a list of records to start XDS - see Doc/INP.txt."""
# decide if we are at a synchrotron if we don't know already...
# that is, the wavelength is around either the Copper or Chromium
# K-alpha edge and this is an image plate.
beam = imageset.get_beam()
from dxtbx.serialize.xds import to_xds, xds_detector_name
converter = to_xds(imageset)
h5_names = ["h5", "nxs"]
if imageset.get_template().split(".")[-1] in h5_names:
if not check_xds_ok_with_h5():
raise RuntimeError("HDF5 input with no converter for XDS")
detector_class_is_square = {
"adsc q4": True,
"adsc q4 2x2 binned": True,
"adsc q210": True,
"adsc q210 2x2 binned": True,
"adsc q270": True,
"adsc q270 2x2 binned": True,
"adsc q315": True,
"adsc q315 2x2 binned": True,
"adsc HF4M": True,
"holton fake 01": True,
"unknown electron 57": True,
"mar 345": False,
"mar 180": False,
"mar 240": False,
"mar 300 ccd": True,
"mar 325 ccd": True,
"mar 225 ccd": True,
"mar ccd 225 hs": True,
"rayonix ccd 165": False,
"rayonix ccd 135": False,
"rayonix ccd 300": True,
"rayonix ccd 325": True,
"rayonix ccd 225": True,
"rayonix ccd 225 hs": True,
"rayonix ccd 300 hs": True,
"mar 165 ccd": False,
"mar 135 ccd": False,
"pilatus 12M": True,
"pilatus 6M": True,
"pilatus 2M": True,
"pilatus 1M": True,
"pilatus 200K": True,
"pilatus 300K": True,
"eiger 4M": True,
"eiger 9M": True,
"eiger 16M": True,
"rigaku saturn 92 2x2 binned": True,
"rigaku saturn 944 2x2 binned": True,
"rigaku saturn 724 2x2 binned": True,
"rigaku saturn 92": True,
"rigaku saturn 944": True,
"rigaku saturn 724": True,
"rigaku saturn a200": True,
"raxis IV": True,
"NOIR1": True,
}
sensor = converter.get_detector()[0].get_type()
fast, slow = converter.detector_size
f, s = converter.pixel_size
df = int(1000 * f)
ds = int(1000 * s)
# FIXME probably need to rotate by pi about the X axis
result = []
from dxtbx.model.detector_helpers_types import detector_helpers_types
detector = xds_detector_name(
detector_helpers_types.get(sensor, fast, slow, df, ds))
trusted = converter.get_detector()[0].get_trusted_range()
# if CCD; undo dxtbx pedestal offset, hard code minimum 1; else use trusted
# range verbatim (i.e. for PAD) (later in pipeline sensor is SENSOR_UNKNOWN
# so additional test)
if sensor == "SENSOR_CCD" or detector == "CCDCHESS":
trusted = 1, trusted[1] - trusted[0]
# XDS upset if we trust < 0 see #193
if trusted[0] < 0:
trusted = 0, trusted[1]
result.append("DETECTOR=%s MINIMUM_VALID_PIXEL_VALUE=%d OVERLOAD=%d" %
(detector, trusted[0], trusted[1]))
result.append("DIRECTION_OF_DETECTOR_X-AXIS=%f %f %f" %
converter.detector_x_axis)
result.append("DIRECTION_OF_DETECTOR_Y-AXIS=%f %f %f" %
converter.detector_y_axis)
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
if params.xds.trusted_region:
result.append("TRUSTED_REGION= %.2f %.2f" %
tuple(params.xds.trusted_region))
elif detector_class_is_square[detector_helpers_types.get(
sensor, fast, slow, df, ds).replace("-", " ")]:
result.append("TRUSTED_REGION=0.0 1.41")
else:
result.append("TRUSTED_REGION=0.0 0.99")
result.append("NX=%d NY=%d QX=%.4f QY=%.4f" % (fast, slow, f, s))
# RAXIS detectors have the distance written negative - why????
# this is ONLY for XDS - SATURN are the same - probably left handed
# goniometer rotation on rigaku X-ray sets.
if refined_distance:
result.append("DETECTOR_DISTANCE=%7.3f" % refined_distance)
else:
result.append("DETECTOR_DISTANCE=%7.3f" % converter.detector_distance)
result.append("OSCILLATION_RANGE=%4.2f" % converter.oscillation_range)
result.append("X-RAY_WAVELENGTH=%8.6f" % converter.wavelength)
# if user specified reversephi and this was not picked up in the
# format class reverse phi: n.b. double-negative warning!
if refined_rotation_axis:
result.append("ROTATION_AXIS= %f %f %f" % refined_rotation_axis)
else:
result.append("ROTATION_AXIS= %.3f %.3f %.3f" %
converter.rotation_axis)
if refined_beam_vector:
result.append("INCIDENT_BEAM_DIRECTION=%f %f %f" % refined_beam_vector)
else:
result.append("INCIDENT_BEAM_DIRECTION= %.3f %.3f %.3f" %
converter.beam_vector)
if hasattr(beam, "get_polarization_fraction"):
R = converter.imagecif_to_xds_transformation_matrix
result.append("FRACTION_OF_POLARIZATION= %.3f" %
beam.get_polarization_fraction())
result.append("POLARIZATION_PLANE_NORMAL= %.3f %.3f %.3f" %
(R * matrix.col(beam.get_polarization_normal())).elems)
# 24/NOV/14 XDS determines the air absorption automatically
# based on wavelength. May be useful to override this for in vacuo exps
# result.append('AIR=0.001')
if detector == "PILATUS":
try:
thickness = converter.get_detector()[0].get_thickness()
if not thickness:
thickness = 0.32
Debug.write(
"Could not determine sensor thickness. Assuming default PILATUS 0.32mm"
)
except e:
thickness = 0.32
Debug.write(
"Error occured during sensor thickness determination. Assuming default PILATUS 0.32mm"
)
result.append("SENSOR_THICKNESS=%f" % thickness)
# FIXME: Sensor absorption coefficient calculation probably requires a more general solution
# if converter.get_detector()[0].get_material() == 'CdTe':
# print "CdTe detector detected. Beam wavelength is %8.6f Angstrom" % converter.wavelength
if len(converter.panel_x_axis) > 1:
for panel_id in range(len(converter.panel_x_axis)):
result.append("")
result.append("!")
result.append("! SEGMENT %d" % (panel_id + 1))
result.append("!")
result.append("SEGMENT= %d %d %d %d" %
converter.panel_limits[panel_id])
result.append("DIRECTION_OF_SEGMENT_X-AXIS= %.3f %.3f %.3f" %
converter.panel_x_axis[panel_id])
result.append("DIRECTION_OF_SEGMENT_Y-AXIS= %.3f %.3f %.3f" %
converter.panel_y_axis[panel_id])
result.append("SEGMENT_DISTANCE= %.3f" %
converter.panel_distance[panel_id])
result.append("SEGMENT_ORGX= %.1f SEGMENT_ORGY= %.1f" %
converter.panel_origin[panel_id])
result.append("")
for f0, s0, f1, s1 in converter.get_detector()[0].get_mask():
result.append("UNTRUSTED_RECTANGLE= %d %d %d %d" %
(f0 - 1, f1 + 1, s0 - 1, s1 + 1))
if params.xds.untrusted_ellipse:
for untrusted_ellipse in params.xds.untrusted_ellipse:
result.append("UNTRUSTED_ELLIPSE= %d %d %d %d" %
tuple(untrusted_ellipse))
Debug.write(result[-1])
if params.xds.untrusted_rectangle:
for untrusted_rectangle in params.xds.untrusted_rectangle:
result.append("UNTRUSTED_RECTANGLE= %d %d %d %d" %
tuple(untrusted_rectangle))
Debug.write(result[-1])
return result
if not os.path.exists(wd):
os.mkdir(wd)
os.chdir(wd)
CommandLine = get_command_line()
xinfo = CommandLine.get_xinfo()
crystals = xinfo.get_crystals()
assert len(crystals) == 1
crystal = crystals.values()[0]
assert len(crystal.get_wavelength_names()) == 1
wavelength = crystal.get_xwavelength(crystal.get_wavelength_names()[0])
sweeps = wavelength.get_sweeps()
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
strategy_params = params.strategy
if not len(strategy_params):
strategy_params = [PhilIndex.get_scope_by_name('strategy')[0].extract()]
from dxtbx.model import MultiAxisGoniometer
gonio = sweeps[0].get_imageset().get_goniometer()
if (isinstance(gonio, MultiAxisGoniometer) and
len(gonio.get_axes()) == 3 and gonio.get_scan_axis() == 2):
from xia2.Wrappers.Dials.AlignCrystal import AlignCrystal
align_crystal = AlignCrystal()
align_crystal.set_experiments_filename(
sweeps[0]._get_integrater().get_integrated_experiments())
align_crystal.set_working_directory(wd)
auto_logfiler(align_crystal)
align_crystal.set_json_filename(
def xia2_main(stop_after=None):
'''Actually process something...'''
Citations.cite('xia2')
# print versions of related software
from dials.util.version import dials_version
Chatter.write(dials_version())
start_time = time.time()
CommandLine = get_command_line()
start_dir = Flags.get_starting_directory()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
no_images = True
for name in xtals.keys():
xtal = xtals[name]
if not xtal.get_all_image_names():
Chatter.write('-----------------------------------' + \
'-' * len(name))
Chatter.write('| No images assigned for crystal %s |' % name)
Chatter.write('-----------------------------------' + '-' \
* len(name))
else:
no_images = False
args = []
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
from libtbx import group_args
xinfo = CommandLine.get_xinfo()
if os.path.exists('xia2.json'):
from xia2.Schema.XProject import XProject
xinfo_new = xinfo
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new.keys():
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names():
wavelength_new = crystals_new[crystal_id].get_xwavelength(wavelength_id)
if wavelength_id not in crystals[crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = [s.get_name() for s in sweeps]
sweep_keys = [
(s.get_directory(), s.get_template(), s.get_image_range())
for s in sweeps]
for sweep in sweeps_new:
if ((sweep.get_directory(), sweep.get_template(),
sweep.get_image_range()) not in sweep_keys):
if sweep.get_name() in sweep_names:
i = 1
while 'SWEEEP%i' %i in sweep_names:
i += 1
sweep._name = 'SWEEP%i' %i
break
wavelength.add_sweep(
name=sweep.get_name(),
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.append(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
if njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
args.append((
group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
),))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(args):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
if mp_params.type == "qsub":
method = "sge"
else:
method = "multiprocessing"
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command
if not qsub_command:
qsub_command = 'qsub'
qsub_command = '%s -V -cwd -pe smp %d' %(qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep, args, processes=njob,
#method=method,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
assert xsweep_dict is not None
if output is not None:
Chatter.write(output)
if not success:
Chatter.write('Sweep failed: removing %s' %sweep.get_name())
remove_sweeps.append(sweep)
else:
Chatter.write('Loading sweep: %s' % sweep.get_name())
from xia2.Schema.XSweep import XSweep
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
else:
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
try:
if stop_after == 'index':
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception, e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(sweep.get_name(), str(e)))
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
def setup(self):
'''Set everything up...'''
# check arguments are all ascii
Debug.write('Start parsing command line: ' + str(sys.argv))
for token in sys.argv:
try:
token.encode('ascii')
except UnicodeDecodeError:
raise RuntimeError('non-ascii characters in input')
self._argv = copy.deepcopy(sys.argv)
replacements = {
'-2d': 'pipeline=2d',
'-2di': 'pipeline=2di',
'-3d': 'pipeline=3d',
'-3di': 'pipeline=3di',
'-3dii': 'pipeline=3dii',
'-3dd': 'pipeline=3dd',
'-dials': 'pipeline=dials',
'-quick': 'dials.fast_mode=true',
'-failover': 'failover=true',
'-small_molecule': 'small_molecule=true'
}
for k, v in replacements.iteritems():
if k in self._argv:
print "***\nCommand line option %s is deprecated.\nPlease use %s instead\n***" % (
k, v)
self._argv[self._argv.index(k)] = v
if '-atom' in self._argv:
idx = self._argv.index('-atom')
element = self._argv[idx + 1]
self._argv[idx:idx + 2] = ['atom=%s' % element]
print "***\nCommand line option -atom %s is deprecated.\nPlease use atom=%s instead\n***" % (
element, element)
# first of all try to interpret arguments as phil parameters/files
from xia2.Handlers.Phil import master_phil
from libtbx.phil import command_line
cmd_line = command_line.argument_interpreter(master_phil=master_phil)
working_phil, self._argv = cmd_line.process_and_fetch(
args=self._argv, custom_processor="collect_remaining")
PhilIndex.merge_phil(working_phil)
try:
params = PhilIndex.get_python_object()
except RuntimeError as e:
raise Sorry(e)
# sanity check / interpret Auto in input
from libtbx import Auto
if params.xia2.settings.input.atom is None:
if params.xia2.settings.input.anomalous is Auto:
PhilIndex.update("xia2.settings.input.anomalous=false")
else:
if params.xia2.settings.input.anomalous == False:
raise Sorry(
'Setting anomalous=false and atom type inconsistent')
params.xia2.settings.input.anomalous = True
PhilIndex.update("xia2.settings.input.anomalous=true")
if params.xia2.settings.resolution.keep_all_reflections is Auto:
if params.xia2.settings.small_molecule == True and \
params.xia2.settings.resolution.d_min is None and \
params.xia2.settings.resolution.d_max is None:
PhilIndex.update(
"xia2.settings.resolution.keep_all_reflections=true")
else:
PhilIndex.update(
"xia2.settings.resolution.keep_all_reflections=false")
if params.xia2.settings.small_molecule == True:
Debug.write('Small molecule selected')
if params.ccp4.pointless.chirality is None:
PhilIndex.update("ccp4.pointless.chirality=nonchiral")
params = PhilIndex.get_python_object()
# pipeline options
self._read_pipeline()
Debug.write('Project: %s' % params.xia2.settings.project)
Debug.write('Crystal: %s' % params.xia2.settings.crystal)
# FIXME add some consistency checks in here e.g. that there are
# images assigned, there is a lattice assigned if cell constants
# are given and so on
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
from xia2.Handlers.Environment import get_number_cpus
if mp_params.mode == 'parallel':
if mp_params.type == 'qsub':
if which('qsub') is None:
raise Sorry('qsub not available')
if mp_params.njob is Auto:
mp_params.njob = get_number_cpus()
if mp_params.nproc is Auto:
mp_params.nproc = 1
elif mp_params.nproc is Auto:
mp_params.nproc = get_number_cpus()
elif mp_params.mode == 'serial':
if mp_params.type == 'qsub':
if which('qsub') is None:
raise Sorry('qsub not available')
if mp_params.njob is Auto:
mp_params.njob = 1
if mp_params.nproc is Auto:
mp_params.nproc = get_number_cpus()
PhilIndex.update("xia2.settings.multiprocessing.njob=%d" %
mp_params.njob)
PhilIndex.update("xia2.settings.multiprocessing.nproc=%d" %
mp_params.nproc)
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
if params.xia2.settings.indexer is not None:
add_preference("indexer", params.xia2.settings.indexer)
if params.xia2.settings.refiner is not None:
add_preference("refiner", params.xia2.settings.refiner)
if params.xia2.settings.integrater is not None:
add_preference("integrater", params.xia2.settings.integrater)
if params.xia2.settings.scaler is not None:
add_preference("scaler", params.xia2.settings.scaler)
if params.xia2.settings.multi_sweep_indexing is Auto:
if params.xia2.settings.small_molecule == True and 'dials' == params.xia2.settings.indexer:
PhilIndex.update("xia2.settings.multi_sweep_indexing=True")
else:
PhilIndex.update("xia2.settings.multi_sweep_indexing=False")
if params.xia2.settings.multi_sweep_indexing == True and \
params.xia2.settings.multiprocessing.mode == 'parallel':
Chatter.write(
'Multi sweep indexing disabled:\nMSI is not available for parallel processing.'
)
PhilIndex.update("xia2.settings.multi_sweep_indexing=False")
input_json = params.xia2.settings.input.json
if (input_json is not None and len(input_json)):
for json_file in input_json:
assert os.path.isfile(json_file)
load_datablock(json_file)
reference_geometry = params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
reference_geometries = "\n".join([
"xia2.settings.input.reference_geometry=%s" %
os.path.abspath(g)
for g in params.xia2.settings.input.reference_geometry
])
Debug.write(reference_geometries)
PhilIndex.update(reference_geometries)
Debug.write("xia2.settings.trust_beam_centre=true")
PhilIndex.update("xia2.settings.trust_beam_centre=true")
params = PhilIndex.get_python_object()
params = PhilIndex.get_python_object()
if params.xia2.settings.input.xinfo is not None:
xinfo_file = os.path.abspath(params.xia2.settings.input.xinfo)
PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file)
params = PhilIndex.get_python_object()
self.set_xinfo(xinfo_file)
# issue #55 if not set ATOM in xinfo but anomalous=true or atom= set
# on commandline, set here, should be idempotent
if params.xia2.settings.input.anomalous is True:
crystals = self._xinfo.get_crystals()
for xname in crystals:
xtal = crystals[xname]
Debug.write("Setting anomalous for crystal %s" % xname)
xtal.set_anomalous(True)
else:
xinfo_file = '%s/automatic.xinfo' % os.path.abspath(os.curdir)
PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file)
params = PhilIndex.get_python_object()
if params.dials.find_spots.phil_file is not None:
PhilIndex.update(
"dials.find_spots.phil_file=%s" %
os.path.abspath(params.dials.find_spots.phil_file))
if params.dials.index.phil_file is not None:
PhilIndex.update("dials.index.phil_file=%s" %
os.path.abspath(params.dials.index.phil_file))
if params.dials.refine.phil_file is not None:
PhilIndex.update("dials.refine.phil_file=%s" %
os.path.abspath(params.dials.refine.phil_file))
if params.dials.integrate.phil_file is not None:
PhilIndex.update("dials.integrate.phil_file=%s" %
os.path.abspath(params.dials.integrate.phil_file))
if params.xds.index.xparm is not None:
Flags.set_xparm(params.xds.index.xparm)
if params.xds.index.xparm_ub is not None:
Flags.set_xparm_ub(params.xds.index.xparm_ub)
if params.xia2.settings.scale.freer_file is not None:
freer_file = os.path.abspath(params.xia2.settings.scale.freer_file)
if not os.path.exists(freer_file):
raise RuntimeError('%s does not exist' % freer_file)
from xia2.Modules.FindFreeFlag import FindFreeFlag
column = FindFreeFlag(freer_file)
Debug.write('FreeR_flag column in %s found: %s' % \
(freer_file, column))
PhilIndex.update("xia2.settings.scale.freer_file=%s" % freer_file)
if params.xia2.settings.scale.reference_reflection_file is not None:
reference_reflection_file = os.path.abspath(
params.xia2.settings.scale.reference_reflection_file)
if not os.path.exists(reference_reflection_file):
raise RuntimeError('%s does not exist' %
reference_reflection_file)
PhilIndex.update(
"xia2.settings.scale.reference_reflection_file=%s" %
reference_reflection_file)
params = PhilIndex.get_python_object()
datasets = unroll_datasets(PhilIndex.params.xia2.settings.input.image)
for dataset in datasets:
start_end = None
if ':' in dataset:
tokens = dataset.split(':')
# cope with windows drives i.e. C:\data\blah\thing_0001.cbf:1:100
if len(tokens[0]) == 1:
tokens = ['%s:%s' % (tokens[0], tokens[1])] + tokens[2:]
if len(tokens) != 3:
raise RuntimeError('/path/to/image_0001.cbf:start:end')
dataset = tokens[0]
start_end = int(tokens[1]), int(tokens[2])
from xia2.Applications.xia2setup import is_hd5f_name
if os.path.exists(os.path.abspath(dataset)):
dataset = os.path.abspath(dataset)
else:
directories = [os.getcwd()] + self._argv[1:]
found = False
for d in directories:
if os.path.exists(os.path.join(d, dataset)):
dataset = os.path.join(d, dataset)
found = True
break
if not found:
raise Sorry('Cound not find %s in %s' % \
(dataset, ' '.join(directories)))
if is_hd5f_name(dataset):
self._hdf5_master_files.append(dataset)
if start_end:
Debug.write('Image range: %d %d' % start_end)
if not dataset in self._default_start_end:
self._default_start_end[dataset] = []
self._default_start_end[dataset].append(start_end)
else:
Debug.write('No image range specified')
else:
template, directory = image2template_directory(
os.path.abspath(dataset))
self._default_template.append(os.path.join(
directory, template))
self._default_directory.append(directory)
Debug.write('Interpreted from image %s:' % dataset)
Debug.write('Template %s' % template)
Debug.write('Directory %s' % directory)
if start_end:
Debug.write('Image range: %d %d' % start_end)
key = os.path.join(directory, template)
if not key in self._default_start_end:
self._default_start_end[key] = []
self._default_start_end[key].append(start_end)
else:
Debug.write('No image range specified')
# finally, check that all arguments were read and raise an exception
# if any of them were nonsense.
with open('xia2-working.phil', 'wb') as f:
print >> f, PhilIndex.working_phil.as_str()
with open('xia2-diff.phil', 'wb') as f:
print >> f, PhilIndex.get_diff().as_str()
Debug.write('\nDifference PHIL:')
Debug.write(PhilIndex.get_diff().as_str(), strip=False)
Debug.write('Working PHIL:')
Debug.write(PhilIndex.working_phil.as_str(), strip=False)
nonsense = 'Unknown command-line options:'
was_nonsense = False
for j, argv in enumerate(self._argv):
if j == 0:
continue
if argv[0] != '-' and '=' not in argv:
continue
if not j in self._understood:
nonsense += ' %s' % argv
was_nonsense = True
if was_nonsense:
raise RuntimeError(nonsense)
def setup(self):
"""Set everything up..."""
# check arguments are all ascii
Debug.write("Start parsing command line: " + str(sys.argv))
for token in sys.argv:
try:
token.encode("ascii")
except UnicodeDecodeError:
raise RuntimeError("non-ascii characters in input")
self._argv = copy.deepcopy(sys.argv)
replacements = {
"-2d": "pipeline=2d",
"-2di": "pipeline=2di",
"-3d": "pipeline=3d",
"-3di": "pipeline=3di",
"-3dii": "pipeline=3dii",
"-3dd": "pipeline=3dd",
"-dials": "pipeline=dials",
"-quick": "dials.fast_mode=true",
"-failover": "failover=true",
"-small_molecule": "small_molecule=true",
}
for k, v in replacements.iteritems():
if k in self._argv:
print(
"***\nCommand line option %s is deprecated.\nPlease use %s instead\n***"
% (k, v))
self._argv[self._argv.index(k)] = v
if "-atom" in self._argv:
idx = self._argv.index("-atom")
element = self._argv[idx + 1]
self._argv[idx:idx + 2] = ["atom=%s" % element]
print(
"***\nCommand line option -atom %s is deprecated.\nPlease use atom=%s instead\n***"
% (element, element))
# first of all try to interpret arguments as phil parameters/files
from xia2.Handlers.Phil import master_phil
from libtbx.phil import command_line
cmd_line = command_line.argument_interpreter(master_phil=master_phil)
working_phil, self._argv = cmd_line.process_and_fetch(
args=self._argv, custom_processor="collect_remaining")
PhilIndex.merge_phil(working_phil)
try:
params = PhilIndex.get_python_object()
except RuntimeError as e:
raise Sorry(e)
# sanity check / interpret Auto in input
from libtbx import Auto
if params.xia2.settings.input.atom is None:
if params.xia2.settings.input.anomalous is Auto:
PhilIndex.update("xia2.settings.input.anomalous=false")
else:
if params.xia2.settings.input.anomalous is False:
raise Sorry(
"Setting anomalous=false and atom type inconsistent")
params.xia2.settings.input.anomalous = True
PhilIndex.update("xia2.settings.input.anomalous=true")
if params.xia2.settings.resolution.keep_all_reflections is Auto:
if (params.xia2.settings.small_molecule is True
and params.xia2.settings.resolution.d_min is None
and params.xia2.settings.resolution.d_max is None):
PhilIndex.update(
"xia2.settings.resolution.keep_all_reflections=true")
else:
PhilIndex.update(
"xia2.settings.resolution.keep_all_reflections=false")
if params.xia2.settings.small_molecule is True:
Debug.write("Small molecule selected")
if params.xia2.settings.symmetry.chirality is None:
PhilIndex.update("xia2.settings.symmetry.chirality=nonchiral")
params = PhilIndex.get_python_object()
# pipeline options
self._read_pipeline()
for (parameter, value) in (
("project", params.xia2.settings.project),
("crystal", params.xia2.settings.crystal),
):
validate_project_crystal_name(parameter, value)
Debug.write("Project: %s" % params.xia2.settings.project)
Debug.write("Crystal: %s" % params.xia2.settings.crystal)
# FIXME add some consistency checks in here e.g. that there are
# images assigned, there is a lattice assigned if cell constants
# are given and so on
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
from xia2.Handlers.Environment import get_number_cpus
if mp_params.mode == "parallel":
if mp_params.type == "qsub":
if which("qsub") is None:
raise Sorry("qsub not available")
if mp_params.njob is Auto:
mp_params.njob = get_number_cpus()
if mp_params.nproc is Auto:
mp_params.nproc = 1
elif mp_params.nproc is Auto:
mp_params.nproc = get_number_cpus()
elif mp_params.mode == "serial":
if mp_params.type == "qsub":
if which("qsub") is None:
raise Sorry("qsub not available")
if mp_params.njob is Auto:
mp_params.njob = 1
if mp_params.nproc is Auto:
mp_params.nproc = get_number_cpus()
PhilIndex.update("xia2.settings.multiprocessing.njob=%d" %
mp_params.njob)
PhilIndex.update("xia2.settings.multiprocessing.nproc=%d" %
mp_params.nproc)
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
if mp_params.nproc > 1 and os.name == "nt":
raise Sorry("nproc > 1 is not supported on Windows.") # #191
if params.xia2.settings.indexer is not None:
add_preference("indexer", params.xia2.settings.indexer)
if params.xia2.settings.refiner is not None:
add_preference("refiner", params.xia2.settings.refiner)
if params.xia2.settings.integrater is not None:
add_preference("integrater", params.xia2.settings.integrater)
if params.xia2.settings.scaler is not None:
add_preference("scaler", params.xia2.settings.scaler)
if params.xia2.settings.multi_sweep_indexing is Auto:
if (params.xia2.settings.small_molecule is True
and "dials" == params.xia2.settings.indexer):
PhilIndex.update("xia2.settings.multi_sweep_indexing=True")
else:
PhilIndex.update("xia2.settings.multi_sweep_indexing=False")
if (params.xia2.settings.multi_sweep_indexing is True
and params.xia2.settings.multiprocessing.mode == "parallel"):
Chatter.write(
"Multi sweep indexing disabled:\nMSI is not available for parallel processing."
)
PhilIndex.update("xia2.settings.multi_sweep_indexing=False")
input_json = params.xia2.settings.input.json
if input_json is not None and len(input_json):
for json_file in input_json:
assert os.path.isfile(json_file)
load_experiments(json_file)
reference_geometry = params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
reference_geometries = "\n".join([
"xia2.settings.input.reference_geometry=%s" %
os.path.abspath(g)
for g in params.xia2.settings.input.reference_geometry
])
Debug.write(reference_geometries)
PhilIndex.update(reference_geometries)
Debug.write("xia2.settings.trust_beam_centre=true")
PhilIndex.update("xia2.settings.trust_beam_centre=true")
params = PhilIndex.get_python_object()
params = PhilIndex.get_python_object()
if params.xia2.settings.input.xinfo is not None:
xinfo_file = os.path.abspath(params.xia2.settings.input.xinfo)
PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file)
params = PhilIndex.get_python_object()
self.set_xinfo(xinfo_file)
# issue #55 if not set ATOM in xinfo but anomalous=true or atom= set
# on commandline, set here, should be idempotent
if params.xia2.settings.input.anomalous is True:
crystals = self._xinfo.get_crystals()
for xname in crystals:
xtal = crystals[xname]
Debug.write("Setting anomalous for crystal %s" % xname)
xtal.set_anomalous(True)
else:
xinfo_file = "%s/automatic.xinfo" % os.path.abspath(os.curdir)
PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file)
params = PhilIndex.get_python_object()
if params.dials.find_spots.phil_file is not None:
PhilIndex.update(
"dials.find_spots.phil_file=%s" %
os.path.abspath(params.dials.find_spots.phil_file))
if params.dials.index.phil_file is not None:
PhilIndex.update("dials.index.phil_file=%s" %
os.path.abspath(params.dials.index.phil_file))
if params.dials.refine.phil_file is not None:
PhilIndex.update("dials.refine.phil_file=%s" %
os.path.abspath(params.dials.refine.phil_file))
if params.dials.integrate.phil_file is not None:
PhilIndex.update("dials.integrate.phil_file=%s" %
os.path.abspath(params.dials.integrate.phil_file))
if params.xds.index.xparm is not None:
Flags.set_xparm(params.xds.index.xparm)
if params.xds.index.xparm_ub is not None:
Flags.set_xparm_ub(params.xds.index.xparm_ub)
if params.xia2.settings.scale.freer_file is not None:
freer_file = os.path.abspath(params.xia2.settings.scale.freer_file)
if not os.path.exists(freer_file):
raise RuntimeError("%s does not exist" % freer_file)
from xia2.Modules.FindFreeFlag import FindFreeFlag
column = FindFreeFlag(freer_file)
Debug.write("FreeR_flag column in %s found: %s" %
(freer_file, column))
PhilIndex.update("xia2.settings.scale.freer_file=%s" % freer_file)
if params.xia2.settings.scale.reference_reflection_file is not None:
reference_reflection_file = os.path.abspath(
params.xia2.settings.scale.reference_reflection_file)
if not os.path.exists(reference_reflection_file):
raise RuntimeError("%s does not exist" %
reference_reflection_file)
PhilIndex.update(
"xia2.settings.scale.reference_reflection_file=%s" %
reference_reflection_file)
params = PhilIndex.get_python_object()
datasets = unroll_datasets(PhilIndex.params.xia2.settings.input.image)
for dataset in datasets:
start_end = None
# here we only care about ':' which are later than C:\
if ":" in dataset[3:]:
tokens = dataset.split(":")
# cope with windows drives i.e. C:\data\blah\thing_0001.cbf:1:100
if len(tokens[0]) == 1:
tokens = ["%s:%s" % (tokens[0], tokens[1])] + tokens[2:]
if len(tokens) != 3:
raise RuntimeError("/path/to/image_0001.cbf:start:end")
dataset = tokens[0]
start_end = int(tokens[1]), int(tokens[2])
from xia2.Applications.xia2setup import is_hd5f_name
if os.path.exists(os.path.abspath(dataset)):
dataset = os.path.abspath(dataset)
else:
directories = [os.getcwd()] + self._argv[1:]
found = False
for d in directories:
if os.path.exists(os.path.join(d, dataset)):
dataset = os.path.join(d, dataset)
found = True
break
if not found:
raise Sorry("Could not find %s in %s" %
(dataset, " ".join(directories)))
if is_hd5f_name(dataset):
self._hdf5_master_files.append(dataset)
if start_end:
Debug.write("Image range: %d %d" % start_end)
if dataset not in self._default_start_end:
self._default_start_end[dataset] = []
self._default_start_end[dataset].append(start_end)
else:
Debug.write("No image range specified")
else:
template, directory = image2template_directory(
os.path.abspath(dataset))
self._default_template.append(os.path.join(
directory, template))
self._default_directory.append(directory)
Debug.write("Interpreted from image %s:" % dataset)
Debug.write("Template %s" % template)
Debug.write("Directory %s" % directory)
if start_end:
Debug.write("Image range: %d %d" % start_end)
key = os.path.join(directory, template)
if key not in self._default_start_end:
self._default_start_end[key] = []
self._default_start_end[key].append(start_end)
else:
Debug.write("No image range specified")
# finally, check that all arguments were read and raise an exception
# if any of them were nonsense.
with open("xia2-working.phil", "wb") as f:
f.write(PhilIndex.working_phil.as_str())
f.write(
os.linesep
) # temporarily required for https://github.com/dials/dials/issues/522
with open("xia2-diff.phil", "wb") as f:
f.write(PhilIndex.get_diff().as_str())
f.write(
os.linesep
) # temporarily required for https://github.com/dials/dials/issues/522
Debug.write("\nDifference PHIL:")
Debug.write(PhilIndex.get_diff().as_str(), strip=False)
Debug.write("Working PHIL:")
Debug.write(PhilIndex.working_phil.as_str(), strip=False)
nonsense = "Unknown command-line options:"
was_nonsense = False
for j, argv in enumerate(self._argv):
if j == 0:
continue
if argv[0] != "-" and "=" not in argv:
continue
if j not in self._understood:
nonsense += " %s" % argv
was_nonsense = True
if was_nonsense:
raise RuntimeError(nonsense)
def imageset_to_xds(imageset, synchrotron = None, refined_beam_vector = None,
refined_rotation_axis = None, refined_distance = None):
'''A function to take an input header dictionary from Diffdump
and generate a list of records to start XDS - see Doc/INP.txt.'''
# decide if we are at a synchrotron if we don't know already...
# that is, the wavelength is around either the Copper or Chromium
# K-alpha edge and this is an image plate.
beam = imageset.get_beam()
from dxtbx.serialize.xds import to_xds, xds_detector_name
converter = to_xds(imageset)
detector_class_is_square = {
'adsc q4':True,
'adsc q4 2x2 binned':True,
'adsc q210':True,
'adsc q210 2x2 binned':True,
'adsc q270':True,
'adsc q270 2x2 binned':True,
'adsc q315':True,
'adsc q315 2x2 binned':True,
'adsc HF4M':True,
'holton fake 01':True,
'mar 345':False,
'mar 180':False,
'mar 240':False,
'mar 300 ccd':True,
'mar 325 ccd':True,
'mar 225 ccd':True,
'mar ccd 225 hs':True,
'rayonix ccd 165':False,
'rayonix ccd 135':False,
'rayonix ccd 300':True,
'rayonix ccd 325':True,
'rayonix ccd 225':True,
'rayonix ccd 225 hs':True,
'rayonix ccd 300 hs':True,
'mar 165 ccd':False,
'mar 135 ccd':False,
'pilatus 12M':True,
'pilatus 6M':True,
'pilatus 2M':True,
'pilatus 1M':True,
'pilatus 200K':True,
'pilatus 300K':True,
'eiger 4M':True,
'eiger 9M':True,
'eiger 16M':True,
'rigaku saturn 92 2x2 binned':True,
'rigaku saturn 944 2x2 binned':True,
'rigaku saturn 724 2x2 binned':True,
'rigaku saturn 92':True,
'rigaku saturn 944':True,
'rigaku saturn 724':True,
'rigaku saturn a200':True,
'raxis IV':True,
'NOIR1':True}
sensor = converter.get_detector()[0].get_type()
fast, slow = converter.detector_size
f, s = converter.pixel_size
df = int(1000 * f)
ds = int(1000 * s)
# FIXME probably need to rotate by pi about the X axis
result = []
from dxtbx.model.detector_helpers_types import detector_helpers_types
detector = xds_detector_name(
detector_helpers_types.get(sensor, fast, slow, df, ds))
trusted = converter.get_detector()[0].get_trusted_range()
# FIXME what follows below should perhaps be 0 for the really weak
# pilatus data sets?
result.append('DETECTOR=%s MINIMUM_VALID_PIXEL_VALUE=%d OVERLOAD=%d' %
(detector, trusted[0] + 1, trusted[1]))
result.append('DIRECTION_OF_DETECTOR_X-AXIS=%f %f %f' %
converter.detector_x_axis)
result.append('DIRECTION_OF_DETECTOR_Y-AXIS=%f %f %f' %
converter.detector_y_axis)
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
if params.xds.trusted_region:
result.append(
'TRUSTED_REGION= %.2f %.2f' % tuple(params.xds.trusted_region))
elif detector_class_is_square[
detector_helpers_types.get(sensor, fast, slow, df, ds).replace('-', ' ')]:
result.append('TRUSTED_REGION=0.0 1.41')
else:
result.append('TRUSTED_REGION=0.0 0.99')
result.append('NX=%d NY=%d QX=%.4f QY=%.4f' % (fast, slow, f, s))
# RAXIS detectors have the distance written negative - why????
# this is ONLY for XDS - SATURN are the same - probably left handed
# goniometer rotation on rigaku X-ray sets.
if refined_distance:
result.append('DETECTOR_DISTANCE=%7.3f' % refined_distance)
else:
result.append('DETECTOR_DISTANCE=%7.3f' % converter.detector_distance)
result.append('OSCILLATION_RANGE=%4.2f' % converter.oscillation_range)
result.append('X-RAY_WAVELENGTH=%8.6f' % converter.wavelength)
# if user specified reversephi and this was not picked up in the
# format class reverse phi: n.b. double-negative warning!
if refined_rotation_axis:
result.append('ROTATION_AXIS= %f %f %f' % \
refined_rotation_axis)
else:
result.append('ROTATION_AXIS= %.3f %.3f %.3f' % \
converter.rotation_axis)
if refined_beam_vector:
result.append('INCIDENT_BEAM_DIRECTION=%f %f %f' % \
refined_beam_vector)
else:
result.append(
'INCIDENT_BEAM_DIRECTION= %.3f %.3f %.3f' % converter.beam_vector)
if hasattr(beam, "get_polarization_fraction"):
R = converter.imagecif_to_xds_transformation_matrix
result.append('FRACTION_OF_POLARIZATION= %.3f' %
beam.get_polarization_fraction())
result.append('POLARIZATION_PLANE_NORMAL= %.3f %.3f %.3f' %
(R * matrix.col(beam.get_polarization_normal())).elems)
# 24/NOV/14 XDS determines the air absorption automatically
# based on wavelength. May be useful to override this for in vacuo exps
# result.append('AIR=0.001')
if detector == 'PILATUS':
try:
thickness = converter.get_detector()[0].get_thickness()
if not thickness:
thickness = 0.32
Debug.write('Could not determine sensor thickness. Assuming default PILATUS 0.32mm')
except e:
thickness = 0.32
Debug.write('Error occured during sensor thickness determination. Assuming default PILATUS 0.32mm')
result.append('SENSOR_THICKNESS=%f' % thickness)
# # FIXME: Sensor absorption coefficient calculation probably requires a more general solution
# if converter.get_detector()[0].get_material() == 'CdTe':
# print "CdTe detector detected. Beam wavelength is %8.6f Angstrom" % converter.wavelength
if len(converter.panel_x_axis) > 1:
for panel_id in range(len(converter.panel_x_axis)):
result.append('')
result.append('!')
result.append('! SEGMENT %d' %(panel_id+1))
result.append('!')
result.append('SEGMENT= %d %d %d %d' % converter.panel_limits[panel_id])
result.append('DIRECTION_OF_SEGMENT_X-AXIS= %.3f %.3f %.3f' % \
converter.panel_x_axis[panel_id])
result.append('DIRECTION_OF_SEGMENT_Y-AXIS= %.3f %.3f %.3f' % \
converter.panel_y_axis[panel_id])
result.append('SEGMENT_DISTANCE= %.3f' % converter.panel_distance[panel_id])
result.append(
'SEGMENT_ORGX= %.1f SEGMENT_ORGY= %.1f' % converter.panel_origin[panel_id])
result.append('')
for f0, s0, f1, s1 in converter.get_detector()[0].get_mask():
result.append('UNTRUSTED_RECTANGLE= %d %d %d %d' %
(f0 - 1, f1 + 1, s0 - 1, s1 + 1))
if params.xds.untrusted_ellipse:
for untrusted_ellipse in params.xds.untrusted_ellipse:
result.append(
'UNTRUSTED_ELLIPSE= %d %d %d %d' % tuple(untrusted_ellipse))
Debug.write(result[-1])
if params.xds.untrusted_rectangle:
for untrusted_rectangle in params.xds.untrusted_rectangle:
result.append(
'UNTRUSTED_RECTANGLE= %d %d %d %d' % tuple(untrusted_rectangle))
Debug.write(result[-1])
return result
def load_imagesets(template, directory, id_image=None, image_range=None,
use_cache=True, reversephi=False):
global imageset_cache
from dxtbx.datablock import DataBlockFactory
from xia2.Applications.xia2setup import known_hdf5_extensions
full_template_path = os.path.join(directory, template)
if full_template_path not in imageset_cache or not use_cache:
from dxtbx.datablock import BeamComparison
from dxtbx.datablock import DetectorComparison
from dxtbx.datablock import GoniometerComparison
params = PhilIndex.params.xia2.settings
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation)
scan_tolerance = params.input.tolerance.scan.oscillation
format_kwargs = {
'dynamic_shadowing' : params.input.format.dynamic_shadowing,
'multi_panel' : params.input.format.multi_panel,
}
if os.path.splitext(full_template_path)[-1] in known_hdf5_extensions:
import glob
g = glob.glob(os.path.join(directory, '*_master.h5'))
master_file = None
for p in g:
substr = longest_common_substring(template, p)
if substr:
if (master_file is None or
(len(substr) > len(longest_common_substring(template, master_file)))):
master_file = p
if master_file is None:
raise RuntimeError("Can't find master file for %s" %full_template_path)
unhandled = []
datablocks = DataBlockFactory.from_filenames(
[master_file], verbose=False, unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs)
assert len(unhandled) == 0, "unhandled image files identified: %s" % \
unhandled
assert len(datablocks) == 1, "1 datablock expected, %d found" % \
len(datablocks)
else:
from dxtbx.sweep_filenames import locate_files_matching_template_string
params = PhilIndex.get_python_object()
read_all_image_headers = params.xia2.settings.read_all_image_headers
if read_all_image_headers:
paths = sorted(locate_files_matching_template_string(full_template_path))
unhandled = []
datablocks = DataBlockFactory.from_filenames(
paths, verbose=False, unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs)
assert len(unhandled) == 0, "unhandled image files identified: %s" % \
unhandled
assert len(datablocks) == 1, "1 datablock expected, %d found" % \
len(datablocks)
else:
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter(
[full_template_path], kwargs=format_kwargs)
datablocks = importer.datablocks
imagesets = datablocks[0].extract_sweeps()
assert len(imagesets) > 0, "no imageset found"
imageset_cache[full_template_path] = OrderedDict()
if reversephi:
for imageset in imagesets:
goniometer = imageset.get_goniometer()
goniometer.set_rotation_axis(
tuple((-g for g in goniometer.get_rotation_axis())))
reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
update_with_reference_geometry(imagesets, reference_geometry)
from dials.command_line.dials_import import ManualGeometryUpdater
update_geometry = ManualGeometryUpdater(
PhilIndex.params.xia2.settings.input)
imagesets = [update_geometry(imageset) for imageset in imagesets]
from scitbx.array_family import flex
for imageset in imagesets:
scan = imageset.get_scan()
exposure_times = scan.get_exposure_times()
epochs = scan.get_epochs()
if exposure_times.all_eq(0):
exposure_times = flex.double(exposure_times.size(), 1)
scan.set_exposure_times(exposure_times)
elif not exposure_times.all_gt(0):
exposure_times = flex.double(exposure_times.size(), exposure_times[0])
scan.set_exposure_times(exposure_times)
if epochs.size() > 1 and not epochs.all_gt(0):
for i in range(1, epochs.size()):
epochs[i] = epochs[i-1] + exposure_times[i-1]
scan.set_epochs(epochs)
_id_image = scan.get_image_range()[0]
imageset_cache[full_template_path][_id_image] = imageset
if id_image is not None:
return [imageset_cache[full_template_path][id_image]]
elif image_range is not None:
for imageset in imageset_cache[full_template_path].values():
scan = imageset.get_scan()
scan_image_range = scan.get_image_range()
if (image_range[0] >= scan_image_range[0] and
image_range[1] <= scan_image_range[1]):
imagesets = [imageset[
image_range[0] - scan_image_range[0]:
image_range[1] + 1 - scan_image_range[0]]]
assert len(imagesets[0]) == image_range[1] - image_range[0] + 1, \
len(imagesets[0])
return imagesets
return imageset_cache[full_template_path].values()
def get_qsub_command():
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
if mp_params.qsub_command:
return mp_params.qsub_command
return None
def __init__(self, name,
wavelength,
sample,
directory = None,
image = None,
beam = None,
reversephi = False,
distance = None,
gain = 0.0,
dmin = 0.0,
dmax = 0.0,
polarization = 0.0,
frames_to_process = None,
user_lattice = None,
user_cell = None,
epoch = 0,
ice = False,
excluded_regions = None):
'''Create a new sweep named name, belonging to XWavelength object
wavelength, representing the images in directory starting with image,
with beam centre optionally defined.'''
if excluded_regions is None:
excluded_regions = []
# + check the wavelength is an XWavelength object
# raise an exception if not... or not...
if not wavelength.__class__.__name__ == 'XWavelength':
pass
# FIXME bug 2221 if DIRECTORY starts with ~/ or ~graeme (say) need to
# interpret this properly - e.g. map it to a full PATH.
directory = expand_path(directory)
self._name = name
self._wavelength = wavelength
self._sample = sample
self._directory = directory
self._image = image
self._reversephi = reversephi
self._epoch = epoch
self._user_lattice = user_lattice
self._user_cell = user_cell
self._header = { }
self._resolution_high = dmin
self._resolution_low = dmax
self._ice = ice
self._excluded_regions = excluded_regions
self._imageset = None
# FIXME in here also need to be able to accumulate the total
# dose from all experimental measurements (complex) and provide
# a _epoch_to_dose dictionary or some such... may be fiddly as
# this will need to parse across multiple templates. c/f Bug # 2798
self._epoch_to_image = { }
self._image_to_epoch = { }
# to allow first, last image for processing to be
# set... c/f integrater interface
self._frames_to_process = frames_to_process
# + derive template, list of images
params = PhilIndex.get_python_object()
if directory and image:
self._template, self._directory = \
image2template_directory(os.path.join(directory,
image))
from xia2.Schema import load_imagesets
imagesets = load_imagesets(
self._template, self._directory, image_range=self._frames_to_process,
reversephi=(params.xia2.settings.input.reverse_phi or self._reversephi))
assert len(imagesets) == 1, "one imageset expected, %d found" % \
len(imagesets)
self._imageset = copy.deepcopy(imagesets[0])
start, end = self._imageset.get_array_range()
self._images = list(range(start+1, end+1))
# FIXME in here check that (1) the list of images is continuous
# and (2) that all of the images are readable. This should also
# take into account frames_to_process if set.
if self._frames_to_process is None:
self._frames_to_process = min(self._images), max(self._images)
start, end = self._frames_to_process
error = False
if params.general.check_image_files_readable:
for j in range(start, end + 1):
image_name = self.get_imageset().get_path(j-start)
if not j in self._images:
Debug.write('image %s missing' %image_name)
error = True
continue
if not os.access(image_name, os.R_OK):
Debug.write('image %s unreadable' %image_name)
error = True
continue
if error:
raise RuntimeError, 'problem with sweep %s' % self._name
# + read the image header information into here?
# or don't I need it? it would be useful for checking
# against wavelength.getWavelength() I guess to make
# sure that the plumbing is all sound.
# check that they match by closer than 0.0001A, if wavelength
# is not None
beam_ = self._imageset.get_beam()
scan = self._imageset.get_scan()
if wavelength is not None:
# FIXME 29/NOV/06 if the wavelength wavelength value
# is 0.0 then first set it to the header value - note
# that this assumes that the header value is correct
# (a reasonable assumption)
if wavelength.get_wavelength() == 0.0:
wavelength.set_wavelength(beam_.get_wavelength())
# FIXME 08/DEC/06 in here need to allow for the fact
# that the wavelength in the image header could be wrong and
# in fact it should be replaced with the input value -
# through the user will need to be warned of this and
# also everything using the FrameProcessor interface
# will also have to respect this!
if math.fabs(beam_.get_wavelength() -
wavelength.get_wavelength()) > 0.0001:
# format = 'wavelength for sweep %s does not ' + \
# 'match wavelength %s'
# raise RuntimeError, format % \
# (name, wavelength.get_name())
format = 'Header wavelength for sweep %s different' + \
' to assigned value (%4.2f vs. %4.2f)'
Chatter.write(format % (name, beam_.get_wavelength(),
wavelength.get_wavelength()))
# also in here look at the image headers to see if we can
# construct a mapping between exposure epoch and image ...
images = []
if self._frames_to_process:
start, end = self._frames_to_process
for j in self._images:
if j >= start and j <= end:
images.append(j)
else:
images = self._images
for j in images:
epoch = scan.get_image_epoch(j)
if epoch == 0.0:
epoch = float(os.stat(self._imageset.get_path(j-images[0])).st_mtime)
self._epoch_to_image[epoch] = j
self._image_to_epoch[j] = epoch
epochs = self._epoch_to_image.keys()
Debug.write('Exposure epoch for sweep %s: %d %d' % \
(self._template, min(epochs), max(epochs)))
self._input_imageset = copy.deepcopy(self._imageset)
# + get the lattice - can this be a pointer, so that when
# this object updates lattice it is globally-for-this-crystal
# updated? The lattice included directly in here includes an
# exact unit cell for data reduction, the crystal lattice
# contains an approximate unit cell which should be
# from the unit cells from all sweeps contained in the
# XCrystal. FIXME should I be using a LatticeInfo object
# in here? See what the Indexer interface produces. ALT:
# just provide an Indexer implementation "hook".
# See Headnote 001 above. See also _get_indexer,
# _get_integrater below.
self._indexer = None
self._refiner = None
self._integrater = None
# I don't need this - it is equivalent to self.getWavelength(
# ).getCrystal().getLattice()
# self._crystal_lattice = None
# this means that this module will have to present largely the
# same interface as Indexer and Integrater so that the calls
# can be appropriately forwarded.
# finally configure the beam if set
if beam is not None:
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
try:
set_mosflm_beam_centre(self.get_imageset().get_detector(),
self.get_imageset().get_beam(),
beam)
except AssertionError, e:
Debug.write('Error setting mosflm beam centre: %s' % e)
def run():
try:
check_environment()
except Exception as e:
with open("xia2-error.txt", "w") as fh:
traceback.print_exc(file=fh)
logger.error('Status: error "%s"', str(e))
if len(sys.argv) < 2 or "-help" in sys.argv:
help()
sys.exit()
cwd = os.getcwd()
try:
from .xia2_main import xia2_main
xia2_main(stop_after="integrate")
logger.info("Status: normal termination")
wd = os.path.join(cwd, "strategy")
if not os.path.exists(wd):
os.mkdir(wd)
os.chdir(wd)
CommandLine = get_command_line()
xinfo = CommandLine.get_xinfo()
crystals = xinfo.get_crystals()
assert len(crystals) == 1
crystal = list(crystals.values())[0]
assert len(crystal.get_wavelength_names()) == 1
wavelength = crystal.get_xwavelength(crystal.get_wavelength_names()[0])
sweeps = wavelength.get_sweeps()
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
strategy_params = params.strategy
if not len(strategy_params):
strategy_params = [
PhilIndex.get_scope_by_name("strategy")[0].extract()
]
from dxtbx.model import MultiAxisGoniometer
gonio = sweeps[0].get_imageset().get_goniometer()
if (isinstance(gonio, MultiAxisGoniometer)
and len(gonio.get_axes()) == 3 and gonio.get_scan_axis() == 2):
from xia2.Wrappers.Dials.AlignCrystal import AlignCrystal
align_crystal = AlignCrystal()
align_crystal.set_experiments_filename(
sweeps[0]._get_integrater().get_integrated_experiments())
align_crystal.set_working_directory(wd)
auto_logfiler(align_crystal)
align_crystal.set_json_filename("%i_align_crystal.json" %
align_crystal.get_xpid())
align_crystal.run()
logger.info("".join(align_crystal.get_all_output()))
results_all = {}
def process_one_strategy(args):
assert len(args) == 4
experiments, reflections, strategy, t_ref = args
from xia2.Wrappers.EMBL import Best
best = Best.BestStrategy()
for isweep, (expt, refl) in enumerate(zip(experiments,
reflections)):
from xia2.Wrappers.Dials.ExportBest import ExportBest
export = ExportBest()
export.set_experiments_filename(expt)
export.set_reflections_filename(refl)
export.set_working_directory(wd)
auto_logfiler(export)
prefix = "%i_best" % export.get_xpid()
export.set_prefix(prefix)
export.run()
if isweep == 0:
best.set_t_ref(t_ref)
best.set_mos_dat("%s.dat" % prefix)
best.set_mos_par("%s.par" % prefix)
best.add_mos_hkl("%s.hkl" % prefix)
best.set_i2s(strategy.i_over_sigi)
best.set_T_max(strategy.max_total_exposure)
best.set_t_min(strategy.min_exposure)
# best.set_trans_ref(25.0)
best.set_S_max(strategy.max_rotation_speed)
best.set_w_min(strategy.min_oscillation_width)
best.set_M_min(strategy.multiplicity)
best.set_C_min(strategy.completeness)
best.set_GpS(strategy.dose_rate)
best.set_shape(strategy.shape)
best.set_susceptibility(strategy.susceptibility)
best.set_anomalous(strategy.anomalous)
best.set_detector("pilatus6m")
best.set_working_directory(wd)
auto_logfiler(best)
xmlout = os.path.join(best.get_working_directory(),
"%i_best.xml" % best.get_xpid())
best.set_xmlout(xmlout)
best.strategy()
results = best.get_results_dict()
results["description"] = strategy.description
if "phi_end" not in results:
results["phi_end"] = str(
float(results["phi_start"]) +
float(results["number_of_images"]) *
float(results["phi_width"]))
from dxtbx.serialize import load
expt = load.experiment_list(experiments[0])[0]
results["spacegroup"] = (
expt.crystal.get_space_group().type().lookup_symbol())
return results
args = []
for istrategy, strategy in enumerate(strategy_params):
imageset = sweeps[0].get_imageset()
scan = imageset.get_scan()
experiments = [
sweep._get_integrater().get_integrated_experiments()
for sweep in sweeps
]
reflections = [
sweep._get_integrater().get_integrated_reflections()
for sweep in sweeps
]
t_ref = scan.get_exposure_times()[0]
args.append((experiments, reflections, strategy, t_ref))
nproc = params.xia2.settings.multiprocessing.nproc
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_strategy,
args,
processes=nproc,
method="multiprocessing",
preserve_order=True,
preserve_exception_message=True,
)
for istrategy, (result,
strategy) in enumerate(zip(results, strategy_params)):
name = strategy.name
description = strategy.description
if name is None:
name = "Strategy%i" % (istrategy + 1)
results_all[name] = result
multiplicity = result["redundancy"]
try:
multiplicity = "%.2f" % multiplicity
except TypeError:
pass
logger.info("Strategy %i", istrategy)
if description is not None:
logger.info(description)
logger.info(
"Start / end / width: %.2f/%.2f/%.2f",
float(result["phi_start"]),
float(result["phi_end"]),
float(result["phi_width"]),
)
logger.info(
"Completeness / multiplicity / resolution: %.2f/%s/%.2f",
float(result["completeness"]),
multiplicity,
float(result["resolution"]),
)
logger.info(
"Transmission / exposure %.3f/%.3f",
float(result["transmission"]),
float(result["exposure_time"]),
)
with open("strategies.json", "wb") as f:
json.dump(results_all, f, indent=2)
except Exception as e:
with open(os.path.join(cwd, "xia2-error.txt"), "w") as fh:
traceback.print_exc(file=fh)
logger.error('Status: error "%s"', str(e))
os.chdir(cwd)
def load_imagesets(template, directory, id_image=None, image_range=None,
use_cache=True, reversephi=False):
global imageset_cache
full_template_path = os.path.join(directory, template)
if full_template_path not in imageset_cache or not use_cache:
from dxtbx.datablock import DataBlockFactory
from dxtbx.sweep_filenames import locate_files_matching_template_string
params = PhilIndex.get_python_object()
read_all_image_headers = params.xia2.settings.read_all_image_headers
if read_all_image_headers:
paths = sorted(locate_files_matching_template_string(full_template_path))
unhandled = []
datablocks = DataBlockFactory.from_filenames(
paths, verbose=False, unhandled=unhandled)
assert len(unhandled) == 0, "unhandled image files identified: %s" % \
unhandled
assert len(datablocks) == 1, "1 datablock expected, %d found" % \
len(datablocks)
else:
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([full_template_path])
datablocks = importer.datablocks
imagesets = datablocks[0].extract_sweeps()
assert len(imagesets) > 0, "no imageset found"
imageset_cache[full_template_path] = OrderedDict()
if reversephi:
for imageset in imagesets:
goniometer = imageset.get_goniometer()
goniometer.set_rotation_axis(
tuple((-g for g in goniometer.get_rotation_axis())))
reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
update_with_reference_geometry(imagesets, reference_geometry)
for imageset in imagesets:
scan = imageset.get_scan()
_id_image = scan.get_image_range()[0]
imageset_cache[full_template_path][_id_image] = imageset
if id_image is not None:
return [imageset_cache[full_template_path][id_image]]
elif image_range is not None:
for imageset in imageset_cache[full_template_path].values():
scan = imageset.get_scan()
scan_image_range = scan.get_image_range()
if (image_range[0] >= scan_image_range[0] and
image_range[1] <= scan_image_range[1]):
imagesets = [imageset[
image_range[0] - scan_image_range[0]:
image_range[1] + 1 - scan_image_range[0]]]
assert len(imagesets[0]) == image_range[1] - image_range[0] + 1, \
len(imagesets[0])
return imagesets
return imageset_cache[full_template_path].values()
