def _updated_aimless(self):
'''Generate a correctly configured Aimless...'''
aimless = None
if not self._scalr_corrections:
aimless = self._factory.Aimless()
else:
aimless = self._factory.Aimless(
partiality_correction = self._scalr_correct_partiality,
absorption_correction = self._scalr_correct_absorption,
decay_correction = self._scalr_correct_decay)
if Flags.get_microcrystal():
# fiddly little data sets - allow more rapid scaling...
aimless.set_scaling_parameters('rotation', 2.0)
if self._scalr_correct_decay:
aimless.set_bfactor(bfactor=True, brotation = 2.0)
if Flags.get_small_molecule():
aimless.set_scaling_parameters('rotation', 15.0)
aimless.set_bfactor(bfactor=False)
aimless.set_surface_tie(PhilIndex.params.ccp4.aimless.surface_tie)
aimless.set_surface_link(PhilIndex.params.ccp4.aimless.surface_link)
return aimless
def _prepare_pointless_hklin(working_directory,
hklin,
phi_width):
'''Prepare some data for pointless - this will take only 180 degrees
of data if there is more than this (through a "rebatch" command) else
will simply return hklin.'''
# also remove blank images?
if not Flags.get_microcrystal() and not Flags.get_small_molecule():
Debug.write('Excluding blank images')
hklout = os.path.join(
working_directory,
'%s_noblank.mtz' % (os.path.split(hklin)[-1][:-4]))
FileHandler.record_temporary_file(hklout)
hklin = remove_blank(hklin, hklout)
# find the number of batches
md = Mtzdump()
md.set_working_directory(working_directory)
auto_logfiler(md)
md.set_hklin(hklin)
md.dump()
batches = max(md.get_batches()) - min(md.get_batches())
phi_limit = 180
if batches * phi_width < phi_limit or Flags.get_small_molecule():
return hklin
hklout = os.path.join(
working_directory,
'%s_prepointless.mtz' % (os.path.split(hklin)[-1][:-4]))
rb = Rebatch()
rb.set_working_directory(working_directory)
auto_logfiler(rb)
rb.set_hklin(hklin)
rb.set_hklout(hklout)
first = min(md.get_batches())
last = first + int(phi_limit / phi_width)
Debug.write('Preparing data for pointless - %d batches (%d degrees)' % \
((last - first), phi_limit))
rb.limit_batches(first, last)
# we will want to delete this one exit
FileHandler.record_temporary_file(hklout)
return hklout
def update(self):
'''Check to see if any more frames have appeared - if they
have update myself and reset.'''
images = find_matching_images(self._template,
self._directory)
if len(images) > len(self._images):
self._images = images
from xia2.Schema import load_imagesets
imagesets = load_imagesets(
self._template, self._directory, id_image=self._id_image,
use_cache=False, reversephi=Flags.get_reversephi())
max_images = 0
best_sweep = None
for imageset in imagesets:
scan = imageset.get_scan()
if scan is None: continue
if imageset.get_scan().get_num_images() > max_images:
best_sweep = imageset
self._imageset = best_sweep
return
def merge(self):
'''Actually merge the already scaled reflections.'''
self.check_hklin()
self.check_hklout()
if not self._onlymerge:
raise RuntimeError, 'for scaling use scale()'
if not self._scalepack:
self.set_task('Merging scaled reflections from %s => %s' % \
(os.path.split(self.get_hklin())[-1],
os.path.split(self.get_hklout())[-1]))
else:
self.set_task('Merging reflections from %s => scalepack %s' % \
(os.path.split(self.get_hklin())[-1],
os.path.split(self.get_hklout())[-1]))
self._xmlout = os.path.join(self.get_working_directory(),
'%d_aimless.xml' % self.get_xpid())
self.start()
self.input('xmlout %d_aimless.xml' % self.get_xpid())
if not Flags.get_small_molecule():
self.input('bins 20')
self.input('run 1 all')
self.input('scales constant')
self.input('initial unity')
self.input('sdcorrection both noadjust 1.0 0.0 0.0')
if self._anomalous:
self.input('anomalous on')
else:
self.input('anomalous off')
if self._scalepack:
self.input('output polish unmerged')
self.input('output unmerged')
self.close_wait()
# check for errors
try:
self.check_for_errors()
self.check_ccp4_errors()
self.check_aimless_errors()
status = self.get_ccp4_status()
if 'Error' in status:
raise RuntimeError, '[AIMLESS] %s' % status
except RuntimeError, e:
try:
os.remove(self.get_hklout())
except:
pass
raise e
def run(self):
from xia2.Handlers.Streams import Debug
Debug.write('Running %s' %self.get_executable())
self.clear_command_line()
self.add_command_line(self._sweep_filename)
self.add_command_line(self._spot_filename)
nproc = Flags.get_parallel()
self.set_cpu_threads(nproc)
self.add_command_line('nproc=%i' % nproc)
for scan_range in self._scan_ranges:
self.add_command_line('scan_range=%d,%d' % scan_range)
if self._phil_file is not None:
self.add_command_line("%s" %self._phil_file)
self._optimized_filename = os.path.join(
self.get_working_directory(), '%d_optimized_datablock.json' %self.get_xpid())
self.add_command_line("output.datablock=%s" %self._optimized_filename)
self.start()
self.close_wait()
self.check_for_errors()
records = self.get_all_output()
assert os.path.exists(self._optimized_filename), self._optimized_filename
return
def resolution_merged_isigma(self, limit=None, log=None):
"""Compute a resolution limit where either Mn(I/sigma) = 1.0 (limit if
set) or the full extent of the data."""
if limit is None:
limit = Flags.get_misigma()
bins, ranges = self.get_resolution_bins()
misigma_s = get_positive_values(
[self.calculate_merged_isigma(bin) for bin in bins]
)
s_s = [1.0 / (r[0] * r[0]) for r in ranges][: len(misigma_s)]
if min(misigma_s) > limit:
return 1.0 / math.sqrt(max(s_s))
misigma_f = log_fit(s_s, misigma_s, 6)
if log:
fout = open(log, "w")
for j, s in enumerate(s_s):
d = 1.0 / math.sqrt(s)
o = misigma_s[j]
m = misigma_f[j]
fout.write("%f %f %f %f\n" % (s, d, o, m))
fout.close()
try:
r_misigma = 1.0 / math.sqrt(interpolate_value(s_s, misigma_f, limit))
except Exception:
r_misigma = 1.0 / math.sqrt(max(s_s))
return r_misigma
def resolution_merged_isigma(self, limit = None, log = None):
'''Compute a resolution limit where either Mn(I/sigma) = 1.0 (limit if
set) or the full extent of the data.'''
if limit is None:
limit = Flags.get_misigma()
bins, ranges = self.get_resolution_bins()
misigma_s = get_positive_values(
[self.calculate_merged_isigma(bin) for bin in bins])
s_s = [1.0 / (r[0] * r[0]) for r in ranges][:len(misigma_s)]
if min(misigma_s) > limit:
return 1.0 / math.sqrt(max(s_s))
misigma_f = log_fit(s_s, misigma_s, 6)
if log:
fout = open(log, 'w')
for j, s in enumerate(s_s):
d = 1.0 / math.sqrt(s)
o = misigma_s[j]
m = misigma_f[j]
fout.write('%f %f %f %f\n' % (s, d, o, m))
fout.close()
try:
r_misigma = 1.0 / math.sqrt(
interpolate_value(s_s, misigma_f, limit))
except:
r_misigma = 1.0 / math.sqrt(max(s_s))
return r_misigma
def __init__(self):
# set up the object ancestors...
DriverInstance.__class__.__init__(self)
# now set myself up...
self._parallel = Flags.get_parallel()
if self._parallel <= 1:
self.set_executable('xscale')
else:
self.set_executable('xscale_par')
self._version = 'new'
# overall information
self._resolution_shells = ''
self._cell = None
self._spacegroup_number = None
self._reindex_matrix = None
# corrections to apply - N.B. default values come from the
# factory function default arguments...
self._correct_decay = correct_decay
self._correct_absorption = correct_absorption
self._correct_modulation = correct_modulation
# input reflections information - including grouping information
# in the same way as the .xinfo files - through the wavelength
# names, which will be used for the output files.
self._input_reflection_files = []
self._input_reflection_wavelength_names = []
self._input_resolution_ranges = []
# these are generated at the run time
self._transposed_input = { }
self._transposed_input_keys = []
# output
self._output_reflection_files = { }
self._remove = []
# decisions about the scaling
self._crystal = None
self._zero_dose = False
self._anomalous = True
self._merge = False
# scale factor output
self._scale_factor = 1.0
# Rmerge values - for the scale model analysis - N.B. get
# one for each data set, obviously...
self._rmerges = { }
return
def reindex(self):
'''Actually perform the reindexing.'''
if PhilIndex.params.ccp4.reindex.program == 'reindex':
return self.reindex_old()
self.check_hklin()
self.check_hklout()
if not self._spacegroup and not self._operator:
raise RuntimeError, 'reindex requires spacegroup or operator'
if self._operator:
self._operator = self._operator.replace('[', '').replace(']', '')
Debug.write('Reindex... %s %s' % (self._spacegroup, self._operator))
if self._spacegroup and Flags.get_small_molecule() and False:
if not self._operator or self._operator.replace(' ', '') == 'h,k,l':
return self.cctbx_reindex()
self.start()
if self._spacegroup:
if type(self._spacegroup) == type(0):
spacegroup = Syminfo.spacegroup_number_to_name(
self._spacegroup)
elif self._spacegroup[0] in '0123456789':
spacegroup = Syminfo.spacegroup_number_to_name(
int(self._spacegroup))
else:
spacegroup = self._spacegroup
self.input('spacegroup \'%s\'' % spacegroup)
if self._operator:
# likewise
self.input('reindex \'%s\'' % self._operator)
else:
self.input('reindex \'h,k,l\'')
self.close_wait()
# check for errors
try:
self.check_for_errors()
except RuntimeError, e:
try:
os.remove(self.get_hklout())
except:
pass
raise e
def _index_prepare(self):
if self._indxr_images == []:
self._index_select_images()
if self._mosflm_autoindex_thresh is None and \
Flags.get_microcrystal():
self._mosflm_autoindex_thresh = 5
return
def __init__(self, params=None):
super(XDSIntegrateWrapper, self).__init__()
# phil parameters
if not params:
from xia2.Handlers.Phil import master_phil
params = master_phil.extract().xds.integrate
self._params = params
# now set myself up...
self._parallel = Flags.get_parallel()
self.set_cpu_threads(self._parallel)
if self._parallel <= 1:
self.set_executable('xds')
else:
self.set_executable('xds_par')
# generic bits
self._data_range = (0, 0)
self._input_data_files = { }
self._output_data_files = { }
self._input_data_files_list = ['X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf',
'BLANK.cbf',
'BKGPIX.cbf',
'GAIN.cbf',
'XPARM.XDS']
self._output_data_files_list = ['FRAME.cbf']
self._refined_xparm = False
self._updates = { }
# note well - INTEGRATE.HKL is not included in this list
# because it is likely to be very large - this is treated
# separately...
self._integrate_hkl = None
# FIXME these will also be wanted by the full integrater
# interface I guess?
self._mean_mosaic = None
self._min_mosaic = None
self._max_mosaic = None
return
def get_cell(self):
'''Compute an average cell.'''
if len(self._cells) < 1:
raise RuntimeError, 'no input unit cell parameters'
# check that the input cells are reasonably uniform -
# be really relaxed and allow 5% variation!
average_cell = [self._cells[0][j] for j in range(6)]
number_cells = 1
for j in range(1, len(self._cells)):
cell = self._cells[j]
for k in range(6):
average = average_cell[k] / number_cells
if math.fabs((cell[k] - average) / average) > 0.05 \
and not Flags.get_relax():
raise RuntimeError, 'incompatible unit cells'
average = average_cell[k] / number_cells
if math.fabs((cell[k] - average) / average) > 0.2:
raise RuntimeError, 'very incompatible unit cells'
# it was ok to remember for later on..
for k in range(6):
average_cell[k] += cell[k]
number_cells += 1
cellparm_inp = open(os.path.join(
self.get_working_directory(), 'CELLPARM.INP'), 'w')
for j in range(len(self._cells)):
cell = self._cells[j]
n_ref = self._n_refs[j]
cellparm_inp.write('UNIT_CELL_CONSTANTS=')
cellparm_inp.write(
'%.3f %.3f %.3f %.3f %.3f %.3f WEIGHT=%d\n' % \
(cell[0], cell[1], cell[2], cell[3], cell[4], cell[5],
n_ref))
cellparm_inp.close()
self.start()
self.close_wait()
# FIXME need to look for errors in here
cellparm_lp = open(os.path.join(
self.get_working_directory(), 'CELLPARM.LP'), 'r')
data = cellparm_lp.readlines()
return map(float, data[-1].split()[:6])
def new_resolution_unmerged_isigma(self, limit = None, log = None):
'''Compute a resolution limit where either I/sigma = 1.0 (limit if
set) or the full extent of the data.'''
if limit is None:
limit = Flags.get_isigma()
bins, ranges = self.get_resolution_bins()
isigma_s = get_positive_values(
[self.calculate_unmerged_isigma(bin) for bin in bins])
s_s = [1.0 / (r[0] * r[0]) for r in ranges][:len(isigma_s)]
if min(isigma_s) > limit:
return 1.0 / math.sqrt(max(s_s))
for _l, s in enumerate(isigma_s):
if s < limit:
break
if _l > 10 and _l < (len(isigma_s) - 10):
start = _l - 10
end = _l + 10
elif _l <= 10:
start = 0
end = 20
elif _l >= (len(isigma_s) - 10):
start = -20
end = -1
_s_s = s_s[start:end]
_isigma_s = isigma_s[start:end]
_isigma_f = log_fit(_s_s, _isigma_s, 3)
if log:
fout = open(log, 'w')
for j, s in enumerate(_s_s):
d = 1.0 / math.sqrt(s)
o = _isigma_s[j]
m = _isigma_f[j]
fout.write('%f %f %f %f\n' % (s, d, o, m))
fout.close()
try:
r_isigma = 1.0 / math.sqrt(interpolate_value(_s_s, _isigma_f,
limit))
except:
r_isigma = 1.0 / math.sqrt(max(_s_s))
return r_isigma
def new_resolution_unmerged_isigma(self, limit=None, log=None):
"""Compute a resolution limit where either I/sigma = 1.0 (limit if
set) or the full extent of the data."""
if limit is None:
limit = Flags.get_isigma()
bins, ranges = self.get_resolution_bins()
isigma_s = get_positive_values(
[self.calculate_unmerged_isigma(bin) for bin in bins]
)
s_s = [1.0 / (r[0] * r[0]) for r in ranges][: len(isigma_s)]
if min(isigma_s) > limit:
return 1.0 / math.sqrt(max(s_s))
for _l, s in enumerate(isigma_s):
if s < limit:
break
if _l > 10 and _l < (len(isigma_s) - 10):
start = _l - 10
end = _l + 10
elif _l <= 10:
start = 0
end = 20
elif _l >= (len(isigma_s) - 10):
start = -20
end = -1
_s_s = s_s[start:end]
_isigma_s = isigma_s[start:end]
_isigma_f = log_fit(_s_s, _isigma_s, 3)
if log:
fout = open(log, "w")
for j, s in enumerate(_s_s):
d = 1.0 / math.sqrt(s)
o = _isigma_s[j]
m = _isigma_f[j]
fout.write("%f %f %f %f\n" % (s, d, o, m))
fout.close()
try:
r_isigma = 1.0 / math.sqrt(interpolate_value(_s_s, _isigma_f, limit))
except Exception:
r_isigma = 1.0 / math.sqrt(max(_s_s))
return r_isigma
def _index_select_images(self):
'''Select correct images based on image headers.'''
if Flags.get_small_molecule():
return self._index_select_images_small_molecule()
if Flags.get_microcrystal():
return self._index_select_images_microcrystal()
phi_width = self.get_phi_width()
images = self.get_matching_images()
if Flags.get_interactive():
selected_images = index_select_images_user(phi_width, images,
Chatter)
else:
selected_images = index_select_images_lone(phi_width, images)
for image in selected_images:
Debug.write('Selected image %s' % image)
self.add_indexer_image_wedge(image)
return
def check_environment():
'''Check the environment we are running in...'''
if sys.hexversion < 0x02070000:
raise RuntimeError('Python versions older than 2.7 are not supported')
import cctbx
executable = sys.executable
cctbx_dir = os.sep.join(cctbx.__file__.split(os.sep)[:-3])
# to help wrapper code - print process id...
Debug.write('Process ID: %d' % os.getpid())
Chatter.write('Environment configuration...')
Chatter.write('Python => %s' % executable)
Chatter.write('CCTBX => %s' % cctbx_dir)
ccp4_keys = ['CCP4', 'CLIBD', 'CCP4_SCR']
for k in ccp4_keys:
v = Environment.getenv(k)
if not v:
raise RuntimeError('%s not defined - is CCP4 set up?' % k)
if not v == v.strip():
raise RuntimeError('spaces around "%s"' % v)
Chatter.write('%s => %s' % (k, v))
from xia2.Handlers.Flags import Flags
Chatter.write('Starting directory: %s' % Flags.get_starting_directory())
Chatter.write('Working directory: %s' % os.getcwd())
# temporary workaround to bug in pointless...
if ' ' in os.getcwd():
raise RuntimeError('Space in working directory ' \
'(https://github.com/xia2/xia2/issues/114)')
Chatter.write('Free space: %.2f GB' % (df() / math.pow(2, 30)))
try:
if os.name == 'nt':
hostname = os.environ['COMPUTERNAME'].split('.')[0]
else:
hostname = os.environ['HOSTNAME'].split('.')[0]
Chatter.write('Host: %s' % hostname)
except KeyError:
pass
Chatter.write('Contact: [email protected]')
Chatter.write(Version)
def __init__(self):
super(XDSScalerA, self).__init__()
self._sweep_information = { }
self._reference = None
# spacegroup and unit cell information - these will be
# derived from an average of all of the sweeps which are
# passed in
self._xds_spacegroup = None
self._factory = CCP4Factory()
self._chef_analysis_groups = { }
self._chef_analysis_times = { }
self._chef_analysis_resolutions = { }
self._user_resolution_limits = { }
# scaling correction choices - may be set one on the command line...
if Flags.get_scale_model():
self._scalr_correct_absorption = Flags.get_scale_model_absorption()
self._scalr_correct_modulation = Flags.get_scale_model_modulation()
self._scalr_correct_decay = Flags.get_scale_model_decay()
self._scalr_corrections = True
else:
self._scalr_correct_decay = True
self._scalr_correct_modulation = True
self._scalr_correct_absorption = True
self._scalr_corrections = True
return
def migrate(self, directory):
'''Migrate (or not) data to a local directory.'''
if not Flags.get_migrate_data():
# we will not migrate this data
return directory
if directory in self._data_migrate.keys():
# we have already migrated this data
return self._data_migrate[directory]
# create a directory to move data to...
self._data_migrate[directory] = tempfile.mkdtemp()
# copy all files in source directory to new directory
# retaining timestamps etc.
start_time = time.time()
migrated = 0
migrated_dir = 0
for f in os.listdir(directory):
# copy over only files....
if os.path.isfile(os.path.join(directory, f)):
shutil.copy2(os.path.join(directory, f),
self._data_migrate[directory])
migrated += 1
elif os.path.isdir(os.path.join(directory, f)):
shutil.copytree(os.path.join(directory, f),
os.path.join(self._data_migrate[directory],
f))
migrated_dir += 1
Debug.write('Migrated %d files from %s to %s' % \
(migrated, directory, self._data_migrate[directory]))
if migrated_dir > 0:
Debug.write('Migrated %d directories from %s to %s' % \
(migrated_dir, directory,
self._data_migrate[directory]))
end_time = time.time()
duration = end_time - start_time
Debug.write('Migration took %s' % \
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
return self._data_migrate[directory]
def _estimate_resolution_limit(self, hklin, batch_range=None):
params = PhilIndex.params.xia2.settings.resolution
m = Merger()
m.set_working_directory(self.get_working_directory())
from xia2.lib.bits import auto_logfiler
auto_logfiler(m)
m.set_hklin(hklin)
m.set_limit_rmerge(params.rmerge)
m.set_limit_completeness(params.completeness)
m.set_limit_cc_half(params.cc_half)
m.set_limit_isigma(params.isigma)
m.set_limit_misigma(params.misigma)
if Flags.get_small_molecule():
m.set_nbins(20)
if batch_range is not None:
start, end = batch_range
m.set_batch_range(start, end)
m.run()
if params.completeness:
r_comp = m.get_resolution_completeness()
else:
r_comp = 0.0
if params.cc_half:
r_cc_half = m.get_resolution_cc_half()
else:
r_cc_half = 0.0
if params.rmerge:
r_rm = m.get_resolution_rmerge()
else:
r_rm = 0.0
if params.isigma:
r_uis = m.get_resolution_isigma()
else:
r_uis = 0.0
if params.misigma:
r_mis = m.get_resolution_misigma()
else:
r_mis = 0.0
resolution = max([r_comp, r_rm, r_uis, r_mis, r_cc_half])
return resolution
def check_environment():
"""Check the environment we are running in..."""
if sys.hexversion < 0x02070000:
raise RuntimeError("Python versions older than 2.7 are not supported")
import cctbx
executable = sys.executable
cctbx_dir = os.sep.join(cctbx.__file__.split(os.sep)[:-3])
# to help wrapper code - print process id...
logger.debug("Process ID: %d", os.getpid())
logger.info("Environment configuration...")
logger.info("Python => %s", executable)
logger.info("CCTBX => %s", cctbx_dir)
ccp4_keys = ["CCP4", "CCP4_SCR"]
for k in ccp4_keys:
v = os.getenv(k)
if not v:
raise RuntimeError("%s not defined - is CCP4 set up?" % k)
if not v == v.strip():
raise RuntimeError('spaces around "%s"' % v)
logger.info(f"{k} => {v}")
from xia2.Handlers.Flags import Flags
logger.info("Starting directory: %s", Flags.get_starting_directory())
logger.info("Working directory: %s", os.getcwd())
logger.info("Free space: %.2f GB", df() / math.pow(2, 30))
hostname = platform.node().split(".")[0]
logger.info("Host: %s", hostname)
logger.info("Contact: [email protected]")
logger.info(Version)
# temporary workaround to bug in pointless...
if " " in os.getcwd():
raise RuntimeError(
"Space in working directory " "(https://github.com/xia2/xia2/issues/114)"
)
def run(self):
from xia2.Handlers.Streams import Debug
Debug.write('Running dials.find_spots')
self.clear_command_line()
self.add_command_line('input.datablock="%s"' % self._input_sweep_filename)
if self._output_sweep_filename is not None:
self.add_command_line(
'output.datablock="%s"' % self._output_sweep_filename)
self.add_command_line('output.reflections="%s"' % self._input_spot_filename)
nproc = Flags.get_parallel()
self.set_cpu_threads(nproc)
self.add_command_line('nproc=%i' % nproc)
for scan_range in self._scan_ranges:
self.add_command_line('spotfinder.scan_range=%d,%d' % scan_range)
if self._min_spot_size is not None:
self.add_command_line('min_spot_size=%i' % self._min_spot_size)
if self._min_local is not None:
self.add_command_line('min_local=%i' % self._min_local)
if self._kernel_size is not None:
self.add_command_line('kernel_size=%i %i' % \
(self._kernel_size, self._kernel_size))
if self._global_threshold is not None:
self.add_command_line('global_threshold=%s' % self._global_threshold)
if self._sigma_strong is not None:
self.add_command_line('sigma_strong=%i' % self._sigma_strong)
if self._filter_ice_rings:
self.add_command_line('ice_rings.filter=%s' % self._filter_ice_rings)
if self._phil_file is not None:
self.add_command_line("%s" % self._phil_file)
if self._write_hot_mask:
self.add_command_line("write_hot_mask=true")
if self._gain:
self.add_command_line("gain=%f" % self._gain)
self.start()
self.close_wait()
self.check_for_errors()
for record in self.get_all_output():
if record.startswith('Saved') and 'reflections to' in record:
self._nspots = int(record.split()[1])
return
def resolution_completeness(self, limit=None, log=None):
'''Compute a resolution limit where completeness < 0.5 (limit if
set) or the full extent of the data. N.B. this completeness is
with respect to the *maximum* completeness in a shell, to reflect
triclinic cases.'''
if limit is None:
limit = Flags.get_completeness()
bins, ranges = self.get_resolution_bins()
s_s = [1.0 / (r[0] * r[0]) for r in reversed(ranges)]
if limit == 0.0:
return 1.0 / math.sqrt(max(s_s))
comp_s = [
self.calculate_completeness(j)
for j, bin in enumerate(reversed(bins))
]
if min(comp_s) > limit:
return 1.0 / math.sqrt(max(s_s))
comp_f = fit(s_s, comp_s, 6)
rlimit = limit * max(comp_s)
if log:
fout = open(log, 'w')
for j, s in enumerate(s_s):
d = 1.0 / math.sqrt(s)
o = comp_s[j]
m = comp_f[j]
fout.write('%f %f %f %f\n' % (s, d, o, m))
fout.close()
try:
r_comp = 1.0 / math.sqrt(interpolate_value(s_s, comp_f, rlimit))
except Exception:
r_comp = 1.0 / math.sqrt(max(s_s))
return r_comp
def resolution_completeness(self, limit = None, log = None):
'''Compute a resolution limit where completeness < 0.5 (limit if
set) or the full extent of the data. N.B. this completeness is
with respect to the *maximum* completeness in a shell, to reflect
triclinic cases.'''
if limit is None:
limit = Flags.get_completeness()
bins, ranges = self.get_resolution_bins()
s_s = [1.0 / (r[0] * r[0]) for r in reversed(ranges)]
if limit == 0.0:
return 1.0 / math.sqrt(max(s_s))
comp_s = [self.calculate_completeness(j) for j, bin in enumerate(
reversed(bins))]
if min(comp_s) > limit:
return 1.0 / math.sqrt(max(s_s))
comp_f = fit(s_s, comp_s, 6)
rlimit = limit * max(comp_s)
if log:
fout = open(log, 'w')
for j, s in enumerate(s_s):
d = 1.0 / math.sqrt(s)
o = comp_s[j]
m = comp_f[j]
fout.write('%f %f %f %f\n' % (s, d, o, m))
fout.close()
try:
r_comp = 1.0 / math.sqrt(
interpolate_value(s_s, comp_f, rlimit))
except:
r_comp = 1.0 / math.sqrt(max(s_s))
return r_comp
def _do_indexing(self, method=None):
indexer = self.Index()
for indxr in self._indxr_indexers:
indexer.add_spot_filename(indxr._indxr_payload["spot_list"])
indexer.add_sweep_filename(indxr._indxr_payload["datablock.json"])
if PhilIndex.params.dials.index.phil_file is not None:
indexer.set_phil_file(PhilIndex.params.dials.index.phil_file)
if PhilIndex.params.dials.index.max_cell:
indexer.set_max_cell(PhilIndex.params.dials.index.max_cell)
if Flags.get_small_molecule():
indexer.set_min_cell(3)
if PhilIndex.params.dials.fix_geometry:
indexer.set_detector_fix('all')
indexer.set_beam_fix('all')
if self._indxr_input_lattice:
indexer.set_indexer_input_lattice(self._indxr_input_lattice)
Debug.write('Set lattice: %s' % self._indxr_input_lattice)
if self._indxr_input_cell:
indexer.set_indexer_input_cell(self._indxr_input_cell)
Debug.write('Set cell: %f %f %f %f %f %f' % \
self._indxr_input_cell)
original_cell = self._indxr_input_cell
if method is None:
if PhilIndex.params.dials.index.method is None:
method = 'fft3d'
Debug.write('Choosing indexing method: %s' % method)
else:
method = PhilIndex.params.dials.index.method
indexer.run(method)
if not os.path.exists(indexer.get_experiments_filename()):
raise RuntimeError("Indexing has failed: %s does not exist."
%indexer.get_experiments_filename())
elif not os.path.exists(indexer.get_indexed_filename()):
raise RuntimeError("Indexing has failed: %s does not exist."
%indexer.get_indexed_filename())
return indexer
def SweepFactory(template, directory, beam = None):
'''A factory which will return a list of sweep objects which match
the input template and directory.'''
sweeps = []
from xia2.Schema import load_imagesets
imagesets = load_imagesets(
template, directory, reversephi=Flags.get_reversephi())
for imageset in imagesets:
scan = imageset.get_scan()
if scan is not None:
sweeps.append(
Sweep(template, directory,
imageset=imageset,
id_image=scan.get_image_range()[0],
beam=beam))
return sweeps
def resolution_rmerge(self, limit=None, log=None):
'''Compute a resolution limit where either rmerge = 1.0 (limit if
set) or the full extent of the data. N.B. this fit is only meaningful
for positive values.'''
if limit is None:
limit = Flags.get_rmerge()
bins, ranges = self.get_resolution_bins()
if limit == 0.0:
return ranges[-1][0]
rmerge_s = get_positive_values(
[self.calculate_rmerge(bin) for bin in bins])
s_s = [1.0 / (r[0] * r[0]) for r in ranges][:len(rmerge_s)]
if limit == 0.0:
return 1.0 / math.sqrt(max(s_s))
if limit > max(rmerge_s):
return 1.0 / math.sqrt(max(s_s))
rmerge_f = log_inv_fit(s_s, rmerge_s, 6)
if log:
fout = open(log, 'w')
for j, s in enumerate(s_s):
d = 1.0 / math.sqrt(s)
o = rmerge_s[j]
m = rmerge_f[j]
fout.write('%f %f %f %f\n' % (s, d, o, m))
fout.close()
try:
r_rmerge = 1.0 / math.sqrt(interpolate_value(s_s, rmerge_f, limit))
except Exception:
r_rmerge = 1.0 / math.sqrt(max(s_s))
return r_rmerge
def resolution_rmerge(self, limit = None, log = None):
'''Compute a resolution limit where either rmerge = 1.0 (limit if
set) or the full extent of the data. N.B. this fit is only meaningful
for positive values.'''
if limit is None:
limit = Flags.get_rmerge()
bins, ranges = self.get_resolution_bins()
if limit == 0.0:
return ranges[-1][0]
rmerge_s = get_positive_values(
[self.calculate_rmerge(bin) for bin in bins])
s_s = [1.0 / (r[0] * r[0]) for r in ranges][:len(rmerge_s)]
if limit == 0.0:
return 1.0 / math.sqrt(max(s_s))
if limit > max(rmerge_s):
return 1.0 / math.sqrt(max(s_s))
rmerge_f = log_inv_fit(s_s, rmerge_s, 6)
if log:
fout = open(log, 'w')
for j, s in enumerate(s_s):
d = 1.0 / math.sqrt(s)
o = rmerge_s[j]
m = rmerge_f[j]
fout.write('%f %f %f %f\n' % (s, d, o, m))
fout.close()
try:
r_rmerge = 1.0 / math.sqrt(interpolate_value(s_s, rmerge_f, limit))
except:
r_rmerge = 1.0 / math.sqrt(max(s_s))
return r_rmerge
def __init__(self, params=None):
super(XDSColspotWrapper, self).__init__()
# phil parameters
if not params:
from xia2.Handlers.Phil import master_phil
params = master_phil.extract().xds.colspot
self._params = params
# now set myself up...
self._parallel = Flags.get_parallel()
self.set_cpu_threads(self._parallel)
if self._parallel <= 1:
self.set_executable('xds')
else:
self.set_executable('xds_par')
# generic bits
self._data_range = (0, 0)
self._spot_range = []
self._background_range = (0, 0)
self._resolution_range = (0, 0)
self._input_data_files = { }
self._output_data_files = { }
self._input_data_files_list = ['X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf',
'BLANK.cbf',
'BKGINIT.cbf',
'GAIN.cbf']
self._output_data_files_list = ['SPOT.XDS']
return
def decide_pointgroup(self):
'''Decide on the correct pointgroup for hklin.'''
if not self._xdsin:
self.check_hklin()
self.set_task('Computing the correct pointgroup for %s' % \
self.get_hklin())
else:
Debug.write('Pointless using XDS input file %s' % \
self._xdsin)
self.set_task('Computing the correct pointgroup for %s' % \
self.get_xdsin())
# FIXME this should probably be a standard CCP4 keyword
if self._xdsin:
self.add_command_line('xdsin')
self.add_command_line(self._xdsin)
self.add_command_line('xmlout')
self.add_command_line('%d_pointless.xml' % self.get_xpid())
if self._hklref:
self.add_command_line('hklref')
self.add_command_line(self._hklref)
self.start()
self.input('systematicabsences off')
self.input('setting symmetry-based')
if self._hklref:
from xia2.Handlers.Phil import PhilIndex
dev = PhilIndex.params.xia2.settings.developmental
if dev.pointless_tolerance > 0.0:
self.input('tolerance %f' % dev.pointless_tolerance)
# may expect more %age variation for small molecule data
if Flags.get_small_molecule() and self._hklref:
self.input('tolerance 5.0')
if Flags.get_small_molecule():
self.input('chirality nonchiral')
if self._input_laue_group:
self.input('lauegroup %s' % self._input_laue_group)
self.close_wait()
# check for errors
self.check_for_errors()
# check for fatal errors
output = self.get_all_output()
for j, record in enumerate(output):
if 'FATAL ERROR message:' in record:
raise RuntimeError, 'Pointless error: %s' % output[j+1].strip()
hklin_spacegroup = ''
hklin_lattice = ''
for o in self.get_all_output():
if 'Spacegroup from HKLIN file' in o:
# hklin_spacegroup = o.split(':')[-1].strip()
hklin_spacegroup = spacegroup_name_xHM_to_old(
o.replace(
'Spacegroup from HKLIN file :', '').strip())
hklin_lattice = Syminfo.get_lattice(hklin_spacegroup)
if 'No alternative indexing possible' in o:
# then the XML file will be broken - no worries...
self._pointgroup = hklin_spacegroup
self._confidence = 1.0
self._totalprob = 1.0
self._reindex_matrix = [1.0, 0.0, 0.0,
0.0, 1.0, 0.0,
0.0, 0.0, 1.0]
self._reindex_operator = 'h,k,l'
return 'ok'
if '**** Incompatible symmetries ****' in o:
raise RuntimeError, \
'reindexing against a reference with different symmetry'
if '***** Stopping because cell discrepancy between files' in o:
raise RuntimeError, 'incompatible unit cells between data sets'
if 'L-test suggests that the data may be twinned' in o:
self._probably_twinned = True
# parse the XML file for the information I need...
xml_file = os.path.join(self.get_working_directory(),
'%d_pointless.xml' % self.get_xpid())
mend_pointless_xml(xml_file)
# catch the case sometimes on ppc mac where pointless adds
# an extra .xml on the end...
if not os.path.exists(xml_file) and \
os.path.exists('%s.xml' % xml_file):
xml_file = '%s.xml' % xml_file
if not self._hklref:
dom = xml.dom.minidom.parse(xml_file)
try:
best = dom.getElementsByTagName('BestSolution')[0]
except IndexError, e:
raise RuntimeError, 'error getting solution from pointless'
self._pointgroup = best.getElementsByTagName(
'GroupName')[0].childNodes[0].data
self._confidence = float(best.getElementsByTagName(
'Confidence')[0].childNodes[0].data)
self._totalprob = float(best.getElementsByTagName(
'TotalProb')[0].childNodes[0].data)
self._reindex_matrix = map(float, best.getElementsByTagName(
'ReindexMatrix')[0].childNodes[0].data.split())
self._reindex_operator = clean_reindex_operator(
best.getElementsByTagName(
'ReindexOperator')[0].childNodes[0].data.strip())
def exercise_dials_integrater(nproc=None):
if not have_dials_regression:
print "Skipping exercise_dials_integrater(): dials_regression not configured"
return
if nproc is not None:
from xia2.Handlers.Flags import Flags
Flags.set_parallel(nproc)
xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data")
template = os.path.join(xia2_demo_data, "insulin_1_###.img")
cwd = os.path.abspath(os.curdir)
tmp_dir = os.path.abspath(open_tmp_directory())
os.chdir(tmp_dir)
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater
from dxtbx.datablock import DataBlockTemplateImporter
indexer = DialsIndexer()
indexer.set_working_directory(tmp_dir)
importer = DataBlockTemplateImporter([template])
datablocks = importer.datablocks
imageset = datablocks[0].extract_imagesets()[0]
indexer.add_indexer_imageset(imageset)
from xia2.Schema.XCrystal import XCrystal
from xia2.Schema.XWavelength import XWavelength
from xia2.Schema.XSweep import XSweep
from xia2.Schema.XSample import XSample
cryst = XCrystal("CRYST1", None)
wav = XWavelength("WAVE1", cryst, imageset.get_beam().get_wavelength())
samp = XSample("X1", cryst)
directory, image = os.path.split(imageset.get_path(1))
sweep = XSweep('SWEEP1', wav, samp, directory=directory, image=image)
indexer.set_indexer_sweep(sweep)
from xia2.Modules.Refiner.DialsRefiner import DialsRefiner
refiner = DialsRefiner()
refiner.set_working_directory(tmp_dir)
refiner.add_refiner_indexer(sweep.get_epoch(1), indexer)
#refiner.refine()
integrater = DialsIntegrater()
integrater.set_working_directory(tmp_dir)
integrater.setup_from_image(imageset.get_path(1))
integrater.set_integrater_refiner(refiner)
#integrater.set_integrater_indexer(indexer)
integrater.set_integrater_sweep(sweep)
integrater.integrate()
integrater_intensities = integrater.get_integrater_intensities()
assert os.path.exists(integrater_intensities)
from iotbx.reflection_file_reader import any_reflection_file
reader = any_reflection_file(integrater_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections()
assert mtz_object.column_labels() == [
'H', 'K', 'L', 'M_ISYM', 'BATCH', 'IPR', 'SIGIPR', 'I', 'SIGI',
'FRACTIONCALC', 'XDET', 'YDET', 'ROT', 'LP', 'DQE']
assert integrater.get_integrater_wedge() == (1, 45)
assert approx_equal(integrater.get_integrater_cell(),
(78.14, 78.14, 78.14, 90, 90, 90), eps=1e-1)
# test serialization of integrater
json_str = integrater.as_json()
#print json_str
integrater2 = DialsIntegrater.from_json(string=json_str)
integrater2.set_integrater_sweep(sweep, reset=False)
integrater2_intensities = integrater.get_integrater_intensities()
assert integrater2_intensities == integrater_intensities
integrater2.set_integrater_finish_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections()
integrater2.set_integrater_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections()
integrater2.set_integrater_prepare_done(False)
integrater2_intensities = integrater2.get_integrater_intensities()
assert os.path.exists(integrater2_intensities)
reader = any_reflection_file(integrater2_intensities)
assert reader.file_type() == "ccp4_mtz"
mtz_object = reader.file_content()
assert abs(mtz_object.n_reflections() - 48117) < 100, mtz_object.n_reflections()
def setup(self):
'''Set everything up...'''
# check arguments are all ascii
Debug.write('Start parsing command line: ' + str(sys.argv))
for token in sys.argv:
try:
token.encode('ascii')
except UnicodeDecodeError:
raise RuntimeError('non-ascii characters in input')
self._argv = copy.deepcopy(sys.argv)
replacements = {
'-2d': 'pipeline=2d',
'-2di': 'pipeline=2di',
'-3d': 'pipeline=3d',
'-3di': 'pipeline=3di',
'-3dii': 'pipeline=3dii',
'-3dd': 'pipeline=3dd',
'-dials': 'pipeline=dials',
'-quick': 'dials.fast_mode=true',
'-failover': 'failover=true',
'-small_molecule': 'small_molecule=true'
}
for k, v in replacements.iteritems():
if k in self._argv:
print "***\nCommand line option %s is deprecated.\nPlease use %s instead\n***" % (
k, v)
self._argv[self._argv.index(k)] = v
if '-atom' in self._argv:
idx = self._argv.index('-atom')
element = self._argv[idx + 1]
self._argv[idx:idx + 2] = ['atom=%s' % element]
print "***\nCommand line option -atom %s is deprecated.\nPlease use atom=%s instead\n***" % (
element, element)
# first of all try to interpret arguments as phil parameters/files
from xia2.Handlers.Phil import master_phil
from libtbx.phil import command_line
cmd_line = command_line.argument_interpreter(master_phil=master_phil)
working_phil, self._argv = cmd_line.process_and_fetch(
args=self._argv, custom_processor="collect_remaining")
PhilIndex.merge_phil(working_phil)
try:
params = PhilIndex.get_python_object()
except RuntimeError as e:
raise Sorry(e)
# sanity check / interpret Auto in input
from libtbx import Auto
if params.xia2.settings.input.atom is None:
if params.xia2.settings.input.anomalous is Auto:
PhilIndex.update("xia2.settings.input.anomalous=false")
else:
if params.xia2.settings.input.anomalous == False:
raise Sorry(
'Setting anomalous=false and atom type inconsistent')
params.xia2.settings.input.anomalous = True
PhilIndex.update("xia2.settings.input.anomalous=true")
if params.xia2.settings.resolution.keep_all_reflections is Auto:
if params.xia2.settings.small_molecule == True and \
params.xia2.settings.resolution.d_min is None and \
params.xia2.settings.resolution.d_max is None:
PhilIndex.update(
"xia2.settings.resolution.keep_all_reflections=true")
else:
PhilIndex.update(
"xia2.settings.resolution.keep_all_reflections=false")
if params.xia2.settings.small_molecule == True:
Debug.write('Small molecule selected')
if params.ccp4.pointless.chirality is None:
PhilIndex.update("ccp4.pointless.chirality=nonchiral")
params = PhilIndex.get_python_object()
# pipeline options
self._read_pipeline()
Debug.write('Project: %s' % params.xia2.settings.project)
Debug.write('Crystal: %s' % params.xia2.settings.crystal)
# FIXME add some consistency checks in here e.g. that there are
# images assigned, there is a lattice assigned if cell constants
# are given and so on
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
from xia2.Handlers.Environment import get_number_cpus
if mp_params.mode == 'parallel':
if mp_params.type == 'qsub':
if which('qsub') is None:
raise Sorry('qsub not available')
if mp_params.njob is Auto:
mp_params.njob = get_number_cpus()
if mp_params.nproc is Auto:
mp_params.nproc = 1
elif mp_params.nproc is Auto:
mp_params.nproc = get_number_cpus()
elif mp_params.mode == 'serial':
if mp_params.type == 'qsub':
if which('qsub') is None:
raise Sorry('qsub not available')
if mp_params.njob is Auto:
mp_params.njob = 1
if mp_params.nproc is Auto:
mp_params.nproc = get_number_cpus()
PhilIndex.update("xia2.settings.multiprocessing.njob=%d" %
mp_params.njob)
PhilIndex.update("xia2.settings.multiprocessing.nproc=%d" %
mp_params.nproc)
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
if params.xia2.settings.indexer is not None:
add_preference("indexer", params.xia2.settings.indexer)
if params.xia2.settings.refiner is not None:
add_preference("refiner", params.xia2.settings.refiner)
if params.xia2.settings.integrater is not None:
add_preference("integrater", params.xia2.settings.integrater)
if params.xia2.settings.scaler is not None:
add_preference("scaler", params.xia2.settings.scaler)
if params.xia2.settings.multi_sweep_indexing is Auto:
if params.xia2.settings.small_molecule == True and 'dials' == params.xia2.settings.indexer:
PhilIndex.update("xia2.settings.multi_sweep_indexing=True")
else:
PhilIndex.update("xia2.settings.multi_sweep_indexing=False")
if params.xia2.settings.multi_sweep_indexing == True and \
params.xia2.settings.multiprocessing.mode == 'parallel':
Chatter.write(
'Multi sweep indexing disabled:\nMSI is not available for parallel processing.'
)
PhilIndex.update("xia2.settings.multi_sweep_indexing=False")
input_json = params.xia2.settings.input.json
if (input_json is not None and len(input_json)):
for json_file in input_json:
assert os.path.isfile(json_file)
load_datablock(json_file)
reference_geometry = params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
reference_geometries = "\n".join([
"xia2.settings.input.reference_geometry=%s" %
os.path.abspath(g)
for g in params.xia2.settings.input.reference_geometry
])
Debug.write(reference_geometries)
PhilIndex.update(reference_geometries)
Debug.write("xia2.settings.trust_beam_centre=true")
PhilIndex.update("xia2.settings.trust_beam_centre=true")
params = PhilIndex.get_python_object()
params = PhilIndex.get_python_object()
if params.xia2.settings.input.xinfo is not None:
xinfo_file = os.path.abspath(params.xia2.settings.input.xinfo)
PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file)
params = PhilIndex.get_python_object()
self.set_xinfo(xinfo_file)
# issue #55 if not set ATOM in xinfo but anomalous=true or atom= set
# on commandline, set here, should be idempotent
if params.xia2.settings.input.anomalous is True:
crystals = self._xinfo.get_crystals()
for xname in crystals:
xtal = crystals[xname]
Debug.write("Setting anomalous for crystal %s" % xname)
xtal.set_anomalous(True)
else:
xinfo_file = '%s/automatic.xinfo' % os.path.abspath(os.curdir)
PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file)
params = PhilIndex.get_python_object()
if params.dials.find_spots.phil_file is not None:
PhilIndex.update(
"dials.find_spots.phil_file=%s" %
os.path.abspath(params.dials.find_spots.phil_file))
if params.dials.index.phil_file is not None:
PhilIndex.update("dials.index.phil_file=%s" %
os.path.abspath(params.dials.index.phil_file))
if params.dials.refine.phil_file is not None:
PhilIndex.update("dials.refine.phil_file=%s" %
os.path.abspath(params.dials.refine.phil_file))
if params.dials.integrate.phil_file is not None:
PhilIndex.update("dials.integrate.phil_file=%s" %
os.path.abspath(params.dials.integrate.phil_file))
if params.xds.index.xparm is not None:
Flags.set_xparm(params.xds.index.xparm)
if params.xds.index.xparm_ub is not None:
Flags.set_xparm_ub(params.xds.index.xparm_ub)
if params.xia2.settings.scale.freer_file is not None:
freer_file = os.path.abspath(params.xia2.settings.scale.freer_file)
if not os.path.exists(freer_file):
raise RuntimeError('%s does not exist' % freer_file)
from xia2.Modules.FindFreeFlag import FindFreeFlag
column = FindFreeFlag(freer_file)
Debug.write('FreeR_flag column in %s found: %s' % \
(freer_file, column))
PhilIndex.update("xia2.settings.scale.freer_file=%s" % freer_file)
if params.xia2.settings.scale.reference_reflection_file is not None:
reference_reflection_file = os.path.abspath(
params.xia2.settings.scale.reference_reflection_file)
if not os.path.exists(reference_reflection_file):
raise RuntimeError('%s does not exist' %
reference_reflection_file)
PhilIndex.update(
"xia2.settings.scale.reference_reflection_file=%s" %
reference_reflection_file)
params = PhilIndex.get_python_object()
datasets = unroll_datasets(PhilIndex.params.xia2.settings.input.image)
for dataset in datasets:
start_end = None
if ':' in dataset:
tokens = dataset.split(':')
# cope with windows drives i.e. C:\data\blah\thing_0001.cbf:1:100
if len(tokens[0]) == 1:
tokens = ['%s:%s' % (tokens[0], tokens[1])] + tokens[2:]
if len(tokens) != 3:
raise RuntimeError('/path/to/image_0001.cbf:start:end')
dataset = tokens[0]
start_end = int(tokens[1]), int(tokens[2])
from xia2.Applications.xia2setup import is_hd5f_name
if os.path.exists(os.path.abspath(dataset)):
dataset = os.path.abspath(dataset)
else:
directories = [os.getcwd()] + self._argv[1:]
found = False
for d in directories:
if os.path.exists(os.path.join(d, dataset)):
dataset = os.path.join(d, dataset)
found = True
break
if not found:
raise Sorry('Cound not find %s in %s' % \
(dataset, ' '.join(directories)))
if is_hd5f_name(dataset):
self._hdf5_master_files.append(dataset)
if start_end:
Debug.write('Image range: %d %d' % start_end)
if not dataset in self._default_start_end:
self._default_start_end[dataset] = []
self._default_start_end[dataset].append(start_end)
else:
Debug.write('No image range specified')
else:
template, directory = image2template_directory(
os.path.abspath(dataset))
self._default_template.append(os.path.join(
directory, template))
self._default_directory.append(directory)
Debug.write('Interpreted from image %s:' % dataset)
Debug.write('Template %s' % template)
Debug.write('Directory %s' % directory)
if start_end:
Debug.write('Image range: %d %d' % start_end)
key = os.path.join(directory, template)
if not key in self._default_start_end:
self._default_start_end[key] = []
self._default_start_end[key].append(start_end)
else:
Debug.write('No image range specified')
# finally, check that all arguments were read and raise an exception
# if any of them were nonsense.
with open('xia2-working.phil', 'wb') as f:
print >> f, PhilIndex.working_phil.as_str()
with open('xia2-diff.phil', 'wb') as f:
print >> f, PhilIndex.get_diff().as_str()
Debug.write('\nDifference PHIL:')
Debug.write(PhilIndex.get_diff().as_str(), strip=False)
Debug.write('Working PHIL:')
Debug.write(PhilIndex.working_phil.as_str(), strip=False)
nonsense = 'Unknown command-line options:'
was_nonsense = False
for j, argv in enumerate(self._argv):
if j == 0:
continue
if argv[0] != '-' and '=' not in argv:
continue
if not j in self._understood:
nonsense += ' %s' % argv
was_nonsense = True
if was_nonsense:
raise RuntimeError(nonsense)
integrater = XDSIntegrater()
Debug.write('Using XDS Integrater in new resolution mode')
except NotAvailableError, e:
if preselection == 'xdsr':
raise RuntimeError, \
'preselected integrater xdsr not available: ' + \
'xds not installed?'
pass
if not integrater:
raise RuntimeError, 'no integrater implementations found'
# check to see if resolution limits were passed in through the
# command line...
dmin = Flags.get_resolution_high()
dmax = Flags.get_resolution_low()
if dmin:
Debug.write('Adding user-assigned resolution limits:')
if dmax:
Debug.write('dmin: %.3f dmax: %.2f' % (dmin, dmax))
integrater.set_integrater_resolution(dmin, dmax, user = True)
else:
Debug.write('dmin: %.3f' % dmin)
integrater.set_integrater_high_resolution(dmin, user = True)
def run(self):
'''Actually run XSCALE.'''
self._write_xscale_inp()
# copy the input file...
shutil.copyfile(os.path.join(self.get_working_directory(),
'XSCALE.INP'),
os.path.join(self.get_working_directory(),
'%d_XSCALE.INP' % self.get_xpid()))
self.start()
self.close_wait()
# copy the LP file
shutil.copyfile(os.path.join(self.get_working_directory(),
'XSCALE.LP'),
os.path.join(self.get_working_directory(),
'%d_XSCALE.LP' % self.get_xpid()))
# now look at XSCALE.LP
xds_check_error(self.get_all_output())
dname = None
# get the outlier reflections... and the overall scale factor
for line in open(os.path.join(
self.get_working_directory(),
'XSCALE.LP'), 'r').readlines():
if '"alien"' in line:
h, k, l = tuple(map(int, line.split()[:3]))
z = float(line.split()[4])
if not (h, k, l, z) in self._remove:
self._remove.append((h, k, l, z))
if 'FACTOR TO PLACE ALL DATA SETS TO ' in line:
self._scale_factor = float(line.split()[-1])
if 'STATISTICS OF SCALED OUTPUT DATA SET' in line:
dname = line.split()[-1].replace('.HKL', '')
if 'total' in line and not dname in self._rmerges:
if len(line.split()) > 5:
self._rmerges[dname] = float(
line.replace('%', '').split()[5])
# trac #419 - if the data sets are not correctly indexed,
# throw an exception. N.B. this will only work if the
# data sets are moderately complete (i.e. there are more
# than a handful of common reflections) - which may not be
# the case in MULTICRYSTAL mode.
if ' !!! WARNING !!! ' in line and \
'CORRELATION FACTORS ARE DANGEROUSLY SMALL' in line:
groups = get_correlation_coefficients_and_group(
os.path.join(self.get_working_directory(),
'XSCALE.LP'))
Debug.write('Low correlations - check data sets')
for j, name in enumerate(groups):
Debug.write('Group %d' % j)
for file_name in groups[name]:
Debug.write(file_name)
if not Flags.get_microcrystal():
raise RuntimeError, 'reindexing error: %s' % \
os.path.join(self.get_working_directory(),
'XSCALE.LP')
return
def _integrate_finish(self):
'''Finish off the integration by running correct.'''
# first run the postrefinement etc with spacegroup P1
# and the current unit cell - this will be used to
# obtain a benchmark rmsd in pixels / phi and also
# cell deviations (this is working towards spotting bad
# indexing solutions) - only do this if we have no
# reindex matrix... and no postrefined cell...
p1_deviations = None
# fix for bug # 3264 -
# if we have not run integration with refined parameters, make it so...
# erm? shouldn't this therefore return if this is the principle, or
# set the flag after we have tested the lattice?
if not self._xds_data_files.has_key('GXPARM.XDS') and \
PhilIndex.params.xds.integrate.reintegrate:
Debug.write(
'Resetting integrater, to ensure refined orientation is used')
self.set_integrater_done(False)
if not self.get_integrater_reindex_matrix() and not self._intgr_cell \
and not Flags.get_no_lattice_test() and \
not self.get_integrater_sweep().get_user_lattice():
correct = self.Correct()
correct.set_data_range(self._intgr_wedge[0],
self._intgr_wedge[1])
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# FIXME should this be using the correctly transformed
# cell or are the results ok without it?!
correct.set_spacegroup_number(1)
correct.set_cell(self._intgr_refiner_cell)
correct.run()
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file('%s %s %s %s CORRECT' % \
(pname, xname, dname, sweep),
os.path.join(
self.get_working_directory(),
'CORRECT.LP'))
FileHandler.record_more_data_file(
'%s %s %s %s CORRECT' % (pname, xname, dname, sweep),
os.path.join(self.get_working_directory(), 'XDS_ASCII.HKL'))
cell = correct.get_result('cell')
cell_esd = correct.get_result('cell_esd')
Debug.write('Postrefinement in P1 results:')
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(cell))
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(cell_esd))
Debug.write('Deviations: %.2f pixels %.2f degrees' % \
(correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi')))
p1_deviations = (correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi'))
# next run the postrefinement etc with the given
# cell / lattice - this will be the assumed result...
correct = self.Correct()
correct.set_data_range(self._intgr_wedge[0],
self._intgr_wedge[1])
if self.get_polarization() > 0.0:
correct.set_polarization(self.get_polarization())
# BUG # 2695 probably comes from here - need to check...
# if the pointless interface comes back with a different
# crystal setting then the unit cell stored in self._intgr_cell
# needs to be set to None...
if self.get_integrater_spacegroup_number():
correct.set_spacegroup_number(
self.get_integrater_spacegroup_number())
if not self._intgr_cell:
raise RuntimeError, 'no unit cell to recycle'
correct.set_cell(self._intgr_cell)
# BUG # 3113 - new version of XDS will try and figure the
# best spacegroup out from the intensities (and get it wrong!)
# unless we set the spacegroup and cell explicitly
if not self.get_integrater_spacegroup_number():
cell = self._intgr_refiner_cell
lattice = self._intgr_refiner.get_refiner_lattice()
spacegroup_number = lattice_to_spacegroup_number(lattice)
# this should not prevent the postrefinement from
# working correctly, else what is above would not
# work correctly (the postrefinement test)
correct.set_spacegroup_number(spacegroup_number)
correct.set_cell(cell)
Debug.write('Setting spacegroup to: %d' % spacegroup_number)
Debug.write(
'Setting cell to: %.2f %.2f %.2f %.2f %.2f %.2f' % tuple(cell))
if self.get_integrater_reindex_matrix():
# bug! if the lattice is not primitive the values in this
# reindex matrix need to be multiplied by a constant which
# depends on the Bravais lattice centering.
lattice = self._intgr_refiner.get_refiner_lattice()
import scitbx.matrix
matrix = self.get_integrater_reindex_matrix()
matrix = scitbx.matrix.sqr(matrix).transpose().elems
matrix = r_to_rt(matrix)
if lattice[1] == 'P':
mult = 1
elif lattice[1] == 'C' or lattice[1] == 'I':
mult = 2
elif lattice[1] == 'R':
mult = 3
elif lattice[1] == 'F':
mult = 4
else:
raise RuntimeError, 'unknown multiplier for lattice %s' % \
lattice
Debug.write('REIDX multiplier for lattice %s: %d' % \
(lattice, mult))
mult_matrix = [mult * m for m in matrix]
Debug.write('REIDX set to %d %d %d %d %d %d %d %d %d %d %d %d' % \
tuple(mult_matrix))
correct.set_reindex_matrix(mult_matrix)
correct.run()
# erm. just to be sure
if self.get_integrater_reindex_matrix() and \
correct.get_reindex_used():
raise RuntimeError, 'Reindex panic!'
# get the reindex operation used, which may be useful if none was
# set but XDS decided to apply one, e.g. #419.
if not self.get_integrater_reindex_matrix() and \
correct.get_reindex_used():
# convert this reindex operation to h, k, l form: n.b. this
# will involve dividing through by the lattice centring multiplier
matrix = rt_to_r(correct.get_reindex_used())
import scitbx.matrix
matrix = scitbx.matrix.sqr(matrix).transpose().elems
lattice = self._intgr_refiner.get_refiner_lattice()
if lattice[1] == 'P':
mult = 1.0
elif lattice[1] == 'C' or lattice[1] == 'I':
mult = 2.0
elif lattice[1] == 'R':
mult = 3.0
elif lattice[1] == 'F':
mult = 4.0
matrix = [m / mult for m in matrix]
reindex_op = mat_to_symop(matrix)
# assign this to self: will this reset?! make for a leaky
# abstraction and just assign this...
# self.set_integrater_reindex_operator(reindex)
self._intgr_reindex_operator = reindex_op
# record the log file -
pname, xname, dname = self.get_integrater_project_info()
sweep = self.get_integrater_sweep_name()
FileHandler.record_log_file('%s %s %s %s CORRECT' % \
(pname, xname, dname, sweep),
os.path.join(self.get_working_directory(),
'CORRECT.LP'))
# should get some interesting stuff from the XDS correct file
# here, for instance the resolution range to use in integration
# (which should be fed back if not fast) and so on...
self._intgr_corrected_hklout = os.path.join(self.get_working_directory(),
'XDS_ASCII.HKL')
# also record the batch range - needed for the analysis of the
# radiation damage in chef...
self._intgr_batches_out = (self._intgr_wedge[0],
self._intgr_wedge[1])
# FIXME perhaps I should also feedback the GXPARM file here??
for file in ['GXPARM.XDS']:
self._xds_data_files[file] = correct.get_output_data_file(file)
# record the postrefined cell parameters
self._intgr_cell = correct.get_result('cell')
self._intgr_n_ref = correct.get_result('n_ref')
Debug.write('Postrefinement in "correct" spacegroup results:')
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(correct.get_result('cell')))
Debug.write('%7.3f %7.3f %7.3f %7.3f %7.3f %7.3f' % \
tuple(correct.get_result('cell_esd')))
Debug.write('Deviations: %.2f pixels %.2f degrees' % \
(correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi')))
Debug.write('Error correction parameters: A=%.3f B=%.3f' % \
correct.get_result('sdcorrection'))
# compute misorientation of axes
xparm_file = os.path.join(self.get_working_directory(), 'GXPARM.XDS')
from iotbx.xds import xparm
handle = xparm.reader()
handle.read_file(xparm_file)
rotn = handle.rotation_axis
beam = handle.beam_vector
dot = sum([rotn[j] * beam[j] for j in range(3)])
r = math.sqrt(sum([rotn[j] * rotn[j] for j in range(3)]))
b = math.sqrt(sum([beam[j] * beam[j] for j in range(3)]))
rtod = 180.0 / math.pi
angle = rtod * math.fabs(0.5 * math.pi - math.acos(dot / (r * b)))
Debug.write('Axis misalignment %.2f degrees' % angle)
correct_deviations = (correct.get_result('rmsd_pixel'),
correct.get_result('rmsd_phi'))
if p1_deviations:
# compare and reject if both > 50% higher - though adding a little
# flexibility - 0.5 pixel / osc width slack.
pixel = p1_deviations[0]
phi = math.sqrt(0.05 * 0.05 + \
p1_deviations[1] * p1_deviations[1])
threshold = Flags.get_rejection_threshold()
Debug.write('RMSD ratio: %.2f' % (correct_deviations[0] / pixel))
Debug.write('RMSPhi ratio: %.2f' % (correct_deviations[1] / phi))
if correct_deviations[0] / pixel > threshold and \
correct_deviations[1] / phi > threshold:
Chatter.write(
'Eliminating this indexing solution as postrefinement')
Chatter.write(
'deviations rather high relative to triclinic')
raise BadLatticeError, \
'high relative deviations in postrefinement'
if not Flags.get_quick() and Flags.get_remove():
# check for alien reflections and perhaps recycle - removing them
if len(correct.get_remove()) > 0:
correct_remove = correct.get_remove()
current_remove = []
final_remove = []
# first ensure that there are no duplicate entries...
if os.path.exists(os.path.join(
self.get_working_directory(),
'REMOVE.HKL')):
for line in open(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'), 'r').readlines():
h, k, l = map(int, line.split()[:3])
z = float(line.split()[3])
if not (h, k, l, z) in current_remove:
current_remove.append((h, k, l, z))
for c in correct_remove:
if c in current_remove:
continue
final_remove.append(c)
Debug.write(
'%d alien reflections are already removed' % \
(len(correct_remove) - len(final_remove)))
else:
# we want to remove all of the new dodgy reflections
final_remove = correct_remove
remove_hkl = open(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'), 'w')
z_min = Flags.get_z_min()
rejected = 0
# write in the old reflections
for remove in current_remove:
z = remove[3]
if z >= z_min:
remove_hkl.write('%d %d %d %f\n' % remove)
else:
rejected += 1
Debug.write('Wrote %d old reflections to REMOVE.HKL' % \
(len(current_remove) - rejected))
Debug.write('Rejected %d as z < %f' % \
(rejected, z_min))
# and the new reflections
rejected = 0
used = 0
for remove in final_remove:
z = remove[3]
if z >= z_min:
used += 1
remove_hkl.write('%d %d %d %f\n' % remove)
else:
rejected += 1
Debug.write('Wrote %d new reflections to REMOVE.HKL' % \
(len(final_remove) - rejected))
Debug.write('Rejected %d as z < %f' % \
(rejected, z_min))
remove_hkl.close()
# we want to rerun the finishing step so...
# unless we have added no new reflections... or unless we
# have not confirmed the point group (see SCI-398)
if used and self.get_integrater_reindex_matrix():
self.set_integrater_finish_done(False)
else:
Debug.write(
'Going quickly so not removing %d outlier reflections...' % \
len(correct.get_remove()))
# Convert INTEGRATE.HKL to MTZ format and reapply any reindexing operations
# spacegroup changes to allow use with CCP4 / Aimless for scaling
integrate_hkl = os.path.join(
self.get_working_directory(), 'INTEGRATE.HKL')
hklout = os.path.splitext(integrate_hkl)[0] + ".mtz"
self._factory.set_working_directory(self.get_working_directory())
pointless = self._factory.Pointless()
pointless.set_xdsin(integrate_hkl)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
integrate_mtz = hklout
if self.get_integrater_reindex_operator() or \
self.get_integrater_spacegroup_number():
Debug.write('Reindexing things to MTZ')
reindex = Reindex()
reindex.set_working_directory(self.get_working_directory())
auto_logfiler(reindex)
if self.get_integrater_reindex_operator():
reindex.set_operator(self.get_integrater_reindex_operator())
if self.get_integrater_spacegroup_number():
reindex.set_spacegroup(self.get_integrater_spacegroup_number())
hklout = '%s_reindex.mtz' % os.path.splitext(integrate_mtz)[0]
reindex.set_hklin(integrate_mtz)
reindex.set_hklout(hklout)
reindex.reindex()
integrate_mtz = hklout
return integrate_mtz
def decide_spacegroup(self):
'''Given data indexed in the correct pointgroup, have a
guess at the spacegroup.'''
if not self._xdsin:
self.check_hklin()
self.set_task('Computing the correct spacegroup for %s' % \
self.get_hklin())
else:
Debug.write('Pointless using XDS input file %s' % \
self._xdsin)
self.set_task('Computing the correct spacegroup for %s' % \
self.get_xdsin())
# FIXME this should probably be a standard CCP4 keyword
if self._xdsin:
self.add_command_line('xdsin')
self.add_command_line(self._xdsin)
self.add_command_line('xmlout')
self.add_command_line('%d_pointless.xml' % self.get_xpid())
self.add_command_line('hklout')
self.add_command_line('pointless.mtz')
self.start()
self.input('lauegroup hklin')
self.input('setting symmetry-based')
if Flags.get_small_molecule():
self.input('chirality nonchiral')
self.close_wait()
# check for errors
self.check_for_errors()
hklin_spacegroup = ''
xml_file = os.path.join(self.get_working_directory(),
'%d_pointless.xml' % self.get_xpid())
mend_pointless_xml(xml_file)
if not os.path.exists(xml_file) and \
os.path.exists('%s.xml' % xml_file):
xml_file = '%s.xml' % xml_file
dom = xml.dom.minidom.parse(xml_file)
sg_list = dom.getElementsByTagName('SpacegroupList')[0]
sg_node = sg_list.getElementsByTagName('Spacegroup')[0]
best_prob = float(sg_node.getElementsByTagName(
'TotalProb')[0].childNodes[0].data.strip())
# FIXME 21/NOV/06 in here record a list of valid spacegroups
# (that is, those which are as likely as the most likely)
# for later use...
self._spacegroup = sg_node.getElementsByTagName(
'SpacegroupName')[0].childNodes[0].data.strip()
self._spacegroup_reindex_operator = sg_node.getElementsByTagName(
'ReindexOperator')[0].childNodes[0].data.strip()
self._spacegroup_reindex_matrix = tuple(
map(float, sg_node.getElementsByTagName(
'ReindexMatrix')[0].childNodes[0].data.split()))
# get a list of "equally likely" spacegroups
for node in sg_list.getElementsByTagName('Spacegroup'):
prob = float(node.getElementsByTagName(
'TotalProb')[0].childNodes[0].data.strip())
name = node.getElementsByTagName(
'SpacegroupName')[0].childNodes[0].data.strip()
if math.fabs(prob - best_prob) < 0.01:
# this is jolly likely!
self._likely_spacegroups.append(name)
# now parse the output looking for the unit cell information -
# this should look familiar from mtzdump
output = self.get_all_output()
length = len(output)
a = 0.0
b = 0.0
c = 0.0
alpha = 0.0
beta = 0.0
gamma = 0.0
self._cell_info['datasets'] = []
self._cell_info['dataset_info'] = { }
for i in range(length):
line = output[i][:-1]
if 'Dataset ID, ' in line:
block = 0
while output[block * 5 + i + 2].strip():
dataset_number = int(
output[5 * block + i + 2].split()[0])
project = output[5 * block + i + 2][10:].strip()
crystal = output[5 * block + i + 3][10:].strip()
dataset = output[5 * block + i + 4][10:].strip()
cell = map(float, output[5 * block + i + 5].strip(
).split())
wavelength = float(output[5 * block + i + 6].strip())
dataset_id = '%s/%s/%s' % \
(project, crystal, dataset)
self._cell_info['datasets'].append(dataset_id)
self._cell_info['dataset_info'][dataset_id] = { }
self._cell_info['dataset_info'][
dataset_id]['wavelength'] = wavelength
self._cell_info['dataset_info'][
dataset_id]['cell'] = cell
self._cell_info['dataset_info'][
dataset_id]['id'] = dataset_number
block += 1
for dataset in self._cell_info['datasets']:
cell = self._cell_info['dataset_info'][dataset]['cell']
a += cell[0]
b += cell[1]
c += cell[2]
alpha += cell[3]
beta += cell[4]
gamma += cell[5]
n = len(self._cell_info['datasets'])
self._cell = (a / n, b / n, c / n, alpha / n, beta / n, gamma / n)
return 'ok'
def _scale(self):
'''Perform all of the operations required to deliver the scaled
data.'''
epochs = self._sweep_handler.get_epochs()
if PhilIndex.params.xia2.settings.optimize_scaling:
self._determine_best_scale_model_8way()
else:
self._scalr_corrections = True
self._scalr_correct_absorption = True
self._scalr_correct_partiality = False
self._scalr_correct_decay = True
if self._scalr_corrections:
Journal.block(
'scaling', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'scaling model':'automatic',
'absorption':self._scalr_correct_absorption,
'tails':self._scalr_correct_partiality,
'decay':self._scalr_correct_decay
})
else:
Journal.block(
'scaling', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'scaling model':'default'})
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_intensities(PhilIndex.params.ccp4.aimless.intensities)
sc.set_chef_unmerged(True)
sc.set_new_scales_file('%s.scales' % self._scalr_xname)
user_resolution_limits = { }
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
intgr = si.get_integrater()
if intgr.get_integrater_user_resolution():
dmin = intgr.get_integrater_high_resolution()
if not user_resolution_limits.has_key((dname, sname)):
user_resolution_limits[(dname, sname)] = dmin
elif dmin < user_resolution_limits[(dname, sname)]:
user_resolution_limits[(dname, sname)] = dmin
start, end = si.get_batch_range()
if (dname, sname) in self._scalr_resolution_limits:
resolution = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude = False,
resolution = resolution, name = sname)
else:
sc.add_run(start, end, name = sname)
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_scaled_test.mtz' % \
(self._scalr_pname, self._scalr_xname)))
if self.get_scaler_anomalous():
sc.set_anomalous()
# what follows, sucks
if Flags.get_failover():
try:
sc.scale()
except RuntimeError, e:
es = str(e)
if 'bad batch' in es or \
'negative scales run' in es or \
'no observations' in es:
# first ID the sweep from the batch no
batch = int(es.split()[-1])
epoch = self._identify_sweep_epoch(batch)
sweep = self._scalr_integraters[
epoch].get_integrater_sweep()
# then remove it from my parent xcrystal
self.get_scaler_xcrystal().remove_sweep(sweep)
# then remove it from the scaler list of intergraters
# - this should really be a scaler interface method
del(self._scalr_integraters[epoch])
# then tell the user what is happening
Chatter.write(
'Sweep %s gave negative scales - removing' % \
sweep.get_name())
# then reset the prepare, do, finish flags
self.set_scaler_prepare_done(False)
self.set_scaler_done(False)
self.set_scaler_finish_done(False)
# and return
return
else:
raise e
def xia2_main(stop_after=None):
'''Actually process something...'''
Citations.cite('xia2')
# print versions of related software
Chatter.write(dials_version())
ccp4_version = get_ccp4_version()
if ccp4_version is not None:
Chatter.write('CCP4 %s' % ccp4_version)
start_time = time.time()
CommandLine = get_command_line()
start_dir = Flags.get_starting_directory()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
no_images = True
for name in xtals.keys():
xtal = xtals[name]
if not xtal.get_all_image_names():
Chatter.write('-----------------------------------' + \
'-' * len(name))
Chatter.write('| No images assigned for crystal %s |' % name)
Chatter.write('-----------------------------------' + '-' \
* len(name))
else:
no_images = False
args = []
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
from libtbx import group_args
xinfo = CommandLine.get_xinfo()
if os.path.exists('xia2.json'):
from xia2.Schema.XProject import XProject
xinfo_new = xinfo
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new.keys():
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names(
):
wavelength_new = crystals_new[crystal_id].get_xwavelength(
wavelength_id)
if wavelength_id not in crystals[
crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(
wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = [s.get_name() for s in sweeps]
sweep_keys = [(s.get_directory(), s.get_template(),
s.get_image_range()) for s in sweeps]
for sweep in sweeps_new:
if ((sweep.get_directory(), sweep.get_template(),
sweep.get_image_range()) not in sweep_keys):
if sweep.get_name() in sweep_names:
i = 1
while 'SWEEEP%i' % i in sweep_names:
i += 1
sweep._name = 'SWEEP%i' % i
break
wavelength.add_sweep(
name=sweep.get_name(),
sample=sweep.get_xsample(),
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.append(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
if mp_params.mode == 'parallel' and njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
args.append((group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
), ))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(args):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
if mp_params.type == "qsub":
method = "sge"
else:
method = "multiprocessing"
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command
if not qsub_command:
qsub_command = 'qsub'
qsub_command = '%s -V -cwd -pe smp %d' % (qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep,
args,
processes=njob,
#method=method,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
if output is not None:
Chatter.write(output)
if not success:
Chatter.write('Sweep failed: removing %s' %
sweep.get_name())
remove_sweeps.append(sweep)
else:
assert xsweep_dict is not None
Chatter.write('Loading sweep: %s' % sweep.get_name())
from xia2.Schema.XSweep import XSweep
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
else:
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
from dials.command_line.show import show_datablocks
from dxtbx.datablock import DataBlock
Debug.write(sweep.get_name())
Debug.write(
show_datablocks([DataBlock([sweep.get_imageset()])]))
try:
if stop_after == 'index':
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception as e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(sweep.get_name(), str(e)))
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
# save intermediate xia2.json file in case scaling step fails
xinfo.as_json(filename='xia2.json')
if stop_after not in ('index', 'integrate'):
Chatter.write(xinfo.get_output(), strip=False)
for crystal in crystals.values():
crystal.serialize()
# save final xia2.json file in case report generation fails
xinfo.as_json(filename='xia2.json')
duration = time.time() - start_time
# write out the time taken in a human readable way
Chatter.write('Processing took %s' % \
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
if stop_after not in ('index', 'integrate'):
# and the summary file
with open('xia2-summary.dat', 'w') as fh:
for record in xinfo.summarise():
fh.write('%s\n' % record)
# looks like this import overwrites the initial command line
# Phil overrides so... for https://github.com/xia2/xia2/issues/150
from xia2.command_line.html import generate_xia2_html
if params.xia2.settings.small_molecule == True:
params.xia2.settings.report.xtriage_analysis = False
params.xia2.settings.report.include_radiation_damage = False
generate_xia2_html(xinfo,
filename='xia2.html',
params=params.xia2.settings.report)
write_citations()
# delete all of the temporary mtz files...
cleanup()
Environment.cleanup()
def Idxref(self):
idxref = _Idxref(params=PhilIndex.params.xds.index)
idxref.set_working_directory(self.get_working_directory())
idxref.setup_from_imageset(self.get_imageset())
if self.get_distance():
idxref.set_distance(self.get_distance())
if self.get_wavelength():
idxref.set_wavelength(self.get_wavelength())
# if we have a refined set of parameters to apply, apply these
if Flags.get_xparm():
idxref.set_refined_origin(Flags.get_xparm_origin())
idxref.set_refined_beam_vector(Flags.get_xparm_beam_vector())
idxref.set_refined_rotation_axis(Flags.get_xparm_rotation_axis())
idxref.set_refined_distance(Flags.get_xparm_distance())
# hacks for Jira 493
if Flags.get_xparm_a():
idxref.set_a_axis(Flags.get_xparm_a())
if Flags.get_xparm_b():
idxref.set_b_axis(Flags.get_xparm_b())
if Flags.get_xparm_c():
idxref.set_c_axis(Flags.get_xparm_c())
auto_logfiler(idxref, "IDXREF")
return idxref
def run(self):
'''Run colspot.'''
#image_header = self.get_header()
## crank through the header dictionary and replace incorrect
## information with updated values through the indexer
## interface if available...
## need to add distance, wavelength - that should be enough...
#if self.get_distance():
#image_header['distance'] = self.get_distance()
#if self.get_wavelength():
#image_header['wavelength'] = self.get_wavelength()
#if self.get_two_theta():
#image_header['two_theta'] = self.get_two_theta()
header = imageset_to_xds(self.get_imageset())
xds_inp = open(os.path.join(self.get_working_directory(),
'XDS.INP'), 'w')
# what are we doing?
xds_inp.write('JOB=COLSPOT\n')
xds_inp.write('MAXIMUM_NUMBER_OF_PROCESSORS=%d\n' % \
self._parallel)
#if image_header['detector'] in ('pilatus', 'dectris'):
if self.get_imageset().get_detector()[0].get_type() == 'SENSOR_PAD':
xds_inp.write('MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT=%d\n' %
self._params.minimum_pixels_per_spot)
for record in header:
xds_inp.write('%s\n' % record)
name_template = os.path.join(self.get_directory(),
self.get_template().replace('#', '?'))
record = 'NAME_TEMPLATE_OF_DATA_FRAMES=%s\n' % \
name_template
xds_inp.write(record)
xds_inp.write('DATA_RANGE=%d %d\n' % self._data_range)
for spot_range in self._spot_range:
xds_inp.write('SPOT_RANGE=%d %d\n' % spot_range)
xds_inp.write('BACKGROUND_RANGE=%d %d\n' % \
self._background_range)
# microcrystals have very mall spots, perhaps?
if Flags.get_microcrystal():
xds_inp.write('MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT=1\n')
if Flags.get_small_molecule():
xds_inp.write('STRONG_PIXEL=5\n')
# FIXME should probably be moved to a phil parameter
xds_inp.close()
# copy the input file...
shutil.copyfile(os.path.join(self.get_working_directory(),
'XDS.INP'),
os.path.join(self.get_working_directory(),
'%d_COLSPOT.INP' % self.get_xpid()))
# write the input data files...
for file_name in self._input_data_files_list:
src = self._input_data_files[file_name]
dst = os.path.join(
self.get_working_directory(), file_name)
if src != dst:
shutil.copyfile(src, dst)
self.start()
self.close_wait()
xds_check_version_supported(self.get_all_output())
# copy the LP file
shutil.copyfile(os.path.join(self.get_working_directory(),
'COLSPOT.LP'),
os.path.join(self.get_working_directory(),
'%d_COLSPOT.LP' % self.get_xpid()))
# gather the output files
for file in self._output_data_files_list:
self._output_data_files[file] = os.path.join(
self.get_working_directory(), file)
return
def setup(self):
"""Set everything up..."""
# check arguments are all ascii
Debug.write("Start parsing command line: " + str(sys.argv))
for token in sys.argv:
try:
token.encode("ascii")
except UnicodeDecodeError:
raise RuntimeError("non-ascii characters in input")
self._argv = copy.deepcopy(sys.argv)
replacements = {
"-2d": "pipeline=2d",
"-2di": "pipeline=2di",
"-3d": "pipeline=3d",
"-3di": "pipeline=3di",
"-3dii": "pipeline=3dii",
"-3dd": "pipeline=3dd",
"-dials": "pipeline=dials",
"-quick": "dials.fast_mode=true",
"-failover": "failover=true",
"-small_molecule": "small_molecule=true",
}
for k, v in replacements.iteritems():
if k in self._argv:
print(
"***\nCommand line option %s is deprecated.\nPlease use %s instead\n***"
% (k, v))
self._argv[self._argv.index(k)] = v
if "-atom" in self._argv:
idx = self._argv.index("-atom")
element = self._argv[idx + 1]
self._argv[idx:idx + 2] = ["atom=%s" % element]
print(
"***\nCommand line option -atom %s is deprecated.\nPlease use atom=%s instead\n***"
% (element, element))
# first of all try to interpret arguments as phil parameters/files
from xia2.Handlers.Phil import master_phil
from libtbx.phil import command_line
cmd_line = command_line.argument_interpreter(master_phil=master_phil)
working_phil, self._argv = cmd_line.process_and_fetch(
args=self._argv, custom_processor="collect_remaining")
PhilIndex.merge_phil(working_phil)
try:
params = PhilIndex.get_python_object()
except RuntimeError as e:
raise Sorry(e)
# sanity check / interpret Auto in input
from libtbx import Auto
if params.xia2.settings.input.atom is None:
if params.xia2.settings.input.anomalous is Auto:
PhilIndex.update("xia2.settings.input.anomalous=false")
else:
if params.xia2.settings.input.anomalous is False:
raise Sorry(
"Setting anomalous=false and atom type inconsistent")
params.xia2.settings.input.anomalous = True
PhilIndex.update("xia2.settings.input.anomalous=true")
if params.xia2.settings.resolution.keep_all_reflections is Auto:
if (params.xia2.settings.small_molecule is True
and params.xia2.settings.resolution.d_min is None
and params.xia2.settings.resolution.d_max is None):
PhilIndex.update(
"xia2.settings.resolution.keep_all_reflections=true")
else:
PhilIndex.update(
"xia2.settings.resolution.keep_all_reflections=false")
if params.xia2.settings.small_molecule is True:
Debug.write("Small molecule selected")
if params.xia2.settings.symmetry.chirality is None:
PhilIndex.update("xia2.settings.symmetry.chirality=nonchiral")
params = PhilIndex.get_python_object()
# pipeline options
self._read_pipeline()
for (parameter, value) in (
("project", params.xia2.settings.project),
("crystal", params.xia2.settings.crystal),
):
validate_project_crystal_name(parameter, value)
Debug.write("Project: %s" % params.xia2.settings.project)
Debug.write("Crystal: %s" % params.xia2.settings.crystal)
# FIXME add some consistency checks in here e.g. that there are
# images assigned, there is a lattice assigned if cell constants
# are given and so on
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
from xia2.Handlers.Environment import get_number_cpus
if mp_params.mode == "parallel":
if mp_params.type == "qsub":
if which("qsub") is None:
raise Sorry("qsub not available")
if mp_params.njob is Auto:
mp_params.njob = get_number_cpus()
if mp_params.nproc is Auto:
mp_params.nproc = 1
elif mp_params.nproc is Auto:
mp_params.nproc = get_number_cpus()
elif mp_params.mode == "serial":
if mp_params.type == "qsub":
if which("qsub") is None:
raise Sorry("qsub not available")
if mp_params.njob is Auto:
mp_params.njob = 1
if mp_params.nproc is Auto:
mp_params.nproc = get_number_cpus()
PhilIndex.update("xia2.settings.multiprocessing.njob=%d" %
mp_params.njob)
PhilIndex.update("xia2.settings.multiprocessing.nproc=%d" %
mp_params.nproc)
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
if mp_params.nproc > 1 and os.name == "nt":
raise Sorry("nproc > 1 is not supported on Windows.") # #191
if params.xia2.settings.indexer is not None:
add_preference("indexer", params.xia2.settings.indexer)
if params.xia2.settings.refiner is not None:
add_preference("refiner", params.xia2.settings.refiner)
if params.xia2.settings.integrater is not None:
add_preference("integrater", params.xia2.settings.integrater)
if params.xia2.settings.scaler is not None:
add_preference("scaler", params.xia2.settings.scaler)
if params.xia2.settings.multi_sweep_indexing is Auto:
if (params.xia2.settings.small_molecule is True
and "dials" == params.xia2.settings.indexer):
PhilIndex.update("xia2.settings.multi_sweep_indexing=True")
else:
PhilIndex.update("xia2.settings.multi_sweep_indexing=False")
if (params.xia2.settings.multi_sweep_indexing is True
and params.xia2.settings.multiprocessing.mode == "parallel"):
Chatter.write(
"Multi sweep indexing disabled:\nMSI is not available for parallel processing."
)
PhilIndex.update("xia2.settings.multi_sweep_indexing=False")
input_json = params.xia2.settings.input.json
if input_json is not None and len(input_json):
for json_file in input_json:
assert os.path.isfile(json_file)
load_experiments(json_file)
reference_geometry = params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
reference_geometries = "\n".join([
"xia2.settings.input.reference_geometry=%s" %
os.path.abspath(g)
for g in params.xia2.settings.input.reference_geometry
])
Debug.write(reference_geometries)
PhilIndex.update(reference_geometries)
Debug.write("xia2.settings.trust_beam_centre=true")
PhilIndex.update("xia2.settings.trust_beam_centre=true")
params = PhilIndex.get_python_object()
params = PhilIndex.get_python_object()
if params.xia2.settings.input.xinfo is not None:
xinfo_file = os.path.abspath(params.xia2.settings.input.xinfo)
PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file)
params = PhilIndex.get_python_object()
self.set_xinfo(xinfo_file)
# issue #55 if not set ATOM in xinfo but anomalous=true or atom= set
# on commandline, set here, should be idempotent
if params.xia2.settings.input.anomalous is True:
crystals = self._xinfo.get_crystals()
for xname in crystals:
xtal = crystals[xname]
Debug.write("Setting anomalous for crystal %s" % xname)
xtal.set_anomalous(True)
else:
xinfo_file = "%s/automatic.xinfo" % os.path.abspath(os.curdir)
PhilIndex.update("xia2.settings.input.xinfo=%s" % xinfo_file)
params = PhilIndex.get_python_object()
if params.dials.find_spots.phil_file is not None:
PhilIndex.update(
"dials.find_spots.phil_file=%s" %
os.path.abspath(params.dials.find_spots.phil_file))
if params.dials.index.phil_file is not None:
PhilIndex.update("dials.index.phil_file=%s" %
os.path.abspath(params.dials.index.phil_file))
if params.dials.refine.phil_file is not None:
PhilIndex.update("dials.refine.phil_file=%s" %
os.path.abspath(params.dials.refine.phil_file))
if params.dials.integrate.phil_file is not None:
PhilIndex.update("dials.integrate.phil_file=%s" %
os.path.abspath(params.dials.integrate.phil_file))
if params.xds.index.xparm is not None:
Flags.set_xparm(params.xds.index.xparm)
if params.xds.index.xparm_ub is not None:
Flags.set_xparm_ub(params.xds.index.xparm_ub)
if params.xia2.settings.scale.freer_file is not None:
freer_file = os.path.abspath(params.xia2.settings.scale.freer_file)
if not os.path.exists(freer_file):
raise RuntimeError("%s does not exist" % freer_file)
from xia2.Modules.FindFreeFlag import FindFreeFlag
column = FindFreeFlag(freer_file)
Debug.write("FreeR_flag column in %s found: %s" %
(freer_file, column))
PhilIndex.update("xia2.settings.scale.freer_file=%s" % freer_file)
if params.xia2.settings.scale.reference_reflection_file is not None:
reference_reflection_file = os.path.abspath(
params.xia2.settings.scale.reference_reflection_file)
if not os.path.exists(reference_reflection_file):
raise RuntimeError("%s does not exist" %
reference_reflection_file)
PhilIndex.update(
"xia2.settings.scale.reference_reflection_file=%s" %
reference_reflection_file)
params = PhilIndex.get_python_object()
datasets = unroll_datasets(PhilIndex.params.xia2.settings.input.image)
for dataset in datasets:
start_end = None
# here we only care about ':' which are later than C:\
if ":" in dataset[3:]:
tokens = dataset.split(":")
# cope with windows drives i.e. C:\data\blah\thing_0001.cbf:1:100
if len(tokens[0]) == 1:
tokens = ["%s:%s" % (tokens[0], tokens[1])] + tokens[2:]
if len(tokens) != 3:
raise RuntimeError("/path/to/image_0001.cbf:start:end")
dataset = tokens[0]
start_end = int(tokens[1]), int(tokens[2])
from xia2.Applications.xia2setup import is_hd5f_name
if os.path.exists(os.path.abspath(dataset)):
dataset = os.path.abspath(dataset)
else:
directories = [os.getcwd()] + self._argv[1:]
found = False
for d in directories:
if os.path.exists(os.path.join(d, dataset)):
dataset = os.path.join(d, dataset)
found = True
break
if not found:
raise Sorry("Could not find %s in %s" %
(dataset, " ".join(directories)))
if is_hd5f_name(dataset):
self._hdf5_master_files.append(dataset)
if start_end:
Debug.write("Image range: %d %d" % start_end)
if dataset not in self._default_start_end:
self._default_start_end[dataset] = []
self._default_start_end[dataset].append(start_end)
else:
Debug.write("No image range specified")
else:
template, directory = image2template_directory(
os.path.abspath(dataset))
self._default_template.append(os.path.join(
directory, template))
self._default_directory.append(directory)
Debug.write("Interpreted from image %s:" % dataset)
Debug.write("Template %s" % template)
Debug.write("Directory %s" % directory)
if start_end:
Debug.write("Image range: %d %d" % start_end)
key = os.path.join(directory, template)
if key not in self._default_start_end:
self._default_start_end[key] = []
self._default_start_end[key].append(start_end)
else:
Debug.write("No image range specified")
# finally, check that all arguments were read and raise an exception
# if any of them were nonsense.
with open("xia2-working.phil", "wb") as f:
f.write(PhilIndex.working_phil.as_str())
f.write(
os.linesep
) # temporarily required for https://github.com/dials/dials/issues/522
with open("xia2-diff.phil", "wb") as f:
f.write(PhilIndex.get_diff().as_str())
f.write(
os.linesep
) # temporarily required for https://github.com/dials/dials/issues/522
Debug.write("\nDifference PHIL:")
Debug.write(PhilIndex.get_diff().as_str(), strip=False)
Debug.write("Working PHIL:")
Debug.write(PhilIndex.working_phil.as_str(), strip=False)
nonsense = "Unknown command-line options:"
was_nonsense = False
for j, argv in enumerate(self._argv):
if j == 0:
continue
if argv[0] != "-" and "=" not in argv:
continue
if j not in self._understood:
nonsense += " %s" % argv
was_nonsense = True
if was_nonsense:
raise RuntimeError(nonsense)
