def xia2_main(stop_after=None):
'''Actually process something...'''
Citations.cite('xia2')
# print versions of related software
Chatter.write(dials_version())
ccp4_version = get_ccp4_version()
if ccp4_version is not None:
Chatter.write('CCP4 %s' % ccp4_version)
start_time = time.time()
CommandLine = get_command_line()
start_dir = Flags.get_starting_directory()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
no_images = True
for name in xtals.keys():
xtal = xtals[name]
if not xtal.get_all_image_names():
Chatter.write('-----------------------------------' + \
'-' * len(name))
Chatter.write('| No images assigned for crystal %s |' % name)
Chatter.write('-----------------------------------' + '-' \
* len(name))
else:
no_images = False
args = []
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
from libtbx import group_args
xinfo = CommandLine.get_xinfo()
if os.path.exists('xia2.json'):
from xia2.Schema.XProject import XProject
xinfo_new = xinfo
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new.keys():
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names(
):
wavelength_new = crystals_new[crystal_id].get_xwavelength(
wavelength_id)
if wavelength_id not in crystals[
crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(
wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = [s.get_name() for s in sweeps]
sweep_keys = [(s.get_directory(), s.get_template(),
s.get_image_range()) for s in sweeps]
for sweep in sweeps_new:
if ((sweep.get_directory(), sweep.get_template(),
sweep.get_image_range()) not in sweep_keys):
if sweep.get_name() in sweep_names:
i = 1
while 'SWEEEP%i' % i in sweep_names:
i += 1
sweep._name = 'SWEEP%i' % i
break
wavelength.add_sweep(
name=sweep.get_name(),
sample=sweep.get_xsample(),
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.append(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
if mp_params.mode == 'parallel' and njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
args.append((group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
), ))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(args):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
if mp_params.type == "qsub":
method = "sge"
else:
method = "multiprocessing"
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command
if not qsub_command:
qsub_command = 'qsub'
qsub_command = '%s -V -cwd -pe smp %d' % (qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep,
args,
processes=njob,
#method=method,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
if output is not None:
Chatter.write(output)
if not success:
Chatter.write('Sweep failed: removing %s' %
sweep.get_name())
remove_sweeps.append(sweep)
else:
assert xsweep_dict is not None
Chatter.write('Loading sweep: %s' % sweep.get_name())
from xia2.Schema.XSweep import XSweep
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
else:
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
from dials.command_line.show import show_datablocks
from dxtbx.datablock import DataBlock
Debug.write(sweep.get_name())
Debug.write(
show_datablocks([DataBlock([sweep.get_imageset()])]))
try:
if stop_after == 'index':
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception as e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(sweep.get_name(), str(e)))
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
# save intermediate xia2.json file in case scaling step fails
xinfo.as_json(filename='xia2.json')
if stop_after not in ('index', 'integrate'):
Chatter.write(xinfo.get_output(), strip=False)
for crystal in crystals.values():
crystal.serialize()
# save final xia2.json file in case report generation fails
xinfo.as_json(filename='xia2.json')
duration = time.time() - start_time
# write out the time taken in a human readable way
Chatter.write('Processing took %s' % \
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
if stop_after not in ('index', 'integrate'):
# and the summary file
with open('xia2-summary.dat', 'w') as fh:
for record in xinfo.summarise():
fh.write('%s\n' % record)
# looks like this import overwrites the initial command line
# Phil overrides so... for https://github.com/xia2/xia2/issues/150
from xia2.command_line.html import generate_xia2_html
if params.xia2.settings.small_molecule == True:
params.xia2.settings.report.xtriage_analysis = False
params.xia2.settings.report.include_radiation_damage = False
generate_xia2_html(xinfo,
filename='xia2.html',
params=params.xia2.settings.report)
write_citations()
# delete all of the temporary mtz files...
cleanup()
Environment.cleanup()
def process_one_sweep(args):
assert len(args) == 1
args = args[0]
# stop_after = args.stop_after
command_line_args = args.command_line_args
nproc = args.nproc
crystal_id = args.crystal_id
wavelength_id = args.wavelength_id
sweep_id = args.sweep_id
failover = args.failover
driver_type = args.driver_type
default_driver_type = DriverFactory.get_driver_type()
DriverFactory.set_driver_type(driver_type)
curdir = os.path.abspath(os.curdir)
if "-xinfo" in command_line_args:
idx = command_line_args.index("-xinfo")
del command_line_args[idx + 1]
del command_line_args[idx]
xia2_integrate = XIA2Integrate()
# import tempfile
# tmpdir = tempfile.mkdtemp(dir=curdir)
tmpdir = os.path.join(curdir, str(uuid.uuid4()))
os.makedirs(tmpdir)
xia2_integrate.set_working_directory(tmpdir)
xia2_integrate.add_command_line_args(args.command_line_args)
xia2_integrate.set_phil_file(os.path.join(curdir, "xia2-working.phil"))
xia2_integrate.add_command_line_args(["sweep.id=%s" % sweep_id])
xia2_integrate.set_nproc(nproc)
xia2_integrate.set_njob(1)
xia2_integrate.set_mp_mode("serial")
auto_logfiler(xia2_integrate)
sweep_tmp_dir = os.path.join(tmpdir, crystal_id, wavelength_id, sweep_id)
sweep_target_dir = os.path.join(curdir, crystal_id, wavelength_id,
sweep_id)
output = None
success = False
xsweep_dict = None
try:
xia2_integrate.run()
output = get_sweep_output_only(xia2_integrate.get_all_output())
success = True
except Exception as e:
logger.warning("Processing sweep %s failed: %s", sweep_id, str(e))
if not failover:
raise
finally:
from xia2.Schema.XProject import XProject
xia2_json = os.path.join(tmpdir, "xia2.json")
json_files = glob.glob(os.path.join(sweep_tmp_dir, "*", "*.json"))
json_files.extend(glob.glob(os.path.join(sweep_tmp_dir, "*",
"*.expt")))
if os.path.exists(xia2_json):
json_files.append(xia2_json)
import fileinput
for line in fileinput.FileInput(files=json_files, inplace=1):
line = line.replace(sweep_tmp_dir, sweep_target_dir)
print(line)
if os.path.exists(xia2_json):
new_json = os.path.join(curdir, "xia2-%s.json" % sweep_id)
shutil.copyfile(xia2_json, new_json)
move_output_folder(sweep_tmp_dir, sweep_target_dir)
if success:
xinfo = XProject.from_json(new_json)
xcryst = list(xinfo.get_crystals().values())[0]
xsweep = xcryst.get_xwavelength(wavelength_id).get_sweeps()[0]
xsweep_dict = xsweep.to_dict()
shutil.rmtree(tmpdir, ignore_errors=True)
if os.path.exists(tmpdir):
shutil.rmtree(tmpdir, ignore_errors=True)
DriverFactory.set_driver_type(default_driver_type)
return success, output, xsweep_dict
def _scale_prepare(self):
'''Prepare the data for scaling - this will reindex it the
reflections to the correct pointgroup and setting, for instance,
and move the reflection files to the scale directory.'''
Citations.cite('xds')
Citations.cite('ccp4')
Citations.cite('pointless')
# GATHER phase - get the reflection files together... note that
# it is not necessary in here to keep the batch information as we
# don't wish to rebatch the reflections prior to scaling.
# FIXME need to think about what I will do about the radiation
# damage analysis in here...
self._sweep_information = { }
# FIXME in here I want to record the batch number to
# epoch mapping as per the CCP4 Scaler implementation.
Journal.block(
'gathering', self.get_scaler_xcrystal().get_name(), 'XDS',
{'working directory':self.get_working_directory()})
for epoch in self._scalr_integraters.keys():
intgr = self._scalr_integraters[epoch]
pname, xname, dname = intgr.get_integrater_project_info()
sname = intgr.get_integrater_sweep_name()
self._sweep_information[epoch] = {
'pname':pname,
'xname':xname,
'dname':dname,
'integrater':intgr,
'corrected_intensities':intgr.get_integrater_corrected_intensities(),
'prepared_reflections':None,
'scaled_reflections':None,
'header':intgr.get_header(),
'batches':intgr.get_integrater_batches(),
'image_to_epoch':intgr.get_integrater_sweep(
).get_image_to_epoch(),
'image_to_dose':{},
'batch_offset':0,
'sname':sname
}
Journal.entry({'adding data from':'%s/%s/%s' % \
(xname, dname, sname)})
# what are these used for?
# pname / xname / dname - dataset identifiers
# image to epoch / batch offset / batches - for RD analysis
Debug.write('For EPOCH %s have:' % str(epoch))
Debug.write('ID = %s/%s/%s' % (pname, xname, dname))
Debug.write('SWEEP = %s' % intgr.get_integrater_sweep_name())
# next work through all of the reflection files and make sure that
# they are XDS_ASCII format...
epochs = self._sweep_information.keys()
epochs.sort()
self._first_epoch = min(epochs)
self._scalr_pname = self._sweep_information[epochs[0]]['pname']
self._scalr_xname = self._sweep_information[epochs[0]]['xname']
for epoch in epochs:
intgr = self._scalr_integraters[epoch]
pname = self._sweep_information[epoch]['pname']
xname = self._sweep_information[epoch]['xname']
dname = self._sweep_information[epoch]['dname']
sname = self._sweep_information[epoch]['sname']
if self._scalr_pname != pname:
raise RuntimeError, 'all data must have a common project name'
xname = self._sweep_information[epoch]['xname']
if self._scalr_xname != xname:
raise RuntimeError, \
'all data for scaling must come from one crystal'
xsh = XDSScalerHelper()
xsh.set_working_directory(self.get_working_directory())
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s_%s_%s_CORRECTED.HKL' %(
pname, xname, dname, sname))
sweep = intgr.get_integrater_sweep()
if sweep.get_frames_to_process() is not None:
offset = intgr.get_frame_offset()
#print "offset: %d" %offset
start, end = sweep.get_frames_to_process()
start -= offset
end -= offset
#end += 1 ????
#print "limiting batches: %d-%d" %(start, end)
xsh.limit_batches(hklin, hklout, start, end)
self._sweep_information[epoch]['corrected_intensities'] = hklout
# if there is more than one sweep then compare the lattices
# and eliminate all but the lowest symmetry examples if
# there are more than one...
# -------------------------------------------------
# Ensure that the integration lattices are the same
# -------------------------------------------------
need_to_return = False
if len(self._sweep_information.keys()) > 1:
lattices = []
# FIXME run this stuff in parallel as well...
for epoch in self._sweep_information.keys():
intgr = self._sweep_information[epoch]['integrater']
hklin = self._sweep_information[epoch]['corrected_intensities']
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
ntr = False
else:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
lattice = Syminfo.get_lattice(pointgroup)
if not lattice in lattices:
lattices.append(lattice)
if ntr:
# if we need to return, we should logically reset
# any reindexing operator right? right here all
# we are talking about is the correctness of
# individual pointgroups?? Bug # 3373
reindex_op = 'h,k,l'
# actually, should this not be done "by magic"
# when a new pointgroup is assigned in the
# pointless indexer jiffy above?!
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
need_to_return = True
# bug # 2433 - need to ensure that all of the lattice
# conclusions were the same...
if len(lattices) > 1:
ordered_lattices = []
for l in lattices_in_order():
if l in lattices:
ordered_lattices.append(l)
correct_lattice = ordered_lattices[0]
Debug.write('Correct lattice asserted to be %s' % \
correct_lattice)
# transfer this information back to the indexers
for epoch in self._sweep_information.keys():
integrater = self._sweep_information[
epoch]['integrater']
refiner = integrater.get_integrater_refiner()
sname = integrater.get_integrater_sweep_name()
if not refiner:
continue
state = refiner.set_refiner_asserted_lattice(
correct_lattice)
if state == refiner.LATTICE_CORRECT:
Debug.write('Lattice %s ok for sweep %s' % \
(correct_lattice, sname))
elif state == refiner.LATTICE_IMPOSSIBLE:
raise RuntimeError, 'Lattice %s impossible for %s' % \
(correct_lattice, sname)
elif state == refiner.LATTICE_POSSIBLE:
Debug.write('Lattice %s assigned for sweep %s' % \
(correct_lattice, sname))
need_to_return = True
# if one or more of them was not in the lowest lattice,
# need to return here to allow reprocessing
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
# next if there is more than one sweep then generate
# a merged reference reflection file to check that the
# setting for all reflection files is the same...
# if we get to here then all data was processed with the same
# lattice
# ----------------------------------------------------------
# next ensure that all sweeps are set in the correct setting
# ----------------------------------------------------------
if self.get_scaler_reference_reflection_file():
self._reference = self.get_scaler_reference_reflection_file()
Debug.write('Using HKLREF %s' % self._reference)
md = self._factory.Mtzdump()
md.set_hklin(self.get_scaler_reference_reflection_file())
md.dump()
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(
md.get_spacegroup())
Debug.write('Spacegroup %d' % self._xds_spacegroup)
elif PhilIndex.params.xia2.settings.scale.reference_reflection_file:
self._reference = PhilIndex.params.xia2.settings.scale.reference_reflection_file
Debug.write('Using HKLREF %s' % self._reference)
md = self._factory.Mtzdump()
md.set_hklin(PhilIndex.params.xia2.settings.scale.reference_reflection_file)
md.dump()
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(
md.get_spacegroup())
Debug.write('Spacegroup %d' % self._xds_spacegroup)
params = PhilIndex.params
use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs
if len(self._sweep_information.keys()) > 1 and use_brehm_diederichs:
brehm_diederichs_files_in = []
for epoch in self._sweep_information.keys():
intgr = self._sweep_information[epoch]['integrater']
hklin = self._sweep_information[epoch]['corrected_intensities']
refiner = intgr.get_integrater_refiner()
# in here need to consider what to do if the user has
# assigned the pointgroup on the command line ...
if not self._scalr_input_pointgroup:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
if ntr:
# Bug # 3373
Debug.write('Reindex to standard (PIJ): %s' % \
reindex_op)
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
else:
# 27/FEB/08 to support user assignment of pointgroups
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
self._sweep_information[epoch]['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# convert the XDS_ASCII for this sweep to mtz - on the next
# get this should be in the correct setting...
dname = self._sweep_information[epoch]['dname']
sname = intgr.get_integrater_sweep_name()
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.mtz' % (dname, sname))
FileHandler.record_temporary_file(hklout)
# now use pointless to make this conversion
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
brehm_diederichs_files_in.append(hklout)
# now run cctbx.brehm_diederichs to figure out the indexing hand for
# each sweep
from xia2.Wrappers.Cctbx.BrehmDiederichs import BrehmDiederichs
brehm_diederichs = BrehmDiederichs()
brehm_diederichs.set_working_directory(self.get_working_directory())
auto_logfiler(brehm_diederichs)
brehm_diederichs.set_input_filenames(brehm_diederichs_files_in)
# 1 or 3? 1 seems to work better?
brehm_diederichs.set_asymmetric(1)
brehm_diederichs.run()
reindexing_dict = brehm_diederichs.get_reindexing_dict()
for epoch in self._sweep_information.keys():
intgr = self._sweep_information[epoch]['integrater']
dname = self._sweep_information[epoch]['dname']
sname = intgr.get_integrater_sweep_name()
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.mtz' % (dname, sname))
# apply the reindexing operator
intgr.set_integrater_reindex_operator(reindex_op)
# and copy the reflection file to the local directory
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
self._sweep_information[epoch][
'prepared_reflections'] = os.path.split(hklout)[-1]
elif len(self._sweep_information.keys()) > 1 and \
not self._reference:
# need to generate a reference reflection file - generate this
# from the reflections in self._first_epoch
#
# FIXME this should really use the Brehm and Diederichs method
# if you have lots of little sweeps...
intgr = self._sweep_information[self._first_epoch]['integrater']
hklin = self._sweep_information[epoch]['corrected_intensities']
refiner = intgr.get_integrater_refiner()
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
ntr = False
reindex_op = 'h,k,l'
else:
pointgroup, reindex_op, ntr = self._pointless_indexer_jiffy(
hklin, refiner)
Debug.write('X1698: %s: %s' % (pointgroup, reindex_op))
reference_reindex_op = intgr.get_integrater_reindex_operator()
if ntr:
# Bug # 3373
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(pointgroup)
# next pass this reindexing operator back to the source
# of the reflections
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
self._sweep_information[epoch]['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
hklin = self._sweep_information[epoch]['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'xds-pointgroup-reference-unsorted.mtz')
FileHandler.record_temporary_file(hklout)
# now use pointless to handle this conversion
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
self._reference = hklout
if self._reference:
from xia2.Driver.DriverFactory import DriverFactory
def run_one_sweep(args):
sweep_information = args[0]
pointless_indexer_jiffy = args[1]
factory = args[2]
job_type = args[3]
if job_type:
DriverFactory.set_driver_type(job_type)
intgr = sweep_information['integrater']
hklin = sweep_information['corrected_intensities']
refiner = intgr.get_integrater_refiner()
# in here need to consider what to do if the user has
# assigned the pointgroup on the command line ...
if not self._scalr_input_pointgroup:
pointgroup, reindex_op, ntr = \
self._pointless_indexer_jiffy(hklin, refiner)
if ntr:
# Bug # 3373
Debug.write('Reindex to standard (PIJ): %s' % \
reindex_op)
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
reindex_op = 'h,k,l'
need_to_return = True
else:
# 27/FEB/08 to support user assignment of pointgroups
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
sweep_information['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# convert the XDS_ASCII for this sweep to mtz - on the next
# get this should be in the correct setting...
hklin = sweep_information['corrected_intensities']
# now use pointless to make this conversion
# try with no conversion?!
pointless = self._factory.Pointless()
pointless.set_xdsin(hklin)
hklout = os.path.join(
self.get_working_directory(),
'%d_xds-pointgroup-unsorted.mtz' %pointless.get_xpid())
FileHandler.record_temporary_file(hklout)
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
pointless = self._factory.Pointless()
pointless.set_hklin(hklout)
pointless.set_hklref(self._reference)
pointless.decide_pointgroup()
pointgroup = pointless.get_pointgroup()
reindex_op = pointless.get_reindex_operator()
# for debugging print out the reindexing operations and
# what have you...
Debug.write('Reindex to standard: %s' % reindex_op)
# this should send back enough information that this
# is in the correct pointgroup (from the call above) and
# also in the correct setting, from the interaction
# with the reference set... - though I guess that the
# spacegroup number should not have changed, right?
# set the reindex operation afterwards... though if the
# spacegroup number is the same this should make no
# difference, right?!
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
intgr.set_integrater_reindex_operator(reindex_op)
sweep_information['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
# and copy the reflection file to the local directory
dname = sweep_information['dname']
sname = intgr.get_integrater_sweep_name()
hklin = sweep_information['corrected_intensities']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
sweep_information['prepared_reflections'] = os.path.split(hklout)[-1]
return sweep_information
from libtbx import easy_mp
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
if njob > 1:
# cache drivertype
drivertype = DriverFactory.get_driver_type()
args = [
(self._sweep_information[epoch], self._pointless_indexer_jiffy,
self._factory, mp_params.type)
for epoch in self._sweep_information.keys()]
results_list = easy_mp.parallel_map(
run_one_sweep, args, params=None,
processes=njob,
method="threading",
asynchronous=True,
callback=None,
preserve_order=True,
preserve_exception_message=True)
# restore drivertype
DriverFactory.set_driver_type(drivertype)
# results should be given back in the same order
for i, epoch in enumerate(self._sweep_information.keys()):
self._sweep_information[epoch] = results_list[i]
else:
for epoch in self._sweep_information.keys():
self._sweep_information[epoch] = run_one_sweep(
(self._sweep_information[epoch], self._pointless_indexer_jiffy,
self._factory, None))
else:
# convert the XDS_ASCII for this sweep to mtz
epoch = self._first_epoch
intgr = self._sweep_information[epoch]['integrater']
refiner = intgr.get_integrater_refiner()
sname = intgr.get_integrater_sweep_name()
hklout = os.path.join(self.get_working_directory(),
'%s-pointless.mtz' % sname)
FileHandler.record_temporary_file(hklout)
pointless = self._factory.Pointless()
pointless.set_xdsin(self._sweep_information[epoch]['corrected_intensities'])
pointless.set_hklout(hklout)
pointless.xds_to_mtz()
# run it through pointless interacting with the
# Indexer which belongs to this sweep
hklin = hklout
if self._scalr_input_pointgroup:
Debug.write('Using input pointgroup: %s' % \
self._scalr_input_pointgroup)
pointgroup = self._scalr_input_pointgroup
ntr = False
reindex_op = 'h,k,l'
else:
pointgroup, reindex_op, ntr = self._pointless_indexer_jiffy(
hklin, refiner)
if ntr:
# if we need to return, we should logically reset
# any reindexing operator right? right here all
# we are talking about is the correctness of
# individual pointgroups?? Bug # 3373
reindex_op = 'h,k,l'
intgr.set_integrater_reindex_operator(
reindex_op, compose = False)
need_to_return = True
self._xds_spacegroup = Syminfo.spacegroup_name_to_number(pointgroup)
# next pass this reindexing operator back to the source
# of the reflections
intgr.set_integrater_reindex_operator(reindex_op)
intgr.set_integrater_spacegroup_number(
Syminfo.spacegroup_name_to_number(pointgroup))
self._sweep_information[epoch]['corrected_intensities'] \
= intgr.get_integrater_corrected_intensities()
hklin = self._sweep_information[epoch]['corrected_intensities']
dname = self._sweep_information[epoch]['dname']
hklout = os.path.join(self.get_working_directory(),
'%s_%s.HKL' % (dname, sname))
# and copy the reflection file to the local
# directory
Debug.write('Copying %s to %s' % (hklin, hklout))
shutil.copyfile(hklin, hklout)
# record just the local file name...
self._sweep_information[epoch][
'prepared_reflections'] = os.path.split(hklout)[-1]
if need_to_return:
self.set_scaler_done(False)
self.set_scaler_prepare_done(False)
return
unit_cell_list = []
for epoch in self._sweep_information.keys():
integrater = self._sweep_information[epoch]['integrater']
cell = integrater.get_integrater_cell()
n_ref = integrater.get_integrater_n_ref()
Debug.write('Cell for %s: %.2f %.2f %.2f %.2f %.2f %.2f' % \
(integrater.get_integrater_sweep_name(),
cell[0], cell[1], cell[2],
cell[3], cell[4], cell[5]))
Debug.write('=> %d reflections' % n_ref)
unit_cell_list.append((cell, n_ref))
self._scalr_cell = compute_average_unit_cell(unit_cell_list)
self._scalr_resolution_limits = { }
Debug.write('Determined unit cell: %.2f %.2f %.2f %.2f %.2f %.2f' % \
tuple(self._scalr_cell))
if os.path.exists(os.path.join(
self.get_working_directory(),
'REMOVE.HKL')):
os.remove(os.path.join(
self.get_working_directory(),
'REMOVE.HKL'))
Debug.write('Deleting REMOVE.HKL at end of scale prepare.')
return
def xia2_main(stop_after=None):
"""Actually process something..."""
Citations.cite("xia2")
# print versions of related software
logger.info(dials_version())
ccp4_version = get_ccp4_version()
if ccp4_version:
logger.info("CCP4 %s", ccp4_version)
start_time = time.time()
CommandLine = get_command_line()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
for name, xtal in xtals.items():
if not xtal.get_all_image_names():
logger.info("-----------------------------------" +
"-" * len(name))
logger.info("| No images assigned for crystal %s |", name)
logger.info("-----------------------------------" +
"-" * len(name))
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
xinfo = CommandLine.get_xinfo()
logger.info("Project directory: %s", xinfo.path)
if (params.xia2.settings.developmental.continue_from_previous_job
and os.path.exists("xia2.json")):
logger.debug("==== Starting from existing xia2.json ====")
xinfo_new = xinfo
xinfo = XProject.from_json(filename="xia2.json")
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new:
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names(
):
wavelength_new = crystals_new[crystal_id].get_xwavelength(
wavelength_id)
if wavelength_id not in crystals[
crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(
wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = {s.get_name() for s in sweeps}
sweep_keys = {(s.get_directory(), s.get_template(),
s.get_image_range())
for s in sweeps}
for sweep in sweeps_new:
if (
sweep.get_directory(),
sweep.get_template(),
sweep.get_image_range(),
) not in sweep_keys:
if sweep.get_name() in sweep_names:
i = 1
while "SWEEEP%i" % i in sweep_names:
i += 1
sweep._name = "SWEEP%i" % i
break
wavelength.add_sweep(
name=sweep.get_name(),
sample=sweep.sample,
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.add(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
with cleanup(xinfo.path):
if mp_params.mode == "parallel" and njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
jobs = []
for crystal_id in crystals:
for wavelength_id in crystals[crystal_id].get_wavelength_names(
):
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
jobs.append((group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
), ))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(jobs):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command or "qsub"
qsub_command = "%s -V -cwd -pe smp %d" % (qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep,
jobs,
processes=njob,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True,
)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals:
for wavelength_id in crystals[crystal_id].get_wavelength_names(
):
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
if output is not None:
logger.info(output)
if not success:
logger.info("Sweep failed: removing %s",
sweep.get_name())
remove_sweeps.append(sweep)
else:
assert xsweep_dict is not None
logger.info("Loading sweep: %s", sweep.get_name())
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.sample
sample.remove_sweep(sweep)
else:
for crystal_id in list(crystals.keys()):
for wavelength_id in crystals[crystal_id].get_wavelength_names(
):
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
from dials.command_line.show import show_experiments
from dxtbx.model.experiment_list import ExperimentListFactory
logger.debug(sweep.get_name())
logger.debug(
show_experiments(
ExperimentListFactory.
from_imageset_and_crystal(
sweep.get_imageset(), None)))
Citations.cite("dials")
try:
if stop_after == "index":
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception as e:
if failover:
logger.info(
"Processing sweep %s failed: %s",
sweep.get_name(),
str(e),
)
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.sample
sample.remove_sweep(sweep)
# save intermediate xia2.json file in case scaling step fails
xinfo.as_json(filename="xia2.json")
if stop_after not in ("index", "integrate"):
logger.info(xinfo.get_output())
for crystal in list(crystals.values()):
crystal.serialize()
# save final xia2.json file in case report generation fails
xinfo.as_json(filename="xia2.json")
if stop_after not in ("index", "integrate"):
# and the summary file
with open("xia2-summary.dat", "w") as fh:
for record in xinfo.summarise():
fh.write("%s\n" % record)
# looks like this import overwrites the initial command line
# Phil overrides so... for https://github.com/xia2/xia2/issues/150
from xia2.command_line.html import generate_xia2_html
if params.xia2.settings.small_molecule:
params.xia2.settings.report.xtriage_analysis = False
params.xia2.settings.report.include_radiation_damage = False
with xia2.Driver.timing.record_step("xia2.report"):
generate_xia2_html(xinfo,
filename="xia2.html",
params=params.xia2.settings.report)
duration = time.time() - start_time
# write out the time taken in a human readable way
logger.info("Processing took %s",
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
write_citations()
def xia2_main(stop_after=None):
'''Actually process something...'''
Citations.cite('xia2')
# print versions of related software
from dials.util.version import dials_version
Chatter.write(dials_version())
start_time = time.time()
CommandLine = get_command_line()
start_dir = Flags.get_starting_directory()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
no_images = True
for name in xtals.keys():
xtal = xtals[name]
if not xtal.get_all_image_names():
Chatter.write('-----------------------------------' + \
'-' * len(name))
Chatter.write('| No images assigned for crystal %s |' % name)
Chatter.write('-----------------------------------' + '-' \
* len(name))
else:
no_images = False
args = []
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
from libtbx import group_args
xinfo = CommandLine.get_xinfo()
if os.path.exists('xia2.json'):
from xia2.Schema.XProject import XProject
xinfo_new = xinfo
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new.keys():
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names():
wavelength_new = crystals_new[crystal_id].get_xwavelength(wavelength_id)
if wavelength_id not in crystals[crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = [s.get_name() for s in sweeps]
sweep_keys = [
(s.get_directory(), s.get_template(), s.get_image_range())
for s in sweeps]
for sweep in sweeps_new:
if ((sweep.get_directory(), sweep.get_template(),
sweep.get_image_range()) not in sweep_keys):
if sweep.get_name() in sweep_names:
i = 1
while 'SWEEEP%i' %i in sweep_names:
i += 1
sweep._name = 'SWEEP%i' %i
break
wavelength.add_sweep(
name=sweep.get_name(),
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.append(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
if njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
args.append((
group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
),))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(args):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
if mp_params.type == "qsub":
method = "sge"
else:
method = "multiprocessing"
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command
if not qsub_command:
qsub_command = 'qsub'
qsub_command = '%s -V -cwd -pe smp %d' %(qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep, args, processes=njob,
#method=method,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
assert xsweep_dict is not None
if output is not None:
Chatter.write(output)
if not success:
Chatter.write('Sweep failed: removing %s' %sweep.get_name())
remove_sweeps.append(sweep)
else:
Chatter.write('Loading sweep: %s' % sweep.get_name())
from xia2.Schema.XSweep import XSweep
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
else:
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
try:
if stop_after == 'index':
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception, e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(sweep.get_name(), str(e)))
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
