finally:
Chatter.uncache()
Debug.uncache()
return s
remove = []
for s in self._sweeps:
# would be nice to put this somewhere else in the hierarchy - not
# sure how to do that though (should be handled in Interfaces?)
try:
result += '%s\n' % s.get_output()
except Exception, e:
if Flags.get_failover():
Chatter.write('Processing sweep %s failed: %s' % \
(s.get_name(), str(e)))
remove.append(s)
else:
raise
for s in remove:
self._sweeps.remove(s)
return result[:-1]
def summarise(self):
summary = ['Wavelength: %s (%7.5f)' % (self._name, self._wavelength)]
def _scale(self):
'''Perform all of the operations required to deliver the scaled
data.'''
epochs = self._sweep_handler.get_epochs()
if PhilIndex.params.xia2.settings.optimize_scaling:
self._determine_best_scale_model_8way()
else:
self._scalr_corrections = True
self._scalr_correct_absorption = True
self._scalr_correct_partiality = False
self._scalr_correct_decay = True
if self._scalr_corrections:
Journal.block(
'scaling', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'scaling model':'automatic',
'absorption':self._scalr_correct_absorption,
'tails':self._scalr_correct_partiality,
'decay':self._scalr_correct_decay
})
else:
Journal.block(
'scaling', self.get_scaler_xcrystal().get_name(), 'CCP4',
{'scaling model':'default'})
sc = self._updated_aimless()
sc.set_hklin(self._prepared_reflections)
sc.set_intensities(PhilIndex.params.ccp4.aimless.intensities)
sc.set_chef_unmerged(True)
sc.set_new_scales_file('%s.scales' % self._scalr_xname)
user_resolution_limits = { }
for epoch in epochs:
si = self._sweep_handler.get_sweep_information(epoch)
pname, xname, dname = si.get_project_info()
sname = si.get_sweep_name()
intgr = si.get_integrater()
if intgr.get_integrater_user_resolution():
dmin = intgr.get_integrater_high_resolution()
if not user_resolution_limits.has_key((dname, sname)):
user_resolution_limits[(dname, sname)] = dmin
elif dmin < user_resolution_limits[(dname, sname)]:
user_resolution_limits[(dname, sname)] = dmin
start, end = si.get_batch_range()
if (dname, sname) in self._scalr_resolution_limits:
resolution = self._scalr_resolution_limits[(dname, sname)]
sc.add_run(start, end, exclude = False,
resolution = resolution, name = sname)
else:
sc.add_run(start, end, name = sname)
sc.set_hklout(os.path.join(self.get_working_directory(),
'%s_%s_scaled_test.mtz' % \
(self._scalr_pname, self._scalr_xname)))
if self.get_scaler_anomalous():
sc.set_anomalous()
# what follows, sucks
if Flags.get_failover():
try:
sc.scale()
except RuntimeError, e:
es = str(e)
if 'bad batch' in es or \
'negative scales run' in es or \
'no observations' in es:
# first ID the sweep from the batch no
batch = int(es.split()[-1])
epoch = self._identify_sweep_epoch(batch)
sweep = self._scalr_integraters[
epoch].get_integrater_sweep()
# then remove it from my parent xcrystal
self.get_scaler_xcrystal().remove_sweep(sweep)
# then remove it from the scaler list of intergraters
# - this should really be a scaler interface method
del(self._scalr_integraters[epoch])
# then tell the user what is happening
Chatter.write(
'Sweep %s gave negative scales - removing' % \
sweep.get_name())
# then reset the prepare, do, finish flags
self.set_scaler_prepare_done(False)
self.set_scaler_done(False)
self.set_scaler_finish_done(False)
# and return
return
else:
raise e
def xia2_main(stop_after=None):
'''Actually process something...'''
#Flags.set_serialize_state(True)
Citations.cite('xia2')
# print versions of related software
from dials.util.version import dials_version
Chatter.write(dials_version())
start_time = time.time()
CommandLine = get_command_line()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
no_images = True
for name in xtals.keys():
xtal = xtals[name]
if not xtal.get_all_image_names():
Chatter.write('-----------------------------------' + \
'-' * len(name))
Chatter.write('| No images assigned for crystal %s |' % name)
Chatter.write('-----------------------------------' + '-' \
* len(name))
else:
no_images = False
args = []
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
from libtbx import group_args
xinfo = CommandLine.get_xinfo()
if os.path.exists('xia2.json'):
from xia2.Schema.XProject import XProject
xinfo_new = xinfo
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new.keys():
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names():
wavelength_new = crystals_new[crystal_id].get_xwavelength(wavelength_id)
if wavelength_id not in crystals[crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = [s.get_name() for s in sweeps]
sweep_keys = [
(s.get_directory(), s.get_template(), s.get_image_range())
for s in sweeps]
for sweep in sweeps_new:
if ((sweep.get_directory(), sweep.get_template(),
sweep.get_image_range()) not in sweep_keys):
if sweep.get_name() in sweep_names:
i = 1
while 'SWEEEP%i' %i in sweep_names:
i += 1
sweep._name = 'SWEEP%i' %i
break
wavelength.add_sweep(
name=sweep.get_name(),
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.append(sweep.get_name())
crystals = xinfo.get_crystals()
if njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
command_line_args = [arg for arg in command_line_args if arg != '-failover']
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
args.append((
group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=Flags.get_failover(),
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
),))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(args):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
if mp_params.type == "qsub":
method = "sge"
else:
method = "multiprocessing"
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command
if not qsub_command:
qsub_command = 'qsub'
qsub_command = '%s -V -cwd -pe smp %d' %(qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep, args, processes=njob,
#method=method,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
assert xsweep_dict is not None
if output is not None:
Chatter.write(output)
if not success:
print "Sweep failed: removing %s" %sweep.get_name()
remove_sweeps.append(sweep)
else:
print "Loading sweep: %s" %sweep.get_name()
from xia2.Schema.XSweep import XSweep
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
else:
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
try:
if stop_after == 'index':
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception, e:
if Flags.get_failover():
Chatter.write('Processing sweep %s failed: %s' % \
(sweep.get_name(), str(e)))
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
else:
raise
