def run():
if os.path.exists("xia2-working.phil"):
sys.argv.append("xia2-working.phil")
try:
check_environment()
except Exception as e:
traceback.print_exc(file=open("xia2.error", "w"))
Chatter.write('Status: error "%s"' % str(e))
# print the version
Chatter.write(Version)
Citations.cite("xia2")
start_time = time.time()
assert os.path.exists("xia2.json")
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename="xia2.json")
crystals = xinfo.get_crystals()
for crystal_id, crystal in crystals.iteritems():
# cwd = os.path.abspath(os.curdir)
from libtbx import Auto
scale_dir = PhilIndex.params.xia2.settings.scale.directory
if scale_dir is Auto:
scale_dir = "scale"
i = 0
while os.path.exists(os.path.join(crystal.get_name(), scale_dir)):
i += 1
scale_dir = "scale%i" % i
PhilIndex.params.xia2.settings.scale.directory = scale_dir
working_directory = Environment.generate_directory(
[crystal.get_name(), scale_dir])
# os.chdir(working_directory)
crystals[crystal_id]._scaler = None # reset scaler
scaler = crystal._get_scaler()
Chatter.write(xinfo.get_output())
crystal.serialize()
duration = time.time() - start_time
# write out the time taken in a human readable way
Chatter.write("Processing took %s" %
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
# delete all of the temporary mtz files...
cleanup()
write_citations()
xinfo.as_json(filename="xia2.json")
Environment.cleanup()
def run():
if os.path.exists("xia2-working.phil"):
sys.argv.append("xia2-working.phil")
try:
check_environment()
except Exception as e:
with open("xia2-error.txt", "w") as fh:
traceback.print_exc(file=fh)
logger.error('Status: error "%s"', str(e))
# print the version
logger.info(Version)
Citations.cite("xia2")
start_time = time.time()
assert os.path.exists("xia2.json")
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename="xia2.json")
with cleanup(xinfo.path):
crystals = xinfo.get_crystals()
for crystal_id, crystal in crystals.items():
scale_dir = PhilIndex.params.xia2.settings.scale.directory
if scale_dir is Auto:
scale_dir = "scale"
i = 0
while os.path.exists(os.path.join(crystal.get_name(), scale_dir)):
i += 1
scale_dir = "scale%i" % i
PhilIndex.params.xia2.settings.scale.directory = scale_dir
# reset scaler
crystals[crystal_id]._scaler = None
crystal._get_scaler()
logger.info(xinfo.get_output())
crystal.serialize()
duration = time.time() - start_time
# write out the time taken in a human readable way
logger.info(
"Processing took %s", time.strftime("%Hh %Mm %Ss", time.gmtime(duration))
)
write_citations()
xinfo.as_json(filename="xia2.json")
def xia2_main(stop_after=None):
'''Actually process something...'''
Citations.cite('xia2')
# print versions of related software
Chatter.write(dials_version())
ccp4_version = get_ccp4_version()
if ccp4_version is not None:
Chatter.write('CCP4 %s' % ccp4_version)
start_time = time.time()
CommandLine = get_command_line()
start_dir = Flags.get_starting_directory()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
no_images = True
for name in xtals.keys():
xtal = xtals[name]
if not xtal.get_all_image_names():
Chatter.write('-----------------------------------' + \
'-' * len(name))
Chatter.write('| No images assigned for crystal %s |' % name)
Chatter.write('-----------------------------------' + '-' \
* len(name))
else:
no_images = False
args = []
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
from libtbx import group_args
xinfo = CommandLine.get_xinfo()
if os.path.exists('xia2.json'):
from xia2.Schema.XProject import XProject
xinfo_new = xinfo
xinfo = XProject.from_json(filename='xia2.json')
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new.keys():
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names(
):
wavelength_new = crystals_new[crystal_id].get_xwavelength(
wavelength_id)
if wavelength_id not in crystals[
crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(
wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = [s.get_name() for s in sweeps]
sweep_keys = [(s.get_directory(), s.get_template(),
s.get_image_range()) for s in sweeps]
for sweep in sweeps_new:
if ((sweep.get_directory(), sweep.get_template(),
sweep.get_image_range()) not in sweep_keys):
if sweep.get_name() in sweep_names:
i = 1
while 'SWEEEP%i' % i in sweep_names:
i += 1
sweep._name = 'SWEEP%i' % i
break
wavelength.add_sweep(
name=sweep.get_name(),
sample=sweep.get_xsample(),
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.append(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
if mp_params.mode == 'parallel' and njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
args.append((group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
), ))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(args):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
if mp_params.type == "qsub":
method = "sge"
else:
method = "multiprocessing"
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command
if not qsub_command:
qsub_command = 'qsub'
qsub_command = '%s -V -cwd -pe smp %d' % (qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep,
args,
processes=njob,
#method=method,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
if output is not None:
Chatter.write(output)
if not success:
Chatter.write('Sweep failed: removing %s' %
sweep.get_name())
remove_sweeps.append(sweep)
else:
assert xsweep_dict is not None
Chatter.write('Loading sweep: %s' % sweep.get_name())
from xia2.Schema.XSweep import XSweep
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
else:
for crystal_id in crystals.keys():
for wavelength_id in crystals[crystal_id].get_wavelength_names():
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
from dials.command_line.show import show_datablocks
from dxtbx.datablock import DataBlock
Debug.write(sweep.get_name())
Debug.write(
show_datablocks([DataBlock([sweep.get_imageset()])]))
try:
if stop_after == 'index':
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception as e:
if failover:
Chatter.write('Processing sweep %s failed: %s' % \
(sweep.get_name(), str(e)))
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.get_xsample()
sample.remove_sweep(sweep)
# save intermediate xia2.json file in case scaling step fails
xinfo.as_json(filename='xia2.json')
if stop_after not in ('index', 'integrate'):
Chatter.write(xinfo.get_output(), strip=False)
for crystal in crystals.values():
crystal.serialize()
# save final xia2.json file in case report generation fails
xinfo.as_json(filename='xia2.json')
duration = time.time() - start_time
# write out the time taken in a human readable way
Chatter.write('Processing took %s' % \
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
if stop_after not in ('index', 'integrate'):
# and the summary file
with open('xia2-summary.dat', 'w') as fh:
for record in xinfo.summarise():
fh.write('%s\n' % record)
# looks like this import overwrites the initial command line
# Phil overrides so... for https://github.com/xia2/xia2/issues/150
from xia2.command_line.html import generate_xia2_html
if params.xia2.settings.small_molecule == True:
params.xia2.settings.report.xtriage_analysis = False
params.xia2.settings.report.include_radiation_damage = False
generate_xia2_html(xinfo,
filename='xia2.html',
params=params.xia2.settings.report)
write_citations()
# delete all of the temporary mtz files...
cleanup()
Environment.cleanup()
PhilIndex.params.xia2.settings.scale.directory = scale_dir
working_directory = Environment.generate_directory(
[crystal.get_name(), scale_dir])
#os.chdir(working_directory)
crystals[crystal_id]._scaler = None # reset scaler
scaler = crystal._get_scaler()
Chatter.write(xinfo.get_output())
crystal.serialize()
duration = time.time() - start_time
# write out the time taken in a human readable way
Chatter.write('Processing took %s' % \
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
# delete all of the temporary mtz files...
cleanup()
write_citations()
xinfo.as_json(filename='xia2.json')
Environment.cleanup()
return
if __name__ == '__main__':
run()
def xia2_main(stop_after=None):
"""Actually process something..."""
Citations.cite("xia2")
# print versions of related software
logger.info(dials_version())
ccp4_version = get_ccp4_version()
if ccp4_version:
logger.info("CCP4 %s", ccp4_version)
start_time = time.time()
CommandLine = get_command_line()
# check that something useful has been assigned for processing...
xtals = CommandLine.get_xinfo().get_crystals()
for name, xtal in xtals.items():
if not xtal.get_all_image_names():
logger.info("-----------------------------------" +
"-" * len(name))
logger.info("| No images assigned for crystal %s |", name)
logger.info("-----------------------------------" +
"-" * len(name))
from xia2.Handlers.Phil import PhilIndex
params = PhilIndex.get_python_object()
mp_params = params.xia2.settings.multiprocessing
njob = mp_params.njob
xinfo = CommandLine.get_xinfo()
logger.info("Project directory: %s", xinfo.path)
if (params.xia2.settings.developmental.continue_from_previous_job
and os.path.exists("xia2.json")):
logger.debug("==== Starting from existing xia2.json ====")
xinfo_new = xinfo
xinfo = XProject.from_json(filename="xia2.json")
crystals = xinfo.get_crystals()
crystals_new = xinfo_new.get_crystals()
for crystal_id in crystals_new:
if crystal_id not in crystals:
crystals[crystal_id] = crystals_new[crystal_id]
continue
crystals[crystal_id]._scaler = None # reset scaler
for wavelength_id in crystals_new[crystal_id].get_wavelength_names(
):
wavelength_new = crystals_new[crystal_id].get_xwavelength(
wavelength_id)
if wavelength_id not in crystals[
crystal_id].get_wavelength_names():
crystals[crystal_id].add_wavelength(
crystals_new[crystal_id].get_xwavelength(
wavelength_new))
continue
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps_new = wavelength_new.get_sweeps()
sweeps = wavelength.get_sweeps()
sweep_names = {s.get_name() for s in sweeps}
sweep_keys = {(s.get_directory(), s.get_template(),
s.get_image_range())
for s in sweeps}
for sweep in sweeps_new:
if (
sweep.get_directory(),
sweep.get_template(),
sweep.get_image_range(),
) not in sweep_keys:
if sweep.get_name() in sweep_names:
i = 1
while "SWEEEP%i" % i in sweep_names:
i += 1
sweep._name = "SWEEP%i" % i
break
wavelength.add_sweep(
name=sweep.get_name(),
sample=sweep.sample,
directory=sweep.get_directory(),
image=sweep.get_image(),
beam=sweep.get_beam_centre(),
reversephi=sweep.get_reversephi(),
distance=sweep.get_distance(),
gain=sweep.get_gain(),
dmin=sweep.get_resolution_high(),
dmax=sweep.get_resolution_low(),
polarization=sweep.get_polarization(),
frames_to_process=sweep.get_frames_to_process(),
user_lattice=sweep.get_user_lattice(),
user_cell=sweep.get_user_cell(),
epoch=sweep._epoch,
ice=sweep._ice,
excluded_regions=sweep._excluded_regions,
)
sweep_names.add(sweep.get_name())
crystals = xinfo.get_crystals()
failover = params.xia2.settings.failover
with cleanup(xinfo.path):
if mp_params.mode == "parallel" and njob > 1:
driver_type = mp_params.type
command_line_args = CommandLine.get_argv()[1:]
jobs = []
for crystal_id in crystals:
for wavelength_id in crystals[crystal_id].get_wavelength_names(
):
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
sweep._get_indexer()
sweep._get_refiner()
sweep._get_integrater()
jobs.append((group_args(
driver_type=driver_type,
stop_after=stop_after,
failover=failover,
command_line_args=command_line_args,
nproc=mp_params.nproc,
crystal_id=crystal_id,
wavelength_id=wavelength_id,
sweep_id=sweep.get_name(),
), ))
from xia2.Driver.DriverFactory import DriverFactory
default_driver_type = DriverFactory.get_driver_type()
# run every nth job on the current computer (no need to submit to qsub)
for i_job, arg in enumerate(jobs):
if (i_job % njob) == 0:
arg[0].driver_type = default_driver_type
nproc = mp_params.nproc
qsub_command = mp_params.qsub_command or "qsub"
qsub_command = "%s -V -cwd -pe smp %d" % (qsub_command, nproc)
from libtbx import easy_mp
results = easy_mp.parallel_map(
process_one_sweep,
jobs,
processes=njob,
method="multiprocessing",
qsub_command=qsub_command,
preserve_order=True,
preserve_exception_message=True,
)
# Hack to update sweep with the serialized indexers/refiners/integraters
i_sweep = 0
for crystal_id in crystals:
for wavelength_id in crystals[crystal_id].get_wavelength_names(
):
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
success, output, xsweep_dict = results[i_sweep]
if output is not None:
logger.info(output)
if not success:
logger.info("Sweep failed: removing %s",
sweep.get_name())
remove_sweeps.append(sweep)
else:
assert xsweep_dict is not None
logger.info("Loading sweep: %s", sweep.get_name())
new_sweep = XSweep.from_dict(xsweep_dict)
sweep._indexer = new_sweep._indexer
sweep._refiner = new_sweep._refiner
sweep._integrater = new_sweep._integrater
i_sweep += 1
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.sample
sample.remove_sweep(sweep)
else:
for crystal_id in list(crystals.keys()):
for wavelength_id in crystals[crystal_id].get_wavelength_names(
):
wavelength = crystals[crystal_id].get_xwavelength(
wavelength_id)
remove_sweeps = []
sweeps = wavelength.get_sweeps()
for sweep in sweeps:
from dials.command_line.show import show_experiments
from dxtbx.model.experiment_list import ExperimentListFactory
logger.debug(sweep.get_name())
logger.debug(
show_experiments(
ExperimentListFactory.
from_imageset_and_crystal(
sweep.get_imageset(), None)))
Citations.cite("dials")
try:
if stop_after == "index":
sweep.get_indexer_cell()
else:
sweep.get_integrater_intensities()
sweep.serialize()
except Exception as e:
if failover:
logger.info(
"Processing sweep %s failed: %s",
sweep.get_name(),
str(e),
)
remove_sweeps.append(sweep)
else:
raise
for sweep in remove_sweeps:
wavelength.remove_sweep(sweep)
sample = sweep.sample
sample.remove_sweep(sweep)
# save intermediate xia2.json file in case scaling step fails
xinfo.as_json(filename="xia2.json")
if stop_after not in ("index", "integrate"):
logger.info(xinfo.get_output())
for crystal in list(crystals.values()):
crystal.serialize()
# save final xia2.json file in case report generation fails
xinfo.as_json(filename="xia2.json")
if stop_after not in ("index", "integrate"):
# and the summary file
with open("xia2-summary.dat", "w") as fh:
for record in xinfo.summarise():
fh.write("%s\n" % record)
# looks like this import overwrites the initial command line
# Phil overrides so... for https://github.com/xia2/xia2/issues/150
from xia2.command_line.html import generate_xia2_html
if params.xia2.settings.small_molecule:
params.xia2.settings.report.xtriage_analysis = False
params.xia2.settings.report.include_radiation_damage = False
with xia2.Driver.timing.record_step("xia2.report"):
generate_xia2_html(xinfo,
filename="xia2.html",
params=params.xia2.settings.report)
duration = time.time() - start_time
# write out the time taken in a human readable way
logger.info("Processing took %s",
time.strftime("%Hh %Mm %Ss", time.gmtime(duration)))
write_citations()