def from_dict(cls, obj): assert obj['__id__'] == 'XWavelength' return_obj = cls(name=None, crystal=None, wavelength=None) for k, v in obj.iteritems(): if k == '_sweeps': v = [XSweep.from_dict(s_dict) for s_dict in v] for sweep in v: sweep._wavelength = return_obj setattr(return_obj, k, v) return return_obj
def exercise_serialization(dials_regression, tmp_dir): xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data") template = os.path.join(xia2_demo_data, "insulin_1_###.img") os.chdir(tmp_dir) from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Refiner.DialsRefiner import DialsRefiner from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA from dxtbx.datablock import DataBlockTemplateImporter importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] from xia2.Schema.XProject import XProject from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample proj = XProject() proj._name = "PROJ1" cryst = XCrystal("CRYST1", proj) proj.add_crystal(cryst) wav = XWavelength("WAVE1", cryst, wavelength=0.98) samp = XSample("X1", cryst) cryst.add_wavelength(wav) cryst.set_ha_info({'atom': 'S'}) directory, image = os.path.split(imageset.get_path(1)) wav.add_sweep(name='SWEEP1', sample=samp, directory=directory, image=image) import json from dxtbx.serialize.load import _decode_dict sweep = wav.get_sweeps()[0] indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) indexer.add_indexer_imageset(imageset) indexer.set_indexer_sweep(sweep) sweep._indexer = indexer refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) sweep._refiner = refiner integrater = DialsIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) #integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.set_integrater_epoch(sweep.get_epoch(1)) integrater.set_integrater_sweep_name(sweep.get_name()) integrater.set_integrater_project_info(cryst.get_name(), wav.get_name(), sweep.get_name()) sweep._integrater = integrater scaler = CCP4ScalerA() scaler.add_scaler_integrater(integrater) scaler.set_scaler_xcrystal(cryst) scaler.set_scaler_project_info(cryst.get_name(), wav.get_name()) scaler._scalr_xcrystal = cryst cryst._scaler = scaler s_dict = sweep.to_dict() s_str = json.dumps(s_dict, ensure_ascii=True) s_dict = json.loads(s_str, object_hook=_decode_dict) xsweep = XSweep.from_dict(s_dict) w_dict = wav.to_dict() w_str = json.dumps(w_dict, ensure_ascii=True) w_dict = json.loads(w_str, object_hook=_decode_dict) xwav = XWavelength.from_dict(w_dict) assert xwav.get_sweeps()[0].get_wavelength() is xwav c_dict = cryst.to_dict() c_str = json.dumps(c_dict, ensure_ascii=True) c_dict = json.loads(c_str, object_hook=_decode_dict) xcryst = XCrystal.from_dict(c_dict) assert xcryst.get_xwavelength( xcryst.get_wavelength_names()[0]).get_crystal() is xcryst p_dict = proj.to_dict() p_str = json.dumps(p_dict, ensure_ascii=True) p_dict = json.loads(p_str, object_hook=_decode_dict) xproj = XProject.from_dict(p_dict) assert xproj.get_crystals().values()[0].get_project() is xproj json_str = proj.as_json() xproj = XProject.from_json(string=json_str) assert xproj.get_crystals().values()[0].get_project() is xproj print(xproj.get_output()) print("\n".join(xproj.summarise())) json_str = xproj.as_json() xproj = XProject.from_json(string=json_str) assert xproj.get_crystals().values()[0].get_project() is xproj xcryst = xproj.get_crystals().values()[0] intgr = xcryst._get_integraters()[0] assert intgr.get_integrater_finish_done() assert xcryst._get_scaler()._sweep_handler.get_sweep_information( intgr.get_integrater_epoch()).get_integrater() is intgr print(xproj.get_output()) print("\n".join(xproj.summarise()))
def xia2_main(stop_after=None): '''Actually process something...''' Citations.cite('xia2') # print versions of related software Chatter.write(dials_version()) ccp4_version = get_ccp4_version() if ccp4_version is not None: Chatter.write('CCP4 %s' % ccp4_version) start_time = time.time() CommandLine = get_command_line() start_dir = Flags.get_starting_directory() # check that something useful has been assigned for processing... xtals = CommandLine.get_xinfo().get_crystals() no_images = True for name in xtals.keys(): xtal = xtals[name] if not xtal.get_all_image_names(): Chatter.write('-----------------------------------' + \ '-' * len(name)) Chatter.write('| No images assigned for crystal %s |' % name) Chatter.write('-----------------------------------' + '-' \ * len(name)) else: no_images = False args = [] from xia2.Handlers.Phil import PhilIndex params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing njob = mp_params.njob from libtbx import group_args xinfo = CommandLine.get_xinfo() if os.path.exists('xia2.json'): from xia2.Schema.XProject import XProject xinfo_new = xinfo xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() crystals_new = xinfo_new.get_crystals() for crystal_id in crystals_new.keys(): if crystal_id not in crystals: crystals[crystal_id] = crystals_new[crystal_id] continue crystals[crystal_id]._scaler = None # reset scaler for wavelength_id in crystals_new[crystal_id].get_wavelength_names( ): wavelength_new = crystals_new[crystal_id].get_xwavelength( wavelength_id) if wavelength_id not in crystals[ crystal_id].get_wavelength_names(): crystals[crystal_id].add_wavelength( crystals_new[crystal_id].get_xwavelength( wavelength_new)) continue wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps_new = wavelength_new.get_sweeps() sweeps = wavelength.get_sweeps() sweep_names = [s.get_name() for s in sweeps] sweep_keys = [(s.get_directory(), s.get_template(), s.get_image_range()) for s in sweeps] for sweep in sweeps_new: if ((sweep.get_directory(), sweep.get_template(), sweep.get_image_range()) not in sweep_keys): if sweep.get_name() in sweep_names: i = 1 while 'SWEEEP%i' % i in sweep_names: i += 1 sweep._name = 'SWEEP%i' % i break wavelength.add_sweep( name=sweep.get_name(), sample=sweep.get_xsample(), directory=sweep.get_directory(), image=sweep.get_image(), beam=sweep.get_beam_centre(), reversephi=sweep.get_reversephi(), distance=sweep.get_distance(), gain=sweep.get_gain(), dmin=sweep.get_resolution_high(), dmax=sweep.get_resolution_low(), polarization=sweep.get_polarization(), frames_to_process=sweep.get_frames_to_process(), user_lattice=sweep.get_user_lattice(), user_cell=sweep.get_user_cell(), epoch=sweep._epoch, ice=sweep._ice, excluded_regions=sweep._excluded_regions, ) sweep_names.append(sweep.get_name()) crystals = xinfo.get_crystals() failover = params.xia2.settings.failover if mp_params.mode == 'parallel' and njob > 1: driver_type = mp_params.type command_line_args = CommandLine.get_argv()[1:] for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps = wavelength.get_sweeps() for sweep in sweeps: sweep._get_indexer() sweep._get_refiner() sweep._get_integrater() args.append((group_args( driver_type=driver_type, stop_after=stop_after, failover=failover, command_line_args=command_line_args, nproc=mp_params.nproc, crystal_id=crystal_id, wavelength_id=wavelength_id, sweep_id=sweep.get_name(), ), )) from xia2.Driver.DriverFactory import DriverFactory default_driver_type = DriverFactory.get_driver_type() # run every nth job on the current computer (no need to submit to qsub) for i_job, arg in enumerate(args): if (i_job % njob) == 0: arg[0].driver_type = default_driver_type if mp_params.type == "qsub": method = "sge" else: method = "multiprocessing" nproc = mp_params.nproc qsub_command = mp_params.qsub_command if not qsub_command: qsub_command = 'qsub' qsub_command = '%s -V -cwd -pe smp %d' % (qsub_command, nproc) from libtbx import easy_mp results = easy_mp.parallel_map( process_one_sweep, args, processes=njob, #method=method, method="multiprocessing", qsub_command=qsub_command, preserve_order=True, preserve_exception_message=True) # Hack to update sweep with the serialized indexers/refiners/integraters i_sweep = 0 for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: success, output, xsweep_dict = results[i_sweep] if output is not None: Chatter.write(output) if not success: Chatter.write('Sweep failed: removing %s' % sweep.get_name()) remove_sweeps.append(sweep) else: assert xsweep_dict is not None Chatter.write('Loading sweep: %s' % sweep.get_name()) from xia2.Schema.XSweep import XSweep new_sweep = XSweep.from_dict(xsweep_dict) sweep._indexer = new_sweep._indexer sweep._refiner = new_sweep._refiner sweep._integrater = new_sweep._integrater i_sweep += 1 for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) else: for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: from dials.command_line.show import show_datablocks from dxtbx.datablock import DataBlock Debug.write(sweep.get_name()) Debug.write( show_datablocks([DataBlock([sweep.get_imageset()])])) try: if stop_after == 'index': sweep.get_indexer_cell() else: sweep.get_integrater_intensities() sweep.serialize() except Exception as e: if failover: Chatter.write('Processing sweep %s failed: %s' % \ (sweep.get_name(), str(e))) remove_sweeps.append(sweep) else: raise for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) # save intermediate xia2.json file in case scaling step fails xinfo.as_json(filename='xia2.json') if stop_after not in ('index', 'integrate'): Chatter.write(xinfo.get_output(), strip=False) for crystal in crystals.values(): crystal.serialize() # save final xia2.json file in case report generation fails xinfo.as_json(filename='xia2.json') duration = time.time() - start_time # write out the time taken in a human readable way Chatter.write('Processing took %s' % \ time.strftime("%Hh %Mm %Ss", time.gmtime(duration))) if stop_after not in ('index', 'integrate'): # and the summary file with open('xia2-summary.dat', 'w') as fh: for record in xinfo.summarise(): fh.write('%s\n' % record) # looks like this import overwrites the initial command line # Phil overrides so... for https://github.com/xia2/xia2/issues/150 from xia2.command_line.html import generate_xia2_html if params.xia2.settings.small_molecule == True: params.xia2.settings.report.xtriage_analysis = False params.xia2.settings.report.include_radiation_damage = False generate_xia2_html(xinfo, filename='xia2.html', params=params.xia2.settings.report) write_citations() # delete all of the temporary mtz files... cleanup() Environment.cleanup()
def exercise_serialization(dials_data, tmp_dir): base_path = pathlib.Path(tmp_dir) template = dials_data("insulin").join("insulin_1_###.img").strpath from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Refiner.DialsRefiner import DialsRefiner from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA from dxtbx.model.experiment_list import ExperimentListTemplateImporter importer = ExperimentListTemplateImporter([template]) experiments = importer.experiments imageset = experiments.imagesets()[0] from xia2.Schema.XProject import XProject from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample proj = XProject(base_path=base_path) proj._name = "PROJ1" cryst = XCrystal("CRYST1", proj) wav = XWavelength("WAVE1", cryst, wavelength=0.98) samp = XSample("X1", cryst) cryst.add_wavelength(wav) cryst.set_ha_info({"atom": "S"}) cryst.add_sample(samp) directory, image = os.path.split(imageset.get_path(1)) sweep = wav.add_sweep(name="SWEEP1", sample=samp, directory=directory, image=image) samp.add_sweep(sweep) from dxtbx.serialize.load import _decode_dict indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) indexer.add_indexer_imageset(imageset) indexer.set_indexer_sweep(sweep) sweep._indexer = indexer refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) refiner.add_refiner_sweep(sweep) sweep._refiner = refiner integrater = DialsIntegrater() integrater.set_output_format("hkl") integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) # integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.set_integrater_epoch(sweep.get_epoch(1)) integrater.set_integrater_sweep_name(sweep.get_name()) integrater.set_integrater_project_info( cryst.get_name(), wav.get_name(), sweep.get_name() ) sweep._integrater = integrater scaler = CCP4ScalerA(base_path=base_path) scaler.add_scaler_integrater(integrater) scaler.set_scaler_xcrystal(cryst) scaler.set_scaler_project_info(cryst.get_name(), wav.get_name()) scaler._scalr_xcrystal = cryst cryst._scaler = scaler proj.add_crystal(cryst) s_dict = sweep.to_dict() s_str = json.dumps(s_dict, ensure_ascii=True) s_dict = json.loads(s_str, object_hook=_decode_dict) xsweep = XSweep.from_dict(s_dict) assert xsweep w_dict = wav.to_dict() w_str = json.dumps(w_dict, ensure_ascii=True) w_dict = json.loads(w_str, object_hook=_decode_dict) xwav = XWavelength.from_dict(w_dict) assert xwav.get_sweeps()[0].get_wavelength() is xwav c_dict = cryst.to_dict() c_str = json.dumps(c_dict, ensure_ascii=True) c_dict = json.loads(c_str, object_hook=_decode_dict) xcryst = XCrystal.from_dict(c_dict) assert ( xcryst.get_xwavelength(xcryst.get_wavelength_names()[0]).get_crystal() is xcryst ) p_dict = proj.to_dict() p_str = json.dumps(p_dict, ensure_ascii=True) p_dict = json.loads(p_str, object_hook=_decode_dict) xproj = XProject.from_dict(p_dict) assert xproj.path == base_path assert list(xproj.get_crystals().values())[0].get_project() is xproj assert list(xproj.get_crystals().values())[0]._scaler._base_path == base_path json_str = proj.as_json() xproj = XProject.from_json(string=json_str) assert xproj.path == base_path assert list(xproj.get_crystals().values())[0].get_project() is xproj print(xproj.get_output()) print("\n".join(xproj.summarise())) json_str = xproj.as_json() xproj = XProject.from_json(string=json_str) assert xproj.path == base_path # Test that we can serialize to json and back again xproj = XProject.from_json(string=xproj.as_json()) assert xproj.path == base_path xcryst = list(xproj.get_crystals().values())[0] assert xcryst.get_project() is xproj intgr = xcryst._get_integraters()[0] assert intgr.get_integrater_finish_done() assert ( xcryst._get_scaler() ._sweep_handler.get_sweep_information(intgr.get_integrater_epoch()) .get_integrater() is intgr ) print(xproj.get_output()) print("\n".join(xproj.summarise()))
def exercise_serialization(): if not have_dials_regression: print "Skipping exercise_serialization(): dials_regression not configured" return from xia2.Handlers.CommandLine import CommandLine xia2_demo_data = os.path.join(dials_regression, "xia2_demo_data") template = os.path.join(xia2_demo_data, "insulin_1_###.img") cwd = os.path.abspath(os.curdir) tmp_dir = os.path.abspath(open_tmp_directory()) os.chdir(tmp_dir) from xia2.Modules.Indexer.DialsIndexer import DialsIndexer from xia2.Modules.Refiner.DialsRefiner import DialsRefiner from xia2.Modules.Integrater.DialsIntegrater import DialsIntegrater from xia2.Modules.Scaler.CCP4ScalerA import CCP4ScalerA from dxtbx.datablock import DataBlockTemplateImporter importer = DataBlockTemplateImporter([template]) datablocks = importer.datablocks imageset = datablocks[0].extract_imagesets()[0] from xia2.Schema.XProject import XProject from xia2.Schema.XCrystal import XCrystal from xia2.Schema.XWavelength import XWavelength from xia2.Schema.XSweep import XSweep from xia2.Schema.XSample import XSample proj = XProject() proj._name = "PROJ1" cryst = XCrystal("CRYST1", proj) proj.add_crystal(cryst) wav = XWavelength("WAVE1", cryst, wavelength=0.98) samp = XSample("X1", cryst) cryst.add_wavelength(wav) cryst.set_ha_info({'atom': 'S'}) directory, image = os.path.split(imageset.get_path(1)) wav.add_sweep(name='SWEEP1', sample=samp, directory=directory, image=image) import json from dxtbx.serialize.load import _decode_dict sweep = wav.get_sweeps()[0] indexer = DialsIndexer() indexer.set_working_directory(tmp_dir) indexer.add_indexer_imageset(imageset) indexer.set_indexer_sweep(sweep) sweep._indexer = indexer refiner = DialsRefiner() refiner.set_working_directory(tmp_dir) refiner.add_refiner_indexer(sweep.get_epoch(1), indexer) sweep._refiner = refiner integrater = DialsIntegrater() integrater.set_working_directory(tmp_dir) integrater.setup_from_image(imageset.get_path(1)) integrater.set_integrater_refiner(refiner) #integrater.set_integrater_indexer(indexer) integrater.set_integrater_sweep(sweep) integrater.set_integrater_epoch(sweep.get_epoch(1)) integrater.set_integrater_sweep_name(sweep.get_name()) integrater.set_integrater_project_info( cryst.get_name(), wav.get_name(), sweep.get_name()) sweep._integrater = integrater scaler = CCP4ScalerA() scaler.add_scaler_integrater(integrater) scaler.set_scaler_xcrystal(cryst) scaler.set_scaler_project_info(cryst.get_name(), wav.get_name()) scaler._scalr_xcrystal = cryst cryst._scaler = scaler s_dict = sweep.to_dict() s_str = json.dumps(s_dict, ensure_ascii=True) s_dict = json.loads(s_str, object_hook=_decode_dict) xsweep = XSweep.from_dict(s_dict) w_dict = wav.to_dict() w_str = json.dumps(w_dict, ensure_ascii=True) w_dict = json.loads(w_str, object_hook=_decode_dict) xwav = XWavelength.from_dict(w_dict) assert xwav.get_sweeps()[0].get_wavelength() is xwav c_dict = cryst.to_dict() c_str = json.dumps(c_dict, ensure_ascii=True) c_dict = json.loads(c_str, object_hook=_decode_dict) xcryst = XCrystal.from_dict(c_dict) assert xcryst.get_xwavelength(xcryst.get_wavelength_names()[0]).get_crystal() is xcryst p_dict = proj.to_dict() p_str = json.dumps(p_dict, ensure_ascii=True) p_dict = json.loads(p_str, object_hook=_decode_dict) xproj = XProject.from_dict(p_dict) assert xproj.get_crystals().values()[0].get_project() is xproj json_str = proj.as_json() xproj = XProject.from_json(string=json_str) assert xproj.get_crystals().values()[0].get_project() is xproj print xproj.get_output() print "\n".join(xproj.summarise()) json_str = xproj.as_json() xproj = XProject.from_json(string=json_str) assert xproj.get_crystals().values()[0].get_project() is xproj xcryst = xproj.get_crystals().values()[0] intgr = xcryst._get_integraters()[0] assert intgr.get_integrater_finish_done() assert xcryst._get_scaler()._sweep_handler.get_sweep_information( intgr.get_integrater_epoch()).get_integrater() is intgr print xproj.get_output() print "\n".join(xproj.summarise()) print
def xia2_main(stop_after=None): """Actually process something...""" Citations.cite("xia2") # print versions of related software logger.info(dials_version()) ccp4_version = get_ccp4_version() if ccp4_version: logger.info("CCP4 %s", ccp4_version) start_time = time.time() CommandLine = get_command_line() # check that something useful has been assigned for processing... xtals = CommandLine.get_xinfo().get_crystals() for name, xtal in xtals.items(): if not xtal.get_all_image_names(): logger.info("-----------------------------------" + "-" * len(name)) logger.info("| No images assigned for crystal %s |", name) logger.info("-----------------------------------" + "-" * len(name)) from xia2.Handlers.Phil import PhilIndex params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing njob = mp_params.njob xinfo = CommandLine.get_xinfo() logger.info("Project directory: %s", xinfo.path) if (params.xia2.settings.developmental.continue_from_previous_job and os.path.exists("xia2.json")): logger.debug("==== Starting from existing xia2.json ====") xinfo_new = xinfo xinfo = XProject.from_json(filename="xia2.json") crystals = xinfo.get_crystals() crystals_new = xinfo_new.get_crystals() for crystal_id in crystals_new: if crystal_id not in crystals: crystals[crystal_id] = crystals_new[crystal_id] continue crystals[crystal_id]._scaler = None # reset scaler for wavelength_id in crystals_new[crystal_id].get_wavelength_names( ): wavelength_new = crystals_new[crystal_id].get_xwavelength( wavelength_id) if wavelength_id not in crystals[ crystal_id].get_wavelength_names(): crystals[crystal_id].add_wavelength( crystals_new[crystal_id].get_xwavelength( wavelength_new)) continue wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps_new = wavelength_new.get_sweeps() sweeps = wavelength.get_sweeps() sweep_names = {s.get_name() for s in sweeps} sweep_keys = {(s.get_directory(), s.get_template(), s.get_image_range()) for s in sweeps} for sweep in sweeps_new: if ( sweep.get_directory(), sweep.get_template(), sweep.get_image_range(), ) not in sweep_keys: if sweep.get_name() in sweep_names: i = 1 while "SWEEEP%i" % i in sweep_names: i += 1 sweep._name = "SWEEP%i" % i break wavelength.add_sweep( name=sweep.get_name(), sample=sweep.sample, directory=sweep.get_directory(), image=sweep.get_image(), beam=sweep.get_beam_centre(), reversephi=sweep.get_reversephi(), distance=sweep.get_distance(), gain=sweep.get_gain(), dmin=sweep.get_resolution_high(), dmax=sweep.get_resolution_low(), polarization=sweep.get_polarization(), frames_to_process=sweep.get_frames_to_process(), user_lattice=sweep.get_user_lattice(), user_cell=sweep.get_user_cell(), epoch=sweep._epoch, ice=sweep._ice, excluded_regions=sweep._excluded_regions, ) sweep_names.add(sweep.get_name()) crystals = xinfo.get_crystals() failover = params.xia2.settings.failover with cleanup(xinfo.path): if mp_params.mode == "parallel" and njob > 1: driver_type = mp_params.type command_line_args = CommandLine.get_argv()[1:] jobs = [] for crystal_id in crystals: for wavelength_id in crystals[crystal_id].get_wavelength_names( ): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps = wavelength.get_sweeps() for sweep in sweeps: sweep._get_indexer() sweep._get_refiner() sweep._get_integrater() jobs.append((group_args( driver_type=driver_type, stop_after=stop_after, failover=failover, command_line_args=command_line_args, nproc=mp_params.nproc, crystal_id=crystal_id, wavelength_id=wavelength_id, sweep_id=sweep.get_name(), ), )) from xia2.Driver.DriverFactory import DriverFactory default_driver_type = DriverFactory.get_driver_type() # run every nth job on the current computer (no need to submit to qsub) for i_job, arg in enumerate(jobs): if (i_job % njob) == 0: arg[0].driver_type = default_driver_type nproc = mp_params.nproc qsub_command = mp_params.qsub_command or "qsub" qsub_command = "%s -V -cwd -pe smp %d" % (qsub_command, nproc) from libtbx import easy_mp results = easy_mp.parallel_map( process_one_sweep, jobs, processes=njob, method="multiprocessing", qsub_command=qsub_command, preserve_order=True, preserve_exception_message=True, ) # Hack to update sweep with the serialized indexers/refiners/integraters i_sweep = 0 for crystal_id in crystals: for wavelength_id in crystals[crystal_id].get_wavelength_names( ): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: success, output, xsweep_dict = results[i_sweep] if output is not None: logger.info(output) if not success: logger.info("Sweep failed: removing %s", sweep.get_name()) remove_sweeps.append(sweep) else: assert xsweep_dict is not None logger.info("Loading sweep: %s", sweep.get_name()) new_sweep = XSweep.from_dict(xsweep_dict) sweep._indexer = new_sweep._indexer sweep._refiner = new_sweep._refiner sweep._integrater = new_sweep._integrater i_sweep += 1 for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.sample sample.remove_sweep(sweep) else: for crystal_id in list(crystals.keys()): for wavelength_id in crystals[crystal_id].get_wavelength_names( ): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: from dials.command_line.show import show_experiments from dxtbx.model.experiment_list import ExperimentListFactory logger.debug(sweep.get_name()) logger.debug( show_experiments( ExperimentListFactory. from_imageset_and_crystal( sweep.get_imageset(), None))) Citations.cite("dials") try: if stop_after == "index": sweep.get_indexer_cell() else: sweep.get_integrater_intensities() sweep.serialize() except Exception as e: if failover: logger.info( "Processing sweep %s failed: %s", sweep.get_name(), str(e), ) remove_sweeps.append(sweep) else: raise for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.sample sample.remove_sweep(sweep) # save intermediate xia2.json file in case scaling step fails xinfo.as_json(filename="xia2.json") if stop_after not in ("index", "integrate"): logger.info(xinfo.get_output()) for crystal in list(crystals.values()): crystal.serialize() # save final xia2.json file in case report generation fails xinfo.as_json(filename="xia2.json") if stop_after not in ("index", "integrate"): # and the summary file with open("xia2-summary.dat", "w") as fh: for record in xinfo.summarise(): fh.write("%s\n" % record) # looks like this import overwrites the initial command line # Phil overrides so... for https://github.com/xia2/xia2/issues/150 from xia2.command_line.html import generate_xia2_html if params.xia2.settings.small_molecule: params.xia2.settings.report.xtriage_analysis = False params.xia2.settings.report.include_radiation_damage = False with xia2.Driver.timing.record_step("xia2.report"): generate_xia2_html(xinfo, filename="xia2.html", params=params.xia2.settings.report) duration = time.time() - start_time # write out the time taken in a human readable way logger.info("Processing took %s", time.strftime("%Hh %Mm %Ss", time.gmtime(duration))) write_citations()
def xia2_main(stop_after=None): '''Actually process something...''' Citations.cite('xia2') # print versions of related software from dials.util.version import dials_version Chatter.write(dials_version()) start_time = time.time() CommandLine = get_command_line() start_dir = Flags.get_starting_directory() # check that something useful has been assigned for processing... xtals = CommandLine.get_xinfo().get_crystals() no_images = True for name in xtals.keys(): xtal = xtals[name] if not xtal.get_all_image_names(): Chatter.write('-----------------------------------' + \ '-' * len(name)) Chatter.write('| No images assigned for crystal %s |' % name) Chatter.write('-----------------------------------' + '-' \ * len(name)) else: no_images = False args = [] from xia2.Handlers.Phil import PhilIndex params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing njob = mp_params.njob from libtbx import group_args xinfo = CommandLine.get_xinfo() if os.path.exists('xia2.json'): from xia2.Schema.XProject import XProject xinfo_new = xinfo xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() crystals_new = xinfo_new.get_crystals() for crystal_id in crystals_new.keys(): if crystal_id not in crystals: crystals[crystal_id] = crystals_new[crystal_id] continue crystals[crystal_id]._scaler = None # reset scaler for wavelength_id in crystals_new[crystal_id].get_wavelength_names(): wavelength_new = crystals_new[crystal_id].get_xwavelength(wavelength_id) if wavelength_id not in crystals[crystal_id].get_wavelength_names(): crystals[crystal_id].add_wavelength( crystals_new[crystal_id].get_xwavelength(wavelength_new)) continue wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) sweeps_new = wavelength_new.get_sweeps() sweeps = wavelength.get_sweeps() sweep_names = [s.get_name() for s in sweeps] sweep_keys = [ (s.get_directory(), s.get_template(), s.get_image_range()) for s in sweeps] for sweep in sweeps_new: if ((sweep.get_directory(), sweep.get_template(), sweep.get_image_range()) not in sweep_keys): if sweep.get_name() in sweep_names: i = 1 while 'SWEEEP%i' %i in sweep_names: i += 1 sweep._name = 'SWEEP%i' %i break wavelength.add_sweep( name=sweep.get_name(), directory=sweep.get_directory(), image=sweep.get_image(), beam=sweep.get_beam_centre(), reversephi=sweep.get_reversephi(), distance=sweep.get_distance(), gain=sweep.get_gain(), dmin=sweep.get_resolution_high(), dmax=sweep.get_resolution_low(), polarization=sweep.get_polarization(), frames_to_process=sweep.get_frames_to_process(), user_lattice=sweep.get_user_lattice(), user_cell=sweep.get_user_cell(), epoch=sweep._epoch, ice=sweep._ice, excluded_regions=sweep._excluded_regions, ) sweep_names.append(sweep.get_name()) crystals = xinfo.get_crystals() failover = params.xia2.settings.failover if njob > 1: driver_type = mp_params.type command_line_args = CommandLine.get_argv()[1:] for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) sweeps = wavelength.get_sweeps() for sweep in sweeps: sweep._get_indexer() sweep._get_refiner() sweep._get_integrater() args.append(( group_args( driver_type=driver_type, stop_after=stop_after, failover=failover, command_line_args=command_line_args, nproc=mp_params.nproc, crystal_id=crystal_id, wavelength_id=wavelength_id, sweep_id=sweep.get_name(), ),)) from xia2.Driver.DriverFactory import DriverFactory default_driver_type = DriverFactory.get_driver_type() # run every nth job on the current computer (no need to submit to qsub) for i_job, arg in enumerate(args): if (i_job % njob) == 0: arg[0].driver_type = default_driver_type if mp_params.type == "qsub": method = "sge" else: method = "multiprocessing" nproc = mp_params.nproc qsub_command = mp_params.qsub_command if not qsub_command: qsub_command = 'qsub' qsub_command = '%s -V -cwd -pe smp %d' %(qsub_command, nproc) from libtbx import easy_mp results = easy_mp.parallel_map( process_one_sweep, args, processes=njob, #method=method, method="multiprocessing", qsub_command=qsub_command, preserve_order=True, preserve_exception_message=True) # Hack to update sweep with the serialized indexers/refiners/integraters i_sweep = 0 for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: success, output, xsweep_dict = results[i_sweep] assert xsweep_dict is not None if output is not None: Chatter.write(output) if not success: Chatter.write('Sweep failed: removing %s' %sweep.get_name()) remove_sweeps.append(sweep) else: Chatter.write('Loading sweep: %s' % sweep.get_name()) from xia2.Schema.XSweep import XSweep new_sweep = XSweep.from_dict(xsweep_dict) sweep._indexer = new_sweep._indexer sweep._refiner = new_sweep._refiner sweep._integrater = new_sweep._integrater i_sweep += 1 for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) else: for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: try: if stop_after == 'index': sweep.get_indexer_cell() else: sweep.get_integrater_intensities() sweep.serialize() except Exception, e: if failover: Chatter.write('Processing sweep %s failed: %s' % \ (sweep.get_name(), str(e))) remove_sweeps.append(sweep) else: raise for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep)