def check_environment(): '''Check the environment we are running in...''' if sys.hexversion < 0x02070000: raise RuntimeError('Python versions older than 2.7 are not supported') import cctbx executable = sys.executable cctbx_dir = os.sep.join(cctbx.__file__.split(os.sep)[:-3]) # to help wrapper code - print process id... Debug.write('Process ID: %d' % os.getpid()) Chatter.write('Environment configuration...') Chatter.write('Python => %s' % executable) Chatter.write('CCTBX => %s' % cctbx_dir) ccp4_keys = ['CCP4', 'CLIBD', 'CCP4_SCR'] for k in ccp4_keys: v = Environment.getenv(k) if not v: raise RuntimeError('%s not defined - is CCP4 set up?' % k) if not v == v.strip(): raise RuntimeError('spaces around "%s"' % v) Chatter.write('%s => %s' % (k, v)) from xia2.Handlers.Flags import Flags Chatter.write('Starting directory: %s' % Flags.get_starting_directory()) Chatter.write('Working directory: %s' % os.getcwd()) # temporary workaround to bug in pointless... if ' ' in os.getcwd(): raise RuntimeError('Space in working directory ' \ '(https://github.com/xia2/xia2/issues/114)') Chatter.write('Free space: %.2f GB' % (df() / math.pow(2, 30))) try: if os.name == 'nt': hostname = os.environ['COMPUTERNAME'].split('.')[0] else: hostname = os.environ['HOSTNAME'].split('.')[0] Chatter.write('Host: %s' % hostname) except KeyError: pass Chatter.write('Contact: [email protected]') Chatter.write(Version)
def check_environment(): """Check the environment we are running in...""" if sys.hexversion < 0x02070000: raise RuntimeError("Python versions older than 2.7 are not supported") import cctbx executable = sys.executable cctbx_dir = os.sep.join(cctbx.__file__.split(os.sep)[:-3]) # to help wrapper code - print process id... logger.debug("Process ID: %d", os.getpid()) logger.info("Environment configuration...") logger.info("Python => %s", executable) logger.info("CCTBX => %s", cctbx_dir) ccp4_keys = ["CCP4", "CCP4_SCR"] for k in ccp4_keys: v = os.getenv(k) if not v: raise RuntimeError("%s not defined - is CCP4 set up?" % k) if not v == v.strip(): raise RuntimeError('spaces around "%s"' % v) logger.info(f"{k} => {v}") from xia2.Handlers.Flags import Flags logger.info("Starting directory: %s", Flags.get_starting_directory()) logger.info("Working directory: %s", os.getcwd()) logger.info("Free space: %.2f GB", df() / math.pow(2, 30)) hostname = platform.node().split(".")[0] logger.info("Host: %s", hostname) logger.info("Contact: [email protected]") logger.info(Version) # temporary workaround to bug in pointless... if " " in os.getcwd(): raise RuntimeError( "Space in working directory " "(https://github.com/xia2/xia2/issues/114)" )
def xia2_main(stop_after=None): '''Actually process something...''' Citations.cite('xia2') # print versions of related software Chatter.write(dials_version()) ccp4_version = get_ccp4_version() if ccp4_version is not None: Chatter.write('CCP4 %s' % ccp4_version) start_time = time.time() CommandLine = get_command_line() start_dir = Flags.get_starting_directory() # check that something useful has been assigned for processing... xtals = CommandLine.get_xinfo().get_crystals() no_images = True for name in xtals.keys(): xtal = xtals[name] if not xtal.get_all_image_names(): Chatter.write('-----------------------------------' + \ '-' * len(name)) Chatter.write('| No images assigned for crystal %s |' % name) Chatter.write('-----------------------------------' + '-' \ * len(name)) else: no_images = False args = [] from xia2.Handlers.Phil import PhilIndex params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing njob = mp_params.njob from libtbx import group_args xinfo = CommandLine.get_xinfo() if os.path.exists('xia2.json'): from xia2.Schema.XProject import XProject xinfo_new = xinfo xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() crystals_new = xinfo_new.get_crystals() for crystal_id in crystals_new.keys(): if crystal_id not in crystals: crystals[crystal_id] = crystals_new[crystal_id] continue crystals[crystal_id]._scaler = None # reset scaler for wavelength_id in crystals_new[crystal_id].get_wavelength_names( ): wavelength_new = crystals_new[crystal_id].get_xwavelength( wavelength_id) if wavelength_id not in crystals[ crystal_id].get_wavelength_names(): crystals[crystal_id].add_wavelength( crystals_new[crystal_id].get_xwavelength( wavelength_new)) continue wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps_new = wavelength_new.get_sweeps() sweeps = wavelength.get_sweeps() sweep_names = [s.get_name() for s in sweeps] sweep_keys = [(s.get_directory(), s.get_template(), s.get_image_range()) for s in sweeps] for sweep in sweeps_new: if ((sweep.get_directory(), sweep.get_template(), sweep.get_image_range()) not in sweep_keys): if sweep.get_name() in sweep_names: i = 1 while 'SWEEEP%i' % i in sweep_names: i += 1 sweep._name = 'SWEEP%i' % i break wavelength.add_sweep( name=sweep.get_name(), sample=sweep.get_xsample(), directory=sweep.get_directory(), image=sweep.get_image(), beam=sweep.get_beam_centre(), reversephi=sweep.get_reversephi(), distance=sweep.get_distance(), gain=sweep.get_gain(), dmin=sweep.get_resolution_high(), dmax=sweep.get_resolution_low(), polarization=sweep.get_polarization(), frames_to_process=sweep.get_frames_to_process(), user_lattice=sweep.get_user_lattice(), user_cell=sweep.get_user_cell(), epoch=sweep._epoch, ice=sweep._ice, excluded_regions=sweep._excluded_regions, ) sweep_names.append(sweep.get_name()) crystals = xinfo.get_crystals() failover = params.xia2.settings.failover if mp_params.mode == 'parallel' and njob > 1: driver_type = mp_params.type command_line_args = CommandLine.get_argv()[1:] for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps = wavelength.get_sweeps() for sweep in sweeps: sweep._get_indexer() sweep._get_refiner() sweep._get_integrater() args.append((group_args( driver_type=driver_type, stop_after=stop_after, failover=failover, command_line_args=command_line_args, nproc=mp_params.nproc, crystal_id=crystal_id, wavelength_id=wavelength_id, sweep_id=sweep.get_name(), ), )) from xia2.Driver.DriverFactory import DriverFactory default_driver_type = DriverFactory.get_driver_type() # run every nth job on the current computer (no need to submit to qsub) for i_job, arg in enumerate(args): if (i_job % njob) == 0: arg[0].driver_type = default_driver_type if mp_params.type == "qsub": method = "sge" else: method = "multiprocessing" nproc = mp_params.nproc qsub_command = mp_params.qsub_command if not qsub_command: qsub_command = 'qsub' qsub_command = '%s -V -cwd -pe smp %d' % (qsub_command, nproc) from libtbx import easy_mp results = easy_mp.parallel_map( process_one_sweep, args, processes=njob, #method=method, method="multiprocessing", qsub_command=qsub_command, preserve_order=True, preserve_exception_message=True) # Hack to update sweep with the serialized indexers/refiners/integraters i_sweep = 0 for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: success, output, xsweep_dict = results[i_sweep] if output is not None: Chatter.write(output) if not success: Chatter.write('Sweep failed: removing %s' % sweep.get_name()) remove_sweeps.append(sweep) else: assert xsweep_dict is not None Chatter.write('Loading sweep: %s' % sweep.get_name()) from xia2.Schema.XSweep import XSweep new_sweep = XSweep.from_dict(xsweep_dict) sweep._indexer = new_sweep._indexer sweep._refiner = new_sweep._refiner sweep._integrater = new_sweep._integrater i_sweep += 1 for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) else: for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: from dials.command_line.show import show_datablocks from dxtbx.datablock import DataBlock Debug.write(sweep.get_name()) Debug.write( show_datablocks([DataBlock([sweep.get_imageset()])])) try: if stop_after == 'index': sweep.get_indexer_cell() else: sweep.get_integrater_intensities() sweep.serialize() except Exception as e: if failover: Chatter.write('Processing sweep %s failed: %s' % \ (sweep.get_name(), str(e))) remove_sweeps.append(sweep) else: raise for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) # save intermediate xia2.json file in case scaling step fails xinfo.as_json(filename='xia2.json') if stop_after not in ('index', 'integrate'): Chatter.write(xinfo.get_output(), strip=False) for crystal in crystals.values(): crystal.serialize() # save final xia2.json file in case report generation fails xinfo.as_json(filename='xia2.json') duration = time.time() - start_time # write out the time taken in a human readable way Chatter.write('Processing took %s' % \ time.strftime("%Hh %Mm %Ss", time.gmtime(duration))) if stop_after not in ('index', 'integrate'): # and the summary file with open('xia2-summary.dat', 'w') as fh: for record in xinfo.summarise(): fh.write('%s\n' % record) # looks like this import overwrites the initial command line # Phil overrides so... for https://github.com/xia2/xia2/issues/150 from xia2.command_line.html import generate_xia2_html if params.xia2.settings.small_molecule == True: params.xia2.settings.report.xtriage_analysis = False params.xia2.settings.report.include_radiation_damage = False generate_xia2_html(xinfo, filename='xia2.html', params=params.xia2.settings.report) write_citations() # delete all of the temporary mtz files... cleanup() Environment.cleanup()
def xia2_main(stop_after=None): '''Actually process something...''' Citations.cite('xia2') # print versions of related software from dials.util.version import dials_version Chatter.write(dials_version()) start_time = time.time() CommandLine = get_command_line() start_dir = Flags.get_starting_directory() # check that something useful has been assigned for processing... xtals = CommandLine.get_xinfo().get_crystals() no_images = True for name in xtals.keys(): xtal = xtals[name] if not xtal.get_all_image_names(): Chatter.write('-----------------------------------' + \ '-' * len(name)) Chatter.write('| No images assigned for crystal %s |' % name) Chatter.write('-----------------------------------' + '-' \ * len(name)) else: no_images = False args = [] from xia2.Handlers.Phil import PhilIndex params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing njob = mp_params.njob from libtbx import group_args xinfo = CommandLine.get_xinfo() if os.path.exists('xia2.json'): from xia2.Schema.XProject import XProject xinfo_new = xinfo xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() crystals_new = xinfo_new.get_crystals() for crystal_id in crystals_new.keys(): if crystal_id not in crystals: crystals[crystal_id] = crystals_new[crystal_id] continue crystals[crystal_id]._scaler = None # reset scaler for wavelength_id in crystals_new[crystal_id].get_wavelength_names(): wavelength_new = crystals_new[crystal_id].get_xwavelength(wavelength_id) if wavelength_id not in crystals[crystal_id].get_wavelength_names(): crystals[crystal_id].add_wavelength( crystals_new[crystal_id].get_xwavelength(wavelength_new)) continue wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) sweeps_new = wavelength_new.get_sweeps() sweeps = wavelength.get_sweeps() sweep_names = [s.get_name() for s in sweeps] sweep_keys = [ (s.get_directory(), s.get_template(), s.get_image_range()) for s in sweeps] for sweep in sweeps_new: if ((sweep.get_directory(), sweep.get_template(), sweep.get_image_range()) not in sweep_keys): if sweep.get_name() in sweep_names: i = 1 while 'SWEEEP%i' %i in sweep_names: i += 1 sweep._name = 'SWEEP%i' %i break wavelength.add_sweep( name=sweep.get_name(), directory=sweep.get_directory(), image=sweep.get_image(), beam=sweep.get_beam_centre(), reversephi=sweep.get_reversephi(), distance=sweep.get_distance(), gain=sweep.get_gain(), dmin=sweep.get_resolution_high(), dmax=sweep.get_resolution_low(), polarization=sweep.get_polarization(), frames_to_process=sweep.get_frames_to_process(), user_lattice=sweep.get_user_lattice(), user_cell=sweep.get_user_cell(), epoch=sweep._epoch, ice=sweep._ice, excluded_regions=sweep._excluded_regions, ) sweep_names.append(sweep.get_name()) crystals = xinfo.get_crystals() failover = params.xia2.settings.failover if njob > 1: driver_type = mp_params.type command_line_args = CommandLine.get_argv()[1:] for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) sweeps = wavelength.get_sweeps() for sweep in sweeps: sweep._get_indexer() sweep._get_refiner() sweep._get_integrater() args.append(( group_args( driver_type=driver_type, stop_after=stop_after, failover=failover, command_line_args=command_line_args, nproc=mp_params.nproc, crystal_id=crystal_id, wavelength_id=wavelength_id, sweep_id=sweep.get_name(), ),)) from xia2.Driver.DriverFactory import DriverFactory default_driver_type = DriverFactory.get_driver_type() # run every nth job on the current computer (no need to submit to qsub) for i_job, arg in enumerate(args): if (i_job % njob) == 0: arg[0].driver_type = default_driver_type if mp_params.type == "qsub": method = "sge" else: method = "multiprocessing" nproc = mp_params.nproc qsub_command = mp_params.qsub_command if not qsub_command: qsub_command = 'qsub' qsub_command = '%s -V -cwd -pe smp %d' %(qsub_command, nproc) from libtbx import easy_mp results = easy_mp.parallel_map( process_one_sweep, args, processes=njob, #method=method, method="multiprocessing", qsub_command=qsub_command, preserve_order=True, preserve_exception_message=True) # Hack to update sweep with the serialized indexers/refiners/integraters i_sweep = 0 for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: success, output, xsweep_dict = results[i_sweep] assert xsweep_dict is not None if output is not None: Chatter.write(output) if not success: Chatter.write('Sweep failed: removing %s' %sweep.get_name()) remove_sweeps.append(sweep) else: Chatter.write('Loading sweep: %s' % sweep.get_name()) from xia2.Schema.XSweep import XSweep new_sweep = XSweep.from_dict(xsweep_dict) sweep._indexer = new_sweep._indexer sweep._refiner = new_sweep._refiner sweep._integrater = new_sweep._integrater i_sweep += 1 for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) else: for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: try: if stop_after == 'index': sweep.get_indexer_cell() else: sweep.get_integrater_intensities() sweep.serialize() except Exception, e: if failover: Chatter.write('Processing sweep %s failed: %s' % \ (sweep.get_name(), str(e))) remove_sweeps.append(sweep) else: raise for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep)