finally: Chatter.uncache() Debug.uncache() return s remove = [] for s in self._sweeps: # would be nice to put this somewhere else in the hierarchy - not # sure how to do that though (should be handled in Interfaces?) try: result += '%s\n' % s.get_output() except Exception, e: if Flags.get_failover(): Chatter.write('Processing sweep %s failed: %s' % \ (s.get_name(), str(e))) remove.append(s) else: raise for s in remove: self._sweeps.remove(s) return result[:-1] def summarise(self): summary = ['Wavelength: %s (%7.5f)' % (self._name, self._wavelength)]
def _scale(self): '''Perform all of the operations required to deliver the scaled data.''' epochs = self._sweep_handler.get_epochs() if PhilIndex.params.xia2.settings.optimize_scaling: self._determine_best_scale_model_8way() else: self._scalr_corrections = True self._scalr_correct_absorption = True self._scalr_correct_partiality = False self._scalr_correct_decay = True if self._scalr_corrections: Journal.block( 'scaling', self.get_scaler_xcrystal().get_name(), 'CCP4', {'scaling model':'automatic', 'absorption':self._scalr_correct_absorption, 'tails':self._scalr_correct_partiality, 'decay':self._scalr_correct_decay }) else: Journal.block( 'scaling', self.get_scaler_xcrystal().get_name(), 'CCP4', {'scaling model':'default'}) sc = self._updated_aimless() sc.set_hklin(self._prepared_reflections) sc.set_intensities(PhilIndex.params.ccp4.aimless.intensities) sc.set_chef_unmerged(True) sc.set_new_scales_file('%s.scales' % self._scalr_xname) user_resolution_limits = { } for epoch in epochs: si = self._sweep_handler.get_sweep_information(epoch) pname, xname, dname = si.get_project_info() sname = si.get_sweep_name() intgr = si.get_integrater() if intgr.get_integrater_user_resolution(): dmin = intgr.get_integrater_high_resolution() if not user_resolution_limits.has_key((dname, sname)): user_resolution_limits[(dname, sname)] = dmin elif dmin < user_resolution_limits[(dname, sname)]: user_resolution_limits[(dname, sname)] = dmin start, end = si.get_batch_range() if (dname, sname) in self._scalr_resolution_limits: resolution = self._scalr_resolution_limits[(dname, sname)] sc.add_run(start, end, exclude = False, resolution = resolution, name = sname) else: sc.add_run(start, end, name = sname) sc.set_hklout(os.path.join(self.get_working_directory(), '%s_%s_scaled_test.mtz' % \ (self._scalr_pname, self._scalr_xname))) if self.get_scaler_anomalous(): sc.set_anomalous() # what follows, sucks if Flags.get_failover(): try: sc.scale() except RuntimeError, e: es = str(e) if 'bad batch' in es or \ 'negative scales run' in es or \ 'no observations' in es: # first ID the sweep from the batch no batch = int(es.split()[-1]) epoch = self._identify_sweep_epoch(batch) sweep = self._scalr_integraters[ epoch].get_integrater_sweep() # then remove it from my parent xcrystal self.get_scaler_xcrystal().remove_sweep(sweep) # then remove it from the scaler list of intergraters # - this should really be a scaler interface method del(self._scalr_integraters[epoch]) # then tell the user what is happening Chatter.write( 'Sweep %s gave negative scales - removing' % \ sweep.get_name()) # then reset the prepare, do, finish flags self.set_scaler_prepare_done(False) self.set_scaler_done(False) self.set_scaler_finish_done(False) # and return return else: raise e
def xia2_main(stop_after=None): '''Actually process something...''' #Flags.set_serialize_state(True) Citations.cite('xia2') # print versions of related software from dials.util.version import dials_version Chatter.write(dials_version()) start_time = time.time() CommandLine = get_command_line() # check that something useful has been assigned for processing... xtals = CommandLine.get_xinfo().get_crystals() no_images = True for name in xtals.keys(): xtal = xtals[name] if not xtal.get_all_image_names(): Chatter.write('-----------------------------------' + \ '-' * len(name)) Chatter.write('| No images assigned for crystal %s |' % name) Chatter.write('-----------------------------------' + '-' \ * len(name)) else: no_images = False args = [] from xia2.Handlers.Phil import PhilIndex params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing njob = mp_params.njob from libtbx import group_args xinfo = CommandLine.get_xinfo() if os.path.exists('xia2.json'): from xia2.Schema.XProject import XProject xinfo_new = xinfo xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() crystals_new = xinfo_new.get_crystals() for crystal_id in crystals_new.keys(): if crystal_id not in crystals: crystals[crystal_id] = crystals_new[crystal_id] continue crystals[crystal_id]._scaler = None # reset scaler for wavelength_id in crystals_new[crystal_id].get_wavelength_names(): wavelength_new = crystals_new[crystal_id].get_xwavelength(wavelength_id) if wavelength_id not in crystals[crystal_id].get_wavelength_names(): crystals[crystal_id].add_wavelength( crystals_new[crystal_id].get_xwavelength(wavelength_new)) continue wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) sweeps_new = wavelength_new.get_sweeps() sweeps = wavelength.get_sweeps() sweep_names = [s.get_name() for s in sweeps] sweep_keys = [ (s.get_directory(), s.get_template(), s.get_image_range()) for s in sweeps] for sweep in sweeps_new: if ((sweep.get_directory(), sweep.get_template(), sweep.get_image_range()) not in sweep_keys): if sweep.get_name() in sweep_names: i = 1 while 'SWEEEP%i' %i in sweep_names: i += 1 sweep._name = 'SWEEP%i' %i break wavelength.add_sweep( name=sweep.get_name(), directory=sweep.get_directory(), image=sweep.get_image(), beam=sweep.get_beam_centre(), reversephi=sweep.get_reversephi(), distance=sweep.get_distance(), gain=sweep.get_gain(), dmin=sweep.get_resolution_high(), dmax=sweep.get_resolution_low(), polarization=sweep.get_polarization(), frames_to_process=sweep.get_frames_to_process(), user_lattice=sweep.get_user_lattice(), user_cell=sweep.get_user_cell(), epoch=sweep._epoch, ice=sweep._ice, excluded_regions=sweep._excluded_regions, ) sweep_names.append(sweep.get_name()) crystals = xinfo.get_crystals() if njob > 1: driver_type = mp_params.type command_line_args = CommandLine.get_argv()[1:] command_line_args = [arg for arg in command_line_args if arg != '-failover'] for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) sweeps = wavelength.get_sweeps() for sweep in sweeps: sweep._get_indexer() sweep._get_refiner() sweep._get_integrater() args.append(( group_args( driver_type=driver_type, stop_after=stop_after, failover=Flags.get_failover(), command_line_args=command_line_args, nproc=mp_params.nproc, crystal_id=crystal_id, wavelength_id=wavelength_id, sweep_id=sweep.get_name(), ),)) from xia2.Driver.DriverFactory import DriverFactory default_driver_type = DriverFactory.get_driver_type() # run every nth job on the current computer (no need to submit to qsub) for i_job, arg in enumerate(args): if (i_job % njob) == 0: arg[0].driver_type = default_driver_type if mp_params.type == "qsub": method = "sge" else: method = "multiprocessing" nproc = mp_params.nproc qsub_command = mp_params.qsub_command if not qsub_command: qsub_command = 'qsub' qsub_command = '%s -V -cwd -pe smp %d' %(qsub_command, nproc) from libtbx import easy_mp results = easy_mp.parallel_map( process_one_sweep, args, processes=njob, #method=method, method="multiprocessing", qsub_command=qsub_command, preserve_order=True, preserve_exception_message=True) # Hack to update sweep with the serialized indexers/refiners/integraters i_sweep = 0 for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: success, output, xsweep_dict = results[i_sweep] assert xsweep_dict is not None if output is not None: Chatter.write(output) if not success: print "Sweep failed: removing %s" %sweep.get_name() remove_sweeps.append(sweep) else: print "Loading sweep: %s" %sweep.get_name() from xia2.Schema.XSweep import XSweep new_sweep = XSweep.from_dict(xsweep_dict) sweep._indexer = new_sweep._indexer sweep._refiner = new_sweep._refiner sweep._integrater = new_sweep._integrater i_sweep += 1 for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) else: for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) sweeps = wavelength.get_sweeps() for sweep in sweeps: try: if stop_after == 'index': sweep.get_indexer_cell() else: sweep.get_integrater_intensities() sweep.serialize() except Exception, e: if Flags.get_failover(): Chatter.write('Processing sweep %s failed: %s' % \ (sweep.get_name(), str(e))) wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) else: raise