def xia2_setup(): """Actually process something...""" Citations.cite("xia2") # print versions of related software logger.info(dials_version()) ccp4_version = get_ccp4_version() if ccp4_version: logger.info("CCP4 %s", ccp4_version) CommandLine = get_command_line() # check that something useful has been assigned for processing... xtals = CommandLine.get_xinfo().get_crystals() for name, xtal in xtals.items(): if not xtal.get_all_image_names(): logger.info("-----------------------------------" + "-" * len(name)) logger.info("| No images assigned for crystal %s |", name) logger.info("-----------------------------------" + "-" * len(name)) from xia2.Handlers.Phil import PhilIndex params = PhilIndex.get_python_object() xinfo = CommandLine.get_xinfo() logger.info(f"Project directory: {xinfo.path}") logger.info(f"xinfo written to: {params.xia2.settings.input.xinfo}") logger.info(f"Parameters: {PhilIndex.get_diff().as_str()}")
def _integrate_prepare(self): """Prepare for integration - in XDS terms this may mean rerunning IDXREF to get the XPARM etc. DEFPIX is considered part of the full integration as it is resolution dependent.""" Citations.cite("dials") # decide what images we are going to process, if not already # specified if not self._intgr_wedge: images = self.get_matching_images() self.set_integrater_wedge(min(images), max(images)) logger.debug("DIALS INTEGRATE PREPARE:") logger.debug("Wavelength: %.6f" % self.get_wavelength()) logger.debug("Distance: %.2f" % self.get_distance()) if not self.get_integrater_low_resolution(): dmax = self._intgr_refiner.get_indexer_low_resolution( self.get_integrater_epoch() ) self.set_integrater_low_resolution(dmax) logger.debug( "Low resolution set to: %s" % self.get_integrater_low_resolution() ) ## copy the data across refiner = self.get_integrater_refiner() # For multi-sweep refinement, get the split experiments from after refinement. if PhilIndex.params.xia2.settings.multi_sweep_refinement: self._intgr_experiments_filename = refiner.get_refiner_payload( f"{self._intgr_sweep._name}_models.expt" ) self._intgr_indexed_filename = refiner.get_refiner_payload( f"{self._intgr_sweep._name}_observations.refl" ) # Otherwise, there should only be a single experiment list and reflection table. else: self._intgr_experiments_filename = refiner.get_refiner_payload( "models.expt" ) self._intgr_indexed_filename = refiner.get_refiner_payload( "observations.refl" ) experiments = load.experiment_list(self._intgr_experiments_filename) experiment = experiments[0] # this is the result of the cell refinement self._intgr_cell = experiment.crystal.get_unit_cell().parameters() logger.debug("Files available at the end of DIALS integrate prepare:") for f in self._data_files: logger.debug("%s" % f) self.set_detector(experiment.detector) self.set_beam_obj(experiment.beam) self.set_goniometer(experiment.goniometer)
def run(args): cmd_line = command_line.argument_interpreter( master_params=master_phil_scope) working_phil, args = cmd_line.process_and_fetch( args=args, custom_processor="collect_remaining") working_phil.show() params = working_phil.extract() if params.unit_cell is not None: unit_cell = params.unit_cell crystal_symmetry = crystal.symmetry(unit_cell=unit_cell) else: crystal_symmetry = None from iotbx.reflection_file_reader import any_reflection_file result = any_reflection_file(args[0]) unmerged_intensities = None batches_all = None for ma in result.as_miller_arrays(merge_equivalents=False, crystal_symmetry=crystal_symmetry): #print ma.info().labels if ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']: assert ma.anomalous_flag() unmerged_intensities = ma elif ma.info().labels == ['I', 'SIGI']: assert not ma.anomalous_flag() unmerged_intensities = ma elif ma.info().labels == ['BATCH']: batches_all = ma assert batches_all is not None assert unmerged_intensities is not None id_to_batches = None if len(params.batch) > 0: id_to_batches = {} for b in params.batch: assert b.id is not None assert b.range is not None assert b.id not in id_to_batches, "Duplicate batch id: %s" % b.id id_to_batches[b.id] = b.range result = delta_cc_half(unmerged_intensities, batches_all, n_bins=params.n_bins, d_min=params.d_min, cc_one_half_method=params.cc_one_half_method, id_to_batches=id_to_batches) hist_filename = 'delta_cc_hist.png' print 'Saving histogram to %s' % hist_filename result.plot_histogram(hist_filename) print result.get_table() from xia2.Handlers.Citations import Citations Citations.cite('delta_cc_half') for citation in Citations.get_citations_acta(): print citation
def __init__(self): # generic things CCP4DriverInstance.__class__.__init__(self) Citations.cite("ccp4") self.set_executable(os.path.join(os.environ.get("CBIN", ""), "freerflag")) self._free_fraction = 0.05
def _integrate(self): '''Implement the integrater interface.''' # cite the program Citations.cite('mosflm') images_str = '%d to %d' % tuple(self._intgr_wedge) cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % tuple(self._intgr_cell) if len(self._fp_directory) <= 50: dirname = self._fp_directory else: dirname = '...%s' % self._fp_directory[-46:] Journal.block( 'integrating', self._intgr_sweep_name, 'mosflm', {'images':images_str, 'cell':cell_str, 'lattice':self.get_integrater_refiner().get_refiner_lattice(), 'template':self._fp_template, 'directory':dirname, 'resolution':'%.2f' % self._intgr_reso_high}) self._mosflm_rerun_integration = False wd = self.get_working_directory() try: if self.get_integrater_sweep_name(): pname, xname, dname = self.get_integrater_project_info() nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc if nproc > 1: Debug.write('Parallel integration: %d jobs' %nproc) self._mosflm_hklout = self._mosflm_parallel_integrate() else: self._mosflm_hklout = self._mosflm_integrate() # record integration output for e.g. BLEND. sweep = self.get_integrater_sweep_name() if sweep: FileHandler.record_more_data_file( '%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep), self._mosflm_hklout) except IntegrationError, e: if 'negative mosaic spread' in str(e): if self._mosflm_postref_fix_mosaic: Chatter.write( 'Negative mosaic spread - stopping integration') raise BadLatticeError, 'negative mosaic spread' Chatter.write( 'Negative mosaic spread - rerunning integration') self.set_integrater_done(False) self._mosflm_postref_fix_mosaic = True
def run(): if os.path.exists("xia2-working.phil"): sys.argv.append("xia2-working.phil") try: check_environment() except Exception as e: traceback.print_exc(file=open("xia2.error", "w")) Chatter.write('Status: error "%s"' % str(e)) # print the version Chatter.write(Version) Citations.cite("xia2") start_time = time.time() assert os.path.exists("xia2.json") from xia2.Schema.XProject import XProject xinfo = XProject.from_json(filename="xia2.json") crystals = xinfo.get_crystals() for crystal_id, crystal in crystals.iteritems(): # cwd = os.path.abspath(os.curdir) from libtbx import Auto scale_dir = PhilIndex.params.xia2.settings.scale.directory if scale_dir is Auto: scale_dir = "scale" i = 0 while os.path.exists(os.path.join(crystal.get_name(), scale_dir)): i += 1 scale_dir = "scale%i" % i PhilIndex.params.xia2.settings.scale.directory = scale_dir working_directory = Environment.generate_directory( [crystal.get_name(), scale_dir]) # os.chdir(working_directory) crystals[crystal_id]._scaler = None # reset scaler scaler = crystal._get_scaler() Chatter.write(xinfo.get_output()) crystal.serialize() duration = time.time() - start_time # write out the time taken in a human readable way Chatter.write("Processing took %s" % time.strftime("%Hh %Mm %Ss", time.gmtime(duration))) # delete all of the temporary mtz files... cleanup() write_citations() xinfo.as_json(filename="xia2.json") Environment.cleanup()
def run(): if os.path.exists("xia2-working.phil"): sys.argv.append("xia2-working.phil") try: check_environment() except Exception as e: with open("xia2-error.txt", "w") as fh: traceback.print_exc(file=fh) logger.error('Status: error "%s"', str(e)) # print the version logger.info(Version) Citations.cite("xia2") start_time = time.time() assert os.path.exists("xia2.json") from xia2.Schema.XProject import XProject xinfo = XProject.from_json(filename="xia2.json") with cleanup(xinfo.path): crystals = xinfo.get_crystals() for crystal_id, crystal in crystals.items(): scale_dir = PhilIndex.params.xia2.settings.scale.directory if scale_dir is Auto: scale_dir = "scale" i = 0 while os.path.exists(os.path.join(crystal.get_name(), scale_dir)): i += 1 scale_dir = "scale%i" % i PhilIndex.params.xia2.settings.scale.directory = scale_dir # reset scaler crystals[crystal_id]._scaler = None crystal._get_scaler() logger.info(xinfo.get_output()) crystal.serialize() duration = time.time() - start_time # write out the time taken in a human readable way logger.info( "Processing took %s", time.strftime("%Hh %Mm %Ss", time.gmtime(duration)) ) write_citations() xinfo.as_json(filename="xia2.json")
def write_citations(): # tell the user which programs were used... logger.info("XIA2 used... %s" % ", ".join(Citations.get_programs())) logger.info("Here are the appropriate citations (BIBTeX in xia2-citations.bib.)") for citation in Citations.get_citations_acta(): logger.info(citation) # and write the bibtex versions out = open("xia2-citations.bib", "w") for citation in Citations.get_citations(): out.write("%s\n" % citation) out.close()
def write_citations(): # tell the user which programs were used... Chatter.write('XIA2 used... %s' % \ ', '.join(Citations.get_programs())) Chatter.write( 'Here are the appropriate citations (BIBTeX in xia2-citations.bib.)') for citation in Citations.get_citations_acta(): Chatter.write(citation) # and write the bibtex versions out = open('xia2-citations.bib', 'w') for citation in Citations.get_citations(): out.write('%s\n' % citation) out.close()
def __init__(self): # generic things DriverInstance.__class__.__init__(self) Citations.cite("ccp4") self.set_executable( os.path.join(os.environ.get("CBIN", ""), "ctruncate")) self._anomalous = False self._nres = 0 self._b_factor = 0.0 self._moments = None # numbers of reflections in and out, and number of absences # counted self._nref_in = 0 self._nref_out = 0 self._nabsent = 0 self._xmlout = None
def citations(xproject): lines = [] lines.append('\n') lines.append('References') lines.append('=' * len(lines[-1])) # Sort citations citations = {} for cdict in Citations.get_citations_dicts(): citations[cdict['acta']] = cdict.get('url') for citation in sorted(citations.iterkeys()): if citations[citation]: lines.append("`%s\n<%s>`_\n" % (citation, citations[citation])) else: lines.append("%s\n" % citation) return lines
def _scale_prepare(self): '''Perform all of the preparation required to deliver the scaled data. This should sort together the reflection files, ensure that they are correctly indexed (via pointless) and generally tidy things up.''' # acknowledge all of the programs we are about to use... Citations.cite('pointless') Citations.cite('aimless') Citations.cite('ccp4') # ---------- GATHER ---------- self._sweep_handler = SweepInformationHandler(self._scalr_integraters) Journal.block( 'gathering', self.get_scaler_xcrystal().get_name(), 'CCP4', {'working directory':self.get_working_directory()}) for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) pname, xname, dname = si.get_project_info() sname = si.get_sweep_name() exclude_sweep = False for sweep in PhilIndex.params.xia2.settings.sweep: if sweep.id == sname and sweep.exclude: exclude_sweep = True break if exclude_sweep: self._sweep_handler.remove_epoch(epoch) Debug.write('Excluding sweep %s' % sname) else: Journal.entry({'adding data from':'%s/%s/%s' % \ (xname, dname, sname)}) # gather data for all images which belonged to the parent # crystal - allowing for the fact that things could go wrong # e.g. epoch information not available, exposure times not in # headers etc... for e in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(e) assert is_mtz_file(si.get_reflections()) p, x = self._sweep_handler.get_project_info() self._scalr_pname = p self._scalr_xname = x # verify that the lattices are consistent, calling eliminate if # they are not N.B. there could be corner cases here need_to_return = False multi_sweep_indexing = \ PhilIndex.params.xia2.settings.multi_sweep_indexing == True if len(self._sweep_handler.get_epochs()) > 1: # if we have multi-sweep-indexing going on then logic says all should # share common lattice & UB definition => this is not used here? if multi_sweep_indexing and not self._scalr_input_pointgroup: pointless_hklins = [] max_batches = 0 for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() batches = MtzUtils.batches_from_mtz(hklin) if 1 + max(batches) - min(batches) > max_batches: max_batches = max(batches) - min(batches) + 1 from xia2.lib.bits import nifty_power_of_ten Debug.write('Biggest sweep has %d batches' % max_batches) max_batches = nifty_power_of_ten(max_batches) counter = 0 refiners = [] for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() integrater = si.get_integrater() refiner = integrater.get_integrater_refiner() refiners.append(refiner) hklin = self._prepare_pointless_hklin( hklin, si.get_integrater().get_phi_width()) hklout = os.path.join(self.get_working_directory(), '%s_%s_%s_%s_prepointless.mtz' % \ (pname, xname, dname, si.get_sweep_name())) # we will want to delete this one exit FileHandler.record_temporary_file(hklout) first_batch = min(si.get_batches()) si.set_batch_offset(counter * max_batches - first_batch + 1) from xia2.Modules.Scaler.rebatch import rebatch new_batches = rebatch( hklin, hklout, first_batch=counter * max_batches + 1, pname=pname, xname=xname, dname=dname) pointless_hklins.append(hklout) # update the counter & recycle counter += 1 s = self._factory.Sortmtz() pointless_hklin = os.path.join(self.get_working_directory(), '%s_%s_prepointless_sorted.mtz' % \ (self._scalr_pname, self._scalr_xname)) s.set_hklout(pointless_hklin) for hklin in pointless_hklins: s.add_hklin(hklin) s.sort() # FIXME xia2-51 in here look at running constant scaling on the # pointless hklin to put the runs on the same scale. Ref=[A] pointless_const = os.path.join(self.get_working_directory(), '%s_%s_prepointless_const.mtz' % \ (self._scalr_pname, self._scalr_xname)) FileHandler.record_temporary_file(pointless_const) aimless_const = self._factory.Aimless() aimless_const.set_hklin(pointless_hklin) aimless_const.set_hklout(pointless_const) aimless_const.const() pointless_const = os.path.join(self.get_working_directory(), '%s_%s_prepointless_const_unmerged.mtz' % \ (self._scalr_pname, self._scalr_xname)) FileHandler.record_temporary_file(pointless_const) pointless_hklin = pointless_const # FIXME xia2-51 in here need to pass all refiners to ensure that the # information is passed back to all of them not just the last one... Debug.write('Running multisweep pointless for %d sweeps' % len(refiners)) pointgroup, reindex_op, ntr, pt = \ self._pointless_indexer_multisweep(pointless_hklin, refiners) Debug.write('X1698: %s: %s' % (pointgroup, reindex_op)) lattices = [Syminfo.get_lattice(pointgroup)] for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) intgr = si.get_integrater() hklin = si.get_reflections() refiner = intgr.get_integrater_refiner() if ntr: intgr.integrater_reset_reindex_operator() need_to_return = True else: lattices = [] for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) intgr = si.get_integrater() hklin = si.get_reflections() refiner = intgr.get_integrater_refiner() if self._scalr_input_pointgroup: pointgroup = self._scalr_input_pointgroup reindex_op = 'h,k,l' ntr = False else: pointless_hklin = self._prepare_pointless_hklin( hklin, si.get_integrater().get_phi_width()) pointgroup, reindex_op, ntr, pt = \ self._pointless_indexer_jiffy( pointless_hklin, refiner) Debug.write('X1698: %s: %s' % (pointgroup, reindex_op)) lattice = Syminfo.get_lattice(pointgroup) if not lattice in lattices: lattices.append(lattice) if ntr: intgr.integrater_reset_reindex_operator() need_to_return = True if len(lattices) > 1: # why not using pointless indexer jiffy??! correct_lattice = sort_lattices(lattices)[0] Chatter.write('Correct lattice asserted to be %s' % \ correct_lattice) # transfer this information back to the indexers for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) refiner = si.get_integrater().get_integrater_refiner() sname = si.get_sweep_name() state = refiner.set_refiner_asserted_lattice( correct_lattice) if state == refiner.LATTICE_CORRECT: Chatter.write('Lattice %s ok for sweep %s' % \ (correct_lattice, sname)) elif state == refiner.LATTICE_IMPOSSIBLE: raise RuntimeError('Lattice %s impossible for %s' \ % (correct_lattice, sname)) elif state == refiner.LATTICE_POSSIBLE: Chatter.write('Lattice %s assigned for sweep %s' % \ (correct_lattice, sname)) need_to_return = True # if one or more of them was not in the lowest lattice, # need to return here to allow reprocessing if need_to_return: self.set_scaler_done(False) self.set_scaler_prepare_done(False) return # ---------- REINDEX ALL DATA TO CORRECT POINTGROUP ---------- # all should share the same pointgroup, unless twinned... in which # case force them to be... pointgroups = {} reindex_ops = {} probably_twinned = False need_to_return = False multi_sweep_indexing = \ PhilIndex.params.xia2.settings.multi_sweep_indexing == True if multi_sweep_indexing and not self._scalr_input_pointgroup: pointless_hklins = [] max_batches = 0 for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() batches = MtzUtils.batches_from_mtz(hklin) if 1 + max(batches) - min(batches) > max_batches: max_batches = max(batches) - min(batches) + 1 from xia2.lib.bits import nifty_power_of_ten Debug.write('Biggest sweep has %d batches' % max_batches) max_batches = nifty_power_of_ten(max_batches) counter = 0 refiners = [] for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() integrater = si.get_integrater() refiner = integrater.get_integrater_refiner() refiners.append(refiner) hklin = self._prepare_pointless_hklin( hklin, si.get_integrater().get_phi_width()) hklout = os.path.join(self.get_working_directory(), '%s_%s_%s_%s_prepointless.mtz' % \ (pname, xname, dname, si.get_sweep_name())) # we will want to delete this one exit FileHandler.record_temporary_file(hklout) first_batch = min(si.get_batches()) si.set_batch_offset(counter * max_batches - first_batch + 1) from xia2.Modules.Scaler.rebatch import rebatch new_batches = rebatch( hklin, hklout, first_batch=counter * max_batches + 1, pname=pname, xname=xname, dname=dname) pointless_hklins.append(hklout) # update the counter & recycle counter += 1 # FIXME related to xia2-51 - this looks very very similar to the logic # in [A] above - is this duplicated logic? s = self._factory.Sortmtz() pointless_hklin = os.path.join(self.get_working_directory(), '%s_%s_prepointless_sorted.mtz' % \ (self._scalr_pname, self._scalr_xname)) s.set_hklout(pointless_hklin) for hklin in pointless_hklins: s.add_hklin(hklin) s.sort() pointless_const = os.path.join(self.get_working_directory(), '%s_%s_prepointless_const.mtz' % \ (self._scalr_pname, self._scalr_xname)) FileHandler.record_temporary_file(pointless_const) aimless_const = self._factory.Aimless() aimless_const.set_hklin(pointless_hklin) aimless_const.set_hklout(pointless_const) aimless_const.const() pointless_const = os.path.join(self.get_working_directory(), '%s_%s_prepointless_const_unmerged.mtz' % \ (self._scalr_pname, self._scalr_xname)) FileHandler.record_temporary_file(pointless_const) pointless_hklin = pointless_const pointgroup, reindex_op, ntr, pt = \ self._pointless_indexer_multisweep( pointless_hklin, refiners) for epoch in self._sweep_handler.get_epochs(): pointgroups[epoch] = pointgroup reindex_ops[epoch] = reindex_op else: for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() integrater = si.get_integrater() refiner = integrater.get_integrater_refiner() if self._scalr_input_pointgroup: Debug.write('Using input pointgroup: %s' % \ self._scalr_input_pointgroup) pointgroup = self._scalr_input_pointgroup reindex_op = 'h,k,l' pt = False else: pointless_hklin = self._prepare_pointless_hklin( hklin, si.get_integrater().get_phi_width()) pointgroup, reindex_op, ntr, pt = \ self._pointless_indexer_jiffy( pointless_hklin, refiner) Debug.write('X1698: %s: %s' % (pointgroup, reindex_op)) if ntr: integrater.integrater_reset_reindex_operator() need_to_return = True if pt and not probably_twinned: probably_twinned = True Debug.write('Pointgroup: %s (%s)' % (pointgroup, reindex_op)) pointgroups[epoch] = pointgroup reindex_ops[epoch] = reindex_op overall_pointgroup = None pointgroup_set = {pointgroups[e] for e in pointgroups} if len(pointgroup_set) > 1 and \ not probably_twinned: raise RuntimeError('non uniform pointgroups') if len(pointgroup_set) > 1: Debug.write('Probably twinned, pointgroups: %s' % \ ' '.join([p.replace(' ', '') for p in \ list(pointgroup_set)])) numbers = [Syminfo.spacegroup_name_to_number(s) for s in \ pointgroup_set] overall_pointgroup = Syminfo.spacegroup_number_to_name(min(numbers)) self._scalr_input_pointgroup = overall_pointgroup Chatter.write('Twinning detected, assume pointgroup %s' % \ overall_pointgroup) need_to_return = True else: overall_pointgroup = pointgroup_set.pop() for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) integrater = si.get_integrater() integrater.set_integrater_spacegroup_number( Syminfo.spacegroup_name_to_number(overall_pointgroup)) integrater.set_integrater_reindex_operator( reindex_ops[epoch], reason='setting point group') # This will give us the reflections in the correct point group si.set_reflections(integrater.get_integrater_intensities()) if need_to_return: self.set_scaler_done(False) self.set_scaler_prepare_done(False) return # in here now optionally work through the data files which should be # indexed with a consistent point group, and transform the orientation # matrices by the lattice symmetry operations (if possible) to get a # consistent definition of U matrix modulo fixed rotations if PhilIndex.params.xia2.settings.unify_setting: from scitbx.matrix import sqr reference_U = None i3 = sqr((1, 0, 0, 0, 1, 0, 0, 0, 1)) for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) intgr = si.get_integrater() fixed = sqr(intgr.get_goniometer().get_fixed_rotation()) u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections()) U = fixed.inverse() * sqr(u).transpose() B = sqr(b) if reference_U is None: reference_U = U continue results = [] for op in s.all_ops(): R = B * sqr(op.r().as_double()).transpose() * B.inverse() nearly_i3 = (U * R).inverse() * reference_U score = sum([abs(_n - _i) for (_n, _i) in zip(nearly_i3, i3)]) results.append((score, op.r().as_hkl(), op)) results.sort() best = results[0] Debug.write('Best reindex: %s %.3f' % (best[1], best[0])) intgr.set_integrater_reindex_operator(best[2].r().inverse().as_hkl(), reason='unifying [U] setting') si.set_reflections(intgr.get_integrater_intensities()) # recalculate to verify u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections()) U = fixed.inverse() * sqr(u).transpose() Debug.write('New reindex: %s' % (U.inverse() * reference_U)) # FIXME I should probably raise an exception at this stage if this # is not about I3... if self.get_scaler_reference_reflection_file(): self._reference = self.get_scaler_reference_reflection_file() Debug.write('Using HKLREF %s' % self._reference) elif PhilIndex.params.xia2.settings.scale.reference_reflection_file: self._reference = PhilIndex.params.xia2.settings.scale.reference_reflection_file Debug.write('Using HKLREF %s' % self._reference) params = PhilIndex.params use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs if len(self._sweep_handler.get_epochs()) > 1 and use_brehm_diederichs: brehm_diederichs_files_in = [] for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() brehm_diederichs_files_in.append(hklin) # now run cctbx.brehm_diederichs to figure out the indexing hand for # each sweep from xia2.Wrappers.Cctbx.BrehmDiederichs import BrehmDiederichs from xia2.lib.bits import auto_logfiler brehm_diederichs = BrehmDiederichs() brehm_diederichs.set_working_directory(self.get_working_directory()) auto_logfiler(brehm_diederichs) brehm_diederichs.set_input_filenames(brehm_diederichs_files_in) # 1 or 3? 1 seems to work better? brehm_diederichs.set_asymmetric(1) brehm_diederichs.run() reindexing_dict = brehm_diederichs.get_reindexing_dict() for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) intgr = si.get_integrater() hklin = si.get_reflections() reindex_op = reindexing_dict.get(os.path.abspath(hklin)) assert reindex_op is not None if 1 or reindex_op != 'h,k,l': # apply the reindexing operator intgr.set_integrater_reindex_operator( reindex_op, reason='match reference') si.set_reflections(intgr.get_integrater_intensities()) elif len(self._sweep_handler.get_epochs()) > 1 and \ not self._reference: first = self._sweep_handler.get_epochs()[0] si = self._sweep_handler.get_sweep_information(first) self._reference = si.get_reflections() if self._reference: md = self._factory.Mtzdump() md.set_hklin(self._reference) md.dump() if md.get_batches() and False: raise RuntimeError('reference reflection file %s unmerged' % \ self._reference) datasets = md.get_datasets() if len(datasets) > 1 and False: raise RuntimeError('more than one dataset in %s' % \ self._reference) # then get the unit cell, lattice etc. reference_lattice = Syminfo.get_lattice(md.get_spacegroup()) reference_cell = md.get_dataset_info(datasets[0])['cell'] # then compute the pointgroup from this... # ---------- REINDEX TO CORRECT (REFERENCE) SETTING ---------- for epoch in self._sweep_handler.get_epochs(): # if we are working with unified UB matrix then this should not # be a problem here (note, *if*; *should*) # what about e.g. alternative P1 settings? # see JIRA MXSW-904 if PhilIndex.params.xia2.settings.unify_setting: continue pl = self._factory.Pointless() si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() pl.set_hklin(self._prepare_pointless_hklin( hklin, si.get_integrater().get_phi_width())) hklout = os.path.join( self.get_working_directory(), '%s_rdx2.mtz' % os.path.split(hklin)[-1][:-4]) # we will want to delete this one exit FileHandler.record_temporary_file(hklout) # now set the initial reflection set as a reference... pl.set_hklref(self._reference) # https://github.com/xia2/xia2/issues/115 - should ideally iteratively # construct a reference or a tree of correlations to ensure correct # reference setting - however if small molecule assume has been # multi-sweep-indexed so can ignore "fatal errors" - temporary hack pl.decide_pointgroup( ignore_errors=PhilIndex.params.xia2.settings.small_molecule) Debug.write('Reindexing analysis of %s' % pl.get_hklin()) pointgroup = pl.get_pointgroup() reindex_op = pl.get_reindex_operator() Debug.write('Operator: %s' % reindex_op) # apply this... integrater = si.get_integrater() integrater.set_integrater_reindex_operator(reindex_op, reason='match reference') integrater.set_integrater_spacegroup_number( Syminfo.spacegroup_name_to_number(pointgroup)) si.set_reflections(integrater.get_integrater_intensities()) md = self._factory.Mtzdump() md.set_hklin(si.get_reflections()) md.dump() datasets = md.get_datasets() if len(datasets) > 1: raise RuntimeError('more than one dataset in %s' % \ si.get_reflections()) # then get the unit cell, lattice etc. lattice = Syminfo.get_lattice(md.get_spacegroup()) cell = md.get_dataset_info(datasets[0])['cell'] if lattice != reference_lattice: raise RuntimeError('lattices differ in %s and %s' % \ (self._reference, si.get_reflections())) Debug.write('Cell: %.2f %.2f %.2f %.2f %.2f %.2f' % cell) Debug.write('Ref: %.2f %.2f %.2f %.2f %.2f %.2f' % reference_cell) for j in range(6): if math.fabs((cell[j] - reference_cell[j]) / reference_cell[j]) > 0.1: raise RuntimeError( \ 'unit cell parameters differ in %s and %s' % \ (self._reference, si.get_reflections())) # ---------- SORT TOGETHER DATA ---------- self._sort_together_data_ccp4() self._scalr_resolution_limits = {} # store central resolution limit estimates batch_ranges = [ self._sweep_handler.get_sweep_information(epoch).get_batch_range() for epoch in self._sweep_handler.get_epochs() ] self._resolution_limit_estimates = ersatz_resolution( self._prepared_reflections, batch_ranges)
def _scale_prepare(self): """Perform all of the preparation required to deliver the scaled data. This should sort together the reflection files, ensure that they are correctly indexed (via pointless) and generally tidy things up.""" # acknowledge all of the programs we are about to use... Citations.cite("pointless") Citations.cite("aimless") Citations.cite("ccp4") # ---------- GATHER ---------- self._sweep_handler = SweepInformationHandler(self._scalr_integraters) for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) pname, xname, dname = si.get_project_info() sname = si.get_sweep_name() exclude_sweep = False for sweep in PhilIndex.params.xia2.settings.sweep: if sweep.id == sname and sweep.exclude: exclude_sweep = True break if exclude_sweep: self._sweep_handler.remove_epoch(epoch) logger.debug("Excluding sweep %s", sname) else: logger.debug("%-30s %s/%s/%s", "adding data from:", xname, dname, sname) # gather data for all images which belonged to the parent # crystal - allowing for the fact that things could go wrong # e.g. epoch information not available, exposure times not in # headers etc... for e in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(e) assert is_mtz_file(si.get_reflections()), repr(si.get_reflections()) p, x = self._sweep_handler.get_project_info() self._scalr_pname = p self._scalr_xname = x # verify that the lattices are consistent, calling eliminate if # they are not N.B. there could be corner cases here need_to_return = False multi_sweep_indexing = PhilIndex.params.xia2.settings.multi_sweep_indexing # START OF if more than one epoch if len(self._sweep_handler.get_epochs()) > 1: # if we have multi-sweep-indexing going on then logic says all should # share common lattice & UB definition => this is not used here? # START OF if multi_sweep indexing and not input pg if multi_sweep_indexing and not self._scalr_input_pointgroup: pointless_hklins = [] max_batches = 0 for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() batches = MtzUtils.batches_from_mtz(hklin) if 1 + max(batches) - min(batches) > max_batches: max_batches = max(batches) - min(batches) + 1 logger.debug("Biggest sweep has %d batches", max_batches) max_batches = nifty_power_of_ten(max_batches) counter = 0 refiners = [] for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() integrater = si.get_integrater() refiner = integrater.get_integrater_refiner() refiners.append(refiner) hklin = self._prepare_pointless_hklin( hklin, si.get_integrater().get_phi_width() ) hklout = os.path.join( self.get_working_directory(), "%s_%s_%s_%s_prepointless.mtz" % (pname, xname, dname, si.get_sweep_name()), ) # we will want to delete this one exit FileHandler.record_temporary_file(hklout) first_batch = min(si.get_batches()) si.set_batch_offset(counter * max_batches - first_batch + 1) rebatch( hklin, hklout, first_batch=counter * max_batches + 1, pname=pname, xname=xname, dname=dname, ) pointless_hklins.append(hklout) # update the counter & recycle counter += 1 # SUMMARY - have added all sweeps to pointless_hklins s = self._factory.Sortmtz() pointless_hklin = os.path.join( self.get_working_directory(), "%s_%s_prepointless_sorted.mtz" % (self._scalr_pname, self._scalr_xname), ) s.set_hklout(pointless_hklin) for hklin in pointless_hklins: s.add_hklin(hklin) s.sort() # FIXME xia2-51 in here look at running constant scaling on the # pointless hklin to put the runs on the same scale. Ref=[A] pointless_const = os.path.join( self.get_working_directory(), "%s_%s_prepointless_const.mtz" % (self._scalr_pname, self._scalr_xname), ) FileHandler.record_temporary_file(pointless_const) aimless_const = self._factory.Aimless() aimless_const.set_hklin(pointless_hklin) aimless_const.set_hklout(pointless_const) aimless_const.const() pointless_const = os.path.join( self.get_working_directory(), "%s_%s_prepointless_const_unmerged.mtz" % (self._scalr_pname, self._scalr_xname), ) FileHandler.record_temporary_file(pointless_const) pointless_hklin = pointless_const # FIXME xia2-51 in here need to pass all refiners to ensure that the # information is passed back to all of them not just the last one... logger.debug( "Running multisweep pointless for %d sweeps", len(refiners) ) pointgroup, reindex_op, ntr, pt = self._pointless_indexer_multisweep( pointless_hklin, refiners ) logger.debug("X1698: %s: %s", pointgroup, reindex_op) lattices = [Syminfo.get_lattice(pointgroup)] for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) intgr = si.get_integrater() hklin = si.get_reflections() refiner = intgr.get_integrater_refiner() if ntr: intgr.integrater_reset_reindex_operator() need_to_return = True # SUMMARY - added all sweeps together into an mtz, ran # _pointless_indexer_multisweep on this, made a list of one lattice # and potentially reset reindex op? # END OF if multi_sweep indexing and not input pg # START OF if not multi_sweep, or input pg given else: lattices = [] for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) intgr = si.get_integrater() hklin = si.get_reflections() refiner = intgr.get_integrater_refiner() if self._scalr_input_pointgroup: pointgroup = self._scalr_input_pointgroup reindex_op = "h,k,l" ntr = False else: pointless_hklin = self._prepare_pointless_hklin( hklin, si.get_integrater().get_phi_width() ) pointgroup, reindex_op, ntr, pt = self._pointless_indexer_jiffy( pointless_hklin, refiner ) logger.debug("X1698: %s: %s", pointgroup, reindex_op) lattice = Syminfo.get_lattice(pointgroup) if lattice not in lattices: lattices.append(lattice) if ntr: intgr.integrater_reset_reindex_operator() need_to_return = True # SUMMARY do pointless_indexer on each sweep, get lattices and make a list # of unique lattices, potentially reset reindex op. # END OF if not multi_sweep, or input pg given # SUMMARY - still within if more than one epoch, now have a list of number # of lattices # START OF if multiple-lattices if len(lattices) > 1: # why not using pointless indexer jiffy??! correct_lattice = sort_lattices(lattices)[0] logger.info("Correct lattice asserted to be %s", correct_lattice) # transfer this information back to the indexers for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) refiner = si.get_integrater().get_integrater_refiner() sname = si.get_sweep_name() state = refiner.set_refiner_asserted_lattice(correct_lattice) if state == refiner.LATTICE_CORRECT: logger.info( "Lattice %s ok for sweep %s", correct_lattice, sname ) elif state == refiner.LATTICE_IMPOSSIBLE: raise RuntimeError( f"Lattice {correct_lattice} impossible for {sname}" ) elif state == refiner.LATTICE_POSSIBLE: logger.info( "Lattice %s assigned for sweep %s", correct_lattice, sname ) need_to_return = True # END OF if multiple-lattices # SUMMARY - forced all lattices to be same and hope its okay. # END OF if more than one epoch # if one or more of them was not in the lowest lattice, # need to return here to allow reprocessing if need_to_return: self.set_scaler_done(False) self.set_scaler_prepare_done(False) return # ---------- REINDEX ALL DATA TO CORRECT POINTGROUP ---------- # all should share the same pointgroup, unless twinned... in which # case force them to be... pointgroups = {} reindex_ops = {} probably_twinned = False need_to_return = False multi_sweep_indexing = PhilIndex.params.xia2.settings.multi_sweep_indexing # START OF if multi-sweep and not input pg if multi_sweep_indexing and not self._scalr_input_pointgroup: pointless_hklins = [] max_batches = 0 for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() batches = MtzUtils.batches_from_mtz(hklin) if 1 + max(batches) - min(batches) > max_batches: max_batches = max(batches) - min(batches) + 1 logger.debug("Biggest sweep has %d batches", max_batches) max_batches = nifty_power_of_ten(max_batches) counter = 0 refiners = [] for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() integrater = si.get_integrater() refiner = integrater.get_integrater_refiner() refiners.append(refiner) hklin = self._prepare_pointless_hklin( hklin, si.get_integrater().get_phi_width() ) hklout = os.path.join( self.get_working_directory(), "%s_%s_%s_%s_prepointless.mtz" % (pname, xname, dname, si.get_sweep_name()), ) # we will want to delete this one exit FileHandler.record_temporary_file(hklout) first_batch = min(si.get_batches()) si.set_batch_offset(counter * max_batches - first_batch + 1) rebatch( hklin, hklout, first_batch=counter * max_batches + 1, pname=pname, xname=xname, dname=dname, ) pointless_hklins.append(hklout) # update the counter & recycle counter += 1 # FIXME related to xia2-51 - this looks very very similar to the logic # in [A] above - is this duplicated logic? s = self._factory.Sortmtz() pointless_hklin = os.path.join( self.get_working_directory(), "%s_%s_prepointless_sorted.mtz" % (self._scalr_pname, self._scalr_xname), ) s.set_hklout(pointless_hklin) for hklin in pointless_hklins: s.add_hklin(hklin) s.sort() pointless_const = os.path.join( self.get_working_directory(), f"{self._scalr_pname}_{self._scalr_xname}_prepointless_const.mtz", ) FileHandler.record_temporary_file(pointless_const) aimless_const = self._factory.Aimless() aimless_const.set_hklin(pointless_hklin) aimless_const.set_hklout(pointless_const) aimless_const.const() pointless_const = os.path.join( self.get_working_directory(), "%s_%s_prepointless_const_unmerged.mtz" % (self._scalr_pname, self._scalr_xname), ) FileHandler.record_temporary_file(pointless_const) pointless_hklin = pointless_const pointgroup, reindex_op, ntr, pt = self._pointless_indexer_multisweep( pointless_hklin, refiners ) for epoch in self._sweep_handler.get_epochs(): pointgroups[epoch] = pointgroup reindex_ops[epoch] = reindex_op # SUMMARY ran pointless multisweep on combined mtz and made a dict # of pointgroups and reindex_ops (all same) # END OF if multi-sweep and not input pg # START OF if not mulit-sweep or pg given else: for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() integrater = si.get_integrater() refiner = integrater.get_integrater_refiner() if self._scalr_input_pointgroup: logger.debug( "Using input pointgroup: %s", self._scalr_input_pointgroup ) pointgroup = self._scalr_input_pointgroup reindex_op = "h,k,l" pt = False else: pointless_hklin = self._prepare_pointless_hklin( hklin, si.get_integrater().get_phi_width() ) pointgroup, reindex_op, ntr, pt = self._pointless_indexer_jiffy( pointless_hklin, refiner ) logger.debug("X1698: %s: %s", pointgroup, reindex_op) if ntr: integrater.integrater_reset_reindex_operator() need_to_return = True if pt and not probably_twinned: probably_twinned = True logger.debug("Pointgroup: %s (%s)", pointgroup, reindex_op) pointgroups[epoch] = pointgroup reindex_ops[epoch] = reindex_op # SUMMARY - for each sweep, run indexer jiffy and get reindex operators # and pointgroups dictionaries (could be different between sweeps) # END OF if not mulit-sweep or pg given overall_pointgroup = None pointgroup_set = {pointgroups[e] for e in pointgroups} if len(pointgroup_set) > 1 and not probably_twinned: raise RuntimeError( "non uniform pointgroups: %s" % str(list(pointgroup_set)) ) if len(pointgroup_set) > 1: logger.debug( "Probably twinned, pointgroups: %s", " ".join(p.replace(" ", "") for p in pointgroup_set), ) numbers = (Syminfo.spacegroup_name_to_number(ps) for ps in pointgroup_set) overall_pointgroup = Syminfo.spacegroup_number_to_name(min(numbers)) self._scalr_input_pointgroup = overall_pointgroup logger.info("Twinning detected, assume pointgroup %s", overall_pointgroup) need_to_return = True else: overall_pointgroup = pointgroup_set.pop() # SUMMARY - Have handled if different pointgroups & chosen an overall_pointgroup # which is the lowest symmetry # Now go through sweeps and do reindexing for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) integrater = si.get_integrater() integrater.set_integrater_spacegroup_number( Syminfo.spacegroup_name_to_number(overall_pointgroup) ) integrater.set_integrater_reindex_operator( reindex_ops[epoch], reason="setting point group" ) # This will give us the reflections in the correct point group si.set_reflections(integrater.get_integrater_intensities()) if need_to_return: self.set_scaler_done(False) self.set_scaler_prepare_done(False) return # in here now optionally work through the data files which should be # indexed with a consistent point group, and transform the orientation # matrices by the lattice symmetry operations (if possible) to get a # consistent definition of U matrix modulo fixed rotations if PhilIndex.params.xia2.settings.unify_setting: self.unify_setting() if self.get_scaler_reference_reflection_file(): self._reference = self.get_scaler_reference_reflection_file() logger.debug("Using HKLREF %s", self._reference) elif PhilIndex.params.xia2.settings.scale.reference_reflection_file: self._reference = ( PhilIndex.params.xia2.settings.scale.reference_reflection_file ) logger.debug("Using HKLREF %s", self._reference) params = PhilIndex.params use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs if len(self._sweep_handler.get_epochs()) > 1 and use_brehm_diederichs: self.brehm_diederichs_reindexing() # If not Brehm-deidrichs, set reference as first sweep elif len(self._sweep_handler.get_epochs()) > 1 and not self._reference: first = self._sweep_handler.get_epochs()[0] si = self._sweep_handler.get_sweep_information(first) self._reference = si.get_reflections() # Now reindex to be consistent with first dataset - run pointless on each # dataset with reference if self._reference: md = self._factory.Mtzdump() md.set_hklin(self._reference) md.dump() datasets = md.get_datasets() # then get the unit cell, lattice etc. reference_lattice = Syminfo.get_lattice(md.get_spacegroup()) reference_cell = md.get_dataset_info(datasets[0])["cell"] # then compute the pointgroup from this... # ---------- REINDEX TO CORRECT (REFERENCE) SETTING ---------- for epoch in self._sweep_handler.get_epochs(): # if we are working with unified UB matrix then this should not # be a problem here (note, *if*; *should*) # what about e.g. alternative P1 settings? # see JIRA MXSW-904 if PhilIndex.params.xia2.settings.unify_setting: continue pl = self._factory.Pointless() si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() pl.set_hklin( self._prepare_pointless_hklin( hklin, si.get_integrater().get_phi_width() ) ) hklout = os.path.join( self.get_working_directory(), "%s_rdx2.mtz" % os.path.split(hklin)[-1][:-4], ) # we will want to delete this one exit FileHandler.record_temporary_file(hklout) # now set the initial reflection set as a reference... pl.set_hklref(self._reference) # https://github.com/xia2/xia2/issues/115 - should ideally iteratively # construct a reference or a tree of correlations to ensure correct # reference setting - however if small molecule assume has been # multi-sweep-indexed so can ignore "fatal errors" - temporary hack pl.decide_pointgroup( ignore_errors=PhilIndex.params.xia2.settings.small_molecule ) logger.debug("Reindexing analysis of %s", pl.get_hklin()) pointgroup = pl.get_pointgroup() reindex_op = pl.get_reindex_operator() logger.debug("Operator: %s", reindex_op) # apply this... integrater = si.get_integrater() integrater.set_integrater_reindex_operator( reindex_op, reason="match reference" ) integrater.set_integrater_spacegroup_number( Syminfo.spacegroup_name_to_number(pointgroup) ) si.set_reflections(integrater.get_integrater_intensities()) md = self._factory.Mtzdump() md.set_hklin(si.get_reflections()) md.dump() datasets = md.get_datasets() if len(datasets) > 1: raise RuntimeError( "more than one dataset in %s" % si.get_reflections() ) # then get the unit cell, lattice etc. lattice = Syminfo.get_lattice(md.get_spacegroup()) cell = md.get_dataset_info(datasets[0])["cell"] if lattice != reference_lattice: raise RuntimeError( "lattices differ in %s and %s" % (self._reference, si.get_reflections()) ) logger.debug("Cell: %.2f %.2f %.2f %.2f %.2f %.2f" % cell) logger.debug("Ref: %.2f %.2f %.2f %.2f %.2f %.2f" % reference_cell) for j in range(6): if ( math.fabs((cell[j] - reference_cell[j]) / reference_cell[j]) > 0.1 ): raise RuntimeError( "unit cell parameters differ in %s and %s" % (self._reference, si.get_reflections()) ) # ---------- SORT TOGETHER DATA ---------- self._sort_together_data_ccp4() self._scalr_resolution_limits = {} # store central resolution limit estimates batch_ranges = [ self._sweep_handler.get_sweep_information(epoch).get_batch_range() for epoch in self._sweep_handler.get_epochs() ] self._resolution_limit_estimates = ersatz_resolution( self._prepared_reflections, batch_ranges )
def _index(self): '''Actually index the diffraction pattern. Note well that this is not going to compute the matrix...''' # acknowledge this program Citations.cite('labelit') Citations.cite('distl') #self.reset() _images = [] for i in self._indxr_images: for j in i: if not j in _images: _images.append(j) _images.sort() images_str = '%d' % _images[0] for i in _images[1:]: images_str += ', %d' % i cell_str = None if self._indxr_input_cell: cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \ self._indxr_input_cell if self._indxr_sweep_name: # then this is a proper autoindexing run - describe this # to the journal entry #if len(self._fp_directory) <= 50: #dirname = self._fp_directory #else: #dirname = '...%s' % self._fp_directory[-46:] dirname = os.path.dirname(self.get_imageset().get_template()) Journal.block( 'autoindexing', self._indxr_sweep_name, 'labelit', {'images':images_str, 'target cell':cell_str, 'target lattice':self._indxr_input_lattice, 'template':self.get_imageset().get_template(), 'directory':dirname}) if len(_images) > 4: raise RuntimeError, 'cannot use more than 4 images' from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex index = LabelitIndex() index.set_working_directory(self.get_working_directory()) auto_logfiler(index) #task = 'Autoindex from images:' #for i in _images: #task += ' %s' % self.get_image_name(i) #self.set_task(task) Debug.write('Indexing from images:') for i in _images: index.add_image(self.get_image_name(i)) Debug.write('%s' % self.get_image_name(i)) xsweep = self.get_indexer_sweep() if xsweep is not None: if xsweep.get_distance() is not None: index.set_distance(xsweep.get_distance()) #if self.get_wavelength_prov() == 'user': #index.set_wavelength(self.get_wavelength()) if xsweep.get_beam_centre() is not None: index.set_beam_centre(xsweep.get_beam_centre()) if self._refine_beam is False: index.set_refine_beam(False) else: index.set_refine_beam(True) index.set_beam_search_scope(self._beam_search_scope) if ((math.fabs(self.get_wavelength() - 1.54) < 0.01) or (math.fabs(self.get_wavelength() - 2.29) < 0.01)): index.set_Cu_KA_or_Cr_KA(True) #sweep = self.get_indexer_sweep_name() #FileHandler.record_log_file( #'%s INDEX' % (sweep), self.get_log_file()) try: index.run() except RuntimeError, e: if self._refine_beam is False: raise e # can we improve the situation? if self._beam_search_scope < 4.0: self._beam_search_scope += 4.0 # try repeating the indexing! self.set_indexer_done(False) return 'failed' # otherwise this is beyond redemption raise e
def _index(self): '''Actually index the diffraction pattern. Note well that this is not going to compute the matrix...''' # acknowledge this program Citations.cite('labelit') Citations.cite('distl') #self.reset() _images = [] for i in self._indxr_images: for j in i: if not j in _images: _images.append(j) _images.sort() images_str = '%d' % _images[0] for i in _images[1:]: images_str += ', %d' % i cell_str = None if self._indxr_input_cell: cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \ self._indxr_input_cell if self._indxr_sweep_name: # then this is a proper autoindexing run - describe this # to the journal entry #if len(self._fp_directory) <= 50: #dirname = self._fp_directory #else: #dirname = '...%s' % self._fp_directory[-46:] dirname = os.path.dirname(self.get_imageset().get_template()) Journal.block( 'autoindexing', self._indxr_sweep_name, 'labelit', { 'images': images_str, 'target cell': cell_str, 'target lattice': self._indxr_input_lattice, 'template': self.get_imageset().get_template(), 'directory': dirname }) if len(_images) > 4: raise RuntimeError('cannot use more than 4 images') from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex index = LabelitIndex() index.set_working_directory(self.get_working_directory()) auto_logfiler(index) #task = 'Autoindex from images:' #for i in _images: #task += ' %s' % self.get_image_name(i) #self.set_task(task) Debug.write('Indexing from images:') for i in _images: index.add_image(self.get_image_name(i)) Debug.write('%s' % self.get_image_name(i)) xsweep = self.get_indexer_sweep() if xsweep is not None: if xsweep.get_distance() is not None: index.set_distance(xsweep.get_distance()) #if self.get_wavelength_prov() == 'user': #index.set_wavelength(self.get_wavelength()) if xsweep.get_beam_centre() is not None: index.set_beam_centre(xsweep.get_beam_centre()) if self._refine_beam is False: index.set_refine_beam(False) else: index.set_refine_beam(True) index.set_beam_search_scope(self._beam_search_scope) if ((math.fabs(self.get_wavelength() - 1.54) < 0.01) or (math.fabs(self.get_wavelength() - 2.29) < 0.01)): index.set_Cu_KA_or_Cr_KA(True) #sweep = self.get_indexer_sweep_name() #FileHandler.record_log_file( #'%s INDEX' % (sweep), self.get_log_file()) try: index.run() except RuntimeError as e: if self._refine_beam is False: raise e # can we improve the situation? if self._beam_search_scope < 4.0: self._beam_search_scope += 4.0 # try repeating the indexing! self.set_indexer_done(False) return 'failed' # otherwise this is beyond redemption raise e self._solutions = index.get_solutions() # FIXME this needs to check the smilie status e.g. # ":)" or ";(" or " ". # FIXME need to check the value of the RMSD and raise an # exception if the P1 solution has an RMSD > 1.0... # Change 27/FEB/08 to support user assigned spacegroups # (euugh!) have to "ignore" solutions with higher symmetry # otherwise the rest of xia will override us. Bummer. for i, solution in self._solutions.iteritems(): if self._indxr_user_input_lattice: if (lattice_to_spacegroup(solution['lattice']) > lattice_to_spacegroup(self._indxr_input_lattice)): Debug.write('Ignoring solution: %s' % solution['lattice']) del self._solutions[i] # check the RMSD from the triclinic unit cell if self._solutions[1]['rmsd'] > 1.0 and False: # don't know when this is useful - but I know when it is not! raise RuntimeError('high RMSD for triclinic solution') # configure the "right" solution self._solution = self.get_solution() # now store also all of the other solutions... keyed by the # lattice - however these should only be added if they # have a smiley in the appropriate record, perhaps? for solution in self._solutions.keys(): lattice = self._solutions[solution]['lattice'] if lattice in self._indxr_other_lattice_cell: if self._indxr_other_lattice_cell[lattice]['goodness'] < \ self._solutions[solution]['metric']: continue self._indxr_other_lattice_cell[lattice] = { 'goodness': self._solutions[solution]['metric'], 'cell': self._solutions[solution]['cell'] } self._indxr_lattice = self._solution['lattice'] self._indxr_cell = tuple(self._solution['cell']) self._indxr_mosaic = self._solution['mosaic'] lms = LabelitMosflmScript() lms.set_working_directory(self.get_working_directory()) lms.set_solution(self._solution['number']) self._indxr_payload['mosflm_orientation_matrix'] = lms.calculate() # get the beam centre from the mosflm script - mosflm # may have inverted the beam centre and labelit will know # this! mosflm_beam_centre = lms.get_mosflm_beam() if mosflm_beam_centre: self._indxr_payload['mosflm_beam_centre'] = tuple( mosflm_beam_centre) import copy detector = copy.deepcopy(self.get_detector()) beam = copy.deepcopy(self.get_beam()) from dxtbx.model.detector_helpers import set_mosflm_beam_centre set_mosflm_beam_centre(detector, beam, mosflm_beam_centre) from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number from scitbx import matrix from cctbx import sgtbx, uctbx from dxtbx.model import CrystalFactory mosflm_matrix = matrix.sqr([ float(i) for line in lms.calculate() for i in line.replace("-", " -").split() ][:9]) space_group = sgtbx.space_group_info( lattice_to_spacegroup_number(self._solution['lattice'])).group() crystal_model = CrystalFactory.from_mosflm_matrix( mosflm_matrix, unit_cell=uctbx.unit_cell(tuple(self._solution['cell'])), space_group=space_group) from dxtbx.model import Experiment, ExperimentList experiment = Experiment( beam=beam, detector=detector, goniometer=self.get_goniometer(), scan=self.get_scan(), crystal=crystal_model, ) experiment_list = ExperimentList([experiment]) self.set_indexer_experiment_list(experiment_list) # also get an estimate of the resolution limit from the # labelit.stats_distl output... FIXME the name is wrong! lsd = LabelitStats_distl() lsd.set_working_directory(self.get_working_directory()) lsd.stats_distl() resolution = 1.0e6 for i in _images: stats = lsd.get_statistics(self.get_image_name(i)) resol = 0.5 * (stats['resol_one'] + stats['resol_two']) if resol < resolution: resolution = resol self._indxr_resolution_estimate = resolution return 'ok'
from xia2.Applications.xia2 import get_command_line, write_citations, help def run(): if os.path.exists('xia2-working.phil'): sys.argv.append('xia2-working.phil') try: check_environment() check() except exceptions.Exception, e: traceback.print_exc(file = open('xia2.error', 'w')) Chatter.write('Status: error "%s"' % str(e)) # print the version Chatter.write(Version) Citations.cite('xia2') start_time = time.time() assert os.path.exists('xia2.json') from xia2.Schema.XProject import XProject xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() for crystal_id, crystal in crystals.iteritems(): #cwd = os.path.abspath(os.curdir) from libtbx import Auto scale_dir = PhilIndex.params.xia2.settings.scale.directory if scale_dir is Auto: scale_dir = 'scale' i = 0
def _index(self): """Actually index the diffraction pattern. Note well that this is not going to compute the matrix...""" # acknowledge this program if not self._indxr_images: raise RuntimeError("No good spots found on any images") Citations.cite("labelit") Citations.cite("distl") _images = [] for i in self._indxr_images: for j in i: if not j in _images: _images.append(j) _images.sort() images_str = "%d" % _images[0] for i in _images[1:]: images_str += ", %d" % i cell_str = None if self._indxr_input_cell: cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % self._indxr_input_cell if self._indxr_sweep_name: # then this is a proper autoindexing run - describe this # to the journal entry if len(self._fp_directory) <= 50: dirname = self._fp_directory else: dirname = "...%s" % self._fp_directory[-46:] Journal.block( "autoindexing", self._indxr_sweep_name, "labelit", { "images": images_str, "target cell": cell_str, "target lattice": self._indxr_input_lattice, "template": self._fp_template, "directory": dirname, }, ) # auto_logfiler(self) from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex index = LabelitIndex() index.set_working_directory(self.get_working_directory()) auto_logfiler(index) # task = 'Autoindex from images:' # for i in _images: # task += ' %s' % self.get_image_name(i) # self.set_task(task) # self.add_command_line('--index_only') Debug.write("Indexing from images:") for i in _images: index.add_image(self.get_image_name(i)) Debug.write("%s" % self.get_image_name(i)) if self._primitive_unit_cell: index.set_primitive_unit_cell(self._primitive_unit_cell) if self._indxr_input_cell: index.set_max_cell(1.25 * max(self._indxr_input_cell[:3])) xsweep = self.get_indexer_sweep() if xsweep is not None: if xsweep.get_distance() is not None: index.set_distance(xsweep.get_distance()) # if self.get_wavelength_prov() == 'user': # index.set_wavelength(self.get_wavelength()) if xsweep.get_beam_centre() is not None: index.set_beam_centre(xsweep.get_beam_centre()) if self._refine_beam is False: index.set_refine_beam(False) else: index.set_refine_beam(True) index.set_beam_search_scope(self._beam_search_scope) if (math.fabs(self.get_wavelength() - 1.54) < 0.01) or (math.fabs(self.get_wavelength() - 2.29) < 0.01): index.set_Cu_KA_or_Cr_KA(True) try: index.run() except RuntimeError as e: if self._refine_beam is False: raise e # can we improve the situation? if self._beam_search_scope < 4.0: self._beam_search_scope += 4.0 # try repeating the indexing! self.set_indexer_done(False) return "failed" # otherwise this is beyond redemption raise e self._solutions = index.get_solutions() # FIXME this needs to check the smilie status e.g. # ":)" or ";(" or " ". # FIXME need to check the value of the RMSD and raise an # exception if the P1 solution has an RMSD > 1.0... # Change 27/FEB/08 to support user assigned spacegroups # (euugh!) have to "ignore" solutions with higher symmetry # otherwise the rest of xia will override us. Bummer. for i, solution in self._solutions.iteritems(): if self._indxr_user_input_lattice: if lattice_to_spacegroup( solution["lattice"]) > lattice_to_spacegroup( self._indxr_input_lattice): Debug.write("Ignoring solution: %s" % solution["lattice"]) del self._solutions[i] # configure the "right" solution self._solution = self.get_solution() # now store also all of the other solutions... keyed by the # lattice - however these should only be added if they # have a smiley in the appropriate record, perhaps? for solution in self._solutions.keys(): lattice = self._solutions[solution]["lattice"] if lattice in self._indxr_other_lattice_cell: if (self._indxr_other_lattice_cell[lattice]["goodness"] < self._solutions[solution]["metric"]): continue self._indxr_other_lattice_cell[lattice] = { "goodness": self._solutions[solution]["metric"], "cell": self._solutions[solution]["cell"], } self._indxr_lattice = self._solution["lattice"] self._indxr_cell = tuple(self._solution["cell"]) self._indxr_mosaic = self._solution["mosaic"] lms = LabelitMosflmMatrix() lms.set_working_directory(self.get_working_directory()) lms.set_solution(self._solution["number"]) self._indxr_payload["mosflm_orientation_matrix"] = lms.calculate() # get the beam centre from the mosflm script - mosflm # may have inverted the beam centre and labelit will know # this! mosflm_beam_centre = lms.get_mosflm_beam() if mosflm_beam_centre: self._indxr_payload["mosflm_beam_centre"] = tuple( mosflm_beam_centre) detector = copy.deepcopy(self.get_detector()) beam = copy.deepcopy(self.get_beam()) from dxtbx.model.detector_helpers import set_mosflm_beam_centre set_mosflm_beam_centre(detector, beam, mosflm_beam_centre) from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number from scitbx import matrix from cctbx import sgtbx, uctbx from dxtbx.model import CrystalFactory mosflm_matrix = matrix.sqr([ float(i) for line in lms.calculate() for i in line.replace("-", " -").split() ][:9]) space_group = sgtbx.space_group_info( lattice_to_spacegroup_number(self._solution["lattice"])).group() crystal_model = CrystalFactory.from_mosflm_matrix( mosflm_matrix, unit_cell=uctbx.unit_cell(tuple(self._solution["cell"])), space_group=space_group, ) from dxtbx.model import Experiment, ExperimentList experiment = Experiment( beam=beam, detector=detector, goniometer=self.get_goniometer(), scan=self.get_scan(), crystal=crystal_model, ) experiment_list = ExperimentList([experiment]) self.set_indexer_experiment_list(experiment_list) # also get an estimate of the resolution limit from the # labelit.stats_distl output... FIXME the name is wrong! lsd = LabelitStats_distl() lsd.set_working_directory(self.get_working_directory()) lsd.stats_distl() resolution = 1.0e6 for i in _images: stats = lsd.get_statistics(self.get_image_name(i)) resol = 0.5 * (stats["resol_one"] + stats["resol_two"]) if resol < resolution: resolution = resol self._indxr_resolution_estimate = resolution return "ok"
def _scale_prepare(self): '''Prepare the data for scaling - this will reindex it the reflections to the correct pointgroup and setting, for instance, and move the reflection files to the scale directory.''' Citations.cite('xds') Citations.cite('ccp4') Citations.cite('pointless') # GATHER phase - get the reflection files together... note that # it is not necessary in here to keep the batch information as we # don't wish to rebatch the reflections prior to scaling. # FIXME need to think about what I will do about the radiation # damage analysis in here... self._sweep_information = { } # FIXME in here I want to record the batch number to # epoch mapping as per the CCP4 Scaler implementation. Journal.block( 'gathering', self.get_scaler_xcrystal().get_name(), 'XDS', {'working directory':self.get_working_directory()}) for epoch in self._scalr_integraters.keys(): intgr = self._scalr_integraters[epoch] pname, xname, dname = intgr.get_integrater_project_info() sname = intgr.get_integrater_sweep_name() self._sweep_information[epoch] = { 'pname':pname, 'xname':xname, 'dname':dname, 'integrater':intgr, 'corrected_intensities':intgr.get_integrater_corrected_intensities(), 'prepared_reflections':None, 'scaled_reflections':None, 'header':intgr.get_header(), 'batches':intgr.get_integrater_batches(), 'image_to_epoch':intgr.get_integrater_sweep( ).get_image_to_epoch(), 'image_to_dose':{}, 'batch_offset':0, 'sname':sname } Journal.entry({'adding data from':'%s/%s/%s' % \ (xname, dname, sname)}) # what are these used for? # pname / xname / dname - dataset identifiers # image to epoch / batch offset / batches - for RD analysis Debug.write('For EPOCH %s have:' % str(epoch)) Debug.write('ID = %s/%s/%s' % (pname, xname, dname)) Debug.write('SWEEP = %s' % intgr.get_integrater_sweep_name()) # next work through all of the reflection files and make sure that # they are XDS_ASCII format... epochs = self._sweep_information.keys() epochs.sort() self._first_epoch = min(epochs) self._scalr_pname = self._sweep_information[epochs[0]]['pname'] self._scalr_xname = self._sweep_information[epochs[0]]['xname'] for epoch in epochs: intgr = self._scalr_integraters[epoch] pname = self._sweep_information[epoch]['pname'] xname = self._sweep_information[epoch]['xname'] dname = self._sweep_information[epoch]['dname'] sname = self._sweep_information[epoch]['sname'] if self._scalr_pname != pname: raise RuntimeError, 'all data must have a common project name' xname = self._sweep_information[epoch]['xname'] if self._scalr_xname != xname: raise RuntimeError, \ 'all data for scaling must come from one crystal' xsh = XDSScalerHelper() xsh.set_working_directory(self.get_working_directory()) hklin = self._sweep_information[epoch]['corrected_intensities'] hklout = os.path.join(self.get_working_directory(), '%s_%s_%s_%s_CORRECTED.HKL' %( pname, xname, dname, sname)) sweep = intgr.get_integrater_sweep() if sweep.get_frames_to_process() is not None: offset = intgr.get_frame_offset() #print "offset: %d" %offset start, end = sweep.get_frames_to_process() start -= offset end -= offset #end += 1 ???? #print "limiting batches: %d-%d" %(start, end) xsh.limit_batches(hklin, hklout, start, end) self._sweep_information[epoch]['corrected_intensities'] = hklout # if there is more than one sweep then compare the lattices # and eliminate all but the lowest symmetry examples if # there are more than one... # ------------------------------------------------- # Ensure that the integration lattices are the same # ------------------------------------------------- need_to_return = False if len(self._sweep_information.keys()) > 1: lattices = [] # FIXME run this stuff in parallel as well... for epoch in self._sweep_information.keys(): intgr = self._sweep_information[epoch]['integrater'] hklin = self._sweep_information[epoch]['corrected_intensities'] refiner = intgr.get_integrater_refiner() if self._scalr_input_pointgroup: pointgroup = self._scalr_input_pointgroup reindex_op = 'h,k,l' ntr = False else: pointgroup, reindex_op, ntr = \ self._pointless_indexer_jiffy(hklin, refiner) Debug.write('X1698: %s: %s' % (pointgroup, reindex_op)) lattice = Syminfo.get_lattice(pointgroup) if not lattice in lattices: lattices.append(lattice) if ntr: # if we need to return, we should logically reset # any reindexing operator right? right here all # we are talking about is the correctness of # individual pointgroups?? Bug # 3373 reindex_op = 'h,k,l' # actually, should this not be done "by magic" # when a new pointgroup is assigned in the # pointless indexer jiffy above?! intgr.set_integrater_reindex_operator( reindex_op, compose = False) need_to_return = True # bug # 2433 - need to ensure that all of the lattice # conclusions were the same... if len(lattices) > 1: ordered_lattices = [] for l in lattices_in_order(): if l in lattices: ordered_lattices.append(l) correct_lattice = ordered_lattices[0] Debug.write('Correct lattice asserted to be %s' % \ correct_lattice) # transfer this information back to the indexers for epoch in self._sweep_information.keys(): integrater = self._sweep_information[ epoch]['integrater'] refiner = integrater.get_integrater_refiner() sname = integrater.get_integrater_sweep_name() if not refiner: continue state = refiner.set_refiner_asserted_lattice( correct_lattice) if state == refiner.LATTICE_CORRECT: Debug.write('Lattice %s ok for sweep %s' % \ (correct_lattice, sname)) elif state == refiner.LATTICE_IMPOSSIBLE: raise RuntimeError, 'Lattice %s impossible for %s' % \ (correct_lattice, sname) elif state == refiner.LATTICE_POSSIBLE: Debug.write('Lattice %s assigned for sweep %s' % \ (correct_lattice, sname)) need_to_return = True # if one or more of them was not in the lowest lattice, # need to return here to allow reprocessing if need_to_return: self.set_scaler_done(False) self.set_scaler_prepare_done(False) return # next if there is more than one sweep then generate # a merged reference reflection file to check that the # setting for all reflection files is the same... # if we get to here then all data was processed with the same # lattice # ---------------------------------------------------------- # next ensure that all sweeps are set in the correct setting # ---------------------------------------------------------- if self.get_scaler_reference_reflection_file(): self._reference = self.get_scaler_reference_reflection_file() Debug.write('Using HKLREF %s' % self._reference) md = self._factory.Mtzdump() md.set_hklin(self.get_scaler_reference_reflection_file()) md.dump() self._xds_spacegroup = Syminfo.spacegroup_name_to_number( md.get_spacegroup()) Debug.write('Spacegroup %d' % self._xds_spacegroup) elif PhilIndex.params.xia2.settings.scale.reference_reflection_file: self._reference = PhilIndex.params.xia2.settings.scale.reference_reflection_file Debug.write('Using HKLREF %s' % self._reference) md = self._factory.Mtzdump() md.set_hklin(PhilIndex.params.xia2.settings.scale.reference_reflection_file) md.dump() self._xds_spacegroup = Syminfo.spacegroup_name_to_number( md.get_spacegroup()) Debug.write('Spacegroup %d' % self._xds_spacegroup) params = PhilIndex.params use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs if len(self._sweep_information.keys()) > 1 and use_brehm_diederichs: brehm_diederichs_files_in = [] for epoch in self._sweep_information.keys(): intgr = self._sweep_information[epoch]['integrater'] hklin = self._sweep_information[epoch]['corrected_intensities'] refiner = intgr.get_integrater_refiner() # in here need to consider what to do if the user has # assigned the pointgroup on the command line ... if not self._scalr_input_pointgroup: pointgroup, reindex_op, ntr = \ self._pointless_indexer_jiffy(hklin, refiner) if ntr: # Bug # 3373 Debug.write('Reindex to standard (PIJ): %s' % \ reindex_op) intgr.set_integrater_reindex_operator( reindex_op, compose = False) reindex_op = 'h,k,l' need_to_return = True else: # 27/FEB/08 to support user assignment of pointgroups Debug.write('Using input pointgroup: %s' % \ self._scalr_input_pointgroup) pointgroup = self._scalr_input_pointgroup reindex_op = 'h,k,l' intgr.set_integrater_reindex_operator(reindex_op) intgr.set_integrater_spacegroup_number( Syminfo.spacegroup_name_to_number(pointgroup)) self._sweep_information[epoch]['corrected_intensities'] \ = intgr.get_integrater_corrected_intensities() # convert the XDS_ASCII for this sweep to mtz - on the next # get this should be in the correct setting... dname = self._sweep_information[epoch]['dname'] sname = intgr.get_integrater_sweep_name() hklin = self._sweep_information[epoch]['corrected_intensities'] hklout = os.path.join(self.get_working_directory(), '%s_%s.mtz' % (dname, sname)) FileHandler.record_temporary_file(hklout) # now use pointless to make this conversion pointless = self._factory.Pointless() pointless.set_xdsin(hklin) pointless.set_hklout(hklout) pointless.xds_to_mtz() brehm_diederichs_files_in.append(hklout) # now run cctbx.brehm_diederichs to figure out the indexing hand for # each sweep from xia2.Wrappers.Cctbx.BrehmDiederichs import BrehmDiederichs brehm_diederichs = BrehmDiederichs() brehm_diederichs.set_working_directory(self.get_working_directory()) auto_logfiler(brehm_diederichs) brehm_diederichs.set_input_filenames(brehm_diederichs_files_in) # 1 or 3? 1 seems to work better? brehm_diederichs.set_asymmetric(1) brehm_diederichs.run() reindexing_dict = brehm_diederichs.get_reindexing_dict() for epoch in self._sweep_information.keys(): intgr = self._sweep_information[epoch]['integrater'] dname = self._sweep_information[epoch]['dname'] sname = intgr.get_integrater_sweep_name() hklin = self._sweep_information[epoch]['corrected_intensities'] hklout = os.path.join(self.get_working_directory(), '%s_%s.mtz' % (dname, sname)) # apply the reindexing operator intgr.set_integrater_reindex_operator(reindex_op) # and copy the reflection file to the local directory hklin = self._sweep_information[epoch]['corrected_intensities'] hklout = os.path.join(self.get_working_directory(), '%s_%s.HKL' % (dname, sname)) Debug.write('Copying %s to %s' % (hklin, hklout)) shutil.copyfile(hklin, hklout) # record just the local file name... self._sweep_information[epoch][ 'prepared_reflections'] = os.path.split(hklout)[-1] elif len(self._sweep_information.keys()) > 1 and \ not self._reference: # need to generate a reference reflection file - generate this # from the reflections in self._first_epoch # # FIXME this should really use the Brehm and Diederichs method # if you have lots of little sweeps... intgr = self._sweep_information[self._first_epoch]['integrater'] hklin = self._sweep_information[epoch]['corrected_intensities'] refiner = intgr.get_integrater_refiner() if self._scalr_input_pointgroup: Debug.write('Using input pointgroup: %s' % \ self._scalr_input_pointgroup) pointgroup = self._scalr_input_pointgroup ntr = False reindex_op = 'h,k,l' else: pointgroup, reindex_op, ntr = self._pointless_indexer_jiffy( hklin, refiner) Debug.write('X1698: %s: %s' % (pointgroup, reindex_op)) reference_reindex_op = intgr.get_integrater_reindex_operator() if ntr: # Bug # 3373 intgr.set_integrater_reindex_operator( reindex_op, compose = False) reindex_op = 'h,k,l' need_to_return = True self._xds_spacegroup = Syminfo.spacegroup_name_to_number(pointgroup) # next pass this reindexing operator back to the source # of the reflections intgr.set_integrater_reindex_operator(reindex_op) intgr.set_integrater_spacegroup_number( Syminfo.spacegroup_name_to_number(pointgroup)) self._sweep_information[epoch]['corrected_intensities'] \ = intgr.get_integrater_corrected_intensities() hklin = self._sweep_information[epoch]['corrected_intensities'] hklout = os.path.join(self.get_working_directory(), 'xds-pointgroup-reference-unsorted.mtz') FileHandler.record_temporary_file(hklout) # now use pointless to handle this conversion pointless = self._factory.Pointless() pointless.set_xdsin(hklin) pointless.set_hklout(hklout) pointless.xds_to_mtz() self._reference = hklout if self._reference: from xia2.Driver.DriverFactory import DriverFactory def run_one_sweep(args): sweep_information = args[0] pointless_indexer_jiffy = args[1] factory = args[2] job_type = args[3] if job_type: DriverFactory.set_driver_type(job_type) intgr = sweep_information['integrater'] hklin = sweep_information['corrected_intensities'] refiner = intgr.get_integrater_refiner() # in here need to consider what to do if the user has # assigned the pointgroup on the command line ... if not self._scalr_input_pointgroup: pointgroup, reindex_op, ntr = \ self._pointless_indexer_jiffy(hklin, refiner) if ntr: # Bug # 3373 Debug.write('Reindex to standard (PIJ): %s' % \ reindex_op) intgr.set_integrater_reindex_operator( reindex_op, compose = False) reindex_op = 'h,k,l' need_to_return = True else: # 27/FEB/08 to support user assignment of pointgroups Debug.write('Using input pointgroup: %s' % \ self._scalr_input_pointgroup) pointgroup = self._scalr_input_pointgroup reindex_op = 'h,k,l' intgr.set_integrater_reindex_operator(reindex_op) intgr.set_integrater_spacegroup_number( Syminfo.spacegroup_name_to_number(pointgroup)) sweep_information['corrected_intensities'] \ = intgr.get_integrater_corrected_intensities() # convert the XDS_ASCII for this sweep to mtz - on the next # get this should be in the correct setting... hklin = sweep_information['corrected_intensities'] # now use pointless to make this conversion # try with no conversion?! pointless = self._factory.Pointless() pointless.set_xdsin(hklin) hklout = os.path.join( self.get_working_directory(), '%d_xds-pointgroup-unsorted.mtz' %pointless.get_xpid()) FileHandler.record_temporary_file(hklout) pointless.set_hklout(hklout) pointless.xds_to_mtz() pointless = self._factory.Pointless() pointless.set_hklin(hklout) pointless.set_hklref(self._reference) pointless.decide_pointgroup() pointgroup = pointless.get_pointgroup() reindex_op = pointless.get_reindex_operator() # for debugging print out the reindexing operations and # what have you... Debug.write('Reindex to standard: %s' % reindex_op) # this should send back enough information that this # is in the correct pointgroup (from the call above) and # also in the correct setting, from the interaction # with the reference set... - though I guess that the # spacegroup number should not have changed, right? # set the reindex operation afterwards... though if the # spacegroup number is the same this should make no # difference, right?! intgr.set_integrater_spacegroup_number( Syminfo.spacegroup_name_to_number(pointgroup)) intgr.set_integrater_reindex_operator(reindex_op) sweep_information['corrected_intensities'] \ = intgr.get_integrater_corrected_intensities() # and copy the reflection file to the local directory dname = sweep_information['dname'] sname = intgr.get_integrater_sweep_name() hklin = sweep_information['corrected_intensities'] hklout = os.path.join(self.get_working_directory(), '%s_%s.HKL' % (dname, sname)) Debug.write('Copying %s to %s' % (hklin, hklout)) shutil.copyfile(hklin, hklout) # record just the local file name... sweep_information['prepared_reflections'] = os.path.split(hklout)[-1] return sweep_information from libtbx import easy_mp params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing njob = mp_params.njob if njob > 1: # cache drivertype drivertype = DriverFactory.get_driver_type() args = [ (self._sweep_information[epoch], self._pointless_indexer_jiffy, self._factory, mp_params.type) for epoch in self._sweep_information.keys()] results_list = easy_mp.parallel_map( run_one_sweep, args, params=None, processes=njob, method="threading", asynchronous=True, callback=None, preserve_order=True, preserve_exception_message=True) # restore drivertype DriverFactory.set_driver_type(drivertype) # results should be given back in the same order for i, epoch in enumerate(self._sweep_information.keys()): self._sweep_information[epoch] = results_list[i] else: for epoch in self._sweep_information.keys(): self._sweep_information[epoch] = run_one_sweep( (self._sweep_information[epoch], self._pointless_indexer_jiffy, self._factory, None)) else: # convert the XDS_ASCII for this sweep to mtz epoch = self._first_epoch intgr = self._sweep_information[epoch]['integrater'] refiner = intgr.get_integrater_refiner() sname = intgr.get_integrater_sweep_name() hklout = os.path.join(self.get_working_directory(), '%s-pointless.mtz' % sname) FileHandler.record_temporary_file(hklout) pointless = self._factory.Pointless() pointless.set_xdsin(self._sweep_information[epoch]['corrected_intensities']) pointless.set_hklout(hklout) pointless.xds_to_mtz() # run it through pointless interacting with the # Indexer which belongs to this sweep hklin = hklout if self._scalr_input_pointgroup: Debug.write('Using input pointgroup: %s' % \ self._scalr_input_pointgroup) pointgroup = self._scalr_input_pointgroup ntr = False reindex_op = 'h,k,l' else: pointgroup, reindex_op, ntr = self._pointless_indexer_jiffy( hklin, refiner) if ntr: # if we need to return, we should logically reset # any reindexing operator right? right here all # we are talking about is the correctness of # individual pointgroups?? Bug # 3373 reindex_op = 'h,k,l' intgr.set_integrater_reindex_operator( reindex_op, compose = False) need_to_return = True self._xds_spacegroup = Syminfo.spacegroup_name_to_number(pointgroup) # next pass this reindexing operator back to the source # of the reflections intgr.set_integrater_reindex_operator(reindex_op) intgr.set_integrater_spacegroup_number( Syminfo.spacegroup_name_to_number(pointgroup)) self._sweep_information[epoch]['corrected_intensities'] \ = intgr.get_integrater_corrected_intensities() hklin = self._sweep_information[epoch]['corrected_intensities'] dname = self._sweep_information[epoch]['dname'] hklout = os.path.join(self.get_working_directory(), '%s_%s.HKL' % (dname, sname)) # and copy the reflection file to the local # directory Debug.write('Copying %s to %s' % (hklin, hklout)) shutil.copyfile(hklin, hklout) # record just the local file name... self._sweep_information[epoch][ 'prepared_reflections'] = os.path.split(hklout)[-1] if need_to_return: self.set_scaler_done(False) self.set_scaler_prepare_done(False) return unit_cell_list = [] for epoch in self._sweep_information.keys(): integrater = self._sweep_information[epoch]['integrater'] cell = integrater.get_integrater_cell() n_ref = integrater.get_integrater_n_ref() Debug.write('Cell for %s: %.2f %.2f %.2f %.2f %.2f %.2f' % \ (integrater.get_integrater_sweep_name(), cell[0], cell[1], cell[2], cell[3], cell[4], cell[5])) Debug.write('=> %d reflections' % n_ref) unit_cell_list.append((cell, n_ref)) self._scalr_cell = compute_average_unit_cell(unit_cell_list) self._scalr_resolution_limits = { } Debug.write('Determined unit cell: %.2f %.2f %.2f %.2f %.2f %.2f' % \ tuple(self._scalr_cell)) if os.path.exists(os.path.join( self.get_working_directory(), 'REMOVE.HKL')): os.remove(os.path.join( self.get_working_directory(), 'REMOVE.HKL')) Debug.write('Deleting REMOVE.HKL at end of scale prepare.') return
def _index(self): '''Implement the indexer interface.''' Citations.cite('mosflm') indexer = MosflmIndex() indexer.set_working_directory(self.get_working_directory()) auto_logfiler(indexer) from xia2.lib.bits import unique_elements _images = unique_elements(self._indxr_images) indexer.set_images(_images) images_str = ', '.join(map(str, _images)) cell_str = None if self._indxr_input_cell: cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \ self._indxr_input_cell if self._indxr_sweep_name: #if len(self._fp_directory) <= 50: #dirname = self._fp_directory #else: #dirname = '...%s' % self._fp_directory[-46:] dirname = os.path.dirname(self.get_imageset().get_template()) Journal.block( 'autoindexing', self._indxr_sweep_name, 'mosflm', {'images':images_str, 'target cell':self._indxr_input_cell, 'target lattice':self._indxr_input_lattice, 'template':self.get_imageset().get_template(), 'directory':dirname}) #task = 'Autoindex from images:' #for i in _images: #task += ' %s' % self.get_image_name(i) #self.set_task(task) indexer.set_template(os.path.basename(self.get_template())) indexer.set_directory(self.get_directory()) xsweep = self.get_indexer_sweep() if xsweep is not None: if xsweep.get_distance() is not None: indexer.set_distance(xsweep.get_distance()) #if self.get_wavelength_prov() == 'user': #index.set_wavelength(self.get_wavelength()) if xsweep.get_beam_centre() is not None: indexer.set_beam_centre(xsweep.get_beam_centre()) if self._indxr_input_cell: indexer.set_unit_cell(self._indxr_input_cell) if self._indxr_input_lattice != None: spacegroup_number = lattice_to_spacegroup( self._indxr_input_lattice) indexer.set_space_group_number(spacegroup_number) if not self._mosflm_autoindex_thresh: try: min_peaks = 200 Debug.write('Aiming for at least %d spots...' % min_peaks) thresholds = [] for i in _images: p = Printpeaks() p.set_working_directory(self.get_working_directory()) auto_logfiler(p) p.set_image(self.get_image_name(i)) thresh = p.threshold(min_peaks) Debug.write('Autoindex threshold for image %d: %d' % \ (i, thresh)) thresholds.append(thresh) thresh = min(thresholds) self._mosflm_autoindex_thresh = thresh except Exception as e: print str(e) #XXX this should disappear! Debug.write('Error computing threshold: %s' % str(e)) Debug.write('Using default of 20.0') thresh = 20.0 else: thresh = self._mosflm_autoindex_thresh Debug.write('Using autoindex threshold: %d' % thresh) if self._mosflm_autoindex_sol: indexer.set_solution_number(self._mosflm_autoindex_sol) indexer.set_threshold(thresh) # now forget this to prevent weird things happening later on if self._mosflm_autoindex_sol: self._mosflm_autoindex_sol = 0 indexer.run() #sweep = self.get_indexer_sweep_name() #FileHandler.record_log_file( #'%s INDEX' % (sweep), self.get_log_file()) indxr_cell = indexer.get_refined_unit_cell() self._indxr_lattice = indexer.get_lattice() space_group_number = indexer.get_indexed_space_group_number() detector_distance = indexer.get_refined_distance() beam_centre = indexer.get_refined_beam_centre() mosaic_spreads = indexer.get_mosaic_spreads() if min(list(indxr_cell)) < 10.0 and \ indxr_cell[2] / indxr_cell[0] > 6: Debug.write( 'Unrealistic autoindexing solution: ' + '%.2f %.2f %.2f %.2f %.2f %.2f' % indxr_cell) # tweak some parameters and try again... self._mosflm_autoindex_thresh *= 1.5 self.set_indexer_done(False) return intgr_params = { } # look up other possible indexing solutions (not well - in # standard settings only!) This is moved earlier as it could # result in returning if Mosflm has selected the wrong # solution! try: self._indxr_other_lattice_cell = indexer.get_solutions() # Change 27/FEB/08 to support user assigned spacegroups if self._indxr_user_input_lattice: lattice_to_spacegroup_dict = { 'aP':1, 'mP':3, 'mC':5, 'oP':16, 'oC':20, 'oF':22, 'oI':23, 'tP':75, 'tI':79, 'hP':143, 'hR':146, 'cP':195, 'cF':196, 'cI':197} for k in self._indxr_other_lattice_cell.keys(): if lattice_to_spacegroup_dict[k] > \ lattice_to_spacegroup_dict[ self._indxr_input_lattice]: del(self._indxr_other_lattice_cell[k]) # check that the selected unit cell matches - and if # not raise a "horrible" exception if self._indxr_input_cell: assert indxr_cell is not None for j in range(6): if math.fabs(self._indxr_input_cell[j] - indxr_cell[j]) > 2.0: Chatter.write( 'Mosflm autoindexing did not select ' + 'correct (target) unit cell') raise RuntimeError, \ 'something horrible happened in indexing' except RuntimeError, e: # check if mosflm rejected a solution we have it if 'horribl' in str(e): # ok it did - time to break out the big guns... if not self._indxr_input_cell: raise RuntimeError, \ 'error in solution selection when not preset' # XXX FIXME self._mosflm_autoindex_sol = _get_indexing_solution_number( indexer.get_all_output(), self._indxr_input_cell, self._indxr_input_lattice) # set the fact that we are not done... self.set_indexer_done(False) # and return - hopefully this will restart everything return else: raise e
def multi_crystal_analysis(stop_after=None): '''Actually process something...''' assert os.path.exists('xia2.json') from xia2.Schema.XProject import XProject xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() for crystal_id, crystal in crystals.iteritems(): cwd = os.path.abspath(os.curdir) working_directory = Environment.generate_directory( [crystal.get_name(), 'analysis']) os.chdir(working_directory) scaler = crystal._get_scaler() #epoch_to_si = {} epoch_to_batches = {} epoch_to_integrated_intensities = {} epoch_to_sweep_name = {} epoch_to_experiments_filename = {} epoch_to_experiments = {} sweep_name_to_epoch = {} epoch_to_first_image = {} from dxtbx.serialize import load try: epochs = scaler._sweep_information.keys() for epoch in epochs: si = scaler._sweep_information[epoch] epoch_to_batches[epoch] = si['batches'] epoch_to_integrated_intensities[epoch] = si[ 'corrected_intensities'] epoch_to_sweep_name[epoch] = si['sname'] sweep_name_to_epoch[si['name']] = epoch intgr = si['integrater'] epoch_to_experiments_filename[ epoch] = intgr.get_integrated_experiments() epoch_to_experiments[epoch] = load.experiment_list( intgr.get_integrated_experiments()) except AttributeError: epochs = scaler._sweep_handler.get_epochs() for epoch in epochs: si = scaler._sweep_handler.get_sweep_information(epoch) epoch_to_batches[epoch] = si.get_batches() epoch_to_integrated_intensities[epoch] = si.get_reflections() epoch_to_sweep_name[epoch] = si.get_sweep_name() sweep_name_to_epoch[si.get_sweep_name()] = epoch intgr = si.get_integrater() epoch_to_experiments_filename[ epoch] = intgr.get_integrated_experiments() epoch_to_experiments[epoch] = load.experiment_list( intgr.get_integrated_experiments()) from xia2.Wrappers.Dials.StereographicProjection import StereographicProjection sp_json_files = {} for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)): sp = StereographicProjection() auto_logfiler(sp) sp.set_working_directory(working_directory) for experiments in epoch_to_experiments_filename.values(): sp.add_experiments(experiments) sp.set_hkl(hkl) sp.run() sp_json_files[hkl] = sp.get_json_filename() unmerged_mtz = scaler.get_scaled_reflections( 'mtz_unmerged').values()[0] from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(unmerged_mtz) from xia2.Wrappers.XIA.PlotMultiplicity import PlotMultiplicity mult_json_files = {} for axis in ('h', 'k', 'l'): pm = PlotMultiplicity() auto_logfiler(pm) pm.set_working_directory(working_directory) pm.set_mtz_filename(unmerged_mtz) pm.set_slice_axis(axis) pm.set_show_missing(True) pm.run() mult_json_files[axis] = pm.get_json_filename() intensities = None batches = None assert reader.file_type() == 'ccp4_mtz' arrays = reader.as_miller_arrays(merge_equivalents=False) for ma in arrays: if ma.info().labels == ['BATCH']: batches = ma elif ma.info().labels == ['I', 'SIGI']: intensities = ma elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']: intensities = ma from xia2.Wrappers.CCP4.Blend import Blend hand_blender = Blend() hand_blender.set_working_directory(working_directory) auto_logfiler(hand_blender) Citations.cite('blend') from xia2.Handlers.Environment import which Rscript_binary = which('Rscript', debug=False) if Rscript_binary is None: Chatter.write('Skipping BLEND analysis: Rscript not available') else: for epoch in epochs: hand_blender.add_hklin(epoch_to_integrated_intensities[epoch], label=epoch_to_sweep_name[epoch]) hand_blender.analysis() Chatter.write("Dendrogram saved to: %s" % hand_blender.get_dendrogram_file()) analysis = hand_blender.get_analysis() summary = hand_blender.get_summary() clusters = hand_blender.get_clusters() ddict = hand_blender.plot_dendrogram() phil_files_dir = 'phil_files' if not os.path.exists(phil_files_dir): os.makedirs(phil_files_dir) rows = [] headers = [ 'Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'LCV', 'aLCV', 'Average unit cell' ] completeness = flex.double() average_unit_cell_params = [] for i, cluster in clusters.iteritems(): print i sel_cluster = flex.bool(batches.size(), False) cluster_uc_params = [flex.double() for k in range(6)] for j in cluster['dataset_ids']: epoch = epochs[j - 1] batch_start, batch_end = epoch_to_batches[epoch] sel_cluster |= ((batches.data() >= batch_start) & (batches.data() <= batch_end)) expts = epoch_to_experiments.get(epoch) assert expts is not None, (epoch) assert len(expts) == 1, len(expts) expt = expts[0] uc_params = expt.crystal.get_unit_cell().parameters() for k in range(6): cluster_uc_params[k].append(uc_params[k]) intensities_cluster = intensities.select(sel_cluster) merging = intensities_cluster.merge_equivalents() merged_intensities = merging.array() multiplicities = merging.redundancies() completeness.append(merged_intensities.completeness()) average_unit_cell_params.append( tuple(flex.mean(p) for p in cluster_uc_params)) dataset_ids = cluster['dataset_ids'] assert min(dataset_ids) > 0 with open( os.path.join(phil_files_dir, 'blend_cluster_%i_images.phil' % i), 'wb') as f: sweep_names = [ hand_blender._labels[dataset_id - 1] for dataset_id in dataset_ids ] for sweep_name in sweep_names: expts = epoch_to_experiments.get( sweep_name_to_epoch.get(sweep_name)) assert expts is not None, ( sweep_name, sweep_name_to_epoch.get(sweep_name)) assert len(expts) == 1, len(expts) expt = expts[0] print >> f, 'xia2.settings.input.image = %s' % expt.imageset.get_path( 0) rows.append([ '%i' % i, ' '.join(['%i'] * len(dataset_ids)) % tuple(dataset_ids), '%.1f' % flex.mean(multiplicities.data().as_double()), '%.2f' % completeness[-1], '%.2f' % cluster['lcv'], '%.2f' % cluster['alcv'], '%g %g %g %g %g %g' % average_unit_cell_params[-1] ]) # sort table by completeness perm = flex.sort_permutation(completeness) rows = [rows[i] for i in perm] print print 'Unit cell clustering summary:' print tabulate(rows, headers, tablefmt='rst') print blend_html = tabulate(rows, headers, tablefmt='html').replace( '<table>', '<table class="table table-hover table-condensed">').replace( '<td>', '<td style="text-align: right;">') # XXX what about multiple wavelengths? with open('batches.phil', 'wb') as f: try: for epoch, si in scaler._sweep_information.iteritems(): print >> f, "batch {" print >> f, " id=%s" % si['sname'] print >> f, " range=%i,%i" % tuple(si['batches']) print >> f, "}" except AttributeError: for epoch in scaler._sweep_handler.get_epochs(): si = scaler._sweep_handler.get_sweep_information(epoch) print >> f, "batch {" print >> f, " id=%s" % si.get_sweep_name() print >> f, " range=%i,%i" % tuple(si.get_batches()) print >> f, "}" from xia2.Wrappers.XIA.MultiCrystalAnalysis import MultiCrystalAnalysis mca = MultiCrystalAnalysis() auto_logfiler(mca, extra="MultiCrystalAnalysis") mca.add_command_line_args([ scaler.get_scaled_reflections(format="sca_unmerged").values()[0], "unit_cell=%s %s %s %s %s %s" % tuple(scaler.get_scaler_cell()), "batches.phil" ]) mca.set_working_directory(working_directory) mca.run() intensity_clusters = mca.get_clusters() rows = [] headers = [ 'Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'Height', 'Average unit cell' ] completeness = flex.double() average_unit_cell_params = [] for i, cluster in intensity_clusters.iteritems(): sel_cluster = flex.bool(batches.size(), False) cluster_uc_params = [flex.double() for k in range(6)] for j in cluster['datasets']: epoch = epochs[j - 1] batch_start, batch_end = epoch_to_batches[epoch] sel_cluster |= ((batches.data() >= batch_start) & (batches.data() <= batch_end)) expts = epoch_to_experiments.get(epoch) assert expts is not None, (epoch) assert len(expts) == 1, len(expts) expt = expts[0] uc_params = expt.crystal.get_unit_cell().parameters() for k in range(6): cluster_uc_params[k].append(uc_params[k]) intensities_cluster = intensities.select(sel_cluster) merging = intensities_cluster.merge_equivalents() merged_intensities = merging.array() multiplicities = merging.redundancies() completeness.append(merged_intensities.completeness()) average_unit_cell_params.append( tuple(flex.mean(p) for p in cluster_uc_params)) dataset_ids = cluster['datasets'] rows.append([ '%i' % int(i), ' '.join(['%i'] * len(dataset_ids)) % tuple(dataset_ids), '%.1f' % flex.mean(multiplicities.data().as_double()), '%.2f' % completeness[-1], '%.2f' % cluster['height'], '%g %g %g %g %g %g' % average_unit_cell_params[-1] ]) # sort table by completeness perm = flex.sort_permutation(completeness) rows = [rows[i] for i in perm] print 'Intensity clustering summary:' print tabulate(rows, headers, tablefmt='rst') print intensity_clustering_html = tabulate( rows, headers, tablefmt='html').replace( '<table>', '<table class="table table-hover table-condensed">').replace( '<td>', '<td style="text-align: right;">') import json json_data = {} if ddict is not None: from xia2.Modules.MultiCrystalAnalysis import scipy_dendrogram_to_plotly_json json_data['blend_dendrogram'] = scipy_dendrogram_to_plotly_json(ddict) else: json_data['blend_dendrogram'] = {'data': [], 'layout': {}} json_data['intensity_clustering'] = mca.get_dict() del json_data['intensity_clustering']['clusters'] for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)): with open(sp_json_files[hkl], 'rb') as f: d = json.load(f) d['layout'][ 'title'] = 'Stereographic projection (hkl=%i%i%i)' % hkl json_data['stereographic_projection_%s%s%s' % hkl] = d for axis in ('h', 'k', 'l'): with open(mult_json_files[axis], 'rb') as f: json_data['multiplicity_%s' % axis] = json.load(f) json_str = json.dumps(json_data, indent=2) javascript = ['var graphs = %s' % (json_str)] javascript.append( 'Plotly.newPlot(blend_dendrogram, graphs.blend_dendrogram.data, graphs.blend_dendrogram.layout);' ) javascript.append( 'Plotly.newPlot(intensity_clustering, graphs.intensity_clustering.data, graphs.intensity_clustering.layout);' ) for hkl in ((1, 0, 0), (0, 1, 0), (0, 0, 1)): javascript.append( 'Plotly.newPlot(stereographic_projection_%(hkl)s, graphs.stereographic_projection_%(hkl)s.data, graphs.stereographic_projection_%(hkl)s.layout);' % ({ 'hkl': "%s%s%s" % hkl })) for axis in ('h', 'k', 'l'): javascript.append( 'Plotly.newPlot(multiplicity_%(axis)s, graphs.multiplicity_%(axis)s.data, graphs.multiplicity_%(axis)s.layout);' % ({ 'axis': axis })) html_header = ''' <head> <!-- Plotly.js --> <script src="https://cdn.plot.ly/plotly-latest.min.js"></script> <meta name="viewport" content="width=device-width, initial-scale=1" charset="UTF-8"> <link rel="stylesheet" href="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css"/> <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script> <script src="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script> <style type="text/css"> body { /*font-family: Helmet, Freesans, Helvetica, Arial, sans-serif;*/ margin: 8px; min-width: 240px; margin-left: 5%; margin-right: 5%; } .plot { float: left; width: 1200px; height: 800px; margin-bottom: 20px; } .square_plot { float: left; width: 800px; height: 800px; margin-bottom: 20px; } </style> </head> ''' html_body = ''' <body> <div class="page-header"> <h1>Multi-crystal analysis report</h1> </div> <div class="panel-group"> <div class="panel panel-default"> <div class="panel-heading" data-toggle="collapse" href="#collapse_multiplicity"> <h4 class="panel-title"> <a>Multiplicity plots</a> </h4> </div> <div id="collapse_multiplicity" class="panel-collapse collapse"> <div class="panel-body"> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_h"></div> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_k"></div> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="multiplicity_l"></div> </div> </div> </div> <div class="panel panel-default"> <div class="panel-heading" data-toggle="collapse" href="#collapse_stereographic_projection"> <h4 class="panel-title"> <a>Stereographic projections</a> </h4> </div> <div id="collapse_stereographic_projection" class="panel-collapse collapse"> <div class="panel-body"> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_100"></div> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_010"></div> <div class="col-xs-12 col-sm-12 col-md-12 square_plot" id="stereographic_projection_001"></div> </div> </div> </div> <div class="panel panel-default"> <div class="panel-heading" data-toggle="collapse" href="#collapse_cell"> <h4 class="panel-title"> <a>Unit cell clustering</a> </h4> </div> <div id="collapse_cell" class="panel-collapse collapse"> <div class="panel-body"> <div class="col-xs-12 col-sm-12 col-md-12 plot" id="blend_dendrogram"></div> <div class="table-responsive" style="width: 800px"> %(blend_html)s </div> </div> </div> </div> <div class="panel panel-default"> <div class="panel-heading" data-toggle="collapse" href="#collapse_intensity"> <h4 class="panel-title"> <a>Intensity clustering</a> </h4> </div> <div id="collapse_intensity" class="panel-collapse collapse"> <div class="panel-body"> <div class="col-xs-12 col-sm-12 col-md-12 plot" id="intensity_clustering" style="height:1000px"></div> <div class="table-responsive" style="width: 800px"> %(intensity_clustering_html)s </div> </div> </div> </div> </div> <script> %(script)s </script> </body> ''' % { 'script': '\n'.join(javascript), 'blend_html': blend_html, 'intensity_clustering_html': intensity_clustering_html } html = '\n'.join([html_header, html_body]) print "Writing html report to: %s" % 'multi-crystal-report.html' with open('multi-crystal-report.html', 'wb') as f: print >> f, html.encode('ascii', 'xmlcharrefreplace') write_citations() Environment.cleanup() return
def _index(self): '''Actually index the diffraction pattern. Note well that this is not going to compute the matrix...''' # acknowledge this program if not self._indxr_images: raise RuntimeError, 'No good spots found on any images' Citations.cite('labelit') Citations.cite('distl') _images = [] for i in self._indxr_images: for j in i: if not j in _images: _images.append(j) _images.sort() images_str = '%d' % _images[0] for i in _images[1:]: images_str += ', %d' % i cell_str = None if self._indxr_input_cell: cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \ self._indxr_input_cell if self._indxr_sweep_name: # then this is a proper autoindexing run - describe this # to the journal entry if len(self._fp_directory) <= 50: dirname = self._fp_directory else: dirname = '...%s' % self._fp_directory[-46:] Journal.block( 'autoindexing', self._indxr_sweep_name, 'labelit', {'images':images_str, 'target cell':cell_str, 'target lattice':self._indxr_input_lattice, 'template':self._fp_template, 'directory':dirname}) #auto_logfiler(self) from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex index = LabelitIndex() index.set_working_directory(self.get_working_directory()) auto_logfiler(index) #task = 'Autoindex from images:' #for i in _images: #task += ' %s' % self.get_image_name(i) #self.set_task(task) #self.add_command_line('--index_only') Debug.write('Indexing from images:') for i in _images: index.add_image(self.get_image_name(i)) Debug.write('%s' % self.get_image_name(i)) if self._indxr_input_lattice and False: index.set_space_group_number( lattice_to_spacegroup(self._indxr_input_lattice)) if self._primitive_unit_cell: index.set_primitive_unit_cell(self._primitive_unit_cell) if self._indxr_input_cell: index.set_max_cell(1.25 * max(self._indxr_input_cell[:3])) xsweep = self.get_indexer_sweep() if xsweep is not None: if xsweep.get_distance() is not None: index.set_distance(xsweep.get_distance()) #if self.get_wavelength_prov() == 'user': #index.set_wavelength(self.get_wavelength()) if xsweep.get_beam_centre() is not None: index.set_beam_centre(xsweep.get_beam_centre()) if self._refine_beam is False: index.set_refine_beam(False) else: index.set_refine_beam(True) index.set_beam_search_scope(self._beam_search_scope) if ((math.fabs(self.get_wavelength() - 1.54) < 0.01) or (math.fabs(self.get_wavelength() - 2.29) < 0.01)): index.set_Cu_KA_or_Cr_KA(True) try: index.run() except RuntimeError, e: if self._refine_beam is False: raise e # can we improve the situation? if self._beam_search_scope < 4.0: self._beam_search_scope += 4.0 # try repeating the indexing! self.set_indexer_done(False) return 'failed' # otherwise this is beyond redemption raise e
def _integrate_prepare(self): '''Prepare for integration - in XDS terms this may mean rerunning IDXREF to get the XPARM etc. DEFPIX is considered part of the full integration as it is resolution dependent.''' from xia2.Handlers.Citations import Citations Citations.cite('dials') # decide what images we are going to process, if not already # specified if not self._intgr_wedge: images = self.get_matching_images() self.set_integrater_wedge(min(images), max(images)) Debug.write('DIALS INTEGRATE PREPARE:') Debug.write('Wavelength: %.6f' % self.get_wavelength()) Debug.write('Distance: %.2f' % self.get_distance()) #if not self._intgr_indexer: #self.set_integrater_indexer(DialsIndexer()) #self.get_integrater_indexer().set_indexer_sweep( #self.get_integrater_sweep()) #self._intgr_indexer.set_working_directory( #self.get_working_directory()) #self._intgr_indexer.setup_from_imageset(self.get_imageset()) #if self.get_frame_wedge(): #wedge = self.get_frame_wedge() #Debug.write('Propogating wedge limit: %d %d' % wedge) #self._intgr_indexer.set_frame_wedge(wedge[0], wedge[1], #apply_offset = False) ## this needs to be set up from the contents of the ## Integrater frame processer - wavelength &c. #if self.get_beam_centre(): #self._intgr_indexer.set_beam_centre(self.get_beam_centre()) #if self.get_distance(): #self._intgr_indexer.set_distance(self.get_distance()) #if self.get_wavelength(): #self._intgr_indexer.set_wavelength( #self.get_wavelength()) ## get the unit cell from this indexer to initiate processing ## if it is new... and also copy out all of the information for ## the Dials indexer if not... #experiments = self._intgr_indexer.get_indexer_experiment_list() #assert len(experiments) == 1 # currently only handle one lattice/sweep #experiment = experiments[0] #crystal_model = experiment.crystal #lattice = self._intgr_indexer.get_indexer_lattice() ## check if the lattice was user assigned... #user_assigned = self._intgr_indexer.get_indexer_user_input_lattice() # XXX check that the indexer is an Dials indexer - if not then # create one... # set a low resolution limit (which isn't really used...) # this should perhaps be done more intelligently from an # analysis of the spot list or something...? if not self.get_integrater_low_resolution(): dmax = self._intgr_refiner.get_indexer_low_resolution( self.get_integrater_epoch()) self.set_integrater_low_resolution(dmax) Debug.write('Low resolution set to: %s' % \ self.get_integrater_low_resolution()) ## copy the data across from dxtbx.serialize import load refiner = self.get_integrater_refiner() self._intgr_experiments_filename = refiner.get_refiner_payload( "experiments.json") experiments = load.experiment_list(self._intgr_experiments_filename) experiment = experiments[0] self._intgr_indexed_filename = refiner.get_refiner_payload( "reflections.pickle") # this is the result of the cell refinement self._intgr_cell = experiment.crystal.get_unit_cell().parameters() Debug.write('Files available at the end of DIALS integrate prepare:') for f in self._data_files.keys(): Debug.write('%s' % f) self.set_detector(experiment.detector) self.set_beam_obj(experiment.beam) self.set_goniometer(experiment.goniometer)
def _index_prepare(self): Citations.cite("dials") # all_images = self.get_matching_images() # first = min(all_images) # last = max(all_images) spot_lists = [] experiments_filenames = [] for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps): logger.notice(banner("Spotfinding %s" % xsweep.get_name())) first, last = imageset.get_scan().get_image_range() # at this stage, break out to run the DIALS code: this sets itself up # now cheat and pass in some information... save re-reading all of the # image headers # FIXME need to adjust this to allow (say) three chunks of images from dxtbx.model.experiment_list import ExperimentListFactory sweep_filename = os.path.join(self.get_working_directory(), "%s_import.expt" % xsweep.get_name()) ExperimentListFactory.from_imageset_and_crystal( imageset, None).as_file(sweep_filename) genmask = self.GenerateMask() genmask.set_input_experiments(sweep_filename) genmask.set_output_experiments( os.path.join( self.get_working_directory(), "%s_%s_masked.expt" % (genmask.get_xpid(), xsweep.get_name()), )) genmask.set_params(PhilIndex.params.dials.masking) sweep_filename, mask_pickle = genmask.run() logger.debug("Generated mask for %s: %s", xsweep.get_name(), mask_pickle) gain = PhilIndex.params.xia2.settings.input.gain if gain is libtbx.Auto: gain_estimater = self.EstimateGain() gain_estimater.set_sweep_filename(sweep_filename) gain_estimater.run() gain = gain_estimater.get_gain() logger.info("Estimated gain: %.2f", gain) PhilIndex.params.xia2.settings.input.gain = gain # FIXME this should really use the assigned spot finding regions # offset = self.get_frame_offset() dfs_params = PhilIndex.params.dials.find_spots spotfinder = self.Spotfinder() if last - first > 10: spotfinder.set_write_hot_mask(True) spotfinder.set_input_sweep_filename(sweep_filename) spotfinder.set_output_sweep_filename( "%s_%s_strong.expt" % (spotfinder.get_xpid(), xsweep.get_name())) spotfinder.set_input_spot_filename( "%s_%s_strong.refl" % (spotfinder.get_xpid(), xsweep.get_name())) if PhilIndex.params.dials.fast_mode: wedges = self._index_select_images_i(imageset) spotfinder.set_scan_ranges(wedges) else: spotfinder.set_scan_ranges([(first, last)]) if dfs_params.phil_file is not None: spotfinder.set_phil_file(dfs_params.phil_file) if dfs_params.min_spot_size is libtbx.Auto: if imageset.get_detector()[0].get_type() == "SENSOR_PAD": dfs_params.min_spot_size = 3 else: dfs_params.min_spot_size = None if dfs_params.min_spot_size is not None: spotfinder.set_min_spot_size(dfs_params.min_spot_size) if dfs_params.min_local is not None: spotfinder.set_min_local(dfs_params.min_local) if dfs_params.sigma_strong: spotfinder.set_sigma_strong(dfs_params.sigma_strong) gain = PhilIndex.params.xia2.settings.input.gain if gain: spotfinder.set_gain(gain) if dfs_params.filter_ice_rings: spotfinder.set_filter_ice_rings(dfs_params.filter_ice_rings) if dfs_params.kernel_size: spotfinder.set_kernel_size(dfs_params.kernel_size) if dfs_params.global_threshold is not None: spotfinder.set_global_threshold(dfs_params.global_threshold) if dfs_params.threshold.algorithm is not None: spotfinder.set_threshold_algorithm( dfs_params.threshold.algorithm) spotfinder.run() spot_filename = spotfinder.get_spot_filename() if not os.path.exists(spot_filename): raise RuntimeError("Spotfinding failed: %s does not exist." % os.path.basename(spot_filename)) spot_lists.append(spot_filename) experiments_filenames.append( spotfinder.get_output_sweep_filename()) refl = flex.reflection_table.from_file(spot_filename) if not len(refl): raise RuntimeError("No spots found in sweep %s" % xsweep.get_name()) logger.info(spot_counts_per_image_plot(refl)) if not PhilIndex.params.dials.fast_mode: detectblanks = self.DetectBlanks() detectblanks.set_sweep_filename(experiments_filenames[-1]) detectblanks.set_reflections_filename(spot_filename) detectblanks.run() json = detectblanks.get_results() blank_regions = json["strong"]["blank_regions"] if len(blank_regions): blank_regions = [(int(s), int(e)) for s, e in blank_regions] for blank_start, blank_end in blank_regions: logger.info( "WARNING: Potential blank images: %i -> %i", blank_start + 1, blank_end, ) if PhilIndex.params.xia2.settings.remove_blanks: non_blanks = [] start, end = imageset.get_array_range() last_blank_end = start for blank_start, blank_end in blank_regions: if blank_start > start: non_blanks.append( (last_blank_end, blank_start)) last_blank_end = blank_end if last_blank_end + 1 < end: non_blanks.append((last_blank_end, end)) xsweep = self.get_indexer_sweep() xwav = xsweep.get_wavelength() xsample = xsweep.sample sweep_name = xsweep.get_name() for i, (nb_start, nb_end) in enumerate(non_blanks): assert i < 26 if i == 0: sub_imageset = imageset[nb_start - start:nb_end - start] xsweep._frames_to_process = (nb_start + 1, nb_end + 1) self.set_indexer_prepare_done(done=False) self._indxr_imagesets[ self._indxr_imagesets.index( imageset)] = sub_imageset xsweep._integrater._setup_from_imageset( sub_imageset) else: min_images = (PhilIndex.params.xia2.settings. input.min_images) if (nb_end - nb_start) < min_images: continue new_name = "_".join( (sweep_name, string.ascii_lowercase[i])) new_sweep = xwav.add_sweep( new_name, xsample, directory=os.path.join( os.path.basename( xsweep.get_directory()), new_name, ), image=imageset.get_path(nb_start - start), frames_to_process=(nb_start + 1, nb_end), beam=xsweep.get_beam_centre(), reversephi=xsweep.get_reversephi(), distance=xsweep.get_distance(), gain=xsweep.get_gain(), dmin=xsweep.get_resolution_high(), dmax=xsweep.get_resolution_low(), polarization=xsweep.get_polarization(), user_lattice=xsweep.get_user_lattice(), user_cell=xsweep.get_user_cell(), epoch=xsweep._epoch, ice=xsweep._ice, excluded_regions=xsweep._excluded_regions, ) logger.info( "Generating new sweep: %s (%s:%i:%i)", new_sweep.get_name(), new_sweep.get_image(), new_sweep.get_frames_to_process()[0], new_sweep.get_frames_to_process()[1], ) return if not PhilIndex.params.xia2.settings.trust_beam_centre: discovery = self.SearchBeamPosition() discovery.set_sweep_filename(experiments_filenames[-1]) discovery.set_spot_filename(spot_filename) # set scan_range to correspond to not more than 180 degrees # if we have > 20000 reflections width = imageset.get_scan().get_oscillation()[1] if (last - first) * width > 180.0 and len(refl) > 20000: end = first + int(round(180.0 / width)) - 1 logger.debug("Using %d to %d for beam search", first, end) discovery.set_image_range((first, end)) try: discovery.run() result = discovery.get_optimized_experiments_filename() # overwrite indexed.expt in experiments list experiments_filenames[-1] = result except Exception as e: logger.debug("DIALS beam centre search failed: %s", str(e), exc_info=True) self.set_indexer_payload("spot_lists", spot_lists) self.set_indexer_payload("experiments", experiments_filenames)
def generate_xia2_html(xinfo, filename="xia2.html", params=None, args=[]): assert params is None or len(args) == 0 if params is None: from xia2.Modules.Analysis import phil_scope interp = phil_scope.command_line_argument_interpreter() params, unhandled = interp.process_and_fetch( args, custom_processor="collect_remaining") params = params.extract() xia2_txt = os.path.join(os.path.abspath(os.path.curdir), "xia2.txt") assert os.path.isfile(xia2_txt), xia2_txt with open(xia2_txt, "r") as f: xia2_output = html.escape(f.read()) styles = {} columns = [] columns.append([ "", "Wavelength (Å)", "Resolution range (Å)", "Completeness (%)", "Multiplicity", "CC-half", "I/sigma", "Rmerge(I)", # anomalous statistics "Anomalous completeness (%)", "Anomalous multiplicity", ]) individual_dataset_reports = {} for cname, xcryst in xinfo.get_crystals().items(): reflection_files = xcryst.get_scaled_merged_reflections() for wname, unmerged_mtz in reflection_files["mtz_unmerged"].items(): xwav = xcryst.get_xwavelength(wname) from xia2.Modules.MultiCrystalAnalysis import batch_phil_scope scope = phil.parse(batch_phil_scope) scaler = xcryst._scaler try: for si in scaler._sweep_information.values(): batch_params = scope.extract().batch[0] batch_params.id = si["sname"] batch_params.range = si["batches"] params.batch.append(batch_params) except AttributeError: for si in scaler._sweep_handler._sweep_information.values(): batch_params = scope.extract().batch[0] batch_params.id = si.get_sweep_name() batch_params.range = si.get_batch_range() params.batch.append(batch_params) report_path = xinfo.path.joinpath(cname, "report") report_path.mkdir(parents=True, exist_ok=True) report = Report.from_unmerged_mtz(unmerged_mtz, params, report_dir=str(report_path)) xtriage_success, xtriage_warnings, xtriage_danger = None, None, None if params.xtriage_analysis: try: ( xtriage_success, xtriage_warnings, xtriage_danger, ) = report.xtriage_report() except Exception as e: params.xtriage_analysis = False logger.debug("Exception running xtriage:") logger.debug(e, exc_info=True) ( overall_stats_table, merging_stats_table, stats_plots, ) = report.resolution_plots_and_stats() d = {} d["merging_statistics_table"] = merging_stats_table d["overall_statistics_table"] = overall_stats_table individual_dataset_reports[wname] = d json_data = {} if params.xtriage_analysis: json_data["xtriage"] = (xtriage_success + xtriage_warnings + xtriage_danger) json_data.update(stats_plots) json_data.update(report.batch_dependent_plots()) json_data.update(report.intensity_stats_plots(run_xtriage=False)) json_data.update(report.pychef_plots()) json_data.update(report.pychef_plots(n_bins=1)) from scitbx.array_family import flex max_points = 500 for g in ( "scale_rmerge_vs_batch", "completeness_vs_dose", "rcp_vs_dose", "scp_vs_dose", "rd_vs_batch_difference", ): for i, data in enumerate(json_data[g]["data"]): x = data["x"] n = len(x) if n > max_points: step = n // max_points sel = (flex.int_range(n) % step) == 0 data["x"] = list(flex.int(data["x"]).select(sel)) data["y"] = list(flex.double(data["y"]).select(sel)) resolution_graphs = OrderedDict((k + "_" + wname, json_data[k]) for k in ( "cc_one_half", "i_over_sig_i", "second_moments", "wilson_intensity_plot", "completeness", "multiplicity_vs_resolution", ) if k in json_data) if params.include_radiation_damage: batch_graphs = OrderedDict((k + "_" + wname, json_data[k]) for k in ( "scale_rmerge_vs_batch", "i_over_sig_i_vs_batch", "completeness_vs_dose", "rcp_vs_dose", "scp_vs_dose", "rd_vs_batch_difference", )) else: batch_graphs = OrderedDict((k + "_" + wname, json_data[k]) for k in ("scale_rmerge_vs_batch", "i_over_sig_i_vs_batch")) misc_graphs = OrderedDict((k, json_data[k]) for k in ( "cumulative_intensity_distribution", "l_test", "multiplicities", ) if k in json_data) for k, v in report.multiplicity_plots().items(): misc_graphs[k + "_" + wname] = {"img": v} d["resolution_graphs"] = resolution_graphs d["batch_graphs"] = batch_graphs d["misc_graphs"] = misc_graphs d["xtriage"] = { "success": xtriage_success, "warnings": xtriage_warnings, "danger": xtriage_danger, } merging_stats = report.merging_stats merging_stats_anom = report.merging_stats_anom overall = merging_stats.overall overall_anom = merging_stats_anom.overall outer_shell = merging_stats.bins[-1] outer_shell_anom = merging_stats_anom.bins[-1] column = [ wname, str(xwav.get_wavelength()), "%.2f - %.2f (%.2f - %.2f)" % (overall.d_max, overall.d_min, outer_shell.d_max, outer_shell.d_min), "%.2f (%.2f)" % (overall.completeness * 100, outer_shell.completeness * 100), f"{overall.mean_redundancy:.2f} ({outer_shell.mean_redundancy:.2f})", f"{overall.cc_one_half:.4f} ({outer_shell.cc_one_half:.4f})", "%.2f (%.2f)" % (overall.i_over_sigma_mean, outer_shell.i_over_sigma_mean), f"{overall.r_merge:.4f} ({outer_shell.r_merge:.4f})", # anomalous statistics "%.2f (%.2f)" % ( overall_anom.anom_completeness * 100, outer_shell_anom.anom_completeness * 100, ), "%.2f (%.2f)" % (overall_anom.mean_redundancy, outer_shell_anom.mean_redundancy), ] columns.append(column) table = [[c[i] for c in columns] for i in range(len(columns[0]))] from cctbx import sgtbx space_groups = xcryst.get_likely_spacegroups() space_groups = [ sgtbx.space_group_info(symbol=str(symbol)) for symbol in space_groups ] space_group = space_groups[0].symbol_and_number() alternative_space_groups = [ sg.symbol_and_number() for sg in space_groups[1:] ] unit_cell = str(report.intensities.unit_cell()) # reflection files for cname, xcryst in xinfo.get_crystals().items(): # hack to replace path to reflection files with DataFiles directory data_dir = os.path.join(os.path.abspath(os.path.curdir), "DataFiles") g = glob.glob(os.path.join(data_dir, "*")) reflection_files = xcryst.get_scaled_merged_reflections() for k, rfile in reflection_files.items(): if isinstance(rfile, str): for datafile in g: if os.path.basename(datafile) == os.path.basename(rfile): reflection_files[k] = datafile break else: for kk in rfile: for datafile in g: if os.path.basename(datafile) == os.path.basename( rfile[kk]): reflection_files[k][kk] = datafile break headers = ["Dataset", "File name"] merged_mtz = reflection_files["mtz"] mtz_files = [ headers, [ "All datasets", '<a href="%s">%s</a>' % (os.path.relpath(merged_mtz), os.path.basename(merged_mtz)), ], ] for wname, unmerged_mtz in reflection_files["mtz_unmerged"].items(): mtz_files.append([ wname, '<a href="%s">%s</a>' % (os.path.relpath(unmerged_mtz), os.path.basename(unmerged_mtz)), ]) sca_files = [headers] if "sca" in reflection_files: for wname, merged_sca in reflection_files["sca"].items(): sca_files.append([ wname, '<a href="%s">%s</a>' % (os.path.relpath(merged_sca), os.path.basename(merged_sca)), ]) unmerged_sca_files = [headers] if "sca_unmerged" in reflection_files: for wname, unmerged_sca in reflection_files["sca_unmerged"].items( ): unmerged_sca_files.append([ wname, '<a href="%s">%s</a>' % ( os.path.relpath(unmerged_sca), os.path.basename(unmerged_sca), ), ]) # other files other_files = [] other_files.append(["File name", "Description"]) for other_file, description in sorted([ ("xia2.cif", "Crystallographic information file"), ("xia2.mmcif", "Macromolecular crystallographic information file"), ("shelxt.hkl", "merged structure factors for SHELXT"), ("shelxt.ins", "SHELXT instruction file"), ] + [(fn, "XPREP input file") for fn in os.listdir(os.path.join(data_dir)) if fn.endswith(".p4p")]): if os.path.exists(os.path.join(data_dir, other_file)): other_files.append([ '<a href="DataFiles/{filename}">{filename}</a>'.format( filename=other_file), description, ]) # log files log_files_table = [] log_dir = os.path.join(os.path.abspath(os.path.curdir), "LogFiles") g = glob.glob(os.path.join(log_dir, "*.log")) for logfile in g: html_file = make_logfile_html(logfile) html_file = os.path.splitext(logfile)[0] + ".html" if os.path.exists(html_file): log_files_table.append([ os.path.basename(logfile), '<a href="%s">original</a>' % os.path.relpath(logfile), '<a href="%s">html</a>' % os.path.relpath(html_file), ]) else: log_files_table.append([ os.path.basename(logfile), '<a href="%s">original</a>' % os.path.relpath(logfile), " ", ]) references = { cdict["acta"]: cdict.get("url") for cdict in Citations.get_citations_dicts() } from jinja2 import Environment, ChoiceLoader, PackageLoader loader = ChoiceLoader([ PackageLoader("xia2", "templates"), PackageLoader("dials", "templates") ]) env = Environment(loader=loader) template = env.get_template("xia2.html") html_source = template.render( page_title="xia2 processing report", xia2_output=xia2_output, space_group=space_group, alternative_space_groups=alternative_space_groups, unit_cell=unit_cell, overall_stats_table=table, cc_half_significance_level=params.cc_half_significance_level, mtz_files=mtz_files, sca_files=sca_files, unmerged_sca_files=unmerged_sca_files, other_files=other_files, log_files_table=log_files_table, individual_dataset_reports=individual_dataset_reports, references=references, styles=styles, ) with open("%s-report.json" % os.path.splitext(filename)[0], "w") as fh: json.dump(json_data, fh, indent=2) with open(filename, "wb") as f: f.write(html_source.encode("utf-8", "xmlcharrefreplace"))
def TwoThetaRefine(DriverType=None): """A factory for RefineWrapper classes.""" from xia2.Handlers.Citations import Citations Citations.cite("dials") from xia2.Driver.DriverFactory import DriverFactory DriverInstance = DriverFactory.Driver(DriverType) class RefineWrapper(DriverInstance.__class__): def __init__(self): DriverInstance.__class__.__init__(self) self.set_executable("dials.two_theta_refine") self._reindexing_operators = None self._reindexed_experiments = None self._reindexed_reflections = None self._experiments = [] self._reflection_files = [] self._phil_file = None self._combine_crystal_models = True # The following are set during run() call: self._output_cif = None self._output_mmcif = None self._output_p4p = None self._output_correlation_plot = None self._output_experiments = None self._crystal = None def set_experiments(self, experiments): self._experiments = experiments def get_experiments(self): return self._experiments def set_reflection_files(self, reflection_files): self._reflection_files = reflection_files def get_reflection_files(self): return self._reflection_files def set_phil_file(self, phil_file): self._phil_file = phil_file def set_combine_crystal_models(self, combine_crystal_models): self._combine_crystal_models = combine_crystal_models def get_output_cif(self): return self._output_cif def get_output_mmcif(self): return self._output_mmcif def set_output_p4p(self, filename): self._output_p4p = filename def get_output_p4p(self): return self._output_p4p def get_output_correlation_plot(self): return self._output_correlation_plot def get_output_experiments(self): return self._output_experiments def get_reindexed_experiments(self): return self._reindexed_experiments def get_reindexed_reflections(self): return self._reindexed_reflections def get_unit_cell(self): return self._crystal.get_unit_cell().parameters() def get_unit_cell_esd(self): return self._crystal.get_cell_parameter_sd() def set_reindex_operators(self, operators): assert len(operators) == len(self._experiments) self._reindexing_operators = operators def run(self): from xia2.Handlers.Streams import Chatter, Debug if self._reindexing_operators: Debug.write("Reindexing sweeps for dials.two_theta_refine") from xia2.lib.bits import auto_logfiler from xia2.Wrappers.Dials.Reindex import Reindex self._reindexed_experiments, self._reindexed_reflections = [], [] for e, p, op in zip( self._experiments, self._reflection_files, self._reindexing_operators, ): reindexer = Reindex() reindexer.set_cb_op(op) reindexer.set_experiments_filename(e) reindexer.set_indexed_filename(p) reindexer.set_working_directory( self.get_working_directory()) auto_logfiler(reindexer) reindexer.run() self._reindexed_experiments.append( reindexer.get_reindexed_experiments_filename()) self._reindexed_reflections.append( reindexer.get_reindexed_reflections_filename()) Debug.write("Running dials.two_theta_refine") self._output_cif = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.cif" % self.get_xpid(), ) self._output_mmcif = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.mmcif" % self.get_xpid(), ) if not self._output_p4p: self._output_p4p = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.p4p" % self.get_xpid(), ) self._output_correlation_plot = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.png" % self.get_xpid(), ) self._output_experiments = os.path.join( self.get_working_directory(), "%s_refined_cell.expt" % self.get_xpid()) self.clear_command_line() if self._reindexing_operators: for experiment in self._reindexed_experiments: self.add_command_line(experiment) for reflection_file in self._reindexed_reflections: self.add_command_line(reflection_file) else: for experiment in self._experiments: self.add_command_line(experiment) for reflection_file in self._reflection_files: self.add_command_line(reflection_file) self.add_command_line("combine_crystal_models=%s" % self._combine_crystal_models) self.add_command_line("output.cif=%s" % self._output_cif) self.add_command_line("output.mmcif=%s" % self._output_mmcif) self.add_command_line("output.p4p=%s" % self._output_p4p) if self._output_correlation_plot is not None: self.add_command_line("output.correlation_plot.filename=%s" % self._output_correlation_plot) if self._output_experiments is not None: self.add_command_line("output.experiments=%s" % self._output_experiments) if self._phil_file is not None: self.add_command_line("%s" % self._phil_file) self.start() self.close_wait() if not os.path.isfile(self._output_cif): Chatter.write( "TwoTheta refinement failed, see log file for more details:\n %s" % self.get_log_file()) raise RuntimeError("unit cell not refined") self.check_for_errors() from dxtbx.model.experiment_list import ExperimentListFactory experiments = ExperimentListFactory.from_json_file( self.get_output_experiments(), check_format=False) self._crystal = experiments.crystals()[0] def import_cif(self): """Import relevant lines from .cif output""" import iotbx.cif cif = iotbx.cif.reader(file_path=self.get_output_cif()).model() block = cif["two_theta_refine"] subset = { k: block[k] for k in block.keys() if k.startswith(("_cell", "_diffrn")) } return subset def import_mmcif(self): """Import relevant lines from .mmcif output""" if os.path.isfile(self.get_output_mmcif()): import iotbx.cif cif = iotbx.cif.reader( file_path=self.get_output_mmcif()).model() block = cif["two_theta_refine"] else: import iotbx.cif.model block = iotbx.cif.model.block() subset = { k: block[k] for k in block.keys() if k.startswith(("_cell", "_diffrn")) } return subset return RefineWrapper()
def load_sweeps_with_common_indexing(): assert os.path.exists('xia2.json') from xia2.Schema.XProject import XProject xinfo = XProject.from_json(filename='xia2.json') import dials # required for gaussian_rs warning from xia2.Wrappers.Dials.Reindex import Reindex Citations.cite('dials') from dxtbx.model.experiment.experiment_list import ExperimentListFactory import cPickle as pickle crystals = xinfo.get_crystals() assert len(crystals) == 1 crystal = next(crystals.itervalues()) working_directory = Environment.generate_directory([crystal.get_name(), 'analysis']) os.chdir(working_directory) scaler = crystal._get_scaler() epoch_to_batches = {} epoch_to_integrated_intensities = {} epoch_to_sweep_name = {} # Aimless only epochs = scaler._sweep_handler.get_epochs() reference_cell = None reference_lattice = None reference_vectors = None reference_wavelength = None # Reindex each sweep to same setting all_miller_indices = flex.miller_index() all_two_thetas = flex.double() for epoch in epochs: si = scaler._sweep_handler.get_sweep_information(epoch) Chatter.smallbanner(si.get_sweep_name(), True) Debug.smallbanner(si.get_sweep_name(), True) intgr = si.get_integrater() experiments_filename = intgr.get_integrated_experiments() reflections_filename = intgr.get_integrated_reflections() refiner = intgr.get_integrater_refiner() Debug.write('experiment: %s' % experiments_filename) Debug.write('reflection: %s' % reflections_filename) # Use setting of first sweep as reference if reference_vectors is None: reference_vectors = experiments_filename # Assume that all sweeps have the same lattice system if reference_lattice is None: reference_lattice = refiner.get_refiner_lattice() else: assert reference_lattice == refiner.get_refiner_lattice() Debug.write("lattice: %s" % refiner.get_refiner_lattice()) # Read .json file for sweep db = ExperimentListFactory.from_json_file(experiments_filename) # Assume that each file only contains a single experiment assert (len(db) == 1) db = db[0] # Get beam vector s0 = db.beam.get_unit_s0() # Use the unit cell of the first sweep as reference if reference_cell is None: reference_cell = db.crystal.get_unit_cell() Debug.write("Reference cell: %s" % str(reference_cell)) dials_reindex = Reindex() dials_reindex.set_working_directory(working_directory) dials_reindex.set_cb_op("auto") dials_reindex.set_reference_filename(reference_vectors) dials_reindex.set_experiments_filename(experiments_filename) dials_reindex.set_indexed_filename(reflections_filename) auto_logfiler(dials_reindex) dials_reindex.run() # Assume that all data are collected at same wavelength if reference_wavelength is None: reference_wavelength = intgr.get_wavelength() else: assert abs(reference_wavelength - intgr.get_wavelength()) < 0.01 Debug.write("wavelength: %f A" % intgr.get_wavelength()) Debug.write("distance: %f mm" % intgr.get_distance()) # Get integrated reflection data import dials with open(dials_reindex.get_reindexed_reflections_filename(), 'rb') as fh: reflections = pickle.load(fh) selection = reflections.get_flags(reflections.flags.used_in_refinement) Chatter.write("Found %d reflections used in refinement (out of %d entries)" % (selection.count(True), len(reflections['miller_index']))) reflections = reflections.select(selection) # Filter bad reflections selection = reflections['intensity.sum.variance'] <= 0 if selection.count(True) > 0: reflections.del_selected(selection) print 'Removing %d reflections with negative variance' % \ selection.count(True) if 'intensity.prf.variance' in reflections: selection = reflections['intensity.prf.variance'] <= 0 if selection.count(True) > 0: reflections.del_selected(selection) print 'Removing %d profile reflections with negative variance' % \ selection.count(True) # Find the observed 2theta angles miller_indices = flex.miller_index() two_thetas_obs = flex.double() for pixel, panel, hkl in zip(reflections['xyzobs.px.value'], reflections['panel'], reflections['miller_index']): assert hkl != (0, 0, 0) two_thetas_obs.append(db.detector[panel].get_two_theta_at_pixel(s0, pixel[0:2])) miller_indices.append(hkl) # Convert observed 2theta angles to degrees two_thetas_obs = two_thetas_obs * 180 / 3.14159265359 Chatter.write("Remaining %d reflections are in 2theta range %.3f - %.3f deg" % (len(miller_indices), min(two_thetas_obs), max(two_thetas_obs))) all_miller_indices.extend(miller_indices) all_two_thetas.extend(two_thetas_obs) return all_miller_indices, all_two_thetas, reference_cell, reference_lattice, reference_wavelength
def run(args=sys.argv[1:]): # Create the parser parser = OptionParser( # usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, # epilog=help_message, ) # Parse the command line params, options, args = parser.parse_args(args=args, show_diff_phil=False, return_unhandled=True) # Configure the logging xia2.Handlers.Streams.setup_logging(logfile=params.output.log, verbose=options.verbose) logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) if params.unit_cell is not None: unit_cell = params.unit_cell crystal_symmetry = crystal.symmetry(unit_cell=unit_cell) else: crystal_symmetry = None if len(params.input.experiments): experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): raise sys.exit( "Mismatched number of experiments and reflection tables found: %s & %s." % (len(experiments), len(reflections))) experiments, reflections = assign_unique_identifiers( experiments, reflections) # transform models into miller arrays intensities, batches = filtered_arrays_from_experiments_reflections( experiments, reflections, outlier_rejection_after_filter=False, partiality_threshold=0.99, return_batches=True, ) if args and os.path.isfile(args[0]): result = any_reflection_file(args[0]) unmerged_intensities = None batches_all = None for ma in result.as_miller_arrays(merge_equivalents=False, crystal_symmetry=crystal_symmetry): if ma.info().labels == ["I(+)", "SIGI(+)", "I(-)", "SIGI(-)"]: assert ma.anomalous_flag() unmerged_intensities = ma elif ma.info().labels == ["I", "SIGI"]: assert not ma.anomalous_flag() unmerged_intensities = ma elif ma.info().labels == ["BATCH"]: batches_all = ma assert batches_all is not None assert unmerged_intensities is not None sel = unmerged_intensities.sigmas() > 0 unmerged_intensities = unmerged_intensities.select(sel).set_info( unmerged_intensities.info()) batches_all = batches_all.select(sel) id_to_batches = None if len(params.batch) > 0: id_to_batches = {} for b in params.batch: assert b.id is not None assert b.range is not None assert b.id not in id_to_batches, "Duplicate batch id: %s" % b.id id_to_batches[b.id] = b.range separate = separate_unmerged(unmerged_intensities, batches_all, id_to_batches=id_to_batches) batches = list(separate.batches.values()) intensities = list(separate.intensities.values()) result = DeltaCcHalf( intensities, batches, n_bins=params.n_bins, d_min=params.d_min, cc_one_half_method=params.cc_one_half_method, group_size=params.group_size, ) logger.info(tabulate(result.get_table(), headers="firstrow")) hist_filename = "delta_cc_hist.png" logger.info("Saving histogram to %s" % hist_filename) result.plot_histogram(hist_filename) normalised_scores_filename = "normalised_scores.png" logger.info("Saving normalised scores to %s" % normalised_scores_filename) result.plot_normalised_scores(normalised_scores_filename) Citations.cite("delta_cc_half") for citation in Citations.get_citations_acta(): logger.info(citation)
def _scale_prepare(self): '''Perform all of the preparation required to deliver the scaled data. This should sort together the reflection files, ensure that they are correctly indexed (via pointless) and generally tidy things up.''' # acknowledge all of the programs we are about to use... Citations.cite('pointless') Citations.cite('aimless') Citations.cite('ccp4') # ---------- GATHER ---------- self._sweep_handler = SweepInformationHandler(self._scalr_integraters) Journal.block( 'gathering', self.get_scaler_xcrystal().get_name(), 'CCP4', {'working directory':self.get_working_directory()}) for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) pname, xname, dname = si.get_project_info() sname = si.get_sweep_name() exclude_sweep = False for sweep in PhilIndex.params.xia2.settings.sweep: if sweep.id == sname and sweep.exclude: exclude_sweep = True break if exclude_sweep: self._sweep_handler.remove_epoch(epoch) Debug.write('Excluding sweep %s' %sname) else: Journal.entry({'adding data from':'%s/%s/%s' % \ (xname, dname, sname)}) # gather data for all images which belonged to the parent # crystal - allowing for the fact that things could go wrong # e.g. epoch information not available, exposure times not in # headers etc... for e in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(e) assert is_mtz_file(si.get_reflections()) p, x = self._sweep_handler.get_project_info() self._scalr_pname = p self._scalr_xname = x # verify that the lattices are consistent, calling eliminate if # they are not N.B. there could be corner cases here need_to_return = False multi_sweep_indexing = \ PhilIndex.params.xia2.settings.developmental.multi_sweep_indexing if len(self._sweep_handler.get_epochs()) > 1: if multi_sweep_indexing and not self._scalr_input_pointgroup: pointless_hklins = [] max_batches = 0 for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() md = self._factory.Mtzdump() md.set_hklin(hklin) md.dump() batches = md.get_batches() if 1 + max(batches) - min(batches) > max_batches: max_batches = max(batches) - min(batches) + 1 datasets = md.get_datasets() Debug.write('In reflection file %s found:' % hklin) for d in datasets: Debug.write('... %s' % d) dataset_info = md.get_dataset_info(datasets[0]) from xia2.lib.bits import nifty_power_of_ten Debug.write('Biggest sweep has %d batches' % max_batches) max_batches = nifty_power_of_ten(max_batches) counter = 0 for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() integrater = si.get_integrater() refiner = integrater.get_integrater_refiner() hklin = self._prepare_pointless_hklin( hklin, si.get_integrater().get_phi_width()) rb = self._factory.Rebatch() hklout = os.path.join(self.get_working_directory(), '%s_%s_%s_%s_prepointless.mtz' % \ (pname, xname, dname, si.get_sweep_name())) # we will want to delete this one exit FileHandler.record_temporary_file(hklout) first_batch = min(si.get_batches()) si.set_batch_offset(counter * max_batches - first_batch + 1) rb.set_hklin(hklin) rb.set_first_batch(counter * max_batches + 1) rb.set_project_info(pname, xname, dname) rb.set_hklout(hklout) new_batches = rb.rebatch() pointless_hklins.append(hklout) # update the counter & recycle counter += 1 s = self._factory.Sortmtz() pointless_hklin = os.path.join(self.get_working_directory(), '%s_%s_prepointless_sorted.mtz' % \ (self._scalr_pname, self._scalr_xname)) s.set_hklout(pointless_hklin) for hklin in pointless_hklins: s.add_hklin(hklin) s.sort() pointgroup, reindex_op, ntr, pt = \ self._pointless_indexer_jiffy( pointless_hklin, refiner) Debug.write('X1698: %s: %s' % (pointgroup, reindex_op)) lattices = [Syminfo.get_lattice(pointgroup)] for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) intgr = si.get_integrater() hklin = si.get_reflections() refiner = intgr.get_integrater_refiner() if ntr: intgr.integrater_reset_reindex_operator() need_to_return = True else: lattices = [] for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) intgr = si.get_integrater() hklin = si.get_reflections() refiner = intgr.get_integrater_refiner() if self._scalr_input_pointgroup: pointgroup = self._scalr_input_pointgroup reindex_op = 'h,k,l' ntr = False else: pointless_hklin = self._prepare_pointless_hklin( hklin, si.get_integrater().get_phi_width()) pointgroup, reindex_op, ntr, pt = \ self._pointless_indexer_jiffy( pointless_hklin, refiner) Debug.write('X1698: %s: %s' % (pointgroup, reindex_op)) lattice = Syminfo.get_lattice(pointgroup) if not lattice in lattices: lattices.append(lattice) if ntr: intgr.integrater_reset_reindex_operator() need_to_return = True if len(lattices) > 1: # why not using pointless indexer jiffy??! correct_lattice = sort_lattices(lattices)[0] Chatter.write('Correct lattice asserted to be %s' % \ correct_lattice) # transfer this information back to the indexers for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) refiner = si.get_integrater().get_integrater_refiner() sname = si.get_sweep_name() state = refiner.set_refiner_asserted_lattice( correct_lattice) if state == refiner.LATTICE_CORRECT: Chatter.write('Lattice %s ok for sweep %s' % \ (correct_lattice, sname)) elif state == refiner.LATTICE_IMPOSSIBLE: raise RuntimeError, 'Lattice %s impossible for %s' \ % (correct_lattice, sname) elif state == refiner.LATTICE_POSSIBLE: Chatter.write('Lattice %s assigned for sweep %s' % \ (correct_lattice, sname)) need_to_return = True # if one or more of them was not in the lowest lattice, # need to return here to allow reprocessing if need_to_return: self.set_scaler_done(False) self.set_scaler_prepare_done(False) return # ---------- REINDEX ALL DATA TO CORRECT POINTGROUP ---------- # all should share the same pointgroup, unless twinned... in which # case force them to be... pointgroups = { } reindex_ops = { } probably_twinned = False need_to_return = False multi_sweep_indexing = \ PhilIndex.params.xia2.settings.developmental.multi_sweep_indexing if multi_sweep_indexing and not self._scalr_input_pointgroup: pointless_hklins = [] max_batches = 0 for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() md = self._factory.Mtzdump() md.set_hklin(hklin) md.dump() batches = md.get_batches() if 1 + max(batches) - min(batches) > max_batches: max_batches = max(batches) - min(batches) + 1 datasets = md.get_datasets() Debug.write('In reflection file %s found:' % hklin) for d in datasets: Debug.write('... %s' % d) dataset_info = md.get_dataset_info(datasets[0]) from xia2.lib.bits import nifty_power_of_ten Debug.write('Biggest sweep has %d batches' % max_batches) max_batches = nifty_power_of_ten(max_batches) counter = 0 for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() integrater = si.get_integrater() refiner = integrater.get_integrater_refiner() hklin = self._prepare_pointless_hklin( hklin, si.get_integrater().get_phi_width()) rb = self._factory.Rebatch() hklout = os.path.join(self.get_working_directory(), '%s_%s_%s_%s_prepointless.mtz' % \ (pname, xname, dname, si.get_sweep_name())) # we will want to delete this one exit FileHandler.record_temporary_file(hklout) first_batch = min(si.get_batches()) si.set_batch_offset(counter * max_batches - first_batch + 1) rb.set_hklin(hklin) rb.set_first_batch(counter * max_batches + 1) rb.set_project_info(pname, xname, dname) rb.set_hklout(hklout) new_batches = rb.rebatch() pointless_hklins.append(hklout) # update the counter & recycle counter += 1 s = self._factory.Sortmtz() pointless_hklin = os.path.join(self.get_working_directory(), '%s_%s_prepointless_sorted.mtz' % \ (self._scalr_pname, self._scalr_xname)) s.set_hklout(pointless_hklin) for hklin in pointless_hklins: s.add_hklin(hklin) s.sort() pointgroup, reindex_op, ntr, pt = \ self._pointless_indexer_jiffy( pointless_hklin, refiner) for epoch in self._sweep_handler.get_epochs(): pointgroups[epoch] = pointgroup reindex_ops[epoch] = reindex_op else: for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() #hklout = os.path.join( #self.get_working_directory(), #os.path.split(hklin)[-1].replace('.mtz', '_rdx.mtz')) #FileHandler.record_temporary_file(hklout) integrater = si.get_integrater() refiner = integrater.get_integrater_refiner() if self._scalr_input_pointgroup: Debug.write('Using input pointgroup: %s' % \ self._scalr_input_pointgroup) pointgroup = self._scalr_input_pointgroup reindex_op = 'h,k,l' pt = False else: pointless_hklin = self._prepare_pointless_hklin( hklin, si.get_integrater().get_phi_width()) pointgroup, reindex_op, ntr, pt = \ self._pointless_indexer_jiffy( pointless_hklin, refiner) Debug.write('X1698: %s: %s' % (pointgroup, reindex_op)) if ntr: integrater.integrater_reset_reindex_operator() need_to_return = True if pt and not probably_twinned: probably_twinned = True Debug.write('Pointgroup: %s (%s)' % (pointgroup, reindex_op)) pointgroups[epoch] = pointgroup reindex_ops[epoch] = reindex_op overall_pointgroup = None pointgroup_set = set([pointgroups[e] for e in pointgroups]) if len(pointgroup_set) > 1 and \ not probably_twinned: raise RuntimeError, 'non uniform pointgroups' if len(pointgroup_set) > 1: Debug.write('Probably twinned, pointgroups: %s' % \ ' '.join([p.replace(' ', '') for p in \ list(pointgroup_set)])) numbers = [Syminfo.spacegroup_name_to_number(s) for s in \ pointgroup_set] overall_pointgroup = Syminfo.spacegroup_number_to_name( min(numbers)) self._scalr_input_pointgroup = overall_pointgroup Chatter.write('Twinning detected, assume pointgroup %s' % \ overall_pointgroup) need_to_return = True else: overall_pointgroup = pointgroup_set.pop() for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) integrater = si.get_integrater() integrater.set_integrater_spacegroup_number( Syminfo.spacegroup_name_to_number(overall_pointgroup)) integrater.set_integrater_reindex_operator( reindex_ops[epoch], reason='setting point group') # This will give us the reflections in the correct point group si.set_reflections(integrater.get_integrater_intensities()) if need_to_return: self.set_scaler_done(False) self.set_scaler_prepare_done(False) return # in here now optinally work through the data files which should be # indexed with a consistent point group, and transform the orientation # matrices by the lattice symmetry operations (if possible) to get a # consistent definition of U matrix modulo fixed rotations if PhilIndex.params.xia2.settings.unify_setting: from scitbx.matrix import sqr reference_U = None i3 = sqr((1, 0, 0, 0, 1, 0, 0, 0, 1)) for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) intgr = si.get_integrater() fixed = sqr(intgr.get_goniometer().get_fixed_rotation()) u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections()) U = fixed.inverse() * sqr(u).transpose() B = sqr(b) if reference_U is None: reference_U = U continue results = [] for op in s.all_ops(): R = B * sqr(op.r().as_double()).transpose() * B.inverse() nearly_i3 = (U * R).inverse() * reference_U score = sum([abs(_n - _i) for (_n, _i) in zip(nearly_i3, i3)]) results.append((score, op.r().as_hkl(), op)) results.sort() best = results[0] Debug.write('Best reindex: %s %.3f' % (best[1], best[0])) intgr.set_integrater_reindex_operator(best[2].r().inverse().as_hkl(), reason='unifying [U] setting') si.set_reflections(intgr.get_integrater_intensities()) # recalculate to verify u, b, s = get_umat_bmat_lattice_symmetry_from_mtz(si.get_reflections()) U = fixed.inverse() * sqr(u).transpose() Debug.write('New reindex: %s' % (U.inverse() * reference_U)) # FIXME I should probably raise an exception at this stage if this # is not about I3... if self.get_scaler_reference_reflection_file(): self._reference = self.get_scaler_reference_reflection_file() Debug.write('Using HKLREF %s' % self._reference) elif Flags.get_reference_reflection_file(): self._reference = Flags.get_reference_reflection_file() Debug.write('Using HKLREF %s' % self._reference) params = PhilIndex.params use_brehm_diederichs = params.xia2.settings.use_brehm_diederichs if len(self._sweep_handler.get_epochs()) > 1 and use_brehm_diederichs: brehm_diederichs_files_in = [] for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() brehm_diederichs_files_in.append(hklin) # now run cctbx.brehm_diederichs to figure out the indexing hand for # each sweep from xia2.Wrappers.Cctbx.BrehmDiederichs import BrehmDiederichs from xia2.lib.bits import auto_logfiler brehm_diederichs = BrehmDiederichs() brehm_diederichs.set_working_directory(self.get_working_directory()) auto_logfiler(brehm_diederichs) brehm_diederichs.set_input_filenames(brehm_diederichs_files_in) # 1 or 3? 1 seems to work better? brehm_diederichs.set_asymmetric(1) brehm_diederichs.run() reindexing_dict = brehm_diederichs.get_reindexing_dict() for epoch in self._sweep_handler.get_epochs(): si = self._sweep_handler.get_sweep_information(epoch) intgr = si.get_integrater() hklin = si.get_reflections() reindex_op = reindexing_dict.get(os.path.abspath(hklin)) assert reindex_op is not None if 1 or reindex_op != 'h,k,l': # apply the reindexing operator intgr.set_integrater_reindex_operator( reindex_op, reason='match reference') si.set_reflections(intgr.get_integrater_intensities()) elif len(self._sweep_handler.get_epochs()) > 1 and \ not self._reference: first = self._sweep_handler.get_epochs()[0] si = self._sweep_handler.get_sweep_information(first) self._reference = si.get_reflections() if self._reference: md = self._factory.Mtzdump() md.set_hklin(self._reference) md.dump() if md.get_batches() and False: raise RuntimeError, 'reference reflection file %s unmerged' % \ self._reference datasets = md.get_datasets() if len(datasets) > 1 and False: raise RuntimeError, 'more than one dataset in %s' % \ self._reference # then get the unit cell, lattice etc. reference_lattice = Syminfo.get_lattice(md.get_spacegroup()) reference_cell = md.get_dataset_info(datasets[0])['cell'] # then compute the pointgroup from this... # ---------- REINDEX TO CORRECT (REFERENCE) SETTING ---------- for epoch in self._sweep_handler.get_epochs(): pl = self._factory.Pointless() si = self._sweep_handler.get_sweep_information(epoch) hklin = si.get_reflections() pl.set_hklin(self._prepare_pointless_hklin( hklin, si.get_integrater().get_phi_width())) hklout = os.path.join( self.get_working_directory(), '%s_rdx2.mtz' % os.path.split(hklin)[-1][:-4]) # we will want to delete this one exit FileHandler.record_temporary_file(hklout) # now set the initial reflection set as a reference... pl.set_hklref(self._reference) # write a pointless log file... pl.decide_pointgroup() Debug.write('Reindexing analysis of %s' % pl.get_hklin()) pointgroup = pl.get_pointgroup() reindex_op = pl.get_reindex_operator() Debug.write('Operator: %s' % reindex_op) # apply this... integrater = si.get_integrater() integrater.set_integrater_reindex_operator(reindex_op, reason='match reference') integrater.set_integrater_spacegroup_number( Syminfo.spacegroup_name_to_number(pointgroup)) si.set_reflections(integrater.get_integrater_intensities()) md = self._factory.Mtzdump() md.set_hklin(si.get_reflections()) md.dump() datasets = md.get_datasets() if len(datasets) > 1: raise RuntimeError, 'more than one dataset in %s' % \ si.get_reflections() # then get the unit cell, lattice etc. lattice = Syminfo.get_lattice(md.get_spacegroup()) cell = md.get_dataset_info(datasets[0])['cell'] if lattice != reference_lattice: raise RuntimeError, 'lattices differ in %s and %s' % \ (self._reference, si.get_reflections()) for j in range(6): if math.fabs((cell[j] - reference_cell[j]) / reference_cell[j]) > 0.1: raise RuntimeError, \ 'unit cell parameters differ in %s and %s' % \ (self._reference, si.get_reflections()) # ---------- SORT TOGETHER DATA ---------- self._sort_together_data_ccp4() self._scalr_resolution_limits = { } # store central resolution limit estimates batch_ranges = [self._sweep_handler.get_sweep_information( epoch).get_batch_range() for epoch in self._sweep_handler.get_epochs()] self._resolution_limit_estimates = erzatz_resolution( self._prepared_reflections, batch_ranges) return
def multi_crystal_analysis(stop_after=None): '''Actually process something...''' assert os.path.exists('xia2.json') from xia2.Schema.XProject import XProject xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() for crystal_id, crystal in crystals.iteritems(): cwd = os.path.abspath(os.curdir) working_directory = Environment.generate_directory( [crystal.get_name(), 'analysis']) os.chdir(working_directory) from xia2.Wrappers.CCP4.Blend import Blend from xia2.lib.bits import auto_logfiler hand_blender = Blend() hand_blender.set_working_directory(working_directory) auto_logfiler(hand_blender) Citations.cite('blend') scaler = crystal._get_scaler() #epoch_to_si = {} epoch_to_batches = {} epoch_to_integrated_intensities = {} epoch_to_sweep_name = {} try: epochs = scaler._sweep_information.keys() for epoch in epochs: si = scaler._sweep_information[epoch] epoch_to_batches[epoch] = si['batches'] epoch_to_integrated_intensities[epoch] = si['corrected_intensities'] epoch_to_sweep_name[epoch] = si['sname'] except AttributeError, e: epochs = scaler._sweep_handler.get_epochs() for epoch in epochs: si = scaler._sweep_handler.get_sweep_information(epoch) epoch_to_batches[epoch] = si.get_batches() epoch_to_integrated_intensities[epoch] = si.get_reflections() epoch_to_sweep_name[epoch] = si.get_sweep_name() unmerged_mtz = scaler.get_scaled_reflections('mtz_unmerged').values()[0] from iotbx.reflection_file_reader import any_reflection_file reader = any_reflection_file(unmerged_mtz) intensities = None batches = None assert reader.file_type() == 'ccp4_mtz' arrays = reader.as_miller_arrays(merge_equivalents=False) for ma in arrays: if ma.info().labels == ['BATCH']: batches = ma elif ma.info().labels == ['I', 'SIGI']: intensities = ma elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']: intensities = ma from xia2.Handlers.Environment import which Rscript_binary = which('Rscript', debug=False) if Rscript_binary is None: Chatter.write('Skipping BLEND analysis: Rscript not available') else: for epoch in epochs: hand_blender.add_hklin(epoch_to_integrated_intensities[epoch], label=epoch_to_sweep_name[epoch]) hand_blender.analysis() Chatter.write("Dendrogram saved to: %s" %hand_blender.get_dendrogram_file()) analysis = hand_blender.get_analysis() summary = hand_blender.get_summary() clusters = hand_blender.get_clusters() linkage_matrix = hand_blender.get_linkage_matrix() ddict = hand_blender.plot_dendrogram() rows = [] headers = ['Cluster', 'Datasets', 'Multiplicity', 'Completeness', 'LCV', 'aLCV'] completeness = flex.double() for i, cluster in clusters.iteritems(): sel_cluster = flex.bool(batches.size(), False) for j in cluster['dataset_ids']: batch_start, batch_end = epoch_to_batches[epochs[j-1]] sel_cluster |= ( (batches.data() >= batch_start) & (batches.data() <= batch_end)) intensities_cluster = intensities.select(sel_cluster) merging = intensities_cluster.merge_equivalents() merged_intensities = merging.array() multiplicities = merging.redundancies() completeness.append(merged_intensities.completeness()) dataset_ids = cluster['dataset_ids'] rows.append( ['%i' %i, ' '.join(['%i'] * len(dataset_ids)) %tuple(dataset_ids), '%.1f' %flex.mean(multiplicities.data().as_double()), '%.2f' %completeness[-1], '%.2f' %cluster['lcv'], '%.2f' %cluster['alcv']]) # sort table by completeness perm = flex.sort_permutation(completeness) rows = [rows[i] for i in perm] print print 'Unit cell clustering summary:' print tabulate(rows, headers, tablefmt='rst') print blend_html = tabulate(rows, headers, tablefmt='html').replace( '<table>', '<table class="table table-hover table-condensed">').replace( '<td>', '<td style="text-align: right;">')
def _integrate_prepare(self): '''Prepare for integration - in XDS terms this may mean rerunning IDXREF to get the XPARM etc. DEFPIX is considered part of the full integration as it is resolution dependent.''' from xia2.Handlers.Citations import Citations Citations.cite('xds') # decide what images we are going to process, if not already # specified if not self._intgr_wedge: images = self.get_matching_images() self.set_integrater_wedge(min(images), max(images)) Debug.write('XDS INTEGRATE PREPARE:') Debug.write('Wavelength: %.6f' % self.get_wavelength()) Debug.write('Distance: %.2f' % self.get_distance()) idxr = self._intgr_refiner.get_refiner_indexer(self.get_integrater_epoch()) if idxr is None: idxr = XDSIndexer() self._intgr_refiner.add_refiner_indexer( self.get_integrater_epoch(), idxr) self.set_integrater_prepare_done(False) #self.set_integrater_indexer() idxr.set_indexer_sweep(self.get_integrater_sweep()) idxr.set_working_directory(self.get_working_directory()) idxr.setup_from_imageset(self.get_imageset()) if self.get_frame_wedge(): wedge = self.get_frame_wedge() Debug.write('Propogating wedge limit: %d %d' % wedge) idxr.set_frame_wedge(wedge[0], wedge[1], apply_offset = False) # this needs to be set up from the contents of the # Integrater frame processer - wavelength &c. if self.get_beam_centre(): idxr.set_beam_centre(self.get_beam_centre()) if self.get_distance(): idxr.set_distance(self.get_distance()) if self.get_wavelength(): idxr.set_wavelength(self.get_wavelength()) # get the unit cell from this indexer to initiate processing # if it is new... and also copy out all of the information for # the XDS indexer if not... # copy the data across self._xds_data_files = copy.deepcopy( self._intgr_refiner.get_refiner_payload(self.get_integrater_epoch())) Debug.write('Files available at the end of XDS integrate prepare:') for f in self._xds_data_files.keys(): Debug.write('%s' % f) experiment = self._intgr_refiner.get_refined_experiment_list( self.get_integrater_epoch())[0] # copy across the trusted_range - it got lost along the way old_detector = self.get_detector() self.set_detector(experiment.detector) for p1, p2 in zip(old_detector, self.get_detector()): p2.set_trusted_range(p1.get_trusted_range()) self.set_beam_obj(experiment.beam) self.set_goniometer(experiment.goniometer) # set a low resolution limit (which isn't really used...) # this should perhaps be done more intelligently from an # analysis of the spot list or something...? if not self.get_integrater_low_resolution(): dmax = self._intgr_refiner.get_indexer_low_resolution( self.get_integrater_epoch()) self.set_integrater_low_resolution(dmax) Debug.write('Low resolution set to: %s' % \ self.get_integrater_low_resolution()) # delete things we should not know e.g. the postrefined cell from # CORRECT - c/f bug # 2695 self._intgr_cell = None self._intgr_spacegroup_number = None return
def TwoThetaRefine(DriverType = None): '''A factory for RefineWrapper classes.''' from xia2.Handlers.Citations import Citations Citations.cite('dials') from xia2.Driver.DriverFactory import DriverFactory DriverInstance = DriverFactory.Driver(DriverType) class RefineWrapper(DriverInstance.__class__): def __init__(self): DriverInstance.__class__.__init__(self) self.set_executable('dials.two_theta_refine') self._reindexing_operator = None self._reindexed_experiments = None self._reindexed_reflections = None self._experiments = [] self._pickles = [] self._phil_file = None # The following are set during run() call: self._output_cif = None self._output_mmcif = None self._output_correlation_plot = None self._output_experiments = None self._crystal = None def set_experiments(self, experiments): self._experiments = experiments def get_experiments(self): return self._experiments def set_pickles(self, pickles): self._pickles = pickles def get_pickles(self): return self._pickles def set_phil_file(self, phil_file): self._phil_file = phil_file def get_output_cif(self): return self._output_cif def get_output_mmcif(self): return self._output_mmcif def get_output_correlation_plot(self): return self._output_correlation_plot def get_output_experiments(self): return self._output_experiments def get_unit_cell(self): return self._crystal.get_unit_cell().parameters() def get_unit_cell_esd(self): return self._crystal.get_cell_parameter_sd() def set_reindex_operator(self, operator): self._reindexing_operator = operator def run(self): from xia2.Handlers.Streams import Chatter, Debug if self._reindexing_operator: Debug.write('Reindexing sweeps for dials.two_theta_refine') from xia2.lib.bits import auto_logfiler from xia2.Wrappers.Dials.Reindex import Reindex self._reindexed_experiments, self._reindexed_reflections = [], [] for e, p in zip(self._experiments, self._pickles): reindexer = Reindex() reindexer.set_cb_op(self._reindexing_operator) reindexer.set_experiments_filename(e) reindexer.set_indexed_filename(p) reindexer.set_working_directory(self.get_working_directory()) auto_logfiler(reindexer) reindexer.run() self._reindexed_experiments.append(reindexer.get_reindexed_experiments_filename()) self._reindexed_reflections.append(reindexer.get_reindexed_reflections_filename()) Debug.write('Running dials.two_theta_refine') self._output_cif = os.path.join( self.get_working_directory(), '%s_dials.two_theta_refine.cif' % self.get_xpid()) self._output_mmcif = os.path.join( self.get_working_directory(), '%s_dials.two_theta_refine.mmcif' % self.get_xpid()) self._output_correlation_plot = os.path.join( self.get_working_directory(), '%s_dials.two_theta_refine.png' % self.get_xpid()) self._output_experiments = os.path.join( self.get_working_directory(), '%s_refined_cell.json' % self.get_xpid()) self.clear_command_line() if self._reindexing_operator: for experiment in self._reindexed_experiments: self.add_command_line(experiment) for pickle in self._reindexed_reflections: self.add_command_line(pickle) else: for experiment in self._experiments: self.add_command_line(experiment) for pickle in self._pickles: self.add_command_line(pickle) self.add_command_line('output.cif=%s' % self._output_cif) self.add_command_line('output.mmcif=%s' % self._output_mmcif) if self._output_correlation_plot is not None: self.add_command_line( 'output.correlation_plot.filename=%s' % self._output_correlation_plot) if self._output_experiments is not None: self.add_command_line( 'output.experiments=%s' % self._output_experiments) if self._phil_file is not None: self.add_command_line('%s' %self._phil_file) self.start() self.close_wait() if not os.path.isfile(self._output_cif): Chatter.write( "TwoTheta refinement failed, see log file for more details:\n %s" % self.get_log_file()) raise RuntimeError, 'unit cell not refined' self.check_for_errors() from dxtbx.model.experiment.experiment_list import ExperimentListFactory experiments = ExperimentListFactory.from_json_file(self.get_output_experiments()) self._crystal = experiments.crystals()[0] def import_cif(self): '''Import relevant lines from .cif output''' import iotbx.cif cif = iotbx.cif.reader(file_path=self.get_output_cif()).model() block = cif['two_theta_refine'] subset = { k: block[k] for k in block.keys() if k.startswith(('_cell', '_diffrn')) } return subset def import_mmcif(self): '''Import relevant lines from .mmcif output''' if os.path.isfile(self.get_output_mmcif()): import iotbx.cif cif = iotbx.cif.reader(file_path=self.get_output_mmcif()).model() block = cif['two_theta_refine'] else: import iotbx.cif.model block = iotbx.cif.model.block() subset = { k: block[k] for k in block.keys() if k.startswith(('_cell', '_diffrn')) } return subset return RefineWrapper()
def generate_xia2_html(xinfo, filename='xia2.html', params=None, args=[]): assert params is None or len(args) == 0 if params is None: from xia2.Modules.Analysis import phil_scope interp = phil_scope.command_line_argument_interpreter() params, unhandled = interp.process_and_fetch( args, custom_processor='collect_remaining') params = params.extract() from xia2.command_line.report import xia2_report crystal = xinfo.get_crystals().values()[0] xia2_txt = os.path.join(os.path.abspath(os.path.curdir), 'xia2.txt') assert os.path.isfile(xia2_txt), xia2_txt with open(xia2_txt, 'rb') as f: xia2_output = f.read().encode('ascii', 'xmlcharrefreplace') xia2_output = cgi.escape(xia2_output) styles = {} reports = [] columns = [] columns.append([ '', u'Wavelength (Å)', u'Resolution range (Å)', 'Completeness (%)', 'Multiplicity', 'CC-half', 'I/sigma', 'Rmerge(I)', # anomalous statistics 'Anomalous completeness (%)', 'Anomalous multiplicity' ]) individual_dataset_reports = {} for cname, xcryst in xinfo.get_crystals().iteritems(): reflection_files = xcryst.get_scaled_merged_reflections() for wname, unmerged_mtz in reflection_files['mtz_unmerged'].iteritems( ): xwav = xcryst.get_xwavelength(wname) report = xia2_report(unmerged_mtz, params) reports.append(report) merging_stats = report.merging_stats merging_stats_anom = report.merging_stats_anom overall = merging_stats.overall overall_anom = merging_stats_anom.overall outer_shell = merging_stats.bins[-1] outer_shell_anom = merging_stats_anom.bins[-1] column = [ wname, str(xwav.get_wavelength()), '%.2f - %.2f (%.2f - %.2f)' % (overall.d_max, overall.d_min, outer_shell.d_max, outer_shell.d_min), '%.2f (%.2f)' % (overall.completeness * 100, outer_shell.completeness * 100), '%.2f (%.2f)' % (overall.mean_redundancy, outer_shell.mean_redundancy), '%.4f (%.4f)' % (overall.cc_one_half, outer_shell.cc_one_half), '%.2f (%.2f)' % (overall.i_over_sigma_mean, outer_shell.i_over_sigma_mean), '%.4f (%.4f)' % (overall.r_merge, outer_shell.r_merge), # anomalous statistics '%.2f (%.2f)' % (overall_anom.anom_completeness * 100, outer_shell_anom.anom_completeness * 100), '%.2f (%.2f)' % (overall_anom.mean_redundancy, outer_shell_anom.mean_redundancy), ] columns.append(column) xtriage_success, xtriage_warnings, xtriage_danger = None, None, None if params.xtriage_analysis: try: xtriage_success, xtriage_warnings, xtriage_danger = report.xtriage_report( ) except Exception as e: from xia2.Handlers.Phil import PhilIndex if PhilIndex.params.xia2.settings.small_molecule == True: print("Xtriage output not available: %s" % str(e)) else: raise d = {} d['merging_statistics_table'] = report.merging_statistics_table() d['overall_statistics_table'] = report.overall_statistics_table() individual_dataset_reports[wname] = d json_data = {} json_data.update(report.multiplicity_vs_resolution_plot()) json_data.update(report.multiplicity_histogram()) json_data.update(report.completeness_plot()) json_data.update(report.scale_rmerge_vs_batch_plot()) json_data.update(report.cc_one_half_plot()) json_data.update(report.i_over_sig_i_plot()) json_data.update(report.i_over_sig_i_vs_batch_plot()) json_data.update(report.second_moments_plot()) json_data.update(report.cumulative_intensity_distribution_plot()) json_data.update(report.l_test_plot()) json_data.update(report.wilson_plot()) json_data.update(report.pychef_plots(n_bins=1)) from scitbx.array_family import flex max_points = 500 for g in ('scale_rmerge_vs_batch', 'completeness_vs_dose', 'rcp_vs_dose', 'scp_vs_dose', 'rd_vs_batch_difference'): for i, data in enumerate(json_data[g]['data']): x = data['x'] n = len(x) if n > max_points: step = n // max_points sel = (flex.int_range(n) % step) == 0 data['x'] = list(flex.int(data['x']).select(sel)) data['y'] = list(flex.double(data['y']).select(sel)) resolution_graphs = collections.OrderedDict( (k + '_' + wname, json_data[k]) for k in ('cc_one_half', 'i_over_sig_i', 'second_moments', 'wilson_intensity_plot', 'completeness', 'multiplicity_vs_resolution') if k in json_data) if params.include_radiation_damage: batch_graphs = collections.OrderedDict( (k + '_' + wname, json_data[k]) for k in ('scale_rmerge_vs_batch', 'i_over_sig_i_vs_batch', 'completeness_vs_dose', 'rcp_vs_dose', 'scp_vs_dose', 'rd_vs_batch_difference')) else: batch_graphs = collections.OrderedDict( (k + '_' + wname, json_data[k]) for k in ('scale_rmerge_vs_batch', 'i_over_sig_i_vs_batch')) misc_graphs = collections.OrderedDict( (k + '_' + wname, json_data[k]) for k in ('cumulative_intensity_distribution', 'l_test', 'multiplicities') if k in json_data) for k, v in report.multiplicity_plots().iteritems(): misc_graphs[k + '_' + wname] = {'img': v} d['resolution_graphs'] = resolution_graphs d['batch_graphs'] = batch_graphs d['misc_graphs'] = misc_graphs d['xtriage'] = { 'success': xtriage_success, 'warnings': xtriage_warnings, 'danger': xtriage_danger } table = [[c[i] for c in columns] for i in range(len(columns[0]))] cell = xcryst.get_cell() from cctbx import sgtbx space_groups = xcryst.get_likely_spacegroups() space_groups = [ sgtbx.space_group_info(symbol=str(symbol)) for symbol in space_groups ] space_group = space_groups[0].symbol_and_number() alternative_space_groups = [ sg.symbol_and_number() for sg in space_groups[1:] ] unit_cell = str(report.intensities.unit_cell()) #twinning_score = xcryst._get_scaler()._scalr_twinning_score #twinning_conclusion = xcryst._get_scaler()._scalr_twinning_conclusion #if twinning_score is not None: #table.append(['','','']) #table.append(['Twinning score', '%.2f' %twinning_score, '']) #if twinning_conclusion is not None: #table.append(['', twinning_conclusion, '']) for row in table: for i in range(len(row)): row[i] = row[i].encode('ascii', 'xmlcharrefreplace') #from libtbx import table_utils #print table_utils.format(rows=table, has_header=True) # reflection files for cname, xcryst in xinfo.get_crystals().iteritems(): # hack to replace path to reflection files with DataFiles directory data_dir = os.path.join(os.path.abspath(os.path.curdir), 'DataFiles') g = glob.glob(os.path.join(data_dir, '*')) reflection_files = xcryst.get_scaled_merged_reflections() for k, rfile in reflection_files.iteritems(): if isinstance(rfile, basestring): for datafile in g: if os.path.basename(datafile) == os.path.basename(rfile): reflection_files[k] = datafile break else: for kk in rfile.keys(): for datafile in g: if os.path.basename(datafile) == os.path.basename( rfile[kk]): reflection_files[k][kk] = datafile break headers = ['Dataset', 'File name'] merged_mtz = reflection_files['mtz'] mtz_files = [ headers, [ 'All datasets', '<a href="%s">%s</a>' % (os.path.relpath(merged_mtz), os.path.basename(merged_mtz)) ] ] for wname, unmerged_mtz in reflection_files['mtz_unmerged'].iteritems( ): mtz_files.append([ wname, '<a href="%s">%s</a>' % (os.path.relpath(unmerged_mtz), os.path.basename(unmerged_mtz)) ]) sca_files = [headers] for wname, merged_sca in reflection_files['sca'].iteritems(): sca_files.append([ wname, '<a href="%s">%s</a>' % (os.path.relpath(merged_sca), os.path.basename(merged_sca)) ]) unmerged_sca_files = [headers] for wname, unmerged_sca in reflection_files['sca_unmerged'].iteritems( ): unmerged_sca_files.append([ wname, '<a href="%s">%s</a>' % (os.path.relpath(unmerged_sca), os.path.basename(unmerged_sca)) ]) # other files other_files = [] other_files.append(['File name', 'Description']) for other_file, description in sorted([ ('xia2.cif', 'Crystallographic information file'), ('xia2.mmcif', 'Macromolecular crystallographic information file'), ('shelxt.hkl', 'merged structure factors for SHELXT'), ('shelxt.ins', 'SHELXT instruction file'), ] + [ (fn, 'XPREP input file') for fn in os.listdir(os.path.join(data_dir)) \ if fn.endswith('.p4p') ]): if os.path.exists(os.path.join(data_dir, other_file)): other_files.append([ '<a href="DataFiles/{filename}">{filename}</a>'.format( filename=other_file), description ]) # log files log_files_table = [] log_dir = os.path.join(os.path.abspath(os.path.curdir), 'LogFiles') g = glob.glob(os.path.join(log_dir, '*.log')) for logfile in g: html_file = make_logfile_html(logfile) html_file = os.path.splitext(logfile)[0] + '.html' if os.path.exists(html_file): log_files_table.append([ os.path.basename(logfile), '<a href="%s">original</a>' % os.path.relpath(logfile), '<a href="%s">html</a>' % os.path.relpath(html_file), ]) else: log_files_table.append([ os.path.basename(logfile), '<a href="%s">original</a>' % os.path.relpath(logfile), ' ', ]) # references references = {} for cdict in Citations.get_citations_dicts(): references[cdict['acta']] = cdict.get('url') from jinja2 import Environment, ChoiceLoader, PackageLoader loader = ChoiceLoader([ PackageLoader('xia2', 'templates'), PackageLoader('dials', 'templates') ]) env = Environment(loader=loader) template = env.get_template('xia2.html') html = template.render( page_title='xia2 processing report', xia2_output=xia2_output, space_group=space_group, alternative_space_groups=alternative_space_groups, unit_cell=unit_cell, xtriage_success=xtriage_success, xtriage_warnings=xtriage_warnings, xtriage_danger=xtriage_danger, overall_stats_table=table, cc_half_significance_level=params.cc_half_significance_level, mtz_files=mtz_files, sca_files=sca_files, unmerged_sca_files=unmerged_sca_files, other_files=other_files, log_files_table=log_files_table, individual_dataset_reports=individual_dataset_reports, references=references, styles=styles) with open(filename, 'wb') as f: f.write(html.encode('ascii', 'xmlcharrefreplace'))
def _integrate(self): """Implement the integrater interface.""" # cite the program Citations.cite("mosflm") images_str = "%d to %d" % tuple(self._intgr_wedge) cell_str = "%.2f %.2f %.2f %.2f %.2f %.2f" % tuple(self._intgr_cell) if len(self._fp_directory) <= 50: dirname = self._fp_directory else: dirname = "...%s" % self._fp_directory[-46:] Journal.block( "integrating", self._intgr_sweep_name, "mosflm", { "images": images_str, "cell": cell_str, "lattice": self.get_integrater_refiner().get_refiner_lattice(), "template": self._fp_template, "directory": dirname, "resolution": "%.2f" % self._intgr_reso_high, }, ) self._mosflm_rerun_integration = False wd = self.get_working_directory() try: if self.get_integrater_sweep_name(): pname, xname, dname = self.get_integrater_project_info() nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc if nproc > 1: Debug.write("Parallel integration: %d jobs" % nproc) self._mosflm_hklout = self._mosflm_parallel_integrate() else: self._mosflm_hklout = self._mosflm_integrate() # record integration output for e.g. BLEND. sweep = self.get_integrater_sweep_name() if sweep: FileHandler.record_more_data_file( "%s %s %s %s INTEGRATE" % (pname, xname, dname, sweep), self._mosflm_hklout, ) except IntegrationError as e: if "negative mosaic spread" in str(e): if self._mosflm_postref_fix_mosaic: Chatter.write( "Negative mosaic spread - stopping integration") raise BadLatticeError("negative mosaic spread") Chatter.write("Negative mosaic spread - rerunning integration") self.set_integrater_done(False) self._mosflm_postref_fix_mosaic = True if self._mosflm_rerun_integration and not PhilIndex.params.dials.fast_mode: # make sure that this is run again... Chatter.write("Need to rerun the integration...") self.set_integrater_done(False) return self._mosflm_hklout
def _index_prepare(self): from xia2.Handlers.Citations import Citations Citations.cite('dials') #all_images = self.get_matching_images() #first = min(all_images) #last = max(all_images) spot_lists = [] datablocks = [] for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps): Chatter.banner('Spotfinding %s' % xsweep.get_name()) first, last = imageset.get_scan().get_image_range() # at this stage, break out to run the DIALS code: this sets itself up # now cheat and pass in some information... save re-reading all of the # image headers # FIXME need to adjust this to allow (say) three chunks of images from dxtbx.serialize import dump from dxtbx.datablock import DataBlock sweep_filename = os.path.join( self.get_working_directory(), '%s_datablock.json' % xsweep.get_name()) dump.datablock(DataBlock([imageset]), sweep_filename) genmask = self.GenerateMask() genmask.set_input_datablock(sweep_filename) genmask.set_output_datablock( os.path.join( self.get_working_directory(), '%s_%s_datablock.json' % (genmask.get_xpid(), xsweep.get_name()))) genmask.set_params(PhilIndex.params.dials.masking) sweep_filename, mask_pickle = genmask.run() Debug.write('Generated mask for %s: %s' % (xsweep.get_name(), mask_pickle)) gain = PhilIndex.params.xia2.settings.input.gain if gain is libtbx.Auto: gain_estimater = self.EstimateGain() gain_estimater.set_sweep_filename(sweep_filename) gain_estimater.run() gain = gain_estimater.get_gain() Chatter.write('Estimated gain: %.2f' % gain) PhilIndex.params.xia2.settings.input.gain = gain # FIXME this should really use the assigned spot finding regions #offset = self.get_frame_offset() spotfinder = self.Spotfinder() if last - first > 10: spotfinder.set_write_hot_mask(True) spotfinder.set_input_sweep_filename(sweep_filename) spotfinder.set_output_sweep_filename( '%s_%s_datablock.json' % (spotfinder.get_xpid(), xsweep.get_name())) spotfinder.set_input_spot_filename( '%s_%s_strong.pickle' % (spotfinder.get_xpid(), xsweep.get_name())) if PhilIndex.params.dials.fast_mode: wedges = self._index_select_images_i(imageset) spotfinder.set_scan_ranges(wedges) else: spotfinder.set_scan_ranges([(first, last)]) if PhilIndex.params.dials.find_spots.phil_file is not None: spotfinder.set_phil_file( PhilIndex.params.dials.find_spots.phil_file) min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size if min_spot_size is libtbx.Auto: if imageset.get_detector()[0].get_type() == 'SENSOR_PAD': min_spot_size = 3 else: min_spot_size = None if min_spot_size is not None: spotfinder.set_min_spot_size(min_spot_size) min_local = PhilIndex.params.dials.find_spots.min_local if min_local is not None: spotfinder.set_min_local(min_local) sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong if sigma_strong: spotfinder.set_sigma_strong(sigma_strong) gain = PhilIndex.params.xia2.settings.input.gain if gain: spotfinder.set_gain(gain) filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings if filter_ice_rings: spotfinder.set_filter_ice_rings(filter_ice_rings) kernel_size = PhilIndex.params.dials.find_spots.kernel_size if kernel_size: spotfinder.set_kernel_size(kernel_size) global_threshold = PhilIndex.params.dials.find_spots.global_threshold if global_threshold is not None: spotfinder.set_global_threshold(global_threshold) spotfinder.run() spot_filename = spotfinder.get_spot_filename() if not os.path.exists(spot_filename): raise RuntimeError("Spotfinding failed: %s does not exist." % os.path.basename(spot_filename)) spot_lists.append(spot_filename) datablocks.append(spotfinder.get_output_sweep_filename()) from libtbx import easy_pickle from dials.util.ascii_art import spot_counts_per_image_plot refl = easy_pickle.load(spot_filename) if not len(refl): raise RuntimeError('No spots found in sweep %s' % xsweep.get_name()) Chatter.write(spot_counts_per_image_plot(refl), strip=False) if not PhilIndex.params.dials.fast_mode: detectblanks = self.DetectBlanks() detectblanks.set_sweep_filename(datablocks[-1]) detectblanks.set_reflections_filename(spot_filename) detectblanks.run() json = detectblanks.get_results() offset = imageset.get_scan().get_image_range()[0] blank_regions = json['strong']['blank_regions'] if len(blank_regions): blank_regions = [(int(s), int(e)) for s, e in blank_regions] for blank_start, blank_end in blank_regions: Chatter.write( 'WARNING: Potential blank images: %i -> %i' % (blank_start + 1, blank_end)) if PhilIndex.params.xia2.settings.remove_blanks: non_blanks = [] start, end = imageset.get_array_range() last_blank_end = start for blank_start, blank_end in blank_regions: if blank_start > start: non_blanks.append( (last_blank_end, blank_start)) last_blank_end = blank_end if last_blank_end + 1 < end: non_blanks.append((last_blank_end, end)) xsweep = self.get_indexer_sweep() xwav = xsweep.get_wavelength() xsample = xsweep.get_xsample() sweep_name = xsweep.get_name() import string for i, (nb_start, nb_end) in enumerate(non_blanks): assert i < 26 if i == 0: sub_imageset = imageset[nb_start - start:nb_end - start] xsweep._frames_to_process = (nb_start + 1, nb_end + 1) self.set_indexer_prepare_done(done=False) self._indxr_imagesets[ self._indxr_imagesets.index( imageset)] = sub_imageset xsweep._integrater._setup_from_imageset( sub_imageset) else: new_name = '_'.join( (sweep_name, string.ascii_lowercase[i])) new_sweep = xwav.add_sweep( new_name, xsample, directory=os.path.join( os.path.basename( xsweep.get_directory()), new_name), image=imageset.get_path(nb_start - start), frames_to_process=(nb_start + 1, nb_end), ) Chatter.write( "Generating new sweep: %s (%s:%i:%i)" % (new_sweep.get_name(), new_sweep.get_image(), new_sweep.get_frames_to_process()[0], new_sweep.get_frames_to_process()[1])) return if not PhilIndex.params.xia2.settings.trust_beam_centre: discovery = self.DiscoverBetterExperimentalModel() discovery.set_sweep_filename(datablocks[-1]) discovery.set_spot_filename(spot_filename) #wedges = self._index_select_images_i(imageset) #discovery.set_scan_ranges(wedges) #discovery.set_scan_ranges([(first + offset, last + offset)]) try: discovery.run() except Exception as e: Debug.write('DIALS beam centre search failed: %s' % str(e)) else: # overwrite datablock.json in datablocks list datablocks[ -1] = discovery.get_optimized_datablock_filename() self.set_indexer_payload("spot_lists", spot_lists) self.set_indexer_payload("datablocks", datablocks) return
def _integrate(self): '''Implement the integrater interface.''' # cite the program Citations.cite('mosflm') images_str = '%d to %d' % tuple(self._intgr_wedge) cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % tuple(self._intgr_cell) if len(self._fp_directory) <= 50: dirname = self._fp_directory else: dirname = '...%s' % self._fp_directory[-46:] Journal.block( 'integrating', self._intgr_sweep_name, 'mosflm', { 'images': images_str, 'cell': cell_str, 'lattice': self.get_integrater_refiner().get_refiner_lattice(), 'template': self._fp_template, 'directory': dirname, 'resolution': '%.2f' % self._intgr_reso_high }) self._mosflm_rerun_integration = False wd = self.get_working_directory() try: if self.get_integrater_sweep_name(): pname, xname, dname = self.get_integrater_project_info() nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc if nproc > 1: Debug.write('Parallel integration: %d jobs' % nproc) self._mosflm_hklout = self._mosflm_parallel_integrate() else: self._mosflm_hklout = self._mosflm_integrate() # record integration output for e.g. BLEND. sweep = self.get_integrater_sweep_name() if sweep: FileHandler.record_more_data_file( '%s %s %s %s INTEGRATE' % (pname, xname, dname, sweep), self._mosflm_hklout) except IntegrationError as e: if 'negative mosaic spread' in str(e): if self._mosflm_postref_fix_mosaic: Chatter.write( 'Negative mosaic spread - stopping integration') raise BadLatticeError('negative mosaic spread') Chatter.write('Negative mosaic spread - rerunning integration') self.set_integrater_done(False) self._mosflm_postref_fix_mosaic = True if self._mosflm_rerun_integration and not PhilIndex.params.dials.fast_mode: # make sure that this is run again... Chatter.write('Need to rerun the integration...') self.set_integrater_done(False) return self._mosflm_hklout
def _integrate_prepare(self): """Prepare for integration - in XDS terms this may mean rerunning IDXREF to get the XPARM etc. DEFPIX is considered part of the full integration as it is resolution dependent.""" Citations.cite("xds") # decide what images we are going to process, if not already # specified if not self._intgr_wedge: images = self.get_matching_images() self.set_integrater_wedge( min(images) + self.get_frame_offset(), max(images) + self.get_frame_offset(), ) logger.debug("XDS INTEGRATE PREPARE:") logger.debug("Wavelength: %.6f" % self.get_wavelength()) logger.debug("Distance: %.2f" % self.get_distance()) idxr = self._intgr_refiner.get_refiner_indexer( self.get_integrater_epoch()) if idxr is None: idxr = XDSIndexer() self._intgr_refiner.add_refiner_indexer( self.get_integrater_epoch(), idxr) self.set_integrater_prepare_done(False) # self.set_integrater_indexer() idxr.set_indexer_sweep(self.get_integrater_sweep()) idxr.set_working_directory(self.get_working_directory()) idxr.setup_from_imageset(self.get_imageset()) if self.get_frame_wedge(): wedge = self.get_frame_wedge() logger.debug("Propogating wedge limit: %d %d" % wedge) idxr.set_frame_wedge(wedge[0], wedge[1], apply_offset=False) # this needs to be set up from the contents of the # Integrater frame processer - wavelength &c. if self.get_beam_centre(): idxr.set_beam_centre(self.get_beam_centre()) if self.get_distance(): idxr.set_distance(self.get_distance()) if self.get_wavelength(): idxr.set_wavelength(self.get_wavelength()) # get the unit cell from this indexer to initiate processing # if it is new... and also copy out all of the information for # the XDS indexer if not... # copy the data across self._xds_data_files = copy.deepcopy( self._intgr_refiner.get_refiner_payload( self.get_integrater_epoch())) if self._xds_data_files is None: self._xds_data_files = {} logger.debug("Files available at the end of XDS integrate prepare:") for f in self._xds_data_files: logger.debug("%s" % f) experiment = self._intgr_refiner.get_refined_experiment_list( self.get_integrater_epoch())[0] # copy across the trusted_range - it got lost along the way old_detector = self.get_detector() self.set_detector(experiment.detector) for p1, p2 in zip(old_detector, self.get_detector()): p2.set_trusted_range(p1.get_trusted_range()) self.set_beam_obj(experiment.beam) self.set_goniometer(experiment.goniometer) # set a low resolution limit (which isn't really used...) # this should perhaps be done more intelligently from an # analysis of the spot list or something...? if not self.get_integrater_low_resolution(): dmax = self._intgr_refiner.get_indexer_low_resolution( self.get_integrater_epoch()) self.set_integrater_low_resolution(dmax) logger.debug("Low resolution set to: %s" % self.get_integrater_low_resolution()) # delete things we should not know e.g. the postrefined cell from # CORRECT - c/f bug # 2695 self._intgr_cell = None self._intgr_spacegroup_number = None
def xia2_main(stop_after=None): '''Actually process something...''' Citations.cite('xia2') # print versions of related software from dials.util.version import dials_version Chatter.write(dials_version()) start_time = time.time() CommandLine = get_command_line() start_dir = Flags.get_starting_directory() # check that something useful has been assigned for processing... xtals = CommandLine.get_xinfo().get_crystals() no_images = True for name in xtals.keys(): xtal = xtals[name] if not xtal.get_all_image_names(): Chatter.write('-----------------------------------' + \ '-' * len(name)) Chatter.write('| No images assigned for crystal %s |' % name) Chatter.write('-----------------------------------' + '-' \ * len(name)) else: no_images = False args = [] from xia2.Handlers.Phil import PhilIndex params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing njob = mp_params.njob from libtbx import group_args xinfo = CommandLine.get_xinfo() if os.path.exists('xia2.json'): from xia2.Schema.XProject import XProject xinfo_new = xinfo xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() crystals_new = xinfo_new.get_crystals() for crystal_id in crystals_new.keys(): if crystal_id not in crystals: crystals[crystal_id] = crystals_new[crystal_id] continue crystals[crystal_id]._scaler = None # reset scaler for wavelength_id in crystals_new[crystal_id].get_wavelength_names(): wavelength_new = crystals_new[crystal_id].get_xwavelength(wavelength_id) if wavelength_id not in crystals[crystal_id].get_wavelength_names(): crystals[crystal_id].add_wavelength( crystals_new[crystal_id].get_xwavelength(wavelength_new)) continue wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) sweeps_new = wavelength_new.get_sweeps() sweeps = wavelength.get_sweeps() sweep_names = [s.get_name() for s in sweeps] sweep_keys = [ (s.get_directory(), s.get_template(), s.get_image_range()) for s in sweeps] for sweep in sweeps_new: if ((sweep.get_directory(), sweep.get_template(), sweep.get_image_range()) not in sweep_keys): if sweep.get_name() in sweep_names: i = 1 while 'SWEEEP%i' %i in sweep_names: i += 1 sweep._name = 'SWEEP%i' %i break wavelength.add_sweep( name=sweep.get_name(), directory=sweep.get_directory(), image=sweep.get_image(), beam=sweep.get_beam_centre(), reversephi=sweep.get_reversephi(), distance=sweep.get_distance(), gain=sweep.get_gain(), dmin=sweep.get_resolution_high(), dmax=sweep.get_resolution_low(), polarization=sweep.get_polarization(), frames_to_process=sweep.get_frames_to_process(), user_lattice=sweep.get_user_lattice(), user_cell=sweep.get_user_cell(), epoch=sweep._epoch, ice=sweep._ice, excluded_regions=sweep._excluded_regions, ) sweep_names.append(sweep.get_name()) crystals = xinfo.get_crystals() failover = params.xia2.settings.failover if njob > 1: driver_type = mp_params.type command_line_args = CommandLine.get_argv()[1:] for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) sweeps = wavelength.get_sweeps() for sweep in sweeps: sweep._get_indexer() sweep._get_refiner() sweep._get_integrater() args.append(( group_args( driver_type=driver_type, stop_after=stop_after, failover=failover, command_line_args=command_line_args, nproc=mp_params.nproc, crystal_id=crystal_id, wavelength_id=wavelength_id, sweep_id=sweep.get_name(), ),)) from xia2.Driver.DriverFactory import DriverFactory default_driver_type = DriverFactory.get_driver_type() # run every nth job on the current computer (no need to submit to qsub) for i_job, arg in enumerate(args): if (i_job % njob) == 0: arg[0].driver_type = default_driver_type if mp_params.type == "qsub": method = "sge" else: method = "multiprocessing" nproc = mp_params.nproc qsub_command = mp_params.qsub_command if not qsub_command: qsub_command = 'qsub' qsub_command = '%s -V -cwd -pe smp %d' %(qsub_command, nproc) from libtbx import easy_mp results = easy_mp.parallel_map( process_one_sweep, args, processes=njob, #method=method, method="multiprocessing", qsub_command=qsub_command, preserve_order=True, preserve_exception_message=True) # Hack to update sweep with the serialized indexers/refiners/integraters i_sweep = 0 for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: success, output, xsweep_dict = results[i_sweep] assert xsweep_dict is not None if output is not None: Chatter.write(output) if not success: Chatter.write('Sweep failed: removing %s' %sweep.get_name()) remove_sweeps.append(sweep) else: Chatter.write('Loading sweep: %s' % sweep.get_name()) from xia2.Schema.XSweep import XSweep new_sweep = XSweep.from_dict(xsweep_dict) sweep._indexer = new_sweep._indexer sweep._refiner = new_sweep._refiner sweep._integrater = new_sweep._integrater i_sweep += 1 for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) else: for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength(wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: try: if stop_after == 'index': sweep.get_indexer_cell() else: sweep.get_integrater_intensities() sweep.serialize() except Exception, e: if failover: Chatter.write('Processing sweep %s failed: %s' % \ (sweep.get_name(), str(e))) remove_sweeps.append(sweep) else: raise for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep)
def xia2_main(stop_after=None): '''Actually process something...''' Citations.cite('xia2') # print versions of related software Chatter.write(dials_version()) ccp4_version = get_ccp4_version() if ccp4_version is not None: Chatter.write('CCP4 %s' % ccp4_version) start_time = time.time() CommandLine = get_command_line() start_dir = Flags.get_starting_directory() # check that something useful has been assigned for processing... xtals = CommandLine.get_xinfo().get_crystals() no_images = True for name in xtals.keys(): xtal = xtals[name] if not xtal.get_all_image_names(): Chatter.write('-----------------------------------' + \ '-' * len(name)) Chatter.write('| No images assigned for crystal %s |' % name) Chatter.write('-----------------------------------' + '-' \ * len(name)) else: no_images = False args = [] from xia2.Handlers.Phil import PhilIndex params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing njob = mp_params.njob from libtbx import group_args xinfo = CommandLine.get_xinfo() if os.path.exists('xia2.json'): from xia2.Schema.XProject import XProject xinfo_new = xinfo xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() crystals_new = xinfo_new.get_crystals() for crystal_id in crystals_new.keys(): if crystal_id not in crystals: crystals[crystal_id] = crystals_new[crystal_id] continue crystals[crystal_id]._scaler = None # reset scaler for wavelength_id in crystals_new[crystal_id].get_wavelength_names( ): wavelength_new = crystals_new[crystal_id].get_xwavelength( wavelength_id) if wavelength_id not in crystals[ crystal_id].get_wavelength_names(): crystals[crystal_id].add_wavelength( crystals_new[crystal_id].get_xwavelength( wavelength_new)) continue wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps_new = wavelength_new.get_sweeps() sweeps = wavelength.get_sweeps() sweep_names = [s.get_name() for s in sweeps] sweep_keys = [(s.get_directory(), s.get_template(), s.get_image_range()) for s in sweeps] for sweep in sweeps_new: if ((sweep.get_directory(), sweep.get_template(), sweep.get_image_range()) not in sweep_keys): if sweep.get_name() in sweep_names: i = 1 while 'SWEEEP%i' % i in sweep_names: i += 1 sweep._name = 'SWEEP%i' % i break wavelength.add_sweep( name=sweep.get_name(), sample=sweep.get_xsample(), directory=sweep.get_directory(), image=sweep.get_image(), beam=sweep.get_beam_centre(), reversephi=sweep.get_reversephi(), distance=sweep.get_distance(), gain=sweep.get_gain(), dmin=sweep.get_resolution_high(), dmax=sweep.get_resolution_low(), polarization=sweep.get_polarization(), frames_to_process=sweep.get_frames_to_process(), user_lattice=sweep.get_user_lattice(), user_cell=sweep.get_user_cell(), epoch=sweep._epoch, ice=sweep._ice, excluded_regions=sweep._excluded_regions, ) sweep_names.append(sweep.get_name()) crystals = xinfo.get_crystals() failover = params.xia2.settings.failover if mp_params.mode == 'parallel' and njob > 1: driver_type = mp_params.type command_line_args = CommandLine.get_argv()[1:] for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps = wavelength.get_sweeps() for sweep in sweeps: sweep._get_indexer() sweep._get_refiner() sweep._get_integrater() args.append((group_args( driver_type=driver_type, stop_after=stop_after, failover=failover, command_line_args=command_line_args, nproc=mp_params.nproc, crystal_id=crystal_id, wavelength_id=wavelength_id, sweep_id=sweep.get_name(), ), )) from xia2.Driver.DriverFactory import DriverFactory default_driver_type = DriverFactory.get_driver_type() # run every nth job on the current computer (no need to submit to qsub) for i_job, arg in enumerate(args): if (i_job % njob) == 0: arg[0].driver_type = default_driver_type if mp_params.type == "qsub": method = "sge" else: method = "multiprocessing" nproc = mp_params.nproc qsub_command = mp_params.qsub_command if not qsub_command: qsub_command = 'qsub' qsub_command = '%s -V -cwd -pe smp %d' % (qsub_command, nproc) from libtbx import easy_mp results = easy_mp.parallel_map( process_one_sweep, args, processes=njob, #method=method, method="multiprocessing", qsub_command=qsub_command, preserve_order=True, preserve_exception_message=True) # Hack to update sweep with the serialized indexers/refiners/integraters i_sweep = 0 for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: success, output, xsweep_dict = results[i_sweep] if output is not None: Chatter.write(output) if not success: Chatter.write('Sweep failed: removing %s' % sweep.get_name()) remove_sweeps.append(sweep) else: assert xsweep_dict is not None Chatter.write('Loading sweep: %s' % sweep.get_name()) from xia2.Schema.XSweep import XSweep new_sweep = XSweep.from_dict(xsweep_dict) sweep._indexer = new_sweep._indexer sweep._refiner = new_sweep._refiner sweep._integrater = new_sweep._integrater i_sweep += 1 for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) else: for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: from dials.command_line.show import show_datablocks from dxtbx.datablock import DataBlock Debug.write(sweep.get_name()) Debug.write( show_datablocks([DataBlock([sweep.get_imageset()])])) try: if stop_after == 'index': sweep.get_indexer_cell() else: sweep.get_integrater_intensities() sweep.serialize() except Exception as e: if failover: Chatter.write('Processing sweep %s failed: %s' % \ (sweep.get_name(), str(e))) remove_sweeps.append(sweep) else: raise for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) # save intermediate xia2.json file in case scaling step fails xinfo.as_json(filename='xia2.json') if stop_after not in ('index', 'integrate'): Chatter.write(xinfo.get_output(), strip=False) for crystal in crystals.values(): crystal.serialize() # save final xia2.json file in case report generation fails xinfo.as_json(filename='xia2.json') duration = time.time() - start_time # write out the time taken in a human readable way Chatter.write('Processing took %s' % \ time.strftime("%Hh %Mm %Ss", time.gmtime(duration))) if stop_after not in ('index', 'integrate'): # and the summary file with open('xia2-summary.dat', 'w') as fh: for record in xinfo.summarise(): fh.write('%s\n' % record) # looks like this import overwrites the initial command line # Phil overrides so... for https://github.com/xia2/xia2/issues/150 from xia2.command_line.html import generate_xia2_html if params.xia2.settings.small_molecule == True: params.xia2.settings.report.xtriage_analysis = False params.xia2.settings.report.include_radiation_damage = False generate_xia2_html(xinfo, filename='xia2.html', params=params.xia2.settings.report) write_citations() # delete all of the temporary mtz files... cleanup() Environment.cleanup()
def xia2_main(stop_after=None): """Actually process something...""" Citations.cite("xia2") # print versions of related software logger.info(dials_version()) ccp4_version = get_ccp4_version() if ccp4_version: logger.info("CCP4 %s", ccp4_version) start_time = time.time() CommandLine = get_command_line() # check that something useful has been assigned for processing... xtals = CommandLine.get_xinfo().get_crystals() for name, xtal in xtals.items(): if not xtal.get_all_image_names(): logger.info("-----------------------------------" + "-" * len(name)) logger.info("| No images assigned for crystal %s |", name) logger.info("-----------------------------------" + "-" * len(name)) from xia2.Handlers.Phil import PhilIndex params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing njob = mp_params.njob xinfo = CommandLine.get_xinfo() logger.info("Project directory: %s", xinfo.path) if (params.xia2.settings.developmental.continue_from_previous_job and os.path.exists("xia2.json")): logger.debug("==== Starting from existing xia2.json ====") xinfo_new = xinfo xinfo = XProject.from_json(filename="xia2.json") crystals = xinfo.get_crystals() crystals_new = xinfo_new.get_crystals() for crystal_id in crystals_new: if crystal_id not in crystals: crystals[crystal_id] = crystals_new[crystal_id] continue crystals[crystal_id]._scaler = None # reset scaler for wavelength_id in crystals_new[crystal_id].get_wavelength_names( ): wavelength_new = crystals_new[crystal_id].get_xwavelength( wavelength_id) if wavelength_id not in crystals[ crystal_id].get_wavelength_names(): crystals[crystal_id].add_wavelength( crystals_new[crystal_id].get_xwavelength( wavelength_new)) continue wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps_new = wavelength_new.get_sweeps() sweeps = wavelength.get_sweeps() sweep_names = {s.get_name() for s in sweeps} sweep_keys = {(s.get_directory(), s.get_template(), s.get_image_range()) for s in sweeps} for sweep in sweeps_new: if ( sweep.get_directory(), sweep.get_template(), sweep.get_image_range(), ) not in sweep_keys: if sweep.get_name() in sweep_names: i = 1 while "SWEEEP%i" % i in sweep_names: i += 1 sweep._name = "SWEEP%i" % i break wavelength.add_sweep( name=sweep.get_name(), sample=sweep.sample, directory=sweep.get_directory(), image=sweep.get_image(), beam=sweep.get_beam_centre(), reversephi=sweep.get_reversephi(), distance=sweep.get_distance(), gain=sweep.get_gain(), dmin=sweep.get_resolution_high(), dmax=sweep.get_resolution_low(), polarization=sweep.get_polarization(), frames_to_process=sweep.get_frames_to_process(), user_lattice=sweep.get_user_lattice(), user_cell=sweep.get_user_cell(), epoch=sweep._epoch, ice=sweep._ice, excluded_regions=sweep._excluded_regions, ) sweep_names.add(sweep.get_name()) crystals = xinfo.get_crystals() failover = params.xia2.settings.failover with cleanup(xinfo.path): if mp_params.mode == "parallel" and njob > 1: driver_type = mp_params.type command_line_args = CommandLine.get_argv()[1:] jobs = [] for crystal_id in crystals: for wavelength_id in crystals[crystal_id].get_wavelength_names( ): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps = wavelength.get_sweeps() for sweep in sweeps: sweep._get_indexer() sweep._get_refiner() sweep._get_integrater() jobs.append((group_args( driver_type=driver_type, stop_after=stop_after, failover=failover, command_line_args=command_line_args, nproc=mp_params.nproc, crystal_id=crystal_id, wavelength_id=wavelength_id, sweep_id=sweep.get_name(), ), )) from xia2.Driver.DriverFactory import DriverFactory default_driver_type = DriverFactory.get_driver_type() # run every nth job on the current computer (no need to submit to qsub) for i_job, arg in enumerate(jobs): if (i_job % njob) == 0: arg[0].driver_type = default_driver_type nproc = mp_params.nproc qsub_command = mp_params.qsub_command or "qsub" qsub_command = "%s -V -cwd -pe smp %d" % (qsub_command, nproc) from libtbx import easy_mp results = easy_mp.parallel_map( process_one_sweep, jobs, processes=njob, method="multiprocessing", qsub_command=qsub_command, preserve_order=True, preserve_exception_message=True, ) # Hack to update sweep with the serialized indexers/refiners/integraters i_sweep = 0 for crystal_id in crystals: for wavelength_id in crystals[crystal_id].get_wavelength_names( ): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: success, output, xsweep_dict = results[i_sweep] if output is not None: logger.info(output) if not success: logger.info("Sweep failed: removing %s", sweep.get_name()) remove_sweeps.append(sweep) else: assert xsweep_dict is not None logger.info("Loading sweep: %s", sweep.get_name()) new_sweep = XSweep.from_dict(xsweep_dict) sweep._indexer = new_sweep._indexer sweep._refiner = new_sweep._refiner sweep._integrater = new_sweep._integrater i_sweep += 1 for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.sample sample.remove_sweep(sweep) else: for crystal_id in list(crystals.keys()): for wavelength_id in crystals[crystal_id].get_wavelength_names( ): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: from dials.command_line.show import show_experiments from dxtbx.model.experiment_list import ExperimentListFactory logger.debug(sweep.get_name()) logger.debug( show_experiments( ExperimentListFactory. from_imageset_and_crystal( sweep.get_imageset(), None))) Citations.cite("dials") try: if stop_after == "index": sweep.get_indexer_cell() else: sweep.get_integrater_intensities() sweep.serialize() except Exception as e: if failover: logger.info( "Processing sweep %s failed: %s", sweep.get_name(), str(e), ) remove_sweeps.append(sweep) else: raise for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.sample sample.remove_sweep(sweep) # save intermediate xia2.json file in case scaling step fails xinfo.as_json(filename="xia2.json") if stop_after not in ("index", "integrate"): logger.info(xinfo.get_output()) for crystal in list(crystals.values()): crystal.serialize() # save final xia2.json file in case report generation fails xinfo.as_json(filename="xia2.json") if stop_after not in ("index", "integrate"): # and the summary file with open("xia2-summary.dat", "w") as fh: for record in xinfo.summarise(): fh.write("%s\n" % record) # looks like this import overwrites the initial command line # Phil overrides so... for https://github.com/xia2/xia2/issues/150 from xia2.command_line.html import generate_xia2_html if params.xia2.settings.small_molecule: params.xia2.settings.report.xtriage_analysis = False params.xia2.settings.report.include_radiation_damage = False with xia2.Driver.timing.record_step("xia2.report"): generate_xia2_html(xinfo, filename="xia2.html", params=params.xia2.settings.report) duration = time.time() - start_time # write out the time taken in a human readable way logger.info("Processing took %s", time.strftime("%Hh %Mm %Ss", time.gmtime(duration))) write_citations()
def _index(self): '''Implement the indexer interface.''' Citations.cite('mosflm') indexer = MosflmIndex() indexer.set_working_directory(self.get_working_directory()) auto_logfiler(indexer) from xia2.lib.bits import unique_elements _images = unique_elements(self._indxr_images) indexer.set_images(_images) images_str = ', '.join(map(str, _images)) cell_str = None if self._indxr_input_cell: cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \ self._indxr_input_cell if self._indxr_sweep_name: #if len(self._fp_directory) <= 50: #dirname = self._fp_directory #else: #dirname = '...%s' % self._fp_directory[-46:] dirname = os.path.dirname(self.get_imageset().get_template()) Journal.block( 'autoindexing', self._indxr_sweep_name, 'mosflm', { 'images': images_str, 'target cell': self._indxr_input_cell, 'target lattice': self._indxr_input_lattice, 'template': self.get_imageset().get_template(), 'directory': dirname }) #task = 'Autoindex from images:' #for i in _images: #task += ' %s' % self.get_image_name(i) #self.set_task(task) indexer.set_template(os.path.basename(self.get_template())) indexer.set_directory(self.get_directory()) xsweep = self.get_indexer_sweep() if xsweep is not None: if xsweep.get_distance() is not None: indexer.set_distance(xsweep.get_distance()) #if self.get_wavelength_prov() == 'user': #index.set_wavelength(self.get_wavelength()) if xsweep.get_beam_centre() is not None: indexer.set_beam_centre(xsweep.get_beam_centre()) if self._indxr_input_cell: indexer.set_unit_cell(self._indxr_input_cell) if self._indxr_input_lattice is not None: spacegroup_number = lattice_to_spacegroup( self._indxr_input_lattice) indexer.set_space_group_number(spacegroup_number) if not self._mosflm_autoindex_thresh: try: min_peaks = 200 Debug.write('Aiming for at least %d spots...' % min_peaks) thresholds = [] for i in _images: p = Printpeaks() p.set_working_directory(self.get_working_directory()) auto_logfiler(p) p.set_image(self.get_image_name(i)) thresh = p.threshold(min_peaks) Debug.write('Autoindex threshold for image %d: %d' % \ (i, thresh)) thresholds.append(thresh) thresh = min(thresholds) self._mosflm_autoindex_thresh = thresh except Exception as e: print str(e) #XXX this should disappear! Debug.write('Error computing threshold: %s' % str(e)) Debug.write('Using default of 20.0') thresh = 20.0 else: thresh = self._mosflm_autoindex_thresh Debug.write('Using autoindex threshold: %d' % thresh) if self._mosflm_autoindex_sol: indexer.set_solution_number(self._mosflm_autoindex_sol) indexer.set_threshold(thresh) # now forget this to prevent weird things happening later on if self._mosflm_autoindex_sol: self._mosflm_autoindex_sol = 0 indexer.run() indxr_cell = indexer.get_refined_unit_cell() self._indxr_lattice = indexer.get_lattice() space_group_number = indexer.get_indexed_space_group_number() detector_distance = indexer.get_refined_distance() beam_centre = indexer.get_refined_beam_centre() mosaic_spreads = indexer.get_mosaic_spreads() if min(list(indxr_cell)) < 10.0 and \ indxr_cell[2] / indxr_cell[0] > 6: Debug.write('Unrealistic autoindexing solution: ' + '%.2f %.2f %.2f %.2f %.2f %.2f' % indxr_cell) # tweak some parameters and try again... self._mosflm_autoindex_thresh *= 1.5 self.set_indexer_done(False) return intgr_params = {} # look up other possible indexing solutions (not well - in # standard settings only!) This is moved earlier as it could # result in returning if Mosflm has selected the wrong # solution! try: self._indxr_other_lattice_cell = indexer.get_solutions() # Change 27/FEB/08 to support user assigned spacegroups if self._indxr_user_input_lattice: lattice_to_spacegroup_dict = { 'aP': 1, 'mP': 3, 'mC': 5, 'oP': 16, 'oC': 20, 'oF': 22, 'oI': 23, 'tP': 75, 'tI': 79, 'hP': 143, 'hR': 146, 'cP': 195, 'cF': 196, 'cI': 197 } for k in self._indxr_other_lattice_cell.keys(): if lattice_to_spacegroup_dict[k] > \ lattice_to_spacegroup_dict[ self._indxr_input_lattice]: del (self._indxr_other_lattice_cell[k]) # check that the selected unit cell matches - and if # not raise a "horrible" exception if self._indxr_input_cell: assert indxr_cell is not None for j in range(6): if math.fabs(self._indxr_input_cell[j] - indxr_cell[j]) > 2.0: Chatter.write('Mosflm autoindexing did not select ' + 'correct (target) unit cell') raise RuntimeError( 'something horrible happened in indexing') except RuntimeError as e: # check if mosflm rejected a solution we have it if 'horribl' in str(e): # ok it did - time to break out the big guns... if not self._indxr_input_cell: raise RuntimeError( 'error in solution selection when not preset') # XXX FIXME self._mosflm_autoindex_sol = _get_indexing_solution_number( indexer.get_all_output(), self._indxr_input_cell, self._indxr_input_lattice) # set the fact that we are not done... self.set_indexer_done(False) # and return - hopefully this will restart everything return else: raise e if len(mosaic_spreads) == 0: # then consider setting it do a default value... # equal to the oscillation width (a good guess) phi_width = self.get_phi_width() Chatter.write( 'Mosaic estimation failed, so guessing at %4.2f' % \ phi_width) # only consider this if we have thus far no idea on the # mosaic spread... mosaic_spreads.append(phi_width) intgr_params['raster'] = indexer.get_raster() intgr_params['separation'] = indexer.get_separation() self._indxr_resolution_estimate = indexer.get_resolution_estimate() # compute mosaic as mean(mosaic_spreads) self._indxr_mosaic = sum(mosaic_spreads) / len(mosaic_spreads) self._indxr_payload['mosflm_integration_parameters'] = intgr_params self._indxr_payload['mosflm_orientation_matrix'] = open( os.path.join(self.get_working_directory(), 'xiaindex.mat'), 'r').readlines() import copy from dxtbx.model.detector_helpers import set_mosflm_beam_centre from xia2.Wrappers.Mosflm.AutoindexHelpers import set_distance from xia2.Wrappers.Mosflm.AutoindexHelpers import crystal_model_from_mosflm_mat from cctbx import sgtbx, uctbx # update the beam centre (i.e. shift the origin of the detector) detector = copy.deepcopy(self.get_detector()) beam = copy.deepcopy(self.get_beam()) set_mosflm_beam_centre(detector, beam, beam_centre) if detector_distance is not None: set_distance(detector, detector_distance) # make a dxtbx crystal_model object from the mosflm matrix space_group = sgtbx.space_group_info(number=space_group_number).group() crystal_model = crystal_model_from_mosflm_mat( self._indxr_payload['mosflm_orientation_matrix'], unit_cell=uctbx.unit_cell(tuple(indxr_cell)), space_group=space_group) # construct an experiment_list from dxtbx.model import Experiment, ExperimentList experiment = Experiment(beam=beam, detector=detector, goniometer=self.get_goniometer(), scan=self.get_scan(), crystal=crystal_model) experiment_list = ExperimentList([experiment]) self.set_indexer_experiment_list(experiment_list)
def _index_prepare(self): from xia2.Handlers.Citations import Citations Citations.cite('dials') #all_images = self.get_matching_images() #first = min(all_images) #last = max(all_images) spot_lists = [] datablocks = [] for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps): Chatter.banner('Spotfinding %s' %xsweep.get_name()) first, last = imageset.get_scan().get_image_range() # at this stage, break out to run the DIALS code: this sets itself up # now cheat and pass in some information... save re-reading all of the # image headers # FIXME need to adjust this to allow (say) three chunks of images from dxtbx.serialize import dump from dxtbx.datablock import DataBlock sweep_filename = os.path.join( self.get_working_directory(), '%s_datablock.json' %xsweep.get_name()) dump.datablock(DataBlock([imageset]), sweep_filename) gain = PhilIndex.params.xia2.settings.input.gain if gain is libtbx.Auto: gain_estimater = self.EstimateGain() gain_estimater.set_sweep_filename(sweep_filename) gain_estimater.run() gain = gain_estimater.get_gain() Chatter.write('Estimated gain: %.2f' %gain) PhilIndex.params.xia2.settings.input.gain = gain # FIXME this should really use the assigned spot finding regions #offset = self.get_frame_offset() spotfinder = self.Spotfinder() if last - first > 10: spotfinder.set_write_hot_mask(True) spotfinder.set_input_sweep_filename(sweep_filename) spotfinder.set_output_sweep_filename( '%s_%s_datablock.json' %(spotfinder.get_xpid(), xsweep.get_name())) spotfinder.set_input_spot_filename( '%s_%s_strong.pickle' %(spotfinder.get_xpid(), xsweep.get_name())) if PhilIndex.params.dials.fast_mode: wedges = self._index_select_images_i(imageset) spotfinder.set_scan_ranges(wedges) else: spotfinder.set_scan_ranges([(first, last)]) if PhilIndex.params.dials.find_spots.phil_file is not None: spotfinder.set_phil_file(PhilIndex.params.dials.find_spots.phil_file) min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size if min_spot_size is libtbx.Auto: if imageset.get_detector()[0].get_type() == 'SENSOR_PAD': min_spot_size = 3 else: min_spot_size = None if min_spot_size is not None: spotfinder.set_min_spot_size(min_spot_size) min_local = PhilIndex.params.dials.find_spots.min_local if min_local is not None: spotfinder.set_min_local(min_local) sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong if sigma_strong: spotfinder.set_sigma_strong(sigma_strong) gain = PhilIndex.params.xia2.settings.input.gain if gain: spotfinder.set_gain(gain) filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings if filter_ice_rings: spotfinder.set_filter_ice_rings(filter_ice_rings) kernel_size = PhilIndex.params.dials.find_spots.kernel_size if kernel_size: spotfinder.set_kernel_size(kernel_size) global_threshold = PhilIndex.params.dials.find_spots.global_threshold if global_threshold is not None: spotfinder.set_global_threshold(global_threshold) spotfinder.run() spot_filename = spotfinder.get_spot_filename() if not os.path.exists(spot_filename): raise RuntimeError("Spotfinding failed: %s does not exist." %os.path.basename(spot_filename)) spot_lists.append(spot_filename) datablocks.append(spotfinder.get_output_sweep_filename()) from libtbx import easy_pickle from dials.util.ascii_art import spot_counts_per_image_plot refl = easy_pickle.load(spot_filename) if not len(refl): raise RuntimeError('No spots found in sweep %s' %xsweep.get_name()) Chatter.write(spot_counts_per_image_plot(refl), strip=False) if not PhilIndex.params.dials.fast_mode: detectblanks = self.DetectBlanks() detectblanks.set_sweep_filename(datablocks[-1]) detectblanks.set_reflections_filename(spot_filename) detectblanks.run() json = detectblanks.get_results() offset = imageset.get_scan().get_image_range()[0] blank_regions = json['strong']['blank_regions'] if len(blank_regions): blank_regions = [(int(s), int(e)) for s, e in blank_regions] for blank_start, blank_end in blank_regions: Chatter.write('WARNING: Potential blank images: %i -> %i' %( blank_start+1, blank_end)) if PhilIndex.params.xia2.settings.remove_blanks: non_blanks = [] start, end = imageset.get_array_range() last_blank_end = start for blank_start, blank_end in blank_regions: if blank_start > start: non_blanks.append((last_blank_end, blank_start)) last_blank_end = blank_end if last_blank_end+1 < end: non_blanks.append((last_blank_end, end)) xsweep = self.get_indexer_sweep() xwav = xsweep.get_wavelength() xsample = xsweep.get_xsample() sweep_name = xsweep.get_name() import string for i, (nb_start, nb_end) in enumerate(non_blanks): assert i < 26 if i == 0: sub_imageset = imageset[nb_start-start:nb_end-start] xsweep._frames_to_process = (nb_start+1, nb_end+1) self.set_indexer_prepare_done(done=False) self._indxr_imagesets[self._indxr_imagesets.index(imageset)] = sub_imageset xsweep._integrater._setup_from_imageset(sub_imageset) else: new_name = '_'.join((sweep_name, string.ascii_lowercase[i])) new_sweep = xwav.add_sweep(new_name, xsample, directory=os.path.join( os.path.basename(xsweep.get_directory()), new_name), image=imageset.get_path(nb_start-start), frames_to_process=(nb_start+1, nb_end), ) Chatter.write("Generating new sweep: %s (%s:%i:%i)" %( new_sweep.get_name(), new_sweep.get_image(), new_sweep.get_frames_to_process()[0], new_sweep.get_frames_to_process()[1])) return if not PhilIndex.params.xia2.settings.trust_beam_centre: discovery = self.DiscoverBetterExperimentalModel() discovery.set_sweep_filename(datablocks[-1]) discovery.set_spot_filename(spot_filename) #wedges = self._index_select_images_i(imageset) #discovery.set_scan_ranges(wedges) #discovery.set_scan_ranges([(first + offset, last + offset)]) try: discovery.run() except Exception, e: Debug.write('DIALS beam centre search failed: %s' %str(e)) else: # overwrite datablock.json in datablocks list datablocks[-1] = discovery.get_optimized_datablock_filename()