def tst_index_spots(): assert (REFLS) global CRYSTALS global RMSD from cxi_xdr_xes.two_color.two_color_indexer import indexer_two_color from cxid9114.spots import spot_utils from libtbx.utils import Sorry for i in [2, 5]: # NOTE: these are the experiments that should index refls_strong = REFLS[i] try: orientAB = indexer_two_color( reflections=spot_utils.as_single_shot_reflections( refls_strong, inplace=False), experiments=EXP_LIST[i:i + 1], params=mad_index_params) orientAB.index() except (Sorry, RuntimeError) as error: print("Failed to index experiment %d:" % i) print(error) continue RMSD.append(orientAB.best_Amat_rmsd) CRYSTALS.append(orientAB.refined_experiments.crystals()[0]) assert (len(RMSD) == 2) assert (all([r < 3.75 for r in RMSD]))
# reflections=spot_utils.as_single_shot_reflections(refls, inplace=False), # imagesets=[isetB], # params=sad_index_params) # orientB.index() # crystalB = orientB.refined_experiments.crystals()[0] # outB = sim_utils.sim_twocolors(crystalB, mos_dom=20, mos_spread=0.1, fracA=fracA, fracB=fracB, plot=False) # B_results.append(outB) # except: # B_results.append(None) # ================================== # 2 color indexer of 2 color pattern # ================================== try: orientAB = indexer_two_color( reflections=spot_utils.as_single_shot_reflections(refls_strong, inplace=False), imagesets=[iset], params=mad_index_params) orientAB.index() except (Sorry, RuntimeError): print("####\nIndexingFailed T_T \n####") failed_shots.append(idx) try: np.savetxt(failed_idx_f, failed_shots, fmt="%d") except: pass continue n_idx += 1 #if not orientAB.refined_experiments.crystals(): # this would probably never happen... # continue print("####\n *_* IndexingWORKED *_* \n####")
waveB = parameters.ENERGY_CONV / en[1] BeamA = deepcopy(iset.get_beam()) BeamB = deepcopy(iset.get_beam()) BeamA.set_wavelength(waveA) BeamB.set_wavelength(waveB) isetA = deepcopy(iset) isetB = deepcopy(iset) isetA.set_beam(BeamA) isetB.set_beam(BeamB) # index two color pattern using fft1d orientA = indexer_base.from_parameters( reflections=spot_utils.as_single_shot_reflections(refls, inplace=False), imagesets=[isetA], params=sad_index_params) orientA.index() crystalA = orientA.refined_experiments.crystals()[0] # try with other color orientB = indexer_base.from_parameters( reflections=spot_utils.as_single_shot_reflections(refls, inplace=False), imagesets=[isetB], params=sad_index_params) orientB.index() crystalB = orientB.refined_experiments.crystals()[0] # ================================== # 2 color indexer of 2 color pattern
BeamA = deepcopy(B) BeamB = deepcopy(B) mask = np.load("mask64.npz")["mask"] BeamA.set_wavelength(parameters.WAVELEN_LOW) BeamB.set_wavelength(parameters.WAVELEN_HIGH) img_data = utils.open_flex('some_imgs.pkl') dblock = utils.datablock_from_numpyarrays( [i.as_numpy_array() for i in img_data["img%d" % shot_idx]], detector=D, beam=BeamA, mask=mask) dump = utils.open_flex("dump_%d.pkl" % shot_idx) refls = dump['refls_strong'] spot_utils.as_single_shot_reflections(refls) iset = dblock.extract_imagesets()[0] if index: orientAB = indexer_two_color(reflections=refls, imagesets=[iset], params=mad_index_params) orientAB.index() C = orientAB.refined_experiments.crystals()[0] utils.save_flex({"C": C}, "C%d.pkl" % shot_idx) from dxtbx.model.experiment_list import Experiment, ExperimentList e = Experiment() e.detector = D e.beam = B
def msi(n_jobs, jid, out_dir, tag, glob_str ): """ msi stands for mad --> spot --> index Here we load a data image that has crystal diffraction then we spot peaks and index the crystal image """ import sys import numpy as np import glob import os from copy import deepcopy from dials.command_line.find_spots import phil_scope as find_spots_phil_scope import dxtbx from dxtbx.datablock import DataBlockFactory from dxtbx.model.experiment_list import ExperimentListFactory, ExperimentList, Experiment from dials.array_family import flex from libtbx.utils import Sorry as Sorry from libtbx.phil import parse from cxi_xdr_xes.two_color import two_color_indexer indexer_two_color = two_color_indexer.indexer_two_color from cxid9114 import utils from cctbx import crystal from cxid9114 import parameters from cxid9114.spots import spot_utils from dials.command_line.stills_process import phil_scope\ as indexer_phil_scope from cxi_xdr_xes.command_line.two_color_process import two_color_phil_scope from dials.algorithms.indexing.lattice_search import basis_vector_search_phil_scope indexer_phil_scope.adopt_scope(two_color_phil_scope) indexer_phil_scope.adopt_scope(basis_vector_search_phil_scope) mad_index_params = indexer_phil_scope.extract() fnames = glob.glob(glob_str) out_dir =os.path.join( out_dir , "job%d" % jid) if not os.path.exists(out_dir): os.makedirs(out_dir) # track shots that indexed, or shots that # had too few spots to index, so can change parameters and try again def load_tracker_f(fname): data = [] if os.path.exists(fname): data = np.loadtxt(fname, str) if data.size and not data.shape: data = list(set(data[None].astype(int))) else: data = list(set(data.astype(int))) return data skip_weak = False skip_failed = False skip_indexed = False weak_shots_f = os.path.join(out_dir, "weak_shots.txt") failed_idx_f = os.path.join(out_dir, "failed_shots.txt") indexed_f = os.path.join(out_dir, "indexed_shots.txt") weak_shots = load_tracker_f(weak_shots_f) failed_shots = load_tracker_f(failed_idx_f) indexed_shots = load_tracker_f(indexed_f) if not os.path.exists(out_dir): os.makedirs(out_dir) # --- spotting parameters spot_par = find_spots_phil_scope.fetch(source=parse("")).extract() spot_par.spotfinder.threshold.dispersion.global_threshold = 40 spot_par.spotfinder.threshold.dispersion.gain = 28 spot_par.spotfinder.threshold.dispersion.kernel_size = [2,2] spot_par.spotfinder.threshold.dispersion.sigma_strong = 1 spot_par.spotfinder.threshold.dispersion.sigma_background = 6 spot_par.spotfinder.filter.min_spot_size = 3 spot_par.spotfinder.force_2d = True # ------ indexing parameters KNOWN_SYMMETRY = crystal.symmetry("79,79,38,90,90,90", "P43212") #KNOWN_SYMMETRY = crystal.symmetry("78.95,78.95,38.13,90,90,90", "P1") #KNOWN_SYMMETRY = crystal.symmetry("78.95,78.95,38.13,90,90,90", "P43212") mad_index_params.refinement.parameterisation.beam.fix = "all" mad_index_params.refinement.parameterisation.detector.fix = "all" mad_index_params.refinement.verbosity = 3 #mad_index_params.refinement.reflections.outlier.algorithm = "null" mad_index_params.indexing.stills.refine_all_candidates = args.stills_refine mad_index_params.indexing.optimise_initial_basis_vectors = args.basis_refine #mad_index_params.indexing.stills.refine_candidates_with_known_symmetry = False mad_index_params.indexing.known_symmetry.space_group = KNOWN_SYMMETRY.space_group_info() mad_index_params.indexing.known_symmetry.unit_cell = KNOWN_SYMMETRY.unit_cell() mad_index_params.indexing.refinement_protocol.d_min_start = None mad_index_params.indexing.debug = True mad_index_params.indexing.real_space_grid_search.characteristic_grid = 0.02 mad_index_params.indexing.known_symmetry.absolute_angle_tolerance = 5.0 mad_index_params.indexing.known_symmetry.relative_length_tolerance = 0.3 mad_index_params.indexing.stills.rmsd_min_px = 20000 mad_index_params.indexing.refinement_protocol.n_macro_cycles = 1 mad_index_params.indexing.multiple_lattice_search.max_lattices = 1 mad_index_params.indexing.basis_vector_combinations.max_refine = 10000000000 #mad_index_params.indexing.basis_vector_combinations.max_combinations = 150 #mad_index_params.indexing.stills.candidate_outlier_rejection = False mad_index_params.indexing.refinement_protocol.mode = "ignore" mad_index_params.indexing.two_color.high_energy = parameters.ENERGY_HIGH mad_index_params.indexing.two_color.low_energy = parameters.ENERGY_LOW mad_index_params.indexing.two_color.avg_energy = parameters.ENERGY_LOW * .5 + parameters.ENERGY_HIGH * .5 #mad_index_params.indexing.two_color.spiral_method = (1., 100000) # 1000000) mad_index_params.indexing.two_color.n_unique_v = 22 #mad_index_params.indexing.two_color.block_size = 25 #mad_index_params.indexing.two_color.filter_by_mag = (10,3) # ------ N = len(fnames) idx_split = np.array_split( np.arange(N), n_jobs) n_idx = 0 # number indexed for idx in idx_split[jid]: img_f = fnames[idx] loader = dxtbx.load(img_f) iset = loader.get_imageset( loader.get_image_file() ) DET = loader.get_detector() BEAM = loader.get_beam() El = ExperimentListFactory.from_imageset_and_crystal( iset, crystal=None) img_data = loader.get_raw_data().as_numpy_array() if idx in weak_shots and skip_weak: print("Skipping weak shots %d" % idx) continue if idx in failed_shots and skip_failed: print("Skipping failed idx shots %d" % idx) continue if idx in indexed_shots and skip_indexed: print("Skipping already idx shots %d" % idx) continue if use_dials_spotter: refls_strong = flex.reflection_table.from_observations(El, spot_par) else: refls_strong = spot_utils.refls_from_sims([img_data], DET, BEAM, thresh=1e-2) refls_strong['id'] = flex.int( np.zeros( len(refls_strong))) if len(refls_strong) < min_spot_per_pattern: print("Not enough spots shot %d, continuing!" % idx) weak_shots.append(idx) try: np.savetxt(weak_shots_f, weak_shots, fmt="%d") except: pass continue # ================================== # 2 color indexer of 2 color pattern # ================================== try: orientAB = indexer_two_color( reflections=spot_utils.as_single_shot_reflections(refls_strong, inplace=False), experiments=El, params=mad_index_params) orientAB.index() except (Sorry, RuntimeError, AssertionError) as error: print("####\nIndexingFailed T_T \n####") print (error) failed_shots.append( idx) try: np.savetxt(failed_idx_f, failed_shots, fmt="%d") except: pass continue print("####\n *_* IndexingWORKED %d *_* \n####" % idx) n_idx += 1 indexed_shots.append(idx) try: np.savetxt(indexed_f, indexed_shots, fmt="%d") except: pass crystalAB = orientAB.refined_experiments.crystals()[0] refl_pkl = os.path.join(out_dir, "refl_%d_%s.pkl" % (idx, tag)) utils.save_flex(refls_strong, refl_pkl) dump = {"crystalAB": crystalAB, "img_f":img_f, "refined_refls_v1": orientAB.refined_reflections, "refls_strong": refls_strong} dump_pkl = os.path.join(out_dir, "dump_%d_%s.pkl" % (idx, tag)) utils.save_flex(dump, dump_pkl)
# ================================== # 2 color indexer of 2 color pattern # ================================== sad_index_params.indexing.multiple_lattice_search.max_lattices = 1 sad_index_params.indexing.stills.refine_all_candidates = False sad_index_params.indexing.stills.refine_candidates_with_known_symmetry = False sad_index_params.indexing.stills.candidate_outlier_rejection = False sad_index_params.indexing.refinement_protocol.mode = "ignore" sad_index_params2 = deepcopy(sad_index_params) #sad_index_params2.indexing.stills.refine_all_candidates = False #sad_index_params2.indexing.stills.refine_candidates_with_known_symmetry = False #sad_index_params2.indexing.stills.candidate_outlier_rejection = False #sad_index_params2.indexing.refinement_protocol.mode = "ignore" orientAB_fft1d = indexer_base.from_parameters( reflections=spot_utils.as_single_shot_reflections(reflAB, inplace=False), imagesets=isetsAB, params=sad_index_params2) orientAB_fft1d.index() crystal_AB_fft1d = orientAB_fft1d.refined_experiments.crystals()[0] orientAB_2_fft1d = indexer_base.from_parameters( reflections=spot_utils.as_single_shot_reflections(reflAB, inplace=False), imagesets=isetsAB_2, params=sad_index_params2) orientAB_2_fft1d.index() crystalAB_2_fft1d = orientAB_2_fft1d.refined_experiments.crystals()[0] # index the 1 color patterns: