def run(self): ''' Parse the options. ''' from dials.util.options import flatten_experiments, flatten_reflections # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) assert len(reflections) == 1 reflections = reflections[0] nvparams = dsp_phil.extract() domain_size = flex.double() mosaic_angle = flex.double() filtered_reflections = flex.reflection_table() for i in range(len(experiments)): refls = reflections.select(reflections['id'] == i) tmp = refls['id'] refls['id'] = flex.int(len(refls), 0) try: nv = NaveParameters(params = nvparams, experiments=experiments[i:i+1], reflections=refls, refinery=None, graph_verbose=False) crystal_model_nv = nv()[0] except Exception as e: print("Error recomputing mosaicity for experiment %d: %s"%(i, str(e))) continue domain_size.append(experiments[i].crystal.get_domain_size_ang() - crystal_model_nv.get_domain_size_ang()) mosaic_angle.append(experiments[i].crystal.get_half_mosaicity_deg() - crystal_model_nv.get_half_mosaicity_deg()) experiments[i].crystal = crystal_model_nv refls['id'] = tmp refls = refls.select(nv.nv_acceptance_flags) filtered_reflections.extend(refls) print("Saving new experiments as %s"%params.output.experiments) experiments.as_file(params.output.experiments) print("Removed %d out of %d reflections as outliers"%(len(reflections) - len(filtered_reflections), len(reflections))) print("Saving filtered reflections as %s"%params.output.experiments) filtered_reflections.as_pickle(params.output.reflections) if params.plot_changes: from matplotlib import pyplot as plt domain_size = domain_size.select((domain_size >= -10) & (domain_size <= 10)) mosaic_angle = mosaic_angle.select((mosaic_angle >= -0.1) & (mosaic_angle <= 0.1)) for d in [domain_size, mosaic_angle]: f = plt.figure() plt.hist(d, bins=30) plt.show()
def __init__(self, img, gain, params, center_intensity=0): """ Initialization and data read-in """ self.gain = gain self.params = params # Read settings from the DIALS target (.phil) file # If none is provided, use default settings (and may God have mercy) if self.params.dials.target != None: with open(self.params.dials.target, 'r') as settings_file: settings_file_contents = settings_file.read() settings = parse(settings_file_contents) current_phil = phil_scope.fetch(sources=[settings]) self.phil = current_phil.extract() else: self.phil = phil_scope.extract() # Modify settings self.phil.output.strong_filename = None self.processor = IOTADialsProcessor(params=self.phil) # Set customized parameters beamX = self.params.image_conversion.beam_center.x beamY = self.params.image_conversion.beam_center.y if beamX != 0 or beamY != 0: self.phil.geometry.detector.slow_fast_beam_centre = '{} {}'.format( beamY, beamX) if self.params.image_conversion.distance != 0: self.phil.geometry.detector.distance = self.params.image_conversion.distance if self.params.advanced.estimate_gain: self.phil.spotfinder.threshold.dispersion.gain = gain if self.params.image_conversion.mask is not None: self.phil.spotfinder.lookup.mask = self.params.image_conversion.mask self.phil.integration.lookup.mask = self.params.image_conversion.mask if self.params.dials.auto_threshold: threshold = int(center_intensity) self.phil.spotfinder.threshold.dispersion.global_threshold = threshold # Convert raw image into single-image datablock with misc.Capturing() as junk_output: self.datablock = DataBlockFactory.from_filenames([img])[0]
def msi(jid): """ msi stands for mad --> spot --> index Here we load a data image that has crystal diffraction then we spot peaks and index the crystal image """ #time.sleep(jid) import sys import numpy as np import glob import os from copy import deepcopy from dials.command_line.find_spots import phil_scope as find_spots_phil_scope from dials.command_line.stills_process import phil_scope\ as indexer_phil_scope from dials.algorithms.indexing.lattice_search import basis_vector_search_phil_scope from dials.array_family import flex import dxtbx from dxtbx.datablock import DataBlockFactory from dxtbx.model.experiment_list import ExperimentListFactory, ExperimentList, Experiment from libtbx.utils import Sorry as Sorry from libtbx.phil import parse from cxi_xdr_xes.command_line.two_color_process import two_color_phil_scope from cxi_xdr_xes.two_color import two_color_indexer indexer_two_color = two_color_indexer.indexer_two_color from cctbx import crystal from cxid9114 import utils from cxid9114 import parameters indexer_phil_scope.adopt_scope(two_color_phil_scope) indexer_phil_scope.adopt_scope(basis_vector_search_phil_scope) mad_index_params = indexer_phil_scope.extract() fnames = glob.glob(glob_str) out_dir = os.path.join( args.out_dir , "job%d" % jid) if not os.path.exists(out_dir): os.makedirs(out_dir) refls_dir = "refls_%s" % tag Els_dir = "Els_%s" % tag dumps_dir = "dumps_%s" % tag for o in [refls_dir, Els_dir, dumps_dir]: sub_o = os.path.join(out_dir, o) if not os.path.exists(sub_o): os.makedirs(sub_o) # track shots that indexed, or shots that # had too few spots to index, so can change parameters and try again def load_tracker_f(fname): data = [] if os.path.exists(fname): data = np.loadtxt(fname, str) if data.size and not data.shape: data = list(set(data[None].astype(int))) else: data = list(set(data.astype(int))) return data skip_weak = False skip_failed = True #False skip_indexed = False weak_shots_f = os.path.join(out_dir, "weak_shots.txt") failed_idx_f = os.path.join(out_dir, "failed_shots.txt") indexed_f = os.path.join(out_dir, "indexed_shots.txt") weak_shots = load_tracker_f(weak_shots_f) failed_shots = load_tracker_f(failed_idx_f) indexed_shots = load_tracker_f(indexed_f) if not os.path.exists(out_dir): os.makedirs(out_dir) # ------ spotting parameters spot_par = find_spots_phil_scope.fetch(source=parse("")).extract() spot_par.spotfinder.threshold.dispersion.global_threshold = 40 spot_par.spotfinder.threshold.dispersion.gain = 28 spot_par.spotfinder.threshold.dispersion.kernel_size = [2,2] spot_par.spotfinder.threshold.dispersion.sigma_strong = 1 spot_par.spotfinder.threshold.dispersion.sigma_background = 6 spot_par.spotfinder.filter.min_spot_size = 3 spot_par.spotfinder.force_2d = True # ------ indexing parameters KNOWN_SYMMETRY = crystal.symmetry("79,79,38,90,90,90", "P1") #KNOWN_SYMMETRY = crystal.symmetry("79,79,38,90,90,90", "P43212") mad_index_params.refinement.parameterisation.beam.fix = "all" mad_index_params.refinement.parameterisation.detector.fix = "all" #mad_index_params.refinement.verbosity = 3 #mad_index_params.refinement.reflections.outlier.algorithm = "null" mad_index_params.indexing.stills.refine_all_candidates = do_stills_refine mad_index_params.indexing.optimise_initial_basis_vectors = do_basis_refine #mad_index_params.indexing.stills.refine_candidates_with_known_symmetry = False mad_index_params.indexing.known_symmetry.space_group = KNOWN_SYMMETRY.space_group_info() mad_index_params.indexing.known_symmetry.unit_cell = KNOWN_SYMMETRY.unit_cell() mad_index_params.indexing.refinement_protocol.d_min_start = None mad_index_params.indexing.debug = True mad_index_params.indexing.real_space_grid_search.characteristic_grid = 0.02 mad_index_params.indexing.known_symmetry.absolute_angle_tolerance = 5.0 mad_index_params.indexing.known_symmetry.relative_length_tolerance = 0.3 mad_index_params.indexing.stills.rmsd_min_px = 20000 mad_index_params.indexing.refinement_protocol.n_macro_cycles = 1 mad_index_params.indexing.multiple_lattice_search.max_lattices = 1 mad_index_params.indexing.basis_vector_combinations.max_refine = 10000000000 #mad_index_params.indexing.basis_vector_combinations.max_combinations = 150 #mad_index_params.indexing.stills.candidate_outlier_rejection = False mad_index_params.indexing.refinement_protocol.mode = None #$"repredict_only" mad_index_params.indexing.two_color.high_energy = parameters.ENERGY_HIGH mad_index_params.indexing.two_color.low_energy = parameters.ENERGY_LOW mad_index_params.indexing.two_color.avg_energy = parameters.ENERGY_LOW * .5 + parameters.ENERGY_HIGH * .5 #mad_index_params.indexing.two_color.spiral_method = (1., 100000) # 1000000) mad_index_params.indexing.two_color.n_unique_v = 22 #mad_index_params.indexing.two_color.block_size = 25 #mad_index_params.indexing.two_color.filter_by_mag = (10,3) # ------ N = len(fnames) idx_split = np.array_split(np.arange(N), n_jobs) n_idx = 0 # number indexed for idx in idx_split[jid]: if args.Nmax is not None: if idx == args.Nmax: sys.exit() img_f = fnames[idx] loader = dxtbx.load(img_f) iset = loader.get_imageset(filenames=[loader.get_image_file()] ) DET = loader.get_detector() BEAM = loader.get_beam() El = ExperimentListFactory.from_imageset_and_crystal(iset, crystal=None) E = El[0] from cxid9114.geom.multi_panel import CSPAD_refined, CSPAD El[0].detector = CSPAD_refined if idx in weak_shots and skip_weak: print("Skipping weak shots %d" % idx) continue if idx in failed_shots and skip_failed: print("Skipping failed idx shots %d" % idx) continue if idx in indexed_shots and skip_indexed: print("Skipping already idx shots %d" % idx) continue refls_strong = flex.reflection_table.from_observations(El, spot_par) if len(refls_strong) < min_spot_per_pattern: print("Not enough spots shot %d, continuing!" % idx) weak_shots.append(idx) try: np.savetxt(weak_shots_f, weak_shots, fmt="%d") except: pass continue # ================================== # 2 color indexer of 2 color pattern # ================================== try: orientAB = indexer_two_color( reflections=refls_strong, experiments=El, params=mad_index_params) orientAB.index() except (Sorry, RuntimeError, AssertionError) as error: print("####\nIndexingFailed T_T \n####") print (error) failed_shots.append(idx) try: np.savetxt(failed_idx_f, failed_shots, fmt="%d") except: pass continue print("####\n *_* IndexingWORKED %d *_* \n####" % idx) n_idx += 1 indexed_shots.append(idx) try: np.savetxt(indexed_f, indexed_shots, fmt="%d") except: pass crystalAB = orientAB.refined_experiments.crystals()[0] refl_pkl = os.path.join(out_dir,refls_dir, "refl_%d_%s.pkl" % (idx, tag)) utils.save_flex(refls_strong, refl_pkl) El[0].crystal = crystalAB El_json = os.path.join(out_dir, Els_dir, "El_%d_%s.json" % (idx, tag)) El.as_json(filename=El_json)
def __init__(self, iparams, write_pickle=True, write_logs=True, last_stage='integrate'): ''' Constructor :param iparams: IOTA params :param write_pickle: Set to True to write out an integration pickle ''' self.iparams = iparams self.write_pickle = write_pickle self.write_logs = write_logs self.last_stage = last_stage self.dlog_bookmark = 0 # Get Processor PHIL and initialize Processor if self.iparams.cctbx_xfel.target: with open(self.iparams.cctbx_xfel.target, 'r') as tf: tphil_string = tf.read() tparams = phil_scope.fetch(source=parse(tphil_string)).extract() else: tparams = phil_scope.extract() Processor.__init__(self, params=tparams) # IOTA-specific settings from here # Turn off all peripheral output self.params.output.experiments_filename = None self.params.output.indexed_filename = None self.params.output.strong_filename = None self.params.output.refined_experiments_filename = None self.params.output.integrated_experiments_filename = None self.params.output.integrated_filename = None self.params.output.profile_filename = None # Set customized parameters beamX = self.iparams.image_import.beam_center.x beamY = self.iparams.image_import.beam_center.y if beamX != 0 or beamY != 0: self.params.geometry.detector.slow_fast_beam_centre = '{} {}'.format( beamY, beamX) if self.iparams.image_import.distance != 0: self.params.geometry.detector.distance = self.iparams.image_import.distance if self.iparams.image_import.mask is not None: self.params.spotfinder.lookup.mask = self.iparams.image_import.mask self.params.integration.lookup.mask = self.iparams.image_import.mask if self.iparams.cctbx_xfel.target_space_group is not None: sg = self.iparams.cctbx_xfel.target_space_group self.params.indexing.known_symmetry.space_group = sg if self.iparams.cctbx_xfel.target_unit_cell is not None: uc = self.iparams.cctbx_xfel.target_unit_cell self.params.indexing.known_symmetry.unit_cell = uc if not self.params.indexing.stills.method_list: self.params.indexing.stills.method_list = [ 'fft1d', 'real_space_grid_search' ] if self.iparams.cctbx_xfel.use_fft3d: self.params.indexing.stills.method_list.insert(2, 'fft3d') if self.iparams.cctbx_xfel.significance_filter.flag_on: sigma = self.iparams.cctbx_xfel.significance_filter.sigma sigma = sigma if sigma else 1 self.params.significance_filter.enable = True self.params.significance_filter.isigi_cutoff = sigma # Load reference geometry self.reference_detector = None if self.iparams.advanced.reference_geometry: from dxtbx.model.experiment_list import ExperimentListFactory try: ref_experiments = ExperimentListFactory.from_json_file( str(self.iparams.advanced.reference_geometry), check_format=False) except Exception as e: print('DEBUG: Could not make exp. list because: ', e) try: import dxtbx img = dxtbx.load( str(self.iparams.advanced.reference_geometry)) except Exception: print("DEBUG: Couldn't load geometry file {}" "".format(self.iparams.advanced.reference_geometry)) else: self.reference_detector = img.get_detector() else: assert len(ref_experiments.detectors()) == 1 self.reference_detector = ref_experiments.detectors()[0]
def test_stills_indexer_multi_lattice_bug_MosaicSauter2014( dials_regression, tmpdir): """Problem: In stills_indexer, before calling the refine function, the experiment list contains a list of dxtbx crystal models (that are not MosaicSauter2014 models). The conversion to MosaicSauter2014 is made during the refine step when functions from nave_parameters is called. If the experiment list contains more than 1 experiment, for eg. multiple lattices, only the first crystal gets assigned mosaicity. In actuality, all crystal models should be assigned mosaicity. This test only compares whether or not all crystal models have been assigned a MosaicSauter2014 model.""" import dxtbx.model from dxtbx.model import Crystal from dxtbx.model.experiment_list import ( Experiment, ExperimentList, ExperimentListFactory, ) from dials.algorithms.indexing.stills_indexer import StillsIndexer from dials.array_family import flex from dials.command_line.stills_process import ( phil_scope as stills_process_phil_scope, ) experiment_data = os.path.join( dials_regression, "refinement_test_data", "cspad_refinement", "cspad_refined_experiments_step6_level2_300.json", ) reflection_data = os.path.join( dials_regression, "refinement_test_data", "cspad_refinement", "cspad_reflections_step7_300.pickle", ) refl = flex.reflection_table.from_file(reflection_data) explist = ExperimentListFactory.from_json_file(experiment_data, check_format=False)[0:2] reflist = refl.select( refl["id"] < 2) # Only use the first 2 for convenience # Construct crystal models that don't have mosaicity. These A,B,C values are the same # as read in from the dials_regression folder # Crystal-0 cs0 = Crystal(explist[0].crystal) exp0 = Experiment( imageset=explist[0].imageset, beam=explist[0].beam, detector=explist[0].detector, goniometer=None, scan=None, crystal=cs0, ) # Crystal-1 cs1 = Crystal(explist[1].crystal) exp1 = Experiment( imageset=explist[1].imageset, beam=explist[1].beam, detector=explist[1].detector, goniometer=None, scan=None, crystal=cs1, ) # Construct a new experiment_list that will be passed on for refinement unrefined_explist = ExperimentList([exp0, exp1]) # Get default params from stills_process and construct StillsIndexer, then run refinement params = stills_process_phil_scope.extract() SI = StillsIndexer(reflist, unrefined_explist, params=params) refined_explist, new_reflist = SI.refine(unrefined_explist, reflist) # Now check whether the models have mosaicity after stills_indexer refinement # Also check that mosaicity values are within expected limits for ii, crys in enumerate(refined_explist.crystals()): assert isinstance(crys, dxtbx.model.MosaicCrystalSauter2014) if ii == 0: assert crys.get_domain_size_ang() == pytest.approx(2242.0, rel=0.1) if ii == 1: assert crys.get_domain_size_ang() == pytest.approx(2689.0, rel=0.1)
def test_stills_indexer_multi_lattice_bug_MosaicSauter2014( dials_regression, tmpdir): """ Problem: In stills_indexer, before calling the refine function, the experiment list contains a list of dxtbx crystal models (that are not MosaicSauter2014 models). The conversion to MosaicSauter2014 is made during the refine step when functions from nave_parameters is called. If the experiment list contains more than 1 experiment, for eg. multiple lattices, only the first crystal gets assigned mosaicity. In actuality, all crystal models should be assigned mosaicity. This test only compares whether or not all crystal models have been assigned a MosaicSauter2014 model. """ import libtbx from dxtbx.model.experiment_list import ExperimentListFactory from dxtbx.model.experiment_list import Experiment, ExperimentList from dials.array_family import flex from dxtbx.model import Crystal from scitbx.matrix import col from dials.algorithms.indexing.stills_indexer import StillsIndexer from dials.command_line.stills_process import ( phil_scope as stills_process_phil_scope, ) import dxtbx_model_ext # needed for comparison of types xfel_regression = libtbx.env.find_in_repositories( relative_path="xfel_regression", test=os.path.isdir) if not xfel_regression: pytest.skip("test requires xfel_regression") experiment_data = os.path.join(xfel_regression, "cspad_cbf_metrology", "input_combined_experiments_300.json") reflection_data = os.path.join(xfel_regression, "cspad_cbf_metrology", "input_combined_reflections_300.pickle") refl = flex.reflection_table.from_file(reflection_data) explist = ExperimentListFactory.from_json_file(experiment_data, check_format=False)[0:2] reflist = refl.select( refl["id"] < 2) # Only use the first 2 for convenience # Construct crystal models that don't have mosaicity. These A,B,C values are the same # as read in from the xfel_regression folder # Crystal-0 sg = "P6122" c = col((-6.273802315592485, 130.0982158871206, -5.79093135302508)) b = col((12.889379382910931, -3.488465493571038, -92.33550227410447)) a = col((73.48274403946694, 6.087949131831335, 57.16094537694445)) cs0 = Crystal(a, b, c, sg) exp0 = Experiment( imageset=explist[0].imageset, beam=explist[0].beam, detector=explist[0].detector, goniometer=None, scan=None, crystal=cs0, ) # Crystal-1 a1 = col((-9.848734136977392, -91.47863873838793, 15.37304224517648)) b1 = col((-61.88040624461757, 60.16978337475523, 35.38476831968059)) c1 = col((-72.13672676543942, -10.447921098609202, -108.38564134527415)) cs1 = Crystal(a1, b1, c1, sg) exp1 = Experiment( imageset=explist[1].imageset, beam=explist[1].beam, detector=explist[1].detector, goniometer=None, scan=None, crystal=cs1, ) # Construct a new experiment_list that will be passed on for refinement unrefined_explist = ExperimentList([exp0, exp1]) # Get default params from stills_process and construct StillsIndexer, then run refinement params = stills_process_phil_scope.extract() SI = StillsIndexer(reflist, unrefined_explist, params=params) refined_explist, new_reflist = SI.refine(unrefined_explist, reflist) # Now check whether the models have mosaicity after stills_indexer refinement for crys in refined_explist.crystals(): assert isinstance(crys, dxtbx_model_ext.MosaicCrystalSauter2014)
from cctbx import crystal import dxtbx from cxid9114.spots import spot_utils from cxid9114 import parameters from libtbx.utils import Sorry #import stackimpact #agent = stackimpact.start( # agent_key = 'agent key here', # app_name = 'MyPythonApp') #agent.start_allocation_profiler() if HAS_TWO_COLOR: indexer_phil_scope.adopt_scope(two_color_phil_scope) params = indexer_phil_scope.extract() MIN_SPOT_PER_HIT = 5 # ===================================== # Parameters show_hits = False INDEXER = two_color_indexer.indexer_two_color KNOWN_SYMMETRY = crystal.symmetry("78.95,78.95,38.13,90,90,90", "P43212") params.refinement.parameterisation.beam.fix = "all" params.refinement.parameterisation.detector.fix = "all" params.indexing.known_symmetry.space_group = KNOWN_SYMMETRY.space_group_info( ) params.refinement.verbosity = 3 params.indexing.refinement_protocol.d_min_start = None params.indexing.known_symmetry.unit_cell = KNOWN_SYMMETRY.unit_cell()