def __init__(self, experiments, reflections, params=None): if params is None: params = phil_scope.extract() self._params = params # transform models into miller arrays n_datasets = len(experiments) datasets = filtered_arrays_from_experiments_reflections( experiments, reflections, outlier_rejection_after_filter=True, partiality_threshold=params.partiality_threshold, ) if len(datasets) != n_datasets: raise ValueError( """Some datasets have no reflection after prefiltering, please check input data and filtering settings e.g partiality_threshold""") result = determine_space_group( datasets, normalisation=self._params.normalisation, d_min=self._params.d_min, min_i_mean_over_sigma_mean=self._params.min_i_mean_over_sigma_mean, lattice_symmetry_max_delta=self._params.lattice_symmetry_max_delta, relative_length_tolerance=self._params.relative_length_tolerance, absolute_angle_tolerance=self._params.absolute_angle_tolerance, ) logger.info(result) if params.output.json is not None: result.as_json(filename=params.output.json) self._export_experiments_reflections(experiments, reflections, result)
def test_filtered_arrays_from_experiments_reflections_with_batches(dials_data): x4wide_dir = dials_data("x4wide_processed") refl = flex.reflection_table.from_file( x4wide_dir.join("AUTOMATIC_DEFAULT_scaled.refl").strpath) expts = ExperimentListFactory.from_serialized_format( x4wide_dir.join("AUTOMATIC_DEFAULT_scaled.expt").strpath, check_format=False) miller_arrays, batches = filtered_arrays_from_experiments_reflections( expts, [refl], return_batches=True) assert len(miller_arrays) == len(batches) == 1 assert miller_arrays[0].size() == batches[0].size() == 62920 assert len(set(batches[0].data())) == 88 assert flex.min(batches[0].data()) == 2 assert flex.max(batches[0].data()) == 89
def delta_cc_half_analysis(self): # transform models into miller arrays intensities, batches = filtered_arrays_from_experiments_reflections( self._data_manager.experiments, parse_multiple_datasets([self._data_manager.reflections]), outlier_rejection_after_filter=False, partiality_threshold=0.99, return_batches=True, ) result = DeltaCcHalf(intensities, batches) d = {} d.update(result.histogram()) d.update(result.normalised_scores()) return d, result.get_table(html=True)
def __init__(self, experiments, reflections, params=None): super(cosym, self).__init__( events=["run_cosym", "performed_unit_cell_clustering"]) if params is None: params = phil_scope.extract() self.params = params self._experiments, self._reflections = self._filter_min_reflections( experiments, reflections) # map experiments and reflections to primitive setting self._experiments, self._reflections = self._map_to_primitive( self._experiments, self._reflections) if len(self._experiments) > 1: # perform unit cell clustering identifiers = self._unit_cell_clustering(self._experiments) if len(identifiers) < len(self._experiments): logger.info( "Selecting subset of %i datasets for cosym analysis: %s", len(identifiers), str(identifiers), ) self._experiments, self._reflections = select_datasets_on_ids( self._experiments, self._reflections, use_datasets=identifiers) self._experiments, self._reflections = self._map_to_minimum_cell( self._experiments, self._reflections) # transform models into miller arrays datasets = filtered_arrays_from_experiments_reflections( self.experiments, self.reflections, outlier_rejection_after_filter=False, partiality_threshold=params.partiality_threshold, ) self.cosym_analysis = CosymAnalysis(datasets, self.params)
def symmetry(experiments, reflection_tables, params=None): """ Run symmetry analysis Args: experiments: An experiment list. reflection_tables: A list of reflection tables. params: The dials.symmetry phil scope. """ result = None if params is None: params = phil_scope.extract() refls_for_sym = [] if params.laue_group is Auto: logger.info("=" * 80) logger.info("") logger.info("Performing Laue group analysis") logger.info("") # Transform models into miller arrays n_datasets = len(experiments) # Map experiments and reflections to minimum cell # Eliminate reflections that are systematically absent due to centring # of the lattice, otherwise they would lead to non-integer miller indices # when reindexing to a primitive setting cb_ops = change_of_basis_ops_to_minimum_cell( experiments, params.lattice_symmetry_max_delta, params.relative_length_tolerance, params.absolute_angle_tolerance, ) reflection_tables = eliminate_sys_absent(experiments, reflection_tables) experiments, reflection_tables = apply_change_of_basis_ops( experiments, reflection_tables, cb_ops) refls_for_sym = get_subset_for_symmetry(experiments, reflection_tables, params.exclude_images) datasets = filtered_arrays_from_experiments_reflections( experiments, refls_for_sym, outlier_rejection_after_filter=True, partiality_threshold=params.partiality_threshold, ) if len(datasets) != n_datasets: raise ValueError( """Some datasets have no reflection after prefiltering, please check input data and filtering settings e.g partiality_threshold""") datasets = [ ma.as_anomalous_array().merge_equivalents().array() for ma in datasets ] result = LaueGroupAnalysis( datasets, normalisation=params.normalisation, d_min=params.d_min, min_i_mean_over_sigma_mean=params.min_i_mean_over_sigma_mean, lattice_symmetry_max_delta=params.lattice_symmetry_max_delta, relative_length_tolerance=params.relative_length_tolerance, absolute_angle_tolerance=params.absolute_angle_tolerance, best_monoclinic_beta=params.best_monoclinic_beta, ) logger.info("") logger.info(result) if params.output.json is not None: d = result.as_dict() d["cb_op_inp_min"] = [str(cb_op) for cb_op in cb_ops] # Copy the "input_symmetry" to "min_cell_symmetry" as it isn't technically # the input symmetry to dials.symmetry d["min_cell_symmetry"] = d["input_symmetry"] del d["input_symmetry"] json_str = json.dumps(d, indent=2) with open(params.output.json, "w") as f: f.write(json_str) # Change of basis operator from input unit cell to best unit cell cb_op_inp_best = result.best_solution.subgroup["cb_op_inp_best"] # Get the best space group. best_subsym = result.best_solution.subgroup["best_subsym"] best_space_group = best_subsym.space_group( ).build_derived_acentric_group() logger.info( tabulate( [[ str(best_subsym.space_group_info()), str(best_space_group.info()) ]], ["Patterson group", "Corresponding MX group"], )) # Reindex the input data experiments, reflection_tables = _reindex_experiments_reflections( experiments, reflection_tables, best_space_group, cb_op_inp_best) elif params.laue_group is not None: if params.change_of_basis_op is not None: cb_op = sgtbx.change_of_basis_op(params.change_of_basis_op) else: cb_op = sgtbx.change_of_basis_op() # Reindex the input data experiments, reflection_tables = _reindex_experiments_reflections( experiments, reflection_tables, params.laue_group.group(), cb_op) if params.systematic_absences.check: logger.info("=" * 80) logger.info("") logger.info("Analysing systematic absences") logger.info("") # Get the laue class from the current space group. space_group = experiments[0].crystal.get_space_group() laue_group = str(space_group.build_derived_patterson_group().info()) logger.info("Laue group: %s", laue_group) if laue_group in ("I m -3", "I m m m"): if laue_group == "I m -3": logger.info( """Space groups I 2 3 & I 21 3 cannot be distinguished with systematic absence analysis, due to lattice centering. Using space group I 2 3, space group I 21 3 is equally likely.\n""") if laue_group == "I m m m": logger.info( """Space groups I 2 2 2 & I 21 21 21 cannot be distinguished with systematic absence analysis, due to lattice centering. Using space group I 2 2 2, space group I 21 21 21 is equally likely.\n""") elif laue_group not in laue_groups_for_absence_analysis: logger.info("No absences to check for this laue group\n") else: if not refls_for_sym: refls_for_sym = get_subset_for_symmetry( experiments, reflection_tables, params.exclude_images) if (params.d_min is Auto) and (result is not None): d_min = result.intensities.resolution_range()[1] elif params.d_min is Auto: d_min = resolution_filter_from_reflections_experiments( refls_for_sym, experiments, params.min_i_mean_over_sigma_mean, params.min_cc_half, ) else: d_min = params.d_min # combine before sys abs test - only triggers if laue_group=None and # multiple input files. if len(reflection_tables) > 1: joint_reflections = flex.reflection_table() for table in refls_for_sym: joint_reflections.extend(table) else: joint_reflections = refls_for_sym[0] merged_reflections = prepare_merged_reflection_table( experiments, joint_reflections, d_min) run_systematic_absences_checks( experiments, merged_reflections, float(params.systematic_absences.significance_level), ) logger.info( "Saving reindexed experiments to %s in space group %s", params.output.experiments, str(experiments[0].crystal.get_space_group().info()), ) experiments.as_file(params.output.experiments) if params.output.reflections is not None: if len(reflection_tables) > 1: joint_reflections = flex.reflection_table() for table in reflection_tables: joint_reflections.extend(table) else: joint_reflections = reflection_tables[0] logger.info( "Saving %s reindexed reflections to %s", len(joint_reflections), params.output.reflections, ) joint_reflections.as_file(params.output.reflections) if params.output.html and params.systematic_absences.check: ScrewAxisObserver().generate_html_report(params.output.html)
def test_filtered_arrays_from_experiments_reflections(): """Test the creating of a miller array from crystal and reflection table.""" refl = generate_integrated_test_reflections() refl["miller_index"] = flex.miller_index([(1, 0, 0), (2, 0, 0), (3, 0, 0), (4, 0, 0), (5, 0, 0), (6, 0, 0)]) experiments = ExperimentList() exp_dict = { "__id__": "crystal", "real_space_a": [1.0, 0.0, 0.0], "real_space_b": [0.0, 1.0, 0.0], "real_space_c": [0.0, 0.0, 2.0], "space_group_hall_symbol": " C 2y", } crystal = Crystal.from_dict(exp_dict) experiments.append(Experiment(crystal=crystal)) miller_set = filtered_arrays_from_experiments_reflections( experiments, [refl])[0] assert isinstance(miller_set, miller.set) assert list(miller_set.data()) == [4.6, 2.4, 2.5] # same as calling filter # for export on scale intensity reducer. # now try for prf del refl["intensity.scale.value"] miller_set = filtered_arrays_from_experiments_reflections( experiments, [refl])[0] assert isinstance(miller_set, miller.set) assert list(miller_set.data()) == [1.0, 2.0, 3.0] # same as calling filter # for export on prf + sum intensity reducer. # now just for sum del refl["intensity.prf.value"] miller_set = filtered_arrays_from_experiments_reflections( experiments, [refl])[0] assert isinstance(miller_set, miller.set) assert list(miller_set.data()) == [11.0, 12.0, 13.0, 14.0] # same as calling # filter for export on prf intensity reducer. # Now try with a bad dataset - should be filtered. refl = generate_integrated_test_reflections() refl["miller_index"] = flex.miller_index([(1, 0, 0), (2, 0, 0), (3, 0, 0), (4, 0, 0), (5, 0, 0), (6, 0, 0)]) # Trigger filtering on prf/sum, but when prf is bad - should proceed with sum refl.unset_flags(flex.bool(6, True), refl.flags.integrated_prf) del refl["intensity.scale.value"] refl2 = generate_integrated_test_reflections() refl2["partiality"] = flex.double(6, 0.0) experiments = ExperimentList() experiments.append(Experiment(crystal=crystal)) experiments.append(Experiment(crystal=crystal)) miller_sets = filtered_arrays_from_experiments_reflections( experiments, [refl, refl2], outlier_rejection_after_filter=True) assert len(miller_sets) == 1 with pytest.raises(AssertionError): miller_sets = filtered_arrays_from_experiments_reflections( experiments, [refl, refl2], return_batches=True) experiments = ExperimentList() experiments.append(Experiment(crystal=crystal)) experiments.append(Experiment(crystal=crystal)) refl2 = generate_integrated_test_reflections() refl2["partiality"] = flex.double(6, 0.0) with pytest.raises(ValueError): refl["partiality"] = flex.double(6, 0.0) _ = filtered_arrays_from_experiments_reflections( experiments, [refl, refl2])
def run(args): import libtbx.load_env from dials.util import Sorry usage = "dials.reindex [options] indexed.expt indexed.refl" parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, options = parser.parse_args(show_diff_phil=True) reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) if len(experiments) == 0 and len(reflections) == 0: parser.print_help() return if params.change_of_basis_op is None: raise Sorry("Please provide a change_of_basis_op.") reference_crystal = None if params.reference.experiments is not None: from dxtbx.serialize import load reference_experiments = load.experiment_list( params.reference.experiments, check_format=False) assert len(reference_experiments.crystals()) == 1 reference_crystal = reference_experiments.crystals()[0] if params.reference.reflections is not None: # First check that we have everything as expected for the reference reindexing # Currently only supports reindexing one dataset at a time if params.reference.experiments is None: raise Sorry( """For reindexing against a reference dataset, a reference experiments file must also be specified with the option: reference= """) if not os.path.exists(params.reference.reflections): raise Sorry("Could not locate reference dataset reflection file") if len(experiments) != 1 or len(reflections) != 1: raise Sorry( "Only one dataset can be reindexed to a reference at a time") reference_reflections = flex.reflection_table().from_file( params.reference.reflections) test_reflections = reflections[0] if (reference_crystal.get_space_group().type().number() != experiments.crystals()[0].get_space_group().type().number()): raise Sorry("Space group of input does not match reference") # Set some flags to allow filtering, if wanting to reindex against # reference with data that has not yet been through integration if (test_reflections.get_flags( test_reflections.flags.integrated_sum).count(True) == 0): assert ( "intensity.sum.value" in test_reflections), "No 'intensity.sum.value' in reflections" test_reflections.set_flags( flex.bool(test_reflections.size(), True), test_reflections.flags.integrated_sum, ) if (reference_reflections.get_flags( reference_reflections.flags.integrated_sum).count(True) == 0): assert ("intensity.sum.value" in test_reflections ), "No 'intensity.sum.value in reference reflections" reference_reflections.set_flags( flex.bool(reference_reflections.size(), True), reference_reflections.flags.integrated_sum, ) # Make miller array of the two datasets try: test_miller_set = filtered_arrays_from_experiments_reflections( experiments, [test_reflections])[0] except ValueError: raise Sorry( "No reflections remain after filtering the test dataset") try: reference_miller_set = filtered_arrays_from_experiments_reflections( reference_experiments, [reference_reflections])[0] except ValueError: raise Sorry( "No reflections remain after filtering the reference dataset") from dials.algorithms.symmetry.reindex_to_reference import ( determine_reindex_operator_against_reference, ) change_of_basis_op = determine_reindex_operator_against_reference( test_miller_set, reference_miller_set) elif len(experiments) and params.change_of_basis_op is libtbx.Auto: if reference_crystal is not None: if len(experiments.crystals()) > 1: raise Sorry("Only one crystal can be processed at a time") from dials.algorithms.indexing.compare_orientation_matrices import ( difference_rotation_matrix_axis_angle, ) cryst = experiments.crystals()[0] R, axis, angle, change_of_basis_op = difference_rotation_matrix_axis_angle( cryst, reference_crystal) print("Change of basis op: %s" % change_of_basis_op) print("Rotation matrix to transform input crystal to reference::") print(R.mathematica_form(format="%.3f", one_row_per_line=True)) print( "Rotation of %.3f degrees" % angle, "about axis (%.3f, %.3f, %.3f)" % axis, ) elif len(reflections): assert len(reflections) == 1 # always re-map reflections to reciprocal space refl_copy = flex.reflection_table() for i, imageset in enumerate(experiments.imagesets()): if "imageset_id" in reflections[0]: sel = reflections[0]["imageset_id"] == i else: sel = reflections[0]["id"] == i refl = reflections[0].select(sel) refl.centroid_px_to_mm(imageset.get_detector(), imageset.get_scan()) refl.map_centroids_to_reciprocal_space( imageset.get_detector(), imageset.get_beam(), imageset.get_goniometer(), ) refl_copy.extend(refl) # index the reflection list using the input experiments list refl_copy["id"] = flex.int(len(refl_copy), -1) index = AssignIndicesGlobal(tolerance=0.2) index(refl_copy, experiments) hkl_expt = refl_copy["miller_index"] hkl_input = reflections[0]["miller_index"] change_of_basis_op = derive_change_of_basis_op(hkl_input, hkl_expt) # reset experiments list since we don't want to reindex this experiments = [] else: change_of_basis_op = sgtbx.change_of_basis_op( params.change_of_basis_op) if len(experiments): for crystal in experiments.crystals(): cryst_orig = copy.deepcopy(crystal) cryst_reindexed = cryst_orig.change_basis(change_of_basis_op) if params.space_group is not None: a, b, c = cryst_reindexed.get_real_space_vectors() A_varying = [ cryst_reindexed.get_A_at_scan_point(i) for i in range(cryst_reindexed.num_scan_points) ] cryst_reindexed = Crystal( a, b, c, space_group=params.space_group.group()) cryst_reindexed.set_A_at_scan_points(A_varying) crystal.update(cryst_reindexed) print("Old crystal:") print(cryst_orig) print() print("New crystal:") print(cryst_reindexed) print() print("Saving reindexed experimental models to %s" % params.output.experiments) experiments.as_file(params.output.experiments) if len(reflections): assert len(reflections) == 1 reflections = reflections[0] miller_indices = reflections["miller_index"] if params.hkl_offset is not None: h, k, l = miller_indices.as_vec3_double().parts() h += params.hkl_offset[0] k += params.hkl_offset[1] l += params.hkl_offset[2] miller_indices = flex.miller_index(h.iround(), k.iround(), l.iround()) non_integral_indices = change_of_basis_op.apply_results_in_non_integral_indices( miller_indices) if non_integral_indices.size() > 0: print( "Removing %i/%i reflections (change of basis results in non-integral indices)" % (non_integral_indices.size(), miller_indices.size())) sel = flex.bool(miller_indices.size(), True) sel.set_selected(non_integral_indices, False) miller_indices_reindexed = change_of_basis_op.apply( miller_indices.select(sel)) reflections["miller_index"].set_selected(sel, miller_indices_reindexed) reflections["miller_index"].set_selected(~sel, (0, 0, 0)) print("Saving reindexed reflections to %s" % params.output.reflections) easy_pickle.dump(params.output.reflections, reflections)
def __init__(self, experiments, reflections, uuid_cache_in, params=None, do_plot=False, i_plot=None, output_dir=None): super(dials_cl_cosym_wrapper, self).__init__( events=["run_cosym", "performed_unit_cell_clustering"]) if params is None: params = phil_scope.extract() self.params = params self._reflections = [] for refl, expt in zip(reflections, experiments): sel = get_selection_for_valid_image_ranges(refl, expt) self._reflections.append(refl.select(sel)) self._experiments, self._reflections = self._filter_min_reflections( experiments, self._reflections, uuid_cache_in) self.ids_to_identifiers_map = {} for table in self._reflections: self.ids_to_identifiers_map.update(table.experiment_identifiers()) self.identifiers_to_ids_map = { value: key for key, value in self.ids_to_identifiers_map.items() } if len(self._experiments) > 1: # perform unit cell clustering identifiers = self._unit_cell_clustering(self._experiments) if len(identifiers) < len(self._experiments): logger.info( "Selecting subset of %i datasets for cosym analysis: %s" % (len(identifiers), str(identifiers))) self._experiments, self._reflections = select_datasets_on_identifiers( self._experiments, self._reflections, use_datasets=identifiers) self.uuid_cache = [ self.uuid_cache[int(id)] for id in identifiers ] # Map experiments and reflections to minimum cell cb_ops = change_of_basis_ops_to_minimum_cell( self._experiments, params.lattice_symmetry_max_delta, params.relative_length_tolerance, params.absolute_angle_tolerance, ) in_cb_ops = len(cb_ops) exclude = [ expt.identifier for expt, cb_op in zip(self._experiments, cb_ops) if not cb_op ] if len(exclude): logger.info( "Rejecting {} datasets from cosym analysis "\ "(couldn't determine consistent cb_op to minimum cell):\n"\ "{}".format(len(exclude), exclude) ) self._experiments, self._reflections = select_datasets_on_identifiers( self._experiments, self._reflections, exclude_datasets=exclude) cb_ops = list(filter(None, cb_ops)) ex_cb_ops = len(cb_ops) #Normally we expect that all the cb_ops are the same (applicable for PSI with P63) assertion_dict = {} for cb_op in cb_ops: key_ = cb_op.as_hkl() assertion_dict[key_] = assertion_dict.get(key_, 0) assertion_dict[key_] += 1 if len(assertion_dict) != 1: # unexpected, there is normally only 1 cb operator to minimum cell from libtbx.mpi4py import MPI mpi_rank = MPI.COMM_WORLD.Get_rank() mpi_size = MPI.COMM_WORLD.Get_size() print( "RANK %02d, # experiments %d, after exclusion %d, unexpectedly there are %d unique cb_ops: %s" % (mpi_rank, in_cb_ops, ex_cb_ops, len(assertion_dict), ", ".join([ "%s:%d" % (key, assertion_dict[key]) for key in assertion_dict ]))) # revisit with different use cases later # In fact we need all cb_ops to match because the user might supply # a custom reindexing operator and we need to consistently tranform # it from the conventional basis into the minimum basis. Therefore, # force them all to match, but make sure user is aware. if not params.single_cb_op_to_minimum: raise RuntimeError( 'There are >1 different cb_ops to minimum and \ cosym.single_cb_op_to_minimum is not True') else: best_cb_op_str = max(assertion_dict, key=assertion_dict.get) best_cb_op = None for cb_op in cb_ops: if cb_op.as_hkl() == best_cb_op_str: best_cb_op = cb_op break assert best_cb_op is not None cb_ops = [best_cb_op] * len(cb_ops) self.cb_op_to_minimum = cb_ops # Eliminate reflections that are systematically absent due to centring # of the lattice, otherwise they would lead to non-integer miller indices # when reindexing to a primitive setting self._reflections = eliminate_sys_absent(self._experiments, self._reflections) self._experiments, self._reflections = apply_change_of_basis_ops( self._experiments, self._reflections, cb_ops) # transform models into miller arrays datasets = filtered_arrays_from_experiments_reflections( self.experiments, self.reflections, outlier_rejection_after_filter=False, partiality_threshold=params.partiality_threshold, ) datasets = [ ma.as_anomalous_array().merge_equivalents().array() for ma in datasets ] # opportunity here to subclass as defined above, instead of the dials-implemented version self.cosym_analysis = CosymAnalysis( datasets, self.params, do_plot=do_plot, i_plot=i_plot, plot_fname=self.params.plot.filename, plot_format=self.params.plot.format, output_dir=output_dir, cb_op=cb_ops[0])
def run(args=None, phil=phil_scope): # type: (List[str], libtbx.phil.scope) -> None usage = "dials.missing_reflections [options] scaled.expt scaled.refl" parser = OptionParser( usage=usage, phil=phil, read_reflections=True, read_experiments=True, check_format=False, epilog=__doc__, ) params, options = parser.parse_args(args=args, show_diff_phil=False) # Configure the logging. dials.util.log.config(options.verbose) logger.info(dials_version()) # Log the difference between the PHIL scope definition and the active PHIL scope, # which will include the parsed user inputs. diff_phil = parser.diff_phil.as_str() if diff_phil: logger.info("The following parameters have been modified:\n%s", diff_phil) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if not experiments or not reflections: parser.print_help() return if len(reflections) != 1 and len(experiments) != len(reflections): sys.exit( "Number of experiments must equal the number of reflection tables") from dials.util.multi_dataset_handling import ( assign_unique_identifiers, parse_multiple_datasets, ) reflections = parse_multiple_datasets(reflections) experiments, reflections = assign_unique_identifiers( experiments, reflections) if all("inverse_scale_factor" in refl for refl in reflections): # Assume all arrays have been scaled miller_array = scaled_data_as_miller_array(reflections, experiments, anomalous_flag=False) else: # Else get the integrated intensities miller_arrays = filtered_arrays_from_experiments_reflections( experiments, reflections, ) miller_array = miller_arrays[0] for ma in miller_arrays[1:]: miller_array = miller_array.concatenate(ma) if params.d_min or params.d_max: miller_array = miller_array.resolution_filter(d_min=params.d_min, d_max=params.d_max) # Print overall summary of input miller array s = io.StringIO() ma_unique = miller_array.unique_under_symmetry() ma_unique.show_comprehensive_summary(f=s) logger.info(f"\n{s.getvalue()}") # Get the regions of missing reflections complete_set, unique_ms = missing_reflections.connected_components( miller_array) unique_ms = [ ms for ms in unique_ms if ms.size() >= params.min_component_size ] # Print some output for user if len(unique_ms): logger.info( "The following connected regions of missing reflections have been identified:" ) n_expected = complete_set.size() rows = [] for ms in unique_ms: d_max, d_min = (uctbx.d_star_sq_as_d(ds2) for ds2 in ms.min_max_d_star_sq()) rows.append([ ms.size(), f"{100 * ms.size() / n_expected:.1f}", f"{d_max:.2f}-{d_min:.2f}", ]) logger.info( tabulate( rows, headers=["# reflections", "% missing", "Resolution range (Å)"])) else: logger.info("No connected regions of missing reflections identified")
def run(args=sys.argv[1:]): # Create the parser parser = OptionParser( # usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, # epilog=help_message, ) # Parse the command line params, options, args = parser.parse_args(args=args, show_diff_phil=False, return_unhandled=True) # Configure the logging xia2.Handlers.Streams.setup_logging(logfile=params.output.log, verbose=options.verbose) logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) if params.unit_cell is not None: unit_cell = params.unit_cell crystal_symmetry = crystal.symmetry(unit_cell=unit_cell) else: crystal_symmetry = None if len(params.input.experiments): experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): raise sys.exit( "Mismatched number of experiments and reflection tables found: %s & %s." % (len(experiments), len(reflections))) experiments, reflections = assign_unique_identifiers( experiments, reflections) # transform models into miller arrays intensities, batches = filtered_arrays_from_experiments_reflections( experiments, reflections, outlier_rejection_after_filter=False, partiality_threshold=0.99, return_batches=True, ) if args and os.path.isfile(args[0]): result = any_reflection_file(args[0]) unmerged_intensities = None batches_all = None for ma in result.as_miller_arrays(merge_equivalents=False, crystal_symmetry=crystal_symmetry): if ma.info().labels == ["I(+)", "SIGI(+)", "I(-)", "SIGI(-)"]: assert ma.anomalous_flag() unmerged_intensities = ma elif ma.info().labels == ["I", "SIGI"]: assert not ma.anomalous_flag() unmerged_intensities = ma elif ma.info().labels == ["BATCH"]: batches_all = ma assert batches_all is not None assert unmerged_intensities is not None sel = unmerged_intensities.sigmas() > 0 unmerged_intensities = unmerged_intensities.select(sel).set_info( unmerged_intensities.info()) batches_all = batches_all.select(sel) id_to_batches = None if len(params.batch) > 0: id_to_batches = {} for b in params.batch: assert b.id is not None assert b.range is not None assert b.id not in id_to_batches, "Duplicate batch id: %s" % b.id id_to_batches[b.id] = b.range separate = separate_unmerged(unmerged_intensities, batches_all, id_to_batches=id_to_batches) batches = list(separate.batches.values()) intensities = list(separate.intensities.values()) result = DeltaCcHalf( intensities, batches, n_bins=params.n_bins, d_min=params.d_min, cc_one_half_method=params.cc_one_half_method, group_size=params.group_size, ) logger.info(tabulate(result.get_table(), headers="firstrow")) hist_filename = "delta_cc_hist.png" logger.info("Saving histogram to %s" % hist_filename) result.plot_histogram(hist_filename) normalised_scores_filename = "normalised_scores.png" logger.info("Saving normalised scores to %s" % normalised_scores_filename) result.plot_normalised_scores(normalised_scores_filename) Citations.cite("delta_cc_half") for citation in Citations.get_citations_acta(): logger.info(citation)
def __init__(self, experiments, reflections, params=None): super(cosym, self).__init__( events=["run_cosym", "performed_unit_cell_clustering"]) if params is None: params = phil_scope.extract() self.params = params self._reflections = [] for refl, expt in zip(reflections, experiments): sel = get_selection_for_valid_image_ranges(refl, expt) self._reflections.append(refl.select(sel)) self._experiments, self._reflections = self._filter_min_reflections( experiments, self._reflections) self.ids_to_identifiers_map = {} for table in self._reflections: self.ids_to_identifiers_map.update(table.experiment_identifiers()) self.identifiers_to_ids_map = { value: key for key, value in self.ids_to_identifiers_map.items() } if len(self._experiments) > 1: # perform unit cell clustering identifiers = self._unit_cell_clustering(self._experiments) if len(identifiers) < len(self._experiments): logger.info( "Selecting subset of %i datasets for cosym analysis: %s", len(identifiers), str(identifiers), ) self._experiments, self._reflections = select_datasets_on_identifiers( self._experiments, self._reflections, use_datasets=identifiers) # Map experiments and reflections to minimum cell cb_ops = change_of_basis_ops_to_minimum_cell( self._experiments, params.lattice_symmetry_max_delta, params.relative_length_tolerance, params.absolute_angle_tolerance, ) exclude = [ expt.identifier for expt, cb_op in zip(self._experiments, cb_ops) if not cb_op ] if len(exclude): logger.info( f"Rejecting {len(exclude)} datasets from cosym analysis " f"(couldn't determine consistent cb_op to minimum cell):\n" f"{exclude}", ) self._experiments, self._reflections = select_datasets_on_identifiers( self._experiments, self._reflections, exclude_datasets=exclude) cb_ops = list(filter(None, cb_ops)) # Eliminate reflections that are systematically absent due to centring # of the lattice, otherwise they would lead to non-integer miller indices # when reindexing to a primitive setting self._reflections = eliminate_sys_absent(self._experiments, self._reflections) self._experiments, self._reflections = apply_change_of_basis_ops( self._experiments, self._reflections, cb_ops) # transform models into miller arrays datasets = filtered_arrays_from_experiments_reflections( self.experiments, self.reflections, outlier_rejection_after_filter=False, partiality_threshold=params.partiality_threshold, ) datasets = [ ma.as_anomalous_array().merge_equivalents().array() for ma in datasets ] self.cosym_analysis = CosymAnalysis(datasets, self.params)
def run(args=None): import libtbx.load_env from dials.util import Sorry usage = "dials.reindex [options] indexed.expt indexed.refl" parser = ArgumentParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, options = parser.parse_args(args, show_diff_phil=True) reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments ) if len(experiments) == 0 and len(reflections) == 0: parser.print_help() return if params.change_of_basis_op is None: raise Sorry("Please provide a change_of_basis_op.") reference_crystal = None if params.reference.experiments is not None: from dxtbx.serialize import load reference_experiments = load.experiment_list( params.reference.experiments, check_format=False ) if len(reference_experiments.crystals()) == 1: reference_crystal = reference_experiments.crystals()[0] else: # first check sg all same sgs = [ expt.crystal.get_space_group().type().number() for expt in experiments ] if len(set(sgs)) > 1: raise Sorry( """The reference experiments have different space groups: space group numbers found: %s Please reanalyse the data so that space groups are consistent, (consider using dials.reindex, dials.symmetry or dials.cosym)""" % ", ".join(map(str, set(sgs))) ) reference_crystal = reference_experiments.crystals()[0] reference_crystal.unit_cell = determine_best_unit_cell( reference_experiments ) if params.reference.reflections is not None: # First check that we have everything as expected for the reference reindexing if params.reference.experiments is None: raise Sorry( """For reindexing against a reference dataset, a reference experiments file must also be specified with the option: reference.experiments= """ ) if not os.path.exists(params.reference.reflections): raise Sorry("Could not locate reference dataset reflection file") reference_reflections = flex.reflection_table().from_file( params.reference.reflections ) test_reflections = reflections[0] if ( reference_crystal.get_space_group().type().number() != experiments.crystals()[0].get_space_group().type().number() ): raise Sorry("Space group of input does not match reference") # Set some flags to allow filtering, if wanting to reindex against # reference with data that has not yet been through integration if ( test_reflections.get_flags(test_reflections.flags.integrated_sum).count( True ) == 0 ): assert ( "intensity.sum.value" in test_reflections ), "No 'intensity.sum.value' in reflections" test_reflections.set_flags( flex.bool(test_reflections.size(), True), test_reflections.flags.integrated_sum, ) if ( reference_reflections.get_flags( reference_reflections.flags.integrated_sum ).count(True) == 0 ): assert ( "intensity.sum.value" in test_reflections ), "No 'intensity.sum.value in reference reflections" reference_reflections.set_flags( flex.bool(reference_reflections.size(), True), reference_reflections.flags.integrated_sum, ) # Make miller array of the two datasets try: test_miller_set = filtered_arrays_from_experiments_reflections( experiments, [test_reflections] )[0] except ValueError: raise Sorry("No reflections remain after filtering the test dataset") try: reference_miller_set = filtered_arrays_from_experiments_reflections( reference_experiments, [reference_reflections] )[0] except ValueError: raise Sorry("No reflections remain after filtering the reference dataset") from dials.algorithms.symmetry.reindex_to_reference import ( determine_reindex_operator_against_reference, ) change_of_basis_op = determine_reindex_operator_against_reference( test_miller_set, reference_miller_set ) elif len(experiments) and params.change_of_basis_op is libtbx.Auto: if reference_crystal is not None: if len(experiments.crystals()) > 1: raise Sorry("Only one crystal can be processed at a time") from dials.algorithms.indexing.compare_orientation_matrices import ( difference_rotation_matrix_axis_angle, ) cryst = experiments.crystals()[0] R, axis, angle, change_of_basis_op = difference_rotation_matrix_axis_angle( cryst, reference_crystal ) print(f"Change of basis op: {change_of_basis_op}") print("Rotation matrix to transform input crystal to reference::") print(R.mathematica_form(format="%.3f", one_row_per_line=True)) print( f"Rotation of {angle:.3f} degrees", "about axis (%.3f, %.3f, %.3f)" % axis, ) elif len(reflections): assert len(reflections) == 1 # always re-map reflections to reciprocal space refl = reflections.deep_copy() refl.centroid_px_to_mm(experiments) refl.map_centroids_to_reciprocal_space(experiments) # index the reflection list using the input experiments list refl["id"] = flex.int(len(refl), -1) index = AssignIndicesGlobal(tolerance=0.2) index(refl, experiments) hkl_expt = refl["miller_index"] hkl_input = reflections[0]["miller_index"] change_of_basis_op = derive_change_of_basis_op(hkl_input, hkl_expt) # reset experiments list since we don't want to reindex this experiments = [] else: change_of_basis_op = sgtbx.change_of_basis_op(params.change_of_basis_op) if len(experiments): space_group = params.space_group if space_group is not None: space_group = space_group.group() try: experiments = reindex_experiments( experiments, change_of_basis_op, space_group=space_group ) except RuntimeError as e: # Only catch specific errors here if "Unsuitable value for rational rotation matrix." in str(e): original_message = str(e).split(":")[-1].strip() sys.exit(f"Error: {original_message} Is your change_of_basis_op valid?") raise print(f"Saving reindexed experimental models to {params.output.experiments}") experiments.as_file(params.output.experiments) if len(reflections): assert len(reflections) == 1 reflections = reflections[0] miller_indices = reflections["miller_index"] if params.hkl_offset is not None: h, k, l = miller_indices.as_vec3_double().parts() h += params.hkl_offset[0] k += params.hkl_offset[1] l += params.hkl_offset[2] miller_indices = flex.miller_index(h.iround(), k.iround(), l.iround()) non_integral_indices = change_of_basis_op.apply_results_in_non_integral_indices( miller_indices ) if non_integral_indices.size() > 0: print( "Removing %i/%i reflections (change of basis results in non-integral indices)" % (non_integral_indices.size(), miller_indices.size()) ) sel = flex.bool(miller_indices.size(), True) sel.set_selected(non_integral_indices, False) miller_indices_reindexed = change_of_basis_op.apply(miller_indices.select(sel)) reflections["miller_index"].set_selected(sel, miller_indices_reindexed) reflections["miller_index"].set_selected(~sel, (0, 0, 0)) print(f"Saving reindexed reflections to {params.output.reflections}") reflections.as_file(params.output.reflections)
def __init__(self, experiments, reflections, uuid_cache_in, params=None, do_plot=False, i_plot=None, output_dir=None): super(dials_cl_cosym_wrapper, self).__init__( events=["run_cosym", "performed_unit_cell_clustering"]) if params is None: params = phil_scope.extract() self.params = params self._reflections = [] for refl, expt in zip(reflections, experiments): sel = get_selection_for_valid_image_ranges(refl, expt) self._reflections.append(refl.select(sel)) self._experiments, self._reflections = self._filter_min_reflections( experiments, self._reflections, uuid_cache_in) self.ids_to_identifiers_map = {} for table in self._reflections: self.ids_to_identifiers_map.update(table.experiment_identifiers()) self.identifiers_to_ids_map = { value: key for key, value in self.ids_to_identifiers_map.items() } if len(self._experiments) > 1: # perform unit cell clustering identifiers = self._unit_cell_clustering(self._experiments) if len(identifiers) < len(self._experiments): logger.info( "Selecting subset of %i datasets for cosym analysis: %s" % (len(identifiers), str(identifiers))) self._experiments, self._reflections = select_datasets_on_identifiers( self._experiments, self._reflections, use_datasets=identifiers) # Map experiments and reflections to minimum cell cb_ops = change_of_basis_ops_to_minimum_cell( self._experiments, params.lattice_symmetry_max_delta, params.relative_length_tolerance, params.absolute_angle_tolerance, ) exclude = [ expt.identifier for expt, cb_op in zip(self._experiments, cb_ops) if not cb_op ] if len(exclude): logger.info( "Rejecting {} datasets from cosym analysis "\ "(couldn't determine consistent cb_op to minimum cell):\n"\ "{}".format(len(exclude), exclude) ) self._experiments, self._reflections = select_datasets_on_identifiers( self._experiments, self._reflections, exclude_datasets=exclude) cb_ops = list(filter(None, cb_ops)) # Eliminate reflections that are systematically absent due to centring # of the lattice, otherwise they would lead to non-integer miller indices # when reindexing to a primitive setting self._reflections = eliminate_sys_absent(self._experiments, self._reflections) self._experiments, self._reflections = apply_change_of_basis_ops( self._experiments, self._reflections, cb_ops) # transform models into miller arrays datasets = filtered_arrays_from_experiments_reflections( self.experiments, self.reflections, outlier_rejection_after_filter=False, partiality_threshold=params.partiality_threshold, ) datasets = [ ma.as_anomalous_array().merge_equivalents().array() for ma in datasets ] # opportunity here to subclass as defined above, instead of the dials-implemented version self.cosym_analysis = CosymAnalysis( datasets, self.params, do_plot=do_plot, i_plot=i_plot, plot_fname=self.params.plot.filename, plot_format=self.params.plot.format, output_dir=output_dir) #Fixed in subclass: parent class apparently erases the knowledge of input-to-minimum cb_ops. # without storing the op in self, we can never trace back to input setting. self.cb_op_to_minimum = cb_ops #Not sure yet, we may be assuming that all the cb_ops are the same (applicable for PSI with P63) assertion_set = set(cb_ops) assert len( assertion_set ) == 1 # guarantees all are the same; revisit with different use cases later