def merge( experiments, reflections, d_min=None, d_max=None, combine_partials=True, partiality_threshold=0.4, best_unit_cell=None, anomalous=True, use_internal_variance=False, assess_space_group=False, n_bins=20, ): """ Merge reflection table data and generate a summary of the merging statistics. This procedure filters the input data, merges the data (normal and optionally anomalous), assesses the space group symmetry and generates a summary of the merging statistics. """ logger.info("\nMerging scaled reflection data\n") # first filter bad reflections using dials.util.filter methods reflections = filter_reflection_table( reflections, intensity_choice=["scale"], d_min=d_min, d_max=d_max, combine_partials=combine_partials, partiality_threshold=partiality_threshold, ) # ^ scale factor has been applied, so now set to 1.0 - okay as not # going to output scale factor in merged mtz. reflections["inverse_scale_factor"] = flex.double(reflections.size(), 1.0) scaled_array = scaled_data_as_miller_array([reflections], experiments, best_unit_cell) # Note, merge_equivalents does not raise an error if data is unique. merged = scaled_array.merge_equivalents( use_internal_variance=use_internal_variance).array() merged_anom = None if anomalous: anomalous_scaled = scaled_array.as_anomalous_array() merged_anom = anomalous_scaled.merge_equivalents( use_internal_variance=use_internal_variance).array() # Before merge, do assessment of the space_group if assess_space_group: merged_reflections = flex.reflection_table() merged_reflections["intensity"] = merged.data() merged_reflections["variance"] = flex.pow2(merged.sigmas()) merged_reflections["miller_index"] = merged.indices() logger.info("Running systematic absences check") run_systematic_absences_checks(experiments, merged_reflections) try: stats, anom_stats = merging_stats_from_scaled_array( scaled_array, n_bins, use_internal_variance, ) except DialsMergingStatisticsError as e: logger.error(e, exc_info=True) stats_summary = None else: stats_summary = make_merging_statistics_summary(stats) stats_summary += table_1_summary(stats, anom_stats) return merged, merged_anom, stats_summary
def symmetry(experiments, reflection_tables, params=None): """ Run symmetry analysis Args: experiments: An experiment list. reflection_tables: A list of reflection tables. params: The dials.symmetry phil scope. """ result = None if params is None: params = phil_scope.extract() refls_for_sym = [] if params.laue_group is Auto: logger.info("=" * 80) logger.info("") logger.info("Performing Laue group analysis") logger.info("") # Transform models into miller arrays n_datasets = len(experiments) # Map experiments and reflections to minimum cell # Eliminate reflections that are systematically absent due to centring # of the lattice, otherwise they would lead to non-integer miller indices # when reindexing to a primitive setting cb_ops = change_of_basis_ops_to_minimum_cell( experiments, params.lattice_symmetry_max_delta, params.relative_length_tolerance, params.absolute_angle_tolerance, ) reflection_tables = eliminate_sys_absent(experiments, reflection_tables) experiments, reflection_tables = apply_change_of_basis_ops( experiments, reflection_tables, cb_ops) refls_for_sym = get_subset_for_symmetry(experiments, reflection_tables, params.exclude_images) datasets = filtered_arrays_from_experiments_reflections( experiments, refls_for_sym, outlier_rejection_after_filter=True, partiality_threshold=params.partiality_threshold, ) if len(datasets) != n_datasets: raise ValueError( """Some datasets have no reflection after prefiltering, please check input data and filtering settings e.g partiality_threshold""") datasets = [ ma.as_anomalous_array().merge_equivalents().array() for ma in datasets ] result = LaueGroupAnalysis( datasets, normalisation=params.normalisation, d_min=params.d_min, min_i_mean_over_sigma_mean=params.min_i_mean_over_sigma_mean, lattice_symmetry_max_delta=params.lattice_symmetry_max_delta, relative_length_tolerance=params.relative_length_tolerance, absolute_angle_tolerance=params.absolute_angle_tolerance, best_monoclinic_beta=params.best_monoclinic_beta, ) logger.info("") logger.info(result) if params.output.json is not None: d = result.as_dict() d["cb_op_inp_min"] = [str(cb_op) for cb_op in cb_ops] # Copy the "input_symmetry" to "min_cell_symmetry" as it isn't technically # the input symmetry to dials.symmetry d["min_cell_symmetry"] = d["input_symmetry"] del d["input_symmetry"] json_str = json.dumps(d, indent=2) with open(params.output.json, "w") as f: f.write(json_str) # Change of basis operator from input unit cell to best unit cell cb_op_inp_best = result.best_solution.subgroup["cb_op_inp_best"] # Get the best space group. best_subsym = result.best_solution.subgroup["best_subsym"] best_space_group = best_subsym.space_group( ).build_derived_acentric_group() logger.info( tabulate( [[ str(best_subsym.space_group_info()), str(best_space_group.info()) ]], ["Patterson group", "Corresponding MX group"], )) # Reindex the input data experiments, reflection_tables = _reindex_experiments_reflections( experiments, reflection_tables, best_space_group, cb_op_inp_best) elif params.laue_group is not None: if params.change_of_basis_op is not None: cb_op = sgtbx.change_of_basis_op(params.change_of_basis_op) else: cb_op = sgtbx.change_of_basis_op() # Reindex the input data experiments, reflection_tables = _reindex_experiments_reflections( experiments, reflection_tables, params.laue_group.group(), cb_op) if params.systematic_absences.check: logger.info("=" * 80) logger.info("") logger.info("Analysing systematic absences") logger.info("") # Get the laue class from the current space group. space_group = experiments[0].crystal.get_space_group() laue_group = str(space_group.build_derived_patterson_group().info()) logger.info("Laue group: %s", laue_group) if laue_group in ("I m -3", "I m m m"): if laue_group == "I m -3": logger.info( """Space groups I 2 3 & I 21 3 cannot be distinguished with systematic absence analysis, due to lattice centering. Using space group I 2 3, space group I 21 3 is equally likely.\n""") if laue_group == "I m m m": logger.info( """Space groups I 2 2 2 & I 21 21 21 cannot be distinguished with systematic absence analysis, due to lattice centering. Using space group I 2 2 2, space group I 21 21 21 is equally likely.\n""") elif laue_group not in laue_groups_for_absence_analysis: logger.info("No absences to check for this laue group\n") else: if not refls_for_sym: refls_for_sym = get_subset_for_symmetry( experiments, reflection_tables, params.exclude_images) if (params.d_min is Auto) and (result is not None): d_min = result.intensities.resolution_range()[1] elif params.d_min is Auto: d_min = resolution_filter_from_reflections_experiments( refls_for_sym, experiments, params.min_i_mean_over_sigma_mean, params.min_cc_half, ) else: d_min = params.d_min # combine before sys abs test - only triggers if laue_group=None and # multiple input files. if len(reflection_tables) > 1: joint_reflections = flex.reflection_table() for table in refls_for_sym: joint_reflections.extend(table) else: joint_reflections = refls_for_sym[0] merged_reflections = prepare_merged_reflection_table( experiments, joint_reflections, d_min) run_systematic_absences_checks( experiments, merged_reflections, float(params.systematic_absences.significance_level), ) logger.info( "Saving reindexed experiments to %s in space group %s", params.output.experiments, str(experiments[0].crystal.get_space_group().info()), ) experiments.as_file(params.output.experiments) if params.output.reflections is not None: if len(reflection_tables) > 1: joint_reflections = flex.reflection_table() for table in reflection_tables: joint_reflections.extend(table) else: joint_reflections = reflection_tables[0] logger.info( "Saving %s reindexed reflections to %s", len(joint_reflections), params.output.reflections, ) joint_reflections.as_file(params.output.reflections) if params.output.html and params.systematic_absences.check: ScrewAxisObserver().generate_html_report(params.output.html)
def merge_and_truncate(params, experiments, reflections): """Filter data, assess space group, run french wilson and Wilson stats.""" logger.info("\nMerging scaled reflection data\n") # first filter bad reflections using dials.util.filter methods reflections = filter_reflection_table( reflections[0], intensity_choice=["scale"], d_min=params.d_min, d_max=params.d_max, combine_partials=params.combine_partials, partiality_threshold=params.partiality_threshold, ) # ^ scale factor has been applied, so now set to 1.0 - okay as not # going to output scale factor in merged mtz. reflections["inverse_scale_factor"] = flex.double(reflections.size(), 1.0) scaled_array = scaled_data_as_miller_array([reflections], experiments) # Note, merge_equivalents does not raise an error if data is unique. if params.anomalous: anomalous_scaled = scaled_array.as_anomalous_array() merged = scaled_array.merge_equivalents( use_internal_variance=params.merging.use_internal_variance).array() merged_anom = None if params.anomalous: merged_anom = anomalous_scaled.merge_equivalents( use_internal_variance=params.merging.use_internal_variance).array( ) # Before merge, do some assessment of the space_group if params.assess_space_group: merged_reflections = flex.reflection_table() merged_reflections["intensity"] = merged.data() merged_reflections["variance"] = flex.pow2(merged.sigmas()) merged_reflections["miller_index"] = merged.indices() logger.info("Running systematic absences check") run_systematic_absences_checks(experiments, merged_reflections) # Run the stats on truncating on anomalous or non anomalous? if params.anomalous: intensities = merged_anom else: intensities = merged assert intensities.is_xray_intensity_array() amplitudes = None anom_amplitudes = None if params.truncate: logger.info("\nScaling input intensities via French-Wilson Method") out = StringIO() if params.anomalous: anom_amplitudes = intensities.french_wilson(params=params, log=out) n_removed = intensities.size() - anom_amplitudes.size() assert anom_amplitudes.is_xray_amplitude_array() amplitudes = anom_amplitudes.as_non_anomalous_array() amplitudes = amplitudes.merge_equivalents().array() else: amplitudes = intensities.french_wilson(params=params, log=out) n_removed = intensities.size() - amplitudes.size() logger.info("Total number of rejected intensities %s", n_removed) logger.debug(out.getvalue()) if params.reporting.wilson_stats: if not intensities.space_group().is_centric(): try: wilson_scaling = data_statistics.wilson_scaling( miller_array=intensities, n_residues=params.n_residues) # XXX default n_residues? except (IndexError, RuntimeError) as e: logger.error( "\n" "Error encountered during Wilson statistics calculation:\n" "Perhaps there are too few unique reflections.\n" "%s", e, exc_info=True, ) else: # Divert output through logger - do with StringIO rather than # info_handle else get way too much whitespace in output. out = StringIO() wilson_scaling.show(out=out) logger.info(out.getvalue()) # Apply wilson B to give absolute scale? # Show merging stats again. if params.reporting.merging_stats: try: stats, anom_stats = merging_stats_from_scaled_array( scaled_array, params.merging.n_bins, params.merging.use_internal_variance, ) except DialsMergingStatisticsError as e: logger.error(e, exc_info=True) else: if params.merging.anomalous and anom_stats: logger.info(make_merging_statistics_summary(anom_stats)) else: logger.info(make_merging_statistics_summary(stats)) logger.info(table_1_summary(stats, anom_stats)) return merged, merged_anom, amplitudes, anom_amplitudes