def print_scaling_summary(self, scaling_script): """Log summary information after scaling.""" if ScalingModelObserver().data: logger.info(ScalingModelObserver().return_model_error_summary()) valid_ranges = get_valid_image_ranges(scaling_script.experiments) image_ranges = get_image_ranges(scaling_script.experiments) msg = [] for (img, valid, refl) in zip(image_ranges, valid_ranges, scaling_script.reflections): if valid: if len(valid) > 1 or valid[0][0] != img[0] or valid[-1][ 1] != img[1]: msg.append( "Excluded images for experiment id: %s, image range: %s, limited range: %s" % ( refl.experiment_identifiers().keys()[0], list(img), list(valid), )) if msg: msg = ["Summary of image ranges removed:"] + msg logger.info("\n".join(msg)) # report on partiality of dataset partials = flex.double() for r in scaling_script.reflections: if "partiality" in r: partials.extend(r["partiality"]) not_full_sel = partials < 0.99 not_zero_sel = partials > 0.01 gt_half = partials > 0.5 lt_half = partials < 0.5 partial_gt_half_sel = not_full_sel & gt_half partial_lt_half_sel = not_zero_sel & lt_half logger.info("Summary of dataset partialities") header = ["Partiality (p)", "n_refl"] rows = [ ["all reflections", str(partials.size())], ["p > 0.99", str(not_full_sel.count(False))], ["0.5 < p < 0.99", str(partial_gt_half_sel.count(True))], ["0.01 < p < 0.5", str(partial_lt_half_sel.count(True))], ["p < 0.01", str(not_zero_sel.count(False))], ] logger.info(tabulate(rows, header)) logger.info( """ Reflections below a partiality_cutoff of %s are not considered for any part of the scaling analysis or for the reporting of merging statistics. Additionally, if applicable, only reflections with a min_partiality > %s were considered for use when refining the scaling model. """, scaling_script.params.cut_data.partiality_cutoff, scaling_script.params.reflection_selection.min_partiality, ) if MergingStatisticsObserver().data: logger.info( make_merging_statistics_summary( MergingStatisticsObserver().data["statistics"]))
def stats_only(reflections, experiments, params): """Calculate and print merging stats.""" best_unit_cell = params.reflection_selection.best_unit_cell if not params.reflection_selection.best_unit_cell: best_unit_cell = determine_best_unit_cell(experiments) scaled_miller_array = scaled_data_as_miller_array( reflections, experiments, best_unit_cell=best_unit_cell) try: res, _ = merging_stats_from_scaled_array( scaled_miller_array, params.output.merging.nbins, params.output.use_internal_variance, ) logger.info(make_merging_statistics_summary(res)) except DialsMergingStatisticsError as e: logger.info(e)
def print_scaling_summary(script): """Log summary information after scaling.""" logger.info(print_scaling_model_error_summary(script.experiments)) valid_ranges = get_valid_image_ranges(script.experiments) image_ranges = get_image_ranges(script.experiments) msg = [] for (img, valid, refl) in zip(image_ranges, valid_ranges, script.reflections): if valid: if len(valid ) > 1 or valid[0][0] != img[0] or valid[-1][1] != img[1]: msg.append( "Excluded images for experiment id: %s, image range: %s, limited range: %s" % ( refl.experiment_identifiers().keys()[0], list(img), list(valid), )) if msg: msg = ["Summary of image ranges removed:"] + msg logger.info("\n".join(msg)) # report on partiality of dataset partials = flex.double() for r in script.reflections: if "partiality" in r: partials.extend(r["partiality"]) not_full_sel = partials < 0.99 not_zero_sel = partials > 0.01 gt_half = partials > 0.5 lt_half = partials < 0.5 partial_gt_half_sel = not_full_sel & gt_half partial_lt_half_sel = not_zero_sel & lt_half logger.info("Summary of dataset partialities") header = ["Partiality (p)", "n_refl"] rows = [ ["all reflections", str(partials.size())], ["p > 0.99", str(not_full_sel.count(False))], ["0.5 < p < 0.99", str(partial_gt_half_sel.count(True))], ["0.01 < p < 0.5", str(partial_lt_half_sel.count(True))], ["p < 0.01", str(not_zero_sel.count(False))], ] logger.info(tabulate(rows, header)) logger.info( """ Reflections below a partiality_cutoff of %s are not considered for any part of the scaling analysis or for the reporting of merging statistics. Additionally, if applicable, only reflections with a min_partiality > %s were considered for use when refining the scaling model. """, script.params.cut_data.partiality_cutoff, script.params.reflection_selection.min_partiality, ) stats = script.merging_statistics_result if stats: anom_stats, cut_stats, cut_anom_stats = (None, None, None) if not script.scaled_miller_array.space_group().is_centric(): anom_stats = script.anom_merging_statistics_result logger.info(make_merging_statistics_summary(stats)) try: d_min = resolution_cc_half(stats, limit=0.3).d_min except RuntimeError as e: logger.debug(f"Resolution fit failed: {e}") else: max_current_res = stats.bins[-1].d_min if d_min and d_min - max_current_res > 0.005: logger.info( "Resolution limit suggested from CC" + "\u00BD" + " fit (limit CC" + "\u00BD" + "=0.3): %.2f", d_min, ) try: cut_stats, cut_anom_stats = merging_stats_from_scaled_array( script.scaled_miller_array.resolution_filter( d_min=d_min), script.params.output.merging.nbins, script.params.output.use_internal_variance, ) except DialsMergingStatisticsError: pass else: if script.scaled_miller_array.space_group().is_centric(): cut_anom_stats = None logger.info( table_1_summary(stats, anom_stats, cut_stats, cut_anom_stats))
def merge( experiments, reflections, d_min=None, d_max=None, combine_partials=True, partiality_threshold=0.4, best_unit_cell=None, anomalous=True, use_internal_variance=False, assess_space_group=False, n_bins=20, ): """ Merge reflection table data and generate a summary of the merging statistics. This procedure filters the input data, merges the data (normal and optionally anomalous), assesses the space group symmetry and generates a summary of the merging statistics. """ logger.info("\nMerging scaled reflection data\n") # first filter bad reflections using dials.util.filter methods reflections = filter_reflection_table( reflections, intensity_choice=["scale"], d_min=d_min, d_max=d_max, combine_partials=combine_partials, partiality_threshold=partiality_threshold, ) # ^ scale factor has been applied, so now set to 1.0 - okay as not # going to output scale factor in merged mtz. reflections["inverse_scale_factor"] = flex.double(reflections.size(), 1.0) scaled_array = scaled_data_as_miller_array([reflections], experiments, best_unit_cell) # Note, merge_equivalents does not raise an error if data is unique. merged = scaled_array.merge_equivalents( use_internal_variance=use_internal_variance).array() merged_anom = None if anomalous: anomalous_scaled = scaled_array.as_anomalous_array() merged_anom = anomalous_scaled.merge_equivalents( use_internal_variance=use_internal_variance).array() # Before merge, do assessment of the space_group if assess_space_group: merged_reflections = flex.reflection_table() merged_reflections["intensity"] = merged.data() merged_reflections["variance"] = flex.pow2(merged.sigmas()) merged_reflections["miller_index"] = merged.indices() logger.info("Running systematic absences check") run_systematic_absences_checks(experiments, merged_reflections) try: stats, anom_stats = merging_stats_from_scaled_array( scaled_array, n_bins, use_internal_variance, ) except DialsMergingStatisticsError as e: logger.error(e, exc_info=True) stats_summary = None else: stats_summary = make_merging_statistics_summary(stats) stats_summary += table_1_summary(stats, anom_stats) return merged, merged_anom, stats_summary
def merge_and_truncate(params, experiments, reflections): """Filter data, assess space group, run french wilson and Wilson stats.""" logger.info("\nMerging scaled reflection data\n") # first filter bad reflections using dials.util.filter methods reflections = filter_reflection_table( reflections[0], intensity_choice=["scale"], d_min=params.d_min, combine_partials=params.combine_partials, partiality_threshold=params.partiality_threshold, ) # ^ scale factor has been applied, so now set to 1.0 - okay as not # going to output scale factor in merged mtz. reflections["inverse_scale_factor"] = flex.double(reflections.size(), 1.0) scaled_array = scaled_data_as_miller_array([reflections], experiments) if params.anomalous: anomalous_scaled = scaled_array.as_anomalous_array() merged = scaled_array.merge_equivalents( use_internal_variance=params.merging.use_internal_variance).array() merged_anom = None if params.anomalous: merged_anom = anomalous_scaled.merge_equivalents( use_internal_variance=params.merging.use_internal_variance).array( ) # Before merge, do some assessment of the space_group if params.assess_space_group: merged_reflections = flex.reflection_table() merged_reflections["intensity"] = merged.data() merged_reflections["variance"] = merged.sigmas()**2 merged_reflections["miller_index"] = merged.indices() logger.info("Running systematic absences check") run_sys_abs_checks(experiments, merged_reflections) # Run the stats on truncating on anomalous or non anomalous? if params.anomalous: intensities = merged_anom else: intensities = merged assert intensities.is_xray_intensity_array() amplitudes = None anom_amplitudes = None if params.truncate: logger.info("\nScaling input intensities via French-Wilson Method") out = StringIO() if params.anomalous: anom_amplitudes = intensities.french_wilson(params=params, log=out) n_removed = intensities.size() - anom_amplitudes.size() assert anom_amplitudes.is_xray_amplitude_array() amplitudes = anom_amplitudes.as_non_anomalous_array() amplitudes = amplitudes.merge_equivalents().array() else: amplitudes = intensities.french_wilson(params=params, log=out) n_removed = intensities.size() - amplitudes.size() logger.info("Total number of rejected intensities %s", n_removed) logger.debug(out.getvalue()) if params.reporting.wilson_stats: if not intensities.space_group().is_centric(): wilson_scaling = data_statistics.wilson_scaling( miller_array=intensities, n_residues=params.n_residues) # XXX default n_residues? # Divert output through logger - do with StringIO rather than # info_handle else get way too much whitespace in output. out = StringIO() wilson_scaling.show(out=out) logger.info(out.getvalue()) # Apply wilson B to give absolute scale? # Show merging stats again. if params.reporting.merging_stats: stats, anom_stats = merging_stats_from_scaled_array( scaled_array, params.merging.n_bins, params.merging.use_internal_variance) if params.merging.anomalous: logger.info(make_merging_statistics_summary(anom_stats)) else: logger.info(make_merging_statistics_summary(stats)) return merged, merged_anom, amplitudes, anom_amplitudes