def merge(
experiments,
reflections,
d_min=None,
d_max=None,
combine_partials=True,
partiality_threshold=0.4,
best_unit_cell=None,
anomalous=True,
use_internal_variance=False,
assess_space_group=False,
n_bins=20,
):
"""
Merge reflection table data and generate a summary of the merging statistics.
This procedure filters the input data, merges the data (normal and optionally
anomalous), assesses the space group symmetry and generates a summary
of the merging statistics.
"""
logger.info("\nMerging scaled reflection data\n")
# first filter bad reflections using dials.util.filter methods
reflections = filter_reflection_table(
reflections,
intensity_choice=["scale"],
d_min=d_min,
d_max=d_max,
combine_partials=combine_partials,
partiality_threshold=partiality_threshold,
)
# ^ scale factor has been applied, so now set to 1.0 - okay as not
# going to output scale factor in merged mtz.
reflections["inverse_scale_factor"] = flex.double(reflections.size(), 1.0)
scaled_array = scaled_data_as_miller_array([reflections], experiments,
best_unit_cell)
# Note, merge_equivalents does not raise an error if data is unique.
merged = scaled_array.merge_equivalents(
use_internal_variance=use_internal_variance).array()
merged_anom = None
if anomalous:
anomalous_scaled = scaled_array.as_anomalous_array()
merged_anom = anomalous_scaled.merge_equivalents(
use_internal_variance=use_internal_variance).array()
# Before merge, do assessment of the space_group
if assess_space_group:
merged_reflections = flex.reflection_table()
merged_reflections["intensity"] = merged.data()
merged_reflections["variance"] = flex.pow2(merged.sigmas())
merged_reflections["miller_index"] = merged.indices()
logger.info("Running systematic absences check")
run_systematic_absences_checks(experiments, merged_reflections)
try:
stats, anom_stats = merging_stats_from_scaled_array(
scaled_array,
n_bins,
use_internal_variance,
)
except DialsMergingStatisticsError as e:
logger.error(e, exc_info=True)
stats_summary = None
else:
stats_summary = make_merging_statistics_summary(stats)
stats_summary += table_1_summary(stats, anom_stats)
return merged, merged_anom, stats_summary
def symmetry(experiments, reflection_tables, params=None):
"""
Run symmetry analysis
Args:
experiments: An experiment list.
reflection_tables: A list of reflection tables.
params: The dials.symmetry phil scope.
"""
result = None
if params is None:
params = phil_scope.extract()
refls_for_sym = []
if params.laue_group is Auto:
logger.info("=" * 80)
logger.info("")
logger.info("Performing Laue group analysis")
logger.info("")
# Transform models into miller arrays
n_datasets = len(experiments)
# Map experiments and reflections to minimum cell
# Eliminate reflections that are systematically absent due to centring
# of the lattice, otherwise they would lead to non-integer miller indices
# when reindexing to a primitive setting
cb_ops = change_of_basis_ops_to_minimum_cell(
experiments,
params.lattice_symmetry_max_delta,
params.relative_length_tolerance,
params.absolute_angle_tolerance,
)
reflection_tables = eliminate_sys_absent(experiments,
reflection_tables)
experiments, reflection_tables = apply_change_of_basis_ops(
experiments, reflection_tables, cb_ops)
refls_for_sym = get_subset_for_symmetry(experiments, reflection_tables,
params.exclude_images)
datasets = filtered_arrays_from_experiments_reflections(
experiments,
refls_for_sym,
outlier_rejection_after_filter=True,
partiality_threshold=params.partiality_threshold,
)
if len(datasets) != n_datasets:
raise ValueError(
"""Some datasets have no reflection after prefiltering, please check
input data and filtering settings e.g partiality_threshold""")
datasets = [
ma.as_anomalous_array().merge_equivalents().array()
for ma in datasets
]
result = LaueGroupAnalysis(
datasets,
normalisation=params.normalisation,
d_min=params.d_min,
min_i_mean_over_sigma_mean=params.min_i_mean_over_sigma_mean,
lattice_symmetry_max_delta=params.lattice_symmetry_max_delta,
relative_length_tolerance=params.relative_length_tolerance,
absolute_angle_tolerance=params.absolute_angle_tolerance,
best_monoclinic_beta=params.best_monoclinic_beta,
)
logger.info("")
logger.info(result)
if params.output.json is not None:
d = result.as_dict()
d["cb_op_inp_min"] = [str(cb_op) for cb_op in cb_ops]
# Copy the "input_symmetry" to "min_cell_symmetry" as it isn't technically
# the input symmetry to dials.symmetry
d["min_cell_symmetry"] = d["input_symmetry"]
del d["input_symmetry"]
json_str = json.dumps(d, indent=2)
with open(params.output.json, "w") as f:
f.write(json_str)
# Change of basis operator from input unit cell to best unit cell
cb_op_inp_best = result.best_solution.subgroup["cb_op_inp_best"]
# Get the best space group.
best_subsym = result.best_solution.subgroup["best_subsym"]
best_space_group = best_subsym.space_group(
).build_derived_acentric_group()
logger.info(
tabulate(
[[
str(best_subsym.space_group_info()),
str(best_space_group.info())
]],
["Patterson group", "Corresponding MX group"],
))
# Reindex the input data
experiments, reflection_tables = _reindex_experiments_reflections(
experiments, reflection_tables, best_space_group, cb_op_inp_best)
elif params.laue_group is not None:
if params.change_of_basis_op is not None:
cb_op = sgtbx.change_of_basis_op(params.change_of_basis_op)
else:
cb_op = sgtbx.change_of_basis_op()
# Reindex the input data
experiments, reflection_tables = _reindex_experiments_reflections(
experiments, reflection_tables, params.laue_group.group(), cb_op)
if params.systematic_absences.check:
logger.info("=" * 80)
logger.info("")
logger.info("Analysing systematic absences")
logger.info("")
# Get the laue class from the current space group.
space_group = experiments[0].crystal.get_space_group()
laue_group = str(space_group.build_derived_patterson_group().info())
logger.info("Laue group: %s", laue_group)
if laue_group in ("I m -3", "I m m m"):
if laue_group == "I m -3":
logger.info(
"""Space groups I 2 3 & I 21 3 cannot be distinguished with systematic absence
analysis, due to lattice centering.
Using space group I 2 3, space group I 21 3 is equally likely.\n""")
if laue_group == "I m m m":
logger.info(
"""Space groups I 2 2 2 & I 21 21 21 cannot be distinguished with systematic absence
analysis, due to lattice centering.
Using space group I 2 2 2, space group I 21 21 21 is equally likely.\n""")
elif laue_group not in laue_groups_for_absence_analysis:
logger.info("No absences to check for this laue group\n")
else:
if not refls_for_sym:
refls_for_sym = get_subset_for_symmetry(
experiments, reflection_tables, params.exclude_images)
if (params.d_min is Auto) and (result is not None):
d_min = result.intensities.resolution_range()[1]
elif params.d_min is Auto:
d_min = resolution_filter_from_reflections_experiments(
refls_for_sym,
experiments,
params.min_i_mean_over_sigma_mean,
params.min_cc_half,
)
else:
d_min = params.d_min
# combine before sys abs test - only triggers if laue_group=None and
# multiple input files.
if len(reflection_tables) > 1:
joint_reflections = flex.reflection_table()
for table in refls_for_sym:
joint_reflections.extend(table)
else:
joint_reflections = refls_for_sym[0]
merged_reflections = prepare_merged_reflection_table(
experiments, joint_reflections, d_min)
run_systematic_absences_checks(
experiments,
merged_reflections,
float(params.systematic_absences.significance_level),
)
logger.info(
"Saving reindexed experiments to %s in space group %s",
params.output.experiments,
str(experiments[0].crystal.get_space_group().info()),
)
experiments.as_file(params.output.experiments)
if params.output.reflections is not None:
if len(reflection_tables) > 1:
joint_reflections = flex.reflection_table()
for table in reflection_tables:
joint_reflections.extend(table)
else:
joint_reflections = reflection_tables[0]
logger.info(
"Saving %s reindexed reflections to %s",
len(joint_reflections),
params.output.reflections,
)
joint_reflections.as_file(params.output.reflections)
if params.output.html and params.systematic_absences.check:
ScrewAxisObserver().generate_html_report(params.output.html)
def merge_and_truncate(params, experiments, reflections):
"""Filter data, assess space group, run french wilson and Wilson stats."""
logger.info("\nMerging scaled reflection data\n")
# first filter bad reflections using dials.util.filter methods
reflections = filter_reflection_table(
reflections[0],
intensity_choice=["scale"],
d_min=params.d_min,
d_max=params.d_max,
combine_partials=params.combine_partials,
partiality_threshold=params.partiality_threshold,
)
# ^ scale factor has been applied, so now set to 1.0 - okay as not
# going to output scale factor in merged mtz.
reflections["inverse_scale_factor"] = flex.double(reflections.size(), 1.0)
scaled_array = scaled_data_as_miller_array([reflections], experiments)
# Note, merge_equivalents does not raise an error if data is unique.
if params.anomalous:
anomalous_scaled = scaled_array.as_anomalous_array()
merged = scaled_array.merge_equivalents(
use_internal_variance=params.merging.use_internal_variance).array()
merged_anom = None
if params.anomalous:
merged_anom = anomalous_scaled.merge_equivalents(
use_internal_variance=params.merging.use_internal_variance).array(
)
# Before merge, do some assessment of the space_group
if params.assess_space_group:
merged_reflections = flex.reflection_table()
merged_reflections["intensity"] = merged.data()
merged_reflections["variance"] = flex.pow2(merged.sigmas())
merged_reflections["miller_index"] = merged.indices()
logger.info("Running systematic absences check")
run_systematic_absences_checks(experiments, merged_reflections)
# Run the stats on truncating on anomalous or non anomalous?
if params.anomalous:
intensities = merged_anom
else:
intensities = merged
assert intensities.is_xray_intensity_array()
amplitudes = None
anom_amplitudes = None
if params.truncate:
logger.info("\nScaling input intensities via French-Wilson Method")
out = StringIO()
if params.anomalous:
anom_amplitudes = intensities.french_wilson(params=params, log=out)
n_removed = intensities.size() - anom_amplitudes.size()
assert anom_amplitudes.is_xray_amplitude_array()
amplitudes = anom_amplitudes.as_non_anomalous_array()
amplitudes = amplitudes.merge_equivalents().array()
else:
amplitudes = intensities.french_wilson(params=params, log=out)
n_removed = intensities.size() - amplitudes.size()
logger.info("Total number of rejected intensities %s", n_removed)
logger.debug(out.getvalue())
if params.reporting.wilson_stats:
if not intensities.space_group().is_centric():
try:
wilson_scaling = data_statistics.wilson_scaling(
miller_array=intensities,
n_residues=params.n_residues) # XXX default n_residues?
except (IndexError, RuntimeError) as e:
logger.error(
"\n"
"Error encountered during Wilson statistics calculation:\n"
"Perhaps there are too few unique reflections.\n"
"%s",
e,
exc_info=True,
)
else:
# Divert output through logger - do with StringIO rather than
# info_handle else get way too much whitespace in output.
out = StringIO()
wilson_scaling.show(out=out)
logger.info(out.getvalue())
# Apply wilson B to give absolute scale?
# Show merging stats again.
if params.reporting.merging_stats:
try:
stats, anom_stats = merging_stats_from_scaled_array(
scaled_array,
params.merging.n_bins,
params.merging.use_internal_variance,
)
except DialsMergingStatisticsError as e:
logger.error(e, exc_info=True)
else:
if params.merging.anomalous and anom_stats:
logger.info(make_merging_statistics_summary(anom_stats))
else:
logger.info(make_merging_statistics_summary(stats))
logger.info(table_1_summary(stats, anom_stats))
return merged, merged_anom, amplitudes, anom_amplitudes