def calculate_completeness(self, resolution_bin=None):
"""Calculate the completeness of observations in a named
resolution bin."""
if resolution_bin is None:
resolution_range = self._mf.get_resolution_range()
hkl_list = list(self._merged_reflections)
else:
resolution_range = self._resolution_ranges[resolution_bin]
hkl_list = self._hkl_ranges[resolution_bin]
uc = self._mf.get_unit_cell()
sg = self._mf.get_space_group()
dmin = min(resolution_range)
dmax = max(resolution_range)
cs = crystal_symmetry(unit_cell=uc, space_group=sg)
hkl_calc = [
hkl for hkl in build_set(cs, False, d_min=dmin, d_max=dmax).indices()
]
# remove systematically absent reflections
hkl_list = [hkl for hkl in itertools.ifilterfalse(sg.is_sys_absent, hkl_list)]
return float(len(hkl_list)) / float(len(hkl_calc))
def calculate_completeness(self, resolution_bin = None):
'''Calculate the completeness of observations in a named
resolution bin.'''
if resolution_bin is None:
resolution_range = self._mf.get_resolution_range()
hkl_list = list(self._merged_reflections)
else:
resolution_range = self._resolution_ranges[resolution_bin]
hkl_list = self._hkl_ranges[resolution_bin]
uc = self._mf.get_unit_cell()
sg = self._mf.get_space_group()
dmin = min(resolution_range)
dmax = max(resolution_range)
cs = crystal_symmetry(unit_cell = uc, space_group = sg)
hkl_calc = [hkl for hkl in build_set(
cs, False, d_min = dmin, d_max = dmax).indices()]
# remove systematically absent reflections
hkl_list = [hkl for hkl in
itertools.ifilterfalse(sg.is_sys_absent, hkl_list)]
return float(len(hkl_list)) / float(len(hkl_calc))
def test_crystal_pointgroup_symmetry(reflections, experiment, params):
crystal = experiment.crystal
# in case we pass in reflections from integration
reflections = reflections.select(reflections["intensity.sum.variance"] > 0)
reflections = reflections.select(reflections["intensity.sum.value"] > 0)
original_miller_indices = reflections["miller_index"]
space_group = crystal.get_space_group()
unit_cell = crystal.get_unit_cell()
cs = crystal_symmetry(unit_cell, space_group.type().lookup_symbol())
ms = miller_set(cs, original_miller_indices)
ms = ms.array(reflections["intensity.sum.value"] /
flex.sqrt(reflections["intensity.sum.variance"]))
if params.d_min or params.d_max:
d_spacings = ms.d_spacings().data()
sel = (d_spacings >= params.d_min) & (d_spacings <= params.d_max)
ms = ms.select(sel)
if params.normalise:
if params.normalise_bins:
ms = normalise_intensities(ms, n_bins=params.normalise_bins)
else:
ms = normalise_intensities(ms)
logger.info("Check symmetry operations on %d reflections:" % ms.size())
logger.info("")
logger.info("%20s %6s %5s" % ("Symop", "Nref", "CC"))
true_symops = []
ccs, n_refs = get_symop_correlation_coefficients(ms)
ses = standard_error_of_pearson_cc(ccs, n_refs)
for smx, cc, n_ref, se in zip(space_group.smx(), ccs, n_refs, ses):
accept = ""
if params.symop_threshold:
if (cc - 2.0 * se) > params.symop_threshold:
true_symops.append(smx)
accept = "***"
cc_str = format_cc_with_standard_error(cc, se)
logger.info("%20s %6d %s %s" % (smx, n_ref, cc_str, accept))
if params.symop_threshold:
sg = sgtbx_space_group()
for symop in true_symops:
sg = sg.expand_smx(symop)
for ltr in space_group.ltr():
sg = sg.expand_ltr(ltr)
sg_symbols = sg.match_tabulated_settings()
logger.info("")
logger.info("Derived point group from symmetry operations: %s" %
sg_symbols.hermann_mauguin())
logger.info("")
return
def get_indexing_offset_correlation_coefficients(
reflections, crystal, grid_search_scope, d_min=None, d_max=None, map_to_asu=False
):
from copy import deepcopy
from dials.array_family import flex
space_group = crystal.get_space_group()
unit_cell = crystal.get_unit_cell()
from cctbx.crystal import symmetry as crystal_symmetry
cs = crystal_symmetry(unit_cell, space_group.type().lookup_symbol())
from cctbx.miller import set as miller_set
data = reflections["intensity.sum.value"] / flex.sqrt(reflections["intensity.sum.variance"])
ccs = flex.double()
offsets = flex.vec3_int()
nref = flex.size_t()
original_miller_indices = reflections["miller_index"]
ms = miller_set(cs, original_miller_indices)
ms = ms.array(data)
if d_min is not None or d_max is not None:
ms = ms.resolution_filter(d_min=d_min, d_max=d_max)
if map_to_asu:
ms = ms.map_to_asu()
g = grid_search_scope
for h in range(-g, g + 1):
for k in range(-g, g + 1):
for l in range(-g, g + 1):
for smx in ["-x,-y,-z"]:
# reindexed = deepcopy(reflections)
# hkl offset doubled as equivalent of h0 + 1, hI - 1
miller_indices = offset_miller_indices(ms.indices(), (2 * h, 2 * k, 2 * l))
reindexed_miller_indices = sgtbx.change_of_basis_op(smx).apply(miller_indices)
rms = miller_set(cs, reindexed_miller_indices)
rms = rms.array(data)
# if params.d_min or params.d_max:
# rms = rms.resolution_filter(d_min=params.d_min, d_max=params.d_max)
# if map_to_asu:
# rms = rms.map_to_asu()
intensity, intensity_rdx = rms.common_sets(ms)
cc = intensity.correlation(intensity_rdx).coefficient()
ccs.append(cc)
offsets.append((h, k, l))
nref.append(intensity.size())
return offsets, ccs, nref
def compute_unique_reflections(unit_cell,
space_group,
anomalous,
high_resolution_limit,
low_resolution_limit = None):
'''Compute the list of unique reflections from the unit cell and space
group.'''
cs = crystal_symmetry(unit_cell = unit_cell,
space_group = space_group)
return [hkl for hkl in build_set(cs, anomalous,
d_min = high_resolution_limit,
d_max = low_resolution_limit).indices()]
def compute_unique_reflections(unit_cell,
space_group,
anomalous,
high_resolution_limit,
low_resolution_limit=None):
'''Compute the list of unique reflections from the unit cell and space
group.'''
cs = crystal_symmetry(unit_cell=unit_cell, space_group=space_group)
return [
hkl for hkl in build_set(cs,
anomalous,
d_min=high_resolution_limit,
d_max=low_resolution_limit).indices()
]
def calculate_completeness(self, resolution_bin = None):
'''Calculate the completeness of observations in a named
resolution bin.'''
if resolution_bin is None:
resolution_range = self._mf.get_resolution_range()
hkl_list = list(self._merged_reflections)
else:
resolution_range = self._resolution_ranges[resolution_bin]
hkl_list = self._hkl_ranges[resolution_bin]
uc = self._mf.get_unit_cell()
sg = self._mf.get_space_group()
d_min = min(resolution_range)
d_max = max(resolution_range)
cs = crystal_symmetry(unit_cell=uc, space_group = sg)
from cctbx import miller
ms = miller.set(cs, flex.miller_index(hkl_list), anomalous_flag=False).remove_systematic_absences()
return ms.completeness(d_max=d_max)
def get_indexing_offset_correlation_coefficients(reflections,
crystal,
grid,
d_min=None,
d_max=None,
map_to_asu=False,
grid_h=0,
grid_k=0,
grid_l=0,
reference=None):
from dials.algorithms.symmetry import origin
if grid:
if grid_h == 0: grid_h = grid
if grid_k == 0: grid_k = grid
if grid_l == 0: grid_l = grid
if True:
return origin.get_hkl_offset_correlation_coefficients(
reflections,
crystal,
map_to_asu=map_to_asu,
grid_h=grid_h,
grid_k=grid_k,
grid_l=grid_l,
reference=reference)
from copy import deepcopy
from dials.array_family import flex
space_group = crystal.get_space_group()
unit_cell = crystal.get_unit_cell()
from cctbx.crystal import symmetry as crystal_symmetry
cs = crystal_symmetry(unit_cell, space_group.type().lookup_symbol())
from cctbx.miller import set as miller_set
data = reflections['intensity.sum.value'] / \
flex.sqrt(reflections['intensity.sum.variance'])
if reference:
reference = reference.select(reference['intensity.sum.variance'] > 0)
reference_data = reference['intensity.sum.value'] / \
flex.sqrt(reference['intensity.sum.variance'])
reference_ms = miller_set(
cs, reference['miller_index']).array(reference_data)
else:
reference_ms = None
ccs = flex.double()
offsets = flex.vec3_int()
nref = flex.size_t()
original_miller_indices = reflections['miller_index']
ms = miller_set(cs, original_miller_indices).array(data)
if d_min is not None or d_max is not None:
ms = ms.resolution_filter(d_min=d_min, d_max=d_max)
gh = gk = gl = grid
if grid_h: gh = grid_h
if grid_k: gk = grid_k
if grid_l: gl = grid_l
# essentially just inversion operation - this *should* have good CC - unless
# we are working on a reference set where we don't reindex
if reference:
cb_op = sgtbx.change_of_basis_op('x,y,z')
else:
cb_op = sgtbx.change_of_basis_op('-x,-y,-z')
for h in range(-gh, gh + 1):
for k in range(-gk, gk + 1):
for l in range(-gl, gl + 1):
miller_indices = offset_miller_indices(ms.indices(), (h, k, l))
reindexed_miller_indices = cb_op.apply(miller_indices)
rms = miller_set(cs, reindexed_miller_indices).array(data)
if reference_ms:
_ms = reference_ms
else:
_ms = miller_set(cs, miller_indices).array(data)
if map_to_asu:
rms = rms.map_to_asu()
_ms = _ms.map_to_asu()
intensity, intensity_rdx = rms.common_sets(_ms)
cc = intensity.correlation(intensity_rdx).coefficient()
ccs.append(cc)
offsets.append((h, k, l))
nref.append(intensity.size())
return offsets, ccs, nref
def test_crystal_pointgroup_symmetry(reflections, experiment, params):
crystal = experiment.crystal
from dials.array_family import flex
# in case we pass in reflections from integration
reflections = reflections.select(reflections['intensity.sum.variance'] > 0)
reflections = reflections.select(reflections['intensity.sum.value'] > 0)
original_miller_indices = reflections['miller_index']
space_group = crystal.get_space_group()
unit_cell = crystal.get_unit_cell()
from cctbx.crystal import symmetry as crystal_symmetry
cs = crystal_symmetry(unit_cell, space_group.type().lookup_symbol())
from cctbx.miller import set as miller_set
ms = miller_set(cs, original_miller_indices)
ms = ms.array(reflections['intensity.sum.value'] /
flex.sqrt(reflections['intensity.sum.variance']))
if params.d_min or params.d_max:
d_spacings = ms.d_spacings().data()
sel = (d_spacings >= params.d_min) & (d_spacings <= params.d_max)
ms = ms.select(sel)
reflections = reflections.select(sel)
if params.normalise:
if params.normalise_bins:
ms = normalise_intensities(ms, n_bins=params.normalise_bins)
else:
ms = normalise_intensities(ms)
logger.info('Check symmetry operations on %d reflections:' % ms.size())
logger.info('')
logger.info('%20s %6s %5s' % ('Symop', 'Nref', 'CC'))
true_symops = []
ccs, n_refs = get_symop_correlation_coefficients(ms)
for smx, cc, n_ref in zip(space_group.smx(), ccs, n_refs):
accept = ''
if params.symop_threshold:
if cc > params.symop_threshold:
true_symops.append(smx)
accept = '***'
logger.info('%20s %6d %.3f %s' % (smx, n_ref, cc, accept))
if params.symop_threshold:
from cctbx.sgtbx import space_group as sgtbx_space_group
sg = sgtbx_space_group()
for symop in true_symops:
sg = sg.expand_smx(symop)
for ltr in space_group.ltr():
sg = sg.expand_ltr(ltr)
sg_symbols = sg.match_tabulated_settings()
logger.info('')
logger.info('Derived point group from symmetry operations: %s' % \
sg_symbols.hermann_mauguin())
logger.info('')
return
def cctbx_crystal_from_dials(crystal):
space_group = crystal.get_space_group()
unit_cell = crystal.get_unit_cell()
from cctbx.crystal import symmetry as crystal_symmetry
return crystal_symmetry(unit_cell, space_group.type().lookup_symbol())
def test_crystal_pointgroup_symmetry(reflections, experiment, params):
crystal = experiment.crystal
from dials.array_family import flex
# in case we pass in reflections from integration
reflections = reflections.select(reflections["intensity.sum.variance"] > 0)
reflections = reflections.select(reflections["intensity.sum.value"] > 0)
original_miller_indices = reflections["miller_index"]
space_group = crystal.get_space_group()
unit_cell = crystal.get_unit_cell()
from cctbx.crystal import symmetry as crystal_symmetry
cs = crystal_symmetry(unit_cell, space_group.type().lookup_symbol())
from cctbx.miller import set as miller_set
ms = miller_set(cs, original_miller_indices)
ms = ms.array(reflections["intensity.sum.value"] / flex.sqrt(reflections["intensity.sum.variance"]))
if params.d_min or params.d_max:
d_spacings = ms.d_spacings().data()
sel = (d_spacings >= params.d_min) & (d_spacings <= params.d_max)
ms = ms.select(sel)
reflections = reflections.select(sel)
if params.normalise:
if params.normalise_bins:
ms = normalise_intensities(ms, n_bins=params.normalise_bins)
else:
ms = normalise_intensities(ms)
print "Check symmetry operations on %d reflections:" % ms.size()
print ""
print "%20s %6s %5s" % ("Symop", "Nref", "CC")
true_symops = []
ccs, n_refs = get_symop_correlation_coefficients(ms)
for smx, cc, n_ref in zip(space_group.smx(), ccs, n_refs):
accept = ""
if params.symop_threshold:
if cc > params.symop_threshold:
true_symops.append(smx)
accept = "***"
print "%20s %6d %.3f %s" % (smx, n_ref, cc, accept)
if params.symop_threshold:
from cctbx.sgtbx import space_group as sgtbx_space_group
sg = sgtbx_space_group()
for symop in true_symops:
sg = sg.expand_smx(symop)
for ltr in space_group.ltr():
sg = sg.expand_ltr(ltr)
sg_symbols = sg.match_tabulated_settings()
print ""
print "Derived point group from symmetry operations: %s" % sg_symbols.hermann_mauguin()
print ""
return
def cctbx_crystal_from_dials(crystal): space_group = crystal.get_space_group() unit_cell = crystal.get_unit_cell() from cctbx.crystal import symmetry as crystal_symmetry return crystal_symmetry(unit_cell, space_group.type().lookup_symbol())
def get_indexing_offset_correlation_coefficients(
reflections, crystal, grid, d_min=None, d_max=None,
map_to_asu=False, grid_h=0, grid_k=0, grid_l=0, reference=None):
from dials.algorithms.symmetry import origin
if grid:
if grid_h == 0: grid_h = grid
if grid_k == 0: grid_k = grid
if grid_l == 0: grid_l = grid
if True:
return origin.get_hkl_offset_correlation_coefficients(
reflections, crystal, map_to_asu=map_to_asu,
grid_h=grid_h, grid_k=grid_k, grid_l=grid_l, reference=reference)
from copy import deepcopy
from dials.array_family import flex
space_group = crystal.get_space_group()
unit_cell = crystal.get_unit_cell()
from cctbx.crystal import symmetry as crystal_symmetry
cs = crystal_symmetry(unit_cell, space_group.type().lookup_symbol())
from cctbx.miller import set as miller_set
data = reflections['intensity.sum.value'] / \
flex.sqrt(reflections['intensity.sum.variance'])
if reference:
reference = reference.select(reference['intensity.sum.variance'] > 0)
reference_data = reference['intensity.sum.value'] / \
flex.sqrt(reference['intensity.sum.variance'])
reference_ms = miller_set(cs, reference['miller_index']).array(
reference_data)
else:
reference_ms = None
ccs = flex.double()
offsets = flex.vec3_int()
nref = flex.size_t()
original_miller_indices = reflections['miller_index']
ms = miller_set(cs, original_miller_indices).array(data)
if d_min is not None or d_max is not None:
ms = ms.resolution_filter(d_min=d_min, d_max=d_max)
gh = gk = gl = grid
if grid_h: gh = grid_h
if grid_k: gk = grid_k
if grid_l: gl = grid_l
# essentially just inversion operation - this *should* have good CC - unless
# we are working on a reference set where we don't reindex
if reference:
cb_op = sgtbx.change_of_basis_op('x,y,z')
else:
cb_op = sgtbx.change_of_basis_op('-x,-y,-z')
for h in range(-gh, gh + 1):
for k in range(-gk, gk + 1):
for l in range(-gl, gl + 1):
miller_indices = offset_miller_indices(ms.indices(), (h, k, l))
reindexed_miller_indices = cb_op.apply(miller_indices)
rms = miller_set(cs, reindexed_miller_indices).array(data)
if reference_ms:
_ms = reference_ms
else:
_ms = miller_set(cs, miller_indices).array(data)
if map_to_asu:
rms = rms.map_to_asu()
_ms = _ms.map_to_asu()
intensity, intensity_rdx = rms.common_sets(_ms)
cc = intensity.correlation(intensity_rdx).coefficient()
ccs.append(cc)
offsets.append((h, k, l))
nref.append(intensity.size())
return offsets, ccs, nref
def run(self):
"""Parse the options."""
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
self.params = params
assert (len(params.input.experiments) == len(params.input.reflections)
== 1), "Provide one experiment list and one reflection table"
assert params.method == "summed_intensity"
experiments = params.input.experiments[0].data
reflections = params.input.reflections[0].data
# Find the aveage unit cell for the crystals in the experiments provided
weighted_relfs = flex.reflection_table()
all_uc = [flex.double() for i in xrange(6)]
space_group = None
for expt_id, experiment in enumerate(experiments):
refls = reflections.select(reflections["id"] == expt_id)
unit_cell = experiment.crystal.get_unit_cell()
for i in xrange(6):
all_uc[i].append(unit_cell.parameters()[i])
if space_group is None:
space_group = experiment.crystal.get_space_group()
else:
assert (space_group.type().lookup_symbol() == experiment.
crystal.get_space_group().type().lookup_symbol())
# Compute the average unit cell and build a miller array with it
unit_cell = uctbx.unit_cell([flex.mean(data) for data in all_uc])
cs = crystal_symmetry(unit_cell, space_group.type().lookup_symbol())
ms = miller_set(cs, reflections["miller_index"], anomalous_flag=False)
ma = ms.array(reflections["intensity.sum.value"] /
flex.sqrt(reflections["intensity.sum.variance"]))
ma.setup_binner(n_bins=10)
binner = ma.binner()
mean_i = flex.double()
reflections["delpsical.weights"] = flex.double(len(reflections), 0)
# Iterate through the bins and compute the Wilson plot, then use it compute the weights
for i in binner.range_all():
sel = binner.selection(i)
if sel.count(True) == 0:
mean_i.append(0)
continue
mean_i.append(
flex.mean(reflections["intensity.sum.value"].select(sel)))
reflections["delpsical.weights"].set_selected(
sel,
reflections["intensity.sum.value"].select(sel) *
(params.summed_intensity.scale_factor / mean_i[i]),
)
if params.show_weight_plots:
fig = plt.figure()
plt.title(str(i))
plt.hist(reflections["delpsical.weights"].select(sel))
# Show unit cell distribution and mean I
print("Average uc +/- std. deviation")
labels = [
"% 6s" % l for l in ["a", "b", "c", "alpha", "beta", "gamma"]
]
for label, data in zip(labels, all_uc):
stats = flex.mean_and_variance(data)
print("%s % 6.1f +/- %6.1f" %
(label, stats.mean(),
stats.unweighted_sample_standard_deviation()))
print("Mean I over all data")
binner.show_data(mean_i, data_fmt="%.1f", show_unused=False)
easy_pickle.dump(params.output.reflections, reflections)
if params.show_weight_plots:
plt.show()
