def __init__(self, strategies, n_bins=8, degrees_per_bin=5):
from cctbx import crystal, miller
import copy
sg = strategies[0].experiment.crystal.get_space_group() \
.build_derived_reflection_intensity_group(anomalous_flag=True)
cs = crystal.symmetry(
unit_cell=strategies[0].experiment.crystal.get_unit_cell(), space_group=sg)
for i, strategy in enumerate(strategies):
if i == 0:
predicted = copy.deepcopy(strategy.predicted)
else:
predicted_ = copy.deepcopy(strategy.predicted)
predicted_['dose'] += (flex.max(predicted['dose']) + 1)
predicted.extend(predicted_)
ms = miller.set(cs, indices=predicted['miller_index'], anomalous_flag=True)
ma = miller.array(ms, data=flex.double(ms.size(),1),
sigmas=flex.double(ms.size(), 1))
if 1:
merging = ma.merge_equivalents()
o = merging.array().customized_copy(
data=merging.redundancies().data().as_double()).as_mtz_dataset('I').mtz_object()
o.write('predicted.mtz')
d_star_sq = ma.d_star_sq().data()
binner = ma.setup_binner_d_star_sq_step(
d_star_sq_step=(flex.max(d_star_sq)-flex.min(d_star_sq)+1e-8)/n_bins)
dose = predicted['dose']
range_width = 1
range_min = flex.min(dose) - range_width
range_max = flex.max(dose)
n_steps = 2 + int((range_max - range_min) - range_width)
binner_non_anom = ma.as_non_anomalous_array().use_binning(
binner)
self.n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])
from xia2.Modules.PyChef2 import ChefStatistics
chef_stats = ChefStatistics(
ma.indices(), ma.data(), ma.sigmas(),
ma.d_star_sq().data(), dose, self.n_complete, binner,
ma.space_group(), ma.anomalous_flag(), n_steps)
def fraction_new(completeness):
# Completeness so far at end of image
completeness_end = completeness[1:]
# Completeness so far at start of image
completeness_start = completeness[:-1]
# Fraction of unique reflections observed for the first time on each image
return completeness_end - completeness_start
self.dose = dose
self.ieither_completeness = chef_stats.ieither_completeness()
self.iboth_completeness = chef_stats.iboth_completeness()
self.frac_new_ref = fraction_new(self.ieither_completeness) / degrees_per_bin
self.frac_new_pairs = fraction_new(self.iboth_completeness) / degrees_per_bin
def __init__(self, strategies, n_bins=8, degrees_per_bin=5):
from cctbx import crystal, miller
import copy
sg = strategies[0].experiment.crystal.get_space_group() \
.build_derived_reflection_intensity_group(anomalous_flag=True)
cs = crystal.symmetry(
unit_cell=strategies[0].experiment.crystal.get_unit_cell(), space_group=sg)
for i, strategy in enumerate(strategies):
if i == 0:
predicted = copy.deepcopy(strategy.predicted)
else:
predicted_ = copy.deepcopy(strategy.predicted)
predicted_['dose'] += (flex.max(predicted['dose']) + 1)
predicted.extend(predicted_)
ms = miller.set(cs, indices=predicted['miller_index'], anomalous_flag=True)
ma = miller.array(ms, data=flex.double(ms.size(),1),
sigmas=flex.double(ms.size(), 1))
if 1:
merging = ma.merge_equivalents()
o = merging.array().customized_copy(
data=merging.redundancies().data().as_double()).as_mtz_dataset('I').mtz_object()
o.write('predicted.mtz')
d_star_sq = ma.d_star_sq().data()
binner = ma.setup_binner_d_star_sq_step(
d_star_sq_step=(flex.max(d_star_sq)-flex.min(d_star_sq)+1e-8)/n_bins)
dose = predicted['dose']
range_width = 1
range_min = flex.min(dose) - range_width
range_max = flex.max(dose)
n_steps = 2 + int((range_max - range_min) - range_width)
binner_non_anom = ma.as_non_anomalous_array().use_binning(
binner)
self.n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])
from xia2.Modules.PyChef2 import ChefStatistics
chef_stats = ChefStatistics(
ma.indices(), ma.data(), ma.sigmas(),
ma.d_star_sq().data(), dose, self.n_complete, binner,
ma.space_group(), ma.anomalous_flag(), n_steps)
def fraction_new(completeness):
# Completeness so far at end of image
completeness_end = completeness[1:]
# Completeness so far at start of image
completeness_start = completeness[:-1]
# Fraction of unique reflections observed for the first time on each image
return completeness_end - completeness_start
self.dose = dose
self.ieither_completeness = chef_stats.ieither_completeness()
self.iboth_completeness = chef_stats.iboth_completeness()
self.frac_new_ref = fraction_new(self.ieither_completeness) / degrees_per_bin
self.frac_new_pairs = fraction_new(self.iboth_completeness) / degrees_per_bin
def __init__(self, intensities, dose, n_bins=8,
range_min=None, range_max=None, range_width=1):
if isinstance(dose, flex.double):
sorted_dose = flex.sorted(dose)
dd = sorted_dose[1:] - sorted_dose[:-1]
dd = dd.select(dd > 0)
if len(dd):
step_size = flex.min(dd)
dose /= step_size
dose = dose.iround()
if flex.min(dose) == 0:
dose += 1
# fix for completeness > 1 if screw axes present
intensities = intensities.customized_copy(
space_group_info=intensities.space_group().build_derived_reflection_intensity_group(
anomalous_flag=intensities.anomalous_flag()).info(),
info=intensities.info())
self.intensities = intensities
self.dose = dose
self.n_bins = n_bins
self.range_min = range_min
self.range_max = range_max
self.range_width = range_width
assert self.range_width > 0
if self.range_min is None:
self.range_min = flex.min(self.dose) - self.range_width
if self.range_max is None:
self.range_max = flex.max(self.dose)
self.n_steps = 2 + int((self.range_max - self.range_min) - self.range_width)
sel = (self.dose.as_double() <= self.range_max) & (self.dose.as_double() >= self.range_min)
self.dose = self.dose.select(sel)
self.intensities = self.intensities.select(sel)
self.d_star_sq = self.intensities.d_star_sq().data()
self.binner = self.intensities.setup_binner_d_star_sq_step(
d_star_sq_step=(flex.max(self.d_star_sq)-flex.min(self.d_star_sq)+1e-8)/self.n_bins)
#self.dose /= range_width
self.dose -= int(self.range_min)
self.dose = flex.size_t(list(self.dose))
binner_non_anom = intensities.as_non_anomalous_array().use_binning(
self.binner)
n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])
from xia2.Modules.PyChef2 import ChefStatistics
chef_stats = ChefStatistics(
intensities.indices(), intensities.data(), intensities.sigmas(),
intensities.d_star_sq().data(), self.dose, n_complete, self.binner,
intensities.space_group(), intensities.anomalous_flag(), self.n_steps)
self.iplus_comp_bins = chef_stats.iplus_completeness_bins()
self.iminus_comp_bins = chef_stats.iminus_completeness_bins()
self.ieither_comp_bins = chef_stats.ieither_completeness_bins()
self.iboth_comp_bins = chef_stats.iboth_completeness_bins()
self.iplus_comp_overall = chef_stats.iplus_completeness()
self.iminus_comp_overall = chef_stats.iminus_completeness()
self.ieither_comp_overall = chef_stats.ieither_completeness()
self.iboth_comp_overall = chef_stats.iboth_completeness()
self.rcp_bins = chef_stats.rcp_bins()
self.rcp = chef_stats.rcp()
self.scp_bins = chef_stats.scp_bins()
self.scp = chef_stats.scp()
self.rd = chef_stats.rd()
def __init__(self, intensities, dose, n_bins=8,
range_min=None, range_max=None, range_width=1):
if isinstance(dose, flex.double):
sorted_dose = flex.sorted(dose)
dd = sorted_dose[1:] - sorted_dose[:-1]
step_size = flex.min(dd.select(dd > 0))
dose /= step_size
dose = dose.iround()
if flex.min(dose) == 0:
dose += 1
# fix for completeness > 1 if screw axes present
intensities = intensities.customized_copy(
space_group_info=intensities.space_group().build_derived_reflection_intensity_group(
anomalous_flag=intensities.anomalous_flag()).info(),
info=intensities.info())
self.intensities = intensities
self.dose = dose
self.n_bins = n_bins
self.range_min = range_min
self.range_max = range_max
self.range_width = range_width
assert self.range_width > 0
if self.range_min is None:
self.range_min = flex.min(self.dose) - self.range_width
if self.range_max is None:
self.range_max = flex.max(self.dose)
self.n_steps = 2 + int((self.range_max - self.range_min) - self.range_width)
sel = (self.dose.as_double() <= self.range_max) & (self.dose.as_double() >= self.range_min)
self.dose = self.dose.select(sel)
self.intensities = self.intensities.select(sel)
self.d_star_sq = self.intensities.d_star_sq().data()
self.binner = self.intensities.setup_binner_d_star_sq_step(
d_star_sq_step=(flex.max(self.d_star_sq)-flex.min(self.d_star_sq)+1e-8)/self.n_bins)
#self.dose /= range_width
self.dose -= int(self.range_min)
self.dose = flex.size_t(list(self.dose))
binner_non_anom = intensities.as_non_anomalous_array().use_binning(
self.binner)
n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])
from xia2.Modules.PyChef2 import ChefStatistics
chef_stats = ChefStatistics(
intensities.indices(), intensities.data(), intensities.sigmas(),
intensities.d_star_sq().data(), self.dose, n_complete, self.binner,
intensities.space_group(), intensities.anomalous_flag(), self.n_steps)
self.iplus_comp_bins = chef_stats.iplus_completeness_bins()
self.iminus_comp_bins = chef_stats.iminus_completeness_bins()
self.ieither_comp_bins = chef_stats.ieither_completeness_bins()
self.iboth_comp_bins = chef_stats.iboth_completeness_bins()
self.iplus_comp_overall = chef_stats.iplus_completeness()
self.iminus_comp_overall = chef_stats.iminus_completeness()
self.ieither_comp_overall = chef_stats.ieither_completeness()
self.iboth_comp_overall = chef_stats.iboth_completeness()
self.rcp_bins = chef_stats.rcp_bins()
self.rcp = chef_stats.rcp()
self.scp_bins = chef_stats.scp_bins()
self.scp = chef_stats.scp()
self.rd = chef_stats.rd()
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
from dials.util.options import flatten_reflections
import libtbx.load_env
usage = "%s [options] experiments.json | integrated.pickle" % (
libtbx.env.dispatcher_name)
parser = OptionParser(usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) == 0 or len(reflections) == 0:
parser.print_help()
exit()
reflections = reflections[0]
cryst = experiments.crystals()[0]
unit_cell = cryst.get_unit_cell()
if params.space_group is not None:
space_group = params.space_group.group()
assert space_group.is_compatible_unit_cell(unit_cell), unit_cell
else:
space_group = cryst.get_space_group()
print space_group.info()
print unit_cell
expt = experiments[0]
from cctbx import miller, crystal
from mmtbx.scaling.data_statistics import wilson_scaling
sel = reflections.get_flags(reflections.flags.integrated_sum)
reflections = reflections.select(sel)
cs = crystal.symmetry(unit_cell=unit_cell, space_group=space_group)
ms = miller.set(cs,
indices=reflections['miller_index'],
anomalous_flag=True)
intensities = miller.array(ms,
data=reflections['intensity.sum.value'],
sigmas=flex.sqrt(
reflections['intensity.sum.variance']))
intensities.set_observation_type_xray_intensity()
d_star_sq = intensities.d_star_sq().data()
n_bins = 20
# binner = intensities.setup_binner_d_star_sq_step(
# d_star_sq_step=(flex.max(d_star_sq)-flex.min(d_star_sq)+1e-8)/n_bins)
binner = intensities.setup_binner_counting_sorted(n_bins=n_bins)
# wilson = intensities.wilson_plot(use_binning=True)
# wilson.show()
# from matplotlib import pyplot
# pyplot.figure()
# pyplot.scatter(wilson.binner.bin_centers(2), wilson.data[1:-1])
# pyplot.show()
intensities = intensities.merge_equivalents().array()
wilson = wilson_scaling(intensities, n_residues=200)
wilson.iso_scale_and_b.show()
from matplotlib import pyplot
pyplot.figure()
pyplot.scatter(wilson.d_star_sq, wilson.mean_I_obs_data, label='Data')
pyplot.plot(wilson.d_star_sq, wilson.mean_I_obs_theory, label='theory')
pyplot.plot(wilson.d_star_sq,
wilson.mean_I_normalisation,
label='smoothed')
pyplot.yscale('log')
pyplot.legend()
pyplot.show()
import copy
import math
# Hack to make the predicter predict reflections outside of the range
# of the scan
expt_input = copy.deepcopy(expt)
scan = expt.scan
image_range = scan.get_image_range()
oscillation = scan.get_oscillation()
scan.set_image_range((1, int(math.ceil(360 / oscillation[1]))))
scan.set_oscillation((0, oscillation[1]))
print scan
# Populate the reflection table with predictions
predicted = flex.reflection_table.from_predictions(
expt, force_static=params.force_static, dmin=params.d_min)
predicted['id'] = flex.int(len(predicted), 0)
print len(predicted)
space_group = space_group.build_derived_reflection_intensity_group(
anomalous_flag=True)
cs = crystal.symmetry(unit_cell=unit_cell, space_group=space_group)
ms = miller.set(cs, indices=predicted['miller_index'], anomalous_flag=True)
ma = miller.array(ms,
data=flex.double(ms.size(), 1),
sigmas=flex.double(ms.size(), 1))
d_star_sq = ma.d_star_sq().data()
n_bins = 1
binner = ma.setup_binner_d_star_sq_step(
d_star_sq_step=(flex.max(d_star_sq) - flex.min(d_star_sq) + 1e-8) /
n_bins)
image_number = predicted['xyzcal.px'].parts()[2]
print flex.min(image_number)
print flex.max(image_number)
#dose = flex.size_t(list(flex.floor(image_number).iround()))
angle_deg = predicted['xyzcal.mm'].parts()[2] * 180 / math.pi
dose = flex.size_t(list(flex.floor(angle_deg).iround()))
range_width = 1
range_min = flex.min(dose) - range_width
range_max = flex.max(dose)
n_steps = 2 + int((range_max - range_min) - range_width)
binner_non_anom = ma.as_non_anomalous_array().use_binning(binner)
n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])
ranges_dict = {}
completeness_levels = [10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 99]
for c in completeness_levels:
ranges_dict[c] = []
from xia2.Modules.PyChef2 import ChefStatistics
step = 1
for i in range(0, 360, step):
sel = dose < step
dose.set_selected(sel, dose.select(sel) + 360)
dose -= flex.min(dose)
chef_stats = ChefStatistics(ma.indices(), ma.data(), ma.sigmas(),
ma.d_star_sq().data(), dose, n_complete,
binner, ma.space_group(),
ma.anomalous_flag(), n_steps)
ieither_completeness = chef_stats.ieither_completeness()
iboth_completeness = chef_stats.iboth_completeness()
for c in completeness_levels:
ranges_dict[c].append(
min((ieither_completeness > (c / 100)).iselection()))
from matplotlib import pyplot
pyplot.figure()
for c in completeness_levels:
pyplot.plot(ranges_dict[c], label=str(c))
# pyplot.plot(range_for_50)
# pyplot.plot(range_for_99)
# pyplot.scatter(range(iboth_completeness.size()), iboth_completeness)
pyplot.legend()
pyplot.show()
def test_exercise_accumulators(xia2_regression):
from xia2.Modules.PyChef2 import PyChef
from xia2.Modules.PyChef2 import ChefStatistics
from iotbx.reflection_file_reader import any_reflection_file
from cctbx.array_family import flex
from libtbx.test_utils import approx_equal
import os
f = os.path.join(xia2_regression, "test/insulin_dials_scaled_unmerged.mtz")
reader = any_reflection_file(f)
assert reader.file_type() == 'ccp4_mtz'
arrays = reader.as_miller_arrays(merge_equivalents=False)
for ma in arrays:
if ma.info().labels == ['BATCH']:
batches = ma
elif ma.info().labels == ['I', 'SIGI']:
intensities = ma
elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
intensities = ma
assert intensities is not None
assert batches is not None
anomalous_flag = True
if anomalous_flag:
intensities = intensities.as_anomalous_array()
pystats = PyChef.PyStatistics(intensities, batches.data())
miller_indices = batches.indices()
sg = batches.space_group()
n_steps = pystats.n_steps
dose = batches.data()
range_width = 1
range_max = flex.max(dose)
range_min = flex.min(dose) - range_width
dose /= range_width
dose -= range_min
binner_non_anom = intensities.as_non_anomalous_array().use_binning(
pystats.binner)
n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])
dose = flex.size_t(list(dose))
chef_stats = ChefStatistics(miller_indices, intensities.data(),
intensities.sigmas(),
intensities.d_star_sq().data(), dose,
n_complete, pystats.binner, sg, anomalous_flag,
n_steps)
# test completeness
assert approx_equal(chef_stats.iplus_completeness(),
pystats.iplus_comp_overall)
assert approx_equal(chef_stats.iminus_completeness(),
pystats.iminus_comp_overall)
assert approx_equal(chef_stats.ieither_completeness(),
pystats.ieither_comp_overall)
assert approx_equal(chef_stats.iboth_completeness(),
pystats.iboth_comp_overall)
# test rcp,scp
assert approx_equal(chef_stats.rcp(), pystats.rcp)
assert approx_equal(chef_stats.scp(), pystats.scp)
# test Rd
assert approx_equal(chef_stats.rd(), pystats.rd)
def exercise_accumulators():
from xia2.Modules.PyChef2 import PyChef
from xia2.Modules.PyChef2 import ChefStatistics
from iotbx.reflection_file_reader import any_reflection_file
from cctbx.array_family import flex
import libtbx.load_env
import os
xia2_regression = libtbx.env.find_in_repositories("xia2_regression")
if xia2_regression is None:
print "Skipping exercise_accumulators(): xia2_regression not available"
return
f = os.path.join(xia2_regression, "test/insulin_dials_scaled_unmerged.mtz")
reader = any_reflection_file(f)
assert reader.file_type() == 'ccp4_mtz'
arrays = reader.as_miller_arrays(merge_equivalents=False)
for ma in arrays:
if ma.info().labels == ['BATCH']:
batches = ma
elif ma.info().labels == ['I', 'SIGI']:
intensities = ma
elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
intensities = ma
assert intensities is not None
assert batches is not None
anomalous_flag = True
if anomalous_flag:
intensities = intensities.as_anomalous_array()
pystats = PyChef.PyStatistics(intensities, batches.data())
miller_indices = batches.indices()
sg = batches.space_group()
n_steps = pystats.n_steps
dose = batches.data()
range_width = 1
range_max = flex.max(dose)
range_min = flex.min(dose) - range_width
dose /= range_width
dose -= range_min
binner_non_anom = intensities.as_non_anomalous_array().use_binning(
pystats.binner)
n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])
dose = flex.size_t(list(dose))
chef_stats = ChefStatistics(
miller_indices, intensities.data(), intensities.sigmas(),
intensities.d_star_sq().data(), dose, n_complete, pystats.binner,
sg, anomalous_flag, n_steps)
# test completeness
assert approx_equal(chef_stats.iplus_completeness(), pystats.iplus_comp_overall)
assert approx_equal(chef_stats.iminus_completeness(), pystats.iminus_comp_overall)
assert approx_equal(chef_stats.ieither_completeness(), pystats.ieither_comp_overall)
assert approx_equal(chef_stats.iboth_completeness(), pystats.iboth_comp_overall)
# test rcp,scp
assert approx_equal(chef_stats.rcp(), pystats.rcp)
assert approx_equal(chef_stats.scp(), pystats.scp)
# test Rd
assert approx_equal(chef_stats.rd(), pystats.rd)
print "OK"