def with_random_intensity(self, N, Imax, Bamax, Bbmax, Bcmax, Bdmax):
""" Generate reflections with a random intensity and background. """
from dials.array_family import flex
if Imax == 0:
I = flex.size_t(N).as_int()
else:
I = flex.random_size_t(N, Imax).as_int()
if Bamax == 0:
Ba = flex.size_t(N).as_int()
else:
Ba = flex.random_size_t(N, Bamax).as_int()
if Bbmax == 0:
Bb = flex.size_t(N).as_int()
else:
Bb = flex.random_size_t(N, Bbmax).as_int()
if Bcmax == 0:
Bc = flex.size_t(N).as_int()
else:
Bc = flex.random_size_t(N, Bcmax).as_int()
if Bdmax == 0:
Bd = flex.size_t(N).as_int()
else:
Bd = flex.random_size_t(N, Bdmax).as_int()
return self.with_individual_given_intensity(N, I, Ba, Bb, Bc, Bd)
def setup_test_sorting():
# Borrowed from tst_reflection_table function tst_find_overlapping
N = 110
r = flex.reflection_table.empty_standard(N)
r["panel"] = flex.size_t([1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0] * 10)
r["id"] = flex.int([1, 2, 1, 1, 2, 0, 1, 1, 1, 0, 1] * 10)
exp_ids = flex.size_t([0, 1])
for i in range(N):
r["miller_index"][i] = (
int(i // 10) - 5,
i % 3,
i % 7,
) # A nice bunch of miller indices
# Filter out reflections to be used by refinement. Sorting of filtered reflections
# require to allow C++ extension modules to give performance benefit. Sorting
# performed within the _filter_reflections step by id, then by panel.
r_sorted = copy.deepcopy(r)
r_sorted.sort("id")
r_sorted.subsort("id", "panel")
# Test that the unfiltered/unsorted table becomes filtered/sorted for id
assert (r_sorted["id"] == r["id"].select(flex.sort_permutation(
r["id"]))).count(False) == 0
# as above for panel within each id
for ii in [0, 1, 2]:
r_id = r.select(r["id"] == ii)
r_sorted_id = r_sorted.select(r_sorted["id"] == ii)
assert (r_sorted_id["panel"] == r_id["panel"].select(
flex.sort_permutation(r_id["panel"]))).count(False) == 0
return (r, r_sorted, exp_ids)
def __call__(self, observed, predicted):
'''i
Match the observed reflections with the predicted.
:param observed: The list of observed reflections.
:param predicted: The list of predicted reflections.
:returns: The list of matched reflections
'''
from dials.array_family import flex
# Find the nearest neighbours and distances
# Command.start('Finding nearest neighbours')
nn, dist = self._find_nearest_neighbours(observed, predicted)
# Command.end('Found nearest neighbours')
# Filter the matches by distance
# Command.start('Filtering matches by distance')
index = self._filter_by_distance(nn, dist)
# Command.end('Filtered {0} matches by distance'.format(len(index)))
# Filter out duplicates to just leave the closest pairs
# Command.start('Removing duplicate matches')
len_index = len(index)
index = self._filter_duplicates(index, nn, dist)
len_diff = len_index - len(index)
# Command.end('Removed {0} duplicate match(es)'.format(len_diff))
# Copy all of the reflection data for the matched reflections
return flex.size_t(index), flex.size_t([nn[i] for i in index])
def test_auto_reduction_parameter_extension_modules_part2(setup_test_sorting):
# Cut-down original algorithm for AutoReduce._unit_cell_surplus_reflections
from dials_refinement_helpers_ext import uc_surpl_iter as uc_surpl
r, r_sorted, exp_ids = setup_test_sorting
isel = flex.size_t()
for exp_id in exp_ids:
isel.extend((r["id"] == exp_id).iselection())
ref = r.select(isel)
h = ref["miller_index"].as_vec3_double()
dB_dp = flex.mat3_double([(1, 2, 3, 4, 5, 6, 7, 8, 9),
(0, 1, 0, 1, 0, 1, 0, 1, 0)])
nref_each_param = []
for der in dB_dp:
tst = (der * h).norms()
nref_each_param.append((tst > 0.0).count(True))
res0 = min(nref_each_param)
# Updated algorithm for _unit_cell_surplus_reflections
res1_unsrt_int = uc_surpl(r["id"], r["miller_index"], exp_ids,
dB_dp).result
res1_int = uc_surpl(r_sorted["id"], r_sorted["miller_index"], exp_ids,
dB_dp).result
res1_sizet = uc_surpl(flex.size_t(list(r_sorted["id"])),
r_sorted["miller_index"], exp_ids, dB_dp).result
assert res0 != res1_unsrt_int
assert res0 == res1_int
assert res0 == res1_sizet
def __call__(self, observed, predicted):
'''i
Match the observed reflections with the predicted.
:param observed: The list of observed reflections.
:param predicted: The list of predicted reflections.
:returns: The list of matched reflections
'''
from dials.array_family import flex
# Find the nearest neighbours and distances
# Command.start('Finding nearest neighbours')
nn, dist = self._find_nearest_neighbours(observed, predicted)
# Command.end('Found nearest neighbours')
# Filter the matches by distance
# Command.start('Filtering matches by distance')
index = self._filter_by_distance(nn, dist)
# Command.end('Filtered {0} matches by distance'.format(len(index)))
# Filter out duplicates to just leave the closest pairs
# Command.start('Removing duplicate matches')
len_index = len(index)
index = self._filter_duplicates(index, nn, dist)
len_diff = len_index - len(index)
# Command.end('Removed {0} duplicate match(es)'.format(len_diff))
# Copy all of the reflection data for the matched reflections
return flex.size_t(index), flex.size_t([nn[i] for i in index])
def import_integrated(integrate_hkl, min_ios=3):
reader = integrate_hkl_as_flex.reader(integrate_hkl, "IOBS,SIGMA,XCAL,YCAL,ZCAL,XOBS,YOBS,ZOBS,ISEG".split(","))
# reference: dials/command_line/import_xds.py
table = flex.reflection_table()
table["id"] = flex.int(len(reader.hkl), 0)
table["panel"] = flex.size_t(len(reader.hkl), 0) # only assuming single panel
table["miller_index"] = reader.hkl
table["xyzcal.px.value"] = flex.vec3_double(reader.data["XCAL"], reader.data["YCAL"], reader.data["ZCAL"])
table["xyzobs.px.value"] = flex.vec3_double(reader.data["XOBS"], reader.data["YOBS"], reader.data["ZOBS"])
table["intensity.cor.value"] = reader.data["IOBS"]
table["intensity.cor.sigma"] = reader.data["SIGMA"] # not valid name, just for making selection
table["flags"] = flex.size_t(len(table), table.flags.indexed | table.flags.strong)
table = table.select(table["intensity.cor.sigma"] > 0)
table = table.select(table["intensity.cor.value"]/table["intensity.cor.sigma"] >= min_ios)
table = table.select(table["xyzobs.px.value"].norms() > 0) # remove invalid xyzobs
# dummy
table["xyzobs.px.variance"] = flex.vec3_double(len(table), (1,1,1)) # TODO appropriate variance value
table["s1"] = flex.vec3_double(len(table), (0,0,0)) # will be updated by set_obs_s1()
del table["intensity.cor.value"]
del table["intensity.cor.sigma"]
return table
def test_select_reflections_for_sigma_calc():
"""Test the reflection selection helper function."""
# first select all if below threshold
reflections = flex.reflection_table()
reflections["id"] = flex.int(range(0, 1000))
reflections.set_flags(flex.bool(1000, True),
reflections.flags.used_in_refinement)
reflections = _select_reflections_for_sigma_calc(reflections, 10000)
assert reflections.size() == 1000
# select used_in_refinement if not all used in refinement and above threshold
reflections = flex.reflection_table()
reflections["id"] = flex.int(range(0, 1000))
good = flex.bool(1000, False)
sel = flex.size_t(i for i in range(0, 1000, 2))
good.set_selected(sel, True)
reflections.set_flags(good, reflections.flags.used_in_refinement)
reflections = _select_reflections_for_sigma_calc(reflections,
min_number_of_refl=200)
assert reflections.size() == 500
assert list(reflections["id"])[0:50] == list(range(0, 100, 2))
# top up if not enough used in refinement compared to threshold
reflections = flex.reflection_table()
reflections["id"] = flex.int(range(0, 1000))
good = flex.bool(1000, False)
sel = flex.size_t(i for i in range(0, 1000, 2))
good.set_selected(sel, True)
reflections.set_flags(good, reflections.flags.used_in_refinement)
reflections = _select_reflections_for_sigma_calc(reflections,
min_number_of_refl=700)
assert reflections.size() > 700
assert reflections.size() < 1000
def test_vs_old(data):
from dials.array_family import flex
r_old = data.reflections
r_new = data.predict_new()
index1 = flex.size_t(
sorted(range(len(r_old)), key=lambda x: r_old["miller_index"][x])
)
index2 = flex.size_t(
sorted(range(len(r_new)), key=lambda x: r_new["miller_index"][x])
)
r_old = r_old.select(index1)
r_new = r_new.select(index2)
assert len(r_old) == len(r_new)
for r1, r2 in zip(r_old.rows(), r_new.rows()):
assert r1["miller_index"] == r2["miller_index"]
assert r1["panel"] == r2["panel"]
assert r1["entering"] == r2["entering"]
assert all(a == pytest.approx(b, abs=1e-7) for a, b in zip(r1["s1"], r2["s1"]))
assert all(
a == pytest.approx(b, abs=1e-7)
for a, b in zip(r1["xyzcal.px"], r2["xyzcal.px"])
)
assert all(
a == pytest.approx(b, abs=1e-7)
for a, b in zip(r1["xyzcal.mm"], r2["xyzcal.mm"])
)
def with_random_intensity(self, N, Imax, Bamax, Bbmax, Bcmax, Bdmax):
""" Generate reflections with a random intensity and background. """
from dials.array_family import flex
if Imax == 0:
I = flex.size_t(N).as_int()
else:
I = flex.random_size_t(N, Imax).as_int()
if Bamax == 0:
Ba = flex.size_t(N).as_int()
else:
Ba = flex.random_size_t(N, Bamax).as_int()
if Bbmax == 0:
Bb = flex.size_t(N).as_int()
else:
Bb = flex.random_size_t(N, Bbmax).as_int()
if Bcmax == 0:
Bc = flex.size_t(N).as_int()
else:
Bc = flex.random_size_t(N, Bcmax).as_int()
if Bdmax == 0:
Bd = flex.size_t(N).as_int()
else:
Bd = flex.random_size_t(N, Bdmax).as_int()
return self.with_individual_given_intensity(N, I, Ba, Bb, Bc, Bd)
def test_with_old_index_generator(data):
from dials.algorithms.spot_prediction import ScanStaticReflectionPredictor
from dials.array_family import flex
predict = ScanStaticReflectionPredictor(data.experiments[0])
r_old = predict.for_ub_old_index_generator(
data.experiments[0].crystal.get_A())
r_new = predict.for_ub(data.experiments[0].crystal.get_A())
index1 = flex.size_t(
sorted(range(len(r_old)), key=lambda x: r_old["miller_index"][x]))
index2 = flex.size_t(
sorted(range(len(r_new)), key=lambda x: r_new["miller_index"][x]))
r_old = r_old.select(index1)
r_new = r_new.select(index2)
assert len(r_old) == len(r_new)
for r1, r2 in zip(r_old.rows(), r_new.rows()):
assert r1["miller_index"] == r2["miller_index"]
assert r1["panel"] == r2["panel"]
assert r1["entering"] == r2["entering"]
assert all(a == pytest.approx(b, abs=1e-7)
for a, b in zip(r1["s1"], r2["s1"]))
assert all(a == pytest.approx(b, abs=1e-7)
for a, b in zip(r1["xyzcal.px"], r2["xyzcal.px"]))
assert all(a == pytest.approx(b, abs=1e-7)
for a, b in zip(r1["xyzcal.mm"], r2["xyzcal.mm"]))
def generate_reflections(self):
"""Use reeke_model to generate indices of reflections near to the Ewald
sphere that might be observed on a still image. Build a reflection_table
of these."""
from cctbx.sgtbx import space_group_info
space_group_type = space_group_info("P 1").group().type()
# create a ReekeIndexGenerator
UB = self.crystal.get_A()
axis = self.goniometer.get_rotation_axis()
s0 = self.beam.get_s0()
dmin = 1.5
# use the same UB at the beginning and end - the margin parameter ensures
# we still have indices close to the Ewald sphere generated
from dials.algorithms.spot_prediction import ReekeIndexGenerator
r = ReekeIndexGenerator(UB, UB, space_group_type, axis, s0, dmin=1.5, margin=1)
# generate indices
hkl = r.to_array()
nref = len(hkl)
# create a reflection table
from dials.array_family import flex
table = flex.reflection_table()
table['flags'] = flex.size_t(nref, 0)
table['id'] = flex.int(nref, 0)
table['panel'] = flex.size_t(nref, 0)
table['miller_index'] = flex.miller_index(hkl)
table['entering'] = flex.bool(nref, True)
table['s1'] = flex.vec3_double(nref)
table['xyzcal.mm'] = flex.vec3_double(nref)
table['xyzcal.px'] = flex.vec3_double(nref)
return table
def tst_with_old_index_generator(self):
from dials.algorithms.spot_prediction import ScanStaticReflectionPredictor
from dials.array_family import flex
predict = ScanStaticReflectionPredictor(self.experiments[0])
r_old = predict.for_ub_old_index_generator(
self.experiments[0].crystal.get_A())
r_new = predict.for_ub(self.experiments[0].crystal.get_A())
index1 = flex.size_t(
sorted(range(len(r_old)), key=lambda x: r_old['miller_index'][x]))
index2 = flex.size_t(
sorted(range(len(r_new)), key=lambda x: r_new['miller_index'][x]))
r_old = r_old.select(index1)
r_new = r_new.select(index2)
assert (len(r_old) == len(r_new))
eps = 1e-7
for r1, r2 in zip(r_old.rows(), r_new.rows()):
assert (r1['miller_index'] == r2['miller_index'])
assert (r1['panel'] == r2['panel'])
assert (r1['entering'] == r2['entering'])
assert (all(abs(a - b) < eps for a, b in zip(r1['s1'], r2['s1'])))
assert (all(
abs(a - b) < eps
for a, b in zip(r1['xyzcal.px'], r2['xyzcal.px'])))
assert (all(
abs(a - b) < eps
for a, b in zip(r1['xyzcal.mm'], r2['xyzcal.mm'])))
print 'OK'
def generate_reflections(self):
"""Use reeke_model to generate indices of reflections near to the Ewald
sphere that might be observed on a still image. Build a reflection_table
of these."""
from cctbx.sgtbx import space_group_info
space_group_type = space_group_info("P 1").group().type()
# create a ReekeIndexGenerator
UB = self.crystal.get_U() * self.crystal.get_B()
axis = self.goniometer.get_rotation_axis()
s0 = self.beam.get_s0()
dmin = 1.5
# use the same UB at the beginning and end - the margin parameter ensures
# we still have indices close to the Ewald sphere generated
from dials.algorithms.spot_prediction import ReekeIndexGenerator
r = ReekeIndexGenerator(UB, UB, space_group_type, axis, s0, dmin=1.5, margin=1)
# generate indices
hkl = r.to_array()
nref = len(hkl)
# create a reflection table
from dials.array_family import flex
table = flex.reflection_table()
table['flags'] = flex.size_t(nref, 0)
table['id'] = flex.int(nref, 0)
table['panel'] = flex.size_t(nref, 0)
table['miller_index'] = flex.miller_index(hkl)
table['entering'] = flex.bool(nref, True)
table['s1'] = flex.vec3_double(nref)
table['xyzcal.mm'] = flex.vec3_double(nref)
table['xyzcal.px'] = flex.vec3_double(nref)
return table
def main(params):
# first generate the reflections - this could be called from elsewhere
rlist = simple_gaussian_spots(params)
correct_intensities = list(rlist["intensity.sum.value"])
del rlist["intensity.sum.value"]
# now integrate those reflections using code from elsewhere
if params.background_method == "xds":
background_xds(rlist)
elif params.background_method == "mosflm":
assert params.pixel_mask != "all"
background_inclined(rlist)
integrate_3d_summation(rlist)
integrated_intensities = list(rlist["intensity.sum.value"])
# now scan through the reflection list and find those where the integration
# gave an apparently duff answer i.e. outside of 3 sigma from correct value
from dials.array_family import flex
overestimates = []
underestimates = []
for j, (c, i) in enumerate(zip(correct_intensities, integrated_intensities)):
sigma = math.sqrt(c)
if math.fabs(c - i) < 3 * sigma:
continue
if i > params.counts:
overestimates.append(j)
else:
underestimates.append(j)
print(
"%d overestimates, %d underestimates"
% (len(overestimates), len(underestimates))
)
overestimates = rlist.select(flex.size_t(overestimates))
underestimates = rlist.select(flex.size_t(underestimates))
# now pickle these, perhaps
import six.moves.cPickle as pickle
if params.output.under:
with open(params.output.under, "wb") as fh:
pickle.dump(underestimates, fh, pickle.HIGHEST_PROTOCOL)
if params.output.over:
with open(params.output.over, "wb") as fh:
pickle.dump(overestimates, fh, pickle.HIGHEST_PROTOCOL)
if params.output.all:
with open(params.output.all, "wb") as fh:
pickle.dump(rlist, fh, pickle.HIGHEST_PROTOCOL)
def merging_reflection_table():
table = flex.reflection_table()
table['miller_index'] = flex.miller_index()
table['miller_index_original'] = flex.miller_index()
table['scaled_intensity'] = flex.double()
table['isigi'] = flex.double()
table['slope'] = flex.double()
table['miller_id'] = flex.size_t()
table['crystal_id'] = flex.size_t()
table['iobs'] = flex.double()
return table
def __init__(self):
from dials.array_family import flex
self.reflections = flex.reflection_table()
self.reflections['panel'] = flex.size_t()
self.reflections['bbox'] = flex.int6()
self.reflections['miller_index'] = flex.miller_index()
self.reflections['s1'] = flex.vec3_double()
self.reflections['xyzcal.px'] = flex.vec3_double()
self.reflections['xyzcal.mm'] = flex.vec3_double()
self.reflections['entering'] = flex.bool()
self.reflections['id'] = flex.int()
self.reflections["flags"] = flex.size_t()
self.npanels = 2
self.width = 1000
self.height = 1000
self.nrefl = 10000
self.array_range = (0, 130)
self.block_size = 20
from random import randint, seed, choice
seed(0)
self.processed = [[] for i in range(12)]
for i in range(self.nrefl):
x0 = randint(0, self.width - 10)
y0 = randint(0, self.height - 10)
zs = randint(2, 9)
x1 = x0 + randint(2, 10)
y1 = y0 + randint(2, 10)
for k, j in enumerate(
[10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]):
m = k + i * 12
pos = choice(["left", "right", "centre"])
if pos == 'left':
z0 = j - zs
z1 = j
elif pos == 'right':
z0 = j
z1 = j + zs
else:
z0 = j - zs // 2
z1 = j + zs // 2
bbox = (x0, x1, y0, y1, z0, z1)
self.reflections.append({
"panel":
randint(0, 1),
"bbox":
bbox,
"flags":
flex.reflection_table.flags.reference_spot
})
self.processed[k].append(m)
def main(params):
# first generate the reflections - this could be called from elsewhere
rlist = simple_gaussian_spots(params)
correct_intensities = [r['intensity.sum.value'] for r in rlist]
del r['intensity.sum.value']
# now integrate those reflections using code from elsewhere
if params.background_method == 'xds':
background_xds(rlist)
elif params.background_method == 'mosflm':
assert (params.pixel_mask != 'all')
background_inclined(rlist)
integrate_3d_summation(rlist)
integrated_intensities = [r['intensity.sum.value'] for r in rlist]
# now scan through the reflection list and find those where the integration
# gave an apparently duff answer i.e. outside of 3 sigma from correct value
from dials.array_family import flex
import math
overestimates = []
underestimates = []
for j, (c, i) in enumerate(zip(correct_intensities,
integrated_intensities)):
sigma = math.sqrt(c)
if math.fabs(c - i) < 3 * sigma:
continue
if i > params.counts:
overestimates.append(j)
else:
underestimates.append(j)
print '%d overestimates, %d underestimates' % (len(overestimates),
len(underestimates))
overestimates = rlist.select(flex.size_t(overestimates))
underestimates = rlist.select(flex.size_t(underestimates))
# now pickle these, perhaps
import cPickle as pickle
if params.output.under:
pickle.dump(underestimates, open(params.output.under, 'w'))
if params.output.over:
pickle.dump(overestimates, open(params.output.over, 'w'))
if params.output.all:
pickle.dump(rlist, open(params.output.all, 'w'))
def main():
parser = OptionParser(read_experiments=True, read_reflections=True)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
data = reflections[0]
expt = experiments[0]
data = data.select(data.get_flags(data.flags.integrated))
# data = data.select(data['d'] > 2.0)
z = data["xyzcal.px"].parts()[2]
shoebox = data["shoebox"]
# make a new reflection table, this one empty - then add columns for
# every event in this set (with some shuffle) - flags = 32 apparently,
# id == id above, intensity.sum.value=1 variance=1 n_signal=1 panel=panel
# xyzobs.px.value = pixel + random.random() - 0.5 variance = 1/12 in each
# direction - then map to reciprocal space
events = flex.vec3_double()
for s in shoebox:
d = s.data
k0, j0, i0 = s.bbox[0], s.bbox[2], s.bbox[4]
k1, j1, i1 = d.focus()
for k in range(k1):
for j in range(j1):
for i in range(i1):
for n in range(int(d[k, j, i])):
if random.random() > 0.1:
continue
z = k + k0 + random.random()
y = j + j0 + random.random()
x = i + i0 + random.random()
events.append((z, y, x))
rt = flex.reflection_table()
rt["xyzobs.px.value"] = events
variance = flex.double(events.size(), 1.0 / 12.0)
rt["xyzobs.px.variance"] = flex.vec3_double(variance, variance, variance)
rt["flags"] = flex.size_t(events.size(), 32)
rt["id"] = flex.int(events.size(), 0)
rt["panel"] = flex.size_t(events.size(), 0)
rt["intensity.sum.value"] = flex.double(events.size(), 1)
rt["intensity.sum.variance"] = flex.double(events.size(), 1)
rt.as_file("events.refl")
def extract_free_set(self, free_set_percentage, offset=0):
"""Extract a free set from all blocks."""
assert not self.free_Ih_table
interval_between_groups = int(100 / free_set_percentage)
free_reflection_table = flex.reflection_table()
free_indices = flex.size_t()
for j, block in enumerate(self.Ih_table_blocks):
n_groups = block.h_index_matrix.n_cols
groups_for_free_set = flex.bool(n_groups, False)
for_free = flex.size_t([
i for i in range(0 + offset, n_groups, interval_between_groups)
])
groups_for_free_set.set_selected(for_free, True)
free_block = block.select_on_groups(groups_for_free_set)
free_reflection_table.extend(free_block.Ih_table)
for sel in free_block.block_selections:
free_indices.extend(sel)
self.Ih_table_blocks[j] = block.select_on_groups(
~groups_for_free_set)
# Now need to update dataset_info dict.
removed_from_each_dataset = [
(free_block.Ih_table["dataset_id"] == i).count(True)
for i in range(0, block.n_datasets)
]
n_removed = 0
for i in range(0, self.Ih_table_blocks[j].n_datasets):
self.Ih_table_blocks[j].dataset_info[i][
"start_index"] -= n_removed
n_removed += removed_from_each_dataset[i]
self.Ih_table_blocks[j].dataset_info[i][
"end_index"] -= n_removed
self.blocked_selection_list = [
block.block_selections for block in self.Ih_table_blocks
]
# now split by dataset and use to instantiate another Ih_table
datasets = set(free_reflection_table["dataset_id"])
tables = []
indices_lists = []
for id_ in datasets:
dataset_sel = free_reflection_table["dataset_id"] == id_
tables.append(free_reflection_table.select(dataset_sel))
indices_lists.append(free_indices.select(dataset_sel))
free_Ih_table = IhTable(tables,
self.space_group,
indices_lists,
nblocks=1,
anomalous=self.anomalous)
# add to blocks list and selection list
self.Ih_table_blocks.append(free_Ih_table.blocked_data_list[0])
self.blocked_selection_list.append(
free_Ih_table.blocked_selection_list[0])
def __call__(self, imageset, pixel_labeller):
"""
Convert the pixel list to shoeboxes
"""
from dxtbx.imageset import ImageSequence
# Extract the pixel lists into a list of reflections
shoeboxes = flex.shoebox()
spotsizes = flex.size_t()
hotpixels = tuple(flex.size_t() for i in range(len(imageset.get_detector())))
if isinstance(imageset, ImageSequence):
scan = imageset.get_scan()
if scan.is_still():
twod = True
else:
twod = False
else:
twod = True
for i, (p, hp) in enumerate(zip(pixel_labeller, hotpixels)):
if p.num_pixels() > 0:
creator = flex.PixelListShoeboxCreator(
p,
i, # panel
0, # zrange
twod, # twod
self.min_spot_size, # min_pixels
self.max_spot_size, # max_pixels
self.write_hot_pixel_mask,
)
shoeboxes.extend(creator.result())
spotsizes.extend(creator.spot_size())
hp.extend(creator.hot_pixels())
logger.info("")
logger.info("Extracted {} spots".format(len(shoeboxes)))
# Get the unallocated spots and print some info
selection = shoeboxes.is_allocated()
shoeboxes = shoeboxes.select(selection)
ntoosmall = (spotsizes < self.min_spot_size).count(True)
ntoolarge = (spotsizes > self.max_spot_size).count(True)
assert ntoosmall + ntoolarge == selection.count(False)
logger.info(
"Removed %d spots with size < %d pixels" % (ntoosmall, self.min_spot_size)
)
logger.info(
"Removed %d spots with size > %d pixels" % (ntoolarge, self.max_spot_size)
)
# Return the shoeboxes
return shoeboxes, hotpixels
def main(params):
# first generate the reflections - this could be called from elsewhere
rlist = simple_gaussian_spots(params)
correct_intensities = [r['intensity.sum.value'] for r in rlist]
del r['intensity.sum.value']
# now integrate those reflections using code from elsewhere
if params.background_method == 'xds':
background_xds(rlist)
elif params.background_method == 'mosflm':
assert(params.pixel_mask != 'all')
background_inclined(rlist)
integrate_3d_summation(rlist)
integrated_intensities = [r['intensity.sum.value'] for r in rlist]
# now scan through the reflection list and find those where the integration
# gave an apparently duff answer i.e. outside of 3 sigma from correct value
from dials.array_family import flex
import math
overestimates = []
underestimates = []
for j, (c, i) in enumerate(zip(correct_intensities, integrated_intensities)):
sigma = math.sqrt(c)
if math.fabs(c - i) < 3 * sigma:
continue
if i > params.counts:
overestimates.append(j)
else:
underestimates.append(j)
print '%d overestimates, %d underestimates' % (len(overestimates),
len(underestimates))
overestimates = rlist.select(flex.size_t(overestimates))
underestimates = rlist.select(flex.size_t(underestimates))
# now pickle these, perhaps
import cPickle as pickle
if params.output.under:
pickle.dump(underestimates, open(params.output.under, 'w'))
if params.output.over:
pickle.dump(overestimates, open(params.output.over, 'w'))
if params.output.all:
pickle.dump(rlist, open(params.output.all, 'w'))
def __init__(self):
from dials.array_family import flex
self.reflections = flex.reflection_table()
self.reflections['panel'] = flex.size_t()
self.reflections['bbox'] = flex.int6()
self.reflections['miller_index'] = flex.miller_index()
self.reflections['s1'] = flex.vec3_double()
self.reflections['xyzcal.px'] = flex.vec3_double()
self.reflections['xyzcal.mm'] = flex.vec3_double()
self.reflections['entering'] = flex.bool()
self.reflections['id'] = flex.int()
self.reflections["flags"] = flex.size_t()
self.npanels = 2
self.width = 1000
self.height = 1000
self.nrefl = 10000
self.array_range = (0, 130)
self.block_size = 20
from random import randint, seed, choice
seed(0)
self.processed = [[] for i in range(12)]
for i in range(self.nrefl):
x0 = randint(0, self.width-10)
y0 = randint(0, self.height-10)
zs = randint(2, 9)
x1 = x0 + randint(2, 10)
y1 = y0 + randint(2, 10)
for k, j in enumerate([10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]):
m = k + i * 12
pos = choice(["left", "right", "centre"])
if pos == 'left':
z0 = j - zs
z1 = j
elif pos == 'right':
z0 = j
z1 = j + zs
else:
z0 = j - zs // 2
z1 = j + zs // 2
bbox = (x0, x1, y0, y1, z0, z1)
self.reflections.append({
"panel" : randint(0,1),
"bbox" : bbox,
"flags" : flex.reflection_table.flags.reference_spot
})
self.processed[k].append(m)
def pair_up(reference, moving, params, R0, t0):
from annlib_ext import AnnAdaptor as ann_adaptor
from dials.array_family import flex
rxyz = reference['xyzobs.px.value'].parts()
mxyz = moving['xyzobs.px.value'].parts()
# apply R0, t0 before performing matching - so should ideally be in almost
# right position
rxy = flex.vec2_double(rxyz[0], rxyz[1])
_mxy = flex.vec2_double(mxyz[0], mxyz[1])
mxy = flex.vec2_double()
for __mxy in _mxy:
mxy.append((R0 * __mxy + t0).elems)
ann = ann_adaptor(rxy.as_double().as_1d(), 2)
ann.query(mxy.as_double().as_1d())
distances = flex.sqrt(ann.distances)
matches = (distances < params.far)
rsel = flex.size_t()
msel = flex.size_t()
xyr = flex.vec2_double()
xym = flex.vec2_double()
for j in range(matches.size()):
if not matches[j]:
continue
msel.append(j)
rsel.append(ann.nn[j])
xym.append(mxy[j])
xyr.append(rxy[ann.nn[j]])
# filter outliers - use IQR etc.
dxy = xym - xyr
dx, dy = dxy.parts()
iqx = IQR(dx.select(flex.sort_permutation(dx)))
iqy = IQR(dy.select(flex.sort_permutation(dy)))
keep_x = (dx > (iqx[0] - iqx[3])) & (dx < (iqx[2] + iqx[3]))
keep_y = (dy > (iqy[0] - iqy[3])) & (dy < (iqy[2] + iqy[3]))
keep = keep_x & keep_y
return rsel.select(keep), msel.select(keep)
def intensities(self):
''' Compare the intensities. '''
from dials.array_family import flex
# Sort by resolution
d = self.refl1['d']
index = flex.size_t(reversed(sorted(range(len(d)), key=lambda x: d[x])))
self.refl1.reorder(index)
self.refl2.reorder(index)
# Get the intensities
I1 = self.refl1['intensity.sum.value']
I2 = self.refl2['intensity.sum.value']
S1 = flex.sqrt(self.refl1['intensity.sum.variance'])
S2 = flex.sqrt(self.refl2['intensity.sum.variance'])
xyz1 = self.refl1['xyzcal.px']
xyz2 = self.refl2['xyzcal.px']
# Compute chunked statistics
corr = []
R = []
scale = []
res = []
for i in range(len(self.refl1) // 1000):
# Get the chunks of data
a = i * 1000
b = (i+1) * 1000
II1 = I1[a:b]
II2 = I2[a:b]
res.append(d[a])
# Compute the mean and standard deviation per chunk
mv1 = flex.mean_and_variance(II1)
mv2 = flex.mean_and_variance(II2)
m1 = mv1.mean()
m2 = mv2.mean()
s1 = mv1.unweighted_sample_standard_deviation()
s2 = mv2.unweighted_sample_standard_deviation()
# compute the correlation coefficient
r = (1/(len(II1) - 1))*sum(((II1[j] - m1) / s1) * ((II2[j] - m2) / s2)
for j in range(len(II1)))
corr.append(r)
# Compute the scale between the chunks
s = sum(II1) / sum(II2)
scale.append(s)
# Compute R between the chunks
r = sum(abs(abs(II1[j]) - abs(s * II2[j])) for j in range(len(II1))) \
/ sum(abs(II1[j]) for j in range(len(II1)))
R.append(r)
from matplotlib import pylab
pylab.plot(corr, label="CC")
pylab.plot(R, label="R")
pylab.plot(scale, label="K")
pylab.legend()
pylab.show()
def _create_block_columns(self):
"""Create a column to contain the block number."""
from scitbx.array_family import flex
self._reflections["block"] = flex.size_t(len(self._reflections))
self._reflections["block_centre"] = flex.double(len(self._reflections))
def match_Ih_values_to_target(self, target_Ih_table):
"""
Use an Ih_table as a target to set Ih values in this table.
Given an Ih table as a target, the common reflections across the tables
are determined and the Ih_values are set to those of the target. If no
matching reflection is found, then the values are removed from the table.
"""
assert target_Ih_table.n_work_blocks == 1
target_asu_Ih_dict = dict(
zip(
target_Ih_table.blocked_data_list[0].asu_miller_index,
target_Ih_table.blocked_data_list[0].Ih_values,
))
new_Ih_values = flex.double(self.size, 0.0)
location_in_unscaled_array = 0
sorted_asu_indices, permuted = get_sorted_asu_indices(
self.Ih_table["asu_miller_index"], target_Ih_table.space_group)
for j, miller_idx in enumerate(OrderedSet(sorted_asu_indices)):
n_in_group = self.h_index_matrix.col(j).non_zeroes
if miller_idx in target_asu_Ih_dict:
i = location_in_unscaled_array
new_Ih_values.set_selected(
flex.size_t(range(i, i + n_in_group)),
flex.double(n_in_group, target_asu_Ih_dict[miller_idx]),
)
location_in_unscaled_array += n_in_group
self.Ih_values.set_selected(permuted, new_Ih_values)
sel = self.Ih_values != 0.0
new_table = self.select(sel)
# now set attributes to update object
self.Ih_table = new_table.Ih_table
self.h_index_matrix = new_table.h_index_matrix
self.h_expand_matrix = new_table.h_expand_matrix
self.block_selections = new_table.block_selections
def generate_exp_list(params, all_exp, all_ref):
if params.n_subset is not None:
subset_all_exp = []
subset_all_ref = []
n_picked = 0
if params.n_subset_method=="random":
while n_picked < params.n_subset:
idx = random.randint(0, len(all_exp)-1)
subset_all_exp.append(all_exp.pop(idx))
subset_all_ref.append(all_ref.pop(idx))
n_picked += 1
elif params.n_subset_method=="n_refl":
from dials.array_family import flex
import cPickle as pickle
len_all_ref = flex.size_t(
[ len(pickle.load(open(A,"rb"))) for A in all_ref ]
)
sort_order = flex.sort_permutation(len_all_ref,reverse=True)
for idx in sort_order[:params.n_subset]:
subset_all_exp.append(all_exp[idx])
subset_all_ref.append(all_ref[idx])
print "Selecting a subset of %d images with highest n_refl out of %d total."%(
params.n_subset, len(len_all_ref))
all_exp = subset_all_exp
all_ref = subset_all_ref
return all_exp, all_ref
def test_multiple_panels():
from dials.array_family import flex
nrefl = 1000
# Generate bboxes
bbox = flex.int6(nrefl)
panel = flex.size_t(nrefl)
for i in range(nrefl):
x0 = random.randint(0, 500)
y0 = random.randint(0, 500)
z0 = random.randint(0, 10)
x1 = x0 + random.randint(2, 10)
y1 = y0 + random.randint(2, 10)
z1 = z0 + random.randint(2, 10)
bbox[i] = (x0, x1, y0, y1, z0, z1)
panel[i] = random.randint(0, 2)
# Find the overlaps
overlaps = find_overlapping(bbox, panel)
assert overlaps.num_vertices() == nrefl
overlaps2 = brute_force(bbox, panel)
assert overlaps.num_edges() == len(overlaps2)
edges = {}
for edge in overlaps2:
edge = (min(edge), max(edge))
edges[edge] = None
for edge in overlaps.edges():
edge = (overlaps.source(edge), overlaps.target(edge))
edge = (min(edge), max(edge))
assert edge in edges
def compute_overall_stats(self):
# Create lookups for elements by miller index
index_lookup = defaultdict(list)
for i, h in enumerate(self.reflection_table["miller_index"]):
index_lookup[h].append(i)
# Compute the Overall Sum(X) and Sum(X^2) for each unique reflection
for h in index_lookup:
sel = flex.size_t(index_lookup[h])
intensities = self.reflection_table["intensity"].select(sel)
n = intensities.size()
sum_x = flex.sum(intensities)
sum_x2 = flex.sum(flex.pow2(intensities))
self.reflection_sums[h] = ReflectionSum(sum_x, sum_x2, n)
# Compute some numbers
self._num_datasets = len(set(self.reflection_table["dataset"]))
self._num_groups = len(set(self.reflection_table["group"]))
self._num_reflections = self.reflection_table.size()
self._num_unique = len(self.reflection_sums)
logger.info(
"""
Summary of input data:
# Datasets: %s
# Groups: %s
# Reflections: %s
# Unique reflections: %s""",
self._num_datasets,
self._num_groups,
self._num_reflections,
self._num_unique,
)
def generate_spots(crystal_model, detector, beam, goniometer=None, scan=None,
sel_fraction=1.0):
import math
experiment = Experiment(beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal_model)
# if we don't set the imageset then from_predictions uses the StillsReflectionPredictor :-(
from dxtbx.imageset import NullReader, ImageSweep
imageset = ImageSweep(NullReader, indices=range(len(scan.get_epochs())), beam=beam, goniometer=goniometer,
detector=detector, scan=scan)
experiment.imageset = imageset
predicted = flex.reflection_table.from_predictions(experiment)
sel = flex.random_selection(len(predicted),
int(math.floor(sel_fraction*len(predicted))))
predicted = predicted.select(sel)
predicted['imageset_id'] = flex.size_t(len(predicted), 0)
predicted['xyzobs.px.value'] = predicted['xyzcal.px']
predicted['xyzobs.px.variance'] = flex.vec3_double(
len(predicted), (0.5,0.5,0.5))
return predicted
def ersatz_MCMC(self, variable_params):
from LS49.adse13_187.adse13_221.case_run import case_job_runner
class ersatz(MCMC_manager, case_job_runner):
pass
self.MCMC = ersatz()
self.MCMC.get_amplitudes(self.dials_model, self.refl_table)
relevant_whitelist_order = flex.size_t()
for sidx in range(len(
self.refl_table["spots_offset"])): #loop through the shoeboxes
for pidx in range(
self.refl_table["spots_offset"][sidx],
self.refl_table["spots_offset"][sidx] +
self.refl_table["spots_size"][sidx]):
relevant_whitelist_order.append(self.spots_pixels[pidx])
self.MCMC.set_whitelist(relevant_whitelist_order)
modality = "job"
return self.MCMC.job_runner(
expt=self.expt,
alt_expt=self.dials_model,
params=variable_params,
mask_array=self.monolithic_mask_whole_detector_as_1D_bool
) # returns simulated image as numpy array
def _xyzcal_mm_to_px(self, experiments, reflections):
# set xyzcal.px field in reflections
reflections["xyzcal.px"] = flex.vec3_double(len(reflections))
for i, expt in enumerate(experiments):
imgset_sel = reflections["imageset_id"] == i
refined_reflections = reflections.select(imgset_sel)
panel_numbers = flex.size_t(refined_reflections["panel"])
xyzcal_mm = refined_reflections["xyzcal.mm"]
x_mm, y_mm, z_rad = xyzcal_mm.parts()
xy_cal_mm = flex.vec2_double(x_mm, y_mm)
xy_cal_px = flex.vec2_double(len(xy_cal_mm))
for i_panel in range(len(expt.detector)):
panel = expt.detector[i_panel]
sel = panel_numbers == i_panel
xy_cal_px.set_selected(
sel, panel.millimeter_to_pixel(xy_cal_mm.select(sel))
)
x_px, y_px = xy_cal_px.parts()
if expt.scan is not None:
z_px = expt.scan.get_array_index_from_angle(z_rad, deg=False)
else:
# must be a still image, z centroid not meaningful
z_px = z_rad
xyzcal_px = flex.vec3_double(x_px, y_px, z_px)
reflections["xyzcal.px"].set_selected(imgset_sel, xyzcal_px)
def model(self, reflections, profiles):
from dials.array_family import flex
indices = flex.size_t(range(len(self)))
weights = flex.double([1.0] * len(self))
for profile in profiles:
self.add(indices, weights, profile)
def display(self, num):
"""
Display some shoeboxes
"""
from dials_scratch.jmp.viewer import show_image_stack_multi_view
from random import sample
from dials.array_family import flex
def simulate(experiments, reflections, parameters):
from dials_scratch.jmp.profile_modelling import MLTarget3D
func = MLTarget3D(experiments[0], reflections)
return [
func.simulate(i, parameters) for i in range(len(reflections))
]
# Sample from reflections
reflections = self.reflections.select(
flex.size_t(sample(range(len(self.reflections)), num)))
# Simulate the reflection profiles from the current model
simulated = simulate(self.experiments, reflections, self.parameters)
# Display stuff
for model, data_sbox in zip(simulated, reflections["shoebox"]):
data = data_sbox.data
show_image_stack_multi_view(model.as_numpy_array(),
vmax=flex.max(model))
show_image_stack_multi_view(data.as_numpy_array(),
vmax=flex.max(data))
def generate_exp_list(params, all_exp, all_ref):
if params.n_subset is not None:
subset_all_exp = []
subset_all_ref = []
n_picked = 0
if params.n_subset_method == "random":
while n_picked < params.n_subset:
idx = random.randint(0, len(all_exp) - 1)
subset_all_exp.append(all_exp.pop(idx))
subset_all_ref.append(all_ref.pop(idx))
n_picked += 1
elif params.n_subset_method == "n_refl":
from dials.array_family import flex
import cPickle as pickle
len_all_ref = flex.size_t(
[len(pickle.load(open(A, "rb"))) for A in all_ref])
sort_order = flex.sort_permutation(len_all_ref, reverse=True)
for idx in sort_order[:params.n_subset]:
subset_all_exp.append(all_exp[idx])
subset_all_ref.append(all_ref[idx])
print "Selecting a subset of %d images with highest n_refl out of %d total." % (
params.n_subset, len(len_all_ref))
all_exp = subset_all_exp
all_ref = subset_all_ref
return all_exp, all_ref
def scale_partial_reflections(integrated_data, min_partiality=0.5):
'''Scale partial reflections (after summation) according to their estimated
partiality - for profile fitted reflections this will result in no change,
for summation integrated reflections will be scaled up by 1 / partiality
with error accordingly scaled. N.B. this will report the scaled up partiality
for the output reflection.'''
# assert: in here there will be no multi-part partial reflections
if not 'partiality' in integrated_data:
return integrated_data
isel = (integrated_data['partiality'] < 1.0).iselection()
if len(isel) == 0:
return integrated_data
delete = flex.size_t()
for j in isel:
if integrated_data['partiality'][j] < min_partiality:
delete.append(j)
continue
inv_p = 1.0 / integrated_data['partiality'][j]
integrated_data['intensity.sum.value'][j] *= inv_p
integrated_data['intensity.sum.variance'][j] *= inv_p
integrated_data['partiality'][j] *= 1.0
integrated_data.del_selected(delete)
return integrated_data
def show_image(c,b,s, BB=None, SS=None):
import numpy.ma
from dials.array_family import flex
N = 100
im = flex.double(flex.grid(N, N))
mask = flex.bool(flex.grid(N, N))
for j in range(N):
for i in range(N):
B = -1.0 + j * 10.0 / N
S = -1.0 + i * 10.0 / N
im[j,i], mask[j,i] = func(c,b,s,B,S)
im[j,i] = -im[j,i]
masked_im = numpy.ma.array(
# im.as_numpy_array(),
flex.exp(im).as_numpy_array(),
mask = mask.as_numpy_array())
mask2 = flex.bool(flex.grid(N, N))
indices = []
for i in range(len(mask)):
if mask[i] == False:
indices.append(i)
indices = flex.size_t(indices)
ind = flex.max_index(im.select(indices))
ind = indices[ind]
maxy = -1.0 + (ind % N) * 10.0 / N
maxx = -1.0 + (ind // N) * 10.0 / N
from matplotlib import pylab
pylab.imshow(masked_im, origin='bottom', extent=[-1.0, 9.0, -1.0, 9.0])
if YY is not None and XX is not None:
pylab.plot(YY, XX)
pylab.scatter([maxy], [maxx])
pylab.show()
def tst_multiple_panels(self):
from dials.array_family import flex
from random import randint
nrefl = 1000
# Generate bboxes
bbox = flex.int6(nrefl)
panel = flex.size_t(nrefl)
for i in range(nrefl):
x0 = randint(0, 500)
y0 = randint(0, 500)
z0 = randint(0, 10)
x1 = x0 + randint(2, 10)
y1 = y0 + randint(2, 10)
z1 = z0 + randint(2, 10)
bbox[i] = (x0, x1, y0, y1, z0, z1)
panel[i] = randint(0,2)
# Find the overlaps
overlaps = find_overlapping(bbox, panel)
assert(overlaps.num_vertices() == nrefl)
overlaps2 = self.brute_force(bbox, panel)
assert(overlaps.num_edges() == len(overlaps2))
edges = {}
for edge in overlaps2:
edge = (min(edge), max(edge))
edges[edge] = None
for edge in overlaps.edges():
edge = (overlaps.source(edge), overlaps.target(edge))
edge = (min(edge), max(edge))
assert(edge in edges)
print 'OK'
def test_match_mismatched_sizes():
n = 100
s = 10
def r():
return random.random()
xyz = flex.vec3_double()
for j in range(n):
xyz.append((r() * s, r() * s, r() * s * 20))
order = list(range(n))
random.shuffle(order)
xyz2 = xyz.select(flex.size_t(order))
a = flex.reflection_table()
b = flex.reflection_table()
a["xyz"] = xyz[:n // 2]
b["xyz"] = xyz2
mm, nn, distance = a.match(b, key="xyz", scale=(1.0, 1.0, 0.05))
a_ = a.select(mm)
b_ = b.select(nn)
for _a, _b in zip(a_["xyz"], b_["xyz"]):
assert _a == pytest.approx(_b)
def _grads_model_loop(
self,
parameterisations,
reflections,
results,
callback=None,
derivatives_fn=None,
):
# loop over the parameterisations
for p in parameterisations:
# Determine (sub)set of reflections affected by this parameterisation
isel = flex.size_t()
for exp_id in p.get_experiment_ids():
isel.extend(self._experiment_to_idx[exp_id])
# Extend derivative vectors for this parameterisation
results = self._extend_gradient_vectors(results,
self._nref,
p.num_free(),
keys=self._grad_names)
if len(isel) == 0:
# if no reflections are in this experiment, skip calculation of
# gradients, but must still process null gradients by a callback
if callback:
for _ in range(p.num_free()):
results[self._iparam] = callback(results[self._iparam])
self._iparam += 1
else:
self._iparam += p.num_free()
continue
w_inv = self._w_inv.select(isel)
u_w_inv = self._u_w_inv.select(isel)
v_w_inv = self._v_w_inv.select(isel)
dpv_dbeam_p, dAngle_dbeam_p = derivatives_fn(
isel, parameterisation=p, reflections=reflections)
# convert to dX/dp, dY/dp and assign the elements of the vectors
# corresponding to this experiment
dX_dbeam_p, dY_dbeam_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp(
w_inv, u_w_inv, v_w_inv, dpv_dbeam_p)
for dX, dY, dAngle in zip(dX_dbeam_p, dY_dbeam_p, dAngle_dbeam_p):
if dX is not None:
results[self._iparam][self._grad_names[0]].set_selected(
isel, dX)
if dY is not None:
results[self._iparam][self._grad_names[1]].set_selected(
isel, dY)
if dAngle is not None:
results[self._iparam][self._grad_names[2]].set_selected(
isel, dAngle)
if callback is not None:
results[self._iparam] = callback(results[self._iparam])
# increment the parameter index pointer
self._iparam += 1
return results
def scale_partial_reflections(integrated_data, min_partiality=0.5):
"""Scale partial reflections (after summation) according to their estimated
partiality - for profile fitted reflections this will result in no change,
for summation integrated reflections will be scaled up by 1 / partiality
with error accordingly scaled. N.B. this will report the scaled up partiality
for the output reflection."""
# assert: in here there will be no multi-part partial reflections
if not "partiality" in integrated_data:
return integrated_data
isel = (integrated_data["partiality"] < 1.0).iselection()
if len(isel) == 0:
return integrated_data
from dials.array_family import flex
delete = flex.size_t()
for j in isel:
if integrated_data["partiality"][j] < min_partiality:
delete.append(j)
continue
inv_p = 1.0 / integrated_data["partiality"][j]
integrated_data["intensity.sum.value"][j] *= inv_p
integrated_data["intensity.sum.variance"][j] *= inv_p
integrated_data["partiality"][j] *= 1.0
integrated_data.del_selected(delete)
return integrated_data
def generate_spots(crystal_model,
detector,
beam,
goniometer=None,
scan=None,
sel_fraction=1.0):
import math
experiment = Experiment(beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal_model)
# if we don't set the imageset then from_predictions uses the StillsReflectionPredictor :-(
from dxtbx.imageset import NullReader, ImageSweep
imageset = ImageSweep(NullReader,
indices=range(len(scan.get_epochs())),
beam=beam,
goniometer=goniometer,
detector=detector,
scan=scan)
experiment.imageset = imageset
predicted = flex.reflection_table.from_predictions(experiment)
sel = flex.random_selection(len(predicted),
int(math.floor(sel_fraction * len(predicted))))
predicted = predicted.select(sel)
predicted['imageset_id'] = flex.size_t(len(predicted), 0)
predicted['xyzobs.px.value'] = predicted['xyzcal.px']
predicted['xyzobs.px.variance'] = flex.vec3_double(len(predicted),
(0.5, 0.5, 0.5))
return predicted
def _create_block_columns(self):
"""Create a column to contain the block number."""
from scitbx.array_family import flex
self._reflections['block'] = flex.size_t(len(self._reflections))
self._reflections['block_centre'] = flex.double(len(self._reflections))
return
def __call__(self, imageset, pixel_labeller):
'''
Convert the pixel list to shoeboxes
'''
from dxtbx.imageset import ImageSweep
from dials.array_family import flex
# Extract the pixel lists into a list of reflections
shoeboxes = flex.shoebox()
spotsizes = flex.size_t()
hotpixels = tuple(flex.size_t() for i in range(len(imageset.get_detector())))
if isinstance(imageset, ImageSweep):
twod = False
else:
twod = True
for i, (p, hp) in enumerate(zip(pixel_labeller, hotpixels)):
if p.num_pixels() > 0:
creator = flex.PixelListShoeboxCreator(
p,
i, # panel
0, # zrange
twod, # twod
self.min_spot_size, # min_pixels
self.max_spot_size, # max_pixels
self.write_hot_pixel_mask)
shoeboxes.extend(creator.result())
spotsizes.extend(creator.spot_size())
hp.extend(creator.hot_pixels())
logger.info('')
logger.info('Extracted {0} spots'.format(len(shoeboxes)))
# Get the unallocated spots and print some info
selection = shoeboxes.is_allocated()
shoeboxes = shoeboxes.select(selection)
ntoosmall = (spotsizes < self.min_spot_size).count(True)
ntoolarge = (spotsizes > self.max_spot_size).count(True)
assert ntoosmall + ntoolarge == selection.count(False)
logger.info('Removed %d spots with size < %d pixels' % (
ntoosmall,
self.min_spot_size))
logger.info('Removed %d spots with size > %d pixels' % (
ntoolarge,
self.max_spot_size))
# Return the shoeboxes
return shoeboxes, hotpixels
def __init__(self, strategies, n_bins=8, degrees_per_bin=5):
from cctbx import crystal, miller
import copy
sg = strategies[0].experiment.crystal.get_space_group() \
.build_derived_reflection_intensity_group(anomalous_flag=True)
cs = crystal.symmetry(
unit_cell=strategies[0].experiment.crystal.get_unit_cell(), space_group=sg)
for i, strategy in enumerate(strategies):
if i == 0:
predicted = copy.deepcopy(strategy.predicted)
else:
predicted_ = copy.deepcopy(strategy.predicted)
predicted_['dose'] += (flex.max(predicted['dose']) + 1)
predicted.extend(predicted_)
ms = miller.set(cs, indices=predicted['miller_index'], anomalous_flag=True)
ma = miller.array(ms, data=flex.double(ms.size(),1),
sigmas=flex.double(ms.size(), 1))
if 1:
merging = ma.merge_equivalents()
o = merging.array().customized_copy(
data=merging.redundancies().data().as_double()).as_mtz_dataset('I').mtz_object()
o.write('predicted.mtz')
d_star_sq = ma.d_star_sq().data()
binner = ma.setup_binner_d_star_sq_step(
d_star_sq_step=(flex.max(d_star_sq)-flex.min(d_star_sq)+1e-8)/n_bins)
dose = predicted['dose']
range_width = 1
range_min = flex.min(dose) - range_width
range_max = flex.max(dose)
n_steps = 2 + int((range_max - range_min) - range_width)
binner_non_anom = ma.as_non_anomalous_array().use_binning(
binner)
self.n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])
from xia2.Modules.PyChef2 import ChefStatistics
chef_stats = ChefStatistics(
ma.indices(), ma.data(), ma.sigmas(),
ma.d_star_sq().data(), dose, self.n_complete, binner,
ma.space_group(), ma.anomalous_flag(), n_steps)
def fraction_new(completeness):
# Completeness so far at end of image
completeness_end = completeness[1:]
# Completeness so far at start of image
completeness_start = completeness[:-1]
# Fraction of unique reflections observed for the first time on each image
return completeness_end - completeness_start
self.dose = dose
self.ieither_completeness = chef_stats.ieither_completeness()
self.iboth_completeness = chef_stats.iboth_completeness()
self.frac_new_ref = fraction_new(self.ieither_completeness) / degrees_per_bin
self.frac_new_pairs = fraction_new(self.iboth_completeness) / degrees_per_bin
def __init__(self, reflections,
experiments,
nref_per_degree=None,
max_sample_size=None,
min_sample_size=0,
close_to_spindle_cutoff=0.1,
outlier_detector=None,
weighting_strategy_override=None,
verbosity=0):
# set verbosity
self._verbosity = verbosity
# keep track of models
self._experiments = experiments
goniometers = [e.goniometer for e in self._experiments]
self._axes = [matrix.col(g.get_rotation_axis()) if g else None for g in goniometers]
self._s0vecs = [matrix.col(e.beam.get_s0()) for e in self._experiments]
# keep track of the original indices of the reflections
reflections['iobs'] = flex.size_t_range(len(reflections))
# set up the reflection inclusion criteria
self._close_to_spindle_cutoff = close_to_spindle_cutoff #too close to spindle
self._outlier_detector = outlier_detector #for outlier rejection
self._nref_per_degree = nref_per_degree #random subsets
self._max_sample_size = max_sample_size #sample size ceiling
self._min_sample_size = min_sample_size #sample size floor
# exclude reflections that fail some inclusion criteria
refs_to_keep = self._id_refs_to_keep(reflections)
self._accepted_refs_size = len(refs_to_keep)
# set entering flags for all reflections
reflections['entering'] = calculate_entering_flags(reflections,
self._experiments)
# set observed frame numbers for all reflections if not already present
calculate_frame_numbers(reflections, self._experiments)
# reset all use flags
self.reset_accepted_reflections(reflections)
# put full list of indexed reflections aside and select only the reflections
# that were not excluded to manage
self._indexed = reflections
self._reflections = reflections.select(flex.size_t(refs_to_keep))
# set weights for all kept reflections
if weighting_strategy_override is not None:
self._weighting_strategy = weighting_strategy_override
self._weighting_strategy.calculate_weights(self._reflections)
# not known until the manager is finalised
self._sample_size = None
return
def __init__(self, experiment, other=None, d_min=None, unit_cell_scale=1, degrees_per_bin=5,
min_frac_new=0.001):
print experiment.goniometer
print experiment.scan
self.experiment = copy.deepcopy(experiment)
self.other = other
self.unit_cell_scale = unit_cell_scale
self.degrees_per_bin = degrees_per_bin
self.min_frac_new = min_frac_new
scan = self.experiment.scan
detector = self.experiment.detector
beam = self.experiment.beam
crystal = self.experiment.crystal
from cctbx import uctbx
s = self.unit_cell_scale
assert s > 0
uc = crystal.get_unit_cell()
a, b, c, alpha, beta, gamma = uc.parameters()
uc_mod = uctbx.unit_cell((a/s, b/s, c/s, alpha, beta, gamma))
crystal.set_B(matrix.sqr(uc_mod.fractionalization_matrix()).transpose())
self.d_min = d_min
if self.d_min is None:
self.d_min = detector.get_max_inscribed_resolution(beam.get_s0())
import math
# bragg's law
sin_theta = 0.5 * beam.get_wavelength()/self.d_min
theta_max_rad = math.asin(sin_theta)
self.theta_max = theta_max_rad * 180/math.pi
print "theta_max (degrees): %.2f" %self.theta_max
Btot = 1 - 3 * (4 * theta_max_rad - math.sin(4 * theta_max_rad))/(32 * (math.sin(theta_max_rad)**3))
print Btot
# Section 2.9, Dauter Acta Cryst. (1999). D55, 1703-1717
#max_rotation = 360 + 2 * self.theta_max
max_rotation = 360
image_range = scan.get_image_range()
oscillation = scan.get_oscillation()
scan.set_image_range((image_range[0],
image_range[0]+int(max_rotation/oscillation[1])))
self.predict()
x, y, z = self.predicted['xyzcal.px'].parts()
self.predicted['dose'] = flex.size_t(
list(flex.ceil(z * oscillation[1]/self.degrees_per_bin).iround()))
self.stats = ComputeStats([self], degrees_per_bin=self.degrees_per_bin)
self.ieither_completeness = self.stats.ieither_completeness
self.iboth_completeness = self.stats.iboth_completeness
self.frac_new_ref = self.stats.frac_new_ref
self.frac_new_pairs = self.stats.frac_new_pairs
self.determine_cutoffs(self.min_frac_new)
self.show()
def __call__(self, params, options):
''' Import the integrate.hkl file. '''
from iotbx.xds import integrate_hkl
from dials.array_family import flex
from dials.util.command_line import Command
from cctbx import sgtbx
# Get the unit cell to calculate the resolution
uc = self._experiment.crystal.get_unit_cell()
# Read the INTEGRATE.HKL file
Command.start('Reading INTEGRATE.HKL')
handle = integrate_hkl.reader()
handle.read_file(self._integrate_hkl)
hkl = flex.miller_index(handle.hkl)
xyzcal = flex.vec3_double(handle.xyzcal)
xyzobs = flex.vec3_double(handle.xyzobs)
iobs = flex.double(handle.iobs)
sigma = flex.double(handle.sigma)
rlp = flex.double(handle.rlp)
peak = flex.double(handle.peak) * 0.01
Command.end('Read %d reflections from INTEGRATE.HKL file.' % len(hkl))
# Derive the reindex matrix
rdx = self.derive_reindex_matrix(handle)
print 'Reindex matrix:\n%d %d %d\n%d %d %d\n%d %d %d' % (rdx.elems)
# Reindex the reflections
Command.start('Reindexing reflections')
cb_op = sgtbx.change_of_basis_op(sgtbx.rt_mx(sgtbx.rot_mx(rdx.elems)))
hkl = cb_op.apply(hkl)
Command.end('Reindexed %d reflections' % len(hkl))
# Create the reflection list
Command.start('Creating reflection table')
table = flex.reflection_table()
table['id'] = flex.int(len(hkl), 0)
table['panel'] = flex.size_t(len(hkl), 0)
table['miller_index'] = hkl
table['xyzcal.px'] = xyzcal
table['xyzobs.px.value'] = xyzobs
table['intensity.cor.value'] = iobs
table['intensity.cor.variance'] = sigma**2
table['intensity.prf.value'] = iobs * peak / rlp
table['intensity.prf.variance'] = (sigma * peak / rlp)**2
table['lp'] = 1.0 / rlp
table['d'] = flex.double(uc.d(h) for h in hkl)
Command.end('Created table with {0} reflections'.format(len(table)))
# Output the table to pickle file
if params.output.filename is None:
params.output.filename = 'integrate_hkl.pickle'
Command.start('Saving reflection table to %s' % params.output.filename)
table.as_pickle(params.output.filename)
Command.end('Saved reflection table to %s' % params.output.filename)
def tst_find_overlapping(self):
from dials.array_family import flex
from random import randint, uniform
from dials.model.data import Shoebox
N = 10000
r = flex.reflection_table(N)
r['bbox'] = flex.int6(N)
r['panel'] = flex.size_t(N)
r['id'] = flex.int(N)
r['imageset_id'] = flex.int(N)
for i in range(N):
x0 = randint(0, 100)
x1 = randint(1, 10) + x0
y0 = randint(0, 100)
y1 = randint(1, 10) + y0
z0 = randint(0, 100)
z1 = randint(1, 10) + z0
panel = randint(0,2)
pid = randint(0,2)
r['bbox'][i] = (x0,x1,y0,y1,z0,z1)
r['panel'][i] = panel
r['id'][i] = pid
r['imageset_id'][i] = pid
def is_overlap(b0, b1, border):
b0 = b0[0]-border,b0[1]+border,b0[2]-border,b0[3]+border,b0[4]-border,b0[5]+border
b1 = b1[0]-border,b1[1]+border,b1[2]-border,b1[3]+border,b1[4]-border,b1[5]+border
if not (b1[0] > b0[1] or
b1[1] < b0[0] or
b1[2] > b0[3] or
b1[3] < b0[2] or
b1[4] > b0[5] or
b1[5] < b0[4]):
return True
return False
for i in [0, 2, 5]:
overlaps = r.find_overlaps(border=i)
for item in overlaps.edges():
i0 = overlaps.source(item)
i1 = overlaps.target(item)
r0 = r[i0]
r1 = r[i1]
p0 = r0['panel']
p1 = r1['panel']
b0 = r0['bbox']
b1 = r1['bbox']
j0 = r0['imageset_id']
j1 = r1['imageset_id']
assert j0 == j1
assert p0 == p1
assert is_overlap(b0,b1,i)
print 'OK'
def run(args):
import libtbx.load_env
usage = "%s experiments.json [options]" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0:
parser.print_help()
return
elif len(experiments) > 1:
raise Sorry("More than one experiment present")
assert len(params.miller_index), "Must specify at least one miller_index to predict."
experiment = experiments[0]
reflections = flex.reflection_table()
miller_indices = flex.miller_index()
entering_flags = flex.bool()
for mi in params.miller_index:
miller_indices.append(mi)
miller_indices.append(mi)
entering_flags.append(True)
entering_flags.append(False)
reflections['miller_index'] = miller_indices
reflections['entering'] = entering_flags
reflections['id'] = flex.size_t(len(reflections), 0)
if params.expand_to_p1:
from cctbx.miller import expand_to_p1_iselection
proxy = expand_to_p1_iselection(
experiment.crystal.get_space_group(),
anomalous_flag=True,
indices=miller_indices,
build_iselection=True)
reflections = reflections.select(proxy.iselection)
reflections['miller_index'] = proxy.indices
from dials.algorithms.refinement.prediction.managed_predictors import ExperimentsPredictor
predictor = ExperimentsPredictor([experiment])
predicted = predictor.predict(reflections)
zmin, zmax = experiment.scan.get_array_range()
z = predicted['xyzcal.px'].parts()[2]
predicted = predicted.select((z >= zmin) & (z <= zmax))
show_predictions(predicted)
def tst_select(self):
from dials.array_family import flex
# The columns as lists
c1 = list(range(10))
c2 = list(range(10))
c3 = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'i', 'j', 'k']
# Create a table with some elements
table = flex.reflection_table()
table['col1'] = flex.int(c1)
table['col2'] = flex.double(c2)
table['col3'] = flex.std_string(c3)
# Select some columns
new_table = table.select(('col1', 'col2'))
assert(new_table.nrows() == 10)
assert(new_table.ncols() == 2)
assert(all(a == b for a, b in zip(new_table['col1'], c1)))
assert(all(a == b for a, b in zip(new_table['col2'], c2)))
print 'OK'
# Select some columns
new_table = table.select(flex.std_string(['col1', 'col2']))
assert(new_table.nrows() == 10)
assert(new_table.ncols() == 2)
assert(all(a == b for a, b in zip(new_table['col1'], c1)))
assert(all(a == b for a, b in zip(new_table['col2'], c2)))
print 'OK'
# Select some rows
index = flex.size_t([0, 1, 5, 8, 9])
cc1 = [c1[i] for i in index]
cc2 = [c2[i] for i in index]
cc3 = [c3[i] for i in index]
new_table = table.select(index)
assert(new_table.nrows() == 5)
assert(new_table.ncols() == 3)
assert(all(a == b for a, b in zip(new_table['col1'], cc1)))
assert(all(a == b for a, b in zip(new_table['col2'], cc2)))
assert(all(a == b for a, b in zip(new_table['col3'], cc3)))
print 'OK'
# Select some rows
index = flex.bool([True, True, False, False, False,
True, False, False, True, True])
new_table = table.select(index)
assert(new_table.nrows() == 5)
assert(new_table.ncols() == 3)
assert(all(a == b for a, b in zip(new_table['col1'], cc1)))
assert(all(a == b for a, b in zip(new_table['col2'], cc2)))
assert(all(a == b for a, b in zip(new_table['col3'], cc3)))
print 'OK'
def assert_basic_mask_is_correct(mask, ninvalid, nforeground):
from scitbx.array_family import flex
from dials.algorithms.shoebox import MaskCode
invalid = flex.size_t(
i for i in range(len(mask)) if not mask[i] & MaskCode.Valid)
foreground = flex.size_t(
i for i in range(len(mask)) if mask[i] & MaskCode.Foreground)
background = flex.size_t(
i for i in range(len(mask)) if mask[i] & MaskCode.Background)
background_used = flex.size_t(
i for i in range(len(mask)) if mask[i] & MaskCode.BackgroundUsed)
background_valid = flex.size_t(
i for i in range(len(mask))
if mask[i] & MaskCode.Valid and mask[i] & MaskCode.Background)
assert(len(invalid) == ninvalid)
assert(len(foreground) == nforeground)
assert(len(background) == len(mask) - len(foreground))
assert(len(background_used) < len(background))
assert(len(set(background).intersection(foreground)) == 0)
assert(len(set(background).intersection(background_used)) == len(background_used))
return invalid, foreground, background, background_used, background_valid
def __call__(self, params, options):
''' Import the spot.xds file. '''
from iotbx.xds import spot_xds
from dials.util.command_line import Command
from dials.array_family import flex
# Read the SPOT.XDS file
Command.start('Reading SPOT.XDS')
handle = spot_xds.reader()
handle.read_file(self._spot_xds)
centroid = handle.centroid
intensity = handle.intensity
try:
miller_index = handle.miller_index
except AttributeError:
miller_index = None
Command.end('Read {0} spots from SPOT.XDS file.'.format(len(centroid)))
# Create the reflection list
Command.start('Creating reflection list')
table = flex.reflection_table()
table['id'] = flex.int(len(centroid), 0)
table['panel'] = flex.size_t(len(centroid), 0)
if miller_index:
table['miller_index'] = flex.miller_index(miller_index)
table['xyzobs.px.value'] = flex.vec3_double(centroid)
table['intensity.sum.value'] = flex.double(intensity)
Command.end('Created reflection list')
# Remove invalid reflections
Command.start('Removing invalid reflections')
if miller_index and params.remove_invalid:
flags = flex.bool([h != (0, 0, 0) for h in table['miller_index']])
table = table.select(flags)
Command.end('Removed invalid reflections, %d remaining' % len(table))
# Fill empty standard columns
if params.add_standard_columns:
Command.start('Adding standard columns')
rt = flex.reflection_table.empty_standard(len(table))
rt.update(table)
table = rt
# set variances to unity
table['xyzobs.mm.variance'] = flex.vec3_double(len(table), (1,1,1))
table['xyzobs.px.variance'] = flex.vec3_double(len(table), (1,1,1))
Command.end('Standard columns added')
# Output the table to pickle file
if params.output.filename is None:
params.output.filename = 'spot_xds.pickle'
Command.start('Saving reflection table to %s' % params.output.filename)
table.as_pickle(params.output.filename)
Command.end('Saved reflection table to %s' % params.output.filename)
def _find_nearest_neighbours(self, observed, predicted):
'''
Find the nearest predicted spot to the observed spot.
:param observed: The observed reflections
:param predicted: The predicted reflections
:returns: (nearest neighbours, distance)
'''
from scitbx.array_family import flex
from logging import warn
# Get the predicted coordinates
predicted_panel = predicted['panel']
predicted_xyz = predicted['xyzcal.px']
observed_panel = observed['panel']
observed_xyz = observed['xyzobs.px.value']
# Get the number of panels
max_panel1 = flex.max(predicted_panel)
max_panel2 = flex.max(observed_panel)
max_panel = max([max_panel1, max_panel2])
nn_all = flex.size_t()
dd_all = flex.double()
for panel in range(max_panel+1):
pind = predicted_panel == panel
oind = observed_panel == panel
pxyz = predicted_xyz.select(pind)
oxyz = observed_xyz.select(oind)
try:
nn, d = self._find_nearest_neighbours_single(oxyz, pxyz)
indices = flex.size_t(range(len(pind))).select(pind)
indices = indices.select(flex.size_t(list(nn)))
nn_all.extend(indices)
dd_all.extend(d)
except Exception:
warn("Unable to match spots on panel %d" % panel)
return nn_all, dd_all
def flex_ios(val, var): ''' Compute I/sigma or return zero for each element. ''' assert(len(val) == len(var)) result = flex.double(len(val),0) indices = flex.size_t(range(len(val))).select(var > 0) val = val.select(indices) var = var.select(indices) assert(var.all_gt(0)) result.set_selected(indices, val / flex.sqrt(var)) return result
def _grads_xl_unit_cell_loop(self, reflections, results, callback=None):
"""Loop over all crystal unit cell parameterisations, calculate gradients
and extend the results"""
# loop over the crystal unit cell parameterisations
for xlucp in self._xl_unit_cell_parameterisations:
# Determine (sub)set of reflections affected by this parameterisation
isel = flex.size_t()
for exp_id in xlucp.get_experiment_ids():
isel.extend(self._experiment_to_idx[exp_id])
# Extend derivative vectors for this crystal unit cell parameterisation
results = self._extend_gradient_vectors(results, self._nref, xlucp.num_free(),
keys=self._grad_names)
if len(isel) == 0:
# if no reflections are in this experiment, skip calculation of
# gradients, but must still process null gradients by a callback
if callback is not None:
for iparam in xrange(xlucp.num_free()):
results[self._iparam] = callback(results[self._iparam])
self._iparam += 1
else:
self._iparam += xlucp.num_free()
continue
w_inv = self._w_inv.select(isel)
u_w_inv = self._u_w_inv.select(isel)
v_w_inv = self._v_w_inv.select(isel)
dpv_dxluc_p, dAngle_dxluc_p = self._xl_unit_cell_derivatives(isel,
parameterisation=xlucp, reflections=reflections)
# convert to dX/dp, dY/dp and assign the elements of the vectors
# corresponding to this experiment
dX_dxluc_p, dY_dxluc_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp(
w_inv, u_w_inv, v_w_inv, dpv_dxluc_p)
for dX, dY, dAngle in zip(dX_dxluc_p, dY_dxluc_p, dAngle_dxluc_p):
if dX is not None:
results[self._iparam][self._grad_names[0]].set_selected(isel, dX)
if dY is not None:
results[self._iparam][self._grad_names[1]].set_selected(isel, dY)
if dAngle is not None:
results[self._iparam][self._grad_names[2]].set_selected(isel, dAngle)
if callback is not None:
results[self._iparam] = callback(results[self._iparam])
# increment the parameter index pointer
self._iparam += 1
return results
def get_hkl_offset_correlation_coefficients(
dials_reflections, dials_crystal, map_to_asu=False,
grid_h=0, grid_k=0, grid_l=0, reference=None):
# N.B. deliberately ignoring d_min, d_max as these are inconsistent with
# changing the miller indices
from dials.array_family import flex
from cctbx.miller import set as miller_set
from cctbx import sgtbx
cs = cctbx_crystal_from_dials(dials_crystal)
ms = cctbx_i_over_sigi_ms_from_dials_data(dials_reflections, cs)
if reference:
reference_ms = cctbx_i_over_sigi_ms_from_dials_data(reference, cs)
else:
reference_ms = None
ccs = flex.double()
offsets = flex.vec3_int()
nref = flex.size_t()
if reference:
cb_op = sgtbx.change_of_basis_op('x,y,z')
else:
cb_op = sgtbx.change_of_basis_op('-x,-y,-z')
hkl_test = [(h, k, l) for h in range(-grid_h, grid_h + 1) \
for k in range(-grid_k, grid_k + 1) \
for l in range(-grid_l, grid_l + 1)]
for hkl in hkl_test:
indices = offset_miller_indices(ms.indices(), hkl)
reindexed_indices = cb_op.apply(indices)
rms = miller_set(cs, reindexed_indices).array(ms.data())
if reference_ms:
_ms = reference_ms
else:
_ms = miller_set(cs, indices).array(ms.data())
n, cc = compute_miller_set_correlation(_ms, rms, map_to_asu=map_to_asu)
ccs.append(cc)
offsets.append(hkl)
nref.append(n)
return offsets, ccs, nref
def sample_predictions(self, experiments, predicted, params):
""" Select a random sample of the predicted reflections to integrate. """
from dials.array_family import flex
nref_per_degree = params.sampling.reflections_per_degree
min_sample_size = params.sampling.minimum_sample_size
max_sample_size = params.sampling.maximum_sample_size
# this code is very similar to David's code in algorithms/refinement/reflection_manager.py!
# constants
from math import pi
RAD2DEG = 180.0 / pi
DEG2RAD = pi / 180.0
working_isel = flex.size_t()
for iexp, exp in enumerate(experiments):
sel = predicted["id"] == iexp
isel = sel.iselection()
# refs = self._reflections.select(sel)
nrefs = sample_size = len(isel)
# set sample size according to nref_per_degree (per experiment)
if exp.scan and nref_per_degree:
sweep_range_rad = exp.scan.get_oscillation_range(deg=False)
width = abs(sweep_range_rad[1] - sweep_range_rad[0]) * RAD2DEG
sample_size = int(nref_per_degree * width)
else:
sweep_range_rad = None
# adjust sample size if below the chosen limit
sample_size = max(sample_size, min_sample_size)
# set maximum sample size if requested
if max_sample_size:
sample_size = min(sample_size, max_sample_size)
# determine subset and collect indices
if sample_size < nrefs:
isel = isel.select(flex.random_selection(nrefs, sample_size))
working_isel.extend(isel)
# create subset
return predicted.select(working_isel)
def _create_working_set(self):
"""Make a subset of the indices of reflections to use in refinement"""
working_isel = flex.size_t()
for iexp, exp in enumerate(self._experiments):
sel = self._reflections['id'] == iexp
isel = sel.iselection()
#refs = self._reflections.select(sel)
nrefs = sample_size = len(isel)
# set sample size according to nref_per_degree (per experiment)
if exp.scan and self._nref_per_degree:
sweep_range_rad = exp.scan.get_oscillation_range(deg=False)
width = abs(sweep_range_rad[1] -
sweep_range_rad[0]) * RAD2DEG
sample_size = int(self._nref_per_degree * width)
else: sweep_range_rad = None
# adjust sample size if below the chosen limit
sample_size = max(sample_size, self._min_sample_size)
# set maximum sample size if requested
if self._max_sample_size:
sample_size = min(sample_size, self._max_sample_size)
# determine subset and collect indices
if sample_size < nrefs:
isel = isel.select(flex.random_selection(nrefs, sample_size))
working_isel.extend(isel)
# create subsets
free_sel = flex.bool(len(self._reflections), True)
free_sel.set_selected(working_isel, False)
self._free_reflections = self._reflections.select(free_sel)
self._reflections = self._reflections.select(working_isel)
return
def __init__(self):
from dials.algorithms.integration.profile import GridSampler2D
from dials.array_family import flex
from random import randint, uniform
# Number of reflections
nrefl = 1000
# Size of the images
width = 1000
height = 1000
# Create the grid
self.grid = GridSampler2D((width, height), (5,5))
# Create the list of xyz and bboxes
self.xyz = flex.vec3_double(nrefl)
self.bbox = flex.int6(nrefl)
self.panel = flex.size_t(nrefl, 0)
for i in range(nrefl):
x0 = randint(0,width-10)
x1 = x0 + randint(3,10)
y0 = randint(0,height-10)
y1 = y0 + randint(3,10)
z0 = randint(0,10)
z1 = z0 + randint(1,10)
b = x0, x1, y0, y1, z0, z1
c = (x1 + x0) / 2, (y1 + y0) / 2, (z1 + z0) / 2
self.xyz[i] = c
self.bbox[i] = b
# Create the array of shoeboxes
self.sbox = flex.shoebox(self.panel, self.bbox)
self.sbox.allocate()
for i in range(len(self.sbox)):
data = self.sbox[i].data
for j in range(len(data)):
data[j] = uniform(0, 100)
