def test_runtime_error_raised_when_not_enough_memory(mock_psutil_swap,
mock_psutil_vm,
mock_flex_max):
mock_flex_max.return_value = 750001
mock_psutil_vm.return_value.available = 1000000
mock_psutil_swap.return_value.free = 0
phil_mock = mock.Mock()
phil_mock.mp.method = "multiprocessing"
phil_mock.mp.nproc = 4
phil_mock.block.max_memory_usage = 0.75
reflections = {"bbox": flex.int6(1000, (0, 1, 0, 1, 0, 1))}
manager = dials.algorithms.integration.processor._Manager(
None, reflections, phil_mock)
manager.jobs = mock.Mock(autospec=JobList)
with pytest.raises(MemoryError) as exc_info:
manager.compute_processors()
assert "Not enough memory to run integration jobs." in exc_info.value.args[
0]
mock_flex_max.assert_called_once_with(
manager.jobs.shoebox_memory.return_value)
# Reduce memory usage by 1 byte, should then pass
mock_flex_max.return_value = 750000
manager.compute_processors()
mock_flex_max.assert_called_with(manager.jobs.shoebox_memory.return_value)
def test_single_panel():
from dials.array_family import flex
nrefl = 1000
# Generate bboxes
bbox = flex.int6(nrefl)
for i in range(nrefl):
x0 = random.randint(0, 500)
y0 = random.randint(0, 500)
z0 = random.randint(0, 10)
x1 = x0 + random.randint(2, 10)
y1 = y0 + random.randint(2, 10)
z1 = z0 + random.randint(2, 10)
bbox[i] = (x0, x1, y0, y1, z0, z1)
# Find the overlaps
overlaps = find_overlapping(bbox)
assert overlaps.num_vertices() == nrefl
overlaps2 = brute_force(bbox)
assert overlaps.num_edges() == len(overlaps2)
edges = {}
for edge in overlaps2:
edge = (min(edge), max(edge))
edges[edge] = None
for edge in overlaps.edges():
edge = overlaps.source(edge), overlaps.target(edge)
edge = (min(edge), max(edge))
assert edge in edges
def tst_multiple_panels(self):
from dials.array_family import flex
from random import randint
nrefl = 1000
# Generate bboxes
bbox = flex.int6(nrefl)
panel = flex.size_t(nrefl)
for i in range(nrefl):
x0 = randint(0, 500)
y0 = randint(0, 500)
z0 = randint(0, 10)
x1 = x0 + randint(2, 10)
y1 = y0 + randint(2, 10)
z1 = z0 + randint(2, 10)
bbox[i] = (x0, x1, y0, y1, z0, z1)
panel[i] = randint(0,2)
# Find the overlaps
overlaps = find_overlapping(bbox, panel)
assert(overlaps.num_vertices() == nrefl)
overlaps2 = self.brute_force(bbox, panel)
assert(overlaps.num_edges() == len(overlaps2))
edges = {}
for edge in overlaps2:
edge = (min(edge), max(edge))
edges[edge] = None
for edge in overlaps.edges():
edge = (overlaps.source(edge), overlaps.target(edge))
edge = (min(edge), max(edge))
assert(edge in edges)
print 'OK'
def tst_multiple_panels(self):
from dials.array_family import flex
from random import randint
nrefl = 1000
# Generate bboxes
bbox = flex.int6(nrefl)
panel = flex.size_t(nrefl)
for i in range(nrefl):
x0 = randint(0, 500)
y0 = randint(0, 500)
z0 = randint(0, 10)
x1 = x0 + randint(2, 10)
y1 = y0 + randint(2, 10)
z1 = z0 + randint(2, 10)
bbox[i] = (x0, x1, y0, y1, z0, z1)
panel[i] = randint(0,2)
# Find the overlaps
overlaps = find_overlapping(bbox, panel)
assert(overlaps.num_vertices() == nrefl)
overlaps2 = self.brute_force(bbox, panel)
assert(overlaps.num_edges() == len(overlaps2))
edges = {}
for edge in overlaps2:
edge = (min(edge), max(edge))
edges[edge] = None
for edge in overlaps.edges():
edge = (overlaps.source(edge), overlaps.target(edge))
edge = (min(edge), max(edge))
assert(edge in edges)
print 'OK'
def test_split_blocks_non_overlapping():
from dials.array_family import flex
from dials.algorithms.integration.integrator import JobList
from scitbx.array_family import shared
blocks = shared.tiny_int_2([(0, 10), (10, 20), (20, 30), (30, 35),
(35, 40), (40, 50), (50, 60), (60, 70),
(70, 80), (80, 90), (90, 100), (100, 110)])
jobs = JobList((0, 1), blocks)
r = flex.reflection_table()
r['value1'] = flex.double()
r['value2'] = flex.int()
r['value3'] = flex.double()
r['bbox'] = flex.int6()
r['id'] = flex.int()
expected = []
for i in range(100):
x0 = random.randint(0, 100)
x1 = x0 + random.randint(1, 10)
y0 = random.randint(0, 100)
y1 = y0 + random.randint(1, 10)
z0 = random.randint(0, 100)
z1 = z0 + random.randint(1, 10)
v1 = random.uniform(0, 100)
v2 = random.randint(0, 100)
v3 = random.uniform(0, 100)
r.append({
'id': 0,
'value1': v1,
'value2': v2,
'value3': v3,
'bbox': (x0, x1, y0, y1, z0, z1)
})
for j in range(len(blocks)):
b0 = blocks[j][0]
b1 = blocks[j][1]
if ((z0 >= b0 and z1 <= b1) or (z0 < b1 and z1 >= b1)
or (z0 < b0 and z1 > b0)):
z00 = max(b0, z0)
z11 = min(b1, z1)
expected.append({
'id': 0,
'value1': v1,
'value2': v2,
'value3': v3,
'bbox': (x0, x1, y0, y1, z00, z11),
'partial_id': i,
})
jobs.split(r)
assert (len(r) == len(expected))
EPS = 1e-7
for r1, r2 in zip(r, expected):
assert (r1['bbox'] == r2['bbox'])
assert (r1['partial_id'] == r2['partial_id'])
assert (abs(r1['value1'] - r2['value1']) < EPS)
assert (r1['value2'] == r2['value2'])
assert (abs(r1['value3'] - r2['value3']) < EPS)
def test_find_overlapping():
N = 10000
r = flex.reflection_table(N)
r["bbox"] = flex.int6(N)
r["panel"] = flex.size_t(N)
r["id"] = flex.int(N)
r["imageset_id"] = flex.int(N)
for i in range(N):
x0 = random.randint(0, 100)
x1 = random.randint(1, 10) + x0
y0 = random.randint(0, 100)
y1 = random.randint(1, 10) + y0
z0 = random.randint(0, 100)
z1 = random.randint(1, 10) + z0
panel = random.randint(0, 2)
pid = random.randint(0, 2)
r["bbox"][i] = (x0, x1, y0, y1, z0, z1)
r["panel"][i] = panel
r["id"][i] = pid
r["imageset_id"][i] = pid
def is_overlap(b0, b1, border):
b0 = (
b0[0] - border,
b0[1] + border,
b0[2] - border,
b0[3] + border,
b0[4] - border,
b0[5] + border,
)
b1 = (
b1[0] - border,
b1[1] + border,
b1[2] - border,
b1[3] + border,
b1[4] - border,
b1[5] + border,
)
if not (b1[0] > b0[1] or b1[1] < b0[0] or b1[2] > b0[3]
or b1[3] < b0[2] or b1[4] > b0[5] or b1[5] < b0[4]):
return True
return False
for i in [0, 2, 5]:
overlaps = r.find_overlaps(border=i)
for item in overlaps.edges():
i0 = overlaps.source(item)
i1 = overlaps.target(item)
r0 = r[i0]
r1 = r[i1]
p0 = r0["panel"]
p1 = r1["panel"]
b0 = r0["bbox"]
b1 = r1["bbox"]
j0 = r0["imageset_id"]
j1 = r1["imageset_id"]
assert j0 == j1
assert p0 == p1
assert is_overlap(b0, b1, i)
def tst_find_overlapping(self):
from dials.array_family import flex
from random import randint, uniform
from dials.model.data import Shoebox
N = 10000
r = flex.reflection_table(N)
r['bbox'] = flex.int6(N)
r['panel'] = flex.size_t(N)
r['id'] = flex.int(N)
r['imageset_id'] = flex.int(N)
for i in range(N):
x0 = randint(0, 100)
x1 = randint(1, 10) + x0
y0 = randint(0, 100)
y1 = randint(1, 10) + y0
z0 = randint(0, 100)
z1 = randint(1, 10) + z0
panel = randint(0,2)
pid = randint(0,2)
r['bbox'][i] = (x0,x1,y0,y1,z0,z1)
r['panel'][i] = panel
r['id'][i] = pid
r['imageset_id'][i] = pid
def is_overlap(b0, b1, border):
b0 = b0[0]-border,b0[1]+border,b0[2]-border,b0[3]+border,b0[4]-border,b0[5]+border
b1 = b1[0]-border,b1[1]+border,b1[2]-border,b1[3]+border,b1[4]-border,b1[5]+border
if not (b1[0] > b0[1] or
b1[1] < b0[0] or
b1[2] > b0[3] or
b1[3] < b0[2] or
b1[4] > b0[5] or
b1[5] < b0[4]):
return True
return False
for i in [0, 2, 5]:
overlaps = r.find_overlaps(border=i)
for item in overlaps.edges():
i0 = overlaps.source(item)
i1 = overlaps.target(item)
r0 = r[i0]
r1 = r[i1]
p0 = r0['panel']
p1 = r1['panel']
b0 = r0['bbox']
b1 = r1['bbox']
j0 = r0['imageset_id']
j1 = r1['imageset_id']
assert j0 == j1
assert p0 == p1
assert is_overlap(b0,b1,i)
print 'OK'
def __init__(self, params, exlist, reference=None,
predicted=None, shoeboxes=None):
'''Initialise the script.'''
assert reference is not None
# Load the extractor based on the input
if shoeboxes is not None:
extractor = self._load_extractor(shoeboxes, params, exlist)
else:
if predicted is None:
predicted = self._predict_reflections(params, exlist)
#predicted = self._filter_reflections(params, exlist, predicted) # FIXME
predicted = self._match_with_reference(predicted, reference)
from annlib_ext import AnnAdaptor
from dials.array_family import flex
import math
matcheddata = predicted.select(predicted.get_flags(predicted.flags.reference_spot))
A = AnnAdaptor(matcheddata['xyzcal.mm'].as_double(), 3, 10)
A.query(predicted['xyzcal.mm'].as_double())
bboxes = flex.int6()
for i, ref in enumerate(predicted):
nn_pred = [matcheddata[A.nn[i*10+j]] for j in xrange(10)]
nn_ref = [reference[reference['miller_index'].first_index(r['miller_index'])] for r in nn_pred]
max_x = max([r['bbox'][1]-r['bbox'][0] for r in nn_ref])
max_y = max([r['bbox'][3]-r['bbox'][2] for r in nn_ref])
max_z = max([r['bbox'][5]-r['bbox'][4] for r in nn_ref])
panel = exlist[ref['id']].detector[ref['panel']]
imgsize_x, imgsize_y = panel.get_image_size()
x1 = int(math.floor(ref['xyzcal.px'][0] - (max_x / 2)))
x2 = int(math.ceil (ref['xyzcal.px'][0] + (max_x / 2)))
y1 = int(math.floor(ref['xyzcal.px'][1] - (max_y / 2)))
y2 = int(math.ceil (ref['xyzcal.px'][1] + (max_y / 2)))
if x1 < 0: x1 = 0
if y1 < 0: y1 = 0
if x2 > imgsize_x: x2 = imgsize_x
if y2 > imgsize_y: y2 = imgsize_y
bboxes.append((x1,x2,y1,y2,0,1))
predicted['bbox'] = bboxes
extractor = self._create_extractor(params, exlist, predicted)
# Initialise the integrator
self._integrator = ReflectionBlockIntegratorStills(params, exlist, reference, extractor)
def __init__(self, params, exlist, reference=None,
predicted=None, shoeboxes=None):
'''Initialise the script.'''
assert reference is not None
# Load the extractor based on the input
if shoeboxes is not None:
extractor = self._load_extractor(shoeboxes, params, exlist)
else:
if predicted is None:
predicted = self._predict_reflections(params, exlist)
#predicted = self._filter_reflections(params, exlist, predicted) # FIXME
predicted = self._match_with_reference(predicted, reference)
from annlib_ext import AnnAdaptor
from dials.array_family import flex
import math
matcheddata = predicted.select(predicted.get_flags(predicted.flags.reference_spot))
A = AnnAdaptor(matcheddata['xyzcal.mm'].as_double(), 3, 10)
A.query(predicted['xyzcal.mm'].as_double())
bboxes = flex.int6()
for i, ref in enumerate(predicted):
nn_pred = [matcheddata[A.nn[i*10+j]] for j in xrange(10)]
nn_ref = [reference[reference['miller_index'].first_index(r['miller_index'])] for r in nn_pred]
max_x = max([r['bbox'][1]-r['bbox'][0] for r in nn_ref])
max_y = max([r['bbox'][3]-r['bbox'][2] for r in nn_ref])
max_z = max([r['bbox'][5]-r['bbox'][4] for r in nn_ref])
panel = exlist[ref['id']].detector[ref['panel']]
imgsize_x, imgsize_y = panel.get_image_size()
x1 = int(math.floor(ref['xyzcal.px'][0] - (max_x / 2)))
x2 = int(math.ceil (ref['xyzcal.px'][0] + (max_x / 2)))
y1 = int(math.floor(ref['xyzcal.px'][1] - (max_y / 2)))
y2 = int(math.ceil (ref['xyzcal.px'][1] + (max_y / 2)))
if x1 < 0: x1 = 0
if y1 < 0: y1 = 0
if x2 > imgsize_x: x2 = imgsize_x
if y2 > imgsize_y: y2 = imgsize_y
bboxes.append((x1,x2,y1,y2,0,1))
predicted['bbox'] = bboxes
extractor = self._create_extractor(params, exlist, predicted)
# Initialise the integrator
self._integrator = ReflectionBlockIntegratorStills(params, exlist, reference, extractor)
def test_split_blocks_1_frame():
from dials.algorithms.integration.integrator import JobList
from dials.array_family import flex
r = flex.reflection_table()
r["value1"] = flex.double()
r["value2"] = flex.int()
r["value3"] = flex.double()
r["bbox"] = flex.int6()
r["id"] = flex.int()
expected = []
for i in range(100):
x0 = random.randint(0, 100)
x1 = x0 + random.randint(1, 10)
y0 = random.randint(0, 100)
y1 = y0 + random.randint(1, 10)
z0 = random.randint(0, 100)
z1 = z0 + random.randint(1, 10)
v1 = random.uniform(0, 100)
v2 = random.randint(0, 100)
v3 = random.uniform(0, 100)
r.append(
{
"id": 0,
"value1": v1,
"value2": v2,
"value3": v3,
"bbox": (x0, x1, y0, y1, z0, z1),
}
)
for z in range(z0, z1):
expected.append(
{
"id": 0,
"value1": v1,
"value2": v2,
"value3": v3,
"bbox": (x0, x1, y0, y1, z, z + 1),
"partial_id": i,
}
)
jobs = JobList()
jobs.add((0, 1), (0, 111), 1, 0)
jobs.split(r)
assert len(r) == len(expected)
EPS = 1e-7
for r1, r2 in zip(r.rows(), expected):
assert r1["bbox"] == r2["bbox"]
assert r1["partial_id"] == r2["partial_id"]
assert abs(r1["value1"] - r2["value1"]) < EPS
assert r1["value2"] == r2["value2"]
assert abs(r1["value3"] - r2["value3"]) < EPS
def __init__(self):
from dials.array_family import flex
self.reflections = flex.reflection_table()
self.reflections['panel'] = flex.size_t()
self.reflections['bbox'] = flex.int6()
self.reflections['miller_index'] = flex.miller_index()
self.reflections['s1'] = flex.vec3_double()
self.reflections['xyzcal.px'] = flex.vec3_double()
self.reflections['xyzcal.mm'] = flex.vec3_double()
self.reflections['entering'] = flex.bool()
self.reflections['id'] = flex.int()
self.reflections["flags"] = flex.size_t()
self.npanels = 2
self.width = 1000
self.height = 1000
self.nrefl = 10000
self.array_range = (0, 130)
self.block_size = 20
from random import randint, seed, choice
seed(0)
self.processed = [[] for i in range(12)]
for i in range(self.nrefl):
x0 = randint(0, self.width - 10)
y0 = randint(0, self.height - 10)
zs = randint(2, 9)
x1 = x0 + randint(2, 10)
y1 = y0 + randint(2, 10)
for k, j in enumerate(
[10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]):
m = k + i * 12
pos = choice(["left", "right", "centre"])
if pos == 'left':
z0 = j - zs
z1 = j
elif pos == 'right':
z0 = j
z1 = j + zs
else:
z0 = j - zs // 2
z1 = j + zs // 2
bbox = (x0, x1, y0, y1, z0, z1)
self.reflections.append({
"panel":
randint(0, 1),
"bbox":
bbox,
"flags":
flex.reflection_table.flags.reference_spot
})
self.processed[k].append(m)
def test_RefinerData(testdata):
experiment = testdata.experiment
reflections = testdata.reflections
panel = experiment.detector[0]
s0_length = matrix.col(experiment.beam.get_s0()).length()
reflections["bbox"] = flex.int6(len(reflections))
reflections["xyzobs.px.value"] = flex.vec3_double(len(reflections))
reflections["s2"] = reflections["s1"].each_normalize() * s0_length
reflections["sp"] = flex.vec3_double(len(reflections))
for i, (x, y, z) in enumerate(reflections["xyzcal.px"]):
x0 = int(x) - 5
x1 = int(x) + 5 + 1
y0 = int(y) - 5
y1 = int(y) + 5 + 1
z0 = int(z)
z1 = z0 + 1
reflections["bbox"][i] = x0, x1, y0, y1, z0, z1
reflections["xyzobs.px.value"][i] = (int(x) + 0.5, int(y) + 0.5,
int(z) + 0.5)
reflections["sp"][i] = (matrix.col(
panel.get_pixel_lab_coord(
reflections["xyzobs.px.value"][i][0:2])).normalize() *
s0_length)
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"],
allocate=True)
shoebox_data = flex.float(flex.grid(1, 11, 11))
shoebox_mask = flex.int(flex.grid(1, 11, 11))
for j in range(11):
for i in range(11):
shoebox_data[0, j, i] = (100 * exp(-0.5 * (j - 5)**2 / 1**2) *
exp(-0.5 * (i - 5)**2 / 1**2))
shoebox_mask[0, j, i] = 5
for sbox in reflections["shoebox"]:
sbox.data = shoebox_data
sbox.mask = shoebox_mask
data = RefinerData.from_reflections(experiment, reflections)
assert tuple(data.s0) == pytest.approx(experiment.beam.get_s0())
assert data.h_list == reflections["miller_index"]
for i, sp in enumerate(reflections["sp"]):
assert data.sp_list[:, i] == pytest.approx(sp)
assert data.ctot_list[0] == sum(shoebox_data)
mobs1 = np.abs(data.mobs_list[0, :])
mobs2 = np.abs(data.mobs_list[1, :])
assert np.max(mobs1) < 1e-6
assert np.max(mobs2) < 1e-6
def tst_split_blocks_1_frame(self):
from dials.array_family import flex
from random import randint, uniform, seed
from dials.algorithms.integration.integrator import JobList
r = flex.reflection_table()
r['value1'] = flex.double()
r['value2'] = flex.int()
r['value3'] = flex.double()
r['bbox'] = flex.int6()
r['id'] = flex.int()
expected = []
for i in range(100):
x0 = randint(0, 100)
x1 = x0 + randint(1, 10)
y0 = randint(0, 100)
y1 = y0 + randint(1, 10)
z0 = randint(0, 100)
z1 = z0 + randint(1, 10)
v1 = uniform(0, 100)
v2 = randint(0, 100)
v3 = uniform(0, 100)
r.append({
'id': 0,
'value1': v1,
'value2': v2,
'value3': v3,
'bbox': (x0, x1, y0, y1, z0, z1)
})
for z in range(z0, z1):
expected.append({
'id': 0,
'value1': v1,
'value2': v2,
'value3': v3,
'bbox': (x0, x1, y0, y1, z, z + 1),
'partial_id': i,
})
jobs = JobList()
jobs.add((0, 1), (0, 111), 1)
jobs.split(r)
assert (len(r) == len(expected))
EPS = 1e-7
for r1, r2 in zip(r, expected):
assert (r1['bbox'] == r2['bbox'])
assert (r1['partial_id'] == r2['partial_id'])
assert (abs(r1['value1'] - r2['value1']) < EPS)
assert (r1['value2'] == r2['value2'])
assert (abs(r1['value3'] - r2['value3']) < EPS)
print 'OK'
def tst_find_overlapping(self):
from dials.array_family import flex
from random import randint, uniform
from dials.model.data import Shoebox
N = 10000
r = flex.reflection_table(N)
r['bbox'] = flex.int6(N)
r['panel'] = flex.size_t(N)
r['id'] = flex.int(N)
r['imageset_id'] = flex.int(N)
for i in range(N):
x0 = randint(0, 100)
x1 = randint(1, 10) + x0
y0 = randint(0, 100)
y1 = randint(1, 10) + y0
z0 = randint(0, 100)
z1 = randint(1, 10) + z0
panel = randint(0, 2)
pid = randint(0, 2)
r['bbox'][i] = (x0, x1, y0, y1, z0, z1)
r['panel'][i] = panel
r['id'][i] = pid
r['imageset_id'][i] = pid
def is_overlap(b0, b1, border):
b0 = b0[0] - border, b0[1] + border, b0[2] - border, b0[
3] + border, b0[4] - border, b0[5] + border
b1 = b1[0] - border, b1[1] + border, b1[2] - border, b1[
3] + border, b1[4] - border, b1[5] + border
if not (b1[0] > b0[1] or b1[1] < b0[0] or b1[2] > b0[3]
or b1[3] < b0[2] or b1[4] > b0[5] or b1[5] < b0[4]):
return True
return False
for i in [0, 2, 5]:
overlaps = r.find_overlaps(border=i)
for item in overlaps.edges():
i0 = overlaps.source(item)
i1 = overlaps.target(item)
r0 = r[i0]
r1 = r[i1]
p0 = r0['panel']
p1 = r1['panel']
b0 = r0['bbox']
b1 = r1['bbox']
j0 = r0['imageset_id']
j1 = r1['imageset_id']
assert j0 == j1
assert p0 == p1
assert is_overlap(b0, b1, i)
print 'OK'
def tst_split_blocks_1_frame(self):
from dials.array_family import flex
from random import randint, uniform, seed
from dials.algorithms.integration.integrator import JobList
r = flex.reflection_table()
r['value1'] = flex.double()
r['value2'] = flex.int()
r['value3'] = flex.double()
r['bbox'] = flex.int6()
r['id'] = flex.int()
expected = []
for i in range(100):
x0 = randint(0, 100)
x1 = x0 + randint(1, 10)
y0 = randint(0, 100)
y1 = y0 + randint(1, 10)
z0 = randint(0, 100)
z1 = z0 + randint(1, 10)
v1 = uniform(0, 100)
v2 = randint(0, 100)
v3 = uniform(0, 100)
r.append({
'id' : 0,
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z0, z1)
})
for z in range(z0, z1):
expected.append({
'id' : 0,
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z, z+1),
'partial_id' : i,
})
jobs = JobList()
jobs.add((0,1), (0, 111), 1)
jobs.split(r)
assert(len(r) == len(expected))
EPS = 1e-7
for r1, r2 in zip(r, expected):
assert(r1['bbox'] == r2['bbox'])
assert(r1['partial_id'] == r2['partial_id'])
assert(abs(r1['value1'] - r2['value1']) < EPS)
assert(r1['value2'] == r2['value2'])
assert(abs(r1['value3'] - r2['value3']) < EPS)
print 'OK'
def run(self):
from dials.algorithms.profile_model.gaussian_rs import \
PartialityCalculator3D
from dials.array_family import flex
calculator = PartialityCalculator3D(
self.experiment.beam,
self.experiment.goniometer,
self.experiment.scan,
self.sigma_m)
predicted = flex.reflection_table.from_predictions_multi(self.experiments)
predicted['bbox'] = predicted.compute_bbox(self.experiments)
# Remove any touching edges of scan to get only fully recorded
x0, x1, y0, y1, z0, z1 = predicted['bbox'].parts()
predicted = predicted.select((z0 > 0) & (z1 < 100))
assert(len(predicted) > 0)
# Compute partiality
partiality = calculator(
predicted['s1'],
predicted['xyzcal.px'].parts()[2],
predicted['bbox'])
# Should have all fully recorded
assert(len(partiality) == len(predicted))
from math import sqrt, erf
three_sigma = 0.5 * (erf(3.0 / sqrt(2.0)) - erf(-3.0 / sqrt(2.0)))
assert(partiality.all_gt(three_sigma))
# Trim bounding boxes
x0, x1, y0, y1, z0, z1 = predicted['bbox'].parts()
z0 = z0 + 1
z1 = z1 - 1
predicted['bbox'] = flex.int6(x0, x1, y0, y1, z0, z1)
predicted = predicted.select(z1 > z0)
assert(len(predicted) > 0)
# Compute partiality
partiality = calculator(
predicted['s1'],
predicted['xyzcal.px'].parts()[2],
predicted['bbox'])
# Should have all partials
assert(len(partiality) == len(predicted))
assert(partiality.all_le(1.0) and partiality.all_gt(0))
print 'OK'
def run(self):
from dials.algorithms.profile_model.gaussian_rs import \
PartialityCalculator3D
from dials.array_family import flex
calculator = PartialityCalculator3D(
self.experiment.beam,
self.experiment.goniometer,
self.experiment.scan,
self.sigma_m)
predicted = flex.reflection_table.from_predictions_multi(self.experiments)
predicted['bbox'] = predicted.compute_bbox(self.experiments)
# Remove any touching edges of scan to get only fully recorded
x0, x1, y0, y1, z0, z1 = predicted['bbox'].parts()
predicted = predicted.select((z0 > 0) & (z1 < 100))
assert(len(predicted) > 0)
# Compute partiality
partiality = calculator(
predicted['s1'],
predicted['xyzcal.px'].parts()[2],
predicted['bbox'])
# Should have all fully recorded
assert(len(partiality) == len(predicted))
from math import sqrt, erf
three_sigma = 0.5 * (erf(3.0 / sqrt(2.0)) - erf(-3.0 / sqrt(2.0)))
assert(partiality.all_gt(three_sigma))
# Trim bounding boxes
x0, x1, y0, y1, z0, z1 = predicted['bbox'].parts()
z0 = z0 + 1
z1 = z1 - 1
predicted['bbox'] = flex.int6(x0, x1, y0, y1, z0, z1)
predicted = predicted.select(z1 > z0)
assert(len(predicted) > 0)
# Compute partiality
partiality = calculator(
predicted['s1'],
predicted['xyzcal.px'].parts()[2],
predicted['bbox'])
# Should have all partials
assert(len(partiality) == len(predicted))
assert(partiality.all_le(1.0) and partiality.all_gt(0))
print 'OK'
def __init__(self):
from dials.array_family import flex
self.reflections = flex.reflection_table()
self.reflections['panel'] = flex.size_t()
self.reflections['bbox'] = flex.int6()
self.reflections['miller_index'] = flex.miller_index()
self.reflections['s1'] = flex.vec3_double()
self.reflections['xyzcal.px'] = flex.vec3_double()
self.reflections['xyzcal.mm'] = flex.vec3_double()
self.reflections['entering'] = flex.bool()
self.reflections['id'] = flex.int()
self.reflections["flags"] = flex.size_t()
self.npanels = 2
self.width = 1000
self.height = 1000
self.nrefl = 10000
self.array_range = (0, 130)
self.block_size = 20
from random import randint, seed, choice
seed(0)
self.processed = [[] for i in range(12)]
for i in range(self.nrefl):
x0 = randint(0, self.width-10)
y0 = randint(0, self.height-10)
zs = randint(2, 9)
x1 = x0 + randint(2, 10)
y1 = y0 + randint(2, 10)
for k, j in enumerate([10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]):
m = k + i * 12
pos = choice(["left", "right", "centre"])
if pos == 'left':
z0 = j - zs
z1 = j
elif pos == 'right':
z0 = j
z1 = j + zs
else:
z0 = j - zs // 2
z1 = j + zs // 2
bbox = (x0, x1, y0, y1, z0, z1)
self.reflections.append({
"panel" : randint(0,1),
"bbox" : bbox,
"flags" : flex.reflection_table.flags.reference_spot
})
self.processed[k].append(m)
def tst_split_partials(self):
from dials.array_family import flex
from random import randint, uniform
r = flex.reflection_table()
r['value1'] = flex.double()
r['value2'] = flex.int()
r['value3'] = flex.double()
r['bbox'] = flex.int6()
expected = []
for i in range(100):
x0 = randint(0, 100)
x1 = x0 + randint(1, 10)
y0 = randint(0, 100)
y1 = y0 + randint(1, 10)
z0 = randint(0, 100)
z1 = z0 + randint(1, 10)
v1 = uniform(0, 100)
v2 = randint(0, 100)
v3 = uniform(0, 100)
r.append({
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z0, z1)
})
for z in range(z0, z1):
expected.append({
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z, z+1),
'partial_id' : i,
})
r.split_partials()
assert(len(r) == len(expected))
EPS = 1e-7
for r1, r2 in zip(r, expected):
assert(abs(r1['value1'] - r2['value1']) < EPS)
assert(r1['value2'] == r2['value2'])
assert(abs(r1['value3'] - r2['value3']) < EPS)
assert(r1['bbox'] == r2['bbox'])
assert(r1['partial_id'] == r2['partial_id'])
print 'OK'
def tst_split_partials(self):
from dials.array_family import flex
from random import randint, uniform
r = flex.reflection_table()
r['value1'] = flex.double()
r['value2'] = flex.int()
r['value3'] = flex.double()
r['bbox'] = flex.int6()
expected = []
for i in range(100):
x0 = randint(0, 100)
x1 = x0 + randint(1, 10)
y0 = randint(0, 100)
y1 = y0 + randint(1, 10)
z0 = randint(0, 100)
z1 = z0 + randint(1, 10)
v1 = uniform(0, 100)
v2 = randint(0, 100)
v3 = uniform(0, 100)
r.append({
'value1': v1,
'value2': v2,
'value3': v3,
'bbox': (x0, x1, y0, y1, z0, z1)
})
for z in range(z0, z1):
expected.append({
'value1': v1,
'value2': v2,
'value3': v3,
'bbox': (x0, x1, y0, y1, z, z + 1),
'partial_id': i,
})
r.split_partials()
assert (len(r) == len(expected))
EPS = 1e-7
for r1, r2 in zip(r, expected):
assert (abs(r1['value1'] - r2['value1']) < EPS)
assert (r1['value2'] == r2['value2'])
assert (abs(r1['value3'] - r2['value3']) < EPS)
assert (r1['bbox'] == r2['bbox'])
assert (r1['partial_id'] == r2['partial_id'])
print 'OK'
def test_bounding_boxes():
from dials.array_family import flex
from dials.model.data import Shoebox
shoebox = flex.shoebox(10)
bbox = flex.int6(10)
for i in range(10):
x0 = random.randint(0, 90)
y0 = random.randint(0, 90)
z0 = random.randint(0, 90)
x1 = random.randint(1, 10) + x0
y1 = random.randint(1, 10) + y0
z1 = random.randint(1, 10) + z0
bbox[i] = (x0, x1, y0, y1, z0, z1)
shoebox[i] = Shoebox(bbox[i])
bbox2 = shoebox.bounding_boxes()
for i in range(10):
assert bbox2[i] == bbox[i]
def refinerdata_testdata(testdata):
experiment = testdata.experiment
reflections = testdata.reflections
panel = experiment.detector[0]
s0_length = matrix.col(experiment.beam.get_s0()).length()
reflections["bbox"] = flex.int6(len(reflections))
reflections["xyzobs.px.value"] = flex.vec3_double(len(reflections))
reflections["s2"] = reflections["s1"].each_normalize() * s0_length
reflections["sp"] = flex.vec3_double(len(reflections))
for i, (x, y, z) in enumerate(reflections["xyzcal.px"]):
x0 = int(x) - 5
x1 = int(x) + 5 + 1
y0 = int(y) - 5
y1 = int(y) + 5 + 1
z0 = int(z)
z1 = z0 + 1
reflections["bbox"][i] = x0, x1, y0, y1, z0, z1
reflections["xyzobs.px.value"][i] = (int(x) + 0.5, int(y) + 0.5,
int(z) + 0.5)
reflections["sp"][i] = (matrix.col(
panel.get_pixel_lab_coord(
reflections["xyzobs.px.value"][i][0:2])).normalize() *
s0_length)
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"],
allocate=True)
shoebox_data = flex.float(flex.grid(1, 11, 11))
shoebox_mask = flex.int(flex.grid(1, 11, 11))
for j in range(11):
for i in range(11):
shoebox_data[0, j, i] = (100 * exp(-0.5 * (j - 5)**2 / 1**2) *
exp(-0.5 * (i - 5)**2 / 1**2))
shoebox_mask[0, j, i] = 5
for sbox in reflections["shoebox"]:
sbox.data = shoebox_data
sbox.mask = shoebox_mask
return RefinerData.from_reflections(experiment, reflections)
def test_split_partials():
r = flex.reflection_table()
r["value1"] = flex.double()
r["value2"] = flex.int()
r["value3"] = flex.double()
r["bbox"] = flex.int6()
expected = []
for i in range(100):
x0 = random.randint(0, 100)
x1 = x0 + random.randint(1, 10)
y0 = random.randint(0, 100)
y1 = y0 + random.randint(1, 10)
z0 = random.randint(0, 100)
z1 = z0 + random.randint(1, 10)
v1 = random.uniform(0, 100)
v2 = random.randint(0, 100)
v3 = random.uniform(0, 100)
r.append({
"value1": v1,
"value2": v2,
"value3": v3,
"bbox": (x0, x1, y0, y1, z0, z1)
})
for z in range(z0, z1):
expected.append({
"value1": v1,
"value2": v2,
"value3": v3,
"bbox": (x0, x1, y0, y1, z, z + 1),
"partial_id": i,
})
r.split_partials()
assert len(r) == len(expected)
EPS = 1e-7
for r1, r2 in zip(r.rows(), expected):
assert abs(r1["value1"] - r2["value1"]) < EPS
assert r1["value2"] == r2["value2"]
assert abs(r1["value3"] - r2["value3"]) < EPS
assert r1["bbox"] == r2["bbox"]
assert r1["partial_id"] == r2["partial_id"]
def test_shoebox_memory_is_a_reasonable_guesstimate(dials_data):
path = dials_data("centroid_test_data").join("experiments.json").strpath
exlist = ExperimentListFactory.from_json_file(path)[0]
exlist.profile = Model(
None,
n_sigma=3,
sigma_b=0.024 * math.pi / 180.0,
sigma_m=0.044 * math.pi / 180.0,
)
rlist = flex.reflection_table.from_predictions(exlist)
rlist["id"] = flex.int(len(rlist), 0)
rlist["bbox"] = flex.int6(rlist.size(), (0, 1, 0, 1, 0, 1))
jobs = JobList()
jobs.add((0, 1), (0, 9), 9)
for flatten in (True, False):
assumed_memory_usage = list(jobs.shoebox_memory(rlist, flatten))
assert len(assumed_memory_usage) == 1
assert assumed_memory_usage[0] == pytest.approx(23952, abs=3000)
def __init__(self):
from dials.algorithms.integration.profile import GridSampler2D
from dials.array_family import flex
from random import randint, uniform
# Number of reflections
nrefl = 1000
# Size of the images
width = 1000
height = 1000
# Create the grid
self.grid = GridSampler2D((width, height), (5, 5))
# Create the list of xyz and bboxes
self.xyz = flex.vec3_double(nrefl)
self.bbox = flex.int6(nrefl)
self.panel = flex.size_t(nrefl, 0)
for i in range(nrefl):
x0 = randint(0, width - 10)
x1 = x0 + randint(3, 10)
y0 = randint(0, height - 10)
y1 = y0 + randint(3, 10)
z0 = randint(0, 10)
z1 = z0 + randint(1, 10)
b = x0, x1, y0, y1, z0, z1
c = (x1 + x0) / 2, (y1 + y0) / 2, (z1 + z0) / 2
self.xyz[i] = c
self.bbox[i] = b
# Create the array of shoeboxes
self.sbox = flex.shoebox(self.panel, self.bbox)
self.sbox.allocate()
for i in range(len(self.sbox)):
data = self.sbox[i].data
for j in range(len(data)):
data[j] = uniform(0, 100)
def __init__(self):
from dials.algorithms.integration.profile import GridSampler2D
from dials.array_family import flex
from random import randint, uniform
# Number of reflections
nrefl = 1000
# Size of the images
width = 1000
height = 1000
# Create the grid
self.grid = GridSampler2D((width, height), (5,5))
# Create the list of xyz and bboxes
self.xyz = flex.vec3_double(nrefl)
self.bbox = flex.int6(nrefl)
self.panel = flex.size_t(nrefl, 0)
for i in range(nrefl):
x0 = randint(0,width-10)
x1 = x0 + randint(3,10)
y0 = randint(0,height-10)
y1 = y0 + randint(3,10)
z0 = randint(0,10)
z1 = z0 + randint(1,10)
b = x0, x1, y0, y1, z0, z1
c = (x1 + x0) / 2, (y1 + y0) / 2, (z1 + z0) / 2
self.xyz[i] = c
self.bbox[i] = b
# Create the array of shoeboxes
self.sbox = flex.shoebox(self.panel, self.bbox)
self.sbox.allocate()
for i in range(len(self.sbox)):
data = self.sbox[i].data
for j in range(len(data)):
data[j] = uniform(0, 100)
def tst_bounding_boxes(self):
from dials.model.data import Shoebox
from random import randint
from dials.array_family import flex
shoebox = flex.shoebox(10)
bbox = flex.int6(10)
for i in range(10):
x0 = randint(0, 90)
y0 = randint(0, 90)
z0 = randint(0, 90)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
bbox[i] = (x0, x1, y0, y1, z0, z1)
shoebox[i] = Shoebox(bbox[i])
bbox2 = shoebox.bounding_boxes()
for i in range(10):
assert(bbox2[i] == bbox[i])
# Test passed
print 'OK'
def refl_bbox_maker(self):
self.reflections['bbox'] = flex.int6(self.length)
def tst_split_partials_with_shoebox(self):
from dials.array_family import flex
from random import randint, uniform
from dials.model.data import Shoebox
r = flex.reflection_table()
r['value1'] = flex.double()
r['value2'] = flex.int()
r['value3'] = flex.double()
r['bbox'] = flex.int6()
r['panel'] = flex.size_t()
r['shoebox'] = flex.shoebox()
expected = []
for i in range(100):
x0 = randint(0, 100)
x1 = x0 + randint(1, 10)
y0 = randint(0, 100)
y1 = y0 + randint(1, 10)
z0 = randint(0, 100)
z1 = z0 + randint(1, 10)
v1 = uniform(0, 100)
v2 = randint(0, 100)
v3 = uniform(0, 100)
sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1))
sbox.allocate()
assert(sbox.is_consistent())
w = x1 - x0
h = y1 - y0
for z in range(z0, z1):
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[z-z0,y-y0,x-x0] = x+y*w+z*w*h
r.append({
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z0, z1),
'panel' : 0,
'shoebox' : sbox
})
for z in range(z0, z1):
sbox = Shoebox(0, (x0, x1, y0, y1, z, z+1))
sbox.allocate()
assert(sbox.is_consistent())
w = x1 - x0
h = y1 - y0
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[0,y-y0,x-x0] = x+y*w+z*w*h
expected.append({
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z, z+1),
'partial_id' : i,
'panel' : 0,
'shoebox' : sbox
})
r.split_partials_with_shoebox()
assert(len(r) == len(expected))
EPS = 1e-7
for r1, r2 in zip(r, expected):
assert(abs(r1['value1'] - r2['value1']) < EPS)
assert(r1['value2'] == r2['value2'])
assert(abs(r1['value3'] - r2['value3']) < EPS)
assert(r1['bbox'] == r2['bbox'])
assert(r1['partial_id'] == r2['partial_id'])
assert(r1['panel'] == r2['panel'])
assert(r1['shoebox'].data.as_double().as_1d().all_approx_equal(
r2['shoebox'].data.as_double().as_1d()))
print 'OK'
def simple_gaussian_spots(params):
from scitbx import matrix
from dials.array_family import flex
r = params.rotation
axis = matrix.col((r.axis.x, r.axis.y, r.axis.z))
if axis.length() > 0:
rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True)
else:
rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))
# generate mask and peak values
from dials.algorithms.shoebox import MaskCode
mask_peak = MaskCode.Valid | MaskCode.Foreground
mask_back = MaskCode.Valid | MaskCode.Background
from dials.util.command_line import ProgressBar
p = ProgressBar(title="Generating reflections")
rlist = flex.reflection_table(params.nrefl)
hkl = flex.miller_index(params.nrefl)
s1 = flex.vec3_double(params.nrefl)
xyzmm = flex.vec3_double(params.nrefl)
xyzpx = flex.vec3_double(params.nrefl)
panel = flex.size_t(params.nrefl)
bbox = flex.int6(params.nrefl)
for j in range(params.nrefl):
p.update(j * 100.0 / params.nrefl)
hkl[j] = (random.randint(0, 20), random.randint(0, 20), random.randint(0, 20))
phi = 2 * math.pi * random.random()
s1[j] = (0, 0, 0)
xyzpx[j] = (0, 0, 0)
xyzmm[j] = (0, 0, phi)
panel[j] = 0
bbox[j] = (
0,
params.shoebox_size.x,
0,
params.shoebox_size.y,
0,
params.shoebox_size.z,
)
p.finished("Generating %d reflections" % params.nrefl)
intensity = flex.double(params.nrefl)
shoebox = flex.shoebox(panel, bbox)
shoebox.allocate_with_value(MaskCode.Valid)
p = ProgressBar(title="Generating shoeboxes")
for i in range(len(rlist)):
p.update(i * 100.0 / params.nrefl)
mask = shoebox[i].mask
if params.pixel_mask == "precise":
# flag everything as background: peak will me assigned later
for j in range(len(mask)):
mask[j] = mask_back
elif params.pixel_mask == "all":
# flag we have no idea what anything is
mask_none = MaskCode.Valid | MaskCode.Foreground | MaskCode.Background
for j in range(len(mask)):
mask[j] = mask_none
elif params.pixel_mask == "static":
from scitbx.array_family import flex
x0 = params.spot_offset.x + params.shoebox_size.x / 2
y0 = params.spot_offset.x + params.shoebox_size.y / 2
z0 = params.spot_offset.x + params.shoebox_size.z / 2
sx = params.mask_nsigma * params.spot_size.x
sy = params.mask_nsigma * params.spot_size.y
sz = params.mask_nsigma * params.spot_size.z
# The x, y, z indices
z, y, x = zip(*itertools.product(*(range(n) for n in mask.all())))
xyz = flex.vec3_double(flex.double(x), flex.double(y), flex.double(z))
# Calculate SUM(((xj - xj0) / sxj)**2) for each element
xyz0 = (x0, y0, z0)
isxyz = (1.0 / sx, 1.0 / sy, 1.0 / sz)
dxyz = sum(
(x * isx) ** 2
for x, isx in zip(((xyz - xyz0) * rotation).parts(), isxyz)
)
# Set the mask values
index = dxyz <= 1.0
index.reshape(mask.accessor())
mask.set_selected(index, MaskCode.Valid | MaskCode.Foreground)
mask.set_selected(not index, MaskCode.Valid | MaskCode.Background)
sbox = shoebox[i].data
# reflection itself, including setting the peak region if we're doing that
# FIXME use flex arrays to make the rotation bit more efficient as this is
# now rather slow...
counts_true = 0
for j in range(params.counts):
_x = random.gauss(0, params.spot_size.x)
_y = random.gauss(0, params.spot_size.y)
_z = random.gauss(0, params.spot_size.z)
Rxyz = rotation * matrix.col((_x, _y, _z)).elems
x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2)
y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2)
z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2)
if x < 0 or x >= params.shoebox_size.x:
continue
if y < 0 or y >= params.shoebox_size.y:
continue
if z < 0 or z >= params.shoebox_size.z:
continue
sbox[z, y, x] += 1
counts_true += 1
if params.pixel_mask == "precise":
mask[z, y, x] = mask_peak
intensity[i] = counts_true
if params.background:
# background:flat;
for j in range(params.background * len(sbox)):
x = random.randint(0, params.shoebox_size.x - 1)
y = random.randint(0, params.shoebox_size.y - 1)
z = random.randint(0, params.shoebox_size.z - 1)
sbox[z, y, x] += 1
else:
# or inclined
random_background_plane(
sbox,
params.background_a,
params.background_b,
params.background_c,
params.background_d,
)
rlist["miller_index"] = hkl
rlist["s1"] = s1
rlist["xyzcal.px"] = xyzpx
rlist["xyzcal.mm"] = xyzmm
rlist["bbox"] = bbox
rlist["panel"] = panel
rlist["shoebox"] = shoebox
rlist["intensity.sum.value"] = intensity
p.finished("Generating %d shoeboxes" % params.nrefl)
return rlist
def simple_gaussian_spots(params):
from dials.array_family import flex
from dials.algorithms import shoebox
import random
import math
from scitbx import matrix
r = params.rotation
axis = matrix.col((r.axis.x, r.axis.y, r.axis.z))
if axis.length() > 0:
rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True)
else:
rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))
# generate mask and peak values
from dials.algorithms.shoebox import MaskCode
mask_peak = MaskCode.Valid|MaskCode.Foreground
mask_back = MaskCode.Valid|MaskCode.Background
from dials.util.command_line import ProgressBar
p = ProgressBar(title = 'Generating reflections')
rlist = flex.reflection_table(params.nrefl)
hkl = flex.miller_index(params.nrefl)
s1 = flex.vec3_double(params.nrefl)
xyzmm = flex.vec3_double(params.nrefl)
xyzpx = flex.vec3_double(params.nrefl)
panel = flex.size_t(params.nrefl)
bbox = flex.int6(params.nrefl)
for j in range(params.nrefl):
p.update(j * 100.0 / params.nrefl)
hkl[j] = (random.randint(0, 20),
random.randint(0, 20),
random.randint(0, 20))
phi = 2 * math.pi * random.random()
s1[j] = (0, 0, 0)
xyzpx[j] = (0, 0, 0)
xyzmm[j] = (0, 0, phi)
panel[j] = 0
bbox[j] = (0, params.shoebox_size.x,
0, params.shoebox_size.y,
0, params.shoebox_size.z)
p.finished('Generating %d reflections' % params.nrefl)
intensity = flex.double(params.nrefl)
shoebox = flex.shoebox(panel, bbox)
shoebox.allocate_with_value(MaskCode.Valid)
p = ProgressBar(title = 'Generating shoeboxes')
for i in range(len(rlist)):
p.update(i * 100.0 / params.nrefl)
mask = shoebox[i].mask
if params.pixel_mask == 'precise':
# flag everything as background: peak will me assigned later
for j in range(len(mask)):
mask[j] = mask_back
elif params.pixel_mask == 'all':
# flag we have no idea what anything is
mask_none = MaskCode.Valid|MaskCode.Foreground|MaskCode.Background
for j in range(len(mask)):
mask[j] = mask_none
elif params.pixel_mask == 'static':
import itertools
from scitbx.array_family import flex
x0 = params.spot_offset.x + params.shoebox_size.x / 2
y0 = params.spot_offset.x + params.shoebox_size.y / 2
z0 = params.spot_offset.x + params.shoebox_size.z / 2
sx = params.mask_nsigma * params.spot_size.x
sy = params.mask_nsigma * params.spot_size.y
sz = params.mask_nsigma * params.spot_size.z
# The x, y, z indices
z, y, x = zip(*itertools.product(*(range(n) for n in mask.all())))
xyz = flex.vec3_double(flex.double(x), flex.double(y), flex.double(z))
# Calculate SUM(((xj - xj0) / sxj)**2) for each element
xyz0 = (x0, y0, z0)
isxyz = (1.0/sx, 1.0/sy, 1.0/sz)
dxyz = sum([(x * isx)**2 for x, isx in
zip(((xyz - xyz0) * rotation).parts(), isxyz)])
# Set the mask values
index = dxyz <= 1.0
index.reshape(mask.accessor())
mask.set_selected(index, MaskCode.Valid | MaskCode.Foreground)
mask.set_selected(index != True, MaskCode.Valid | MaskCode.Background)
sbox = shoebox[i].data
# reflection itself, including setting the peak region if we're doing that
# FIXME use flex arrays to make the rotation bit more efficient as this is
# now rather slow...
counts_true = 0
for j in range(params.counts):
_x = random.gauss(0, params.spot_size.x)
_y = random.gauss(0, params.spot_size.y)
_z = random.gauss(0, params.spot_size.z)
Rxyz = rotation * matrix.col((_x, _y, _z)).elems
x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2)
y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2)
z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2)
if x < 0 or x >= params.shoebox_size.x:
continue
if y < 0 or y >= params.shoebox_size.y:
continue
if z < 0 or z >= params.shoebox_size.z:
continue
sbox[z, y, x] += 1
counts_true += 1
if params.pixel_mask == 'precise':
mask[z, y, x] = mask_peak
intensity[i] = counts_true
if params.background:
# background:flat;
for j in range(params.background * len(sbox)):
x = random.randint(0, params.shoebox_size.x - 1)
y = random.randint(0, params.shoebox_size.y - 1)
z = random.randint(0, params.shoebox_size.z - 1)
sbox[z, y, x] += 1
else:
# or inclined
random_background_plane(sbox, params.background_a, params.background_b,
params.background_c, params.background_d)
rlist['miller_index'] = hkl
rlist['s1'] = s1
rlist['xyzcal.px'] = xyzpx
rlist['xyzcal.mm'] = xyzmm
rlist['bbox'] = bbox
rlist['panel'] = panel
rlist['shoebox'] = shoebox
rlist['intensity.sum.value'] = intensity
p.finished('Generating %d shoeboxes' % params.nrefl)
return rlist
def test_to_from_msgpack(tmpdir):
from dials.model.data import Shoebox
def gen_shoebox():
shoebox = Shoebox(0, (0, 4, 0, 3, 0, 1))
shoebox.allocate()
for k in range(1):
for j in range(3):
for i in range(4):
shoebox.data[k, j, i] = i + j + k + 0.1
shoebox.mask[k, j, i] = i % 2
shoebox.background[k, j, i] = i * j + 0.2
return shoebox
def compare(a, b):
assert a.is_consistent()
assert b.is_consistent()
assert a.panel == b.panel
assert a.bbox == b.bbox
for aa, bb in zip(a.data, b.data):
if abs(aa - bb) > 1e-9:
return False
for aa, bb in zip(a.background, b.background):
if abs(aa - bb) > 1e-9:
return False
for aa, bb in zip(a.mask, b.mask):
if aa != bb:
return False
return True
# The columns as lists
c1 = list(range(10))
c2 = list(range(10))
c3 = ["a", "b", "c", "d", "e", "f", "g", "i", "j", "k"]
c4 = [True, False, True, False, True] * 2
c5 = list(range(10))
c6 = [(i + 1, i + 2) for i in range(10)]
c7 = [(i + 1, i + 2, i + 3) for i in range(10)]
c8 = [tuple(i + j for j in range(9)) for i in range(10)]
c9 = [tuple(i + j for j in range(6)) for i in range(10)]
c10 = [(i + 1, i + 2, i + 3) for i in range(10)]
c11 = [gen_shoebox() for i in range(10)]
# Create a table with some elements
table = flex.reflection_table()
table["col1"] = flex.int(c1)
table["col2"] = flex.double(c2)
table["col3"] = flex.std_string(c3)
table["col4"] = flex.bool(c4)
table["col5"] = flex.size_t(c5)
table["col6"] = flex.vec2_double(c6)
table["col7"] = flex.vec3_double(c7)
table["col8"] = flex.mat3_double(c8)
table["col9"] = flex.int6(c9)
table["col10"] = flex.miller_index(c10)
table["col11"] = flex.shoebox(c11)
obj = table.as_msgpack()
new_table = flex.reflection_table.from_msgpack(obj)
assert new_table.is_consistent()
assert new_table.nrows() == 10
assert new_table.ncols() == 11
assert all(tuple(a == b for a, b in zip(new_table["col1"], c1)))
assert all(tuple(a == b for a, b in zip(new_table["col2"], c2)))
assert all(tuple(a == b for a, b in zip(new_table["col3"], c3)))
assert all(tuple(a == b for a, b in zip(new_table["col4"], c4)))
assert all(tuple(a == b for a, b in zip(new_table["col5"], c5)))
assert all(tuple(a == b for a, b in zip(new_table["col6"], c6)))
assert all(tuple(a == b for a, b in zip(new_table["col7"], c7)))
assert all(tuple(a == b for a, b in zip(new_table["col8"], c8)))
assert all(tuple(a == b for a, b in zip(new_table["col9"], c9)))
assert all(tuple(a == b for a, b in zip(new_table["col10"], c10)))
assert all(tuple(compare(a, b) for a, b in zip(new_table["col11"], c11)))
table.as_msgpack_file(tmpdir.join("reflections.mpack").strpath)
new_table = flex.reflection_table.from_msgpack_file(
tmpdir.join("reflections.mpack").strpath)
assert new_table.is_consistent()
assert new_table.nrows() == 10
assert new_table.ncols() == 11
assert all(tuple(a == b for a, b in zip(new_table["col1"], c1)))
assert all(tuple(a == b for a, b in zip(new_table["col2"], c2)))
assert all(tuple(a == b for a, b in zip(new_table["col3"], c3)))
assert all(tuple(a == b for a, b in zip(new_table["col4"], c4)))
assert all(tuple(a == b for a, b in zip(new_table["col5"], c5)))
assert all(tuple(a == b for a, b in zip(new_table["col6"], c6)))
assert all(tuple(a == b for a, b in zip(new_table["col7"], c7)))
assert all(tuple(a == b for a, b in zip(new_table["col8"], c8)))
assert all(tuple(a == b for a, b in zip(new_table["col9"], c9)))
assert all(tuple(a == b for a, b in zip(new_table["col10"], c10)))
assert all(tuple(compare(a, b) for a, b in zip(new_table["col11"], c11)))
return decoder.decode(self._handle)
def set_reflections(self, reflections):
'''Set the reflection data.'''
self.set_data(reflections, ReflectionListEncoder())
def get_reflections(self):
'''Get the reflection data.'''
return self.get_data(ReflectionListDecoder())
if __name__ == '__main__':
from dials.array_family import flex
reflections = flex.reflection_table([('hkl', flex.miller_index(10)),
('s1', flex.vec3_double(10)),
('bbox', flex.int6(10)),
('id', flex.int(10)),
('shoebox', flex.shoebox(10))])
for i in range(10):
reflections['shoebox'][i].data = flex.double(flex.grid(10, 10, 10))
reflections['shoebox'][i].mask = flex.int(flex.grid(10, 10, 10))
reflections['shoebox'][i].background = flex.double(
flex.grid(10, 10, 10))
for i in range(10):
print reflections['shoebox'][i].data.all()
writer = NexusFile('temp.h5', 'w')
writer.set_reflections(reflections)
writer.close()
def _find_in_imageset(self, imageset):
"""
Do the spot finding.
:param imageset: The imageset to process
:return: The observed spots
"""
from dials.util.masking import MaskGenerator
from dials.array_family import flex
from dials.util.command_line import Command
from dxtbx.imageset import ImageSweep
from logging import info
# The input mask
mask = self.mask_generator.generate(imageset)
if self.mask is not None:
mask = tuple(m1 & m2 for m1, m2 in zip(mask, self.mask))
# Set the spot finding algorithm
extract_spots = ExtractSpots(
threshold_function=self.threshold_function,
mask=mask,
region_of_interest=self.region_of_interest,
max_strong_pixel_fraction=self.max_strong_pixel_fraction,
mp_method=self.mp_method,
nproc=self.nproc,
mp_chunksize=self.mp_chunksize,
min_spot_size=self.min_spot_size,
max_spot_size=self.max_spot_size,
)
# Get the max scan range
if isinstance(imageset, ImageSweep):
max_scan_range = imageset.get_array_range()
else:
max_scan_range = (0, len(imageset))
# Get list of scan ranges
if not self.scan_range or self.scan_range[0] is None:
scan_range = [(max_scan_range[0] + 1, max_scan_range[1])]
else:
scan_range = self.scan_range
# Get spots from bits of scan
spots_all = []
for scan in scan_range:
j0, j1 = scan
assert j1 >= j0 and j0 > max_scan_range[0] and j1 <= max_scan_range[1]
info("\nFinding spots in image {0} to {1}...".format(j0, j1))
j0 -= 1
if isinstance(imageset, ImageSweep):
j0 -= imageset.get_array_range()[0]
j1 -= imageset.get_array_range()[0]
spots_all.extend(extract_spots(imageset[j0:j1]))
# Get the list of shoeboxes
shoeboxes = flex.shoebox(spots_all)
# Calculate the spot centroids
centroid = shoeboxes.centroid_valid()
info("Calculated {0} spot centroids".format(len(shoeboxes)))
# Calculate the spot intensities
intensity = shoeboxes.summed_intensity()
info("Calculated {0} spot intensities".format(len(shoeboxes)))
# Create the observations
observed = flex.observation(shoeboxes.panels(), centroid, intensity)
# Write the hot mask
if self.write_hot_mask:
# Find spots which cover the whole scan range
bbox = flex.int6([sbox.bbox for sbox in shoeboxes])
z0, z1 = bbox.parts()[4:6]
zr = z1 - z0
assert zr.all_gt(0)
possible_hot_spots = zr == len(imageset)
num_possible_hot_spots = possible_hot_spots.count(True)
info("Found %d possible hot spots" % num_possible_hot_spots)
# Create the hot pixel mask
hot_mask = tuple(flex.bool(flex.grid(p.get_image_size()[::-1]), True) for p in imageset.get_detector())
if num_possible_hot_spots > 0:
hot_shoeboxes = shoeboxes.select(possible_hot_spots)
for sbox in hot_shoeboxes:
x0, x1, y0, y1 = sbox.bbox[0:4]
m = sbox.mask
p = sbox.panel
for y in range(m.all()[1]):
for x in range(m.all()[2]):
if m[:, y : y + 1, x : x + 1].all_ne(0):
hot_mask[p][y0 + y, x0 + x] = False
info("Found %d possible hot pixel(s)" % hot_mask.count(False))
else:
hot_mask = None
# Filter the reflections and select only the desired spots
flags = self.filter_spots(None, sweep=imageset, observations=observed, shoeboxes=shoeboxes)
observed = observed.select(flags)
shoeboxes = shoeboxes.select(flags)
# Return as a reflection list
return flex.reflection_table(observed, shoeboxes), hot_mask
def tst_split_blocks_overlapping(self):
from dials.array_family import flex
from random import randint, uniform, seed
from dials.algorithms.integration.integrator import JobList
from scitbx.array_family import shared
blocks = shared.tiny_int_2([
(0, 10),
(5, 15),
(10, 20),
(15, 25),
(20, 30),
(25, 35),
(30, 40),
(35, 45),
(40, 50),
(45, 55),
(50, 60),
(55, 65),
(60, 70),
(65, 75),
(70, 80),
(75, 85),
(80, 90),
(85, 95),
(90, 100),
(95, 105),
(100, 110)])
jobs = JobList((0, 1), blocks)
r = flex.reflection_table()
r['value1'] = flex.double()
r['value2'] = flex.int()
r['value3'] = flex.double()
r['bbox'] = flex.int6()
r['id'] = flex.int()
expected = []
for i in range(100):
x0 = randint(0, 100)
x1 = x0 + randint(1, 10)
y0 = randint(0, 100)
y1 = y0 + randint(1, 10)
z0 = randint(0, 90)
z1 = z0 + randint(1, 20)
v1 = uniform(0, 100)
v2 = randint(0, 100)
v3 = uniform(0, 100)
r.append({
'id' : 0,
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z0, z1)
})
expected.append({
'id' : 0,
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z0, z1)
})
jobs.split(r)
assert(len(r) > 100)
for r1 in r:
v1 = r1['value1']
v2 = r1['value2']
v3 = r1['value3']
bbox = r1['bbox']
pid = r1['partial_id']
z0 = bbox[4]
z1 = bbox[5]
success = False
for i in range(len(blocks)):
b0 = blocks[i][0]
b1 = blocks[i][1]
if z0 >= b0 and z1 <= b1:
success = True
break
assert(success)
v11 = expected[pid]['value1']
v22 = expected[pid]['value2']
v33 = expected[pid]['value3']
bb = expected[pid]['bbox']
assert(v11 == v1)
assert(v22 == v2)
assert(v33 == v3)
assert(bb[0] == bbox[0])
assert(bb[1] == bbox[1])
assert(bb[2] == bbox[2])
assert(bb[3] == bbox[3])
print 'OK'
def tst_split_blocks_non_overlapping(self):
from dials.array_family import flex
from random import randint, uniform, seed
from dials.algorithms.integration.integrator import JobList
from scitbx.array_family import shared
blocks = shared.tiny_int_2([
(0, 10),
(10, 20),
(20, 30),
(30, 35),
(35, 40),
(40, 50),
(50, 60),
(60, 70),
(70, 80),
(80, 90),
(90, 100),
(100, 110)])
jobs = JobList((0, 1), blocks)
r = flex.reflection_table()
r['value1'] = flex.double()
r['value2'] = flex.int()
r['value3'] = flex.double()
r['bbox'] = flex.int6()
r['id'] = flex.int()
expected = []
for i in range(100):
x0 = randint(0, 100)
x1 = x0 + randint(1, 10)
y0 = randint(0, 100)
y1 = y0 + randint(1, 10)
z0 = randint(0, 100)
z1 = z0 + randint(1, 10)
v1 = uniform(0, 100)
v2 = randint(0, 100)
v3 = uniform(0, 100)
r.append({
'id' : 0,
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z0, z1)
})
for j in range(len(blocks)):
b0 = blocks[j][0]
b1 = blocks[j][1]
if ((z0 >= b0 and z1 <= b1) or
(z0 < b1 and z1 >= b1) or
(z0 < b0 and z1 > b0)):
z00 = max(b0, z0)
z11 = min(b1, z1)
expected.append({
'id' : 0,
'value1' : v1,
'value2' : v2,
'value3' : v3,
'bbox' : (x0, x1, y0, y1, z00, z11),
'partial_id' : i,
})
jobs.split(r)
assert(len(r) == len(expected))
EPS = 1e-7
for r1, r2 in zip(r, expected):
assert(r1['bbox'] == r2['bbox'])
assert(r1['partial_id'] == r2['partial_id'])
assert(abs(r1['value1'] - r2['value1']) < EPS)
assert(r1['value2'] == r2['value2'])
assert(abs(r1['value3'] - r2['value3']) < EPS)
print 'OK'
def read(handle, key):
from dials.array_family import flex
import numpy as np
if key == 'miller_index':
h = flex.int(handle['h'][:].astype(np.int32))
k = flex.int(handle['k'][:].astype(np.int32))
l = flex.int(handle['l'][:].astype(np.int32))
return flex.miller_index(h,k,l)
elif key == 'id':
return flex.int(handle['id'][:].astype(int))
elif key == 'partial_id':
return flex.size_t(handle['reflection_id'][:].astype(int))
elif key == 'entering':
return flex.bool(handle['entering'][:])
elif key == 'flags':
return flex.size_t(handle['flags'][:].astype(int))
elif key == 'panel':
return flex.size_t(handle['det_module'][:].astype(int))
elif key == 'd':
return flex.double(handle['d'][:])
elif key == 'partiality':
return flex.double(handle['partiality'][:])
elif key == 'xyzcal.px':
x = flex.double(handle['prd_px_x'][:])
y = flex.double(handle['prd_px_y'][:])
z = flex.double(handle['prd_frame'][:])
return flex.vec3_double(x, y, z)
elif key == 'xyzcal.mm':
x = flex.double(handle['prd_mm_x'][:])
y = flex.double(handle['prd_mm_y'][:])
z = flex.double(handle['prd_phi'][:])
return flex.vec3_double(x, y, z)
elif key == 'bbox':
x0 = flex.int(handle['bbx0'][:].astype(np.int32))
x1 = flex.int(handle['bbx1'][:].astype(np.int32))
y0 = flex.int(handle['bby0'][:].astype(np.int32))
y1 = flex.int(handle['bby1'][:].astype(np.int32))
z0 = flex.int(handle['bbz0'][:].astype(np.int32))
z1 = flex.int(handle['bbz1'][:].astype(np.int32))
return flex.int6(x0, x1, y0, y1, z0, z1)
elif key == 'xyzobs.px.value':
x = flex.double(handle['obs_px_x_val'][:])
y = flex.double(handle['obs_px_y_val'][:])
z = flex.double(handle['obs_frame_val'][:])
return flex.vec3_double(x, y, z)
elif key == 'xyzobs.px.variance':
x = flex.double(handle['obs_px_x_var'][:])
y = flex.double(handle['obs_px_y_var'][:])
z = flex.double(handle['obs_frame_var'][:])
return flex.vec3_double(x, y, z)
elif key == 'xyzobs.mm.value':
x = flex.double(handle['obs_mm_x_val'][:])
y = flex.double(handle['obs_mm_y_val'][:])
z = flex.double(handle['obs_phi_val'][:])
return flex.vec3_double(x, y, z)
elif key == 'xyzobs.mm.variance':
x = flex.double(handle['obs_mm_x_var'][:])
y = flex.double(handle['obs_mm_y_var'][:])
z = flex.double(handle['obs_phi_var'][:])
return flex.vec3_double(x, y, z)
elif key == 'background.mean':
return flex.double(handle['bkg_mean'][:])
elif key == 'intensity.sum.value':
return flex.double(handle['int_sum_val'][:])
elif key == 'intensity.sum.variance':
return flex.double(handle['int_sum_var'][:])
elif key == 'intensity.prf.value':
return flex.double(handle['int_prf_val'][:])
elif key == 'intensity.prf.variance':
return flex.double(handle['int_prf_var'][:])
elif key == 'profile.correlation':
return flex.double(handle['prf_cc'][:])
elif key == 'lp':
return flex.double(handle['lp'][:])
elif key == 'num_pixels.background':
return flex.int(handle['num_bg'][:].astype(np.int32))
elif key == 'num_pixels.background_used':
return flex.int(handle['num_bg_used'][:].astype(np.int32))
elif key == 'num_pixels.foreground':
return flex.int(handle['num_fg'][:].astype(np.int32))
elif key == 'num_pixels.valid':
return flex.int(handle['num_valid'][:].astype(np.int32))
elif key == 'profile.rmsd':
return flex.double(handle['prf_rmsd'][:])
else:
raise KeyError('Column %s not read from file' % key)
def test_export_dials():
from dials.array_family import flex
print 'Creating dummy reflection table...'
table = flex.reflection_table()
table['miller_index'] = flex.miller_index(100)
table['id'] = flex.int(100)
table['intensity.sum.value'] = flex.double(100)
table['intensity.sum.variance'] = flex.double(100)
table['intensity.prf.value'] = flex.double(100)
table['intensity.prf.variance'] = flex.double(100)
table['lp'] = flex.double(100)
table['panel'] = flex.size_t(100)
table['bbox'] = flex.int6(100)
table['xyzcal.px'] = flex.vec3_double(100)
table['xyzcal.mm'] = flex.vec3_double(100)
table['xyzobs.px.value'] = flex.vec3_double(100)
table['xyzobs.px.variance'] = flex.vec3_double(100)
table['xyzobs.mm.value'] = flex.vec3_double(100)
table['xyzobs.mm.variance'] = flex.vec3_double(100)
table['partiality'] = flex.double(100)
table['d'] = flex.double(100)
table['s1'] = flex.vec3_double(100)
table['rlp'] = flex.vec3_double(100)
table['background.mean'] = flex.double(100)
table['entering'] = flex.bool(100)
table['flags'] = flex.size_t(100)
table['profile.correlation'] = flex.double(100)
# Open the file
outfile = File('test_file.mtz2', 'w')
# Get the entry
entry = outfile.entry
print 'Writing reflection table stuff...'
# Save some processed data
diffraction = entry.diffraction
# Set the experiments
experiment = diffraction.experiments[0]
experiment['beam'] = 0
experiment['detector'] = 0
experiment['goniometer'] = 0
experiment['scan'] = 0
experiment['crystal'] =0
# Get columns into array
col1, col2, col3 = zip(*list(table['miller_index']))
col4 = table['id']
col5 = table['intensity.sum.value']
col6 = table['intensity.sum.variance']
col7 = table['intensity.prf.value']
col8 = table['intensity.prf.variance']
col9 = table['lp']
col10 = table['panel']
col11, col12, col13, col14, col15, col16 = table['bbox'].parts()
col17, col18, col19 = table['xyzcal.px'].parts()
col20, col21, col22 = table['xyzcal.mm'].parts()
col23, col24, col25 = table['xyzobs.px.value'].parts()
col26, col27, col28 = table['xyzobs.px.variance'].parts()
col29, col30, col31 = table['xyzobs.mm.value'].parts()
col32, col33, col34 = table['xyzobs.mm.variance'].parts()
col35 = table['partiality']
col36 = table['d']
col37 = table['background.mean']
col38 = table['entering']
col39 = table['flags']
col40 = table['profile.correlation']
# Some data
diffraction['h'] = col1
diffraction['k'] = col2
diffraction['l'] = col3
diffraction['id'] = col4
diffraction['int_sum_val'] = col5
diffraction['int_sum_var'] = col6
diffraction['int_prf_val'] = col7
diffraction['int_prf_var'] = col8
diffraction['lp'] = col9
diffraction['det_module'] = col10
diffraction['bbx0'] = col11
diffraction['bbx1'] = col12
diffraction['bby0'] = col13
diffraction['bby1'] = col14
diffraction['bbz0'] = col15
diffraction['bbz1'] = col16
diffraction['prd_px_x'] = col17
diffraction['prd_px_y'] = col18
diffraction['prd_frame'] = col19
diffraction['prd_mm_x'] = col20
diffraction['prd_mm_y'] = col21
diffraction['prd_phi'] = col22
diffraction['obs_px_x_val'] = col23
diffraction['obs_px_x_var'] = col24
diffraction['obs_px_y_val'] = col25
diffraction['obs_px_y_var'] = col26
diffraction['obs_frame_val'] = col27
diffraction['obs_frame_var'] = col28
diffraction['obs_mm_x_val'] = col29
diffraction['obs_mm_x_var'] = col30
diffraction['obs_mm_y_val'] = col31
diffraction['obs_mm_y_var'] = col32
diffraction['obs_phi_val'] = col33
diffraction['obs_phi_var'] = col34
diffraction['partiality'] = col35
diffraction['d'] = col36
diffraction['bkg_mean'] = col37
diffraction['entering'] = col38
diffraction['flags'] = col39
diffraction['prf_cc'] = col40
# Flush the file
outfile.flush()
def run(self):
''' Extract the shoeboxes. '''
from dials.util.options import flatten_reflections
from dials.util.options import flatten_experiments
from dials.util.options import flatten_datablocks
from dials.util import log
from dials.array_family import flex
from libtbx.utils import Sorry
from logging import info
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# Configure logging
log.config()
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
info('The following parameters have been modified:\n')
info(diff_phil)
# Get the data
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
datablocks = flatten_datablocks(params.input.datablock)
if len(experiments) == 0 and len(datablocks) == 0 and len(reflections) == 0:
self.parser.print_help()
exit(0)
elif (len(experiments) != 0 and len(datablocks) != 0):
raise Sorry('Both experiment list and datablocks set')
elif len(experiments) > 1:
raise Sorry('More than 1 experiment set')
elif len(datablocks) > 1:
raise Sorry('More than 1 datablock set')
elif len(experiments) == 1:
imageset = experiments[0].imageset
elif len(datablocks) == 1:
imagesets = datablocks[0].extract_imagesets()
if len(imagesets) != 1:
raise Sorry('Need 1 imageset, got %d' % len(imagesets))
imageset = imagesets[0]
if len(reflections) != 1:
raise Sorry('Need 1 reflection table, got %d' % len(reflections))
else:
reflections = reflections[0]
# Check the reflections contain the necessary stuff
assert("bbox" in reflections)
assert("panel" in reflections)
# Get some models
detector = imageset.get_detector()
scan = imageset.get_scan()
frame0, frame1 = scan.get_array_range()
# Add some padding but limit to image volume
if params.padding > 0:
info('Adding %d pixels as padding' % params.padding)
x0, x1, y0, y1, z0, z1 = reflections['bbox'].parts()
x0 -= params.padding
x1 += params.padding
y0 -= params.padding
y1 += params.padding
z0 -= params.padding
z1 += params.padding
panel = reflections['panel']
for i in range(len(reflections)):
width, height = detector[panel[i]].get_image_size()
if z0[i] < frame0: z0[i] = frame0
if z1[i] > frame1: z1[i] = frame1
reflections['bbox'] = flex.int6(x0, x1, y0, y1, z0, z1)
# Save the old shoeboxes
if "shoebox" in reflections:
old_shoebox = reflections['shoebox']
else:
old_shoebox = None
# Allocate the shoeboxes
reflections["shoebox"] = flex.shoebox(
reflections["panel"],
reflections["bbox"],
allocate=True)
# Extract the shoeboxes
reflections.extract_shoeboxes(imageset, verbose=True)
# Preserve masking
if old_shoebox is not None:
info("Applying old shoebox mask")
new_shoebox = reflections['shoebox']
for i in range(len(reflections)):
bbox0 = old_shoebox[i].bbox
bbox1 = new_shoebox[i].bbox
mask0 = old_shoebox[i].mask
mask1 = new_shoebox[i].mask
mask2 = flex.int(mask1.accessor(), 0)
x0 = bbox0[0] - bbox1[0]
x1 = bbox0[1] - bbox0[0] + x0
y0 = bbox0[2] - bbox1[2]
y1 = bbox0[3] - bbox0[2] + y0
z0 = bbox0[4] - bbox1[4]
z1 = bbox0[5] - bbox0[4] + z0
mask2[z0:z1,y0:y1,x0:x1] = mask0
mask1 = mask1.as_1d() | mask2.as_1d()
mask1.reshape(new_shoebox[i].mask.accessor())
new_shoebox[i].mask = mask1
# Saving the reflections to disk
filename = params.output.reflections
info('Saving %d reflections to %s' % (len(reflections), filename))
reflections.as_pickle(filename)
# rlist = ReflectionList()
pi = 3.14159265358
n_x = 80
n_y = 86
# n_x = 80
# n_y = 86
num_ref = n_y * n_x
ref_table = flex.reflection_table()
t_shoebox = flex.shoebox(num_ref)
t_intensity = flex.double(num_ref)
t_intensity_var = flex.double(num_ref)
t_bbox = flex.int6(num_ref)
t_xyzobs = flex.vec3_double(num_ref)
t_xyzcal = flex.vec3_double(num_ref)
t_row = 0
for ypos in range(n_y):
for xpos in range(n_x):
# if xpos/32.0 == int(xpos/32) and ypos/32.0 == int(ypos/32):
# if ypos/6.0 == int(ypos/6):
# if xpos/3.0 == int(xpos/3) and ypos/3.0 == int(ypos/3):
row_str = ypos * nrow
col_str = xpos * ncol
dx = col_str - 1200
dy = row_str - 1300
def read(handle, key):
from dxtbx.format.nexus import convert_units
if key == "miller_index":
h = flex.int(handle["h"][:].astype(np.int32))
k = flex.int(handle["k"][:].astype(np.int32))
l = flex.int(handle["l"][:].astype(np.int32))
return flex.miller_index(h, k, l)
elif key == "id":
return flex.int(handle["id"][:].astype(int))
elif key == "partial_id":
return flex.size_t(handle["reflection_id"][:].astype(int))
elif key == "entering":
return flex.bool(handle["entering"][:])
elif key == "flags":
return flex.size_t(handle["flags"][:].astype(int))
elif key == "panel":
return flex.size_t(handle["det_module"][:].astype(int))
elif key == "d":
return flex.double(handle["d"][:])
elif key == "partiality":
return flex.double(handle["partiality"][:])
elif key == "xyzcal.px":
x = flex.double(handle["predicted_px_x"][:])
y = flex.double(handle["predicted_px_y"][:])
z = flex.double(handle["predicted_frame"][:])
return flex.vec3_double(x, y, z)
elif key == "xyzcal.mm":
x = convert_units(
flex.double(handle["predicted_x"][:]),
handle["predicted_x"].attrs["units"],
"mm",
)
y = convert_units(
flex.double(handle["predicted_y"][:]),
handle["predicted_y"].attrs["units"],
"mm",
)
z = convert_units(
flex.double(handle["predicted_phi"][:]),
handle["predicted_phi"].attrs["units"],
"rad",
)
return flex.vec3_double(x, y, z)
elif key == "bbox":
b = flex.int(handle["bounding_box"][:].astype(np.int32))
return flex.int6(b.as_1d())
elif key == "xyzobs.px.value":
x = flex.double(handle["observed_px_x"][:])
y = flex.double(handle["observed_px_y"][:])
z = flex.double(handle["observed_frame"][:])
return flex.vec3_double(x, y, z)
elif key == "xyzobs.px.variance":
x = flex.double(handle["observed_px_x_var"][:])
y = flex.double(handle["observed_px_y_var"][:])
z = flex.double(handle["observed_frame_var"][:])
return flex.vec3_double(x, y, z)
elif key == "xyzobs.mm.value":
x = convert_units(
flex.double(handle["observed_x"][:]),
handle["observed_x"].attrs["units"],
"mm",
)
y = convert_units(
flex.double(handle["observed_y"][:]),
handle["observed_y"].attrs["units"],
"mm",
)
z = convert_units(
flex.double(handle["observed_phi"][:]),
handle["observed_phi"].attrs["units"],
"rad",
)
return flex.vec3_double(x, y, z)
elif key == "xyzobs.mm.variance":
x = convert_units(
flex.double(handle["observed_x_var"][:]),
handle["observed_x_var"].attrs["units"],
"mm",
)
y = convert_units(
flex.double(handle["observed_y_var"][:]),
handle["observed_y_var"].attrs["units"],
"mm",
)
z = convert_units(
flex.double(handle["observed_phi_var"][:]),
handle["observed_phi_var"].attrs["units"],
"rad",
)
return flex.vec3_double(x, y, z)
elif key == "background.mean":
return flex.double(handle["background_mean"][:])
elif key == "intensity.sum.value":
return flex.double(handle["int_sum"][:])
elif key == "intensity.sum.variance":
return flex.double(handle["int_sum_var"][:])
elif key == "intensity.prf.value":
return flex.double(handle["int_prf"][:])
elif key == "intensity.prf.variance":
return flex.double(handle["int_prf_var"][:])
elif key == "profile.correlation":
return flex.double(handle["prf_cc"][:])
elif key == "lp":
return flex.double(handle["lp"][:])
elif key == "num_pixels.background":
return flex.int(handle["num_bg"][:].astype(np.int32))
elif key == "num_pixels.background_used":
return flex.int(handle["num_bg_used"][:].astype(np.int32))
elif key == "num_pixels.foreground":
return flex.int(handle["num_fg"][:].astype(np.int32))
elif key == "num_pixels.valid":
return flex.int(handle["num_valid"][:].astype(np.int32))
elif key == "profile.rmsd":
return flex.double(handle["prf_rmsd"][:])
else:
raise KeyError("Column %s not read from file" % key)
def integrate(experiment):
from dials.algorithms.spot_prediction import PixelToMillerIndex
from dials.array_family import flex
from math import floor, sqrt
from collections import defaultdict
detector = experiment.detector
assert len(detector) == 1
panel = detector[0]
xsize, ysize = panel.get_image_size()
transform = PixelToMillerIndex(experiment.beam, experiment.detector,
experiment.crystal)
data = experiment.imageset.get_raw_data(0)[0]
mask = flex.bool(flex.grid(ysize, xsize), False)
reflections = defaultdict(list)
print("Doing pixel labelling")
for j in range(ysize):
for i in range(xsize):
h = transform.h(0, i, j)
h0 = tuple(map(lambda x: int(floor(x + 0.5)), h))
d = sqrt(sum(map(lambda x, y: (x - y)**2, h, h0)))
# if not hasattr(reflections[h0], "xd"):
# reflections[h0].xd = d
# reflections[h0].xc = i
# reflections[h0].yc = j
# elif reflections[h0].xd > d:
# reflections[h0].xd = d
# reflections[h0].xc = i
# reflections[h0].yc = j
if d < 0.3:
mask[j, i] = True
reflections[h0].append((j, i))
# from matplotlib import pylab
# #pylab.imshow(mask.as_numpy_array(), interpolation='none')
# pylab.show()
print("Integrating reflections")
miller_index = flex.miller_index()
intensity = flex.double()
variance = flex.double()
bbox = flex.int6()
xyz = flex.vec3_double()
for h, r in reflections.iteritems():
# xc = r.xc
# yc = r.yc
b_sum = 0
f_sum = 0
b_cnt = 0
f_cnt = 0
for i in range(len(r)):
y, x = r[i]
m = mask[y, x]
if data[y, x] >= 0:
if m:
f_sum += data[y, x]
f_cnt += 1
else:
b_sum += data[y, x]
b_cnt += 1
Y, X = zip(*r)
x0, x1, y0, y1 = min(X), max(X), min(Y), max(Y)
if f_cnt > 0 and b_cnt > 0:
B = b_sum / b_cnt
I = f_sum - B * f_cnt
V = f_sum + B * (1 + f_cnt / b_cnt)
miller_index.append(h)
intensity.append(I)
variance.append(V)
bbox.append((x0, x1, y0, y1, 0, 1))
# xyz.append((xc, yc, 0))
print("Integrated %d reflections" % len(reflections))
print(flex.min(intensity), flex.max(intensity), flex.mean(intensity))
reflections = flex.reflection_table()
reflections["miller_index"] = miller_index
reflections["intensity.sum.value"] = intensity
reflections["intensity.sum.variance"] = variance
reflections["bbox"] = bbox
reflections["panel"] = flex.size_t(len(reflections), 0)
reflections["id"] = flex.size_t(len(reflections), 0)
# reflections["xyzcal.px"] = xyz
# reflections["xyzobs.px"] = xyz
reflections.set_flags(flex.size_t(range(len(reflections))),
reflections.flags.integrated_sum)
return reflections
def set_reflections(self, reflections):
'''Set the reflection data.'''
self.set_data(reflections, ReflectionListEncoder())
def get_reflections(self):
'''Get the reflection data.'''
return self.get_data(ReflectionListDecoder())
if __name__ == '__main__':
from dials.array_family import flex
reflections = flex.reflection_table([
('hkl', flex.miller_index(10)),
('s1', flex.vec3_double(10)),
('bbox', flex.int6(10)),
('id', flex.int(10)),
('shoebox', flex.shoebox(10))
])
for i in range(10):
reflections['shoebox'][i].data = flex.double(flex.grid(10,10,10))
reflections['shoebox'][i].mask = flex.int(flex.grid(10,10,10))
reflections['shoebox'][i].background = flex.double(flex.grid(10,10,10))
for i in range(10):
print reflections['shoebox'][i].data.all()
writer = NexusFile('temp.h5', 'w')
writer.set_reflections(reflections)
writer.close()
def tst_extract_shoeboxes(self):
from dials.array_family import flex
from random import randint, seed
from dials.algorithms.shoebox import MaskCode
import sys
seed(0)
reflections = flex.reflection_table()
reflections['panel'] = flex.size_t()
reflections['bbox'] = flex.int6()
npanels = 2
width = 1000
height = 1000
nrefl = 10000
frame0 = 10
frame1 = 100
nrefl = 1000
for i in range(nrefl):
xs = randint(5, 10)
ys = randint(5, 10)
x0 = randint(-xs+1, width-1)
y0 = randint(-ys+1, height-1)
z0 = randint(frame0, frame1-1)
x1 = x0 + xs
y1 = y0 + ys
z1 = min([z0 + randint(1, 10), frame1])
assert(x1 > x0)
assert(y1 > y0)
assert(z1 > z0)
assert(z0 >= frame0 and z1 <= frame1)
bbox = (x0, x1, y0, y1, z0, z1)
reflections.append({
"panel" : randint(0,1),
"bbox" : bbox,
})
reflections['shoebox'] = flex.shoebox(
reflections['panel'],
reflections['bbox'])
reflections['shoebox'].allocate()
class FakeImageSet(object):
def __init__(self):
from dials.array_family import flex
self.data = flex.int(range(height*width))
self.data.reshape(flex.grid(height,width))
def get_array_range(self):
return (frame0, frame1)
def get_detector(self):
class FakeDetector(object):
def __len__(self):
return npanels
def __getitem__(self, index):
class FakePanel(object):
def get_trusted_range(self):
return (-1, 1000000)
return FakePanel()
return FakeDetector()
def __len__(self):
return frame1 - frame0
def __getitem__(self, index):
f = frame0+index
return (self.data + f*1, self.data + f*2)
def get_corrected_data(self, index):
f = frame0+index
return (self.data + f*1, self.data + f*2)
def get_mask(self, index):
image = self.get_corrected_data(index)
return tuple(im >= 0 for im in image)
imageset = FakeImageSet()
stdout = sys.stdout
class DevNull(object):
def write(self, *args):
pass
def flush(self):
pass
sys.stdout = DevNull()
reflections.extract_shoeboxes(imageset)
sys.stdout = stdout
for i in range(len(reflections)):
sbox = reflections[i]["shoebox"]
assert(sbox.is_consistent())
mask = sbox.mask
data = sbox.data
bbox = sbox.bbox
panel = sbox.panel
x0, x1, y0, y1, z0, z1 = bbox
for z in range(z1 - z0):
for y in range(y1 - y0):
for x in range(x1 - x0):
v1 = data[z,y,x]
m1 = mask[z,y,x]
if (x0 + x >= 0 and y0 + y >= 0 and
x0 + x < width and y0 + y < height):
v2 = imageset.data[y+y0,x+x0] + (z+z0)*(panel+1)
m2 = MaskCode.Valid
assert(v1 == v2)
assert(m1 == m2)
else:
assert(v1 == 0)
assert(m1 == 0)
print 'OK'
def run(self):
''' Extract the shoeboxes. '''
from dials.util.options import flatten_reflections
from dials.util.options import flatten_experiments
from dials.util.options import flatten_datablocks
from dials.util import log
from dials.array_family import flex
from libtbx.utils import Sorry
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# Configure logging
log.config()
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Get the data
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
datablocks = flatten_datablocks(params.input.datablock)
if not any([experiments, datablocks, reflections]):
self.parser.print_help()
exit(0)
elif experiments and datablocks:
raise Sorry('Both experiment list and datablocks set')
elif len(experiments) > 1:
raise Sorry('More than 1 experiment set')
elif len(datablocks) > 1:
raise Sorry('More than 1 datablock set')
elif len(experiments) == 1:
imageset = experiments[0].imageset
elif len(datablocks) == 1:
imagesets = datablocks[0].extract_imagesets()
if len(imagesets) != 1:
raise Sorry('Need 1 imageset, got %d' % len(imagesets))
imageset = imagesets[0]
if len(reflections) != 1:
raise Sorry('Need 1 reflection table, got %d' % len(reflections))
else:
reflections = reflections[0]
# Check the reflections contain the necessary stuff
assert("bbox" in reflections)
assert("panel" in reflections)
# Get some models
detector = imageset.get_detector()
scan = imageset.get_scan()
frame0, frame1 = scan.get_array_range()
# Add some padding but limit to image volume
if params.padding > 0:
logger.info('Adding %d pixels as padding' % params.padding)
x0, x1, y0, y1, z0, z1 = reflections['bbox'].parts()
x0 -= params.padding
x1 += params.padding
y0 -= params.padding
y1 += params.padding
# z0 -= params.padding
# z1 += params.padding
panel = reflections['panel']
for i in range(len(reflections)):
width, height = detector[panel[i]].get_image_size()
if x0[i] < 0: x0[i] = 0
if x1[i] > width: x1[i] = width
if y0[i] < 0: y0[i] = 0
if y1[i] > height: y1[i] = height
if z0[i] < frame0: z0[i] = frame0
if z1[i] > frame1: z1[i] = frame1
reflections['bbox'] = flex.int6(x0, x1, y0, y1, z0, z1)
# Save the old shoeboxes
if "shoebox" in reflections:
old_shoebox = reflections['shoebox']
else:
old_shoebox = None
# Allocate the shoeboxes
reflections["shoebox"] = flex.shoebox(
reflections["panel"],
reflections["bbox"],
allocate=True)
# Extract the shoeboxes
reflections.extract_shoeboxes(imageset, verbose=True)
# Preserve masking
if old_shoebox is not None:
from dials.algorithms.shoebox import MaskCode
logger.info("Applying old shoebox mask")
new_shoebox = reflections['shoebox']
for i in range(len(reflections)):
bbox0 = old_shoebox[i].bbox
bbox1 = new_shoebox[i].bbox
mask0 = old_shoebox[i].mask
mask1 = new_shoebox[i].mask
mask2 = flex.int(mask1.accessor(), 0)
x0 = bbox0[0] - bbox1[0]
x1 = bbox0[1] - bbox0[0] + x0
y0 = bbox0[2] - bbox1[2]
y1 = bbox0[3] - bbox0[2] + y0
z0 = bbox0[4] - bbox1[4]
z1 = bbox0[5] - bbox0[4] + z0
mask2[z0:z1,y0:y1,x0:x1] = mask0
mask1 = mask1.as_1d() | mask2.as_1d()
if params.padding_is_background:
selection = flex.size_t(range(len(mask1))).select(mask1 == MaskCode.Valid)
values = flex.int(len(selection), MaskCode.Valid | MaskCode.Background)
mask1.set_selected(selection, values)
mask1.reshape(new_shoebox[i].mask.accessor())
new_shoebox[i].mask = mask1
# Saving the reflections to disk
filename = params.output.reflections
logger.info('Saving %d reflections to %s' % (len(reflections), filename))
reflections.as_pickle(filename)
def test_extract_shoeboxes():
from dials.algorithms.shoebox import MaskCode
random.seed(0)
reflections = flex.reflection_table()
reflections["panel"] = flex.size_t()
reflections["bbox"] = flex.int6()
npanels = 2
width = 1000
height = 1000
frame0 = 10
frame1 = 100
nrefl = 1000
for i in range(nrefl):
xs = random.randint(5, 10)
ys = random.randint(5, 10)
x0 = random.randint(-xs + 1, width - 1)
y0 = random.randint(-ys + 1, height - 1)
z0 = random.randint(frame0, frame1 - 1)
x1 = x0 + xs
y1 = y0 + ys
z1 = min([z0 + random.randint(1, 10), frame1])
assert x1 > x0
assert y1 > y0
assert z1 > z0
assert z0 >= frame0 and z1 <= frame1
bbox = (x0, x1, y0, y1, z0, z1)
reflections.append({"panel": random.randint(0, 1), "bbox": bbox})
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"])
reflections["shoebox"].allocate()
class FakeImageSet(object):
def __init__(self):
self.data = flex.int(range(height * width))
self.data.reshape(flex.grid(height, width))
def get_array_range(self):
return (frame0, frame1)
def get_detector(self):
class FakeDetector(object):
def __len__(self):
return npanels
def __getitem__(self, index):
class FakePanel(object):
def get_trusted_range(self):
return (-1, 1000000)
return FakePanel()
return FakeDetector()
def __len__(self):
return frame1 - frame0
def __getitem__(self, index):
f = frame0 + index
return (self.data + f * 1, self.data + f * 2)
def get_corrected_data(self, index):
f = frame0 + index
return (self.data + f * 1, self.data + f * 2)
def get_mask(self, index):
image = self.get_corrected_data(index)
return tuple(im >= 0 for im in image)
imageset = FakeImageSet()
reflections.extract_shoeboxes(imageset)
for i in range(len(reflections)):
sbox = reflections[i]["shoebox"]
assert sbox.is_consistent()
mask = sbox.mask
data = sbox.data
bbox = sbox.bbox
panel = sbox.panel
x0, x1, y0, y1, z0, z1 = bbox
for z in range(z1 - z0):
for y in range(y1 - y0):
for x in range(x1 - x0):
v1 = data[z, y, x]
m1 = mask[z, y, x]
if (x0 + x >= 0 and y0 + y >= 0 and x0 + x < width
and y0 + y < height):
v2 = imageset.data[y + y0,
x + x0] + (z + z0) * (panel + 1)
m2 = MaskCode.Valid
assert v1 == v2
assert m1 == m2
else:
assert v1 == 0
assert m1 == 0
def test_export_dials():
from dials.array_family import flex
print("Creating dummy reflection table...")
table = flex.reflection_table()
table["miller_index"] = flex.miller_index(100)
table["id"] = flex.int(100)
table["intensity.sum.value"] = flex.double(100)
table["intensity.sum.variance"] = flex.double(100)
table["intensity.prf.value"] = flex.double(100)
table["intensity.prf.variance"] = flex.double(100)
table["lp"] = flex.double(100)
table["panel"] = flex.size_t(100)
table["bbox"] = flex.int6(100)
table["xyzcal.px"] = flex.vec3_double(100)
table["xyzcal.mm"] = flex.vec3_double(100)
table["xyzobs.px.value"] = flex.vec3_double(100)
table["xyzobs.px.variance"] = flex.vec3_double(100)
table["xyzobs.mm.value"] = flex.vec3_double(100)
table["xyzobs.mm.variance"] = flex.vec3_double(100)
table["partiality"] = flex.double(100)
table["d"] = flex.double(100)
table["s1"] = flex.vec3_double(100)
table["rlp"] = flex.vec3_double(100)
table["background.mean"] = flex.double(100)
table["entering"] = flex.bool(100)
table["flags"] = flex.size_t(100)
table["profile.correlation"] = flex.double(100)
# Open the file
outfile = File("test_file.mtz2", "w")
# Get the entry
entry = outfile.entry
print("Writing reflection table stuff...")
# Save some processed data
diffraction = entry.diffraction
# Set the experiments
experiment = diffraction.experiments[0]
experiment["beam"] = 0
experiment["detector"] = 0
experiment["goniometer"] = 0
experiment["scan"] = 0
experiment["crystal"] = 0
# Get columns into array
col1, col2, col3 = zip(*list(table["miller_index"]))
col4 = table["id"]
col5 = table["intensity.sum.value"]
col6 = table["intensity.sum.variance"]
col7 = table["intensity.prf.value"]
col8 = table["intensity.prf.variance"]
col9 = table["lp"]
col10 = table["panel"]
col11, col12, col13, col14, col15, col16 = table["bbox"].parts()
col17, col18, col19 = table["xyzcal.px"].parts()
col20, col21, col22 = table["xyzcal.mm"].parts()
col23, col24, col25 = table["xyzobs.px.value"].parts()
col26, col27, col28 = table["xyzobs.px.variance"].parts()
col29, col30, col31 = table["xyzobs.mm.value"].parts()
col32, col33, col34 = table["xyzobs.mm.variance"].parts()
col35 = table["partiality"]
col36 = table["d"]
col37 = table["background.mean"]
col38 = table["entering"]
col39 = table["flags"]
col40 = table["profile.correlation"]
# Some data
diffraction["h"] = col1
diffraction["k"] = col2
diffraction["l"] = col3
diffraction["id"] = col4
diffraction["int_sum_val"] = col5
diffraction["int_sum_var"] = col6
diffraction["int_prf_val"] = col7
diffraction["int_prf_var"] = col8
diffraction["lp"] = col9
diffraction["det_module"] = col10
diffraction["bbx0"] = col11
diffraction["bbx1"] = col12
diffraction["bby0"] = col13
diffraction["bby1"] = col14
diffraction["bbz0"] = col15
diffraction["bbz1"] = col16
diffraction["prd_px_x"] = col17
diffraction["prd_px_y"] = col18
diffraction["prd_frame"] = col19
diffraction["prd_mm_x"] = col20
diffraction["prd_mm_y"] = col21
diffraction["prd_phi"] = col22
diffraction["obs_px_x_val"] = col23
diffraction["obs_px_x_var"] = col24
diffraction["obs_px_y_val"] = col25
diffraction["obs_px_y_var"] = col26
diffraction["obs_frame_val"] = col27
diffraction["obs_frame_var"] = col28
diffraction["obs_mm_x_val"] = col29
diffraction["obs_mm_x_var"] = col30
diffraction["obs_mm_y_val"] = col31
diffraction["obs_mm_y_var"] = col32
diffraction["obs_phi_val"] = col33
diffraction["obs_phi_var"] = col34
diffraction["partiality"] = col35
diffraction["d"] = col36
diffraction["bkg_mean"] = col37
diffraction["entering"] = col38
diffraction["flags"] = col39
diffraction["prf_cc"] = col40
# Flush the file
outfile.flush()
def test_split_partials_with_shoebox():
from dials.model.data import Shoebox
r = flex.reflection_table()
r["value1"] = flex.double()
r["value2"] = flex.int()
r["value3"] = flex.double()
r["bbox"] = flex.int6()
r["panel"] = flex.size_t()
r["shoebox"] = flex.shoebox()
expected = []
for i in range(100):
x0 = random.randint(0, 100)
x1 = x0 + random.randint(1, 10)
y0 = random.randint(0, 100)
y1 = y0 + random.randint(1, 10)
z0 = random.randint(0, 100)
z1 = z0 + random.randint(1, 10)
v1 = random.uniform(0, 100)
v2 = random.randint(0, 100)
v3 = random.uniform(0, 100)
sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1))
sbox.allocate()
assert sbox.is_consistent()
w = x1 - x0
h = y1 - y0
for z in range(z0, z1):
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[z - z0, y - y0, x - x0] = x + y * w + z * w * h
r.append({
"value1": v1,
"value2": v2,
"value3": v3,
"bbox": (x0, x1, y0, y1, z0, z1),
"panel": 0,
"shoebox": sbox,
})
for z in range(z0, z1):
sbox = Shoebox(0, (x0, x1, y0, y1, z, z + 1))
sbox.allocate()
assert sbox.is_consistent()
w = x1 - x0
h = y1 - y0
for y in range(y0, y1):
for x in range(x0, x1):
sbox.data[0, y - y0, x - x0] = x + y * w + z * w * h
expected.append({
"value1": v1,
"value2": v2,
"value3": v3,
"bbox": (x0, x1, y0, y1, z, z + 1),
"partial_id": i,
"panel": 0,
"shoebox": sbox,
})
r.split_partials_with_shoebox()
assert len(r) == len(expected)
EPS = 1e-7
for r1, r2 in zip(r.rows(), expected):
assert abs(r1["value1"] - r2["value1"]) < EPS
assert r1["value2"] == r2["value2"]
assert abs(r1["value3"] - r2["value3"]) < EPS
assert r1["bbox"] == r2["bbox"]
assert r1["partial_id"] == r2["partial_id"]
assert r1["panel"] == r2["panel"]
assert (r1["shoebox"].data.as_double().as_1d().all_approx_equal(
r2["shoebox"].data.as_double().as_1d()))
def test_split_blocks_overlapping():
from scitbx.array_family import shared
from dials.algorithms.integration.integrator import JobList
from dials.array_family import flex
blocks = shared.tiny_int_2([
(0, 10),
(5, 15),
(10, 20),
(15, 25),
(20, 30),
(25, 35),
(30, 40),
(35, 45),
(40, 50),
(45, 55),
(50, 60),
(55, 65),
(60, 70),
(65, 75),
(70, 80),
(75, 85),
(80, 90),
(85, 95),
(90, 100),
(95, 105),
(100, 110),
])
jobs = JobList((0, 1), blocks)
r = flex.reflection_table()
r["value1"] = flex.double()
r["value2"] = flex.int()
r["value3"] = flex.double()
r["bbox"] = flex.int6()
r["id"] = flex.int()
expected = []
for i in range(100):
x0 = random.randint(0, 100)
x1 = x0 + random.randint(1, 10)
y0 = random.randint(0, 100)
y1 = y0 + random.randint(1, 10)
z0 = random.randint(0, 90)
z1 = z0 + random.randint(1, 20)
v1 = random.uniform(0, 100)
v2 = random.randint(0, 100)
v3 = random.uniform(0, 100)
r.append({
"id": 0,
"value1": v1,
"value2": v2,
"value3": v3,
"bbox": (x0, x1, y0, y1, z0, z1),
})
expected.append({
"id": 0,
"value1": v1,
"value2": v2,
"value3": v3,
"bbox": (x0, x1, y0, y1, z0, z1),
})
jobs.split(r)
assert len(r) > 100
for r1 in r.rows():
v1 = r1["value1"]
v2 = r1["value2"]
v3 = r1["value3"]
bbox = r1["bbox"]
pid = r1["partial_id"]
z0 = bbox[4]
z1 = bbox[5]
success = False
for i in range(len(blocks)):
b0 = blocks[i][0]
b1 = blocks[i][1]
if z0 >= b0 and z1 <= b1:
success = True
break
assert success
v11 = expected[pid]["value1"]
v22 = expected[pid]["value2"]
v33 = expected[pid]["value3"]
bb = expected[pid]["bbox"]
assert v11 == v1
assert v22 == v2
assert v33 == v3
assert bb[0] == bbox[0]
assert bb[1] == bbox[1]
assert bb[2] == bbox[2]
assert bb[3] == bbox[3]
def test_reflection_manager():
from dials.array_family import flex
reflections = flex.reflection_table()
reflections["panel"] = flex.size_t()
reflections["bbox"] = flex.int6()
reflections["miller_index"] = flex.miller_index()
reflections["s1"] = flex.vec3_double()
reflections["xyzcal.px"] = flex.vec3_double()
reflections["xyzcal.mm"] = flex.vec3_double()
reflections["entering"] = flex.bool()
reflections["id"] = flex.int()
reflections["flags"] = flex.size_t()
width = 1000
height = 1000
nrefl = 10000
array_range = (0, 130)
block_size = 20
block_overlap = 10
random.seed(0)
processed = [[] for i in range(12)]
for i in range(nrefl):
x0 = random.randint(0, width - 10)
y0 = random.randint(0, height - 10)
zs = random.randint(2, 9)
x1 = x0 + random.randint(2, 10)
y1 = y0 + random.randint(2, 10)
for k, j in enumerate(
[10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]):
m = k + i * 12
pos = random.choice(["left", "right", "centre"])
if pos == "left":
z0 = j - zs
z1 = j
elif pos == "right":
z0 = j
z1 = j + zs
else:
z0 = j - zs // 2
z1 = j + zs // 2
bbox = (x0, x1, y0, y1, z0, z1)
reflections.append({
"panel":
random.randint(0, 1),
"bbox":
bbox,
"flags":
flex.reflection_table.flags.reference_spot,
})
processed[k].append(m)
# Add reflection to ignore
# zc = random.choice([10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120])
# z0 = zc - 11
# z1 = zc + 11
# bbox = (x0, x1, y0, y1, z0, z1)
# reflections.append({
# "panel" : randint(0,1),
# "bbox" : bbox,
# "flags" : flex.reflection_table.flags.reference_spot
# })
from dials.algorithms.integration.integrator import JobList, ReflectionManager
jobs = JobList()
jobs.add((0, 1), array_range, block_size, block_overlap)
# Create the executor
executor = ReflectionManager(jobs, reflections)
# Ensure the tasks make sense
jobs = [executor.job(i) for i in range(len(executor))]
assert len(executor) == 12
assert not executor.finished()
assert len(jobs) == 12
assert jobs[0].frames() == (0, 20)
assert jobs[1].frames() == (10, 30)
assert jobs[2].frames() == (20, 40)
assert jobs[3].frames() == (30, 50)
assert jobs[4].frames() == (40, 60)
assert jobs[5].frames() == (50, 70)
assert jobs[6].frames() == (60, 80)
assert jobs[7].frames() == (70, 90)
assert jobs[8].frames() == (80, 100)
assert jobs[9].frames() == (90, 110)
assert jobs[10].frames() == (100, 120)
assert jobs[11].frames() == (110, 130)
# Get the task specs
data0 = executor.split(0)
data1 = executor.split(1)
data2 = executor.split(2)
data3 = executor.split(3)
data4 = executor.split(4)
data5 = executor.split(5)
data6 = executor.split(6)
data7 = executor.split(7)
data8 = executor.split(8)
data9 = executor.split(9)
data10 = executor.split(10)
data11 = executor.split(11)
assert len(data0) == len(processed[0])
assert len(data1) == len(processed[1])
assert len(data2) == len(processed[2])
assert len(data3) == len(processed[3])
assert len(data4) == len(processed[4])
assert len(data5) == len(processed[5])
assert len(data6) == len(processed[6])
assert len(data7) == len(processed[7])
assert len(data8) == len(processed[8])
assert len(data9) == len(processed[9])
assert len(data10) == len(processed[10])
assert len(data11) == len(processed[11])
# Add some results
data0["data"] = flex.double(len(data0), 1)
data1["data"] = flex.double(len(data1), 2)
data2["data"] = flex.double(len(data2), 3)
data3["data"] = flex.double(len(data3), 4)
data4["data"] = flex.double(len(data4), 5)
data5["data"] = flex.double(len(data5), 6)
data6["data"] = flex.double(len(data6), 7)
data7["data"] = flex.double(len(data7), 8)
data8["data"] = flex.double(len(data8), 9)
data9["data"] = flex.double(len(data9), 10)
data10["data"] = flex.double(len(data10), 11)
data11["data"] = flex.double(len(data11), 12)
# Accumulate the data again
assert not executor.finished()
executor.accumulate(0, data0)
executor.accumulate(1, data1)
executor.accumulate(2, data2)
executor.accumulate(3, data3)
executor.accumulate(4, data4)
executor.accumulate(5, data5)
executor.accumulate(6, data6)
executor.accumulate(7, data7)
executor.accumulate(8, data8)
executor.accumulate(9, data9)
executor.accumulate(10, data10)
executor.accumulate(11, data11)
assert executor.finished()
# Get results and check they're as expected
data = executor.data()
result = data["data"]
for i in range(len(processed)):
for j in range(len(processed[i])):
assert result[processed[i][j]] == i + 1
def refl_bbox_maker(self): self.reflections['bbox'] = flex.int6(self.length)
