def test_runtime_error_raised_when_not_enough_memory(mock_psutil_swap, mock_psutil_vm, mock_flex_max): mock_flex_max.return_value = 750001 mock_psutil_vm.return_value.available = 1000000 mock_psutil_swap.return_value.free = 0 phil_mock = mock.Mock() phil_mock.mp.method = "multiprocessing" phil_mock.mp.nproc = 4 phil_mock.block.max_memory_usage = 0.75 reflections = {"bbox": flex.int6(1000, (0, 1, 0, 1, 0, 1))} manager = dials.algorithms.integration.processor._Manager( None, reflections, phil_mock) manager.jobs = mock.Mock(autospec=JobList) with pytest.raises(MemoryError) as exc_info: manager.compute_processors() assert "Not enough memory to run integration jobs." in exc_info.value.args[ 0] mock_flex_max.assert_called_once_with( manager.jobs.shoebox_memory.return_value) # Reduce memory usage by 1 byte, should then pass mock_flex_max.return_value = 750000 manager.compute_processors() mock_flex_max.assert_called_with(manager.jobs.shoebox_memory.return_value)
def test_single_panel(): from dials.array_family import flex nrefl = 1000 # Generate bboxes bbox = flex.int6(nrefl) for i in range(nrefl): x0 = random.randint(0, 500) y0 = random.randint(0, 500) z0 = random.randint(0, 10) x1 = x0 + random.randint(2, 10) y1 = y0 + random.randint(2, 10) z1 = z0 + random.randint(2, 10) bbox[i] = (x0, x1, y0, y1, z0, z1) # Find the overlaps overlaps = find_overlapping(bbox) assert overlaps.num_vertices() == nrefl overlaps2 = brute_force(bbox) assert overlaps.num_edges() == len(overlaps2) edges = {} for edge in overlaps2: edge = (min(edge), max(edge)) edges[edge] = None for edge in overlaps.edges(): edge = overlaps.source(edge), overlaps.target(edge) edge = (min(edge), max(edge)) assert edge in edges
def tst_multiple_panels(self): from dials.array_family import flex from random import randint nrefl = 1000 # Generate bboxes bbox = flex.int6(nrefl) panel = flex.size_t(nrefl) for i in range(nrefl): x0 = randint(0, 500) y0 = randint(0, 500) z0 = randint(0, 10) x1 = x0 + randint(2, 10) y1 = y0 + randint(2, 10) z1 = z0 + randint(2, 10) bbox[i] = (x0, x1, y0, y1, z0, z1) panel[i] = randint(0,2) # Find the overlaps overlaps = find_overlapping(bbox, panel) assert(overlaps.num_vertices() == nrefl) overlaps2 = self.brute_force(bbox, panel) assert(overlaps.num_edges() == len(overlaps2)) edges = {} for edge in overlaps2: edge = (min(edge), max(edge)) edges[edge] = None for edge in overlaps.edges(): edge = (overlaps.source(edge), overlaps.target(edge)) edge = (min(edge), max(edge)) assert(edge in edges) print 'OK'
def test_split_blocks_non_overlapping(): from dials.array_family import flex from dials.algorithms.integration.integrator import JobList from scitbx.array_family import shared blocks = shared.tiny_int_2([(0, 10), (10, 20), (20, 30), (30, 35), (35, 40), (40, 50), (50, 60), (60, 70), (70, 80), (80, 90), (90, 100), (100, 110)]) jobs = JobList((0, 1), blocks) r = flex.reflection_table() r['value1'] = flex.double() r['value2'] = flex.int() r['value3'] = flex.double() r['bbox'] = flex.int6() r['id'] = flex.int() expected = [] for i in range(100): x0 = random.randint(0, 100) x1 = x0 + random.randint(1, 10) y0 = random.randint(0, 100) y1 = y0 + random.randint(1, 10) z0 = random.randint(0, 100) z1 = z0 + random.randint(1, 10) v1 = random.uniform(0, 100) v2 = random.randint(0, 100) v3 = random.uniform(0, 100) r.append({ 'id': 0, 'value1': v1, 'value2': v2, 'value3': v3, 'bbox': (x0, x1, y0, y1, z0, z1) }) for j in range(len(blocks)): b0 = blocks[j][0] b1 = blocks[j][1] if ((z0 >= b0 and z1 <= b1) or (z0 < b1 and z1 >= b1) or (z0 < b0 and z1 > b0)): z00 = max(b0, z0) z11 = min(b1, z1) expected.append({ 'id': 0, 'value1': v1, 'value2': v2, 'value3': v3, 'bbox': (x0, x1, y0, y1, z00, z11), 'partial_id': i, }) jobs.split(r) assert (len(r) == len(expected)) EPS = 1e-7 for r1, r2 in zip(r, expected): assert (r1['bbox'] == r2['bbox']) assert (r1['partial_id'] == r2['partial_id']) assert (abs(r1['value1'] - r2['value1']) < EPS) assert (r1['value2'] == r2['value2']) assert (abs(r1['value3'] - r2['value3']) < EPS)
def test_find_overlapping(): N = 10000 r = flex.reflection_table(N) r["bbox"] = flex.int6(N) r["panel"] = flex.size_t(N) r["id"] = flex.int(N) r["imageset_id"] = flex.int(N) for i in range(N): x0 = random.randint(0, 100) x1 = random.randint(1, 10) + x0 y0 = random.randint(0, 100) y1 = random.randint(1, 10) + y0 z0 = random.randint(0, 100) z1 = random.randint(1, 10) + z0 panel = random.randint(0, 2) pid = random.randint(0, 2) r["bbox"][i] = (x0, x1, y0, y1, z0, z1) r["panel"][i] = panel r["id"][i] = pid r["imageset_id"][i] = pid def is_overlap(b0, b1, border): b0 = ( b0[0] - border, b0[1] + border, b0[2] - border, b0[3] + border, b0[4] - border, b0[5] + border, ) b1 = ( b1[0] - border, b1[1] + border, b1[2] - border, b1[3] + border, b1[4] - border, b1[5] + border, ) if not (b1[0] > b0[1] or b1[1] < b0[0] or b1[2] > b0[3] or b1[3] < b0[2] or b1[4] > b0[5] or b1[5] < b0[4]): return True return False for i in [0, 2, 5]: overlaps = r.find_overlaps(border=i) for item in overlaps.edges(): i0 = overlaps.source(item) i1 = overlaps.target(item) r0 = r[i0] r1 = r[i1] p0 = r0["panel"] p1 = r1["panel"] b0 = r0["bbox"] b1 = r1["bbox"] j0 = r0["imageset_id"] j1 = r1["imageset_id"] assert j0 == j1 assert p0 == p1 assert is_overlap(b0, b1, i)
def tst_find_overlapping(self): from dials.array_family import flex from random import randint, uniform from dials.model.data import Shoebox N = 10000 r = flex.reflection_table(N) r['bbox'] = flex.int6(N) r['panel'] = flex.size_t(N) r['id'] = flex.int(N) r['imageset_id'] = flex.int(N) for i in range(N): x0 = randint(0, 100) x1 = randint(1, 10) + x0 y0 = randint(0, 100) y1 = randint(1, 10) + y0 z0 = randint(0, 100) z1 = randint(1, 10) + z0 panel = randint(0,2) pid = randint(0,2) r['bbox'][i] = (x0,x1,y0,y1,z0,z1) r['panel'][i] = panel r['id'][i] = pid r['imageset_id'][i] = pid def is_overlap(b0, b1, border): b0 = b0[0]-border,b0[1]+border,b0[2]-border,b0[3]+border,b0[4]-border,b0[5]+border b1 = b1[0]-border,b1[1]+border,b1[2]-border,b1[3]+border,b1[4]-border,b1[5]+border if not (b1[0] > b0[1] or b1[1] < b0[0] or b1[2] > b0[3] or b1[3] < b0[2] or b1[4] > b0[5] or b1[5] < b0[4]): return True return False for i in [0, 2, 5]: overlaps = r.find_overlaps(border=i) for item in overlaps.edges(): i0 = overlaps.source(item) i1 = overlaps.target(item) r0 = r[i0] r1 = r[i1] p0 = r0['panel'] p1 = r1['panel'] b0 = r0['bbox'] b1 = r1['bbox'] j0 = r0['imageset_id'] j1 = r1['imageset_id'] assert j0 == j1 assert p0 == p1 assert is_overlap(b0,b1,i) print 'OK'
def __init__(self, params, exlist, reference=None, predicted=None, shoeboxes=None): '''Initialise the script.''' assert reference is not None # Load the extractor based on the input if shoeboxes is not None: extractor = self._load_extractor(shoeboxes, params, exlist) else: if predicted is None: predicted = self._predict_reflections(params, exlist) #predicted = self._filter_reflections(params, exlist, predicted) # FIXME predicted = self._match_with_reference(predicted, reference) from annlib_ext import AnnAdaptor from dials.array_family import flex import math matcheddata = predicted.select(predicted.get_flags(predicted.flags.reference_spot)) A = AnnAdaptor(matcheddata['xyzcal.mm'].as_double(), 3, 10) A.query(predicted['xyzcal.mm'].as_double()) bboxes = flex.int6() for i, ref in enumerate(predicted): nn_pred = [matcheddata[A.nn[i*10+j]] for j in xrange(10)] nn_ref = [reference[reference['miller_index'].first_index(r['miller_index'])] for r in nn_pred] max_x = max([r['bbox'][1]-r['bbox'][0] for r in nn_ref]) max_y = max([r['bbox'][3]-r['bbox'][2] for r in nn_ref]) max_z = max([r['bbox'][5]-r['bbox'][4] for r in nn_ref]) panel = exlist[ref['id']].detector[ref['panel']] imgsize_x, imgsize_y = panel.get_image_size() x1 = int(math.floor(ref['xyzcal.px'][0] - (max_x / 2))) x2 = int(math.ceil (ref['xyzcal.px'][0] + (max_x / 2))) y1 = int(math.floor(ref['xyzcal.px'][1] - (max_y / 2))) y2 = int(math.ceil (ref['xyzcal.px'][1] + (max_y / 2))) if x1 < 0: x1 = 0 if y1 < 0: y1 = 0 if x2 > imgsize_x: x2 = imgsize_x if y2 > imgsize_y: y2 = imgsize_y bboxes.append((x1,x2,y1,y2,0,1)) predicted['bbox'] = bboxes extractor = self._create_extractor(params, exlist, predicted) # Initialise the integrator self._integrator = ReflectionBlockIntegratorStills(params, exlist, reference, extractor)
def test_split_blocks_1_frame(): from dials.algorithms.integration.integrator import JobList from dials.array_family import flex r = flex.reflection_table() r["value1"] = flex.double() r["value2"] = flex.int() r["value3"] = flex.double() r["bbox"] = flex.int6() r["id"] = flex.int() expected = [] for i in range(100): x0 = random.randint(0, 100) x1 = x0 + random.randint(1, 10) y0 = random.randint(0, 100) y1 = y0 + random.randint(1, 10) z0 = random.randint(0, 100) z1 = z0 + random.randint(1, 10) v1 = random.uniform(0, 100) v2 = random.randint(0, 100) v3 = random.uniform(0, 100) r.append( { "id": 0, "value1": v1, "value2": v2, "value3": v3, "bbox": (x0, x1, y0, y1, z0, z1), } ) for z in range(z0, z1): expected.append( { "id": 0, "value1": v1, "value2": v2, "value3": v3, "bbox": (x0, x1, y0, y1, z, z + 1), "partial_id": i, } ) jobs = JobList() jobs.add((0, 1), (0, 111), 1, 0) jobs.split(r) assert len(r) == len(expected) EPS = 1e-7 for r1, r2 in zip(r.rows(), expected): assert r1["bbox"] == r2["bbox"] assert r1["partial_id"] == r2["partial_id"] assert abs(r1["value1"] - r2["value1"]) < EPS assert r1["value2"] == r2["value2"] assert abs(r1["value3"] - r2["value3"]) < EPS
def __init__(self): from dials.array_family import flex self.reflections = flex.reflection_table() self.reflections['panel'] = flex.size_t() self.reflections['bbox'] = flex.int6() self.reflections['miller_index'] = flex.miller_index() self.reflections['s1'] = flex.vec3_double() self.reflections['xyzcal.px'] = flex.vec3_double() self.reflections['xyzcal.mm'] = flex.vec3_double() self.reflections['entering'] = flex.bool() self.reflections['id'] = flex.int() self.reflections["flags"] = flex.size_t() self.npanels = 2 self.width = 1000 self.height = 1000 self.nrefl = 10000 self.array_range = (0, 130) self.block_size = 20 from random import randint, seed, choice seed(0) self.processed = [[] for i in range(12)] for i in range(self.nrefl): x0 = randint(0, self.width - 10) y0 = randint(0, self.height - 10) zs = randint(2, 9) x1 = x0 + randint(2, 10) y1 = y0 + randint(2, 10) for k, j in enumerate( [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]): m = k + i * 12 pos = choice(["left", "right", "centre"]) if pos == 'left': z0 = j - zs z1 = j elif pos == 'right': z0 = j z1 = j + zs else: z0 = j - zs // 2 z1 = j + zs // 2 bbox = (x0, x1, y0, y1, z0, z1) self.reflections.append({ "panel": randint(0, 1), "bbox": bbox, "flags": flex.reflection_table.flags.reference_spot }) self.processed[k].append(m)
def test_RefinerData(testdata): experiment = testdata.experiment reflections = testdata.reflections panel = experiment.detector[0] s0_length = matrix.col(experiment.beam.get_s0()).length() reflections["bbox"] = flex.int6(len(reflections)) reflections["xyzobs.px.value"] = flex.vec3_double(len(reflections)) reflections["s2"] = reflections["s1"].each_normalize() * s0_length reflections["sp"] = flex.vec3_double(len(reflections)) for i, (x, y, z) in enumerate(reflections["xyzcal.px"]): x0 = int(x) - 5 x1 = int(x) + 5 + 1 y0 = int(y) - 5 y1 = int(y) + 5 + 1 z0 = int(z) z1 = z0 + 1 reflections["bbox"][i] = x0, x1, y0, y1, z0, z1 reflections["xyzobs.px.value"][i] = (int(x) + 0.5, int(y) + 0.5, int(z) + 0.5) reflections["sp"][i] = (matrix.col( panel.get_pixel_lab_coord( reflections["xyzobs.px.value"][i][0:2])).normalize() * s0_length) reflections["shoebox"] = flex.shoebox(reflections["panel"], reflections["bbox"], allocate=True) shoebox_data = flex.float(flex.grid(1, 11, 11)) shoebox_mask = flex.int(flex.grid(1, 11, 11)) for j in range(11): for i in range(11): shoebox_data[0, j, i] = (100 * exp(-0.5 * (j - 5)**2 / 1**2) * exp(-0.5 * (i - 5)**2 / 1**2)) shoebox_mask[0, j, i] = 5 for sbox in reflections["shoebox"]: sbox.data = shoebox_data sbox.mask = shoebox_mask data = RefinerData.from_reflections(experiment, reflections) assert tuple(data.s0) == pytest.approx(experiment.beam.get_s0()) assert data.h_list == reflections["miller_index"] for i, sp in enumerate(reflections["sp"]): assert data.sp_list[:, i] == pytest.approx(sp) assert data.ctot_list[0] == sum(shoebox_data) mobs1 = np.abs(data.mobs_list[0, :]) mobs2 = np.abs(data.mobs_list[1, :]) assert np.max(mobs1) < 1e-6 assert np.max(mobs2) < 1e-6
def tst_split_blocks_1_frame(self): from dials.array_family import flex from random import randint, uniform, seed from dials.algorithms.integration.integrator import JobList r = flex.reflection_table() r['value1'] = flex.double() r['value2'] = flex.int() r['value3'] = flex.double() r['bbox'] = flex.int6() r['id'] = flex.int() expected = [] for i in range(100): x0 = randint(0, 100) x1 = x0 + randint(1, 10) y0 = randint(0, 100) y1 = y0 + randint(1, 10) z0 = randint(0, 100) z1 = z0 + randint(1, 10) v1 = uniform(0, 100) v2 = randint(0, 100) v3 = uniform(0, 100) r.append({ 'id': 0, 'value1': v1, 'value2': v2, 'value3': v3, 'bbox': (x0, x1, y0, y1, z0, z1) }) for z in range(z0, z1): expected.append({ 'id': 0, 'value1': v1, 'value2': v2, 'value3': v3, 'bbox': (x0, x1, y0, y1, z, z + 1), 'partial_id': i, }) jobs = JobList() jobs.add((0, 1), (0, 111), 1) jobs.split(r) assert (len(r) == len(expected)) EPS = 1e-7 for r1, r2 in zip(r, expected): assert (r1['bbox'] == r2['bbox']) assert (r1['partial_id'] == r2['partial_id']) assert (abs(r1['value1'] - r2['value1']) < EPS) assert (r1['value2'] == r2['value2']) assert (abs(r1['value3'] - r2['value3']) < EPS) print 'OK'
def tst_find_overlapping(self): from dials.array_family import flex from random import randint, uniform from dials.model.data import Shoebox N = 10000 r = flex.reflection_table(N) r['bbox'] = flex.int6(N) r['panel'] = flex.size_t(N) r['id'] = flex.int(N) r['imageset_id'] = flex.int(N) for i in range(N): x0 = randint(0, 100) x1 = randint(1, 10) + x0 y0 = randint(0, 100) y1 = randint(1, 10) + y0 z0 = randint(0, 100) z1 = randint(1, 10) + z0 panel = randint(0, 2) pid = randint(0, 2) r['bbox'][i] = (x0, x1, y0, y1, z0, z1) r['panel'][i] = panel r['id'][i] = pid r['imageset_id'][i] = pid def is_overlap(b0, b1, border): b0 = b0[0] - border, b0[1] + border, b0[2] - border, b0[ 3] + border, b0[4] - border, b0[5] + border b1 = b1[0] - border, b1[1] + border, b1[2] - border, b1[ 3] + border, b1[4] - border, b1[5] + border if not (b1[0] > b0[1] or b1[1] < b0[0] or b1[2] > b0[3] or b1[3] < b0[2] or b1[4] > b0[5] or b1[5] < b0[4]): return True return False for i in [0, 2, 5]: overlaps = r.find_overlaps(border=i) for item in overlaps.edges(): i0 = overlaps.source(item) i1 = overlaps.target(item) r0 = r[i0] r1 = r[i1] p0 = r0['panel'] p1 = r1['panel'] b0 = r0['bbox'] b1 = r1['bbox'] j0 = r0['imageset_id'] j1 = r1['imageset_id'] assert j0 == j1 assert p0 == p1 assert is_overlap(b0, b1, i) print 'OK'
def tst_split_blocks_1_frame(self): from dials.array_family import flex from random import randint, uniform, seed from dials.algorithms.integration.integrator import JobList r = flex.reflection_table() r['value1'] = flex.double() r['value2'] = flex.int() r['value3'] = flex.double() r['bbox'] = flex.int6() r['id'] = flex.int() expected = [] for i in range(100): x0 = randint(0, 100) x1 = x0 + randint(1, 10) y0 = randint(0, 100) y1 = y0 + randint(1, 10) z0 = randint(0, 100) z1 = z0 + randint(1, 10) v1 = uniform(0, 100) v2 = randint(0, 100) v3 = uniform(0, 100) r.append({ 'id' : 0, 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z0, z1) }) for z in range(z0, z1): expected.append({ 'id' : 0, 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z, z+1), 'partial_id' : i, }) jobs = JobList() jobs.add((0,1), (0, 111), 1) jobs.split(r) assert(len(r) == len(expected)) EPS = 1e-7 for r1, r2 in zip(r, expected): assert(r1['bbox'] == r2['bbox']) assert(r1['partial_id'] == r2['partial_id']) assert(abs(r1['value1'] - r2['value1']) < EPS) assert(r1['value2'] == r2['value2']) assert(abs(r1['value3'] - r2['value3']) < EPS) print 'OK'
def run(self): from dials.algorithms.profile_model.gaussian_rs import \ PartialityCalculator3D from dials.array_family import flex calculator = PartialityCalculator3D( self.experiment.beam, self.experiment.goniometer, self.experiment.scan, self.sigma_m) predicted = flex.reflection_table.from_predictions_multi(self.experiments) predicted['bbox'] = predicted.compute_bbox(self.experiments) # Remove any touching edges of scan to get only fully recorded x0, x1, y0, y1, z0, z1 = predicted['bbox'].parts() predicted = predicted.select((z0 > 0) & (z1 < 100)) assert(len(predicted) > 0) # Compute partiality partiality = calculator( predicted['s1'], predicted['xyzcal.px'].parts()[2], predicted['bbox']) # Should have all fully recorded assert(len(partiality) == len(predicted)) from math import sqrt, erf three_sigma = 0.5 * (erf(3.0 / sqrt(2.0)) - erf(-3.0 / sqrt(2.0))) assert(partiality.all_gt(three_sigma)) # Trim bounding boxes x0, x1, y0, y1, z0, z1 = predicted['bbox'].parts() z0 = z0 + 1 z1 = z1 - 1 predicted['bbox'] = flex.int6(x0, x1, y0, y1, z0, z1) predicted = predicted.select(z1 > z0) assert(len(predicted) > 0) # Compute partiality partiality = calculator( predicted['s1'], predicted['xyzcal.px'].parts()[2], predicted['bbox']) # Should have all partials assert(len(partiality) == len(predicted)) assert(partiality.all_le(1.0) and partiality.all_gt(0)) print 'OK'
def __init__(self): from dials.array_family import flex self.reflections = flex.reflection_table() self.reflections['panel'] = flex.size_t() self.reflections['bbox'] = flex.int6() self.reflections['miller_index'] = flex.miller_index() self.reflections['s1'] = flex.vec3_double() self.reflections['xyzcal.px'] = flex.vec3_double() self.reflections['xyzcal.mm'] = flex.vec3_double() self.reflections['entering'] = flex.bool() self.reflections['id'] = flex.int() self.reflections["flags"] = flex.size_t() self.npanels = 2 self.width = 1000 self.height = 1000 self.nrefl = 10000 self.array_range = (0, 130) self.block_size = 20 from random import randint, seed, choice seed(0) self.processed = [[] for i in range(12)] for i in range(self.nrefl): x0 = randint(0, self.width-10) y0 = randint(0, self.height-10) zs = randint(2, 9) x1 = x0 + randint(2, 10) y1 = y0 + randint(2, 10) for k, j in enumerate([10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]): m = k + i * 12 pos = choice(["left", "right", "centre"]) if pos == 'left': z0 = j - zs z1 = j elif pos == 'right': z0 = j z1 = j + zs else: z0 = j - zs // 2 z1 = j + zs // 2 bbox = (x0, x1, y0, y1, z0, z1) self.reflections.append({ "panel" : randint(0,1), "bbox" : bbox, "flags" : flex.reflection_table.flags.reference_spot }) self.processed[k].append(m)
def tst_split_partials(self): from dials.array_family import flex from random import randint, uniform r = flex.reflection_table() r['value1'] = flex.double() r['value2'] = flex.int() r['value3'] = flex.double() r['bbox'] = flex.int6() expected = [] for i in range(100): x0 = randint(0, 100) x1 = x0 + randint(1, 10) y0 = randint(0, 100) y1 = y0 + randint(1, 10) z0 = randint(0, 100) z1 = z0 + randint(1, 10) v1 = uniform(0, 100) v2 = randint(0, 100) v3 = uniform(0, 100) r.append({ 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z0, z1) }) for z in range(z0, z1): expected.append({ 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z, z+1), 'partial_id' : i, }) r.split_partials() assert(len(r) == len(expected)) EPS = 1e-7 for r1, r2 in zip(r, expected): assert(abs(r1['value1'] - r2['value1']) < EPS) assert(r1['value2'] == r2['value2']) assert(abs(r1['value3'] - r2['value3']) < EPS) assert(r1['bbox'] == r2['bbox']) assert(r1['partial_id'] == r2['partial_id']) print 'OK'
def tst_split_partials(self): from dials.array_family import flex from random import randint, uniform r = flex.reflection_table() r['value1'] = flex.double() r['value2'] = flex.int() r['value3'] = flex.double() r['bbox'] = flex.int6() expected = [] for i in range(100): x0 = randint(0, 100) x1 = x0 + randint(1, 10) y0 = randint(0, 100) y1 = y0 + randint(1, 10) z0 = randint(0, 100) z1 = z0 + randint(1, 10) v1 = uniform(0, 100) v2 = randint(0, 100) v3 = uniform(0, 100) r.append({ 'value1': v1, 'value2': v2, 'value3': v3, 'bbox': (x0, x1, y0, y1, z0, z1) }) for z in range(z0, z1): expected.append({ 'value1': v1, 'value2': v2, 'value3': v3, 'bbox': (x0, x1, y0, y1, z, z + 1), 'partial_id': i, }) r.split_partials() assert (len(r) == len(expected)) EPS = 1e-7 for r1, r2 in zip(r, expected): assert (abs(r1['value1'] - r2['value1']) < EPS) assert (r1['value2'] == r2['value2']) assert (abs(r1['value3'] - r2['value3']) < EPS) assert (r1['bbox'] == r2['bbox']) assert (r1['partial_id'] == r2['partial_id']) print 'OK'
def test_bounding_boxes(): from dials.array_family import flex from dials.model.data import Shoebox shoebox = flex.shoebox(10) bbox = flex.int6(10) for i in range(10): x0 = random.randint(0, 90) y0 = random.randint(0, 90) z0 = random.randint(0, 90) x1 = random.randint(1, 10) + x0 y1 = random.randint(1, 10) + y0 z1 = random.randint(1, 10) + z0 bbox[i] = (x0, x1, y0, y1, z0, z1) shoebox[i] = Shoebox(bbox[i]) bbox2 = shoebox.bounding_boxes() for i in range(10): assert bbox2[i] == bbox[i]
def refinerdata_testdata(testdata): experiment = testdata.experiment reflections = testdata.reflections panel = experiment.detector[0] s0_length = matrix.col(experiment.beam.get_s0()).length() reflections["bbox"] = flex.int6(len(reflections)) reflections["xyzobs.px.value"] = flex.vec3_double(len(reflections)) reflections["s2"] = reflections["s1"].each_normalize() * s0_length reflections["sp"] = flex.vec3_double(len(reflections)) for i, (x, y, z) in enumerate(reflections["xyzcal.px"]): x0 = int(x) - 5 x1 = int(x) + 5 + 1 y0 = int(y) - 5 y1 = int(y) + 5 + 1 z0 = int(z) z1 = z0 + 1 reflections["bbox"][i] = x0, x1, y0, y1, z0, z1 reflections["xyzobs.px.value"][i] = (int(x) + 0.5, int(y) + 0.5, int(z) + 0.5) reflections["sp"][i] = (matrix.col( panel.get_pixel_lab_coord( reflections["xyzobs.px.value"][i][0:2])).normalize() * s0_length) reflections["shoebox"] = flex.shoebox(reflections["panel"], reflections["bbox"], allocate=True) shoebox_data = flex.float(flex.grid(1, 11, 11)) shoebox_mask = flex.int(flex.grid(1, 11, 11)) for j in range(11): for i in range(11): shoebox_data[0, j, i] = (100 * exp(-0.5 * (j - 5)**2 / 1**2) * exp(-0.5 * (i - 5)**2 / 1**2)) shoebox_mask[0, j, i] = 5 for sbox in reflections["shoebox"]: sbox.data = shoebox_data sbox.mask = shoebox_mask return RefinerData.from_reflections(experiment, reflections)
def test_split_partials(): r = flex.reflection_table() r["value1"] = flex.double() r["value2"] = flex.int() r["value3"] = flex.double() r["bbox"] = flex.int6() expected = [] for i in range(100): x0 = random.randint(0, 100) x1 = x0 + random.randint(1, 10) y0 = random.randint(0, 100) y1 = y0 + random.randint(1, 10) z0 = random.randint(0, 100) z1 = z0 + random.randint(1, 10) v1 = random.uniform(0, 100) v2 = random.randint(0, 100) v3 = random.uniform(0, 100) r.append({ "value1": v1, "value2": v2, "value3": v3, "bbox": (x0, x1, y0, y1, z0, z1) }) for z in range(z0, z1): expected.append({ "value1": v1, "value2": v2, "value3": v3, "bbox": (x0, x1, y0, y1, z, z + 1), "partial_id": i, }) r.split_partials() assert len(r) == len(expected) EPS = 1e-7 for r1, r2 in zip(r.rows(), expected): assert abs(r1["value1"] - r2["value1"]) < EPS assert r1["value2"] == r2["value2"] assert abs(r1["value3"] - r2["value3"]) < EPS assert r1["bbox"] == r2["bbox"] assert r1["partial_id"] == r2["partial_id"]
def test_shoebox_memory_is_a_reasonable_guesstimate(dials_data): path = dials_data("centroid_test_data").join("experiments.json").strpath exlist = ExperimentListFactory.from_json_file(path)[0] exlist.profile = Model( None, n_sigma=3, sigma_b=0.024 * math.pi / 180.0, sigma_m=0.044 * math.pi / 180.0, ) rlist = flex.reflection_table.from_predictions(exlist) rlist["id"] = flex.int(len(rlist), 0) rlist["bbox"] = flex.int6(rlist.size(), (0, 1, 0, 1, 0, 1)) jobs = JobList() jobs.add((0, 1), (0, 9), 9) for flatten in (True, False): assumed_memory_usage = list(jobs.shoebox_memory(rlist, flatten)) assert len(assumed_memory_usage) == 1 assert assumed_memory_usage[0] == pytest.approx(23952, abs=3000)
def __init__(self): from dials.algorithms.integration.profile import GridSampler2D from dials.array_family import flex from random import randint, uniform # Number of reflections nrefl = 1000 # Size of the images width = 1000 height = 1000 # Create the grid self.grid = GridSampler2D((width, height), (5, 5)) # Create the list of xyz and bboxes self.xyz = flex.vec3_double(nrefl) self.bbox = flex.int6(nrefl) self.panel = flex.size_t(nrefl, 0) for i in range(nrefl): x0 = randint(0, width - 10) x1 = x0 + randint(3, 10) y0 = randint(0, height - 10) y1 = y0 + randint(3, 10) z0 = randint(0, 10) z1 = z0 + randint(1, 10) b = x0, x1, y0, y1, z0, z1 c = (x1 + x0) / 2, (y1 + y0) / 2, (z1 + z0) / 2 self.xyz[i] = c self.bbox[i] = b # Create the array of shoeboxes self.sbox = flex.shoebox(self.panel, self.bbox) self.sbox.allocate() for i in range(len(self.sbox)): data = self.sbox[i].data for j in range(len(data)): data[j] = uniform(0, 100)
def __init__(self): from dials.algorithms.integration.profile import GridSampler2D from dials.array_family import flex from random import randint, uniform # Number of reflections nrefl = 1000 # Size of the images width = 1000 height = 1000 # Create the grid self.grid = GridSampler2D((width, height), (5,5)) # Create the list of xyz and bboxes self.xyz = flex.vec3_double(nrefl) self.bbox = flex.int6(nrefl) self.panel = flex.size_t(nrefl, 0) for i in range(nrefl): x0 = randint(0,width-10) x1 = x0 + randint(3,10) y0 = randint(0,height-10) y1 = y0 + randint(3,10) z0 = randint(0,10) z1 = z0 + randint(1,10) b = x0, x1, y0, y1, z0, z1 c = (x1 + x0) / 2, (y1 + y0) / 2, (z1 + z0) / 2 self.xyz[i] = c self.bbox[i] = b # Create the array of shoeboxes self.sbox = flex.shoebox(self.panel, self.bbox) self.sbox.allocate() for i in range(len(self.sbox)): data = self.sbox[i].data for j in range(len(data)): data[j] = uniform(0, 100)
def tst_bounding_boxes(self): from dials.model.data import Shoebox from random import randint from dials.array_family import flex shoebox = flex.shoebox(10) bbox = flex.int6(10) for i in range(10): x0 = randint(0, 90) y0 = randint(0, 90) z0 = randint(0, 90) x1 = randint(1, 10) + x0 y1 = randint(1, 10) + y0 z1 = randint(1, 10) + z0 bbox[i] = (x0, x1, y0, y1, z0, z1) shoebox[i] = Shoebox(bbox[i]) bbox2 = shoebox.bounding_boxes() for i in range(10): assert(bbox2[i] == bbox[i]) # Test passed print 'OK'
def refl_bbox_maker(self): self.reflections['bbox'] = flex.int6(self.length)
def tst_split_partials_with_shoebox(self): from dials.array_family import flex from random import randint, uniform from dials.model.data import Shoebox r = flex.reflection_table() r['value1'] = flex.double() r['value2'] = flex.int() r['value3'] = flex.double() r['bbox'] = flex.int6() r['panel'] = flex.size_t() r['shoebox'] = flex.shoebox() expected = [] for i in range(100): x0 = randint(0, 100) x1 = x0 + randint(1, 10) y0 = randint(0, 100) y1 = y0 + randint(1, 10) z0 = randint(0, 100) z1 = z0 + randint(1, 10) v1 = uniform(0, 100) v2 = randint(0, 100) v3 = uniform(0, 100) sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1)) sbox.allocate() assert(sbox.is_consistent()) w = x1 - x0 h = y1 - y0 for z in range(z0, z1): for y in range(y0, y1): for x in range(x0, x1): sbox.data[z-z0,y-y0,x-x0] = x+y*w+z*w*h r.append({ 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z0, z1), 'panel' : 0, 'shoebox' : sbox }) for z in range(z0, z1): sbox = Shoebox(0, (x0, x1, y0, y1, z, z+1)) sbox.allocate() assert(sbox.is_consistent()) w = x1 - x0 h = y1 - y0 for y in range(y0, y1): for x in range(x0, x1): sbox.data[0,y-y0,x-x0] = x+y*w+z*w*h expected.append({ 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z, z+1), 'partial_id' : i, 'panel' : 0, 'shoebox' : sbox }) r.split_partials_with_shoebox() assert(len(r) == len(expected)) EPS = 1e-7 for r1, r2 in zip(r, expected): assert(abs(r1['value1'] - r2['value1']) < EPS) assert(r1['value2'] == r2['value2']) assert(abs(r1['value3'] - r2['value3']) < EPS) assert(r1['bbox'] == r2['bbox']) assert(r1['partial_id'] == r2['partial_id']) assert(r1['panel'] == r2['panel']) assert(r1['shoebox'].data.as_double().as_1d().all_approx_equal( r2['shoebox'].data.as_double().as_1d())) print 'OK'
def simple_gaussian_spots(params): from scitbx import matrix from dials.array_family import flex r = params.rotation axis = matrix.col((r.axis.x, r.axis.y, r.axis.z)) if axis.length() > 0: rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True) else: rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1)) # generate mask and peak values from dials.algorithms.shoebox import MaskCode mask_peak = MaskCode.Valid | MaskCode.Foreground mask_back = MaskCode.Valid | MaskCode.Background from dials.util.command_line import ProgressBar p = ProgressBar(title="Generating reflections") rlist = flex.reflection_table(params.nrefl) hkl = flex.miller_index(params.nrefl) s1 = flex.vec3_double(params.nrefl) xyzmm = flex.vec3_double(params.nrefl) xyzpx = flex.vec3_double(params.nrefl) panel = flex.size_t(params.nrefl) bbox = flex.int6(params.nrefl) for j in range(params.nrefl): p.update(j * 100.0 / params.nrefl) hkl[j] = (random.randint(0, 20), random.randint(0, 20), random.randint(0, 20)) phi = 2 * math.pi * random.random() s1[j] = (0, 0, 0) xyzpx[j] = (0, 0, 0) xyzmm[j] = (0, 0, phi) panel[j] = 0 bbox[j] = ( 0, params.shoebox_size.x, 0, params.shoebox_size.y, 0, params.shoebox_size.z, ) p.finished("Generating %d reflections" % params.nrefl) intensity = flex.double(params.nrefl) shoebox = flex.shoebox(panel, bbox) shoebox.allocate_with_value(MaskCode.Valid) p = ProgressBar(title="Generating shoeboxes") for i in range(len(rlist)): p.update(i * 100.0 / params.nrefl) mask = shoebox[i].mask if params.pixel_mask == "precise": # flag everything as background: peak will me assigned later for j in range(len(mask)): mask[j] = mask_back elif params.pixel_mask == "all": # flag we have no idea what anything is mask_none = MaskCode.Valid | MaskCode.Foreground | MaskCode.Background for j in range(len(mask)): mask[j] = mask_none elif params.pixel_mask == "static": from scitbx.array_family import flex x0 = params.spot_offset.x + params.shoebox_size.x / 2 y0 = params.spot_offset.x + params.shoebox_size.y / 2 z0 = params.spot_offset.x + params.shoebox_size.z / 2 sx = params.mask_nsigma * params.spot_size.x sy = params.mask_nsigma * params.spot_size.y sz = params.mask_nsigma * params.spot_size.z # The x, y, z indices z, y, x = zip(*itertools.product(*(range(n) for n in mask.all()))) xyz = flex.vec3_double(flex.double(x), flex.double(y), flex.double(z)) # Calculate SUM(((xj - xj0) / sxj)**2) for each element xyz0 = (x0, y0, z0) isxyz = (1.0 / sx, 1.0 / sy, 1.0 / sz) dxyz = sum( (x * isx) ** 2 for x, isx in zip(((xyz - xyz0) * rotation).parts(), isxyz) ) # Set the mask values index = dxyz <= 1.0 index.reshape(mask.accessor()) mask.set_selected(index, MaskCode.Valid | MaskCode.Foreground) mask.set_selected(not index, MaskCode.Valid | MaskCode.Background) sbox = shoebox[i].data # reflection itself, including setting the peak region if we're doing that # FIXME use flex arrays to make the rotation bit more efficient as this is # now rather slow... counts_true = 0 for j in range(params.counts): _x = random.gauss(0, params.spot_size.x) _y = random.gauss(0, params.spot_size.y) _z = random.gauss(0, params.spot_size.z) Rxyz = rotation * matrix.col((_x, _y, _z)).elems x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2) y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2) z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2) if x < 0 or x >= params.shoebox_size.x: continue if y < 0 or y >= params.shoebox_size.y: continue if z < 0 or z >= params.shoebox_size.z: continue sbox[z, y, x] += 1 counts_true += 1 if params.pixel_mask == "precise": mask[z, y, x] = mask_peak intensity[i] = counts_true if params.background: # background:flat; for j in range(params.background * len(sbox)): x = random.randint(0, params.shoebox_size.x - 1) y = random.randint(0, params.shoebox_size.y - 1) z = random.randint(0, params.shoebox_size.z - 1) sbox[z, y, x] += 1 else: # or inclined random_background_plane( sbox, params.background_a, params.background_b, params.background_c, params.background_d, ) rlist["miller_index"] = hkl rlist["s1"] = s1 rlist["xyzcal.px"] = xyzpx rlist["xyzcal.mm"] = xyzmm rlist["bbox"] = bbox rlist["panel"] = panel rlist["shoebox"] = shoebox rlist["intensity.sum.value"] = intensity p.finished("Generating %d shoeboxes" % params.nrefl) return rlist
def simple_gaussian_spots(params): from dials.array_family import flex from dials.algorithms import shoebox import random import math from scitbx import matrix r = params.rotation axis = matrix.col((r.axis.x, r.axis.y, r.axis.z)) if axis.length() > 0: rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True) else: rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1)) # generate mask and peak values from dials.algorithms.shoebox import MaskCode mask_peak = MaskCode.Valid|MaskCode.Foreground mask_back = MaskCode.Valid|MaskCode.Background from dials.util.command_line import ProgressBar p = ProgressBar(title = 'Generating reflections') rlist = flex.reflection_table(params.nrefl) hkl = flex.miller_index(params.nrefl) s1 = flex.vec3_double(params.nrefl) xyzmm = flex.vec3_double(params.nrefl) xyzpx = flex.vec3_double(params.nrefl) panel = flex.size_t(params.nrefl) bbox = flex.int6(params.nrefl) for j in range(params.nrefl): p.update(j * 100.0 / params.nrefl) hkl[j] = (random.randint(0, 20), random.randint(0, 20), random.randint(0, 20)) phi = 2 * math.pi * random.random() s1[j] = (0, 0, 0) xyzpx[j] = (0, 0, 0) xyzmm[j] = (0, 0, phi) panel[j] = 0 bbox[j] = (0, params.shoebox_size.x, 0, params.shoebox_size.y, 0, params.shoebox_size.z) p.finished('Generating %d reflections' % params.nrefl) intensity = flex.double(params.nrefl) shoebox = flex.shoebox(panel, bbox) shoebox.allocate_with_value(MaskCode.Valid) p = ProgressBar(title = 'Generating shoeboxes') for i in range(len(rlist)): p.update(i * 100.0 / params.nrefl) mask = shoebox[i].mask if params.pixel_mask == 'precise': # flag everything as background: peak will me assigned later for j in range(len(mask)): mask[j] = mask_back elif params.pixel_mask == 'all': # flag we have no idea what anything is mask_none = MaskCode.Valid|MaskCode.Foreground|MaskCode.Background for j in range(len(mask)): mask[j] = mask_none elif params.pixel_mask == 'static': import itertools from scitbx.array_family import flex x0 = params.spot_offset.x + params.shoebox_size.x / 2 y0 = params.spot_offset.x + params.shoebox_size.y / 2 z0 = params.spot_offset.x + params.shoebox_size.z / 2 sx = params.mask_nsigma * params.spot_size.x sy = params.mask_nsigma * params.spot_size.y sz = params.mask_nsigma * params.spot_size.z # The x, y, z indices z, y, x = zip(*itertools.product(*(range(n) for n in mask.all()))) xyz = flex.vec3_double(flex.double(x), flex.double(y), flex.double(z)) # Calculate SUM(((xj - xj0) / sxj)**2) for each element xyz0 = (x0, y0, z0) isxyz = (1.0/sx, 1.0/sy, 1.0/sz) dxyz = sum([(x * isx)**2 for x, isx in zip(((xyz - xyz0) * rotation).parts(), isxyz)]) # Set the mask values index = dxyz <= 1.0 index.reshape(mask.accessor()) mask.set_selected(index, MaskCode.Valid | MaskCode.Foreground) mask.set_selected(index != True, MaskCode.Valid | MaskCode.Background) sbox = shoebox[i].data # reflection itself, including setting the peak region if we're doing that # FIXME use flex arrays to make the rotation bit more efficient as this is # now rather slow... counts_true = 0 for j in range(params.counts): _x = random.gauss(0, params.spot_size.x) _y = random.gauss(0, params.spot_size.y) _z = random.gauss(0, params.spot_size.z) Rxyz = rotation * matrix.col((_x, _y, _z)).elems x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2) y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2) z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2) if x < 0 or x >= params.shoebox_size.x: continue if y < 0 or y >= params.shoebox_size.y: continue if z < 0 or z >= params.shoebox_size.z: continue sbox[z, y, x] += 1 counts_true += 1 if params.pixel_mask == 'precise': mask[z, y, x] = mask_peak intensity[i] = counts_true if params.background: # background:flat; for j in range(params.background * len(sbox)): x = random.randint(0, params.shoebox_size.x - 1) y = random.randint(0, params.shoebox_size.y - 1) z = random.randint(0, params.shoebox_size.z - 1) sbox[z, y, x] += 1 else: # or inclined random_background_plane(sbox, params.background_a, params.background_b, params.background_c, params.background_d) rlist['miller_index'] = hkl rlist['s1'] = s1 rlist['xyzcal.px'] = xyzpx rlist['xyzcal.mm'] = xyzmm rlist['bbox'] = bbox rlist['panel'] = panel rlist['shoebox'] = shoebox rlist['intensity.sum.value'] = intensity p.finished('Generating %d shoeboxes' % params.nrefl) return rlist
def test_to_from_msgpack(tmpdir): from dials.model.data import Shoebox def gen_shoebox(): shoebox = Shoebox(0, (0, 4, 0, 3, 0, 1)) shoebox.allocate() for k in range(1): for j in range(3): for i in range(4): shoebox.data[k, j, i] = i + j + k + 0.1 shoebox.mask[k, j, i] = i % 2 shoebox.background[k, j, i] = i * j + 0.2 return shoebox def compare(a, b): assert a.is_consistent() assert b.is_consistent() assert a.panel == b.panel assert a.bbox == b.bbox for aa, bb in zip(a.data, b.data): if abs(aa - bb) > 1e-9: return False for aa, bb in zip(a.background, b.background): if abs(aa - bb) > 1e-9: return False for aa, bb in zip(a.mask, b.mask): if aa != bb: return False return True # The columns as lists c1 = list(range(10)) c2 = list(range(10)) c3 = ["a", "b", "c", "d", "e", "f", "g", "i", "j", "k"] c4 = [True, False, True, False, True] * 2 c5 = list(range(10)) c6 = [(i + 1, i + 2) for i in range(10)] c7 = [(i + 1, i + 2, i + 3) for i in range(10)] c8 = [tuple(i + j for j in range(9)) for i in range(10)] c9 = [tuple(i + j for j in range(6)) for i in range(10)] c10 = [(i + 1, i + 2, i + 3) for i in range(10)] c11 = [gen_shoebox() for i in range(10)] # Create a table with some elements table = flex.reflection_table() table["col1"] = flex.int(c1) table["col2"] = flex.double(c2) table["col3"] = flex.std_string(c3) table["col4"] = flex.bool(c4) table["col5"] = flex.size_t(c5) table["col6"] = flex.vec2_double(c6) table["col7"] = flex.vec3_double(c7) table["col8"] = flex.mat3_double(c8) table["col9"] = flex.int6(c9) table["col10"] = flex.miller_index(c10) table["col11"] = flex.shoebox(c11) obj = table.as_msgpack() new_table = flex.reflection_table.from_msgpack(obj) assert new_table.is_consistent() assert new_table.nrows() == 10 assert new_table.ncols() == 11 assert all(tuple(a == b for a, b in zip(new_table["col1"], c1))) assert all(tuple(a == b for a, b in zip(new_table["col2"], c2))) assert all(tuple(a == b for a, b in zip(new_table["col3"], c3))) assert all(tuple(a == b for a, b in zip(new_table["col4"], c4))) assert all(tuple(a == b for a, b in zip(new_table["col5"], c5))) assert all(tuple(a == b for a, b in zip(new_table["col6"], c6))) assert all(tuple(a == b for a, b in zip(new_table["col7"], c7))) assert all(tuple(a == b for a, b in zip(new_table["col8"], c8))) assert all(tuple(a == b for a, b in zip(new_table["col9"], c9))) assert all(tuple(a == b for a, b in zip(new_table["col10"], c10))) assert all(tuple(compare(a, b) for a, b in zip(new_table["col11"], c11))) table.as_msgpack_file(tmpdir.join("reflections.mpack").strpath) new_table = flex.reflection_table.from_msgpack_file( tmpdir.join("reflections.mpack").strpath) assert new_table.is_consistent() assert new_table.nrows() == 10 assert new_table.ncols() == 11 assert all(tuple(a == b for a, b in zip(new_table["col1"], c1))) assert all(tuple(a == b for a, b in zip(new_table["col2"], c2))) assert all(tuple(a == b for a, b in zip(new_table["col3"], c3))) assert all(tuple(a == b for a, b in zip(new_table["col4"], c4))) assert all(tuple(a == b for a, b in zip(new_table["col5"], c5))) assert all(tuple(a == b for a, b in zip(new_table["col6"], c6))) assert all(tuple(a == b for a, b in zip(new_table["col7"], c7))) assert all(tuple(a == b for a, b in zip(new_table["col8"], c8))) assert all(tuple(a == b for a, b in zip(new_table["col9"], c9))) assert all(tuple(a == b for a, b in zip(new_table["col10"], c10))) assert all(tuple(compare(a, b) for a, b in zip(new_table["col11"], c11)))
return decoder.decode(self._handle) def set_reflections(self, reflections): '''Set the reflection data.''' self.set_data(reflections, ReflectionListEncoder()) def get_reflections(self): '''Get the reflection data.''' return self.get_data(ReflectionListDecoder()) if __name__ == '__main__': from dials.array_family import flex reflections = flex.reflection_table([('hkl', flex.miller_index(10)), ('s1', flex.vec3_double(10)), ('bbox', flex.int6(10)), ('id', flex.int(10)), ('shoebox', flex.shoebox(10))]) for i in range(10): reflections['shoebox'][i].data = flex.double(flex.grid(10, 10, 10)) reflections['shoebox'][i].mask = flex.int(flex.grid(10, 10, 10)) reflections['shoebox'][i].background = flex.double( flex.grid(10, 10, 10)) for i in range(10): print reflections['shoebox'][i].data.all() writer = NexusFile('temp.h5', 'w') writer.set_reflections(reflections) writer.close()
def _find_in_imageset(self, imageset): """ Do the spot finding. :param imageset: The imageset to process :return: The observed spots """ from dials.util.masking import MaskGenerator from dials.array_family import flex from dials.util.command_line import Command from dxtbx.imageset import ImageSweep from logging import info # The input mask mask = self.mask_generator.generate(imageset) if self.mask is not None: mask = tuple(m1 & m2 for m1, m2 in zip(mask, self.mask)) # Set the spot finding algorithm extract_spots = ExtractSpots( threshold_function=self.threshold_function, mask=mask, region_of_interest=self.region_of_interest, max_strong_pixel_fraction=self.max_strong_pixel_fraction, mp_method=self.mp_method, nproc=self.nproc, mp_chunksize=self.mp_chunksize, min_spot_size=self.min_spot_size, max_spot_size=self.max_spot_size, ) # Get the max scan range if isinstance(imageset, ImageSweep): max_scan_range = imageset.get_array_range() else: max_scan_range = (0, len(imageset)) # Get list of scan ranges if not self.scan_range or self.scan_range[0] is None: scan_range = [(max_scan_range[0] + 1, max_scan_range[1])] else: scan_range = self.scan_range # Get spots from bits of scan spots_all = [] for scan in scan_range: j0, j1 = scan assert j1 >= j0 and j0 > max_scan_range[0] and j1 <= max_scan_range[1] info("\nFinding spots in image {0} to {1}...".format(j0, j1)) j0 -= 1 if isinstance(imageset, ImageSweep): j0 -= imageset.get_array_range()[0] j1 -= imageset.get_array_range()[0] spots_all.extend(extract_spots(imageset[j0:j1])) # Get the list of shoeboxes shoeboxes = flex.shoebox(spots_all) # Calculate the spot centroids centroid = shoeboxes.centroid_valid() info("Calculated {0} spot centroids".format(len(shoeboxes))) # Calculate the spot intensities intensity = shoeboxes.summed_intensity() info("Calculated {0} spot intensities".format(len(shoeboxes))) # Create the observations observed = flex.observation(shoeboxes.panels(), centroid, intensity) # Write the hot mask if self.write_hot_mask: # Find spots which cover the whole scan range bbox = flex.int6([sbox.bbox for sbox in shoeboxes]) z0, z1 = bbox.parts()[4:6] zr = z1 - z0 assert zr.all_gt(0) possible_hot_spots = zr == len(imageset) num_possible_hot_spots = possible_hot_spots.count(True) info("Found %d possible hot spots" % num_possible_hot_spots) # Create the hot pixel mask hot_mask = tuple(flex.bool(flex.grid(p.get_image_size()[::-1]), True) for p in imageset.get_detector()) if num_possible_hot_spots > 0: hot_shoeboxes = shoeboxes.select(possible_hot_spots) for sbox in hot_shoeboxes: x0, x1, y0, y1 = sbox.bbox[0:4] m = sbox.mask p = sbox.panel for y in range(m.all()[1]): for x in range(m.all()[2]): if m[:, y : y + 1, x : x + 1].all_ne(0): hot_mask[p][y0 + y, x0 + x] = False info("Found %d possible hot pixel(s)" % hot_mask.count(False)) else: hot_mask = None # Filter the reflections and select only the desired spots flags = self.filter_spots(None, sweep=imageset, observations=observed, shoeboxes=shoeboxes) observed = observed.select(flags) shoeboxes = shoeboxes.select(flags) # Return as a reflection list return flex.reflection_table(observed, shoeboxes), hot_mask
def tst_split_blocks_overlapping(self): from dials.array_family import flex from random import randint, uniform, seed from dials.algorithms.integration.integrator import JobList from scitbx.array_family import shared blocks = shared.tiny_int_2([ (0, 10), (5, 15), (10, 20), (15, 25), (20, 30), (25, 35), (30, 40), (35, 45), (40, 50), (45, 55), (50, 60), (55, 65), (60, 70), (65, 75), (70, 80), (75, 85), (80, 90), (85, 95), (90, 100), (95, 105), (100, 110)]) jobs = JobList((0, 1), blocks) r = flex.reflection_table() r['value1'] = flex.double() r['value2'] = flex.int() r['value3'] = flex.double() r['bbox'] = flex.int6() r['id'] = flex.int() expected = [] for i in range(100): x0 = randint(0, 100) x1 = x0 + randint(1, 10) y0 = randint(0, 100) y1 = y0 + randint(1, 10) z0 = randint(0, 90) z1 = z0 + randint(1, 20) v1 = uniform(0, 100) v2 = randint(0, 100) v3 = uniform(0, 100) r.append({ 'id' : 0, 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z0, z1) }) expected.append({ 'id' : 0, 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z0, z1) }) jobs.split(r) assert(len(r) > 100) for r1 in r: v1 = r1['value1'] v2 = r1['value2'] v3 = r1['value3'] bbox = r1['bbox'] pid = r1['partial_id'] z0 = bbox[4] z1 = bbox[5] success = False for i in range(len(blocks)): b0 = blocks[i][0] b1 = blocks[i][1] if z0 >= b0 and z1 <= b1: success = True break assert(success) v11 = expected[pid]['value1'] v22 = expected[pid]['value2'] v33 = expected[pid]['value3'] bb = expected[pid]['bbox'] assert(v11 == v1) assert(v22 == v2) assert(v33 == v3) assert(bb[0] == bbox[0]) assert(bb[1] == bbox[1]) assert(bb[2] == bbox[2]) assert(bb[3] == bbox[3]) print 'OK'
def tst_split_blocks_non_overlapping(self): from dials.array_family import flex from random import randint, uniform, seed from dials.algorithms.integration.integrator import JobList from scitbx.array_family import shared blocks = shared.tiny_int_2([ (0, 10), (10, 20), (20, 30), (30, 35), (35, 40), (40, 50), (50, 60), (60, 70), (70, 80), (80, 90), (90, 100), (100, 110)]) jobs = JobList((0, 1), blocks) r = flex.reflection_table() r['value1'] = flex.double() r['value2'] = flex.int() r['value3'] = flex.double() r['bbox'] = flex.int6() r['id'] = flex.int() expected = [] for i in range(100): x0 = randint(0, 100) x1 = x0 + randint(1, 10) y0 = randint(0, 100) y1 = y0 + randint(1, 10) z0 = randint(0, 100) z1 = z0 + randint(1, 10) v1 = uniform(0, 100) v2 = randint(0, 100) v3 = uniform(0, 100) r.append({ 'id' : 0, 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z0, z1) }) for j in range(len(blocks)): b0 = blocks[j][0] b1 = blocks[j][1] if ((z0 >= b0 and z1 <= b1) or (z0 < b1 and z1 >= b1) or (z0 < b0 and z1 > b0)): z00 = max(b0, z0) z11 = min(b1, z1) expected.append({ 'id' : 0, 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z00, z11), 'partial_id' : i, }) jobs.split(r) assert(len(r) == len(expected)) EPS = 1e-7 for r1, r2 in zip(r, expected): assert(r1['bbox'] == r2['bbox']) assert(r1['partial_id'] == r2['partial_id']) assert(abs(r1['value1'] - r2['value1']) < EPS) assert(r1['value2'] == r2['value2']) assert(abs(r1['value3'] - r2['value3']) < EPS) print 'OK'
def read(handle, key): from dials.array_family import flex import numpy as np if key == 'miller_index': h = flex.int(handle['h'][:].astype(np.int32)) k = flex.int(handle['k'][:].astype(np.int32)) l = flex.int(handle['l'][:].astype(np.int32)) return flex.miller_index(h,k,l) elif key == 'id': return flex.int(handle['id'][:].astype(int)) elif key == 'partial_id': return flex.size_t(handle['reflection_id'][:].astype(int)) elif key == 'entering': return flex.bool(handle['entering'][:]) elif key == 'flags': return flex.size_t(handle['flags'][:].astype(int)) elif key == 'panel': return flex.size_t(handle['det_module'][:].astype(int)) elif key == 'd': return flex.double(handle['d'][:]) elif key == 'partiality': return flex.double(handle['partiality'][:]) elif key == 'xyzcal.px': x = flex.double(handle['prd_px_x'][:]) y = flex.double(handle['prd_px_y'][:]) z = flex.double(handle['prd_frame'][:]) return flex.vec3_double(x, y, z) elif key == 'xyzcal.mm': x = flex.double(handle['prd_mm_x'][:]) y = flex.double(handle['prd_mm_y'][:]) z = flex.double(handle['prd_phi'][:]) return flex.vec3_double(x, y, z) elif key == 'bbox': x0 = flex.int(handle['bbx0'][:].astype(np.int32)) x1 = flex.int(handle['bbx1'][:].astype(np.int32)) y0 = flex.int(handle['bby0'][:].astype(np.int32)) y1 = flex.int(handle['bby1'][:].astype(np.int32)) z0 = flex.int(handle['bbz0'][:].astype(np.int32)) z1 = flex.int(handle['bbz1'][:].astype(np.int32)) return flex.int6(x0, x1, y0, y1, z0, z1) elif key == 'xyzobs.px.value': x = flex.double(handle['obs_px_x_val'][:]) y = flex.double(handle['obs_px_y_val'][:]) z = flex.double(handle['obs_frame_val'][:]) return flex.vec3_double(x, y, z) elif key == 'xyzobs.px.variance': x = flex.double(handle['obs_px_x_var'][:]) y = flex.double(handle['obs_px_y_var'][:]) z = flex.double(handle['obs_frame_var'][:]) return flex.vec3_double(x, y, z) elif key == 'xyzobs.mm.value': x = flex.double(handle['obs_mm_x_val'][:]) y = flex.double(handle['obs_mm_y_val'][:]) z = flex.double(handle['obs_phi_val'][:]) return flex.vec3_double(x, y, z) elif key == 'xyzobs.mm.variance': x = flex.double(handle['obs_mm_x_var'][:]) y = flex.double(handle['obs_mm_y_var'][:]) z = flex.double(handle['obs_phi_var'][:]) return flex.vec3_double(x, y, z) elif key == 'background.mean': return flex.double(handle['bkg_mean'][:]) elif key == 'intensity.sum.value': return flex.double(handle['int_sum_val'][:]) elif key == 'intensity.sum.variance': return flex.double(handle['int_sum_var'][:]) elif key == 'intensity.prf.value': return flex.double(handle['int_prf_val'][:]) elif key == 'intensity.prf.variance': return flex.double(handle['int_prf_var'][:]) elif key == 'profile.correlation': return flex.double(handle['prf_cc'][:]) elif key == 'lp': return flex.double(handle['lp'][:]) elif key == 'num_pixels.background': return flex.int(handle['num_bg'][:].astype(np.int32)) elif key == 'num_pixels.background_used': return flex.int(handle['num_bg_used'][:].astype(np.int32)) elif key == 'num_pixels.foreground': return flex.int(handle['num_fg'][:].astype(np.int32)) elif key == 'num_pixels.valid': return flex.int(handle['num_valid'][:].astype(np.int32)) elif key == 'profile.rmsd': return flex.double(handle['prf_rmsd'][:]) else: raise KeyError('Column %s not read from file' % key)
def test_export_dials(): from dials.array_family import flex print 'Creating dummy reflection table...' table = flex.reflection_table() table['miller_index'] = flex.miller_index(100) table['id'] = flex.int(100) table['intensity.sum.value'] = flex.double(100) table['intensity.sum.variance'] = flex.double(100) table['intensity.prf.value'] = flex.double(100) table['intensity.prf.variance'] = flex.double(100) table['lp'] = flex.double(100) table['panel'] = flex.size_t(100) table['bbox'] = flex.int6(100) table['xyzcal.px'] = flex.vec3_double(100) table['xyzcal.mm'] = flex.vec3_double(100) table['xyzobs.px.value'] = flex.vec3_double(100) table['xyzobs.px.variance'] = flex.vec3_double(100) table['xyzobs.mm.value'] = flex.vec3_double(100) table['xyzobs.mm.variance'] = flex.vec3_double(100) table['partiality'] = flex.double(100) table['d'] = flex.double(100) table['s1'] = flex.vec3_double(100) table['rlp'] = flex.vec3_double(100) table['background.mean'] = flex.double(100) table['entering'] = flex.bool(100) table['flags'] = flex.size_t(100) table['profile.correlation'] = flex.double(100) # Open the file outfile = File('test_file.mtz2', 'w') # Get the entry entry = outfile.entry print 'Writing reflection table stuff...' # Save some processed data diffraction = entry.diffraction # Set the experiments experiment = diffraction.experiments[0] experiment['beam'] = 0 experiment['detector'] = 0 experiment['goniometer'] = 0 experiment['scan'] = 0 experiment['crystal'] =0 # Get columns into array col1, col2, col3 = zip(*list(table['miller_index'])) col4 = table['id'] col5 = table['intensity.sum.value'] col6 = table['intensity.sum.variance'] col7 = table['intensity.prf.value'] col8 = table['intensity.prf.variance'] col9 = table['lp'] col10 = table['panel'] col11, col12, col13, col14, col15, col16 = table['bbox'].parts() col17, col18, col19 = table['xyzcal.px'].parts() col20, col21, col22 = table['xyzcal.mm'].parts() col23, col24, col25 = table['xyzobs.px.value'].parts() col26, col27, col28 = table['xyzobs.px.variance'].parts() col29, col30, col31 = table['xyzobs.mm.value'].parts() col32, col33, col34 = table['xyzobs.mm.variance'].parts() col35 = table['partiality'] col36 = table['d'] col37 = table['background.mean'] col38 = table['entering'] col39 = table['flags'] col40 = table['profile.correlation'] # Some data diffraction['h'] = col1 diffraction['k'] = col2 diffraction['l'] = col3 diffraction['id'] = col4 diffraction['int_sum_val'] = col5 diffraction['int_sum_var'] = col6 diffraction['int_prf_val'] = col7 diffraction['int_prf_var'] = col8 diffraction['lp'] = col9 diffraction['det_module'] = col10 diffraction['bbx0'] = col11 diffraction['bbx1'] = col12 diffraction['bby0'] = col13 diffraction['bby1'] = col14 diffraction['bbz0'] = col15 diffraction['bbz1'] = col16 diffraction['prd_px_x'] = col17 diffraction['prd_px_y'] = col18 diffraction['prd_frame'] = col19 diffraction['prd_mm_x'] = col20 diffraction['prd_mm_y'] = col21 diffraction['prd_phi'] = col22 diffraction['obs_px_x_val'] = col23 diffraction['obs_px_x_var'] = col24 diffraction['obs_px_y_val'] = col25 diffraction['obs_px_y_var'] = col26 diffraction['obs_frame_val'] = col27 diffraction['obs_frame_var'] = col28 diffraction['obs_mm_x_val'] = col29 diffraction['obs_mm_x_var'] = col30 diffraction['obs_mm_y_val'] = col31 diffraction['obs_mm_y_var'] = col32 diffraction['obs_phi_val'] = col33 diffraction['obs_phi_var'] = col34 diffraction['partiality'] = col35 diffraction['d'] = col36 diffraction['bkg_mean'] = col37 diffraction['entering'] = col38 diffraction['flags'] = col39 diffraction['prf_cc'] = col40 # Flush the file outfile.flush()
def run(self): ''' Extract the shoeboxes. ''' from dials.util.options import flatten_reflections from dials.util.options import flatten_experiments from dials.util.options import flatten_datablocks from dials.util import log from dials.array_family import flex from libtbx.utils import Sorry from logging import info # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) # Configure logging log.config() # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) # Get the data reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) datablocks = flatten_datablocks(params.input.datablock) if len(experiments) == 0 and len(datablocks) == 0 and len(reflections) == 0: self.parser.print_help() exit(0) elif (len(experiments) != 0 and len(datablocks) != 0): raise Sorry('Both experiment list and datablocks set') elif len(experiments) > 1: raise Sorry('More than 1 experiment set') elif len(datablocks) > 1: raise Sorry('More than 1 datablock set') elif len(experiments) == 1: imageset = experiments[0].imageset elif len(datablocks) == 1: imagesets = datablocks[0].extract_imagesets() if len(imagesets) != 1: raise Sorry('Need 1 imageset, got %d' % len(imagesets)) imageset = imagesets[0] if len(reflections) != 1: raise Sorry('Need 1 reflection table, got %d' % len(reflections)) else: reflections = reflections[0] # Check the reflections contain the necessary stuff assert("bbox" in reflections) assert("panel" in reflections) # Get some models detector = imageset.get_detector() scan = imageset.get_scan() frame0, frame1 = scan.get_array_range() # Add some padding but limit to image volume if params.padding > 0: info('Adding %d pixels as padding' % params.padding) x0, x1, y0, y1, z0, z1 = reflections['bbox'].parts() x0 -= params.padding x1 += params.padding y0 -= params.padding y1 += params.padding z0 -= params.padding z1 += params.padding panel = reflections['panel'] for i in range(len(reflections)): width, height = detector[panel[i]].get_image_size() if z0[i] < frame0: z0[i] = frame0 if z1[i] > frame1: z1[i] = frame1 reflections['bbox'] = flex.int6(x0, x1, y0, y1, z0, z1) # Save the old shoeboxes if "shoebox" in reflections: old_shoebox = reflections['shoebox'] else: old_shoebox = None # Allocate the shoeboxes reflections["shoebox"] = flex.shoebox( reflections["panel"], reflections["bbox"], allocate=True) # Extract the shoeboxes reflections.extract_shoeboxes(imageset, verbose=True) # Preserve masking if old_shoebox is not None: info("Applying old shoebox mask") new_shoebox = reflections['shoebox'] for i in range(len(reflections)): bbox0 = old_shoebox[i].bbox bbox1 = new_shoebox[i].bbox mask0 = old_shoebox[i].mask mask1 = new_shoebox[i].mask mask2 = flex.int(mask1.accessor(), 0) x0 = bbox0[0] - bbox1[0] x1 = bbox0[1] - bbox0[0] + x0 y0 = bbox0[2] - bbox1[2] y1 = bbox0[3] - bbox0[2] + y0 z0 = bbox0[4] - bbox1[4] z1 = bbox0[5] - bbox0[4] + z0 mask2[z0:z1,y0:y1,x0:x1] = mask0 mask1 = mask1.as_1d() | mask2.as_1d() mask1.reshape(new_shoebox[i].mask.accessor()) new_shoebox[i].mask = mask1 # Saving the reflections to disk filename = params.output.reflections info('Saving %d reflections to %s' % (len(reflections), filename)) reflections.as_pickle(filename)
# rlist = ReflectionList() pi = 3.14159265358 n_x = 80 n_y = 86 # n_x = 80 # n_y = 86 num_ref = n_y * n_x ref_table = flex.reflection_table() t_shoebox = flex.shoebox(num_ref) t_intensity = flex.double(num_ref) t_intensity_var = flex.double(num_ref) t_bbox = flex.int6(num_ref) t_xyzobs = flex.vec3_double(num_ref) t_xyzcal = flex.vec3_double(num_ref) t_row = 0 for ypos in range(n_y): for xpos in range(n_x): # if xpos/32.0 == int(xpos/32) and ypos/32.0 == int(ypos/32): # if ypos/6.0 == int(ypos/6): # if xpos/3.0 == int(xpos/3) and ypos/3.0 == int(ypos/3): row_str = ypos * nrow col_str = xpos * ncol dx = col_str - 1200 dy = row_str - 1300
def read(handle, key): from dxtbx.format.nexus import convert_units if key == "miller_index": h = flex.int(handle["h"][:].astype(np.int32)) k = flex.int(handle["k"][:].astype(np.int32)) l = flex.int(handle["l"][:].astype(np.int32)) return flex.miller_index(h, k, l) elif key == "id": return flex.int(handle["id"][:].astype(int)) elif key == "partial_id": return flex.size_t(handle["reflection_id"][:].astype(int)) elif key == "entering": return flex.bool(handle["entering"][:]) elif key == "flags": return flex.size_t(handle["flags"][:].astype(int)) elif key == "panel": return flex.size_t(handle["det_module"][:].astype(int)) elif key == "d": return flex.double(handle["d"][:]) elif key == "partiality": return flex.double(handle["partiality"][:]) elif key == "xyzcal.px": x = flex.double(handle["predicted_px_x"][:]) y = flex.double(handle["predicted_px_y"][:]) z = flex.double(handle["predicted_frame"][:]) return flex.vec3_double(x, y, z) elif key == "xyzcal.mm": x = convert_units( flex.double(handle["predicted_x"][:]), handle["predicted_x"].attrs["units"], "mm", ) y = convert_units( flex.double(handle["predicted_y"][:]), handle["predicted_y"].attrs["units"], "mm", ) z = convert_units( flex.double(handle["predicted_phi"][:]), handle["predicted_phi"].attrs["units"], "rad", ) return flex.vec3_double(x, y, z) elif key == "bbox": b = flex.int(handle["bounding_box"][:].astype(np.int32)) return flex.int6(b.as_1d()) elif key == "xyzobs.px.value": x = flex.double(handle["observed_px_x"][:]) y = flex.double(handle["observed_px_y"][:]) z = flex.double(handle["observed_frame"][:]) return flex.vec3_double(x, y, z) elif key == "xyzobs.px.variance": x = flex.double(handle["observed_px_x_var"][:]) y = flex.double(handle["observed_px_y_var"][:]) z = flex.double(handle["observed_frame_var"][:]) return flex.vec3_double(x, y, z) elif key == "xyzobs.mm.value": x = convert_units( flex.double(handle["observed_x"][:]), handle["observed_x"].attrs["units"], "mm", ) y = convert_units( flex.double(handle["observed_y"][:]), handle["observed_y"].attrs["units"], "mm", ) z = convert_units( flex.double(handle["observed_phi"][:]), handle["observed_phi"].attrs["units"], "rad", ) return flex.vec3_double(x, y, z) elif key == "xyzobs.mm.variance": x = convert_units( flex.double(handle["observed_x_var"][:]), handle["observed_x_var"].attrs["units"], "mm", ) y = convert_units( flex.double(handle["observed_y_var"][:]), handle["observed_y_var"].attrs["units"], "mm", ) z = convert_units( flex.double(handle["observed_phi_var"][:]), handle["observed_phi_var"].attrs["units"], "rad", ) return flex.vec3_double(x, y, z) elif key == "background.mean": return flex.double(handle["background_mean"][:]) elif key == "intensity.sum.value": return flex.double(handle["int_sum"][:]) elif key == "intensity.sum.variance": return flex.double(handle["int_sum_var"][:]) elif key == "intensity.prf.value": return flex.double(handle["int_prf"][:]) elif key == "intensity.prf.variance": return flex.double(handle["int_prf_var"][:]) elif key == "profile.correlation": return flex.double(handle["prf_cc"][:]) elif key == "lp": return flex.double(handle["lp"][:]) elif key == "num_pixels.background": return flex.int(handle["num_bg"][:].astype(np.int32)) elif key == "num_pixels.background_used": return flex.int(handle["num_bg_used"][:].astype(np.int32)) elif key == "num_pixels.foreground": return flex.int(handle["num_fg"][:].astype(np.int32)) elif key == "num_pixels.valid": return flex.int(handle["num_valid"][:].astype(np.int32)) elif key == "profile.rmsd": return flex.double(handle["prf_rmsd"][:]) else: raise KeyError("Column %s not read from file" % key)
def integrate(experiment): from dials.algorithms.spot_prediction import PixelToMillerIndex from dials.array_family import flex from math import floor, sqrt from collections import defaultdict detector = experiment.detector assert len(detector) == 1 panel = detector[0] xsize, ysize = panel.get_image_size() transform = PixelToMillerIndex(experiment.beam, experiment.detector, experiment.crystal) data = experiment.imageset.get_raw_data(0)[0] mask = flex.bool(flex.grid(ysize, xsize), False) reflections = defaultdict(list) print("Doing pixel labelling") for j in range(ysize): for i in range(xsize): h = transform.h(0, i, j) h0 = tuple(map(lambda x: int(floor(x + 0.5)), h)) d = sqrt(sum(map(lambda x, y: (x - y)**2, h, h0))) # if not hasattr(reflections[h0], "xd"): # reflections[h0].xd = d # reflections[h0].xc = i # reflections[h0].yc = j # elif reflections[h0].xd > d: # reflections[h0].xd = d # reflections[h0].xc = i # reflections[h0].yc = j if d < 0.3: mask[j, i] = True reflections[h0].append((j, i)) # from matplotlib import pylab # #pylab.imshow(mask.as_numpy_array(), interpolation='none') # pylab.show() print("Integrating reflections") miller_index = flex.miller_index() intensity = flex.double() variance = flex.double() bbox = flex.int6() xyz = flex.vec3_double() for h, r in reflections.iteritems(): # xc = r.xc # yc = r.yc b_sum = 0 f_sum = 0 b_cnt = 0 f_cnt = 0 for i in range(len(r)): y, x = r[i] m = mask[y, x] if data[y, x] >= 0: if m: f_sum += data[y, x] f_cnt += 1 else: b_sum += data[y, x] b_cnt += 1 Y, X = zip(*r) x0, x1, y0, y1 = min(X), max(X), min(Y), max(Y) if f_cnt > 0 and b_cnt > 0: B = b_sum / b_cnt I = f_sum - B * f_cnt V = f_sum + B * (1 + f_cnt / b_cnt) miller_index.append(h) intensity.append(I) variance.append(V) bbox.append((x0, x1, y0, y1, 0, 1)) # xyz.append((xc, yc, 0)) print("Integrated %d reflections" % len(reflections)) print(flex.min(intensity), flex.max(intensity), flex.mean(intensity)) reflections = flex.reflection_table() reflections["miller_index"] = miller_index reflections["intensity.sum.value"] = intensity reflections["intensity.sum.variance"] = variance reflections["bbox"] = bbox reflections["panel"] = flex.size_t(len(reflections), 0) reflections["id"] = flex.size_t(len(reflections), 0) # reflections["xyzcal.px"] = xyz # reflections["xyzobs.px"] = xyz reflections.set_flags(flex.size_t(range(len(reflections))), reflections.flags.integrated_sum) return reflections
def set_reflections(self, reflections): '''Set the reflection data.''' self.set_data(reflections, ReflectionListEncoder()) def get_reflections(self): '''Get the reflection data.''' return self.get_data(ReflectionListDecoder()) if __name__ == '__main__': from dials.array_family import flex reflections = flex.reflection_table([ ('hkl', flex.miller_index(10)), ('s1', flex.vec3_double(10)), ('bbox', flex.int6(10)), ('id', flex.int(10)), ('shoebox', flex.shoebox(10)) ]) for i in range(10): reflections['shoebox'][i].data = flex.double(flex.grid(10,10,10)) reflections['shoebox'][i].mask = flex.int(flex.grid(10,10,10)) reflections['shoebox'][i].background = flex.double(flex.grid(10,10,10)) for i in range(10): print reflections['shoebox'][i].data.all() writer = NexusFile('temp.h5', 'w') writer.set_reflections(reflections) writer.close()
def tst_extract_shoeboxes(self): from dials.array_family import flex from random import randint, seed from dials.algorithms.shoebox import MaskCode import sys seed(0) reflections = flex.reflection_table() reflections['panel'] = flex.size_t() reflections['bbox'] = flex.int6() npanels = 2 width = 1000 height = 1000 nrefl = 10000 frame0 = 10 frame1 = 100 nrefl = 1000 for i in range(nrefl): xs = randint(5, 10) ys = randint(5, 10) x0 = randint(-xs+1, width-1) y0 = randint(-ys+1, height-1) z0 = randint(frame0, frame1-1) x1 = x0 + xs y1 = y0 + ys z1 = min([z0 + randint(1, 10), frame1]) assert(x1 > x0) assert(y1 > y0) assert(z1 > z0) assert(z0 >= frame0 and z1 <= frame1) bbox = (x0, x1, y0, y1, z0, z1) reflections.append({ "panel" : randint(0,1), "bbox" : bbox, }) reflections['shoebox'] = flex.shoebox( reflections['panel'], reflections['bbox']) reflections['shoebox'].allocate() class FakeImageSet(object): def __init__(self): from dials.array_family import flex self.data = flex.int(range(height*width)) self.data.reshape(flex.grid(height,width)) def get_array_range(self): return (frame0, frame1) def get_detector(self): class FakeDetector(object): def __len__(self): return npanels def __getitem__(self, index): class FakePanel(object): def get_trusted_range(self): return (-1, 1000000) return FakePanel() return FakeDetector() def __len__(self): return frame1 - frame0 def __getitem__(self, index): f = frame0+index return (self.data + f*1, self.data + f*2) def get_corrected_data(self, index): f = frame0+index return (self.data + f*1, self.data + f*2) def get_mask(self, index): image = self.get_corrected_data(index) return tuple(im >= 0 for im in image) imageset = FakeImageSet() stdout = sys.stdout class DevNull(object): def write(self, *args): pass def flush(self): pass sys.stdout = DevNull() reflections.extract_shoeboxes(imageset) sys.stdout = stdout for i in range(len(reflections)): sbox = reflections[i]["shoebox"] assert(sbox.is_consistent()) mask = sbox.mask data = sbox.data bbox = sbox.bbox panel = sbox.panel x0, x1, y0, y1, z0, z1 = bbox for z in range(z1 - z0): for y in range(y1 - y0): for x in range(x1 - x0): v1 = data[z,y,x] m1 = mask[z,y,x] if (x0 + x >= 0 and y0 + y >= 0 and x0 + x < width and y0 + y < height): v2 = imageset.data[y+y0,x+x0] + (z+z0)*(panel+1) m2 = MaskCode.Valid assert(v1 == v2) assert(m1 == m2) else: assert(v1 == 0) assert(m1 == 0) print 'OK'
def run(self): ''' Extract the shoeboxes. ''' from dials.util.options import flatten_reflections from dials.util.options import flatten_experiments from dials.util.options import flatten_datablocks from dials.util import log from dials.array_family import flex from libtbx.utils import Sorry # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) # Configure logging log.config() # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Get the data reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) datablocks = flatten_datablocks(params.input.datablock) if not any([experiments, datablocks, reflections]): self.parser.print_help() exit(0) elif experiments and datablocks: raise Sorry('Both experiment list and datablocks set') elif len(experiments) > 1: raise Sorry('More than 1 experiment set') elif len(datablocks) > 1: raise Sorry('More than 1 datablock set') elif len(experiments) == 1: imageset = experiments[0].imageset elif len(datablocks) == 1: imagesets = datablocks[0].extract_imagesets() if len(imagesets) != 1: raise Sorry('Need 1 imageset, got %d' % len(imagesets)) imageset = imagesets[0] if len(reflections) != 1: raise Sorry('Need 1 reflection table, got %d' % len(reflections)) else: reflections = reflections[0] # Check the reflections contain the necessary stuff assert("bbox" in reflections) assert("panel" in reflections) # Get some models detector = imageset.get_detector() scan = imageset.get_scan() frame0, frame1 = scan.get_array_range() # Add some padding but limit to image volume if params.padding > 0: logger.info('Adding %d pixels as padding' % params.padding) x0, x1, y0, y1, z0, z1 = reflections['bbox'].parts() x0 -= params.padding x1 += params.padding y0 -= params.padding y1 += params.padding # z0 -= params.padding # z1 += params.padding panel = reflections['panel'] for i in range(len(reflections)): width, height = detector[panel[i]].get_image_size() if x0[i] < 0: x0[i] = 0 if x1[i] > width: x1[i] = width if y0[i] < 0: y0[i] = 0 if y1[i] > height: y1[i] = height if z0[i] < frame0: z0[i] = frame0 if z1[i] > frame1: z1[i] = frame1 reflections['bbox'] = flex.int6(x0, x1, y0, y1, z0, z1) # Save the old shoeboxes if "shoebox" in reflections: old_shoebox = reflections['shoebox'] else: old_shoebox = None # Allocate the shoeboxes reflections["shoebox"] = flex.shoebox( reflections["panel"], reflections["bbox"], allocate=True) # Extract the shoeboxes reflections.extract_shoeboxes(imageset, verbose=True) # Preserve masking if old_shoebox is not None: from dials.algorithms.shoebox import MaskCode logger.info("Applying old shoebox mask") new_shoebox = reflections['shoebox'] for i in range(len(reflections)): bbox0 = old_shoebox[i].bbox bbox1 = new_shoebox[i].bbox mask0 = old_shoebox[i].mask mask1 = new_shoebox[i].mask mask2 = flex.int(mask1.accessor(), 0) x0 = bbox0[0] - bbox1[0] x1 = bbox0[1] - bbox0[0] + x0 y0 = bbox0[2] - bbox1[2] y1 = bbox0[3] - bbox0[2] + y0 z0 = bbox0[4] - bbox1[4] z1 = bbox0[5] - bbox0[4] + z0 mask2[z0:z1,y0:y1,x0:x1] = mask0 mask1 = mask1.as_1d() | mask2.as_1d() if params.padding_is_background: selection = flex.size_t(range(len(mask1))).select(mask1 == MaskCode.Valid) values = flex.int(len(selection), MaskCode.Valid | MaskCode.Background) mask1.set_selected(selection, values) mask1.reshape(new_shoebox[i].mask.accessor()) new_shoebox[i].mask = mask1 # Saving the reflections to disk filename = params.output.reflections logger.info('Saving %d reflections to %s' % (len(reflections), filename)) reflections.as_pickle(filename)
def test_extract_shoeboxes(): from dials.algorithms.shoebox import MaskCode random.seed(0) reflections = flex.reflection_table() reflections["panel"] = flex.size_t() reflections["bbox"] = flex.int6() npanels = 2 width = 1000 height = 1000 frame0 = 10 frame1 = 100 nrefl = 1000 for i in range(nrefl): xs = random.randint(5, 10) ys = random.randint(5, 10) x0 = random.randint(-xs + 1, width - 1) y0 = random.randint(-ys + 1, height - 1) z0 = random.randint(frame0, frame1 - 1) x1 = x0 + xs y1 = y0 + ys z1 = min([z0 + random.randint(1, 10), frame1]) assert x1 > x0 assert y1 > y0 assert z1 > z0 assert z0 >= frame0 and z1 <= frame1 bbox = (x0, x1, y0, y1, z0, z1) reflections.append({"panel": random.randint(0, 1), "bbox": bbox}) reflections["shoebox"] = flex.shoebox(reflections["panel"], reflections["bbox"]) reflections["shoebox"].allocate() class FakeImageSet(object): def __init__(self): self.data = flex.int(range(height * width)) self.data.reshape(flex.grid(height, width)) def get_array_range(self): return (frame0, frame1) def get_detector(self): class FakeDetector(object): def __len__(self): return npanels def __getitem__(self, index): class FakePanel(object): def get_trusted_range(self): return (-1, 1000000) return FakePanel() return FakeDetector() def __len__(self): return frame1 - frame0 def __getitem__(self, index): f = frame0 + index return (self.data + f * 1, self.data + f * 2) def get_corrected_data(self, index): f = frame0 + index return (self.data + f * 1, self.data + f * 2) def get_mask(self, index): image = self.get_corrected_data(index) return tuple(im >= 0 for im in image) imageset = FakeImageSet() reflections.extract_shoeboxes(imageset) for i in range(len(reflections)): sbox = reflections[i]["shoebox"] assert sbox.is_consistent() mask = sbox.mask data = sbox.data bbox = sbox.bbox panel = sbox.panel x0, x1, y0, y1, z0, z1 = bbox for z in range(z1 - z0): for y in range(y1 - y0): for x in range(x1 - x0): v1 = data[z, y, x] m1 = mask[z, y, x] if (x0 + x >= 0 and y0 + y >= 0 and x0 + x < width and y0 + y < height): v2 = imageset.data[y + y0, x + x0] + (z + z0) * (panel + 1) m2 = MaskCode.Valid assert v1 == v2 assert m1 == m2 else: assert v1 == 0 assert m1 == 0
def test_export_dials(): from dials.array_family import flex print("Creating dummy reflection table...") table = flex.reflection_table() table["miller_index"] = flex.miller_index(100) table["id"] = flex.int(100) table["intensity.sum.value"] = flex.double(100) table["intensity.sum.variance"] = flex.double(100) table["intensity.prf.value"] = flex.double(100) table["intensity.prf.variance"] = flex.double(100) table["lp"] = flex.double(100) table["panel"] = flex.size_t(100) table["bbox"] = flex.int6(100) table["xyzcal.px"] = flex.vec3_double(100) table["xyzcal.mm"] = flex.vec3_double(100) table["xyzobs.px.value"] = flex.vec3_double(100) table["xyzobs.px.variance"] = flex.vec3_double(100) table["xyzobs.mm.value"] = flex.vec3_double(100) table["xyzobs.mm.variance"] = flex.vec3_double(100) table["partiality"] = flex.double(100) table["d"] = flex.double(100) table["s1"] = flex.vec3_double(100) table["rlp"] = flex.vec3_double(100) table["background.mean"] = flex.double(100) table["entering"] = flex.bool(100) table["flags"] = flex.size_t(100) table["profile.correlation"] = flex.double(100) # Open the file outfile = File("test_file.mtz2", "w") # Get the entry entry = outfile.entry print("Writing reflection table stuff...") # Save some processed data diffraction = entry.diffraction # Set the experiments experiment = diffraction.experiments[0] experiment["beam"] = 0 experiment["detector"] = 0 experiment["goniometer"] = 0 experiment["scan"] = 0 experiment["crystal"] = 0 # Get columns into array col1, col2, col3 = zip(*list(table["miller_index"])) col4 = table["id"] col5 = table["intensity.sum.value"] col6 = table["intensity.sum.variance"] col7 = table["intensity.prf.value"] col8 = table["intensity.prf.variance"] col9 = table["lp"] col10 = table["panel"] col11, col12, col13, col14, col15, col16 = table["bbox"].parts() col17, col18, col19 = table["xyzcal.px"].parts() col20, col21, col22 = table["xyzcal.mm"].parts() col23, col24, col25 = table["xyzobs.px.value"].parts() col26, col27, col28 = table["xyzobs.px.variance"].parts() col29, col30, col31 = table["xyzobs.mm.value"].parts() col32, col33, col34 = table["xyzobs.mm.variance"].parts() col35 = table["partiality"] col36 = table["d"] col37 = table["background.mean"] col38 = table["entering"] col39 = table["flags"] col40 = table["profile.correlation"] # Some data diffraction["h"] = col1 diffraction["k"] = col2 diffraction["l"] = col3 diffraction["id"] = col4 diffraction["int_sum_val"] = col5 diffraction["int_sum_var"] = col6 diffraction["int_prf_val"] = col7 diffraction["int_prf_var"] = col8 diffraction["lp"] = col9 diffraction["det_module"] = col10 diffraction["bbx0"] = col11 diffraction["bbx1"] = col12 diffraction["bby0"] = col13 diffraction["bby1"] = col14 diffraction["bbz0"] = col15 diffraction["bbz1"] = col16 diffraction["prd_px_x"] = col17 diffraction["prd_px_y"] = col18 diffraction["prd_frame"] = col19 diffraction["prd_mm_x"] = col20 diffraction["prd_mm_y"] = col21 diffraction["prd_phi"] = col22 diffraction["obs_px_x_val"] = col23 diffraction["obs_px_x_var"] = col24 diffraction["obs_px_y_val"] = col25 diffraction["obs_px_y_var"] = col26 diffraction["obs_frame_val"] = col27 diffraction["obs_frame_var"] = col28 diffraction["obs_mm_x_val"] = col29 diffraction["obs_mm_x_var"] = col30 diffraction["obs_mm_y_val"] = col31 diffraction["obs_mm_y_var"] = col32 diffraction["obs_phi_val"] = col33 diffraction["obs_phi_var"] = col34 diffraction["partiality"] = col35 diffraction["d"] = col36 diffraction["bkg_mean"] = col37 diffraction["entering"] = col38 diffraction["flags"] = col39 diffraction["prf_cc"] = col40 # Flush the file outfile.flush()
def test_split_partials_with_shoebox(): from dials.model.data import Shoebox r = flex.reflection_table() r["value1"] = flex.double() r["value2"] = flex.int() r["value3"] = flex.double() r["bbox"] = flex.int6() r["panel"] = flex.size_t() r["shoebox"] = flex.shoebox() expected = [] for i in range(100): x0 = random.randint(0, 100) x1 = x0 + random.randint(1, 10) y0 = random.randint(0, 100) y1 = y0 + random.randint(1, 10) z0 = random.randint(0, 100) z1 = z0 + random.randint(1, 10) v1 = random.uniform(0, 100) v2 = random.randint(0, 100) v3 = random.uniform(0, 100) sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1)) sbox.allocate() assert sbox.is_consistent() w = x1 - x0 h = y1 - y0 for z in range(z0, z1): for y in range(y0, y1): for x in range(x0, x1): sbox.data[z - z0, y - y0, x - x0] = x + y * w + z * w * h r.append({ "value1": v1, "value2": v2, "value3": v3, "bbox": (x0, x1, y0, y1, z0, z1), "panel": 0, "shoebox": sbox, }) for z in range(z0, z1): sbox = Shoebox(0, (x0, x1, y0, y1, z, z + 1)) sbox.allocate() assert sbox.is_consistent() w = x1 - x0 h = y1 - y0 for y in range(y0, y1): for x in range(x0, x1): sbox.data[0, y - y0, x - x0] = x + y * w + z * w * h expected.append({ "value1": v1, "value2": v2, "value3": v3, "bbox": (x0, x1, y0, y1, z, z + 1), "partial_id": i, "panel": 0, "shoebox": sbox, }) r.split_partials_with_shoebox() assert len(r) == len(expected) EPS = 1e-7 for r1, r2 in zip(r.rows(), expected): assert abs(r1["value1"] - r2["value1"]) < EPS assert r1["value2"] == r2["value2"] assert abs(r1["value3"] - r2["value3"]) < EPS assert r1["bbox"] == r2["bbox"] assert r1["partial_id"] == r2["partial_id"] assert r1["panel"] == r2["panel"] assert (r1["shoebox"].data.as_double().as_1d().all_approx_equal( r2["shoebox"].data.as_double().as_1d()))
def test_split_blocks_overlapping(): from scitbx.array_family import shared from dials.algorithms.integration.integrator import JobList from dials.array_family import flex blocks = shared.tiny_int_2([ (0, 10), (5, 15), (10, 20), (15, 25), (20, 30), (25, 35), (30, 40), (35, 45), (40, 50), (45, 55), (50, 60), (55, 65), (60, 70), (65, 75), (70, 80), (75, 85), (80, 90), (85, 95), (90, 100), (95, 105), (100, 110), ]) jobs = JobList((0, 1), blocks) r = flex.reflection_table() r["value1"] = flex.double() r["value2"] = flex.int() r["value3"] = flex.double() r["bbox"] = flex.int6() r["id"] = flex.int() expected = [] for i in range(100): x0 = random.randint(0, 100) x1 = x0 + random.randint(1, 10) y0 = random.randint(0, 100) y1 = y0 + random.randint(1, 10) z0 = random.randint(0, 90) z1 = z0 + random.randint(1, 20) v1 = random.uniform(0, 100) v2 = random.randint(0, 100) v3 = random.uniform(0, 100) r.append({ "id": 0, "value1": v1, "value2": v2, "value3": v3, "bbox": (x0, x1, y0, y1, z0, z1), }) expected.append({ "id": 0, "value1": v1, "value2": v2, "value3": v3, "bbox": (x0, x1, y0, y1, z0, z1), }) jobs.split(r) assert len(r) > 100 for r1 in r.rows(): v1 = r1["value1"] v2 = r1["value2"] v3 = r1["value3"] bbox = r1["bbox"] pid = r1["partial_id"] z0 = bbox[4] z1 = bbox[5] success = False for i in range(len(blocks)): b0 = blocks[i][0] b1 = blocks[i][1] if z0 >= b0 and z1 <= b1: success = True break assert success v11 = expected[pid]["value1"] v22 = expected[pid]["value2"] v33 = expected[pid]["value3"] bb = expected[pid]["bbox"] assert v11 == v1 assert v22 == v2 assert v33 == v3 assert bb[0] == bbox[0] assert bb[1] == bbox[1] assert bb[2] == bbox[2] assert bb[3] == bbox[3]
def test_reflection_manager(): from dials.array_family import flex reflections = flex.reflection_table() reflections["panel"] = flex.size_t() reflections["bbox"] = flex.int6() reflections["miller_index"] = flex.miller_index() reflections["s1"] = flex.vec3_double() reflections["xyzcal.px"] = flex.vec3_double() reflections["xyzcal.mm"] = flex.vec3_double() reflections["entering"] = flex.bool() reflections["id"] = flex.int() reflections["flags"] = flex.size_t() width = 1000 height = 1000 nrefl = 10000 array_range = (0, 130) block_size = 20 block_overlap = 10 random.seed(0) processed = [[] for i in range(12)] for i in range(nrefl): x0 = random.randint(0, width - 10) y0 = random.randint(0, height - 10) zs = random.randint(2, 9) x1 = x0 + random.randint(2, 10) y1 = y0 + random.randint(2, 10) for k, j in enumerate( [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]): m = k + i * 12 pos = random.choice(["left", "right", "centre"]) if pos == "left": z0 = j - zs z1 = j elif pos == "right": z0 = j z1 = j + zs else: z0 = j - zs // 2 z1 = j + zs // 2 bbox = (x0, x1, y0, y1, z0, z1) reflections.append({ "panel": random.randint(0, 1), "bbox": bbox, "flags": flex.reflection_table.flags.reference_spot, }) processed[k].append(m) # Add reflection to ignore # zc = random.choice([10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]) # z0 = zc - 11 # z1 = zc + 11 # bbox = (x0, x1, y0, y1, z0, z1) # reflections.append({ # "panel" : randint(0,1), # "bbox" : bbox, # "flags" : flex.reflection_table.flags.reference_spot # }) from dials.algorithms.integration.integrator import JobList, ReflectionManager jobs = JobList() jobs.add((0, 1), array_range, block_size, block_overlap) # Create the executor executor = ReflectionManager(jobs, reflections) # Ensure the tasks make sense jobs = [executor.job(i) for i in range(len(executor))] assert len(executor) == 12 assert not executor.finished() assert len(jobs) == 12 assert jobs[0].frames() == (0, 20) assert jobs[1].frames() == (10, 30) assert jobs[2].frames() == (20, 40) assert jobs[3].frames() == (30, 50) assert jobs[4].frames() == (40, 60) assert jobs[5].frames() == (50, 70) assert jobs[6].frames() == (60, 80) assert jobs[7].frames() == (70, 90) assert jobs[8].frames() == (80, 100) assert jobs[9].frames() == (90, 110) assert jobs[10].frames() == (100, 120) assert jobs[11].frames() == (110, 130) # Get the task specs data0 = executor.split(0) data1 = executor.split(1) data2 = executor.split(2) data3 = executor.split(3) data4 = executor.split(4) data5 = executor.split(5) data6 = executor.split(6) data7 = executor.split(7) data8 = executor.split(8) data9 = executor.split(9) data10 = executor.split(10) data11 = executor.split(11) assert len(data0) == len(processed[0]) assert len(data1) == len(processed[1]) assert len(data2) == len(processed[2]) assert len(data3) == len(processed[3]) assert len(data4) == len(processed[4]) assert len(data5) == len(processed[5]) assert len(data6) == len(processed[6]) assert len(data7) == len(processed[7]) assert len(data8) == len(processed[8]) assert len(data9) == len(processed[9]) assert len(data10) == len(processed[10]) assert len(data11) == len(processed[11]) # Add some results data0["data"] = flex.double(len(data0), 1) data1["data"] = flex.double(len(data1), 2) data2["data"] = flex.double(len(data2), 3) data3["data"] = flex.double(len(data3), 4) data4["data"] = flex.double(len(data4), 5) data5["data"] = flex.double(len(data5), 6) data6["data"] = flex.double(len(data6), 7) data7["data"] = flex.double(len(data7), 8) data8["data"] = flex.double(len(data8), 9) data9["data"] = flex.double(len(data9), 10) data10["data"] = flex.double(len(data10), 11) data11["data"] = flex.double(len(data11), 12) # Accumulate the data again assert not executor.finished() executor.accumulate(0, data0) executor.accumulate(1, data1) executor.accumulate(2, data2) executor.accumulate(3, data3) executor.accumulate(4, data4) executor.accumulate(5, data5) executor.accumulate(6, data6) executor.accumulate(7, data7) executor.accumulate(8, data8) executor.accumulate(9, data9) executor.accumulate(10, data10) executor.accumulate(11, data11) assert executor.finished() # Get results and check they're as expected data = executor.data() result = data["data"] for i in range(len(processed)): for j in range(len(processed[i])): assert result[processed[i][j]] == i + 1