def with_random_intensity(self, N, Imax, Bamax, Bbmax, Bcmax, Bdmax): """ Generate reflections with a random intensity and background. """ from dials.array_family import flex if Imax == 0: I = flex.size_t(N).as_int() else: I = flex.random_size_t(N, Imax).as_int() if Bamax == 0: Ba = flex.size_t(N).as_int() else: Ba = flex.random_size_t(N, Bamax).as_int() if Bbmax == 0: Bb = flex.size_t(N).as_int() else: Bb = flex.random_size_t(N, Bbmax).as_int() if Bcmax == 0: Bc = flex.size_t(N).as_int() else: Bc = flex.random_size_t(N, Bcmax).as_int() if Bdmax == 0: Bd = flex.size_t(N).as_int() else: Bd = flex.random_size_t(N, Bdmax).as_int() return self.with_individual_given_intensity(N, I, Ba, Bb, Bc, Bd)
def setup_test_sorting(): # Borrowed from tst_reflection_table function tst_find_overlapping N = 110 r = flex.reflection_table.empty_standard(N) r["panel"] = flex.size_t([1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0] * 10) r["id"] = flex.int([1, 2, 1, 1, 2, 0, 1, 1, 1, 0, 1] * 10) exp_ids = flex.size_t([0, 1]) for i in range(N): r["miller_index"][i] = ( int(i // 10) - 5, i % 3, i % 7, ) # A nice bunch of miller indices # Filter out reflections to be used by refinement. Sorting of filtered reflections # require to allow C++ extension modules to give performance benefit. Sorting # performed within the _filter_reflections step by id, then by panel. r_sorted = copy.deepcopy(r) r_sorted.sort("id") r_sorted.subsort("id", "panel") # Test that the unfiltered/unsorted table becomes filtered/sorted for id assert (r_sorted["id"] == r["id"].select(flex.sort_permutation( r["id"]))).count(False) == 0 # as above for panel within each id for ii in [0, 1, 2]: r_id = r.select(r["id"] == ii) r_sorted_id = r_sorted.select(r_sorted["id"] == ii) assert (r_sorted_id["panel"] == r_id["panel"].select( flex.sort_permutation(r_id["panel"]))).count(False) == 0 return (r, r_sorted, exp_ids)
def __call__(self, observed, predicted): '''i Match the observed reflections with the predicted. :param observed: The list of observed reflections. :param predicted: The list of predicted reflections. :returns: The list of matched reflections ''' from dials.array_family import flex # Find the nearest neighbours and distances # Command.start('Finding nearest neighbours') nn, dist = self._find_nearest_neighbours(observed, predicted) # Command.end('Found nearest neighbours') # Filter the matches by distance # Command.start('Filtering matches by distance') index = self._filter_by_distance(nn, dist) # Command.end('Filtered {0} matches by distance'.format(len(index))) # Filter out duplicates to just leave the closest pairs # Command.start('Removing duplicate matches') len_index = len(index) index = self._filter_duplicates(index, nn, dist) len_diff = len_index - len(index) # Command.end('Removed {0} duplicate match(es)'.format(len_diff)) # Copy all of the reflection data for the matched reflections return flex.size_t(index), flex.size_t([nn[i] for i in index])
def test_auto_reduction_parameter_extension_modules_part2(setup_test_sorting): # Cut-down original algorithm for AutoReduce._unit_cell_surplus_reflections from dials_refinement_helpers_ext import uc_surpl_iter as uc_surpl r, r_sorted, exp_ids = setup_test_sorting isel = flex.size_t() for exp_id in exp_ids: isel.extend((r["id"] == exp_id).iselection()) ref = r.select(isel) h = ref["miller_index"].as_vec3_double() dB_dp = flex.mat3_double([(1, 2, 3, 4, 5, 6, 7, 8, 9), (0, 1, 0, 1, 0, 1, 0, 1, 0)]) nref_each_param = [] for der in dB_dp: tst = (der * h).norms() nref_each_param.append((tst > 0.0).count(True)) res0 = min(nref_each_param) # Updated algorithm for _unit_cell_surplus_reflections res1_unsrt_int = uc_surpl(r["id"], r["miller_index"], exp_ids, dB_dp).result res1_int = uc_surpl(r_sorted["id"], r_sorted["miller_index"], exp_ids, dB_dp).result res1_sizet = uc_surpl(flex.size_t(list(r_sorted["id"])), r_sorted["miller_index"], exp_ids, dB_dp).result assert res0 != res1_unsrt_int assert res0 == res1_int assert res0 == res1_sizet
def import_integrated(integrate_hkl, min_ios=3): reader = integrate_hkl_as_flex.reader(integrate_hkl, "IOBS,SIGMA,XCAL,YCAL,ZCAL,XOBS,YOBS,ZOBS,ISEG".split(",")) # reference: dials/command_line/import_xds.py table = flex.reflection_table() table["id"] = flex.int(len(reader.hkl), 0) table["panel"] = flex.size_t(len(reader.hkl), 0) # only assuming single panel table["miller_index"] = reader.hkl table["xyzcal.px.value"] = flex.vec3_double(reader.data["XCAL"], reader.data["YCAL"], reader.data["ZCAL"]) table["xyzobs.px.value"] = flex.vec3_double(reader.data["XOBS"], reader.data["YOBS"], reader.data["ZOBS"]) table["intensity.cor.value"] = reader.data["IOBS"] table["intensity.cor.sigma"] = reader.data["SIGMA"] # not valid name, just for making selection table["flags"] = flex.size_t(len(table), table.flags.indexed | table.flags.strong) table = table.select(table["intensity.cor.sigma"] > 0) table = table.select(table["intensity.cor.value"]/table["intensity.cor.sigma"] >= min_ios) table = table.select(table["xyzobs.px.value"].norms() > 0) # remove invalid xyzobs # dummy table["xyzobs.px.variance"] = flex.vec3_double(len(table), (1,1,1)) # TODO appropriate variance value table["s1"] = flex.vec3_double(len(table), (0,0,0)) # will be updated by set_obs_s1() del table["intensity.cor.value"] del table["intensity.cor.sigma"] return table
def test_select_reflections_for_sigma_calc(): """Test the reflection selection helper function.""" # first select all if below threshold reflections = flex.reflection_table() reflections["id"] = flex.int(range(0, 1000)) reflections.set_flags(flex.bool(1000, True), reflections.flags.used_in_refinement) reflections = _select_reflections_for_sigma_calc(reflections, 10000) assert reflections.size() == 1000 # select used_in_refinement if not all used in refinement and above threshold reflections = flex.reflection_table() reflections["id"] = flex.int(range(0, 1000)) good = flex.bool(1000, False) sel = flex.size_t(i for i in range(0, 1000, 2)) good.set_selected(sel, True) reflections.set_flags(good, reflections.flags.used_in_refinement) reflections = _select_reflections_for_sigma_calc(reflections, min_number_of_refl=200) assert reflections.size() == 500 assert list(reflections["id"])[0:50] == list(range(0, 100, 2)) # top up if not enough used in refinement compared to threshold reflections = flex.reflection_table() reflections["id"] = flex.int(range(0, 1000)) good = flex.bool(1000, False) sel = flex.size_t(i for i in range(0, 1000, 2)) good.set_selected(sel, True) reflections.set_flags(good, reflections.flags.used_in_refinement) reflections = _select_reflections_for_sigma_calc(reflections, min_number_of_refl=700) assert reflections.size() > 700 assert reflections.size() < 1000
def test_vs_old(data): from dials.array_family import flex r_old = data.reflections r_new = data.predict_new() index1 = flex.size_t( sorted(range(len(r_old)), key=lambda x: r_old["miller_index"][x]) ) index2 = flex.size_t( sorted(range(len(r_new)), key=lambda x: r_new["miller_index"][x]) ) r_old = r_old.select(index1) r_new = r_new.select(index2) assert len(r_old) == len(r_new) for r1, r2 in zip(r_old.rows(), r_new.rows()): assert r1["miller_index"] == r2["miller_index"] assert r1["panel"] == r2["panel"] assert r1["entering"] == r2["entering"] assert all(a == pytest.approx(b, abs=1e-7) for a, b in zip(r1["s1"], r2["s1"])) assert all( a == pytest.approx(b, abs=1e-7) for a, b in zip(r1["xyzcal.px"], r2["xyzcal.px"]) ) assert all( a == pytest.approx(b, abs=1e-7) for a, b in zip(r1["xyzcal.mm"], r2["xyzcal.mm"]) )
def test_with_old_index_generator(data): from dials.algorithms.spot_prediction import ScanStaticReflectionPredictor from dials.array_family import flex predict = ScanStaticReflectionPredictor(data.experiments[0]) r_old = predict.for_ub_old_index_generator( data.experiments[0].crystal.get_A()) r_new = predict.for_ub(data.experiments[0].crystal.get_A()) index1 = flex.size_t( sorted(range(len(r_old)), key=lambda x: r_old["miller_index"][x])) index2 = flex.size_t( sorted(range(len(r_new)), key=lambda x: r_new["miller_index"][x])) r_old = r_old.select(index1) r_new = r_new.select(index2) assert len(r_old) == len(r_new) for r1, r2 in zip(r_old.rows(), r_new.rows()): assert r1["miller_index"] == r2["miller_index"] assert r1["panel"] == r2["panel"] assert r1["entering"] == r2["entering"] assert all(a == pytest.approx(b, abs=1e-7) for a, b in zip(r1["s1"], r2["s1"])) assert all(a == pytest.approx(b, abs=1e-7) for a, b in zip(r1["xyzcal.px"], r2["xyzcal.px"])) assert all(a == pytest.approx(b, abs=1e-7) for a, b in zip(r1["xyzcal.mm"], r2["xyzcal.mm"]))
def generate_reflections(self): """Use reeke_model to generate indices of reflections near to the Ewald sphere that might be observed on a still image. Build a reflection_table of these.""" from cctbx.sgtbx import space_group_info space_group_type = space_group_info("P 1").group().type() # create a ReekeIndexGenerator UB = self.crystal.get_A() axis = self.goniometer.get_rotation_axis() s0 = self.beam.get_s0() dmin = 1.5 # use the same UB at the beginning and end - the margin parameter ensures # we still have indices close to the Ewald sphere generated from dials.algorithms.spot_prediction import ReekeIndexGenerator r = ReekeIndexGenerator(UB, UB, space_group_type, axis, s0, dmin=1.5, margin=1) # generate indices hkl = r.to_array() nref = len(hkl) # create a reflection table from dials.array_family import flex table = flex.reflection_table() table['flags'] = flex.size_t(nref, 0) table['id'] = flex.int(nref, 0) table['panel'] = flex.size_t(nref, 0) table['miller_index'] = flex.miller_index(hkl) table['entering'] = flex.bool(nref, True) table['s1'] = flex.vec3_double(nref) table['xyzcal.mm'] = flex.vec3_double(nref) table['xyzcal.px'] = flex.vec3_double(nref) return table
def tst_with_old_index_generator(self): from dials.algorithms.spot_prediction import ScanStaticReflectionPredictor from dials.array_family import flex predict = ScanStaticReflectionPredictor(self.experiments[0]) r_old = predict.for_ub_old_index_generator( self.experiments[0].crystal.get_A()) r_new = predict.for_ub(self.experiments[0].crystal.get_A()) index1 = flex.size_t( sorted(range(len(r_old)), key=lambda x: r_old['miller_index'][x])) index2 = flex.size_t( sorted(range(len(r_new)), key=lambda x: r_new['miller_index'][x])) r_old = r_old.select(index1) r_new = r_new.select(index2) assert (len(r_old) == len(r_new)) eps = 1e-7 for r1, r2 in zip(r_old.rows(), r_new.rows()): assert (r1['miller_index'] == r2['miller_index']) assert (r1['panel'] == r2['panel']) assert (r1['entering'] == r2['entering']) assert (all(abs(a - b) < eps for a, b in zip(r1['s1'], r2['s1']))) assert (all( abs(a - b) < eps for a, b in zip(r1['xyzcal.px'], r2['xyzcal.px']))) assert (all( abs(a - b) < eps for a, b in zip(r1['xyzcal.mm'], r2['xyzcal.mm']))) print 'OK'
def generate_reflections(self): """Use reeke_model to generate indices of reflections near to the Ewald sphere that might be observed on a still image. Build a reflection_table of these.""" from cctbx.sgtbx import space_group_info space_group_type = space_group_info("P 1").group().type() # create a ReekeIndexGenerator UB = self.crystal.get_U() * self.crystal.get_B() axis = self.goniometer.get_rotation_axis() s0 = self.beam.get_s0() dmin = 1.5 # use the same UB at the beginning and end - the margin parameter ensures # we still have indices close to the Ewald sphere generated from dials.algorithms.spot_prediction import ReekeIndexGenerator r = ReekeIndexGenerator(UB, UB, space_group_type, axis, s0, dmin=1.5, margin=1) # generate indices hkl = r.to_array() nref = len(hkl) # create a reflection table from dials.array_family import flex table = flex.reflection_table() table['flags'] = flex.size_t(nref, 0) table['id'] = flex.int(nref, 0) table['panel'] = flex.size_t(nref, 0) table['miller_index'] = flex.miller_index(hkl) table['entering'] = flex.bool(nref, True) table['s1'] = flex.vec3_double(nref) table['xyzcal.mm'] = flex.vec3_double(nref) table['xyzcal.px'] = flex.vec3_double(nref) return table
def main(params): # first generate the reflections - this could be called from elsewhere rlist = simple_gaussian_spots(params) correct_intensities = list(rlist["intensity.sum.value"]) del rlist["intensity.sum.value"] # now integrate those reflections using code from elsewhere if params.background_method == "xds": background_xds(rlist) elif params.background_method == "mosflm": assert params.pixel_mask != "all" background_inclined(rlist) integrate_3d_summation(rlist) integrated_intensities = list(rlist["intensity.sum.value"]) # now scan through the reflection list and find those where the integration # gave an apparently duff answer i.e. outside of 3 sigma from correct value from dials.array_family import flex overestimates = [] underestimates = [] for j, (c, i) in enumerate(zip(correct_intensities, integrated_intensities)): sigma = math.sqrt(c) if math.fabs(c - i) < 3 * sigma: continue if i > params.counts: overestimates.append(j) else: underestimates.append(j) print( "%d overestimates, %d underestimates" % (len(overestimates), len(underestimates)) ) overestimates = rlist.select(flex.size_t(overestimates)) underestimates = rlist.select(flex.size_t(underestimates)) # now pickle these, perhaps import six.moves.cPickle as pickle if params.output.under: with open(params.output.under, "wb") as fh: pickle.dump(underestimates, fh, pickle.HIGHEST_PROTOCOL) if params.output.over: with open(params.output.over, "wb") as fh: pickle.dump(overestimates, fh, pickle.HIGHEST_PROTOCOL) if params.output.all: with open(params.output.all, "wb") as fh: pickle.dump(rlist, fh, pickle.HIGHEST_PROTOCOL)
def merging_reflection_table(): table = flex.reflection_table() table['miller_index'] = flex.miller_index() table['miller_index_original'] = flex.miller_index() table['scaled_intensity'] = flex.double() table['isigi'] = flex.double() table['slope'] = flex.double() table['miller_id'] = flex.size_t() table['crystal_id'] = flex.size_t() table['iobs'] = flex.double() return table
def __init__(self): from dials.array_family import flex self.reflections = flex.reflection_table() self.reflections['panel'] = flex.size_t() self.reflections['bbox'] = flex.int6() self.reflections['miller_index'] = flex.miller_index() self.reflections['s1'] = flex.vec3_double() self.reflections['xyzcal.px'] = flex.vec3_double() self.reflections['xyzcal.mm'] = flex.vec3_double() self.reflections['entering'] = flex.bool() self.reflections['id'] = flex.int() self.reflections["flags"] = flex.size_t() self.npanels = 2 self.width = 1000 self.height = 1000 self.nrefl = 10000 self.array_range = (0, 130) self.block_size = 20 from random import randint, seed, choice seed(0) self.processed = [[] for i in range(12)] for i in range(self.nrefl): x0 = randint(0, self.width - 10) y0 = randint(0, self.height - 10) zs = randint(2, 9) x1 = x0 + randint(2, 10) y1 = y0 + randint(2, 10) for k, j in enumerate( [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]): m = k + i * 12 pos = choice(["left", "right", "centre"]) if pos == 'left': z0 = j - zs z1 = j elif pos == 'right': z0 = j z1 = j + zs else: z0 = j - zs // 2 z1 = j + zs // 2 bbox = (x0, x1, y0, y1, z0, z1) self.reflections.append({ "panel": randint(0, 1), "bbox": bbox, "flags": flex.reflection_table.flags.reference_spot }) self.processed[k].append(m)
def main(params): # first generate the reflections - this could be called from elsewhere rlist = simple_gaussian_spots(params) correct_intensities = [r['intensity.sum.value'] for r in rlist] del r['intensity.sum.value'] # now integrate those reflections using code from elsewhere if params.background_method == 'xds': background_xds(rlist) elif params.background_method == 'mosflm': assert (params.pixel_mask != 'all') background_inclined(rlist) integrate_3d_summation(rlist) integrated_intensities = [r['intensity.sum.value'] for r in rlist] # now scan through the reflection list and find those where the integration # gave an apparently duff answer i.e. outside of 3 sigma from correct value from dials.array_family import flex import math overestimates = [] underestimates = [] for j, (c, i) in enumerate(zip(correct_intensities, integrated_intensities)): sigma = math.sqrt(c) if math.fabs(c - i) < 3 * sigma: continue if i > params.counts: overestimates.append(j) else: underestimates.append(j) print '%d overestimates, %d underestimates' % (len(overestimates), len(underestimates)) overestimates = rlist.select(flex.size_t(overestimates)) underestimates = rlist.select(flex.size_t(underestimates)) # now pickle these, perhaps import cPickle as pickle if params.output.under: pickle.dump(underestimates, open(params.output.under, 'w')) if params.output.over: pickle.dump(overestimates, open(params.output.over, 'w')) if params.output.all: pickle.dump(rlist, open(params.output.all, 'w'))
def main(): parser = OptionParser(read_experiments=True, read_reflections=True) params, options = parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) data = reflections[0] expt = experiments[0] data = data.select(data.get_flags(data.flags.integrated)) # data = data.select(data['d'] > 2.0) z = data["xyzcal.px"].parts()[2] shoebox = data["shoebox"] # make a new reflection table, this one empty - then add columns for # every event in this set (with some shuffle) - flags = 32 apparently, # id == id above, intensity.sum.value=1 variance=1 n_signal=1 panel=panel # xyzobs.px.value = pixel + random.random() - 0.5 variance = 1/12 in each # direction - then map to reciprocal space events = flex.vec3_double() for s in shoebox: d = s.data k0, j0, i0 = s.bbox[0], s.bbox[2], s.bbox[4] k1, j1, i1 = d.focus() for k in range(k1): for j in range(j1): for i in range(i1): for n in range(int(d[k, j, i])): if random.random() > 0.1: continue z = k + k0 + random.random() y = j + j0 + random.random() x = i + i0 + random.random() events.append((z, y, x)) rt = flex.reflection_table() rt["xyzobs.px.value"] = events variance = flex.double(events.size(), 1.0 / 12.0) rt["xyzobs.px.variance"] = flex.vec3_double(variance, variance, variance) rt["flags"] = flex.size_t(events.size(), 32) rt["id"] = flex.int(events.size(), 0) rt["panel"] = flex.size_t(events.size(), 0) rt["intensity.sum.value"] = flex.double(events.size(), 1) rt["intensity.sum.variance"] = flex.double(events.size(), 1) rt.as_file("events.refl")
def extract_free_set(self, free_set_percentage, offset=0): """Extract a free set from all blocks.""" assert not self.free_Ih_table interval_between_groups = int(100 / free_set_percentage) free_reflection_table = flex.reflection_table() free_indices = flex.size_t() for j, block in enumerate(self.Ih_table_blocks): n_groups = block.h_index_matrix.n_cols groups_for_free_set = flex.bool(n_groups, False) for_free = flex.size_t([ i for i in range(0 + offset, n_groups, interval_between_groups) ]) groups_for_free_set.set_selected(for_free, True) free_block = block.select_on_groups(groups_for_free_set) free_reflection_table.extend(free_block.Ih_table) for sel in free_block.block_selections: free_indices.extend(sel) self.Ih_table_blocks[j] = block.select_on_groups( ~groups_for_free_set) # Now need to update dataset_info dict. removed_from_each_dataset = [ (free_block.Ih_table["dataset_id"] == i).count(True) for i in range(0, block.n_datasets) ] n_removed = 0 for i in range(0, self.Ih_table_blocks[j].n_datasets): self.Ih_table_blocks[j].dataset_info[i][ "start_index"] -= n_removed n_removed += removed_from_each_dataset[i] self.Ih_table_blocks[j].dataset_info[i][ "end_index"] -= n_removed self.blocked_selection_list = [ block.block_selections for block in self.Ih_table_blocks ] # now split by dataset and use to instantiate another Ih_table datasets = set(free_reflection_table["dataset_id"]) tables = [] indices_lists = [] for id_ in datasets: dataset_sel = free_reflection_table["dataset_id"] == id_ tables.append(free_reflection_table.select(dataset_sel)) indices_lists.append(free_indices.select(dataset_sel)) free_Ih_table = IhTable(tables, self.space_group, indices_lists, nblocks=1, anomalous=self.anomalous) # add to blocks list and selection list self.Ih_table_blocks.append(free_Ih_table.blocked_data_list[0]) self.blocked_selection_list.append( free_Ih_table.blocked_selection_list[0])
def __call__(self, imageset, pixel_labeller): """ Convert the pixel list to shoeboxes """ from dxtbx.imageset import ImageSequence # Extract the pixel lists into a list of reflections shoeboxes = flex.shoebox() spotsizes = flex.size_t() hotpixels = tuple(flex.size_t() for i in range(len(imageset.get_detector()))) if isinstance(imageset, ImageSequence): scan = imageset.get_scan() if scan.is_still(): twod = True else: twod = False else: twod = True for i, (p, hp) in enumerate(zip(pixel_labeller, hotpixels)): if p.num_pixels() > 0: creator = flex.PixelListShoeboxCreator( p, i, # panel 0, # zrange twod, # twod self.min_spot_size, # min_pixels self.max_spot_size, # max_pixels self.write_hot_pixel_mask, ) shoeboxes.extend(creator.result()) spotsizes.extend(creator.spot_size()) hp.extend(creator.hot_pixels()) logger.info("") logger.info("Extracted {} spots".format(len(shoeboxes))) # Get the unallocated spots and print some info selection = shoeboxes.is_allocated() shoeboxes = shoeboxes.select(selection) ntoosmall = (spotsizes < self.min_spot_size).count(True) ntoolarge = (spotsizes > self.max_spot_size).count(True) assert ntoosmall + ntoolarge == selection.count(False) logger.info( "Removed %d spots with size < %d pixels" % (ntoosmall, self.min_spot_size) ) logger.info( "Removed %d spots with size > %d pixels" % (ntoolarge, self.max_spot_size) ) # Return the shoeboxes return shoeboxes, hotpixels
def main(params): # first generate the reflections - this could be called from elsewhere rlist = simple_gaussian_spots(params) correct_intensities = [r['intensity.sum.value'] for r in rlist] del r['intensity.sum.value'] # now integrate those reflections using code from elsewhere if params.background_method == 'xds': background_xds(rlist) elif params.background_method == 'mosflm': assert(params.pixel_mask != 'all') background_inclined(rlist) integrate_3d_summation(rlist) integrated_intensities = [r['intensity.sum.value'] for r in rlist] # now scan through the reflection list and find those where the integration # gave an apparently duff answer i.e. outside of 3 sigma from correct value from dials.array_family import flex import math overestimates = [] underestimates = [] for j, (c, i) in enumerate(zip(correct_intensities, integrated_intensities)): sigma = math.sqrt(c) if math.fabs(c - i) < 3 * sigma: continue if i > params.counts: overestimates.append(j) else: underestimates.append(j) print '%d overestimates, %d underestimates' % (len(overestimates), len(underestimates)) overestimates = rlist.select(flex.size_t(overestimates)) underestimates = rlist.select(flex.size_t(underestimates)) # now pickle these, perhaps import cPickle as pickle if params.output.under: pickle.dump(underestimates, open(params.output.under, 'w')) if params.output.over: pickle.dump(overestimates, open(params.output.over, 'w')) if params.output.all: pickle.dump(rlist, open(params.output.all, 'w'))
def __init__(self): from dials.array_family import flex self.reflections = flex.reflection_table() self.reflections['panel'] = flex.size_t() self.reflections['bbox'] = flex.int6() self.reflections['miller_index'] = flex.miller_index() self.reflections['s1'] = flex.vec3_double() self.reflections['xyzcal.px'] = flex.vec3_double() self.reflections['xyzcal.mm'] = flex.vec3_double() self.reflections['entering'] = flex.bool() self.reflections['id'] = flex.int() self.reflections["flags"] = flex.size_t() self.npanels = 2 self.width = 1000 self.height = 1000 self.nrefl = 10000 self.array_range = (0, 130) self.block_size = 20 from random import randint, seed, choice seed(0) self.processed = [[] for i in range(12)] for i in range(self.nrefl): x0 = randint(0, self.width-10) y0 = randint(0, self.height-10) zs = randint(2, 9) x1 = x0 + randint(2, 10) y1 = y0 + randint(2, 10) for k, j in enumerate([10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]): m = k + i * 12 pos = choice(["left", "right", "centre"]) if pos == 'left': z0 = j - zs z1 = j elif pos == 'right': z0 = j z1 = j + zs else: z0 = j - zs // 2 z1 = j + zs // 2 bbox = (x0, x1, y0, y1, z0, z1) self.reflections.append({ "panel" : randint(0,1), "bbox" : bbox, "flags" : flex.reflection_table.flags.reference_spot }) self.processed[k].append(m)
def pair_up(reference, moving, params, R0, t0): from annlib_ext import AnnAdaptor as ann_adaptor from dials.array_family import flex rxyz = reference['xyzobs.px.value'].parts() mxyz = moving['xyzobs.px.value'].parts() # apply R0, t0 before performing matching - so should ideally be in almost # right position rxy = flex.vec2_double(rxyz[0], rxyz[1]) _mxy = flex.vec2_double(mxyz[0], mxyz[1]) mxy = flex.vec2_double() for __mxy in _mxy: mxy.append((R0 * __mxy + t0).elems) ann = ann_adaptor(rxy.as_double().as_1d(), 2) ann.query(mxy.as_double().as_1d()) distances = flex.sqrt(ann.distances) matches = (distances < params.far) rsel = flex.size_t() msel = flex.size_t() xyr = flex.vec2_double() xym = flex.vec2_double() for j in range(matches.size()): if not matches[j]: continue msel.append(j) rsel.append(ann.nn[j]) xym.append(mxy[j]) xyr.append(rxy[ann.nn[j]]) # filter outliers - use IQR etc. dxy = xym - xyr dx, dy = dxy.parts() iqx = IQR(dx.select(flex.sort_permutation(dx))) iqy = IQR(dy.select(flex.sort_permutation(dy))) keep_x = (dx > (iqx[0] - iqx[3])) & (dx < (iqx[2] + iqx[3])) keep_y = (dy > (iqy[0] - iqy[3])) & (dy < (iqy[2] + iqy[3])) keep = keep_x & keep_y return rsel.select(keep), msel.select(keep)
def intensities(self): ''' Compare the intensities. ''' from dials.array_family import flex # Sort by resolution d = self.refl1['d'] index = flex.size_t(reversed(sorted(range(len(d)), key=lambda x: d[x]))) self.refl1.reorder(index) self.refl2.reorder(index) # Get the intensities I1 = self.refl1['intensity.sum.value'] I2 = self.refl2['intensity.sum.value'] S1 = flex.sqrt(self.refl1['intensity.sum.variance']) S2 = flex.sqrt(self.refl2['intensity.sum.variance']) xyz1 = self.refl1['xyzcal.px'] xyz2 = self.refl2['xyzcal.px'] # Compute chunked statistics corr = [] R = [] scale = [] res = [] for i in range(len(self.refl1) // 1000): # Get the chunks of data a = i * 1000 b = (i+1) * 1000 II1 = I1[a:b] II2 = I2[a:b] res.append(d[a]) # Compute the mean and standard deviation per chunk mv1 = flex.mean_and_variance(II1) mv2 = flex.mean_and_variance(II2) m1 = mv1.mean() m2 = mv2.mean() s1 = mv1.unweighted_sample_standard_deviation() s2 = mv2.unweighted_sample_standard_deviation() # compute the correlation coefficient r = (1/(len(II1) - 1))*sum(((II1[j] - m1) / s1) * ((II2[j] - m2) / s2) for j in range(len(II1))) corr.append(r) # Compute the scale between the chunks s = sum(II1) / sum(II2) scale.append(s) # Compute R between the chunks r = sum(abs(abs(II1[j]) - abs(s * II2[j])) for j in range(len(II1))) \ / sum(abs(II1[j]) for j in range(len(II1))) R.append(r) from matplotlib import pylab pylab.plot(corr, label="CC") pylab.plot(R, label="R") pylab.plot(scale, label="K") pylab.legend() pylab.show()
def _create_block_columns(self): """Create a column to contain the block number.""" from scitbx.array_family import flex self._reflections["block"] = flex.size_t(len(self._reflections)) self._reflections["block_centre"] = flex.double(len(self._reflections))
def match_Ih_values_to_target(self, target_Ih_table): """ Use an Ih_table as a target to set Ih values in this table. Given an Ih table as a target, the common reflections across the tables are determined and the Ih_values are set to those of the target. If no matching reflection is found, then the values are removed from the table. """ assert target_Ih_table.n_work_blocks == 1 target_asu_Ih_dict = dict( zip( target_Ih_table.blocked_data_list[0].asu_miller_index, target_Ih_table.blocked_data_list[0].Ih_values, )) new_Ih_values = flex.double(self.size, 0.0) location_in_unscaled_array = 0 sorted_asu_indices, permuted = get_sorted_asu_indices( self.Ih_table["asu_miller_index"], target_Ih_table.space_group) for j, miller_idx in enumerate(OrderedSet(sorted_asu_indices)): n_in_group = self.h_index_matrix.col(j).non_zeroes if miller_idx in target_asu_Ih_dict: i = location_in_unscaled_array new_Ih_values.set_selected( flex.size_t(range(i, i + n_in_group)), flex.double(n_in_group, target_asu_Ih_dict[miller_idx]), ) location_in_unscaled_array += n_in_group self.Ih_values.set_selected(permuted, new_Ih_values) sel = self.Ih_values != 0.0 new_table = self.select(sel) # now set attributes to update object self.Ih_table = new_table.Ih_table self.h_index_matrix = new_table.h_index_matrix self.h_expand_matrix = new_table.h_expand_matrix self.block_selections = new_table.block_selections
def generate_exp_list(params, all_exp, all_ref): if params.n_subset is not None: subset_all_exp = [] subset_all_ref = [] n_picked = 0 if params.n_subset_method=="random": while n_picked < params.n_subset: idx = random.randint(0, len(all_exp)-1) subset_all_exp.append(all_exp.pop(idx)) subset_all_ref.append(all_ref.pop(idx)) n_picked += 1 elif params.n_subset_method=="n_refl": from dials.array_family import flex import cPickle as pickle len_all_ref = flex.size_t( [ len(pickle.load(open(A,"rb"))) for A in all_ref ] ) sort_order = flex.sort_permutation(len_all_ref,reverse=True) for idx in sort_order[:params.n_subset]: subset_all_exp.append(all_exp[idx]) subset_all_ref.append(all_ref[idx]) print "Selecting a subset of %d images with highest n_refl out of %d total."%( params.n_subset, len(len_all_ref)) all_exp = subset_all_exp all_ref = subset_all_ref return all_exp, all_ref
def test_multiple_panels(): from dials.array_family import flex nrefl = 1000 # Generate bboxes bbox = flex.int6(nrefl) panel = flex.size_t(nrefl) for i in range(nrefl): x0 = random.randint(0, 500) y0 = random.randint(0, 500) z0 = random.randint(0, 10) x1 = x0 + random.randint(2, 10) y1 = y0 + random.randint(2, 10) z1 = z0 + random.randint(2, 10) bbox[i] = (x0, x1, y0, y1, z0, z1) panel[i] = random.randint(0, 2) # Find the overlaps overlaps = find_overlapping(bbox, panel) assert overlaps.num_vertices() == nrefl overlaps2 = brute_force(bbox, panel) assert overlaps.num_edges() == len(overlaps2) edges = {} for edge in overlaps2: edge = (min(edge), max(edge)) edges[edge] = None for edge in overlaps.edges(): edge = (overlaps.source(edge), overlaps.target(edge)) edge = (min(edge), max(edge)) assert edge in edges
def compute_overall_stats(self): # Create lookups for elements by miller index index_lookup = defaultdict(list) for i, h in enumerate(self.reflection_table["miller_index"]): index_lookup[h].append(i) # Compute the Overall Sum(X) and Sum(X^2) for each unique reflection for h in index_lookup: sel = flex.size_t(index_lookup[h]) intensities = self.reflection_table["intensity"].select(sel) n = intensities.size() sum_x = flex.sum(intensities) sum_x2 = flex.sum(flex.pow2(intensities)) self.reflection_sums[h] = ReflectionSum(sum_x, sum_x2, n) # Compute some numbers self._num_datasets = len(set(self.reflection_table["dataset"])) self._num_groups = len(set(self.reflection_table["group"])) self._num_reflections = self.reflection_table.size() self._num_unique = len(self.reflection_sums) logger.info( """ Summary of input data: # Datasets: %s # Groups: %s # Reflections: %s # Unique reflections: %s""", self._num_datasets, self._num_groups, self._num_reflections, self._num_unique, )
def generate_spots(crystal_model, detector, beam, goniometer=None, scan=None, sel_fraction=1.0): import math experiment = Experiment(beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal_model) # if we don't set the imageset then from_predictions uses the StillsReflectionPredictor :-( from dxtbx.imageset import NullReader, ImageSweep imageset = ImageSweep(NullReader, indices=range(len(scan.get_epochs())), beam=beam, goniometer=goniometer, detector=detector, scan=scan) experiment.imageset = imageset predicted = flex.reflection_table.from_predictions(experiment) sel = flex.random_selection(len(predicted), int(math.floor(sel_fraction*len(predicted)))) predicted = predicted.select(sel) predicted['imageset_id'] = flex.size_t(len(predicted), 0) predicted['xyzobs.px.value'] = predicted['xyzcal.px'] predicted['xyzobs.px.variance'] = flex.vec3_double( len(predicted), (0.5,0.5,0.5)) return predicted
def ersatz_MCMC(self, variable_params): from LS49.adse13_187.adse13_221.case_run import case_job_runner class ersatz(MCMC_manager, case_job_runner): pass self.MCMC = ersatz() self.MCMC.get_amplitudes(self.dials_model, self.refl_table) relevant_whitelist_order = flex.size_t() for sidx in range(len( self.refl_table["spots_offset"])): #loop through the shoeboxes for pidx in range( self.refl_table["spots_offset"][sidx], self.refl_table["spots_offset"][sidx] + self.refl_table["spots_size"][sidx]): relevant_whitelist_order.append(self.spots_pixels[pidx]) self.MCMC.set_whitelist(relevant_whitelist_order) modality = "job" return self.MCMC.job_runner( expt=self.expt, alt_expt=self.dials_model, params=variable_params, mask_array=self.monolithic_mask_whole_detector_as_1D_bool ) # returns simulated image as numpy array
def _xyzcal_mm_to_px(self, experiments, reflections): # set xyzcal.px field in reflections reflections["xyzcal.px"] = flex.vec3_double(len(reflections)) for i, expt in enumerate(experiments): imgset_sel = reflections["imageset_id"] == i refined_reflections = reflections.select(imgset_sel) panel_numbers = flex.size_t(refined_reflections["panel"]) xyzcal_mm = refined_reflections["xyzcal.mm"] x_mm, y_mm, z_rad = xyzcal_mm.parts() xy_cal_mm = flex.vec2_double(x_mm, y_mm) xy_cal_px = flex.vec2_double(len(xy_cal_mm)) for i_panel in range(len(expt.detector)): panel = expt.detector[i_panel] sel = panel_numbers == i_panel xy_cal_px.set_selected( sel, panel.millimeter_to_pixel(xy_cal_mm.select(sel)) ) x_px, y_px = xy_cal_px.parts() if expt.scan is not None: z_px = expt.scan.get_array_index_from_angle(z_rad, deg=False) else: # must be a still image, z centroid not meaningful z_px = z_rad xyzcal_px = flex.vec3_double(x_px, y_px, z_px) reflections["xyzcal.px"].set_selected(imgset_sel, xyzcal_px)
def model(self, reflections, profiles): from dials.array_family import flex indices = flex.size_t(range(len(self))) weights = flex.double([1.0] * len(self)) for profile in profiles: self.add(indices, weights, profile)
def display(self, num): """ Display some shoeboxes """ from dials_scratch.jmp.viewer import show_image_stack_multi_view from random import sample from dials.array_family import flex def simulate(experiments, reflections, parameters): from dials_scratch.jmp.profile_modelling import MLTarget3D func = MLTarget3D(experiments[0], reflections) return [ func.simulate(i, parameters) for i in range(len(reflections)) ] # Sample from reflections reflections = self.reflections.select( flex.size_t(sample(range(len(self.reflections)), num))) # Simulate the reflection profiles from the current model simulated = simulate(self.experiments, reflections, self.parameters) # Display stuff for model, data_sbox in zip(simulated, reflections["shoebox"]): data = data_sbox.data show_image_stack_multi_view(model.as_numpy_array(), vmax=flex.max(model)) show_image_stack_multi_view(data.as_numpy_array(), vmax=flex.max(data))
def generate_exp_list(params, all_exp, all_ref): if params.n_subset is not None: subset_all_exp = [] subset_all_ref = [] n_picked = 0 if params.n_subset_method == "random": while n_picked < params.n_subset: idx = random.randint(0, len(all_exp) - 1) subset_all_exp.append(all_exp.pop(idx)) subset_all_ref.append(all_ref.pop(idx)) n_picked += 1 elif params.n_subset_method == "n_refl": from dials.array_family import flex import cPickle as pickle len_all_ref = flex.size_t( [len(pickle.load(open(A, "rb"))) for A in all_ref]) sort_order = flex.sort_permutation(len_all_ref, reverse=True) for idx in sort_order[:params.n_subset]: subset_all_exp.append(all_exp[idx]) subset_all_ref.append(all_ref[idx]) print "Selecting a subset of %d images with highest n_refl out of %d total." % ( params.n_subset, len(len_all_ref)) all_exp = subset_all_exp all_ref = subset_all_ref return all_exp, all_ref
def scale_partial_reflections(integrated_data, min_partiality=0.5): '''Scale partial reflections (after summation) according to their estimated partiality - for profile fitted reflections this will result in no change, for summation integrated reflections will be scaled up by 1 / partiality with error accordingly scaled. N.B. this will report the scaled up partiality for the output reflection.''' # assert: in here there will be no multi-part partial reflections if not 'partiality' in integrated_data: return integrated_data isel = (integrated_data['partiality'] < 1.0).iselection() if len(isel) == 0: return integrated_data delete = flex.size_t() for j in isel: if integrated_data['partiality'][j] < min_partiality: delete.append(j) continue inv_p = 1.0 / integrated_data['partiality'][j] integrated_data['intensity.sum.value'][j] *= inv_p integrated_data['intensity.sum.variance'][j] *= inv_p integrated_data['partiality'][j] *= 1.0 integrated_data.del_selected(delete) return integrated_data
def show_image(c,b,s, BB=None, SS=None): import numpy.ma from dials.array_family import flex N = 100 im = flex.double(flex.grid(N, N)) mask = flex.bool(flex.grid(N, N)) for j in range(N): for i in range(N): B = -1.0 + j * 10.0 / N S = -1.0 + i * 10.0 / N im[j,i], mask[j,i] = func(c,b,s,B,S) im[j,i] = -im[j,i] masked_im = numpy.ma.array( # im.as_numpy_array(), flex.exp(im).as_numpy_array(), mask = mask.as_numpy_array()) mask2 = flex.bool(flex.grid(N, N)) indices = [] for i in range(len(mask)): if mask[i] == False: indices.append(i) indices = flex.size_t(indices) ind = flex.max_index(im.select(indices)) ind = indices[ind] maxy = -1.0 + (ind % N) * 10.0 / N maxx = -1.0 + (ind // N) * 10.0 / N from matplotlib import pylab pylab.imshow(masked_im, origin='bottom', extent=[-1.0, 9.0, -1.0, 9.0]) if YY is not None and XX is not None: pylab.plot(YY, XX) pylab.scatter([maxy], [maxx]) pylab.show()
def tst_multiple_panels(self): from dials.array_family import flex from random import randint nrefl = 1000 # Generate bboxes bbox = flex.int6(nrefl) panel = flex.size_t(nrefl) for i in range(nrefl): x0 = randint(0, 500) y0 = randint(0, 500) z0 = randint(0, 10) x1 = x0 + randint(2, 10) y1 = y0 + randint(2, 10) z1 = z0 + randint(2, 10) bbox[i] = (x0, x1, y0, y1, z0, z1) panel[i] = randint(0,2) # Find the overlaps overlaps = find_overlapping(bbox, panel) assert(overlaps.num_vertices() == nrefl) overlaps2 = self.brute_force(bbox, panel) assert(overlaps.num_edges() == len(overlaps2)) edges = {} for edge in overlaps2: edge = (min(edge), max(edge)) edges[edge] = None for edge in overlaps.edges(): edge = (overlaps.source(edge), overlaps.target(edge)) edge = (min(edge), max(edge)) assert(edge in edges) print 'OK'
def test_match_mismatched_sizes(): n = 100 s = 10 def r(): return random.random() xyz = flex.vec3_double() for j in range(n): xyz.append((r() * s, r() * s, r() * s * 20)) order = list(range(n)) random.shuffle(order) xyz2 = xyz.select(flex.size_t(order)) a = flex.reflection_table() b = flex.reflection_table() a["xyz"] = xyz[:n // 2] b["xyz"] = xyz2 mm, nn, distance = a.match(b, key="xyz", scale=(1.0, 1.0, 0.05)) a_ = a.select(mm) b_ = b.select(nn) for _a, _b in zip(a_["xyz"], b_["xyz"]): assert _a == pytest.approx(_b)
def _grads_model_loop( self, parameterisations, reflections, results, callback=None, derivatives_fn=None, ): # loop over the parameterisations for p in parameterisations: # Determine (sub)set of reflections affected by this parameterisation isel = flex.size_t() for exp_id in p.get_experiment_ids(): isel.extend(self._experiment_to_idx[exp_id]) # Extend derivative vectors for this parameterisation results = self._extend_gradient_vectors(results, self._nref, p.num_free(), keys=self._grad_names) if len(isel) == 0: # if no reflections are in this experiment, skip calculation of # gradients, but must still process null gradients by a callback if callback: for _ in range(p.num_free()): results[self._iparam] = callback(results[self._iparam]) self._iparam += 1 else: self._iparam += p.num_free() continue w_inv = self._w_inv.select(isel) u_w_inv = self._u_w_inv.select(isel) v_w_inv = self._v_w_inv.select(isel) dpv_dbeam_p, dAngle_dbeam_p = derivatives_fn( isel, parameterisation=p, reflections=reflections) # convert to dX/dp, dY/dp and assign the elements of the vectors # corresponding to this experiment dX_dbeam_p, dY_dbeam_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp( w_inv, u_w_inv, v_w_inv, dpv_dbeam_p) for dX, dY, dAngle in zip(dX_dbeam_p, dY_dbeam_p, dAngle_dbeam_p): if dX is not None: results[self._iparam][self._grad_names[0]].set_selected( isel, dX) if dY is not None: results[self._iparam][self._grad_names[1]].set_selected( isel, dY) if dAngle is not None: results[self._iparam][self._grad_names[2]].set_selected( isel, dAngle) if callback is not None: results[self._iparam] = callback(results[self._iparam]) # increment the parameter index pointer self._iparam += 1 return results
def scale_partial_reflections(integrated_data, min_partiality=0.5): """Scale partial reflections (after summation) according to their estimated partiality - for profile fitted reflections this will result in no change, for summation integrated reflections will be scaled up by 1 / partiality with error accordingly scaled. N.B. this will report the scaled up partiality for the output reflection.""" # assert: in here there will be no multi-part partial reflections if not "partiality" in integrated_data: return integrated_data isel = (integrated_data["partiality"] < 1.0).iselection() if len(isel) == 0: return integrated_data from dials.array_family import flex delete = flex.size_t() for j in isel: if integrated_data["partiality"][j] < min_partiality: delete.append(j) continue inv_p = 1.0 / integrated_data["partiality"][j] integrated_data["intensity.sum.value"][j] *= inv_p integrated_data["intensity.sum.variance"][j] *= inv_p integrated_data["partiality"][j] *= 1.0 integrated_data.del_selected(delete) return integrated_data
def generate_spots(crystal_model, detector, beam, goniometer=None, scan=None, sel_fraction=1.0): import math experiment = Experiment(beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal_model) # if we don't set the imageset then from_predictions uses the StillsReflectionPredictor :-( from dxtbx.imageset import NullReader, ImageSweep imageset = ImageSweep(NullReader, indices=range(len(scan.get_epochs())), beam=beam, goniometer=goniometer, detector=detector, scan=scan) experiment.imageset = imageset predicted = flex.reflection_table.from_predictions(experiment) sel = flex.random_selection(len(predicted), int(math.floor(sel_fraction * len(predicted)))) predicted = predicted.select(sel) predicted['imageset_id'] = flex.size_t(len(predicted), 0) predicted['xyzobs.px.value'] = predicted['xyzcal.px'] predicted['xyzobs.px.variance'] = flex.vec3_double(len(predicted), (0.5, 0.5, 0.5)) return predicted
def _create_block_columns(self): """Create a column to contain the block number.""" from scitbx.array_family import flex self._reflections['block'] = flex.size_t(len(self._reflections)) self._reflections['block_centre'] = flex.double(len(self._reflections)) return
def __call__(self, imageset, pixel_labeller): ''' Convert the pixel list to shoeboxes ''' from dxtbx.imageset import ImageSweep from dials.array_family import flex # Extract the pixel lists into a list of reflections shoeboxes = flex.shoebox() spotsizes = flex.size_t() hotpixels = tuple(flex.size_t() for i in range(len(imageset.get_detector()))) if isinstance(imageset, ImageSweep): twod = False else: twod = True for i, (p, hp) in enumerate(zip(pixel_labeller, hotpixels)): if p.num_pixels() > 0: creator = flex.PixelListShoeboxCreator( p, i, # panel 0, # zrange twod, # twod self.min_spot_size, # min_pixels self.max_spot_size, # max_pixels self.write_hot_pixel_mask) shoeboxes.extend(creator.result()) spotsizes.extend(creator.spot_size()) hp.extend(creator.hot_pixels()) logger.info('') logger.info('Extracted {0} spots'.format(len(shoeboxes))) # Get the unallocated spots and print some info selection = shoeboxes.is_allocated() shoeboxes = shoeboxes.select(selection) ntoosmall = (spotsizes < self.min_spot_size).count(True) ntoolarge = (spotsizes > self.max_spot_size).count(True) assert ntoosmall + ntoolarge == selection.count(False) logger.info('Removed %d spots with size < %d pixels' % ( ntoosmall, self.min_spot_size)) logger.info('Removed %d spots with size > %d pixels' % ( ntoolarge, self.max_spot_size)) # Return the shoeboxes return shoeboxes, hotpixels
def __init__(self, strategies, n_bins=8, degrees_per_bin=5): from cctbx import crystal, miller import copy sg = strategies[0].experiment.crystal.get_space_group() \ .build_derived_reflection_intensity_group(anomalous_flag=True) cs = crystal.symmetry( unit_cell=strategies[0].experiment.crystal.get_unit_cell(), space_group=sg) for i, strategy in enumerate(strategies): if i == 0: predicted = copy.deepcopy(strategy.predicted) else: predicted_ = copy.deepcopy(strategy.predicted) predicted_['dose'] += (flex.max(predicted['dose']) + 1) predicted.extend(predicted_) ms = miller.set(cs, indices=predicted['miller_index'], anomalous_flag=True) ma = miller.array(ms, data=flex.double(ms.size(),1), sigmas=flex.double(ms.size(), 1)) if 1: merging = ma.merge_equivalents() o = merging.array().customized_copy( data=merging.redundancies().data().as_double()).as_mtz_dataset('I').mtz_object() o.write('predicted.mtz') d_star_sq = ma.d_star_sq().data() binner = ma.setup_binner_d_star_sq_step( d_star_sq_step=(flex.max(d_star_sq)-flex.min(d_star_sq)+1e-8)/n_bins) dose = predicted['dose'] range_width = 1 range_min = flex.min(dose) - range_width range_max = flex.max(dose) n_steps = 2 + int((range_max - range_min) - range_width) binner_non_anom = ma.as_non_anomalous_array().use_binning( binner) self.n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1]) from xia2.Modules.PyChef2 import ChefStatistics chef_stats = ChefStatistics( ma.indices(), ma.data(), ma.sigmas(), ma.d_star_sq().data(), dose, self.n_complete, binner, ma.space_group(), ma.anomalous_flag(), n_steps) def fraction_new(completeness): # Completeness so far at end of image completeness_end = completeness[1:] # Completeness so far at start of image completeness_start = completeness[:-1] # Fraction of unique reflections observed for the first time on each image return completeness_end - completeness_start self.dose = dose self.ieither_completeness = chef_stats.ieither_completeness() self.iboth_completeness = chef_stats.iboth_completeness() self.frac_new_ref = fraction_new(self.ieither_completeness) / degrees_per_bin self.frac_new_pairs = fraction_new(self.iboth_completeness) / degrees_per_bin
def __init__(self, reflections, experiments, nref_per_degree=None, max_sample_size=None, min_sample_size=0, close_to_spindle_cutoff=0.1, outlier_detector=None, weighting_strategy_override=None, verbosity=0): # set verbosity self._verbosity = verbosity # keep track of models self._experiments = experiments goniometers = [e.goniometer for e in self._experiments] self._axes = [matrix.col(g.get_rotation_axis()) if g else None for g in goniometers] self._s0vecs = [matrix.col(e.beam.get_s0()) for e in self._experiments] # keep track of the original indices of the reflections reflections['iobs'] = flex.size_t_range(len(reflections)) # set up the reflection inclusion criteria self._close_to_spindle_cutoff = close_to_spindle_cutoff #too close to spindle self._outlier_detector = outlier_detector #for outlier rejection self._nref_per_degree = nref_per_degree #random subsets self._max_sample_size = max_sample_size #sample size ceiling self._min_sample_size = min_sample_size #sample size floor # exclude reflections that fail some inclusion criteria refs_to_keep = self._id_refs_to_keep(reflections) self._accepted_refs_size = len(refs_to_keep) # set entering flags for all reflections reflections['entering'] = calculate_entering_flags(reflections, self._experiments) # set observed frame numbers for all reflections if not already present calculate_frame_numbers(reflections, self._experiments) # reset all use flags self.reset_accepted_reflections(reflections) # put full list of indexed reflections aside and select only the reflections # that were not excluded to manage self._indexed = reflections self._reflections = reflections.select(flex.size_t(refs_to_keep)) # set weights for all kept reflections if weighting_strategy_override is not None: self._weighting_strategy = weighting_strategy_override self._weighting_strategy.calculate_weights(self._reflections) # not known until the manager is finalised self._sample_size = None return
def __init__(self, experiment, other=None, d_min=None, unit_cell_scale=1, degrees_per_bin=5, min_frac_new=0.001): print experiment.goniometer print experiment.scan self.experiment = copy.deepcopy(experiment) self.other = other self.unit_cell_scale = unit_cell_scale self.degrees_per_bin = degrees_per_bin self.min_frac_new = min_frac_new scan = self.experiment.scan detector = self.experiment.detector beam = self.experiment.beam crystal = self.experiment.crystal from cctbx import uctbx s = self.unit_cell_scale assert s > 0 uc = crystal.get_unit_cell() a, b, c, alpha, beta, gamma = uc.parameters() uc_mod = uctbx.unit_cell((a/s, b/s, c/s, alpha, beta, gamma)) crystal.set_B(matrix.sqr(uc_mod.fractionalization_matrix()).transpose()) self.d_min = d_min if self.d_min is None: self.d_min = detector.get_max_inscribed_resolution(beam.get_s0()) import math # bragg's law sin_theta = 0.5 * beam.get_wavelength()/self.d_min theta_max_rad = math.asin(sin_theta) self.theta_max = theta_max_rad * 180/math.pi print "theta_max (degrees): %.2f" %self.theta_max Btot = 1 - 3 * (4 * theta_max_rad - math.sin(4 * theta_max_rad))/(32 * (math.sin(theta_max_rad)**3)) print Btot # Section 2.9, Dauter Acta Cryst. (1999). D55, 1703-1717 #max_rotation = 360 + 2 * self.theta_max max_rotation = 360 image_range = scan.get_image_range() oscillation = scan.get_oscillation() scan.set_image_range((image_range[0], image_range[0]+int(max_rotation/oscillation[1]))) self.predict() x, y, z = self.predicted['xyzcal.px'].parts() self.predicted['dose'] = flex.size_t( list(flex.ceil(z * oscillation[1]/self.degrees_per_bin).iround())) self.stats = ComputeStats([self], degrees_per_bin=self.degrees_per_bin) self.ieither_completeness = self.stats.ieither_completeness self.iboth_completeness = self.stats.iboth_completeness self.frac_new_ref = self.stats.frac_new_ref self.frac_new_pairs = self.stats.frac_new_pairs self.determine_cutoffs(self.min_frac_new) self.show()
def __call__(self, params, options): ''' Import the integrate.hkl file. ''' from iotbx.xds import integrate_hkl from dials.array_family import flex from dials.util.command_line import Command from cctbx import sgtbx # Get the unit cell to calculate the resolution uc = self._experiment.crystal.get_unit_cell() # Read the INTEGRATE.HKL file Command.start('Reading INTEGRATE.HKL') handle = integrate_hkl.reader() handle.read_file(self._integrate_hkl) hkl = flex.miller_index(handle.hkl) xyzcal = flex.vec3_double(handle.xyzcal) xyzobs = flex.vec3_double(handle.xyzobs) iobs = flex.double(handle.iobs) sigma = flex.double(handle.sigma) rlp = flex.double(handle.rlp) peak = flex.double(handle.peak) * 0.01 Command.end('Read %d reflections from INTEGRATE.HKL file.' % len(hkl)) # Derive the reindex matrix rdx = self.derive_reindex_matrix(handle) print 'Reindex matrix:\n%d %d %d\n%d %d %d\n%d %d %d' % (rdx.elems) # Reindex the reflections Command.start('Reindexing reflections') cb_op = sgtbx.change_of_basis_op(sgtbx.rt_mx(sgtbx.rot_mx(rdx.elems))) hkl = cb_op.apply(hkl) Command.end('Reindexed %d reflections' % len(hkl)) # Create the reflection list Command.start('Creating reflection table') table = flex.reflection_table() table['id'] = flex.int(len(hkl), 0) table['panel'] = flex.size_t(len(hkl), 0) table['miller_index'] = hkl table['xyzcal.px'] = xyzcal table['xyzobs.px.value'] = xyzobs table['intensity.cor.value'] = iobs table['intensity.cor.variance'] = sigma**2 table['intensity.prf.value'] = iobs * peak / rlp table['intensity.prf.variance'] = (sigma * peak / rlp)**2 table['lp'] = 1.0 / rlp table['d'] = flex.double(uc.d(h) for h in hkl) Command.end('Created table with {0} reflections'.format(len(table))) # Output the table to pickle file if params.output.filename is None: params.output.filename = 'integrate_hkl.pickle' Command.start('Saving reflection table to %s' % params.output.filename) table.as_pickle(params.output.filename) Command.end('Saved reflection table to %s' % params.output.filename)
def tst_find_overlapping(self): from dials.array_family import flex from random import randint, uniform from dials.model.data import Shoebox N = 10000 r = flex.reflection_table(N) r['bbox'] = flex.int6(N) r['panel'] = flex.size_t(N) r['id'] = flex.int(N) r['imageset_id'] = flex.int(N) for i in range(N): x0 = randint(0, 100) x1 = randint(1, 10) + x0 y0 = randint(0, 100) y1 = randint(1, 10) + y0 z0 = randint(0, 100) z1 = randint(1, 10) + z0 panel = randint(0,2) pid = randint(0,2) r['bbox'][i] = (x0,x1,y0,y1,z0,z1) r['panel'][i] = panel r['id'][i] = pid r['imageset_id'][i] = pid def is_overlap(b0, b1, border): b0 = b0[0]-border,b0[1]+border,b0[2]-border,b0[3]+border,b0[4]-border,b0[5]+border b1 = b1[0]-border,b1[1]+border,b1[2]-border,b1[3]+border,b1[4]-border,b1[5]+border if not (b1[0] > b0[1] or b1[1] < b0[0] or b1[2] > b0[3] or b1[3] < b0[2] or b1[4] > b0[5] or b1[5] < b0[4]): return True return False for i in [0, 2, 5]: overlaps = r.find_overlaps(border=i) for item in overlaps.edges(): i0 = overlaps.source(item) i1 = overlaps.target(item) r0 = r[i0] r1 = r[i1] p0 = r0['panel'] p1 = r1['panel'] b0 = r0['bbox'] b1 = r1['bbox'] j0 = r0['imageset_id'] j1 = r1['imageset_id'] assert j0 == j1 assert p0 == p1 assert is_overlap(b0,b1,i) print 'OK'
def run(args): import libtbx.load_env usage = "%s experiments.json [options]" %libtbx.env.dispatcher_name parser = OptionParser( usage=usage, phil=phil_scope, read_experiments=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) if len(experiments) == 0: parser.print_help() return elif len(experiments) > 1: raise Sorry("More than one experiment present") assert len(params.miller_index), "Must specify at least one miller_index to predict." experiment = experiments[0] reflections = flex.reflection_table() miller_indices = flex.miller_index() entering_flags = flex.bool() for mi in params.miller_index: miller_indices.append(mi) miller_indices.append(mi) entering_flags.append(True) entering_flags.append(False) reflections['miller_index'] = miller_indices reflections['entering'] = entering_flags reflections['id'] = flex.size_t(len(reflections), 0) if params.expand_to_p1: from cctbx.miller import expand_to_p1_iselection proxy = expand_to_p1_iselection( experiment.crystal.get_space_group(), anomalous_flag=True, indices=miller_indices, build_iselection=True) reflections = reflections.select(proxy.iselection) reflections['miller_index'] = proxy.indices from dials.algorithms.refinement.prediction.managed_predictors import ExperimentsPredictor predictor = ExperimentsPredictor([experiment]) predicted = predictor.predict(reflections) zmin, zmax = experiment.scan.get_array_range() z = predicted['xyzcal.px'].parts()[2] predicted = predicted.select((z >= zmin) & (z <= zmax)) show_predictions(predicted)
def tst_select(self): from dials.array_family import flex # The columns as lists c1 = list(range(10)) c2 = list(range(10)) c3 = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'i', 'j', 'k'] # Create a table with some elements table = flex.reflection_table() table['col1'] = flex.int(c1) table['col2'] = flex.double(c2) table['col3'] = flex.std_string(c3) # Select some columns new_table = table.select(('col1', 'col2')) assert(new_table.nrows() == 10) assert(new_table.ncols() == 2) assert(all(a == b for a, b in zip(new_table['col1'], c1))) assert(all(a == b for a, b in zip(new_table['col2'], c2))) print 'OK' # Select some columns new_table = table.select(flex.std_string(['col1', 'col2'])) assert(new_table.nrows() == 10) assert(new_table.ncols() == 2) assert(all(a == b for a, b in zip(new_table['col1'], c1))) assert(all(a == b for a, b in zip(new_table['col2'], c2))) print 'OK' # Select some rows index = flex.size_t([0, 1, 5, 8, 9]) cc1 = [c1[i] for i in index] cc2 = [c2[i] for i in index] cc3 = [c3[i] for i in index] new_table = table.select(index) assert(new_table.nrows() == 5) assert(new_table.ncols() == 3) assert(all(a == b for a, b in zip(new_table['col1'], cc1))) assert(all(a == b for a, b in zip(new_table['col2'], cc2))) assert(all(a == b for a, b in zip(new_table['col3'], cc3))) print 'OK' # Select some rows index = flex.bool([True, True, False, False, False, True, False, False, True, True]) new_table = table.select(index) assert(new_table.nrows() == 5) assert(new_table.ncols() == 3) assert(all(a == b for a, b in zip(new_table['col1'], cc1))) assert(all(a == b for a, b in zip(new_table['col2'], cc2))) assert(all(a == b for a, b in zip(new_table['col3'], cc3))) print 'OK'
def assert_basic_mask_is_correct(mask, ninvalid, nforeground): from scitbx.array_family import flex from dials.algorithms.shoebox import MaskCode invalid = flex.size_t( i for i in range(len(mask)) if not mask[i] & MaskCode.Valid) foreground = flex.size_t( i for i in range(len(mask)) if mask[i] & MaskCode.Foreground) background = flex.size_t( i for i in range(len(mask)) if mask[i] & MaskCode.Background) background_used = flex.size_t( i for i in range(len(mask)) if mask[i] & MaskCode.BackgroundUsed) background_valid = flex.size_t( i for i in range(len(mask)) if mask[i] & MaskCode.Valid and mask[i] & MaskCode.Background) assert(len(invalid) == ninvalid) assert(len(foreground) == nforeground) assert(len(background) == len(mask) - len(foreground)) assert(len(background_used) < len(background)) assert(len(set(background).intersection(foreground)) == 0) assert(len(set(background).intersection(background_used)) == len(background_used)) return invalid, foreground, background, background_used, background_valid
def __call__(self, params, options): ''' Import the spot.xds file. ''' from iotbx.xds import spot_xds from dials.util.command_line import Command from dials.array_family import flex # Read the SPOT.XDS file Command.start('Reading SPOT.XDS') handle = spot_xds.reader() handle.read_file(self._spot_xds) centroid = handle.centroid intensity = handle.intensity try: miller_index = handle.miller_index except AttributeError: miller_index = None Command.end('Read {0} spots from SPOT.XDS file.'.format(len(centroid))) # Create the reflection list Command.start('Creating reflection list') table = flex.reflection_table() table['id'] = flex.int(len(centroid), 0) table['panel'] = flex.size_t(len(centroid), 0) if miller_index: table['miller_index'] = flex.miller_index(miller_index) table['xyzobs.px.value'] = flex.vec3_double(centroid) table['intensity.sum.value'] = flex.double(intensity) Command.end('Created reflection list') # Remove invalid reflections Command.start('Removing invalid reflections') if miller_index and params.remove_invalid: flags = flex.bool([h != (0, 0, 0) for h in table['miller_index']]) table = table.select(flags) Command.end('Removed invalid reflections, %d remaining' % len(table)) # Fill empty standard columns if params.add_standard_columns: Command.start('Adding standard columns') rt = flex.reflection_table.empty_standard(len(table)) rt.update(table) table = rt # set variances to unity table['xyzobs.mm.variance'] = flex.vec3_double(len(table), (1,1,1)) table['xyzobs.px.variance'] = flex.vec3_double(len(table), (1,1,1)) Command.end('Standard columns added') # Output the table to pickle file if params.output.filename is None: params.output.filename = 'spot_xds.pickle' Command.start('Saving reflection table to %s' % params.output.filename) table.as_pickle(params.output.filename) Command.end('Saved reflection table to %s' % params.output.filename)
def _find_nearest_neighbours(self, observed, predicted): ''' Find the nearest predicted spot to the observed spot. :param observed: The observed reflections :param predicted: The predicted reflections :returns: (nearest neighbours, distance) ''' from scitbx.array_family import flex from logging import warn # Get the predicted coordinates predicted_panel = predicted['panel'] predicted_xyz = predicted['xyzcal.px'] observed_panel = observed['panel'] observed_xyz = observed['xyzobs.px.value'] # Get the number of panels max_panel1 = flex.max(predicted_panel) max_panel2 = flex.max(observed_panel) max_panel = max([max_panel1, max_panel2]) nn_all = flex.size_t() dd_all = flex.double() for panel in range(max_panel+1): pind = predicted_panel == panel oind = observed_panel == panel pxyz = predicted_xyz.select(pind) oxyz = observed_xyz.select(oind) try: nn, d = self._find_nearest_neighbours_single(oxyz, pxyz) indices = flex.size_t(range(len(pind))).select(pind) indices = indices.select(flex.size_t(list(nn))) nn_all.extend(indices) dd_all.extend(d) except Exception: warn("Unable to match spots on panel %d" % panel) return nn_all, dd_all
def flex_ios(val, var): ''' Compute I/sigma or return zero for each element. ''' assert(len(val) == len(var)) result = flex.double(len(val),0) indices = flex.size_t(range(len(val))).select(var > 0) val = val.select(indices) var = var.select(indices) assert(var.all_gt(0)) result.set_selected(indices, val / flex.sqrt(var)) return result
def _grads_xl_unit_cell_loop(self, reflections, results, callback=None): """Loop over all crystal unit cell parameterisations, calculate gradients and extend the results""" # loop over the crystal unit cell parameterisations for xlucp in self._xl_unit_cell_parameterisations: # Determine (sub)set of reflections affected by this parameterisation isel = flex.size_t() for exp_id in xlucp.get_experiment_ids(): isel.extend(self._experiment_to_idx[exp_id]) # Extend derivative vectors for this crystal unit cell parameterisation results = self._extend_gradient_vectors(results, self._nref, xlucp.num_free(), keys=self._grad_names) if len(isel) == 0: # if no reflections are in this experiment, skip calculation of # gradients, but must still process null gradients by a callback if callback is not None: for iparam in xrange(xlucp.num_free()): results[self._iparam] = callback(results[self._iparam]) self._iparam += 1 else: self._iparam += xlucp.num_free() continue w_inv = self._w_inv.select(isel) u_w_inv = self._u_w_inv.select(isel) v_w_inv = self._v_w_inv.select(isel) dpv_dxluc_p, dAngle_dxluc_p = self._xl_unit_cell_derivatives(isel, parameterisation=xlucp, reflections=reflections) # convert to dX/dp, dY/dp and assign the elements of the vectors # corresponding to this experiment dX_dxluc_p, dY_dxluc_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp( w_inv, u_w_inv, v_w_inv, dpv_dxluc_p) for dX, dY, dAngle in zip(dX_dxluc_p, dY_dxluc_p, dAngle_dxluc_p): if dX is not None: results[self._iparam][self._grad_names[0]].set_selected(isel, dX) if dY is not None: results[self._iparam][self._grad_names[1]].set_selected(isel, dY) if dAngle is not None: results[self._iparam][self._grad_names[2]].set_selected(isel, dAngle) if callback is not None: results[self._iparam] = callback(results[self._iparam]) # increment the parameter index pointer self._iparam += 1 return results
def get_hkl_offset_correlation_coefficients( dials_reflections, dials_crystal, map_to_asu=False, grid_h=0, grid_k=0, grid_l=0, reference=None): # N.B. deliberately ignoring d_min, d_max as these are inconsistent with # changing the miller indices from dials.array_family import flex from cctbx.miller import set as miller_set from cctbx import sgtbx cs = cctbx_crystal_from_dials(dials_crystal) ms = cctbx_i_over_sigi_ms_from_dials_data(dials_reflections, cs) if reference: reference_ms = cctbx_i_over_sigi_ms_from_dials_data(reference, cs) else: reference_ms = None ccs = flex.double() offsets = flex.vec3_int() nref = flex.size_t() if reference: cb_op = sgtbx.change_of_basis_op('x,y,z') else: cb_op = sgtbx.change_of_basis_op('-x,-y,-z') hkl_test = [(h, k, l) for h in range(-grid_h, grid_h + 1) \ for k in range(-grid_k, grid_k + 1) \ for l in range(-grid_l, grid_l + 1)] for hkl in hkl_test: indices = offset_miller_indices(ms.indices(), hkl) reindexed_indices = cb_op.apply(indices) rms = miller_set(cs, reindexed_indices).array(ms.data()) if reference_ms: _ms = reference_ms else: _ms = miller_set(cs, indices).array(ms.data()) n, cc = compute_miller_set_correlation(_ms, rms, map_to_asu=map_to_asu) ccs.append(cc) offsets.append(hkl) nref.append(n) return offsets, ccs, nref
def sample_predictions(self, experiments, predicted, params): """ Select a random sample of the predicted reflections to integrate. """ from dials.array_family import flex nref_per_degree = params.sampling.reflections_per_degree min_sample_size = params.sampling.minimum_sample_size max_sample_size = params.sampling.maximum_sample_size # this code is very similar to David's code in algorithms/refinement/reflection_manager.py! # constants from math import pi RAD2DEG = 180.0 / pi DEG2RAD = pi / 180.0 working_isel = flex.size_t() for iexp, exp in enumerate(experiments): sel = predicted["id"] == iexp isel = sel.iselection() # refs = self._reflections.select(sel) nrefs = sample_size = len(isel) # set sample size according to nref_per_degree (per experiment) if exp.scan and nref_per_degree: sweep_range_rad = exp.scan.get_oscillation_range(deg=False) width = abs(sweep_range_rad[1] - sweep_range_rad[0]) * RAD2DEG sample_size = int(nref_per_degree * width) else: sweep_range_rad = None # adjust sample size if below the chosen limit sample_size = max(sample_size, min_sample_size) # set maximum sample size if requested if max_sample_size: sample_size = min(sample_size, max_sample_size) # determine subset and collect indices if sample_size < nrefs: isel = isel.select(flex.random_selection(nrefs, sample_size)) working_isel.extend(isel) # create subset return predicted.select(working_isel)
def _create_working_set(self): """Make a subset of the indices of reflections to use in refinement""" working_isel = flex.size_t() for iexp, exp in enumerate(self._experiments): sel = self._reflections['id'] == iexp isel = sel.iselection() #refs = self._reflections.select(sel) nrefs = sample_size = len(isel) # set sample size according to nref_per_degree (per experiment) if exp.scan and self._nref_per_degree: sweep_range_rad = exp.scan.get_oscillation_range(deg=False) width = abs(sweep_range_rad[1] - sweep_range_rad[0]) * RAD2DEG sample_size = int(self._nref_per_degree * width) else: sweep_range_rad = None # adjust sample size if below the chosen limit sample_size = max(sample_size, self._min_sample_size) # set maximum sample size if requested if self._max_sample_size: sample_size = min(sample_size, self._max_sample_size) # determine subset and collect indices if sample_size < nrefs: isel = isel.select(flex.random_selection(nrefs, sample_size)) working_isel.extend(isel) # create subsets free_sel = flex.bool(len(self._reflections), True) free_sel.set_selected(working_isel, False) self._free_reflections = self._reflections.select(free_sel) self._reflections = self._reflections.select(working_isel) return
def __init__(self): from dials.algorithms.integration.profile import GridSampler2D from dials.array_family import flex from random import randint, uniform # Number of reflections nrefl = 1000 # Size of the images width = 1000 height = 1000 # Create the grid self.grid = GridSampler2D((width, height), (5,5)) # Create the list of xyz and bboxes self.xyz = flex.vec3_double(nrefl) self.bbox = flex.int6(nrefl) self.panel = flex.size_t(nrefl, 0) for i in range(nrefl): x0 = randint(0,width-10) x1 = x0 + randint(3,10) y0 = randint(0,height-10) y1 = y0 + randint(3,10) z0 = randint(0,10) z1 = z0 + randint(1,10) b = x0, x1, y0, y1, z0, z1 c = (x1 + x0) / 2, (y1 + y0) / 2, (z1 + z0) / 2 self.xyz[i] = c self.bbox[i] = b # Create the array of shoeboxes self.sbox = flex.shoebox(self.panel, self.bbox) self.sbox.allocate() for i in range(len(self.sbox)): data = self.sbox[i].data for j in range(len(data)): data[j] = uniform(0, 100)