def run(self): ''' Parse the options. ''' from dials.util.options import flatten_experiments, flatten_reflections # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) self.params = params experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) assert len(reflections) == len(experiments) == 1 reflections = reflections[0] exp = experiments[0] from dials.algorithms.indexing import index_reflections from dials.algorithms.indexing.indexer import indexer_base reflections['id'] = flex.int(len(reflections), -1) reflections['imageset_id'] = flex.int(len(reflections), 0) reflections = indexer_base.map_spots_pixel_to_mm_rad(reflections, exp.detector, exp.scan) indexer_base.map_centroids_to_reciprocal_space( reflections, exp.detector, exp.beam, exp.goniometer,) index_reflections(reflections, experiments, params.d_min, tolerance=0.3) indexed_reflections = reflections.select(reflections['miller_index'] != (0,0,0)) print "Indexed %d reflections out of %d"%(len(indexed_reflections), len(reflections)) easy_pickle.dump("indexedstrong.pickle", indexed_reflections)
def with_given_intensity(self, N, I, Ba, Bb, Bc, Bd): """ Generate reflections with a given intensity and background. """ from dials.array_family import flex return self.with_individual_given_intensity( N, flex.int(N, I), flex.int(N, Ba), flex.int(N, Bb), flex.int(N, Bc), flex.int(N, Bd) )
def tst_find_overlapping(self): from dials.array_family import flex from random import randint, uniform from dials.model.data import Shoebox N = 10000 r = flex.reflection_table(N) r['bbox'] = flex.int6(N) r['panel'] = flex.size_t(N) r['id'] = flex.int(N) r['imageset_id'] = flex.int(N) for i in range(N): x0 = randint(0, 100) x1 = randint(1, 10) + x0 y0 = randint(0, 100) y1 = randint(1, 10) + y0 z0 = randint(0, 100) z1 = randint(1, 10) + z0 panel = randint(0,2) pid = randint(0,2) r['bbox'][i] = (x0,x1,y0,y1,z0,z1) r['panel'][i] = panel r['id'][i] = pid r['imageset_id'][i] = pid def is_overlap(b0, b1, border): b0 = b0[0]-border,b0[1]+border,b0[2]-border,b0[3]+border,b0[4]-border,b0[5]+border b1 = b1[0]-border,b1[1]+border,b1[2]-border,b1[3]+border,b1[4]-border,b1[5]+border if not (b1[0] > b0[1] or b1[1] < b0[0] or b1[2] > b0[3] or b1[3] < b0[2] or b1[4] > b0[5] or b1[5] < b0[4]): return True return False for i in [0, 2, 5]: overlaps = r.find_overlaps(border=i) for item in overlaps.edges(): i0 = overlaps.source(item) i1 = overlaps.target(item) r0 = r[i0] r1 = r[i1] p0 = r0['panel'] p1 = r1['panel'] b0 = r0['bbox'] b1 = r1['bbox'] j0 = r0['imageset_id'] j1 = r1['imageset_id'] assert j0 == j1 assert p0 == p1 assert is_overlap(b0,b1,i) print 'OK'
def tst_split_blocks_1_frame(self): from dials.array_family import flex from random import randint, uniform, seed from dials.algorithms.integration.integrator import JobList r = flex.reflection_table() r['value1'] = flex.double() r['value2'] = flex.int() r['value3'] = flex.double() r['bbox'] = flex.int6() r['id'] = flex.int() expected = [] for i in range(100): x0 = randint(0, 100) x1 = x0 + randint(1, 10) y0 = randint(0, 100) y1 = y0 + randint(1, 10) z0 = randint(0, 100) z1 = z0 + randint(1, 10) v1 = uniform(0, 100) v2 = randint(0, 100) v3 = uniform(0, 100) r.append({ 'id' : 0, 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z0, z1) }) for z in range(z0, z1): expected.append({ 'id' : 0, 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z, z+1), 'partial_id' : i, }) jobs = JobList() jobs.add((0,1), (0, 111), 1) jobs.split(r) assert(len(r) == len(expected)) EPS = 1e-7 for r1, r2 in zip(r, expected): assert(r1['bbox'] == r2['bbox']) assert(r1['partial_id'] == r2['partial_id']) assert(abs(r1['value1'] - r2['value1']) < EPS) assert(r1['value2'] == r2['value2']) assert(abs(r1['value3'] - r2['value3']) < EPS) print 'OK'
def tst_copy(self): import copy from dials.array_family import flex # Create a table table = flex.reflection_table([ ('col1', flex.int(range(10)))]) # Make a shallow copy of the table shallow = copy.copy(table) shallow['col2'] = flex.double(range(10)) assert(table.ncols() == 2) assert(table.is_consistent()) print 'OK' # Make a deep copy of the table deep = copy.deepcopy(table) deep['col3'] = flex.std_string(10) assert(table.ncols() == 2) assert(deep.ncols() == 3) assert(table.is_consistent()) assert(deep.is_consistent()) table2 = table.copy() table2['col3'] = flex.std_string(10) assert(table.ncols() == 2) assert(table2.ncols() == 3) assert(table.is_consistent()) assert(table2.is_consistent()) print 'OK'
def tst_serialize(self): from dials.array_family import flex # The columns as lists c1 = list(range(10)) c2 = list(range(10)) c3 = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'i', 'j', 'k'] # Create a table with some elements table = flex.reflection_table() table['col1'] = flex.int(c1) table['col2'] = flex.double(c2) table['col3'] = flex.std_string(c3) # Pickle, then unpickle import cPickle as pickle obj = pickle.dumps(table) new_table = pickle.loads(obj) assert(new_table.is_consistent()) assert(new_table.nrows() == 10) assert(new_table.ncols() == 3) assert(all(a == b for a, b in zip(new_table['col1'], c1))) assert(all(a == b for a, b in zip(new_table['col2'], c2))) assert(all(a == b for a, b in zip(new_table['col3'], c3))) print 'OK'
def decode(self, handle): '''Decode the reflection data.''' from dials.array_family import flex # Get the group containing the reflection data g = handle['entry/data_processing'] # Create the list of reflections rl = flex.reflection_table(int(g.attrs['num_reflections'])) # Decode all the columns for key in g: item = g[key] name = item.attrs['flex_type'] if name == 'shoebox': flex_type = getattr(flex, name) data = item['data'] mask = item['mask'] background = item['background'] col = flex_type(len(rl)) for i in range(len(rl)): dd = data['%d' % i].value col[i].data = flex.double(data['%d' % i].value) col[i].mask = flex.int(mask['%d' % i].value) col[i].background = flex.double(background['%d' % i].value) else: flex_type = getattr(flex, name) col = self.decode_column(flex_type, item) rl[str(key)] = col # Return the list of reflections return rl
def filter_frames(self, data): data = data[0] lp = gaussian_filter(data, 100) hp = data - lp # poormans background subtraction hp -= np.min(hp) sh = hp.shape print "here" hp = hp.astype('uint32') hp = flex.int(hp) print "here now" mask = flex.bool(np.ones_like(hp).astype('bool')) print "here now" result1 = flex.bool(np.zeros_like(hp).astype('bool')) spots = np.zeros_like(hp).astype('bool') print "here now" for i in range(3, self.parameters['spotsize'], 5): print "here now" algorithm = DispersionThreshold(sh, (i, i), 1, 1, 0, -1) print "here now" print type(hp), type(mask), type(result1) thing = algorithm(hp, mask, result1) print "here now" spots = spots + result1.as_numpy_array() return [data, spots*data]
def tst_count_mask_values(self): from dials.model.data import Shoebox from random import randint, sample from dials.array_family import flex shoebox = flex.shoebox(10) num = flex.int(10) value = (1 << 2) for i in range(10): x0 = randint(0, 90) y0 = randint(0, 90) z0 = randint(0, 90) x1 = randint(1, 10) + x0 y1 = randint(1, 10) + y0 z1 = randint(1, 10) + z0 shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1)) shoebox[i].allocate() maxnum = len(shoebox[i].mask) num[i] = randint(1, maxnum) indices = sample(list(range(maxnum)), num[i]) for j in indices: shoebox[i].mask[j] = value assert(shoebox.count_mask_values(value) == num) # Test passed print 'OK'
def tst_for_dataset(self, creator, filename): from dials.array_family import flex from dials.algorithms.shoebox import MaskCode print filename rlist = flex.reflection_table.from_pickle(filename) shoebox = rlist['shoebox'] # FIXME doesn't work for single image zr = flex.int([s.bbox[5]-s.bbox[4] for s in shoebox]) rlist = rlist.select(zr > 1) shoebox = rlist['shoebox'] background = [sb.background.deep_copy() for sb in shoebox] success = creator(shoebox) assert(success.count(True) == len(success)) diff = [] for i in range(len(rlist)): mask = flex.bool([(m & MaskCode.Foreground) != 0 for m in shoebox[i].mask]) px1 = background[i].select(mask) px2 = shoebox[i].background.select(mask) den = max([flex.mean(px1), 1.0]) diff.append(flex.mean(px2 - px1) / den) diff = flex.double(diff) mv = flex.mean_and_variance(flex.double(diff)) mean = mv.mean() sdev = mv.unweighted_sample_standard_deviation() try: assert(abs(mean) < 0.01) except Exception: print "Mean: %f, Sdev: %f", mean, sdev #from matplotlib import pylab #pylab.hist(diff) #pylab.show() raise
def prepare_reflection_list(self,detector): spots = self.triclinic.get_observations_with_outlier_removal() ordinary_python_list_of_indexed_observations = [ { "id":0, "panel":0, "miller_index":item["pred"], "xyzobs.px.value":(spots[item["spot"]].ctr_mass_x(),spots[item["spot"]].ctr_mass_y(),0.0), "xyzobs.px.variance":(0.25,0.25,0.25), "spotfinder_lookup":item["spot"] } for item in self.triclinic_pairs ] self.length = len(ordinary_python_list_of_indexed_observations) R= flex.reflection_table.empty_standard(self.length) R['miller_index'] = flex.miller_index([item["miller_index"] for item in ordinary_python_list_of_indexed_observations]) R['xyzobs.px.value'] = flex.vec3_double([item["xyzobs.px.value"] for item in ordinary_python_list_of_indexed_observations]) R['xyzobs.px.variance'] = flex.vec3_double([item["xyzobs.px.variance"] for item in ordinary_python_list_of_indexed_observations]) R['spotfinder_lookup'] = flex.int([item["spotfinder_lookup"] for item in ordinary_python_list_of_indexed_observations]) R['xyzobs.mm.value'] = flex.vec3_double(self.length) R['xyzobs.mm.variance'] = flex.vec3_double(self.length) pxlsz = detector[0].get_pixel_size() for idx in xrange(self.length): R['xyzobs.mm.value'][idx] = (R['xyzobs.px.value'][idx][0]*pxlsz[0], R['xyzobs.px.value'][idx][1]*pxlsz[1], R['xyzobs.px.value'][idx][2]) R['xyzobs.mm.variance'][idx] = (R['xyzobs.px.variance'][idx][0]*pxlsz[0], R['xyzobs.px.variance'][idx][1]*pxlsz[1], R['xyzobs.px.variance'][idx][2]) return R
def generate_reflections(self): """Use reeke_model to generate indices of reflections near to the Ewald sphere that might be observed on a still image. Build a reflection_table of these.""" from cctbx.sgtbx import space_group_info space_group_type = space_group_info("P 1").group().type() # create a ReekeIndexGenerator UB = self.crystal.get_U() * self.crystal.get_B() axis = self.goniometer.get_rotation_axis() s0 = self.beam.get_s0() dmin = 1.5 # use the same UB at the beginning and end - the margin parameter ensures # we still have indices close to the Ewald sphere generated from dials.algorithms.spot_prediction import ReekeIndexGenerator r = ReekeIndexGenerator(UB, UB, space_group_type, axis, s0, dmin=1.5, margin=1) # generate indices hkl = r.to_array() nref = len(hkl) # create a reflection table from dials.array_family import flex table = flex.reflection_table() table['flags'] = flex.size_t(nref, 0) table['id'] = flex.int(nref, 0) table['panel'] = flex.size_t(nref, 0) table['miller_index'] = flex.miller_index(hkl) table['entering'] = flex.bool(nref, True) table['s1'] = flex.vec3_double(nref) table['xyzcal.mm'] = flex.vec3_double(nref) table['xyzcal.px'] = flex.vec3_double(nref) return table
def predict(self): # Populate the reflection table with predictions self.predicted = flex.reflection_table.from_predictions( self.experiment, force_static=True, dmin=self.d_min) self.predicted['id'] = flex.int(len(self.predicted), 0)
def plot_scale_vs_x_y(self): from scitbx.array_family import flex from math import ceil print "Getting scale" points = [(int(xyz[1] / 8), int(xyz[0] / 8)) for xyz in self.xyz] scale = [x / d for x, d in zip(self.i_xds, self.i_dials)] print "Creating Grid" image_size = self.sweep.get_detector()[0].get_image_size()[::-1] image_size = (int(ceil(image_size[0] / 8)), int(ceil(image_size[1] / 8))) grid = flex.double(flex.grid(image_size)) count = flex.int(flex.grid(image_size)) for p, s in zip(points, scale): grid[p] += s count[p] += 1 for i in range(len(grid)): if count[i] > 0: grid[i] /= count[i] # grid_points = [(j,i) for j in range(image_size[0]) for i in range(image_size[1])] # grid = griddata(points, scale, grid_points) # grid.shape = image_size from matplotlib import pyplot fig, ax = pyplot.subplots() pyplot.title("scale vs x/y") cax = pyplot.imshow(grid.as_numpy_array()) cbar = fig.colorbar(cax) pyplot.savefig("plot-scale-vs-xy.png") pyplot.close()
def spot_count_per_panel(self, rlist): ''' Analyse the spot count per panel. ''' from os.path import join panel = rlist['panel'] if flex.max(panel) == 0: # only one panel, don't bother generating a plot return n_panels = int(flex.max(panel)) spot_count_per_panel = flex.int() for i in range(n_panels): sel = (panel >= i) & (panel < (i+1)) spot_count_per_panel.append(sel.count(True)) from matplotlib import pyplot fig = pyplot.figure() ax = fig.add_subplot(111) ax.set_title("Spot count per panel") ax.scatter( list(range(len(spot_count_per_panel))), spot_count_per_panel, s=10, color='blue', marker='o', alpha=0.4) ax.set_xlabel("Panel #") ax.set_ylabel("# spots") pyplot.savefig(join(self.directory, "spots_per_panel.png")) pyplot.close()
def spot_count_per_image(self, rlist): ''' Analyse the spot count per image. ''' from os.path import join x,y,z = rlist['xyzobs.px.value'].parts() max_z = int(math.ceil(flex.max(z))) ids = rlist['id'] spot_count_per_image = [] for j in range(flex.max(ids)+1): spot_count_per_image.append(flex.int()) zsel = z.select(ids == j) for i in range(max_z): sel = (zsel >= i) & (zsel < (i+1)) spot_count_per_image[j].append(sel.count(True)) colours = ['blue', 'red', 'green', 'orange', 'purple', 'black'] * 10 assert len(spot_count_per_image) <= colours from matplotlib import pyplot fig = pyplot.figure() ax = fig.add_subplot(111) ax.set_title("Spot count per image") for j in range(len(spot_count_per_image)): ax.scatter( list(range(len(spot_count_per_image[j]))), spot_count_per_image[j], s=5, color=colours[j], marker='o', alpha=0.4) ax.set_xlabel("Image #") ax.set_ylabel("# spots") pyplot.savefig(join(self.directory, "spots_per_image.png")) pyplot.close()
def run(self): from dials.algorithms.profile_model.modeller import ProfileModellerIface from dials.algorithms.profile_model.modeller import MultiExpProfileModeller from dials.array_family import flex class Modeller(ProfileModellerIface): def __init__(self, index, expected): self.index = index self.accumulated = False self.finalized = False self.expected = expected super(Modeller, self).__init__() def model(self, reflections): assert(reflections['id'].all_eq(self.index)) assert(len(reflections) == self.expected) def accumulate(self, other): self.accumulated = True assert(self.index == other.index) def finalize(self): assert(self.accumulated == True) self.finalized = True # The expected number of reflections expected = [100, 200, 300, 400, 500] # Create some reflections reflections = flex.reflection_table() reflections["id"] = flex.int() for idx in range(len(expected)): for n in range(expected[idx]): reflections.append({ "id" : idx }) # Create two modellers modeller1 = MultiExpProfileModeller() modeller2 = MultiExpProfileModeller() for idx in range(len(expected)): modeller1.add(Modeller(idx, expected[idx])) modeller2.add(Modeller(idx, expected[idx])) # Model the reflections modeller1.model(reflections) modeller2.model(reflections) # Accumulate modeller1.accumulate(modeller2) # Finalize modeller1.finalize() # Check finalized assert(modeller1.finalized) # Test passed print 'OK'
def run(self): from dials.array_family import flex #self.test_for_reference() for filename in self.refl_filenames: refl = flex.reflection_table.from_pickle(filename) refl.compute_partiality(self.experiments) refl['id'] = flex.int(len(refl),0) self.test_for_reflections(refl, filename)
def plot_multirun_stats(runs, run_numbers, d_min, ratio_cutoff=1, n_strong_cutoff=40, run_tags=[], run_statuses=[], interactive=False, compress_runs=True, xsize=30, ysize=10, high_vis=False): tset = flex.double() two_theta_low_set = flex.double() two_theta_high_set = flex.double() nset = flex.int() I_sig_I_low_set = flex.double() I_sig_I_high_set = flex.double() boundaries = [] lengths = [] runs_with_data = [] offset = 0 for idx in xrange(len(runs)): r = runs[idx] if len(r[0]) > 0: if compress_runs: tslice = r[0] - r[0][0] + offset offset += (r[0][-1] - r[0][0]) else: tslice = r[0] last_end = r[0][-1] tset.extend(tslice) two_theta_low_set.extend(r[1]) two_theta_high_set.extend(r[2]) nset.extend(r[3]) I_sig_I_low_set.extend(r[4]) I_sig_I_high_set.extend(r[5]) boundaries.append(tslice[0]) boundaries.append(tslice[-1]) lengths.append(len(tslice)) runs_with_data.append(run_numbers[idx]) else: boundaries.extend([None]*2) stats_tuple = get_run_stats(tset, two_theta_low_set, two_theta_high_set, nset, I_sig_I_low_set, I_sig_I_high_set, tuple(boundaries), tuple(lengths), runs_with_data, ratio_cutoff=ratio_cutoff, n_strong_cutoff=n_strong_cutoff) png = plot_run_stats(stats_tuple, d_min, run_tags=run_tags, run_statuses=run_statuses, interactive=interactive, xsize=xsize, ysize=ysize, high_vis=high_vis) return png
def __call__(self, params, options): ''' Import the integrate.hkl file. ''' from iotbx.xds import integrate_hkl from dials.array_family import flex from dials.util.command_line import Command from cctbx import sgtbx # Get the unit cell to calculate the resolution uc = self._experiment.crystal.get_unit_cell() # Read the INTEGRATE.HKL file Command.start('Reading INTEGRATE.HKL') handle = integrate_hkl.reader() handle.read_file(self._integrate_hkl) hkl = flex.miller_index(handle.hkl) xyzcal = flex.vec3_double(handle.xyzcal) xyzobs = flex.vec3_double(handle.xyzobs) iobs = flex.double(handle.iobs) sigma = flex.double(handle.sigma) rlp = flex.double(handle.rlp) peak = flex.double(handle.peak) * 0.01 Command.end('Read %d reflections from INTEGRATE.HKL file.' % len(hkl)) # Derive the reindex matrix rdx = self.derive_reindex_matrix(handle) print 'Reindex matrix:\n%d %d %d\n%d %d %d\n%d %d %d' % (rdx.elems) # Reindex the reflections Command.start('Reindexing reflections') cb_op = sgtbx.change_of_basis_op(sgtbx.rt_mx(sgtbx.rot_mx(rdx.elems))) hkl = cb_op.apply(hkl) Command.end('Reindexed %d reflections' % len(hkl)) # Create the reflection list Command.start('Creating reflection table') table = flex.reflection_table() table['id'] = flex.int(len(hkl), 0) table['panel'] = flex.size_t(len(hkl), 0) table['miller_index'] = hkl table['xyzcal.px'] = xyzcal table['xyzobs.px.value'] = xyzobs table['intensity.cor.value'] = iobs table['intensity.cor.variance'] = sigma**2 table['intensity.prf.value'] = iobs * peak / rlp table['intensity.prf.variance'] = (sigma * peak / rlp)**2 table['lp'] = 1.0 / rlp table['d'] = flex.double(uc.d(h) for h in hkl) Command.end('Created table with {0} reflections'.format(len(table))) # Output the table to pickle file if params.output.filename is None: params.output.filename = 'integrate_hkl.pickle' Command.start('Saving reflection table to %s' % params.output.filename) table.as_pickle(params.output.filename) Command.end('Saved reflection table to %s' % params.output.filename)
def _predict_reflections(self, params, experiments): ''' Predict all the reflections. ''' from dials.array_family import flex n_sigma = params.integration.shoebox.n_sigma result = flex.reflection_table() for i, experiment in enumerate(experiments): predicted = flex.reflection_table.from_predictions(experiment) predicted['id'] = flex.int(len(predicted), i) result.extend(predicted) return result
def tst_select(self): from dials.array_family import flex # The columns as lists c1 = list(range(10)) c2 = list(range(10)) c3 = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'i', 'j', 'k'] # Create a table with some elements table = flex.reflection_table() table['col1'] = flex.int(c1) table['col2'] = flex.double(c2) table['col3'] = flex.std_string(c3) # Select some columns new_table = table.select(('col1', 'col2')) assert(new_table.nrows() == 10) assert(new_table.ncols() == 2) assert(all(a == b for a, b in zip(new_table['col1'], c1))) assert(all(a == b for a, b in zip(new_table['col2'], c2))) print 'OK' # Select some columns new_table = table.select(flex.std_string(['col1', 'col2'])) assert(new_table.nrows() == 10) assert(new_table.ncols() == 2) assert(all(a == b for a, b in zip(new_table['col1'], c1))) assert(all(a == b for a, b in zip(new_table['col2'], c2))) print 'OK' # Select some rows index = flex.size_t([0, 1, 5, 8, 9]) cc1 = [c1[i] for i in index] cc2 = [c2[i] for i in index] cc3 = [c3[i] for i in index] new_table = table.select(index) assert(new_table.nrows() == 5) assert(new_table.ncols() == 3) assert(all(a == b for a, b in zip(new_table['col1'], cc1))) assert(all(a == b for a, b in zip(new_table['col2'], cc2))) assert(all(a == b for a, b in zip(new_table['col3'], cc3))) print 'OK' # Select some rows index = flex.bool([True, True, False, False, False, True, False, False, True, True]) new_table = table.select(index) assert(new_table.nrows() == 5) assert(new_table.ncols() == 3) assert(all(a == b for a, b in zip(new_table['col1'], cc1))) assert(all(a == b for a, b in zip(new_table['col2'], cc2))) assert(all(a == b for a, b in zip(new_table['col3'], cc3))) print 'OK'
def build_np_img(nrow=64, ncol=64): data2d = flex.double(flex.grid(nrow, ncol), 0) mask2d = flex.int(flex.grid(nrow, ncol), 0) for x in range(nrow): for y in range(ncol): data2d[x, y] += (x * 1.00000000001 + y * 2.5555555555555) * 0.00000000001 #print "number to see(aprox) =", data2d[x, y] return data2d, mask2d
def __call__(self, params, options): ''' Import the spot.xds file. ''' from iotbx.xds import spot_xds from dials.util.command_line import Command from dials.array_family import flex # Read the SPOT.XDS file Command.start('Reading SPOT.XDS') handle = spot_xds.reader() handle.read_file(self._spot_xds) centroid = handle.centroid intensity = handle.intensity try: miller_index = handle.miller_index except AttributeError: miller_index = None Command.end('Read {0} spots from SPOT.XDS file.'.format(len(centroid))) # Create the reflection list Command.start('Creating reflection list') table = flex.reflection_table() table['id'] = flex.int(len(centroid), 0) table['panel'] = flex.size_t(len(centroid), 0) if miller_index: table['miller_index'] = flex.miller_index(miller_index) table['xyzobs.px.value'] = flex.vec3_double(centroid) table['intensity.sum.value'] = flex.double(intensity) Command.end('Created reflection list') # Remove invalid reflections Command.start('Removing invalid reflections') if miller_index and params.remove_invalid: flags = flex.bool([h != (0, 0, 0) for h in table['miller_index']]) table = table.select(flags) Command.end('Removed invalid reflections, %d remaining' % len(table)) # Fill empty standard columns if params.add_standard_columns: Command.start('Adding standard columns') rt = flex.reflection_table.empty_standard(len(table)) rt.update(table) table = rt # set variances to unity table['xyzobs.mm.variance'] = flex.vec3_double(len(table), (1,1,1)) table['xyzobs.px.variance'] = flex.vec3_double(len(table), (1,1,1)) Command.end('Standard columns added') # Output the table to pickle file if params.output.filename is None: params.output.filename = 'spot_xds.pickle' Command.start('Saving reflection table to %s' % params.output.filename) table.as_pickle(params.output.filename) Command.end('Saved reflection table to %s' % params.output.filename)
def tst_updating(self): from dials.array_family import flex # The columns as lists c1 = list(range(10)) c2 = list(range(10)) c3 = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'i', 'j', 'k'] # Create a table with some elements table0 = flex.reflection_table() table1 = flex.reflection_table() table2 = flex.reflection_table() table1['col1'] = flex.int(c1) table1['col2'] = flex.double(c2) table2['col3'] = flex.std_string(c3) # Update from zero columns table0.update(table1) assert(table0.is_consistent()) assert(table0.nrows() == 10) assert(table0.ncols() == 2) print 'OK' # Update table1 with table2 columns table1.update(table2) assert(table1.is_consistent()) assert(table1.nrows() == 10) assert(table1.ncols() == 3) assert(table2.is_consistent()) assert(table2.nrows() == 10) assert(table2.ncols() == 1) print 'OK' # Update trable1 with invalid table c3 = ['a', 'b', 'c'] # Create a table with some elements table2 = flex.reflection_table() table2['col3'] = flex.std_string(c3) try: table1.update(table2) assert(False) except Exception: pass assert(table1.is_consistent()) assert(table1.nrows() == 10) assert(table1.ncols() == 3) assert(table2.is_consistent()) assert(table2.nrows() == 3) assert(table2.ncols() == 1) print 'OK'
def tst_init(self): from dials.array_family import flex # test default table = flex.reflection_table() assert(table.is_consistent()) assert(table.nrows() == 0) assert(table.ncols() == 0) assert(table.empty()) print 'Ok' # test with nrows table = flex.reflection_table(10) assert(table.is_consistent()) assert(table.nrows() == 10) assert(table.ncols() == 0) assert(table.empty()) print 'OK' # test with valid columns table = flex.reflection_table([ ('col1', flex.int(10)), ('col2', flex.double(10)), ('col3', flex.std_string(10))]) assert(table.is_consistent()) assert(table.nrows() == 10) assert(table.ncols() == 3) assert(not table.empty()) print 'OK' # test with invalid columns try: table = flex.reflection_table([ ('col1', flex.int(10)), ('col2', flex.double(20)), ('col3', flex.std_string(10))]) assert(false) except Exception: pass print 'OK'
def _filter_by_distance(self, nn, dist): ''' Filter the matches by distance. :param nn: The nearest neighbour list :param dist: The distances :returns: A reduced list of nearest neighbours ''' from scitbx.array_family import flex index = range(len(nn)) return flex.int([i for i in index if dist[i] <= self._max_separation])
def tst_delete(self): from dials.array_family import flex # Test del item table = flex.reflection_table() table['col1'] = flex.int([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) table['col2'] = flex.int([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) table['col3'] = flex.int([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) del table['col3'] assert(table.is_consistent()) assert(table.nrows() == 10) assert(table.ncols() == 2) assert(not "col3" in table) print 'OK' # Test del row del table[5] assert(table.is_consistent()) assert(table.nrows() == 9) assert(table.ncols() == 2) assert(all(a==b for a, b in zip(list(table['col1']), [0, 1, 2, 3, 4, 6, 7, 8, 9]))) print 'OK' # Test del slice del table[0:10:2] assert(table.is_consistent()) assert(table.nrows() == 4) assert(table.ncols() == 2) assert(all(a==b for a, b in zip(list(table['col1']), [1, 3, 6, 8]))) print 'OK' # Test del slice del table[:] assert(table.is_consistent()) assert(table.nrows() == 0) assert(table.ncols() == 2) print 'OK'
def __init__(self, experiment, reflections, expected_miller_indices): from dials.algorithms.indexing \ import index_reflections, index_reflections_local import copy # index reflections using simple "global" method self.reflections_global = copy.deepcopy(reflections) self.reflections_global['id'] = flex.int(len(self.reflections_global), -1) self.reflections_global['imageset_id'] = flex.int(len(self.reflections_global), 0) index_reflections( self.reflections_global, ExperimentList([experiment])) non_zero_sel = (self.reflections_global['miller_index'] != (0,0,0)) assert self.reflections_global['id'].select(~non_zero_sel).all_eq(-1) self.misindexed_global = ( expected_miller_indices == self.reflections_global['miller_index']).select( non_zero_sel).count(False) self.correct_global = ( expected_miller_indices == self.reflections_global['miller_index']).count(True) # index reflections using xds-style "local" method self.reflections_local = copy.deepcopy(reflections) self.reflections_local['id'] = flex.int(len(self.reflections_local), -1) index_reflections_local( self.reflections_local, ExperimentList([experiment])) non_zero_sel = (self.reflections_local['miller_index'] != (0,0,0)) assert self.reflections_local['id'].select(~non_zero_sel).all_eq(-1) self.misindexed_local = ( expected_miller_indices == self.reflections_local['miller_index']).select( non_zero_sel).count(False) self.correct_local = ( expected_miller_indices == self.reflections_local['miller_index']).count(True) print self.misindexed_global, self.correct_global, len(self.reflections_global) print self.misindexed_local, self.correct_local, len(self.reflections_local)
def __call__(self, datablock): ''' Do the spot finding. :param datablock: The datablock to process :return: The observed spots ''' from dials.array_family import flex import cPickle as pickle # Loop through all the imagesets and find the strong spots reflections = flex.reflection_table() for i, imageset in enumerate(datablock.extract_imagesets()): # Find the strong spots in the sweep logger.info('-' * 80) logger.info('Finding strong spots in imageset %d' % i) logger.info('-' * 80) logger.info('') table, hot_mask = self._find_spots_in_imageset(imageset) table['id'] = flex.int(table.nrows(), i) reflections.extend(table) # Write a hot pixel mask if self.write_hot_mask: if imageset.external_lookup.mask.data is not None: and_mask = [] for m1, m2 in zip(imageset.external_lookup.mask.data, hot_mask): and_mask.append(m1 & m2) imageset.external_lookup.mask.data = tuple(and_mask) else: imageset.external_lookup.mask.data = hot_mask imageset.external_lookup.mask.filename = "hot_mask_%d.pickle" % i # Write the hot mask with open(imageset.external_lookup.mask.filename, "wb") as outfile: pickle.dump(hot_mask, outfile, protocol=pickle.HIGHEST_PROTOCOL) # Set the strong spot flag reflections.set_flags( flex.size_t_range(len(reflections)), reflections.flags.strong) # Check for overloads reflections.is_overloaded(datablock) # Return the reflections return reflections
def write(handle, key, data): from dials.array_family import flex if key == 'miller_index': col1, col2, col3 = zip(*list(data)) col1 = flex.int(col1) col2 = flex.int(col2) col3 = flex.int(col3) dsc1 = 'The h component of the miller index' dsc2 = 'The k component of the miller index' dsc3 = 'The l component of the miller index' make_int(handle, "h", col1, dsc1) make_int(handle, "k", col2, dsc2) make_int(handle, "l", col3, dsc3) elif key == 'id': col = data dsc = 'The experiment id' make_int(handle, "id", col, dsc) elif key == 'partial_id': col = data desc = 'The reflection id' make_uint(handle, "reflection_id", col, desc) elif key == 'entering': col = data dsc = 'Entering or exiting the Ewald sphere' make_bool(handle, 'entering', col, dsc) elif key == 'flags': col = data dsc = 'Status of the reflection in processing' make_uint(handle, 'flags', col, dsc) elif key == 'panel': col = data dsc = 'The detector module on which the reflection was recorded' make_uint(handle, 'det_module', col, dsc) elif key == 'd': col = data dsc = 'The resolution of the reflection' make_float(handle, 'd', col, dsc) elif key == 'partiality': col = data dsc = 'The partiality of the reflection' make_float(handle, 'partiality', col, dsc) elif key == 'xyzcal.px': col1, col2, col3 = data.parts() dsc1 = 'The predicted bragg peak fast pixel location' dsc2 = 'The predicted bragg peak slow pixel location' dsc3 = 'The predicted bragg peak frame number' make_float(handle, 'predicted_px_x', col1, dsc1) make_float(handle, 'predicted_px_y', col2, dsc2) make_float(handle, 'predicted_frame', col3, dsc3) elif key == 'xyzcal.mm': col1, col2, col3 = data.parts() dsc1 = 'The predicted bragg peak fast millimeter location' dsc2 = 'The predicted bragg peak slow millimeter location' dsc3 = 'The predicted bragg peak rotation angle number' make_float(handle, 'predicted_x', col1, dsc1, units='mm') make_float(handle, 'predicted_y', col2, dsc2, units='mm') make_float(handle, 'predicted_phi', col3, dsc3, units='rad') elif key == 'bbox': d = data.as_int() d.reshape(flex.grid((len(data)), 6)) make_int(handle, 'bounding_box', d, 'The reflection bounding box') elif key == 'xyzobs.px.value': col1, col2, col3 = data.parts() dsc1 = 'The observed centroid fast pixel value' dsc2 = 'The observed centroid slow pixel value' dsc3 = 'The observed centroid frame value' make_float(handle, 'observed_px_x', col1, dsc1) make_float(handle, 'observed_px_y', col2, dsc2) make_float(handle, 'observed_frame', col3, dsc3) elif key == 'xyzobs.px.variance': col1, col2, col3 = data.parts() dsc1 = 'The observed centroid fast pixel variance' dsc2 = 'The observed centroid slow pixel variance' dsc3 = 'The observed centroid frame variance' make_float(handle, 'observed_px_x_var', col1, dsc1) make_float(handle, 'observed_px_y_var', col2, dsc2) make_float(handle, 'observed_frame_var', col3, dsc3) elif key == 'xyzobs.mm.value': col1, col2, col3 = data.parts() dsc1 = 'The observed centroid fast pixel value' dsc2 = 'The observed centroid slow pixel value' dsc3 = 'The observed centroid phi value' make_float(handle, 'observed_x', col1, dsc1, units='mm') make_float(handle, 'observed_y', col2, dsc2, units='mm') make_float(handle, 'observed_phi', col3, dsc3, units='rad') elif key == 'xyzobs.mm.variance': col1, col2, col3 = data.parts() dsc1 = 'The observed centroid fast pixel variance' dsc2 = 'The observed centroid slow pixel variance' dsc3 = 'The observed centroid phi variance' make_float(handle, 'observed_x_var', col1, dsc1, units='mm') make_float(handle, 'observed_y_var', col2, dsc2, units='mm') make_float(handle, 'observed_phi_var', col3, dsc3, units='rad') elif key == 'background.mean': col = data dsc = 'The mean background value' make_float(handle, 'background_mean', col, dsc) elif key == 'intensity.sum.value': col = data dsc = 'The value of the summed intensity' make_float(handle, 'int_sum', col, dsc) elif key == 'intensity.sum.variance': col = data dsc = 'The variance of the summed intensity' make_float(handle, 'int_sum_var', col, dsc) elif key == 'intensity.prf.value': col = data dsc = 'The value of the profile fitted intensity' make_float(handle, 'int_prf', col, dsc) elif key == 'intensity.prf.variance': col = data dsc = 'The variance of the profile fitted intensity' make_float(handle, 'int_prf_var', col, dsc) elif key == 'profile.correlation': col = data dsc = 'Profile fitting correlations' make_float(handle, 'prf_cc', col, dsc) elif key == 'lp': col = data dsc = 'The lorentz-polarization correction factor' make_float(handle, 'lp', col, dsc) elif key == 'num_pixels.background': col = data dsc = 'Number of background pixels' make_int(handle, 'num_bg', col, dsc) elif key == 'num_pixels.background_used': col = data dsc = 'Number of background pixels used' make_int(handle, 'num_bg_used', col, dsc) elif key == 'num_pixels.foreground': col = data dsc = 'Number of foreground pixels' make_int(handle, 'num_fg', col, dsc) elif key == 'num_pixels.valid': col = data dsc = 'Number of valid pixels' make_int(handle, 'num_valid', col, dsc) elif key == 'profile.rmsd': col = data dsc = 'Profile rmsd' make_float(handle, 'prf_rmsd', col, dsc) else: raise KeyError('Column %s not written to file' % key)
def run(self, experiments, reflections): self.logger.log_step_time("POSTREFINEMENT") if not self.params.postrefinement.enable: self.logger.log("Postrefinement was not done") if self.mpi_helper.rank == 0: self.logger.main_log("Postrefinement was not done") return experiments, reflections target_symm = symmetry( unit_cell=self.params.scaling.unit_cell, space_group_info=self.params.scaling.space_group) i_model = self.params.scaling.i_model miller_set = self.params.scaling.miller_set # Ensure that match_multi_indices() will return identical results # when a frame's observations are matched against the # pre-generated Miller set, self.miller_set, and the reference # data set, self.i_model. The implication is that the same match # can be used to map Miller indices to array indices for intensity # accumulation, and for determination of the correlation # coefficient in the presence of a scaling reference. assert len(i_model.indices()) == len(miller_set.indices()) assert (i_model.indices() == miller_set.indices()).count(False) == 0 new_experiments = ExperimentList() new_reflections = flex.reflection_table() experiments_rejected_by_reason = {} # reason:how_many_rejected for experiment in experiments: exp_reflections = reflections.select( reflections['exp_id'] == experiment.identifier) # Build a miller array for the experiment reflections with original miller indexes exp_miller_indices_original = miller.set( target_symm, exp_reflections['miller_index'], not self.params.merging.merge_anomalous) observations_original_index = miller.array( exp_miller_indices_original, exp_reflections['intensity.sum.value'], flex.double( flex.sqrt(exp_reflections['intensity.sum.variance']))) assert exp_reflections.size() == exp_miller_indices_original.size() assert observations_original_index.size( ) == exp_miller_indices_original.size() # Build a miller array for the experiment reflections with asu miller indexes exp_miller_indices_asu = miller.set( target_symm, exp_reflections['miller_index_asymmetric'], True) observations = miller.array( exp_miller_indices_asu, exp_reflections['intensity.sum.value'], flex.double( flex.sqrt(exp_reflections['intensity.sum.variance']))) matches = miller.match_multi_indices( miller_indices_unique=miller_set.indices(), miller_indices=observations.indices()) pair1 = flex.int([pair[1] for pair in matches.pairs() ]) # refers to the observations pair0 = flex.int([pair[0] for pair in matches.pairs() ]) # refers to the model assert exp_reflections.size() == exp_miller_indices_original.size() assert observations_original_index.size( ) == exp_miller_indices_original.size() # narrow things down to the set that matches, only observations_pair1_selected = observations.customized_copy( indices=flex.miller_index( [observations.indices()[p] for p in pair1]), data=flex.double([observations.data()[p] for p in pair1]), sigmas=flex.double([observations.sigmas()[p] for p in pair1])) observations_original_index_pair1_selected = observations_original_index.customized_copy( indices=flex.miller_index( [observations_original_index.indices()[p] for p in pair1]), data=flex.double( [observations_original_index.data()[p] for p in pair1]), sigmas=flex.double( [observations_original_index.sigmas()[p] for p in pair1])) I_observed = observations_pair1_selected.data() MILLER = observations_original_index_pair1_selected.indices() ORI = crystal_orientation(experiment.crystal.get_A(), basis_type.reciprocal) Astar = matrix.sqr(ORI.reciprocal_matrix()) Astar_from_experiment = matrix.sqr(experiment.crystal.get_A()) assert Astar == Astar_from_experiment WAVE = experiment.beam.get_wavelength() BEAM = matrix.col((0.0, 0.0, -1. / WAVE)) BFACTOR = 0. MOSAICITY_DEG = experiment.crystal.get_half_mosaicity_deg() DOMAIN_SIZE_A = experiment.crystal.get_domain_size_ang() # calculation of correlation here I_reference = flex.double( [i_model.data()[pair[0]] for pair in matches.pairs()]) I_invalid = flex.bool( [i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()]) use_weights = False # New facility for getting variance-weighted correlation if use_weights: # variance weighting I_weight = flex.double([ 1. / (observations_pair1_selected.sigmas()[pair[1]])**2 for pair in matches.pairs() ]) else: I_weight = flex.double( len(observations_pair1_selected.sigmas()), 1.) I_weight.set_selected(I_invalid, 0.) """Explanation of 'include_negatives' semantics as originally implemented in cxi.merge postrefinement: include_negatives = True + and - reflections both used for Rh distribution for initial estimate of RS parameter + and - reflections both used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) include_negatives = False + and - reflections both used for Rh distribution for initial estimate of RS parameter + reflections only used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) """ # RB: By design, for MPI-Merge "include negatives" is implicitly True SWC = simple_weighted_correlation(I_weight, I_reference, I_observed) if self.params.output.log_level == 0: self.logger.log("Old correlation is: %f" % SWC.corr) if self.params.postrefinement.algorithm == "rs": Rhall = flex.double() for mill in MILLER: H = matrix.col(mill) Xhkl = Astar * H Rh = (Xhkl + BEAM).length() - (1. / WAVE) Rhall.append(Rh) Rs = math.sqrt(flex.mean(Rhall * Rhall)) RS = 1. / 10000. # reciprocal effective domain size of 1 micron RS = Rs # try this empirically determined approximate, monochrome, a-mosaic value current = flex.double([SWC.slope, BFACTOR, RS, 0., 0.]) parameterization_class = rs_parameterization refinery = rs_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC=I_reference, IOBSVEC=I_observed) elif self.params.postrefinement.algorithm == "eta_deff": eta_init = 2. * MOSAICITY_DEG * math.pi / 180. D_eff_init = 2. * DOMAIN_SIZE_A current = flex.double( [SWC.slope, BFACTOR, eta_init, 0., 0., D_eff_init]) parameterization_class = eta_deff_parameterization refinery = eta_deff_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC=I_reference, IOBSVEC=I_observed) func = refinery.fvec_callable(parameterization_class(current)) functional = flex.sum(func * func) if self.params.output.log_level == 0: self.logger.log("functional: %f" % functional) self.current = current self.parameterization_class = parameterization_class self.refinery = refinery self.observations_pair1_selected = observations_pair1_selected self.observations_original_index_pair1_selected = observations_original_index_pair1_selected error_detected = False try: self.run_plain() result_observations_original_index, result_observations, result_matches = self.result_for_cxi_merge( ) assert result_observations_original_index.size( ) == result_observations.size() assert result_matches.pairs().size( ) == result_observations_original_index.size() except (AssertionError, ValueError, RuntimeError) as e: error_detected = True reason = repr(e) if not reason: reason = "Unknown error" if not reason in experiments_rejected_by_reason: experiments_rejected_by_reason[reason] = 1 else: experiments_rejected_by_reason[reason] += 1 if not error_detected: new_experiments.append(experiment) new_exp_reflections = flex.reflection_table() new_exp_reflections[ 'miller_index_asymmetric'] = flex.miller_index( result_observations.indices()) new_exp_reflections['intensity.sum.value'] = flex.double( result_observations.data()) new_exp_reflections['intensity.sum.variance'] = flex.double( flex.pow(result_observations.sigmas(), 2)) new_exp_reflections['exp_id'] = flex.std_string( len(new_exp_reflections), experiment.identifier) new_reflections.extend(new_exp_reflections) ''' # debugging elif reason.startswith("ValueError"): self.logger.log("Rejected b/c of value error exp id: %s; unit cell: %s"%(exp_id, str(experiment.crystal.get_unit_cell())) ) ''' # report rejected experiments, reflections experiments_rejected_by_postrefinement = len(experiments) - len( new_experiments) reflections_rejected_by_postrefinement = reflections.size( ) - new_reflections.size() self.logger.log("Experiments rejected by post-refinement: %d" % experiments_rejected_by_postrefinement) self.logger.log("Reflections rejected by post-refinement: %d" % reflections_rejected_by_postrefinement) all_reasons = [] for reason, count in six.iteritems(experiments_rejected_by_reason): self.logger.log("Experiments rejected due to %s: %d" % (reason, count)) all_reasons.append(reason) comm = self.mpi_helper.comm MPI = self.mpi_helper.MPI # Collect all rejection reasons from all ranks. Use allreduce to let each rank have all reasons. all_reasons = comm.allreduce(all_reasons, MPI.SUM) all_reasons = set(all_reasons) # Now that each rank has all reasons from all ranks, we can treat the reasons in a uniform way. total_experiments_rejected_by_reason = {} for reason in all_reasons: rejected_experiment_count = 0 if reason in experiments_rejected_by_reason: rejected_experiment_count = experiments_rejected_by_reason[ reason] total_experiments_rejected_by_reason[reason] = comm.reduce( rejected_experiment_count, MPI.SUM, 0) total_accepted_experiment_count = comm.reduce(len(new_experiments), MPI.SUM, 0) # how many reflections have we rejected due to post-refinement? rejected_reflections = len(reflections) - len(new_reflections) total_rejected_reflections = self.mpi_helper.sum(rejected_reflections) if self.mpi_helper.rank == 0: for reason, count in six.iteritems( total_experiments_rejected_by_reason): self.logger.main_log( "Total experiments rejected due to %s: %d" % (reason, count)) self.logger.main_log("Total experiments accepted: %d" % total_accepted_experiment_count) self.logger.main_log( "Total reflections rejected due to post-refinement: %d" % total_rejected_reflections) self.logger.log_step_time("POSTREFINEMENT", True) return new_experiments, new_reflections
def run(args): import libtbx.load_env from dials.util import Sorry usage = "dials.reindex [options] indexed.expt indexed.refl" parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, options = parser.parse_args(show_diff_phil=True) reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) if len(experiments) == 0 and len(reflections) == 0: parser.print_help() return if params.change_of_basis_op is None: raise Sorry("Please provide a change_of_basis_op.") reference_crystal = None if params.reference.experiments is not None: from dxtbx.serialize import load reference_experiments = load.experiment_list( params.reference.experiments, check_format=False) assert len(reference_experiments.crystals()) == 1 reference_crystal = reference_experiments.crystals()[0] if params.reference.reflections is not None: # First check that we have everything as expected for the reference reindexing # Currently only supports reindexing one dataset at a time if params.reference.experiments is None: raise Sorry( """For reindexing against a reference dataset, a reference experiments file must also be specified with the option: reference= """) if not os.path.exists(params.reference.reflections): raise Sorry("Could not locate reference dataset reflection file") if len(experiments) != 1 or len(reflections) != 1: raise Sorry( "Only one dataset can be reindexed to a reference at a time") reference_reflections = flex.reflection_table().from_file( params.reference.reflections) test_reflections = reflections[0] if (reference_crystal.get_space_group().type().number() != experiments.crystals()[0].get_space_group().type().number()): raise Sorry("Space group of input does not match reference") # Set some flags to allow filtering, if wanting to reindex against # reference with data that has not yet been through integration if (test_reflections.get_flags( test_reflections.flags.integrated_sum).count(True) == 0): assert ( "intensity.sum.value" in test_reflections), "No 'intensity.sum.value' in reflections" test_reflections.set_flags( flex.bool(test_reflections.size(), True), test_reflections.flags.integrated_sum, ) if (reference_reflections.get_flags( reference_reflections.flags.integrated_sum).count(True) == 0): assert ("intensity.sum.value" in test_reflections ), "No 'intensity.sum.value in reference reflections" reference_reflections.set_flags( flex.bool(reference_reflections.size(), True), reference_reflections.flags.integrated_sum, ) # Make miller array of the two datasets try: test_miller_set = filtered_arrays_from_experiments_reflections( experiments, [test_reflections])[0] except ValueError: raise Sorry( "No reflections remain after filtering the test dataset") try: reference_miller_set = filtered_arrays_from_experiments_reflections( reference_experiments, [reference_reflections])[0] except ValueError: raise Sorry( "No reflections remain after filtering the reference dataset") from dials.algorithms.symmetry.reindex_to_reference import ( determine_reindex_operator_against_reference, ) change_of_basis_op = determine_reindex_operator_against_reference( test_miller_set, reference_miller_set) elif len(experiments) and params.change_of_basis_op is libtbx.Auto: if reference_crystal is not None: if len(experiments.crystals()) > 1: raise Sorry("Only one crystal can be processed at a time") from dials.algorithms.indexing.compare_orientation_matrices import ( difference_rotation_matrix_axis_angle, ) cryst = experiments.crystals()[0] R, axis, angle, change_of_basis_op = difference_rotation_matrix_axis_angle( cryst, reference_crystal) print("Change of basis op: %s" % change_of_basis_op) print("Rotation matrix to transform input crystal to reference::") print(R.mathematica_form(format="%.3f", one_row_per_line=True)) print( "Rotation of %.3f degrees" % angle, "about axis (%.3f, %.3f, %.3f)" % axis, ) elif len(reflections): assert len(reflections) == 1 # always re-map reflections to reciprocal space refl_copy = flex.reflection_table() for i, imageset in enumerate(experiments.imagesets()): if "imageset_id" in reflections[0]: sel = reflections[0]["imageset_id"] == i else: sel = reflections[0]["id"] == i refl = reflections[0].select(sel) refl.centroid_px_to_mm(imageset.get_detector(), imageset.get_scan()) refl.map_centroids_to_reciprocal_space( imageset.get_detector(), imageset.get_beam(), imageset.get_goniometer(), ) refl_copy.extend(refl) # index the reflection list using the input experiments list refl_copy["id"] = flex.int(len(refl_copy), -1) from dials.algorithms.indexing import index_reflections index_reflections(refl_copy, experiments, tolerance=0.2) hkl_expt = refl_copy["miller_index"] hkl_input = reflections[0]["miller_index"] change_of_basis_op = derive_change_of_basis_op(hkl_input, hkl_expt) # reset experiments list since we don't want to reindex this experiments = [] else: change_of_basis_op = sgtbx.change_of_basis_op( params.change_of_basis_op) if len(experiments): for crystal in experiments.crystals(): cryst_orig = copy.deepcopy(crystal) cryst_reindexed = cryst_orig.change_basis(change_of_basis_op) if params.space_group is not None: a, b, c = cryst_reindexed.get_real_space_vectors() A_varying = [ cryst_reindexed.get_A_at_scan_point(i) for i in range(cryst_reindexed.num_scan_points) ] cryst_reindexed = Crystal( a, b, c, space_group=params.space_group.group()) cryst_reindexed.set_A_at_scan_points(A_varying) crystal.update(cryst_reindexed) print("Old crystal:") print(cryst_orig) print() print("New crystal:") print(cryst_reindexed) print() print("Saving reindexed experimental models to %s" % params.output.experiments) dump.experiment_list(experiments, params.output.experiments) if len(reflections): assert len(reflections) == 1 reflections = reflections[0] miller_indices = reflections["miller_index"] if params.hkl_offset is not None: h, k, l = miller_indices.as_vec3_double().parts() h += params.hkl_offset[0] k += params.hkl_offset[1] l += params.hkl_offset[2] miller_indices = flex.miller_index(h.iround(), k.iround(), l.iround()) non_integral_indices = change_of_basis_op.apply_results_in_non_integral_indices( miller_indices) if non_integral_indices.size() > 0: print( "Removing %i/%i reflections (change of basis results in non-integral indices)" % (non_integral_indices.size(), miller_indices.size())) sel = flex.bool(miller_indices.size(), True) sel.set_selected(non_integral_indices, False) miller_indices_reindexed = change_of_basis_op.apply( miller_indices.select(sel)) reflections["miller_index"].set_selected(sel, miller_indices_reindexed) reflections["miller_index"].set_selected(~sel, (0, 0, 0)) print("Saving reindexed reflections to %s" % params.output.reflections) easy_pickle.dump(params.output.reflections, reflections)
def find_spots(self, experiments: ExperimentList) -> flex.reflection_table: """ Do spotfinding for a set of experiments. Args: experiments: The experiment list to process Returns: A new reflection table of found reflections """ # Loop through all the experiments and get the unique imagesets imagesets = [] for experiment in experiments: if experiment.imageset not in imagesets: imagesets.append(experiment.imageset) # Loop through all the imagesets and find the strong spots reflections = flex.reflection_table() for j, imageset in enumerate(imagesets): # Find the strong spots in the sequence logger.info( "-" * 80 + "\nFinding strong spots in imageset %d\n" + "-" * 80, j) table, hot_mask = self._find_spots_in_imageset(imageset) # Fix up the experiment ID's now table["id"] = flex.int(table.nrows(), -1) for i, experiment in enumerate(experiments): if experiment.imageset is not imageset: continue if not self.is_stills and experiment.scan: z0, z1 = experiment.scan.get_array_range() z = table["xyzobs.px.value"].parts()[2] table["id"].set_selected((z > z0) & (z < z1), i) if experiment.identifier: table.experiment_identifiers( )[i] = experiment.identifier else: table["id"] = flex.int(table.nrows(), j) if experiment.identifier: table.experiment_identifiers( )[j] = experiment.identifier missed = table["id"] == -1 assert missed.count( True) == 0, "Failed to remap {} experiment IDs".format( missed.count(True)) reflections.extend(table) # Write a hot pixel mask if self.write_hot_mask: if not imageset.external_lookup.mask.data.empty(): for m1, m2 in zip(hot_mask, imageset.external_lookup.mask.data): m1 &= m2.data() imageset.external_lookup.mask.data = ImageBool(hot_mask) else: imageset.external_lookup.mask.data = ImageBool(hot_mask) imageset.external_lookup.mask.filename = "%s_%d.pickle" % ( self.hot_mask_prefix, i, ) # Write the hot mask with open(imageset.external_lookup.mask.filename, "wb") as outfile: pickle.dump(hot_mask, outfile, protocol=pickle.HIGHEST_PROTOCOL) # Set the strong spot flag reflections.set_flags(flex.size_t_range(len(reflections)), reflections.flags.strong) # Check for overloads reflections.is_overloaded(experiments) # Return the reflections return reflections
def split_for_scan_range(experiments, reference, scan_range): """Update experiments when scan range is set. Args: experiments: An experiment list reference: A reflection table of reference reflections scan_range (tuple): Range of scan images to be processed Returns: experiments: A new experiment list with the requested scan ranges reference: A reflection table with data from the scan ranges Raises: ValueError: If bad input for scan range. """ # Only do anything is the scan range is set if scan_range is not None and len(scan_range) > 0: # Ensure that all experiments have the same imageset and scan iset = [e.imageset for e in experiments] scan = [e.scan for e in experiments] assert all(x == iset[0] for x in iset) assert all(x == scan[0] for x in scan) # Get the imageset and scan iset = experiments[0].imageset scan = experiments[0].scan # Get the array range if scan is not None: frames_start, frames_end = scan.get_array_range() assert scan.get_num_images() == len(iset) else: frames_start, frames_end = (0, len(iset)) # Create the new lists new_experiments = ExperimentList() new_reference_all = reference.split_by_experiment_id() new_reference = flex.reflection_table() for i in range(len(new_reference_all) - len(experiments)): new_reference_all.append(flex.reflection_table()) assert len(new_reference_all) == len(experiments) # Loop through all the scan ranges and create a new experiment list with # the requested scan ranges. for scan_start, scan_end in scan_range: # Validate the requested scan range if scan_end == scan_start: raise ValueError( "Scan range end must be higher than start; pass {},{} for single image" .format(scan_start, scan_start + 1)) if scan_end < scan_start: raise ValueError("Scan range must be in ascending order") elif scan_start < frames_start or scan_end > frames_end: raise ValueError( "Scan range must be within image range {}..{}".format( frames_start, frames_end)) assert scan_end > scan_start assert scan_start >= frames_start assert scan_end <= frames_end index_start = scan_start - frames_start index_end = index_start + (scan_end - scan_start) assert index_start < index_end assert index_start >= 0 assert index_end <= len(iset) new_iset = iset[index_start:index_end] if scan is None: new_scan = None else: new_scan = scan[index_start:index_end] for i, e1 in enumerate(experiments): e2 = Experiment() e2.beam = e1.beam e2.detector = e1.detector e2.goniometer = e1.goniometer e2.crystal = slice_crystal(e1.crystal, (index_start, index_end)) e2.profile = e1.profile e2.imageset = new_iset e2.scan = new_scan new_reference_all[i]["id"] = flex.int( len(new_reference_all[i]), len(new_experiments)) new_reference.extend(new_reference_all[i]) new_experiments.append(e2) experiments = new_experiments reference = new_reference # Print some information logger.info( "Modified experiment list to integrate over requested scan range") for scan_start, scan_end in scan_range: logger.info(" scan_range = %d -> %d", scan_start, scan_end) # Return the experiments return experiments, reference
def run(self): ''' Parse the options. ''' from dials.util.options import flatten_experiments, flatten_reflections from dxtbx.model import ExperimentList from scitbx.math import five_number_summary # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) self.params = params experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) assert len(reflections) == 1 reflections = reflections[0] print "Found", len(reflections), "reflections", "and", len( experiments), "experiments" filtered_reflections = flex.reflection_table() filtered_experiments = ExperimentList() skipped_reflections = flex.reflection_table() skipped_experiments = ExperimentList() if params.detector is not None: culled_reflections = flex.reflection_table() culled_experiments = ExperimentList() detector = experiments.detectors()[params.detector] for expt_id, experiment in enumerate(experiments): refls = reflections.select(reflections['id'] == expt_id) if experiment.detector is detector: culled_experiments.append(experiment) refls['id'] = flex.int(len(refls), len(culled_experiments) - 1) culled_reflections.extend(refls) else: skipped_experiments.append(experiment) refls['id'] = flex.int(len(refls), len(skipped_experiments) - 1) skipped_reflections.extend(refls) print "RMSD filtering %d experiments using detector %d, out of %d" % ( len(culled_experiments), params.detector, len(experiments)) reflections = culled_reflections experiments = culled_experiments difference_vector_norms = (reflections['xyzcal.mm'] - reflections['xyzobs.mm.value']).norms() if params.max_delta is not None: sel = difference_vector_norms <= params.max_delta reflections = reflections.select(sel) difference_vector_norms = difference_vector_norms.select(sel) data = flex.double() counts = flex.double() for i in xrange(len(experiments)): dvns = difference_vector_norms.select(reflections['id'] == i) counts.append(len(dvns)) if len(dvns) == 0: data.append(0) continue rmsd = math.sqrt(flex.sum_sq(dvns) / len(dvns)) data.append(rmsd) data *= 1000 subset = data.select(counts > 0) print len(subset), "experiments with > 0 reflections" if params.show_plots: h = flex.histogram(subset, n_slots=40) fig = plt.figure() ax = fig.add_subplot('111') ax.plot(h.slot_centers().as_numpy_array(), h.slots().as_numpy_array(), '-') plt.title("Histogram of %d image RMSDs" % len(subset)) fig = plt.figure() plt.boxplot(subset, vert=False) plt.title("Boxplot of %d image RMSDs" % len(subset)) plt.show() outliers = counts == 0 min_x, q1_x, med_x, q3_x, max_x = five_number_summary(subset) print "Five number summary of RMSDs (microns): min %.1f, q1 %.1f, med %.1f, q3 %.1f, max %.1f" % ( min_x, q1_x, med_x, q3_x, max_x) iqr_x = q3_x - q1_x cut_x = params.iqr_multiplier * iqr_x outliers.set_selected(data > q3_x + cut_x, True) #outliers.set_selected(col < q1_x - cut_x, True) # Don't throw away the images that are outliers in the 'good' direction! for i in xrange(len(experiments)): if outliers[i]: continue refls = reflections.select(reflections['id'] == i) refls['id'] = flex.int(len(refls), len(filtered_experiments)) filtered_reflections.extend(refls) filtered_experiments.append(experiments[i]) zeroes = counts == 0 n_zero = len(counts.select(zeroes)) print "Removed %d bad experiments and %d experiments with zero reflections, out of %d (%%%.1f)" % ( len(experiments) - len(filtered_experiments) - n_zero, n_zero, len(experiments), 100 * ((len(experiments) - len(filtered_experiments)) / len(experiments))) if params.detector is not None: crystals = filtered_experiments.crystals() for expt_id, experiment in enumerate(skipped_experiments): if experiment.crystal in crystals: filtered_experiments.append(experiment) refls = skipped_reflections.select( skipped_reflections['id'] == expt_id) refls['id'] = flex.int(len(refls), len(filtered_experiments) - 1) filtered_reflections.extend(refls) if params.delta_psi_filter is not None: delta_psi = filtered_reflections['delpsical.rad'] * 180 / math.pi sel = (delta_psi <= params.delta_psi_filter) & ( delta_psi >= -params.delta_psi_filter) l = len(filtered_reflections) filtered_reflections = filtered_reflections.select(sel) print "Filtering by delta psi, removing %d out of %d reflections" % ( l - len(filtered_reflections), l) print "Final experiment count", len(filtered_experiments) from dxtbx.model.experiment_list import ExperimentListDumper dump = ExperimentListDumper(filtered_experiments) dump.as_json(params.output.filtered_experiments) filtered_reflections.as_pickle(params.output.filtered_reflections)
def test_average_bbox_size(): """Test behaviour of function for obtaining average bbox size.""" reflections = flex.reflection_table() reflections["bbox"] = flex.int6(*(flex.int(10, i) for i in range(6))) assert _average_bbox_size(reflections) == (1, 1, 1)
def test_set_selected(): from dials.array_family import flex # The columns as lists c1 = list(range(10)) c2 = list(range(10)) c3 = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'i', 'j', 'k'] # Create a table with some elements table1 = flex.reflection_table() table2 = flex.reflection_table() table1['col1'] = flex.int(c1) table2['col2'] = flex.double(c2) table2['col3'] = flex.std_string(c3) # Set selected columns table1.set_selected(('col3', 'col2'), table2) assert(table1.nrows() == 10) assert(table1.ncols() == 3) assert(all(a == b for a, b in zip(table1['col1'], c1))) assert(all(a == b for a, b in zip(table1['col2'], c2))) assert(all(a == b for a, b in zip(table1['col3'], c3))) # Set selected columns table1 = flex.reflection_table() table1['col1'] = flex.int(c1) table1.set_selected(flex.std_string(['col3', 'col2']), table2) assert(table1.nrows() == 10) assert(table1.ncols() == 3) assert(all(a == b for a, b in zip(table1['col1'], c1))) assert(all(a == b for a, b in zip(table1['col2'], c2))) assert(all(a == b for a, b in zip(table1['col3'], c3))) cc1 = list(range(10, 15)) cc2 = list(range(10, 15)) cc3 = ['l', 'm', 'n', 'o', 'p'] # Set selected rows table2 = flex.reflection_table() table2['col1'] = flex.int(cc1) table2['col2'] = flex.double(cc2) table2['col3'] = flex.std_string(cc3) index = flex.size_t([0, 1, 5, 8, 9]) ccc1 = copy.deepcopy(c1) ccc2 = copy.deepcopy(c2) ccc3 = copy.deepcopy(c3) for j, i in enumerate(index): ccc1[i] = cc1[j] ccc2[i] = cc2[j] ccc3[i] = cc3[j] table1.set_selected(index, table2) assert(all(a == b for a, b in zip(table1['col1'], ccc1))) assert(all(a == b for a, b in zip(table1['col2'], ccc2))) assert(all(a == b for a, b in zip(table1['col3'], ccc3))) # Set selected rows table2 = flex.reflection_table() table2['col1'] = flex.int(cc1) table2['col2'] = flex.double(cc2) table2['col3'] = flex.std_string(cc3) flags = flex.bool([True, True, False, False, False, True, False, False, True, True]) table1.set_selected(index, table2) assert(all(a == b for a, b in zip(table1['col1'], ccc1))) assert(all(a == b for a, b in zip(table1['col2'], ccc2))) assert(all(a == b for a, b in zip(table1['col3'], ccc3)))
def test_slicing(): from dials.array_family import flex # The columns as lists c1 = list(range(10)) c2 = list(range(10)) c3 = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'i', 'j', 'k'] # Create a table with some elements table = flex.reflection_table() table['col1'] = flex.int(c1) table['col2'] = flex.double(c2) table['col3'] = flex.std_string(c3) # Try forward slicing new_table = table[2:7:2] assert(new_table.ncols() == 3) assert(new_table.nrows() == 3) assert(new_table.is_consistent()) c11 = c1[2:7:2] c22 = c2[2:7:2] c33 = c3[2:7:2] assert(all(a == b for a, b in zip(new_table['col1'], c11))) assert(all(a == b for a, b in zip(new_table['col2'], c22))) assert(all(a == b for a, b in zip(new_table['col3'], c33))) # Try backward slicing new_table = table[7:2:-2] assert(new_table.ncols() == 3) assert(new_table.nrows() == 3) assert(new_table.is_consistent()) c11 = c1[7:2:-2] c22 = c2[7:2:-2] c33 = c3[7:2:-2] assert(all(a == b for a, b in zip(new_table['col1'], c11))) assert(all(a == b for a, b in zip(new_table['col2'], c22))) assert(all(a == b for a, b in zip(new_table['col3'], c33))) # Try setting forward slicing table[2:7:2] = new_table assert(table.ncols() == 3) assert(table.nrows() == 10) assert(table.is_consistent()) c1[2:7:2] = c11 c2[2:7:2] = c22 c3[2:7:2] = c33 assert(all(a == b for a, b in zip(table['col1'], c1))) assert(all(a == b for a, b in zip(table['col2'], c2))) assert(all(a == b for a, b in zip(table['col3'], c3))) # Try setting backward slicing table[7:2:-2] = new_table assert(table.ncols() == 3) assert(table.nrows() == 10) assert(table.is_consistent()) c1[7:2:-2] = c11 c2[7:2:-2] = c22 c3[7:2:-2] = c33 assert(all(a == b for a, b in zip(table['col1'], c1))) assert(all(a == b for a, b in zip(table['col2'], c2))) assert(all(a == b for a, b in zip(table['col3'], c3)))
def test_merge_multi_wavelength(dials_data, tmpdir): """Test that merge handles multi-wavelength data suitably - should be exported into an mtz with seprate columns for each wavelength.""" mean_labels = [ "%sIMEAN_WAVE%s" % (pre, i) for i in [1, 2] for pre in ["", "SIG"] ] anom_labels = [ "%sI_WAVE%s(%s)" % (pre, i, sgn) for i in [1, 2] for pre in ["", "SIG"] for sgn in ["+", "-"] ] amp_labels = [ "%sF_WAVE%s" % (pre, i) for i in [1, 2] for pre in ["", "SIG"] ] anom_amp_labels = [ "%sF_WAVE%s(%s)" % (pre, i, sgn) for i in [1, 2] for pre in ["", "SIG"] for sgn in ["+", "-"] ] location = dials_data("l_cysteine_4_sweeps_scaled") refl1 = location.join("scaled_30.refl").strpath expt1 = location.join("scaled_30.expt").strpath refl2 = location.join("scaled_35.refl").strpath expt2 = location.join("scaled_35.expt").strpath expts1 = ExperimentListFactory.from_json_file(expt1, check_format=False) expts1[0].beam.set_wavelength(0.5) expts2 = ExperimentListFactory.from_json_file(expt2, check_format=False) expts1.extend(expts2) tmp_expt = tmpdir.join("tmp.expt").strpath expts1.as_json(tmp_expt) reflections1 = flex.reflection_table.from_pickle(refl1) reflections2 = flex.reflection_table.from_pickle(refl2) # first need to resolve identifiers - usually done on loading reflections2["id"] = flex.int(reflections2.size(), 1) del reflections2.experiment_identifiers()[0] reflections2.experiment_identifiers()[1] = "3" reflections1.extend(reflections2) tmp_refl = tmpdir.join("tmp.refl").strpath reflections1.as_file(tmp_refl) # Can now run after creating our 'fake' multiwavelength dataset command = [ "dials.merge", tmp_refl, tmp_expt, "truncate=True", "anomalous=True" ] result = procrunner.run(command, working_directory=tmpdir) assert not result.returncode and not result.stderr assert tmpdir.join("merged.mtz").check() m = mtz.object(tmpdir.join("merged.mtz").strpath) labels = [] for ma in m.as_miller_arrays(merge_equivalents=False): labels.extend(ma.info().labels) assert all(x in labels for x in mean_labels) assert all(x in labels for x in anom_labels) assert all(x in labels for x in amp_labels) assert all(x in labels for x in anom_amp_labels) # 5 miller arrays for each dataset assert m.as_miller_arrays()[0].info().wavelength == pytest.approx(0.5) assert m.as_miller_arrays()[5].info().wavelength == pytest.approx(0.6889)
def test_experiment_identifiers(): from dials.array_family import flex from dxtbx.model import ExperimentList, Experiment table = flex.reflection_table() table['id'] = flex.int([0,1,2,3]) assert table.are_experiment_identifiers_consistent() == True identifiers = table.experiment_identifiers() identifiers[0] = 'abcd' identifiers[1] = 'efgh' identifiers[2] = 'ijkl' identifiers[3] = 'mnop' assert identifiers[0] == 'abcd' assert identifiers[1] == 'efgh' assert identifiers[2] == 'ijkl' assert identifiers[3] == 'mnop' for k, v in identifiers: if k == 0: assert v == 'abcd' if k == 1: assert v == 'efgh' if k == 2: assert v == 'ijkl' if k == 3: assert v == 'mnop' assert tuple(identifiers.keys()) == (0, 1, 2, 3) assert tuple(identifiers.values()) == ("abcd", "efgh", "ijkl", "mnop") assert table.are_experiment_identifiers_consistent() == True experiments = ExperimentList() experiments.append(Experiment(identifier="abcd")) experiments.append(Experiment(identifier="efgh")) experiments.append(Experiment(identifier="ijkl")) experiments.append(Experiment(identifier="mnop")) assert table.are_experiment_identifiers_consistent(experiments) == True experiments = ExperimentList() experiments.append(Experiment(identifier="abcd")) experiments.append(Experiment(identifier="efgh")) experiments.append(Experiment(identifier="ijkl")) experiments.append(Experiment(identifier="mnop")) experiments[3].identifier = "ijkl" assert table.are_experiment_identifiers_consistent(experiments) == False identifiers = table.experiment_identifiers() identifiers[0] = 'abcd' identifiers[1] = 'efgh' identifiers[2] = 'ijkl' identifiers[3] = 'ijkl' assert table.are_experiment_identifiers_consistent() == False identifiers[4] = 'mnop' import six.moves.cPickle as pickle pickled = pickle.dumps(table) table2 = pickle.loads(pickled) id1 = table.experiment_identifiers() id2 = table2.experiment_identifiers() for i in id1.keys(): assert id1[i] == id2[i] other_table = flex.reflection_table() other_table['id'] = flex.int([3, 4]) assert other_table.are_experiment_identifiers_consistent() == True identifiers = other_table.experiment_identifiers() identifiers[3] = 'mnop' identifiers[4] = 'qrst' table.extend(other_table) assert len(table.experiment_identifiers()) == 5 assert table.experiment_identifiers()[0] == 'abcd' assert table.experiment_identifiers()[1] == 'efgh' assert table.experiment_identifiers()[2] == 'ijkl' assert table.experiment_identifiers()[3] == 'mnop' assert table.experiment_identifiers()[4] == 'qrst'
def run(self): import datetime time_now = datetime.datetime.now() self.mpi_logger.log(str(time_now)) if self.mpi_helper.rank == 0: self.mpi_logger.main_log(str(time_now)) self.mpi_logger.log_step_time("TOTAL") self.mpi_logger.log_step_time("PARSE_INPUT_PARAMS") self.parse_input() self.mpi_logger.log_step_time("PARSE_INPUT_PARAMS", True) if self.params.mp.debug.cProfile: import cProfile pr = cProfile.Profile() pr.enable() # Create the workers using the factories self.mpi_logger.log_step_time("CREATE_WORKERS") from xfel.merging import application import importlib, copy self._resolve_persistent_columns() workers = [] self.params.dispatch.step_list = self.params.dispatch.step_list or default_steps for step in self.params.dispatch.step_list: step_factory_name = step step_additional_info = [] step_info = step.split('_') assert len(step_info) > 0 if len(step_info) > 1: step_factory_name = step_info[0] step_additional_info = step_info[1:] try: factory = importlib.import_module('xfel.merging.application.' + step_factory_name + '.factory') except ModuleNotFoundError: # remember the system path so the custom worker can temporarily modify it sys_path = copy.deepcopy(sys.path) pathstr = os.path.join('~', '.cctbx.xfel', 'merging', 'application', step_factory_name, 'factory.py') pathstr = os.path.expanduser(pathstr) modulename = 'xfel.merging.application.' + step_factory_name + '.factory' spec = importlib.util.spec_from_file_location( modulename, pathstr) factory = importlib.util.module_from_spec(spec) spec.loader.exec_module(factory) # reset the path sys.path = sys_path workers.extend( factory.factory.from_parameters(self.params, step_additional_info, mpi_helper=self.mpi_helper, mpi_logger=self.mpi_logger)) # Perform phil validation up front for worker in workers: worker.validate() self.mpi_logger.log_step_time("CREATE_WORKERS", True) # Do the work experiments = reflections = None step = 0 while (workers): worker = workers.pop(0) self.mpi_logger.log_step_time("STEP_" + worker.__repr__()) # Log worker name, i.e. execution step name step += 1 if step > 1: self.mpi_logger.log('') step_desc = "STEP %d: %s" % (step, worker) self.mpi_logger.log(step_desc) if self.mpi_helper.rank == 0: if step > 1: self.mpi_logger.main_log('') self.mpi_logger.main_log(step_desc) # Execute worker experiments, reflections = worker.run(experiments, reflections) self.mpi_logger.log_step_time("STEP_" + worker.__repr__(), True) if experiments: self.mpi_logger.log("Ending step with %d experiments" % len(experiments)) if self.params.output.save_experiments_and_reflections: if len(reflections) and 'id' not in reflections: from dials.array_family import flex id_ = flex.int(len(reflections), -1) if experiments: for expt_number, expt in enumerate(experiments): sel = reflections['exp_id'] == expt.identifier id_.set_selected(sel, expt_number) else: for expt_number, exp_id in enumerate( set(reflections['exp_id'])): sel = reflections['exp_id'] == exp_id id_.set_selected(sel, expt_number) reflections['id'] = id_ assert (reflections['id'] == -1).count(True) == 0, (( reflections['id'] == -1).count(True), len(reflections)) if self.mpi_helper.size == 1: filename_suffix = "" else: filename_suffix = "_%06d" % self.mpi_helper.rank if len(reflections): reflections.as_pickle( os.path.join( self.params.output.output_dir, "%s%s.refl" % (self.params.output.prefix, filename_suffix))) if experiments: experiments.as_file( os.path.join( self.params.output.output_dir, "%s%s.expt" % (self.params.output.prefix, filename_suffix))) self.mpi_logger.log_step_time("TOTAL", True) if self.params.mp.debug.cProfile: pr.disable() pr.dump_stats( os.path.join( self.params.output.output_dir, "cpu_%s_%d.prof" % (self.params.output.prefix, self.mpi_helper.rank)))
import scipy.linalg # import dependency except ImportError, e: pass ''' if __name__ == "__main__": lst_flex = [] lst_flex_norm = [] for size_xyz in range(8, 6, -1): size_x = size_xyz * 2 data_xyz_flex = flex.double(flex.grid(size_xyz, size_xyz, size_x), 15) data_flex_norm = flex.double(flex.grid(size_xyz, size_xyz, size_x), 15) data_flex_mask = flex.int(flex.grid(size_xyz, size_xyz, size_x), 0) tot = 0.0 for frm in range(size_xyz): for row in range(size_xyz): for col in range(size_x): data_xyz_flex[frm, row, col] += (row * 2 + col * 2 + frm * 2) tot += data_xyz_flex[frm, row, col] if row > 1 and row < size_xyz - 2 and col > 1 and col < size_x - 2: data_flex_mask[frm, row, col] = MaskCode.Foreground different_mask_values = ''' MaskCode.Valid = "\\\\\\" MaskCode.Foreground = "//////" MaskCode.Background = "||||||"
def test_split_partials_with_shoebox(): from dials.array_family import flex from random import randint, uniform from dials.model.data import Shoebox r = flex.reflection_table() r['value1'] = flex.double() r['value2'] = flex.int() r['value3'] = flex.double() r['bbox'] = flex.int6() r['panel'] = flex.size_t() r['shoebox'] = flex.shoebox() expected = [] for i in range(100): x0 = randint(0, 100) x1 = x0 + randint(1, 10) y0 = randint(0, 100) y1 = y0 + randint(1, 10) z0 = randint(0, 100) z1 = z0 + randint(1, 10) v1 = uniform(0, 100) v2 = randint(0, 100) v3 = uniform(0, 100) sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1)) sbox.allocate() assert(sbox.is_consistent()) w = x1 - x0 h = y1 - y0 for z in range(z0, z1): for y in range(y0, y1): for x in range(x0, x1): sbox.data[z-z0,y-y0,x-x0] = x+y*w+z*w*h r.append({ 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z0, z1), 'panel' : 0, 'shoebox' : sbox }) for z in range(z0, z1): sbox = Shoebox(0, (x0, x1, y0, y1, z, z+1)) sbox.allocate() assert(sbox.is_consistent()) w = x1 - x0 h = y1 - y0 for y in range(y0, y1): for x in range(x0, x1): sbox.data[0,y-y0,x-x0] = x+y*w+z*w*h expected.append({ 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z, z+1), 'partial_id' : i, 'panel' : 0, 'shoebox' : sbox }) r.split_partials_with_shoebox() assert(len(r) == len(expected)) EPS = 1e-7 for r1, r2 in zip(r, expected): assert(abs(r1['value1'] - r2['value1']) < EPS) assert(r1['value2'] == r2['value2']) assert(abs(r1['value3'] - r2['value3']) < EPS) assert(r1['bbox'] == r2['bbox']) assert(r1['partial_id'] == r2['partial_id']) assert(r1['panel'] == r2['panel']) assert(r1['shoebox'].data.as_double().as_1d().all_approx_equal( r2['shoebox'].data.as_double().as_1d()))
def run_integration(params, experiments, reference=None): """Perform the integration. Returns: experiments: The integrated experiments reflections: The integrated reflections report(optional): An integration report. Raises: ValueError: For a number of bad inputs RuntimeError: If the profile model creation fails """ predicted = None rubbish = None for abs_params in params.absorption_correction: if abs_params.apply: if not (params.integration.debug.output and not params.integration.debug.separate_files): raise ValueError( "Shoeboxes must be saved to integration intermediates to apply an absorption correction. " + "Set integration.debug.output=True, integration.debug.separate_files=False and " + "integration.debug.delete_shoeboxes=True to temporarily store shoeboxes." ) # Print if we're using a mask for i, exp in enumerate(experiments): mask = exp.imageset.external_lookup.mask if mask.filename is not None: if mask.data: logger.info("Using external mask: %s", mask.filename) for tile in mask.data: logger.info(" Mask has %d pixels masked", tile.data().count(False)) # Print the experimental models for i, exp in enumerate(experiments): summary = "\n".join(( "", "=" * 80, "", "Experiments", "", "Models for experiment %d" % i, "", str(exp.beam), str(exp.detector), )) if exp.goniometer: summary += str(exp.goniometer) + "\n" if exp.scan: summary += str(exp.scan) + "\n" summary += str(exp.crystal) logger.info(summary) logger.info("\n".join(("", "=" * 80, ""))) logger.info(heading("Initialising")) # Load the data if reference: reference, rubbish = process_reference(reference) # Check pixels don't belong to neighbours if exp.goniometer is not None and exp.scan is not None: reference = filter_reference_pixels(reference, experiments) # Modify experiment list if scan range is set. experiments, reference = split_for_scan_range(experiments, reference, params.scan_range) # Modify experiment list if exclude images is set if params.exclude_images: for experiment in experiments: for index in params.exclude_images: experiment.imageset.mark_for_rejection(index, True) # Predict the reflections logger.info("\n".join(("", "=" * 80, ""))) logger.info(heading("Predicting reflections")) predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=params.prediction.d_min, dmax=params.prediction.d_max, margin=params.prediction.margin, force_static=params.prediction.force_static, padding=params.prediction.padding, ) # Match reference with predicted if reference: matched, reference, unmatched = predicted.match_with_reference( reference) assert len(matched) == len(predicted) assert matched.count(True) <= len(reference) if matched.count(True) == 0: raise ValueError(""" Invalid input for reference reflections. Zero reference spots were matched to predictions """) elif unmatched: msg = ( "Warning: %d reference spots were not matched to predictions" % unmatched.size()) border = "\n".join(("", "*" * 80, "")) logger.info("".join((border, msg, border))) rubbish.extend(unmatched) if len(experiments) > 1: # filter out any experiments without matched reference reflections # f_: filtered f_reference = flex.reflection_table() f_predicted = flex.reflection_table() f_rubbish = flex.reflection_table() f_experiments = ExperimentList() good_expt_count = 0 def refl_extend(src, dest, eid): old_id = eid new_id = good_expt_count tmp = src.select(src["id"] == old_id) tmp["id"] = flex.int(len(tmp), good_expt_count) if old_id in tmp.experiment_identifiers(): identifier = tmp.experiment_identifiers()[old_id] del tmp.experiment_identifiers()[old_id] tmp.experiment_identifiers()[new_id] = identifier dest.extend(tmp) for expt_id, experiment in enumerate(experiments): if len(reference.select(reference["id"] == expt_id)) != 0: refl_extend(reference, f_reference, expt_id) refl_extend(predicted, f_predicted, expt_id) refl_extend(rubbish, f_rubbish, expt_id) f_experiments.append(experiment) good_expt_count += 1 else: logger.info( "Removing experiment %d: no reference reflections matched to predictions", expt_id, ) reference = f_reference predicted = f_predicted experiments = f_experiments rubbish = f_rubbish # Select a random sample of the predicted reflections if not params.sampling.integrate_all_reflections: predicted = sample_predictions(experiments, predicted, params) # Compute the profile model - either load existing or compute # can raise RuntimeError experiments = ProfileModelFactory.create(params, experiments, reference) for expr in experiments: if expr.profile is None: raise ValueError("No profile information in experiment list") del reference # Compute the bounding box predicted.compute_bbox(experiments) # Create the integrator integrator = create_integrator(params, experiments, predicted) # Integrate the reflections reflections = integrator.integrate() # Remove unintegrated reflections if not params.output.output_unintegrated_reflections: keep = reflections.get_flags(reflections.flags.integrated, all=False) logger.info("Removing %d unintegrated reflections of %d total" % (keep.count(False), keep.size())) reflections = reflections.select(keep) # Append rubbish data onto the end if rubbish is not None and params.output.include_bad_reference: mask = flex.bool(len(rubbish), True) rubbish.unset_flags(mask, rubbish.flags.integrated_sum) rubbish.unset_flags(mask, rubbish.flags.integrated_prf) rubbish.set_flags(mask, rubbish.flags.bad_reference) reflections.extend(rubbish) # Correct integrated intensities for absorption correction, if necessary for abs_params in params.absorption_correction: if abs_params.apply and abs_params.algorithm == "fuller_kapton": from dials.algorithms.integration.kapton_correction import ( multi_kapton_correction, ) experiments, reflections = multi_kapton_correction( experiments, reflections, abs_params.fuller_kapton, logger=logger)() if params.significance_filter.enable: from dials.algorithms.integration.stills_significance_filter import ( SignificanceFilter, ) sig_filter = SignificanceFilter(params) filtered_refls = sig_filter(experiments, reflections) accepted_expts = ExperimentList() accepted_refls = flex.reflection_table() logger.info( "Removed %d reflections out of %d when applying significance filter", (reflections.size() - filtered_refls.size()), reflections.size(), ) for expt_id, expt in enumerate(experiments): refls = filtered_refls.select(filtered_refls["id"] == expt_id) if refls: accepted_expts.append(expt) current_id = expt_id new_id = len(accepted_expts) - 1 refls["id"] = flex.int(len(refls), new_id) if expt.identifier: del refls.experiment_identifiers()[current_id] refls.experiment_identifiers()[new_id] = expt.identifier accepted_refls.extend(refls) else: logger.info( "Removed experiment %d which has no reflections left after applying significance filter", expt_id, ) if not accepted_refls: raise ValueError( "No reflections left after applying significance filter") experiments = accepted_expts reflections = accepted_refls # Write a report if requested report = None if params.output.report is not None: report = integrator.report() return experiments, reflections, report
def run(i, imp): from random import randint from dials.array_family import flex #building a reflection table num_ref = 5 ref_table = flex.reflection_table() shoebox = flex.shoebox(num_ref) ref_table['shoebox'] = shoebox intensity = flex.double(num_ref) ref_table['intensity.sum.value'] = intensity intensity_var = flex.double(num_ref) ref_table['intensity.sum.variance'] = intensity_var iterate = ref_table['shoebox'] i_to_compare = [] # bulding the shoebox with a desired content # which is a reflection with noise included n = 0 for arr in iterate: img = flex.double(flex.grid(3, 3, 3)) bkg = flex.double(flex.grid(3, 3, 3)) msk = flex.int(flex.grid(3, 3, 3)) for row in range(3): for col in range(3): for fra in range(3): img[row, col, fra] = row + col + fra + n * 9 + randint(0, i) bkg[row, col, fra] = 0.0 msk[row, col, fra] = 3 n += 1 msk[1, 1, 1] = 5 tmp_i = n * n * n * 3 i_to_compare.append(tmp_i) img[1, 1, 1] += tmp_i arr.data = img[:, :, :] arr.background = bkg[:, :, :] arr.mask = msk[:, :, :] # calling the functions that we need to test # first select the algorithm for background calculation if imp == "inclined": print("testing inclined_background_subtractor") from dials.algorithms.background.inclined_background_subtractor \ import layering_and_background_plane layering_and_background_plane(ref_table) elif imp == "flat": print("testing flat_background_subtractor") from dials.algorithms.background.flat_background_subtractor \ import layering_and_background_avg layering_and_background_avg(ref_table) elif imp == "curved": print("testing curved_background_subtractor") from dials.algorithms.background.curved_background_subtractor \ import layering_and_background_modl layering_and_background_modl(ref_table) # no matter which algorithm was used for background calculation # the integration summation must remain compatible from dials.algorithms.integration.summation2d \ import flex_2d_layering_n_integrating flex_2d_layering_n_integrating(ref_table) # comparing results result = "OK" resl_its = ref_table['intensity.sum.value'] resl_var = ref_table['intensity.sum.variance'] for n_its in range(len(resl_its)): if resl_its[n_its] <= i_to_compare[n_its] + i and \ resl_its[n_its] >= i_to_compare[n_its] - i and \ resl_var[n_its] > resl_its[n_its]: print("Ok ", n_its) else: print("Wrong num", n_its) print("i =", i) print("resl_its[n_its] =", resl_its[n_its]) print("i_to_compare[n_its] =", i_to_compare[n_its]) print("resl_var[n_its] =", resl_var[n_its]) result = "wrong" raise RuntimeError('wrong result') return result
def __init__(self, reflections, av_callback=flex.mean, debug=False): # flags to indicate at what level the analysis has been performed self._average_residuals = False self._spectral_analysis = False self._av_callback = av_callback # Remove invalid reflections reflections = reflections.select(~(reflections["miller_index"] == (0, 0, 0))) x, y, z = reflections["xyzcal.mm"].parts() sel = (x == 0) & (y == 0) reflections = reflections.select(~sel) self._nexp = flex.max(reflections["id"]) + 1 # Ensure required keys are present if not all(k in reflections for k in ["x_resid", "y_resid", "phi_resid"]): x_obs, y_obs, phi_obs = reflections["xyzobs.mm.value"].parts() x_cal, y_cal, phi_cal = reflections["xyzcal.mm"].parts() # do not wrap around multiples of 2*pi; keep the full rotation # from zero to differentiate repeat observations. TWO_PI = 2.0 * math.pi resid = phi_cal - (flex.fmod_positive(phi_obs, TWO_PI)) # ensure this is the smaller of two possibilities resid = flex.fmod_positive((resid + math.pi), TWO_PI) - math.pi phi_cal = phi_obs + resid reflections["x_resid"] = x_cal - x_obs reflections["y_resid"] = y_cal - y_obs reflections["phi_resid"] = phi_cal - phi_obs # create empty results list self._results = [] # first, just determine a suitable block size for analysis for iexp in range(self._nexp): ref_this_exp = reflections.select(reflections["id"] == iexp) if len(ref_this_exp) == 0: # can't do anything, just keep an empty dictionary self._results.append({}) continue phi_obs_deg = ref_this_exp["xyzobs.mm.value"].parts()[2] * RAD2DEG phi_range = flex.min(phi_obs_deg), flex.max(phi_obs_deg) phi_width = phi_range[1] - phi_range[0] ideal_block_size = 1.0 old_nblocks = 0 while True: nblocks = int(phi_width // ideal_block_size) if nblocks == old_nblocks: nblocks -= 1 nblocks = max(nblocks, 1) block_size = phi_width / nblocks nr = flex.int() for i in range(nblocks - 1): blk_start = phi_range[0] + i * block_size blk_end = blk_start + block_size sel = (phi_obs_deg >= blk_start) & (phi_obs_deg < blk_end) nref_in_block = sel.count(True) nr.append(nref_in_block) # include max phi in the final block blk_start = phi_range[0] + (nblocks - 1) * block_size blk_end = phi_range[1] sel = (phi_obs_deg >= blk_start) & (phi_obs_deg <= blk_end) nref_in_block = sel.count(True) nr.append(nref_in_block) # Break if there are enough reflections, otherwise increase block size, # unless only one block remains if nblocks == 1: break min_nr = flex.min(nr) if min_nr >= 50: break if min_nr < 5: fac = 2 else: fac = 50 / min_nr ideal_block_size *= fac old_nblocks = nblocks # collect the basic data for this experiment self._results.append( { "block_size": block_size, "nref_per_block": nr, "nblocks": nblocks, "phi_range": phi_range, } ) # keep reflections for analysis self._reflections = reflections # for debugging, write out reflections used if debug: self._reflections.as_file("centroid_analysis.refl")
def run(self, args=None): """Execute the script.""" from dials.util.options import reflections_and_experiments_from_files # Parse the command line params, options = self.parser.parse_args(args, show_diff_phil=True) reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) # Try to load the models and data slice_exps = len(experiments) > 0 slice_refs = len(reflections) > 0 # Catch case of nothing to do if not slice_exps and not slice_refs: print("No suitable input provided") self.parser.print_help() return if reflections: if len(reflections) > 1: raise Sorry( "Only one reflections list can be imported at present") reflections = reflections[0] # calculate frame numbers if needed if experiments: reflections = calculate_frame_numbers(reflections, experiments) # if we still don't have the right column give up if "xyzobs.px.value" not in reflections: raise Sorry( "These reflections do not have frame numbers set, and " "there are no experiments provided to calculate these.") # set trivial case where no scan range is provided at all if not params.image_range: params.image_range = [None] # check if slicing into blocks if params.block_size is not None: if not slice_exps: raise Sorry( "For slicing into blocks, an experiment file must be provided" ) if len(experiments) > 1: raise Sorry( "For slicing into blocks please provide a single scan only" ) scan = experiments[0].scan # Having extracted the scan, calculate the blocks params.image_range = calculate_block_ranges( scan, params.block_size) # Do the slicing then recombine sliced = [ slice_experiments(experiments, [sr])[0] for sr in params.image_range ] generate_experiment_identifiers(sliced) sliced_experiments = ExperimentList(sliced) # slice reflections if present if slice_refs: sliced = [ slice_reflections(reflections, [sr]) for sr in params.image_range ] sliced_reflections = flex.reflection_table() identifiers = sliced_experiments.identifiers() # resetting experiment identifiers for i, rt in enumerate(sliced): for k in rt.experiment_identifiers().keys(): del rt.experiment_identifiers()[k] rt["id"] = flex.int(rt.size(), i) # set id rt.experiment_identifiers()[i] = identifiers[i] sliced_reflections.extend(rt) else: # slice each dataset into the requested subset if slice_exps: sliced_experiments = slice_experiments(experiments, params.image_range) if slice_refs: sliced_reflections = slice_reflections(reflections, params.image_range) # Save sliced experiments if slice_exps: output_experiments_filename = params.output.experiments_filename if output_experiments_filename is None: # take first filename as template bname = basename(params.input.experiments[0].filename) bname = splitext(bname)[0] if not bname: bname = "experiments" if len(params.image_range ) == 1 and params.image_range[0] is not None: ext = "_{0}_{1}.expt".format(*params.image_range[0]) else: ext = "_sliced.expt" output_experiments_filename = bname + ext print("Saving sliced experiments to {}".format( output_experiments_filename)) sliced_experiments.as_file(output_experiments_filename) # Save sliced reflections if slice_refs: output_reflections_filename = params.output.reflections_filename if output_reflections_filename is None: # take first filename as template bname = basename(params.input.reflections[0].filename) bname = splitext(bname)[0] if not bname: bname = "reflections" if len(params.image_range ) == 1 and params.image_range[0] is not None: ext = "_{0}_{1}.refl".format(*params.image_range[0]) else: ext = "_sliced.refl" output_reflections_filename = bname + ext print("Saving sliced reflections to {0}".format( output_reflections_filename)) sliced_reflections.as_file(output_reflections_filename) return
def test_assign_indices(dials_regression, space_group_symbol): experiments_json = os.path.join(dials_regression, "indexing_test_data", "i04_weak_data", "datablock_orig.json") experiments = load.experiment_list(experiments_json, check_format=False) sweep = experiments.imagesets()[0] sweep = sweep[:20] # set random seeds so tests more reliable seed = 54321 random.seed(seed) flex.set_random_seed(seed) space_group_info = sgtbx.space_group_info(symbol=space_group_symbol) space_group = space_group_info.group() unit_cell = space_group_info.any_compatible_unit_cell( volume=random.uniform(1e4, 1e6)) crystal_symmetry = crystal.symmetry(unit_cell=unit_cell, space_group=space_group) crystal_symmetry.show_summary() # the reciprocal matrix B = matrix.sqr(unit_cell.fractionalization_matrix()).transpose() U = random_rotation() A = U * B direct_matrix = A.inverse() cryst_model = Crystal( direct_matrix[0:3], direct_matrix[3:6], direct_matrix[6:9], space_group=space_group, ) experiment = Experiment( imageset=sweep, beam=sweep.get_beam(), detector=sweep.get_detector(), goniometer=sweep.get_goniometer(), scan=sweep.get_scan(), crystal=cryst_model, ) predicted_reflections = flex.reflection_table.from_predictions(experiment) use_fraction = 0.3 use_sel = flex.random_selection( len(predicted_reflections), int(use_fraction * len(predicted_reflections))) predicted_reflections = predicted_reflections.select(use_sel) miller_indices = predicted_reflections["miller_index"] predicted_reflections["xyzobs.mm.value"] = predicted_reflections[ "xyzcal.mm"] predicted_reflections["id"] = flex.int(len(predicted_reflections), 0) predicted_reflections.map_centroids_to_reciprocal_space( sweep.get_detector(), sweep.get_beam(), sweep.get_goniometer()) # check that local and global indexing worked equally well in absence of errors result = CompareGlobalLocal(experiment, predicted_reflections, miller_indices) assert result.misindexed_local == 0 assert result.misindexed_global == 0 a, b, c = map(matrix.col, cryst_model.get_real_space_vectors()) relative_error = 0.02 a *= 1 + relative_error b *= 1 + relative_error c *= 1 + relative_error cryst_model2 = Crystal(a, b, c, space_group=space_group) experiment.crystal = cryst_model2 result = CompareGlobalLocal(experiment, predicted_reflections, miller_indices) # check that the local indexing did a better job given the errors in the basis vectors # assert result.misindexed_local < result.misindexed_global assert result.misindexed_local == 0 assert result.correct_local > result.correct_global # usually the number misindexed is much smaller than this assert result.misindexed_local < (0.001 * len(result.reflections_local)) # the reciprocal matrix A = matrix.sqr(cryst_model.get_A()) A = random_rotation(angle_max=0.5) * A direct_matrix = A.inverse() cryst_model2 = Crystal( direct_matrix[0:3], direct_matrix[3:6], direct_matrix[6:9], space_group=space_group, ) experiment.crystal = cryst_model2 result = CompareGlobalLocal(experiment, predicted_reflections, miller_indices) # check that the local indexing did a better job given the errors in the basis vectors assert result.misindexed_local <= result.misindexed_global, ( result.misindexed_local, result.misindexed_global, ) assert result.misindexed_local < 0.01 * result.correct_local assert result.correct_local >= result.correct_global # usually the number misindexed is much smaller than this assert result.misindexed_local < (0.001 * len(result.reflections_local))
def run(self): ''' Extract the shoeboxes. ''' from dials.util.options import flatten_reflections from dials.util.options import flatten_experiments from dials.util.options import flatten_datablocks from dials.util import log from dials.array_family import flex from libtbx.utils import Sorry # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) # Configure logging log.config() # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Get the data reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) datablocks = flatten_datablocks(params.input.datablock) if not any([experiments, datablocks, reflections]): self.parser.print_help() exit(0) elif experiments and datablocks: raise Sorry('Both experiment list and datablocks set') elif len(experiments) > 1: raise Sorry('More than 1 experiment set') elif len(datablocks) > 1: raise Sorry('More than 1 datablock set') elif len(experiments) == 1: imageset = experiments[0].imageset elif len(datablocks) == 1: imagesets = datablocks[0].extract_imagesets() if len(imagesets) != 1: raise Sorry('Need 1 imageset, got %d' % len(imagesets)) imageset = imagesets[0] if len(reflections) != 1: raise Sorry('Need 1 reflection table, got %d' % len(reflections)) else: reflections = reflections[0] # Check the reflections contain the necessary stuff assert("bbox" in reflections) assert("panel" in reflections) # Get some models detector = imageset.get_detector() scan = imageset.get_scan() frame0, frame1 = scan.get_array_range() # Add some padding but limit to image volume if params.padding > 0: logger.info('Adding %d pixels as padding' % params.padding) x0, x1, y0, y1, z0, z1 = reflections['bbox'].parts() x0 -= params.padding x1 += params.padding y0 -= params.padding y1 += params.padding # z0 -= params.padding # z1 += params.padding panel = reflections['panel'] for i in range(len(reflections)): width, height = detector[panel[i]].get_image_size() if x0[i] < 0: x0[i] = 0 if x1[i] > width: x1[i] = width if y0[i] < 0: y0[i] = 0 if y1[i] > height: y1[i] = height if z0[i] < frame0: z0[i] = frame0 if z1[i] > frame1: z1[i] = frame1 reflections['bbox'] = flex.int6(x0, x1, y0, y1, z0, z1) # Save the old shoeboxes if "shoebox" in reflections: old_shoebox = reflections['shoebox'] else: old_shoebox = None # Allocate the shoeboxes reflections["shoebox"] = flex.shoebox( reflections["panel"], reflections["bbox"], allocate=True) # Extract the shoeboxes reflections.extract_shoeboxes(imageset, verbose=True) # Preserve masking if old_shoebox is not None: from dials.algorithms.shoebox import MaskCode logger.info("Applying old shoebox mask") new_shoebox = reflections['shoebox'] for i in range(len(reflections)): bbox0 = old_shoebox[i].bbox bbox1 = new_shoebox[i].bbox mask0 = old_shoebox[i].mask mask1 = new_shoebox[i].mask mask2 = flex.int(mask1.accessor(), 0) x0 = bbox0[0] - bbox1[0] x1 = bbox0[1] - bbox0[0] + x0 y0 = bbox0[2] - bbox1[2] y1 = bbox0[3] - bbox0[2] + y0 z0 = bbox0[4] - bbox1[4] z1 = bbox0[5] - bbox0[4] + z0 mask2[z0:z1,y0:y1,x0:x1] = mask0 mask1 = mask1.as_1d() | mask2.as_1d() if params.padding_is_background: selection = flex.size_t(range(len(mask1))).select(mask1 == MaskCode.Valid) values = flex.int(len(selection), MaskCode.Valid | MaskCode.Background) mask1.set_selected(selection, values) mask1.reshape(new_shoebox[i].mask.accessor()) new_shoebox[i].mask = mask1 # Saving the reflections to disk filename = params.output.reflections logger.info('Saving %d reflections to %s' % (len(reflections), filename)) reflections.as_pickle(filename)
def reflections_1(): """Test reflection table with batch""" r = flex.reflection_table() r["batch"] = flex.int([1, 11, 21, 31, 41, 51, 61, 71, 81, 91]) r.set_flags(flex.bool(10, False), r.flags.user_excluded_in_scaling) return r
def reflections_2(): """Test reflection table with batch""" r = flex.reflection_table() r["batch"] = flex.int([201, 211, 221, 231, 241, 251, 261, 271, 281, 291]) r.set_flags(flex.bool(10, False), r.flags.user_excluded_in_scaling) return r
def batch_plot_shapes_and_annotations(self): light_grey = "#d3d3d3" grey = "#808080" shapes = [] annotations = [] batches = flex.int(self.batches) text = flex.std_string(batches.size()) for i, batch in enumerate(self.batch_params): fillcolor = [light_grey, grey][i % 2] # alternate colours shapes.append({ "type": "rect", # x-reference is assigned to the x-values "xref": "x", # y-reference is assigned to the plot paper [0,1] "yref": "paper", "x0": self._batch_increments[i], "y0": 0, "x1": self._batch_increments[i] + (batch["range"][1] - batch["range"][0]), "y1": 1, "fillcolor": fillcolor, "opacity": 0.2, "line": { "width": 0 }, }) annotations.append({ # x-reference is assigned to the x-values "xref": "x", # y-reference is assigned to the plot paper [0,1] "yref": "paper", "x": self._batch_increments[i] + (batch["range"][1] - batch["range"][0]) / 2, "y": 1, "text": f"{batch['id']}", "showarrow": False, "yshift": 20, # 'arrowhead': 7, # 'ax': 0, # 'ay': -40 }) sel = (batches >= batch["range"][0]) & (batches <= batch["range"][1]) text.set_selected( sel, flex.std_string([ f"{batch['id']}: {j - batch['range'][0] + 1}" for j in batches.select(sel) ]), ) return shapes, annotations, list(text)
def assign_hkl_to_reflections(self, reflections, experiments, d_min=None, tolerance=0.3, debug=False): ''' Function to assign hkl values to reflections on a shot. Uses underlying c++ function AssignIndices() available in DIALS''' from cctbx.array_family import flex reciprocal_lattice_points = reflections['rlp'] reflections['miller_index'] = flex.miller_index( len(reflections), (0, 0, 0)) # IOTA reflections['fractional_miller_index'] = flex.vec3_double( len(reflections), (0.0, 0.0, 0.0)) if d_min is not None: d_spacings = 1 / reciprocal_lattice_points.norms() inside_resolution_limit = d_spacings > d_min else: inside_resolution_limit = flex.bool( reciprocal_lattice_points.size(), True) sel = inside_resolution_limit & (reflections['id'] == -1) isel = sel.iselection() rlps = reciprocal_lattice_points.select(isel) refs = reflections.select(isel) phi = refs['xyzobs.mm.value'].parts()[2] diffs = [] norms = [] hkl_ints = [] UB_matrices = flex.mat3_double( [cm.get_A() for cm in experiments.crystals()]) imgset_ids = reflections['imageset_id'].select(sel) for i_imgset, imgset in enumerate(experiments.imagesets()): sel_imgset = (imgset_ids == i_imgset) result = AssignIndices(rlps.select(sel_imgset), phi.select(sel_imgset), UB_matrices, tolerance=tolerance) miller_indices = result.miller_indices() crystal_ids = result.crystal_ids() expt_ids = flex.int(crystal_ids.size(), -1) for i_cryst, cryst in enumerate(experiments.crystals()): sel_cryst = (crystal_ids == i_cryst) for i_expt in experiments.where(crystal=cryst, imageset=imgset): expt_ids.set_selected(sel_cryst, i_expt) reflections['miller_index'].set_selected(isel.select(sel_imgset), miller_indices) reflections['id'].set_selected(isel.select(sel_imgset), expt_ids) reflections.set_flags(reflections['miller_index'] != (0, 0, 0), reflections.flags.indexed) reflections['id'].set_selected( reflections['miller_index'] == (0, 0, 0), -1)
def run(self, all_experiments, all_reflections): """ Load all the data using MPI """ from dxtbx.model.experiment_list import ExperimentList from dials.array_family import flex # Both must be none or not none test = [all_experiments is None, all_reflections is None].count(True) assert test in [0, 2] if test == 2: all_experiments = ExperimentList() all_reflections = flex.reflection_table() starting_expts_count = starting_refls_count = 0 else: starting_expts_count = len(all_experiments) starting_refls_count = len(all_reflections) self.logger.log( "Initial number of experiments: %d; Initial number of reflections: %d" % (starting_expts_count, starting_refls_count)) # Generate and send a list of file paths to each worker if self.mpi_helper.rank == 0: file_list = self.get_list() self.logger.log( "Built an input list of %d json/pickle file pairs" % (len(file_list))) self.params.input.path = None # Rank 0 has already parsed the input parameters per_rank_file_list = file_load_calculator(self.params, file_list, self.logger).\ calculate_file_load(available_rank_count = self.mpi_helper.size) self.logger.log( 'Transmitting a list of %d lists of json/pickle file pairs' % (len(per_rank_file_list))) transmitted = per_rank_file_list else: transmitted = None self.logger.log_step_time("BROADCAST_FILE_LIST") transmitted = self.mpi_helper.comm.bcast(transmitted, root=0) new_file_list = transmitted[ self.mpi_helper. rank] if self.mpi_helper.rank < len(transmitted) else None self.logger.log_step_time("BROADCAST_FILE_LIST", True) # Load the data self.logger.log_step_time("LOAD") if new_file_list is not None: self.logger.log("Received a list of %d json/pickle file pairs" % len(new_file_list)) for experiments_filename, reflections_filename in new_file_list: self.logger.log("Reading %s %s" % (experiments_filename, reflections_filename)) experiments = ExperimentListFactory.from_json_file( experiments_filename, check_format=False) reflections = flex.reflection_table.from_file( reflections_filename) self.logger.log("Data read, prepping") if 'intensity.sum.value' in reflections: reflections[ 'intensity.sum.value.unmodified'] = reflections[ 'intensity.sum.value'] * 1 if 'intensity.sum.variance' in reflections: reflections[ 'intensity.sum.variance.unmodified'] = reflections[ 'intensity.sum.variance'] * 1 new_ids = flex.int(len(reflections), -1) new_identifiers = flex.std_string(len(reflections)) eid = reflections.experiment_identifiers() for k in eid.keys(): del eid[k] for experiment_id, experiment in enumerate(experiments): # select reflections of the current experiment refls_sel = reflections['id'] == experiment_id if refls_sel.count(True) == 0: continue if experiment.identifier is None or len( experiment.identifier) == 0: experiment.identifier = create_experiment_identifier( experiment, experiments_filename, experiment_id) if not self.params.input.keep_imagesets: experiment.imageset = None all_experiments.append(experiment) # Reflection experiment 'id' is unique within this rank; 'exp_id' (i.e. experiment identifier) is unique globally new_identifiers.set_selected(refls_sel, experiment.identifier) new_id = len(all_experiments) - 1 eid[new_id] = experiment.identifier new_ids.set_selected(refls_sel, new_id) assert (new_ids < 0 ).count(True) == 0, "Not all reflections accounted for" reflections['id'] = new_ids reflections['exp_id'] = new_identifiers all_reflections.extend(reflections) else: self.logger.log("Received a list of 0 json/pickle file pairs") self.logger.log_step_time("LOAD", True) self.logger.log('Read %d experiments consisting of %d reflections' % (len(all_experiments) - starting_expts_count, len(all_reflections) - starting_refls_count)) self.logger.log("Memory usage: %d MB" % get_memory_usage()) all_reflections = self.prune_reflection_table_keys(all_reflections) # Do we have any data? from xfel.merging.application.utils.data_counter import data_counter data_counter(self.params).count(all_experiments, all_reflections) return all_experiments, all_reflections
def index(self, provided_experiments=None, debug=False): ''' This step does 1. find_lattices (via a method like fft1d) 2. Assign hkl indices (through index_reflections) 3. Housekeeping like apply_symmetry, discard too similar models ''' experiments = ExperimentList() have_similar_crystal_models = False self.d_min = self.params.refinement_protocol.d_min_start # Find lattices i.e the basis vectors & unit cell params # Possible to index multiple lattices ?? while True: max_lattices = self.params.multiple_lattice_search.max_lattices if max_lattices is not None and len(experiments) >= max_lattices: break n_lattices_previous_cycle = len(experiments) if len(experiments) == 0: experiments.extend(self.find_lattices()) else: try: new = self.find_lattices() experiments.extend(new) except Sorry: print('Indexing remaining reflections failed') break if len(experiments) == 0: raise Sorry("No suitable lattice could be found.") # Initialize id values as -1 since no indexing has been done yet self.reflections['id'] = flex.int(len(self.reflections), -1) # Now index reflections self.index_reflections(experiments, self.reflections, debug=debug) # Housekeeping. Apply symmetry self._apply_symmetry_post_indexing(experiments, self.reflections, n_lattices_previous_cycle) # Aug_Refactor :: probably unnecessary to remove stuff below but still doing so to adapt to new style # Never mind, might keep it to have IOTA stuff working ''' target_space_group = self.target_symmetry_primitive.space_group() for i_cryst, cryst in enumerate(experiments.crystals()): if i_cryst >= n_lattices_previous_cycle: new_cryst, cb_op_to_primitive = self.apply_symmetry( cryst, target_space_group) if provided_experiments is None: if self.cb_op_primitive_inp is not None: new_cryst = new_cryst.change_basis(self.cb_op_primitive_inp) logger.info(new_cryst.get_space_group().info()) cryst.update(new_cryst) cryst.set_space_group( self.params.known_symmetry.space_group.group()) for i_expt, expt in enumerate(experiments): if expt.crystal is not cryst: continue if not cb_op_to_primitive.is_identity_op(): miller_indices = self.reflections['miller_index'].select( self.reflections['id'] == i_expt) if provided_experiments is None: miller_indices = cb_op_to_primitive.apply(miller_indices) self.reflections['miller_index'].set_selected( self.reflections['id'] == i_expt, miller_indices) if self.cb_op_primitive_inp is not None: miller_indices = self.reflections['miller_index'].select( self.reflections['id'] == i_expt) if provided_experiments is None: miller_indices = self.cb_op_primitive_inp.apply(miller_indices) self.reflections['miller_index'].set_selected( self.reflections['id'] == i_expt, miller_indices) # IOTA from scitbx.matrix import sqr hklfrac=flex.mat3_double(len(miller_indices), sqr(cryst.get_A()).inverse())*self.reflections['rlp'].select(self.reflections['id']==i_expt) self.reflections['fractional_miller_index'].set_selected(self.reflections['id']==i_expt, hklfrac) ''' logger.info("\nIndexed crystal models:") self.show_experiments(experiments, self.reflections, d_min=self.d_min) # Discard nearly overlapping lattices # difference_rotation_matrix_axis_angle function is there still in DIALS 2.0 so no need to change anything below if len(experiments) > 1: from dials.algorithms.indexing.compare_orientation_matrices \ import difference_rotation_matrix_axis_angle cryst_b = experiments.crystals()[-1] have_similar_crystal_models = False for i_a, cryst_a in enumerate(experiments.crystals()[:-1]): R_ab, axis, angle, cb_op_ab = \ difference_rotation_matrix_axis_angle(cryst_a, cryst_b) min_angle = self.params.multiple_lattice_search.minimum_angular_separation if abs(angle) < min_angle: # degrees logger.info( "Crystal models too similar, rejecting crystal %i:" % (len(experiments))) logger.info( "Rotation matrix to transform crystal %i to crystal %i" % (i_a + 1, len(experiments))) logger.info(R_ab) logger.info("Rotation of %.3f degrees" % angle + " about axis (%.3f, %.3f, %.3f)" % axis) have_similar_crystal_models = True del experiments[-1] break self.indexed_reflections = (self.reflections['id'] > -1) self.experiments = experiments
def test_del_selected(): from dials.array_family import flex # The columns as lists c1 = list(range(10)) c2 = list(range(10)) c3 = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'i', 'j', 'k'] # Create a table with some elements table1 = flex.reflection_table() table1['col1'] = flex.int(c1) table1['col2'] = flex.double(c2) table1['col3'] = flex.std_string(c3) # Del selected columns table1.del_selected(('col3', 'col2')) assert(table1.nrows() == 10) assert(table1.ncols() == 1) assert("col1" in table1) assert("col2" not in table1) assert("col3" not in table1) assert(all(a == b for a, b in zip(table1['col1'], c1))) # Del selected columns table1 = flex.reflection_table() table1['col1'] = flex.int(c1) table1['col2'] = flex.double(c2) table1['col3'] = flex.std_string(c3) table1.del_selected(flex.std_string(['col3', 'col2'])) assert(table1.nrows() == 10) assert(table1.ncols() == 1) assert("col1" in table1) assert("col2" not in table1) assert("col3" not in table1) assert(all(a == b for a, b in zip(table1['col1'], c1))) # Del selected rows table1 = flex.reflection_table() table1['col1'] = flex.int(c1) table1['col2'] = flex.double(c2) table1['col3'] = flex.std_string(c3) index = flex.size_t([0, 1, 5, 8, 9]) index2 = range(10) for i in index: index2.remove(i) ccc1 = [c1[i] for i in index2] ccc2 = [c2[i] for i in index2] ccc3 = [c3[i] for i in index2] table1.del_selected(index) assert(table1.nrows() == len(ccc1)) assert(all(a == b for a, b in zip(table1['col1'], ccc1))) assert(all(a == b for a, b in zip(table1['col2'], ccc2))) assert(all(a == b for a, b in zip(table1['col3'], ccc3))) # Del selected rows table1 = flex.reflection_table() table1['col1'] = flex.int(c1) table1['col2'] = flex.double(c2) table1['col3'] = flex.std_string(c3) flags = flex.bool([True, True, False, False, False, True, False, False, True, True]) table1.del_selected(index) assert(table1.nrows() == len(ccc1)) assert(all(a == b for a, b in zip(table1['col1'], ccc1))) assert(all(a == b for a, b in zip(table1['col2'], ccc2))) assert(all(a == b for a, b in zip(table1['col3'], ccc3)))
def _refine(self): for epoch, idxr in self._refinr_indexers.items(): experiments = idxr.get_indexer_experiment_list() indexed_experiments = idxr.get_indexer_payload( "experiments_filename") indexed_reflections = idxr.get_indexer_payload("indexed_filename") if len(experiments) > 1: xsweeps = idxr._indxr_sweeps assert len(xsweeps) == len(experiments) assert (len(self._refinr_sweeps) == 1 ) # don't currently support joint refinement xsweep = self._refinr_sweeps[0] i = xsweeps.index(xsweep) experiments = experiments[i:i + 1] # Extract and output experiment and reflections for current sweep indexed_experiments = os.path.join( self.get_working_directory(), "%s_indexed.expt" % xsweep.get_name()) indexed_reflections = os.path.join( self.get_working_directory(), "%s_indexed.refl" % xsweep.get_name()) experiments.as_file(indexed_experiments) reflections = flex.reflection_table.from_file( idxr.get_indexer_payload("indexed_filename")) sel = reflections["id"] == i assert sel.count(True) > 0 imageset_id = reflections["imageset_id"].select(sel) assert imageset_id.all_eq(imageset_id[0]) sel = reflections["imageset_id"] == imageset_id[0] reflections = reflections.select(sel) # set indexed reflections to id == 0 and imageset_id == 0 reflections["id"].set_selected(reflections["id"] == i, 0) reflections["imageset_id"] = flex.int(len(reflections), 0) reflections.as_file(indexed_reflections) assert (len(experiments.crystals()) == 1 ) # currently only handle one lattice/sweep scan_static = PhilIndex.params.dials.refine.scan_static # Avoid doing scan-varying refinement on narrow wedges. start, end = experiments[0].scan.get_oscillation_range() total_oscillation_range = end - start if (PhilIndex.params.dials.refine.scan_varying and total_oscillation_range > 5 and not PhilIndex.params.dials.fast_mode): scan_varying = PhilIndex.params.dials.refine.scan_varying else: scan_varying = False if scan_static: refiner = self.Refine() refiner.set_experiments_filename(indexed_experiments) refiner.set_indexed_filename(indexed_reflections) refiner.set_scan_varying(False) refiner.run() self._refinr_experiments_filename = ( refiner.get_refined_experiments_filename()) self._refinr_indexed_filename = refiner.get_refined_filename() else: self._refinr_experiments_filename = indexed_experiments self._refinr_indexed_filename = indexed_reflections if scan_varying: refiner = self.Refine() refiner.set_experiments_filename( self._refinr_experiments_filename) refiner.set_indexed_filename(self._refinr_indexed_filename) if total_oscillation_range < 36: refiner.set_interval_width_degrees( total_oscillation_range / 2) refiner.run() self._refinr_experiments_filename = ( refiner.get_refined_experiments_filename()) self._refinr_indexed_filename = refiner.get_refined_filename() if scan_static or scan_varying: FileHandler.record_log_file( "%s REFINE" % idxr.get_indexer_full_name(), refiner.get_log_file()) report = self.Report() report.set_experiments_filename( self._refinr_experiments_filename) report.set_reflections_filename(self._refinr_indexed_filename) html_filename = os.path.join( self.get_working_directory(), "%i_dials.refine.report.html" % report.get_xpid(), ) report.set_html_filename(html_filename) report.run() FileHandler.record_html_file( "%s REFINE" % idxr.get_indexer_full_name(), html_filename) experiments = load.experiment_list( self._refinr_experiments_filename) self.set_refiner_payload("models.expt", self._refinr_experiments_filename) self.set_refiner_payload("observations.refl", self._refinr_indexed_filename) # this is the result of the cell refinement self._refinr_cell = experiments.crystals()[0].get_unit_cell( ).parameters()
def test_row_operations(): from dials.array_family import flex # The columns as lists c1 = list(range(10)) c2 = list(range(10)) c3 = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'i', 'j', 'k'] # Create a table with some elements table = flex.reflection_table() table['col1'] = flex.int(c1) table['col2'] = flex.double(c2) table['col3'] = flex.std_string(c3) # Extend the table table.extend(table) c1 = c1 * 2 c2 = c2 * 2 c3 = c3 * 2 assert(table.nrows() == 20) assert(table.ncols() == 3) assert(table.is_consistent()) assert(all(a == b for a, b in zip(table['col1'], c1))) assert(all(a == b for a, b in zip(table['col2'], c2))) assert(all(a == b for a, b in zip(table['col3'], c3))) # Append some rows to the table row = { 'col1' : 10 } c1 = c1 + [10] c2 = c2 + [0] c3 = c3 + [''] table.append(row) assert(table.nrows() == 21) assert(table.ncols() == 3) assert(table.is_consistent()) assert(all(a == b for a, b in zip(table['col1'], c1))) assert(all(a == b for a, b in zip(table['col2'], c2))) assert(all(a == b for a, b in zip(table['col3'], c3))) row = { 'col2' : 11 } c1 = c1 + [0] c2 = c2 + [11] c3 = c3 + [''] table.append(row) assert(table.nrows() == 22) assert(table.ncols() == 3) assert(table.is_consistent()) assert(all(a == b for a, b in zip(table['col1'], c1))) assert(all(a == b for a, b in zip(table['col2'], c2))) assert(all(a == b for a, b in zip(table['col3'], c3))) row = { 'col1' : 12, 'col2' : 12, 'col3' : 'l' } c1 = c1 + [12] c2 = c2 + [12] c3 = c3 + ['l'] table.append(row) assert(table.nrows() == 23) assert(table.ncols() == 3) assert(table.is_consistent()) assert(all(a == b for a, b in zip(table['col1'], c1))) assert(all(a == b for a, b in zip(table['col2'], c2))) assert(all(a == b for a, b in zip(table['col3'], c3))) # Try inserting some rows row = { 'col1' : -1 } c1.insert(5, -1) c2.insert(5, 0) c3.insert(5, '') table.insert(5, row) assert(table.nrows() == 24) assert(table.ncols() == 3) assert(table.is_consistent()) assert(all(a == b for a, b in zip(table['col1'], c1))) assert(all(a == b for a, b in zip(table['col2'], c2))) assert(all(a == b for a, b in zip(table['col3'], c3))) row = { 'col1' : -2, 'col2' : -3, 'col3' : 'abc' } c1.insert(2, -2) c2.insert(2, -3) c3.insert(2, 'abc') table.insert(2, row) assert(table.nrows() == 25) assert(table.ncols() == 3) assert(table.is_consistent()) assert(all(a == b for a, b in zip(table['col1'], c1))) assert(all(a == b for a, b in zip(table['col2'], c2))) assert(all(a == b for a, b in zip(table['col3'], c3))) # Try iterating through table rows for i in range(table.nrows()): row = table[i] assert(row['col1'] == c1[i]) assert(row['col2'] == c2[i]) assert(row['col3'] == c3[i]) # Trying setting some rows row = { 'col1' : 100 } table[2] = row assert(table[2]['col1'] == 100) assert(table[2]['col2'] == c2[2]) assert(table[2]['col3'] == c3[2]) row = { 'col1' : 1000, 'col2' : 2000, 'col3' : 'hello' } table[10] = row assert(table[10]['col1'] == 1000) assert(table[10]['col2'] == 2000) assert(table[10]['col3'] == 'hello')
def __init__(self): self.data = flex.int(range(height*width)) self.data.reshape(flex.grid(height,width))