def plot_valid(b, s): from dials.array_family import flex b = [0.1, 0.2, 0.3, 0.4, 0.5] s = [0.1, 0.3, 0.5, 0.3, 0.1] v1 = flex.bool(flex.grid(100, 100)) v2 = flex.bool(flex.grid(100, 100)) v3 = flex.bool(flex.grid(100, 100)) r = [float(ss) / float(bb) for ss, bb in zip(s, b)] for BB in range(0, 100): for SS in range(0, 100): B = -5.0 + BB / 10.0 S = -5.0 + SS / 10.0 V1 = True V2 = True V3 = True for i in range(len(b)): if B*b[i]+S*s[i] <= 0: V1 = False break for i in range(len(b)): if B*b[i] <= -S*s[i]: V2 = False break v1[BB,SS] = V1 v2[BB,SS] = V2 from matplotlib import pylab pylab.imshow(v1.as_numpy_array()) pylab.show() exit(0)
def show_image(c,b,s, BB=None, SS=None): import numpy.ma from dials.array_family import flex N = 100 im = flex.double(flex.grid(N, N)) mask = flex.bool(flex.grid(N, N)) for j in range(N): for i in range(N): B = -1.0 + j * 10.0 / N S = -1.0 + i * 10.0 / N im[j,i], mask[j,i] = func(c,b,s,B,S) im[j,i] = -im[j,i] masked_im = numpy.ma.array( # im.as_numpy_array(), flex.exp(im).as_numpy_array(), mask = mask.as_numpy_array()) mask2 = flex.bool(flex.grid(N, N)) indices = [] for i in range(len(mask)): if mask[i] == False: indices.append(i) indices = flex.size_t(indices) ind = flex.max_index(im.select(indices)) ind = indices[ind] maxy = -1.0 + (ind % N) * 10.0 / N maxx = -1.0 + (ind // N) * 10.0 / N from matplotlib import pylab pylab.imshow(masked_im, origin='bottom', extent=[-1.0, 9.0, -1.0, 9.0]) if YY is not None and XX is not None: pylab.plot(YY, XX) pylab.scatter([maxy], [maxx]) pylab.show()
def filter_frames(self, data): data = data[0] lp = gaussian_filter(data, 100) hp = data - lp # poormans background subtraction hp -= np.min(hp) sh = hp.shape print "here" hp = hp.astype('uint32') hp = flex.int(hp) print "here now" mask = flex.bool(np.ones_like(hp).astype('bool')) print "here now" result1 = flex.bool(np.zeros_like(hp).astype('bool')) spots = np.zeros_like(hp).astype('bool') print "here now" for i in range(3, self.parameters['spotsize'], 5): print "here now" algorithm = DispersionThreshold(sh, (i, i), 1, 1, 0, -1) print "here now" print type(hp), type(mask), type(result1) thing = algorithm(hp, mask, result1) print "here now" spots = spots + result1.as_numpy_array() return [data, spots*data]
def run(self): '''Execute the script.''' from dials.array_family import flex from dials.util.options import flatten_reflections from libtbx.utils import Sorry # Parse the command line params, options = self.parser.parse_args(show_diff_phil=True) reflections = flatten_reflections(params.input.reflections) if len(reflections) == 0: self.parser.print_help() raise Sorry('No valid reflection file given') if len(reflections) != 1: self.parser.print_help() raise Sorry('Exactly 1 reflection file must be specified') reflections = reflections[0] print "{0} reflections loaded".format(len(reflections)) if len(params.inclusions.flag) == 0: self.parser.print_help() raise Sorry('No inclusion criteria given') # Build up the initial inclusion selection inc = flex.bool(len(reflections)) for flag in params.inclusions.flag: sel = reflections.get_flags(getattr(reflections.flags, flag)) inc = inc | sel reflections = reflections.select(inc) print "{0} reflections selected to form the working set".format(len(reflections)) # Make requested exclusions from the current selection exc = flex.bool(len(reflections)) for flag in params.exclusions.flag: print flag sel = reflections.get_flags(getattr(reflections.flags, flag)) exc = exc | sel reflections = reflections.select(~exc) print "{0} reflections excluded from the working set".format(exc.count(True)) # Save filtered reflections to file if params.output.reflections: print "Saving {0} reflections to {1}".format(len(reflections), params.output.reflections) reflections.as_pickle(params.output.reflections) return
def process_reference(reference): ''' Load the reference spots. ''' from dials.util.command_line import Command from dials.array_family import flex if reference is None: return None assert("miller_index" in reference) Command.start('Removing reference spots with invalid coordinates') mask = flex.bool([x == (0, 0, 0) for x in reference['xyzcal.mm']]) reference.del_selected(mask) mask = flex.bool([h == (0, 0, 0) for h in reference['miller_index']]) reference.del_selected(mask) Command.end('Removed reference spots with invalid coordinates, %d remaining' % len(reference)) return reference
def _create_hot_mask(self, imageset, hot_pixels): ''' Find hot pixels in images ''' from dials.array_family import flex # Write the hot mask if self.write_hot_mask: # Create the hot pixel mask hot_mask = tuple(flex.bool(flex.grid(p.get_image_size()[::-1]), True) for p in imageset.get_detector()) num_hot = 0 if hot_pixels > 0: for hp, hm in zip(hot_pixels, hot_mask): for i in range(len(hp)): hm[hp[i]] = False num_hot += len(hp) logger.info('Found %d possible hot pixel(s)' % num_hot) else: hot_mask = None # Return the hot mask return hot_mask
def tst_for_dataset(self, creator, filename): from dials.array_family import flex from dials.algorithms.shoebox import MaskCode print filename rlist = flex.reflection_table.from_pickle(filename) shoebox = rlist['shoebox'] background = [sb.background.deep_copy() for sb in shoebox] success = creator(shoebox) assert(success.count(True) == len(success)) diff = [] for i in range(len(rlist)): mask = flex.bool([(m & MaskCode.Foreground) != 0 for m in shoebox[i].mask]) px1 = background[i].select(mask) px2 = shoebox[i].background.select(mask) den = max([flex.mean(px1), 1.0]) diff.append(flex.mean(px2 - px1) / den) diff = flex.double(diff) mv = flex.mean_and_variance(flex.double(diff)) mean = mv.mean() sdev = mv.unweighted_sample_standard_deviation() try: assert(abs(mean) < 0.01) except Exception: print "Mean: %f, Sdev: %f", mean, sdev from matplotlib import pylab pylab.hist(diff) pylab.show() raise
def plot_prob_for_zero(c, b, s): from math import log, exp, factorial from dials.array_family import flex L = flex.double(flex.grid(100, 100)) MASK = flex.bool(flex.grid(100, 100)) c = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] b = [bb / sum(b) for bb in b] s = [ss / sum(s) for ss in s] for BB in range(0, 100): for SS in range(0, 100): B = 0 + BB / 10000.0 S = 0 + SS / 40.0 LL = 0 for i in range(len(b)): if B*b[i] + S*s[i] <= 0: MASK[BB, SS] = True LL = -999999 break else: LL += c[i]*log(B*b[i]+S*s[i]) - log(factorial(c[i])) - B*b[i] - S*s[i] L[BB, SS] = LL index = flex.max_index(L) i = index % 100 j = index // 100 B = 0 + j / 10000.0 S = 0 + i / 40.0 print flex.max(L), B, S from matplotlib import pylab import numpy im = numpy.ma.masked_array(flex.exp(L).as_numpy_array(), mask=MASK.as_numpy_array()) pylab.imshow(im) pylab.show() exit(0)
def generate_reflections(self): """Use reeke_model to generate indices of reflections near to the Ewald sphere that might be observed on a still image. Build a reflection_table of these.""" from cctbx.sgtbx import space_group_info space_group_type = space_group_info("P 1").group().type() # create a ReekeIndexGenerator UB = self.crystal.get_U() * self.crystal.get_B() axis = self.goniometer.get_rotation_axis() s0 = self.beam.get_s0() dmin = 1.5 # use the same UB at the beginning and end - the margin parameter ensures # we still have indices close to the Ewald sphere generated from dials.algorithms.spot_prediction import ReekeIndexGenerator r = ReekeIndexGenerator(UB, UB, space_group_type, axis, s0, dmin=1.5, margin=1) # generate indices hkl = r.to_array() nref = len(hkl) # create a reflection table from dials.array_family import flex table = flex.reflection_table() table['flags'] = flex.size_t(nref, 0) table['id'] = flex.int(nref, 0) table['panel'] = flex.size_t(nref, 0) table['miller_index'] = flex.miller_index(hkl) table['entering'] = flex.bool(nref, True) table['s1'] = flex.vec3_double(nref) table['xyzcal.mm'] = flex.vec3_double(nref) table['xyzcal.px'] = flex.vec3_double(nref) return table
def refine(self, experiments, reflections): sel = ((reflections['id'] >= -1)) refl = reflections.select(sel) acceptance_flags = self.identify_outliers(self.all_params, experiments, refl) #create a new "indexed" list with outliers thrown out: refl = refl.select(acceptance_flags) print "$$$ stills_indexer::refine" R = e_refine(params = self.all_params, experiments=experiments, reflections=refl, graph_verbose=False) ref_experiments = R.get_experiments() # try to improve the outcome with a second round of outlier rejection post-initial refinement: acceptance_flags = self.identify_outliers(self.all_params, ref_experiments, refl) # insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection nv0 = nave_parameters(params = self.all_params, experiments=ref_experiments, reflections=refl, refinery=R, graph_verbose=False) crystal_model_nv0 = nv0() acceptance_flags_nv0 = nv0.nv_acceptance_flags refl = refl.select(acceptance_flags & acceptance_flags_nv0) print "$$$ stills_indexer::refine after positional and delta-psi outlier rejection" refiner = e_refine(params = self.all_params, experiments=ref_experiments, reflections=refl, graph_verbose=False) matches = refiner.get_matches() xyzcal_mm = flex.vec3_double(len(refl)) xyzcal_mm.set_selected(matches['iobs'], matches['xyzcal.mm']) refl['xyzcal.mm'] = xyzcal_mm refl.set_flags(matches['iobs'], refl.flags.used_in_refinement) refl['entering'] = flex.bool(len(refl), False) return refiner.get_experiments(), refl
def outlier_rejection(reflections): # http://scripts.iucr.org/cgi-bin/paper?ba0032 if len(reflections) == 1: return reflections intensities = reflections['intensity.sum.value'] variances = reflections['intensity.sum.variance'] i_max = flex.max_index(intensities) sel = flex.bool(len(reflections), True) sel[i_max] = False i_test = intensities[i_max] var_test = variances[i_max] intensities_subset = intensities.select(sel) var_subset = variances.select(sel) var_prior = var_test + 1/flex.sum(1/var_subset) p_prior = 1/math.sqrt(2*math.pi * var_prior) * math.exp( -(i_test - flex.mean(intensities_subset))**2/(2 * var_prior)) #print p_prior if p_prior > 1e-10: return reflections return outlier_rejection(reflections.select(sel))
def __init__(self, frames, size): from dials.array_family import flex self.frames = frames nframes = frames[1] - frames[0] self.data = [] self.mask = [] for sz in size: self.data.append(flex.double(flex.grid(nframes, sz[0], sz[1]))) self.mask.append(flex.bool(flex.grid(nframes, sz[0], sz[1])))
def main(reflections, experiment, params): nref0 = len(reflections) method = params.method ids = params.id if 'intensity.prf.variance' in reflections: selection = reflections.get_flags( reflections.flags.integrated, all=True) else: selection = reflections.get_flags( reflections.flags.integrated_sum) reflections = reflections.select(selection) # filter according to rules if params.min_isum: selection = reflections['intensity.sum.value'] > params.min_isum reflections = reflections.select(selection) if params.num > len(reflections): raise RuntimeError, 'you asked for too many reflections sorry' if params.seed > 0 and params.num > 0: import random from dials.array_family import flex random.seed(params.seed) selected = flex.bool(len(reflections), False) while len(selected.iselection()) < params.num: selected[random.randint(0, len(reflections))] = True reflections = reflections.select(selected) nref1 = len(reflections) print 'Removed %d reflections, %d remain' % (nref0 - nref1, nref1) results = [] for j, reflection in enumerate(reflections): result = None if ids: if j in ids: result = globals()['model_reflection_%s' % method](reflection, experiment, params) elif params.id_start and params.id_end: if j < params.id_start or j >= params.id_end: continue print j result = globals()['model_reflection_%s' % method](reflection, experiment, params) else: print j result = globals()['model_reflection_%s' % method](reflection, experiment, params) if result is not None: results.append(result) return results
def run(self): ''' Perform the integration. ''' from dials.util.command_line import heading from dials.util.options import flatten_experiments from dials.util import log from time import time from libtbx.utils import Sorry from dials.array_family import flex # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) experiments = flatten_experiments(params.input.experiments) if len(experiments) == 0: self.parser.print_help() return assert len(experiments) == 1 imageset = experiments[0].imageset beam = experiments[0].beam detector = experiments[0].detector goniometer = experiments[0].goniometer assert len(detector) == 1 # Configure logging log.config() from dials.algorithms.background.gmodel import PolarTransform import cPickle as pickle model = pickle.load(open(params.model)) image = model.data(0) mask = flex.bool(image.accessor(), True) # Do the transformation transform = PolarTransform(beam, detector[0], goniometer) result = transform.to_polar(image, mask) data = result.data() mask = result.mask() pickle.dump((data, mask), open(params.output.data, "w")) from matplotlib import pylab vmax = sorted(list(data))[int(0.99 * len(data))] figure = pylab.figure(figsize=(6,4)) pylab.imshow( data.as_numpy_array(), interpolation = 'none', vmin = 0, vmax = vmax) ax1 = pylab.gca() ax1.get_xaxis().set_visible(False) ax1.get_yaxis().set_visible(False) cb = pylab.colorbar() cb.ax.tick_params(labelsize=8) logger.info("Saving polar model %s" % (params.output.image)) pylab.savefig("%s" % (params.output.image), dpi=600, bbox_inches='tight')
def run(args): import libtbx.load_env usage = "%s experiments.json [options]" %libtbx.env.dispatcher_name parser = OptionParser( usage=usage, phil=phil_scope, read_experiments=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) if len(experiments) == 0: parser.print_help() return elif len(experiments) > 1: raise Sorry("More than one experiment present") assert len(params.miller_index), "Must specify at least one miller_index to predict." experiment = experiments[0] reflections = flex.reflection_table() miller_indices = flex.miller_index() entering_flags = flex.bool() for mi in params.miller_index: miller_indices.append(mi) miller_indices.append(mi) entering_flags.append(True) entering_flags.append(False) reflections['miller_index'] = miller_indices reflections['entering'] = entering_flags reflections['id'] = flex.size_t(len(reflections), 0) if params.expand_to_p1: from cctbx.miller import expand_to_p1_iselection proxy = expand_to_p1_iselection( experiment.crystal.get_space_group(), anomalous_flag=True, indices=miller_indices, build_iselection=True) reflections = reflections.select(proxy.iselection) reflections['miller_index'] = proxy.indices from dials.algorithms.refinement.prediction.managed_predictors import ExperimentsPredictor predictor = ExperimentsPredictor([experiment]) predicted = predictor.predict(reflections) zmin, zmax = experiment.scan.get_array_range() z = predicted['xyzcal.px'].parts()[2] predicted = predicted.select((z >= zmin) & (z <= zmax)) show_predictions(predicted)
def tst_select(self): from dials.array_family import flex # The columns as lists c1 = list(range(10)) c2 = list(range(10)) c3 = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'i', 'j', 'k'] # Create a table with some elements table = flex.reflection_table() table['col1'] = flex.int(c1) table['col2'] = flex.double(c2) table['col3'] = flex.std_string(c3) # Select some columns new_table = table.select(('col1', 'col2')) assert(new_table.nrows() == 10) assert(new_table.ncols() == 2) assert(all(a == b for a, b in zip(new_table['col1'], c1))) assert(all(a == b for a, b in zip(new_table['col2'], c2))) print 'OK' # Select some columns new_table = table.select(flex.std_string(['col1', 'col2'])) assert(new_table.nrows() == 10) assert(new_table.ncols() == 2) assert(all(a == b for a, b in zip(new_table['col1'], c1))) assert(all(a == b for a, b in zip(new_table['col2'], c2))) print 'OK' # Select some rows index = flex.size_t([0, 1, 5, 8, 9]) cc1 = [c1[i] for i in index] cc2 = [c2[i] for i in index] cc3 = [c3[i] for i in index] new_table = table.select(index) assert(new_table.nrows() == 5) assert(new_table.ncols() == 3) assert(all(a == b for a, b in zip(new_table['col1'], cc1))) assert(all(a == b for a, b in zip(new_table['col2'], cc2))) assert(all(a == b for a, b in zip(new_table['col3'], cc3))) print 'OK' # Select some rows index = flex.bool([True, True, False, False, False, True, False, False, True, True]) new_table = table.select(index) assert(new_table.nrows() == 5) assert(new_table.ncols() == 3) assert(all(a == b for a, b in zip(new_table['col1'], cc1))) assert(all(a == b for a, b in zip(new_table['col2'], cc2))) assert(all(a == b for a, b in zip(new_table['col3'], cc3))) print 'OK'
def __call__(self, params, options): ''' Import the spot.xds file. ''' from iotbx.xds import spot_xds from dials.util.command_line import Command from dials.array_family import flex # Read the SPOT.XDS file Command.start('Reading SPOT.XDS') handle = spot_xds.reader() handle.read_file(self._spot_xds) centroid = handle.centroid intensity = handle.intensity try: miller_index = handle.miller_index except AttributeError: miller_index = None Command.end('Read {0} spots from SPOT.XDS file.'.format(len(centroid))) # Create the reflection list Command.start('Creating reflection list') table = flex.reflection_table() table['id'] = flex.int(len(centroid), 0) table['panel'] = flex.size_t(len(centroid), 0) if miller_index: table['miller_index'] = flex.miller_index(miller_index) table['xyzobs.px.value'] = flex.vec3_double(centroid) table['intensity.sum.value'] = flex.double(intensity) Command.end('Created reflection list') # Remove invalid reflections Command.start('Removing invalid reflections') if miller_index and params.remove_invalid: flags = flex.bool([h != (0, 0, 0) for h in table['miller_index']]) table = table.select(flags) Command.end('Removed invalid reflections, %d remaining' % len(table)) # Fill empty standard columns if params.add_standard_columns: Command.start('Adding standard columns') rt = flex.reflection_table.empty_standard(len(table)) rt.update(table) table = rt # set variances to unity table['xyzobs.mm.variance'] = flex.vec3_double(len(table), (1,1,1)) table['xyzobs.px.variance'] = flex.vec3_double(len(table), (1,1,1)) Command.end('Standard columns added') # Output the table to pickle file if params.output.filename is None: params.output.filename = 'spot_xds.pickle' Command.start('Saving reflection table to %s' % params.output.filename) table.as_pickle(params.output.filename) Command.end('Saved reflection table to %s' % params.output.filename)
def _detect_outliers(self, cols): from scitbx.math import five_number_summary outliers = flex.bool(len(cols[0]), False) for col in cols: min_x, q1_x, med_x, q3_x, max_x = five_number_summary(col) iqr_x = q3_x - q1_x cut_x = self._iqr_multiplier * iqr_x outliers.set_selected(col > q3_x + cut_x, True) outliers.set_selected(col < q1_x - cut_x, True) return outliers
def __call__(self, d): ''' True if within powder ring. :param d: The resolution :return: True/False if within a powder ring ''' from dials.array_family import flex result = flex.bool(len(d), False) for filter in self: result = result | filter(d) return result
def tst_does_bbox_contain_bad_pixels(self): from dials.array_family import flex from dials.model.data import Shoebox from random import randint mask = flex.bool(flex.grid(100, 100), True) for j in range(100): for i in range(40, 60): mask[j,i] = False mask[i,j] = False shoebox = flex.shoebox(1000) res = flex.bool(1000) for i in range(1000): x0 = randint(0, 90) y0 = randint(0, 90) z0 = randint(0, 90) x1 = randint(1, 10) + x0 y1 = randint(1, 10) + y0 z1 = randint(1, 10) + z0 shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1)) res2 = False if x0 >= 40 and x0 < 60: res2 = True if x1 > 40 and x1 <= 60: res2 = True if y0 >= 40 and y0 < 60: res2 = True if y1 > 40 and y1 <= 60: res2 = True res[i] = res2 assert(shoebox.does_bbox_contain_bad_pixels(mask) == res) # Test passed print 'OK'
def __call__(self, d): ''' True if within powder ring. :param d: The resolution :return: True/False in powder ring ''' from dials.array_family import flex result = flex.bool(len(d), False) for d_spacing in self.d_spacings: result = result | (flex.abs(d - d_spacing) < self.half_width) return result
def _id_refs_to_keep(self, obs_data): """Create a selection of observations that pass certain conditions. This step includes rejection of reflections too close to the spindle, reflections measured outside the scan range, rejection of the (0,0,0) Miller index and rejection of reflections with the overload flag set. Outlier rejection is done later.""" # first exclude reflections with miller index set to 0,0,0 sel1 = obs_data['miller_index'] != (0,0,0) # exclude reflections with overloads, as these have worse centroids sel2 = ~obs_data.get_flags(obs_data.flags.overloaded) # combine selections sel = sel1 & sel2 inc = flex.size_t_range(len(obs_data)).select(sel) obs_data = obs_data.select(sel) # Default to True to pass the following test if there is no rotation axis # for a particular experiment to_keep = flex.bool(len(inc), True) for iexp, exp in enumerate(self._experiments): axis = self._axes[iexp] if not axis or exp.scan is None: continue if exp.scan.get_oscillation()[1] == 0.0: continue sel = obs_data['id'] == iexp s0 = self._s0vecs[iexp] s1 = obs_data['s1'].select(sel) phi = obs_data['xyzobs.mm.value'].parts()[2].select(sel) # first test: reject reflections for which the parallelepiped formed # between the gonio axis, s0 and s1 has a volume of less than the cutoff. # Those reflections are by definition closer to the spindle-beam # plane and for low values of the cutoff are troublesome to # integrate anyway. p_vol = flex.abs(s1.cross(flex.vec3_double(s1.size(), s0)).dot(axis)) passed1 = p_vol > self._close_to_spindle_cutoff # second test: reject reflections that lie outside the scan range phi_min, phi_max = exp.scan.get_oscillation_range(deg=False) passed2 = (phi >= phi_min) & (phi <= phi_max) # combine tests to_update = passed1 & passed2 to_keep.set_selected(sel, to_update) inc = inc.select(to_keep) return inc
def test1(): # work in a temporary directory cwd = os.path.abspath(os.curdir) tmp_dir = open_tmp_directory(suffix="test_dials_filter_reflections") os.chdir(tmp_dir) # make a dummy reflection table for the test, setting some flags from dials.array_family import flex rt = flex.reflection_table.empty_standard(6) rt["iobs"] = flex.size_t_range(len(rt)) mask1 = flex.bool([True] * 3 + [False] * 3) mask2 = flex.bool([True, False] * 3) rt.set_flags(mask1, rt.flags.integrated) rt.set_flags(mask2, rt.flags.bad_spot) rt_name = "test_refs.pickle" rt.as_pickle(rt_name) cmd = "dev.dials.filter_reflections " + rt_name + " inclusions.flag=integrated" + " exclusions.flag=bad_spot" print cmd try: result = easy_run.fully_buffered(command=cmd).raise_if_errors() # load results ref = flex.reflection_table.from_pickle("filtered.pickle") finally: os.chdir(cwd) # clean up tmp dir shutil.rmtree(tmp_dir) # The test selects only 1 reflection assert len(ref) == 1 assert list(ref["iobs"]) == [1] print "OK" return
def write_integration_pickles(self, integrated, experiments, callback = None): """ Write a serialized python dictionary with integrated intensities and other information suitible for use by cxi.merge or prime.postrefine. @param integrated Reflection table with integrated intensities @param experiments Experiment list. One integration pickle for each experiment will be created. @param callback Deriving classes can use callback to make further modifications to the dictionary before it is serialized. Callback should be a function with this signature: def functionname(params, outfile, frame), where params is the phil scope, outfile is the path to the pickle that will be saved, and frame is the python dictionary to be serialized. """ try: picklefilename = self.params.output.integration_pickle except AttributeError: return if self.params.output.integration_pickle is not None: from libtbx import easy_pickle import os from xfel.command_line.frame_extractor import ConstructFrame from dials.array_family import flex # Split everything into separate experiments for pickling for e_number in xrange(len(experiments)): experiment = experiments[e_number] e_selection = flex.bool( [r['id']==e_number for r in integrated]) reflections = integrated.select(e_selection) frame = ConstructFrame(reflections, experiment).make_frame() frame["pixel_size"] = experiment.detector[0].get_pixel_size()[0] try: # if the data was a file on disc, get the path event_timestamp = os.path.splitext(experiments[0].imageset.paths()[0])[0] except NotImplementedError: # if the data is in memory only, check if the reader set a timestamp on the format object event_timestamp = experiment.imageset.reader().get_format(0).timestamp event_timestamp = os.path.basename(event_timestamp) if event_timestamp.find("shot-")==0: event_timestamp = os.path.splitext(event_timestamp)[0] # micromanage the file name if hasattr(self.params.output, "output_dir"): outfile = os.path.join(self.params.output.output_dir, self.params.output.integration_pickle%(e_number,event_timestamp)) else: outfile = os.path.join(os.path.dirname(self.params.output.integration_pickle), self.params.output.integration_pickle%(e_number,event_timestamp)) if callback is not None: callback(self.params, outfile, frame) easy_pickle.dump(outfile, frame)
def __call__(self, d): ''' True if within powder ring. :param d: The resolution :return: True/False in powder ring ''' from dials.array_family import flex result = flex.bool(len(d), False) d2 = 1.0 / d**2 for ice_ring in self.ice_rings: result = result | (d2 >= ice_ring[0]) & (d2 <= ice_ring[1]) return result
def hot_pixel_mask(imageset, reflections): depth = imageset.get_array_range()[1] - imageset.get_array_range()[0] xylist = filter_reflections(reflections, depth) from dials.array_family import flex mask = flex.bool(flex.grid(reversed(imageset.get_image_size())), True) for x, y in xylist: mask[y, x] = False print 'Found %d hot pixels' % len(xylist) return (mask,)
def __init__(self): from dials.array_family import flex self.reflections = flex.reflection_table() self.reflections['panel'] = flex.size_t() self.reflections['bbox'] = flex.int6() self.reflections['miller_index'] = flex.miller_index() self.reflections['s1'] = flex.vec3_double() self.reflections['xyzcal.px'] = flex.vec3_double() self.reflections['xyzcal.mm'] = flex.vec3_double() self.reflections['entering'] = flex.bool() self.reflections['id'] = flex.int() self.reflections["flags"] = flex.size_t() self.npanels = 2 self.width = 1000 self.height = 1000 self.nrefl = 10000 self.array_range = (0, 130) self.block_size = 20 from random import randint, seed, choice seed(0) self.processed = [[] for i in range(12)] for i in range(self.nrefl): x0 = randint(0, self.width-10) y0 = randint(0, self.height-10) zs = randint(2, 9) x1 = x0 + randint(2, 10) y1 = y0 + randint(2, 10) for k, j in enumerate([10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]): m = k + i * 12 pos = choice(["left", "right", "centre"]) if pos == 'left': z0 = j - zs z1 = j elif pos == 'right': z0 = j z1 = j + zs else: z0 = j - zs // 2 z1 = j + zs // 2 bbox = (x0, x1, y0, y1, z0, z1) self.reflections.append({ "panel" : randint(0,1), "bbox" : bbox, "flags" : flex.reflection_table.flags.reference_spot }) self.processed[k].append(m)
def tst_consistent(self): from random import randint from dials.model.data import Shoebox from dials.array_family import flex shoebox = flex.shoebox(10) for i in range(10): x0 = randint(0, 1000) y0 = randint(0, 1000) z0 = randint(0, 1000) x1 = randint(1, 10) + x0 y1 = randint(1, 10) + y0 z1 = randint(1, 10) + z0 shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1)) assert(shoebox.is_consistent() == flex.bool(10, False)) for i in range(10): shoebox[i].allocate() assert(shoebox.is_consistent() == flex.bool(10, True)) for i in [0, 2, 4, 6, 8]: shoebox[i].data.resize(flex.grid(10, 10, 10)) assert(shoebox.is_consistent() == flex.bool([False, True] * 5)) for i in range(10): shoebox[i].deallocate() assert(shoebox.is_consistent() == flex.bool(10, False)) # Test passed print 'OK'
def filter_integrated_centroids(reflections): '''Filter reflections to include only those with the integrated and the strong flag set, but only if there are apparently some integrated reflections''' orig_len = len(reflections) inc = flex.bool(orig_len, False) mask = reflections.get_flags(reflections.flags.integrated) if mask.count(True) == 0: return reflections reflections = reflections.select(mask) mask = reflections.get_flags(reflections.flags.strong) reflections = reflections.select(mask) logger.info('{0} out of {1} reflections remain after filtering to keep only strong' ' and integrated centroids'.format(len(reflections), orig_len)) return reflections
def d2f(I): mask = flex.bool(flex.grid(9,9,9), False) for k in range(9): for j in range(9): for i in range(9): dx = 5 * (i - 4.5) / 4.5 dy = 5 * (j - 4.5) / 4.5 dz = 5 * (k - 4.5) / 4.5 dd = sqrt(dx**2 + dy**2 + dz**2) if dd <= 3: mask[k,j,i] = True mask = mask.as_1d() & (ref_P.as_1d() > 0) p = ref_P.as_1d().select(mask) c = max_P.as_1d().select(mask) return flex.sum(2*c*c*p*p / (p*I)**3)
def load_inputs(self, pandas_table, miller_data=None, refls_key='predictions'): """ :param pandas_table: contains path to the experiments (pandas column exp_name) to be loaded the pandas table is expected to have been written by diffBragg.hopper or diffBragg.hopper_process . See method save_to_pandas in simtbx/command_line/hopper.py For example, if the outputdir of diffBragg.hopper was set to `all_shots`, then there should be a golder all_shots/pandas created which contains all of the per-shot pandas dataframes. They should be concatenated as follows, forming a suitable argument for this method >> import glob,pandas >> fnames = glob.glob("all_shots/pandas/rank*/*pkl") >> df = pandas.concat([ pandas.read_pickle(f) for f in fnames]) >> df.reset_index(inplace=True, drop=True) >> df.to_pickle("all_shots.pkl") >> # Then later, as part of an MPI application, the following will load all data: >> RefineLauncher_instance.load_inputs(df, refls_key="stage1_refls") :param miller_data: Optional miller array for the structure factor component of the model :param refls_key: key specifying the reflection tables in the pandas table Modeled pixels will lie in shoeboxes centered on each x,y,z in xyzobs.px.value :return: """ COMM.Barrier() num_exp = len(pandas_table) first_exper_file = pandas_table.exp_name.values[0] detector = ExperimentListFactory.from_json_file(first_exper_file, check_format=False)[0].detector if detector is None and self.params.refiner.reference_geom is None: raise RuntimeError("No detector in experiment, must provide a reference geom.") # TODO verify all shots have the same detector ? if self.params.refiner.reference_geom is not None: detector = ExperimentListFactory.from_json_file(self.params.refiner.reference_geom, check_format=False)[0].detector print("Using reference geom from expt %s" % self.params.refiner.reference_geom) if COMM.size > num_exp: raise ValueError("Requested %d MPI ranks to process %d shots. Reduce number of ranks to %d" % (COMM.size, num_exp, num_exp)) self._init_panel_group_information(detector) self.verbose = False if COMM.rank == 0: self.verbose = self.params.refiner.verbose > 0 if self.params.refiner.gather_dir is not None and not os.path.exists(self.params.refiner.gather_dir): os.makedirs(self.params.refiner.gather_dir) LOGGER.info("MADE GATHER DIR %s" % self.params.refiner.gather_dir) COMM.barrier() shot_idx = 0 # each rank keeps index of the shots local to it rank_panel_groups_refined = set() exper_names = pandas_table.exp_name assert len(exper_names) == len(set(exper_names)) # TODO assert all exper are single-file, probably way before this point LOGGER.info("EVENT: begin loading inputs") for i_exp, exper_name in enumerate(exper_names): if i_exp % COMM.size != COMM.rank: continue LOGGER.info("EVENT: BEGIN loading experiment list") expt_list = ExperimentListFactory.from_json_file(exper_name, check_format=self.params.refiner.check_expt_format) LOGGER.info("EVENT: DONE loading experiment list") if len(expt_list) != 1: print("Input experiments need to have length 1, %s does not" % exper_name) expt = expt_list[0] expt.detector = detector # in case of supplied ref geom self._check_experiment_integrity(expt) exper_dataframe = pandas_table.query("exp_name=='%s'" % exper_name) refl_name = exper_dataframe[refls_key].values[0] refls = flex.reflection_table.from_file(refl_name) # FIXME need to remove (0,0,0) bboxes good_sel = flex.bool([h != (0, 0, 0) for h in list(refls["miller_index"])]) refls = refls.select(good_sel) #UcellMan = utils.manager_from_crystal(expt.crystal) opt_uc_param = exper_dataframe[["a","b","c","al","be","ga"]].values[0] UcellMan = utils.manager_from_params(opt_uc_param) if self.symbol is None: if self.params.refiner.force_symbol is not None: self.symbol = self.params.refiner.force_symbol else: self.symbol = expt.crystal.get_space_group().type().lookup_symbol() else: if self.params.refiner.force_symbol is None: if expt.crystal.get_space_group().type().lookup_symbol() != self.symbol: raise ValueError("Crystals should all have the same space group symmetry") if shot_idx == 0: # each rank initializes a simulator only once if self.params.simulator.init_scale != 1: print("WARNING: For stage_two , it is assumed that total scale is stored in the pandas dataframe") print("WARNING: resetting params.simulator.init_scale to 1!") self.params.simulator.init_scale = 1 self._init_simulator(expt, miller_data) if self.params.profile: self.SIM.record_timings = True if self.params.refiner.stage_two.Fref_mtzname is not None: self.Fref = utils.open_mtz(self.params.refiner.stage_two.Fref_mtzname, self.params.refiner.stage_two.Fref_mtzcol) LOGGER.info("EVENT: LOADING ROI DATA") shot_modeler = hopper_utils.DataModeler(self.params) if self.params.refiner.load_data_from_refl: gathered = shot_modeler.GatherFromReflectionTable(expt, refls, sg_symbol=self.symbol) else: gathered = shot_modeler.GatherFromExperiment(expt, refls, sg_symbol=self.symbol) if not gathered: raise IOError("Failed to gather data from experiment %s", exper_name) if self.params.refiner.gather_dir is not None: gathered_name = os.path.splitext(os.path.basename(exper_name))[0] gathered_name += "_withData.refl" gathered_name = os.path.join(self.params.refiner.gather_dir, gathered_name ) shot_modeler.dump_gathered_to_refl(gathered_name, do_xyobs_sanity_check=False) #True) LOGGER.info("SAVED ROI DATA TO %s" % gathered_name) if self.params.refiner.test_gathered_file: all_data = shot_modeler.all_data.copy() all_roi_id = shot_modeler.roi_id.copy() all_bg = shot_modeler.all_background.copy() all_trusted = shot_modeler.all_trusted.copy() all_pids = np.array(shot_modeler.pids) all_rois = np.array(shot_modeler.rois) new_Modeler = hopper_utils.DataModeler(self.params) assert new_Modeler.GatherFromReflectionTable(exper_name, gathered_name, sg_symbol=self.symbol) assert np.allclose(new_Modeler.all_data, all_data) assert np.allclose(new_Modeler.all_background, all_bg) assert np.allclose(new_Modeler.rois, all_rois) assert np.allclose(new_Modeler.pids, all_pids) assert np.allclose(new_Modeler.all_trusted, all_trusted) assert np.allclose(new_Modeler.roi_id, all_roi_id) LOGGER.info("Gathered file approved!") self.Hi[shot_idx] = shot_modeler.Hi self.Hi_asu[shot_idx] = shot_modeler.Hi_asu LOGGER.info("EVENT: DONE LOADING ROI") shot_modeler.ucell_man = UcellMan self.SIM.num_ucell_param = len(shot_modeler.ucell_man.variables) # for convenience loaded_spectra = False if self.params.spectrum_from_imageset: try: shot_spectra = hopper_utils.downsamp_spec(self.SIM, self.params, expt, return_and_dont_set=True) loaded_spectra = True except Exception as err: LOGGER.warning("spectrum_from_imageset is set to True, however failed to load spectra: %s" % err) loaded_spectra = False if not loaded_spectra: if "spectrum_filename" in list(exper_dataframe) and exper_dataframe.spectrum_filename.values[0] is not None: shot_spectra = utils.load_spectra_from_dataframe(exper_dataframe) LOGGER.debug("Loaded specta from %s" % exper_dataframe.spectrum_filename.values[0]) else: total_flux = exper_dataframe.total_flux.values[0] if total_flux is None: total_flux = self.params.simulator.total_flux shot_spectra = [(expt.beam.get_wavelength(), total_flux)] shot_modeler.spectra = shot_spectra if self.params.refiner.gather_dir is not None and not self.params.refiner.load_data_from_refl: spec_wave, spec_weights = map(np.array, zip(*shot_spectra)) spec_filename = os.path.splitext(os.path.basename(exper_name))[0] spec_filename = os.path.join(self.params.refiner.gather_dir, spec_filename+".lam") utils.save_spectra_file(spec_filename, spec_wave, spec_weights) LOGGER.info("saved spectra filename %s" % spec_filename) LOGGER.info("Will simulate %d energy channels" % len(shot_spectra)) if "detz_shift_mm" in list(exper_dataframe): shot_modeler.originZ_init = exper_dataframe.detz_shift_mm.values[0]*1e-3 else: shot_modeler.originZ_init = 0 shot_modeler.exper_name = exper_name shot_modeler.refl_name = refl_name shot_panel_groups_refined = self.determine_refined_panel_groups(shot_modeler.pids) rank_panel_groups_refined = rank_panel_groups_refined.union(set(shot_panel_groups_refined)) shot_idx += 1 if COMM.rank == 0: self._mem_usage() print("Finished loading image %d / %d" % (i_exp+1, len(exper_names)), flush=True) shot_modeler.PAR = PAR_from_params(self.params, expt, best=exper_dataframe) self.Modelers[i_exp] = shot_modeler LOGGER.info("DONE LOADING DATA; ENTER BARRIER") COMM.Barrier() LOGGER.info("DONE LOADING DATA; EXIT BARRIER") self.shot_roi_darkRMS = None # TODO warn that per_spot_scale refinement not intended for ensemble mode all_refined_groups = COMM.gather(rank_panel_groups_refined) panel_groups_refined = None if COMM.rank == 0: panel_groups_refined = set() for set_of_panels in all_refined_groups: panel_groups_refined = panel_groups_refined.union(set_of_panels) self.panel_groups_refined = list(COMM.bcast(panel_groups_refined)) LOGGER.info("EVENT: Gathering global HKL information") self._gather_Hi_information() LOGGER.info("EVENT: FINISHED gather global HKL information") if self.params.roi.cache_dir_only: print("Done creating cache directory and cache_dir_only=True, so goodbye.") sys.exit() # in case of GPU LOGGER.info("BEGIN DETERMINE MAX PIX") self.NPIX_TO_ALLOC = self._determine_per_rank_max_num_pix() # TODO in case of randomize devices, shouldnt this be total max across all ranks? n = COMM.gather(self.NPIX_TO_ALLOC) if COMM.rank == 0: n = max(n) self.NPIX_TO_ALLOC = COMM.bcast(n) LOGGER.info("DONE DETERMINE MAX PIX") self.DEVICE_ID = COMM.rank % self.params.refiner.num_devices self._mem_usage()
def generate_simple_table(prf=True): """Generate a reflection table for testing intensity combination. The numbers are contrived to make sum intensities agree well at high intensity but terribly at low and vice versa for profile intensities.""" reflections = flex.reflection_table() reflections["miller_index"] = flex.miller_index( [ (0, 0, 1), (0, 0, 1), (0, 0, 1), (0, 0, 1), (0, 0, 1), (0, 0, 2), (0, 0, 2), (0, 0, 2), (0, 0, 2), (0, 0, 2), (0, 0, 3), (0, 0, 3), (0, 0, 3), (0, 0, 3), (0, 0, 3), (0, 0, 4), (0, 0, 4), (0, 0, 4), (0, 0, 4), (0, 0, 4), (0, 0, 5), (0, 0, 5), (0, 0, 5), (0, 0, 5), (0, 0, 5), ] ) reflections["inverse_scale_factor"] = flex.double(25, 1.0) # Contrive an example that should give the best cc12 when combined. # make sum intensities agree well at high intensity but terribly at low # and vice versa for profile intensities. # profile less consistent at high intensity here # sumless consistent at low intensity here reflections["intensity.sum.value"] = flex.double( [ 10000.0, 11000.0, 9000.0, 8000.0, 12000.0, 500.0, 5600.0, 5500.0, 2000.0, 6000.0, 100.0, 50.0, 150.0, 75.0, 125.0, 30.0, 10.0, 2.0, 35.0, 79.0, 1.0, 10.0, 20.0, 10.0, 5.0, ] ) reflections["intensity.sum.variance"] = flex.double( [10000] * 5 + [5000] * 5 + [100] * 5 + [30] * 5 + [10] * 5 ) reflections.set_flags(flex.bool(25, False), reflections.flags.outlier_in_scaling) reflections.set_flags(flex.bool(25, True), reflections.flags.integrated) reflections["lp"] = flex.double(25, 0.5) if prf: reflections["intensity.prf.value"] = flex.double( [ 10000.0, 16000.0, 12000.0, 6000.0, 9000.0, 5000.0, 2000.0, 1500.0, 1300.0, 9000.0, 100.0, 80.0, 120.0, 90.0, 100.0, 30.0, 40.0, 50.0, 30.0, 30.0, 10.0, 12.0, 9.0, 8.0, 10.0, ] ) reflections["intensity.prf.variance"] = flex.double( [1000] * 5 + [500] * 5 + [10] * 5 + [3] * 5 + [1] * 5 ) reflections = calculate_prescaling_correction(reflections) return reflections
def read(handle, key): from dials.array_family import flex from dxtbx.format.nexus import convert_units import numpy as np if key == 'miller_index': h = flex.int(handle['h'][:].astype(np.int32)) k = flex.int(handle['k'][:].astype(np.int32)) l = flex.int(handle['l'][:].astype(np.int32)) return flex.miller_index(h, k, l) elif key == 'id': return flex.int(handle['id'][:].astype(int)) elif key == 'partial_id': return flex.size_t(handle['reflection_id'][:].astype(int)) elif key == 'entering': return flex.bool(handle['entering'][:]) elif key == 'flags': return flex.size_t(handle['flags'][:].astype(int)) elif key == 'panel': return flex.size_t(handle['det_module'][:].astype(int)) elif key == 'd': return flex.double(handle['d'][:]) elif key == 'partiality': return flex.double(handle['partiality'][:]) elif key == 'xyzcal.px': x = flex.double(handle['predicted_px_x'][:]) y = flex.double(handle['predicted_px_y'][:]) z = flex.double(handle['predicted_frame'][:]) return flex.vec3_double(x, y, z) elif key == 'xyzcal.mm': x = convert_units(flex.double(handle['predicted_x'][:]), handle['predicted_x'].attrs['units'], 'mm') y = convert_units(flex.double(handle['predicted_y'][:]), handle['predicted_y'].attrs['units'], 'mm') z = convert_units(flex.double(handle['predicted_phi'][:]), handle['predicted_phi'].attrs['units'], 'rad') return flex.vec3_double(x, y, z) elif key == 'bbox': b = flex.int(handle['bounding_box'][:].astype(np.int32)) return flex.int6(b.as_1d()) elif key == 'xyzobs.px.value': x = flex.double(handle['observed_px_x'][:]) y = flex.double(handle['observed_px_y'][:]) z = flex.double(handle['observed_frame'][:]) return flex.vec3_double(x, y, z) elif key == 'xyzobs.px.variance': x = flex.double(handle['observed_px_x_var'][:]) y = flex.double(handle['observed_px_y_var'][:]) z = flex.double(handle['observed_frame_var'][:]) return flex.vec3_double(x, y, z) elif key == 'xyzobs.mm.value': x = convert_units(flex.double(handle['observed_x'][:]), handle['observed_x'].attrs['units'], 'mm') y = convert_units(flex.double(handle['observed_y'][:]), handle['observed_y'].attrs['units'], 'mm') z = convert_units(flex.double(handle['observed_phi'][:]), handle['observed_phi'].attrs['units'], 'rad') return flex.vec3_double(x, y, z) elif key == 'xyzobs.mm.variance': x = convert_units(flex.double(handle['observed_x_var'][:]), handle['observed_x_var'].attrs['units'], 'mm') y = convert_units(flex.double(handle['observed_y_var'][:]), handle['observed_y_var'].attrs['units'], 'mm') z = convert_units(flex.double(handle['observed_phi_var'][:]), handle['observed_phi_var'].attrs['units'], 'rad') return flex.vec3_double(x, y, z) elif key == 'background.mean': return flex.double(handle['background_mean'][:]) elif key == 'intensity.sum.value': return flex.double(handle['int_sum'][:]) elif key == 'intensity.sum.variance': return flex.double(handle['int_sum_var'][:]) elif key == 'intensity.prf.value': return flex.double(handle['int_prf'][:]) elif key == 'intensity.prf.variance': return flex.double(handle['int_prf_var'][:]) elif key == 'profile.correlation': return flex.double(handle['prf_cc'][:]) elif key == 'lp': return flex.double(handle['lp'][:]) elif key == 'num_pixels.background': return flex.int(handle['num_bg'][:].astype(np.int32)) elif key == 'num_pixels.background_used': return flex.int(handle['num_bg_used'][:].astype(np.int32)) elif key == 'num_pixels.foreground': return flex.int(handle['num_fg'][:].astype(np.int32)) elif key == 'num_pixels.valid': return flex.int(handle['num_valid'][:].astype(np.int32)) elif key == 'profile.rmsd': return flex.double(handle['prf_rmsd'][:]) else: raise KeyError('Column %s not read from file' % key)
def _multiplicity_mean_error_stddev( self, calculate_variances=False, keep_singles=False ): """ " Calculate aggregate properties of grouped symmetry-equivalent reflections. Populate the reflection table of observations with the following properties: * ``multiplicity`` — Multiplicity of observations of a given reflection in the asymmetric unit; :type: `dials.array_family_flex_ext.int` array * ``intensity.mean.value`` — Mean of symmetry-equivalent reflections, weighted by measurement error; :type: `dials.array_family_flex_ext.double` array * ``intensity.mean.std_error`` — Standard error on the weighted mean; :type: `dials.array_family_flex_ext.double` array * (optional) ``intensity.mean.variance`` — variance of symmetry-equivalent reflections, weighted by measurement error; :type: `dials.array_family_flex_ext.double` array :param calculate_variances: Elect whether to calculate the weighted variances. Defaults to False, to spare an expensive computation. :type calculate_variances: bool :param keep_singles: Choose whether to keep single-multiplicity reflections. :type keep_singles: bool """ for key, rtable in self.rtables.items(): # Sort the reflection table for speedier iteration. rtable.sort("miller_index.asu") # Record the positions of any multiplicity-1 reflections. if not keep_singles: singles = flex.size_t() # Record the multiplicities. multiplicity = flex.int() # For weighted averaging. weights = 1 / rtable["intensity.sum.variance"] sum_weights = flex.double() if calculate_variances: sum_square_weights = flex.double() # Calculate the weighted mean intensities. i_means = flex.double() # Calculate the standard deviations from unbiased weighted variances. variances = flex.double() # Iterate over the reflections, grouping by equivalent Miller index, # to calculate multiplicities, weighted mean intensities, etc.. # Some time can be saved by only calculating variances if necessary. # Initial values: prev_index = None count = 1 # The following will be set during loop iteration i_sum, sum_weight, sum_square_weight = None, None, None # One big loop through the entire reflection table: for j in range(rtable.size()): index = rtable["miller_index.asu"][j] weight = weights[j] # Aggregate within a symmetry-equivalent group of reflections: if index == prev_index: count += 1 i_sum += weight * rtable["intensity.sum.value"][j] sum_weight += weight if calculate_variances: sum_square_weight += weight * weight # Record the aggregated values for the group: elif prev_index: if count == 1 and not keep_singles: singles.append(j - 1) multiplicity.extend(flex.int(count, count)) i_means.extend(flex.double(count, i_sum / sum_weight)) sum_weights.extend(flex.double(count, sum_weight)) if calculate_variances: sum_square_weights.extend(flex.double(count, sum_square_weight)) # And reinitialise: prev_index = index count = 1 i_sum = weight * rtable["intensity.sum.value"][j] sum_weight = weight if calculate_variances: sum_square_weight = weight * weight # Handle the first row: else: prev_index = rtable["miller_index.asu"][j] i_sum = weight * rtable["intensity.sum.value"][j] sum_weight = weight if calculate_variances: sum_square_weight = weight * weight # Record the aggregated values for the last group: if count == 1 and not keep_singles: singles.append(rtable.size() - 1) multiplicity.extend(flex.int(count, count)) i_means.extend(flex.double(count, i_sum / sum_weight)) sum_weights.extend(flex.double(count, sum_weight)) if calculate_variances: sum_square_weights.extend(flex.double(count, sum_square_weight)) # Discard singletons: if not keep_singles: singles_del = flex.bool(rtable.size(), True) singles_del.set_selected(singles, False) multiplicity, weights, sum_weights, i_means = [ a.select(singles_del) for a in (multiplicity, weights, sum_weights, i_means) ] rtable.del_selected(singles) if calculate_variances: sum_square_weights = sum_square_weights.select(singles_del) # Record the multiplicities in the reflection table. rtable["multiplicity"] = multiplicity # Record the weighted mean intensities in the reflection table. rtable["intensity.mean.value"] = i_means # Record the standard errors on the means in the reflection table. rtable["intensity.mean.std_error"] = flex.sqrt(1 / sum_weights) if calculate_variances: # Initialise values: prev_index = None weighted_sum_square_residual = None for j in range(rtable.size()): index = rtable["miller_index.asu"][j] weight = weights[j] residual = rtable["intensity.sum.value"][j] - i_means[j] # Aggregate within a symmetry-equivalent group of reflections: if index == prev_index: count += 1 weighted_sum_square_residual += weight * residual * residual # Record the aggregated value for the group: elif prev_index: # The weighted variance is undefined for multiplicity=1, # use the measured variance instead in this case. if count == 1: variances.append(rtable["intensity.sum.variance"][j - 1]) else: sum_weight = sum_weights[j - 1] var_weight = 1 / ( sum_weight - sum_square_weights[j - 1] / sum_weight ) variances.extend( flex.double( count, weighted_sum_square_residual * var_weight ) ) # Reinitialise: prev_index = index count = 1 weighted_sum_square_residual = weight * residual * residual # Handle the first row: else: prev_index = rtable["miller_index.asu"][j] count = 1 weighted_sum_square_residual = weight * residual * residual # Record the aggregated values for the last group: # The weighted variance is undefined for multiplicity=1, # use the measured variance instead in this case. if count == 1: variances.append(rtable["intensity.sum.variance"][-1]) else: sum_weight = sum_weights[-1] var_weight = 1 / (sum_weight - sum_square_weights[-1] / sum_weight) variances.extend( flex.double(count, weighted_sum_square_residual * var_weight) ) # Record the variances in the reflection table. rtable["intensity.mean.variance"] = variances self.rtables[key] = rtable
def test_combine_intensities(test_exp_P1): """Test the combine intensities function for a single dataset""" reflections = generate_simple_table() scaler = Mock() scaler.reflection_table = reflections scaler.suitable_refl_for_scaling_sel = flex.bool(reflections.size(), True) scaler.outliers = flex.bool(reflections.size(), False) scaler.experiment = test_exp_P1 scaler.space_group = test_exp_P1.crystal.get_space_group() scaler.params.reflection_selection.combine.Imid = None combiner = SingleDatasetIntensityCombiner(scaler) Imid = combiner.max_key intensity, variance = combiner.calculate_suitable_combined_intensities() # Imid being 1200.0 should be best for this contrived example assert Imid == 1200.0 # Due to nature of crossover, just require 2% tolerance for this example assert list(intensity[0:5]) == pytest.approx( list( reflections["intensity.sum.value"][0:5] * reflections["prescaling_correction"][0:5] ), rel=2e-2, ) assert list(intensity[20:25]) == pytest.approx( list( reflections["intensity.prf.value"][20:25] * reflections["prescaling_correction"][20:25] ), rel=2e-2, ) assert list(variance[0:5]) == pytest.approx( list( reflections["intensity.sum.variance"][0:5] * reflections["prescaling_correction"][0:5] ** 2 ), rel=2e-2, ) assert list(variance[20:25]) == pytest.approx( list( reflections["intensity.prf.variance"][20:25] * reflections["prescaling_correction"][20:25] ** 2 ), rel=2e-2, ) combiner.max_key = 0 # prf intensity, variance = combiner.calculate_suitable_combined_intensities() assert list(intensity) == pytest.approx( list(reflections["intensity.prf.value"] * reflections["prescaling_correction"]), rel=2e-2, ) assert list(variance) == pytest.approx( list( reflections["intensity.prf.variance"] * reflections["prescaling_correction"] ** 2 ), rel=2e-2, ) combiner.max_key = 1 # sum intensity, variance = combiner.calculate_suitable_combined_intensities() assert list(intensity) == pytest.approx( list(reflections["intensity.sum.value"] * reflections["prescaling_correction"]), rel=2e-2, ) assert list(variance) == pytest.approx( list( reflections["intensity.sum.variance"] * reflections["prescaling_correction"] ** 2 ), rel=2e-2, )