def plot_valid(b, s):
from dials.array_family import flex
b = [0.1, 0.2, 0.3, 0.4, 0.5]
s = [0.1, 0.3, 0.5, 0.3, 0.1]
v1 = flex.bool(flex.grid(100, 100))
v2 = flex.bool(flex.grid(100, 100))
v3 = flex.bool(flex.grid(100, 100))
r = [float(ss) / float(bb) for ss, bb in zip(s, b)]
for BB in range(0, 100):
for SS in range(0, 100):
B = -5.0 + BB / 10.0
S = -5.0 + SS / 10.0
V1 = True
V2 = True
V3 = True
for i in range(len(b)):
if B*b[i]+S*s[i] <= 0:
V1 = False
break
for i in range(len(b)):
if B*b[i] <= -S*s[i]:
V2 = False
break
v1[BB,SS] = V1
v2[BB,SS] = V2
from matplotlib import pylab
pylab.imshow(v1.as_numpy_array())
pylab.show()
exit(0)
def show_image(c,b,s, BB=None, SS=None):
import numpy.ma
from dials.array_family import flex
N = 100
im = flex.double(flex.grid(N, N))
mask = flex.bool(flex.grid(N, N))
for j in range(N):
for i in range(N):
B = -1.0 + j * 10.0 / N
S = -1.0 + i * 10.0 / N
im[j,i], mask[j,i] = func(c,b,s,B,S)
im[j,i] = -im[j,i]
masked_im = numpy.ma.array(
# im.as_numpy_array(),
flex.exp(im).as_numpy_array(),
mask = mask.as_numpy_array())
mask2 = flex.bool(flex.grid(N, N))
indices = []
for i in range(len(mask)):
if mask[i] == False:
indices.append(i)
indices = flex.size_t(indices)
ind = flex.max_index(im.select(indices))
ind = indices[ind]
maxy = -1.0 + (ind % N) * 10.0 / N
maxx = -1.0 + (ind // N) * 10.0 / N
from matplotlib import pylab
pylab.imshow(masked_im, origin='bottom', extent=[-1.0, 9.0, -1.0, 9.0])
if YY is not None and XX is not None:
pylab.plot(YY, XX)
pylab.scatter([maxy], [maxx])
pylab.show()
def filter_frames(self, data):
data = data[0]
lp = gaussian_filter(data, 100)
hp = data - lp # poormans background subtraction
hp -= np.min(hp)
sh = hp.shape
print "here"
hp = hp.astype('uint32')
hp = flex.int(hp)
print "here now"
mask = flex.bool(np.ones_like(hp).astype('bool'))
print "here now"
result1 = flex.bool(np.zeros_like(hp).astype('bool'))
spots = np.zeros_like(hp).astype('bool')
print "here now"
for i in range(3, self.parameters['spotsize'], 5):
print "here now"
algorithm = DispersionThreshold(sh, (i, i), 1, 1, 0, -1)
print "here now"
print type(hp), type(mask), type(result1)
thing = algorithm(hp, mask, result1)
print "here now"
spots = spots + result1.as_numpy_array()
return [data, spots*data]
def run(self):
'''Execute the script.'''
from dials.array_family import flex
from dials.util.options import flatten_reflections
from libtbx.utils import Sorry
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
if len(reflections) == 0:
self.parser.print_help()
raise Sorry('No valid reflection file given')
if len(reflections) != 1:
self.parser.print_help()
raise Sorry('Exactly 1 reflection file must be specified')
reflections = reflections[0]
print "{0} reflections loaded".format(len(reflections))
if len(params.inclusions.flag) == 0:
self.parser.print_help()
raise Sorry('No inclusion criteria given')
# Build up the initial inclusion selection
inc = flex.bool(len(reflections))
for flag in params.inclusions.flag:
sel = reflections.get_flags(getattr(reflections.flags, flag))
inc = inc | sel
reflections = reflections.select(inc)
print "{0} reflections selected to form the working set".format(len(reflections))
# Make requested exclusions from the current selection
exc = flex.bool(len(reflections))
for flag in params.exclusions.flag:
print flag
sel = reflections.get_flags(getattr(reflections.flags, flag))
exc = exc | sel
reflections = reflections.select(~exc)
print "{0} reflections excluded from the working set".format(exc.count(True))
# Save filtered reflections to file
if params.output.reflections:
print "Saving {0} reflections to {1}".format(len(reflections),
params.output.reflections)
reflections.as_pickle(params.output.reflections)
return
def process_reference(reference):
''' Load the reference spots. '''
from dials.util.command_line import Command
from dials.array_family import flex
if reference is None:
return None
assert("miller_index" in reference)
Command.start('Removing reference spots with invalid coordinates')
mask = flex.bool([x == (0, 0, 0) for x in reference['xyzcal.mm']])
reference.del_selected(mask)
mask = flex.bool([h == (0, 0, 0) for h in reference['miller_index']])
reference.del_selected(mask)
Command.end('Removed reference spots with invalid coordinates, %d remaining' %
len(reference))
return reference
def _create_hot_mask(self, imageset, hot_pixels):
'''
Find hot pixels in images
'''
from dials.array_family import flex
# Write the hot mask
if self.write_hot_mask:
# Create the hot pixel mask
hot_mask = tuple(flex.bool(flex.grid(p.get_image_size()[::-1]), True)
for p in imageset.get_detector())
num_hot = 0
if hot_pixels > 0:
for hp, hm in zip(hot_pixels, hot_mask):
for i in range(len(hp)):
hm[hp[i]] = False
num_hot += len(hp)
logger.info('Found %d possible hot pixel(s)' % num_hot)
else:
hot_mask = None
# Return the hot mask
return hot_mask
def tst_for_dataset(self, creator, filename):
from dials.array_family import flex
from dials.algorithms.shoebox import MaskCode
print filename
rlist = flex.reflection_table.from_pickle(filename)
shoebox = rlist['shoebox']
background = [sb.background.deep_copy() for sb in shoebox]
success = creator(shoebox)
assert(success.count(True) == len(success))
diff = []
for i in range(len(rlist)):
mask = flex.bool([(m & MaskCode.Foreground) != 0 for m in shoebox[i].mask])
px1 = background[i].select(mask)
px2 = shoebox[i].background.select(mask)
den = max([flex.mean(px1), 1.0])
diff.append(flex.mean(px2 - px1) / den)
diff = flex.double(diff)
mv = flex.mean_and_variance(flex.double(diff))
mean = mv.mean()
sdev = mv.unweighted_sample_standard_deviation()
try:
assert(abs(mean) < 0.01)
except Exception:
print "Mean: %f, Sdev: %f", mean, sdev
from matplotlib import pylab
pylab.hist(diff)
pylab.show()
raise
def plot_prob_for_zero(c, b, s):
from math import log, exp, factorial
from dials.array_family import flex
L = flex.double(flex.grid(100, 100))
MASK = flex.bool(flex.grid(100, 100))
c = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]
b = [bb / sum(b) for bb in b]
s = [ss / sum(s) for ss in s]
for BB in range(0, 100):
for SS in range(0, 100):
B = 0 + BB / 10000.0
S = 0 + SS / 40.0
LL = 0
for i in range(len(b)):
if B*b[i] + S*s[i] <= 0:
MASK[BB, SS] = True
LL = -999999
break
else:
LL += c[i]*log(B*b[i]+S*s[i]) - log(factorial(c[i])) - B*b[i] - S*s[i]
L[BB, SS] = LL
index = flex.max_index(L)
i = index % 100
j = index // 100
B = 0 + j / 10000.0
S = 0 + i / 40.0
print flex.max(L), B, S
from matplotlib import pylab
import numpy
im = numpy.ma.masked_array(flex.exp(L).as_numpy_array(), mask=MASK.as_numpy_array())
pylab.imshow(im)
pylab.show()
exit(0)
def generate_reflections(self):
"""Use reeke_model to generate indices of reflections near to the Ewald
sphere that might be observed on a still image. Build a reflection_table
of these."""
from cctbx.sgtbx import space_group_info
space_group_type = space_group_info("P 1").group().type()
# create a ReekeIndexGenerator
UB = self.crystal.get_U() * self.crystal.get_B()
axis = self.goniometer.get_rotation_axis()
s0 = self.beam.get_s0()
dmin = 1.5
# use the same UB at the beginning and end - the margin parameter ensures
# we still have indices close to the Ewald sphere generated
from dials.algorithms.spot_prediction import ReekeIndexGenerator
r = ReekeIndexGenerator(UB, UB, space_group_type, axis, s0, dmin=1.5, margin=1)
# generate indices
hkl = r.to_array()
nref = len(hkl)
# create a reflection table
from dials.array_family import flex
table = flex.reflection_table()
table['flags'] = flex.size_t(nref, 0)
table['id'] = flex.int(nref, 0)
table['panel'] = flex.size_t(nref, 0)
table['miller_index'] = flex.miller_index(hkl)
table['entering'] = flex.bool(nref, True)
table['s1'] = flex.vec3_double(nref)
table['xyzcal.mm'] = flex.vec3_double(nref)
table['xyzcal.px'] = flex.vec3_double(nref)
return table
def refine(self, experiments, reflections):
sel = ((reflections['id'] >= -1))
refl = reflections.select(sel)
acceptance_flags = self.identify_outliers(self.all_params, experiments, refl)
#create a new "indexed" list with outliers thrown out:
refl = refl.select(acceptance_flags)
print "$$$ stills_indexer::refine"
R = e_refine(params = self.all_params, experiments=experiments, reflections=refl, graph_verbose=False)
ref_experiments = R.get_experiments()
# try to improve the outcome with a second round of outlier rejection post-initial refinement:
acceptance_flags = self.identify_outliers(self.all_params, ref_experiments, refl)
# insert a round of Nave-outlier rejection on top of the r.m.s.d. rejection
nv0 = nave_parameters(params = self.all_params, experiments=ref_experiments, reflections=refl, refinery=R, graph_verbose=False)
crystal_model_nv0 = nv0()
acceptance_flags_nv0 = nv0.nv_acceptance_flags
refl = refl.select(acceptance_flags & acceptance_flags_nv0)
print "$$$ stills_indexer::refine after positional and delta-psi outlier rejection"
refiner = e_refine(params = self.all_params, experiments=ref_experiments, reflections=refl, graph_verbose=False)
matches = refiner.get_matches()
xyzcal_mm = flex.vec3_double(len(refl))
xyzcal_mm.set_selected(matches['iobs'], matches['xyzcal.mm'])
refl['xyzcal.mm'] = xyzcal_mm
refl.set_flags(matches['iobs'], refl.flags.used_in_refinement)
refl['entering'] = flex.bool(len(refl), False)
return refiner.get_experiments(), refl
def outlier_rejection(reflections):
# http://scripts.iucr.org/cgi-bin/paper?ba0032
if len(reflections) == 1:
return reflections
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
i_max = flex.max_index(intensities)
sel = flex.bool(len(reflections), True)
sel[i_max] = False
i_test = intensities[i_max]
var_test = variances[i_max]
intensities_subset = intensities.select(sel)
var_subset = variances.select(sel)
var_prior = var_test + 1/flex.sum(1/var_subset)
p_prior = 1/math.sqrt(2*math.pi * var_prior) * math.exp(
-(i_test - flex.mean(intensities_subset))**2/(2 * var_prior))
#print p_prior
if p_prior > 1e-10:
return reflections
return outlier_rejection(reflections.select(sel))
def __init__(self, frames, size):
from dials.array_family import flex
self.frames = frames
nframes = frames[1] - frames[0]
self.data = []
self.mask = []
for sz in size:
self.data.append(flex.double(flex.grid(nframes, sz[0], sz[1])))
self.mask.append(flex.bool(flex.grid(nframes, sz[0], sz[1])))
def main(reflections, experiment, params):
nref0 = len(reflections)
method = params.method
ids = params.id
if 'intensity.prf.variance' in reflections:
selection = reflections.get_flags(
reflections.flags.integrated,
all=True)
else:
selection = reflections.get_flags(
reflections.flags.integrated_sum)
reflections = reflections.select(selection)
# filter according to rules
if params.min_isum:
selection = reflections['intensity.sum.value'] > params.min_isum
reflections = reflections.select(selection)
if params.num > len(reflections):
raise RuntimeError, 'you asked for too many reflections sorry'
if params.seed > 0 and params.num > 0:
import random
from dials.array_family import flex
random.seed(params.seed)
selected = flex.bool(len(reflections), False)
while len(selected.iselection()) < params.num:
selected[random.randint(0, len(reflections))] = True
reflections = reflections.select(selected)
nref1 = len(reflections)
print 'Removed %d reflections, %d remain' % (nref0 - nref1, nref1)
results = []
for j, reflection in enumerate(reflections):
result = None
if ids:
if j in ids:
result = globals()['model_reflection_%s' % method](reflection, experiment, params)
elif params.id_start and params.id_end:
if j < params.id_start or j >= params.id_end:
continue
print j
result = globals()['model_reflection_%s' % method](reflection, experiment, params)
else:
print j
result = globals()['model_reflection_%s' % method](reflection, experiment, params)
if result is not None:
results.append(result)
return results
def run(self):
''' Perform the integration. '''
from dials.util.command_line import heading
from dials.util.options import flatten_experiments
from dials.util import log
from time import time
from libtbx.utils import Sorry
from dials.array_family import flex
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0:
self.parser.print_help()
return
assert len(experiments) == 1
imageset = experiments[0].imageset
beam = experiments[0].beam
detector = experiments[0].detector
goniometer = experiments[0].goniometer
assert len(detector) == 1
# Configure logging
log.config()
from dials.algorithms.background.gmodel import PolarTransform
import cPickle as pickle
model = pickle.load(open(params.model))
image = model.data(0)
mask = flex.bool(image.accessor(), True)
# Do the transformation
transform = PolarTransform(beam, detector[0], goniometer)
result = transform.to_polar(image, mask)
data = result.data()
mask = result.mask()
pickle.dump((data, mask), open(params.output.data, "w"))
from matplotlib import pylab
vmax = sorted(list(data))[int(0.99 * len(data))]
figure = pylab.figure(figsize=(6,4))
pylab.imshow(
data.as_numpy_array(),
interpolation = 'none',
vmin = 0,
vmax = vmax)
ax1 = pylab.gca()
ax1.get_xaxis().set_visible(False)
ax1.get_yaxis().set_visible(False)
cb = pylab.colorbar()
cb.ax.tick_params(labelsize=8)
logger.info("Saving polar model %s" % (params.output.image))
pylab.savefig("%s" % (params.output.image), dpi=600, bbox_inches='tight')
def run(args):
import libtbx.load_env
usage = "%s experiments.json [options]" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0:
parser.print_help()
return
elif len(experiments) > 1:
raise Sorry("More than one experiment present")
assert len(params.miller_index), "Must specify at least one miller_index to predict."
experiment = experiments[0]
reflections = flex.reflection_table()
miller_indices = flex.miller_index()
entering_flags = flex.bool()
for mi in params.miller_index:
miller_indices.append(mi)
miller_indices.append(mi)
entering_flags.append(True)
entering_flags.append(False)
reflections['miller_index'] = miller_indices
reflections['entering'] = entering_flags
reflections['id'] = flex.size_t(len(reflections), 0)
if params.expand_to_p1:
from cctbx.miller import expand_to_p1_iselection
proxy = expand_to_p1_iselection(
experiment.crystal.get_space_group(),
anomalous_flag=True,
indices=miller_indices,
build_iselection=True)
reflections = reflections.select(proxy.iselection)
reflections['miller_index'] = proxy.indices
from dials.algorithms.refinement.prediction.managed_predictors import ExperimentsPredictor
predictor = ExperimentsPredictor([experiment])
predicted = predictor.predict(reflections)
zmin, zmax = experiment.scan.get_array_range()
z = predicted['xyzcal.px'].parts()[2]
predicted = predicted.select((z >= zmin) & (z <= zmax))
show_predictions(predicted)
def tst_select(self):
from dials.array_family import flex
# The columns as lists
c1 = list(range(10))
c2 = list(range(10))
c3 = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'i', 'j', 'k']
# Create a table with some elements
table = flex.reflection_table()
table['col1'] = flex.int(c1)
table['col2'] = flex.double(c2)
table['col3'] = flex.std_string(c3)
# Select some columns
new_table = table.select(('col1', 'col2'))
assert(new_table.nrows() == 10)
assert(new_table.ncols() == 2)
assert(all(a == b for a, b in zip(new_table['col1'], c1)))
assert(all(a == b for a, b in zip(new_table['col2'], c2)))
print 'OK'
# Select some columns
new_table = table.select(flex.std_string(['col1', 'col2']))
assert(new_table.nrows() == 10)
assert(new_table.ncols() == 2)
assert(all(a == b for a, b in zip(new_table['col1'], c1)))
assert(all(a == b for a, b in zip(new_table['col2'], c2)))
print 'OK'
# Select some rows
index = flex.size_t([0, 1, 5, 8, 9])
cc1 = [c1[i] for i in index]
cc2 = [c2[i] for i in index]
cc3 = [c3[i] for i in index]
new_table = table.select(index)
assert(new_table.nrows() == 5)
assert(new_table.ncols() == 3)
assert(all(a == b for a, b in zip(new_table['col1'], cc1)))
assert(all(a == b for a, b in zip(new_table['col2'], cc2)))
assert(all(a == b for a, b in zip(new_table['col3'], cc3)))
print 'OK'
# Select some rows
index = flex.bool([True, True, False, False, False,
True, False, False, True, True])
new_table = table.select(index)
assert(new_table.nrows() == 5)
assert(new_table.ncols() == 3)
assert(all(a == b for a, b in zip(new_table['col1'], cc1)))
assert(all(a == b for a, b in zip(new_table['col2'], cc2)))
assert(all(a == b for a, b in zip(new_table['col3'], cc3)))
print 'OK'
def __call__(self, params, options):
''' Import the spot.xds file. '''
from iotbx.xds import spot_xds
from dials.util.command_line import Command
from dials.array_family import flex
# Read the SPOT.XDS file
Command.start('Reading SPOT.XDS')
handle = spot_xds.reader()
handle.read_file(self._spot_xds)
centroid = handle.centroid
intensity = handle.intensity
try:
miller_index = handle.miller_index
except AttributeError:
miller_index = None
Command.end('Read {0} spots from SPOT.XDS file.'.format(len(centroid)))
# Create the reflection list
Command.start('Creating reflection list')
table = flex.reflection_table()
table['id'] = flex.int(len(centroid), 0)
table['panel'] = flex.size_t(len(centroid), 0)
if miller_index:
table['miller_index'] = flex.miller_index(miller_index)
table['xyzobs.px.value'] = flex.vec3_double(centroid)
table['intensity.sum.value'] = flex.double(intensity)
Command.end('Created reflection list')
# Remove invalid reflections
Command.start('Removing invalid reflections')
if miller_index and params.remove_invalid:
flags = flex.bool([h != (0, 0, 0) for h in table['miller_index']])
table = table.select(flags)
Command.end('Removed invalid reflections, %d remaining' % len(table))
# Fill empty standard columns
if params.add_standard_columns:
Command.start('Adding standard columns')
rt = flex.reflection_table.empty_standard(len(table))
rt.update(table)
table = rt
# set variances to unity
table['xyzobs.mm.variance'] = flex.vec3_double(len(table), (1,1,1))
table['xyzobs.px.variance'] = flex.vec3_double(len(table), (1,1,1))
Command.end('Standard columns added')
# Output the table to pickle file
if params.output.filename is None:
params.output.filename = 'spot_xds.pickle'
Command.start('Saving reflection table to %s' % params.output.filename)
table.as_pickle(params.output.filename)
Command.end('Saved reflection table to %s' % params.output.filename)
def _detect_outliers(self, cols):
from scitbx.math import five_number_summary
outliers = flex.bool(len(cols[0]), False)
for col in cols:
min_x, q1_x, med_x, q3_x, max_x = five_number_summary(col)
iqr_x = q3_x - q1_x
cut_x = self._iqr_multiplier * iqr_x
outliers.set_selected(col > q3_x + cut_x, True)
outliers.set_selected(col < q1_x - cut_x, True)
return outliers
def __call__(self, d):
'''
True if within powder ring.
:param d: The resolution
:return: True/False if within a powder ring
'''
from dials.array_family import flex
result = flex.bool(len(d), False)
for filter in self:
result = result | filter(d)
return result
def tst_does_bbox_contain_bad_pixels(self):
from dials.array_family import flex
from dials.model.data import Shoebox
from random import randint
mask = flex.bool(flex.grid(100, 100), True)
for j in range(100):
for i in range(40, 60):
mask[j,i] = False
mask[i,j] = False
shoebox = flex.shoebox(1000)
res = flex.bool(1000)
for i in range(1000):
x0 = randint(0, 90)
y0 = randint(0, 90)
z0 = randint(0, 90)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
res2 = False
if x0 >= 40 and x0 < 60:
res2 = True
if x1 > 40 and x1 <= 60:
res2 = True
if y0 >= 40 and y0 < 60:
res2 = True
if y1 > 40 and y1 <= 60:
res2 = True
res[i] = res2
assert(shoebox.does_bbox_contain_bad_pixels(mask) == res)
# Test passed
print 'OK'
def __call__(self, d):
'''
True if within powder ring.
:param d: The resolution
:return: True/False in powder ring
'''
from dials.array_family import flex
result = flex.bool(len(d), False)
for d_spacing in self.d_spacings:
result = result | (flex.abs(d - d_spacing) < self.half_width)
return result
def _id_refs_to_keep(self, obs_data):
"""Create a selection of observations that pass certain conditions.
This step includes rejection of reflections too close to the spindle,
reflections measured outside the scan range, rejection of the (0,0,0)
Miller index and rejection of reflections with the overload flag set.
Outlier rejection is done later."""
# first exclude reflections with miller index set to 0,0,0
sel1 = obs_data['miller_index'] != (0,0,0)
# exclude reflections with overloads, as these have worse centroids
sel2 = ~obs_data.get_flags(obs_data.flags.overloaded)
# combine selections
sel = sel1 & sel2
inc = flex.size_t_range(len(obs_data)).select(sel)
obs_data = obs_data.select(sel)
# Default to True to pass the following test if there is no rotation axis
# for a particular experiment
to_keep = flex.bool(len(inc), True)
for iexp, exp in enumerate(self._experiments):
axis = self._axes[iexp]
if not axis or exp.scan is None: continue
if exp.scan.get_oscillation()[1] == 0.0: continue
sel = obs_data['id'] == iexp
s0 = self._s0vecs[iexp]
s1 = obs_data['s1'].select(sel)
phi = obs_data['xyzobs.mm.value'].parts()[2].select(sel)
# first test: reject reflections for which the parallelepiped formed
# between the gonio axis, s0 and s1 has a volume of less than the cutoff.
# Those reflections are by definition closer to the spindle-beam
# plane and for low values of the cutoff are troublesome to
# integrate anyway.
p_vol = flex.abs(s1.cross(flex.vec3_double(s1.size(), s0)).dot(axis))
passed1 = p_vol > self._close_to_spindle_cutoff
# second test: reject reflections that lie outside the scan range
phi_min, phi_max = exp.scan.get_oscillation_range(deg=False)
passed2 = (phi >= phi_min) & (phi <= phi_max)
# combine tests
to_update = passed1 & passed2
to_keep.set_selected(sel, to_update)
inc = inc.select(to_keep)
return inc
def test1():
# work in a temporary directory
cwd = os.path.abspath(os.curdir)
tmp_dir = open_tmp_directory(suffix="test_dials_filter_reflections")
os.chdir(tmp_dir)
# make a dummy reflection table for the test, setting some flags
from dials.array_family import flex
rt = flex.reflection_table.empty_standard(6)
rt["iobs"] = flex.size_t_range(len(rt))
mask1 = flex.bool([True] * 3 + [False] * 3)
mask2 = flex.bool([True, False] * 3)
rt.set_flags(mask1, rt.flags.integrated)
rt.set_flags(mask2, rt.flags.bad_spot)
rt_name = "test_refs.pickle"
rt.as_pickle(rt_name)
cmd = "dev.dials.filter_reflections " + rt_name + " inclusions.flag=integrated" + " exclusions.flag=bad_spot"
print cmd
try:
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
# load results
ref = flex.reflection_table.from_pickle("filtered.pickle")
finally:
os.chdir(cwd)
# clean up tmp dir
shutil.rmtree(tmp_dir)
# The test selects only 1 reflection
assert len(ref) == 1
assert list(ref["iobs"]) == [1]
print "OK"
return
def write_integration_pickles(self, integrated, experiments, callback = None):
"""
Write a serialized python dictionary with integrated intensities and other information
suitible for use by cxi.merge or prime.postrefine.
@param integrated Reflection table with integrated intensities
@param experiments Experiment list. One integration pickle for each experiment will be created.
@param callback Deriving classes can use callback to make further modifications to the dictionary
before it is serialized. Callback should be a function with this signature:
def functionname(params, outfile, frame), where params is the phil scope, outfile is the path
to the pickle that will be saved, and frame is the python dictionary to be serialized.
"""
try:
picklefilename = self.params.output.integration_pickle
except AttributeError:
return
if self.params.output.integration_pickle is not None:
from libtbx import easy_pickle
import os
from xfel.command_line.frame_extractor import ConstructFrame
from dials.array_family import flex
# Split everything into separate experiments for pickling
for e_number in xrange(len(experiments)):
experiment = experiments[e_number]
e_selection = flex.bool( [r['id']==e_number for r in integrated])
reflections = integrated.select(e_selection)
frame = ConstructFrame(reflections, experiment).make_frame()
frame["pixel_size"] = experiment.detector[0].get_pixel_size()[0]
try:
# if the data was a file on disc, get the path
event_timestamp = os.path.splitext(experiments[0].imageset.paths()[0])[0]
except NotImplementedError:
# if the data is in memory only, check if the reader set a timestamp on the format object
event_timestamp = experiment.imageset.reader().get_format(0).timestamp
event_timestamp = os.path.basename(event_timestamp)
if event_timestamp.find("shot-")==0:
event_timestamp = os.path.splitext(event_timestamp)[0] # micromanage the file name
if hasattr(self.params.output, "output_dir"):
outfile = os.path.join(self.params.output.output_dir, self.params.output.integration_pickle%(e_number,event_timestamp))
else:
outfile = os.path.join(os.path.dirname(self.params.output.integration_pickle), self.params.output.integration_pickle%(e_number,event_timestamp))
if callback is not None:
callback(self.params, outfile, frame)
easy_pickle.dump(outfile, frame)
def __call__(self, d):
'''
True if within powder ring.
:param d: The resolution
:return: True/False in powder ring
'''
from dials.array_family import flex
result = flex.bool(len(d), False)
d2 = 1.0 / d**2
for ice_ring in self.ice_rings:
result = result | (d2 >= ice_ring[0]) & (d2 <= ice_ring[1])
return result
def hot_pixel_mask(imageset, reflections):
depth = imageset.get_array_range()[1] - imageset.get_array_range()[0]
xylist = filter_reflections(reflections, depth)
from dials.array_family import flex
mask = flex.bool(flex.grid(reversed(imageset.get_image_size())), True)
for x, y in xylist:
mask[y, x] = False
print 'Found %d hot pixels' % len(xylist)
return (mask,)
def __init__(self):
from dials.array_family import flex
self.reflections = flex.reflection_table()
self.reflections['panel'] = flex.size_t()
self.reflections['bbox'] = flex.int6()
self.reflections['miller_index'] = flex.miller_index()
self.reflections['s1'] = flex.vec3_double()
self.reflections['xyzcal.px'] = flex.vec3_double()
self.reflections['xyzcal.mm'] = flex.vec3_double()
self.reflections['entering'] = flex.bool()
self.reflections['id'] = flex.int()
self.reflections["flags"] = flex.size_t()
self.npanels = 2
self.width = 1000
self.height = 1000
self.nrefl = 10000
self.array_range = (0, 130)
self.block_size = 20
from random import randint, seed, choice
seed(0)
self.processed = [[] for i in range(12)]
for i in range(self.nrefl):
x0 = randint(0, self.width-10)
y0 = randint(0, self.height-10)
zs = randint(2, 9)
x1 = x0 + randint(2, 10)
y1 = y0 + randint(2, 10)
for k, j in enumerate([10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]):
m = k + i * 12
pos = choice(["left", "right", "centre"])
if pos == 'left':
z0 = j - zs
z1 = j
elif pos == 'right':
z0 = j
z1 = j + zs
else:
z0 = j - zs // 2
z1 = j + zs // 2
bbox = (x0, x1, y0, y1, z0, z1)
self.reflections.append({
"panel" : randint(0,1),
"bbox" : bbox,
"flags" : flex.reflection_table.flags.reference_spot
})
self.processed[k].append(m)
def tst_consistent(self):
from random import randint
from dials.model.data import Shoebox
from dials.array_family import flex
shoebox = flex.shoebox(10)
for i in range(10):
x0 = randint(0, 1000)
y0 = randint(0, 1000)
z0 = randint(0, 1000)
x1 = randint(1, 10) + x0
y1 = randint(1, 10) + y0
z1 = randint(1, 10) + z0
shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1))
assert(shoebox.is_consistent() == flex.bool(10, False))
for i in range(10):
shoebox[i].allocate()
assert(shoebox.is_consistent() == flex.bool(10, True))
for i in [0, 2, 4, 6, 8]:
shoebox[i].data.resize(flex.grid(10, 10, 10))
assert(shoebox.is_consistent() == flex.bool([False, True] * 5))
for i in range(10):
shoebox[i].deallocate()
assert(shoebox.is_consistent() == flex.bool(10, False))
# Test passed
print 'OK'
def filter_integrated_centroids(reflections):
'''Filter reflections to include only those with the integrated and the
strong flag set, but only if there are apparently some integrated
reflections'''
orig_len = len(reflections)
inc = flex.bool(orig_len, False)
mask = reflections.get_flags(reflections.flags.integrated)
if mask.count(True) == 0: return reflections
reflections = reflections.select(mask)
mask = reflections.get_flags(reflections.flags.strong)
reflections = reflections.select(mask)
logger.info('{0} out of {1} reflections remain after filtering to keep only strong'
' and integrated centroids'.format(len(reflections), orig_len))
return reflections
def d2f(I):
mask = flex.bool(flex.grid(9,9,9), False)
for k in range(9):
for j in range(9):
for i in range(9):
dx = 5 * (i - 4.5) / 4.5
dy = 5 * (j - 4.5) / 4.5
dz = 5 * (k - 4.5) / 4.5
dd = sqrt(dx**2 + dy**2 + dz**2)
if dd <= 3:
mask[k,j,i] = True
mask = mask.as_1d() & (ref_P.as_1d() > 0)
p = ref_P.as_1d().select(mask)
c = max_P.as_1d().select(mask)
return flex.sum(2*c*c*p*p / (p*I)**3)
def load_inputs(self, pandas_table, miller_data=None, refls_key='predictions'):
"""
:param pandas_table: contains path to the experiments (pandas column exp_name) to be loaded
the pandas table is expected to have been written by diffBragg.hopper or
diffBragg.hopper_process . See method save_to_pandas in simtbx/command_line/hopper.py
For example, if the outputdir of diffBragg.hopper was set to `all_shots`, then
there should be a golder all_shots/pandas created which contains all of the per-shot pandas
dataframes. They should be concatenated as follows, forming a suitable argument for this method
>> import glob,pandas
>> fnames = glob.glob("all_shots/pandas/rank*/*pkl")
>> df = pandas.concat([ pandas.read_pickle(f) for f in fnames])
>> df.reset_index(inplace=True, drop=True)
>> df.to_pickle("all_shots.pkl")
>> # Then later, as part of an MPI application, the following will load all data:
>> RefineLauncher_instance.load_inputs(df, refls_key="stage1_refls")
:param miller_data: Optional miller array for the structure factor component of the model
:param refls_key: key specifying the reflection tables in the pandas table
Modeled pixels will lie in shoeboxes centered on each x,y,z in xyzobs.px.value
:return:
"""
COMM.Barrier()
num_exp = len(pandas_table)
first_exper_file = pandas_table.exp_name.values[0]
detector = ExperimentListFactory.from_json_file(first_exper_file, check_format=False)[0].detector
if detector is None and self.params.refiner.reference_geom is None:
raise RuntimeError("No detector in experiment, must provide a reference geom.")
# TODO verify all shots have the same detector ?
if self.params.refiner.reference_geom is not None:
detector = ExperimentListFactory.from_json_file(self.params.refiner.reference_geom, check_format=False)[0].detector
print("Using reference geom from expt %s" % self.params.refiner.reference_geom)
if COMM.size > num_exp:
raise ValueError("Requested %d MPI ranks to process %d shots. Reduce number of ranks to %d"
% (COMM.size, num_exp, num_exp))
self._init_panel_group_information(detector)
self.verbose = False
if COMM.rank == 0:
self.verbose = self.params.refiner.verbose > 0
if self.params.refiner.gather_dir is not None and not os.path.exists(self.params.refiner.gather_dir):
os.makedirs(self.params.refiner.gather_dir)
LOGGER.info("MADE GATHER DIR %s" % self.params.refiner.gather_dir)
COMM.barrier()
shot_idx = 0 # each rank keeps index of the shots local to it
rank_panel_groups_refined = set()
exper_names = pandas_table.exp_name
assert len(exper_names) == len(set(exper_names))
# TODO assert all exper are single-file, probably way before this point
LOGGER.info("EVENT: begin loading inputs")
for i_exp, exper_name in enumerate(exper_names):
if i_exp % COMM.size != COMM.rank:
continue
LOGGER.info("EVENT: BEGIN loading experiment list")
expt_list = ExperimentListFactory.from_json_file(exper_name, check_format=self.params.refiner.check_expt_format)
LOGGER.info("EVENT: DONE loading experiment list")
if len(expt_list) != 1:
print("Input experiments need to have length 1, %s does not" % exper_name)
expt = expt_list[0]
expt.detector = detector # in case of supplied ref geom
self._check_experiment_integrity(expt)
exper_dataframe = pandas_table.query("exp_name=='%s'" % exper_name)
refl_name = exper_dataframe[refls_key].values[0]
refls = flex.reflection_table.from_file(refl_name)
# FIXME need to remove (0,0,0) bboxes
good_sel = flex.bool([h != (0, 0, 0) for h in list(refls["miller_index"])])
refls = refls.select(good_sel)
#UcellMan = utils.manager_from_crystal(expt.crystal)
opt_uc_param = exper_dataframe[["a","b","c","al","be","ga"]].values[0]
UcellMan = utils.manager_from_params(opt_uc_param)
if self.symbol is None:
if self.params.refiner.force_symbol is not None:
self.symbol = self.params.refiner.force_symbol
else:
self.symbol = expt.crystal.get_space_group().type().lookup_symbol()
else:
if self.params.refiner.force_symbol is None:
if expt.crystal.get_space_group().type().lookup_symbol() != self.symbol:
raise ValueError("Crystals should all have the same space group symmetry")
if shot_idx == 0: # each rank initializes a simulator only once
if self.params.simulator.init_scale != 1:
print("WARNING: For stage_two , it is assumed that total scale is stored in the pandas dataframe")
print("WARNING: resetting params.simulator.init_scale to 1!")
self.params.simulator.init_scale = 1
self._init_simulator(expt, miller_data)
if self.params.profile:
self.SIM.record_timings = True
if self.params.refiner.stage_two.Fref_mtzname is not None:
self.Fref = utils.open_mtz(self.params.refiner.stage_two.Fref_mtzname,
self.params.refiner.stage_two.Fref_mtzcol)
LOGGER.info("EVENT: LOADING ROI DATA")
shot_modeler = hopper_utils.DataModeler(self.params)
if self.params.refiner.load_data_from_refl:
gathered = shot_modeler.GatherFromReflectionTable(expt, refls, sg_symbol=self.symbol)
else:
gathered = shot_modeler.GatherFromExperiment(expt, refls, sg_symbol=self.symbol)
if not gathered:
raise IOError("Failed to gather data from experiment %s", exper_name)
if self.params.refiner.gather_dir is not None:
gathered_name = os.path.splitext(os.path.basename(exper_name))[0]
gathered_name += "_withData.refl"
gathered_name = os.path.join(self.params.refiner.gather_dir, gathered_name )
shot_modeler.dump_gathered_to_refl(gathered_name, do_xyobs_sanity_check=False) #True)
LOGGER.info("SAVED ROI DATA TO %s" % gathered_name)
if self.params.refiner.test_gathered_file:
all_data = shot_modeler.all_data.copy()
all_roi_id = shot_modeler.roi_id.copy()
all_bg = shot_modeler.all_background.copy()
all_trusted = shot_modeler.all_trusted.copy()
all_pids = np.array(shot_modeler.pids)
all_rois = np.array(shot_modeler.rois)
new_Modeler = hopper_utils.DataModeler(self.params)
assert new_Modeler.GatherFromReflectionTable(exper_name, gathered_name, sg_symbol=self.symbol)
assert np.allclose(new_Modeler.all_data, all_data)
assert np.allclose(new_Modeler.all_background, all_bg)
assert np.allclose(new_Modeler.rois, all_rois)
assert np.allclose(new_Modeler.pids, all_pids)
assert np.allclose(new_Modeler.all_trusted, all_trusted)
assert np.allclose(new_Modeler.roi_id, all_roi_id)
LOGGER.info("Gathered file approved!")
self.Hi[shot_idx] = shot_modeler.Hi
self.Hi_asu[shot_idx] = shot_modeler.Hi_asu
LOGGER.info("EVENT: DONE LOADING ROI")
shot_modeler.ucell_man = UcellMan
self.SIM.num_ucell_param = len(shot_modeler.ucell_man.variables) # for convenience
loaded_spectra = False
if self.params.spectrum_from_imageset:
try:
shot_spectra = hopper_utils.downsamp_spec(self.SIM, self.params, expt, return_and_dont_set=True)
loaded_spectra = True
except Exception as err:
LOGGER.warning("spectrum_from_imageset is set to True, however failed to load spectra: %s" % err)
loaded_spectra = False
if not loaded_spectra:
if "spectrum_filename" in list(exper_dataframe) and exper_dataframe.spectrum_filename.values[0] is not None:
shot_spectra = utils.load_spectra_from_dataframe(exper_dataframe)
LOGGER.debug("Loaded specta from %s" % exper_dataframe.spectrum_filename.values[0])
else:
total_flux = exper_dataframe.total_flux.values[0]
if total_flux is None:
total_flux = self.params.simulator.total_flux
shot_spectra = [(expt.beam.get_wavelength(), total_flux)]
shot_modeler.spectra = shot_spectra
if self.params.refiner.gather_dir is not None and not self.params.refiner.load_data_from_refl:
spec_wave, spec_weights = map(np.array, zip(*shot_spectra))
spec_filename = os.path.splitext(os.path.basename(exper_name))[0]
spec_filename = os.path.join(self.params.refiner.gather_dir, spec_filename+".lam")
utils.save_spectra_file(spec_filename, spec_wave, spec_weights)
LOGGER.info("saved spectra filename %s" % spec_filename)
LOGGER.info("Will simulate %d energy channels" % len(shot_spectra))
if "detz_shift_mm" in list(exper_dataframe):
shot_modeler.originZ_init = exper_dataframe.detz_shift_mm.values[0]*1e-3
else:
shot_modeler.originZ_init = 0
shot_modeler.exper_name = exper_name
shot_modeler.refl_name = refl_name
shot_panel_groups_refined = self.determine_refined_panel_groups(shot_modeler.pids)
rank_panel_groups_refined = rank_panel_groups_refined.union(set(shot_panel_groups_refined))
shot_idx += 1
if COMM.rank == 0:
self._mem_usage()
print("Finished loading image %d / %d" % (i_exp+1, len(exper_names)), flush=True)
shot_modeler.PAR = PAR_from_params(self.params, expt, best=exper_dataframe)
self.Modelers[i_exp] = shot_modeler
LOGGER.info("DONE LOADING DATA; ENTER BARRIER")
COMM.Barrier()
LOGGER.info("DONE LOADING DATA; EXIT BARRIER")
self.shot_roi_darkRMS = None
# TODO warn that per_spot_scale refinement not intended for ensemble mode
all_refined_groups = COMM.gather(rank_panel_groups_refined)
panel_groups_refined = None
if COMM.rank == 0:
panel_groups_refined = set()
for set_of_panels in all_refined_groups:
panel_groups_refined = panel_groups_refined.union(set_of_panels)
self.panel_groups_refined = list(COMM.bcast(panel_groups_refined))
LOGGER.info("EVENT: Gathering global HKL information")
self._gather_Hi_information()
LOGGER.info("EVENT: FINISHED gather global HKL information")
if self.params.roi.cache_dir_only:
print("Done creating cache directory and cache_dir_only=True, so goodbye.")
sys.exit()
# in case of GPU
LOGGER.info("BEGIN DETERMINE MAX PIX")
self.NPIX_TO_ALLOC = self._determine_per_rank_max_num_pix()
# TODO in case of randomize devices, shouldnt this be total max across all ranks?
n = COMM.gather(self.NPIX_TO_ALLOC)
if COMM.rank == 0:
n = max(n)
self.NPIX_TO_ALLOC = COMM.bcast(n)
LOGGER.info("DONE DETERMINE MAX PIX")
self.DEVICE_ID = COMM.rank % self.params.refiner.num_devices
self._mem_usage()
def generate_simple_table(prf=True):
"""Generate a reflection table for testing intensity combination.
The numbers are contrived to make sum intensities agree well at high
intensity but terribly at low and vice versa for profile intensities."""
reflections = flex.reflection_table()
reflections["miller_index"] = flex.miller_index(
[
(0, 0, 1),
(0, 0, 1),
(0, 0, 1),
(0, 0, 1),
(0, 0, 1),
(0, 0, 2),
(0, 0, 2),
(0, 0, 2),
(0, 0, 2),
(0, 0, 2),
(0, 0, 3),
(0, 0, 3),
(0, 0, 3),
(0, 0, 3),
(0, 0, 3),
(0, 0, 4),
(0, 0, 4),
(0, 0, 4),
(0, 0, 4),
(0, 0, 4),
(0, 0, 5),
(0, 0, 5),
(0, 0, 5),
(0, 0, 5),
(0, 0, 5),
]
)
reflections["inverse_scale_factor"] = flex.double(25, 1.0)
# Contrive an example that should give the best cc12 when combined.
# make sum intensities agree well at high intensity but terribly at low
# and vice versa for profile intensities.
# profile less consistent at high intensity here
# sumless consistent at low intensity here
reflections["intensity.sum.value"] = flex.double(
[
10000.0,
11000.0,
9000.0,
8000.0,
12000.0,
500.0,
5600.0,
5500.0,
2000.0,
6000.0,
100.0,
50.0,
150.0,
75.0,
125.0,
30.0,
10.0,
2.0,
35.0,
79.0,
1.0,
10.0,
20.0,
10.0,
5.0,
]
)
reflections["intensity.sum.variance"] = flex.double(
[10000] * 5 + [5000] * 5 + [100] * 5 + [30] * 5 + [10] * 5
)
reflections.set_flags(flex.bool(25, False), reflections.flags.outlier_in_scaling)
reflections.set_flags(flex.bool(25, True), reflections.flags.integrated)
reflections["lp"] = flex.double(25, 0.5)
if prf:
reflections["intensity.prf.value"] = flex.double(
[
10000.0,
16000.0,
12000.0,
6000.0,
9000.0,
5000.0,
2000.0,
1500.0,
1300.0,
9000.0,
100.0,
80.0,
120.0,
90.0,
100.0,
30.0,
40.0,
50.0,
30.0,
30.0,
10.0,
12.0,
9.0,
8.0,
10.0,
]
)
reflections["intensity.prf.variance"] = flex.double(
[1000] * 5 + [500] * 5 + [10] * 5 + [3] * 5 + [1] * 5
)
reflections = calculate_prescaling_correction(reflections)
return reflections
def read(handle, key):
from dials.array_family import flex
from dxtbx.format.nexus import convert_units
import numpy as np
if key == 'miller_index':
h = flex.int(handle['h'][:].astype(np.int32))
k = flex.int(handle['k'][:].astype(np.int32))
l = flex.int(handle['l'][:].astype(np.int32))
return flex.miller_index(h, k, l)
elif key == 'id':
return flex.int(handle['id'][:].astype(int))
elif key == 'partial_id':
return flex.size_t(handle['reflection_id'][:].astype(int))
elif key == 'entering':
return flex.bool(handle['entering'][:])
elif key == 'flags':
return flex.size_t(handle['flags'][:].astype(int))
elif key == 'panel':
return flex.size_t(handle['det_module'][:].astype(int))
elif key == 'd':
return flex.double(handle['d'][:])
elif key == 'partiality':
return flex.double(handle['partiality'][:])
elif key == 'xyzcal.px':
x = flex.double(handle['predicted_px_x'][:])
y = flex.double(handle['predicted_px_y'][:])
z = flex.double(handle['predicted_frame'][:])
return flex.vec3_double(x, y, z)
elif key == 'xyzcal.mm':
x = convert_units(flex.double(handle['predicted_x'][:]),
handle['predicted_x'].attrs['units'], 'mm')
y = convert_units(flex.double(handle['predicted_y'][:]),
handle['predicted_y'].attrs['units'], 'mm')
z = convert_units(flex.double(handle['predicted_phi'][:]),
handle['predicted_phi'].attrs['units'], 'rad')
return flex.vec3_double(x, y, z)
elif key == 'bbox':
b = flex.int(handle['bounding_box'][:].astype(np.int32))
return flex.int6(b.as_1d())
elif key == 'xyzobs.px.value':
x = flex.double(handle['observed_px_x'][:])
y = flex.double(handle['observed_px_y'][:])
z = flex.double(handle['observed_frame'][:])
return flex.vec3_double(x, y, z)
elif key == 'xyzobs.px.variance':
x = flex.double(handle['observed_px_x_var'][:])
y = flex.double(handle['observed_px_y_var'][:])
z = flex.double(handle['observed_frame_var'][:])
return flex.vec3_double(x, y, z)
elif key == 'xyzobs.mm.value':
x = convert_units(flex.double(handle['observed_x'][:]),
handle['observed_x'].attrs['units'], 'mm')
y = convert_units(flex.double(handle['observed_y'][:]),
handle['observed_y'].attrs['units'], 'mm')
z = convert_units(flex.double(handle['observed_phi'][:]),
handle['observed_phi'].attrs['units'], 'rad')
return flex.vec3_double(x, y, z)
elif key == 'xyzobs.mm.variance':
x = convert_units(flex.double(handle['observed_x_var'][:]),
handle['observed_x_var'].attrs['units'], 'mm')
y = convert_units(flex.double(handle['observed_y_var'][:]),
handle['observed_y_var'].attrs['units'], 'mm')
z = convert_units(flex.double(handle['observed_phi_var'][:]),
handle['observed_phi_var'].attrs['units'], 'rad')
return flex.vec3_double(x, y, z)
elif key == 'background.mean':
return flex.double(handle['background_mean'][:])
elif key == 'intensity.sum.value':
return flex.double(handle['int_sum'][:])
elif key == 'intensity.sum.variance':
return flex.double(handle['int_sum_var'][:])
elif key == 'intensity.prf.value':
return flex.double(handle['int_prf'][:])
elif key == 'intensity.prf.variance':
return flex.double(handle['int_prf_var'][:])
elif key == 'profile.correlation':
return flex.double(handle['prf_cc'][:])
elif key == 'lp':
return flex.double(handle['lp'][:])
elif key == 'num_pixels.background':
return flex.int(handle['num_bg'][:].astype(np.int32))
elif key == 'num_pixels.background_used':
return flex.int(handle['num_bg_used'][:].astype(np.int32))
elif key == 'num_pixels.foreground':
return flex.int(handle['num_fg'][:].astype(np.int32))
elif key == 'num_pixels.valid':
return flex.int(handle['num_valid'][:].astype(np.int32))
elif key == 'profile.rmsd':
return flex.double(handle['prf_rmsd'][:])
else:
raise KeyError('Column %s not read from file' % key)
def _multiplicity_mean_error_stddev(
self, calculate_variances=False, keep_singles=False
):
""" "
Calculate aggregate properties of grouped symmetry-equivalent reflections.
Populate the reflection table of observations with the following
properties:
* ``multiplicity`` — Multiplicity of observations of a given reflection
in the asymmetric unit;
:type: `dials.array_family_flex_ext.int` array
* ``intensity.mean.value`` — Mean of symmetry-equivalent reflections,
weighted by measurement error;
:type: `dials.array_family_flex_ext.double` array
* ``intensity.mean.std_error`` — Standard error on the weighted mean;
:type: `dials.array_family_flex_ext.double` array
* (optional) ``intensity.mean.variance`` — variance of
symmetry-equivalent reflections, weighted by measurement error;
:type: `dials.array_family_flex_ext.double` array
:param calculate_variances: Elect whether to calculate the weighted
variances. Defaults to False, to spare an expensive computation.
:type calculate_variances: bool
:param keep_singles: Choose whether to keep single-multiplicity
reflections.
:type keep_singles: bool
"""
for key, rtable in self.rtables.items():
# Sort the reflection table for speedier iteration.
rtable.sort("miller_index.asu")
# Record the positions of any multiplicity-1 reflections.
if not keep_singles:
singles = flex.size_t()
# Record the multiplicities.
multiplicity = flex.int()
# For weighted averaging.
weights = 1 / rtable["intensity.sum.variance"]
sum_weights = flex.double()
if calculate_variances:
sum_square_weights = flex.double()
# Calculate the weighted mean intensities.
i_means = flex.double()
# Calculate the standard deviations from unbiased weighted variances.
variances = flex.double()
# Iterate over the reflections, grouping by equivalent Miller index,
# to calculate multiplicities, weighted mean intensities, etc..
# Some time can be saved by only calculating variances if necessary.
# Initial values:
prev_index = None
count = 1
# The following will be set during loop iteration
i_sum, sum_weight, sum_square_weight = None, None, None
# One big loop through the entire reflection table:
for j in range(rtable.size()):
index = rtable["miller_index.asu"][j]
weight = weights[j]
# Aggregate within a symmetry-equivalent group of reflections:
if index == prev_index:
count += 1
i_sum += weight * rtable["intensity.sum.value"][j]
sum_weight += weight
if calculate_variances:
sum_square_weight += weight * weight
# Record the aggregated values for the group:
elif prev_index:
if count == 1 and not keep_singles:
singles.append(j - 1)
multiplicity.extend(flex.int(count, count))
i_means.extend(flex.double(count, i_sum / sum_weight))
sum_weights.extend(flex.double(count, sum_weight))
if calculate_variances:
sum_square_weights.extend(flex.double(count, sum_square_weight))
# And reinitialise:
prev_index = index
count = 1
i_sum = weight * rtable["intensity.sum.value"][j]
sum_weight = weight
if calculate_variances:
sum_square_weight = weight * weight
# Handle the first row:
else:
prev_index = rtable["miller_index.asu"][j]
i_sum = weight * rtable["intensity.sum.value"][j]
sum_weight = weight
if calculate_variances:
sum_square_weight = weight * weight
# Record the aggregated values for the last group:
if count == 1 and not keep_singles:
singles.append(rtable.size() - 1)
multiplicity.extend(flex.int(count, count))
i_means.extend(flex.double(count, i_sum / sum_weight))
sum_weights.extend(flex.double(count, sum_weight))
if calculate_variances:
sum_square_weights.extend(flex.double(count, sum_square_weight))
# Discard singletons:
if not keep_singles:
singles_del = flex.bool(rtable.size(), True)
singles_del.set_selected(singles, False)
multiplicity, weights, sum_weights, i_means = [
a.select(singles_del)
for a in (multiplicity, weights, sum_weights, i_means)
]
rtable.del_selected(singles)
if calculate_variances:
sum_square_weights = sum_square_weights.select(singles_del)
# Record the multiplicities in the reflection table.
rtable["multiplicity"] = multiplicity
# Record the weighted mean intensities in the reflection table.
rtable["intensity.mean.value"] = i_means
# Record the standard errors on the means in the reflection table.
rtable["intensity.mean.std_error"] = flex.sqrt(1 / sum_weights)
if calculate_variances:
# Initialise values:
prev_index = None
weighted_sum_square_residual = None
for j in range(rtable.size()):
index = rtable["miller_index.asu"][j]
weight = weights[j]
residual = rtable["intensity.sum.value"][j] - i_means[j]
# Aggregate within a symmetry-equivalent group of reflections:
if index == prev_index:
count += 1
weighted_sum_square_residual += weight * residual * residual
# Record the aggregated value for the group:
elif prev_index:
# The weighted variance is undefined for multiplicity=1,
# use the measured variance instead in this case.
if count == 1:
variances.append(rtable["intensity.sum.variance"][j - 1])
else:
sum_weight = sum_weights[j - 1]
var_weight = 1 / (
sum_weight - sum_square_weights[j - 1] / sum_weight
)
variances.extend(
flex.double(
count, weighted_sum_square_residual * var_weight
)
)
# Reinitialise:
prev_index = index
count = 1
weighted_sum_square_residual = weight * residual * residual
# Handle the first row:
else:
prev_index = rtable["miller_index.asu"][j]
count = 1
weighted_sum_square_residual = weight * residual * residual
# Record the aggregated values for the last group:
# The weighted variance is undefined for multiplicity=1,
# use the measured variance instead in this case.
if count == 1:
variances.append(rtable["intensity.sum.variance"][-1])
else:
sum_weight = sum_weights[-1]
var_weight = 1 / (sum_weight - sum_square_weights[-1] / sum_weight)
variances.extend(
flex.double(count, weighted_sum_square_residual * var_weight)
)
# Record the variances in the reflection table.
rtable["intensity.mean.variance"] = variances
self.rtables[key] = rtable
def test_combine_intensities(test_exp_P1):
"""Test the combine intensities function for a single dataset"""
reflections = generate_simple_table()
scaler = Mock()
scaler.reflection_table = reflections
scaler.suitable_refl_for_scaling_sel = flex.bool(reflections.size(), True)
scaler.outliers = flex.bool(reflections.size(), False)
scaler.experiment = test_exp_P1
scaler.space_group = test_exp_P1.crystal.get_space_group()
scaler.params.reflection_selection.combine.Imid = None
combiner = SingleDatasetIntensityCombiner(scaler)
Imid = combiner.max_key
intensity, variance = combiner.calculate_suitable_combined_intensities()
# Imid being 1200.0 should be best for this contrived example
assert Imid == 1200.0
# Due to nature of crossover, just require 2% tolerance for this example
assert list(intensity[0:5]) == pytest.approx(
list(
reflections["intensity.sum.value"][0:5]
* reflections["prescaling_correction"][0:5]
),
rel=2e-2,
)
assert list(intensity[20:25]) == pytest.approx(
list(
reflections["intensity.prf.value"][20:25]
* reflections["prescaling_correction"][20:25]
),
rel=2e-2,
)
assert list(variance[0:5]) == pytest.approx(
list(
reflections["intensity.sum.variance"][0:5]
* reflections["prescaling_correction"][0:5] ** 2
),
rel=2e-2,
)
assert list(variance[20:25]) == pytest.approx(
list(
reflections["intensity.prf.variance"][20:25]
* reflections["prescaling_correction"][20:25] ** 2
),
rel=2e-2,
)
combiner.max_key = 0 # prf
intensity, variance = combiner.calculate_suitable_combined_intensities()
assert list(intensity) == pytest.approx(
list(reflections["intensity.prf.value"] * reflections["prescaling_correction"]),
rel=2e-2,
)
assert list(variance) == pytest.approx(
list(
reflections["intensity.prf.variance"]
* reflections["prescaling_correction"] ** 2
),
rel=2e-2,
)
combiner.max_key = 1 # sum
intensity, variance = combiner.calculate_suitable_combined_intensities()
assert list(intensity) == pytest.approx(
list(reflections["intensity.sum.value"] * reflections["prescaling_correction"]),
rel=2e-2,
)
assert list(variance) == pytest.approx(
list(
reflections["intensity.sum.variance"]
* reflections["prescaling_correction"] ** 2
),
rel=2e-2,
)
