def find_nearest_neighbour(image, mask, reflections):
from annlib_ext import AnnAdaptor
from scitbx.array_family import flex
from math import sqrt
import numpy
# Get the predicted coords
pred_xyz = []
for r in reflections:
x = r.image_coord_px[0]
y = r.image_coord_px[1]
z = r.frame_number
pred_xyz.append((x, y, z))
# Create the KD Tree
ann = AnnAdaptor(flex.double(pred_xyz).as_1d(), 3)
pixel_xyz = []
ind = numpy.where(mask != 0)
z = ind[0]
y = ind[1]
x = ind[2]
ann.query(flex.double(zip(x, y, z)).as_1d())
# for i in xrange(len(ann.nn)):
# print "Neighbor of {0}, index {1} distance {2}".format(
# obs_xyz[i], ann.nn[i], sqrt(ann.distances[i]))
owner = numpy.zeros(shape=mask.shape, dtype=numpy.int32)
owner[ind] = ann.nn.as_numpy_array()
return owner
def find_pixel_nearest_neighbour(image, mask, reflections):
from annlib_ext import AnnAdaptor
from scitbx.array_family import flex
from math import sqrt
import numpy
# Get the predicted coords
pred_xyz = []
for r in reflections:
x = r.image_coord_px[0] * 0.172
y = r.image_coord_px[1] * 0.172
z = r.frame_number * 0.2
pred_xyz.append((x, y, z))
# Create the KD Tree
ann = AnnAdaptor(flex.double(pred_xyz).as_1d(), 3)
pixel_xyz = []
ind = numpy.where(mask != 0)
z = ind[0] * 0.2
y = ind[1] * 0.172
x = ind[2] * 0.172
ann.query(flex.double(zip(x, y, z)).as_1d())
# for i in xrange(len(ann.nn)):
# print "Neighbor of {0}, index {1} distance {2}".format(
# obs_xyz[i], ann.nn[i], sqrt(ann.distances[i]))
owner = numpy.zeros(shape=mask.shape, dtype=numpy.int32)
owner[ind] = ann.nn.as_numpy_array()
return owner
def find_nearest_neighbour(obs_xyz, reflections):
from annlib_ext import AnnAdaptor
from scitbx.array_family import flex
from math import sqrt
# Get the predicted coords
pred_xyz = []
for r in reflections:
x = r.image_coord_px[0]# * 0.172
y = r.image_coord_px[1]# * 0.172
z = r.frame_number# * 0.2
pred_xyz.append((x, y, z))
# Create the KD Tree
ann = AnnAdaptor(flex.double(pred_xyz).as_1d(), 3)
# obs_xyz2 = []
# for x, y, z in obs_xyz:
# x = x * 0.172
# y = y * 0.172
# z = z * 0.2
# obs_xyz2.append((x, y, z))
ann.query(flex.double(obs_xyz).as_1d())
# for i in xrange(len(ann.nn)):
# print "Neighbor of {0}, index {1} distance {2}".format(
# obs_xyz[i], ann.nn[i], sqrt(ann.distances[i]))
return ann.nn, ann.distances
def interpolate_x_value(self, y):
# return np.random.normal(1000., 10)
# if self.N < 20:
lookup_table_cdf, lookup_table_x = self.create_exgauss_lookup_table()
# else:
#lookup_table_cdf, lookup_table_x = self.create_smarter_lookup_table(y)
lookup_table_cdf = flex.double(lookup_table_cdf)
# print 'Length of lookup table = ', len(lookup_table_cdf)
# del(lookup_table_cdf)
# del(lookup_table_x)
# return np.random.normal(1000., 10)
from annlib_ext import AnnAdaptor
A = AnnAdaptor(lookup_table_cdf, 1)
A.query([y])
idx = A.nn[0]
try:
y1 = lookup_table_cdf[idx]
x1 = lookup_table_x[idx]
if y > y1:
y2 = lookup_table_cdf[idx + 1]
x2 = lookup_table_x[idx + 1]
else:
y2 = lookup_table_cdf[idx - 1]
x2 = lookup_table_x[idx - 1]
x = ((x2 - x1) / (y2 - y1)) * (y - y1) + x1
# print 'true cdf value from interpol',self.exgauss_cdf(x)
return x
except:
print 'in the except block', y
# print 'true cdf value from interpol',self.exgauss_cdf(x)
return lookup_table_x[idx]
def find_nearest_neighbour(obs_xyz, reflections):
from annlib_ext import AnnAdaptor
from scitbx.array_family import flex
from math import sqrt
# Get the predicted coords
pred_xyz = []
for r in reflections:
x = r.image_coord_px[0] # * 0.172
y = r.image_coord_px[1] # * 0.172
z = r.frame_number # * 0.2
pred_xyz.append((x, y, z))
# Create the KD Tree
ann = AnnAdaptor(flex.double(pred_xyz).as_1d(), 3)
# obs_xyz2 = []
# for x, y, z in obs_xyz:
# x = x * 0.172
# y = y * 0.172
# z = z * 0.2
# obs_xyz2.append((x, y, z))
ann.query(flex.double(obs_xyz).as_1d())
# for i in xrange(len(ann.nn)):
# print "Neighbor of {0}, index {1} distance {2}".format(
# obs_xyz[i], ann.nn[i], sqrt(ann.distances[i]))
return ann.nn, ann.distances
def _find_nearest_neighbours(self, observed, predicted):
'''Find the nearest predicted spot to the observed spot.
Params:
observed The observed reflections
predicted The predicted reflections
Returns:
(nearest neighbours, distance)
'''
from annlib_ext import AnnAdaptor
from scitbx.array_family import flex
from math import sqrt
# Get the predicted coordinates
predicted_xyz = []
for r in predicted:
x, y = r.image_coord_px
z = r.frame_number
predicted_xyz.append((x, y, z))
# Create the KD Tree
ann = AnnAdaptor(flex.double(predicted_xyz).as_1d(), 3)
# Get the observed coordinates
observed_xyz = [r.centroid_position for r in observed]
# Query to find all the nearest neighbours
ann.query(flex.double(observed_xyz).as_1d())
# Return the nearest neighbours and distances
return ann.nn, flex.sqrt(ann.distances)
def find_x_from_expdata_annlib(self, data, y):
''' find the corresponding x value of a desired y_cdf value, given exp data
'''
exgauss_rand = []
lookup_table_x = np.sort(data)
lookup_table_cdf = np.array(range(1,
len(lookup_table_x) + 1)) / float(
len(lookup_table_x))
lookup_table_cdf[:] = [
z - 0.5 / len(lookup_table_cdf) for z in lookup_table_cdf
]
y_cdf = lookup_table_cdf
lookup_table_cdf = flex.double(lookup_table_cdf)
from annlib_ext import AnnAdaptor
A = AnnAdaptor(lookup_table_cdf, 1)
A.query([y])
idx = A.nn[0]
try:
y1 = lookup_table_cdf[idx]
x1 = lookup_table_x[idx]
if y > y1:
y2 = lookup_table_cdf[idx + 1]
x2 = lookup_table_x[idx + 1]
else:
y2 = lookup_table_cdf[idx - 1]
x2 = lookup_table_x[idx - 1]
x = ((x2 - x1) / (y2 - y1)) * (y - y1) + x1
# print 'true cdf value from interpol',self.exgauss_cdf(x)
return (x, y_cdf)
except:
print 'in the except block of find_x_from_expdata_annlib', y
# print 'true cdf value from interpol',self.exgauss_cdf(x)
return (lookup_table_x[idx], y_cdf)
def __init__(self, params, exlist, reference=None,
predicted=None, shoeboxes=None):
'''Initialise the script.'''
assert reference is not None
# Load the extractor based on the input
if shoeboxes is not None:
extractor = self._load_extractor(shoeboxes, params, exlist)
else:
if predicted is None:
predicted = self._predict_reflections(params, exlist)
#predicted = self._filter_reflections(params, exlist, predicted) # FIXME
predicted = self._match_with_reference(predicted, reference)
from annlib_ext import AnnAdaptor
from dials.array_family import flex
import math
matcheddata = predicted.select(predicted.get_flags(predicted.flags.reference_spot))
A = AnnAdaptor(matcheddata['xyzcal.mm'].as_double(), 3, 10)
A.query(predicted['xyzcal.mm'].as_double())
bboxes = flex.int6()
for i, ref in enumerate(predicted):
nn_pred = [matcheddata[A.nn[i*10+j]] for j in xrange(10)]
nn_ref = [reference[reference['miller_index'].first_index(r['miller_index'])] for r in nn_pred]
max_x = max([r['bbox'][1]-r['bbox'][0] for r in nn_ref])
max_y = max([r['bbox'][3]-r['bbox'][2] for r in nn_ref])
max_z = max([r['bbox'][5]-r['bbox'][4] for r in nn_ref])
panel = exlist[ref['id']].detector[ref['panel']]
imgsize_x, imgsize_y = panel.get_image_size()
x1 = int(math.floor(ref['xyzcal.px'][0] - (max_x / 2)))
x2 = int(math.ceil (ref['xyzcal.px'][0] + (max_x / 2)))
y1 = int(math.floor(ref['xyzcal.px'][1] - (max_y / 2)))
y2 = int(math.ceil (ref['xyzcal.px'][1] + (max_y / 2)))
if x1 < 0: x1 = 0
if y1 < 0: y1 = 0
if x2 > imgsize_x: x2 = imgsize_x
if y2 > imgsize_y: y2 = imgsize_y
bboxes.append((x1,x2,y1,y2,0,1))
predicted['bbox'] = bboxes
extractor = self._create_extractor(params, exlist, predicted)
# Initialise the integrator
self._integrator = ReflectionBlockIntegratorStills(params, exlist, reference, extractor)
def __init__(self, params, exlist, reference=None,
predicted=None, shoeboxes=None):
'''Initialise the script.'''
assert reference is not None
# Load the extractor based on the input
if shoeboxes is not None:
extractor = self._load_extractor(shoeboxes, params, exlist)
else:
if predicted is None:
predicted = self._predict_reflections(params, exlist)
#predicted = self._filter_reflections(params, exlist, predicted) # FIXME
predicted = self._match_with_reference(predicted, reference)
from annlib_ext import AnnAdaptor
from dials.array_family import flex
import math
matcheddata = predicted.select(predicted.get_flags(predicted.flags.reference_spot))
A = AnnAdaptor(matcheddata['xyzcal.mm'].as_double(), 3, 10)
A.query(predicted['xyzcal.mm'].as_double())
bboxes = flex.int6()
for i, ref in enumerate(predicted):
nn_pred = [matcheddata[A.nn[i*10+j]] for j in xrange(10)]
nn_ref = [reference[reference['miller_index'].first_index(r['miller_index'])] for r in nn_pred]
max_x = max([r['bbox'][1]-r['bbox'][0] for r in nn_ref])
max_y = max([r['bbox'][3]-r['bbox'][2] for r in nn_ref])
max_z = max([r['bbox'][5]-r['bbox'][4] for r in nn_ref])
panel = exlist[ref['id']].detector[ref['panel']]
imgsize_x, imgsize_y = panel.get_image_size()
x1 = int(math.floor(ref['xyzcal.px'][0] - (max_x / 2)))
x2 = int(math.ceil (ref['xyzcal.px'][0] + (max_x / 2)))
y1 = int(math.floor(ref['xyzcal.px'][1] - (max_y / 2)))
y2 = int(math.ceil (ref['xyzcal.px'][1] + (max_y / 2)))
if x1 < 0: x1 = 0
if y1 < 0: y1 = 0
if x2 > imgsize_x: x2 = imgsize_x
if y2 > imgsize_y: y2 = imgsize_y
bboxes.append((x1,x2,y1,y2,0,1))
predicted['bbox'] = bboxes
extractor = self._create_extractor(params, exlist, predicted)
# Initialise the integrator
self._integrator = ReflectionBlockIntegratorStills(params, exlist, reference, extractor)
def __init__(self, rs_vectors, percentile=0.05):
from scitbx.array_family import flex
NEAR = 10
self.NNBIN = 5 # target number of neighbors per histogram bin
# nearest neighbor analysis
from annlib_ext import AnnAdaptor
query = flex.double()
for spot in rs_vectors: # spots, in reciprocal space xyz
query.append(spot[0])
query.append(spot[1])
query.append(spot[2])
assert len(
rs_vectors) > NEAR # Can't do nearest neighbor with too few spots
IS_adapt = AnnAdaptor(data=query, dim=3, k=1)
IS_adapt.query(query)
direct = flex.double()
for i in range(len(rs_vectors)):
direct.append(1.0 / math.sqrt(IS_adapt.distances[i]))
# determine the most probable nearest neighbor distance (direct space)
hst = flex.histogram(direct, n_slots=int(len(rs_vectors) / self.NNBIN))
centers = hst.slot_centers()
islot = hst.slots()
highest_bin_height = flex.max(islot)
most_probable_neighbor = centers[list(islot).index(highest_bin_height)]
if False: # to print out the histogramming analysis
smin, smax = flex.min(direct), flex.max(direct)
stats = flex.mean_and_variance(direct)
import sys
out = sys.stdout
print(" range: %6.2f - %.2f" % (smin, smax), file=out)
print(" mean: %6.2f +/- %6.2f on N = %d" %
(stats.mean(), stats.unweighted_sample_standard_deviation(),
direct.size()),
file=out)
hst.show(f=out, prefix=" ", format_cutoffs="%6.2f")
print("", file=out)
# determine the 5th-percentile direct-space distance
perm = flex.sort_permutation(direct, reverse=True)
percentile = direct[perm[int(percentile * len(rs_vectors))]]
MAXTOL = 1.5 # Margin of error for max unit cell estimate
self.max_cell = max(MAXTOL * most_probable_neighbor,
MAXTOL * percentile)
if False:
self.plot(direct)
def excercise_nearest_neighbor():
data,query = data_from_files()
S = StringIO()
A = AnnAdaptor(data,2) # construct k-d tree for reference set
A.query(query) # find nearest neighbors of query points
for i in range(len(A.nn)):
print >>S,"Neighbor of (%7.1f,%7.1f), index %6d distance %4.1f"%(
query[2*i],query[2*i+1],A.nn[i],math.sqrt(A.distances[i]))
return S.getvalue()
def excercise_nearest_neighbor():
data,query = data_from_files()
S = StringIO.StringIO()
A = AnnAdaptor(data,2) # construct k-d tree for reference set
A.query(query) # find nearest neighbors of query points
for i in xrange(len(A.nn)):
print >>S,"Neighbor of (%7.1f,%7.1f), index %6d distance %4.1f"%(
query[2*i],query[2*i+1],A.nn[i],math.sqrt(A.distances[i]))
return S.getvalue()
def __init__(self, rs_vectors, percentile=0.05):
from scitbx.array_family import flex
NEAR = 10
self.NNBIN = 5 # target number of neighbors per histogram bin
# nearest neighbor analysis
from annlib_ext import AnnAdaptor
query = flex.double()
for spot in rs_vectors: # spots, in reciprocal space xyz
query.append(spot[0])
query.append(spot[1])
query.append(spot[2])
assert len(rs_vectors)>NEAR # Can't do nearest neighbor with too few spots
IS_adapt = AnnAdaptor(data=query,dim=3,k=1)
IS_adapt.query(query)
direct = flex.double()
for i in xrange(len(rs_vectors)):
direct.append(1.0/math.sqrt(IS_adapt.distances[i]))
# determine the most probable nearest neighbor distance (direct space)
hst = flex.histogram(direct, n_slots=int(len(rs_vectors)/self.NNBIN))
centers = hst.slot_centers()
islot = hst.slots()
highest_bin_height = flex.max(islot)
most_probable_neighbor = centers[list(islot).index(highest_bin_height)]
if False: # to print out the histogramming analysis
smin, smax = flex.min(direct), flex.max(direct)
stats = flex.mean_and_variance(direct)
import sys
out = sys.stdout
print >> out, " range: %6.2f - %.2f" % (smin, smax)
print >> out, " mean: %6.2f +/- %6.2f on N = %d" % (
stats.mean(), stats.unweighted_sample_standard_deviation(), direct.size())
hst.show(f=out, prefix=" ", format_cutoffs="%6.2f")
print >> out, ""
# determine the 5th-percentile direct-space distance
perm = flex.sort_permutation(direct, reverse=True)
percentile = direct[perm[int(percentile * len(rs_vectors))]]
MAXTOL = 1.5 # Margin of error for max unit cell estimate
self.max_cell = max( MAXTOL * most_probable_neighbor,
MAXTOL * percentile)
if False:
self.plot(direct)
def __init__(self,IT):
from scitbx import matrix
self.IT = IT
from annlib_ext import AnnAdaptor
reference = flex.double()
for i in range(len(IT)//4):
UL = matrix.col((float(IT[4*i]),float(IT[4*i+1])))
LR = matrix.col((float(IT[4*i+2]),float(IT[4*i+3])))
center = (UL+LR)/2.
reference.append(center[0])
reference.append(center[1])
self.adapt = AnnAdaptor(data=reference,dim=2,k=self.NEAR)
class active_area_filter:
NEAR = 2
def __init__(self, IT):
from scitbx import matrix
self.IT = IT
from annlib_ext import AnnAdaptor
reference = flex.double()
for i in xrange(len(IT) // 4):
UL = matrix.col((float(IT[4 * i]), float(IT[4 * i + 1])))
LR = matrix.col((float(IT[4 * i + 2]), float(IT[4 * i + 3])))
center = (UL + LR) / 2.0
reference.append(center[0])
reference.append(center[1])
self.adapt = AnnAdaptor(data=reference, dim=2, k=self.NEAR)
def __call__(self, predictions, hkllist, pxlsz):
if len(self.IT) == 4:
# We have only one tile, AnnAdaptor chokes in this case but then there is
# only one choice of nearest neighbour anyway!
nearest_neighbours = flex.int(len(predictions) * self.NEAR, 0)
else:
query = flex.double()
for pred in predictions:
query.append(pred[0] / pxlsz)
query.append(pred[1] / pxlsz)
self.adapt.query(query)
assert len(self.adapt.nn) == len(predictions) * self.NEAR
nearest_neighbours = self.adapt.nn
selection = flex.bool()
self.tile_id = flex.int()
for p in xrange(len(predictions)):
is_in_active_area = False
for n in xrange(self.NEAR):
itile = nearest_neighbours[p * self.NEAR + n]
if (
self.IT[4 * itile] < predictions[p][0] / pxlsz < self.IT[4 * itile + 2]
and self.IT[4 * itile + 1] < predictions[p][1] / pxlsz < self.IT[4 * itile + 3]
):
is_in_active_area = True
break
if is_in_active_area:
self.tile_id.append(itile)
selection.append(is_in_active_area)
assert selection.count(True) == len(self.tile_id)
return predictions.select(selection), hkllist.select(selection)
def restrict_target_angles_to_quick_winners(self, quick_sampling):
# find which target_grid directions are near neighbors to
# the quick_sampling winners from the first round of testing.
# in the second round, restrict the calculation of FFTs to those directions
self.restricted = flex.Direction()
target = flex.double()
for iz in self.angles:
# fairly inefficient in Python; expect big improvement in C++
v = iz.dvec
target.append(v[0])
target.append(v[1])
target.append(v[2])
kn = int(2 * self.second_round_sampling)
#construct k-d tree for the reference set
A = AnnAdaptor(data=target, dim=3, k=kn)
query = flex.double()
for j in quick_sampling:
v = j.dvec
query.append(v[0])
query.append(v[1])
query.append(v[2])
if abs((math.pi / 2.) -
j.psi) < 0.0001: #take care of equatorial boundary
query.append(-v[0])
query.append(-v[1])
query.append(-v[2])
A.query(query) #find nearest neighbors of query points
neighbors = flex.sqrt(A.distances)
neighborid = A.nn
accept_flag = flex.bool(len(self.angles))
for idx in range(len(neighbors)):
# use small angle approximation to test if target is within desired radius
if neighbors[idx] < self.quick_grid:
accept_flag[neighborid[idx]] = True
#go through all of the original target angles
for iz in range(len(self.angles)):
if accept_flag[iz]:
self.restricted.append(self.angles[iz])
self.angles = self.restricted
class active_area_filter:
NEAR = 2
def __init__(self, IT):
from scitbx import matrix
self.IT = IT
from annlib_ext import AnnAdaptor
reference = flex.double()
for i in range(len(IT) // 4):
UL = matrix.col((float(IT[4 * i]), float(IT[4 * i + 1])))
LR = matrix.col((float(IT[4 * i + 2]), float(IT[4 * i + 3])))
center = (UL + LR) / 2.
reference.append(center[0])
reference.append(center[1])
self.adapt = AnnAdaptor(data=reference, dim=2, k=self.NEAR)
def __call__(self, predictions, hkllist, pxlsz):
if len(self.IT) == 4:
# We have only one tile, AnnAdaptor chokes in this case but then there is
# only one choice of nearest neighbour anyway!
nearest_neighbours = flex.int(len(predictions) * self.NEAR, 0)
else:
query = flex.double()
for pred in predictions:
query.append(pred[0] / pxlsz)
query.append(pred[1] / pxlsz)
self.adapt.query(query)
assert len(self.adapt.nn) == len(predictions) * self.NEAR
nearest_neighbours = self.adapt.nn
selection = flex.bool()
self.tile_id = flex.int()
for p in range(len(predictions)):
is_in_active_area = False
for n in range(self.NEAR):
itile = nearest_neighbours[p * self.NEAR + n]
if self.IT[4*itile]<predictions[p][0]/pxlsz<self.IT[4*itile+2] and\
self.IT[4*itile+1]<predictions[p][1]/pxlsz<self.IT[4*itile+3]:
is_in_active_area = True
break
if is_in_active_area:
self.tile_id.append(itile)
selection.append(is_in_active_area)
assert selection.count(True) == len(self.tile_id)
return predictions.select(selection), hkllist.select(selection)
def _find_nearest_neighbours_single(self, oxyz, pxyz):
"""
Find the nearest predicted spot to the observed spot.
:param observed: The observed reflections
:param predicted: The predicted reflections
:returns: (nearest neighbours, distance)
"""
from annlib_ext import AnnAdaptor
# Create the KD Tree
ann = AnnAdaptor(pxyz.as_double().as_1d(), 3)
# Query to find all the nearest neighbours
ann.query(oxyz.as_double().as_1d())
# Return the nearest neighbours and distances
return ann.nn, flex.sqrt(ann.distances)
def rand_annlib(self, seed):
exgauss_rand = []
np.random.seed(seed)
query = np.random.rand(self.N)
lookup_table_cdf, lookup_table_x = self.create_exgauss_lookup_table()
lookup_table_cdf = flex.double(lookup_table_cdf)
# t1 = time.time()
from annlib_ext import AnnAdaptor
A = AnnAdaptor(lookup_table_cdf, 1)
A.query(query)
# ================== If you want to test timing comment out below ================
# for i in xrange(len(A.nn)):
# print "Neighbor of (%12.7f), index %6d distance %12.5f"%(
# query[i],A.nn[i],math.sqrt(A.distances[i]))
# t2 = time.time()
# print 'Time Taken by Annlib = %12.7f'%(t2-t1)
# lookup_table_cdf = lookup_table_cdf.as_numpy_array().tolist()
# t1 = time.time()
# for y in query:
# idx = lookup_table_cdf.index(min(lookup_table_cdf, key=lambda y0:abs(y0-y)))
# print 'From Naive Search (%12.7f), index %6d'%(y, idx)
# t2 = time.time()
# print 'Time taken by Naive = %12.7f'%(t2-t1)
## ====================== Test Over ================================================
for i in xrange(len(A.nn)):
idx = A.nn[i]
y = query[i]
try:
y1 = lookup_table_cdf[idx]
x1 = lookup_table_x[idx]
if y > y1:
y2 = lookup_table_cdf[idx + 1]
x2 = lookup_table_x[idx + 1]
else:
y2 = lookup_table_cdf[idx - 1]
x2 = lookup_table_x[idx - 1]
x = ((x2 - x1) / (y2 - y1)) * (y - y1) + x1
exgauss_rand.append(x)
except:
print 'in the except block'
exgauss_rand.append(lookup_table_x[idx])
return exgauss_rand
def interpolate_x_value(self, y):
lookup_table_cdf, lookup_table_x = self.create_exgauss_lookup_table()
lookup_table_cdf = flex.double(lookup_table_cdf)
A = AnnAdaptor(lookup_table_cdf, 1)
A.query([y])
idx = A.nn[0]
try:
y1 = lookup_table_cdf[idx]
x1 = lookup_table_x[idx]
if y > y1:
y2 = lookup_table_cdf[idx + 1]
x2 = lookup_table_x[idx + 1]
else:
y2 = lookup_table_cdf[idx - 1]
x2 = lookup_table_x[idx - 1]
x = ((x2 - x1) / (y2 - y1)) * (y - y1) + x1
return x
except:
print 'in the except block', y
return lookup_table_x[idx]
def _find_nearest_neighbours_single(self, oxyz, pxyz):
'''
Find the nearest predicted spot to the observed spot.
:param observed: The observed reflections
:param predicted: The predicted reflections
:returns: (nearest neighbours, distance)
'''
from annlib_ext import AnnAdaptor
from scitbx.array_family import flex
# Create the KD Tree
ann = AnnAdaptor(pxyz.as_double().as_1d(), 3)
# Query to find all the nearest neighbours
ann.query(oxyz.as_double().as_1d())
# Return the nearest neighbours and distances
return ann.nn, flex.sqrt(ann.distances)
def restrict_target_angles_to_quick_winners(self,quick_sampling):
# find which target_grid directions are near neighbors to
# the quick_sampling winners from the first round of testing.
# in the second round, restrict the calculation of FFTs to those directions
self.restricted = flex.Direction()
target = flex.double()
for iz in self.angles:
# fairly inefficient in Python; expect big improvement in C++
v = iz.dvec
target.append(v[0]);target.append(v[1]);target.append(v[2]);
kn = int(2 *self.second_round_sampling)
#construct k-d tree for the reference set
A = AnnAdaptor(data = target, dim = 3, k = kn)
query = flex.double()
for j in quick_sampling:
v = j.dvec
query.append(v[0]);query.append(v[1]); query.append(v[2]);
if abs((math.pi/2.) - j.psi) < 0.0001: #take care of equatorial boundary
query.append(-v[0]);query.append(-v[1]); query.append(-v[2]);
A.query(query) #find nearest neighbors of query points
neighbors = flex.sqrt(A.distances)
neighborid = A.nn
accept_flag = flex.bool(len(self.angles))
for idx in xrange(len(neighbors)):
# use small angle approximation to test if target is within desired radius
if neighbors[idx] < self.quick_grid:
accept_flag[neighborid[idx]] = True
#go through all of the original target angles
for iz in xrange(len(self.angles)):
if accept_flag[iz]:
self.restricted.append(self.angles[iz])
self.angles = self.restricted
def specific_libann_cluster(data, intensity_cutoff=25, distance_cutoff=17):
from annlib_ext import AnnAdaptor
#construct a new data structure containing only pixels > intensity_cutoff
#input data structure is a flex array
info = {}
shape = data.accessor().focus()
for slow in range(shape[0]):
for fast in range(shape[1]):
if data[(slow, fast)] > intensity_cutoff:
info[(slow, fast)] = data[(slow, fast)]
Ktree = int(distance_cutoff * distance_cutoff * pi)
Sq_cut = distance_cutoff * distance_cutoff # distance < distance_cutoff => the two points are clustered
all_pixels = flex.double()
all_keys = info.keys()
for key in info.keys():
all_pixels.append(key[0])
all_pixels.append(key[1])
distance_tree = AnnAdaptor(data=all_pixels, dim=2, k=Ktree)
distance_tree.query(all_pixels)
clusters = []
membership_lookup = {}
for i_query_pt in range(len(all_keys)):
query_coords = all_keys[i_query_pt]
query_ok = True
for i_target_pt in range(Ktree):
target_coords = all_keys[distance_tree.nn[Ktree * i_query_pt +
i_target_pt]]
if distance_tree.distances[Ktree * i_query_pt +
i_target_pt] < Sq_cut:
if info[query_coords] < info[target_coords]:
query_ok = False
break
if query_ok:
membership_lookup[query_coords] = info[query_coords]
return membership_lookup
def specific_libann_cluster(data,intensity_cutoff = 25,distance_cutoff=17):
from annlib_ext import AnnAdaptor
#construct a new data structure containing only pixels > intensity_cutoff
#input data structure is a flex array
info = {}
shape = data.accessor().focus()
for slow in xrange(shape[0]):
for fast in xrange(shape[1]):
if data[(slow,fast)] > intensity_cutoff:
info[(slow,fast)] = data[(slow,fast)]
Ktree = int(distance_cutoff*distance_cutoff*pi)
Sq_cut = distance_cutoff*distance_cutoff # distance < distance_cutoff => the two points are clustered
all_pixels = flex.double()
all_keys = info.keys()
for key in info.keys():
all_pixels.append(key[0]); all_pixels.append(key[1])
distance_tree = AnnAdaptor(data=all_pixels,dim=2,k=Ktree)
distance_tree.query(all_pixels)
clusters = []
membership_lookup = {}
for i_query_pt in xrange(len(all_keys)):
query_coords = all_keys[i_query_pt]
query_ok = True
for i_target_pt in xrange(Ktree):
target_coords = all_keys[distance_tree.nn[Ktree*i_query_pt+i_target_pt]]
if distance_tree.distances[Ktree*i_query_pt+i_target_pt] < Sq_cut:
if info[query_coords] < info[target_coords]:
query_ok = False
break
if query_ok:
membership_lookup[query_coords]=info[query_coords]
return membership_lookup
def __init__(self, IT):
from scitbx import matrix
self.IT = IT
from annlib_ext import AnnAdaptor
reference = flex.double()
for i in xrange(len(IT) // 4):
UL = matrix.col((float(IT[4 * i]), float(IT[4 * i + 1])))
LR = matrix.col((float(IT[4 * i + 2]), float(IT[4 * i + 3])))
center = (UL + LR) / 2.0
reference.append(center[0])
reference.append(center[1])
self.adapt = AnnAdaptor(data=reference, dim=2, k=self.NEAR)
def excercise_nearest_neighbor():
data = data_from_files()
A = AnnAdaptor(data, 2) # construct k-d tree for reference set
A.query(data) # find nearest neighbors of query points
for i in range(len(A.nn)):
#print "Neighbor of (%7.1f,%7.1f), index %6d distance %4.1f"%(
#data[2*i],data[2*i+1],A.nn[i],math.sqrt(A.distances[i]))
assert A.nn[i] != i
A = AnnAdaptorSelfInclude(data, 2) # construct k-d tree for reference set
A.query(data) # find nearest neighbors of query points
for i in range(len(A.nn)):
#print "Neighbor of (%7.1f,%7.1f), index %6d distance %4.1f"%(
#data[2*i],data[2*i+1],A.nn[i],math.sqrt(A.distances[i]))
assert A.nn[i] == i
def excercise_nearest_neighbor():
data = data_from_files()
A = AnnAdaptor(data,2)# construct k-d tree for reference set
A.query(data) # find nearest neighbors of query points
for i in range(len(A.nn)):
#print "Neighbor of (%7.1f,%7.1f), index %6d distance %4.1f"%(
#data[2*i],data[2*i+1],A.nn[i],math.sqrt(A.distances[i]))
assert A.nn[i]!=i
A = AnnAdaptorSelfInclude(data,2)# construct k-d tree for reference set
A.query(data) # find nearest neighbors of query points
for i in range(len(A.nn)):
#print "Neighbor of (%7.1f,%7.1f), index %6d distance %4.1f"%(
#data[2*i],data[2*i+1],A.nn[i],math.sqrt(A.distances[i]))
assert A.nn[i]==i
def test3(dials_regression, tmpdir):
"""Strict check for scan-varying refinement using automated outlier rejection
block width and interval width setting"""
# use the i04_weak_data for this test
data_dir = os.path.join(dials_regression, "refinement_test_data",
"centroid")
experiments_path = os.path.join(data_dir,
"experiments_XPARM_REGULARIZED.json")
pickle_path = os.path.join(data_dir, "spot_all_xds.pickle")
for pth in (experiments_path, pickle_path):
assert os.path.exists(pth)
result = procrunner.run(
(
"dials.refine",
experiments_path,
pickle_path,
"scan_varying=true",
"max_iterations=5",
"output.history=history.json",
"crystal.orientation.smoother.interval_width_degrees=auto",
"crystal.unit_cell.smoother.interval_width_degrees=auto",
),
working_directory=tmpdir,
)
assert not result.returncode and not result.stderr
# load and check results
history = Journal.from_json_file(tmpdir.join("history.json").strpath)
expected_rmsds = [
[0.619507829, 0.351326044, 0.006955399],
[0.174024575, 0.113486044, 0.004704006],
[0.098351363, 0.084052519, 0.002660408],
[0.069202909, 0.072796782, 0.001451734],
[0.064305277, 0.071560831, 0.001165639],
[0.062955462, 0.071315612, 0.001074453],
]
for a, b in zip(history["rmsd"], expected_rmsds):
assert a == pytest.approx(b, abs=1e-6)
# check the refined unit cell
ref_exp = ExperimentListFactory.from_json_file(
tmpdir.join("refined.expt").strpath, check_format=False)[0]
unit_cell = ref_exp.crystal.get_unit_cell().parameters()
assert unit_cell == pytest.approx(
[42.27482, 42.27482, 39.66893, 90.00000, 90.00000, 90.00000], abs=1e-3)
refined_refl = flex.reflection_table.from_file(
tmpdir.join("refined.refl").strpath)
# re-predict reflections using the refined experiments
predicted = flex.reflection_table.from_predictions_multi([ref_exp])
matched, reference, unmatched = predicted.match_with_reference(
refined_refl)
# assert most refined reflections are matched with predictions
assert reference.size() > (0.997 * refined_refl.size())
# second check with nearest neighbour matching that the predictions match up
ann = AnnAdaptor(data=predicted["xyzcal.px"].as_double(), dim=3, k=1)
ann.query(refined_refl["xyzcal.px"].as_double())
assert (ann.distances < 0.5).count(True) > (0.998 * refined_refl.size())
def __init__(self,
reflections,
step_size=45,
tolerance=1.5,
max_height_fraction=0.25,
percentile=None,
histogram_binning='linear',
nn_per_bin=5):
self.tolerance = tolerance # Margin of error for max unit cell estimate
from scitbx.array_family import flex
NEAR = 10
self.NNBIN = nn_per_bin # target number of neighbors per histogram bin
self.histogram_binning = histogram_binning
direct = flex.double()
if 'entering' in reflections:
entering_flags = reflections['entering']
else:
entering_flags = flex.bool(reflections.size(), True)
rs_vectors = reflections['rlp']
phi_deg = reflections['xyzobs.mm.value'].parts()[2] * (180 / math.pi)
d_spacings = flex.double()
# nearest neighbor analysis
from annlib_ext import AnnAdaptor
for imageset_id in range(flex.max(reflections['imageset_id']) + 1):
sel_imageset = reflections['imageset_id'] == imageset_id
if sel_imageset.count(True) == 0:
continue
phi_min = flex.min(phi_deg.select(sel_imageset))
phi_max = flex.max(phi_deg.select(sel_imageset))
d_phi = phi_max - phi_min
n_steps = max(int(math.ceil(d_phi / step_size)), 1)
for n in range(n_steps):
sel_step = sel_imageset & (phi_deg >=
(phi_min + n * step_size)) & (
phi_deg <
(phi_min + (n + 1) * step_size))
for entering in (True, False):
sel_entering = sel_step & (entering_flags == entering)
if sel_entering.count(True) == 0:
continue
query = flex.double()
query.extend(rs_vectors.select(sel_entering).as_double())
if query.size() == 0:
continue
IS_adapt = AnnAdaptor(data=query, dim=3, k=1)
IS_adapt.query(query)
direct.extend(1 / flex.sqrt(IS_adapt.distances))
d_spacings.extend(1 / rs_vectors.norms())
assert len(direct) > NEAR, (
"Too few spots (%d) for nearest neighbour analysis." % len(direct))
perm = flex.sort_permutation(direct)
direct = direct.select(perm)
d_spacings = d_spacings.select(perm)
# eliminate nonsensical direct space distances
sel = direct > 1
direct = direct.select(sel)
d_spacings = d_spacings.select(sel)
if percentile is None:
# reject top 1% of longest distances to hopefully get rid of any outliers
n = int(math.floor(0.99 * len(direct)))
direct = direct[:n]
d_spacings = d_spacings[:n]
# determine the most probable nearest neighbor distance (direct space)
if self.histogram_binning == 'log':
hst = flex.histogram(flex.log10(direct),
n_slots=int(len(direct) / self.NNBIN))
else:
hst = flex.histogram(direct, n_slots=int(len(direct) / self.NNBIN))
centers = hst.slot_centers()
if self.histogram_binning == 'log':
self.slot_start = flex.double(
[10**(s - 0.5 * hst.slot_width()) for s in hst.slot_centers()])
self.slot_end = flex.double(
[10**(s + 0.5 * hst.slot_width()) for s in hst.slot_centers()])
self.slot_width = self.slot_end - self.slot_start
else:
self.slot_start = hst.slot_centers() - 0.5 * hst.slot_width()
self.slot_end = hst.slot_centers() + 0.5 * hst.slot_width()
self.slot_width = hst.slot_width()
self.relative_frequency = hst.slots().as_double() / self.slot_width
highest_bin_height = flex.max(self.relative_frequency)
if False: # to print out the histogramming analysis
smin, smax = flex.min(direct), flex.max(direct)
stats = flex.mean_and_variance(direct)
import sys
out = sys.stdout
print >> out, " range: %6.2f - %.2f" % (smin, smax)
print >> out, " mean: %6.2f +/- %6.2f on N = %d" % (
stats.mean(), stats.unweighted_sample_standard_deviation(),
direct.size())
hst.show(f=out, prefix=" ", format_cutoffs="%6.2f")
print >> out, ""
if percentile is not None:
# determine the nth-percentile direct-space distance
perm = flex.sort_permutation(direct, reverse=True)
self.max_cell = self.tolerance * direct[perm[int(
(1 - percentile) * len(direct))]]
else:
# choose a max cell based on bins above a given fraction of the highest bin height
# given multiple
isel = (self.relative_frequency.as_double() >
(max_height_fraction * highest_bin_height)).iselection()
self.max_cell = (self.tolerance *
self.slot_end[int(flex.max(isel.as_double()))])
self.reciprocal_lattice_vectors = rs_vectors
self.d_spacings = d_spacings
self.direct = direct
self.histogram = hst
def integration_concept(self,
image_number=0,
cb_op_to_primitive=None,
verbose=False,
**kwargs):
self.image_number = image_number
NEAR = 10
pxlsz = self.pixel_size
self.get_predictions_accounting_for_centering(cb_op_to_primitive,
**kwargs)
FWMOSAICITY = self.inputai.getMosaicity()
DOMAIN_SZ_ANG = kwargs.get("domain_size_ang",
self.__dict__.get("actual", 0))
refineflag = {True: 0, False: 1}[kwargs.get("domain_size_ang", 0) == 0]
self.inputpd["symmetry"].show_summary(
prefix="EXCURSION%1d REPORT FWMOS= %6.4f DOMAIN= %6.1f " %
(refineflag, FWMOSAICITY, DOMAIN_SZ_ANG))
from annlib_ext import AnnAdaptor
self.cell = self.inputai.getOrientation().unit_cell()
query = flex.double()
for pred in self.predicted: # predicted spot coord in pixels
query.append(pred[0] / pxlsz)
query.append(pred[1] / pxlsz)
self.reserve_hkllist_for_signal_search = self.hkllist
reference = flex.double()
spots = self.get_observations_with_outlier_removal()
assert len(
spots) > NEAR # Can't do spot/pred matching with too few spots
for spot in spots:
reference.append(spot.ctr_mass_x())
reference.append(spot.ctr_mass_y())
IS_adapt = AnnAdaptor(data=reference, dim=2, k=NEAR)
IS_adapt.query(query)
print "Calculate correction vectors for %d observations & %d predictions" % (
len(spots), len(self.predicted))
indexed_pairs_provisional = []
correction_vectors_provisional = []
c_v_p_flex = flex.vec3_double()
idx_cutoff = float(min(self.mask_focus[image_number]))
if verbose:
print "idx_cutoff distance in pixels", idx_cutoff
if not self.horizons_phil.integration.enable_one_to_one_safeguard:
# legacy code, no safeguard against many-to-one predicted-to-observation mapping
for i in range(len(self.predicted)): # loop over predicteds
#for n in range(NEAR): # loop over near spotfinder spots
for n in range(1): # only consider the nearest spotfinder spot
Match = dict(spot=IS_adapt.nn[i * NEAR + n], pred=i)
if n == 0 and math.sqrt(
IS_adapt.distances[i * NEAR + n]) < idx_cutoff:
indexed_pairs_provisional.append(Match)
vector = matrix.col([
spots[Match["spot"]].ctr_mass_x() -
self.predicted[Match["pred"]][0] / pxlsz,
spots[Match["spot"]].ctr_mass_y() -
self.predicted[Match["pred"]][1] / pxlsz
])
correction_vectors_provisional.append(vector)
c_v_p_flex.append((vector[0], vector[1], 0.))
else:
one_to_one = {}
for i in range(len(self.predicted)): # loop over predicteds
annresultidx = i * NEAR
obsidx = IS_adapt.nn[annresultidx]
this_distancesq = IS_adapt.distances[annresultidx]
if obsidx not in one_to_one or \
this_distancesq < one_to_one[obsidx]["distancesq"]:
if math.sqrt(this_distancesq) < idx_cutoff:
one_to_one[obsidx] = dict(spot=obsidx,
pred=i,
distancesq=this_distancesq)
for key, value in one_to_one.items():
indexed_pairs_provisional.append(value)
vector = matrix.col([
spots[value["spot"]].ctr_mass_x() -
self.predicted[value["pred"]][0] / pxlsz,
spots[value["spot"]].ctr_mass_y() -
self.predicted[value["pred"]][1] / pxlsz
])
correction_vectors_provisional.append(vector)
c_v_p_flex.append((vector[0], vector[1], 0.))
print "... %d provisional matches" % len(
correction_vectors_provisional),
print "r.m.s.d. in pixels: %5.2f" % (math.sqrt(
flex.mean(c_v_p_flex.dot(c_v_p_flex))))
if self.horizons_phil.integration.enable_residual_scatter:
from matplotlib import pyplot as plt
fig = plt.figure()
for cv in correction_vectors_provisional:
plt.plot([cv[1]], [-cv[0]], "b.")
plt.title(" %d matches, r.m.s.d. %5.2f pixels" %
(len(correction_vectors_provisional),
math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))))
plt.axes().set_aspect("equal")
self.show_figure(plt, fig, "res")
plt.close()
if self.horizons_phil.integration.enable_residual_map:
from matplotlib import pyplot as plt
fig = plt.figure()
for match, cv in zip(indexed_pairs_provisional,
correction_vectors_provisional):
plt.plot([spots[match["spot"]].ctr_mass_y()],
[-spots[match["spot"]].ctr_mass_x()], "r.")
plt.plot([self.predicted[match["pred"]][1] / pxlsz],
[-self.predicted[match["pred"]][0] / pxlsz], "g.")
plt.plot([
spots[match["spot"]].ctr_mass_y(),
spots[match["spot"]].ctr_mass_y() + 10. * cv[1]
], [
-spots[match["spot"]].ctr_mass_x(),
-spots[match["spot"]].ctr_mass_x() - 10. * cv[0]
], 'b-')
plt.xlim([0, float(self.inputpd["size2"])])
plt.ylim([-float(self.inputpd["size1"]), 0])
plt.title(" %d matches, r.m.s.d. %5.2f pixels" %
(len(correction_vectors_provisional),
math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))))
plt.axes().set_aspect("equal")
self.show_figure(plt, fig, "map")
plt.close()
# insert code here to remove correction length outliers...
# they are causing terrible
# problems for finding legitimate correction vectors (print out the list)
# also remove outliers for the purpose of reporting RMS
outlier_rejection = True
cache_refinement_spots = getattr(slip_callbacks.slip_callback,
"requires_refinement_spots", False)
if outlier_rejection:
correction_lengths = flex.double(
[v.length() for v in correction_vectors_provisional])
clorder = flex.sort_permutation(correction_lengths)
sorted_cl = correction_lengths.select(clorder)
ACCEPTABLE_LIMIT = 2
limit = int(
0.33 * len(sorted_cl)
) # best 1/3 of data are assumed to be correctly modeled.
if (limit <= ACCEPTABLE_LIMIT):
raise Sorry(
"Not enough indexed spots to reject outliers; have %d need >%d"
% (limit, ACCEPTABLE_LIMIT))
y_data = flex.double(len(sorted_cl))
for i in range(len(y_data)):
y_data[i] = float(i) / float(len(y_data))
# ideas are explained in Sauter & Poon (2010) J Appl Cryst 43, 611-616.
from rstbx.outlier_spots.fit_distribution import fit_cdf, rayleigh
fitted_rayleigh = fit_cdf(x_data=sorted_cl[0:limit],
y_data=y_data[0:limit],
distribution=rayleigh)
inv_cdf = [
fitted_rayleigh.distribution.inv_cdf(cdf) for cdf in y_data
]
#print "SORTED LIST OF ",len(sorted_cl), "with sigma",fitted_rayleigh.distribution.sigma
indexed_pairs = []
correction_vectors = []
self.correction_vectors = []
for icand in range(len(sorted_cl)):
# somewhat arbitrary sigma = 1.0 cutoff for outliers
if (sorted_cl[icand] - inv_cdf[icand]
) / fitted_rayleigh.distribution.sigma > 1.0:
break
indexed_pairs.append(indexed_pairs_provisional[clorder[icand]])
correction_vectors.append(
correction_vectors_provisional[clorder[icand]])
if cache_refinement_spots:
self.spotfinder.images[self.frame_numbers[
self.image_number]]["refinement_spots"].append(
spots[indexed_pairs[-1]["spot"]])
if kwargs.get("verbose_cv") == True:
print "CV OBSCENTER %7.2f %7.2f REFINEDCENTER %7.2f %7.2f" % (
float(self.inputpd["size1"]) / 2.,
float(self.inputpd["size2"]) / 2.,
self.inputai.xbeam() / pxlsz,
self.inputai.ybeam() / pxlsz),
print "OBSSPOT %7.2f %7.2f PREDSPOT %7.2f %7.2f" % (
spots[indexed_pairs[-1]["spot"]].ctr_mass_x(),
spots[indexed_pairs[-1]["spot"]].ctr_mass_y(),
self.predicted[indexed_pairs[-1]["pred"]][0] / pxlsz,
self.predicted[indexed_pairs[-1]["pred"]][1] / pxlsz),
the_hkl = self.hkllist[indexed_pairs[-1]["pred"]]
print "HKL %4d %4d %4d" % the_hkl, "%2d" % self.setting_id,
radial, azimuthal = spots[indexed_pairs[-1][
"spot"]].get_radial_and_azimuthal_size(
self.inputai.xbeam() / pxlsz,
self.inputai.ybeam() / pxlsz)
print "RADIALpx %5.3f AZIMUTpx %5.3f" % (radial, azimuthal)
# Store a list of correction vectors in self.
radial, azimuthal = spots[
indexed_pairs[-1]['spot']].get_radial_and_azimuthal_size(
self.inputai.xbeam() / pxlsz,
self.inputai.ybeam() / pxlsz)
self.correction_vectors.append(
dict(obscenter=(float(self.inputpd['size1']) / 2,
float(self.inputpd['size2']) / 2),
refinedcenter=(self.inputai.xbeam() / pxlsz,
self.inputai.ybeam() / pxlsz),
obsspot=(
spots[indexed_pairs[-1]['spot']].ctr_mass_x(),
spots[indexed_pairs[-1]['spot']].ctr_mass_y()),
predspot=(
self.predicted[indexed_pairs[-1]['pred']][0] /
pxlsz,
self.predicted[indexed_pairs[-1]['pred']][1] /
pxlsz),
hkl=(self.hkllist[indexed_pairs[-1]['pred']][0],
self.hkllist[indexed_pairs[-1]['pred']][1],
self.hkllist[indexed_pairs[-1]['pred']][2]),
setting_id=self.setting_id,
radial=radial,
azimuthal=azimuthal))
print "After outlier rejection %d indexed spotfinder spots remain." % len(
indexed_pairs)
if False:
rayleigh_cdf = [
fitted_rayleigh.distribution.cdf(x=sorted_cl[c])
for c in range(len(sorted_cl))
]
from matplotlib import pyplot as plt
plt.plot(sorted_cl, y_data, "r+")
#plt.plot(sorted_cl,rayleigh_cdf,"g.")
plt.plot(inv_cdf, y_data, "b.")
plt.show()
else:
indexed_pairs = indexed_pairs_provisional
correction_vectors = correction_vectors_provisional
########### finished with outlier rejection
self.inputpd["symmetry"].show_summary(prefix="SETTING ")
is_triclinic = (self.setting_id == 1)
if is_triclinic:
self.triclinic_pairs = [
dict(pred=self.hkllist[a["pred"]], spot=a["spot"])
for a in indexed_pairs
]
if self.horizons_phil.integration.model == "user_supplied":
if kwargs.get("user-reentrant", None) == None:
from cxi_user import post_outlier_rejection
self.indexed_pairs = indexed_pairs
self.spots = spots
post_outlier_rejection(self, image_number, cb_op_to_primitive,
self.horizons_phil, kwargs)
return
########### finished with user-supplied code
if self.horizons_phil.integration.spot_shape_verbose:
from rstbx.new_horizons.spot_shape import spot_shape_verbose
spot_shape_verbose(rawdata=self.imagefiles.images[
self.image_number].linearintdata,
beam_center_pix=matrix.col(
(self.inputai.xbeam() / pxlsz,
self.inputai.ybeam() / pxlsz)),
indexed_pairs=indexed_pairs,
spotfinder_observations=spots,
distance_mm=self.inputai.distance(),
mm_per_pixel=pxlsz,
hkllist=self.hkllist,
unit_cell=self.cell,
wavelength_ang=self.inputai.wavelength)
#Other checks to be implemented (future):
# spot is within active area of detector on a circular detector such as the Mar IP
# integration masks do not overlap; or deconvolute
correction_lengths = flex.double(
[v.length() for v in correction_vectors])
if verbose:
print "average correction %5.2f over %d vectors" % (
flex.mean(correction_lengths), len(correction_lengths)),
print "or %5.2f mm." % (pxlsz * flex.mean(correction_lengths))
self.r_residual = pxlsz * flex.mean(correction_lengths)
#assert len(indexed_pairs)>NEAR # must have enough indexed spots
if (len(indexed_pairs) <= NEAR):
raise Sorry("Not enough indexed spots, only found %d, need %d" %
(len(indexed_pairs), NEAR))
reference = flex.double()
for item in indexed_pairs:
reference.append(spots[item["spot"]].ctr_mass_x())
reference.append(spots[item["spot"]].ctr_mass_y())
PS_adapt = AnnAdaptor(data=reference, dim=2, k=NEAR)
PS_adapt.query(query)
self.BSmasks = []
#self.null_correction_mapping( predicted=self.predicted,
# correction_vectors = correction_vectors,
# IS_adapt = IS_adapt,
# spots = spots)
self.positional_correction_mapping(
predicted=self.predicted,
correction_vectors=correction_vectors,
PS_adapt=PS_adapt,
IS_adapt=IS_adapt,
spots=spots)
# which spots are close enough to interfere with background?
MAXOVER = 6
OS_adapt = AnnAdaptor(data=query, dim=2, k=MAXOVER) #six near nbrs
OS_adapt.query(query)
if self.mask_focus[image_number] is None:
raise Sorry(
"No observed/predicted spot agreement; no Spotfinder masks; skip integration"
)
nbr_cutoff = 2.0 * max(self.mask_focus[image_number])
FRAME = int(nbr_cutoff / 2)
#print "The overlap cutoff is %d pixels"%nbr_cutoff
nbr_cutoff_sq = nbr_cutoff * nbr_cutoff
#print "Optimized C++ section...",
self.set_frame(FRAME)
self.set_background_factor(kwargs["background_factor"])
self.set_nbr_cutoff_sq(nbr_cutoff_sq)
self.set_guard_width_sq(self.horizons_phil.integration.guard_width_sq)
self.set_detector_gain(self.horizons_phil.integration.detector_gain)
flex_sorted = flex.int()
for item in self.sorted:
flex_sorted.append(item[0])
flex_sorted.append(item[1])
if self.horizons_phil.integration.mask_pixel_value is not None:
self.set_mask_pixel_val(
self.horizons_phil.integration.mask_pixel_value)
image_obj = self.imagefiles.imageindex(
self.frame_numbers[self.image_number])
image_obj.read()
rawdata = image_obj.linearintdata # assume image #1
if self.inputai.active_areas != None:
self.detector_xy_draft = self.safe_background(
rawdata=rawdata,
predicted=self.predicted,
OS_adapt=OS_adapt,
sorted=flex_sorted,
tiles=self.inputai.active_areas.IT,
tile_id=self.inputai.active_areas.tile_id)
else:
self.detector_xy_draft = self.safe_background(
rawdata=rawdata,
predicted=self.predicted,
OS_adapt=OS_adapt,
sorted=flex_sorted)
for i in range(len(self.predicted)): # loop over predicteds
B_S_mask = {}
keys = self.get_bsmask(i)
for k in range(0, len(keys), 2):
B_S_mask[(keys[k], keys[k + 1])] = True
self.BSmasks.append(B_S_mask)
#print "Done"
return
def oneImage(self, framenumber, pd, image):
self.reporters[framenumber] = []
# The only way to get pixel size & pixel dimensions (currently) is from header
pimage = image
pimage.read()
#print "Detector type",type(pimage)
if 'endstation' not in pd:
from iotbx.detectors.context import endstation
pd['endstation'] = endstation.EndStation_from_ImageObject(
pimage, self.phil_params)
for key in pd['endstation'].mosflm():
pd[key] = pd['endstation'].mosflm()[key]
pd['vendortype'] = pimage.vendortype
pd['binning'] = "%d" % pimage.bin
pd['pixel_size'] = "%f" % pimage.pixel_size
self.pixel_size = float(pd['pixel_size'])
pd['size1'] = "%d" % pimage.size1
self.size1 = float(pd['size1'])
pd['size2'] = "%d" % pimage.size2
self.size2 = float(pd['size2'])
if 'osc_start' not in pd: pd['osc_start'] = {}
pd['osc_start'][framenumber] = "%f" % pimage.osc_start
if 'file' not in pd: pd['file'] = {}
pd['file'][framenumber] = pimage.filename
self.two_theta_degrees = float(pd['twotheta'])
if 'xbeam' not in pd or 'ybeam' not in pd:
raise SpotfinderError(
"Deprecation warning: inputs had no beam position", pd)
self.complex_nominal_center = complex(float(pd["xbeam"]),
float(pd["ybeam"]))
arguments = "" # distl_aggressive no longer supported, eg "-s2 5 -s3 8"
#allow for self.phil_params.distl_highres_limit
if self.phil_params.distl_highres_limit != None:
arguments = arguments + " -ro %.3f" % self.phil_params.distl_highres_limit
# test implementation of parallel processing for spotfinder
#from labelit.webice_support import parallel_distl
#pimage,pd = parallel_distl.split_image(pimage,pd)
try:
sf = Distl(
arguments,
pimage,
pd,
report_overloads=self.phil_params.distl_report_overloads,
params=self.phil_params)
except Sorry as e:
raise e
except Exception as e:
raise SpotfinderError("Spotfinder cannot analyze image %s :" %
pimage.filename + e.message)
#To support sublattice detection, make pixel-wise Z-scores persistent
if self.phil_params.distl_keep_Zdata:
pimage.linear_Z_data = sf.Z_data(
) #potentially uses a lot of memory
#************************************************************
#
# Very important. For the mar image plate (and allother circular detectors,
# must specify that these are embedded circular detectors and so adjust the
# percentage underloads. Or else libdistl must be changed so as not to
# search in these regions: yes, this would be better because I propose to
# change the search loop anyway. However, the getUnderload function must
# also be modified!!!
#
# ice ring search must also be modified to take this into account, but this
# part must be more precise than the above.
#
#************************************************************
if self.two_theta_degrees == 0.0:
mm_minimum_radius = resol_to_radius(
self.phil_params.distl_lowres_limit, pd)
sfa = SpotFilterAgent(
pixel_size=pimage.pixel_size,
xbeam=float(pd["xbeam"]),
ybeam=float(pd["ybeam"]),
distance=float(pd['distance']),
wavelength=float(pd['wavelength']),
icerings=sf.icerings,
)
#from libtbx import easy_pickle
#easy_pickle.dump("file.dmp",sfa)
#easy_pickle.load("file.dmp")
fstats = ListManager(masterlist=sf.spots,
masterkey='spots_total',
spotfilter=sfa)
fstats['distl_resolution'] = sf.imgresol()
# 1. Get all spots
fstats.alias(oldkey='spots_total', newkey='goodspots')
if VERBOSE_COUNT: print "total DISTL spots", fstats['N_spots_total']
fstats.c_spot_filter('goodspots', 'spots_non-ice', 'ice_ring_test')
fstats['ice-ring_impact'] = sf.nicerings()
fstats['ice-ring_bounds'] = [
(sf.icerings[i].lowerresol, sf.icerings[i].upperresol)
for i in xrange(fstats['ice-ring_impact'])
]
#**********************************************************************
# Known parts of code that are inefficient: use 35000-spot HK97 example
# 1. 3.8 seconds: sorting the resolution spots (item #4 in tnear2) (Corrected)
# 2. 9.7 seconds: spreadsheet bookkeeping; method2_resolution.py, xrow loop (Partly corrected 5/09)
# 3. 4.4 seconds: ice2::RingFinder::filtered() (Corrected)
# 3. omit spots too close to the beamstop
if self.two_theta_degrees == 0.0:
fstats.c_spot_filter('spots_non-ice',
'hi_pass_resolution_spots',
'resolution_test',
arguments=[
mm_minimum_radius,
])
else:
fstats.precompute_resolution(
self.two_theta_degrees * math.pi / 180., #two theta radians
pd['endstation'].rotation_axis(),
pd['endstation'].camera_convention())
fstats.c_spot_filter(
'spots_non-ice',
'hi_pass_resolution_spots',
'resolution_test_nztt',
arguments=[self.phil_params.distl_lowres_limit])
if VERBOSE_COUNT:
print "after lowres filter", fstats["N_hi_pass_resolution_spots"]
#start here.
#In the end, make sure these work:
#interface with mosflm: "TWOTHETA" keyword fails; "TILT" fix works with fudge factor
#James Holton's spots index (make unit test); anything with resolution_mm
#diffimage display!: ring_markup() method of webice_support/__init__
#report the two theta value in stats index
# 4. Calculate resolution cutoff
if self.two_theta_degrees == 0.0:
sorted_order, sorted_resolutions = fstats.resolution_sort(
"hi_pass_resolution_spots")
else:
sorted_order, sorted_resolutions = fstats.resolution_sort_nztt(
"hi_pass_resolution_spots")
# targetBinNumber: first try number of candidate Bragg spots per bin
targetBinNumber = max(self.BinMin, len(sorted_order) // 20)
# cutoff threshhold: expected number spots in highest resolution bin
cutoff_ratio = 100. / self.phil_params.distl.method2_cutoff_percentage
fstats['resolution_divisor'] = cutoff_ratio
lowerCutoffBinNumber = targetBinNumber / cutoff_ratio
if len(sorted_order) < targetBinNumber:
# So few spots that there is only one bin
# no resolution determination possible
fstats['resolution'] = None
fstats['resolution_detail'] = len(sorted_order)
elif False and sorted_resolutions[self.BinMin - 1] < 4.0:
'''This filter (that essentially says you must have low
resolution spots) hindered the ability to index the case
ana/procrun0000084148/sphN1_*.mar2300. Further investigation
showed that not a single one of the 94 reference cases in the
regression database was affected by this test, so the test is
being provisionally removed. It is not known if the test has a
beneficial effect on cases with no protein diffraction, i.e., just
ice rings.'''
# This test indicates that there are so few spots at low
# resolution that the first bin extends past 4.0 Angstroms,
# into the region where ice rings might be found
# since there are too few low resolution data
# conclude that these are false Bragg spots
fstats['resolution'] = None
sorted_resolutions = []
# deprecated; would need to be recoded:
fstats['N_spots_resolution'] = 0
else:
from spotfinder.diffraction.geometry import Geom2d
frac_calc = Geom2d(pd)
#spot-based ice-ring filtering, added Aug 2004
'''The case ana/procrun0000084148/sphN1_*.mar2300 (image 090) shows the
limits of this filter as presently implemented. In that particular
case, this ice-ring filter eliminates spots from a large area in the
2-3 Angstrom resolution range. However, to be believed, the purported
ice-spots should define a circle or ellipse centered at the beam position
with a resolution spread that is very narrow. Code could be
written much more effectively to search and elimate these rings'''
from spotfinder.applications.heuristic_tbx.ice2 import RingFinder
from spotfinder.applications.heuristic_tbx.ice_nztt import RingFinder_nztt
if (abs(float(pd['twotheta'])) > 0.0):
# Very inefficient--8 seconds per call; will need to be optimized
#PP = Profiler("nztt")
Ring = RingFinder_nztt(sorted_resolutions, targetBinNumber,
frac_calc, image)
#del PP
else:
Ring = RingFinder(sorted_resolutions, targetBinNumber,
frac_calc)
fstats.add_child("hi_pass_resolution_spots",
"ice_free_resolution_spots",
Ring.filtered(sorted_order))
fstats['ice-ring_impact'] += Ring.ice_ring_impact()
fstats['ice-ring_bounds'] += Ring.ice_ring_bounds()
#*************************the filtering of existing spots
if VERBOSE_COUNT:
print "after spot_based ice-ring filter", fstats[
'N_ice_free_resolution_spots']
if fstats['N_ice_free_resolution_spots'] < targetBinNumber:
# So few spots that there is only one bin
# no resolution determination possible
fstats['resolution'] = None
fstats['resolution_detail'] = fstats[
'N_ice_free_resolution_spots']
else:
from spotfinder.applications.heuristic_tbx.method2_resolution\
import ResolutionShells
sorted_resolutions = fstats.spotfilter.get_resolution(
fstats.master,
fstats.get_indices('ice_free_resolution_spots'))
Shell = ResolutionShells(
sorted_resolutions,
sorted_resolutions[targetBinNumber - 1], frac_calc)
#Shell.show()
if self.phil_params.distl.bins.verbose:
ShellR = spotreporter(
sorted_resolutions,
self.phil_params,
sorted_resolutions[targetBinNumber - 1],
wavelength=float(pd['wavelength']),
max_total_rows=self.phil_params.distl.bins.N,
fractionCalculator=frac_calc,
use_binning_of=self)
ShellR.total_signal(
fstats,
fstats.get_indices('ice_free_resolution_spots'))
ShellR.background(sf)
ShellR.sigma_analysis()
print
ShellR.show(
message=
"Analysis of spots after ice removal, but prior to resolution cutoff, for image \n%s"
% pimage.filename)
self.reporters[framenumber].append(ShellR)
# slight adjustment so that we don't focus on the lowest
# resolution data shell (spends too much time in Ewald sphere)
# But don't make this adjustment if we are limited by low spot count
if Shell.rows() > 2 and \
Shell.Population[1] > 2*lowerCutoffBinNumber and \
Shell.Population[1] > self.BinMin:
lowerCutoffBinNumber = Shell.Population[1] / cutoff_ratio
# first determination of cutoff ignoring corner effect
for x in xrange(Shell.rows()):
idx = Shell.rows() - x - 1
if Shell.Population[idx] > lowerCutoffBinNumber:
lastshell = idx
break
# eliminate pathological case where there are a lot of low resolution
# spots and then some ice rings at high resolution. If there are ten
# empty shells in a row (empty defined as having fewer than
# VetterCut spots), redetermine lastshell.
#
# Originally, VetterCut := lowerCutoffBinNumber
# Modify the heuristic to cover two extremes:
# Case 1. Based on the HK97 virus work; there is a class of diffraction
# cases showing a distinct dip in diffraction intensity in the
# 5-Angstrom regime. If lowerCutoffBinNumber is used (usually
# 20% of the spot count in the 2nd bin), the algorithm can
# misbehave and reject all the high-resolution Bragg spots.
# For one case in particular a tiny change in beam position
# flips the resolution cutoff from a 6.5- to 4.4-Angstrom.
# Case 2. There are many cases where the diffraction clearly
# drops off, and at higher resolutions there are random-signal
# spots plus ice rings. The spot count in these high-res bins
# is typically small, <20. Use "20" as a heuristic cutoff
# between Case 1 & Case 2.
# In future, it may be productive to focus on smarter algorithms to
# execute ellipse recognition in the "Ringfinder" section above
if lowerCutoffBinNumber <= 20:
VetterCut = lowerCutoffBinNumber
else:
VetterCut = 20 + lowerCutoffBinNumber // 5
lastshell = Shell.vetter(VetterCut, lastshell)
#option for overriding the resolution analysis based on falloff
# of spot count. Force spots at least this far out
if self.phil_params.force_method2_resolution_limit is not None:
for x in xrange(Shell.rows()):
if Shell.Limit[
x] < self.phil_params.force_method2_resolution_limit and lastshell < x:
lastshell = x
break
# extend resolution cutoff taking into account corner cutoff
while Shell.Fract[
lastshell] < 1.0 and lastshell + 1 < Shell.rows():
nextshell = lastshell + 1
if Shell.adjustPop[nextshell] > lowerCutoffBinNumber:
lastshell += 1
else:
break
fstats['resolution'] = Shell.Limit[lastshell]
if self.phil_params.distl_highres_limit != None:
fstats['resolution'] = max(
fstats['resolution'],
self.phil_params.distl_highres_limit)
for x in xrange(lastshell + 1):
if self.phil_params.spotfinder_verbose:
print "(%.2f,%d)" % (Shell.Limit[x],
Shell.Population[x])
if self.two_theta_degrees == 0.0:
fstats.c_spot_filter( #hi-resolution radius
'ice_free_resolution_spots',
'lo_pass_resolution_spots',
'lo_pass_resolution_test',
arguments=[
resol_to_radius(fstats['resolution'], self.pd),
])
else:
fstats.c_spot_filter( #hi-resolution radius
'ice_free_resolution_spots',
'lo_pass_resolution_spots',
'lo_pass_resolution_test_nztt',
arguments=[
fstats['resolution'],
])
if VERBOSE_COUNT:
print "ice_free_resolution_spots ", fstats[
'N_ice_free_resolution_spots']
fstats['shells'] = Shell
if VERBOSE_COUNT:
print "after resolution-shell cutoff", fstats[
'N_lo_pass_resolution_spots']
fstats['resolution_mm'] = resol_to_radius(
fstats['resolution'], pd)
fstats['saturation'] = self.calculate_saturation(fstats, image)
if self.phil_params.distl.bins.verbose:
try:
subset = fstats.spotfilter.get_resolution(
fstats.master,
fstats.get_indices('lo_pass_resolution_spots'))
ShellR = spotreporter(subset,
self.phil_params,
use_binning_of=self)
ShellR.total_signal(
fstats, fstats.get_indices('lo_pass_resolution_spots'))
ShellR.background(sf)
ShellR.sigma_analysis()
print
ShellR.show(
message=
"Analysis of spots after resolution filtering, but prior to spot quality heuristics, for image \n%s"
% pimage.filename)
self.reporters[framenumber].append(ShellR)
except Exception: # in case the low-pass filter step was skipped; e.g., blank image.
pass
# 5. eliminate multi-modal spots & punctate spots
fstats.c_spot_filter(fstats.most_recent_child(),
'spots_unimodal',
'modal_test',
arguments=[
self.phil_params.distl_profile_bumpiness,
self.phil_params.distl.minimum_spot_area
or sf.spotbasesize()
])
# parameters are bumpiness(max number of local maxima in peak),minimum pixels
if VERBOSE_COUNT:
print "not bumpy & not punctate", fstats['N_spots_unimodal']
# 6. Compute distributions for outlier rejection
#Y = Timer("Inliers")#try to get this down from 232 seconds to 8 seconds
#(For HK97 sz=4)
inlier_idx_raw = flex.int()
inlier_neigh = []
unimodal_spots = fstats['spots_unimodal']
if fstats[
'N_spots_unimodal'] > 2: # avoids divide-by-zero error in stats calls, below
intensities = flex.double([s.intensity() for s in unimodal_spots])
areas = flex.double([s.bodypixels.size() for s in unimodal_spots])
# Deprecate shapes because code is unstable; in rare cases spots can have
# body pixels but no border pixels ( RAW/APS_07_2005/pEI4/pEI4_8_1.0001 )
#shapes = flex.double([s.shape() for s in unimodal_spots])
eccentricity = flex.double(
[s.model_eccentricity() for s in unimodal_spots])
skewness = fstats.get_property(fstats.most_recent_child(),
'skewness')
fstats['intensity'] = (i_ave, i_std) = scitbx_stats(intensities)
fstats['area'] = (a_ave, a_std) = scitbx_stats(areas)
#fstats['shape'] = (s_ave,s_std) = scitbx_stats(shapes)
fstats['eccen'] = (e_ave, e_std) = scitbx_stats(eccentricity)
fstats['skewness'] = (k_ave, k_std) = scitbx_stats(skewness)
#Filtering on skewness is important when using center-of-mass positions
# for autoindexing. The goal is to eliminate pairs of unresolved Bragg
# spots. Condition is flagged when the max_pixel to center-of-mass
# vector is more than 45% of the semi-major axis
fstats.c_spot_filter(fstats.most_recent_child(),
'spots_low_skew',
'low_skew',
arguments=[
min(0.45, 2.0 * k_std + k_ave),
])
#Filter on the intensity distribution.
fstats.c_spot_filter(fstats.most_recent_child(),
'spots_good_intensity',
'intensity_inlier',
arguments=[
i_ave - 5.0 * i_std,
i_ave + 5.0 * i_std,
image.saturation,
])
#special code for expecting a high MOSFLM RESID based on the
# presence of very high signal/noise ratio (essentially lysozyme strength).
# This is all just a supposition based on one dataset, offset_1 from Ana.
# The assumption may be wrong; e.g. the high resid may be due to very low
# background instead of high s/n; or due to some geometrical distortion.
#This breaks encapsulation and will have to be re-organized in the future.
if 'special_resid' not in pd: pd['special_resid'] = ''
if fstats['intensity'][0] > 160.:
pd['special_resid'] = 'RESID 10.0 #High s/n'
#The most unusual thing about the procrun0000077831/TMC114_WT2_run77831_1_001
#dataset is its large differential between spot areas of the largest spots
# and the smallest spots. Hypothesize that this also leads to high
# weighted residuals.
if stats_profile(areas) > 10.:
pd['special_resid'] = 'RESID 10.0 #High s/n'
from annlib_ext import AnnAdaptor
data = fstats.get_property(
'goodspots', self.phil_params.distl_spotcenter_algorithm)
query = fstats.get_property(
fstats.most_recent_child(),
self.phil_params.distl_spotcenter_algorithm)
A = AnnAdaptor(data, 2) # construct k-d tree for reference set
A.query(query) # find nearest neighbors of query points
neighbors = (self.pixel_size) * flex.sqrt(A.distances)
"""Explanation: Distance to the nearest neighbor is math.sqrt(A.distances[i]), in
units of pixels. The vector to the nearest neighbor, in units of pixels, is
( fstats.master[A.nn[i]].x()-query[2*i], fstats.master[A.nn[i]].y()-query[2*i+1])
"""
fstats['neighbor'] = (n_ave, n_std) = scitbx_stats(neighbors)
sep_input_spots = fstats[fstats.most_recent_child()]
sep_input_indices = fstats.get_indices(fstats.most_recent_child())
overlapping_count = 0
for idx in xrange(len(sep_input_spots)):
try:
if float(pd['pixel_size'])*sep_input_spots[idx].majoraxis() * \
self.phil_params.overlapping_spot_criterion > neighbors[idx]:
overlapping_count += 1
except Exception:
pass
if len(sep_input_spots) == 0: percent_overlap = 0
else:
percent_overlap = 100 * overlapping_count / len(
sep_input_spots)
#print "overlap %2.0f%% vs. cutoff %2.0f%%"%(percent_overlap,self.phil_params.percent_overlap_forcing_detail)
from spotfinder.core_toolbox.close_spots_detail import NearNeighborVectors
Afull = AnnAdaptor(data, 2)
Afull.query(data)
NV = NearNeighborVectors(ave_area=a_ave,
query_centers=data,
fstats=fstats,
ann_adaptor=Afull)
#NV.show_vector_map()
#NV.show_maxima()
DEVELOP_ASSERT = True
self.force_detail = percent_overlap > self.phil_params.percent_overlap_forcing_detail
if DEVELOP_ASSERT or self.force_detail:
self.overlapping = True
#extraordinary procedure, when many spots are close. Include closely
# spaced spots in autoindexing, if it appears that they truly
# reflect lattice spacing.
if self.phil_params.spotfinder_verbose:
print len(sep_input_spots
), "spot count before close neighbor analysis;",
print overlapping_count, "(%2.0f%%) rejected on neighbors;" % (
percent_overlap)
pmax = NV.vectors()
#filter out spots that are potentially large enough to contain
#two spots, now that we know a candidate projected unit-cell vector
# expect big time savings if this section is pushed down to C++
compact_idx = []
if 0 < len(pmax) <= 3:
sq_vectors = [v[0] * v[0] + v[1] * v[1] for v in pmax]
for idx in xrange(len(sep_input_spots)):
spot_compact = True
for iv, vector in enumerate(pmax):
thisspot = sep_input_spots[idx]
#more efficient code--disallowed spots (based on nearest neighbor
# vector) are now flagged based on the width of the ellipse model
# rather than a more time-consuming body-pixel match:
#calculate angle theta between major axis and neighbor vector
dot = thisspot.eigenvector(1)[0] * vector[
0] + thisspot.eigenvector(1)[1] * vector[1]
costheta = dot / math.sqrt(sq_vectors[iv])
costhetasq = min(costheta * costheta, 1.0)
sintheta = math.sqrt(1 - costhetasq)
spota = thisspot.a()
spotb = thisspot.b()
a_sin_theta = spota * sintheta
b_cos_theta = spotb * costheta
#evaluate the ellipse full width along neighbor vector direction
width_sq = 4. * (spota * spota * spotb * spotb /
(a_sin_theta * a_sin_theta +
b_cos_theta * b_cos_theta))
if width_sq >= sq_vectors[iv]:
spot_compact = False
break
if spot_compact: compact_idx.append(idx)
else:
pass #print "eliminate the spot at",thisspot.x(),thisspot.y()
else:
compact_idx = xrange(len(sep_input_spots))
if self.phil_params.spotfinder_verbose:
print len(sep_input_spots) - len(
compact_idx), "large spots rejected"
# finally, allow certain close spots (but not all of them) to be included
for idx in compact_idx:
try:
S = (int(fstats.master[A.nn[idx]].max_pxl_x() -
query[2 * idx]),
int(fstats.master[A.nn[idx]].max_pxl_y() -
query[2 * idx + 1]))
#accept spot by default
reject_spot = False
#if close to nearest neighbor reject
if sep_input_spots[idx].majoraxis() * \
self.phil_params.overlapping_spot_criterion > math.sqrt(S[0]*S[0]+S[1]*S[1]):
# with normal procedure, spot would be rejected at this point
reject_spot = True
if len(pmax) <= 3:
for vector in pmax:
if abs(vector[0] - S[0]) <= 1 and abs(
vector[1] - S[1]) <= 1:
#but allow if the proximity is to a candidate cell near neighbor
reject_spot = False
break
if reject_spot: continue
inlier_idx_raw.append(sep_input_indices[idx])
inlier_neigh.append(neighbors[idx])
except Exception:
pass
if self.phil_params.spotfinder_verbose:
print len(compact_idx) - len(
inlier_idx_raw), "close spots rejected"
print len(sep_input_spots
), "input for spot separation analysis;",
jj = len(sep_input_spots) - len(inlier_idx_raw)
print jj, "(%2.0f%%) rejected on special criteria;" % (
100. * jj / len(sep_input_spots))
else: #normal procedure, assuming not too many close spots
for idx in xrange(len(sep_input_spots)):
try:
if math.fabs(intensities[idx]-i_ave) <= 5.0*i_std and \
intensities[idx]<image.saturation and \
float(pd['pixel_size'])*sep_input_spots[idx].majoraxis() * \
self.phil_params.overlapping_spot_criterion<=neighbors[idx]:
inlier_idx_raw.append(sep_input_indices[idx])
inlier_neigh.append(neighbors[idx])
except Exception:
#print "REJECT spot on exception"
pass #sometimes throw an error when majoraxis is requested (edge spots)
if len(inlier_idx_raw) < self.NspotMin:
for idx in xrange(len(sep_input_spots)):
try:
proximal_radius = 2.0 * self.phil_params.overlapping_spot_criterion * float(
pd['pixel_size']
) * sep_input_spots[idx].majoraxis()
if math.fabs(intensities[idx]-i_ave) <= 5.0*i_std and \
intensities[idx]<image.saturation and \
float(pd['pixel_size'])*sep_input_spots[idx].majoraxis() *self.phil_params.overlapping_spot_criterion > neighbors[idx] and \
sf.isIsolated(sep_input_spots[idx],proximal_radius):
#print "Very few Bragg spots; forced to accept spot with neighbor distance",neighbors[idx]/(float(pd['pixel_size'])*sep_input_spots[idx].majoraxis())
inlier_idx_raw.append(sep_input_indices[idx])
inlier_neigh.append(neighbors[idx])
except Exception:
print "REJECT spot on exception"
pass #sometimes throw an error when majoraxis is requested (edge spots)
if fstats.has_extended_key('lo_pass_resolution_spots'):
fstats.alias('lo_pass_resolution_spots', 'spots_resolution')
elif fstats.has_extended_key('ice_free_resolution_spots'):
fstats.alias('ice_free_resolution_spots', 'spots_resolution')
else:
fstats.alias('hi_pass_resolution_spots', 'spots_resolution')
fstats.add_child('spots_unimodal', 'spots_separated', inlier_idx_raw)
fstats.alias(fstats.most_recent_child(), 'spots_inlier')
fstats.alias('spots_inlier',
'inlier_spots') #for backward compatibility
if self.phil_params.distl.bins.verbose:
try:
subset = fstats.spotfilter.get_resolution(
fstats.master, fstats.get_indices('inlier_spots'))
ShellR = spotreporter(subset,
self.phil_params,
use_binning_of=self)
ShellR.total_signal(fstats, fstats.get_indices('inlier_spots'))
ShellR.background(sf)
ShellR.sigma_analysis()
print
ShellR.show(
message=
"Analysis of good Bragg spots after quality heuristics, for image \n%s"
% pimage.filename)
self.reporters[framenumber].append(ShellR)
except Exception:
pass #if there aren't enough spots, just skip the tabular printout
if 'masks' not in pd: pd['masks'] = {}
pd['masks'][framenumber] = None
if fstats['N_spots_inlier'] > ( #guard against C++ hard-coded minimum
self.phil_params.codecamp.minimum_spot_count or 25):
Msk = fstats.single_mask('inlier_spots', self.phil_params)
pd['masks'][framenumber] = [Msk.x, Msk.y]
if VERBOSE_COUNT: print "inlier spots", fstats["N_inlier_spots"]
#need this for later calculation of maxcell
fstats['neighbors'] = flex.double(inlier_neigh)
return fstats
def integration_concept_detail(self, experiments, reflections, spots,
image_number, cb_op_to_primitive, **kwargs):
detector = experiments[0].detector
crystal = experiments[0].crystal
from cctbx.crystal import symmetry
c_symmetry = symmetry(space_group=crystal.get_space_group(),
unit_cell=crystal.get_unit_cell())
self.image_number = image_number
NEAR = 10
pxlsz = detector[0].get_pixel_size()
Predicted = self.get_predictions_accounting_for_centering(
experiments, reflections, cb_op_to_primitive, **kwargs)
FWMOSAICITY = self.inputai.getMosaicity()
self.DOMAIN_SZ_ANG = kwargs.get("domain_size_ang",
self.__dict__.get("actual", 0))
refineflag = {True: 0, False: 1}[kwargs.get("domain_size_ang", 0) == 0]
c_symmetry.show_summary(
prefix="EXCURSION%1d REPORT FWMOS= %6.4f DOMAIN= %6.1f " %
(refineflag, FWMOSAICITY, self.DOMAIN_SZ_ANG))
from annlib_ext import AnnAdaptor
self.cell = c_symmetry.unit_cell()
query = flex.double()
print len(self.predicted)
for pred in self.predicted: # predicted spot coord in pixels
query.append(pred[0] / pxlsz[0])
query.append(pred[1] / pxlsz[1])
self.reserve_hkllist_for_signal_search = self.hkllist
reference = flex.double()
assert self.length > NEAR # Can't do spot/pred matching with too few spots
for spot in spots:
reference.append(spot.ctr_mass_x())
reference.append(spot.ctr_mass_y())
IS_adapt = AnnAdaptor(data=reference, dim=2, k=NEAR)
IS_adapt.query(query)
idx_cutoff = float(min(self.mask_focus[image_number]))
from rstbx.apps.slip_helpers import slip_callbacks
cache_refinement_spots = getattr(slip_callbacks.slip_callback,
"requires_refinement_spots", False)
indexed_pairs_provisional = []
correction_vectors_provisional = []
c_v_p_flex = flex.vec3_double()
this_setting_matched_indices = reflections["miller_index"]
for j, item in enumerate(this_setting_matched_indices):
this_setting_index = self.hkllist.first_index(item)
if this_setting_index:
Match = dict(spot=j, pred=this_setting_index)
indexed_pairs_provisional.append(Match)
vector = matrix.col([
reflections["xyzobs.px.value"][j][0] -
self.predicted[Match["pred"]][0] / pxlsz[0],
reflections["xyzobs.px.value"][j][1] -
self.predicted[Match["pred"]][1] / pxlsz[1]
])
correction_vectors_provisional.append(vector)
c_v_p_flex.append((vector[0], vector[1], 0.))
self.N_correction_vectors = len(correction_vectors_provisional)
self.rmsd_px = math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))
print "... %d provisional matches" % self.N_correction_vectors,
print "r.m.s.d. in pixels: %6.3f" % (self.rmsd_px)
if self.horizons_phil.integration.enable_residual_scatter:
from matplotlib import pyplot as plt
fig = plt.figure()
for cv in correction_vectors_provisional:
plt.plot([cv[1]], [-cv[0]], "r.")
plt.title(" %d matches, r.m.s.d. %5.2f pixels" %
(len(correction_vectors_provisional),
math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))))
plt.axes().set_aspect("equal")
self.show_figure(plt, fig, "res")
plt.close()
if self.horizons_phil.integration.enable_residual_map:
from matplotlib import pyplot as plt
PX = reflections["xyzobs.px.value"]
fig = plt.figure()
for match, cv in zip(indexed_pairs_provisional,
correction_vectors_provisional):
plt.plot([PX[match["spot"]][1]], [-PX[match["spot"]][0]], "r.")
plt.plot([self.predicted[match["pred"]][1] / pxlsz[1]],
[-self.predicted[match["pred"]][0] / pxlsz[0]], "g.")
plt.plot(
[PX[match["spot"]][1], PX[match["spot"]][1] + 10. * cv[1]],
[
-PX[match["spot"]][0],
-PX[match["spot"]][0] - 10. * cv[0]
], 'r-')
if kwargs.get("user-reentrant") != None and self.horizons_phil.integration.spot_prediction == "dials" \
and self.horizons_phil.integration.enable_residual_map_deltapsi:
from rstbx.apps.stills.util import residual_map_special_deltapsi_add_on
residual_map_special_deltapsi_add_on(
reflections=self.dials_spot_prediction,
matches=indexed_pairs_provisional,
experiments=experiments,
hkllist=self.hkllist,
predicted=self.predicted,
plot=plt,
eta_deg=FWMOSAICITY,
deff=self.DOMAIN_SZ_ANG)
plt.xlim([0, detector[0].get_image_size()[1]])
plt.ylim([-detector[0].get_image_size()[0], 0])
plt.title(" %d matches, r.m.s.d. %5.2f pixels" %
(len(correction_vectors_provisional),
math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))))
plt.axes().set_aspect("equal")
self.show_figure(plt, fig, "map")
plt.close()
indexed_pairs = indexed_pairs_provisional
correction_vectors = correction_vectors_provisional
########### skip outlier rejection for this derived class
### However must retain the ability to write out correction vectiors.
if True: # at Aaron's request; test later
correction_lengths = flex.double(
[v.length() for v in correction_vectors_provisional])
clorder = flex.sort_permutation(correction_lengths)
sorted_cl = correction_lengths.select(clorder)
indexed_pairs = []
correction_vectors = []
self.correction_vectors = []
for icand in xrange(len(sorted_cl)):
# somewhat arbitrary sigma = 1.0 cutoff for outliers
indexed_pairs.append(indexed_pairs_provisional[clorder[icand]])
correction_vectors.append(
correction_vectors_provisional[clorder[icand]])
if cache_refinement_spots:
self.spotfinder.images[self.frame_numbers[
self.image_number]]["refinement_spots"].append(
spots[reflections[indexed_pairs[-1]["spot"]]
['spotfinder_lookup']])
if kwargs.get("verbose_cv") == True:
print "CV OBSCENTER %7.2f %7.2f REFINEDCENTER %7.2f %7.2f" % (
float(self.inputpd["size1"]) / 2.,
float(self.inputpd["size2"]) / 2.,
self.inputai.xbeam() / pxlsz[0],
self.inputai.ybeam() / pxlsz[1]),
print "OBSSPOT %7.2f %7.2f PREDSPOT %7.2f %7.2f" % (
reflections[indexed_pairs[-1]["spot"]]
['xyzobs.px.value'][0], reflections[
indexed_pairs[-1]["spot"]]['xyzobs.px.value'][1],
self.predicted[indexed_pairs[-1]["pred"]][0] /
pxlsz[0], self.predicted[indexed_pairs[-1]["pred"]][1]
/ pxlsz[1]),
the_hkl = self.hkllist[indexed_pairs[-1]["pred"]]
print "HKL %4d %4d %4d" % the_hkl, "%2d" % self.setting_id,
radial, azimuthal = spots[indexed_pairs[-1][
"spot"]].get_radial_and_azimuthal_size(
self.inputai.xbeam() / pxlsz[0],
self.inputai.ybeam() / pxlsz[1])
print "RADIALpx %5.3f AZIMUTpx %5.3f" % (radial, azimuthal)
# Store a list of correction vectors in self.
radial, azimuthal = spots[
indexed_pairs[-1]['spot']].get_radial_and_azimuthal_size(
self.inputai.xbeam() / pxlsz[0],
self.inputai.ybeam() / pxlsz[1])
self.correction_vectors.append(
dict(obscenter=(float(self.inputpd['size1']) / 2,
float(self.inputpd['size2']) / 2),
refinedcenter=(self.inputai.xbeam() / pxlsz[0],
self.inputai.ybeam() / pxlsz[1]),
obsspot=(reflections[indexed_pairs[-1]
['spot']]['xyzobs.px.value'][0],
reflections[indexed_pairs[-1]
['spot']]['xyzobs.px.value'][1]),
predspot=(
self.predicted[indexed_pairs[-1]['pred']][0] /
pxlsz[0],
self.predicted[indexed_pairs[-1]['pred']][1] /
pxlsz[1]),
hkl=(self.hkllist[indexed_pairs[-1]['pred']][0],
self.hkllist[indexed_pairs[-1]['pred']][1],
self.hkllist[indexed_pairs[-1]['pred']][2]),
setting_id=self.setting_id,
radial=radial,
azimuthal=azimuthal))
self.inputpd["symmetry"] = c_symmetry
self.inputpd["symmetry"].show_summary(prefix="SETTING ")
if self.horizons_phil.integration.model == "user_supplied":
# Not certain of whether the reentrant_* dictionary keys create a memory leak
if kwargs.get("user-reentrant", None) == None:
kwargs["reentrant_experiments"] = experiments
kwargs["reentrant_reflections"] = reflections
from cxi_user import post_outlier_rejection
self.indexed_pairs = indexed_pairs
self.spots = spots
post_outlier_rejection(self, image_number, cb_op_to_primitive,
self.horizons_phil, kwargs)
return
########### finished with user-supplied code
correction_lengths = flex.double(
[v.length() for v in correction_vectors])
self.r_residual = pxlsz[0] * flex.mean(correction_lengths)
#assert len(indexed_pairs)>NEAR # must have enough indexed spots
if (len(indexed_pairs) <= NEAR):
raise Sorry("Not enough indexed spots, only found %d, need %d" %
(len(indexed_pairs), NEAR))
reference = flex.double()
for item in indexed_pairs:
reference.append(spots[item["spot"]].ctr_mass_x())
reference.append(spots[item["spot"]].ctr_mass_y())
PS_adapt = AnnAdaptor(data=reference, dim=2, k=NEAR)
PS_adapt.query(query)
self.BSmasks = []
# do not use null: self.null_correction_mapping( predicted=self.predicted,
self.positional_correction_mapping(
predicted=self.predicted,
correction_vectors=correction_vectors,
PS_adapt=PS_adapt,
IS_adapt=IS_adapt,
spots=spots)
# which spots are close enough to interfere with background?
MAXOVER = 6
OS_adapt = AnnAdaptor(data=query, dim=2, k=MAXOVER) #six near nbrs
OS_adapt.query(query)
if self.mask_focus[image_number] is None:
raise Sorry(
"No observed/predicted spot agreement; no Spotfinder masks; skip integration"
)
nbr_cutoff = 2.0 * max(self.mask_focus[image_number])
FRAME = int(nbr_cutoff / 2)
#print "The overlap cutoff is %d pixels"%nbr_cutoff
nbr_cutoff_sq = nbr_cutoff * nbr_cutoff
#print "Optimized C++ section...",
self.set_frame(FRAME)
self.set_background_factor(kwargs["background_factor"])
self.set_nbr_cutoff_sq(nbr_cutoff_sq)
self.set_guard_width_sq(self.horizons_phil.integration.guard_width_sq)
self.set_detector_gain(self.horizons_phil.integration.detector_gain)
flex_sorted = flex.int()
for item in self.sorted:
flex_sorted.append(item[0])
flex_sorted.append(item[1])
if self.horizons_phil.integration.mask_pixel_value is not None:
self.set_mask_pixel_val(
self.horizons_phil.integration.mask_pixel_value)
image_obj = self.imagefiles.imageindex(
self.frame_numbers[self.image_number])
image_obj.read()
rawdata = image_obj.linearintdata # assume image #1
if self.inputai.active_areas != None:
self.detector_xy_draft = self.safe_background(
rawdata=rawdata,
predicted=self.predicted,
OS_adapt=OS_adapt,
sorted=flex_sorted,
tiles=self.inputai.active_areas.IT,
tile_id=self.inputai.active_areas.tile_id)
else:
self.detector_xy_draft = self.safe_background(
rawdata=rawdata,
predicted=self.predicted,
OS_adapt=OS_adapt,
sorted=flex_sorted)
for i in xrange(len(self.predicted)): # loop over predicteds
B_S_mask = {}
keys = self.get_bsmask(i)
for k in xrange(0, len(keys), 2):
B_S_mask[(keys[k], keys[k + 1])] = True
self.BSmasks.append(B_S_mask)
#print "Done"
return
def __init__(self,
reflections,
step_size=45,
tolerance=1.5,
max_height_fraction=0.25,
percentile=0.05):
self.tolerance = tolerance # Margin of error for max unit cell estimate
from scitbx.array_family import flex
NEAR = 10
self.NNBIN = 5 # target number of neighbors per histogram bin
direct = flex.double()
if 'entering' in reflections:
entering_flags = reflections['entering']
else:
entering_flags = flex.bool(reflections.size(), True)
rs_vectors = reflections['rlp']
phi_deg = reflections['xyzobs.mm.value'].parts()[2] * (180 / math.pi)
# nearest neighbor analysis
from annlib_ext import AnnAdaptor
for imageset_id in range(flex.max(reflections['imageset_id']) + 1):
sel = reflections['imageset_id'] == imageset_id
phi_min = flex.min(phi_deg.select(sel))
phi_max = flex.max(phi_deg.select(sel))
d_phi = phi_max - phi_min
n_steps = max(int(math.ceil(d_phi / step_size)), 1)
for n in range(n_steps):
sel &= (phi_deg >=
(phi_min + n * step_size)) & (phi_deg <
(phi_min +
(n + 1) * step_size))
for entering in (True, False):
sel &= entering_flags == entering
if sel.count(True) == 0:
continue
query = flex.double()
query.extend(rs_vectors.select(sel).as_double())
if query.size() == 0:
continue
IS_adapt = AnnAdaptor(data=query, dim=3, k=1)
IS_adapt.query(query)
direct.extend(1 / flex.sqrt(IS_adapt.distances))
assert len(direct) > NEAR, (
"Too few spots (%d) for nearest neighbour analysis." % len(direct))
# reject top 1% of longest distances to hopefully get rid of any outliers
direct = direct.select(
flex.sort_permutation(direct)[:int(math.floor(0.99 *
len(direct)))])
# determine the most probable nearest neighbor distance (direct space)
hst = flex.histogram(direct, n_slots=int(len(direct) / self.NNBIN))
centers = hst.slot_centers()
islot = hst.slots()
highest_bin_height = flex.max(islot)
most_probable_neighbor = centers[list(islot).index(highest_bin_height)]
if False: # to print out the histogramming analysis
smin, smax = flex.min(direct), flex.max(direct)
stats = flex.mean_and_variance(direct)
import sys
out = sys.stdout
print >> out, " range: %6.2f - %.2f" % (smin, smax)
print >> out, " mean: %6.2f +/- %6.2f on N = %d" % (
stats.mean(), stats.unweighted_sample_standard_deviation(),
direct.size())
hst.show(f=out, prefix=" ", format_cutoffs="%6.2f")
print >> out, ""
# choose a max cell based on bins above a given fraction of the highest bin height
# given multiple
isel = (islot.as_double() >
(max_height_fraction * highest_bin_height)).iselection()
self.max_cell = (
self.tolerance * centers[int(flex.max(isel.as_double()))] +
0.5 * hst.slot_width())
# determine the 5th-percentile direct-space distance
perm = flex.sort_permutation(direct, reverse=True)
self.percentile = direct[perm[int(percentile * len(direct))]]
#MAXTOL = 1.5 # Margin of error for max unit cell estimate
#self.max_cell = max(MAXTOL * most_probable_neighbor,
#MAXTOL * self.percentile)
if False:
self.plot(direct)
def check_memory():
data,query = data_from_files()
for x in xrange(1000):
AnnAdaptor(data,2).query(query)
#This breaks encapsulation and will have to be re-organized in the future.
if not pd.has_key('special_resid'): pd['special_resid']=''
if fstats['intensity'][0]>160.: pd['special_resid']='RESID 10.0 #High s/n'
#The most unusual thing about the procrun0000077831/TMC114_WT2_run77831_1_001
#dataset is its large differential between spot areas of the largest spots
# and the smallest spots. Hypothesize that this also leads to high
# weighted residuals.
if stats_profile(areas)>10.: pd['special_resid']='RESID 10.0 #High s/n'
from annlib_ext import AnnAdaptor
data = fstats.get_property('goodspots',
self.phil_params.distl_spotcenter_algorithm)
query = fstats.get_property(fstats.most_recent_child(),
self.phil_params.distl_spotcenter_algorithm)
A = AnnAdaptor(data,2) # construct k-d tree for reference set
A.query(query) # find nearest neighbors of query points
neighbors = (self.pixel_size) * flex.sqrt(A.distances)
"""Explanation: Distance to the nearest neighbor is math.sqrt(A.distances[i]), in
units of pixels. The vector to the nearest neighbor, in units of pixels, is
( fstats.master[A.nn[i]].x()-query[2*i], fstats.master[A.nn[i]].y()-query[2*i+1])
"""
fstats['neighbor'] = (n_ave,n_std) = scitbx_stats(neighbors)
sep_input_spots = fstats[fstats.most_recent_child()]
sep_input_indices = fstats.get_indices(fstats.most_recent_child())
overlapping_count = 0
def do_connected_components(
experiments, # type: ExperimentList
reflection_tables, # type: flex.reflection_table
min_component_size=50, # type: int
): # type: (...) -> [{}]
"""
Write the behaviour of the program as functions and classes outside run().
Don't include file output here, remember that this function may be re-used
elsewhere by someone who doesn't need the output written immediately to file.
It can be especially helpful to document any expected exceptions that might be
raised, in order to keep track of what needs to be handled in any code that
re-uses this function.
Args:
experiments: An experiment list.
reflections: A reflection table.
"""
miller_array = scaled_data_as_miller_array(
reflection_tables, experiments,
anomalous_flag=False).primitive_setting()
unique = miller_array.unique_under_symmetry().map_to_asu()
unique = unique.generate_bijvoet_mates()
complete_set = unique.complete_set()
missing_set = complete_set.lone_set(unique)
missing_set = missing_set.expand_to_p1().customized_copy(
crystal_symmetry=missing_set.crystal_symmetry())
mi = missing_set.indices().as_vec3_double().as_double()
k = 6
ann = AnnAdaptor(data=mi, dim=3, k=k)
ann.query(mi)
G = nx.Graph()
distance_cutoff = 2**0.5
for i in range(missing_set.size()):
ik = i * k
for i_ann in range(k):
if ann.distances[ik + i_ann] <= distance_cutoff:
j = ann.nn[ik + i_ann]
G.add_edge(i, j)
conn = sorted(nx.connected_components(G), key=len, reverse=True)
unique_mi = []
unique_ms = []
for i, c in enumerate(conn):
ms = (missing_set.select(flex.size_t(list(c))).customized_copy(
crystal_symmetry=miller_array).as_non_anomalous_set().map_to_asu())
ms = ms.unique_under_symmetry()
mi = set(ms.indices())
if mi not in unique_mi:
unique_ms.append(ms)
unique_mi.append(mi)
n_expected = unique.as_non_anomalous_set().complete_set().size()
unique_ms = sorted(unique_ms, key=lambda ms: ms.size(), reverse=True)
unique_ms = [ms for ms in unique_ms if ms.size() > min_component_size]
for ms in unique_ms:
d_max, d_min = (uctbx.d_star_sq_as_d(ds2)
for ds2 in ms.min_max_d_star_sq())
logger.info("%i reflections (%.1f%%): %.2f-%.2f Å" %
(ms.size(), 100 * ms.size() / n_expected, d_max, d_min))
return unique_ms
def integration_concept_detail(self, experiments, reflections, spots,image_number,cb_op_to_primitive,**kwargs):
detector = experiments[0].detector
crystal = experiments[0].crystal
from cctbx.crystal import symmetry
c_symmetry = symmetry(space_group = crystal.get_space_group(), unit_cell = crystal.get_unit_cell())
self.image_number = image_number
NEAR = 10
pxlsz = detector[0].get_pixel_size()
Predicted = self.get_predictions_accounting_for_centering(experiments,reflections,cb_op_to_primitive,**kwargs)
FWMOSAICITY = self.inputai.getMosaicity()
self.DOMAIN_SZ_ANG = kwargs.get("domain_size_ang", self.__dict__.get("actual",0) )
refineflag = {True:0,False:1}[kwargs.get("domain_size_ang",0)==0]
c_symmetry.show_summary(prefix="EXCURSION%1d REPORT FWMOS= %6.4f DOMAIN= %6.1f "%(refineflag,FWMOSAICITY,self.DOMAIN_SZ_ANG))
from annlib_ext import AnnAdaptor
self.cell = c_symmetry.unit_cell()
query = flex.double()
print len(self.predicted)
for pred in self.predicted: # predicted spot coord in pixels
query.append(pred[0]/pxlsz[0])
query.append(pred[1]/pxlsz[1])
self.reserve_hkllist_for_signal_search = self.hkllist
reference = flex.double()
assert self.length>NEAR# Can't do spot/pred matching with too few spots
for spot in spots:
reference.append(spot.ctr_mass_x())
reference.append(spot.ctr_mass_y())
IS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR)
IS_adapt.query(query)
idx_cutoff = float(min(self.mask_focus[image_number]))
from rstbx.apps.slip_helpers import slip_callbacks
cache_refinement_spots = getattr(slip_callbacks.slip_callback,"requires_refinement_spots",False)
indexed_pairs_provisional = []
correction_vectors_provisional = []
c_v_p_flex = flex.vec3_double()
this_setting_matched_indices = reflections["miller_index"]
for j,item in enumerate(this_setting_matched_indices):
this_setting_index = self.hkllist.first_index(item)
if this_setting_index:
Match = dict(spot=j,pred=this_setting_index)
indexed_pairs_provisional.append(Match)
vector = matrix.col(
[reflections["xyzobs.px.value"][j][0] - self.predicted[Match["pred"]][0]/pxlsz[0],
reflections["xyzobs.px.value"][j][1] - self.predicted[Match["pred"]][1]/pxlsz[1]])
correction_vectors_provisional.append(vector)
c_v_p_flex.append((vector[0],vector[1],0.))
self.N_correction_vectors = len(correction_vectors_provisional)
self.rmsd_px = math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))
print "... %d provisional matches"%self.N_correction_vectors,
print "r.m.s.d. in pixels: %6.3f"%(self.rmsd_px)
if self.horizons_phil.integration.enable_residual_scatter:
from matplotlib import pyplot as plt
fig = plt.figure()
for cv in correction_vectors_provisional:
plt.plot([cv[1]],[-cv[0]],"r.")
plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))))
plt.axes().set_aspect("equal")
self.show_figure(plt,fig,"res")
plt.close()
if self.horizons_phil.integration.enable_residual_map:
from matplotlib import pyplot as plt
PX = reflections["xyzobs.px.value"]
fig = plt.figure()
for match,cv in zip(indexed_pairs_provisional,correction_vectors_provisional):
plt.plot([PX[match["spot"]][1]],[-PX[match["spot"]][0]],"r.")
plt.plot([self.predicted[match["pred"]][1]/pxlsz[1]],[-self.predicted[match["pred"]][0]/pxlsz[0]],"g.")
plt.plot([PX[match["spot"]][1], PX[match["spot"]][1] + 10.*cv[1]],
[-PX[match["spot"]][0], -PX[match["spot"]][0] - 10.*cv[0]],'r-')
if kwargs.get("user-reentrant") != None and self.horizons_phil.integration.spot_prediction == "dials" \
and self.horizons_phil.integration.enable_residual_map_deltapsi:
from rstbx.apps.stills.util import residual_map_special_deltapsi_add_on
residual_map_special_deltapsi_add_on(
reflections = self.dials_spot_prediction,
matches = indexed_pairs_provisional, experiments=experiments,
hkllist = self.hkllist,
predicted = self.predicted, plot=plt, eta_deg=FWMOSAICITY, deff=self.DOMAIN_SZ_ANG
)
plt.xlim([0,detector[0].get_image_size()[1]])
plt.ylim([-detector[0].get_image_size()[0],0])
plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))))
plt.axes().set_aspect("equal")
self.show_figure(plt,fig,"map")
plt.close()
indexed_pairs = indexed_pairs_provisional
correction_vectors = correction_vectors_provisional
########### skip outlier rejection for this derived class
### However must retain the ability to write out correction vectiors.
if True: # at Aaron's request; test later
correction_lengths = flex.double([v.length() for v in correction_vectors_provisional])
clorder = flex.sort_permutation(correction_lengths)
sorted_cl = correction_lengths.select(clorder)
indexed_pairs = []
correction_vectors = []
self.correction_vectors = []
for icand in xrange(len(sorted_cl)):
# somewhat arbitrary sigma = 1.0 cutoff for outliers
indexed_pairs.append(indexed_pairs_provisional[clorder[icand]])
correction_vectors.append(correction_vectors_provisional[clorder[icand]])
if cache_refinement_spots:
self.spotfinder.images[self.frame_numbers[self.image_number]]["refinement_spots"].append(
spots[reflections[indexed_pairs[-1]["spot"]]['spotfinder_lookup']])
if kwargs.get("verbose_cv")==True:
print "CV OBSCENTER %7.2f %7.2f REFINEDCENTER %7.2f %7.2f"%(
float(self.inputpd["size1"])/2.,float(self.inputpd["size2"])/2.,
self.inputai.xbeam()/pxlsz[0], self.inputai.ybeam()/pxlsz[1]),
print "OBSSPOT %7.2f %7.2f PREDSPOT %7.2f %7.2f"%(
reflections[indexed_pairs[-1]["spot"]]['xyzobs.px.value'][0],
reflections[indexed_pairs[-1]["spot"]]['xyzobs.px.value'][1],
self.predicted[indexed_pairs[-1]["pred"]][0]/pxlsz[0],
self.predicted[indexed_pairs[-1]["pred"]][1]/pxlsz[1]),
the_hkl = self.hkllist[indexed_pairs[-1]["pred"]]
print "HKL %4d %4d %4d"%the_hkl,"%2d"%self.setting_id,
radial, azimuthal = spots[indexed_pairs[-1]["spot"]].get_radial_and_azimuthal_size(
self.inputai.xbeam()/pxlsz[0], self.inputai.ybeam()/pxlsz[1])
print "RADIALpx %5.3f AZIMUTpx %5.3f"%(radial,azimuthal)
# Store a list of correction vectors in self.
radial, azimuthal = spots[indexed_pairs[-1]['spot']].get_radial_and_azimuthal_size(
self.inputai.xbeam()/pxlsz[0], self.inputai.ybeam()/pxlsz[1])
self.correction_vectors.append(
dict(obscenter=(float(self.inputpd['size1']) / 2,
float(self.inputpd['size2']) / 2),
refinedcenter=(self.inputai.xbeam() / pxlsz[0],
self.inputai.ybeam() / pxlsz[1]),
obsspot=(reflections[indexed_pairs[-1]['spot']]['xyzobs.px.value'][0],
reflections[indexed_pairs[-1]['spot']]['xyzobs.px.value'][1]),
predspot=(self.predicted[indexed_pairs[-1]['pred']][0] / pxlsz[0],
self.predicted[indexed_pairs[-1]['pred']][1] / pxlsz[1]),
hkl=(self.hkllist[indexed_pairs[-1]['pred']][0],
self.hkllist[indexed_pairs[-1]['pred']][1],
self.hkllist[indexed_pairs[-1]['pred']][2]),
setting_id=self.setting_id,
radial=radial,
azimuthal=azimuthal))
self.inputpd["symmetry"] = c_symmetry
self.inputpd["symmetry"].show_summary(prefix="SETTING ")
if self.horizons_phil.integration.model == "user_supplied":
# Not certain of whether the reentrant_* dictionary keys create a memory leak
if kwargs.get("user-reentrant",None)==None:
kwargs["reentrant_experiments"] = experiments
kwargs["reentrant_reflections"] = reflections
from cxi_user import post_outlier_rejection
self.indexed_pairs = indexed_pairs
self.spots = spots
post_outlier_rejection(self,image_number,cb_op_to_primitive,self.horizons_phil,kwargs)
return
########### finished with user-supplied code
correction_lengths=flex.double([v.length() for v in correction_vectors])
self.r_residual = pxlsz[0]*flex.mean(correction_lengths)
#assert len(indexed_pairs)>NEAR # must have enough indexed spots
if (len(indexed_pairs) <= NEAR):
raise Sorry("Not enough indexed spots, only found %d, need %d" % (len(indexed_pairs), NEAR))
reference = flex.double()
for item in indexed_pairs:
reference.append(spots[item["spot"]].ctr_mass_x())
reference.append(spots[item["spot"]].ctr_mass_y())
PS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR)
PS_adapt.query(query)
self.BSmasks = []
# do not use null: self.null_correction_mapping( predicted=self.predicted,
self.positional_correction_mapping( predicted=self.predicted,
correction_vectors = correction_vectors,
PS_adapt = PS_adapt,
IS_adapt = IS_adapt,
spots = spots)
# which spots are close enough to interfere with background?
MAXOVER=6
OS_adapt = AnnAdaptor(data=query,dim=2,k=MAXOVER) #six near nbrs
OS_adapt.query(query)
if self.mask_focus[image_number] is None:
raise Sorry("No observed/predicted spot agreement; no Spotfinder masks; skip integration")
nbr_cutoff = 2.0* max(self.mask_focus[image_number])
FRAME = int(nbr_cutoff/2)
#print "The overlap cutoff is %d pixels"%nbr_cutoff
nbr_cutoff_sq = nbr_cutoff * nbr_cutoff
#print "Optimized C++ section...",
self.set_frame(FRAME)
self.set_background_factor(kwargs["background_factor"])
self.set_nbr_cutoff_sq(nbr_cutoff_sq)
self.set_guard_width_sq(self.horizons_phil.integration.guard_width_sq)
self.set_detector_gain(self.horizons_phil.integration.detector_gain)
flex_sorted = flex.int()
for item in self.sorted:
flex_sorted.append(item[0]);flex_sorted.append(item[1]);
if self.horizons_phil.integration.mask_pixel_value is not None:
self.set_mask_pixel_val(self.horizons_phil.integration.mask_pixel_value)
image_obj = self.imagefiles.imageindex(self.frame_numbers[self.image_number])
image_obj.read()
rawdata = image_obj.linearintdata # assume image #1
if self.inputai.active_areas != None:
self.detector_xy_draft = self.safe_background( rawdata=rawdata,
predicted=self.predicted,
OS_adapt=OS_adapt,
sorted=flex_sorted,
tiles=self.inputai.active_areas.IT,
tile_id=self.inputai.active_areas.tile_id);
else:
self.detector_xy_draft = self.safe_background( rawdata=rawdata,
predicted=self.predicted,
OS_adapt=OS_adapt,
sorted=flex_sorted);
for i in xrange(len(self.predicted)): # loop over predicteds
B_S_mask = {}
keys = self.get_bsmask(i)
for k in xrange(0,len(keys),2):
B_S_mask[(keys[k],keys[k+1])]=True
self.BSmasks.append(B_S_mask)
#print "Done"
return
def connected_components(miller_array: cctbx.miller.array, ) -> [{}]:
"""
Identify connected regions of missing reflections in the asymmetric unit.
This is achieved by first generating the complete set of possible miller indices,
then performing connected components analysis on a graph of nearest neighbours in
the list of missing reflections.
Args:
miller_array: The input list of reflections.
Returns:
The list of miller sets for each connected region of missing reflections. The
first item in the list will be the complete set of all possible miller indices.
"""
# Map to primitive setting for centred cells, otherwise true missing reflections
# won't be identified as connected as a result of being separated by systematically
# absent reflections.
cb_op_to_primitive = miller_array.change_of_basis_op_to_primitive_setting()
miller_array = miller_array.change_basis(cb_op_to_primitive)
# First generate the missing_set of reflections. We want the full sphere of missing
# reflections to allow us to find connected regions that cross the boundary of the
# asu.
unique = miller_array.unique_under_symmetry().map_to_asu()
unique = unique.generate_bijvoet_mates()
complete_set = unique.complete_set()
missing_set = complete_set.lone_set(unique)
missing_set = missing_set.expand_to_p1().customized_copy(
crystal_symmetry=missing_set.crystal_symmetry())
if missing_set.size() == 0:
return complete_set, []
# Now find the nearest neighbours.
mi = missing_set.indices().as_vec3_double().as_double()
k = 6
ann = AnnAdaptor(data=mi, dim=3, k=k)
ann.query(mi)
# Construct the graph of connected missing reflections
g = graph.adjacency_list(
graph_type="undirected",
vertex_type="vector",
edge_type="set",
)
distance_cutoff = 2**0.5
for i in range(missing_set.size()):
ik = i * k
for i_ann in range(k):
if ann.distances[ik + i_ann] <= distance_cutoff:
j = ann.nn[ik + i_ann]
g.add_edge(i, j)
# Now do the connected components analysis, filtering out lone missing reflections
components = [c for c in cca.connected_components(graph=g) if len(c) > 1]
# Determine the unique miller indices for each component within the asu
unique_mi = []
unique_ms = []
for i, c in enumerate(components):
ms = (missing_set.select(flex.size_t(list(c))).customized_copy(
crystal_symmetry=miller_array).as_non_anomalous_set().map_to_asu())
ms = ms.unique_under_symmetry()
mi = set(ms.indices())
if mi not in unique_mi:
unique_ms.append(ms)
unique_mi.append(mi)
# Sort connected regions by size
unique_ms = sorted(unique_ms, key=lambda ms: ms.size(), reverse=True)
# Map indices back to input setting
cb_op_primitive_inp = cb_op_to_primitive.inverse()
return (
unique.as_non_anomalous_set().complete_set().change_basis(
cb_op_primitive_inp),
[ms.change_basis(cb_op_primitive_inp) for ms in unique_ms],
)
def __init__(self, reflections, step_size=45, tolerance=1.5,
max_height_fraction=0.25, percentile=0.05,
histogram_binning='linear'):
self.tolerance = tolerance # Margin of error for max unit cell estimate
from scitbx.array_family import flex
NEAR = 10
self.NNBIN = 5 # target number of neighbors per histogram bin
self.histogram_binning = histogram_binning
direct = flex.double()
if 'entering' in reflections:
entering_flags = reflections['entering']
else:
entering_flags = flex.bool(reflections.size(), True)
rs_vectors = reflections['rlp']
phi_deg = reflections['xyzobs.mm.value'].parts()[2] * (180/math.pi)
d_spacings = flex.double()
# nearest neighbor analysis
from annlib_ext import AnnAdaptor
for imageset_id in range(flex.max(reflections['imageset_id'])+1):
sel = reflections['imageset_id'] == imageset_id
if sel.count(True) == 0:
continue
phi_min = flex.min(phi_deg.select(sel))
phi_max = flex.max(phi_deg.select(sel))
d_phi = phi_max - phi_min
n_steps = max(int(math.ceil(d_phi / step_size)), 1)
for n in range(n_steps):
sel &= (phi_deg >= (phi_min+n*step_size)) & (phi_deg < (phi_min+(n+1)*step_size))
for entering in (True, False):
sel &= entering_flags == entering
if sel.count(True) == 0:
continue
query = flex.double()
query.extend(rs_vectors.select(sel).as_double())
if query.size() == 0:
continue
IS_adapt = AnnAdaptor(data=query,dim=3,k=1)
IS_adapt.query(query)
direct.extend(1/flex.sqrt(IS_adapt.distances))
d_spacings.extend(1/rs_vectors.norms())
assert len(direct)>NEAR, (
"Too few spots (%d) for nearest neighbour analysis." %len(direct))
perm = flex.sort_permutation(direct)
direct = direct.select(perm)
d_spacings = d_spacings.select(perm)
# reject top 1% of longest distances to hopefully get rid of any outliers
n = int(math.floor(0.99*len(direct)))
direct = direct[:n]
d_spacings = d_spacings[:n]
# determine the most probable nearest neighbor distance (direct space)
if self.histogram_binning == 'log':
hst = flex.histogram(
flex.log10(direct), n_slots=int(len(direct)/self.NNBIN))
else:
hst = flex.histogram(direct, n_slots=int(len(direct)/self.NNBIN))
centers = hst.slot_centers()
if self.histogram_binning == 'log':
self.slot_start = flex.double(
[10**s for s in hst.slot_centers() - 0.5 * hst.slot_width()])
self.slot_end = flex.double(
[10**s for s in hst.slot_centers() + 0.5 * hst.slot_width()])
self.slot_width = self.slot_end - self.slot_start
else:
self.slot_start = hst.slot_centers() - 0.5 * hst.slot_width()
self.slot_end = hst.slot_centers() + 0.5 * hst.slot_width()
self.slot_width = hst.slot_width()
self.relative_frequency = hst.slots().as_double()/self.slot_width
highest_bin_height = flex.max(self.relative_frequency)
if False: # to print out the histogramming analysis
smin, smax = flex.min(direct), flex.max(direct)
stats = flex.mean_and_variance(direct)
import sys
out = sys.stdout
print >> out, " range: %6.2f - %.2f" % (smin, smax)
print >> out, " mean: %6.2f +/- %6.2f on N = %d" % (
stats.mean(), stats.unweighted_sample_standard_deviation(), direct.size())
hst.show(f=out, prefix=" ", format_cutoffs="%6.2f")
print >> out, ""
# choose a max cell based on bins above a given fraction of the highest bin height
# given multiple
isel = (self.relative_frequency.as_double() > (
max_height_fraction * highest_bin_height)).iselection()
self.max_cell = (
self.tolerance * self.slot_end[int(flex.max(isel.as_double()))])
# determine the 5th-percentile direct-space distance
perm = flex.sort_permutation(direct, reverse=True)
self.percentile = direct[perm[int(percentile * len(direct))]]
self.reciprocal_lattice_vectors = rs_vectors
self.d_spacings = d_spacings
self.direct = direct
self.histogram = hst
def integration_concept(self,image_number=0,cb_op_to_primitive=None,verbose=False,**kwargs):
self.image_number = image_number
NEAR = 10
pxlsz = self.pixel_size
self.get_predictions_accounting_for_centering(cb_op_to_primitive,**kwargs)
FWMOSAICITY = self.inputai.getMosaicity()
DOMAIN_SZ_ANG = kwargs.get("domain_size_ang", self.__dict__.get("actual",0) )
refineflag = {True:0,False:1}[kwargs.get("domain_size_ang",0)==0]
self.inputpd["symmetry"].show_summary(prefix="EXCURSION%1d REPORT FWMOS= %6.4f DOMAIN= %6.1f "%(refineflag,FWMOSAICITY,DOMAIN_SZ_ANG))
from annlib_ext import AnnAdaptor
self.cell = self.inputai.getOrientation().unit_cell()
query = flex.double()
for pred in self.predicted: # predicted spot coord in pixels
query.append(pred[0]/pxlsz)
query.append(pred[1]/pxlsz)
self.reserve_hkllist_for_signal_search = self.hkllist
reference = flex.double()
spots = self.get_observations_with_outlier_removal()
assert len(spots)>NEAR# Can't do spot/pred matching with too few spots
for spot in spots:
reference.append(spot.ctr_mass_x())
reference.append(spot.ctr_mass_y())
IS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR)
IS_adapt.query(query)
print "Calculate correction vectors for %d observations & %d predictions"%(len(spots),len(self.predicted))
indexed_pairs_provisional = []
correction_vectors_provisional = []
c_v_p_flex = flex.vec3_double()
idx_cutoff = float(min(self.mask_focus[image_number]))
if verbose:
print "idx_cutoff distance in pixels",idx_cutoff
if not self.horizons_phil.integration.enable_one_to_one_safeguard:
# legacy code, no safeguard against many-to-one predicted-to-observation mapping
for i in xrange(len(self.predicted)): # loop over predicteds
#for n in xrange(NEAR): # loop over near spotfinder spots
for n in xrange(1): # only consider the nearest spotfinder spot
Match = dict(spot=IS_adapt.nn[i*NEAR+n],pred=i)
if n==0 and math.sqrt(IS_adapt.distances[i*NEAR+n]) < idx_cutoff:
indexed_pairs_provisional.append(Match)
vector = matrix.col(
[spots[Match["spot"]].ctr_mass_x() - self.predicted[Match["pred"]][0]/pxlsz,
spots[Match["spot"]].ctr_mass_y() - self.predicted[Match["pred"]][1]/pxlsz])
correction_vectors_provisional.append(vector)
c_v_p_flex.append((vector[0],vector[1],0.))
else:
one_to_one = {}
for i in xrange(len(self.predicted)): # loop over predicteds
annresultidx = i*NEAR
obsidx = IS_adapt.nn[annresultidx]
this_distancesq = IS_adapt.distances[annresultidx]
if not one_to_one.has_key(obsidx) or \
this_distancesq < one_to_one[obsidx]["distancesq"]:
if math.sqrt(this_distancesq) < idx_cutoff:
one_to_one[obsidx] = dict(spot=obsidx,pred=i,distancesq=this_distancesq)
for key,value in one_to_one.items():
indexed_pairs_provisional.append(value)
vector = matrix.col(
[spots[value["spot"]].ctr_mass_x() - self.predicted[value["pred"]][0]/pxlsz,
spots[value["spot"]].ctr_mass_y() - self.predicted[value["pred"]][1]/pxlsz])
correction_vectors_provisional.append(vector)
c_v_p_flex.append((vector[0],vector[1],0.))
print "... %d provisional matches"%len(correction_vectors_provisional),
print "r.m.s.d. in pixels: %5.2f"%(math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))
if self.horizons_phil.integration.enable_residual_scatter:
from matplotlib import pyplot as plt
fig = plt.figure()
for cv in correction_vectors_provisional:
plt.plot([cv[1]],[-cv[0]],"b.")
plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))))
plt.axes().set_aspect("equal")
self.show_figure(plt,fig,"res")
plt.close()
if self.horizons_phil.integration.enable_residual_map:
from matplotlib import pyplot as plt
fig = plt.figure()
for match,cv in zip(indexed_pairs_provisional,correction_vectors_provisional):
plt.plot([spots[match["spot"]].ctr_mass_y()],[-spots[match["spot"]].ctr_mass_x()],"r.")
plt.plot([self.predicted[match["pred"]][1]/pxlsz],[-self.predicted[match["pred"]][0]/pxlsz],"g.")
plt.plot([spots[match["spot"]].ctr_mass_y(), spots[match["spot"]].ctr_mass_y() + 10.*cv[1]],
[-spots[match["spot"]].ctr_mass_x(), -spots[match["spot"]].ctr_mass_x() - 10.*cv[0]],'b-')
plt.xlim([0,float(self.inputpd["size2"])])
plt.ylim([-float(self.inputpd["size1"]),0])
plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))))
plt.axes().set_aspect("equal")
self.show_figure(plt,fig,"map")
plt.close()
# insert code here to remove correction length outliers...
# they are causing terrible
# problems for finding legitimate correction vectors (print out the list)
# also remove outliers for the purpose of reporting RMS
outlier_rejection = True
cache_refinement_spots = getattr(slip_callbacks.slip_callback,"requires_refinement_spots",False)
if outlier_rejection:
correction_lengths = flex.double([v.length() for v in correction_vectors_provisional])
clorder = flex.sort_permutation(correction_lengths)
sorted_cl = correction_lengths.select(clorder)
ACCEPTABLE_LIMIT = 2
limit = int(0.33 * len(sorted_cl)) # best 1/3 of data are assumed to be correctly modeled.
if (limit <= ACCEPTABLE_LIMIT):
raise Sorry("Not enough indexed spots to reject outliers; have %d need >%d" % (limit, ACCEPTABLE_LIMIT))
y_data = flex.double(len(sorted_cl))
for i in xrange(len(y_data)):
y_data[i] = float(i)/float(len(y_data))
# ideas are explained in Sauter & Poon (2010) J Appl Cryst 43, 611-616.
from rstbx.outlier_spots.fit_distribution import fit_cdf,rayleigh
fitted_rayleigh = fit_cdf(x_data = sorted_cl[0:limit],
y_data = y_data[0:limit],
distribution=rayleigh)
inv_cdf = [fitted_rayleigh.distribution.inv_cdf(cdf) for cdf in y_data]
#print "SORTED LIST OF ",len(sorted_cl), "with sigma",fitted_rayleigh.distribution.sigma
indexed_pairs = []
correction_vectors = []
self.correction_vectors = []
for icand in xrange(len(sorted_cl)):
# somewhat arbitrary sigma = 1.0 cutoff for outliers
if (sorted_cl[icand]-inv_cdf[icand])/fitted_rayleigh.distribution.sigma > 1.0:
break
indexed_pairs.append(indexed_pairs_provisional[clorder[icand]])
correction_vectors.append(correction_vectors_provisional[clorder[icand]])
if cache_refinement_spots:
self.spotfinder.images[self.frame_numbers[self.image_number]]["refinement_spots"].append(
spots[indexed_pairs[-1]["spot"]])
if kwargs.get("verbose_cv")==True:
print "CV OBSCENTER %7.2f %7.2f REFINEDCENTER %7.2f %7.2f"%(
float(self.inputpd["size1"])/2.,float(self.inputpd["size2"])/2.,
self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz),
print "OBSSPOT %7.2f %7.2f PREDSPOT %7.2f %7.2f"%(
spots[indexed_pairs[-1]["spot"]].ctr_mass_x(),
spots[indexed_pairs[-1]["spot"]].ctr_mass_y(),
self.predicted[indexed_pairs[-1]["pred"]][0]/pxlsz,
self.predicted[indexed_pairs[-1]["pred"]][1]/pxlsz),
the_hkl = self.hkllist[indexed_pairs[-1]["pred"]]
print "HKL %4d %4d %4d"%the_hkl,"%2d"%self.setting_id,
radial, azimuthal = spots[indexed_pairs[-1]["spot"]].get_radial_and_azimuthal_size(
self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz)
print "RADIALpx %5.3f AZIMUTpx %5.3f"%(radial,azimuthal)
# Store a list of correction vectors in self.
radial, azimuthal = spots[indexed_pairs[-1]['spot']].get_radial_and_azimuthal_size(
self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz)
self.correction_vectors.append(
dict(obscenter=(float(self.inputpd['size1']) / 2,
float(self.inputpd['size2']) / 2),
refinedcenter=(self.inputai.xbeam() / pxlsz,
self.inputai.ybeam() / pxlsz),
obsspot=(spots[indexed_pairs[-1]['spot']].ctr_mass_x(),
spots[indexed_pairs[-1]['spot']].ctr_mass_y()),
predspot=(self.predicted[indexed_pairs[-1]['pred']][0] / pxlsz,
self.predicted[indexed_pairs[-1]['pred']][1] / pxlsz),
hkl=(self.hkllist[indexed_pairs[-1]['pred']][0],
self.hkllist[indexed_pairs[-1]['pred']][1],
self.hkllist[indexed_pairs[-1]['pred']][2]),
setting_id=self.setting_id,
radial=radial,
azimuthal=azimuthal))
print "After outlier rejection %d indexed spotfinder spots remain."%len(indexed_pairs)
if False:
rayleigh_cdf = [
fitted_rayleigh.distribution.cdf(x=sorted_cl[c]) for c in xrange(len(sorted_cl))]
from matplotlib import pyplot as plt
plt.plot(sorted_cl,y_data,"r+")
#plt.plot(sorted_cl,rayleigh_cdf,"g.")
plt.plot(inv_cdf,y_data,"b.")
plt.show()
else:
indexed_pairs = indexed_pairs_provisional
correction_vectors = correction_vectors_provisional
########### finished with outlier rejection
self.inputpd["symmetry"].show_summary(prefix="SETTING ")
is_triclinic = (self.setting_id==1)
if is_triclinic:
self.triclinic_pairs = [ dict(pred=self.hkllist[a["pred"]],spot=a["spot"])
for a in indexed_pairs ]
if self.horizons_phil.integration.model == "user_supplied":
if kwargs.get("user-reentrant",None)==None:
from cxi_user import post_outlier_rejection
self.indexed_pairs = indexed_pairs
self.spots = spots
post_outlier_rejection(self,image_number,cb_op_to_primitive,self.horizons_phil,kwargs)
return
########### finished with user-supplied code
if self.horizons_phil.integration.spot_shape_verbose:
from rstbx.new_horizons.spot_shape import spot_shape_verbose
spot_shape_verbose(rawdata = self.imagefiles.images[self.image_number].linearintdata,
beam_center_pix = matrix.col((self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz)),
indexed_pairs = indexed_pairs,
spotfinder_observations = spots,
distance_mm = self.inputai.distance(),
mm_per_pixel = pxlsz,
hkllist = self.hkllist,
unit_cell = self.cell,
wavelength_ang = self.inputai.wavelength
)
#Other checks to be implemented (future):
# spot is within active area of detector on a circular detector such as the Mar IP
# integration masks do not overlap; or deconvolute
correction_lengths=flex.double([v.length() for v in correction_vectors])
if verbose:
print "average correction %5.2f over %d vectors"%(flex.mean(correction_lengths),
len(correction_lengths)),
print "or %5.2f mm."%(pxlsz*flex.mean(correction_lengths))
self.r_residual = pxlsz*flex.mean(correction_lengths)
#assert len(indexed_pairs)>NEAR # must have enough indexed spots
if (len(indexed_pairs) <= NEAR):
raise Sorry("Not enough indexed spots, only found %d, need %d" % (len(indexed_pairs), NEAR))
reference = flex.double()
for item in indexed_pairs:
reference.append(spots[item["spot"]].ctr_mass_x())
reference.append(spots[item["spot"]].ctr_mass_y())
PS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR)
PS_adapt.query(query)
self.BSmasks = []
#self.null_correction_mapping( predicted=self.predicted,
# correction_vectors = correction_vectors,
# IS_adapt = IS_adapt,
# spots = spots)
self.positional_correction_mapping( predicted=self.predicted,
correction_vectors = correction_vectors,
PS_adapt = PS_adapt,
IS_adapt = IS_adapt,
spots = spots)
# which spots are close enough to interfere with background?
MAXOVER=6
OS_adapt = AnnAdaptor(data=query,dim=2,k=MAXOVER) #six near nbrs
OS_adapt.query(query)
if self.mask_focus[image_number] is None:
raise Sorry("No observed/predicted spot agreement; no Spotfinder masks; skip integration")
nbr_cutoff = 2.0* max(self.mask_focus[image_number])
FRAME = int(nbr_cutoff/2)
#print "The overlap cutoff is %d pixels"%nbr_cutoff
nbr_cutoff_sq = nbr_cutoff * nbr_cutoff
#print "Optimized C++ section...",
self.set_frame(FRAME)
self.set_background_factor(kwargs["background_factor"])
self.set_nbr_cutoff_sq(nbr_cutoff_sq)
self.set_guard_width_sq(self.horizons_phil.integration.guard_width_sq)
self.set_detector_gain(self.horizons_phil.integration.detector_gain)
flex_sorted = flex.int()
for item in self.sorted:
flex_sorted.append(item[0]);flex_sorted.append(item[1]);
if self.horizons_phil.integration.mask_pixel_value is not None:
self.set_mask_pixel_val(self.horizons_phil.integration.mask_pixel_value)
image_obj = self.imagefiles.imageindex(self.frame_numbers[self.image_number])
image_obj.read()
rawdata = image_obj.linearintdata # assume image #1
if self.inputai.active_areas != None:
self.detector_xy_draft = self.safe_background( rawdata=rawdata,
predicted=self.predicted,
OS_adapt=OS_adapt,
sorted=flex_sorted,
tiles=self.inputai.active_areas.IT,
tile_id=self.inputai.active_areas.tile_id);
else:
self.detector_xy_draft = self.safe_background( rawdata=rawdata,
predicted=self.predicted,
OS_adapt=OS_adapt,
sorted=flex_sorted);
for i in xrange(len(self.predicted)): # loop over predicteds
B_S_mask = {}
keys = self.get_bsmask(i)
for k in xrange(0,len(keys),2):
B_S_mask[(keys[k],keys[k+1])]=True
self.BSmasks.append(B_S_mask)
#print "Done"
return
