def get_bounded_data(self, data, bounds):
assert len(bounds) == 4
x = [b[0] for b in bounds]
y = [b[1] for b in bounds]
left = sorted(x)[1]
right = sorted(x)[2]
top = sorted(y)[2]
bottom = sorted(y)[1]
origin = col((left, bottom))
scale_x = right-left
scale_y = top-bottom
scale = min(scale_x, scale_y)
data_max_x = flex.max(data.parts()[0])
data_min_x = flex.min(data.parts()[0])
data_max_y = flex.max(data.parts()[1])
data_min_y = flex.min(data.parts()[1])
data_scale_x = data_max_x - data_min_x
data_scale_y = data_max_y - data_min_y
if data_scale_x == 0 or data_scale_y == 0:
print "WARNING bad scale"
return data
return flex.vec2_double(data.parts()[0] * (scale/abs(data_scale_x)),
data.parts()[1] * (scale/abs(data_scale_y))) + origin
def __init__(self, strategies, n_bins=8, degrees_per_bin=5):
from cctbx import crystal, miller
import copy
sg = strategies[0].experiment.crystal.get_space_group() \
.build_derived_reflection_intensity_group(anomalous_flag=True)
cs = crystal.symmetry(
unit_cell=strategies[0].experiment.crystal.get_unit_cell(), space_group=sg)
for i, strategy in enumerate(strategies):
if i == 0:
predicted = copy.deepcopy(strategy.predicted)
else:
predicted_ = copy.deepcopy(strategy.predicted)
predicted_['dose'] += (flex.max(predicted['dose']) + 1)
predicted.extend(predicted_)
ms = miller.set(cs, indices=predicted['miller_index'], anomalous_flag=True)
ma = miller.array(ms, data=flex.double(ms.size(),1),
sigmas=flex.double(ms.size(), 1))
if 1:
merging = ma.merge_equivalents()
o = merging.array().customized_copy(
data=merging.redundancies().data().as_double()).as_mtz_dataset('I').mtz_object()
o.write('predicted.mtz')
d_star_sq = ma.d_star_sq().data()
binner = ma.setup_binner_d_star_sq_step(
d_star_sq_step=(flex.max(d_star_sq)-flex.min(d_star_sq)+1e-8)/n_bins)
dose = predicted['dose']
range_width = 1
range_min = flex.min(dose) - range_width
range_max = flex.max(dose)
n_steps = 2 + int((range_max - range_min) - range_width)
binner_non_anom = ma.as_non_anomalous_array().use_binning(
binner)
self.n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])
from xia2.Modules.PyChef2 import ChefStatistics
chef_stats = ChefStatistics(
ma.indices(), ma.data(), ma.sigmas(),
ma.d_star_sq().data(), dose, self.n_complete, binner,
ma.space_group(), ma.anomalous_flag(), n_steps)
def fraction_new(completeness):
# Completeness so far at end of image
completeness_end = completeness[1:]
# Completeness so far at start of image
completeness_start = completeness[:-1]
# Fraction of unique reflections observed for the first time on each image
return completeness_end - completeness_start
self.dose = dose
self.ieither_completeness = chef_stats.ieither_completeness()
self.iboth_completeness = chef_stats.iboth_completeness()
self.frac_new_ref = fraction_new(self.ieither_completeness) / degrees_per_bin
self.frac_new_pairs = fraction_new(self.iboth_completeness) / degrees_per_bin
def get_min_max_xy(self, rlist):
xc, yc, zc = rlist['xyzcal.px'].parts()
xo, yo, zo = rlist['xyzobs.px.value'].parts()
min_x = math.floor(min(flex.min(xc), flex.min(xo)))
min_y = math.floor(min(flex.min(yc), flex.min(yo)))
max_x = math.ceil(max(flex.max(xc), flex.max(xo)))
max_y = math.ceil(max(flex.max(yc), flex.max(yo)))
return min_x, max_x, min_y, max_y
def get_min_max_xy(self, rlist):
xc, yc, zc = rlist['xyzcal.px'].parts()
xo, yo, zo = rlist['xyzobs.px.value'].parts()
dx = xc - xo
dz = zc - zo
min_x = math.floor(flex.min(dx))
min_y = math.floor(flex.min(dz))
max_x = math.ceil(flex.max(dx))
max_y = math.ceil(flex.max(dz))
return min_x, max_x, min_y, max_y
def per_image(self):
"""Set one block per image for all experiments"""
self._create_block_columns()
# get observed phi in radians
phi_obs = self._reflections['xyzobs.mm.value'].parts()[2]
for iexp, exp in enumerate(self._experiments):
sel = self._reflections['id'] == iexp
isel = sel.iselection()
exp_phi = phi_obs.select(isel)
# convert phi to integer frames
frames = exp.scan.get_array_index_from_angle(exp_phi, deg=False)
frames = flex.floor(frames).iround()
start, stop = flex.min(frames), flex.max(frames)
frame_range = range(start, stop + 1)
for f_num, f in enumerate(frame_range):
sub_isel = isel.select(frames == f)
self._reflections['block'].set_selected(sub_isel, f_num)
self._reflections['block_centre'].set_selected(sub_isel, f_num)
return self._reflections
def profile2d(p, vmin=None, vmax=None):
from dials.array_family import flex
import string
if vmin is None:
vmin = flex.min(p)
if vmax is None:
vmax = flex.max(p)
assert(vmax >= vmin)
dv = vmax - vmin
if dv == 0:
c = 0
m = 0
else:
m = 35.0 / dv
c = -m * vmin
lookup = string.digits + string.ascii_uppercase
ny, nx = p.all()
text = ''
for j in range(ny):
for i in range(nx):
v = int(m * p[j,i] + c)
if v < 0:
v = 0
elif v > 35:
v = 35
t = lookup[v]
text += t + ' '
text += '\n'
return text
def compose(self, reflections):
"""Compose scan-varying crystal parameterisations at the specified image
number, for the specified experiment, for each image. Put the U, B and
UB matrices in the reflection table, and cache the derivatives."""
self._prepare_for_compose(reflections)
for iexp, exp in enumerate(self._experiments):
# select the reflections of interest
sel = reflections['id'] == iexp
isel = sel.iselection()
blocks = reflections['block'].select(isel)
# identify which crystal parameterisations to use for this experiment
xl_op = self._get_xl_orientation_parameterisation(iexp)
xl_ucp = self._get_xl_unit_cell_parameterisation(iexp)
# get state and derivatives for each block
for block in xrange(flex.min(blocks),
flex.max(blocks) + 1):
# determine the subset of reflections this affects
subsel = isel.select(blocks == block)
if len(subsel) == 0: continue
# get the integer frame number nearest the centre of that block
frames = reflections['block_centre'].select(subsel)
# can only be false if original block assignment has gone wrong
assert frames.all_eq(frames[0]), \
"Failing: a block contains reflections that shouldn't be there"
frame = int(floor(frames[0]))
# model states at current frame
U = self._get_state_from_parameterisation(xl_op, frame)
if U is None: U = exp.crystal.get_U()
B = self._get_state_from_parameterisation(xl_ucp, frame)
if B is None: B = exp.crystal.get_B()
# set states
reflections['u_matrix'].set_selected(subsel, U.elems)
reflections['b_matrix'].set_selected(subsel, B.elems)
# set derivatives of the states
if xl_op is not None:
for j, dU in enumerate(xl_op.get_ds_dp()):
colname = "dU_dp{0}".format(j)
reflections[colname].set_selected(subsel, dU)
if xl_ucp is not None:
for j, dB in enumerate(xl_ucp.get_ds_dp()):
colname = "dB_dp{0}".format(j)
reflections[colname].set_selected(subsel, dB)
# set the UB matrices for prediction
reflections['ub_matrix'] = reflections['u_matrix'] * reflections['b_matrix']
return
def _local_setup(self, reflections):
"""Setup additional attributes used in gradients calculation. These are
specific to scans-type prediction parameterisations"""
# Spindle rotation matrices for every reflection
#R = self._axis.axis_and_angle_as_r3_rotation_matrix(phi)
#R = flex.mat3_double(len(reflections))
# NB for now use flex.vec3_double.rotate_around_origin each time I need the
# rotation matrix R.
# r is the reciprocal lattice vector, in the lab frame
self._phi_calc = reflections['xyzcal.mm'].parts()[2]
q = self._fixed_rotation * (self._UB * self._h)
self._r = self._setting_rotation * q.rotate_around_origin(self._axis, self._phi_calc)
# All of the derivatives of phi have a common denominator, given by
# (e X r).s0, where e is the rotation axis. Calculate this once, here.
self._e_X_r = (self._setting_rotation * self._axis).cross(self._r)
self._e_r_s0 = (self._e_X_r).dot(self._s0)
# Note that e_r_s0 -> 0 when the rotation axis, beam vector and
# relp are coplanar. This occurs when a reflection just touches
# the Ewald sphere.
#
# There is a relationship between e_r_s0 and zeta_factor.
# Uncommenting the code below shows that
# s0.(e X r) = zeta * |s X s0|
#from dials.algorithms.profile_model.gaussian_rs import zeta_factor
#from libtbx.test_utils import approx_equal
#s = matrix.col(reflections['s1'][0])
#z = zeta_factor(axis[0], s0[0], s)
#ss0 = (s.cross(matrix.col(s0[0]))).length()
#assert approx_equal(e_r_s0[0], z * ss0)
# catch small values of e_r_s0
e_r_s0_mag = flex.abs(self._e_r_s0)
try:
assert flex.min(e_r_s0_mag) > 1.e-6
except AssertionError as e:
imin = flex.min_index(e_r_s0_mag)
print "(e X r).s0 too small:"
print "for", (e_r_s0_mag <= 1.e-6).count(True), "reflections"
print "out of", len(e_r_s0_mag), "total"
print "such as", reflections['miller_index'][imin]
print "with scattering vector", reflections['s1'][imin]
print "where r =", self._r[imin]
print "e =", self._axis[imin]
print "s0 =", self._s0[imin]
print ("this reflection forms angle with the equatorial plane "
"normal:")
vecn = matrix.col(self._s0[imin]).cross(matrix.col(self._axis[imin])).normalize()
print matrix.col(reflections['s1'][imin]).accute_angle(vecn)
raise e
return
def generate_mmcif(crystal, refiner, file):
logger.info('Saving mmCIF information to %s' % file)
from cctbx import miller
import datetime
import iotbx.cif.model
import math
block = iotbx.cif.model.block()
block["_audit.creation_method"] = dials_version()
block["_audit.creation_date"] = datetime.date.today().isoformat()
# block["_publ.section_references"] = '' # once there is a reference...
for cell, esd, cifname in zip(crystal.get_unit_cell().parameters(),
crystal.get_cell_parameter_sd(),
['length_a', 'length_b', 'length_c', 'angle_alpha', 'angle_beta', 'angle_gamma']):
block['_cell.%s' % cifname] = "%.8f" % cell
block['_cell.%s_esd' % cifname] = "%.8f" % esd
block['_cell.volume'] = "%f" % crystal.get_unit_cell().volume()
block['_cell.volume_esd'] = "%f" % crystal.get_cell_volume_sd()
used_reflections = refiner.get_matches()
block['_cell_measurement.reflns_used'] = len(used_reflections)
block['_cell_measurement.theta_min'] = flex.min(used_reflections['2theta_obs.rad']) * 180 / math.pi / 2
block['_cell_measurement.theta_max'] = flex.max(used_reflections['2theta_obs.rad']) * 180 / math.pi / 2
block['_diffrn_reflns.number'] = len(used_reflections)
miller_span = miller.index_span(used_reflections['miller_index'])
min_h, min_k, min_l = miller_span.min()
max_h, max_k, max_l = miller_span.max()
block['_diffrn_reflns.limit_h_min'] = min_h
block['_diffrn_reflns.limit_h_max'] = max_h
block['_diffrn_reflns.limit_k_min'] = min_k
block['_diffrn_reflns.limit_k_max'] = max_k
block['_diffrn_reflns.limit_l_min'] = min_l
block['_diffrn_reflns.limit_l_max'] = max_l
block['_diffrn_reflns.theta_min'] = flex.min(used_reflections['2theta_obs.rad']) * 180 / math.pi / 2
block['_diffrn_reflns.theta_max'] = flex.max(used_reflections['2theta_obs.rad']) * 180 / math.pi / 2
cif = iotbx.cif.model.cif()
cif['two_theta_refine'] = block
with open(file, 'w') as fh:
cif.show(out=fh)
def profile3d(p, vmin=None, vmax=None):
''' Print a 3D profile. '''
from dials.array_family import flex
if vmin is None:
vmin = flex.min(p)
if vmax is None:
vmax = flex.max(p)
nz, ny, nx = p.all()
text = []
for k in range(nz):
p2 = p[k:k+1,:,:]
p2.reshape(flex.grid(ny, nx))
text.append(profile2d(p2, vmin=vmin, vmax=vmax))
return '\n'.join(text)
def ice_rings_selection(reflections):
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
from dials.algorithms.integration import filtering
unit_cell = uctbx.unit_cell((4.498,4.498,7.338,90,90,120))
space_group = sgtbx.space_group_info(number=194).group()
width = 0.06
ice_filter = filtering.PowderRingFilter(
unit_cell, space_group, flex.min(d_spacings)-width, width)
ice_sel = ice_filter(d_spacings)
return ice_sel
def get_normalized_colors(self, data, vmin=None, vmax=None):
if vmax is None:
vmax = self.params.residuals.plot_max
if vmax is None:
vmax = flex.max(data)
if vmin is None:
vmin = flex.min(data)
# initialize the color map
norm = Normalize(vmin=vmin, vmax=vmax)
cmap = plt.cm.get_cmap(self.params.colormap)
sm = cm.ScalarMappable(norm=norm, cmap=cmap)
color_vals = np.linspace(vmin, vmax, 11)
sm.set_array(color_vals) # needed for colorbar
return norm, cmap, color_vals, sm
def plot_statistics(statistics, prefix='', degrees_per_bin=5,
cutoff_anom=None, cutoff_non_anom=None):
range_width = 1
range_min = flex.min(statistics.dose) - range_width
range_max = flex.max(statistics.dose)
n_steps = 2 + int((range_max - range_min) - range_width)
x = flex.double_range(n_steps) * range_width + range_min
x *= degrees_per_bin
dpi = 300
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt
try: plt.style.use('ggplot')
except AttributeError: pass
line1, = plt.plot(x, statistics.ieither_completeness, label='Unique reflections')
line2, = plt.plot(x, statistics.iboth_completeness, label='Bijvoet pairs')
if cutoff_non_anom is not None:
plt.plot([cutoff_non_anom, cutoff_non_anom], plt.ylim(), c=line1.get_color(), linestyle='dashed')
if cutoff_anom is not None:
plt.plot([cutoff_anom, cutoff_anom], plt.ylim(), c=line2.get_color(), linestyle='dotted')
plt.xlim(0, plt.xlim()[1])
plt.xlabel('Scan angle (degrees)')
plt.ylabel('Completeness (%)')
plt.ylim(0, 1)
plt.legend(loc='lower right', fontsize='small')
plt.savefig('%scompleteness_vs_scan_angle.png' %prefix, dpi=dpi)
plt.clf()
line1, = plt.plot(x[1:], 100 * statistics.frac_new_ref, label='Unique reflections')
line2, = plt.plot(x[1:], 100 * statistics.frac_new_pairs, label='Bijvoet pairs')
ylim = plt.ylim()
if cutoff_non_anom is not None:
plt.plot([cutoff_non_anom, cutoff_non_anom], ylim, c=line1.get_color(), linestyle='dashed')
if cutoff_anom is not None:
plt.plot([cutoff_anom, cutoff_anom], ylim, c=line2.get_color(), linestyle='dotted')
plt.ylim(ylim)
plt.xlim(0, plt.xlim()[1])
plt.xlabel('Scan angle (degrees)')
plt.ylabel('% new reflections per degree')
plt.legend(loc='upper right', fontsize='small')
plt.savefig('%spercent_new_reflections_vs_scan_angle.png' %prefix, dpi=dpi)
plt.clf()
def plot_one_model(self,nrow,out):
fig = plt.subplot(self.gs[nrow*self.ncols])
two_thetas = self.reduction.get_two_theta_deg()
degrees = self.reduction.get_delta_psi_deg()
if self.color_encoding=="conventional":
positive = (self.reduction.i_sigi>=0.)
fig.plot(two_thetas.select(positive), degrees.select(positive), "bo")
fig.plot(two_thetas.select(~positive), degrees.select(~positive), "r+")
elif self.color_encoding=="I/sigma":
positive = (self.reduction.i_sigi>=0.)
tt_selected = two_thetas.select(positive)
dp_selected = degrees.select(positive)
i_sigi_select = self.reduction.i_sigi.select(positive)
order = flex.sort_permutation(i_sigi_select)
tt_selected = tt_selected.select(order)
dp_selected = dp_selected.select(order)
i_sigi_selected = i_sigi_select.select(order)
from matplotlib.colors import Normalize
dnorm = Normalize()
dcolors = i_sigi_selected.as_numpy_array()
dnorm.autoscale(dcolors)
N = len(dcolors)
CMAP = plt.get_cmap("rainbow")
if self.refined.get("partiality_array",None) is None:
for n in xrange(N):
fig.plot([tt_selected[n]],[dp_selected[n]],
color=CMAP(dnorm(dcolors[n])),marker=".", markersize=10)
else:
partials = self.refined.get("partiality_array")
partials_select = partials.select(positive)
partials_selected = partials_select.select(order)
assert len(partials)==len(positive)
for n in xrange(N):
fig.plot([tt_selected[n]],[dp_selected[n]],
color=CMAP(dnorm(dcolors[n])),marker=".", markersize=20*partials_selected[n])
# change the markersize to indicate partiality.
negative = (self.reduction.i_sigi<0.)
fig.plot(two_thetas.select(negative), degrees.select(negative), "r+", linewidth=1)
else:
strong = (self.reduction.i_sigi>=10.)
positive = ((~strong) & (self.reduction.i_sigi>=0.))
negative = (self.reduction.i_sigi<0.)
assert (strong.count(True)+positive.count(True)+negative.count(True) ==
len(self.reduction.i_sigi))
fig.plot(two_thetas.select(positive), degrees.select(positive), "bo")
fig.plot(two_thetas.select(strong), degrees.select(strong), marker='.',linestyle='None',
markerfacecolor='#00ee00', markersize=10)
fig.plot(two_thetas.select(negative), degrees.select(negative), "r+")
# indicate the imposed resolution filter
wavelength = self.reduction.experiment.beam.get_wavelength()
imposed_res_filter = self.reduction.get_imposed_res_filter(out)
resolution_markers = [
a for a in [imposed_res_filter,self.reduction.measurements.d_min()] if a is not None]
for RM in resolution_markers:
two_th = (180./math.pi)*2.*math.asin(wavelength/(2.*RM))
plt.plot([two_th, two_th],[self.AD1TF7B_MAXDP*-0.8,self.AD1TF7B_MAXDP*0.8],'k-')
plt.text(two_th,self.AD1TF7B_MAXDP*-0.9,"%4.2f"%RM)
#indicate the linefit
mean = flex.mean(degrees)
minplot = flex.min(two_thetas)
plt.plot([0,minplot],[mean,mean],"k-")
LR = flex.linear_regression(two_thetas, degrees)
model_y = LR.slope()*two_thetas + LR.y_intercept()
plt.plot(two_thetas, model_y, "k-")
#Now let's take care of the red and green lines.
half_mosaic_rotation_deg = self.refined["half_mosaic_rotation_deg"]
mosaic_domain_size_ang = self.refined["mosaic_domain_size_ang"]
red_curve_domain_size_ang = self.refined.get("red_curve_domain_size_ang",mosaic_domain_size_ang)
a_step = self.AD1TF7B_MAX2T / 50.
a_range = flex.double([a_step*x for x in xrange(1,50)]) # domain two-theta array
#Bragg law [d=L/2sinTH]
d_spacing = (wavelength/(2.*flex.sin(math.pi*a_range/360.)))
# convert two_theta to a delta-psi. Formula for Deffective [Dpsi=d/2Deff]
inner_phi_deg = flex.asin((d_spacing / (2.*red_curve_domain_size_ang)) )*(180./math.pi)
outer_phi_deg = flex.asin((d_spacing / (2.*mosaic_domain_size_ang)) + \
half_mosaic_rotation_deg*math.pi/180. )*(180./math.pi)
plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots\n%s"%(
2.*half_mosaic_rotation_deg, mosaic_domain_size_ang, len(two_thetas),
os.path.basename(self.reduction.filename)))
plt.plot(a_range, inner_phi_deg, "r-")
plt.plot(a_range,-inner_phi_deg, "r-")
plt.plot(a_range, outer_phi_deg, "g-")
plt.plot(a_range, -outer_phi_deg, "g-")
plt.xlim([0,self.AD1TF7B_MAX2T])
plt.ylim([-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP])
#second plot shows histogram
fig = plt.subplot(self.gs[1+nrow*self.ncols])
plt.xlim([-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP])
nbins = 50
n,bins,patches = plt.hist(dp_selected, nbins,
range=(-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP),
weights=self.reduction.i_sigi.select(positive),
normed=0, facecolor="orange", alpha=0.75)
#ersatz determine the median i_sigi point:
isi_positive = self.reduction.i_sigi.select(positive)
isi_order = flex.sort_permutation(isi_positive)
reordered = isi_positive.select(isi_order)
isi_median = reordered[int(len(isi_positive)*0.9)]
isi_top_half_selection = (isi_positive>isi_median)
n,bins,patches = plt.hist(dp_selected.select(isi_top_half_selection), nbins,
range=(-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP),
weights=isi_positive.select(isi_top_half_selection),
normed=0, facecolor="#ff0000", alpha=0.75)
plt.xlabel("(degrees)")
plt.title("Weighted histogram of Delta-psi")
patch[(y, x)] += 1.0 * iw / scale
cc = profile_correlation(data, patch)
if params.show:
print 'Simulated reflection (flattened in Z):'
print
for j in range(dy):
for i in range(dx):
print '%5d' % int(patch[(j, i)]),
print
print 'Correlation coefficient: %.3f isum: %.1f ' % (cc, i0)
import numpy as np
maxx = flex.max(all_pix.parts()[0])
maxy = flex.max(all_pix.parts()[1])
minx = flex.min(all_pix.parts()[0])
miny = flex.min(all_pix.parts()[1])
dx = (maxx-minx)/2
dy = (maxy-miny)/2
medx = minx + (dx)
medy = miny + (dy)
if maxx-minx > maxy-miny:
miny = medy-dx
maxy = medy+dx
else:
minx = medx-dy
maxx = medx+dy
limits = [[minx, maxx], [miny, maxy]]
reflection['xyzsim.mm'] = (flex.mean_weighted(all_pix.parts()[0], all_iw),
flex.mean_weighted(all_pix.parts()[1], all_iw), 0.0)
def spot_counts_per_image_plot(reflections, char='*', width=60, height=10):
from dials.array_family import flex
if len(reflections) == 0:
return '\n'
assert isinstance(char, basestring)
assert len(char) == 1
x,y,z = reflections['xyzobs.px.value'].parts()
min_z = flex.min(z)
max_z = flex.max(z)
# image numbers to display on x-axis label
xlab = (int(round(min_z + 0.5)), int(round(max_z + 0.5)))
# estimate the total number of images
image_count = xlab[1] - xlab[0] + 1
z_range = max_z - min_z + 1
if z_range <= 1:
return '%i spots found on 1 image' %len(reflections)
width = int(min(z_range, width))
z_step = z_range / width
z_bound = min_z + z_step - 0.5
# print [round(i * 10) / 10 for i in sorted(z)]
counts = flex.double()
sel = (z < z_bound)
counts.append(sel.count(True))
# print 0, ('-', z_bound), sel.count(True)
for i in range(1, width-1):
sel = ((z >= z_bound) & (z < (z_bound + z_step)))
counts.append(sel.count(True))
# print i, (z_bound, z_bound + z_step), sel.count(True)
z_bound += z_step
sel = (z >= z_bound)
# print i + 1, (z_bound, '-'), sel.count(True)
counts.append(sel.count(True))
max_count = flex.max(counts)
total_counts = flex.sum(counts)
assert total_counts == len(z)
counts *= (height/max_count)
counts = counts.iround()
rows = []
rows.append('%i spots found on %i images (max %i / bin)' %(
total_counts, image_count, max_count))
for i in range(height, 0, -1):
row = []
for j, c in enumerate(counts):
if c > (i - 1):
row.append(char)
else:
row.append(' ')
rows.append(''.join(row))
padding = width - len(str(xlab[0])) - len(str(xlab[1]))
rows.append('%i%s%i' % (xlab[0],
(' ' if padding < 7 else 'image').center(padding),
xlab[1]))
return '\n'.join(rows)
def __call__(self, reflections):
"""Identify outliers in the input and set the centroid_outlier flag.
Return True if any outliers were detected, otherwise False"""
if self._verbosity > 0:
logger.info("Detecting centroid outliers using the {0} algorithm".format(type(self).__name__))
# check the columns are present
for col in self._cols:
assert col in reflections
sel = reflections.get_flags(reflections.flags.used_in_refinement)
all_data = reflections.select(sel)
all_data_indices = sel.iselection()
nexp = flex.max(all_data["id"]) + 1
jobs = []
if self._separate_experiments:
# split the data set by experiment id
for iexp in xrange(nexp):
sel = all_data["id"] == iexp
job = {
"id": iexp,
"panel": "all",
"data": all_data.select(sel),
"indices": all_data_indices.select(sel),
}
jobs.append(job)
else:
# keep the whole dataset across all experiment ids
job = {"id": "all", "panel": "all", "data": all_data, "indices": all_data_indices}
jobs.append(job)
jobs2 = []
if self._separate_panels:
# split further by panel id
for job in jobs:
data = job["data"]
iexp = job["id"]
indices = job["indices"]
for ipanel in xrange(flex.max(data["panel"]) + 1):
sel = data["panel"] == ipanel
job = {"id": iexp, "panel": ipanel, "data": data.select(sel), "indices": indices.select(sel)}
jobs2.append(job)
else:
# keep the splits as they are
jobs2 = jobs
jobs3 = []
if self.get_block_width() is not None:
# split into equal-sized phi ranges
for job in jobs2:
data = job["data"]
iexp = job["id"]
ipanel = job["panel"]
indices = job["indices"]
phi = data["xyzobs.mm.value"].parts()[2]
if len(phi) == 0: # detect no data in the job
jobs3.append(job)
continue
phi_low = flex.min(phi)
phi_range = flex.max(phi) - phi_low
if phi_range == 0.0: # detect stills and do not split
jobs3.append(job)
continue
bw = self.get_block_width(iexp)
if bw is None: # detect no split for this experiment
jobs3.append(job)
continue
nblocks = int(round(RAD2DEG * phi_range / bw))
nblocks = max(1, nblocks)
real_width = phi_range / nblocks
block_end = 0.0
for iblock in xrange(nblocks - 1): # all except the last block
block_start = iblock * real_width
block_end = (iblock + 1) * real_width
sel = (phi >= (phi_low + block_start)) & (phi < (phi_low + block_end))
job = {
"id": iexp,
"panel": ipanel,
"data": data.select(sel),
"indices": indices.select(sel),
"phi_start": RAD2DEG * (phi_low + block_start),
"phi_end": RAD2DEG * (phi_low + block_end),
}
jobs3.append(job)
# now last block
sel = phi >= (phi_low + block_end)
job = {
"id": iexp,
"panel": ipanel,
"data": data.select(sel),
"indices": indices.select(sel),
"phi_start": RAD2DEG * (phi_low + block_end),
"phi_end": RAD2DEG * (phi_low + phi_range),
}
jobs3.append(job)
else:
# keep the splits as they are
jobs3 = jobs2
# Work out the format of the jobs table
if self._verbosity > 0:
header = ["Job"]
if self._separate_experiments:
header.append("Exp\nid")
if self._separate_panels:
header.append("Panel\nid")
if self.get_block_width() is not None:
header.append("Block range\n(deg)")
header.extend(["Nref", "Nout", "%out"])
rows = []
# now loop over the lowest level of splits
for i, job in enumerate(jobs3):
data = job["data"]
indices = job["indices"]
iexp = job["id"]
ipanel = job["panel"]
nref = len(indices)
if nref >= self._min_num_obs:
# get the subset of data as a list of columns
cols = [data[col] for col in self._cols]
# determine the position of outliers on this sub-dataset
outliers = self._detect_outliers(cols)
# get positions of outliers from the original matches
ioutliers = indices.select(outliers)
elif nref > 0:
# too few reflections in the job
msg = "For job {0}, fewer than {1} reflections are present.".format(i + 1, self._min_num_obs)
msg += " All reflections flagged as possible outliers."
if self._verbosity > 0:
logger.debug(msg)
ioutliers = indices
else:
# no reflections in the job
ioutliers = indices
# set the centroid_outlier flag in the original reflection table
nout = len(ioutliers)
if nout > 0:
reflections.set_flags(ioutliers, reflections.flags.centroid_outlier)
self.nreject += nout
# Add job data to the table
if self._verbosity > 0:
row = [str(i + 1)]
if self._separate_experiments:
row.append(str(iexp))
if self._separate_panels:
row.append(str(ipanel))
if self.get_block_width() is not None:
try:
row.append("{phi_start:.2f} - {phi_end:.2f}".format(**job))
except KeyError:
row.append("{0:.2f} - {1:.2f}".format(0.0, 0.0))
if nref == 0:
p100 = 0
else:
p100 = nout / nref * 100.0
if p100 > 30.0:
msg = ("{0:3.1f}% of reflections were flagged as outliers from job" " {1}").format(p100, i + 1)
row.extend([str(nref), str(nout), "%3.1f" % p100])
rows.append(row)
if self.nreject == 0:
return False
if self._verbosity > 0:
logger.info("{0} reflections have been flagged as outliers".format(self.nreject))
logger.debug("Outlier rejections per job:")
st = simple_table(rows, header)
logger.debug(st.format())
return True
def detector_plot_dict(self,
detector,
data,
title,
units_str,
show=True,
reverse_colormap=False):
"""
Use matplotlib to plot a detector, color coding panels according to data
@param detector detector reference detector object
@param data python dictionary of panel names as keys and numbers as values
@param title title string for plot
@param units_str string with a formatting statment for units on each panel
"""
# initialize the color map
values = flex.double(data.values())
norm = Normalize(vmin=flex.min(values), vmax=flex.max(values))
if reverse_colormap:
cmap = plt.cm.get_cmap(self.params.colormap + "_r")
else:
cmap = plt.cm.get_cmap(self.params.colormap)
sm = cm.ScalarMappable(norm=norm, cmap=cmap)
if len(values) == 0:
print "no values"
return
elif len(values) == 1:
sm.set_array(np.arange(values[0], values[0],
1)) # needed for colorbar
else:
sm.set_array(
np.arange(flex.min(values), flex.max(values),
(flex.max(values) - flex.min(values)) /
20)) # needed for colorbar
fig = plt.figure()
ax = fig.add_subplot(111, aspect='equal')
max_dim = 0
root = detector.hierarchy()
rf = col(root.get_fast_axis())
rs = col(root.get_slow_axis())
for pg_id, pg in enumerate(
iterate_detector_at_level(root, 0,
self.params.hierarchy_level)):
if pg.get_name() not in data:
continue
# get panel coordinates
p0, p1, p2, p3 = get_bounds(root, pg)
v1 = p1 - p0
v2 = p3 - p0
vcen = ((v2 / 2) + (v1 / 2)) + p0
# add the panel to the plot
ax.add_patch(
Polygon((p0[0:2], p1[0:2], p2[0:2], p3[0:2]),
closed=True,
color=sm.to_rgba(data[pg.get_name()]),
fill=True))
ax.annotate("%d %s" % (pg_id, units_str % data[pg.get_name()]),
vcen[0:2],
ha='center')
if self.params.draw_normal_arrows:
pgn = col(pg.get_normal())
v = col((rf.dot(pgn), rs.dot(pgn), 0))
v *= 10000
ax.arrow(vcen[0],
vcen[1],
v[0],
v[1],
head_width=5.0,
head_length=10.0,
fc='k',
ec='k')
# find the plot maximum dimensions
for p in [p0, p1, p2, p3]:
for c in p[0:2]:
if abs(c) > max_dim:
max_dim = abs(c)
# plot the results
ax.set_xlim((-max_dim, max_dim))
ax.set_ylim((-max_dim, max_dim))
ax.set_xlabel("mm")
ax.set_ylabel("mm")
fig.colorbar(sm)
plt.title(title)
if show:
plt.show()
def read_mtzfile(filename, batch_offset=None):
"""
Read the mtz file
"""
miller_arrays = mtz.object(file_name=filename).as_miller_arrays(
merge_equivalents=False)
# Select the desired columns
intensities = None
batches = None
for array in miller_arrays:
if array.info().labels == ["I", "SIGI"]:
intensities = array
if array.info().labels == ["BATCH"]:
batches = array
if not intensities:
raise KeyError(
"Intensities not found in mtz file, expected labels I, SIGI")
if not batches:
raise KeyError("Batch values not found")
if batches.data().size() != intensities.data().size():
raise ValueError("Batch and intensity array sizes do not match")
# Get the unit cell and space group
unit_cell = intensities.unit_cell()
space_group = intensities.crystal_symmetry().space_group()
# The reflection data
table = flex.reflection_table()
table["miller_index"] = intensities.indices()
table["intensity"] = intensities.data()
table["variance"] = flex.pow2(intensities.sigmas())
# Create unit cell list
zeroed_batches = batches.data() - flex.min(batches.data())
dataset = flex.int(table.size(), 0)
sorted_batches = flex.sorted(zeroed_batches)
sel_perm = flex.sort_permutation(zeroed_batches)
if not batch_offset:
previous = 0
potential_batch_offsets = flex.double()
for i, b in enumerate(sorted_batches):
if b - previous > 1:
potential_batch_offsets.append(b - previous)
previous = b
potential = flex.sorted(potential_batch_offsets)
# potential is a list of low numbers (where images may not have any spots)
# and larger numbers between batches.
if len(potential) == 1:
batch_offset = potential[0]
logger.info(
"""
Using a batch offset of %s to split datasets.
Batch offset can be specified with mtz.batch_offset=
""",
batch_offset,
)
elif len(potential) > 1:
diffs = flex.double([
potential[i + 1] - p for i, p in enumerate(potential[:-1])
])
i = flex.sort_permutation(diffs)[-1]
batch_offset = int(potential[i + 1] - (0.2 * diffs[i]))
logger.info(
"""
Using an approximate batch offset of %s to split datasets.
Batch offset can be specified with mtz.batch_offset=
""",
batch_offset,
)
else:
batch_offset = 1
previous = 0
dataset_no = 0
for i, b in enumerate(sorted_batches):
if b - previous > batch_offset - 1:
dataset_no += 1
dataset[i] = dataset_no
previous = b
table["dataset"] = flex.int(table.size(), 0)
table["dataset"].set_selected(sel_perm, dataset)
return table, unit_cell, space_group
avr2.reshape(avr.accessor()) avr = avr2 std2 = stddev.as_1d() std2.set_selected(indices, flex.double(len(indices), 0)) std2.reshape(stddev.accessor()) stddev = std2 iod = flex.double(len(avr)) mask2 = avr2.as_1d() > 0 indices2 = flex.size_t(range(len(mask))).select(mask2) avr2 = avr2.as_1d().select(mask2) var2 = std2.as_1d().select(mask2)**2 iod.set_selected(indices2, var2 / avr2) iod.reshape(avr.accessor()) mask.reshape(avr.accessor()) print flex.max(avr), flex.min(avr) from matplotlib import pylab pylab.imshow(iod.as_numpy_array(), vmax=3, interpolation='none') pylab.colorbar() pylab.show() exit(0) from matplotlib import pylab pylab.imshow(avr.as_numpy_array(), interpolation='none') pylab.colorbar() pylab.show() pylab.hist(avr, bins=100) pylab.show() #avr = mean_filter(avr, mask, (1,1), 2) #print flex.max(avr), flex.min(avr)
def onclick(event): import math ts = event.xdata diffs = flex.abs(t - ts) ts = t[flex.first_index(diffs, flex.min(diffs))] print get_paths_from_timestamps([ts], tag="shot")[0]
def index(self):
experiments = ExperimentList()
had_refinement_error = False
have_similar_crystal_models = False
while True:
if had_refinement_error or have_similar_crystal_models:
break
max_lattices = self.params.multiple_lattice_search.max_lattices
if max_lattices is not None and len(experiments) >= max_lattices:
break
if len(experiments) > 0:
cutoff_fraction = (self.params.multiple_lattice_search.
recycle_unindexed_reflections_cutoff)
d_spacings = 1 / self.reflections["rlp"].norms()
d_min_indexed = flex.min(
d_spacings.select(self.indexed_reflections))
min_reflections_for_indexing = cutoff_fraction * len(
self.reflections.select(d_spacings > d_min_indexed))
crystal_ids = self.reflections.select(
d_spacings > d_min_indexed)["id"]
if (crystal_ids
== -1).count(True) < min_reflections_for_indexing:
logger.info(
"Finish searching for more lattices: %i unindexed reflections remaining."
% ((crystal_ids == -1).count(True)))
break
n_lattices_previous_cycle = len(experiments)
if self.d_min is None:
self.d_min = self.params.refinement_protocol.d_min_start
if len(experiments) == 0:
new_expts = self.find_lattices()
generate_experiment_identifiers(new_expts)
experiments.extend(new_expts)
else:
try:
new = self.find_lattices()
generate_experiment_identifiers(new)
experiments.extend(new)
except DialsIndexError:
logger.info("Indexing remaining reflections failed")
if self.params.refinement_protocol.d_min_step is libtbx.Auto:
n_cycles = self.params.refinement_protocol.n_macro_cycles
if self.d_min is None or n_cycles == 1:
self.params.refinement_protocol.d_min_step = 0
else:
d_spacings = 1 / self.reflections["rlp"].norms()
d_min_all = flex.min(d_spacings)
self.params.refinement_protocol.d_min_step = (
self.d_min - d_min_all) / (n_cycles - 1)
logger.info("Using d_min_step %.1f" %
self.params.refinement_protocol.d_min_step)
if len(experiments) == 0:
raise DialsIndexError("No suitable lattice could be found.")
elif len(experiments) == n_lattices_previous_cycle:
# no more lattices found
break
for i_cycle in range(
self.params.refinement_protocol.n_macro_cycles):
if (i_cycle > 0 and self.d_min is not None
and self.params.refinement_protocol.d_min_step > 0):
d_min = self.d_min - self.params.refinement_protocol.d_min_step
d_min = max(d_min, 0)
if self.params.refinement_protocol.d_min_final is not None:
d_min = max(
d_min, self.params.refinement_protocol.d_min_final)
if d_min >= 0:
self.d_min = d_min
logger.info("Increasing resolution to %.2f Angstrom" %
d_min)
# reset reflection lattice flags
# the lattice a given reflection belongs to: a value of -1 indicates
# that a reflection doesn't belong to any lattice so far
self.reflections["id"] = flex.int(len(self.reflections), -1)
self.index_reflections(experiments, self.reflections)
if i_cycle == 0 and self.params.known_symmetry.space_group is not None:
self._apply_symmetry_post_indexing(
experiments, self.reflections,
n_lattices_previous_cycle)
logger.info("\nIndexed crystal models:")
self.show_experiments(experiments,
self.reflections,
d_min=self.d_min)
if self._check_have_similar_crystal_models(experiments):
have_similar_crystal_models = True
break
logger.info("")
logger.info("#" * 80)
logger.info("Starting refinement (macro-cycle %i)" %
(i_cycle + 1))
logger.info("#" * 80)
logger.info("")
self.indexed_reflections = self.reflections["id"] > -1
sel = flex.bool(len(self.reflections), False)
lengths = 1 / self.reflections["rlp"].norms()
if self.d_min is not None:
isel = (lengths <= self.d_min).iselection()
sel.set_selected(isel, True)
sel.set_selected(self.reflections["id"] == -1, True)
self.reflections.unset_flags(sel,
self.reflections.flags.indexed)
self.unindexed_reflections = self.reflections.select(sel)
reflections_for_refinement = self.reflections.select(
self.indexed_reflections)
if self.params.refinement_protocol.mode == "repredict_only":
refined_experiments, refined_reflections = (
experiments,
reflections_for_refinement,
)
from dials.algorithms.refinement.prediction.managed_predictors import (
ExperimentsPredictorFactory, )
ref_predictor = ExperimentsPredictorFactory.from_experiments(
experiments,
spherical_relp=self.all_params.refinement.
parameterisation.spherical_relp_model,
)
ref_predictor(refined_reflections)
else:
try:
refined_experiments, refined_reflections = self.refine(
experiments, reflections_for_refinement)
except (DialsRefineConfigError,
DialsRefineRuntimeError) as e:
if len(experiments) == 1:
raise DialsIndexRefineError(str(e))
had_refinement_error = True
logger.info("Refinement failed:")
logger.info(e)
del experiments[-1]
break
self._unit_cell_volume_sanity_check(experiments,
refined_experiments)
self.refined_reflections = refined_reflections
self.refined_reflections.unset_flags(
self.refined_reflections["id"] < 0,
self.refined_reflections.flags.indexed,
)
for i, expt in enumerate(self.experiments):
ref_sel = self.refined_reflections.select(
self.refined_reflections["imageset_id"] == i)
ref_sel = ref_sel.select(ref_sel["id"] >= 0)
for i_expt in set(ref_sel["id"]):
refined_expt = refined_experiments[i_expt]
expt.detector = refined_expt.detector
expt.beam = refined_expt.beam
expt.goniometer = refined_expt.goniometer
expt.scan = refined_expt.scan
refined_expt.imageset = expt.imageset
if not (self.all_params.refinement.parameterisation.beam.fix
== "all" and self.all_params.refinement.
parameterisation.detector.fix == "all"):
# Experimental geometry may have changed - re-map centroids to
# reciprocal space
self.reflections.map_centroids_to_reciprocal_space(
self.experiments)
# update for next cycle
experiments = refined_experiments
self.refined_experiments = refined_experiments
logger.info("\nRefined crystal models:")
self.show_experiments(self.refined_experiments,
self.reflections,
d_min=self.d_min)
if (i_cycle >= 2 and self.d_min
== self.params.refinement_protocol.d_min_final):
logger.info(
"Target d_min_final reached: finished with refinement")
break
if self.refined_experiments is None:
raise DialsIndexRefineError(
"None of the experiments could refine.")
if len(self.refined_experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices import (
rotation_matrix_differences, )
logger.info(
rotation_matrix_differences(
self.refined_experiments.crystals()))
self._xyzcal_mm_to_px(self.experiments, self.refined_reflections)
def __call__(self):
"""Determine optimal mosaicity and domain size model (monochromatic)"""
RR = self.refinery.predict_for_reflection_table(self.reflections)
excursion_rad = RR["delpsical.rad"]
delta_psi_deg = excursion_rad * 180./math.pi
print
print flex.max(delta_psi_deg), flex.min(delta_psi_deg)
mean_excursion = flex.mean(delta_psi_deg)
print "The mean excursion is %7.3f degrees, r.m.s.d %7.3f"%(mean_excursion, math.sqrt(flex.mean(RR["delpsical2"])))
crystal = self.experiments[0].crystal
beam = self.experiments[0].beam
miller_indices = self.reflections["miller_index"]
# FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection
two_thetas = crystal.get_unit_cell().two_theta(miller_indices,beam.get_wavelength(),deg=True)
dspacings = crystal.get_unit_cell().d(miller_indices)
dspace_sq = dspacings * dspacings
# First -- try to get a reasonable envelope for the observed excursions.
## minimum of three regions; maximum of 50 measurements in each bin
print "fitting parameters on %d spots"%len(excursion_rad)
n_bins = min(max(3, len(excursion_rad)//25),50)
bin_sz = len(excursion_rad)//n_bins
print "nbins",n_bins,"bin_sz",bin_sz
order = flex.sort_permutation(two_thetas)
two_thetas_env = flex.double()
dspacings_env = flex.double()
excursion_rads_env = flex.double()
for x in xrange(0,n_bins):
subset = order[x*bin_sz:(x+1)*bin_sz]
two_thetas_env.append(flex.mean(two_thetas.select(subset)))
dspacings_env.append(flex.mean(dspacings.select(subset)))
excursion_rads_env.append(flex.max(flex.abs(excursion_rad.select(subset))))
# Second -- parameter fit
## solve the normal equations
sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
sum_inv_u = flex.sum(dspacings_env)
sum_te_u = flex.sum(dspacings_env * excursion_rads_env)
sum_te = flex.sum(excursion_rads_env)
Normal_Mat = sqr((sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
Vector = col((sum_te_u, sum_te))
solution = Normal_Mat.inverse() * Vector
s_ang = 1./(2*solution[0])
print "Best LSQ fit Scheerer domain size is %9.2f ang"%(
s_ang)
tan_phi_rad = dspacings / (2. * s_ang)
tan_phi_deg = tan_phi_rad * 180./math.pi
k_degrees = solution[1]* 180./math.pi
print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f"%(2*k_degrees, k_degrees)
tan_outer_deg = tan_phi_deg + k_degrees
from xfel.mono_simulation.max_like import minimizer
# coerce the estimates to be positive for max-likelihood
lower_limit_domain_size = math.pow(crystal.get_unit_cell().volume(),
1./3.)*3 # params.refinement.domain_size_lower_limit
d_estimate = max(s_ang, lower_limit_domain_size)
M = minimizer(d_i = dspacings, psi_i = excursion_rad, eta_rad = abs(2. * solution[1]),
Deff = d_estimate)
print "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas))
tan_phi_rad_ML = dspacings / (2. / M.x[0])
tan_phi_deg_ML = tan_phi_rad_ML * 180./math.pi
tan_outer_deg_ML = tan_phi_deg_ML + 0.5*M.x[1]*180./math.pi
self.nv_acceptance_flags = flex.abs(delta_psi_deg) < tan_outer_deg_ML
if self.graph_verbose: #params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit
AD1TF7B_MAX2T = 30.
AD1TF7B_MAXDP = 1.
from matplotlib import pyplot as plt
plt.plot(two_thetas, delta_psi_deg, "bo")
minplot = flex.min(two_thetas)
plt.plot([0,minplot],[mean_excursion,mean_excursion],"k-")
LR = flex.linear_regression(two_thetas, delta_psi_deg)
model_y = LR.slope()*two_thetas + LR.y_intercept()
plt.plot(two_thetas, model_y, "k-")
plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas)))
plt.plot(two_thetas, tan_phi_deg_ML, "r.")
plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
plt.plot(two_thetas, tan_outer_deg_ML, "g.")
plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
plt.xlim([0,AD1TF7B_MAX2T])
plt.ylim([-AD1TF7B_MAXDP,AD1TF7B_MAXDP])
plt.show()
plt.close()
from xfel.mono_simulation.util import green_curve_area
self.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML)
print "The green curve area is ", self.green_curve_area
crystal._ML_half_mosaicity_deg = M.x[1]*180./(2.*math.pi)
crystal._ML_domain_size_ang = 2./M.x[0]
self._ML_full_mosaicity_rad = M.x[1]
self._ML_domain_size_ang = 2./M.x[0]
#params.refinement.mosaic.model_expansion_factor
"""The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes
as close as possible to # of observed reflections input, in the last integration call. Determine this by
inspecting the output log file interactively. Do not exceed the bare minimum threshold needed.
The intention is to find an optimal value, global for a given dataset."""
model_expansion_factor = 1.4
crystal._ML_half_mosaicity_deg *= model_expansion_factor
crystal._ML_domain_size_ang /= model_expansion_factor
return crystal
def finalize(self, data, mask):
'''
Finalize the model
:param data: The data array
:param mask: The mask array
'''
from dials.algorithms.image.filter import median_filter, mean_filter
from dials.algorithms.image.fill_holes import diffusion_fill
from dials.algorithms.image.fill_holes import simple_fill
from dials.array_family import flex
# Print some image properties
sub_data = data.as_1d().select(mask.as_1d())
logger.info('Raw image statistics:')
logger.info(' min: %d' % int(flex.min(sub_data)))
logger.info(' max: %d' % int(flex.max(sub_data)))
logger.info(' mean: %d' % int(flex.mean(sub_data)))
logger.info('')
# Transform to polar
logger.info('Transforming image data to polar grid')
result = self.transform.to_polar(data, mask)
data = result.data()
mask = result.mask()
sub_data = data.as_1d().select(mask.as_1d())
logger.info('Polar image statistics:')
logger.info(' min: %d' % int(flex.min(sub_data)))
logger.info(' max: %d' % int(flex.max(sub_data)))
logger.info(' mean: %d' % int(flex.mean(sub_data)))
logger.info('')
# Filter the image to remove noise
if self.kernel_size > 0:
if self.filter_type == 'median':
logger.info('Applying median filter')
data = median_filter(data, mask, (self.kernel_size, 0))
sub_data = data.as_1d().select(mask.as_1d())
logger.info('Median polar image statistics:')
logger.info(' min: %d' % int(flex.min(sub_data)))
logger.info(' max: %d' % int(flex.max(sub_data)))
logger.info(' mean: %d' % int(flex.mean(sub_data)))
logger.info('')
elif self.filter_type == 'mean':
logger.info('Applying mean filter')
mask_as_int = mask.as_1d().as_int()
mask_as_int.reshape(mask.accessor())
data = mean_filter(data, mask_as_int, (self.kernel_size, 0), 1)
sub_data = data.as_1d().select(mask.as_1d())
logger.info('Mean polar image statistics:')
logger.info(' min: %d' % int(flex.min(sub_data)))
logger.info(' max: %d' % int(flex.max(sub_data)))
logger.info(' mean: %d' % int(flex.mean(sub_data)))
logger.info('')
else:
raise RuntimeError('Unknown filter_type: %s' % self.filter_type)
# Fill any remaining holes
logger.info("Filling holes")
data = simple_fill(data, mask)
data = diffusion_fill(data, mask, self.niter)
mask = flex.bool(data.accessor(), True)
sub_data = data.as_1d().select(mask.as_1d())
logger.info('Filled polar image statistics:')
logger.info(' min: %d' % int(flex.min(sub_data)))
logger.info(' max: %d' % int(flex.max(sub_data)))
logger.info(' mean: %d' % int(flex.mean(sub_data)))
logger.info('')
# Transform back
logger.info('Transforming image data from polar grid')
result = self.transform.from_polar(data, mask)
data = result.data()
mask = result.mask()
sub_data = data.as_1d().select(mask.as_1d())
logger.info('Final image statistics:')
logger.info(' min: %d' % int(flex.min(sub_data)))
logger.info(' max: %d' % int(flex.max(sub_data)))
logger.info(' mean: %d' % int(flex.mean(sub_data)))
logger.info('')
# Fill in any discontinuities
# FIXME NEED TO HANDLE DISCONTINUITY
# mask = ~self.transform.discontinuity()[:-1,:-1]
# data = diffusion_fill(data, mask, self.niter)
# Get and apply the mask
mask = self.experiment.imageset.get_mask(0)[0]
mask = mask.as_1d().as_int().as_double()
mask.reshape(data.accessor())
data *= mask
# Return the result
return data
def _get_gradients_core(self, reflections, D, s0, U, B, axis, fixed_rotation, callback=None):
"""Calculate gradients of the prediction formula with respect to
each of the parameters of the contained models, for reflection h
that reflects at rotation angle phi with scattering vector s that
intersects panel panel_id. That is, calculate dX/dp, dY/dp and
dphi/dp"""
# Spindle rotation matrices for every reflection
#R = self._axis.axis_and_angle_as_r3_rotation_matrix(phi)
#R = flex.mat3_double(len(reflections))
# NB for now use flex.vec3_double.rotate_around_origin each time I need the
# rotation matrix R.
self._axis = axis
self._fixed_rotation = fixed_rotation
self._s0 = s0
# pv is the 'projection vector' for the ray along s1.
self._D = D
self._s1 = reflections['s1']
self._pv = D * self._s1
# also need quantities derived from pv, precalculated for efficiency
u, v, w = self._pv.parts()
self._w_inv = 1/w
self._u_w_inv = u * self._w_inv
self._v_w_inv = v * self._w_inv
self._UB = U * B
self._U = U
self._B = B
# r is the reciprocal lattice vector, in the lab frame
self._h = reflections['miller_index'].as_vec3_double()
self._phi_calc = reflections['xyzcal.mm'].parts()[2]
self._r = (self._fixed_rotation * (self._UB * self._h)).rotate_around_origin(self._axis, self._phi_calc)
# All of the derivatives of phi have a common denominator, given by
# (e X r).s0, where e is the rotation axis. Calculate this once, here.
self._e_X_r = self._axis.cross(self._r)
self._e_r_s0 = (self._e_X_r).dot(self._s0)
# Note that e_r_s0 -> 0 when the rotation axis, beam vector and
# relp are coplanar. This occurs when a reflection just touches
# the Ewald sphere.
#
# There is a relationship between e_r_s0 and zeta_factor.
# Uncommenting the code below shows that
# s0.(e X r) = zeta * |s X s0|
#from dials.algorithms.profile_model.gaussian_rs import zeta_factor
#from libtbx.test_utils import approx_equal
#s = matrix.col(reflections['s1'][0])
#z = zeta_factor(axis[0], s0[0], s)
#ss0 = (s.cross(matrix.col(s0[0]))).length()
#assert approx_equal(e_r_s0[0], z * ss0)
# catch small values of e_r_s0
e_r_s0_mag = flex.abs(self._e_r_s0)
try:
assert flex.min(e_r_s0_mag) > 1.e-6
except AssertionError as e:
imin = flex.min_index(e_r_s0_mag)
print "(e X r).s0 too small:"
print "for", (e_r_s0_mag <= 1.e-6).count(True), "reflections"
print "out of", len(e_r_s0_mag), "total"
print "such as", reflections['miller_index'][imin]
print "with scattering vector", reflections['s1'][imin]
print "where r =", self._r[imin]
print "e =", self._axis[imin]
print "s0 =", self._s0[imin]
print ("this reflection forms angle with the equatorial plane "
"normal:")
vecn = matrix.col(self._s0[imin]).cross(matrix.col(self._axis[imin])).normalize()
print matrix.col(reflections['s1'][imin]).accute_angle(vecn)
raise e
# Set up empty list in which to store gradients
m = len(reflections)
results = []
# determine experiment to indices mappings once, here
experiment_to_idx = []
for iexp, exp in enumerate(self._experiments):
sel = reflections['id'] == iexp
isel = sel.iselection()
experiment_to_idx.append(isel)
# reset a pointer to the parameter number
self._iparam = 0
### Work through the parameterisations, calculating their contributions
### to derivatives d[pv]/dp and d[phi]/dp
# loop over the detector parameterisations
for dp in self._detector_parameterisations:
# Determine (sub)set of reflections affected by this parameterisation
isel = flex.size_t()
for exp_id in dp.get_experiment_ids():
isel.extend(experiment_to_idx[exp_id])
# Access the detector model being parameterised
detector = dp.get_model()
# Get panel numbers of the affected reflections
panel = reflections['panel'].select(isel)
# Extend derivative vectors for this detector parameterisation
results = self._extend_gradient_vectors(results, m, dp.num_free(),
keys=self._grad_names)
# loop through the panels in this detector
for panel_id, _ in enumerate(exp.detector):
# get the right subset of array indices to set for this panel
sub_isel = isel.select(panel == panel_id)
if len(sub_isel) == 0:
# if no reflections intersect this panel, skip calculation
continue
sub_pv = self._pv.select(sub_isel)
sub_D = self._D.select(sub_isel)
dpv_ddet_p = self._detector_derivatives(dp, sub_pv, sub_D, panel_id)
# convert to dX/dp, dY/dp and assign the elements of the vectors
# corresponding to this experiment and panel
sub_w_inv = self._w_inv.select(sub_isel)
sub_u_w_inv = self._u_w_inv.select(sub_isel)
sub_v_w_inv = self._v_w_inv.select(sub_isel)
dX_ddet_p, dY_ddet_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp(
sub_w_inv, sub_u_w_inv, sub_v_w_inv, dpv_ddet_p)
# use a local parameter index pointer because we set all derivatives
# for this panel before moving on to the next
iparam = self._iparam
for dX, dY in zip(dX_ddet_p, dY_ddet_p):
if dX is not None:
results[iparam]['dX_dp'].set_selected(sub_isel, dX)
if dY is not None:
results[iparam]['dY_dp'].set_selected(sub_isel, dY)
# increment the local parameter index pointer
iparam += 1
if callback is not None:
iparam = self._iparam
for i in range(dp.num_free()):
results[iparam] = callback(results[iparam])
iparam += 1
# increment the parameter index pointer to the last detector parameter
self._iparam += dp.num_free()
# loop over the beam parameterisations
for bp in self._beam_parameterisations:
# Determine (sub)set of reflections affected by this parameterisation
isel = flex.size_t()
for exp_id in bp.get_experiment_ids():
isel.extend(experiment_to_idx[exp_id])
# Extend derivative vectors for this beam parameterisation
results = self._extend_gradient_vectors(results, m, bp.num_free(),
keys=self._grad_names)
if len(isel) == 0:
# if no reflections are in this experiment, skip calculation
self._iparam += bp.num_free()
continue
# Get required data from those reflections
r = self._r.select(isel)
e_X_r = self._e_X_r.select(isel)
e_r_s0 = self._e_r_s0.select(isel)
D = self._D.select(isel)
w_inv = self._w_inv.select(isel)
u_w_inv = self._u_w_inv.select(isel)
v_w_inv = self._v_w_inv.select(isel)
dpv_dbeam_p, dphi_dbeam_p = self._beam_derivatives(bp, r, e_X_r, e_r_s0, D)
# convert to dX/dp, dY/dp and assign the elements of the vectors
# corresponding to this experiment
dX_dbeam_p, dY_dbeam_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp(
w_inv, u_w_inv, v_w_inv, dpv_dbeam_p)
for dX, dY, dphi in zip(dX_dbeam_p, dY_dbeam_p, dphi_dbeam_p):
results[self._iparam][self._grad_names[0]].set_selected(isel, dX)
results[self._iparam][self._grad_names[1]].set_selected(isel, dY)
results[self._iparam][self._grad_names[2]].set_selected(isel, dphi)
if callback is not None:
results[self._iparam] = callback(results[self._iparam])
# increment the parameter index pointer
self._iparam += 1
# loop over the crystal orientation parameterisations
for xlop in self._xl_orientation_parameterisations:
# Determine (sub)set of reflections affected by this parameterisation
isel = flex.size_t()
for exp_id in xlop.get_experiment_ids():
isel.extend(experiment_to_idx[exp_id])
# Extend derivative vectors for this crystal orientation parameterisation
results = self._extend_gradient_vectors(results, m, xlop.num_free(),
keys=self._grad_names)
if len(isel) == 0:
# if no reflections are in this experiment, skip calculation
self._iparam += xlop.num_free()
continue
# Get required data from those reflections
axis = self._axis.select(isel)
fixed_rotation = self._fixed_rotation.select(isel)
phi_calc = self._phi_calc.select(isel)
h = self._h.select(isel)
s1 = self._s1.select(isel)
e_X_r = self._e_X_r.select(isel)
e_r_s0 = self._e_r_s0.select(isel)
B = self._B.select(isel)
D = self._D.select(isel)
w_inv = self._w_inv.select(isel)
u_w_inv = self._u_w_inv.select(isel)
v_w_inv = self._v_w_inv.select(isel)
# get derivatives of the U matrix wrt the parameters
dU_dxlo_p = [reflections["dU_dp{0}".format(i)].select(isel) \
for i in range(xlop.num_free())]
dpv_dxlo_p, dphi_dxlo_p = self._xl_orientation_derivatives(
dU_dxlo_p, axis, fixed_rotation, phi_calc, h, s1, e_X_r, e_r_s0, B, D)
# convert to dX/dp, dY/dp and assign the elements of the vectors
# corresponding to this experiment
dX_dxlo_p, dY_dxlo_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp(
w_inv, u_w_inv, v_w_inv, dpv_dxlo_p)
for dX, dY, dphi in zip(dX_dxlo_p, dY_dxlo_p, dphi_dxlo_p):
results[self._iparam][self._grad_names[0]].set_selected(isel, dX)
results[self._iparam][self._grad_names[1]].set_selected(isel, dY)
results[self._iparam][self._grad_names[2]].set_selected(isel, dphi)
if callback is not None:
results[self._iparam] = callback(results[self._iparam])
# increment the parameter index pointer
self._iparam += 1
# loop over the crystal unit cell parameterisations
for xlucp in self._xl_unit_cell_parameterisations:
# Determine (sub)set of reflections affected by this parameterisation
isel = flex.size_t()
for exp_id in xlucp.get_experiment_ids():
isel.extend(experiment_to_idx[exp_id])
# Extend derivative vectors for this crystal unit cell parameterisation
results = self._extend_gradient_vectors(results, m, xlucp.num_free(),
keys=self._grad_names)
if len(isel) == 0:
# if no reflections are in this experiment, skip calculation
self._iparam += xlucp.num_free()
continue
# Get required data from those reflections
axis = self._axis.select(isel)
fixed_rotation = self._fixed_rotation.select(isel)
phi_calc = self._phi_calc.select(isel)
h = self._h.select(isel)
s1 = self._s1.select(isel)
e_X_r = self._e_X_r.select(isel)
e_r_s0 = self._e_r_s0.select(isel)
U = self._U.select(isel)
D = self._D.select(isel)
w_inv = self._w_inv.select(isel)
u_w_inv = self._u_w_inv.select(isel)
v_w_inv = self._v_w_inv.select(isel)
dB_dxluc_p = [reflections["dB_dp{0}".format(i)].select(isel) \
for i in range(xlucp.num_free())]
dpv_dxluc_p, dphi_dxluc_p = self._xl_unit_cell_derivatives(
dB_dxluc_p, axis, fixed_rotation, phi_calc, h, s1, e_X_r, e_r_s0, U, D)
# convert to dX/dp, dY/dp and assign the elements of the vectors
# corresponding to this experiment
dX_dxluc_p, dY_dxluc_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp(
w_inv, u_w_inv, v_w_inv, dpv_dxluc_p)
for dX, dY, dphi in zip(dX_dxluc_p, dY_dxluc_p, dphi_dxluc_p):
results[self._iparam][self._grad_names[0]].set_selected(isel, dX)
results[self._iparam][self._grad_names[1]].set_selected(isel, dY)
results[self._iparam][self._grad_names[2]].set_selected(isel, dphi)
if callback is not None:
results[self._iparam] = callback(results[self._iparam])
# increment the parameter index pointer
self._iparam += 1
return results
integrated_data['hash'] = hash_value
return integrated_data
if __name__ == '__main__':
import sys
if len(sys.argv) != 2:
raise RuntimeError, '%s strong.pickle' % sys.argv[0]
import cPickle as pickle
from dials.array_family import flex
integrated_data = pickle.load(open(sys.argv[1], 'rb'))
# keep only flagged as integrated reflections
sel = integrated_data.get_flags(integrated_data.flags.integrated)
integrated_data = integrated_data.select(sel)
# only fully recorded reflections
sel = integrated_data['partiality'] > 0.99
integrated_data = integrated_data.select(sel)
integrated_data = add_hash(integrated_data)
hash_value = integrated_data['hash']
for h in hash_value:
sel = hash_value == h
assert(sel.count(True) == 1)
print flex.min(hash_value), flex.max(hash_value)
def paper_test(B, S):
from numpy.random import poisson
from math import exp
background_shape = [1 for i in range(20)]
signal_shape = [1 if i >= 6 and i < 15 else 0 for i in range(20)]
background = [poisson(bb * B,1)[0] for bb in background_shape]
signal = [poisson(ss * S, 1)[0] for ss in signal_shape]
# background = [bb * B for bb in background_shape]
# signal = [ss * S for ss in signal_shape]
total = [b + s for b, s in zip(background, signal)]
# from matplotlib import pylab
# pylab.plot(total)
# pylab.plot(signal)
# pylab.plot(background)
# pylab.show()
total = [0, 1, 0, 0, 0, 0, 3, 1, 3, 3, 6, 6, 4, 1, 4, 0, 2, 0, 1, 1]
total = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]
# plot_prob_for_zero(total, background_shape, signal_shape)
# signal_shape = [exp(-(x - 10.0)**2 / (2*3.0**2)) for x in range(20)]
# signal_shape = [ss / sum(signal_shape) for ss in signal_shape]
# print signal_shape
#plot_valid(background_shape, signal_shape)
B = 155.0 / 296.0
from dials.array_family import flex
from math import log, factorial
V = flex.double(flex.grid(100, 100))
L = flex.double(flex.grid(100, 100))
DB = flex.double(flex.grid(100, 100))
DS = flex.double(flex.grid(100, 100))
P = flex.double(flex.grid(100, 100))
Fb = sum(background_shape)
Fs = sum(signal_shape)
SV = []
MASK = flex.bool(flex.grid(100, 100), False)
for BB in range(100):
for SS in range(100):
B = -5.0 + (BB) / 10.0
S = -5.0 + (SS) / 10.0
# SV.append(S)
VV = 0
LL = 0
DDB = 0
DDS = 0
for i in range(20):
s = signal_shape[i]
b = background_shape[i]
c = total[i]
if B*b + S*s <= 0:
# MASK[BB, SS] = True
LL = 0
if b == 0:
DDB += 0
else:
DDB += 1e7
if s == 0:
DDS += 0
else:
DDS += 1e7
# break
else:
# VV += (b + s)*c / (B*b + S*s)
# LL += c*log(B*b+S*s) - log(factorial(c)) - B*b - S*s
DDB += c*b/(B*b+S*s) - b
DDS += c*s/(B*b+S*s) - s
VV -= (Fb + Fs)
# print B, S, VV
# V[BB, SS] = abs(VV)
L[BB, SS] = LL
DB[BB,SS] = DDB
DS[BB,SS] = DDS
max_ind = flex.max_index(L)
j = max_ind // 100
i = max_ind % 100
print "Approx: ", (j+1) / 20.0, (i+1) / 20.0
print "Min/Max DB: ", flex.min(DB), flex.max(DB)
print "Min/Max DS: ", flex.min(DS), flex.max(DS)
from matplotlib import pylab
# pylab.imshow(flex.log(V).as_numpy_array(), extent=[0.05, 5.05, 5.05, 0.05])
# pylab.plot(SV, V)
# pylab.plot(SV, [0] * 100)
# pylab.show()
im = flex.exp(L).as_numpy_array()
import numpy
# im = numpy.ma.masked_array(im, mask=MASK.as_numpy_array())
# pylab.imshow(im)#, extent=[-5.0, 5.0, 5.0, -5.0], origin='lower')
pylab.imshow(DB.as_numpy_array(), vmin=-100, vmax=100)#, extent=[-5.0, 5.0, 5.0, -5.0], origin='lower')
pylab.contour(DB.as_numpy_array(), levels=[0], colors=['red'])
pylab.contour(DS.as_numpy_array(), levels=[0], colors=['black'])
pylab.show()
# im = numpy.ma.masked_array(DB.as_numpy_array(), mask=MASK.as_numpy_array())
# pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100)
# pylab.show()
# im = numpy.ma.masked_array(DS.as_numpy_array(), mask=MASK.as_numpy_array())
# pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100)
# pylab.show()
# exit(0)
S1, B1 = value(total, background_shape, signal_shape)
exit(0)
try:
S1, B1 = value(total, background_shape, signal_shape)
print "Result:"
print S1, B1
exit(0)
except Exception, e:
raise e
import sys
import traceback
traceback.print_exc()
from dials.array_family import flex
Fs = sum(signal_shape)
Fb = sum(background_shape)
Rs = flex.double(flex.grid(100, 100))
print "-----"
print B, S
# from matplotlib import pylab
# pylab.plot(total)
# pylab.show()
print background_shape
print signal_shape
print total
from math import exp, factorial, log
minx = -1
miny = -1
minr = 9999
for BB in range(0, 100):
for SS in range(0, 100):
B = -10 + (BB) / 5.0
S = -10 + (SS) / 5.0
L = 0
Fb2 = 0
Fs2 = 0
for i in range(len(total)):
c = total[i]
b = background_shape[i]
s = signal_shape[i]
# P = exp(-(B*b + S*s)) * (B*b+S*s)**c / factorial(c)
# print P
# if P > 0:
# L += log(P)
den = B*b + S*s
num1 = b*c
num2 = s*c
if den != 0:
Fb2 += num1 / den
Fs2 += num2 / den
R = (Fb2 - Fb)**2 + (Fs2 - Fs)**2
if R > 1000:
R = 0
# Rs[BB,SS] = L#R#Fs2 - Fs
Rs[BB,SS] = R#Fs2 - Fs
from matplotlib import pylab
pylab.imshow(flex.log(Rs).as_numpy_array(), extent=[-5,5,5,-5])
pylab.show()
exit(0)
def blank_integrated_analysis(reflections, scan, phi_step, fractional_loss):
prf_sel = reflections.get_flags(reflections.flags.integrated_prf)
if prf_sel.count(True) > 0:
reflections = reflections.select(prf_sel)
intensities = reflections["intensity.prf.value"]
variances = reflections["intensity.prf.variance"]
else:
sum_sel = reflections.get_flags(reflections.flags.integrated_sum)
reflections = reflections.select(sum_sel)
intensities = reflections["intensity.sum.value"]
variances = reflections["intensity.sum.variance"]
i_sigi = intensities / flex.sqrt(variances)
xyz_px = reflections["xyzobs.px.value"]
x_px, y_px, z_px = xyz_px.parts()
phi = scan.get_angle_from_array_index(z_px)
osc = scan.get_oscillation()[1]
n_images_per_step = iceil(phi_step / osc)
phi_step = n_images_per_step * osc
array_range = scan.get_array_range()
phi_min = flex.min(phi)
phi_max = flex.max(phi)
n_steps = int(round((phi_max - phi_min) / phi_step))
hist = flex.histogram(z_px,
data_min=array_range[0],
data_max=array_range[1],
n_slots=n_steps)
logger.debug("Histogram:")
logger.debug(hist.as_str())
mean_i_sigi = flex.double()
for i, slot_info in enumerate(hist.slot_infos()):
sel = (z_px >= slot_info.low_cutoff) & (z_px < slot_info.high_cutoff)
if sel.count(True) == 0:
mean_i_sigi.append(0)
else:
mean_i_sigi.append(flex.mean(i_sigi.select(sel)))
potential_blank_sel = mean_i_sigi <= (fractional_loss *
flex.max(mean_i_sigi))
xmin, xmax = zip(*[(slot_info.low_cutoff, slot_info.high_cutoff)
for slot_info in hist.slot_infos()])
d = {
"data": [{
"x": list(hist.slot_centers()),
"y": list(mean_i_sigi),
"xlow": xmin,
"xhigh": xmax,
"blank": list(potential_blank_sel),
"type": "bar",
"name": "blank_counts_analysis",
}],
"layout": {
"xaxis": {
"title": "z observed (images)"
},
"yaxis": {
"title": "Number of reflections"
},
"bargap": 0,
},
}
blank_regions = blank_regions_from_sel(d["data"][0])
d["blank_regions"] = blank_regions
return d
def compose(self, reflections, skip_derivatives=False):
"""Compose scan-varying crystal parameterisations at the specified image
number, for the specified experiment, for each image. Put the U, B and
UB matrices in the reflection table, and cache the derivatives."""
self._prepare_for_compose(reflections, skip_derivatives)
for iexp, exp in enumerate(self._experiments):
# select the reflections of interest
sel = reflections['id'] == iexp
isel = sel.iselection()
blocks = reflections['block'].select(isel)
# identify which parameterisations to use for this experiment
xl_op = self._get_xl_orientation_parameterisation(iexp)
xl_ucp = self._get_xl_unit_cell_parameterisation(iexp)
bp = self._get_beam_parameterisation(iexp)
dp = self._get_detector_parameterisation(iexp)
# reset current frame cache for scan-varying parameterisations
self._current_frame = {}
# get state and derivatives for each block
for block in xrange(flex.min(blocks),
flex.max(blocks) + 1):
# determine the subset of reflections this affects
subsel = isel.select(blocks == block)
if len(subsel) == 0: continue
# get the panels hit by these reflections
panels = reflections['panel'].select(subsel)
# get the integer frame number nearest the centre of that block
frames = reflections['block_centre'].select(subsel)
# can only be false if original block assignment has gone wrong
assert frames.all_eq(frames[0]), \
"Failing: a block contains reflections that shouldn't be there"
frame = int(floor(frames[0]))
# model states at current frame
U = self._get_state_from_parameterisation(xl_op, frame)
if U is None: U = exp.crystal.get_U()
B = self._get_state_from_parameterisation(xl_ucp, frame)
if B is None: B = exp.crystal.get_B()
s0 = self._get_state_from_parameterisation(bp, frame)
if s0 is None: s0 = exp.beam.get_s0()
# set states for crystal and beam
reflections['u_matrix'].set_selected(subsel, U.elems)
reflections['b_matrix'].set_selected(subsel, B.elems)
reflections['s0_vector'].set_selected(subsel, s0.elems)
# set states and derivatives for multi-panel detector
if dp is not None and dp.is_multi_state():
# loop through the panels in this detector
for panel_id, _ in enumerate(exp.detector):
# get the right subset of array indices to set for this panel
subsel2 = subsel.select(panels == panel_id)
if len(subsel2) == 0:
# if no reflections intersect this panel, skip calculation
continue
dmat = self._get_state_from_parameterisation(dp,
frame, multi_state_elt=panel_id)
if dmat is None: dmat = exp.detector[panel_id].get_d_matrix()
Dmat = exp.detector[panel_id].get_D_matrix()
reflections['d_matrix'].set_selected(subsel2, dmat)
reflections['D_matrix'].set_selected(subsel2, Dmat)
if dp is not None and self._varying_detectors and not skip_derivatives:
for j, dd in enumerate(dp.get_ds_dp(multi_state_elt=panel_id,
use_none_as_null=True)):
if dd is None: continue
colname = "dd_dp{0}".format(j)
reflections[colname].set_selected(subsel, dd)
else: # set states and derivatives for single panel detector
dmat = self._get_state_from_parameterisation(dp, frame)
if dmat is None: dmat = exp.detector[0].get_d_matrix()
Dmat = exp.detector[0].get_D_matrix()
reflections['d_matrix'].set_selected(subsel, dmat)
reflections['D_matrix'].set_selected(subsel, Dmat)
if dp is not None and self._varying_detectors and not skip_derivatives:
for j, dd in enumerate(dp.get_ds_dp(use_none_as_null=True)):
if dd is None: continue
colname = "dd_dp{0}".format(j)
reflections[colname].set_selected(subsel, dd)
# set derivatives of the states for crystal and beam
if not skip_derivatives:
if xl_op is not None and self._varying_xl_orientations:
for j, dU in enumerate(xl_op.get_ds_dp(use_none_as_null=True)):
if dU is None: continue
colname = "dU_dp{0}".format(j)
reflections[colname].set_selected(subsel, dU)
if xl_ucp is not None and self._varying_xl_unit_cells:
for j, dB in enumerate(xl_ucp.get_ds_dp(use_none_as_null=True)):
if dB is None: continue
colname = "dB_dp{0}".format(j)
reflections[colname].set_selected(subsel, dB)
if bp is not None and self._varying_beams:
for j, ds0 in enumerate(bp.get_ds_dp(use_none_as_null=True)):
if ds0 is None: continue
colname = "ds0_dp{0}".format(j)
reflections[colname].set_selected(subsel, ds0)
# set the UB matrices for prediction
reflections['ub_matrix'] = reflections['u_matrix'] * reflections['b_matrix']
return
def centroid_mean_diff_vs_phi(self, rlist, threshold):
from os.path import join
import math
I = rlist['intensity.sum.value']
I_sig = flex.sqrt(rlist['intensity.sum.variance'])
I_over_S = I / I_sig
mask = I_over_S > threshold
rlist = rlist.select(mask)
assert(len(rlist) > 0)
xc, yc, zc = rlist['xyzcal.mm'].parts()
xo, yo, zo = rlist['xyzobs.mm.value'].parts()
dx = xc - xo
dy = yc - yo
dphi = (zc - zo) * RAD2DEG
mean_residuals_x = flex.double()
mean_residuals_y = flex.double()
mean_residuals_phi = flex.double()
rmsd_x = flex.double()
rmsd_y = flex.double()
rmsd_phi = flex.double()
frame = []
phi_obs_deg = RAD2DEG * zo
phi = []
for i_phi in range(int(math.floor(flex.min(phi_obs_deg))),
int(math.ceil(flex.max(phi_obs_deg)))):
sel = (phi_obs_deg >= i_phi) & (phi_obs_deg < (i_phi+1))
if sel.count(True) == 0:
continue
mean_residuals_x.append(flex.mean(dx.select(sel)))
mean_residuals_y.append(flex.mean(dy.select(sel)))
mean_residuals_phi.append(flex.mean(dphi.select(sel)))
rmsd_x.append(math.sqrt(flex.mean_sq(dx.select(sel))))
rmsd_y.append(math.sqrt(flex.mean_sq(dy.select(sel))))
rmsd_phi.append(math.sqrt(flex.mean_sq(dphi.select(sel))))
phi.append(i_phi)
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(311)
#fig.subplots_adjust(hspace=0.5)
pyplot.axhline(0, color='grey')
ax.scatter(phi, mean_residuals_x)
ax.set_xlabel('phi (deg)')
ax.set_ylabel('mean $\Delta$ x (mm)')
ax = fig.add_subplot(312)
pyplot.axhline(0, color='grey')
ax.scatter(phi, mean_residuals_y)
ax.set_xlabel('phi (deg)')
ax.set_ylabel('mean $\Delta$ y (mm)')
ax = fig.add_subplot(313)
pyplot.axhline(0, color='grey')
ax.scatter(phi, mean_residuals_phi)
ax.set_xlabel('phi (deg)')
ax.set_ylabel('mean $\Delta$ phi (deg)')
pyplot.savefig(join(self.directory, "centroid_mean_diff_vs_phi.png"))
pyplot.close()
fig = pyplot.figure()
ax = fig.add_subplot(311)
#fig.subplots_adjust(hspace=0.5)
pyplot.axhline(flex.mean(rmsd_x), color='grey')
ax.scatter(phi, rmsd_x)
ax.set_xlabel('phi (deg)')
ax.set_ylabel('rmsd x (mm)')
ax = fig.add_subplot(312)
pyplot.axhline(flex.mean(rmsd_y), color='grey')
ax.scatter(phi, rmsd_y)
ax.set_xlabel('phi (deg)')
ax.set_ylabel('rmsd y (mm)')
ax = fig.add_subplot(313)
pyplot.axhline(flex.mean(rmsd_phi), color='grey')
ax.scatter(phi, rmsd_phi)
ax.set_xlabel('phi (deg)')
ax.set_ylabel('rmsd phi (deg)')
pyplot.savefig(join(self.directory, "centroid_rmsd_vs_phi.png"))
pyplot.close()
exit(0)
from dials.array_family import flex
im1 = flex.double(flex.grid(100, 100))
im2 = flex.double(flex.grid(100, 100))
for j in range(100):
for i in range(100):
t1 = -90.0 + i
t2 = -90.0 + j
d1 = dt1(c, b, s, t1, t2, mu0, mu1, mu2)
d2 = dt2(c, b, s, t1, t2, mu0, mu1, mu2)
im1[j,i] = d1
im2[j,i] = d2
print flex.min(im1), flex.max(im1)
print flex.min(im2), flex.max(im2)
from matplotlib import pylab
pylab.imshow(im1.as_numpy_array())
pylab.contour(im1.as_numpy_array(), levels=[0])
pylab.contour(im2.as_numpy_array(), levels=[0])
pylab.show()
# print mu0, mu1, mu2
# b = [bb / sum(b) for bb in b]
# s = [ss / sum(s) for ss in s]
# K1 = 0
# K2 = 9.0 / 20.0
def __init__(self, reflections, av_callback=flex.mean):
# flags to indicate at what level the analysis has been performed
self._average_residuals = False
self._spectral_analysis = False
self._av_callback = av_callback
# Remove invalid reflections
reflections = reflections.select(~(reflections['miller_index'] == (0,0,0)))
# FIXME - better way to recognise non-predictions. Can't rely on flags
# in e.g. indexed.pickle I think.
x, y, z = reflections['xyzcal.mm'].parts()
sel = (x == 0) & (y == 0)
reflections = reflections.select(~sel)
self._nexp = flex.max(reflections['id']) + 1
# Ensure required keys are present
if not all([k in reflections for k in ['x_resid', 'y_resid', 'phi_resid']]):
x_obs, y_obs, phi_obs = reflections['xyzobs.mm.value'].parts()
x_cal, y_cal, phi_cal = reflections['xyzcal.mm'].parts()
# do not wrap around multiples of 2*pi; keep the full rotation
# from zero to differentiate repeat observations.
from math import pi
TWO_PI = 2.0 * pi
resid = phi_cal - (flex.fmod_positive(phi_obs, TWO_PI))
# ensure this is the smaller of two possibilities
resid = flex.fmod_positive((resid + pi), TWO_PI) - pi
phi_cal = phi_obs + resid
reflections['x_resid'] = x_cal - x_obs
reflections['y_resid'] = y_cal - y_obs
reflections['phi_resid'] = phi_cal - phi_obs
# create empty results list
self._results = []
# first, just determine a suitable block size for analysis
for iexp in range(self._nexp):
ref_this_exp = reflections.select(reflections['id'] == iexp)
if len(ref_this_exp) == 0:
# can't do anything, just keep an empty dictionary
self._results.append({})
continue
phi_obs_deg = ref_this_exp['xyzobs.mm.value'].parts()[2] * RAD2DEG
phi_range = flex.min(phi_obs_deg), flex.max(phi_obs_deg)
phi_width = phi_range[1] - phi_range[0]
ideal_block_size = 1.0
old_nblocks = 0
while True:
nblocks = int(phi_width // ideal_block_size)
if nblocks == old_nblocks: nblocks -= 1
nblocks = max(nblocks, 1)
block_size = phi_width / nblocks
nr = flex.int()
for i in range(nblocks - 1):
blk_start = phi_range[0] + i * block_size
blk_end = blk_start + block_size
sel = (phi_obs_deg >= blk_start) & (phi_obs_deg < blk_end)
nref_in_block = sel.count(True)
nr.append(nref_in_block)
# include max phi in the final block
blk_start = phi_range[0] + (nblocks - 1) * block_size
blk_end = phi_range[1]
sel = (phi_obs_deg >= blk_start) & (phi_obs_deg <= blk_end)
nref_in_block = sel.count(True)
nr.append(nref_in_block)
# Break if there are enough reflections, otherwise increase block size,
# unless only one block remains
if nblocks == 1: break
min_nr = flex.min(nr)
if min_nr >= 50: break
if min_nr < 5:
fac = 2
else:
fac = 50 / min_nr
ideal_block_size *= fac
old_nblocks = nblocks
# collect the basic data for this experiment
self._results.append({'block_size':block_size,
'nref_per_block':nr,
'nblocks':nblocks,
'phi_range':phi_range})
# keep reflections for analysis
self._reflections = reflections
#from matplotlib import pylab
#fig = pylab.figure(dpi=300)
#pylab.imshow(scale_data.as_numpy_array(), vmin=0, vmax=2, interpolation='none')
#pylab.colorbar()
#pylab.savefig("scale_%d.png" % frame)
#pylab.clf()
##pylab.show()
#exit(0)
#pylab.hist(scale_data.as_1d().select(scale_mask.as_1d()).as_numpy_array(),
# bins=100)
#pylab.show()
sd1 = scale_data.as_1d()
sm1 = scale_mask.as_1d()
scale_min = flex.min(sd1.select(sm1))
scale_max = flex.max(sd1.select(sm1))
scale_avr = flex.sum(sd1.select(sm1)) / sm1.count(True)
background = model_data * scale_data
reflections['shoebox'].select(indices).apply_pixel_data(
data.as_double(),
background,
raw_mask,
frame,
1)
subset = reflections.select(indices3)
if len(subset) > 0:
subset.compute_summed_intensity()
def show_reflections(
reflections,
show_intensities=False,
show_profile_fit=False,
show_centroids=False,
show_all_reflection_data=False,
show_flags=False,
max_reflections=None,
show_identifiers=False,
):
text = []
from orderedset import OrderedSet
formats = collections.OrderedDict((
("miller_index", "%i, %i, %i"),
("d", "%.2f"),
("qe", "%.3f"),
("dqe", "%.3f"),
("id", "%i"),
("imageset_id", "%i"),
("panel", "%i"),
("flags", "%i"),
("background.mean", "%.1f"),
("background.dispersion", "%.1f"),
("background.mse", "%.1f"),
("background.sum.value", "%.1f"),
("background.sum.variance", "%.1f"),
("intensity.prf.value", "%.1f"),
("intensity.prf.variance", "%.1f"),
("intensity.sum.value", "%.1f"),
("intensity.sum.variance", "%.1f"),
("intensity.cor.value", "%.1f"),
("intensity.cor.variance", "%.1f"),
("intensity.scale.value", "%.1f"),
("intensity.scale.variance", "%.1f"),
("Ih_values", "%.1f"),
("lp", "%.3f"),
("num_pixels.background", "%i"),
("num_pixels.background_used", "%i"),
("num_pixels.foreground", "%i"),
("num_pixels.valid", "%i"),
("partial_id", "%i"),
("partiality", "%.4f"),
("profile.correlation", "%.3f"),
("profile.rmsd", "%.3f"),
("xyzcal.mm", "%.2f, %.2f, %.2f"),
("xyzcal.px", "%.2f, %.2f, %.2f"),
("delpsical.rad", "%.3f"),
("delpsical2", "%.3f"),
("delpsical.weights", "%.3f"),
("xyzobs.mm.value", "%.2f, %.2f, %.2f"),
("xyzobs.mm.variance", "%.4e, %.4e, %.4e"),
("xyzobs.px.value", "%.2f, %.2f, %.2f"),
("xyzobs.px.variance", "%.4f, %.4f, %.4f"),
("s1", "%.4f, %.4f, %.4f"),
("s2", "%.4f, %.4f, %.4f"),
("shoebox", "%.1f"),
("rlp", "%.4f, %.4f, %.4f"),
("zeta", "%.3f"),
("x_resid", "%.3f"),
("x_resid2", "%.3f"),
("y_resid", "%.3f"),
("y_resid2", "%.3f"),
("kapton_absorption_correction", "%.3f"),
("kapton_absorption_correction_sigmas", "%.3f"),
("inverse_scale_factor", "%.3f"),
("inverse_scale_factor_variance", "%.3f"),
))
for rlist in reflections:
from dials.algorithms.shoebox import MaskCode
foreground_valid = MaskCode.Valid | MaskCode.Foreground
text.append("")
text.append("Reflection list contains %i reflections" % (len(rlist)))
if len(rlist) == 0:
continue
rows = [["Column", "min", "max", "mean"]]
for k, col in rlist.cols():
if k in formats and "%" not in formats.get(k, "%s"):
# Allow blanking out of entries that wouldn't make sense
rows.append([
k,
formats.get(k, "%s"),
formats.get(k, "%s"),
formats.get(k, "%s"),
])
elif type(col) in (flex.double, flex.int, flex.size_t):
if type(col) in (flex.int, flex.size_t):
col = col.as_double()
rows.append([
k,
formats.get(k, "%s") % flex.min(col),
formats.get(k, "%s") % flex.max(col),
formats.get(k, "%s") % flex.mean(col),
])
elif type(col) in (flex.vec3_double, flex.miller_index):
if isinstance(col, flex.miller_index):
col = col.as_vec3_double()
rows.append([
k,
formats.get(k, "%s") % col.min(),
formats.get(k, "%s") % col.max(),
formats.get(k, "%s") % col.mean(),
])
elif isinstance(col, flex.shoebox):
rows.append([k, "", "", ""])
si = col.summed_intensity().observed_value()
rows.append([
" summed I",
formats.get(k, "%s") % flex.min(si),
formats.get(k, "%s") % flex.max(si),
formats.get(k, "%s") % flex.mean(si),
])
x1, x2, y1, y2, z1, z2 = col.bounding_boxes().parts()
bbox_sizes = ((z2 - z1) * (y2 - y1) * (x2 - x1)).as_double()
rows.append([
" N pix",
formats.get(k, "%s") % flex.min(bbox_sizes),
formats.get(k, "%s") % flex.max(bbox_sizes),
formats.get(k, "%s") % flex.mean(bbox_sizes),
])
fore_valid = col.count_mask_values(
foreground_valid).as_double()
rows.append([
" N valid foreground pix",
formats.get(k, "%s") % flex.min(fore_valid),
formats.get(k, "%s") % flex.max(fore_valid),
formats.get(k, "%s") % flex.mean(fore_valid),
])
text.append(tabulate(rows, headers="firstrow"))
if show_flags:
text.append(_create_flag_count_table(rlist))
if show_identifiers:
if rlist.experiment_identifiers():
text.append(
"""Experiment identifiers id-map values:\n%s""" %
("\n".join(
"id:" + str(k) + " -> experiment identifier:" +
str(rlist.experiment_identifiers()[k])
for k in rlist.experiment_identifiers().keys())))
intensity_keys = (
"miller_index",
"d",
"intensity.prf.value",
"intensity.prf.variance",
"intensity.sum.value",
"intensity.sum.variance",
"background.mean",
"profile.correlation",
"profile.rmsd",
)
profile_fit_keys = ("miller_index", "d")
centroid_keys = (
"miller_index",
"d",
"xyzcal.mm",
"xyzcal.px",
"xyzobs.mm.value",
"xyzobs.mm.variance",
"xyzobs.px.value",
"xyzobs.px.variance",
)
keys_to_print = OrderedSet()
if show_intensities:
for k in intensity_keys:
keys_to_print.add(k)
if show_profile_fit:
for k in profile_fit_keys:
keys_to_print.add(k)
if show_centroids:
for k in centroid_keys:
keys_to_print.add(k)
if show_all_reflection_data:
for k in formats:
keys_to_print.add(k)
def format_column(key, data, format_strings=None):
if isinstance(data, flex.vec3_double):
c_strings = [
c.as_string(format_strings[i].strip())
for i, c in enumerate(data.parts())
]
elif isinstance(data, flex.miller_index):
c_strings = [
c.as_string(format_strings[i].strip())
for i, c in enumerate(data.as_vec3_double().parts())
]
elif isinstance(data, flex.size_t):
c_strings = [data.as_int().as_string(format_strings[0].strip())]
elif isinstance(data, flex.shoebox):
x1, x2, y1, y2, z1, z2 = data.bounding_boxes().parts()
bbox_sizes = ((z2 - z1) * (y2 - y1) * (x2 - x1)).as_double()
c_strings = [bbox_sizes.as_string(format_strings[0].strip())]
key += " (N pix)"
else:
c_strings = [data.as_string(format_strings[0].strip())]
column = flex.std_string()
max_element_lengths = [c.max_element_length() for c in c_strings]
for i in range(len(c_strings[0])):
column.append(("%%%is" % len(key)) % ", ".join(
("%%%is" % max_element_lengths[j]) % c_strings[j][i]
for j in range(len(c_strings))))
return column
if keys_to_print:
keys = [k for k in keys_to_print if k in rlist]
if max_reflections is not None:
max_reflections = min(len(rlist), max_reflections)
else:
max_reflections = len(rlist)
columns = []
for k in keys:
columns.append(
format_column(k,
rlist[k],
format_strings=formats[k].split(",")))
text.append("")
text.append("Printing %i of %i reflections:" %
(max_reflections, len(rlist)))
line = []
for j in range(len(columns)):
key = keys[j]
if key == "shoebox":
key += " (N pix)"
width = max(len(key), columns[j].max_element_length())
line.append("%%%is" % width % key)
text.append(" ".join(line))
for i in range(max_reflections):
line = (c[i] for c in columns)
text.append(" ".join(line))
return "\n".join(text)