def image_correlation(a, b): a = a.as_1d().as_double() b = b.as_1d().as_double() sel = (a > 0) & (b > 0) _a = a.select(sel) _b = b.select(sel) return sel.count(True), flex.linear_correlation(x=a, y=b).coefficient()
def correlation(a, b):
from astrotbx.input_output.loader import load_image_gs
from dials.array_family import flex
data_a = load_image_gs(a).as_1d()
data_b = load_image_gs(b).as_1d()
c = flex.linear_correlation(data_a, data_b)
print(c.coefficient())
def image_correlation(a, b):
sig_a = extract_signal_mask(a)
sig_b = extract_signal_mask(b)
a = a.as_1d().as_double()
b = b.as_1d().as_double()
sel = (a > 0) & (b > 0) & sig_a & sig_b
_a = a.select(sel)
_b = b.select(sel)
return sel.count(True), flex.linear_correlation(x=a, y=b).coefficient()
def image_correlation(a, b): sig_a = extract_signal_mask(a) sig_b = extract_signal_mask(b) a = a.as_1d().as_double() b = b.as_1d().as_double() sel = (a > 0) & (b > 0) & sig_a & sig_b _a = a.select(sel) _b = b.select(sel) return sel.count(True), flex.linear_correlation(x=a, y=b).coefficient()
def correlation(self, x):
"""
The correlation of the Jacobian
"""
J = self.jacobian(x)
C = flex.double(flex.grid(J.all()[1], J.all()[1]))
for j in range(C.all()[0]):
for i in range(C.all()[1]):
a = J[:, i:i + 1].as_1d()
b = J[:, j:j + 1].as_1d()
C[j, i] = flex.linear_correlation(a, b).coefficient()
return C
def profile_correlation(data, model):
"""Compute CC between reflection profiles data and model."""
assert data.focus() == model.focus()
sel = data.as_1d() > 0
model = model.as_1d().select(sel)
data = data.as_1d().select(sel).as_double()
assert len(data) == len(model)
correlation = flex.linear_correlation(data, model)
return correlation.coefficient()
def profile_correlation(data, model):
'''Compute CC between reflection profiles data and model.'''
from dials.array_family import flex
assert (data.focus() == model.focus())
sel = data.as_1d() > 0
model = model.as_1d().select(sel)
data = data.as_1d().select(sel).as_double()
assert (len(data) == len(model))
correlation = flex.linear_correlation(data, model)
return correlation.coefficient()
def profile_correlation(data, model): '''Compute CC between reflection profiles data and model.''' from dials.array_family import flex assert(data.focus() == model.focus()) sel = data.as_1d() > 0 model = model.as_1d().select(sel) data = data.as_1d().select(sel).as_double() assert(len(data) == len(model)) correlation = flex.linear_correlation(data, model) return correlation.coefficient()
def compare_pickle_with_mtz(params):
"""Compare intensities in pickle file with the scaled and merged intensities
provided through the mtz file."""
from libtbx.utils import Sorry
from iotbx import mtz
from dials.array_family import flex
import cPickle as pickle
m = mtz.object(params.reference)
d = pickle.load(open(params.data))
# fnd the right reference data
mad = m.as_miller_arrays_dict(merge_equivalents=False)
i = None
for k in mad.keys():
if k[2] == "IMEAN":
i = mad[k].as_non_anomalous_array().expand_to_p1()
assert not i is None
# print reference information
print("Reference")
i.show_summary()
# clean up - remove non-integrated data - TODO look at just strong spots
if params.filter_integrated:
d = d.select(d.get_flags(d.flags.integrated))
if params.filter_partiality:
d = d.select(d["partiality"] > params.filter_partiality)
# apply scale factors from input file, if present
# if requested, scale data to reference using simple kB model
if params.kb_scale:
raise Sorry("No can do, implementing kB scaling is on the to do")
# match data to reference
from cctbx import miller
mi = miller.set(
crystal_symmetry=i.crystal_symmetry(),
anomalous_flag=False,
indices=d["miller_index"],
).expand_to_p1()
match = mi.match_indices(i)
pairs = match.pairs()
i1 = flex.double()
i2 = flex.double()
scl = flex.double()
# TODO remove outliers here from the paired up list => do not need to
# worry about them biasing the correlation
for p0, p1 in pairs:
i1.append(d["intensity.sum.value"][p0])
i2.append(i.data()[p1])
if "partiality" in d:
scl.append(d["partiality"][p0])
else:
scl.append(1.0)
corr = flex.linear_correlation(i1, i2)
corr2 = flex.linear_correlation(i1 / scl, i2)
print("Correlation:", corr.coefficient(), corr2.coefficient(),
pairs.size(), d.size())
if params.out:
with open(params.out, "w") as f:
for iis in zip(i1, i2, scl):
f.write("%f %f %f\n" % iis)
def run(self):
''' Parse the options. '''
from dials.util.options import flatten_experiments, flatten_reflections
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
self.params = params
experiments = flatten_experiments(params.input.experiments)
# Find all detector objects
detectors = experiments.detectors()
# Verify inputs
if len(params.input.reflections) == len(detectors) and len(detectors) > 1:
# case for passing in multiple images on the command line
assert len(params.input.reflections) == len(detectors)
reflections = flex.reflection_table()
for expt_id in xrange(len(detectors)):
subset = params.input.reflections[expt_id].data
subset['id'] = flex.int(len(subset), expt_id)
reflections.extend(subset)
else:
# case for passing in combined experiments and reflections
reflections = flatten_reflections(params.input.reflections)[0]
detector = detectors[0]
#from dials.algorithms.refinement.prediction import ExperimentsPredictor
#ref_predictor = ExperimentsPredictor(experiments, force_stills=experiments.all_stills())
print "N reflections total:", len(reflections)
if params.residuals.exclude_outliers:
reflections = reflections.select(reflections.get_flags(reflections.flags.used_in_refinement))
print "N reflections used in refinement:", len(reflections)
print "Reporting only on those reflections used in refinement"
if self.params.residuals.i_sigi_cutoff is not None:
sel = (reflections['intensity.sum.value']/flex.sqrt(reflections['intensity.sum.variance'])) >= self.params.residuals.i_sigi_cutoff
reflections = reflections.select(sel)
print "After filtering by I/sigi cutoff of %f, there are %d reflections left"%(self.params.residuals.i_sigi_cutoff,len(reflections))
reflections['difference_vector_norms'] = (reflections['xyzcal.mm']-reflections['xyzobs.mm.value']).norms()
n = len(reflections)
rmsd = self.get_weighted_rmsd(reflections)
print "Dataset RMSD (microns)", rmsd * 1000
if params.tag is None:
tag = ''
else:
tag = '%s '%params.tag
# set up delta-psi ratio heatmap
p = flex.int() # positive
n = flex.int() # negative
for i in set(reflections['id']):
exprefls = reflections.select(reflections['id']==i)
p.append(len(exprefls.select(exprefls['delpsical.rad']>0)))
n.append(len(exprefls.select(exprefls['delpsical.rad']<0)))
plt.hist2d(p, n, bins=30)
cb = plt.colorbar()
cb.set_label("N images")
plt.title(r"%s2D histogram of pos vs. neg $\Delta\Psi$ per image"%tag)
plt.xlabel(r"N reflections with $\Delta\Psi$ > 0")
plt.ylabel(r"N reflections with $\Delta\Psi$ < 0")
self.delta_scalar = 50
# Iterate through the detectors, computing detector statistics at the per-panel level (IE one statistic per panel)
# Per panel dictionaries
rmsds = {}
refl_counts = {}
transverse_rmsds = {}
radial_rmsds = {}
ttdpcorr = {}
pg_bc_dists = {}
mean_delta_two_theta = {}
# per panelgroup flex arrays
pg_rmsds = flex.double()
pg_r_rmsds = flex.double()
pg_t_rmsds = flex.double()
pg_refls_count = flex.int()
pg_refls_count_d = {}
table_header = ["PG id", "RMSD","Radial", "Transverse", "N refls"]
table_header2 = ["","(um)","RMSD (um)","RMSD (um)",""]
table_data = []
table_data.append(table_header)
table_data.append(table_header2)
# Compute a set of radial and transverse displacements for each reflection
print "Setting up stats..."
tmp = flex.reflection_table()
# Need to construct a variety of vectors
for panel_id, panel in enumerate(detector):
panel_refls = reflections.select(reflections['panel'] == panel_id)
bcl = flex.vec3_double()
tto = flex.double()
ttc = flex.double()
# Compute the beam center in lab space (a vector pointing from the origin to where the beam would intersect
# the panel, if it did intersect the panel)
for expt_id in set(panel_refls['id']):
beam = experiments[expt_id].beam
s0 = beam.get_s0()
expt_refls = panel_refls.select(panel_refls['id'] == expt_id)
beam_centre = panel.get_beam_centre_lab(s0)
bcl.extend(flex.vec3_double(len(expt_refls), beam_centre))
obs_x, obs_y, _ = expt_refls['xyzobs.px.value'].parts()
cal_x, cal_y, _ = expt_refls['xyzcal.px'].parts()
tto.extend(flex.double([panel.get_two_theta_at_pixel(s0, (obs_x[i], obs_y[i])) for i in xrange(len(expt_refls))]))
ttc.extend(flex.double([panel.get_two_theta_at_pixel(s0, (cal_x[i], cal_y[i])) for i in xrange(len(expt_refls))]))
panel_refls['beam_centre_lab'] = bcl
panel_refls['two_theta_obs'] = tto * (180/math.pi)
panel_refls['two_theta_cal'] = ttc * (180/math.pi) #+ (0.5*panel_refls['delpsical.rad']*panel_refls['two_theta_obs'])
# Compute obs in lab space
x, y, _ = panel_refls['xyzobs.mm.value'].parts()
c = flex.vec2_double(x, y)
panel_refls['obs_lab_coords'] = panel.get_lab_coord(c)
# Compute deltaXY in panel space. This vector is relative to the panel origin
x, y, _ = (panel_refls['xyzcal.mm'] - panel_refls['xyzobs.mm.value']).parts()
# Convert deltaXY to lab space, subtracting off of the panel origin
panel_refls['delta_lab_coords'] = panel.get_lab_coord(flex.vec2_double(x,y)) - panel.get_origin()
tmp.extend(panel_refls)
reflections = tmp
# The radial vector points from the center of the reflection to the beam center
radial_vectors = (reflections['obs_lab_coords'] - reflections['beam_centre_lab']).each_normalize()
# The transverse vector is orthogonal to the radial vector and the beam vector
transverse_vectors = radial_vectors.cross(reflections['beam_centre_lab']).each_normalize()
# Compute the raidal and transverse components of each deltaXY
reflections['radial_displacements'] = reflections['delta_lab_coords'].dot(radial_vectors)
reflections['transverse_displacements'] = reflections['delta_lab_coords'].dot(transverse_vectors)
# Iterate through the detector at the specified hierarchy level
for pg_id, pg in enumerate(iterate_detector_at_level(detector.hierarchy(), 0, params.hierarchy_level)):
pg_msd_sum = 0
pg_r_msd_sum = 0
pg_t_msd_sum = 0
pg_refls = 0
pg_delpsi = flex.double()
pg_deltwotheta = flex.double()
for p in iterate_panels(pg):
panel_id = id_from_name(detector, p.get_name())
panel_refls = reflections.select(reflections['panel'] == panel_id)
n = len(panel_refls)
pg_refls += n
delta_x = panel_refls['xyzcal.mm'].parts()[0] - panel_refls['xyzobs.mm.value'].parts()[0]
delta_y = panel_refls['xyzcal.mm'].parts()[1] - panel_refls['xyzobs.mm.value'].parts()[1]
tmp = flex.sum((delta_x**2)+(delta_y**2))
pg_msd_sum += tmp
r = panel_refls['radial_displacements']
t = panel_refls['transverse_displacements']
pg_r_msd_sum += flex.sum_sq(r)
pg_t_msd_sum += flex.sum_sq(t)
pg_delpsi.extend(panel_refls['delpsical.rad']*180/math.pi)
pg_deltwotheta.extend(panel_refls['two_theta_obs'] - panel_refls['two_theta_cal'])
bc = col(pg.get_beam_centre_lab(s0))
ori = get_center(pg)
pg_bc_dists[pg.get_name()] = (ori-bc).length()
if len(pg_deltwotheta) > 0:
mean_delta_two_theta[pg.get_name()] = flex.mean(pg_deltwotheta)
else:
mean_delta_two_theta[pg.get_name()] = 0
if pg_refls == 0:
pg_rmsd = pg_r_rmsd = pg_t_rmsd = 0
else:
pg_rmsd = math.sqrt(pg_msd_sum/pg_refls) * 1000
pg_r_rmsd = math.sqrt(pg_r_msd_sum/pg_refls) * 1000
pg_t_rmsd = math.sqrt(pg_t_msd_sum/pg_refls) * 1000
pg_rmsds.append(pg_rmsd)
pg_r_rmsds.append(pg_r_rmsd)
pg_t_rmsds.append(pg_t_rmsd)
pg_refls_count.append(pg_refls)
pg_refls_count_d[pg.get_name()] = pg_refls
table_data.append(["%d"%pg_id, "%.1f"%pg_rmsd, "%.1f"%pg_r_rmsd, "%.1f"%pg_t_rmsd, "%6d"%pg_refls])
refl_counts[pg.get_name()] = pg_refls
if pg_refls == 0:
rmsds[p.get_name()] = -1
radial_rmsds[p.get_name()] = -1
transverse_rmsds[p.get_name()] = -1
ttdpcorr[pg.get_name()] = -1
else:
rmsds[pg.get_name()] = pg_rmsd
radial_rmsds[pg.get_name()] = pg_r_rmsd
transverse_rmsds[pg.get_name()] = pg_t_rmsd
lc = flex.linear_correlation(pg_delpsi, pg_deltwotheta)
ttdpcorr[pg.get_name()] = lc.coefficient()
r1 = ["Weighted mean"]
r2 = ["Weighted stddev"]
if len(pg_rmsds) > 1:
stats = flex.mean_and_variance(pg_rmsds, pg_refls_count.as_double())
r1.append("%.1f"%stats.mean())
r2.append("%.1f"%stats.gsl_stats_wsd())
stats = flex.mean_and_variance(pg_r_rmsds, pg_refls_count.as_double())
r1.append("%.1f"%stats.mean())
r2.append("%.1f"%stats.gsl_stats_wsd())
stats = flex.mean_and_variance(pg_t_rmsds, pg_refls_count.as_double())
r1.append("%.1f"%stats.mean())
r2.append("%.1f"%stats.gsl_stats_wsd())
else:
r1.extend([""]*3)
r2.extend([""]*3)
r1.append("")
r2.append("")
table_data.append(r1)
table_data.append(r2)
table_data.append(["Mean", "", "", "", "%8.1f"%flex.mean(pg_refls_count.as_double())])
from libtbx import table_utils
print "Detector statistics. Angles in degrees, RMSDs in microns"
print table_utils.format(table_data,has_header=2,justify='center',delim=" ")
self.histogram(reflections, '%sDifference vector norms (mm)'%tag)
if params.show_plots:
if self.params.tag is None:
t = ""
else:
t = "%s "%self.params.tag
self.image_rmsd_histogram(reflections, tag)
# Plots! these are plots with callbacks to draw on individual panels
self.detector_plot_refls(detector, reflections, '%sOverall positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_deltas)
self.detector_plot_refls(detector, reflections, '%sRadial positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_radial_deltas)
self.detector_plot_refls(detector, reflections, '%sTransverse positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_transverse_deltas)
self.detector_plot_refls(detector, reflections, r'%s$\Delta\Psi$'%tag, show=False, plot_callback=self.plot_obs_colored_by_deltapsi, colorbar_units=r"$\circ$")
self.detector_plot_refls(detector, reflections, r'%s$\Delta$XY*%s'%(tag, self.delta_scalar), show=False, plot_callback=self.plot_deltas)
self.detector_plot_refls(detector, reflections, '%sSP Manual CDF'%tag, show=False, plot_callback=self.plot_cdf_manually)
self.detector_plot_refls(detector, reflections, r'%s$\Delta$XY Histograms'%tag, show=False, plot_callback=self.plot_histograms)
self.detector_plot_refls(detector, reflections, r'%sRadial displacements vs. $\Delta\Psi$, colored by $\Delta$XY'%tag, show=False, plot_callback=self.plot_radial_displacements_vs_deltapsi)
self.detector_plot_refls(detector, reflections, r'%sDistance vector norms'%tag, show=False, plot_callback=self.plot_difference_vector_norms_histograms)
# Plot intensity vs. radial_displacement
fig = plt.figure()
panel_id = 15
panel_refls = reflections.select(reflections['panel'] == panel_id)
a = panel_refls['radial_displacements']
b = panel_refls['intensity.sum.value']
sel = (a > -0.2) & (a < 0.2) & (b < 50000)
plt.hist2d(a.select(sel), b.select(sel), bins=100)
plt.title("%s2D histogram of intensity vs. radial displacement for panel %d"%(tag, panel_id))
plt.xlabel("Radial displacement (mm)")
plt.ylabel("Intensity")
ax = plt.colorbar()
ax.set_label("Counts")
# Plot delta 2theta vs. deltapsi
n_bins = 10
bin_size = len(reflections)//n_bins
bin_low = []
bin_high = []
data = flex.sorted(reflections['two_theta_obs'])
for i in xrange(n_bins):
bin_low = data[i*bin_size]
if (i+1)*bin_size >= len(reflections):
bin_high = data[-1]
else:
bin_high = data[(i+1)*bin_size]
refls = reflections.select((reflections['two_theta_obs'] >= bin_low) &
(reflections['two_theta_obs'] <= bin_high))
a = refls['delpsical.rad']*180/math.pi
b = refls['two_theta_obs'] - refls['two_theta_cal']
fig = plt.figure()
sel = (a > -0.2) & (a < 0.2) & (b > -0.05) & (b < 0.05)
plt.hist2d(a.select(sel), b.select(sel), bins=50, range = [[-0.2, 0.2], [-0.05, 0.05]])
cb = plt.colorbar()
cb.set_label("N reflections")
plt.title(r'%sBin %d (%.02f, %.02f 2$\Theta$) $\Delta2\Theta$ vs. $\Delta\Psi$. Showing %d of %d refls'%(tag,i,bin_low,bin_high,len(a.select(sel)),len(a)))
plt.xlabel(r'$\Delta\Psi \circ$')
plt.ylabel(r'$\Delta2\Theta \circ$')
# Plot delta 2theta vs. 2theta
a = reflections['two_theta_obs']#[:71610]
b = reflections['two_theta_obs'] - reflections['two_theta_cal']
fig = plt.figure()
limits = -0.05, 0.05
sel = (b > limits[0]) & (b < limits[1])
plt.hist2d(a.select(sel), b.select(sel), bins=100, range=((0,50), limits))
plt.clim((0,100))
cb = plt.colorbar()
cb.set_label("N reflections")
plt.title(r'%s$\Delta2\Theta$ vs. 2$\Theta$. Showing %d of %d refls'%(tag,len(a.select(sel)),len(a)))
plt.xlabel(r'2$\Theta \circ$')
plt.ylabel(r'$\Delta2\Theta \circ$')
# calc the trendline
z = np.polyfit(a.select(sel), b.select(sel), 1)
print 'y=%.7fx+(%.7f)'%(z[0],z[1])
# Plots with single values per panel
self.detector_plot_dict(detector, refl_counts, u"%s N reflections"%t, u"%6d", show=False)
self.detector_plot_dict(detector, rmsds, "%s Positional RMSDs (microns)"%t, u"%4.1f", show=False)
self.detector_plot_dict(detector, radial_rmsds, "%s Radial RMSDs (microns)"%t, u"%4.1f", show=False)
self.detector_plot_dict(detector, transverse_rmsds, "%s Transverse RMSDs (microns)"%t, u"%4.1f", show=False)
self.detector_plot_dict(detector, ttdpcorr, r"%s $\Delta2\Theta$ vs. $\Delta\Psi$ CC"%t, u"%5.3f", show=False)
self.plot_unitcells(experiments)
self.plot_data_by_two_theta(reflections, tag)
# Plot data by panel group
sorted_values = sorted(pg_bc_dists.values())
vdict = {}
for k in pg_bc_dists:
vdict[pg_bc_dists[k]] = k
sorted_keys = [vdict[v] for v in sorted_values if vdict[v] in rmsds]
x = [sorted_values[i] for i in xrange(len(sorted_values)) if pg_bc_dists.keys()[i] in rmsds]
self.plot_multi_data(x,
[[pg_refls_count_d[k] for k in sorted_keys],
([rmsds[k] for k in sorted_keys],
[radial_rmsds[k] for k in sorted_keys],
[transverse_rmsds[k] for k in sorted_keys]),
[radial_rmsds[k]/transverse_rmsds[k] for k in sorted_keys],
[mean_delta_two_theta[k] for k in sorted_keys]],
"Panel group distance from beam center (mm)",
["N reflections",
("Overall RMSD",
"Radial RMSD",
"Transverse RMSD"),
"R/T RMSD ratio",
"Delta two theta"],
["N reflections",
"RMSD (microns)",
"R/T RMSD ratio",
"Delta two theta (degrees)"],
"%sData by panelgroup"%tag)
if self.params.save_pdf:
pp = PdfPages('residuals_%s.pdf'%(tag.strip()))
for i in plt.get_fignums():
pp.savefig(plt.figure(i))
pp.close()
else:
plt.show()
def compute_local_cc_vs_ref(reflections, reference, kernel_size):
"""
Compute the local cc vs a reference set of intrnsities
"""
H, K, L, I1, I2 = match_reflections(reflections, reference)
# Get the range of miller indices
min_H, max_H = min(H), max(H)
min_K, max_K = min(K), max(K)
min_L, max_L = min(L), max(L)
n_H = max_H - min_H + 1
n_K = max_K - min_K + 1
n_L = max_L - min_L + 1
mask = flex.bool(flex.grid(n_L, n_K, n_H))
data1 = flex.double(mask.accessor())
data2 = flex.double(mask.accessor())
H_array = flex.double(mask.accessor())
K_array = flex.double(mask.accessor())
L_array = flex.double(mask.accessor())
for h, k, l, intensity1, intensity2 in zip(H, K, L, I1, I2):
x = h - min_H
y = k - min_K
z = l - min_L
assert x >= 0 and y >= 0 and z >= 0
mask[z, y, x] = True
data1[z, y, x] = intensity1
data2[z, y, x] = intensity2
H_array[z, y, x] = h
K_array[z, y, x] = k
L_array[z, y, x] = l
indices = []
for k in range(mask.all()[0]):
for j in range(mask.all()[1]):
for i in range(mask.all()[2]):
if mask[k, j, i] > 0:
indices.append((k, j, i))
cc_array = flex.double(mask.accessor())
mask_out = flex.bool(mask.accessor())
for k, j, i in indices:
k0 = k - kernel_size
k1 = k + kernel_size + 1
j0 = j - kernel_size
j1 = j + kernel_size + 1
i0 = i - kernel_size
i1 = i + kernel_size + 1
if k0 < 0:
k0 = 0
if j0 < 0:
j0 = 0
if i0 < 0:
i0 = 0
if k1 > mask.all()[0]:
k1 = mask.all()[0]
if j1 > mask.all()[1]:
j1 = mask.all()[1]
if i1 > mask.all()[2]:
i1 = mask.all()[2]
X = flex.double()
Y = flex.double()
for kk in range(k0, k1):
for jj in range(j0, j1):
for ii in range(i0, i1):
if mask[kk, jj, ii]:
X.append(data1[kk, jj, ii])
Y.append(data2[kk, jj, ii])
if len(X) > 0.25 * (2 * kernel_size + 1)**3:
c = flex.linear_correlation(X, Y)
cc_array[k, j, i] = c.coefficient()
print(c.coefficient())
mask_out[k, j, i] = True
selection = (mask_out == True).as_1d()
H_sub = H_array.as_1d().select(selection)
K_sub = K_array.as_1d().select(selection)
L_sub = L_array.as_1d().select(selection)
R_sub = cc_array.as_1d().select(selection)
return H_sub, K_sub, L_sub, R_sub
