def tilting_plane(img, mask=None, zscore=2):
"""
fit tilting plane to img data, used for background subtraction of spots
:param img: numpy image
:param mask: boolean mask, same shape as img, True is good pixels
mask should include strong spot pixels and bad pixels, e.g. zingers
:param zscore: modified z-score for outlier detection, lower increases number of outliers
:return: tilting plane, same shape as img
"""
from cxid9114 import utils
Y, X = np.indices(img.shape)
YY, XX = Y.ravel(), X.ravel()
img1d = img.ravel()
if mask is None:
mask = np.ones(img.shape, bool)
mask1d = mask.ravel()
out1d = np.zeros(mask1d.shape, bool)
out1d[mask1d] = utils.is_outlier(img1d[mask1d].ravel(), zscore)
out2d = out1d.reshape(img.shape)
fit_sel = np.logical_and(
~out2d, mask) # fit plane to these points, no outliers, no masked
x, y, z = X[fit_sel], Y[fit_sel], img[fit_sel]
guess = np.array([np.ones_like(x), x, y]).T
coeff, r, rank, s = np.linalg.lstsq(guess, z)
ev = (coeff[0] + coeff[1] * XX + coeff[2] * YY)
return ev.reshape(img.shape), out2d, coeff
def plot_img(refls,
spot_dataA,
spot_dataB,
detector,
beamA,
beamB,
crystal,
iset,
name,
bad=None):
from cxid9114 import utils
import numpy as np
import pylab as plt
d, dvecs, best = indexing_residuals_twocolor(spot_dataA, spot_dataB, refls,
detector)
HA, HiA = spot_utils.refls_to_hkl(refls, detector, beamA, crystal)
HB, HiB = spot_utils.refls_to_hkl(refls, detector, beamB, crystal)
Q = spot_utils.refls_to_q(refls, detector, beamA)
Qmag = np.linalg.norm(Q, axis=1)
res = 1. / Qmag
HAres = np.round((HA - HiA), 2)
HBres = np.round((HB - HiB), 2)
plot_dvecs(d, dvecs, HAres, HBres)
if bad is None:
bad = np.where(utils.is_outlier(d, 4))[0]
for i_b, b in enumerate(bad):
r = refls[b]
reso = res[b]
panel = r['panel']
HAres = np.round((HA - HiA)[b], 2)
HBres = np.round((HB - HiB)[b], 2)
# continue
yA, xA = zip(*spot_dataA[panel]['comIpos'])
yB, xB = zip(*spot_dataB[panel]['comIpos'])
xp, yp, _ = r['xyzobs.px.value']
img = iset.get_raw_data(0)[panel].as_numpy_array()
plt.figure()
plt.imshow(img, vmax=150)
plt.plot(xA, yA, 'o', mfc='none', color='lightblue', ms=10, mew=2)
plt.plot(xB, yB, 's', mfc='none', color='C3', ms=10, mew=2)
plt.plot(xp, yp, 'o', color='C2', mfc='none', ms=10, mew=2)
HAres = np.sqrt(np.sum(HAres**2))
HBres = np.sqrt(np.sum(HBres**2))
title_s = "fhklA: %.2f, fhklB: %.2f" % (HAres, HBres)
title_s += "\nresolution of spot: ~ %.2f Angstrom" % reso
plt.gca().set_title(title_s)
plt.draw()
plt.pause(5)
plt.savefig("%s_%d.png" % (name, i_b))
plt.close()
def plot_dvecs(d, dvecs, HA, HB):
import pylab as plt
import numpy as np
from cxid9114 import utils
bad = np.where(utils.is_outlier(d, 4))[0]
plt.figure()
plt.plot(dvecs[:, 0] / .10992, dvecs[:, 1] / .10992, 'o', color='C0', ms=3)
ax = plt.gca()
ax.add_patch(
plt.Circle(xy=(0, 0), radius=2, ec='C1', fc='none', ls='dashed'))
for b in bad:
hA = np.sqrt(np.sum(HA[b]**2))
hB = np.sqrt(np.sum(HB[b]**2))
h = min([hA, hB])
s = "frac_h: %.2f" % h
i, j, _ = dvecs[b] / .10992
t = ax.text(i, j + 2, s=s)
t.set_bbox(dict(facecolor='w', alpha=0.3, edgecolor='w'))
plt.plot(i, j, 'D', mec='C2', mfc='none', mew=2, ms=10)
#ax.add_patch(plt.Circle(xy=(i,j), radius=.5, ec='C2', fc='none',lw=2))
plt.xlabel("$\Delta_X$ (pixels)", fontsize=18)
plt.ylabel("$\Delta_Y$ (pixels)", fontsize=18)
ax.tick_params(labelsize=15)
import h5py
import os
from dxtbx.model.crystal import Crystal
from dxtbx.model.experiment_list import ExperimentList
from dxtbx.model.experiment_list import ExperimentList, ExperimentListFactory
from copy import deepcopy
from cxid9114 import utils
print("Loading pandas pickle")
all_df = pandas.concat([pandas.read_pickle(f) for f in glob.glob(args.glob)])
#df = pandas.read_pickle("refine_kaladin_2_gpu_refined_3.pkl")
print("I found data on %d experiments" % len(all_df))
is_bad_expt = utils.is_outlier(all_df.image_corr, args.thresh)
df = all_df.loc[~is_bad_expt]
df_bad = all_df.loc[is_bad_expt]
print("I removed %d experiments that were apparently diverging" % len(df_bad))
print("Average model correlation of kept experiments= %.3f" %
df.image_corr.mean())
print("Average model correlation of removed experiments= %.3f" %
df_bad.image_corr.mean())
print("Loading Amatrices optimized from the agg file")
u_proc_fnames = {f: h5py.File(f, 'r') for f in df.proc_fnames.unique()}
#Amat = { fname: h5["Amatrices_preopt2"] for fname, h5 in u_proc_fnames.items()}
img_paths = {fname: h5["h5_path"] for fname, h5 in u_proc_fnames.items()}
print("Adding basenames to the dataframe ")
def filter_outliers(dhkl, use_median=True, thresh=2.5, fit_gauss=True,
nsig=3, gain_key="gain"):
# reflections where A channel scattering only is present
dAnotB = dhkl.query("PA > 0 and PB == 0")
I_AnotB = dAnotB.D / dAnotB[gain_key] / dAnotB.LA / dAnotB.PA * dAnotB.K
# reflections where B channel scattering only present
dBnotA = dhkl.query("PA == 0 and PB > 0")
I_BnotA = dBnotA.D / dBnotA[gain_key] / dBnotA.LB / dBnotA.PB * dBnotA.K
# reflections where both A and B channels present
# here we assume half the scattering is A channel and the other half B channel, only
# need to be approximate here as we are guessing the form factor value we will refine
p = dhkl.query("PA > 0 and PB > 0")
valsA = p.D/p.LA/p.PA*p.K / p[gain_key]/2
valsB = p.D/p.LB/p.PB*p.K / p[gain_key]/2
# combine all estimates of the form factor
all_vals = np.hstack([I_AnotB, I_BnotA, valsA, valsB])
all_vals2 = np.sqrt(all_vals)
# remove outliers using median absolute deviation filter
outliers = is_outlier(all_vals2, thresh)
# now combine the inlier rows into a new dataframe
N_AnotB = len(I_AnotB)
out_AnotB = outliers[:N_AnotB]
N_BnotA = len(I_BnotA)
out_BnotA = outliers[N_AnotB: N_AnotB + N_BnotA]
n = N_AnotB + N_BnotA
out_AandB_1 = outliers[n: n + len(valsA)]
out_AandB_2 = outliers[n + len(valsA):]
out_AandB = np.logical_or(out_AandB_1, out_AandB_2)
# sanity check
#n1 = out_AnotB.sum() + out_BnotA.sum() + out_AandB_1.sum() + out_AandB_2.sum()
#n2 = outliers.sum()
#assert (n1 == n2), "%d , %d" % (n1, n2)
good_vals = all_vals2[~outliers]
if use_median:
best_val = np.median(good_vals)**2
else:
best_val = np.mean(good_vals)**2
d1 = dAnotB.loc[~out_AnotB]
I1 = I_AnotB[~out_AnotB].values
d2 = dBnotA.loc[~out_BnotA]
I2 = I_BnotA[~out_BnotA].values
o3 = ~out_AandB
d3 = p.loc[o3]
I3 = valsA[o3].values*.5 + valsB[o3].values*.5
dhkl_filt = pandas.concat([d1, d2, d3])
dhkl_filt['Iestimate'] = np.concatenate([I1, I2, I3])
if fit_gauss:
# fit a Gaussian to good_vals
mu = np.median(good_vals)
sig = np.std(good_vals)
bins = np.linspace(mu - nsig * sig, mu + nsig * sig, len(dhkl_filt) / 2)
xdata = .5 * bins[1:] + .5 * bins[:-1]
ydata, _ = np.histogram(good_vals, bins=bins)
try:
amp = ydata.max()
pFit, cov = curve_fit(Gauss, xdata, ydata, p0=(amp, mu, sig))
_, muFit, sigFit = pFit
W1 = abs(np.sqrt(I1) - muFit) / sigFit
W2 = abs(np.sqrt(I2) - muFit) / sigFit
W3 = abs(np.sqrt(I3) - muFit) / sigFit
from IPython import embed
embed()
except (RuntimeError, TypeError, ValueError):
pFit, cov = None, None
W1 = np.zeros_like(I1)
W2 = np.zeros_like(I2)
W3 = np.zeros_like(I3)
else:
pFit = cov = None
W1 = np.zeros_like(I1)
W2 = np.zeros_like(I2)
W3 = np.zeros_like(I3)
dhkl_filt['weights'] = np.concatenate([W1, W2, W3])
#dhkl_filt.weights /= dhkl_filt.weights.max()
return best_val, dhkl_filt, pFit, cov