def __init__(self, calib_dir, offline=False, denom=1023):
self.denom = denom
try:
wgt = load_weights(calib_dir)
self.height, self.width = wgt.shape
self.weights = (wgt.astype("f4") * self.denom).astype("i4")
except IOError:
self.width, self.height = load_res(calib_dir)
self.weights = np.full((self.height, self.width),
self.denom,
dtype='i4')
print('Could not load lens-shading map')
try:
self.hot = load_hot(calib_dir, offline=offline)
hot_x = self.hot % self.width
hot_y = self.hot // self.width
self.weights[hot_y, hot_x] = 0
except IOError:
print('Could not load hotcells')
try:
_, self.blk_lvl, _ = load_electrons(calib_dir)
except IOError:
self.blk_lvl = 0
print('Could not load black level')
def find_dark(px, py, thresh=None, calib='calib', commit=False):
# load the image geometry from the calibrations:
width, height = load_res(calib)
fgain = np.load(os.path.join(calib, 'gain.npz'))
gain = fgain.f.gain.reshape(height, width)
rsq = fgain.f.rsq.reshape(height, width)
fgain.close()
# clean outliers
gain[gain < 0] = np.nan
# crop to fit the dark pixel period
crop_x = (width // px - 2) * px
crop_y = (height // py - 2) * py
idxx = (width - crop_x) // 2
idxy = (height - crop_y) // 2
gain_crop = gain[idxy:-idxy, idxx:-idxx]
gain_ds = gain_crop.reshape(crop_y // py, py, crop_x // px, px)
# normalize and center at 0
gain_norm = np.nanmean(gain_ds, axis=(1, 3))
gain_ds /= gain_norm.reshape(crop_y // py, 1, crop_x // px, 1)
gain_ds -= 1
gain_mean = np.nanmean(gain_ds, axis=(0, 2))
gain_std = np.nanstd(gain_ds, axis=(0, 2))
plt.figure(figsize=(12, 6))
plt.subplot(121)
plt.imshow(gain_mean, cmap='coolwarm')
plt.title('Mean')
plt.colorbar()
plt.subplot(122)
plt.imshow(gain_std, cmap='coolwarm')
plt.title('Standard deviation')
plt.colorbar()
plt.show()
if not thresh: return
dark_y, dark_x = np.argwhere(gain_mean < -thresh).transpose()
dark_x = (dark_x + idxx) % px
dark_y = (dark_y + idxy) % py
print(dark_x.size, 'dark pixels found')
if commit:
outname = os.path.join(calib, 'dark.npz')
print('saving to', outname)
np.savez(outname, dark_x=dark_x, dark_y=dark_y, px=px, py=py)
def load_dark(calib):
width, height = load_res(calib)
fdark = np.load(os.path.join(calib, 'dark.npz'))
dark_x = fdark['dark_x']
dark_y = fdark['dark_y']
px = fdark['px']
py = fdark['py']
fdark.close()
dark = np.zeros(width * height, dtype=bool)
index = np.arange(width * height)
x = index % width
y = index // width
for ix, iy in zip(dark_x, dark_y):
dark[(x % px == ix) & (y % py == iy)] = True
return dark
for z in args.lyso_z:
x.append(LYSO_X)
y.append(LYSO_Y)
zminus.append(z - LYSO_Z / 2)
zplus.append(z + LYSO_Z / 2)
track_thresh.append(args.lyso_thresh)
sig_xy.append(args.lyso_dxy)
sig_z.append(args.lyso_dz)
sig_thresh.append(args.lyso_thresh_err)
if not args.phone_z is None:
sys.path.insert(1, '../pixelstats')
from geometry import load_res
cmos_size = args.pix_pitch / 1000 * np.array(load_res(args.calib))
if args.rot:
x.append(cmos_size[0])
y.append(cmos_size[1])
else:
x.append(cmos_size[1])
y.append(cmos_size[0])
zminus.append(args.phone_z - 0.001)
zplus.append(args.phone_z + 0.001)
track_thresh.append(0)
sig_xy.append(args.phone_dxy)
sig_z.append(args.phone_dz)
sig_thresh.append(0)
x = np.array(x)
try:
# find first-order correction for varying saturation level
# from weighting, i.e. fraction of the sensor with saturation
# below each calibrated value
wgt = load_weights(args.calib)
print(wgt.shape)
npix = (wgt.shape[1] -
2 * args.xborder) * wgt.shape[0] if args.xborder else wgt.size
hist, _ = np.histogram(wgt * (1023 - blk_lvl) + 1,
bins=np.arange(1025 - blk_lvl))
cumsum = np.cumsum(hist)[::args.bin_sz]
sat_frac = cumsum[:-1] / cumsum[-1]
except FileNotFoundError:
print('Weights not found. Using equal weights.')
sat_frac = np.zeros((1023 - blk_lvl) // args.bin_sz)
width, height = load_res(args.calib)
npix = (width - 2 * args.xborder) * height
bins = np.arange(args.bin_sz, 1024 - blk_lvl, args.bin_sz)
f_src = r.TFile(args.src)
src = f_src.Get('triggers')
src_tot = src.GetEntries() + f_src.Get('nontriggers').GetEntries()
src_min = min([getattr(trig, args.stat)
for trig in src]) - blk_lvl * COUNTS[args.stat]
f_bg = r.TFile(args.bg)
bg = f_bg.Get('triggers')
bg_tot = bg.GetEntries() + f_bg.Get('nontriggers').GetEntries()
bg_min = min([getattr(trig, args.stat)
for trig in bg]) - blk_lvl * COUNTS[args.stat]
def downsample(calib, rsq_thresh=0, calib_dark=True, calib_hot=True, ds=4):
print('loading image geometry')
width, height = load_res(calib)
index = np.arange(width*height,dtype=int)
xpos = index % width
ypos = index / width
# load the pixel gains:
try:
print('loading pixel gains')
filename = os.path.join(calib, "gain.npz")
fgains = np.load(filename)
except:
print("could not process file ", filename, " as .npz file. Run gain.py with --commit option first?")
return
gain = fgains['gain']
intercept = fgains['intercept']
rsq = fgains['rsq']
fgains.close()
# set bad pixels to NaN
infs = np.isinf(gain) | np.isinf(intercept) | np.isnan(intercept)
gain[infs] = np.nan
intercept[infs] = np.nan
if rsq_thresh:
gain[rsq < rsq_thresh] = np.nan
intercept[rsq < rsq_thresh] = np.nan
if calib_dark:
try:
print('loading dark pixels')
dark = load_dark(calib)
gain[dark] = np.nan
intercept[dark] = np.nan
except IOError:
print("dark pixel file could not be processed")
if calib_hot:
from hot_pixels import load_hot
try:
print('loading hot pixels')
hot = load_hot(calib)
gain[hot] = np.nan
intercept[hot] = np.nan
except IOError:
print("hot pixel file could not be processed")
print('beginning to downsample')
nx = width // ds
ny = height // ds
# take mean of each n x n block
gain_ds = gain.reshape(ny, ds, nx, ds)
intercept_ds = intercept.reshape(ny, ds, nx, ds)
g_av = np.nanmedian(gain_ds, axis=(1,3))
b_av = np.nanmedian(intercept_ds, axis=(1,3))
print('downsampling completed, displaying information:')
print('nx: ', nx)
print('ny: ', ny)
print('lens.shape: ', g_av.shape)
print('NaN found: ', np.sum(np.isnan(g_av)))
print()
print('computed lens shading attributes')
return g_av, b_av
def process(filename, args):
# load data, from either raw file directly from phone, or as output from combine.py utility:
if args.raw:
version,header,sum,ssq,max,second = unpack_all(filename)
index = get_pixel_indices(header)
images = interpret_header(header, "images")
num = np.full(index.size, images)
width = interpret_header(header, "width")
height = interpret_header(header, "height")
else:
# load the image geometry from the calibrations:
width, height = load_res(args.calib)
try:
npz = np.load(filename)
except:
print("could not process file ", filename, " as .npz file. Use --raw option?")
return
sum = npz['sum']
ssq = npz['ssq']
num = npz['num']
cmean = sum / num
cvari = (ssq / num - cmean**2) * num / (num-1)
# apply gains if appropriate
if args.gain:
try:
wgt = load_weights(args.calib).flatten()
except IOError:
print("weights not found.")
return
cmean = cmean * wgt
cvari = cvari * wgt**2
# select pixels to plot
index = np.arange(sum.size)
xpos = index % width
ypos = index // width
rpos = np.sqrt((xpos - xpos.mean())**2 + (ypos - ypos.mean())**2)
keep = np.ones(width*height, dtype=bool)
if args.no_dark or args.all_dark:
try:
dark = load_dark(args.calib)
except IOError:
print("dark pixel file does not exist.")
return
if args.no_dark:
keep &= np.logical_not(dark)
if args.all_dark:
keep &= dark
if args.hot:
max_mean = args.hot[0]
max_vari = args.hot[1]
print("saving hot pixel list from mean > ", max_mean, " or var > ", max_vari)
hot = ((cmean > max_mean) + (cvari > max_vari))
hot_list = index[hot]
hotfile = os.path.join(args.calib, "hot_online.npz")
print("saving ", hot_list.size, "hot pixels to file ", hotfile)
print("total pixels in device: ", width * height)
frac = hot_list.size / (width*height)
print("faction of hot pixels: ", frac)
np.savez(hotfile, hot_list=hot_list)
keep &= (hot == False)
# first show spatial distribution
plt.figure(1, figsize=(6,8))
if args.by_radius:
plt.subplot(211)
plt.hist2d(rpos, cmean,norm=LogNorm(),bins=[500,500],range=[[0,rpos.max()],[0,args.max_mean]], cmap='seismic')
plt.xlabel('radius')
plt.ylabel('mean')
plt.subplot(212)
plt.hist2d(rpos, cvari,norm=LogNorm(),bins=[500,500],range=[[0,rpos.max()],[0,args.max_var]], cmap='seismic')
plt.xlabel('radius')
plt.ylabel('variance')
else:
plt.subplot(211)
plt.imshow(cmean.reshape(height, width),
cmap='seismic', vmax=args.max_mean)
plt.colorbar()
plt.subplot(212)
plt.imshow(cvari.reshape(height, width), #norm=LogNorm(),
cmap='seismic', vmax=args.max_var)
plt.colorbar()
# now do 2D histogram(s) for mean and variance
plt.figure(2, figsize=(10,8))
if args.by_filter:
# 4 subplots
for i in range(4):
ix = i % 2
iy = i // 2
pos = keep * ((xpos%2)==ix) * ((ypos%2)==iy)
plt.subplot(2,2,i+1)
plt.hist2d(cmean[pos],cvari[pos],norm=LogNorm(), bins=(500,500), range=((0,args.max_mean),(0, args.max_var)))
plt.xlabel("mean")
plt.ylabel("variance")
else:
plt.hist2d(cmean,cvari,norm=LogNorm(),bins=[500,500],range=[[0,args.max_mean],[0,args.max_var]])
plt.xlabel("mean")
plt.ylabel("variance")
if args.save_plot:
plot_name = "plots/mean_var_calib.pdf" if args.calib \
else "plots/mean_var.pdf"
print("saving plot to file: ", plot_name)
plt.savefig(plot_name)
plt.show()
return
h,bins = np.histogram(np.clip(cmean,0,10), bins=100, range=(0,10))
err = h**0.5
cbins = 0.5*(bins[:-1] + bins[1:])
plt.errorbar(cbins,h,yerr=err,color="black",fmt="o")
plt.ylim(1.0,1E7)
plt.ylabel("pixels")
plt.xlabel("mean")
plt.yscale('log')
plt.savefig("hmean.pdf")
plt.show()
h,bins = np.histogram(np.clip(cvari,0,100), bins=100, range=(0,100))
err = h**0.5
cbins = 0.5*(bins[:-1] + bins[1:])
plt.errorbar(cbins,h,yerr=err,color="black",fmt="o")
plt.ylim(1.0,1E7)
plt.ylabel("pixels")
plt.xlabel("variance")
plt.yscale('log')
plt.savefig("hvari.pdf")
plt.show()