def measure_peak_dist(fp_images, det_size=3e-3):
allbright = []
distances = []
centroids = []
for i, fp in enumerate(fp_images):
# Define a threshold
x = fp[~np.isnan(fp)] # Remove NAN values
mean, std = ft.meancut(x, nsig=7)
# std = np.nanstd(quart_fp)
threshold = 3 * std
print('threshold: ', threshold)
# Keep only pixels above threshold
bright = fp > threshold
brightpixels = np.column_stack(np.where(bright.T))
# Apply DBSCAN
clustering = DBSCAN(eps=1, min_samples=3).fit(brightpixels)
labels = clustering.labels_
nfound = len(np.unique(np.sort(labels)))
print('nfound: ', nfound)
if nfound == 3:
allbright.append(bright)
centers = np.zeros((nfound, 2))
for k in range(nfound):
ok = labels == k
centers[k, :] = np.mean(brightpixels[ok, :], axis=0)
centroids.append(centers)
# Distance calculation
a = np.sqrt((centers[0, 0] - centers[1, 0])**2 +
(centers[0, 1] - centers[1, 1])**2) * det_size
distances.append(a)
return allbright, np.array(centroids), distances
for asic in [1, 2]:
a.read_qubicstudio_dataset(thedir, asic=asic)
bar = progress_bar(128, 'ASIC #{}'.format(AsicNum))
for tes in np.arange(128) + 1:
t_data, data, FREQ_SAMPLING, freq_f, spectrum_f, amp_peak, okfit, res = get_amp_first_harmonic(
a, tes, freq_mod)
bar.update()
tes_index = (tes - 1) + 128 * (asic - 1)
allres[tes_index, :] = res[1]
allerr[tes_index, :] = res[2]
allamp_peak[tes_index] = amp_peak
amps = allres[:, 2]
# amps = allamp_peak
img_one = ft.image_asics(all1=amps)
mm, ss = ft.meancut(amps, 3)
imshow(img_one, vmin=0, vmax=mm + 3 * ss, interpolation='nearest')
colorbar()
# ============== Loop on all files =================
allres_tot = np.zeros((len(dirs), 256, 4))
allerr_tot = np.zeros((len(dirs), 256, 4))
allamp_peak_tot = np.zeros((len(dirs), 256))
for idir in range(len(dirs)):
thedir = dirs[idir]
for asic in [1, 2]:
a.read_qubicstudio_dataset(thedir, asic=asic)
bar = progress_bar(128, 'file #{0} ASIC #{1}'.format(idir, AsicNum))
for tes in np.arange(128) + 1:
t_data, data, FREQ_SAMPLING, freq_f, spectrum_f, amp_peak, okfit, res = get_amp_first_harmonic(
def do_some_dets(detnums,
d,
p,
directory,
fittedpeakfile,
az,
el,
proj_name,
custom=False,
q=None,
nside=None,
tol=5e-3,
refit=False,
resample=False,
newsize=70,
doplot=True,
verbose=True,
sbfitmodel=None,
angs=None,
usepeaks=None,
azmin=None,
azmax=None,
remove=None,
fitted_directory=None,
weighted=False,
nf_sub_rec=1,
lowcut=1e-3,
highcut=0.3):
if nside is not None:
d['nside'] = nside
s = qubic.QubicScene(d)
if q == None:
q = qubic.QubicMultibandInstrument(d)
if len(q) == 1:
xgrf, ygrf, FP_index, index_q = sc.get_TES_Instru_coords(q,
frame='GRF',
verbose=False)
else:
xgrf, ygrf, FP_index, index_q = sc.get_TES_Instru_coords(q[0],
frame='GRF',
verbose=False)
# Create TES, index and ASIC numbers assuming detnums is continuos TES number (1-248).
tes, asic = np.zeros((len(detnums), ), dtype=int), np.zeros(
(len(detnums), ), dtype=int)
qpix = np.zeros((len(detnums), ), dtype=int)
for j, npix in enumerate(detnums):
tes[j], asic[j] = (npix, 1) if (npix < 128) else (npix - 128, 2)
qpix[j] = tes2pix(tes[j], asic[j]) - 1
if verbose:
print("DETNUM{} TES{} ASIC{} QPIX{}".format(
npix, tes[j], asic[j], qpix[j]))
#Center of FOV
azcen_fov = np.mean(az)
elcen_fov = np.mean(el)
#Directory where the maps are:
mapsdir = directory
#File where the fitted peaks are:
peaksdir = fittedpeakfile
if not custom:
if verbose:
print('')
print('Normal Reconstruction')
qcut = select_det(qubic.QubicMultibandInstrument(d), detnums)
#qcut = select_det(qubic.QubicMultibandInstrument(d),[145])
else:
if verbose:
print('')
print('Custom Reconstruction')
### Refit or not the locations of the peaks
### from the synthesized beam images
### First instantiate a jchinstrument (modified from instrument
### to be able to read peaks from a file)
print("Generating jchinstrument")
qcut = select_det(
jcinst.QubicMultibandInstrument(d,
peakfile=fittedpeakfile,
tes=tes,
asic=asic), qpix)
print("LEN(qcut) and detector", len(qcut),
np.shape(qcut[0].detector.center))
print("Generating jchinstrument"[::-1])
### In the present case, we use the peak measurements at 150 GHz
### So we assume its index is len(qcut)//2
id150 = len(qcut) // 2
nu = qcut[id150].filter.nu
synthbeam = qcut[id150].synthbeam
horn = getattr(qcut[id150], 'horn', None)
primary_beam = getattr(qcut[id150], 'primary_beam', None)
# Cosine projection with elevation center of the FOV considering symetric scan in azimuth for each elevation step
thecos = np.cos(np.radians(elcen_fov))
#Read map (flat or healpy) for each detector
print("TES ASIC", tes, asic)
filemaps = readmaps(directory,
tes,
asic,
az,
el,
p,
proj_name=proj_name,
nside=d['nside'],
verbose=verbose)
# Compute measured coordenates
allphis_M, allthetas_M, allvals_M = get_data_Mrefsyst(
detnums,
filemaps,
az,
el,
fitted_directory,
fittedpeakfile,
proj_name,
resample=resample,
newsize=newsize,
azmin=azmin,
azmax=azmax,
remove=remove,
sbfitmodel=sbfitmodel,
refit=refit,
verbose=verbose)
if doplot:
_plot_onemap(filemaps,
az,
el, [allphis_M, allthetas_M],
proj_name,
makerotation=False)
### Now we want to perform the rotation to go to boresight
### reference frame (used internally by QubicSoft)
if verbose:
print("Solid angles synthbeam = {:.3e}, QubicScene {:.3e}".format(
synthbeam.peak150.solid_angle, s.solid_angle))
allphis_Q, allthetas_Q, allvals_Q, numpeak = convert_M2Q(
detnums,
allphis_M,
allthetas_M,
allvals_M,
elcen_fov,
solid_angle=synthbeam.peak150.solid_angle,
nu=nu,
horn=horn,
solid_angle_s=s.solid_angle,
angs=angs,
verbose=verbose)
print("allphis_Q , allvals_Q shapes", np.shape(allphis_Q),
np.shape(allvals_Q))
if doplot:
plt.title("xy coord. of peaks in M and Q ref system")
plt.plot(np.cos(allphis_M[0]) * np.sin(allthetas_M[0]),
np.sin(allphis_M[0]) * np.sin(allthetas_M[0]),
'r+',
ms=10,
label="M ref syst")
plt.plot(np.cos(allphis_Q[0]) * np.sin(allthetas_Q[0]),
np.sin(allphis_Q[0]) * np.sin(allthetas_Q[0]),
'b+',
ms=10,
label="Q ref syst")
plt.legend()
plt.show()
if doplot:
# Plot peaks in measured reference frame.
_plot_grfRF(detnums, xgrf, ygrf, qcut, numpeak)
for idet in range(len(detnums)):
position = np.ravel(qcut[0].detector[idet].center)
position = -position / np.sqrt(np.sum(position**2))
theta_center = np.arcsin(np.sqrt(position[0]**2 + position[1]**2))
phi_center = np.arctan2(position[1], position[0])
rav_thQ = np.ravel(allthetas_Q[idet, :])
rav_phQ = np.ravel(allphis_Q[idet, :])
## Now we identify the nearest peak to the theoretical Line Of Sight
angdist = np.zeros(len(rav_phQ))
for k in range(len(rav_phQ)):
angdist[k] = sbfit.ang_dist([theta_center, phi_center],
[rav_thQ[k], rav_phQ[k]])
print(k, np.degrees(angdist[k]))
idxmin = np.argmin(angdist)
numpeak[idet] = idxmin
### We nowwrite the temporary file that contains the peaks locations to be used
if usepeaks is None:
peaknums = np.arange(9)
else:
peaknums = usepeaks
data = [
allthetas_Q[:, peaknums], allphis_Q[:, peaknums] - np.pi,
allvals_Q[:, peaknums], numpeak
]
file = open(os.environ['QUBIC_PEAKS'] + 'peaks.pk', 'wb')
pickle.dump(data, file)
file.close()
### Make the TODs from the measured synthesized beams
realTOD = np.zeros((len(detnums), len(p)))
sigmaTOD = np.zeros(len(detnums))
if weighted:
sumweight = 0.
allimg = []
for i in range(len(detnums)):
if proj_name == "flat":
filename = directory + 'flat_ASIC{}TES{}.fits'.format(
asic[i], tes[i])
img = FitsArray(filename)
elif proj_name == "healpix":
filename = directory + '../../Flat/synth_beam/flat_ASIC{}TES{}.fits'.format(
asic[i], tes[i])
img = FitsArray(filename)
allimg.append(img)
fact = 1 #5e-28
realTOD[i, :] = np.ravel(img) * fact
if weighted: ## Not to be used - old test...
realTOD[i, :] *= 1. / ss**2
sumweight += 1. / ss**2
print("All img", np.shape(allimg))
### new code multiband
plt.figure()
for i in range(len(detnums)):
plt.plot(realTOD[i, :], label='TES#{0:}'.format(detnums[i]))
plt.legend()
plt.xlabel('Samples')
plt.ylabel('TOD')
plt.show()
plt.figure()
for i in range(len(detnums)):
spectrum_f, freq_f = ft.power_spectrum(np.arange(len(realTOD[i, :])),
realTOD[i, :])
pl = plt.plot(freq_f,
spectrum_f,
label='TES#{0:}'.format(detnums[i]),
alpha=0.5)
plt.xscale('log')
plt.yscale('log')
plt.legend()
plt.xlabel('Fourier mode')
plt.ylabel('Power Spectrum')
plt.title('Before Filtering')
plt.show()
for i in range(len(detnums)):
realTOD[i, :] = ft.filter_data(np.arange(len(realTOD[i, :])),
realTOD[i, :], lowcut, highcut)
mm, ss = ft.meancut(realTOD[i, :], 3)
sigmaTOD[i] = ss
if doplot:
for i in range(len(detnums)):
plt.figure()
plt.subplot(1, 2, 1)
plt.imshow(allimg[i] * fact,
vmin=mm - 3 * ss,
vmax=mm + 3 * ss,
extent=[
np.max(az) * thecos,
np.min(az) * thecos,
np.min(el),
np.max(el)
],
aspect='equal')
plt.colorbar()
plt.title('Init - TOD {0:} RMS={1:5.2g}'.format(
detnums[i], sigmaTOD[i]))
plt.subplot(1, 2, 2)
plt.imshow(np.reshape(realTOD[i, :], np.shape(img)),
vmin=mm - 3 * ss,
vmax=mm + 3 * ss,
extent=[
np.max(az) * thecos,
np.min(az) * thecos,
np.min(el),
np.max(el)
],
aspect='equal')
plt.colorbar()
plt.title('Filtered - TOD {0:} RMS={1:5.2g}'.format(
detnums[i], sigmaTOD[i]))
plt.figure()
for i in range(len(detnums)):
spectrum_f, freq_f = ft.power_spectrum(np.arange(len(realTOD[i, :])),
realTOD[i, :])
pl = plt.plot(freq_f,
spectrum_f,
label='TES#{0:} Var*2pi={1:5.2g}'.format(
detnums[i], sigmaTOD[i]**2 * 2 * np.pi),
alpha=0.5)
plt.plot(freq_f,
freq_f * 0 + sigmaTOD[i]**2 * 2 * np.pi,
color=pl[0].get_color())
plt.xscale('log')
plt.yscale('log')
plt.ylim(np.min(sigmaTOD**2 * 2 * np.pi),
np.max(sigmaTOD**2 * 2 * np.pi) * 1e9)
plt.legend()
plt.xlabel('Fourier mode')
plt.ylabel('Power Spectrum')
plt.title('After Filtering')
if lowcut:
plt.axvline(x=lowcut, color='k')
if highcut:
plt.axvline(x=highcut, color='k')
plt.show()
plt.figure()
plt.clf()
print('%%%%%%%%%%%%%%%%%%%%%%')
ax = plt.subplot(111, projection='polar')
print("len qcut", len(qcut[0].detector.center))
print("=?===============================")
print(np.shape(realTOD), np.shape(p), nf_sub_rec, np.shape(qcut))
print("=?===============================")
maps_recon, cov, nus, nus_edge = si.reconstruct_maps(
realTOD,
d,
p,
nf_sub_rec,
x0=None,
instrument=qcut, #verbose=True,
forced_tes_sigma=sigmaTOD)
ax.set_rmax(0.5)
#legend(fontsize=8)
if weighted:
maps_recon /= sumweight / len(detnums)
return maps_recon, qcut, np.mean(cov, axis=0), nus, nus_edge
def fit_sb(flatmap_init,
az_init,
el_init,
model,
newsize=70,
dmax=5.,
az_center=0.,
el_center=50.,
doplot=False,
resample=False,
verbose=False,
extra_title='',
return_fitted=False,
precision=None,
nsiglo=1.,
nsighi=3.,
figsave=None):
#### If requested, resample the iage in order to speedup the fitting
if resample:
### Resample input map to have less pixels to deal with for fitting
flatmap = scsig.resample(scsig.resample(flatmap_init, newsize, axis=0),
newsize,
axis=1)
az = np.linspace(np.min(az_init), np.max(az_init), newsize)
el = np.linspace(np.min(el_init), np.max(el_init), newsize)
else:
flatmap = flatmap_init
az = np.array(az_init)
el = np.array(el_init)
az2d, el2d = np.meshgrid(az * np.cos(np.radians(el_center)), np.flip(el))
# if verbose:
# print('fit_sb: Model Name = ',model.name)
# print('Initial Parameters:')
# model.print_start()
### First find the location of the maximum closest to the center
distance_max = dmax
mask = np.array(
np.sqrt((az2d - az_center)**2 +
(el2d - el_center)**2) < distance_max).astype(int)
### We use a smoothed version of the map in order to avoid glitches
smoothed_flatmap = f.gaussian_filter(flatmap, 2.)
wmax = np.where((smoothed_flatmap * mask) == np.max(smoothed_flatmap *
mask))
maxval = flatmap[wmax][0]
# print('Maximum of map is {0:5.2g} and was found at: az={1:5.2f}, el={2:5.2f}'.format(maxval,az2d[wmax][0], el2d[wmax][0]))
### Get the fixed parameters
fixpars = model.fixpars
### Create range for the fitting around the initial values
ranges = model.ranges.T
### Update initial parameters with the values found on the map
parsinit = model.startpars
parsinit[0] = az2d[wmax][0]
parsinit[1] = el2d[wmax][0]
parsinit[9] = maxval
if model.name == 'SbModelIndepPeaksAmp':
parsinit[9:] = maxval
ranges[9:, 1] = maxval * 2
elif model.name == 'SbModelIndepPeaksAmpFWHM':
for i in range(model.npeaks):
parsinit[9 + 2 * i] = maxval
ranges[9 + 2 * i, 1] = maxval * 2
elif model.name == 'SbModelIndepPeaks':
for i in range(model.npeaks):
parsinit[9 + 4 * i] = maxval
ranges[9 + 4 * i, 1] = np.abs(maxval) * 2
### Run the fitting
x = [az2d, el2d]
mm, ss = ft.meancut(flatmap, 3)
if verbose:
print('Running Minuit with model: {}'.format(model.name))
model.print_start()
mychi2 = ft.MyChi2_nocov(x, np.ravel(flatmap),
np.zeros_like(np.ravel(flatmap)) + ss, model)
fit = ft.do_minuit(x,
np.ravel(flatmap),
np.zeros_like(np.ravel(flatmap)) + ss,
parsinit,
functname=model,
chi2=mychi2,
rangepars=ranges,
fixpars=fixpars,
force_chi2_ndf=False,
verbose=False,
nohesse=True,
precision=precision)
fitpars = fit[1]
fiterrs = fit[2]
### Get the peaks positions and amplitudes
themap, newxxyy = model(x, fitpars, return_peaks=True)
### Put the fitted amplitude values to zero when needed (when peak ouside the input image)
if model.name == 'SbModelIndepPeaksAmp':
xmin = np.min(az2d)
xmax = np.max(az2d)
ymin = np.min(el2d)
ymax = np.max(el2d)
for i in range(model.npeaks):
if (((newxxyy[0, i] < xmin) or (newxxyy[0, i] > xmax))
or ((newxxyy[1, i] < ymin) or (newxxyy[1, i] > ymax))):
fitpars[9 + i] = 0
fiterrs[9 + i] = 0
### Now get the map again with updated parameters
themap, newxxyy = model(x, fitpars, return_peaks=True)
elif model.name == 'SbModelIndepPeaksAmpFWHM':
xmin = np.min(az2d)
xmax = np.max(az2d)
ymin = np.min(el2d)
ymax = np.max(el2d)
for i in range(model.npeaks):
if (((newxxyy[0, i] < xmin) or (newxxyy[0, i] > xmax))
or ((newxxyy[1, i] < ymin) or (newxxyy[1, i] > ymax))):
fitpars[9 + 2 * i] = 0
fiterrs[9 + 2 * i] = 0
### Now get the map again with updated parameters
themap, newxxyy = model(x, fitpars, return_peaks=True)
elif model.name == 'SbModelIndepPeaks':
xmin = np.min(az2d)
xmax = np.max(az2d)
ymin = np.min(el2d)
ymax = np.max(el2d)
for i in range(model.npeaks):
if (((newxxyy[0, i] < xmin) or (newxxyy[0, i] > xmax))
or ((newxxyy[1, i] < ymin) or (newxxyy[1, i] > ymax))):
fitpars[9 + 4 * i] = 0
fiterrs[9 + 4 * i] = 0
if verbose:
print('===========================================================')
print('Fitted values:')
print('-----------------------------------------------------------')
for i in range(len(parsinit)):
print('{0:<20}: {1:>12.6f} +/- {2:>12.6f}'.format(
model.parnames[i], fitpars[i], fiterrs[i]))
print('-----------------------------------------------------------')
print('Residuals**2/pix : {0:^8.5g}'.format(
np.sum((flatmap - themap)**2) / np.size(flatmap)))
print('===========================================================')
if doplot:
rc('figure', figsize=(18, 4))
sh = np.shape(newxxyy)
mm, ss = ft.meancut(flatmap, 3)
subplot(1, 4, 1)
imshow(themap,
extent=[
np.min(az) * np.cos(np.radians(50)),
np.max(az) * np.cos(np.radians(50)),
np.min(el),
np.max(el)
],
vmin=mm - ss,
vmax=mm + 10 * ss)
colorbar()
title('Fitted Map ' + extra_title)
xlabel('Angle in Az direction [deg.]')
ylabel('Elevation [deg.]')
mm, ss = ft.meancut(flatmap - themap, 3)
subplot(1, 4, 2)
imshow(flatmap,
extent=[
np.min(az) * np.cos(np.radians(50)),
np.max(az) * np.cos(np.radians(50)),
np.min(el),
np.max(el)
],
vmin=mm - nsiglo * ss,
vmax=mm + nsighi * ss)
xlim(
np.min(az) * np.cos(np.radians(50)),
np.max(az) * np.cos(np.radians(50)))
ylim(np.min(el), np.max(el))
colorbar()
for i in range(sh[1]):
ax = plot(newxxyy[0, i], newxxyy[1, i], 'r.')
title('Input Map ' + extra_title)
xlabel('Angle in Az direction [deg.]')
ylabel('Elevation [deg.]')
subplot(1, 4, 3)
imshow(themap,
extent=[
np.min(az) * np.cos(np.radians(50)),
np.max(az) * np.cos(np.radians(50)),
np.min(el),
np.max(el)
],
vmin=mm - nsiglo * ss,
vmax=mm + nsighi * ss)
colorbar()
title('Fitted Map ' + extra_title)
xlabel('Angle in Az direction [deg.]')
ylabel('Elevation [deg.]')
subplot(1, 4, 4)
imshow(flatmap - themap,
extent=[
np.min(az) * np.cos(np.radians(50)),
np.max(az) * np.cos(np.radians(50)),
np.min(el),
np.max(el)
],
vmin=mm - nsiglo * ss,
vmax=mm + nsighi * ss)
colorbar()
title('Residual Map ' + extra_title)
xlabel('Angle in Az direction [deg.]')
ylabel('Elevation [deg.]')
tight_layout()
if figsave:
savefig(figsave)
show()
if return_fitted:
return fit, newxxyy, themap
else:
return fit, newxxyy
def Allplots(fib,
allparams,
allparams1,
allparams2,
okfinal,
okfinal1,
okfinal2,
asic,
med=False,
rng=None,
cmap='viridis'):
figure()
subplot(2, 2, 1)
mmt, sst = ft.meancut(allparams[okfinal, 1], 3)
hist(allparams[okfinal, 1],
range=[0, mmt + 4 * sst],
bins=15,
label='All ({}) '.format(okfinal.sum()) +
ft.statstr(allparams[okfinal, 1] * 1000, median=med) + ' ms',
alpha=0.5)
hist(allparams1[okfinal1, 1],
range=[0, mmt + 4 * sst],
bins=15,
label='Asic1 ({})'.format(okfinal1.sum()) +
ft.statstr(allparams1[okfinal1, 1] * 1000, median=med) + ' ms',
alpha=0.5)
hist(allparams1[okfinal2, 1],
range=[0, mmt + 4 * sst],
bins=15,
label='Asic2 ({})'.format(okfinal2.sum()) +
ft.statstr(allparams2[okfinal2, 1] * 1000, median=med) + ' ms',
alpha=0.5)
xlabel('Tau [sec]')
legend(fontsize=7, frameon=False)
title('Fib {} - Tau [s]'.format(fib))
subplot(2, 2, 2)
mma, ssa = ft.meancut(allparams[okfinal, 3], 3)
hist(allparams[okfinal, 3],
range=[0, mma + 4 * ssa],
bins=15,
label='All ({}) '.format(okfinal.sum()) +
ft.statstr(allparams[okfinal, 3], median=med) + ' nA',
alpha=0.5)
hist(allparams1[okfinal1, 3],
range=[0, mma + 4 * ssa],
bins=15,
label='Asic1 ({}) '.format(okfinal1.sum()) +
ft.statstr(allparams1[okfinal1, 3], median=med) + ' nA',
alpha=0.5)
hist(allparams1[okfinal2, 3],
range=[0, mma + 4 * ssa],
bins=15,
label='Asic2 ({}) '.format(okfinal2.sum()) +
ft.statstr(allparams2[okfinal2, 3], median=med) + ' nA',
alpha=0.5)
xlabel('Amp [nA]')
legend(fontsize=7, frameon=False)
title('Fib {} - Amp [nA]'.format(fib))
subplot(2, 2, 3)
imtau = ft.image_asics(data1=allparams1[:, 1], data2=allparams2[:, 1])
imshow(imtau,
vmin=0,
vmax=mmt + 4 * sst,
interpolation='nearest',
cmap=cmap)
title('Tau [s] - Fiber {}'.format(fib, asic))
colorbar()
subplot(2, 2, 4)
imamp = ft.image_asics(data1=allparams1[:, 3], data2=allparams2[:, 3])
imshow(imamp,
vmin=0,
vmax=mma + 4 * ssa,
interpolation='nearest',
cmap=cmap)
title('Amp [nA] - Fiber {}'.format(fib, asic))
colorbar()
tight_layout()
return