def FitAtmosAndGround(self, tod, az, el, niter=100):
# Fit gradients
dlength = tod.size
templates = np.ones((3, tod.size))
templates[0, :] = az
if np.abs(np.max(az) - np.min(az)) > 180:
high = templates[0, :] > 180
templates[0, high] -= 360
templates[0, :] -= np.median(templates[0, :])
templates[1, :] = 1. / np.sin(el * np.pi / 180.)
a_all = np.zeros((niter, templates.shape[0]))
for a_iter in range(niter):
sel = np.random.uniform(low=0, high=dlength,
size=dlength).astype(int)
cov = np.sum(templates[:, None, sel] * templates[None, :, sel],
axis=-1)
z = np.sum(templates[:, sel] * tod[None, sel], axis=1)
try:
a_all[a_iter, :] = np.linalg.solve(cov, z).flatten()
except:
a_all[a_iter, :] = np.nan
fits, errs = np.nanmedian(a_all, axis=0), stats.MAD(a_all, axis=0)
tod_filter = np.sum(templates[:, :] * fits[:, None], axis=0)
return tod_filter, fits, errs
def smooth_gains(obsid, gains, errs):
"""
Smooth the calibration functions over all time to get the best signal-to-noise
"""
minobsid = np.min(obsid)
maxobsid = np.max(obsid)
count = int(maxobsid) - int(minobsid) + 1
allobs = np.zeros(count)
allerrs = np.zeros(count)
allgain = np.zeros(count)
idx = (obsid - minobsid).astype(int)
allobs[idx] = obsid
allgain[idx] = gains
allerrs[idx] = errs
idx_sort = np.argsort(allobs)
#allobs = np.arange(minobsid,maxobsid+1)
allgain = allgain[idx_sort]
allobs = allobs[idx_sort]
allerrs = allerrs[idx_sort]
med = np.nanmedian(gains)
bd = (allgain < 0.5) | (allgain > 1) | (np.abs(allgain - med) >
2) | (allerrs > 1e-3)
allgain[bd] = np.nan
gd = np.isfinite(allgain)
#pyplot.errorbar(allobs[gd],allgain[gd],fmt='.',yerr=allerrs[gd],capsize=3)
try:
allgain[~gd] = np.interp(allobs[~gd], allobs[gd], allgain[gd])
allgain_mdl = filters.median(allgain, filters.window('boxcar', 51))
except ValueError:
return allobs, allgain
rms = stats.MAD(allgain[gd] - allgain_mdl[gd])
bd = (allgain < 0.5) | (allgain > 1) | (np.abs(allgain - med) > 2) | (
allerrs > 1e-3) | (np.abs(allgain - allgain_mdl) > 3 * rms)
allgain[bd] = np.nan
gd = np.isfinite(allgain)
try:
allgain[~gd] = np.interp(allobs[~gd], allobs[gd], allgain[gd])
allgain = filters.median(allgain, filters.window('boxcar', 51))
except ValueError:
pass
return allobs, allgain
def construct_calibration_table(self):
"""
0) If calibration table exists and no overwrite - read and skip
1) Find all astro calibration files.
2) Transform each source measurement into Gain factors
3*) <Correct for atmospheric absorption if not using vane>
4) Store all gain factors per day into a file
"""
#if os.path.exists(self.calibration_table_file) & (not self.overwrite):
## self.table_data = h5py.File(self.calibration_table_file,'r')
# return
# Get files in correct format
testfile = 'test_source_data.npy'
if not os.path.exists(testfile):
gains = self.build_gains()
np.save(testfile, [gains])
else:
gains = np.load(testfile, allow_pickle=True).flatten()[0]
# Flatten Gains into days
allgains = np.concatenate([v['gain'] for k, v in gains.items()])
alltimes = np.concatenate([v['mjd'] for k, v in gains.items()])
allfrequency = np.concatenate(
[v['frequency'] for k, v in gains.items()])
allobsid = np.concatenate([v['obsid'] for k, v in gains.items()])
allcodes = np.concatenate([v['sourcecode'] for k, v in gains.items()])
print(allfrequency.shape)
# Remove bad fits
nObs, nFeed, nBand = allgains.shape
meds = np.zeros((nFeed, nBand))
mads = np.zeros((nFeed, nBand))
scale = 3
pyplot.figure(figsize=(20, 20))
for ifeed in range(nFeed):
for iband in range(nBand):
meds[ifeed, iband] = np.nanmedian(allgains[:, ifeed, iband])
mads[ifeed, iband] = stats.MAD(allgains[:, ifeed, iband])
bad = np.abs(allgains[:, ifeed, iband] -
meds[ifeed, iband]) > mads[ifeed, iband] * scale
allgains[bad, ifeed, iband] = np.nan
if ifeed < 5:
ax1 = pyplot.subplot(2, 2, 1)
pyplot.plot(allfrequency[0, :],
meds[ifeed, :],
'.',
label=(ifeed + 1))
elif (ifeed >= 5) & (ifeed < 10):
ax2 = pyplot.subplot(2, 2, 2)
pyplot.plot(allfrequency[0, :],
meds[ifeed, :],
'.',
label=(ifeed + 1))
elif (ifeed >= 10) & (ifeed < 15):
ax3 = pyplot.subplot(2, 2, 3)
pyplot.plot(allfrequency[0, :],
meds[ifeed, :],
'.',
label=(ifeed + 1))
elif ifeed >= 15:
ax4 = pyplot.subplot(2, 2, 4)
pyplot.plot(allfrequency[0, :],
meds[ifeed, :],
'.',
label=(ifeed + 1))
for ax in [ax1, ax2, ax3, ax4]:
pyplot.sca(ax)
pyplot.legend(loc='upper left', prop={'size': 10})
pyplot.ylim(0.6, 0.9)
pyplot.grid()
pyplot.xlabel('Frequency (GHz)')
pyplot.ylabel('Gain')
pyplot.savefig('figures/AstroCal/average_gains.png')
pyplot.clf()
# Write a table to AstroCal directory
self.write_gain_table(alltimes, allgains, allfrequency, allobsid,
allcodes)
#def plots(self):
nFeed = 20
nBand = 8
colors = ['C1', 'C2', 'C3', 'C4']
for ifeed in range(nFeed):
fig = pyplot.figure(figsize=(20, 12))
for iband in range(nBand):
pyplot.subplot(2, 4, 1 + iband)
data = []
labels = []
for i, (k, v) in enumerate(gains.items()):
data += [v['gain'][:, ifeed, iband]]
labels += [k]
pyplot.hist(data,
30,
range=[0.5, 1.5],
density=True,
stacked=True,
label=labels,
color=colors)
data = np.concatenate(data)
pyplot.axvline(np.nanmedian(data), color='k', ls='--', lw=2)
pyplot.legend()
pyplot.suptitle(f'Feed: {ifeed+1}')
pyplot.tight_layout()
pyplot.savefig(f'figures/AstroCal/{ifeed+1:02d}_histogram.png')
pyplot.close(fig)
fig = pyplot.figure()
for ifeed in range(nFeed):
fig = pyplot.figure(figsize=(20, 12))
for iband in range(nBand):
pyplot.subplot(2, 4, 1 + iband)
for i, (k, v) in enumerate(gains.items()):
pyplot.errorbar(v['time'],
v['gain'][:, ifeed, iband],
fmt='.',
capsize=3,
color=colors[i],
yerr=v['error'][:, ifeed, iband],
label=k)
pyplot.legend()
pyplot.ylim(0.5, 1.5)
fig.autofmt_xdate()
pyplot.title(iband)
pyplot.grid()
pyplot.suptitle('Feed: {ifeed+1}')
pyplot.savefig(f'figures/AstroCal/{ifeed+1:02d}_timeline.png')
pyplot.close(fig)
def __call__(self,
P0_dict,
xy,
z,
covariance,
P0_priors={},
limfunc=None,
nwalkers=32,
samples=5000,
discard=100,
thin=15,
return_array=False):
normalise = 1 #np.max(z)
z /= normalise
covariance /= normalise**2
self.P0_priors = P0_priors
if isinstance(limfunc, type(None)):
self.limfunc = self.limfunc
else:
self.limfunc = limfunc
P0 = [v for k, v in P0_dict.items()]
self.idx = {k: i for i, k in enumerate(P0_dict.keys())}
if self.use_bootstrap:
self.niter = 100
results = np.zeros((self.niter, self.nparams))
for i in tqdm(range(self.niter)):
sel = np.random.uniform(low=0, high=z.size,
size=z.size).astype(int)
results[i] = minimize(self.minimize_errfunc,
P0,
args=((xy[0][sel], xy[1][sel]), z[sel],
covariance[sel]),
method='CG').x
error = stats.MAD(results, axis=0)
result = np.nanmedian(results, axis=0)
elif self.use_leastsqs:
result = minimize(self.minimize_errfunc,
P0,
args=(xy, z, covariance),
method='CG').x
error = result * 0.
flat_samples = np.zeros(1)
min_chi2 = self.emcee_errfunc(result, xy, z, covariance)
ddof = len(z)
else:
# Perform the least-sqaures fit
result = minimize(self.minimize_errfunc,
P0,
args=(xy, z, covariance),
method='Nelder-Mead')
pos = result.x + 1e-4 * np.random.normal(size=(nwalkers,
len(result.x)))
sampler = emcee.EnsembleSampler(nwalkers,
len(result.x),
self.emcee_errfunc,
args=(xy, z, covariance))
sampler.run_mcmc(pos, samples, progress=True)
flat_samples = sampler.get_chain(discard=discard,
thin=thin,
flat=True)
result = np.nanmedian(flat_samples, axis=0)
error = stats.MAD(flat_samples, axis=0)
zchi = ((flat_samples - result[None, :]) / error[None, :])**2
zchi = np.max(zchi, axis=1)
gd = (zchi < 15)
flat_samples = flat_samples[gd, :]
result = np.nanmedian(flat_samples, axis=0)
error = stats.MAD(flat_samples, axis=0)
min_chi2 = self.emcee_errfunc(result, xy, z, covariance)
ddof = len(z)
z *= normalise
covariance *= normalise**2
Value_dict = {k: result[i] for k, i in self.idx.items()}
Error_dict = {k: error[i] for k, i in self.idx.items()}
for k in ['A', 'B']:
Value_dict[k] *= normalise
Error_dict[k] *= normalise
if return_array:
return result, error, flat_samples, min_chi2, ddof
else:
return Value_dict, Error_dict, flat_samples, min_chi2, ddof