def run(self, data):
"""
Expects a level2 file structure to be passed.
"""
# First we need:
# 1) The TOD data
# 2) The feature bits to select just the observing period
# 3) Elevation to remove the atmospheric component
tod = np.nanmean(data['level1/spectrometer/band_average'][...], axis=1)
feeds = data['level1/spectrometer/feeds'][:]
nFeeds, nSamples = tod.shape
scan_edges = self.getGroup(data, data,
f'{self.level2}/Statistics/scan_edges')
self.flags = np.zeros(tod.shape).astype(bool)
for ifeed in range(nFeeds):
for (s, e) in scan_edges:
rtod = tod[ifeed, s:e] - self.median_filter(
tod[ifeed, s:e], self.medfilt_stepsize)
rms = stats.AutoRMS(rtod)
f = gaussian_filter1d(
(rtod > rms * self.sigma_clip_value).astype(np.float), 50)
f /= np.max(f)
self.flags[ifeed, s:e] = (f > 0.5)
def findHotCold(self, mtod):
"""
Find the hot/cold sections of the ambient load scan
"""
nSamps = mtod.size
# Assuming there is a big step we take mid-value
midVal = (np.nanmax(mtod)+np.nanmin(mtod))/2.
# Sort the tod...
mtodSort = np.sort(mtod)
mtodArgsort = np.argsort(mtod)
# Find the whitenoise rms
rms = stats.AutoRMS(mtod[:,None]).flatten()
# Where is it hot? where is it cold?
groupHot = (mtodSort-midVal) > rms*5
groupCold = (mtodSort-midVal) < rms*5
# Find indices where vane is in
hotTod = mtod[(mtodArgsort[groupHot])]
X = np.abs((hotTod - np.median(hotTod))/rms) < 1
if np.sum(X) == 0:
raise NoHotError('No hot load data found')
snips = int(np.min(np.where(X)[0])), int(np.max(np.where(X)[0]))
idHot = (mtodArgsort[groupHot])[X]
idHot = np.sort(idHot)
# Find indices where vane is out
coldTod = mtod[(mtodArgsort[groupCold])]
X = np.abs((coldTod - np.median(coldTod))/rms) < 1
if np.sum(X) == 0:
raise NoColdError('No cold load data found')
snips = int(np.min(np.where(X)[0])), int(np.max(np.where(X)[0]))
idCold = (mtodArgsort[groupCold])[X]
idCold = np.sort(idCold)
# Just find cold AFTER calvane
diffCold = idCold[1:] - idCold[:-1]
jumps = np.where((diffCold > 50))[0]
if len(jumps) == 0:
snip = 0
else:
snip = int(jumps[0])
idHot = np.sort(idHot)
idCold = np.sort(idCold[snip:])
idCold = idCold[idCold > min(idHot)]
return idHot, idCold
def FitPowerSpectrum(self, tod):
"""
Calculate the power spectrum of the data, fits a 1/f noise curve, returns parameters
"""
auto_rms = stats.AutoRMS(tod)
nu, ps, counts = self.PowerSpectrum(tod)
# Only select non-nan values
# You may want to increase min counts,
# as the power spectrum is non-gaussian for small counts
good = (counts > 50) & ((nu < 0.03) | (nu > 0.05))
args = (nu[good], ps[good], auto_rms / np.sqrt(counts[good]), auto_rms)
bounds = [[None, None], [-3, 0]]
P0 = [0, -1]
P1 = minimize(self.Error, P0, args=args, bounds=bounds)
return ps, nu, P1.x, auto_rms
def create_maps(self, data, tod, filters, sel):
"""
Bin maps into instrument frame centred on source
"""
mjd = data['level1/spectrometer/MJD'][:]
# We do Jupiter in the Az/El frame but celestial in sky frame
#if self.source.upper() == 'JUPITER':
az = data['level1/spectrometer/pixel_pointing/pixel_az'][:]
el = data['level1/spectrometer/pixel_pointing/pixel_el'][:]
N = az.shape[1] // 2 * 2
daz = np.gradient(az[0, :]) * 50.
daz = daz[sel]
az = az[:, sel]
el = el[:, sel]
cw = daz > 1e-2
ccw = daz < 1e-2
mjd = mjd[sel]
npix = self.Nx * self.Ny
temp_maps = {
'map': np.zeros(npix, dtype=np.float64),
'cov': np.zeros(npix, dtype=np.float64)
}
maps = {
'map':
np.zeros(
(tod.shape[0], tod.shape[1], tod.shape[2], self.Nx, self.Ny)),
'cov':
np.zeros(
(tod.shape[0], tod.shape[1], tod.shape[2], self.Nx, self.Ny))
}
feed_avg = {
'map': np.zeros((tod.shape[0], self.Nx, self.Ny)),
'cov': np.zeros((tod.shape[0], self.Nx, self.Ny))
}
scan_maps = {
'CW': {
'map': np.zeros((self.Nx, self.Ny)),
'cov': np.zeros((self.Nx, self.Ny))
},
'CCW': {
'map': np.zeros((self.Nx, self.Ny)),
'cov': np.zeros((self.Nx, self.Ny))
}
}
azSource, elSource, raSource, decSource = Coordinates.sourcePosition(
self.source, mjd, self.lon, self.lat)
self.src_el = np.mean(elSource)
self.src_az = np.mean(azSource)
for ifeed in tqdm(self.feedlist,
desc=f'{self.name}:create_maps:{self.source}'):
feed_tod = tod[ifeed, ...]
#if self.source.upper() == 'JUPITER':
x, y = Coordinates.Rotate(azSource, elSource, az[ifeed, :],
el[ifeed, :], 0)
pixels, pX, pY = self.getpixels(x, y, self.dx, self.dy, self.Nx,
self.Ny)
mask = np.ones(pixels.size, dtype=int)
for isb in range(tod.shape[1]):
for ichan in range(1, tod.shape[2] - 1): # Always skip edges
for k in temp_maps.keys():
temp_maps[k][:] = 0.
z = (feed_tod[isb, ichan, sel] -
filters[ifeed, isb, ichan])
mask[:] = 1
mask[(pixels == -1) | np.isnan(z) | np.isinf(z)] = 0
if np.sum(np.isfinite(z)) == 0:
continue
rms = stats.AutoRMS(z)
weights = {
'map': z.astype(np.float64) / rms**2,
'cov': np.ones(z.size) / rms**2
}
for k in temp_maps.keys():
binFuncs.binValues(temp_maps[k],
pixels,
weights=weights[k],
mask=mask)
maps[k][ifeed, isb, ichan,
...] = np.reshape(temp_maps[k],
(self.Ny, self.Nx))
feed_avg[k][ifeed,
...] += np.reshape(temp_maps[k],
(self.Ny, self.Nx))
if (ifeed == 0):
for (key, direction) in zip(['CW', 'CCW'], [cw, ccw]):
for k in temp_maps.keys():
temp_maps[k][:] = 0.
binFuncs.binValues(
temp_maps[k],
pixels[direction],
weights=weights[k][direction],
mask=mask[direction])
scan_maps[key][k] += np.reshape(
temp_maps[k], (self.Ny, self.Nx))
xygrid = np.meshgrid(
(np.arange(self.Nx) + 0.5) * self.dx - self.Nx * self.dx / 2.,
(np.arange(self.Ny) + 0.5) * self.dy - self.Ny * self.dy / 2.)
feed_avg['xygrid'] = xygrid
maps['xygrid'] = xygrid
feed_avg = self.average_maps(feed_avg)
for key in scan_maps.keys():
scan_maps[key] = self.average_maps(scan_maps[key])
scan_maps[key]['xygrid'] = xygrid
map_axes = np.array([a for a in maps['map'].shape])
map_axes[2] = int(map_axes[2] / self.binwidth)
map_axes = np.insert(map_axes, 3, self.binwidth)
maps['map'] = np.nansum(np.reshape(maps['map'], map_axes), axis=3)
maps['cov'] = np.nansum(np.reshape(maps['cov'], map_axes), axis=3)
maps = self.average_maps(maps)
self.map_freqs = np.mean(np.reshape(
data[f'{self.level2}/frequency'][...], map_axes[1:4]),
axis=-1)
return maps, feed_avg, scan_maps
def create_maps(self, data, tod, filters, sel):
"""
Bin maps into instrument frame centred on source
"""
mjd = data['spectrometer/MJD'][:]
# We do Jupiter in the Az/El frame but celestial in sky frame
#if self.source.upper() == 'JUPITER':
az = data['spectrometer/pixel_pointing/pixel_az'][:]
el = data['spectrometer/pixel_pointing/pixel_el'][:]
N = az.shape[1] // 2 * 2
daz = np.gradient(az[0, :]) * 50.
daz = daz[sel]
az = az[:, sel]
el = el[:, sel]
cw = daz > 1e-2
ccw = daz < 1e-2
mjd = mjd[sel]
npix = self.Nx * self.Ny
temp_maps = {
'map': np.zeros(npix, dtype=np.float64),
'cov': np.zeros(npix, dtype=np.float64)
}
maps = {
'maps': {
'map': np.zeros(
(tod.shape[0], tod.shape[1], self.Nx, self.Ny)),
'cov': np.zeros((tod.shape[0], tod.shape[1], self.Nx, self.Ny))
}
}
maps['feed_avg'] = {
'map': np.zeros((tod.shape[0], 1, self.Nx, self.Ny)),
'cov': np.zeros((tod.shape[0], 1, self.Nx, self.Ny))
}
maps['CW'] = {
'map': np.zeros((1, 1, self.Nx, self.Ny)),
'cov': np.zeros((1, 1, self.Nx, self.Ny))
}
maps['CCW'] = {
'map': np.zeros((1, 1, self.Nx, self.Ny)),
'cov': np.zeros((1, 1, self.Nx, self.Ny))
}
selections = {
k: selection
for k, selection in zip(maps.keys(), [
np.ones(az.shape[-1], dtype=bool),
np.ones(az.shape[-1], dtype=bool), cw, ccw
])
}
slices = {
k: sl
for k, sl in zip(maps.keys(), [
lambda ifeed, isb: [
slice(ifeed, ifeed + 1),
slice(isb, isb + 1),
slice(None),
slice(None)
], lambda ifeed, isb: [
slice(ifeed, ifeed + 1),
slice(None),
slice(None),
slice(None)
], lambda ifeed, isb:
[slice(None),
slice(None),
slice(None),
slice(None)], lambda ifeed, isb:
[slice(None),
slice(None),
slice(None),
slice(None)]
])
}
self.source_positions = {
k: a
for k, a in zip(['az', 'el', 'ra', 'dec'],
Coordinates.sourcePosition(self.source, mjd,
self.lon, self.lat))
}
self.source_positions['mean_el'] = np.mean(self.source_positions['el'])
self.source_positions['mean_az'] = np.mean(self.source_positions['az'])
for ifeed in tqdm(self.feedlist,
desc=f'{self.name}:create_maps:{self.source}'):
feed_tod = tod[ifeed, ...]
pixels = self.get_pixel_positions(self.source_positions['az'],
self.source_positions['el'],
az[ifeed, :], el[ifeed, :])
mask = np.ones(pixels.size, dtype=int)
for isb in range(tod.shape[1]):
for k in temp_maps.keys():
temp_maps[k][:] = 0.
z = (feed_tod[isb, sel] - filters[ifeed, isb])
mask[:] = 1
mask[(pixels == -1) | np.isnan(z) | np.isinf(z)] = 0
if np.sum(np.isfinite(z)) == 0:
continue
rms = stats.AutoRMS(z)
weights = {
'map': z.astype(np.float64) / rms**2,
'cov': np.ones(z.size) / rms**2
}
for k in temp_maps.keys():
for mode, map_data in maps.items():
if ('CW' in mode) & (ifeed > 1):
continue
binFuncs.binValues(
temp_maps[k],
pixels[selections[mode]],
weights=weights[k][selections[mode]],
mask=mask[selections[mode]])
maps[mode][k][slices[mode](ifeed, isb)] = np.reshape(
temp_maps[k], (self.Ny, self.Nx))
xygrid = np.meshgrid(
(np.arange(self.Nx) + 0.5) * self.dx - self.Nx * self.dx / 2.,
(np.arange(self.Ny) + 0.5) * self.dy - self.Ny * self.dy / 2.)
for k, v in maps.items():
maps[k] = self.average_maps(maps[k])
maps[k]['xygrid'] = xygrid
return maps
def run(self, data):
"""
Expects a level2 file structure to be passed.
"""
fname = data.filename.split('/')[-1]
# First we need:
# 1) The TOD data
# 2) The feature bits to select just the observing period
# 3) Elevation to remove the atmospheric component
tod = data[f'{self.level2}/averaged_tod'][...]
az = data['level1/spectrometer/pixel_pointing/pixel_az'][...]
el = data['level1/spectrometer/pixel_pointing/pixel_el'][...]
feeds = data['level1/spectrometer/feeds'][:]
bands = [
b.decode('ascii') for b in data['level1/spectrometer/bands'][:]
]
statistics = self.getGroup(data, data, f'{self.level2}/Statistics')
scan_edges = self.getGroup(data, statistics, 'scan_edges')
# Looping over Feed - Band - Channel, perform 1/f noise fit
nFeeds, nBands, nChannels, nSamples = tod.shape
#if 20 in feeds:
# nFeeds -= 1
nScans = len(scan_edges)
self.powerspectra = np.zeros((nFeeds, nBands, nScans, self.nbins))
self.freqspectra = np.zeros((nFeeds, nBands, nScans, self.nbins))
self.fnoise_fits = np.zeros((nFeeds, nBands, nScans, 3))
self.wnoise_auto = np.zeros((nFeeds, nBands, nChannels, nScans, 1))
self.atmos = np.zeros((nFeeds, nBands, nScans, 3))
self.atmos_errs = np.zeros((nFeeds, nBands, nScans, 3))
self.filter_tods = [] # Store as a list of arrays, one for each "scan"
self.filter_coefficients = np.zeros(
(nFeeds, nBands, nChannels, nScans,
1)) # Stores the per channel gradient of the median filter
self.atmos_coefficients = np.zeros(
(nFeeds, nBands, nChannels, nScans,
1)) # Stores the per channel gradient of the median filter
pbar = tqdm(total=(nFeeds * nBands * nChannels * nScans),
desc=self.name)
for iscan, (start, end) in enumerate(scan_edges):
local_filter_tods = np.zeros((nFeeds, nBands, end - start))
for ifeed in range(nFeeds):
if feeds[ifeed] == 20:
continue
for iband in range(nBands):
band_average = np.nanmean(tod[ifeed, iband, 3:-3,
start:end],
axis=0)
atmos_filter, atmos, atmos_errs = self.FitAtmosAndGround(
band_average, az[ifeed, start:end], el[ifeed,
start:end])
local_filter_tods[ifeed, iband, :] = self.median_filter(
band_average - atmos_filter)[:band_average.size]
self.atmos[ifeed, iband, iscan, :] = atmos
self.atmos_errs[ifeed, iband, iscan, :] = atmos_errs
ps, nu, f_fits, w_auto = self.FitPowerSpectrum(
band_average - atmos_filter -
local_filter_tods[ifeed, iband, :])
self.powerspectra[ifeed, iband, iscan, :] = ps
self.freqspectra[ifeed, iband, iscan, :] = nu
self.fnoise_fits[ifeed, iband, iscan, 0] = w_auto
self.fnoise_fits[ifeed, iband, iscan, 1:] = f_fits
#self.logger(f'{fname}:{self.name}: Feed {feeds[ifeed]} Band {bands[iband]} RMS - {w_auto:.3f}K')
#self.logger(f'{fname}:{self.name}: Feed {feeds[ifeed]} Band {bands[iband]} Knee - {f_fits[0]:.3f}')
#self.logger(f'{fname}:{self.name}: Feed {feeds[ifeed]} Band {bands[iband]} Spec - {f_fits[1]:.3f}')
for ichan in range(nChannels):
if np.nansum(tod[ifeed, iband, ichan, start:end]) == 0:
continue
# Check atmosphere coefficients
atmos_coeff, med_coeff, offset = self.coefficient_jointfit(
tod[ifeed, iband, ichan, start:end], atmos_filter,
local_filter_tods[ifeed, iband, :])
w_auto = stats.AutoRMS(tod[ifeed, iband, ichan,
start:end])
self.wnoise_auto[ifeed, iband, ichan,
iscan, :] = w_auto
self.filter_coefficients[ifeed, iband, ichan,
iscan, :] = med_coeff
self.atmos_coefficients[ifeed, iband, ichan,
iscan, :] = atmos_coeff
pbar.update(1)
self.filter_tods += [local_filter_tods]
pbar.close()