def findOutliers(self, fracAmp=None, thrldPhase=None):
"""
@brief Find outliers within the analyzed detectors.
@param fracAmp Only accept amplitude values such that:
(1-fracAmp)*mAmp < Amp < (1+fracAmp)*mAmp
where mAmp = median(Amps)
@param thrldPhase Only accept phase values such that:
mPhase-thrldPhase < Phase < mPhase+thrldPhase
where mPhase = median(Phases)
"""
params = self.getDefaultInitParams(self.tod.info.array)
if fracAmp is None:
fracAmp = params['fracAmp']
if thrldPhase is None:
thrldPhase = params['thrldPhase']
self.mask = numpy.zeros(self.ndet)
zeroSel = (self.amp == 0.0)
self.mask[zeroSel] = -2
m_amp = numpy.median(self.amp[self.mask == 0])
m_phase = numpy.median(self.phase[self.mask == 0])
self.mask[self.amp > 1e5] = 1
for c in range(32):
sel = (self.col == c) * ~zeroSel
if len(self.col[sel]) > 2:
m = numpy.median(self.amp[sel])
self.mask[sel * (self.amp > m * (1 + fracAmp))] = 1
self.mask[sel * (self.amp < m * (1 - fracAmp))] = 1
elif (len(self.col[sel]) == 2) and \
((self.amp[sel][0] > self.amp[sel][1]*(1+fracAmp)) or \
(self.amp[sel][0] < self.amp[sel][1]*(1-fracAmp))):
if numpy.abs(self.amp[sel][0] -
m_amp) > numpy.abs(self.amp[sel][1] - m_amp):
self.mask[numpy.where(sel)[0][0]] = 1
else:
self.mask[numpy.where(sel)[0][1]] = 1
sel *= (self.mask == 0)
if len(self.col[sel]) > 2:
m = numpy.median(self.phase[sel])
self.mask[sel * (self.phase > m + thrldPhase)] = 2
self.mask[sel * (self.phase < m - thrldPhase)] = 2
elif (len(self.col[sel]) == 2) and \
((self.phase[sel][0] > self.phase[sel][1]+thrldPhase) or \
(self.phase[sel][0] < self.phase[sel][1]-thrldPhase)):
if numpy.abs(self.phase[sel][0] -
m_phase) > numpy.abs(self.phase[sel][1] -
m_phase):
self.mask[numpy.where(sel)[0][0]] = 2
else:
self.mask[numpy.where(sel)[0][1]] = 2
sel *= (self.mask == 0)
if not (numpy.any(self.mask[self.col == c] == 0)):
self.badcols.append(c)
psLib.trace("moby", 2,
"WARNING: No amplitudes left for COL " + str(c))
def noiseFilter(self, data=None):
psLib.trace( "moby", 3, "simpleFilter.noiseFilter begins")
if data is None:
data = tod.data
#d = self.tod.cuts.get_uncut()
moby2.tod.remove_mean(data=data)
moby2.tod.detrend_tod(data=data)
hpf = moby2.tod.TODFilter()
hpf.add(moby2.tod.filters.highPassButterworth,
{'fc':self.params['noiseHighPassFc']})
hpf.apply(self.tod)
def applyFilter(self, data=None):
"""
@brief Filter the TOD. Performs the pcg (1-C)^TN^{-1}(1-C) step.
"""
psLib.trace("moby", 2, "Filtering")
self.removeCorrelations(
data) # don't need to repeat 1-C b/c (1-C)^T(1-C) = 1-C
if self.params['filterNoise']:
psLib.trace("moby", 3, "Noise Filtering")
self.noiseFilter(data)
if not (self.params['useNoiseInCorr']):
self.removeCorrelations(data)
def get_sources_in_patch(ctime=None,
ra_lims=None,
dec_lims=None,
tod=None,
source_list=None,
add_source_list=None):
"""
Find sources in a region of RA and dec. The usual planets will be
checked, unless a list of particular objects is passed in
source_list. The RA and dec. limits will be determined from the
tod argument unless they are passed in explicitly (as (min,max)
tuples). Since these things tend not to move very fast, we do the
computation at a single ctime (which defaults to
tod.ctime.mean()).
Returns a list of matches, with each match a tuple (source_name, ra, dec).
"""
if tod is not None:
mask = tod.pointing_mask
if ra_lims is None or dec_lims is None:
wand_eq = moby2.pointing.ArrayWand.for_tod(tod, coords='ra_dec')
# Get ra, dec limits covered by this TOD. Just use the array center, buffered by ~.5 deg.
cos_dec0 = np.cos(wand_eq.dec.mean())
if ra_lims is None:
ra_lims = (wand_eq.ra[mask].min() - 0.5 * DEG / cos_dec0,
wand_eq.ra[mask].max() + 0.5 * DEG / cos_dec0)
if dec_lims is None:
dec_lims = (wand_eq.dec[mask].min() - 0.5 * DEG,
wand_eq.dec[mask].max() + 0.5 * DEG)
if ctime is None:
ctime = tod.ctime[mask].mean()
if source_list is None:
source_list = calibration_sources
if add_source_list is not None:
source_list.extend(add_source_list)
ephem = ACTEphem()
ephem.set_ctime(ctime)
matches = []
for source_name, source_type in source_list:
s_ra, s_dec = get_source_coords(source_name, ctime)
psLib.trace(
'moby', 3, 'Source = %s, RA = %f, Dec = %f' %
(source_name, s_ra / DEG, s_dec / DEG))
if angles.is_within(s_ra, ra_lims[0], ra_lims[1], 2*np.pi) and \
angles.is_within(s_dec, dec_lims[0], dec_lims[1], 2*np.pi):
matches.append(
(source_name, float(s_ra), float(s_dec), source_type))
return matches
def for_tod_source_coords(cls,
tod,
ref_coord=(0., 0.),
polarized=False,
warning_distance=10. / 60,
scan_coords=False,
hwp=None):
"""
Get ArrayWand for tod, in tangent plane coordinates centered
on the provided ref_coord=(ra,dec), and possibly including a
rotation that makes the X axis parallel to the scan direction.
A warning is issued if the TOD does not scan within
warning_distance (degrees) of the ref_coord.
"""
if isinstance(ref_coord, basestring):
ra_src, dec_src = moby2.ephem.get_source_coords(
ref_coord, tod.ctime.mean())
gamma_in = 0.
elif len(ref_coord) == 3:
ra_src, dec_src, gamma_in = ref_coord
elif len(ref_coord) == 2:
ra_src, dec_src = ref_coord
gamma_in = 0.
else:
raise ValueError(
"ref_coord must consist of 2 or 3 values, or a string to decode."
)
# Preliminary computation, no rotation
wand0 = cls.for_tod(tod,
coords='tangent',
polarized=polarized,
ref_coord=(ra_src, dec_src, gamma_in),
hwp=hwp)
if not scan_coords:
return wand0
# Get index of sample that is nearest the ref_coord
distance = (wand0.y**2 + wand0.z**2)**.5
ic0 = np.argmin(distance)
assert tod.ctime[ic0] != 0 # To be dealt with elsewhere...
if distance[ic0] * (180 / np.pi) > warning_distance:
logger.trace(
'moby', 0, 'Source at [%f, %f] is not in %s' %
(ra_src, dec_src, tod.info.name))
gamma_src = np.arctan2(wand0.sin_gamma[ic0], wand0.cos_gamma[ic0])
# Recompute with this rotation
return cls.for_tod(tod,
coords='tangent',
polarized=polarized,
ref_coord=(ra_src, dec_src, gamma_in + gamma_src),
hwp=hwp)
def removeCorrelations( self ):
"""
@brief Remove detector-detector correlations from the TOD.
"""
psLib.trace( "moby", 3, "Removing Correlations." )
if self.params['useNoiseInCorr'] and self.params['filterNoise']: noise = self.noise
else: noise = None
if self.params['removeLiveCorr']:
if self.params['removeArrayCorr'] and self.params['arrayCorrParams']['cmtype'] == 'poly':
if self.params['useNoiseInCorr']:
psLib.trace( "moby", 2, "WARNING: PolyCommonMode not implemented with noise." )
self.polyCommonMode.removeModes()
if len(self.modes.modes) > 0:
self.modes.removeModeSet( noise = noise, forceFit = True )
def condenseModes( self, tod, lc, dc ):
psLib.trace( "moby", 3, "Condensing Modes." )
dets, rows, cols = tod.listUncut()
self.modes = correlations.modeSet(tod, [], numpy.array(dets))
self.polyCommonMode = None
if self.params['removeLiveCorr']:
if self.params['removeArrayCorr']:
if self.params['arrayCorrParams']['cmtype'] == 'poly':
self.polyCommonMode = lc.commonMode
else: self.modes.append(lc.commonMode.modes)
if self.params['removeColCorr']:
self.modes.append(lc.liveColModes.modes)
if self.params['removeRowCorr']:
self.modes.append(lc.liveRowModes.modes)
if self.params['removeSVDCorr']:
self.modes.append(lc.liveSVDModes.modes)
if self.params['removeDarkCorr']:
self.modes.append(dc.darkModes.modes, orthogonalize = True)
def get_source_cuts(tod,
ra,
dec,
map_size=None,
map_pix=None,
radius=10. / 60,
offset=(0., 0.)):
"""
Return TODCuts for tod that excise a circle around (ra, dec).
ra, dec should be provided in radians. All other parameters have
units of degrees.
"""
if map_size is None:
map_size = radius * 2
if map_pix is None:
map_pix = .003 * map_size
# Create a small map centered on source
g, dg = map_size, map_pix
maskMap = moby2.mapping.fits_map.spaceMap.simpleMap((-g, g), (-g, g),
(dg, dg),
dtype='float32',
wtype='int32')
x0, y0 = offset
dx, dy = maskMap.x - x0, maskMap.y - y0
my_mask = (dx**2 + dy**2 < radius**2)
maskMap.data[my_mask] = 1.
ny, nx = maskMap.data.shape
psLib.trace('moby', 3, 'get_source_cuts map has %i x %i pixels' % (ny, nx))
psLib.trace('moby', 3, ' centered at %f %f' % (x0, y0))
psLib.trace(
'moby', 3,
' map mask fraction is %f' % (float(my_mask.sum()) / (nx * ny)))
# Project into TOD
wand = moby2.pointing.ArrayWand.for_tod_source_coords(tod,
ref_coord=(ra, dec),
scan_coords=True)
gridding = moby2.pointing.GridPixelization.forFitsMap(maskMap)
proj = moby2.pointing.WandProjector(wand, gridding, tod.fplane)
# Warning this creates TOD-sized data...
mask_vect = proj.deproject_from_map(maskMap.data)
# Get cuts that is the same dets as focal plane entries.
cuts = moby2.TODCuts(nsamps=tod.nsamps,
det_uid=tod.fplane.det_uid,
sample_offset=tod.info.sample_index)
for i, d in enumerate(mask_vect):
cuts.add_cuts(i, d, mask=True)
return cuts
def findAndRemoveCorrelations( self, lc, dc ):
"""
@brief Find and Remove detector-detector correlations from the TOD. Only to initialize modes.
"""
#Remove correlations
if self.params['removeLiveCorr']:
psLib.trace( "moby", 3, "Removing Live Detector Correlations." )
if self.params['removeArrayCorr']:
psLib.trace( "moby", 3, "Removing Live CommonMode, params %s" % \
str(self.params['arrayCorrParams']) )
lc.removeArrayCorr( **self.params['arrayCorrParams'] )
if self.params['removeColCorr']:
psLib.trace( "moby", 3, "Removing Column Correlations, params %s" % \
str(self.params['colCorrParams']) )
lc.removeColCorr( **self.params['colCorrParams'] )
if self.params['removeRowCorr']:
psLib.trace( "moby", 3, "Removing Row Correlations, params %s" % \
str(self.params['rowCorrParams']) )
lc.removeRowCorr( **self.params['rowCorrParams'] )
if self.params['removeSVDCorr']:
psLib.trace( "moby", 3, "Removing Live SVD Modes, params %s" % \
str(self.params['SVDCorrParams']) )
lc.removeSVDCorr( **self.params['SVDCorrParams'] )
if self.params['removeDarkCorr']:
psLib.trace("moby", 3, "Removing Dark Detector Correlation, params %s" % \
str(self.params['darkCorrParams']))
dc.removeDarkCorr( **self.params['darkCorrParams'] )
def trace(level, text):
psLib.trace('moby', level, text)
def trace(self, unit, level, msg):
psLib.trace(unit, level, msg)
def addTOD(self,
tod,
projDict,
tod_name=None,
modesObject=None,
setInitConditions=False,
exportProjections=False,
outPath=".",
modesLevel=0.0001):
"""
@brief add a tod and associated projectors to the pcg
@param tod the TOD.TOD to add to the pcg
@param projDict a dictionary with map name keys and projector values
"""
if tod_name is None:
try:
tod_name = tod.info.name
if tod_name is None:
raise
except:
tod_name = str(id(tod))
self.trace("moby", 2,
"Assigning name '%s' to unknown TOD" % tod_name)
self.ntod += 1
self.tods[tod_name] = tod
for mn in list(projDict.keys()):
self.proj[self._createProjKey(tod_name, mn)] = projDict[mn]
if self.filterClass is not None:
self.trace("moby", 3, "Estimating noise for %s." % tod_name)
data = tod.data.copy()
mapsZero = True
for mapName in self.getMapNamesForTOD(tod_name):
if self.x[mapName].max() > 0. or self.x[mapName].min() < 0.:
mapsZero = False
if not mapsZero:
self.trace("moby", 3,
"Subtracting initial maps from %s." % tod_name)
# Remove maps from data before estimating noise filters
tod.data[:] *= -1
self.loadMaps(weight=False)
self.projectMapsToTOD(tod)
tod.data[:] *= -1
#now estimate filters
self.filters[tod_name] = self.filterClass(tod, self.filterParams)
tod.data[:] = data[:]
del data
self.filters[tod_name].setTOD(tod)
self.filters[tod_name].applyFilter()
self.filters[tod_name].setTOD(None)
else:
self.filters[tod_name] = None
psLib.trace("moby", 3, "Initial projection of %s." % tod_name)
self.clearMaps()
self.projectTODToMaps(tod_name)
mapNames = self.getMapNamesForTOD(tod_name)
for mapName in mapNames:
map2numpy(self.maps[mapName], self.b[mapName], accum=True)
self.weights[mapName][:] += np.array(self.maps[mapName].weight[:])
if exportProjections:
filename = '%s/%s_proj.fits' % (outPath, tod_name)
m = self.maps[mapName].trim()
m.write(filename, weight=True)
# If solving for correlated modes, add modesObject
if modesObject is not None:
# Cut sections of the TOD that fall outside the maps area.
print("Datacopy2")
data = tod.data.copy()
self.cutTODexcess(tod)
tod.data[:] = data[:]
del data
mo = self.modes[tod_name] = modesObject
mo.loadModes(tod)
nPix = 0
for mapName in mapNames:
mo.addMap(mapName)
mo.b[mapName] = mo.projectModes(initialize=True,
divideByWeights=False)
nPix += self.maps[mapName].npix
mo.setWeights(nPix=nPix)
for mapName in mapNames:
mo.b[mapName] /= mo.weight
# Set initial conditions
if setInitConditions:
mo.a[mapName] = mo.b[mapName].copy()
mo.p[mapName] = mo.a[mapName].copy(
) # For use in applyInverseCovariance
#print "Datacopy3"
#data = tod.data.copy()
mo.modes.coeff[:] = mo.b[mapName][:] * mo.weight
mo.modes.removeModeSet(forceFit=False,
dets=None,
noise=None)
self.clearMaps()
self.projectTODToMaps(tod_name)
map2numpy(self.maps[mapName], self.x[mapName], accum=True)
#tod.data[:] = data[:]
#del data
mo.unloadModes()
import numpy as np
import moby2.util.log as psLib
#import mobyLib
#import Mapset, preconditioner, prior
#from moby.todUtils import TOD, cuts
from moby2.tod import TOD, TODCuts
#from moby.pointing import pointingOffset
#from moby.database import result
from .preconditioner import preconditionerList, preconditioner_registry
try:
if os.environ['MB_USE_MPI']:
psLib.trace(
"moby", 3,
"###################### pcg initializing MPI #########################"
)
from mpi4py import MPI
from moby.utilities import mbMPI
except:
pass
def map2numpy(m, n, accum=False):
if accum:
n[:] += m.data[:]
else:
n[:] = m.data[:]
def numpy2map(n, m, accum=False):
def applyFilter(self, data=None):
psLib.trace("moby", 6, "Indentity Filter")
def fill_cuts_old(tod=None,
cuts=None,
data=None,
neighborhood=40,
do_all=False,
filterScaling=1.0):
"""
@brief fill regions of cut data with synthetic data
@param tod TOD to fill
@param cuts TODCuts to use (defaults to tod.cuts)
@param data data array (defaults to tod.dta)
@param neighborhood int: number of data points two either side of a cut
region to consider when estimating baseline and rms.
"""
# Just passing in tod is enough.
if cuts is None:
cuts = tod.cuts
if data is None:
data = tod.data
if hasattr(tod, 'info'):
tod_name = tod.info.name
else:
tod_name = 'data'
if do_all:
det_list = range(data.shape[0])
else:
det_list = cuts.get_uncut()
psLib.trace("moby", 3, "cuts.fill_cuts begins...")
nsamps = data.shape[1]
for deti in det_list:
for cfirst, clast in cuts.cuts[deti]:
if cfirst >= nsamps:
break
# This .trace call is not what makes the cuts slow:
#psLib.trace("moby",6,"Cut between %d (%f sec), %d (%f sec)." % \
# ( cfirst, tod.ctime[cfirst], clast,
# tod.ctime[clast-1] ) )
# Left and right side sample windows and interior region.
xStart = np.arange(max(0, cfirst - neighborhood), cfirst)
xStop = np.arange(clast, min(clast + neighborhood, nsamps))
xGap = np.arange(cfirst, min(nsamps, clast + 1))
extr = (len(xStart) == 0) or (len(xStop) == 0)
x = np.concatenate((xStart, xStop))
if len(x) == 0 or len(xGap) == 0:
c = c.__next__
continue
# Linear component, if both ends anchored.
y = data[deti, x]
if extr:
m = 0.0
b = y.mean()
else:
m, b = np.polyfit(x, y, 1)
resid = y - b - x * m
rms = resid.std()
if rms == 0:
psLib.trace("moby", 4, "fillCuts: Gap RMS 0 (%s: det %d, i0 %d, i1 %d)" % \
(tod_name, deti, cfirst, clast))
yGap = b + xGap * m
if filterScaling != 0.0 and rms > 0:
yGap += np.random.normal(loc=0, scale=rms, size=len(xGap))
data[deti, xGap] = yGap
psLib.trace("moby", 3, "cuts.fill_cuts ends.")
if tod is not None:
tod.abuses += [{'name': 'fillCuts', 'neighborhood': neighborhood}]
def __init__( self, noiseTOD, filterParams ):
"""
@param noiseTOD TOD.TOD containing the best estimate of the noise, will be altered!
@param filterParams dictionary with parameters for filters and correlations, they are:
filterNoise: True or False. Whether to apply an inverse noise filter.
noiseParams: Dictionary with parameters of the inverse noise filter.
useNoiseInCorr: True or False. Whether to apply the (m'N^{-1}m)^{-1} term for mode removal.
removeLiveCorr: True or False. Whether to remove live correlation modes.
removeArrayCorr: True or False. Whether to remove live common mode.
arrayCorrParams: Dictionary with parameters defining the kind of common mode to use.
removeColCorr: True or False. Whether to remove live column correlation modes.
colCorrParams: Dictionary with parameters for the column correlations.
removeRowCorr: True or False. Whether to remove live row correlation modes.
rowCorrParams: Dictionary with parameters for the row correlations.
removeSVDCorr: True or False. Whether to remove live SVD correlation modes.
SVDCorrParams: Dictionary with parameters for the SVD correlations.
removeDarkCorr: True or False. Whether to remove live dark correlation modes.
darkCorrParams: Dictionary with parameters for the dark correlations.
loadExternal: True or False. Whether to read in the noise and correlations from a depot
specified in the params
corrDepot: if loadExternal, look in this dir (under constants.CORR_DEPOT_ROOT) for
correlation objects
corrTag: location in corrDepot for correlation objects
noiseTag: location in the noiseDepot for noise objects, yo
"""
self.params = filterParams
lc = None; dc = None
if self.params['removeLiveCorr']:
if self.params['loadExternal']:
wd = result.setDepot( constants.CORR_DEPOT_ROOT+self.params['corrDepot'] )
lc = result.readTODResult( correlations.liveCorrObj, noiseTOD, self.params['corrTag'] )
result.setDepot( wd )
else:
lc = correlations.liveCorrObj(noiseTOD)
if self.params['removeDarkCorr']:
if self.params['loadExternal']:
wd = result.setDepot( constants.CORR_DEPOT_ROOT+self.params['corrDepot'] )
dc = result.readTODResult( correlations.darkCorrObj, noiseTOD, self.params['corrTag'] )
result.setDepot( wd )
else:
dc = correlations.darkCorrObj(noiseTOD)
if not self.params['loadExternal']:
self.findAndRemoveCorrelations( lc, dc )
if self.params['filterNoise']:
if self.params['loadExternal']:
wd = result.setDepot( constants.NOISE_DEPOT_ROOT+self.params['noiseDepot'] )
self.noise = result.readTODResult( noise.noise, noiseTOD, self.params['noiseTag'] )
result.setDepot( wd )
psLib.trace( "moby", 3, "Read old noise")
else:
self.noise = noise.noise( noiseTOD, **self.params['noiseParams'] )
self.noise.inverseFilterTOD( useCutDet = True )
psLib.trace( "moby", 3, "Found new noise")
psLib.trace( "moby", 2, "Initialized noise for %s" % noiseTOD.name)
else:
self.noise = None
self.condenseModes( noiseTOD, lc, dc )
if self.params['useNoiseInCorr'] and self.params['filterNoise']:
psLib.trace( "moby", 2, "Obtaining noise coefficient for %s" % noiseTOD.name)
if len(self.modes.modes) > 0:
self.modes.generateNoiseCoeff( self.noise )
def removeCorrelations(self, data=None):
psLib.trace( "moby", 3, "simpleFilter.removeCorrelations begins")
if data is None:
data = self.tod.data
self.filter=True
psLib.trace( "moby", 3, "get uncut")
psLib.trace( "moby", 3, "remove mean")
moby2.tod.remove_mean(data=data, dets=self.uncut_dets)
psLib.trace( "moby", 3, "computing common mode")
cm = moby2.libactpol.data_mean_axis0(
data, self.uncut_dets.astype('int32'))
psLib.trace( "moby", 3, "create filter")
cmfilt = self.params.get('commonModeFilter', 'boxcar')
if cmfilt == 'boxcar':
fc = 1./self.params['commonModeSmoothing']
nw = int(self.params['commonModeSmoothing'] / self.dt)
# Mimic behaviour of windowFilter by using only odd window widths.
nw = nw//2*2 + 1
filt = moby2.tod.filters2.boxcar(width_samples=nw) \
.getExecFor(data=data, dt=self.dt)
elif cmfilt == 'butterworth-low':
fc = 1./self.params['commonModeSmoothing']
filt = moby2.tod.filters2.lowPassButterworth(fc=fc, order=4) \
.getExecFor(data=data, dt=self.dt)
psLib.trace( "moby", 3, "filtering common mode")
cm = filt.applyTime(cm)
psLib.trace( "moby", 3, "removing common mode")
modes = cm.reshape((1,-1)).astype('float32')
amps = numpy.ones((1,len(self.uncut_dets)), dtype='float64')
moby2.libactpol.remove_modes(
data, self.uncut_dets.astype('int32'), modes, amps)
psLib.trace( "moby", 3, "simpleFilter.removeCorrelations ends")
def fillBlanks(self, homogenize=False):
"""
@brief Puts the median of the elements of the rest of the column to the value of the
pixels that were labeled as outliers.
@param homogenize Uses the same value to all detectors in a colunm, using the mean of the
unmasked values for that column.
"""
amp = self.amp - 0.0
phase = self.phase - 0.0
rms = self.rms - 0.0
if homogenize:
for ind in range(32):
if numpy.any(self.col[self.mask == 0] == ind):
amp[self.col == ind] = \
numpy.median(self.amp[(self.col == ind)*(self.mask == 0)])
phase[self.col == ind] = \
numpy.median(self.phase[(self.col == ind)*(self.mask == 0)])
rms[self.col == ind] = \
numpy.median(self.rms[(self.col == ind)*(self.mask == 0)])
else:
amp[self.col == ind] = \
numpy.median(self.amp[self.mask == 0])
phase[self.col == ind] = \
numpy.median(self.phase[self.mask == 0])
rms[self.col == ind] = \
numpy.median(self.rms[self.mask == 0])
psLib.trace("moby", 3, "WARNING: Missing information for COL" + str(ind) + \
", using median of whole array.")
else:
#dets, rows, cols = self.tod.listUncut()
## This is admittedly somewhat more work...
dets = self.tod.cuts.get_uncut()
cols = self.tod.info.array_data['col'][self.tod.det_uid][dets]
for ind in range(0, max(cols) + 1):
if numpy.any(self.col[self.mask == 0] == ind):
amp[(self.col == ind)*(self.mask != 0)] = \
numpy.median(self.amp[(self.col == ind)*(self.mask == 0)])
phase[(self.col == ind)*(self.mask != 0)] = \
numpy.median(self.phase[(self.col == ind)*(self.mask == 0)])
rms[(self.col == ind)*(self.mask != 0)] = \
numpy.median(self.rms[(self.col == ind)*(self.mask == 0)])
else:
sel = (numpy.array(cols) == ind)
if numpy.any(sel):
clm = numpy.empty(self.tod.ndata)
mobyLib.transposeMedian(self.tod.ctod,
list(numpy.array(dets)[sel]),
clm)
tamp, tphase = fitSine(
clm,
2 * numpy.pi * self.scanFreq * self.sampleTime)
amp[(self.col == ind) * (self.mask != 0)] = tamp
phase[(self.col == ind) * (self.mask != 0)] = tphase
rms[(self.col == ind) * (self.mask != 0)] = 0.0
psLib.trace(
"moby", 1,
"WARNING: Missing information for COL %d,"
" fitting from live data" % ind)
return amp, phase, rms
def fill_cuts(tod=None,
cuts=None,
data=None,
neighborhood=40,
do_all=False,
filterScaling=1.0,
extrapolate=False,
sample_index=0,
no_noise=False):
"""Replace certain samples of a TOD with a straight line plus some
white noise. The samples to fill are specified by a TODCuts
object, which is passed through the cuts= argument, or obtained
from tod.cuts. The data array to fill is passed through thte
data= argument, or else tod.data is used.
The cut samples will be replaced with a straight line and white
noise, with the linear fit and RMS taken from a "neighborhood" of
samples on either side of the cut region. The white noise can be
modulated by the filterScaling argument, or turned off entirely by
passing no_noise=True.
The extrapolate argument affects how cut regions at the beginning
or end of a timestream are handled; False means that the linear
fill will have its slope forced to zero.
"""
psLib.trace("moby", 4, "cuts.fill_cuts begins...")
# Just passing in tod is enough.
if cuts is None:
cuts = tod.cuts
if data is None:
data = tod.data
si = tod.info.sample_index
else:
si = sample_index
if hasattr(tod, 'info'):
tod_name = tod.info.name
else:
tod_name = 'data'
nsamps = data.shape[-1]
# Check alignment between cuts and data.
if si != cuts.sample_offset:
print(
"TOD was loaded from sample %i but cuts have sample offset %i. The first samples will be cut"
% (si, cuts.sample_offset))
else:
assert ((data.shape[-1] == cuts.nsamps) |
(data.shape[-1] == cuts.nsamps + 1))
# Prevent det_uid mismatch.
assert (data.shape[0] == len(cuts.det_uid))
if data is None:
assert (np.all(tod.det_uid == cuts.det_uid))
if do_all:
det_list = range(data.shape[0])
else:
det_list = cuts.get_uncut()
for deti in det_list:
mask = cuts.cuts[deti].get_mask()
offset = cuts.sample_offset - si
total_mask = np.ones(nsamps, dtype=np.bool)
total_mask[max(0,offset):min(nsamps,cuts.nsamps+offset)] = \
mask[max(0,-offset):min(nsamps-offset,cuts.nsamps)]
cuts_list = CutsVector.from_mask(total_mask)
if no_noise:
noise = None
else:
ncut = np.dot([-1, 1], cuts_list.sum(axis=0))
noise = np.random.normal(
size=ncut).astype('float32') * filterScaling
libactpol.fill_cuts(data[deti], cuts_list, noise, neighborhood,
extrapolate)
if tod is not None:
tod.abuses += [{'name': 'fillCuts', 'neighborhood': neighborhood}]
psLib.trace("moby", 4, "cuts.fill_cuts exits.")
