def _compute_getraras(self, toread, uvdatoptions):
seenaps = set()
rarawork = {}
previnp = None
gen = uvdat.setupAndRead(toread,
'a3',
False,
nopass=True,
nocal=True,
nopol=True,
**uvdatoptions)
for inp, preamble, data, flags in gen:
if previnp is None or inp is not previnp:
instr = inp.getScalarItem('arfinstr', 'UNDEF')
if instr != 'UNDEF':
if self.instr is None:
self.instr = instr
elif instr != self.instr:
util.die(
'input instrument changed from "%s" to '
'"%s" in "%s"', self.instr, instr, inp.path())
previnp = inp
# the 'a' uvdat options limits us to autocorrelations,
# but includes 1X-1Y-type autocorrelations
bp = util.mir2bp(inp, preamble)
if not util.bpIsInten(bp):
continue
ap = bp[0]
t = preamble[3]
w = np.where(flags)[0]
if not w.size:
continue
data = data[w]
rara = np.sqrt((data.real**2).mean())
seenaps.add(ap)
ag = rarawork.get(ap)
if ag is None:
ag = rarawork[ap] = ArrayGrower(3)
# We record w.size as a weight for the RARA measurement,
# but it's essentially noise-free.
ag.add(t, rara, w.size)
self.saps = sorted(seenaps)
raras = self.raras = {}
apidxs = dict((t[1], t[0]) for t in enumerate(self.saps))
for ap in rarawork.keys():
raradata = rarawork[ap].finish().T
apidx = apidxs[ap]
sidx = np.argsort(raradata[0])
raras[apidx] = raradata[:, sidx]
def record (self, state):
inp = state.inp
track = self.track
systemps = self.systemps
prefactor = self.prefactor
bps = self.bps
bpfactors = self.bpfactors
if track is None:
track = inp.makeVarTracker ().track ('systemp', 'sdf', 'inttime')
self.track = track
if track.updated ():
nants = inp.getVarInt ('nants')
systemps = self.systemps = inp.getVarFloat ('systemp', nants)
nspect = inp.getVarInt ('nspect')
sdf = inp.getVarDouble ('sdf', nspect)
if nspect > 1:
sdf = sdf[0]
inttime = inp.getVarFloat ('inttime')
prefactor = self.prefactor = 2 * inttime * sdf * 1e9
if bpfactors is None or bpfactors.size != state.nbp:
bpfactors = self.bpfactors = np.empty (state.nbp)
bps = self.bps = np.empty ((state.nbp, 2), dtype=np.int)
ap1, ap2 = mir2bp (inp, state.preamble)
if ap1 == ap2:
# Autocorrelation! Get the RARA (raw autocorrelation
# RMS amplitude)
w = np.where (state.flags)[0]
if w.size == 0:
self.raras.pop (ap1, 0)
else:
self.raras[ap1] = np.sqrt ((state.data.real[w]**2).mean ())
# Get the factor that will go in front of the RARAs for
# jyperk computations. Do this for autocorrs too because
# there's no reason not to.
if state.instr not in self.byinstr:
raise Exception ('need information for instrument ' +
state.instr)
byap, default = self.byinstr[state.instr]
cal1 = byap.get (ap1, default)
cal2 = byap.get (ap2, default)
ant1, ant2 = apAnt (ap1), apAnt (ap2)
tsys1, tsys2 = systemps[ant1 - 1], systemps[ant2 - 1]
bpindex = state.bpindex
bps[bpindex,0] = ap1
bps[bpindex,1] = ap2
bpfactors[bpindex] = prefactor * cal1 * cal2 / (tsys1 * tsys2)
def record(self, state):
inp = state.inp
track = self.track
systemps = self.systemps
prefactor = self.prefactor
bps = self.bps
bpfactors = self.bpfactors
if track is None:
track = inp.makeVarTracker().track('systemp', 'sdf', 'inttime')
self.track = track
if track.updated():
nants = inp.getVarInt('nants')
systemps = self.systemps = inp.getVarFloat('systemp', nants)
nspect = inp.getVarInt('nspect')
sdf = inp.getVarDouble('sdf', nspect)
if nspect > 1:
sdf = sdf[0]
inttime = inp.getVarFloat('inttime')
prefactor = self.prefactor = 2 * inttime * sdf * 1e9
if bpfactors is None or bpfactors.size != state.nbp:
bpfactors = self.bpfactors = np.empty(state.nbp)
bps = self.bps = np.empty((state.nbp, 2), dtype=np.int)
ap1, ap2 = mir2bp(inp, state.preamble)
if ap1 == ap2:
# Autocorrelation! Get the RARA (raw autocorrelation
# RMS amplitude)
w = np.where(state.flags)[0]
if w.size == 0:
self.raras.pop(ap1, 0)
else:
self.raras[ap1] = np.sqrt((state.data.real[w]**2).mean())
# Get the factor that will go in front of the RARAs for
# jyperk computations. Do this for autocorrs too because
# there's no reason not to.
if state.instr not in self.byinstr:
raise Exception('need information for instrument ' +
state.instr)
byap, default = self.byinstr[state.instr]
cal1 = byap.get(ap1, default)
cal2 = byap.get(ap2, default)
ant1, ant2 = apAnt(ap1), apAnt(ap2)
tsys1, tsys2 = systemps[ant1 - 1], systemps[ant2 - 1]
bpindex = state.bpindex
bps[bpindex, 0] = ap1
bps[bpindex, 1] = ap2
bpfactors[bpindex] = prefactor * cal1 * cal2 / (tsys1 * tsys2)
def _compute_getraras (self, toread, uvdatoptions):
seenaps = set ()
rarawork = {}
previnp = None
gen = uvdat.setupAndRead (toread, 'a3', False,
nopass=True, nocal=True, nopol=True,
**uvdatoptions)
for inp, preamble, data, flags in gen:
if previnp is None or inp is not previnp:
instr = inp.getScalarItem ('arfinstr', 'UNDEF')
if instr != 'UNDEF':
if self.instr is None:
self.instr = instr
elif instr != self.instr:
util.die ('input instrument changed from "%s" to '
'"%s" in "%s"', self.instr, instr,
inp.path ())
previnp = inp
# the 'a' uvdat options limits us to autocorrelations,
# but includes 1X-1Y-type autocorrelations
bp = util.mir2bp (inp, preamble)
if not util.bpIsInten (bp):
continue
ap = bp[0]
t = preamble[3]
w = np.where (flags)[0]
if not w.size:
continue
data = data[w]
rara = np.sqrt ((data.real**2).mean ())
seenaps.add (ap)
ag = rarawork.get (ap)
if ag is None:
ag = rarawork[ap] = ArrayGrower (3)
# We record w.size as a weight for the RARA measurement,
# but it's essentially noise-free.
ag.add (t, rara, w.size)
self.saps = sorted (seenaps)
raras = self.raras = {}
apidxs = dict ((t[1], t[0]) for t in enumerate (self.saps))
for ap in rarawork.keys ():
raradata = rarawork[ap].finish ().T
apidx = apidxs[ap]
sidx = np.argsort (raradata[0])
raras[apidx] = raradata[:,sidx]
def _compute_getbpvs (self, toread, uvdatoptions):
raras = self.raras
apidxs = dict ((t[1], t[0]) for t in enumerate (self.saps))
sqbps = ArrayGrower (2, dtype=np.int)
bpdata = ArrayGrower (5)
nnorara = 0
seenidxs = set ()
gen = uvdat.setupAndRead (toread, 'x3', False,
nopass=False, nocal=False, nopol=False,
**uvdatoptions)
for inp, preamble, data, flags in gen:
bp = util.mir2bp (inp, preamble)
t = preamble[3]
w = np.where (flags)[0]
if not w.size:
continue
idx1, idx2 = apidxs.get (bp[0]), apidxs.get (bp[1])
if idx1 is None or idx2 is None:
nnorara += 1
continue
rdata1, rdata2 = raras[idx1], raras[idx2]
tidx1 = rdata1[0].searchsorted (t)
tidx2 = rdata2[0].searchsorted (t)
if (tidx1 < 0 or tidx1 >= rdata1.shape[1] or
tidx2 < 0 or tidx2 >= rdata2.shape[1]):
nnorara += 1
continue
tr1, rara1, rwt1 = rdata1[:,tidx1]
tr2, rara2, rwt2 = rdata2[:,tidx2]
dt1 = np.abs (tr1 - t)
dt2 = np.abs (tr2 - t)
if max (dt1, dt2) > TTOL:
nnorara += 1
continue
# We propagate the weight for crara but once again,
# it's basically noiseless.
crara = rara1 * rara2
cwt = 1. / (rara2**2 / rwt1 + rara1**2 / rwt2)
data = data[w]
rvar = data.real.var (ddof=1)
ivar = data.imag.var (ddof=1)
var = 0.5 * (rvar + ivar)
# The weight of the computed variance (i.e., the
# inverse square of the maximum-likelihood variance
# in the variance measurement) is 0.5 * (nsamp - ndof) / var**2.
# We have 2*w.size samples and 2 degrees of freedom, so:
wtvar = (w.size - 1) / var**2
seenidxs.add (idx1)
seenidxs.add (idx2)
sqbps.add (idx1, idx2)
bpdata.add (t, var, wtvar, crara, cwt)
sqbps = self.sqbps = sqbps.finish ()
self.bpdata = bpdata.finish ()
nsamp = sqbps.shape[0]
if nnorara > nsamp * 0.02:
print >>sys.stderr, ('Warning: no autocorr data for %d/%d '
'(%.0f%%) of samples' % (nnorara, nsamp,
100. * nnorara / nsamp))
if len (seenidxs) == len (apidxs):
return
# There exist antpols that we got autocorrelations for but
# have no contributing baselines. This can happen if there are
# no gains for the given antpol. To avoid running into a
# singular matrix in the solve step, we have to eliminate
# them. To avoid a bunch of baggage in the analysis code, we
# rewrite our data structures, which actually isn't so bad.
mapping = {}
newsaps = []
newraras = {}
for idx in xrange (len (apidxs)):
if idx in seenidxs:
mapping[idx] = len (mapping)
newsaps.append (self.saps[idx])
newraras[mapping[idx]] = raras[idx]
self.saps = newsaps
self.raras = newraras
for i in xrange (nsamp):
sqbps[i,0] = mapping[sqbps[i,0]]
sqbps[i,1] = mapping[sqbps[i,1]]
def rewrite (self, ncorr, invis, outvis, **uvdargs):
psources = [s for s in self.sources if s.major is None]
gsources = [s for s in self.sources if s.major is not None]
outhnd = outvis.open ('c')
outhnd.setPreambleType ('uvw', 'time', 'baseline')
first = True
curhnd = None
npol = prevnpol = 0
neednpol = True
npolvaried = False
nproc = 0
nrec = 0
prevtime = None
for inhnd, preamble, data, flags in invis.readLowlevel ('3', False, **uvdargs):
if first:
inhnd.copyItem (outhnd, 'history')
outhnd.openHistory ()
outhnd.writeHistory (self.banner)
first = False
if inhnd is not curhnd:
inhnd.initVarsAsInput ('channel')
outhnd.initVarsAsOutput (inhnd, 'channel')
#outhnd.setCorrelationType ('r')
curhnd = inhnd
freqsupdated = True # assuming fixed freqs per dataset
if freqsupdated:
freqs = inhnd.getSkyFrequencies ()
alphas = pbalphas (freqs)
dwork = np.empty (freqs.shape, dtype=np.double)
dwork2 = np.empty (freqs.shape, dtype=np.double)
cwork = np.empty (freqs.shape, dtype=np.complex)
freqsupdated = False
if prevtime is None or preamble[3] != prevtime:
# Need to determine new squint distances as PB rotates on the sky.
lst = inhnd.getVarDouble ('lst')
lat = inhnd.getVarDouble ('latitud')
az, el = util.equToHorizon (inhnd.getVarDouble ('ra'),
inhnd.getVarDouble ('dec'),
lst, lat)
for ap, (sqmag, sqpa) in self.squints.iteritems ():
oel, oaz = sphofs (el, az, sqmag, sqpa)
ora, odec = util.horizonToEqu (oaz, oel, lst, lat)
for s in self.sources:
# We take half of the squared distance because
# that's what's going to be relevant in the PB
# attenuation (half in exponent -> sqrt ->
# geometric mean)
s.hpbsqdists[ap] = 0.5 * sphdist (s.dec, s.ra, odec, ora)**2
prevtime = preamble[3]
if nrec % recmod == 0:
print >>sys.stderr, '%5.1f%% (%d of %d)' % (100.* nproc / ncorr,
nproc, ncorr)
if npol == 0:
npol = inhnd.getNPol ()
neednpol = True
if neednpol:
if npol != prevnpol:
outhnd.writeVarInt ('npol', npol)
npolvaried = npolvaried or (prevnpol != 0)
prevnpol = npol
neednpol = False
ap1, ap2 = util.mir2bp (inhnd, preamble)
data.fill (0)
flags.fill (1)
for s in psources:
# geometric mean of effective fluxes, attenuated by
# (maybe) freq-dependent PB falloff.
multiply (twopii * dot (s.lmn, preamble[:3]), freqs, cwork)
if pbfreqdep:
multiply (s.hpbsqdists[ap1] + s.hpbsqdists[ap2], alphas, dwork)
add (dwork, cwork, cwork)
else:
add ((s.hpbsqdists[ap1] + s.hpbsqdists[ap2]) * alphas[0], cwork, cwork)
exp (cwork, cwork)
multiply (s.totflux, cwork, cwork)
data += cwork
for s in gsources:
# As above, with additional gaussian amplitude falloff term.
cp = np.cos (s.pa)
sp = np.sin (s.pa)
g = gfac * ((s.minor * (preamble[0] * cp - preamble[1] * sp))**2 +
(s.major * (preamble[0] * sp + preamble[1] * cp))**2)
square (freqs, dwork2)
multiply (g, dwork2, dwork2)
multiply (twopii * dot (s.lmn, preamble[:3]), freqs, cwork)
if pbfreqdep:
multiply (s.hpbsqdists[ap1] + s.hpbsqdists[ap2], alphas, dwork)
add (dwork, cwork, cwork)
else:
add ((s.hpbsqdists[ap1] + s.hpbsqdists[ap2]) * alphas[0], cwork, cwork)
add (dwork2, cwork, cwork)
exp (cwork, cwork)
multiply (s.totflux, cwork, cwork)
data += cwork
inhnd.copyLineVars (outhnd)
outhnd.writeVarInt ('pol', inhnd.getPol ())
outhnd.write (preamble, data, flags)
npol -= 1
nrec += 1
nproc += data.size
if not npolvaried:
outhnd.setScalarItem ('npol', np.int32, prevnpol)
outhnd.closeHistory ()
outhnd.close ()
def _compute_getbpvs(self, toread, uvdatoptions):
raras = self.raras
apidxs = dict((t[1], t[0]) for t in enumerate(self.saps))
sqbps = ArrayGrower(2, dtype=np.int)
bpdata = ArrayGrower(5)
nnorara = 0
seenidxs = set()
gen = uvdat.setupAndRead(toread,
'x3',
False,
nopass=False,
nocal=False,
nopol=False,
**uvdatoptions)
for inp, preamble, data, flags in gen:
bp = util.mir2bp(inp, preamble)
t = preamble[3]
w = np.where(flags)[0]
if not w.size:
continue
idx1, idx2 = apidxs.get(bp[0]), apidxs.get(bp[1])
if idx1 is None or idx2 is None:
nnorara += 1
continue
rdata1, rdata2 = raras[idx1], raras[idx2]
tidx1 = rdata1[0].searchsorted(t)
tidx2 = rdata2[0].searchsorted(t)
if (tidx1 < 0 or tidx1 >= rdata1.shape[1] or tidx2 < 0
or tidx2 >= rdata2.shape[1]):
nnorara += 1
continue
tr1, rara1, rwt1 = rdata1[:, tidx1]
tr2, rara2, rwt2 = rdata2[:, tidx2]
dt1 = np.abs(tr1 - t)
dt2 = np.abs(tr2 - t)
if max(dt1, dt2) > TTOL:
nnorara += 1
continue
# We propagate the weight for crara but once again,
# it's basically noiseless.
crara = rara1 * rara2
cwt = 1. / (rara2**2 / rwt1 + rara1**2 / rwt2)
data = data[w]
rvar = data.real.var(ddof=1)
ivar = data.imag.var(ddof=1)
var = 0.5 * (rvar + ivar)
# The weight of the computed variance (i.e., the
# inverse square of the maximum-likelihood variance
# in the variance measurement) is 0.5 * (nsamp - ndof) / var**2.
# We have 2*w.size samples and 2 degrees of freedom, so:
wtvar = (w.size - 1) / var**2
seenidxs.add(idx1)
seenidxs.add(idx2)
sqbps.add(idx1, idx2)
bpdata.add(t, var, wtvar, crara, cwt)
sqbps = self.sqbps = sqbps.finish()
self.bpdata = bpdata.finish()
nsamp = sqbps.shape[0]
if nnorara > nsamp * 0.02:
print >> sys.stderr, ('Warning: no autocorr data for %d/%d '
'(%.0f%%) of samples' %
(nnorara, nsamp, 100. * nnorara / nsamp))
if len(seenidxs) == len(apidxs):
return
# There exist antpols that we got autocorrelations for but
# have no contributing baselines. This can happen if there are
# no gains for the given antpol. To avoid running into a
# singular matrix in the solve step, we have to eliminate
# them. To avoid a bunch of baggage in the analysis code, we
# rewrite our data structures, which actually isn't so bad.
mapping = {}
newsaps = []
newraras = {}
for idx in xrange(len(apidxs)):
if idx in seenidxs:
mapping[idx] = len(mapping)
newsaps.append(self.saps[idx])
newraras[mapping[idx]] = raras[idx]
self.saps = newsaps
self.raras = newraras
for i in xrange(nsamp):
sqbps[i, 0] = mapping[sqbps[i, 0]]
sqbps[i, 1] = mapping[sqbps[i, 1]]
