def channelAverageWithSetup (toread, out, naver, slop=DEFAULT_SLOP,
banner=DEFAULT_BANNER, **uvdargs):
"""Read UV data and channel average into an output dataset.
toread: dataset handle or iterable thereof; input dataset(s)
out: dataset handle; the output dataset to be created
naver: int; the number of channels to average together (see task docs)
slop: float; tolerance for partially flagged bins (see task docs)
banner: string; a message to write into the output's history
**uvdargs: keyword arguments passed through to the uvdat subsystem
initialization (mirtask.uvdat.setupAndRead)
returns: None
Contrast with channelAverage, which performs no extra initialization
of the uvdat subsystem.
"""
if naver < 1:
raise ValueError ('must average at least one channel (got naver=%d)' % naver)
if slop < 0 or slop > 1:
raise ValueError ('slop must be between 0 and 1 (got slop=%f)' % slop)
try:
gen = uvdat.setupAndRead (toread, UVDAT_OPTIONS, False, **uvdargs)
args = ['vis=' + ','.join (str (x) for x in ensureiterable (toread))]
args += ['%s=%s' % (k, uvdargs[k]) for k in sorted (uvdargs.iterkeys ())]
_channelAverage (gen, out, naver, slop, banner, args)
except _CreateFailedError, e:
# Don't delete the existing dataset!
raise e.subexc
def _compute_getraras(self, toread, uvdatoptions):
seenaps = set()
rarawork = {}
previnp = None
gen = uvdat.setupAndRead(toread,
'a3',
False,
nopass=True,
nocal=True,
nopol=True,
**uvdatoptions)
for inp, preamble, data, flags in gen:
if previnp is None or inp is not previnp:
instr = inp.getScalarItem('arfinstr', 'UNDEF')
if instr != 'UNDEF':
if self.instr is None:
self.instr = instr
elif instr != self.instr:
util.die(
'input instrument changed from "%s" to '
'"%s" in "%s"', self.instr, instr, inp.path())
previnp = inp
# the 'a' uvdat options limits us to autocorrelations,
# but includes 1X-1Y-type autocorrelations
bp = util.mir2bp(inp, preamble)
if not util.bpIsInten(bp):
continue
ap = bp[0]
t = preamble[3]
w = np.where(flags)[0]
if not w.size:
continue
data = data[w]
rara = np.sqrt((data.real**2).mean())
seenaps.add(ap)
ag = rarawork.get(ap)
if ag is None:
ag = rarawork[ap] = ArrayGrower(3)
# We record w.size as a weight for the RARA measurement,
# but it's essentially noise-free.
ag.add(t, rara, w.size)
self.saps = sorted(seenaps)
raras = self.raras = {}
apidxs = dict((t[1], t[0]) for t in enumerate(self.saps))
for ap in rarawork.keys():
raradata = rarawork[ap].finish().T
apidx = apidxs[ap]
sidx = np.argsort(raradata[0])
raras[apidx] = raradata[:, sidx]
def _compute_getraras (self, toread, uvdatoptions):
seenaps = set ()
rarawork = {}
previnp = None
gen = uvdat.setupAndRead (toread, 'a3', False,
nopass=True, nocal=True, nopol=True,
**uvdatoptions)
for inp, preamble, data, flags in gen:
if previnp is None or inp is not previnp:
instr = inp.getScalarItem ('arfinstr', 'UNDEF')
if instr != 'UNDEF':
if self.instr is None:
self.instr = instr
elif instr != self.instr:
util.die ('input instrument changed from "%s" to '
'"%s" in "%s"', self.instr, instr,
inp.path ())
previnp = inp
# the 'a' uvdat options limits us to autocorrelations,
# but includes 1X-1Y-type autocorrelations
bp = util.mir2bp (inp, preamble)
if not util.bpIsInten (bp):
continue
ap = bp[0]
t = preamble[3]
w = np.where (flags)[0]
if not w.size:
continue
data = data[w]
rara = np.sqrt ((data.real**2).mean ())
seenaps.add (ap)
ag = rarawork.get (ap)
if ag is None:
ag = rarawork[ap] = ArrayGrower (3)
# We record w.size as a weight for the RARA measurement,
# but it's essentially noise-free.
ag.add (t, rara, w.size)
self.saps = sorted (seenaps)
raras = self.raras = {}
apidxs = dict ((t[1], t[0]) for t in enumerate (self.saps))
for ap in rarawork.keys ():
raradata = rarawork[ap].finish ().T
apidx = apidxs[ap]
sidx = np.argsort (raradata[0])
raras[apidx] = raradata[:,sidx]
def tui_checkcal (args):
import omega as om, scipy.stats as SS
toread = []
uvdatoptions = {}
for arg in args:
if '=' in arg:
key, value = arg.split ('=', 1)
uvdatoptions[key] = value
else:
toread.append (VisData (arg))
if len (toread) < 1:
util.die ('usage: <vis1> [... visn] [uvdat options]')
samples = VectorGrower ()
gen = uvdat.setupAndRead (toread, 'x3', False, **uvdatoptions)
for inp, pream, data, flags in gen:
w = np.where (flags)[0]
if w.size == 0:
continue
data = data[w]
var = 0.5 * (data.real.var (ddof=1) + data.imag.var (ddof=1))
uvar = np.sqrt (1. / (w.size - 1)) * var # uncert in variance msmt
thy = inp.getVariance ()
samples.add ((var - thy) / uvar)
samples = samples.finish ()
n = samples.size
m = samples.mean ()
s = samples.std ()
med = np.median (samples)
smadm = 1.4826 * np.median (np.abs (samples - med)) # see comment below
print ' Number of samples:', n
print 'Mean normalized error (should be 0):', m
print ' Median:', med
print ' Normalized std. dev. (should be 1):', s
print ' SMADM:', smadm
print 'Probability that samples are normal:', SS.normaltest (samples)[1]
bins = 50
rng = -5, 5
p = om.quickHist (samples, keyText='Samples', bins=bins, range=rng)
x = np.linspace (-5, 5, 200)
area = 10. / bins * n
y = area / np.sqrt (2 * np.pi) * np.exp (-0.5 * x**2)
p.addXY (x, y, 'Ideal')
p.rebound (False, False)
p.show ()
return 0
def tui_checkcal(args):
import omega as om, scipy.stats as SS
toread = []
uvdatoptions = {}
for arg in args:
if '=' in arg:
key, value = arg.split('=', 1)
uvdatoptions[key] = value
else:
toread.append(VisData(arg))
if len(toread) < 1:
util.die('usage: <vis1> [... visn] [uvdat options]')
samples = VectorGrower()
gen = uvdat.setupAndRead(toread, 'x3', False, **uvdatoptions)
for inp, pream, data, flags in gen:
w = np.where(flags)[0]
if w.size == 0:
continue
data = data[w]
var = 0.5 * (data.real.var(ddof=1) + data.imag.var(ddof=1))
uvar = np.sqrt(1. / (w.size - 1)) * var # uncert in variance msmt
thy = inp.getVariance()
samples.add((var - thy) / uvar)
samples = samples.finish()
n = samples.size
m = samples.mean()
s = samples.std()
med = np.median(samples)
smadm = 1.4826 * np.median(np.abs(samples - med)) # see comment below
print ' Number of samples:', n
print 'Mean normalized error (should be 0):', m
print ' Median:', med
print ' Normalized std. dev. (should be 1):', s
print ' SMADM:', smadm
print 'Probability that samples are normal:', SS.normaltest(samples)[1]
bins = 50
rng = -5, 5
p = om.quickHist(samples, keyText='Samples', bins=bins, range=rng)
x = np.linspace(-5, 5, 200)
area = 10. / bins * n
y = area / np.sqrt(2 * np.pi) * np.exp(-0.5 * x**2)
p.addXY(x, y, 'Ideal')
p.rebound(False, False)
p.show()
return 0
def readLowlevel (self, uvdOptions, saveFlags, **kwargs):
"""Directly access the visibility data.
:arg uvdOptions: options controlling the behavior of the UVDAT subsystem
:type uvdOptions: :class:`str`
:arg saveFlags: whether to save modified UV flags to the dataset
as it is read
:type saveFlags: :class:`bool`
:arg kwargs: extra arguments to
:func:`mirtask.uvdat.setupAndRead`.
:rtype: generator of ``(handle, preamble, data, flags)``
:returns: generates a sequence of UV information tuples. See the
documentation of :func:`mirtask.uvdat.setupAndRead` for more
information.
Calls :func:`mirtask.uvdat.setupAndRead` on this dataset with the
specified arguments.
"""
from mirtask import uvdat
return uvdat.setupAndRead (self, uvdOptions, saveFlags, **kwargs)
def readLowlevel (self, uvdOptions, saveFlags, **kwargs):
"""Directly access the visibility data.
:arg uvdOptions: options controlling the behavior of the UVDAT subsystem
:type uvdOptions: :class:`str`
:arg saveFlags: whether to save modified UV flags to the dataset
as it is read
:type saveFlags: :class:`bool`
:arg kwargs: extra arguments to
:func:`mirtask.uvdat.setupAndRead`.
:rtype: generator of ``(handle, preamble, data, flags)``
:returns: generates a sequence of UV information tuples. See the
documentation of :func:`mirtask.uvdat.setupAndRead` for more
information.
Calls :func:`mirtask.uvdat.setupAndRead` on this dataset with the
specified arguments.
"""
from mirtask import uvdat
return uvdat.setupAndRead (self, uvdOptions, saveFlags, **kwargs)
def _compute_getbpvs (self, toread, uvdatoptions):
raras = self.raras
apidxs = dict ((t[1], t[0]) for t in enumerate (self.saps))
sqbps = ArrayGrower (2, dtype=np.int)
bpdata = ArrayGrower (5)
nnorara = 0
seenidxs = set ()
gen = uvdat.setupAndRead (toread, 'x3', False,
nopass=False, nocal=False, nopol=False,
**uvdatoptions)
for inp, preamble, data, flags in gen:
bp = util.mir2bp (inp, preamble)
t = preamble[3]
w = np.where (flags)[0]
if not w.size:
continue
idx1, idx2 = apidxs.get (bp[0]), apidxs.get (bp[1])
if idx1 is None or idx2 is None:
nnorara += 1
continue
rdata1, rdata2 = raras[idx1], raras[idx2]
tidx1 = rdata1[0].searchsorted (t)
tidx2 = rdata2[0].searchsorted (t)
if (tidx1 < 0 or tidx1 >= rdata1.shape[1] or
tidx2 < 0 or tidx2 >= rdata2.shape[1]):
nnorara += 1
continue
tr1, rara1, rwt1 = rdata1[:,tidx1]
tr2, rara2, rwt2 = rdata2[:,tidx2]
dt1 = np.abs (tr1 - t)
dt2 = np.abs (tr2 - t)
if max (dt1, dt2) > TTOL:
nnorara += 1
continue
# We propagate the weight for crara but once again,
# it's basically noiseless.
crara = rara1 * rara2
cwt = 1. / (rara2**2 / rwt1 + rara1**2 / rwt2)
data = data[w]
rvar = data.real.var (ddof=1)
ivar = data.imag.var (ddof=1)
var = 0.5 * (rvar + ivar)
# The weight of the computed variance (i.e., the
# inverse square of the maximum-likelihood variance
# in the variance measurement) is 0.5 * (nsamp - ndof) / var**2.
# We have 2*w.size samples and 2 degrees of freedom, so:
wtvar = (w.size - 1) / var**2
seenidxs.add (idx1)
seenidxs.add (idx2)
sqbps.add (idx1, idx2)
bpdata.add (t, var, wtvar, crara, cwt)
sqbps = self.sqbps = sqbps.finish ()
self.bpdata = bpdata.finish ()
nsamp = sqbps.shape[0]
if nnorara > nsamp * 0.02:
print >>sys.stderr, ('Warning: no autocorr data for %d/%d '
'(%.0f%%) of samples' % (nnorara, nsamp,
100. * nnorara / nsamp))
if len (seenidxs) == len (apidxs):
return
# There exist antpols that we got autocorrelations for but
# have no contributing baselines. This can happen if there are
# no gains for the given antpol. To avoid running into a
# singular matrix in the solve step, we have to eliminate
# them. To avoid a bunch of baggage in the analysis code, we
# rewrite our data structures, which actually isn't so bad.
mapping = {}
newsaps = []
newraras = {}
for idx in xrange (len (apidxs)):
if idx in seenidxs:
mapping[idx] = len (mapping)
newsaps.append (self.saps[idx])
newraras[mapping[idx]] = raras[idx]
self.saps = newsaps
self.raras = newraras
for i in xrange (nsamp):
sqbps[i,0] = mapping[sqbps[i,0]]
sqbps[i,1] = mapping[sqbps[i,1]]
def _compute_getbpvs(self, toread, uvdatoptions):
raras = self.raras
apidxs = dict((t[1], t[0]) for t in enumerate(self.saps))
sqbps = ArrayGrower(2, dtype=np.int)
bpdata = ArrayGrower(5)
nnorara = 0
seenidxs = set()
gen = uvdat.setupAndRead(toread,
'x3',
False,
nopass=False,
nocal=False,
nopol=False,
**uvdatoptions)
for inp, preamble, data, flags in gen:
bp = util.mir2bp(inp, preamble)
t = preamble[3]
w = np.where(flags)[0]
if not w.size:
continue
idx1, idx2 = apidxs.get(bp[0]), apidxs.get(bp[1])
if idx1 is None or idx2 is None:
nnorara += 1
continue
rdata1, rdata2 = raras[idx1], raras[idx2]
tidx1 = rdata1[0].searchsorted(t)
tidx2 = rdata2[0].searchsorted(t)
if (tidx1 < 0 or tidx1 >= rdata1.shape[1] or tidx2 < 0
or tidx2 >= rdata2.shape[1]):
nnorara += 1
continue
tr1, rara1, rwt1 = rdata1[:, tidx1]
tr2, rara2, rwt2 = rdata2[:, tidx2]
dt1 = np.abs(tr1 - t)
dt2 = np.abs(tr2 - t)
if max(dt1, dt2) > TTOL:
nnorara += 1
continue
# We propagate the weight for crara but once again,
# it's basically noiseless.
crara = rara1 * rara2
cwt = 1. / (rara2**2 / rwt1 + rara1**2 / rwt2)
data = data[w]
rvar = data.real.var(ddof=1)
ivar = data.imag.var(ddof=1)
var = 0.5 * (rvar + ivar)
# The weight of the computed variance (i.e., the
# inverse square of the maximum-likelihood variance
# in the variance measurement) is 0.5 * (nsamp - ndof) / var**2.
# We have 2*w.size samples and 2 degrees of freedom, so:
wtvar = (w.size - 1) / var**2
seenidxs.add(idx1)
seenidxs.add(idx2)
sqbps.add(idx1, idx2)
bpdata.add(t, var, wtvar, crara, cwt)
sqbps = self.sqbps = sqbps.finish()
self.bpdata = bpdata.finish()
nsamp = sqbps.shape[0]
if nnorara > nsamp * 0.02:
print >> sys.stderr, ('Warning: no autocorr data for %d/%d '
'(%.0f%%) of samples' %
(nnorara, nsamp, 100. * nnorara / nsamp))
if len(seenidxs) == len(apidxs):
return
# There exist antpols that we got autocorrelations for but
# have no contributing baselines. This can happen if there are
# no gains for the given antpol. To avoid running into a
# singular matrix in the solve step, we have to eliminate
# them. To avoid a bunch of baggage in the analysis code, we
# rewrite our data structures, which actually isn't so bad.
mapping = {}
newsaps = []
newraras = {}
for idx in xrange(len(apidxs)):
if idx in seenidxs:
mapping[idx] = len(mapping)
newsaps.append(self.saps[idx])
newraras[mapping[idx]] = raras[idx]
self.saps = newsaps
self.raras = newraras
for i in xrange(nsamp):
sqbps[i, 0] = mapping[sqbps[i, 0]]
sqbps[i, 1] = mapping[sqbps[i, 1]]
