def task (args):
"""Run as a command-line task.
args: list of strings; command-line arguments, not including the
program name (no traditional argv[0]).
returns: None
Will call mirtask.util.die on some errors.
"""
util.checkusage (__doc__, ['dummy'] + args)
ks = keys.KeySpec ()
ks.keyword ('out', 'f', ' ')
ks.keyword ('naver', 'i', -1)
ks.keyword ('slop', 'd', DEFAULT_SLOP)
ks.uvdat (UVDAT_OPTIONS + 'dslr')
opts = ks.process (args)
if opts.out == ' ':
util.wrongusage (__doc__, 'must specify an output filename (out=...)')
out = VisData (opts.out)
if opts.naver == -1:
util.wrongusage (__doc__,
'must specify the number of channels to average (naver=...)')
try:
channelAverage (out, opts.naver, opts.slop, banner=DEFAULT_BANNER, args=args)
except (InputStructureError, ValueError), e:
util.die (str (e))
def tui_checkabs (args):
import omega as om
if len (args) != 1:
util.die ('usage: <datfile>')
nc = NoiseCal ()
nc.load (args[0])
saps = nc.saps
sqbps = nc.sqbps
vals = nc.bpdata[:,1]
uncerts = nc.bpdata[:,2]**-0.5
modelvals = nc.bpdata[:,3] * nc.svals
muncerts = nc.suncerts * nc.bpdata[:,3]
pg = om.quickPager ([])
for idx in xrange (len (saps)):
w = np.where ((sqbps[:,0] == idx) | (sqbps[:,1] == idx))
p = om.quickXYErr (vals[w], modelvals[w], muncerts[w],
util.fmtAP (saps[idx]), lines=False)
p.setLabels ('Measured variance (Jy²)', 'Modeled variance (Jy²)')
pg.send (p)
pg.done ()
return 0
def tui_checkabs(args):
import omega as om
if len(args) != 1:
util.die('usage: <datfile>')
nc = NoiseCal()
nc.load(args[0])
saps = nc.saps
sqbps = nc.sqbps
vals = nc.bpdata[:, 1]
uncerts = nc.bpdata[:, 2]**-0.5
modelvals = nc.bpdata[:, 3] * nc.svals
muncerts = nc.suncerts * nc.bpdata[:, 3]
pg = om.quickPager([])
for idx in xrange(len(saps)):
w = np.where((sqbps[:, 0] == idx) | (sqbps[:, 1] == idx))
p = om.quickXYErr(vals[w],
modelvals[w],
muncerts[w],
util.fmtAP(saps[idx]),
lines=False)
p.setLabels('Measured variance (Jy²)', 'Modeled variance (Jy²)')
pg.send(p)
pg.done()
return 0
def tui_sefds(args):
if len(args) != 1:
util.die('usage: <datfile>')
nc = NoiseCal()
nc.load(args[0])
nc.printsefdinfo()
return 0
def compute (self, toread, **uvdatoptions):
for vis in toread:
if not os.path.exists (vis.path ('gains')):
util.die ('input "%s" has no gain calibration tables', vis)
self._compute_getraras (toread, uvdatoptions)
self._compute_getbpvs (toread, uvdatoptions)
self._compute_solve ()
def tui_sefds (args):
if len (args) != 1:
util.die ('usage: <datfile>')
nc = NoiseCal ()
nc.load (args[0])
nc.printsefdinfo ()
return 0
def compute(self, toread, **uvdatoptions):
for vis in toread:
if not os.path.exists(vis.path('gains')):
util.die('input "%s" has no gain calibration tables', vis)
self._compute_getraras(toread, uvdatoptions)
self._compute_getbpvs(toread, uvdatoptions)
self._compute_solve()
def _compute_getraras(self, toread, uvdatoptions):
seenaps = set()
rarawork = {}
previnp = None
gen = uvdat.setupAndRead(toread,
'a3',
False,
nopass=True,
nocal=True,
nopol=True,
**uvdatoptions)
for inp, preamble, data, flags in gen:
if previnp is None or inp is not previnp:
instr = inp.getScalarItem('arfinstr', 'UNDEF')
if instr != 'UNDEF':
if self.instr is None:
self.instr = instr
elif instr != self.instr:
util.die(
'input instrument changed from "%s" to '
'"%s" in "%s"', self.instr, instr, inp.path())
previnp = inp
# the 'a' uvdat options limits us to autocorrelations,
# but includes 1X-1Y-type autocorrelations
bp = util.mir2bp(inp, preamble)
if not util.bpIsInten(bp):
continue
ap = bp[0]
t = preamble[3]
w = np.where(flags)[0]
if not w.size:
continue
data = data[w]
rara = np.sqrt((data.real**2).mean())
seenaps.add(ap)
ag = rarawork.get(ap)
if ag is None:
ag = rarawork[ap] = ArrayGrower(3)
# We record w.size as a weight for the RARA measurement,
# but it's essentially noise-free.
ag.add(t, rara, w.size)
self.saps = sorted(seenaps)
raras = self.raras = {}
apidxs = dict((t[1], t[0]) for t in enumerate(self.saps))
for ap in rarawork.keys():
raradata = rarawork[ap].finish().T
apidx = apidxs[ap]
sidx = np.argsort(raradata[0])
raras[apidx] = raradata[:, sidx]
def tui_export(args):
if len(args) < 1:
util.die('usage: <dat1> [... datn]')
nc = NoiseCal()
for path in args:
nc.load(path, partial=True)
nc.export(sys.stdout)
return 0
def tui_export (args):
if len (args) < 1:
util.die ('usage: <dat1> [... datn]')
nc = NoiseCal ()
for path in args:
nc.load (path, partial=True)
nc.export (sys.stdout)
return 0
def _compute_getraras (self, toread, uvdatoptions):
seenaps = set ()
rarawork = {}
previnp = None
gen = uvdat.setupAndRead (toread, 'a3', False,
nopass=True, nocal=True, nopol=True,
**uvdatoptions)
for inp, preamble, data, flags in gen:
if previnp is None or inp is not previnp:
instr = inp.getScalarItem ('arfinstr', 'UNDEF')
if instr != 'UNDEF':
if self.instr is None:
self.instr = instr
elif instr != self.instr:
util.die ('input instrument changed from "%s" to '
'"%s" in "%s"', self.instr, instr,
inp.path ())
previnp = inp
# the 'a' uvdat options limits us to autocorrelations,
# but includes 1X-1Y-type autocorrelations
bp = util.mir2bp (inp, preamble)
if not util.bpIsInten (bp):
continue
ap = bp[0]
t = preamble[3]
w = np.where (flags)[0]
if not w.size:
continue
data = data[w]
rara = np.sqrt ((data.real**2).mean ())
seenaps.add (ap)
ag = rarawork.get (ap)
if ag is None:
ag = rarawork[ap] = ArrayGrower (3)
# We record w.size as a weight for the RARA measurement,
# but it's essentially noise-free.
ag.add (t, rara, w.size)
self.saps = sorted (seenaps)
raras = self.raras = {}
apidxs = dict ((t[1], t[0]) for t in enumerate (self.saps))
for ap in rarawork.keys ():
raradata = rarawork[ap].finish ().T
apidx = apidxs[ap]
sidx = np.argsort (raradata[0])
raras[apidx] = raradata[:,sidx]
def tui_checkcal (args):
import omega as om, scipy.stats as SS
toread = []
uvdatoptions = {}
for arg in args:
if '=' in arg:
key, value = arg.split ('=', 1)
uvdatoptions[key] = value
else:
toread.append (VisData (arg))
if len (toread) < 1:
util.die ('usage: <vis1> [... visn] [uvdat options]')
samples = VectorGrower ()
gen = uvdat.setupAndRead (toread, 'x3', False, **uvdatoptions)
for inp, pream, data, flags in gen:
w = np.where (flags)[0]
if w.size == 0:
continue
data = data[w]
var = 0.5 * (data.real.var (ddof=1) + data.imag.var (ddof=1))
uvar = np.sqrt (1. / (w.size - 1)) * var # uncert in variance msmt
thy = inp.getVariance ()
samples.add ((var - thy) / uvar)
samples = samples.finish ()
n = samples.size
m = samples.mean ()
s = samples.std ()
med = np.median (samples)
smadm = 1.4826 * np.median (np.abs (samples - med)) # see comment below
print ' Number of samples:', n
print 'Mean normalized error (should be 0):', m
print ' Median:', med
print ' Normalized std. dev. (should be 1):', s
print ' SMADM:', smadm
print 'Probability that samples are normal:', SS.normaltest (samples)[1]
bins = 50
rng = -5, 5
p = om.quickHist (samples, keyText='Samples', bins=bins, range=rng)
x = np.linspace (-5, 5, 200)
area = 10. / bins * n
y = area / np.sqrt (2 * np.pi) * np.exp (-0.5 * x**2)
p.addXY (x, y, 'Ideal')
p.rebound (False, False)
p.show ()
return 0
def tui_checkcal(args):
import omega as om, scipy.stats as SS
toread = []
uvdatoptions = {}
for arg in args:
if '=' in arg:
key, value = arg.split('=', 1)
uvdatoptions[key] = value
else:
toread.append(VisData(arg))
if len(toread) < 1:
util.die('usage: <vis1> [... visn] [uvdat options]')
samples = VectorGrower()
gen = uvdat.setupAndRead(toread, 'x3', False, **uvdatoptions)
for inp, pream, data, flags in gen:
w = np.where(flags)[0]
if w.size == 0:
continue
data = data[w]
var = 0.5 * (data.real.var(ddof=1) + data.imag.var(ddof=1))
uvar = np.sqrt(1. / (w.size - 1)) * var # uncert in variance msmt
thy = inp.getVariance()
samples.add((var - thy) / uvar)
samples = samples.finish()
n = samples.size
m = samples.mean()
s = samples.std()
med = np.median(samples)
smadm = 1.4826 * np.median(np.abs(samples - med)) # see comment below
print ' Number of samples:', n
print 'Mean normalized error (should be 0):', m
print ' Median:', med
print ' Normalized std. dev. (should be 1):', s
print ' SMADM:', smadm
print 'Probability that samples are normal:', SS.normaltest(samples)[1]
bins = 50
rng = -5, 5
p = om.quickHist(samples, keyText='Samples', bins=bins, range=rng)
x = np.linspace(-5, 5, 200)
area = 10. / bins * n
y = area / np.sqrt(2 * np.pi) * np.exp(-0.5 * x**2)
p.addXY(x, y, 'Ideal')
p.rebound(False, False)
p.show()
return 0
def task(args=None):
banner = util.printBannerGit('pwflux', 'calculate flux from UV data',
IDENT)
ks = keys.KeySpec()
ks.keyword('interval', 'd', 1)
ks.mkeyword('offset', 'd', 2)
ks.keyword('textapp', 'f', ' ')
ks.option('plot')
ks.uvdat('dsl3w', True)
opts = ks.process(args)
interval = opts.interval / (24. * 60.)
if interval <= 0:
print >> sys.stderr, 'Error: averaging interval must be positive, not', opts.interval
return 1
if len(opts.offset) == 0:
offset = np.zeros(2)
elif len(opts.offset) == 2:
offset = np.asarray(opts.offset) / 206265.
else:
print >> sys.stderr, (
'Error: zero or two values must be specified for source offset;'
' got'), opts.offset
return 1
if opts.plot:
try:
import omega
except ImportError:
util.die('cannot import Python module "omega" for plotting')
f = Fluxer(interval, offset)
onflush = flushPrint
if opts.textapp != ' ':
appobj = TabularAppender(file(opts.textapp, 'a'), onflush)
onflush = appobj.onFlush
if opts.plot:
plotacc = PlotAccumulator(onflush)
onflush = plotacc.onFlush
f.onFlush(onflush)
f.process()
if opts.plot:
plotacc.plot().show()
return 0
def tui_checkap(args):
import omega as om
if len(args) != 2:
util.die('usage: <datfile> <antpol>')
nc = NoiseCal()
nc.load(args[0])
ap = util.parseAP(args[1])
apidx = nc.saps.index(ap)
if apidx < 0:
util.die('no antpol %s in data file!', util.fmtAP(ap))
sqbps = nc.sqbps
vals = nc.bpdata[:, 1]
modelvals = nc.bpdata[:, 3] * nc.svals
resids = vals - modelvals
runcerts = np.sqrt(1. / nc.bpdata[:, 2] +
(nc.suncerts * nc.bpdata[:, 3])**2)
resids /= runcerts
w = np.where((sqbps[:, 0] == apidx) | (sqbps[:, 1] == apidx))
resids = resids[w]
n = resids.size
mn = resids.mean()
s = resids.std()
md = np.median(resids)
smadm = 1.4826 * np.median(np.abs(resids - md)) # see comment below
print ' Number of samples:', n
print ' Norm. mean residal:', mn
print ' Median:', md
print 'Norm. residual std. dev.:', s
print ' SMADM:', smadm
bins = 50
rng = -5, 5
p = om.quickHist(resids,
keyText='%s Residuals' % util.fmtAP(ap),
bins=bins,
range=rng)
x = np.linspace(-5, 5, 200)
area = 10. / bins * n
y = area / np.sqrt(2 * np.pi) * np.exp(-0.5 * x**2)
p.addXY(x, y, 'Ideal')
p.rebound(False, False)
p.show()
return 0
def tui_checkap (args):
import omega as om
if len (args) != 2:
util.die ('usage: <datfile> <antpol>')
nc = NoiseCal ()
nc.load (args[0])
ap = util.parseAP (args[1])
apidx = nc.saps.index (ap)
if apidx < 0:
util.die ('no antpol %s in data file!', util.fmtAP (ap))
sqbps = nc.sqbps
vals = nc.bpdata[:,1]
modelvals = nc.bpdata[:,3] * nc.svals
resids = vals - modelvals
runcerts = np.sqrt (1./nc.bpdata[:,2] + (nc.suncerts * nc.bpdata[:,3])**2)
resids /= runcerts
w = np.where ((sqbps[:,0] == apidx) | (sqbps[:,1] == apidx))
resids = resids[w]
n = resids.size
mn = resids.mean ()
s = resids.std ()
md = np.median (resids)
smadm = 1.4826 * np.median (np.abs (resids - md)) # see comment below
print ' Number of samples:', n
print ' Norm. mean residal:', mn
print ' Median:', md
print 'Norm. residual std. dev.:', s
print ' SMADM:', smadm
bins = 50
rng = -5, 5
p = om.quickHist (resids, keyText='%s Residuals' % util.fmtAP (ap),
bins=bins, range=rng)
x = np.linspace (-5, 5, 200)
area = 10. / bins * n
y = area / np.sqrt (2 * np.pi) * np.exp (-0.5 * x**2)
p.addXY (x, y, 'Ideal')
p.rebound (False, False)
p.show ()
return 0
def task (args=None):
banner = util.printBannerGit ('pwflux', 'calculate flux from UV data', IDENT)
ks = keys.KeySpec ()
ks.keyword ('interval', 'd', 1)
ks.mkeyword ('offset', 'd', 2)
ks.keyword ('textapp', 'f', ' ')
ks.option ('plot')
ks.uvdat ('dsl3w', True)
opts = ks.process (args)
interval = opts.interval / (24. * 60.)
if interval <= 0:
print >>sys.stderr, 'Error: averaging interval must be positive, not', opts.interval
return 1
if len (opts.offset) == 0:
offset = np.zeros (2)
elif len (opts.offset) == 2:
offset = np.asarray (opts.offset) / 206265.
else:
print >>sys.stderr, ('Error: zero or two values must be specified for source offset;'
' got'), opts.offset
return 1
if opts.plot:
try:
import omega
except ImportError:
util.die ('cannot import Python module "omega" for plotting')
f = Fluxer (interval, offset)
onflush = flushPrint
if opts.textapp != ' ':
appobj = TabularAppender (file (opts.textapp, 'a'), onflush)
onflush = appobj.onFlush
if opts.plot:
plotacc = PlotAccumulator (onflush)
onflush = plotacc.onFlush
f.onFlush (onflush)
f.process ()
if opts.plot:
plotacc.plot ().show ()
return 0
def tui_compute(args):
toread = []
uvdatoptions = {}
for arg in args:
if '=' in arg:
key, value = arg.split('=', 1)
uvdatoptions[key] = value
else:
toread.append(arg)
if len(toread) < 2:
util.die('usage: <vis1> [... visn] [uvdat options] <output name>')
outpath = toread[-1]
toread = [VisData(x) for x in toread[:-1]]
try:
tmp = open(outpath, 'w')
tmp.close()
except Exception, e:
util.die('cannot open output path "%s" for writing: %s', outpath, e)
def tui_compute (args):
toread = []
uvdatoptions = {}
for arg in args:
if '=' in arg:
key, value = arg.split ('=', 1)
uvdatoptions[key] = value
else:
toread.append (arg)
if len (toread) < 2:
util.die ('usage: <vis1> [... visn] [uvdat options] <output name>')
outpath = toread[-1]
toread = [VisData (x) for x in toread[:-1]]
try:
tmp = open (outpath, 'w')
tmp.close ()
except Exception, e:
util.die ('cannot open output path "%s" for writing: %s',
outpath, e)
def task(args):
from miriad import CalData
from mirtask import cliutil, keys
banner = util.printBannerGit('gptext', 'convert gains to and from text',
IDENT)
ks = keys.KeySpec()
ks.keyword('vis', 'f', ' ')
ks.keyword('out', 'f', ' ')
ks.keymatch('mode', 1, ['totext', 'applyvis', 'createvis'])
opts = ks.process(args)
if opts.out == ' ':
util.die('must specify an output filename (out=...)')
if opts.vis == ' ':
util.die('must specify an input filename (vis=...)')
mode = opts.mode[0]
gi = GainsInfo()
if mode == 'totext':
gi.fromDataset(CalData(opts.vis).open('rw'))
if opts.out == '-':
gi.toText(sys.stdout)
else:
gi.toText(open(opts.out, 'w'))
elif mode == 'applyvis':
gi.fromText(open(opts.vis))
h = CalData(opts.out).open('rw')
gi.toDataset(h)
h.openHistory()
h.writeHistory(banner)
h.logInvocation('PYTHON gptext', args)
h.closeHistory()
h.close()
elif mode == 'createvis':
gi.fromText(open(opts.vis))
h = CalData(opts.out).open('c')
gi.toDataset(h)
h.openHistory()
h.writeHistory(banner)
h.logInvocation('PYTHON gptext', args)
h.closeHistory()
h.close()
else:
util.die('unrecognized mode "%s"', mode)
def task (args):
from miriad import CalData
from mirtask import cliutil, keys
banner = util.printBannerGit ('gptext', 'convert gains to and from text',
IDENT)
ks = keys.KeySpec ()
ks.keyword ('vis', 'f', ' ')
ks.keyword ('out', 'f', ' ')
ks.keymatch ('mode', 1, ['totext', 'applyvis', 'createvis'])
opts = ks.process (args)
if opts.out == ' ':
util.die ('must specify an output filename (out=...)')
if opts.vis == ' ':
util.die ('must specify an input filename (vis=...)')
mode = opts.mode[0]
gi = GainsInfo ()
if mode == 'totext':
gi.fromDataset (CalData (opts.vis).open ('rw'))
if opts.out == '-':
gi.toText (sys.stdout)
else:
gi.toText (open (opts.out, 'w'))
elif mode == 'applyvis':
gi.fromText (open (opts.vis))
h = CalData (opts.out).open ('rw')
gi.toDataset (h)
h.openHistory ()
h.writeHistory (banner)
h.logInvocation ('PYTHON gptext', args)
h.closeHistory ()
h.close ()
elif mode == 'createvis':
gi.fromText (open (opts.vis))
h = CalData (opts.out).open ('c')
gi.toDataset (h)
h.openHistory ()
h.writeHistory (banner)
h.logInvocation ('PYTHON gptext', args)
h.closeHistory ()
h.close ()
else:
util.die ('unrecognized mode "%s"', mode)
def tui_checkfit(args):
import omega as om
if len(args) != 1:
util.die('usage: <datfile>')
nc = NoiseCal()
nc.load(args[0])
vals = nc.bpdata[:, 1]
modelvals = nc.bpdata[:, 3] * nc.svals
resids = vals - modelvals
runcerts = np.sqrt(1. / nc.bpdata[:, 2] +
(nc.suncerts * nc.bpdata[:, 3])**2)
normresids = resids / runcerts
n = normresids.size
mn = normresids.mean()
s = normresids.std()
md = np.median(normresids)
smadm = 1.4826 * np.median(np.abs(normresids - md)) # see comment below
print ' Number of samples:', n
print ' Normalized mean residal:', mn
print ' Median:', md
print 'Normalized std. dev. (should be 1):', s
print ' SMADM:', smadm
# Check for problematic antpols and basepols
saps = nc.saps
sqbps = nc.sqbps
nap = len(saps)
dumbnbp = nap**2
apcounts = np.zeros(nap, dtype=np.int)
apsumsqresids = np.zeros(nap)
bpcounts = np.zeros(dumbnbp, dtype=np.int)
bpsumsqresids = np.zeros(dumbnbp)
for i in xrange(n):
idx1, idx2 = sqbps[i]
apcounts[idx1] += 1
apsumsqresids[idx1] += normresids[i]**2
apcounts[idx2] += 1
apsumsqresids[idx2] += normresids[i]**2
bpidx = idx1 * nap + idx2
bpcounts[bpidx] += 1
bpsumsqresids[bpidx] += normresids[i]**2
aprmsresids = np.sqrt(apsumsqresids / apcounts)
sapresids = np.argsort(aprmsresids)
print
print 'Extreme residual RMS by antpol:'
for i in xrange(5):
idx = sapresids[i]
print ' %10s %8.2f' % (util.fmtAP(saps[idx]), aprmsresids[idx])
print ' ....'
for i in xrange(5):
idx = sapresids[i - 5]
print ' %10s %8.2f' % (util.fmtAP(saps[idx]), aprmsresids[idx])
wbpgood = np.where(bpcounts)[0]
wbpbad = np.where(bpcounts == 0)[0]
bpcounts[wbpbad] = 1
bprmsresids = bpsumsqresids / bpcounts
sbpresids = np.argsort(bprmsresids[wbpgood])
print
print 'Extreme residual RMS by basepol:'
for i in xrange(3):
idx = wbpgood[sbpresids[i]]
ap2 = saps[idx % nap]
ap1 = saps[idx // nap]
print ' %10s %8.2f' % (util.fmtBP((ap1, ap2)), bprmsresids[idx])
print ' ....'
for i in xrange(7):
idx = wbpgood[sbpresids[i - 7]]
ap2 = saps[idx % nap]
ap1 = saps[idx // nap]
print ' %10s %8.2f' % (util.fmtBP((ap1, ap2)), bprmsresids[idx])
# Plot the distribution of residuals
bins = 50
rng = -5, 5
p = om.quickHist(normresids, keyText='Residuals', bins=bins, range=rng)
x = np.linspace(-5, 5, 200)
area = 10. / bins * n
y = area / np.sqrt(2 * np.pi) * np.exp(-0.5 * x**2)
p.addXY(x, y, 'Ideal')
p.rebound(False, False)
p.show()
return 0
def _compute_solve(self):
nap = len(self.saps)
sqbps = self.sqbps
bpdata = self.bpdata
nsamp = bpdata.shape[0]
# Get approximate solution by using a linear least squares
# solver on the log of our equation.
coeffs = np.zeros((nap, nsamp))
values = np.empty(nsamp)
invsigmas = np.empty(nsamp)
for i in xrange(nsamp):
idx1, idx2 = sqbps[i]
var, wtvar, crara = bpdata[i, 1:4]
coeffs[idx1, i] = coeffs[idx2, i] = 1
values[i] = np.log(var / crara)
# Here we ignore the noise in crara.
invsigmas[i] = crara * np.sqrt(wtvar)
soln = util.linLeastSquares(coeffs, values)
# Now refine and get uncertainties with a nonlinear solver.
soln = np.exp(soln)
values = np.exp(values)
modelvals = np.empty(nsamp)
def nonlin(params):
for i in xrange(nsamp):
idx1, idx2 = sqbps[i]
modelvals[i] = params[idx1] * params[idx2]
return (values - modelvals) * invsigmas
from scipy.optimize import leastsq
soln, cov, misc, mesg, flag = leastsq(nonlin, soln, full_output=True)
if flag < 1 or flag > 4 or cov is None:
if cov is None:
expln = 'encountered singular matrix'
else:
expln = 'return flag was %d' % flag
if 'CALNOISE_DEBUG' in os.environ:
print >> sys.stderr, 'Nonlinear fit failed! Saving anyway!'
print >> sys.stderr, 'expln:', expln
print >> sys.stderr, 'mesg:', mesg
cov = None
else:
util.die('nonlinear fit failed: %s; mesg: %s', expln, mesg)
rchisq = (((values - modelvals) * invsigmas)**2).sum() / (nsamp - nap)
print 'reduced chi squared: %.3f for %d DOF' % (rchisq, nsamp - nap)
if cov is None:
cov = suncerts = np.empty(0)
else:
cov *= rchisq # copying scipy.optimize.curve_fit
suncerts = np.empty(nsamp)
for i in xrange(nsamp):
idx1, idx2 = sqbps[i]
suncerts[i] = np.sqrt((soln[idx1]**2 * cov[idx2, idx2] +
soln[idx2]**2 * cov[idx1, idx1]))
self.solution = soln
self.covar = cov
self.svals = modelvals
self.suncerts = suncerts
def dualSelfCal (vis, out, usemself=False, ttol=DEFAULT_TTOL, outexists=False,
serial=False, select='', postinterval=None,
banner='PYTHON dualSelfCal', **kwargs):
vis = ensureiterable (vis)
if len (vis) == 0:
util.die ('must specify at least one visibility file')
for ivis in vis:
if not ivis.exists:
util.die ('visibility input file %s does not exist', ivis)
if usemself:
task = TaskMSelfCal ()
else:
task = TaskSelfCal ()
if 'verbose' in kwargs and kwargs['verbose']:
util.die ('option "verbose" only supported by mselfcal')
if ttol <= 0:
util.die ('parameter "ttol" must be positive')
if out is None:
if len (vis) != 1:
util.die ('must specify "out" if more than '
'one visibility input specified')
dest = vis[0].open ('rw')
else:
if outexists:
dest = out.open ('rw')
else:
dest = out.open ('c')
src1 = vis[0].open ('rw')
src1.copyItem (dest, 'history')
src1.close ()
if len (select) == 0:
selformat = 'pol(%s)'
else:
selformat = select + ',pol(%s)'
task.vis = vis
task.set (**kwargs)
# Go
worksets = []
procs = []
for pol in 'xx', 'yy':
workset = vis[0].makeVariant ('sc' + pol, CalData)
workset.delete ()
worksets.append (workset)
task.select = selformat % pol
task.out = workset
if serial:
task.run ()
else:
procs.append (task.launch ())
for proc in procs:
proc.checkwait ()
src1 = worksets[0].open ('rw')
src2 = worksets[1].open ('rw')
merge (str (worksets[0]), src1, str (worksets[1]), src2, dest, banner, ttol)
for workset in worksets:
workset.delete ()
if postinterval is not None:
dest.setScalarItem ('interval', np.float32, postinterval / (60. * 24))
util.die('usage: <dat1> [... datn]')
nc = NoiseCal()
for path in args:
nc.load(path, partial=True)
nc.export(sys.stdout)
return 0
def tui_sefds(args):
if len(args) != 1:
util.die('usage: <datfile>')
nc = NoiseCal()
nc.load(args[0])
nc.printsefdinfo()
return 0
if __name__ == '__main__':
if len(sys.argv) == 1:
util.die(
'add a subcommand: one of "checkabs checkap checkcal checkfit '
'compute export sefds"')
subcommand = sys.argv[1]
if 'tui_' + subcommand not in globals():
util.die('unknown subcommand "%s"', subcommand)
sys.exit(globals()['tui_' + subcommand](sys.argv[2:]))
if len (args) < 1:
util.die ('usage: <dat1> [... datn]')
nc = NoiseCal ()
for path in args:
nc.load (path, partial=True)
nc.export (sys.stdout)
return 0
def tui_sefds (args):
if len (args) != 1:
util.die ('usage: <datfile>')
nc = NoiseCal ()
nc.load (args[0])
nc.printsefdinfo ()
return 0
if __name__ == '__main__':
if len (sys.argv) == 1:
util.die ('add a subcommand: one of "checkabs checkap checkcal checkfit '
'compute export sefds"')
subcommand = sys.argv[1]
if 'tui_' + subcommand not in globals ():
util.die ('unknown subcommand "%s"', subcommand)
sys.exit (globals ()['tui_' + subcommand] (sys.argv[2:]))
def tui_checkfit (args):
import omega as om
if len (args) != 1:
util.die ('usage: <datfile>')
nc = NoiseCal ()
nc.load (args[0])
vals = nc.bpdata[:,1]
modelvals = nc.bpdata[:,3] * nc.svals
resids = vals - modelvals
runcerts = np.sqrt (1./nc.bpdata[:,2] + (nc.suncerts * nc.bpdata[:,3])**2)
normresids = resids / runcerts
n = normresids.size
mn = normresids.mean ()
s = normresids.std ()
md = np.median (normresids)
smadm = 1.4826 * np.median (np.abs (normresids - md)) # see comment below
print ' Number of samples:', n
print ' Normalized mean residal:', mn
print ' Median:', md
print 'Normalized std. dev. (should be 1):', s
print ' SMADM:', smadm
# Check for problematic antpols and basepols
saps = nc.saps
sqbps = nc.sqbps
nap = len (saps)
dumbnbp = nap**2
apcounts = np.zeros (nap, dtype=np.int)
apsumsqresids = np.zeros (nap)
bpcounts = np.zeros (dumbnbp, dtype=np.int)
bpsumsqresids = np.zeros (dumbnbp)
for i in xrange (n):
idx1, idx2 = sqbps[i]
apcounts[idx1] += 1
apsumsqresids[idx1] += normresids[i]**2
apcounts[idx2] += 1
apsumsqresids[idx2] += normresids[i]**2
bpidx = idx1 * nap + idx2
bpcounts[bpidx] += 1
bpsumsqresids[bpidx] += normresids[i]**2
aprmsresids = np.sqrt (apsumsqresids / apcounts)
sapresids = np.argsort (aprmsresids)
print
print 'Extreme residual RMS by antpol:'
for i in xrange (5):
idx = sapresids[i]
print ' %10s %8.2f' % (util.fmtAP (saps[idx]), aprmsresids[idx])
print ' ....'
for i in xrange (5):
idx = sapresids[i - 5]
print ' %10s %8.2f' % (util.fmtAP (saps[idx]), aprmsresids[idx])
wbpgood = np.where (bpcounts)[0]
wbpbad = np.where (bpcounts == 0)[0]
bpcounts[wbpbad] = 1
bprmsresids = bpsumsqresids / bpcounts
sbpresids = np.argsort (bprmsresids[wbpgood])
print
print 'Extreme residual RMS by basepol:'
for i in xrange (3):
idx = wbpgood[sbpresids[i]]
ap2 = saps[idx % nap]
ap1 = saps[idx // nap]
print ' %10s %8.2f' % (util.fmtBP ((ap1, ap2)), bprmsresids[idx])
print ' ....'
for i in xrange (7):
idx = wbpgood[sbpresids[i - 7]]
ap2 = saps[idx % nap]
ap1 = saps[idx // nap]
print ' %10s %8.2f' % (util.fmtBP ((ap1, ap2)), bprmsresids[idx])
# Plot the distribution of residuals
bins = 50
rng = -5, 5
p = om.quickHist (normresids, keyText='Residuals', bins=bins, range=rng)
x = np.linspace (-5, 5, 200)
area = 10. / bins * n
y = area / np.sqrt (2 * np.pi) * np.exp (-0.5 * x**2)
p.addXY (x, y, 'Ideal')
p.rebound (False, False)
p.show ()
return 0
def _compute_solve (self):
nap = len (self.saps)
sqbps = self.sqbps
bpdata = self.bpdata
nsamp = bpdata.shape[0]
# Get approximate solution by using a linear least squares
# solver on the log of our equation.
coeffs = np.zeros ((nap, nsamp))
values = np.empty (nsamp)
invsigmas = np.empty (nsamp)
for i in xrange (nsamp):
idx1, idx2 = sqbps[i]
var, wtvar, crara = bpdata[i,1:4]
coeffs[idx1,i] = coeffs[idx2,i] = 1
values[i] = np.log (var / crara)
# Here we ignore the noise in crara.
invsigmas[i] = crara * np.sqrt (wtvar)
soln = util.linLeastSquares (coeffs, values)
# Now refine and get uncertainties with a nonlinear solver.
soln = np.exp (soln)
values = np.exp (values)
modelvals = np.empty (nsamp)
def nonlin (params):
for i in xrange (nsamp):
idx1, idx2 = sqbps[i]
modelvals[i] = params[idx1] * params[idx2]
return (values - modelvals) * invsigmas
from scipy.optimize import leastsq
soln, cov, misc, mesg, flag = leastsq (nonlin, soln, full_output=True)
if flag < 1 or flag > 4 or cov is None:
if cov is None:
expln = 'encountered singular matrix'
else:
expln = 'return flag was %d' % flag
if 'CALNOISE_DEBUG' in os.environ:
print >>sys.stderr, 'Nonlinear fit failed! Saving anyway!'
print >>sys.stderr, 'expln:', expln
print >>sys.stderr, 'mesg:', mesg
cov = None
else:
util.die ('nonlinear fit failed: %s; mesg: %s', expln, mesg)
rchisq = (((values - modelvals) * invsigmas)**2).sum () / (nsamp - nap)
print 'reduced chi squared: %.3f for %d DOF' % (rchisq, nsamp - nap)
if cov is None:
cov = suncerts = np.empty (0)
else:
cov *= rchisq # copying scipy.optimize.curve_fit
suncerts = np.empty (nsamp)
for i in xrange (nsamp):
idx1, idx2 = sqbps[i]
suncerts[i] = np.sqrt ((soln[idx1]**2 * cov[idx2,idx2] +
soln[idx2]**2 * cov[idx1,idx1]))
self.solution = soln
self.covar = cov
self.svals = modelvals
self.suncerts = suncerts
