def task (args): """Run as a command-line task. args: list of strings; command-line arguments, not including the program name (no traditional argv[0]). returns: None Will call mirtask.util.die on some errors. """ util.checkusage (__doc__, ['dummy'] + args) ks = keys.KeySpec () ks.keyword ('out', 'f', ' ') ks.keyword ('naver', 'i', -1) ks.keyword ('slop', 'd', DEFAULT_SLOP) ks.uvdat (UVDAT_OPTIONS + 'dslr') opts = ks.process (args) if opts.out == ' ': util.wrongusage (__doc__, 'must specify an output filename (out=...)') out = VisData (opts.out) if opts.naver == -1: util.wrongusage (__doc__, 'must specify the number of channels to average (naver=...)') try: channelAverage (out, opts.naver, opts.slop, banner=DEFAULT_BANNER, args=args) except (InputStructureError, ValueError), e: util.die (str (e))
def tui_checkabs (args): import omega as om if len (args) != 1: util.die ('usage: <datfile>') nc = NoiseCal () nc.load (args[0]) saps = nc.saps sqbps = nc.sqbps vals = nc.bpdata[:,1] uncerts = nc.bpdata[:,2]**-0.5 modelvals = nc.bpdata[:,3] * nc.svals muncerts = nc.suncerts * nc.bpdata[:,3] pg = om.quickPager ([]) for idx in xrange (len (saps)): w = np.where ((sqbps[:,0] == idx) | (sqbps[:,1] == idx)) p = om.quickXYErr (vals[w], modelvals[w], muncerts[w], util.fmtAP (saps[idx]), lines=False) p.setLabels ('Measured variance (Jy²)', 'Modeled variance (Jy²)') pg.send (p) pg.done () return 0
def tui_checkabs(args): import omega as om if len(args) != 1: util.die('usage: <datfile>') nc = NoiseCal() nc.load(args[0]) saps = nc.saps sqbps = nc.sqbps vals = nc.bpdata[:, 1] uncerts = nc.bpdata[:, 2]**-0.5 modelvals = nc.bpdata[:, 3] * nc.svals muncerts = nc.suncerts * nc.bpdata[:, 3] pg = om.quickPager([]) for idx in xrange(len(saps)): w = np.where((sqbps[:, 0] == idx) | (sqbps[:, 1] == idx)) p = om.quickXYErr(vals[w], modelvals[w], muncerts[w], util.fmtAP(saps[idx]), lines=False) p.setLabels('Measured variance (Jy²)', 'Modeled variance (Jy²)') pg.send(p) pg.done() return 0
def tui_sefds(args): if len(args) != 1: util.die('usage: <datfile>') nc = NoiseCal() nc.load(args[0]) nc.printsefdinfo() return 0
def compute (self, toread, **uvdatoptions): for vis in toread: if not os.path.exists (vis.path ('gains')): util.die ('input "%s" has no gain calibration tables', vis) self._compute_getraras (toread, uvdatoptions) self._compute_getbpvs (toread, uvdatoptions) self._compute_solve ()
def tui_sefds (args): if len (args) != 1: util.die ('usage: <datfile>') nc = NoiseCal () nc.load (args[0]) nc.printsefdinfo () return 0
def compute(self, toread, **uvdatoptions): for vis in toread: if not os.path.exists(vis.path('gains')): util.die('input "%s" has no gain calibration tables', vis) self._compute_getraras(toread, uvdatoptions) self._compute_getbpvs(toread, uvdatoptions) self._compute_solve()
def _compute_getraras(self, toread, uvdatoptions): seenaps = set() rarawork = {} previnp = None gen = uvdat.setupAndRead(toread, 'a3', False, nopass=True, nocal=True, nopol=True, **uvdatoptions) for inp, preamble, data, flags in gen: if previnp is None or inp is not previnp: instr = inp.getScalarItem('arfinstr', 'UNDEF') if instr != 'UNDEF': if self.instr is None: self.instr = instr elif instr != self.instr: util.die( 'input instrument changed from "%s" to ' '"%s" in "%s"', self.instr, instr, inp.path()) previnp = inp # the 'a' uvdat options limits us to autocorrelations, # but includes 1X-1Y-type autocorrelations bp = util.mir2bp(inp, preamble) if not util.bpIsInten(bp): continue ap = bp[0] t = preamble[3] w = np.where(flags)[0] if not w.size: continue data = data[w] rara = np.sqrt((data.real**2).mean()) seenaps.add(ap) ag = rarawork.get(ap) if ag is None: ag = rarawork[ap] = ArrayGrower(3) # We record w.size as a weight for the RARA measurement, # but it's essentially noise-free. ag.add(t, rara, w.size) self.saps = sorted(seenaps) raras = self.raras = {} apidxs = dict((t[1], t[0]) for t in enumerate(self.saps)) for ap in rarawork.keys(): raradata = rarawork[ap].finish().T apidx = apidxs[ap] sidx = np.argsort(raradata[0]) raras[apidx] = raradata[:, sidx]
def tui_export(args): if len(args) < 1: util.die('usage: <dat1> [... datn]') nc = NoiseCal() for path in args: nc.load(path, partial=True) nc.export(sys.stdout) return 0
def tui_export (args): if len (args) < 1: util.die ('usage: <dat1> [... datn]') nc = NoiseCal () for path in args: nc.load (path, partial=True) nc.export (sys.stdout) return 0
def _compute_getraras (self, toread, uvdatoptions): seenaps = set () rarawork = {} previnp = None gen = uvdat.setupAndRead (toread, 'a3', False, nopass=True, nocal=True, nopol=True, **uvdatoptions) for inp, preamble, data, flags in gen: if previnp is None or inp is not previnp: instr = inp.getScalarItem ('arfinstr', 'UNDEF') if instr != 'UNDEF': if self.instr is None: self.instr = instr elif instr != self.instr: util.die ('input instrument changed from "%s" to ' '"%s" in "%s"', self.instr, instr, inp.path ()) previnp = inp # the 'a' uvdat options limits us to autocorrelations, # but includes 1X-1Y-type autocorrelations bp = util.mir2bp (inp, preamble) if not util.bpIsInten (bp): continue ap = bp[0] t = preamble[3] w = np.where (flags)[0] if not w.size: continue data = data[w] rara = np.sqrt ((data.real**2).mean ()) seenaps.add (ap) ag = rarawork.get (ap) if ag is None: ag = rarawork[ap] = ArrayGrower (3) # We record w.size as a weight for the RARA measurement, # but it's essentially noise-free. ag.add (t, rara, w.size) self.saps = sorted (seenaps) raras = self.raras = {} apidxs = dict ((t[1], t[0]) for t in enumerate (self.saps)) for ap in rarawork.keys (): raradata = rarawork[ap].finish ().T apidx = apidxs[ap] sidx = np.argsort (raradata[0]) raras[apidx] = raradata[:,sidx]
def tui_checkcal (args): import omega as om, scipy.stats as SS toread = [] uvdatoptions = {} for arg in args: if '=' in arg: key, value = arg.split ('=', 1) uvdatoptions[key] = value else: toread.append (VisData (arg)) if len (toread) < 1: util.die ('usage: <vis1> [... visn] [uvdat options]') samples = VectorGrower () gen = uvdat.setupAndRead (toread, 'x3', False, **uvdatoptions) for inp, pream, data, flags in gen: w = np.where (flags)[0] if w.size == 0: continue data = data[w] var = 0.5 * (data.real.var (ddof=1) + data.imag.var (ddof=1)) uvar = np.sqrt (1. / (w.size - 1)) * var # uncert in variance msmt thy = inp.getVariance () samples.add ((var - thy) / uvar) samples = samples.finish () n = samples.size m = samples.mean () s = samples.std () med = np.median (samples) smadm = 1.4826 * np.median (np.abs (samples - med)) # see comment below print ' Number of samples:', n print 'Mean normalized error (should be 0):', m print ' Median:', med print ' Normalized std. dev. (should be 1):', s print ' SMADM:', smadm print 'Probability that samples are normal:', SS.normaltest (samples)[1] bins = 50 rng = -5, 5 p = om.quickHist (samples, keyText='Samples', bins=bins, range=rng) x = np.linspace (-5, 5, 200) area = 10. / bins * n y = area / np.sqrt (2 * np.pi) * np.exp (-0.5 * x**2) p.addXY (x, y, 'Ideal') p.rebound (False, False) p.show () return 0
def tui_checkcal(args): import omega as om, scipy.stats as SS toread = [] uvdatoptions = {} for arg in args: if '=' in arg: key, value = arg.split('=', 1) uvdatoptions[key] = value else: toread.append(VisData(arg)) if len(toread) < 1: util.die('usage: <vis1> [... visn] [uvdat options]') samples = VectorGrower() gen = uvdat.setupAndRead(toread, 'x3', False, **uvdatoptions) for inp, pream, data, flags in gen: w = np.where(flags)[0] if w.size == 0: continue data = data[w] var = 0.5 * (data.real.var(ddof=1) + data.imag.var(ddof=1)) uvar = np.sqrt(1. / (w.size - 1)) * var # uncert in variance msmt thy = inp.getVariance() samples.add((var - thy) / uvar) samples = samples.finish() n = samples.size m = samples.mean() s = samples.std() med = np.median(samples) smadm = 1.4826 * np.median(np.abs(samples - med)) # see comment below print ' Number of samples:', n print 'Mean normalized error (should be 0):', m print ' Median:', med print ' Normalized std. dev. (should be 1):', s print ' SMADM:', smadm print 'Probability that samples are normal:', SS.normaltest(samples)[1] bins = 50 rng = -5, 5 p = om.quickHist(samples, keyText='Samples', bins=bins, range=rng) x = np.linspace(-5, 5, 200) area = 10. / bins * n y = area / np.sqrt(2 * np.pi) * np.exp(-0.5 * x**2) p.addXY(x, y, 'Ideal') p.rebound(False, False) p.show() return 0
def task(args=None): banner = util.printBannerGit('pwflux', 'calculate flux from UV data', IDENT) ks = keys.KeySpec() ks.keyword('interval', 'd', 1) ks.mkeyword('offset', 'd', 2) ks.keyword('textapp', 'f', ' ') ks.option('plot') ks.uvdat('dsl3w', True) opts = ks.process(args) interval = opts.interval / (24. * 60.) if interval <= 0: print >> sys.stderr, 'Error: averaging interval must be positive, not', opts.interval return 1 if len(opts.offset) == 0: offset = np.zeros(2) elif len(opts.offset) == 2: offset = np.asarray(opts.offset) / 206265. else: print >> sys.stderr, ( 'Error: zero or two values must be specified for source offset;' ' got'), opts.offset return 1 if opts.plot: try: import omega except ImportError: util.die('cannot import Python module "omega" for plotting') f = Fluxer(interval, offset) onflush = flushPrint if opts.textapp != ' ': appobj = TabularAppender(file(opts.textapp, 'a'), onflush) onflush = appobj.onFlush if opts.plot: plotacc = PlotAccumulator(onflush) onflush = plotacc.onFlush f.onFlush(onflush) f.process() if opts.plot: plotacc.plot().show() return 0
def tui_checkap(args): import omega as om if len(args) != 2: util.die('usage: <datfile> <antpol>') nc = NoiseCal() nc.load(args[0]) ap = util.parseAP(args[1]) apidx = nc.saps.index(ap) if apidx < 0: util.die('no antpol %s in data file!', util.fmtAP(ap)) sqbps = nc.sqbps vals = nc.bpdata[:, 1] modelvals = nc.bpdata[:, 3] * nc.svals resids = vals - modelvals runcerts = np.sqrt(1. / nc.bpdata[:, 2] + (nc.suncerts * nc.bpdata[:, 3])**2) resids /= runcerts w = np.where((sqbps[:, 0] == apidx) | (sqbps[:, 1] == apidx)) resids = resids[w] n = resids.size mn = resids.mean() s = resids.std() md = np.median(resids) smadm = 1.4826 * np.median(np.abs(resids - md)) # see comment below print ' Number of samples:', n print ' Norm. mean residal:', mn print ' Median:', md print 'Norm. residual std. dev.:', s print ' SMADM:', smadm bins = 50 rng = -5, 5 p = om.quickHist(resids, keyText='%s Residuals' % util.fmtAP(ap), bins=bins, range=rng) x = np.linspace(-5, 5, 200) area = 10. / bins * n y = area / np.sqrt(2 * np.pi) * np.exp(-0.5 * x**2) p.addXY(x, y, 'Ideal') p.rebound(False, False) p.show() return 0
def tui_checkap (args): import omega as om if len (args) != 2: util.die ('usage: <datfile> <antpol>') nc = NoiseCal () nc.load (args[0]) ap = util.parseAP (args[1]) apidx = nc.saps.index (ap) if apidx < 0: util.die ('no antpol %s in data file!', util.fmtAP (ap)) sqbps = nc.sqbps vals = nc.bpdata[:,1] modelvals = nc.bpdata[:,3] * nc.svals resids = vals - modelvals runcerts = np.sqrt (1./nc.bpdata[:,2] + (nc.suncerts * nc.bpdata[:,3])**2) resids /= runcerts w = np.where ((sqbps[:,0] == apidx) | (sqbps[:,1] == apidx)) resids = resids[w] n = resids.size mn = resids.mean () s = resids.std () md = np.median (resids) smadm = 1.4826 * np.median (np.abs (resids - md)) # see comment below print ' Number of samples:', n print ' Norm. mean residal:', mn print ' Median:', md print 'Norm. residual std. dev.:', s print ' SMADM:', smadm bins = 50 rng = -5, 5 p = om.quickHist (resids, keyText='%s Residuals' % util.fmtAP (ap), bins=bins, range=rng) x = np.linspace (-5, 5, 200) area = 10. / bins * n y = area / np.sqrt (2 * np.pi) * np.exp (-0.5 * x**2) p.addXY (x, y, 'Ideal') p.rebound (False, False) p.show () return 0
def task (args=None): banner = util.printBannerGit ('pwflux', 'calculate flux from UV data', IDENT) ks = keys.KeySpec () ks.keyword ('interval', 'd', 1) ks.mkeyword ('offset', 'd', 2) ks.keyword ('textapp', 'f', ' ') ks.option ('plot') ks.uvdat ('dsl3w', True) opts = ks.process (args) interval = opts.interval / (24. * 60.) if interval <= 0: print >>sys.stderr, 'Error: averaging interval must be positive, not', opts.interval return 1 if len (opts.offset) == 0: offset = np.zeros (2) elif len (opts.offset) == 2: offset = np.asarray (opts.offset) / 206265. else: print >>sys.stderr, ('Error: zero or two values must be specified for source offset;' ' got'), opts.offset return 1 if opts.plot: try: import omega except ImportError: util.die ('cannot import Python module "omega" for plotting') f = Fluxer (interval, offset) onflush = flushPrint if opts.textapp != ' ': appobj = TabularAppender (file (opts.textapp, 'a'), onflush) onflush = appobj.onFlush if opts.plot: plotacc = PlotAccumulator (onflush) onflush = plotacc.onFlush f.onFlush (onflush) f.process () if opts.plot: plotacc.plot ().show () return 0
def tui_compute(args): toread = [] uvdatoptions = {} for arg in args: if '=' in arg: key, value = arg.split('=', 1) uvdatoptions[key] = value else: toread.append(arg) if len(toread) < 2: util.die('usage: <vis1> [... visn] [uvdat options] <output name>') outpath = toread[-1] toread = [VisData(x) for x in toread[:-1]] try: tmp = open(outpath, 'w') tmp.close() except Exception, e: util.die('cannot open output path "%s" for writing: %s', outpath, e)
def tui_compute (args): toread = [] uvdatoptions = {} for arg in args: if '=' in arg: key, value = arg.split ('=', 1) uvdatoptions[key] = value else: toread.append (arg) if len (toread) < 2: util.die ('usage: <vis1> [... visn] [uvdat options] <output name>') outpath = toread[-1] toread = [VisData (x) for x in toread[:-1]] try: tmp = open (outpath, 'w') tmp.close () except Exception, e: util.die ('cannot open output path "%s" for writing: %s', outpath, e)
def task(args): from miriad import CalData from mirtask import cliutil, keys banner = util.printBannerGit('gptext', 'convert gains to and from text', IDENT) ks = keys.KeySpec() ks.keyword('vis', 'f', ' ') ks.keyword('out', 'f', ' ') ks.keymatch('mode', 1, ['totext', 'applyvis', 'createvis']) opts = ks.process(args) if opts.out == ' ': util.die('must specify an output filename (out=...)') if opts.vis == ' ': util.die('must specify an input filename (vis=...)') mode = opts.mode[0] gi = GainsInfo() if mode == 'totext': gi.fromDataset(CalData(opts.vis).open('rw')) if opts.out == '-': gi.toText(sys.stdout) else: gi.toText(open(opts.out, 'w')) elif mode == 'applyvis': gi.fromText(open(opts.vis)) h = CalData(opts.out).open('rw') gi.toDataset(h) h.openHistory() h.writeHistory(banner) h.logInvocation('PYTHON gptext', args) h.closeHistory() h.close() elif mode == 'createvis': gi.fromText(open(opts.vis)) h = CalData(opts.out).open('c') gi.toDataset(h) h.openHistory() h.writeHistory(banner) h.logInvocation('PYTHON gptext', args) h.closeHistory() h.close() else: util.die('unrecognized mode "%s"', mode)
def task (args): from miriad import CalData from mirtask import cliutil, keys banner = util.printBannerGit ('gptext', 'convert gains to and from text', IDENT) ks = keys.KeySpec () ks.keyword ('vis', 'f', ' ') ks.keyword ('out', 'f', ' ') ks.keymatch ('mode', 1, ['totext', 'applyvis', 'createvis']) opts = ks.process (args) if opts.out == ' ': util.die ('must specify an output filename (out=...)') if opts.vis == ' ': util.die ('must specify an input filename (vis=...)') mode = opts.mode[0] gi = GainsInfo () if mode == 'totext': gi.fromDataset (CalData (opts.vis).open ('rw')) if opts.out == '-': gi.toText (sys.stdout) else: gi.toText (open (opts.out, 'w')) elif mode == 'applyvis': gi.fromText (open (opts.vis)) h = CalData (opts.out).open ('rw') gi.toDataset (h) h.openHistory () h.writeHistory (banner) h.logInvocation ('PYTHON gptext', args) h.closeHistory () h.close () elif mode == 'createvis': gi.fromText (open (opts.vis)) h = CalData (opts.out).open ('c') gi.toDataset (h) h.openHistory () h.writeHistory (banner) h.logInvocation ('PYTHON gptext', args) h.closeHistory () h.close () else: util.die ('unrecognized mode "%s"', mode)
def tui_checkfit(args): import omega as om if len(args) != 1: util.die('usage: <datfile>') nc = NoiseCal() nc.load(args[0]) vals = nc.bpdata[:, 1] modelvals = nc.bpdata[:, 3] * nc.svals resids = vals - modelvals runcerts = np.sqrt(1. / nc.bpdata[:, 2] + (nc.suncerts * nc.bpdata[:, 3])**2) normresids = resids / runcerts n = normresids.size mn = normresids.mean() s = normresids.std() md = np.median(normresids) smadm = 1.4826 * np.median(np.abs(normresids - md)) # see comment below print ' Number of samples:', n print ' Normalized mean residal:', mn print ' Median:', md print 'Normalized std. dev. (should be 1):', s print ' SMADM:', smadm # Check for problematic antpols and basepols saps = nc.saps sqbps = nc.sqbps nap = len(saps) dumbnbp = nap**2 apcounts = np.zeros(nap, dtype=np.int) apsumsqresids = np.zeros(nap) bpcounts = np.zeros(dumbnbp, dtype=np.int) bpsumsqresids = np.zeros(dumbnbp) for i in xrange(n): idx1, idx2 = sqbps[i] apcounts[idx1] += 1 apsumsqresids[idx1] += normresids[i]**2 apcounts[idx2] += 1 apsumsqresids[idx2] += normresids[i]**2 bpidx = idx1 * nap + idx2 bpcounts[bpidx] += 1 bpsumsqresids[bpidx] += normresids[i]**2 aprmsresids = np.sqrt(apsumsqresids / apcounts) sapresids = np.argsort(aprmsresids) print print 'Extreme residual RMS by antpol:' for i in xrange(5): idx = sapresids[i] print ' %10s %8.2f' % (util.fmtAP(saps[idx]), aprmsresids[idx]) print ' ....' for i in xrange(5): idx = sapresids[i - 5] print ' %10s %8.2f' % (util.fmtAP(saps[idx]), aprmsresids[idx]) wbpgood = np.where(bpcounts)[0] wbpbad = np.where(bpcounts == 0)[0] bpcounts[wbpbad] = 1 bprmsresids = bpsumsqresids / bpcounts sbpresids = np.argsort(bprmsresids[wbpgood]) print print 'Extreme residual RMS by basepol:' for i in xrange(3): idx = wbpgood[sbpresids[i]] ap2 = saps[idx % nap] ap1 = saps[idx // nap] print ' %10s %8.2f' % (util.fmtBP((ap1, ap2)), bprmsresids[idx]) print ' ....' for i in xrange(7): idx = wbpgood[sbpresids[i - 7]] ap2 = saps[idx % nap] ap1 = saps[idx // nap] print ' %10s %8.2f' % (util.fmtBP((ap1, ap2)), bprmsresids[idx]) # Plot the distribution of residuals bins = 50 rng = -5, 5 p = om.quickHist(normresids, keyText='Residuals', bins=bins, range=rng) x = np.linspace(-5, 5, 200) area = 10. / bins * n y = area / np.sqrt(2 * np.pi) * np.exp(-0.5 * x**2) p.addXY(x, y, 'Ideal') p.rebound(False, False) p.show() return 0
def _compute_solve(self): nap = len(self.saps) sqbps = self.sqbps bpdata = self.bpdata nsamp = bpdata.shape[0] # Get approximate solution by using a linear least squares # solver on the log of our equation. coeffs = np.zeros((nap, nsamp)) values = np.empty(nsamp) invsigmas = np.empty(nsamp) for i in xrange(nsamp): idx1, idx2 = sqbps[i] var, wtvar, crara = bpdata[i, 1:4] coeffs[idx1, i] = coeffs[idx2, i] = 1 values[i] = np.log(var / crara) # Here we ignore the noise in crara. invsigmas[i] = crara * np.sqrt(wtvar) soln = util.linLeastSquares(coeffs, values) # Now refine and get uncertainties with a nonlinear solver. soln = np.exp(soln) values = np.exp(values) modelvals = np.empty(nsamp) def nonlin(params): for i in xrange(nsamp): idx1, idx2 = sqbps[i] modelvals[i] = params[idx1] * params[idx2] return (values - modelvals) * invsigmas from scipy.optimize import leastsq soln, cov, misc, mesg, flag = leastsq(nonlin, soln, full_output=True) if flag < 1 or flag > 4 or cov is None: if cov is None: expln = 'encountered singular matrix' else: expln = 'return flag was %d' % flag if 'CALNOISE_DEBUG' in os.environ: print >> sys.stderr, 'Nonlinear fit failed! Saving anyway!' print >> sys.stderr, 'expln:', expln print >> sys.stderr, 'mesg:', mesg cov = None else: util.die('nonlinear fit failed: %s; mesg: %s', expln, mesg) rchisq = (((values - modelvals) * invsigmas)**2).sum() / (nsamp - nap) print 'reduced chi squared: %.3f for %d DOF' % (rchisq, nsamp - nap) if cov is None: cov = suncerts = np.empty(0) else: cov *= rchisq # copying scipy.optimize.curve_fit suncerts = np.empty(nsamp) for i in xrange(nsamp): idx1, idx2 = sqbps[i] suncerts[i] = np.sqrt((soln[idx1]**2 * cov[idx2, idx2] + soln[idx2]**2 * cov[idx1, idx1])) self.solution = soln self.covar = cov self.svals = modelvals self.suncerts = suncerts
def dualSelfCal (vis, out, usemself=False, ttol=DEFAULT_TTOL, outexists=False, serial=False, select='', postinterval=None, banner='PYTHON dualSelfCal', **kwargs): vis = ensureiterable (vis) if len (vis) == 0: util.die ('must specify at least one visibility file') for ivis in vis: if not ivis.exists: util.die ('visibility input file %s does not exist', ivis) if usemself: task = TaskMSelfCal () else: task = TaskSelfCal () if 'verbose' in kwargs and kwargs['verbose']: util.die ('option "verbose" only supported by mselfcal') if ttol <= 0: util.die ('parameter "ttol" must be positive') if out is None: if len (vis) != 1: util.die ('must specify "out" if more than ' 'one visibility input specified') dest = vis[0].open ('rw') else: if outexists: dest = out.open ('rw') else: dest = out.open ('c') src1 = vis[0].open ('rw') src1.copyItem (dest, 'history') src1.close () if len (select) == 0: selformat = 'pol(%s)' else: selformat = select + ',pol(%s)' task.vis = vis task.set (**kwargs) # Go worksets = [] procs = [] for pol in 'xx', 'yy': workset = vis[0].makeVariant ('sc' + pol, CalData) workset.delete () worksets.append (workset) task.select = selformat % pol task.out = workset if serial: task.run () else: procs.append (task.launch ()) for proc in procs: proc.checkwait () src1 = worksets[0].open ('rw') src2 = worksets[1].open ('rw') merge (str (worksets[0]), src1, str (worksets[1]), src2, dest, banner, ttol) for workset in worksets: workset.delete () if postinterval is not None: dest.setScalarItem ('interval', np.float32, postinterval / (60. * 24))
util.die('usage: <dat1> [... datn]') nc = NoiseCal() for path in args: nc.load(path, partial=True) nc.export(sys.stdout) return 0 def tui_sefds(args): if len(args) != 1: util.die('usage: <datfile>') nc = NoiseCal() nc.load(args[0]) nc.printsefdinfo() return 0 if __name__ == '__main__': if len(sys.argv) == 1: util.die( 'add a subcommand: one of "checkabs checkap checkcal checkfit ' 'compute export sefds"') subcommand = sys.argv[1] if 'tui_' + subcommand not in globals(): util.die('unknown subcommand "%s"', subcommand) sys.exit(globals()['tui_' + subcommand](sys.argv[2:]))
if len (args) < 1: util.die ('usage: <dat1> [... datn]') nc = NoiseCal () for path in args: nc.load (path, partial=True) nc.export (sys.stdout) return 0 def tui_sefds (args): if len (args) != 1: util.die ('usage: <datfile>') nc = NoiseCal () nc.load (args[0]) nc.printsefdinfo () return 0 if __name__ == '__main__': if len (sys.argv) == 1: util.die ('add a subcommand: one of "checkabs checkap checkcal checkfit ' 'compute export sefds"') subcommand = sys.argv[1] if 'tui_' + subcommand not in globals (): util.die ('unknown subcommand "%s"', subcommand) sys.exit (globals ()['tui_' + subcommand] (sys.argv[2:]))
def tui_checkfit (args): import omega as om if len (args) != 1: util.die ('usage: <datfile>') nc = NoiseCal () nc.load (args[0]) vals = nc.bpdata[:,1] modelvals = nc.bpdata[:,3] * nc.svals resids = vals - modelvals runcerts = np.sqrt (1./nc.bpdata[:,2] + (nc.suncerts * nc.bpdata[:,3])**2) normresids = resids / runcerts n = normresids.size mn = normresids.mean () s = normresids.std () md = np.median (normresids) smadm = 1.4826 * np.median (np.abs (normresids - md)) # see comment below print ' Number of samples:', n print ' Normalized mean residal:', mn print ' Median:', md print 'Normalized std. dev. (should be 1):', s print ' SMADM:', smadm # Check for problematic antpols and basepols saps = nc.saps sqbps = nc.sqbps nap = len (saps) dumbnbp = nap**2 apcounts = np.zeros (nap, dtype=np.int) apsumsqresids = np.zeros (nap) bpcounts = np.zeros (dumbnbp, dtype=np.int) bpsumsqresids = np.zeros (dumbnbp) for i in xrange (n): idx1, idx2 = sqbps[i] apcounts[idx1] += 1 apsumsqresids[idx1] += normresids[i]**2 apcounts[idx2] += 1 apsumsqresids[idx2] += normresids[i]**2 bpidx = idx1 * nap + idx2 bpcounts[bpidx] += 1 bpsumsqresids[bpidx] += normresids[i]**2 aprmsresids = np.sqrt (apsumsqresids / apcounts) sapresids = np.argsort (aprmsresids) print print 'Extreme residual RMS by antpol:' for i in xrange (5): idx = sapresids[i] print ' %10s %8.2f' % (util.fmtAP (saps[idx]), aprmsresids[idx]) print ' ....' for i in xrange (5): idx = sapresids[i - 5] print ' %10s %8.2f' % (util.fmtAP (saps[idx]), aprmsresids[idx]) wbpgood = np.where (bpcounts)[0] wbpbad = np.where (bpcounts == 0)[0] bpcounts[wbpbad] = 1 bprmsresids = bpsumsqresids / bpcounts sbpresids = np.argsort (bprmsresids[wbpgood]) print print 'Extreme residual RMS by basepol:' for i in xrange (3): idx = wbpgood[sbpresids[i]] ap2 = saps[idx % nap] ap1 = saps[idx // nap] print ' %10s %8.2f' % (util.fmtBP ((ap1, ap2)), bprmsresids[idx]) print ' ....' for i in xrange (7): idx = wbpgood[sbpresids[i - 7]] ap2 = saps[idx % nap] ap1 = saps[idx // nap] print ' %10s %8.2f' % (util.fmtBP ((ap1, ap2)), bprmsresids[idx]) # Plot the distribution of residuals bins = 50 rng = -5, 5 p = om.quickHist (normresids, keyText='Residuals', bins=bins, range=rng) x = np.linspace (-5, 5, 200) area = 10. / bins * n y = area / np.sqrt (2 * np.pi) * np.exp (-0.5 * x**2) p.addXY (x, y, 'Ideal') p.rebound (False, False) p.show () return 0
def _compute_solve (self): nap = len (self.saps) sqbps = self.sqbps bpdata = self.bpdata nsamp = bpdata.shape[0] # Get approximate solution by using a linear least squares # solver on the log of our equation. coeffs = np.zeros ((nap, nsamp)) values = np.empty (nsamp) invsigmas = np.empty (nsamp) for i in xrange (nsamp): idx1, idx2 = sqbps[i] var, wtvar, crara = bpdata[i,1:4] coeffs[idx1,i] = coeffs[idx2,i] = 1 values[i] = np.log (var / crara) # Here we ignore the noise in crara. invsigmas[i] = crara * np.sqrt (wtvar) soln = util.linLeastSquares (coeffs, values) # Now refine and get uncertainties with a nonlinear solver. soln = np.exp (soln) values = np.exp (values) modelvals = np.empty (nsamp) def nonlin (params): for i in xrange (nsamp): idx1, idx2 = sqbps[i] modelvals[i] = params[idx1] * params[idx2] return (values - modelvals) * invsigmas from scipy.optimize import leastsq soln, cov, misc, mesg, flag = leastsq (nonlin, soln, full_output=True) if flag < 1 or flag > 4 or cov is None: if cov is None: expln = 'encountered singular matrix' else: expln = 'return flag was %d' % flag if 'CALNOISE_DEBUG' in os.environ: print >>sys.stderr, 'Nonlinear fit failed! Saving anyway!' print >>sys.stderr, 'expln:', expln print >>sys.stderr, 'mesg:', mesg cov = None else: util.die ('nonlinear fit failed: %s; mesg: %s', expln, mesg) rchisq = (((values - modelvals) * invsigmas)**2).sum () / (nsamp - nap) print 'reduced chi squared: %.3f for %d DOF' % (rchisq, nsamp - nap) if cov is None: cov = suncerts = np.empty (0) else: cov *= rchisq # copying scipy.optimize.curve_fit suncerts = np.empty (nsamp) for i in xrange (nsamp): idx1, idx2 = sqbps[i] suncerts[i] = np.sqrt ((soln[idx1]**2 * cov[idx2,idx2] + soln[idx2]**2 * cov[idx1,idx1])) self.solution = soln self.covar = cov self.svals = modelvals self.suncerts = suncerts