def channelAverageWithSetup (toread, out, naver, slop=DEFAULT_SLOP, banner=DEFAULT_BANNER, **uvdargs): """Read UV data and channel average into an output dataset. toread: dataset handle or iterable thereof; input dataset(s) out: dataset handle; the output dataset to be created naver: int; the number of channels to average together (see task docs) slop: float; tolerance for partially flagged bins (see task docs) banner: string; a message to write into the output's history **uvdargs: keyword arguments passed through to the uvdat subsystem initialization (mirtask.uvdat.setupAndRead) returns: None Contrast with channelAverage, which performs no extra initialization of the uvdat subsystem. """ if naver < 1: raise ValueError ('must average at least one channel (got naver=%d)' % naver) if slop < 0 or slop > 1: raise ValueError ('slop must be between 0 and 1 (got slop=%f)' % slop) try: gen = uvdat.setupAndRead (toread, UVDAT_OPTIONS, False, **uvdargs) args = ['vis=' + ','.join (str (x) for x in ensureiterable (toread))] args += ['%s=%s' % (k, uvdargs[k]) for k in sorted (uvdargs.iterkeys ())] _channelAverage (gen, out, naver, slop, banner, args) except _CreateFailedError, e: # Don't delete the existing dataset! raise e.subexc
def _compute_getraras(self, toread, uvdatoptions): seenaps = set() rarawork = {} previnp = None gen = uvdat.setupAndRead(toread, 'a3', False, nopass=True, nocal=True, nopol=True, **uvdatoptions) for inp, preamble, data, flags in gen: if previnp is None or inp is not previnp: instr = inp.getScalarItem('arfinstr', 'UNDEF') if instr != 'UNDEF': if self.instr is None: self.instr = instr elif instr != self.instr: util.die( 'input instrument changed from "%s" to ' '"%s" in "%s"', self.instr, instr, inp.path()) previnp = inp # the 'a' uvdat options limits us to autocorrelations, # but includes 1X-1Y-type autocorrelations bp = util.mir2bp(inp, preamble) if not util.bpIsInten(bp): continue ap = bp[0] t = preamble[3] w = np.where(flags)[0] if not w.size: continue data = data[w] rara = np.sqrt((data.real**2).mean()) seenaps.add(ap) ag = rarawork.get(ap) if ag is None: ag = rarawork[ap] = ArrayGrower(3) # We record w.size as a weight for the RARA measurement, # but it's essentially noise-free. ag.add(t, rara, w.size) self.saps = sorted(seenaps) raras = self.raras = {} apidxs = dict((t[1], t[0]) for t in enumerate(self.saps)) for ap in rarawork.keys(): raradata = rarawork[ap].finish().T apidx = apidxs[ap] sidx = np.argsort(raradata[0]) raras[apidx] = raradata[:, sidx]
def _compute_getraras (self, toread, uvdatoptions): seenaps = set () rarawork = {} previnp = None gen = uvdat.setupAndRead (toread, 'a3', False, nopass=True, nocal=True, nopol=True, **uvdatoptions) for inp, preamble, data, flags in gen: if previnp is None or inp is not previnp: instr = inp.getScalarItem ('arfinstr', 'UNDEF') if instr != 'UNDEF': if self.instr is None: self.instr = instr elif instr != self.instr: util.die ('input instrument changed from "%s" to ' '"%s" in "%s"', self.instr, instr, inp.path ()) previnp = inp # the 'a' uvdat options limits us to autocorrelations, # but includes 1X-1Y-type autocorrelations bp = util.mir2bp (inp, preamble) if not util.bpIsInten (bp): continue ap = bp[0] t = preamble[3] w = np.where (flags)[0] if not w.size: continue data = data[w] rara = np.sqrt ((data.real**2).mean ()) seenaps.add (ap) ag = rarawork.get (ap) if ag is None: ag = rarawork[ap] = ArrayGrower (3) # We record w.size as a weight for the RARA measurement, # but it's essentially noise-free. ag.add (t, rara, w.size) self.saps = sorted (seenaps) raras = self.raras = {} apidxs = dict ((t[1], t[0]) for t in enumerate (self.saps)) for ap in rarawork.keys (): raradata = rarawork[ap].finish ().T apidx = apidxs[ap] sidx = np.argsort (raradata[0]) raras[apidx] = raradata[:,sidx]
def tui_checkcal (args): import omega as om, scipy.stats as SS toread = [] uvdatoptions = {} for arg in args: if '=' in arg: key, value = arg.split ('=', 1) uvdatoptions[key] = value else: toread.append (VisData (arg)) if len (toread) < 1: util.die ('usage: <vis1> [... visn] [uvdat options]') samples = VectorGrower () gen = uvdat.setupAndRead (toread, 'x3', False, **uvdatoptions) for inp, pream, data, flags in gen: w = np.where (flags)[0] if w.size == 0: continue data = data[w] var = 0.5 * (data.real.var (ddof=1) + data.imag.var (ddof=1)) uvar = np.sqrt (1. / (w.size - 1)) * var # uncert in variance msmt thy = inp.getVariance () samples.add ((var - thy) / uvar) samples = samples.finish () n = samples.size m = samples.mean () s = samples.std () med = np.median (samples) smadm = 1.4826 * np.median (np.abs (samples - med)) # see comment below print ' Number of samples:', n print 'Mean normalized error (should be 0):', m print ' Median:', med print ' Normalized std. dev. (should be 1):', s print ' SMADM:', smadm print 'Probability that samples are normal:', SS.normaltest (samples)[1] bins = 50 rng = -5, 5 p = om.quickHist (samples, keyText='Samples', bins=bins, range=rng) x = np.linspace (-5, 5, 200) area = 10. / bins * n y = area / np.sqrt (2 * np.pi) * np.exp (-0.5 * x**2) p.addXY (x, y, 'Ideal') p.rebound (False, False) p.show () return 0
def tui_checkcal(args): import omega as om, scipy.stats as SS toread = [] uvdatoptions = {} for arg in args: if '=' in arg: key, value = arg.split('=', 1) uvdatoptions[key] = value else: toread.append(VisData(arg)) if len(toread) < 1: util.die('usage: <vis1> [... visn] [uvdat options]') samples = VectorGrower() gen = uvdat.setupAndRead(toread, 'x3', False, **uvdatoptions) for inp, pream, data, flags in gen: w = np.where(flags)[0] if w.size == 0: continue data = data[w] var = 0.5 * (data.real.var(ddof=1) + data.imag.var(ddof=1)) uvar = np.sqrt(1. / (w.size - 1)) * var # uncert in variance msmt thy = inp.getVariance() samples.add((var - thy) / uvar) samples = samples.finish() n = samples.size m = samples.mean() s = samples.std() med = np.median(samples) smadm = 1.4826 * np.median(np.abs(samples - med)) # see comment below print ' Number of samples:', n print 'Mean normalized error (should be 0):', m print ' Median:', med print ' Normalized std. dev. (should be 1):', s print ' SMADM:', smadm print 'Probability that samples are normal:', SS.normaltest(samples)[1] bins = 50 rng = -5, 5 p = om.quickHist(samples, keyText='Samples', bins=bins, range=rng) x = np.linspace(-5, 5, 200) area = 10. / bins * n y = area / np.sqrt(2 * np.pi) * np.exp(-0.5 * x**2) p.addXY(x, y, 'Ideal') p.rebound(False, False) p.show() return 0
def readLowlevel (self, uvdOptions, saveFlags, **kwargs): """Directly access the visibility data. :arg uvdOptions: options controlling the behavior of the UVDAT subsystem :type uvdOptions: :class:`str` :arg saveFlags: whether to save modified UV flags to the dataset as it is read :type saveFlags: :class:`bool` :arg kwargs: extra arguments to :func:`mirtask.uvdat.setupAndRead`. :rtype: generator of ``(handle, preamble, data, flags)`` :returns: generates a sequence of UV information tuples. See the documentation of :func:`mirtask.uvdat.setupAndRead` for more information. Calls :func:`mirtask.uvdat.setupAndRead` on this dataset with the specified arguments. """ from mirtask import uvdat return uvdat.setupAndRead (self, uvdOptions, saveFlags, **kwargs)
def _compute_getbpvs (self, toread, uvdatoptions): raras = self.raras apidxs = dict ((t[1], t[0]) for t in enumerate (self.saps)) sqbps = ArrayGrower (2, dtype=np.int) bpdata = ArrayGrower (5) nnorara = 0 seenidxs = set () gen = uvdat.setupAndRead (toread, 'x3', False, nopass=False, nocal=False, nopol=False, **uvdatoptions) for inp, preamble, data, flags in gen: bp = util.mir2bp (inp, preamble) t = preamble[3] w = np.where (flags)[0] if not w.size: continue idx1, idx2 = apidxs.get (bp[0]), apidxs.get (bp[1]) if idx1 is None or idx2 is None: nnorara += 1 continue rdata1, rdata2 = raras[idx1], raras[idx2] tidx1 = rdata1[0].searchsorted (t) tidx2 = rdata2[0].searchsorted (t) if (tidx1 < 0 or tidx1 >= rdata1.shape[1] or tidx2 < 0 or tidx2 >= rdata2.shape[1]): nnorara += 1 continue tr1, rara1, rwt1 = rdata1[:,tidx1] tr2, rara2, rwt2 = rdata2[:,tidx2] dt1 = np.abs (tr1 - t) dt2 = np.abs (tr2 - t) if max (dt1, dt2) > TTOL: nnorara += 1 continue # We propagate the weight for crara but once again, # it's basically noiseless. crara = rara1 * rara2 cwt = 1. / (rara2**2 / rwt1 + rara1**2 / rwt2) data = data[w] rvar = data.real.var (ddof=1) ivar = data.imag.var (ddof=1) var = 0.5 * (rvar + ivar) # The weight of the computed variance (i.e., the # inverse square of the maximum-likelihood variance # in the variance measurement) is 0.5 * (nsamp - ndof) / var**2. # We have 2*w.size samples and 2 degrees of freedom, so: wtvar = (w.size - 1) / var**2 seenidxs.add (idx1) seenidxs.add (idx2) sqbps.add (idx1, idx2) bpdata.add (t, var, wtvar, crara, cwt) sqbps = self.sqbps = sqbps.finish () self.bpdata = bpdata.finish () nsamp = sqbps.shape[0] if nnorara > nsamp * 0.02: print >>sys.stderr, ('Warning: no autocorr data for %d/%d ' '(%.0f%%) of samples' % (nnorara, nsamp, 100. * nnorara / nsamp)) if len (seenidxs) == len (apidxs): return # There exist antpols that we got autocorrelations for but # have no contributing baselines. This can happen if there are # no gains for the given antpol. To avoid running into a # singular matrix in the solve step, we have to eliminate # them. To avoid a bunch of baggage in the analysis code, we # rewrite our data structures, which actually isn't so bad. mapping = {} newsaps = [] newraras = {} for idx in xrange (len (apidxs)): if idx in seenidxs: mapping[idx] = len (mapping) newsaps.append (self.saps[idx]) newraras[mapping[idx]] = raras[idx] self.saps = newsaps self.raras = newraras for i in xrange (nsamp): sqbps[i,0] = mapping[sqbps[i,0]] sqbps[i,1] = mapping[sqbps[i,1]]
def _compute_getbpvs(self, toread, uvdatoptions): raras = self.raras apidxs = dict((t[1], t[0]) for t in enumerate(self.saps)) sqbps = ArrayGrower(2, dtype=np.int) bpdata = ArrayGrower(5) nnorara = 0 seenidxs = set() gen = uvdat.setupAndRead(toread, 'x3', False, nopass=False, nocal=False, nopol=False, **uvdatoptions) for inp, preamble, data, flags in gen: bp = util.mir2bp(inp, preamble) t = preamble[3] w = np.where(flags)[0] if not w.size: continue idx1, idx2 = apidxs.get(bp[0]), apidxs.get(bp[1]) if idx1 is None or idx2 is None: nnorara += 1 continue rdata1, rdata2 = raras[idx1], raras[idx2] tidx1 = rdata1[0].searchsorted(t) tidx2 = rdata2[0].searchsorted(t) if (tidx1 < 0 or tidx1 >= rdata1.shape[1] or tidx2 < 0 or tidx2 >= rdata2.shape[1]): nnorara += 1 continue tr1, rara1, rwt1 = rdata1[:, tidx1] tr2, rara2, rwt2 = rdata2[:, tidx2] dt1 = np.abs(tr1 - t) dt2 = np.abs(tr2 - t) if max(dt1, dt2) > TTOL: nnorara += 1 continue # We propagate the weight for crara but once again, # it's basically noiseless. crara = rara1 * rara2 cwt = 1. / (rara2**2 / rwt1 + rara1**2 / rwt2) data = data[w] rvar = data.real.var(ddof=1) ivar = data.imag.var(ddof=1) var = 0.5 * (rvar + ivar) # The weight of the computed variance (i.e., the # inverse square of the maximum-likelihood variance # in the variance measurement) is 0.5 * (nsamp - ndof) / var**2. # We have 2*w.size samples and 2 degrees of freedom, so: wtvar = (w.size - 1) / var**2 seenidxs.add(idx1) seenidxs.add(idx2) sqbps.add(idx1, idx2) bpdata.add(t, var, wtvar, crara, cwt) sqbps = self.sqbps = sqbps.finish() self.bpdata = bpdata.finish() nsamp = sqbps.shape[0] if nnorara > nsamp * 0.02: print >> sys.stderr, ('Warning: no autocorr data for %d/%d ' '(%.0f%%) of samples' % (nnorara, nsamp, 100. * nnorara / nsamp)) if len(seenidxs) == len(apidxs): return # There exist antpols that we got autocorrelations for but # have no contributing baselines. This can happen if there are # no gains for the given antpol. To avoid running into a # singular matrix in the solve step, we have to eliminate # them. To avoid a bunch of baggage in the analysis code, we # rewrite our data structures, which actually isn't so bad. mapping = {} newsaps = [] newraras = {} for idx in xrange(len(apidxs)): if idx in seenidxs: mapping[idx] = len(mapping) newsaps.append(self.saps[idx]) newraras[mapping[idx]] = raras[idx] self.saps = newsaps self.raras = newraras for i in xrange(nsamp): sqbps[i, 0] = mapping[sqbps[i, 0]] sqbps[i, 1] = mapping[sqbps[i, 1]]