def _compute_getraras(self, toread, uvdatoptions): seenaps = set() rarawork = {} previnp = None gen = uvdat.setupAndRead(toread, 'a3', False, nopass=True, nocal=True, nopol=True, **uvdatoptions) for inp, preamble, data, flags in gen: if previnp is None or inp is not previnp: instr = inp.getScalarItem('arfinstr', 'UNDEF') if instr != 'UNDEF': if self.instr is None: self.instr = instr elif instr != self.instr: util.die( 'input instrument changed from "%s" to ' '"%s" in "%s"', self.instr, instr, inp.path()) previnp = inp # the 'a' uvdat options limits us to autocorrelations, # but includes 1X-1Y-type autocorrelations bp = util.mir2bp(inp, preamble) if not util.bpIsInten(bp): continue ap = bp[0] t = preamble[3] w = np.where(flags)[0] if not w.size: continue data = data[w] rara = np.sqrt((data.real**2).mean()) seenaps.add(ap) ag = rarawork.get(ap) if ag is None: ag = rarawork[ap] = ArrayGrower(3) # We record w.size as a weight for the RARA measurement, # but it's essentially noise-free. ag.add(t, rara, w.size) self.saps = sorted(seenaps) raras = self.raras = {} apidxs = dict((t[1], t[0]) for t in enumerate(self.saps)) for ap in rarawork.keys(): raradata = rarawork[ap].finish().T apidx = apidxs[ap] sidx = np.argsort(raradata[0]) raras[apidx] = raradata[:, sidx]
def record (self, state): inp = state.inp track = self.track systemps = self.systemps prefactor = self.prefactor bps = self.bps bpfactors = self.bpfactors if track is None: track = inp.makeVarTracker ().track ('systemp', 'sdf', 'inttime') self.track = track if track.updated (): nants = inp.getVarInt ('nants') systemps = self.systemps = inp.getVarFloat ('systemp', nants) nspect = inp.getVarInt ('nspect') sdf = inp.getVarDouble ('sdf', nspect) if nspect > 1: sdf = sdf[0] inttime = inp.getVarFloat ('inttime') prefactor = self.prefactor = 2 * inttime * sdf * 1e9 if bpfactors is None or bpfactors.size != state.nbp: bpfactors = self.bpfactors = np.empty (state.nbp) bps = self.bps = np.empty ((state.nbp, 2), dtype=np.int) ap1, ap2 = mir2bp (inp, state.preamble) if ap1 == ap2: # Autocorrelation! Get the RARA (raw autocorrelation # RMS amplitude) w = np.where (state.flags)[0] if w.size == 0: self.raras.pop (ap1, 0) else: self.raras[ap1] = np.sqrt ((state.data.real[w]**2).mean ()) # Get the factor that will go in front of the RARAs for # jyperk computations. Do this for autocorrs too because # there's no reason not to. if state.instr not in self.byinstr: raise Exception ('need information for instrument ' + state.instr) byap, default = self.byinstr[state.instr] cal1 = byap.get (ap1, default) cal2 = byap.get (ap2, default) ant1, ant2 = apAnt (ap1), apAnt (ap2) tsys1, tsys2 = systemps[ant1 - 1], systemps[ant2 - 1] bpindex = state.bpindex bps[bpindex,0] = ap1 bps[bpindex,1] = ap2 bpfactors[bpindex] = prefactor * cal1 * cal2 / (tsys1 * tsys2)
def record(self, state): inp = state.inp track = self.track systemps = self.systemps prefactor = self.prefactor bps = self.bps bpfactors = self.bpfactors if track is None: track = inp.makeVarTracker().track('systemp', 'sdf', 'inttime') self.track = track if track.updated(): nants = inp.getVarInt('nants') systemps = self.systemps = inp.getVarFloat('systemp', nants) nspect = inp.getVarInt('nspect') sdf = inp.getVarDouble('sdf', nspect) if nspect > 1: sdf = sdf[0] inttime = inp.getVarFloat('inttime') prefactor = self.prefactor = 2 * inttime * sdf * 1e9 if bpfactors is None or bpfactors.size != state.nbp: bpfactors = self.bpfactors = np.empty(state.nbp) bps = self.bps = np.empty((state.nbp, 2), dtype=np.int) ap1, ap2 = mir2bp(inp, state.preamble) if ap1 == ap2: # Autocorrelation! Get the RARA (raw autocorrelation # RMS amplitude) w = np.where(state.flags)[0] if w.size == 0: self.raras.pop(ap1, 0) else: self.raras[ap1] = np.sqrt((state.data.real[w]**2).mean()) # Get the factor that will go in front of the RARAs for # jyperk computations. Do this for autocorrs too because # there's no reason not to. if state.instr not in self.byinstr: raise Exception('need information for instrument ' + state.instr) byap, default = self.byinstr[state.instr] cal1 = byap.get(ap1, default) cal2 = byap.get(ap2, default) ant1, ant2 = apAnt(ap1), apAnt(ap2) tsys1, tsys2 = systemps[ant1 - 1], systemps[ant2 - 1] bpindex = state.bpindex bps[bpindex, 0] = ap1 bps[bpindex, 1] = ap2 bpfactors[bpindex] = prefactor * cal1 * cal2 / (tsys1 * tsys2)
def _compute_getraras (self, toread, uvdatoptions): seenaps = set () rarawork = {} previnp = None gen = uvdat.setupAndRead (toread, 'a3', False, nopass=True, nocal=True, nopol=True, **uvdatoptions) for inp, preamble, data, flags in gen: if previnp is None or inp is not previnp: instr = inp.getScalarItem ('arfinstr', 'UNDEF') if instr != 'UNDEF': if self.instr is None: self.instr = instr elif instr != self.instr: util.die ('input instrument changed from "%s" to ' '"%s" in "%s"', self.instr, instr, inp.path ()) previnp = inp # the 'a' uvdat options limits us to autocorrelations, # but includes 1X-1Y-type autocorrelations bp = util.mir2bp (inp, preamble) if not util.bpIsInten (bp): continue ap = bp[0] t = preamble[3] w = np.where (flags)[0] if not w.size: continue data = data[w] rara = np.sqrt ((data.real**2).mean ()) seenaps.add (ap) ag = rarawork.get (ap) if ag is None: ag = rarawork[ap] = ArrayGrower (3) # We record w.size as a weight for the RARA measurement, # but it's essentially noise-free. ag.add (t, rara, w.size) self.saps = sorted (seenaps) raras = self.raras = {} apidxs = dict ((t[1], t[0]) for t in enumerate (self.saps)) for ap in rarawork.keys (): raradata = rarawork[ap].finish ().T apidx = apidxs[ap] sidx = np.argsort (raradata[0]) raras[apidx] = raradata[:,sidx]
def _compute_getbpvs (self, toread, uvdatoptions): raras = self.raras apidxs = dict ((t[1], t[0]) for t in enumerate (self.saps)) sqbps = ArrayGrower (2, dtype=np.int) bpdata = ArrayGrower (5) nnorara = 0 seenidxs = set () gen = uvdat.setupAndRead (toread, 'x3', False, nopass=False, nocal=False, nopol=False, **uvdatoptions) for inp, preamble, data, flags in gen: bp = util.mir2bp (inp, preamble) t = preamble[3] w = np.where (flags)[0] if not w.size: continue idx1, idx2 = apidxs.get (bp[0]), apidxs.get (bp[1]) if idx1 is None or idx2 is None: nnorara += 1 continue rdata1, rdata2 = raras[idx1], raras[idx2] tidx1 = rdata1[0].searchsorted (t) tidx2 = rdata2[0].searchsorted (t) if (tidx1 < 0 or tidx1 >= rdata1.shape[1] or tidx2 < 0 or tidx2 >= rdata2.shape[1]): nnorara += 1 continue tr1, rara1, rwt1 = rdata1[:,tidx1] tr2, rara2, rwt2 = rdata2[:,tidx2] dt1 = np.abs (tr1 - t) dt2 = np.abs (tr2 - t) if max (dt1, dt2) > TTOL: nnorara += 1 continue # We propagate the weight for crara but once again, # it's basically noiseless. crara = rara1 * rara2 cwt = 1. / (rara2**2 / rwt1 + rara1**2 / rwt2) data = data[w] rvar = data.real.var (ddof=1) ivar = data.imag.var (ddof=1) var = 0.5 * (rvar + ivar) # The weight of the computed variance (i.e., the # inverse square of the maximum-likelihood variance # in the variance measurement) is 0.5 * (nsamp - ndof) / var**2. # We have 2*w.size samples and 2 degrees of freedom, so: wtvar = (w.size - 1) / var**2 seenidxs.add (idx1) seenidxs.add (idx2) sqbps.add (idx1, idx2) bpdata.add (t, var, wtvar, crara, cwt) sqbps = self.sqbps = sqbps.finish () self.bpdata = bpdata.finish () nsamp = sqbps.shape[0] if nnorara > nsamp * 0.02: print >>sys.stderr, ('Warning: no autocorr data for %d/%d ' '(%.0f%%) of samples' % (nnorara, nsamp, 100. * nnorara / nsamp)) if len (seenidxs) == len (apidxs): return # There exist antpols that we got autocorrelations for but # have no contributing baselines. This can happen if there are # no gains for the given antpol. To avoid running into a # singular matrix in the solve step, we have to eliminate # them. To avoid a bunch of baggage in the analysis code, we # rewrite our data structures, which actually isn't so bad. mapping = {} newsaps = [] newraras = {} for idx in xrange (len (apidxs)): if idx in seenidxs: mapping[idx] = len (mapping) newsaps.append (self.saps[idx]) newraras[mapping[idx]] = raras[idx] self.saps = newsaps self.raras = newraras for i in xrange (nsamp): sqbps[i,0] = mapping[sqbps[i,0]] sqbps[i,1] = mapping[sqbps[i,1]]
def rewrite (self, ncorr, invis, outvis, **uvdargs): psources = [s for s in self.sources if s.major is None] gsources = [s for s in self.sources if s.major is not None] outhnd = outvis.open ('c') outhnd.setPreambleType ('uvw', 'time', 'baseline') first = True curhnd = None npol = prevnpol = 0 neednpol = True npolvaried = False nproc = 0 nrec = 0 prevtime = None for inhnd, preamble, data, flags in invis.readLowlevel ('3', False, **uvdargs): if first: inhnd.copyItem (outhnd, 'history') outhnd.openHistory () outhnd.writeHistory (self.banner) first = False if inhnd is not curhnd: inhnd.initVarsAsInput ('channel') outhnd.initVarsAsOutput (inhnd, 'channel') #outhnd.setCorrelationType ('r') curhnd = inhnd freqsupdated = True # assuming fixed freqs per dataset if freqsupdated: freqs = inhnd.getSkyFrequencies () alphas = pbalphas (freqs) dwork = np.empty (freqs.shape, dtype=np.double) dwork2 = np.empty (freqs.shape, dtype=np.double) cwork = np.empty (freqs.shape, dtype=np.complex) freqsupdated = False if prevtime is None or preamble[3] != prevtime: # Need to determine new squint distances as PB rotates on the sky. lst = inhnd.getVarDouble ('lst') lat = inhnd.getVarDouble ('latitud') az, el = util.equToHorizon (inhnd.getVarDouble ('ra'), inhnd.getVarDouble ('dec'), lst, lat) for ap, (sqmag, sqpa) in self.squints.iteritems (): oel, oaz = sphofs (el, az, sqmag, sqpa) ora, odec = util.horizonToEqu (oaz, oel, lst, lat) for s in self.sources: # We take half of the squared distance because # that's what's going to be relevant in the PB # attenuation (half in exponent -> sqrt -> # geometric mean) s.hpbsqdists[ap] = 0.5 * sphdist (s.dec, s.ra, odec, ora)**2 prevtime = preamble[3] if nrec % recmod == 0: print >>sys.stderr, '%5.1f%% (%d of %d)' % (100.* nproc / ncorr, nproc, ncorr) if npol == 0: npol = inhnd.getNPol () neednpol = True if neednpol: if npol != prevnpol: outhnd.writeVarInt ('npol', npol) npolvaried = npolvaried or (prevnpol != 0) prevnpol = npol neednpol = False ap1, ap2 = util.mir2bp (inhnd, preamble) data.fill (0) flags.fill (1) for s in psources: # geometric mean of effective fluxes, attenuated by # (maybe) freq-dependent PB falloff. multiply (twopii * dot (s.lmn, preamble[:3]), freqs, cwork) if pbfreqdep: multiply (s.hpbsqdists[ap1] + s.hpbsqdists[ap2], alphas, dwork) add (dwork, cwork, cwork) else: add ((s.hpbsqdists[ap1] + s.hpbsqdists[ap2]) * alphas[0], cwork, cwork) exp (cwork, cwork) multiply (s.totflux, cwork, cwork) data += cwork for s in gsources: # As above, with additional gaussian amplitude falloff term. cp = np.cos (s.pa) sp = np.sin (s.pa) g = gfac * ((s.minor * (preamble[0] * cp - preamble[1] * sp))**2 + (s.major * (preamble[0] * sp + preamble[1] * cp))**2) square (freqs, dwork2) multiply (g, dwork2, dwork2) multiply (twopii * dot (s.lmn, preamble[:3]), freqs, cwork) if pbfreqdep: multiply (s.hpbsqdists[ap1] + s.hpbsqdists[ap2], alphas, dwork) add (dwork, cwork, cwork) else: add ((s.hpbsqdists[ap1] + s.hpbsqdists[ap2]) * alphas[0], cwork, cwork) add (dwork2, cwork, cwork) exp (cwork, cwork) multiply (s.totflux, cwork, cwork) data += cwork inhnd.copyLineVars (outhnd) outhnd.writeVarInt ('pol', inhnd.getPol ()) outhnd.write (preamble, data, flags) npol -= 1 nrec += 1 nproc += data.size if not npolvaried: outhnd.setScalarItem ('npol', np.int32, prevnpol) outhnd.closeHistory () outhnd.close ()
def _compute_getbpvs(self, toread, uvdatoptions): raras = self.raras apidxs = dict((t[1], t[0]) for t in enumerate(self.saps)) sqbps = ArrayGrower(2, dtype=np.int) bpdata = ArrayGrower(5) nnorara = 0 seenidxs = set() gen = uvdat.setupAndRead(toread, 'x3', False, nopass=False, nocal=False, nopol=False, **uvdatoptions) for inp, preamble, data, flags in gen: bp = util.mir2bp(inp, preamble) t = preamble[3] w = np.where(flags)[0] if not w.size: continue idx1, idx2 = apidxs.get(bp[0]), apidxs.get(bp[1]) if idx1 is None or idx2 is None: nnorara += 1 continue rdata1, rdata2 = raras[idx1], raras[idx2] tidx1 = rdata1[0].searchsorted(t) tidx2 = rdata2[0].searchsorted(t) if (tidx1 < 0 or tidx1 >= rdata1.shape[1] or tidx2 < 0 or tidx2 >= rdata2.shape[1]): nnorara += 1 continue tr1, rara1, rwt1 = rdata1[:, tidx1] tr2, rara2, rwt2 = rdata2[:, tidx2] dt1 = np.abs(tr1 - t) dt2 = np.abs(tr2 - t) if max(dt1, dt2) > TTOL: nnorara += 1 continue # We propagate the weight for crara but once again, # it's basically noiseless. crara = rara1 * rara2 cwt = 1. / (rara2**2 / rwt1 + rara1**2 / rwt2) data = data[w] rvar = data.real.var(ddof=1) ivar = data.imag.var(ddof=1) var = 0.5 * (rvar + ivar) # The weight of the computed variance (i.e., the # inverse square of the maximum-likelihood variance # in the variance measurement) is 0.5 * (nsamp - ndof) / var**2. # We have 2*w.size samples and 2 degrees of freedom, so: wtvar = (w.size - 1) / var**2 seenidxs.add(idx1) seenidxs.add(idx2) sqbps.add(idx1, idx2) bpdata.add(t, var, wtvar, crara, cwt) sqbps = self.sqbps = sqbps.finish() self.bpdata = bpdata.finish() nsamp = sqbps.shape[0] if nnorara > nsamp * 0.02: print >> sys.stderr, ('Warning: no autocorr data for %d/%d ' '(%.0f%%) of samples' % (nnorara, nsamp, 100. * nnorara / nsamp)) if len(seenidxs) == len(apidxs): return # There exist antpols that we got autocorrelations for but # have no contributing baselines. This can happen if there are # no gains for the given antpol. To avoid running into a # singular matrix in the solve step, we have to eliminate # them. To avoid a bunch of baggage in the analysis code, we # rewrite our data structures, which actually isn't so bad. mapping = {} newsaps = [] newraras = {} for idx in xrange(len(apidxs)): if idx in seenidxs: mapping[idx] = len(mapping) newsaps.append(self.saps[idx]) newraras[mapping[idx]] = raras[idx] self.saps = newsaps self.raras = newraras for i in xrange(nsamp): sqbps[i, 0] = mapping[sqbps[i, 0]] sqbps[i, 1] = mapping[sqbps[i, 1]]