def get_pix_ranges(shape, wcs, horbox, daz, nt=4, azdown=1, ndet=1.0): (t1,t2),(az1,az2),el = horbox[:,0], horbox[:,1], np.mean(horbox[:,2]) nphi = np.abs(utils.nint(360/wcs.wcs.cdelt[0])) # Find the pixel coordinates of first az sweep naz = utils.nint(np.abs(az2-az1)/daz)/azdown ahor = np.zeros([3,naz]) ahor[0] = utils.ctime2mjd(t1) ahor[1] = np.linspace(az1,az2,naz) ahor[2] = el acel = coordinates.transform("hor","cel",ahor[1:],time=ahor[0],site=site) y, x1 = upscale(fixx(utils.nint(enmap.sky2pix(shape, wcs, acel[::-1])),nphi),azdown) # Reduce to unique y values _, uinds, hits = np.unique(y, return_index=True, return_counts=True) y, x1 = y[uinds], x1[uinds] # Find the pixel coordinates of time drift thor = np.zeros([3,nt]) thor[0] = utils.ctime2mjd(np.linspace(t1,t2,nt)) thor[1] = az1 thor[2] = el tcel = coordinates.transform("hor","cel",thor[1:],time=thor[0],site=site) _, tx = utils.nint(fixx(enmap.sky2pix(shape, wcs, tcel[::-1]),nphi)) x2 = x1 + tx[-1]-tx[0] x1, x2 = np.minimum(x1,x2), np.maximum(x1,x2) pix_ranges = np.concatenate([y[:,None],x1[:,None],x2[:,None]],1) # Weight per pixel in pix ranges. If ndet=1 this corresponds to # telescope time per output pixel weights = (t2-t1)/(naz*azdown)/(x2-x1)*ndet * hits return pix_ranges, weights
def calc_driftangle(hor, t, site): hor = np.atleast_2d(hor).T t = np.atleast_1d(t) equ = coordinates.transform("hor", "equ", hor, time=utils.ctime2mjd(t), site=site) hor_drift = utils.rewind(coordinates.transform("equ","hor", equ, time=utils.ctime2mjd(t+1), site=site),hor,2*np.pi) vec_drift = hor_drift-hor # Compute angle between this vector and the az axis angle = np.arctan2(vec_drift[1],vec_drift[0]*np.cos(hor[1]))%np.pi return angle
def setup_extra_transforms(param): extra = [] if "p9" in param: # Planet 9 search coordinate system: p9=elemfile:tref. Includes # both parallax and motion compensation toks = param["p9"].split(":") elemfile = toks[0] tref = float(toks[1]) if len(toks)>1 else 1380000000.0 tref = utils.ctime2mjd(tref) obj = ephemeris.read_object(elemfile) p9 = planet9.MotionCompensator(obj) def trf(pos, time): # We ignore the polarization rotation for now opos = pos.copy() opos[:2] = p9.compensate(pos[:2], time, tref) return opos extra.append(trf) if "parallax" in param: # Simple parallax compensation. parallax=dist, with dist in AU dist = float(param["parallax"]) def trf(pos, time): opos = pos.copy() opos[:2] = parallax.earth2sun(pos[:2], time, dist) return opos extra.append(trf) return extra
def __call__(self, sys1, sys2, pos): mjd = utils.ctime2mjd(self.data["t"]) bore = np.array([self.data["az"], self.data["el"]]) * utils.degree hor = coordinates.transform( sys1, sys2, np.asarray(pos) * utils.degree, mjd, bore=bore) / utils.degree return hor
def avoidance_cut_old(bore, det_offs, site, name_or_pos, margin): """Cut samples that get too close to the specified object (e.g. "Sun" or "Moon") or celestial position ([ra,dec] in racians). Margin specifies how much to avoid the object by.""" cmargin = np.cos(margin) mjd = utils.ctime2mjd(bore[0]) obj_pos = coordinates.interpol_pos("cel", "hor", name_or_pos, mjd, site) obj_rect = utils.ang2rect(obj_pos, zenith=False) # Only cut if above horizon above_horizon = obj_pos[1] > 0 if np.all(~above_horizon): return sampcut.empty(det_offs.shape[0], bore.shape[1]) cuts = [] for di, off in enumerate(det_offs): det_pos = bore[1:] + off[:, None] #det_rect1 = utils.ang2rect(det_pos, zenith=False) # slow # Not sure defining an ang2rect in array_ops is worth it.. It's ugly, # (angle convention hard coded) and only gives factor 2 on laptop. det_rect = array_ops.ang2rect(np.ascontiguousarray(det_pos.T)).T cdist = np.sum(obj_rect * det_rect, 0) # Cut samples above horizon that are too close bad = (cdist > cmargin) & above_horizon cuts.append(sampcut.from_mask(bad)) res = sampcut.stack(cuts) return res
def __call__(self, fname): mjd = utils.ctime2mjd(self.data["t"]) mjd[~np.isfinite(mjd)] = 0 obj = ephemeris.read_object(fname) pos = ephemeris.ephem_pos(obj, mjd)[:2] pos /= utils.degree return pos
def calc_driftangle(hor, t, site): hor = np.atleast_2d(hor).T t = np.atleast_1d(t) equ = coordinates.transform("hor", "equ", hor, time=utils.ctime2mjd(t), site=site) hor_drift = utils.rewind( coordinates.transform("equ", "hor", equ, time=utils.ctime2mjd(t + 1), site=site), hor, 2 * np.pi) vec_drift = hor_drift - hor # Compute angle between this vector and the az axis angle = np.arctan2(vec_drift[1], vec_drift[0] * np.cos(hor[1])) % np.pi return angle
def summarize(self, chain): """Given a chain of lpars, compute a summary in the same format as returned by SrcFitterML.""" dposs = np.array([c.dpos for c in chain]) poss_cel = np.array([c.poss for c in chain]) poss_hor = poss_cel * 0 for i in range(len(self.sdata)): poss_hor[:, i] = coordinates.transform("cel", "hor", poss_cel[:, i].T, time=utils.ctime2mjd( self.sdata[i].ctime), site=self.sdata[i].site).T ampss = np.array([c.amps for c in chain]) vampss = np.array([c.vamps for c in chain]) chisq0 = chain[0].chisq0 chisq = np.mean([c.chisq for c in chain]) # Compute means dpos = np.mean(dposs, 0) pos_cel = np.mean(poss_cel, 0) pos_hor = np.mean(poss_hor, 0) amps = np.mean(ampss, 0) # And uncertainties dpos_cov = np.cov(dposs.T) ddpos = np.diag(dpos_cov)**0.5 pcorr = dpos_cov[0, 1] / ddpos[0] / ddpos[1] # mean([da0_i + da_ij]**2). Uncorrelated, so this is # mean(da0**2) + mean(da**2) arel = ampss + np.random.standard_normal(ampss.shape) * vampss**0.5 amps = np.mean(arel, 0) vamps = np.var(arel, 0) #vamps = np.var(ampss,0)+np.mean(vampss,0) damps = vamps**0.5 models = self.lik.eval(dpos).models nsigma = (chisq0 - chisq)**0.5 # We want how much to offset detector by, not how much to offset # source by res = bunch.Bunch(dpos=dpos, poss_cel=pos_cel, poss_hor=pos_hor, ddpos=ddpos, amps=amps, damps=damps, pcorr=pcorr, nsrc=len(self.sdata), models=models, nsigma=nsigma, chisq0=chisq0, chisq=chisq, npix=self.lik.npix) return res
def get_sids_in_tod(id, src_pos, bounds, ind, isids=None, src_sys="cel"): if isids is None: isids = list(range(src_pos.shape[-1])) if bounds is not None: poly = bounds[:,:,ind]*utils.degree poly[0] = utils.rewind(poly[0],poly[0,0]) # bounds are defined in celestial coordinates. Must convert srcpos for comparison mjd = utils.ctime2mjd(float(id.split(".")[0])) srccel = coordinates.transform(src_sys, "cel", src_pos, time=mjd) srccel[0] = utils.rewind(srccel[0], poly[0,0]) poly = pad_polygon(poly.T, poly_pad).T accepted = np.where(utils.point_in_polygon(srccel.T, poly.T))[0] sids = [isids[i] for i in accepted] else: sids = isids return sids
def build_rangedata(tod, rcut, d, ivar): nmax = np.max(rcut.ranges[:, 1] - rcut.ranges[:, 0]) nrange = rcut.nrange rdata = bunch.Bunch() rdata.detmap = np.zeros(nrange, int) rdata.tod = np.zeros([nrange, nmax], dtype) rdata.pos = np.zeros([nrange, nmax, 2]) rdata.ivar = np.zeros([nrange, nmax], dtype) # Build our detector mapping for di in range(rcut.ndet): rdata.detmap[rcut.detmap[di]:rcut.detmap[di + 1]] = di rdata.n = rcut.ranges[:, 1] - rcut.ranges[:, 0] # Extract our tod samples and coordinates for i, r in enumerate(rcut.ranges): di = rdata.detmap[i] rn = r[1] - r[0] rdata.tod[i, :rn] = tod[di, r[0]:r[1]] bore = d.boresight[:, r[0]:r[1]] mjd = utils.ctime2mjd(bore[0]) pos_hor = bore[1:] + d.point_offset[di, :, None] pos_rel = coordinates.transform(tod_sys, sys, pos_hor, time=mjd, site=d.site) rdata.pos[i, :rn] = pos_rel.T # Expand noise ivar too, including the effect of our normal data cut rdata.ivar[ i, :rn] = ivar[di] * (1 - d.cut[di:di + 1, r[0]:r[1]].to_mask()[0]) # Precompute our fourier space units rdata.freqs = fft.rfftfreq(nmax, 1 / d.srate) # And precompute out butterworth filter rdata.butter = filters.mce_filter(rdata.freqs, d.mce_fsamp, d.mce_params) # Store the fiducial time constants for reference rdata.tau = d.tau # These are also nice to have rdata.dsens = ivar**-0.5 / d.srate**0.5 rdata.asens = np.sum(ivar)**-0.5 / d.srate**0.5 rdata.srate = d.srate rdata.dets = d.dets rdata.beam = d.beam rdata.id = d.entry.id return rdata
def calc_full_result(self, dpos, marginalize=True): res = self.lik.eval(dpos) res.poss_cel = res.poss res.poss_hor = np.array([ coordinates.transform("cel", "hor", res.poss_cel[i], utils.ctime2mjd(self.sdata[i].ctime), site=self.sdata[i].site) for i in range(self.nsrc) ]) # Get the position uncertainty hess = self.calc_hessian(dpos, step=0.1 * utils.arcmin) hess = 0.5 * (hess + hess.T) try: pcov = np.linalg.inv(0.5 * hess) except np.linalg.LinAlgError: pcov = np.diag([np.inf, np.inf]) dchisq = res.chisq0 - res.chisq # Apply marginalization correction if marginalize: R = self.lik.rmax A = np.pi * R**2 Abeam = 2 * np.pi * self.fwhm**2 / (8 * np.log(2)) npoint = int(np.round(A / Abeam * self.nsrc)) # Correct position and uncertainty dpos, pcov = marg_pos(dpos, pcov, res.chisq0 - res.chisq, npoint, R) # Correct total chisquare prob = stats.norm.cdf(-dchisq**0.5) if prob > 1e-10: prob = 1 - (1 - prob)**npoint dchisq = stats.norm.ppf(prob)**2 ddpos = np.diag(pcov)**0.5 pcorr = pcov[0, 1] / ddpos[0] / ddpos[1] res.dpos = dpos res.ddpos = ddpos res.damps = res.vamps**0.5 res.pcorr = pcorr res.pcov = pcov res.nsrc = self.nsrc res.dchisq = dchisq res.nsigma = dchisq**0.5 return res
def __init__(self, sdat): self.mjd = utils.ctime2mjd(sdat.ctime) self.site = sdat.site self.ref_cel = sdat.srcpos # Find the boresight pointing that defines our focalplane # coordinates. This is not exact for two reasons: # 1. The array center is offset from the boresight, by about 1 degree. # 2. The detectors are offset from the array center by half that. # We could store the former to improve our accuracy a bit, but # to get #2 we would need a time-domain fit, which is what we're # trying to avoid here. The typical error from using the array center # instead would be about 1' offset * 1 degree error = 1.05 arcsec error. # We *can* easily get the boresight elevation since we have constant # elevation scans, so that removes half the error. That should be # good enough. self.bore= [ coordinates.transform("cel","hor",sdat.srcpos,time=self.mjd,site=self.site)[0], sdat.el ] self.ref_foc = cel2foc(self.ref_cel, self.site, self.mjd, self.bore)
def avoidance_cut(bore, det_offs, site, name_or_pos, margin): """Cut samples that get too close to the specified object (e.g. "Sun" or "Moon") or celestial position ([ra,dec] in racians). Margin specifies how much to avoid the object by.""" cmargin = np.cos(margin) mjd = utils.ctime2mjd(bore[0]) obj_pos = coordinates.interpol_pos("cel", "hor", name_or_pos, mjd, site) obj_pos[0] = utils.rewind(obj_pos[0], bore[1]) cosel = np.cos(obj_pos[1]) # Only cut if above horizon above_horizon = obj_pos[1] > 0 null_cut = sampcut.empty(det_offs.shape[0], bore.shape[1]) if np.all(~above_horizon): return null_cut # Find center of array, and radius arr_center = np.mean(det_offs, 0) arr_rad = np.max(np.sum((det_offs - arr_center)**2, 1)**0.5) def calc_mask(det_pos, rad, mask=slice(None)): offs = (det_pos - obj_pos[:, mask]) offs[0] *= cosel[mask] dists2 = np.sum(offs**2, 0) return dists2 < rad**2 # Find samples that could possibly be near object for any detector cand_mask = calc_mask(arr_center[:, None] + bore[1:], margin + arr_rad) cand_mask &= above_horizon cand_inds = np.where(cand_mask)[0] if len(cand_inds) == 0: return null_cut # Loop through all detectors and find out if each candidate actually intersects cuts = [] for di, off in enumerate(det_offs): det_pos = bore[1:, cand_inds] + off[:, None] det_mask = calc_mask(det_pos, margin, cand_inds) # Expand mask to full set det_mask_full = np.zeros(bore.shape[1], bool) det_mask_full[cand_inds] = det_mask # And use this to build actual cuts cuts.append(sampcut.from_mask(det_mask_full)) res = sampcut.stack(cuts) return res
def build_rangedata(tod, rcut, d, ivar): nmax = np.max(rcut.ranges[:,1]-rcut.ranges[:,0]) nrange = rcut.nrange rdata = bunch.Bunch() rdata.detmap = np.zeros(nrange,int) rdata.tod = np.zeros([nrange,nmax],dtype) rdata.pos = np.zeros([nrange,nmax,2]) rdata.ivar= np.zeros([nrange,nmax],dtype) # Build our detector mapping for di in range(rcut.ndet): rdata.detmap[rcut.detmap[di]:rcut.detmap[di+1]] = di rdata.n = rcut.ranges[:,1]-rcut.ranges[:,0] # Extract our tod samples and coordinates for i, r in enumerate(rcut.ranges): di = rdata.detmap[i] rn = r[1]-r[0] rdata.tod[i,:rn] = tod[di,r[0]:r[1]] bore = d.boresight[:,r[0]:r[1]] mjd = utils.ctime2mjd(bore[0]) pos_hor = bore[1:] + d.point_offset[di,:,None] pos_rel = coordinates.transform(tod_sys, sys, pos_hor, time=mjd, site=d.site) rdata.pos[i,:rn] = pos_rel.T # Expand noise ivar too, including the effect of our normal data cut rdata.ivar[i,:rn] = ivar[di] * (1-d.cut[di:di+1,r[0]:r[1]].to_mask()[0]) # Precompute our fourier space units rdata.freqs = fft.rfftfreq(nmax, 1/d.srate) # And precompute out butterworth filter rdata.butter = filters.mce_filter(rdata.freqs, d.mce_fsamp, d.mce_params) # Store the fiducial time constants for reference rdata.tau = d.tau # These are also nice to have rdata.dsens = ivar**-0.5 / d.srate**0.5 rdata.asens = np.sum(ivar)**-0.5 / d.srate**0.5 rdata.srate = d.srate rdata.dets = d.dets rdata.beam = d.beam rdata.id = d.entry.id return rdata
def summarize(self, chain): """Given a chain of lpars, compute a summary in the same format as returned by SrcFitterML.""" dposs = np.array([c.dpos for c in chain]) poss_cel = np.array([c.poss for c in chain]) poss_hor = poss_cel*0 for i in range(len(self.sdata)): poss_hor[:,i] = coordinates.transform("cel","hor",poss_cel[:,i].T, time=utils.ctime2mjd(self.sdata[i].ctime), site=self.sdata[i].site).T ampss = np.array([c.amps for c in chain]) vampss = np.array([c.vamps for c in chain]) chisq0 = chain[0].chisq0 chisq = np.mean([c.chisq for c in chain]) # Compute means dpos = np.mean(dposs,0) pos_cel = np.mean(poss_cel,0) pos_hor = np.mean(poss_hor,0) amps = np.mean(ampss,0) # And uncertainties dpos_cov= np.cov(dposs.T) ddpos = np.diag(dpos_cov)**0.5 pcorr = dpos_cov[0,1]/ddpos[0]/ddpos[1] # mean([da0_i + da_ij]**2). Uncorrelated, so this is # mean(da0**2) + mean(da**2) arel = ampss + np.random.standard_normal(ampss.shape)*vampss**0.5 amps = np.mean(arel,0) vamps = np.var(arel,0) #vamps = np.var(ampss,0)+np.mean(vampss,0) damps = vamps**0.5 models = self.lik.eval(dpos).models nsigma = (chisq0-chisq)**0.5 # We want how much to offset detector by, not how much to offset # source by res = bunch.Bunch( dpos = dpos, poss_cel=pos_cel, poss_hor=pos_hor, ddpos= ddpos, amps=amps, damps=damps, pcorr=pcorr, nsrc = len(self.sdata), models=models, nsigma = nsigma, chisq0 = chisq0, chisq = chisq, npix=self.lik.npix) return res
def calc_full_result(self, dpos, marginalize=True): res = self.lik.eval(dpos) res.poss_cel = res.poss res.poss_hor = np.array([ coordinates.transform("cel","hor",res.poss_cel[i],utils.ctime2mjd(self.sdata[i].ctime),site=self.sdata[i].site) for i in range(self.nsrc) ]) # Get the position uncertainty hess = self.calc_hessian(dpos, step=0.1*utils.arcmin) hess = 0.5*(hess+hess.T) try: pcov = np.linalg.inv(0.5*hess) except np.linalg.LinAlgError: pcov = np.diag([np.inf,np.inf]) dchisq = res.chisq0-res.chisq # Apply marginalization correction if marginalize: R = self.lik.rmax A = np.pi*R**2 Abeam = 2*np.pi*self.fwhm**2/(8*np.log(2)) npoint= int(np.round(A/Abeam * self.nsrc)) # Correct position and uncertainty dpos, pcov = marg_pos(dpos, pcov, res.chisq0-res.chisq, npoint, R) # Correct total chisquare prob = stats.norm.cdf(-dchisq**0.5) if prob > 1e-10: prob = 1-(1-prob)**npoint dchisq = stats.norm.ppf(prob)**2 ddpos = np.diag(pcov)**0.5 pcorr = pcov[0,1]/ddpos[0]/ddpos[1] res.dpos = dpos res.ddpos = ddpos res.damps = res.vamps**0.5 res.pcorr = pcorr res.pcov = pcov res.nsrc = self.nsrc res.dchisq= dchisq res.nsigma= dchisq**0.5 return res
def avoidance_cut(bore, det_offs, site, name_or_pos, margin): """Cut samples that get too close to the specified object (e.g. "Sun" or "Moon") or celestial position ([ra,dec] in racians). Margin specifies how much to avoid the object by.""" cmargin = np.cos(margin) mjd = utils.ctime2mjd(bore[0]) obj_pos = coordinates.interpol_pos("cel","hor",name_or_pos,mjd,site) obj_pos[0] = utils.rewind(obj_pos[0], bore[1]) cosel = np.cos(obj_pos[1]) # Only cut if above horizon above_horizon = obj_pos[1]>0 null_cut = sampcut.empty(det_offs.shape[0], bore.shape[1]) if np.all(~above_horizon): return null_cut # Find center of array, and radius arr_center = np.mean(det_offs,0) arr_rad = np.max(np.sum((det_offs-arr_center)**2,1)**0.5) def calc_mask(det_pos, rad, mask=slice(None)): offs = (det_pos-obj_pos[:,mask]) offs[0] *= cosel[mask] dists2= np.sum(offs**2,0) return dists2 < rad**2 # Find samples that could possibly be near object for any detector cand_mask = calc_mask(arr_center[:,None] + bore[1:], margin+arr_rad) cand_mask &= above_horizon cand_inds = np.where(cand_mask)[0] if len(cand_inds) == 0: return null_cut # Loop through all detectors and find out if each candidate actually intersects cuts = [] for di, off in enumerate(det_offs): det_pos = bore[1:,cand_inds]+off[:,None] det_mask = calc_mask(det_pos, margin, cand_inds) # Expand mask to full set det_mask_full = np.zeros(bore.shape[1], bool) det_mask_full[cand_inds] = det_mask # And use this to build actual cuts cuts.append(sampcut.from_mask(det_mask_full)) res = sampcut.stack(cuts) return res
def __init__(self, sdat): self.mjd = utils.ctime2mjd(sdat.ctime) self.site = sdat.site self.ref_cel = sdat.srcpos # Find the boresight pointing that defines our focalplane # coordinates. This is not exact for two reasons: # 1. The array center is offset from the boresight, by about 1 degree. # 2. The detectors are offset from the array center by half that. # We could store the former to improve our accuracy a bit, but # to get #2 we would need a time-domain fit, which is what we're # trying to avoid here. The typical error from using the array center # instead would be about 1' offset * 1 degree error = 1.05 arcsec error. # We *can* easily get the boresight elevation since we have constant # elevation scans, so that removes half the error. That should be # good enough. self.bore = [ coordinates.transform("cel", "hor", sdat.srcpos, time=self.mjd, site=self.site)[0], sdat.el ] self.ref_foc = cel2foc(self.ref_cel, self.site, self.mjd, self.bore)
def avoidance_cut_old(bore, det_offs, site, name_or_pos, margin): """Cut samples that get too close to the specified object (e.g. "Sun" or "Moon") or celestial position ([ra,dec] in racians). Margin specifies how much to avoid the object by.""" cmargin = np.cos(margin) mjd = utils.ctime2mjd(bore[0]) obj_pos = coordinates.interpol_pos("cel","hor",name_or_pos,mjd,site) obj_rect = utils.ang2rect(obj_pos, zenith=False) # Only cut if above horizon above_horizon = obj_pos[1]>0 if np.all(~above_horizon): return sampcut.empty(det_offs.shape[0], bore.shape[1]) cuts = [] for di, off in enumerate(det_offs): det_pos = bore[1:]+off[:,None] #det_rect1 = utils.ang2rect(det_pos, zenith=False) # slow # Not sure defining an ang2rect in array_ops is worth it.. It's ugly, # (angle convention hard coded) and only gives factor 2 on laptop. det_rect = array_ops.ang2rect(np.ascontiguousarray(det_pos.T)).T cdist = np.sum(obj_rect*det_rect,0) # Cut samples above horizon that are too close bad = (cdist > cmargin) & above_horizon cuts.append(sampcut.from_mask(bad)) res = sampcut.stack(cuts) return res
only = [int(word) for word in args.only.split(",")] if args.only else [] # Should we use distributed maps? npix = shape[-2] * shape[-1] use_dmap = npix > 5e7 utils.mkdir(root + "log") logfile = root + "log/log%03d.txt" % comm.rank log_level = log.verbosity2level(config.get("verbosity")) L = log.init(level=log_level, file=logfile, rank=comm.rank) filedb.init() db = filedb.scans.select(args.sel) ids = db.ids mjd = utils.ctime2mjd(db.data["t"]) chunks = utils.find_equal_groups(mjd // args.dt) chunks = [np.sort(chunk) for chunk in chunks] chunks = [chunks[i] for i in np.argsort([c[0] for c in chunks])] corr_pos = planet9.choose_corr_points(shape, wcs, args.corr_spacing * utils.degree) if args.inject: inject_params = np.loadtxt(args.inject, ndmin=2) # [:,{ra0,dec0,R,vx,vy,flux}] asteroids = planet9.get_asteroids(args.asteroid_file, args.asteroid_list) # How to parallelize? Could do it over chunks. Usually there will be more chunks than # mpi tasks. But there will still be many tods per chunk too (about 6 tods per hour # and 72 hours per chunk gives 432 tods per chunk). That's quite a bit for one mpi
return res srcs = load_srcs(args.srcs) # Find out which sources are hit by each tod db = filedb.scans.select(filedb.scans[args.sel]) tod_srcs = {} for sid, src in enumerate(srcs): if args.src is not None and sid != args.src: continue if src.type == "planet": # This is a bit hacky, but sometimes the "t" member is unavailable # in the database. This also ignores the planet movement during the # TOD. We will take that into account in the final step in the mapping, though. t = np.char.partition(db.ids, ".")[:, 0].astype(float) + 300 ra, dec = ephemeris.ephem_pos(src.name, utils.ctime2mjd(t), dt=0)[:2] else: ra, dec = src.ra, src.dec points = np.array([ra, dec]) polys = db.data["bounds"] * utils.degree polys[0] = utils.rewind(polys[0], points[0]) polys[0] = utils.rewind(polys[0], polys[0, 0]) inside = utils.point_in_polygon(points.T, polys.T) dists = utils.poly_edge_dist(points.T, polys.T) dists = np.where(inside, 0, dists) hit = np.where(dists < args.hit_tol * utils.degree)[0] for id in db.ids[hit]: if not id in tod_srcs: tod_srcs[id] = [] tod_srcs[id].append(sid) # Prune those those that are done
res.type = "fixed" return res srcs = load_srcs(args.srcs) # Find out which sources are hit by each tod db = filedb.scans.select(filedb.scans[args.sel]) tod_srcs = {} for sid, src in enumerate(srcs): if args.src is not None and sid != args.src: continue if src.type == "planet": # This is a bit hacky, but sometimes the "t" member is unavailable # in the database. This also ignores the planet movement during the # TOD. We will take that into account in the final step in the mapping, though. t = np.char.partition(db.ids,".")[:,0].astype(float)+300 ra, dec = ephemeris.ephem_pos(src.name, utils.ctime2mjd(t), dt=0)[:2] else: ra, dec = src.ra, src.dec points = np.array([ra, dec]) polys = db.data["bounds"]*utils.degree polys[0] = utils.rewind(polys[0], points[0]) polys[0] = utils.rewind(polys[0], polys[0,0]) inside = utils.point_in_polygon(points.T, polys.T) dists = utils.poly_edge_dist(points.T, polys.T) dists = np.where(inside, 0, dists) hit = np.where(dists < args.hit_tol*utils.degree)[0] for id in db.ids[hit]: if not id in tod_srcs: tod_srcs[id] = [] tod_srcs[id].append(sid) # Prune those those that are done if args.cont:
try: d = actdata.read(entry, ["point_offsets","boresight","site","array_info"]) d = actdata.calibrate(d, exclude=["autocut"]) except (errors.DataMissing, AttributeError) as e: print("Skipping %s (%s)" % (id, e)) continue # Build a projector between samples and mask. This # requires us to massage d into scan form. It's getting # annoying that scan and data objects aren't compatible. bore = d.boresight.T.copy() bore[:,0] -= bore[0,0] scan = enscan.Scan( boresight = bore, offsets = np.concatenate([np.zeros(d.ndet)[:,None],d.point_offset],1), comps = np.concatenate([np.ones(d.ndet)[:,None],np.zeros((d.ndet,3))],1), mjd0 = utils.ctime2mjd(d.boresight[0,0]), sys = "hor", site = d.site) scan.hwp_phase = np.zeros([len(bore),2]) bore_box = np.array([np.min(d.boresight,1),np.max(d.boresight,1)]) bore_corners = utils.box2corners(bore_box) scan.entry = d.entry # Is the source above the horizon? If not, it doesn't matter how close # it is. mjd = utils.ctime2mjd(utils.mjd2ctime(scan.mjd0)+scan.boresight[::100,0]) try: object_pos = coordinates.interpol_pos("cel","hor", args.objname, mjd, site=scan.site) except AttributeError as e: print("Unexpected error in interpol_pos for %s. mid time was %.5f. message: %s. skipping" % (id, mjd[len(mjd)//2], e)) continue visible = np.any(object_pos[1] >= -margin) if not visible:
def build_tod_stats(entry, Naz=8, Nt=2): """Collect summary information for the tod in the given entry, returning it as a bunch. If some information can't be found, then those fields will be set to a placeholder value (usually NaN), but the fields will still all be present.""" # At the very least we need the pointing, so no try catch around this d = actdata.read(entry, ["boresight", "site"]) d += actdata.read_point_offsets(entry, no_correction=True) d = actdata.calibrate(d, exclude=["autocut"]) # Get the array center and radius acenter = np.mean(d.point_offset, 0) arad = np.mean((d.point_offset - acenter)**2, 0)**0.5 t, baz, bel = 0.5 * (np.min(d.boresight, 1) + np.max(d.boresight, 1)) #t, baz, bel = np.mean(d.boresight,1) az = baz + acenter[0] el = bel + acenter[1] dur, waz, wel = np.max(d.boresight, 1) - np.min(d.boresight, 1) if waz > 180 * utils.degree: print("bad waz %8.3f for %s" % (waz / utils.degree, entry.id)) mjd = utils.ctime2mjd(t) hour = t / 3600. % 24 day = hour >= day_range[0] and hour < day_range[1] night = not day jon = (t - jon_ref) / (3600 * 24) ra, dec = coordinates.transform(tsys, "cel", [az, el], mjd, site=d.site) # Get the array center bounds on the sky, assuming constant elevation ts = utils.ctime2mjd(t + dur / 2 * np.linspace(-1, 1, Nt)) azs = az + waz / 2 * np.linspace(-1, 1, Naz) E1 = coordinates.transform(tsys, "cel", [azs, [el] * Naz], time=[ts[0]] * Naz, site=d.site)[:, 1:] E2 = coordinates.transform(tsys, "cel", [[azs[-1]] * Nt, [el] * Nt], time=ts, site=d.site)[:, 1:] E3 = coordinates.transform(tsys, "cel", [azs[::-1], [el] * Naz], time=[ts[-1]] * Naz, site=d.site)[:, 1:] E4 = coordinates.transform(tsys, "cel", [[azs[0]] * Nt, [el] * Nt], time=ts[::-1], site=d.site)[:, 1:] bounds = np.concatenate([E1, E2, E3, E4], 1) bounds[0] = utils.rewind(bounds[0]) ## Grow bounds by array radius #bmid = np.mean(bounds,1) #for i in range(2): # bounds[i,bounds[i]<bmid[i]] -= arad[i] # bounds[i,bounds[i]>bmid[i]] += arad[i] tot_id = entry.id + (":" + entry.tag if entry.tag else "") res = bunch.Bunch(id=tot_id, nsamp=d.nsamp, t=t, mjd=mjd, jon=jon, hour=hour, day=day, night=night, dur=dur, az=az / utils.degree, el=el / utils.degree, baz=baz / utils.degree, bel=bel / utils.degree, waz=waz / utils.degree, wel=wel / utils.degree, ra=ra / utils.degree, dec=dec / utils.degree, bounds=bounds / utils.degree) if "gseason" in entry: res[entry.gseason] = True # Planets for obj in [ "Sun", "Moon", "Mercury", "Venus", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune" ]: res[obj] = coordinates.ephem_pos(obj, utils.ctime2mjd(t)) / utils.degree # Get our weather information, if available try: d += actdata.read(entry, ["apex"]) d = actdata.calibrate_apex(d) res["pwv"] = d.apex.pwv res["wx"] = d.apex.wind[0] res["wy"] = d.apex.wind[1] res["wind_speed"] = d.apex.wind_speed res["T"] = d.apex.temperature except errors.DataMissing: res["pwv"] = np.NaN res["wx"] = np.NaN res["wy"] = np.NaN res["wind_speed"] = np.NaN res["T"] = np.NaN # Try to get our cut info, so that we can select on # number of detectors and cut fraction try: npre = d.nsamp * d.ndet d += actdata.read(entry, ["cut"]) res["ndet"] = d.ndet res["cut"] = 1 - d.nsamp * d.ndet / float(npre) except errors.DataMissing: res["ndet"] = 0 res["cut"] = 1.0 # Try to get hwp info res["hwp"] = False res["hwp_name"] = "none" try: epochs = actdata.try_read(files.read_hwp_epochs, "hwp_epochs", entry.hwp_epochs) t, _, ar = entry.id.split(".") t = float(t) if ar in epochs: for epoch in epochs[ar]: if t >= epoch[0] and t < epoch[1]: res["hwp"] = True res["hwp_name"] = epoch[2] except errors.DataMissing: pass return res
def __call__(self, pos): mjd = utils.ctime2mjd(self.data["t"]) hor = coordinates.transform("cel", "hor", pos * utils.degree, mjd) / utils.degree return hor
def get_pix_ranges(shape, wcs, horbox, daz, nt=4, ndet=1.0, site=None): """An appropriate daz for this function is about 1 degree""" # For each row in the map we want to know the hit density for that row, # as well as its start and end. In the original function we got one # sample per row by oversampling and then using unique. This is unreliable, # and also results in quantized steps in the depth. We can instead # do a coarse equispaced az -> ra,dec -> y,x. We can then interpolate # this to get exactly one sample per y. To get the density properly, # we just need dy/dt = dy/daz * daz/dt, where we assume daz/dt is constant. # We get dy/daz from the coarse stuff, and interpolate that too, which gives # the density per row. (t1, t2), (az1, az2), el = horbox[:, 0], horbox[:, 1], np.mean(horbox[:, 2]) nphi = np.abs(utils.nint(360 / wcs.wcs.cdelt[0])) # First produce the coarse single scan naz = utils.nint(np.abs(az2 - az1) / daz) if naz <= 1: return None, None ahor = np.zeros([3, naz]) ahor[0] = utils.ctime2mjd(t1) ahor[1] = np.linspace(az1, az2, naz) ahor[2] = el acel = coordinates.transform("hor", "cel", ahor[1:], time=ahor[0], site=site) ylow, x1low = fixx(enmap.sky2pix(shape, wcs, acel[::-1]), nphi) if ylow[1] < ylow[0]: ylow, x1low = ylow[::-1], x1low[::-1] # Find dy/daz for these points glow = np.gradient(ylow) * (naz - 1) / (az2 - az1) # Now interpolate to full resolution y = np.arange(utils.nint(ylow[0]), utils.nint(ylow[-1]) + 1) if len(y) == 0: print "Why is y empty?", naz, ylow[0], ylow[1] return None, None x1 = np.interp(y, ylow, x1low) grad = np.interp(y, ylow, glow) # Now we just need the width of the rows, x2, which comes # from the time drift thor = np.zeros([3, nt]) thor[0] = utils.ctime2mjd(np.linspace(t1, t2, nt)) thor[1] = az1 thor[2] = el tcel = coordinates.transform("hor", "cel", thor[1:], time=thor[0], site=site) _, tx = utils.nint(fixx(enmap.sky2pix(shape, wcs, tcel[::-1]), nphi)) x2 = x1 + tx[-1] - tx[0] x1, x2 = np.minimum(x1, x2), np.maximum(x1, x2) pix_ranges = utils.nint( np.concatenate([y[:, None], x1[:, None], x2[:, None]], 1)) # Weight per pixel. We want this to be in units of seconds of # observing time per pixel if ndet=1. We know the total number of pixels # hit (ny*nx) and the total time (t2-t1), and we know the relative # weight per row (1/grad), so we can just normalize things ny, nx = len(y), x2[0] - x1[0] npix = ny * nx if npix == 0 or np.any(grad <= 0): return pix_ranges, grad * 0 else: weights = 1 / grad weights *= (t2 - t1) / (np.sum(weights) * nx) * ndet # *nx because weight is per row return pix_ranges, weights
entry, fields=["gain", "cut", "point_offsets", "boresight", "site"]) d = data.calibrate(d) except (zipfile.BadZipfile, errors.DataMissing) as e: print "#%s error: %s" % (id, e.message) #print "%s %8.3f %7.3f %8.3f %7.3f %s" % (id, np.nan, np.nan, np.nan, np.nan, "nodata") continue hour = info[ind].fields.hour tags = sorted(list(set(info[ind].tags) - set([id]))) # Get input pointing bore = d.boresight[:, ::sstep] offs = d.point_offset.T[:, ::dstep] ipoint = np.zeros(bore.shape + offs.shape[1:]) ipoint[0] = utils.ctime2mjd(bore[0, :, None]) ipoint[1:] = bore[1:, :, None] + offs[:, None, :] ipoint = ipoint.reshape(3, -1) iref = np.mean(ipoint[1:], 1) # Transform to equ opoint = coordinates.transform("hor", "equ", ipoint[1:], time=ipoint[0], site=d.site) oref = np.mean(opoint, 1) print "%s %4.1f %8.3f %7.3f %8.3f %7.3f" % ( id, hour, iref[0] / utils.degree, iref[1] / utils.degree, oref[0] / utils.degree, oref[1] / utils.degree),
def build_workspace_geometry(wid, bore, point_offset, global_wcs, site=None, tagger=None, padding=100, max_ra_width=2.5*utils.degree, ncomp=3, dtype=np.float64): if tagger is None: tagger = WorkspaceTagger() if isinstance(wid, basestring): wid = tagger.analyze(wid) if not valid_az_range(wid[0], wid[1]): raise WorkspaceError("Azimuth crosses north/south") trans = TransformPos2Pospix(global_wcs, site=site) az1, az2, el, ra1 = wid # Extract the workspace definition of the tag name ra_ref = ra1 + tagger.ra_step/2 # We want ra(dec) for up- and down-sweeps for the middle of # the workspace. First find a t that will result in a sweep # that crosses through the middle of the workspace. foc_offset = np.mean(point_offset,0) t0 = utils.ctime2mjd(bore[0,0]) t_ref = find_t_giving_ra(az1+foc_offset[0], el+foc_offset[1], ra_ref, site=site, t0=t0) # We also need the corners of the full workspace area. t1 = find_t_giving_ra(az1+foc_offset[0], el+foc_offset[1], ra1, site=site, t0=t0) t2 = find_t_giving_ra(az1+foc_offset[0], el+foc_offset[1], ra1+tagger.ra_step+max_ra_width, site=site, t0=t0) #print "t1", t1, "t2", t2 #print "az1", az1/utils.degree, "az2", az2/utils.degree #print "ra", ra1/utils.degree, (ra1+tagger.ra_step+max_ra_width)/utils.degree bore_box_hor = np.array([[t1,az1,el],[t2,az2,el]]) bore_corners_hor = utils.box2corners(bore_box_hor) work_corners_hor = bore_corners_hor[None,:,:] + (point_offset[:,[0,0,1]] * [0,1,1])[:,None,:] work_corners_hor = work_corners_hor.T.reshape(3,-1) work_corners = trans(work_corners_hor[1:], time=work_corners_hor[0]) ixcorn, iycorn = np.round(work_corners[2:]).astype(int) iybox = np.array([np.min(iycorn)-padding,np.max(iycorn)+1+padding]) # Generate an up and down sweep srate = get_srate(bore[0]) period = pmat.get_scan_period(bore[1], srate) dmjd = period/2./24/3600 xshifts = [] yshifts = [] work_dazs = [] nwxs, nwys = [], [] for si, (afrom,ato) in enumerate([[az1,az2],[az2,az1]]): sweep = generate_sweep_by_dec_pix( [[ t_ref, afrom+foc_offset[0],el+foc_offset[1]], [t_ref+dmjd,ato +foc_offset[0],el+foc_offset[1]] ],iybox,trans) # Get the shift in ra pix per dec pix. At this point, # the shifts are just relative to the lowest-dec pixel xshift = np.round(sweep[5]-sweep[5,0,None]).astype(int) # Get the shift in dec pix per dec pix. These tell us where # each working pixel starts as a function of normal dec pixel. # For example [0,1,3,6] would mean that the work to normal pixel # mapping is [0,1,1,2,2,2]. This is done to make dwdec/daz approximately # constant daz = np.abs(sweep[1,1:]-sweep[1,:-1]) daz_ratio = np.maximum(1,daz/np.min(daz[1:-1])) yshift = np.round(utils.cumsum(daz_ratio, endpoint=True)).astype(int) yshift -= yshift[0] # Now that we have the x and y mapping, we can determine the # bounds of our workspace by transforming the corners of our # input coordinates. #print "iyc", iycorn-iybox[0] #print "ixc", ixcorn #for i in np.argsort(iycorn): # print "A %6d %6d" % (iycorn[i], ixcorn[i]) #print "min(ixc)", np.min(ixcorn) #print "max(ixc)", np.max(ixcorn) #print "xshift", xshift[iycorn-iybox[0]] wycorn = ixcorn - xshift[iycorn-iybox[0]] #print "wycorn", wycorn #print "min(wyc)", np.min(wycorn) #print "max(wyc)", np.max(wycorn) # Modify the shifts so that any scan in this workspace is always transformed # to valid positions. wx and wy are transposed relative to x and y. # Padding is needed because of the rounding involved in recovering the # az and el from the wid. xshift += np.min(wycorn) xshift -= padding wycorn2= ixcorn - xshift[iycorn-iybox[0]] #print "wycorn2", wycorn2 #print "min(wyc2)", np.min(wycorn2) #print "max(wyc2)", np.max(wycorn2) #sys.stdout.flush() nwy = np.max(wycorn)-np.min(wycorn)+1 + 2*padding nwx = yshift[-1]+1 # Get the average azimuth spacing in wx work_daz = (sweep[1,-1]-sweep[1,0])/(yshift[-1]-yshift[0]) print work_daz/utils.degree # And collect so we can pass them to the Workspace construtor later xshifts.append(xshift) yshifts.append(yshift) nwxs.append(nwx) nwys.append(nwy) work_dazs.append(work_daz) # The shifts from each sweep are guaranteed to have the same length, # since they are based on the same iybox. nwx = np.max(nwxs) # To translate the noise properties, we need a mapping from the x and t # fourier spaces. For this we need the azimuth scanning speed. scan_speed = 2*(az2-az1)/period work_daz = np.mean(work_dazs) wgeo = WorkspaceGeometry(nwys, nwx, xshifts, yshifts, iybox[0], scan_speed, work_daz, global_wcs, ncomp=ncomp, dtype=dtype) return wgeo
def __init__(self, entry, subdets=None, d=None, verbose=False, dark=False): self.fields = [ "gain", "mce_filter", "tags", "polangle", "tconst", "hwp", "cut", "point_offsets", "boresight", "site", "tod_shape", "array_info", "beam", "pointsrcs", "buddies" ] if dark: self.fields += ["dark"] if config.get("noise_model") == "file": self.fields += ["noise"] else: if config.get("cut_noise_whiteness"): self.fields += ["noise_cut"] if config.get("cut_spikes"): self.fields += ["spikes"] if d is None: d = actdata.read(entry, self.fields, verbose=verbose) d = actdata.calibrate(d, verbose=verbose) if subdets is not None: d.restrict(dets=d.dets[subdets]) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("No data in scan") ndet = d.ndet # Necessary components for Scan interface self.mjd0 = utils.ctime2mjd(d.boresight[0, 0]) self.boresight = np.ascontiguousarray( d.boresight.T.copy()) # [nsamp,{t,az,el}] self.boresight[:, 0] -= self.boresight[0, 0] self.offsets = np.zeros([ndet, self.boresight.shape[1]]) self.offsets[:, 1:] = d.point_offset self.cut = d.cut.copy() self.cut_noiseest = d.cut_noiseest.copy() self.comps = np.zeros([ndet, 4]) self.beam = d.beam self.pointsrcs = d.pointsrcs self.comps = d.det_comps self.hwp = d.hwp self.hwp_phase = d.hwp_phase self.dets = d.dets self.dgrid = (d.array_info.nrow, d.array_info.ncol) self.array_info = d.array_info self.sys = config.get("tod_sys", entry.tod_sys if "tod_sys" in entry else None) self.site = d.site self.speed = d.speed if "noise" in d: self.noise = d.noise else: spikes = d.spikes[:2].T if "spikes" in d else None self.noise = nmat_measure.NmatBuildDelayed( model=config.get("noise_model"), spikes=spikes, cut=self.cut_noiseest) if "dark_tod" in d: self.dark_tod = d.dark_tod if "dark_cut" in d: self.dark_cut = d.dark_cut if "buddy_comps" in d: # Expand buddy_offs to {dt,daz,ddec} self.buddy_comps = d.buddy_comps self.buddy_offs = np.concatenate( [d.buddy_offs[..., :1] * 0, d.buddy_offs], -1) self.autocut = d.autocut if "autocut" in d else [] # Implementation details. d is our DataSet, which we keep around in # because we need it to read tod consistently later. It will *not* # take part in any sample slicing operations, as that might make the # delayed tod read inconsistent with the rest. It could take part in # detector slicing as long as calibrate_tod operates on each detector # independently. This is true now, but would not be so if we did stuff # like common mode subtraction there. On the other hand, not doing this # would prevent slicing before reading from giving any speedup or memory # savings. I don't think allowing this should be a serious problem. self.d = d self.entry = entry def fmt_id(entry): if isinstance(entry, list): return "+".join([fmt_id(e) for e in entry]) else: if entry.tag: return entry.id + ":" + entry.tag else: return entry.id self.id = fmt_id(entry) self.sampslices = [] self.mapping = None # FIXME: debug test if config.get("dummy_cut") > 0: nmax = int(config.get("dummy_cut_len")) # Power law between 1 and nmax, with slope -1. # C(w) = log(w)/log(nmax) # P(w) = w**-1/log(nmax) # w(C) = n**C # Mean: (nmax-1)/log(nmax) nmean = (nmax - 1) / np.log(nmax) ncut = int(self.nsamp * config.get("dummy_cut") / nmean) cut_ranges = np.zeros([self.ndet, ncut, 2], int) w = (nmax**np.random.uniform(0, 1, size=[self.ndet, ncut])).astype(int) np.clip(w, 1, nmax) cut_ranges[:, :, 0] = np.random.uniform(0, self.nsamp, size=[self.ndet, ncut]).astype(int) cut_ranges[:, :, 0] = np.sort(cut_ranges[:, :, 0], 1) cut_ranges[:, :, 1] = cut_ranges[:, :, 0] + w np.clip(cut_ranges[:, :, 1], 0, self.nsamp) cut_dummy = sampcut.from_list(cut_ranges, self.nsamp) print(np.mean(w), nmean, nmax, ncut) print("cut fraction before", float(self.cut.sum()) / self.cut.size) self.cut *= cut_dummy print("cut fraction after", float(self.cut.sum()) / self.cut.size)
def __init__(self, entry, subdets=None, d=None, verbose=False, dark=False): self.fields = ["gain","mce_filter","tags","polangle","tconst","hwp","cut","point_offsets","boresight","site","tod_shape","array_info","beam","pointsrcs", "buddies"] if dark: self.fields += ["dark"] if config.get("noise_model") == "file": self.fields += ["noise"] else: if config.get("cut_noise_whiteness"): self.fields += ["noise_cut"] if config.get("cut_spikes"): self.fields += ["spikes"] if d is None: d = actdata.read(entry, self.fields, verbose=verbose) d = actdata.calibrate(d, verbose=verbose) if subdets is not None: d.restrict(dets=d.dets[subdets]) if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("No data in scan") ndet = d.ndet # Necessary components for Scan interface self.mjd0 = utils.ctime2mjd(d.boresight[0,0]) self.boresight = np.ascontiguousarray(d.boresight.T.copy()) # [nsamp,{t,az,el}] self.boresight[:,0] -= self.boresight[0,0] self.offsets = np.zeros([ndet,self.boresight.shape[1]]) self.offsets[:,1:] = d.point_offset self.cut = d.cut.copy() self.cut_noiseest = d.cut_noiseest.copy() self.comps = np.zeros([ndet,4]) self.beam = d.beam self.pointsrcs = d.pointsrcs self.comps = d.det_comps self.hwp = d.hwp self.hwp_phase = d.hwp_phase self.dets = d.dets self.dgrid = (d.array_info.nrow, d.array_info.ncol) self.array_info = d.array_info self.sys = config.get("tod_sys") self.site = d.site self.speed = d.speed if "noise" in d: self.noise = d.noise else: spikes = d.spikes[:2].T if "spikes" in d else None self.noise = nmat_measure.NmatBuildDelayed(model = config.get("noise_model"), spikes=spikes, cut=self.cut_noiseest) if "dark_tod" in d: self.dark_tod = d.dark_tod if "dark_cut" in d: self.dark_cut = d.dark_cut if "buddy_comps" in d: # Expand buddy_offs to {dt,daz,ddec} self.buddy_comps = d.buddy_comps self.buddy_offs = np.concatenate([d.buddy_offs[...,:1]*0,d.buddy_offs],-1) self.autocut = d.autocut if "autocut" in d else [] # Implementation details. d is our DataSet, which we keep around in # because we need it to read tod consistently later. It will *not* # take part in any sample slicing operations, as that might make the # delayed tod read inconsistent with the rest. It could take part in # detector slicing as long as calibrate_tod operates on each detector # independently. This is true now, but would not be so if we did stuff # like common mode subtraction there. On the other hand, not doing this # would prevent slicing before reading from giving any speedup or memory # savings. I don't think allowing this should be a serious problem. self.d = d self.entry = entry def fmt_id(entry): if isinstance(entry, list): return "+".join([fmt_id(e) for e in entry]) else: if entry.tag: return entry.id + ":" + entry.tag else: return entry.id self.id = fmt_id(entry) self.sampslices = [] self.mapping = None # FIXME: debug test if config.get("dummy_cut") > 0: nmax = int(config.get("dummy_cut_len")) # Power law between 1 and nmax, with slope -1. # C(w) = log(w)/log(nmax) # P(w) = w**-1/log(nmax) # w(C) = n**C # Mean: (nmax-1)/log(nmax) nmean = (nmax-1)/np.log(nmax) ncut = int(self.nsamp * config.get("dummy_cut") / nmean) cut_ranges = np.zeros([self.ndet, ncut, 2],int) w = (nmax**np.random.uniform(0, 1, size=[self.ndet, ncut])).astype(int) np.clip(w, 1, nmax) cut_ranges[:,:,0] = np.random.uniform(0, self.nsamp, size=[self.ndet, ncut]).astype(int) cut_ranges[:,:,0] = np.sort(cut_ranges[:,:,0],1) cut_ranges[:,:,1] = cut_ranges[:,:,0] + w np.clip(cut_ranges[:,:,1], 0, self.nsamp) cut_dummy = sampcut.from_list(cut_ranges, self.nsamp) print np.mean(w), nmean, nmax, ncut print "cut fraction before", float(self.cut.sum())/self.cut.size self.cut *= cut_dummy print "cut fraction after", float(self.cut.sum())/self.cut.size
# Get the scan el and az bounds az1 = np.min(d.boresight[1]) az2 = np.max(d.boresight[1]) el = np.mean(d.boresight[2]) if not valid_az_range(az1, az2): L.debug("Skipped %s (%s)" % (id, "Azimuth crosses poles")) continue # Then get the ra block we live in. This is set by the lowest RA- # detector at the lowest az of the scan at the earliest time in # the scan. So transform all the detectors. ipoint = np.zeros([2, d.ndet]) ipoint[0] = az1 + d.point_offset[:, 0] ipoint[1] = el + d.point_offset[:, 1] mjd = utils.ctime2mjd(d.boresight[0, 0]) opoint = coordinates.transform("hor", "cel", ipoint, time=mjd, site=d.site) ra1 = np.min(opoint[0]) wid = tagger.build(az1, az2, el, ra1) print "%s %s" % (id, wid) sys.stdout.flush() elif command == "build": # Given a list of id tag, loop over tags, and project tods on # a work space per tag. parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc") parser.add_argument("command")
["point_offsets", "boresight", "site", "array_info"]) d = actdata.calibrate(d, exclude=["autocut"]) except (errors.DataMissing, AttributeError) as e: print "Skipping %s (%s)" % (id, e.message) continue # Build a projector between samples and mask. This # requires us to massage d into scan form. It's getting # annoying that scan and data objects aren't compatible. bore = d.boresight.T.copy() bore[:, 0] -= bore[0, 0] scan = enscan.Scan( boresight=bore, offsets=np.concatenate([np.zeros(d.ndet)[:, None], d.point_offset], 1), comps=np.concatenate([np.ones(d.ndet)[:, None], np.zeros((d.ndet, 3))], 1), mjd0=utils.ctime2mjd(d.boresight[0, 0]), sys="hor", site=d.site) scan.hwp_phase = np.zeros([len(bore), 2]) bore_box = np.array([np.min(d.boresight, 1), np.max(d.boresight, 1)]) bore_corners = utils.box2corners(bore_box) scan.entry = d.entry # Is the source above the horizon? If not, it doesn't matter how close # it is. mjd = utils.ctime2mjd( utils.mjd2ctime(scan.mjd0) + scan.boresight[::100, 0]) object_pos = coordinates.interpol_pos("cel", "hor", args.objname, mjd, site=scan.site)
def build_workspace_geometry(wid, bore, point_offset, global_wcs, site=None, tagger=None, padding=100, max_ra_width=2.5 * utils.degree, ncomp=3, dtype=np.float64): if tagger is None: tagger = WorkspaceTagger() if isinstance(wid, basestring): wid = tagger.analyze(wid) if not valid_az_range(wid[0], wid[1]): raise WorkspaceError("Azimuth crosses north/south") trans = TransformPos2Pospix(global_wcs, site=site) az1, az2, el, ra1 = wid # Extract the workspace definition of the tag name ra_ref = ra1 + tagger.ra_step / 2 # We want ra(dec) for up- and down-sweeps for the middle of # the workspace. First find a t that will result in a sweep # that crosses through the middle of the workspace. foc_offset = np.mean(point_offset, 0) t0 = utils.ctime2mjd(bore[0, 0]) t_ref = find_t_giving_ra(az1 + foc_offset[0], el + foc_offset[1], ra_ref, site=site, t0=t0) # We also need the corners of the full workspace area. t1 = find_t_giving_ra(az1 + foc_offset[0], el + foc_offset[1], ra1, site=site, t0=t0) t2 = find_t_giving_ra(az1 + foc_offset[0], el + foc_offset[1], ra1 + tagger.ra_step + max_ra_width, site=site, t0=t0) #print "t1", t1, "t2", t2 #print "az1", az1/utils.degree, "az2", az2/utils.degree #print "ra", ra1/utils.degree, (ra1+tagger.ra_step+max_ra_width)/utils.degree bore_box_hor = np.array([[t1, az1, el], [t2, az2, el]]) bore_corners_hor = utils.box2corners(bore_box_hor) work_corners_hor = bore_corners_hor[None, :, :] + ( point_offset[:, [0, 0, 1]] * [0, 1, 1])[:, None, :] work_corners_hor = work_corners_hor.T.reshape(3, -1) work_corners = trans(work_corners_hor[1:], time=work_corners_hor[0]) ixcorn, iycorn = np.round(work_corners[2:]).astype(int) iybox = np.array([np.min(iycorn) - padding, np.max(iycorn) + 1 + padding]) # Generate an up and down sweep srate = get_srate(bore[0]) period = pmat.get_scan_period(bore[1], srate) dmjd = period / 2. / 24 / 3600 xshifts = [] yshifts = [] work_dazs = [] nwxs, nwys = [], [] for si, (afrom, ato) in enumerate([[az1, az2], [az2, az1]]): sweep = generate_sweep_by_dec_pix( [[t_ref, afrom + foc_offset[0], el + foc_offset[1]], [t_ref + dmjd, ato + foc_offset[0], el + foc_offset[1]]], iybox, trans) # Get the shift in ra pix per dec pix. At this point, # the shifts are just relative to the lowest-dec pixel xshift = np.round(sweep[5] - sweep[5, 0, None]).astype(int) # Get the shift in dec pix per dec pix. These tell us where # each working pixel starts as a function of normal dec pixel. # For example [0,1,3,6] would mean that the work to normal pixel # mapping is [0,1,1,2,2,2]. This is done to make dwdec/daz approximately # constant daz = np.abs(sweep[1, 1:] - sweep[1, :-1]) daz_ratio = np.maximum(1, daz / np.min(daz[1:-1])) yshift = np.round(utils.cumsum(daz_ratio, endpoint=True)).astype(int) yshift -= yshift[0] # Now that we have the x and y mapping, we can determine the # bounds of our workspace by transforming the corners of our # input coordinates. #print "iyc", iycorn-iybox[0] #print "ixc", ixcorn #for i in np.argsort(iycorn): # print "A %6d %6d" % (iycorn[i], ixcorn[i]) #print "min(ixc)", np.min(ixcorn) #print "max(ixc)", np.max(ixcorn) #print "xshift", xshift[iycorn-iybox[0]] wycorn = ixcorn - xshift[iycorn - iybox[0]] #print "wycorn", wycorn #print "min(wyc)", np.min(wycorn) #print "max(wyc)", np.max(wycorn) # Modify the shifts so that any scan in this workspace is always transformed # to valid positions. wx and wy are transposed relative to x and y. # Padding is needed because of the rounding involved in recovering the # az and el from the wid. xshift += np.min(wycorn) xshift -= padding wycorn2 = ixcorn - xshift[iycorn - iybox[0]] #print "wycorn2", wycorn2 #print "min(wyc2)", np.min(wycorn2) #print "max(wyc2)", np.max(wycorn2) #sys.stdout.flush() nwy = np.max(wycorn) - np.min(wycorn) + 1 + 2 * padding nwx = yshift[-1] + 1 # Get the average azimuth spacing in wx work_daz = (sweep[1, -1] - sweep[1, 0]) / (yshift[-1] - yshift[0]) print work_daz / utils.degree # And collect so we can pass them to the Workspace construtor later xshifts.append(xshift) yshifts.append(yshift) nwxs.append(nwx) nwys.append(nwy) work_dazs.append(work_daz) # The shifts from each sweep are guaranteed to have the same length, # since they are based on the same iybox. nwx = np.max(nwxs) # To translate the noise properties, we need a mapping from the x and t # fourier spaces. For this we need the azimuth scanning speed. scan_speed = 2 * (az2 - az1) / period work_daz = np.mean(work_dazs) wgeo = WorkspaceGeometry(nwys, nwx, xshifts, yshifts, iybox[0], scan_speed, work_daz, global_wcs, ncomp=ncomp, dtype=dtype) return wgeo
# Get the scan el and az bounds az1 = np.min(d.boresight[1]) az2 = np.max(d.boresight[1]) el = np.mean(d.boresight[2]) if not valid_az_range(az1, az2): L.debug("Skipped %s (%s)" % (id, "Azimuth crosses poles")) continue # Then get the ra block we live in. This is set by the lowest RA- # detector at the lowest az of the scan at the earliest time in # the scan. So transform all the detectors. ipoint = np.zeros([2, d.ndet]) ipoint[0] = az1 + d.point_offset[:,0] ipoint[1] = el + d.point_offset[:,1] mjd = utils.ctime2mjd(d.boresight[0,0]) opoint = coordinates.transform("hor","cel",ipoint,time=mjd,site=d.site) ra1 = np.min(opoint[0]) wid = tagger.build(az1,az2,el,ra1) print "%s %s" % (id, wid) sys.stdout.flush() elif command == "build": # Given a list of id tag, loop over tags, and project tods on # a work space per tag. parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc") parser.add_argument("command") parser.add_argument("todtags") parser.add_argument("odir") args = parser.parse_args() filedb.init()
try: d = data.read(entry, fields=["gain","cut","point_offsets","boresight","site"]) d = data.calibrate(d) except (zipfile.BadZipfile, errors.DataMissing) as e: print "#%s error: %s" % (id,e.message) #print "%s %8.3f %7.3f %8.3f %7.3f %s" % (id, np.nan, np.nan, np.nan, np.nan, "nodata") continue hour = info[ind].fields.hour tags = sorted(list(set(info[ind].tags)-set([id]))) # Get input pointing bore = d.boresight[:,::sstep] offs = d.point_offset.T[:,::dstep] ipoint = np.zeros(bore.shape + offs.shape[1:]) ipoint[0] = utils.ctime2mjd(bore[0,:,None]) ipoint[1:]= bore[1:,:,None]+offs[:,None,:] ipoint = ipoint.reshape(3,-1) iref = np.mean(ipoint[1:],1) # Transform to equ opoint = coordinates.transform("hor","equ", ipoint[1:], time=ipoint[0], site=d.site) oref = np.mean(opoint,1) print "%s %4.1f %8.3f %7.3f %8.3f %7.3f" % (id, hour, iref[0]/utils.degree, iref[1]/utils.degree, oref[0]/utils.degree, oref[1]/utils.degree), # Compute position of each object, and distance to it orect = coordinates.ang2rect(opoint, zenith=False) for obj in objs: objpos = coordinates.ephem_pos(obj, ipoint[0,0]) objrect = coordinates.ang2rect(objpos, zenith=False)
def __call__(self, name): mjd = utils.ctime2mjd(self.data["t"]) mjd[~np.isfinite(mjd)] = 0 pos = coordinates.ephem_pos(name, mjd) pos /= utils.degree return pos
print "%s %9.3f %9.3f %9.3f %9.3f %s" % (id, np.nan, np.nan, np.nan, np.nan, "badzip") continue if bore.shape[0] < 3 or bore.shape[1] < 1: print "%s %9.3f %9.3f %9.3f %9.3f %s" % (id, np.nan, np.nan, np.nan, np.nan, "nobore") continue # Compute mean pointing in hor and equ t = np.median(bore[0, ::10]) hor = np.median(bore[1:, ::10], 1) try: equ = coordinates.transform("hor", "equ", hor[None] * utils.degree, time=utils.ctime2mjd(t[None]), site=site)[0] / utils.degree except AttributeError: equ = np.array([np.nan, np.nan]) hour = t / 3600 % 24 bsub = bore[:, 50::100].copy() bsub = bsub[:, np.any(~np.isnan(bsub), 0)] bsub[0] = utils.ctime2mjd(bsub[0]) bsub[1:3] = coordinates.transform("hor", "equ", bsub[1:3] * utils.degree, time=bsub[0], site=site) # Compute matching object targdb = targets.TargetDB(entry.targets)
f = open(args.odir + "/fits_%03d.txt" % comm.rank, "w") # Iterate over tods for ind in range(comm.rank, len(ids), comm.size): id = ids[ind] oid = id.replace(":","_") # Check if we hit any of the sources. We first make sure # there's no angle wraps in the bounds, and then move the sources # to the same side of the sky. bounds are pretty approximate, so # might not actually hit all these sources if bounds is not None: poly = bounds[:,:,ind]*utils.degree poly[0] = utils.rewind(poly[0],poly[0,0]) # bounds are defined in celestial coordinates. Must convert srcpos for comparison mjd = utils.ctime2mjd(float(id.split(".")[0])) srccel = coordinates.transform(src_sys, "cel", srcpos, time=mjd) srccel[0] = utils.rewind(srccel[0], poly[0,0]) poly = pad_polygon(poly.T, poly_pad).T sids = np.where(utils.point_in_polygon(srccel.T, poly.T))[0] sids = sorted(list(set(sids)&allowed)) else: sids = sorted(list(allowed)) if len(sids) == 0: print("%s has 0 srcs: skipping" % id) continue try: nsrc = len(sids) print("%s has %d srcs: %s" % (id,nsrc,", ".join(["%d (%.1f)" % (i,a) for i,a in zip(sids,amps[sids])]))) except TypeError as e: print("Weird: %s" % e)