def read_large_galaxies(survey, targetwcs): from astrometry.libkd.spherematch import tree_open, tree_search_radec galfn = survey.find_file('large-galaxies') radius = 1. rc, dc = targetwcs.radec_center() kd = tree_open(galfn, 'largegals') I = tree_search_radec(kd, rc, dc, radius) debug(len(I), 'large galaxies within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (rc, dc)) if len(I) == 0: return None # Read only the rows within range. galaxies = fits_table( galfn, rows=I, columns=['ra', 'dec', 'd25', 'mag', 'lslga_id', 'ba', 'pa']) del kd # # D25 is diameter in arcmin galaxies.radius = galaxies.d25 / 2. / 60. # John told me to do this... #galaxies.radius *= 1.2 ...and then John taketh away. galaxies.delete_column('d25') galaxies.rename('lslga_id', 'ref_id') galaxies.ref_cat = np.array(['L2'] * len(galaxies)) galaxies.islargegalaxy = np.ones(len(galaxies), bool) return galaxies
def other_passes(self, tile, tiles): ''' Given tile number *tile*, return the obstatus rows for the other passes on this tile center. Returns: *otherpasses*, table object ''' if tiles != self.tiles: from astrometry.libkd.spherematch import match_radec # Could also use the fact that the passes are offset from each other # by a fixed index (15872 for decam)... # Max separation is about 0.6 degrees for DECam... #### FIXME for Mosaic this is much smaller... and the three passes #### for a tile are not necessarily relatively close to each other. I,J,d = match_radec(tile.ra, tile.dec, tiles.ra, tiles.dec, 1.) # Omit 'tile' itself... K = np.flatnonzero(tiles.tileid[J] != tile.tileid) J = J[K] return tiles[J] if self.tiletree is None: from astrometry.libkd.spherematch import tree_build_radec self.tiletree = tree_build_radec(self.tiles.ra, self.tiles.dec) from astrometry.libkd.spherematch import tree_search_radec J = tree_search_radec(self.tiletree, tile.ra, tile.dec, 1.0) # Omit 'tile' itself... K = np.flatnonzero(tiles.tileid[J] != tile.tileid) J = J[K] return tiles[J]
def get_healpix_rangesearch_catalogs(self, healpixes, rc, dc, rad): cats = [] for hp in healpixes: (kd, tab) = self.get_healpix_tree(hp) I = tree_search_radec(kd, rc, dc, rad) if len(I): cats.append(tab[I]) if len(cats) == 1: return cats[0] #.copy() return merge_tables(cats)
def cat_kd(req, ver, tag, fn): tag = 'spec' ralo = float(req.GET['ralo']) rahi = float(req.GET['rahi']) declo = float(req.GET['declo']) dechi = float(req.GET['dechi']) ver = int(ver) if not ver in catversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) import numpy as np from astrometry.util.fits import fits_table, merge_tables from astrometry.libkd.spherematch import tree_open, tree_search_radec from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between xyz1 = radectoxyz(ralo, declo) xyz2 = radectoxyz(rahi, dechi) xyz = (xyz1 + xyz2) / 2. xyz /= np.sqrt(np.sum(xyz**2)) rc, dc = xyztoradec(xyz) rc = rc[0] dc = dc[0] rad = degrees_between(rc, dc, ralo, declo) kd = tree_open(fn) I = tree_search_radec(kd, rc, dc, rad) print('Matched', len(I), 'from', fn) if len(I) == 0: return None T = fits_table(fn, rows=I) debug(len(T), 'spectra') if ralo > rahi: # RA wrap T.cut( np.logical_or(T.ra > ralo, T.ra < rahi) * (T.dec > declo) * (T.dec < dechi)) else: T.cut( (T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi)) debug(len(T), 'in cut') return T
def bricks_touching_radec_box(self, bricks, ralo, rahi, declo, dechi): ''' Returns an index vector of the bricks that touch the given RA,Dec box. ''' if bricks is None: bricks = self.get_bricks_readonly() if self.cache_tree and bricks == self.bricks: from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec # Use kdtree if self.bricktree is None: self.bricktree = tree_build_radec(bricks.ra, bricks.dec) # brick size radius = np.sqrt(2.)/2. * self.bricksize # + RA,Dec box size radius = radius + degrees_between(ralo, declo, rahi, dechi) / 2. dec = (dechi + declo) / 2. c = (np.cos(np.deg2rad(rahi)) + np.cos(np.deg2rad(ralo))) / 2. s = (np.sin(np.deg2rad(rahi)) + np.sin(np.deg2rad(ralo))) / 2. ra = np.rad2deg(np.arctan2(s, c)) J = tree_search_radec(self.bricktree, ra, dec, radius) I = J[np.nonzero((bricks.ra1[J] <= rahi ) * (bricks.ra2[J] >= ralo) * (bricks.dec1[J] <= dechi) * (bricks.dec2[J] >= declo))[0]] return I if rahi < ralo: # Wrap-around print('In Dec slice:', len(np.flatnonzero((bricks.dec1 <= dechi) * (bricks.dec2 >= declo)))) print('Above RAlo=', ralo, ':', len(np.flatnonzero(bricks.ra2 >= ralo))) print('Below RAhi=', rahi, ':', len(np.flatnonzero(bricks.ra1 <= rahi))) print('In RA slice:', len(np.nonzero(np.logical_or(bricks.ra2 >= ralo, bricks.ra1 <= rahi)))) I, = np.nonzero(np.logical_or(bricks.ra2 >= ralo, bricks.ra1 <= rahi) * (bricks.dec1 <= dechi) * (bricks.dec2 >= declo)) print('In RA&Dec slice', len(I)) else: I, = np.nonzero((bricks.ra1 <= rahi ) * (bricks.ra2 >= ralo) * (bricks.dec1 <= dechi) * (bricks.dec2 >= declo)) return I
def bricks_touching_radec_box(self, bricks, ralo, rahi, declo, dechi): ''' Returns an index vector of the bricks that touch the given RA,Dec box. ''' if bricks is None: bricks = self.get_bricks_readonly() if self.cache_tree and bricks == self.bricks: from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec # Use kdtree if self.bricktree is None: self.bricktree = tree_build_radec(bricks.ra, bricks.dec) # brick size radius = np.sqrt(2.)/2. * self.bricksize # + RA,Dec box size radius = radius + degrees_between(ralo, declo, rahi, dechi) / 2. dec = (dechi + declo) / 2. c = (np.cos(np.deg2rad(rahi)) + np.cos(np.deg2rad(ralo))) / 2. s = (np.sin(np.deg2rad(rahi)) + np.sin(np.deg2rad(ralo))) / 2. ra = np.rad2deg(np.arctan2(s, c)) J = tree_search_radec(self.bricktree, ra, dec, radius) I = J[np.nonzero((bricks.ra1[J] <= rahi ) * (bricks.ra2[J] >= ralo) * (bricks.dec1[J] <= dechi) * (bricks.dec2[J] >= declo))[0]] return I if rahi < ralo: # Wrap-around print('In Dec slice:', len(np.flatnonzero((bricks.dec1 <= dechi) * (bricks.dec2 >= declo)))) print('Above RAlo=', ralo, ':', len(np.flatnonzero(bricks.ra2 >= ralo))) print('Below RAhi=', rahi, ':', len(np.flatnonzero(bricks.ra1 <= rahi))) print('In RA slice:', len(np.nonzero(np.logical_or(bricks.ra2 >= ralo, bricks.ra1 <= rahi)))) I, = np.nonzero(np.logical_or(bricks.ra2 >= ralo, bricks.ra1 <= rahi) * (bricks.dec1 <= dechi) * (bricks.dec2 >= declo)) print('In RA&Dec slice', len(I)) else: I, = np.nonzero((bricks.ra1 <= rahi ) * (bricks.ra2 >= ralo) * (bricks.dec1 <= dechi) * (bricks.dec2 >= declo)) return I
def cat_kd(req, ver, tag, fn): tag = 'spec' ralo = float(req.GET['ralo']) rahi = float(req.GET['rahi']) declo = float(req.GET['declo']) dechi = float(req.GET['dechi']) ver = int(ver) if not ver in catversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) import numpy as np from astrometry.util.fits import fits_table, merge_tables from astrometry.libkd.spherematch import tree_open, tree_search_radec from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between xyz1 = radectoxyz(ralo, declo) xyz2 = radectoxyz(rahi, dechi) xyz = (xyz1 + xyz2)/2. xyz /= np.sqrt(np.sum(xyz**2)) rc,dc = xyztoradec(xyz) rc = rc[0] dc = dc[0] rad = degrees_between(rc, dc, ralo, declo) kd = tree_open(fn) I = tree_search_radec(kd, rc, dc, rad) print('Matched', len(I), 'from', fn) if len(I) == 0: return None T = fits_table(fn, rows=I) debug(len(T), 'spectra') if ralo > rahi: # RA wrap T.cut(np.logical_or(T.ra > ralo, T.ra < rahi) * (T.dec > declo) * (T.dec < dechi)) else: T.cut((T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi)) debug(len(T), 'in cut') return T
def cat_targets_drAB(req, ver, cats=[], tag='', bgs=False, sky=False, bright=False, dark=False, color_name_func=desitarget_color_names): ''' color_name_func: function that selects names and colors for targets (eg based on targeting bit values) ''' import json ralo = float(req.GET['ralo']) rahi = float(req.GET['rahi']) declo = float(req.GET['declo']) dechi = float(req.GET['dechi']) ver = int(ver) if not ver in catversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) from astrometry.util.fits import fits_table, merge_tables from astrometry.libkd.spherematch import tree_open, tree_search_radec import numpy as np from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between xyz1 = radectoxyz(ralo, declo) xyz2 = radectoxyz(rahi, dechi) xyz = (xyz1 + xyz2) / 2. xyz /= np.sqrt(np.sum(xyz**2)) rc, dc = xyztoradec(xyz) rc = rc[0] dc = dc[0] rad = degrees_between(rc, dc, ralo, declo) ''' startree -i /project/projectdirs/desi/target/catalogs/targets-dr4-0.20.0.fits -o data/targets-dr4-0.20.0.kd.fits -P -k -T ''' TT = [] for fn in cats: kd = tree_open(fn) I = tree_search_radec(kd, rc, dc, rad) print('Matched', len(I), 'from', fn) if len(I) == 0: continue T = fits_table(fn, rows=I) TT.append(T) if len(TT) == 0: return HttpResponse(json.dumps(dict(rd=[], name=[])), content_type='application/json') T = merge_tables(TT, columns='fillzero') if bgs: T.cut(T.bgs_target > 0) if bright: T.cut(np.logical_or(T.bgs_target > 0, T.mws_target > 0)) if dark: T.cut(T.desi_target > 0) names = None colors = None if color_name_func is not None: names, colors = color_name_func(T) if sky: fluxes = [ dict(g=float(g), r=float(r), z=float(z)) for (g, r, z) in zip(T.apflux_g[:, 0], T.apflux_r[:, 0], T.apflux_z[:, 0]) ] nobs = None else: fluxes = [ dict(g=float(g), r=float(r), z=float(z), W1=float(W1), W2=float(W2)) for (g, r, z, W1, W2) in zip(T.flux_g, T.flux_r, T.flux_z, T.flux_w1, T.flux_w2) ] nobs = [ dict(g=int(g), r=int(r), z=int(z)) for g, r, z in zip(T.nobs_g, T.nobs_r, T.nobs_z) ], rtn = dict( rd=[(t.ra, t.dec) for t in T], targetid=[int(t) for t in T.targetid], fluxes=fluxes, ) if names is not None: rtn.update(name=names) if colors is not None: rtn.update(color=colors) if nobs is not None: rtn.update(nobs=nobs) return HttpResponse(json.dumps(rtn), content_type='application/json')
def cat_targets_drAB(req, ver, cats=None, tag='', bgs=False, sky=False, bright=False, dark=False, color_name_func=desitarget_color_names): ''' color_name_func: function that selects names and colors for targets (eg based on targeting bit values) ''' if cats is None: cats = [] import json ralo = float(req.GET['ralo']) rahi = float(req.GET['rahi']) declo = float(req.GET['declo']) dechi = float(req.GET['dechi']) ver = int(ver) if not ver in catversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) from astrometry.util.fits import fits_table, merge_tables from astrometry.libkd.spherematch import tree_open, tree_search_radec import numpy as np from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between xyz1 = radectoxyz(ralo, declo) xyz2 = radectoxyz(rahi, dechi) xyz = (xyz1 + xyz2)/2. xyz /= np.sqrt(np.sum(xyz**2)) rc,dc = xyztoradec(xyz) rc = rc[0] dc = dc[0] rad = degrees_between(rc, dc, ralo, declo) ''' startree -i /project/projectdirs/desi/target/catalogs/targets-dr4-0.20.0.fits -o data/targets-dr4-0.20.0.kd.fits -P -k -T ''' TT = [] for fn in cats: kd = tree_open(fn) I = tree_search_radec(kd, rc, dc, rad) print('Matched', len(I), 'from', fn) if len(I) == 0: continue T = fits_table(fn, rows=I) TT.append(T) if len(TT) == 0: return HttpResponse(json.dumps(dict(rd=[], name=[])), content_type='application/json') T = merge_tables(TT, columns='fillzero') if bgs: T.cut(T.bgs_target > 0) if bright: T.cut(np.logical_or(T.bgs_target > 0, T.mws_target > 0)) if dark: T.cut(T.desi_target > 0) names = None colors = None if color_name_func is not None: names,colors = color_name_func(T) if sky: fluxes = [dict(g=float(g), r=float(r), z=float(z)) for (g,r,z) in zip(T.apflux_g[:,0], T.apflux_r[:,0], T.apflux_z[:,0])] nobs = None else: fluxes = [dict(g=float(g), r=float(r), z=float(z), W1=float(W1), W2=float(W2)) for (g,r,z,W1,W2) in zip(T.flux_g, T.flux_r, T.flux_z, T.flux_w1, T.flux_w2)] nobs=[dict(g=int(g), r=int(r), z=int(z)) for g,r,z in zip(T.nobs_g, T.nobs_r, T.nobs_z)], rtn = dict(rd=[(t.ra, t.dec) for t in T], targetid=[int(t) for t in T.targetid], fluxes=fluxes, ) if names is not None: rtn.update(name=names) if colors is not None: rtn.update(color=colors) if nobs is not None: rtn.update(nobs=nobs) # Convert targetid to string to prevent rounding errors rtn['targetid'] = [str(s) for s in rtn['targetid']] return HttpResponse(json.dumps(rtn), content_type='application/json')
def main(): import argparse parser = argparse.ArgumentParser( description='This script creates small self-contained data sets that ' 'are useful for test cases of the pipeline codes.') parser.add_argument('ccds', help='CCDs table describing region to grab') parser.add_argument('outdir', help='Output directory name') parser.add_argument('brick', help='Brick containing these images') parser.add_argument('--survey-dir', type=str, default=None) parser.add_argument('--cache-dir', type=str, default=None, help='Directory to search for cached files') parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image') parser.add_argument('--wise-wcs-hdu', help='For WISE outputs, the HDU to read the WCS from in the file given by --wise.', type=int, default=0) parser.add_argument('--fpack', action='store_true', default=False) parser.add_argument('--gzip', action='store_true', default=False) parser.add_argument('--pad', action='store_true', default=False, help='Keep original image size, but zero out pixels outside ROI') args = parser.parse_args() v = 'SKY_TEMPLATE_DIR' if v in os.environ: del os.environ[v] C = fits_table(args.ccds) print(len(C), 'CCDs in', args.ccds) C.camera = np.array([c.strip() for c in C.camera]) survey = LegacySurveyData(cache_dir=args.cache_dir, survey_dir=args.survey_dir) if ',' in args.brick: ra,dec = args.brick.split(',') ra = float(ra) dec = float(dec) fakebricks = fits_table() fakebricks.brickname = np.array([('custom-%06i%s%05i' % (int(1000*ra), 'm' if dec < 0 else 'p', int(1000*np.abs(dec))))]) fakebricks.ra = np.array([ra]) fakebricks.dec = np.array([dec]) bricks = fakebricks outbricks = bricks else: bricks = survey.get_bricks_readonly() outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])] assert(len(outbricks) == 1) outsurvey = LegacySurveyData(survey_dir = args.outdir) trymakedirs(args.outdir) outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz')) targetwcs = wcs_for_brick(outbricks[0]) H,W = targetwcs.shape tycho2fn = survey.find_file('tycho2') kd = tree_open(tycho2fn, 'stars') radius = 1. rc,dc = targetwcs.radec_center() I = tree_search_radec(kd, rc, dc, radius) print(len(I), 'Tycho-2 stars within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (rc,dc)) # Read only the rows within range. tycho = fits_table(tycho2fn, rows=I) del kd print('Read', len(tycho), 'Tycho-2 stars') ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec) #margin = 100 #tycho.cut(ok * (tx > -margin) * (tx < W+margin) * # (ty > -margin) * (ty < H+margin)) print('Cut to', len(tycho), 'Tycho-2 stars within brick') del ok,tx,ty #tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz')) f,tfn = tempfile.mkstemp(suffix='.fits') os.close(f) tycho.writeto(tfn) outfn = os.path.join(args.outdir, 'tycho2.kd.fits') cmd = 'startree -i %s -o %s -P -k -n stars -T' % (tfn, outfn) print(cmd) rtn = os.system(cmd) assert(rtn == 0) os.unlink(tfn) from legacypipe.gaiacat import GaiaCatalog gcat = GaiaCatalog() # from ps1cat.py: wcs = targetwcs step=100. margin=10. # Grid the CCD in pixel space W,H = wcs.get_width(), wcs.get_height() xx,yy = np.meshgrid( np.linspace(1-margin, W+margin, 2+int((W+2*margin)/step)), np.linspace(1-margin, H+margin, 2+int((H+2*margin)/step))) # Convert to RA,Dec and then to unique healpixes ra,dec = wcs.pixelxy2radec(xx.ravel(), yy.ravel()) healpixes = set() for r,d in zip(ra,dec): healpixes.add(gcat.healpix_for_radec(r, d)) for hp in healpixes: hpcat = gcat.get_healpix_catalog(hp) ok,xx,yy = wcs.radec2pixelxy(hpcat.ra, hpcat.dec) onccd = np.flatnonzero((xx >= 1.-margin) * (xx <= W+margin) * (yy >= 1.-margin) * (yy <= H+margin)) hpcat.cut(onccd) if len(hpcat): outfn = os.path.join(args.outdir, 'gaia', 'chunk-%05d.fits' % hp) trymakedirs(os.path.join(args.outdir, 'gaia')) hpcat.writeto(outfn) outccds = C.copy() cols = outccds.get_columns() for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1', 'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1', 'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver', 'psfplver' ]: if c in cols: outccds.delete_column(c) outccds.image_hdu[:] = 1 # Convert to list to avoid truncating filenames outccds.image_filename = [fn for fn in outccds.image_filename] for iccd,ccd in enumerate(C): decam = (ccd.camera.strip() == 'decam') bok = (ccd.camera.strip() == '90prime') im = survey.get_image_object(ccd) print('Got', im) if survey.cache_dir is not None: im.check_for_cached_files(survey) slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1)) psfkwargs = dict(pixPsf=True, gaussPsf=False, hybridPsf=False, normalizePsf=False) tim = im.get_tractor_image(slc, pixPsf=True, subsky=False, nanomaggies=False, no_remap_invvar=True, old_calibs_ok=True) print('Tim:', tim.shape) psfrow = psfhdr = None if args.pad: psf = im.read_psf_model(0, 0, w=im.width, h=im.height, **psfkwargs) psfex = psf.psfex else: psf = tim.getPsf() psfex = psf.psfex # Did the PSF model come from a merged file? for fn in [im.merged_psffn, im.psffn] + im.old_merged_psffns: if not os.path.exists(fn): continue T = fits_table(fn) I, = np.nonzero((T.expnum == im.expnum) * np.array([c.strip() == im.ccdname for c in T.ccdname])) if len(I) != 1: continue psfrow = T[I] x0 = ccd.ccd_x0 y0 = ccd.ccd_y0 psfrow.polzero1[0] -= x0 psfrow.polzero2[0] -= y0 #psfhdr = fitsio.read_header(im.merged_psffn) break psfex.fwhm = tim.psf_fwhm #### HACK #psfrow = None assert(psfrow is not None) if psfrow is not None: print('PSF row:', psfrow) #else: # print('PSF:', psf) # print('PsfEx:', psfex) skyrow = skyhdr = None if args.pad: primhdr = fitsio.read_header(im.imgfn) imghdr = fitsio.read_header(im.imgfn, hdu=im.hdu) sky = im.read_sky_model(splinesky=True, primhdr=primhdr, imghdr=imghdr) #skyhdr = fitsio.read_header(im.splineskyfn) #msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True) else: sky = tim.getSky() # Did the sky model come from a merged file? #msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True) print('merged skyfn:', im.merged_skyfn) print('single skyfn:', im.skyfn) print('old merged skyfns:', im.old_merged_skyfns) for fn in [im.merged_skyfn, im.skyfn] + im.old_merged_skyfns: if not os.path.exists(fn): continue T = fits_table(fn) I, = np.nonzero((T.expnum == im.expnum) * np.array([c.strip() == im.ccdname for c in T.ccdname])) skyrow = T[I] skyrow.x0[0] = ccd.ccd_x0 skyrow.y0[0] = ccd.ccd_y0 # s_med = skyrow.sky_med[0] # s_john = skyrow.sky_john[0] # skyhdr = fitsio.read_header(fn) assert(skyrow is not None) ### HACK #skyrow = None if skyrow is not None: print('Sky row:', skyrow) else: print('Sky:', sky) # Output filename format: fn = ccd.image_filename.strip() ccd.image_filename = os.path.join(os.path.dirname(fn), '%s.%s.fits' % (os.path.basename(fn).split('.')[0], ccd.ccdname.strip())) outim = outsurvey.get_image_object(ccd) print('Output image:', outim) print('Image filename:', outim.imgfn) trymakedirs(outim.imgfn, dir=True) imgdata = tim.getImage() ivdata = tim.getInvvar() # Since we remap DQ codes (always with Mosaic and Bok, sometimes with DECam), # re-read from the FITS file rather than using tim.dq. print('Reading data quality from', im.dqfn, 'hdu', im.hdu) dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice) print('Tim shape:', tim.shape, 'Slice', tim.slice) print('image shape:', imgdata.shape, 'iv', ivdata.shape, 'DQ', dqdata.shape) from collections import Counter dqvals = Counter(dqdata.ravel()) print('DQ pixel counts:') for k,n in dqvals.most_common(): print(' 0x%x' % k, ':', n) if args.pad: # Create zero image of full size, copy in data. fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype) fullsize[slc] = imgdata imgdata = fullsize fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype) fullsize[slc] = dqdata dqdata = fullsize fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype) fullsize[slc] = ivdata ivdata = fullsize else: # Adjust the header WCS by x0,y0 crpix1 = tim.hdr['CRPIX1'] crpix2 = tim.hdr['CRPIX2'] tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0 tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0 # Add image extension to filename # fitsio doesn't compress .fz by default, so drop .fz suffix #outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits') if args.gzip: outim.imgfn = outim.imgfn.replace('.fits', '.fits.gz') #outim.wtfn = outim.wtfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.wtfn = outim.wtfn.replace('.fits.fz', '.fits') if args.gzip: outim.wtfn = outim.wtfn.replace('.fits', '.fits.gz') if outim.dqfn is not None: #outim.dqfn = outim.dqfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.dqfn = outim.dqfn.replace('.fits.fz', '.fits') if args.gzip: outim.dqfn = outim.dqfn.replace('.fits', '.fits.gz') if bok: outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname) ccdfn = outim.imgfn ccdfn = ccdfn.replace(outsurvey.get_image_dir(), '') if ccdfn.startswith('/'): ccdfn = ccdfn[1:] outccds.image_filename[iccd] = ccdfn print('Changed output filenames to:') print(outim.imgfn) print(outim.dqfn) ofn = outim.imgfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fits = fitsio.FITS(ofn, 'rw', clobber=True) fits.write(None, header=tim.primhdr) fits.write(imgdata, header=tim.hdr, extname=ccd.ccdname) fits.close() if args.fpack: cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) h,w = tim.shape if not args.pad: outccds.width[iccd] = w outccds.height[iccd] = h outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0 outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0 wcs = Tan(*[float(x) for x in [ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2, ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]]) if args.pad: subwcs = wcs else: subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h) outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center() print('Weight filename:', outim.wtfn) wfn = outim.wtfn trymakedirs(wfn, dir=True) ofn = wfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fits = fitsio.FITS(ofn, 'rw', clobber=True) fits.write(None, header=tim.primhdr) fits.write(ivdata, header=tim.hdr, extname=ccd.ccdname) fits.close() if args.fpack: cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) if outim.dqfn is not None: print('DQ filename', outim.dqfn) trymakedirs(outim.dqfn, dir=True) ofn = outim.dqfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fits = fitsio.FITS(ofn, 'rw', clobber=True) fits.write(None, header=tim.primhdr) fits.write(dqdata, header=tim.hdr, extname=ccd.ccdname) fits.close() if args.fpack: cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) psfout = outim.psffn #if psfrow: # psfout = outim.merged_psffn print('PSF output filename:', psfout) trymakedirs(psfout, dir=True) if psfrow: psfrow.writeto(psfout, primhdr=psfhdr) else: print('Writing PsfEx:', psfout) psfex.writeto(psfout) # update header F = fitsio.FITS(psfout, 'rw') F[0].write_keys([dict(name='EXPNUM', value=ccd.expnum), dict(name='PLVER', value=psf.plver), dict(name='PROCDATE', value=psf.procdate), dict(name='PLPROCID', value=psf.plprocid),]) F.close() skyout = outim.skyfn #if skyrow: # skyout = outim.merged_splineskyfn print('Sky output filename:', skyout) trymakedirs(skyout, dir=True) if skyrow is not None: skyrow.writeto(skyout, primhdr=skyhdr) else: primhdr = fitsio.FITSHDR() primhdr['PLVER'] = sky.plver primhdr['PLPROCID'] = sky.plprocid primhdr['PROCDATE'] = sky.procdate primhdr['EXPNUM'] = ccd.expnum primhdr['IMGDSUM'] = sky.datasum primhdr['S_MED'] = s_med primhdr['S_JOHN'] = s_john sky.write_fits(skyout, primhdr=primhdr) # HACK -- check result immediately. outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz')) outsurvey.ccds = None outC = outsurvey.get_ccds_readonly() occd = outC[iccd] outim = outsurvey.get_image_object(occd) print('Got output image:', outim) otim = outim.get_tractor_image(pixPsf=True, hybridPsf=True, old_calibs_ok=True) print('Got output tim:', otim) outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz')) # WISE if args.wise is not None: from wise.forcedphot import unwise_tiles_touching_wcs from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs, get_unwise_tractor_image, get_unwise_tile_dir) # Read WCS... print('Reading TAN wcs header from', args.wise, 'HDU', args.wise_wcs_hdu) targetwcs = Tan(args.wise, args.wise_wcs_hdu) tiles = unwise_tiles_touching_wcs(targetwcs) print('Cut to', len(tiles), 'unWISE tiles') H,W = targetwcs.shape r,d = targetwcs.pixelxy2radec(np.array([1, W, W/2, W/2]), np.array([H/2, H/2, 1, H ])) roiradec = [r[0], r[1], d[2], d[3]] unwise_dir = os.environ['UNWISE_COADDS_DIR'] wise_out = os.path.join(args.outdir, 'images', 'unwise') print('Will write WISE outputs to', wise_out) unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR'] wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr') print('Will write WISE time-resolved outputs to', wise_tr_out) trymakedirs(wise_tr_out) W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_atlas.fits')) print('Read', len(W), 'time-resolved WISE coadd tiles') W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id])) print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth') # Write the time-resolved index subset. W.writeto(os.path.join(wise_tr_out, 'time_resolved_atlas.fits')) # this ought to be enough for anyone =) _,Nepochs = W.epoch_bitmask.shape print('N epochs in time-resolved atlas:', Nepochs) wisedata = [] # full depth for band in [1,2,3,4]: wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band, True)) # time-resolved for band in [1,2]: # W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2) bitmask = (1 << (band-1)) for e in range(Nepochs): # Which tiles have images for this epoch? I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask) if len(I) == 0: continue print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I]) edir = os.path.join(unwise_tr_dir, 'e%03i' % e) eoutdir = os.path.join(wise_tr_out, 'e%03i' % e) wisedata.append((edir, eoutdir, tiles.coadd_id[I], band, False)) wrote_masks = set() model_dir = os.environ.get('UNWISE_MODEL_SKY_DIR') if model_dir is not None: model_dir_out = os.path.join(args.outdir, 'images', 'unwise-mod') trymakedirs(model_dir_out) for indir, outdir, tiles, band, fulldepth in wisedata: for tile in tiles: wanyband = 'w' tim = get_unwise_tractor_image(indir, tile, band, bandname=wanyband, roiradecbox=roiradec) print('Got unWISE tim', tim) print(tim.shape) if model_dir is not None and fulldepth and band in [1,2]: print('ROI', tim.roi) #0387p575.1.mod.fits fn = '%s.%i.mod.fits' % (tile, band) print('Filename', fn) F = fitsio.FITS(os.path.join(model_dir, fn)) x0,x1,y0,y1 = tim.roi slc = slice(y0,y1),slice(x0,x1) phdr = F[0].read_header() outfn = os.path.join(model_dir_out, fn) for e,extname in [(1,'MODEL'), (2,'SKY')]: pix = F[e][slc] hdr = F[e].read_header() crpix1 = hdr['CRPIX1'] crpix2 = hdr['CRPIX2'] hdr['CRPIX1'] -= x0 hdr['CRPIX2'] -= y0 #print('mod', mod) #print('Model', mod.shape) if e == 1: fitsio.write(outfn, None, clobber=True, header=phdr) fitsio.write(outfn, pix, header=hdr, extname=extname) print('Wrote', outfn) thisdir = get_unwise_tile_dir(outdir, tile) print('Directory for this WISE tile:', thisdir) base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band)) print('Base filename:', base) masked = True mu = 'm' if masked else 'u' imfn = base + 'img-%s.fits' % mu ivfn = base + 'invvar-%s.fits.gz' % mu nifn = base + 'n-%s.fits.gz' % mu nufn = base + 'n-u.fits.gz' #print('WISE image header:', tim.hdr) # Adjust the header WCS by x0,y0 wcs = tim.wcs.wcs tim.hdr['CRPIX1'] = wcs.crpix[0] tim.hdr['CRPIX2'] = wcs.crpix[1] H,W = tim.shape tim.hdr['IMAGEW'] = W tim.hdr['IMAGEH'] = H print('WCS:', wcs) print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2']) trymakedirs(imfn, dir=True) fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True) print('Wrote', imfn) fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True) print('Wrote', ivfn) fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True) print('Wrote', nifn) fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True) print('Wrote', nufn) if not (indir,tile) in wrote_masks: print('Looking for mask file for', indir, tile) # record that we tried this dir/tile combo wrote_masks.add((indir,tile)) for idir in indir.split(':'): tdir = get_unwise_tile_dir(idir, tile) maskfn = 'unwise-%s-msk.fits.gz' % tile fn = os.path.join(tdir, maskfn) print('Mask file:', fn) if os.path.exists(fn): print('Reading', fn) (x0,x1,y0,y1) = tim.roi roislice = (slice(y0,y1), slice(x0,x1)) F = fitsio.FITS(fn)[0] hdr = F.read_header() M = F[roislice] outfn = os.path.join(thisdir, maskfn) fitsio.write(outfn, M, header=tim.hdr, clobber=True) print('Wrote', outfn) break outC = outsurvey.get_ccds_readonly() for iccd,ccd in enumerate(outC): outim = outsurvey.get_image_object(ccd) print('Got output image:', outim) otim = outim.get_tractor_image(pixPsf=True, hybridPsf=True, old_calibs_ok=True) print('Got output tim:', otim)
print('Kd data:', spherematch.tree_data(kd, np.array([0,3,5]).astype(np.uint32))) print('Kd data:', kd.get_data(np.array([0,3,5]).astype(np.uint32))) print('Permute:', spherematch.tree_permute(kd, np.array([3,5,7]).astype(np.int32))) print('Permute:', kd.permute(np.array([0,99,199]).astype(np.int32))) ra,dec = np.meshgrid(np.arange(0, 360), np.arange(-90, 91, 1)) ra1 = ra.ravel() dec1 = dec.ravel() rdkd1 = spherematch.tree_build_radec(ra1, dec1) print('RdKd:', rdkd1.n, rdkd1.bbox) ra2 = np.random.uniform(-10, 10, size=1000) dec2 = np.random.uniform(-10, 10, size=1000) rdkd2 = spherematch.tree_build_radec(ra2, dec2) I = spherematch.tree_search_radec(rdkd1, ra2[0], dec2[0], 2.) print('search_radec:', I) I,J,d = spherematch.match_radec(ra1, dec1, ra2, dec2, 1.) print('Matches:', len(I)) I,J,d = spherematch.match_radec(ra1, dec1, ra2, dec2, 1., nearest=True) print('Nearest matches:', len(I)) I = spherematch.match_radec(ra1, dec1, ra2, dec2, 1., indexlist=True) print('Index lists matches:', len(I))
def map_decam_depth(req, ver, zoom, x, y, savecache=False, band=None, ignoreCached=False): global Tdepth global Tdepthkd if band is None: band = req.GET.get('band') if not band in ['g', 'r', 'z']: raise RuntimeError('Invalid band') tag = 'decam-depth-%s' % band zoom = int(zoom) zoomscale = 2.**zoom x = int(x) y = int(y) if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale: raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom, x, y)) ver = int(ver) if not ver in tileversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) basedir = settings.DATA_DIR tilefn = os.path.join(basedir, 'tiles', tag, '%i/%i/%i/%i.jpg' % (ver, zoom, x, y)) if os.path.exists(tilefn) and not ignoreCached: print('Cached:', tilefn) return send_file(tilefn, 'image/jpeg', expires=oneyear, modsince=req.META.get('HTTP_IF_MODIFIED_SINCE')) from astrometry.util.util import Tan from astrometry.libkd.spherematch import match_radec from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec from astrometry.util.fits import fits_table from astrometry.util.starutil_numpy import degrees_between import numpy as np import fitsio try: wcs, W, H, zoomscale, zoom, x, y = get_tile_wcs(zoom, x, y) except RuntimeError as e: return HttpResponse(e.strerror) rlo, d = wcs.pixelxy2radec(W, H / 2)[-2:] rhi, d = wcs.pixelxy2radec(1, H / 2)[-2:] r, d1 = wcs.pixelxy2radec(W / 2, 1)[-2:] r, d2 = wcs.pixelxy2radec(W / 2, H)[-2:] r, d = wcs.pixelxy2radec(W / 2, H / 2)[-2:] rad = max(degrees_between(r, d, rlo, d1), degrees_between(r, d, rhi, d2)) if Tdepth is None: T = fits_table(os.path.join(basedir, 'decals-zpt-nondecals.fits'), columns=[ 'ccdra', 'ccddec', 'arawgain', 'avsky', 'ccdzpt', 'filter', 'crpix1', 'crpix2', 'crval1', 'crval2', 'cd1_1', 'cd1_2', 'cd2_1', 'cd2_2', 'naxis1', 'naxis2', 'exptime', 'fwhm' ]) T.rename('ccdra', 'ra') T.rename('ccddec', 'dec') Tdepth = {} Tdepthkd = {} for b in ['g', 'r', 'z']: Tdepth[b] = T[T.filter == b] Tdepthkd[b] = tree_build_radec(Tdepth[b].ra, Tdepth[b].dec) T = Tdepth[band] Tkd = Tdepthkd[band] #I,J,d = match_radec(T.ra, T.dec, r, d, rad + 0.2) I = tree_search_radec(Tkd, r, d, rad + 0.2) print(len(I), 'CCDs in range') if len(I) == 0: from django.http import HttpResponseRedirect return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg') depthiv = np.zeros((H, W), np.float32) for t in T[I]: twcs = Tan(*[ float(x) for x in [ t.crval1, t.crval2, t.crpix1, t.crpix2, t.cd1_1, t.cd1_2, t.cd2_1, t.cd2_2, t.naxis1, t.naxis2 ] ]) w, h = t.naxis1, t.naxis2 r, d = twcs.pixelxy2radec([1, 1, w, w], [1, h, h, 1]) ok, x, y = wcs.radec2pixelxy(r, d) #print 'x,y coords of CCD:', x, y x0 = int(x.min()) x1 = int(x.max()) y0 = int(y.min()) y1 = int(y.max()) if y1 < 0 or x1 < 0 or x0 >= W or y0 >= H: continue readnoise = 10. # e-; 7.0 to 15.0 according to DECam Data Handbook skysig = np.sqrt(t.avsky * t.arawgain + readnoise**2) / t.arawgain zpscale = 10.**((t.ccdzpt - 22.5) / 2.5) * t.exptime sig1 = skysig / zpscale psf_sigma = t.fwhm / 2.35 # point-source depth psfnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma) detsig1 = sig1 / psfnorm #print '5-sigma point-source depth:', NanoMaggies.nanomaggiesToMag(detsig1 * 5.) div = 1 / detsig1**2 depthiv[max(y0, 0):min(y1, H), max(x0, 0):min(x1, W)] += div ptsrc = -2.5 * (np.log10(np.sqrt(1. / depthiv) * 5) - 9) ptsrc[depthiv == 0] = 0. if savecache: trymakedirs(tilefn) else: import tempfile f, tilefn = tempfile.mkstemp(suffix='.jpg') os.close(f) import pylab as plt plt.imsave(tilefn, ptsrc, vmin=22., vmax=25., cmap='hot') #nipy_spectral') return send_file(tilefn, 'image/jpeg', unlink=(not savecache))
def map_sdss(req, ver, zoom, x, y, savecache=None, tag='sdss', get_images=False, ignoreCached=False, wcs=None, forcecache=False, forcescale=None, bestOnly=False, bands='gri', **kwargs): from decals import settings if savecache is None: savecache = settings.SAVE_CACHE zoom = int(zoom) zoomscale = 2.**zoom x = int(x) y = int(y) if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale: raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom, x, y)) ver = int(ver) if not ver in tileversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) basedir = settings.DATA_DIR tilefn = os.path.join(basedir, 'tiles', tag, '%i/%i/%i/%i.jpg' % (ver, zoom, x, y)) if os.path.exists(tilefn) and not ignoreCached: if get_images: return None return send_file(tilefn, 'image/jpeg', expires=oneyear, modsince=req.META.get('HTTP_IF_MODIFIED_SINCE')) if not savecache: import tempfile f, tilefn = tempfile.mkstemp(suffix='.jpg') os.close(f) if wcs is None: try: wcs, W, H, zoomscale, zoom, x, y = get_tile_wcs(zoom, x, y) except RuntimeError as e: if get_images: return None return HttpResponse(e.strerror) else: W = wcs.get_width() H = wcs.get_height() from astrometry.util.fits import fits_table import numpy as np from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec from astrometry.util.starutil_numpy import degrees_between, arcsec_between from astrometry.util.resample import resample_with_wcs, OverlapError from astrometry.util.util import Tan, Sip import fitsio print('Tile wcs: center', wcs.radec_center(), 'pixel scale', wcs.pixel_scale()) global w_flist global w_flist_tree if w_flist is None: w_flist = fits_table( os.path.join(settings.DATA_DIR, 'sdss', 'window_flist.fits'), columns=['run', 'rerun', 'camcol', 'field', 'ra', 'dec', 'score']) print('Read', len(w_flist), 'window_flist entries') w_flist.cut(w_flist.rerun == '301') print('Cut to', len(w_flist), 'in rerun 301') w_flist_tree = tree_build_radec(w_flist.ra, w_flist.dec) # SDSS field size radius = 1.01 * np.hypot(10., 14.) / 2. / 60. # leaflet tile size ra, dec = wcs.pixelxy2radec(W / 2., H / 2.)[-2:] r0, d0 = wcs.pixelxy2radec(1, 1)[-2:] r1, d1 = wcs.pixelxy2radec(W, H)[-2:] radius = radius + max(degrees_between(ra, dec, r0, d0), degrees_between(ra, dec, r1, d1)) J = tree_search_radec(w_flist_tree, ra, dec, radius) print(len(J), 'overlapping SDSS fields found') if len(J) == 0: if get_images: return None if forcecache: # create symlink to blank.jpg! trymakedirs(tilefn) src = os.path.join(settings.STATIC_ROOT, 'blank.jpg') if os.path.exists(tilefn): os.unlink(tilefn) os.symlink(src, tilefn) print('Symlinked', tilefn, '->', src) from django.http import HttpResponseRedirect return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg') ww = [1, W * 0.25, W * 0.5, W * 0.75, W] hh = [1, H * 0.25, H * 0.5, H * 0.75, H] r, d = wcs.pixelxy2radec( [1] * len(hh) + ww + [W] * len(hh) + list(reversed(ww)), hh + [1] * len(ww) + list(reversed(hh)) + [H] * len(ww))[-2:] scaled = 0 scalepat = None scaledir = 'sdss' if zoom <= 13 and forcescale is None: # Get *actual* pixel scales at the top & bottom r1, d1 = wcs.pixelxy2radec(W / 2., H)[-2:] r2, d2 = wcs.pixelxy2radec(W / 2., H - 1.)[-2:] r3, d3 = wcs.pixelxy2radec(W / 2., 1.)[-2:] r4, d4 = wcs.pixelxy2radec(W / 2., 2.)[-2:] # Take the min = most zoomed-in scale = min(arcsec_between(r1, d1, r2, d2), arcsec_between(r3, d3, r4, d4)) native_scale = 0.396 scaled = int(np.floor(np.log2(scale / native_scale))) print('Zoom:', zoom, 'x,y', x, y, 'Tile pixel scale:', scale, 'Scale step:', scaled) scaled = np.clip(scaled, 1, 7) dirnm = os.path.join(basedir, 'scaled', scaledir) scalepat = os.path.join( dirnm, '%(scale)i%(band)s', '%(rerun)s', '%(run)i', '%(camcol)i', 'sdss-%(run)i-%(camcol)i-%(field)i-%(band)s.fits') if forcescale is not None: scaled = forcescale rimgs = [np.zeros((H, W), np.float32) for band in bands] rns = [np.zeros((H, W), np.float32) for band in bands] from astrometry.sdss import AsTransWrapper, DR9 sdss = DR9(basedir=settings.SDSS_DIR) sdss.saveUnzippedFiles(settings.SDSS_DIR) #sdss.setFitsioReadBZ2() if settings.SDSS_PHOTOOBJS: sdss.useLocalTree(photoObjs=settings.SDSS_PHOTOOBJS, resolve=settings.SDSS_RESOLVE) for jnum, j in enumerate(J): print('SDSS field', jnum, 'of', len(J), 'for zoom', zoom, 'x', x, 'y', y) im = w_flist[j] if im.score >= 0.5: weight = 1. else: weight = 0.001 for band, rimg, rn in zip(bands, rimgs, rns): if im.rerun != '301': continue tmpsuff = '.tmp%08i' % np.random.randint(100000000) basefn = sdss.retrieve('frame', im.run, im.camcol, field=im.field, band=band, rerun=im.rerun, tempsuffix=tmpsuff) if scaled > 0: fnargs = dict(band=band, rerun=im.rerun, run=im.run, camcol=im.camcol, field=im.field) fn = get_scaled(scalepat, fnargs, scaled, basefn, read_base_wcs=read_astrans, read_wcs=read_sip_wcs) print('get_scaled:', fn) else: fn = basefn frame = None if fn == basefn: frame = sdss.readFrame(im.run, im.camcol, im.field, band, filename=fn) h, w = frame.getImageShape() # Trim off the overlapping top of the image # Wimp out and instead of trimming 128 pix, trim 124! trim = 124 subh = h - trim astrans = frame.getAsTrans() fwcs = AsTransWrapper(astrans, w, subh) fullimg = frame.getImage() fullimg = fullimg[:-trim, :] else: fwcs = Sip(fn) fitsimg = fitsio.FITS(fn)[0] h, w = fitsimg.get_info()['dims'] fullimg = fitsimg.read() try: #Yo,Xo,Yi,Xi,nil = resample_with_wcs(wcs, fwcs, [], 3) Yo, Xo, Yi, Xi, [resamp ] = resample_with_wcs(wcs, fwcs, [fullimg], 2) except OverlapError: continue if len(Xi) == 0: #print 'No overlap' continue if sdssps is not None: x0 = Xi.min() x1 = Xi.max() y0 = Yi.min() y1 = Yi.max() slc = (slice(y0, y1 + 1), slice(x0, x1 + 1)) if frame is not None: img = frame.getImageSlice(slc) else: img = fitsimg[slc] #rimg[Yo,Xo] += img[Yi-y0, Xi-x0] if bestOnly: K = np.flatnonzero(weight > rn[Yo, Xo]) print('Updating', len(K), 'of', len(Yo), 'pixels') if len(K): rimg[Yo[K], Xo[K]] = resamp[K] * weight rn[Yo[K], Xo[K]] = weight else: rimg[Yo, Xo] += resamp * weight rn[Yo, Xo] += weight if sdssps is not None: # goodpix = np.ones(img.shape, bool) # fpM = sdss.readFpM(im.run, im.camcol, im.field, band) # for plane in [ 'INTERP', 'SATUR', 'CR', 'GHOST' ]: # fpM.setMaskedPixels(plane, goodpix, False, roi=[x0,x1,y0,y1]) plt.clf() #ima = dict(vmin=-0.05, vmax=0.5) #ima = dict(vmin=-0.5, vmax=2.) ima = dict(vmax=np.percentile(img, 99)) plt.subplot(2, 3, 1) dimshow(img, ticks=False, **ima) plt.title('image') rthis = np.zeros_like(rimg) #rthis[Yo,Xo] += img[Yi-y0, Xi-x0] rthis[Yo, Xo] += resamp plt.subplot(2, 3, 2) dimshow(rthis, ticks=False, **ima) plt.title('resampled') # plt.subplot(2,3,3) # dimshow(goodpix, ticks=False, vmin=0, vmax=1) # plt.title('good pix') plt.subplot(2, 3, 4) dimshow(rimg / np.maximum(rn, 1), ticks=False, **ima) plt.title('coadd') plt.subplot(2, 3, 5) dimshow(rn, vmin=0, ticks=False) plt.title('coverage: max %i' % rn.max()) plt.subplot(2, 3, 6) rgb = sdss_rgb( [rimg / np.maximum(rn, 1) for rimg, rn in zip(rimgs, rns)], bands) dimshow(rgb) plt.suptitle('SDSS %s, R/C/F %i/%i/%i' % (band, im.run, im.camcol, im.field)) sdssps.savefig() for rimg, rn in zip(rimgs, rns): rimg /= np.maximum(rn, 1e-3) del rns if get_images: return rimgs rgb = sdss_rgb(rimgs, bands) trymakedirs(tilefn) save_jpeg(tilefn, rgb) print('Wrote', tilefn) return send_file(tilefn, 'image/jpeg', unlink=(not savecache))
def run_tiles(X): tiles, tag = X print('Running', tag, '-', len(tiles), 'tiles') # Aaron's file has all images share the boresight CRVAL, so they have large CRPIX values. T = fits_table( '/global/cfs/cdirs/desi/users/ameisner/GFA/gfa_reduce_etc/gfa_wcs+focus.bigtan-zenith.fits' ) Nbright = 10 tiles_ann = fits_table() tiles_ann.index = tiles.index gfa_regions = [] maxr = 0. #t.cd[0,0], t.cd[0,1], t.cd[1,0], t.cd[1,1], for t in T: wcs = Tan(0., 0., t.crpix[0], t.crpix[1], t.cd[0, 0], t.cd[1, 0], t.cd[0, 1], t.cd[1, 1], float(t.naxis[0]), float(t.naxis[1])) ctype = t.extname[:5] cnum = int(t.extname[5]) h, w = wcs.shape x, y = [1, 1, w, w, 1], [1, h, h, 1, 1] r, d = wcs.pixelxy2radec(x, y) dists = degrees_between(0., 0., r, d) maxr = max(maxr, max(dists)) # x0, y0, x1, y1 rois = [] if ctype == 'FOCUS': # add the two half-chips. # wcs.get_subimage moves the CRPIX, but leave CRVAL unchanged, so tx,ty still work unchanged. # Aaron's WCS templates correct for the overscans #wcs_subs.append((cstr, cnum, 'a', wcs.get_subimage(0, 0, 1024, h))) #wcs_subs.append((cstr, cnum, 'b', wcs.get_subimage(1024, 0, 1024, h))) #all_sub_wcs[(cstr, cnum, 1)] = (tx, ty, wcs.get_subimage(50, 0, 1024, 1032)) #all_sub_wcs[(cstr, cnum, 2)] = (tx, ty, wcs.get_subimage(1174, 0, 1024, 1032)) # Add (negative) margin for donut size and telescope pointing uncertainty. # ~10" for donuts and ~10" for telescope pointing #margin = 100 #wcs_subs.append((cstr, cnum, 'a_margin', wcs.get_subimage(margin, margin, 1024-2*margin, h-2*margin))) #wcs_subs.append((cstr, cnum, 'b_margin', wcs.get_subimage(1024+margin, margin, 1024-2*margin, h-2*margin))) # Also add a positive margin for bright-star reflections off filters #margin = 125 #wcs_subs.append((cstr, cnum, 'expanded', wcs.get_subimage(-margin, -margin, w+2*margin, h+2*margin))) rois.append(('a', 0, 0, 1024, h)) rois.append(('b', 1024, 0, 2048, h)) margin = 100 rois.append( ('a_margin', margin, margin, 1024 - margin, h - margin)) rois.append( ('b_margin', 1024 + margin, margin, 2048 - margin, h - margin)) margin = 125 rois.append(('expanded', -margin, -margin, w + margin, h + margin)) else: # Guide chips include overscan pixels -- including a blank region in the middle. #print(cstr,cnum, 'shape', wcs.shape) #wcs_subs.append((cstr, cnum, 'ccd', wcs)) rois.append(('ccd', 0, 0, w, h)) # Add expanded GUIDE chips -- 25" margin / 0.2"/pix = 125 pix margin = 125 #wcs_subs.append((cstr, cnum, 'expanded', wcs.get_subimage(-margin, -margin, w+2*margin, h+2*margin))) rois.append(('expanded', -margin, -margin, w + margin, h + margin)) margin = 125 expwcs = wcs.get_subimage(-margin, -margin, w + 2 * margin, h + 2 * margin) newrois = [] for tag, x0, y0, x1, y1 in rois: name = '%s_%i_%s' % (ctype.lower(), cnum, tag) arr = np.zeros(len(tiles), (np.float32, Nbright)) tiles_ann.set('brightest_' + name, arr) # (the rois have zero-indexed x0,y0, and non-inclusive x1,y1!) newrois.append((name, arr, 1 + x0, 1 + y0, x1, y1)) gfa_regions.append((ctype, cnum, wcs, expwcs, newrois)) # DEBUG WCS # s = [] # for ctype,cnum,wcs,expwcs,rois in gfa_regions: # WCS = wcs # h,w = WCS.shape # #print('Expwcs:', w, 'x', h) # x = [1,1,w,w,1] # y = [1,h,h,1,1] # r,d = WCS.pixelxy2radec(x, y) # p = ','.join(['%.4f,%.4f' % (rr,dd) for rr,dd in zip(r,d)]) # s.append(p) # s = ';'.join(s) # print('http://legacysurvey.org/viewer/?ra=0&dec=0&poly='+s) # sys.exit(0) gaia = CachingGaiaCatalog(columns=[ 'ra', 'dec', 'phot_g_mean_mag', 'phot_bp_mean_mag', 'phot_rp_mean_mag', 'astrometric_excess_noise', 'astrometric_params_solved', 'source_id', 'pmra_error', 'pmdec_error', 'parallax_error', 'ra_error', 'dec_error', 'pmra', 'pmdec', 'parallax', 'ref_epoch' ]) tyc2fn = '/global/cfs/cdirs/cosmo/staging/tycho2/tycho2.kd.fits' tycho_kd = tree_open(tyc2fn) tycho_cat = fits_table(tyc2fn) maxrad = maxr * 1.05 for itile, tile in enumerate(tiles): #if not tile.in_imaging: # continue #if tile.centerid % 10 == 0: #print('tile program', tile.program, 'pass', tile.get('pass'), 'id', tile.centerid, gaia.get_healpix_tree.cache_info()) I = tree_search_radec(tycho_kd, tile.ra, tile.dec, maxrad) tycstars = tycho_cat[I] fix_tycho(tycstars) for cstr, cname, chipwcs, bigwcs, rois in gfa_regions: h, w = chipwcs.shape chipwcs.set_crval(tile.ra, tile.dec) bigwcs.set_crval(tile.ra, tile.dec) gstars = gaia.get_catalog_in_wcs(bigwcs, step=1032, margin=0) fix_gaia(gstars) bh, bw = bigwcs.shape ok, x, y = bigwcs.radec2pixelxy(tycstars.ra, tycstars.dec) tstars = tycstars[(x >= 1) * (y >= 1) * (x <= bw) * (y <= bh)] #print('Tile', tile.program, 'p', tile.get('pass'), tile.centerid, # 'GFA', cstr, cname, ':', len(gstars), 'Gaia stars', len(tstars), 'Tycho-2 stars') if len(gstars) + len(tstars) == 0: print('No stars in tile centerid', tile.centerid, 'chip', name) continue if len(gstars) > 0 and len(tstars) > 0: merge_gaia_tycho(gstars, tstars) stars = merge_tables([gstars, tstars], columns='fillzero') elif len(tstars) > 0: stars = tstars else: stars = gstars ok, x, y = chipwcs.radec2pixelxy(stars.ra, stars.dec) for name, arr, x0, y0, x1, y1 in rois: J = np.flatnonzero( (x >= x0) * (x <= x1) * (y >= y0) * (y <= y1)) mags = stars.mag[J] #print(' ', len(mags), 'in name') K = np.argsort(mags) K = K[:Nbright] arr[itile, :len(K)] = mags[K] #tiles.add_columns_from(tiles_ann) return tiles_ann
print('Kd data:', spherematch.tree_data(kd, np.array([0,3,5]).astype(np.uint32))) print('Kd data:', kd.get_data(np.array([0,3,5]).astype(np.uint32))) print('Permute:', spherematch.tree_permute(kd, np.array([3,5,7]).astype(np.int32))) print('Permute:', kd.permute(np.array([0,99,199]).astype(np.int32))) ra,dec = np.meshgrid(np.arange(0, 360), np.arange(-90, 91, 1)) ra1 = ra.ravel() dec1 = dec.ravel() rdkd1 = spherematch.tree_build_radec(ra1, dec1) print('RdKd:', rdkd1.n, rdkd1.bbox) ra2 = np.random.uniform(-10, 10, size=1000) dec2 = np.random.uniform(-10, 10, size=1000) rdkd2 = spherematch.tree_build_radec(ra2, dec2) I = spherematch.tree_search_radec(rdkd1, ra2[0], dec2[0], 2.) print('search_radec:', I) I,J,d = spherematch.match_radec(ra1, dec1, ra2, dec2, 1.) print('Matches:', len(I)) I,J,d = spherematch.match_radec(ra1, dec1, ra2, dec2, 1., nearest=True) print('Nearest matches:', len(I)) I = spherematch.match_radec(ra1, dec1, ra2, dec2, 1., indexlist=True) print('Index lists matches:', len(I))
def map_sdss(req, ver, zoom, x, y, savecache=None, tag='sdss', get_images=False, ignoreCached=False, wcs=None, forcecache=False, forcescale=None, **kwargs): from decals import settings if savecache is None: savecache = settings.SAVE_CACHE zoom = int(zoom) zoomscale = 2.**zoom x = int(x) y = int(y) if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale: raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom,x,y)) ver = int(ver) if not ver in tileversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) basedir = settings.DATA_DIR tilefn = os.path.join(basedir, 'tiles', tag, '%i/%i/%i/%i.jpg' % (ver, zoom, x, y)) if os.path.exists(tilefn) and not ignoreCached: if get_images: return None return send_file(tilefn, 'image/jpeg', expires=oneyear, modsince=req.META.get('HTTP_IF_MODIFIED_SINCE')) if not savecache: import tempfile f,tilefn = tempfile.mkstemp(suffix='.jpg') os.close(f) if wcs is None: try: wcs, W, H, zoomscale, zoom,x,y = get_tile_wcs(zoom, x, y) except RuntimeError as e: if get_images: return None return HttpResponse(e.strerror) else: W = wcs.get_width() H = wcs.get_height() from astrometry.util.fits import fits_table import numpy as np from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec from astrometry.util.starutil_numpy import degrees_between, arcsec_between from astrometry.util.resample import resample_with_wcs, OverlapError from astrometry.util.util import Tan, Sip import fitsio print 'Tile wcs: center', wcs.radec_center(), 'pixel scale', wcs.pixel_scale() global w_flist global w_flist_tree if w_flist is None: w_flist = fits_table(os.path.join(settings.DATA_DIR, 'sdss', 'window_flist.fits'), columns=['run','rerun','camcol','field','ra','dec','score']) print 'Read', len(w_flist), 'window_flist entries' w_flist.cut(w_flist.rerun == '301') print 'Cut to', len(w_flist), 'in rerun 301' w_flist_tree = tree_build_radec(w_flist.ra, w_flist.dec) # SDSS field size radius = 1.01 * np.hypot(10., 14.)/2. / 60. # leaflet tile size ra,dec = wcs.pixelxy2radec(W/2., H/2.)[-2:] r0,d0 = wcs.pixelxy2radec(1, 1)[-2:] r1,d1 = wcs.pixelxy2radec(W, H)[-2:] radius = radius + max(degrees_between(ra,dec, r0,d0), degrees_between(ra,dec, r1,d1)) J = tree_search_radec(w_flist_tree, ra, dec, radius) print len(J), 'overlapping SDSS fields found' if len(J) == 0: if get_images: return None if forcecache: # create symlink to blank.jpg! trymakedirs(tilefn) src = os.path.join(settings.STATIC_ROOT, 'blank.jpg') if os.path.exists(tilefn): os.unlink(tilefn) os.symlink(src, tilefn) print 'Symlinked', tilefn, '->', src from django.http import HttpResponseRedirect return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg') ww = [1, W*0.25, W*0.5, W*0.75, W] hh = [1, H*0.25, H*0.5, H*0.75, H] r,d = wcs.pixelxy2radec( [1]*len(hh) + ww + [W]*len(hh) + list(reversed(ww)), hh + [1]*len(ww) + list(reversed(hh)) + [H]*len(ww))[-2:] scaled = 0 scalepat = None scaledir = 'sdss' if zoom <= 13 and forcescale is None: # Get *actual* pixel scales at the top & bottom r1,d1 = wcs.pixelxy2radec(W/2., H)[-2:] r2,d2 = wcs.pixelxy2radec(W/2., H-1.)[-2:] r3,d3 = wcs.pixelxy2radec(W/2., 1.)[-2:] r4,d4 = wcs.pixelxy2radec(W/2., 2.)[-2:] # Take the min = most zoomed-in scale = min(arcsec_between(r1,d1, r2,d2), arcsec_between(r3,d3, r4,d4)) native_scale = 0.396 scaled = int(np.floor(np.log2(scale / native_scale))) print 'Zoom:', zoom, 'x,y', x,y, 'Tile pixel scale:', scale, 'Scale step:', scaled scaled = np.clip(scaled, 1, 7) dirnm = os.path.join(basedir, 'scaled', scaledir) scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(rerun)s', '%(run)i', '%(camcol)i', 'sdss-%(run)i-%(camcol)i-%(field)i-%(band)s.fits') if forcescale is not None: scaled = forcescale bands = 'gri' rimgs = [np.zeros((H,W), np.float32) for band in bands] rns = [np.zeros((H,W), np.float32) for band in bands] from astrometry.sdss import AsTransWrapper, DR9 sdss = DR9(basedir=settings.SDSS_DIR) sdss.saveUnzippedFiles(settings.SDSS_DIR) #sdss.setFitsioReadBZ2() if settings.SDSS_PHOTOOBJS: sdss.useLocalTree(photoObjs=settings.SDSS_PHOTOOBJS, resolve=settings.SDSS_RESOLVE) for jnum,j in enumerate(J): print 'SDSS field', jnum, 'of', len(J), 'for zoom', zoom, 'x', x, 'y', y im = w_flist[j] if im.score >= 0.5: weight = 1. else: weight = 0.001 for band,rimg,rn in zip(bands, rimgs, rns): if im.rerun != '301': continue tmpsuff = '.tmp%08i' % np.random.randint(100000000) basefn = sdss.retrieve('frame', im.run, im.camcol, field=im.field, band=band, rerun=im.rerun, tempsuffix=tmpsuff) if scaled > 0: fnargs = dict(band=band, rerun=im.rerun, run=im.run, camcol=im.camcol, field=im.field) fn = get_scaled(scalepat, fnargs, scaled, basefn, read_base_wcs=read_astrans, read_wcs=_read_sip_wcs) print 'get_scaled:', fn else: fn = basefn frame = None if fn == basefn: frame = sdss.readFrame(im.run, im.camcol, im.field, band, filename=fn) h,w = frame.getImageShape() # Trim off the overlapping top of the image # Wimp out and instead of trimming 128 pix, trim 124! trim = 124 subh = h - trim astrans = frame.getAsTrans() fwcs = AsTransWrapper(astrans, w, subh) fullimg = frame.getImage() fullimg = fullimg[:-trim,:] else: fwcs = Sip(fn) fitsimg = fitsio.FITS(fn)[0] h,w = fitsimg.get_info()['dims'] fullimg = fitsimg.read() try: #Yo,Xo,Yi,Xi,nil = resample_with_wcs(wcs, fwcs, [], 3) Yo,Xo,Yi,Xi,[resamp] = resample_with_wcs(wcs, fwcs, [fullimg], 2) except OverlapError: continue if len(Xi) == 0: #print 'No overlap' continue if sdssps is not None: x0 = Xi.min() x1 = Xi.max() y0 = Yi.min() y1 = Yi.max() slc = (slice(y0,y1+1), slice(x0,x1+1)) if frame is not None: img = frame.getImageSlice(slc) else: img = fitsimg[slc] #rimg[Yo,Xo] += img[Yi-y0, Xi-x0] rimg[Yo,Xo] += resamp * weight rn [Yo,Xo] += weight if sdssps is not None: # goodpix = np.ones(img.shape, bool) # fpM = sdss.readFpM(im.run, im.camcol, im.field, band) # for plane in [ 'INTERP', 'SATUR', 'CR', 'GHOST' ]: # fpM.setMaskedPixels(plane, goodpix, False, roi=[x0,x1,y0,y1]) plt.clf() #ima = dict(vmin=-0.05, vmax=0.5) #ima = dict(vmin=-0.5, vmax=2.) ima = dict(vmax=np.percentile(img, 99)) plt.subplot(2,3,1) dimshow(img, ticks=False, **ima) plt.title('image') rthis = np.zeros_like(rimg) #rthis[Yo,Xo] += img[Yi-y0, Xi-x0] rthis[Yo,Xo] += resamp plt.subplot(2,3,2) dimshow(rthis, ticks=False, **ima) plt.title('resampled') # plt.subplot(2,3,3) # dimshow(goodpix, ticks=False, vmin=0, vmax=1) # plt.title('good pix') plt.subplot(2,3,4) dimshow(rimg / np.maximum(rn, 1), ticks=False, **ima) plt.title('coadd') plt.subplot(2,3,5) dimshow(rn, vmin=0, ticks=False) plt.title('coverage: max %i' % rn.max()) plt.subplot(2,3,6) rgb = sdss_rgb([rimg/np.maximum(rn,1) for rimg,rn in zip(rimgs,rns)], bands) dimshow(rgb) plt.suptitle('SDSS %s, R/C/F %i/%i/%i' % (band, im.run, im.camcol, im.field)) sdssps.savefig() for rimg,rn in zip(rimgs, rns): rimg /= np.maximum(rn, 1e-3) del rns if get_images: return rimgs rgb = sdss_rgb(rimgs, bands) trymakedirs(tilefn) save_jpeg(tilefn, rgb) print 'Wrote', tilefn return send_file(tilefn, 'image/jpeg', unlink=(not savecache))
def read_tycho2(survey, targetwcs): from astrometry.libkd.spherematch import tree_open, tree_search_radec tycho2fn = survey.find_file('tycho2') radius = 1. ra, dec = targetwcs.radec_center() # John added the "isgalaxy" flag 2018-05-10, from the Metz & # Geffert (04) catalog. # Eddie added the "zguess" column 2019-03-06, by matching with # 2MASS and estimating z based on APASS. # The "tycho2.kd.fits" file read here was produced by: # # fitscopy ~schlafly/legacysurvey/tycho-isgalaxyflag-2mass.fits"[col \ # tyc1;tyc2;tyc3;ra;dec;sigma_ra;sigma_dec;mean_ra;mean_dec;pm_ra;pm_dec; \ # sigma_pm_ra;sigma_pm_dec;epoch_ra;epoch_dec;mag_bt;mag_vt;mag_hp; \ # isgalaxy;Jmag;Hmag;Kmag,zguess]" /tmp/tycho2-astrom.fits # startree -P -k -n stars -T -i /tmp/tycho2-astrom.fits \ # -o /global/project/projectdirs/cosmo/staging/tycho2/tycho2.kd.fits kd = tree_open(tycho2fn, 'stars') I = tree_search_radec(kd, ra, dec, radius) debug(len(I), 'Tycho-2 stars within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (ra, dec)) if len(I) == 0: return None # Read only the rows within range. tycho = fits_table(tycho2fn, rows=I) del kd if 'isgalaxy' in tycho.get_columns(): tycho.cut(tycho.isgalaxy == 0) debug('Cut to', len(tycho), 'Tycho-2 stars on isgalaxy==0') else: print('Warning: no "isgalaxy" column in Tycho-2 catalog') tycho.ref_cat = np.array(['T2'] * len(tycho)) # tyc1: [1,9537], tyc2: [1,12121], tyc3: [1,3] tycho.ref_id = (tycho.tyc1.astype(np.int64) * 1000000 + tycho.tyc2.astype(np.int64) * 10 + tycho.tyc3.astype(np.int64)) with np.errstate(divide='ignore'): tycho.pmra_ivar = 1. / tycho.sigma_pm_ra**2 tycho.pmdec_ivar = 1. / tycho.sigma_pm_dec**2 tycho.ra_ivar = 1. / tycho.sigma_ra**2 tycho.dec_ivar = 1. / tycho.sigma_dec**2 tycho.rename('pm_ra', 'pmra') tycho.rename('pm_dec', 'pmdec') for c in ['pmra', 'pmdec', 'pmra_ivar', 'pmdec_ivar']: X = tycho.get(c) X[np.logical_not(np.isfinite(X))] = 0. tycho.mag = tycho.mag_vt # Patch missing mag values... tycho.mag[tycho.mag == 0] = tycho.mag_hp[tycho.mag == 0] tycho.mag[tycho.mag == 0] = tycho.mag_bt[tycho.mag == 0] # Per discussion in issue #306 -- cut on mag < 13. This drops only 13k/2.5M stars tycho.cut(tycho.mag < 13.) # See note on gaia.radius above -- don't change the 0.262 to # targetwcs.pixel_scale()! tycho.radius = np.minimum(1800., 150. * 2.5**( (11. - tycho.mag) / 3.)) * 0.262 / 3600. for c in [ 'tyc1', 'tyc2', 'tyc3', 'mag_bt', 'mag_vt', 'mag_hp', 'mean_ra', 'mean_dec', 'sigma_pm_ra', 'sigma_pm_dec', 'sigma_ra', 'sigma_dec' ]: tycho.delete_column(c) # add Gaia-style columns # No parallaxes in Tycho-2 tycho.parallax = np.zeros(len(tycho), np.float32) # Tycho-2 has separate epoch_ra and epoch_dec. # Move source to the mean epoch. tycho.ref_epoch = (tycho.epoch_ra + tycho.epoch_dec) / 2. cosdec = np.cos(np.deg2rad(tycho.dec)) tycho.ra += (tycho.ref_epoch - tycho.epoch_ra) * tycho.pmra / 3600. / cosdec tycho.dec += (tycho.ref_epoch - tycho.epoch_dec) * tycho.pmdec / 3600. # Tycho-2 proper motions are in arcsec/yr; Gaia are mas/yr. tycho.pmra *= 1000. tycho.pmdec *= 1000. # We already cut on John's "isgalaxy" flag tycho.pointsource = np.ones(len(tycho), bool) # phot_g_mean_mag -- for initial brightness of source tycho.phot_g_mean_mag = tycho.mag tycho.delete_column('epoch_ra') tycho.delete_column('epoch_dec') tycho.isbright = np.ones(len(tycho), bool) tycho.ismedium = np.ones(len(tycho), bool) return tycho
def read_large_galaxies(survey, targetwcs, bands, clean_columns=True, max_radius=2.): # Note, max_radius must include the brick radius! from astrometry.libkd.spherematch import tree_open, tree_search_radec galfn = survey.find_file('large-galaxies') if galfn is None: debug('No large-galaxies catalog file') return None radius = max_radius rc, dc = targetwcs.radec_center() debug('Reading', galfn) try: kd = tree_open(galfn, 'stars') except: kd = tree_open(galfn, 'largegals') I = tree_search_radec(kd, rc, dc, radius) debug('%i large galaxies within %.3g deg of RA,Dec (%.3f, %.3f)' % (len(I), radius, rc, dc)) if len(I) == 0: return None # Read only the rows within range. galaxies = fits_table(galfn, rows=I) del kd refcat, preburn = get_large_galaxy_version(galfn) debug('Large galaxies version: "%s", preburned?' % refcat, preburn) if preburn: # SGA ellipse catalog # NOTE: fields such as ref_cat, preburned, etc, already exist in the # "galaxies" catalog read from disk. # The galaxies we want to appear in MASKBITS get # 'islargegalaxy' set. This includes both pre-burned # galaxies, and ones where the preburning failed and we want # to fall back to the SGA-parent ellipse for masking. galaxies.islargegalaxy = ((galaxies.ref_cat == refcat) * (galaxies.sga_id > -1)) # The pre-fit galaxies whose parameters will stay fixed galaxies.freezeparams = (galaxies.preburned * galaxies.freeze) else: # SGA parent catalog galaxies.ref_cat = np.array([refcat] * len(galaxies)) galaxies.islargegalaxy = np.ones(len(galaxies), bool) galaxies.freezeparams = np.zeros(len(galaxies), bool) galaxies.preburned = np.zeros(len(galaxies), bool) galaxies.rename('sga_id', 'ref_id') galaxies.rename('mag_leda', 'mag') # Pre-burned, frozen but non-SGA sources have diam=-1. galaxies.radius = np.maximum(0., galaxies.diam / 2. / 60.) # [degree] galaxies.keep_radius = 2. * galaxies.radius galaxies.sources = np.empty(len(galaxies), object) galaxies.sources[:] = None if bands is not None: galaxies.sources[:] = get_galaxy_sources(galaxies, bands) if clean_columns: keep_columns = [ 'ra', 'dec', 'radius', 'mag', 'ref_cat', 'ref_id', 'ba', 'pa', 'sources', 'islargegalaxy', 'freezeparams', 'keep_radius' ] for c in galaxies.get_columns(): if not c in keep_columns: galaxies.delete_column(c) return galaxies
def read_tycho2(survey, targetwcs, bands): from astrometry.libkd.spherematch import tree_open, tree_search_radec from legacypipe.survey import GaiaSource tycho2fn = survey.find_file('tycho2') radius = 1. ra, dec = targetwcs.radec_center() # John added the "isgalaxy" flag 2018-05-10, from the Metz & # Geffert (04) catalog. # Eddie added the "zguess" column 2019-03-06, by matching with # 2MASS and estimating z based on APASS. # The "tycho2.kd.fits" file read here was produced by: # # fitscopy ~schlafly/legacysurvey/tycho-isgalaxyflag-2mass.fits"[col \ # tyc1;tyc2;tyc3;ra;dec;sigma_ra;sigma_dec;mean_ra;mean_dec;pm_ra;pm_dec; \ # sigma_pm_ra;sigma_pm_dec;epoch_ra;epoch_dec;mag_bt;mag_vt;mag_hp; \ # isgalaxy;Jmag;Hmag;Kmag,zguess]" /tmp/tycho2-astrom.fits # startree -P -k -n stars -T -i /tmp/tycho2-astrom.fits \ # -o /global/project/projectdirs/cosmo/staging/tycho2/tycho2.kd.fits kd = tree_open(tycho2fn, 'stars') I = tree_search_radec(kd, ra, dec, radius) debug(len(I), 'Tycho-2 stars within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (ra, dec)) if len(I) == 0: return None # Read only the rows within range. tycho = fits_table(tycho2fn, rows=I) del kd if 'isgalaxy' in tycho.get_columns(): tycho.cut(tycho.isgalaxy == 0) debug('Cut to', len(tycho), 'Tycho-2 stars on isgalaxy==0') else: print('Warning: no "isgalaxy" column in Tycho-2 catalog') tycho.ref_cat = np.array(['T2'] * len(tycho)) # tyc1: [1,9537], tyc2: [1,12121], tyc3: [1,3] tycho.ref_id = (tycho.tyc1.astype(np.int64) * 1000000 + tycho.tyc2.astype(np.int64) * 10 + tycho.tyc3.astype(np.int64)) with np.errstate(divide='ignore'): # In our Tycho-2 catalog, sigma_pm_* are in *arcsec/yr*, Gaia is in mas/yr. tycho.pmra_ivar = 1. / (tycho.sigma_pm_ra * 1000.)**2 tycho.pmdec_ivar = 1. / (tycho.sigma_pm_dec * 1000.)**2 tycho.ra_ivar = 1. / tycho.sigma_ra**2 tycho.dec_ivar = 1. / tycho.sigma_dec**2 tycho.rename('pm_ra', 'pmra') tycho.rename('pm_dec', 'pmdec') for c in ['pmra', 'pmdec', 'pmra_ivar', 'pmdec_ivar']: X = tycho.get(c) X[np.logical_not(np.isfinite(X))] = 0. tycho.mag = tycho.mag_vt # Patch missing mag values... tycho.mag[tycho.mag == 0] = tycho.mag_hp[tycho.mag == 0] tycho.mag[tycho.mag == 0] = tycho.mag_bt[tycho.mag == 0] # Use zguess tycho.mask_mag = tycho.mag with np.errstate(invalid='ignore'): I = np.flatnonzero( np.isfinite(tycho.zguess) * (tycho.zguess + 1. < tycho.mag)) tycho.mask_mag[I] = tycho.zguess[I] # Per discussion in issue #306 -- cut on mag < 13. # This drops only 13k/2.5M stars. tycho.cut(tycho.mask_mag < 13.) tycho.radius = mask_radius_for_mag(tycho.mask_mag) tycho.keep_radius = 2. * tycho.radius for c in [ 'tyc1', 'tyc2', 'tyc3', 'mag_bt', 'mag_vt', 'mag_hp', 'mean_ra', 'mean_dec', 'sigma_pm_ra', 'sigma_pm_dec', 'sigma_ra', 'sigma_dec' ]: tycho.delete_column(c) # add Gaia-style columns # No parallaxes in Tycho-2 tycho.parallax = np.zeros(len(tycho), np.float32) # Tycho-2 "supplement" stars, from Hipparcos and Tycho-1 catalogs, have # ref_epoch = 0. Fill in with the 1991.25 epoch of those catalogs. tycho.epoch_ra[tycho.epoch_ra == 0] = 1991.25 tycho.epoch_dec[tycho.epoch_dec == 0] = 1991.25 # Tycho-2 has separate epoch_ra and epoch_dec. # Move source to the mean epoch. tycho.ref_epoch = (tycho.epoch_ra + tycho.epoch_dec) / 2. cosdec = np.cos(np.deg2rad(tycho.dec)) tycho.ra += (tycho.ref_epoch - tycho.epoch_ra) * tycho.pmra / 3600. / cosdec tycho.dec += (tycho.ref_epoch - tycho.epoch_dec) * tycho.pmdec / 3600. # Tycho-2 proper motions are in arcsec/yr; Gaia are mas/yr. tycho.pmra *= 1000. tycho.pmdec *= 1000. # We already cut on John's "isgalaxy" flag tycho.pointsource = np.ones(len(tycho), bool) # phot_g_mean_mag -- for initial brightness of source tycho.phot_g_mean_mag = tycho.mag tycho.delete_column('epoch_ra') tycho.delete_column('epoch_dec') tycho.istycho = np.ones(len(tycho), bool) tycho.isbright = np.ones(len(tycho), bool) tycho.ismedium = np.ones(len(tycho), bool) tycho.donotfit = np.zeros(len(tycho), bool) tycho.sources = np.empty(len(tycho), object) if bands is not None: for i, t in enumerate(tycho): tycho.sources[i] = GaiaSource.from_catalog(t, bands) return tycho
def map_decam_depth(req, ver, zoom, x, y, savecache=False, band=None, ignoreCached=False): global Tdepth global Tdepthkd if band is None: band = req.GET.get('band') if not band in ['g','r','z']: raise RuntimeError('Invalid band') tag = 'decam-depth-%s' % band zoom = int(zoom) zoomscale = 2.**zoom x = int(x) y = int(y) if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale: raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom,x,y)) ver = int(ver) if not ver in tileversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) basedir = settings.DATA_DIR tilefn = os.path.join(basedir, 'tiles', tag, '%i/%i/%i/%i.jpg' % (ver, zoom, x, y)) if os.path.exists(tilefn) and not ignoreCached: print 'Cached:', tilefn return send_file(tilefn, 'image/jpeg', expires=oneyear, modsince=req.META.get('HTTP_IF_MODIFIED_SINCE')) from astrometry.util.util import Tan from astrometry.libkd.spherematch import match_radec from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec from astrometry.util.fits import fits_table from astrometry.util.starutil_numpy import degrees_between import numpy as np import fitsio try: wcs, W, H, zoomscale, zoom,x,y = get_tile_wcs(zoom, x, y) except RuntimeError as e: return HttpResponse(e.strerror) rlo,d = wcs.pixelxy2radec(W, H/2)[-2:] rhi,d = wcs.pixelxy2radec(1, H/2)[-2:] r,d1 = wcs.pixelxy2radec(W/2, 1)[-2:] r,d2 = wcs.pixelxy2radec(W/2, H)[-2:] r,d = wcs.pixelxy2radec(W/2, H/2)[-2:] rad = max(degrees_between(r, d, rlo, d1), degrees_between(r, d, rhi, d2)) if Tdepth is None: T = fits_table(os.path.join(basedir, 'decals-zpt-nondecals.fits'), columns=['ccdra','ccddec','arawgain', 'avsky', 'ccdzpt', 'filter', 'crpix1','crpix2', 'crval1','crval2','cd1_1','cd1_2', 'cd2_1','cd2_2', 'naxis1', 'naxis2', 'exptime', 'fwhm']) T.rename('ccdra', 'ra') T.rename('ccddec', 'dec') Tdepth = {} Tdepthkd = {} for b in ['g','r','z']: Tdepth[b] = T[T.filter == b] Tdepthkd[b] = tree_build_radec(Tdepth[b].ra, Tdepth[b].dec) T = Tdepth[band] Tkd = Tdepthkd[band] #I,J,d = match_radec(T.ra, T.dec, r, d, rad + 0.2) I = tree_search_radec(Tkd, r, d, rad + 0.2) print len(I), 'CCDs in range' if len(I) == 0: from django.http import HttpResponseRedirect return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg') depthiv = np.zeros((H,W), np.float32) for t in T[I]: twcs = Tan(*[float(x) for x in [ t.crval1, t.crval2, t.crpix1, t.crpix2, t.cd1_1, t.cd1_2, t.cd2_1, t.cd2_2, t.naxis1, t.naxis2]]) w,h = t.naxis1, t.naxis2 r,d = twcs.pixelxy2radec([1,1,w,w], [1,h,h,1]) ok,x,y = wcs.radec2pixelxy(r, d) #print 'x,y coords of CCD:', x, y x0 = int(x.min()) x1 = int(x.max()) y0 = int(y.min()) y1 = int(y.max()) if y1 < 0 or x1 < 0 or x0 >= W or y0 >= H: continue readnoise = 10. # e-; 7.0 to 15.0 according to DECam Data Handbook skysig = np.sqrt(t.avsky * t.arawgain + readnoise**2) / t.arawgain zpscale = 10.**((t.ccdzpt - 22.5)/2.5) * t.exptime sig1 = skysig / zpscale psf_sigma = t.fwhm / 2.35 # point-source depth psfnorm = 1./(2. * np.sqrt(np.pi) * psf_sigma) detsig1 = sig1 / psfnorm #print '5-sigma point-source depth:', NanoMaggies.nanomaggiesToMag(detsig1 * 5.) div = 1 / detsig1**2 depthiv[max(y0,0):min(y1,H), max(x0,0):min(x1,W)] += div ptsrc = -2.5 * (np.log10(np.sqrt(1./depthiv) * 5) - 9) ptsrc[depthiv == 0] = 0. if savecache: trymakedirs(tilefn) else: import tempfile f,tilefn = tempfile.mkstemp(suffix='.jpg') os.close(f) import pylab as plt plt.imsave(tilefn, ptsrc, vmin=22., vmax=25., cmap='hot')#nipy_spectral') return send_file(tilefn, 'image/jpeg', unlink=(not savecache))