def read_large_galaxies(survey, targetwcs): from astrometry.libkd.spherematch import tree_open, tree_search_radec galfn = survey.find_file('large-galaxies') radius = 1. rc, dc = targetwcs.radec_center() kd = tree_open(galfn, 'largegals') I = tree_search_radec(kd, rc, dc, radius) debug(len(I), 'large galaxies within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (rc, dc)) if len(I) == 0: return None # Read only the rows within range. galaxies = fits_table( galfn, rows=I, columns=['ra', 'dec', 'd25', 'mag', 'lslga_id', 'ba', 'pa']) del kd # # D25 is diameter in arcmin galaxies.radius = galaxies.d25 / 2. / 60. # John told me to do this... #galaxies.radius *= 1.2 ...and then John taketh away. galaxies.delete_column('d25') galaxies.rename('lslga_id', 'ref_id') galaxies.ref_cat = np.array(['L2'] * len(galaxies)) galaxies.islargegalaxy = np.ones(len(galaxies), bool) return galaxies
def match_kdtree_catalog(wcs, catfn): from astrometry.libkd.spherematch import ( tree_open, tree_close, tree_build_radec, tree_free, trees_match, tree_permute, ) from astrometry.libkd import spherematch_c from astrometry.util.starutil_numpy import deg2dist, xyztoradec import numpy as np import sys rc, dc = wcs.get_center() rr = wcs.get_radius() kd = tree_open(catfn) kd2 = tree_build_radec(np.array([rc]), np.array([dc])) r = deg2dist(rr) I, J, nil = trees_match(kd, kd2, r, permuted=False) # HACK # I2,J,d = trees_match(kd, kd2, r) xyz = spherematch_c.kdtree_get_positions(kd, I.astype(np.uint32)) # print 'I', I I2 = tree_permute(kd, I) # print 'I2', I2 tree_free(kd2) tree_close(kd) tra, tdec = xyztoradec(xyz) return tra, tdec, I2
def get_sky_template_filename(self, old_calibs_ok=False): import os from astrometry.util.fits import fits_table dirnm = os.environ.get('SKY_TEMPLATE_DIR', None) if dirnm is None: info('decam: no SKY_TEMPLATE_DIR environment variable set.') return None ''' # Create this sky-scales.kd.fits file via: python legacypipe/create-sky-template-kdtree.py skyscales_ccds.fits \ sky-scales.kd.fits ''' fn = os.path.join(dirnm, 'sky-scales.kd.fits') if not os.path.exists(fn): info('decam: no $SKY_TEMPLATE_DIR/sky-scales.kd.fits file.') return None from astrometry.libkd.spherematch import tree_open kd = tree_open(fn, 'expnum') I = kd.search(np.array([self.expnum]), 0.5, 0, 0) if len(I) == 0: info('decam: expnum %i not found in file %s' % (self.expnum, fn)) return None # Read only the CCD-table rows within range. S = fits_table(fn, rows=I) S.cut(np.array([c.strip() == self.ccdname for c in S.ccdname])) if len(S) == 0: info('decam: ccdname %s, expnum %i not found in file %s' % (self.ccdname, self.expnum, fn)) return None assert (len(S) == 1) sky = S[0] if sky.run == -1: debug('sky template: run=-1 for expnum %i, ccdname %s' % (self.expnum, self.ccdname)) return None # Check PLPROCID only if not validate_version( fn, 'table', self.expnum, None, self.plprocid, data=S): txt = ( 'Sky template for expnum=%i, ccdname=%s did not pass consistency validation (EXPNUM, PLPROCID) -- image %i,%s vs template table %i,%s' % (self.expnum, self.ccdname, self.expnum, self.plprocid, sky.expnum, sky.plprocid)) if old_calibs_ok: info('Warning:', txt, '-- but old_calibs_ok') else: raise RuntimeError(txt) assert (self.band == sky.filter) tfn = os.path.join(dirnm, 'sky_templates', 'sky_template_%s_%i.fits.fz' % (self.band, sky.run)) if not os.path.exists(tfn): info('WARNING: Sky template file %s does not exist' % tfn) return None return dict(template_filename=tfn, sky_template_dir=dirnm, sky_obj=sky, skyscales_fn=fn)
def match_kdtree_catalog(wcs, catfn): from astrometry.libkd.spherematch import tree_open, tree_close, tree_build_radec, tree_free, trees_match, tree_permute from astrometry.libkd import spherematch_c from astrometry.util.starutil_numpy import deg2dist, xyztoradec import numpy as np import sys rc,dc = wcs.get_center() rr = wcs.get_radius() kd = tree_open(catfn) kd2 = tree_build_radec(np.array([rc]), np.array([dc])) r = deg2dist(rr) I,J,nil = trees_match(kd, kd2, r, permuted=False) del kd2 xyz = kd.get_data(I.astype(np.uint32)) I = kd.permute(I) del kd tra,tdec = xyztoradec(xyz) return tra, tdec, I
def match_kdtree_catalog(wcs, catfn): from astrometry.libkd.spherematch import tree_open, tree_close, tree_build_radec, tree_free, trees_match, tree_permute from astrometry.libkd import spherematch_c from astrometry.util.starutil_numpy import deg2dist, xyztoradec import numpy as np import sys rc,dc = wcs.get_center() rr = wcs.get_radius() kd = tree_open(catfn) kd2 = tree_build_radec(np.array([rc]), np.array([dc])) r = deg2dist(rr) I,J,nil = trees_match(kd, kd2, r, permuted=False) del kd2 xyz = kd.get_data(I.astype(np.uint32)) I = kd.permute(I) del kd tra,tdec = xyztoradec(xyz) return tra, tdec, I
def cat_kd(req, ver, tag, fn): tag = 'spec' ralo = float(req.GET['ralo']) rahi = float(req.GET['rahi']) declo = float(req.GET['declo']) dechi = float(req.GET['dechi']) ver = int(ver) if not ver in catversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) import numpy as np from astrometry.util.fits import fits_table, merge_tables from astrometry.libkd.spherematch import tree_open, tree_search_radec from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between xyz1 = radectoxyz(ralo, declo) xyz2 = radectoxyz(rahi, dechi) xyz = (xyz1 + xyz2) / 2. xyz /= np.sqrt(np.sum(xyz**2)) rc, dc = xyztoradec(xyz) rc = rc[0] dc = dc[0] rad = degrees_between(rc, dc, ralo, declo) kd = tree_open(fn) I = tree_search_radec(kd, rc, dc, rad) print('Matched', len(I), 'from', fn) if len(I) == 0: return None T = fits_table(fn, rows=I) debug(len(T), 'spectra') if ralo > rahi: # RA wrap T.cut( np.logical_or(T.ra > ralo, T.ra < rahi) * (T.dec > declo) * (T.dec < dechi)) else: T.cut( (T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi)) debug(len(T), 'in cut') return T
def cat_kd(req, ver, tag, fn): tag = 'spec' ralo = float(req.GET['ralo']) rahi = float(req.GET['rahi']) declo = float(req.GET['declo']) dechi = float(req.GET['dechi']) ver = int(ver) if not ver in catversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) import numpy as np from astrometry.util.fits import fits_table, merge_tables from astrometry.libkd.spherematch import tree_open, tree_search_radec from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between xyz1 = radectoxyz(ralo, declo) xyz2 = radectoxyz(rahi, dechi) xyz = (xyz1 + xyz2)/2. xyz /= np.sqrt(np.sum(xyz**2)) rc,dc = xyztoradec(xyz) rc = rc[0] dc = dc[0] rad = degrees_between(rc, dc, ralo, declo) kd = tree_open(fn) I = tree_search_radec(kd, rc, dc, rad) print('Matched', len(I), 'from', fn) if len(I) == 0: return None T = fits_table(fn, rows=I) debug(len(T), 'spectra') if ralo > rahi: # RA wrap T.cut(np.logical_or(T.ra > ralo, T.ra < rahi) * (T.dec > declo) * (T.dec < dechi)) else: T.cut((T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi)) debug(len(T), 'in cut') return T
def match_kdtree_catalog(wcs, catfn): from astrometry.libkd.spherematch import tree_open, tree_close, tree_build_radec, tree_free, trees_match, tree_permute from astrometry.libkd import spherematch_c from astrometry.util.starutil_numpy import deg2dist, xyztoradec import numpy as np import sys rc, dc = wcs.get_center() rr = wcs.get_radius() kd = tree_open(catfn) kd2 = tree_build_radec(np.array([rc]), np.array([dc])) r = deg2dist(rr) I, J, nil = trees_match(kd, kd2, r, permuted=False) # HACK #I2,J,d = trees_match(kd, kd2, r) xyz = spherematch_c.kdtree_get_positions(kd, I.astype(np.uint32)) #print 'I', I I2 = tree_permute(kd, I) #print 'I2', I2 tree_free(kd2) tree_close(kd) tra, tdec = xyztoradec(xyz) return tra, tdec, I2
def plot_wcs_outline(wcsfn, plotfn, W=256, H=256, width=36, zoom=True, zoomwidth=3.6, grid=10, hd=False, hd_labels=False, tycho2=False): anutil.log_init(3) #anutil.log_set_level(3) wcs = anutil.Tan(wcsfn, 0) ra, dec = wcs.radec_center() plot = ps.Plotstuff(outformat='png', size=(W, H), rdw=(ra, dec, width)) plot.linestep = 1. plot.color = 'verydarkblue' plot.plot('fill') plot.fontsize = 12 #plot.color = 'gray' # dark gray plot.rgb = (0.3, 0.3, 0.3) if grid is not None: plot.plot_grid(*([grid] * 4)) plot.rgb = (0.4, 0.6, 0.4) ann = plot.annotations ann.NGC = ann.bright = ann.HD = 0 ann.constellations = 1 ann.constellation_labels = 1 ann.constellation_labels_long = 1 plot.plot('annotations') plot.stroke() ann.constellation_labels = 0 ann.constellation_labels_long = 0 ann.constellation_lines = 0 ann.constellation_markers = 1 plot.markersize = 3 plot.rgb = (0.4, 0.6, 0.4) plot.plot('annotations') plot.fill() ann.constellation_markers = 0 ann.bright_labels = False ann.bright = True plot.markersize = 2 if zoom >= 2: ann.bright_labels = True plot.plot('annotations') ann.bright = False ann.bright_labels = False plot.fill() if hd: ann.HD = True ann.HD_labels = hd_labels ps.plot_annotations_set_hd_catalog(ann, settings.HENRY_DRAPER_CAT) plot.plot('annotations') plot.stroke() ann.HD = False ann.HD_labels = False if tycho2 and settings.TYCHO2_KD: from astrometry.libkd.spherematch import tree_open, tree_close, tree_build_radec, tree_free, trees_match from astrometry.libkd import spherematch_c from astrometry.util.starutil_numpy import deg2dist, xyztoradec import numpy as np import sys kd = tree_open(settings.TYCHO2_KD) # this is a bit silly: build a tree with a single point, then do match() kd2 = tree_build_radec(np.array([ra]), np.array([dec])) r = deg2dist(width * np.sqrt(2.) / 2.) #r = deg2dist(wcs.radius()) I, nil, nil = trees_match(kd, kd2, r, permuted=False) del nil #print 'Matched', len(I) xyz = spherematch_c.kdtree_get_positions(kd, I) del I tree_free(kd2) tree_close(kd) #print >>sys.stderr, 'Got', xyz.shape, xyz tra, tdec = xyztoradec(xyz) #print >>sys.stderr, 'RA,Dec', ra,dec plot.apply_settings() for r, d in zip(tra, tdec): plot.marker_radec(r, d) plot.fill() ann.NGC = 1 plot.plot('annotations') ann.NGC = 0 plot.color = 'white' plot.lw = 3 out = plot.outline out.wcs_file = wcsfn plot.plot('outline') if zoom: # MAGIC width, height are arbitrary zoomwcs = anutil.anwcs_create_box(ra, dec, zoomwidth, 1000, 1000) out.wcs = zoomwcs plot.lw = 1 plot.dashed(3) plot.plot('outline') plot.write(plotfn)
def plot_wcs_outline(wcsfn, plotfn, W=256, H=256, width=36, zoom=True, zoomwidth=3.6, grid=10, hd=False, hd_labels=False, tycho2=False): anutil.log_init(3) #anutil.log_set_level(3) wcs = anutil.Tan(wcsfn, 0) ra,dec = wcs.radec_center() plot = ps.Plotstuff(outformat='png', size=(W, H), rdw=(ra,dec,width)) plot.linestep = 1. plot.color = 'verydarkblue' plot.plot('fill') plot.fontsize = 12 #plot.color = 'gray' # dark gray plot.rgb = (0.3,0.3,0.3) if grid is not None: plot.plot_grid(*([grid]*4)) plot.rgb = (0.4, 0.6, 0.4) ann = plot.annotations ann.NGC = ann.bright = ann.HD = 0 ann.constellations = 1 ann.constellation_labels = 1 ann.constellation_labels_long = 1 plot.plot('annotations') plot.stroke() ann.constellation_labels = 0 ann.constellation_labels_long = 0 ann.constellation_lines = 0 ann.constellation_markers = 1 plot.markersize = 3 plot.rgb = (0.4, 0.6, 0.4) plot.plot('annotations') plot.fill() ann.constellation_markers = 0 ann.bright_labels = False ann.bright = True plot.markersize = 2 if zoom >= 2: ann.bright_labels = True plot.plot('annotations') ann.bright = False ann.bright_labels = False plot.fill() if hd: ann.HD = True ann.HD_labels = hd_labels ps.plot_annotations_set_hd_catalog(ann, settings.HENRY_DRAPER_CAT) plot.plot('annotations') plot.stroke() ann.HD = False ann.HD_labels = False if tycho2: from astrometry.libkd.spherematch import tree_open, tree_close, tree_build_radec, tree_free, trees_match from astrometry.libkd import spherematch_c from astrometry.util.starutil_numpy import deg2dist, xyztoradec import numpy as np import sys kd = tree_open(settings.TYCHO2_KD) # this is a bit silly: build a tree with a single point, then do match() kd2 = tree_build_radec(np.array([ra]), np.array([dec])) r = deg2dist(width * np.sqrt(2.) / 2.) #r = deg2dist(wcs.radius()) I,J,d = trees_match(kd, kd2, r, permuted=False) del J del d #print 'Matched', len(I) xyz = spherematch_c.kdtree_get_positions(kd, I) del I tree_free(kd2) tree_close(kd) #print >>sys.stderr, 'Got', xyz.shape, xyz tra,tdec = xyztoradec(xyz) #print >>sys.stderr, 'RA,Dec', ra,dec plot.apply_settings() for r,d in zip(tra,tdec): plot.marker_radec(r,d) plot.fill() ann.NGC = 1 plot.plot('annotations') ann.NGC = 0 plot.color = 'white' plot.lw = 3 out = plot.outline out.wcs_file = wcsfn plot.plot('outline') if zoom: # MAGIC width, height are arbitrary zoomwcs = anutil.anwcs_create_box(ra, dec, zoomwidth, 1000,1000) out.wcs = zoomwcs plot.lw = 1 plot.dashed(3) plot.plot('outline') plot.write(plotfn)
def cat_targets_drAB(req, ver, cats=[], tag='', bgs=False, sky=False, bright=False, dark=False, color_name_func=desitarget_color_names): ''' color_name_func: function that selects names and colors for targets (eg based on targeting bit values) ''' import json ralo = float(req.GET['ralo']) rahi = float(req.GET['rahi']) declo = float(req.GET['declo']) dechi = float(req.GET['dechi']) ver = int(ver) if not ver in catversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) from astrometry.util.fits import fits_table, merge_tables from astrometry.libkd.spherematch import tree_open, tree_search_radec import numpy as np from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between xyz1 = radectoxyz(ralo, declo) xyz2 = radectoxyz(rahi, dechi) xyz = (xyz1 + xyz2) / 2. xyz /= np.sqrt(np.sum(xyz**2)) rc, dc = xyztoradec(xyz) rc = rc[0] dc = dc[0] rad = degrees_between(rc, dc, ralo, declo) ''' startree -i /project/projectdirs/desi/target/catalogs/targets-dr4-0.20.0.fits -o data/targets-dr4-0.20.0.kd.fits -P -k -T ''' TT = [] for fn in cats: kd = tree_open(fn) I = tree_search_radec(kd, rc, dc, rad) print('Matched', len(I), 'from', fn) if len(I) == 0: continue T = fits_table(fn, rows=I) TT.append(T) if len(TT) == 0: return HttpResponse(json.dumps(dict(rd=[], name=[])), content_type='application/json') T = merge_tables(TT, columns='fillzero') if bgs: T.cut(T.bgs_target > 0) if bright: T.cut(np.logical_or(T.bgs_target > 0, T.mws_target > 0)) if dark: T.cut(T.desi_target > 0) names = None colors = None if color_name_func is not None: names, colors = color_name_func(T) if sky: fluxes = [ dict(g=float(g), r=float(r), z=float(z)) for (g, r, z) in zip(T.apflux_g[:, 0], T.apflux_r[:, 0], T.apflux_z[:, 0]) ] nobs = None else: fluxes = [ dict(g=float(g), r=float(r), z=float(z), W1=float(W1), W2=float(W2)) for (g, r, z, W1, W2) in zip(T.flux_g, T.flux_r, T.flux_z, T.flux_w1, T.flux_w2) ] nobs = [ dict(g=int(g), r=int(r), z=int(z)) for g, r, z in zip(T.nobs_g, T.nobs_r, T.nobs_z) ], rtn = dict( rd=[(t.ra, t.dec) for t in T], targetid=[int(t) for t in T.targetid], fluxes=fluxes, ) if names is not None: rtn.update(name=names) if colors is not None: rtn.update(color=colors) if nobs is not None: rtn.update(nobs=nobs) return HttpResponse(json.dumps(rtn), content_type='application/json')
def in_footprint(parent, nside=2048, dr='dr9'): """Find all galaxies in the DESI footprint. """ import time import healpy as hp import legacyhalos.misc #tiles = SGA.io.read_desi_tiles(verbose=verbose) #indesi = SGA.misc.is_point_in_desi(tiles, parent['RA'], parent['DEC']).astype(bool) parentpix = legacyhalos.misc.radec2pix(nside, parent['RA'], parent['DEC']) #parentpix = np.hstack((parentpix, hp.pixelfunc.get_all_neighbours(nside, parentpix, nest=True).flatten())) drdir = os.path.join(sample_dir(), dr) bands = ('g', 'r', 'z') camera = ('90prime', 'mosaic', 'decam') indesi = dict() for cam in camera: for band in bands: indesi.update( {'{}_{}'.format(cam, band): np.zeros(len(parent), dtype=bool)}) #indesi = np.zeros(len(parent), dtype=bool) t0 = time.time() for cam, radius in zip(camera, (0.44, 0.21, 0.17)): if False: from astrometry.libkd.spherematch import trees_match, tree_open kdccds = tree_open( os.path.join(drdir, 'survey-ccds-{}-{}.kd.fits'.format(cam, dr))) I, J, dd = trees_match(kdparent, kdccds, np.radians(radius)) #, nearest=True) else: ccdsfile = os.path.join( drdir, 'survey-ccds-{}-{}.kd.fits'.format(cam, dr)) ccds = fitsio.read(ccdsfile) ccds = ccds[ccds['ccd_cuts'] == 0] print('Read {} CCDs from {}'.format(len(ccds), ccdsfile)) for band in bands: ww = ccds['filter'] == band if np.sum(ww) > 0: # add the neighboring healpixels to protect against edge effects ccdpix = legacyhalos.misc.radec2pix( nside, ccds['ra'][ww], ccds['dec'][ww]) ccdpix = np.hstack( (ccdpix, hp.pixelfunc.get_all_neighbours(nside, ccdpix, nest=True).flatten())) if np.sum( ccdpix == -1 ) > 0: # remove the "no neighbors" healpixel, if it exists ccdpix = np.delete(ccdpix, np.where(ccdpix == -1)[0]) I = np.isin(parentpix, ccdpix) indesi['{}_{}'.format(cam, band)][I] = True else: I = [False] #print('Found {} galaxies in {} {} footprint in {:.1f} sec'.format(np.sum(I), cam, time.time() - t0)) print(' Found {} galaxies in {} {} footprint.'.format( np.sum(I), cam, band)) print('Total time to find galaxies in footprint = {:.1f} sec'.format( time.time() - t0)) parent['IN_FOOTPRINT_NORTH'] = indesi['90prime_g'] | indesi[ '90prime_r'] | indesi['mosaic_z'] parent['IN_FOOTPRINT_NORTH_GRZ'] = indesi['90prime_g'] & indesi[ '90prime_r'] & indesi['mosaic_z'] parent['IN_FOOTPRINT_SOUTH'] = indesi['decam_g'] | indesi[ 'decam_r'] | indesi['decam_z'] parent['IN_FOOTPRINT_SOUTH_GRZ'] = indesi['decam_g'] & indesi[ 'decam_r'] & indesi['decam_z'] parent['IN_FOOTPRINT'] = parent['IN_FOOTPRINT_NORTH'] | parent[ 'IN_FOOTPRINT_SOUTH'] parent['IN_FOOTPRINT_GRZ'] = parent['IN_FOOTPRINT_NORTH_GRZ'] | parent[ 'IN_FOOTPRINT_SOUTH_GRZ'] #plt.scatter(parent['RA'], parent['DEC'], s=1) #plt.scatter(parent['RA'][indesi], parent['DEC'][indesi], s=1) #plt.xlim(360, 0) #plt.show() #bb = parent[parent['IN_FOOTPRINT_NORTH_GRZ'] & parent['IN_FOOTPRINT_SOUTH_GRZ']] #plt.scatter(bb['RA'], bb['DEC'], s=1) #plt.xlim(300, 90) ; plt.ylim(30, 36) #plt.axhline(y=32.375, color='k') #plt.xlabel('RA') ; plt.ylabel('Dec') #plt.show() print( ' Identified {}/{} ({:.2f}%) galaxies inside and {}/{} ({:.2f}%) galaxies outside the DESI footprint.' .format(np.sum(parent['IN_FOOTPRINT']), len(parent), 100 * np.sum(parent['IN_FOOTPRINT']) / len(parent), np.sum(~parent['IN_FOOTPRINT']), len(parent), 100 * np.sum(~parent['IN_FOOTPRINT']) / len(parent))) return parent
T = fits_table(opt.abellcat) for i in range(len(T)): if not plot.wcs.is_inside(T.ra[i], T.dec[i]): continue ann.add_target(T.ra[i], T.dec[i], 'Abell %i' % T.aco[i]) if opt.t2cat: from astrometry.libkd.spherematch import tree_open, tree_close, tree_build_radec, tree_free, trees_match from astrometry.libkd import spherematch_c from astrometry.util.starutil_numpy import deg2dist, xyztoradec import numpy as np import sys wcs = plot.wcs rc,dc = wcs.get_center() rr = wcs.get_radius() kd = tree_open(opt.t2cat) kd2 = tree_build_radec(np.array([rc]), np.array([dc])) r = deg2dist(rr) I,J,d = trees_match(kd, kd2, r, permuted=False) # HACK I2,J,d = trees_match(kd, kd2, r) xyz = spherematch_c.kdtree_get_positions(kd, I) tree_free(kd2) tree_close(kd) tra,tdec = xyztoradec(xyz) T = fits_table(opt.t2cat, hdu=6) for r,d,t1,t2,t3 in zip(tra,tdec, T.tyc1[I2], T.tyc2[I2], T.tyc3[I2]): if not plot.wcs.is_inside(r, d): continue ann.add_target(r, d, 'Tycho-2 %i-%i-%i' % (t1,t2,t3))
def main(): import argparse parser = argparse.ArgumentParser( description='This script creates small self-contained data sets that ' 'are useful for test cases of the pipeline codes.') parser.add_argument('ccds', help='CCDs table describing region to grab') parser.add_argument('outdir', help='Output directory name') parser.add_argument('brick', help='Brick containing these images') parser.add_argument('--survey-dir', type=str, default=None) parser.add_argument('--cache-dir', type=str, default=None, help='Directory to search for cached files') parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image') parser.add_argument('--wise-wcs-hdu', help='For WISE outputs, the HDU to read the WCS from in the file given by --wise.', type=int, default=0) parser.add_argument('--fpack', action='store_true', default=False) parser.add_argument('--gzip', action='store_true', default=False) parser.add_argument('--pad', action='store_true', default=False, help='Keep original image size, but zero out pixels outside ROI') args = parser.parse_args() v = 'SKY_TEMPLATE_DIR' if v in os.environ: del os.environ[v] C = fits_table(args.ccds) print(len(C), 'CCDs in', args.ccds) C.camera = np.array([c.strip() for c in C.camera]) survey = LegacySurveyData(cache_dir=args.cache_dir, survey_dir=args.survey_dir) if ',' in args.brick: ra,dec = args.brick.split(',') ra = float(ra) dec = float(dec) fakebricks = fits_table() fakebricks.brickname = np.array([('custom-%06i%s%05i' % (int(1000*ra), 'm' if dec < 0 else 'p', int(1000*np.abs(dec))))]) fakebricks.ra = np.array([ra]) fakebricks.dec = np.array([dec]) bricks = fakebricks outbricks = bricks else: bricks = survey.get_bricks_readonly() outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])] assert(len(outbricks) == 1) outsurvey = LegacySurveyData(survey_dir = args.outdir) trymakedirs(args.outdir) outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz')) targetwcs = wcs_for_brick(outbricks[0]) H,W = targetwcs.shape tycho2fn = survey.find_file('tycho2') kd = tree_open(tycho2fn, 'stars') radius = 1. rc,dc = targetwcs.radec_center() I = tree_search_radec(kd, rc, dc, radius) print(len(I), 'Tycho-2 stars within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (rc,dc)) # Read only the rows within range. tycho = fits_table(tycho2fn, rows=I) del kd print('Read', len(tycho), 'Tycho-2 stars') ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec) #margin = 100 #tycho.cut(ok * (tx > -margin) * (tx < W+margin) * # (ty > -margin) * (ty < H+margin)) print('Cut to', len(tycho), 'Tycho-2 stars within brick') del ok,tx,ty #tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz')) f,tfn = tempfile.mkstemp(suffix='.fits') os.close(f) tycho.writeto(tfn) outfn = os.path.join(args.outdir, 'tycho2.kd.fits') cmd = 'startree -i %s -o %s -P -k -n stars -T' % (tfn, outfn) print(cmd) rtn = os.system(cmd) assert(rtn == 0) os.unlink(tfn) from legacypipe.gaiacat import GaiaCatalog gcat = GaiaCatalog() # from ps1cat.py: wcs = targetwcs step=100. margin=10. # Grid the CCD in pixel space W,H = wcs.get_width(), wcs.get_height() xx,yy = np.meshgrid( np.linspace(1-margin, W+margin, 2+int((W+2*margin)/step)), np.linspace(1-margin, H+margin, 2+int((H+2*margin)/step))) # Convert to RA,Dec and then to unique healpixes ra,dec = wcs.pixelxy2radec(xx.ravel(), yy.ravel()) healpixes = set() for r,d in zip(ra,dec): healpixes.add(gcat.healpix_for_radec(r, d)) for hp in healpixes: hpcat = gcat.get_healpix_catalog(hp) ok,xx,yy = wcs.radec2pixelxy(hpcat.ra, hpcat.dec) onccd = np.flatnonzero((xx >= 1.-margin) * (xx <= W+margin) * (yy >= 1.-margin) * (yy <= H+margin)) hpcat.cut(onccd) if len(hpcat): outfn = os.path.join(args.outdir, 'gaia', 'chunk-%05d.fits' % hp) trymakedirs(os.path.join(args.outdir, 'gaia')) hpcat.writeto(outfn) outccds = C.copy() cols = outccds.get_columns() for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1', 'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1', 'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver', 'psfplver' ]: if c in cols: outccds.delete_column(c) outccds.image_hdu[:] = 1 # Convert to list to avoid truncating filenames outccds.image_filename = [fn for fn in outccds.image_filename] for iccd,ccd in enumerate(C): decam = (ccd.camera.strip() == 'decam') bok = (ccd.camera.strip() == '90prime') im = survey.get_image_object(ccd) print('Got', im) if survey.cache_dir is not None: im.check_for_cached_files(survey) slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1)) psfkwargs = dict(pixPsf=True, gaussPsf=False, hybridPsf=False, normalizePsf=False) tim = im.get_tractor_image(slc, pixPsf=True, subsky=False, nanomaggies=False, no_remap_invvar=True, old_calibs_ok=True) print('Tim:', tim.shape) psfrow = psfhdr = None if args.pad: psf = im.read_psf_model(0, 0, w=im.width, h=im.height, **psfkwargs) psfex = psf.psfex else: psf = tim.getPsf() psfex = psf.psfex # Did the PSF model come from a merged file? for fn in [im.merged_psffn, im.psffn] + im.old_merged_psffns: if not os.path.exists(fn): continue T = fits_table(fn) I, = np.nonzero((T.expnum == im.expnum) * np.array([c.strip() == im.ccdname for c in T.ccdname])) if len(I) != 1: continue psfrow = T[I] x0 = ccd.ccd_x0 y0 = ccd.ccd_y0 psfrow.polzero1[0] -= x0 psfrow.polzero2[0] -= y0 #psfhdr = fitsio.read_header(im.merged_psffn) break psfex.fwhm = tim.psf_fwhm #### HACK #psfrow = None assert(psfrow is not None) if psfrow is not None: print('PSF row:', psfrow) #else: # print('PSF:', psf) # print('PsfEx:', psfex) skyrow = skyhdr = None if args.pad: primhdr = fitsio.read_header(im.imgfn) imghdr = fitsio.read_header(im.imgfn, hdu=im.hdu) sky = im.read_sky_model(splinesky=True, primhdr=primhdr, imghdr=imghdr) #skyhdr = fitsio.read_header(im.splineskyfn) #msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True) else: sky = tim.getSky() # Did the sky model come from a merged file? #msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True) print('merged skyfn:', im.merged_skyfn) print('single skyfn:', im.skyfn) print('old merged skyfns:', im.old_merged_skyfns) for fn in [im.merged_skyfn, im.skyfn] + im.old_merged_skyfns: if not os.path.exists(fn): continue T = fits_table(fn) I, = np.nonzero((T.expnum == im.expnum) * np.array([c.strip() == im.ccdname for c in T.ccdname])) skyrow = T[I] skyrow.x0[0] = ccd.ccd_x0 skyrow.y0[0] = ccd.ccd_y0 # s_med = skyrow.sky_med[0] # s_john = skyrow.sky_john[0] # skyhdr = fitsio.read_header(fn) assert(skyrow is not None) ### HACK #skyrow = None if skyrow is not None: print('Sky row:', skyrow) else: print('Sky:', sky) # Output filename format: fn = ccd.image_filename.strip() ccd.image_filename = os.path.join(os.path.dirname(fn), '%s.%s.fits' % (os.path.basename(fn).split('.')[0], ccd.ccdname.strip())) outim = outsurvey.get_image_object(ccd) print('Output image:', outim) print('Image filename:', outim.imgfn) trymakedirs(outim.imgfn, dir=True) imgdata = tim.getImage() ivdata = tim.getInvvar() # Since we remap DQ codes (always with Mosaic and Bok, sometimes with DECam), # re-read from the FITS file rather than using tim.dq. print('Reading data quality from', im.dqfn, 'hdu', im.hdu) dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice) print('Tim shape:', tim.shape, 'Slice', tim.slice) print('image shape:', imgdata.shape, 'iv', ivdata.shape, 'DQ', dqdata.shape) from collections import Counter dqvals = Counter(dqdata.ravel()) print('DQ pixel counts:') for k,n in dqvals.most_common(): print(' 0x%x' % k, ':', n) if args.pad: # Create zero image of full size, copy in data. fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype) fullsize[slc] = imgdata imgdata = fullsize fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype) fullsize[slc] = dqdata dqdata = fullsize fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype) fullsize[slc] = ivdata ivdata = fullsize else: # Adjust the header WCS by x0,y0 crpix1 = tim.hdr['CRPIX1'] crpix2 = tim.hdr['CRPIX2'] tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0 tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0 # Add image extension to filename # fitsio doesn't compress .fz by default, so drop .fz suffix #outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits') if args.gzip: outim.imgfn = outim.imgfn.replace('.fits', '.fits.gz') #outim.wtfn = outim.wtfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.wtfn = outim.wtfn.replace('.fits.fz', '.fits') if args.gzip: outim.wtfn = outim.wtfn.replace('.fits', '.fits.gz') if outim.dqfn is not None: #outim.dqfn = outim.dqfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.dqfn = outim.dqfn.replace('.fits.fz', '.fits') if args.gzip: outim.dqfn = outim.dqfn.replace('.fits', '.fits.gz') if bok: outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname) ccdfn = outim.imgfn ccdfn = ccdfn.replace(outsurvey.get_image_dir(), '') if ccdfn.startswith('/'): ccdfn = ccdfn[1:] outccds.image_filename[iccd] = ccdfn print('Changed output filenames to:') print(outim.imgfn) print(outim.dqfn) ofn = outim.imgfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fits = fitsio.FITS(ofn, 'rw', clobber=True) fits.write(None, header=tim.primhdr) fits.write(imgdata, header=tim.hdr, extname=ccd.ccdname) fits.close() if args.fpack: cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) h,w = tim.shape if not args.pad: outccds.width[iccd] = w outccds.height[iccd] = h outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0 outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0 wcs = Tan(*[float(x) for x in [ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2, ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]]) if args.pad: subwcs = wcs else: subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h) outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center() print('Weight filename:', outim.wtfn) wfn = outim.wtfn trymakedirs(wfn, dir=True) ofn = wfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fits = fitsio.FITS(ofn, 'rw', clobber=True) fits.write(None, header=tim.primhdr) fits.write(ivdata, header=tim.hdr, extname=ccd.ccdname) fits.close() if args.fpack: cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) if outim.dqfn is not None: print('DQ filename', outim.dqfn) trymakedirs(outim.dqfn, dir=True) ofn = outim.dqfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fits = fitsio.FITS(ofn, 'rw', clobber=True) fits.write(None, header=tim.primhdr) fits.write(dqdata, header=tim.hdr, extname=ccd.ccdname) fits.close() if args.fpack: cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) psfout = outim.psffn #if psfrow: # psfout = outim.merged_psffn print('PSF output filename:', psfout) trymakedirs(psfout, dir=True) if psfrow: psfrow.writeto(psfout, primhdr=psfhdr) else: print('Writing PsfEx:', psfout) psfex.writeto(psfout) # update header F = fitsio.FITS(psfout, 'rw') F[0].write_keys([dict(name='EXPNUM', value=ccd.expnum), dict(name='PLVER', value=psf.plver), dict(name='PROCDATE', value=psf.procdate), dict(name='PLPROCID', value=psf.plprocid),]) F.close() skyout = outim.skyfn #if skyrow: # skyout = outim.merged_splineskyfn print('Sky output filename:', skyout) trymakedirs(skyout, dir=True) if skyrow is not None: skyrow.writeto(skyout, primhdr=skyhdr) else: primhdr = fitsio.FITSHDR() primhdr['PLVER'] = sky.plver primhdr['PLPROCID'] = sky.plprocid primhdr['PROCDATE'] = sky.procdate primhdr['EXPNUM'] = ccd.expnum primhdr['IMGDSUM'] = sky.datasum primhdr['S_MED'] = s_med primhdr['S_JOHN'] = s_john sky.write_fits(skyout, primhdr=primhdr) # HACK -- check result immediately. outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz')) outsurvey.ccds = None outC = outsurvey.get_ccds_readonly() occd = outC[iccd] outim = outsurvey.get_image_object(occd) print('Got output image:', outim) otim = outim.get_tractor_image(pixPsf=True, hybridPsf=True, old_calibs_ok=True) print('Got output tim:', otim) outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz')) # WISE if args.wise is not None: from wise.forcedphot import unwise_tiles_touching_wcs from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs, get_unwise_tractor_image, get_unwise_tile_dir) # Read WCS... print('Reading TAN wcs header from', args.wise, 'HDU', args.wise_wcs_hdu) targetwcs = Tan(args.wise, args.wise_wcs_hdu) tiles = unwise_tiles_touching_wcs(targetwcs) print('Cut to', len(tiles), 'unWISE tiles') H,W = targetwcs.shape r,d = targetwcs.pixelxy2radec(np.array([1, W, W/2, W/2]), np.array([H/2, H/2, 1, H ])) roiradec = [r[0], r[1], d[2], d[3]] unwise_dir = os.environ['UNWISE_COADDS_DIR'] wise_out = os.path.join(args.outdir, 'images', 'unwise') print('Will write WISE outputs to', wise_out) unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR'] wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr') print('Will write WISE time-resolved outputs to', wise_tr_out) trymakedirs(wise_tr_out) W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_atlas.fits')) print('Read', len(W), 'time-resolved WISE coadd tiles') W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id])) print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth') # Write the time-resolved index subset. W.writeto(os.path.join(wise_tr_out, 'time_resolved_atlas.fits')) # this ought to be enough for anyone =) _,Nepochs = W.epoch_bitmask.shape print('N epochs in time-resolved atlas:', Nepochs) wisedata = [] # full depth for band in [1,2,3,4]: wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band, True)) # time-resolved for band in [1,2]: # W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2) bitmask = (1 << (band-1)) for e in range(Nepochs): # Which tiles have images for this epoch? I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask) if len(I) == 0: continue print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I]) edir = os.path.join(unwise_tr_dir, 'e%03i' % e) eoutdir = os.path.join(wise_tr_out, 'e%03i' % e) wisedata.append((edir, eoutdir, tiles.coadd_id[I], band, False)) wrote_masks = set() model_dir = os.environ.get('UNWISE_MODEL_SKY_DIR') if model_dir is not None: model_dir_out = os.path.join(args.outdir, 'images', 'unwise-mod') trymakedirs(model_dir_out) for indir, outdir, tiles, band, fulldepth in wisedata: for tile in tiles: wanyband = 'w' tim = get_unwise_tractor_image(indir, tile, band, bandname=wanyband, roiradecbox=roiradec) print('Got unWISE tim', tim) print(tim.shape) if model_dir is not None and fulldepth and band in [1,2]: print('ROI', tim.roi) #0387p575.1.mod.fits fn = '%s.%i.mod.fits' % (tile, band) print('Filename', fn) F = fitsio.FITS(os.path.join(model_dir, fn)) x0,x1,y0,y1 = tim.roi slc = slice(y0,y1),slice(x0,x1) phdr = F[0].read_header() outfn = os.path.join(model_dir_out, fn) for e,extname in [(1,'MODEL'), (2,'SKY')]: pix = F[e][slc] hdr = F[e].read_header() crpix1 = hdr['CRPIX1'] crpix2 = hdr['CRPIX2'] hdr['CRPIX1'] -= x0 hdr['CRPIX2'] -= y0 #print('mod', mod) #print('Model', mod.shape) if e == 1: fitsio.write(outfn, None, clobber=True, header=phdr) fitsio.write(outfn, pix, header=hdr, extname=extname) print('Wrote', outfn) thisdir = get_unwise_tile_dir(outdir, tile) print('Directory for this WISE tile:', thisdir) base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band)) print('Base filename:', base) masked = True mu = 'm' if masked else 'u' imfn = base + 'img-%s.fits' % mu ivfn = base + 'invvar-%s.fits.gz' % mu nifn = base + 'n-%s.fits.gz' % mu nufn = base + 'n-u.fits.gz' #print('WISE image header:', tim.hdr) # Adjust the header WCS by x0,y0 wcs = tim.wcs.wcs tim.hdr['CRPIX1'] = wcs.crpix[0] tim.hdr['CRPIX2'] = wcs.crpix[1] H,W = tim.shape tim.hdr['IMAGEW'] = W tim.hdr['IMAGEH'] = H print('WCS:', wcs) print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2']) trymakedirs(imfn, dir=True) fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True) print('Wrote', imfn) fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True) print('Wrote', ivfn) fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True) print('Wrote', nifn) fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True) print('Wrote', nufn) if not (indir,tile) in wrote_masks: print('Looking for mask file for', indir, tile) # record that we tried this dir/tile combo wrote_masks.add((indir,tile)) for idir in indir.split(':'): tdir = get_unwise_tile_dir(idir, tile) maskfn = 'unwise-%s-msk.fits.gz' % tile fn = os.path.join(tdir, maskfn) print('Mask file:', fn) if os.path.exists(fn): print('Reading', fn) (x0,x1,y0,y1) = tim.roi roislice = (slice(y0,y1), slice(x0,x1)) F = fitsio.FITS(fn)[0] hdr = F.read_header() M = F[roislice] outfn = os.path.join(thisdir, maskfn) fitsio.write(outfn, M, header=tim.hdr, clobber=True) print('Wrote', outfn) break outC = outsurvey.get_ccds_readonly() for iccd,ccd in enumerate(outC): outim = outsurvey.get_image_object(ccd) print('Got output image:', outim) otim = outim.get_tractor_image(pixPsf=True, hybridPsf=True, old_calibs_ok=True) print('Got output tim:', otim)
def run_tiles(X): tiles, tag = X print('Running', tag, '-', len(tiles), 'tiles') # Aaron's file has all images share the boresight CRVAL, so they have large CRPIX values. T = fits_table( '/global/cfs/cdirs/desi/users/ameisner/GFA/gfa_reduce_etc/gfa_wcs+focus.bigtan-zenith.fits' ) Nbright = 10 tiles_ann = fits_table() tiles_ann.index = tiles.index gfa_regions = [] maxr = 0. #t.cd[0,0], t.cd[0,1], t.cd[1,0], t.cd[1,1], for t in T: wcs = Tan(0., 0., t.crpix[0], t.crpix[1], t.cd[0, 0], t.cd[1, 0], t.cd[0, 1], t.cd[1, 1], float(t.naxis[0]), float(t.naxis[1])) ctype = t.extname[:5] cnum = int(t.extname[5]) h, w = wcs.shape x, y = [1, 1, w, w, 1], [1, h, h, 1, 1] r, d = wcs.pixelxy2radec(x, y) dists = degrees_between(0., 0., r, d) maxr = max(maxr, max(dists)) # x0, y0, x1, y1 rois = [] if ctype == 'FOCUS': # add the two half-chips. # wcs.get_subimage moves the CRPIX, but leave CRVAL unchanged, so tx,ty still work unchanged. # Aaron's WCS templates correct for the overscans #wcs_subs.append((cstr, cnum, 'a', wcs.get_subimage(0, 0, 1024, h))) #wcs_subs.append((cstr, cnum, 'b', wcs.get_subimage(1024, 0, 1024, h))) #all_sub_wcs[(cstr, cnum, 1)] = (tx, ty, wcs.get_subimage(50, 0, 1024, 1032)) #all_sub_wcs[(cstr, cnum, 2)] = (tx, ty, wcs.get_subimage(1174, 0, 1024, 1032)) # Add (negative) margin for donut size and telescope pointing uncertainty. # ~10" for donuts and ~10" for telescope pointing #margin = 100 #wcs_subs.append((cstr, cnum, 'a_margin', wcs.get_subimage(margin, margin, 1024-2*margin, h-2*margin))) #wcs_subs.append((cstr, cnum, 'b_margin', wcs.get_subimage(1024+margin, margin, 1024-2*margin, h-2*margin))) # Also add a positive margin for bright-star reflections off filters #margin = 125 #wcs_subs.append((cstr, cnum, 'expanded', wcs.get_subimage(-margin, -margin, w+2*margin, h+2*margin))) rois.append(('a', 0, 0, 1024, h)) rois.append(('b', 1024, 0, 2048, h)) margin = 100 rois.append( ('a_margin', margin, margin, 1024 - margin, h - margin)) rois.append( ('b_margin', 1024 + margin, margin, 2048 - margin, h - margin)) margin = 125 rois.append(('expanded', -margin, -margin, w + margin, h + margin)) else: # Guide chips include overscan pixels -- including a blank region in the middle. #print(cstr,cnum, 'shape', wcs.shape) #wcs_subs.append((cstr, cnum, 'ccd', wcs)) rois.append(('ccd', 0, 0, w, h)) # Add expanded GUIDE chips -- 25" margin / 0.2"/pix = 125 pix margin = 125 #wcs_subs.append((cstr, cnum, 'expanded', wcs.get_subimage(-margin, -margin, w+2*margin, h+2*margin))) rois.append(('expanded', -margin, -margin, w + margin, h + margin)) margin = 125 expwcs = wcs.get_subimage(-margin, -margin, w + 2 * margin, h + 2 * margin) newrois = [] for tag, x0, y0, x1, y1 in rois: name = '%s_%i_%s' % (ctype.lower(), cnum, tag) arr = np.zeros(len(tiles), (np.float32, Nbright)) tiles_ann.set('brightest_' + name, arr) # (the rois have zero-indexed x0,y0, and non-inclusive x1,y1!) newrois.append((name, arr, 1 + x0, 1 + y0, x1, y1)) gfa_regions.append((ctype, cnum, wcs, expwcs, newrois)) # DEBUG WCS # s = [] # for ctype,cnum,wcs,expwcs,rois in gfa_regions: # WCS = wcs # h,w = WCS.shape # #print('Expwcs:', w, 'x', h) # x = [1,1,w,w,1] # y = [1,h,h,1,1] # r,d = WCS.pixelxy2radec(x, y) # p = ','.join(['%.4f,%.4f' % (rr,dd) for rr,dd in zip(r,d)]) # s.append(p) # s = ';'.join(s) # print('http://legacysurvey.org/viewer/?ra=0&dec=0&poly='+s) # sys.exit(0) gaia = CachingGaiaCatalog(columns=[ 'ra', 'dec', 'phot_g_mean_mag', 'phot_bp_mean_mag', 'phot_rp_mean_mag', 'astrometric_excess_noise', 'astrometric_params_solved', 'source_id', 'pmra_error', 'pmdec_error', 'parallax_error', 'ra_error', 'dec_error', 'pmra', 'pmdec', 'parallax', 'ref_epoch' ]) tyc2fn = '/global/cfs/cdirs/cosmo/staging/tycho2/tycho2.kd.fits' tycho_kd = tree_open(tyc2fn) tycho_cat = fits_table(tyc2fn) maxrad = maxr * 1.05 for itile, tile in enumerate(tiles): #if not tile.in_imaging: # continue #if tile.centerid % 10 == 0: #print('tile program', tile.program, 'pass', tile.get('pass'), 'id', tile.centerid, gaia.get_healpix_tree.cache_info()) I = tree_search_radec(tycho_kd, tile.ra, tile.dec, maxrad) tycstars = tycho_cat[I] fix_tycho(tycstars) for cstr, cname, chipwcs, bigwcs, rois in gfa_regions: h, w = chipwcs.shape chipwcs.set_crval(tile.ra, tile.dec) bigwcs.set_crval(tile.ra, tile.dec) gstars = gaia.get_catalog_in_wcs(bigwcs, step=1032, margin=0) fix_gaia(gstars) bh, bw = bigwcs.shape ok, x, y = bigwcs.radec2pixelxy(tycstars.ra, tycstars.dec) tstars = tycstars[(x >= 1) * (y >= 1) * (x <= bw) * (y <= bh)] #print('Tile', tile.program, 'p', tile.get('pass'), tile.centerid, # 'GFA', cstr, cname, ':', len(gstars), 'Gaia stars', len(tstars), 'Tycho-2 stars') if len(gstars) + len(tstars) == 0: print('No stars in tile centerid', tile.centerid, 'chip', name) continue if len(gstars) > 0 and len(tstars) > 0: merge_gaia_tycho(gstars, tstars) stars = merge_tables([gstars, tstars], columns='fillzero') elif len(tstars) > 0: stars = tstars else: stars = gstars ok, x, y = chipwcs.radec2pixelxy(stars.ra, stars.dec) for name, arr, x0, y0, x1, y1 in rois: J = np.flatnonzero( (x >= x0) * (x <= x1) * (y >= y0) * (y <= y1)) mags = stars.mag[J] #print(' ', len(mags), 'in name') K = np.argsort(mags) K = K[:Nbright] arr[itile, :len(K)] = mags[K] #tiles.add_columns_from(tiles_ann) return tiles_ann
def read_tycho2(survey, targetwcs, bands): from astrometry.libkd.spherematch import tree_open, tree_search_radec from legacypipe.survey import GaiaSource tycho2fn = survey.find_file('tycho2') radius = 1. ra, dec = targetwcs.radec_center() # John added the "isgalaxy" flag 2018-05-10, from the Metz & # Geffert (04) catalog. # Eddie added the "zguess" column 2019-03-06, by matching with # 2MASS and estimating z based on APASS. # The "tycho2.kd.fits" file read here was produced by: # # fitscopy ~schlafly/legacysurvey/tycho-isgalaxyflag-2mass.fits"[col \ # tyc1;tyc2;tyc3;ra;dec;sigma_ra;sigma_dec;mean_ra;mean_dec;pm_ra;pm_dec; \ # sigma_pm_ra;sigma_pm_dec;epoch_ra;epoch_dec;mag_bt;mag_vt;mag_hp; \ # isgalaxy;Jmag;Hmag;Kmag,zguess]" /tmp/tycho2-astrom.fits # startree -P -k -n stars -T -i /tmp/tycho2-astrom.fits \ # -o /global/project/projectdirs/cosmo/staging/tycho2/tycho2.kd.fits kd = tree_open(tycho2fn, 'stars') I = tree_search_radec(kd, ra, dec, radius) debug(len(I), 'Tycho-2 stars within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (ra, dec)) if len(I) == 0: return None # Read only the rows within range. tycho = fits_table(tycho2fn, rows=I) del kd if 'isgalaxy' in tycho.get_columns(): tycho.cut(tycho.isgalaxy == 0) debug('Cut to', len(tycho), 'Tycho-2 stars on isgalaxy==0') else: print('Warning: no "isgalaxy" column in Tycho-2 catalog') tycho.ref_cat = np.array(['T2'] * len(tycho)) # tyc1: [1,9537], tyc2: [1,12121], tyc3: [1,3] tycho.ref_id = (tycho.tyc1.astype(np.int64) * 1000000 + tycho.tyc2.astype(np.int64) * 10 + tycho.tyc3.astype(np.int64)) with np.errstate(divide='ignore'): # In our Tycho-2 catalog, sigma_pm_* are in *arcsec/yr*, Gaia is in mas/yr. tycho.pmra_ivar = 1. / (tycho.sigma_pm_ra * 1000.)**2 tycho.pmdec_ivar = 1. / (tycho.sigma_pm_dec * 1000.)**2 tycho.ra_ivar = 1. / tycho.sigma_ra**2 tycho.dec_ivar = 1. / tycho.sigma_dec**2 tycho.rename('pm_ra', 'pmra') tycho.rename('pm_dec', 'pmdec') for c in ['pmra', 'pmdec', 'pmra_ivar', 'pmdec_ivar']: X = tycho.get(c) X[np.logical_not(np.isfinite(X))] = 0. tycho.mag = tycho.mag_vt # Patch missing mag values... tycho.mag[tycho.mag == 0] = tycho.mag_hp[tycho.mag == 0] tycho.mag[tycho.mag == 0] = tycho.mag_bt[tycho.mag == 0] # Use zguess tycho.mask_mag = tycho.mag with np.errstate(invalid='ignore'): I = np.flatnonzero( np.isfinite(tycho.zguess) * (tycho.zguess + 1. < tycho.mag)) tycho.mask_mag[I] = tycho.zguess[I] # Per discussion in issue #306 -- cut on mag < 13. # This drops only 13k/2.5M stars. tycho.cut(tycho.mask_mag < 13.) tycho.radius = mask_radius_for_mag(tycho.mask_mag) tycho.keep_radius = 2. * tycho.radius for c in [ 'tyc1', 'tyc2', 'tyc3', 'mag_bt', 'mag_vt', 'mag_hp', 'mean_ra', 'mean_dec', 'sigma_pm_ra', 'sigma_pm_dec', 'sigma_ra', 'sigma_dec' ]: tycho.delete_column(c) # add Gaia-style columns # No parallaxes in Tycho-2 tycho.parallax = np.zeros(len(tycho), np.float32) # Tycho-2 "supplement" stars, from Hipparcos and Tycho-1 catalogs, have # ref_epoch = 0. Fill in with the 1991.25 epoch of those catalogs. tycho.epoch_ra[tycho.epoch_ra == 0] = 1991.25 tycho.epoch_dec[tycho.epoch_dec == 0] = 1991.25 # Tycho-2 has separate epoch_ra and epoch_dec. # Move source to the mean epoch. tycho.ref_epoch = (tycho.epoch_ra + tycho.epoch_dec) / 2. cosdec = np.cos(np.deg2rad(tycho.dec)) tycho.ra += (tycho.ref_epoch - tycho.epoch_ra) * tycho.pmra / 3600. / cosdec tycho.dec += (tycho.ref_epoch - tycho.epoch_dec) * tycho.pmdec / 3600. # Tycho-2 proper motions are in arcsec/yr; Gaia are mas/yr. tycho.pmra *= 1000. tycho.pmdec *= 1000. # We already cut on John's "isgalaxy" flag tycho.pointsource = np.ones(len(tycho), bool) # phot_g_mean_mag -- for initial brightness of source tycho.phot_g_mean_mag = tycho.mag tycho.delete_column('epoch_ra') tycho.delete_column('epoch_dec') tycho.istycho = np.ones(len(tycho), bool) tycho.isbright = np.ones(len(tycho), bool) tycho.ismedium = np.ones(len(tycho), bool) tycho.donotfit = np.zeros(len(tycho), bool) tycho.sources = np.empty(len(tycho), object) if bands is not None: for i, t in enumerate(tycho): tycho.sources[i] = GaiaSource.from_catalog(t, bands) return tycho
def read_tycho2(survey, targetwcs): from astrometry.libkd.spherematch import tree_open, tree_search_radec tycho2fn = survey.find_file('tycho2') radius = 1. ra, dec = targetwcs.radec_center() # John added the "isgalaxy" flag 2018-05-10, from the Metz & # Geffert (04) catalog. # Eddie added the "zguess" column 2019-03-06, by matching with # 2MASS and estimating z based on APASS. # The "tycho2.kd.fits" file read here was produced by: # # fitscopy ~schlafly/legacysurvey/tycho-isgalaxyflag-2mass.fits"[col \ # tyc1;tyc2;tyc3;ra;dec;sigma_ra;sigma_dec;mean_ra;mean_dec;pm_ra;pm_dec; \ # sigma_pm_ra;sigma_pm_dec;epoch_ra;epoch_dec;mag_bt;mag_vt;mag_hp; \ # isgalaxy;Jmag;Hmag;Kmag,zguess]" /tmp/tycho2-astrom.fits # startree -P -k -n stars -T -i /tmp/tycho2-astrom.fits \ # -o /global/project/projectdirs/cosmo/staging/tycho2/tycho2.kd.fits kd = tree_open(tycho2fn, 'stars') I = tree_search_radec(kd, ra, dec, radius) debug(len(I), 'Tycho-2 stars within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (ra, dec)) if len(I) == 0: return None # Read only the rows within range. tycho = fits_table(tycho2fn, rows=I) del kd if 'isgalaxy' in tycho.get_columns(): tycho.cut(tycho.isgalaxy == 0) debug('Cut to', len(tycho), 'Tycho-2 stars on isgalaxy==0') else: print('Warning: no "isgalaxy" column in Tycho-2 catalog') tycho.ref_cat = np.array(['T2'] * len(tycho)) # tyc1: [1,9537], tyc2: [1,12121], tyc3: [1,3] tycho.ref_id = (tycho.tyc1.astype(np.int64) * 1000000 + tycho.tyc2.astype(np.int64) * 10 + tycho.tyc3.astype(np.int64)) with np.errstate(divide='ignore'): tycho.pmra_ivar = 1. / tycho.sigma_pm_ra**2 tycho.pmdec_ivar = 1. / tycho.sigma_pm_dec**2 tycho.ra_ivar = 1. / tycho.sigma_ra**2 tycho.dec_ivar = 1. / tycho.sigma_dec**2 tycho.rename('pm_ra', 'pmra') tycho.rename('pm_dec', 'pmdec') for c in ['pmra', 'pmdec', 'pmra_ivar', 'pmdec_ivar']: X = tycho.get(c) X[np.logical_not(np.isfinite(X))] = 0. tycho.mag = tycho.mag_vt # Patch missing mag values... tycho.mag[tycho.mag == 0] = tycho.mag_hp[tycho.mag == 0] tycho.mag[tycho.mag == 0] = tycho.mag_bt[tycho.mag == 0] # Per discussion in issue #306 -- cut on mag < 13. This drops only 13k/2.5M stars tycho.cut(tycho.mag < 13.) # See note on gaia.radius above -- don't change the 0.262 to # targetwcs.pixel_scale()! tycho.radius = np.minimum(1800., 150. * 2.5**( (11. - tycho.mag) / 3.)) * 0.262 / 3600. for c in [ 'tyc1', 'tyc2', 'tyc3', 'mag_bt', 'mag_vt', 'mag_hp', 'mean_ra', 'mean_dec', 'sigma_pm_ra', 'sigma_pm_dec', 'sigma_ra', 'sigma_dec' ]: tycho.delete_column(c) # add Gaia-style columns # No parallaxes in Tycho-2 tycho.parallax = np.zeros(len(tycho), np.float32) # Tycho-2 has separate epoch_ra and epoch_dec. # Move source to the mean epoch. tycho.ref_epoch = (tycho.epoch_ra + tycho.epoch_dec) / 2. cosdec = np.cos(np.deg2rad(tycho.dec)) tycho.ra += (tycho.ref_epoch - tycho.epoch_ra) * tycho.pmra / 3600. / cosdec tycho.dec += (tycho.ref_epoch - tycho.epoch_dec) * tycho.pmdec / 3600. # Tycho-2 proper motions are in arcsec/yr; Gaia are mas/yr. tycho.pmra *= 1000. tycho.pmdec *= 1000. # We already cut on John's "isgalaxy" flag tycho.pointsource = np.ones(len(tycho), bool) # phot_g_mean_mag -- for initial brightness of source tycho.phot_g_mean_mag = tycho.mag tycho.delete_column('epoch_ra') tycho.delete_column('epoch_dec') tycho.isbright = np.ones(len(tycho), bool) tycho.ismedium = np.ones(len(tycho), bool) return tycho
def read_large_galaxies(survey, targetwcs, bands, clean_columns=True, max_radius=2.): # Note, max_radius must include the brick radius! from astrometry.libkd.spherematch import tree_open, tree_search_radec galfn = survey.find_file('large-galaxies') if galfn is None: debug('No large-galaxies catalog file') return None radius = max_radius rc, dc = targetwcs.radec_center() debug('Reading', galfn) try: kd = tree_open(galfn, 'stars') except: kd = tree_open(galfn, 'largegals') I = tree_search_radec(kd, rc, dc, radius) debug('%i large galaxies within %.3g deg of RA,Dec (%.3f, %.3f)' % (len(I), radius, rc, dc)) if len(I) == 0: return None # Read only the rows within range. galaxies = fits_table(galfn, rows=I) del kd refcat, preburn = get_large_galaxy_version(galfn) debug('Large galaxies version: "%s", preburned?' % refcat, preburn) if preburn: # SGA ellipse catalog # NOTE: fields such as ref_cat, preburned, etc, already exist in the # "galaxies" catalog read from disk. # The galaxies we want to appear in MASKBITS get # 'islargegalaxy' set. This includes both pre-burned # galaxies, and ones where the preburning failed and we want # to fall back to the SGA-parent ellipse for masking. galaxies.islargegalaxy = ((galaxies.ref_cat == refcat) * (galaxies.sga_id > -1)) # The pre-fit galaxies whose parameters will stay fixed galaxies.freezeparams = (galaxies.preburned * galaxies.freeze) else: # SGA parent catalog galaxies.ref_cat = np.array([refcat] * len(galaxies)) galaxies.islargegalaxy = np.ones(len(galaxies), bool) galaxies.freezeparams = np.zeros(len(galaxies), bool) galaxies.preburned = np.zeros(len(galaxies), bool) galaxies.rename('sga_id', 'ref_id') galaxies.rename('mag_leda', 'mag') # Pre-burned, frozen but non-SGA sources have diam=-1. galaxies.radius = np.maximum(0., galaxies.diam / 2. / 60.) # [degree] galaxies.keep_radius = 2. * galaxies.radius galaxies.sources = np.empty(len(galaxies), object) galaxies.sources[:] = None if bands is not None: galaxies.sources[:] = get_galaxy_sources(galaxies, bands) if clean_columns: keep_columns = [ 'ra', 'dec', 'radius', 'mag', 'ref_cat', 'ref_id', 'ba', 'pa', 'sources', 'islargegalaxy', 'freezeparams', 'keep_radius' ] for c in galaxies.get_columns(): if not c in keep_columns: galaxies.delete_column(c) return galaxies
inds = inds[order] dists = dists[order] ok = np.array_equal(pairs, inds) print('Indices equal:', ok) print('Build uint64 tree...') kd = spherematch.tree_build(ux1) kd.print() data = kd.get_data(np.array([0,3,5]).astype(np.uint32)) assert(data.dtype == np.uint64) print('Kd data:', data.dtype, data) kd.write('kd-u64.fits') kd2 = spherematch.tree_open('kd-u64.fits') data2 = kd2.get_data(np.array([0,3,5]).astype(np.uint32)) assert(data2.dtype == np.uint64) print('Kd data2:', data2.dtype, data2) assert(np.all(data == data2)) del kd del kd2 ### t0 = time() (inds,dists) = spherematch.nearest(x1, x2, r) dt = time() - t0
def cat_targets_drAB(req, ver, cats=None, tag='', bgs=False, sky=False, bright=False, dark=False, color_name_func=desitarget_color_names): ''' color_name_func: function that selects names and colors for targets (eg based on targeting bit values) ''' if cats is None: cats = [] import json ralo = float(req.GET['ralo']) rahi = float(req.GET['rahi']) declo = float(req.GET['declo']) dechi = float(req.GET['dechi']) ver = int(ver) if not ver in catversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) from astrometry.util.fits import fits_table, merge_tables from astrometry.libkd.spherematch import tree_open, tree_search_radec import numpy as np from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between xyz1 = radectoxyz(ralo, declo) xyz2 = radectoxyz(rahi, dechi) xyz = (xyz1 + xyz2)/2. xyz /= np.sqrt(np.sum(xyz**2)) rc,dc = xyztoradec(xyz) rc = rc[0] dc = dc[0] rad = degrees_between(rc, dc, ralo, declo) ''' startree -i /project/projectdirs/desi/target/catalogs/targets-dr4-0.20.0.fits -o data/targets-dr4-0.20.0.kd.fits -P -k -T ''' TT = [] for fn in cats: kd = tree_open(fn) I = tree_search_radec(kd, rc, dc, rad) print('Matched', len(I), 'from', fn) if len(I) == 0: continue T = fits_table(fn, rows=I) TT.append(T) if len(TT) == 0: return HttpResponse(json.dumps(dict(rd=[], name=[])), content_type='application/json') T = merge_tables(TT, columns='fillzero') if bgs: T.cut(T.bgs_target > 0) if bright: T.cut(np.logical_or(T.bgs_target > 0, T.mws_target > 0)) if dark: T.cut(T.desi_target > 0) names = None colors = None if color_name_func is not None: names,colors = color_name_func(T) if sky: fluxes = [dict(g=float(g), r=float(r), z=float(z)) for (g,r,z) in zip(T.apflux_g[:,0], T.apflux_r[:,0], T.apflux_z[:,0])] nobs = None else: fluxes = [dict(g=float(g), r=float(r), z=float(z), W1=float(W1), W2=float(W2)) for (g,r,z,W1,W2) in zip(T.flux_g, T.flux_r, T.flux_z, T.flux_w1, T.flux_w2)] nobs=[dict(g=int(g), r=int(r), z=int(z)) for g,r,z in zip(T.nobs_g, T.nobs_r, T.nobs_z)], rtn = dict(rd=[(t.ra, t.dec) for t in T], targetid=[int(t) for t in T.targetid], fluxes=fluxes, ) if names is not None: rtn.update(name=names) if colors is not None: rtn.update(color=colors) if nobs is not None: rtn.update(nobs=nobs) # Convert targetid to string to prevent rounding errors rtn['targetid'] = [str(s) for s in rtn['targetid']] return HttpResponse(json.dumps(rtn), content_type='application/json')