def no_overlapping_radec(ra,dec, bounds, random_state=None, dist=5.0/3600): '''resamples ra,dec where they are within dist of each other ra,dec -- numpy arrays [degrees] bounds-- list of [ramin,ramax,decmin,decmax] random_state-- np.random.RandomState() object dist-- separation in degrees, sqrt(x^2+y^2) where x=(dec2-dec1), y=cos(avg(dec2,dec1))*(ra2-ra1) ''' log = logging.getLogger('decals_sim') if random_state is None: random_state = np.random.RandomState() # ra,dec indices of just neighbors within "dist" away, just nerest neighbor of those m1, m2, d12 = match_radec(ra.copy(),dec.copy(), ra.copy(),dec.copy(),\ dist, nearest=True,notself=True) cnt = 1 log.info("after iter=%d, have overlapping ra,dec %d/%d", cnt, len(m2),ra.shape[0]) while len(m2) > 0: ra[m2]= random_state.uniform(bounds[0], bounds[1], len(m2)) dec[m2]= random_state.uniform(bounds[2], bounds[3], len(m2)) m1, m2, d12 = match_radec(ra.copy(),dec.copy(), ra.copy(),dec.copy(),\ dist, nearest=True,notself=True) cnt += 1 log.info("after iter=%d, have overlapping ra,dec %d/%d", cnt, len(m2),ra.shape[0]) if cnt > 30: log.error('Crash, could not get non-overlapping ra,dec in 30 iterations') raise ValueError return ra, dec
def all_matches_near(catfn, radius, atlasfn, photdir, outfn, projname, **kwargs): ''' radius in arcsec ''' T = fits_table(atlasfn) print(len(T), 'tiles') M = fits_table(catfn) print(len(M), 'targets') M.set('%s_index' % projname, np.arange(len(M))) I,J,d = match_radec(M.ra, M.dec, T.ra, T.dec, 1.2) #keep = np.zeros(len(M), bool) #keep[I] = True #assert(np.all(keep)) keep = np.zeros(len(T), bool) keep[J] = True T.cut(keep) print(len(T), 'tiles near targets') WW = [] for t in T: fn = os.path.join(photdir, 'phot-%s.fits' % t.coadd_id) if not os.path.exists(fn): print('Does not exist:', fn) continue W = fits_table(fn) print(len(W), 'from', fn) if len(W) == 0: continue I,J,d = match_radec(M.ra, M.dec, W.ra, W.dec, radius/3600., **kwargs) print(len(I), 'matched') if len(I) == 0: continue W.cut(J) MI = M[I] MI.rename('ra', 'ra_%s' % projname) MI.rename('dec', 'dec_%s' % projname) W.add_columns_from(MI) WW.append(W) W = merge_tables(WW) print(len(W), 'total matches') W = W[np.argsort(W.get('%s_index' % projname))] print(len(np.unique(W.objid)), 'unique objids') W.writeto(outfn)
def main(): from thesis_code import timing from astrometry.libkd.spherematch import match_radec parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='DECaLS simulations.') args = parser.parse_args() rand = np.random.RandomState(10) n_ref,n_other=10000,5000 ra_ref = rand.uniform(150.,150.25,n_ref) dec_ref = rand.uniform(20.,20.25,n_ref) ra = rand.uniform(150.,150.25,n_other) dec = rand.uniform(20.,20.25,n_other) i_ref,i_other,ds={},{},{} t1=timing.now() i_ref['astrom'],i_other['astrom'],ds['astrom']= match_radec(ra_ref.copy(),dec_ref.copy(), ra.copy(),dec.copy(), 1./3600) t2=timing.now() #i_ref['n2'],i_other['n2'],ds['n2']= n2_match(ra_ref.copy(),dec_ref.copy(), ra.copy(),dec.copy(), dsmax=5./3600) i_ref['kd'],i_other['kd'],ds['kd']= kdtree(ra_ref.copy(),dec_ref.copy(), ra.copy(),dec.copy(), dsmax=1./3600) t3=timing.now() print 'time for astrom[sec]=',timing.diff(t1,t2) print 'time for kd[sec]=',timing.diff(t2,t3) for key in ['astrom','kd']: print('%s: matched %d/%d' % (key,len(i_ref[key]),len(ra_ref))) i=np.argsort(i_ref[key]) i_ref[key]= i_ref[key][i] i_other[key]= i_other[key][i]
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument("--name1", help="Name for first data set") parser.add_argument("--name2", help="Name for second data set") parser.add_argument("--plot-prefix", default="compare", help='Prefix for plot filenames; default "%default"') parser.add_argument("--match", default=1.0, help="Astrometric cross-match distance in arcsec") parser.add_argument("dir1", help="First directory to compare") parser.add_argument("dir2", help="Second directory to compare") opt = parser.parse_args() ps = PlotSequence(opt.plot_prefix) name1 = opt.name1 if name1 is None: name1 = os.path.basename(opt.dir1) if not len(name1): name1 = os.path.basename(os.path.dirname(opt.dir1)) name2 = opt.name2 if name2 is None: name2 = os.path.basename(opt.dir2) if not len(name2): name2 = os.path.basename(os.path.dirname(opt.dir2)) tt = "Comparing %s to %s" % (name1, name2) # regex for tractor-*.fits catalog filename catre = re.compile("tractor-.*.fits") cat1, cat2 = [], [] for basedir, cat in [(opt.dir1, cat1), (opt.dir2, cat2)]: for dirpath, dirnames, filenames in os.walk(basedir, followlinks=True): for fn in filenames: if not catre.match(fn): print("Skipping", fn, "due to filename") continue fn = os.path.join(dirpath, fn) t = fits_table(fn) print(len(t), "from", fn) cat.append(t) cat1 = merge_tables(cat1, columns="fillzero") cat2 = merge_tables(cat2, columns="fillzero") print("Total of", len(cat1), "from", name1) print("Total of", len(cat2), "from", name2) cat1.cut(cat1.brick_primary) cat2.cut(cat2.brick_primary) print("Total of", len(cat1), "BRICK_PRIMARY from", name1) print("Total of", len(cat2), "BRICK_PRIMARY from", name2) cat1.cut((cat1.decam_anymask[:, 1] == 0) * (cat1.decam_anymask[:, 2] == 0) * (cat1.decam_anymask[:, 4] == 0)) cat2.cut((cat2.decam_anymask[:, 1] == 0) * (cat2.decam_anymask[:, 2] == 0) * (cat2.decam_anymask[:, 4] == 0)) print("Total of", len(cat1), "unmasked from", name1) print("Total of", len(cat2), "unmasked from", name2) I, J, d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match / 3600.0, nearest=True) print(len(I), "matched") matched1 = cat1[I] matched2 = cat2[J]
def match_two_cats(ref_cats_file,test_cats_file): # Set the debugging level lvl = logging.INFO logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout) log = logging.getLogger('__name__') #get lists of tractor cats to compare fns_1= read_lines(ref_cats_file) fns_2= read_lines(test_cats_file) log.info('Comparing tractor catalogues: ') for one,two in zip(fns_1,fns_2): log.info("%s -- %s" % (one,two)) #if fns_1.size == 1: fns_1,fns_2= [fns_1],[fns_2] #object to store concatenated matched tractor cats ref_matched = [] ref_missed = [] test_matched = [] test_missed = [] d_matched= 0. deg2= dict(ref=0.,test=0.,matched=0.) #for cnt,cat1,cat2 in zip(range(len(fns_1)),fns_1,fns_2): for cnt,cat1,cat2 in zip([0],[fns_1[0]],[fns_2[0]]): log.info('Reading %s -- %s' % (cat1,cat2)) ref_tractor = Table(fits.getdata(cat1, 1)) test_tractor = Table(fits.getdata(cat2, 1)) m1, m2, d12 = match_radec(ref_tractor['ra'].copy(), ref_tractor['dec'].copy(),\ test_tractor['ra'].copy(), test_tractor['dec'].copy(), \ 1.0/3600.0) miss1 = np.delete(np.arange(len(ref_tractor)), m1, axis=0) miss2 = np.delete(np.arange(len(test_tractor)), m2, axis=0) log.info('matched %d/%d' % (len(m2),len(test_tractor['ra']))) # Build combined catalogs if len(ref_matched) == 0: ref_matched = ref_tractor[m1] ref_missed = ref_tractor[miss1] test_matched = test_tractor[m2] test_missed = test_tractor[miss2] d_matched= d12 else: ref_matched = vstack((ref_matched, ref_tractor[m1])) ref_missed = vstack((ref_missed, ref_tractor[miss1])) test_matched = vstack((test_matched, test_tractor[m2])) test_missed = vstack((test_missed, test_tractor[miss2])) d_matched= np.concatenate([d_matched, d12]) deg2['ref']+= deg2_lower_limit(ref_tractor['ra'],ref_tractor['dec']) deg2['test']+= deg2_lower_limit(test_tractor['ra'],test_tractor['dec']) deg2['matched']+= deg2_lower_limit(ref_matched['ra'],ref_matched['dec']) return dict(ref_matched = ref_matched, ref_missed = ref_missed, test_matched = test_matched, test_missed = test_missed, d_matched= d_matched, deg2= deg2)
def match_it(cat1, cat2): """cat1,2 are tractor catalogue to match objects between""" # match cats data_1 = targets.read_from_tractor_cat(cat1) data_2 = targets.read_from_tractor_cat(cat2) # deg2 spanned by objects in each data set deg2_decam = deg2_lower_limit(data_1) deg2_bokmos = deg2_lower_limit(data_2) # all the 'all1' objects that have match in 'all2' m1, m2, d12 = match_radec(data_1["ra"], data_1["dec"], data_2["ra"], data_2["dec"], 1.0 / 3600.0, nearest=True) m1_unm = np.delete(np.arange(len(data_1["ra"])), m1, axis=0) m2_unm = np.delete(np.arange(len(data_2["ra"])), m2, axis=0) return data_1, data_2, m1, m2, m1_unm, m2_unm, d12, deg2_decam, deg2_bokmos
def tychoMatch(ra,dec,rad): cat = pyf.open("tycho2-cut.fits") data = cat[1].data RA=data.field('RA') DEC=data.field('DEC') MAG=data.field('MAG') ra1=ra dec1=dec matchrad = rad I1,I2,d = sm.match_radec(ra1,dec1, RA,DEC, matchrad) cat.close() return RA[I2],DEC[I2],MAG[I2]
def on_bricks_dependencies(brick, survey, bricks=None): # Find nearby bricks from earlier brick phases if bricks is None: bricks = survey.get_bricks_readonly() print(len(bricks), 'bricks') bricks = bricks[bricks.brickq < brick.brickq] print(len(bricks), 'from phases before this brickq:', brick.brickq) if len(bricks) == 0: return [] from astrometry.libkd.spherematch import match_radec radius = survey.bricksize * np.sqrt(2.) * 1.01 bricks.cut(np.abs(brick.dec - bricks.dec) < radius) #print(len(bricks), 'within %.2f degree of Dec' % radius) I,J,d = match_radec(brick.ra, brick.dec, bricks.ra, bricks.dec, radius) bricks.cut(J) print(len(bricks), 'within', radius, 'degrees') return bricks
def bricks_touching_wcs(targetwcs, survey=None, B=None, margin=20): ''' Finds LegacySurvey bricks touching a given WCS header object. Parameters ---------- targetwcs : astrometry.util.Tan object or similar The region of sky to search survey : legacypipe.survey.LegacySurveyData object From which the brick table will be retrieved B : FITS table The table of brick objects to search margin : int Margin in pixels around the outside of the WCS Returns ------- A table (subset of B, if given) containing the bricks touching the given WCS region + margin. ''' from astrometry.libkd.spherematch import match_radec if B is None: assert(survey is not None) B = survey.get_bricks_readonly() ra,dec = targetwcs.radec_center() radius = targetwcs.radius() # MAGIC 0.4 degree search radius = # DECam hypot(1024,2048)*0.27/3600 + Brick hypot(0.25, 0.25) ~= 0.35 + margin I,J,d = match_radec(B.ra, B.dec, ra, dec, radius + np.hypot(0.25,0.25)/2. + 0.05) print(len(I), 'bricks nearby') keep = [] for i in I: b = B[i] brickwcs = wcs_for_brick(b) clip = clip_wcs(targetwcs, brickwcs) if len(clip) == 0: print('No overlap with brick', b.brickname) continue keep.append(i) return B[np.array(keep)]
def choose_field2(): import astrometry.libkd.spherematch as spherematch fields = fits_table('window_flist.fits') print len(ngc2000), 'NGC/IC objects' goodngc = [n for n in ngc2000 if n.get('classification', None) == 'Gx'] print len(goodngc), 'NGC galaxies' nra = np.array([n['ra'] for n in goodngc]) ndec = np.array([n['dec'] for n in goodngc]) rad = 8./60. (I,J,dist) = spherematch.match_radec(nra, ndec, fields.ra, fields.dec, rad) #sdss = DR7() for i in np.unique(I): ii = (I == i) n = goodngc[i] isngc = n['is_ngc'] num = n['id'] if (sum(ii) > 10) & (n['dec'] > 2.0): print '<p>' print ('NGC' if isngc else 'IC'), num, 'has', sum(ii), 'fields, and is at RA = ', n['ra'], 'Dec=', n['dec'] print '</p>' ff = fields[J[ii]] print 'rcfi = [', for f in ff: print '(', f.run, ',', f.camcol, ',', f.field, ',', f.incl, ')', print ']' for f in ff: print 'Incl', f.incl, '<br />' print ('<img src="%s" /><br /><br />' % ('http://skyservice.pha.jhu.edu/DR8/ImgCutout/getjpegcodec.aspx?R=%i&C=%i&F=%i&Z=25' % (f.run, f.camcol, f.field))) print '<br />'
def match_two_cats(ref_cats_file, test_cats_file, debug=False): '''return dict containing astropy Table of concatenated tractor cats one Table for matched and missed reference and test objects, each''' # Set the debugging level lvl = logging.INFO logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout) log = logging.getLogger('__name__') #get lists of tractor cats to compare fns_1 = read_lines(ref_cats_file) fns_2 = read_lines(test_cats_file) log.info('Comparing tractor catalogues: ') for one, two in zip(fns_1, fns_2): log.info("%s -- %s" % (one, two)) #if fns_1.size == 1: fns_1,fns_2= [fns_1],[fns_2] #object to store concatenated matched tractor cats ref_matched = [] ref_missed = [] test_matched = [] test_missed = [] d_matched = 0. deg2 = dict(ref=0., test=0., matched=0.) # One catalogue for quick debugging if debug: fns_1, fns_2 = fns_1[:1], fns_2[:1] # Loop over cats for cnt, cat1, cat2 in zip(range(len(fns_1)), fns_1, fns_2): log.info('Reading %s -- %s' % (cat1, cat2)) ref_tractor = Table(fits.getdata(cat1, 1), masked=True) test_tractor = Table(fits.getdata(cat2, 1), masked=True) m1, m2, d12 = match_radec(ref_tractor['ra'].data.copy(), ref_tractor['dec'].data.copy(),\ test_tractor['ra'].data.copy(), test_tractor['dec'].data.copy(), \ 1.0/3600.0) #m1, m2, d12= kdtree_match(ref_tractor['ra'].copy(), ref_tractor['dec'].copy(),\ # test_tractor['ra'].copy(), test_tractor['dec'].copy(),\ # k=1, dsmax=1./3600) print("Matched: %d/%d objects" % (m1.size, len(ref_tractor['ra']))) miss1 = np.delete(np.arange(len(ref_tractor)), m1, axis=0) miss2 = np.delete(np.arange(len(test_tractor)), m2, axis=0) # Build combined catalogs if len(ref_matched) == 0: ref_matched = ref_tractor[m1] ref_missed = ref_tractor[miss1] test_matched = test_tractor[m2] test_missed = test_tractor[miss2] d_matched = d12 else: ref_matched = vstack((ref_matched, ref_tractor[m1]), metadata_conflicts='error') ref_missed = vstack((ref_missed, ref_tractor[miss1]), metadata_conflicts='error') test_matched = vstack((test_matched, test_tractor[m2]), metadata_conflicts='error') test_missed = vstack((test_missed, test_tractor[miss2]), metadata_conflicts='error') d_matched = np.concatenate([d_matched, d12]) deg2['ref'] += deg2_lower_limit(ref_tractor['ra'], ref_tractor['dec']) deg2['test'] += deg2_lower_limit(test_tractor['ra'], test_tractor['dec']) deg2['matched'] += deg2_lower_limit(ref_matched['ra'], ref_matched['dec']) return dict(ref_matched = ref_matched, ref_missed = ref_missed, test_matched = test_matched, test_missed = test_missed), \ dict(d_matched= d_matched, deg2= deg2)
sig1 = 1. / np.sqrt(np.median(invvar[invvar > 0])) sigoff = 3 img += sig1 * sigoff # convert to sigmas img /= sig1 invvar *= sig1**2 H, W = img.shape sz = 22 # Read sources detected in DECam image too T = fits_table(im.sefn, hdu=2) print 'Got', len(T), 'DECam sources' T.about() T.ra, T.dec = wcs.pixelxy2radec(T.x_image, T.y_image) I, J, d = match_radec(S.ra, S.dec, T.ra, T.dec, 1. / 3600., nearest=True) print 'Matched', len(I) # Replace SDSS RA,Dec by DECam RA,Dec S.cut(I) S.ra = T.ra[J] S.dec = T.dec[J] ok, S.x, S.y = wcs.radec2pixelxy(S.ra, S.dec) S.x -= 1 S.y -= 1 S.ix, S.iy = np.round(S.x).astype(int), np.round(S.y).astype(int) S.cut((S.ix >= sz) * (S.iy >= sz) * (S.ix < W - sz) * (S.iy < H - sz)) print len(S), 'SDSS stars in bounds' S.cut(invvar[S.iy, S.ix] > 0)
def _get_sources(run, camcol, field, bandname='r', sdss=None, release='DR7', objs=None, retrieve=True, checkFiles=True, curl=False, roi=None, radecroi=None, radecrad=None, bands=None, badmag=25, nanomaggies=False, getobjs=False, getsourceobjs=False, getobjinds=False, extrabands=None, fixedComposites=False, forcePointSources=False, useObjcType=False, objCuts=True, classmap={}, ellipse=GalaxyShape, cutToPrimary=False): ''' If set, radecrad = (ra,dec,rad) returns sources within "rad" degrees of the given RA,Dec (in degrees) center. WARNING, this method alters the "objs" argument, if given. Consider calling objs.copy() before calling. -"bandname" is the SDSS band used to cut on position, select star/gal/exp/dev, and set galaxy shapes. -"bands" are the bands to include in the returned Source objects; they will be initialized from the SDSS bands. -"extrabands" are also included in the returned Source objects; they will be initialized to the SDSS flux for either the first of "bands", if given, or "bandname". ''' from astrometry.sdss import (DR7, DR8, DR9, band_names, band_index, photo_flags1_map) # brightPointSourceThreshold=0.): if sdss is None: dr = dict(DR7=DR7, DR8=DR8, DR9=DR9)[release] sdss = dr(curl=curl) drnum = sdss.getDRNumber() isdr7 = (drnum == 7) if bands is None: bands = band_names() bandnum = band_index(bandname) bandnums = np.array([band_index(b) for b in bands]) bandnames = bands if extrabands is None: extrabands = [] if objs is None: from astrometry.util.fits import fits_table if isdr7: # FIXME rerun = 0 if checkFiles: _check_sdss_files(sdss, run, camcol, field, bandnum, ['tsObj', 'tsField'], retrieve=retrieve, tryopen=True) tsf = sdss.readTsField(run, camcol, field, rerun) objfn = sdss.getPath('tsObj', run, camcol, field, bandname, rerun=rerun) else: if checkFiles: _check_sdss_files(sdss, run, camcol, field, bandnum, ['photoObj'], retrieve=retrieve, tryopen=True) objfn = sdss.getPath('photoObj', run, camcol, field) objs = fits_table(objfn) if objs is None: print('No sources in SDSS file', objfn) return [] objs.index = np.arange(len(objs)) if getobjs: allobjs = objs.copy() if roi is not None: x0, x1, y0, y1 = roi # FIXME -- we keep only the sources whose centers are within # the ROI box. Should instead do some ellipse-overlaps # geometry. x = objs.colc[:, bandnum] y = objs.rowc[:, bandnum] objs.cut((x >= x0) * (x < x1) * (y >= y0) * (y < y1)) if radecroi is not None: r0, r1, d0, d1 = radecroi objs.cut((objs.ra >= r0) * (objs.ra <= r1) * (objs.dec >= d0) * (objs.dec <= d1)) if radecrad is not None: from astrometry.libkd.spherematch import match_radec (ra, dec, rad) = radecrad I, J, d = match_radec(ra, dec, objs.ra, objs.dec, rad) objs.cut(J) del I del d if objCuts: # Only deblended children; # No BRIGHT sources bright = photo_flags1_map.get('BRIGHT') objs.cut((objs.nchild == 0) * ((objs.objc_flags & bright) == 0)) if cutToPrimary: objs.cut((objs.resolve_status & 256) > 0) if len(objs) == 0: sources = [] if not (getobjs or getobjinds or getsourceobjs): return sources rtn = [sources] if getobjs: rtn.append(None) if getobjinds: rtn.append(None) if getsourceobjs: rtn.append(None) return rtn if isdr7: objs.rename('phi_dev', 'phi_dev_deg') objs.rename('phi_exp', 'phi_exp_deg') objs.rename('r_dev', 'theta_dev') objs.rename('r_exp', 'theta_exp') # SDSS and Tractor have different opinions on which way this rotation goes objs.phi_dev_deg *= -1. objs.phi_exp_deg *= -1. # MAGIC -- minimum size of galaxy. objs.theta_dev = np.maximum(objs.theta_dev, 1. / 30.) objs.theta_exp = np.maximum(objs.theta_exp, 1. / 30.) if forcePointSources: Lstar = np.ones(len(objs), float) Lgal = np.zeros(len(objs), float) Ldev = Lexp = Lgal else: if useObjcType: objs.cut(np.logical_or(objs.objc_type == 6, objs.objc_type == 3)) Lstar = (objs.objc_type == 6) Lgal = (objs.objc_type == 3) else: Lstar = (objs.prob_psf[:, bandnum] == 1) * 1.0 Lgal = (objs.prob_psf[:, bandnum] == 0) * 1.0 if isdr7: fracdev = objs.fracpsf[:, bandnum] else: fracdev = objs.fracdev[:, bandnum] Ldev = Lgal * fracdev Lexp = Lgal * (1. - fracdev) if isdr7: from .sdss_dr7 import _dr7_getBrightness if nanomaggies: raise RuntimeError('Nanomaggies not supported for DR7 (yet)') def lup2bright(lups): counts = [ tsf.luptitude_to_counts(lup, j) for j, lup in enumerate(lups) ] counts = np.array(counts) bright = _dr7_getBrightness(counts, tsf, bandnames, extrabands) return bright flux2bright = lup2bright starflux = objs.psfcounts compflux = objs.counts_model devflux = objs.counts_dev expflux = objs.counts_exp def comp2bright(lups, Ldev, Lexp): counts = [ tsf.luptitude_to_counts(lup, j) for j, lup in enumerate(lups) ] counts = np.array(counts) dcounts = counts * Ldev ecounts = counts * Lexp dbright = _dr7_getBrightness(dcounts, tsf, bands, extrabands) ebright = _dr7_getBrightness(ecounts, tsf, bands, extrabands) return dbright, ebright else: def nmgy2bright(flux): if len(bandnums): flux = flux[bandnums] else: flux = flux[np.array([bandnum])] bb = bandnames + extrabands if nanomaggies: if len(extrabands): if len(bandnums) == 0: # Only "extrabands", no SDSS bands. flux = np.zeros(len(extrabands)) + flux[0] else: flux = np.append(flux, np.zeros(len(extrabands))) bright = NanoMaggies(order=bb, **dict(zip(bb, flux))) else: I = (flux > 0) mag = np.zeros_like(flux) + badmag mag[I] = sdss.nmgy_to_mag(flux[I]) if len(extrabands): mag = np.append(mag, np.zeros(len(extrabands)) + badmag) bright = Mags(order=bb, **dict(zip(bb, mag))) return bright def comp2bright(flux, Ldev, Lexp): dflux = flux * Ldev eflux = flux * Lexp dbright = nmgy2bright(dflux) ebright = nmgy2bright(eflux) return dbright, ebright flux2bright = nmgy2bright starflux = objs.psfflux compflux = objs.cmodelflux devflux = objs.devflux expflux = objs.expflux sources = [] nstars, ndev, nexp, ncomp = 0, 0, 0, 0 isources = [] ptsrcclass = classmap.get(PointSource, PointSource) for i in range(len(objs)): if Lstar[i]: pos = RaDecPos(objs.ra[i], objs.dec[i]) flux = starflux[i, :] bright = flux2bright(flux) # This should work, I just don't feel like testing it now... # if brightPointSourceThreshold: # ps = SdssPointSource(pos, bright, thresh=brightPointSourceThreshold) # else: # ps = PointSource(pos, bright) sources.append(ptsrcclass(pos, bright)) nstars += 1 isources.append(i) continue hasdev = (Ldev[i] > 0) hasexp = (Lexp[i] > 0) iscomp = (hasdev and hasexp) pos = RaDecPos(objs.ra[i], objs.dec[i]) if iscomp: flux = compflux[i, :] elif hasdev: flux = devflux[i, :] elif hasexp: flux = expflux[i, :] else: print( 'Skipping object with Lstar = %g, Ldev = %g, Lexp = %g (fracdev=%g)' % (Lstar[i], Ldev[i], Lexp[i], fracdev[i])) continue isources.append(i) if iscomp: if fixedComposites: bright = flux2bright(flux) fdev = (Ldev[i] / (Ldev[i] + Lexp[i])) else: dbright, ebright = comp2bright(flux, Ldev[i], Lexp[i]) else: bright = flux2bright(flux) if hasdev: re = objs.theta_dev[i, bandnum] ab = objs.ab_dev[i, bandnum] phi = objs.phi_dev_deg[i, bandnum] dshape = ellipse(re, ab, phi) if hasexp: re = objs.theta_exp[i, bandnum] ab = objs.ab_exp[i, bandnum] phi = objs.phi_exp_deg[i, bandnum] eshape = ellipse(re, ab, phi) if iscomp: if fixedComposites: gal = FixedCompositeGalaxy(pos, bright, fdev, eshape, dshape) else: gal = CompositeGalaxy(pos, ebright, eshape, dbright, dshape) ncomp += 1 elif hasdev: gal = DevGalaxy(pos, bright, dshape) ndev += 1 elif hasexp: gal = ExpGalaxy(pos, bright, eshape) nexp += 1 sources.append(gal) print( 'Created', ndev, 'deV,', nexp, 'exp,', ncomp, 'composite', ) print('(total %i) galaxies and %i stars' % (ndev + nexp + ncomp, nstars)) if not (getobjs or getobjinds or getsourceobjs): return sources if nstars + ndev + nexp + ncomp < len(objs): objs = objs[np.array(isources)] rtn = [sources] if getobjs: rtn.append(allobjs) if getobjinds: rtn.append(objs.index if len(objs) else np.array([])) if getsourceobjs: rtn.append(objs) return rtn
def get_footprint_object(self): """Returns footprint object 'sfd'""" # work with SFD map and Decals/Mzls tiles # lonlat from SFD healpix is in galactic coords, convert this to Celestial hdu = fitsio.FITS(os.path.join(self.map_dir, 'lambda_sfd_ebv.fits')) sfd = EmptyClass() temp = hdu[1].read() sfd.temp = temp['TEMPERATURE'] npix = Healpix().get_nside(len(sfd.temp)) assert (npix == 512) sfd.l_indeg, sfd.b_indeg = hp.pix2ang(512, np.where(sfd.temp > 0)[0], nest=True, lonlat=True) #inPlane= np.where((sfd_gal_dec > -20) & (sfd_gal_dec < 20))[0] trans = Galactic(l=sfd.l_indeg * units.degree, b=sfd.b_indeg * units.degree) radec = trans.transform_to(ICRS) sfd.ra, sfd.dec = radec.ra.value, radec.dec.value all_tiles = fits_table( os.path.join(self.tile_dir, 'mosaic-tiles_obstatus.fits')) wdes_tiles = fits_table( os.path.join(self.tile_dir, 'decam-tiles_obstatus.fits')) inDESI = ((all_tiles.in_desi_orig == 1) | (all_tiles.in_desi == 1)) inDecals = ((inDESI) & (all_tiles.dec <= 30.)) #(mzls_decals.in_des == 0)) inMzls = ((inDESI) & (all_tiles.dec > 30.)) #(mzls_decals.in_des == 0)) inDes = ((wdes_tiles.in_desi_orig == 1) | (wdes_tiles.in_desi == 1)) inDes = ((inDes) & (wdes_tiles.in_des == 1)) #above30= mzls.dec > 30. #inDESI= ( (mzls.in_desi_orig == 1) | # (mzls.in_desi == 1)) #inMzls= ( (above30) & # (inDESI)) #desi= merge_tables([mzls,decals],columns='fillzero') des = wdes_tiles.copy() del wdes_tiles des.cut(inDes) mzls = all_tiles.copy() decals = all_tiles.copy() del all_tiles mzls.cut(inMzls) decals.cut(inDecals) ps = Healpix().get_pixscale(len(sfd.temp), unit='deg') # match_radec(ref,obs): for each point in ref, return matching point in obs print('matching tiles to healpix centers') I, J, d = match_radec(mzls.ra, mzls.dec, sfd.ra, sfd.dec, ps * 8) sfd.ipix_mzls = list(set(J)) I, J, d = match_radec(decals.ra, decals.dec, sfd.ra, sfd.dec, ps * 8) sfd.ipix_decals = list(set(J)) I, J, d = match_radec(des.ra, des.dec, sfd.ra, sfd.dec, ps * 8) sfd.ipix_des = list(set(J)) return sfd # legasurvey pts fill in in ps*3 I, J, d = match_radec(legsurvey.ra, legsurvey.dec, sfd.ra, sfd.dec, ps * 3) sfd.ipix_legsurvey = set(J) # des fills in with ps*8 I, J, d = match_radec(des.ra, des.dec, sfd.ra, sfd.dec, ps * 8) sfd.ipix_legsurvey.union(set(J)) sfd.ipix_legsurvey = list(sfd.ipix_legsurvey) return sfd
def main(args): """Main program. """ import argparse parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).") parser.add_argument('--calibs', action='store_true', help='Output CCDs that need to be calibrated.') parser.add_argument('--nper', type=int, default=None, help='Batch N calibs per line') parser.add_argument('--byexp', action='store_true', default=False, help='Run one whole exposure per job (not one CCD per job)') parser.add_argument('--forced', action='store_true', help='Output forced-photometry commands') parser.add_argument('--lsb', action='store_true', help='Output Low-Surface-Brightness commands') parser.add_argument('--stage', help='Stage image files to given directory') parser.add_argument('--touching', action='store_true', help='Cut to only CCDs touching selected bricks') parser.add_argument('--near', action='store_true', help='Quick cut to only CCDs near selected bricks') parser.add_argument('--check-coadd', action='store_true', help='Check which coadds actually need to run.') parser.add_argument('--out', help='Output filename for calibs, default %(default)s', default='jobs') parser.add_argument('--command', action='store_true', help='Write out full command-line to run calib') parser.add_argument('--opt', help='With --command, extra options to add') parser.add_argument('--maxra', type=float, help='Maximum RA to run') parser.add_argument('--minra', type=float, help='Minimum RA to run') parser.add_argument('--maxdec', type=float, help='Maximum Dec to run') parser.add_argument('--mindec', type=float, help='Minimum Dec to run') parser.add_argument('--region', help='Region to select') parser.add_argument('--bricks', help='Set bricks.fits file to load') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--ignore_cuts', action='store_true',default=False,help='no photometric cuts') parser.add_argument('--save_to_fits', action='store_true',default=False,help='save cut brick,ccd to fits table') parser.add_argument('--name', action='store',default='dr3',help='save with this suffix, e.g. refers to ccds table') parser.add_argument('--delete-sky', action='store_true', help='Delete any existing sky calibration files') parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?') parser.add_argument('--nccds', action='store_true', default=False, help='Prints number of CCDs per brick') parser.add_argument('--bands', default='g,r,z', help='Set bands to keep: comma-separated list.') opt = parser.parse_args(args) want_ccds = (opt.calibs or opt.forced or opt.lsb) want_bricks = not want_ccds survey = LegacySurveyData() if opt.bricks is not None: B = fits_table(opt.bricks) log('Read', len(B), 'from', opt.bricks) else: B = survey.get_bricks() log('Bricks Dec range:', B.dec.min(), B.dec.max()) if opt.ccds is not None: T = fits_table(opt.ccds) log('Read', len(T), 'from', opt.ccds) else: T = survey.get_ccds() log(len(T), 'CCDs') T.index = np.arange(len(T)) if opt.ignore_cuts == False: log('Applying CCD cuts...') if 'ccd_cuts' in T.columns(): T.cut(T.ccd_cuts == 0) log(len(T), 'CCDs survive cuts') bands = opt.bands.split(',') log('Filters:', np.unique(T.filter)) T.cut(np.flatnonzero(np.array([f in bands for f in T.filter]))) log('Cut to', len(T), 'CCDs in filters', bands) log('CCDs Dec range:', T.dec.min(), T.dec.max()) # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True) # plt.clf() # plt.hist(counts, counts.max()+1) # plt.savefig('bricks.png') # B.cut(I[counts >= 9]) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # #plt.scatter(B.ra[I], B.dec[I], c=counts) # plt.savefig('bricks2.png') # DES Stripe82 #rlo,rhi = 350.,360. # rlo,rhi = 300., 10. # dlo,dhi = -6., 4. # TINY bit #rlo,rhi = 350.,351.1 #dlo,dhi = 0., 1.1 # EDR+ # 860 bricks # ~10,000 CCDs #rlo,rhi = 239,246 #dlo,dhi = 5, 13 # DR1 #rlo,rhi = 0, 360 # part 1 #dlo,dhi = 25, 40 # part 2 #dlo,dhi = 20,25 # part 3 #dlo,dhi = 15,20 # part 4 #dlo,dhi = 10,15 # part 5 #dlo,dhi = 5,10 # the rest #dlo,dhi = -11, 5 #dlo,dhi = 15,25.5 dlo,dhi = -25, 40 rlo,rhi = 0, 360 # Arjun says 3x3 coverage area is roughly # RA=240-252 DEC=6-12 (but not completely rectangular) # COSMOS #rlo,rhi = 148.9, 151.2 #dlo,dhi = 0.9, 3.5 # A nice well-behaved region (EDR2/3) # rlo,rhi = 243.6, 244.6 # dlo,dhi = 8.1, 8.6 # 56 bricks, ~725 CCDs #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7)) # 240 bricks, ~3000 CCDs #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9)) # 535 bricks, ~7000 CCDs #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12)) if opt.region in ['test1', 'test2', 'test3', 'test4']: nm = dict(test1='2446p115', # weird stuff around bright star test2='1183p292', # faint sources around bright galaxy test3='3503p005', # DES test4='1163p277', # Pollux )[opt.region] B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname]))) log('Cut to', len(B), 'bricks') log(B.ra, B.dec) dlo,dhi = -90,90 rlo,rhi = 0, 360 elif opt.region == 'edr': # EDR: # 535 bricks, ~7000 CCDs rlo,rhi = 240,245 dlo,dhi = 5, 12 elif opt.region == 'dr8-decam': rlo,rhi = 0, 360 dlo,dhi = -70, 40 log('DR8-DECam region') elif opt.region == 'edrplus': rlo,rhi = 235,248 dlo,dhi = 5, 15 elif opt.region == 'edr-south': rlo,rhi = 240,245 dlo,dhi = 5, 10 elif opt.region == 'cosmos1': # 16 bricks in the core of the COSMOS field. rlo,rhi = 149.75, 150.75 dlo,dhi = 1.6, 2.6 elif opt.region == 'pristine': # Stream? rlo,rhi = 240,250 dlo,dhi = 10,15 elif opt.region == 'des': dlo, dhi = -6., 4. rlo, rhi = 317., 7. T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage]))) log('Cut to', len(T), 'CCDs with "CPDES82" in filename') elif opt.region == 'subdes': rlo,rhi = 320., 360. dlo,dhi = -1.25, 1.25 elif opt.region == 'northwest': rlo,rhi = 240,360 dlo,dhi = 20,40 elif opt.region == 'north': rlo,rhi = 120,240 dlo,dhi = 20,40 elif opt.region == 'northeast': rlo,rhi = 0,120 dlo,dhi = 20,40 elif opt.region == 'southwest': rlo,rhi = 240,360 dlo,dhi = -20,0 elif opt.region == 'south': rlo,rhi = 120,240 dlo,dhi = -20,0 elif opt.region == 'southeast': rlo,rhi = 0,120 dlo,dhi = -20,0 elif opt.region == 'southsoutheast': rlo,rhi = 0,120 dlo,dhi = -20,-10 elif opt.region == 'midwest': rlo,rhi = 240,360 dlo,dhi = 0,20 elif opt.region == 'middle': rlo,rhi = 120,240 dlo,dhi = 0,20 elif opt.region == 'mideast': rlo,rhi = 0,120 dlo,dhi = 0,20 elif opt.region == 'grz': # Bricks with grz coverage. # Be sure to use --bricks survey-bricks-in-dr1.fits # which has_[grz] columns. B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)) log('Cut to', len(B), 'bricks with grz coverage') elif opt.region == 'nogrz': # Bricks without grz coverage. # Be sure to use --bricks survey-bricks-in-dr1.fits # which has_[grz] columns. B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))) log('Cut to', len(B), 'bricks withOUT grz coverage') elif opt.region == 'deep2': rlo,rhi = 250,260 dlo,dhi = 30,35 elif opt.region == 'deep2f2': rlo,rhi = 251.4, 254.4 dlo,dhi = 34.6, 35.3 elif opt.region == 'deep2f3': rlo,rhi = 351.25, 353.75 dlo,dhi = 0, 0.5 elif opt.region == 'deep3': rlo,rhi = 214,216 dlo,dhi = 52.25,53.25 elif opt.region == 'virgo': rlo,rhi = 185,190 dlo,dhi = 10, 15 elif opt.region == 'virgo2': rlo,rhi = 182,192 dlo,dhi = 8, 18 elif opt.region == 'coma': # van Dokkum et al Coma cluster ultra-diffuse galaxies: 3x3 field centered on Coma cluster rc,dc = 195., 28. dd = 1.5 cosdec = np.cos(np.deg2rad(dc)) rlo,rhi = rc - dd/cosdec, rc + dd/cosdec dlo,dhi = dc - dd, dc + dd elif opt.region == 'lsb': rlo,rhi = 147.2, 147.8 dlo,dhi = -0.4, 0.4 elif opt.region == 'eboss-sgc': # generous boundaries to make sure get all relevant images # RA -45 to +45 # Dec -5 to +7 rlo,rhi = 310., 50. dlo,dhi = -6., 6. elif opt.region == 'eboss-ngc': # generous boundaries to make sure get all relevant images # NGC ELGs # RA 115 to 175 # Dec 15 to 30 # rlo,rhi = 122., 177. # dlo,dhi = 12., 32. rlo,rhi = 126., 168. dlo,dhi = 18., 33. elif opt.region == 'mzls': dlo,dhi = -10., 90. # -10: pull in Stripe 82 data too elif opt.region == 'dr4-bootes': # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched #dlo,dhi = 34., 35. #rlo,rhi = 209.5, 210.5 dlo,dhi = 33., 36. rlo,rhi = 216.5, 219.5 elif opt.region == 'des-sn-x3': #rlo,rhi = 36., 37. #dlo,dhi = -5., -4. rlo,rhi = 36., 36.5 dlo,dhi = -4.5, -4. elif opt.region == 'ngc2632': # open cluster rlo,rhi = 129.0, 131.0 dlo,dhi = 19.0, 20.5 elif opt.region == 'dr8sky': rlo,rhi = 35.0, 37.0 dlo,dhi = -3.0, -1.0 # ADM DR8 test regions, see, e.g.: # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR8#Testregions elif opt.region == 'dr8-test-s82': rlo, rhi = 0, 45 dlo, dhi = -1.25, 1.25 elif opt.region == 'dr8-test-hsc-sgc': rlo, rhi = 30, 40 dlo, dhi = -6.5, -1.25 elif opt.region == 'dr8-test-hsc-ngc': rlo, rhi = 177.5, 182.5 dlo, dhi = -1, 1 elif opt.region == 'dr8-test-edr': rlo, rhi = 240, 245 dlo, dhi = 5, 12 elif opt.region == 'dr8-test-hsc-north': rlo, rhi = 240, 250 dlo, dhi = 42, 45 elif opt.region == 'dr8-test-deep2-egs': rlo, rhi = 213, 216.5 dlo, dhi = 52, 54 elif opt.region == 'dr8-test-overlap': rlo, rhi = 132, 140.5 dlo, dhi = 31.5, 35 if opt.mindec is not None: dlo = opt.mindec if opt.maxdec is not None: dhi = opt.maxdec if opt.minra is not None: rlo = opt.minra if opt.maxra is not None: rhi = opt.maxra if rlo < rhi: B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) else: # RA wrap B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) log(len(B), 'bricks in range; cut Dec range', B.dec.min(), B.dec.max()) #for name in B.get('brickname'): # print(name) #B.writeto('bricks-cut.fits') bricksize = 0.25 # A bit more than 0.25-degree brick radius + Bok image radius ~ 0.57 search_radius = 1.05 * np.sqrt(2.) * (bricksize + (0.455 * 4096 / 3600.))/2. log(len(T), 'CCDs') log(len(B), 'Bricks') I,J,_ = match_radec(B.ra, B.dec, T.ra, T.dec, search_radius, nearest=True) B.cut(I) log('Cut to', len(B), 'bricks near CCDs') log('Bricks Dec range:', B.dec.min(), B.dec.max()) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # plt.title('DR3 bricks') # plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1]) # plt.savefig('bricks.png') if opt.touching: I,J,_ = match_radec(T.ra, T.dec, B.ra, B.dec, search_radius, nearest=True) # list the ones that will be cut # drop = np.ones(len(T)) # drop[I] = False # for i in np.flatnonzero(drop): # from astrometry.util.starutil_numpy import degrees_between # dists = degrees_between(B.ra, B.dec, T.ra[i], T.dec[i]) # mindist = min(dists) # print('Dropping:', T.ra[i], T.dec[i], 'min dist', mindist, 'search_radius', search_radius) T.cut(I) log('Cut to', len(T), 'CCDs near bricks') if opt.forced: log('Writing forced-photometry commands to', opt.out) f = open(opt.out,'w') log('Total of', len(T), 'CCDs') #T.cut(allI) camexp = set([(c,e) for c,e in zip(T.camera, T.expnum)]) print(len(camexp), 'unique camera/exposure pairs') for cam,exp in camexp: #expstr = '%08i' % exp #outfn = os.path.join('forced', cam, expstr[:5], 'forced-%s-%s.fits' % (cam, exp)) #f.write('%s %s all %s\n' % (cam, exp, outfn)) f.write('%s %s\n' % (cam, exp)) f.close() log('Wrote', opt.out) return 0 # sort by RA increasing B.cut(np.argsort(B.ra)) if opt.save_to_fits: assert(opt.touching) # Write cut tables to file for tab,typ in zip([B,T],['bricks','ccds']): fn='%s-%s-cut.fits' % (typ,opt.region) if os.path.exists(fn): os.remove(fn) tab.writeto(fn) log('Wrote %s' % fn) # Write text files listing ccd and filename names # nm1,nm2= 'ccds-%s.txt'% opt.region,'filenames-%s.txt' % opt.region # if os.path.exists(nm1): # os.remove(nm1) # if os.path.exists(nm2): # os.remove(nm2) # f1,f2=open(nm1,'w'),open(nm2,'w') # fns= list(set(T.get('image_filename'))) # for fn in fns: # f2.write('%s\n' % fn.strip()) # for ti in T: # f1.write('%s\n' % ti.get('image_filename').strip()) # f1.close() # f2.close() # log('Wrote *-names.txt') if opt.touching: if want_bricks: # Shortcut the list of bricks that are definitely touching CCDs -- # a brick-ccd pair within this radius must be touching. closest_radius = 0.95 * (bricksize + 0.262 * 2048 / 3600.) / 2. J1,_,_ = match_radec(B.ra, B.dec, T.ra, T.dec, closest_radius, nearest=True) log(len(J1), 'of', len(B), 'bricks definitely touch CCDs') tocheck = np.ones(len(B), bool) tocheck[J1] = False J2 = [] for j in np.flatnonzero(tocheck): b = B[j] wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickname) if len(I) == 0: continue J2.append(j) J = np.hstack((J1, J2)) J = np.sort(J).astype(int) B.cut(J) log('Cut to', len(B), 'bricks touching CCDs') else: J = [] allI = set() for j,b in enumerate(B): wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickname) if len(I) == 0: continue allI.update(I) J.append(j) allI = list(allI) allI.sort() B.cut(np.array(J)) log('Cut to', len(B), 'bricks touching CCDs') elif opt.near: # Find CCDs near bricks allI,_,_ = match_radec(T.ra, T.dec, B.ra, B.dec, search_radius, nearest=True) # Find bricks near CCDs J,_,_ = match_radec(B.ra, B.dec, T.ra, T.dec, search_radius, nearest=True) B.cut(J) log('Cut to', len(B), 'bricks near CCDs') else: allI = np.arange(len(T)) if opt.byexp: _,eI = np.unique(T.expnum[allI], return_index=True) allI = allI[eI] print('Cut to', len(allI), 'expnums') if opt.nccds: from queue import Queue from threading import Thread log('Checking number of CCDs per brick') def worker(): while True: i = q.get() if i is None: break b = B[i] wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(b.brickname, len(I)) q.task_done() q = Queue() num_threads = 24 threads = [] for i in range(num_threads): t = Thread(target=worker) t.start() threads.append(t) for i in range(len(B)): q.put(i) q.join() for i in range(num_threads): q.put(None) for t in threads: t.join() if opt.write_ccds: T[allI].writeto(opt.write_ccds) log('Wrote', opt.write_ccds) if want_bricks: # Print the list of bricks and exit. for b in B: print(b.brickname) if opt.save_to_fits: B.writeto('bricks-%s-touching.fits' % opt.region) if not want_ccds: return 0 ## Be careful here -- T has been cut; we want to write out T.index. ## 'allI' contains indices into T. if opt.stage is not None: cmd_pat = 'rsync -LRarv %s %s' fns = set() for iccd in allI: im = survey.get_image_object(T[iccd]) fns.update([im.imgfn, im.wtfn, im.dqfn, im.psffn, im.merged_psffn, im.merged_skyfn, im.skyfn]) for i,fn in enumerate(fns): print('File', i+1, 'of', len(fns), ':', fn) if not os.path.exists(fn): print('No such file:', fn) continue base = survey.get_survey_dir() if base.endswith('/'): base = base[:-1] rel = os.path.relpath(fn, base) dest = os.path.join(opt.stage, rel) print('Dest:', dest) if os.path.exists(dest): print('Exists:', dest) continue cmd = cmd_pat % ('%s/./%s' % (base, rel), opt.stage) print(cmd) rtn = os.system(cmd) assert(rtn == 0) return 0 if opt.lsb: log('Writing LSB commands to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') for j,i in enumerate(allI): exp = T.expnum[i] ext = T.ccdname[i].strip() outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext) f.write('python legacyanalysis/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext)) f.close() log('Wrote', opt.out) return 0 log('Writing calibs to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') batch = [] def write_batch(f, batch, cmd): if cmd is None: cmd = '' f.write(cmd + ' '.join(batch) + '\n') cmd = None if opt.command: cmd = 'python legacypipe/run-calib.py ' if opt.opt is not None: cmd += opt.opt + ' ' for j,i in enumerate(allI): if opt.delete_sky: log(j+1, 'of', len(allI)) im = survey.get_image_object(T[i]) if opt.delete_sky and os.path.exists(im.skyfn): log(' deleting:', im.skyfn) os.unlink(im.skyfn) if opt.command: if opt.byexp: s = '--expnum %i' % (T.expnum[i]) else: s = '%i-%s' % (T.expnum[i], T.ccdname[i]) prefix = 'python legacypipe/run-calib.py ' if opt.opt is not None: prefix = prefix + opt.opt #('python legacypipe/run-calib.py --expnum %i --ccdname %s' % # (T.expnum[i], T.ccdname[i])) else: s = '%i' % T.index[i] prefix = '' if j < 10: print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i], 'filename', T.image_filename[i]) if not opt.nper: f.write(prefix + s + '\n') else: batch.append(s) if len(batch) >= opt.nper: write_batch(f, batch, cmd) batch = [] if len(batch): write_batch(f, batch, cmd) f.close() log('Wrote', opt.out) return 0
plt.title('Coverage (RGB = z/r/g)') plt.xlabel('RA (deg)') plt.ylabel('Dec (deg)') ps.savefig() overfn = 'overlapping-ccds.fits' if os.path.exists(overfn): M = fits_table(overfn) M.rename('i','I') M.rename('j','J') else: from astrometry.libkd.spherematch import match_radec radius = np.hypot(2048, 4096) * 0.262/3600. M = fits_table() M.I,M.J,d = match_radec(A.ra, A.dec, A.ra, A.dec, radius, notself=True) M.cut(M.I < M.J) # ra_centers vs dra cosd = np.cos(np.deg2rad(A.dec_center[M.I])) x1 = np.cos(np.deg2rad(A.ra_center[M.I])) y1 = np.sin(np.deg2rad(A.ra_center[M.I])) x2 = np.cos(np.deg2rad(A.ra_center[M.J])) y2 = np.sin(np.deg2rad(A.ra_center[M.J])) dra = np.rad2deg(cosd * np.hypot(x1 - x2, y1 - y2)) # widths - dra = overlap M.raoverlap = A.dra[M.I] + A.dra[M.J] - dra M.cut(M.raoverlap > 0) ddec = np.abs(A.dec_center[M.I] - A.dec_center[M.J]) # heights - ddec = overlap M.decoverlap = A.ddec[M.I] + A.ddec[M.J] - ddec
print 'Looking for', sefn morph = fits_table(sefn, hdu=2) wcs = Sip(im.wcsfn) if len(sdss) == 0: print 'EMPTY:', im.sdssfn continue if len(morph) == 0: print 'EMPTY:', im.morphfn continue print len(sdss), 'SDSS sources from', im.sdssfn print len(morph), 'SE sources from', sefn morph.ra,morph.dec = wcs.pixelxy2radec(morph.x_image, morph.y_image) I,J,d = match_radec(morph.ra, morph.dec, sdss.ra, sdss.dec, 0.5/3600.) corr = sdss[J] corr.add_columns_from(morph[I]) chipnames.append(im.extname) #corr = fits_table(im.corrfn) corrs.append(corr) print im, ':', len(corr), 'correspondences' for col,cut in ([('flux_auto',None)] + [('flux_aper',i) for i in range(3)] + [('flux_psf',None), ('flux_model',None)]): dmags = [] smags = [] for corr in corrs: if not col in corr.get_columns():
sig1 = 1.0 / np.sqrt(np.median(invvar[invvar > 0])) sigoff = 3 img += sig1 * sigoff # convert to sigmas img /= sig1 invvar *= sig1 ** 2 H, W = img.shape sz = 22 # Read sources detected in DECam image too T = fits_table(im.sefn, hdu=2) print "Got", len(T), "DECam sources" T.about() T.ra, T.dec = wcs.pixelxy2radec(T.x_image, T.y_image) I, J, d = match_radec(S.ra, S.dec, T.ra, T.dec, 1.0 / 3600.0, nearest=True) print "Matched", len(I) # Replace SDSS RA,Dec by DECam RA,Dec S.cut(I) S.ra = T.ra[J] S.dec = T.dec[J] ok, S.x, S.y = wcs.radec2pixelxy(S.ra, S.dec) S.x -= 1 S.y -= 1 S.ix, S.iy = np.round(S.x).astype(int), np.round(S.y).astype(int) S.cut((S.ix >= sz) * (S.iy >= sz) * (S.ix < W - sz) * (S.iy < H - sz)) print len(S), "SDSS stars in bounds" S.cut(invvar[S.iy, S.ix] > 0)
def process_file(self, path): print('Reading', path) F = fitsio.FITS(path) primhdr = F[0].read_header() print(len(F), 'extensions') # measure_raw . DECamMeasurer or Mosaic3Measurer meas_class = get_measurer_class_for_file(path) if meas_class is None: print('Failed to identify camera in', path) return meas = meas_class(path, 0, self.nom) # We read the WCS headers from all extensions and then spherematch # against the catalog. rr, dd = [], [] exts = np.arange(1, len(F)) keep_exts = [] for ext in exts: hdr = F[ext].read_header() extname = hdr['EXTNAME'].strip() # HACK -- skip DECam focus chips. if 'F' in extname: continue meas.ext = ext meas.primhdr = primhdr wcs = meas.get_wcs(hdr) #print('WCS:', wcs) rc, dc = wcs.radec_center() radius = wcs.radius() #print('RA,Dec center', rc, dc, 'radius', radius) rr.append(rc) dd.append(dc) keep_exts.append(ext) exts = keep_exts # we use the last extension's 'radius' here... rr = np.array(rr) dd = np.array(dd) if self.match_ngc(rr, dd, radius, exts, F, primhdr, meas): return print('Not doing SDSS spectro jazz.') return # No match with NGC catalog -- look at SDSS spectro objects. I, J, d = match_radec(rr, dd, self.spec.ra, self.spec.dec, radius) print(len(I), 'matches to spectra') if len(I) == 0: return False #meas_exts = set() #K = [] measures = {} specobjs = [] for k, (i, j) in enumerate(zip(I, J)): ext = exts[i] obj = self.spec[j] obj.name = obj.label.strip() hdr = F[ext].read_header() extname = hdr['EXTNAME'].strip() expnum = primhdr['EXPNUM'] wcs = meas.get_wcs(hdr) ok, x, y = wcs.radec2pixelxy(obj.ra, obj.dec) x = x - 1 y = y - 1 # Choose cutout area pixrad = 1.4 * 50 r = pixrad tt = '%s in exp %i ext %s (%i)' % (obj.name, expnum, extname, ext) print(tt) # Find the cutout region... does it actually overlap the chip? H, W = wcs.shape xl, xh = int(np.clip(x - r, 0, W - 1)), int(np.clip(x + r, 0, W - 1)) yl, yh = int(np.clip(y - r, 0, H - 1)), int(np.clip(y + r, 0, H - 1)) if xl == xh or yl == yh: print('no actual overlap with image') continue sh, sw = yh - yl, xh - xl if sh < 25 or sw < 25: print('tiny overlap', sw, 'x', sh) continue #meas_exts.add(ext) #K.append(k) if ext in measures: M = measures[ext] else: # Measure the image! meas.ext = ext meas.edge_trim = 20 try: M = meas.run(n_fwhm=1, verbose=False, get_image=True) except KeyboardInterrupt: sys.exit(0) except: import traceback print('Failed to measure file', path, 'ext', ext, ':') traceback.print_exc() continue measures[ext] = M raw = M['image'] # Now repeat the cutout check with the trimmed image wcs = M['wcs'] # Trim WCS to trimmed raw image shape trim_x0, trim_y0 = M['trim_x0'], M['trim_y0'] H, W = raw.shape wcs = wcs.get_subimage(trim_x0, trim_y0, W, H) ok, x, y = wcs.radec2pixelxy(obj.ra, obj.dec) x = x - 1 y = y - 1 xl, xh = int(np.clip(x - r, 0, W - 1)), int(np.clip(x + r, 0, W - 1)) yl, yh = int(np.clip(y - r, 0, H - 1)), int(np.clip(y + r, 0, H - 1)) if xl == xh or yl == yh: print('no actual overlap with image') continue subimg = raw[yl:yh, xl:xh] sh, sw = subimg.shape if sh < 25 or sw < 25: print('tiny overlap', sw, 'x', sh) continue subwcs = wcs.get_subimage(xl, yl, sw, sh) # Astrometric shifts dx = M['dx'] dy = M['dy'] aff = M['affine'] x = (xl + xh) / 2 y = (yl + yh) / 2 #print('Affine correction terms:', aff) adx = x - aff[0] ady = y - aff[1] corrx = aff[2] + aff[3] * adx + aff[4] * ady - adx corry = aff[5] + aff[6] * adx + aff[7] * ady - ady print('Affine correction', corrx, corry) # Shift the 'subwcs' to account for astrometric offset cx, cy = subwcs.get_crpix() subwcs.set_crpix((cx - dx - corrx, cy - dy - corry)) specobjs.append((tt, subwcs, subimg)) # # What size of image are we going to request? # scale = 1. # # rh,rw = int(np.ceil(sh/scale)),int(np.ceil(sw/scale)) # # make it square # mx = max(rh, rw) # rw = rh = mx # # fitsimgs = [] # # We'll resample the new image into the existing-image WCS. # newimg = None # # # HACK # imgs = [] # layer = 'sdssco' # bands = 'gri' # for band in bands: # fn = 'sdssco-1679p492-%s.fits' % band # fitsfile = fitsio.FITS(fn) # wcs = Tan(fn, 0) # hh,ww = wcs.shape # ok,x,y = wcs.radec2pixelxy(obj.ra, obj.dec) # x = x - 1 # y = y - 1 # print('x,y', x,y) # sz = pixrad * 0.262/0.396 # xl,xh = int(np.clip(x-sz, 0, ww-1)), int(np.clip(x+sz, 0, ww-1)) # yl,yh = int(np.clip(y-sz, 0, hh-1)), int(np.clip(y+sz, 0, hh-1)) # if xl == xh or yl == yh: # print('no overlap with SDSS image') # continue # # img = fitsfile[1][yl:yh+1, xl:xh+1] # s = (subwcs.pixel_scale() / 0.396) # img *= s**2 # imgs.append(img) # thiswcs = wcs # # if len(imgs) == 0: # continue # # fitsimgs.append((layer, bands, imgs)) # # # Resample the new image to this layer's WCS # newimg = np.zeros((rh,rw), dtype=subimg.dtype) # try: # # Laczos # Yo,Xo,Yi,Xi,rims = resample_with_wcs(thiswcs, subwcs, [subimg]) # except: # continue # newimg[Yo,Xo] = rims[0] # # if len(fitsimgs) == 0: # # No overlap with existing surveys # continue # # newband = primhdr['FILTER'][0] # I = I[K] # J = J[K] # print('Cut to', len(I), 'matches in', len(meas_exts), 'extensions') # if len(I) == 0: # return # measures = {} # for ext in meas_ext: # measures[ext] = print(len(specobjs), 'objects') def my_rgb(imgs, bands, **kwargs): return sdss_rgb(imgs, bands, scales=dict(g=6.0, r=3.4, i=2.5, z=2.2), m=-0.02, clip=False, **kwargs) def grayscale(img, band): rgb = my_rgb([img, img, img], [band, band, band]) index = 'zrg'.index(newband) gray = rgb[:, :, index] return gray newband = primhdr['FILTER'][0] plt.figure(2) plt.clf() plt.subplots_adjust(left=0.03, right=0.97, bottom=0.03, hspace=0, wspace=0) N = len(specobjs) C = int(np.ceil(np.sqrt(N * 1.2))) R = int(np.ceil(N / float(C))) print('Rows,cols, N', R, C, N) gray = grayscale(raw, newband) hi = np.percentile(gray, 99.9) grayargs = dict(interpolation='nearest', origin='lower', vmin=0., vmax=hi, cmap='gray') print('Plotting:') plt.clf() for i, (tt, subwcs, subimg) in enumerate(specobjs): plt.subplot(R, C, i + 1) print(' ', tt) newimg = subimg # New Image plot newgray = grayscale(newimg, newband) #hi = np.percentile(newgray, 99.9) plt.imshow(newgray, **grayargs) plt.xticks([]) plt.yticks([]) ps.savefig()
def write_cat_files(T, outdir): """ writes out single source files for each object in T, to outdir """ # Write out Stripe82 measurements... radius = np.sqrt(2.) * pixradius * pixscale / 3600. for i in range(len(T)): # looks for sources nearby T[i], within radius. it returns index in # both first (short) list and second (long) list I,J,d = asphere.match_radec(np.array([T.ra[i]]), np.array([T.dec[i]]), T.ra, T.dec, radius) print len(J), 'matched within', radius*3600., 'arcsec' t = T[J] print len(t), 'matched within', radius*3600., 'arcsec' tt = aufits.fits_table() cols = ['ra','dec','run','camcol','field',#'probpsf', #'flags', #'type', 'fracdev_r', #'probpsf_r', 'devrad_r','devraderr_r', 'devab_r', 'devaberr_r', 'devphi_r', 'devphierr_r', 'exprad_r','expraderr_r', 'expab_r', 'expaberr_r', 'expphi_r', 'expphierr_r', ] for c in cols: cout = c # drop "_r" from dev/exp shapes if cout.endswith('_r'): cout = cout[:-2] coutmap = dict(devrad='theta_dev', devphi='phi_dev', devab ='ab_dev', devraderr='theta_dev_err', devphierr='phi_dev_err', devaberr ='ab_dev_err', exprad='theta_exp', expphi='phi_exp', expab ='ab_exp', expraderr='theta_exp_err', expphierr='phi_exp_err', expaberr ='ab_exp_err', fracdev='frac_dev') cout = coutmap.get(cout, cout) tt.set(cout, t.get(c)) tt.is_star = (t.type == 6) for magname in ['psf', 'dev', 'exp']: for band in 'ugriz': mag = t.get('%smag_%s' % (magname, band)) magerr = t.get('%smagerr_%s' % (magname, band)) ### FIXME -- arcsinh mags?? flux = tsdss.NanoMaggies.magToNanomaggies(mag) dflux = np.abs(flux * np.log(10.)/-2.5 * magerr) tt.set('%sflux_%s' % (magname, band), flux) tt.set('%sfluxerr_%s' % (magname, band), dflux) for band in 'ugriz': # http://www.sdss3.org/dr10/algorithms/magnitudes.php#cmodel fexp = tt.get('expflux_%s' % band) fdev = tt.get('expflux_%s' % band) fracdev = t.get('fracdev_%s' % band) tt.set('cmodelflux_%s' % band, fracdev * fdev + (1.-fracdev) * fexp) catfn = os.path.join(outdir, 'cat-s82-%.4f-%.4f.fits' % (t.ra[0], t.dec[0])) tt.writeto(catfn) print 'Wrote', catfn
def sed_matched_figs(detect_sn, good, img, sedlist, DES, g_det, r_det, i_det, wcs): x,y = detect_sources(detect_sn, 100.) sources = fits_table() sources.x = x sources.y = y sources.cut(good[sources.y, sources.x]) print('Cut to', len(sources), 'good sources') sz = 20 H,W = good.shape sources.cut((sources.x > sz) * (sources.y > sz) * (sources.x < (W-sz)) * (sources.y < (H-sz))) print(len(sources), 'not near edges') for s in sedlist: sources.set(s.tname, s.snmap[sources.y, sources.x]) # sources.g_sn = (g_det[sources.y, sources.x] * np.sqrt(g_detiv[sources.y, sources.x])) # sources.r_sn = (r_det[sources.y, sources.x] * np.sqrt(r_detiv[sources.y, sources.x])) # sources.i_sn = (i_det[sources.y, sources.x] * np.sqrt(i_detiv[sources.y, sources.x])) sources.g_flux = g_det[sources.y, sources.x] sources.r_flux = r_det[sources.y, sources.x] sources.i_flux = i_det[sources.y, sources.x] sources.ra,sources.dec = wcs.pixelxy2radec(sources.x+1, sources.y+1) sources.g_mag = -2.5*(np.log10(sources.g_flux) - 9) sources.r_mag = -2.5*(np.log10(sources.r_flux) - 9) sources.i_mag = -2.5*(np.log10(sources.i_flux) - 9) sources.imax = np.argmax(np.vstack([sources.get(s.tname) for s in sedlist]), axis=0) plt.figure(figsize=(5,4)) plt.subplots_adjust(left=0.15, right=0.97, bottom=0.12, top=0.98) plt.clf() for i,s in enumerate(sedlist): if not np.all(s.sed > 0): continue I = np.flatnonzero(sources.imax == i) plt.plot(sources.g_mag[I] - sources.r_mag[I], sources.r_mag[I] - sources.i_mag[I], s.plotsym, label=s.name, color=s.plotcolor, alpha=0.5, mfc='none', ms=5) gr = -2.5 * np.log10(s.sed[0] / s.sed[1]) ri = -2.5 * np.log10(s.sed[1] / s.sed[2]) plt.plot(gr, ri, 'o', color='k', mfc='none', ms=8, mew=3) plt.axis([-0.5, 2.5, -0.5, 2]) plt.xlabel('g - r (mag)') plt.ylabel('r - i (mag)') plt.legend(loc='upper left') plt.savefig('best-color.pdf') plt.figure(figsize=(4,4)) plt.subplots_adjust(left=0.01, right=0.99, bottom=0.01, top=0.99) plt.clf() xlo,xhi = 500,1100 ylo,yhi = 500,1100 plt.imshow(img[ylo:yhi, xlo:xhi], origin='lower', interpolation='nearest', extent=[xlo,xhi,ylo,yhi]) ax = plt.axis() for i,s in enumerate(sedlist): if not np.all(s.sed > 0): continue I = np.flatnonzero((sources.imax == i) * (sources.x >= xlo) * (sources.x <= xhi) * (sources.y >= ylo) * (sources.y <= yhi)) print(len(I), s.name) plt.plot(sources.x[I], sources.y[I], s.plotsym, label=s.name, color=s.plotcolor, alpha=0.5, mfc='none', ms=15) plt.axis(ax) plt.xticks([]) plt.yticks([]) plt.savefig('image-sources.pdf') for i,s in enumerate(sedlist): I = np.flatnonzero(sources.imax == i) J = np.argsort(-sources.get(s.tname)[I]) plt.clf() show_sources(sources[I[J]], img) plt.savefig('best-%s.pdf' % s.name.lower()) ##### Run detection at different thresholds #### tsedlist = [] for i,s in enumerate(sedlist): if not np.all(s.sed > 0): continue tsedlist.append(s) snmap = None for i,s in enumerate(tsedlist): if snmap is None: snmap = s.snmap else: snmap = np.maximum(snmap, s.snmap) for thresh in [30]: #10, 30, 100]: x,y = detect_sources(snmap, thresh) tsources = fits_table() tsources.x = x tsources.y = y #tsources.cut(good[tsources.y, tsources.x]) print('Threshold', thresh) print('Cut to', len(tsources), 'good sources') sz = 20 H,W = good.shape tsources.cut((tsources.x > sz) * (tsources.y > sz) * (tsources.x < (W-sz)) * (tsources.y < (H-sz))) print(len(tsources), 'not near edges') for s in tsedlist: tsources.set(s.tname, s.snmap[tsources.y, tsources.x]) tsources.g_flux = g_det[tsources.y, tsources.x] tsources.r_flux = r_det[tsources.y, tsources.x] tsources.i_flux = i_det[tsources.y, tsources.x] tsources.ra,tsources.dec = wcs.pixelxy2radec(tsources.x+1, tsources.y+1) tsources.g_mag = -2.5*(np.log10(tsources.g_flux) - 9) tsources.r_mag = -2.5*(np.log10(tsources.r_flux) - 9) tsources.i_mag = -2.5*(np.log10(tsources.i_flux) - 9) tsources.imax = np.argmax(np.vstack([tsources.get(s.tname) for s in tsedlist]), axis=0) plt.clf() xlo,xhi = 500,1100 ylo,yhi = 500,1100 plt.imshow(img[ylo:yhi, xlo:xhi], origin='lower', interpolation='nearest', extent=[xlo,xhi,ylo,yhi]) ax = plt.axis() for i,s in enumerate(tsedlist): I = np.flatnonzero((tsources.imax == i) * (tsources.x >= xlo) * (tsources.x <= xhi) * (tsources.y >= ylo) * (tsources.y <= yhi)) print(len(I), s.name) plt.plot(tsources.x[I], tsources.y[I], s.plotsym, label=s.name, color=s.plotcolor, alpha=0.5, mfc='none', ms=15, mew=2) plt.axis(ax) plt.xticks([]) plt.yticks([]) plt.savefig('image-sources-%i.pdf' % thresh) # boundary = (snmap > thresh) # boundary = np.logical_xor(boundary, binary_dilation(boundary, # structure=np.ones((3,3)))) # rgb = img[ylo:yhi, xlo:xhi].copy() # rgb[:,:,1][boundary[ylo:yhi, xlo:xhi]] = 255 # plt.clf() # plt.imshow(rgb, origin='lower', interpolation='nearest', # extent=[xlo,xhi,ylo,yhi]) # plt.xticks([]) # plt.yticks([]) # plt.savefig('image-blobs-%i.pdf' % thresh) plt.figure(figsize=(5,4)) plt.subplots_adjust(left=0.15, right=0.97, bottom=0.12, top=0.98) plt.clf() lp,lt = [],[] for i,s in enumerate(tsedlist): I = np.flatnonzero(tsources.imax == i) plt.plot(tsources.g_mag[I] - tsources.r_mag[I], tsources.r_mag[I] - tsources.i_mag[I], s.plotsym, color=s.plotcolor, alpha=0.2, mfc='none', ms=5) # For the legend p = plt.plot(-1, -1, s.plotsym, color=s.plotcolor, mfc='none', ms=5) lp.append(p[0]) lt.append(s.name) gr = -2.5 * np.log10(s.sed[0] / s.sed[1]) ri = -2.5 * np.log10(s.sed[1] / s.sed[2]) plt.plot(gr, ri, 'o', color='k', mfc='none', ms=8, mew=3) plt.axis([-0.5, 2.5, -0.5, 2]) plt.xlabel('g - r (mag)') plt.ylabel('r - i (mag)') plt.legend(lp, lt, loc='upper left') plt.savefig('best-color-%i.pdf' % thresh) ############################## # Artifacts from single-band detections I = np.hstack((np.flatnonzero(sources.imax == 3)[:6], np.flatnonzero(sources.imax == 4)[:18], np.flatnonzero(sources.imax == 5)[:12])) plt.figure(figsize=(4,4)) plt.subplots_adjust(left=0.01, right=0.99, bottom=0.01, top=0.99) plt.clf() show_sources(sources[I], img, R=6, C=6, sz=30, divider=1) plt.savefig('singleband.pdf') MI,MJ,d = match_radec(sources.ra, sources.dec, DES.ra, DES.dec, 1./3600, nearest=True) print(len(MI), 'matches') MDES = DES[MJ] Msources = sources[MI] ## FIXME -- select only isolated stars? colorbins = np.linspace(-0.5, 4.0, 10) II = [] K = [] DES.gi = DES.mag_auto_g - DES.mag_auto_i for clo,chi in zip(colorbins, colorbins[1:]): C = np.flatnonzero((DES.gi >= clo) * (DES.gi < chi)) minmag = np.vstack((DES.mag_auto_g, DES.mag_auto_r, DES.mag_auto_i)).max(axis=0)[C] C = C[np.argsort(np.abs(minmag - 17.9))] C = C[DES.spread_model_r[C] < 0.01] II.extend(C[:10]) K.append(C[0]) fw,fh = 6,4 sl,sr,sb,st = 0.15, 0.98, 0.12, 0.98 plt.figure(figsize=(fw,fh)) plt.subplots_adjust(left=sl, right=sr, bottom=sb, top=st) plt.clf() plt.axhline(1., color='orange', lw=5) plt.axhline(1., color='k', alpha=0.5) plt.axvline(0., color='b', lw=2, alpha=0.2) plt.axvline(1.3, color='orange', lw=2, alpha=0.2) plt.axvline(2.5, color='r', lw=2, alpha=0.2) plt.plot(MDES.mag_auto_g - MDES.mag_auto_i, Msources.blue_sn / Msources.yellow_sn, 'bD', alpha=0.3, label='Blue SED-matched filter', ms=3) plt.plot(MDES.mag_auto_g - MDES.mag_auto_i, Msources.red_sn / Msources.yellow_sn, 'rs', alpha=0.5, label='Red SED-matched filter', ms=3) plt.xlabel('DES g - i color (mag)') plt.ylabel('Relative strength of SED filter vs Yellow') plt.legend(loc='upper left') # Position the postage stamp images just right... # axes width,height w = fw * (sr-sl) h = fh * (st-sb) n = len(colorbins)-1 # image size s = w/n # fraction of vertical axis devoted to image fim = s/h # fraction devoted to plot fplot = 1.-fim # scale ys = (1.3 - 0.7) / fplot # lower limit ymin = 1.3 - ys # image top ymax = ymin + ys * fim ax = [-0.5, 4.0, ymin, 1.3] plt.axis(ax) aspect = plt.gca().get_aspect() for clo,chi,k in zip(colorbins, colorbins[1:], K): x,y = DES.x[k], DES.y[k] plt.imshow(img[y-sz:y+sz+1, x-sz:x+sz+1], interpolation='nearest', origin='lower', extent=[clo,chi,ymin,ymax], zorder=20) plt.yticks([0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3]) plt.axis(ax) plt.gca().set_aspect(aspect) plt.savefig('strength.pdf')
def _get_sources(run, camcol, field, bandname='r', sdss=None, release='DR7', objs=None, retrieve=True, curl=False, roi=None, radecroi=None, radecrad=None, bands=None, badmag=25, nanomaggies=False, getobjs=False, getsourceobjs=False, getobjinds=False, extrabands=None, fixedComposites=False, forcePointSources=False, useObjcType=False, objCuts=True, classmap={}, ellipse=GalaxyShape, cutToPrimary=False): ''' If set, radecrad = (ra,dec,rad) returns sources within "rad" degrees of the given RA,Dec (in degrees) center. WARNING, this method alters the "objs" argument, if given. Consider calling objs.copy() before calling. -"bandname" is the SDSS band used to cut on position, select star/gal/exp/dev, and set galaxy shapes. -"bands" are the bands to include in the returned Source objects; they will be initialized from the SDSS bands. -"extrabands" are also included in the returned Source objects; they will be initialized to the SDSS flux for either the first of "bands", if given, or "bandname". ''' from astrometry.sdss import (DR7, DR8, DR9, band_names, band_index, photo_flags1_map) # brightPointSourceThreshold=0.): if sdss is None: dr = dict(DR7=DR7, DR8=DR8, DR9=DR9)[release] sdss = dr(curl=curl) drnum = sdss.getDRNumber() isdr7 = (drnum == 7) if bands is None: bands = band_names() bandnum = band_index(bandname) bandnums = np.array([band_index(b) for b in bands]) bandnames = bands if extrabands is None: extrabands = [] if objs is None: from astrometry.util.fits import fits_table if isdr7: # FIXME rerun = 0 _check_sdss_files(sdss, run, camcol, field, bandnum, ['tsObj', 'tsField'], retrieve=retrieve, tryopen=True) tsf = sdss.readTsField(run, camcol, field, rerun) objfn = sdss.getPath('tsObj', run, camcol, field, bandname, rerun=rerun) else: _check_sdss_files(sdss, run, camcol, field, bandnum, ['photoObj'], tryopen=True, retrieve=retrieve) objfn = sdss.getPath('photoObj', run, camcol, field) objs = fits_table(objfn) if objs is None: print('No sources in SDSS file', objfn) return [] objs.index = np.arange(len(objs)) if getobjs: allobjs = objs.copy() if roi is not None: x0,x1,y0,y1 = roi # FIXME -- we keep only the sources whose centers are within # the ROI box. Should instead do some ellipse-overlaps # geometry. x = objs.colc[:,bandnum] y = objs.rowc[:,bandnum] objs.cut((x >= x0) * (x < x1) * (y >= y0) * (y < y1)) if radecroi is not None: r0,r1,d0,d1 = radecroi objs.cut((objs.ra >= r0) * (objs.ra <= r1) * (objs.dec >= d0) * (objs.dec <= d1)) if radecrad is not None: from astrometry.libkd.spherematch import match_radec (ra,dec,rad) = radecrad I,J,d = match_radec(ra, dec, objs.ra, objs.dec, rad) objs.cut(J) del I del d if objCuts: # Only deblended children; # No BRIGHT sources bright = photo_flags1_map.get('BRIGHT') objs.cut((objs.nchild == 0) * ((objs.objc_flags & bright) == 0)) if cutToPrimary: objs.cut((objs.resolve_status & 256) > 0) if isdr7: objs.rename('phi_dev', 'phi_dev_deg') objs.rename('phi_exp', 'phi_exp_deg') objs.rename('r_dev', 'theta_dev') objs.rename('r_exp', 'theta_exp') # SDSS and Tractor have different opinions on which way this rotation goes objs.phi_dev_deg *= -1. objs.phi_exp_deg *= -1. # MAGIC -- minimum size of galaxy. objs.theta_dev = np.maximum(objs.theta_dev, 1./30.) objs.theta_exp = np.maximum(objs.theta_exp, 1./30.) if forcePointSources: Lstar = np.ones(len(objs), float) Lgal = np.zeros(len(objs), float) Ldev = Lexp = Lgal else: if useObjcType: objs.cut(np.logical_or(objs.objc_type == 6, objs.objc_type == 3)) Lstar = (objs.objc_type == 6) Lgal = (objs.objc_type == 3) else: Lstar = (objs.prob_psf[:,bandnum] == 1) * 1.0 Lgal = (objs.prob_psf[:,bandnum] == 0) if isdr7: fracdev = objs.fracpsf[:,bandnum] else: fracdev = objs.fracdev[:,bandnum] Ldev = Lgal * fracdev Lexp = Lgal * (1. - fracdev) if isdr7: if nanomaggies: raise RuntimeError('Nanomaggies not supported for DR7 (yet)') def lup2bright(lups): counts = [tsf.luptitude_to_counts(lup,j) for j,lup in enumerate(lups)] counts = np.array(counts) bright = _getBrightness(counts, tsf, bandnames, extrabands) return bright flux2bright = lup2bright starflux = objs.psfcounts compflux = objs.counts_model devflux = objs.counts_dev expflux = objs.counts_exp def comp2bright(lups, Ldev, Lexp): counts = [tsf.luptitude_to_counts(lup,j) for j,lup in enumerate(lups)] counts = np.array(counts) dcounts = counts * Ldev ecounts = counts * Lexp dbright = _getBrightness(dcounts, tsf, bands, extrabands) ebright = _getBrightness(ecounts, tsf, bands, extrabands) return dbright, ebright else: def nmgy2bright(flux): if len(bandnums): flux = flux[bandnums] else: flux = flux[np.array([bandnum])] bb = bandnames + extrabands if nanomaggies: if len(extrabands): if len(bandnums) == 0: # Only "extrabands", no SDSS bands. flux = np.zeros(len(extrabands)) + flux[0] else: flux = np.append(flux, np.zeros(len(extrabands))) bright = NanoMaggies(order=bb, **dict(zip(bb, flux))) else: I = (flux > 0) mag = np.zeros_like(flux) + badmag mag[I] = sdss.nmgy_to_mag(flux[I]) if len(extrabands): mag = np.append(mag, np.zeros(len(extrabands)) + badmag) bright = Mags(order=bb, **dict(zip(bb,mag))) return bright def comp2bright(flux, Ldev, Lexp): dflux = flux * Ldev eflux = flux * Lexp dbright = nmgy2bright(dflux) ebright = nmgy2bright(eflux) return dbright, ebright flux2bright = nmgy2bright starflux = objs.psfflux compflux = objs.cmodelflux devflux = objs.devflux expflux = objs.expflux sources = [] nstars, ndev, nexp, ncomp = 0, 0, 0, 0 isources = [] ptsrcclass = classmap.get(PointSource, PointSource) for i in range(len(objs)): if Lstar[i]: pos = RaDecPos(objs.ra[i], objs.dec[i]) flux = starflux[i,:] bright = flux2bright(flux) # This should work, I just don't feel like testing it now... # if brightPointSourceThreshold: # ps = SdssPointSource(pos, bright, thresh=brightPointSourceThreshold) # else: # ps = PointSource(pos, bright) sources.append(ptsrcclass(pos, bright)) nstars += 1 isources.append(i) continue hasdev = (Ldev[i] > 0) hasexp = (Lexp[i] > 0) iscomp = (hasdev and hasexp) pos = RaDecPos(objs.ra[i], objs.dec[i]) if iscomp: flux = compflux[i,:] elif hasdev: flux = devflux[i,:] elif hasexp: flux = expflux[i,:] else: print('Skipping object with Lstar = %g, Ldev = %g, Lexp = %g (fracdev=%g)' % (Lstar[i], Ldev[i], Lexp[i], fracdev[i])) continue isources.append(i) if iscomp: if fixedComposites: bright = flux2bright(flux) fdev = (Ldev[i] / (Ldev[i] + Lexp[i])) else: dbright,ebright = comp2bright(flux, Ldev[i], Lexp[i]) else: bright = flux2bright(flux) if hasdev: re = objs.theta_dev [i,bandnum] ab = objs.ab_dev [i,bandnum] phi = objs.phi_dev_deg[i,bandnum] dshape = ellipse(re, ab, phi) if hasexp: re = objs.theta_exp [i,bandnum] ab = objs.ab_exp [i,bandnum] phi = objs.phi_exp_deg[i,bandnum] eshape = ellipse(re, ab, phi) if iscomp: if fixedComposites: gal = FixedCompositeGalaxy(pos, bright, fdev, eshape, dshape) else: gal = CompositeGalaxy(pos, ebright, eshape, dbright, dshape) ncomp += 1 elif hasdev: gal = DevGalaxy(pos, bright, dshape) ndev += 1 elif hasexp: gal = ExpGalaxy(pos, bright, eshape) nexp += 1 sources.append(gal) print('Created', ndev, 'deV,', nexp, 'exp,', ncomp, 'composite',) print('(total %i) galaxies and %i stars' % (ndev+nexp+ncomp, nstars)) if not (getobjs or getobjinds or getsourceobjs): return sources if nstars + ndev + nexp + ncomp < len(objs): objs = objs[np.array(isources)] rtn = [sources] if getobjs: rtn.append(allobjs) if getobjinds: rtn.append(objs.index if len(objs) else np.array([])) if getsourceobjs: rtn.append(objs) return rtn
def forced_phot(): # Forced phot ps = PlotSequence('kick') plt.subplots_adjust(top=0.95, bottom=0.1, left=0.1, right=0.95) ff = [ ('decam-348227-S15-g-forced.fits', 348227, 'S15', 'g'), ('decam-348248-S16-g-forced.fits', 348248, 'S16', 'g'), ('decam-348271-S14-g-forced.fits', 348271, 'S14', 'g'), ('decam-348660-S15-r-forced.fits', 348660, 'S15', 'r'), ('decam-348684-S16-r-forced.fits', 348684, 'S16', 'r'), ('decam-348712-S14-r-forced.fits', 348712, 'S14', 'r'), ('decam-349154-S15-z-forced.fits', 349154, 'S15', 'z'), ('decam-349183-S14-z-forced.fits', 349183, 'S14', 'z'), ('decam-346630-S16-z-forced.fits', 346630, 'S16', 'z'), ] FF = [] for fn, expnum, extname, band in ff: F = fits_table(fn) print len(F), 'from', fn F.expnum = np.array([expnum] * len(F)) F.extname = np.array([extname] * len(F)) FF.append(F) F = merge_tables(FF) bricks = np.unique(F.brickname) print 'Bricks:', bricks T = merge_tables([ fits_table(os.path.join('dr1', 'tractor', b[:3], 'tractor-%s.fits' % b)) for b in bricks ]) print 'Total of', len(T), 'sources' T.cut(T.brick_primary == 1) print 'Cut to', len(T), 'brick_primary' Tall = T.copy() cutouts = [] I, J, d = match_radec(Tall.ra, Tall.dec, Tall.ra, Tall.dec, 5. / 3600., notself=True) K = np.flatnonzero(I < J) I = I[K] J = J[K] # randomize K = np.random.randint(2, size=len(I)) I, J = I * K + J * (1 - K), I * (1 - K) + J * K plt.clf() plothist( 3600. * (Tall.ra[I] - Tall.ra[J]) * np.cos(np.deg2rad(Tall.dec[I])), 3600. * (Tall.dec[I] - Tall.dec[J]), 200) #, range=((-2,2),(-2,2))) plt.xlabel('dRA (arcsec)') plt.ylabel('dDec (arcsec)') plt.title('%i sources, %i matches' % (len(Tall), len(I))) ps.savefig() dist = 3600. * np.hypot( (Tall.ra[I] - Tall.ra[J]) * np.cos(np.deg2rad(Tall.dec[I])), Tall.dec[I] - Tall.dec[J]) cutouts.append((Tall, I[dist < 0.5], 'Close pairs', None)) plt.clf() plt.hist(dist, 50, histtype='step', color='b') plt.xlabel('Match distance (arcsec)') plt.title('%i sources, %i matches' % (len(Tall), len(I))) ps.savefig() plt.clf() plothist(Tall.bx0[I] - Tall.bx0[J], Tall.by0[I] - Tall.by0[J], 200, range=((-20, 20), (-20, 20))) plt.xlabel('d(x0)') plt.ylabel('d(y0)') plt.title('%i sources, %i matches' % (len(Tall), len(I))) ps.savefig() # plt.clf() # plt.plot(np.vstack((Tall.bx0[I], Tall.bx0[J])), # np.vstack((Tall.by0[I], Tall.by0[J])), 'b-') # ps.savefig() T.srcid = (T.brickid.astype(np.int64) << 32 | T.objid) F.srcid = (F.brickid.astype(np.int64) << 32 | F.objid) print 'Total of', len(F), 'forced' print len(np.unique(F.srcid)), 'unique source in forced' tmap = dict([(s, i) for i, s in enumerate(T.srcid)]) T.forced_g = [[] for i in range(len(T))] T.forced_r = [[] for i in range(len(T))] T.forced_z = [[] for i in range(len(T))] for f in F: i = tmap[f.srcid] forced = T.get('forced_%s' % f.filter)[i] forced.append(f.flux) allbands = 'ugrizY' # pack into arrays bands = 'grz' for band in bands: flist = T.get('forced_%s' % band) nmax = max([len(f) for f in flist]) arr = np.zeros((len(T), nmax), np.float32) for i, f in enumerate(flist): arr[i, :len(f)] = f T.set('forced_%s' % band, arr) ib = allbands.index(band) flux = T.decam_flux[:, ib] arr = np.zeros(len(T), np.float32) for i, f in enumerate(flist): arr[i] = np.mean([(fi - flux[i])**2 for fi in f]) arr = np.sqrt(arr) T.set('forced_rms_%s' % band, arr) T.set('forced_n_%s' % band, np.array([len(f) for f in flist])) # Cut to sources with forced phot T.cut( np.flatnonzero( reduce(np.logical_or, [ np.any(T.get('forced_%s' % band) > 0, axis=1) for band in bands ]))) print 'Cut to', len(T), 'sources with forced phot' flux = T.decam_flux[:, 1] forced = T.forced_g rms = T.forced_rms_g N = T.forced_n_g I = np.flatnonzero((N > 1) * (flux > 1e3)) print len(I), 'with flux > 10^3' #print 'flux', flux[I] #print 'forced' for f, ff, r, n in zip(flux[I], forced[I, :], rms[I], N[I]): print 'flux', f, 'n', n, 'forced RMS', r, 'forced', ff # Compute some summary stats # allbands = 'ugrizY' # for b in bands: # ib = allbands.index(b) # forced = T.get('forced_%s' % b) # flux = T.decam_flux[:,ib] # T.set('forced_rms_%s' % b, np.sqrt([np.mean([(fi - fl)**2 for fi in flist if fi != 0]) # for flist,fl in zip(forced, flux)])) imgs = {} allbands = 'ugrizY' for b in bands: ib = allbands.index(b) plt.clf() f = T.get('forced_%s' % b) n, nf = f.shape flux = T.decam_flux[:, ib] lo, hi = 1e-2, 1e5 plt.plot([lo, hi], [lo, hi], 'r-') plt.errorbar(flux, flux, fmt='none', yerr=1. / np.sqrt(T.decam_flux_ivar[:, ib]), ecolor='r') for i in range(nf): plt.plot(T.decam_flux[:, ib], f[:, i], 'b.', alpha=0.25) plt.axis([lo, hi, lo, hi]) plt.xscale('log') plt.yscale('log') plt.xlabel('Combined flux') plt.ylabel('Forced-photometry flux') plt.title('Forced phot: %s band' % b) ps.savefig() for b in bands: ib = allbands.index(b) plt.clf() f = T.get('forced_%s' % b) n, nf = f.shape flux = T.decam_flux[:, ib] lo, hi = -0.5, 4 plt.axhline(1., color='r') plt.axhline(0., color='k', alpha=0.25) for i in range(nf): plt.plot(flux, np.clip(f[:, i] / flux, lo, hi), 'b.', alpha=0.25) plt.ylim(lo - 0.1, hi + 0.1) plt.xlim(1e-2, 1e5) plt.xscale('log') #plt.yscale('log') plt.xlabel('Combined flux') plt.ylabel('Relative forced-photometry flux') plt.title('Forced phot: %s band' % b) ps.savefig() # Large relative flux I = np.flatnonzero((flux > 0.) * ((f[:, i] / flux) > 4.)) #print 'Relative fluxes:', (f[:,i]/flux)[I] #print 'Fluxes:', flux[I] I = I[np.argsort(-flux[I])] #print 'Sorted fluxes:', flux[I] labels = [] for t in T[I]: fluxes = t.get('forced_%s' % b) ib = allbands.index(b) flux = t.decam_flux[ib] txt = ('%.1f / %.1f / %.1f | %.1f' % (fluxes[0], fluxes[1], fluxes[2], flux)) labels.append(txt) cutouts.append((T, I, 'Large relative flux: %s band' % b, labels)) for b in bands: ib = allbands.index(b) plt.clf() rms = T.get('forced_rms_%s' % b) N = T.get('forced_n_%s' % b) flux = T.decam_flux[:, ib] lo, hi = -0.5, 4 #plt.axhline(1., color='r') #plt.axhline(0., color='k', alpha=0.25) I = np.flatnonzero(N > 1) print len(I), 'of', len(rms), 'have >1 exposure' plt.plot(flux[I], rms[I], 'b.', alpha=0.25) #plt.ylim(lo-0.1, hi+0.1) plt.xlim(1e-2, 1e5) plt.xscale('log') plt.yscale('log') plt.xlabel('Combined flux') plt.ylabel('Forced-photometry flux RMS') plt.title('Forced phot: %s band' % b) ps.savefig() # Large relative RMS I = np.flatnonzero((flux > 10.) * (N > 1) * ((rms / flux) > 0.1)) I = I[np.argsort(-flux[I])] cutouts.append((T, I, 'Large relative RMS: %s band' % b, None)) T[I].writeto('large-rms-%s.fits' % b) # Create a fake "brick" WCS bounding the forced-phot objects # rlo = T.ra.min() # rhi = T.ra.max() # dlo = T.dec.min() # dhi = T.dec.max() rlo = Tall.ra.min() rhi = Tall.ra.max() dlo = Tall.dec.min() dhi = Tall.dec.max() if rhi - rlo > 180: print 'No RA wrap-around support' sys.exit(0) dec = (dlo + dhi) / 2. ra = (rlo + rhi) / 2. pixscale = 0.262 ddec = (dhi - dlo) H = ddec * 3600. / pixscale dra = (rhi - rlo) W = dra * np.cos(np.deg2rad(dec)) * 3600. / pixscale H = int(np.ceil(H)) W = int(np.ceil(W)) print 'Target image shape', (H, W) targetwcs = Tan(ra, dec, W / 2. + 0.5, H / 2. + 0.5, -pixscale / 3600., 0., 0., pixscale / 3600., float(W), float(H)) img = np.zeros((H, W, 3), np.uint8) print 'img', img.shape decals = Decals() for brickname in bricks: brick = decals.get_brick_by_name(brickname) brickwcs = wcs_for_brick(brick) try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, brickwcs, [], 3) except: continue brickimg = plt.imread( os.path.join('dr1', 'coadd', brickname[:3], brickname, 'decals-%s-image.jpg' % brickname)) print 'brick image', brickimg.shape, brickimg.dtype brickimg = brickimg[::-1, :, :] img[Yo, Xo, :] = brickimg[Yi, Xi, :] # Now fake the "bx,by" coords to refer to 'targetwcs' / 'img' ok, x, y = targetwcs.radec2pixelxy(T.ra, T.dec) T.bx = x - 1 T.by = y - 1 ok, x, y = targetwcs.radec2pixelxy(Tall.ra, Tall.dec) Tall.bx = x - 1 Tall.by = y - 1 print 'Tall:', len(Tall), 'sources' # plt.clf() # dimshow(img) # ps.savefig() # # ax = plt.axis() # plt.plot(Tall.bx, Tall.by, 'r.') # plt.axis(ax) # ps.savefig() for TT, I, desc, labels in cutouts: plt.subplots_adjust(left=0.05, right=0.95, bottom=0.05, top=0.95, hspace=0.05, wspace=0.05) kwa = dict(rows=6, cols=9) plt.clf() plot_objects(TT[I], None, img, targetwcs, **kwa) plt.suptitle(desc) ps.savefig() plt.clf() plot_objects(TT[I], Tall, img, targetwcs, labels=labels, **kwa) plt.suptitle(desc) ps.savefig()
def plot_unmatched(): from bigboss_test import radecroi ''' select run, rerun, camcol, field, nChild, probPSF, psfFlux_u, psfFlux_g, psfFlux_r, psfFlux_i, psfFlux_z, deVRad_u, deVRad_g, deVRad_r, deVRad_i, deVRad_z, deVAB_u, deVAB_g, deVAB_r, deVAB_i, deVAB_z, deVPhi_u, deVPhi_g, deVPhi_r, deVPhi_i, deVPhi_z, deVFlux_u, deVFlux_g, deVFlux_r, deVFlux_i, deVFlux_z, expRad_u, expRad_g, expRad_r, expRad_i, expRad_z, expAB_u, expAB_g, expAB_r, expAB_i, expAB_z, expPhi_u, expPhi_g, expPhi_r, expPhi_i, expPhi_z, expFlux_u, expFlux_g, expFlux_r, expFlux_i, expFlux_z, fracDeV_u, fracDeV_g, fracDeV_r, fracDeV_i, fracDeV_z, flags_u, flags_g, flags_r, flags_i, flags_z, probPSF_u, probPSF_g, probPSF_r, probPSF_i, probPSF_z, ra, dec from PhotoPrimary where ra between 333.5 and 335.5 and dec between -0.5 and 1.5 ''' ''' -> 124k rows. (sdss-cas-testarea.fits) Distinct runs: 2585, 2728, 7712, 4203, 2583, 4192, 4207, 4184, 2662, 7717 Run #sources 2583 663 2585 675 2662 4 2728 156 4184 762 4192 36135 4203 5 4207 44078 7712 12047 7717 29911 ''' ''' select run, rerun, camcol, field, nChild, probPSF, psfFlux_u, psfFlux_g, psfFlux_r, psfFlux_i, psfFlux_z, deVRad_u, deVRad_g, deVRad_r, deVRad_i, deVRad_z, deVAB_u, deVAB_g, deVAB_r, deVAB_i, deVAB_z, deVPhi_u, deVPhi_g, deVPhi_r, deVPhi_i, deVPhi_z, deVFlux_u, deVFlux_g, deVFlux_r, deVFlux_i, deVFlux_z, expRad_u, expRad_g, expRad_r, expRad_i, expRad_z, expAB_u, expAB_g, expAB_r, expAB_i, expAB_z, expPhi_u, expPhi_g, expPhi_r, expPhi_i, expPhi_z, expFlux_u, expFlux_g, expFlux_r, expFlux_i, expFlux_z, fracDeV_u, fracDeV_g, fracDeV_r, fracDeV_i, fracDeV_z, flags_u, flags_g, flags_r, flags_i, flags_z, probPSF_u, probPSF_g, probPSF_r, probPSF_i, probPSF_z, ra, dec, resolveStatus, score into mydb.wisetest from PhotoObjAll where ra between 333.5 and 335.5 and dec between -0.5 and 1.5 and (resolveStatus & ( dbo.fResolveStatus('SURVEY_PRIMARY') | dbo.fResolveStatus('SURVEY_BADFIELD') | dbo.fResolveStatus('SURVEY_EDGE'))) != 0 --> sdss-cas-testarea-2.fits select run, rerun, camcol, field, nChild, probPSF, psfFlux_u, psfFlux_g, psfFlux_r, psfFlux_i, psfFlux_z, deVRad_u, deVRad_g, deVRad_r, deVRad_i, deVRad_z, deVAB_u, deVAB_g, deVAB_r, deVAB_i, deVAB_z, deVPhi_u, deVPhi_g, deVPhi_r, deVPhi_i, deVPhi_z, deVFlux_u, deVFlux_g, deVFlux_r, deVFlux_i, deVFlux_z, expRad_u, expRad_g, expRad_r, expRad_i, expRad_z, expAB_u, expAB_g, expAB_r, expAB_i, expAB_z, expPhi_u, expPhi_g, expPhi_r, expPhi_i, expPhi_z, expFlux_u, expFlux_g, expFlux_r, expFlux_i, expFlux_z, fracDeV_u, fracDeV_g, fracDeV_r, fracDeV_i, fracDeV_z, flags_u, flags_g, flags_r, flags_i, flags_z, probPSF_u, probPSF_g, probPSF_r, probPSF_i, probPSF_z, ra, dec, resolveStatus, score into mydb.wisetest from PhotoObjAll where ra between 333.5 and 335.5 and dec between -0.5 and 1.5 and (resolveStatus & ( dbo.fResolveStatus('SURVEY_PRIMARY') | dbo.fResolveStatus('SURVEY_BADFIELD') | dbo.fResolveStatus('SURVEY_EDGE') | dbo.fResolveStatus('SURVEY_BEST') )) != 0 --> sdss-cas-testarea-3.fits ''' ''' Check it out: spatial source density looks fine. No overlap between runs. ''' rng = ((333.5, 335.5), (-0.5, 1.5)) from astrometry.util.plotutils import PlotSequence ps = PlotSequence('sdss') if False: r, d = np.mean(rng[0]), np.mean(rng[1]) RCF = radec_to_sdss_rcf(r, d, radius=2. * 60., tablefn='window_flist-DR9.fits') print('Found', len(RCF), 'fields in range') for run, c, f, ra, dec in RCF: print(' ', run, c, f, 'at', ra, dec) from astrometry.blind.plotstuff import PlotSequence plot = Plotstuff(rdw=(r, d, 10), size=(1000, 1000), outformat='png') plot.color = 'white' plot.alpha = 0.5 plot.apply_settings() T = fits_table('window_flist-DR9.fits') I, J, d = match_radec(T.ra, T.dec, r, d, 10) T.cut(I) print('Plotting', len(T)) for i, (m0, m1, n0, n1, node, incl) in enumerate(zip(T.mu_start, T.mu_end, T.nu_start, T.nu_end, T.node, T.incl)): #rr,dd = [],[] for j, (m, n) in enumerate([(m0, n0), (m0, n1), (m1, n1), (m1, n0), (m0, n0)]): ri, di = munu_to_radec_deg(m, n, node, incl) # rr.append(ri) # dd.append(di) if j == 0: plot.move_to_radec(ri, di) else: plot.line_to_radec(ri, di) plot.stroke() plot.plot_grid(2, 2, 5, 5) plot.write('fields.png') # CAS PhotoObjAll.resolveStatus bits sprim = 0x100 sbad = 0x800 sedge = 0x1000 sbest = 0x200 if False: T = fits_table('sdss-cas-testarea.fits') plt.clf() plothist(T.ra, T.dec, 200, range=rng) plt.title('PhotoPrimary') ps.savefig() T = fits_table('sdss-cas-testarea-2.fits') plt.clf() plothist(T.ra, T.dec, 200, range=rng) plt.title('PhotoObjAll: SURVEY_PRIMARY | SURVEY_BADFIELD | SURVEY_EDGE') ps.savefig() T = fits_table('sdss-cas-testarea-3.fits') plt.clf() plothist(T.ra, T.dec, 200, range=rng) plt.title( 'PhotoObjAll: SURVEY_PRIMARY | SURVEY_BADFIELD | SURVEY_EDGE | SURVEY_BEST') ps.savefig() for j, (flags, txt) in enumerate([(sprim, 'PRIM'), (sbad, 'BAD'), (sedge, 'EDGE'), (sbest, 'BEST')]): I = np.flatnonzero( (T.resolvestatus & (sprim | sbad | sedge | sbest)) == flags) print(len(I), 'with', txt) if len(I) == 0: continue plt.clf() plothist(T.ra[I], T.dec[I], 200, range=rng) plt.title('%i with %s' % (len(I), txt)) ps.savefig() for j, (flags, txt) in enumerate([(sprim | sbad, 'PRIM + BAD'), (sprim | sedge, 'PRIM + EDGE'), (sprim | sbest, 'PRIM + BEST')]): I = np.flatnonzero((T.resolvestatus & flags) > 0) print(len(I), 'with', txt) if len(I) == 0: continue plt.clf() plothist(T.ra[I], T.dec[I], 200, range=rng) plt.title('%i with %s' % (len(I), txt)) ps.savefig() # for run in np.unique(T.run): # I = (T.run == run) # plt.clf() # plothist(T.ra[I], T.dec[I], 200, range=rng) # plt.title('Run %i' % run) # ps.savefig() R = 1. / 3600. I, J, d = match_radec(T.ra, T.dec, T.ra, T.dec, R, notself=True) print(len(I), 'matches') plt.clf() loghist((T.ra[I] - T.ra[J]) * 3600., (T.dec[I] - T.dec[J]) * 3600., 200, range=((-1, 1), (-1, 1))) ps.savefig() ps = PlotSequence('forced') basedir = os.environ.get('BIGBOSS_DATA', '/project/projectdirs/bigboss') wisedatadir = os.path.join(basedir, 'data', 'wise') wisecat = fits_table(os.path.join( wisedatadir, 'catalogs', 'wisecat2.fits')) # plt.clf() #plothist(wisecat.ra, wisecat.dec, 200, range=rng) # plt.savefig('wisecat.png') (ra0, ra1, dec0, dec1) = radecroi ra = (ra0 + ra1) / 2. dec = (dec0 + dec1) / 2. #cas = fits_table('sdss-cas-testarea.fits') #cas = fits_table('sdss-cas-testarea-2.fits') cas = fits_table('sdss-cas-testarea-3.fits') print('Read', len(cas), 'CAS sources') cas.cut((cas.resolvestatus & sedge) == 0) print('Cut to ', len(cas), 'without SURVEY_EDGE set') # Check out WISE / SDSS matches. wise = wisecat sdss = cas print(len(sdss), 'SDSS sources') print(len(wise), 'WISE sources') R = 10. I, J, d = match_radec(wise.ra, wise.dec, sdss.ra, sdss.dec, R / 3600., nearest=True) print(len(I), 'matches') print('max dist:', d.max()) plt.clf() plt.hist(d * 3600., 100, range=(0, R), log=True) plt.xlabel('Match distance (arcsec)') plt.ylabel('Number of matches') plt.title('SDSS-WISE astrometric matches') ps.savefig() plt.clf() loghist((wise.ra[I] - sdss.ra[J]) * 3600., (wise.dec[I] - sdss.dec[J]) * 3600., 200, range=((-R, R), (-R, R))) plt.title('SDSS-WISE astrometric matches') plt.xlabel('dRA (arcsec)') plt.ylabel('dDec (arcsec)') ps.savefig() R = 4. I, J, d = match_radec(wise.ra, wise.dec, sdss.ra, sdss.dec, R / 3600., nearest=True) print(len(I), 'matches') unmatched = np.ones(len(wise), bool) unmatched[I] = False wun = wise[unmatched] plt.clf() plothist(sdss.ra, sdss.dec, 200, range=rng) plt.title('SDSS source density') ps.savefig() plt.clf() plothist(wise.ra, wise.dec, 200, range=rng) plt.title('WISE source density') ps.savefig() plt.clf() #plt.plot(wun.ra, wun.dec, 'r.') #plt.axis(rng[0] + rng[1]) plothist(wun.ra, wun.dec, 200, range=rng) plt.title('Unmatched WISE sources') ps.savefig() for band in 'ugriz': sdss.set('psfmag_' + band, NanoMaggies.nanomaggiesToMag(sdss.get('psfflux_' + band))) # plt.clf() # loghist(wise.w1mpro[I], sdss.psfmag_r[J], 200) # plt.xlabel('WISE w1mpro') # plt.ylabel('SDSS psfflux_r') # ps.savefig() for band in 'riz': ax = [0, 10, 25, 5] plt.clf() mag = sdss.get('psfmag_' + band)[J] loghist(mag - wise.w1mpro[I], mag, 200, range=((ax[0], ax[1]), (ax[3], ax[2]))) plt.xlabel('SDSS %s - WISE w1' % band) plt.ylabel('SDSS ' + band) plt.axis(ax) ps.savefig() for w, t in [(wise[I], 'Matched'), (wun, 'Unmatched')]: plt.clf() w1 = w.get('w1mpro') w2 = w.get('w2mpro') ax = [-1, 3, 18, 6] loghist(w1 - w2, w1, 200, range=((ax[0], ax[1]), (ax[3], ax[2]))) plt.xlabel('W1 - W2') plt.ylabel('W1') plt.title('WISE CMD for %s sources' % t) plt.axis(ax) ps.savefig() sdssobj = DR9() band = 'r' RCF = np.unique(zip(sdss.run, sdss.camcol, sdss.field)) wcses = [] fns = [] pfn = 'wcses.pickle' if os.path.exists(pfn): print('Reading', pfn) wcses, fns = unpickle_from_file(pfn) else: for r, c, f in RCF: fn = sdssobj.retrieve('frame', r, c, f, band) print('got', fn) fns.append(fn) wcs = Tan(fn, 0) print('got wcs', wcs) wcses.append(wcs) pickle_to_file((wcses, fns), pfn) print('Wrote to', pfn) wisefns = glob(os.path.join(wisedatadir, 'level3', '*w1-int-3.fits')) wisewcs = [] for fn in wisefns: print('Reading', fn) wcs = anwcs(fn, 0) print('Got', wcs) wisewcs.append(wcs) I = np.argsort(wun.w1mpro) wun.cut(I) for i in range(len(wun)): ra, dec = wun.ra[i], wun.dec[i] insdss = -1 for j, wcs in enumerate(wcses): if wcs.is_inside(ra, dec): insdss = j break inwise = -1 for j, wcs in enumerate(wisewcs): if wcs.is_inside(ra, dec): inwise = j break N = 0 if insdss != -1: N += 1 if inwise != -1: N += 1 if N == 0: continue if N != 2: continue plt.clf() ss = 1 plt.subplot(2, N, ss) ss += 1 M = 0.02 I = np.flatnonzero((sdss.ra > (ra - M)) * (sdss.ra < (ra + M)) * (sdss.dec > (dec - M)) * (sdss.dec < (dec + M))) sdssnear = sdss[I] plt.plot(sdss.ra[I], sdss.dec[I], 'b.', alpha=0.7) I = np.flatnonzero((wise.ra > (ra - M)) * (wise.ra < (ra + M)) * (wise.dec > (dec - M)) * (wise.dec < (dec + M))) wisenear = wise[I] plt.plot(wise.ra[I], wise.dec[I], 'rx', alpha=0.7) if insdss: wcs = wcses[j] w, h = wcs.imagew, wcs.imageh rd = np.array([wcs.pixelxy2radec(x, y) for x, y in [(1, 1), (w, 1), (w, h), (1, h), (1, 1)]]) plt.plot(rd[:, 0], rd[:, 1], 'b-', alpha=0.5) if inwise: wcs = wisewcs[j] w, h = wcs.imagew, wcs.imageh rd = np.array([wcs.pixelxy2radec(x, y) for x, y in [(1, 1), (w, 1), (w, h), (1, h), (1, 1)]]) plt.plot(rd[:, 0], rd[:, 1], 'r-', alpha=0.5) plt.plot([ra], [dec], 'o', mec='k', mfc='none', mew=3, ms=20, alpha=0.5) #plt.axis([ra+M, ra-M, dec-M, dec+M]) plt.axis([ra - M, ra + M, dec - M, dec + M]) plt.xticks([ra], ['RA = %0.3f' % ra]) plt.yticks([dec], ['Dec = %0.3f' % dec]) SW = 20 ss = N + 1 plt.subplot(2, N, ss) if insdss != -1: ss += 1 j = insdss wcs = wcses[j] xc, yc = wcs.radec2pixelxy(ra, dec) r, c, f = RCF[j] frame = sdssobj.readFrame(r, c, f, band) #S = 50 S = SW * 3.472 im = frame.image H, W = im.shape y0, x0 = max(0, yc - S), max(0, xc - S) y1, x1 = min(H, yc + S), min(W, xc + S) subim = im[y0:y1, x0:x1] # plt.imshow(subim, interpolation='nearest', origin='lower', # vmax=0.3, extent=[x0,x1,y0,y1]) plt.imshow(subim.T, interpolation='nearest', origin='lower', vmax=0.3, extent=[y0, y1, x0, x1]) # vmax=subim.max()*1.01) ax = plt.axis() sx, sy = wcs.radec2pixelxy(sdssnear.ra, sdssnear.dec) #plt.plot(x, y, 'o', mec='b', mfc='none', ms=15) plt.plot(sy, sx, 'o', mec='b', mfc='none', ms=15) x, y = wcs.radec2pixelxy(wisenear.ra, wisenear.dec) #plt.plot(x, y, 'rx', ms=10) plt.plot(y, x, 'rx', ms=10) # Which way up? x, y = wcs.radec2pixelxy(np.array([ra, ra]), np.array( [0.5, 2.0]) * S * 0.396 / 3600. + dec) #plt.plot(x, y, 'b-', alpha=0.5, lw=2) plt.plot(y, x, 'b-', alpha=0.5, lw=2) plt.axis(ax) plt.gray() plt.title('SDSS %s (%i/%i/%i)' % (band, r, c, f)) # Try to guess the PRIMARY run from the nearest object. for I in np.argsort((sx - xc)**2 + (sy - yc)**2): r, c, f = sdssnear.run[I], sdssnear.camcol[I], sdssnear.field[I] jj = None for j, (ri, ci, fi) in enumerate(RCF): if ri == r and ci == c and fi == f: jj = j break assert(jj is not None) wcs = wcses[jj] xc, yc = wcs.radec2pixelxy(ra, dec) frame = sdssobj.readFrame(r, c, f, band) S = SW * 3.472 im = frame.image H, W = im.shape y0, x0 = max(0, yc - S), max(0, xc - S) y1, x1 = min(H, yc + S), min(W, xc + S) subim = im[y0:y1, x0:x1] if np.prod(subim.shape) == 0: continue print('subim shape', subim.shape) plt.subplot(2, N, 2) plt.imshow(subim.T, interpolation='nearest', origin='lower', vmax=0.3, extent=[y0, y1, x0, x1]) plt.gray() plt.title('SDSS %s (%i/%i/%i)' % (band, r, c, f)) break if inwise != -1: plt.subplot(2, N, ss) ss += 1 j = inwise wcs = wisewcs[j] ok, x, y = wcs.radec2pixelxy(ra, dec) im = pyfits.open(wisefns[j])[0].data S = SW H, W = im.shape y0, x0 = max(0, y - S), max(0, x - S) subim = im[y0: min(H, y + S), x0: min(W, x + S)] plt.imshow(subim, interpolation='nearest', origin='lower', vmax=subim.max() * 1.01) ax = plt.axis() x, y = [], [] for r, d in zip(wisenear.ra, wisenear.dec): ok, xi, yi = wcs.radec2pixelxy(r, d) x.append(xi) y.append(yi) x = np.array(x) y = np.array(y) plt.plot(x - x0, y - y0, 'rx', ms=15) x, y = [], [] for r, d in zip(sdssnear.ra, sdssnear.dec): ok, xi, yi = wcs.radec2pixelxy(r, d) x.append(xi) y.append(yi) x = np.array(x) y = np.array(y) plt.plot(x - x0, y - y0, 'o', mec='b', mfc='none', ms=10) # Which way up? pixscale = 1.375 / 3600. ok, x1, y1 = wcs.radec2pixelxy(ra, dec + 0.5 * S * pixscale) ok, x2, y2 = wcs.radec2pixelxy(ra, dec + 2.0 * S * pixscale) plt.plot([x1 - x0, x2 - x0], [y1 - y0, y2 - y0], 'r-', alpha=0.5, lw=2) plt.axis([ax[1], ax[0], ax[2], ax[3]]) # plt.axis(ax) plt.gray() plt.title('WISE W1 (coadd)') plt.suptitle('WISE unmatched source: w1=%.1f, RA,Dec = (%.3f, %.3f)' % (wun.w1mpro[i], ra, dec)) ps.savefig() rcfs = zip(sdssnear.run, sdssnear.camcol, sdssnear.field) print('Nearby SDSS sources are from:', np.unique(rcfs)) return
st = np.array(st) plt.loglog([nm0[I],nm0[I]], [np.maximum(1e-6, (mn-st) / nm0[I]), np.maximum(1e-6, (mn+st) / nm0[I])], 'b-', alpha=0.5) plt.axhline(1., color='k', lw=2, alpha=0.5) plt.xlabel('WISE brightness (nanomaggies)') plt.ylabel('Tractor-measured brightness / WISE brightness') plt.ylim(0.1, 10.) ps.savefig() # Match to WISE sources r = 4./3600. #I,J,d = match_radec(R.ra, R.dec, W.ra, W.dec, r) #RW = R[I] #WR = W[J] I,J,d = match_radec(T.ra, T.dec, W.ra, W.dec, r) TW = T[I] WT = W[J] WT.nm1 = NanoMaggies.magToNanomaggies(WT.w1mpro) print 'Matched', len(TW), 'Schlegel sources to WISE' if opt.match: r = 4./3600. #r = 10./3600. I,J,d = match_radec(T.ra, T.dec, R.ra, R.dec, r, nearest=opt.nearest) print 'Matched', len(I) T.cut(I) R.cut(J) T.I = I R.J = J
def match_ngc(self, rr, dd, radius, exts, F, primhdr, meas): ''' rr, dd: np arrays: RA,Dec centers of chips/amps radius: scalar, radius of chips/amps exts: list (same len as rr,dd) of extension indices F: fitsio.FITS object meas: measurement class ''' I, J, d = match_radec(rr, dd, self.cat.ra, self.cat.dec, radius) # plt.clf() # angles = np.linspace(0, 2.*np.pi, 40) # for r,d in zip(rr, dd): # plt.plot(r + radius * np.sin(angles) / np.cos(np.deg2rad(d)), d + radius * np.cos(angles), 'b-') # plt.plot(rr, dd, 'bo') # ax = plt.axis() # plt.plot(self.cat.ra, self.cat.dec, 'r.') # plt.axis(ax) # ps.savefig() print('Matched', len(I), 'NGC objects') if len(I) == 0: return False for j in J: info = ngc_typenames.get(self.cat.classification[j].strip(), '') print(' ', self.cat.name[j], info) # Potential tweet texts and plot filenames tweets = [] goodplots = [] for i, j in zip(I, J): ext = exts[i] obj = self.cat[j] obj.name = obj.name.strip() hdr = F[ext].read_header() extname = hdr['EXTNAME'].strip() expnum = primhdr['EXPNUM'] wcs = meas.get_wcs(hdr) ok, x, y = wcs.radec2pixelxy(obj.ra, obj.dec) x = x - 1 y = y - 1 # Choose cutout area pixrad = 1.4 * obj.radius * 3600. / wcs.pixel_scale() pixrad = max(pixrad, 100) # print('radius:', obj.radius, 'pixel radius:', pixrad) # HACK #pixrad *= 3 r = pixrad tt = '%s in exp %i ext %s (%i)' % (obj.name, expnum, extname, ext) print(tt) # Find the cutout region... does it actually overlap the chip? H, W = wcs.shape xl, xh = int(np.clip(x - r, 0, W - 1)), int(np.clip(x + r, 0, W - 1)) yl, yh = int(np.clip(y - r, 0, H - 1)), int(np.clip(y + r, 0, H - 1)) if xl == xh or yl == yh: print('no actual overlap with image') continue sh, sw = yh - yl, xh - xl if sh < 25 or sw < 25: print('tiny overlap', sw, 'x', sh) continue # Measure the image! flat = None if self.read_flats: band = meas.get_band(hdr) flat = self.get_flat(band, ext, meas) if flat is not None: print('flat: range', flat.min(), flat.max(), 'median', np.median(flat)) meas.ext = ext meas.edge_trim = 20 debugps = None if self.opt.ps: debugps = ps try: M = meas.run(n_fwhm=1, verbose=False, get_image=True, flat=flat, ps=debugps) except KeyboardInterrupt: sys.exit(0) except: import traceback print('Failed to measure file', meas.fn, 'ext', ext, ':') traceback.print_exc() continue #print('Measured:', M.keys()) raw = M['image'] # Now repeat the cutout check with the trimmed image wcs = M['wcs'] # Trim WCS to trimmed raw image shape trim_x0, trim_y0 = M['trim_x0'], M['trim_y0'] H, W = raw.shape wcs = wcs.get_subimage(trim_x0, trim_y0, W, H) ok, x, y = wcs.radec2pixelxy(obj.ra, obj.dec) x = x - 1 y = y - 1 xl, xh = int(np.clip(x - r, 0, W - 1)), int(np.clip(x + r, 0, W - 1)) yl, yh = int(np.clip(y - r, 0, H - 1)), int(np.clip(y + r, 0, H - 1)) if xl == xh or yl == yh: print('no actual overlap with image') continue subimg = raw[yl:yh, xl:xh] sh, sw = subimg.shape if sh < 25 or sw < 25: print('tiny overlap', sw, 'x', sh) continue subwcs = wcs.get_subimage(xl, yl, sw, sh) if False: # Flat image for the same CCD region. flatfn = '/tmp/mzls/mos3.127506.fits' # meas_class = get_measurer_class_for_file(flatfn) # if meas_class is None: # print('Failed to identify camera in', flatfn) # return # flatmeas = meas_class(flatfn, 0, self.nom) # print('Flat meas:', flatmeas) # flatmeas.ext = ext # flathdr = fitsio.read_header(flatfn) # flatmeas.primhdr = flathdr # flatmeas.edge_trim = 20 # FM = flatmeas.run(n_fwhm=1, verbose=False, get_image=True) # flatraw = FM['image'] F = fitsio.FITS(flatfn) flatraw, flathdr = meas.read_raw(F, ext) flatsub = flatraw[yl:yh, xl:xh] zerofn = '/tmp/mzls/mos3.127522.fits' F = fitsio.FITS(zerofn) zeroraw, zerohdr = meas.read_raw(F, ext) zerosub = zeroraw[yl:yh, xl:xh] rawfn = meas.fn F = fitsio.FITS(rawfn) rawimg, rawhdr = meas.read_raw(F, ext) rawsub = rawimg[yl:yh, xl:xh] # Astrometric shifts dx = M['dx'] dy = M['dy'] aff = M['affine'] x = (xl + xh) / 2 y = (yl + yh) / 2 #print('Affine correction terms:', aff) adx = x - aff[0] ady = y - aff[1] corrx = aff[2] + aff[3] * adx + aff[4] * ady - adx corry = aff[5] + aff[6] * adx + aff[7] * ady - ady print('Affine correction', corrx, corry) # Shift the 'subwcs' to account for astrometric offset cx, cy = subwcs.get_crpix() subwcs.set_crpix((cx - dx - corrx, cy - dy - corry)) # Now grab existing data from the LegacySurvey site # What size of image are we going to request? scale = 1. if max(sh, sw) > 1024: scale = 4. elif max(sh, sw) > 512: scale = 2. rh, rw = int(np.ceil(sh / scale)), int(np.ceil(sw / scale)) # make it square mx = max(rh, rw) rw = rh = mx fitsimgs = [] # We'll resample the new image into the existing-image WCS. newimg = None for layer in legacy_survey_layers: url = ( 'http://legacysurvey.org/viewer-dev/fits-cutout/?ra=%.4f&dec=%.4f&pixscale=%.3f&width=%i&height=%i&layer=%s' % (obj.ra, obj.dec, subwcs.pixel_scale() * scale, rw, rh, layer)) print('URL:', url) r = requests.get(url) ftmp = tempfile.NamedTemporaryFile() ftmp.write(r.content) ftmp.flush() #fits,hdr = fitsio.read(ftmp.name, header=True) fitsfile = fitsio.FITS(ftmp.name) ftmp.close() print('FITS file:', len(fitsfile), 'extensions') hdr = fitsfile[0].read_header() fits = fitsfile[0].read() #print('fits:', fits) if fits is None: print('no coverage in layer', layer) continue # If you need to keep a copy (debugging...) # f,tmpfn = tempfile.mkstemp(suffix='.fits') # os.write(f, r.content) # os.close(f) # fits,hdr = fitsio.read(tmpfn, header=True) # print('Wrote FITS to', tmpfn) if np.all(fits == 0): continue ### HACK -- surface brightness correction... if layer == 'sdssco': s = (subwcs.pixel_scale() / 0.396) fits *= s**2 N, ww, hh = fits.shape # pull out the image planes imgs = [fits[n, :, :] for n in range(N)] bands = hdr['BANDS'].strip() fitsimgs.append((layer, bands, imgs)) # Resample the new image to this layer's WCS if newimg is not None: continue thiswcs = Tan(hdr) newimg = np.zeros((rh, rw), dtype=subimg.dtype) try: #Yo,Xo,Yi,Xi,rims = resample_with_wcs(thiswcs, subwcs) # Laczos Yo, Xo, Yi, Xi, rims = resample_with_wcs( thiswcs, subwcs, [subimg]) except: continue #newimg[Yo,Xo] = subimg[Yi,Xi] newimg[Yo, Xo] = rims[0] if len(fitsimgs) == 0: # No overlap with existing surveys continue #print() newband = primhdr['FILTER'][0] #print('New image is', newband) if False: mn, mx = np.percentile(flatsub, [25, 99]) plt.clf() #plt.imshow(newflat, interpolation='nearest', origin='lower', plt.imshow(flatsub, interpolation='nearest', origin='lower', vmin=mn, vmax=mx) #plt.colorbar() plt.colorbar() plt.savefig('ngcbot-flat.png') med = np.median(newflat.ravel()) #mn,mx = np.percentile(newimg, [25,99]) #mn,mx = np.percentile(newimg, [25,90]) #mn,mx = np.percentile(rawsub, [25,95]) mn, mx = np.percentile(rawsub, [50, 95]) plt.clf() # plt.subplot(1,2,1) #plt.imshow(newimg, interpolation='nearest', origin='lower', # plt.imshow(subimg, interpolation='nearest', origin='lower', plt.imshow(rawsub, interpolation='nearest', origin='lower', vmin=mn, vmax=mx) plt.colorbar() plt.savefig('ngcbot-unflattened.png') plt.clf() #plt.subplot(1,2,2) #plt.imshow(newimg / (newflat / med), interpolation='nearest', origin='lower', #plt.imshow(subimg / (flatsub / med), interpolation='nearest', origin='lower', plt.imshow(rawsub / (flatsub / med), interpolation='nearest', origin='lower', vmin=mn, vmax=mx) plt.colorbar() plt.savefig('ngcbot-flattened.png') mn, mx = np.percentile(zerosub, [5, 95]) plt.clf() plt.imshow(zerosub, interpolation='nearest', origin='lower', vmin=mn, vmax=mx) plt.colorbar() plt.savefig('ngcbot-zero.png') zp = M['zp'] zpscale = 10.**((zp - 22.5) / 2.5) exptime = primhdr['EXPTIME'] newimg /= (zpscale * exptime) nh, nw = newimg.shape coverage = np.sum(newimg != 0) / float(nw * nh) print('Fraction', coverage, 'of new image has data') def my_rgb(imgs, bands, **kwargs): return sdss_rgb(imgs, bands, scales=dict(g=6.0, r=3.4, i=2.5, z=2.2), m=-0.02, clip=False, **kwargs) def grayscale(img, band): rgb = my_rgb([img, img, img], [band, band, band]) index = 'zrg'.index(newband) gray = rgb[:, :, index] return gray # plot title args targs = dict(fontsize=8) # DEBUG if self.opt.ps: ocx, ocy = subwcs.get_crpix() subwcs.set_crpix((cx, cy)) newimgA = np.zeros((rh, rw), dtype=subimg.dtype) Yo, Xo, Yi, Xi, rims = resample_with_wcs( thiswcs, subwcs, [subimg]) newimgA[Yo, Xo] = rims[0] subwcs.set_crpix((cx - dx, cy - dy)) newimgB = np.zeros((rh, rw), dtype=subimg.dtype) Yo, Xo, Yi, Xi, rims = resample_with_wcs( thiswcs, subwcs, [subimg]) newimgB[Yo, Xo] = rims[0] subwcs.set_crpix((cx - dx - corrx, cy - dy - corry)) newimgC = np.zeros((rh, rw), dtype=subimg.dtype) Yo, Xo, Yi, Xi, rims = resample_with_wcs( thiswcs, subwcs, [subimg]) newimgC[Yo, Xo] = rims[0] #newgray = grayscale(newimg, newband) #hi = np.percentile(newgray, 99.9) grayargs = dict(interpolation='nearest', origin='lower', cmap='gray', vmin=0.) #vmin=0., vmax=hi, cmap='gray') (layer, bands, imgs) = fitsimgs[0] nicelayer = nicelayernames.get(layer, layer) for band, img in zip(bands, imgs): newbands = ['g', 'r', 'z'] newindex = dict(g=0, r=1, i=2, z=2) if newindex[band] == newindex[newband]: oldgray = grayscale(img, band) plt.clf() plt.imshow(oldgray, **grayargs) plt.title(nicelayer) ps.savefig() plt.clf() plt.imshow(grayscale(newimgA, newband), **grayargs) #plt.imshow(newimgA, **grayargs) plt.title('No astromety correction') ps.savefig() plt.clf() plt.imshow(grayscale(newimgB, newband), **grayargs) #plt.imshow(newimgB, **grayargs) plt.title('Astromety shift only correction') ps.savefig() plt.clf() plt.imshow(grayscale(newimgC, newband), **grayargs) #plt.imshow(newimgC, **grayargs) plt.title('Astromety shift+affine correction') ps.savefig() subwcs.set_crpix((ocx, ocy)) plt.clf() plt.subplots_adjust(left=0.03, right=0.97, bottom=0.03) NC = 1 + len(fitsimgs) NR = 2 # New Image plot newgray = grayscale(newimg, newband) hi = np.percentile(newgray, 99.9) grayargs = dict(interpolation='nearest', origin='lower', vmin=0., vmax=hi, cmap='gray') plt.subplot(NR, NC, 1) plt.imshow(newgray, **grayargs) plt.xticks([]) plt.yticks([]) plt.title('New image (%s)' % newband, **targs) # Select images & bands for the New RGB image. zeroimg = np.zeros_like(newimg) newimgs = [zeroimg, zeroimg, zeroimg] # sdss_rgb reverses the order, so do like grz. newbands = ['g', 'r', 'z'] newindex = dict(g=0, r=1, i=2, z=2) # Start with the new image plane of the New RGB image j = newindex[newband] newimgs[j] = newimg newbands[j] = newband #print('Setting band', newband, '(index %i)'%j, 'to new image') # We wait to choose the scaling until after the "New RGB" image # has been built, so we store the RGB images along the way... rgbs = [] # In the link to the legacysurvey viewer, we select the best # imaging layer. The DECaLS layer is listed first, so if it # has three bands, it wins. bestdata = None bestn = 0 for i, (layer, bands, imgs) in enumerate(fitsimgs): nicelayer = nicelayernames.get(layer, layer) # For DECaLS missing bands: "--z" nicebands = '' goodbands = [] goodimgs = [] for band, img in zip(bands, imgs): if np.all(img == 0.): print('Band', band, 'of', nicelayer, 'is all zero') nicebands += '-' continue goodbands.append(band) goodimgs.append(img) nicebands += band #print(' ', nicelayer, band) j = newindex[band] if newimgs[j] is zeroimg: #print(' Setting band', band, '(index %i)'%j, 'to', nicelayer) newimgs[j] = img newbands[j] = band elif band == newbands[j]: # patch empty regions if same band #print(' Patching index %i from' % j, nicelayer, 'band', band) Z = (newimgs[j] == 0) newimgs[j][Z] = img[Z] # z -> i if newindex[band] == newindex[newband]: # Old image grayscale oldgray = grayscale(img, band) plt.subplot(NR, NC, 2 + i) plt.imshow(oldgray, **grayargs) plt.xticks([]) plt.yticks([]) plt.title('%s (%s)' % (nicelayer, band), **targs) if len(goodbands) == 1: #rgb = grayscale(goodimgs[0], goodbands[0]) img, band = goodimgs[0], goodbands[0] rgb = my_rgb([img, img, img], [band, band, band]) else: rgb = my_rgb(imgs, bands) if len(goodbands) > bestn: bestn = len(goodbands) bestdata = layer # print('bands for', nicelayer, ':', bands, ', actually', nicebands) rgbs.append( (2 + i + NC, rgb, '%s (%s)' % (nicelayer, nicebands))) # list to string newbands = ''.join(newbands) #print('Newbands:', newbands) # New RGB plt.subplot(NR, NC, 1 + NC) rgb = my_rgb(newimgs, newbands) lo = 0. hi = np.percentile(rgb.ravel(), 99.9) rgb = np.clip((rgb - lo) / (hi - lo), 0., 1.) plt.imshow(rgb, interpolation='nearest', origin='lower') plt.xticks([]) plt.yticks([]) plt.title('New+Old (%s)' % newbands, **targs) # Old RGBs for sp, rgb, tt in rgbs: plt.subplot(NR, NC, sp) rgb = np.clip((rgb - lo) / (hi - lo), 0., 1.) plt.imshow(rgb, interpolation='nearest', origin='lower') plt.xticks([]) plt.yticks([]) plt.title(tt, **targs) # NGC classification info = '' info = ngc_typenames.get(obj.classification.strip(), '') if len(info): info = '(' + info + ') ' plt.suptitle( '%s %sin %s %i-%s: %s band' % (obj.name, info, nice_camera_name, expnum, extname, newband)) # Save / show plot if self.opt.show: plt.draw() plt.show(block=False) plt.pause(0.001) plotfn = ('ngcbot-%i-%s-%s.png' % (expnum, extname, obj.name.replace(' ', '_'))) if self.opt.plotdir: plotfn = os.path.join(self.opt.plotdir, plotfn) plt.savefig(plotfn) print('Saved', plotfn) good_coverage = 0.75 if self.opt.coverage is not None: good_coverage = self.opt.coverage if coverage > good_coverage: goodplots.append(plotfn) # Compose tweet text if self.opt.tweet: import urllib url = 'http://legacysurvey.org/viewer/?ra=%.3f&dec=%.3f' % ( obj.ra, obj.dec) if bestdata is not None: url += '&layer=%s' % bestdata url2 = ('http://ned.ipac.caltech.edu/cgi-bin/objsearch?' + urllib.urlencode( dict(objname=obj.name, corr_z=1, list_limit=5, img_stamp='YES'))) dateobs = primhdr['DATE-OBS'].strip() # 2017-03-06T00:15:40.101482 dateobs = dateobs[:19] dateobs = dateobs.replace('T', ' ') txt = ('%s observed %s %sin image %i-%s: %s band' % (nice_camera_name, obj.name, info, expnum, extname, newband) + ' at %s UT' % dateobs + '\n' + url + '\n' + url2) print('Tweet text:', txt) if coverage > good_coverage: tweets.append((txt, plotfn)) else: print('Coverage', coverage, 'less than target', good_coverage, 'so not tweeting') # Select a random good-looking plot if len(goodplots): irandom = np.random.randint(0, len(goodplots)) # Copy that plot to ngcbot-latest.png plotfn = goodplots[irandom] import shutil latest = 'ngcbot-latest.png' print('Copying', plotfn, 'to', latest) shutil.copy(plotfn, latest) # Tweet one NGC object per exposure, chosen randomly. if len(tweets): assert (len(tweets) == len(goodplots)) txt, plotfn = tweets[irandom] if self.opt.tweet: send_tweet(txt, plotfn) return True
def star_profiles(ps): # Run an example CCD, 292604-N4, with fairly large difference vs PS1. # python -c "from astrometry.util.fits import *; T = merge_tables([fits_table('/project/projectdirs/desiproc/dr3/tractor/244/tractor-244%s.fits' % b) for b in ['2p065','4p065', '7p065']]); T.writeto('tst-cat.fits')" # python legacypipe/forced_photom_decam.py --save-data tst-data.fits --save-model tst-model.fits 292604 N4 tst-cat.fits tst-phot.fits # -> tst-{model,data,phot}.fits datafn = 'tst-data.fits' modfn = 'tst-model.fits' photfn = 'tst-phot.fits' catfn = 'tst-cat.fits' img = fitsio.read(datafn) mod = fitsio.read(modfn) phot = fits_table(photfn) cat = fits_table(catfn) print(len(phot), 'forced-photometry results') margin = 25 phot.cut((phot.x > 0+margin) * (phot.x < 2046-margin) * (phot.y > 0+margin) * (phot.y < 4096-margin)) print(len(phot), 'in bounds') cmap = dict([((b,o),i) for i,(b,o) in enumerate(zip(cat.brickname, cat.objid))]) I = np.array([cmap.get((b,o), -1) for b,o in zip(phot.brickname, phot.objid)]) print(np.sum(I >= 0), 'forced-phot matched cat') phot.type = cat.type[I] wcs = Sip(datafn) phot.ra,phot.dec = wcs.pixelxy2radec(phot.x+1, phot.y+1) phot.cut(np.argsort(phot.flux)) phot.sn = phot.flux * np.sqrt(phot.flux_ivar) phot.cut(phot.sn > 5) print(len(phot), 'with S/N > 5') ps1 = ps1cat(ccdwcs=wcs) stars = ps1.get_stars() print(len(stars), 'PS1 sources') # Now cut to just *stars* with good colors stars.gicolor = stars.median[:,0] - stars.median[:,2] keep = (stars.gicolor > 0.4) * (stars.gicolor < 2.7) stars.cut(keep) print(len(stars), 'PS1 stars with good colors') stars.cut(np.minimum(stars.stdev[:,1], stars.stdev[:,2]) < 0.05) print(len(stars), 'PS1 stars with min stdev(r,i) < 0.05') I,J,d = match_radec(phot.ra, phot.dec, stars.ra, stars.dec, 1./3600.) print(len(I), 'matches') plt.clf() ha=dict(histtype='step', bins=20, range=(0,100), normed=True) plt.hist(phot.flux, color='b', **ha) plt.hist(phot.flux[I], color='r', **ha) ps.savefig() plt.clf() plt.hist(phot.flux * np.sqrt(phot.flux_ivar), bins=100, range=(-10, 50)) plt.xlabel('Flux S/N') ps.savefig() K = np.argsort(phot.flux[I]) I = I[K] J = J[K] ix = np.round(phot.x).astype(int) iy = np.round(phot.y).astype(int) sz = 10 P = np.flatnonzero(phot.type == 'PSF ') print(len(P), 'PSFs') imed = len(P)/2 i1 = int(len(P) * 0.75) i2 = int(len(P) * 0.25) N = 401 allmods = [] allimgs = [] for II,tt in [#(I[:len(I)/2], 'faint matches to PS1'), #(I[len(I)/2:], 'bright matches to PS1'), #(P[i2: i2+N], '25th pct PSFs'), #(P[imed: imed+N], 'median PSFs'), #(P[i1: i1+N], '75th pct PSFs'), #(P[-25:], 'brightest PSFs'), (P[i2:imed], '2nd quartile of PSFs'), (P[imed:i1], '3rd quartile of PSFs'), #(P[:len(P)/2], 'faint half of PSFs'), #(P[len(P)/2:], 'bright half of PSFs'), ]: imgs = [] mods = [] shimgs = [] shmods = [] imgsum = modsum = 0 #plt.clf() for i in II: from astrometry.util.util import lanczos_shift_image dy = phot.y[i] - iy[i] dx = phot.x[i] - ix[i] sub = img[iy[i]-sz : iy[i]+sz+1, ix[i]-sz : ix[i]+sz+1] shimg = lanczos_shift_image(sub, -dx, -dy) sub = mod[iy[i]-sz : iy[i]+sz+1, ix[i]-sz : ix[i]+sz+1] shmod = lanczos_shift_image(sub, -dx, -dy) iyslice = img[iy[i], ix[i]-sz : ix[i]+sz+1] myslice = mod[iy[i], ix[i]-sz : ix[i]+sz+1] ixslice = img[iy[i]-sz : iy[i]+sz+1, ix[i]] mxslice = mod[iy[i]-sz : iy[i]+sz+1, ix[i]] mx = iyslice.max() # plt.plot(iyslice/mx, 'b-', alpha=0.1) # plt.plot(myslice/mx, 'r-', alpha=0.1) # plt.plot(ixslice/mx, 'b-', alpha=0.1) # plt.plot(mxslice/mx, 'r-', alpha=0.1) siyslice = shimg[sz, :] sixslice = shimg[:, sz] smyslice = shmod[sz, :] smxslice = shmod[:, sz] shimgs.append(siyslice/mx) shimgs.append(sixslice/mx) shmods.append(smyslice/mx) shmods.append(smxslice/mx) imgs.append(iyslice/mx) imgs.append(ixslice/mx) mods.append(myslice/mx) mods.append(mxslice/mx) imgsum = imgsum + ixslice + iyslice modsum = modsum + mxslice + myslice # plt.ylim(-0.1, 1.1) # plt.title(tt) # ps.savefig() mimg = np.median(np.array(imgs), axis=0) mmod = np.median(np.array(mods), axis=0) mshim = np.median(np.array(shimgs), axis=0) mshmo = np.median(np.array(shmods), axis=0) allmods.append(mshmo) allimgs.append(mshim) plt.clf() # plt.plot(mimg, 'b-') # plt.plot(mmod, 'r-') plt.plot(mshim, 'g-') plt.plot(mshmo, 'm-') plt.ylim(-0.1, 1.1) plt.title(tt + ': median; sums %.3f/%.3f' % (np.sum(mimg), np.sum(mmod))) ps.savefig() # plt.clf() # mx = imgsum.max() # plt.plot(imgsum/mx, 'b-') # plt.plot(modsum/mx, 'r-') # plt.ylim(-0.1, 1.1) # plt.title(tt + ': sum') # ps.savefig() plt.clf() plt.plot((mimg + 0.01) / (mmod + 0.01), 'k-') plt.plot((imgsum/mx + 0.01) / (modsum/mx + 0.01), 'g-') plt.plot((mshim + 0.01) / (mshmo + 0.01), 'm-') plt.ylabel('(img + 0.01) / (mod + 0.01)') plt.title(tt) ps.savefig() iq2,iq3 = allimgs mq2,mq3 = allmods plt.clf() plt.plot(iq2, 'r-') plt.plot(mq2, 'm-') plt.plot(iq3, 'b-') plt.plot(mq3, 'g-') plt.title('Q2 vs Q3') ps.savefig()
def decam(): ''' cp ~/cosmo/staging/decam/DECam_CP/CP20170731/c4d_170801_080516_oki_g_v1.fits.fz /tmp funpack /tmp/c4d_170801_080516_oki_g_v1.fits.fz fitsgetext -i /tmp/c4d_170801_080516_oki_g_v1.fits -o decam-%02i.wcs -a -H cat decam-??.wcs > decam.wcs for ((i=1; i<=61; i++)); do modhead decam.wcs+$i NAXIS2 0; done ''' plt.figure(figsize=(4,3)) plt.subplots_adjust(left=0.15, right=0.99, top=0.99, bottom=0.15) T = fits_table('obstatus/decam-tiles_obstatus.fits') T.rename('pass', 'passnum') print('Passes:', np.unique(T.passnum)) for p in np.unique(T.passnum): I = np.flatnonzero(T.passnum == p) plt.clf() plt.plot(T.ra[I], T.dec[I], 'b.') plt.title('Pass %s' % p) plt.savefig('pass%i.png' % p) ra,dec = 0.933, 0. I,J,d = match_radec(T.ra, T.dec, ra, dec, 5.) #2.8) print(len(I), 'tiles near 0,0') T.cut(I) T.dist = d print('dists:', d) print('Passes:', T.passnum) F = fitsio.FITS(os.path.join(os.path.dirname(__file__), 'decam.wcs')) wcs = [] for i in range(1, len(F)): hdr = F[i].read_header() wcs.append(wcs_pv2sip_hdr(hdr, W=2046, H=4094)) W,H = 5000, 5000 pixsc = 4./3600. targetwcs = Tan(ra, dec, W/2.+0.5, H/2.+0.5, -pixsc, 0., 0., pixsc, float(W), float(H)) II = np.lexsort((T.dist, T.passnum)) # This is for making the (vector) PDF format tiling images. for maxit in [0, 6, 30, 31, 37, 61, 62, 68, 90]: #mx = { 1: 2, 2: 4, 3: 6 }[t.passnum] plot = Plotstuff(outformat='pdf', ra=ra, dec=dec, width=W*pixsc, size=(W,H), outfn='tile-%02i.pdf' % maxit) plot.color = 'white' plot.alpha = 1. plot.plot('fill') out = plot.outline out.fill = True out.stepsize = 1024. plot.color = 'black' plot.alpha = 0.4 plot.apply_settings() for it,t in enumerate(T[II]): print('Tile', it, 'pass', t.passnum) for w in wcs: w.set_crval((t.ra, t.dec)) out.wcs = anwcs_new_sip(w) plot.plot('outline') if it == maxit: print('Writing', it) plot.write() break # And this is for PNG-format tiling images and histograms. cov = np.zeros((H,W), np.uint8) for it,t in enumerate(T[II]): print('Tile', it, 'pass', t.passnum) for w in wcs: w.set_crval((t.ra, t.dec)) #print('WCS:', w) try: Yo,Xo,Yi,Xi,nil = resample_with_wcs(targetwcs, w) except: #import traceback #traceback.print_exc() continue cov[Yo,Xo] += 1 if it in [0, 6, 30, 31, 37, 61, 62, 68, 90]: mx = { 1: 2, 2: 4, 3: 6 }[t.passnum] # plt.clf() # plt.imshow(cov, interpolation='nearest', origin='lower', vmin=0, vmax=mx) # plt.colorbar() # plt.savefig('tile-%02i.png' % it) plt.imsave('tile-%02i.png' % it, cov, origin='lower', vmin=0, vmax=mx, cmap=antigray) #plt.imsave('tile-%02i.pdf' % it, cov, origin='lower', vmin=0, vmax=mx, cmap=antigray, format='pdf') if it in [30, 61, 90]: from collections import Counter print('Coverage counts:', Counter(cov.ravel()).most_common()) bins = -0.5 + np.arange(8) plt.clf() ncov = cov.ravel() ncov = ncov / np.sum(ncov) n,b,p = plt.hist(ncov, bins=bins) # Cumulative histogram from the right... xx,yy = [],[] for blo,bhi,ni in reversed(list(zip(bins, bins[1:], n))): nc = float(np.sum(cov.ravel() > blo)) / len(cov.ravel()) yy.extend([nc,nc]) xx.extend([bhi,blo]) if ni > 0: if nc != ni: if nc > ni+0.03: # If there's room, label the histogram bin above, else below plt.text((blo+bhi)/2., ni, '%.1f \%%' % (100.*ni), ha='center', va='bottom', color='k') else: plt.text((blo+bhi)/2., ni-0.01, '%.1f \%%' % (100.*ni), ha='center', va='top', color='k') plt.text((blo+bhi)/2., nc, '%.1f \%%' % (100.*nc), ha='center', va='bottom', color='k') plt.plot(xx, yy, 'k-') plt.xlim(bins.min(), bins.max()) plt.ylim(0., 1.1) plt.xlabel('Number of exposures') plt.ylabel('Fraction of sky') #plt.title('DECaLS tiling, %i pass%s' % (t.passnum, t.passnum > 1 and 'es' or '')) #plt.savefig('hist-%02i.png' % it) plt.savefig('hist-%02i.pdf' % it)
def main(): """Main program. """ import argparse parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).") parser.add_argument('--calibs', action='store_true', help='Output CCDs that need to be calibrated.') parser.add_argument('--nper', type=int, default=None, help='Batch N calibs per line') parser.add_argument('--forced', action='store_true', help='Output forced-photometry commands') parser.add_argument('--lsb', action='store_true', help='Output Low-Surface-Brightness commands') parser.add_argument('--touching', action='store_true', help='Cut to only CCDs touching selected bricks') parser.add_argument('--check', action='store_true', help='Check which calibrations actually need to run.') parser.add_argument('--check-coadd', action='store_true', help='Check which caoadds actually need to run.') parser.add_argument('--out', help='Output filename for calibs, default %(default)s', default='jobs') parser.add_argument('--command', action='store_true', help='Write out full command-line to run calib') parser.add_argument('--maxdec', type=float, help='Maximum Dec to run') parser.add_argument('--mindec', type=float, help='Minimum Dec to run') parser.add_argument('--region', help='Region to select') parser.add_argument('--bricks', help='Set bricks.fits file to load') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--delete-sky', action='store_true', help='Delete any existing sky calibration files') parser.add_argument('--delete-pvastrom', action='store_true', help='Delete any existing PV WCS calibration files') parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?') opt = parser.parse_args() decals = Decals() if opt.bricks is not None: B = fits_table(opt.bricks) log('Read', len(B), 'from', opt.bricks) else: B = decals.get_bricks() if opt.ccds is not None: T = fits_table(opt.ccds) log('Read', len(T), 'from', opt.ccds) else: T = decals.get_ccds() log(len(T), 'CCDs') T.index = np.arange(len(T)) # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True) # plt.clf() # plt.hist(counts, counts.max()+1) # plt.savefig('bricks.png') # B.cut(I[counts >= 9]) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # #plt.scatter(B.ra[I], B.dec[I], c=counts) # plt.savefig('bricks2.png') # DES Stripe82 #rlo,rhi = 350.,360. # rlo,rhi = 300., 10. # dlo,dhi = -6., 4. # TINY bit #rlo,rhi = 350.,351.1 #dlo,dhi = 0., 1.1 # EDR+ # 860 bricks # ~10,000 CCDs #rlo,rhi = 239,246 #dlo,dhi = 5, 13 # DR1 #rlo,rhi = 0, 360 # part 1 #dlo,dhi = 25, 40 # part 2 #dlo,dhi = 20,25 # part 3 #dlo,dhi = 15,20 # part 4 #dlo,dhi = 10,15 # part 5 #dlo,dhi = 5,10 # the rest #dlo,dhi = -11, 5 #dlo,dhi = 15,25.5 dlo,dhi = -15, 40 rlo,rhi = 0, 360 # Arjun says 3x3 coverage area is roughly # RA=240-252 DEC=6-12 (but not completely rectangular) # COSMOS #rlo,rhi = 148.9, 151.2 #dlo,dhi = 0.9, 3.5 # A nice well-behaved region (EDR2/3) # rlo,rhi = 243.6, 244.6 # dlo,dhi = 8.1, 8.6 # 56 bricks, ~725 CCDs #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7)) # 240 bricks, ~3000 CCDs #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9)) # 535 bricks, ~7000 CCDs #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12)) if opt.region in ['test1', 'test2', 'test3', 'test4']: nm = dict(test1='2446p115', # weird stuff around bright star test2='1183p292', # faint sources around bright galaxy test3='3503p005', # DES test4='1163p277', # Pollux )[opt.region] B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname]))) log('Cut to', len(B), 'bricks') log(B.ra, B.dec) dlo,dhi = -90,90 rlo,rhi = 0, 360 elif opt.region == 'badsky': # A handful of exposures in DR2 with inconsistent sky estimates. #C = decals.get_ccds() #log(len(C), 'CCDs') #C.cut((C.expnum >= 257400) * (C.expnum < 257500)) T.cut(np.array([e in [257460, 257461, 257462, 257463, 257464, 257465, 257466, 257467, 257469, 257472, 257483, 257496] for e in T.expnum])) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25)/2. I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'badsky2': # UGH, missed this one in original 'badsky' definition. T.cut(T.expnum == 257466) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25)/2. I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'edr': # EDR: # 535 bricks, ~7000 CCDs rlo,rhi = 240,245 dlo,dhi = 5, 12 elif opt.region == 'edr-south': rlo,rhi = 240,245 dlo,dhi = 5, 10 elif opt.region == 'cosmos1': # 16 bricks in the core of the COSMOS field. rlo,rhi = 149.75, 150.75 dlo,dhi = 1.6, 2.6 elif opt.region == 'pristine': # Stream? rlo,rhi = 240,250 dlo,dhi = 10,15 elif opt.region == 'des': dlo, dhi = -6., 4. rlo, rhi = 317., 7. T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage]))) log('Cut to', len(T), 'CCDs with "CPDES82" in filename') elif opt.region == 'subdes': rlo,rhi = 320., 360. dlo,dhi = -1.25, 1.25 elif opt.region == 'northwest': rlo,rhi = 240,360 dlo,dhi = 20,40 elif opt.region == 'north': rlo,rhi = 120,240 dlo,dhi = 20,40 elif opt.region == 'northeast': rlo,rhi = 0,120 dlo,dhi = 20,40 elif opt.region == 'southwest': rlo,rhi = 240,360 dlo,dhi = -20,0 elif opt.region == 'south': rlo,rhi = 120,240 dlo,dhi = -20,0 elif opt.region == 'southeast': rlo,rhi = 0,120 dlo,dhi = -20,0 elif opt.region == 'midwest': rlo,rhi = 240,360 dlo,dhi = 0,20 elif opt.region == 'middle': rlo,rhi = 120,240 dlo,dhi = 0,20 elif opt.region == 'mideast': rlo,rhi = 0,120 dlo,dhi = 0,20 elif opt.region == 'grz': # Bricks with grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)) log('Cut to', len(B), 'bricks with grz coverage') elif opt.region == 'nogrz': # Bricks without grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))) log('Cut to', len(B), 'bricks withOUT grz coverage') elif opt.region == 'deep2': rlo,rhi = 250,260 dlo,dhi = 30,35 elif opt.region == 'virgo': rlo,rhi = 185,190 dlo,dhi = 10, 15 elif opt.region == 'virgo2': rlo,rhi = 182,192 dlo,dhi = 8, 18 elif opt.region == 'lsb': rlo,rhi = 147.2, 147.8 dlo,dhi = -0.4, 0.4 if opt.mindec is not None: dlo = opt.mindec if opt.maxdec is not None: dhi = opt.maxdec if rlo < rhi: B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) else: # RA wrap B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) log(len(B), 'bricks in range') I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs') if opt.touching: keep = np.zeros(len(T), bool) for j in J: keep[j] = True T.cut(keep) log('Cut to', len(T), 'CCDs near bricks') # Aside -- how many near DR1=1 CCDs? if False: T2 = D.get_ccds() log(len(T2), 'CCDs') T2.cut(T2.dr1 == 1) log(len(T2), 'CCDs marked DR1=1') log(len(B), 'bricks in range') I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B2 = B[keep] log('Total of', len(B2), 'bricks near CCDs with DR1=1') for band in 'grz': Tb = T2[T2.filter == band] log(len(Tb), 'in filter', band) I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, 0.25) good = np.zeros(len(B2), np.uint8) for i in I: good[i] = 1 B2.set('has_' + band, good) B2.writeto('decals-bricks-in-dr1.fits') sys.exit(0) # sort by dec decreasing B.cut(np.argsort(-B.dec)) for b in B: if opt.check: fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue if opt.check_coadd: fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue print(b.brickname) if not (opt.calibs or opt.forced or opt.lsb): sys.exit(0) bands = 'grz' log('Filters:', np.unique(T.filter)) T.cut(np.flatnonzero(np.array([f in bands for f in T.filter]))) log('Cut to', len(T), 'CCDs in filters', bands) if opt.touching: allI = set() for b in B: wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec)) if len(I) == 0: continue allI.update(I) allI = list(allI) allI.sort() else: allI = np.arange(len(T)) if opt.write_ccds: T[allI].writeto(opt.write_ccds) log('Wrote', opt.write_ccds) ## Be careful here -- T has been cut; we want to write out T.index. ## 'allI' contains indices into T. if opt.forced: log('Writing forced-photometry commands to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') for j,i in enumerate(allI): expstr = '%08i' % T.expnum[i] #outdir = os.path.join('forced', expstr[:5], expstr) #trymakedirs(outdir) outfn = os.path.join('forced', expstr[:5], expstr, 'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i])) imgfn = os.path.join(decals.decals_dir, 'images', T.image_filename[i].strip()) if (not os.path.exists(imgfn) and imgfn.endswith('.fz') and os.path.exists(imgfn[:-3])): imgfn = imgfn[:-3] f.write('python legacypipe/forced-photom-decam.py %s %i DR1 %s\n' % (imgfn, T.cpimage_hdu[i], outfn)) f.close() log('Wrote', opt.out) sys.exit(0) if opt.lsb: log('Writing LSB commands to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') for j,i in enumerate(allI): exp = T.expnum[i] ext = T.ccdname[i].strip() outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext) f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext)) f.close() log('Wrote', opt.out) sys.exit(0) log('Writing calibs to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') batch = [] def write_batch(f, batch, cmd): if opt.command: s = '; '.join(batch) else: s = ' '.join(batch) f.write(s + '\n') for j,i in enumerate(allI): if opt.delete_sky or opt.delete_pvastrom: log(j+1, 'of', len(allI)) im = decals.get_image_object(T[i]) if opt.delete_sky and os.path.exists(im.skyfn): log(' deleting:', im.skyfn) os.unlink(im.skyfn) if opt.delete_pvastrom and os.path.exists(im.pvwcsfn): log(' deleting:', im.pvwcsfn) os.unlink(im.pvwcsfn) if opt.check: log(j+1, 'of', len(allI)) im = decals.get_image_object(T[i]) if not im.run_calibs(im, just_check=True): log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done') continue if opt.command: s = ('python legacypipe/run-calib.py --expnum %i --ccdname %s' % (T.expnum[i], T.ccdname[i])) else: s = '%i' % T.index[i] if not opt.nper: f.write(s + '\n') else: batch.append(s) if len(batch) >= opt.nper: write_batch(f, batch, opt.command) batch = [] if opt.check: f.flush() if len(batch): write_batch(f, batch, opt.command) f.close() log('Wrote', opt.out) return 0
def map_decals_wl(req, ver, zoom, x, y): tag = 'decals-wl' ignoreCached = False filename = None forcecache = False from decals import settings savecache = settings.SAVE_CACHE zoom = int(zoom) zoomscale = 2.**zoom x = int(x) y = int(y) if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale: raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom, x, y)) ver = int(ver) if not ver in tileversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) basedir = settings.DATA_DIR tilefn = os.path.join(basedir, 'tiles', tag, '%i/%i/%i/%i.jpg' % (ver, zoom, x, y)) if os.path.exists(tilefn) and not ignoreCached: print('Cached:', tilefn) return send_file(tilefn, 'image/jpeg', expires=oneyear, modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'), filename=filename) else: print('Tile image does not exist:', tilefn) from astrometry.util.resample import resample_with_wcs, OverlapError from astrometry.util.util import Tan from astrometry.libkd.spherematch import match_radec from astrometry.util.fits import fits_table from astrometry.util.starutil_numpy import degrees_between import numpy as np import fitsio try: wcs, W, H, zoomscale, zoom, x, y = get_tile_wcs(zoom, x, y) except RuntimeError as e: return HttpResponse(e.strerror) mydir = os.path.join(basedir, 'coadd', 'weak-lensing') rlo, d = wcs.pixelxy2radec(W, H / 2)[-2:] rhi, d = wcs.pixelxy2radec(1, H / 2)[-2:] r, d1 = wcs.pixelxy2radec(W / 2, 1)[-2:] r, d2 = wcs.pixelxy2radec(W / 2, H)[-2:] #dlo = min(d1, d2) #dhi = max(d1, d2) r, d = wcs.pixelxy2radec(W / 2, H / 2)[-2:] rad = degrees_between(r, d, rlo, d1) fn = os.path.join(mydir, 'index.fits') if not os.path.exists(fn): # ii, rr, dd = [], [], [] for i in range(1, 52852 + 1): imgfn = os.path.join(mydir, 'map%i.fits' % i) hdr = fitsio.read_header(imgfn) r = hdr['CRVAL1'] d = hdr['CRVAL2'] ii.append(i) rr.append(r) dd.append(d) T = fits_table() T.ra = np.array(rr) T.dec = np.array(dd) T.i = np.array(ii) T.writeto(fn) T = fits_table(fn) I, J, d = match_radec(T.ra, T.dec, r, d, rad + 0.2) T.cut(I) print(len(T), 'weak-lensing maps in range') if len(I) == 0: from django.http import HttpResponseRedirect if forcecache: # create symlink to blank.jpg! trymakedirs(tilefn) src = os.path.join(settings.STATIC_ROOT, 'blank.jpg') if os.path.exists(tilefn): os.unlink(tilefn) os.symlink(src, tilefn) print('Symlinked', tilefn, '->', src) return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg') r, d = wcs.pixelxy2radec([1, 1, 1, W / 2, W, W, W, W / 2], [1, H / 2, H, H, H, H / 2, 1, 1])[-2:] foundany = False rimg = np.zeros((H, W), np.float32) rn = np.zeros((H, W), np.uint8) for tilei in T.i: fn = os.path.join(mydir, 'map%i.fits' % tilei) try: bwcs = read_tan_wcs(fn, 0) except: print('Failed to read WCS:', fn) savecache = False import traceback import sys traceback.print_exc(None, sys.stdout) continue foundany = True print('Reading', fn) ok, xx, yy = bwcs.radec2pixelxy(r, d) xx = xx.astype(np.int) yy = yy.astype(np.int) imW, imH = int(bwcs.get_width()), int(bwcs.get_height()) M = 10 xlo = np.clip(xx.min() - M, 0, imW) xhi = np.clip(xx.max() + M, 0, imW) ylo = np.clip(yy.min() - M, 0, imH) yhi = np.clip(yy.max() + M, 0, imH) if xlo >= xhi or ylo >= yhi: continue subwcs = bwcs.get_subimage(xlo, ylo, xhi - xlo, yhi - ylo) slc = slice(ylo, yhi), slice(xlo, xhi) try: f = fitsio.FITS(fn)[0] img = f[slc] del f except: print('Failed to read image and WCS:', fn) savecache = False import traceback import sys traceback.print_exc(None, sys.stdout) continue try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(wcs, subwcs, [], 3) except OverlapError: print('Resampling exception') continue rimg[Yo, Xo] += img[Yi, Xi] rn[Yo, Xo] += 1 rimg /= np.maximum(rn, 1) if forcecache: savecache = True if savecache: trymakedirs(tilefn) else: import tempfile f, tilefn = tempfile.mkstemp(suffix='.jpg') os.close(f) import pylab as plt # S/N #lo,hi = 1.5, 5.0 lo, hi = 0, 5.0 rgb = plt.cm.hot((rimg - lo) / (hi - lo)) plt.imsave(tilefn, rgb) print('Wrote', tilefn) return send_file(tilefn, 'image/jpeg', unlink=(not savecache), filename=filename)
def plots(opt): from astrometry.util.plotutils import antigray import tractor.sfd T = fits_table(opt.files[0]) print('Read', len(T), 'bricks summarized in', opt.files[0]) import pylab as plt import matplotlib B = fits_table('survey-bricks.fits.gz') print('Looking up brick bounds') ibrick = dict([(n, i) for i, n in enumerate(B.brickname)]) bi = np.array([ibrick[n] for n in T.brickname]) T.ra1 = B.ra1[bi] T.ra2 = B.ra2[bi] T.dec1 = B.dec1[bi] T.dec2 = B.dec2[bi] assert (np.all(T.ra2 > T.ra1)) T.area = ((T.ra2 - T.ra1) * (T.dec2 - T.dec1) * np.cos(np.deg2rad((T.dec1 + T.dec2) / 2.))) del B del bi del ibrick print('Total sources:', sum(T.nobjs)) print('Approx area:', len(T) / 16., 'sq deg') print('Area:', np.sum(T.area)) print('g,r,z coverage:', sum((T.nexp_g > 0) * (T.nexp_r > 0) * (T.nexp_z > 0)) / 16.) decam = True if decam: release = 'DECaLS DR8' else: release = 'BASS+MzLS DR8' if decam: # DECam #ax = [360, 0, -21, 36] ax = [300, -60, -75, 36] def map_ra(r): return r + (-360 * (r > 300)) else: # MzLS+BASS ax = [310, 90, 30, 80] def map_ra(r): return r udec = np.unique(T.dec) print('Number of unique Dec values:', len(udec)) print('Number of unique Dec values in range', ax[2], ax[3], ':', np.sum((udec >= ax[2]) * (udec <= ax[3]))) def radec_plot(): plt.axis(ax) plt.xlabel('RA (deg)') if decam: # plt.xticks(np.arange(0, 361, 45)) #tt = np.arange(0, 361, 60) #plt.xticks(tt, map_ra(tt)) plt.xticks([-60, 0, 60, 120, 180, 240, 300], [300, 0, 60, 120, 180, 240, 300]) else: plt.xticks(np.arange(90, 311, 30)) plt.ylabel('Dec (deg)') def plot_broken(rr, dd, *args, **kwargs): dr = np.abs(np.diff(rr)) I = np.flatnonzero(dr > 90) #print('breaks:', rr[I]) #print('breaks:', rr[I+1]) if len(I) == 0: plt.plot(rr, dd, *args, **kwargs) return for lo, hi in zip(np.append([0], I + 1), np.append(I + 1, -1)): #print('Cut:', lo, ':', hi, '->', rr[lo], rr[hi-1]) plt.plot(rr[lo:hi], dd[lo:hi], *args, **kwargs) # Galactic plane lines gl = np.arange(361) gb = np.zeros_like(gl) from astrometry.util.starutil_numpy import lbtoradec rr, dd = lbtoradec(gl, gb) plot_broken(map_ra(rr), dd, 'k-', alpha=0.5, lw=1) rr, dd = lbtoradec(gl, gb + 10) plot_broken(map_ra(rr), dd, 'k-', alpha=0.25, lw=1) rr, dd = lbtoradec(gl, gb - 10) plot_broken(map_ra(rr), dd, 'k-', alpha=0.25, lw=1) plt.figure(1, figsize=(8, 5)) plt.subplots_adjust(left=0.1, right=0.98, top=0.93) plt.figure(2, figsize=(8, 4)) #plt.subplots_adjust(left=0.06, right=0.98, top=0.98) plt.subplots_adjust(left=0.08, right=0.98, top=0.98) plt.figure(1) # Map of the tile centers we want to observe... """ if decam: O = fits_table('obstatus/decam-tiles_obstatus.fits') else: O = fits_table('mosaic-tiles_obstatus.fits') """ # File from the "observing" svn repo: from pkg_resources import resource_filename if decam: tilefile = resource_filename('legacyzpts', 'data/decam-tiles_obstatus.fits') else: tilefile = resource_filename('legacyzpts', 'data/mosaic-tiles_obstatus.fits') O = fits_table(tilefile) O.cut(O.in_desi == 1) rr, dd = np.meshgrid(np.linspace(ax[1], ax[0], 700), np.linspace(ax[2], ax[3], 200)) from astrometry.libkd.spherematch import match_radec I, J, d = match_radec(O.ra, O.dec, rr.ravel(), dd.ravel(), 1.) desimap = np.zeros(rr.shape, bool) desimap.flat[J] = True # Smoothed DESI boundary contours from scipy.ndimage.filters import gaussian_filter from scipy.ndimage.morphology import binary_dilation C = plt.contour(gaussian_filter( binary_dilation(desimap).astype(np.float32), 2), [0.5], extent=[ax[1], ax[0], ax[2], ax[3]]) plt.clf() desi_map_boundaries = C.collections[0] def desi_map_outline(): segs = desi_map_boundaries.get_segments() for seg in segs: plt.plot(seg[:, 0], seg[:, 1], 'b-') def desi_map(): # Show the DESI tile map in the background. plt.imshow(desimap, origin='lower', interpolation='nearest', extent=[ax[1], ax[0], ax[2], ax[3]], aspect='auto', cmap=antigray, vmax=8) base_cmap = 'viridis' # Dust map -- B&W version nr, nd = 610, 350 plt.figure(2) plt.clf() dmap = np.zeros((nd, nr)) rr = np.linspace(ax[0], ax[1], nr) dd = np.linspace(ax[2], ax[3], nd) rr = rr[:-1] + 0.5 * (rr[1] - rr[0]) dd = dd[:-1] + 0.5 * (dd[1] - dd[0]) rr, dd = np.meshgrid(rr, dd) I, J, d = match_radec(rr.ravel(), dd.ravel(), O.ra, O.dec, 1.0, nearest=True) iy, ix = np.unravel_index(I, rr.shape) #dmap[iy,ix] = O.ebv_med[J] sfd = tractor.sfd.SFDMap() ebv = sfd.ebv(rr[iy, ix], dd[iy, ix]) dmap[iy, ix] = ebv mx = np.percentile(dmap[dmap > 0], 98) plt.imshow(dmap, extent=[ax[0], ax[1], ax[2], ax[3]], interpolation='nearest', origin='lower', aspect='auto', cmap='Greys', vmin=0, vmax=mx) #desi_map_outline() radec_plot() cax = colorbar_axes(plt.gca(), frac=0.12) cbar = plt.colorbar(cax=cax) cbar.set_label('Extinction E(B-V)') plt.savefig('ext-bw.pdf') plt.clf() dmap = sfd.ebv(rr.ravel(), dd.ravel()).reshape(rr.shape) plt.imshow(dmap, extent=[ax[0], ax[1], ax[2], ax[3]], interpolation='nearest', origin='lower', aspect='auto', cmap='Greys', vmin=0, vmax=0.25) desi_map_outline() radec_plot() cax = colorbar_axes(plt.gca(), frac=0.12) cbar = plt.colorbar(cax=cax) cbar.set_label('Extinction E(B-V)') plt.savefig('ext-bw-2.pdf') plt.figure(1) #sys.exit(0) plt.clf() depthlo, depthhi = 21.5, 25.5 for band in 'grz': depth = T.get('galdepth_%s' % band) ha = dict(histtype='step', bins=50, range=(depthlo, depthhi)) ccmap = dict(g='g', r='r', z='m') plt.hist(depth[depth > 0], label='%s band' % band, color=ccmap[band], **ha) plt.xlim(depthlo, depthhi) plt.xlabel('Galaxy depth (median per brick) (mag)') plt.ylabel('Number of Bricks') plt.title(release) plt.savefig('galdepths.png') for band in 'grz': depth = T.get('galdepth_%s' % band) nexp = T.get('nexp_%s' % band) #lo,hi = 22.0-0.05, 24.2+0.05 lo, hi = depthlo - 0.05, depthhi + 0.05 nbins = 1 + int((depthhi - depthlo) / 0.1) ha = dict(histtype='step', bins=nbins, range=(lo, hi)) ccmap = dict(g='g', r='r', z='m') area = 0.25**2 plt.clf() I = np.flatnonzero((depth > 0) * (nexp == 1)) plt.hist(depth[I], label='%s band, 1 exposure' % band, color=ccmap[band], lw=1, weights=area * np.ones_like(depth[I]), **ha) I = np.flatnonzero((depth > 0) * (nexp == 2)) plt.hist(depth[I], label='%s band, 2 exposures' % band, color=ccmap[band], lw=2, alpha=0.5, weights=area * np.ones_like(depth[I]), **ha) I = np.flatnonzero((depth > 0) * (nexp >= 3)) plt.hist(depth[I], label='%s band, 3+ exposures' % band, color=ccmap[band], lw=3, alpha=0.3, weights=area * np.ones_like(depth[I]), **ha) plt.title('%s: galaxy depths, %s band' % (release, band)) plt.xlabel('5-sigma galaxy depth (mag)') plt.ylabel('Square degrees') plt.xlim(lo, hi) plt.xticks(np.arange(depthlo, depthhi + 0.01, 0.2)) plt.legend(loc='upper right') plt.savefig('depth-hist-%s.png' % band) for band in 'grz': plt.clf() desi_map() N = T.get('nexp_%s' % band) I = np.flatnonzero(N > 0) #cm = matplotlib.cm.get_cmap('jet', 6) #cm = matplotlib.cm.get_cmap('winter', 5) mx = 10 cm = cmap_discretize(base_cmap, mx) plt.scatter(map_ra(T.ra[I]), T.dec[I], c=N[I], s=3, edgecolors='none', vmin=0.5, vmax=mx + 0.5, cmap=cm) radec_plot() cax = colorbar_axes(plt.gca(), frac=0.08) plt.colorbar(cax=cax, ticks=range(mx + 1)) plt.title('%s: Number of exposures in %s' % (release, band)) plt.savefig('nexp-%s.png' % band) #cmap = cmap_discretize(base_cmap, 15) cmap = cmap_discretize(base_cmap, 10) plt.clf() desi_map() psf = T.get('psfsize_%s' % band) I = np.flatnonzero(psf > 0) plt.scatter(map_ra(T.ra[I]), T.dec[I], c=psf[I], s=3, edgecolors='none', cmap=cmap, vmin=0.5, vmax=2.5) #vmin=0, vmax=3.) radec_plot() plt.colorbar() plt.title('%s: PSF size, band %s' % (release, band)) plt.savefig('psfsize-%s.png' % band) plt.clf() desi_map() depth = T.get('galdepth_%s' % band) - T.get('ext_%s' % band) mn, mx = np.percentile(depth[depth > 0], [10, 98]) mn = np.floor(mn * 10) / 10. mx = np.ceil(mx * 10) / 10. cmap = cmap_discretize(base_cmap, 1 + int((mx - mn + 0.001) / 0.1)) I = (depth > 0) plt.scatter(map_ra(T.ra[I]), T.dec[I], c=depth[I], s=3, edgecolors='none', vmin=mn - 0.05, vmax=mx + 0.05, cmap=cmap) radec_plot() plt.colorbar() plt.title( '%s: galaxy depth, band %s, median per brick, extinction-corrected' % (release, band)) plt.savefig('galdepth-%s.png' % band) # B&W version plt.figure(2) plt.clf() mn, mx = np.percentile(depth[depth > 0], [2, 98]) print('Raw mn,mx', mn, mx) mn = np.floor((mn + 0.05) * 10) / 10. - 0.05 mx = np.ceil((mx - 0.05) * 10) / 10. + 0.05 print('rounded mn,mx', mn, mx) nsteps = int((mx - mn + 0.001) / 0.1) print('discretizing into', nsteps, 'colormap bins') #nsteps = 1+int((mx-mn+0.001)/0.1) cmap = cmap_discretize(antigray, nsteps) nr, nd = 610, 228 dmap = np.zeros((nd, nr)) rr = np.linspace(ax[0], ax[1], nr) dd = np.linspace(ax[2], ax[3], nd) rr = rr[:-1] + 0.5 * (rr[1] - rr[0]) dd = dd[:-1] + 0.5 * (dd[1] - dd[0]) rr, dd = np.meshgrid(rr, dd) I, J, d = match_radec(rr.ravel(), dd.ravel(), T.ra, T.dec, 0.2, nearest=True) iy, ix = np.unravel_index(I, rr.shape) dmap[iy, ix] = depth[J] plt.imshow(dmap, extent=[ax[0], ax[1], ax[2], ax[3]], interpolation='nearest', origin='lower', aspect='auto', cmap=cmap, vmin=mn, vmax=mx) desi_map_outline() radec_plot() cax = colorbar_axes(plt.gca(), frac=0.12) cbar = plt.colorbar( cax=cax, ticks=np.arange(20, 26, 0.5) ) #ticks=np.arange(np.floor(mn/5.)*5., 0.1+np.ceil(mx/5.)*5, 0.2)) cbar.set_label('Depth (5-sigma, galaxy profile, AB mag)') plt.savefig('galdepth-bw-%s.pdf' % band) plt.figure(1) plt.clf() desi_map() ext = T.get('ext_%s' % band) mn = 0. mx = 0.5 cmap = 'hot' cmap = cmap_discretize(cmap, 10) #cmap = cmap_discretize(base_cmap, 1+int((mx-mn+0.001)/0.1)) plt.scatter(map_ra(T.ra), T.dec, c=ext, s=3, edgecolors='none', vmin=mn, vmax=mx, cmap=cmap) radec_plot() plt.colorbar() plt.title('%s: extinction, band %s' % (release, band)) plt.savefig('ext-%s.png' % band) T.ngal = T.nsimp + T.nrex + T.nexp + T.ndev + T.ncomp for col in [ 'nobjs', 'npsf', 'nsimp', 'nrex', 'nexp', 'ndev', 'ncomp', 'ngal' ]: if not col in T.get_columns(): continue plt.clf() desi_map() N = T.get(col) / T.area mx = np.percentile(N, 99.5) plt.scatter(map_ra(T.ra), T.dec, c=N, s=3, edgecolors='none', vmin=0, vmax=mx) radec_plot() cbar = plt.colorbar() cbar.set_label('Objects per square degree') tt = 'of type %s' % col[1:] if col == 'nobjs': tt = 'total' plt.title('%s: Number of objects %s' % (release, tt)) plt.savefig('nobjs-%s.png' % col[1:]) # B&W version plt.figure(2) plt.clf() # plt.scatter(map_ra(T.ra), T.dec, c=N, s=3, # edgecolors='none', vmin=0, vmax=mx, cmap=antigray) # Approximate pixel size in PNG plot # This doesn't work correctly -- we've already binned to brick resolution, so get moire patterns # nobjs,xe,ye = np.histogram2d(map_ra(T.ra), T.dec, weights=T.get(col), # bins=(nr,nd), range=((ax[1],ax[0]),(ax[2],ax[3]))) # nobjs = nobjs.T # area = np.diff(xe)[np.newaxis,:] * (np.diff(ye) * np.cos(np.deg2rad(ye[:-1])))[:,np.newaxis] # nobjs /= area # plt.imshow(nobjs, extent=[ax[1],ax[0],ax[2],ax[3]], interpolation='nearest', origin='lower', # aspect='auto') #print('Computing neighbours for nobjs plot...') nr, nd = 610, 228 nobjs = np.zeros((nd, nr)) rr = np.linspace(ax[0], ax[1], nr) dd = np.linspace(ax[2], ax[3], nd) rr = rr[:-1] + 0.5 * (rr[1] - rr[0]) dd = dd[:-1] + 0.5 * (dd[1] - dd[0]) rr, dd = np.meshgrid(rr, dd) I, J, d = match_radec(rr.ravel(), dd.ravel(), T.ra, T.dec, 0.2, nearest=True) iy, ix = np.unravel_index(I, rr.shape) nobjs[iy, ix] = T.get(col)[J] / T.area[J] #print('done') #mx = 2. * np.median(nobjs[nobjs > 0]) mx = np.percentile(N, 99) plt.imshow(nobjs, extent=[ax[0], ax[1], ax[2], ax[3]], interpolation='nearest', origin='lower', aspect='auto', cmap='Greys', vmin=0, vmax=mx) desi_map_outline() radec_plot() #cax = colorbar_axes(plt.gca(), frac=0.08) cax = colorbar_axes(plt.gca(), frac=0.12) cbar = plt.colorbar(cax=cax, format=matplotlib.ticker.FuncFormatter( lambda x, p: format(int(x), ','))) cbar.set_label('Objects per square degree') plt.savefig('nobjs-bw-%s.pdf' % col[1:]) #plt.savefig('nobjs-bw-%s.png' % col[1:]) plt.figure(1) Ntot = T.nobjs for col in ['npsf', 'nsimp', 'nrex', 'nexp', 'ndev', 'ncomp', 'ngal']: if not col in T.get_columns(): continue plt.clf() desi_map() N = T.get(col) / (Ntot.astype(np.float32)) N[Ntot == 0] = 0. print(col, 'max frac:', N.max()) mx = np.percentile(N, 99.5) print('mx', mx) plt.scatter(map_ra(T.ra), T.dec, c=N, s=3, edgecolors='none', vmin=0, vmax=mx) radec_plot() plt.colorbar() plt.title('%s: Fraction of objects of type %s' % (release, col[1:])) plt.savefig('fobjs-%s.png' % col[1:]) # B&W version plt.figure(2) plt.clf() #plt.scatter(map_ra(T.ra), T.dec, c=N * 100., s=3, # edgecolors='none', vmin=0, vmax=mx*100., cmap=antigray) fobjs = np.zeros((nd, nr)) rr = np.linspace(ax[0], ax[1], nr) dd = np.linspace(ax[2], ax[3], nd) rr = rr[:-1] + 0.5 * (rr[1] - rr[0]) dd = dd[:-1] + 0.5 * (dd[1] - dd[0]) rr, dd = np.meshgrid(rr, dd) I, J, d = match_radec(rr.ravel(), dd.ravel(), T.ra, T.dec, 0.2, nearest=True) iy, ix = np.unravel_index(I, rr.shape) fobjs[iy, ix] = N[J] * 100. #mx = 2. * np.median(fobjs[fobjs > 0]) mx = np.percentile(N * 100., 99) plt.imshow(fobjs, extent=[ax[0], ax[1], ax[2], ax[3]], interpolation='nearest', origin='lower', aspect='auto', cmap='Greys', vmin=0, vmax=mx) desi_map_outline() radec_plot() cax = colorbar_axes(plt.gca(), frac=0.12) cbar = plt.colorbar( cax=cax, format=matplotlib.ticker.FuncFormatter(lambda x, p: '%.2g' % x)) cbar.set_label('Percentage of objects of type %s' % col[1:].upper()) plt.savefig('fobjs-bw-%s.pdf' % col[1:]) #plt.savefig('fobjs-bw-%s.png' % col[1:]) plt.figure(1) return 0
print('Kd data:', kd.get_data(np.array([0, 3, 5]).astype(np.uint32))) print('Permute:', spherematch.tree_permute(kd, np.array([3, 5, 7]).astype(np.int32))) print('Permute:', kd.permute(np.array([0, 99, 199]).astype(np.int32))) ra, dec = np.meshgrid(np.arange(0, 360), np.arange(-90, 91, 1)) ra1 = ra.ravel() dec1 = dec.ravel() rdkd1 = spherematch.tree_build_radec(ra1, dec1) print('RdKd:', rdkd1.n, rdkd1.bbox) ra2 = np.random.uniform(-10, 10, size=1000) dec2 = np.random.uniform(-10, 10, size=1000) rdkd2 = spherematch.tree_build_radec(ra2, dec2) I = spherematch.tree_search_radec(rdkd1, ra2[0], dec2[0], 2.) print('search_radec:', I) I, J, d = spherematch.match_radec(ra1, dec1, ra2, dec2, 1.) print('Matches:', len(I)) I, J, d = spherematch.match_radec(ra1, dec1, ra2, dec2, 1., nearest=True) print('Nearest matches:', len(I)) I = spherematch.match_radec(ra1, dec1, ra2, dec2, 1., indexlist=True) print('Index lists matches:', len(I))
def galex_coadds(onegal, galaxy=None, radius_mosaic=30, radius_mask=None, pixscale=1.5, ref_pixscale=0.262, output_dir=None, galex_dir=None, log=None, centrals=True, verbose=False): '''Generate custom GALEX cutouts. radius_mosaic and radius_mask in arcsec pixscale: GALEX pixel scale in arcsec/pixel. ''' import fitsio import matplotlib.pyplot as plt from astrometry.libkd.spherematch import match_radec from astrometry.util.resample import resample_with_wcs, OverlapError from tractor import (Tractor, NanoMaggies, Image, LinearPhotoCal, NCircularGaussianPSF, ConstantFitsWcs, ConstantSky) from legacypipe.survey import imsave_jpeg from legacypipe.catalog import read_fits_catalog if galaxy is None: galaxy = 'galaxy' if galex_dir is None: galex_dir = os.environ.get('GALEX_DIR') if output_dir is None: output_dir = '.' if radius_mask is None: radius_mask = radius_mosaic radius_search = 5.0 # [arcsec] else: radius_search = radius_mask W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int') # [pixels] targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0, -pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W), float(H)) # Read the custom Tractor catalog tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy)) if not os.path.isfile(tractorfile): print('Missing Tractor catalog {}'.format(tractorfile)) return 0 cat = fits_table(tractorfile) print('Read {} sources from {}'.format(len(cat), tractorfile), flush=True, file=log) keep = np.ones(len(cat)).astype(bool) if centrals: # Find the large central galaxy and mask out (ignore) all the models # which are within its elliptical mask. # This algorithm will have to change for mosaics not centered on large # galaxies, e.g., in galaxy groups. m1, m2, d12 = match_radec(cat.ra, cat.dec, onegal['RA'], onegal['DEC'], radius_search / 3600.0, nearest=False) if len(m1) == 0: print('No central galaxies found at the central coordinates!', flush=True, file=log) else: pixfactor = ref_pixscale / pixscale # shift the optical Tractor positions for mm in m1: morphtype = cat.type[mm].strip() if morphtype == 'EXP' or morphtype == 'COMP': e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[ mm], cat.shapeexp_r[mm] # [arcsec] elif morphtype == 'DEV' or morphtype == 'COMP': e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[ mm], cat.shapedev_r[mm] # [arcsec] else: r50 = None if r50: majoraxis = r50 * 5 / pixscale # [pixels] ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2) these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis, ba * majoraxis, np.radians(phi), cat.bx * pixfactor, cat.by * pixfactor) if np.sum(these) > 0: #keep[these] = False pass print('Hack!') keep[mm] = False #srcs = read_fits_catalog(cat) #_srcs = np.array(srcs)[~keep].tolist() #mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0) #import matplotlib.pyplot as plt ##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png') #plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png') #pdb.set_trace() srcs = read_fits_catalog(cat) for src in srcs: src.freezeAllBut('brightness') #srcs_nocentral = np.array(srcs)[keep].tolist() # Find all overlapping GALEX tiles and then read the tims. galex_tiles = _read_galex_tiles(targetwcs, galex_dir, log=log, verbose=verbose) gbands = ['n', 'f'] nicegbands = ['NUV', 'FUV'] zps = dict(n=20.08, f=18.82) coimgs, comods, coresids, coimgs_central, comods_nocentral = [], [], [], [], [] for niceband, band in zip(nicegbands, gbands): J = np.flatnonzero(galex_tiles.get('has_' + band)) print(len(J), 'GALEX tiles have coverage in band', band) coimg = np.zeros((H, W), np.float32) comod = np.zeros((H, W), np.float32) cowt = np.zeros((H, W), np.float32) comod_nocentral = np.zeros((H, W), np.float32) for src in srcs: src.setBrightness(NanoMaggies(**{band: 1})) for j in J: brick = galex_tiles[j] fn = os.path.join( galex_dir, brick.tilename.strip(), '%s-%sd-intbgsub.fits.gz' % (brick.brickname, band)) #print(fn) gwcs = Tan(*[ float(f) for f in [ brick.crval1, brick.crval2, brick.crpix1, brick.crpix2, brick.cdelt1, 0., 0., brick.cdelt2, 3840., 3840. ] ]) img = fitsio.read(fn) #print('Read', img.shape) try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, gwcs, [], 3) except OverlapError: continue K = np.flatnonzero(img[Yi, Xi] != 0.) if len(K) == 0: continue Yo, Xo, Yi, Xi = Yo[K], Xo[K], Yi[K], Xi[K] wt = brick.get(band + 'exptime') coimg[Yo, Xo] += wt * img[Yi, Xi] cowt[Yo, Xo] += wt x0, x1, y0, y1 = min(Xi), max(Xi), min(Yi), max(Yi) subwcs = gwcs.get_subimage(x0, y0, x1 - x0 + 1, y1 - y0 + 1) twcs = ConstantFitsWcs(subwcs) timg = img[y0:y1 + 1, x0:x1 + 1] tie = np.ones_like(timg) ## HACK! #hdr = fitsio.read_header(fn) #zp = hdr[''] zp = zps[band] photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp), band=band) tsky = ConstantSky(0.0) # HACK -- circular Gaussian PSF of fixed size... # in arcsec #fwhms = dict(NUV=6.0, FUV=6.0) # -> sigma in pixels #sig = fwhms[band] / 2.35 / twcs.pixel_scale() sig = 6.0 / np.sqrt(8 * np.log(2)) / twcs.pixel_scale() tpsf = NCircularGaussianPSF([sig], [1.]) tim = Image(data=timg, inverr=tie, psf=tpsf, wcs=twcs, sky=tsky, photocal=photocal, name='GALEX ' + band + brick.brickname) ## Build the model image with and without the central galaxy model. tractor = Tractor([tim], srcs) mod = tractor.getModelImage(0) tractor.freezeParam('images') tractor.optimize_forced_photometry(priors=False, shared_params=False) mod = tractor.getModelImage(0) srcs_nocentral = np.array(srcs)[keep].tolist() #srcs_nocentral = np.array(srcs)[nocentral].tolist() tractor_nocentral = Tractor([tim], srcs_nocentral) mod_nocentral = tractor_nocentral.getModelImage(0) comod[Yo, Xo] += wt * mod[Yi - y0, Xi - x0] comod_nocentral[Yo, Xo] += wt * mod_nocentral[Yi - y0, Xi - x0] coimg /= np.maximum(cowt, 1e-18) comod /= np.maximum(cowt, 1e-18) comod_nocentral /= np.maximum(cowt, 1e-18) coresid = coimg - comod # Subtract the model image which excludes the central (comod_nocentral) # from the data (coimg) to isolate the light of the central # (coimg_central). coimg_central = coimg - comod_nocentral coimgs.append(coimg) comods.append(comod) coresids.append(coresid) comods_nocentral.append(comod_nocentral) coimgs_central.append(coimg_central) # Write out the final images with and without the central, making sure # to apply the zeropoint to go from counts/s to AB nanomaggies. # https://asd.gsfc.nasa.gov/archive/galex/FAQ/counts_background.html for thisimg, imtype in zip((coimg, comod, comod_nocentral), ('image', 'model', 'model-nocentral')): fitsfile = os.path.join( output_dir, '{}-{}-{}.fits'.format(galaxy, imtype, niceband)) if verbose: print('Writing {}'.format(fitsfile)) fitsio.write(fitsfile, thisimg * 10**(-0.4 * (zp - 22.5)), clobber=True) # Build a color mosaic (but note that the images here are in units of # background-subtracted counts/s). #_galex_rgb = _galex_rgb_moustakas #_galex_rgb = _galex_rgb_dstn _galex_rgb = _galex_rgb_official for imgs, imtype in zip( (coimgs, comods, coresids, comods_nocentral, coimgs_central), ('image', 'model', 'resid', 'model-nocentral', 'image-central')): rgb = _galex_rgb(imgs) jpgfile = os.path.join(output_dir, '{}-{}-FUVNUV.jpg'.format(galaxy, imtype)) if verbose: print('Writing {}'.format(jpgfile)) imsave_jpeg(jpgfile, rgb, origin='lower') return 1
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='DECaLS simulations.') parser.add_argument('-b', '--brick', type=str, default='2428p117', metavar='', help='process this brick (required input)') parser.add_argument('-o', '--objtype', type=str, default='ELG', metavar='', help='object type (STAR, ELG, LRG, BGS)') parser.add_argument('-v', '--verbose', action='store_true', help='toggle on verbose output') args = parser.parse_args() if args.brick is None: parser.print_help() sys.exit(1) # Set the debugging level if args.verbose: lvl = logging.DEBUG else: lvl = logging.INFO logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout) log = logging.getLogger('__name__') brickname = args.brick objtype = args.objtype.upper() lobjtype = objtype.lower() log.info('Analyzing objtype {} on brick {}'.format(objtype, brickname)) if 'DECALS_SIM_DIR' in os.environ: decals_sim_dir = os.getenv('DECALS_SIM_DIR') else: decals_sim_dir = '.' # Plotting preferences sns.set(style='white', font_scale=1.6, palette='dark') #,font='fantasy') col = sns.color_palette('dark') # Read the meta-catalog. metafile = os.path.join(decals_sim_dir, brickname, 'metacat-' + brickname + '-' + lobjtype + '.fits') log.info('Reading {}'.format(metafile)) meta = fits.getdata(metafile, 1) # We need this for our histograms below magbinsz = 0.2 rminmax = np.array(meta['rmag_range'][0], meta['rmag_range'][1]) nmagbin = long((rminmax[1] - rminmax[0]) / magbinsz) # Work in chunks. nchunk = meta['nchunk'] for ichunk in range(nchunk): log.info('Working on chunk {:02d}/{:02d}'.format(ichunk + 1, nchunk)) chunksuffix = '{:02d}'.format(ichunk) # Read the simulated object catalog simcatfile = os.path.join( decals_sim_dir, brickname, 'simcat-' + brickname + '-' + lobjtype + '-' + chunksuffix + '.fits') #log.info('Reading {}'.format(simcatfile)) simcat = fits.getdata(simcatfile, 1) # Read and match to the Tractor catalog tractorfile = os.path.join( decals_sim_dir, brickname, 'tractor-' + brickname + '-' + lobjtype + '-' + chunksuffix + '.fits') #log.info('Reading {}'.format(tractorfile)) tractor = fits.getdata(tractorfile, 1) m1, m2, d12 = match_radec(tractor['ra'], tractor['dec'], simcat['ra'], simcat['dec'], 1.0 / 3600.0) missing = np.delete(np.arange(len(simcat)), m2, axis=0) good = np.where( (np.abs(tractor['decam_flux'][m1, 2] / simcat['rflux'][m2] - 1) < 0.3) * 1) # Flux residuals vs r-band magnitude rmag = simcat['r'][m2] gflux_sim = simcat['gflux'][m2] rflux_sim = simcat['rflux'][m2] zflux_sim = simcat['zflux'][m2] gflux_tra = tractor['decam_flux'][m1, 1] rflux_tra = tractor['decam_flux'][m1, 2] zflux_tra = tractor['decam_flux'][m1, 4] fig, ax = plt.subplots(3, sharex=True, figsize=(6, 8)) ax[0].scatter(rmag, gflux_sim / gflux_tra - 1, color=col[0], s=10) ax[1].scatter(rmag, rflux_sim / rflux_tra - 1, color=col[1], s=10) ax[2].scatter(rmag, zflux_sim / zflux_tra - 1, color=col[2], s=10) [thisax.set_ylim(-0.7, 0.7) for thisax in ax] [thisax.set_xlim(rminmax + [-0.1, 0.0]) for thisax in ax] [thisax.axhline(y=0.0, lw=2, ls='solid', color='gray') for thisax in ax] for ix, thisband in enumerate(['g', 'r', 'z']): ax[ix].text(0.05, 0.05, thisband, horizontalalignment='left', verticalalignment='bottom', transform=ax[ix].transAxes, fontsize=16) ax[1].set_ylabel('Input Flux / Tractor Flux - 1') ax[2].set_xlabel('Input r magnitude (AB mag)') fig.subplots_adjust(left=0.18, hspace=0.1) qafile = os.path.join(decals_sim_dir, 'qa-' + brickname + '-' + lobjtype + '-flux.png') log.info('Writing {}'.format(qafile)) plt.savefig(qafile) # Color residuals gr_tra = -2.5 * np.log10(gflux_tra / rflux_tra) rz_tra = -2.5 * np.log10(rflux_tra / zflux_tra) gr_sim = -2.5 * np.log10(gflux_sim / rflux_sim) rz_sim = -2.5 * np.log10(rflux_sim / zflux_sim) fig, ax = plt.subplots(2, sharex=True, figsize=(6, 8)) ax[0].scatter(rmag, gr_tra - gr_sim, color=col[0], s=10) ax[1].scatter(rmag, rz_tra - rz_sim, color=col[1], s=10) [thisax.set_ylim(-0.7, 0.7) for thisax in ax] [thisax.set_xlim(rminmax + [-0.1, 0.0]) for thisax in ax] [thisax.axhline(y=0.0, lw=2, ls='solid', color='gray') for thisax in ax] ax[0].set_ylabel('$\Delta$(g - r) (Tractor minus Input)') ax[1].set_ylabel('$\Delta$(r - z) (Tractor minus Input)') ax[1].set_xlabel('Input r magnitude (AB mag)') fig.subplots_adjust(left=0.18, hspace=0.1) qafile = os.path.join(decals_sim_dir, 'qa-' + brickname + '-' + lobjtype + '-color.png') log.info('Writing {}'.format(qafile)) plt.savefig(qafile) sys.exit(1) # Get cutouts of the missing sources imfile = os.path.join( decals_sim_dir, 'qa-' + brickname + '-' + lobjtype + '-image-' + chunksuffix + '.jpg') hw = 30 # half-width [pixels] ncols = 5 nrows = 5 nthumb = ncols * nrows dims = (ncols * hw * 2, nrows * hw * 2) mosaic = Image.new('RGB', dims) miss = missing[np.argsort(simcat['r'][missing])] print(simcat['r'][miss]) xpos, ypos = np.meshgrid(np.arange(0, dims[0], hw * 2, dtype='int'), np.arange(0, dims[1], hw * 2, dtype='int')) im = Image.open(imfile) sz = im.size iobj = 0 for ic in range(ncols): for ir in range(nrows): mm = miss[iobj] xx = int(simcat['X'][mm]) yy = int(sz[1] - simcat['Y'][mm]) crop = (xx - hw, yy - hw, xx + hw, yy + hw) box = (xpos[ir, ic], ypos[ir, ic]) thumb = im.crop(crop) mosaic.paste(thumb, box) iobj = iobj + 1 # Add a border draw = ImageDraw.Draw(mosaic) for ic in range(ncols): for ir in range(nrows): draw.rectangle([(xpos[ir, ic], ypos[ir, ic]), (xpos[ir, ic] + hw * 2, ypos[ir, ic] + hw * 2)]) qafile = os.path.join(decals_sim_dir, 'qa-' + brickname + '-' + lobjtype + '-missing.png') log.info('Writing {}'.format(qafile)) mosaic.save(qafile) # Modify the coadd image and residual files so the simulated sources # are labeled. rad = 15 imfile = os.path.join( decals_sim_dir, 'qa-' + brickname + '-' + lobjtype + '-image-' + chunksuffix + '.jpg') imfile = [imfile, imfile.replace('-image', '-resid')] for ifile in imfile: im = Image.open(ifile) sz = im.size draw = ImageDraw.Draw(im) [ draw.ellipse((cat['X'] - rad, sz[1] - cat['Y'] - rad, cat['X'] + rad, sz[1] - cat['Y'] + rad)) for cat in simcat ] im.save(ifile) # Fraction of matching sources rmaghist, magbins = np.histogram(simcat['r'], bins=nmagbin, range=rminmax) cmagbins = (magbins[:-1] + magbins[1:]) / 2.0 ymatch, binsmatch = np.histogram(simcat['r'][m2], bins=nmagbin, range=rminmax) ymatchgood, binsgood = np.histogram(simcat['r'][m2[good]], bins=nmagbin, range=rminmax) fig, ax = plt.subplots(1, figsize=(8, 6)) ax.step(cmagbins, 1.0 * ymatch / rmaghist, lw=3, alpha=0.5, label='All objects') ax.step(cmagbins, 1.0 * ymatchgood / rmaghist, lw=3, ls='dashed', label='|$\Delta$m|<0.3') ax.axhline(y=1.0, lw=2, ls='dashed', color='gray') ax.set_xlabel('Input r magnitude (AB mag)') ax.set_ylabel('Fraction of Matching ' + objtype + 's') ax.set_ylim([0.0, 1.1]) ax.legend(loc='lower left') fig.subplots_adjust(bottom=0.15) qafile = os.path.join(decals_sim_dir, 'qa-' + brickname + '-' + lobjtype + '-frac.png') log.info('Writing {}'.format(qafile)) plt.savefig(qafile) # Distribution of object types fig = plt.figure(figsize=(8, 6)) ax = fig.gca() rmaghist, magbins = np.histogram(simcat['r'][m2], bins=nmagbin, range=rminmax) cmagbins = (magbins[:-1] + magbins[1:]) / 2.0 tractortype = tractor['TYPE'][m1].strip() for otype in ['PSF', 'EXP', 'DEV', 'COMP']: these = np.where(tractortype == otype)[0] if len(these) > 0: yobj, binsobj = np.histogram(simcat['r'][m2[these]], bins=nmagbin, range=rminmax) #plt.step(cmagbins,1.0*yobj,lw=3,alpha=0.5,label=otype) plt.step(cmagbins, 1.0 * yobj / rmaghist, lw=3, alpha=0.5, label=otype) plt.axhline(y=1.0, lw=2, ls='dashed', color='gray') plt.xlabel('Input r magnitude (AB mag)') #plt.ylabel('Number of Objects') plt.ylabel('Fraction of ' + objtype + 's classified') plt.ylim([0.0, 1.1]) plt.legend(loc='center left', bbox_to_anchor=(0.08, 0.5)) fig.subplots_adjust(bottom=0.15) qafile = os.path.join(decals_sim_dir, 'qa-' + brickname + '-' + lobjtype + '-type.png') log.info('Writing {}'.format(qafile)) plt.savefig(qafile) # Morphology plots if objtype == 'ELGo': fig = plt.figure(figsize=(8, 4)) plt.subplot(1, 3, 1) plt.plot(rmag, deltam, 's', markersize=3) plt.axhline(y=0.0, lw=2, ls='solid', color='gray') plt.xlim(rminmax) plt.xlabel('r (AB mag)') plt.subplot(1, 3, 2) plt.plot(simcat['R50_1'][m2], deltam, 's', markersize=3) plt.axhline(y=0.0, lw=2, ls='solid', color='gray') plt.xlabel('$r_{50}$ (arcsec)') plt.subplot(1, 3, 3) plt.plot(simcat['BA_1'][m2], deltam, 's', markersize=3) plt.axhline(y=0.0, lw=2, ls='solid', color='gray') plt.xlabel('b/a') plt.xlim([0.2, 1.0]) fig.subplots_adjust(bottom=0.18) qafile = os.path.join( decals_sim_dir, 'qa-' + brickname + '-' + lobjtype + '-morph.png') log.info('Writing {}'.format(qafile)) plt.savefig(qafile)
def mosaic(): ''' > cp ~/cosmo/staging/mosaicz/MZLS_CP/CP20180102/k4m_180103_040423_ooi_zd_v1.fits.fz /tmp > funpack /tmp/k4m_180103_040423_ooi_zd_v1.fits.fz > fitsgetext -i /tmp/k4m_180103_040423_ooi_zd_v1.fits -o mosaic-%02i.wcs -a -H > cat mosaic-??.wcs > mosaic.wcs > for ((i=1; i<=4; i++)); do modhead mosaic.wcs+$i NAXIS2; modhead mosaic.wcs+$i NAXIS2 0; done NAXIS2 = 4079 / Axis length NAXIS2 = 4079 / Axis length NAXIS2 = 4079 / Axis length NAXIS2 = 4061 / Axis length ''' plt.figure(figsize=(4,3)) plt.subplots_adjust(left=0.15, right=0.99, top=0.99, bottom=0.15) T = fits_table('obstatus/mosaic-tiles_obstatus.fits') print(len(T), 'tiles') T.rename('pass', 'passnum') T.cut(T.passnum <= 3) print(len(T), 'tiles with passnum <= 3') ra,dec = 180.216, 40.191 #ra,dec = 180., 40. I,J,d = match_radec(T.ra, T.dec, ra, dec, 2.) print(len(I), 'tiles near', ra,dec) T.cut(I) T.dist = d print('dists:', d) print('Passes:', T.passnum) F = fitsio.FITS(os.path.join(os.path.dirname(__file__), 'mosaic.wcs')) wcs = [] heights = [ 4079, 4079, 4079, 4061 ] for i in range(1, len(F)): hdr = F[i].read_header() W = hdr['NAXIS1'] wcs.append(wcs_pv2sip_hdr(hdr, H=heights[i-1], W=W)) print('WCS:', wcs[-1]) # Rendering canvas W,H = 2200, 2200 pixsc = 4./3600. targetwcs = Tan(ra, dec, W/2.+0.5, H/2.+0.5, -pixsc, 0., 0., pixsc, float(W), float(H)) II = np.lexsort((T.dist, T.passnum)) print('First tile center:', T.ra[II[0]], T.dec[II[0]]) # This is for making the (vector) PDF format tiling images. for maxit in [0, 8, 36, 37, 44, 73, 74, 82, 112]: plot = Plotstuff(outformat='pdf', ra=ra, dec=dec, width=W*pixsc, size=(W,H), outfn='tile-mosaic-%02i.pdf' % maxit) plot.color = 'white' plot.alpha = 1. plot.plot('fill') out = plot.outline out.fill = True out.stepsize = 1024. plot.color = 'black' plot.alpha = 0.4 plot.apply_settings() for it,t in enumerate(T[II]): print('Tile', it, 'pass', t.passnum) for w in wcs: w.set_crval((t.ra, t.dec)) out.wcs = anwcs_new_sip(w) plot.plot('outline') if it == maxit: print('Writing', it) plot.write() break # # And this is for PNG-format tiling images and histograms. cov = np.zeros((H,W), np.uint8) for it,t in enumerate(T[II]): print('Tile', it, 'pass', t.passnum) for w in wcs: w.set_crval((t.ra, t.dec)) try: Yo,Xo,Yi,Xi,nil = resample_with_wcs(targetwcs, w) except: continue cov[Yo,Xo] += 1 if it in [0, 8, 36, 37, 44, 73, 74, 82, 112]: mx = { 1: 2, 2: 4, 3: 6 }[t.passnum] # plt.clf() # plt.imshow(cov, interpolation='nearest', origin='lower', vmin=0, vmax=mx) # plt.colorbar() # plt.savefig('tile-%02i.png' % it) plt.imsave('tile-mosaic-%02i.png' % it, cov, origin='lower', vmin=0, vmax=mx, cmap=antigray) if it in [36, 73, 112]: from collections import Counter print('Coverage counts:', Counter(cov.ravel()).most_common()) bins = -0.5 + np.arange(8) plt.clf() n,b,p = plt.hist(cov.ravel(), bins=bins, normed=True) #plt.hist(cov.ravel(), bins=bins, normed=True, cumulative=True, histtype='step') # Cumulative histogram from the right... xx,yy = [],[] for blo,bhi,ni in reversed(list(zip(bins, bins[1:], n))): nc = float(np.sum(cov.ravel() > blo)) / len(cov.ravel()) yy.extend([nc,nc]) xx.extend([bhi,blo]) if ni > 0: if nc != ni: if nc > ni+0.03: # If there's room, label the histogram bin above, else below plt.text((blo+bhi)/2., ni, '%.1f \%%' % (100.*ni), ha='center', va='bottom', color='k') else: plt.text((blo+bhi)/2., ni-0.01, '%.1f \%%' % (100.*ni), ha='center', va='top', color='k') plt.text((blo+bhi)/2., nc, '%.1f \%%' % (100.*nc), ha='center', va='bottom', color='k') plt.plot(xx, yy, 'k-') plt.xlim(bins.min(), bins.max()) plt.ylim(0., 1.1) plt.xlabel('Number of exposures') plt.ylabel('Fraction of sky') plt.savefig('hist-mosaic-%02i.pdf' % it)
names=True, dtype=None) d25 = table['d25'] name_cal, ra_cal, dec_cal, a_cal, b_cal, pa_cal, ty_cal = read_file( 'sample_tfc_all.glga', n_skip=4, seprator=' ') #print name_cal #print ra_cal #print dec_cal # Python equivalent of matchlength radius_in_deg = 1. / 60. # 1 arcmin match (m1, m2, d) = spherematch.match_radec(ra_cal, dec_cal, ra, dec, radius_in_deg, notself=False, nearest=True) print len(m1) #for m in range(len(m1)): #i = m1[m] #j = m2[m] #if d[m]*3600>=30.: #print name_cal[i], ra_cal[i], dec_cal[i], pgc[j], ra[j], dec[j], d[m]*3600 db_dir = "/home/ehsan/db_esn/data/"
morph = fits_table(sefn, hdu=2) wcs = Sip(im.wcsfn) if len(sdss) == 0: print 'EMPTY:', im.sdssfn continue if len(morph) == 0: print 'EMPTY:', im.morphfn continue print len(sdss), 'SDSS sources from', im.sdssfn print len(morph), 'SE sources from', sefn morph.ra, morph.dec = wcs.pixelxy2radec(morph.x_image, morph.y_image) I, J, d = match_radec(morph.ra, morph.dec, sdss.ra, sdss.dec, 0.5 / 3600.) corr = sdss[J] corr.add_columns_from(morph[I]) chipnames.append(im.extname) #corr = fits_table(im.corrfn) corrs.append(corr) print im, ':', len(corr), 'correspondences' for col, cut in ([('flux_auto', None)] + [('flux_aper', i) for i in range(3)] + [('flux_psf', None), ('flux_model', None)]): dmags = [] smags = [] for corr in corrs:
def map_decals_wl(req, ver, zoom, x, y): tag = 'decals-wl' ignoreCached = False filename = None forcecache = False from decals import settings savecache = settings.SAVE_CACHE zoom = int(zoom) zoomscale = 2.**zoom x = int(x) y = int(y) if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale: raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom,x,y)) ver = int(ver) if not ver in tileversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) basedir = settings.DATA_DIR tilefn = os.path.join(basedir, 'tiles', tag, '%i/%i/%i/%i.jpg' % (ver, zoom, x, y)) if os.path.exists(tilefn) and not ignoreCached: print('Cached:', tilefn) return send_file(tilefn, 'image/jpeg', expires=oneyear, modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'), filename=filename) else: print('Tile image does not exist:', tilefn) from astrometry.util.resample import resample_with_wcs, OverlapError from astrometry.util.util import Tan from astrometry.libkd.spherematch import match_radec from astrometry.util.fits import fits_table from astrometry.util.starutil_numpy import degrees_between import numpy as np import fitsio try: wcs, W, H, zoomscale, zoom,x,y = get_tile_wcs(zoom, x, y) except RuntimeError as e: return HttpResponse(e.strerror) mydir = os.path.join(basedir, 'coadd', 'weak-lensing') rlo,d = wcs.pixelxy2radec(W, H/2)[-2:] rhi,d = wcs.pixelxy2radec(1, H/2)[-2:] r,d1 = wcs.pixelxy2radec(W/2, 1)[-2:] r,d2 = wcs.pixelxy2radec(W/2, H)[-2:] #dlo = min(d1, d2) #dhi = max(d1, d2) r,d = wcs.pixelxy2radec(W/2, H/2)[-2:] rad = degrees_between(r, d, rlo, d1) fn = os.path.join(mydir, 'index.fits') if not os.path.exists(fn): # ii,rr,dd = [],[],[] for i in range(1, 52852+1): imgfn = os.path.join(mydir, 'map%i.fits' % i) hdr = fitsio.read_header(imgfn) r = hdr['CRVAL1'] d = hdr['CRVAL2'] ii.append(i) rr.append(r) dd.append(d) T = fits_table() T.ra = np.array(rr) T.dec = np.array(dd) T.i = np.array(ii) T.writeto(fn) T = fits_table(fn) I,J,d = match_radec(T.ra, T.dec, r, d, rad + 0.2) T.cut(I) print(len(T), 'weak-lensing maps in range') if len(I) == 0: from django.http import HttpResponseRedirect if forcecache: # create symlink to blank.jpg! trymakedirs(tilefn) src = os.path.join(settings.STATIC_ROOT, 'blank.jpg') if os.path.exists(tilefn): os.unlink(tilefn) os.symlink(src, tilefn) print('Symlinked', tilefn, '->', src) return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg') r,d = wcs.pixelxy2radec([1,1,1,W/2,W,W,W,W/2], [1,H/2,H,H,H,H/2,1,1])[-2:] foundany = False rimg = np.zeros((H,W), np.float32) rn = np.zeros((H,W), np.uint8) for tilei in T.i: fn = os.path.join(mydir, 'map%i.fits' % tilei) try: bwcs = _read_tan_wcs(fn, 0) except: print('Failed to read WCS:', fn) savecache = False import traceback import sys traceback.print_exc(None, sys.stdout) continue foundany = True print('Reading', fn) ok,xx,yy = bwcs.radec2pixelxy(r, d) xx = xx.astype(np.int) yy = yy.astype(np.int) imW,imH = int(bwcs.get_width()), int(bwcs.get_height()) M = 10 xlo = np.clip(xx.min() - M, 0, imW) xhi = np.clip(xx.max() + M, 0, imW) ylo = np.clip(yy.min() - M, 0, imH) yhi = np.clip(yy.max() + M, 0, imH) if xlo >= xhi or ylo >= yhi: continue subwcs = bwcs.get_subimage(xlo, ylo, xhi-xlo, yhi-ylo) slc = slice(ylo,yhi), slice(xlo,xhi) try: f = fitsio.FITS(fn)[0] img = f[slc] del f except: print('Failed to read image and WCS:', fn) savecache = False import traceback import sys traceback.print_exc(None, sys.stdout) continue try: Yo,Xo,Yi,Xi,nil = resample_with_wcs(wcs, subwcs, [], 3) except OverlapError: print('Resampling exception') continue rimg[Yo,Xo] += img[Yi,Xi] rn [Yo,Xo] += 1 rimg /= np.maximum(rn, 1) if forcecache: savecache = True if savecache: trymakedirs(tilefn) else: import tempfile f,tilefn = tempfile.mkstemp(suffix='.jpg') os.close(f) import pylab as plt # S/N #lo,hi = 1.5, 5.0 lo,hi = 0, 5.0 rgb = plt.cm.hot((rimg - lo) / (hi - lo)) plt.imsave(tilefn, rgb) print('Wrote', tilefn) return send_file(tilefn, 'image/jpeg', unlink=(not savecache), filename=filename)
def main(): """Main program. """ import argparse parser = argparse.ArgumentParser( description= "This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)." ) parser.add_argument('--calibs', action='store_true', help='Output CCDs that need to be calibrated.') parser.add_argument('--nper', type=int, default=None, help='Batch N calibs per line') parser.add_argument('--forced', action='store_true', help='Output forced-photometry commands') parser.add_argument('--lsb', action='store_true', help='Output Low-Surface-Brightness commands') parser.add_argument('--touching', action='store_true', help='Cut to only CCDs touching selected bricks') parser.add_argument('--check', action='store_true', help='Check which calibrations actually need to run.') parser.add_argument('--check-coadd', action='store_true', help='Check which caoadds actually need to run.') parser.add_argument('--out', help='Output filename for calibs, default %(default)s', default='jobs') parser.add_argument('--command', action='store_true', help='Write out full command-line to run calib') parser.add_argument('--maxdec', type=float, help='Maximum Dec to run') parser.add_argument('--mindec', type=float, help='Minimum Dec to run') parser.add_argument('--region', help='Region to select') parser.add_argument('--bricks', help='Set bricks.fits file to load') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--delete-sky', action='store_true', help='Delete any existing sky calibration files') parser.add_argument('--delete-pvastrom', action='store_true', help='Delete any existing PV WCS calibration files') parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?') opt = parser.parse_args() decals = Decals() if opt.bricks is not None: B = fits_table(opt.bricks) log('Read', len(B), 'from', opt.bricks) else: B = decals.get_bricks() if opt.ccds is not None: T = fits_table(opt.ccds) log('Read', len(T), 'from', opt.ccds) else: T = decals.get_ccds() log(len(T), 'CCDs') T.index = np.arange(len(T)) # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True) # plt.clf() # plt.hist(counts, counts.max()+1) # plt.savefig('bricks.png') # B.cut(I[counts >= 9]) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # #plt.scatter(B.ra[I], B.dec[I], c=counts) # plt.savefig('bricks2.png') # DES Stripe82 #rlo,rhi = 350.,360. # rlo,rhi = 300., 10. # dlo,dhi = -6., 4. # TINY bit #rlo,rhi = 350.,351.1 #dlo,dhi = 0., 1.1 # EDR+ # 860 bricks # ~10,000 CCDs #rlo,rhi = 239,246 #dlo,dhi = 5, 13 # DR1 #rlo,rhi = 0, 360 # part 1 #dlo,dhi = 25, 40 # part 2 #dlo,dhi = 20,25 # part 3 #dlo,dhi = 15,20 # part 4 #dlo,dhi = 10,15 # part 5 #dlo,dhi = 5,10 # the rest #dlo,dhi = -11, 5 #dlo,dhi = 15,25.5 dlo, dhi = -15, 40 rlo, rhi = 0, 360 # Arjun says 3x3 coverage area is roughly # RA=240-252 DEC=6-12 (but not completely rectangular) # COSMOS #rlo,rhi = 148.9, 151.2 #dlo,dhi = 0.9, 3.5 # A nice well-behaved region (EDR2/3) # rlo,rhi = 243.6, 244.6 # dlo,dhi = 8.1, 8.6 # 56 bricks, ~725 CCDs #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7)) # 240 bricks, ~3000 CCDs #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9)) # 535 bricks, ~7000 CCDs #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12)) if opt.region in ['test1', 'test2', 'test3', 'test4']: nm = dict( test1='2446p115', # weird stuff around bright star test2='1183p292', # faint sources around bright galaxy test3='3503p005', # DES test4='1163p277', # Pollux )[opt.region] B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname]))) log('Cut to', len(B), 'bricks') log(B.ra, B.dec) dlo, dhi = -90, 90 rlo, rhi = 0, 360 elif opt.region == 'badsky': # A handful of exposures in DR2 with inconsistent sky estimates. #C = decals.get_ccds() #log(len(C), 'CCDs') #C.cut((C.expnum >= 257400) * (C.expnum < 257500)) T.cut( np.array([ e in [ 257460, 257461, 257462, 257463, 257464, 257465, 257466, 257467, 257469, 257472, 257483, 257496 ] for e in T.expnum ])) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25) / 2. I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'badsky2': # UGH, missed this one in original 'badsky' definition. T.cut(T.expnum == 257466) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25) / 2. I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'edr': # EDR: # 535 bricks, ~7000 CCDs rlo, rhi = 240, 245 dlo, dhi = 5, 12 elif opt.region == 'edr-south': rlo, rhi = 240, 245 dlo, dhi = 5, 10 elif opt.region == 'cosmos1': # 16 bricks in the core of the COSMOS field. rlo, rhi = 149.75, 150.75 dlo, dhi = 1.6, 2.6 elif opt.region == 'pristine': # Stream? rlo, rhi = 240, 250 dlo, dhi = 10, 15 elif opt.region == 'des': dlo, dhi = -6., 4. rlo, rhi = 317., 7. T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage]))) log('Cut to', len(T), 'CCDs with "CPDES82" in filename') elif opt.region == 'subdes': rlo, rhi = 320., 360. dlo, dhi = -1.25, 1.25 elif opt.region == 'northwest': rlo, rhi = 240, 360 dlo, dhi = 20, 40 elif opt.region == 'north': rlo, rhi = 120, 240 dlo, dhi = 20, 40 elif opt.region == 'northeast': rlo, rhi = 0, 120 dlo, dhi = 20, 40 elif opt.region == 'southwest': rlo, rhi = 240, 360 dlo, dhi = -20, 0 elif opt.region == 'south': rlo, rhi = 120, 240 dlo, dhi = -20, 0 elif opt.region == 'southeast': rlo, rhi = 0, 120 dlo, dhi = -20, 0 elif opt.region == 'midwest': rlo, rhi = 240, 360 dlo, dhi = 0, 20 elif opt.region == 'middle': rlo, rhi = 120, 240 dlo, dhi = 0, 20 elif opt.region == 'mideast': rlo, rhi = 0, 120 dlo, dhi = 0, 20 elif opt.region == 'grz': # Bricks with grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)) log('Cut to', len(B), 'bricks with grz coverage') elif opt.region == 'nogrz': # Bricks without grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))) log('Cut to', len(B), 'bricks withOUT grz coverage') elif opt.region == 'deep2': rlo, rhi = 250, 260 dlo, dhi = 30, 35 elif opt.region == 'virgo': rlo, rhi = 185, 190 dlo, dhi = 10, 15 elif opt.region == 'virgo2': rlo, rhi = 182, 192 dlo, dhi = 8, 18 elif opt.region == 'lsb': rlo, rhi = 147.2, 147.8 dlo, dhi = -0.4, 0.4 if opt.mindec is not None: dlo = opt.mindec if opt.maxdec is not None: dhi = opt.maxdec if rlo < rhi: B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) else: # RA wrap B.cut( np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) log(len(B), 'bricks in range') I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs') if opt.touching: keep = np.zeros(len(T), bool) for j in J: keep[j] = True T.cut(keep) log('Cut to', len(T), 'CCDs near bricks') # Aside -- how many near DR1=1 CCDs? if False: T2 = D.get_ccds() log(len(T2), 'CCDs') T2.cut(T2.dr1 == 1) log(len(T2), 'CCDs marked DR1=1') log(len(B), 'bricks in range') I, J, d = match_radec(B.ra, B.dec, T2.ra, T2.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B2 = B[keep] log('Total of', len(B2), 'bricks near CCDs with DR1=1') for band in 'grz': Tb = T2[T2.filter == band] log(len(Tb), 'in filter', band) I, J, d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, 0.25) good = np.zeros(len(B2), np.uint8) for i in I: good[i] = 1 B2.set('has_' + band, good) B2.writeto('decals-bricks-in-dr1.fits') sys.exit(0) # sort by dec decreasing B.cut(np.argsort(-B.dec)) for b in B: if opt.check: fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue if opt.check_coadd: fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % ( b.brickname[:3], b.brickname, b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue print(b.brickname) if not (opt.calibs or opt.forced or opt.lsb): sys.exit(0) bands = 'grz' log('Filters:', np.unique(T.filter)) T.cut(np.flatnonzero(np.array([f in bands for f in T.filter]))) log('Cut to', len(T), 'CCDs in filters', bands) if opt.touching: allI = set() for b in B: wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec)) if len(I) == 0: continue allI.update(I) allI = list(allI) allI.sort() else: allI = np.arange(len(T)) if opt.write_ccds: T[allI].writeto(opt.write_ccds) log('Wrote', opt.write_ccds) ## Be careful here -- T has been cut; we want to write out T.index. ## 'allI' contains indices into T. if opt.forced: log('Writing forced-photometry commands to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') for j, i in enumerate(allI): expstr = '%08i' % T.expnum[i] #outdir = os.path.join('forced', expstr[:5], expstr) #trymakedirs(outdir) outfn = os.path.join( 'forced', expstr[:5], expstr, 'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i])) imgfn = os.path.join(decals.decals_dir, 'images', T.image_filename[i].strip()) if (not os.path.exists(imgfn) and imgfn.endswith('.fz') and os.path.exists(imgfn[:-3])): imgfn = imgfn[:-3] f.write('python legacypipe/forced-photom-decam.py %s %i DR1 %s\n' % (imgfn, T.cpimage_hdu[i], outfn)) f.close() log('Wrote', opt.out) sys.exit(0) if opt.lsb: log('Writing LSB commands to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') for j, i in enumerate(allI): exp = T.expnum[i] ext = T.ccdname[i].strip() outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext) f.write( 'python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext)) f.close() log('Wrote', opt.out) sys.exit(0) log('Writing calibs to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') batch = [] def write_batch(f, batch, cmd): if opt.command: s = '; '.join(batch) else: s = ' '.join(batch) f.write(s + '\n') for j, i in enumerate(allI): if opt.delete_sky or opt.delete_pvastrom: log(j + 1, 'of', len(allI)) im = decals.get_image_object(T[i]) if opt.delete_sky and os.path.exists(im.skyfn): log(' deleting:', im.skyfn) os.unlink(im.skyfn) if opt.delete_pvastrom and os.path.exists(im.pvwcsfn): log(' deleting:', im.pvwcsfn) os.unlink(im.pvwcsfn) if opt.check: log(j + 1, 'of', len(allI)) im = decals.get_image_object(T[i]) if not im.run_calibs(im, just_check=True): log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done') continue if opt.command: s = ('python legacypipe/run-calib.py --expnum %i --ccdname %s' % (T.expnum[i], T.ccdname[i])) else: s = '%i' % T.index[i] if not opt.nper: f.write(s + '\n') else: batch.append(s) if len(batch) >= opt.nper: write_batch(f, batch, opt.command) batch = [] if opt.check: f.flush() if len(batch): write_batch(f, batch, opt.command) f.close() log('Wrote', opt.out) return 0
def get_reference_sources(survey, targetwcs, pixscale, bands, tycho_stars=True, gaia_stars=True, large_galaxies=True, star_clusters=True, clean_columns=True, plots=False, ps=None, gaia_margin=None, galaxy_margin=None): # If bands = None, does not create sources. H, W = targetwcs.shape H, W = int(H), int(W) # How big of a margin to search for bright stars and star clusters -- # this should be based on the maximum radius they are considered to # affect. In degrees. if gaia_margin is not None: ref_margin = gaia_margin else: ref_margin = mask_radius_for_mag(0.) mpix = int(np.ceil(ref_margin * 3600. / pixscale)) marginwcs = targetwcs.get_subimage(-mpix, -mpix, W + 2 * mpix, H + 2 * mpix) # Table of reference-source properties, including a field 'sources', # with tractor source objects. refs = [] # Tycho-2 stars tycho = [] if tycho_stars: tycho = read_tycho2(survey, marginwcs, bands) if tycho and len(tycho): refs.append(tycho) # Add Gaia stars gaia = None if gaia_stars: from astrometry.libkd.spherematch import match_radec gaia = read_gaia(marginwcs, bands) if gaia is not None: # Handle sources that appear in both Gaia and Tycho-2 by # dropping the entry from Tycho-2. if len(gaia) and len(tycho): # Before matching, apply proper motions to bring them to # the same epoch. We want to use the more-accurate Gaia # proper motions, so rewind Gaia positions to the Tycho # epoch. (Note that in read_tycho2, we massaged the # epochs) cosdec = np.cos(np.deg2rad(gaia.dec)) # First do a coarse matching with the approximate epoch: dt = 1991.5 - gaia.ref_epoch gra = gaia.ra + dt * gaia.pmra / (3600. * 1000.) / cosdec gdec = gaia.dec + dt * gaia.pmdec / (3600. * 1000.) # Max Tycho-2 PM is 10"/yr, max |epoch_ra,epoch_dec - mean| = 0.5 I, J, _ = match_radec(tycho.ra, tycho.dec, gra, gdec, 10. / 3600., nearest=True) debug('Initially matched', len(I), 'Tycho-2 stars to Gaia stars (10").') if plots: import pylab as plt plt.clf() plt.plot(gra, gdec, 'bo', label='Gaia (1991.5)') plt.plot(gaia.ra, gaia.dec, 'gx', label='Gaia (2015.5)') plt.plot([gaia.ra, gra], [gaia.dec, gdec], 'k-') plt.plot([tycho.ra[I], gra[J]], [tycho.dec[I], gdec[J]], 'r-') plt.plot(tycho.ra, tycho.dec, 'rx', label='Tycho-2') plt.plot(tycho.ra[I], tycho.dec[I], 'o', mec='r', ms=8, mfc='none', label='Tycho-2 matched') plt.legend() r0, r1, d0, d1 = targetwcs.radec_bounds() plt.axis([r0, r1, d0, d1]) plt.title('Initial (10") matching') ps.savefig() dt = tycho.ref_epoch[I] - gaia.ref_epoch[J] cosdec = np.cos(np.deg2rad(gaia.dec[J])) gra = gaia.ra[J] + dt * gaia.pmra[J] / (3600. * 1000.) / cosdec gdec = gaia.dec[J] + dt * gaia.pmdec[J] / (3600. * 1000.) dists = np.hypot((gra - tycho.ra[I]) * cosdec, gdec - tycho.dec[I]) K = np.flatnonzero(dists <= 1. / 3600.) if len(K) < len(I): debug('Unmatched Tycho-2 - Gaia stars: dists', dists[dists > 1. / 3600.] * 3600.) I = I[K] J = J[K] debug('Matched', len(I), 'Tycho-2 stars to Gaia stars.') if len(I): keep = np.ones(len(tycho), bool) keep[I] = False tycho.cut(keep) gaia.isbright[J] = True gaia.istycho[J] = True if gaia is not None and len(gaia) > 0: refs.append(gaia) # Read the catalog of star (open and globular) clusters and add them to the # set of reference stars (with the isbright bit set). if star_clusters: clusters = read_star_clusters(marginwcs) if clusters is not None: debug('Found', len(clusters), 'star clusters nearby') refs.append(clusters) # Read large galaxies nearby. if large_galaxies: kw = {} if galaxy_margin is not None: kw.update(max_radius=galaxy_margin + np.hypot(H, W) / 2. * pixscale / 3600) galaxies = read_large_galaxies(survey, targetwcs, bands, clean_columns=clean_columns, **kw) if galaxies is not None: # Resolve possible Gaia-large-galaxy duplicates if gaia and len(gaia): I, J, _ = match_radec(galaxies.ra, galaxies.dec, gaia.ra, gaia.dec, 2. / 3600., nearest=True) info('Matched', len(I), 'large galaxies to Gaia stars.') if len(I): gaia.donotfit[J] = True # Resolve possible Tycho2-large-galaxy duplicates (with larger radius) if tycho and len(tycho): I, J, _ = match_radec(galaxies.ra, galaxies.dec, tycho.ra, tycho.dec, 5. / 3600., nearest=True) info('Matched', len(I), 'large galaxies to Tycho-2 stars.') if len(I): tycho.donotfit[J] = True refs.append(galaxies) if len(refs): refs = merge_tables([r for r in refs if r is not None], columns='fillzero') if len(refs) == 0: return None, None # these x,y are in the margin-padded WCS; not useful. # See ibx,iby computed below instead. for c in ['x', 'y']: if c in refs.get_columns(): refs.delete_column(c) # radius / radius_pix are used to set the MASKBITS shapes; # keep_radius determines which sources are kept (because we subtract # stellar halos out to N x their radii) refs.radius_pix = np.ceil(refs.radius * 3600. / pixscale).astype(int) debug('Increasing radius for', np.sum(refs.keep_radius > refs.radius), 'ref sources based on keep_radius') keeprad = np.maximum(refs.keep_radius, refs.radius) # keeprad to pix keeprad = np.ceil(keeprad * 3600. / pixscale).astype(int) _, xx, yy = targetwcs.radec2pixelxy(refs.ra, refs.dec) # ibx = integer brick coords refs.ibx = np.round(xx - 1.).astype(np.int32) refs.iby = np.round(yy - 1.).astype(np.int32) # cut ones whose position + radius are outside the brick bounds. refs.cut((xx > -keeprad) * (xx < W + keeprad) * (yy > -keeprad) * (yy < H + keeprad)) # mark ones that are actually inside the brick area. refs.in_bounds = ((refs.ibx >= 0) * (refs.ibx < W) * (refs.iby >= 0) * (refs.iby < H)) # ensure bool columns for col in [ 'isbright', 'ismedium', 'islargegalaxy', 'iscluster', 'isgaia', 'istycho', 'donotfit', 'freezeparams' ]: if not col in refs.get_columns(): refs.set(col, np.zeros(len(refs), bool)) # Copy flags from the 'refs' table to the source objects themselves. sources = refs.sources refs.delete_column('sources') for i, (donotfit, freeze) in enumerate(zip(refs.donotfit, refs.freezeparams)): if donotfit: sources[i] = None if sources[i] is None: continue sources[i].is_reference_source = True if freeze: sources[i].freezeparams = True return refs, sources
def real_one_tile((args)): #(itile,tile,udecs,P3,T,tilewcs,exps,bad_expids,tileid_to_depth) = args (itile,tile) = args print() print('Tile', itile+1, ':', tile.tileid, 'at', tile.ra, tile.dec) i = np.nonzero(tile.dec == udecs)[0][0] if i == 0 or i == len(udecs)-1: print('Endpoint Dec; skipping for now') return None print(' Decs:', udecs[i-1], tile.dec, udecs[i+1]) declo = (udecs[i-1] + tile.dec) / 2. dechi = (udecs[i+1] + tile.dec) / 2. row = P3[P3.dec == tile.dec] print(' ', len(row), 'tiles in this Dec row') ras = np.sort(row.ra) i = np.nonzero(tile.ra == ras)[0][0] if i == 0 or i == len(ras)-1: print(' Endpoint RA; skipping for now') return None print(' RAs:', ras[i-1], tile.ra, ras[i+1]) ralo = (ras[i-1] + tile.ra) / 2. rahi = (ras[i+1] + tile.ra) / 2. pixscale = 2. #pixscale = 0.262 H = int(np.ceil((dechi - declo) / (pixscale/3600.))) W = int(np.ceil((rahi - ralo) * np.cos(np.deg2rad(tile.dec)) / (pixscale/3600.))) print(' Dec height', dechi-declo, 'RA width', rahi-ralo, '-> pix', W, 'x', H) cd = pixscale/3600. thiswcs = Tan(tile.ra, tile.dec, (W+1)/2., (H+1)/2., -cd, 0., 0., cd, float(W), float(H)) # Find surrounding tiles radius = np.hypot(W, H) * pixscale / 3600. I,J,d = match_radec(T.ra, T.dec, tile.ra, tile.dec, radius) print(' ', len(I), 'tiles with measured depths nearby') if len(I) == 0: return None # Now we need to take a tile boresight position and map it to # boxes in RA,Dec of the good portions of the CCDs. depth = np.zeros((H,W), np.float32) nexp = np.zeros((H,W), np.uint8) matched_exposures = set() #print(' tileids', T.tileid[I]) #print(' expnums', T.z_expnum[I]) for ii in I: if T.z_expnum[ii] in bad_expids: print(' skipping bad exp num', T.z_expnum[ii]) continue # Get depth from CCDs file, if available. zdepth = tileid_to_depth.get(T.tileid[ii], 0.) if zdepth == 0.: zdepth = T.z_depth[ii] matched_exposures.add(T.z_expnum[ii]) for twcs in tilewcs: twcs.set_crval((T.ra[ii], T.dec[ii])) try: Yo,Xo,Yi,Xi,rims = resample_with_wcs(thiswcs, twcs) except OverlapError: continue dflux = 10.**((zdepth - 22.5)/-2.5) div = 1./dflux**2 depth[Yo,Xo] += div nexp[Yo,Xo] += 1 # Now also look for entries in the CCDs (exposures) table not previously found. I,J,d = match_radec(exps.ra_bore, exps.dec_bore, tile.ra, tile.dec, radius) print(' ', len(I), 'exposures from CCDs file nearby') if len(I): I = np.array([i for i,expnum,gd in zip(I, exps.expnum[I], exps.galdepth[I]) if (not expnum in matched_exposures) and gd > 0]) print(' ', len(I), 'exposures that were not in tile file') # Drop exposures from this pass, except for previous exposures of this tile! # if len(I): # I = I[np.logical_or(exps.tilepass[I] != 3, # exps.tileid[I] == tile.tileid)] # print(' ', len(I), 'exposures not in pass 3') # if len(I): # print(' objects:', [o.strip() for o in exps.object[I]]) # print(' tileids:', exps.tileid[I]) # print(' expnums:', exps.expnum[I]) # print(' passes:', exps.tilepass[I]) for ii in I: if exps.expnum[ii] in bad_expids: print(' skipping bad exp num', exps.expnum[ii]) continue zdepth = exps.galdepth[ii] for twcs in tilewcs: twcs.set_crval((exps.ra_bore[ii], exps.dec_bore[ii])) try: Yo,Xo,Yi,Xi,rims = resample_with_wcs(thiswcs, twcs) except OverlapError: continue dflux = 10.**((zdepth - 22.5)/-2.5) div = 1./dflux**2 depth[Yo,Xo] += div nexp[Yo,Xo] += 1 # Convert depth map from depth-iv back to mag. # flux with np.errstate(divide='ignore'): dflux = np.sqrt(1./depth) dflux[depth == 0] = 0. depth = -2.5 * (np.log10(dflux) - 9.) depth[dflux == 0] = 0. if depth.max() == 0: print(' Actually no overlap') return None # Extinction correction for this tile... ext_z = 1.211 * tile.ebv_med print(' Applying extinction correction', ext_z, 'mag') depth[depth != 0] -= ext_z #pcts = [0,10,20,30,40,50,60,70,80,90,100] pcts = np.arange(0, 101) depths = np.percentile(depth, pcts) target = 22.5 req_pcts = [0, 2, 2, 5, 5, 10, 10, 100] req_depths = [0, 0, target-0.6, target-0.6, target-0.3, target-0.3, target, target] print(' Depths at 2, 5, and 10th percentile vs target:', '%.2f' % (depths[2] - (target - 0.6)), '%.2f' % (depths[5] - (target - 0.3)), '%.2f' % (depths[10] - target)) return depths
minra = np.min(allra) maxra = np.max(allra) mindec = np.min(alldec) maxdec = np.max(alldec) print('RA,Dec range:', minra, maxra, mindec, maxdec) plothist(allra, alldec) plt.axis([maxra, minra, mindec, maxdec]) plt.xlabel('RA (deg)') plt.ylabel('Dec (deg)') plt.savefig('match-all.png') Tref = fits_table('gaia.fits') r_arcsec = 0.2 I, J, d = match_radec(Tref.ra, Tref.dec, allra, alldec, r_arcsec / 3600.) dec = alldec[J] cosdec = np.cos(np.deg2rad(dec)) dr = (Tref.ra[I] - allra[J]) * cosdec * 3600. dd = (Tref.dec[I] - alldec[J]) * 3600. plt.clf() rr = (-r_arcsec * 1000, +r_arcsec * 1000) plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr)) plt.xlabel('dRA (milli-arcsec)') plt.ylabel('dDec (milli-arcsec)') plt.savefig('match-all-ref-before.png') # Initial matching of all stars r_arcsec = 0.2 I, J, d = match_radec(allra, alldec,
def main(): """Main program. """ import argparse parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).") parser.add_argument('--calibs', action='store_true', help='Output CCDs that need to be calibrated.') parser.add_argument('--nper', type=int, default=None, help='Batch N calibs per line') parser.add_argument('--forced', action='store_true', help='Output forced-photometry commands') parser.add_argument('--lsb', action='store_true', help='Output Low-Surface-Brightness commands') parser.add_argument('--touching', action='store_true', help='Cut to only CCDs touching selected bricks') parser.add_argument('--near', action='store_true', help='Quick cut to only CCDs near selected bricks') parser.add_argument('--check', action='store_true', help='Check which calibrations actually need to run.') parser.add_argument('--check-coadd', action='store_true', help='Check which caoadds actually need to run.') parser.add_argument('--out', help='Output filename for calibs, default %(default)s', default='jobs') parser.add_argument('--command', action='store_true', help='Write out full command-line to run calib') parser.add_argument('--opt', help='With --command, extra options to add') parser.add_argument('--maxdec', type=float, help='Maximum Dec to run') parser.add_argument('--mindec', type=float, help='Minimum Dec to run') parser.add_argument('--region', help='Region to select') parser.add_argument('--bricks', help='Set bricks.fits file to load') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--ignore_cuts', action='store_true',default=False,help='no photometric or blacklist cuts') parser.add_argument('--save_to_fits', action='store_true',default=False,help='save cut brick,ccd to fits table') parser.add_argument('--name', action='store',default='dr3',help='save with this suffix, e.g. refers to ccds table') parser.add_argument('--delete-sky', action='store_true', help='Delete any existing sky calibration files') parser.add_argument('--delete-pvastrom', action='store_true', help='Delete any existing PV WCS calibration files') parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?') parser.add_argument('--brickq', type=int, default=None, help='Queue only bricks with the given "brickq" value [0 to 3]') parser.add_argument('--brickq-deps', action='store_true', default=False, help='Queue bricks directly using qdo API, setting brickq dependencies') parser.add_argument('--queue', default='bricks', help='With --brickq-deps, the QDO queue name to use') opt = parser.parse_args() survey = LegacySurveyData() if opt.bricks is not None: B = fits_table(opt.bricks) log('Read', len(B), 'from', opt.bricks) else: B = survey.get_bricks() if opt.ccds is not None: T = fits_table(opt.ccds) log('Read', len(T), 'from', opt.ccds) else: T = survey.get_ccds() log(len(T), 'CCDs') T.index = np.arange(len(T)) if opt.ignore_cuts == False: I = survey.photometric_ccds(T) print(len(I), 'CCDs are photometric') T.cut(I) I = survey.apply_blacklist(T) print(len(I), 'CCDs are not blacklisted') T.cut(I) print(len(T), 'CCDs remain') # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True) # plt.clf() # plt.hist(counts, counts.max()+1) # plt.savefig('bricks.png') # B.cut(I[counts >= 9]) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # #plt.scatter(B.ra[I], B.dec[I], c=counts) # plt.savefig('bricks2.png') # DES Stripe82 #rlo,rhi = 350.,360. # rlo,rhi = 300., 10. # dlo,dhi = -6., 4. # TINY bit #rlo,rhi = 350.,351.1 #dlo,dhi = 0., 1.1 # EDR+ # 860 bricks # ~10,000 CCDs #rlo,rhi = 239,246 #dlo,dhi = 5, 13 # DR1 #rlo,rhi = 0, 360 # part 1 #dlo,dhi = 25, 40 # part 2 #dlo,dhi = 20,25 # part 3 #dlo,dhi = 15,20 # part 4 #dlo,dhi = 10,15 # part 5 #dlo,dhi = 5,10 # the rest #dlo,dhi = -11, 5 #dlo,dhi = 15,25.5 dlo,dhi = -25, 40 rlo,rhi = 0, 360 # Arjun says 3x3 coverage area is roughly # RA=240-252 DEC=6-12 (but not completely rectangular) # COSMOS #rlo,rhi = 148.9, 151.2 #dlo,dhi = 0.9, 3.5 # A nice well-behaved region (EDR2/3) # rlo,rhi = 243.6, 244.6 # dlo,dhi = 8.1, 8.6 # 56 bricks, ~725 CCDs #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7)) # 240 bricks, ~3000 CCDs #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9)) # 535 bricks, ~7000 CCDs #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12)) if opt.region in ['test1', 'test2', 'test3', 'test4']: nm = dict(test1='2446p115', # weird stuff around bright star test2='1183p292', # faint sources around bright galaxy test3='3503p005', # DES test4='1163p277', # Pollux )[opt.region] B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname]))) log('Cut to', len(B), 'bricks') log(B.ra, B.dec) dlo,dhi = -90,90 rlo,rhi = 0, 360 elif opt.region == 'edr': # EDR: # 535 bricks, ~7000 CCDs rlo,rhi = 240,245 dlo,dhi = 5, 12 elif opt.region == 'edrplus': rlo,rhi = 235,248 dlo,dhi = 5, 15 elif opt.region == 'edr-south': rlo,rhi = 240,245 dlo,dhi = 5, 10 elif opt.region == 'cosmos1': # 16 bricks in the core of the COSMOS field. rlo,rhi = 149.75, 150.75 dlo,dhi = 1.6, 2.6 elif opt.region == 'pristine': # Stream? rlo,rhi = 240,250 dlo,dhi = 10,15 elif opt.region == 'des': dlo, dhi = -6., 4. rlo, rhi = 317., 7. T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage]))) log('Cut to', len(T), 'CCDs with "CPDES82" in filename') elif opt.region == 'subdes': rlo,rhi = 320., 360. dlo,dhi = -1.25, 1.25 elif opt.region == 'northwest': rlo,rhi = 240,360 dlo,dhi = 20,40 elif opt.region == 'north': rlo,rhi = 120,240 dlo,dhi = 20,40 elif opt.region == 'northeast': rlo,rhi = 0,120 dlo,dhi = 20,40 elif opt.region == 'southwest': rlo,rhi = 240,360 dlo,dhi = -20,0 elif opt.region == 'south': rlo,rhi = 120,240 dlo,dhi = -20,0 elif opt.region == 'southeast': rlo,rhi = 0,120 dlo,dhi = -20,0 elif opt.region == 'southsoutheast': rlo,rhi = 0,120 dlo,dhi = -20,-10 elif opt.region == 'midwest': rlo,rhi = 240,360 dlo,dhi = 0,20 elif opt.region == 'middle': rlo,rhi = 120,240 dlo,dhi = 0,20 elif opt.region == 'mideast': rlo,rhi = 0,120 dlo,dhi = 0,20 elif opt.region == 'grz': # Bricks with grz coverage. # Be sure to use --bricks survey-bricks-in-dr1.fits # which has_[grz] columns. B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)) log('Cut to', len(B), 'bricks with grz coverage') elif opt.region == 'nogrz': # Bricks without grz coverage. # Be sure to use --bricks survey-bricks-in-dr1.fits # which has_[grz] columns. B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))) log('Cut to', len(B), 'bricks withOUT grz coverage') elif opt.region == 'deep2': rlo,rhi = 250,260 dlo,dhi = 30,35 elif opt.region == 'deep2f3': rlo,rhi = 351.25, 353.75 dlo,dhi = 0, 0.5 elif opt.region == 'virgo': rlo,rhi = 185,190 dlo,dhi = 10, 15 elif opt.region == 'virgo2': rlo,rhi = 182,192 dlo,dhi = 8, 18 elif opt.region == 'lsb': rlo,rhi = 147.2, 147.8 dlo,dhi = -0.4, 0.4 elif opt.region == 'eboss-elg': # RA -45 to +45 # Dec -5 to +7 rlo,rhi = 315., 45. dlo,dhi = -5., 7. elif opt.region == 'eboss-ngc': # NGC ELGs # RA 115 to 175 # Dec 15 to 30 rlo,rhi = 115., 175. dlo,dhi = 15., 30. elif opt.region == 'mzls': dlo,dhi = 30., 90. elif opt.region == 'dr4-bootes': # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched #dlo,dhi = 34., 35. #rlo,rhi = 209.5, 210.5 dlo,dhi = 33., 36. rlo,rhi = 216.5, 219.5 if opt.mindec is not None: dlo = opt.mindec if opt.maxdec is not None: dhi = opt.maxdec if rlo < rhi: B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) else: # RA wrap B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) log(len(B), 'bricks in range') for name in B.get('brickname'): print(name) B.writeto('bricks-cut.fits') I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs') plt.clf() plt.plot(B.ra, B.dec, 'b.') plt.title('DR3 bricks') plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1]) plt.savefig('bricks.png') if opt.brickq is not None: B.cut(B.brickq == opt.brickq) log('Cut to', len(B), 'with brickq =', opt.brickq) if opt.touching: keep = np.zeros(len(T), bool) for j in J: keep[j] = True T.cut(keep) log('Cut to', len(T), 'CCDs near bricks') # Aside -- how many near DR1=1 CCDs? if False: T2 = D.get_ccds() log(len(T2), 'CCDs') T2.cut(T2.dr1 == 1) log(len(T2), 'CCDs marked DR1=1') log(len(B), 'bricks in range') I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B2 = B[keep] log('Total of', len(B2), 'bricks near CCDs with DR1=1') for band in 'grz': Tb = T2[T2.filter == band] log(len(Tb), 'in filter', band) I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, survey.bricksize) good = np.zeros(len(B2), np.uint8) for i in I: good[i] = 1 B2.set('has_' + band, good) B2.writeto('survey-bricks-in-dr1.fits') sys.exit(0) # sort by dec decreasing #B.cut(np.argsort(-B.dec)) # RA increasing B.cut(np.argsort(B.ra)) for b in B: if opt.check: fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue if opt.check_coadd: fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue print(b.brickname) if opt.save_to_fits: assert(opt.touching) # Write cut tables to file for tab,typ in zip([B,T],['bricks','ccds']): fn='%s-%s-cut.fits' % (typ,opt.name) if os.path.exists(fn): os.remove(fn) tab.writeto(fn) print('Wrote %s' % fn) # Write text files listing ccd and filename names nm1,nm2= 'ccds-%s.txt'% opt.name,'filenames-%s.txt' % opt.name if os.path.exists(nm1): os.remove(nm1) if os.path.exists(nm2): os.remove(nm2) f1,f2=open(nm1,'w'),open(nm2,'w') fns= list(set(T.get('image_filename'))) for fn in fns: f2.write('%s\n' % fn.strip()) for ti in T: f1.write('%s\n' % ti.get('image_filename').strip()) f1.close() f2.close() print('Wrote *-names.txt') if opt.brickq_deps: import qdo from legacypipe.survey import on_bricks_dependencies #... find Queue... q = qdo.connect(opt.queue, create_ok=True) print('Connected to QDO queue', opt.queue, q) brick_to_task = dict() I = survey.photometric_ccds(T) print(len(I), 'CCDs are photometric') T.cut(I) I = survey.apply_blacklist(T) print(len(I), 'CCDs are not blacklisted') T.cut(I) print(len(T), 'CCDs remaining') T.wra = T.ra + (T.ra > 180) * -360 wra = rlo - 360 plt.clf() plt.plot(T.wra, T.dec, 'b.') ax = [wra, rhi, dlo, dhi] plt.axis(ax) plt.title('CCDs') plt.savefig('q-ccds.png') B.wra = B.ra + (B.ra > 180) * -360 # this slight overestimate (for DECam images) is fine radius = 0.3 Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius, indexlist=True) ikeep = [] for ib,(b,Iccd) in enumerate(zip(B, Iccds)): if Iccd is None or len(Iccd) == 0: print('No matched CCDs to brick', b.brickname) continue wcs = wcs_for_brick(b) cI = ccds_touching_wcs(wcs, T[np.array(Iccd)]) print(len(cI), 'CCDs touching brick', b.brickname) if len(cI) == 0: continue ikeep.append(ib) B.cut(np.array(ikeep)) print('Cut to', len(B), 'bricks touched by CCDs') for brickq in range(4): I = np.flatnonzero(B.brickq == brickq) print(len(I), 'bricks with brickq =', brickq) J = np.flatnonzero(B.brickq < brickq) preB = B[J] reqs = [] if brickq > 0: for b in B[I]: # find brick dependencies brickdeps = on_bricks_dependencies(b, survey, bricks=preB) # convert to task ids taskdeps = [brick_to_task.get(b.brickname,None) for b in brickdeps] # If we dropped a dependency brick from a previous brickq because # of no overlapping CCDs, it won't appear in the brick_to_task map. taskdeps = [t for t in taskdeps if t is not None] reqs.append(taskdeps) plt.clf() plt.plot(B.wra, B.dec, '.', color='0.5') plt.plot(B.wra[I], B.dec[I], 'b.') plt.axis(ax) plt.title('Bricks: brickq=%i' % brickq) plt.savefig('q-bricks-%i.png' % brickq) # submit to qdo queue print('Queuing', len(B[I]), 'bricks') if brickq == 0: reqs = None else: assert(len(I) == len(reqs)) taskids = q.add_multiple(B.brickname[I], requires=reqs) assert(len(taskids) == len(I)) print('Queued', len(taskids), 'bricks') brick_to_task.update(dict(zip(B.brickname[I], taskids))) if not (opt.calibs or opt.forced or opt.lsb): sys.exit(0) bands = 'grz' log('Filters:', np.unique(T.filter)) T.cut(np.flatnonzero(np.array([f in bands for f in T.filter]))) log('Cut to', len(T), 'CCDs in filters', bands) if opt.touching: allI = set() for b in B: wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec)) if len(I) == 0: continue allI.update(I) allI = list(allI) allI.sort() elif opt.near: # Roughly brick radius + DECam image radius radius = 0.35 allI,nil,nil = match_radec(T.ra, T.dec, B.ra, B.dec, radius, nearest=True) else: allI = np.arange(len(T)) if opt.write_ccds: T[allI].writeto(opt.write_ccds) log('Wrote', opt.write_ccds) ## Be careful here -- T has been cut; we want to write out T.index. ## 'allI' contains indices into T. if opt.forced: log('Writing forced-photometry commands to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') for j,i in enumerate(allI): expstr = '%08i' % T.expnum[i] outfn = os.path.join('forced', expstr[:5], expstr, 'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i])) imgfn = os.path.join(survey.survey_dir, 'images', T.image_filename[i].strip()) if (not os.path.exists(imgfn) and imgfn.endswith('.fz') and os.path.exists(imgfn[:-3])): imgfn = imgfn[:-3] #f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' % # (imgfn, T.image_hdu[i], outfn)) f.write('python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n' % (T.expnum[i], T.ccdname[i], outfn)) f.close() log('Wrote', opt.out) sys.exit(0) if opt.lsb: log('Writing LSB commands to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') for j,i in enumerate(allI): exp = T.expnum[i] ext = T.ccdname[i].strip() outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext) f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext)) f.close() log('Wrote', opt.out) sys.exit(0) log('Writing calibs to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') batch = [] def write_batch(f, batch, cmd): if cmd is None: cmd = '' f.write(cmd + ' '.join(batch) + '\n') cmd = None if opt.command: cmd = 'python legacypipe/run-calib.py ' if opt.opt is not None: cmd += opt.opt + ' ' for j,i in enumerate(allI): if opt.delete_sky or opt.delete_pvastrom: log(j+1, 'of', len(allI)) im = survey.get_image_object(T[i]) if opt.delete_sky and os.path.exists(im.skyfn): log(' deleting:', im.skyfn) os.unlink(im.skyfn) if opt.delete_pvastrom and os.path.exists(im.pvwcsfn): log(' deleting:', im.pvwcsfn) os.unlink(im.pvwcsfn) if opt.check: log(j+1, 'of', len(allI)) im = survey.get_image_object(T[i]) if not im.run_calibs(im, just_check=True): log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done') continue if opt.command: s = '%i-%s' % (T.expnum[i], T.ccdname[i]) prefix = 'python legacypipe/run-calib.py ' + opt.opt #('python legacypipe/run-calib.py --expnum %i --ccdname %s' % # (T.expnum[i], T.ccdname[i])) else: s = '%i' % T.index[i] prefix = '' if j < 10: print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i], 'filename', T.image_filename[i]) if not opt.nper: f.write(prefix + s + '\n') else: batch.append(s) if len(batch) >= opt.nper: write_batch(f, batch, cmd) batch = [] if opt.check: f.flush() if len(batch): write_batch(f, batch, cmd) f.close() log('Wrote', opt.out) return 0
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('--name1', help='Name for first data set') parser.add_argument('--name2', help='Name for second data set') parser.add_argument('--plot-prefix', default='compare', help='Prefix for plot filenames; default "%default"') parser.add_argument('--match', default=1.0, help='Astrometric cross-match distance in arcsec') parser.add_argument('dir1', help='First directory to compare') parser.add_argument('dir2', help='Second directory to compare') opt = parser.parse_args() ps = PlotSequence(opt.plot_prefix) name1 = opt.name1 if name1 is None: name1 = os.path.basename(opt.dir1) if not len(name1): name1 = os.path.basename(os.path.dirname(opt.dir1)) name2 = opt.name2 if name2 is None: name2 = os.path.basename(opt.dir2) if not len(name2): name2 = os.path.basename(os.path.dirname(opt.dir2)) tt = 'Comparing %s to %s' % (name1, name2) # regex for tractor-*.fits catalog filename catre = re.compile('tractor-.*.fits') cat1, cat2 = [], [] for basedir, cat in [(opt.dir1, cat1), (opt.dir2, cat2)]: for dirpath, dirnames, filenames in os.walk(basedir, followlinks=True): for fn in filenames: if not catre.match(fn): print('Skipping', fn, 'due to filename') continue fn = os.path.join(dirpath, fn) t = fits_table(fn) print(len(t), 'from', fn) cat.append(t) cat1 = merge_tables(cat1, columns='fillzero') cat2 = merge_tables(cat2, columns='fillzero') print('Total of', len(cat1), 'from', name1) print('Total of', len(cat2), 'from', name2) cat1.cut(cat1.brick_primary) cat2.cut(cat2.brick_primary) print('Total of', len(cat1), 'BRICK_PRIMARY from', name1) print('Total of', len(cat2), 'BRICK_PRIMARY from', name2) cat1.cut((cat1.decam_anymask[:, 1] == 0) * (cat1.decam_anymask[:, 2] == 0) * (cat1.decam_anymask[:, 4] == 0)) cat2.cut((cat2.decam_anymask[:, 1] == 0) * (cat2.decam_anymask[:, 2] == 0) * (cat2.decam_anymask[:, 4] == 0)) print('Total of', len(cat1), 'unmasked from', name1) print('Total of', len(cat2), 'unmasked from', name2) I, J, d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match / 3600., nearest=True) print(len(I), 'matched') matched1 = cat1[I] matched2 = cat2[J]
def get_reference_sources(survey, targetwcs, pixscale, bands, tycho_stars=True, gaia_stars=True, large_galaxies=True, star_clusters=True): from legacypipe.survey import GaiaSource from legacypipe.survey import LegacyEllipseWithPriors from tractor import NanoMaggies, RaDecPos from tractor.galaxy import ExpGalaxy from tractor.ellipses import EllipseESoft H, W = targetwcs.shape H, W = int(H), int(W) # How big of a margin to search for bright stars and star clusters -- # this should be based on the maximum radius they are considered to # affect. ref_margin = 0.125 mpix = int(np.ceil(ref_margin * 3600. / pixscale)) marginwcs = targetwcs.get_subimage(-mpix, -mpix, W + 2 * mpix, H + 2 * mpix) refs = [] # Tycho-2 stars if tycho_stars: tycho = read_tycho2(survey, marginwcs) if len(tycho): refs.append(tycho) # Add Gaia stars gaia = None if gaia_stars: from astrometry.libkd.spherematch import match_radec gaia = read_gaia(marginwcs) if gaia is not None: gaia.isbright = (gaia.phot_g_mean_mag < 13.) gaia.ismedium = (gaia.phot_g_mean_mag < 16.) gaia.donotfit = np.zeros(len(gaia), bool) # Handle sources that appear in both Gaia and Tycho-2 by # dropping the entry from Tycho-2. if len(gaia) and len(tycho): # Before matching, apply proper motions to bring them to # the same epoch. We want to use the more-accurate Gaia # proper motions, so rewind Gaia positions to the # approximate epoch of Tycho-2: 1991.5. cosdec = np.cos(np.deg2rad(gaia.dec)) gra = gaia.ra + (1991.5 - gaia.ref_epoch) * gaia.pmra / ( 3600. * 1000.) / cosdec gdec = gaia.dec + (1991.5 - gaia.ref_epoch) * gaia.pmdec / (3600. * 1000.) I, J, _ = match_radec(tycho.ra, tycho.dec, gra, gdec, 1. / 3600., nearest=True) debug('Matched', len(I), 'Tycho-2 stars to Gaia stars.') if len(I): keep = np.ones(len(tycho), bool) keep[I] = False tycho.cut(keep) gaia.isbright[J] = True if gaia is not None and len(gaia) > 0: refs.append(gaia) # Read the catalog of star (open and globular) clusters and add them to the # set of reference stars (with the isbright bit set). if star_clusters: clusters = read_star_clusters(marginwcs) if clusters is not None: debug('Found', len(clusters), 'star clusters nearby') clusters.iscluster = np.ones(len(clusters), bool) refs.append(clusters) # Read large galaxies nearby. if large_galaxies: galaxies = read_large_galaxies(survey, targetwcs) if galaxies is not None: # Resolve possible Gaia-large-galaxy duplicates if gaia and len(gaia): I, J, _ = match_radec(galaxies.ra, galaxies.dec, gaia.ra, gaia.dec, 2. / 3600., nearest=True) print('Matched', len(I), 'large galaxies to Gaia stars.') if len(I): gaia.donotfit[J] = True refs.append(galaxies) refcat = None if len(refs): refs = merge_tables([r for r in refs if r is not None], columns='fillzero') if len(refs) == 0: return None, None refs.radius_pix = np.ceil(refs.radius * 3600. / pixscale).astype(int) ok, xx, yy = targetwcs.radec2pixelxy(refs.ra, refs.dec) # ibx = integer brick coords refs.ibx = np.round(xx - 1.).astype(int) refs.iby = np.round(yy - 1.).astype(int) # cut ones whose position + radius are outside the brick bounds. refs.cut((xx > -refs.radius_pix) * (xx < W + refs.radius_pix) * (yy > -refs.radius_pix) * (yy < H + refs.radius_pix)) # mark ones that are actually inside the brick area. refs.in_bounds = ((refs.ibx >= 0) * (refs.ibx < W) * (refs.iby >= 0) * (refs.iby < H)) for col in [ 'isbright', 'ismedium', 'islargegalaxy', 'iscluster', 'donotfit' ]: if not col in refs.get_columns(): refs.set(col, np.zeros(len(refs), bool)) ## Create Tractor sources from reference stars refcat = [] for g in refs: if g.donotfit or g.iscluster: refcat.append(None) elif g.islargegalaxy: fluxes = dict([(band, NanoMaggies.magToNanomaggies(g.mag)) for band in bands]) assert (np.all(np.isfinite(list(fluxes.values())))) rr = g.radius * 3600. / 0.5 # factor of two accounts for R(25)-->reff pa = 180 - g.pa if not np.isfinite(pa): pa = 0. logr, ee1, ee2 = EllipseESoft.rAbPhiToESoft(rr, g.ba, pa) gal = ExpGalaxy(RaDecPos(g.ra, g.dec), NanoMaggies(order=bands, **fluxes), LegacyEllipseWithPriors(logr, ee1, ee2)) refcat.append(gal) else: # Gaia star -- which we want to create a source for, regardless of # whether it is marked medium | bright (or neither). refcat.append(GaiaSource.from_catalog(g, bands)) for src in refcat: if src: src.is_reference_source = True return refs, refcat
splinesky=True, coadd_bw=True, forceAll=True, writePickles=False) print('Reading', outfn) cat = fits_table(outfn) primhdr = fitsio.read_header(outfn) iband = survey.index_of_band(im.band) cat.flux = cat.decam_flux[:, iband] cat.apflux = cat.decam_apflux[:, iband, :] cat.mag = -2.5 * (np.log10(cat.flux) - 9.) cat.apmag = -2.5 * (np.log10(cat.apflux) - 9.) I,J,d = match_radec(stars.ra, stars.dec, cat.ra, cat.dec, 1./3600., nearest=True) print(len(I), 'matches to PS1 stars') colorterm = ps1_to_decam(stars.median, im.band) ipsband = ps1cat.ps1band[im.band] stars.mag = stars.median[:, ipsband] + colorterm stars.flux = 10.**((stars.mag - 22.5) / -2.5) mstars = stars[I] mcat = cat[J] print(np.sum(mcat.type == 'PSF '), 'matched sources are PSF') mcat.ispsf = (mcat.type == 'PSF ') mcat.unmasked = (np.sum(mcat.decam_anymask, axis=1) == 0) P = np.flatnonzero(mcat.ispsf * mcat.unmasked) print(len(P), 'matched, unmasked, PSFs')
def read_forcedphot_ccds(ccds, survey): ccds.mdiff = np.zeros(len(ccds)) ccds.mscatter = np.zeros(len(ccds)) Nap = 8 ccds.apdiff = np.zeros((len(ccds), Nap)) ccds.apscatter = np.zeros((len(ccds), Nap)) ccds.nforced = np.zeros(len(ccds), np.int16) ccds.nunmasked = np.zeros(len(ccds), np.int16) ccds.nmatched = np.zeros(len(ccds), np.int16) ccds.nps1 = np.zeros(len(ccds), np.int16) brickcache = {} FF = [] for iccd,ccd in enumerate(ccds): print('CCD', iccd, 'of', len(ccds)) F = fits_table(ccd.path) print(len(F), 'sources in', ccd.path) ccds.nforced[iccd] = len(F) # arr, have to match with brick sources to get RA,Dec. F.ra = np.zeros(len(F)) F.dec = np.zeros(len(F)) F.masked = np.zeros(len(F), bool) maglo,maghi = 14.,21. maxdmag = 1. F.mag = -2.5 * (np.log10(F.flux) - 9) F.cut((F.flux > 0) * (F.mag > maglo-maxdmag) * (F.mag < maghi+maxdmag)) print(len(F), 'sources between', (maglo-maxdmag), 'and', (maghi+maxdmag), 'mag') im = survey.get_image_object(ccd) print('Reading DQ image for', im) dq = im.read_dq() H,W = dq.shape ix = np.clip(np.round(F.x), 0, W-1).astype(int) iy = np.clip(np.round(F.y), 0, H-1).astype(int) F.mask = dq[iy,ix] print(np.sum(F.mask != 0), 'sources are masked') for brickname in np.unique(F.brickname): if not brickname in brickcache: brickcache[brickname] = fits_table(survey.find_file('tractor', brick=brickname)) T = brickcache[brickname] idtoindex = np.zeros(T.objid.max()+1, int) - 1 idtoindex[T.objid] = np.arange(len(T)) I = np.flatnonzero(F.brickname == brickname) J = idtoindex[F.objid[I]] assert(np.all(J >= 0)) F.ra [I] = T.ra [J] F.dec[I] = T.dec[J] F.masked[I] = (T.decam_anymask[J,:].max(axis=1) > 0) #F.cut(F.masked == False) #print(len(F), 'not masked') print(np.sum(F.masked), 'masked in ANYMASK') ccds.nunmasked[iccd] = len(F) wcs = Tan(*[float(x) for x in [ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2, ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]]) ps1 = ps1cat(ccdwcs=wcs) stars = ps1.get_stars() print(len(stars), 'PS1 sources') ccds.nps1[iccd] = len(stars) # Now cut to just *stars* with good colors stars.gicolor = stars.median[:,0] - stars.median[:,2] keep = (stars.gicolor > 0.4) * (stars.gicolor < 2.7) stars.cut(keep) print(len(stars), 'PS1 stars with good colors') stars.cut(np.minimum(stars.stdev[:,1], stars.stdev[:,2]) < 0.05) print(len(stars), 'PS1 stars with min stdev(r,i) < 0.05') I,J,d = match_radec(F.ra, F.dec, stars.ra, stars.dec, 1./3600.) print(len(I), 'matches') band = ccd.filter colorterm = ps1_to_decam(stars.median[J], band) F.cut(I) F.psmag = stars.median[J, ps1.ps1band[band]] + colorterm K = np.flatnonzero((F.psmag > maglo) * (F.psmag < maghi)) print(len(K), 'with mag', maglo, 'to', maghi) F.cut(K) K = np.flatnonzero(np.abs(F.mag - F.psmag) < maxdmag) print(len(K), 'with good mag matches (<', maxdmag, 'mag difference)') ccds.nmatched[iccd] = len(K) if len(K) == 0: continue F.cut(K) ccds.mdiff[iccd] = np.median(F.mag - F.psmag) ccds.mscatter[iccd] = (np.percentile(F.mag - F.psmag, 84) - np.percentile(F.mag - F.psmag, 16))/2. for i in range(Nap): apmag = -2.5 * (np.log10(F.apflux[:, i]) - 9) ccds.apdiff[iccd,i] = np.median(apmag - F.psmag) ccds.apscatter[iccd,i] = (np.percentile(apmag - F.psmag, 84) - np.percentile(apmag - F.psmag, 16))/2. #F.about() for c in ['apflux_ivar', 'brickid', 'flux_ivar', 'mjd', 'objid', 'fracflux', 'rchi2', 'x','y']: F.delete_column(c) F.expnum = np.zeros(len(F), np.int32) + ccd.expnum F.ccdname = np.array([ccd.ccdname] * len(F)) F.iforced = np.zeros(len(F), np.int32) + iccd FF.append(F) FF = merge_tables(FF) return FF bricks = survey.get_bricks_readonly() bricks = bricks[(bricks.ra > ralo) * (bricks.ra < rahi) * (bricks.dec > declo) * (bricks.dec < dechi)] print(len(bricks), 'bricks') I, = np.nonzero([os.path.exists(survey.find_file('tractor', brick=b.brickname)) for b in bricks]) print(len(I), 'bricks with catalogs') bricks.cut(I) for band in bands: bricks.set('diff_%s' % band, np.zeros(len(bricks), np.float32)) bricks.set('psfsize_%s' % band, np.zeros(len(bricks), np.float32)) diffs = dict([(b,[]) for b in bands]) for ibrick,b in enumerate(bricks): fn = survey.find_file('tractor', brick=b.brickname) T = fits_table(fn) print(len(T), 'sources in', b.brickname) brickwcs = wcs_for_brick(b) ps1 = ps1cat(ccdwcs=brickwcs) stars = ps1.get_stars() print(len(stars), 'PS1 sources') # Now cut to just *stars* with good colors stars.gicolor = stars.median[:,0] - stars.median[:,2] keep = (stars.gicolor > 0.4) * (stars.gicolor < 2.7) stars.cut(keep) print(len(stars), 'PS1 stars with good colors') I,J,d = match_radec(T.ra, T.dec, stars.ra, stars.dec, 1./3600.) print(len(I), 'matches') for band in bands: bricks.get('psfsize_%s' % band)[ibrick] = np.median( T.decam_psfsize[:, survey.index_of_band(band)]) colorterm = ps1_to_decam(stars.median[J], band) psmag = stars.median[J, ps1.ps1band[band]] psmag += colorterm decflux = T.decam_flux[I, survey.index_of_band(band)] decmag = -2.5 * (np.log10(decflux) - 9) #K = np.flatnonzero((psmag > 14) * (psmag < 24)) #print(len(K), 'with mag 14 to 24') K = np.flatnonzero((psmag > 14) * (psmag < 21)) print(len(K), 'with mag 14 to 21') decmag = decmag[K] psmag = psmag [K] K = np.flatnonzero(np.abs(decmag - psmag) < 1) print(len(K), 'with good mag matches (< 1 mag difference)') decmag = decmag[K] psmag = psmag [K] if False and ibrick == 0: plt.clf() #plt.plot(psmag, decmag, 'b.') plt.plot(psmag, decmag - psmag, 'b.') plt.xlabel('PS1 mag') plt.xlabel('DECam - PS1 mag') plt.title('PS1 matches for %s band, brick %s' % (band, b.brickname)) ps.savefig() mdiff = np.median(decmag - psmag) diffs[band].append(mdiff) print('Median difference:', mdiff) bricks.get('diff_%s' % band)[ibrick] = mdiff for band in bands: d = diffs[band] plt.clf() plt.hist(d, bins=20, range=(-0.02, 0.02), histtype='step') plt.xlabel('Median mag difference per brick') plt.title('DR3 EDR PS1 vs DECaLS: %s band' % band) ps.savefig() print('Median differences in', band, 'band:', np.median(d)) if False: plt.clf() plt.hist(diffs['g'], bins=20, range=(-0.02, 0.02), histtype='step', color='g') plt.hist(diffs['r'], bins=20, range=(-0.02, 0.02), histtype='step', color='r') plt.hist(diffs['z'], bins=20, range=(-0.02, 0.02), histtype='step', color='m') plt.xlabel('Median mag difference per brick') plt.title('DR3 EDR PS1 vs DECaLS') ps.savefig() rr,dd = np.meshgrid(np.linspace(ralo,rahi, 400), np.linspace(declo,dechi, 400)) I,J,d = match_radec(rr.ravel(), dd.ravel(), bricks.ra, bricks.dec, 0.18, nearest=True) print(len(I), 'matches') for band in bands: plt.clf() dmag = np.zeros_like(rr) - 1. dmag.ravel()[I] = bricks.get('diff_%s' % band)[J] plt.imshow(dmag, interpolation='nearest', origin='lower', vmin=-0.01, vmax=0.01, cmap='hot', extent=(ralo,rahi,declo,dechi)) plt.colorbar() plt.title('DR3 EDR PS1 vs DECaLS: %s band' % band) plt.xlabel('RA (deg)') plt.ylabel('Dec (deg)') plt.axis([ralo,rahi,declo,dechi]) ps.savefig() plt.clf() # reuse 'dmag' map... dmag = np.zeros_like(rr) dmag.ravel()[I] = bricks.get('psfsize_%s' % band)[J] plt.imshow(dmag, interpolation='nearest', origin='lower', cmap='hot', extent=(ralo,rahi,declo,dechi)) plt.colorbar() plt.title('DR3 EDR: DECaLS PSF size: %s band' % band) plt.xlabel('RA (deg)') plt.ylabel('Dec (deg)') plt.axis([ralo,rahi,declo,dechi]) ps.savefig() if False: for band in bands: plt.clf() plt.scatter(bricks.ra, bricks.dec, c=bricks.get('diff_%s' % band), vmin=-0.01, vmax=0.01, edgecolors='face', s=200) plt.colorbar() plt.title('DR3 EDR PS1 vs DECaLS: %s band' % band) plt.xlabel('RA (deg)') plt.ylabel('Dec (deg)') plt.axis('scaled') plt.axis([ralo,rahi,declo,dechi]) ps.savefig() plt.clf() plt.plot(bricks.psfsize_g, bricks.diff_g, 'g.') plt.plot(bricks.psfsize_r, bricks.diff_r, 'r.') plt.plot(bricks.psfsize_z, bricks.diff_z, 'm.') plt.xlabel('PSF size (arcsec)') plt.ylabel('DECaLS PSF - PS1 (mag)') plt.title('DR3 EDR') ps.savefig()
def plot_light_curves(pfn, ucal=False): lightcurves = unpickle_from_file(pfn) if ucal: tag = 'ucal-' else: tag = '' survey = LegacySurveyData() brickname = '0364m042' catfn = survey.find_file('tractor', brick=brickname) print('Reading catalog from', catfn) cat = fits_table(catfn) print(len(cat), 'catalog entries') cat.cut(cat.brick_primary) print(len(cat), 'brick primary') I = [] for i, oid in enumerate(cat.objid): if (brickname, oid) in lightcurves: I.append(i) I = np.array(I) cat.cut(I) print('Cut to', len(cat), 'with light curves') S = fits_table('specObj-dr12-trim-2.fits') from astrometry.libkd.spherematch import match_radec I, J, d = match_radec(S.ra, S.dec, cat.ra, cat.dec, 2. / 3600.) print('Matched', len(I), 'to spectra') plt.subplots_adjust(hspace=0) movie_jpegs = [] movie_wcs = None for i in range(28): fn = os.path.join('des-sn-movie', 'epoch%i' % i, 'coadd', brickname[:3], brickname, 'legacysurvey-%s-image.jpg' % brickname) print(fn) if not os.path.exists(fn): continue img = plt.imread(fn) img = np.flipud(img) h, w, d = img.shape fn = os.path.join('des-sn-movie', 'epoch%i' % i, 'coadd', brickname[:3], brickname, 'legacysurvey-%s-image-r.fits' % brickname) if not os.path.exists(fn): continue wcs = Tan(fn) movie_jpegs.append(img) movie_wcs = wcs plt.figure(figsize=(8, 6), dpi=100) n = 0 fluxtags = [('flux', 'flux_ivar', '', 'a')] if ucal: fluxtags.append(('uflux', 'uflux_ivar', ': ucal', 'b')) for oid, ii in zip(cat.objid[J], I): print('Objid', oid) spec = S[ii] k = (brickname, oid) v = lightcurves[k] # Cut bad CCDs v.cut(np.array([e not in [230151, 230152, 230153] for e in v.expnum])) plt.clf() print('obj', k, 'has', len(v), 'measurements') T = v for fluxtag, fluxivtag, fluxname, plottag in fluxtags: plt.clf() filts = np.unique(T.filter) for i, f in enumerate(filts): from tractor.brightness import NanoMaggies plt.subplot(len(filts), 1, i + 1) fluxes = np.hstack( [T.get(ft[0])[T.filter == f] for ft in fluxtags]) fluxes = fluxes[np.isfinite(fluxes)] mn, mx = np.percentile(fluxes, [5, 95]) print('Flux percentiles for filter', f, ':', mn, mx) # note swap mn, mx = NanoMaggies.nanomaggiesToMag( mx), NanoMaggies.nanomaggiesToMag(mn) print('-> mags', mn, mx) cut = (T.filter == f) * (T.flux_ivar > 0) if ucal: cut *= np.isfinite(T.uflux) I = np.flatnonzero(cut) print(' ', len(I), 'in', f, 'band') I = I[np.argsort(T.mjd[I])] mediv = np.median(T.flux_ivar[I]) # cut really noisy ones I = I[T.flux_ivar[I] > 0.25 * mediv] #plt.plot(T.mjd[I], T.flux[I], '.-', color=dict(g='g',r='r',z='m')[f]) # plt.errorbar(T.mjd[I], T.flux[I], yerr=1/np.sqrt(T.fluxiv[I]), # fmt='.-', color=dict(g='g',r='r',z='m')[f]) #plt.errorbar(T.mjd[I], T.flux[I], yerr=1/np.sqrt(T.fluxiv[I]), # fmt='.', color=dict(g='g',r='r',z='m')[f]) # if ucal: # mag,dmag = NanoMaggies.fluxErrorsToMagErrors(T.flux[I], T.flux_ivar[I]) # else: # mag,dmag = NanoMaggies.fluxErrorsToMagErrors(T.uflux[I], T.uflux_ivar[I]) mag, dmag = NanoMaggies.fluxErrorsToMagErrors( T.get(fluxtag)[I], T.get(fluxivtag)[I]) plt.errorbar(T.mjd[I], mag, yerr=dmag, fmt='.', color=dict(g='g', r='r', z='m')[f]) #yl,yh = plt.ylim() #plt.ylim(yh,yl) plt.ylim(mx, mn) plt.ylabel(f) if i + 1 < len(filts): plt.xticks([]) #plt.yscale('symlog') outfn = 'cutout_%.4f_%.4f.jpg' % (spec.ra, spec.dec) if not os.path.exists(outfn): url = 'http://legacysurvey.org/viewer/jpeg-cutout/?ra=%.4f&dec=%.4f&zoom=14&layer=sdssco&size=128' % ( spec.ra, spec.dec) cmd = 'wget -O %s "%s"' % (outfn, url) print(cmd) os.system(cmd) pix = plt.imread(outfn) h, w, d = pix.shape fig = plt.gcf() #print('fig bbox:', fig.bbox) #print('xmax, ymax', fig.bbox.xmax, fig.bbox.ymax) #plt.figimage(pix, 0, fig.bbox.ymax - h, zorder=10) #plt.figimage(pix, 0, fig.bbox.ymax, zorder=10) #plt.figimage(pix, fig.bbox.xmax - w, fig.bbox.ymax, zorder=10) plt.figimage(pix, fig.bbox.xmax - (w + 2), fig.bbox.ymax - (h + 2), zorder=10) plt.suptitle('SDSS spectro object: %s at (%.4f, %.4f)%s' % (spec.label.strip(), spec.ra, spec.dec, fluxname)) plt.savefig('forced-%s%i-%s.png' % (tag, n, plottag)) ok, x, y = movie_wcs.radec2pixelxy(spec.ra, spec.dec) x = int(np.round(x - 1)) y = int(np.round(y - 1)) sz = 32 plt.clf() plt.subplots_adjust(hspace=0, wspace=0) k = 1 for i, img in enumerate(movie_jpegs): stamp = img[y - sz:y + sz + 1, x - sz:x + sz + 1] plt.subplot(5, 6, k) plt.imshow(stamp, interpolation='nearest', origin='lower') plt.xticks([]) plt.yticks([]) k += 1 plt.suptitle('SDSS spectro object: %s at (%.4f, %.4f): DES images' % (spec.label.strip(), spec.ra, spec.dec)) plt.savefig('forced-%s%i-c.png' % (tag, n)) n += 1
def main(): parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='DECaLS simulations.') parser.add_argument('-b', '--brick', type=str, default='2428p117', metavar='', help='process this brick (required input)') parser.add_argument('-o', '--objtype', type=str, default='ELG', metavar='', help='object type (STAR, ELG, LRG, BGS)') parser.add_argument('-v', '--verbose', action='store_true', help='toggle on verbose output') args = parser.parse_args() if args.brick is None: parser.print_help() sys.exit(1) # Set the debugging level if args.verbose: lvl = logging.DEBUG else: lvl = logging.INFO logging.basicConfig(format='%(message)s',level=lvl,stream=sys.stdout) log = logging.getLogger('__name__') brickname = args.brick objtype = args.objtype.upper() lobjtype = objtype.lower() log.info('Analyzing objtype {} on brick {}'.format(objtype,brickname)) if 'DECALS_SIM_DIR' in os.environ: decals_sim_dir = os.getenv('DECALS_SIM_DIR') else: decals_sim_dir = '.' # Plotting preferences sns.set(style='white',font_scale=1.6,palette='dark')#,font='fantasy') col = sns.color_palette('dark') # Read the meta-catalog. metafile = os.path.join(decals_sim_dir,brickname,'metacat-'+brickname+'-'+lobjtype+'.fits') log.info('Reading {}'.format(metafile)) meta = fits.getdata(metafile,1) # We need this for our histograms below magbinsz = 0.2 rminmax = np.array(meta['rmag_range'][0],meta['rmag_range'][1]) nmagbin = long((rminmax[1]-rminmax[0])/magbinsz) # Work in chunks. nchunk = meta['nchunk'] for ichunk in range(nchunk): log.info('Working on chunk {:02d}/{:02d}'.format(ichunk+1,nchunk)) chunksuffix = '{:02d}'.format(ichunk) # Read the simulated object catalog simcatfile = os.path.join(decals_sim_dir,brickname,'simcat-'+brickname+'-'+ lobjtype+'-'+chunksuffix+'.fits') #log.info('Reading {}'.format(simcatfile)) simcat = fits.getdata(simcatfile, 1) # Read and match to the Tractor catalog tractorfile = os.path.join(decals_sim_dir,brickname,'tractor-'+brickname+'-'+ lobjtype+'-'+chunksuffix+'.fits') #log.info('Reading {}'.format(tractorfile)) tractor = fits.getdata(tractorfile, 1) m1, m2, d12 = match_radec(tractor['ra'],tractor['dec'], simcat['ra'],simcat['dec'],1.0/3600.0) missing = np.delete(np.arange(len(simcat)),m2,axis=0) good = np.where((np.abs(tractor['decam_flux'][m1,2]/simcat['rflux'][m2]-1)<0.3)*1) # Flux residuals vs r-band magnitude rmag = simcat['r'][m2] gflux_sim = simcat['gflux'][m2] rflux_sim = simcat['rflux'][m2] zflux_sim = simcat['zflux'][m2] gflux_tra = tractor['decam_flux'][m1,1] rflux_tra = tractor['decam_flux'][m1,2] zflux_tra = tractor['decam_flux'][m1,4] fig, ax = plt.subplots(3, sharex=True, figsize=(6,8)) ax[0].scatter(rmag,gflux_sim/gflux_tra-1,color=col[0],s=10) ax[1].scatter(rmag,rflux_sim/rflux_tra-1,color=col[1],s=10) ax[2].scatter(rmag,zflux_sim/zflux_tra-1,color=col[2],s=10) [thisax.set_ylim(-0.7,0.7) for thisax in ax] [thisax.set_xlim(rminmax+[-0.1,0.0]) for thisax in ax] [thisax.axhline(y=0.0,lw=2,ls='solid',color='gray') for thisax in ax] for ix, thisband in enumerate(['g','r','z']): ax[ix].text(0.05,0.05,thisband,horizontalalignment='left', verticalalignment='bottom',transform=ax[ix].transAxes, fontsize=16) ax[1].set_ylabel('Input Flux / Tractor Flux - 1') ax[2].set_xlabel('Input r magnitude (AB mag)') fig.subplots_adjust(left=0.18,hspace=0.1) qafile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+ lobjtype+'-flux.png') log.info('Writing {}'.format(qafile)) plt.savefig(qafile) # Color residuals gr_tra = -2.5*np.log10(gflux_tra/rflux_tra) rz_tra = -2.5*np.log10(rflux_tra/zflux_tra) gr_sim = -2.5*np.log10(gflux_sim/rflux_sim) rz_sim = -2.5*np.log10(rflux_sim/zflux_sim) fig, ax = plt.subplots(2,sharex=True,figsize=(6,8)) ax[0].scatter(rmag,gr_tra-gr_sim,color=col[0],s=10) ax[1].scatter(rmag,rz_tra-rz_sim,color=col[1],s=10) [thisax.set_ylim(-0.7,0.7) for thisax in ax] [thisax.set_xlim(rminmax+[-0.1,0.0]) for thisax in ax] [thisax.axhline(y=0.0,lw=2,ls='solid',color='gray') for thisax in ax] ax[0].set_ylabel('$\Delta$(g - r) (Tractor minus Input)') ax[1].set_ylabel('$\Delta$(r - z) (Tractor minus Input)') ax[1].set_xlabel('Input r magnitude (AB mag)') fig.subplots_adjust(left=0.18,hspace=0.1) qafile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+ lobjtype+'-color.png') log.info('Writing {}'.format(qafile)) plt.savefig(qafile) sys.exit(1) # Get cutouts of the missing sources imfile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+lobjtype+ '-image-'+chunksuffix+'.jpg') hw = 30 # half-width [pixels] ncols = 5 nrows = 5 nthumb = ncols*nrows dims = (ncols*hw*2,nrows*hw*2) mosaic = Image.new('RGB',dims) miss = missing[np.argsort(simcat['r'][missing])] print(simcat['r'][miss]) xpos, ypos = np.meshgrid(np.arange(0,dims[0],hw*2,dtype='int'), np.arange(0,dims[1],hw*2,dtype='int')) im = Image.open(imfile) sz = im.size iobj = 0 for ic in range(ncols): for ir in range(nrows): mm = miss[iobj] xx = int(simcat['X'][mm]) yy = int(sz[1]-simcat['Y'][mm]) crop = (xx-hw,yy-hw,xx+hw,yy+hw) box = (xpos[ir,ic],ypos[ir,ic]) thumb = im.crop(crop) mosaic.paste(thumb,box) iobj = iobj+1 # Add a border draw = ImageDraw.Draw(mosaic) for ic in range(ncols): for ir in range(nrows): draw.rectangle([(xpos[ir,ic],ypos[ir,ic]), (xpos[ir,ic]+hw*2,ypos[ir,ic]+hw*2)]) qafile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+lobjtype+'-missing.png') log.info('Writing {}'.format(qafile)) mosaic.save(qafile) # Modify the coadd image and residual files so the simulated sources # are labeled. rad = 15 imfile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+lobjtype+ '-image-'+chunksuffix+'.jpg') imfile = [imfile,imfile.replace('-image','-resid')] for ifile in imfile: im = Image.open(ifile) sz = im.size draw = ImageDraw.Draw(im) [draw.ellipse((cat['X']-rad, sz[1]-cat['Y']-rad,cat['X']+rad, sz[1]-cat['Y']+rad)) for cat in simcat] im.save(ifile) # Fraction of matching sources rmaghist, magbins = np.histogram(simcat['r'],bins=nmagbin,range=rminmax) cmagbins = (magbins[:-1] + magbins[1:]) / 2.0 ymatch, binsmatch = np.histogram(simcat['r'][m2],bins=nmagbin,range=rminmax) ymatchgood, binsgood = np.histogram(simcat['r'][m2[good]],bins=nmagbin,range=rminmax) fig, ax = plt.subplots(1,figsize=(8,6)) ax.step(cmagbins,1.0*ymatch/rmaghist,lw=3,alpha=0.5,label='All objects') ax.step(cmagbins,1.0*ymatchgood/rmaghist,lw=3,ls='dashed',label='|$\Delta$m|<0.3') ax.axhline(y=1.0,lw=2,ls='dashed',color='gray') ax.set_xlabel('Input r magnitude (AB mag)') ax.set_ylabel('Fraction of Matching '+objtype+'s') ax.set_ylim([0.0,1.1]) ax.legend(loc='lower left') fig.subplots_adjust(bottom=0.15) qafile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+lobjtype+'-frac.png') log.info('Writing {}'.format(qafile)) plt.savefig(qafile) # Distribution of object types fig = plt.figure(figsize=(8,6)) ax = fig.gca() rmaghist, magbins = np.histogram(simcat['r'][m2],bins=nmagbin,range=rminmax) cmagbins = (magbins[:-1] + magbins[1:]) / 2.0 tractortype = tractor['TYPE'][m1].strip() for otype in ['PSF','EXP','DEV','COMP']: these = np.where(tractortype==otype)[0] if len(these)>0: yobj, binsobj = np.histogram(simcat['r'][m2[these]],bins=nmagbin,range=rminmax) #plt.step(cmagbins,1.0*yobj,lw=3,alpha=0.5,label=otype) plt.step(cmagbins,1.0*yobj/rmaghist,lw=3,alpha=0.5,label=otype) plt.axhline(y=1.0,lw=2,ls='dashed',color='gray') plt.xlabel('Input r magnitude (AB mag)') #plt.ylabel('Number of Objects') plt.ylabel('Fraction of '+objtype+'s classified') plt.ylim([0.0,1.1]) plt.legend(loc='center left',bbox_to_anchor=(0.08,0.5)) fig.subplots_adjust(bottom=0.15) qafile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+lobjtype+'-type.png') log.info('Writing {}'.format(qafile)) plt.savefig(qafile) # Morphology plots if objtype=='ELGo': fig = plt.figure(figsize=(8,4)) plt.subplot(1,3,1) plt.plot(rmag,deltam,'s',markersize=3) plt.axhline(y=0.0,lw=2,ls='solid',color='gray') plt.xlim(rminmax) plt.xlabel('r (AB mag)') plt.subplot(1,3,2) plt.plot(simcat['R50_1'][m2],deltam,'s',markersize=3) plt.axhline(y=0.0,lw=2,ls='solid',color='gray') plt.xlabel('$r_{50}$ (arcsec)') plt.subplot(1,3,3) plt.plot(simcat['BA_1'][m2],deltam,'s',markersize=3) plt.axhline(y=0.0,lw=2,ls='solid',color='gray') plt.xlabel('b/a') plt.xlim([0.2,1.0]) fig.subplots_adjust(bottom=0.18) qafile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+ lobjtype+'-morph.png') log.info('Writing {}'.format(qafile)) plt.savefig(qafile)
def qsocuts(SW): in1 = ( ((SW.gpsf - SW.ipsf) < 1.5) * (SW.optpsf > 17.) * (SW.optpsf < 22.) * ((SW.optmod - SW.wise) > ((SW.gpsf - SW.ipsf) + 3)) * np.logical_or(SW.ispsf, (SW.optpsf - SW.optmod) < 0.1) ) I = np.flatnonzero(in1) print('Selected', len(I)) in2 = in1 * (SW.w1mag < 25.) * (SW.w2mag < 25.) I2 = np.flatnonzero(in2) print('With w1,w2 < 25:', len(I2)) SW.w1rchi2 = SW.w1_prochi2 / SW.w1_pronpix in3 = in2 * (SW.w1rchi2 < 10.) I3 = np.flatnonzero(in3) print('And chi2/pix < 10:', len(I3)) I = I2 # Check against WISE catalog #wfn = 'w3-wise.fits' #WC = fits_table(wfn) # SDSS matched to WISE catalog swfn = 'eboss-w3-wise-cat-dr9.fits' #SWC = fits_table(swfn) S = fits_table('objs-eboss-w3-dr9.fits') print('Read', len(S), 'SDSS objects') wfn = 'w3-wise.fits' WC = fits_table(wfn) print('Read', len(WC), 'WISE catalog objects') R = 4./3600. I,J,d = match_radec(S.ra, S.dec, WC.ra, WC.dec, R, nearest=True) print(len(I), 'matches of SDSS to WISE catalog') SWC = S[I] for k in WC.columns(): if k in ['ra','dec']: outkey = k+'_wise' else: outkey = k X = WC.get(k) SWC.set(outkey, X[J]) SWC.writeto(swfn) SWC.gpsf = fluxtomag(SWC.psfflux[:,1]) SWC.rpsf = fluxtomag(SWC.psfflux[:,2]) SWC.ipsf = fluxtomag(SWC.psfflux[:,3]) SWC.ispsf = (SWC.objc_type == 6) SWC.isgal = (SWC.objc_type == 3) SWC.optpsf = fluxtomag((SWC.psfflux[:,1] * 0.8 + SWC.psfflux[:,2] * 0.6 + SWC.psfflux[:,3] * 1.0) / 2.4) SWC.optmod = fluxtomag((SWC.modelflux[:,1] * 0.8 + SWC.modelflux[:,2] * 0.6 + SWC.modelflux[:,3] * 1.0) / 2.4) SWC.wise = fluxtomag((SWC.w1mpro * 1.0 + SWC.w2mpro * 0.5) / 1.5) in1 = ( ((SWC.gpsf - SWC.ipsf) < 1.5) * (SWC.optpsf > 17.) * (SWC.optpsf < 22.) * ((SWC.optpsf - SWC.wise) > ((SWC.gpsf - SWC.ipsf) + 3)) * np.logical_or(SWC.ispsf, (SWC.optpsf - SWC.optmod) < 0.1) ) I = np.flatnonzero(in1) print('Selected', len(I), 'from WISE catalog') sys.exit(0) worstI = I[np.argsort(-SW.w1rchi2[I])] print('Worst:') mp = multiproc(1) for wi,i in enumerate(worstI): print(' %8.3f, %8.3f, chi2/npix %g' % (SW.ra[i], SW.dec[i], SW.w1rchi2[i])) ra, dec = SW.ra[i], SW.dec[i] ri = int((ra - r0) / (rr[1]-rr[0])) di = int((dec - d0) / (dd[1]-dd[0])) print(' ri %i, di %i' % (ri,di)) if SW.w1rchi2[i] > 25: continue opt = myopts() wbasefn = 'eboss-w3-worst%04i-r%02i-d%02i' % (wi, ri,di) opt.picklepat = '%s-stage%%0i.pickle' % wbasefn opt.ps = wbasefn opt.minflux = None opt.bandnum = 1 opt.osources = None opt.sources = 'objs-eboss-w3-dr9.fits' opt.ptsrc = False opt.pixpsf = False # opt.minsb = 0.05 opt.minsb = 0.005 opt.write = True opt.force = [205] opt.ri = ri opt.di = di import wise3 import tractor pcat = [] pcat.append(tractor.PointSource(RaDecPos(ra, dec), None)) R = wise3.runtostage(205, opt, mp, rr[ri],rr[ri+1],dd[di],dd[di+1], ttsuf='chi2/npix %g' % SW.w1rchi2[i], pcat=pcat, addSky=True) ### Try to use the stored solved values -- actually doesn't make things ### much faster. # R = wise3.runtostage(103, opt, mp, rr[ri],rr[ri+1],dd[di],dd[di+1], # ttsuf='chi2/npix %g' % SW.w1rchi2[i], # pcat=pcat) # t = R['tractor'] # T = fits_table('ebossw3-v4-r%02i-d%02i-w1.fits' % (ri, di)) # assert(len(t.catalog) == len(T)) # for i,src in enumerate(t.catalog): # print('Source', src) # assert(src.getPosition().ra == T.ra [i]) # assert(src.getPosition().dec == T.dec[i]) # t.catalog.freezeParamsRecursive('*') # t.catalog.thawPathsTo('w1') # assert(len(t.catalog.getParams()) == len(T)) # t.catalog.setParams(T.w1) # R2 = wise3.stage204(opt=opt, mp=mp, ri=opt.ri, di=opt.di, **R) # R2['ims1'] = R2['ims0'] # ps = PlotSequence(wbasefn) # R3 = wise3.stage205(opt=opt, mp=mp, ri=opt.ri, di=opt.di, ps=ps, **R2) plt.clf() plt.hist(SW.w1mag, 100, range=(10,30), histtype='step', color='b', log=True) plt.hist(SW.w1mag[I], 100, range=(10,30), histtype='step', color='r', log=True) plt.xlabel('W1 mag') ylo,yhi = plt.ylim() plt.ylim(0.3, yhi) ps.savefig() plt.clf() plt.hist(SW.w2mag, 100, range=(10,30), histtype='step', color='b', log=True) plt.hist(SW.w2mag[I], 100, range=(10,30), histtype='step', color='r', log=True) plt.xlabel('W2 mag') ylo,yhi = plt.ylim() plt.ylim(0.3, yhi) ps.savefig() plt.clf() plt.hist(np.log10(SW.w1_prochi2 / SW.w1_pronpix), 100, range=(0,3), log=True, histtype='step', color='b') plt.hist(np.log10(SW.w1_prochi2[I] / SW.w1_pronpix[I]), 100, range=(0,3), log=True, histtype='step', color='r') plt.xlabel('log chi2/npix') ylo,yhi = plt.ylim() plt.ylim(0.3, yhi) ps.savefig()
def main(): parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='DECaLS simulations.') parser.add_argument('-b', '--brick', type=str, default='2428p117', metavar='', help='process this brick (required input)') parser.add_argument('-o', '--objtype', type=str, choices=['STAR', 'ELG', 'LRG', 'BGS'], default='STAR', metavar='', help='object type (STAR, ELG, LRG, BGS)') parser.add_argument('-out', '--output_dir', type=str, default=None, metavar='', help='relative path to output directory') parser.add_argument('-extra', '--extra_plots', action='store_true', help='make missing, annotated, coadded plots, dont if many chunks') parser.add_argument('-v', '--verbose', action='store_true', help='toggle on verbose output') args = parser.parse_args() if args.brick is None: parser.print_help() sys.exit(1) if args.extra_plots: extra_plots= True else: extra_plots= False print('extra_plots=',extra_plots) # Set the debugging level if args.verbose: lvl = logging.DEBUG else: lvl = logging.INFO logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout) log = logging.getLogger('__name__') brickname = args.brick objtype = args.objtype.upper() lobjtype = objtype.lower() log.info('Analyzing objtype {} on brick {}'.format(objtype, brickname)) if 'DECALS_SIM_DIR' in os.environ: decals_sim_dir = os.getenv('DECALS_SIM_DIR') else: decals_sim_dir = '.' input_dir= os.path.join(decals_sim_dir,brickname,lobjtype) if args.output_dir is None: output_dir= os.path.join(decals_sim_dir,brickname,'qaplots_'+lobjtype) else: output_dir= args.output_dir if not os.path.exists(output_dir): os.makedirs(output_dir) # Plotting preferences #sns.set(style='white',font_scale=1.6,palette='dark')#,font='fantasy') #col = sns.color_palette('dark') col = ['b', 'k', 'c', 'm', 'y', 0.8] # Read metadata catalog. metafile = os.path.join(input_dir, 'metacat-{}-{}.fits'.format(brickname, lobjtype)) log.info('Reading {}'.format(metafile)) meta = fits.getdata(metafile, 1) # We need this for our histograms below magbinsz = 0.2 rminmax = np.squeeze(meta['RMAG_RANGE']) nmagbin = long((rminmax[1]-rminmax[0])/magbinsz) # Work in chunks. allsimcat = [] bigsimcat = [] bigsimcat_missed = [] bigtractor = [] chunk_dirs= glob.glob(os.path.join(input_dir,'0*')) nchunk= len(chunk_dirs) if nchunk == 0: raise ValueError # Loop through chunk dirs 000,001,...,999 for ichunk,cdir in enumerate([chunk_dirs[0]]): chunksuffix = os.path.basename(cdir) #'{:02d}'.format(ichunk) log.info('Working on chunk {:02d}/{:02d}'.format(ichunk+1, nchunk)) # Read the simulated object catalog simcatfile = os.path.join(cdir, 'simcat-{}-{}-{:02d}.fits'.format(brickname, lobjtype, int(chunksuffix))) log.info('Reading {}'.format(simcatfile)) simcat = Table(fits.getdata(simcatfile, 1)) # Read Tractor catalog tractorfile = os.path.join(cdir, 'tractor-{}-{}-{:02d}.fits'.format(brickname, lobjtype, int(chunksuffix))) log.info('Reading {}'.format(tractorfile)) tractor = Table(fits.getdata(tractorfile, 1)) # Match m1, m2, d12 = match_radec(tractor['ra'].copy(), tractor['dec'].copy(), simcat['RA'].copy(), simcat['DEC'].copy(), 1.0/3600.0) #m1, m2, d12 = matching.johan_tree(tractor['ra'].copy(), tractor['dec'].copy(),\ # simcat['RA'].copy(), simcat['DEC'].copy(), dsmax=1.0/3600.0) print('matched %d/%d' % (len(m2),len(simcat['RA']))) missing = np.delete(np.arange(len(simcat)), m2, axis=0) log.info('Missing {}/{} sources'.format(len(missing), len(simcat))) #good = np.where((np.abs(tractor['decam_flux'][m1,2]/simcat['rflux'][m2]-1)<0.3)*1) # Build matching catalogs for the plots below. if len(bigsimcat) == 0: bigsimcat = simcat[m2] bigtractor = tractor[m1] bigsimcat_missing = simcat[missing] else: bigsimcat = vstack((bigsimcat, simcat[m2])) bigtractor = vstack((bigtractor, tractor[m1])) bigsimcat_missing = vstack((bigsimcat_missing, simcat[missing])) if len(allsimcat) == 0: allsimcat = simcat else: allsimcat = vstack((allsimcat, simcat)) # Get cutouts of the bright matched sources with small/large delta mag if extra_plots: for img_name in ['image','resid','simscoadd']: # Indices of large and small dmag junk,i_large_dmag,i_small_dmag= bright_dmag_cut(simcat[m2],tractor[m1]) # Large dmag cutouts qafile = os.path.join(output_dir, 'qa-{}-{}-{}-bright-large-dmag-{:02d}.png'.format(\ brickname, lobjtype, img_name, int(chunksuffix))) plot_cutouts_by_index(simcat,i_large_dmag, brickname,lobjtype,chunksuffix, \ indir=cdir,img_name=img_name,qafile=qafile) log.info('Wrote {}'.format(qafile)) # Small dmag cutouts qafile = os.path.join(output_dir, 'qa-{}-{}-{}-bright-small-dmag-{:02d}.png'.format(\ brickname, lobjtype, img_name, int(chunksuffix))) plot_cutouts_by_index(simcat,i_small_dmag, brickname,lobjtype,chunksuffix, \ indir=cdir,img_name=img_name,qafile=qafile) log.info('Wrote {}'.format(qafile)) # Get cutouts of the missing sources in each chunk (if any) if len(missing) > 0 and extra_plots: for img_name in ['image']: #,'simscoadd']: qafile = os.path.join(output_dir, 'qa-{}-{}-{}-missing-{:02d}.png'.format(\ brickname, lobjtype, img_name, int(chunksuffix))) miss = missing[np.argsort(simcat['R'][missing])] plot_cutouts_by_index(simcat,miss, brickname,lobjtype,chunksuffix, \ indir=cdir,img_name=img_name,qafile=qafile) log.info('Wrote {}'.format(qafile)) # Annotate the coadd image and residual files so the simulated sources # are labeled. if extra_plots: for img_name in ('simscoadd','image', 'resid'): qafile = os.path.join(output_dir, 'qa-{}-{}-{}-{:02d}-annot.png'.format(\ brickname, lobjtype,img_name, int(chunksuffix))) plot_annotated_coadds(simcat, brickname, lobjtype, chunksuffix, \ indir=cdir,img_name=img_name,qafile=qafile) log.info('Wrote {}'.format(qafile)) # now operate on concatenated catalogues from multiple chunks # Grab flags b_good,b_bad= basic_cut(bigtractor) # mags and colors of ALL injected sources plot_injected_mags(allsimcat, log, qafile=\ os.path.join(output_dir, 'qa-{}-{}-injected-mags.png'.format(brickname, lobjtype))) # number of detected sources that are bad, good and number of undetected, binned by r mag plot_good_bad_ugly(allsimcat,bigsimcat,bigsimcat_missing, nmagbin,rminmax, b_good,b_bad, log, qafile=\ os.path.join(output_dir, 'qa-{}-{}-N-good-bad-missed.png'.format(brickname, lobjtype))) # Flux residuals vs input magnitude plot_tractor_minus_answer(bigsimcat,bigtractor, b_good,rminmax, log, qafile=\ os.path.join(output_dir, 'qa-{}-{}-good-flux.png'.format(brickname, lobjtype))) # chi plots: Flux residual / estimated Flux error plot_chi(bigsimcat,bigtractor, b_good,rminmax, log, qafile=\ os.path.join(output_dir, 'qa-{}-{}-chi-good.png'.format(brickname, lobjtype))) # Color residuals plot_color_tractor_minus_answer(bigtractor,bigsimcat, rminmax, brickname,lobjtype, log, qafile =\ os.path.join(output_dir, 'qa-{}-{}-color.png'.format(brickname, lobjtype))) # Fraction of recovered sources plot_fraction_recovered(allsimcat,bigsimcat, nmagbin,rminmax, brickname, lobjtype, log, qafile =\ os.path.join(output_dir, 'qa-{}-{}-frac.png'.format(brickname, lobjtype))) # S/N of recovered sources (S/N band vs. AB mag band) plot_sn_recovered(allsimcat,bigsimcat,bigtractor, brickname, lobjtype, log, qafile =\ os.path.join(output_dir, 'qa-{}-{}-SN.png'.format(brickname, lobjtype))) # Distribution of object types for matching sources. plot_recovered_types(bigsimcat,bigtractor, nmagbin,rminmax, objtype,log, qafile =\ os.path.join(output_dir, 'qa-{}-{}-type.png'.format(brickname, lobjtype))) # Confusion matrix for distribution of object types # Basic cm, use slim=False types= ['PSF ', 'SIMP', 'EXP ', 'DEV ', 'COMP'] cm,all_names= create_confusion_matrix(np.array(['PSF ']*bigtractor['ra'].data[b_good].shape[0]), bigtractor['type'].data[b_good], \ types=types,slim=False) qafile = os.path.join(output_dir, 'qa-{}-{}-{}-confusion.png'.format(brickname, lobjtype,'good')) plot_confusion_matrix(cm,all_names,all_names, log,qafile) # Now a stacked confusion matrix # Compute a row for each r mag range and stack rows make_stacked_cm(bigsimcat,bigtractor, b_good, log,qafile =\ os.path.join(output_dir, 'qa-{}-{}-good-confusion-stack.png'.format(brickname, lobjtype))) '''
# plothist(W.ra, W.dec, 100, range=((r0,r1),(d0,d1))) # ps.savefig() # # plt.clf() # plothist(WC.ra, WC.dec, 100, range=((r0,r1),(d0,d1))) # ps.savefig() #xlo,xhi = 7,26 xlo,xhi = 10,25 R = 4.0 I,J,d = match_radec(W.ra, W.dec, WC.ra, WC.dec, R, nearest=True) loghist(W.w1mag[I], WC.w1mpro[J], 200, range=((xlo,xhi),(xlo,xhi))) plt.plot([xlo,xhi],[xlo,xhi], '-', color='1') plt.xlabel('Tractor w1mag') plt.ylabel('Catalog w1mpro') plt.axis([xlo,xhi,xlo,xhi]) ps.savefig() lo,hi = 0,0.25 loghist(W.w1magerr[I], WC.w1sigmpro[J], 200, range=((lo,hi),(lo,hi))) plt.plot([lo,hi],[lo,hi], '-', color='1') plt.xlabel('Tractor w1mag err') plt.ylabel('Catalog w1sigmpro') plt.axis([lo,hi,lo,hi])
def main(): ps = PlotSequence('shotgun') decals = Decals() C = fits_table('decals-ccds-annotated.fits') print(len(C), 'CCDs') C.cut(C.photometric) C.cut(C.blacklist_ok) print(len(C), 'photometric and not blacklisted') # HACK print('FIXME not cutting on DECALS') #C.cut(C.tilepass > 0) #print(len(C), 'taken by DECaLS') targets = dict(g=24.0, r=23.4, z=22.5) def ivtomag(iv, nsigma=5.): return -2.5 * (np.log10(nsigma / np.sqrt(iv)) - 9) def band_index(band): allbands = 'ugrizY' return allbands.index(band) ccmap = dict(g='g', r='r', z='m') ceil_exptime = dict(g=125., r=125., z=250.) #plt.clf() bands = 'grz' for band in bands: tmag = targets[band] print() print(band, 'band, target depth', tmag) ccds = C[C.filter == band] ccdarea = (2046*4094*(0.262/3600.)**2) print(len(ccds), 'CCDs, total exptime', np.sum(ccds.exptime), '(mean %.1f)' % np.mean(ccds.exptime), 'total area', len(ccds)*ccdarea, 'sq.deg') detsig1 = ccds.sig1 / ccds.galnorm_mean totiv = np.sum(1. / detsig1**2) # depth we would have if we had all exposure time in one CCD # print('5-sigma galaxy depth if concentrated in one CCD:', ivtomag(totiv)) # # mean depth # print('5-sigma galaxy depth if spread equally among', len(ccds), 'CCDs:', ivtomag(totiv / len(ccds))) # print('vs median depth', np.median(ccds.galdepth)) # print('5-sigma galaxy depth if spread equally among %i/2' % (len(ccds)), 'CCDs:', ivtomag(totiv / (len(ccds)/2))) # print('5-sigma galaxy depth if spread equally among %i/3' % (len(ccds)), 'CCDs:', ivtomag(totiv / (len(ccds)/3))) # spread over 6000 sq deg sqdeg = 6000 avgiv = totiv * ccdarea / sqdeg #print('5-sigma galaxy depth if spread over', sqdeg, 'sqdeg:', ivtomag(avgiv)) tflux = 10.**(tmag / -2.5 + 9) tiv = 1. / (tflux / 5)**2 #print('Fraction of', sqdeg, 'sqdeg survey complete:', avgiv / tiv) iband = band_index(band) ext = ccds.decam_extinction[:,iband] medext = np.median(ext) print('With extinction (median %.2f mag):' % medext) transmission = 10.**(-ext / 2.5) detsig1 = ccds.sig1 / ccds.galnorm_mean / transmission totiv = np.sum(1. / detsig1**2) # depth we would have if we had all exposure time in one CCD print('5-sigma galaxy depth if concentrated in one CCD: %.3f' % ivtomag(totiv)) # mean depth print('5-sigma galaxy depth if spread equally among', len(ccds), 'CCDs: %.3f' % ivtomag(totiv / len(ccds))) print('vs median depth: %.3f' % np.median(ccds.galdepth - ext)) print('5-sigma galaxy depth if spread equally among %i/2' % (len(ccds)), 'CCDs: %.3f' % ivtomag(totiv / (len(ccds)/2))) print('5-sigma galaxy depth if spread equally among %i/3' % (len(ccds)), 'CCDs: %.3f' % ivtomag(totiv / (len(ccds)/3))) # spread over 6000 sq deg sqdeg = 6000 avgiv = totiv * ccdarea / sqdeg print('5-sigma galaxy depth if spread over', sqdeg, 'sqdeg: %.3f' % ivtomag(avgiv)) print('Fraction of', sqdeg, 'sqdeg survey complete: %.3f' % (avgiv / tiv)) # plt.hist(ccds.exptime, range=(0,250), bins=50, histtype='step', color=ccmap[band]) # I = np.flatnonzero(ccds.exptime < (ceil_exptime[band] - 1.)) # ccds.cut(I) # print('Cutting out exposures with ceil exposure time:', len(ccds)) # # plt.hist(ccds.exptime, bins=25, histtype='step', color=ccmap[band], # linestyle='dotted', linewidth=3, alpha=0.3) # # transmission = transmission[I] # ext = ext[I] # # detsig1 = ccds.sig1 / ccds.galnorm_mean / transmission # totiv = np.sum(1. / detsig1**2) # # depth we would have if we had all exposure time in one CCD # print('5-sigma galaxy depth if concentrated in one CCD:', ivtomag(totiv)) # # mean depth # print('5-sigma galaxy depth if spread equally among', len(ccds), 'CCDs:', ivtomag(totiv / len(ccds))) # print('vs median depth', np.median(ccds.galdepth - ext)) # print('5-sigma galaxy depth if spread equally among %i/2' % (len(ccds)), 'CCDs:', ivtomag(totiv / (len(ccds)/2))) # print('5-sigma galaxy depth if spread equally among %i/3' % (len(ccds)), 'CCDs:', ivtomag(totiv / (len(ccds)/3))) # # spread over 6000 sq deg # sqdeg = 6000 # avgiv = totiv * ccdarea / sqdeg # print('5-sigma galaxy depth if spread over', sqdeg, 'sqdeg:', ivtomag(avgiv)) # print('Fraction of', sqdeg, 'sqdeg survey complete:', avgiv / tiv) # plt.xlabel('Exposure time (s)') # ps.savefig() print() dra = 4094 / 2. / 3600 * 0.262 ddec = 2046 / 2. / 3600 * 0.262 ralo = max( 0, min(C.ra - dra / np.cos(np.deg2rad(C.dec)))) rahi = min(360, max(C.ra + dra / np.cos(np.deg2rad(C.dec)))) declo = max(-90, min(C.dec - ddec)) dechi = min( 90, max(C.dec + ddec)) # brick 0001m002 #ralo,rahi = 0., 0.25 #declo,dechi = -0.375, -0.125 #ralo, rahi = 0, 1 #declo, dechi = 0, 1 ralo, rahi = 0, 0.5 declo, dechi = 0, 0.5 print('RA,Dec range', (ralo, rahi), (declo, dechi)) N = 10000 nbatch = 1000 rr,dd = [],[] ntotal = 0 while ntotal < N: ru = np.random.uniform(size=nbatch) d = np.random.uniform(low=declo, high=dechi, size=nbatch) # Taper the accepted width in RA based on Dec cosd = np.cos(np.deg2rad(d)) I = np.flatnonzero(ru < cosd) if len(I) == 0: continue r = ralo + (rahi - ralo) * ru[I]/cosd[I] d = d[I] rr.append(r) dd.append(d) ntotal += len(r) print('Kept', len(r), 'of', nbatch) ra = np.hstack(rr) dec = np.hstack(dd) del rr del dd ra = ra [:N] dec = dec[:N] print('RA,Dec ranges of samples:', (ra.min(), ra.max()), (dec.min(), dec.max())) # plt.clf() # plt.plot(ra, dec, 'b.', alpha=0.1) # ps.savefig() # CCD size margin = 10 * 0.262 / 3600. # C.dra is in degrees on the sphere, not delta-RA # dra = C.dra / np.cos(np.deg2rad(C.dec)) # ccds = C[(C.ra + dra + margin > ralo) * # (C.ra - dra - margin < rahi) * # (C.dec + C.ddec + margin > declo) * # (C.dec - C.ddec - margin < dechi)] I,J,d = match_radec(C.ra, C.dec, (ralo+rahi)/2., (declo+dechi)/2., degrees_between(ralo,declo, rahi,dechi)/2. + 1., nearest=True) ccds = C[I] print(len(ccds), 'nearby') assert(len(ccds)) print('RA range', ccds.ra.min(), ccds.ra.max()) radius = np.hypot(2046, 4096) / 2. * 0.262 / 3600 * 1.1 II = match_radec(ccds.ra, ccds.dec, ra, dec, radius, indexlist=True) #print('Matching:', II) depthrange = [20,25] depthbins = 100 depthbins2 = 500 galdepths = dict([(b, np.zeros(len(ra))) for b in bands]) iccds = [] for iccd,I in enumerate(II): if I is None: continue ccd = ccds[iccd] print('Matched to CCD %s: %i' % (ccd.expid, len(I))) # Actually inside CCD RA,Dec box? r = ra[I] d = dec[I] # print('degrees_between: ccd ra,dec', ccd.ra, ccd.dec) # for ri,di in zip(r, degrees_between(r, ccd.dec + np.zeros_like(r), # ccd.ra, ccd.dec)): # print('ri: di', ri, di) J = np.flatnonzero((degrees_between(r, ccd.dec+np.zeros_like(r), ccd.ra, ccd.dec) < dra) * (np.abs(d - ccd.dec) < ddec)) print('Actually inside CCD RA,Dec box:', len(J)) if len(J) == 0: continue I = np.array(I)[J] # j = J[0] # print('deg between', degrees_between(r[j], ccd.dec, # ccd.ra, ccd.dec), 'vs', dra) # print(' dec', d[j], 'dist', np.abs(d[j] - ccd.dec), 'vs', ddec) iccds.append((iccd,I[0])) band = ccd.filter gd = galdepths[band] print('Gal depth in', band, ':', ccd.galdepth) # mag -> 5sig1 -> iv cgd = 10.**((ccd.galdepth - 22.5) / -2.5) cgd = 1. / cgd**2 gd[I] += cgd plt.clf() # plt.plot(ra, dec, 'k.', alpha=0.1) # C1 = fits_table('decals-ccds-annotated.fits') # ii,jj,dd = match_radec(C1.ra, C1.dec, (ralo+rahi)/2.,(declo+dechi)/2,0.5) # C1.cut(ii) # C1.ra -= (C1.ra > 270)*360 # for ccd in C1: # r,dr = ccd.ra, ccd.dra / np.cos(np.deg2rad(ccd.dec)) # d,dd = ccd.dec, ccd.ddec # sty = dict(color='k', alpha=0.1) # if not (ccd.photometric and ccd.blacklist_ok): # sty = dict(color='c', lw=3) # plt.plot([r-dr, r+dr, r+dr, r-dr, r-dr], [d-dd,d-dd,d+dd,d+dd,d-dd], # '-', **sty) for iccd,i in iccds: ccd = ccds[iccd] r,dr = ccd.ra, dra / np.cos(np.deg2rad(ccd.dec)) # HACK r = r + (r > 270)*-360. d,dd = ccd.dec, ddec plt.plot([r-dr, r+dr, r+dr, r-dr, r-dr], [d-dd,d-dd,d+dd,d+dd,d-dd], '-', color=ccmap[ccd.filter], alpha=0.5) plt.plot(ra[i], dec[i], 'k.') plt.plot([ralo,ralo,rahi,rahi,ralo],[declo,dechi,dechi,declo,declo], 'k-', lw=2) #plt.axis([-0.1, 0.6, -0.1, 0.6]) plt.axis([-0.5, 1., -0.5, 1.]) plt.xlabel('RA') plt.ylabel('Dec') ps.savefig() for band in bands: gd = galdepths[band] # iv -> 5sig1 -> mag gd = 1. / np.sqrt(gd) gd = -2.5 * (np.log10(gd) - 9.) plt.clf() plt.hist(gd, range=depthrange, bins=depthbins, histtype='step', color=ccmap[band]) plt.xlabel('5-sigma Galaxy depth (mag)') plt.title('%s band' % band) ps.savefig() print('Reading extinction values for sample points...') sfd = SFDMap() filts = ['%s %s' % ('DES', f) for f in bands] ebv,extinction = sfd.extinction(filts, ra, dec, get_ebv=True) print('Extinction:', extinction.shape) B = decals.get_bricks_readonly() I = decals.bricks_touching_radec_box(None, ralo, rahi, declo, dechi) B.cut(I) print(len(B), 'bricks touching RA,Dec box') depth_hists = {} depth_hists_2 = {} for brick in B: print('Brick', brick.brickname) I = np.flatnonzero((ra > brick.ra1 ) * (ra < brick.ra2) * (dec > brick.dec1) * (dec < brick.dec2)) print(len(I), 'samples in brick') if len(I) == 0: continue wcs = wcs_for_brick(brick) ok,x,y = wcs.radec2pixelxy(ra[I], dec[I]) x = np.round(x - 1).astype(int) y = np.round(y - 1).astype(int) for iband,band in enumerate(bands): fn = decals.find_file('nexp', brick=brick.brickname, band=band) print('Reading', fn) if not os.path.exists(fn): print('Missing:', fn) continue nexp = fitsio.read(fn) nexp = nexp[y, x] ext = extinction[I, iband] fn = decals.find_file('galdepth', brick=brick.brickname, band=band) print('Reading', fn) galdepth = fitsio.read(fn) galdepth = galdepth[y, x] # iv -> mag galdepth = -2.5 * (np.log10(5. / np.sqrt(galdepth)) - 9) # extinction-corrected galdepth -= ext un = np.unique(nexp) print('Numbers of exposures:', un) for ne in un: if ne == 0: continue J = np.flatnonzero(nexp == ne) gd = galdepth[J] key = (ne, band) H = depth_hists.get(key, 0) h,e = np.histogram(gd, range=depthrange, bins=depthbins) H = h + H depth_hists[key] = H H = depth_hists_2.get(key, 0) h,e = np.histogram(gd, range=depthrange, bins=depthbins2) H = h + H depth_hists_2[key] = H dlo,dhi = depthrange left = dlo + np.arange(depthbins) * (dhi-dlo) / float(depthbins) binwidth = left[1]-left[0] # for k,H in depth_hists.items(): # plt.clf() # plt.bar(left, H, width=binwidth) # plt.xlabel('Galaxy depth (mag)') # (ne,band) = k # plt.title('%s band, %i exposures' % (band, ne)) # plt.xlim(dlo, dhi) # fn = 'depth-%s-%i.png' % (band, ne) # plt.savefig(fn) # print('Wrote', fn) rainbow = ['r', '#ffa000', 'y', 'g', 'b', 'm'] for band in bands: plt.clf() for ne in range(1,10): key = (ne, band) if not key in depth_hists: continue H = depth_hists[key] print('hist length:', len(H)) plt.plot(np.vstack((left, left+binwidth)).T.ravel(), np.repeat(H, 2), '-', color=rainbow[ne-1 % len(rainbow)], label='%i exp' % ne) plt.title('%s band' % band) plt.xlabel('Galaxy depth (mag)') plt.legend(loc='upper left') ps.savefig() left2 = dlo + np.arange(depthbins2) * (dhi-dlo) / float(depthbins2) binwidth2 = left2[1]-left2[0] for band in bands: hsum = 0 for ne in range(1,10): key = (ne, band) if not key in depth_hists_2: continue hsum += depth_hists_2[key] print('%s band:' % band) print('Total number of counts in histogram:', sum(hsum)) # [-1::-1] = reversed hsum = np.cumsum(hsum[-1::-1])[-1::-1] hsum = hsum * 100. / float(N) plt.clf() #plt.plot(left2+binwidth/2, hsum, 'k-') plt.plot(left2, hsum, 'k-') plt.xlabel('Galaxy depth (mag)') plt.ylabel('Cumulative fraction (%)') plt.title('%s band' % band) # 90% to full depth # 95% to full depth - 0.3 mag # 98% to full depth - 0.6 mag for y,x,c in [ (90, targets[band], 'r'), (95, targets[band] - 0.3, 'g'), (98, targets[band] - 0.6, 'b')]: xf = (x - dlo) / (dhi - dlo) yf = y / 100. plt.axvline(x, ymax=yf, color=c, lw=3, alpha=0.3) plt.axhline(y, xmax=xf, color=c, lw=3, alpha=0.3) #mid = left2 + binwidth2/2 print('target : %.1f %% deeper than %.2f' % (y, x)) # ??? left or midpoint? ibin = np.flatnonzero(left2 > x)[0] pct = hsum[ibin] status = ' '*20 if pct >= y: status += '-> pass' else: status += '-> FAIL' plt.axhline(pct, color=c, alpha=0.5) #xmax=xf, print('achieved: %.1f %% %s' % (pct, status)) # deeper than %.2f' % (hsum[ibin], x)) ii = np.flatnonzero(hsum > y) if len(ii): ibin = ii[-1] print(' %.1f %% deeper than %.2f' % (y, left2[ibin])) else: print(' Total coverage only %.1f %%' % max(hsum)) print() plt.xlim(dlo, dhi) plt.ylim(0., 100.) ps.savefig()
def main(): sbfn = 'skybricks.fits' SB = fits_table(sbfn) Bnorth = fits_table( '/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/north/survey-bricks-dr9-north.fits.gz' ) Bsouth = fits_table( '/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/survey-bricks-dr9-south.fits.gz' ) Bnorth.cut(Bnorth.survey_primary) Bsouth.cut(Bsouth.survey_primary) #Bsouth.cut(Bsouth.dec > -30) Bnorth.hemi = np.array(['north'] * len(Bnorth)) Bsouth.hemi = np.array(['south'] * len(Bsouth)) B = merge_tables([Bnorth, Bsouth]) # Rough cut the skybricks to those near bricks. I, J, d = match_radec(SB.ra, SB.dec, B.ra, B.dec, 1., nearest=True) SB.cut(I) import argparse parser = argparse.ArgumentParser() #parser.add_argument('--brick', help='Sky brick name') parser.add_argument('--minra', type=float, help='Cut to a minimum RA range of sky bricks') parser.add_argument('--maxra', type=float, help='Cut to a maximum RA range of sky bricks') parser.add_argument('--threads', type=int, help='Parallelize on this many cores') opt = parser.parse_args() if opt.minra: SB.cut(SB.ra >= opt.minra) if opt.maxra: SB.cut(SB.ra <= opt.maxra) version = get_git_version(os.getcwd()) print('Version string:', version) # Find bricks near skybricks, as a rough cut. # (magic 1. degree > hypot(skybrick radius, brick radiu) ~= 0.9) Inear = match_radec(SB.ra, SB.dec, B.ra, B.dec, 1., indexlist=True) args = [] k = 0 Isb = [] for isb, (sb, inear) in enumerate(zip(SB, Inear)): if inear is None: continue args.append((k, sb, B[np.array(inear)], version)) k += 1 Isb.append(isb) print(len(args), 'sky bricks') SB.cut(np.array(Isb)) if opt.threads: mp = multiproc(opt.threads) else: mp = multiproc() exist = mp.map(run_one, args) if opt.minra is None and opt.maxra is None: exist = np.array(exist) SB[exist].writeto('skybricks-exist.fits') return
def forcedphot(): T1 = fits_table('cs82data/cas-primary-DR8.fits') print(len(T1), 'primary') T1.cut(T1.nchild == 0) print(len(T1), 'children') rl, rh = T1.ra.min(), T1.ra.max() dl, dh = T1.dec.min(), T1.dec.max() tims = [] # Coadd basedir = os.path.join('cs82data', 'wise', 'level3') basefn = os.path.join(basedir, '3342p000_ab41-w1') tim = read_wise_coadd(basefn, radecroi=[rl, rh, dl, dh], nanomaggies=True) tims.append(tim) # Individuals basedir = os.path.join('cs82data', 'wise', 'level1b') for fn in ['04933b137-w1', '04937b137-w1', '04941b137-w1', '04945b137-w1', '04948a112-w1', '04949b137-w1', '04952a112-w1', '04953b137-w1', '04956a112-w1', '04960a112-w1', '04964a112-w1', '04968a112-w1', '05204a106-w1']: basefn = os.path.join(basedir, fn) tim = read_wise_image( basefn, radecroi=[rl, rh, dl, dh], nanomaggies=True) tims.append(tim) # tractor.Image's setMask() does a binary dilation on bad pixels! for tim in tims: # tim.mask = np.zeros(tim.shape, dtype=bool) tim.invvar = tim.origInvvar tim.mask = np.zeros(tim.shape, dtype=bool) print('tim:', tim) ps = PlotSequence('forced') plt.clf() plt.plot(T1.ra, T1.dec, 'r.') for tim in tims: wcs = tim.getWcs() H, W = tim.shape rr, dd = [], [] for x, y in zip([1, 1, W, W, 1], [1, H, H, 1, 1]): rd = wcs.pixelToPosition(x, y) rr.append(rd.ra) dd.append(rd.dec) plt.plot(rr, dd, 'k-', alpha=0.5) # setRadecAxes(rl,rh,dl,dh) ps.savefig() T2 = fits_table('wise-cut.fits') T2.w1 = T2.w1mpro R = 1. / 3600. I, J, d = match_radec(T1.ra, T1.dec, T2.ra, T2.dec, R) print(len(I), 'matches') refband = 'r' #bandnum = band_index('r') Lstar = (T1.probpsf == 1) * 1.0 Lgal = (T1.probpsf == 0) fracdev = T1.get('fracdev_%s' % refband) Ldev = Lgal * fracdev Lexp = Lgal * (1. - fracdev) ndev, nexp, ncomp, nstar = 0, 0, 0, 0 cat = Catalog() # for i,t in enumerate(T1): jmatch = np.zeros(len(T1)) jmatch[:] = -1 jmatch[I] = J for i in range(len(T1)): j = jmatch[i] if j >= 0: # match source: grab WISE catalog mag w1 = T2.w1[j] else: # unmatched: set it faint w1 = 18. bright = NanoMaggies(w1=NanoMaggies.magToNanomaggies(w1)) pos = RaDecPos(T1.ra[i], T1.dec[i]) if Lstar[i] > 0: # Star star = PointSource(pos, bright) cat.append(star) nstar += 1 continue hasdev = (Ldev[i] > 0) hasexp = (Lexp[i] > 0) iscomp = (hasdev and hasexp) if iscomp: dbright = bright * Ldev[i] ebright = bright * Lexp[i] elif hasdev: dbright = bright elif hasexp: ebright = bright else: assert(False) if hasdev: re = T1.get('devrad_%s' % refband)[i] ab = T1.get('devab_%s' % refband)[i] phi = T1.get('devphi_%s' % refband)[i] dshape = GalaxyShape(re, ab, phi) if hasexp: re = T1.get('exprad_%s' % refband)[i] ab = T1.get('expab_%s' % refband)[i] phi = T1.get('expphi_%s' % refband)[i] eshape = GalaxyShape(re, ab, phi) if iscomp: gal = CompositeGalaxy(pos, ebright, eshape, dbright, dshape) ncomp += 1 elif hasdev: gal = DevGalaxy(pos, dbright, dshape) ndev += 1 elif hasexp: gal = ExpGalaxy(pos, ebright, eshape) nexp += 1 cat.append(gal) print('Created', ndev, 'pure deV', nexp, 'pure exp and', end=' ') print(ncomp, 'composite galaxies', end=' ') print('and', nstar, 'stars') tractor = Tractor(tims, cat) for i, tim in enumerate(tims): ima = dict(interpolation='nearest', origin='lower', vmin=tim.zr[0], vmax=tim.zr[1]) mod = tractor.getModelImage(i) plt.clf() plt.imshow(mod, **ima) plt.gray() plt.title('model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(tim.getImage(), **ima) plt.gray() plt.title('data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(tim.getInvvar(), interpolation='nearest', origin='lower') plt.gray() plt.title('invvar') ps.savefig() # for tim in tims: wcs = tim.getWcs() H, W = tim.shape poly = [] for r, d in zip([rl, rl, rh, rh, rl], [dl, dh, dh, dl, dl]): x, y = wcs.positionToPixel(RaDecPos(r, d)) poly.append((x, y)) xx, yy = np.meshgrid(np.arange(W), np.arange(H)) xy = np.vstack((xx.flat, yy.flat)).T grid = points_inside_poly(xy, poly) grid = grid.reshape((H, W)) tim.setInvvar(tim.getInvvar() * grid) # plt.clf() # plt.imshow(grid, interpolation='nearest', origin='lower') # plt.gray() # ps.savefig() plt.clf() plt.imshow(tim.getInvvar(), interpolation='nearest', origin='lower') plt.gray() plt.title('invvar') ps.savefig() if i == 1: plt.clf() plt.imshow(tim.goodmask, interpolation='nearest', origin='lower') plt.gray() plt.title('goodmask') ps.savefig() miv = (1. / (tim.uncplane)**2) for bit in range(-1, 32): if bit >= 0: miv[(tim.maskplane & (1 << bit)) != 0] = 0. if bit == 31: plt.clf() plt.imshow(miv, interpolation='nearest', origin='lower') plt.gray() plt.title( 'invvar with mask bits up to %i blanked out' % bit) ps.savefig()
def find_alignments(fns, wcsfns, gaia_fn, aff_fn, aligned_fn): from astrometry.libkd.spherematch import tree_build_radec, trees_match from astrometry.libkd.spherematch import match_radec from astrometry.util.plotutils import plothist from astrometry.util.util import Tan import fitsio from astrom_common import getwcsoutline from singles import find_overlaps if True: WCS = [] for fn in wcsfns: wcs = Tan(fn) WCS.append(wcs) names = [fn.replace('-bright.fits', '') for fn in fns] outlines = [getwcsoutline(wcs) for wcs in WCS] overlaps, areas = find_overlaps(outlines) print('Reading tables...') TT = [fits_table(fn) for fn in fns] print('Building trees...') kds = [tree_build_radec(T.ra, T.dec) for T in TT] for T, name in zip(TT, names): T.name = np.array([name] * len(T)) allra = np.hstack([T.ra for T in TT]) alldec = np.hstack([T.dec for T in TT]) minra = np.min(allra) maxra = np.max(allra) mindec = np.min(alldec) maxdec = np.max(alldec) print('RA,Dec range:', minra, maxra, mindec, maxdec) plothist(allra, alldec) plt.axis([maxra, minra, mindec, maxdec]) plt.xlabel('RA (deg)') plt.ylabel('Dec (deg)') plt.savefig('match-all.png') Tref = fits_table(gaia_fn) r_arcsec = 0.2 I, J, d = match_radec(Tref.ra, Tref.dec, allra, alldec, r_arcsec / 3600.) dec = alldec[J] cosdec = np.cos(np.deg2rad(dec)) dr = (Tref.ra[I] - allra[J]) * cosdec * 3600. dd = (Tref.dec[I] - alldec[J]) * 3600. plt.clf() rr = (-r_arcsec * 1000, +r_arcsec * 1000) plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr)) plt.xlabel('dRA (milli-arcsec)') plt.ylabel('dDec (milli-arcsec)') plt.savefig('match-all-ref-before.png') # Initial matching of all stars r_arcsec = 0.2 I, J, d = match_radec(allra, alldec, allra, alldec, r_arcsec / 3600., notself=True) dec = alldec[I] cosdec = np.cos(np.deg2rad(dec)) dr = (allra[I] - allra[J]) * cosdec * 3600. dd = (alldec[I] - alldec[J]) * 3600. plt.clf() rr = (-r_arcsec * 1000, +r_arcsec * 1000) plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr)) plt.xlabel('dRA (milli-arcsec)') plt.ylabel('dDec (milli-arcsec)') plt.savefig('match-all-before.png') hulls = [] from scipy.spatial import ConvexHull for T in TT: hull = ConvexHull(np.vstack((T.ra, T.dec)).T) ra = T.ra[hull.vertices] ra = np.append(ra, ra[0]) dec = T.dec[hull.vertices] dec = np.append(dec, dec[0]) hulls.append((ra, dec)) aligns = {} #for i in []: for i in range(len(kds)): for j in range(i + 1, len(kds)): print('Matching trees', i, 'and', j) r_arcsec = 0.2 radius = np.deg2rad(r_arcsec / 3600) I, J, d2 = trees_match(kds[i], kds[j], radius) print(len(I), 'matches') if len(I) == 0: continue Ti = TT[i] Tj = TT[j] dec = Ti[I].dec cosdec = np.cos(np.deg2rad(dec)) dr = (Ti[I].ra - Tj[J].ra) * cosdec * 3600. dd = (Ti[I].dec - Tj[J].dec) * 3600. if False: al = Alignment(Ti, Tj, searchradius=r_arcsec) print('Aligning...') if not al.shift(): print('Failed to find Alignment between fields') continue aligns[(i, j)] = al plt.clf() plotalignment(al) plt.savefig('match-align-%02i-%02i.png' % (i, j)) plt.clf() #plothist(np.append(Ti.ra, Tj.ra), np.append(Ti.dec, Tj.dec), docolorbar=False, doclf=False, dohot=False, # imshowargs=dict(cmap=antigray)) plothist(Ti.ra[I], Ti.dec[I], docolorbar=False, doclf=False) r, d = hulls[i] plt.plot(r, d, 'r-') r, d = hulls[j] plt.plot(r, d, 'b-') mra = Ti.ra[I] mdec = Ti.dec[I] mnra = np.min(mra) mxra = np.max(mra) mndec = np.min(mdec) mxdec = np.max(mdec) plt.plot([mnra, mnra, mxra, mxra, mnra], [mndec, mxdec, mxdec, mndec, mndec], 'g-') plt.axis([maxra, minra, mindec, maxdec]) plt.xlabel('RA (deg)') plt.ylabel('Dec (deg)') plt.savefig('match-radec-%02i-%02i.png' % (i, j)) plt.clf() rr = (-r_arcsec, +r_arcsec) plothist(dr, dd, nbins=100, range=(rr, rr)) plt.xlabel('dRA (arcsec)') plt.ylabel('dDec (arcsec)') plt.savefig('match-dradec-%02i-%02i.png' % (i, j)) #for roundi,(Nk,R) in enumerate(NkeepRads): refrad = 0.15 targetrad = 0.005 ps = PlotSequence('shift') from astrom_intra import intrabrickshift from singles import plot_all_alignments #Rads = [0.25, 0.1] Rads = [0.2, 0.050, 0.020] #Rads = [0.1] affs = None # this is the reference point around which rotations take place, NOT reference catalog stars. refrd = None for roundi, R in enumerate(Rads): if roundi > 0: refrad = 0.050 TT1 = TT nb = int(np.ceil(R / targetrad)) nb = max(nb, 5) if nb % 2 == 0: nb += 1 print('Round', roundi + 1, ': matching with radius', R) print('Nbins:', nb) # kwargs to pass to intrabrickshift ikwargs = {} minoverlap = 0.01 tryoverlaps = (overlaps > minoverlap) ikwargs.update( do_affine=True, #mp=mp, #alignplotargs=dict(bins=25), alignplotargs=dict(bins=50), overlaps=tryoverlaps) ikwargs.update(ref=Tref, refrad=refrad) # kwargs to pass to Alignment akwargs = {} i1 = intrabrickshift(TT1, matchradius=R, refradecs=refrd, align_kwargs=dict(histbins=nb, **akwargs), **ikwargs) refrd = i1.get_reference_radecs() filts = ['' for n in names] ap = i1.alplotgrid Nk = 100000 plot_all_alignments(ap, R * 1000, refrad * 1000, roundi + 1, names, filts, ps, overlaps, outlines, Nk) for T, aff in zip(TT, i1.affines): T.ra, T.dec = aff.apply(T.ra, T.dec) if affs is None: affs = i1.affines else: for a, a2 in zip(affs, i1.affines): a.add(a2) from astrom_common import Affine T = Affine.toTable(affs) T.filenames = fns #T.flt = fltfns #T.gst = gstfns #T.chip = chips # FAKE -- used as a name in alignment_plots T.gst = np.array([n + '.gst.fits' for n in names]) T.writeto(aff_fn) # Final matching of all stars allra = np.hstack([T.ra for T in TT]) alldec = np.hstack([T.dec for T in TT]) r_arcsec = 0.2 I, J, d = match_radec(allra, alldec, allra, alldec, r_arcsec / 3600., notself=True) dec = alldec[I] cosdec = np.cos(np.deg2rad(dec)) dr = (allra[I] - allra[J]) * cosdec * 3600. dd = (alldec[I] - alldec[J]) * 3600. plt.clf() rr = (-r_arcsec * 1000, +r_arcsec * 1000) plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr)) plt.xlabel('dRA (milli-arcsec)') plt.ylabel('dDec (milli-arcsec)') plt.savefig('match-all-after.png') I, J, d = match_radec(Tref.ra, Tref.dec, allra, alldec, r_arcsec / 3600.) dec = alldec[J] cosdec = np.cos(np.deg2rad(dec)) dr = (Tref.ra[I] - allra[J]) * cosdec * 3600. dd = (Tref.dec[I] - alldec[J]) * 3600. plt.clf() rr = (-r_arcsec * 1000, +r_arcsec * 1000) plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr)) plt.xlabel('dRA (milli-arcsec)') plt.ylabel('dDec (milli-arcsec)') plt.savefig('match-all-ref-after.png') r_arcsec = 0.02 I, J, d = match_radec(allra, alldec, allra, alldec, r_arcsec / 3600., notself=True) dec = alldec[I] cosdec = np.cos(np.deg2rad(dec)) dr = (allra[I] - allra[J]) * cosdec * 3600. dd = (alldec[I] - alldec[J]) * 3600. plt.clf() rr = (-r_arcsec * 1000, +r_arcsec * 1000) plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr)) plt.xlabel('dRA (milli-arcsec)') plt.ylabel('dDec (milli-arcsec)') plt.savefig('match-all-after2.png') T = fits_table() T.ra = allra T.dec = alldec for col in [ 'f814w_vega', 'f475w_vega', 'f336w_vega', 'f275w_vega', 'f110w_vega', 'f160w_vega', 'name' ]: T.set(col, np.hstack([t.get(col) for t in TT])) T.writeto(aligned_fn) if False: from singles import alignment_plots dataset = 'M31' Nkeep = 100000 R = 0.1 minoverlap = 0.01 perfield = False nocache = True from astrometry.util.multiproc import multiproc mp = multiproc() filts = ['F475W' for n in names] chips = [-1] * len(names) exptimes = [1] * len(names) Nall = [0] * len(names) rd = (minra, maxra, mindec, maxdec) cnames = names meta = (chips, names, cnames, filts, exptimes, Nall, rd) alignment_plots(afffn, dataset, Nkeep, 0, R, minoverlap, perfield, nocache, mp, 0, tables=(TT, outlines, meta), lexsort=False)
def tractor_vs_cat(): # Tractor/SDSS vs WISE/SDSS comparisons R = 4./3600. # NOTE, this matches all W entries (ie, same RA,Dec as SDSS), not just # the ones with photometry. I,J,d = match_radec(W.ra, W.dec, WC.ra, WC.dec, R, nearest=True) print(len(I), 'matches to WISE catalog') I2 = np.flatnonzero(np.logical_or(W.w1 > 0, W.w2 > 0)) T = S[I2] T.add_columns_from(W[I2]) C = WC[J] C.add_columns_from(S[I]) print(len(T), 'Tractor-SDSS matches') print(len(C), 'WISE-SDSS matches') I = np.flatnonzero((T.gpsf != T.ipsf) * (T.w1mag < 25)) wcuts = [17,18,19,20,21] plothist(T.gpsf[I] - T.ipsf[I], T.imod[I] - T.w1mag[I], 200, range=((-1,5),(0,7))) plt.xlabel('g - i (psf)') plt.ylabel('i - W1 (model)') plt.title('Tractor photometry') ps.savefig() for wcut in wcuts: J = I[(T.w1mag[I] < wcut)] plothist(T.gpsf[J] - T.ipsf[J], T.imod[J] - T.w1mag[J], 200, range=((-1,5),(0,7))) plt.xlabel('g - i (psf)') plt.ylabel('i - W1 (model)') plt.title('Tractor photometry: W1 < %i' % wcut) ps.savefig() plothist(C.gpsf - C.ipsf, C.ipsf - C.w1mag, 200, range=((-1,5),(0,7))) plt.xlabel('g - i (psf)') plt.ylabel('i - W1 (psf)') plt.title('WISE catalog photometry') ps.savefig() IT = np.logical_and(T.w1mag < 25, T.w2mag < 25) for l1,x1,l2,x2 in [ #('g (psf)', 'gpsf')*2, ('r (psf)', 'rpsf')*2, ('W1', 'w1mag', 'W1', 'w1mpro'), ('W2', 'w2mag', 'W2', 'w2mpro')]: if l1.startswith('W'): rng = ((10,20),(-2,2)) else: rng = ((15,25),(-2,2)) plothist(T.get(x1)[IT], (T.w1mag - T.w2mag)[IT], 200, range=rng) plt.xlabel(l1) plt.ylabel('W1 - W2') plt.title('Tractor photometry') ps.savefig() for wcut in wcuts: J = IT[(T.w1mag[IT] < wcut)] plothist(T.get(x1)[J], (T.w1mag - T.w2mag)[J], 200, range=rng) plt.xlabel(l1) plt.ylabel('W1 - W2') plt.title('Tractor photometry: W1 < %i' % wcut) ps.savefig() plothist(C.get(x2), C.w1mpro - C.w2mpro, 200, range=rng) plt.xlabel(l2) plt.ylabel('W1 - W2') plt.title('WISE catalog photometry') ps.savefig()
def orig_code(data, nmatch): nside = Healpix().get_nside(len(data)) _, lo, hi = sigmaclip(data[data != 0], low=3, high=3) flag = np.logical_or(data < lo, data > hi) flag *= (nmatch > 20) ra, dec = hp.pix2ang(nside, np.where(flag)[0], lonlat=True) # PLOTTING ralim = [ra.min(), ra.max()] declim = [dec.min(), dec.max()] my_mollzoom(ra, dec, data[flag], 'outliers', ralim=ralim, declim=declim, vlim=(lo, hi)) temp_ra, temp_dec = hp.pix2ang(nside, np.where(np.ones(len(data), bool))[0], lonlat=True) keep= (temp_ra >= ralim[0])*\ (temp_ra <= ralim[1])*\ (temp_dec >= declim[0])*\ (temp_dec <= declim[1]) my_mollzoom(temp_ra[keep], temp_dec[keep], data[keep], 'all', ralim=ralim, declim=declim, vlim=(lo, hi)) keep *= (nmatch > 20) my_mollzoom(temp_ra[keep], temp_dec[keep], data[keep], 'nmatch_gt20', ralim=ralim, declim=declim, vlim=(lo, hi)) # Match bricks #heal= fits_table() #for col,arr in zip(['ra','dec'],[ra,dec]): # heal.set(col, arr) brick = fits_table( os.path.join(args.targz_dir, 'legacysurveydir', 'survey-bricks.fits.gz')) brick.cut( (brick.dec > -40)*\ (brick.dec < 40)) #deg_per_healpix= get_pixscale(npix,unit='deg') deg_per_brick = 0.25 #imatch,imiss,d2d= Matcher().match_within(heal,brick, dist= deg_per_brick/2) I, J, d = match_radec(ra, dec, brick.ra, brick.dec, deg_per_brick / 2, nearest=True) #raise ValueError brick.cut(imatch['obs']) my_scatter(brick.ra, brick.dec, 'bricks', ralim=ralim, declim=declim) id = fn.replace('/', '').replace('.fits', '') savenm = os.path.join(args.outdir, 'brick_table_%s.fits' % id) brick.writeto(savenm) print('Wrote %s' % savenm)
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('--name1', help='Name for first data set') parser.add_argument('--name2', help='Name for second data set') parser.add_argument('--plot-prefix', default='compare', help='Prefix for plot filenames; default "%default"') parser.add_argument('--match', default=1.0, help='Astrometric cross-match distance in arcsec') parser.add_argument('dir1', help='First directory to compare') parser.add_argument('dir2', help='Second directory to compare') opt = parser.parse_args() ps = PlotSequence(opt.plot_prefix) name1 = opt.name1 if name1 is None: name1 = os.path.basename(opt.dir1) if not len(name1): name1 = os.path.basename(os.path.dirname(opt.dir1)) name2 = opt.name2 if name2 is None: name2 = os.path.basename(opt.dir2) if not len(name2): name2 = os.path.basename(os.path.dirname(opt.dir2)) tt = 'Comparing %s to %s' % (name1, name2) # regex for tractor-*.fits catalog filename catre = re.compile('tractor-.*.fits') cat1,cat2 = [],[] for basedir,cat in [(opt.dir1, cat1), (opt.dir2, cat2)]: for dirpath,dirnames,filenames in os.walk(basedir, followlinks=True): for fn in filenames: if not catre.match(fn): print('Skipping', fn, 'due to filename') continue fn = os.path.join(dirpath, fn) t = fits_table(fn) print(len(t), 'from', fn) cat.append(t) cat1 = merge_tables(cat1, columns='fillzero') cat2 = merge_tables(cat2, columns='fillzero') print('Total of', len(cat1), 'from', name1) print('Total of', len(cat2), 'from', name2) cat1.cut(cat1.brick_primary) cat2.cut(cat2.brick_primary) print('Total of', len(cat1), 'BRICK_PRIMARY from', name1) print('Total of', len(cat2), 'BRICK_PRIMARY from', name2) cat1.cut((cat1.decam_anymask[:,1] == 0) * (cat1.decam_anymask[:,2] == 0) * (cat1.decam_anymask[:,4] == 0)) cat2.cut((cat2.decam_anymask[:,1] == 0) * (cat2.decam_anymask[:,2] == 0) * (cat2.decam_anymask[:,4] == 0)) print('Total of', len(cat1), 'unmasked from', name1) print('Total of', len(cat2), 'unmasked from', name2) I,J,d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match/3600., nearest=True) print(len(I), 'matched') plt.clf() plt.hist(d * 3600., 100) plt.xlabel('Match distance (arcsec)') plt.title(tt) ps.savefig() matched1 = cat1[I] matched2 = cat2[J] for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]: K = np.flatnonzero((matched1.decam_flux_ivar[:,iband] > 0) * (matched2.decam_flux_ivar[:,iband] > 0)) print('Median mw_trans', band, 'is', np.median(matched1.decam_mw_transmission[:,iband])) plt.clf() plt.errorbar(matched1.decam_flux[K,iband], matched2.decam_flux[K,iband], fmt='.', color=cc, xerr=1./np.sqrt(matched1.decam_flux_ivar[K,iband]), yerr=1./np.sqrt(matched2.decam_flux_ivar[K,iband]), alpha=0.1, ) plt.xlabel('%s flux: %s' % (name1, band)) plt.ylabel('%s flux: %s' % (name2, band)) plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.) plt.axis([-100, 1000, -100, 1000]) plt.title(tt) ps.savefig() for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]: good = ((matched1.decam_flux_ivar[:,iband] > 0) * (matched2.decam_flux_ivar[:,iband] > 0)) K = np.flatnonzero(good) psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') P = np.flatnonzero(good * psf1 * psf2) mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband]) iv1 = matched1.decam_flux_ivar[:, iband] iv2 = matched2.decam_flux_ivar[:, iband] std = np.sqrt(1./iv1 + 1./iv2) plt.clf() plt.plot(mag1[K], (matched2.decam_flux[K,iband] - matched1.decam_flux[K,iband]) / std[K], '.', alpha=0.1, color=cc) plt.plot(mag1[P], (matched2.decam_flux[P,iband] - matched1.decam_flux[P,iband]) / std[P], '.', alpha=0.1, color='k') plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' % (name2, name1, band)) plt.xlabel('%s mag: %s' % (name1, band)) plt.axhline(0, color='k', alpha=0.5) plt.axis([24, 16, -10, 10]) plt.title(tt) ps.savefig() plt.clf() lp,lt = [],[] for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]: good = ((matched1.decam_flux_ivar[:,iband] > 0) * (matched2.decam_flux_ivar[:,iband] > 0)) #good = True psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband]) iv1 = matched1.decam_flux_ivar[:, iband] iv2 = matched2.decam_flux_ivar[:, iband] std = np.sqrt(1./iv1 + 1./iv2) #std = np.hypot(std, 0.01) G = np.flatnonzero(good * psf1 * psf2 * np.isfinite(mag1) * (mag1 >= 20) * (mag1 < dict(g=24, r=23.5, z=22.5)[band])) n,b,p = plt.hist((matched2.decam_flux[G,iband] - matched1.decam_flux[G,iband]) / std[G], range=(-4, 4), bins=50, histtype='step', color=cc, normed=True) sig = (matched2.decam_flux[G,iband] - matched1.decam_flux[G,iband]) / std[G] print('Raw mean and std of points:', np.mean(sig), np.std(sig)) med = np.median(sig) rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2. print('Median and percentile-based sigma:', med, rsigma) lp.append(p[0]) lt.append('%s: %.2f +- %.2f' % (band, med, rsigma)) bins = [] gaussint = [] for blo,bhi in zip(b, b[1:]): c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo) c /= (bhi - blo) #bins.extend([blo,bhi]) #gaussint.extend([c,c]) bins.append((blo+bhi)/2.) gaussint.append(c) plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5) plt.title(tt) plt.xlabel('Flux difference / error (sigma)') plt.axvline(0, color='k', alpha=0.1) plt.ylim(0, 0.45) plt.legend(lp, lt, loc='upper right') ps.savefig() for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]: plt.clf() mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband]) mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors( matched2.decam_flux[:,iband], matched2.decam_flux_ivar[:,iband]) meanmag = NanoMaggies.nanomaggiesToMag(( matched1.decam_flux[:,iband] + matched2.decam_flux[:,iband]) / 2.) psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') good = ((matched1.decam_flux_ivar[:,iband] > 0) * (matched2.decam_flux_ivar[:,iband] > 0) * np.isfinite(mag1) * np.isfinite(mag2)) K = np.flatnonzero(good) P = np.flatnonzero(good * psf1 * psf2) plt.errorbar(mag1[K], mag2[K], fmt='.', color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1) plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('%s %s (mag)' % (name2, band)) plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.) plt.axis([24, 16, 24, 16]) plt.title(tt) ps.savefig() plt.clf() plt.errorbar(mag1[K], mag2[K] - mag1[K], fmt='.', color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1) plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axis([24, 16, -1, 1]) plt.title(tt) ps.savefig() magbins = np.arange(16, 24.001, 0.5) plt.clf() plt.plot(mag1[K], (mag2[K]-mag1[K]) / np.hypot(magerr1[K], magerr2[K]), '.', color=cc, alpha=0.1) plt.plot(mag1[P], (mag2[P]-mag1[P]) / np.hypot(magerr1[P], magerr2[P]), 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('(%s %s - %s %s) / errors (sigma)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axis([24, 16, -10, 10]) plt.title(tt) ps.savefig() y = (mag2 - mag1) / np.hypot(magerr1, magerr2) plt.clf() plt.plot(meanmag[P], y[P], 'k.', alpha=0.1) midmag = [] vals = np.zeros((len(magbins)-1, 5)) median_err1 = [] iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25) # FIXME -- should we do some stats after taking off the mean difference? for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])): I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)] midmag.append((mlo+mhi)/2.) median_err1.append(np.median(magerr1[I])) if len(I) == 0: continue # median and +- 1 sigma quantiles ybin = y[I] vals[bini,0] = np.percentile(ybin, 16) vals[bini,1] = np.median(ybin) vals[bini,2] = np.percentile(ybin, 84) # +- 2 sigma quantiles vals[bini,3] = np.percentile(ybin, 2.3) vals[bini,4] = np.percentile(ybin, 97.7) iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25) print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points') # if iqd > iqd_gauss: # # What error adding in quadrature would you need to make the IQD match? # err = median_err1[-1] # target_err = err * (iqd / iqd_gauss) # sys_err = np.sqrt(target_err**2 - err**2) # print('--> add systematic error', sys_err) # ~ Johan's cuts mlo = 21. mhi = dict(g=24., r=23.5, z=22.5)[band] I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)] ybin = y[I] iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25) print('Mag bin', mlo, mhi, 'band', band, ': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points') if iqd > iqd_gauss: # What error adding in quadrature would you need to make # the IQD match? err = np.median(np.hypot(magerr1[I], magerr2[I])) print('Median error (hypot):', err) target_err = err * (iqd / iqd_gauss) print('Target:', target_err) sys_err = np.sqrt((target_err**2 - err**2) / 2.) print('--> add systematic error', sys_err) # check... err_sys = np.hypot(np.hypot(magerr1, sys_err), np.hypot(magerr2, sys_err)) ysys = (mag2 - mag1) / err_sys ysys = ysys[I] print('Resulting median error:', np.median(err_sys[I])) iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25) print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian') # Hmmm, this doesn't work... totally overshoots. plt.errorbar(midmag, vals[:,1], fmt='o', color='b', yerr=(vals[:,1]-vals[:,0], vals[:,2]-vals[:,1]), capthick=3, zorder=20) plt.errorbar(midmag, vals[:,1], fmt='o', color='b', yerr=(vals[:,1]-vals[:,3], vals[:,4]-vals[:,1]), capthick=2, zorder=20) plt.axhline( 1., color='b', alpha=0.2) plt.axhline(-1., color='b', alpha=0.2) plt.axhline( 2., color='b', alpha=0.2) plt.axhline(-2., color='b', alpha=0.2) for mag,err,y in zip(midmag, median_err1, vals[:,3]): if not np.isfinite(err): continue if y < -6: continue plt.text(mag, y-0.1, '%.3f' % err, va='top', ha='center', color='k', fontsize=10) plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band)) plt.ylabel('(%s %s - %s %s) / errors (sigma)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axvline(21, color='k', alpha=0.3) plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3) plt.axis([24.1, 16, -6, 6]) plt.title(tt) ps.savefig() #magbins = np.append([16, 18], np.arange(20, 24.001, 0.5)) if band == 'g': magbins = [20, 24] elif band == 'r': magbins = [20, 23.5] elif band == 'z': magbins = [20, 22.5] slo,shi = -5,5 plt.clf() ha = dict(bins=25, range=(slo,shi), histtype='step', normed=True) y = (mag2 - mag1) / np.hypot(magerr1, magerr2) midmag = [] nn = [] rgbs = [] lt,lp = [],[] for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])): I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)] if len(I) == 0: continue ybin = y[I] rgb = [0.,0.,0.] rgb[0] = float(bini) / (len(magbins)-1) rgb[2] = 1. - rgb[0] n,b,p = plt.hist(ybin, color=rgb, **ha) lt.append('mag %g to %g' % (mlo,mhi)) lp.append(p[0]) midmag.append((mlo+mhi)/2.) nn.append(n) rgbs.append(rgb) bins = [] gaussint = [] for blo,bhi in zip(b, b[1:]): #midbin.append((blo+bhi)/2.) #gaussint.append(scipy.stats.norm.cdf(bhi) - # scipy.stats.norm.cdf(blo)) c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo) c /= (bhi - blo) bins.extend([blo,bhi]) gaussint.extend([c,c]) plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5) plt.legend(lp, lt) plt.title(tt) plt.xlim(slo,shi) ps.savefig() bincenters = b[:-1] + (b[1]-b[0])/2. plt.clf() lp = [] for n,rgb,mlo,mhi in zip(nn, rgbs, magbins, magbins[1:]): p = plt.plot(bincenters, n, '-', color=rgb) lp.append(p[0]) plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2) plt.legend(lp, lt) plt.title(tt) plt.xlim(slo,shi) ps.savefig()
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('-o', '--out', dest='outfn', help='Output filename', default='TMP/nexp.fits') parser.add_argument('--merge', action='store_true', help='Merge sub-tables') parser.add_argument('--plot', action='store_true', help='Plot results') parser.add_argument('files', metavar='nexp-file.fits.gz', nargs='+', help='List of nexp files to process') opt = parser.parse_args() fns = opt.files if opt.merge: from astrometry.util.fits import merge_tables TT = [] for fn in fns: T = fits_table(fn) print(fn, '->', len(T)) TT.append(T) T = merge_tables(TT) T.writeto(opt.outfn) print('Wrote', opt.outfn) if opt.plot: T = fits_table(opt.files[0]) import pylab as plt import matplotlib ax = [360, 0, -21, 36] def radec_plot(): plt.axis(ax) plt.xlabel('RA (deg)') plt.xticks(np.arange(0, 361, 45)) plt.ylabel('Dec (deg)') gl = np.arange(361) gb = np.zeros_like(gl) from astrometry.util.starutil_numpy import lbtoradec rr,dd = lbtoradec(gl, gb) plt.plot(rr, dd, 'k-', alpha=0.5, lw=1) rr,dd = lbtoradec(gl, gb+10) plt.plot(rr, dd, 'k-', alpha=0.25, lw=1) rr,dd = lbtoradec(gl, gb-10) plt.plot(rr, dd, 'k-', alpha=0.25, lw=1) plt.figure(figsize=(8,5)) plt.subplots_adjust(left=0.1, right=0.98, top=0.93) # Map of the tile centers we want to observe... O = fits_table('obstatus/decam-tiles_obstatus.fits') O.cut(O.in_desi == 1) rr,dd = np.meshgrid(np.linspace(ax[1],ax[0], 700), np.linspace(ax[2],ax[3], 200)) from astrometry.libkd.spherematch import match_radec I,J,d = match_radec(O.ra, O.dec, rr.ravel(), dd.ravel(), 1.) desimap = np.zeros(rr.shape, bool) desimap.flat[J] = True def desi_map(): # Show the DESI tile map in the background. from astrometry.util.plotutils import antigray plt.imshow(desimap, origin='lower', interpolation='nearest', extent=[ax[1],ax[0],ax[2],ax[3]], aspect='auto', cmap=antigray, vmax=8) for band in 'grz': plt.clf() desi_map() N = T.get('nexp_%s' % band) I = np.flatnonzero(N > 0) #cm = matplotlib.cm.get_cmap('jet', 6) #cm = matplotlib.cm.get_cmap('winter', 5) cm = matplotlib.cm.viridis cm = matplotlib.cm.get_cmap(cm, 5) plt.scatter(T.ra[I], T.dec[I], c=N[I], s=2, edgecolors='none', vmin=0.5, vmax=5.5, cmap=cm) radec_plot() cax = colorbar_axes(plt.gca(), frac=0.06) plt.colorbar(cax=cax, ticks=range(6)) #plt.colorbar(ticks=range(6)) plt.title('DECaLS DR3: Number of exposures in %s' % band) plt.savefig('nexp-%s.png' % band) plt.clf() desi_map() plt.scatter(T.ra, T.dec, c=T.get('nexp_%s' % band), s=2, edgecolors='none', vmin=0, vmax=2.) radec_plot() plt.colorbar() plt.title('DECaLS DR3: PSF size, band %s' % band) plt.savefig('psfsize-%s.png' % band) return 0 for col in ['nobjs', 'npsf', 'nsimp', 'nexp', 'ndev', 'ncomp']: plt.clf() desi_map() N = T.get(col) mx = np.percentile(N, 99.5) plt.scatter(T.ra, T.dec, c=N, s=2, edgecolors='none', vmin=0, vmax=mx) radec_plot() plt.colorbar() plt.title('DECaLS DR3: Number of objects of type %s' % col[1:]) plt.savefig('nobjs-%s.png' % col[1:]) Ntot = T.nobjs for col in ['npsf', 'nsimp', 'nexp', 'ndev', 'ncomp']: plt.clf() desi_map() N = T.get(col) / Ntot.astype(np.float32) mx = np.percentile(N, 99.5) plt.scatter(T.ra, T.dec, c=N, s=2, edgecolors='none', vmin=0, vmax=mx) radec_plot() plt.colorbar() plt.title('DECaLS DR3: Fraction of objects of type %s' % col[1:]) plt.savefig('fobjs-%s.png' % col[1:]) return 0 # fnpats = opt.files # fns = [] # for pat in fnpats: # pfns = glob(pat) # fns.extend(pfns) # print('Pattern', pat, '->', len(pfns), 'files') #fns = glob('coadd/*/*/*-nexp*') #fns = glob('coadd/000/*/*-nexp*') #fns = glob('coadd/000/0001*/*-nexp*') fns.sort() print(len(fns), 'nexp files') brickset = set() bricklist = [] gn = [] rn = [] zn = [] gnhist = [] rnhist = [] znhist = [] nnhist = 6 gdepth = [] rdepth = [] zdepth = [] ibricks = [] nsrcs = [] npsf = [] nsimp = [] nexp = [] ndev = [] ncomp = [] gpsfsize = [] rpsfsize = [] zpsfsize = [] ebv = [] gtrans = [] rtrans = [] ztrans = [] bricks = fits_table('survey-bricks.fits.gz') #sfd = SFDMap() W = H = 3600 # H=3600 # xx,yy = np.meshgrid(np.arange(W), np.arange(H)) unique = np.ones((H,W), bool) tlast = 0 for fn in fns: print('File', fn) words = fn.split('/') dirprefix = '/'.join(words[:-4]) print('Directory prefix:', dirprefix) words = words[-4:] brick = words[2] print('Brick', brick) if not brick in brickset: brickset.add(brick) bricklist.append(brick) gn.append(0) rn.append(0) zn.append(0) gnhist.append([0 for i in range(nnhist)]) rnhist.append([0 for i in range(nnhist)]) znhist.append([0 for i in range(nnhist)]) index = -1 ibrick = np.nonzero(bricks.brickname == brick)[0][0] ibricks.append(ibrick) tfn = os.path.join(dirprefix, 'tractor', brick[:3], 'tractor-%s.fits'%brick) print('Tractor filename', tfn) T = fits_table(tfn, columns=['brick_primary', 'type', 'decam_psfsize', 'ebv', 'decam_mw_transmission']) T.cut(T.brick_primary) nsrcs.append(len(T)) types = Counter([t.strip() for t in T.type]) npsf.append(types['PSF']) nsimp.append(types['SIMP']) nexp.append(types['EXP']) ndev.append(types['DEV']) ncomp.append(types['COMP']) print('N sources', nsrcs[-1]) gpsfsize.append(np.median(T.decam_psfsize[:,1])) rpsfsize.append(np.median(T.decam_psfsize[:,2])) zpsfsize.append(np.median(T.decam_psfsize[:,4])) ebv.append(np.median(T.ebv)) gtrans.append(np.median(T.decam_mw_transmission[:,1])) rtrans.append(np.median(T.decam_mw_transmission[:,2])) ztrans.append(np.median(T.decam_mw_transmission[:,4])) br = bricks[ibrick] print('Computing unique brick pixels...') #wcs = Tan(fn, 0) #W,H = int(wcs.get_width()), int(wcs.get_height()) pixscale = 0.262/3600. wcs = Tan(br.ra, br.dec, W/2.+0.5, H/2.+0.5, -pixscale, 0., 0., pixscale, float(W), float(H)) import time t0 = time.clock() unique[:,:] = True find_unique_pixels(wcs, W, H, unique, br.ra1, br.ra2, br.dec1, br.dec2) # for i in range(W/2): # allin = True # lo,hi = i, W-i-1 # # one slice per side # side = slice(lo,hi+1) # top = (lo, side) # bot = (hi, side) # left = (side, lo) # right = (side, hi) # for slc in [top, bot, left, right]: # #print('xx,yy', xx[slc], yy[slc]) # rr,dd = wcs.pixelxy2radec(xx[slc]+1, yy[slc]+1) # U = (rr >= br.ra1 ) * (rr < br.ra2 ) * (dd >= br.dec1) * (dd < br.dec2) # #print('Pixel', i, ':', np.sum(U), 'of', len(U), 'pixels are unique') # allin *= np.all(U) # unique[slc] = U # if allin: # print('Scanned to pixel', i) # break t1 = time.clock() U = np.flatnonzero(unique) t2 = time.clock() print(len(U), 'of', W*H, 'pixels are unique to this brick') # #t3 = time.clock() #rr,dd = wcs.pixelxy2radec(xx+1, yy+1) # #t4 = time.clock() # #u = (rr >= br.ra1 ) * (rr < br.ra2 ) * (dd >= br.dec1) * (dd < br.dec2) # #t5 = time.clock() # #U2 = np.flatnonzero(u) #U2 = np.flatnonzero((rr >= br.ra1 ) * (rr < br.ra2 ) * # (dd >= br.dec1) * (dd < br.dec2)) #assert(np.all(U == U2)) #assert(len(U) == len(U2)) # #t6 = time.clock() # print(len(U2), 'of', W*H, 'pixels are unique to this brick') # #print(t0-tlast, 'other time') #tlast = time.clock() #t2 #print('t1:', t1-t0, 't2', t2-t1) # #print('t4:', t4-t3, 't5', t5-t4, 't6', t6-t5) # else: index = bricklist.index(brick) assert(index == len(bricklist)-1) index = bricklist.index(brick) assert(index == len(bricklist)-1) filepart = words[-1] filepart = filepart.replace('.fits.gz', '') print('File:', filepart) band = filepart[-1] assert(band in 'grz') nlist,nhist = dict(g=(gn,gnhist), r=(rn,rnhist), z=(zn,znhist))[band] upix = fitsio.read(fn).flat[U] med = np.median(upix) print('Band', band, ': Median', med) nlist[index] = med hist = nhist[index] for i in range(nnhist): if i < nnhist-1: hist[i] = np.sum(upix == i) else: hist[i] = np.sum(upix >= i) assert(sum(hist) == len(upix)) print('Number of exposures histogram:', hist) ibricks = np.array(ibricks) print('Maximum number of sources:', max(nsrcs)) T = fits_table() T.brickname = np.array(bricklist) T.ra = bricks.ra [ibricks] T.dec = bricks.dec[ibricks] T.nexp_g = np.array(gn).astype(np.int16) T.nexp_r = np.array(rn).astype(np.int16) T.nexp_z = np.array(zn).astype(np.int16) T.nexphist_g = np.array(gnhist).astype(np.int32) T.nexphist_r = np.array(rnhist).astype(np.int32) T.nexphist_z = np.array(znhist).astype(np.int32) T.nobjs = np.array(nsrcs).astype(np.int16) T.npsf = np.array(npsf ).astype(np.int16) T.nsimp = np.array(nsimp).astype(np.int16) T.nexp = np.array(nexp ).astype(np.int16) T.ndev = np.array(ndev ).astype(np.int16) T.ncomp = np.array(ncomp).astype(np.int16) T.psfsize_g = np.array(gpsfsize).astype(np.float32) T.psfsize_r = np.array(rpsfsize).astype(np.float32) T.psfsize_z = np.array(zpsfsize).astype(np.float32) T.ebv = np.array(ebv).astype(np.float32) T.trans_g = np.array(gtrans).astype(np.float32) T.trans_r = np.array(rtrans).astype(np.float32) T.trans_z = np.array(ztrans).astype(np.float32) T.writeto(opt.outfn)