def no_overlapping_radec(ra,dec, bounds, random_state=None, dist=5.0/3600):
'''resamples ra,dec where they are within dist of each other
ra,dec -- numpy arrays [degrees]
bounds-- list of [ramin,ramax,decmin,decmax]
random_state-- np.random.RandomState() object
dist-- separation in degrees,
sqrt(x^2+y^2) where x=(dec2-dec1), y=cos(avg(dec2,dec1))*(ra2-ra1)
'''
log = logging.getLogger('decals_sim')
if random_state is None:
random_state = np.random.RandomState()
# ra,dec indices of just neighbors within "dist" away, just nerest neighbor of those
m1, m2, d12 = match_radec(ra.copy(),dec.copy(), ra.copy(),dec.copy(),\
dist, nearest=True,notself=True)
cnt = 1
log.info("after iter=%d, have overlapping ra,dec %d/%d", cnt, len(m2),ra.shape[0])
while len(m2) > 0:
ra[m2]= random_state.uniform(bounds[0], bounds[1], len(m2))
dec[m2]= random_state.uniform(bounds[2], bounds[3], len(m2))
m1, m2, d12 = match_radec(ra.copy(),dec.copy(), ra.copy(),dec.copy(),\
dist, nearest=True,notself=True)
cnt += 1
log.info("after iter=%d, have overlapping ra,dec %d/%d", cnt, len(m2),ra.shape[0])
if cnt > 30:
log.error('Crash, could not get non-overlapping ra,dec in 30 iterations')
raise ValueError
return ra, dec
def all_matches_near(catfn, radius, atlasfn, photdir, outfn, projname, **kwargs):
'''
radius in arcsec
'''
T = fits_table(atlasfn)
print(len(T), 'tiles')
M = fits_table(catfn)
print(len(M), 'targets')
M.set('%s_index' % projname, np.arange(len(M)))
I,J,d = match_radec(M.ra, M.dec, T.ra, T.dec, 1.2)
#keep = np.zeros(len(M), bool)
#keep[I] = True
#assert(np.all(keep))
keep = np.zeros(len(T), bool)
keep[J] = True
T.cut(keep)
print(len(T), 'tiles near targets')
WW = []
for t in T:
fn = os.path.join(photdir, 'phot-%s.fits' % t.coadd_id)
if not os.path.exists(fn):
print('Does not exist:', fn)
continue
W = fits_table(fn)
print(len(W), 'from', fn)
if len(W) == 0:
continue
I,J,d = match_radec(M.ra, M.dec, W.ra, W.dec, radius/3600., **kwargs)
print(len(I), 'matched')
if len(I) == 0:
continue
W.cut(J)
MI = M[I]
MI.rename('ra', 'ra_%s' % projname)
MI.rename('dec', 'dec_%s' % projname)
W.add_columns_from(MI)
WW.append(W)
W = merge_tables(WW)
print(len(W), 'total matches')
W = W[np.argsort(W.get('%s_index' % projname))]
print(len(np.unique(W.objid)), 'unique objids')
W.writeto(outfn)
def main():
from thesis_code import timing
from astrometry.libkd.spherematch import match_radec
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description='DECaLS simulations.')
args = parser.parse_args()
rand = np.random.RandomState(10)
n_ref,n_other=10000,5000
ra_ref = rand.uniform(150.,150.25,n_ref)
dec_ref = rand.uniform(20.,20.25,n_ref)
ra = rand.uniform(150.,150.25,n_other)
dec = rand.uniform(20.,20.25,n_other)
i_ref,i_other,ds={},{},{}
t1=timing.now()
i_ref['astrom'],i_other['astrom'],ds['astrom']= match_radec(ra_ref.copy(),dec_ref.copy(), ra.copy(),dec.copy(), 1./3600)
t2=timing.now()
#i_ref['n2'],i_other['n2'],ds['n2']= n2_match(ra_ref.copy(),dec_ref.copy(), ra.copy(),dec.copy(), dsmax=5./3600)
i_ref['kd'],i_other['kd'],ds['kd']= kdtree(ra_ref.copy(),dec_ref.copy(), ra.copy(),dec.copy(), dsmax=1./3600)
t3=timing.now()
print 'time for astrom[sec]=',timing.diff(t1,t2)
print 'time for kd[sec]=',timing.diff(t2,t3)
for key in ['astrom','kd']:
print('%s: matched %d/%d' % (key,len(i_ref[key]),len(ra_ref)))
i=np.argsort(i_ref[key])
i_ref[key]= i_ref[key][i]
i_other[key]= i_other[key][i]
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--name1", help="Name for first data set")
parser.add_argument("--name2", help="Name for second data set")
parser.add_argument("--plot-prefix", default="compare", help='Prefix for plot filenames; default "%default"')
parser.add_argument("--match", default=1.0, help="Astrometric cross-match distance in arcsec")
parser.add_argument("dir1", help="First directory to compare")
parser.add_argument("dir2", help="Second directory to compare")
opt = parser.parse_args()
ps = PlotSequence(opt.plot_prefix)
name1 = opt.name1
if name1 is None:
name1 = os.path.basename(opt.dir1)
if not len(name1):
name1 = os.path.basename(os.path.dirname(opt.dir1))
name2 = opt.name2
if name2 is None:
name2 = os.path.basename(opt.dir2)
if not len(name2):
name2 = os.path.basename(os.path.dirname(opt.dir2))
tt = "Comparing %s to %s" % (name1, name2)
# regex for tractor-*.fits catalog filename
catre = re.compile("tractor-.*.fits")
cat1, cat2 = [], []
for basedir, cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
for dirpath, dirnames, filenames in os.walk(basedir, followlinks=True):
for fn in filenames:
if not catre.match(fn):
print("Skipping", fn, "due to filename")
continue
fn = os.path.join(dirpath, fn)
t = fits_table(fn)
print(len(t), "from", fn)
cat.append(t)
cat1 = merge_tables(cat1, columns="fillzero")
cat2 = merge_tables(cat2, columns="fillzero")
print("Total of", len(cat1), "from", name1)
print("Total of", len(cat2), "from", name2)
cat1.cut(cat1.brick_primary)
cat2.cut(cat2.brick_primary)
print("Total of", len(cat1), "BRICK_PRIMARY from", name1)
print("Total of", len(cat2), "BRICK_PRIMARY from", name2)
cat1.cut((cat1.decam_anymask[:, 1] == 0) * (cat1.decam_anymask[:, 2] == 0) * (cat1.decam_anymask[:, 4] == 0))
cat2.cut((cat2.decam_anymask[:, 1] == 0) * (cat2.decam_anymask[:, 2] == 0) * (cat2.decam_anymask[:, 4] == 0))
print("Total of", len(cat1), "unmasked from", name1)
print("Total of", len(cat2), "unmasked from", name2)
I, J, d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match / 3600.0, nearest=True)
print(len(I), "matched")
matched1 = cat1[I]
matched2 = cat2[J]
def match_two_cats(ref_cats_file,test_cats_file):
# Set the debugging level
lvl = logging.INFO
logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout)
log = logging.getLogger('__name__')
#get lists of tractor cats to compare
fns_1= read_lines(ref_cats_file)
fns_2= read_lines(test_cats_file)
log.info('Comparing tractor catalogues: ')
for one,two in zip(fns_1,fns_2): log.info("%s -- %s" % (one,two))
#if fns_1.size == 1: fns_1,fns_2= [fns_1],[fns_2]
#object to store concatenated matched tractor cats
ref_matched = []
ref_missed = []
test_matched = []
test_missed = []
d_matched= 0.
deg2= dict(ref=0.,test=0.,matched=0.)
#for cnt,cat1,cat2 in zip(range(len(fns_1)),fns_1,fns_2):
for cnt,cat1,cat2 in zip([0],[fns_1[0]],[fns_2[0]]):
log.info('Reading %s -- %s' % (cat1,cat2))
ref_tractor = Table(fits.getdata(cat1, 1))
test_tractor = Table(fits.getdata(cat2, 1))
m1, m2, d12 = match_radec(ref_tractor['ra'].copy(), ref_tractor['dec'].copy(),\
test_tractor['ra'].copy(), test_tractor['dec'].copy(), \
1.0/3600.0)
miss1 = np.delete(np.arange(len(ref_tractor)), m1, axis=0)
miss2 = np.delete(np.arange(len(test_tractor)), m2, axis=0)
log.info('matched %d/%d' % (len(m2),len(test_tractor['ra'])))
# Build combined catalogs
if len(ref_matched) == 0:
ref_matched = ref_tractor[m1]
ref_missed = ref_tractor[miss1]
test_matched = test_tractor[m2]
test_missed = test_tractor[miss2]
d_matched= d12
else:
ref_matched = vstack((ref_matched, ref_tractor[m1]))
ref_missed = vstack((ref_missed, ref_tractor[miss1]))
test_matched = vstack((test_matched, test_tractor[m2]))
test_missed = vstack((test_missed, test_tractor[miss2]))
d_matched= np.concatenate([d_matched, d12])
deg2['ref']+= deg2_lower_limit(ref_tractor['ra'],ref_tractor['dec'])
deg2['test']+= deg2_lower_limit(test_tractor['ra'],test_tractor['dec'])
deg2['matched']+= deg2_lower_limit(ref_matched['ra'],ref_matched['dec'])
return dict(ref_matched = ref_matched,
ref_missed = ref_missed,
test_matched = test_matched,
test_missed = test_missed,
d_matched= d_matched,
deg2= deg2)
def match_it(cat1, cat2):
"""cat1,2 are tractor catalogue to match objects between"""
# match cats
data_1 = targets.read_from_tractor_cat(cat1)
data_2 = targets.read_from_tractor_cat(cat2)
# deg2 spanned by objects in each data set
deg2_decam = deg2_lower_limit(data_1)
deg2_bokmos = deg2_lower_limit(data_2)
# all the 'all1' objects that have match in 'all2'
m1, m2, d12 = match_radec(data_1["ra"], data_1["dec"], data_2["ra"], data_2["dec"], 1.0 / 3600.0, nearest=True)
m1_unm = np.delete(np.arange(len(data_1["ra"])), m1, axis=0)
m2_unm = np.delete(np.arange(len(data_2["ra"])), m2, axis=0)
return data_1, data_2, m1, m2, m1_unm, m2_unm, d12, deg2_decam, deg2_bokmos
def tychoMatch(ra,dec,rad):
cat = pyf.open("tycho2-cut.fits")
data = cat[1].data
RA=data.field('RA')
DEC=data.field('DEC')
MAG=data.field('MAG')
ra1=ra
dec1=dec
matchrad = rad
I1,I2,d = sm.match_radec(ra1,dec1, RA,DEC, matchrad)
cat.close()
return RA[I2],DEC[I2],MAG[I2]
def on_bricks_dependencies(brick, survey, bricks=None):
# Find nearby bricks from earlier brick phases
if bricks is None:
bricks = survey.get_bricks_readonly()
print(len(bricks), 'bricks')
bricks = bricks[bricks.brickq < brick.brickq]
print(len(bricks), 'from phases before this brickq:', brick.brickq)
if len(bricks) == 0:
return []
from astrometry.libkd.spherematch import match_radec
radius = survey.bricksize * np.sqrt(2.) * 1.01
bricks.cut(np.abs(brick.dec - bricks.dec) < radius)
#print(len(bricks), 'within %.2f degree of Dec' % radius)
I,J,d = match_radec(brick.ra, brick.dec, bricks.ra, bricks.dec, radius)
bricks.cut(J)
print(len(bricks), 'within', radius, 'degrees')
return bricks
def bricks_touching_wcs(targetwcs, survey=None, B=None, margin=20):
'''
Finds LegacySurvey bricks touching a given WCS header object.
Parameters
----------
targetwcs : astrometry.util.Tan object or similar
The region of sky to search
survey : legacypipe.survey.LegacySurveyData object
From which the brick table will be retrieved
B : FITS table
The table of brick objects to search
margin : int
Margin in pixels around the outside of the WCS
Returns
-------
A table (subset of B, if given) containing the bricks touching the
given WCS region + margin.
'''
from astrometry.libkd.spherematch import match_radec
if B is None:
assert(survey is not None)
B = survey.get_bricks_readonly()
ra,dec = targetwcs.radec_center()
radius = targetwcs.radius()
# MAGIC 0.4 degree search radius =
# DECam hypot(1024,2048)*0.27/3600 + Brick hypot(0.25, 0.25) ~= 0.35 + margin
I,J,d = match_radec(B.ra, B.dec, ra, dec,
radius + np.hypot(0.25,0.25)/2. + 0.05)
print(len(I), 'bricks nearby')
keep = []
for i in I:
b = B[i]
brickwcs = wcs_for_brick(b)
clip = clip_wcs(targetwcs, brickwcs)
if len(clip) == 0:
print('No overlap with brick', b.brickname)
continue
keep.append(i)
return B[np.array(keep)]
def choose_field2():
import astrometry.libkd.spherematch as spherematch
fields = fits_table('window_flist.fits')
print len(ngc2000), 'NGC/IC objects'
goodngc = [n for n in ngc2000 if n.get('classification', None) == 'Gx']
print len(goodngc), 'NGC galaxies'
nra = np.array([n['ra'] for n in goodngc])
ndec = np.array([n['dec'] for n in goodngc])
rad = 8./60.
(I,J,dist) = spherematch.match_radec(nra, ndec, fields.ra, fields.dec, rad)
#sdss = DR7()
for i in np.unique(I):
ii = (I == i)
n = goodngc[i]
isngc = n['is_ngc']
num = n['id']
if (sum(ii) > 10) & (n['dec'] > 2.0):
print '<p>'
print ('NGC' if isngc else 'IC'), num, 'has', sum(ii), 'fields, and is at RA = ', n['ra'], 'Dec=', n['dec']
print '</p>'
ff = fields[J[ii]]
print 'rcfi = [',
for f in ff:
print '(', f.run, ',', f.camcol, ',', f.field, ',', f.incl, ')',
print ']'
for f in ff:
print 'Incl', f.incl, '<br />'
print ('<img src="%s" /><br /><br />' %
('http://skyservice.pha.jhu.edu/DR8/ImgCutout/getjpegcodec.aspx?R=%i&C=%i&F=%i&Z=25' % (f.run, f.camcol, f.field)))
print '<br />'
def match_two_cats(ref_cats_file, test_cats_file, debug=False):
'''return dict containing astropy Table of concatenated tractor cats
one Table for matched and missed reference and test objects, each'''
# Set the debugging level
lvl = logging.INFO
logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout)
log = logging.getLogger('__name__')
#get lists of tractor cats to compare
fns_1 = read_lines(ref_cats_file)
fns_2 = read_lines(test_cats_file)
log.info('Comparing tractor catalogues: ')
for one, two in zip(fns_1, fns_2):
log.info("%s -- %s" % (one, two))
#if fns_1.size == 1: fns_1,fns_2= [fns_1],[fns_2]
#object to store concatenated matched tractor cats
ref_matched = []
ref_missed = []
test_matched = []
test_missed = []
d_matched = 0.
deg2 = dict(ref=0., test=0., matched=0.)
# One catalogue for quick debugging
if debug: fns_1, fns_2 = fns_1[:1], fns_2[:1]
# Loop over cats
for cnt, cat1, cat2 in zip(range(len(fns_1)), fns_1, fns_2):
log.info('Reading %s -- %s' % (cat1, cat2))
ref_tractor = Table(fits.getdata(cat1, 1), masked=True)
test_tractor = Table(fits.getdata(cat2, 1), masked=True)
m1, m2, d12 = match_radec(ref_tractor['ra'].data.copy(), ref_tractor['dec'].data.copy(),\
test_tractor['ra'].data.copy(), test_tractor['dec'].data.copy(), \
1.0/3600.0)
#m1, m2, d12= kdtree_match(ref_tractor['ra'].copy(), ref_tractor['dec'].copy(),\
# test_tractor['ra'].copy(), test_tractor['dec'].copy(),\
# k=1, dsmax=1./3600)
print("Matched: %d/%d objects" % (m1.size, len(ref_tractor['ra'])))
miss1 = np.delete(np.arange(len(ref_tractor)), m1, axis=0)
miss2 = np.delete(np.arange(len(test_tractor)), m2, axis=0)
# Build combined catalogs
if len(ref_matched) == 0:
ref_matched = ref_tractor[m1]
ref_missed = ref_tractor[miss1]
test_matched = test_tractor[m2]
test_missed = test_tractor[miss2]
d_matched = d12
else:
ref_matched = vstack((ref_matched, ref_tractor[m1]),
metadata_conflicts='error')
ref_missed = vstack((ref_missed, ref_tractor[miss1]),
metadata_conflicts='error')
test_matched = vstack((test_matched, test_tractor[m2]),
metadata_conflicts='error')
test_missed = vstack((test_missed, test_tractor[miss2]),
metadata_conflicts='error')
d_matched = np.concatenate([d_matched, d12])
deg2['ref'] += deg2_lower_limit(ref_tractor['ra'], ref_tractor['dec'])
deg2['test'] += deg2_lower_limit(test_tractor['ra'],
test_tractor['dec'])
deg2['matched'] += deg2_lower_limit(ref_matched['ra'],
ref_matched['dec'])
return dict(ref_matched = ref_matched,
ref_missed = ref_missed,
test_matched = test_matched,
test_missed = test_missed), \
dict(d_matched= d_matched, deg2= deg2)
sig1 = 1. / np.sqrt(np.median(invvar[invvar > 0]))
sigoff = 3
img += sig1 * sigoff
# convert to sigmas
img /= sig1
invvar *= sig1**2
H, W = img.shape
sz = 22
# Read sources detected in DECam image too
T = fits_table(im.sefn, hdu=2)
print 'Got', len(T), 'DECam sources'
T.about()
T.ra, T.dec = wcs.pixelxy2radec(T.x_image, T.y_image)
I, J, d = match_radec(S.ra, S.dec, T.ra, T.dec, 1. / 3600., nearest=True)
print 'Matched', len(I)
# Replace SDSS RA,Dec by DECam RA,Dec
S.cut(I)
S.ra = T.ra[J]
S.dec = T.dec[J]
ok, S.x, S.y = wcs.radec2pixelxy(S.ra, S.dec)
S.x -= 1
S.y -= 1
S.ix, S.iy = np.round(S.x).astype(int), np.round(S.y).astype(int)
S.cut((S.ix >= sz) * (S.iy >= sz) * (S.ix < W - sz) * (S.iy < H - sz))
print len(S), 'SDSS stars in bounds'
S.cut(invvar[S.iy, S.ix] > 0)
def _get_sources(run,
camcol,
field,
bandname='r',
sdss=None,
release='DR7',
objs=None,
retrieve=True,
checkFiles=True,
curl=False,
roi=None,
radecroi=None,
radecrad=None,
bands=None,
badmag=25,
nanomaggies=False,
getobjs=False,
getsourceobjs=False,
getobjinds=False,
extrabands=None,
fixedComposites=False,
forcePointSources=False,
useObjcType=False,
objCuts=True,
classmap={},
ellipse=GalaxyShape,
cutToPrimary=False):
'''
If set,
radecrad = (ra,dec,rad)
returns sources within "rad" degrees of the given RA,Dec (in degrees)
center.
WARNING, this method alters the "objs" argument, if given.
Consider calling objs.copy() before calling.
-"bandname" is the SDSS band used to cut on position, select
star/gal/exp/dev, and set galaxy shapes.
-"bands" are the bands to include in the returned Source objects;
they will be initialized from the SDSS bands.
-"extrabands" are also included in the returned Source objects;
they will be initialized to the SDSS flux for either the first of
"bands", if given, or "bandname".
'''
from astrometry.sdss import (DR7, DR8, DR9, band_names, band_index,
photo_flags1_map)
# brightPointSourceThreshold=0.):
if sdss is None:
dr = dict(DR7=DR7, DR8=DR8, DR9=DR9)[release]
sdss = dr(curl=curl)
drnum = sdss.getDRNumber()
isdr7 = (drnum == 7)
if bands is None:
bands = band_names()
bandnum = band_index(bandname)
bandnums = np.array([band_index(b) for b in bands])
bandnames = bands
if extrabands is None:
extrabands = []
if objs is None:
from astrometry.util.fits import fits_table
if isdr7:
# FIXME
rerun = 0
if checkFiles:
_check_sdss_files(sdss,
run,
camcol,
field,
bandnum, ['tsObj', 'tsField'],
retrieve=retrieve,
tryopen=True)
tsf = sdss.readTsField(run, camcol, field, rerun)
objfn = sdss.getPath('tsObj',
run,
camcol,
field,
bandname,
rerun=rerun)
else:
if checkFiles:
_check_sdss_files(sdss,
run,
camcol,
field,
bandnum, ['photoObj'],
retrieve=retrieve,
tryopen=True)
objfn = sdss.getPath('photoObj', run, camcol, field)
objs = fits_table(objfn)
if objs is None:
print('No sources in SDSS file', objfn)
return []
objs.index = np.arange(len(objs))
if getobjs:
allobjs = objs.copy()
if roi is not None:
x0, x1, y0, y1 = roi
# FIXME -- we keep only the sources whose centers are within
# the ROI box. Should instead do some ellipse-overlaps
# geometry.
x = objs.colc[:, bandnum]
y = objs.rowc[:, bandnum]
objs.cut((x >= x0) * (x < x1) * (y >= y0) * (y < y1))
if radecroi is not None:
r0, r1, d0, d1 = radecroi
objs.cut((objs.ra >= r0) * (objs.ra <= r1) * (objs.dec >= d0) *
(objs.dec <= d1))
if radecrad is not None:
from astrometry.libkd.spherematch import match_radec
(ra, dec, rad) = radecrad
I, J, d = match_radec(ra, dec, objs.ra, objs.dec, rad)
objs.cut(J)
del I
del d
if objCuts:
# Only deblended children;
# No BRIGHT sources
bright = photo_flags1_map.get('BRIGHT')
objs.cut((objs.nchild == 0) * ((objs.objc_flags & bright) == 0))
if cutToPrimary:
objs.cut((objs.resolve_status & 256) > 0)
if len(objs) == 0:
sources = []
if not (getobjs or getobjinds or getsourceobjs):
return sources
rtn = [sources]
if getobjs:
rtn.append(None)
if getobjinds:
rtn.append(None)
if getsourceobjs:
rtn.append(None)
return rtn
if isdr7:
objs.rename('phi_dev', 'phi_dev_deg')
objs.rename('phi_exp', 'phi_exp_deg')
objs.rename('r_dev', 'theta_dev')
objs.rename('r_exp', 'theta_exp')
# SDSS and Tractor have different opinions on which way this rotation goes
objs.phi_dev_deg *= -1.
objs.phi_exp_deg *= -1.
# MAGIC -- minimum size of galaxy.
objs.theta_dev = np.maximum(objs.theta_dev, 1. / 30.)
objs.theta_exp = np.maximum(objs.theta_exp, 1. / 30.)
if forcePointSources:
Lstar = np.ones(len(objs), float)
Lgal = np.zeros(len(objs), float)
Ldev = Lexp = Lgal
else:
if useObjcType:
objs.cut(np.logical_or(objs.objc_type == 6, objs.objc_type == 3))
Lstar = (objs.objc_type == 6)
Lgal = (objs.objc_type == 3)
else:
Lstar = (objs.prob_psf[:, bandnum] == 1) * 1.0
Lgal = (objs.prob_psf[:, bandnum] == 0) * 1.0
if isdr7:
fracdev = objs.fracpsf[:, bandnum]
else:
fracdev = objs.fracdev[:, bandnum]
Ldev = Lgal * fracdev
Lexp = Lgal * (1. - fracdev)
if isdr7:
from .sdss_dr7 import _dr7_getBrightness
if nanomaggies:
raise RuntimeError('Nanomaggies not supported for DR7 (yet)')
def lup2bright(lups):
counts = [
tsf.luptitude_to_counts(lup, j) for j, lup in enumerate(lups)
]
counts = np.array(counts)
bright = _dr7_getBrightness(counts, tsf, bandnames, extrabands)
return bright
flux2bright = lup2bright
starflux = objs.psfcounts
compflux = objs.counts_model
devflux = objs.counts_dev
expflux = objs.counts_exp
def comp2bright(lups, Ldev, Lexp):
counts = [
tsf.luptitude_to_counts(lup, j) for j, lup in enumerate(lups)
]
counts = np.array(counts)
dcounts = counts * Ldev
ecounts = counts * Lexp
dbright = _dr7_getBrightness(dcounts, tsf, bands, extrabands)
ebright = _dr7_getBrightness(ecounts, tsf, bands, extrabands)
return dbright, ebright
else:
def nmgy2bright(flux):
if len(bandnums):
flux = flux[bandnums]
else:
flux = flux[np.array([bandnum])]
bb = bandnames + extrabands
if nanomaggies:
if len(extrabands):
if len(bandnums) == 0:
# Only "extrabands", no SDSS bands.
flux = np.zeros(len(extrabands)) + flux[0]
else:
flux = np.append(flux, np.zeros(len(extrabands)))
bright = NanoMaggies(order=bb, **dict(zip(bb, flux)))
else:
I = (flux > 0)
mag = np.zeros_like(flux) + badmag
mag[I] = sdss.nmgy_to_mag(flux[I])
if len(extrabands):
mag = np.append(mag, np.zeros(len(extrabands)) + badmag)
bright = Mags(order=bb, **dict(zip(bb, mag)))
return bright
def comp2bright(flux, Ldev, Lexp):
dflux = flux * Ldev
eflux = flux * Lexp
dbright = nmgy2bright(dflux)
ebright = nmgy2bright(eflux)
return dbright, ebright
flux2bright = nmgy2bright
starflux = objs.psfflux
compflux = objs.cmodelflux
devflux = objs.devflux
expflux = objs.expflux
sources = []
nstars, ndev, nexp, ncomp = 0, 0, 0, 0
isources = []
ptsrcclass = classmap.get(PointSource, PointSource)
for i in range(len(objs)):
if Lstar[i]:
pos = RaDecPos(objs.ra[i], objs.dec[i])
flux = starflux[i, :]
bright = flux2bright(flux)
# This should work, I just don't feel like testing it now...
# if brightPointSourceThreshold:
# ps = SdssPointSource(pos, bright, thresh=brightPointSourceThreshold)
# else:
# ps = PointSource(pos, bright)
sources.append(ptsrcclass(pos, bright))
nstars += 1
isources.append(i)
continue
hasdev = (Ldev[i] > 0)
hasexp = (Lexp[i] > 0)
iscomp = (hasdev and hasexp)
pos = RaDecPos(objs.ra[i], objs.dec[i])
if iscomp:
flux = compflux[i, :]
elif hasdev:
flux = devflux[i, :]
elif hasexp:
flux = expflux[i, :]
else:
print(
'Skipping object with Lstar = %g, Ldev = %g, Lexp = %g (fracdev=%g)'
% (Lstar[i], Ldev[i], Lexp[i], fracdev[i]))
continue
isources.append(i)
if iscomp:
if fixedComposites:
bright = flux2bright(flux)
fdev = (Ldev[i] / (Ldev[i] + Lexp[i]))
else:
dbright, ebright = comp2bright(flux, Ldev[i], Lexp[i])
else:
bright = flux2bright(flux)
if hasdev:
re = objs.theta_dev[i, bandnum]
ab = objs.ab_dev[i, bandnum]
phi = objs.phi_dev_deg[i, bandnum]
dshape = ellipse(re, ab, phi)
if hasexp:
re = objs.theta_exp[i, bandnum]
ab = objs.ab_exp[i, bandnum]
phi = objs.phi_exp_deg[i, bandnum]
eshape = ellipse(re, ab, phi)
if iscomp:
if fixedComposites:
gal = FixedCompositeGalaxy(pos, bright, fdev, eshape, dshape)
else:
gal = CompositeGalaxy(pos, ebright, eshape, dbright, dshape)
ncomp += 1
elif hasdev:
gal = DevGalaxy(pos, bright, dshape)
ndev += 1
elif hasexp:
gal = ExpGalaxy(pos, bright, eshape)
nexp += 1
sources.append(gal)
print(
'Created',
ndev,
'deV,',
nexp,
'exp,',
ncomp,
'composite',
)
print('(total %i) galaxies and %i stars' % (ndev + nexp + ncomp, nstars))
if not (getobjs or getobjinds or getsourceobjs):
return sources
if nstars + ndev + nexp + ncomp < len(objs):
objs = objs[np.array(isources)]
rtn = [sources]
if getobjs:
rtn.append(allobjs)
if getobjinds:
rtn.append(objs.index if len(objs) else np.array([]))
if getsourceobjs:
rtn.append(objs)
return rtn
def get_footprint_object(self):
"""Returns footprint object 'sfd'"""
# work with SFD map and Decals/Mzls tiles
# lonlat from SFD healpix is in galactic coords, convert this to Celestial
hdu = fitsio.FITS(os.path.join(self.map_dir, 'lambda_sfd_ebv.fits'))
sfd = EmptyClass()
temp = hdu[1].read()
sfd.temp = temp['TEMPERATURE']
npix = Healpix().get_nside(len(sfd.temp))
assert (npix == 512)
sfd.l_indeg, sfd.b_indeg = hp.pix2ang(512,
np.where(sfd.temp > 0)[0],
nest=True,
lonlat=True)
#inPlane= np.where((sfd_gal_dec > -20) & (sfd_gal_dec < 20))[0]
trans = Galactic(l=sfd.l_indeg * units.degree,
b=sfd.b_indeg * units.degree)
radec = trans.transform_to(ICRS)
sfd.ra, sfd.dec = radec.ra.value, radec.dec.value
all_tiles = fits_table(
os.path.join(self.tile_dir, 'mosaic-tiles_obstatus.fits'))
wdes_tiles = fits_table(
os.path.join(self.tile_dir, 'decam-tiles_obstatus.fits'))
inDESI = ((all_tiles.in_desi_orig == 1) | (all_tiles.in_desi == 1))
inDecals = ((inDESI) & (all_tiles.dec <= 30.))
#(mzls_decals.in_des == 0))
inMzls = ((inDESI) & (all_tiles.dec > 30.))
#(mzls_decals.in_des == 0))
inDes = ((wdes_tiles.in_desi_orig == 1) | (wdes_tiles.in_desi == 1))
inDes = ((inDes) & (wdes_tiles.in_des == 1))
#above30= mzls.dec > 30.
#inDESI= ( (mzls.in_desi_orig == 1) |
# (mzls.in_desi == 1))
#inMzls= ( (above30) &
# (inDESI))
#desi= merge_tables([mzls,decals],columns='fillzero')
des = wdes_tiles.copy()
del wdes_tiles
des.cut(inDes)
mzls = all_tiles.copy()
decals = all_tiles.copy()
del all_tiles
mzls.cut(inMzls)
decals.cut(inDecals)
ps = Healpix().get_pixscale(len(sfd.temp), unit='deg')
# match_radec(ref,obs): for each point in ref, return matching point in obs
print('matching tiles to healpix centers')
I, J, d = match_radec(mzls.ra, mzls.dec, sfd.ra, sfd.dec, ps * 8)
sfd.ipix_mzls = list(set(J))
I, J, d = match_radec(decals.ra, decals.dec, sfd.ra, sfd.dec, ps * 8)
sfd.ipix_decals = list(set(J))
I, J, d = match_radec(des.ra, des.dec, sfd.ra, sfd.dec, ps * 8)
sfd.ipix_des = list(set(J))
return sfd
# legasurvey pts fill in in ps*3
I, J, d = match_radec(legsurvey.ra, legsurvey.dec, sfd.ra, sfd.dec,
ps * 3)
sfd.ipix_legsurvey = set(J)
# des fills in with ps*8
I, J, d = match_radec(des.ra, des.dec, sfd.ra, sfd.dec, ps * 8)
sfd.ipix_legsurvey.union(set(J))
sfd.ipix_legsurvey = list(sfd.ipix_legsurvey)
return sfd
def main(args):
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).")
parser.add_argument('--calibs', action='store_true',
help='Output CCDs that need to be calibrated.')
parser.add_argument('--nper', type=int, default=None,
help='Batch N calibs per line')
parser.add_argument('--byexp', action='store_true', default=False,
help='Run one whole exposure per job (not one CCD per job)')
parser.add_argument('--forced', action='store_true',
help='Output forced-photometry commands')
parser.add_argument('--lsb', action='store_true',
help='Output Low-Surface-Brightness commands')
parser.add_argument('--stage', help='Stage image files to given directory')
parser.add_argument('--touching', action='store_true',
help='Cut to only CCDs touching selected bricks')
parser.add_argument('--near', action='store_true',
help='Quick cut to only CCDs near selected bricks')
parser.add_argument('--check-coadd', action='store_true',
help='Check which coadds actually need to run.')
parser.add_argument('--out', help='Output filename for calibs, default %(default)s',
default='jobs')
parser.add_argument('--command', action='store_true',
help='Write out full command-line to run calib')
parser.add_argument('--opt', help='With --command, extra options to add')
parser.add_argument('--maxra', type=float, help='Maximum RA to run')
parser.add_argument('--minra', type=float, help='Minimum RA to run')
parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
parser.add_argument('--mindec', type=float, help='Minimum Dec to run')
parser.add_argument('--region', help='Region to select')
parser.add_argument('--bricks', help='Set bricks.fits file to load')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--ignore_cuts', action='store_true',default=False,help='no photometric cuts')
parser.add_argument('--save_to_fits', action='store_true',default=False,help='save cut brick,ccd to fits table')
parser.add_argument('--name', action='store',default='dr3',help='save with this suffix, e.g. refers to ccds table')
parser.add_argument('--delete-sky', action='store_true',
help='Delete any existing sky calibration files')
parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')
parser.add_argument('--nccds', action='store_true', default=False, help='Prints number of CCDs per brick')
parser.add_argument('--bands', default='g,r,z', help='Set bands to keep: comma-separated list.')
opt = parser.parse_args(args)
want_ccds = (opt.calibs or opt.forced or opt.lsb)
want_bricks = not want_ccds
survey = LegacySurveyData()
if opt.bricks is not None:
B = fits_table(opt.bricks)
log('Read', len(B), 'from', opt.bricks)
else:
B = survey.get_bricks()
log('Bricks Dec range:', B.dec.min(), B.dec.max())
if opt.ccds is not None:
T = fits_table(opt.ccds)
log('Read', len(T), 'from', opt.ccds)
else:
T = survey.get_ccds()
log(len(T), 'CCDs')
T.index = np.arange(len(T))
if opt.ignore_cuts == False:
log('Applying CCD cuts...')
if 'ccd_cuts' in T.columns():
T.cut(T.ccd_cuts == 0)
log(len(T), 'CCDs survive cuts')
bands = opt.bands.split(',')
log('Filters:', np.unique(T.filter))
T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
log('Cut to', len(T), 'CCDs in filters', bands)
log('CCDs Dec range:', T.dec.min(), T.dec.max())
# I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
# plt.clf()
# plt.hist(counts, counts.max()+1)
# plt.savefig('bricks.png')
# B.cut(I[counts >= 9])
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# #plt.scatter(B.ra[I], B.dec[I], c=counts)
# plt.savefig('bricks2.png')
# DES Stripe82
#rlo,rhi = 350.,360.
# rlo,rhi = 300., 10.
# dlo,dhi = -6., 4.
# TINY bit
#rlo,rhi = 350.,351.1
#dlo,dhi = 0., 1.1
# EDR+
# 860 bricks
# ~10,000 CCDs
#rlo,rhi = 239,246
#dlo,dhi = 5, 13
# DR1
#rlo,rhi = 0, 360
# part 1
#dlo,dhi = 25, 40
# part 2
#dlo,dhi = 20,25
# part 3
#dlo,dhi = 15,20
# part 4
#dlo,dhi = 10,15
# part 5
#dlo,dhi = 5,10
# the rest
#dlo,dhi = -11, 5
#dlo,dhi = 15,25.5
dlo,dhi = -25, 40
rlo,rhi = 0, 360
# Arjun says 3x3 coverage area is roughly
# RA=240-252 DEC=6-12 (but not completely rectangular)
# COSMOS
#rlo,rhi = 148.9, 151.2
#dlo,dhi = 0.9, 3.5
# A nice well-behaved region (EDR2/3)
# rlo,rhi = 243.6, 244.6
# dlo,dhi = 8.1, 8.6
# 56 bricks, ~725 CCDs
#B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
# 240 bricks, ~3000 CCDs
#B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
# 535 bricks, ~7000 CCDs
#B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))
if opt.region in ['test1', 'test2', 'test3', 'test4']:
nm = dict(test1='2446p115', # weird stuff around bright star
test2='1183p292', # faint sources around bright galaxy
test3='3503p005', # DES
test4='1163p277', # Pollux
)[opt.region]
B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
log('Cut to', len(B), 'bricks')
log(B.ra, B.dec)
dlo,dhi = -90,90
rlo,rhi = 0, 360
elif opt.region == 'edr':
# EDR:
# 535 bricks, ~7000 CCDs
rlo,rhi = 240,245
dlo,dhi = 5, 12
elif opt.region == 'dr8-decam':
rlo,rhi = 0, 360
dlo,dhi = -70, 40
log('DR8-DECam region')
elif opt.region == 'edrplus':
rlo,rhi = 235,248
dlo,dhi = 5, 15
elif opt.region == 'edr-south':
rlo,rhi = 240,245
dlo,dhi = 5, 10
elif opt.region == 'cosmos1':
# 16 bricks in the core of the COSMOS field.
rlo,rhi = 149.75, 150.75
dlo,dhi = 1.6, 2.6
elif opt.region == 'pristine':
# Stream?
rlo,rhi = 240,250
dlo,dhi = 10,15
elif opt.region == 'des':
dlo, dhi = -6., 4.
rlo, rhi = 317., 7.
T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
log('Cut to', len(T), 'CCDs with "CPDES82" in filename')
elif opt.region == 'subdes':
rlo,rhi = 320., 360.
dlo,dhi = -1.25, 1.25
elif opt.region == 'northwest':
rlo,rhi = 240,360
dlo,dhi = 20,40
elif opt.region == 'north':
rlo,rhi = 120,240
dlo,dhi = 20,40
elif opt.region == 'northeast':
rlo,rhi = 0,120
dlo,dhi = 20,40
elif opt.region == 'southwest':
rlo,rhi = 240,360
dlo,dhi = -20,0
elif opt.region == 'south':
rlo,rhi = 120,240
dlo,dhi = -20,0
elif opt.region == 'southeast':
rlo,rhi = 0,120
dlo,dhi = -20,0
elif opt.region == 'southsoutheast':
rlo,rhi = 0,120
dlo,dhi = -20,-10
elif opt.region == 'midwest':
rlo,rhi = 240,360
dlo,dhi = 0,20
elif opt.region == 'middle':
rlo,rhi = 120,240
dlo,dhi = 0,20
elif opt.region == 'mideast':
rlo,rhi = 0,120
dlo,dhi = 0,20
elif opt.region == 'grz':
# Bricks with grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
log('Cut to', len(B), 'bricks with grz coverage')
elif opt.region == 'nogrz':
# Bricks without grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
log('Cut to', len(B), 'bricks withOUT grz coverage')
elif opt.region == 'deep2':
rlo,rhi = 250,260
dlo,dhi = 30,35
elif opt.region == 'deep2f2':
rlo,rhi = 251.4, 254.4
dlo,dhi = 34.6, 35.3
elif opt.region == 'deep2f3':
rlo,rhi = 351.25, 353.75
dlo,dhi = 0, 0.5
elif opt.region == 'deep3':
rlo,rhi = 214,216
dlo,dhi = 52.25,53.25
elif opt.region == 'virgo':
rlo,rhi = 185,190
dlo,dhi = 10, 15
elif opt.region == 'virgo2':
rlo,rhi = 182,192
dlo,dhi = 8, 18
elif opt.region == 'coma':
# van Dokkum et al Coma cluster ultra-diffuse galaxies: 3x3 field centered on Coma cluster
rc,dc = 195., 28.
dd = 1.5
cosdec = np.cos(np.deg2rad(dc))
rlo,rhi = rc - dd/cosdec, rc + dd/cosdec
dlo,dhi = dc - dd, dc + dd
elif opt.region == 'lsb':
rlo,rhi = 147.2, 147.8
dlo,dhi = -0.4, 0.4
elif opt.region == 'eboss-sgc':
# generous boundaries to make sure get all relevant images
# RA -45 to +45
# Dec -5 to +7
rlo,rhi = 310., 50.
dlo,dhi = -6., 6.
elif opt.region == 'eboss-ngc':
# generous boundaries to make sure get all relevant images
# NGC ELGs
# RA 115 to 175
# Dec 15 to 30
# rlo,rhi = 122., 177.
# dlo,dhi = 12., 32.
rlo,rhi = 126., 168.
dlo,dhi = 18., 33.
elif opt.region == 'mzls':
dlo,dhi = -10., 90. # -10: pull in Stripe 82 data too
elif opt.region == 'dr4-bootes':
# https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
#dlo,dhi = 34., 35.
#rlo,rhi = 209.5, 210.5
dlo,dhi = 33., 36.
rlo,rhi = 216.5, 219.5
elif opt.region == 'des-sn-x3':
#rlo,rhi = 36., 37.
#dlo,dhi = -5., -4.
rlo,rhi = 36., 36.5
dlo,dhi = -4.5, -4.
elif opt.region == 'ngc2632':
# open cluster
rlo,rhi = 129.0, 131.0
dlo,dhi = 19.0, 20.5
elif opt.region == 'dr8sky':
rlo,rhi = 35.0, 37.0
dlo,dhi = -3.0, -1.0
# ADM DR8 test regions, see, e.g.:
# https://desi.lbl.gov/trac/wiki/DecamLegacy/DR8#Testregions
elif opt.region == 'dr8-test-s82':
rlo, rhi = 0, 45
dlo, dhi = -1.25, 1.25
elif opt.region == 'dr8-test-hsc-sgc':
rlo, rhi = 30, 40
dlo, dhi = -6.5, -1.25
elif opt.region == 'dr8-test-hsc-ngc':
rlo, rhi = 177.5, 182.5
dlo, dhi = -1, 1
elif opt.region == 'dr8-test-edr':
rlo, rhi = 240, 245
dlo, dhi = 5, 12
elif opt.region == 'dr8-test-hsc-north':
rlo, rhi = 240, 250
dlo, dhi = 42, 45
elif opt.region == 'dr8-test-deep2-egs':
rlo, rhi = 213, 216.5
dlo, dhi = 52, 54
elif opt.region == 'dr8-test-overlap':
rlo, rhi = 132, 140.5
dlo, dhi = 31.5, 35
if opt.mindec is not None:
dlo = opt.mindec
if opt.maxdec is not None:
dhi = opt.maxdec
if opt.minra is not None:
rlo = opt.minra
if opt.maxra is not None:
rhi = opt.maxra
if rlo < rhi:
B.cut((B.ra >= rlo) * (B.ra <= rhi) *
(B.dec >= dlo) * (B.dec <= dhi))
else: # RA wrap
B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) *
(B.dec >= dlo) * (B.dec <= dhi))
log(len(B), 'bricks in range; cut Dec range', B.dec.min(), B.dec.max())
#for name in B.get('brickname'):
# print(name)
#B.writeto('bricks-cut.fits')
bricksize = 0.25
# A bit more than 0.25-degree brick radius + Bok image radius ~ 0.57
search_radius = 1.05 * np.sqrt(2.) * (bricksize +
(0.455 * 4096 / 3600.))/2.
log(len(T), 'CCDs')
log(len(B), 'Bricks')
I,J,_ = match_radec(B.ra, B.dec, T.ra, T.dec, search_radius,
nearest=True)
B.cut(I)
log('Cut to', len(B), 'bricks near CCDs')
log('Bricks Dec range:', B.dec.min(), B.dec.max())
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# plt.title('DR3 bricks')
# plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
# plt.savefig('bricks.png')
if opt.touching:
I,J,_ = match_radec(T.ra, T.dec, B.ra, B.dec, search_radius,
nearest=True)
# list the ones that will be cut
# drop = np.ones(len(T))
# drop[I] = False
# for i in np.flatnonzero(drop):
# from astrometry.util.starutil_numpy import degrees_between
# dists = degrees_between(B.ra, B.dec, T.ra[i], T.dec[i])
# mindist = min(dists)
# print('Dropping:', T.ra[i], T.dec[i], 'min dist', mindist, 'search_radius', search_radius)
T.cut(I)
log('Cut to', len(T), 'CCDs near bricks')
if opt.forced:
log('Writing forced-photometry commands to', opt.out)
f = open(opt.out,'w')
log('Total of', len(T), 'CCDs')
#T.cut(allI)
camexp = set([(c,e) for c,e in zip(T.camera, T.expnum)])
print(len(camexp), 'unique camera/exposure pairs')
for cam,exp in camexp:
#expstr = '%08i' % exp
#outfn = os.path.join('forced', cam, expstr[:5], 'forced-%s-%s.fits' % (cam, exp))
#f.write('%s %s all %s\n' % (cam, exp, outfn))
f.write('%s %s\n' % (cam, exp))
f.close()
log('Wrote', opt.out)
return 0
# sort by RA increasing
B.cut(np.argsort(B.ra))
if opt.save_to_fits:
assert(opt.touching)
# Write cut tables to file
for tab,typ in zip([B,T],['bricks','ccds']):
fn='%s-%s-cut.fits' % (typ,opt.region)
if os.path.exists(fn):
os.remove(fn)
tab.writeto(fn)
log('Wrote %s' % fn)
# Write text files listing ccd and filename names
# nm1,nm2= 'ccds-%s.txt'% opt.region,'filenames-%s.txt' % opt.region
# if os.path.exists(nm1):
# os.remove(nm1)
# if os.path.exists(nm2):
# os.remove(nm2)
# f1,f2=open(nm1,'w'),open(nm2,'w')
# fns= list(set(T.get('image_filename')))
# for fn in fns:
# f2.write('%s\n' % fn.strip())
# for ti in T:
# f1.write('%s\n' % ti.get('image_filename').strip())
# f1.close()
# f2.close()
# log('Wrote *-names.txt')
if opt.touching:
if want_bricks:
# Shortcut the list of bricks that are definitely touching CCDs --
# a brick-ccd pair within this radius must be touching.
closest_radius = 0.95 * (bricksize + 0.262 * 2048 / 3600.) / 2.
J1,_,_ = match_radec(B.ra, B.dec, T.ra, T.dec, closest_radius, nearest=True)
log(len(J1), 'of', len(B), 'bricks definitely touch CCDs')
tocheck = np.ones(len(B), bool)
tocheck[J1] = False
J2 = []
for j in np.flatnonzero(tocheck):
b = B[j]
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickname)
if len(I) == 0:
continue
J2.append(j)
J = np.hstack((J1, J2))
J = np.sort(J).astype(int)
B.cut(J)
log('Cut to', len(B), 'bricks touching CCDs')
else:
J = []
allI = set()
for j,b in enumerate(B):
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickname)
if len(I) == 0:
continue
allI.update(I)
J.append(j)
allI = list(allI)
allI.sort()
B.cut(np.array(J))
log('Cut to', len(B), 'bricks touching CCDs')
elif opt.near:
# Find CCDs near bricks
allI,_,_ = match_radec(T.ra, T.dec, B.ra, B.dec, search_radius, nearest=True)
# Find bricks near CCDs
J,_,_ = match_radec(B.ra, B.dec, T.ra, T.dec, search_radius, nearest=True)
B.cut(J)
log('Cut to', len(B), 'bricks near CCDs')
else:
allI = np.arange(len(T))
if opt.byexp:
_,eI = np.unique(T.expnum[allI], return_index=True)
allI = allI[eI]
print('Cut to', len(allI), 'expnums')
if opt.nccds:
from queue import Queue
from threading import Thread
log('Checking number of CCDs per brick')
def worker():
while True:
i = q.get()
if i is None:
break
b = B[i]
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(b.brickname, len(I))
q.task_done()
q = Queue()
num_threads = 24
threads = []
for i in range(num_threads):
t = Thread(target=worker)
t.start()
threads.append(t)
for i in range(len(B)):
q.put(i)
q.join()
for i in range(num_threads):
q.put(None)
for t in threads:
t.join()
if opt.write_ccds:
T[allI].writeto(opt.write_ccds)
log('Wrote', opt.write_ccds)
if want_bricks:
# Print the list of bricks and exit.
for b in B:
print(b.brickname)
if opt.save_to_fits:
B.writeto('bricks-%s-touching.fits' % opt.region)
if not want_ccds:
return 0
## Be careful here -- T has been cut; we want to write out T.index.
## 'allI' contains indices into T.
if opt.stage is not None:
cmd_pat = 'rsync -LRarv %s %s'
fns = set()
for iccd in allI:
im = survey.get_image_object(T[iccd])
fns.update([im.imgfn, im.wtfn, im.dqfn, im.psffn, im.merged_psffn,
im.merged_skyfn, im.skyfn])
for i,fn in enumerate(fns):
print('File', i+1, 'of', len(fns), ':', fn)
if not os.path.exists(fn):
print('No such file:', fn)
continue
base = survey.get_survey_dir()
if base.endswith('/'):
base = base[:-1]
rel = os.path.relpath(fn, base)
dest = os.path.join(opt.stage, rel)
print('Dest:', dest)
if os.path.exists(dest):
print('Exists:', dest)
continue
cmd = cmd_pat % ('%s/./%s' % (base, rel), opt.stage)
print(cmd)
rtn = os.system(cmd)
assert(rtn == 0)
return 0
if opt.lsb:
log('Writing LSB commands to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
for j,i in enumerate(allI):
exp = T.expnum[i]
ext = T.ccdname[i].strip()
outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
f.write('python legacyanalysis/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext))
f.close()
log('Wrote', opt.out)
return 0
log('Writing calibs to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
batch = []
def write_batch(f, batch, cmd):
if cmd is None:
cmd = ''
f.write(cmd + ' '.join(batch) + '\n')
cmd = None
if opt.command:
cmd = 'python legacypipe/run-calib.py '
if opt.opt is not None:
cmd += opt.opt + ' '
for j,i in enumerate(allI):
if opt.delete_sky:
log(j+1, 'of', len(allI))
im = survey.get_image_object(T[i])
if opt.delete_sky and os.path.exists(im.skyfn):
log(' deleting:', im.skyfn)
os.unlink(im.skyfn)
if opt.command:
if opt.byexp:
s = '--expnum %i' % (T.expnum[i])
else:
s = '%i-%s' % (T.expnum[i], T.ccdname[i])
prefix = 'python legacypipe/run-calib.py '
if opt.opt is not None:
prefix = prefix + opt.opt
#('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
# (T.expnum[i], T.ccdname[i]))
else:
s = '%i' % T.index[i]
prefix = ''
if j < 10:
print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i],
'filename', T.image_filename[i])
if not opt.nper:
f.write(prefix + s + '\n')
else:
batch.append(s)
if len(batch) >= opt.nper:
write_batch(f, batch, cmd)
batch = []
if len(batch):
write_batch(f, batch, cmd)
f.close()
log('Wrote', opt.out)
return 0
plt.title('Coverage (RGB = z/r/g)')
plt.xlabel('RA (deg)')
plt.ylabel('Dec (deg)')
ps.savefig()
overfn = 'overlapping-ccds.fits'
if os.path.exists(overfn):
M = fits_table(overfn)
M.rename('i','I')
M.rename('j','J')
else:
from astrometry.libkd.spherematch import match_radec
radius = np.hypot(2048, 4096) * 0.262/3600.
M = fits_table()
M.I,M.J,d = match_radec(A.ra, A.dec, A.ra, A.dec, radius, notself=True)
M.cut(M.I < M.J)
# ra_centers vs dra
cosd = np.cos(np.deg2rad(A.dec_center[M.I]))
x1 = np.cos(np.deg2rad(A.ra_center[M.I]))
y1 = np.sin(np.deg2rad(A.ra_center[M.I]))
x2 = np.cos(np.deg2rad(A.ra_center[M.J]))
y2 = np.sin(np.deg2rad(A.ra_center[M.J]))
dra = np.rad2deg(cosd * np.hypot(x1 - x2, y1 - y2))
# widths - dra = overlap
M.raoverlap = A.dra[M.I] + A.dra[M.J] - dra
M.cut(M.raoverlap > 0)
ddec = np.abs(A.dec_center[M.I] - A.dec_center[M.J])
# heights - ddec = overlap
M.decoverlap = A.ddec[M.I] + A.ddec[M.J] - ddec
print 'Looking for', sefn
morph = fits_table(sefn, hdu=2)
wcs = Sip(im.wcsfn)
if len(sdss) == 0:
print 'EMPTY:', im.sdssfn
continue
if len(morph) == 0:
print 'EMPTY:', im.morphfn
continue
print len(sdss), 'SDSS sources from', im.sdssfn
print len(morph), 'SE sources from', sefn
morph.ra,morph.dec = wcs.pixelxy2radec(morph.x_image, morph.y_image)
I,J,d = match_radec(morph.ra, morph.dec, sdss.ra, sdss.dec, 0.5/3600.)
corr = sdss[J]
corr.add_columns_from(morph[I])
chipnames.append(im.extname)
#corr = fits_table(im.corrfn)
corrs.append(corr)
print im, ':', len(corr), 'correspondences'
for col,cut in ([('flux_auto',None)] + [('flux_aper',i) for i in range(3)]
+ [('flux_psf',None), ('flux_model',None)]):
dmags = []
smags = []
for corr in corrs:
if not col in corr.get_columns():
sig1 = 1.0 / np.sqrt(np.median(invvar[invvar > 0]))
sigoff = 3
img += sig1 * sigoff
# convert to sigmas
img /= sig1
invvar *= sig1 ** 2
H, W = img.shape
sz = 22
# Read sources detected in DECam image too
T = fits_table(im.sefn, hdu=2)
print "Got", len(T), "DECam sources"
T.about()
T.ra, T.dec = wcs.pixelxy2radec(T.x_image, T.y_image)
I, J, d = match_radec(S.ra, S.dec, T.ra, T.dec, 1.0 / 3600.0, nearest=True)
print "Matched", len(I)
# Replace SDSS RA,Dec by DECam RA,Dec
S.cut(I)
S.ra = T.ra[J]
S.dec = T.dec[J]
ok, S.x, S.y = wcs.radec2pixelxy(S.ra, S.dec)
S.x -= 1
S.y -= 1
S.ix, S.iy = np.round(S.x).astype(int), np.round(S.y).astype(int)
S.cut((S.ix >= sz) * (S.iy >= sz) * (S.ix < W - sz) * (S.iy < H - sz))
print len(S), "SDSS stars in bounds"
S.cut(invvar[S.iy, S.ix] > 0)
def process_file(self, path):
print('Reading', path)
F = fitsio.FITS(path)
primhdr = F[0].read_header()
print(len(F), 'extensions')
# measure_raw . DECamMeasurer or Mosaic3Measurer
meas_class = get_measurer_class_for_file(path)
if meas_class is None:
print('Failed to identify camera in', path)
return
meas = meas_class(path, 0, self.nom)
# We read the WCS headers from all extensions and then spherematch
# against the catalog.
rr, dd = [], []
exts = np.arange(1, len(F))
keep_exts = []
for ext in exts:
hdr = F[ext].read_header()
extname = hdr['EXTNAME'].strip()
# HACK -- skip DECam focus chips.
if 'F' in extname:
continue
meas.ext = ext
meas.primhdr = primhdr
wcs = meas.get_wcs(hdr)
#print('WCS:', wcs)
rc, dc = wcs.radec_center()
radius = wcs.radius()
#print('RA,Dec center', rc, dc, 'radius', radius)
rr.append(rc)
dd.append(dc)
keep_exts.append(ext)
exts = keep_exts
# we use the last extension's 'radius' here...
rr = np.array(rr)
dd = np.array(dd)
if self.match_ngc(rr, dd, radius, exts, F, primhdr, meas):
return
print('Not doing SDSS spectro jazz.')
return
# No match with NGC catalog -- look at SDSS spectro objects.
I, J, d = match_radec(rr, dd, self.spec.ra, self.spec.dec, radius)
print(len(I), 'matches to spectra')
if len(I) == 0:
return False
#meas_exts = set()
#K = []
measures = {}
specobjs = []
for k, (i, j) in enumerate(zip(I, J)):
ext = exts[i]
obj = self.spec[j]
obj.name = obj.label.strip()
hdr = F[ext].read_header()
extname = hdr['EXTNAME'].strip()
expnum = primhdr['EXPNUM']
wcs = meas.get_wcs(hdr)
ok, x, y = wcs.radec2pixelxy(obj.ra, obj.dec)
x = x - 1
y = y - 1
# Choose cutout area
pixrad = 1.4 * 50
r = pixrad
tt = '%s in exp %i ext %s (%i)' % (obj.name, expnum, extname, ext)
print(tt)
# Find the cutout region... does it actually overlap the chip?
H, W = wcs.shape
xl, xh = int(np.clip(x - r, 0,
W - 1)), int(np.clip(x + r, 0, W - 1))
yl, yh = int(np.clip(y - r, 0,
H - 1)), int(np.clip(y + r, 0, H - 1))
if xl == xh or yl == yh:
print('no actual overlap with image')
continue
sh, sw = yh - yl, xh - xl
if sh < 25 or sw < 25:
print('tiny overlap', sw, 'x', sh)
continue
#meas_exts.add(ext)
#K.append(k)
if ext in measures:
M = measures[ext]
else:
# Measure the image!
meas.ext = ext
meas.edge_trim = 20
try:
M = meas.run(n_fwhm=1, verbose=False, get_image=True)
except KeyboardInterrupt:
sys.exit(0)
except:
import traceback
print('Failed to measure file', path, 'ext', ext, ':')
traceback.print_exc()
continue
measures[ext] = M
raw = M['image']
# Now repeat the cutout check with the trimmed image
wcs = M['wcs']
# Trim WCS to trimmed raw image shape
trim_x0, trim_y0 = M['trim_x0'], M['trim_y0']
H, W = raw.shape
wcs = wcs.get_subimage(trim_x0, trim_y0, W, H)
ok, x, y = wcs.radec2pixelxy(obj.ra, obj.dec)
x = x - 1
y = y - 1
xl, xh = int(np.clip(x - r, 0,
W - 1)), int(np.clip(x + r, 0, W - 1))
yl, yh = int(np.clip(y - r, 0,
H - 1)), int(np.clip(y + r, 0, H - 1))
if xl == xh or yl == yh:
print('no actual overlap with image')
continue
subimg = raw[yl:yh, xl:xh]
sh, sw = subimg.shape
if sh < 25 or sw < 25:
print('tiny overlap', sw, 'x', sh)
continue
subwcs = wcs.get_subimage(xl, yl, sw, sh)
# Astrometric shifts
dx = M['dx']
dy = M['dy']
aff = M['affine']
x = (xl + xh) / 2
y = (yl + yh) / 2
#print('Affine correction terms:', aff)
adx = x - aff[0]
ady = y - aff[1]
corrx = aff[2] + aff[3] * adx + aff[4] * ady - adx
corry = aff[5] + aff[6] * adx + aff[7] * ady - ady
print('Affine correction', corrx, corry)
# Shift the 'subwcs' to account for astrometric offset
cx, cy = subwcs.get_crpix()
subwcs.set_crpix((cx - dx - corrx, cy - dy - corry))
specobjs.append((tt, subwcs, subimg))
# # What size of image are we going to request?
# scale = 1.
#
# rh,rw = int(np.ceil(sh/scale)),int(np.ceil(sw/scale))
# # make it square
# mx = max(rh, rw)
# rw = rh = mx
#
# fitsimgs = []
# # We'll resample the new image into the existing-image WCS.
# newimg = None
#
# # HACK
# imgs = []
# layer = 'sdssco'
# bands = 'gri'
# for band in bands:
# fn = 'sdssco-1679p492-%s.fits' % band
# fitsfile = fitsio.FITS(fn)
# wcs = Tan(fn, 0)
# hh,ww = wcs.shape
# ok,x,y = wcs.radec2pixelxy(obj.ra, obj.dec)
# x = x - 1
# y = y - 1
# print('x,y', x,y)
# sz = pixrad * 0.262/0.396
# xl,xh = int(np.clip(x-sz, 0, ww-1)), int(np.clip(x+sz, 0, ww-1))
# yl,yh = int(np.clip(y-sz, 0, hh-1)), int(np.clip(y+sz, 0, hh-1))
# if xl == xh or yl == yh:
# print('no overlap with SDSS image')
# continue
#
# img = fitsfile[1][yl:yh+1, xl:xh+1]
# s = (subwcs.pixel_scale() / 0.396)
# img *= s**2
# imgs.append(img)
# thiswcs = wcs
#
# if len(imgs) == 0:
# continue
#
# fitsimgs.append((layer, bands, imgs))
#
# # Resample the new image to this layer's WCS
# newimg = np.zeros((rh,rw), dtype=subimg.dtype)
# try:
# # Laczos
# Yo,Xo,Yi,Xi,rims = resample_with_wcs(thiswcs, subwcs, [subimg])
# except:
# continue
# newimg[Yo,Xo] = rims[0]
#
# if len(fitsimgs) == 0:
# # No overlap with existing surveys
# continue
#
# newband = primhdr['FILTER'][0]
# I = I[K]
# J = J[K]
# print('Cut to', len(I), 'matches in', len(meas_exts), 'extensions')
# if len(I) == 0:
# return
# measures = {}
# for ext in meas_ext:
# measures[ext] =
print(len(specobjs), 'objects')
def my_rgb(imgs, bands, **kwargs):
return sdss_rgb(imgs,
bands,
scales=dict(g=6.0, r=3.4, i=2.5, z=2.2),
m=-0.02,
clip=False,
**kwargs)
def grayscale(img, band):
rgb = my_rgb([img, img, img], [band, band, band])
index = 'zrg'.index(newband)
gray = rgb[:, :, index]
return gray
newband = primhdr['FILTER'][0]
plt.figure(2)
plt.clf()
plt.subplots_adjust(left=0.03,
right=0.97,
bottom=0.03,
hspace=0,
wspace=0)
N = len(specobjs)
C = int(np.ceil(np.sqrt(N * 1.2)))
R = int(np.ceil(N / float(C)))
print('Rows,cols, N', R, C, N)
gray = grayscale(raw, newband)
hi = np.percentile(gray, 99.9)
grayargs = dict(interpolation='nearest',
origin='lower',
vmin=0.,
vmax=hi,
cmap='gray')
print('Plotting:')
plt.clf()
for i, (tt, subwcs, subimg) in enumerate(specobjs):
plt.subplot(R, C, i + 1)
print(' ', tt)
newimg = subimg
# New Image plot
newgray = grayscale(newimg, newband)
#hi = np.percentile(newgray, 99.9)
plt.imshow(newgray, **grayargs)
plt.xticks([])
plt.yticks([])
ps.savefig()
def write_cat_files(T, outdir):
""" writes out single source files for each object in T, to outdir
"""
# Write out Stripe82 measurements...
radius = np.sqrt(2.) * pixradius * pixscale / 3600.
for i in range(len(T)):
# looks for sources nearby T[i], within radius. it returns index in
# both first (short) list and second (long) list
I,J,d = asphere.match_radec(np.array([T.ra[i]]), np.array([T.dec[i]]),
T.ra, T.dec, radius)
print len(J), 'matched within', radius*3600., 'arcsec'
t = T[J]
print len(t), 'matched within', radius*3600., 'arcsec'
tt = aufits.fits_table()
cols = ['ra','dec','run','camcol','field',#'probpsf',
#'flags', #'type',
'fracdev_r', #'probpsf_r',
'devrad_r','devraderr_r', 'devab_r', 'devaberr_r',
'devphi_r', 'devphierr_r',
'exprad_r','expraderr_r', 'expab_r', 'expaberr_r',
'expphi_r', 'expphierr_r',
]
for c in cols:
cout = c
# drop "_r" from dev/exp shapes
if cout.endswith('_r'):
cout = cout[:-2]
coutmap = dict(devrad='theta_dev',
devphi='phi_dev',
devab ='ab_dev',
devraderr='theta_dev_err',
devphierr='phi_dev_err',
devaberr ='ab_dev_err',
exprad='theta_exp',
expphi='phi_exp',
expab ='ab_exp',
expraderr='theta_exp_err',
expphierr='phi_exp_err',
expaberr ='ab_exp_err',
fracdev='frac_dev')
cout = coutmap.get(cout, cout)
tt.set(cout, t.get(c))
tt.is_star = (t.type == 6)
for magname in ['psf', 'dev', 'exp']:
for band in 'ugriz':
mag = t.get('%smag_%s' % (magname, band))
magerr = t.get('%smagerr_%s' % (magname, band))
### FIXME -- arcsinh mags??
flux = tsdss.NanoMaggies.magToNanomaggies(mag)
dflux = np.abs(flux * np.log(10.)/-2.5 * magerr)
tt.set('%sflux_%s' % (magname, band), flux)
tt.set('%sfluxerr_%s' % (magname, band), dflux)
for band in 'ugriz':
# http://www.sdss3.org/dr10/algorithms/magnitudes.php#cmodel
fexp = tt.get('expflux_%s' % band)
fdev = tt.get('expflux_%s' % band)
fracdev = t.get('fracdev_%s' % band)
tt.set('cmodelflux_%s' % band, fracdev * fdev + (1.-fracdev) * fexp)
catfn = os.path.join(outdir, 'cat-s82-%.4f-%.4f.fits' % (t.ra[0], t.dec[0]))
tt.writeto(catfn)
print 'Wrote', catfn
def sed_matched_figs(detect_sn, good, img, sedlist, DES, g_det, r_det, i_det,
wcs):
x,y = detect_sources(detect_sn, 100.)
sources = fits_table()
sources.x = x
sources.y = y
sources.cut(good[sources.y, sources.x])
print('Cut to', len(sources), 'good sources')
sz = 20
H,W = good.shape
sources.cut((sources.x > sz) * (sources.y > sz) *
(sources.x < (W-sz)) * (sources.y < (H-sz)))
print(len(sources), 'not near edges')
for s in sedlist:
sources.set(s.tname, s.snmap[sources.y, sources.x])
# sources.g_sn = (g_det[sources.y, sources.x] * np.sqrt(g_detiv[sources.y, sources.x]))
# sources.r_sn = (r_det[sources.y, sources.x] * np.sqrt(r_detiv[sources.y, sources.x]))
# sources.i_sn = (i_det[sources.y, sources.x] * np.sqrt(i_detiv[sources.y, sources.x]))
sources.g_flux = g_det[sources.y, sources.x]
sources.r_flux = r_det[sources.y, sources.x]
sources.i_flux = i_det[sources.y, sources.x]
sources.ra,sources.dec = wcs.pixelxy2radec(sources.x+1, sources.y+1)
sources.g_mag = -2.5*(np.log10(sources.g_flux) - 9)
sources.r_mag = -2.5*(np.log10(sources.r_flux) - 9)
sources.i_mag = -2.5*(np.log10(sources.i_flux) - 9)
sources.imax = np.argmax(np.vstack([sources.get(s.tname) for s in sedlist]), axis=0)
plt.figure(figsize=(5,4))
plt.subplots_adjust(left=0.15, right=0.97, bottom=0.12, top=0.98)
plt.clf()
for i,s in enumerate(sedlist):
if not np.all(s.sed > 0):
continue
I = np.flatnonzero(sources.imax == i)
plt.plot(sources.g_mag[I] - sources.r_mag[I],
sources.r_mag[I] - sources.i_mag[I],
s.plotsym, label=s.name, color=s.plotcolor, alpha=0.5,
mfc='none', ms=5)
gr = -2.5 * np.log10(s.sed[0] / s.sed[1])
ri = -2.5 * np.log10(s.sed[1] / s.sed[2])
plt.plot(gr, ri, 'o', color='k', mfc='none', ms=8, mew=3)
plt.axis([-0.5, 2.5, -0.5, 2])
plt.xlabel('g - r (mag)')
plt.ylabel('r - i (mag)')
plt.legend(loc='upper left')
plt.savefig('best-color.pdf')
plt.figure(figsize=(4,4))
plt.subplots_adjust(left=0.01, right=0.99, bottom=0.01, top=0.99)
plt.clf()
xlo,xhi = 500,1100
ylo,yhi = 500,1100
plt.imshow(img[ylo:yhi, xlo:xhi], origin='lower', interpolation='nearest',
extent=[xlo,xhi,ylo,yhi])
ax = plt.axis()
for i,s in enumerate(sedlist):
if not np.all(s.sed > 0):
continue
I = np.flatnonzero((sources.imax == i) *
(sources.x >= xlo) * (sources.x <= xhi) *
(sources.y >= ylo) * (sources.y <= yhi))
print(len(I), s.name)
plt.plot(sources.x[I], sources.y[I],
s.plotsym, label=s.name, color=s.plotcolor, alpha=0.5,
mfc='none', ms=15)
plt.axis(ax)
plt.xticks([])
plt.yticks([])
plt.savefig('image-sources.pdf')
for i,s in enumerate(sedlist):
I = np.flatnonzero(sources.imax == i)
J = np.argsort(-sources.get(s.tname)[I])
plt.clf()
show_sources(sources[I[J]], img)
plt.savefig('best-%s.pdf' % s.name.lower())
##### Run detection at different thresholds ####
tsedlist = []
for i,s in enumerate(sedlist):
if not np.all(s.sed > 0):
continue
tsedlist.append(s)
snmap = None
for i,s in enumerate(tsedlist):
if snmap is None:
snmap = s.snmap
else:
snmap = np.maximum(snmap, s.snmap)
for thresh in [30]: #10, 30, 100]:
x,y = detect_sources(snmap, thresh)
tsources = fits_table()
tsources.x = x
tsources.y = y
#tsources.cut(good[tsources.y, tsources.x])
print('Threshold', thresh)
print('Cut to', len(tsources), 'good sources')
sz = 20
H,W = good.shape
tsources.cut((tsources.x > sz) * (tsources.y > sz) *
(tsources.x < (W-sz)) * (tsources.y < (H-sz)))
print(len(tsources), 'not near edges')
for s in tsedlist:
tsources.set(s.tname, s.snmap[tsources.y, tsources.x])
tsources.g_flux = g_det[tsources.y, tsources.x]
tsources.r_flux = r_det[tsources.y, tsources.x]
tsources.i_flux = i_det[tsources.y, tsources.x]
tsources.ra,tsources.dec = wcs.pixelxy2radec(tsources.x+1, tsources.y+1)
tsources.g_mag = -2.5*(np.log10(tsources.g_flux) - 9)
tsources.r_mag = -2.5*(np.log10(tsources.r_flux) - 9)
tsources.i_mag = -2.5*(np.log10(tsources.i_flux) - 9)
tsources.imax = np.argmax(np.vstack([tsources.get(s.tname)
for s in tsedlist]), axis=0)
plt.clf()
xlo,xhi = 500,1100
ylo,yhi = 500,1100
plt.imshow(img[ylo:yhi, xlo:xhi], origin='lower', interpolation='nearest',
extent=[xlo,xhi,ylo,yhi])
ax = plt.axis()
for i,s in enumerate(tsedlist):
I = np.flatnonzero((tsources.imax == i) *
(tsources.x >= xlo) * (tsources.x <= xhi) *
(tsources.y >= ylo) * (tsources.y <= yhi))
print(len(I), s.name)
plt.plot(tsources.x[I], tsources.y[I],
s.plotsym, label=s.name, color=s.plotcolor, alpha=0.5,
mfc='none', ms=15, mew=2)
plt.axis(ax)
plt.xticks([])
plt.yticks([])
plt.savefig('image-sources-%i.pdf' % thresh)
# boundary = (snmap > thresh)
# boundary = np.logical_xor(boundary, binary_dilation(boundary,
# structure=np.ones((3,3))))
# rgb = img[ylo:yhi, xlo:xhi].copy()
# rgb[:,:,1][boundary[ylo:yhi, xlo:xhi]] = 255
# plt.clf()
# plt.imshow(rgb, origin='lower', interpolation='nearest',
# extent=[xlo,xhi,ylo,yhi])
# plt.xticks([])
# plt.yticks([])
# plt.savefig('image-blobs-%i.pdf' % thresh)
plt.figure(figsize=(5,4))
plt.subplots_adjust(left=0.15, right=0.97, bottom=0.12, top=0.98)
plt.clf()
lp,lt = [],[]
for i,s in enumerate(tsedlist):
I = np.flatnonzero(tsources.imax == i)
plt.plot(tsources.g_mag[I] - tsources.r_mag[I],
tsources.r_mag[I] - tsources.i_mag[I],
s.plotsym, color=s.plotcolor, alpha=0.2,
mfc='none', ms=5)
# For the legend
p = plt.plot(-1, -1, s.plotsym, color=s.plotcolor, mfc='none', ms=5)
lp.append(p[0])
lt.append(s.name)
gr = -2.5 * np.log10(s.sed[0] / s.sed[1])
ri = -2.5 * np.log10(s.sed[1] / s.sed[2])
plt.plot(gr, ri, 'o', color='k', mfc='none', ms=8, mew=3)
plt.axis([-0.5, 2.5, -0.5, 2])
plt.xlabel('g - r (mag)')
plt.ylabel('r - i (mag)')
plt.legend(lp, lt, loc='upper left')
plt.savefig('best-color-%i.pdf' % thresh)
##############################
# Artifacts from single-band detections
I = np.hstack((np.flatnonzero(sources.imax == 3)[:6],
np.flatnonzero(sources.imax == 4)[:18],
np.flatnonzero(sources.imax == 5)[:12]))
plt.figure(figsize=(4,4))
plt.subplots_adjust(left=0.01, right=0.99, bottom=0.01, top=0.99)
plt.clf()
show_sources(sources[I], img, R=6, C=6, sz=30, divider=1)
plt.savefig('singleband.pdf')
MI,MJ,d = match_radec(sources.ra, sources.dec, DES.ra, DES.dec, 1./3600, nearest=True)
print(len(MI), 'matches')
MDES = DES[MJ]
Msources = sources[MI]
## FIXME -- select only isolated stars?
colorbins = np.linspace(-0.5, 4.0, 10)
II = []
K = []
DES.gi = DES.mag_auto_g - DES.mag_auto_i
for clo,chi in zip(colorbins, colorbins[1:]):
C = np.flatnonzero((DES.gi >= clo) * (DES.gi < chi))
minmag = np.vstack((DES.mag_auto_g, DES.mag_auto_r, DES.mag_auto_i)).max(axis=0)[C]
C = C[np.argsort(np.abs(minmag - 17.9))]
C = C[DES.spread_model_r[C] < 0.01]
II.extend(C[:10])
K.append(C[0])
fw,fh = 6,4
sl,sr,sb,st = 0.15, 0.98, 0.12, 0.98
plt.figure(figsize=(fw,fh))
plt.subplots_adjust(left=sl, right=sr, bottom=sb, top=st)
plt.clf()
plt.axhline(1., color='orange', lw=5)
plt.axhline(1., color='k', alpha=0.5)
plt.axvline(0., color='b', lw=2, alpha=0.2)
plt.axvline(1.3, color='orange', lw=2, alpha=0.2)
plt.axvline(2.5, color='r', lw=2, alpha=0.2)
plt.plot(MDES.mag_auto_g - MDES.mag_auto_i, Msources.blue_sn / Msources.yellow_sn, 'bD', alpha=0.3,
label='Blue SED-matched filter', ms=3)
plt.plot(MDES.mag_auto_g - MDES.mag_auto_i, Msources.red_sn / Msources.yellow_sn, 'rs', alpha=0.5,
label='Red SED-matched filter', ms=3)
plt.xlabel('DES g - i color (mag)')
plt.ylabel('Relative strength of SED filter vs Yellow')
plt.legend(loc='upper left')
# Position the postage stamp images just right...
# axes width,height
w = fw * (sr-sl)
h = fh * (st-sb)
n = len(colorbins)-1
# image size
s = w/n
# fraction of vertical axis devoted to image
fim = s/h
# fraction devoted to plot
fplot = 1.-fim
# scale
ys = (1.3 - 0.7) / fplot
# lower limit
ymin = 1.3 - ys
# image top
ymax = ymin + ys * fim
ax = [-0.5, 4.0, ymin, 1.3]
plt.axis(ax)
aspect = plt.gca().get_aspect()
for clo,chi,k in zip(colorbins, colorbins[1:], K):
x,y = DES.x[k], DES.y[k]
plt.imshow(img[y-sz:y+sz+1, x-sz:x+sz+1], interpolation='nearest', origin='lower',
extent=[clo,chi,ymin,ymax], zorder=20)
plt.yticks([0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3])
plt.axis(ax)
plt.gca().set_aspect(aspect)
plt.savefig('strength.pdf')
def _get_sources(run, camcol, field, bandname='r', sdss=None, release='DR7',
objs=None,
retrieve=True, curl=False, roi=None,
radecroi=None,
radecrad=None,
bands=None,
badmag=25, nanomaggies=False,
getobjs=False, getsourceobjs=False, getobjinds=False,
extrabands=None,
fixedComposites=False,
forcePointSources=False,
useObjcType=False,
objCuts=True,
classmap={},
ellipse=GalaxyShape,
cutToPrimary=False):
'''
If set,
radecrad = (ra,dec,rad)
returns sources within "rad" degrees of the given RA,Dec (in degrees)
center.
WARNING, this method alters the "objs" argument, if given.
Consider calling objs.copy() before calling.
-"bandname" is the SDSS band used to cut on position, select
star/gal/exp/dev, and set galaxy shapes.
-"bands" are the bands to include in the returned Source objects;
they will be initialized from the SDSS bands.
-"extrabands" are also included in the returned Source objects;
they will be initialized to the SDSS flux for either the first of
"bands", if given, or "bandname".
'''
from astrometry.sdss import (DR7, DR8, DR9, band_names, band_index,
photo_flags1_map)
# brightPointSourceThreshold=0.):
if sdss is None:
dr = dict(DR7=DR7, DR8=DR8, DR9=DR9)[release]
sdss = dr(curl=curl)
drnum = sdss.getDRNumber()
isdr7 = (drnum == 7)
if bands is None:
bands = band_names()
bandnum = band_index(bandname)
bandnums = np.array([band_index(b) for b in bands])
bandnames = bands
if extrabands is None:
extrabands = []
if objs is None:
from astrometry.util.fits import fits_table
if isdr7:
# FIXME
rerun = 0
_check_sdss_files(sdss, run, camcol, field, bandnum,
['tsObj', 'tsField'],
retrieve=retrieve, tryopen=True)
tsf = sdss.readTsField(run, camcol, field, rerun)
objfn = sdss.getPath('tsObj', run, camcol, field,
bandname, rerun=rerun)
else:
_check_sdss_files(sdss, run, camcol, field, bandnum, ['photoObj'],
tryopen=True, retrieve=retrieve)
objfn = sdss.getPath('photoObj', run, camcol, field)
objs = fits_table(objfn)
if objs is None:
print('No sources in SDSS file', objfn)
return []
objs.index = np.arange(len(objs))
if getobjs:
allobjs = objs.copy()
if roi is not None:
x0,x1,y0,y1 = roi
# FIXME -- we keep only the sources whose centers are within
# the ROI box. Should instead do some ellipse-overlaps
# geometry.
x = objs.colc[:,bandnum]
y = objs.rowc[:,bandnum]
objs.cut((x >= x0) * (x < x1) * (y >= y0) * (y < y1))
if radecroi is not None:
r0,r1,d0,d1 = radecroi
objs.cut((objs.ra >= r0) * (objs.ra <= r1) *
(objs.dec >= d0) * (objs.dec <= d1))
if radecrad is not None:
from astrometry.libkd.spherematch import match_radec
(ra,dec,rad) = radecrad
I,J,d = match_radec(ra, dec, objs.ra, objs.dec, rad)
objs.cut(J)
del I
del d
if objCuts:
# Only deblended children;
# No BRIGHT sources
bright = photo_flags1_map.get('BRIGHT')
objs.cut((objs.nchild == 0) * ((objs.objc_flags & bright) == 0))
if cutToPrimary:
objs.cut((objs.resolve_status & 256) > 0)
if isdr7:
objs.rename('phi_dev', 'phi_dev_deg')
objs.rename('phi_exp', 'phi_exp_deg')
objs.rename('r_dev', 'theta_dev')
objs.rename('r_exp', 'theta_exp')
# SDSS and Tractor have different opinions on which way this rotation goes
objs.phi_dev_deg *= -1.
objs.phi_exp_deg *= -1.
# MAGIC -- minimum size of galaxy.
objs.theta_dev = np.maximum(objs.theta_dev, 1./30.)
objs.theta_exp = np.maximum(objs.theta_exp, 1./30.)
if forcePointSources:
Lstar = np.ones(len(objs), float)
Lgal = np.zeros(len(objs), float)
Ldev = Lexp = Lgal
else:
if useObjcType:
objs.cut(np.logical_or(objs.objc_type == 6,
objs.objc_type == 3))
Lstar = (objs.objc_type == 6)
Lgal = (objs.objc_type == 3)
else:
Lstar = (objs.prob_psf[:,bandnum] == 1) * 1.0
Lgal = (objs.prob_psf[:,bandnum] == 0)
if isdr7:
fracdev = objs.fracpsf[:,bandnum]
else:
fracdev = objs.fracdev[:,bandnum]
Ldev = Lgal * fracdev
Lexp = Lgal * (1. - fracdev)
if isdr7:
if nanomaggies:
raise RuntimeError('Nanomaggies not supported for DR7 (yet)')
def lup2bright(lups):
counts = [tsf.luptitude_to_counts(lup,j)
for j,lup in enumerate(lups)]
counts = np.array(counts)
bright = _getBrightness(counts, tsf, bandnames, extrabands)
return bright
flux2bright = lup2bright
starflux = objs.psfcounts
compflux = objs.counts_model
devflux = objs.counts_dev
expflux = objs.counts_exp
def comp2bright(lups, Ldev, Lexp):
counts = [tsf.luptitude_to_counts(lup,j)
for j,lup in enumerate(lups)]
counts = np.array(counts)
dcounts = counts * Ldev
ecounts = counts * Lexp
dbright = _getBrightness(dcounts, tsf, bands, extrabands)
ebright = _getBrightness(ecounts, tsf, bands, extrabands)
return dbright, ebright
else:
def nmgy2bright(flux):
if len(bandnums):
flux = flux[bandnums]
else:
flux = flux[np.array([bandnum])]
bb = bandnames + extrabands
if nanomaggies:
if len(extrabands):
if len(bandnums) == 0:
# Only "extrabands", no SDSS bands.
flux = np.zeros(len(extrabands)) + flux[0]
else:
flux = np.append(flux, np.zeros(len(extrabands)))
bright = NanoMaggies(order=bb, **dict(zip(bb, flux)))
else:
I = (flux > 0)
mag = np.zeros_like(flux) + badmag
mag[I] = sdss.nmgy_to_mag(flux[I])
if len(extrabands):
mag = np.append(mag, np.zeros(len(extrabands)) + badmag)
bright = Mags(order=bb, **dict(zip(bb,mag)))
return bright
def comp2bright(flux, Ldev, Lexp):
dflux = flux * Ldev
eflux = flux * Lexp
dbright = nmgy2bright(dflux)
ebright = nmgy2bright(eflux)
return dbright, ebright
flux2bright = nmgy2bright
starflux = objs.psfflux
compflux = objs.cmodelflux
devflux = objs.devflux
expflux = objs.expflux
sources = []
nstars, ndev, nexp, ncomp = 0, 0, 0, 0
isources = []
ptsrcclass = classmap.get(PointSource, PointSource)
for i in range(len(objs)):
if Lstar[i]:
pos = RaDecPos(objs.ra[i], objs.dec[i])
flux = starflux[i,:]
bright = flux2bright(flux)
# This should work, I just don't feel like testing it now...
# if brightPointSourceThreshold:
# ps = SdssPointSource(pos, bright, thresh=brightPointSourceThreshold)
# else:
# ps = PointSource(pos, bright)
sources.append(ptsrcclass(pos, bright))
nstars += 1
isources.append(i)
continue
hasdev = (Ldev[i] > 0)
hasexp = (Lexp[i] > 0)
iscomp = (hasdev and hasexp)
pos = RaDecPos(objs.ra[i], objs.dec[i])
if iscomp:
flux = compflux[i,:]
elif hasdev:
flux = devflux[i,:]
elif hasexp:
flux = expflux[i,:]
else:
print('Skipping object with Lstar = %g, Ldev = %g, Lexp = %g (fracdev=%g)'
% (Lstar[i], Ldev[i], Lexp[i], fracdev[i]))
continue
isources.append(i)
if iscomp:
if fixedComposites:
bright = flux2bright(flux)
fdev = (Ldev[i] / (Ldev[i] + Lexp[i]))
else:
dbright,ebright = comp2bright(flux, Ldev[i], Lexp[i])
else:
bright = flux2bright(flux)
if hasdev:
re = objs.theta_dev [i,bandnum]
ab = objs.ab_dev [i,bandnum]
phi = objs.phi_dev_deg[i,bandnum]
dshape = ellipse(re, ab, phi)
if hasexp:
re = objs.theta_exp [i,bandnum]
ab = objs.ab_exp [i,bandnum]
phi = objs.phi_exp_deg[i,bandnum]
eshape = ellipse(re, ab, phi)
if iscomp:
if fixedComposites:
gal = FixedCompositeGalaxy(pos, bright, fdev, eshape, dshape)
else:
gal = CompositeGalaxy(pos, ebright, eshape, dbright, dshape)
ncomp += 1
elif hasdev:
gal = DevGalaxy(pos, bright, dshape)
ndev += 1
elif hasexp:
gal = ExpGalaxy(pos, bright, eshape)
nexp += 1
sources.append(gal)
print('Created', ndev, 'deV,', nexp, 'exp,', ncomp, 'composite',)
print('(total %i) galaxies and %i stars' % (ndev+nexp+ncomp, nstars))
if not (getobjs or getobjinds or getsourceobjs):
return sources
if nstars + ndev + nexp + ncomp < len(objs):
objs = objs[np.array(isources)]
rtn = [sources]
if getobjs:
rtn.append(allobjs)
if getobjinds:
rtn.append(objs.index if len(objs) else np.array([]))
if getsourceobjs:
rtn.append(objs)
return rtn
def forced_phot():
# Forced phot
ps = PlotSequence('kick')
plt.subplots_adjust(top=0.95, bottom=0.1, left=0.1, right=0.95)
ff = [
('decam-348227-S15-g-forced.fits', 348227, 'S15', 'g'),
('decam-348248-S16-g-forced.fits', 348248, 'S16', 'g'),
('decam-348271-S14-g-forced.fits', 348271, 'S14', 'g'),
('decam-348660-S15-r-forced.fits', 348660, 'S15', 'r'),
('decam-348684-S16-r-forced.fits', 348684, 'S16', 'r'),
('decam-348712-S14-r-forced.fits', 348712, 'S14', 'r'),
('decam-349154-S15-z-forced.fits', 349154, 'S15', 'z'),
('decam-349183-S14-z-forced.fits', 349183, 'S14', 'z'),
('decam-346630-S16-z-forced.fits', 346630, 'S16', 'z'),
]
FF = []
for fn, expnum, extname, band in ff:
F = fits_table(fn)
print len(F), 'from', fn
F.expnum = np.array([expnum] * len(F))
F.extname = np.array([extname] * len(F))
FF.append(F)
F = merge_tables(FF)
bricks = np.unique(F.brickname)
print 'Bricks:', bricks
T = merge_tables([
fits_table(os.path.join('dr1', 'tractor', b[:3],
'tractor-%s.fits' % b)) for b in bricks
])
print 'Total of', len(T), 'sources'
T.cut(T.brick_primary == 1)
print 'Cut to', len(T), 'brick_primary'
Tall = T.copy()
cutouts = []
I, J, d = match_radec(Tall.ra,
Tall.dec,
Tall.ra,
Tall.dec,
5. / 3600.,
notself=True)
K = np.flatnonzero(I < J)
I = I[K]
J = J[K]
# randomize
K = np.random.randint(2, size=len(I))
I, J = I * K + J * (1 - K), I * (1 - K) + J * K
plt.clf()
plothist(
3600. * (Tall.ra[I] - Tall.ra[J]) * np.cos(np.deg2rad(Tall.dec[I])),
3600. * (Tall.dec[I] - Tall.dec[J]), 200) #, range=((-2,2),(-2,2)))
plt.xlabel('dRA (arcsec)')
plt.ylabel('dDec (arcsec)')
plt.title('%i sources, %i matches' % (len(Tall), len(I)))
ps.savefig()
dist = 3600. * np.hypot(
(Tall.ra[I] - Tall.ra[J]) * np.cos(np.deg2rad(Tall.dec[I])),
Tall.dec[I] - Tall.dec[J])
cutouts.append((Tall, I[dist < 0.5], 'Close pairs', None))
plt.clf()
plt.hist(dist, 50, histtype='step', color='b')
plt.xlabel('Match distance (arcsec)')
plt.title('%i sources, %i matches' % (len(Tall), len(I)))
ps.savefig()
plt.clf()
plothist(Tall.bx0[I] - Tall.bx0[J],
Tall.by0[I] - Tall.by0[J],
200,
range=((-20, 20), (-20, 20)))
plt.xlabel('d(x0)')
plt.ylabel('d(y0)')
plt.title('%i sources, %i matches' % (len(Tall), len(I)))
ps.savefig()
# plt.clf()
# plt.plot(np.vstack((Tall.bx0[I], Tall.bx0[J])),
# np.vstack((Tall.by0[I], Tall.by0[J])), 'b-')
# ps.savefig()
T.srcid = (T.brickid.astype(np.int64) << 32 | T.objid)
F.srcid = (F.brickid.astype(np.int64) << 32 | F.objid)
print 'Total of', len(F), 'forced'
print len(np.unique(F.srcid)), 'unique source in forced'
tmap = dict([(s, i) for i, s in enumerate(T.srcid)])
T.forced_g = [[] for i in range(len(T))]
T.forced_r = [[] for i in range(len(T))]
T.forced_z = [[] for i in range(len(T))]
for f in F:
i = tmap[f.srcid]
forced = T.get('forced_%s' % f.filter)[i]
forced.append(f.flux)
allbands = 'ugrizY'
# pack into arrays
bands = 'grz'
for band in bands:
flist = T.get('forced_%s' % band)
nmax = max([len(f) for f in flist])
arr = np.zeros((len(T), nmax), np.float32)
for i, f in enumerate(flist):
arr[i, :len(f)] = f
T.set('forced_%s' % band, arr)
ib = allbands.index(band)
flux = T.decam_flux[:, ib]
arr = np.zeros(len(T), np.float32)
for i, f in enumerate(flist):
arr[i] = np.mean([(fi - flux[i])**2 for fi in f])
arr = np.sqrt(arr)
T.set('forced_rms_%s' % band, arr)
T.set('forced_n_%s' % band, np.array([len(f) for f in flist]))
# Cut to sources with forced phot
T.cut(
np.flatnonzero(
reduce(np.logical_or, [
np.any(T.get('forced_%s' % band) > 0, axis=1) for band in bands
])))
print 'Cut to', len(T), 'sources with forced phot'
flux = T.decam_flux[:, 1]
forced = T.forced_g
rms = T.forced_rms_g
N = T.forced_n_g
I = np.flatnonzero((N > 1) * (flux > 1e3))
print len(I), 'with flux > 10^3'
#print 'flux', flux[I]
#print 'forced'
for f, ff, r, n in zip(flux[I], forced[I, :], rms[I], N[I]):
print 'flux', f, 'n', n, 'forced RMS', r, 'forced', ff
# Compute some summary stats
# allbands = 'ugrizY'
# for b in bands:
# ib = allbands.index(b)
# forced = T.get('forced_%s' % b)
# flux = T.decam_flux[:,ib]
# T.set('forced_rms_%s' % b, np.sqrt([np.mean([(fi - fl)**2 for fi in flist if fi != 0])
# for flist,fl in zip(forced, flux)]))
imgs = {}
allbands = 'ugrizY'
for b in bands:
ib = allbands.index(b)
plt.clf()
f = T.get('forced_%s' % b)
n, nf = f.shape
flux = T.decam_flux[:, ib]
lo, hi = 1e-2, 1e5
plt.plot([lo, hi], [lo, hi], 'r-')
plt.errorbar(flux,
flux,
fmt='none',
yerr=1. / np.sqrt(T.decam_flux_ivar[:, ib]),
ecolor='r')
for i in range(nf):
plt.plot(T.decam_flux[:, ib], f[:, i], 'b.', alpha=0.25)
plt.axis([lo, hi, lo, hi])
plt.xscale('log')
plt.yscale('log')
plt.xlabel('Combined flux')
plt.ylabel('Forced-photometry flux')
plt.title('Forced phot: %s band' % b)
ps.savefig()
for b in bands:
ib = allbands.index(b)
plt.clf()
f = T.get('forced_%s' % b)
n, nf = f.shape
flux = T.decam_flux[:, ib]
lo, hi = -0.5, 4
plt.axhline(1., color='r')
plt.axhline(0., color='k', alpha=0.25)
for i in range(nf):
plt.plot(flux, np.clip(f[:, i] / flux, lo, hi), 'b.', alpha=0.25)
plt.ylim(lo - 0.1, hi + 0.1)
plt.xlim(1e-2, 1e5)
plt.xscale('log')
#plt.yscale('log')
plt.xlabel('Combined flux')
plt.ylabel('Relative forced-photometry flux')
plt.title('Forced phot: %s band' % b)
ps.savefig()
# Large relative flux
I = np.flatnonzero((flux > 0.) * ((f[:, i] / flux) > 4.))
#print 'Relative fluxes:', (f[:,i]/flux)[I]
#print 'Fluxes:', flux[I]
I = I[np.argsort(-flux[I])]
#print 'Sorted fluxes:', flux[I]
labels = []
for t in T[I]:
fluxes = t.get('forced_%s' % b)
ib = allbands.index(b)
flux = t.decam_flux[ib]
txt = ('%.1f / %.1f / %.1f | %.1f' %
(fluxes[0], fluxes[1], fluxes[2], flux))
labels.append(txt)
cutouts.append((T, I, 'Large relative flux: %s band' % b, labels))
for b in bands:
ib = allbands.index(b)
plt.clf()
rms = T.get('forced_rms_%s' % b)
N = T.get('forced_n_%s' % b)
flux = T.decam_flux[:, ib]
lo, hi = -0.5, 4
#plt.axhline(1., color='r')
#plt.axhline(0., color='k', alpha=0.25)
I = np.flatnonzero(N > 1)
print len(I), 'of', len(rms), 'have >1 exposure'
plt.plot(flux[I], rms[I], 'b.', alpha=0.25)
#plt.ylim(lo-0.1, hi+0.1)
plt.xlim(1e-2, 1e5)
plt.xscale('log')
plt.yscale('log')
plt.xlabel('Combined flux')
plt.ylabel('Forced-photometry flux RMS')
plt.title('Forced phot: %s band' % b)
ps.savefig()
# Large relative RMS
I = np.flatnonzero((flux > 10.) * (N > 1) * ((rms / flux) > 0.1))
I = I[np.argsort(-flux[I])]
cutouts.append((T, I, 'Large relative RMS: %s band' % b, None))
T[I].writeto('large-rms-%s.fits' % b)
# Create a fake "brick" WCS bounding the forced-phot objects
# rlo = T.ra.min()
# rhi = T.ra.max()
# dlo = T.dec.min()
# dhi = T.dec.max()
rlo = Tall.ra.min()
rhi = Tall.ra.max()
dlo = Tall.dec.min()
dhi = Tall.dec.max()
if rhi - rlo > 180:
print 'No RA wrap-around support'
sys.exit(0)
dec = (dlo + dhi) / 2.
ra = (rlo + rhi) / 2.
pixscale = 0.262
ddec = (dhi - dlo)
H = ddec * 3600. / pixscale
dra = (rhi - rlo)
W = dra * np.cos(np.deg2rad(dec)) * 3600. / pixscale
H = int(np.ceil(H))
W = int(np.ceil(W))
print 'Target image shape', (H, W)
targetwcs = Tan(ra, dec, W / 2. + 0.5, H / 2. + 0.5, -pixscale / 3600., 0.,
0., pixscale / 3600., float(W), float(H))
img = np.zeros((H, W, 3), np.uint8)
print 'img', img.shape
decals = Decals()
for brickname in bricks:
brick = decals.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brick)
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, brickwcs, [], 3)
except:
continue
brickimg = plt.imread(
os.path.join('dr1', 'coadd', brickname[:3], brickname,
'decals-%s-image.jpg' % brickname))
print 'brick image', brickimg.shape, brickimg.dtype
brickimg = brickimg[::-1, :, :]
img[Yo, Xo, :] = brickimg[Yi, Xi, :]
# Now fake the "bx,by" coords to refer to 'targetwcs' / 'img'
ok, x, y = targetwcs.radec2pixelxy(T.ra, T.dec)
T.bx = x - 1
T.by = y - 1
ok, x, y = targetwcs.radec2pixelxy(Tall.ra, Tall.dec)
Tall.bx = x - 1
Tall.by = y - 1
print 'Tall:', len(Tall), 'sources'
# plt.clf()
# dimshow(img)
# ps.savefig()
#
# ax = plt.axis()
# plt.plot(Tall.bx, Tall.by, 'r.')
# plt.axis(ax)
# ps.savefig()
for TT, I, desc, labels in cutouts:
plt.subplots_adjust(left=0.05,
right=0.95,
bottom=0.05,
top=0.95,
hspace=0.05,
wspace=0.05)
kwa = dict(rows=6, cols=9)
plt.clf()
plot_objects(TT[I], None, img, targetwcs, **kwa)
plt.suptitle(desc)
ps.savefig()
plt.clf()
plot_objects(TT[I], Tall, img, targetwcs, labels=labels, **kwa)
plt.suptitle(desc)
ps.savefig()
def plot_unmatched():
from bigboss_test import radecroi
'''
select
run, rerun, camcol, field, nChild, probPSF,
psfFlux_u, psfFlux_g, psfFlux_r, psfFlux_i, psfFlux_z,
deVRad_u, deVRad_g, deVRad_r, deVRad_i, deVRad_z,
deVAB_u, deVAB_g, deVAB_r, deVAB_i, deVAB_z,
deVPhi_u, deVPhi_g, deVPhi_r, deVPhi_i, deVPhi_z,
deVFlux_u, deVFlux_g, deVFlux_r, deVFlux_i, deVFlux_z,
expRad_u, expRad_g, expRad_r, expRad_i, expRad_z,
expAB_u, expAB_g, expAB_r, expAB_i, expAB_z,
expPhi_u, expPhi_g, expPhi_r, expPhi_i, expPhi_z,
expFlux_u, expFlux_g, expFlux_r, expFlux_i, expFlux_z,
fracDeV_u, fracDeV_g, fracDeV_r, fracDeV_i, fracDeV_z,
flags_u, flags_g, flags_r, flags_i, flags_z,
probPSF_u, probPSF_g, probPSF_r, probPSF_i, probPSF_z,
ra, dec
from PhotoPrimary
where ra between 333.5 and 335.5 and dec between -0.5 and 1.5
'''
''' -> 124k rows. (sdss-cas-testarea.fits) Distinct runs:
2585, 2728, 7712, 4203, 2583, 4192, 4207, 4184, 2662, 7717
Run #sources
2583 663
2585 675
2662 4
2728 156
4184 762
4192 36135
4203 5
4207 44078
7712 12047
7717 29911
'''
'''
select
run, rerun, camcol, field, nChild, probPSF,
psfFlux_u, psfFlux_g, psfFlux_r, psfFlux_i, psfFlux_z,
deVRad_u, deVRad_g, deVRad_r, deVRad_i, deVRad_z,
deVAB_u, deVAB_g, deVAB_r, deVAB_i, deVAB_z,
deVPhi_u, deVPhi_g, deVPhi_r, deVPhi_i, deVPhi_z,
deVFlux_u, deVFlux_g, deVFlux_r, deVFlux_i, deVFlux_z,
expRad_u, expRad_g, expRad_r, expRad_i, expRad_z,
expAB_u, expAB_g, expAB_r, expAB_i, expAB_z,
expPhi_u, expPhi_g, expPhi_r, expPhi_i, expPhi_z,
expFlux_u, expFlux_g, expFlux_r, expFlux_i, expFlux_z,
fracDeV_u, fracDeV_g, fracDeV_r, fracDeV_i, fracDeV_z,
flags_u, flags_g, flags_r, flags_i, flags_z,
probPSF_u, probPSF_g, probPSF_r, probPSF_i, probPSF_z,
ra, dec, resolveStatus, score into mydb.wisetest from PhotoObjAll
where ra between 333.5 and 335.5 and dec between -0.5 and 1.5
and (resolveStatus & (
dbo.fResolveStatus('SURVEY_PRIMARY') |
dbo.fResolveStatus('SURVEY_BADFIELD') |
dbo.fResolveStatus('SURVEY_EDGE'))) != 0
--> sdss-cas-testarea-2.fits
select
run, rerun, camcol, field, nChild, probPSF,
psfFlux_u, psfFlux_g, psfFlux_r, psfFlux_i, psfFlux_z,
deVRad_u, deVRad_g, deVRad_r, deVRad_i, deVRad_z,
deVAB_u, deVAB_g, deVAB_r, deVAB_i, deVAB_z,
deVPhi_u, deVPhi_g, deVPhi_r, deVPhi_i, deVPhi_z,
deVFlux_u, deVFlux_g, deVFlux_r, deVFlux_i, deVFlux_z,
expRad_u, expRad_g, expRad_r, expRad_i, expRad_z,
expAB_u, expAB_g, expAB_r, expAB_i, expAB_z,
expPhi_u, expPhi_g, expPhi_r, expPhi_i, expPhi_z,
expFlux_u, expFlux_g, expFlux_r, expFlux_i, expFlux_z,
fracDeV_u, fracDeV_g, fracDeV_r, fracDeV_i, fracDeV_z,
flags_u, flags_g, flags_r, flags_i, flags_z,
probPSF_u, probPSF_g, probPSF_r, probPSF_i, probPSF_z,
ra, dec, resolveStatus, score into mydb.wisetest from PhotoObjAll
where ra between 333.5 and 335.5 and dec between -0.5 and 1.5
and (resolveStatus & (
dbo.fResolveStatus('SURVEY_PRIMARY') |
dbo.fResolveStatus('SURVEY_BADFIELD') |
dbo.fResolveStatus('SURVEY_EDGE') |
dbo.fResolveStatus('SURVEY_BEST')
)) != 0
--> sdss-cas-testarea-3.fits
'''
''' Check it out: spatial source density looks fine. No overlap between runs.
'''
rng = ((333.5, 335.5), (-0.5, 1.5))
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('sdss')
if False:
r, d = np.mean(rng[0]), np.mean(rng[1])
RCF = radec_to_sdss_rcf(r, d, radius=2. * 60.,
tablefn='window_flist-DR9.fits')
print('Found', len(RCF), 'fields in range')
for run, c, f, ra, dec in RCF:
print(' ', run, c, f, 'at', ra, dec)
from astrometry.blind.plotstuff import PlotSequence
plot = Plotstuff(rdw=(r, d, 10), size=(1000, 1000), outformat='png')
plot.color = 'white'
plot.alpha = 0.5
plot.apply_settings()
T = fits_table('window_flist-DR9.fits')
I, J, d = match_radec(T.ra, T.dec, r, d, 10)
T.cut(I)
print('Plotting', len(T))
for i, (m0, m1, n0, n1, node, incl) in enumerate(zip(T.mu_start, T.mu_end, T.nu_start, T.nu_end, T.node, T.incl)):
#rr,dd = [],[]
for j, (m, n) in enumerate([(m0, n0), (m0, n1), (m1, n1), (m1, n0), (m0, n0)]):
ri, di = munu_to_radec_deg(m, n, node, incl)
# rr.append(ri)
# dd.append(di)
if j == 0:
plot.move_to_radec(ri, di)
else:
plot.line_to_radec(ri, di)
plot.stroke()
plot.plot_grid(2, 2, 5, 5)
plot.write('fields.png')
# CAS PhotoObjAll.resolveStatus bits
sprim = 0x100
sbad = 0x800
sedge = 0x1000
sbest = 0x200
if False:
T = fits_table('sdss-cas-testarea.fits')
plt.clf()
plothist(T.ra, T.dec, 200, range=rng)
plt.title('PhotoPrimary')
ps.savefig()
T = fits_table('sdss-cas-testarea-2.fits')
plt.clf()
plothist(T.ra, T.dec, 200, range=rng)
plt.title('PhotoObjAll: SURVEY_PRIMARY | SURVEY_BADFIELD | SURVEY_EDGE')
ps.savefig()
T = fits_table('sdss-cas-testarea-3.fits')
plt.clf()
plothist(T.ra, T.dec, 200, range=rng)
plt.title(
'PhotoObjAll: SURVEY_PRIMARY | SURVEY_BADFIELD | SURVEY_EDGE | SURVEY_BEST')
ps.savefig()
for j, (flags, txt) in enumerate([(sprim, 'PRIM'), (sbad, 'BAD'), (sedge, 'EDGE'),
(sbest, 'BEST')]):
I = np.flatnonzero(
(T.resolvestatus & (sprim | sbad | sedge | sbest)) == flags)
print(len(I), 'with', txt)
if len(I) == 0:
continue
plt.clf()
plothist(T.ra[I], T.dec[I], 200, range=rng)
plt.title('%i with %s' % (len(I), txt))
ps.savefig()
for j, (flags, txt) in enumerate([(sprim | sbad, 'PRIM + BAD'),
(sprim | sedge, 'PRIM + EDGE'),
(sprim | sbest, 'PRIM + BEST')]):
I = np.flatnonzero((T.resolvestatus & flags) > 0)
print(len(I), 'with', txt)
if len(I) == 0:
continue
plt.clf()
plothist(T.ra[I], T.dec[I], 200, range=rng)
plt.title('%i with %s' % (len(I), txt))
ps.savefig()
# for run in np.unique(T.run):
# I = (T.run == run)
# plt.clf()
# plothist(T.ra[I], T.dec[I], 200, range=rng)
# plt.title('Run %i' % run)
# ps.savefig()
R = 1. / 3600.
I, J, d = match_radec(T.ra, T.dec, T.ra, T.dec, R, notself=True)
print(len(I), 'matches')
plt.clf()
loghist((T.ra[I] - T.ra[J]) * 3600., (T.dec[I] - T.dec[J])
* 3600., 200, range=((-1, 1), (-1, 1)))
ps.savefig()
ps = PlotSequence('forced')
basedir = os.environ.get('BIGBOSS_DATA', '/project/projectdirs/bigboss')
wisedatadir = os.path.join(basedir, 'data', 'wise')
wisecat = fits_table(os.path.join(
wisedatadir, 'catalogs', 'wisecat2.fits'))
# plt.clf()
#plothist(wisecat.ra, wisecat.dec, 200, range=rng)
# plt.savefig('wisecat.png')
(ra0, ra1, dec0, dec1) = radecroi
ra = (ra0 + ra1) / 2.
dec = (dec0 + dec1) / 2.
#cas = fits_table('sdss-cas-testarea.fits')
#cas = fits_table('sdss-cas-testarea-2.fits')
cas = fits_table('sdss-cas-testarea-3.fits')
print('Read', len(cas), 'CAS sources')
cas.cut((cas.resolvestatus & sedge) == 0)
print('Cut to ', len(cas), 'without SURVEY_EDGE set')
# Check out WISE / SDSS matches.
wise = wisecat
sdss = cas
print(len(sdss), 'SDSS sources')
print(len(wise), 'WISE sources')
R = 10.
I, J, d = match_radec(wise.ra, wise.dec, sdss.ra, sdss.dec,
R / 3600., nearest=True)
print(len(I), 'matches')
print('max dist:', d.max())
plt.clf()
plt.hist(d * 3600., 100, range=(0, R), log=True)
plt.xlabel('Match distance (arcsec)')
plt.ylabel('Number of matches')
plt.title('SDSS-WISE astrometric matches')
ps.savefig()
plt.clf()
loghist((wise.ra[I] - sdss.ra[J]) * 3600., (wise.dec[I] - sdss.dec[J]) * 3600.,
200, range=((-R, R), (-R, R)))
plt.title('SDSS-WISE astrometric matches')
plt.xlabel('dRA (arcsec)')
plt.ylabel('dDec (arcsec)')
ps.savefig()
R = 4.
I, J, d = match_radec(wise.ra, wise.dec, sdss.ra, sdss.dec,
R / 3600., nearest=True)
print(len(I), 'matches')
unmatched = np.ones(len(wise), bool)
unmatched[I] = False
wun = wise[unmatched]
plt.clf()
plothist(sdss.ra, sdss.dec, 200, range=rng)
plt.title('SDSS source density')
ps.savefig()
plt.clf()
plothist(wise.ra, wise.dec, 200, range=rng)
plt.title('WISE source density')
ps.savefig()
plt.clf()
#plt.plot(wun.ra, wun.dec, 'r.')
#plt.axis(rng[0] + rng[1])
plothist(wun.ra, wun.dec, 200, range=rng)
plt.title('Unmatched WISE sources')
ps.savefig()
for band in 'ugriz':
sdss.set('psfmag_' + band,
NanoMaggies.nanomaggiesToMag(sdss.get('psfflux_' + band)))
# plt.clf()
# loghist(wise.w1mpro[I], sdss.psfmag_r[J], 200)
# plt.xlabel('WISE w1mpro')
# plt.ylabel('SDSS psfflux_r')
# ps.savefig()
for band in 'riz':
ax = [0, 10, 25, 5]
plt.clf()
mag = sdss.get('psfmag_' + band)[J]
loghist(mag - wise.w1mpro[I], mag, 200,
range=((ax[0], ax[1]), (ax[3], ax[2])))
plt.xlabel('SDSS %s - WISE w1' % band)
plt.ylabel('SDSS ' + band)
plt.axis(ax)
ps.savefig()
for w, t in [(wise[I], 'Matched'), (wun, 'Unmatched')]:
plt.clf()
w1 = w.get('w1mpro')
w2 = w.get('w2mpro')
ax = [-1, 3, 18, 6]
loghist(w1 - w2, w1, 200,
range=((ax[0], ax[1]), (ax[3], ax[2])))
plt.xlabel('W1 - W2')
plt.ylabel('W1')
plt.title('WISE CMD for %s sources' % t)
plt.axis(ax)
ps.savefig()
sdssobj = DR9()
band = 'r'
RCF = np.unique(zip(sdss.run, sdss.camcol, sdss.field))
wcses = []
fns = []
pfn = 'wcses.pickle'
if os.path.exists(pfn):
print('Reading', pfn)
wcses, fns = unpickle_from_file(pfn)
else:
for r, c, f in RCF:
fn = sdssobj.retrieve('frame', r, c, f, band)
print('got', fn)
fns.append(fn)
wcs = Tan(fn, 0)
print('got wcs', wcs)
wcses.append(wcs)
pickle_to_file((wcses, fns), pfn)
print('Wrote to', pfn)
wisefns = glob(os.path.join(wisedatadir, 'level3', '*w1-int-3.fits'))
wisewcs = []
for fn in wisefns:
print('Reading', fn)
wcs = anwcs(fn, 0)
print('Got', wcs)
wisewcs.append(wcs)
I = np.argsort(wun.w1mpro)
wun.cut(I)
for i in range(len(wun)):
ra, dec = wun.ra[i], wun.dec[i]
insdss = -1
for j, wcs in enumerate(wcses):
if wcs.is_inside(ra, dec):
insdss = j
break
inwise = -1
for j, wcs in enumerate(wisewcs):
if wcs.is_inside(ra, dec):
inwise = j
break
N = 0
if insdss != -1:
N += 1
if inwise != -1:
N += 1
if N == 0:
continue
if N != 2:
continue
plt.clf()
ss = 1
plt.subplot(2, N, ss)
ss += 1
M = 0.02
I = np.flatnonzero((sdss.ra > (ra - M)) * (sdss.ra < (ra + M)) *
(sdss.dec > (dec - M)) * (sdss.dec < (dec + M)))
sdssnear = sdss[I]
plt.plot(sdss.ra[I], sdss.dec[I], 'b.', alpha=0.7)
I = np.flatnonzero((wise.ra > (ra - M)) * (wise.ra < (ra + M)) *
(wise.dec > (dec - M)) * (wise.dec < (dec + M)))
wisenear = wise[I]
plt.plot(wise.ra[I], wise.dec[I], 'rx', alpha=0.7)
if insdss:
wcs = wcses[j]
w, h = wcs.imagew, wcs.imageh
rd = np.array([wcs.pixelxy2radec(x, y) for x, y in
[(1, 1), (w, 1), (w, h), (1, h), (1, 1)]])
plt.plot(rd[:, 0], rd[:, 1], 'b-', alpha=0.5)
if inwise:
wcs = wisewcs[j]
w, h = wcs.imagew, wcs.imageh
rd = np.array([wcs.pixelxy2radec(x, y) for x, y in
[(1, 1), (w, 1), (w, h), (1, h), (1, 1)]])
plt.plot(rd[:, 0], rd[:, 1], 'r-', alpha=0.5)
plt.plot([ra], [dec], 'o', mec='k',
mfc='none', mew=3, ms=20, alpha=0.5)
#plt.axis([ra+M, ra-M, dec-M, dec+M])
plt.axis([ra - M, ra + M, dec - M, dec + M])
plt.xticks([ra], ['RA = %0.3f' % ra])
plt.yticks([dec], ['Dec = %0.3f' % dec])
SW = 20
ss = N + 1
plt.subplot(2, N, ss)
if insdss != -1:
ss += 1
j = insdss
wcs = wcses[j]
xc, yc = wcs.radec2pixelxy(ra, dec)
r, c, f = RCF[j]
frame = sdssobj.readFrame(r, c, f, band)
#S = 50
S = SW * 3.472
im = frame.image
H, W = im.shape
y0, x0 = max(0, yc - S), max(0, xc - S)
y1, x1 = min(H, yc + S), min(W, xc + S)
subim = im[y0:y1, x0:x1]
# plt.imshow(subim, interpolation='nearest', origin='lower',
# vmax=0.3, extent=[x0,x1,y0,y1])
plt.imshow(subim.T, interpolation='nearest', origin='lower',
vmax=0.3, extent=[y0, y1, x0, x1])
# vmax=subim.max()*1.01)
ax = plt.axis()
sx, sy = wcs.radec2pixelxy(sdssnear.ra, sdssnear.dec)
#plt.plot(x, y, 'o', mec='b', mfc='none', ms=15)
plt.plot(sy, sx, 'o', mec='b', mfc='none', ms=15)
x, y = wcs.radec2pixelxy(wisenear.ra, wisenear.dec)
#plt.plot(x, y, 'rx', ms=10)
plt.plot(y, x, 'rx', ms=10)
# Which way up?
x, y = wcs.radec2pixelxy(np.array([ra, ra]), np.array(
[0.5, 2.0]) * S * 0.396 / 3600. + dec)
#plt.plot(x, y, 'b-', alpha=0.5, lw=2)
plt.plot(y, x, 'b-', alpha=0.5, lw=2)
plt.axis(ax)
plt.gray()
plt.title('SDSS %s (%i/%i/%i)' % (band, r, c, f))
# Try to guess the PRIMARY run from the nearest object.
for I in np.argsort((sx - xc)**2 + (sy - yc)**2):
r, c, f = sdssnear.run[I], sdssnear.camcol[I], sdssnear.field[I]
jj = None
for j, (ri, ci, fi) in enumerate(RCF):
if ri == r and ci == c and fi == f:
jj = j
break
assert(jj is not None)
wcs = wcses[jj]
xc, yc = wcs.radec2pixelxy(ra, dec)
frame = sdssobj.readFrame(r, c, f, band)
S = SW * 3.472
im = frame.image
H, W = im.shape
y0, x0 = max(0, yc - S), max(0, xc - S)
y1, x1 = min(H, yc + S), min(W, xc + S)
subim = im[y0:y1, x0:x1]
if np.prod(subim.shape) == 0:
continue
print('subim shape', subim.shape)
plt.subplot(2, N, 2)
plt.imshow(subim.T, interpolation='nearest', origin='lower',
vmax=0.3, extent=[y0, y1, x0, x1])
plt.gray()
plt.title('SDSS %s (%i/%i/%i)' % (band, r, c, f))
break
if inwise != -1:
plt.subplot(2, N, ss)
ss += 1
j = inwise
wcs = wisewcs[j]
ok, x, y = wcs.radec2pixelxy(ra, dec)
im = pyfits.open(wisefns[j])[0].data
S = SW
H, W = im.shape
y0, x0 = max(0, y - S), max(0, x - S)
subim = im[y0: min(H, y + S), x0: min(W, x + S)]
plt.imshow(subim, interpolation='nearest', origin='lower',
vmax=subim.max() * 1.01)
ax = plt.axis()
x, y = [], []
for r, d in zip(wisenear.ra, wisenear.dec):
ok, xi, yi = wcs.radec2pixelxy(r, d)
x.append(xi)
y.append(yi)
x = np.array(x)
y = np.array(y)
plt.plot(x - x0, y - y0, 'rx', ms=15)
x, y = [], []
for r, d in zip(sdssnear.ra, sdssnear.dec):
ok, xi, yi = wcs.radec2pixelxy(r, d)
x.append(xi)
y.append(yi)
x = np.array(x)
y = np.array(y)
plt.plot(x - x0, y - y0, 'o', mec='b', mfc='none', ms=10)
# Which way up?
pixscale = 1.375 / 3600.
ok, x1, y1 = wcs.radec2pixelxy(ra, dec + 0.5 * S * pixscale)
ok, x2, y2 = wcs.radec2pixelxy(ra, dec + 2.0 * S * pixscale)
plt.plot([x1 - x0, x2 - x0], [y1 - y0, y2 - y0],
'r-', alpha=0.5, lw=2)
plt.axis([ax[1], ax[0], ax[2], ax[3]])
# plt.axis(ax)
plt.gray()
plt.title('WISE W1 (coadd)')
plt.suptitle('WISE unmatched source: w1=%.1f, RA,Dec = (%.3f, %.3f)' %
(wun.w1mpro[i], ra, dec))
ps.savefig()
rcfs = zip(sdssnear.run, sdssnear.camcol, sdssnear.field)
print('Nearby SDSS sources are from:', np.unique(rcfs))
return
st = np.array(st)
plt.loglog([nm0[I],nm0[I]], [np.maximum(1e-6, (mn-st) / nm0[I]),
np.maximum(1e-6, (mn+st) / nm0[I])], 'b-', alpha=0.5)
plt.axhline(1., color='k', lw=2, alpha=0.5)
plt.xlabel('WISE brightness (nanomaggies)')
plt.ylabel('Tractor-measured brightness / WISE brightness')
plt.ylim(0.1, 10.)
ps.savefig()
# Match to WISE sources
r = 4./3600.
#I,J,d = match_radec(R.ra, R.dec, W.ra, W.dec, r)
#RW = R[I]
#WR = W[J]
I,J,d = match_radec(T.ra, T.dec, W.ra, W.dec, r)
TW = T[I]
WT = W[J]
WT.nm1 = NanoMaggies.magToNanomaggies(WT.w1mpro)
print 'Matched', len(TW), 'Schlegel sources to WISE'
if opt.match:
r = 4./3600.
#r = 10./3600.
I,J,d = match_radec(T.ra, T.dec, R.ra, R.dec, r, nearest=opt.nearest)
print 'Matched', len(I)
T.cut(I)
R.cut(J)
T.I = I
R.J = J
def match_ngc(self, rr, dd, radius, exts, F, primhdr, meas):
'''
rr, dd: np arrays: RA,Dec centers of chips/amps
radius: scalar, radius of chips/amps
exts: list (same len as rr,dd) of extension indices
F: fitsio.FITS object
meas: measurement class
'''
I, J, d = match_radec(rr, dd, self.cat.ra, self.cat.dec, radius)
# plt.clf()
# angles = np.linspace(0, 2.*np.pi, 40)
# for r,d in zip(rr, dd):
# plt.plot(r + radius * np.sin(angles) / np.cos(np.deg2rad(d)), d + radius * np.cos(angles), 'b-')
# plt.plot(rr, dd, 'bo')
# ax = plt.axis()
# plt.plot(self.cat.ra, self.cat.dec, 'r.')
# plt.axis(ax)
# ps.savefig()
print('Matched', len(I), 'NGC objects')
if len(I) == 0:
return False
for j in J:
info = ngc_typenames.get(self.cat.classification[j].strip(), '')
print(' ', self.cat.name[j], info)
# Potential tweet texts and plot filenames
tweets = []
goodplots = []
for i, j in zip(I, J):
ext = exts[i]
obj = self.cat[j]
obj.name = obj.name.strip()
hdr = F[ext].read_header()
extname = hdr['EXTNAME'].strip()
expnum = primhdr['EXPNUM']
wcs = meas.get_wcs(hdr)
ok, x, y = wcs.radec2pixelxy(obj.ra, obj.dec)
x = x - 1
y = y - 1
# Choose cutout area
pixrad = 1.4 * obj.radius * 3600. / wcs.pixel_scale()
pixrad = max(pixrad, 100)
# print('radius:', obj.radius, 'pixel radius:', pixrad)
# HACK
#pixrad *= 3
r = pixrad
tt = '%s in exp %i ext %s (%i)' % (obj.name, expnum, extname, ext)
print(tt)
# Find the cutout region... does it actually overlap the chip?
H, W = wcs.shape
xl, xh = int(np.clip(x - r, 0,
W - 1)), int(np.clip(x + r, 0, W - 1))
yl, yh = int(np.clip(y - r, 0,
H - 1)), int(np.clip(y + r, 0, H - 1))
if xl == xh or yl == yh:
print('no actual overlap with image')
continue
sh, sw = yh - yl, xh - xl
if sh < 25 or sw < 25:
print('tiny overlap', sw, 'x', sh)
continue
# Measure the image!
flat = None
if self.read_flats:
band = meas.get_band(hdr)
flat = self.get_flat(band, ext, meas)
if flat is not None:
print('flat: range', flat.min(), flat.max(), 'median',
np.median(flat))
meas.ext = ext
meas.edge_trim = 20
debugps = None
if self.opt.ps:
debugps = ps
try:
M = meas.run(n_fwhm=1,
verbose=False,
get_image=True,
flat=flat,
ps=debugps)
except KeyboardInterrupt:
sys.exit(0)
except:
import traceback
print('Failed to measure file', meas.fn, 'ext', ext, ':')
traceback.print_exc()
continue
#print('Measured:', M.keys())
raw = M['image']
# Now repeat the cutout check with the trimmed image
wcs = M['wcs']
# Trim WCS to trimmed raw image shape
trim_x0, trim_y0 = M['trim_x0'], M['trim_y0']
H, W = raw.shape
wcs = wcs.get_subimage(trim_x0, trim_y0, W, H)
ok, x, y = wcs.radec2pixelxy(obj.ra, obj.dec)
x = x - 1
y = y - 1
xl, xh = int(np.clip(x - r, 0,
W - 1)), int(np.clip(x + r, 0, W - 1))
yl, yh = int(np.clip(y - r, 0,
H - 1)), int(np.clip(y + r, 0, H - 1))
if xl == xh or yl == yh:
print('no actual overlap with image')
continue
subimg = raw[yl:yh, xl:xh]
sh, sw = subimg.shape
if sh < 25 or sw < 25:
print('tiny overlap', sw, 'x', sh)
continue
subwcs = wcs.get_subimage(xl, yl, sw, sh)
if False:
# Flat image for the same CCD region.
flatfn = '/tmp/mzls/mos3.127506.fits'
# meas_class = get_measurer_class_for_file(flatfn)
# if meas_class is None:
# print('Failed to identify camera in', flatfn)
# return
# flatmeas = meas_class(flatfn, 0, self.nom)
# print('Flat meas:', flatmeas)
# flatmeas.ext = ext
# flathdr = fitsio.read_header(flatfn)
# flatmeas.primhdr = flathdr
# flatmeas.edge_trim = 20
# FM = flatmeas.run(n_fwhm=1, verbose=False, get_image=True)
# flatraw = FM['image']
F = fitsio.FITS(flatfn)
flatraw, flathdr = meas.read_raw(F, ext)
flatsub = flatraw[yl:yh, xl:xh]
zerofn = '/tmp/mzls/mos3.127522.fits'
F = fitsio.FITS(zerofn)
zeroraw, zerohdr = meas.read_raw(F, ext)
zerosub = zeroraw[yl:yh, xl:xh]
rawfn = meas.fn
F = fitsio.FITS(rawfn)
rawimg, rawhdr = meas.read_raw(F, ext)
rawsub = rawimg[yl:yh, xl:xh]
# Astrometric shifts
dx = M['dx']
dy = M['dy']
aff = M['affine']
x = (xl + xh) / 2
y = (yl + yh) / 2
#print('Affine correction terms:', aff)
adx = x - aff[0]
ady = y - aff[1]
corrx = aff[2] + aff[3] * adx + aff[4] * ady - adx
corry = aff[5] + aff[6] * adx + aff[7] * ady - ady
print('Affine correction', corrx, corry)
# Shift the 'subwcs' to account for astrometric offset
cx, cy = subwcs.get_crpix()
subwcs.set_crpix((cx - dx - corrx, cy - dy - corry))
# Now grab existing data from the LegacySurvey site
# What size of image are we going to request?
scale = 1.
if max(sh, sw) > 1024:
scale = 4.
elif max(sh, sw) > 512:
scale = 2.
rh, rw = int(np.ceil(sh / scale)), int(np.ceil(sw / scale))
# make it square
mx = max(rh, rw)
rw = rh = mx
fitsimgs = []
# We'll resample the new image into the existing-image WCS.
newimg = None
for layer in legacy_survey_layers:
url = (
'http://legacysurvey.org/viewer-dev/fits-cutout/?ra=%.4f&dec=%.4f&pixscale=%.3f&width=%i&height=%i&layer=%s'
% (obj.ra, obj.dec, subwcs.pixel_scale() * scale, rw, rh,
layer))
print('URL:', url)
r = requests.get(url)
ftmp = tempfile.NamedTemporaryFile()
ftmp.write(r.content)
ftmp.flush()
#fits,hdr = fitsio.read(ftmp.name, header=True)
fitsfile = fitsio.FITS(ftmp.name)
ftmp.close()
print('FITS file:', len(fitsfile), 'extensions')
hdr = fitsfile[0].read_header()
fits = fitsfile[0].read()
#print('fits:', fits)
if fits is None:
print('no coverage in layer', layer)
continue
# If you need to keep a copy (debugging...)
# f,tmpfn = tempfile.mkstemp(suffix='.fits')
# os.write(f, r.content)
# os.close(f)
# fits,hdr = fitsio.read(tmpfn, header=True)
# print('Wrote FITS to', tmpfn)
if np.all(fits == 0):
continue
### HACK -- surface brightness correction...
if layer == 'sdssco':
s = (subwcs.pixel_scale() / 0.396)
fits *= s**2
N, ww, hh = fits.shape
# pull out the image planes
imgs = [fits[n, :, :] for n in range(N)]
bands = hdr['BANDS'].strip()
fitsimgs.append((layer, bands, imgs))
# Resample the new image to this layer's WCS
if newimg is not None:
continue
thiswcs = Tan(hdr)
newimg = np.zeros((rh, rw), dtype=subimg.dtype)
try:
#Yo,Xo,Yi,Xi,rims = resample_with_wcs(thiswcs, subwcs)
# Laczos
Yo, Xo, Yi, Xi, rims = resample_with_wcs(
thiswcs, subwcs, [subimg])
except:
continue
#newimg[Yo,Xo] = subimg[Yi,Xi]
newimg[Yo, Xo] = rims[0]
if len(fitsimgs) == 0:
# No overlap with existing surveys
continue
#print()
newband = primhdr['FILTER'][0]
#print('New image is', newband)
if False:
mn, mx = np.percentile(flatsub, [25, 99])
plt.clf()
#plt.imshow(newflat, interpolation='nearest', origin='lower',
plt.imshow(flatsub,
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
#plt.colorbar()
plt.colorbar()
plt.savefig('ngcbot-flat.png')
med = np.median(newflat.ravel())
#mn,mx = np.percentile(newimg, [25,99])
#mn,mx = np.percentile(newimg, [25,90])
#mn,mx = np.percentile(rawsub, [25,95])
mn, mx = np.percentile(rawsub, [50, 95])
plt.clf()
# plt.subplot(1,2,1)
#plt.imshow(newimg, interpolation='nearest', origin='lower',
# plt.imshow(subimg, interpolation='nearest', origin='lower',
plt.imshow(rawsub,
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
plt.colorbar()
plt.savefig('ngcbot-unflattened.png')
plt.clf()
#plt.subplot(1,2,2)
#plt.imshow(newimg / (newflat / med), interpolation='nearest', origin='lower',
#plt.imshow(subimg / (flatsub / med), interpolation='nearest', origin='lower',
plt.imshow(rawsub / (flatsub / med),
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
plt.colorbar()
plt.savefig('ngcbot-flattened.png')
mn, mx = np.percentile(zerosub, [5, 95])
plt.clf()
plt.imshow(zerosub,
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
plt.colorbar()
plt.savefig('ngcbot-zero.png')
zp = M['zp']
zpscale = 10.**((zp - 22.5) / 2.5)
exptime = primhdr['EXPTIME']
newimg /= (zpscale * exptime)
nh, nw = newimg.shape
coverage = np.sum(newimg != 0) / float(nw * nh)
print('Fraction', coverage, 'of new image has data')
def my_rgb(imgs, bands, **kwargs):
return sdss_rgb(imgs,
bands,
scales=dict(g=6.0, r=3.4, i=2.5, z=2.2),
m=-0.02,
clip=False,
**kwargs)
def grayscale(img, band):
rgb = my_rgb([img, img, img], [band, band, band])
index = 'zrg'.index(newband)
gray = rgb[:, :, index]
return gray
# plot title args
targs = dict(fontsize=8)
# DEBUG
if self.opt.ps:
ocx, ocy = subwcs.get_crpix()
subwcs.set_crpix((cx, cy))
newimgA = np.zeros((rh, rw), dtype=subimg.dtype)
Yo, Xo, Yi, Xi, rims = resample_with_wcs(
thiswcs, subwcs, [subimg])
newimgA[Yo, Xo] = rims[0]
subwcs.set_crpix((cx - dx, cy - dy))
newimgB = np.zeros((rh, rw), dtype=subimg.dtype)
Yo, Xo, Yi, Xi, rims = resample_with_wcs(
thiswcs, subwcs, [subimg])
newimgB[Yo, Xo] = rims[0]
subwcs.set_crpix((cx - dx - corrx, cy - dy - corry))
newimgC = np.zeros((rh, rw), dtype=subimg.dtype)
Yo, Xo, Yi, Xi, rims = resample_with_wcs(
thiswcs, subwcs, [subimg])
newimgC[Yo, Xo] = rims[0]
#newgray = grayscale(newimg, newband)
#hi = np.percentile(newgray, 99.9)
grayargs = dict(interpolation='nearest',
origin='lower',
cmap='gray',
vmin=0.)
#vmin=0., vmax=hi, cmap='gray')
(layer, bands, imgs) = fitsimgs[0]
nicelayer = nicelayernames.get(layer, layer)
for band, img in zip(bands, imgs):
newbands = ['g', 'r', 'z']
newindex = dict(g=0, r=1, i=2, z=2)
if newindex[band] == newindex[newband]:
oldgray = grayscale(img, band)
plt.clf()
plt.imshow(oldgray, **grayargs)
plt.title(nicelayer)
ps.savefig()
plt.clf()
plt.imshow(grayscale(newimgA, newband), **grayargs)
#plt.imshow(newimgA, **grayargs)
plt.title('No astromety correction')
ps.savefig()
plt.clf()
plt.imshow(grayscale(newimgB, newband), **grayargs)
#plt.imshow(newimgB, **grayargs)
plt.title('Astromety shift only correction')
ps.savefig()
plt.clf()
plt.imshow(grayscale(newimgC, newband), **grayargs)
#plt.imshow(newimgC, **grayargs)
plt.title('Astromety shift+affine correction')
ps.savefig()
subwcs.set_crpix((ocx, ocy))
plt.clf()
plt.subplots_adjust(left=0.03, right=0.97, bottom=0.03)
NC = 1 + len(fitsimgs)
NR = 2
# New Image plot
newgray = grayscale(newimg, newband)
hi = np.percentile(newgray, 99.9)
grayargs = dict(interpolation='nearest',
origin='lower',
vmin=0.,
vmax=hi,
cmap='gray')
plt.subplot(NR, NC, 1)
plt.imshow(newgray, **grayargs)
plt.xticks([])
plt.yticks([])
plt.title('New image (%s)' % newband, **targs)
# Select images & bands for the New RGB image.
zeroimg = np.zeros_like(newimg)
newimgs = [zeroimg, zeroimg, zeroimg]
# sdss_rgb reverses the order, so do like grz.
newbands = ['g', 'r', 'z']
newindex = dict(g=0, r=1, i=2, z=2)
# Start with the new image plane of the New RGB image
j = newindex[newband]
newimgs[j] = newimg
newbands[j] = newband
#print('Setting band', newband, '(index %i)'%j, 'to new image')
# We wait to choose the scaling until after the "New RGB" image
# has been built, so we store the RGB images along the way...
rgbs = []
# In the link to the legacysurvey viewer, we select the best
# imaging layer. The DECaLS layer is listed first, so if it
# has three bands, it wins.
bestdata = None
bestn = 0
for i, (layer, bands, imgs) in enumerate(fitsimgs):
nicelayer = nicelayernames.get(layer, layer)
# For DECaLS missing bands: "--z"
nicebands = ''
goodbands = []
goodimgs = []
for band, img in zip(bands, imgs):
if np.all(img == 0.):
print('Band', band, 'of', nicelayer, 'is all zero')
nicebands += '-'
continue
goodbands.append(band)
goodimgs.append(img)
nicebands += band
#print(' ', nicelayer, band)
j = newindex[band]
if newimgs[j] is zeroimg:
#print(' Setting band', band, '(index %i)'%j, 'to', nicelayer)
newimgs[j] = img
newbands[j] = band
elif band == newbands[j]:
# patch empty regions if same band
#print(' Patching index %i from' % j, nicelayer, 'band', band)
Z = (newimgs[j] == 0)
newimgs[j][Z] = img[Z]
# z -> i
if newindex[band] == newindex[newband]:
# Old image grayscale
oldgray = grayscale(img, band)
plt.subplot(NR, NC, 2 + i)
plt.imshow(oldgray, **grayargs)
plt.xticks([])
plt.yticks([])
plt.title('%s (%s)' % (nicelayer, band), **targs)
if len(goodbands) == 1:
#rgb = grayscale(goodimgs[0], goodbands[0])
img, band = goodimgs[0], goodbands[0]
rgb = my_rgb([img, img, img], [band, band, band])
else:
rgb = my_rgb(imgs, bands)
if len(goodbands) > bestn:
bestn = len(goodbands)
bestdata = layer
# print('bands for', nicelayer, ':', bands, ', actually', nicebands)
rgbs.append(
(2 + i + NC, rgb, '%s (%s)' % (nicelayer, nicebands)))
# list to string
newbands = ''.join(newbands)
#print('Newbands:', newbands)
# New RGB
plt.subplot(NR, NC, 1 + NC)
rgb = my_rgb(newimgs, newbands)
lo = 0.
hi = np.percentile(rgb.ravel(), 99.9)
rgb = np.clip((rgb - lo) / (hi - lo), 0., 1.)
plt.imshow(rgb, interpolation='nearest', origin='lower')
plt.xticks([])
plt.yticks([])
plt.title('New+Old (%s)' % newbands, **targs)
# Old RGBs
for sp, rgb, tt in rgbs:
plt.subplot(NR, NC, sp)
rgb = np.clip((rgb - lo) / (hi - lo), 0., 1.)
plt.imshow(rgb, interpolation='nearest', origin='lower')
plt.xticks([])
plt.yticks([])
plt.title(tt, **targs)
# NGC classification
info = ''
info = ngc_typenames.get(obj.classification.strip(), '')
if len(info):
info = '(' + info + ') '
plt.suptitle(
'%s %sin %s %i-%s: %s band' %
(obj.name, info, nice_camera_name, expnum, extname, newband))
# Save / show plot
if self.opt.show:
plt.draw()
plt.show(block=False)
plt.pause(0.001)
plotfn = ('ngcbot-%i-%s-%s.png' %
(expnum, extname, obj.name.replace(' ', '_')))
if self.opt.plotdir:
plotfn = os.path.join(self.opt.plotdir, plotfn)
plt.savefig(plotfn)
print('Saved', plotfn)
good_coverage = 0.75
if self.opt.coverage is not None:
good_coverage = self.opt.coverage
if coverage > good_coverage:
goodplots.append(plotfn)
# Compose tweet text
if self.opt.tweet:
import urllib
url = 'http://legacysurvey.org/viewer/?ra=%.3f&dec=%.3f' % (
obj.ra, obj.dec)
if bestdata is not None:
url += '&layer=%s' % bestdata
url2 = ('http://ned.ipac.caltech.edu/cgi-bin/objsearch?' +
urllib.urlencode(
dict(objname=obj.name,
corr_z=1,
list_limit=5,
img_stamp='YES')))
dateobs = primhdr['DATE-OBS'].strip()
# 2017-03-06T00:15:40.101482
dateobs = dateobs[:19]
dateobs = dateobs.replace('T', ' ')
txt = ('%s observed %s %sin image %i-%s: %s band' %
(nice_camera_name, obj.name, info, expnum, extname,
newband) + ' at %s UT' % dateobs + '\n' + url + '\n' +
url2)
print('Tweet text:', txt)
if coverage > good_coverage:
tweets.append((txt, plotfn))
else:
print('Coverage', coverage, 'less than target',
good_coverage, 'so not tweeting')
# Select a random good-looking plot
if len(goodplots):
irandom = np.random.randint(0, len(goodplots))
# Copy that plot to ngcbot-latest.png
plotfn = goodplots[irandom]
import shutil
latest = 'ngcbot-latest.png'
print('Copying', plotfn, 'to', latest)
shutil.copy(plotfn, latest)
# Tweet one NGC object per exposure, chosen randomly.
if len(tweets):
assert (len(tweets) == len(goodplots))
txt, plotfn = tweets[irandom]
if self.opt.tweet:
send_tweet(txt, plotfn)
return True
def star_profiles(ps):
# Run an example CCD, 292604-N4, with fairly large difference vs PS1.
# python -c "from astrometry.util.fits import *; T = merge_tables([fits_table('/project/projectdirs/desiproc/dr3/tractor/244/tractor-244%s.fits' % b) for b in ['2p065','4p065', '7p065']]); T.writeto('tst-cat.fits')"
# python legacypipe/forced_photom_decam.py --save-data tst-data.fits --save-model tst-model.fits 292604 N4 tst-cat.fits tst-phot.fits
# -> tst-{model,data,phot}.fits
datafn = 'tst-data.fits'
modfn = 'tst-model.fits'
photfn = 'tst-phot.fits'
catfn = 'tst-cat.fits'
img = fitsio.read(datafn)
mod = fitsio.read(modfn)
phot = fits_table(photfn)
cat = fits_table(catfn)
print(len(phot), 'forced-photometry results')
margin = 25
phot.cut((phot.x > 0+margin) * (phot.x < 2046-margin) *
(phot.y > 0+margin) * (phot.y < 4096-margin))
print(len(phot), 'in bounds')
cmap = dict([((b,o),i) for i,(b,o) in enumerate(zip(cat.brickname, cat.objid))])
I = np.array([cmap.get((b,o), -1) for b,o in zip(phot.brickname, phot.objid)])
print(np.sum(I >= 0), 'forced-phot matched cat')
phot.type = cat.type[I]
wcs = Sip(datafn)
phot.ra,phot.dec = wcs.pixelxy2radec(phot.x+1, phot.y+1)
phot.cut(np.argsort(phot.flux))
phot.sn = phot.flux * np.sqrt(phot.flux_ivar)
phot.cut(phot.sn > 5)
print(len(phot), 'with S/N > 5')
ps1 = ps1cat(ccdwcs=wcs)
stars = ps1.get_stars()
print(len(stars), 'PS1 sources')
# Now cut to just *stars* with good colors
stars.gicolor = stars.median[:,0] - stars.median[:,2]
keep = (stars.gicolor > 0.4) * (stars.gicolor < 2.7)
stars.cut(keep)
print(len(stars), 'PS1 stars with good colors')
stars.cut(np.minimum(stars.stdev[:,1], stars.stdev[:,2]) < 0.05)
print(len(stars), 'PS1 stars with min stdev(r,i) < 0.05')
I,J,d = match_radec(phot.ra, phot.dec, stars.ra, stars.dec, 1./3600.)
print(len(I), 'matches')
plt.clf()
ha=dict(histtype='step', bins=20, range=(0,100), normed=True)
plt.hist(phot.flux, color='b', **ha)
plt.hist(phot.flux[I], color='r', **ha)
ps.savefig()
plt.clf()
plt.hist(phot.flux * np.sqrt(phot.flux_ivar), bins=100,
range=(-10, 50))
plt.xlabel('Flux S/N')
ps.savefig()
K = np.argsort(phot.flux[I])
I = I[K]
J = J[K]
ix = np.round(phot.x).astype(int)
iy = np.round(phot.y).astype(int)
sz = 10
P = np.flatnonzero(phot.type == 'PSF ')
print(len(P), 'PSFs')
imed = len(P)/2
i1 = int(len(P) * 0.75)
i2 = int(len(P) * 0.25)
N = 401
allmods = []
allimgs = []
for II,tt in [#(I[:len(I)/2], 'faint matches to PS1'),
#(I[len(I)/2:], 'bright matches to PS1'),
#(P[i2: i2+N], '25th pct PSFs'),
#(P[imed: imed+N], 'median PSFs'),
#(P[i1: i1+N], '75th pct PSFs'),
#(P[-25:], 'brightest PSFs'),
(P[i2:imed], '2nd quartile of PSFs'),
(P[imed:i1], '3rd quartile of PSFs'),
#(P[:len(P)/2], 'faint half of PSFs'),
#(P[len(P)/2:], 'bright half of PSFs'),
]:
imgs = []
mods = []
shimgs = []
shmods = []
imgsum = modsum = 0
#plt.clf()
for i in II:
from astrometry.util.util import lanczos_shift_image
dy = phot.y[i] - iy[i]
dx = phot.x[i] - ix[i]
sub = img[iy[i]-sz : iy[i]+sz+1, ix[i]-sz : ix[i]+sz+1]
shimg = lanczos_shift_image(sub, -dx, -dy)
sub = mod[iy[i]-sz : iy[i]+sz+1, ix[i]-sz : ix[i]+sz+1]
shmod = lanczos_shift_image(sub, -dx, -dy)
iyslice = img[iy[i], ix[i]-sz : ix[i]+sz+1]
myslice = mod[iy[i], ix[i]-sz : ix[i]+sz+1]
ixslice = img[iy[i]-sz : iy[i]+sz+1, ix[i]]
mxslice = mod[iy[i]-sz : iy[i]+sz+1, ix[i]]
mx = iyslice.max()
# plt.plot(iyslice/mx, 'b-', alpha=0.1)
# plt.plot(myslice/mx, 'r-', alpha=0.1)
# plt.plot(ixslice/mx, 'b-', alpha=0.1)
# plt.plot(mxslice/mx, 'r-', alpha=0.1)
siyslice = shimg[sz, :]
sixslice = shimg[:, sz]
smyslice = shmod[sz, :]
smxslice = shmod[:, sz]
shimgs.append(siyslice/mx)
shimgs.append(sixslice/mx)
shmods.append(smyslice/mx)
shmods.append(smxslice/mx)
imgs.append(iyslice/mx)
imgs.append(ixslice/mx)
mods.append(myslice/mx)
mods.append(mxslice/mx)
imgsum = imgsum + ixslice + iyslice
modsum = modsum + mxslice + myslice
# plt.ylim(-0.1, 1.1)
# plt.title(tt)
# ps.savefig()
mimg = np.median(np.array(imgs), axis=0)
mmod = np.median(np.array(mods), axis=0)
mshim = np.median(np.array(shimgs), axis=0)
mshmo = np.median(np.array(shmods), axis=0)
allmods.append(mshmo)
allimgs.append(mshim)
plt.clf()
# plt.plot(mimg, 'b-')
# plt.plot(mmod, 'r-')
plt.plot(mshim, 'g-')
plt.plot(mshmo, 'm-')
plt.ylim(-0.1, 1.1)
plt.title(tt + ': median; sums %.3f/%.3f' % (np.sum(mimg), np.sum(mmod)))
ps.savefig()
# plt.clf()
# mx = imgsum.max()
# plt.plot(imgsum/mx, 'b-')
# plt.plot(modsum/mx, 'r-')
# plt.ylim(-0.1, 1.1)
# plt.title(tt + ': sum')
# ps.savefig()
plt.clf()
plt.plot((mimg + 0.01) / (mmod + 0.01), 'k-')
plt.plot((imgsum/mx + 0.01) / (modsum/mx + 0.01), 'g-')
plt.plot((mshim + 0.01) / (mshmo + 0.01), 'm-')
plt.ylabel('(img + 0.01) / (mod + 0.01)')
plt.title(tt)
ps.savefig()
iq2,iq3 = allimgs
mq2,mq3 = allmods
plt.clf()
plt.plot(iq2, 'r-')
plt.plot(mq2, 'm-')
plt.plot(iq3, 'b-')
plt.plot(mq3, 'g-')
plt.title('Q2 vs Q3')
ps.savefig()
def decam():
'''
cp ~/cosmo/staging/decam/DECam_CP/CP20170731/c4d_170801_080516_oki_g_v1.fits.fz /tmp
funpack /tmp/c4d_170801_080516_oki_g_v1.fits.fz
fitsgetext -i /tmp/c4d_170801_080516_oki_g_v1.fits -o decam-%02i.wcs -a -H
cat decam-??.wcs > decam.wcs
for ((i=1; i<=61; i++)); do modhead decam.wcs+$i NAXIS2 0; done
'''
plt.figure(figsize=(4,3))
plt.subplots_adjust(left=0.15, right=0.99, top=0.99, bottom=0.15)
T = fits_table('obstatus/decam-tiles_obstatus.fits')
T.rename('pass', 'passnum')
print('Passes:', np.unique(T.passnum))
for p in np.unique(T.passnum):
I = np.flatnonzero(T.passnum == p)
plt.clf()
plt.plot(T.ra[I], T.dec[I], 'b.')
plt.title('Pass %s' % p)
plt.savefig('pass%i.png' % p)
ra,dec = 0.933, 0.
I,J,d = match_radec(T.ra, T.dec, ra, dec, 5.) #2.8)
print(len(I), 'tiles near 0,0')
T.cut(I)
T.dist = d
print('dists:', d)
print('Passes:', T.passnum)
F = fitsio.FITS(os.path.join(os.path.dirname(__file__), 'decam.wcs'))
wcs = []
for i in range(1, len(F)):
hdr = F[i].read_header()
wcs.append(wcs_pv2sip_hdr(hdr, W=2046, H=4094))
W,H = 5000, 5000
pixsc = 4./3600.
targetwcs = Tan(ra, dec, W/2.+0.5, H/2.+0.5, -pixsc, 0., 0., pixsc,
float(W), float(H))
II = np.lexsort((T.dist, T.passnum))
# This is for making the (vector) PDF format tiling images.
for maxit in [0, 6, 30, 31, 37, 61, 62, 68, 90]:
#mx = { 1: 2, 2: 4, 3: 6 }[t.passnum]
plot = Plotstuff(outformat='pdf', ra=ra, dec=dec, width=W*pixsc,
size=(W,H), outfn='tile-%02i.pdf' % maxit)
plot.color = 'white'
plot.alpha = 1.
plot.plot('fill')
out = plot.outline
out.fill = True
out.stepsize = 1024.
plot.color = 'black'
plot.alpha = 0.4
plot.apply_settings()
for it,t in enumerate(T[II]):
print('Tile', it, 'pass', t.passnum)
for w in wcs:
w.set_crval((t.ra, t.dec))
out.wcs = anwcs_new_sip(w)
plot.plot('outline')
if it == maxit:
print('Writing', it)
plot.write()
break
# And this is for PNG-format tiling images and histograms.
cov = np.zeros((H,W), np.uint8)
for it,t in enumerate(T[II]):
print('Tile', it, 'pass', t.passnum)
for w in wcs:
w.set_crval((t.ra, t.dec))
#print('WCS:', w)
try:
Yo,Xo,Yi,Xi,nil = resample_with_wcs(targetwcs, w)
except:
#import traceback
#traceback.print_exc()
continue
cov[Yo,Xo] += 1
if it in [0, 6, 30, 31, 37, 61, 62, 68, 90]:
mx = { 1: 2, 2: 4, 3: 6 }[t.passnum]
# plt.clf()
# plt.imshow(cov, interpolation='nearest', origin='lower', vmin=0, vmax=mx)
# plt.colorbar()
# plt.savefig('tile-%02i.png' % it)
plt.imsave('tile-%02i.png' % it, cov, origin='lower', vmin=0, vmax=mx, cmap=antigray)
#plt.imsave('tile-%02i.pdf' % it, cov, origin='lower', vmin=0, vmax=mx, cmap=antigray, format='pdf')
if it in [30, 61, 90]:
from collections import Counter
print('Coverage counts:', Counter(cov.ravel()).most_common())
bins = -0.5 + np.arange(8)
plt.clf()
ncov = cov.ravel()
ncov = ncov / np.sum(ncov)
n,b,p = plt.hist(ncov, bins=bins)
# Cumulative histogram from the right...
xx,yy = [],[]
for blo,bhi,ni in reversed(list(zip(bins, bins[1:], n))):
nc = float(np.sum(cov.ravel() > blo)) / len(cov.ravel())
yy.extend([nc,nc])
xx.extend([bhi,blo])
if ni > 0:
if nc != ni:
if nc > ni+0.03:
# If there's room, label the histogram bin above, else below
plt.text((blo+bhi)/2., ni, '%.1f \%%' % (100.*ni), ha='center', va='bottom', color='k')
else:
plt.text((blo+bhi)/2., ni-0.01, '%.1f \%%' % (100.*ni), ha='center', va='top', color='k')
plt.text((blo+bhi)/2., nc, '%.1f \%%' % (100.*nc), ha='center', va='bottom', color='k')
plt.plot(xx, yy, 'k-')
plt.xlim(bins.min(), bins.max())
plt.ylim(0., 1.1)
plt.xlabel('Number of exposures')
plt.ylabel('Fraction of sky')
#plt.title('DECaLS tiling, %i pass%s' % (t.passnum, t.passnum > 1 and 'es' or ''))
#plt.savefig('hist-%02i.png' % it)
plt.savefig('hist-%02i.pdf' % it)
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).")
parser.add_argument('--calibs', action='store_true',
help='Output CCDs that need to be calibrated.')
parser.add_argument('--nper', type=int, default=None,
help='Batch N calibs per line')
parser.add_argument('--forced', action='store_true',
help='Output forced-photometry commands')
parser.add_argument('--lsb', action='store_true',
help='Output Low-Surface-Brightness commands')
parser.add_argument('--touching', action='store_true',
help='Cut to only CCDs touching selected bricks')
parser.add_argument('--check', action='store_true',
help='Check which calibrations actually need to run.')
parser.add_argument('--check-coadd', action='store_true',
help='Check which caoadds actually need to run.')
parser.add_argument('--out', help='Output filename for calibs, default %(default)s',
default='jobs')
parser.add_argument('--command', action='store_true',
help='Write out full command-line to run calib')
parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
parser.add_argument('--mindec', type=float, help='Minimum Dec to run')
parser.add_argument('--region', help='Region to select')
parser.add_argument('--bricks', help='Set bricks.fits file to load')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--delete-sky', action='store_true',
help='Delete any existing sky calibration files')
parser.add_argument('--delete-pvastrom', action='store_true',
help='Delete any existing PV WCS calibration files')
parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')
opt = parser.parse_args()
decals = Decals()
if opt.bricks is not None:
B = fits_table(opt.bricks)
log('Read', len(B), 'from', opt.bricks)
else:
B = decals.get_bricks()
if opt.ccds is not None:
T = fits_table(opt.ccds)
log('Read', len(T), 'from', opt.ccds)
else:
T = decals.get_ccds()
log(len(T), 'CCDs')
T.index = np.arange(len(T))
# I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
# plt.clf()
# plt.hist(counts, counts.max()+1)
# plt.savefig('bricks.png')
# B.cut(I[counts >= 9])
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# #plt.scatter(B.ra[I], B.dec[I], c=counts)
# plt.savefig('bricks2.png')
# DES Stripe82
#rlo,rhi = 350.,360.
# rlo,rhi = 300., 10.
# dlo,dhi = -6., 4.
# TINY bit
#rlo,rhi = 350.,351.1
#dlo,dhi = 0., 1.1
# EDR+
# 860 bricks
# ~10,000 CCDs
#rlo,rhi = 239,246
#dlo,dhi = 5, 13
# DR1
#rlo,rhi = 0, 360
# part 1
#dlo,dhi = 25, 40
# part 2
#dlo,dhi = 20,25
# part 3
#dlo,dhi = 15,20
# part 4
#dlo,dhi = 10,15
# part 5
#dlo,dhi = 5,10
# the rest
#dlo,dhi = -11, 5
#dlo,dhi = 15,25.5
dlo,dhi = -15, 40
rlo,rhi = 0, 360
# Arjun says 3x3 coverage area is roughly
# RA=240-252 DEC=6-12 (but not completely rectangular)
# COSMOS
#rlo,rhi = 148.9, 151.2
#dlo,dhi = 0.9, 3.5
# A nice well-behaved region (EDR2/3)
# rlo,rhi = 243.6, 244.6
# dlo,dhi = 8.1, 8.6
# 56 bricks, ~725 CCDs
#B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
# 240 bricks, ~3000 CCDs
#B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
# 535 bricks, ~7000 CCDs
#B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))
if opt.region in ['test1', 'test2', 'test3', 'test4']:
nm = dict(test1='2446p115', # weird stuff around bright star
test2='1183p292', # faint sources around bright galaxy
test3='3503p005', # DES
test4='1163p277', # Pollux
)[opt.region]
B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
log('Cut to', len(B), 'bricks')
log(B.ra, B.dec)
dlo,dhi = -90,90
rlo,rhi = 0, 360
elif opt.region == 'badsky':
# A handful of exposures in DR2 with inconsistent sky estimates.
#C = decals.get_ccds()
#log(len(C), 'CCDs')
#C.cut((C.expnum >= 257400) * (C.expnum < 257500))
T.cut(np.array([e in [257460, 257461, 257462, 257463, 257464,
257465, 257466, 257467, 257469, 257472,
257483, 257496]
for e in T.expnum]))
log(len(T), 'CCDs with bad sky')
# CCD radius
radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600.
# Brick radius
radius += np.hypot(0.25, 0.25)/2.
I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05)
keep = np.zeros(len(B), bool)
keep[I] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs with bad sky')
elif opt.region == 'badsky2':
# UGH, missed this one in original 'badsky' definition.
T.cut(T.expnum == 257466)
log(len(T), 'CCDs with bad sky')
# CCD radius
radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600.
# Brick radius
radius += np.hypot(0.25, 0.25)/2.
I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05)
keep = np.zeros(len(B), bool)
keep[I] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs with bad sky')
elif opt.region == 'edr':
# EDR:
# 535 bricks, ~7000 CCDs
rlo,rhi = 240,245
dlo,dhi = 5, 12
elif opt.region == 'edr-south':
rlo,rhi = 240,245
dlo,dhi = 5, 10
elif opt.region == 'cosmos1':
# 16 bricks in the core of the COSMOS field.
rlo,rhi = 149.75, 150.75
dlo,dhi = 1.6, 2.6
elif opt.region == 'pristine':
# Stream?
rlo,rhi = 240,250
dlo,dhi = 10,15
elif opt.region == 'des':
dlo, dhi = -6., 4.
rlo, rhi = 317., 7.
T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
log('Cut to', len(T), 'CCDs with "CPDES82" in filename')
elif opt.region == 'subdes':
rlo,rhi = 320., 360.
dlo,dhi = -1.25, 1.25
elif opt.region == 'northwest':
rlo,rhi = 240,360
dlo,dhi = 20,40
elif opt.region == 'north':
rlo,rhi = 120,240
dlo,dhi = 20,40
elif opt.region == 'northeast':
rlo,rhi = 0,120
dlo,dhi = 20,40
elif opt.region == 'southwest':
rlo,rhi = 240,360
dlo,dhi = -20,0
elif opt.region == 'south':
rlo,rhi = 120,240
dlo,dhi = -20,0
elif opt.region == 'southeast':
rlo,rhi = 0,120
dlo,dhi = -20,0
elif opt.region == 'midwest':
rlo,rhi = 240,360
dlo,dhi = 0,20
elif opt.region == 'middle':
rlo,rhi = 120,240
dlo,dhi = 0,20
elif opt.region == 'mideast':
rlo,rhi = 0,120
dlo,dhi = 0,20
elif opt.region == 'grz':
# Bricks with grz coverage.
# Be sure to use --bricks decals-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
log('Cut to', len(B), 'bricks with grz coverage')
elif opt.region == 'nogrz':
# Bricks without grz coverage.
# Be sure to use --bricks decals-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
log('Cut to', len(B), 'bricks withOUT grz coverage')
elif opt.region == 'deep2':
rlo,rhi = 250,260
dlo,dhi = 30,35
elif opt.region == 'virgo':
rlo,rhi = 185,190
dlo,dhi = 10, 15
elif opt.region == 'virgo2':
rlo,rhi = 182,192
dlo,dhi = 8, 18
elif opt.region == 'lsb':
rlo,rhi = 147.2, 147.8
dlo,dhi = -0.4, 0.4
if opt.mindec is not None:
dlo = opt.mindec
if opt.maxdec is not None:
dhi = opt.maxdec
if rlo < rhi:
B.cut((B.ra >= rlo) * (B.ra <= rhi) *
(B.dec >= dlo) * (B.dec <= dhi))
else: # RA wrap
B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) *
(B.dec >= dlo) * (B.dec <= dhi))
log(len(B), 'bricks in range')
I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, 0.25)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs')
if opt.touching:
keep = np.zeros(len(T), bool)
for j in J:
keep[j] = True
T.cut(keep)
log('Cut to', len(T), 'CCDs near bricks')
# Aside -- how many near DR1=1 CCDs?
if False:
T2 = D.get_ccds()
log(len(T2), 'CCDs')
T2.cut(T2.dr1 == 1)
log(len(T2), 'CCDs marked DR1=1')
log(len(B), 'bricks in range')
I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, 0.25)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B2 = B[keep]
log('Total of', len(B2), 'bricks near CCDs with DR1=1')
for band in 'grz':
Tb = T2[T2.filter == band]
log(len(Tb), 'in filter', band)
I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, 0.25)
good = np.zeros(len(B2), np.uint8)
for i in I:
good[i] = 1
B2.set('has_' + band, good)
B2.writeto('decals-bricks-in-dr1.fits')
sys.exit(0)
# sort by dec decreasing
B.cut(np.argsort(-B.dec))
for b in B:
if opt.check:
fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
if opt.check_coadd:
fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
print(b.brickname)
if not (opt.calibs or opt.forced or opt.lsb):
sys.exit(0)
bands = 'grz'
log('Filters:', np.unique(T.filter))
T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
log('Cut to', len(T), 'CCDs in filters', bands)
if opt.touching:
allI = set()
for b in B:
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
if len(I) == 0:
continue
allI.update(I)
allI = list(allI)
allI.sort()
else:
allI = np.arange(len(T))
if opt.write_ccds:
T[allI].writeto(opt.write_ccds)
log('Wrote', opt.write_ccds)
## Be careful here -- T has been cut; we want to write out T.index.
## 'allI' contains indices into T.
if opt.forced:
log('Writing forced-photometry commands to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
for j,i in enumerate(allI):
expstr = '%08i' % T.expnum[i]
#outdir = os.path.join('forced', expstr[:5], expstr)
#trymakedirs(outdir)
outfn = os.path.join('forced', expstr[:5], expstr,
'decam-%s-%s-forced.fits' %
(expstr, T.ccdname[i]))
imgfn = os.path.join(decals.decals_dir, 'images',
T.image_filename[i].strip())
if (not os.path.exists(imgfn) and
imgfn.endswith('.fz') and
os.path.exists(imgfn[:-3])):
imgfn = imgfn[:-3]
f.write('python legacypipe/forced-photom-decam.py %s %i DR1 %s\n' %
(imgfn, T.cpimage_hdu[i], outfn))
f.close()
log('Wrote', opt.out)
sys.exit(0)
if opt.lsb:
log('Writing LSB commands to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
for j,i in enumerate(allI):
exp = T.expnum[i]
ext = T.ccdname[i].strip()
outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext))
f.close()
log('Wrote', opt.out)
sys.exit(0)
log('Writing calibs to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
batch = []
def write_batch(f, batch, cmd):
if opt.command:
s = '; '.join(batch)
else:
s = ' '.join(batch)
f.write(s + '\n')
for j,i in enumerate(allI):
if opt.delete_sky or opt.delete_pvastrom:
log(j+1, 'of', len(allI))
im = decals.get_image_object(T[i])
if opt.delete_sky and os.path.exists(im.skyfn):
log(' deleting:', im.skyfn)
os.unlink(im.skyfn)
if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
log(' deleting:', im.pvwcsfn)
os.unlink(im.pvwcsfn)
if opt.check:
log(j+1, 'of', len(allI))
im = decals.get_image_object(T[i])
if not im.run_calibs(im, just_check=True):
log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done')
continue
if opt.command:
s = ('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
(T.expnum[i], T.ccdname[i]))
else:
s = '%i' % T.index[i]
if not opt.nper:
f.write(s + '\n')
else:
batch.append(s)
if len(batch) >= opt.nper:
write_batch(f, batch, opt.command)
batch = []
if opt.check:
f.flush()
if len(batch):
write_batch(f, batch, opt.command)
f.close()
log('Wrote', opt.out)
return 0
def map_decals_wl(req, ver, zoom, x, y):
tag = 'decals-wl'
ignoreCached = False
filename = None
forcecache = False
from decals import settings
savecache = settings.SAVE_CACHE
zoom = int(zoom)
zoomscale = 2.**zoom
x = int(x)
y = int(y)
if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale:
raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom, x, y))
ver = int(ver)
if not ver in tileversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
basedir = settings.DATA_DIR
tilefn = os.path.join(basedir, 'tiles', tag,
'%i/%i/%i/%i.jpg' % (ver, zoom, x, y))
if os.path.exists(tilefn) and not ignoreCached:
print('Cached:', tilefn)
return send_file(tilefn,
'image/jpeg',
expires=oneyear,
modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'),
filename=filename)
else:
print('Tile image does not exist:', tilefn)
from astrometry.util.resample import resample_with_wcs, OverlapError
from astrometry.util.util import Tan
from astrometry.libkd.spherematch import match_radec
from astrometry.util.fits import fits_table
from astrometry.util.starutil_numpy import degrees_between
import numpy as np
import fitsio
try:
wcs, W, H, zoomscale, zoom, x, y = get_tile_wcs(zoom, x, y)
except RuntimeError as e:
return HttpResponse(e.strerror)
mydir = os.path.join(basedir, 'coadd', 'weak-lensing')
rlo, d = wcs.pixelxy2radec(W, H / 2)[-2:]
rhi, d = wcs.pixelxy2radec(1, H / 2)[-2:]
r, d1 = wcs.pixelxy2radec(W / 2, 1)[-2:]
r, d2 = wcs.pixelxy2radec(W / 2, H)[-2:]
#dlo = min(d1, d2)
#dhi = max(d1, d2)
r, d = wcs.pixelxy2radec(W / 2, H / 2)[-2:]
rad = degrees_between(r, d, rlo, d1)
fn = os.path.join(mydir, 'index.fits')
if not os.path.exists(fn):
#
ii, rr, dd = [], [], []
for i in range(1, 52852 + 1):
imgfn = os.path.join(mydir, 'map%i.fits' % i)
hdr = fitsio.read_header(imgfn)
r = hdr['CRVAL1']
d = hdr['CRVAL2']
ii.append(i)
rr.append(r)
dd.append(d)
T = fits_table()
T.ra = np.array(rr)
T.dec = np.array(dd)
T.i = np.array(ii)
T.writeto(fn)
T = fits_table(fn)
I, J, d = match_radec(T.ra, T.dec, r, d, rad + 0.2)
T.cut(I)
print(len(T), 'weak-lensing maps in range')
if len(I) == 0:
from django.http import HttpResponseRedirect
if forcecache:
# create symlink to blank.jpg!
trymakedirs(tilefn)
src = os.path.join(settings.STATIC_ROOT, 'blank.jpg')
if os.path.exists(tilefn):
os.unlink(tilefn)
os.symlink(src, tilefn)
print('Symlinked', tilefn, '->', src)
return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg')
r, d = wcs.pixelxy2radec([1, 1, 1, W / 2, W, W, W, W / 2],
[1, H / 2, H, H, H, H / 2, 1, 1])[-2:]
foundany = False
rimg = np.zeros((H, W), np.float32)
rn = np.zeros((H, W), np.uint8)
for tilei in T.i:
fn = os.path.join(mydir, 'map%i.fits' % tilei)
try:
bwcs = read_tan_wcs(fn, 0)
except:
print('Failed to read WCS:', fn)
savecache = False
import traceback
import sys
traceback.print_exc(None, sys.stdout)
continue
foundany = True
print('Reading', fn)
ok, xx, yy = bwcs.radec2pixelxy(r, d)
xx = xx.astype(np.int)
yy = yy.astype(np.int)
imW, imH = int(bwcs.get_width()), int(bwcs.get_height())
M = 10
xlo = np.clip(xx.min() - M, 0, imW)
xhi = np.clip(xx.max() + M, 0, imW)
ylo = np.clip(yy.min() - M, 0, imH)
yhi = np.clip(yy.max() + M, 0, imH)
if xlo >= xhi or ylo >= yhi:
continue
subwcs = bwcs.get_subimage(xlo, ylo, xhi - xlo, yhi - ylo)
slc = slice(ylo, yhi), slice(xlo, xhi)
try:
f = fitsio.FITS(fn)[0]
img = f[slc]
del f
except:
print('Failed to read image and WCS:', fn)
savecache = False
import traceback
import sys
traceback.print_exc(None, sys.stdout)
continue
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(wcs, subwcs, [], 3)
except OverlapError:
print('Resampling exception')
continue
rimg[Yo, Xo] += img[Yi, Xi]
rn[Yo, Xo] += 1
rimg /= np.maximum(rn, 1)
if forcecache:
savecache = True
if savecache:
trymakedirs(tilefn)
else:
import tempfile
f, tilefn = tempfile.mkstemp(suffix='.jpg')
os.close(f)
import pylab as plt
# S/N
#lo,hi = 1.5, 5.0
lo, hi = 0, 5.0
rgb = plt.cm.hot((rimg - lo) / (hi - lo))
plt.imsave(tilefn, rgb)
print('Wrote', tilefn)
return send_file(tilefn,
'image/jpeg',
unlink=(not savecache),
filename=filename)
def plots(opt):
from astrometry.util.plotutils import antigray
import tractor.sfd
T = fits_table(opt.files[0])
print('Read', len(T), 'bricks summarized in', opt.files[0])
import pylab as plt
import matplotlib
B = fits_table('survey-bricks.fits.gz')
print('Looking up brick bounds')
ibrick = dict([(n, i) for i, n in enumerate(B.brickname)])
bi = np.array([ibrick[n] for n in T.brickname])
T.ra1 = B.ra1[bi]
T.ra2 = B.ra2[bi]
T.dec1 = B.dec1[bi]
T.dec2 = B.dec2[bi]
assert (np.all(T.ra2 > T.ra1))
T.area = ((T.ra2 - T.ra1) * (T.dec2 - T.dec1) *
np.cos(np.deg2rad((T.dec1 + T.dec2) / 2.)))
del B
del bi
del ibrick
print('Total sources:', sum(T.nobjs))
print('Approx area:', len(T) / 16., 'sq deg')
print('Area:', np.sum(T.area))
print('g,r,z coverage:',
sum((T.nexp_g > 0) * (T.nexp_r > 0) * (T.nexp_z > 0)) / 16.)
decam = True
if decam:
release = 'DECaLS DR8'
else:
release = 'BASS+MzLS DR8'
if decam:
# DECam
#ax = [360, 0, -21, 36]
ax = [300, -60, -75, 36]
def map_ra(r):
return r + (-360 * (r > 300))
else:
# MzLS+BASS
ax = [310, 90, 30, 80]
def map_ra(r):
return r
udec = np.unique(T.dec)
print('Number of unique Dec values:', len(udec))
print('Number of unique Dec values in range', ax[2], ax[3], ':',
np.sum((udec >= ax[2]) * (udec <= ax[3])))
def radec_plot():
plt.axis(ax)
plt.xlabel('RA (deg)')
if decam:
# plt.xticks(np.arange(0, 361, 45))
#tt = np.arange(0, 361, 60)
#plt.xticks(tt, map_ra(tt))
plt.xticks([-60, 0, 60, 120, 180, 240, 300],
[300, 0, 60, 120, 180, 240, 300])
else:
plt.xticks(np.arange(90, 311, 30))
plt.ylabel('Dec (deg)')
def plot_broken(rr, dd, *args, **kwargs):
dr = np.abs(np.diff(rr))
I = np.flatnonzero(dr > 90)
#print('breaks:', rr[I])
#print('breaks:', rr[I+1])
if len(I) == 0:
plt.plot(rr, dd, *args, **kwargs)
return
for lo, hi in zip(np.append([0], I + 1), np.append(I + 1, -1)):
#print('Cut:', lo, ':', hi, '->', rr[lo], rr[hi-1])
plt.plot(rr[lo:hi], dd[lo:hi], *args, **kwargs)
# Galactic plane lines
gl = np.arange(361)
gb = np.zeros_like(gl)
from astrometry.util.starutil_numpy import lbtoradec
rr, dd = lbtoradec(gl, gb)
plot_broken(map_ra(rr), dd, 'k-', alpha=0.5, lw=1)
rr, dd = lbtoradec(gl, gb + 10)
plot_broken(map_ra(rr), dd, 'k-', alpha=0.25, lw=1)
rr, dd = lbtoradec(gl, gb - 10)
plot_broken(map_ra(rr), dd, 'k-', alpha=0.25, lw=1)
plt.figure(1, figsize=(8, 5))
plt.subplots_adjust(left=0.1, right=0.98, top=0.93)
plt.figure(2, figsize=(8, 4))
#plt.subplots_adjust(left=0.06, right=0.98, top=0.98)
plt.subplots_adjust(left=0.08, right=0.98, top=0.98)
plt.figure(1)
# Map of the tile centers we want to observe...
"""
if decam:
O = fits_table('obstatus/decam-tiles_obstatus.fits')
else:
O = fits_table('mosaic-tiles_obstatus.fits')
"""
# File from the "observing" svn repo:
from pkg_resources import resource_filename
if decam:
tilefile = resource_filename('legacyzpts',
'data/decam-tiles_obstatus.fits')
else:
tilefile = resource_filename('legacyzpts',
'data/mosaic-tiles_obstatus.fits')
O = fits_table(tilefile)
O.cut(O.in_desi == 1)
rr, dd = np.meshgrid(np.linspace(ax[1], ax[0], 700),
np.linspace(ax[2], ax[3], 200))
from astrometry.libkd.spherematch import match_radec
I, J, d = match_radec(O.ra, O.dec, rr.ravel(), dd.ravel(), 1.)
desimap = np.zeros(rr.shape, bool)
desimap.flat[J] = True
# Smoothed DESI boundary contours
from scipy.ndimage.filters import gaussian_filter
from scipy.ndimage.morphology import binary_dilation
C = plt.contour(gaussian_filter(
binary_dilation(desimap).astype(np.float32), 2), [0.5],
extent=[ax[1], ax[0], ax[2], ax[3]])
plt.clf()
desi_map_boundaries = C.collections[0]
def desi_map_outline():
segs = desi_map_boundaries.get_segments()
for seg in segs:
plt.plot(seg[:, 0], seg[:, 1], 'b-')
def desi_map():
# Show the DESI tile map in the background.
plt.imshow(desimap,
origin='lower',
interpolation='nearest',
extent=[ax[1], ax[0], ax[2], ax[3]],
aspect='auto',
cmap=antigray,
vmax=8)
base_cmap = 'viridis'
# Dust map -- B&W version
nr, nd = 610, 350
plt.figure(2)
plt.clf()
dmap = np.zeros((nd, nr))
rr = np.linspace(ax[0], ax[1], nr)
dd = np.linspace(ax[2], ax[3], nd)
rr = rr[:-1] + 0.5 * (rr[1] - rr[0])
dd = dd[:-1] + 0.5 * (dd[1] - dd[0])
rr, dd = np.meshgrid(rr, dd)
I, J, d = match_radec(rr.ravel(),
dd.ravel(),
O.ra,
O.dec,
1.0,
nearest=True)
iy, ix = np.unravel_index(I, rr.shape)
#dmap[iy,ix] = O.ebv_med[J]
sfd = tractor.sfd.SFDMap()
ebv = sfd.ebv(rr[iy, ix], dd[iy, ix])
dmap[iy, ix] = ebv
mx = np.percentile(dmap[dmap > 0], 98)
plt.imshow(dmap,
extent=[ax[0], ax[1], ax[2], ax[3]],
interpolation='nearest',
origin='lower',
aspect='auto',
cmap='Greys',
vmin=0,
vmax=mx)
#desi_map_outline()
radec_plot()
cax = colorbar_axes(plt.gca(), frac=0.12)
cbar = plt.colorbar(cax=cax)
cbar.set_label('Extinction E(B-V)')
plt.savefig('ext-bw.pdf')
plt.clf()
dmap = sfd.ebv(rr.ravel(), dd.ravel()).reshape(rr.shape)
plt.imshow(dmap,
extent=[ax[0], ax[1], ax[2], ax[3]],
interpolation='nearest',
origin='lower',
aspect='auto',
cmap='Greys',
vmin=0,
vmax=0.25)
desi_map_outline()
radec_plot()
cax = colorbar_axes(plt.gca(), frac=0.12)
cbar = plt.colorbar(cax=cax)
cbar.set_label('Extinction E(B-V)')
plt.savefig('ext-bw-2.pdf')
plt.figure(1)
#sys.exit(0)
plt.clf()
depthlo, depthhi = 21.5, 25.5
for band in 'grz':
depth = T.get('galdepth_%s' % band)
ha = dict(histtype='step', bins=50, range=(depthlo, depthhi))
ccmap = dict(g='g', r='r', z='m')
plt.hist(depth[depth > 0],
label='%s band' % band,
color=ccmap[band],
**ha)
plt.xlim(depthlo, depthhi)
plt.xlabel('Galaxy depth (median per brick) (mag)')
plt.ylabel('Number of Bricks')
plt.title(release)
plt.savefig('galdepths.png')
for band in 'grz':
depth = T.get('galdepth_%s' % band)
nexp = T.get('nexp_%s' % band)
#lo,hi = 22.0-0.05, 24.2+0.05
lo, hi = depthlo - 0.05, depthhi + 0.05
nbins = 1 + int((depthhi - depthlo) / 0.1)
ha = dict(histtype='step', bins=nbins, range=(lo, hi))
ccmap = dict(g='g', r='r', z='m')
area = 0.25**2
plt.clf()
I = np.flatnonzero((depth > 0) * (nexp == 1))
plt.hist(depth[I],
label='%s band, 1 exposure' % band,
color=ccmap[band],
lw=1,
weights=area * np.ones_like(depth[I]),
**ha)
I = np.flatnonzero((depth > 0) * (nexp == 2))
plt.hist(depth[I],
label='%s band, 2 exposures' % band,
color=ccmap[band],
lw=2,
alpha=0.5,
weights=area * np.ones_like(depth[I]),
**ha)
I = np.flatnonzero((depth > 0) * (nexp >= 3))
plt.hist(depth[I],
label='%s band, 3+ exposures' % band,
color=ccmap[band],
lw=3,
alpha=0.3,
weights=area * np.ones_like(depth[I]),
**ha)
plt.title('%s: galaxy depths, %s band' % (release, band))
plt.xlabel('5-sigma galaxy depth (mag)')
plt.ylabel('Square degrees')
plt.xlim(lo, hi)
plt.xticks(np.arange(depthlo, depthhi + 0.01, 0.2))
plt.legend(loc='upper right')
plt.savefig('depth-hist-%s.png' % band)
for band in 'grz':
plt.clf()
desi_map()
N = T.get('nexp_%s' % band)
I = np.flatnonzero(N > 0)
#cm = matplotlib.cm.get_cmap('jet', 6)
#cm = matplotlib.cm.get_cmap('winter', 5)
mx = 10
cm = cmap_discretize(base_cmap, mx)
plt.scatter(map_ra(T.ra[I]),
T.dec[I],
c=N[I],
s=3,
edgecolors='none',
vmin=0.5,
vmax=mx + 0.5,
cmap=cm)
radec_plot()
cax = colorbar_axes(plt.gca(), frac=0.08)
plt.colorbar(cax=cax, ticks=range(mx + 1))
plt.title('%s: Number of exposures in %s' % (release, band))
plt.savefig('nexp-%s.png' % band)
#cmap = cmap_discretize(base_cmap, 15)
cmap = cmap_discretize(base_cmap, 10)
plt.clf()
desi_map()
psf = T.get('psfsize_%s' % band)
I = np.flatnonzero(psf > 0)
plt.scatter(map_ra(T.ra[I]),
T.dec[I],
c=psf[I],
s=3,
edgecolors='none',
cmap=cmap,
vmin=0.5,
vmax=2.5)
#vmin=0, vmax=3.)
radec_plot()
plt.colorbar()
plt.title('%s: PSF size, band %s' % (release, band))
plt.savefig('psfsize-%s.png' % band)
plt.clf()
desi_map()
depth = T.get('galdepth_%s' % band) - T.get('ext_%s' % band)
mn, mx = np.percentile(depth[depth > 0], [10, 98])
mn = np.floor(mn * 10) / 10.
mx = np.ceil(mx * 10) / 10.
cmap = cmap_discretize(base_cmap, 1 + int((mx - mn + 0.001) / 0.1))
I = (depth > 0)
plt.scatter(map_ra(T.ra[I]),
T.dec[I],
c=depth[I],
s=3,
edgecolors='none',
vmin=mn - 0.05,
vmax=mx + 0.05,
cmap=cmap)
radec_plot()
plt.colorbar()
plt.title(
'%s: galaxy depth, band %s, median per brick, extinction-corrected'
% (release, band))
plt.savefig('galdepth-%s.png' % band)
# B&W version
plt.figure(2)
plt.clf()
mn, mx = np.percentile(depth[depth > 0], [2, 98])
print('Raw mn,mx', mn, mx)
mn = np.floor((mn + 0.05) * 10) / 10. - 0.05
mx = np.ceil((mx - 0.05) * 10) / 10. + 0.05
print('rounded mn,mx', mn, mx)
nsteps = int((mx - mn + 0.001) / 0.1)
print('discretizing into', nsteps, 'colormap bins')
#nsteps = 1+int((mx-mn+0.001)/0.1)
cmap = cmap_discretize(antigray, nsteps)
nr, nd = 610, 228
dmap = np.zeros((nd, nr))
rr = np.linspace(ax[0], ax[1], nr)
dd = np.linspace(ax[2], ax[3], nd)
rr = rr[:-1] + 0.5 * (rr[1] - rr[0])
dd = dd[:-1] + 0.5 * (dd[1] - dd[0])
rr, dd = np.meshgrid(rr, dd)
I, J, d = match_radec(rr.ravel(),
dd.ravel(),
T.ra,
T.dec,
0.2,
nearest=True)
iy, ix = np.unravel_index(I, rr.shape)
dmap[iy, ix] = depth[J]
plt.imshow(dmap,
extent=[ax[0], ax[1], ax[2], ax[3]],
interpolation='nearest',
origin='lower',
aspect='auto',
cmap=cmap,
vmin=mn,
vmax=mx)
desi_map_outline()
radec_plot()
cax = colorbar_axes(plt.gca(), frac=0.12)
cbar = plt.colorbar(
cax=cax, ticks=np.arange(20, 26, 0.5)
) #ticks=np.arange(np.floor(mn/5.)*5., 0.1+np.ceil(mx/5.)*5, 0.2))
cbar.set_label('Depth (5-sigma, galaxy profile, AB mag)')
plt.savefig('galdepth-bw-%s.pdf' % band)
plt.figure(1)
plt.clf()
desi_map()
ext = T.get('ext_%s' % band)
mn = 0.
mx = 0.5
cmap = 'hot'
cmap = cmap_discretize(cmap, 10)
#cmap = cmap_discretize(base_cmap, 1+int((mx-mn+0.001)/0.1))
plt.scatter(map_ra(T.ra),
T.dec,
c=ext,
s=3,
edgecolors='none',
vmin=mn,
vmax=mx,
cmap=cmap)
radec_plot()
plt.colorbar()
plt.title('%s: extinction, band %s' % (release, band))
plt.savefig('ext-%s.png' % band)
T.ngal = T.nsimp + T.nrex + T.nexp + T.ndev + T.ncomp
for col in [
'nobjs', 'npsf', 'nsimp', 'nrex', 'nexp', 'ndev', 'ncomp', 'ngal'
]:
if not col in T.get_columns():
continue
plt.clf()
desi_map()
N = T.get(col) / T.area
mx = np.percentile(N, 99.5)
plt.scatter(map_ra(T.ra),
T.dec,
c=N,
s=3,
edgecolors='none',
vmin=0,
vmax=mx)
radec_plot()
cbar = plt.colorbar()
cbar.set_label('Objects per square degree')
tt = 'of type %s' % col[1:]
if col == 'nobjs':
tt = 'total'
plt.title('%s: Number of objects %s' % (release, tt))
plt.savefig('nobjs-%s.png' % col[1:])
# B&W version
plt.figure(2)
plt.clf()
# plt.scatter(map_ra(T.ra), T.dec, c=N, s=3,
# edgecolors='none', vmin=0, vmax=mx, cmap=antigray)
# Approximate pixel size in PNG plot
# This doesn't work correctly -- we've already binned to brick resolution, so get moire patterns
# nobjs,xe,ye = np.histogram2d(map_ra(T.ra), T.dec, weights=T.get(col),
# bins=(nr,nd), range=((ax[1],ax[0]),(ax[2],ax[3])))
# nobjs = nobjs.T
# area = np.diff(xe)[np.newaxis,:] * (np.diff(ye) * np.cos(np.deg2rad(ye[:-1])))[:,np.newaxis]
# nobjs /= area
# plt.imshow(nobjs, extent=[ax[1],ax[0],ax[2],ax[3]], interpolation='nearest', origin='lower',
# aspect='auto')
#print('Computing neighbours for nobjs plot...')
nr, nd = 610, 228
nobjs = np.zeros((nd, nr))
rr = np.linspace(ax[0], ax[1], nr)
dd = np.linspace(ax[2], ax[3], nd)
rr = rr[:-1] + 0.5 * (rr[1] - rr[0])
dd = dd[:-1] + 0.5 * (dd[1] - dd[0])
rr, dd = np.meshgrid(rr, dd)
I, J, d = match_radec(rr.ravel(),
dd.ravel(),
T.ra,
T.dec,
0.2,
nearest=True)
iy, ix = np.unravel_index(I, rr.shape)
nobjs[iy, ix] = T.get(col)[J] / T.area[J]
#print('done')
#mx = 2. * np.median(nobjs[nobjs > 0])
mx = np.percentile(N, 99)
plt.imshow(nobjs,
extent=[ax[0], ax[1], ax[2], ax[3]],
interpolation='nearest',
origin='lower',
aspect='auto',
cmap='Greys',
vmin=0,
vmax=mx)
desi_map_outline()
radec_plot()
#cax = colorbar_axes(plt.gca(), frac=0.08)
cax = colorbar_axes(plt.gca(), frac=0.12)
cbar = plt.colorbar(cax=cax,
format=matplotlib.ticker.FuncFormatter(
lambda x, p: format(int(x), ',')))
cbar.set_label('Objects per square degree')
plt.savefig('nobjs-bw-%s.pdf' % col[1:])
#plt.savefig('nobjs-bw-%s.png' % col[1:])
plt.figure(1)
Ntot = T.nobjs
for col in ['npsf', 'nsimp', 'nrex', 'nexp', 'ndev', 'ncomp', 'ngal']:
if not col in T.get_columns():
continue
plt.clf()
desi_map()
N = T.get(col) / (Ntot.astype(np.float32))
N[Ntot == 0] = 0.
print(col, 'max frac:', N.max())
mx = np.percentile(N, 99.5)
print('mx', mx)
plt.scatter(map_ra(T.ra),
T.dec,
c=N,
s=3,
edgecolors='none',
vmin=0,
vmax=mx)
radec_plot()
plt.colorbar()
plt.title('%s: Fraction of objects of type %s' % (release, col[1:]))
plt.savefig('fobjs-%s.png' % col[1:])
# B&W version
plt.figure(2)
plt.clf()
#plt.scatter(map_ra(T.ra), T.dec, c=N * 100., s=3,
# edgecolors='none', vmin=0, vmax=mx*100., cmap=antigray)
fobjs = np.zeros((nd, nr))
rr = np.linspace(ax[0], ax[1], nr)
dd = np.linspace(ax[2], ax[3], nd)
rr = rr[:-1] + 0.5 * (rr[1] - rr[0])
dd = dd[:-1] + 0.5 * (dd[1] - dd[0])
rr, dd = np.meshgrid(rr, dd)
I, J, d = match_radec(rr.ravel(),
dd.ravel(),
T.ra,
T.dec,
0.2,
nearest=True)
iy, ix = np.unravel_index(I, rr.shape)
fobjs[iy, ix] = N[J] * 100.
#mx = 2. * np.median(fobjs[fobjs > 0])
mx = np.percentile(N * 100., 99)
plt.imshow(fobjs,
extent=[ax[0], ax[1], ax[2], ax[3]],
interpolation='nearest',
origin='lower',
aspect='auto',
cmap='Greys',
vmin=0,
vmax=mx)
desi_map_outline()
radec_plot()
cax = colorbar_axes(plt.gca(), frac=0.12)
cbar = plt.colorbar(
cax=cax,
format=matplotlib.ticker.FuncFormatter(lambda x, p: '%.2g' % x))
cbar.set_label('Percentage of objects of type %s' % col[1:].upper())
plt.savefig('fobjs-bw-%s.pdf' % col[1:])
#plt.savefig('fobjs-bw-%s.png' % col[1:])
plt.figure(1)
return 0
print('Kd data:', kd.get_data(np.array([0, 3, 5]).astype(np.uint32)))
print('Permute:',
spherematch.tree_permute(kd,
np.array([3, 5, 7]).astype(np.int32)))
print('Permute:', kd.permute(np.array([0, 99, 199]).astype(np.int32)))
ra, dec = np.meshgrid(np.arange(0, 360), np.arange(-90, 91, 1))
ra1 = ra.ravel()
dec1 = dec.ravel()
rdkd1 = spherematch.tree_build_radec(ra1, dec1)
print('RdKd:', rdkd1.n, rdkd1.bbox)
ra2 = np.random.uniform(-10, 10, size=1000)
dec2 = np.random.uniform(-10, 10, size=1000)
rdkd2 = spherematch.tree_build_radec(ra2, dec2)
I = spherematch.tree_search_radec(rdkd1, ra2[0], dec2[0], 2.)
print('search_radec:', I)
I, J, d = spherematch.match_radec(ra1, dec1, ra2, dec2, 1.)
print('Matches:', len(I))
I, J, d = spherematch.match_radec(ra1, dec1, ra2, dec2, 1., nearest=True)
print('Nearest matches:', len(I))
I = spherematch.match_radec(ra1, dec1, ra2, dec2, 1., indexlist=True)
print('Index lists matches:', len(I))
def galex_coadds(onegal,
galaxy=None,
radius_mosaic=30,
radius_mask=None,
pixscale=1.5,
ref_pixscale=0.262,
output_dir=None,
galex_dir=None,
log=None,
centrals=True,
verbose=False):
'''Generate custom GALEX cutouts.
radius_mosaic and radius_mask in arcsec
pixscale: GALEX pixel scale in arcsec/pixel.
'''
import fitsio
import matplotlib.pyplot as plt
from astrometry.libkd.spherematch import match_radec
from astrometry.util.resample import resample_with_wcs, OverlapError
from tractor import (Tractor, NanoMaggies, Image, LinearPhotoCal,
NCircularGaussianPSF, ConstantFitsWcs, ConstantSky)
from legacypipe.survey import imsave_jpeg
from legacypipe.catalog import read_fits_catalog
if galaxy is None:
galaxy = 'galaxy'
if galex_dir is None:
galex_dir = os.environ.get('GALEX_DIR')
if output_dir is None:
output_dir = '.'
if radius_mask is None:
radius_mask = radius_mosaic
radius_search = 5.0 # [arcsec]
else:
radius_search = radius_mask
W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int') # [pixels]
targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0,
-pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W),
float(H))
# Read the custom Tractor catalog
tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy))
if not os.path.isfile(tractorfile):
print('Missing Tractor catalog {}'.format(tractorfile))
return 0
cat = fits_table(tractorfile)
print('Read {} sources from {}'.format(len(cat), tractorfile),
flush=True,
file=log)
keep = np.ones(len(cat)).astype(bool)
if centrals:
# Find the large central galaxy and mask out (ignore) all the models
# which are within its elliptical mask.
# This algorithm will have to change for mosaics not centered on large
# galaxies, e.g., in galaxy groups.
m1, m2, d12 = match_radec(cat.ra,
cat.dec,
onegal['RA'],
onegal['DEC'],
radius_search / 3600.0,
nearest=False)
if len(m1) == 0:
print('No central galaxies found at the central coordinates!',
flush=True,
file=log)
else:
pixfactor = ref_pixscale / pixscale # shift the optical Tractor positions
for mm in m1:
morphtype = cat.type[mm].strip()
if morphtype == 'EXP' or morphtype == 'COMP':
e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[
mm], cat.shapeexp_r[mm] # [arcsec]
elif morphtype == 'DEV' or morphtype == 'COMP':
e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[
mm], cat.shapedev_r[mm] # [arcsec]
else:
r50 = None
if r50:
majoraxis = r50 * 5 / pixscale # [pixels]
ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2)
these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis,
ba * majoraxis,
np.radians(phi),
cat.bx * pixfactor,
cat.by * pixfactor)
if np.sum(these) > 0:
#keep[these] = False
pass
print('Hack!')
keep[mm] = False
#srcs = read_fits_catalog(cat)
#_srcs = np.array(srcs)[~keep].tolist()
#mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0)
#import matplotlib.pyplot as plt
##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png')
#plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png')
#pdb.set_trace()
srcs = read_fits_catalog(cat)
for src in srcs:
src.freezeAllBut('brightness')
#srcs_nocentral = np.array(srcs)[keep].tolist()
# Find all overlapping GALEX tiles and then read the tims.
galex_tiles = _read_galex_tiles(targetwcs,
galex_dir,
log=log,
verbose=verbose)
gbands = ['n', 'f']
nicegbands = ['NUV', 'FUV']
zps = dict(n=20.08, f=18.82)
coimgs, comods, coresids, coimgs_central, comods_nocentral = [], [], [], [], []
for niceband, band in zip(nicegbands, gbands):
J = np.flatnonzero(galex_tiles.get('has_' + band))
print(len(J), 'GALEX tiles have coverage in band', band)
coimg = np.zeros((H, W), np.float32)
comod = np.zeros((H, W), np.float32)
cowt = np.zeros((H, W), np.float32)
comod_nocentral = np.zeros((H, W), np.float32)
for src in srcs:
src.setBrightness(NanoMaggies(**{band: 1}))
for j in J:
brick = galex_tiles[j]
fn = os.path.join(
galex_dir, brick.tilename.strip(),
'%s-%sd-intbgsub.fits.gz' % (brick.brickname, band))
#print(fn)
gwcs = Tan(*[
float(f) for f in [
brick.crval1, brick.crval2, brick.crpix1, brick.crpix2,
brick.cdelt1, 0., 0., brick.cdelt2, 3840., 3840.
]
])
img = fitsio.read(fn)
#print('Read', img.shape)
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, gwcs, [], 3)
except OverlapError:
continue
K = np.flatnonzero(img[Yi, Xi] != 0.)
if len(K) == 0:
continue
Yo, Xo, Yi, Xi = Yo[K], Xo[K], Yi[K], Xi[K]
wt = brick.get(band + 'exptime')
coimg[Yo, Xo] += wt * img[Yi, Xi]
cowt[Yo, Xo] += wt
x0, x1, y0, y1 = min(Xi), max(Xi), min(Yi), max(Yi)
subwcs = gwcs.get_subimage(x0, y0, x1 - x0 + 1, y1 - y0 + 1)
twcs = ConstantFitsWcs(subwcs)
timg = img[y0:y1 + 1, x0:x1 + 1]
tie = np.ones_like(timg) ## HACK!
#hdr = fitsio.read_header(fn)
#zp = hdr['']
zp = zps[band]
photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp),
band=band)
tsky = ConstantSky(0.0)
# HACK -- circular Gaussian PSF of fixed size...
# in arcsec
#fwhms = dict(NUV=6.0, FUV=6.0)
# -> sigma in pixels
#sig = fwhms[band] / 2.35 / twcs.pixel_scale()
sig = 6.0 / np.sqrt(8 * np.log(2)) / twcs.pixel_scale()
tpsf = NCircularGaussianPSF([sig], [1.])
tim = Image(data=timg,
inverr=tie,
psf=tpsf,
wcs=twcs,
sky=tsky,
photocal=photocal,
name='GALEX ' + band + brick.brickname)
## Build the model image with and without the central galaxy model.
tractor = Tractor([tim], srcs)
mod = tractor.getModelImage(0)
tractor.freezeParam('images')
tractor.optimize_forced_photometry(priors=False,
shared_params=False)
mod = tractor.getModelImage(0)
srcs_nocentral = np.array(srcs)[keep].tolist()
#srcs_nocentral = np.array(srcs)[nocentral].tolist()
tractor_nocentral = Tractor([tim], srcs_nocentral)
mod_nocentral = tractor_nocentral.getModelImage(0)
comod[Yo, Xo] += wt * mod[Yi - y0, Xi - x0]
comod_nocentral[Yo, Xo] += wt * mod_nocentral[Yi - y0, Xi - x0]
coimg /= np.maximum(cowt, 1e-18)
comod /= np.maximum(cowt, 1e-18)
comod_nocentral /= np.maximum(cowt, 1e-18)
coresid = coimg - comod
# Subtract the model image which excludes the central (comod_nocentral)
# from the data (coimg) to isolate the light of the central
# (coimg_central).
coimg_central = coimg - comod_nocentral
coimgs.append(coimg)
comods.append(comod)
coresids.append(coresid)
comods_nocentral.append(comod_nocentral)
coimgs_central.append(coimg_central)
# Write out the final images with and without the central, making sure
# to apply the zeropoint to go from counts/s to AB nanomaggies.
# https://asd.gsfc.nasa.gov/archive/galex/FAQ/counts_background.html
for thisimg, imtype in zip((coimg, comod, comod_nocentral),
('image', 'model', 'model-nocentral')):
fitsfile = os.path.join(
output_dir, '{}-{}-{}.fits'.format(galaxy, imtype, niceband))
if verbose:
print('Writing {}'.format(fitsfile))
fitsio.write(fitsfile,
thisimg * 10**(-0.4 * (zp - 22.5)),
clobber=True)
# Build a color mosaic (but note that the images here are in units of
# background-subtracted counts/s).
#_galex_rgb = _galex_rgb_moustakas
#_galex_rgb = _galex_rgb_dstn
_galex_rgb = _galex_rgb_official
for imgs, imtype in zip(
(coimgs, comods, coresids, comods_nocentral, coimgs_central),
('image', 'model', 'resid', 'model-nocentral', 'image-central')):
rgb = _galex_rgb(imgs)
jpgfile = os.path.join(output_dir,
'{}-{}-FUVNUV.jpg'.format(galaxy, imtype))
if verbose:
print('Writing {}'.format(jpgfile))
imsave_jpeg(jpgfile, rgb, origin='lower')
return 1
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description='DECaLS simulations.')
parser.add_argument('-b',
'--brick',
type=str,
default='2428p117',
metavar='',
help='process this brick (required input)')
parser.add_argument('-o',
'--objtype',
type=str,
default='ELG',
metavar='',
help='object type (STAR, ELG, LRG, BGS)')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='toggle on verbose output')
args = parser.parse_args()
if args.brick is None:
parser.print_help()
sys.exit(1)
# Set the debugging level
if args.verbose:
lvl = logging.DEBUG
else:
lvl = logging.INFO
logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout)
log = logging.getLogger('__name__')
brickname = args.brick
objtype = args.objtype.upper()
lobjtype = objtype.lower()
log.info('Analyzing objtype {} on brick {}'.format(objtype, brickname))
if 'DECALS_SIM_DIR' in os.environ:
decals_sim_dir = os.getenv('DECALS_SIM_DIR')
else:
decals_sim_dir = '.'
# Plotting preferences
sns.set(style='white', font_scale=1.6, palette='dark') #,font='fantasy')
col = sns.color_palette('dark')
# Read the meta-catalog.
metafile = os.path.join(decals_sim_dir, brickname,
'metacat-' + brickname + '-' + lobjtype + '.fits')
log.info('Reading {}'.format(metafile))
meta = fits.getdata(metafile, 1)
# We need this for our histograms below
magbinsz = 0.2
rminmax = np.array(meta['rmag_range'][0], meta['rmag_range'][1])
nmagbin = long((rminmax[1] - rminmax[0]) / magbinsz)
# Work in chunks.
nchunk = meta['nchunk']
for ichunk in range(nchunk):
log.info('Working on chunk {:02d}/{:02d}'.format(ichunk + 1, nchunk))
chunksuffix = '{:02d}'.format(ichunk)
# Read the simulated object catalog
simcatfile = os.path.join(
decals_sim_dir, brickname, 'simcat-' + brickname + '-' + lobjtype +
'-' + chunksuffix + '.fits')
#log.info('Reading {}'.format(simcatfile))
simcat = fits.getdata(simcatfile, 1)
# Read and match to the Tractor catalog
tractorfile = os.path.join(
decals_sim_dir, brickname, 'tractor-' + brickname + '-' +
lobjtype + '-' + chunksuffix + '.fits')
#log.info('Reading {}'.format(tractorfile))
tractor = fits.getdata(tractorfile, 1)
m1, m2, d12 = match_radec(tractor['ra'], tractor['dec'], simcat['ra'],
simcat['dec'], 1.0 / 3600.0)
missing = np.delete(np.arange(len(simcat)), m2, axis=0)
good = np.where(
(np.abs(tractor['decam_flux'][m1, 2] / simcat['rflux'][m2] - 1) <
0.3) * 1)
# Flux residuals vs r-band magnitude
rmag = simcat['r'][m2]
gflux_sim = simcat['gflux'][m2]
rflux_sim = simcat['rflux'][m2]
zflux_sim = simcat['zflux'][m2]
gflux_tra = tractor['decam_flux'][m1, 1]
rflux_tra = tractor['decam_flux'][m1, 2]
zflux_tra = tractor['decam_flux'][m1, 4]
fig, ax = plt.subplots(3, sharex=True, figsize=(6, 8))
ax[0].scatter(rmag, gflux_sim / gflux_tra - 1, color=col[0], s=10)
ax[1].scatter(rmag, rflux_sim / rflux_tra - 1, color=col[1], s=10)
ax[2].scatter(rmag, zflux_sim / zflux_tra - 1, color=col[2], s=10)
[thisax.set_ylim(-0.7, 0.7) for thisax in ax]
[thisax.set_xlim(rminmax + [-0.1, 0.0]) for thisax in ax]
[thisax.axhline(y=0.0, lw=2, ls='solid', color='gray') for thisax in ax]
for ix, thisband in enumerate(['g', 'r', 'z']):
ax[ix].text(0.05,
0.05,
thisband,
horizontalalignment='left',
verticalalignment='bottom',
transform=ax[ix].transAxes,
fontsize=16)
ax[1].set_ylabel('Input Flux / Tractor Flux - 1')
ax[2].set_xlabel('Input r magnitude (AB mag)')
fig.subplots_adjust(left=0.18, hspace=0.1)
qafile = os.path.join(decals_sim_dir,
'qa-' + brickname + '-' + lobjtype + '-flux.png')
log.info('Writing {}'.format(qafile))
plt.savefig(qafile)
# Color residuals
gr_tra = -2.5 * np.log10(gflux_tra / rflux_tra)
rz_tra = -2.5 * np.log10(rflux_tra / zflux_tra)
gr_sim = -2.5 * np.log10(gflux_sim / rflux_sim)
rz_sim = -2.5 * np.log10(rflux_sim / zflux_sim)
fig, ax = plt.subplots(2, sharex=True, figsize=(6, 8))
ax[0].scatter(rmag, gr_tra - gr_sim, color=col[0], s=10)
ax[1].scatter(rmag, rz_tra - rz_sim, color=col[1], s=10)
[thisax.set_ylim(-0.7, 0.7) for thisax in ax]
[thisax.set_xlim(rminmax + [-0.1, 0.0]) for thisax in ax]
[thisax.axhline(y=0.0, lw=2, ls='solid', color='gray') for thisax in ax]
ax[0].set_ylabel('$\Delta$(g - r) (Tractor minus Input)')
ax[1].set_ylabel('$\Delta$(r - z) (Tractor minus Input)')
ax[1].set_xlabel('Input r magnitude (AB mag)')
fig.subplots_adjust(left=0.18, hspace=0.1)
qafile = os.path.join(decals_sim_dir,
'qa-' + brickname + '-' + lobjtype + '-color.png')
log.info('Writing {}'.format(qafile))
plt.savefig(qafile)
sys.exit(1)
# Get cutouts of the missing sources
imfile = os.path.join(
decals_sim_dir,
'qa-' + brickname + '-' + lobjtype + '-image-' + chunksuffix + '.jpg')
hw = 30 # half-width [pixels]
ncols = 5
nrows = 5
nthumb = ncols * nrows
dims = (ncols * hw * 2, nrows * hw * 2)
mosaic = Image.new('RGB', dims)
miss = missing[np.argsort(simcat['r'][missing])]
print(simcat['r'][miss])
xpos, ypos = np.meshgrid(np.arange(0, dims[0], hw * 2, dtype='int'),
np.arange(0, dims[1], hw * 2, dtype='int'))
im = Image.open(imfile)
sz = im.size
iobj = 0
for ic in range(ncols):
for ir in range(nrows):
mm = miss[iobj]
xx = int(simcat['X'][mm])
yy = int(sz[1] - simcat['Y'][mm])
crop = (xx - hw, yy - hw, xx + hw, yy + hw)
box = (xpos[ir, ic], ypos[ir, ic])
thumb = im.crop(crop)
mosaic.paste(thumb, box)
iobj = iobj + 1
# Add a border
draw = ImageDraw.Draw(mosaic)
for ic in range(ncols):
for ir in range(nrows):
draw.rectangle([(xpos[ir, ic], ypos[ir, ic]),
(xpos[ir, ic] + hw * 2, ypos[ir, ic] + hw * 2)])
qafile = os.path.join(decals_sim_dir,
'qa-' + brickname + '-' + lobjtype + '-missing.png')
log.info('Writing {}'.format(qafile))
mosaic.save(qafile)
# Modify the coadd image and residual files so the simulated sources
# are labeled.
rad = 15
imfile = os.path.join(
decals_sim_dir,
'qa-' + brickname + '-' + lobjtype + '-image-' + chunksuffix + '.jpg')
imfile = [imfile, imfile.replace('-image', '-resid')]
for ifile in imfile:
im = Image.open(ifile)
sz = im.size
draw = ImageDraw.Draw(im)
[
draw.ellipse((cat['X'] - rad, sz[1] - cat['Y'] - rad,
cat['X'] + rad, sz[1] - cat['Y'] + rad))
for cat in simcat
]
im.save(ifile)
# Fraction of matching sources
rmaghist, magbins = np.histogram(simcat['r'], bins=nmagbin, range=rminmax)
cmagbins = (magbins[:-1] + magbins[1:]) / 2.0
ymatch, binsmatch = np.histogram(simcat['r'][m2],
bins=nmagbin,
range=rminmax)
ymatchgood, binsgood = np.histogram(simcat['r'][m2[good]],
bins=nmagbin,
range=rminmax)
fig, ax = plt.subplots(1, figsize=(8, 6))
ax.step(cmagbins,
1.0 * ymatch / rmaghist,
lw=3,
alpha=0.5,
label='All objects')
ax.step(cmagbins,
1.0 * ymatchgood / rmaghist,
lw=3,
ls='dashed',
label='|$\Delta$m|<0.3')
ax.axhline(y=1.0, lw=2, ls='dashed', color='gray')
ax.set_xlabel('Input r magnitude (AB mag)')
ax.set_ylabel('Fraction of Matching ' + objtype + 's')
ax.set_ylim([0.0, 1.1])
ax.legend(loc='lower left')
fig.subplots_adjust(bottom=0.15)
qafile = os.path.join(decals_sim_dir,
'qa-' + brickname + '-' + lobjtype + '-frac.png')
log.info('Writing {}'.format(qafile))
plt.savefig(qafile)
# Distribution of object types
fig = plt.figure(figsize=(8, 6))
ax = fig.gca()
rmaghist, magbins = np.histogram(simcat['r'][m2],
bins=nmagbin,
range=rminmax)
cmagbins = (magbins[:-1] + magbins[1:]) / 2.0
tractortype = tractor['TYPE'][m1].strip()
for otype in ['PSF', 'EXP', 'DEV', 'COMP']:
these = np.where(tractortype == otype)[0]
if len(these) > 0:
yobj, binsobj = np.histogram(simcat['r'][m2[these]],
bins=nmagbin,
range=rminmax)
#plt.step(cmagbins,1.0*yobj,lw=3,alpha=0.5,label=otype)
plt.step(cmagbins,
1.0 * yobj / rmaghist,
lw=3,
alpha=0.5,
label=otype)
plt.axhline(y=1.0, lw=2, ls='dashed', color='gray')
plt.xlabel('Input r magnitude (AB mag)')
#plt.ylabel('Number of Objects')
plt.ylabel('Fraction of ' + objtype + 's classified')
plt.ylim([0.0, 1.1])
plt.legend(loc='center left', bbox_to_anchor=(0.08, 0.5))
fig.subplots_adjust(bottom=0.15)
qafile = os.path.join(decals_sim_dir,
'qa-' + brickname + '-' + lobjtype + '-type.png')
log.info('Writing {}'.format(qafile))
plt.savefig(qafile)
# Morphology plots
if objtype == 'ELGo':
fig = plt.figure(figsize=(8, 4))
plt.subplot(1, 3, 1)
plt.plot(rmag, deltam, 's', markersize=3)
plt.axhline(y=0.0, lw=2, ls='solid', color='gray')
plt.xlim(rminmax)
plt.xlabel('r (AB mag)')
plt.subplot(1, 3, 2)
plt.plot(simcat['R50_1'][m2], deltam, 's', markersize=3)
plt.axhline(y=0.0, lw=2, ls='solid', color='gray')
plt.xlabel('$r_{50}$ (arcsec)')
plt.subplot(1, 3, 3)
plt.plot(simcat['BA_1'][m2], deltam, 's', markersize=3)
plt.axhline(y=0.0, lw=2, ls='solid', color='gray')
plt.xlabel('b/a')
plt.xlim([0.2, 1.0])
fig.subplots_adjust(bottom=0.18)
qafile = os.path.join(
decals_sim_dir, 'qa-' + brickname + '-' + lobjtype + '-morph.png')
log.info('Writing {}'.format(qafile))
plt.savefig(qafile)
def mosaic():
'''
> cp ~/cosmo/staging/mosaicz/MZLS_CP/CP20180102/k4m_180103_040423_ooi_zd_v1.fits.fz /tmp
> funpack /tmp/k4m_180103_040423_ooi_zd_v1.fits.fz
> fitsgetext -i /tmp/k4m_180103_040423_ooi_zd_v1.fits -o mosaic-%02i.wcs -a -H
> cat mosaic-??.wcs > mosaic.wcs
> for ((i=1; i<=4; i++)); do modhead mosaic.wcs+$i NAXIS2; modhead mosaic.wcs+$i NAXIS2 0; done
NAXIS2 = 4079 / Axis length
NAXIS2 = 4079 / Axis length
NAXIS2 = 4079 / Axis length
NAXIS2 = 4061 / Axis length
'''
plt.figure(figsize=(4,3))
plt.subplots_adjust(left=0.15, right=0.99, top=0.99, bottom=0.15)
T = fits_table('obstatus/mosaic-tiles_obstatus.fits')
print(len(T), 'tiles')
T.rename('pass', 'passnum')
T.cut(T.passnum <= 3)
print(len(T), 'tiles with passnum <= 3')
ra,dec = 180.216, 40.191
#ra,dec = 180., 40.
I,J,d = match_radec(T.ra, T.dec, ra, dec, 2.)
print(len(I), 'tiles near', ra,dec)
T.cut(I)
T.dist = d
print('dists:', d)
print('Passes:', T.passnum)
F = fitsio.FITS(os.path.join(os.path.dirname(__file__), 'mosaic.wcs'))
wcs = []
heights = [ 4079, 4079, 4079, 4061 ]
for i in range(1, len(F)):
hdr = F[i].read_header()
W = hdr['NAXIS1']
wcs.append(wcs_pv2sip_hdr(hdr, H=heights[i-1], W=W))
print('WCS:', wcs[-1])
# Rendering canvas
W,H = 2200, 2200
pixsc = 4./3600.
targetwcs = Tan(ra, dec, W/2.+0.5, H/2.+0.5, -pixsc, 0., 0., pixsc,
float(W), float(H))
II = np.lexsort((T.dist, T.passnum))
print('First tile center:', T.ra[II[0]], T.dec[II[0]])
# This is for making the (vector) PDF format tiling images.
for maxit in [0, 8, 36, 37, 44, 73, 74, 82, 112]:
plot = Plotstuff(outformat='pdf', ra=ra, dec=dec, width=W*pixsc,
size=(W,H), outfn='tile-mosaic-%02i.pdf' % maxit)
plot.color = 'white'
plot.alpha = 1.
plot.plot('fill')
out = plot.outline
out.fill = True
out.stepsize = 1024.
plot.color = 'black'
plot.alpha = 0.4
plot.apply_settings()
for it,t in enumerate(T[II]):
print('Tile', it, 'pass', t.passnum)
for w in wcs:
w.set_crval((t.ra, t.dec))
out.wcs = anwcs_new_sip(w)
plot.plot('outline')
if it == maxit:
print('Writing', it)
plot.write()
break
# # And this is for PNG-format tiling images and histograms.
cov = np.zeros((H,W), np.uint8)
for it,t in enumerate(T[II]):
print('Tile', it, 'pass', t.passnum)
for w in wcs:
w.set_crval((t.ra, t.dec))
try:
Yo,Xo,Yi,Xi,nil = resample_with_wcs(targetwcs, w)
except:
continue
cov[Yo,Xo] += 1
if it in [0, 8, 36, 37, 44, 73, 74, 82, 112]:
mx = { 1: 2, 2: 4, 3: 6 }[t.passnum]
# plt.clf()
# plt.imshow(cov, interpolation='nearest', origin='lower', vmin=0, vmax=mx)
# plt.colorbar()
# plt.savefig('tile-%02i.png' % it)
plt.imsave('tile-mosaic-%02i.png' % it, cov, origin='lower', vmin=0, vmax=mx, cmap=antigray)
if it in [36, 73, 112]:
from collections import Counter
print('Coverage counts:', Counter(cov.ravel()).most_common())
bins = -0.5 + np.arange(8)
plt.clf()
n,b,p = plt.hist(cov.ravel(), bins=bins, normed=True)
#plt.hist(cov.ravel(), bins=bins, normed=True, cumulative=True, histtype='step')
# Cumulative histogram from the right...
xx,yy = [],[]
for blo,bhi,ni in reversed(list(zip(bins, bins[1:], n))):
nc = float(np.sum(cov.ravel() > blo)) / len(cov.ravel())
yy.extend([nc,nc])
xx.extend([bhi,blo])
if ni > 0:
if nc != ni:
if nc > ni+0.03:
# If there's room, label the histogram bin above, else below
plt.text((blo+bhi)/2., ni, '%.1f \%%' % (100.*ni), ha='center', va='bottom', color='k')
else:
plt.text((blo+bhi)/2., ni-0.01, '%.1f \%%' % (100.*ni), ha='center', va='top', color='k')
plt.text((blo+bhi)/2., nc, '%.1f \%%' % (100.*nc), ha='center', va='bottom', color='k')
plt.plot(xx, yy, 'k-')
plt.xlim(bins.min(), bins.max())
plt.ylim(0., 1.1)
plt.xlabel('Number of exposures')
plt.ylabel('Fraction of sky')
plt.savefig('hist-mosaic-%02i.pdf' % it)
names=True,
dtype=None)
d25 = table['d25']
name_cal, ra_cal, dec_cal, a_cal, b_cal, pa_cal, ty_cal = read_file(
'sample_tfc_all.glga', n_skip=4, seprator=' ')
#print name_cal
#print ra_cal
#print dec_cal
# Python equivalent of matchlength
radius_in_deg = 1. / 60. # 1 arcmin match
(m1, m2, d) = spherematch.match_radec(ra_cal,
dec_cal,
ra,
dec,
radius_in_deg,
notself=False,
nearest=True)
print len(m1)
#for m in range(len(m1)):
#i = m1[m]
#j = m2[m]
#if d[m]*3600>=30.:
#print name_cal[i], ra_cal[i], dec_cal[i], pgc[j], ra[j], dec[j], d[m]*3600
db_dir = "/home/ehsan/db_esn/data/"
morph = fits_table(sefn, hdu=2)
wcs = Sip(im.wcsfn)
if len(sdss) == 0:
print 'EMPTY:', im.sdssfn
continue
if len(morph) == 0:
print 'EMPTY:', im.morphfn
continue
print len(sdss), 'SDSS sources from', im.sdssfn
print len(morph), 'SE sources from', sefn
morph.ra, morph.dec = wcs.pixelxy2radec(morph.x_image,
morph.y_image)
I, J, d = match_radec(morph.ra, morph.dec, sdss.ra, sdss.dec,
0.5 / 3600.)
corr = sdss[J]
corr.add_columns_from(morph[I])
chipnames.append(im.extname)
#corr = fits_table(im.corrfn)
corrs.append(corr)
print im, ':', len(corr), 'correspondences'
for col, cut in ([('flux_auto', None)] + [('flux_aper', i)
for i in range(3)] +
[('flux_psf', None), ('flux_model', None)]):
dmags = []
smags = []
for corr in corrs:
def map_decals_wl(req, ver, zoom, x, y):
tag = 'decals-wl'
ignoreCached = False
filename = None
forcecache = False
from decals import settings
savecache = settings.SAVE_CACHE
zoom = int(zoom)
zoomscale = 2.**zoom
x = int(x)
y = int(y)
if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale:
raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom,x,y))
ver = int(ver)
if not ver in tileversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
basedir = settings.DATA_DIR
tilefn = os.path.join(basedir, 'tiles', tag,
'%i/%i/%i/%i.jpg' % (ver, zoom, x, y))
if os.path.exists(tilefn) and not ignoreCached:
print('Cached:', tilefn)
return send_file(tilefn, 'image/jpeg', expires=oneyear,
modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'),
filename=filename)
else:
print('Tile image does not exist:', tilefn)
from astrometry.util.resample import resample_with_wcs, OverlapError
from astrometry.util.util import Tan
from astrometry.libkd.spherematch import match_radec
from astrometry.util.fits import fits_table
from astrometry.util.starutil_numpy import degrees_between
import numpy as np
import fitsio
try:
wcs, W, H, zoomscale, zoom,x,y = get_tile_wcs(zoom, x, y)
except RuntimeError as e:
return HttpResponse(e.strerror)
mydir = os.path.join(basedir, 'coadd', 'weak-lensing')
rlo,d = wcs.pixelxy2radec(W, H/2)[-2:]
rhi,d = wcs.pixelxy2radec(1, H/2)[-2:]
r,d1 = wcs.pixelxy2radec(W/2, 1)[-2:]
r,d2 = wcs.pixelxy2radec(W/2, H)[-2:]
#dlo = min(d1, d2)
#dhi = max(d1, d2)
r,d = wcs.pixelxy2radec(W/2, H/2)[-2:]
rad = degrees_between(r, d, rlo, d1)
fn = os.path.join(mydir, 'index.fits')
if not os.path.exists(fn):
#
ii,rr,dd = [],[],[]
for i in range(1, 52852+1):
imgfn = os.path.join(mydir, 'map%i.fits' % i)
hdr = fitsio.read_header(imgfn)
r = hdr['CRVAL1']
d = hdr['CRVAL2']
ii.append(i)
rr.append(r)
dd.append(d)
T = fits_table()
T.ra = np.array(rr)
T.dec = np.array(dd)
T.i = np.array(ii)
T.writeto(fn)
T = fits_table(fn)
I,J,d = match_radec(T.ra, T.dec, r, d, rad + 0.2)
T.cut(I)
print(len(T), 'weak-lensing maps in range')
if len(I) == 0:
from django.http import HttpResponseRedirect
if forcecache:
# create symlink to blank.jpg!
trymakedirs(tilefn)
src = os.path.join(settings.STATIC_ROOT, 'blank.jpg')
if os.path.exists(tilefn):
os.unlink(tilefn)
os.symlink(src, tilefn)
print('Symlinked', tilefn, '->', src)
return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg')
r,d = wcs.pixelxy2radec([1,1,1,W/2,W,W,W,W/2],
[1,H/2,H,H,H,H/2,1,1])[-2:]
foundany = False
rimg = np.zeros((H,W), np.float32)
rn = np.zeros((H,W), np.uint8)
for tilei in T.i:
fn = os.path.join(mydir, 'map%i.fits' % tilei)
try:
bwcs = _read_tan_wcs(fn, 0)
except:
print('Failed to read WCS:', fn)
savecache = False
import traceback
import sys
traceback.print_exc(None, sys.stdout)
continue
foundany = True
print('Reading', fn)
ok,xx,yy = bwcs.radec2pixelxy(r, d)
xx = xx.astype(np.int)
yy = yy.astype(np.int)
imW,imH = int(bwcs.get_width()), int(bwcs.get_height())
M = 10
xlo = np.clip(xx.min() - M, 0, imW)
xhi = np.clip(xx.max() + M, 0, imW)
ylo = np.clip(yy.min() - M, 0, imH)
yhi = np.clip(yy.max() + M, 0, imH)
if xlo >= xhi or ylo >= yhi:
continue
subwcs = bwcs.get_subimage(xlo, ylo, xhi-xlo, yhi-ylo)
slc = slice(ylo,yhi), slice(xlo,xhi)
try:
f = fitsio.FITS(fn)[0]
img = f[slc]
del f
except:
print('Failed to read image and WCS:', fn)
savecache = False
import traceback
import sys
traceback.print_exc(None, sys.stdout)
continue
try:
Yo,Xo,Yi,Xi,nil = resample_with_wcs(wcs, subwcs, [], 3)
except OverlapError:
print('Resampling exception')
continue
rimg[Yo,Xo] += img[Yi,Xi]
rn [Yo,Xo] += 1
rimg /= np.maximum(rn, 1)
if forcecache:
savecache = True
if savecache:
trymakedirs(tilefn)
else:
import tempfile
f,tilefn = tempfile.mkstemp(suffix='.jpg')
os.close(f)
import pylab as plt
# S/N
#lo,hi = 1.5, 5.0
lo,hi = 0, 5.0
rgb = plt.cm.hot((rimg - lo) / (hi - lo))
plt.imsave(tilefn, rgb)
print('Wrote', tilefn)
return send_file(tilefn, 'image/jpeg', unlink=(not savecache),
filename=filename)
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(
description=
"This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)."
)
parser.add_argument('--calibs',
action='store_true',
help='Output CCDs that need to be calibrated.')
parser.add_argument('--nper',
type=int,
default=None,
help='Batch N calibs per line')
parser.add_argument('--forced',
action='store_true',
help='Output forced-photometry commands')
parser.add_argument('--lsb',
action='store_true',
help='Output Low-Surface-Brightness commands')
parser.add_argument('--touching',
action='store_true',
help='Cut to only CCDs touching selected bricks')
parser.add_argument('--check',
action='store_true',
help='Check which calibrations actually need to run.')
parser.add_argument('--check-coadd',
action='store_true',
help='Check which caoadds actually need to run.')
parser.add_argument('--out',
help='Output filename for calibs, default %(default)s',
default='jobs')
parser.add_argument('--command',
action='store_true',
help='Write out full command-line to run calib')
parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
parser.add_argument('--mindec', type=float, help='Minimum Dec to run')
parser.add_argument('--region', help='Region to select')
parser.add_argument('--bricks', help='Set bricks.fits file to load')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--delete-sky',
action='store_true',
help='Delete any existing sky calibration files')
parser.add_argument('--delete-pvastrom',
action='store_true',
help='Delete any existing PV WCS calibration files')
parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')
opt = parser.parse_args()
decals = Decals()
if opt.bricks is not None:
B = fits_table(opt.bricks)
log('Read', len(B), 'from', opt.bricks)
else:
B = decals.get_bricks()
if opt.ccds is not None:
T = fits_table(opt.ccds)
log('Read', len(T), 'from', opt.ccds)
else:
T = decals.get_ccds()
log(len(T), 'CCDs')
T.index = np.arange(len(T))
# I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
# plt.clf()
# plt.hist(counts, counts.max()+1)
# plt.savefig('bricks.png')
# B.cut(I[counts >= 9])
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# #plt.scatter(B.ra[I], B.dec[I], c=counts)
# plt.savefig('bricks2.png')
# DES Stripe82
#rlo,rhi = 350.,360.
# rlo,rhi = 300., 10.
# dlo,dhi = -6., 4.
# TINY bit
#rlo,rhi = 350.,351.1
#dlo,dhi = 0., 1.1
# EDR+
# 860 bricks
# ~10,000 CCDs
#rlo,rhi = 239,246
#dlo,dhi = 5, 13
# DR1
#rlo,rhi = 0, 360
# part 1
#dlo,dhi = 25, 40
# part 2
#dlo,dhi = 20,25
# part 3
#dlo,dhi = 15,20
# part 4
#dlo,dhi = 10,15
# part 5
#dlo,dhi = 5,10
# the rest
#dlo,dhi = -11, 5
#dlo,dhi = 15,25.5
dlo, dhi = -15, 40
rlo, rhi = 0, 360
# Arjun says 3x3 coverage area is roughly
# RA=240-252 DEC=6-12 (but not completely rectangular)
# COSMOS
#rlo,rhi = 148.9, 151.2
#dlo,dhi = 0.9, 3.5
# A nice well-behaved region (EDR2/3)
# rlo,rhi = 243.6, 244.6
# dlo,dhi = 8.1, 8.6
# 56 bricks, ~725 CCDs
#B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
# 240 bricks, ~3000 CCDs
#B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
# 535 bricks, ~7000 CCDs
#B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))
if opt.region in ['test1', 'test2', 'test3', 'test4']:
nm = dict(
test1='2446p115', # weird stuff around bright star
test2='1183p292', # faint sources around bright galaxy
test3='3503p005', # DES
test4='1163p277', # Pollux
)[opt.region]
B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
log('Cut to', len(B), 'bricks')
log(B.ra, B.dec)
dlo, dhi = -90, 90
rlo, rhi = 0, 360
elif opt.region == 'badsky':
# A handful of exposures in DR2 with inconsistent sky estimates.
#C = decals.get_ccds()
#log(len(C), 'CCDs')
#C.cut((C.expnum >= 257400) * (C.expnum < 257500))
T.cut(
np.array([
e in [
257460, 257461, 257462, 257463, 257464, 257465, 257466,
257467, 257469, 257472, 257483, 257496
] for e in T.expnum
]))
log(len(T), 'CCDs with bad sky')
# CCD radius
radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600.
# Brick radius
radius += np.hypot(0.25, 0.25) / 2.
I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05)
keep = np.zeros(len(B), bool)
keep[I] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs with bad sky')
elif opt.region == 'badsky2':
# UGH, missed this one in original 'badsky' definition.
T.cut(T.expnum == 257466)
log(len(T), 'CCDs with bad sky')
# CCD radius
radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600.
# Brick radius
radius += np.hypot(0.25, 0.25) / 2.
I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05)
keep = np.zeros(len(B), bool)
keep[I] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs with bad sky')
elif opt.region == 'edr':
# EDR:
# 535 bricks, ~7000 CCDs
rlo, rhi = 240, 245
dlo, dhi = 5, 12
elif opt.region == 'edr-south':
rlo, rhi = 240, 245
dlo, dhi = 5, 10
elif opt.region == 'cosmos1':
# 16 bricks in the core of the COSMOS field.
rlo, rhi = 149.75, 150.75
dlo, dhi = 1.6, 2.6
elif opt.region == 'pristine':
# Stream?
rlo, rhi = 240, 250
dlo, dhi = 10, 15
elif opt.region == 'des':
dlo, dhi = -6., 4.
rlo, rhi = 317., 7.
T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
log('Cut to', len(T), 'CCDs with "CPDES82" in filename')
elif opt.region == 'subdes':
rlo, rhi = 320., 360.
dlo, dhi = -1.25, 1.25
elif opt.region == 'northwest':
rlo, rhi = 240, 360
dlo, dhi = 20, 40
elif opt.region == 'north':
rlo, rhi = 120, 240
dlo, dhi = 20, 40
elif opt.region == 'northeast':
rlo, rhi = 0, 120
dlo, dhi = 20, 40
elif opt.region == 'southwest':
rlo, rhi = 240, 360
dlo, dhi = -20, 0
elif opt.region == 'south':
rlo, rhi = 120, 240
dlo, dhi = -20, 0
elif opt.region == 'southeast':
rlo, rhi = 0, 120
dlo, dhi = -20, 0
elif opt.region == 'midwest':
rlo, rhi = 240, 360
dlo, dhi = 0, 20
elif opt.region == 'middle':
rlo, rhi = 120, 240
dlo, dhi = 0, 20
elif opt.region == 'mideast':
rlo, rhi = 0, 120
dlo, dhi = 0, 20
elif opt.region == 'grz':
# Bricks with grz coverage.
# Be sure to use --bricks decals-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
log('Cut to', len(B), 'bricks with grz coverage')
elif opt.region == 'nogrz':
# Bricks without grz coverage.
# Be sure to use --bricks decals-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
log('Cut to', len(B), 'bricks withOUT grz coverage')
elif opt.region == 'deep2':
rlo, rhi = 250, 260
dlo, dhi = 30, 35
elif opt.region == 'virgo':
rlo, rhi = 185, 190
dlo, dhi = 10, 15
elif opt.region == 'virgo2':
rlo, rhi = 182, 192
dlo, dhi = 8, 18
elif opt.region == 'lsb':
rlo, rhi = 147.2, 147.8
dlo, dhi = -0.4, 0.4
if opt.mindec is not None:
dlo = opt.mindec
if opt.maxdec is not None:
dhi = opt.maxdec
if rlo < rhi:
B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi))
else: # RA wrap
B.cut(
np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) *
(B.dec <= dhi))
log(len(B), 'bricks in range')
I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, 0.25)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs')
if opt.touching:
keep = np.zeros(len(T), bool)
for j in J:
keep[j] = True
T.cut(keep)
log('Cut to', len(T), 'CCDs near bricks')
# Aside -- how many near DR1=1 CCDs?
if False:
T2 = D.get_ccds()
log(len(T2), 'CCDs')
T2.cut(T2.dr1 == 1)
log(len(T2), 'CCDs marked DR1=1')
log(len(B), 'bricks in range')
I, J, d = match_radec(B.ra, B.dec, T2.ra, T2.dec, 0.25)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B2 = B[keep]
log('Total of', len(B2), 'bricks near CCDs with DR1=1')
for band in 'grz':
Tb = T2[T2.filter == band]
log(len(Tb), 'in filter', band)
I, J, d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, 0.25)
good = np.zeros(len(B2), np.uint8)
for i in I:
good[i] = 1
B2.set('has_' + band, good)
B2.writeto('decals-bricks-in-dr1.fits')
sys.exit(0)
# sort by dec decreasing
B.cut(np.argsort(-B.dec))
for b in B:
if opt.check:
fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3],
b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
if opt.check_coadd:
fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (
b.brickname[:3], b.brickname, b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
print(b.brickname)
if not (opt.calibs or opt.forced or opt.lsb):
sys.exit(0)
bands = 'grz'
log('Filters:', np.unique(T.filter))
T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
log('Cut to', len(T), 'CCDs in filters', bands)
if opt.touching:
allI = set()
for b in B:
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickid,
'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
if len(I) == 0:
continue
allI.update(I)
allI = list(allI)
allI.sort()
else:
allI = np.arange(len(T))
if opt.write_ccds:
T[allI].writeto(opt.write_ccds)
log('Wrote', opt.write_ccds)
## Be careful here -- T has been cut; we want to write out T.index.
## 'allI' contains indices into T.
if opt.forced:
log('Writing forced-photometry commands to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
for j, i in enumerate(allI):
expstr = '%08i' % T.expnum[i]
#outdir = os.path.join('forced', expstr[:5], expstr)
#trymakedirs(outdir)
outfn = os.path.join(
'forced', expstr[:5], expstr,
'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i]))
imgfn = os.path.join(decals.decals_dir, 'images',
T.image_filename[i].strip())
if (not os.path.exists(imgfn) and imgfn.endswith('.fz')
and os.path.exists(imgfn[:-3])):
imgfn = imgfn[:-3]
f.write('python legacypipe/forced-photom-decam.py %s %i DR1 %s\n' %
(imgfn, T.cpimage_hdu[i], outfn))
f.close()
log('Wrote', opt.out)
sys.exit(0)
if opt.lsb:
log('Writing LSB commands to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
for j, i in enumerate(allI):
exp = T.expnum[i]
ext = T.ccdname[i].strip()
outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
f.write(
'python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n'
% (exp, ext, outfn, exp, ext))
f.close()
log('Wrote', opt.out)
sys.exit(0)
log('Writing calibs to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
batch = []
def write_batch(f, batch, cmd):
if opt.command:
s = '; '.join(batch)
else:
s = ' '.join(batch)
f.write(s + '\n')
for j, i in enumerate(allI):
if opt.delete_sky or opt.delete_pvastrom:
log(j + 1, 'of', len(allI))
im = decals.get_image_object(T[i])
if opt.delete_sky and os.path.exists(im.skyfn):
log(' deleting:', im.skyfn)
os.unlink(im.skyfn)
if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
log(' deleting:', im.pvwcsfn)
os.unlink(im.pvwcsfn)
if opt.check:
log(j + 1, 'of', len(allI))
im = decals.get_image_object(T[i])
if not im.run_calibs(im, just_check=True):
log('Calibs for', im.expnum, im.ccdname, im.calname,
'already done')
continue
if opt.command:
s = ('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
(T.expnum[i], T.ccdname[i]))
else:
s = '%i' % T.index[i]
if not opt.nper:
f.write(s + '\n')
else:
batch.append(s)
if len(batch) >= opt.nper:
write_batch(f, batch, opt.command)
batch = []
if opt.check:
f.flush()
if len(batch):
write_batch(f, batch, opt.command)
f.close()
log('Wrote', opt.out)
return 0
def get_reference_sources(survey,
targetwcs,
pixscale,
bands,
tycho_stars=True,
gaia_stars=True,
large_galaxies=True,
star_clusters=True,
clean_columns=True,
plots=False,
ps=None,
gaia_margin=None,
galaxy_margin=None):
# If bands = None, does not create sources.
H, W = targetwcs.shape
H, W = int(H), int(W)
# How big of a margin to search for bright stars and star clusters --
# this should be based on the maximum radius they are considered to
# affect. In degrees.
if gaia_margin is not None:
ref_margin = gaia_margin
else:
ref_margin = mask_radius_for_mag(0.)
mpix = int(np.ceil(ref_margin * 3600. / pixscale))
marginwcs = targetwcs.get_subimage(-mpix, -mpix, W + 2 * mpix,
H + 2 * mpix)
# Table of reference-source properties, including a field 'sources',
# with tractor source objects.
refs = []
# Tycho-2 stars
tycho = []
if tycho_stars:
tycho = read_tycho2(survey, marginwcs, bands)
if tycho and len(tycho):
refs.append(tycho)
# Add Gaia stars
gaia = None
if gaia_stars:
from astrometry.libkd.spherematch import match_radec
gaia = read_gaia(marginwcs, bands)
if gaia is not None:
# Handle sources that appear in both Gaia and Tycho-2 by
# dropping the entry from Tycho-2.
if len(gaia) and len(tycho):
# Before matching, apply proper motions to bring them to
# the same epoch. We want to use the more-accurate Gaia
# proper motions, so rewind Gaia positions to the Tycho
# epoch. (Note that in read_tycho2, we massaged the
# epochs)
cosdec = np.cos(np.deg2rad(gaia.dec))
# First do a coarse matching with the approximate epoch:
dt = 1991.5 - gaia.ref_epoch
gra = gaia.ra + dt * gaia.pmra / (3600. * 1000.) / cosdec
gdec = gaia.dec + dt * gaia.pmdec / (3600. * 1000.)
# Max Tycho-2 PM is 10"/yr, max |epoch_ra,epoch_dec - mean| = 0.5
I, J, _ = match_radec(tycho.ra,
tycho.dec,
gra,
gdec,
10. / 3600.,
nearest=True)
debug('Initially matched', len(I),
'Tycho-2 stars to Gaia stars (10").')
if plots:
import pylab as plt
plt.clf()
plt.plot(gra, gdec, 'bo', label='Gaia (1991.5)')
plt.plot(gaia.ra, gaia.dec, 'gx', label='Gaia (2015.5)')
plt.plot([gaia.ra, gra], [gaia.dec, gdec], 'k-')
plt.plot([tycho.ra[I], gra[J]], [tycho.dec[I], gdec[J]], 'r-')
plt.plot(tycho.ra, tycho.dec, 'rx', label='Tycho-2')
plt.plot(tycho.ra[I],
tycho.dec[I],
'o',
mec='r',
ms=8,
mfc='none',
label='Tycho-2 matched')
plt.legend()
r0, r1, d0, d1 = targetwcs.radec_bounds()
plt.axis([r0, r1, d0, d1])
plt.title('Initial (10") matching')
ps.savefig()
dt = tycho.ref_epoch[I] - gaia.ref_epoch[J]
cosdec = np.cos(np.deg2rad(gaia.dec[J]))
gra = gaia.ra[J] + dt * gaia.pmra[J] / (3600. * 1000.) / cosdec
gdec = gaia.dec[J] + dt * gaia.pmdec[J] / (3600. * 1000.)
dists = np.hypot((gra - tycho.ra[I]) * cosdec, gdec - tycho.dec[I])
K = np.flatnonzero(dists <= 1. / 3600.)
if len(K) < len(I):
debug('Unmatched Tycho-2 - Gaia stars: dists',
dists[dists > 1. / 3600.] * 3600.)
I = I[K]
J = J[K]
debug('Matched', len(I), 'Tycho-2 stars to Gaia stars.')
if len(I):
keep = np.ones(len(tycho), bool)
keep[I] = False
tycho.cut(keep)
gaia.isbright[J] = True
gaia.istycho[J] = True
if gaia is not None and len(gaia) > 0:
refs.append(gaia)
# Read the catalog of star (open and globular) clusters and add them to the
# set of reference stars (with the isbright bit set).
if star_clusters:
clusters = read_star_clusters(marginwcs)
if clusters is not None:
debug('Found', len(clusters), 'star clusters nearby')
refs.append(clusters)
# Read large galaxies nearby.
if large_galaxies:
kw = {}
if galaxy_margin is not None:
kw.update(max_radius=galaxy_margin +
np.hypot(H, W) / 2. * pixscale / 3600)
galaxies = read_large_galaxies(survey,
targetwcs,
bands,
clean_columns=clean_columns,
**kw)
if galaxies is not None:
# Resolve possible Gaia-large-galaxy duplicates
if gaia and len(gaia):
I, J, _ = match_radec(galaxies.ra,
galaxies.dec,
gaia.ra,
gaia.dec,
2. / 3600.,
nearest=True)
info('Matched', len(I), 'large galaxies to Gaia stars.')
if len(I):
gaia.donotfit[J] = True
# Resolve possible Tycho2-large-galaxy duplicates (with larger radius)
if tycho and len(tycho):
I, J, _ = match_radec(galaxies.ra,
galaxies.dec,
tycho.ra,
tycho.dec,
5. / 3600.,
nearest=True)
info('Matched', len(I), 'large galaxies to Tycho-2 stars.')
if len(I):
tycho.donotfit[J] = True
refs.append(galaxies)
if len(refs):
refs = merge_tables([r for r in refs if r is not None],
columns='fillzero')
if len(refs) == 0:
return None, None
# these x,y are in the margin-padded WCS; not useful.
# See ibx,iby computed below instead.
for c in ['x', 'y']:
if c in refs.get_columns():
refs.delete_column(c)
# radius / radius_pix are used to set the MASKBITS shapes;
# keep_radius determines which sources are kept (because we subtract
# stellar halos out to N x their radii)
refs.radius_pix = np.ceil(refs.radius * 3600. / pixscale).astype(int)
debug('Increasing radius for', np.sum(refs.keep_radius > refs.radius),
'ref sources based on keep_radius')
keeprad = np.maximum(refs.keep_radius, refs.radius)
# keeprad to pix
keeprad = np.ceil(keeprad * 3600. / pixscale).astype(int)
_, xx, yy = targetwcs.radec2pixelxy(refs.ra, refs.dec)
# ibx = integer brick coords
refs.ibx = np.round(xx - 1.).astype(np.int32)
refs.iby = np.round(yy - 1.).astype(np.int32)
# cut ones whose position + radius are outside the brick bounds.
refs.cut((xx > -keeprad) * (xx < W + keeprad) * (yy > -keeprad) *
(yy < H + keeprad))
# mark ones that are actually inside the brick area.
refs.in_bounds = ((refs.ibx >= 0) * (refs.ibx < W) * (refs.iby >= 0) *
(refs.iby < H))
# ensure bool columns
for col in [
'isbright', 'ismedium', 'islargegalaxy', 'iscluster', 'isgaia',
'istycho', 'donotfit', 'freezeparams'
]:
if not col in refs.get_columns():
refs.set(col, np.zeros(len(refs), bool))
# Copy flags from the 'refs' table to the source objects themselves.
sources = refs.sources
refs.delete_column('sources')
for i, (donotfit,
freeze) in enumerate(zip(refs.donotfit, refs.freezeparams)):
if donotfit:
sources[i] = None
if sources[i] is None:
continue
sources[i].is_reference_source = True
if freeze:
sources[i].freezeparams = True
return refs, sources
def real_one_tile((args)):
#(itile,tile,udecs,P3,T,tilewcs,exps,bad_expids,tileid_to_depth) = args
(itile,tile) = args
print()
print('Tile', itile+1, ':', tile.tileid, 'at', tile.ra, tile.dec)
i = np.nonzero(tile.dec == udecs)[0][0]
if i == 0 or i == len(udecs)-1:
print('Endpoint Dec; skipping for now')
return None
print(' Decs:', udecs[i-1], tile.dec, udecs[i+1])
declo = (udecs[i-1] + tile.dec) / 2.
dechi = (udecs[i+1] + tile.dec) / 2.
row = P3[P3.dec == tile.dec]
print(' ', len(row), 'tiles in this Dec row')
ras = np.sort(row.ra)
i = np.nonzero(tile.ra == ras)[0][0]
if i == 0 or i == len(ras)-1:
print(' Endpoint RA; skipping for now')
return None
print(' RAs:', ras[i-1], tile.ra, ras[i+1])
ralo = (ras[i-1] + tile.ra) / 2.
rahi = (ras[i+1] + tile.ra) / 2.
pixscale = 2.
#pixscale = 0.262
H = int(np.ceil((dechi - declo) / (pixscale/3600.)))
W = int(np.ceil((rahi - ralo) * np.cos(np.deg2rad(tile.dec)) / (pixscale/3600.)))
print(' Dec height', dechi-declo, 'RA width', rahi-ralo, '-> pix', W, 'x', H)
cd = pixscale/3600.
thiswcs = Tan(tile.ra, tile.dec, (W+1)/2., (H+1)/2.,
-cd, 0., 0., cd, float(W), float(H))
# Find surrounding tiles
radius = np.hypot(W, H) * pixscale / 3600.
I,J,d = match_radec(T.ra, T.dec, tile.ra, tile.dec, radius)
print(' ', len(I), 'tiles with measured depths nearby')
if len(I) == 0:
return None
# Now we need to take a tile boresight position and map it to
# boxes in RA,Dec of the good portions of the CCDs.
depth = np.zeros((H,W), np.float32)
nexp = np.zeros((H,W), np.uint8)
matched_exposures = set()
#print(' tileids', T.tileid[I])
#print(' expnums', T.z_expnum[I])
for ii in I:
if T.z_expnum[ii] in bad_expids:
print(' skipping bad exp num', T.z_expnum[ii])
continue
# Get depth from CCDs file, if available.
zdepth = tileid_to_depth.get(T.tileid[ii], 0.)
if zdepth == 0.:
zdepth = T.z_depth[ii]
matched_exposures.add(T.z_expnum[ii])
for twcs in tilewcs:
twcs.set_crval((T.ra[ii], T.dec[ii]))
try:
Yo,Xo,Yi,Xi,rims = resample_with_wcs(thiswcs, twcs)
except OverlapError:
continue
dflux = 10.**((zdepth - 22.5)/-2.5)
div = 1./dflux**2
depth[Yo,Xo] += div
nexp[Yo,Xo] += 1
# Now also look for entries in the CCDs (exposures) table not previously found.
I,J,d = match_radec(exps.ra_bore, exps.dec_bore, tile.ra, tile.dec, radius)
print(' ', len(I), 'exposures from CCDs file nearby')
if len(I):
I = np.array([i for i,expnum,gd in zip(I, exps.expnum[I], exps.galdepth[I])
if (not expnum in matched_exposures) and gd > 0])
print(' ', len(I), 'exposures that were not in tile file')
# Drop exposures from this pass, except for previous exposures of this tile!
# if len(I):
# I = I[np.logical_or(exps.tilepass[I] != 3,
# exps.tileid[I] == tile.tileid)]
# print(' ', len(I), 'exposures not in pass 3')
# if len(I):
# print(' objects:', [o.strip() for o in exps.object[I]])
# print(' tileids:', exps.tileid[I])
# print(' expnums:', exps.expnum[I])
# print(' passes:', exps.tilepass[I])
for ii in I:
if exps.expnum[ii] in bad_expids:
print(' skipping bad exp num', exps.expnum[ii])
continue
zdepth = exps.galdepth[ii]
for twcs in tilewcs:
twcs.set_crval((exps.ra_bore[ii], exps.dec_bore[ii]))
try:
Yo,Xo,Yi,Xi,rims = resample_with_wcs(thiswcs, twcs)
except OverlapError:
continue
dflux = 10.**((zdepth - 22.5)/-2.5)
div = 1./dflux**2
depth[Yo,Xo] += div
nexp[Yo,Xo] += 1
# Convert depth map from depth-iv back to mag.
# flux
with np.errstate(divide='ignore'):
dflux = np.sqrt(1./depth)
dflux[depth == 0] = 0.
depth = -2.5 * (np.log10(dflux) - 9.)
depth[dflux == 0] = 0.
if depth.max() == 0:
print(' Actually no overlap')
return None
# Extinction correction for this tile...
ext_z = 1.211 * tile.ebv_med
print(' Applying extinction correction', ext_z, 'mag')
depth[depth != 0] -= ext_z
#pcts = [0,10,20,30,40,50,60,70,80,90,100]
pcts = np.arange(0, 101)
depths = np.percentile(depth, pcts)
target = 22.5
req_pcts = [0, 2, 2, 5, 5, 10, 10, 100]
req_depths = [0, 0, target-0.6, target-0.6,
target-0.3, target-0.3, target, target]
print(' Depths at 2, 5, and 10th percentile vs target:',
'%.2f' % (depths[2] - (target - 0.6)),
'%.2f' % (depths[5] - (target - 0.3)),
'%.2f' % (depths[10] - target))
return depths
minra = np.min(allra)
maxra = np.max(allra)
mindec = np.min(alldec)
maxdec = np.max(alldec)
print('RA,Dec range:', minra, maxra, mindec, maxdec)
plothist(allra, alldec)
plt.axis([maxra, minra, mindec, maxdec])
plt.xlabel('RA (deg)')
plt.ylabel('Dec (deg)')
plt.savefig('match-all.png')
Tref = fits_table('gaia.fits')
r_arcsec = 0.2
I, J, d = match_radec(Tref.ra, Tref.dec, allra, alldec, r_arcsec / 3600.)
dec = alldec[J]
cosdec = np.cos(np.deg2rad(dec))
dr = (Tref.ra[I] - allra[J]) * cosdec * 3600.
dd = (Tref.dec[I] - alldec[J]) * 3600.
plt.clf()
rr = (-r_arcsec * 1000, +r_arcsec * 1000)
plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr))
plt.xlabel('dRA (milli-arcsec)')
plt.ylabel('dDec (milli-arcsec)')
plt.savefig('match-all-ref-before.png')
# Initial matching of all stars
r_arcsec = 0.2
I, J, d = match_radec(allra,
alldec,
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).")
parser.add_argument('--calibs', action='store_true',
help='Output CCDs that need to be calibrated.')
parser.add_argument('--nper', type=int, default=None,
help='Batch N calibs per line')
parser.add_argument('--forced', action='store_true',
help='Output forced-photometry commands')
parser.add_argument('--lsb', action='store_true',
help='Output Low-Surface-Brightness commands')
parser.add_argument('--touching', action='store_true',
help='Cut to only CCDs touching selected bricks')
parser.add_argument('--near', action='store_true',
help='Quick cut to only CCDs near selected bricks')
parser.add_argument('--check', action='store_true',
help='Check which calibrations actually need to run.')
parser.add_argument('--check-coadd', action='store_true',
help='Check which caoadds actually need to run.')
parser.add_argument('--out', help='Output filename for calibs, default %(default)s',
default='jobs')
parser.add_argument('--command', action='store_true',
help='Write out full command-line to run calib')
parser.add_argument('--opt', help='With --command, extra options to add')
parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
parser.add_argument('--mindec', type=float, help='Minimum Dec to run')
parser.add_argument('--region', help='Region to select')
parser.add_argument('--bricks', help='Set bricks.fits file to load')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--ignore_cuts', action='store_true',default=False,help='no photometric or blacklist cuts')
parser.add_argument('--save_to_fits', action='store_true',default=False,help='save cut brick,ccd to fits table')
parser.add_argument('--name', action='store',default='dr3',help='save with this suffix, e.g. refers to ccds table')
parser.add_argument('--delete-sky', action='store_true',
help='Delete any existing sky calibration files')
parser.add_argument('--delete-pvastrom', action='store_true',
help='Delete any existing PV WCS calibration files')
parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')
parser.add_argument('--brickq', type=int, default=None,
help='Queue only bricks with the given "brickq" value [0 to 3]')
parser.add_argument('--brickq-deps', action='store_true', default=False,
help='Queue bricks directly using qdo API, setting brickq dependencies')
parser.add_argument('--queue', default='bricks',
help='With --brickq-deps, the QDO queue name to use')
opt = parser.parse_args()
survey = LegacySurveyData()
if opt.bricks is not None:
B = fits_table(opt.bricks)
log('Read', len(B), 'from', opt.bricks)
else:
B = survey.get_bricks()
if opt.ccds is not None:
T = fits_table(opt.ccds)
log('Read', len(T), 'from', opt.ccds)
else:
T = survey.get_ccds()
log(len(T), 'CCDs')
T.index = np.arange(len(T))
if opt.ignore_cuts == False:
I = survey.photometric_ccds(T)
print(len(I), 'CCDs are photometric')
T.cut(I)
I = survey.apply_blacklist(T)
print(len(I), 'CCDs are not blacklisted')
T.cut(I)
print(len(T), 'CCDs remain')
# I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
# plt.clf()
# plt.hist(counts, counts.max()+1)
# plt.savefig('bricks.png')
# B.cut(I[counts >= 9])
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# #plt.scatter(B.ra[I], B.dec[I], c=counts)
# plt.savefig('bricks2.png')
# DES Stripe82
#rlo,rhi = 350.,360.
# rlo,rhi = 300., 10.
# dlo,dhi = -6., 4.
# TINY bit
#rlo,rhi = 350.,351.1
#dlo,dhi = 0., 1.1
# EDR+
# 860 bricks
# ~10,000 CCDs
#rlo,rhi = 239,246
#dlo,dhi = 5, 13
# DR1
#rlo,rhi = 0, 360
# part 1
#dlo,dhi = 25, 40
# part 2
#dlo,dhi = 20,25
# part 3
#dlo,dhi = 15,20
# part 4
#dlo,dhi = 10,15
# part 5
#dlo,dhi = 5,10
# the rest
#dlo,dhi = -11, 5
#dlo,dhi = 15,25.5
dlo,dhi = -25, 40
rlo,rhi = 0, 360
# Arjun says 3x3 coverage area is roughly
# RA=240-252 DEC=6-12 (but not completely rectangular)
# COSMOS
#rlo,rhi = 148.9, 151.2
#dlo,dhi = 0.9, 3.5
# A nice well-behaved region (EDR2/3)
# rlo,rhi = 243.6, 244.6
# dlo,dhi = 8.1, 8.6
# 56 bricks, ~725 CCDs
#B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
# 240 bricks, ~3000 CCDs
#B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
# 535 bricks, ~7000 CCDs
#B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))
if opt.region in ['test1', 'test2', 'test3', 'test4']:
nm = dict(test1='2446p115', # weird stuff around bright star
test2='1183p292', # faint sources around bright galaxy
test3='3503p005', # DES
test4='1163p277', # Pollux
)[opt.region]
B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
log('Cut to', len(B), 'bricks')
log(B.ra, B.dec)
dlo,dhi = -90,90
rlo,rhi = 0, 360
elif opt.region == 'edr':
# EDR:
# 535 bricks, ~7000 CCDs
rlo,rhi = 240,245
dlo,dhi = 5, 12
elif opt.region == 'edrplus':
rlo,rhi = 235,248
dlo,dhi = 5, 15
elif opt.region == 'edr-south':
rlo,rhi = 240,245
dlo,dhi = 5, 10
elif opt.region == 'cosmos1':
# 16 bricks in the core of the COSMOS field.
rlo,rhi = 149.75, 150.75
dlo,dhi = 1.6, 2.6
elif opt.region == 'pristine':
# Stream?
rlo,rhi = 240,250
dlo,dhi = 10,15
elif opt.region == 'des':
dlo, dhi = -6., 4.
rlo, rhi = 317., 7.
T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
log('Cut to', len(T), 'CCDs with "CPDES82" in filename')
elif opt.region == 'subdes':
rlo,rhi = 320., 360.
dlo,dhi = -1.25, 1.25
elif opt.region == 'northwest':
rlo,rhi = 240,360
dlo,dhi = 20,40
elif opt.region == 'north':
rlo,rhi = 120,240
dlo,dhi = 20,40
elif opt.region == 'northeast':
rlo,rhi = 0,120
dlo,dhi = 20,40
elif opt.region == 'southwest':
rlo,rhi = 240,360
dlo,dhi = -20,0
elif opt.region == 'south':
rlo,rhi = 120,240
dlo,dhi = -20,0
elif opt.region == 'southeast':
rlo,rhi = 0,120
dlo,dhi = -20,0
elif opt.region == 'southsoutheast':
rlo,rhi = 0,120
dlo,dhi = -20,-10
elif opt.region == 'midwest':
rlo,rhi = 240,360
dlo,dhi = 0,20
elif opt.region == 'middle':
rlo,rhi = 120,240
dlo,dhi = 0,20
elif opt.region == 'mideast':
rlo,rhi = 0,120
dlo,dhi = 0,20
elif opt.region == 'grz':
# Bricks with grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
log('Cut to', len(B), 'bricks with grz coverage')
elif opt.region == 'nogrz':
# Bricks without grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
log('Cut to', len(B), 'bricks withOUT grz coverage')
elif opt.region == 'deep2':
rlo,rhi = 250,260
dlo,dhi = 30,35
elif opt.region == 'deep2f3':
rlo,rhi = 351.25, 353.75
dlo,dhi = 0, 0.5
elif opt.region == 'virgo':
rlo,rhi = 185,190
dlo,dhi = 10, 15
elif opt.region == 'virgo2':
rlo,rhi = 182,192
dlo,dhi = 8, 18
elif opt.region == 'lsb':
rlo,rhi = 147.2, 147.8
dlo,dhi = -0.4, 0.4
elif opt.region == 'eboss-elg':
# RA -45 to +45
# Dec -5 to +7
rlo,rhi = 315., 45.
dlo,dhi = -5., 7.
elif opt.region == 'eboss-ngc':
# NGC ELGs
# RA 115 to 175
# Dec 15 to 30
rlo,rhi = 115., 175.
dlo,dhi = 15., 30.
elif opt.region == 'mzls':
dlo,dhi = 30., 90.
elif opt.region == 'dr4-bootes':
# https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
#dlo,dhi = 34., 35.
#rlo,rhi = 209.5, 210.5
dlo,dhi = 33., 36.
rlo,rhi = 216.5, 219.5
if opt.mindec is not None:
dlo = opt.mindec
if opt.maxdec is not None:
dhi = opt.maxdec
if rlo < rhi:
B.cut((B.ra >= rlo) * (B.ra <= rhi) *
(B.dec >= dlo) * (B.dec <= dhi))
else: # RA wrap
B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) *
(B.dec >= dlo) * (B.dec <= dhi))
log(len(B), 'bricks in range')
for name in B.get('brickname'):
print(name)
B.writeto('bricks-cut.fits')
I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs')
plt.clf()
plt.plot(B.ra, B.dec, 'b.')
plt.title('DR3 bricks')
plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
plt.savefig('bricks.png')
if opt.brickq is not None:
B.cut(B.brickq == opt.brickq)
log('Cut to', len(B), 'with brickq =', opt.brickq)
if opt.touching:
keep = np.zeros(len(T), bool)
for j in J:
keep[j] = True
T.cut(keep)
log('Cut to', len(T), 'CCDs near bricks')
# Aside -- how many near DR1=1 CCDs?
if False:
T2 = D.get_ccds()
log(len(T2), 'CCDs')
T2.cut(T2.dr1 == 1)
log(len(T2), 'CCDs marked DR1=1')
log(len(B), 'bricks in range')
I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B2 = B[keep]
log('Total of', len(B2), 'bricks near CCDs with DR1=1')
for band in 'grz':
Tb = T2[T2.filter == band]
log(len(Tb), 'in filter', band)
I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, survey.bricksize)
good = np.zeros(len(B2), np.uint8)
for i in I:
good[i] = 1
B2.set('has_' + band, good)
B2.writeto('survey-bricks-in-dr1.fits')
sys.exit(0)
# sort by dec decreasing
#B.cut(np.argsort(-B.dec))
# RA increasing
B.cut(np.argsort(B.ra))
for b in B:
if opt.check:
fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
if opt.check_coadd:
fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
print(b.brickname)
if opt.save_to_fits:
assert(opt.touching)
# Write cut tables to file
for tab,typ in zip([B,T],['bricks','ccds']):
fn='%s-%s-cut.fits' % (typ,opt.name)
if os.path.exists(fn):
os.remove(fn)
tab.writeto(fn)
print('Wrote %s' % fn)
# Write text files listing ccd and filename names
nm1,nm2= 'ccds-%s.txt'% opt.name,'filenames-%s.txt' % opt.name
if os.path.exists(nm1):
os.remove(nm1)
if os.path.exists(nm2):
os.remove(nm2)
f1,f2=open(nm1,'w'),open(nm2,'w')
fns= list(set(T.get('image_filename')))
for fn in fns:
f2.write('%s\n' % fn.strip())
for ti in T:
f1.write('%s\n' % ti.get('image_filename').strip())
f1.close()
f2.close()
print('Wrote *-names.txt')
if opt.brickq_deps:
import qdo
from legacypipe.survey import on_bricks_dependencies
#... find Queue...
q = qdo.connect(opt.queue, create_ok=True)
print('Connected to QDO queue', opt.queue, q)
brick_to_task = dict()
I = survey.photometric_ccds(T)
print(len(I), 'CCDs are photometric')
T.cut(I)
I = survey.apply_blacklist(T)
print(len(I), 'CCDs are not blacklisted')
T.cut(I)
print(len(T), 'CCDs remaining')
T.wra = T.ra + (T.ra > 180) * -360
wra = rlo - 360
plt.clf()
plt.plot(T.wra, T.dec, 'b.')
ax = [wra, rhi, dlo, dhi]
plt.axis(ax)
plt.title('CCDs')
plt.savefig('q-ccds.png')
B.wra = B.ra + (B.ra > 180) * -360
# this slight overestimate (for DECam images) is fine
radius = 0.3
Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius,
indexlist=True)
ikeep = []
for ib,(b,Iccd) in enumerate(zip(B, Iccds)):
if Iccd is None or len(Iccd) == 0:
print('No matched CCDs to brick', b.brickname)
continue
wcs = wcs_for_brick(b)
cI = ccds_touching_wcs(wcs, T[np.array(Iccd)])
print(len(cI), 'CCDs touching brick', b.brickname)
if len(cI) == 0:
continue
ikeep.append(ib)
B.cut(np.array(ikeep))
print('Cut to', len(B), 'bricks touched by CCDs')
for brickq in range(4):
I = np.flatnonzero(B.brickq == brickq)
print(len(I), 'bricks with brickq =', brickq)
J = np.flatnonzero(B.brickq < brickq)
preB = B[J]
reqs = []
if brickq > 0:
for b in B[I]:
# find brick dependencies
brickdeps = on_bricks_dependencies(b, survey, bricks=preB)
# convert to task ids
taskdeps = [brick_to_task.get(b.brickname,None) for b in brickdeps]
# If we dropped a dependency brick from a previous brickq because
# of no overlapping CCDs, it won't appear in the brick_to_task map.
taskdeps = [t for t in taskdeps if t is not None]
reqs.append(taskdeps)
plt.clf()
plt.plot(B.wra, B.dec, '.', color='0.5')
plt.plot(B.wra[I], B.dec[I], 'b.')
plt.axis(ax)
plt.title('Bricks: brickq=%i' % brickq)
plt.savefig('q-bricks-%i.png' % brickq)
# submit to qdo queue
print('Queuing', len(B[I]), 'bricks')
if brickq == 0:
reqs = None
else:
assert(len(I) == len(reqs))
taskids = q.add_multiple(B.brickname[I], requires=reqs)
assert(len(taskids) == len(I))
print('Queued', len(taskids), 'bricks')
brick_to_task.update(dict(zip(B.brickname[I], taskids)))
if not (opt.calibs or opt.forced or opt.lsb):
sys.exit(0)
bands = 'grz'
log('Filters:', np.unique(T.filter))
T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
log('Cut to', len(T), 'CCDs in filters', bands)
if opt.touching:
allI = set()
for b in B:
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
if len(I) == 0:
continue
allI.update(I)
allI = list(allI)
allI.sort()
elif opt.near:
# Roughly brick radius + DECam image radius
radius = 0.35
allI,nil,nil = match_radec(T.ra, T.dec, B.ra, B.dec, radius, nearest=True)
else:
allI = np.arange(len(T))
if opt.write_ccds:
T[allI].writeto(opt.write_ccds)
log('Wrote', opt.write_ccds)
## Be careful here -- T has been cut; we want to write out T.index.
## 'allI' contains indices into T.
if opt.forced:
log('Writing forced-photometry commands to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
for j,i in enumerate(allI):
expstr = '%08i' % T.expnum[i]
outfn = os.path.join('forced', expstr[:5], expstr,
'decam-%s-%s-forced.fits' %
(expstr, T.ccdname[i]))
imgfn = os.path.join(survey.survey_dir, 'images',
T.image_filename[i].strip())
if (not os.path.exists(imgfn) and
imgfn.endswith('.fz') and
os.path.exists(imgfn[:-3])):
imgfn = imgfn[:-3]
#f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' %
# (imgfn, T.image_hdu[i], outfn))
f.write('python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n' %
(T.expnum[i], T.ccdname[i], outfn))
f.close()
log('Wrote', opt.out)
sys.exit(0)
if opt.lsb:
log('Writing LSB commands to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
for j,i in enumerate(allI):
exp = T.expnum[i]
ext = T.ccdname[i].strip()
outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext))
f.close()
log('Wrote', opt.out)
sys.exit(0)
log('Writing calibs to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
batch = []
def write_batch(f, batch, cmd):
if cmd is None:
cmd = ''
f.write(cmd + ' '.join(batch) + '\n')
cmd = None
if opt.command:
cmd = 'python legacypipe/run-calib.py '
if opt.opt is not None:
cmd += opt.opt + ' '
for j,i in enumerate(allI):
if opt.delete_sky or opt.delete_pvastrom:
log(j+1, 'of', len(allI))
im = survey.get_image_object(T[i])
if opt.delete_sky and os.path.exists(im.skyfn):
log(' deleting:', im.skyfn)
os.unlink(im.skyfn)
if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
log(' deleting:', im.pvwcsfn)
os.unlink(im.pvwcsfn)
if opt.check:
log(j+1, 'of', len(allI))
im = survey.get_image_object(T[i])
if not im.run_calibs(im, just_check=True):
log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done')
continue
if opt.command:
s = '%i-%s' % (T.expnum[i], T.ccdname[i])
prefix = 'python legacypipe/run-calib.py ' + opt.opt
#('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
# (T.expnum[i], T.ccdname[i]))
else:
s = '%i' % T.index[i]
prefix = ''
if j < 10:
print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i],
'filename', T.image_filename[i])
if not opt.nper:
f.write(prefix + s + '\n')
else:
batch.append(s)
if len(batch) >= opt.nper:
write_batch(f, batch, cmd)
batch = []
if opt.check:
f.flush()
if len(batch):
write_batch(f, batch, cmd)
f.close()
log('Wrote', opt.out)
return 0
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--name1', help='Name for first data set')
parser.add_argument('--name2', help='Name for second data set')
parser.add_argument('--plot-prefix',
default='compare',
help='Prefix for plot filenames; default "%default"')
parser.add_argument('--match',
default=1.0,
help='Astrometric cross-match distance in arcsec')
parser.add_argument('dir1', help='First directory to compare')
parser.add_argument('dir2', help='Second directory to compare')
opt = parser.parse_args()
ps = PlotSequence(opt.plot_prefix)
name1 = opt.name1
if name1 is None:
name1 = os.path.basename(opt.dir1)
if not len(name1):
name1 = os.path.basename(os.path.dirname(opt.dir1))
name2 = opt.name2
if name2 is None:
name2 = os.path.basename(opt.dir2)
if not len(name2):
name2 = os.path.basename(os.path.dirname(opt.dir2))
tt = 'Comparing %s to %s' % (name1, name2)
# regex for tractor-*.fits catalog filename
catre = re.compile('tractor-.*.fits')
cat1, cat2 = [], []
for basedir, cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
for dirpath, dirnames, filenames in os.walk(basedir, followlinks=True):
for fn in filenames:
if not catre.match(fn):
print('Skipping', fn, 'due to filename')
continue
fn = os.path.join(dirpath, fn)
t = fits_table(fn)
print(len(t), 'from', fn)
cat.append(t)
cat1 = merge_tables(cat1, columns='fillzero')
cat2 = merge_tables(cat2, columns='fillzero')
print('Total of', len(cat1), 'from', name1)
print('Total of', len(cat2), 'from', name2)
cat1.cut(cat1.brick_primary)
cat2.cut(cat2.brick_primary)
print('Total of', len(cat1), 'BRICK_PRIMARY from', name1)
print('Total of', len(cat2), 'BRICK_PRIMARY from', name2)
cat1.cut((cat1.decam_anymask[:, 1] == 0) *
(cat1.decam_anymask[:, 2] == 0) * (cat1.decam_anymask[:, 4] == 0))
cat2.cut((cat2.decam_anymask[:, 1] == 0) *
(cat2.decam_anymask[:, 2] == 0) * (cat2.decam_anymask[:, 4] == 0))
print('Total of', len(cat1), 'unmasked from', name1)
print('Total of', len(cat2), 'unmasked from', name2)
I, J, d = match_radec(cat1.ra,
cat1.dec,
cat2.ra,
cat2.dec,
opt.match / 3600.,
nearest=True)
print(len(I), 'matched')
matched1 = cat1[I]
matched2 = cat2[J]
def get_reference_sources(survey,
targetwcs,
pixscale,
bands,
tycho_stars=True,
gaia_stars=True,
large_galaxies=True,
star_clusters=True):
from legacypipe.survey import GaiaSource
from legacypipe.survey import LegacyEllipseWithPriors
from tractor import NanoMaggies, RaDecPos
from tractor.galaxy import ExpGalaxy
from tractor.ellipses import EllipseESoft
H, W = targetwcs.shape
H, W = int(H), int(W)
# How big of a margin to search for bright stars and star clusters --
# this should be based on the maximum radius they are considered to
# affect.
ref_margin = 0.125
mpix = int(np.ceil(ref_margin * 3600. / pixscale))
marginwcs = targetwcs.get_subimage(-mpix, -mpix, W + 2 * mpix,
H + 2 * mpix)
refs = []
# Tycho-2 stars
if tycho_stars:
tycho = read_tycho2(survey, marginwcs)
if len(tycho):
refs.append(tycho)
# Add Gaia stars
gaia = None
if gaia_stars:
from astrometry.libkd.spherematch import match_radec
gaia = read_gaia(marginwcs)
if gaia is not None:
gaia.isbright = (gaia.phot_g_mean_mag < 13.)
gaia.ismedium = (gaia.phot_g_mean_mag < 16.)
gaia.donotfit = np.zeros(len(gaia), bool)
# Handle sources that appear in both Gaia and Tycho-2 by
# dropping the entry from Tycho-2.
if len(gaia) and len(tycho):
# Before matching, apply proper motions to bring them to
# the same epoch. We want to use the more-accurate Gaia
# proper motions, so rewind Gaia positions to the
# approximate epoch of Tycho-2: 1991.5.
cosdec = np.cos(np.deg2rad(gaia.dec))
gra = gaia.ra + (1991.5 - gaia.ref_epoch) * gaia.pmra / (
3600. * 1000.) / cosdec
gdec = gaia.dec + (1991.5 - gaia.ref_epoch) * gaia.pmdec / (3600. *
1000.)
I, J, _ = match_radec(tycho.ra,
tycho.dec,
gra,
gdec,
1. / 3600.,
nearest=True)
debug('Matched', len(I), 'Tycho-2 stars to Gaia stars.')
if len(I):
keep = np.ones(len(tycho), bool)
keep[I] = False
tycho.cut(keep)
gaia.isbright[J] = True
if gaia is not None and len(gaia) > 0:
refs.append(gaia)
# Read the catalog of star (open and globular) clusters and add them to the
# set of reference stars (with the isbright bit set).
if star_clusters:
clusters = read_star_clusters(marginwcs)
if clusters is not None:
debug('Found', len(clusters), 'star clusters nearby')
clusters.iscluster = np.ones(len(clusters), bool)
refs.append(clusters)
# Read large galaxies nearby.
if large_galaxies:
galaxies = read_large_galaxies(survey, targetwcs)
if galaxies is not None:
# Resolve possible Gaia-large-galaxy duplicates
if gaia and len(gaia):
I, J, _ = match_radec(galaxies.ra,
galaxies.dec,
gaia.ra,
gaia.dec,
2. / 3600.,
nearest=True)
print('Matched', len(I), 'large galaxies to Gaia stars.')
if len(I):
gaia.donotfit[J] = True
refs.append(galaxies)
refcat = None
if len(refs):
refs = merge_tables([r for r in refs if r is not None],
columns='fillzero')
if len(refs) == 0:
return None, None
refs.radius_pix = np.ceil(refs.radius * 3600. / pixscale).astype(int)
ok, xx, yy = targetwcs.radec2pixelxy(refs.ra, refs.dec)
# ibx = integer brick coords
refs.ibx = np.round(xx - 1.).astype(int)
refs.iby = np.round(yy - 1.).astype(int)
# cut ones whose position + radius are outside the brick bounds.
refs.cut((xx > -refs.radius_pix) * (xx < W + refs.radius_pix) *
(yy > -refs.radius_pix) * (yy < H + refs.radius_pix))
# mark ones that are actually inside the brick area.
refs.in_bounds = ((refs.ibx >= 0) * (refs.ibx < W) * (refs.iby >= 0) *
(refs.iby < H))
for col in [
'isbright', 'ismedium', 'islargegalaxy', 'iscluster', 'donotfit'
]:
if not col in refs.get_columns():
refs.set(col, np.zeros(len(refs), bool))
## Create Tractor sources from reference stars
refcat = []
for g in refs:
if g.donotfit or g.iscluster:
refcat.append(None)
elif g.islargegalaxy:
fluxes = dict([(band, NanoMaggies.magToNanomaggies(g.mag))
for band in bands])
assert (np.all(np.isfinite(list(fluxes.values()))))
rr = g.radius * 3600. / 0.5 # factor of two accounts for R(25)-->reff
pa = 180 - g.pa
if not np.isfinite(pa):
pa = 0.
logr, ee1, ee2 = EllipseESoft.rAbPhiToESoft(rr, g.ba, pa)
gal = ExpGalaxy(RaDecPos(g.ra, g.dec),
NanoMaggies(order=bands, **fluxes),
LegacyEllipseWithPriors(logr, ee1, ee2))
refcat.append(gal)
else:
# Gaia star -- which we want to create a source for, regardless of
# whether it is marked medium | bright (or neither).
refcat.append(GaiaSource.from_catalog(g, bands))
for src in refcat:
if src:
src.is_reference_source = True
return refs, refcat
splinesky=True,
coadd_bw=True,
forceAll=True,
writePickles=False)
print('Reading', outfn)
cat = fits_table(outfn)
primhdr = fitsio.read_header(outfn)
iband = survey.index_of_band(im.band)
cat.flux = cat.decam_flux[:, iband]
cat.apflux = cat.decam_apflux[:, iband, :]
cat.mag = -2.5 * (np.log10(cat.flux) - 9.)
cat.apmag = -2.5 * (np.log10(cat.apflux) - 9.)
I,J,d = match_radec(stars.ra, stars.dec, cat.ra, cat.dec, 1./3600.,
nearest=True)
print(len(I), 'matches to PS1 stars')
colorterm = ps1_to_decam(stars.median, im.band)
ipsband = ps1cat.ps1band[im.band]
stars.mag = stars.median[:, ipsband] + colorterm
stars.flux = 10.**((stars.mag - 22.5) / -2.5)
mstars = stars[I]
mcat = cat[J]
print(np.sum(mcat.type == 'PSF '), 'matched sources are PSF')
mcat.ispsf = (mcat.type == 'PSF ')
mcat.unmasked = (np.sum(mcat.decam_anymask, axis=1) == 0)
P = np.flatnonzero(mcat.ispsf * mcat.unmasked)
print(len(P), 'matched, unmasked, PSFs')
def read_forcedphot_ccds(ccds, survey):
ccds.mdiff = np.zeros(len(ccds))
ccds.mscatter = np.zeros(len(ccds))
Nap = 8
ccds.apdiff = np.zeros((len(ccds), Nap))
ccds.apscatter = np.zeros((len(ccds), Nap))
ccds.nforced = np.zeros(len(ccds), np.int16)
ccds.nunmasked = np.zeros(len(ccds), np.int16)
ccds.nmatched = np.zeros(len(ccds), np.int16)
ccds.nps1 = np.zeros(len(ccds), np.int16)
brickcache = {}
FF = []
for iccd,ccd in enumerate(ccds):
print('CCD', iccd, 'of', len(ccds))
F = fits_table(ccd.path)
print(len(F), 'sources in', ccd.path)
ccds.nforced[iccd] = len(F)
# arr, have to match with brick sources to get RA,Dec.
F.ra = np.zeros(len(F))
F.dec = np.zeros(len(F))
F.masked = np.zeros(len(F), bool)
maglo,maghi = 14.,21.
maxdmag = 1.
F.mag = -2.5 * (np.log10(F.flux) - 9)
F.cut((F.flux > 0) * (F.mag > maglo-maxdmag) * (F.mag < maghi+maxdmag))
print(len(F), 'sources between', (maglo-maxdmag), 'and', (maghi+maxdmag), 'mag')
im = survey.get_image_object(ccd)
print('Reading DQ image for', im)
dq = im.read_dq()
H,W = dq.shape
ix = np.clip(np.round(F.x), 0, W-1).astype(int)
iy = np.clip(np.round(F.y), 0, H-1).astype(int)
F.mask = dq[iy,ix]
print(np.sum(F.mask != 0), 'sources are masked')
for brickname in np.unique(F.brickname):
if not brickname in brickcache:
brickcache[brickname] = fits_table(survey.find_file('tractor', brick=brickname))
T = brickcache[brickname]
idtoindex = np.zeros(T.objid.max()+1, int) - 1
idtoindex[T.objid] = np.arange(len(T))
I = np.flatnonzero(F.brickname == brickname)
J = idtoindex[F.objid[I]]
assert(np.all(J >= 0))
F.ra [I] = T.ra [J]
F.dec[I] = T.dec[J]
F.masked[I] = (T.decam_anymask[J,:].max(axis=1) > 0)
#F.cut(F.masked == False)
#print(len(F), 'not masked')
print(np.sum(F.masked), 'masked in ANYMASK')
ccds.nunmasked[iccd] = len(F)
wcs = Tan(*[float(x) for x in [ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2,
ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2,
ccd.width, ccd.height]])
ps1 = ps1cat(ccdwcs=wcs)
stars = ps1.get_stars()
print(len(stars), 'PS1 sources')
ccds.nps1[iccd] = len(stars)
# Now cut to just *stars* with good colors
stars.gicolor = stars.median[:,0] - stars.median[:,2]
keep = (stars.gicolor > 0.4) * (stars.gicolor < 2.7)
stars.cut(keep)
print(len(stars), 'PS1 stars with good colors')
stars.cut(np.minimum(stars.stdev[:,1], stars.stdev[:,2]) < 0.05)
print(len(stars), 'PS1 stars with min stdev(r,i) < 0.05')
I,J,d = match_radec(F.ra, F.dec, stars.ra, stars.dec, 1./3600.)
print(len(I), 'matches')
band = ccd.filter
colorterm = ps1_to_decam(stars.median[J], band)
F.cut(I)
F.psmag = stars.median[J, ps1.ps1band[band]] + colorterm
K = np.flatnonzero((F.psmag > maglo) * (F.psmag < maghi))
print(len(K), 'with mag', maglo, 'to', maghi)
F.cut(K)
K = np.flatnonzero(np.abs(F.mag - F.psmag) < maxdmag)
print(len(K), 'with good mag matches (<', maxdmag, 'mag difference)')
ccds.nmatched[iccd] = len(K)
if len(K) == 0:
continue
F.cut(K)
ccds.mdiff[iccd] = np.median(F.mag - F.psmag)
ccds.mscatter[iccd] = (np.percentile(F.mag - F.psmag, 84) -
np.percentile(F.mag - F.psmag, 16))/2.
for i in range(Nap):
apmag = -2.5 * (np.log10(F.apflux[:, i]) - 9)
ccds.apdiff[iccd,i] = np.median(apmag - F.psmag)
ccds.apscatter[iccd,i] = (np.percentile(apmag - F.psmag, 84) -
np.percentile(apmag - F.psmag, 16))/2.
#F.about()
for c in ['apflux_ivar', 'brickid', 'flux_ivar',
'mjd', 'objid', 'fracflux', 'rchi2', 'x','y']:
F.delete_column(c)
F.expnum = np.zeros(len(F), np.int32) + ccd.expnum
F.ccdname = np.array([ccd.ccdname] * len(F))
F.iforced = np.zeros(len(F), np.int32) + iccd
FF.append(F)
FF = merge_tables(FF)
return FF
bricks = survey.get_bricks_readonly()
bricks = bricks[(bricks.ra > ralo) * (bricks.ra < rahi) *
(bricks.dec > declo) * (bricks.dec < dechi)]
print(len(bricks), 'bricks')
I, = np.nonzero([os.path.exists(survey.find_file('tractor', brick=b.brickname))
for b in bricks])
print(len(I), 'bricks with catalogs')
bricks.cut(I)
for band in bands:
bricks.set('diff_%s' % band, np.zeros(len(bricks), np.float32))
bricks.set('psfsize_%s' % band, np.zeros(len(bricks), np.float32))
diffs = dict([(b,[]) for b in bands])
for ibrick,b in enumerate(bricks):
fn = survey.find_file('tractor', brick=b.brickname)
T = fits_table(fn)
print(len(T), 'sources in', b.brickname)
brickwcs = wcs_for_brick(b)
ps1 = ps1cat(ccdwcs=brickwcs)
stars = ps1.get_stars()
print(len(stars), 'PS1 sources')
# Now cut to just *stars* with good colors
stars.gicolor = stars.median[:,0] - stars.median[:,2]
keep = (stars.gicolor > 0.4) * (stars.gicolor < 2.7)
stars.cut(keep)
print(len(stars), 'PS1 stars with good colors')
I,J,d = match_radec(T.ra, T.dec, stars.ra, stars.dec, 1./3600.)
print(len(I), 'matches')
for band in bands:
bricks.get('psfsize_%s' % band)[ibrick] = np.median(
T.decam_psfsize[:, survey.index_of_band(band)])
colorterm = ps1_to_decam(stars.median[J], band)
psmag = stars.median[J, ps1.ps1band[band]]
psmag += colorterm
decflux = T.decam_flux[I, survey.index_of_band(band)]
decmag = -2.5 * (np.log10(decflux) - 9)
#K = np.flatnonzero((psmag > 14) * (psmag < 24))
#print(len(K), 'with mag 14 to 24')
K = np.flatnonzero((psmag > 14) * (psmag < 21))
print(len(K), 'with mag 14 to 21')
decmag = decmag[K]
psmag = psmag [K]
K = np.flatnonzero(np.abs(decmag - psmag) < 1)
print(len(K), 'with good mag matches (< 1 mag difference)')
decmag = decmag[K]
psmag = psmag [K]
if False and ibrick == 0:
plt.clf()
#plt.plot(psmag, decmag, 'b.')
plt.plot(psmag, decmag - psmag, 'b.')
plt.xlabel('PS1 mag')
plt.xlabel('DECam - PS1 mag')
plt.title('PS1 matches for %s band, brick %s' % (band, b.brickname))
ps.savefig()
mdiff = np.median(decmag - psmag)
diffs[band].append(mdiff)
print('Median difference:', mdiff)
bricks.get('diff_%s' % band)[ibrick] = mdiff
for band in bands:
d = diffs[band]
plt.clf()
plt.hist(d, bins=20, range=(-0.02, 0.02), histtype='step')
plt.xlabel('Median mag difference per brick')
plt.title('DR3 EDR PS1 vs DECaLS: %s band' % band)
ps.savefig()
print('Median differences in', band, 'band:', np.median(d))
if False:
plt.clf()
plt.hist(diffs['g'], bins=20, range=(-0.02, 0.02), histtype='step', color='g')
plt.hist(diffs['r'], bins=20, range=(-0.02, 0.02), histtype='step', color='r')
plt.hist(diffs['z'], bins=20, range=(-0.02, 0.02), histtype='step', color='m')
plt.xlabel('Median mag difference per brick')
plt.title('DR3 EDR PS1 vs DECaLS')
ps.savefig()
rr,dd = np.meshgrid(np.linspace(ralo,rahi, 400), np.linspace(declo,dechi, 400))
I,J,d = match_radec(rr.ravel(), dd.ravel(), bricks.ra, bricks.dec, 0.18, nearest=True)
print(len(I), 'matches')
for band in bands:
plt.clf()
dmag = np.zeros_like(rr) - 1.
dmag.ravel()[I] = bricks.get('diff_%s' % band)[J]
plt.imshow(dmag, interpolation='nearest', origin='lower',
vmin=-0.01, vmax=0.01, cmap='hot',
extent=(ralo,rahi,declo,dechi))
plt.colorbar()
plt.title('DR3 EDR PS1 vs DECaLS: %s band' % band)
plt.xlabel('RA (deg)')
plt.ylabel('Dec (deg)')
plt.axis([ralo,rahi,declo,dechi])
ps.savefig()
plt.clf()
# reuse 'dmag' map...
dmag = np.zeros_like(rr)
dmag.ravel()[I] = bricks.get('psfsize_%s' % band)[J]
plt.imshow(dmag, interpolation='nearest', origin='lower',
cmap='hot', extent=(ralo,rahi,declo,dechi))
plt.colorbar()
plt.title('DR3 EDR: DECaLS PSF size: %s band' % band)
plt.xlabel('RA (deg)')
plt.ylabel('Dec (deg)')
plt.axis([ralo,rahi,declo,dechi])
ps.savefig()
if False:
for band in bands:
plt.clf()
plt.scatter(bricks.ra, bricks.dec, c=bricks.get('diff_%s' % band), vmin=-0.01, vmax=0.01,
edgecolors='face', s=200)
plt.colorbar()
plt.title('DR3 EDR PS1 vs DECaLS: %s band' % band)
plt.xlabel('RA (deg)')
plt.ylabel('Dec (deg)')
plt.axis('scaled')
plt.axis([ralo,rahi,declo,dechi])
ps.savefig()
plt.clf()
plt.plot(bricks.psfsize_g, bricks.diff_g, 'g.')
plt.plot(bricks.psfsize_r, bricks.diff_r, 'r.')
plt.plot(bricks.psfsize_z, bricks.diff_z, 'm.')
plt.xlabel('PSF size (arcsec)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.title('DR3 EDR')
ps.savefig()
def plot_light_curves(pfn, ucal=False):
lightcurves = unpickle_from_file(pfn)
if ucal:
tag = 'ucal-'
else:
tag = ''
survey = LegacySurveyData()
brickname = '0364m042'
catfn = survey.find_file('tractor', brick=brickname)
print('Reading catalog from', catfn)
cat = fits_table(catfn)
print(len(cat), 'catalog entries')
cat.cut(cat.brick_primary)
print(len(cat), 'brick primary')
I = []
for i, oid in enumerate(cat.objid):
if (brickname, oid) in lightcurves:
I.append(i)
I = np.array(I)
cat.cut(I)
print('Cut to', len(cat), 'with light curves')
S = fits_table('specObj-dr12-trim-2.fits')
from astrometry.libkd.spherematch import match_radec
I, J, d = match_radec(S.ra, S.dec, cat.ra, cat.dec, 2. / 3600.)
print('Matched', len(I), 'to spectra')
plt.subplots_adjust(hspace=0)
movie_jpegs = []
movie_wcs = None
for i in range(28):
fn = os.path.join('des-sn-movie', 'epoch%i' % i, 'coadd',
brickname[:3], brickname,
'legacysurvey-%s-image.jpg' % brickname)
print(fn)
if not os.path.exists(fn):
continue
img = plt.imread(fn)
img = np.flipud(img)
h, w, d = img.shape
fn = os.path.join('des-sn-movie', 'epoch%i' % i, 'coadd',
brickname[:3], brickname,
'legacysurvey-%s-image-r.fits' % brickname)
if not os.path.exists(fn):
continue
wcs = Tan(fn)
movie_jpegs.append(img)
movie_wcs = wcs
plt.figure(figsize=(8, 6), dpi=100)
n = 0
fluxtags = [('flux', 'flux_ivar', '', 'a')]
if ucal:
fluxtags.append(('uflux', 'uflux_ivar', ': ucal', 'b'))
for oid, ii in zip(cat.objid[J], I):
print('Objid', oid)
spec = S[ii]
k = (brickname, oid)
v = lightcurves[k]
# Cut bad CCDs
v.cut(np.array([e not in [230151, 230152, 230153] for e in v.expnum]))
plt.clf()
print('obj', k, 'has', len(v), 'measurements')
T = v
for fluxtag, fluxivtag, fluxname, plottag in fluxtags:
plt.clf()
filts = np.unique(T.filter)
for i, f in enumerate(filts):
from tractor.brightness import NanoMaggies
plt.subplot(len(filts), 1, i + 1)
fluxes = np.hstack(
[T.get(ft[0])[T.filter == f] for ft in fluxtags])
fluxes = fluxes[np.isfinite(fluxes)]
mn, mx = np.percentile(fluxes, [5, 95])
print('Flux percentiles for filter', f, ':', mn, mx)
# note swap
mn, mx = NanoMaggies.nanomaggiesToMag(
mx), NanoMaggies.nanomaggiesToMag(mn)
print('-> mags', mn, mx)
cut = (T.filter == f) * (T.flux_ivar > 0)
if ucal:
cut *= np.isfinite(T.uflux)
I = np.flatnonzero(cut)
print(' ', len(I), 'in', f, 'band')
I = I[np.argsort(T.mjd[I])]
mediv = np.median(T.flux_ivar[I])
# cut really noisy ones
I = I[T.flux_ivar[I] > 0.25 * mediv]
#plt.plot(T.mjd[I], T.flux[I], '.-', color=dict(g='g',r='r',z='m')[f])
# plt.errorbar(T.mjd[I], T.flux[I], yerr=1/np.sqrt(T.fluxiv[I]),
# fmt='.-', color=dict(g='g',r='r',z='m')[f])
#plt.errorbar(T.mjd[I], T.flux[I], yerr=1/np.sqrt(T.fluxiv[I]),
# fmt='.', color=dict(g='g',r='r',z='m')[f])
# if ucal:
# mag,dmag = NanoMaggies.fluxErrorsToMagErrors(T.flux[I], T.flux_ivar[I])
# else:
# mag,dmag = NanoMaggies.fluxErrorsToMagErrors(T.uflux[I], T.uflux_ivar[I])
mag, dmag = NanoMaggies.fluxErrorsToMagErrors(
T.get(fluxtag)[I],
T.get(fluxivtag)[I])
plt.errorbar(T.mjd[I],
mag,
yerr=dmag,
fmt='.',
color=dict(g='g', r='r', z='m')[f])
#yl,yh = plt.ylim()
#plt.ylim(yh,yl)
plt.ylim(mx, mn)
plt.ylabel(f)
if i + 1 < len(filts):
plt.xticks([])
#plt.yscale('symlog')
outfn = 'cutout_%.4f_%.4f.jpg' % (spec.ra, spec.dec)
if not os.path.exists(outfn):
url = 'http://legacysurvey.org/viewer/jpeg-cutout/?ra=%.4f&dec=%.4f&zoom=14&layer=sdssco&size=128' % (
spec.ra, spec.dec)
cmd = 'wget -O %s "%s"' % (outfn, url)
print(cmd)
os.system(cmd)
pix = plt.imread(outfn)
h, w, d = pix.shape
fig = plt.gcf()
#print('fig bbox:', fig.bbox)
#print('xmax, ymax', fig.bbox.xmax, fig.bbox.ymax)
#plt.figimage(pix, 0, fig.bbox.ymax - h, zorder=10)
#plt.figimage(pix, 0, fig.bbox.ymax, zorder=10)
#plt.figimage(pix, fig.bbox.xmax - w, fig.bbox.ymax, zorder=10)
plt.figimage(pix,
fig.bbox.xmax - (w + 2),
fig.bbox.ymax - (h + 2),
zorder=10)
plt.suptitle('SDSS spectro object: %s at (%.4f, %.4f)%s' %
(spec.label.strip(), spec.ra, spec.dec, fluxname))
plt.savefig('forced-%s%i-%s.png' % (tag, n, plottag))
ok, x, y = movie_wcs.radec2pixelxy(spec.ra, spec.dec)
x = int(np.round(x - 1))
y = int(np.round(y - 1))
sz = 32
plt.clf()
plt.subplots_adjust(hspace=0, wspace=0)
k = 1
for i, img in enumerate(movie_jpegs):
stamp = img[y - sz:y + sz + 1, x - sz:x + sz + 1]
plt.subplot(5, 6, k)
plt.imshow(stamp, interpolation='nearest', origin='lower')
plt.xticks([])
plt.yticks([])
k += 1
plt.suptitle('SDSS spectro object: %s at (%.4f, %.4f): DES images' %
(spec.label.strip(), spec.ra, spec.dec))
plt.savefig('forced-%s%i-c.png' % (tag, n))
n += 1
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description='DECaLS simulations.')
parser.add_argument('-b', '--brick', type=str, default='2428p117', metavar='',
help='process this brick (required input)')
parser.add_argument('-o', '--objtype', type=str, default='ELG', metavar='',
help='object type (STAR, ELG, LRG, BGS)')
parser.add_argument('-v', '--verbose', action='store_true',
help='toggle on verbose output')
args = parser.parse_args()
if args.brick is None:
parser.print_help()
sys.exit(1)
# Set the debugging level
if args.verbose:
lvl = logging.DEBUG
else:
lvl = logging.INFO
logging.basicConfig(format='%(message)s',level=lvl,stream=sys.stdout)
log = logging.getLogger('__name__')
brickname = args.brick
objtype = args.objtype.upper()
lobjtype = objtype.lower()
log.info('Analyzing objtype {} on brick {}'.format(objtype,brickname))
if 'DECALS_SIM_DIR' in os.environ:
decals_sim_dir = os.getenv('DECALS_SIM_DIR')
else:
decals_sim_dir = '.'
# Plotting preferences
sns.set(style='white',font_scale=1.6,palette='dark')#,font='fantasy')
col = sns.color_palette('dark')
# Read the meta-catalog.
metafile = os.path.join(decals_sim_dir,brickname,'metacat-'+brickname+'-'+lobjtype+'.fits')
log.info('Reading {}'.format(metafile))
meta = fits.getdata(metafile,1)
# We need this for our histograms below
magbinsz = 0.2
rminmax = np.array(meta['rmag_range'][0],meta['rmag_range'][1])
nmagbin = long((rminmax[1]-rminmax[0])/magbinsz)
# Work in chunks.
nchunk = meta['nchunk']
for ichunk in range(nchunk):
log.info('Working on chunk {:02d}/{:02d}'.format(ichunk+1,nchunk))
chunksuffix = '{:02d}'.format(ichunk)
# Read the simulated object catalog
simcatfile = os.path.join(decals_sim_dir,brickname,'simcat-'+brickname+'-'+
lobjtype+'-'+chunksuffix+'.fits')
#log.info('Reading {}'.format(simcatfile))
simcat = fits.getdata(simcatfile, 1)
# Read and match to the Tractor catalog
tractorfile = os.path.join(decals_sim_dir,brickname,'tractor-'+brickname+'-'+
lobjtype+'-'+chunksuffix+'.fits')
#log.info('Reading {}'.format(tractorfile))
tractor = fits.getdata(tractorfile, 1)
m1, m2, d12 = match_radec(tractor['ra'],tractor['dec'],
simcat['ra'],simcat['dec'],1.0/3600.0)
missing = np.delete(np.arange(len(simcat)),m2,axis=0)
good = np.where((np.abs(tractor['decam_flux'][m1,2]/simcat['rflux'][m2]-1)<0.3)*1)
# Flux residuals vs r-band magnitude
rmag = simcat['r'][m2]
gflux_sim = simcat['gflux'][m2]
rflux_sim = simcat['rflux'][m2]
zflux_sim = simcat['zflux'][m2]
gflux_tra = tractor['decam_flux'][m1,1]
rflux_tra = tractor['decam_flux'][m1,2]
zflux_tra = tractor['decam_flux'][m1,4]
fig, ax = plt.subplots(3, sharex=True, figsize=(6,8))
ax[0].scatter(rmag,gflux_sim/gflux_tra-1,color=col[0],s=10)
ax[1].scatter(rmag,rflux_sim/rflux_tra-1,color=col[1],s=10)
ax[2].scatter(rmag,zflux_sim/zflux_tra-1,color=col[2],s=10)
[thisax.set_ylim(-0.7,0.7) for thisax in ax]
[thisax.set_xlim(rminmax+[-0.1,0.0]) for thisax in ax]
[thisax.axhline(y=0.0,lw=2,ls='solid',color='gray') for thisax in ax]
for ix, thisband in enumerate(['g','r','z']):
ax[ix].text(0.05,0.05,thisband,horizontalalignment='left',
verticalalignment='bottom',transform=ax[ix].transAxes,
fontsize=16)
ax[1].set_ylabel('Input Flux / Tractor Flux - 1')
ax[2].set_xlabel('Input r magnitude (AB mag)')
fig.subplots_adjust(left=0.18,hspace=0.1)
qafile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+
lobjtype+'-flux.png')
log.info('Writing {}'.format(qafile))
plt.savefig(qafile)
# Color residuals
gr_tra = -2.5*np.log10(gflux_tra/rflux_tra)
rz_tra = -2.5*np.log10(rflux_tra/zflux_tra)
gr_sim = -2.5*np.log10(gflux_sim/rflux_sim)
rz_sim = -2.5*np.log10(rflux_sim/zflux_sim)
fig, ax = plt.subplots(2,sharex=True,figsize=(6,8))
ax[0].scatter(rmag,gr_tra-gr_sim,color=col[0],s=10)
ax[1].scatter(rmag,rz_tra-rz_sim,color=col[1],s=10)
[thisax.set_ylim(-0.7,0.7) for thisax in ax]
[thisax.set_xlim(rminmax+[-0.1,0.0]) for thisax in ax]
[thisax.axhline(y=0.0,lw=2,ls='solid',color='gray') for thisax in ax]
ax[0].set_ylabel('$\Delta$(g - r) (Tractor minus Input)')
ax[1].set_ylabel('$\Delta$(r - z) (Tractor minus Input)')
ax[1].set_xlabel('Input r magnitude (AB mag)')
fig.subplots_adjust(left=0.18,hspace=0.1)
qafile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+
lobjtype+'-color.png')
log.info('Writing {}'.format(qafile))
plt.savefig(qafile)
sys.exit(1)
# Get cutouts of the missing sources
imfile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+lobjtype+
'-image-'+chunksuffix+'.jpg')
hw = 30 # half-width [pixels]
ncols = 5
nrows = 5
nthumb = ncols*nrows
dims = (ncols*hw*2,nrows*hw*2)
mosaic = Image.new('RGB',dims)
miss = missing[np.argsort(simcat['r'][missing])]
print(simcat['r'][miss])
xpos, ypos = np.meshgrid(np.arange(0,dims[0],hw*2,dtype='int'),
np.arange(0,dims[1],hw*2,dtype='int'))
im = Image.open(imfile)
sz = im.size
iobj = 0
for ic in range(ncols):
for ir in range(nrows):
mm = miss[iobj]
xx = int(simcat['X'][mm])
yy = int(sz[1]-simcat['Y'][mm])
crop = (xx-hw,yy-hw,xx+hw,yy+hw)
box = (xpos[ir,ic],ypos[ir,ic])
thumb = im.crop(crop)
mosaic.paste(thumb,box)
iobj = iobj+1
# Add a border
draw = ImageDraw.Draw(mosaic)
for ic in range(ncols):
for ir in range(nrows):
draw.rectangle([(xpos[ir,ic],ypos[ir,ic]),
(xpos[ir,ic]+hw*2,ypos[ir,ic]+hw*2)])
qafile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+lobjtype+'-missing.png')
log.info('Writing {}'.format(qafile))
mosaic.save(qafile)
# Modify the coadd image and residual files so the simulated sources
# are labeled.
rad = 15
imfile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+lobjtype+
'-image-'+chunksuffix+'.jpg')
imfile = [imfile,imfile.replace('-image','-resid')]
for ifile in imfile:
im = Image.open(ifile)
sz = im.size
draw = ImageDraw.Draw(im)
[draw.ellipse((cat['X']-rad, sz[1]-cat['Y']-rad,cat['X']+rad,
sz[1]-cat['Y']+rad)) for cat in simcat]
im.save(ifile)
# Fraction of matching sources
rmaghist, magbins = np.histogram(simcat['r'],bins=nmagbin,range=rminmax)
cmagbins = (magbins[:-1] + magbins[1:]) / 2.0
ymatch, binsmatch = np.histogram(simcat['r'][m2],bins=nmagbin,range=rminmax)
ymatchgood, binsgood = np.histogram(simcat['r'][m2[good]],bins=nmagbin,range=rminmax)
fig, ax = plt.subplots(1,figsize=(8,6))
ax.step(cmagbins,1.0*ymatch/rmaghist,lw=3,alpha=0.5,label='All objects')
ax.step(cmagbins,1.0*ymatchgood/rmaghist,lw=3,ls='dashed',label='|$\Delta$m|<0.3')
ax.axhline(y=1.0,lw=2,ls='dashed',color='gray')
ax.set_xlabel('Input r magnitude (AB mag)')
ax.set_ylabel('Fraction of Matching '+objtype+'s')
ax.set_ylim([0.0,1.1])
ax.legend(loc='lower left')
fig.subplots_adjust(bottom=0.15)
qafile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+lobjtype+'-frac.png')
log.info('Writing {}'.format(qafile))
plt.savefig(qafile)
# Distribution of object types
fig = plt.figure(figsize=(8,6))
ax = fig.gca()
rmaghist, magbins = np.histogram(simcat['r'][m2],bins=nmagbin,range=rminmax)
cmagbins = (magbins[:-1] + magbins[1:]) / 2.0
tractortype = tractor['TYPE'][m1].strip()
for otype in ['PSF','EXP','DEV','COMP']:
these = np.where(tractortype==otype)[0]
if len(these)>0:
yobj, binsobj = np.histogram(simcat['r'][m2[these]],bins=nmagbin,range=rminmax)
#plt.step(cmagbins,1.0*yobj,lw=3,alpha=0.5,label=otype)
plt.step(cmagbins,1.0*yobj/rmaghist,lw=3,alpha=0.5,label=otype)
plt.axhline(y=1.0,lw=2,ls='dashed',color='gray')
plt.xlabel('Input r magnitude (AB mag)')
#plt.ylabel('Number of Objects')
plt.ylabel('Fraction of '+objtype+'s classified')
plt.ylim([0.0,1.1])
plt.legend(loc='center left',bbox_to_anchor=(0.08,0.5))
fig.subplots_adjust(bottom=0.15)
qafile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+lobjtype+'-type.png')
log.info('Writing {}'.format(qafile))
plt.savefig(qafile)
# Morphology plots
if objtype=='ELGo':
fig = plt.figure(figsize=(8,4))
plt.subplot(1,3,1)
plt.plot(rmag,deltam,'s',markersize=3)
plt.axhline(y=0.0,lw=2,ls='solid',color='gray')
plt.xlim(rminmax)
plt.xlabel('r (AB mag)')
plt.subplot(1,3,2)
plt.plot(simcat['R50_1'][m2],deltam,'s',markersize=3)
plt.axhline(y=0.0,lw=2,ls='solid',color='gray')
plt.xlabel('$r_{50}$ (arcsec)')
plt.subplot(1,3,3)
plt.plot(simcat['BA_1'][m2],deltam,'s',markersize=3)
plt.axhline(y=0.0,lw=2,ls='solid',color='gray')
plt.xlabel('b/a')
plt.xlim([0.2,1.0])
fig.subplots_adjust(bottom=0.18)
qafile = os.path.join(decals_sim_dir,'qa-'+brickname+'-'+
lobjtype+'-morph.png')
log.info('Writing {}'.format(qafile))
plt.savefig(qafile)
def qsocuts(SW):
in1 = ( ((SW.gpsf - SW.ipsf) < 1.5) *
(SW.optpsf > 17.) *
(SW.optpsf < 22.) *
((SW.optmod - SW.wise) > ((SW.gpsf - SW.ipsf) + 3)) *
np.logical_or(SW.ispsf, (SW.optpsf - SW.optmod) < 0.1) )
I = np.flatnonzero(in1)
print('Selected', len(I))
in2 = in1 * (SW.w1mag < 25.) * (SW.w2mag < 25.)
I2 = np.flatnonzero(in2)
print('With w1,w2 < 25:', len(I2))
SW.w1rchi2 = SW.w1_prochi2 / SW.w1_pronpix
in3 = in2 * (SW.w1rchi2 < 10.)
I3 = np.flatnonzero(in3)
print('And chi2/pix < 10:', len(I3))
I = I2
# Check against WISE catalog
#wfn = 'w3-wise.fits'
#WC = fits_table(wfn)
# SDSS matched to WISE catalog
swfn = 'eboss-w3-wise-cat-dr9.fits'
#SWC = fits_table(swfn)
S = fits_table('objs-eboss-w3-dr9.fits')
print('Read', len(S), 'SDSS objects')
wfn = 'w3-wise.fits'
WC = fits_table(wfn)
print('Read', len(WC), 'WISE catalog objects')
R = 4./3600.
I,J,d = match_radec(S.ra, S.dec, WC.ra, WC.dec, R, nearest=True)
print(len(I), 'matches of SDSS to WISE catalog')
SWC = S[I]
for k in WC.columns():
if k in ['ra','dec']:
outkey = k+'_wise'
else:
outkey = k
X = WC.get(k)
SWC.set(outkey, X[J])
SWC.writeto(swfn)
SWC.gpsf = fluxtomag(SWC.psfflux[:,1])
SWC.rpsf = fluxtomag(SWC.psfflux[:,2])
SWC.ipsf = fluxtomag(SWC.psfflux[:,3])
SWC.ispsf = (SWC.objc_type == 6)
SWC.isgal = (SWC.objc_type == 3)
SWC.optpsf = fluxtomag((SWC.psfflux[:,1] * 0.8 +
SWC.psfflux[:,2] * 0.6 +
SWC.psfflux[:,3] * 1.0) / 2.4)
SWC.optmod = fluxtomag((SWC.modelflux[:,1] * 0.8 +
SWC.modelflux[:,2] * 0.6 +
SWC.modelflux[:,3] * 1.0) / 2.4)
SWC.wise = fluxtomag((SWC.w1mpro * 1.0 +
SWC.w2mpro * 0.5) / 1.5)
in1 = ( ((SWC.gpsf - SWC.ipsf) < 1.5) *
(SWC.optpsf > 17.) *
(SWC.optpsf < 22.) *
((SWC.optpsf - SWC.wise) > ((SWC.gpsf - SWC.ipsf) + 3)) *
np.logical_or(SWC.ispsf, (SWC.optpsf - SWC.optmod) < 0.1) )
I = np.flatnonzero(in1)
print('Selected', len(I), 'from WISE catalog')
sys.exit(0)
worstI = I[np.argsort(-SW.w1rchi2[I])]
print('Worst:')
mp = multiproc(1)
for wi,i in enumerate(worstI):
print(' %8.3f, %8.3f, chi2/npix %g' % (SW.ra[i], SW.dec[i], SW.w1rchi2[i]))
ra, dec = SW.ra[i], SW.dec[i]
ri = int((ra - r0) / (rr[1]-rr[0]))
di = int((dec - d0) / (dd[1]-dd[0]))
print(' ri %i, di %i' % (ri,di))
if SW.w1rchi2[i] > 25:
continue
opt = myopts()
wbasefn = 'eboss-w3-worst%04i-r%02i-d%02i' % (wi, ri,di)
opt.picklepat = '%s-stage%%0i.pickle' % wbasefn
opt.ps = wbasefn
opt.minflux = None
opt.bandnum = 1
opt.osources = None
opt.sources = 'objs-eboss-w3-dr9.fits'
opt.ptsrc = False
opt.pixpsf = False
# opt.minsb = 0.05
opt.minsb = 0.005
opt.write = True
opt.force = [205]
opt.ri = ri
opt.di = di
import wise3
import tractor
pcat = []
pcat.append(tractor.PointSource(RaDecPos(ra, dec), None))
R = wise3.runtostage(205, opt, mp, rr[ri],rr[ri+1],dd[di],dd[di+1],
ttsuf='chi2/npix %g' % SW.w1rchi2[i],
pcat=pcat, addSky=True)
### Try to use the stored solved values -- actually doesn't make things
### much faster.
# R = wise3.runtostage(103, opt, mp, rr[ri],rr[ri+1],dd[di],dd[di+1],
# ttsuf='chi2/npix %g' % SW.w1rchi2[i],
# pcat=pcat)
# t = R['tractor']
# T = fits_table('ebossw3-v4-r%02i-d%02i-w1.fits' % (ri, di))
# assert(len(t.catalog) == len(T))
# for i,src in enumerate(t.catalog):
# print('Source', src)
# assert(src.getPosition().ra == T.ra [i])
# assert(src.getPosition().dec == T.dec[i])
# t.catalog.freezeParamsRecursive('*')
# t.catalog.thawPathsTo('w1')
# assert(len(t.catalog.getParams()) == len(T))
# t.catalog.setParams(T.w1)
# R2 = wise3.stage204(opt=opt, mp=mp, ri=opt.ri, di=opt.di, **R)
# R2['ims1'] = R2['ims0']
# ps = PlotSequence(wbasefn)
# R3 = wise3.stage205(opt=opt, mp=mp, ri=opt.ri, di=opt.di, ps=ps, **R2)
plt.clf()
plt.hist(SW.w1mag, 100, range=(10,30), histtype='step', color='b', log=True)
plt.hist(SW.w1mag[I], 100, range=(10,30), histtype='step', color='r', log=True)
plt.xlabel('W1 mag')
ylo,yhi = plt.ylim()
plt.ylim(0.3, yhi)
ps.savefig()
plt.clf()
plt.hist(SW.w2mag, 100, range=(10,30), histtype='step', color='b', log=True)
plt.hist(SW.w2mag[I], 100, range=(10,30), histtype='step', color='r', log=True)
plt.xlabel('W2 mag')
ylo,yhi = plt.ylim()
plt.ylim(0.3, yhi)
ps.savefig()
plt.clf()
plt.hist(np.log10(SW.w1_prochi2 / SW.w1_pronpix), 100, range=(0,3),
log=True, histtype='step', color='b')
plt.hist(np.log10(SW.w1_prochi2[I] / SW.w1_pronpix[I]), 100, range=(0,3),
log=True, histtype='step', color='r')
plt.xlabel('log chi2/npix')
ylo,yhi = plt.ylim()
plt.ylim(0.3, yhi)
ps.savefig()
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description='DECaLS simulations.')
parser.add_argument('-b', '--brick', type=str, default='2428p117', metavar='',
help='process this brick (required input)')
parser.add_argument('-o', '--objtype', type=str, choices=['STAR', 'ELG', 'LRG', 'BGS'], default='STAR', metavar='',
help='object type (STAR, ELG, LRG, BGS)')
parser.add_argument('-out', '--output_dir', type=str, default=None, metavar='',
help='relative path to output directory')
parser.add_argument('-extra', '--extra_plots', action='store_true',
help='make missing, annotated, coadded plots, dont if many chunks')
parser.add_argument('-v', '--verbose', action='store_true',
help='toggle on verbose output')
args = parser.parse_args()
if args.brick is None:
parser.print_help()
sys.exit(1)
if args.extra_plots: extra_plots= True
else: extra_plots= False
print('extra_plots=',extra_plots)
# Set the debugging level
if args.verbose:
lvl = logging.DEBUG
else:
lvl = logging.INFO
logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout)
log = logging.getLogger('__name__')
brickname = args.brick
objtype = args.objtype.upper()
lobjtype = objtype.lower()
log.info('Analyzing objtype {} on brick {}'.format(objtype, brickname))
if 'DECALS_SIM_DIR' in os.environ:
decals_sim_dir = os.getenv('DECALS_SIM_DIR')
else:
decals_sim_dir = '.'
input_dir= os.path.join(decals_sim_dir,brickname,lobjtype)
if args.output_dir is None: output_dir= os.path.join(decals_sim_dir,brickname,'qaplots_'+lobjtype)
else: output_dir= args.output_dir
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# Plotting preferences
#sns.set(style='white',font_scale=1.6,palette='dark')#,font='fantasy')
#col = sns.color_palette('dark')
col = ['b', 'k', 'c', 'm', 'y', 0.8]
# Read metadata catalog.
metafile = os.path.join(input_dir, 'metacat-{}-{}.fits'.format(brickname, lobjtype))
log.info('Reading {}'.format(metafile))
meta = fits.getdata(metafile, 1)
# We need this for our histograms below
magbinsz = 0.2
rminmax = np.squeeze(meta['RMAG_RANGE'])
nmagbin = long((rminmax[1]-rminmax[0])/magbinsz)
# Work in chunks.
allsimcat = []
bigsimcat = []
bigsimcat_missed = []
bigtractor = []
chunk_dirs= glob.glob(os.path.join(input_dir,'0*'))
nchunk= len(chunk_dirs)
if nchunk == 0: raise ValueError
# Loop through chunk dirs 000,001,...,999
for ichunk,cdir in enumerate([chunk_dirs[0]]):
chunksuffix = os.path.basename(cdir) #'{:02d}'.format(ichunk)
log.info('Working on chunk {:02d}/{:02d}'.format(ichunk+1, nchunk))
# Read the simulated object catalog
simcatfile = os.path.join(cdir, 'simcat-{}-{}-{:02d}.fits'.format(brickname, lobjtype, int(chunksuffix)))
log.info('Reading {}'.format(simcatfile))
simcat = Table(fits.getdata(simcatfile, 1))
# Read Tractor catalog
tractorfile = os.path.join(cdir, 'tractor-{}-{}-{:02d}.fits'.format(brickname, lobjtype, int(chunksuffix)))
log.info('Reading {}'.format(tractorfile))
tractor = Table(fits.getdata(tractorfile, 1))
# Match
m1, m2, d12 = match_radec(tractor['ra'].copy(), tractor['dec'].copy(),
simcat['RA'].copy(), simcat['DEC'].copy(), 1.0/3600.0)
#m1, m2, d12 = matching.johan_tree(tractor['ra'].copy(), tractor['dec'].copy(),\
# simcat['RA'].copy(), simcat['DEC'].copy(), dsmax=1.0/3600.0)
print('matched %d/%d' % (len(m2),len(simcat['RA'])))
missing = np.delete(np.arange(len(simcat)), m2, axis=0)
log.info('Missing {}/{} sources'.format(len(missing), len(simcat)))
#good = np.where((np.abs(tractor['decam_flux'][m1,2]/simcat['rflux'][m2]-1)<0.3)*1)
# Build matching catalogs for the plots below.
if len(bigsimcat) == 0:
bigsimcat = simcat[m2]
bigtractor = tractor[m1]
bigsimcat_missing = simcat[missing]
else:
bigsimcat = vstack((bigsimcat, simcat[m2]))
bigtractor = vstack((bigtractor, tractor[m1]))
bigsimcat_missing = vstack((bigsimcat_missing, simcat[missing]))
if len(allsimcat) == 0:
allsimcat = simcat
else:
allsimcat = vstack((allsimcat, simcat))
# Get cutouts of the bright matched sources with small/large delta mag
if extra_plots:
for img_name in ['image','resid','simscoadd']:
# Indices of large and small dmag
junk,i_large_dmag,i_small_dmag= bright_dmag_cut(simcat[m2],tractor[m1])
# Large dmag cutouts
qafile = os.path.join(output_dir, 'qa-{}-{}-{}-bright-large-dmag-{:02d}.png'.format(\
brickname, lobjtype, img_name, int(chunksuffix)))
plot_cutouts_by_index(simcat,i_large_dmag, brickname,lobjtype,chunksuffix, \
indir=cdir,img_name=img_name,qafile=qafile)
log.info('Wrote {}'.format(qafile))
# Small dmag cutouts
qafile = os.path.join(output_dir, 'qa-{}-{}-{}-bright-small-dmag-{:02d}.png'.format(\
brickname, lobjtype, img_name, int(chunksuffix)))
plot_cutouts_by_index(simcat,i_small_dmag, brickname,lobjtype,chunksuffix, \
indir=cdir,img_name=img_name,qafile=qafile)
log.info('Wrote {}'.format(qafile))
# Get cutouts of the missing sources in each chunk (if any)
if len(missing) > 0 and extra_plots:
for img_name in ['image']: #,'simscoadd']:
qafile = os.path.join(output_dir, 'qa-{}-{}-{}-missing-{:02d}.png'.format(\
brickname, lobjtype, img_name, int(chunksuffix)))
miss = missing[np.argsort(simcat['R'][missing])]
plot_cutouts_by_index(simcat,miss, brickname,lobjtype,chunksuffix, \
indir=cdir,img_name=img_name,qafile=qafile)
log.info('Wrote {}'.format(qafile))
# Annotate the coadd image and residual files so the simulated sources
# are labeled.
if extra_plots:
for img_name in ('simscoadd','image', 'resid'):
qafile = os.path.join(output_dir, 'qa-{}-{}-{}-{:02d}-annot.png'.format(\
brickname, lobjtype,img_name, int(chunksuffix)))
plot_annotated_coadds(simcat, brickname, lobjtype, chunksuffix, \
indir=cdir,img_name=img_name,qafile=qafile)
log.info('Wrote {}'.format(qafile))
# now operate on concatenated catalogues from multiple chunks
# Grab flags
b_good,b_bad= basic_cut(bigtractor)
# mags and colors of ALL injected sources
plot_injected_mags(allsimcat, log, qafile=\
os.path.join(output_dir, 'qa-{}-{}-injected-mags.png'.format(brickname, lobjtype)))
# number of detected sources that are bad, good and number of undetected, binned by r mag
plot_good_bad_ugly(allsimcat,bigsimcat,bigsimcat_missing, nmagbin,rminmax, b_good,b_bad, log, qafile=\
os.path.join(output_dir, 'qa-{}-{}-N-good-bad-missed.png'.format(brickname, lobjtype)))
# Flux residuals vs input magnitude
plot_tractor_minus_answer(bigsimcat,bigtractor, b_good,rminmax, log, qafile=\
os.path.join(output_dir, 'qa-{}-{}-good-flux.png'.format(brickname, lobjtype)))
# chi plots: Flux residual / estimated Flux error
plot_chi(bigsimcat,bigtractor, b_good,rminmax, log, qafile=\
os.path.join(output_dir, 'qa-{}-{}-chi-good.png'.format(brickname, lobjtype)))
# Color residuals
plot_color_tractor_minus_answer(bigtractor,bigsimcat, rminmax, brickname,lobjtype, log, qafile =\
os.path.join(output_dir, 'qa-{}-{}-color.png'.format(brickname, lobjtype)))
# Fraction of recovered sources
plot_fraction_recovered(allsimcat,bigsimcat, nmagbin,rminmax, brickname, lobjtype, log, qafile =\
os.path.join(output_dir, 'qa-{}-{}-frac.png'.format(brickname, lobjtype)))
# S/N of recovered sources (S/N band vs. AB mag band)
plot_sn_recovered(allsimcat,bigsimcat,bigtractor, brickname, lobjtype, log, qafile =\
os.path.join(output_dir, 'qa-{}-{}-SN.png'.format(brickname, lobjtype)))
# Distribution of object types for matching sources.
plot_recovered_types(bigsimcat,bigtractor, nmagbin,rminmax, objtype,log, qafile =\
os.path.join(output_dir, 'qa-{}-{}-type.png'.format(brickname, lobjtype)))
# Confusion matrix for distribution of object types
# Basic cm, use slim=False
types= ['PSF ', 'SIMP', 'EXP ', 'DEV ', 'COMP']
cm,all_names= create_confusion_matrix(np.array(['PSF ']*bigtractor['ra'].data[b_good].shape[0]),
bigtractor['type'].data[b_good], \
types=types,slim=False)
qafile = os.path.join(output_dir, 'qa-{}-{}-{}-confusion.png'.format(brickname, lobjtype,'good'))
plot_confusion_matrix(cm,all_names,all_names, log,qafile)
# Now a stacked confusion matrix
# Compute a row for each r mag range and stack rows
make_stacked_cm(bigsimcat,bigtractor, b_good, log,qafile =\
os.path.join(output_dir, 'qa-{}-{}-good-confusion-stack.png'.format(brickname, lobjtype)))
'''
# plothist(W.ra, W.dec, 100, range=((r0,r1),(d0,d1)))
# ps.savefig()
#
# plt.clf()
# plothist(WC.ra, WC.dec, 100, range=((r0,r1),(d0,d1)))
# ps.savefig()
#xlo,xhi = 7,26
xlo,xhi = 10,25
R = 4.0
I,J,d = match_radec(W.ra, W.dec, WC.ra, WC.dec, R, nearest=True)
loghist(W.w1mag[I], WC.w1mpro[J], 200, range=((xlo,xhi),(xlo,xhi)))
plt.plot([xlo,xhi],[xlo,xhi], '-', color='1')
plt.xlabel('Tractor w1mag')
plt.ylabel('Catalog w1mpro')
plt.axis([xlo,xhi,xlo,xhi])
ps.savefig()
lo,hi = 0,0.25
loghist(W.w1magerr[I], WC.w1sigmpro[J], 200, range=((lo,hi),(lo,hi)))
plt.plot([lo,hi],[lo,hi], '-', color='1')
plt.xlabel('Tractor w1mag err')
plt.ylabel('Catalog w1sigmpro')
plt.axis([lo,hi,lo,hi])
def main():
ps = PlotSequence('shotgun')
decals = Decals()
C = fits_table('decals-ccds-annotated.fits')
print(len(C), 'CCDs')
C.cut(C.photometric)
C.cut(C.blacklist_ok)
print(len(C), 'photometric and not blacklisted')
# HACK
print('FIXME not cutting on DECALS')
#C.cut(C.tilepass > 0)
#print(len(C), 'taken by DECaLS')
targets = dict(g=24.0, r=23.4, z=22.5)
def ivtomag(iv, nsigma=5.):
return -2.5 * (np.log10(nsigma / np.sqrt(iv)) - 9)
def band_index(band):
allbands = 'ugrizY'
return allbands.index(band)
ccmap = dict(g='g', r='r', z='m')
ceil_exptime = dict(g=125., r=125., z=250.)
#plt.clf()
bands = 'grz'
for band in bands:
tmag = targets[band]
print()
print(band, 'band, target depth', tmag)
ccds = C[C.filter == band]
ccdarea = (2046*4094*(0.262/3600.)**2)
print(len(ccds), 'CCDs, total exptime', np.sum(ccds.exptime),
'(mean %.1f)' % np.mean(ccds.exptime), 'total area',
len(ccds)*ccdarea, 'sq.deg')
detsig1 = ccds.sig1 / ccds.galnorm_mean
totiv = np.sum(1. / detsig1**2)
# depth we would have if we had all exposure time in one CCD
# print('5-sigma galaxy depth if concentrated in one CCD:', ivtomag(totiv))
# # mean depth
# print('5-sigma galaxy depth if spread equally among', len(ccds), 'CCDs:', ivtomag(totiv / len(ccds)))
# print('vs median depth', np.median(ccds.galdepth))
# print('5-sigma galaxy depth if spread equally among %i/2' % (len(ccds)), 'CCDs:', ivtomag(totiv / (len(ccds)/2)))
# print('5-sigma galaxy depth if spread equally among %i/3' % (len(ccds)), 'CCDs:', ivtomag(totiv / (len(ccds)/3)))
# spread over 6000 sq deg
sqdeg = 6000
avgiv = totiv * ccdarea / sqdeg
#print('5-sigma galaxy depth if spread over', sqdeg, 'sqdeg:', ivtomag(avgiv))
tflux = 10.**(tmag / -2.5 + 9)
tiv = 1. / (tflux / 5)**2
#print('Fraction of', sqdeg, 'sqdeg survey complete:', avgiv / tiv)
iband = band_index(band)
ext = ccds.decam_extinction[:,iband]
medext = np.median(ext)
print('With extinction (median %.2f mag):' % medext)
transmission = 10.**(-ext / 2.5)
detsig1 = ccds.sig1 / ccds.galnorm_mean / transmission
totiv = np.sum(1. / detsig1**2)
# depth we would have if we had all exposure time in one CCD
print('5-sigma galaxy depth if concentrated in one CCD: %.3f' % ivtomag(totiv))
# mean depth
print('5-sigma galaxy depth if spread equally among', len(ccds), 'CCDs: %.3f' % ivtomag(totiv / len(ccds)))
print('vs median depth: %.3f' % np.median(ccds.galdepth - ext))
print('5-sigma galaxy depth if spread equally among %i/2' % (len(ccds)), 'CCDs: %.3f' % ivtomag(totiv / (len(ccds)/2)))
print('5-sigma galaxy depth if spread equally among %i/3' % (len(ccds)), 'CCDs: %.3f' % ivtomag(totiv / (len(ccds)/3)))
# spread over 6000 sq deg
sqdeg = 6000
avgiv = totiv * ccdarea / sqdeg
print('5-sigma galaxy depth if spread over', sqdeg, 'sqdeg: %.3f' % ivtomag(avgiv))
print('Fraction of', sqdeg, 'sqdeg survey complete: %.3f' % (avgiv / tiv))
# plt.hist(ccds.exptime, range=(0,250), bins=50, histtype='step', color=ccmap[band])
# I = np.flatnonzero(ccds.exptime < (ceil_exptime[band] - 1.))
# ccds.cut(I)
# print('Cutting out exposures with ceil exposure time:', len(ccds))
#
# plt.hist(ccds.exptime, bins=25, histtype='step', color=ccmap[band],
# linestyle='dotted', linewidth=3, alpha=0.3)
#
# transmission = transmission[I]
# ext = ext[I]
#
# detsig1 = ccds.sig1 / ccds.galnorm_mean / transmission
# totiv = np.sum(1. / detsig1**2)
# # depth we would have if we had all exposure time in one CCD
# print('5-sigma galaxy depth if concentrated in one CCD:', ivtomag(totiv))
# # mean depth
# print('5-sigma galaxy depth if spread equally among', len(ccds), 'CCDs:', ivtomag(totiv / len(ccds)))
# print('vs median depth', np.median(ccds.galdepth - ext))
# print('5-sigma galaxy depth if spread equally among %i/2' % (len(ccds)), 'CCDs:', ivtomag(totiv / (len(ccds)/2)))
# print('5-sigma galaxy depth if spread equally among %i/3' % (len(ccds)), 'CCDs:', ivtomag(totiv / (len(ccds)/3)))
# # spread over 6000 sq deg
# sqdeg = 6000
# avgiv = totiv * ccdarea / sqdeg
# print('5-sigma galaxy depth if spread over', sqdeg, 'sqdeg:', ivtomag(avgiv))
# print('Fraction of', sqdeg, 'sqdeg survey complete:', avgiv / tiv)
# plt.xlabel('Exposure time (s)')
# ps.savefig()
print()
dra = 4094 / 2. / 3600 * 0.262
ddec = 2046 / 2. / 3600 * 0.262
ralo = max( 0, min(C.ra - dra / np.cos(np.deg2rad(C.dec))))
rahi = min(360, max(C.ra + dra / np.cos(np.deg2rad(C.dec))))
declo = max(-90, min(C.dec - ddec))
dechi = min( 90, max(C.dec + ddec))
# brick 0001m002
#ralo,rahi = 0., 0.25
#declo,dechi = -0.375, -0.125
#ralo, rahi = 0, 1
#declo, dechi = 0, 1
ralo, rahi = 0, 0.5
declo, dechi = 0, 0.5
print('RA,Dec range', (ralo, rahi), (declo, dechi))
N = 10000
nbatch = 1000
rr,dd = [],[]
ntotal = 0
while ntotal < N:
ru = np.random.uniform(size=nbatch)
d = np.random.uniform(low=declo, high=dechi, size=nbatch)
# Taper the accepted width in RA based on Dec
cosd = np.cos(np.deg2rad(d))
I = np.flatnonzero(ru < cosd)
if len(I) == 0:
continue
r = ralo + (rahi - ralo) * ru[I]/cosd[I]
d = d[I]
rr.append(r)
dd.append(d)
ntotal += len(r)
print('Kept', len(r), 'of', nbatch)
ra = np.hstack(rr)
dec = np.hstack(dd)
del rr
del dd
ra = ra [:N]
dec = dec[:N]
print('RA,Dec ranges of samples:', (ra.min(), ra.max()), (dec.min(), dec.max()))
# plt.clf()
# plt.plot(ra, dec, 'b.', alpha=0.1)
# ps.savefig()
# CCD size
margin = 10 * 0.262 / 3600.
# C.dra is in degrees on the sphere, not delta-RA
# dra = C.dra / np.cos(np.deg2rad(C.dec))
# ccds = C[(C.ra + dra + margin > ralo) *
# (C.ra - dra - margin < rahi) *
# (C.dec + C.ddec + margin > declo) *
# (C.dec - C.ddec - margin < dechi)]
I,J,d = match_radec(C.ra, C.dec,
(ralo+rahi)/2., (declo+dechi)/2.,
degrees_between(ralo,declo, rahi,dechi)/2. + 1.,
nearest=True)
ccds = C[I]
print(len(ccds), 'nearby')
assert(len(ccds))
print('RA range', ccds.ra.min(), ccds.ra.max())
radius = np.hypot(2046, 4096) / 2. * 0.262 / 3600 * 1.1
II = match_radec(ccds.ra, ccds.dec, ra, dec,
radius, indexlist=True)
#print('Matching:', II)
depthrange = [20,25]
depthbins = 100
depthbins2 = 500
galdepths = dict([(b, np.zeros(len(ra))) for b in bands])
iccds = []
for iccd,I in enumerate(II):
if I is None:
continue
ccd = ccds[iccd]
print('Matched to CCD %s: %i' % (ccd.expid, len(I)))
# Actually inside CCD RA,Dec box?
r = ra[I]
d = dec[I]
# print('degrees_between: ccd ra,dec', ccd.ra, ccd.dec)
# for ri,di in zip(r, degrees_between(r, ccd.dec + np.zeros_like(r),
# ccd.ra, ccd.dec)):
# print('ri: di', ri, di)
J = np.flatnonzero((degrees_between(r, ccd.dec+np.zeros_like(r),
ccd.ra, ccd.dec) < dra) *
(np.abs(d - ccd.dec) < ddec))
print('Actually inside CCD RA,Dec box:', len(J))
if len(J) == 0:
continue
I = np.array(I)[J]
# j = J[0]
# print('deg between', degrees_between(r[j], ccd.dec,
# ccd.ra, ccd.dec), 'vs', dra)
# print(' dec', d[j], 'dist', np.abs(d[j] - ccd.dec), 'vs', ddec)
iccds.append((iccd,I[0]))
band = ccd.filter
gd = galdepths[band]
print('Gal depth in', band, ':', ccd.galdepth)
# mag -> 5sig1 -> iv
cgd = 10.**((ccd.galdepth - 22.5) / -2.5)
cgd = 1. / cgd**2
gd[I] += cgd
plt.clf()
# plt.plot(ra, dec, 'k.', alpha=0.1)
# C1 = fits_table('decals-ccds-annotated.fits')
# ii,jj,dd = match_radec(C1.ra, C1.dec, (ralo+rahi)/2.,(declo+dechi)/2,0.5)
# C1.cut(ii)
# C1.ra -= (C1.ra > 270)*360
# for ccd in C1:
# r,dr = ccd.ra, ccd.dra / np.cos(np.deg2rad(ccd.dec))
# d,dd = ccd.dec, ccd.ddec
# sty = dict(color='k', alpha=0.1)
# if not (ccd.photometric and ccd.blacklist_ok):
# sty = dict(color='c', lw=3)
# plt.plot([r-dr, r+dr, r+dr, r-dr, r-dr], [d-dd,d-dd,d+dd,d+dd,d-dd],
# '-', **sty)
for iccd,i in iccds:
ccd = ccds[iccd]
r,dr = ccd.ra, dra / np.cos(np.deg2rad(ccd.dec))
# HACK
r = r + (r > 270)*-360.
d,dd = ccd.dec, ddec
plt.plot([r-dr, r+dr, r+dr, r-dr, r-dr], [d-dd,d-dd,d+dd,d+dd,d-dd],
'-', color=ccmap[ccd.filter], alpha=0.5)
plt.plot(ra[i], dec[i], 'k.')
plt.plot([ralo,ralo,rahi,rahi,ralo],[declo,dechi,dechi,declo,declo],
'k-', lw=2)
#plt.axis([-0.1, 0.6, -0.1, 0.6])
plt.axis([-0.5, 1., -0.5, 1.])
plt.xlabel('RA')
plt.ylabel('Dec')
ps.savefig()
for band in bands:
gd = galdepths[band]
# iv -> 5sig1 -> mag
gd = 1. / np.sqrt(gd)
gd = -2.5 * (np.log10(gd) - 9.)
plt.clf()
plt.hist(gd, range=depthrange, bins=depthbins, histtype='step',
color=ccmap[band])
plt.xlabel('5-sigma Galaxy depth (mag)')
plt.title('%s band' % band)
ps.savefig()
print('Reading extinction values for sample points...')
sfd = SFDMap()
filts = ['%s %s' % ('DES', f) for f in bands]
ebv,extinction = sfd.extinction(filts, ra, dec, get_ebv=True)
print('Extinction:', extinction.shape)
B = decals.get_bricks_readonly()
I = decals.bricks_touching_radec_box(None, ralo, rahi, declo, dechi)
B.cut(I)
print(len(B), 'bricks touching RA,Dec box')
depth_hists = {}
depth_hists_2 = {}
for brick in B:
print('Brick', brick.brickname)
I = np.flatnonzero((ra > brick.ra1 ) * (ra < brick.ra2) *
(dec > brick.dec1) * (dec < brick.dec2))
print(len(I), 'samples in brick')
if len(I) == 0:
continue
wcs = wcs_for_brick(brick)
ok,x,y = wcs.radec2pixelxy(ra[I], dec[I])
x = np.round(x - 1).astype(int)
y = np.round(y - 1).astype(int)
for iband,band in enumerate(bands):
fn = decals.find_file('nexp', brick=brick.brickname, band=band)
print('Reading', fn)
if not os.path.exists(fn):
print('Missing:', fn)
continue
nexp = fitsio.read(fn)
nexp = nexp[y, x]
ext = extinction[I, iband]
fn = decals.find_file('galdepth', brick=brick.brickname, band=band)
print('Reading', fn)
galdepth = fitsio.read(fn)
galdepth = galdepth[y, x]
# iv -> mag
galdepth = -2.5 * (np.log10(5. / np.sqrt(galdepth)) - 9)
# extinction-corrected
galdepth -= ext
un = np.unique(nexp)
print('Numbers of exposures:', un)
for ne in un:
if ne == 0:
continue
J = np.flatnonzero(nexp == ne)
gd = galdepth[J]
key = (ne, band)
H = depth_hists.get(key, 0)
h,e = np.histogram(gd, range=depthrange, bins=depthbins)
H = h + H
depth_hists[key] = H
H = depth_hists_2.get(key, 0)
h,e = np.histogram(gd, range=depthrange, bins=depthbins2)
H = h + H
depth_hists_2[key] = H
dlo,dhi = depthrange
left = dlo + np.arange(depthbins) * (dhi-dlo) / float(depthbins)
binwidth = left[1]-left[0]
# for k,H in depth_hists.items():
# plt.clf()
# plt.bar(left, H, width=binwidth)
# plt.xlabel('Galaxy depth (mag)')
# (ne,band) = k
# plt.title('%s band, %i exposures' % (band, ne))
# plt.xlim(dlo, dhi)
# fn = 'depth-%s-%i.png' % (band, ne)
# plt.savefig(fn)
# print('Wrote', fn)
rainbow = ['r', '#ffa000', 'y', 'g', 'b', 'm']
for band in bands:
plt.clf()
for ne in range(1,10):
key = (ne, band)
if not key in depth_hists:
continue
H = depth_hists[key]
print('hist length:', len(H))
plt.plot(np.vstack((left, left+binwidth)).T.ravel(), np.repeat(H, 2), '-',
color=rainbow[ne-1 % len(rainbow)], label='%i exp' % ne)
plt.title('%s band' % band)
plt.xlabel('Galaxy depth (mag)')
plt.legend(loc='upper left')
ps.savefig()
left2 = dlo + np.arange(depthbins2) * (dhi-dlo) / float(depthbins2)
binwidth2 = left2[1]-left2[0]
for band in bands:
hsum = 0
for ne in range(1,10):
key = (ne, band)
if not key in depth_hists_2:
continue
hsum += depth_hists_2[key]
print('%s band:' % band)
print('Total number of counts in histogram:', sum(hsum))
# [-1::-1] = reversed
hsum = np.cumsum(hsum[-1::-1])[-1::-1]
hsum = hsum * 100. / float(N)
plt.clf()
#plt.plot(left2+binwidth/2, hsum, 'k-')
plt.plot(left2, hsum, 'k-')
plt.xlabel('Galaxy depth (mag)')
plt.ylabel('Cumulative fraction (%)')
plt.title('%s band' % band)
# 90% to full depth
# 95% to full depth - 0.3 mag
# 98% to full depth - 0.6 mag
for y,x,c in [
(90, targets[band], 'r'),
(95, targets[band] - 0.3, 'g'),
(98, targets[band] - 0.6, 'b')]:
xf = (x - dlo) / (dhi - dlo)
yf = y / 100.
plt.axvline(x, ymax=yf, color=c, lw=3, alpha=0.3)
plt.axhline(y, xmax=xf, color=c, lw=3, alpha=0.3)
#mid = left2 + binwidth2/2
print('target : %.1f %% deeper than %.2f' % (y, x))
# ??? left or midpoint?
ibin = np.flatnonzero(left2 > x)[0]
pct = hsum[ibin]
status = ' '*20
if pct >= y:
status += '-> pass'
else:
status += '-> FAIL'
plt.axhline(pct, color=c, alpha=0.5) #xmax=xf,
print('achieved: %.1f %% %s' % (pct, status)) # deeper than %.2f' % (hsum[ibin], x))
ii = np.flatnonzero(hsum > y)
if len(ii):
ibin = ii[-1]
print(' %.1f %% deeper than %.2f' % (y, left2[ibin]))
else:
print(' Total coverage only %.1f %%' % max(hsum))
print()
plt.xlim(dlo, dhi)
plt.ylim(0., 100.)
ps.savefig()
def main():
sbfn = 'skybricks.fits'
SB = fits_table(sbfn)
Bnorth = fits_table(
'/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/north/survey-bricks-dr9-north.fits.gz'
)
Bsouth = fits_table(
'/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/survey-bricks-dr9-south.fits.gz'
)
Bnorth.cut(Bnorth.survey_primary)
Bsouth.cut(Bsouth.survey_primary)
#Bsouth.cut(Bsouth.dec > -30)
Bnorth.hemi = np.array(['north'] * len(Bnorth))
Bsouth.hemi = np.array(['south'] * len(Bsouth))
B = merge_tables([Bnorth, Bsouth])
# Rough cut the skybricks to those near bricks.
I, J, d = match_radec(SB.ra, SB.dec, B.ra, B.dec, 1., nearest=True)
SB.cut(I)
import argparse
parser = argparse.ArgumentParser()
#parser.add_argument('--brick', help='Sky brick name')
parser.add_argument('--minra',
type=float,
help='Cut to a minimum RA range of sky bricks')
parser.add_argument('--maxra',
type=float,
help='Cut to a maximum RA range of sky bricks')
parser.add_argument('--threads',
type=int,
help='Parallelize on this many cores')
opt = parser.parse_args()
if opt.minra:
SB.cut(SB.ra >= opt.minra)
if opt.maxra:
SB.cut(SB.ra <= opt.maxra)
version = get_git_version(os.getcwd())
print('Version string:', version)
# Find bricks near skybricks, as a rough cut.
# (magic 1. degree > hypot(skybrick radius, brick radiu) ~= 0.9)
Inear = match_radec(SB.ra, SB.dec, B.ra, B.dec, 1., indexlist=True)
args = []
k = 0
Isb = []
for isb, (sb, inear) in enumerate(zip(SB, Inear)):
if inear is None:
continue
args.append((k, sb, B[np.array(inear)], version))
k += 1
Isb.append(isb)
print(len(args), 'sky bricks')
SB.cut(np.array(Isb))
if opt.threads:
mp = multiproc(opt.threads)
else:
mp = multiproc()
exist = mp.map(run_one, args)
if opt.minra is None and opt.maxra is None:
exist = np.array(exist)
SB[exist].writeto('skybricks-exist.fits')
return
def forcedphot():
T1 = fits_table('cs82data/cas-primary-DR8.fits')
print(len(T1), 'primary')
T1.cut(T1.nchild == 0)
print(len(T1), 'children')
rl, rh = T1.ra.min(), T1.ra.max()
dl, dh = T1.dec.min(), T1.dec.max()
tims = []
# Coadd
basedir = os.path.join('cs82data', 'wise', 'level3')
basefn = os.path.join(basedir, '3342p000_ab41-w1')
tim = read_wise_coadd(basefn, radecroi=[rl, rh, dl, dh], nanomaggies=True)
tims.append(tim)
# Individuals
basedir = os.path.join('cs82data', 'wise', 'level1b')
for fn in ['04933b137-w1', '04937b137-w1', '04941b137-w1', '04945b137-w1', '04948a112-w1',
'04949b137-w1', '04952a112-w1', '04953b137-w1', '04956a112-w1', '04960a112-w1',
'04964a112-w1', '04968a112-w1', '05204a106-w1']:
basefn = os.path.join(basedir, fn)
tim = read_wise_image(
basefn, radecroi=[rl, rh, dl, dh], nanomaggies=True)
tims.append(tim)
# tractor.Image's setMask() does a binary dilation on bad pixels!
for tim in tims:
# tim.mask = np.zeros(tim.shape, dtype=bool)
tim.invvar = tim.origInvvar
tim.mask = np.zeros(tim.shape, dtype=bool)
print('tim:', tim)
ps = PlotSequence('forced')
plt.clf()
plt.plot(T1.ra, T1.dec, 'r.')
for tim in tims:
wcs = tim.getWcs()
H, W = tim.shape
rr, dd = [], []
for x, y in zip([1, 1, W, W, 1], [1, H, H, 1, 1]):
rd = wcs.pixelToPosition(x, y)
rr.append(rd.ra)
dd.append(rd.dec)
plt.plot(rr, dd, 'k-', alpha=0.5)
# setRadecAxes(rl,rh,dl,dh)
ps.savefig()
T2 = fits_table('wise-cut.fits')
T2.w1 = T2.w1mpro
R = 1. / 3600.
I, J, d = match_radec(T1.ra, T1.dec, T2.ra, T2.dec, R)
print(len(I), 'matches')
refband = 'r'
#bandnum = band_index('r')
Lstar = (T1.probpsf == 1) * 1.0
Lgal = (T1.probpsf == 0)
fracdev = T1.get('fracdev_%s' % refband)
Ldev = Lgal * fracdev
Lexp = Lgal * (1. - fracdev)
ndev, nexp, ncomp, nstar = 0, 0, 0, 0
cat = Catalog()
# for i,t in enumerate(T1):
jmatch = np.zeros(len(T1))
jmatch[:] = -1
jmatch[I] = J
for i in range(len(T1)):
j = jmatch[i]
if j >= 0:
# match source: grab WISE catalog mag
w1 = T2.w1[j]
else:
# unmatched: set it faint
w1 = 18.
bright = NanoMaggies(w1=NanoMaggies.magToNanomaggies(w1))
pos = RaDecPos(T1.ra[i], T1.dec[i])
if Lstar[i] > 0:
# Star
star = PointSource(pos, bright)
cat.append(star)
nstar += 1
continue
hasdev = (Ldev[i] > 0)
hasexp = (Lexp[i] > 0)
iscomp = (hasdev and hasexp)
if iscomp:
dbright = bright * Ldev[i]
ebright = bright * Lexp[i]
elif hasdev:
dbright = bright
elif hasexp:
ebright = bright
else:
assert(False)
if hasdev:
re = T1.get('devrad_%s' % refband)[i]
ab = T1.get('devab_%s' % refband)[i]
phi = T1.get('devphi_%s' % refband)[i]
dshape = GalaxyShape(re, ab, phi)
if hasexp:
re = T1.get('exprad_%s' % refband)[i]
ab = T1.get('expab_%s' % refband)[i]
phi = T1.get('expphi_%s' % refband)[i]
eshape = GalaxyShape(re, ab, phi)
if iscomp:
gal = CompositeGalaxy(pos, ebright, eshape, dbright, dshape)
ncomp += 1
elif hasdev:
gal = DevGalaxy(pos, dbright, dshape)
ndev += 1
elif hasexp:
gal = ExpGalaxy(pos, ebright, eshape)
nexp += 1
cat.append(gal)
print('Created', ndev, 'pure deV', nexp, 'pure exp and', end=' ')
print(ncomp, 'composite galaxies', end=' ')
print('and', nstar, 'stars')
tractor = Tractor(tims, cat)
for i, tim in enumerate(tims):
ima = dict(interpolation='nearest', origin='lower',
vmin=tim.zr[0], vmax=tim.zr[1])
mod = tractor.getModelImage(i)
plt.clf()
plt.imshow(mod, **ima)
plt.gray()
plt.title('model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(tim.getImage(), **ima)
plt.gray()
plt.title('data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(tim.getInvvar(), interpolation='nearest', origin='lower')
plt.gray()
plt.title('invvar')
ps.savefig()
# for tim in tims:
wcs = tim.getWcs()
H, W = tim.shape
poly = []
for r, d in zip([rl, rl, rh, rh, rl], [dl, dh, dh, dl, dl]):
x, y = wcs.positionToPixel(RaDecPos(r, d))
poly.append((x, y))
xx, yy = np.meshgrid(np.arange(W), np.arange(H))
xy = np.vstack((xx.flat, yy.flat)).T
grid = points_inside_poly(xy, poly)
grid = grid.reshape((H, W))
tim.setInvvar(tim.getInvvar() * grid)
# plt.clf()
# plt.imshow(grid, interpolation='nearest', origin='lower')
# plt.gray()
# ps.savefig()
plt.clf()
plt.imshow(tim.getInvvar(), interpolation='nearest', origin='lower')
plt.gray()
plt.title('invvar')
ps.savefig()
if i == 1:
plt.clf()
plt.imshow(tim.goodmask, interpolation='nearest', origin='lower')
plt.gray()
plt.title('goodmask')
ps.savefig()
miv = (1. / (tim.uncplane)**2)
for bit in range(-1, 32):
if bit >= 0:
miv[(tim.maskplane & (1 << bit)) != 0] = 0.
if bit == 31:
plt.clf()
plt.imshow(miv, interpolation='nearest', origin='lower')
plt.gray()
plt.title(
'invvar with mask bits up to %i blanked out' % bit)
ps.savefig()
def find_alignments(fns, wcsfns, gaia_fn, aff_fn, aligned_fn):
from astrometry.libkd.spherematch import tree_build_radec, trees_match
from astrometry.libkd.spherematch import match_radec
from astrometry.util.plotutils import plothist
from astrometry.util.util import Tan
import fitsio
from astrom_common import getwcsoutline
from singles import find_overlaps
if True:
WCS = []
for fn in wcsfns:
wcs = Tan(fn)
WCS.append(wcs)
names = [fn.replace('-bright.fits', '') for fn in fns]
outlines = [getwcsoutline(wcs) for wcs in WCS]
overlaps, areas = find_overlaps(outlines)
print('Reading tables...')
TT = [fits_table(fn) for fn in fns]
print('Building trees...')
kds = [tree_build_radec(T.ra, T.dec) for T in TT]
for T, name in zip(TT, names):
T.name = np.array([name] * len(T))
allra = np.hstack([T.ra for T in TT])
alldec = np.hstack([T.dec for T in TT])
minra = np.min(allra)
maxra = np.max(allra)
mindec = np.min(alldec)
maxdec = np.max(alldec)
print('RA,Dec range:', minra, maxra, mindec, maxdec)
plothist(allra, alldec)
plt.axis([maxra, minra, mindec, maxdec])
plt.xlabel('RA (deg)')
plt.ylabel('Dec (deg)')
plt.savefig('match-all.png')
Tref = fits_table(gaia_fn)
r_arcsec = 0.2
I, J, d = match_radec(Tref.ra, Tref.dec, allra, alldec, r_arcsec / 3600.)
dec = alldec[J]
cosdec = np.cos(np.deg2rad(dec))
dr = (Tref.ra[I] - allra[J]) * cosdec * 3600.
dd = (Tref.dec[I] - alldec[J]) * 3600.
plt.clf()
rr = (-r_arcsec * 1000, +r_arcsec * 1000)
plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr))
plt.xlabel('dRA (milli-arcsec)')
plt.ylabel('dDec (milli-arcsec)')
plt.savefig('match-all-ref-before.png')
# Initial matching of all stars
r_arcsec = 0.2
I, J, d = match_radec(allra,
alldec,
allra,
alldec,
r_arcsec / 3600.,
notself=True)
dec = alldec[I]
cosdec = np.cos(np.deg2rad(dec))
dr = (allra[I] - allra[J]) * cosdec * 3600.
dd = (alldec[I] - alldec[J]) * 3600.
plt.clf()
rr = (-r_arcsec * 1000, +r_arcsec * 1000)
plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr))
plt.xlabel('dRA (milli-arcsec)')
plt.ylabel('dDec (milli-arcsec)')
plt.savefig('match-all-before.png')
hulls = []
from scipy.spatial import ConvexHull
for T in TT:
hull = ConvexHull(np.vstack((T.ra, T.dec)).T)
ra = T.ra[hull.vertices]
ra = np.append(ra, ra[0])
dec = T.dec[hull.vertices]
dec = np.append(dec, dec[0])
hulls.append((ra, dec))
aligns = {}
#for i in []:
for i in range(len(kds)):
for j in range(i + 1, len(kds)):
print('Matching trees', i, 'and', j)
r_arcsec = 0.2
radius = np.deg2rad(r_arcsec / 3600)
I, J, d2 = trees_match(kds[i], kds[j], radius)
print(len(I), 'matches')
if len(I) == 0:
continue
Ti = TT[i]
Tj = TT[j]
dec = Ti[I].dec
cosdec = np.cos(np.deg2rad(dec))
dr = (Ti[I].ra - Tj[J].ra) * cosdec * 3600.
dd = (Ti[I].dec - Tj[J].dec) * 3600.
if False:
al = Alignment(Ti, Tj, searchradius=r_arcsec)
print('Aligning...')
if not al.shift():
print('Failed to find Alignment between fields')
continue
aligns[(i, j)] = al
plt.clf()
plotalignment(al)
plt.savefig('match-align-%02i-%02i.png' % (i, j))
plt.clf()
#plothist(np.append(Ti.ra, Tj.ra), np.append(Ti.dec, Tj.dec), docolorbar=False, doclf=False, dohot=False,
# imshowargs=dict(cmap=antigray))
plothist(Ti.ra[I], Ti.dec[I], docolorbar=False, doclf=False)
r, d = hulls[i]
plt.plot(r, d, 'r-')
r, d = hulls[j]
plt.plot(r, d, 'b-')
mra = Ti.ra[I]
mdec = Ti.dec[I]
mnra = np.min(mra)
mxra = np.max(mra)
mndec = np.min(mdec)
mxdec = np.max(mdec)
plt.plot([mnra, mnra, mxra, mxra, mnra],
[mndec, mxdec, mxdec, mndec, mndec], 'g-')
plt.axis([maxra, minra, mindec, maxdec])
plt.xlabel('RA (deg)')
plt.ylabel('Dec (deg)')
plt.savefig('match-radec-%02i-%02i.png' % (i, j))
plt.clf()
rr = (-r_arcsec, +r_arcsec)
plothist(dr, dd, nbins=100, range=(rr, rr))
plt.xlabel('dRA (arcsec)')
plt.ylabel('dDec (arcsec)')
plt.savefig('match-dradec-%02i-%02i.png' % (i, j))
#for roundi,(Nk,R) in enumerate(NkeepRads):
refrad = 0.15
targetrad = 0.005
ps = PlotSequence('shift')
from astrom_intra import intrabrickshift
from singles import plot_all_alignments
#Rads = [0.25, 0.1]
Rads = [0.2, 0.050, 0.020]
#Rads = [0.1]
affs = None
# this is the reference point around which rotations take place, NOT reference catalog stars.
refrd = None
for roundi, R in enumerate(Rads):
if roundi > 0:
refrad = 0.050
TT1 = TT
nb = int(np.ceil(R / targetrad))
nb = max(nb, 5)
if nb % 2 == 0:
nb += 1
print('Round', roundi + 1, ': matching with radius', R)
print('Nbins:', nb)
# kwargs to pass to intrabrickshift
ikwargs = {}
minoverlap = 0.01
tryoverlaps = (overlaps > minoverlap)
ikwargs.update(
do_affine=True, #mp=mp,
#alignplotargs=dict(bins=25),
alignplotargs=dict(bins=50),
overlaps=tryoverlaps)
ikwargs.update(ref=Tref, refrad=refrad)
# kwargs to pass to Alignment
akwargs = {}
i1 = intrabrickshift(TT1,
matchradius=R,
refradecs=refrd,
align_kwargs=dict(histbins=nb, **akwargs),
**ikwargs)
refrd = i1.get_reference_radecs()
filts = ['' for n in names]
ap = i1.alplotgrid
Nk = 100000
plot_all_alignments(ap, R * 1000, refrad * 1000, roundi + 1, names,
filts, ps, overlaps, outlines, Nk)
for T, aff in zip(TT, i1.affines):
T.ra, T.dec = aff.apply(T.ra, T.dec)
if affs is None:
affs = i1.affines
else:
for a, a2 in zip(affs, i1.affines):
a.add(a2)
from astrom_common import Affine
T = Affine.toTable(affs)
T.filenames = fns
#T.flt = fltfns
#T.gst = gstfns
#T.chip = chips
# FAKE -- used as a name in alignment_plots
T.gst = np.array([n + '.gst.fits' for n in names])
T.writeto(aff_fn)
# Final matching of all stars
allra = np.hstack([T.ra for T in TT])
alldec = np.hstack([T.dec for T in TT])
r_arcsec = 0.2
I, J, d = match_radec(allra,
alldec,
allra,
alldec,
r_arcsec / 3600.,
notself=True)
dec = alldec[I]
cosdec = np.cos(np.deg2rad(dec))
dr = (allra[I] - allra[J]) * cosdec * 3600.
dd = (alldec[I] - alldec[J]) * 3600.
plt.clf()
rr = (-r_arcsec * 1000, +r_arcsec * 1000)
plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr))
plt.xlabel('dRA (milli-arcsec)')
plt.ylabel('dDec (milli-arcsec)')
plt.savefig('match-all-after.png')
I, J, d = match_radec(Tref.ra, Tref.dec, allra, alldec, r_arcsec / 3600.)
dec = alldec[J]
cosdec = np.cos(np.deg2rad(dec))
dr = (Tref.ra[I] - allra[J]) * cosdec * 3600.
dd = (Tref.dec[I] - alldec[J]) * 3600.
plt.clf()
rr = (-r_arcsec * 1000, +r_arcsec * 1000)
plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr))
plt.xlabel('dRA (milli-arcsec)')
plt.ylabel('dDec (milli-arcsec)')
plt.savefig('match-all-ref-after.png')
r_arcsec = 0.02
I, J, d = match_radec(allra,
alldec,
allra,
alldec,
r_arcsec / 3600.,
notself=True)
dec = alldec[I]
cosdec = np.cos(np.deg2rad(dec))
dr = (allra[I] - allra[J]) * cosdec * 3600.
dd = (alldec[I] - alldec[J]) * 3600.
plt.clf()
rr = (-r_arcsec * 1000, +r_arcsec * 1000)
plothist(dr * 1000., dd * 1000., nbins=100, range=(rr, rr))
plt.xlabel('dRA (milli-arcsec)')
plt.ylabel('dDec (milli-arcsec)')
plt.savefig('match-all-after2.png')
T = fits_table()
T.ra = allra
T.dec = alldec
for col in [
'f814w_vega', 'f475w_vega', 'f336w_vega', 'f275w_vega',
'f110w_vega', 'f160w_vega', 'name'
]:
T.set(col, np.hstack([t.get(col) for t in TT]))
T.writeto(aligned_fn)
if False:
from singles import alignment_plots
dataset = 'M31'
Nkeep = 100000
R = 0.1
minoverlap = 0.01
perfield = False
nocache = True
from astrometry.util.multiproc import multiproc
mp = multiproc()
filts = ['F475W' for n in names]
chips = [-1] * len(names)
exptimes = [1] * len(names)
Nall = [0] * len(names)
rd = (minra, maxra, mindec, maxdec)
cnames = names
meta = (chips, names, cnames, filts, exptimes, Nall, rd)
alignment_plots(afffn,
dataset,
Nkeep,
0,
R,
minoverlap,
perfield,
nocache,
mp,
0,
tables=(TT, outlines, meta),
lexsort=False)
def tractor_vs_cat():
# Tractor/SDSS vs WISE/SDSS comparisons
R = 4./3600.
# NOTE, this matches all W entries (ie, same RA,Dec as SDSS), not just
# the ones with photometry.
I,J,d = match_radec(W.ra, W.dec, WC.ra, WC.dec, R, nearest=True)
print(len(I), 'matches to WISE catalog')
I2 = np.flatnonzero(np.logical_or(W.w1 > 0, W.w2 > 0))
T = S[I2]
T.add_columns_from(W[I2])
C = WC[J]
C.add_columns_from(S[I])
print(len(T), 'Tractor-SDSS matches')
print(len(C), 'WISE-SDSS matches')
I = np.flatnonzero((T.gpsf != T.ipsf) * (T.w1mag < 25))
wcuts = [17,18,19,20,21]
plothist(T.gpsf[I] - T.ipsf[I], T.imod[I] - T.w1mag[I], 200, range=((-1,5),(0,7)))
plt.xlabel('g - i (psf)')
plt.ylabel('i - W1 (model)')
plt.title('Tractor photometry')
ps.savefig()
for wcut in wcuts:
J = I[(T.w1mag[I] < wcut)]
plothist(T.gpsf[J] - T.ipsf[J], T.imod[J] - T.w1mag[J], 200, range=((-1,5),(0,7)))
plt.xlabel('g - i (psf)')
plt.ylabel('i - W1 (model)')
plt.title('Tractor photometry: W1 < %i' % wcut)
ps.savefig()
plothist(C.gpsf - C.ipsf, C.ipsf - C.w1mag, 200, range=((-1,5),(0,7)))
plt.xlabel('g - i (psf)')
plt.ylabel('i - W1 (psf)')
plt.title('WISE catalog photometry')
ps.savefig()
IT = np.logical_and(T.w1mag < 25, T.w2mag < 25)
for l1,x1,l2,x2 in [
#('g (psf)', 'gpsf')*2,
('r (psf)', 'rpsf')*2,
('W1', 'w1mag', 'W1', 'w1mpro'),
('W2', 'w2mag', 'W2', 'w2mpro')]:
if l1.startswith('W'):
rng = ((10,20),(-2,2))
else:
rng = ((15,25),(-2,2))
plothist(T.get(x1)[IT], (T.w1mag - T.w2mag)[IT], 200,
range=rng)
plt.xlabel(l1)
plt.ylabel('W1 - W2')
plt.title('Tractor photometry')
ps.savefig()
for wcut in wcuts:
J = IT[(T.w1mag[IT] < wcut)]
plothist(T.get(x1)[J], (T.w1mag - T.w2mag)[J], 200,
range=rng)
plt.xlabel(l1)
plt.ylabel('W1 - W2')
plt.title('Tractor photometry: W1 < %i' % wcut)
ps.savefig()
plothist(C.get(x2), C.w1mpro - C.w2mpro, 200, range=rng)
plt.xlabel(l2)
plt.ylabel('W1 - W2')
plt.title('WISE catalog photometry')
ps.savefig()
def orig_code(data, nmatch):
nside = Healpix().get_nside(len(data))
_, lo, hi = sigmaclip(data[data != 0], low=3, high=3)
flag = np.logical_or(data < lo, data > hi)
flag *= (nmatch > 20)
ra, dec = hp.pix2ang(nside, np.where(flag)[0], lonlat=True)
# PLOTTING
ralim = [ra.min(), ra.max()]
declim = [dec.min(), dec.max()]
my_mollzoom(ra,
dec,
data[flag],
'outliers',
ralim=ralim,
declim=declim,
vlim=(lo, hi))
temp_ra, temp_dec = hp.pix2ang(nside,
np.where(np.ones(len(data), bool))[0],
lonlat=True)
keep= (temp_ra >= ralim[0])*\
(temp_ra <= ralim[1])*\
(temp_dec >= declim[0])*\
(temp_dec <= declim[1])
my_mollzoom(temp_ra[keep],
temp_dec[keep],
data[keep],
'all',
ralim=ralim,
declim=declim,
vlim=(lo, hi))
keep *= (nmatch > 20)
my_mollzoom(temp_ra[keep],
temp_dec[keep],
data[keep],
'nmatch_gt20',
ralim=ralim,
declim=declim,
vlim=(lo, hi))
# Match bricks
#heal= fits_table()
#for col,arr in zip(['ra','dec'],[ra,dec]):
# heal.set(col, arr)
brick = fits_table(
os.path.join(args.targz_dir, 'legacysurveydir',
'survey-bricks.fits.gz'))
brick.cut( (brick.dec > -40)*\
(brick.dec < 40))
#deg_per_healpix= get_pixscale(npix,unit='deg')
deg_per_brick = 0.25
#imatch,imiss,d2d= Matcher().match_within(heal,brick, dist= deg_per_brick/2)
I, J, d = match_radec(ra,
dec,
brick.ra,
brick.dec,
deg_per_brick / 2,
nearest=True)
#raise ValueError
brick.cut(imatch['obs'])
my_scatter(brick.ra, brick.dec, 'bricks', ralim=ralim, declim=declim)
id = fn.replace('/', '').replace('.fits', '')
savenm = os.path.join(args.outdir, 'brick_table_%s.fits' % id)
brick.writeto(savenm)
print('Wrote %s' % savenm)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--name1', help='Name for first data set')
parser.add_argument('--name2', help='Name for second data set')
parser.add_argument('--plot-prefix', default='compare',
help='Prefix for plot filenames; default "%default"')
parser.add_argument('--match', default=1.0,
help='Astrometric cross-match distance in arcsec')
parser.add_argument('dir1', help='First directory to compare')
parser.add_argument('dir2', help='Second directory to compare')
opt = parser.parse_args()
ps = PlotSequence(opt.plot_prefix)
name1 = opt.name1
if name1 is None:
name1 = os.path.basename(opt.dir1)
if not len(name1):
name1 = os.path.basename(os.path.dirname(opt.dir1))
name2 = opt.name2
if name2 is None:
name2 = os.path.basename(opt.dir2)
if not len(name2):
name2 = os.path.basename(os.path.dirname(opt.dir2))
tt = 'Comparing %s to %s' % (name1, name2)
# regex for tractor-*.fits catalog filename
catre = re.compile('tractor-.*.fits')
cat1,cat2 = [],[]
for basedir,cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
for dirpath,dirnames,filenames in os.walk(basedir, followlinks=True):
for fn in filenames:
if not catre.match(fn):
print('Skipping', fn, 'due to filename')
continue
fn = os.path.join(dirpath, fn)
t = fits_table(fn)
print(len(t), 'from', fn)
cat.append(t)
cat1 = merge_tables(cat1, columns='fillzero')
cat2 = merge_tables(cat2, columns='fillzero')
print('Total of', len(cat1), 'from', name1)
print('Total of', len(cat2), 'from', name2)
cat1.cut(cat1.brick_primary)
cat2.cut(cat2.brick_primary)
print('Total of', len(cat1), 'BRICK_PRIMARY from', name1)
print('Total of', len(cat2), 'BRICK_PRIMARY from', name2)
cat1.cut((cat1.decam_anymask[:,1] == 0) *
(cat1.decam_anymask[:,2] == 0) *
(cat1.decam_anymask[:,4] == 0))
cat2.cut((cat2.decam_anymask[:,1] == 0) *
(cat2.decam_anymask[:,2] == 0) *
(cat2.decam_anymask[:,4] == 0))
print('Total of', len(cat1), 'unmasked from', name1)
print('Total of', len(cat2), 'unmasked from', name2)
I,J,d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match/3600.,
nearest=True)
print(len(I), 'matched')
plt.clf()
plt.hist(d * 3600., 100)
plt.xlabel('Match distance (arcsec)')
plt.title(tt)
ps.savefig()
matched1 = cat1[I]
matched2 = cat2[J]
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
K = np.flatnonzero((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0))
print('Median mw_trans', band, 'is',
np.median(matched1.decam_mw_transmission[:,iband]))
plt.clf()
plt.errorbar(matched1.decam_flux[K,iband],
matched2.decam_flux[K,iband],
fmt='.', color=cc,
xerr=1./np.sqrt(matched1.decam_flux_ivar[K,iband]),
yerr=1./np.sqrt(matched2.decam_flux_ivar[K,iband]),
alpha=0.1,
)
plt.xlabel('%s flux: %s' % (name1, band))
plt.ylabel('%s flux: %s' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.)
plt.axis([-100, 1000, -100, 1000])
plt.title(tt)
ps.savefig()
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
good = ((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0))
K = np.flatnonzero(good)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
P = np.flatnonzero(good * psf1 * psf2)
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1./iv1 + 1./iv2)
plt.clf()
plt.plot(mag1[K],
(matched2.decam_flux[K,iband] - matched1.decam_flux[K,iband]) / std[K],
'.', alpha=0.1, color=cc)
plt.plot(mag1[P],
(matched2.decam_flux[P,iband] - matched1.decam_flux[P,iband]) / std[P],
'.', alpha=0.1, color='k')
plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' % (name2, name1, band))
plt.xlabel('%s mag: %s' % (name1, band))
plt.axhline(0, color='k', alpha=0.5)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
plt.clf()
lp,lt = [],[]
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
good = ((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0))
#good = True
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1./iv1 + 1./iv2)
#std = np.hypot(std, 0.01)
G = np.flatnonzero(good * psf1 * psf2 *
np.isfinite(mag1) *
(mag1 >= 20) * (mag1 < dict(g=24, r=23.5, z=22.5)[band]))
n,b,p = plt.hist((matched2.decam_flux[G,iband] -
matched1.decam_flux[G,iband]) / std[G],
range=(-4, 4), bins=50, histtype='step', color=cc,
normed=True)
sig = (matched2.decam_flux[G,iband] -
matched1.decam_flux[G,iband]) / std[G]
print('Raw mean and std of points:', np.mean(sig), np.std(sig))
med = np.median(sig)
rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.
print('Median and percentile-based sigma:', med, rsigma)
lp.append(p[0])
lt.append('%s: %.2f +- %.2f' % (band, med, rsigma))
bins = []
gaussint = []
for blo,bhi in zip(b, b[1:]):
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
#bins.extend([blo,bhi])
#gaussint.extend([c,c])
bins.append((blo+bhi)/2.)
gaussint.append(c)
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.title(tt)
plt.xlabel('Flux difference / error (sigma)')
plt.axvline(0, color='k', alpha=0.1)
plt.ylim(0, 0.45)
plt.legend(lp, lt, loc='upper right')
ps.savefig()
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
plt.clf()
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
matched2.decam_flux[:,iband], matched2.decam_flux_ivar[:,iband])
meanmag = NanoMaggies.nanomaggiesToMag((
matched1.decam_flux[:,iband] + matched2.decam_flux[:,iband]) / 2.)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
good = ((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0) *
np.isfinite(mag1) * np.isfinite(mag2))
K = np.flatnonzero(good)
P = np.flatnonzero(good * psf1 * psf2)
plt.errorbar(mag1[K], mag2[K], fmt='.', color=cc,
xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s (mag)' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.)
plt.axis([24, 16, 24, 16])
plt.title(tt)
ps.savefig()
plt.clf()
plt.errorbar(mag1[K], mag2[K] - mag1[K], fmt='.', color=cc,
xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -1, 1])
plt.title(tt)
ps.savefig()
magbins = np.arange(16, 24.001, 0.5)
plt.clf()
plt.plot(mag1[K], (mag2[K]-mag1[K]) / np.hypot(magerr1[K], magerr2[K]),
'.', color=cc, alpha=0.1)
plt.plot(mag1[P], (mag2[P]-mag1[P]) / np.hypot(magerr1[P], magerr2[P]),
'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
plt.clf()
plt.plot(meanmag[P], y[P], 'k.', alpha=0.1)
midmag = []
vals = np.zeros((len(magbins)-1, 5))
median_err1 = []
iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)
# FIXME -- should we do some stats after taking off the mean difference?
for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
midmag.append((mlo+mhi)/2.)
median_err1.append(np.median(magerr1[I]))
if len(I) == 0:
continue
# median and +- 1 sigma quantiles
ybin = y[I]
vals[bini,0] = np.percentile(ybin, 16)
vals[bini,1] = np.median(ybin)
vals[bini,2] = np.percentile(ybin, 84)
# +- 2 sigma quantiles
vals[bini,3] = np.percentile(ybin, 2.3)
vals[bini,4] = np.percentile(ybin, 97.7)
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss,
'vs expected for Gaussian;', len(ybin), 'points')
# if iqd > iqd_gauss:
# # What error adding in quadrature would you need to make the IQD match?
# err = median_err1[-1]
# target_err = err * (iqd / iqd_gauss)
# sys_err = np.sqrt(target_err**2 - err**2)
# print('--> add systematic error', sys_err)
# ~ Johan's cuts
mlo = 21.
mhi = dict(g=24., r=23.5, z=22.5)[band]
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
ybin = y[I]
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', mlo, mhi, 'band', band, ': IQD is factor',
iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points')
if iqd > iqd_gauss:
# What error adding in quadrature would you need to make
# the IQD match?
err = np.median(np.hypot(magerr1[I], magerr2[I]))
print('Median error (hypot):', err)
target_err = err * (iqd / iqd_gauss)
print('Target:', target_err)
sys_err = np.sqrt((target_err**2 - err**2) / 2.)
print('--> add systematic error', sys_err)
# check...
err_sys = np.hypot(np.hypot(magerr1, sys_err),
np.hypot(magerr2, sys_err))
ysys = (mag2 - mag1) / err_sys
ysys = ysys[I]
print('Resulting median error:', np.median(err_sys[I]))
iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian')
# Hmmm, this doesn't work... totally overshoots.
plt.errorbar(midmag, vals[:,1], fmt='o', color='b',
yerr=(vals[:,1]-vals[:,0], vals[:,2]-vals[:,1]),
capthick=3, zorder=20)
plt.errorbar(midmag, vals[:,1], fmt='o', color='b',
yerr=(vals[:,1]-vals[:,3], vals[:,4]-vals[:,1]),
capthick=2, zorder=20)
plt.axhline( 1., color='b', alpha=0.2)
plt.axhline(-1., color='b', alpha=0.2)
plt.axhline( 2., color='b', alpha=0.2)
plt.axhline(-2., color='b', alpha=0.2)
for mag,err,y in zip(midmag, median_err1, vals[:,3]):
if not np.isfinite(err):
continue
if y < -6:
continue
plt.text(mag, y-0.1, '%.3f' % err, va='top', ha='center', color='k',
fontsize=10)
plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axvline(21, color='k', alpha=0.3)
plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3)
plt.axis([24.1, 16, -6, 6])
plt.title(tt)
ps.savefig()
#magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
if band == 'g':
magbins = [20, 24]
elif band == 'r':
magbins = [20, 23.5]
elif band == 'z':
magbins = [20, 22.5]
slo,shi = -5,5
plt.clf()
ha = dict(bins=25, range=(slo,shi), histtype='step', normed=True)
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
midmag = []
nn = []
rgbs = []
lt,lp = [],[]
for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
if len(I) == 0:
continue
ybin = y[I]
rgb = [0.,0.,0.]
rgb[0] = float(bini) / (len(magbins)-1)
rgb[2] = 1. - rgb[0]
n,b,p = plt.hist(ybin, color=rgb, **ha)
lt.append('mag %g to %g' % (mlo,mhi))
lp.append(p[0])
midmag.append((mlo+mhi)/2.)
nn.append(n)
rgbs.append(rgb)
bins = []
gaussint = []
for blo,bhi in zip(b, b[1:]):
#midbin.append((blo+bhi)/2.)
#gaussint.append(scipy.stats.norm.cdf(bhi) -
# scipy.stats.norm.cdf(blo))
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
bins.extend([blo,bhi])
gaussint.extend([c,c])
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo,shi)
ps.savefig()
bincenters = b[:-1] + (b[1]-b[0])/2.
plt.clf()
lp = []
for n,rgb,mlo,mhi in zip(nn, rgbs, magbins, magbins[1:]):
p = plt.plot(bincenters, n, '-', color=rgb)
lp.append(p[0])
plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo,shi)
ps.savefig()
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-o', '--out', dest='outfn', help='Output filename',
default='TMP/nexp.fits')
parser.add_argument('--merge', action='store_true', help='Merge sub-tables')
parser.add_argument('--plot', action='store_true', help='Plot results')
parser.add_argument('files', metavar='nexp-file.fits.gz', nargs='+',
help='List of nexp files to process')
opt = parser.parse_args()
fns = opt.files
if opt.merge:
from astrometry.util.fits import merge_tables
TT = []
for fn in fns:
T = fits_table(fn)
print(fn, '->', len(T))
TT.append(T)
T = merge_tables(TT)
T.writeto(opt.outfn)
print('Wrote', opt.outfn)
if opt.plot:
T = fits_table(opt.files[0])
import pylab as plt
import matplotlib
ax = [360, 0, -21, 36]
def radec_plot():
plt.axis(ax)
plt.xlabel('RA (deg)')
plt.xticks(np.arange(0, 361, 45))
plt.ylabel('Dec (deg)')
gl = np.arange(361)
gb = np.zeros_like(gl)
from astrometry.util.starutil_numpy import lbtoradec
rr,dd = lbtoradec(gl, gb)
plt.plot(rr, dd, 'k-', alpha=0.5, lw=1)
rr,dd = lbtoradec(gl, gb+10)
plt.plot(rr, dd, 'k-', alpha=0.25, lw=1)
rr,dd = lbtoradec(gl, gb-10)
plt.plot(rr, dd, 'k-', alpha=0.25, lw=1)
plt.figure(figsize=(8,5))
plt.subplots_adjust(left=0.1, right=0.98, top=0.93)
# Map of the tile centers we want to observe...
O = fits_table('obstatus/decam-tiles_obstatus.fits')
O.cut(O.in_desi == 1)
rr,dd = np.meshgrid(np.linspace(ax[1],ax[0], 700),
np.linspace(ax[2],ax[3], 200))
from astrometry.libkd.spherematch import match_radec
I,J,d = match_radec(O.ra, O.dec, rr.ravel(), dd.ravel(), 1.)
desimap = np.zeros(rr.shape, bool)
desimap.flat[J] = True
def desi_map():
# Show the DESI tile map in the background.
from astrometry.util.plotutils import antigray
plt.imshow(desimap, origin='lower', interpolation='nearest',
extent=[ax[1],ax[0],ax[2],ax[3]], aspect='auto',
cmap=antigray, vmax=8)
for band in 'grz':
plt.clf()
desi_map()
N = T.get('nexp_%s' % band)
I = np.flatnonzero(N > 0)
#cm = matplotlib.cm.get_cmap('jet', 6)
#cm = matplotlib.cm.get_cmap('winter', 5)
cm = matplotlib.cm.viridis
cm = matplotlib.cm.get_cmap(cm, 5)
plt.scatter(T.ra[I], T.dec[I], c=N[I], s=2,
edgecolors='none',
vmin=0.5, vmax=5.5, cmap=cm)
radec_plot()
cax = colorbar_axes(plt.gca(), frac=0.06)
plt.colorbar(cax=cax, ticks=range(6))
#plt.colorbar(ticks=range(6))
plt.title('DECaLS DR3: Number of exposures in %s' % band)
plt.savefig('nexp-%s.png' % band)
plt.clf()
desi_map()
plt.scatter(T.ra, T.dec, c=T.get('nexp_%s' % band), s=2,
edgecolors='none', vmin=0, vmax=2.)
radec_plot()
plt.colorbar()
plt.title('DECaLS DR3: PSF size, band %s' % band)
plt.savefig('psfsize-%s.png' % band)
return 0
for col in ['nobjs', 'npsf', 'nsimp', 'nexp', 'ndev', 'ncomp']:
plt.clf()
desi_map()
N = T.get(col)
mx = np.percentile(N, 99.5)
plt.scatter(T.ra, T.dec, c=N, s=2,
edgecolors='none', vmin=0, vmax=mx)
radec_plot()
plt.colorbar()
plt.title('DECaLS DR3: Number of objects of type %s' % col[1:])
plt.savefig('nobjs-%s.png' % col[1:])
Ntot = T.nobjs
for col in ['npsf', 'nsimp', 'nexp', 'ndev', 'ncomp']:
plt.clf()
desi_map()
N = T.get(col) / Ntot.astype(np.float32)
mx = np.percentile(N, 99.5)
plt.scatter(T.ra, T.dec, c=N, s=2,
edgecolors='none', vmin=0, vmax=mx)
radec_plot()
plt.colorbar()
plt.title('DECaLS DR3: Fraction of objects of type %s' % col[1:])
plt.savefig('fobjs-%s.png' % col[1:])
return 0
# fnpats = opt.files
# fns = []
# for pat in fnpats:
# pfns = glob(pat)
# fns.extend(pfns)
# print('Pattern', pat, '->', len(pfns), 'files')
#fns = glob('coadd/*/*/*-nexp*')
#fns = glob('coadd/000/*/*-nexp*')
#fns = glob('coadd/000/0001*/*-nexp*')
fns.sort()
print(len(fns), 'nexp files')
brickset = set()
bricklist = []
gn = []
rn = []
zn = []
gnhist = []
rnhist = []
znhist = []
nnhist = 6
gdepth = []
rdepth = []
zdepth = []
ibricks = []
nsrcs = []
npsf = []
nsimp = []
nexp = []
ndev = []
ncomp = []
gpsfsize = []
rpsfsize = []
zpsfsize = []
ebv = []
gtrans = []
rtrans = []
ztrans = []
bricks = fits_table('survey-bricks.fits.gz')
#sfd = SFDMap()
W = H = 3600
# H=3600
# xx,yy = np.meshgrid(np.arange(W), np.arange(H))
unique = np.ones((H,W), bool)
tlast = 0
for fn in fns:
print('File', fn)
words = fn.split('/')
dirprefix = '/'.join(words[:-4])
print('Directory prefix:', dirprefix)
words = words[-4:]
brick = words[2]
print('Brick', brick)
if not brick in brickset:
brickset.add(brick)
bricklist.append(brick)
gn.append(0)
rn.append(0)
zn.append(0)
gnhist.append([0 for i in range(nnhist)])
rnhist.append([0 for i in range(nnhist)])
znhist.append([0 for i in range(nnhist)])
index = -1
ibrick = np.nonzero(bricks.brickname == brick)[0][0]
ibricks.append(ibrick)
tfn = os.path.join(dirprefix, 'tractor', brick[:3], 'tractor-%s.fits'%brick)
print('Tractor filename', tfn)
T = fits_table(tfn, columns=['brick_primary', 'type', 'decam_psfsize',
'ebv', 'decam_mw_transmission'])
T.cut(T.brick_primary)
nsrcs.append(len(T))
types = Counter([t.strip() for t in T.type])
npsf.append(types['PSF'])
nsimp.append(types['SIMP'])
nexp.append(types['EXP'])
ndev.append(types['DEV'])
ncomp.append(types['COMP'])
print('N sources', nsrcs[-1])
gpsfsize.append(np.median(T.decam_psfsize[:,1]))
rpsfsize.append(np.median(T.decam_psfsize[:,2]))
zpsfsize.append(np.median(T.decam_psfsize[:,4]))
ebv.append(np.median(T.ebv))
gtrans.append(np.median(T.decam_mw_transmission[:,1]))
rtrans.append(np.median(T.decam_mw_transmission[:,2]))
ztrans.append(np.median(T.decam_mw_transmission[:,4]))
br = bricks[ibrick]
print('Computing unique brick pixels...')
#wcs = Tan(fn, 0)
#W,H = int(wcs.get_width()), int(wcs.get_height())
pixscale = 0.262/3600.
wcs = Tan(br.ra, br.dec, W/2.+0.5, H/2.+0.5,
-pixscale, 0., 0., pixscale,
float(W), float(H))
import time
t0 = time.clock()
unique[:,:] = True
find_unique_pixels(wcs, W, H, unique,
br.ra1, br.ra2, br.dec1, br.dec2)
# for i in range(W/2):
# allin = True
# lo,hi = i, W-i-1
# # one slice per side
# side = slice(lo,hi+1)
# top = (lo, side)
# bot = (hi, side)
# left = (side, lo)
# right = (side, hi)
# for slc in [top, bot, left, right]:
# #print('xx,yy', xx[slc], yy[slc])
# rr,dd = wcs.pixelxy2radec(xx[slc]+1, yy[slc]+1)
# U = (rr >= br.ra1 ) * (rr < br.ra2 ) * (dd >= br.dec1) * (dd < br.dec2)
# #print('Pixel', i, ':', np.sum(U), 'of', len(U), 'pixels are unique')
# allin *= np.all(U)
# unique[slc] = U
# if allin:
# print('Scanned to pixel', i)
# break
t1 = time.clock()
U = np.flatnonzero(unique)
t2 = time.clock()
print(len(U), 'of', W*H, 'pixels are unique to this brick')
# #t3 = time.clock()
#rr,dd = wcs.pixelxy2radec(xx+1, yy+1)
# #t4 = time.clock()
# #u = (rr >= br.ra1 ) * (rr < br.ra2 ) * (dd >= br.dec1) * (dd < br.dec2)
# #t5 = time.clock()
# #U2 = np.flatnonzero(u)
#U2 = np.flatnonzero((rr >= br.ra1 ) * (rr < br.ra2 ) *
# (dd >= br.dec1) * (dd < br.dec2))
#assert(np.all(U == U2))
#assert(len(U) == len(U2))
# #t6 = time.clock()
# print(len(U2), 'of', W*H, 'pixels are unique to this brick')
#
#print(t0-tlast, 'other time')
#tlast = time.clock() #t2
#print('t1:', t1-t0, 't2', t2-t1)
# #print('t4:', t4-t3, 't5', t5-t4, 't6', t6-t5)
#
else:
index = bricklist.index(brick)
assert(index == len(bricklist)-1)
index = bricklist.index(brick)
assert(index == len(bricklist)-1)
filepart = words[-1]
filepart = filepart.replace('.fits.gz', '')
print('File:', filepart)
band = filepart[-1]
assert(band in 'grz')
nlist,nhist = dict(g=(gn,gnhist), r=(rn,rnhist), z=(zn,znhist))[band]
upix = fitsio.read(fn).flat[U]
med = np.median(upix)
print('Band', band, ': Median', med)
nlist[index] = med
hist = nhist[index]
for i in range(nnhist):
if i < nnhist-1:
hist[i] = np.sum(upix == i)
else:
hist[i] = np.sum(upix >= i)
assert(sum(hist) == len(upix))
print('Number of exposures histogram:', hist)
ibricks = np.array(ibricks)
print('Maximum number of sources:', max(nsrcs))
T = fits_table()
T.brickname = np.array(bricklist)
T.ra = bricks.ra [ibricks]
T.dec = bricks.dec[ibricks]
T.nexp_g = np.array(gn).astype(np.int16)
T.nexp_r = np.array(rn).astype(np.int16)
T.nexp_z = np.array(zn).astype(np.int16)
T.nexphist_g = np.array(gnhist).astype(np.int32)
T.nexphist_r = np.array(rnhist).astype(np.int32)
T.nexphist_z = np.array(znhist).astype(np.int32)
T.nobjs = np.array(nsrcs).astype(np.int16)
T.npsf = np.array(npsf ).astype(np.int16)
T.nsimp = np.array(nsimp).astype(np.int16)
T.nexp = np.array(nexp ).astype(np.int16)
T.ndev = np.array(ndev ).astype(np.int16)
T.ncomp = np.array(ncomp).astype(np.int16)
T.psfsize_g = np.array(gpsfsize).astype(np.float32)
T.psfsize_r = np.array(rpsfsize).astype(np.float32)
T.psfsize_z = np.array(zpsfsize).astype(np.float32)
T.ebv = np.array(ebv).astype(np.float32)
T.trans_g = np.array(gtrans).astype(np.float32)
T.trans_r = np.array(rtrans).astype(np.float32)
T.trans_z = np.array(ztrans).astype(np.float32)
T.writeto(opt.outfn)
