def build_simcat(Samp=None,brickwcs=None, meta=None):
"""Creates the simulated source catalog for a given brick (not CCD).
The WCS for the brick (not CCD) is used to convert ra,dec of source
to x,y pixel location in brickspace
Args:
Samp: fits_table for the properties of sources in the brick
usually a subset of all sources in the brick determined by
rowstart (rs)
brickwcs: WCS object for the brick
meta: 'metacat' table
fits_table with configuration-like params for the simulated sources
Returns:
tuple of
cat:
skipping_ids:
"""
log = logging.getLogger('decals_sim')
#rand = np.random.RandomState(seed)
# Assign central coordinates uniformly but remove simulated sources which
# are too near to one another. Iterate until we have the requisite number
# of objects.
#bounds = brickwcs.radec_bounds()
#ra = rand.uniform(bounds[0], bounds[1], nobj)
#dec = rand.uniform(bounds[2], bounds[3], nobj)
i_keep,i_skip= flag_nearest_neighbors(Samp, radius_in_deg=5./3600)
skipping_ids= Samp.get('id')[i_skip]
log.info('sources %d, keeping %d, flagged as nearby %d' % (len(Samp),len(i_keep),len(i_skip)))
Samp.cut(i_keep)
xxyy = brickwcs.radec2pixelxy(Samp.ra,Samp.dec)
#cat = Table()
#cat['ID'] = Column(Samp.get('id'),dtype='i4') #np.arange(nobj, dtype='i4'))
#cat['RA'] = Column(Samp.ra, dtype='f8')
#cat['DEC'] = Column(Samp.dec, dtype='f8')
#cat['X'] = Column(xxyy[1][:], dtype='f4')
#cat['Y'] = Column(xxyy[2][:], dtype='f4')
cat = fits_table()
for key in ['id','ra','dec']:
cat.set(key, Samp.get(key))
cat.set('x', xxyy[1][:])
cat.set('y', xxyy[2][:])
typ=meta.get('objtype')[0]
# Mags
filts = ['%s %s' % ('DES', f) for f in 'grz']
for band in ['g','r','z']:
nanomag= 1E9*10**(-0.4*Samp.get(band))
# Add extinction (to stars too, b/c "decam-chatter 6517")
mw_transmission= SFDMap().extinction(['DES %s' % band],
Samp.ra, Samp.dec)
mw_transmission= 10**(-mw_transmission[:,0].astype(np.float32)/2.5)
cat.set('%sflux' % band, nanomag * mw_transmission)
cat.set('mw_transmission_%s' % band, mw_transmission)
# Galaxy Properties
if typ in ['elg','lrg']:
# Convert to e1,e2 if given ba,pa
if ('ba' in Samp.get_columns()) & ('pa' in Samp.get_columns()):
e1,e2= get_e1_e2(Samp.get('ba'),Samp.get('pa'))
Samp.set('e1',e1)
Samp.set('e2',e2)
for key in ['n','rhalf','e1','e2']:
cat.set(key, Samp.get(key))
# Sersic n: GALSIM n = [0.3,6.2] for numerical stability,see
# https://github.com/GalSim-developers/GalSim/issues/{325,450}
return cat, skipping_ids
def main():
ps = PlotSequence('shotgun')
decals = Decals()
C = fits_table('decals-ccds-annotated.fits')
print(len(C), 'CCDs')
C.cut(C.photometric)
C.cut(C.blacklist_ok)
print(len(C), 'photometric and not blacklisted')
# HACK
print('FIXME not cutting on DECALS')
#C.cut(C.tilepass > 0)
#print(len(C), 'taken by DECaLS')
targets = dict(g=24.0, r=23.4, z=22.5)
def ivtomag(iv, nsigma=5.):
return -2.5 * (np.log10(nsigma / np.sqrt(iv)) - 9)
def band_index(band):
allbands = 'ugrizY'
return allbands.index(band)
ccmap = dict(g='g', r='r', z='m')
ceil_exptime = dict(g=125., r=125., z=250.)
#plt.clf()
bands = 'grz'
for band in bands:
tmag = targets[band]
print()
print(band, 'band, target depth', tmag)
ccds = C[C.filter == band]
ccdarea = (2046*4094*(0.262/3600.)**2)
print(len(ccds), 'CCDs, total exptime', np.sum(ccds.exptime),
'(mean %.1f)' % np.mean(ccds.exptime), 'total area',
len(ccds)*ccdarea, 'sq.deg')
detsig1 = ccds.sig1 / ccds.galnorm_mean
totiv = np.sum(1. / detsig1**2)
# depth we would have if we had all exposure time in one CCD
# print('5-sigma galaxy depth if concentrated in one CCD:', ivtomag(totiv))
# # mean depth
# print('5-sigma galaxy depth if spread equally among', len(ccds), 'CCDs:', ivtomag(totiv / len(ccds)))
# print('vs median depth', np.median(ccds.galdepth))
# print('5-sigma galaxy depth if spread equally among %i/2' % (len(ccds)), 'CCDs:', ivtomag(totiv / (len(ccds)/2)))
# print('5-sigma galaxy depth if spread equally among %i/3' % (len(ccds)), 'CCDs:', ivtomag(totiv / (len(ccds)/3)))
# spread over 6000 sq deg
sqdeg = 6000
avgiv = totiv * ccdarea / sqdeg
#print('5-sigma galaxy depth if spread over', sqdeg, 'sqdeg:', ivtomag(avgiv))
tflux = 10.**(tmag / -2.5 + 9)
tiv = 1. / (tflux / 5)**2
#print('Fraction of', sqdeg, 'sqdeg survey complete:', avgiv / tiv)
iband = band_index(band)
ext = ccds.decam_extinction[:,iband]
medext = np.median(ext)
print('With extinction (median %.2f mag):' % medext)
transmission = 10.**(-ext / 2.5)
detsig1 = ccds.sig1 / ccds.galnorm_mean / transmission
totiv = np.sum(1. / detsig1**2)
# depth we would have if we had all exposure time in one CCD
print('5-sigma galaxy depth if concentrated in one CCD: %.3f' % ivtomag(totiv))
# mean depth
print('5-sigma galaxy depth if spread equally among', len(ccds), 'CCDs: %.3f' % ivtomag(totiv / len(ccds)))
print('vs median depth: %.3f' % np.median(ccds.galdepth - ext))
print('5-sigma galaxy depth if spread equally among %i/2' % (len(ccds)), 'CCDs: %.3f' % ivtomag(totiv / (len(ccds)/2)))
print('5-sigma galaxy depth if spread equally among %i/3' % (len(ccds)), 'CCDs: %.3f' % ivtomag(totiv / (len(ccds)/3)))
# spread over 6000 sq deg
sqdeg = 6000
avgiv = totiv * ccdarea / sqdeg
print('5-sigma galaxy depth if spread over', sqdeg, 'sqdeg: %.3f' % ivtomag(avgiv))
print('Fraction of', sqdeg, 'sqdeg survey complete: %.3f' % (avgiv / tiv))
# plt.hist(ccds.exptime, range=(0,250), bins=50, histtype='step', color=ccmap[band])
# I = np.flatnonzero(ccds.exptime < (ceil_exptime[band] - 1.))
# ccds.cut(I)
# print('Cutting out exposures with ceil exposure time:', len(ccds))
#
# plt.hist(ccds.exptime, bins=25, histtype='step', color=ccmap[band],
# linestyle='dotted', linewidth=3, alpha=0.3)
#
# transmission = transmission[I]
# ext = ext[I]
#
# detsig1 = ccds.sig1 / ccds.galnorm_mean / transmission
# totiv = np.sum(1. / detsig1**2)
# # depth we would have if we had all exposure time in one CCD
# print('5-sigma galaxy depth if concentrated in one CCD:', ivtomag(totiv))
# # mean depth
# print('5-sigma galaxy depth if spread equally among', len(ccds), 'CCDs:', ivtomag(totiv / len(ccds)))
# print('vs median depth', np.median(ccds.galdepth - ext))
# print('5-sigma galaxy depth if spread equally among %i/2' % (len(ccds)), 'CCDs:', ivtomag(totiv / (len(ccds)/2)))
# print('5-sigma galaxy depth if spread equally among %i/3' % (len(ccds)), 'CCDs:', ivtomag(totiv / (len(ccds)/3)))
# # spread over 6000 sq deg
# sqdeg = 6000
# avgiv = totiv * ccdarea / sqdeg
# print('5-sigma galaxy depth if spread over', sqdeg, 'sqdeg:', ivtomag(avgiv))
# print('Fraction of', sqdeg, 'sqdeg survey complete:', avgiv / tiv)
# plt.xlabel('Exposure time (s)')
# ps.savefig()
print()
dra = 4094 / 2. / 3600 * 0.262
ddec = 2046 / 2. / 3600 * 0.262
ralo = max( 0, min(C.ra - dra / np.cos(np.deg2rad(C.dec))))
rahi = min(360, max(C.ra + dra / np.cos(np.deg2rad(C.dec))))
declo = max(-90, min(C.dec - ddec))
dechi = min( 90, max(C.dec + ddec))
# brick 0001m002
#ralo,rahi = 0., 0.25
#declo,dechi = -0.375, -0.125
#ralo, rahi = 0, 1
#declo, dechi = 0, 1
ralo, rahi = 0, 0.5
declo, dechi = 0, 0.5
print('RA,Dec range', (ralo, rahi), (declo, dechi))
N = 10000
nbatch = 1000
rr,dd = [],[]
ntotal = 0
while ntotal < N:
ru = np.random.uniform(size=nbatch)
d = np.random.uniform(low=declo, high=dechi, size=nbatch)
# Taper the accepted width in RA based on Dec
cosd = np.cos(np.deg2rad(d))
I = np.flatnonzero(ru < cosd)
if len(I) == 0:
continue
r = ralo + (rahi - ralo) * ru[I]/cosd[I]
d = d[I]
rr.append(r)
dd.append(d)
ntotal += len(r)
print('Kept', len(r), 'of', nbatch)
ra = np.hstack(rr)
dec = np.hstack(dd)
del rr
del dd
ra = ra [:N]
dec = dec[:N]
print('RA,Dec ranges of samples:', (ra.min(), ra.max()), (dec.min(), dec.max()))
# plt.clf()
# plt.plot(ra, dec, 'b.', alpha=0.1)
# ps.savefig()
# CCD size
margin = 10 * 0.262 / 3600.
# C.dra is in degrees on the sphere, not delta-RA
# dra = C.dra / np.cos(np.deg2rad(C.dec))
# ccds = C[(C.ra + dra + margin > ralo) *
# (C.ra - dra - margin < rahi) *
# (C.dec + C.ddec + margin > declo) *
# (C.dec - C.ddec - margin < dechi)]
I,J,d = match_radec(C.ra, C.dec,
(ralo+rahi)/2., (declo+dechi)/2.,
degrees_between(ralo,declo, rahi,dechi)/2. + 1.,
nearest=True)
ccds = C[I]
print(len(ccds), 'nearby')
assert(len(ccds))
print('RA range', ccds.ra.min(), ccds.ra.max())
radius = np.hypot(2046, 4096) / 2. * 0.262 / 3600 * 1.1
II = match_radec(ccds.ra, ccds.dec, ra, dec,
radius, indexlist=True)
#print('Matching:', II)
depthrange = [20,25]
depthbins = 100
depthbins2 = 500
galdepths = dict([(b, np.zeros(len(ra))) for b in bands])
iccds = []
for iccd,I in enumerate(II):
if I is None:
continue
ccd = ccds[iccd]
print('Matched to CCD %s: %i' % (ccd.expid, len(I)))
# Actually inside CCD RA,Dec box?
r = ra[I]
d = dec[I]
# print('degrees_between: ccd ra,dec', ccd.ra, ccd.dec)
# for ri,di in zip(r, degrees_between(r, ccd.dec + np.zeros_like(r),
# ccd.ra, ccd.dec)):
# print('ri: di', ri, di)
J = np.flatnonzero((degrees_between(r, ccd.dec+np.zeros_like(r),
ccd.ra, ccd.dec) < dra) *
(np.abs(d - ccd.dec) < ddec))
print('Actually inside CCD RA,Dec box:', len(J))
if len(J) == 0:
continue
I = np.array(I)[J]
# j = J[0]
# print('deg between', degrees_between(r[j], ccd.dec,
# ccd.ra, ccd.dec), 'vs', dra)
# print(' dec', d[j], 'dist', np.abs(d[j] - ccd.dec), 'vs', ddec)
iccds.append((iccd,I[0]))
band = ccd.filter
gd = galdepths[band]
print('Gal depth in', band, ':', ccd.galdepth)
# mag -> 5sig1 -> iv
cgd = 10.**((ccd.galdepth - 22.5) / -2.5)
cgd = 1. / cgd**2
gd[I] += cgd
plt.clf()
# plt.plot(ra, dec, 'k.', alpha=0.1)
# C1 = fits_table('decals-ccds-annotated.fits')
# ii,jj,dd = match_radec(C1.ra, C1.dec, (ralo+rahi)/2.,(declo+dechi)/2,0.5)
# C1.cut(ii)
# C1.ra -= (C1.ra > 270)*360
# for ccd in C1:
# r,dr = ccd.ra, ccd.dra / np.cos(np.deg2rad(ccd.dec))
# d,dd = ccd.dec, ccd.ddec
# sty = dict(color='k', alpha=0.1)
# if not (ccd.photometric and ccd.blacklist_ok):
# sty = dict(color='c', lw=3)
# plt.plot([r-dr, r+dr, r+dr, r-dr, r-dr], [d-dd,d-dd,d+dd,d+dd,d-dd],
# '-', **sty)
for iccd,i in iccds:
ccd = ccds[iccd]
r,dr = ccd.ra, dra / np.cos(np.deg2rad(ccd.dec))
# HACK
r = r + (r > 270)*-360.
d,dd = ccd.dec, ddec
plt.plot([r-dr, r+dr, r+dr, r-dr, r-dr], [d-dd,d-dd,d+dd,d+dd,d-dd],
'-', color=ccmap[ccd.filter], alpha=0.5)
plt.plot(ra[i], dec[i], 'k.')
plt.plot([ralo,ralo,rahi,rahi,ralo],[declo,dechi,dechi,declo,declo],
'k-', lw=2)
#plt.axis([-0.1, 0.6, -0.1, 0.6])
plt.axis([-0.5, 1., -0.5, 1.])
plt.xlabel('RA')
plt.ylabel('Dec')
ps.savefig()
for band in bands:
gd = galdepths[band]
# iv -> 5sig1 -> mag
gd = 1. / np.sqrt(gd)
gd = -2.5 * (np.log10(gd) - 9.)
plt.clf()
plt.hist(gd, range=depthrange, bins=depthbins, histtype='step',
color=ccmap[band])
plt.xlabel('5-sigma Galaxy depth (mag)')
plt.title('%s band' % band)
ps.savefig()
print('Reading extinction values for sample points...')
sfd = SFDMap()
filts = ['%s %s' % ('DES', f) for f in bands]
ebv,extinction = sfd.extinction(filts, ra, dec, get_ebv=True)
print('Extinction:', extinction.shape)
B = decals.get_bricks_readonly()
I = decals.bricks_touching_radec_box(None, ralo, rahi, declo, dechi)
B.cut(I)
print(len(B), 'bricks touching RA,Dec box')
depth_hists = {}
depth_hists_2 = {}
for brick in B:
print('Brick', brick.brickname)
I = np.flatnonzero((ra > brick.ra1 ) * (ra < brick.ra2) *
(dec > brick.dec1) * (dec < brick.dec2))
print(len(I), 'samples in brick')
if len(I) == 0:
continue
wcs = wcs_for_brick(brick)
ok,x,y = wcs.radec2pixelxy(ra[I], dec[I])
x = np.round(x - 1).astype(int)
y = np.round(y - 1).astype(int)
for iband,band in enumerate(bands):
fn = decals.find_file('nexp', brick=brick.brickname, band=band)
print('Reading', fn)
if not os.path.exists(fn):
print('Missing:', fn)
continue
nexp = fitsio.read(fn)
nexp = nexp[y, x]
ext = extinction[I, iband]
fn = decals.find_file('galdepth', brick=brick.brickname, band=band)
print('Reading', fn)
galdepth = fitsio.read(fn)
galdepth = galdepth[y, x]
# iv -> mag
galdepth = -2.5 * (np.log10(5. / np.sqrt(galdepth)) - 9)
# extinction-corrected
galdepth -= ext
un = np.unique(nexp)
print('Numbers of exposures:', un)
for ne in un:
if ne == 0:
continue
J = np.flatnonzero(nexp == ne)
gd = galdepth[J]
key = (ne, band)
H = depth_hists.get(key, 0)
h,e = np.histogram(gd, range=depthrange, bins=depthbins)
H = h + H
depth_hists[key] = H
H = depth_hists_2.get(key, 0)
h,e = np.histogram(gd, range=depthrange, bins=depthbins2)
H = h + H
depth_hists_2[key] = H
dlo,dhi = depthrange
left = dlo + np.arange(depthbins) * (dhi-dlo) / float(depthbins)
binwidth = left[1]-left[0]
# for k,H in depth_hists.items():
# plt.clf()
# plt.bar(left, H, width=binwidth)
# plt.xlabel('Galaxy depth (mag)')
# (ne,band) = k
# plt.title('%s band, %i exposures' % (band, ne))
# plt.xlim(dlo, dhi)
# fn = 'depth-%s-%i.png' % (band, ne)
# plt.savefig(fn)
# print('Wrote', fn)
rainbow = ['r', '#ffa000', 'y', 'g', 'b', 'm']
for band in bands:
plt.clf()
for ne in range(1,10):
key = (ne, band)
if not key in depth_hists:
continue
H = depth_hists[key]
print('hist length:', len(H))
plt.plot(np.vstack((left, left+binwidth)).T.ravel(), np.repeat(H, 2), '-',
color=rainbow[ne-1 % len(rainbow)], label='%i exp' % ne)
plt.title('%s band' % band)
plt.xlabel('Galaxy depth (mag)')
plt.legend(loc='upper left')
ps.savefig()
left2 = dlo + np.arange(depthbins2) * (dhi-dlo) / float(depthbins2)
binwidth2 = left2[1]-left2[0]
for band in bands:
hsum = 0
for ne in range(1,10):
key = (ne, band)
if not key in depth_hists_2:
continue
hsum += depth_hists_2[key]
print('%s band:' % band)
print('Total number of counts in histogram:', sum(hsum))
# [-1::-1] = reversed
hsum = np.cumsum(hsum[-1::-1])[-1::-1]
hsum = hsum * 100. / float(N)
plt.clf()
#plt.plot(left2+binwidth/2, hsum, 'k-')
plt.plot(left2, hsum, 'k-')
plt.xlabel('Galaxy depth (mag)')
plt.ylabel('Cumulative fraction (%)')
plt.title('%s band' % band)
# 90% to full depth
# 95% to full depth - 0.3 mag
# 98% to full depth - 0.6 mag
for y,x,c in [
(90, targets[band], 'r'),
(95, targets[band] - 0.3, 'g'),
(98, targets[band] - 0.6, 'b')]:
xf = (x - dlo) / (dhi - dlo)
yf = y / 100.
plt.axvline(x, ymax=yf, color=c, lw=3, alpha=0.3)
plt.axhline(y, xmax=xf, color=c, lw=3, alpha=0.3)
#mid = left2 + binwidth2/2
print('target : %.1f %% deeper than %.2f' % (y, x))
# ??? left or midpoint?
ibin = np.flatnonzero(left2 > x)[0]
pct = hsum[ibin]
status = ' '*20
if pct >= y:
status += '-> pass'
else:
status += '-> FAIL'
plt.axhline(pct, color=c, alpha=0.5) #xmax=xf,
print('achieved: %.1f %% %s' % (pct, status)) # deeper than %.2f' % (hsum[ibin], x))
ii = np.flatnonzero(hsum > y)
if len(ii):
ibin = ii[-1]
print(' %.1f %% deeper than %.2f' % (y, left2[ibin]))
else:
print(' Total coverage only %.1f %%' % max(hsum))
print()
plt.xlim(dlo, dhi)
plt.ylim(0., 100.)
ps.savefig()
def format_catalog(T,
hdr,
primhdr,
allbands,
outfn,
in_flux_prefix='',
flux_prefix=''):
# Retrieve the bands in this catalog.
bands = []
for i in range(10):
b = primhdr.get('BAND%i' % i)
if b is None:
break
b = b.strip()
bands.append(b)
print('Bands in this catalog:', bands)
primhdr.add_record(
dict(name='ALLBANDS',
value=allbands,
comment='Band order in array values'))
has_wise = 'wise_flux' in T.columns()
has_wise_lc = 'wise_lc_flux' in T.columns()
has_ap = 'apflux' in T.columns()
# Expand out FLUX and related fields.
B = np.array([allbands.index(band) for band in bands])
keys = [
'flux', 'flux_ivar', 'rchi2', 'fracflux', 'fracmasked', 'fracin',
'nobs', 'anymask', 'allmask', 'psfsize', 'depth', 'galdepth'
]
if has_ap:
keys.extend(['apflux', 'apflux_resid', 'apflux_ivar'])
for k in keys:
incol = '%s%s' % (in_flux_prefix, k)
X = T.get(incol)
# print('Column', k, 'has shape', X.shape)
# apflux array columns...
sh = X.shape
if len(sh) == 3:
nt, nb, N = sh
A = np.zeros((len(T), len(allbands), N), X.dtype)
A[:, B, :] = X
else:
A = np.zeros((len(T), len(allbands)), X.dtype)
A[:, B] = X
T.delete_column(incol)
T.set('%s%s' % (flux_prefix, k), A)
from tractor.sfd import SFDMap
print('Reading SFD maps...')
sfd = SFDMap()
filts = ['%s %s' % ('DES', f) for f in allbands]
wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4']
ebv, ext = sfd.extinction(filts + wisebands, T.ra, T.dec, get_ebv=True)
T.ebv = ebv.astype(np.float32)
ext = ext.astype(np.float32)
decam_ext = ext[:, :len(allbands)]
T.decam_mw_transmission = 10.**(-decam_ext / 2.5)
if has_wise:
wise_ext = ext[:, len(allbands):]
T.wise_mw_transmission = 10.**(-wise_ext / 2.5)
# Column ordering...
cols = [
'brickid',
'brickname',
'objid',
'brick_primary',
'blob',
'ninblob',
'tycho2inblob',
'type',
'ra',
'ra_ivar',
'dec',
'dec_ivar',
'bx',
'by',
'bx0',
'by0',
'left_blob',
'out_of_bounds',
'dchisq',
'ebv',
'cpu_source',
'cpu_blob',
'blob_width',
'blob_height',
'blob_npix',
'blob_nimages',
'blob_totalpix',
'mjd_min',
'mjd_max',
]
cols.extend([flux_prefix + c for c in ['flux', 'flux_ivar']])
if has_ap:
cols.extend([
flux_prefix + c for c in ['apflux', 'apflux_resid', 'apflux_ivar']
])
cols.append('decam_mw_transmission')
cols.extend([
flux_prefix + c for c in [
'nobs', 'rchi2', 'fracflux', 'fracmasked', 'fracin', 'anymask',
'allmask', 'psfsize', 'depth', 'galdepth'
]
])
if has_wise:
cols.extend([
'wise_coadd_id', 'wise_flux', 'wise_flux_ivar', 'wise_mask',
'wise_mw_transmission', 'wise_nobs', 'wise_fracflux', 'wise_rchi2'
])
if has_wise_lc:
cols.extend([
'wise_lc_flux', 'wise_lc_flux_ivar', 'wise_lc_nobs',
'wise_lc_fracflux', 'wise_lc_rchi2', 'wise_lc_mjd'
])
cols.extend([
'fracdev',
'fracdev_ivar',
'shapeexp_r',
'shapeexp_r_ivar',
'shapeexp_e1',
'shapeexp_e1_ivar',
'shapeexp_e2',
'shapeexp_e2_ivar',
'shapedev_r',
'shapedev_r_ivar',
'shapedev_e1',
'shapedev_e1_ivar',
'shapedev_e2',
'shapedev_e2_ivar',
])
print('Columns:', cols)
print('T columns:', T.columns())
# match case to T.
cc = T.get_columns()
cclower = [c.lower() for c in cc]
for i, c in enumerate(cols):
if (not c in cc) and c in cclower:
j = cclower.index(c)
cols[i] = cc[j]
# Units
deg = 'deg'
degiv = '1/deg^2'
flux = 'nanomaggy'
fluxiv = '1/nanomaggy^2'
units = dict(ra=deg,
dec=deg,
ra_ivar=degiv,
dec_ivar=degiv,
ebv='mag',
wise_flux=flux,
wise_flux_ivar=fluxiv,
wise_lc_flux=flux,
wise_lc_flux_ivar=fluxiv,
shapeexp_r='arcsec',
shapeexp_r_ivar='1/arcsec^2',
shapedev_r='arcsec',
shapedev_r_ivar='1/arcsec^2')
# Fields that take prefixes
funits = dict(flux=flux,
flux_ivar=fluxiv,
apflux=flux,
apflux_ivar=fluxiv,
apflux_resid=flux,
depth=fluxiv,
galdepth=fluxiv)
units.update([('%s%s' % (flux_prefix, k), v) for k, v in funits.items()])
# Reformat as list aligned with cols
units = [units.get(c, '') for c in cols]
T.writeto(outfn, columns=cols, header=hdr, primheader=primhdr, units=units)
def format_catalog(self,
T,
allbands,
outfn,
in_flux_prefix='',
flux_prefix='',
write=False):
# args: hdr, primhdr
# Retrieve the bands in this catalog.
bands = allbands
print('Bands in this catalog:', bands)
has_wise = 'wise_flux' in T.columns()
has_wise_lc = 'wise_lc_flux' in T.columns()
has_ap = 'decam_apflux' in T.columns()
# Expand out FLUX and related fields from grz arrays to 'allbands'
# (eg, ugrizY) arrays.
B = np.array([allbands.index(band) for band in bands])
keys = [
'flux', 'flux_ivar', 'rchi2', 'fracflux', 'fracmasked', 'fracin',
'nobs', 'anymask', 'allmask', 'psfsize', 'depth', 'galdepth'
]
if has_ap:
keys.extend(['apflux', 'apflux_resid', 'apflux_ivar'])
for k in keys:
incol = '%s%s' % (in_flux_prefix, k)
X = T.get(incol)
# Handle array columns (eg, apflux)
sh = X.shape
if len(sh) == 3:
nt, nb, N = sh
A = np.zeros((len(T), len(allbands), N), X.dtype)
A[:, B, :] = X
else:
A = np.zeros((len(T), len(allbands)), X.dtype)
# If there is only one band, these can show up as scalar arrays.
if len(sh) == 1:
A[:, B] = X[:, np.newaxis]
else:
A[:, B] = X
T.delete_column(incol)
# FLUX_b for each band, rather than array columns.
for i, b in enumerate(allbands):
T.set('%s%s_%s' % (flux_prefix, k, b), A[:, i])
# WISE
wise_prefix = 'wise_'
wise_keys = []
if has_wise:
wise_keys.extend(
['flux', 'flux_ivar', 'nobs', 'rchi2', 'fracflux'])
if has_wise_lc:
wise_keys.extend([
'lc_flux', 'lc_flux_ivar', 'lc_nobs', 'lc_rchi2',
'lc_fracflux', 'lc_mjd'
])
for k in wise_keys:
if 'lc_' in k:
wisebands = ['w1', 'w2']
else:
wisebands = ['w1', 'w2', 'w3', 'w4']
B_wise = np.array([wisebands.index(band) for band in wisebands])
incol = '%s%s' % (wise_prefix, k)
#raise ValueError
X = T.get(incol)
# Handle array columns (eg, apflux)
sh = X.shape
if len(sh) == 3:
nt, nb, N = sh
A = np.zeros((len(T), len(wisebands), N), X.dtype)
A[:, B_wise, :] = X
else:
A = np.zeros((len(T), len(wisebands)), X.dtype)
# If there is only one band, these can show up as scalar arrays.
if len(sh) == 1:
A[:, B_wise] = X[:, np.newaxis]
else:
A[:, B_wise] = X
T.delete_column(incol)
# FLUX_b for each band, rather than array columns.
for i, b in enumerate(wisebands):
T.set('%s%s_%s' % ('', k, b), A[:, i])
### Done WISE
from tractor.sfd import SFDMap
print('Reading SFD maps...')
sfd = SFDMap()
filts = ['%s %s' % ('DES', f) for f in allbands]
wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4']
ebv, ext = sfd.extinction(filts + wisebands, T.ra, T.dec, get_ebv=True)
T.ebv = ebv.astype(np.float32)
ext = ext.astype(np.float32)
decam_ext = ext[:, :len(allbands)]
if has_wise:
wise_ext = ext[:, len(allbands):]
wbands = ['w1', 'w2', 'w3', 'w4']
trans_cols_opt = []
trans_cols_wise = []
# No MW_TRANSMISSION_* columns at all
for i, b in enumerate(allbands):
col = 'mw_transmission_%s' % b
T.set(col, 10.**(-decam_ext[:, i] / 2.5))
trans_cols_opt.append(col)
if has_wise:
for i, b in enumerate(wbands):
col = 'mw_transmission_%s' % b
T.set(col, 10.**(-wise_ext[:, i] / 2.5))
trans_cols_wise.append(col)
from legacypipe.survey import release_number
T.release = np.zeros(len(T), np.int16) + release_number
# Column ordering...
cols = [
'release', 'brickid', 'brickname', 'objid', 'brick_primary',
'type', 'ra', 'dec', 'ra_ivar', 'dec_ivar', 'bx', 'by', 'dchisq',
'ebv', 'mjd_min', 'mjd_max'
]
def add_fluxlike(c):
for b in allbands:
cols.append('%s%s_%s' % (flux_prefix, c, b))
def add_wiselike(c, bands=wbands):
for b in bands:
cols.append('%s_%s' % (c, b))
add_fluxlike('flux')
if has_wise:
add_wiselike('flux')
add_fluxlike('flux_ivar')
if has_wise:
add_wiselike('flux_ivar')
if has_ap:
for c in ['apflux', 'apflux_resid', 'apflux_ivar']:
add_fluxlike(c)
cols.extend(trans_cols_opt)
cols.extend(trans_cols_wise)
for c in ['nobs', 'rchisq', 'fracflux']:
add_fluxlike(c)
if has_wise:
add_wiselike(c)
for c in ['fracmasked', 'fracin', 'anymask', 'allmask']:
add_fluxlike(c)
if has_wise:
pass # Only DR4 on has a wise mask
#for i,b in enumerate(wbands[:2]):
# col = 'wisemask_%s' % (b)
# T.set(col, T.wise_mask[:,i])
# cols.append(col)
for c in ['psfsize', 'psfdepth', 'galdepth']:
add_fluxlike(c)
if has_wise:
cols.append('wise_coadd_id')
if has_wise_lc:
for c in [
'lc_flux', 'lc_flux_ivar', 'lc_nobs', 'lc_fracflux',
'lc_rchisq', 'lc_mjd'
]:
add_wiselike(c, bands=['w1', 'w2'])
cols.extend([
'fracdev',
'fracdev_ivar',
'shapeexp_r',
'shapeexp_r_ivar',
'shapeexp_e1',
'shapeexp_e1_ivar',
'shapeexp_e2',
'shapeexp_e2_ivar',
'shapedev_r',
'shapedev_r_ivar',
'shapedev_e1',
'shapedev_e1_ivar',
'shapedev_e2',
'shapedev_e2_ivar',
])
# match case to T.
cc = T.get_columns()
cclower = [c.lower() for c in cc]
for i, c in enumerate(cols):
if (not c in cc) and c in cclower:
j = cclower.index(c)
cols[i] = cc[j]
# Units
deg = 'deg'
degiv = '1/deg^2'
arcsec = 'arcsec'
flux = 'nanomaggy'
fluxiv = '1/nanomaggy^2'
units = dict(ra=deg,
dec=deg,
ra_ivar=degiv,
dec_ivar=degiv,
ebv='mag',
shapeexp_r=arcsec,
shapeexp_r_ivar='1/arcsec^2',
shapedev_r=arcsec,
shapedev_r_ivar='1/arcsec^2')
# WISE fields
wunits = dict(flux=flux,
flux_ivar=fluxiv,
lc_flux=flux,
lc_flux_ivar=fluxiv)
# Fields that take prefixes (and have bands)
funits = dict(flux=flux,
flux_ivar=fluxiv,
apflux=flux,
apflux_ivar=fluxiv,
apflux_resid=flux,
psfdepth=fluxiv,
galdepth=fluxiv,
psfsize=arcsec)
# add prefixes
units.update([('%s%s' % (flux_prefix, k), v)
for k, v in funits.items()])
# add bands
for b in allbands:
units.update([('%s%s_%s' % (flux_prefix, k, b), v)
for k, v in funits.items()])
# add WISE bands
for b in wbands:
units.update([('%s_%s' % (k, b), v) for k, v in wunits.items()])
# Create a list of units aligned with 'cols'
units = [units.get(c, '') for c in cols]
# Cleanup
# Catalogue
for k_new, k_old in zip(['psfdepth', 'rchisq'], ['depth', 'rchi2']):
for b in bands:
T.rename('%s_%s' % (k_old, b), '%s_%s' % (k_new, b))
#old_col= '%s_%s' % (k_old,b)
#if old_col in T.get_columns():
# T.rename(old_col,'%s_%s' % (k_new,b))
#else:
# print('warning'.upper() + ' %s not in fits_table' % old_col)
for k_new, k_old in zip(['rchisq'], ['rchi2']):
for b in ['w1', 'w2', 'w3', 'w4']:
T.rename('%s_%s' % (k_old, b), '%s_%s' % (k_new, b))
for k_new, k_old in zip(['lc_rchisq'], ['lc_rchi2']):
for b in ['w1', 'w2']:
T.rename('%s_%s' % (k_old, b), '%s_%s' % (k_new, b))
for d_key in ['decam_mw_transmission']:
if d_key in T.get_columns():
T.delete_column(d_key)
# ONLY in DR4 on
for col in ['mjd_min', 'mjd_max', 'wise_coadd_id']:
if col in cols:
i = cols.index(col)
cols.pop(i)
units.pop(i)
# check that cleanup worked
for col in cols:
if not col in T.get_columns():
raise KeyError('%s in col but not in T' % col)
#T.writeto(outfn, columns=cols, header=hdr, primheader=primhdr, units=units)
if write:
T.writeto(outfn, columns=cols, units=units)
print('Wrote %s' % outfn)
else:
print('Skipping writing of %s' % outfn)
return T
def format_catalog(T,
hdr,
primhdr,
allbands,
outfn,
in_flux_prefix='',
flux_prefix='',
write_kwargs={},
N_wise_epochs=None,
motions=True,
gaia_tagalong=False):
# Retrieve the bands in this catalog.
bands = []
for i in range(10):
b = primhdr.get('BAND%i' % i)
if b is None:
break
b = b.strip()
bands.append(b)
print('Bands in this catalog:', bands)
has_wise = 'flux_w1' in T.columns()
has_wise_lc = 'lc_flux_w1' in T.columns()
has_ap = 'apflux' in T.columns()
# Nans,Infs
# Ivar -> 0
ivar_nans = [
'ra_ivar', 'dec_ivar', 'shapeexp_r_ivar', 'shapeexp_e1_ivar',
'shapeexp_e2_ivar'
]
for key in ivar_nans:
ind = np.isfinite(T.get(key)) == False
if np.any(ind):
T.get(key)[ind] = 0.
# Other --> NaN (PostgreSQL can work with NaNs)
other_nans = ['dchisq', 'rchisq', 'mjd_min', 'mjd_max']
for key in other_nans:
ind = np.isfinite(T.get(key)) == False
if np.any(ind):
T.get(key)[ind] = np.nan
# Expand out FLUX and related fields from grz arrays to 'allbands'
# (eg, ugrizY) arrays.
B = np.array([allbands.index(band) for band in bands])
keys = [
'flux', 'flux_ivar', 'rchisq', 'fracflux', 'fracmasked', 'fracin',
'nobs', 'anymask', 'allmask', 'psfsize', 'psfdepth', 'galdepth'
]
if has_ap:
keys.extend(['apflux', 'apflux_resid', 'apflux_ivar'])
for k in keys:
incol = '%s%s' % (in_flux_prefix, k)
X = T.get(incol)
# Handle array columns (eg, apflux)
sh = X.shape
if len(sh) == 3:
nt, nb, N = sh
A = np.zeros((len(T), len(allbands), N), X.dtype)
A[:, B, :] = X
else:
A = np.zeros((len(T), len(allbands)), X.dtype)
# If there is only one band, these can show up as scalar arrays.
if len(sh) == 1:
A[:, B] = X[:, np.newaxis]
else:
A[:, B] = X
T.delete_column(incol)
# FLUX_b for each band, rather than array columns.
for i, b in enumerate(allbands):
T.set('%s%s_%s' % (flux_prefix, k, b), A[:, i])
from tractor.sfd import SFDMap
print('Reading SFD maps...')
sfd = SFDMap()
filts = ['%s %s' % ('DES', f) for f in allbands]
wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4']
ebv, ext = sfd.extinction(filts + wisebands, T.ra, T.dec, get_ebv=True)
T.ebv = ebv.astype(np.float32)
ext = ext.astype(np.float32)
decam_ext = ext[:, :len(allbands)]
if has_wise:
wise_ext = ext[:, len(allbands):]
wbands = ['w1', 'w2', 'w3', 'w4']
trans_cols_opt = []
trans_cols_wise = []
# No MW_TRANSMISSION_* columns at all
for i, b in enumerate(allbands):
col = 'mw_transmission_%s' % b
T.set(col, 10.**(-decam_ext[:, i] / 2.5))
trans_cols_opt.append(col)
if has_wise:
for i, b in enumerate(wbands):
col = 'mw_transmission_%s' % b
T.set(col, 10.**(-wise_ext[:, i] / 2.5))
trans_cols_wise.append(col)
from legacypipe.survey import release_number
T.release = np.zeros(len(T), np.int16) + release_number
# Column ordering...
cols = [
'release', 'brickid', 'brickname', 'objid', 'brick_primary',
'brightstarinblob', 'type', 'ra', 'dec', 'ra_ivar', 'dec_ivar', 'bx',
'by', 'dchisq', 'ebv', 'mjd_min', 'mjd_max', 'ref_cat', 'ref_id'
]
if motions:
cols.extend([
'pmra',
'pmdec',
'parallax',
'pmra_ivar',
'pmdec_ivar',
'parallax_ivar',
'ref_epoch',
])
# Add zero columns if these are missing (eg, if there are no
# Tycho-2 or Gaia stars in the brick).
tcols = T.get_columns()
for c in cols:
if not c in tcols:
T.set(c, np.zeros(len(T), np.float32))
if gaia_tagalong:
gaia_cols = [ #('source_id', np.int64), # already have this in ref_id
('pointsource', bool), # did we force it to be a point source?
('phot_g_mean_mag', np.float32),
('phot_g_mean_flux_over_error', np.float32),
('phot_g_n_obs', np.int16),
('phot_bp_mean_mag', np.float32),
('phot_bp_mean_flux_over_error', np.float32),
('phot_bp_n_obs', np.int16),
('phot_rp_mean_mag', np.float32),
('phot_rp_mean_flux_over_error', np.float32),
('phot_rp_n_obs', np.int16),
('phot_variable_flag', bool),
('astrometric_excess_noise', np.float32),
('astrometric_excess_noise_sig', np.float32),
('astrometric_n_obs_al', np.int16),
('astrometric_n_good_obs_al', np.int16),
('astrometric_weight_al', np.float32),
('duplicated_source', bool),
]
tcols = T.get_columns()
for c, t in gaia_cols:
gc = 'gaia_' + c
if not c in tcols:
T.set(gc, np.zeros(len(T), t))
else:
T.rename(c, gc)
cols.append(gc)
def add_fluxlike(c):
for b in allbands:
cols.append('%s%s_%s' % (flux_prefix, c, b))
def add_wiselike(c, bands=wbands):
for b in bands:
cols.append('%s_%s' % (c, b))
add_fluxlike('flux')
if has_wise:
add_wiselike('flux')
add_fluxlike('flux_ivar')
if has_wise:
add_wiselike('flux_ivar')
if has_ap:
for c in ['apflux', 'apflux_resid', 'apflux_ivar']:
add_fluxlike(c)
cols.extend(trans_cols_opt)
cols.extend(trans_cols_wise)
for c in ['nobs', 'rchisq', 'fracflux']:
add_fluxlike(c)
if has_wise:
add_wiselike(c)
for c in ['fracmasked', 'fracin', 'anymask', 'allmask']:
add_fluxlike(c)
if has_wise:
for i, b in enumerate(wbands[:2]):
col = 'wisemask_%s' % (b)
T.set(col, T.wise_mask[:, i])
cols.append(col)
for c in ['psfsize', 'psfdepth', 'galdepth']:
add_fluxlike(c)
if has_wise:
cols.append('wise_coadd_id')
if has_wise_lc:
lc_cols = [
'lc_flux', 'lc_flux_ivar', 'lc_nobs', 'lc_fracflux', 'lc_rchisq',
'lc_mjd'
]
for c in lc_cols:
add_wiselike(c, bands=['w1', 'w2'])
# Cut down to a fixed number of WISE time-resolved epochs?
if N_wise_epochs is not None:
for col in lc_cols:
for band in ['w1', 'w2']:
colname = col + '_' + band
# Cut or pad old value to have N_wise_epochs-length arrays
oldval = T.get(colname)
n, ne = oldval.shape
newval = np.zeros((n, N_wise_epochs), oldval.dtype)
ncopy = min(N_wise_epochs, ne)
newval[:, :ncopy] = oldval[:, :ncopy]
T.set(colname, newval)
cols.extend([
'fracdev',
'fracdev_ivar',
'shapeexp_r',
'shapeexp_r_ivar',
'shapeexp_e1',
'shapeexp_e1_ivar',
'shapeexp_e2',
'shapeexp_e2_ivar',
'shapedev_r',
'shapedev_r_ivar',
'shapedev_e1',
'shapedev_e1_ivar',
'shapedev_e2',
'shapedev_e2_ivar',
])
print('Columns:', cols)
print('T columns:', T.columns())
# match case to T.
cc = T.get_columns()
cclower = [c.lower() for c in cc]
for i, c in enumerate(cols):
if (not c in cc) and c in cclower:
j = cclower.index(c)
cols[i] = cc[j]
# Units
deg = 'deg'
degiv = '1/deg^2'
arcsec = 'arcsec'
flux = 'nanomaggy'
fluxiv = '1/nanomaggy^2'
units = dict(ra=deg,
dec=deg,
ra_ivar=degiv,
dec_ivar=degiv,
ebv='mag',
shapeexp_r=arcsec,
shapeexp_r_ivar='1/arcsec^2',
shapedev_r=arcsec,
shapedev_r_ivar='1/arcsec^2')
# WISE fields
wunits = dict(flux=flux,
flux_ivar=fluxiv,
lc_flux=flux,
lc_flux_ivar=fluxiv)
# Fields that take prefixes (and have bands)
funits = dict(flux=flux,
flux_ivar=fluxiv,
apflux=flux,
apflux_ivar=fluxiv,
apflux_resid=flux,
psfdepth=fluxiv,
galdepth=fluxiv,
psfsize=arcsec)
# add prefixes
units.update([('%s%s' % (flux_prefix, k), v) for k, v in funits.items()])
# add bands
for b in allbands:
units.update([('%s%s_%s' % (flux_prefix, k, b), v)
for k, v in funits.items()])
# add WISE bands
for b in wbands:
units.update([('%s_%s' % (k, b), v) for k, v in wunits.items()])
# Create a list of units aligned with 'cols'
units = [units.get(c, '') for c in cols]
T.writeto(outfn,
columns=cols,
header=hdr,
primheader=primhdr,
units=units,
**write_kwargs)
def main(cmdlineargs=None, get_copilot=False):
global gSFD
import optparse
parser = optparse.OptionParser(usage='%prog')
# Mosaic or Decam?
from camera import (nominal_cal, ephem_observer, default_extension,
tile_path)
nom = nominal_cal
obs = ephem_observer()
plotfn_default = 'recent.png'
parser.add_option('--ext', default=default_extension,
help='Extension to read for computing observing conditions: default %default')
parser.add_option('--extnum', type=int, help='Integer extension to read')
parser.add_option('--rawdata', help='Directory to monitor for new images: default $MOS3_DATA if set, else "rawdata"', default=None)
parser.add_option('--n-fwhm', default=None, type=int, help='Number of stars on which to measure FWHM')
parser.add_option('--no-db', dest='db', default=True, action='store_false',
help='Do not append results to database')
parser.add_option('--no-focus', dest='focus', default=True,
action='store_false', help='Do not analyze focus frames')
parser.add_option('--fits', help='Write database to given FITS table')
parser.add_option('--plot', action='store_true',
help='Plot recent data and quit')
parser.add_option('--plot-filename', default=None,
help='Save plot to given file, default %s' % plotfn_default)
parser.add_option('--nightplot', '--night', action='store_true',
help="Plot tonight's data and quit")
parser.add_option('--qa-plots', dest='doplots', default=False,
action='store_true', help='Create QA plots')
parser.add_option('--keep-plots', action='store_true',
help='Do not remove PNG-format plots (normally merged into PDF)')
parser.add_option('--mjdstart', type=float, default=None,
help='MJD (UTC) at which to start plot')
now = mjdnow()
parser.add_option('--mjdend', type=float, default=None,
help='MJD (UTC) at which to end plot (default: now, which is %.3f)' % now)
parser.add_option('--skip', action='store_true',
help='Skip images that already exist in the database')
parser.add_option('--threads', type=int, default=None,
help='Run multi-threaded when processing list of files on command-line')
parser.add_option('--fix-db', action='store_true')
parser.add_option('--tiles', default=tile_path,
help='Tiles table, default %default')
parser.add_option('--no-show', dest='show', default=True, action='store_false',
help='Do not show plot window, just save it.')
if cmdlineargs is None:
opt,args = parser.parse_args()
else:
opt,args = parser.parse_args(cmdlineargs)
if not opt.show:
import matplotlib
matplotlib.use('Agg')
imagedir = opt.rawdata
if imagedir is None:
imagedir = os.environ.get('MOS3_DATA', 'rawdata')
rawext = opt.ext
if opt.extnum is not None:
rawext = opt.extnum
assert(rawext is not None)
from astrometry.util.fits import fits_table
tiles = fits_table(opt.tiles)
from django.conf import settings
import obsdb
import pylab as plt
plt.figure(figsize=(8,10))
markmjds = []
if opt.nightplot:
opt.plot = True
if opt.plot_filename is None:
opt.plot_filename = 'night.png'
# Are we at Tololo or Kitt Peak? Look for latest image.
o = obsdb.MeasuredCCD.objects.all().order_by('-mjd_obs')
cam = o[0].camera
print('Camera:', cam)
if opt.mjdstart is not None:
sdate = ephem.Date(mjdtodate(opt.mjdend))
else:
sdate = ephem.Date(datenow())
(sunset, eve12, eve18, morn18, morn12, sunrise) = get_twilight(
cam, sdate)
if opt.mjdstart is None:
opt.mjdstart = ephemdate_to_mjd(sunset)
print('Set mjd start to sunset:', sunset, opt.mjdstart)
if opt.mjdend is None:
opt.mjdend = ephemdate_to_mjd(sunrise)
print('Set mjd end to sunrise', sunrise, opt.mjdend)
markmjds.append((ephemdate_to_mjd(eve18),'b'))
print('Evening twi18:', eve18, markmjds[-1])
markmjds.append((ephemdate_to_mjd(morn18),'b'))
print('Morning twi18:', morn18, markmjds[-1])
markmjds.append((ephemdate_to_mjd(eve12),'g'))
print('Evening twi12:', eve12, markmjds[-1])
markmjds.append((ephemdate_to_mjd(morn12),'g'))
print('Morning twi12:', morn12, markmjds[-1])
if opt.plot_filename is None:
opt.plot_filename = plotfn_default
if opt.fits:
ccds = obsdb.MeasuredCCD.objects.all()
print(ccds.count(), 'measured CCDs')
T = db_to_fits(ccds)
T.writeto(opt.fits)
print('Wrote', opt.fits)
return 0
if opt.fix_db:
from astrometry.util.fits import fits_table
tiles = fits_table('obstatus/mosaic-tiles_obstatus.fits')
now = mjdnow()
#ccds = obsdb.MeasuredCCD.objects.all()
#ccds = obsdb.MeasuredCCD.objects.all().filter(mjd_obs__gt=now - 0.25)
ccds = obsdb.MeasuredCCD.objects.all().filter(mjd_obs__gt=57434)
print(ccds.count(), 'measured CCDs')
for ccd in ccds:
try:
hdr = fitsio.read_header(ccd.filename, ext=0)
# band = hdr['FILTER']
# band = band.split()[0]
# ccd.band = band
set_tile_fields(ccd, hdr, tiles)
ccd.save()
print('Fixed', ccd.filename)
except:
import traceback
traceback.print_exc()
return 0
if opt.plot:
plot_recent(opt, nom, tiles=tiles, markmjds=markmjds, show_plot=False)
return 0
print('Loading SFD maps...')
sfd = SFDMap()
if len(args) > 0:
mp = None
if opt.threads > 1:
gSFD = sfd
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads)
if opt.skip:
fns = skip_existing_files(args, rawext)
else:
fns = args
if mp is None:
for fn in fns:
process_image(fn, rawext, nom, sfd, opt, obs, tiles)
else:
sfd = None
mp.map(bounce_process_image,
[(fn, rawext, nom, sfd, opt, obs, tiles) for fn in fns])
plot_recent(opt, nom, tiles=tiles, markmjds=markmjds, show_plot=False)
return 0
copilot = Copilot(imagedir, rawext, opt, nom, sfd, obs, tiles)
# for testability
if get_copilot:
return copilot
copilot.run()
return 0
plt.clf()
plt.plot(C.ra, C.dec, 'bo', alpha=0.2)
plt.savefig('rd.png')
# Find the unique exposures (not CCDs), save as E.
C.galnorm = C.galnorm_mean
nil, I = np.unique(C.expnum, return_index=True)
E = C[I]
print(len(E), 'exposures')
if False:
# Find the extinction in the center of the COSMOS region and apply it
# as a correction to our target depths (so that we reach that depth
# for de-reddened mags).
print('Reading SFD maps...')
sfd = SFDMap()
filts = ['%s %s' % ('DES', f) for f in bands]
ext = sfd.extinction(filts, 150., 2.2)
print('Extinction:', ext)
# -> Extinction: [[ 0.06293296 0.04239261 0.02371245]]
E.index = np.arange(len(E))
E.passnum = np.zeros(len(E), np.uint8)
E.depthfraction = np.zeros(len(E), np.float32)
# Compute which pass number each exposure would be called.
zp0 = DecamImage.nominal_zeropoints()
# HACK -- copied from obsbot
kx = dict(
g=0.178,
r=0.094,
def format_catalog(T,
hdr,
primhdr,
bands,
allbands,
outfn,
release,
write_kwargs=None,
N_wise_epochs=None,
motions=True,
gaia_tagalong=False):
if write_kwargs is None:
write_kwargs = {}
has_wise = 'flux_w1' in T.columns()
has_wise_lc = 'lc_flux_w1' in T.columns()
has_galex = 'flux_nuv' in T.columns()
has_ap = 'apflux' in T.columns()
# Nans,Infs
# Ivar -> 0
ivar_nans = [
'ra_ivar', 'dec_ivar', 'shape_r_ivar', 'shape_e1_ivar', 'shape_e2_ivar'
]
for key in ivar_nans:
ind = np.isfinite(T.get(key)) == False
if np.any(ind):
T.get(key)[ind] = 0.
# Other --> NaN (PostgreSQL can work with NaNs)
other_nans = ['dchisq', 'rchisq', 'mjd_min', 'mjd_max']
for key in other_nans:
ind = np.isfinite(T.get(key)) == False
if np.any(ind):
T.get(key)[ind] = np.nan
# Expand out FLUX and related fields from grz arrays to 'allbands'
# (eg, ugrizY) arrays.
keys = [
'flux', 'flux_ivar', 'rchisq', 'fracflux', 'fracmasked', 'fracin',
'nobs', 'anymask', 'allmask', 'psfsize', 'psfdepth', 'galdepth',
'fiberflux', 'fibertotflux'
]
if has_ap:
keys.extend([
'apflux', 'apflux_resid', 'apflux_blobresid', 'apflux_ivar',
'apflux_masked'
])
_expand_flux_columns(T, bands, allbands, keys)
from tractor.sfd import SFDMap
info('Reading SFD maps...')
sfd = SFDMap()
filts = ['%s %s' % ('DES', f) for f in allbands]
wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4']
ebv, ext = sfd.extinction(filts + wisebands, T.ra, T.dec, get_ebv=True)
T.ebv = ebv.astype(np.float32)
ext = ext.astype(np.float32)
decam_ext = ext[:, :len(allbands)]
if has_wise:
wise_ext = ext[:, len(allbands):]
wbands = ['w1', 'w2', 'w3', 'w4']
gbands = ['nuv', 'fuv']
trans_cols_opt = []
trans_cols_wise = []
for i, b in enumerate(allbands):
col = 'mw_transmission_%s' % b
T.set(col, 10.**(-decam_ext[:, i] / 2.5))
trans_cols_opt.append(col)
if has_wise:
for i, b in enumerate(wbands):
col = 'mw_transmission_%s' % b
T.set(col, 10.**(-wise_ext[:, i] / 2.5))
trans_cols_wise.append(col)
T.release = np.zeros(len(T), np.int16) + release
# Column ordering...
cols = [
'release', 'brickid', 'brickname', 'objid', 'brick_primary',
'maskbits', 'fitbits', 'type', 'ra', 'dec', 'ra_ivar', 'dec_ivar',
'bx', 'by', 'dchisq', 'ebv', 'mjd_min', 'mjd_max', 'ref_cat', 'ref_id'
]
if motions:
cols.extend([
'pmra',
'pmdec',
'parallax',
'pmra_ivar',
'pmdec_ivar',
'parallax_ivar',
'ref_epoch',
])
# Add zero columns if these are missing (eg, if there are no
# Tycho-2 or Gaia stars in the brick).
tcols = T.get_columns()
for c in cols:
if not c in tcols:
T.set(c, np.zeros(len(T), np.float32))
if gaia_tagalong:
gaia_cols = [
('phot_g_mean_mag', np.float32),
('phot_g_mean_flux_over_error', np.float32),
('phot_g_n_obs', np.int16),
('phot_bp_mean_mag', np.float32),
('phot_bp_mean_flux_over_error', np.float32),
('phot_bp_n_obs', np.int16),
('phot_rp_mean_mag', np.float32),
('phot_rp_mean_flux_over_error', np.float32),
('phot_rp_n_obs', np.int16),
('phot_variable_flag', bool),
('astrometric_excess_noise', np.float32),
('astrometric_excess_noise_sig', np.float32),
('astrometric_n_obs_al', np.int16),
('astrometric_n_good_obs_al', np.int16),
('astrometric_weight_al', np.float32),
('duplicated_source', bool),
('a_g_val', np.float32),
('e_bp_min_rp_val', np.float32),
('phot_bp_rp_excess_factor', np.float32),
('astrometric_sigma5d_max', np.float32),
('astrometric_params_solved', np.uint8),
]
tcols = T.get_columns()
for c, t in gaia_cols:
gc = 'gaia_' + c
if not c in tcols:
T.set(gc, np.zeros(len(T), t))
else:
T.rename(c, gc)
cols.append(gc)
def add_fluxlike(c):
for b in allbands:
cols.append('%s_%s' % (c, b))
def add_wiselike(c, bands=None):
if bands is None:
bands = wbands
for b in bands:
cols.append('%s_%s' % (c, b))
def add_galexlike(c):
for b in gbands:
cols.append('%s_%s' % (c, b))
add_fluxlike('flux')
if has_wise:
add_wiselike('flux')
if has_galex:
add_galexlike('flux')
add_fluxlike('flux_ivar')
if has_wise:
add_wiselike('flux_ivar')
if has_galex:
add_galexlike('flux_ivar')
add_fluxlike('fiberflux')
add_fluxlike('fibertotflux')
if has_ap:
for c in [
'apflux', 'apflux_resid', 'apflux_blobresid', 'apflux_ivar',
'apflux_masked'
]:
add_fluxlike(c)
if has_wise and has_ap and 'apflux_w1' in T.get_columns():
add_wiselike('apflux')
add_wiselike('apflux_resid')
add_wiselike('apflux_ivar')
if has_galex and has_ap and 'apflux_nuv' in T.get_columns():
add_galexlike('apflux')
add_galexlike('apflux_resid')
add_galexlike('apflux_ivar')
cols.extend(trans_cols_opt)
cols.extend(trans_cols_wise)
for c in ['nobs', 'rchisq', 'fracflux']:
add_fluxlike(c)
if has_wise:
add_wiselike(c)
for c in ['fracmasked', 'fracin', 'anymask', 'allmask']:
add_fluxlike(c)
if has_wise:
for i, b in enumerate(wbands[:2]):
col = 'wisemask_%s' % (b)
T.set(col, T.wise_mask[:, i])
cols.append(col)
for c in ['psfsize', 'psfdepth', 'galdepth', 'nea', 'blob_nea']:
add_fluxlike(c)
if has_wise:
add_wiselike('psfdepth')
if has_wise:
cols.extend(['wise_coadd_id', 'wise_x', 'wise_y'])
if has_wise_lc:
trbands = ['w1', 'w2']
lc_cols = [
'lc_flux', 'lc_flux_ivar', 'lc_nobs', 'lc_fracflux', 'lc_rchisq',
'lc_mjd'
]
for c in lc_cols:
add_wiselike(c, bands=trbands)
add_wiselike('lc_epoch_index', bands=trbands)
T.lc_epoch_index_w1 = np.empty((len(T), N_wise_epochs), np.int16)
T.lc_epoch_index_w2 = np.empty((len(T), N_wise_epochs), np.int16)
# initialize...
T.lc_epoch_index_w1[:] = -1
T.lc_epoch_index_w2[:] = -1
# Cut down to a fixed number of WISE time-resolved epochs?
if N_wise_epochs is not None:
# mapping from (nobs, mjd) to indices to keep
keep_epochs = {}
newvals = {}
for band in trbands:
# initialize new (cut) arrays
for col in lc_cols:
colname = col + '_' + band
oldval = T.get(colname)
n, _ = oldval.shape
newval = np.zeros((n, N_wise_epochs), oldval.dtype)
newvals[colname] = newval
lc_nobs = T.get('lc_nobs_%s' % band)
lc_mjd = T.get('lc_mjd_%s' % band)
lc_epoch = T.get('lc_epoch_index_%s' % band)
# Check each row (source) individually, since coverage isn't
# uniform across a brick
for row, (nobs, mjd) in enumerate(zip(lc_nobs, lc_mjd)):
key = tuple(nobs) + tuple(mjd)
if key not in keep_epochs:
I = one_lightcurve_bitmask(nobs,
mjd,
n_final=N_wise_epochs)
# convert to integer index list
I = np.flatnonzero(I)
keep_epochs[key] = I
else:
I = keep_epochs[key]
for col in lc_cols:
colname = col + '_' + band
oldval = T.get(colname)
newval = newvals[colname]
newval[row, :len(I)] = oldval[row, I]
assert (np.all(I) < 255)
lc_epoch[row, :len(I)] = I.astype(np.int16)
for k, v in newvals.items():
T.set(k, v)
del keep_epochs
cols.extend([
'sersic', 'sersic_ivar', 'shape_r', 'shape_r_ivar', 'shape_e1',
'shape_e1_ivar', 'shape_e2', 'shape_e2_ivar'
])
debug('Columns:', cols)
debug('T columns:', T.columns())
# match case to T.
cc = T.get_columns()
cclower = [c.lower() for c in cc]
for i, c in enumerate(cols):
if (not c in cc) and c in cclower:
j = cclower.index(c)
cols[i] = cc[j]
units = get_units_for_columns(cols, list(allbands) + wbands + gbands)
T.writeto(outfn,
columns=cols,
header=hdr,
primheader=primhdr,
units=units,
**write_kwargs)