def compare_problematic(self):
# Assumes j results from self.rename_john_zptfile()
j=fits_table('/project/projectdirs/desi/users/burleigh/test_data/old/john_combined_renamed.fits')
a=fits_table('/project/projectdirs/desi/users/burleigh/test_data/old/arjun_combined.fits')
# Compare
discrep_keys=['zpt','zptavg',\
'skycounts','skyrms','skymag',\
'transp','phoff','rarms','decrms','raoff','decoff']
panels=len(discrep_keys)
cols=3
if panels % cols == 0:
rows=panels/cols
else:
rows=panels/cols+1
fig,axes= plt.subplots(rows,cols,figsize=(20,10))
ax=axes.flatten()
plt.subplots_adjust(hspace=0.4,wspace=0.3)
cnt=-1
for key in discrep_keys:
cnt+=1
arjun= a.get( self.trans.j2a[key] )
john= j.get( self.trans.j2a[key] )
if key == 'skymag':
arjun=arjun[ j.get('exptime') > 50]
john=john[ j.get('exptime') > 50]
ax[cnt].scatter(john,arjun)
ylab=ax[cnt].set_ylabel('Arjun',fontsize='large')
xlab=ax[cnt].set_xlabel('%s (John)' % key,fontsize='large')
def setHdus(self, p):
self.hdus = p
t = fits_table(p[6].data)
# the table has only one row...
assert(len(t) == 1)
t = t[0]
#self.table = t
self.gain = t.gain
self.dark_variance = t.dark_variance
self.sky = t.sky
self.skyerr = t.skyerr
self.psp_status = t.status
# Double-Gaussian PSF params
self.dgpsf_s1 = t.psf_sigma1_2g
self.dgpsf_s2 = t.psf_sigma2_2g
self.dgpsf_b = t.psf_b_2g
# summary PSF width (sigmas)
self.psf_fwhm = t.psf_width * (2.*np.sqrt(2.*np.log(2.)))
# 2-gaussian plus power-law PSF params
self.plpsf_s1 = t.psf_sigma1
self.plpsf_s2 = t.psf_sigma2
self.plpsf_b = t.psf_b
self.plpsf_p0 = t.psf_p0
self.plpsf_beta = t.psf_beta
self.plpsf_sigmap = t.psf_sigmap
t = fits_table(p[8].data)
self.per_run_apcorrs = t.ap_corr_run
def cat_vcc(req, ver):
import json
tag = 'ngc'
ralo = float(req.GET['ralo'])
rahi = float(req.GET['rahi'])
declo = float(req.GET['declo'])
dechi = float(req.GET['dechi'])
ver = int(ver)
if not ver in catversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
from astrometry.util.fits import fits_table, merge_tables
import numpy as np
from decals import settings
TT = []
T = fits_table(os.path.join(settings.DATA_DIR, 'virgo-cluster-cat-2.fits'))
print(len(T), 'in VCC 2; ra', ralo, rahi, 'dec', declo, dechi)
T.cut((T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi))
print(len(T), 'in cut')
TT.append(T)
T = fits_table(os.path.join(settings.DATA_DIR, 'virgo-cluster-cat-3.fits'))
print(len(T), 'in VCC 3; ra', ralo, rahi, 'dec', declo, dechi)
T.cut((T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi))
print(len(T), 'in cut')
T.evcc_id = np.array(['-']*len(T))
T.rename('id', 'vcc_id')
TT.append(T)
T = merge_tables(TT)
rd = list((float(r),float(d)) for r,d in zip(T.ra, T.dec))
names = []
for t in T:
evcc = t.evcc_id.strip()
vcc = t.vcc_id.strip()
ngc = t.ngc.strip()
nms = []
if evcc != '-':
nms.append('EVCC ' + evcc)
if vcc != '-':
nms.append('VCC ' + vcc)
if ngc != '-':
nms.append('NGC ' + ngc)
names.append(' / '.join(nms))
return HttpResponse(json.dumps(dict(rd=rd, name=names)),
content_type='application/json')
def decals_dr3_check_wcsfailed():
import fitsio
basedir = os.environ['LEGACY_SURVEY_DIR']
image_basedir = os.path.join(basedir, 'images')
for fn in [#'survey-ccds-decals.fits.gz',
'survey-ccds-nondecals.fits.gz',
'survey-ccds-extra.fits.gz',
]:
T = fits_table(fn)
T.wcscal = np.zeros(len(T), bool)
fns = np.unique(np.array([f.strip() for f in T.image_filename]))
H = fits_table()
H.image_filename = fns
H.primary_header = []
for ifn,f in enumerate(fns):
imgfn = os.path.join(image_basedir, f)
print('Reading', imgfn)
ff = open(imgfn, 'r')
h = ff.read(32768)
ff.close()
hdr = fitsio.FITSHDR()
hdrstring = ''
while True:
line = h[:80]
h = h[80:]
# fitsio apparently can't handle CONTINUE
if line[:8] != 'CONTINUE':
hdr.add_record(line)
hdrstring += line
if line == ('END' + ' '*77):
break
H.primary_header.append(hdrstring)
#hdr = fitsio.read_header(imgfn)
expnum = hdr['EXPNUM']
wcscal = hdr['WCSCAL']
wcsok = (wcscal == 'Successful')
print('File', f, 'expnum', expnum, 'WCS cal', wcscal, 'ok', wcsok)
I = np.flatnonzero(T.expnum == expnum)
T.wcscal[I] = wcsok
T.wcsok = T.wcscal.astype(np.uint8)
T.writeto('new-' + fn)
H.primary_header = np.array(H.primary_header)
H.writeto('headers-' + fn)
def getMaskPlane(self, name):
# Mask planes are described in HDU 11 (the last HDU)
if self.maskmap is None:
self.maskmap = {}
T = fits_table(self.hdus[-1].data)
T.cut(T.defname == 'S_MASKTYPE')
for k,v in zip(T.attributename, T.value):
k = k.replace('S_MASK_', '')
if k == 'S_NMASK_TYPES':
continue
self.maskmap[k] = v
if not name in self.maskmap:
raise RuntimeError('Unknown mask plane \"%s\"' % name)
return fits_table(self.hdus[1 + self.maskmap[name]].data)
def mzls_to_20160315():
basedir = os.environ['LEGACY_SURVEY_DIR']
cam = 'mosaic'
image_basedir = os.path.join(basedir, 'images')
TT = []
for fn,dirnms in [
('/global/homes/a/arjundey/ZeroPoints/mzls-zpt-all.fits',
['CP20160202','CP20160203','CP20160204','CP20160205','CP20160206','CP20160208',
'CP20160209','CP20160210','CP20160211','CP20160212','CP20160213','CP20160214',
'CP20160215','CP20160216','CP20160217','CP20160219','CP20160224','CP20160225',
'CP20160226','CP20160227','CP20160228','CP20160229','CP20160301','CP20160302',
'CP20160303','CP20160304','CP20160305','CP20160306','CP20160308','CP20160309',
'CP20160310','CP20160311','CP20160312','CP20160313','CP20160314','CP20160315',
'CP20160316','CP20160317','CP20160318','CP20160319','CP20160320','CP20160325',
'CP20160326','CP20160327','CP20160328','CP20160330','CP20160331','CP20160401',
'CP20160402','CP20160403','CP20160404','CP20160408',]),
]:
T = fits_table(fn)
normalize_zeropoints(fn, dirnms, image_basedir, cam, T=T)
TT.append(T)
T = merge_tables(TT)
I = np.flatnonzero(T.fwhm == 0)
if len(I):
T.fwhm[I] = T.seeing[I] / 0.262
outfn = 'survey-ccds-mzls-to-20160315.fits'
T.writeto(outfn)
print('Wrote', outfn)
for fn in [outfn]:
os.system('gzip --best ' + fn)
def mzls_dr4(v2=True):
if v2:
zpname='arjuns-ccds-mzls-v2thruMarch19.fits'
savefn='survey-ccds-mzls-v2thruMarch19.fits.gz'
else:
zpname='arjuns-ccds-mzls-v3.fits'
savefn='survey-ccds-mzls-v3.fits.gz'
basedir = os.environ['LEGACY_SURVEY_DIR']
cam = 'mosaic'
image_basedir = os.path.join(basedir, 'images')
TT = []
# mzls_cpdirs_...txt is list of all possible CP directories
for fn,dirnms in [
(zpname,
list(np.loadtxt('mzls_cpdirs_thruMarch19.txt',dtype=str))),
]:
T = fits_table(fn)
normalize_zeropoints(fn, dirnms, image_basedir, cam, T=T)
TT.append(T)
T = merge_tables(TT)
I = np.flatnonzero(T.fwhm == 0)
if len(I):
T.fwhm[I] = T.seeing[I] / 0.262
T.writeto(savefn)
print('Wrote', savefn)
def decals_dr3_plus():
basedir = os.environ['LEGACY_SURVEY_DIR']
cam = 'decam'
image_basedir = os.path.join(basedir, 'images')
TT = []
expnums = [547257, 535154, 535106]
for fn,dirnms in [
('/global/homes/a/arjundey/ZeroPoints/decals-zpt-20160407.fits',
['CP20160407',]),
('/global/homes/a/arjundey/ZeroPoints/decals-zpt-20160606.fits',
['CP20160606',]),
]:
T = fits_table(fn)
T.cut(np.nonzero([e not in expnums for e in T.expnum])[0])
normalize_zeropoints(fn, dirnms, image_basedir, cam, T=T)
TT.append(T)
dirnms = ['CP20160407','CP20160606']
for fn in [
'/global/cscratch1/sd/arjundey/ZeroPoints/zeropoint-c4d_160408_023844_oki_r_v1.fits',
'/global/cscratch1/sd/arjundey/ZeroPoints/zeropoint-c4d_160408_001343_oki_r_v1.fits',
'/global/cscratch1/sd/arjundey/ZeroPoints/zeropoint-c4d_160607_023641_oki_g_v1.fits',
]:
T = normalize_zeropoints(fn, dirnms, image_basedir, cam)
TT.append(T)
T = merge_tables(TT)
outfn = 'survey-ccds-dr3plus.fits'
T.writeto(outfn)
print('Wrote', outfn)
for fn in [outfn]:
os.system('gzip --best ' + fn)
def cat_phat_clusters(req, ver):
import json
from astrometry.util.fits import fits_table, merge_tables
tag = 'phat-clusters'
ralo = float(req.GET['ralo'])
rahi = float(req.GET['rahi'])
declo = float(req.GET['declo'])
dechi = float(req.GET['dechi'])
ver = int(ver)
if not ver in catversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
cat = fits_table(os.path.join(settings.DATA_DIR, 'phat-clusters.fits'))
cat.cut((cat.ra >= ralo ) * (cat.ra <= rahi) *
(cat.dec >= declo) * (cat.dec <= dechi))
return HttpResponse(json.dumps(dict(
name=[str(s.strip()) for s in cat.name],
rd=[(float(o.ra),float(o.dec)) for o in cat],
mag=[float(o.mag) for o in cat],
young=[bool(o.young) for o in cat],
velocity=[float(o.velocity) for o in cat],
metallicity=[float(o.metallicity) for o in cat],
)),
content_type='application/json')
def catalog_to_fits(cat):
from astrometry.util.fits import fits_table
import numpy as np
cols = ['ra','dec','g','r','z','w1','w2','w3','w4',
'brickid', 'brickname', 'objid', 'type',]
values = cat.values_list(*cols)
def convert_nan(x):
if x is None:
return np.nan
return x
T = fits_table()
for i,c in enumerate(cols):
v = values[0][i]
convert = None
if isinstance(v, unicode):
convert = str
if isinstance(v, float):
convert = convert_nan
#print('Type of column', c, 'is', type(v), 'eg', v)
cname = c
if convert is None:
T.set(cname, np.array([v[i] for v in values]))
else:
T.set(cname, np.array([convert(v[i]) for v in values]))
return T
def __init__(self, fn=None, ext=1):
if fn is not None:
from astrometry.util.fits import fits_table
T = fits_table(fn, ext=ext)
ims = T.psf_mask[0]
print('Got', ims.shape, 'PSF images')
hdr = T.get_header()
# PSF distortion bases are polynomials of x,y
assert(hdr['POLNAME1'].strip() == 'X_IMAGE')
assert(hdr['POLNAME2'].strip() == 'Y_IMAGE')
assert(hdr['POLGRP1'] == 1)
assert(hdr['POLGRP2'] == 1)
assert(hdr['POLNGRP' ] == 1)
x0 = hdr.get('POLZERO1')
xscale = hdr.get('POLSCAL1')
y0 = hdr.get('POLZERO2')
yscale = hdr.get('POLSCAL2')
degree = hdr.get('POLDEG1')
self.sampling = hdr.get('PSF_SAMP')
print('PsfEx sampling:', self.sampling)
# number of terms in polynomial
ne = (degree + 1) * (degree + 2) / 2
assert(hdr['PSFAXIS3'] == ne)
assert(len(ims.shape) == 3)
assert(ims.shape[0] == ne)
self.psfbases = ims
self.xscale, self.yscale = xscale, yscale
self.degree = degree
print('PsfEx degree:', self.degree)
bh,bw = self.psfbases[0].shape
self.radius = (bh+1)/2.
self.x0,self.y0 = x0,y0
def get_ccds(self):
'''
Returns the table of CCDs.
'''
fn = os.path.join(self.decals_dir, 'decals-ccds.fits')
if not os.path.exists(fn):
fn += '.gz'
print('Reading CCDs from', fn)
T = fits_table(fn)
print('Got', len(T), 'CCDs')
cols = T.columns()
# Make DR1 CCDs table somewhat compatible with DR2
if 'extname' in cols and not 'ccdname' in cols:
T.ccdname = T.extname
if not 'camera' in cols:
T.camera = np.array(['decam'] * len(T))
if 'cpimage' in cols and not 'image_filename' in cols:
T.image_filename = T.cpimage
if 'cpimage_hdu' in cols and not 'image_hdu' in cols:
T.image_hdu = T.cpimage_hdu
if 'ccdname' in T.columns():
# "N4 " -> "N4"
T.ccdname = np.array([s.strip() for s in T.ccdname])
return T
def add_depth_tag(decals, brick, outdir, overwrite=False):
outfn = os.path.join(outdir, 'tractor', brick[:3], 'tractor-%s.fits' % brick)
if os.path.exists(outfn) and not overwrite:
print 'Exists:', outfn
return
fn = decals.find_file('tractor', brick=brick)
if not os.path.exists(fn):
print 'Does not exist:', fn
return
T = fits_table(fn, lower=False)
primhdr = fitsio.read_header(fn)
hdr = fitsio.read_header(fn, ext=1)
print 'Read', len(T), 'from', fn
T.decam_depth = np.zeros((len(T), len(decals.allbands)), np.float32)
T.decam_galdepth = np.zeros((len(T), len(decals.allbands)), np.float32)
bands = 'grz'
ibands = [decals.index_of_band(b) for b in bands]
ix = np.clip(np.round(T.bx).astype(int), 0, 3599)
iy = np.clip(np.round(T.by).astype(int), 0, 3599)
for iband,band in zip(ibands, bands):
fn = decals.find_file('depth', brick=brick, band=band)
if os.path.exists(fn):
print 'Reading', fn
img = fitsio.read(fn)
T.decam_depth[:,iband] = img[iy, ix]
fn = decals.find_file('galdepth', brick=brick, band=band)
if os.path.exists(fn):
print 'Reading', fn
img = fitsio.read(fn)
T.decam_galdepth[:,iband] = img[iy, ix]
outfn = os.path.join(outdir, 'tractor', brick[:3], 'tractor-%s.fits' % brick)
trymakedirs(outfn, dir=True)
for s in [
'Data product of the DECam Legacy Survey (DECaLS)',
'Full documentation at http://legacysurvey.org',
]:
primhdr.add_record(dict(name='COMMENT', value=s, comment=s))
# print 'Header:', hdr
# T.writeto(outfn, header=hdr, primheader=primhdr)
# Yuck, all this to get the units right
tmpfn = outfn + '.tmp'
fits = fitsio.FITS(tmpfn, 'rw', clobber=True)
fits.write(None, header=primhdr)
cols = T.get_columns()
units = []
for i in range(1, len(cols)+1):
u = hdr.get('TUNIT%i' % i, '')
units.append(u)
# decam_depth units
fluxiv = '1/nanomaggy^2'
units[-2] = fluxiv
units[-1] = fluxiv
fits.write([T.get(c) for c in cols], names=cols, header=hdr, units=units)
fits.close()
os.rename(tmpfn, outfn)
print 'Wrote', outfn
def make_fits_images(run, camcol, field):
"""gets field files from local cache (or sdss), returns UGRIZ dict of
fits images"""
print """==================================================\n\n
Grabbing image files from the cache.
TODO: turn off the tractor printing... """
imgs = {}
for band in BANDS:
print "reading in band %s" % band
imgs[band] = sdss.get_tractor_image_dr9(run, camcol, field, band)
fn = asdss.DR9().retrieve('photoField', run, camcol, field)
F = aufits.fits_table(fn)
# convert to FitsImage's
imgfits = {}
for iband,band in enumerate(BANDS):
print "converting images %s" % band
frame = asdss.DR9().readFrame(run, camcol, field, band)
calib = np.median(frame.getCalibVec())
gain = F[0].gain[iband]
darkvar = F[0].dark_variance[iband]
sky = np.median(frame.getSky())
imgfits[band] = celeste.FitsImage(band,
timg=imgs[band],
calib=calib,
gain=gain,
darkvar=darkvar,
sky=sky)
return imgfits
def write_fits(self, filename, hdr=None, primhdr=None):
tt = type(self)
sky_type = '%s.%s' % (tt.__module__, tt.__name__)
if hdr is None:
import fitsio
hdr = fitsio.FITSHDR()
if primhdr is None:
import fitsio
primhdr = fitsio.FITSHDR()
hdr.add_record(dict(name='SKY', value=sky_type,
comment='Sky class'))
primhdr.add_record(dict(name='SKY', value=sky_type,
comment='Sky class'))
#hdr.add_record(dict(name='SPL_ORD', value=self.order,
# comment='Spline sky order'))
# this writes all params as header cards
#self.toFitsHeader(hdr, prefix='SKY_')
#fits = fitsio.FITS(filename, 'rw')
#fits.write(None, header=primhdr, clobber=True)
#fits.write(self.c, header=hdr)
#fits.close()
from astrometry.util.fits import fits_table
T = fits_table()
T.xgrid = np.atleast_2d(self.xgrid).astype(np.int32)
T.ygrid = np.atleast_2d(self.ygrid).astype(np.int32)
T.x0 = np.atleast_1d(self.x0).astype(np.int32)
T.y0 = np.atleast_1d(self.y0).astype(np.int32)
gridvals = self.spl(self.xgrid, self.ygrid).T
T.gridvals = np.array([gridvals]).astype(np.float32)
T.order = np.atleast_1d(self.order).astype(np.uint8)
assert(len(T) == 1)
T.writeto(filename, header=hdr, primheader=primhdr)
def from_fits(cls, filename, header, row=0):
from astrometry.util.fits import fits_table
T = fits_table(filename)
T = T[row]
sky = cls(T.xgrid, T.ygrid, T.gridvals, order=T.order)
sky.shift(T.x0, T.y0)
return sky
def __init__(self, fn=None, ext=1):
if fn is not None:
from astrometry.util.fits import fits_table
T = fits_table(fn, ext=ext)
ims = T.psf_mask[0]
print("Got", ims.shape, "PSF images")
hdr = T.get_header()
# PSF distortion bases are polynomials of x,y
assert hdr["POLNAME1"].strip() == "X_IMAGE"
assert hdr["POLNAME2"].strip() == "Y_IMAGE"
assert hdr["POLGRP1"] == 1
assert hdr["POLGRP2"] == 1
assert hdr["POLNGRP"] == 1
x0 = hdr.get("POLZERO1")
xscale = hdr.get("POLSCAL1")
y0 = hdr.get("POLZERO2")
yscale = hdr.get("POLSCAL2")
degree = hdr.get("POLDEG1")
self.sampling = hdr.get("PSF_SAMP")
print("PsfEx sampling:", self.sampling)
# number of terms in polynomial
ne = (degree + 1) * (degree + 2) / 2
assert hdr["PSFAXIS3"] == ne
assert len(ims.shape) == 3
assert ims.shape[0] == ne
self.psfbases = ims
self.xscale, self.yscale = xscale, yscale
self.degree = degree
print("PsfEx degree:", self.degree)
bh, bw = self.psfbases[0].shape
self.radius = (bh + 1) / 2.0
self.x0, self.y0 = x0, y0
def read(fn, F=None, primhdr=None, table=None):
'''
F: fitsio.FITS object to use an already-open file.
primhdr: FITS header object for the primary HDU.
table: astrometry.util.fits table
'''
if F is None:
import fitsio
F = fitsio.FITS(fn)
if primhdr is None:
primhdr = F[0].read_header()
band = primhdr['FILTER'].strip()
run = primhdr['RUN']
camcol = primhdr['CAMCOL']
field = 0 # 'FRAME' != field
if table is None:
tab = fits_table(F[3].read(lower=True))
else:
tab = table
assert(len(tab) == 1)
tab = tab[0]
return AsTrans(run, camcol, field, band,
node=np.deg2rad(tab.node),
incl=np.deg2rad(tab.incl),
astrans=tab, cut_to_band=False)
def tractor_cat(fn):
'''uses Dustin's fits_table() function to read his tractor catalog then returns it as dictionary of np arrays
his fits_table handles booleans with weird values like 84 for "brick_primary", and converts that to True/False'''
temp = fits_table(fn)
data={}
for key in temp.columns(): data[key]= temp.get(key)
return data
def read_intermediate_catalog(self, brick, **kwargs):
'''
Reads the intermediate tractor catalog for the given brickname.
*kwargs*: passed to self.find_file()
Returns (T, hdr, primhdr)
'''
fn = self.find_file('tractor-intermediate', brick=brick, **kwargs)
T = fits_table(fn)
hdr = T.get_header()
primhdr = fitsio.read_header(fn)
in_flux_prefix = ''
# Ensure flux arrays are 2d (N x 1)
keys = ['flux', 'flux_ivar', 'rchi2', 'fracflux', 'fracmasked',
'fracin', 'nobs', 'anymask', 'allmask', 'psfsize', 'depth',
'galdepth']
for k in keys:
incol = '%s%s' % (in_flux_prefix, k)
X = T.get(incol)
# Hmm, if we need to reshape one of these arrays, we will
# need to do all of them.
if len(X.shape) == 1:
X = X[:, np.newaxis]
T.set(incol, X)
return T, hdr, primhdr
def setHdus(self, p):
self.hdus = p
self.table = fits_table(self.hdus[1].data)[0]
T = self.table
self.aa = T.aa.astype(float)
self.kk = T.kk.astype(float)
self.airmass = T.airmass
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--force', action='store_true',
help='Run calib processes even if files already exist?')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--expnum', type=int, help='Cut to a single exposure')
parser.add_argument('--extname', '--ccdname', help='Cut to a single extension/CCD name')
parser.add_argument('--no-astrom', dest='astrom', action='store_false',
help='Do not compute astrometric calibs')
parser.add_argument('--no-psf', dest='psfex', action='store_false',
help='Do not compute PsfEx calibs')
parser.add_argument('--no-sky', dest='sky', action='store_false',
help='Do not compute sky models')
parser.add_argument('--run-se', action='store_true', help='Run SourceExtractor')
parser.add_argument('--splinesky', action='store_true', help='Spline sky, not constant')
parser.add_argument('args',nargs=argparse.REMAINDER)
opt = parser.parse_args()
D = Decals()
if opt.ccds is not None:
T = fits_table(opt.ccds)
print('Read', len(T), 'from', opt.ccds)
else:
T = D.get_ccds()
#print len(T), 'CCDs'
if len(opt.args) == 0:
if opt.expnum is not None:
T.cut(T.expnum == opt.expnum)
print('Cut to', len(T), 'with expnum =', opt.expnum)
if opt.extname is not None:
T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname]))
print('Cut to', len(T), 'with extname =', opt.extname)
opt.args = range(len(T))
for a in opt.args:
i = int(a)
t = T[i]
im = D.get_image_object(t)
print('Running', im.calname)
kwargs = dict(pvastrom=opt.astrom, psfex=opt.psfex, sky=opt.sky)
if opt.force:
kwargs.update(force=True)
if opt.run_se:
kwargs.update(se=True)
if opt.splinesky:
kwargs.update(splinesky=True)
run_calibs((im, kwargs))
return 0
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--name1", help="Name for first data set")
parser.add_argument("--name2", help="Name for second data set")
parser.add_argument("--plot-prefix", default="compare", help='Prefix for plot filenames; default "%default"')
parser.add_argument("--match", default=1.0, help="Astrometric cross-match distance in arcsec")
parser.add_argument("dir1", help="First directory to compare")
parser.add_argument("dir2", help="Second directory to compare")
opt = parser.parse_args()
ps = PlotSequence(opt.plot_prefix)
name1 = opt.name1
if name1 is None:
name1 = os.path.basename(opt.dir1)
if not len(name1):
name1 = os.path.basename(os.path.dirname(opt.dir1))
name2 = opt.name2
if name2 is None:
name2 = os.path.basename(opt.dir2)
if not len(name2):
name2 = os.path.basename(os.path.dirname(opt.dir2))
tt = "Comparing %s to %s" % (name1, name2)
# regex for tractor-*.fits catalog filename
catre = re.compile("tractor-.*.fits")
cat1, cat2 = [], []
for basedir, cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
for dirpath, dirnames, filenames in os.walk(basedir, followlinks=True):
for fn in filenames:
if not catre.match(fn):
print("Skipping", fn, "due to filename")
continue
fn = os.path.join(dirpath, fn)
t = fits_table(fn)
print(len(t), "from", fn)
cat.append(t)
cat1 = merge_tables(cat1, columns="fillzero")
cat2 = merge_tables(cat2, columns="fillzero")
print("Total of", len(cat1), "from", name1)
print("Total of", len(cat2), "from", name2)
cat1.cut(cat1.brick_primary)
cat2.cut(cat2.brick_primary)
print("Total of", len(cat1), "BRICK_PRIMARY from", name1)
print("Total of", len(cat2), "BRICK_PRIMARY from", name2)
cat1.cut((cat1.decam_anymask[:, 1] == 0) * (cat1.decam_anymask[:, 2] == 0) * (cat1.decam_anymask[:, 4] == 0))
cat2.cut((cat2.decam_anymask[:, 1] == 0) * (cat2.decam_anymask[:, 2] == 0) * (cat2.decam_anymask[:, 4] == 0))
print("Total of", len(cat1), "unmasked from", name1)
print("Total of", len(cat2), "unmasked from", name2)
I, J, d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match / 3600.0, nearest=True)
print(len(I), "matched")
matched1 = cat1[I]
matched2 = cat2[J]
def bricks_near_radec(ra,dec,d_ra=1.,d_dec=1.):
'''given bricks table, return all bricknames with brick centers [ra-d_ra,ra+d_ra],[dec-d_dec,dec+d_dec]'''
b=fits_table('survey-bricks.fits.gz')
ra1,ra2= ra-d_ra, ra+d_ra
dec1,dec2= dec-d_dec, dec+d_dec
ind= np.all((b.get('ra') >= ra1,b.get('ra') <= ra2,b.get('dec') >= dec1,b.get('dec') <= dec2),axis=0)
print 'these bricks have centers between %.1f < ra < %.1f and %.1f < dec < %.1f' % (ra1,ra2,dec1,dec2)
return b.get('brickname')[ind]
def get_bricks(self):
'''
Returns a table of bricks. The caller owns the table.
For read-only purposes, see *get_bricks_readonly()*, which
uses a cached version.
'''
return fits_table(self.find_file('bricks'))
def get_bricks(self):
'''
Returns a table of bricks. The caller owns the table.
For read-only purposes, see *get_bricks_readonly()*, which
uses a cached version.
'''
return fits_table(os.path.join(self.decals_dir, 'decals-bricks.fits'))
def readPhotoObj(self, run, camcol, field, filename=None):
obj = PhotoObj(run, camcol, field)
if filename is None:
fn = self.getPath('photoObj', run, camcol, field)
else:
fn = filename
obj.table = fits_table(fn)
return obj
def overlap_bricks(bricks_fn,ra,dec,dx):
'''given bricks table, return all bricknames with brick centers [ra-dx,ra+dx],[dec-dx,dec+dx]'''
b=fits_table(bricks_fn)
ra1,ra2= ra-dx, ra+dx
dec1,dec2= dec-dx, dec+dx
ind= np.all((b.get('ra') >= ra1,b.get('ra') <= ra2,b.get('dec') >= dec1,b.get('dec') <= dec2),axis=0)
print 'these bricks have centers between %.1f < ra < %.1f and %.1f < dec < %.1f' % (ra1,ra2,dec1,dec2)
for name,ra,dec in zip(b.get('brickname')[ind],b.get('ra')[ind],b.get('dec')[ind]): print name,ra,dec
def upload_cat(req):
import tempfile
from astrometry.util.fits import fits_table
from django.http import HttpResponseRedirect
from map.views import index
if req.method != 'POST':
return HttpResponse('POST only')
print('Files:', req.FILES)
cat = req.FILES['catalog']
dirnm = settings.USER_QUERY_DIR
if not os.path.exists(dirnm):
try:
os.makedirs(dirnm)
except:
pass
f,tmpfn = tempfile.mkstemp(suffix='.fits', dir=dirnm)
os.close(f)
os.unlink(tmpfn)
print('Saving to', tmpfn)
with open(tmpfn, 'wb+') as destination:
for chunk in cat.chunks():
destination.write(chunk)
print('Wrote', tmpfn)
try:
T = fits_table(tmpfn)
except:
return HttpResponse('Must upload FITS format catalog including "RA", "Dec", optionally "Name" columns')
# Rename and resave columns if necessary
if rename_cols(T):
T.write_to(tmpfn)
cols = T.columns()
if not (('ra' in cols) and ('dec' in cols)):
return HttpResponse('Must upload catalog including "RA", "Dec", optionally "Name" columns')
ra,dec = T.ra[0], T.dec[0]
catname = tmpfn.replace(dirnm, '').replace('.fits', '')
if catname.startswith('/'):
catname = catname[1:]
try:
import fitsio
primhdr = fitsio.read_header(tmpfn)
name = primhdr.get('CATNAME', '')
color = primhdr.get('CATCOLOR', '')
if len(name):
catname = catname + '-n%s' % name.strip().replace(' ','_')
if len(color):
catname = catname + '-c%s' % color.strip()
except:
pass
return HttpResponseRedirect(reverse(index) +
'?ra=%.4f&dec=%.4f&catalog=%s' % (ra, dec, catname))
def main():
brickname = '2428p117'
objtype = 'STAR'
lobjtype = objtype.lower()
decals_sim_dir = '/Users/ioannis/decals_sim_dir/star/2428p117'
zoom = [1800,2400,1800,2400]
#zoom = [1800,2000,1800,2000]
#zoom = None
decals = Decals()
brickinfo = decals.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale()
if zoom is not None:
# See also runbrick.stage_tims()
#(x0,x1,y0,y1) = args.zoom
(x0,x1,y0,y1) = zoom
W = x1-x0
H = y1-y0
targetwcs = brickwcs.get_subimage(x0, y0, W, H)
bounds = brickwcs.radec_bounds()
ra_range = bounds[1]-bounds[0]
dec_range = bounds[3]-bounds[2]
radec_center = brickwcs.radec_center()
print(radec_center, ra_range, dec_range)
corners = np.array([brickwcs.pixelxy2radec(x,y) for x,y in
[(1,1),(W,1),(W,H),(1,H),(1,1)]])
# Identify the CCDs in the region of interest.
ccdinfo = decals.ccds_touching_wcs(brickwcs)
ccdinfo.about()
log.info('Got {} CCDs'.format(len(ccdinfo)))
# Generate the catalog of simulated sources here!
simcat = fits_table('simcat-{}-{}-00.fits'.format(brickname,lobjtype))
#simcat = fits.getdata('simcat-{}-{}-00.fits'.format(brickname,lobjtype))
simcat.about()
blobxy = zip(simcat.x,simcat.y)
simdecals = SimDecals(sim_sources=simcat)
# For testing!
#sim = SimImage(simdecals,ccdinfo[0])
#tim = sim.get_tractor_image(const2psf=True)
run_brick(brickname, decals=simdecals, outdir=decals_sim_dir,
threads=8, zoom=zoom, wise=False, sdssInit=False,
forceAll=True, writePickles=False, blobxy=blobxy,
pixPsf=False, stages=['tims','srcs','fitblobs',
'fitblobs_finish','coadds',
'writecat'])
def main(args):
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).")
parser.add_argument('--calibs', action='store_true',
help='Output CCDs that need to be calibrated.')
parser.add_argument('--nper', type=int, default=None,
help='Batch N calibs per line')
parser.add_argument('--byexp', action='store_true', default=False,
help='Run one whole exposure per job (not one CCD per job)')
parser.add_argument('--forced', action='store_true',
help='Output forced-photometry commands')
parser.add_argument('--lsb', action='store_true',
help='Output Low-Surface-Brightness commands')
parser.add_argument('--stage', help='Stage image files to given directory')
parser.add_argument('--touching', action='store_true',
help='Cut to only CCDs touching selected bricks')
parser.add_argument('--near', action='store_true',
help='Quick cut to only CCDs near selected bricks')
parser.add_argument('--check-coadd', action='store_true',
help='Check which coadds actually need to run.')
parser.add_argument('--out', help='Output filename for calibs, default %(default)s',
default='jobs')
parser.add_argument('--command', action='store_true',
help='Write out full command-line to run calib')
parser.add_argument('--opt', help='With --command, extra options to add')
parser.add_argument('--maxra', type=float, help='Maximum RA to run')
parser.add_argument('--minra', type=float, help='Minimum RA to run')
parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
parser.add_argument('--mindec', type=float, help='Minimum Dec to run')
parser.add_argument('--region', help='Region to select')
parser.add_argument('--bricks', help='Set bricks.fits file to load')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--ignore_cuts', action='store_true',default=False,help='no photometric cuts')
parser.add_argument('--save_to_fits', action='store_true',default=False,help='save cut brick,ccd to fits table')
parser.add_argument('--name', action='store',default='dr3',help='save with this suffix, e.g. refers to ccds table')
parser.add_argument('--delete-sky', action='store_true',
help='Delete any existing sky calibration files')
parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')
parser.add_argument('--nccds', action='store_true', default=False, help='Prints number of CCDs per brick')
parser.add_argument('--bands', default='g,r,z', help='Set bands to keep: comma-separated list.')
opt = parser.parse_args(args)
want_ccds = (opt.calibs or opt.forced or opt.lsb)
want_bricks = not want_ccds
survey = LegacySurveyData()
if opt.bricks is not None:
B = fits_table(opt.bricks)
log('Read', len(B), 'from', opt.bricks)
else:
B = survey.get_bricks()
log('Bricks Dec range:', B.dec.min(), B.dec.max())
if opt.ccds is not None:
T = fits_table(opt.ccds)
log('Read', len(T), 'from', opt.ccds)
else:
T = survey.get_ccds()
log(len(T), 'CCDs')
T.index = np.arange(len(T))
if opt.ignore_cuts == False:
log('Applying CCD cuts...')
if 'ccd_cuts' in T.columns():
T.cut(T.ccd_cuts == 0)
log(len(T), 'CCDs survive cuts')
bands = opt.bands.split(',')
log('Filters:', np.unique(T.filter))
T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
log('Cut to', len(T), 'CCDs in filters', bands)
log('CCDs Dec range:', T.dec.min(), T.dec.max())
# I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
# plt.clf()
# plt.hist(counts, counts.max()+1)
# plt.savefig('bricks.png')
# B.cut(I[counts >= 9])
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# #plt.scatter(B.ra[I], B.dec[I], c=counts)
# plt.savefig('bricks2.png')
# DES Stripe82
#rlo,rhi = 350.,360.
# rlo,rhi = 300., 10.
# dlo,dhi = -6., 4.
# TINY bit
#rlo,rhi = 350.,351.1
#dlo,dhi = 0., 1.1
# EDR+
# 860 bricks
# ~10,000 CCDs
#rlo,rhi = 239,246
#dlo,dhi = 5, 13
# DR1
#rlo,rhi = 0, 360
# part 1
#dlo,dhi = 25, 40
# part 2
#dlo,dhi = 20,25
# part 3
#dlo,dhi = 15,20
# part 4
#dlo,dhi = 10,15
# part 5
#dlo,dhi = 5,10
# the rest
#dlo,dhi = -11, 5
#dlo,dhi = 15,25.5
dlo,dhi = -25, 40
rlo,rhi = 0, 360
# Arjun says 3x3 coverage area is roughly
# RA=240-252 DEC=6-12 (but not completely rectangular)
# COSMOS
#rlo,rhi = 148.9, 151.2
#dlo,dhi = 0.9, 3.5
# A nice well-behaved region (EDR2/3)
# rlo,rhi = 243.6, 244.6
# dlo,dhi = 8.1, 8.6
# 56 bricks, ~725 CCDs
#B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
# 240 bricks, ~3000 CCDs
#B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
# 535 bricks, ~7000 CCDs
#B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))
if opt.region in ['test1', 'test2', 'test3', 'test4']:
nm = dict(test1='2446p115', # weird stuff around bright star
test2='1183p292', # faint sources around bright galaxy
test3='3503p005', # DES
test4='1163p277', # Pollux
)[opt.region]
B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
log('Cut to', len(B), 'bricks')
log(B.ra, B.dec)
dlo,dhi = -90,90
rlo,rhi = 0, 360
elif opt.region == 'edr':
# EDR:
# 535 bricks, ~7000 CCDs
rlo,rhi = 240,245
dlo,dhi = 5, 12
elif opt.region == 'dr8-decam':
rlo,rhi = 0, 360
dlo,dhi = -70, 40
log('DR8-DECam region')
elif opt.region == 'edrplus':
rlo,rhi = 235,248
dlo,dhi = 5, 15
elif opt.region == 'edr-south':
rlo,rhi = 240,245
dlo,dhi = 5, 10
elif opt.region == 'cosmos1':
# 16 bricks in the core of the COSMOS field.
rlo,rhi = 149.75, 150.75
dlo,dhi = 1.6, 2.6
elif opt.region == 'pristine':
# Stream?
rlo,rhi = 240,250
dlo,dhi = 10,15
elif opt.region == 'des':
dlo, dhi = -6., 4.
rlo, rhi = 317., 7.
T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
log('Cut to', len(T), 'CCDs with "CPDES82" in filename')
elif opt.region == 'subdes':
rlo,rhi = 320., 360.
dlo,dhi = -1.25, 1.25
elif opt.region == 'northwest':
rlo,rhi = 240,360
dlo,dhi = 20,40
elif opt.region == 'north':
rlo,rhi = 120,240
dlo,dhi = 20,40
elif opt.region == 'northeast':
rlo,rhi = 0,120
dlo,dhi = 20,40
elif opt.region == 'southwest':
rlo,rhi = 240,360
dlo,dhi = -20,0
elif opt.region == 'south':
rlo,rhi = 120,240
dlo,dhi = -20,0
elif opt.region == 'southeast':
rlo,rhi = 0,120
dlo,dhi = -20,0
elif opt.region == 'southsoutheast':
rlo,rhi = 0,120
dlo,dhi = -20,-10
elif opt.region == 'midwest':
rlo,rhi = 240,360
dlo,dhi = 0,20
elif opt.region == 'middle':
rlo,rhi = 120,240
dlo,dhi = 0,20
elif opt.region == 'mideast':
rlo,rhi = 0,120
dlo,dhi = 0,20
elif opt.region == 'grz':
# Bricks with grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
log('Cut to', len(B), 'bricks with grz coverage')
elif opt.region == 'nogrz':
# Bricks without grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
log('Cut to', len(B), 'bricks withOUT grz coverage')
elif opt.region == 'deep2':
rlo,rhi = 250,260
dlo,dhi = 30,35
elif opt.region == 'deep2f2':
rlo,rhi = 251.4, 254.4
dlo,dhi = 34.6, 35.3
elif opt.region == 'deep2f3':
rlo,rhi = 351.25, 353.75
dlo,dhi = 0, 0.5
elif opt.region == 'deep3':
rlo,rhi = 214,216
dlo,dhi = 52.25,53.25
elif opt.region == 'virgo':
rlo,rhi = 185,190
dlo,dhi = 10, 15
elif opt.region == 'virgo2':
rlo,rhi = 182,192
dlo,dhi = 8, 18
elif opt.region == 'coma':
# van Dokkum et al Coma cluster ultra-diffuse galaxies: 3x3 field centered on Coma cluster
rc,dc = 195., 28.
dd = 1.5
cosdec = np.cos(np.deg2rad(dc))
rlo,rhi = rc - dd/cosdec, rc + dd/cosdec
dlo,dhi = dc - dd, dc + dd
elif opt.region == 'lsb':
rlo,rhi = 147.2, 147.8
dlo,dhi = -0.4, 0.4
elif opt.region == 'eboss-sgc':
# generous boundaries to make sure get all relevant images
# RA -45 to +45
# Dec -5 to +7
rlo,rhi = 310., 50.
dlo,dhi = -6., 6.
elif opt.region == 'eboss-ngc':
# generous boundaries to make sure get all relevant images
# NGC ELGs
# RA 115 to 175
# Dec 15 to 30
# rlo,rhi = 122., 177.
# dlo,dhi = 12., 32.
rlo,rhi = 126., 168.
dlo,dhi = 18., 33.
elif opt.region == 'mzls':
dlo,dhi = -10., 90. # -10: pull in Stripe 82 data too
elif opt.region == 'dr4-bootes':
# https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
#dlo,dhi = 34., 35.
#rlo,rhi = 209.5, 210.5
dlo,dhi = 33., 36.
rlo,rhi = 216.5, 219.5
elif opt.region == 'des-sn-x3':
#rlo,rhi = 36., 37.
#dlo,dhi = -5., -4.
rlo,rhi = 36., 36.5
dlo,dhi = -4.5, -4.
elif opt.region == 'ngc2632':
# open cluster
rlo,rhi = 129.0, 131.0
dlo,dhi = 19.0, 20.5
elif opt.region == 'dr8sky':
rlo,rhi = 35.0, 37.0
dlo,dhi = -3.0, -1.0
# ADM DR8 test regions, see, e.g.:
# https://desi.lbl.gov/trac/wiki/DecamLegacy/DR8#Testregions
elif opt.region == 'dr8-test-s82':
rlo, rhi = 0, 45
dlo, dhi = -1.25, 1.25
elif opt.region == 'dr8-test-hsc-sgc':
rlo, rhi = 30, 40
dlo, dhi = -6.5, -1.25
elif opt.region == 'dr8-test-hsc-ngc':
rlo, rhi = 177.5, 182.5
dlo, dhi = -1, 1
elif opt.region == 'dr8-test-edr':
rlo, rhi = 240, 245
dlo, dhi = 5, 12
elif opt.region == 'dr8-test-hsc-north':
rlo, rhi = 240, 250
dlo, dhi = 42, 45
elif opt.region == 'dr8-test-deep2-egs':
rlo, rhi = 213, 216.5
dlo, dhi = 52, 54
elif opt.region == 'dr8-test-overlap':
rlo, rhi = 132, 140.5
dlo, dhi = 31.5, 35
if opt.mindec is not None:
dlo = opt.mindec
if opt.maxdec is not None:
dhi = opt.maxdec
if opt.minra is not None:
rlo = opt.minra
if opt.maxra is not None:
rhi = opt.maxra
if rlo < rhi:
B.cut((B.ra >= rlo) * (B.ra <= rhi) *
(B.dec >= dlo) * (B.dec <= dhi))
else: # RA wrap
B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) *
(B.dec >= dlo) * (B.dec <= dhi))
log(len(B), 'bricks in range; cut Dec range', B.dec.min(), B.dec.max())
#for name in B.get('brickname'):
# print(name)
#B.writeto('bricks-cut.fits')
bricksize = 0.25
# A bit more than 0.25-degree brick radius + Bok image radius ~ 0.57
search_radius = 1.05 * np.sqrt(2.) * (bricksize +
(0.455 * 4096 / 3600.))/2.
log(len(T), 'CCDs')
log(len(B), 'Bricks')
I,J,_ = match_radec(B.ra, B.dec, T.ra, T.dec, search_radius,
nearest=True)
B.cut(I)
log('Cut to', len(B), 'bricks near CCDs')
log('Bricks Dec range:', B.dec.min(), B.dec.max())
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# plt.title('DR3 bricks')
# plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
# plt.savefig('bricks.png')
if opt.touching:
I,J,_ = match_radec(T.ra, T.dec, B.ra, B.dec, search_radius,
nearest=True)
# list the ones that will be cut
# drop = np.ones(len(T))
# drop[I] = False
# for i in np.flatnonzero(drop):
# from astrometry.util.starutil_numpy import degrees_between
# dists = degrees_between(B.ra, B.dec, T.ra[i], T.dec[i])
# mindist = min(dists)
# print('Dropping:', T.ra[i], T.dec[i], 'min dist', mindist, 'search_radius', search_radius)
T.cut(I)
log('Cut to', len(T), 'CCDs near bricks')
if opt.forced:
log('Writing forced-photometry commands to', opt.out)
f = open(opt.out,'w')
log('Total of', len(T), 'CCDs')
#T.cut(allI)
camexp = set([(c,e) for c,e in zip(T.camera, T.expnum)])
print(len(camexp), 'unique camera/exposure pairs')
for cam,exp in camexp:
#expstr = '%08i' % exp
#outfn = os.path.join('forced', cam, expstr[:5], 'forced-%s-%s.fits' % (cam, exp))
#f.write('%s %s all %s\n' % (cam, exp, outfn))
f.write('%s %s\n' % (cam, exp))
f.close()
log('Wrote', opt.out)
return 0
# sort by RA increasing
B.cut(np.argsort(B.ra))
if opt.save_to_fits:
assert(opt.touching)
# Write cut tables to file
for tab,typ in zip([B,T],['bricks','ccds']):
fn='%s-%s-cut.fits' % (typ,opt.region)
if os.path.exists(fn):
os.remove(fn)
tab.writeto(fn)
log('Wrote %s' % fn)
# Write text files listing ccd and filename names
# nm1,nm2= 'ccds-%s.txt'% opt.region,'filenames-%s.txt' % opt.region
# if os.path.exists(nm1):
# os.remove(nm1)
# if os.path.exists(nm2):
# os.remove(nm2)
# f1,f2=open(nm1,'w'),open(nm2,'w')
# fns= list(set(T.get('image_filename')))
# for fn in fns:
# f2.write('%s\n' % fn.strip())
# for ti in T:
# f1.write('%s\n' % ti.get('image_filename').strip())
# f1.close()
# f2.close()
# log('Wrote *-names.txt')
if opt.touching:
if want_bricks:
# Shortcut the list of bricks that are definitely touching CCDs --
# a brick-ccd pair within this radius must be touching.
closest_radius = 0.95 * (bricksize + 0.262 * 2048 / 3600.) / 2.
J1,_,_ = match_radec(B.ra, B.dec, T.ra, T.dec, closest_radius, nearest=True)
log(len(J1), 'of', len(B), 'bricks definitely touch CCDs')
tocheck = np.ones(len(B), bool)
tocheck[J1] = False
J2 = []
for j in np.flatnonzero(tocheck):
b = B[j]
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickname)
if len(I) == 0:
continue
J2.append(j)
J = np.hstack((J1, J2))
J = np.sort(J).astype(int)
B.cut(J)
log('Cut to', len(B), 'bricks touching CCDs')
else:
J = []
allI = set()
for j,b in enumerate(B):
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickname)
if len(I) == 0:
continue
allI.update(I)
J.append(j)
allI = list(allI)
allI.sort()
B.cut(np.array(J))
log('Cut to', len(B), 'bricks touching CCDs')
elif opt.near:
# Find CCDs near bricks
allI,_,_ = match_radec(T.ra, T.dec, B.ra, B.dec, search_radius, nearest=True)
# Find bricks near CCDs
J,_,_ = match_radec(B.ra, B.dec, T.ra, T.dec, search_radius, nearest=True)
B.cut(J)
log('Cut to', len(B), 'bricks near CCDs')
else:
allI = np.arange(len(T))
if opt.byexp:
_,eI = np.unique(T.expnum[allI], return_index=True)
allI = allI[eI]
print('Cut to', len(allI), 'expnums')
if opt.nccds:
from queue import Queue
from threading import Thread
log('Checking number of CCDs per brick')
def worker():
while True:
i = q.get()
if i is None:
break
b = B[i]
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(b.brickname, len(I))
q.task_done()
q = Queue()
num_threads = 24
threads = []
for i in range(num_threads):
t = Thread(target=worker)
t.start()
threads.append(t)
for i in range(len(B)):
q.put(i)
q.join()
for i in range(num_threads):
q.put(None)
for t in threads:
t.join()
if opt.write_ccds:
T[allI].writeto(opt.write_ccds)
log('Wrote', opt.write_ccds)
if want_bricks:
# Print the list of bricks and exit.
for b in B:
print(b.brickname)
if opt.save_to_fits:
B.writeto('bricks-%s-touching.fits' % opt.region)
if not want_ccds:
return 0
## Be careful here -- T has been cut; we want to write out T.index.
## 'allI' contains indices into T.
if opt.stage is not None:
cmd_pat = 'rsync -LRarv %s %s'
fns = set()
for iccd in allI:
im = survey.get_image_object(T[iccd])
fns.update([im.imgfn, im.wtfn, im.dqfn, im.psffn, im.merged_psffn,
im.merged_skyfn, im.skyfn])
for i,fn in enumerate(fns):
print('File', i+1, 'of', len(fns), ':', fn)
if not os.path.exists(fn):
print('No such file:', fn)
continue
base = survey.get_survey_dir()
if base.endswith('/'):
base = base[:-1]
rel = os.path.relpath(fn, base)
dest = os.path.join(opt.stage, rel)
print('Dest:', dest)
if os.path.exists(dest):
print('Exists:', dest)
continue
cmd = cmd_pat % ('%s/./%s' % (base, rel), opt.stage)
print(cmd)
rtn = os.system(cmd)
assert(rtn == 0)
return 0
if opt.lsb:
log('Writing LSB commands to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
for j,i in enumerate(allI):
exp = T.expnum[i]
ext = T.ccdname[i].strip()
outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
f.write('python legacyanalysis/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext))
f.close()
log('Wrote', opt.out)
return 0
log('Writing calibs to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
batch = []
def write_batch(f, batch, cmd):
if cmd is None:
cmd = ''
f.write(cmd + ' '.join(batch) + '\n')
cmd = None
if opt.command:
cmd = 'python legacypipe/run-calib.py '
if opt.opt is not None:
cmd += opt.opt + ' '
for j,i in enumerate(allI):
if opt.delete_sky:
log(j+1, 'of', len(allI))
im = survey.get_image_object(T[i])
if opt.delete_sky and os.path.exists(im.skyfn):
log(' deleting:', im.skyfn)
os.unlink(im.skyfn)
if opt.command:
if opt.byexp:
s = '--expnum %i' % (T.expnum[i])
else:
s = '%i-%s' % (T.expnum[i], T.ccdname[i])
prefix = 'python legacypipe/run-calib.py '
if opt.opt is not None:
prefix = prefix + opt.opt
#('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
# (T.expnum[i], T.ccdname[i]))
else:
s = '%i' % T.index[i]
prefix = ''
if j < 10:
print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i],
'filename', T.image_filename[i])
if not opt.nper:
f.write(prefix + s + '\n')
else:
batch.append(s)
if len(batch) >= opt.nper:
write_batch(f, batch, cmd)
batch = []
if len(batch):
write_batch(f, batch, cmd)
f.close()
log('Wrote', opt.out)
return 0
print(len(C), 'CCDs in', region)
C.cut(np.array([f in bands for f in C.filter]))
print(len(C), 'in', bands)
C.cut(C.exptime >= 50.)
print(len(C), 'with exptime >= 50 sec')
C.cut(survey.photometric_ccds(C))
print(len(C), 'photometric')
C.cut(np.lexsort((C.expnum, C.filter)))
C.writeto(cfn)
else:
C = fits_table(cfn)
plt.clf()
plt.plot(C.ra, C.dec, 'bo', alpha=0.2)
plt.savefig('rd.png')
# Find the unique exposures (not CCDs), save as E.
C.galnorm = C.galnorm_mean
nil, I = np.unique(C.expnum, return_index=True)
E = C[I]
print(len(E), 'exposures')
if False:
# Find the extinction in the center of the COSMOS region and apply it
# as a correction to our target depths (so that we reach that depth
# for de-reddened mags).
plot.plot('outline')
mjd0 = 57789.0
db = ComputedExptime.objects.filter(starttime__gt=mjd0,
starttime__lt=mjd0 + 0.8)
print(db.count(), 'entries from', mjd0)
adj = db.filter(adjfactor__gt=1.)
print(adj.count(), 'entries with adjfactor > 1')
#adj = db_to_fits(adj)
ps = PlotSequence('adjust', format='%03i')
tiles = fits_table('obstatus/mosaic-tiles_obstatus.fits')
tiles.cut(tiles.get('pass') <= 3)
tiles.cut(tiles.dec >= 30)
for a in adj[:10]:
tileid = a.tileid
print()
print('tile id', tileid)
i = np.flatnonzero(tiles.tileid == tileid)
tile = tiles[i[0]]
ra, dec = tile.ra, tile.dec
tilepass = tile.get('pass')
others = a.otherpasses_set.all()
print(others.count(), 'other passes involved')
others = others.filter(depth__gt=1, depth__lt=30)
def __init__(self, fn=None, ext=1, Ti=None):
'''
If *fn* is not None, reads a PsfEx file from the given filename and *ext*.
Else if *Ti* is not None, reads from one row of a merged PsfEx table.
'''
if fn is not None:
from astrometry.util.fits import fits_table
T = fits_table(fn, ext=ext)
ims = T.psf_mask[0]
ims = ims.astype(np.float32)
#print('Got', ims.shape, 'PSF images')
hdr = T.get_header()
ngrp = hdr['POLNGRP']
assert (ngrp in [0, 1])
if ngrp == 0:
# Constant PSF.
degree = 0
x0 = y0 = 0.
xscale = yscale = 1.
else:
assert (hdr['POLNGRP'] == 1)
# PSF distortion bases are polynomials of x,y
assert (hdr['POLNAME1'].strip() == 'X_IMAGE')
assert (hdr['POLNAME2'].strip() == 'Y_IMAGE')
assert (hdr['POLGRP1'] == 1)
assert (hdr['POLGRP2'] == 1)
x0 = hdr.get('POLZERO1')
xscale = hdr.get('POLSCAL1')
y0 = hdr.get('POLZERO2')
yscale = hdr.get('POLSCAL2')
degree = hdr.get('POLDEG1')
self.sampling = hdr.get('PSF_SAMP')
#print('PsfEx sampling:', self.sampling)
# number of terms in polynomial
ne = (degree + 1) * (degree + 2) // 2
assert (hdr['PSFAXIS3'] == ne)
assert (len(ims.shape) == 3)
assert (ims.shape[0] == ne)
self.psfbases = ims
self.xscale, self.yscale = xscale, yscale
self.degree = degree
#print('PsfEx degree:', self.degree)
bh, bw = self.psfbases[0].shape
self.radius = (bh + 1) / 2.
self.x0, self.y0 = x0, y0
self.fwhm = hdr.get('PSF_FWHM')
elif Ti is not None:
self.sampling = Ti.psf_samp
degree = Ti.poldeg1
# PSF distortion bases are polynomials of x,y
if degree > 0:
assert (Ti.polname1.strip() == 'X_IMAGE')
assert (Ti.polname2.strip() == 'Y_IMAGE')
assert (Ti.polgrp1 == 1)
assert (Ti.polgrp2 == 1)
assert (Ti.polngrp == 1)
self.x0 = Ti.polzero1
self.y0 = Ti.polzero2
self.xscale = Ti.polscal1
self.yscale = Ti.polscal2
self.degree = degree
# number of terms in polynomial
ne = (degree + 1) * (degree + 2) / 2
assert (Ti.psfaxis3 == ne)
ims = Ti.psf_mask
ims = ims.astype(np.float32)
assert (len(ims.shape) == 3)
assert (ims.shape[0] == ne)
self.psfbases = ims
bh, bw = self.psfbases[0].shape
self.radius = (bh + 1) / 2.
self.fwhm = Ti.psf_fwhm
ps.savefig()
sys.exit(0)
# Test RexGalaxy
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6-rex'
main(args=['--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise',
#'--rex', #'--plots',
'--survey-dir', surveydir,
'--outdir', outdir] + extra_args)
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert(os.path.exists(fn))
T = fits_table(fn)
assert(len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert(T.type[0] == 'PSF ')
cmd = ('(cd %s && sha256sum -c %s)' %
(outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum')))
print(cmd)
rtn = os.system(cmd)
assert(rtn == 0)
# Test with a Tycho-2 star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6'
main(args=['--brick', '1102p240', '--zoom', '500', '600', '650', '750',
def main():
# I read a DESI DR8 target catalog, cut to ELGs, then took a narrow
# r-mag slice around the peak of that distribution;
# desi/target/catalogs/dr8/0.31.1/targets/main/resolve/targets-dr8-hp-1,5,11,50,55,60,83,84,86,89,91,98,119,155,158,174,186,187,190.fits')
# Then took the median g and z mags
# And opened the coadd invvar mags for a random brick in that footprint
# (0701p000) to find the median per-pixel invvars.
r = 23.0
g = 23.4
z = 22.2
# Selecting EXPs, the peak of the shapeexp_r was ~ 0.75".
r_e = 0.75
# Image properties:
giv = 77000.
riv = 27000.
ziv = 8000.
# PSF sizes were roughly equal, 1.16, 1.17, 1.19"
# -> sigma = 1.9 DECam pixels
psf_sigma = 1.9
H, W = 1000, 1000
seed = 42
np.random.seed(seed)
ra, dec = 70., 1.
brick = BrickDuck(ra, dec, 'custom-0700p010')
wcs = wcs_for_brick(brick, W=W, H=H)
bands = 'grz'
tims = []
for band, iv in zip(bands, [giv, riv, ziv]):
img = np.zeros((H, W), np.float32)
tiv = np.zeros_like(img) + iv
s = psf_sigma**2
psf = GaussianMixturePSF(1., 0., 0., s, s, 0.)
twcs = ConstantFitsWcs(wcs)
sky = ConstantSky(0.)
photocal = LinearPhotoCal(1., band=band)
tai = TAITime(None, mjd=55700.)
tim = tractor.Image(data=img,
invvar=tiv,
psf=psf,
wcs=twcs,
sky=sky,
photocal=photocal,
name='fake %s' % band,
time=tai)
tim.skyver = ('1', '1')
tim.psfver = ('1', '1')
tim.plver = '1'
tim.x0 = tim.y0 = 0
tim.subwcs = wcs
tim.psfnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
tim.galnorm = tim.psfnorm
tim.propid = '2020A-000'
tim.band = band
tim.dq = None
tim.sig1 = 1. / np.sqrt(iv)
tim.psf_sigma = psf_sigma
tim.primhdr = fitsio.FITSHDR()
tims.append(tim)
# Simulated catalog
gflux = NanoMaggies.magToNanomaggies(g)
rflux = NanoMaggies.magToNanomaggies(r)
zflux = NanoMaggies.magToNanomaggies(z)
box = 50
CX, CY = np.meshgrid(np.arange(box // 2, W, box),
np.arange(box // 2, H, box))
ny, nx = CX.shape
BA, PHI = np.meshgrid(np.linspace(0.1, 1.0, nx), np.linspace(0., 180., ny))
cat = []
for cx, cy, ba, phi in zip(CX.ravel(), CY.ravel(), BA.ravel(),
PHI.ravel()):
#print('x,y %.1f,%.1f, ba %.2f, phi %.2f' % (cx, cy, ba, phi))
r, d = wcs.pixelxy2radec(cx + 1, cy + 1)
src = ExpGalaxy(RaDecPos(r, d),
NanoMaggies(order=bands, g=gflux, r=rflux, z=zflux),
EllipseE.fromRAbPhi(r_e, ba, phi))
cat.append(src)
from legacypipe.catalog import prepare_fits_catalog
TT = fits_table()
TT.bx = CX.ravel()
TT.by = CY.ravel()
TT.ba = BA.ravel()
TT.phi = PHI.ravel()
tcat = tractor.Catalog(*cat)
T2 = prepare_fits_catalog(tcat, None, TT, bands, save_invvars=False)
T2.writeto('sim-cat.fits')
tr = Tractor(tims, cat)
for band, tim in zip(bands, tims):
mod = tr.getModelImage(tim)
mod += np.random.standard_normal(
size=tim.shape) * 1. / tim.getInvError()
fitsio.write('sim-%s.fits' % band, mod, clobber=True)
tim.data = mod
ccds = fits_table()
ccds.filter = np.array([f for f in bands])
ccds.ccd_cuts = np.zeros(len(ccds), np.int16)
ccds.imgfn = np.array([tim.name for tim in tims])
ccds.propid = np.array(['2020A-000'] * len(ccds))
ccds.fwhm = np.zeros(len(ccds), np.float32) + psf_sigma * 2.35
ccds.mjd_obs = np.zeros(len(ccds))
ccds.camera = np.array(['fake'] * len(ccds))
survey = FakeLegacySurvey(ccds, tims)
import logging
verbose = False
if verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
run_brick(None,
survey,
radec=(ra, dec),
width=W,
height=H,
do_calibs=False,
gaia_stars=False,
large_galaxies=False,
tycho_stars=False,
forceall=True,
outliers=False) #, stages=['image_coadds'])
def unwise_forcedphot(cat,
tiles,
band=1,
roiradecbox=None,
use_ceres=True,
ceres_block=8,
save_fits=False,
get_models=False,
ps=None,
psf_broadening=None,
pixelized_psf=False,
get_masks=None,
move_crpix=False,
modelsky_dir=None,
tag=None):
'''
Given a list of tractor sources *cat*
and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id)
runs forced photometry, returning a FITS table the same length as *cat*.
*get_masks*: the WCS to resample mask bits into.
'''
from tractor import PointSource, Tractor, ExpGalaxy, DevGalaxy
from tractor.sersic import SersicGalaxy
if tag is None:
tag = ''
else:
tag = tag + ': '
if not pixelized_psf and psf_broadening is None:
# PSF broadening in post-reactivation data, by band.
# Newer version from Aaron's email to decam-chatter, 2018-06-14.
broadening = {1: 1.0405, 2: 1.0346, 3: None, 4: None}
psf_broadening = broadening[band]
if False:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('wise-forced-w%i' % band)
plots = (ps is not None)
if plots:
import pylab as plt
wantims = (plots or save_fits or get_models)
wanyband = 'w'
if get_models:
models = []
wband = 'w%i' % band
Nsrcs = len(cat)
phot = fits_table()
# Filled in based on unique tile overlap
phot.wise_coadd_id = np.array([' '] * Nsrcs, dtype='U8')
phot.wise_x = np.zeros(Nsrcs, np.float32)
phot.wise_y = np.zeros(Nsrcs, np.float32)
phot.set('psfdepth_%s' % wband, np.zeros(Nsrcs, np.float32))
nexp = np.zeros(Nsrcs, np.int16)
mjd = np.zeros(Nsrcs, np.float64)
central_flux = np.zeros(Nsrcs, np.float32)
ra = np.array([src.getPosition().ra for src in cat])
dec = np.array([src.getPosition().dec for src in cat])
fskeys = ['prochi2', 'profracflux']
fitstats = {}
if get_masks:
mh, mw = get_masks.shape
maskmap = np.zeros((mh, mw), np.uint32)
tims = []
for tile in tiles:
info(tag + 'Reading WISE tile', tile.coadd_id, 'band', band)
tim = get_unwise_tractor_image(tile.unwise_dir,
tile.coadd_id,
band,
bandname=wanyband,
roiradecbox=roiradecbox)
if tim is None:
debug('Actually, no overlap with WISE coadd tile', tile.coadd_id)
continue
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage(),
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data' % tag)
ps.savefig()
plt.clf()
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
range=(-5, 10),
bins=100)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.title(tag)
ps.savefig()
if move_crpix and band in [1, 2]:
realwcs = tim.wcs.wcs
x, y = realwcs.crpix
tile_crpix = tile.get('crpix_w%i' % band)
dx = tile_crpix[0] - 1024.5
dy = tile_crpix[1] - 1024.5
realwcs.set_crpix(x + dx, y + dy)
debug('unWISE', tile.coadd_id, 'band', band, 'CRPIX', x, y,
'shift by', dx, dy, 'to', realwcs.crpix)
if modelsky_dir and band in [1, 2]:
fn = os.path.join(modelsky_dir,
'%s.%i.mod.fits' % (tile.coadd_id, band))
if not os.path.exists(fn):
raise RuntimeError('WARNING: does not exist:', fn)
x0, x1, y0, y1 = tim.roi
bg = fitsio.FITS(fn)[2][y0:y1, x0:x1]
assert (bg.shape == tim.shape)
if plots:
plt.clf()
plt.subplot(1, 2, 1)
plt.imshow(tim.getImage(),
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=5 * sig1)
plt.subplot(1, 2, 2)
plt.imshow(bg,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=5 * sig1)
tag = '%s W%i' % (tile.coadd_id, band)
plt.suptitle(tag)
ps.savefig()
plt.clf()
ha = dict(range=(-5, 10), bins=100, histtype='step')
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
color='b',
label='Original',
**ha)
plt.hist(((tim.getImage() - bg) *
tim.inverr)[tim.inverr > 0].ravel(),
color='g',
label='Minus Background',
**ha)
plt.axvline(0, color='k', alpha=0.5)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.legend()
plt.title(tag + ': background')
ps.savefig()
# Actually subtract the background!
tim.data -= bg
# Floor the per-pixel variances,
# and add Poisson contribution from sources
if band in [1, 2]:
# in Vega nanomaggies per pixel
floor_sigma = {1: 0.5, 2: 2.0}
poissons = {1: 0.15, 2: 0.3}
with np.errstate(divide='ignore'):
new_ie = 1. / np.sqrt(
(1. / tim.inverr)**2 + floor_sigma[band] +
poissons[band]**2 * np.maximum(0., tim.data))
new_ie[tim.inverr == 0] = 0.
if plots:
plt.clf()
plt.plot((1. / tim.inverr[tim.inverr > 0]).ravel(),
(1. / new_ie[tim.inverr > 0]).ravel(), 'b.')
plt.title('unWISE per-pixel error: %s band %i' %
(tile.coadd_id, band))
plt.xlabel('original')
plt.ylabel('floored')
ps.savefig()
assert (np.all(np.isfinite(new_ie)))
assert (np.all(new_ie >= 0.))
tim.inverr = new_ie
# Expand a 3-pixel radius around weight=0 (saturated) pixels
# from Eddie via crowdsource
# https://github.com/schlafly/crowdsource/blob/7069da3e7d9d3124be1cbbe1d21ffeb63fc36dcc/python/wise_proc.py#L74
## FIXME -- W3/W4 ??
satlimit = 85000
msat = ((tim.data > satlimit) | ((tim.nims == 0) &
(tim.nuims > 1)))
from scipy.ndimage.morphology import binary_dilation
xx, yy = np.mgrid[-3:3 + 1, -3:3 + 1]
dilate = xx**2 + yy**2 <= 3**2
msat = binary_dilation(msat, dilate)
nbefore = np.sum(tim.inverr == 0)
tim.inverr[msat] = 0
nafter = np.sum(tim.inverr == 0)
debug('Masking an additional', (nafter - nbefore),
'near-saturated pixels in unWISE', tile.coadd_id, 'band',
band)
# Read mask file?
if get_masks:
from astrometry.util.resample import resample_with_wcs, OverlapError
# unwise_dir can be a colon-separated list of paths
tilemask = None
for d in tile.unwise_dir.split(':'):
fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id,
'unwise-%s-msk.fits.gz' % tile.coadd_id)
if os.path.exists(fn):
debug('Reading unWISE mask file', fn)
x0, x1, y0, y1 = tim.roi
tilemask = fitsio.FITS(fn)[0][y0:y1, x0:x1]
break
if tilemask is None:
info('unWISE mask file for tile', tile.coadd_id,
'does not exist')
else:
try:
tanwcs = tim.wcs.wcs
assert (tanwcs.shape == tilemask.shape)
Yo, Xo, Yi, Xi, _ = resample_with_wcs(get_masks,
tanwcs,
intType=np.int16)
# Only deal with mask pixels that are set.
I, = np.nonzero(tilemask[Yi, Xi] > 0)
# Trim to unique area for this tile
rr, dd = get_masks.pixelxy2radec(Xo[I] + 1, Yo[I] + 1)
good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2,
tile.dec1, tile.dec2)
I = I[good]
maskmap[Yo[I], Xo[I]] = tilemask[Yi[I], Xi[I]]
except OverlapError:
# Shouldn't happen by this point
print('Warning: no overlap between WISE tile',
tile.coadd_id, 'and brick')
if plots:
plt.clf()
plt.imshow(tilemask, interpolation='nearest', origin='lower')
plt.title('Tile %s: mask' % tile.coadd_id)
ps.savefig()
plt.clf()
plt.imshow(maskmap, interpolation='nearest', origin='lower')
plt.title('Tile %s: accumulated maskmap' % tile.coadd_id)
ps.savefig()
# The tiles have some overlap, so zero out pixels outside the
# tile's unique area.
th, tw = tim.shape
xx, yy = np.meshgrid(np.arange(tw), np.arange(th))
rr, dd = tim.wcs.wcs.pixelxy2radec(xx + 1, yy + 1)
unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1,
tile.dec2)
debug('Tile', tile.coadd_id, '- total of', np.sum(unique),
'unique pixels out of', len(unique.flat), 'total pixels')
if get_models:
# Save the inverr before blanking out non-unique pixels, for making coadds with no gaps!
# (actually, slightly more subtly, expand unique area by 1 pixel)
from scipy.ndimage.morphology import binary_dilation
du = binary_dilation(unique)
tim.coadd_inverr = tim.inverr * du
tim.inverr[unique == False] = 0.
del xx, yy, rr, dd, unique
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage() * (tim.inverr > 0),
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data (unique)' % tag)
ps.savefig()
if pixelized_psf:
from unwise_psf import unwise_psf
if (band == 1) or (band == 2):
# we only have updated PSFs for W1 and W2
psfimg = unwise_psf.get_unwise_psf(band,
tile.coadd_id,
modelname='neo6_unwisecat')
else:
psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id)
if band == 4:
# oversample (the unwise_psf models are at native W4 5.5"/pix,
# while the unWISE coadds are made at 2.75"/pix.
ph, pw = psfimg.shape
subpsf = np.zeros((ph * 2 - 1, pw * 2 - 1), np.float32)
from astrometry.util.util import lanczos3_interpolate
xx, yy = np.meshgrid(
np.arange(0., pw - 0.51, 0.5, dtype=np.float32),
np.arange(0., ph - 0.51, 0.5, dtype=np.float32))
xx = xx.ravel()
yy = yy.ravel()
ix = xx.astype(np.int32)
iy = yy.astype(np.int32)
dx = (xx - ix).astype(np.float32)
dy = (yy - iy).astype(np.float32)
psfimg = psfimg.astype(np.float32)
rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat],
[psfimg])
if plots:
plt.clf()
plt.imshow(psfimg, interpolation='nearest', origin='lower')
plt.title('Original PSF model')
ps.savefig()
plt.clf()
plt.imshow(subpsf, interpolation='nearest', origin='lower')
plt.title('Subsampled PSF model')
ps.savefig()
psfimg = subpsf
del xx, yy, ix, iy, dx, dy
from tractor.psf import PixelizedPSF
psfimg /= psfimg.sum()
fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0}
psfimg *= fluxrescales[band]
tim.psf = PixelizedPSF(psfimg)
if psf_broadening is not None and not pixelized_psf:
# psf_broadening is a factor by which the PSF FWHMs
# should be scaled; the PSF is a little wider
# post-reactivation.
psf = tim.getPsf()
from tractor import GaussianMixturePSF
if isinstance(psf, GaussianMixturePSF):
debug('Broadening PSF: from', psf)
p0 = psf.getParams()
pnames = psf.getParamNames()
p1 = [
p * psf_broadening**2 if 'var' in name else p
for (p, name) in zip(p0, pnames)
]
psf.setParams(p1)
debug('Broadened PSF:', psf)
else:
print(
'WARNING: cannot apply psf_broadening to WISE PSF of type',
type(psf))
wcs = tim.wcs.wcs
_, fx, fy = wcs.radec2pixelxy(ra, dec)
x = np.round(fx - 1.).astype(int)
y = np.round(fy - 1.).astype(int)
good = (x >= 0) * (x < tw) * (y >= 0) * (y < th)
# Which sources are in this brick's unique area?
usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1,
tile.dec2)
I, = np.nonzero(good * usrc)
nexp[I] = tim.nuims[y[I], x[I]]
if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'):
mjd[I] = (tim.mjdmin + tim.mjdmax) / 2.
phot.wise_coadd_id[I] = tile.coadd_id
phot.wise_x[I] = fx[I] - 1.
phot.wise_y[I] = fy[I] - 1.
central_flux[I] = tim.getImage()[y[I], x[I]]
del x, y, good, usrc
# PSF norm for depth
psf = tim.getPsf()
h, w = tim.shape
patch = psf.getPointSourcePatch(h // 2, w // 2).patch
psfnorm = np.sqrt(np.sum(patch**2))
# To handle zero-depth, we return 1/nanomaggies^2 units rather than mags.
# In the small empty patches of the sky (eg W4 in 0922p702), we get sig1 = NaN
if np.isfinite(tim.sig1):
phot.get('psfdepth_%s' % wband)[I] = 1. / (tim.sig1 / psfnorm)**2
tim.tile = tile
tims.append(tim)
if plots:
plt.clf()
mn, mx = 0.1, 20000
plt.hist(np.log10(np.clip(central_flux, mn, mx)),
bins=100,
range=(np.log10(mn), np.log10(mx)))
logt = np.arange(0, 5)
plt.xticks(logt, ['%i' % i for i in 10.**logt])
plt.title('Central fluxes (W%i)' % band)
plt.axvline(np.log10(20000), color='k')
plt.axvline(np.log10(1000), color='k')
ps.savefig()
# Eddie's non-secret recipe:
#- central pixel <= 1000: 19x19 pix box size
#- central pixel in 1000 - 20000: 59x59 box size
#- central pixel > 20000 or saturated: 149x149 box size
#- object near "bright star": 299x299 box size
nbig = nmedium = nsmall = 0
for src, cflux in zip(cat, central_flux):
if cflux > 20000:
R = 100
nbig += 1
elif cflux > 1000:
R = 30
nmedium += 1
else:
R = 15
nsmall += 1
if isinstance(src, PointSource):
src.fixedRadius = R
else:
### FIXME -- sizes for galaxies..... can we set PSF size separately?
galrad = 0
# RexGalaxy is a subclass of ExpGalaxy
if isinstance(src, (ExpGalaxy, DevGalaxy, SersicGalaxy)):
galrad = src.shape.re
pixscale = 2.75
src.halfsize = int(np.hypot(R, galrad * 5 / pixscale))
debug('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall,
'small')
tractor = Tractor(tims, cat)
if use_ceres:
from tractor.ceres_optimizer import CeresOptimizer
tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block)
tractor.freezeParamsRecursive('*')
tractor.thawPathsTo(wanyband)
t0 = Time()
R = tractor.optimize_forced_photometry(fitstats=True,
variance=True,
shared_params=False,
wantims=wantims)
info(tag + 'unWISE forced photometry took', Time() - t0)
if use_ceres:
term = R.ceres_status['termination']
# Running out of memory can cause failure to converge and term
# status = 2. Fail completely in this case.
if term != 0:
info(tag + 'Ceres termination status:', term)
raise RuntimeError('Ceres terminated with status %i' % term)
if wantims:
ims1 = R.ims1
# can happen if empty source list (we still want to generate coadds)
if ims1 is None:
ims1 = R.ims0
flux_invvars = R.IV
if R.fitstats is not None:
for k in fskeys:
x = getattr(R.fitstats, k)
fitstats[k] = np.array(x).astype(np.float32)
if save_fits:
for i, tim in enumerate(tims):
tile = tim.tile
(dat, mod, _, chi, _) = ims1[i]
wcshdr = fitsio.FITSHDR()
tim.wcs.wcs.add_to_header(wcshdr)
tag = 'fit-%s-w%i' % (tile.coadd_id, band)
fitsio.write('%s-data.fits' % tag,
dat,
clobber=True,
header=wcshdr)
fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr)
fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr)
if plots:
# Create models for just the brightest sources
bright_cat = [
src for src in cat if src.getBrightness().getBand(wanyband) > 1000
]
debug('Bright soures:', len(bright_cat))
btr = Tractor(tims, bright_cat)
for tim in tims:
mod = btr.getModelImage(tim)
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
sig1 = tim.sig1
plt.clf()
plt.imshow(mod,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=25 * sig1)
plt.colorbar()
plt.title('%s: bright-star models' % tag)
ps.savefig()
if get_models:
for i, tim in enumerate(tims):
tile = tim.tile
(dat, mod, _, _, _) = ims1[i]
models.append(
(tile.coadd_id, band, tim.wcs.wcs, dat, mod, tim.coadd_inverr))
if plots:
for i, tim in enumerate(tims):
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
(dat, mod, _, chi, _) = ims1[i]
sig1 = tim.sig1
plt.clf()
plt.imshow(dat,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=25 * sig1)
plt.colorbar()
plt.title('%s: data' % tag)
ps.savefig()
plt.clf()
plt.imshow(mod,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=25 * sig1)
plt.colorbar()
plt.title('%s: model' % tag)
ps.savefig()
plt.clf()
plt.imshow(chi,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-5,
vmax=+5)
plt.colorbar()
plt.title('%s: chi' % tag)
ps.savefig()
nm = np.array([src.getBrightness().getBand(wanyband) for src in cat])
nm_ivar = flux_invvars
# Sources out of bounds, eg, never change from their initial
# fluxes. Zero them out instead.
nm[nm_ivar == 0] = 0.
phot.set('flux_%s' % wband, nm.astype(np.float32))
phot.set('flux_ivar_%s' % wband, nm_ivar.astype(np.float32))
for k in fskeys:
phot.set(k + '_' + wband,
fitstats.get(k, np.zeros(len(phot), np.float32)))
phot.set('nobs_%s' % wband, nexp)
phot.set('mjd_%s' % wband, mjd)
rtn = wphotduck()
rtn.phot = phot
rtn.models = None
rtn.maskmap = None
if get_models:
rtn.models = models
if get_masks:
rtn.maskmap = maskmap
return rtn
def toFits(self, fn, data=None, hdr=None, merge=False):
'''
If merge: merge params "amp0", "amp1", ... into an "amp" array.
'''
from astrometry.util.fits import fits_table
if hdr is None:
import fitsio
hdr = fitsio.FITSHDR()
hdr.add_record(
dict(name='PSFEX_T',
value=typestring(type(self)),
comment='PsfEx type'))
hdr.add_record(
dict(name='PSF_TYPE',
value=typestring(self.psfclass),
comment='PsfEx PSF type'))
hdr.add_record(dict(name='PSF_W', value=self.W, comment='Image width'))
hdr.add_record(dict(name='PSF_H', value=self.H,
comment='Image height'))
# hdr.add_record(dict(name='PSF_SCALING',
hdr.add_record(
dict(name='PSF_K',
value=self.K,
comment='Number of PSF components'))
hdr.add_record(
dict(name='PSF_NX',
value=self.nx,
comment='Number of X grid points'))
hdr.add_record(
dict(name='PSF_NY',
value=self.ny,
comment='Number of Y grid points'))
if data is None:
data = self.splinedata
(pp, XX, YY) = data
ny, nx, nparams = pp.shape
assert (ny == self.ny)
assert (nx == self.nx)
X = self.psfclass(*pp[0, 0])
names = X.getParamNames()
hdr.add_record(
dict(name='PSF_NA',
value=len(names),
comment='PSF number of params'))
for i, nm in enumerate(names):
hdr.add_record(
dict(name='PSF_A%i' % i, value=nm, comment='PSF param name'))
T = fits_table()
T.xx = XX.reshape((1, len(XX)))
T.yy = YY.reshape((1, len(YY)))
if merge:
# find like params and group them together.
# assume names like "amp0"
assert (self.K < 10)
pnames = set()
for nm in names:
assert (nm[-1] in '0123456789'[:self.K])
pnames.add(nm[:-1])
assert (len(pnames) * self.K == nparams)
pnames = list(pnames)
pnames.sort()
print('Pnames:', pnames)
namemap = dict([(nm, i) for i, nm in enumerate(names)])
for i, nm in enumerate(pnames):
X = np.empty((1, self.K, ny, nx))
for k in range(self.K):
X[0, k, :, :] = pp[:, :, namemap['%s%i' % (nm, k)]]
T.set(nm, X)
# X = np.dstack([pp[:,:,namemap['%s%i' % (nm, k)]] for k in range(self.K)])
# print 'pname', nm, 'array:', X.shape
# T.set(nm, X.reshape((1,self.K,ny,nx)))
else:
for i, nm in enumerate(names):
T.set(nm, pp[:, :, i].reshape((1, ny, nx)))
T.writeto(fn, header=hdr)
def _bounce_main((i, ccds)):
try:
outfn = 'ccds-annotated/ccds-annotated-%03i.fits' % i
if os.path.exists(outfn):
print('Already exists:', outfn)
return
main(outfn=outfn, ccds=ccds)
except:
import traceback
traceback.print_exc()
if __name__ == '__main__':
TT = [
fits_table('ccds-annotated/ccds-annotated-%03i.fits' % i)
for i in range(515)
]
T = merge_tables(TT)
T.writeto('ccds-annotated.fits')
import sys
sys.exit()
#sys.exit(main())
decals = Decals()
ccds = decals.get_ccds()
from astrometry.util.multiproc import *
#mp = multiproc(8)
mp = multiproc(4)
N = 1000
def run_tiles(X):
tiles, tag = X
print('Running', tag, '-', len(tiles), 'tiles')
# Aaron's file has all images share the boresight CRVAL, so they have large CRPIX values.
T = fits_table(
'/global/cfs/cdirs/desi/users/ameisner/GFA/gfa_reduce_etc/gfa_wcs+focus.bigtan-zenith.fits'
)
Nbright = 10
tiles_ann = fits_table()
tiles_ann.index = tiles.index
gfa_regions = []
maxr = 0.
#t.cd[0,0], t.cd[0,1], t.cd[1,0], t.cd[1,1],
for t in T:
wcs = Tan(0., 0., t.crpix[0], t.crpix[1], t.cd[0, 0], t.cd[1, 0],
t.cd[0, 1], t.cd[1, 1], float(t.naxis[0]), float(t.naxis[1]))
ctype = t.extname[:5]
cnum = int(t.extname[5])
h, w = wcs.shape
x, y = [1, 1, w, w, 1], [1, h, h, 1, 1]
r, d = wcs.pixelxy2radec(x, y)
dists = degrees_between(0., 0., r, d)
maxr = max(maxr, max(dists))
# x0, y0, x1, y1
rois = []
if ctype == 'FOCUS':
# add the two half-chips.
# wcs.get_subimage moves the CRPIX, but leave CRVAL unchanged, so tx,ty still work unchanged.
# Aaron's WCS templates correct for the overscans
#wcs_subs.append((cstr, cnum, 'a', wcs.get_subimage(0, 0, 1024, h)))
#wcs_subs.append((cstr, cnum, 'b', wcs.get_subimage(1024, 0, 1024, h)))
#all_sub_wcs[(cstr, cnum, 1)] = (tx, ty, wcs.get_subimage(50, 0, 1024, 1032))
#all_sub_wcs[(cstr, cnum, 2)] = (tx, ty, wcs.get_subimage(1174, 0, 1024, 1032))
# Add (negative) margin for donut size and telescope pointing uncertainty.
# ~10" for donuts and ~10" for telescope pointing
#margin = 100
#wcs_subs.append((cstr, cnum, 'a_margin', wcs.get_subimage(margin, margin, 1024-2*margin, h-2*margin)))
#wcs_subs.append((cstr, cnum, 'b_margin', wcs.get_subimage(1024+margin, margin, 1024-2*margin, h-2*margin)))
# Also add a positive margin for bright-star reflections off filters
#margin = 125
#wcs_subs.append((cstr, cnum, 'expanded', wcs.get_subimage(-margin, -margin, w+2*margin, h+2*margin)))
rois.append(('a', 0, 0, 1024, h))
rois.append(('b', 1024, 0, 2048, h))
margin = 100
rois.append(
('a_margin', margin, margin, 1024 - margin, h - margin))
rois.append(
('b_margin', 1024 + margin, margin, 2048 - margin, h - margin))
margin = 125
rois.append(('expanded', -margin, -margin, w + margin, h + margin))
else:
# Guide chips include overscan pixels -- including a blank region in the middle.
#print(cstr,cnum, 'shape', wcs.shape)
#wcs_subs.append((cstr, cnum, 'ccd', wcs))
rois.append(('ccd', 0, 0, w, h))
# Add expanded GUIDE chips -- 25" margin / 0.2"/pix = 125 pix
margin = 125
#wcs_subs.append((cstr, cnum, 'expanded', wcs.get_subimage(-margin, -margin, w+2*margin, h+2*margin)))
rois.append(('expanded', -margin, -margin, w + margin, h + margin))
margin = 125
expwcs = wcs.get_subimage(-margin, -margin, w + 2 * margin,
h + 2 * margin)
newrois = []
for tag, x0, y0, x1, y1 in rois:
name = '%s_%i_%s' % (ctype.lower(), cnum, tag)
arr = np.zeros(len(tiles), (np.float32, Nbright))
tiles_ann.set('brightest_' + name, arr)
# (the rois have zero-indexed x0,y0, and non-inclusive x1,y1!)
newrois.append((name, arr, 1 + x0, 1 + y0, x1, y1))
gfa_regions.append((ctype, cnum, wcs, expwcs, newrois))
# DEBUG WCS
# s = []
# for ctype,cnum,wcs,expwcs,rois in gfa_regions:
# WCS = wcs
# h,w = WCS.shape
# #print('Expwcs:', w, 'x', h)
# x = [1,1,w,w,1]
# y = [1,h,h,1,1]
# r,d = WCS.pixelxy2radec(x, y)
# p = ','.join(['%.4f,%.4f' % (rr,dd) for rr,dd in zip(r,d)])
# s.append(p)
# s = ';'.join(s)
# print('http://legacysurvey.org/viewer/?ra=0&dec=0&poly='+s)
# sys.exit(0)
gaia = CachingGaiaCatalog(columns=[
'ra', 'dec', 'phot_g_mean_mag', 'phot_bp_mean_mag', 'phot_rp_mean_mag',
'astrometric_excess_noise', 'astrometric_params_solved', 'source_id',
'pmra_error', 'pmdec_error', 'parallax_error', 'ra_error', 'dec_error',
'pmra', 'pmdec', 'parallax', 'ref_epoch'
])
tyc2fn = '/global/cfs/cdirs/cosmo/staging/tycho2/tycho2.kd.fits'
tycho_kd = tree_open(tyc2fn)
tycho_cat = fits_table(tyc2fn)
maxrad = maxr * 1.05
for itile, tile in enumerate(tiles):
#if not tile.in_imaging:
# continue
#if tile.centerid % 10 == 0:
#print('tile program', tile.program, 'pass', tile.get('pass'), 'id', tile.centerid, gaia.get_healpix_tree.cache_info())
I = tree_search_radec(tycho_kd, tile.ra, tile.dec, maxrad)
tycstars = tycho_cat[I]
fix_tycho(tycstars)
for cstr, cname, chipwcs, bigwcs, rois in gfa_regions:
h, w = chipwcs.shape
chipwcs.set_crval(tile.ra, tile.dec)
bigwcs.set_crval(tile.ra, tile.dec)
gstars = gaia.get_catalog_in_wcs(bigwcs, step=1032, margin=0)
fix_gaia(gstars)
bh, bw = bigwcs.shape
ok, x, y = bigwcs.radec2pixelxy(tycstars.ra, tycstars.dec)
tstars = tycstars[(x >= 1) * (y >= 1) * (x <= bw) * (y <= bh)]
#print('Tile', tile.program, 'p', tile.get('pass'), tile.centerid,
# 'GFA', cstr, cname, ':', len(gstars), 'Gaia stars', len(tstars), 'Tycho-2 stars')
if len(gstars) + len(tstars) == 0:
print('No stars in tile centerid', tile.centerid, 'chip', name)
continue
if len(gstars) > 0 and len(tstars) > 0:
merge_gaia_tycho(gstars, tstars)
stars = merge_tables([gstars, tstars], columns='fillzero')
elif len(tstars) > 0:
stars = tstars
else:
stars = gstars
ok, x, y = chipwcs.radec2pixelxy(stars.ra, stars.dec)
for name, arr, x0, y0, x1, y1 in rois:
J = np.flatnonzero(
(x >= x0) * (x <= x1) * (y >= y0) * (y <= y1))
mags = stars.mag[J]
#print(' ', len(mags), 'in name')
K = np.argsort(mags)
K = K[:Nbright]
arr[itile, :len(K)] = mags[K]
#tiles.add_columns_from(tiles_ann)
return tiles_ann
def get_healpix_tree(self, healpix):
from astrometry.util.fits import fits_table
fname = self.fnpattern % dict(hp=healpix)
tab = fits_table(fname, columns=self.columns)
kd = tree_build_radec(tab.ra, tab.dec)
return (kd, tab)
def main(fn, mp):
basefn = os.path.basename(fn)
tiles = fits_table(fn)
#I = np.flatnonzero(tiles.in_imaging)
#tiles.cut(I)
### Split the tiles into nearby chunks of work for multi-processing.
from astrometry.util.util import radecdegtohealpix
nside = 8
Nhp = 12 * nside**2
Ihps = [[] for i in range(Nhp)]
for i, (r, d) in enumerate(zip(tiles.ra, tiles.dec)):
hp = radecdegtohealpix(r, d, nside)
assert (hp >= 0)
Ihps[hp].append(i)
args = []
tiles.index = np.arange(len(tiles))
for hp, I in enumerate(Ihps):
if len(I) == 0:
continue
args.append((tiles[np.array(I)], 'HP %i' % hp))
print(len(args), 'big healpixes are populated')
R = mp.map(run_tiles, args)
tiles_ann = merge_tables(R)
print(len(tiles), 'tiles')
print(len(tiles_ann), 'annotated')
tiles_ann.about()
# unpermute
I = np.zeros(len(tiles_ann), np.int32)
I[tiles_ann.index] = np.arange(len(tiles_ann))
tiles_ann.cut(I)
tiles.add_columns_from(tiles_ann)
tiles.delete_column('index')
outfn = basefn.replace('.fits', '-brightest.fits')
tiles.writeto(outfn)
# Nudge (only inside imaging footprint)
from functools import reduce
brightest = reduce(np.minimum, [
tiles.brightest_guide_0_expanded[:, 0],
tiles.brightest_guide_2_expanded[:, 0],
tiles.brightest_guide_3_expanded[:, 0],
tiles.brightest_guide_5_expanded[:, 0],
tiles.brightest_guide_7_expanded[:, 0],
tiles.brightest_guide_8_expanded[:, 0],
tiles.brightest_focus_1_expanded[:, 0],
tiles.brightest_focus_4_expanded[:, 0],
tiles.brightest_focus_6_expanded[:, 0],
tiles.brightest_focus_9_expanded[:, 0],
])
if 'in_imaging' in tiles.get_columns():
I = np.flatnonzero((brightest < 6.) * tiles.in_imaging)
else:
I = np.flatnonzero((brightest < 6.))
print(len(I), 'tiles with G<6')
tiles.nudge_ra = np.zeros(len(tiles), np.float32)
tiles.nudge_dec = np.zeros(len(tiles), np.float32)
tiles.nudge_brightest = np.zeros(len(tiles), np.float32)
tiles.index = np.arange(len(tiles))
for nudge in range(1, 20):
if len(I) == 0:
break
print('Nudging', len(I), 'by', nudge)
ddec = 10. / 3600.
dra = ddec / np.cos(np.deg2rad(tiles.dec[I]))
# copy tiles
nudgetiles = tiles[np.repeat(I, 4)]
nudgetiles.nudge_ra[0::4] = +nudge * dra
nudgetiles.nudge_ra[1::4] = -nudge * dra
nudgetiles.nudge_dec[2::4] = +nudge * ddec
nudgetiles.nudge_dec[3::4] = -nudge * ddec
nudgetiles.ra += nudgetiles.nudge_ra
nudgetiles.dec += nudgetiles.nudge_dec
#ann = run_tiles((nudgetiles, 'nudge%i' % nudge))
# split into subsets
args = []
isplit = np.linspace(0, len(nudgetiles), 33, dtype=int)
for i0, i1 in zip(isplit, isplit[1:]):
if i0 == i1:
continue
args.append((nudgetiles[i0:i1], 'nudge%i+%i' % (nudge, i0)))
A = mp.map(run_tiles, args)
ann = merge_tables(A)
ann.brightest = reduce(np.minimum, [
ann.brightest_guide_0_expanded[:, 0],
ann.brightest_guide_2_expanded[:, 0],
ann.brightest_guide_3_expanded[:, 0],
ann.brightest_guide_5_expanded[:, 0],
ann.brightest_guide_7_expanded[:, 0],
ann.brightest_guide_8_expanded[:, 0],
ann.brightest_focus_1_expanded[:, 0],
ann.brightest_focus_4_expanded[:, 0],
ann.brightest_focus_6_expanded[:, 0],
ann.brightest_focus_9_expanded[:, 0],
])
ok = (ann.brightest > 6)
found = np.zeros(len(I), bool)
for idir, dirok in enumerate([ok[0::4], ok[1::4], ok[2::4], ok[3::4]]):
J = np.flatnonzero(np.logical_not(found) * dirok)
print('Nudge dir', idir, ':', len(J), 'are okay')
found[J] = True
tiles.nudge_ra[I[J]] = nudgetiles.nudge_ra[J * 4 + idir]
tiles.nudge_dec[I[J]] = nudgetiles.nudge_dec[J * 4 + idir]
tiles.nudge_brightest[I[J]] = ann.brightest[J * 4 + idir]
I = I[np.flatnonzero(found == False)]
outfn = basefn.replace('.fits', '-brightest-nudged.fits')
tiles.writeto(outfn)
def main(outfn='ccds-annotated.fits', ccds=None):
decals = Decals()
if ccds is None:
ccds = decals.get_ccds()
# File from the "observing" svn repo:
# https://desi.lbl.gov/svn/decam/code/observing/trunk
tiles = fits_table('decam-tiles_obstatus.fits')
#ccds.cut(np.arange(100))
#print("HACK!")
#ccds.cut(np.array([name in ['N15', 'N16', 'N21', 'N9']
# for name in ccds.ccdname]) *
# ccds.expnum == 229683)
I = decals.photometric_ccds(ccds)
ccds.photometric = np.zeros(len(ccds), bool)
ccds.photometric[I] = True
I = decals.apply_blacklist(ccds)
ccds.blacklist_ok = np.zeros(len(ccds), bool)
ccds.blacklist_ok[I] = True
ccds.good_region = np.empty((len(ccds), 4), np.int16)
ccds.good_region[:, :] = -1
ccds.ra0 = np.zeros(len(ccds), np.float64)
ccds.dec0 = np.zeros(len(ccds), np.float64)
ccds.ra1 = np.zeros(len(ccds), np.float64)
ccds.dec1 = np.zeros(len(ccds), np.float64)
ccds.ra2 = np.zeros(len(ccds), np.float64)
ccds.dec2 = np.zeros(len(ccds), np.float64)
ccds.ra3 = np.zeros(len(ccds), np.float64)
ccds.dec3 = np.zeros(len(ccds), np.float64)
ccds.dra = np.zeros(len(ccds), np.float32)
ccds.ddec = np.zeros(len(ccds), np.float32)
ccds.ra_center = np.zeros(len(ccds), np.float64)
ccds.dec_center = np.zeros(len(ccds), np.float64)
ccds.sig1 = np.zeros(len(ccds), np.float32)
ccds.meansky = np.zeros(len(ccds), np.float32)
ccds.stdsky = np.zeros(len(ccds), np.float32)
ccds.maxsky = np.zeros(len(ccds), np.float32)
ccds.minsky = np.zeros(len(ccds), np.float32)
ccds.pixscale_mean = np.zeros(len(ccds), np.float32)
ccds.pixscale_std = np.zeros(len(ccds), np.float32)
ccds.pixscale_max = np.zeros(len(ccds), np.float32)
ccds.pixscale_min = np.zeros(len(ccds), np.float32)
ccds.psfnorm_mean = np.zeros(len(ccds), np.float32)
ccds.psfnorm_std = np.zeros(len(ccds), np.float32)
ccds.galnorm_mean = np.zeros(len(ccds), np.float32)
ccds.galnorm_std = np.zeros(len(ccds), np.float32)
gaussgalnorm = np.zeros(len(ccds), np.float32)
# 2nd moments
ccds.psf_mx2 = np.zeros(len(ccds), np.float32)
ccds.psf_my2 = np.zeros(len(ccds), np.float32)
ccds.psf_mxy = np.zeros(len(ccds), np.float32)
#
ccds.psf_a = np.zeros(len(ccds), np.float32)
ccds.psf_b = np.zeros(len(ccds), np.float32)
ccds.psf_theta = np.zeros(len(ccds), np.float32)
ccds.psf_ell = np.zeros(len(ccds), np.float32)
ccds.humidity = np.zeros(len(ccds), np.float32)
ccds.outtemp = np.zeros(len(ccds), np.float32)
ccds.tileid = np.zeros(len(ccds), np.int32)
ccds.tilepass = np.zeros(len(ccds), np.uint8)
ccds.tileebv = np.zeros(len(ccds), np.float32)
plvers = []
for iccd, ccd in enumerate(ccds):
im = decals.get_image_object(ccd)
print('Reading CCD %i of %i:' % (iccd + 1, len(ccds)), im)
X = im.get_good_image_subregion()
for i, x in enumerate(X):
if x is not None:
ccds.good_region[iccd, i] = x
W, H = ccd.width, ccd.height
psf = None
wcs = None
sky = None
try:
tim = im.get_tractor_image(pixPsf=True,
splinesky=True,
subsky=False,
pixels=False)
except:
import traceback
traceback.print_exc()
plvers.append('')
continue
if tim is None:
plvers.append('')
continue
psf = tim.psf
wcs = tim.wcs.wcs
sky = tim.sky
hdr = tim.primhdr
# print('Got PSF', psf)
# print('Got sky', type(sky))
# print('Got WCS', wcs)
ccds.humidity[iccd] = hdr.get('HUMIDITY')
ccds.outtemp[iccd] = hdr.get('OUTTEMP')
ccds.sig1[iccd] = tim.sig1
plvers.append(tim.plver)
obj = hdr.get('OBJECT')
# parse 'DECaLS_15150_r'
words = obj.split('_')
tile = None
if len(words) == 3 and words[0] == 'DECaLS':
try:
tileid = int(words[1])
tile = tiles[tileid - 1]
if tile.tileid != tileid:
I = np.flatnonzero(tile.tileid == tileid)
tile = tiles[I[0]]
except:
pass
if tile is not None:
ccds.tileid[iccd] = tile.tileid
ccds.tilepass[iccd] = tile.get('pass')
ccds.tileebv[iccd] = tile.ebv_med
# Instantiate PSF on a grid
S = 32
xx = np.linspace(1 + S, W - S, 5)
yy = np.linspace(1 + S, H - S, 5)
xx, yy = np.meshgrid(xx, yy)
psfnorms = []
galnorms = []
for x, y in zip(xx.ravel(), yy.ravel()):
p = im.psf_norm(tim, x=x, y=y)
g = im.galaxy_norm(tim, x=x, y=y)
psfnorms.append(p)
galnorms.append(g)
ccds.psfnorm_mean[iccd] = np.mean(psfnorms)
ccds.psfnorm_std[iccd] = np.std(psfnorms)
ccds.galnorm_mean[iccd] = np.mean(galnorms)
ccds.galnorm_std[iccd] = np.std(galnorms)
# PSF in center of field
cx, cy = (W + 1) / 2., (H + 1) / 2.
p = psf.getPointSourcePatch(cx, cy).patch
ph, pw = p.shape
px, py = np.meshgrid(np.arange(pw), np.arange(ph))
psum = np.sum(p)
# print('psum', psum)
p /= psum
# centroids
cenx = np.sum(p * px)
ceny = np.sum(p * py)
# print('cenx,ceny', cenx,ceny)
# second moments
x2 = np.sum(p * (px - cenx)**2)
y2 = np.sum(p * (py - ceny)**2)
xy = np.sum(p * (px - cenx) * (py - ceny))
# semi-major/minor axes and position angle
theta = np.rad2deg(np.arctan2(2 * xy, x2 - y2) / 2.)
theta = np.abs(theta) * np.sign(xy)
s = np.sqrt(((x2 - y2) / 2.)**2 + xy**2)
a = np.sqrt((x2 + y2) / 2. + s)
b = np.sqrt((x2 + y2) / 2. - s)
ell = 1. - b / a
# print('PSF second moments', x2, y2, xy)
# print('PSF position angle', theta)
# print('PSF semi-axes', a, b)
# print('PSF ellipticity', ell)
ccds.psf_mx2[iccd] = x2
ccds.psf_my2[iccd] = y2
ccds.psf_mxy[iccd] = xy
ccds.psf_a[iccd] = a
ccds.psf_b[iccd] = b
ccds.psf_theta[iccd] = theta
ccds.psf_ell[iccd] = ell
# Galaxy norm using Gaussian approximation of PSF.
realpsf = tim.psf
tim.psf = im.read_psf_model(0,
0,
gaussPsf=True,
psf_sigma=tim.psf_sigma)
gaussgalnorm[iccd] = im.galaxy_norm(tim, x=cx, y=cy)
tim.psf = realpsf
# Sky
mod = np.zeros((ccd.height, ccd.width), np.float32)
sky.addTo(mod)
ccds.meansky[iccd] = np.mean(mod)
ccds.stdsky[iccd] = np.std(mod)
ccds.maxsky[iccd] = mod.max()
ccds.minsky[iccd] = mod.min()
# WCS
ccds.ra0[iccd], ccds.dec0[iccd] = wcs.pixelxy2radec(1, 1)
ccds.ra1[iccd], ccds.dec1[iccd] = wcs.pixelxy2radec(1, H)
ccds.ra2[iccd], ccds.dec2[iccd] = wcs.pixelxy2radec(W, H)
ccds.ra3[iccd], ccds.dec3[iccd] = wcs.pixelxy2radec(W, 1)
midx, midy = (W + 1) / 2., (H + 1) / 2.
rc, dc = wcs.pixelxy2radec(midx, midy)
ra, dec = wcs.pixelxy2radec([1, W, midx, midx], [midy, midy, 1, H])
ccds.dra[iccd] = max(
degrees_between(ra, dc + np.zeros_like(ra), rc, dc))
ccds.ddec[iccd] = max(
degrees_between(rc + np.zeros_like(dec), dec, rc, dc))
ccds.ra_center[iccd] = rc
ccds.dec_center[iccd] = dc
# Compute scale change across the chip
# how many pixels to step
step = 10
xx = np.linspace(1 + step, W - step, 5)
yy = np.linspace(1 + step, H - step, 5)
xx, yy = np.meshgrid(xx, yy)
pixscale = []
for x, y in zip(xx.ravel(), yy.ravel()):
sx = [x - step, x - step, x + step, x + step, x - step]
sy = [y - step, y + step, y + step, y - step, y - step]
sr, sd = wcs.pixelxy2radec(sx, sy)
rc, dc = wcs.pixelxy2radec(x, y)
# project around a tiny little TAN WCS at (x,y), with 1" pixels
locwcs = Tan(rc, dc, 0., 0., 1. / 3600, 0., 0., 1. / 3600, 1., 1.)
ok, lx, ly = locwcs.radec2pixelxy(sr, sd)
#print('local x,y:', lx, ly)
A = polygon_area((lx, ly))
pixscale.append(np.sqrt(A / (2 * step)**2))
# print('Pixel scales:', pixscale)
ccds.pixscale_mean[iccd] = np.mean(pixscale)
ccds.pixscale_min[iccd] = min(pixscale)
ccds.pixscale_max[iccd] = max(pixscale)
ccds.pixscale_std[iccd] = np.std(pixscale)
ccds.plver = np.array(plvers)
sfd = tractor.sfd.SFDMap()
allbands = 'ugrizY'
filts = ['%s %s' % ('DES', f) for f in allbands]
wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4']
ebv, ext = sfd.extinction(filts + wisebands,
ccds.ra_center,
ccds.dec_center,
get_ebv=True)
ext = ext.astype(np.float32)
ccds.ebv = ebv.astype(np.float32)
ccds.decam_extinction = ext[:, :len(allbands)]
ccds.wise_extinction = ext[:, len(allbands):]
# Depth
detsig1 = ccds.sig1 / ccds.psfnorm_mean
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.psfdepth = -2.5 * (np.log10(depth) - 9)
detsig1 = ccds.sig1 / ccds.galnorm_mean
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.galdepth = -2.5 * (np.log10(depth) - 9)
# Depth using Gaussian FWHM.
psf_sigma = ccds.fwhm / 2.35
gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
detsig1 = ccds.sig1 / gnorm
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.gausspsfdepth = -2.5 * (np.log10(depth) - 9)
# Gaussian galaxy depth
detsig1 = ccds.sig1 / gaussgalnorm
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.gaussgaldepth = -2.5 * (np.log10(depth) - 9)
ccds.writeto(outfn)
def prepare_fits_catalog(cat,
invvars,
T,
bands,
allbands=None,
prefix='',
save_invvars=True,
force_keep=None):
if T is None:
from astrometry.util.fits import fits_table
T = fits_table()
if allbands is None:
allbands = bands
params0 = cat.getParams()
flux = np.zeros((len(T), len(allbands)), np.float32)
flux_ivar = np.zeros((len(T), len(allbands)), np.float32)
for band in bands:
i = allbands.index(band)
for j, src in enumerate(cat):
if src is not None:
flux[j,
i] = sum(b.getFlux(band) for b in src.getBrightnesses())
if invvars is None:
continue
# Oh my, this is tricky... set parameter values to the variance
# vector so that we can read off the parameter variances via the
# python object apis.
cat.setParams(invvars)
for j, src in enumerate(cat):
if src is not None:
flux_ivar[j, i] = sum(
b.getFlux(band) for b in src.getBrightnesses())
cat.setParams(params0)
T.set('%sflux' % prefix, flux)
if save_invvars:
T.set('%sflux_ivar' % prefix, flux_ivar)
_get_tractor_fits_values(T, cat, '%s%%s' % prefix)
if save_invvars:
if invvars is not None:
cat.setParams(invvars)
else:
cat.setParams(np.zeros(cat.numberOfParams()))
_get_tractor_fits_values(T, cat, '%s%%s_ivar' % prefix)
# Heh, "no uncertainty here!"
T.delete_column('%stype_ivar' % prefix)
cat.setParams(params0)
# mod RA
ra = T.get('%sra' % prefix)
ra += (ra < 0) * 360.
ra -= (ra > 360) * 360.
# Downconvert RA,Dec invvars to float32
if save_invvars:
for c in ['ra', 'dec']:
col = '%s%s_ivar' % (prefix, c)
T.set(col, T.get(col).astype(np.float32))
# Zero out unconstrained values
flux = T.get('%s%s' % (prefix, 'flux'))
iv = T.get('%s%s' % (prefix, 'flux_ivar'))
if force_keep is not None:
flux[(iv == 0) * np.logical_not(force_keep[:, np.newaxis])] = 0.
else:
flux[iv == 0] = 0.
return T
def main():
import optparse
import sys
parser = optparse.OptionParser(usage='%prog <json>')
parser.add_option('--base', default='plan', help='Plot base filename')
parser.add_option('-t', '--obstatus', help='Show already-observed tiles?')
parser.add_option(
'--bands',
help='Plot only already-observed tiles in the given bands',
default='g,r,z')
parser.add_option('--sgc', action='store_true', help='Center on SGC?')
parser.add_option('--ralo', type=float, default=None)
parser.add_option('--rahi', type=float, default=None)
parser.add_option('--declo', type=float, default=None)
parser.add_option('--dechi', type=float, default=None)
parser.add_option(
'--scaled',
action='store_true',
default=False,
help='Scale plot so that 1 deg RA = 1 deg Dec (no COS term)')
parser.add_option('--wide',
action='store_true',
default=False,
help='Make wider plots?')
parser.add_option('--also',
action='append',
default=[],
help='Also plot the plan from the given filename.')
parser.add_option('--mosaic',
action='store_true',
help='Set defaults for Mosaic survey')
parser.add_option(
'--start-time',
help=
'Start time for this plan, HH:MM:SS UTC. Default: 12-degree twilight tonight.'
)
parser.add_option('--start-date',
help='Start date for this plan, YYYY-MM-DD UTC.')
parser.add_option(
'--stop-time',
help='Stop time for this plan, HH:MM:SS UTC. Default: no limit.')
parser.add_option(
'--second-half',
action='store_true',
help='This plan starts at the start of the second half-night.')
parser.add_option('--skip',
type=int,
default=1,
help='Write every Nth plot only')
parser.add_option('--threads', type=int, help='Multi-processing?')
opt, args = parser.parse_args()
if len(args) != 1:
parser.print_help()
sys.exit(-1)
if opt.mosaic:
dd = dict(ralo=0, rahi=360, declo=30, dechi=88)
else:
dd = dict(ralo=0, rahi=360, declo=-10, dechi=35)
for k in dd.keys():
if getattr(opt, k, None) is None:
setattr(opt, k, dd[k])
start_date_specified = (opt.start_date is not None)
if opt.start_date is None:
# Get date at start of night, where we define a new day as
# starting at noon UTC.
now = datetime.datetime.utcnow()
# noon
nightstart = now - datetime.timedelta(0, 12 * 3600)
d = nightstart.date()
opt.start_date = '%04i-%02i-%02i' % (d.year, d.month, d.day)
print('Set start date to', opt.start_date)
if opt.mosaic:
from camera_mosaic import ephem_observer
else:
from camera_decam import ephem_observer
obs = ephem_observer()
obs.temp = 10.0 # deg celsius; average temp for August
obs.pressure = 780.0 # mbar
### HACK
obs.date = ephem.Date(opt.start_date + ' 8:00:00')
#print('Obs date:', obs.date)
daystart = obs.date
obs.horizon = -ephem.degrees('12:00:00.0')
sun = ephem.Sun()
eve_twi = obs.next_setting(sun)
obs.date = eve_twi
morn_twi = obs.next_rising(sun)
print('Evening twilight:', eve_twi)
print('Morning twilight:', morn_twi)
assert (morn_twi > eve_twi)
obs.horizon = 0.
print('Eve twi:', eve_twi, 'Morning:', morn_twi)
if opt.second_half:
# Set start-time to the midpoint between 12-degree twilights.
obs.date = ephem.Date((eve_twi + morn_twi) / 2.)
print('Second half starts at', obs.date)
elif opt.start_time is None:
# 12-degree twilight on start_date
obs.date = eve_twi
else:
obs.date = ephem.Date(opt.start_date + ' ' + opt.start_time)
if not start_date_specified and obs.date < daystart:
# If --start-date is, eg, 2am, assume it's during the night starting on daystart.
obs.date = ephem.Date(float(obs.date) + 1.)
print('Start date:', obs.date)
if opt.stop_time is not None:
# The date should be unambiguous -- try the same as obs.date =
# start time, add one day if necessary.
date = obs.date.datetime()
stopdate = ephem.Date('%04i-%02i-%02i' %
(date.year, date.month, date.day) + ' ' +
opt.stop_time)
if stopdate < obs.date:
stopdate = ephem.Date(float(stopdate) + 1.)
print('Stop date:', stopdate)
jfn = args[0]
print('Reading JSON file', jfn)
J = json.loads(open(jfn, 'rb').read())
print(len(J), 'entries')
Jalso = [json.loads(open(fn, 'rb').read()) for fn in opt.also]
# Get times when exposures should occur.
times = []
LSTs = []
# If the JSON files include estimated times, use those
if 'approx_datetime' in J[0]:
for i, j in enumerate(J):
obs.date = ephem.Date(str(j['approx_datetime']))
if opt.stop_time is not None and obs.date > stopdate:
print('Tile', i, 'is after --stopdate')
J = J[:i]
assert (len(J) == len(times))
break
times.append(ephem.Date(obs.date))
LSTs.append(np.rad2deg(float(obs.sidereal_time())))
print('Date', obs.date)
print('LST', obs.sidereal_time())
else:
# Predict overheads
lastra, lastdec = None, None
for i in range(len(J)):
print('Exposure', i, 'should start at', str(obs.date))
if opt.stop_time is not None and obs.date > stopdate:
print('Tile', J[i], 'is after --stopdate')
break
times.append(ephem.Date(obs.date))
LSTs.append(np.rad2deg(float(obs.sidereal_time())))
overhead = 30.
if lastra is not None:
slew = degrees_between(lastra, lastdec, ras[i], decs[i])
lastra = ras[i]
lastdec = decs[i]
# Add 3 seconds per degree for slews longer than 2 degrees
overhead += np.maximum(0, slew - 2.) * 3.
# Add overhead
print('Adding', exptime[i], 'seconds exptime plus', overhead,
'seconds overhead')
obs.date += (exptime[i] + overhead) / (24 * 3600.)
tiles = None
if opt.obstatus is not None:
from astrometry.util.fits import fits_table
tiles = fits_table(opt.obstatus)
print('Read', len(tiles), 'tiles')
tiles = tiles[(tiles.in_des == 0) * np.logical_or(
(tiles.in_sdss == 1), (tiles.in_sdss == 0) * (tiles.in_desi == 1))]
print(len(tiles), 'in footprint')
fcmap = dict(g='g', r='r', z='m', zd='m')
ddecmap = dict(g=-0.2, r=0, z=0.2, zd=0.2)
ras = np.array([j['RA'] for j in J])
decs = np.array([j['dec'] for j in J])
filts = np.array([j['filter'] for j in J])
exptime = np.array([j['expTime'] for j in J])
fieldname = [j['object'] for j in J]
passnum = np.zeros(len(J), int)
filtcc = np.array([fcmap[f] for f in filts])
ddecs = np.array([ddecmap[f] for f in filts])
# passmap = { 1: dict(marker='.'),
# 2: dict(marker='o', mfc='none'),
# 3: dict(marker='x') }
passmap = {
1: dict(marker='.'),
2: dict(marker='.'),
3: dict(marker='.'),
}
opt.bands = opt.bands.split(',')
if len(opt.bands) == 1:
filtddec = {'g': 0, 'r': 0, 'z': 0}
else:
ddec = 0.4
filtddec = {'g': -ddec, 'r': 0, 'z': ddec}
seqmap = ['r', 'y', 'g', 'b', 'm']
#seqcc = np.array([seqmap[s % len(seqmap)] for s in seqnum])
#seqcc = np.array([seqmap[s % len(seqmap)] for s in seqid])
ax = [
transform_ra(opt.rahi, opt),
transform_ra(opt.ralo, opt), opt.declo, opt.dechi
]
alsocolors = 'kbr'
also = []
for Ja in Jalso:
# We assume the --also plan files contain approx_datetime...
atimes = np.array([ephem.Date(str(j['approx_datetime'])) for j in Ja])
aras = np.array([j['RA'] for j in Ja])
adecs = np.array([j['dec'] for j in Ja])
afilts = np.array([j['filter'] for j in Ja])
aexptime = np.array([j['expTime'] for j in Ja])
afieldname = [j['object'] for j in Ja]
apassnum = np.zeros(len(Ja), int)
if tiles is not None:
for i, f in enumerate(afieldname):
tile = get_tile_from_name(f, tiles)
if tile is None:
continue
pa = tile.get('pass')
apassnum[i] = pa
also.append(
(atimes, aras, adecs, afilts, aexptime, afieldname, apassnum))
# Try to get the pass number via parsing the field name to get tile id
# and looking up the pass number in the tiles table.
if tiles is not None:
for i, f in enumerate(fieldname):
tile = get_tile_from_name(f, tiles)
if tile is None:
continue
pa = tile.get('pass')
passnum[i] = pa
print('Field', f, 'tileid', tile.tileid, 'pass', pa)
allargs = []
for i in reversed(range(0, len(J), opt.skip)):
#print('Exposure', i, 'of', len(J))
fn = '%s-%03i.png' % (opt.base, i)
fn = os.path.join(os.path.dirname(args[0]), fn)
pargs = (opt, ax, tiles, filtddec, fcmap, passmap, also, LSTs, times,
ras, decs, ddecs, fieldname, passnum, exptime, i, filtcc,
alsocolors, ddecmap, fn)
allargs.append(pargs)
if opt.threads:
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads, init=plot_init)
mp.map(plot_one, allargs)
else:
plot_init()
map(plot_one, allargs)
#plot_one(pargs)
print()
cmd = 'avconv -r 4 -i %s-%%03d.png -y %s.mov' % (opt.base, opt.base)
print(cmd)
os.system(cmd)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
'-b',
'--brick',
help='Brick name to run; required unless --radec is given')
parser.add_argument(
'--survey-dir',
type=str,
default=None,
help='Override the $LEGACY_SURVEY_DIR environment variable')
parser.add_argument('-d',
'--outdir',
dest='output_dir',
help='Set output base directory, default "."')
parser.add_argument(
'--out',
help='Output filename -- if not set, defaults to path within --outdir.'
)
parser.add_argument('-r',
'--run',
default=None,
help='Set the run type to execute (for images)')
parser.add_argument(
'--catalog',
help=
'Use the given FITS catalog file, rather than reading from a data release directory'
)
parser.add_argument('--catalog-dir',
help='Set LEGACY_SURVEY_DIR to use to read catalogs')
parser.add_argument(
'--catalog-dir-north',
help='Set LEGACY_SURVEY_DIR to use to read Northern catalogs')
parser.add_argument(
'--catalog-dir-south',
help='Set LEGACY_SURVEY_DIR to use to read Southern catalogs')
parser.add_argument(
'--catalog-resolve-dec-ngc',
type=float,
help=
'Dec at which to switch from Northern to Southern catalogs (NGC only)',
default=32.375)
parser.add_argument('-v',
'--verbose',
dest='verbose',
action='count',
default=0,
help='Make more verbose')
opt = parser.parse_args()
if opt.brick is None:
parser.print_help()
return -1
verbose = opt.verbose
if verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
from legacypipe.runs import get_survey
survey = get_survey(opt.run,
survey_dir=opt.survey_dir,
output_dir=opt.output_dir)
columns = [
'release',
'brickid',
'objid',
]
cat = None
catsurvey = survey
if opt.catalog is not None:
cat = fits_table(opt.catalog, columns=columns)
print('Read', len(cat), 'sources from', opt.catalog)
else:
from astrometry.util.starutil_numpy import radectolb
# The "north" and "south" directories often don't have
# 'survey-bricks" files of their own -- use the 'survey' one
# instead.
brick = None
for s in [survey, catsurvey]:
try:
brick = s.get_brick_by_name(opt.brick)
break
except:
import traceback
traceback.print_exc()
pass
l, b = radectolb(brick.ra, brick.dec)
# NGC and above resolve line? -> north
if b > 0 and brick.dec >= opt.catalog_resolve_dec_ngc:
if opt.catalog_dir_north:
catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir_north)
else:
if opt.catalog_dir_south:
catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir_south)
fn = catsurvey.find_file('tractor', brick=opt.brick)
cat = fits_table(fn, columns=columns)
print('Read', len(cat), 'sources from', fn)
program_name = sys.argv[0]
## FIXME -- from catalog?
release = 9999
version_hdr = get_version_header(program_name, opt.survey_dir, release)
from legacypipe.utils import add_bits
from legacypipe.bits import DQ_BITS
add_bits(version_hdr, DQ_BITS, 'DQMASK', 'DQ', 'D')
from legacyzpts.psfzpt_cuts import CCD_CUT_BITS
add_bits(version_hdr, CCD_CUT_BITS, 'CCD_CUTS', 'CC', 'C')
for i, ap in enumerate(apertures_arcsec):
version_hdr.add_record(
dict(name='APRAD%i' % i,
value=ap,
comment='(optical) Aperture radius, in arcsec'))
cat, forced = merge_forced(survey, opt.brick, cat)
units = []
for i, col in enumerate(forced.get_columns()):
units.append(forced._header.get('TUNIT%i' % (i + 1), ''))
cols = forced.get_columns()
if opt.out:
cat.writeto(opt.out, primheader=version_hdr)
forced.writeto(opt.out, append=True, units=units, columns=cols)
else:
with survey.write_output('forced-brick', brick=opt.brick) as out:
cat.writeto(None, fits_object=out.fits, primheader=version_hdr)
forced.writeto(None,
fits_object=out.fits,
append=True,
units=units,
columns=cols)
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--force', action='store_true',
help='Run calib processes even if files already exist?')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--expnum', type=int, help='Cut to a single exposure')
parser.add_argument('--extname', '--ccdname', help='Cut to a single extension/CCD name')
parser.add_argument('--no-psf', dest='psfex', action='store_false',
help='Do not compute PsfEx calibs')
parser.add_argument('--no-sky', dest='sky', action='store_false',
help='Do not compute sky models')
parser.add_argument('--run-se', action='store_true', help='Run SourceExtractor')
parser.add_argument('--splinesky', action='store_true', help='Spline sky, not constant')
parser.add_argument('--threads', type=int, help='Run multi-threaded', default=None)
parser.add_argument('args',nargs=argparse.REMAINDER)
opt = parser.parse_args()
survey = LegacySurveyData()
if opt.ccds is not None:
T = fits_table(opt.ccds)
print('Read', len(T), 'from', opt.ccds)
else:
T = survey.get_ccds()
#print len(T), 'CCDs'
if len(opt.args) == 0:
if opt.expnum is not None:
T.cut(T.expnum == opt.expnum)
print('Cut to', len(T), 'with expnum =', opt.expnum)
if opt.extname is not None:
T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname]))
print('Cut to', len(T), 'with extname =', opt.extname)
opt.args = range(len(T))
args = []
for a in opt.args:
# Check for "expnum-ccdname" format.
if '-' in str(a):
words = a.split('-')
assert(len(words) == 2)
expnum = int(words[0])
ccdname = words[1]
I = np.flatnonzero((T.expnum == expnum) * (T.ccdname == ccdname))
if len(I) != 1:
print('Found', len(I), 'CCDs for expnum', expnum, 'CCDname', ccdname, ':', I)
assert(len(I) == 1)
t = T[I[0]]
else:
i = int(a)
print('Index', i)
t = T[i]
print('CCDnmatch', t.ccdnmatch)
if t.ccdnmatch < 20 and not opt.force:
print('Skipping ccdnmatch = %i' % t.ccdnmatch)
continue
im = survey.get_image_object(t)
print('Running', im.calname)
kwargs = dict(psfex=opt.psfex, sky=opt.sky)
if opt.force:
kwargs.update(force=True)
if opt.run_se:
kwargs.update(se=True)
if opt.splinesky:
kwargs.update(splinesky=True)
if opt.threads:
args.append((im, kwargs))
else:
run_calibs((im, kwargs))
if opt.threads:
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads)
mp.map(run_calibs, args)
return 0
def get_footprint_object(self):
"""Returns footprint object 'sfd'"""
# work with SFD map and Decals/Mzls tiles
# lonlat from SFD healpix is in galactic coords, convert this to Celestial
hdu = fitsio.FITS(os.path.join(self.map_dir, 'lambda_sfd_ebv.fits'))
sfd = EmptyClass()
temp = hdu[1].read()
sfd.temp = temp['TEMPERATURE']
npix = Healpix().get_nside(len(sfd.temp))
assert (npix == 512)
sfd.l_indeg, sfd.b_indeg = hp.pix2ang(512,
np.where(sfd.temp > 0)[0],
nest=True,
lonlat=True)
#inPlane= np.where((sfd_gal_dec > -20) & (sfd_gal_dec < 20))[0]
trans = Galactic(l=sfd.l_indeg * units.degree,
b=sfd.b_indeg * units.degree)
radec = trans.transform_to(ICRS)
sfd.ra, sfd.dec = radec.ra.value, radec.dec.value
all_tiles = fits_table(
os.path.join(self.tile_dir, 'mosaic-tiles_obstatus.fits'))
wdes_tiles = fits_table(
os.path.join(self.tile_dir, 'decam-tiles_obstatus.fits'))
inDESI = ((all_tiles.in_desi_orig == 1) | (all_tiles.in_desi == 1))
inDecals = ((inDESI) & (all_tiles.dec <= 30.))
#(mzls_decals.in_des == 0))
inMzls = ((inDESI) & (all_tiles.dec > 30.))
#(mzls_decals.in_des == 0))
inDes = ((wdes_tiles.in_desi_orig == 1) | (wdes_tiles.in_desi == 1))
inDes = ((inDes) & (wdes_tiles.in_des == 1))
#above30= mzls.dec > 30.
#inDESI= ( (mzls.in_desi_orig == 1) |
# (mzls.in_desi == 1))
#inMzls= ( (above30) &
# (inDESI))
#desi= merge_tables([mzls,decals],columns='fillzero')
des = wdes_tiles.copy()
del wdes_tiles
des.cut(inDes)
mzls = all_tiles.copy()
decals = all_tiles.copy()
del all_tiles
mzls.cut(inMzls)
decals.cut(inDecals)
ps = Healpix().get_pixscale(len(sfd.temp), unit='deg')
# match_radec(ref,obs): for each point in ref, return matching point in obs
print('matching tiles to healpix centers')
I, J, d = match_radec(mzls.ra, mzls.dec, sfd.ra, sfd.dec, ps * 8)
sfd.ipix_mzls = list(set(J))
I, J, d = match_radec(decals.ra, decals.dec, sfd.ra, sfd.dec, ps * 8)
sfd.ipix_decals = list(set(J))
I, J, d = match_radec(des.ra, des.dec, sfd.ra, sfd.dec, ps * 8)
sfd.ipix_des = list(set(J))
return sfd
# legasurvey pts fill in in ps*3
I, J, d = match_radec(legsurvey.ra, legsurvey.dec, sfd.ra, sfd.dec,
ps * 3)
sfd.ipix_legsurvey = set(J)
# des fills in with ps*8
I, J, d = match_radec(des.ra, des.dec, sfd.ra, sfd.dec, ps * 8)
sfd.ipix_legsurvey.union(set(J))
sfd.ipix_legsurvey = list(sfd.ipix_legsurvey)
return sfd
action='store',
default=True)
parser.add_argument('--dowhat',
choices=[
'bricks_notdone', 'time_per_brick', 'nersc_time',
'sanity_tractors'
],
action='store',
default=True)
parser.add_argument('--fn', action='store', default=False)
args = parser.parse_args()
if args.dowhat == 'bricks_notdone':
if args.therun == 'obiwan':
b = fits_table(
os.path.join(os.environ['LEGACY_SURVEY_DIR'],
'survey-bricks-eboss-ngc.fits.gz'))
elif args.therun == 'dr4':
b = fits_table(
os.path.join(os.environ['LEGACY_SURVEY_DIR'],
'survey-bricks-dr4.fits.gz'))
don = np.loadtxt(args.fn, dtype=str)
fout = args.fn.replace('_done.tmp', '_notdone.tmp')
if os.path.exists(fout):
os.remove(fout)
# Bricks not finished
with open(fout, 'w') as fil:
for brick in list(set(b.brickname).difference(set(don))):
fil.write('%s\n' % brick)
print('Wrote %s' % fout)
# All Bricks
def from_fits(cls, filename, header, row=0):
from astrometry.util.fits import fits_table
T = fits_table(filename)
T = T[row]
return cls.from_fits_row(T)
cmd += "%d)" % ids[-1]
else:
# Slower
vals = ""
for i in ids[:-1]:
vals += "(%d)," % i
vals += "(%d)" % ids[-1]
cmd = (cmd + " FROM %s as db RIGHT JOIN (values %s) " %
(db_randoms_table, vals) + "as v(id) on (db.id=v.id)")
#print('cmd= %s' % cmd)
db.cur.execute(cmd)
# List of tuples [(id,reshift,...),(id,reshift,...)]
a = db.cur.fetchall()
# Tuple of lists (ids,reshifts,...)
tup = zip(*a)
#tup[ith_col])
return {col: np.array(vals) for col, vals in zip(columns, tup)}
#return np.array(sql_ids),np.array(sql_redshift)
if __name__ == '__main__':
T = getSrcsInBrick('1765p247', 'elg', db_table='obiwan_elg_ra175')
simcat = fits_table(
"/global/cscratch1/sd/kaylanb/obiwan_out/elg_9deg2_ra175/elg/176/1765p247/rs0/obiwan/simcat-elg-1765p247.fits"
)
data_dict = all_psqlcols_for_ids(simcat.id,
db_randoms_table='obiwan_elg_ra175')
for i in range(10):
print(data_dict['id'][i], data_dict['redshift'][i])
def main():
print("Running with args %s" % sys.argv)
import optparse
parser = optparse.OptionParser()
parser.add_option(
"--server",
dest="server",
default=Client.default_url,
help="Set server base URL (eg, %default)",
)
parser.add_option(
"--apikey",
"-k",
dest="apikey",
help="API key for Astrometry.net web service; if not given will check AN_API_KEY environment variable",
)
parser.add_option("--upload", "-u", dest="upload", help="Upload a file")
parser.add_option(
"--upload-xy", dest="upload_xy", help="Upload a FITS x,y table as JSON"
)
parser.add_option(
"--wait",
"-w",
dest="wait",
action="store_true",
help="After submitting, monitor job status",
)
parser.add_option(
"--wcs",
dest="wcs",
help="Download resulting wcs.fits file, saving to given filename; implies --wait if --urlupload or --upload",
)
parser.add_option(
"--newfits",
dest="newfits",
help="Download resulting new-image.fits file, saving to given filename; implies --wait if --urlupload or --upload",
)
parser.add_option(
"--kmz",
dest="kmz",
help="Download resulting kmz file, saving to given filename; implies --wait if --urlupload or --upload",
)
parser.add_option(
"--annotate",
"-a",
dest="annotate",
help="store information about annotations in give file, JSON format; implies --wait if --urlupload or --upload",
)
parser.add_option(
"--urlupload", "-U", dest="upload_url", help="Upload a file at specified url"
)
parser.add_option(
"--scale-units",
dest="scale_units",
choices=("arcsecperpix", "arcminwidth", "degwidth", "focalmm"),
help="Units for scale estimate",
)
# parser.add_option('--scale-type', dest='scale_type',
# choices=('ul', 'ev'), help='Scale bounds: lower/upper or estimate/error')
parser.add_option(
"--scale-lower", dest="scale_lower", type=float, help="Scale lower-bound"
)
parser.add_option(
"--scale-upper", dest="scale_upper", type=float, help="Scale upper-bound"
)
parser.add_option(
"--scale-est", dest="scale_est", type=float, help="Scale estimate"
)
parser.add_option(
"--scale-err",
dest="scale_err",
type=float,
help='Scale estimate error (in PERCENT), eg "10" if you estimate can be off by 10%',
)
parser.add_option("--ra", dest="center_ra", type=float, help="RA center")
parser.add_option("--dec", dest="center_dec", type=float, help="Dec center")
parser.add_option(
"--radius", dest="radius", type=float, help="Search radius around RA,Dec center"
)
parser.add_option(
"--downsample",
dest="downsample_factor",
type=int,
help="Downsample image by this factor",
)
parser.add_option(
"--positional_error",
dest="positional_error",
type=float,
help="How many pixels a star may be from where it should be.",
)
parser.add_option(
"--parity", dest="parity", choices=("0", "1"), help="Parity (flip) of image"
)
parser.add_option(
"--tweak-order",
dest="tweak_order",
type=int,
help="SIP distortion order (default: 2)",
)
parser.add_option(
"--crpix-center",
dest="crpix_center",
action="store_true",
default=None,
help="Set reference point to center of image?",
)
parser.add_option(
"--invert",
action="store_true",
default=None,
help="Invert image before detecting sources -- for white-sky, black-stars images",
)
parser.add_option("--image-width", type=int, help="Set image width for x,y lists")
parser.add_option("--image-height", type=int, help="Set image height for x,y lists")
parser.add_option(
"--sdss",
dest="sdss_wcs",
nargs=2,
help="Plot SDSS image for the given WCS file; write plot to given PNG filename",
)
parser.add_option(
"--galex",
dest="galex_wcs",
nargs=2,
help="Plot GALEX image for the given WCS file; write plot to given PNG filename",
)
parser.add_option(
"--jobid",
"-i",
dest="solved_id",
type=int,
help="retrieve result for jobId instead of submitting new image",
)
parser.add_option(
"--substatus", "-s", dest="sub_id", help="Get status of a submission"
)
parser.add_option("--jobstatus", "-j", dest="job_id", help="Get status of a job")
parser.add_option(
"--jobs", "-J", dest="myjobs", action="store_true", help="Get all my jobs"
)
parser.add_option(
"--jobsbyexacttag",
"-T",
dest="jobs_by_exact_tag",
help="Get a list of jobs associated with a given tag--exact match",
)
parser.add_option(
"--jobsbytag",
"-t",
dest="jobs_by_tag",
help="Get a list of jobs associated with a given tag",
)
parser.add_option(
"--private",
"-p",
dest="public",
action="store_const",
const="n",
default="y",
help="Hide this submission from other users",
)
parser.add_option(
"--allow_mod_sa",
"-m",
dest="allow_mod",
action="store_const",
const="sa",
default="d",
help="Select license to allow derivative works of submission, but only if shared under same conditions of original license",
)
parser.add_option(
"--no_mod",
"-M",
dest="allow_mod",
action="store_const",
const="n",
default="d",
help="Select license to disallow derivative works of submission",
)
parser.add_option(
"--no_commercial",
"-c",
dest="allow_commercial",
action="store_const",
const="n",
default="d",
help="Select license to disallow commercial use of submission",
)
opt, args = parser.parse_args()
if opt.apikey is None:
# try the environment
opt.apikey = os.environ.get("AN_API_KEY", None)
if opt.apikey is None:
parser.print_help()
print()
print("You must either specify --apikey or set AN_API_KEY")
sys.exit(-1)
args = {}
args["apiurl"] = opt.server
c = Client(**args)
c.login(opt.apikey)
if opt.upload or opt.upload_url or opt.upload_xy:
if opt.wcs or opt.kmz or opt.newfits or opt.annotate:
opt.wait = True
kwargs = dict(
allow_commercial_use=opt.allow_commercial,
allow_modifications=opt.allow_mod,
publicly_visible=opt.public,
)
if opt.scale_lower and opt.scale_upper:
kwargs.update(
scale_lower=opt.scale_lower,
scale_upper=opt.scale_upper,
scale_type="ul",
)
elif opt.scale_est and opt.scale_err:
kwargs.update(
scale_est=opt.scale_est, scale_err=opt.scale_err, scale_type="ev"
)
elif opt.scale_lower or opt.scale_upper:
kwargs.update(scale_type="ul")
if opt.scale_lower:
kwargs.update(scale_lower=opt.scale_lower)
if opt.scale_upper:
kwargs.update(scale_upper=opt.scale_upper)
for key in [
"scale_units",
"center_ra",
"center_dec",
"radius",
"downsample_factor",
"positional_error",
"tweak_order",
"crpix_center",
]:
if getattr(opt, key) is not None:
kwargs[key] = getattr(opt, key)
if opt.parity is not None:
kwargs.update(parity=int(opt.parity))
if opt.upload:
upres = c.upload(opt.upload, **kwargs)
if opt.upload_xy:
from astrometry.util.fits import fits_table
T = fits_table(opt.upload_xy)
kwargs.update(x=[float(x) for x in T.x], y=[float(y) for y in T.y])
upres = c.upload(**kwargs)
if opt.upload_url:
upres = c.url_upload(opt.upload_url, **kwargs)
stat = upres["status"]
if stat != "success":
print("Upload failed: status", stat)
print(upres)
sys.exit(-1)
opt.sub_id = upres["subid"]
if opt.wait:
if opt.solved_id is None:
if opt.sub_id is None:
print("Can't --wait without a submission id or job id!")
sys.exit(-1)
while True:
stat = c.sub_status(opt.sub_id, justdict=True)
print("Got status:", stat)
jobs = stat.get("jobs", [])
if len(jobs):
for j in jobs:
if j is not None:
break
if j is not None:
print("Selecting job id", j)
opt.solved_id = j
break
time.sleep(5)
while True:
stat = c.job_status(opt.solved_id, justdict=True)
print("Got job status:", stat)
if stat.get("status", "") in ["success"]:
success = stat["status"] == "success"
break
time.sleep(5)
if opt.solved_id:
# we have a jobId for retrieving results
retrieveurls = []
if opt.wcs:
# We don't need the API for this, just construct URL
url = opt.server.replace("/api/", "/wcs_file/%i" % opt.solved_id)
retrieveurls.append((url, opt.wcs))
if opt.kmz:
url = opt.server.replace("/api/", "/kml_file/%i/" % opt.solved_id)
retrieveurls.append((url, opt.kmz))
if opt.newfits:
url = opt.server.replace("/api/", "/new_fits_file/%i/" % opt.solved_id)
retrieveurls.append((url, opt.newfits))
for url, fn in retrieveurls:
print("Retrieving file from", url, "to", fn)
f = urlopen(url)
txt = f.read()
w = open(fn, "wb")
w.write(txt)
w.close()
print("Wrote to", fn)
if opt.annotate:
result = c.annotate_data(opt.solved_id)
with open(opt.annotate, "w") as f:
f.write(python2json(result))
if opt.wait:
# behaviour as in old implementation
opt.sub_id = None
if opt.sdss_wcs:
(wcsfn, outfn) = opt.sdss_wcs
c.sdss_plot(outfn, wcsfn)
if opt.galex_wcs:
(wcsfn, outfn) = opt.galex_wcs
c.galex_plot(outfn, wcsfn)
if opt.sub_id:
print(c.sub_status(opt.sub_id))
if opt.job_id:
print(c.job_status(opt.job_id))
if opt.jobs_by_tag:
tag = opt.jobs_by_tag
print(c.jobs_by_tag(tag, None))
if opt.jobs_by_exact_tag:
tag = opt.jobs_by_exact_tag
print(c.jobs_by_tag(tag, "yes"))
if opt.myjobs:
jobs = c.myjobs()
print(jobs)
help='Search radius, in pixels (default 50)')
parser.set_defaults(xcol='X',
ycol='Y',
nimage=0,
cells=0,
cellsize=0,
rxcol='X',
rycol='Y',
nref=0)
opt, args = parser.parse_args()
if len(args) != 3:
parser.print_help()
sys.exit(-1)
imxy = fits_table(args[0])
refxy = fits_table(args[1])
outfn = args[2]
kwargs = {}
if opt.cells:
kwargs['ncells'] = opt.cells
if opt.cellsize:
kwargs['dcell'] = opt.cellsize
ix = imxy.getcolumn(opt.xcol)
iy = imxy.getcolumn(opt.ycol)
ixy = vstack((ix, iy)).T
if opt.nimage:
ixy = ixy[:opt.nimage, :]
def galex_coadds(onegal,
galaxy=None,
radius_mosaic=30,
radius_mask=None,
pixscale=1.5,
ref_pixscale=0.262,
output_dir=None,
galex_dir=None,
log=None,
centrals=True,
verbose=False):
'''Generate custom GALEX cutouts.
radius_mosaic and radius_mask in arcsec
pixscale: GALEX pixel scale in arcsec/pixel.
'''
import fitsio
import matplotlib.pyplot as plt
from astrometry.libkd.spherematch import match_radec
from astrometry.util.resample import resample_with_wcs, OverlapError
from tractor import (Tractor, NanoMaggies, Image, LinearPhotoCal,
NCircularGaussianPSF, ConstantFitsWcs, ConstantSky)
from legacypipe.survey import imsave_jpeg
from legacypipe.catalog import read_fits_catalog
if galaxy is None:
galaxy = 'galaxy'
if galex_dir is None:
galex_dir = os.environ.get('GALEX_DIR')
if output_dir is None:
output_dir = '.'
if radius_mask is None:
radius_mask = radius_mosaic
radius_search = 5.0 # [arcsec]
else:
radius_search = radius_mask
W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int') # [pixels]
targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0,
-pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W),
float(H))
# Read the custom Tractor catalog
tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy))
if not os.path.isfile(tractorfile):
print('Missing Tractor catalog {}'.format(tractorfile))
return 0
cat = fits_table(tractorfile)
print('Read {} sources from {}'.format(len(cat), tractorfile),
flush=True,
file=log)
keep = np.ones(len(cat)).astype(bool)
if centrals:
# Find the large central galaxy and mask out (ignore) all the models
# which are within its elliptical mask.
# This algorithm will have to change for mosaics not centered on large
# galaxies, e.g., in galaxy groups.
m1, m2, d12 = match_radec(cat.ra,
cat.dec,
onegal['RA'],
onegal['DEC'],
radius_search / 3600.0,
nearest=False)
if len(m1) == 0:
print('No central galaxies found at the central coordinates!',
flush=True,
file=log)
else:
pixfactor = ref_pixscale / pixscale # shift the optical Tractor positions
for mm in m1:
morphtype = cat.type[mm].strip()
if morphtype == 'EXP' or morphtype == 'COMP':
e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[
mm], cat.shapeexp_r[mm] # [arcsec]
elif morphtype == 'DEV' or morphtype == 'COMP':
e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[
mm], cat.shapedev_r[mm] # [arcsec]
else:
r50 = None
if r50:
majoraxis = r50 * 5 / pixscale # [pixels]
ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2)
these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis,
ba * majoraxis,
np.radians(phi),
cat.bx * pixfactor,
cat.by * pixfactor)
if np.sum(these) > 0:
#keep[these] = False
pass
print('Hack!')
keep[mm] = False
#srcs = read_fits_catalog(cat)
#_srcs = np.array(srcs)[~keep].tolist()
#mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0)
#import matplotlib.pyplot as plt
##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png')
#plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png')
#pdb.set_trace()
srcs = read_fits_catalog(cat)
for src in srcs:
src.freezeAllBut('brightness')
#srcs_nocentral = np.array(srcs)[keep].tolist()
# Find all overlapping GALEX tiles and then read the tims.
galex_tiles = _read_galex_tiles(targetwcs,
galex_dir,
log=log,
verbose=verbose)
gbands = ['n', 'f']
nicegbands = ['NUV', 'FUV']
zps = dict(n=20.08, f=18.82)
coimgs, comods, coresids, coimgs_central, comods_nocentral = [], [], [], [], []
for niceband, band in zip(nicegbands, gbands):
J = np.flatnonzero(galex_tiles.get('has_' + band))
print(len(J), 'GALEX tiles have coverage in band', band)
coimg = np.zeros((H, W), np.float32)
comod = np.zeros((H, W), np.float32)
cowt = np.zeros((H, W), np.float32)
comod_nocentral = np.zeros((H, W), np.float32)
for src in srcs:
src.setBrightness(NanoMaggies(**{band: 1}))
for j in J:
brick = galex_tiles[j]
fn = os.path.join(
galex_dir, brick.tilename.strip(),
'%s-%sd-intbgsub.fits.gz' % (brick.brickname, band))
#print(fn)
gwcs = Tan(*[
float(f) for f in [
brick.crval1, brick.crval2, brick.crpix1, brick.crpix2,
brick.cdelt1, 0., 0., brick.cdelt2, 3840., 3840.
]
])
img = fitsio.read(fn)
#print('Read', img.shape)
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, gwcs, [], 3)
except OverlapError:
continue
K = np.flatnonzero(img[Yi, Xi] != 0.)
if len(K) == 0:
continue
Yo, Xo, Yi, Xi = Yo[K], Xo[K], Yi[K], Xi[K]
wt = brick.get(band + 'exptime')
coimg[Yo, Xo] += wt * img[Yi, Xi]
cowt[Yo, Xo] += wt
x0, x1, y0, y1 = min(Xi), max(Xi), min(Yi), max(Yi)
subwcs = gwcs.get_subimage(x0, y0, x1 - x0 + 1, y1 - y0 + 1)
twcs = ConstantFitsWcs(subwcs)
timg = img[y0:y1 + 1, x0:x1 + 1]
tie = np.ones_like(timg) ## HACK!
#hdr = fitsio.read_header(fn)
#zp = hdr['']
zp = zps[band]
photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp),
band=band)
tsky = ConstantSky(0.0)
# HACK -- circular Gaussian PSF of fixed size...
# in arcsec
#fwhms = dict(NUV=6.0, FUV=6.0)
# -> sigma in pixels
#sig = fwhms[band] / 2.35 / twcs.pixel_scale()
sig = 6.0 / np.sqrt(8 * np.log(2)) / twcs.pixel_scale()
tpsf = NCircularGaussianPSF([sig], [1.])
tim = Image(data=timg,
inverr=tie,
psf=tpsf,
wcs=twcs,
sky=tsky,
photocal=photocal,
name='GALEX ' + band + brick.brickname)
## Build the model image with and without the central galaxy model.
tractor = Tractor([tim], srcs)
mod = tractor.getModelImage(0)
tractor.freezeParam('images')
tractor.optimize_forced_photometry(priors=False,
shared_params=False)
mod = tractor.getModelImage(0)
srcs_nocentral = np.array(srcs)[keep].tolist()
#srcs_nocentral = np.array(srcs)[nocentral].tolist()
tractor_nocentral = Tractor([tim], srcs_nocentral)
mod_nocentral = tractor_nocentral.getModelImage(0)
comod[Yo, Xo] += wt * mod[Yi - y0, Xi - x0]
comod_nocentral[Yo, Xo] += wt * mod_nocentral[Yi - y0, Xi - x0]
coimg /= np.maximum(cowt, 1e-18)
comod /= np.maximum(cowt, 1e-18)
comod_nocentral /= np.maximum(cowt, 1e-18)
coresid = coimg - comod
# Subtract the model image which excludes the central (comod_nocentral)
# from the data (coimg) to isolate the light of the central
# (coimg_central).
coimg_central = coimg - comod_nocentral
coimgs.append(coimg)
comods.append(comod)
coresids.append(coresid)
comods_nocentral.append(comod_nocentral)
coimgs_central.append(coimg_central)
# Write out the final images with and without the central, making sure
# to apply the zeropoint to go from counts/s to AB nanomaggies.
# https://asd.gsfc.nasa.gov/archive/galex/FAQ/counts_background.html
for thisimg, imtype in zip((coimg, comod, comod_nocentral),
('image', 'model', 'model-nocentral')):
fitsfile = os.path.join(
output_dir, '{}-{}-{}.fits'.format(galaxy, imtype, niceband))
if verbose:
print('Writing {}'.format(fitsfile))
fitsio.write(fitsfile,
thisimg * 10**(-0.4 * (zp - 22.5)),
clobber=True)
# Build a color mosaic (but note that the images here are in units of
# background-subtracted counts/s).
#_galex_rgb = _galex_rgb_moustakas
#_galex_rgb = _galex_rgb_dstn
_galex_rgb = _galex_rgb_official
for imgs, imtype in zip(
(coimgs, comods, coresids, comods_nocentral, coimgs_central),
('image', 'model', 'resid', 'model-nocentral', 'image-central')):
rgb = _galex_rgb(imgs)
jpgfile = os.path.join(output_dir,
'{}-{}-FUVNUV.jpg'.format(galaxy, imtype))
if verbose:
print('Writing {}'.format(jpgfile))
imsave_jpeg(jpgfile, rgb, origin='lower')
return 1
#summary_plots(summaryfn, ps)
#sys.exit(0)
#fns = glob('/project/projectdirs/cosmo/work/legacysurvey/dr3/coadd/*/*/*-depth.fits')
fns = glob(
'/project/projectdirs/cosmo/data/legacysurvey/dr3/coadd/*/*/*-depth.fits'
)
fns.sort()
print(len(fns), 'depth files')
fn = fns.pop(0)
print('Reading', fn)
# We'll keep all files for merging...
TT = []
T = fits_table(fn)
# Create / upgrade the count columns to int64.
for band in 'grz':
for pro in ['ptsrc', 'gal']:
col = 'counts_%s_%s' % (pro, band)
if not col in T.columns():
v = np.zeros(len(T), np.int64)
else:
v = T.get(col).astype(np.int64)
T.set(col, v)
T.brickname = np.array([brickname_from_filename(fn)] * len(T))
TT.append(T.copy())
for ifn, fn in enumerate(fns):
print('Reading', ifn, 'of', len(fns), ':', fn)
t = fits_table(fn)
def run_sed_matched_filters(SEDs,
bands,
detmaps,
detivs,
omit_xy,
targetwcs,
nsigma=5,
saturated_pix=None,
plots=False,
ps=None,
mp=None):
'''
Runs a given set of SED-matched filters.
Parameters
----------
SEDs : list of (name, sed) tuples
The SEDs to run. The `sed` values are lists the same length
as `bands`.
bands : list of string
The band names of `detmaps` and `detivs`.
detmaps : numpy array, float
Detection maps for each of the listed `bands`.
detivs : numpy array, float
Inverse-variances of the `detmaps`.
omit_xy : None, or (xx,yy) tuple
Existing sources to avoid.
targetwcs : WCS object
WCS object to use to convert pixel values into RA,Decs for the
returned Tractor PointSource objects.
nsigma : float, optional
Detection threshold
saturated_pix : None or numpy array, boolean
Passed through to sed_matched_detection.
A map of pixels that are always considered "hot" when
determining whether a new source touches hot pixels of an
existing source.
plots : boolean, optional
Create plots?
ps : PlotSequence object
Create plots?
mp : multiproc object
Multiprocessing
Returns
-------
Tnew : fits_table
Table of new sources detected
newcat : list of PointSource objects
Newly detected objects, with positions and fluxes, as Tractor
PointSource objects.
hot : numpy array of bool
"Hot pixels" containing sources.
See also
--------
sed_matched_detection : run a single SED-matched filter.
'''
if omit_xy is not None:
xx, yy = omit_xy
n0 = len(xx)
else:
xx, yy = [], []
n0 = 0
H, W = detmaps[0].shape
hot = np.zeros((H, W), bool)
peaksn = []
apsn = []
for sedname, sed in SEDs:
print('SED', sedname)
if plots:
pps = ps
else:
pps = None
t0 = Time()
sedhot, px, py, peakval, apval = sed_matched_detection(
sedname,
sed,
detmaps,
detivs,
bands,
xx,
yy,
nsigma=nsigma,
saturated_pix=saturated_pix,
ps=pps)
print('SED took', Time() - t0)
if sedhot is None:
continue
print(len(px), 'new peaks')
hot |= sedhot
# With an empty xx, np.append turns it into a double!
xx = np.append(xx, px).astype(int)
yy = np.append(yy, py).astype(int)
peaksn.extend(peakval)
apsn.extend(apval)
# New peaks:
peakx = xx[n0:]
peaky = yy[n0:]
if len(peakx) == 0:
return None, None, None
# Add sources for the new peaks we found
pr, pd = targetwcs.pixelxy2radec(peakx + 1, peaky + 1)
print('Adding', len(pr), 'new sources')
# Also create FITS table for new sources
Tnew = fits_table()
Tnew.ra = pr
Tnew.dec = pd
Tnew.tx = peakx
Tnew.ty = peaky
assert (len(peaksn) == len(Tnew))
assert (len(apsn) == len(Tnew))
Tnew.peaksn = np.array(peaksn)
Tnew.apsn = np.array(apsn)
Tnew.itx = np.clip(np.round(Tnew.tx), 0, W - 1).astype(int)
Tnew.ity = np.clip(np.round(Tnew.ty), 0, H - 1).astype(int)
newcat = []
for i, (r, d, x, y) in enumerate(zip(pr, pd, peakx, peaky)):
fluxes = dict([(band, detmap[Tnew.ity[i], Tnew.itx[i]])
for band, detmap in zip(bands, detmaps)])
newcat.append(
PointSource(RaDecPos(r, d), NanoMaggies(order=bands, **fluxes)))
return Tnew, newcat, hot
def __init__(self, targz_dir, decals=True):
self.bricks = fits_table(
os.path.join(targz_dir, 'legacysurveydir',
'survey-bricks.fits.gz'))
if decals:
self.bricks.cut((self.bricks.dec > -30) & (self.bricks.dec < 30))
def orig_code(data, nmatch):
nside = Healpix().get_nside(len(data))
_, lo, hi = sigmaclip(data[data != 0], low=3, high=3)
flag = np.logical_or(data < lo, data > hi)
flag *= (nmatch > 20)
ra, dec = hp.pix2ang(nside, np.where(flag)[0], lonlat=True)
# PLOTTING
ralim = [ra.min(), ra.max()]
declim = [dec.min(), dec.max()]
my_mollzoom(ra,
dec,
data[flag],
'outliers',
ralim=ralim,
declim=declim,
vlim=(lo, hi))
temp_ra, temp_dec = hp.pix2ang(nside,
np.where(np.ones(len(data), bool))[0],
lonlat=True)
keep= (temp_ra >= ralim[0])*\
(temp_ra <= ralim[1])*\
(temp_dec >= declim[0])*\
(temp_dec <= declim[1])
my_mollzoom(temp_ra[keep],
temp_dec[keep],
data[keep],
'all',
ralim=ralim,
declim=declim,
vlim=(lo, hi))
keep *= (nmatch > 20)
my_mollzoom(temp_ra[keep],
temp_dec[keep],
data[keep],
'nmatch_gt20',
ralim=ralim,
declim=declim,
vlim=(lo, hi))
# Match bricks
#heal= fits_table()
#for col,arr in zip(['ra','dec'],[ra,dec]):
# heal.set(col, arr)
brick = fits_table(
os.path.join(args.targz_dir, 'legacysurveydir',
'survey-bricks.fits.gz'))
brick.cut( (brick.dec > -40)*\
(brick.dec < 40))
#deg_per_healpix= get_pixscale(npix,unit='deg')
deg_per_brick = 0.25
#imatch,imiss,d2d= Matcher().match_within(heal,brick, dist= deg_per_brick/2)
I, J, d = match_radec(ra,
dec,
brick.ra,
brick.dec,
deg_per_brick / 2,
nearest=True)
#raise ValueError
brick.cut(imatch['obs'])
my_scatter(brick.ra, brick.dec, 'bricks', ralim=ralim, declim=declim)
id = fn.replace('/', '').replace('.fits', '')
savenm = os.path.join(args.outdir, 'brick_table_%s.fits' % id)
brick.writeto(savenm)
print('Wrote %s' % savenm)
def get_healpix_catalog(self, healpix):
#return super().get_healpix_catalog(healpix)
from astrometry.util.fits import fits_table
fname = self.fnpattern % dict(hp=healpix)
#print('Reading', fname)
return fits_table(fname, columns=self.columns)
def merge_forced(survey, brickname, cat, bands='grz'):
ccdfn = survey.find_file('ccds-table', brick=brickname)
CCDs = fits_table(ccdfn)
print('Read', len(CCDs), 'CCDs')
# objects in the catalog: (release,brickid,objid)
catobjs = set([(r, b, o)
for r, b, o in zip(cat.release, cat.brickid, cat.objid)])
# (release, brickid, objid, band) -> [ index in forced-phot table ]
phot_index = {}
camexp = set()
FF = []
for ccd in CCDs:
cam = ccd.camera.strip()
key = (cam, ccd.expnum)
if key in camexp:
# already read this camera-expnum
continue
camexp.add(key)
ffn = survey.find_file('forced', camera=cam, expnum=ccd.expnum)
print('Forced phot filename:', ffn)
F = fits_table(ffn)
print('Read', len(F), 'forced-phot entries for CCD')
ikeep = []
for i, (r, b, o) in enumerate(zip(F.release, F.brickid, F.objid)):
if (r, b, o) in catobjs:
ikeep.append(i)
if len(ikeep) == 0:
print('No catalog objects found in this forced-phot table.')
continue
F.cut(np.array(ikeep))
print('Cut to', len(F), 'phot entries matching catalog')
FF.append(F)
F = merge_tables(FF)
F._header = FF[0]._header
del FF
I = np.lexsort(
(F.expnum, F.camera, F.filter, F.objid, F.brickid, F.release))
F.cut(I)
for i, (r, brick, o,
band) in enumerate(zip(F.release, F.brickid, F.objid, F.filter)):
key = (r, brick, o, band)
if not key in phot_index:
phot_index[key] = []
phot_index[key].append(i)
# find largest number of photometry measurements!
Nmax = max([len(inds) for inds in phot_index.values()])
print('Maximum number of photometry entries:', Nmax)
for band in bands:
nobs = np.zeros(len(cat), np.int32)
indx = np.empty(len(cat), np.int32)
indx[:] = -1
cat.set('nobs_%s' % band, nobs)
cat.set('index_%s' % band, indx)
for i, (r, brick,
o) in enumerate(zip(cat.release, cat.brickid, cat.objid)):
key = (r, brick, o, band)
try:
inds = phot_index[key]
except KeyError:
continue
nobs[i] = len(inds)
# Indices are all contiguous (so we only need store the first one)
assert (np.all(np.array(inds) == (np.arange(len(inds)) + inds[0])))
indx[i] = inds[0]
return cat, F
