def skyplot():
plot = Plotstuff(size=(800, 800), rdw=(103.1, 37.45, 0.8), outformat='png')
plot.color = 'verydarkblue'
plot.plot('fill')
for ext in range(1, 17):
fn = 'mos3.68488.fits'
hdr = fitsio.read_header(fn, ext=ext)
wcs = wcs_pv2sip_hdr(hdr)
plot.color = 'red'
plot.outline.wcs = anwcs_new_sip(wcs)
plot.plot('outline')
plot.color = 'white'
plot.apply_settings()
rc, dc = wcs.radec_center()
plot.text_radec(rc, dc, hdr['EXTNAME'])
plot.color = 'white'
for ext in range(1, 17):
fn = 'an2/mos3.68488.ext%02i.wcs' % ext
plot.outline.wcs = anwcs(fn)
plot.plot('outline')
plot.rgb = (0.2, 0.2, 0.2)
plot.plot_grid(0.1, 0.1)
plot.color = 'gray'
plot.plot_grid(0.5, 0.5, 0.5, 0.5)
plot.write('plot.png')
def get_wcs(self, hdr):
# Older images have ZPX, newer TPV.
if hdr['CTYPE1'] == 'RA---TPV':
from astrometry.util.util import wcs_pv2sip_hdr
wcs = wcs_pv2sip_hdr(hdr)
else:
from astrometry.util.util import Tan
hdr['CTYPE1'] = 'RA---TAN'
hdr['CTYPE2'] = 'DEC--TAN'
wcs = Tan(hdr)
return wcs
def get_wcs(self):
hdr = fitsio.read_header(self.imgfn, self.hdu)
wcs = wcs_pv2sip_hdr(hdr)
phdr = fitsio.read_header(self.imgfn, 0)
dra, ddec = self.decals.get_astrometric_zeropoint_for(self)
r, d = wcs.get_crval()
print('Applying astrometric zeropoint:', (dra, ddec))
wcs.set_crval((r + dra, d + ddec))
wcs.version = ''
wcs.plver = phdr.get('PLVER', '').strip()
return wcs
def get_wcs(self):
hdr = fitsio.read_header(self.imgfn, self.hdu)
wcs = wcs_pv2sip_hdr(hdr)
phdr = fitsio.read_header(self.imgfn, 0)
dra,ddec = self.decals.get_astrometric_zeropoint_for(self)
r,d = wcs.get_crval()
print('Applying astrometric zeropoint:', (dra,ddec))
wcs.set_crval((r + dra, d + ddec))
wcs.version = ''
wcs.plver = phdr.get('PLVER', '').strip()
return wcs
def get_wcs(hdr, tpv_to_sip=True):
# Thanks Dstn :)
# https://github.com/legacysurvey/legacypipe/blob/master/py/legacypipe/cpimage.py#102
width = hdr['NAXIS1']
height = hdr['NAXIS2']
if not (tpv_to_sip):
# extract wcs from fits header directly
wcs= Tan(hdr['CRVAL1'], hdr['CRVAL2'],hdr['CRPIX1'],hdr['CRPIX2'],\
hdr['CD1_1'],hdr['CD1_2'],hdr['CD2_1'],hdr['CD2_2'],\
float(width),float(height))
else:
# Make sure the PV-to-SIP converter samples enough points for small
# images
stepsize = 0
if min(width, height) < 600:
stepsize = min(width, height) / 10.
wcs = wcs_pv2sip_hdr(hdr, stepsize=stepsize)
# Dstn adds an offset correction i.e.
# Correction: ccd,ccdraoff, decoff from zeropoints file
# Should I do this?
return wcs
def get_wcs(self):
# Make sure the PV-to-SIP converter samples enough points for small
# images
stepsize = 0
if min(self.width, self.height) < 600:
stepsize = min(self.width, self.height) / 10.;
hdr = self.read_image_header()
if self.camera == '90prime':
# WCS is in myriad of formats
# Don't support TNX yet, use TAN for now
hdr = self.read_image_header()
hdr['CTYPE1'] = 'RA---TAN'
hdr['CTYPE2'] = 'DEC--TAN'
wcs = wcs_pv2sip_hdr(hdr, stepsize=stepsize)
# Correctoin: ccd,ccdraoff, decoff from zeropoints file
dra,ddec = self.dradec
print('Applying astrometric zeropoint:', (dra,ddec))
r,d = wcs.get_crval()
wcs.set_crval((r + dra, d + ddec))
wcs.version = ''
phdr = self.read_image_primary_header()
wcs.plver = phdr.get('PLVER', '').strip()
return wcs
def get_wcs(self,hdr=None):
# Make sure the PV-to-SIP converter samples enough points for small
# images
stepsize = 0
if min(self.width, self.height) < 600:
stepsize = min(self.width, self.height) / 10.;
if hdr is None:
hdr = self.read_image_header()
#if self.camera == '90prime':
# WCS is in myriad of formats
# Don't support TNX yet, use TAN for now
# hdr = self.read_image_header()
# hdr['CTYPE1'] = 'RA---TAN'
# hdr['CTYPE2'] = 'DEC--TAN'
wcs = wcs_pv2sip_hdr(hdr, stepsize=stepsize)
# Correctoin: ccd,ccdraoff, decoff from zeropoints file
dra,ddec = self.dradec
print('Applying astrometric zeropoint:', (dra,ddec))
r,d = wcs.get_crval()
wcs.set_crval((r + dra, d + ddec))
wcs.version = ''
phdr = self.read_image_primary_header()
wcs.plver = phdr.get('PLVER', '').strip()
return wcs
3.)
Tnear = T[J]
print('Found', len(Tnear), 'tiles nearby')
# sort by distance from center
I = np.argsort(d)
Tnear.cut(I)
Tnear.rename('pass', 'passnum')
# MzLS_594295_z
ps = PlotSequence('tile', format='%03i')
wcses = []
for ext in range(1, 4 + 1):
hdr = fitsio.read_header(fn, ext=ext)
wcs = wcs_pv2sip_hdr(hdr)
wcses.append(wcs)
print('WCS', wcs)
PW, PH = 800, 800
plot = Plotstuff(size=(PW, PH), rdw=(tile.ra, tile.dec, 2), outformat='png')
plot.color = 'verydarkblue'
plot.plot('fill')
plot.color = 'white'
plot.alpha = 0.25
plot.outline.fill = True
plot.apply_settings()
for passnum in [1, 2, 3]:
def main():
# Where are the data?
datadir = os.path.join(os.path.dirname(__file__), 'data-decam')
name = 'decam-520206-S16'
imagefn = os.path.join(datadir, '%s-image-sub.fits' % name)
invvarfn = os.path.join(datadir, '%s-invvar-sub.fits' % name)
psfexfn = os.path.join(datadir, '%s-psfex.fits' % name)
catfn = os.path.join(datadir, 'tractor-1816p325-sub.fits')
# Read the image and inverse-variance maps.
image = fitsio.read(imagefn)
invvar = fitsio.read(invvarfn)
# The DECam inverse-variance maps are unfortunately corrupted
# by fpack, causing zeros to become negative. Fix those.
invvar[invvar < np.median(invvar) * 0.1] = 0.
H, W = image.shape
print('Subimage size:', image.shape)
# For the PSF model, we need to know what subimage region this is:
subimage_offset = (35, 1465)
# We also need the calibrated zeropoint.
zeropoint = 24.7787
# What filter was this image taken in? (z)
prim_header = fitsio.read_header(imagefn)
band = prim_header['FILTER'].strip()[0]
print('Band:', band)
# These DECam images were calibrated so that the zeropoints need
# an exposure-time factor, so add that in.
exptime = prim_header['EXPTIME']
zeropoint += 2.5 * np.log10(exptime)
# Read the PsfEx model file
psf = PixelizedPsfEx(psfexfn)
# Instantiate a constant pixelized PSF at the image center
# (of the subimage)
x0, y0 = subimage_offset
psf = psf.constantPsfAt(x0 + W / 2., y0 + H / 2.)
# Load the WCS model from the header
# We convert from the RA---TPV type to RA---SIP
header = fitsio.read_header(imagefn, ext=1)
wcsobj = wcs_pv2sip_hdr(header, stepsize=10)
# We'll just use a rough sky estimate...
skyval = np.median(image)
# Create the Tractor Image (tim).
tim = Image(data=image,
invvar=invvar,
psf=psf,
wcs=ConstantFitsWcs(wcsobj),
sky=ConstantSky(skyval),
photocal=LinearPhotoCal(
NanoMaggies.zeropointToScale(zeropoint), band=band))
# Read the official DECaLS DR3 catalog -- it has only two sources in this subimage.
catalog = fits_table(catfn)
print('Read', len(catalog), 'sources')
print('Source types:', catalog.type)
# Create Tractor sources corresponding to these two catalog
# entries.
# In DECaLS, the "SIMP" type is a round Exponential galaxy with a
# fixed 0.45" radius, but we'll treat it as a general Exp galaxy.
sources = []
for c in catalog:
# Create a "position" object given the catalog RA,Dec
position = RaDecPos(c.ra, c.dec)
# Create a "brightness" object; in the catalog, the fluxes are
# stored in a [ugrizY] array, so pull out the right index
band_index = 'ugrizY'.index(band)
flux = c.decam_flux[band_index]
brightness = NanoMaggies(**{band: flux})
# Depending on the source classification in the catalog, pull
# out different fields for the galaxy shape, and for the
# galaxy type. The DECaLS catalogs, conveniently, store
# galaxy shapes as (radius, e1, e2) ellipses.
if c.type.strip() == 'DEV':
shape = EllipseE(c.shapedev_r, c.shapedev_e1, c.shapedev_e2)
galclass = DevGalaxy
elif c.type.strip() == 'SIMP':
shape = EllipseE(c.shapeexp_r, c.shapeexp_e1, c.shapeexp_e2)
galclass = ExpGalaxy
else:
assert (False)
# Create the tractor galaxy object
source = galclass(position, brightness, shape)
print('Created', source)
sources.append(source)
# Create the Tractor object -- a list of tractor Images and a list of tractor sources.
tractor = Tractor([tim], sources)
# Render the initial model image.
print('Getting initial model...')
mod = tractor.getModelImage(0)
make_plot(tim, mod, 'Initial Scene', 'mod0.png')
# Instantiate a new source at the location of the unmodelled peak.
print('Adding new source...')
# Find the peak very naively...
ipeak = np.argmax((image - mod) * tim.inverr)
iy, ix = np.unravel_index(ipeak, tim.shape)
print('Residual peak at', ix, iy)
# Compute the RA,Dec location of the peak...
radec = tim.getWcs().pixelToPosition(ix, iy)
print('RA,Dec', radec)
# Try modelling it as a point source.
# We'll initialize the brightness arbitrarily to 1 nanomaggy (= mag 22.5)
brightness = NanoMaggies(**{band: 1.})
source = PointSource(radec, brightness)
# Add it to the catalog!
tractor.catalog.append(source)
# Render the new model image with this source added.
mod = tractor.getModelImage(0)
make_plot(tim, mod, 'New Source (Before Fit)', 'mod1.png')
print('Fitting new source...')
# Now we're going to fit for the properties of the new source we
# added.
# We don't want to fit for any of the image calibration properties:
tractor.freezeParam('images')
# And we don't (yet) want to fit the existing sources. The new
# source is index number 2, so freeze everything else in the catalog.
tractor.catalog.freezeAllBut(2)
print('Fitting parameters:')
tractor.printThawedParams()
# Do the actual optimization:
tractor.optimize_loop()
mod = tractor.getModelImage(0)
make_plot(tim, mod, 'New Source Fit', 'mod2.png')
print('Fitting sources simultaneously...')
# Now let's unfreeze all the sources and fit them simultaneously.
tractor.catalog.thawAllParams()
tractor.printThawedParams()
tractor.optimize_loop()
mod = tractor.getModelImage(0)
make_plot(tim, mod, 'Simultaneous Fit', 'mod3.png')
def mosaic():
'''
> cp ~/cosmo/staging/mosaicz/MZLS_CP/CP20180102/k4m_180103_040423_ooi_zd_v1.fits.fz /tmp
> funpack /tmp/k4m_180103_040423_ooi_zd_v1.fits.fz
> fitsgetext -i /tmp/k4m_180103_040423_ooi_zd_v1.fits -o mosaic-%02i.wcs -a -H
> cat mosaic-??.wcs > mosaic.wcs
> for ((i=1; i<=4; i++)); do modhead mosaic.wcs+$i NAXIS2; modhead mosaic.wcs+$i NAXIS2 0; done
NAXIS2 = 4079 / Axis length
NAXIS2 = 4079 / Axis length
NAXIS2 = 4079 / Axis length
NAXIS2 = 4061 / Axis length
'''
plt.figure(figsize=(4, 3))
plt.subplots_adjust(left=0.15, right=0.99, top=0.99, bottom=0.15)
T = fits_table('obstatus/mosaic-tiles_obstatus.fits')
print(len(T), 'tiles')
T.rename('pass', 'passnum')
T.cut(T.passnum <= 3)
print(len(T), 'tiles with passnum <= 3')
ra, dec = 180.216, 40.191
#ra,dec = 180., 40.
I, J, d = match_radec(T.ra, T.dec, ra, dec, 2.)
print(len(I), 'tiles near', ra, dec)
T.cut(I)
T.dist = d
print('dists:', d)
print('Passes:', T.passnum)
F = fitsio.FITS(os.path.join(os.path.dirname(__file__), 'mosaic.wcs'))
wcs = []
heights = [4079, 4079, 4079, 4061]
for i in range(1, len(F)):
hdr = F[i].read_header()
W = hdr['NAXIS1']
wcs.append(wcs_pv2sip_hdr(hdr, H=heights[i - 1], W=W))
print('WCS:', wcs[-1])
# Rendering canvas
W, H = 2200, 2200
pixsc = 4. / 3600.
targetwcs = Tan(ra, dec, W / 2. + 0.5, H / 2. + 0.5, -pixsc, 0., 0., pixsc,
float(W), float(H))
II = np.lexsort((T.dist, T.passnum))
print('First tile center:', T.ra[II[0]], T.dec[II[0]])
# This is for making the (vector) PDF format tiling images.
for maxit in [0, 8, 36, 37, 44, 73, 74, 82, 112]:
plot = Plotstuff(outformat='pdf',
ra=ra,
dec=dec,
width=W * pixsc,
size=(W, H),
outfn='tile-mosaic-%02i.pdf' % maxit)
plot.color = 'white'
plot.alpha = 1.
plot.plot('fill')
out = plot.outline
out.fill = True
out.stepsize = 1024.
plot.color = 'black'
plot.alpha = 0.4
plot.apply_settings()
for it, t in enumerate(T[II]):
print('Tile', it, 'pass', t.passnum)
for w in wcs:
w.set_crval((t.ra, t.dec))
out.wcs = anwcs_new_sip(w)
plot.plot('outline')
if it == maxit:
print('Writing', it)
plot.write()
break
# # And this is for PNG-format tiling images and histograms.
cov = np.zeros((H, W), np.uint8)
for it, t in enumerate(T[II]):
print('Tile', it, 'pass', t.passnum)
for w in wcs:
w.set_crval((t.ra, t.dec))
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, w)
except:
continue
cov[Yo, Xo] += 1
if it in [0, 8, 36, 37, 44, 73, 74, 82, 112]:
mx = {1: 2, 2: 4, 3: 6}[t.passnum]
# plt.clf()
# plt.imshow(cov, interpolation='nearest', origin='lower', vmin=0, vmax=mx)
# plt.colorbar()
# plt.savefig('tile-%02i.png' % it)
plt.imsave('tile-mosaic-%02i.png' % it,
cov,
origin='lower',
vmin=0,
vmax=mx,
cmap=antigray)
if it in [36, 73, 112]:
from collections import Counter
print('Coverage counts:', Counter(cov.ravel()).most_common())
bins = -0.5 + np.arange(8)
plt.clf()
n, b, p = plt.hist(cov.ravel(), bins=bins, normed=True)
#plt.hist(cov.ravel(), bins=bins, normed=True, cumulative=True, histtype='step')
# Cumulative histogram from the right...
xx, yy = [], []
for blo, bhi, ni in reversed(list(zip(bins, bins[1:], n))):
nc = float(np.sum(cov.ravel() > blo)) / len(cov.ravel())
yy.extend([nc, nc])
xx.extend([bhi, blo])
if ni > 0:
if nc != ni:
if nc > ni + 0.03:
# If there's room, label the histogram bin above, else below
plt.text((blo + bhi) / 2.,
ni,
'%.1f \%%' % (100. * ni),
ha='center',
va='bottom',
color='k')
else:
plt.text((blo + bhi) / 2.,
ni - 0.01,
'%.1f \%%' % (100. * ni),
ha='center',
va='top',
color='k')
plt.text((blo + bhi) / 2.,
nc,
'%.1f \%%' % (100. * nc),
ha='center',
va='bottom',
color='k')
plt.plot(xx, yy, 'k-')
plt.xlim(bins.min(), bins.max())
plt.ylim(0., 1.1)
plt.xlabel('Number of exposures')
plt.ylabel('Fraction of sky')
plt.savefig('hist-mosaic-%02i.pdf' % it)
def decam():
'''
cp ~/cosmo/staging/decam/DECam_CP/CP20170731/c4d_170801_080516_oki_g_v1.fits.fz /tmp
funpack /tmp/c4d_170801_080516_oki_g_v1.fits.fz
fitsgetext -i /tmp/c4d_170801_080516_oki_g_v1.fits -o decam-%02i.wcs -a -H
cat decam-??.wcs > decam.wcs
for ((i=1; i<=61; i++)); do modhead decam.wcs+$i NAXIS2 0; done
'''
plt.figure(figsize=(4, 3))
plt.subplots_adjust(left=0.15, right=0.99, top=0.99, bottom=0.15)
T = fits_table('obstatus/decam-tiles_obstatus.fits')
T.rename('pass', 'passnum')
ra, dec = 0.933, 0.
I, J, d = match_radec(T.ra, T.dec, ra, dec, 5.) #2.8)
print(len(I), 'tiles near 0,0')
T.cut(I)
T.dist = d
print('dists:', d)
print('Passes:', T.passnum)
F = fitsio.FITS(os.path.join(os.path.dirname(__file__), 'decam.wcs'))
wcs = []
for i in range(1, len(F)):
hdr = F[i].read_header()
wcs.append(wcs_pv2sip_hdr(hdr, W=2046, H=4094))
W, H = 5000, 5000
pixsc = 4. / 3600.
targetwcs = Tan(ra, dec, W / 2. + 0.5, H / 2. + 0.5, -pixsc, 0., 0., pixsc,
float(W), float(H))
II = np.lexsort((T.dist, T.passnum))
# This is for making the (vector) PDF format tiling images.
for maxit in [0, 6, 30, 31, 37, 61, 62, 68, 90]:
#mx = { 1: 2, 2: 4, 3: 6 }[t.passnum]
plot = Plotstuff(outformat='pdf',
ra=ra,
dec=dec,
width=W * pixsc,
size=(W, H),
outfn='tile-%02i.pdf' % maxit)
plot.color = 'white'
plot.alpha = 1.
plot.plot('fill')
out = plot.outline
out.fill = True
out.stepsize = 1024.
plot.color = 'black'
plot.alpha = 0.4
plot.apply_settings()
for it, t in enumerate(T[II]):
print('Tile', it, 'pass', t.passnum)
for w in wcs:
w.set_crval((t.ra, t.dec))
out.wcs = anwcs_new_sip(w)
plot.plot('outline')
if it == maxit:
print('Writing', it)
plot.write()
break
# And this is for PNG-format tiling images and histograms.
cov = np.zeros((H, W), np.uint8)
for it, t in enumerate(T[II]):
print('Tile', it, 'pass', t.passnum)
for w in wcs:
w.set_crval((t.ra, t.dec))
#print('WCS:', w)
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, w)
except:
#import traceback
#traceback.print_exc()
continue
cov[Yo, Xo] += 1
if it in [0, 6, 30, 31, 37, 61, 62, 68, 90]:
mx = {1: 2, 2: 4, 3: 6}[t.passnum]
# plt.clf()
# plt.imshow(cov, interpolation='nearest', origin='lower', vmin=0, vmax=mx)
# plt.colorbar()
# plt.savefig('tile-%02i.png' % it)
plt.imsave('tile-%02i.png' % it,
cov,
origin='lower',
vmin=0,
vmax=mx,
cmap=antigray)
#plt.imsave('tile-%02i.pdf' % it, cov, origin='lower', vmin=0, vmax=mx, cmap=antigray, format='pdf')
if it in [30, 61, 90]:
from collections import Counter
print('Coverage counts:', Counter(cov.ravel()).most_common())
bins = -0.5 + np.arange(8)
plt.clf()
n, b, p = plt.hist(cov.ravel(), bins=bins, normed=True)
#plt.hist(cov.ravel(), bins=bins, normed=True, cumulative=True, histtype='step')
# Cumulative histogram from the right...
xx, yy = [], []
for blo, bhi, ni in reversed(zip(bins, bins[1:], n)):
nc = float(np.sum(cov.ravel() > blo)) / len(cov.ravel())
yy.extend([nc, nc])
xx.extend([bhi, blo])
if ni > 0:
if nc != ni:
if nc > ni + 0.03:
# If there's room, label the histogram bin above, else below
plt.text((blo + bhi) / 2.,
ni,
'%.1f \%%' % (100. * ni),
ha='center',
va='bottom',
color='k')
else:
plt.text((blo + bhi) / 2.,
ni - 0.01,
'%.1f \%%' % (100. * ni),
ha='center',
va='top',
color='k')
plt.text((blo + bhi) / 2.,
nc,
'%.1f \%%' % (100. * nc),
ha='center',
va='bottom',
color='k')
plt.plot(xx, yy, 'k-')
plt.xlim(bins.min(), bins.max())
plt.ylim(0., 1.1)
plt.xlabel('Number of exposures')
plt.ylabel('Fraction of sky')
#plt.title('DECaLS tiling, %i pass%s' % (t.passnum, t.passnum > 1 and 'es' or ''))
#plt.savefig('hist-%02i.png' % it)
plt.savefig('hist-%02i.pdf' % it)
img,imghdr = fitsio.read(imgfn, ext=ext, header=True)
print('Read image', img.shape, img.dtype)
img = img.astype(np.float32)
H,W = img.shape
band = imghdr['FILTER'][0]
print('Band:', band)
photocal = CfhtLinearPhotoCal(imghdr, band)
headers = parse_head_file(headfn)
print('Read headers:', len(headers))
wcshdr = headers[ext-1]
wcshdr['IMAGEW'] = W
wcshdr['IMAGEH'] = H
wcs = wcs_pv2sip_hdr(wcshdr)
print('WCS pixel scale:', wcs.pixel_scale())
ok,xx,yy = wcs.radec2pixelxy(cat.ra, cat.dec)
print('Ext', ext, 'x range', int(xx.min()), int(xx.max()), 'y range', int(yy.min()), int(yy.max()))
cat.xx = xx
cat.yy = yy
# # Estimate per-pixel noise via Blanton's 5-pixel MAD
slice1 = (slice(0,-5,10),slice(0,-5,10))
slice2 = (slice(5,None,10),slice(5,None,10))
mad = np.median(np.abs(img[slice1] - img[slice2]).ravel())
sig1 = 1.4826 * mad / np.sqrt(2.)
print('sig1 estimate:', sig1)
inverr = np.ones_like(img) / sig1
def get_wcs(self, hdr):
from astrometry.util.util import wcs_pv2sip_hdr
# HACK -- convert TPV WCS header to SIP.
wcs = wcs_pv2sip_hdr(hdr)
#print('Converted WCS to', wcs)
return wcs
img, imghdr = fitsio.read(imgfn, ext=ext, header=True)
print('Read image', img.shape, img.dtype)
img = img.astype(np.float32)
H, W = img.shape
band = imghdr['FILTER'][0]
print('Band:', band)
photocal = CfhtLinearPhotoCal(imghdr, band)
headers = parse_head_file(headfn)
print('Read headers:', len(headers))
wcshdr = headers[ext - 1]
wcshdr['IMAGEW'] = W
wcshdr['IMAGEH'] = H
wcs = wcs_pv2sip_hdr(wcshdr)
print('WCS pixel scale:', wcs.pixel_scale())
ok, xx, yy = wcs.radec2pixelxy(cat.ra, cat.dec)
print('Ext', ext, 'x range', int(xx.min()), int(xx.max()), 'y range',
int(yy.min()), int(yy.max()))
cat.xx = xx
cat.yy = yy
# # Estimate per-pixel noise via Blanton's 5-pixel MAD
slice1 = (slice(0, -5, 10), slice(0, -5, 10))
slice2 = (slice(5, None, 10), slice(5, None, 10))
mad = np.median(np.abs(img[slice1] - img[slice2]).ravel())
sig1 = 1.4826 * mad / np.sqrt(2.)
print('sig1 estimate:', sig1)
def main(passnum, threads):
global udecs
global P3
global T
global tilewcs
global exps
global bad_expids
global tileid_to_depth
ps = PlotSequence('covfill-p%i' % passnum)
retirablefn = 'retirable-p%i.fits' % passnum
depthsfn = 'all-depths-p%i.fits' % passnum
if os.path.exists(retirablefn):
R = fits_table(retirablefn)
pcts = np.arange(0, 101)
target = 22.5
req_pcts = [0, 2, 2, 5, 5, 10, 10, 100]
req_depths = [0, 0, target-0.6, target-0.6,
target-0.3, target-0.3, target, target]
maglo, maghi = 21,23
plt.clf()
for depths in R.depths:
plt.plot(pcts, np.clip(depths, maglo, maghi), 'b-',
alpha=0.1)
plt.plot(req_pcts, np.clip(req_depths, maglo, maghi), 'k-',
lw=2, alpha=0.5)
plt.ylim(maglo, maghi)
plt.xlim(0, 100)
plt.xlabel('Coverage fraction')
plt.ylabel('Existing depth')
plt.suptitle('MzLS: retirable pass-%i tiles: %i' % (passnum,len(R)))
ps.savefig()
# Where are they on the sky?
T = fits_table('obstatus/mosaic-tiles_obstatus.fits')
T.cut(T.in_desi == 1)
T.cut(T.get('pass') <= 3)
tileid_to_index = np.zeros(T.tileid.max()+1, int)
tileid_to_index[T.tileid] = np.arange(len(T))
R.ra = T.ra [tileid_to_index[R.tileid]]
R.dec = T.dec[tileid_to_index[R.tileid]]
plt.clf()
plt.plot(T.ra, T.dec, 'k.', alpha=0.02)
I = (T.z_done == 1)
plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.1)
plt.plot(R.ra, R.dec, 'b.')
ax = [310,80,30,85]
#xl,xh = plt.xlim()
#plt.xlim(xh,xl)
plt.xlabel('RA (deg)')
plt.ylabel('Dec (deg)')
plt.title('MzLS: retirable pass-%i tiles' % passnum)
plt.axis(ax)
ps.savefig()
for p in [1,2,3]:
plt.clf()
plt.plot(T.ra, T.dec, 'k.', alpha=0.02)
#plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.1)
I = np.flatnonzero((T.get('pass') == p) * (T.z_done == 1)
* (T.z_depth > 1) * (T.z_depth < 30))
plt.scatter(T.ra[I], T.dec[I], c=T.z_depth[I],
vmin=20, vmax=23, s=4)
I = np.flatnonzero((T.get('pass') == p) * (T.z_done == 1)
* (T.z_depth == 30))
plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.5)
plt.colorbar()
plt.title('MzLS: Finished tiles in pass %i' % p)
plt.axis(ax)
ps.savefig()
sys.exit(0)
# NERSC: export LEGACY_SURVEY_DIR=/global/cscratch1/sd/dstn/dr4plus
# (dstn laptop: export LEGACY_SURVEY_DIR=~/legacypipe-dir-mzls/)
survey = LegacySurveyData()
ccds = survey.get_annotated_ccds()
print('Annotated CCDs:', len(ccds))
ccds.cut(ccds.camera == 'mosaic')
print(len(ccds), 'Mosaic')
print('Unique exposures:', len(np.unique(ccds.expnum)))
ccds.cut(ccds.exptime > 60)
print('Exptime > 60 sec:', len(ccds))
nccds = Counter(ccds.expnum)
for k,v in nccds.most_common():
if v <= 4:
break
print('Expnum', k, 'appears', v, 'times')
print('Tile pass numbers:', Counter(ccds.tilepass).most_common())
# Fix parsing of OBJECT field to tileid...
from obsbot import get_tile_id_from_name
tileids = []
for o in ccds.object:
tid = get_tile_id_from_name(o.strip())
if tid is None:
tid = 0
tileids.append(tid)
tileids = np.array(tileids)
print(len(np.unique(tileids)), 'unique tile ids in annotated file, from OBJECT')
print(len(np.unique(ccds.tileid)), 'unique tile ids in ann file from TILEID')
D = np.flatnonzero(tileids != ccds.tileid)
print(len(D), 'different tileids')
print('From OBJECT:', tileids[D])
print('From TILEID:', ccds.tileid[D])
ccds.tileid = tileids
T = fits_table('obstatus/mosaic-tiles_obstatus.fits')
f = open('obstatus/bad_expid.txt')
bad_expids = []
for line in f:
line = line.strip()
if len(line) == 0:
continue
if line[0] == '#':
continue
words = line.split()
try:
expnum = int(words[0])
except:
print('Skipping line:', line)
continue
bad_expids.append(expnum)
print('Read', len(bad_expids), 'bad exposure numbers')
# Update ccds.tilepass from ccds.tileid
tileidtopass = dict(zip(T.tileid, T.get('pass')))
tileidtoebv = dict(zip(T.tileid, T.ebv_med))
ccds.tilepass = np.array([tileidtopass.get(tileid, 0)
for tileid in ccds.tileid])
ccds.tileebv = np.array([tileidtoebv.get(tileid, 0)
for tileid in ccds.tileid])
print('Tile pass numbers after update:', Counter(ccds.tilepass).most_common())
e,I = np.unique(ccds.expnum, return_index=True)
exps = ccds[I]
#print('Object names,exptimes for tilepass==0:', zip(exps.object[exps.tilepass == 0], exps.exptime[exps.tilepass == 0]))
# Average the depth per exposure
for j,expnum in enumerate(exps.expnum):
I = np.flatnonzero(ccds.expnum == expnum)
if len(I) != 4:
print('Exposure', expnum, 'has', len(I), 'CCD entries')
continue
# Don't include zeros in computing average depths!
Igood = I[(ccds.galdepth[I] > 0) * (ccds.ccdzpt[I] < 30)]
if len(Igood) > 0:
exps.galdepth[j] = np.mean(ccds.galdepth[Igood])
else:
exps.galdepth[j] = 0.
# CCDs-table-based mapping from tileid to depth.
I = np.flatnonzero((exps.tilepass > 0) * (exps.galdepth > 0) *
(exps.tileid > 0))
tileid_to_depth = dict(zip(exps.tileid[I], exps.galdepth[I]))
T.cut(T.in_desi == 1)
T.cut(T.get('pass') <= 3)
# The tiles we'll examine
P3 = T[T.get('pass') == passnum]
print(len(P3), 'pass', passnum, 'and in DESI')
todo = P3[P3.z_done == 0]
print(len(todo), 'pass', passnum, 'tiles to do (Z_DONE=0)')
# Tiles with measured depths
T.cut((T.z_depth > 15) * (T.z_depth < 30))
print(len(T), 'tiles with measured depths')
# Passes other than 3... they ~ only barely overlap, so don't
# contribute significant depth.
#T.cut(T.get('pass') < 3)
udecs = np.unique(P3.dec)
print(len(udecs), 'unique Dec values in pass', passnum)
# Grab an arbitrary weight-map image and use that as a proxy!
wtfn = 'k4m_170501_112501_oow_zd_v1.fits.fz'
F = fitsio.FITS(wtfn)
tilewcs = []
# Read the WCS headers for each chip.
# They make the CRVAL be the boresight for all 4 chips... perfect!
# (because this means we can just set CRVAL = RA,Dec to shift the WCSes)
for i in range(1, len(F)):
hdr = F[i].read_header()
wcs = wcs_pv2sip_hdr(hdr)
tilewcs.append(wcs)
mp = multiproc(threads)
#args = [(i,t,udecs,P3,T,tilewcs,exps,bad_expids,tileid_to_depth)
args = [(i,t)
for i,t in enumerate(todo)]
thedepths = mp.map(one_tile, args)
alldepths = []
retirable = []
for arg,depths in zip(args, thedepths):
if depths is None:
continue
itile,tile = arg[:2]
target = 22.5
req_pcts = [0, 2, 2, 5, 5, 10, 10, 100]
req_depths = [0, 0, target-0.6, target-0.6,
target-0.3, target-0.3, target, target]
print(' Depths at 2, 5, and 10th percentile vs target:',
'%.2f' % (depths[2] - (target - 0.6)),
'%.2f' % (depths[5] - (target - 0.3)),
'%.2f' % (depths[10] - target))
alldepths.append((tile.tileid, depths))
if not ((depths[2] > target - 0.6) and
(depths[5] > target - 0.3) and
(depths[10] > target)):
continue
retirable.append((tile.tileid, depths))
#if len(retirable) == 10:
# break
# if ps.ploti >= 100:
# continue
#
# maglo, maghi = 21,23
#
# plt.clf()
# #plt.subplot(1,2,1)
# plt.subplot2grid((2,2), (0,0))
# plt.imshow(depth, interpolation='nearest', origin='lower',
# vmin=maglo, vmax=maghi)
# plt.colorbar(ticks=[np.arange(maglo, maghi+0.01, 0.5)])
# plt.xticks([]); plt.yticks([])
# #plt.subplot(1,2,2)
# plt.subplot2grid((2,2), (1,0))
# plt.imshow(nexp, interpolation='nearest', origin='lower',
# vmin=0, vmax=4)
# plt.colorbar(ticks=[0,1,2,3,4])
# plt.xticks([]); plt.yticks([])
#
# ax = plt.subplot2grid((2,2), (0,1), rowspan=2)
# plt.plot(req_pcts, np.clip(req_depths, maglo, maghi), 'k-',
# lw=2, alpha=0.5)
# plt.plot(pcts, np.clip(depths, maglo, maghi), 'b-')
# ax.yaxis.tick_right()
# plt.ylim(maglo, maghi)
# plt.xlim(0, 100)
# plt.xlabel('Coverage fraction')
# plt.ylabel('Existing depth')
# plt.suptitle('Tile %i' % tile.tileid)
# ps.savefig()
#if ps.ploti == 100:
# break
# print('Tiles that could be retired:')
# print('# Tileid 0th-percentile-extcorr-depth 1st-pctile 2nd-pctile ...')
# for tileid, depths in retirable:
# print(tileid, ' '.join(['%.3f' % d for d in depths]))
R = fits_table()
R.tileid = np.array ([t for t,d in alldepths])
R.depths = np.vstack([d for t,d in alldepths])
R.writeto(depthsfn)
if len(retirable):
R = fits_table()
R.tileid = np.array([t for t,d in retirable])
R.depths = np.vstack([d for t,d in retirable])
R.writeto(retirablefn)
else:
print('No tiles in pass', passnum, 'are retirable')
