def main():
survey = LegacySurveyData()
ccds = survey.get_ccds_readonly()
print(len(ccds), 'CCDs')
ccds = ccds[ccds.ccd_cuts == 0]
print(len(ccds), 'good CCDs')
# Find bricks touched by >=1 CCD
bricks = survey.get_bricks_readonly()
bricks = bricks[(bricks.dec > -20) * (bricks.dec < 35.)]
print(len(bricks), 'bricks in Dec range')
I, J, d = match_radec(bricks.ra,
bricks.dec,
ccds.ra,
ccds.dec,
0.5,
nearest=True)
bricks = bricks[I]
print(len(bricks), 'bricks')
bands = ['g', 'r', 'z']
nexps = {}
for b in bands:
ne = np.zeros(len(bricks), np.int16)
nexps[b] = ne
bricks.set('nexp_' + b, ne)
npix = {}
for b in bands:
n = np.zeros(len(bricks), np.int64)
npix[b] = n
bricks.set('npix_' + b, n)
for b in bands:
n = np.zeros(len(bricks), np.float32)
bricks.set('psfdepth_' + b, n)
args = enumerate(bricks)
mp = multiproc(8)
R = mp.map(one_brick, args)
for ibrick, res in enumerate(R):
if res is None:
continue
(npix, nexps, depths) = res
for band in bands:
bricks.get('npix_' + band)[ibrick] = npix[band]
bricks.get('nexp_' + band)[ibrick] = nexps[band]
bricks.get('psfdepth_' + band)[ibrick] = depths[band]
bricks.cut((bricks.nexp_g + bricks.nexp_r + bricks.nexp_z) > 0)
bricks.writeto('/global/cscratch1/sd/dstn/bricks-nexp.fits')
allstate = []
alltasks = []
# allra.append(ra)
# alldec.append(dec)
# allstate.append([state] * len(ra))
# alltasks.append(tasks)
ra = np.hstack(allra)
dec = np.hstack(alldec)
state = np.hstack(allstate)
tasks = np.hstack(alltasks)
# Match to actual table of bricks to get brickq.
survey = LegacySurveyData()
bricks = survey.get_bricks_readonly()
I,J,d = match_radec(ra, dec, bricks.ra, bricks.dec, 0.2, nearest=True)
print(len(ra), 'jobs')
print(len(I), 'matches')
ra = ra[I]
dec = dec[I]
state = state[I]
tasks = tasks[I]
brickq = bricks.brickq[J]
for q in [0,1,2,3]:
print()
print('Brickq', q)
plt.clf()
lp,lt = [],[]
def main():
# indir = '/global/cscratch1/sd/dstn/dr8test-1'
# name = 'dr8-test1'
# pretty = 'DR8 test1'
# indir = '/scratch1/scratchdirs/desiproc/dr8test002/'
# name = 'dr8-test2'
# pretty = 'DR8 test2 (outliers)'
# indir = '/scratch1/scratchdirs/desiproc/dr8test003/'
# name = 'dr8-test3'
# pretty = 'DR8 test3 (outliers)'
#
# indir = '/scratch1/scratchdirs/desiproc/dr8test004/'
# name = 'dr8-test4'
# pretty = 'DR8 test4 (large-galaxies)'
# indir = '/global/cscratch1/sd/dstn/dr8test005/'
# name = 'dr8-test5'
# pretty = 'DR8 test5 (trident)'
# indir = '/global/cscratch1/sd/dstn/dr8test006/'
# name = 'dr8-test6'
# pretty = 'DR8 test6 (sky)'
# indir = '/global/cscratch1/sd/dstn/dr8test007/'
# name = 'dr8-test7'
# pretty = 'DR8 test7 (outliers)'
#indir = '/global/cscratch1/sd/dstn/dr8test14/'
#name = 'dr8-test14'
#pretty = 'DR8 test14 (rc)'
#indir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8a/'
#name = 'dr8a'
#pretty = 'DR8a (rc)'
if False:
indir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-decam/'
name = 'dr8b-decam'
pretty = 'DR8b DECam'
survey_dir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-decam'
if True:
indir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-90prime-mosaic/'
name = 'dr8b-90p-mos'
pretty = 'DR8b BASS+MzLS'
survey_dir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-90prime-mosaic'
# ln -s /global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-decam/coadds-only/coadd/ .
sublayers = ['', '-model', '-resid']
subpretty = {'': ' images', '-model': ' models', '-resid': ' residuals'}
# survey_dir = '/global/cscratch1/sd/desiproc/dr7'
# sublayers = ['']
# subpretty = {'':' images'}
#survey_dir = '/global/cscratch1/sd/dstn/dr8-depthcut'
#survey_dir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8a/'
rsync = False
datadir = 'data'
survey = LegacySurveyData(survey_dir=survey_dir)
fn = 'map/test_layers.py'
txt = open(fn).read()
for x in sublayers:
txt = txt + '\n' + 'test_layers.append(("%s%s", "%s%s"))\n' % (
name, x, pretty, subpretty[x])
open(fn, 'wb').write(txt.encode())
print('Wrote', fn)
basedir = os.path.join(datadir, name)
if rsync:
cmd = 'rsync -LRarv %s/./{coadd/*/*/*-{image-,model-,ccds}*.fits*,tractor} %s/%s' % (
indir, datadir, name)
print(cmd)
os.system(cmd)
# ...?
cmd = 'rsync -Rarv %s/./{images,survey-ccds*.fits} %s/%s' % (
survey_dir, datadir, name)
print(cmd)
os.system(cmd)
else:
# symlink
if os.path.exists(basedir):
print('Not symlinking', indir, 'to', basedir, ': already exists!')
else:
os.makedirs(basedir)
for subdir in ['coadd', 'tractor']:
os.symlink(os.path.join(indir, subdir),
os.path.join(basedir, subdir),
target_is_directory=True)
for fn in ['images', 'calib']:
os.symlink(os.path.join(indir, subdir),
os.path.join(basedir, subdir),
target_is_directory=False)
for pat in ['survey-ccds-*']:
for fn in [
os.path.basename(f)
for f in glob(os.path.join(indir, pat))
]:
os.symlink(os.path.join(indir, subdir),
os.path.join(basedir, subdir),
target_is_directory=False)
allbricks = survey.get_bricks_readonly()
imagefns = glob(os.path.join(basedir, 'coadd', '*', '*',
'*-image-*.fits*'))
extraimagefns = glob(
os.path.join(basedir, 'extra-images', 'coadd', '*', '*',
'*-image-*.fits*'))
print('Image filenames:', len(imagefns), 'plus', len(extraimagefns),
'extras')
imagefns += extraimagefns
brickset = set()
for fn in imagefns:
dirs = fn.split('/')
brickname = dirs[-2]
brickset.add(brickname)
print(len(brickset), 'bricks found')
I, = np.nonzero([b in brickset for b in allbricks.brickname])
bricks = allbricks[I]
brickfn = os.path.join(basedir, 'survey-bricks.fits.gz')
bricks.writeto(brickfn)
print('Wrote', brickfn)
threads = 8
tharg = '--threads %i ' % threads
#tharg = ''
for x in sublayers:
cmd = 'python -u render-tiles.py --kind %s%s --bricks' % (name, x)
print(cmd)
os.system(cmd)
# images
for scale in range(1, 8):
cmd = 'python -u render-tiles.py --kind %s --scale --zoom %i %s' % (
name, scale, tharg)
print(cmd)
os.system(cmd)
# models
for scale in range(1, 8):
cmd = 'python -u render-tiles.py --kind %s-model --scale --zoom %i %s' % (
name, scale, tharg)
print(cmd)
os.system(cmd)
# resids
for scale in range(1, 8):
cmd = 'python -u render-tiles.py --kind %s-resid --scale --zoom %i %s' % (
name, scale, tharg)
print(cmd)
os.system(cmd)
for x in sublayers:
cmd = 'python -u render-tiles.py --kind %s%s --top' % (name, x)
print(cmd)
os.system(cmd)
def main():
indir = '/global/cscratch1/sd/dstn/dr8test-1'
name = 'dr8-test1'
pretty = 'DR8 test1'
sublayers = ['', '-model', '-resid']
subpretty = {'': ' images', '-model': ' models', '-resid': ' residuals'}
survey_dir = '/global/cscratch1/sd/desiproc/dr7'
datadir = 'data'
survey = LegacySurveyData(survey_dir=survey_dir)
fn = 'map/test_layers.py'
txt = open(fn).read()
for x in sublayers:
txt = txt + '\n' + 'test_layers.append(("%s%s", "%s%s"))\n' % (
name, x, pretty, subpretty[x])
open(fn, 'wb').write(txt.encode())
print('Wrote', fn)
cmd = 'rsync -LRarv %s/./{coadd/*/*/*-{image,model}-*.fits*,tractor} %s/%s' % (
indir, datadir, name)
print(cmd)
os.system(cmd)
basedir = os.path.join(datadir, name)
allbricks = survey.get_bricks_readonly()
imagefns = glob(os.path.join(basedir, 'coadd', '*', '*',
'*-image-*.fits*'))
print('Image filenames:', len(imagefns))
brickset = set()
for fn in imagefns:
dirs = fn.split('/')
brickname = dirs[-2]
brickset.add(brickname)
print(len(brickset), 'bricks found')
I, = np.nonzero([b in brickset for b in allbricks.brickname])
bricks = allbricks[I]
brickfn = os.path.join(basedir, 'survey-bricks.fits.gz')
bricks.writeto(brickfn)
print('Wrote', brickfn)
threads = 8
tharg = '--threads %i ' % threads
# images
for scale in range(1, 8):
cmd = 'python -u render-tiles.py --kind %s --scale --zoom %i %s' % (
name, scale, tharg)
print(cmd)
os.system(cmd)
# models
for scale in range(1, 8):
cmd = 'python -u render-tiles.py --kind %s-model --scale --zoom %i %s' % (
name, scale, tharg)
print(cmd)
os.system(cmd)
for x in sublayers:
cmd = 'python -u render-tiles.py --kind %s%s --top' % (name, x)
print(cmd)
os.system(cmd)
'''
After running forced_photom.py on a set of CCDs, this script merges
the results back into a catalog.
'''
from astrometry.util.fits import *
import numpy as np
from glob import glob
from collections import Counter
from legacypipe.survey import LegacySurveyData
fns = glob('forced/*/*/forced-*.fits')
F = merge_tables([fits_table(fn) for fn in fns])
dr6 = LegacySurveyData('/project/projectdirs/cosmo/data/legacysurvey/dr6')
B = dr6.get_bricks_readonly()
I = np.flatnonzero((B.ra1 < F.ra.max()) * (B.ra2 > F.ra.min()) * (B.dec1 < F.dec.max()) * (B.dec2 > F.dec.min()))
print(len(I), 'bricks')
T = merge_tables([fits_table(dr6.find_file('tractor', brick=B.brickname[i])) for i in I])
print(len(T), 'sources')
T.cut(T.brick_primary)
print(len(T), 'primary')
# map from F to T index
imap = dict([((b,o),i) for i,(b,o) in enumerate(zip(T.brickid, T.objid))])
F.tindex = np.array([imap[(b,o)] for b,o in zip(F.brickid, F.objid)])
assert(np.all(T.brickid[F.tindex] == F.brickid))
assert(np.all(T.objid[F.tindex] == F.objid))
fcols = 'apflux apflux_ivar camera expnum ccdname exptime flux flux_ivar fracflux mask mjd rchi2 x y brickid objid'.split()
def main():
import argparse
parser = argparse.ArgumentParser(
description='This script creates small self-contained data sets that '
'are useful for test cases of the pipeline codes.')
parser.add_argument('ccds', help='CCDs table describing region to grab')
parser.add_argument('outdir', help='Output directory name')
parser.add_argument('brick', help='Brick containing these images')
parser.add_argument('--survey-dir', type=str, default=None)
parser.add_argument('--cache-dir', type=str, default=None,
help='Directory to search for cached files')
parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image')
parser.add_argument('--wise-wcs-hdu', help='For WISE outputs, the HDU to read the WCS from in the file given by --wise.', type=int, default=0)
parser.add_argument('--fpack', action='store_true', default=False)
parser.add_argument('--gzip', action='store_true', default=False)
parser.add_argument('--pad', action='store_true', default=False,
help='Keep original image size, but zero out pixels outside ROI')
args = parser.parse_args()
v = 'SKY_TEMPLATE_DIR'
if v in os.environ:
del os.environ[v]
C = fits_table(args.ccds)
print(len(C), 'CCDs in', args.ccds)
C.camera = np.array([c.strip() for c in C.camera])
survey = LegacySurveyData(cache_dir=args.cache_dir, survey_dir=args.survey_dir)
if ',' in args.brick:
ra,dec = args.brick.split(',')
ra = float(ra)
dec = float(dec)
fakebricks = fits_table()
fakebricks.brickname = np.array([('custom-%06i%s%05i' %
(int(1000*ra), 'm' if dec < 0 else 'p',
int(1000*np.abs(dec))))])
fakebricks.ra = np.array([ra])
fakebricks.dec = np.array([dec])
bricks = fakebricks
outbricks = bricks
else:
bricks = survey.get_bricks_readonly()
outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])]
assert(len(outbricks) == 1)
outsurvey = LegacySurveyData(survey_dir = args.outdir)
trymakedirs(args.outdir)
outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz'))
targetwcs = wcs_for_brick(outbricks[0])
H,W = targetwcs.shape
tycho2fn = survey.find_file('tycho2')
kd = tree_open(tycho2fn, 'stars')
radius = 1.
rc,dc = targetwcs.radec_center()
I = tree_search_radec(kd, rc, dc, radius)
print(len(I), 'Tycho-2 stars within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (rc,dc))
# Read only the rows within range.
tycho = fits_table(tycho2fn, rows=I)
del kd
print('Read', len(tycho), 'Tycho-2 stars')
ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec)
#margin = 100
#tycho.cut(ok * (tx > -margin) * (tx < W+margin) *
# (ty > -margin) * (ty < H+margin))
print('Cut to', len(tycho), 'Tycho-2 stars within brick')
del ok,tx,ty
#tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz'))
f,tfn = tempfile.mkstemp(suffix='.fits')
os.close(f)
tycho.writeto(tfn)
outfn = os.path.join(args.outdir, 'tycho2.kd.fits')
cmd = 'startree -i %s -o %s -P -k -n stars -T' % (tfn, outfn)
print(cmd)
rtn = os.system(cmd)
assert(rtn == 0)
os.unlink(tfn)
from legacypipe.gaiacat import GaiaCatalog
gcat = GaiaCatalog()
# from ps1cat.py:
wcs = targetwcs
step=100.
margin=10.
# Grid the CCD in pixel space
W,H = wcs.get_width(), wcs.get_height()
xx,yy = np.meshgrid(
np.linspace(1-margin, W+margin, 2+int((W+2*margin)/step)),
np.linspace(1-margin, H+margin, 2+int((H+2*margin)/step)))
# Convert to RA,Dec and then to unique healpixes
ra,dec = wcs.pixelxy2radec(xx.ravel(), yy.ravel())
healpixes = set()
for r,d in zip(ra,dec):
healpixes.add(gcat.healpix_for_radec(r, d))
for hp in healpixes:
hpcat = gcat.get_healpix_catalog(hp)
ok,xx,yy = wcs.radec2pixelxy(hpcat.ra, hpcat.dec)
onccd = np.flatnonzero((xx >= 1.-margin) * (xx <= W+margin) *
(yy >= 1.-margin) * (yy <= H+margin))
hpcat.cut(onccd)
if len(hpcat):
outfn = os.path.join(args.outdir, 'gaia', 'chunk-%05d.fits' % hp)
trymakedirs(os.path.join(args.outdir, 'gaia'))
hpcat.writeto(outfn)
outccds = C.copy()
cols = outccds.get_columns()
for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1',
'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1',
'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver',
'psfplver' ]:
if c in cols:
outccds.delete_column(c)
outccds.image_hdu[:] = 1
# Convert to list to avoid truncating filenames
outccds.image_filename = [fn for fn in outccds.image_filename]
for iccd,ccd in enumerate(C):
decam = (ccd.camera.strip() == 'decam')
bok = (ccd.camera.strip() == '90prime')
im = survey.get_image_object(ccd)
print('Got', im)
if survey.cache_dir is not None:
im.check_for_cached_files(survey)
slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1))
psfkwargs = dict(pixPsf=True, gaussPsf=False, hybridPsf=False,
normalizePsf=False)
tim = im.get_tractor_image(slc, pixPsf=True,
subsky=False, nanomaggies=False,
no_remap_invvar=True, old_calibs_ok=True)
print('Tim:', tim.shape)
psfrow = psfhdr = None
if args.pad:
psf = im.read_psf_model(0, 0, w=im.width, h=im.height, **psfkwargs)
psfex = psf.psfex
else:
psf = tim.getPsf()
psfex = psf.psfex
# Did the PSF model come from a merged file?
for fn in [im.merged_psffn, im.psffn] + im.old_merged_psffns:
if not os.path.exists(fn):
continue
T = fits_table(fn)
I, = np.nonzero((T.expnum == im.expnum) *
np.array([c.strip() == im.ccdname for c in T.ccdname]))
if len(I) != 1:
continue
psfrow = T[I]
x0 = ccd.ccd_x0
y0 = ccd.ccd_y0
psfrow.polzero1[0] -= x0
psfrow.polzero2[0] -= y0
#psfhdr = fitsio.read_header(im.merged_psffn)
break
psfex.fwhm = tim.psf_fwhm
#### HACK
#psfrow = None
assert(psfrow is not None)
if psfrow is not None:
print('PSF row:', psfrow)
#else:
# print('PSF:', psf)
# print('PsfEx:', psfex)
skyrow = skyhdr = None
if args.pad:
primhdr = fitsio.read_header(im.imgfn)
imghdr = fitsio.read_header(im.imgfn, hdu=im.hdu)
sky = im.read_sky_model(splinesky=True, primhdr=primhdr, imghdr=imghdr)
#skyhdr = fitsio.read_header(im.splineskyfn)
#msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True)
else:
sky = tim.getSky()
# Did the sky model come from a merged file?
#msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True)
print('merged skyfn:', im.merged_skyfn)
print('single skyfn:', im.skyfn)
print('old merged skyfns:', im.old_merged_skyfns)
for fn in [im.merged_skyfn, im.skyfn] + im.old_merged_skyfns:
if not os.path.exists(fn):
continue
T = fits_table(fn)
I, = np.nonzero((T.expnum == im.expnum) *
np.array([c.strip() == im.ccdname for c in T.ccdname]))
skyrow = T[I]
skyrow.x0[0] = ccd.ccd_x0
skyrow.y0[0] = ccd.ccd_y0
# s_med = skyrow.sky_med[0]
# s_john = skyrow.sky_john[0]
# skyhdr = fitsio.read_header(fn)
assert(skyrow is not None)
### HACK
#skyrow = None
if skyrow is not None:
print('Sky row:', skyrow)
else:
print('Sky:', sky)
# Output filename format:
fn = ccd.image_filename.strip()
ccd.image_filename = os.path.join(os.path.dirname(fn),
'%s.%s.fits' % (os.path.basename(fn).split('.')[0], ccd.ccdname.strip()))
outim = outsurvey.get_image_object(ccd)
print('Output image:', outim)
print('Image filename:', outim.imgfn)
trymakedirs(outim.imgfn, dir=True)
imgdata = tim.getImage()
ivdata = tim.getInvvar()
# Since we remap DQ codes (always with Mosaic and Bok, sometimes with DECam),
# re-read from the FITS file rather than using tim.dq.
print('Reading data quality from', im.dqfn, 'hdu', im.hdu)
dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice)
print('Tim shape:', tim.shape, 'Slice', tim.slice)
print('image shape:', imgdata.shape, 'iv', ivdata.shape, 'DQ', dqdata.shape)
from collections import Counter
dqvals = Counter(dqdata.ravel())
print('DQ pixel counts:')
for k,n in dqvals.most_common():
print(' 0x%x' % k, ':', n)
if args.pad:
# Create zero image of full size, copy in data.
fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype)
fullsize[slc] = imgdata
imgdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype)
fullsize[slc] = dqdata
dqdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype)
fullsize[slc] = ivdata
ivdata = fullsize
else:
# Adjust the header WCS by x0,y0
crpix1 = tim.hdr['CRPIX1']
crpix2 = tim.hdr['CRPIX2']
tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0
tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0
# Add image extension to filename
# fitsio doesn't compress .fz by default, so drop .fz suffix
#outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits')
if args.gzip:
outim.imgfn = outim.imgfn.replace('.fits', '.fits.gz')
#outim.wtfn = outim.wtfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.wtfn = outim.wtfn.replace('.fits.fz', '.fits')
if args.gzip:
outim.wtfn = outim.wtfn.replace('.fits', '.fits.gz')
if outim.dqfn is not None:
#outim.dqfn = outim.dqfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.dqfn = outim.dqfn.replace('.fits.fz', '.fits')
if args.gzip:
outim.dqfn = outim.dqfn.replace('.fits', '.fits.gz')
if bok:
outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname)
ccdfn = outim.imgfn
ccdfn = ccdfn.replace(outsurvey.get_image_dir(), '')
if ccdfn.startswith('/'):
ccdfn = ccdfn[1:]
outccds.image_filename[iccd] = ccdfn
print('Changed output filenames to:')
print(outim.imgfn)
print(outim.dqfn)
ofn = outim.imgfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fits = fitsio.FITS(ofn, 'rw', clobber=True)
fits.write(None, header=tim.primhdr)
fits.write(imgdata, header=tim.hdr, extname=ccd.ccdname)
fits.close()
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
h,w = tim.shape
if not args.pad:
outccds.width[iccd] = w
outccds.height[iccd] = h
outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0
outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0
wcs = Tan(*[float(x) for x in
[ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2,
ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]])
if args.pad:
subwcs = wcs
else:
subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h)
outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center()
print('Weight filename:', outim.wtfn)
wfn = outim.wtfn
trymakedirs(wfn, dir=True)
ofn = wfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fits = fitsio.FITS(ofn, 'rw', clobber=True)
fits.write(None, header=tim.primhdr)
fits.write(ivdata, header=tim.hdr, extname=ccd.ccdname)
fits.close()
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
if outim.dqfn is not None:
print('DQ filename', outim.dqfn)
trymakedirs(outim.dqfn, dir=True)
ofn = outim.dqfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fits = fitsio.FITS(ofn, 'rw', clobber=True)
fits.write(None, header=tim.primhdr)
fits.write(dqdata, header=tim.hdr, extname=ccd.ccdname)
fits.close()
if args.fpack:
cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
psfout = outim.psffn
#if psfrow:
# psfout = outim.merged_psffn
print('PSF output filename:', psfout)
trymakedirs(psfout, dir=True)
if psfrow:
psfrow.writeto(psfout, primhdr=psfhdr)
else:
print('Writing PsfEx:', psfout)
psfex.writeto(psfout)
# update header
F = fitsio.FITS(psfout, 'rw')
F[0].write_keys([dict(name='EXPNUM', value=ccd.expnum),
dict(name='PLVER', value=psf.plver),
dict(name='PROCDATE', value=psf.procdate),
dict(name='PLPROCID', value=psf.plprocid),])
F.close()
skyout = outim.skyfn
#if skyrow:
# skyout = outim.merged_splineskyfn
print('Sky output filename:', skyout)
trymakedirs(skyout, dir=True)
if skyrow is not None:
skyrow.writeto(skyout, primhdr=skyhdr)
else:
primhdr = fitsio.FITSHDR()
primhdr['PLVER'] = sky.plver
primhdr['PLPROCID'] = sky.plprocid
primhdr['PROCDATE'] = sky.procdate
primhdr['EXPNUM'] = ccd.expnum
primhdr['IMGDSUM'] = sky.datasum
primhdr['S_MED'] = s_med
primhdr['S_JOHN'] = s_john
sky.write_fits(skyout, primhdr=primhdr)
# HACK -- check result immediately.
outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))
outsurvey.ccds = None
outC = outsurvey.get_ccds_readonly()
occd = outC[iccd]
outim = outsurvey.get_image_object(occd)
print('Got output image:', outim)
otim = outim.get_tractor_image(pixPsf=True,
hybridPsf=True, old_calibs_ok=True)
print('Got output tim:', otim)
outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))
# WISE
if args.wise is not None:
from wise.forcedphot import unwise_tiles_touching_wcs
from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs,
get_unwise_tractor_image, get_unwise_tile_dir)
# Read WCS...
print('Reading TAN wcs header from', args.wise, 'HDU', args.wise_wcs_hdu)
targetwcs = Tan(args.wise, args.wise_wcs_hdu)
tiles = unwise_tiles_touching_wcs(targetwcs)
print('Cut to', len(tiles), 'unWISE tiles')
H,W = targetwcs.shape
r,d = targetwcs.pixelxy2radec(np.array([1, W, W/2, W/2]),
np.array([H/2, H/2, 1, H ]))
roiradec = [r[0], r[1], d[2], d[3]]
unwise_dir = os.environ['UNWISE_COADDS_DIR']
wise_out = os.path.join(args.outdir, 'images', 'unwise')
print('Will write WISE outputs to', wise_out)
unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR']
wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr')
print('Will write WISE time-resolved outputs to', wise_tr_out)
trymakedirs(wise_tr_out)
W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_atlas.fits'))
print('Read', len(W), 'time-resolved WISE coadd tiles')
W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id]))
print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth')
# Write the time-resolved index subset.
W.writeto(os.path.join(wise_tr_out, 'time_resolved_atlas.fits'))
# this ought to be enough for anyone =)
_,Nepochs = W.epoch_bitmask.shape
print('N epochs in time-resolved atlas:', Nepochs)
wisedata = []
# full depth
for band in [1,2,3,4]:
wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band, True))
# time-resolved
for band in [1,2]:
# W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2)
bitmask = (1 << (band-1))
for e in range(Nepochs):
# Which tiles have images for this epoch?
I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask)
if len(I) == 0:
continue
print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I])
edir = os.path.join(unwise_tr_dir, 'e%03i' % e)
eoutdir = os.path.join(wise_tr_out, 'e%03i' % e)
wisedata.append((edir, eoutdir, tiles.coadd_id[I], band, False))
wrote_masks = set()
model_dir = os.environ.get('UNWISE_MODEL_SKY_DIR')
if model_dir is not None:
model_dir_out = os.path.join(args.outdir, 'images', 'unwise-mod')
trymakedirs(model_dir_out)
for indir, outdir, tiles, band, fulldepth in wisedata:
for tile in tiles:
wanyband = 'w'
tim = get_unwise_tractor_image(indir, tile, band,
bandname=wanyband, roiradecbox=roiradec)
print('Got unWISE tim', tim)
print(tim.shape)
if model_dir is not None and fulldepth and band in [1,2]:
print('ROI', tim.roi)
#0387p575.1.mod.fits
fn = '%s.%i.mod.fits' % (tile, band)
print('Filename', fn)
F = fitsio.FITS(os.path.join(model_dir, fn))
x0,x1,y0,y1 = tim.roi
slc = slice(y0,y1),slice(x0,x1)
phdr = F[0].read_header()
outfn = os.path.join(model_dir_out, fn)
for e,extname in [(1,'MODEL'), (2,'SKY')]:
pix = F[e][slc]
hdr = F[e].read_header()
crpix1 = hdr['CRPIX1']
crpix2 = hdr['CRPIX2']
hdr['CRPIX1'] -= x0
hdr['CRPIX2'] -= y0
#print('mod', mod)
#print('Model', mod.shape)
if e == 1:
fitsio.write(outfn, None, clobber=True, header=phdr)
fitsio.write(outfn, pix, header=hdr, extname=extname)
print('Wrote', outfn)
thisdir = get_unwise_tile_dir(outdir, tile)
print('Directory for this WISE tile:', thisdir)
base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band))
print('Base filename:', base)
masked = True
mu = 'm' if masked else 'u'
imfn = base + 'img-%s.fits' % mu
ivfn = base + 'invvar-%s.fits.gz' % mu
nifn = base + 'n-%s.fits.gz' % mu
nufn = base + 'n-u.fits.gz'
#print('WISE image header:', tim.hdr)
# Adjust the header WCS by x0,y0
wcs = tim.wcs.wcs
tim.hdr['CRPIX1'] = wcs.crpix[0]
tim.hdr['CRPIX2'] = wcs.crpix[1]
H,W = tim.shape
tim.hdr['IMAGEW'] = W
tim.hdr['IMAGEH'] = H
print('WCS:', wcs)
print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2'])
trymakedirs(imfn, dir=True)
fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True)
print('Wrote', imfn)
fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True)
print('Wrote', ivfn)
fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True)
print('Wrote', nifn)
fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True)
print('Wrote', nufn)
if not (indir,tile) in wrote_masks:
print('Looking for mask file for', indir, tile)
# record that we tried this dir/tile combo
wrote_masks.add((indir,tile))
for idir in indir.split(':'):
tdir = get_unwise_tile_dir(idir, tile)
maskfn = 'unwise-%s-msk.fits.gz' % tile
fn = os.path.join(tdir, maskfn)
print('Mask file:', fn)
if os.path.exists(fn):
print('Reading', fn)
(x0,x1,y0,y1) = tim.roi
roislice = (slice(y0,y1), slice(x0,x1))
F = fitsio.FITS(fn)[0]
hdr = F.read_header()
M = F[roislice]
outfn = os.path.join(thisdir, maskfn)
fitsio.write(outfn, M, header=tim.hdr, clobber=True)
print('Wrote', outfn)
break
outC = outsurvey.get_ccds_readonly()
for iccd,ccd in enumerate(outC):
outim = outsurvey.get_image_object(ccd)
print('Got output image:', outim)
otim = outim.get_tractor_image(pixPsf=True,
hybridPsf=True, old_calibs_ok=True)
print('Got output tim:', otim)
def main():
# indir = '/global/cscratch1/sd/dstn/dr8test-1'
# name = 'dr8-test1'
# pretty = 'DR8 test1'
# indir = '/scratch1/scratchdirs/desiproc/dr8test002/'
# name = 'dr8-test2'
# pretty = 'DR8 test2 (outliers)'
# indir = '/scratch1/scratchdirs/desiproc/dr8test003/'
# name = 'dr8-test3'
# pretty = 'DR8 test3 (outliers)'
#
# indir = '/scratch1/scratchdirs/desiproc/dr8test004/'
# name = 'dr8-test4'
# pretty = 'DR8 test4 (large-galaxies)'
# indir = '/global/cscratch1/sd/dstn/dr8test005/'
# name = 'dr8-test5'
# pretty = 'DR8 test5 (trident)'
# indir = '/global/cscratch1/sd/dstn/dr8test006/'
# name = 'dr8-test6'
# pretty = 'DR8 test6 (sky)'
# indir = '/global/cscratch1/sd/dstn/dr8test007/'
# name = 'dr8-test7'
# pretty = 'DR8 test7 (outliers)'
#indir = '/global/cscratch1/sd/dstn/dr8test14/'
#name = 'dr8-test14'
#pretty = 'DR8 test14 (rc)'
#indir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8a/'
#name = 'dr8a'
#pretty = 'DR8a (rc)'
# indir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-decam/'
# name = 'dr8b-decam'
# pretty = 'DR8b DECam'
# survey_dir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-decam'
# indir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-90prime-mosaic/'
# name = 'dr8b-90p-mos'
# pretty = 'DR8b BASS+MzLS'
# survey_dir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-90prime-mosaic'
rsync = False
#indir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8c/90prime-mosaic/'
#name = 'dr8c-90p-mos'
#pretty = 'DR8c BASS+MzLS'
survey_dir = '/global/cscratch1/sd/landriau/dr8'
# ln -s /global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-decam/coadds-only/coadd/ .
#indir = '/scratch1/scratchdirs/desiproc/dr8/decam/'
indir = 'data/dr8c-decam'
name = 'dr8c-decam'
pretty = 'DR8c DECam'
#rsync = True
indir = 'data/dr8i-decam'
name = 'dr8i-decam'
pretty = 'DR8i DECam'
indir = 'data/dr8i-90p-mos'
name = 'dr8i-90p-mos'
pretty = 'DR8i MzLS+BASS'
sublayers = ['', '-model', '-resid']
subpretty = {'':' images', '-model':' models', '-resid':' residuals'}
# #indir = '/global/cscratch1/sd/ziyaoz/dr9c/'
# #indir = '/global/cscratch1/sd/dstn/dr9c-fpack/'
# #rsync = True
# indir = 'data/dr9c'
# name = 'dr9c'
# pretty = 'DR9c'
# survey_dir = indir
#
# indir = '/global/cscratch1/sd/ziyaoz/dr9d-south/'
# #rsync = True
# name = 'dr9d-south'
# pretty = 'DR9d south'
# survey_dir = indir
# indir = '/global/cscratch1/sd/ziyaoz/dr9d-north/'
# #rsync = True
# name = 'dr9d-north'
# pretty = 'DR9d north'
# survey_dir = indir
# code runs:
# rsync -LRarv /global/cscratch1/sd/ziyaoz/dr9d-south//./{coadd/*/*/*-{image-,model-,ccds}*.fits*,tractor} data/dr9d-south
# add my image-coadds:
# rsync -LRarv /global/cscratch1/sd/dstn/dr9d-coadds/./coadd/*/*/*-{image-,ccds}*.fits* data/dr9d-south
# survey_dir = '/global/cscratch1/sd/desiproc/dr7'
# sublayers = ['']
# subpretty = {'':' images'}
#survey_dir = '/global/cscratch1/sd/dstn/dr8-depthcut'
#survey_dir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8a/'
rsync = True
#survey_dir = '/global/cfs/cdirs/cosmo/work/legacysurv
side = 'north'
#side = 'south'
survey_dir = '/global/cscratch1/sd/ziyaoz/dr9e4/%s' % side
indir = survey_dir
name = 'dr9sv-%s' % side
pretty = 'DR9-SV %s' % side
rsync = False
survey_dir = '/global/cfs/cdirs/cosmo/work/legacysurvey/dr9'
indir = '/global/cscratch1/sd/dstn/fornax'
name = 'fornax'
pretty = 'Fornax'
rsync = True
survey_dir = '/global/cfs/cdirs/cosmo/work/legacysurvey/dr9'
indir = '/global/cscratch1/sd/dstn/dr9.2'
name = 'dr9-test-9.2'
pretty = 'DR9.2 test'
survey_dir = '/global/cscratch1/sd/ziyaoz/dr9j/south'
indir = survey_dir
name = 'dr9j-south'
pretty = 'DR9j south'
rsync = False
if True:
#indir = '/global/cscratch1/sd/ziyaoz/dr9m/north/'
indir = '/global/cfs/cdirs/cosmo/work/legacysurvey/dr9m/north'
name = 'dr9m-north'
pretty = 'DR9m-north'
survey_dir = '/global/cfs/cdirs/cosmo/work/legacysurvey/dr9m'
if False:
#indir = '/global/cscratch1/sd/ziyaoz/dr9m/south/'
indir = '/global/cfs/cdirs/cosmo/work/legacysurvey/dr9m/south'
#name = 'dr9m-south'
name = 'ls-dr9-south'
pretty = 'DR9m-south'
survey_dir = '/global/cfs/cdirs/cosmo/work/legacysurvey/dr9m'
update = True
#update = False
queue = False
# rsync = True
# update = False
# queue = False
# indir = '/global/cscratch1/sd/dstn/m33-2/south/'
# name = 'dr9-m33'
# pretty = 'DR9m-M33'
# survey_dir = '/global/cfs/cdirs/cosmo/work/legacysurvey/dr9m'
datadir = 'data'
survey = LegacySurveyData(survey_dir=survey_dir)
allbricks = survey.get_bricks_readonly()
basedir = os.path.join(datadir, name)
from astrometry.util.fits import fits_table
if update:
old_bricks_dir = None
for i in range(100):
old_bricks_dir = os.path.join(basedir, 'old-bricks-%i' % i)
if os.path.exists(old_bricks_dir):
#print('exists:', old_bricks_dir)
continue
os.makedirs(old_bricks_dir)
print('Created', old_bricks_dir)
break
if old_bricks_dir is None:
sys.exit(-1)
old_bricks = []
for scale in range(8):
if scale == 0:
fn = 'survey-bricks.fits.gz'
else:
fn = 'survey-bricks-%i.fits.gz' % scale
pathfn = os.path.join(basedir, fn)
if os.path.exists(pathfn):
T = fits_table(pathfn)
print('Read', len(T), 'old bricks from', pathfn)
old_bricks.append(T)
#### os.rename(pathfn, os.path.join(old_bricks_dir, fn))
if rsync:
for sub in ['image-g', 'image-r', 'image-z', 'model-g', 'model-r', 'model-z', 'ccds']:
cmd = 'rsync -LRarv %s/./coadd/*/*/*-%s*.fits* %s/%s' % (indir, sub, datadir, name)
print(cmd)
os.system(cmd)
cmd = 'rsync -LRarv %s/./tractor %s/%s' % (indir, datadir, name)
print(cmd)
os.system(cmd)
# cmd = 'rsync -LRarv %s/./{coadd/*/*/*-{image-,model-,ccds}*.fits*,tractor} %s/%s' % (indir, datadir, name)
# print(cmd)
# os.system(cmd)
# ...?
cmd = 'rsync -Rarv %s/./{images,survey-ccds*.fits} %s/%s' % (survey_dir, datadir, name)
print(cmd)
os.system(cmd)
else:
# symlink
if os.path.exists(basedir):
print('Not symlinking', indir, 'to', basedir, ': already exists!')
else:
os.makedirs(basedir)
for subdir in ['coadd', 'tractor']:
os.symlink(os.path.join(indir, subdir), os.path.join(basedir, subdir), target_is_directory=True)
for subdir in ['images', 'calib']:
os.symlink(os.path.join(indir, subdir), os.path.join(basedir, subdir), target_is_directory=True)
for pat in ['survey-ccds-*']:
fns = glob(os.path.join(indir, pat))
print('fns', fns)
for fn in [os.path.basename(f) for f in fns]:
print('symlink', os.path.join(indir, subdir), os.path.join(basedir, subdir))
os.symlink(os.path.join(indir, fn), os.path.join(basedir, fn), target_is_directory=False)
# Find new available bricks
print('Searching for new extra-image files...')
extraimagefns = glob(os.path.join(basedir, 'extra-images', 'coadd', '*', '*', '*-image-*.fits*'))
print('Found', len(extraimagefns), 'extra images')
# Update all bricks in extra-images...
if update:
brickset = set()
# Read list of new bricks
f = open('bricks.txt')
for line in f.readlines():
brickname = line.strip()
brickset.add(brickname)
# for fn in extraimagefns:
# dirs = fn.split('/')
# brickname = dirs[-2]
# brickset.add(brickname)
print(len(brickset), 'bricks found')
I, = np.nonzero([b in brickset for b in allbricks.brickname])
bricks = allbricks[I]
# Find tiles that overlap each brick.
if True:
ii = 0
for zoom in range(6, 15):
zoomscale = 2.**zoom
W,H = 256,256
xyset = set()
for brick in bricks:
x1 = int(np.floor((360. - brick.ra2)/360. * zoomscale))
x2 = int(np.floor((360. - brick.ra1)/360. * zoomscale))
y1 = int(zoomscale * 1./(2.*np.pi) * (np.pi - np.log(np.tan(np.pi/4. + np.deg2rad(brick.dec2)/2.))))
y2 = int(zoomscale * 1./(2.*np.pi) * (np.pi - np.log(np.tan(np.pi/4. + np.deg2rad(brick.dec1)/2.))))
#print('Brick', brick.brickname, 'RA,Dec', brick.ra, brick.dec, ': zoom', zoom, 'x range', x1, x2, 'y range', y1, y2)
#print('# brick ', brick.brickname)
for x in range(x1, x2+1):
for y in range(y1, y2+1):
xyset.add((x,y))
xyset = list(xyset)
xyset.sort()
for x,y in xyset:
#print('%i/%i/%i.jpg' % (zoom, x, y))
#print('python render-tiles.py --kind ls-dr9-north -z %i -x %i -y %i --ignore > /dev/null 2>&1 &' % (zoom, x, y))
# cat update3.txt | xargs -n 6 -P 32 python render-tiles.py --kind ls-dr9-north --ignore
print('-z %i -x %i -y %i' % (zoom, x, y))
# ii += 1
# if ii % 32 == 0:
# pass
# #print('wait')
#print('wait')
#delete_scaled_images(name, old_bricks, bricks)
sys.exit(0)
print('Searching for new coadd image files...')
imagefns = glob(os.path.join(basedir, 'coadd', '*', '*', '*-image-*.fits*'))
print('Image filenames:', len(imagefns), 'plus', len(extraimagefns), 'extras')
imagefns += extraimagefns
brickset = set()
for fn in imagefns:
dirs = fn.split('/')
brickname = dirs[-2]
brickset.add(brickname)
print(len(brickset), 'bricks found')
I, = np.nonzero([b in brickset for b in allbricks.brickname])
bricks = allbricks[I]
brickfn = os.path.join(basedir, 'survey-bricks.fits.gz')
bricks.writeto(brickfn)
print('Wrote', brickfn)
for x in sublayers:
cmd = 'python3 -u render-tiles.py --kind %s%s --bricks' % (name, x)
print(cmd)
os.system(cmd)
if update:
# Find and remove existing scaled images touching new bricks.
old = old_bricks[0]
old_names = set([str(b) for b in old.brickname])
I_new = np.array([i for i,b in enumerate(bricks.brickname)
if not str(b) in old_names])
# Newly added bricks
new_bricks = bricks[I_new]
print('Added', len(new_bricks), 'bricks')
delete_scaled_images(name, old_bricks, new_bricks)
fn = 'map/test_layers.py'
txt = open(fn).read()
for x in sublayers:
txt = txt + '\n' + 'test_layers.append(("%s%s", "%s%s"))\n' % (name, x, pretty, subpretty[x])
open(fn, 'wb').write(txt.encode())
print('Wrote', fn)
threads = 32
tharg = '--threads %i ' % threads
#tharg = ''
if queue:
# from map.views import get_layer
# imglayer = get_layer(name)
# modlayer = get_layer(name + '-model')
ras = np.linspace(0, 360, 361)
for scale in range(1,8):
#for layer,layerobj in [(name,imglayer), (name+'-model',modlayer)]:
for layer in [name, name+'-model']:
for ralo,rahi in zip(ras, ras[1:]):
cmd = 'python3 -u render-tiles.py --kind %s --scale --zoom %i --minra %f --maxra %f' % (layer, scale, ralo, rahi)
print(cmd)
return
# images
for scale in range(1,8):
cmd = 'python3 -u render-tiles.py --kind %s --scale --zoom %i %s' % (name, scale, tharg)
print(cmd)
os.system(cmd)
# models
for scale in range(1,8):
cmd = 'python3 -u render-tiles.py --kind %s-model --scale --zoom %i %s' % (name, scale, tharg)
print(cmd)
os.system(cmd)
# resids
for scale in range(1,8):
cmd = 'python3 -u render-tiles.py --kind %s-resid --scale --zoom %i %s' % (name, scale, tharg)
print(cmd)
os.system(cmd)
for x in sublayers:
cmd = 'python3 -u render-tiles.py --kind %s%s --top' % (name, x)
print(cmd)
os.system(cmd)
def main():
import argparse
parser = argparse.ArgumentParser(
description='This script creates small self-contained data sets that '
'are useful for test cases of the pipeline codes.')
parser.add_argument('ccds', help='CCDs table describing region to grab')
parser.add_argument('outdir', help='Output directory name')
parser.add_argument('brick', help='Brick containing these images')
parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image')
parser.add_argument('--fpack', action='store_true', default=False)
parser.add_argument('--pad', action='store_true', default=False,
help='Keep original image size, but zero out pixels outside ROI')
args = parser.parse_args()
C = fits_table(args.ccds)
print(len(C), 'CCDs in', args.ccds)
C.camera = np.array([c.strip() for c in C.camera])
survey = LegacySurveyData()
bricks = survey.get_bricks_readonly()
outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])]
assert(len(outbricks) == 1)
outsurvey = LegacySurveyData(survey_dir = args.outdir)
trymakedirs(args.outdir)
outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz'))
targetwcs = wcs_for_brick(outbricks[0])
H,W = targetwcs.shape
tycho = fits_table(os.path.join(survey.get_survey_dir(), 'tycho2.fits.gz'))
print('Read', len(tycho), 'Tycho-2 stars')
ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec)
margin = 100
tycho.cut(ok * (tx > -margin) * (tx < W+margin) *
(ty > -margin) * (ty < H+margin))
print('Cut to', len(tycho), 'Tycho-2 stars within brick')
del ok,tx,ty
tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz'))
outccds = C.copy()
for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1',
'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1',
'plver', 'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver',
'psfplver' ]:
outccds.delete_column(c)
outccds.image_hdu[:] = 1
# Convert to list to avoid truncating filenames
outccds.image_filename = [fn for fn in outccds.image_filename]
for iccd,ccd in enumerate(C):
decam = (ccd.camera.strip() == 'decam')
bok = (ccd.camera.strip() == '90prime')
im = survey.get_image_object(ccd)
print('Got', im)
slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1))
tim = im.get_tractor_image(slc, pixPsf=True, splinesky=True,
subsky=False, nanomaggies=False)
print('Tim:', tim.shape)
psf = tim.getPsf()
print('PSF:', psf)
psfex = psf.psfex
print('PsfEx:', psfex)
outim = outsurvey.get_image_object(ccd)
print('Output image:', outim)
print('Image filename:', outim.imgfn)
trymakedirs(outim.imgfn, dir=True)
imgdata = tim.getImage()
dqdata = tim.dq
if decam:
# DECam-specific code remaps the DQ codes... re-read raw
print('Reading data quality from', im.dqfn, 'hdu', im.hdu)
dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice)
ivdata = tim.getInvvar()
if args.pad:
# Create zero image of full size, copy in data.
fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype)
fullsize[slc] = imgdata
imgdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype)
fullsize[slc] = dqdata
dqdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype)
fullsize[slc] = ivdata
ivdata = fullsize
else:
# Adjust the header WCS by x0,y0
crpix1 = tim.hdr['CRPIX1']
crpix2 = tim.hdr['CRPIX2']
tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0
tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0
# Add image extension to filename
# fitsio doesn't compress .fz by default, so drop .fz suffix
outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits')
# if bok:
# outim.whtfn = outim.whtfn .replace('.wht.fits', '-%s.wht.fits' % im.ccdname)
# if not args.fpack:
# outim.whtfn = outim.whtfn .replace('.fits.fz', '.fits')
# else:
if True:
outim.wtfn = outim.wtfn .replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.wtfn = outim.wtfn .replace('.fits.fz', '.fits')
if outim.dqfn is not None:
outim.dqfn = outim.dqfn .replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.dqfn = outim.dqfn .replace('.fits.fz', '.fits')
if bok:
outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname)
ccdfn = outim.imgfn
ccdfn = ccdfn.replace(outsurvey.get_image_dir(),'')
if ccdfn.startswith('/'):
ccdfn = ccdfn[1:]
outccds.image_filename[iccd] = ccdfn
print('Changed output filenames to:')
print(outim.imgfn)
print(outim.dqfn)
ofn = outim.imgfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fitsio.write(ofn, None, header=tim.primhdr, clobber=True)
fitsio.write(ofn, imgdata, header=tim.hdr, extname=ccd.ccdname)
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
h,w = tim.shape
if not args.pad:
outccds.width[iccd] = w
outccds.height[iccd] = h
outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0
outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0
wcs = Tan(*[float(x) for x in
[ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2,
ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]])
if args.pad:
subwcs = wcs
else:
subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h)
outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center()
#if not bok:
if True:
print('Weight filename:', outim.wtfn)
wfn = outim.wtfn
# else:
# print('Weight filename:', outim.whtfn)
# wfn = outim.whtfn
trymakedirs(wfn, dir=True)
ofn = wfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fitsio.write(ofn, None, header=tim.primhdr, clobber=True)
fitsio.write(ofn, ivdata, header=tim.hdr, extname=ccd.ccdname)
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
if outim.dqfn is not None:
print('DQ filename', outim.dqfn)
trymakedirs(outim.dqfn, dir=True)
ofn = outim.dqfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fitsio.write(ofn, None, header=tim.primhdr, clobber=True)
fitsio.write(ofn, dqdata, header=tim.hdr, extname=ccd.ccdname)
if args.fpack:
cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
print('PSF filename:', outim.psffn)
trymakedirs(outim.psffn, dir=True)
psfex.writeto(outim.psffn)
if not bok:
print('Sky filename:', outim.splineskyfn)
sky = tim.getSky()
print('Sky:', sky)
trymakedirs(outim.splineskyfn, dir=True)
sky.write_fits(outim.splineskyfn)
outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))
# WISE
if args.wise is not None:
from wise.forcedphot import unwise_tiles_touching_wcs
from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs,
get_unwise_tractor_image, get_unwise_tile_dir)
# Read WCS...
print('Reading TAN wcs header from', args.wise)
targetwcs = Tan(args.wise)
tiles = unwise_tiles_touching_wcs(targetwcs)
print('Cut to', len(tiles), 'unWISE tiles')
H,W = targetwcs.shape
r,d = targetwcs.pixelxy2radec(np.array([1, W, W/2, W/2]),
np.array([H/2, H/2, 1, H ]))
roiradec = [r[0], r[1], d[2], d[3]]
unwise_dir = os.environ['UNWISE_COADDS_DIR']
wise_out = os.path.join(args.outdir, 'images', 'unwise')
print('Will write WISE outputs to', wise_out)
unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR']
wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr')
print('Will write WISE time-resolved outputs to', wise_tr_out)
W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_neo1-atlas.fits'))
print('Read', len(W), 'time-resolved WISE coadd tiles')
W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id]))
print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth')
# Write the time-resolved index subset.
W.writeto(os.path.join(wise_tr_out, 'time_resolved_neo1-atlas.fits'))
# this ought to be enough for anyone =)
Nepochs = 5
wisedata = []
# full depth
for band in [1,2,3,4]:
wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band))
# time-resolved
for band in [1,2]:
# W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2)
bitmask = (1 << (band-1))
for e in range(Nepochs):
# Which tiles have images for this epoch?
I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask)
if len(I) == 0:
continue
print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I])
edir = os.path.join(unwise_tr_dir, 'e%03i' % e)
eoutdir = os.path.join(wise_tr_out, 'e%03i' % e)
wisedata.append((edir, eoutdir, tiles.coadd_id[I], band))
wrote_masks = set()
for indir, outdir, tiles, band in wisedata:
for tile in tiles:
wanyband = 'w'
tim = get_unwise_tractor_image(indir, tile, band,
bandname=wanyband, roiradecbox=roiradec)
print('Got unWISE tim', tim)
print(tim.shape)
thisdir = get_unwise_tile_dir(outdir, tile)
print('Directory for this WISE tile:', thisdir)
base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band))
print('Base filename:', base)
masked = True
mu = 'm' if masked else 'u'
imfn = base + 'img-%s.fits' % mu
ivfn = base + 'invvar-%s.fits.gz' % mu
nifn = base + 'n-%s.fits.gz' % mu
nufn = base + 'n-u.fits.gz'
#print('WISE image header:', tim.hdr)
# Adjust the header WCS by x0,y0
wcs = tim.wcs.wcs
tim.hdr['CRPIX1'] = wcs.crpix[0]
tim.hdr['CRPIX2'] = wcs.crpix[1]
H,W = tim.shape
tim.hdr['IMAGEW'] = W
tim.hdr['IMAGEH'] = H
print('WCS:', wcs)
print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2'])
trymakedirs(imfn, dir=True)
fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True)
print('Wrote', imfn)
fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True)
print('Wrote', ivfn)
fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True)
print('Wrote', nifn)
fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True)
print('Wrote', nufn)
if not (indir,tile) in wrote_masks:
print('Looking for mask file for', indir, tile)
# record that we tried this dir/tile combo
wrote_masks.add((indir,tile))
for idir in indir.split(':'):
tdir = get_unwise_tile_dir(idir, tile)
maskfn = 'unwise-%s-msk.fits.gz' % tile
fn = os.path.join(tdir, maskfn)
print('Mask file:', fn)
if os.path.exists(fn):
print('Reading', fn)
(x0,x1,y0,y1) = tim.roi
roislice = (slice(y0,y1), slice(x0,x1))
F = fitsio.FITS(fn)[0]
hdr = F.read_header()
M = F[roislice]
outfn = os.path.join(thisdir, maskfn)
fitsio.write(outfn, M, header=tim.hdr, clobber=True)
print('Wrote', outfn)
break
outC = outsurvey.get_ccds_readonly()
for iccd,ccd in enumerate(outC):
outim = outsurvey.get_image_object(ccd)
print('Got output image:', outim)
otim = outim.get_tractor_image(pixPsf=True, splinesky=True)
print('Got output tim:', otim)
