def _build_model_image(cat, tims, survey=None, log=None):
"""Generate a model image by rendering each source.
"""
from legacypipe.catalog import read_fits_catalog
from legacypipe.runbrick import _get_mod
print('Creating tractor sources...', flush=True, file=log)
srcs = read_fits_catalog(cat, fluxPrefix='')
if False:
print('Sources:')
[print(' ', src) for src in srcs]
print('Rendering model images...', flush=True, file=log)
mods = [_get_mod((tim, srcs)) for tim in tims]
return mods
def run_one_ccd(survey, catsurvey_north, catsurvey_south, resolve_dec, ccd,
opt, zoomslice, ps):
from functools import reduce
from legacypipe.bits import DQ_BITS
tlast = Time()
#print('Opt:', opt)
im = survey.get_image_object(ccd)
print('Run_one_ccd: checking cache', survey.cache_dir)
if survey.cache_dir is not None:
im.check_for_cached_files(survey)
if opt.do_calib:
im.run_calibs(splinesky=True)
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
constant_invvar=opt.constant_invvar,
hybridPsf=opt.hybrid_psf,
normalizePsf=opt.normalize_psf,
old_calibs_ok=True,
trim_edges=False)
print('Got tim:', tim) #, 'x0,y0', tim.x0, tim.y0)
chipwcs = tim.subwcs
H, W = tim.shape
tnow = Time()
print('Read image:', tnow - tlast)
tlast = tnow
if ccd.camera == 'decam':
# Halo subtraction
from legacypipe.halos import subtract_one
from legacypipe.reference import mask_radius_for_mag, read_gaia
ref_margin = mask_radius_for_mag(0.)
mpix = int(np.ceil(ref_margin * 3600. / chipwcs.pixel_scale()))
marginwcs = chipwcs.get_subimage(-mpix, -mpix, W + 2 * mpix,
H + 2 * mpix)
gaia = read_gaia(marginwcs, None)
keeprad = np.ceil(gaia.keep_radius * 3600. /
chipwcs.pixel_scale()).astype(int)
_, xx, yy = chipwcs.radec2pixelxy(gaia.ra, gaia.dec)
# cut to those touching the chip
gaia.cut((xx > -keeprad) * (xx < W + keeprad) * (yy > -keeprad) *
(yy < H + keeprad))
Igaia, = np.nonzero(gaia.isgaia * gaia.pointsource)
halostars = gaia[Igaia]
print('Got', len(gaia), 'Gaia stars,', len(halostars),
'for halo subtraction')
moffat = True
halos = subtract_one((tim, halostars, moffat))
tim.data -= halos
# The "north" and "south" directories often don't have
# 'survey-bricks" files of their own -- use the 'survey' one
# instead.
if catsurvey_south is not None:
try:
catsurvey_south.get_bricks_readonly()
except:
catsurvey_south.bricks = survey.get_bricks_readonly()
if catsurvey_north is not None:
try:
catsurvey_north.get_bricks_readonly()
except:
catsurvey_north.bricks = survey.get_bricks_readonly()
# Apply outlier masks
outlier_header = None
outlier_mask = None
posneg_mask = None
if opt.outlier_mask is not None:
posneg_mask = np.zeros(tim.shape, np.uint8)
# Outliers masks are computed within a survey (north/south for dr9), and are stored
# in a brick-oriented way, in the results directories.
north_ccd = (ccd.camera.strip() != 'decam')
catsurvey = catsurvey_north
if not north_ccd and catsurvey_south is not None:
catsurvey = catsurvey_south
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
for b in bricks:
print(
'Reading outlier mask for brick', b.brickname, ':',
catsurvey.find_file('outliers_mask',
brick=b.brickname,
output=False))
ok = read_outlier_mask_file(catsurvey, [tim],
b.brickname,
pos_neg_mask=posneg_mask,
subimage=False,
output=False,
ps=ps)
if not ok:
print('WARNING: failed to read outliers mask file for brick',
b.brickname)
if opt.outlier_mask is not None:
outlier_mask = np.zeros((ccd.height, ccd.width), np.uint8)
outlier_mask[tim.y0:tim.y0 + H, tim.x0:tim.x0 + W] = posneg_mask
del posneg_mask
# Grab original image headers (including WCS)
im = survey.get_image_object(ccd)
imhdr = im.read_image_header()
imhdr['CAMERA'] = ccd.camera
imhdr['EXPNUM'] = ccd.expnum
imhdr['CCDNAME'] = ccd.ccdname
imhdr['IMGFILE'] = ccd.image_filename.strip()
outlier_header = imhdr
if opt.catalog:
T = fits_table(opt.catalog)
else:
chipwcs = tim.subwcs
T = get_catalog_in_wcs(chipwcs,
survey,
catsurvey_north,
catsurvey_south=catsurvey_south,
resolve_dec=resolve_dec)
if T is None:
print('No sources to photometer.')
return None
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
surveydir = survey.get_survey_dir()
del survey
if opt.move_gaia:
# Gaia stars: move RA,Dec to the epoch of this image.
I = np.flatnonzero(T.ref_epoch > 0)
if len(I):
print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd())
ra, dec = radec_at_mjd(T.ra[I], T.dec[I],
T.ref_epoch[I].astype(float), T.pmra[I],
T.pmdec[I], T.parallax[I], tim.time.toMjd())
T.ra[I] = ra
T.dec[I] = dec
tnow = Time()
print('Read catalog:', tnow - tlast)
tlast = tnow
# Find SGA galaxies outside this chip and subtract them before we begin.
chipwcs = tim.subwcs
_, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
sga_out = (T.ref_cat == 'L3') * np.logical_not(
(xx >= 1) * (xx <= W) * (yy >= 1) * (yy <= H))
I = np.flatnonzero(sga_out)
if len(I):
print(len(I), 'SGA galaxies are outside the image. Subtracting...')
cat = read_fits_catalog(T[I], bands=[tim.band])
tr = Tractor([tim], cat)
mod = tr.getModelImage(0)
tim.data -= mod
I = np.flatnonzero(np.logical_not(sga_out))
T.cut(I)
cat = read_fits_catalog(T, bands='r')
# Replace the brightness (which will be a NanoMaggies with g,r,z)
# with a NanoMaggies with this image's band only.
for src in cat:
src.brightness = NanoMaggies(**{tim.band: 1.})
tnow = Time()
print('Parse catalog:', tnow - tlast)
tlast = tnow
print('Forced photom...')
F = run_forced_phot(cat,
tim,
ceres=opt.ceres,
derivs=opt.derivs,
fixed_also=True,
agn=opt.agn,
do_forced=opt.forced,
do_apphot=opt.apphot,
get_model=opt.save_model,
ps=ps,
timing=True,
ceres_threads=opt.ceres_threads)
if opt.save_model:
# unpack results
F, model_img = F
F.release = T.release
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.camera = np.array([ccd.camera] * len(F))
F.expnum = np.array([im.expnum] * len(F), dtype=np.int64)
F.ccdname = np.array([im.ccdname] * len(F))
# "Denormalizing"
F.filter = np.array([tim.band] * len(F))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(F))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(F), dtype=np.float32)
F.psfsize = np.array([tim.psf_fwhm * tim.imobj.pixscale] * len(F),
dtype=np.float32)
F.ccd_cuts = np.array([ccd.ccd_cuts] * len(F))
F.airmass = np.array([ccd.airmass] * len(F), dtype=np.float32)
### --> also add units to the dict below so the FITS headers have units
F.sky = np.array([tim.midsky / tim.zpscale / tim.imobj.pixscale**2] *
len(F),
dtype=np.float32)
# in the same units as the depth maps -- flux inverse-variance.
F.psfdepth = np.array([(1. / (tim.sig1 / tim.psfnorm)**2)] * len(F),
dtype=np.float32)
F.galdepth = np.array([(1. / (tim.sig1 / tim.galnorm)**2)] * len(F),
dtype=np.float32)
F.fwhm = np.array([tim.psf_fwhm] * len(F), dtype=np.float32)
F.skyrms = np.array([ccd.skyrms] * len(F), dtype=np.float32)
F.ccdzpt = np.array([ccd.ccdzpt] * len(F), dtype=np.float32)
F.ccdrarms = np.array([ccd.ccdrarms] * len(F), dtype=np.float32)
F.ccddecrms = np.array([ccd.ccddecrms] * len(F), dtype=np.float32)
F.ccdphrms = np.array([ccd.ccdphrms] * len(F), dtype=np.float32)
if opt.derivs:
cosdec = np.cos(np.deg2rad(T.dec))
with np.errstate(divide='ignore', invalid='ignore'):
F.dra = (F.flux_dra / F.flux) * 3600. / cosdec
F.ddec = (F.flux_ddec / F.flux) * 3600.
F.dra[F.flux == 0] = 0.
F.ddec[F.flux == 0] = 0.
F.dra_ivar = F.flux_dra_ivar * (F.flux / 3600. * cosdec)**2
F.ddec_ivar = F.flux_ddec_ivar * (F.flux / 3600.)**2
F.delete_column('flux_dra')
F.delete_column('flux_ddec')
F.delete_column('flux_dra_ivar')
F.delete_column('flux_ddec_ivar')
F.flux = F.flux_fixed
F.flux_ivar = F.flux_fixed_ivar
F.delete_column('flux_fixed')
F.delete_column('flux_fixed_ivar')
for c in ['dra', 'ddec', 'dra_ivar', 'ddec_ivar', 'flux', 'flux_ivar']:
F.set(c, F.get(c).astype(np.float32))
F.ra = T.ra
F.dec = T.dec
_, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
h, w = tim.shape
ix = np.round(F.x).astype(int)
iy = np.round(F.y).astype(int)
F.dqmask = tim.dq[np.clip(iy, 0, h - 1), np.clip(ix, 0, w - 1)]
# Set an OUT-OF-BOUNDS bit.
F.dqmask[reduce(np.logical_or,
[ix < 0, ix >= w, iy < 0, iy >= h])] |= DQ_BITS['edge2']
program_name = sys.argv[0]
## FIXME -- from catalog?
release = 9999
version_hdr = get_version_header(program_name, surveydir, release)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn.strip())
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(
dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(
dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='PLVER', value=ccd.plver, comment='CP pipeline version'))
version_hdr.add_record(
dict(name='PLPROCID', value=ccd.plprocid, comment='CP pipeline id'))
version_hdr.add_record(
dict(name='PROCDATE', value=ccd.procdate, comment='CP image DATE'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
fitsio.write(opt.save_model, model_img, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
tnow = Time()
print('Forced phot:', tnow - tlast)
return F, version_hdr, outlier_mask, outlier_header
def main():
survey = LegacySurveyData()
brickname = '2351p137'
# RA,Dec = 235.0442, 13.7125
bx, by = 3300, 1285
sz = 50
bbox = [bx - sz, bx + sz, by - sz, by + sz]
objid = 1394
bands = ['g', 'r', 'z']
from legacypipe.runbrick import stage_tims, _get_mod, rgbkwargs, rgbkwargs_resid
from legacypipe.survey import get_rgb, imsave_jpeg
from legacypipe.coadds import make_coadds
from astrometry.util.multiproc import multiproc
from astrometry.util.fits import fits_table
from legacypipe.catalog import read_fits_catalog
# brick = survey.get_brick_by_name(brickname)
# # Get WCS object describing brick
# targetwcs = wcs_for_brick(brick)
# (x0,x1,y0,y1) = bbox
# W = x1-x0
# H = y1-y0
# targetwcs = targetwcs.get_subimage(x0, y0, W, H)
# H,W = targetwcs.shape
mp = multiproc()
P = stage_tims(brickname=brickname,
survey=survey,
target_extent=bbox,
pixPsf=True,
hybridPsf=True,
depth_cut=False,
mp=mp)
print('Got', P.keys())
tims = P['tims']
targetwcs = P['targetwcs']
H, W = targetwcs.shape
# Read Tractor catalog
fn = survey.find_file('tractor', brick=brickname)
print('Trying to read catalog', fn)
cat = fits_table(fn)
print('Read', len(cat), 'sources')
ok, xx, yy = targetwcs.radec2pixelxy(cat.ra, cat.dec)
I = np.flatnonzero((xx > 0) * (xx < W) * (yy > 0) * (yy < H))
cat.cut(I)
print('Cut to', len(cat), 'sources within box')
I = np.flatnonzero(cat.objid != objid)
cat.cut(I)
print('Cut to', len(cat), 'sources with objid !=', objid)
#cat.about()
# Convert FITS catalog into tractor source objects
print('Creating tractor sources...')
srcs = read_fits_catalog(cat, fluxPrefix='')
print('Sources:')
for src in srcs:
print(' ', src)
print('Rendering model images...')
mods = [_get_mod((tim, srcs)) for tim in tims]
print('Producing coadds...')
C = make_coadds(tims, bands, targetwcs, mods=mods, mp=mp)
print('Coadds:', dir(C))
coadd_list = [('image', C.coimgs, rgbkwargs),
('model', C.comods, rgbkwargs),
('resid', C.coresids, rgbkwargs_resid)]
#C.coimgs, C.comods, C.coresids
for name, ims, rgbkw in coadd_list:
rgb = get_rgb(ims, bands, **rgbkw)
kwa = {}
#with survey.write_output(name + '-jpeg', brick=brickname) as out:
# imsave_jpeg(out.fn, rgb, origin='lower', **kwa)
# print('Wrote', out.fn)
outfn = name + '.jpg'
imsave_jpeg(outfn, rgb, origin='lower', **kwa)
del rgb
def rbmain():
travis = 'travis' in sys.argv
extra_args = [
'--old-calibs-ok',
#'--verbose',
]
if travis:
extra_args.extend(
['--no-wise-ceres', '--no-gaia', '--no-large-galaxies'])
if 'ceres' in sys.argv:
surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--ceres',
'--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres',
'--no-depth-cut'
])
sys.exit(0)
# demo RexGalaxy, with plots
if False:
from legacypipe.survey import RexGalaxy
from tractor import NanoMaggies, PixPos
from tractor import Image, GaussianMixturePSF, LinearPhotoCal
from legacypipe.survey import LogRadius
rex = RexGalaxy(PixPos(1., 2.), NanoMaggies(r=3.), LogRadius(0.))
print('Rex:', rex)
print('Rex params:', rex.getParams())
print('Rex nparams:', rex.numberOfParams())
H, W = 100, 100
tim = Image(data=np.zeros((H, W), np.float32),
inverr=np.ones((H, W), np.float32),
psf=GaussianMixturePSF(1., 0., 0., 4., 4., 0.),
photocal=LinearPhotoCal(1., band='r'))
derivs = rex.getParamDerivatives(tim)
print('Derivs:', len(derivs))
print('Rex params:', rex.getParamNames())
import pylab as plt
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('rex')
for d, nm in zip(derivs, rex.getParamNames()):
plt.clf()
plt.imshow(d.patch, interpolation='nearest', origin='lower')
plt.title('Derivative %s' % nm)
ps.savefig()
sys.exit(0)
# Test RexGalaxy
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6-rex'
main(args=[
'--brick',
'1102p240',
'--zoom',
'500',
'600',
'650',
'750',
'--force-all',
'--no-write',
'--no-wise',
#'--rex', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
] + extra_args)
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0] == 'PSF ')
cmd = (
'(cd %s && sha256sum -c %s)' %
(outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum')))
print(cmd)
rtn = os.system(cmd)
assert (rtn == 0)
# Test with a Tycho-2 star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6'
main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--survey-dir', surveydir,
'--outdir', outdir
] + extra_args)
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0] == 'PSF ')
# Test that we can run splinesky calib if required...
from legacypipe.decam import DecamImage
DecamImage.splinesky_boxsize = 128
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
outdir = 'out-testcase4'
fn = os.path.join(surveydir, 'calib', 'decam', 'splinesky', '00431',
'00431608', 'decam-00431608-N3.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
os.path.join(surveydir, 'images', 'unwise-tr'), '--blob-image',
'--no-hybrid-psf', '--survey-dir', surveydir, '--outdir', outdir
] + extra_args + ['-v'])
print('Checking for calib file', fn)
assert (os.path.exists(fn))
# Wrap-around, hybrid PSF
surveydir = os.path.join(os.path.dirname(__file__), 'testcase8')
outdir = 'out-testcase8'
main(args=[
'--brick',
'1209p050',
'--zoom',
'720',
'1095',
'3220',
'3500',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
] + extra_args)
# Test with a Tycho-2 star + another saturated star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase7')
outdir = 'out-testcase7'
# remove --no-gaia
my_extra_args = [a for a in extra_args if a != '--no-gaia']
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia-dr2')
os.environ['GAIA_CAT_VER'] = '2'
main(args=[
'--brick',
'1102p240',
'--zoom',
'250',
'350',
'1550',
'1650',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
] + my_extra_args)
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 4)
# Check skipping blobs outside the brick's unique area.
# (this now doesn't detect any sources at all, reasonably)
# surveydir = os.path.join(os.path.dirname(__file__), 'testcase5')
# outdir = 'out-testcase5'
#
# fn = os.path.join(outdir, 'tractor', '186', 'tractor-1867p255.fits')
# if os.path.exists(fn):
# os.unlink(fn)
#
# main(args=['--brick', '1867p255', '--zoom', '0', '150', '0', '150',
# '--force-all', '--no-write', '--coadd-bw',
# '--survey-dir', surveydir,
# '--early-coadds',
# '--outdir', outdir] + extra_args)
#
# assert(os.path.exists(fn))
# T = fits_table(fn)
# assert(len(T) == 1)
# Custom RA,Dec; blob ra,dec.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
outdir = 'out-testcase4b'
# Catalog written with one entry (--blobradec)
fn = os.path.join(outdir, 'tractor', 'cus',
'tractor-custom-186743p25461.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--radec', '186.743965', '25.461788', '--width', '250', '--height',
'250', '--force-all', '--no-write', '--no-wise', '--blobradec',
'186.740369', '25.453855', '--survey-dir', surveydir, '--outdir',
outdir
] + extra_args)
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 1)
surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
outdir = 'out-testcase3'
checkpoint_fn = os.path.join(outdir, 'checkpoint.pickle')
if os.path.exists(checkpoint_fn):
os.unlink(checkpoint_fn)
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
] + extra_args)
# Read catalog into Tractor sources to test read_fits_catalog
from legacypipe.catalog import read_fits_catalog
from legacypipe.survey import LegacySurveyData, GaiaSource
from tractor.galaxy import DevGalaxy
from tractor import PointSource
survey = LegacySurveyData(survey_dir=outdir)
fn = survey.find_file('tractor', brick='2447p120')
print('Checking', fn)
T = fits_table(fn)
cat = read_fits_catalog(T, fluxPrefix='')
print('Read catalog:', cat)
assert (len(cat) == 2)
src = cat[0]
assert (type(src) == DevGalaxy)
assert (np.abs(src.pos.ra - 244.77973) < 0.00001)
assert (np.abs(src.pos.dec - 12.07233) < 0.00001)
src = cat[1]
print('Source', src)
assert (type(src) in [PointSource, GaiaSource])
assert (np.abs(src.pos.ra - 244.77830) < 0.00001)
assert (np.abs(src.pos.dec - 12.07250) < 0.00001)
# DevGalaxy(pos=RaDecPos[244.77975494973529, 12.072348111713127], brightness=NanoMaggies: g=19.2, r=17.9, z=17.1, shape=re=2.09234, e1=-0.198453, e2=0.023652,
# PointSource(RaDecPos[244.77833280764278, 12.072521274981987], NanoMaggies: g=25, r=23, z=21.7)
# Check that we can run again, using that checkpoint file.
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
] + extra_args)
# Assert...... something?
# Test --checkpoint without --threads
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1'
] + extra_args)
# MzLS + BASS data
# surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass')
# main(args=['--brick', '3521p002', '--zoom', '2400', '2450', '1200', '1250',
# '--no-wise', '--force-all', '--no-write',
# '--survey-dir', surveydir2,
# '--outdir', 'out-mzlsbass'])
# From Kaylan's Bootes pre-DR4 run
# surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass3')
# main(args=['--brick', '2173p350', '--zoom', '100', '200', '100', '200',
# '--no-wise', '--force-all', '--no-write',
# '--survey-dir', surveydir2,
# '--outdir', 'out-mzlsbass3'] + extra_args)
# With plots!
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', 'out-testcase3', '--plots', '--nblobs', '1'
] + extra_args)
# Decals Image Simulations
# Uncomment WHEN galsim build for Travis
#os.environ["DECALS_SIM_DIR"]= os.path.join(os.path.dirname(__file__),'image_sims')
#brick= '2447p120'
#sim_main(args=['--brick', brick, '-n', '2', '-o', 'STAR', \
# '-ic', '1', '--rmag-range', '18', '26', '--threads', '1',\
# '--zoom', '1020', '1070', '2775', '2815'])
# Check if correct files written out
#rt_dir= os.path.join(os.getenv('DECALS_SIM_DIR'),brick,'star','001')
#assert( os.path.exists(os.path.join(rt_dir,'../','metacat-'+brick+'-star.fits')) )
#for fn in ['tractor-%s-star-01.fits' % brick,'simcat-%s-star-01.fits' % brick]:
# assert( os.path.exists(os.path.join(rt_dir,fn)) )
#for fn in ['image','model','resid','simscoadd']:
# assert( os.path.exists(os.path.join(rt_dir,'qa-'+brick+'-star-'+fn+'-01.jpg')) )
if not travis:
# With ceres
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--ceres',
'--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres'
] + extra_args)
def unwise_coadds(onegal,
galaxy=None,
radius_mosaic=30,
radius_mask=None,
pixscale=2.75,
ref_pixscale=0.262,
output_dir=None,
unwise_dir=None,
verbose=False,
log=None,
centrals=True):
'''Generate custom unWISE cutouts.
radius_mosaic and radius_mask in arcsec
pixscale: WISE pixel scale in arcsec/pixel; make this smaller than 2.75
to oversample.
'''
import fitsio
import matplotlib.pyplot as plt
from astrometry.util.util import Tan
from astrometry.util.fits import fits_table
from astrometry.libkd.spherematch import match_radec
from astrometry.util.resample import resample_with_wcs, ResampleError
from wise.forcedphot import unwise_tiles_touching_wcs
from wise.unwise import get_unwise_tractor_image
from tractor import Tractor, Image, NanoMaggies
from legacypipe.survey import imsave_jpeg
from legacypipe.catalog import read_fits_catalog
if galaxy is None:
galaxy = 'galaxy'
if output_dir is None:
output_dir = '.'
if unwise_dir is None:
unwise_dir = os.environ.get('UNWISE_COADDS_DIR')
if radius_mask is None:
radius_mask = radius_mosaic
radius_search = 5.0 # [arcsec]
else:
radius_search = radius_mask
# Initialize the WCS object.
W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int') # [pixels]
targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0,
-pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W),
float(H))
# Read the custom Tractor catalog.
tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy))
if not os.path.isfile(tractorfile):
print('Missing Tractor catalog {}'.format(tractorfile),
flush=True,
file=log)
return 0
primhdr = fitsio.read_header(tractorfile)
cat = fits_table(tractorfile)
print('Read {} sources from {}'.format(len(cat), tractorfile),
flush=True,
file=log)
keep = np.ones(len(cat)).astype(bool)
if centrals:
# Find the large central galaxy and mask out (ignore) all the models
# which are within its elliptical mask.
# This algorithm will have to change for mosaics not centered on large
# galaxies, e.g., in galaxy groups.
m1, m2, d12 = match_radec(cat.ra,
cat.dec,
onegal['RA'],
onegal['DEC'],
radius_search / 3600.0,
nearest=False)
if len(m1) == 0:
print('No central galaxies found at the central coordinates!',
flush=True,
file=log)
else:
pixfactor = ref_pixscale / pixscale # shift the optical Tractor positions
for mm in m1:
morphtype = cat.type[mm].strip()
if morphtype == 'EXP' or morphtype == 'COMP':
e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[
mm], cat.shapeexp_r[mm] # [arcsec]
elif morphtype == 'DEV' or morphtype == 'COMP':
e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[
mm], cat.shapedev_r[mm] # [arcsec]
else:
r50 = None
if r50:
majoraxis = r50 * 5 / pixscale # [pixels]
ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2)
these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis,
ba * majoraxis,
np.radians(phi),
cat.bx * pixfactor,
cat.by * pixfactor)
if np.sum(these) > 0:
#keep[these] = False
pass
print('Hack!')
keep[mm] = False
#srcs = read_fits_catalog(cat)
#_srcs = np.array(srcs)[~keep].tolist()
#mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0)
#import matplotlib.pyplot as plt
##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png')
#plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png')
#pdb.set_trace()
srcs = read_fits_catalog(cat)
for src in srcs:
src.freezeAllBut('brightness')
#srcs_nocentral = np.array(srcs)[keep].tolist()
cat_nocentral = cat[keep]
## Find and remove all the objects within XX arcsec of the target
## coordinates.
#m1, m2, d12 = match_radec(T.ra, T.dec, onegal['RA'], onegal['DEC'], 5/3600.0, nearest=False)
#if len(d12) == 0:
# print('No matching galaxies found -- probably not what you wanted.')
# #raise ValueError
# nocentral = np.ones(len(T)).astype(bool)
#else:
# nocentral = ~np.isin(T.objid, T[m1].objid)
#T_nocentral = T[nocentral]
# Find and read the overlapping unWISE tiles. Assume the targetwcs is
# axis-aligned and that the edge midpoints yield the RA, Dec limits (true
# for TAN). Note: the way the roiradec box is used, the min/max order
# doesn't matter.
r, d = targetwcs.pixelxy2radec(np.array([1, W, W / 2, W / 2]),
np.array([H / 2, H / 2, 1, H]))
roiradec = [r[0], r[1], d[2], d[3]]
tiles = unwise_tiles_touching_wcs(targetwcs)
wbands = [1, 2, 3, 4]
wanyband = 'w'
vega_to_ab = dict(w1=2.699, w2=3.339, w3=5.174, w4=6.620)
# Convert the AB WISE fluxes in the Tractor catalog to Vega nanomaggies so
# they're consistent with the coadds, below.
for band in wbands:
f = cat.get('flux_w{}'.format(band))
e = cat.get('flux_ivar_w{}'.format(band))
print('Setting negative fluxes equal to zero!')
f[f < 0] = 0
#f[f/e < 3] = 0
f *= 10**(0.4 * vega_to_ab['w{}'.format(band)])
coimgs = [np.zeros((H, W), np.float32) for b in wbands]
comods = [np.zeros((H, W), np.float32) for b in wbands]
comods_nocentral = [np.zeros((H, W), np.float32) for b in wbands]
con = [np.zeros((H, W), np.uint8) for b in wbands]
for iband, band in enumerate(wbands):
for ii, src in enumerate(srcs):
src.setBrightness(
NanoMaggies(
**{wanyband: cat.get('flux_w{}'.format(band))[ii]}))
srcs_nocentral = np.array(srcs)[keep].tolist()
#srcs_nocentral = np.array(srcs)[nocentral].tolist()
# The tiles have some overlap, so for each source, keep the fit in the
# tile whose center is closest to the source.
for tile in tiles:
#print('Reading tile {}'.format(tile.coadd_id))
tim = get_unwise_tractor_image(unwise_dir,
tile.coadd_id,
band,
bandname=wanyband,
roiradecbox=roiradec)
if tim is None:
print('Actually, no overlap with tile {}'.format(
tile.coadd_id))
continue
print('Read image {} with shape {}'.format(tile.coadd_id,
tim.shape))
def _unwise_mod(tim, use_cat, use_srcs, margin=10):
# Select sources in play.
wisewcs = tim.wcs.wcs
timH, timW = tim.shape
ok, x, y = wisewcs.radec2pixelxy(use_cat.ra, use_cat.dec)
x = (x - 1.).astype(np.float32)
y = (y - 1.).astype(np.float32)
I = np.flatnonzero((x >= -margin) * (x < timW + margin) *
(y >= -margin) * (y < timH + margin))
#print('Found {} sources within the image + margin = {} pixels'.format(len(I), margin))
subcat = [use_srcs[i] for i in I]
tractor = Tractor([tim], subcat)
mod = tractor.getModelImage(0)
return mod
mod = _unwise_mod(tim, cat, srcs)
mod_nocentral = _unwise_mod(tim, cat_nocentral, srcs_nocentral)
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, tim.wcs.wcs)
except ResampleError:
continue
if len(Yo) == 0:
continue
# The models are already in AB nanomaggies, but the tiles / tims are
# in Vega nanomaggies, so convert them here.
coimgs[iband][Yo, Xo] += tim.getImage()[Yi, Xi]
comods[iband][Yo, Xo] += mod[Yi, Xi]
comods_nocentral[iband][Yo, Xo] += mod_nocentral[Yi, Xi]
con[iband][Yo, Xo] += 1
## Convert back to nanomaggies.
#vega2ab = vega_to_ab['w{}'.format(band)]
#coimgs[iband] *= 10**(-0.4 * vega2ab)
#comods[iband] *= 10**(-0.4 * vega2ab)
#comods_nocentral[iband] *= 10**(-0.4 * vega2ab)
for img, mod, mod_nocentral, n in zip(coimgs, comods, comods_nocentral,
con):
img /= np.maximum(n, 1)
mod /= np.maximum(n, 1)
mod_nocentral /= np.maximum(n, 1)
coresids = [img - mod for img, mod in list(zip(coimgs, comods))]
# Subtract the model image which excludes the central (comod_nocentral)
# from the data (coimg) to isolate the light of the central
# (coimg_central).
coimgs_central = [
img - mod for img, mod in list(zip(coimgs, comods_nocentral))
]
# Write out the final images with and without the central and converted into
# AB nanomaggies.
for coadd, imtype in zip((coimgs, comods, comods_nocentral),
('image', 'model', 'model-nocentral')):
for img, band in zip(coadd, wbands):
vega2ab = vega_to_ab['w{}'.format(band)]
fitsfile = os.path.join(
output_dir, '{}-{}-W{}.fits'.format(galaxy, imtype, band))
if verbose:
print('Writing {}'.format(fitsfile))
fitsio.write(fitsfile, img * 10**(-0.4 * vega2ab), clobber=True)
# Generate color WISE images.
kwa = dict(mn=-1, mx=100, arcsinh=0.5)
#kwa = dict(mn=-0.05, mx=1., arcsinh=0.5)
#kwa = dict(mn=-0.1, mx=2., arcsinh=None)
for imgs, imtype in zip(
(coimgs, comods, coresids, comods_nocentral, coimgs_central),
('image', 'model', 'resid', 'model-nocentral', 'image-central')):
rgb = _unwise_to_rgb(imgs[:2], **kwa) # W1, W2
jpgfile = os.path.join(output_dir,
'{}-{}-W1W2.jpg'.format(galaxy, imtype))
if verbose:
print('Writing {}'.format(jpgfile))
imsave_jpeg(jpgfile, rgb, origin='lower')
return 1
'--outdir', outdir,
'--checkpoint', checkpoint_fn,
'--checkpoint-period', '1',
'--threads', '2'])
# Read catalog into Tractor sources to test read_fits_catalog
from legacypipe.catalog import read_fits_catalog
from legacypipe.survey import LegacySurveyData
from astrometry.util.fits import fits_table
from tractor.galaxy import DevGalaxy
from tractor import PointSource
survey = LegacySurveyData(survey_dir=outdir)
fn = survey.find_file('tractor', brick='2447p120')
T = fits_table(fn)
cat = read_fits_catalog(T)
print('Read catalog:', cat)
assert(len(cat) == 2)
src = cat[0]
assert(type(src) == DevGalaxy)
assert(np.abs(src.pos.ra - 244.77975) < 0.00001)
assert(np.abs(src.pos.dec - 12.07234) < 0.00001)
src = cat[1]
assert(type(src) == PointSource)
assert(np.abs(src.pos.ra - 244.77833) < 0.00001)
assert(np.abs(src.pos.dec - 12.07252) < 0.00001)
# DevGalaxy(pos=RaDecPos[244.77975494973529, 12.072348111713127], brightness=NanoMaggies: g=19.2, r=17.9, z=17.1, shape=re=2.09234, e1=-0.198453, e2=0.023652,
# PointSource(RaDecPos[244.77833280764278, 12.072521274981987], NanoMaggies: g=25, r=23, z=21.7)
# Check that we can run again, using that checkpoint file.
main(args=['--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
else:
bricknames.append('%s_sg%02i' % (tile.strip(), subvis))
galex.brickname = np.array(bricknames)
# bricks_touching_radec_box(self, ralo, rahi, declo, dechi, scale=None):
I, = np.nonzero((galex.dec1 <= dechi) * (galex.dec2 >= declo))
ok = ra_ranges_overlap(ralo, rahi, galex.ra1[I], galex.ra2[I])
I = I[ok]
galex.cut(I)
print('-> bricks', galex.brickname)
gbands = ['n', 'f']
coimgs = []
comods = []
srcs = read_fits_catalog(T)
for src in srcs:
src.freezeAllBut('brightness')
for band in gbands:
J = np.flatnonzero(galex.get('has_' + band))
print(len(J), 'GALEX tiles have coverage in band', band)
coimg = np.zeros((H, W), np.float32)
comod = np.zeros((H, W), np.float32)
cowt = np.zeros((H, W), np.float32)
for src in srcs:
src.setBrightness(NanoMaggies(**{band: 1}))
for j in J:
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--catalog', help='Catalog to render')
parser.add_argument('--brick-coadd', help='Produce a coadd of the images overlapping the given brickname.')
parser.add_argument('--ccds', help='Use this table of CCDs')
parser.add_argument('--brick-wcs', help='File containing a WCS header describing the coadd WCS to render.')
parser.add_argument('--brick-wcs-ext', type=int, help='FITS file extension containing a WCS header describing the coadd WCS to render.')
parser.add_argument('--outlier-mask-brick', help='Comma-separated list of bricknames from which outlier masks should be read.')
parser.add_argument('--out', help='Filename pattern ("BAND" will be replaced by band name) of output images.')
parser.add_argument('--resid', help='Filename pattern ("BAND" will be replaced by band name) of residual images.')
parser.add_argument('--jpeg', help='Write RGB image to this filename')
parser.add_argument('--resid-jpeg', help='Write RGB residual image to this filename')
opt = parser.parse_args()
if opt.catalog is None:
print('Need catalog!')
return -1
cat = fits_table(opt.catalog)
if opt.ccds is None:
if opt.brick_coadd is None:
print('Need brick catalog!')
return -1
brickname = opt.brick_coadd
survey = LegacySurveyData()
if opt.brick_wcs is None:
print('FIXME')
return -1
else:
wcs = Tan(opt.brick_wcs, opt.brick_wcs_ext)
tcat = read_fits_catalog(cat)
if opt.ccds:
ccdfn = opt.ccds
else:
ccdfn = survey.find_file('ccds-table', brick=brickname)
print('Reading', ccdfn)
ccds = fits_table(ccdfn)
H,W = wcs.shape
targetrd = np.array([wcs.pixelxy2radec(x,y) for x,y in
[(1,1),(W,1),(W,H),(1,H),(1,1)]])
tims = []
for ccd in ccds:
im = survey.get_image_object(ccd)
#slc = slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1)
#tim = im.get_tractor_image(slc=slc)
tim = im.get_tractor_image(radecpoly=targetrd)
print('Read', tim)
tims.append(tim)
if opt.outlier_mask_brick is not None:
bricks = opt.outlier_mask_brick.split(',')
for b in bricks:
print('Reading outlier mask for brick', b,
':', survey.find_file('outliers_mask', brick=b, output=False))
ok = read_outlier_mask_file(survey, tims, b,
subimage=True, output=False)
tr = Tractor(tims, tcat)
mods = list(tr.getModelImages())
bands = 'grz'
def write_model(band, cowimg=None, cowmod=None, **kwargs):
if cowmod is None:
print('No model for', band)
return
outfn = opt.out.replace('BAND', band)
fitsio.write(outfn, cowmod, clobber=True)
print('Wrote model for', band, 'to', outfn)
if opt.resid:
outfn = opt.resid.replace('BAND', band)
fitsio.write(outfn, cowimg - cowmod, clobber=True)
print('Wrote resid for', band, 'to', outfn)
C = make_coadds(tims, bands, wcs, mods=mods,
callback=write_model)
if opt.jpeg:
from legacypipe.survey import get_rgb
import pylab as plt
plt.imsave(opt.jpeg, get_rgb(C.comods, bands), origin='lower')
if opt.resid_jpeg:
from legacypipe.survey import get_rgb
import pylab as plt
plt.imsave(opt.resid_jpeg,
get_rgb([im-mod for im,mod in zip(C.coimgs, C.comods)], bands),
origin='lower')
'--checkpoint', checkpoint_fn,
'--checkpoint-period', '1',
'--threads', '2'] + extra_args)
# Read catalog into Tractor sources to test read_fits_catalog
from legacypipe.catalog import read_fits_catalog
from legacypipe.survey import LegacySurveyData, GaiaSource
from astrometry.util.fits import fits_table
from tractor.galaxy import DevGalaxy
from tractor import PointSource
survey = LegacySurveyData(survey_dir=outdir)
fn = survey.find_file('tractor', brick='2447p120')
print('Checking', fn)
T = fits_table(fn)
cat = read_fits_catalog(T, fluxPrefix='')
print('Read catalog:', cat)
assert(len(cat) == 2)
src = cat[0]
assert(type(src) == DevGalaxy)
assert(np.abs(src.pos.ra - 244.77975) < 0.00001)
assert(np.abs(src.pos.dec - 12.07234) < 0.00001)
src = cat[1]
print('Source', src)
assert(type(src) in [PointSource, GaiaSource])
assert(np.abs(src.pos.ra - 244.77833) < 0.00001)
assert(np.abs(src.pos.dec - 12.07252) < 0.00001)
# DevGalaxy(pos=RaDecPos[244.77975494973529, 12.072348111713127], brightness=NanoMaggies: g=19.2, r=17.9, z=17.1, shape=re=2.09234, e1=-0.198453, e2=0.023652,
# PointSource(RaDecPos[244.77833280764278, 12.072521274981987], NanoMaggies: g=25, r=23, z=21.7)
# Check that we can run again, using that checkpoint file.
def galex_coadds(onegal,
galaxy=None,
radius_mosaic=30,
radius_mask=None,
pixscale=1.5,
ref_pixscale=0.262,
output_dir=None,
galex_dir=None,
log=None,
centrals=True,
verbose=False):
'''Generate custom GALEX cutouts.
radius_mosaic and radius_mask in arcsec
pixscale: GALEX pixel scale in arcsec/pixel.
'''
import fitsio
import matplotlib.pyplot as plt
from astrometry.libkd.spherematch import match_radec
from astrometry.util.resample import resample_with_wcs, OverlapError
from tractor import (Tractor, NanoMaggies, Image, LinearPhotoCal,
NCircularGaussianPSF, ConstantFitsWcs, ConstantSky)
from legacypipe.survey import imsave_jpeg
from legacypipe.catalog import read_fits_catalog
if galaxy is None:
galaxy = 'galaxy'
if galex_dir is None:
galex_dir = os.environ.get('GALEX_DIR')
if output_dir is None:
output_dir = '.'
if radius_mask is None:
radius_mask = radius_mosaic
radius_search = 5.0 # [arcsec]
else:
radius_search = radius_mask
W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int') # [pixels]
targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0,
-pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W),
float(H))
# Read the custom Tractor catalog
tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy))
if not os.path.isfile(tractorfile):
print('Missing Tractor catalog {}'.format(tractorfile))
return 0
cat = fits_table(tractorfile)
print('Read {} sources from {}'.format(len(cat), tractorfile),
flush=True,
file=log)
keep = np.ones(len(cat)).astype(bool)
if centrals:
# Find the large central galaxy and mask out (ignore) all the models
# which are within its elliptical mask.
# This algorithm will have to change for mosaics not centered on large
# galaxies, e.g., in galaxy groups.
m1, m2, d12 = match_radec(cat.ra,
cat.dec,
onegal['RA'],
onegal['DEC'],
radius_search / 3600.0,
nearest=False)
if len(m1) == 0:
print('No central galaxies found at the central coordinates!',
flush=True,
file=log)
else:
pixfactor = ref_pixscale / pixscale # shift the optical Tractor positions
for mm in m1:
morphtype = cat.type[mm].strip()
if morphtype == 'EXP' or morphtype == 'COMP':
e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[
mm], cat.shapeexp_r[mm] # [arcsec]
elif morphtype == 'DEV' or morphtype == 'COMP':
e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[
mm], cat.shapedev_r[mm] # [arcsec]
else:
r50 = None
if r50:
majoraxis = r50 * 5 / pixscale # [pixels]
ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2)
these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis,
ba * majoraxis,
np.radians(phi),
cat.bx * pixfactor,
cat.by * pixfactor)
if np.sum(these) > 0:
#keep[these] = False
pass
print('Hack!')
keep[mm] = False
#srcs = read_fits_catalog(cat)
#_srcs = np.array(srcs)[~keep].tolist()
#mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0)
#import matplotlib.pyplot as plt
##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png')
#plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png')
#pdb.set_trace()
srcs = read_fits_catalog(cat)
for src in srcs:
src.freezeAllBut('brightness')
#srcs_nocentral = np.array(srcs)[keep].tolist()
# Find all overlapping GALEX tiles and then read the tims.
galex_tiles = _read_galex_tiles(targetwcs,
galex_dir,
log=log,
verbose=verbose)
gbands = ['n', 'f']
nicegbands = ['NUV', 'FUV']
zps = dict(n=20.08, f=18.82)
coimgs, comods, coresids, coimgs_central, comods_nocentral = [], [], [], [], []
for niceband, band in zip(nicegbands, gbands):
J = np.flatnonzero(galex_tiles.get('has_' + band))
print(len(J), 'GALEX tiles have coverage in band', band)
coimg = np.zeros((H, W), np.float32)
comod = np.zeros((H, W), np.float32)
cowt = np.zeros((H, W), np.float32)
comod_nocentral = np.zeros((H, W), np.float32)
for src in srcs:
src.setBrightness(NanoMaggies(**{band: 1}))
for j in J:
brick = galex_tiles[j]
fn = os.path.join(
galex_dir, brick.tilename.strip(),
'%s-%sd-intbgsub.fits.gz' % (brick.brickname, band))
#print(fn)
gwcs = Tan(*[
float(f) for f in [
brick.crval1, brick.crval2, brick.crpix1, brick.crpix2,
brick.cdelt1, 0., 0., brick.cdelt2, 3840., 3840.
]
])
img = fitsio.read(fn)
#print('Read', img.shape)
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, gwcs, [], 3)
except OverlapError:
continue
K = np.flatnonzero(img[Yi, Xi] != 0.)
if len(K) == 0:
continue
Yo, Xo, Yi, Xi = Yo[K], Xo[K], Yi[K], Xi[K]
wt = brick.get(band + 'exptime')
coimg[Yo, Xo] += wt * img[Yi, Xi]
cowt[Yo, Xo] += wt
x0, x1, y0, y1 = min(Xi), max(Xi), min(Yi), max(Yi)
subwcs = gwcs.get_subimage(x0, y0, x1 - x0 + 1, y1 - y0 + 1)
twcs = ConstantFitsWcs(subwcs)
timg = img[y0:y1 + 1, x0:x1 + 1]
tie = np.ones_like(timg) ## HACK!
#hdr = fitsio.read_header(fn)
#zp = hdr['']
zp = zps[band]
photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp),
band=band)
tsky = ConstantSky(0.0)
# HACK -- circular Gaussian PSF of fixed size...
# in arcsec
#fwhms = dict(NUV=6.0, FUV=6.0)
# -> sigma in pixels
#sig = fwhms[band] / 2.35 / twcs.pixel_scale()
sig = 6.0 / np.sqrt(8 * np.log(2)) / twcs.pixel_scale()
tpsf = NCircularGaussianPSF([sig], [1.])
tim = Image(data=timg,
inverr=tie,
psf=tpsf,
wcs=twcs,
sky=tsky,
photocal=photocal,
name='GALEX ' + band + brick.brickname)
## Build the model image with and without the central galaxy model.
tractor = Tractor([tim], srcs)
mod = tractor.getModelImage(0)
tractor.freezeParam('images')
tractor.optimize_forced_photometry(priors=False,
shared_params=False)
mod = tractor.getModelImage(0)
srcs_nocentral = np.array(srcs)[keep].tolist()
#srcs_nocentral = np.array(srcs)[nocentral].tolist()
tractor_nocentral = Tractor([tim], srcs_nocentral)
mod_nocentral = tractor_nocentral.getModelImage(0)
comod[Yo, Xo] += wt * mod[Yi - y0, Xi - x0]
comod_nocentral[Yo, Xo] += wt * mod_nocentral[Yi - y0, Xi - x0]
coimg /= np.maximum(cowt, 1e-18)
comod /= np.maximum(cowt, 1e-18)
comod_nocentral /= np.maximum(cowt, 1e-18)
coresid = coimg - comod
# Subtract the model image which excludes the central (comod_nocentral)
# from the data (coimg) to isolate the light of the central
# (coimg_central).
coimg_central = coimg - comod_nocentral
coimgs.append(coimg)
comods.append(comod)
coresids.append(coresid)
comods_nocentral.append(comod_nocentral)
coimgs_central.append(coimg_central)
# Write out the final images with and without the central, making sure
# to apply the zeropoint to go from counts/s to AB nanomaggies.
# https://asd.gsfc.nasa.gov/archive/galex/FAQ/counts_background.html
for thisimg, imtype in zip((coimg, comod, comod_nocentral),
('image', 'model', 'model-nocentral')):
fitsfile = os.path.join(
output_dir, '{}-{}-{}.fits'.format(galaxy, imtype, niceband))
if verbose:
print('Writing {}'.format(fitsfile))
fitsio.write(fitsfile,
thisimg * 10**(-0.4 * (zp - 22.5)),
clobber=True)
# Build a color mosaic (but note that the images here are in units of
# background-subtracted counts/s).
#_galex_rgb = _galex_rgb_moustakas
#_galex_rgb = _galex_rgb_dstn
_galex_rgb = _galex_rgb_official
for imgs, imtype in zip(
(coimgs, comods, coresids, comods_nocentral, coimgs_central),
('image', 'model', 'resid', 'model-nocentral', 'image-central')):
rgb = _galex_rgb(imgs)
jpgfile = os.path.join(output_dir,
'{}-{}-FUVNUV.jpg'.format(galaxy, imtype))
if verbose:
print('Writing {}'.format(jpgfile))
imsave_jpeg(jpgfile, rgb, origin='lower')
return 1
def main():
fn = '/global/cscratch1/sd/dstn/c4d_190730_024955_ori/c4d_190730_024955_ori.52.fits'
survey_dir = '/global/cscratch1/sd/dstn/subtractor-survey-dir'
imagedir = os.path.join(survey_dir, 'images')
trymakedirs(imagedir)
calibdir = os.path.join(survey_dir, 'calib')
psfexdir = os.path.join(calibdir, 'decam', 'psfex-merged')
trymakedirs(psfexdir)
skydir = os.path.join(calibdir, 'decam', 'splinesky-merged')
trymakedirs(skydir)
basename = os.path.basename(fn)
basename = basename.replace('.fits', '')
# Output filenames for legacyzpts calibration/zeropoint files
f, photfn = tempfile.mkstemp()
os.close(f)
surveyfn = os.path.join(survey_dir, 'survey-ccds-%s.fits.gz' % basename)
annfn = os.path.join(survey_dir, 'annotated-%s.fits' % basename)
mp = multiproc()
survey = LegacySurveyData(survey_dir)
# Use the subclass above to handle DECam images!
survey.image_typemap.update(decam=GoldsteinDecamImage)
# Grab the exposure number and CCD name
hdr = fitsio.read_header(fn)
expnum = hdr['EXPNUM']
ccdname = hdr['EXTNAME'].strip()
print('Exposure', expnum, 'CCD', ccdname)
import logging
lvl = logging.INFO
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
if not os.path.exists(surveyfn):
# Run calibrations and zeropoints
runit(fn,
photfn,
surveyfn,
annfn,
mp,
survey=survey,
camera='decam',
debug=False,
choose_ccd=ccdname,
splinesky=True,
calibdir=calibdir,
measureclass=GoldsteinDecamMeasurer)
# Find catalog sources touching this CCD
ccds = survey.find_ccds(expnum=expnum, ccdname=ccdname)
assert (len(ccds) == 1)
ccd = ccds[0]
print('Got CCD', ccd)
# Create Tractor image
im = survey.get_image_object(ccd)
print('Got image:', im)
# Look at this sub-image, or the whole chip?
#zoomslice=None
zoomslice = (slice(0, 1000), slice(0, 1000))
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
splinesky=True,
hybridPsf=True,
normalizePsf=True,
old_calibs_ok=True)
print('Got tim:', tim)
# Read catalog files touching this CCD
catsurvey = LegacySurveyData(
'/global/project/projectdirs/cosmo/work/legacysurvey/dr8/south')
T = get_catalog_in_wcs(tim.subwcs, catsurvey)
print('Got', len(T), 'DR8 catalog sources within CCD')
# Gaia stars: move RA,Dec to the epoch of this image.
I = np.flatnonzero(T.ref_epoch > 0)
if len(I):
from legacypipe.survey import radec_at_mjd
print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd())
ra, dec = radec_at_mjd(T.ra[I], T.dec[I], T.ref_epoch[I].astype(float),
T.pmra[I], T.pmdec[I], T.parallax[I],
tim.time.toMjd())
T.ra[I] = ra
T.dec[I] = dec
# Create Tractor Source objects from the catalog
cat = read_fits_catalog(T, bands=tim.band)
print('Created', len(cat), 'source objects')
# Render model image!
tr = Tractor([tim], cat)
mod = tr.getModelImage(0)
# plots
ima = dict(interpolation='nearest',
origin='lower',
vmin=-2 * tim.sig1,
vmax=10 * tim.sig1,
cmap='gray')
plt.clf()
plt.imshow(tim.getImage(), **ima)
plt.title('Image')
plt.savefig('img.jpg')
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model')
plt.savefig('mod.jpg')
plt.clf()
plt.imshow(tim.getImage() - mod, **ima)
plt.title('Residual')
plt.savefig('res.jpg')
def rbmain():
from legacypipe.catalog import read_fits_catalog
from legacypipe.survey import LegacySurveyData, wcs_for_brick
from tractor.galaxy import DevGalaxy
from tractor import PointSource, Catalog
from tractor import GaussianMixturePSF
from legacypipe.survey import BrickDuck
from legacypipe.forced_photom import main as forced_main
from astrometry.util.file import trymakedirs
import shutil
ceres = 'ceres' in sys.argv
psfex = 'psfex' in sys.argv
for v in [
'UNWISE_COADDS_TIMERESOLVED_DIR', 'SKY_TEMPLATE_DIR',
'LARGEGALAXIES_CAT', 'GAIA_CAT_DIR', 'TYCHO2_KD_DIR'
]:
if v in os.environ:
del os.environ[v]
oldargs = sys.argv
sys.argv = [sys.argv[0]]
main()
sys.argv = oldargs
# Test create_kdtree and (reading CCD kd-tree)!
indir = os.path.join(os.path.dirname(__file__), 'testcase6')
with tempfile.TemporaryDirectory() as surveydir:
files = [
'calib', 'gaia', 'images', 'survey-bricks.fits.gz',
'tycho2.kd.fits'
]
for fn in files:
src = os.path.join(indir, fn)
dst = os.path.join(surveydir, fn)
#trymakedirs(dst, dir=True)
print('Copy', src, dst)
if os.path.isfile(src):
shutil.copy(src, dst)
else:
shutil.copytree(src, dst)
from legacypipe.create_kdtrees import create_kdtree
infn = os.path.join(indir, 'survey-ccds-1.fits.gz')
outfn = os.path.join(surveydir, 'survey-ccds-1.kd.fits')
create_kdtree(infn, outfn, False)
os.environ['TYCHO2_KD_DIR'] = surveydir
outdir = 'out-testcase6-kd'
main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--no-gaia',
'--survey-dir', surveydir, '--outdir', outdir
])
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0].strip() == 'PSF')
assert (T.type[1].strip() == 'PSF')
# There is a Tycho-2 star in the blob.
I = np.flatnonzero(T.ref_cat == 'T2')
assert (len(I) == 1)
assert (T.ref_id[I][0] == 1909016711)
cat = read_fits_catalog(T)
assert (len(cat) == 2)
assert (isinstance(cat[0], PointSource))
assert (isinstance(cat[1], PointSource))
cat, ivs = read_fits_catalog(T, invvars=True)
assert (len(cat) == 2)
assert (isinstance(cat[0], PointSource))
assert (isinstance(cat[1], PointSource))
cat2 = Catalog(*cat)
assert (len(ivs) == len(cat2.getParams()))
# test --fit-on-coadds
outdir = 'out-testcase6-coadds'
main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--no-gaia',
'--survey-dir', surveydir, '--fit-on-coadds', '--outdir', outdir
])
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0].strip() == 'PSF')
assert (T.type[1].strip() == 'PSF')
# There is a Tycho-2 star in the blob.
I = np.flatnonzero(T.ref_cat == 'T2')
assert (len(I) == 1)
assert (T.ref_id[I][0] == 1909016711)
del os.environ['TYCHO2_KD_DIR']
# test --skip-coadds
r = main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--no-gaia',
'--survey-dir', surveydir, '--outdir', outdir, '--skip-coadd'
])
assert (r == 0)
# test --skip
r = main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--no-gaia',
'--survey-dir', surveydir, '--outdir', outdir, '--skip'
])
assert (r == 0)
# NothingToDoError (neighbouring brick)
r = main(args=[
'--brick', '1102p240', '--zoom', '0', '100', '0', '100',
'--force-all', '--no-write', '--no-wise', '--no-gaia',
'--survey-dir', surveydir, '--outdir', outdir
])
assert (r == 0)
surveydir = os.path.join(os.path.dirname(__file__), 'testcase9')
# Test for some get_tractor_image kwargs
survey = LegacySurveyData(surveydir)
fakebrick = BrickDuck(9.1228, 3.3975, 'quack')
wcs = wcs_for_brick(fakebrick, W=100, H=100)
ccds = survey.ccds_touching_wcs(wcs)
ccd = ccds[0]
im = survey.get_image_object(ccd)
H, W = wcs.shape
targetrd = np.array([
wcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
tim = im.get_tractor_image(radecpoly=targetrd)
assert (tim.getImage() is not None)
assert (tim.getInvError() is not None)
assert (tim.dq is not None)
tim2 = im.get_tractor_image(radecpoly=targetrd, pixels=False)
assert (np.all(tim2.getImage() == 0.))
tim4 = im.get_tractor_image(radecpoly=targetrd, invvar=False)
u = np.unique(tim4.inverr)
assert (len(u) == 1)
u = u[0]
target = tim4.zpscale / tim4.sig1
assert (np.abs(u / target - 1.) < 0.001)
tim3 = im.get_tractor_image(radecpoly=targetrd, invvar=False, dq=False)
assert (not hasattr(tim3, 'dq'))
tim5 = im.get_tractor_image(radecpoly=targetrd, gaussPsf=True)
print(tim5.getPsf())
assert (isinstance(tim5.getPsf(), GaussianMixturePSF))
surveydir = os.path.join(os.path.dirname(__file__), 'testcase12')
os.environ['TYCHO2_KD_DIR'] = surveydir
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
os.environ['UNWISE_MODEL_SKY_DIR'] = os.path.join(surveydir, 'images',
'unwise-mod')
#python legacypipe/runbrick.py --radec --width 100 --height 100 --outdir dup5b --survey-dir test/testcase12 --force-all --no-wise
unwdir = os.path.join(surveydir, 'images', 'unwise')
main(args=[
'--radec', '346.684', '12.791', '--width', '100', '--height', '100',
'--no-wise-ceres', '--unwise-dir', unwdir, '--survey-dir', surveydir,
'--outdir', 'out-testcase12', '--skip-coadd', '--force-all'
])
# --plots for stage_wise_forced
main(args=[
'--radec', '346.684', '12.791', '--width', '100', '--height', '100',
'--no-wise-ceres', '--unwise-dir', unwdir, '--survey-dir', surveydir,
'--outdir', 'out-testcase12', '--stage', 'wise_forced', '--plots'
])
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
del os.environ['TYCHO2_KD_DIR']
del os.environ['UNWISE_MODEL_SKY_DIR']
M = fitsio.read(
'out-testcase12/coadd/cus/custom-346684p12791/legacysurvey-custom-346684p12791-maskbits.fits.fz'
)
# Count masked & unmasked bits (the cluster splits this 100x100 field)
from collections import Counter
c = Counter(M.ravel())
from legacypipe.bits import MASKBITS
assert (c[0] >= 4000)
assert (c[MASKBITS['CLUSTER']] >= 4000)
surveydir = os.path.join(os.path.dirname(__file__), 'testcase9')
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
os.environ['LARGEGALAXIES_CAT'] = os.path.join(surveydir,
'sga-sub.kd.fits')
main(args=[
'--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
'--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir',
surveydir, '--outdir', 'out-testcase9', '--skip', '--force-all',
'--ps', 'tc9-ps.fits', '--ps-t0',
str(int(time.time()))
])
# (omit --force-all --no-write... reading from pickles below!)
# Test with --apodize
main(args=[
'--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
'--old-calibs-ok', '--no-wise', '--force-all', '--no-write',
'--survey-dir', surveydir, '--outdir', 'out-testcase9-ap', '--apodize'
])
main(args=[
'--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
'--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir',
surveydir, '--outdir', 'out-testcase9', '--plots', '--stage', 'halos'
])
main(args=[
'--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
'--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir',
surveydir, '--outdir', 'out-testcase9-coadds', '--stage',
'image_coadds', '--blob-image'
])
T = fits_table(
'out-testcase9/tractor/cus/tractor-custom-009122p03397.fits')
assert (len(T) == 4)
# Gaia star becomes a DUP!
assert (np.sum([t == 'DUP' for t in T.type]) == 1)
# LSLGA galaxy exists!
Igal = np.flatnonzero([r == 'L3' for r in T.ref_cat])
assert (len(Igal) == 1)
assert (np.all(T.ref_id[Igal] > 0))
assert (T.type[Igal[0]] == 'SER')
# --brick and --zoom rather than --radec --width --height
main(args=[
'--survey-dir', surveydir, '--outdir', 'out-testcase9b', '--zoom',
'1950', '2050', '340', '440', '--brick', '0091p035', '--force-all'
])
# test forced phot??
shutil.copy('test/testcase9/survey-bricks.fits.gz', 'out-testcase9b')
forced_main(args=[
'--survey-dir', surveydir, '--no-ceres', '--catalog-dir',
'out-testcase9b', '372546', 'N26', 'forced1.fits'
])
assert (os.path.exists('forced1.fits'))
_ = fits_table('forced1.fits')
# ... more tests...!
forced_main(args=[
'--survey-dir', surveydir, '--no-ceres', '--catalog-dir',
'out-testcase9b', '--derivs', '--threads', '2', '--apphot', '372547',
'N26', 'forced2.fits'
])
assert (os.path.exists('forced2.fits'))
_ = fits_table('forced2.fits')
forced_main(args=[
'--survey-dir', surveydir, '--no-ceres', '--catalog-dir',
'out-testcase9b', '--agn', '257266', 'S21', 'forced3.fits'
])
assert (os.path.exists('forced3.fits'))
_ = fits_table('forced3.fits')
if ceres:
forced_main(args=[
'--survey-dir', surveydir, '--catalog-dir', 'out-testcase9b',
'--derivs', '--threads', '2', '--apphot', '372546', 'N26',
'forced4.fits'
])
assert (os.path.exists('forced4.fits'))
_ = fits_table('forced4.fits')
# Test cache_dir
with tempfile.TemporaryDirectory() as cachedir, \
tempfile.TemporaryDirectory() as tempsurveydir:
files = []
for dirpath, _, filenames in os.walk(surveydir):
for fn in filenames:
path = os.path.join(dirpath, fn)
relpath = os.path.relpath(path, surveydir)
files.append(relpath)
# cache or no?
files.sort()
files_cache = files[::2]
files_nocache = files[1::2]
# Survey-ccds *must* be in nocache.
fn = 'survey-ccds-1.kd.fits'
if fn in files_cache:
files_cache.remove(fn)
files_nocache.append(fn)
for fn in files_cache:
src = os.path.join(surveydir, fn)
dst = os.path.join(cachedir, fn)
trymakedirs(dst, dir=True)
print('Copy', src, dst)
shutil.copy(src, dst)
for fn in files_nocache:
src = os.path.join(surveydir, fn)
dst = os.path.join(tempsurveydir, fn)
trymakedirs(dst, dir=True)
print('Copy', src, dst)
shutil.copy(src, dst)
main(args=[
'--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
'--no-wise', '--survey-dir', tempsurveydir, '--cache-dir',
cachedir, '--outdir', 'out-testcase9cache', '--force-all'
])
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
del os.environ['LARGEGALAXIES_CAT']
# if ceres:
# surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
# main(args=['--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
# '--no-wise', '--force-all', '--no-write', '--ceres',
# '--survey-dir', surveydir,
# '--outdir', 'out-testcase3-ceres',
# '--no-depth-cut'])
# MzLS + BASS data
# python legacypipe/runbrick.py --run north --brick 1773p595 --zoom 1300 1500 700 900 --survey-dir dr9-north -s coadds
# fitscopy coadd/177/1773p595/legacysurvey-1773p595-ccds.fits"[#row<3 || #row==12]" cx.fits
# python legacyanalysis/create_testcase.py cx.fits test/mzlsbass2 1773p595 --survey-dir dr9-north/ --fpack
surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass2')
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir2, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
main(args=[
'--brick', '1773p595', '--zoom', '1300', '1500', '700', '900',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir2,
'--outdir', 'out-mzlsbass2'
])
T = fits_table('out-mzlsbass2/tractor/177/tractor-1773p595.fits')
assert (np.sum(T.ref_cat == 'G2') == 3)
assert (np.sum(T.ref_id > 0) == 3)
# Test --max-blobsize, --checkpoint, --bail-out
outdir = 'out-mzlsbass2b'
chk = 'checkpoint-mzb2b.p'
if os.path.exists(chk):
os.unlink(chk)
main(args=[
'--brick', '1773p595', '--zoom', '1300', '1500', '700', '900',
'--no-wise', '--force-all', '--stage', 'fitblobs', '--write-stage',
'srcs', '--survey-dir', surveydir2, '--outdir', outdir, '--checkpoint',
chk, '--nblobs', '3'
])
# err... --max-blobsize does not result in bailed-out blobs masked,
# because it treats large blobs as *completed*...
#'--max-blobsize', '3000',
outdir = 'out-mzlsbass2c'
main(args=[
'--brick', '1773p595', '--zoom', '1300', '1500', '700', '900',
'--no-wise', '--force-all', '--survey-dir', surveydir2, '--outdir',
outdir, '--bail-out', '--checkpoint', chk, '--no-write'
])
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
M = fitsio.read(
os.path.join(outdir, 'coadd', '177', '1773p595',
'legacysurvey-1773p595-maskbits.fits.fz'))
assert (np.sum((M & MASKBITS['BAILOUT']) > 0) >= 1000)
# Test RexGalaxy
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6-rex'
the_args = [
'--brick',
'1102p240',
'--zoom',
'500',
'600',
'650',
'750',
'--force-all',
'--no-write',
'--no-wise',
'--skip-calibs',
#'--rex', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
]
print('python legacypipe/runbrick.py', ' '.join(the_args))
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
main(args=the_args)
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0].strip() == 'PSF')
cmd = (
'(cd %s && sha256sum -c %s)' %
(outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum')))
print(cmd)
rtn = os.system(cmd)
assert (rtn == 0)
# Test with a Tycho-2 star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6'
main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--survey-dir', surveydir,
'--outdir', outdir
])
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0].strip() == 'PSF')
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
# Test that we can run splinesky calib if required...
from legacypipe.decam import DecamImage
DecamImage.splinesky_boxsize = 128
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
survey = LegacySurveyData(surveydir)
# get brick by id
brickid = 473357
brick = survey.get_brick(brickid)
assert (brick.brickname == '1867p255')
assert (brick.brickid == brickid)
outdir = 'out-testcase4'
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
fn = os.path.join(surveydir, 'calib', 'sky-single', 'decam', 'CP',
'V4.8.2', 'CP20170315', 'c4d_170316_062107_ooi_z_ls9',
'c4d_170316_062107_ooi_z_ls9-N2-splinesky.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
os.path.join(surveydir, 'images',
'unwise-tr'), '--blob-image', '--no-hybrid-psf',
'--survey-dir', surveydir, '--outdir', outdir, '-v', '--no-wise-ceres'
])
print('Checking for calib file', fn)
assert (os.path.exists(fn))
# Test with blob-masking when creating sky calib.
os.unlink(fn)
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--blob-mask-dir',
surveydir, '--survey-dir', surveydir, '--stage', 'image_coadds',
'--outdir', 'out-testcase4b', '--plots'
])
print('Checking for calib file', fn)
assert (os.path.exists(fn))
if ceres:
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
os.path.join(surveydir, 'images', 'unwise-tr'), '--survey-dir',
surveydir, '--outdir', outdir
])
if psfex:
# Check that we can regenerate PsfEx files if necessary.
fn = os.path.join(surveydir, 'calib', 'psfex', 'decam', 'CP', 'V4.8.2',
'CP20170315',
'c4d_170316_062107_ooi_z_ls9-psfex.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
os.path.join(surveydir, 'images', 'unwise-tr'), '--blob-image',
'--survey-dir', surveydir, '--outdir', outdir, '-v'
])
print('After generating PsfEx calib:')
os.system('find %s' % (os.path.join(surveydir, 'calib')))
# Wrap-around, hybrid PSF
surveydir = os.path.join(os.path.dirname(__file__), 'testcase8')
outdir = 'out-testcase8'
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
main(args=[
'--brick',
'1209p050',
'--zoom',
'720',
'1095',
'3220',
'3500',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
])
# Test with a Tycho-2 star + another saturated star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase7')
outdir = 'out-testcase7'
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
main(args=[
'--brick',
'1102p240',
'--zoom',
'250',
'350',
'1550',
'1650',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
])
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 4)
# Check skipping blobs outside the brick's unique area.
# (this now doesn't detect any sources at all, reasonably)
# surveydir = os.path.join(os.path.dirname(__file__), 'testcase5')
# outdir = 'out-testcase5'
#
# fn = os.path.join(outdir, 'tractor', '186', 'tractor-1867p255.fits')
# if os.path.exists(fn):
# os.unlink(fn)
#
# main(args=['--brick', '1867p255', '--zoom', '0', '150', '0', '150',
# '--force-all', '--no-write', '--coadd-bw',
# '--survey-dir', surveydir,
# '--early-coadds',
# '--outdir', outdir] + extra_args)
#
# assert(os.path.exists(fn))
# T = fits_table(fn)
# assert(len(T) == 1)
# Custom RA,Dec; blob ra,dec.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
outdir = 'out-testcase4b'
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
# Catalog written with one entry (--blobradec)
fn = os.path.join(outdir, 'tractor', 'cus',
'tractor-custom-186743p25461.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--radec', '186.743965', '25.461788', '--width', '250', '--height',
'250', '--force-all', '--no-write', '--no-wise', '--blobradec',
'186.740369', '25.453855', '--survey-dir', surveydir, '--outdir',
outdir
])
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 1)
surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
outdir = 'out-testcase3'
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
checkpoint_fn = os.path.join(outdir, 'checkpoint.pickle')
if os.path.exists(checkpoint_fn):
os.unlink(checkpoint_fn)
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
])
# Read catalog into Tractor sources to test read_fits_catalog
survey = LegacySurveyData(survey_dir=outdir)
fn = survey.find_file('tractor', brick='2447p120')
print('Checking', fn)
T = fits_table(fn)
cat = read_fits_catalog(T)
print('Read catalog:', cat)
assert (len(cat) == 2)
src = cat[1]
print('Source0', src)
from tractor.sersic import SersicGalaxy
assert (type(src) in [DevGalaxy, SersicGalaxy])
assert (np.abs(src.pos.ra - 244.77973) < 0.00001)
assert (np.abs(src.pos.dec - 12.07234) < 0.00002)
src = cat[0]
print('Source1', src)
assert (type(src) == PointSource)
assert (np.abs(src.pos.ra - 244.77828) < 0.00001)
assert (np.abs(src.pos.dec - 12.07250) < 0.00001)
# Check that we can run again, using that checkpoint file.
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
])
# Assert...... something?
# Test --checkpoint without --threads
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1'
])
# From Kaylan's Bootes pre-DR4 run
# surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass3')
# main(args=['--brick', '2173p350', '--zoom', '100', '200', '100', '200',
# '--no-wise', '--force-all', '--no-write',
# '--survey-dir', surveydir2,
# '--outdir', 'out-mzlsbass3'] + extra_args)
# With plots!
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', 'out-testcase3', '--plots', '--nblobs', '1'
])
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
])
# Read catalog into Tractor sources to test read_fits_catalog
from legacypipe.catalog import read_fits_catalog
from legacypipe.survey import LegacySurveyData
from astrometry.util.fits import fits_table
from tractor.galaxy import DevGalaxy
from tractor import PointSource
survey = LegacySurveyData(survey_dir=outdir)
fn = survey.find_file('tractor', brick='2447p120')
T = fits_table(fn)
cat = read_fits_catalog(T)
print('Read catalog:', cat)
assert (len(cat) == 2)
src = cat[0]
assert (type(src) == DevGalaxy)
assert (np.abs(src.pos.ra - 244.77975) < 0.00001)
assert (np.abs(src.pos.dec - 12.07234) < 0.00001)
src = cat[1]
assert (type(src) == PointSource)
assert (np.abs(src.pos.ra - 244.77833) < 0.00001)
assert (np.abs(src.pos.dec - 12.07252) < 0.00001)
# DevGalaxy(pos=RaDecPos[244.77975494973529, 12.072348111713127], brightness=NanoMaggies: g=19.2, r=17.9, z=17.1, shape=re=2.09234, e1=-0.198453, e2=0.023652,
# PointSource(RaDecPos[244.77833280764278, 12.072521274981987], NanoMaggies: g=25, r=23, z=21.7)
# Check that we can run again, using that checkpoint file.
main(args=[
def run_one_ccd(survey, catsurvey_north, catsurvey_south, resolve_dec, ccd,
opt, zoomslice, ps):
tlast = Time()
im = survey.get_image_object(ccd)
if opt.do_calib:
im.run_calibs(splinesky=True)
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
splinesky=True,
constant_invvar=opt.constant_invvar,
hybridPsf=opt.hybrid_psf,
normalizePsf=opt.normalize_psf,
old_calibs_ok=True)
print('Got tim:', tim, 'x0,y0', tim.x0, tim.y0)
tnow = Time()
print('Read image:', tnow - tlast)
tlast = tnow
# Apply outlier masks
if True:
# Outliers masks are computed within a survey (north/south for dr8), and are stored
# in a brick-oriented way, in the results directories.
north_ccd = (ccd.camera.strip() != 'decam')
catsurvey = catsurvey_north
if not north_ccd and catsurvey_south is not None:
catsurvey = catsurvey_south
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
for b in bricks:
from legacypipe.outliers import read_outlier_mask_file
print('Reading outlier mask for brick', b.brickname)
ok = read_outlier_mask_file(catsurvey, [tim],
b.brickname,
subimage=False,
output=False,
ps=ps)
if not ok:
print('WARNING: failed to read outliers mask file for brick',
b.brickname)
if opt.catalog:
T = fits_table(opt.catalog)
else:
chipwcs = tim.subwcs
T = get_catalog_in_wcs(chipwcs,
catsurvey_north,
catsurvey_south=catsurvey_south,
resolve_dec=resolve_dec)
if T is None:
print('No sources to photometer.')
return None
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
surveydir = survey.get_survey_dir()
del survey
if opt.move_gaia:
# Gaia stars: move RA,Dec to the epoch of this image.
I = np.flatnonzero(T.ref_epoch > 0)
if len(I):
from legacypipe.survey import radec_at_mjd
print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd())
ra, dec = radec_at_mjd(T.ra[I], T.dec[I],
T.ref_epoch[I].astype(float), T.pmra[I],
T.pmdec[I], T.parallax[I], tim.time.toMjd())
T.ra[I] = ra
T.dec[I] = dec
tnow = Time()
print('Read catalog:', tnow - tlast)
tlast = tnow
cat = read_fits_catalog(T, bands='r')
# Replace the brightness (which will be a NanoMaggies with g,r,z)
# with a NanoMaggies with this image's band only.
for src in cat:
src.brightness = NanoMaggies(**{tim.band: 1.})
tnow = Time()
print('Parse catalog:', tnow - tlast)
tlast = tnow
print('Forced photom...')
F = run_forced_phot(cat,
tim,
ceres=opt.ceres,
derivs=opt.derivs,
fixed_also=True,
agn=opt.agn,
do_forced=opt.forced,
do_apphot=opt.apphot,
get_model=opt.save_model,
ps=ps,
timing=True,
ceres_threads=opt.ceres_threads)
if opt.save_model:
# unpack results
F, model_img = F
F.release = T.release
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.camera = np.array([ccd.camera] * len(F))
F.expnum = np.array([im.expnum] * len(F)).astype(np.int64)
F.ccdname = np.array([im.ccdname] * len(F))
# "Denormalizing"
F.filter = np.array([tim.band] * len(F))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(F))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(F)).astype(np.float32)
F.psfsize = np.array([tim.psf_fwhm * tim.imobj.pixscale] * len(F)).astype(
np.float32)
F.ccd_cuts = np.array([ccd.ccd_cuts] * len(F))
F.airmass = np.array([ccd.airmass] * len(F))
### --> also add units to the dict below so the FITS headers have units
F.sky = np.array([tim.midsky / tim.zpscale / tim.imobj.pixscale**2] *
len(F)).astype(np.float32)
# in the same units as the depth maps -- flux inverse-variance.
F.psfdepth = np.array([(1. / (tim.sig1 / tim.psfnorm)**2)] *
len(F)).astype(np.float32)
F.galdepth = np.array([(1. / (tim.sig1 / tim.galnorm)**2)] *
len(F)).astype(np.float32)
# F.psfdepth = np.array([-2.5 * (np.log10(5. * tim.sig1 / tim.psfnorm) - 9)] * len(F)).astype(np.float32)
# F.galdepth = np.array([-2.5 * (np.log10(5. * tim.sig1 / tim.galnorm) - 9)] * len(F)).astype(np.float32)
# super units questions here
if opt.derivs:
cosdec = np.cos(np.deg2rad(T.dec))
F.dra = (F.flux_dra / F.flux) * 3600. / cosdec
F.ddec = (F.flux_ddec / F.flux) * 3600.
F.dra_ivar = F.flux_dra_ivar * (F.flux / 3600. * cosdec)**2
F.ddec_ivar = F.flux_ddec_ivar * (F.flux / 3600.)**2
F.delete_column('flux_dra')
F.delete_column('flux_ddec')
F.delete_column('flux_dra_ivar')
F.delete_column('flux_ddec_ivar')
F.flux = F.flux_fixed
F.flux_ivar = F.flux_fixed_ivar
F.delete_column('flux_fixed')
F.delete_column('flux_fixed_ivar')
for c in ['dra', 'ddec', 'dra_ivar', 'ddec_ivar', 'flux', 'flux_ivar']:
F.set(c, F.get(c).astype(np.float32))
F.ra = T.ra
F.dec = T.dec
ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
h, w = tim.shape
F.dqmask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h - 1),
np.clip(np.round(F.x).astype(int), 0, w - 1)]
program_name = sys.argv[0]
## FIXME -- from catalog?
release = 8002
version_hdr = get_version_header(program_name, surveydir, release)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn.strip())
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(
dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(
dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='PLVER', value=ccd.plver, comment='CP pipeline version'))
version_hdr.add_record(
dict(name='PLPROCID', value=ccd.plprocid, comment='CP pipeline id'))
version_hdr.add_record(
dict(name='PROCDATE', value=ccd.procdate, comment='CP image DATE'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
fitsio.write(opt.save_model, model_img, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
tnow = Time()
print('Forced phot:', tnow - tlast)
return F, version_hdr
def wise_cutouts(ra,
dec,
radius,
ps,
pixscale=2.75,
survey_dir=None,
unwise_dir=None):
'''
radius in arcsec.
pixscale: WISE pixel scale in arcsec/pixel;
make this smaller than 2.75 to oversample.
'''
if unwise_dir is None:
unwise_dir = os.environ.get('UNWISE_COADDS_DIR')
npix = int(np.ceil(radius / pixscale))
print('Image size:', npix)
W = H = npix
pix = pixscale / 3600.
wcs = Tan(ra, dec, (W + 1) / 2., (H + 1) / 2., -pix, 0., 0., pix, float(W),
float(H))
# Find DECaLS bricks overlapping
survey = LegacySurveyData(survey_dir=survey_dir)
B = bricks_touching_wcs(wcs, survey=survey)
print('Found', len(B), 'bricks overlapping')
TT = []
for b in B.brickname:
fn = survey.find_file('tractor', brick=b)
T = fits_table(fn)
print('Read', len(T), 'from', b)
primhdr = fitsio.read_header(fn)
TT.append(T)
T = merge_tables(TT)
print('Total of', len(T), 'sources')
T.cut(T.brick_primary)
print(len(T), 'primary')
margin = 20
ok, xx, yy = wcs.radec2pixelxy(T.ra, T.dec)
I = np.flatnonzero((xx > -margin) * (yy > -margin) * (xx < W + margin) *
(yy < H + margin))
T.cut(I)
print(len(T), 'within ROI')
#return wcs,T
# Pull out DECaLS coadds (image, model, resid).
dwcs = wcs.scale(2. * pixscale / 0.262)
dh, dw = dwcs.shape
print('DECaLS resampled shape:', dh, dw)
tags = ['image', 'model', 'resid']
coimgs = [np.zeros((dh, dw, 3), np.uint8) for t in tags]
for b in B.brickname:
fn = survey.find_file('image', brick=b, band='r')
bwcs = Tan(fn, 1) # ext 1: .fz
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(dwcs, bwcs)
except ResampleError:
continue
if len(Yo) == 0:
continue
print('Resampling', len(Yo), 'pixels from', b)
xl, xh, yl, yh = Xi.min(), Xi.max(), Yi.min(), Yi.max()
#print('python legacypipe/runbrick.py -b %s --zoom %i %i %i %i --outdir cluster --pixpsf --splinesky --pipe --no-early-coadds' %
# (b, xl-5, xh+5, yl-5, yh+5) + ' -P \'pickles/cluster-%(brick)s-%%(stage)s.pickle\'')
for i, tag in enumerate(tags):
fn = survey.find_file(tag + '-jpeg', brick=b)
img = plt.imread(fn)
img = np.flipud(img)
coimgs[i][Yo, Xo, :] = img[Yi, Xi, :]
tt = dict(image='Image', model='Model', resid='Resid')
for img, tag in zip(coimgs, tags):
plt.clf()
dimshow(img, ticks=False)
plt.title('DECaLS grz %s' % tt[tag])
ps.savefig()
# Find unWISE tiles overlapping
tiles = unwise_tiles_touching_wcs(wcs)
print('Cut to', len(tiles), 'unWISE tiles')
# Here we assume the targetwcs is axis-aligned and that the
# edge midpoints yield the RA,Dec limits (true for TAN).
r, d = wcs.pixelxy2radec(np.array([1, W, W / 2, W / 2]),
np.array([H / 2, H / 2, 1, H]))
# the way the roiradec box is used, the min/max order doesn't matter
roiradec = [r[0], r[1], d[2], d[3]]
ra, dec = T.ra, T.dec
srcs = read_fits_catalog(T)
wbands = [1, 2, 3, 4]
wanyband = 'w'
for band in wbands:
f = T.get('flux_w%i' % band)
f *= 10.**(primhdr['WISEAB%i' % band] / 2.5)
coimgs = [np.zeros((H, W), np.float32) for b in wbands]
comods = [np.zeros((H, W), np.float32) for b in wbands]
con = [np.zeros((H, W), np.uint8) for b in wbands]
for iband, band in enumerate(wbands):
print('Photometering WISE band', band)
wband = 'w%i' % band
for i, src in enumerate(srcs):
#print('Source', src, 'brightness', src.getBrightness(), 'params', src.getBrightness().getParams())
#src.getBrightness().setParams([T.wise_flux[i, band-1]])
src.setBrightness(
NanoMaggies(**{wanyband: T.get('flux_w%i' % band)[i]}))
# print('Set source brightness:', src.getBrightness())
# The tiles have some overlap, so for each source, keep the
# fit in the tile whose center is closest to the source.
for tile in tiles:
print('Reading tile', tile.coadd_id)
tim = get_unwise_tractor_image(unwise_dir,
tile.coadd_id,
band,
bandname=wanyband,
roiradecbox=roiradec)
if tim is None:
print('Actually, no overlap with tile', tile.coadd_id)
continue
print('Read image with shape', tim.shape)
# Select sources in play.
wisewcs = tim.wcs.wcs
H, W = tim.shape
ok, x, y = wisewcs.radec2pixelxy(ra, dec)
x = (x - 1.).astype(np.float32)
y = (y - 1.).astype(np.float32)
margin = 10.
I = np.flatnonzero((x >= -margin) * (x < W + margin) *
(y >= -margin) * (y < H + margin))
print(len(I), 'within the image + margin')
subcat = [srcs[i] for i in I]
tractor = Tractor([tim], subcat)
mod = tractor.getModelImage(0)
# plt.clf()
# dimshow(tim.getImage(), ticks=False)
# plt.title('WISE %s %s' % (tile.coadd_id, wband))
# ps.savefig()
# plt.clf()
# dimshow(mod, ticks=False)
# plt.title('WISE %s %s' % (tile.coadd_id, wband))
# ps.savefig()
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(wcs, tim.wcs.wcs)
except ResampleError:
continue
if len(Yo) == 0:
continue
print('Resampling', len(Yo), 'pixels from WISE', tile.coadd_id,
band)
coimgs[iband][Yo, Xo] += tim.getImage()[Yi, Xi]
comods[iband][Yo, Xo] += mod[Yi, Xi]
con[iband][Yo, Xo] += 1
for img, mod, n in zip(coimgs, comods, con):
img /= np.maximum(n, 1)
mod /= np.maximum(n, 1)
for band, img, mod in zip(wbands, coimgs, comods):
lo, hi = np.percentile(img, [25, 99])
plt.clf()
dimshow(img, vmin=lo, vmax=hi, ticks=False)
plt.title('WISE W%i Data' % band)
ps.savefig()
plt.clf()
dimshow(mod, vmin=lo, vmax=hi, ticks=False)
plt.title('WISE W%i Model' % band)
ps.savefig()
resid = img - mod
mx = np.abs(resid).max()
plt.clf()
dimshow(resid, vmin=-mx, vmax=mx, ticks=False)
plt.title('WISE W%i Resid' % band)
ps.savefig()
#kwa = dict(mn=-0.1, mx=2., arcsinh = 1.)
kwa = dict(mn=-0.1, mx=2., arcsinh=None)
rgb = _unwise_to_rgb(coimgs[:2], **kwa)
plt.clf()
dimshow(rgb, ticks=False)
plt.title('WISE W1/W2 Data')
ps.savefig()
rgb = _unwise_to_rgb(comods[:2], **kwa)
plt.clf()
dimshow(rgb, ticks=False)
plt.title('WISE W1/W2 Model')
ps.savefig()
kwa = dict(mn=-1, mx=1, arcsinh=None)
rgb = _unwise_to_rgb(
[img - mod for img, mod in list(zip(coimgs, comods))[:2]], **kwa)
plt.clf()
dimshow(rgb, ticks=False)
plt.title('WISE W1/W2 Resid')
ps.savefig()
return wcs, T
def stage_1(expnum=431202,
extname='S19',
plotprefix='lsb',
plots=False,
brightstars='bright.fits',
pixscale=0.27,
**kwa):
if plots:
ps = PlotSequence(plotprefix)
else:
ps = None
survey = LegacySurveyData()
C = survey.find_ccds(expnum=expnum, ccdname=extname)
print len(C), 'CCDs'
im = survey.get_image_object(C[0])
print 'im', im
#(x0,x1,y0,y1) = opt.zoom
#zoomslice = (slice(y0,y1), slice(x0,x1))
zoomslice = None
#tim = im.get_tractor_image(gaussPsf=True, splinesky=True, slc=zoomslice)
tim = im.get_tractor_image(hybridPsf=True, splinesky=True, slc=zoomslice)
print 'Tim', tim
cats = []
bricks = bricks_touching_wcs(tim.subwcs, survey=survey)
bricknames = bricks.brickname
for b in bricknames:
fn = survey.find_file('tractor', brick=b)
if not os.path.exists(fn):
print 'WARNING: file does not exist:', fn
continue
print 'Reading', fn
cat = fits_table(fn)
print 'Read', len(cat), 'sources'
if cat is None or len(cat) == 0:
continue
cats.append(cat)
if len(cats):
T = merge_tables(cats)
T._header = cats[0]._header
# margin
M = 20
ok, x, y = tim.subwcs.radec2pixelxy(T.ra, T.dec)
x -= 1.
y -= 1.
T.x = x
T.y = y
H, W = tim.shape
T.cut((x > -M) * (x < (W + M)) * (y > -M) * (y < (H + M)))
print 'Cut to', len(T), 'within image bounds'
T.cut(T.brick_primary)
print 'Cut to', len(T), 'brick_primary'
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print 'Cut to', len(T), 'not out-of-bound or left-blob'
print 'Brightest z-band:', np.max(T.decam_flux[:, 4])
print 'Brightest r-band:', np.max(T.decam_flux[:, 2])
orig_catalog = T.copy()
# Cut to compact sources
T.cut(np.maximum(T.shapeexp_r, T.shapedev_r) < 3.)
print 'Cut to', len(T), 'compact catalog objects'
cat = read_fits_catalog(T, allbands='ugrizY')
else:
cat = []
orig_catalog = fits_table()
print len(cat), 'catalog objects'
if plots:
plt.clf()
img = tim.getImage()
mn, mx = np.percentile(img.ravel(), [25, 99])
print('Image plot range:', mn, mx)
tim.ima = dict(interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
plt.imshow(tim.getImage(), **tim.ima)
plt.title('Orig data')
ps.savefig()
# Mask out bright pixels.
mask = np.zeros(tim.shape, np.bool)
bright = fits_table(brightstars)
print 'Read', len(bright), 'SDSS bright stars'
ok, bx, by = tim.subwcs.radec2pixelxy(bright.ra, bright.dec)
bx = np.round(bx).astype(int)
by = np.round(by).astype(int)
H, W = mask.shape
bright.modelmag = 22.5 - 2.5 * np.log10(bright.modelflux)
mag = bright.modelmag[:, 2]
radius = (10.**(3.5 - 0.15 * mag) / pixscale).astype(np.int)
I = np.flatnonzero(ok * (radius > 0) * (bx + radius > 0) *
(bx - radius < W) * (by + radius > 0) *
(by - radius < H))
print len(I), 'bright stars are near the image'
xx, yy = np.meshgrid(np.arange(W), np.arange(H))
for x, y, r in zip(bx[I], by[I], radius[I]):
mask[(xx - x)**2 + (yy - y)**2 < r**2] = True
mask[tim.inverr == 0] = True
tim.inverr[mask] = 0.
tim.data[mask] = 0.
if plots:
plt.clf()
plt.imshow(mask,
interpolation='nearest',
origin='lower',
vmin=0,
vmax=1,
cmap='gray')
plt.title('Mask')
ps.savefig()
plt.clf()
plt.imshow(tim.getImage(), **tim.ima)
plt.title('Masked')
ps.savefig()
tr = Tractor([tim], cat)
mod = tr.getModelImage(tim)
print('Model range:', mod.min(), mod.max())
# print('Model counts:', [tim.getPhotoCal().brightnessToCounts(src.getBrightness())
# for src in cat])
# print('Catalog:')
# for src in cat[:10]:
# print(' ', src)
if False:
# OLD DEBUGGING
print 'Model median:', np.median(mod)
rawimg = fitsio.read(
'decals-lsb/images/decam/CP20150407/c4d_150410_035040_ooi_z_v1.fits.fz',
ext=im.hdu)
print 'Image median:', np.median(rawimg)
print 'mid sky', tim.midsky
rawmod = mod * tim.zpscale + tim.midsky
print 'Model median:', np.median(rawmod)
fitsio.write('model.fits', rawmod, clobber=True)
if plots:
plt.clf()
plt.imshow(mod, **tim.ima)
plt.title('Model')
ps.savefig()
ax = plt.axis()
ok, xx, yy = tim.subwcs.radec2pixelxy(
[src.getPosition().ra for src in cat],
[src.getPosition().dec for src in cat])
plt.plot(xx, yy, 'r+')
plt.axis(ax)
ps.savefig()
mod[mask] = 0.
if plots:
plt.clf()
plt.imshow(mod, **tim.ima)
plt.title('Masked model')
ps.savefig()
imchi = dict(interpolation='nearest',
origin='lower',
vmin=-5,
vmax=5,
cmap='RdBu')
# plt.clf()
# plt.imshow((tim.getImage() - mod) * tim.getInvError(), **imchi)
# plt.title('Chi')
# plt.colorbar()
# ps.savefig()
plt.clf()
plt.imshow((tim.getImage() - mod), **tim.ima)
plt.title('Residuals')
ps.savefig()
resid = tim.getImage() - mod
sky = np.zeros_like(resid)
median_smooth(resid, mask, 256, sky)
if plots:
plt.clf()
plt.imshow(sky, **tim.ima)
plt.title('Smoothed residuals (sky)')
ps.savefig()
resid -= sky
# Re-apply mask
resid[mask] = 0.
if plots:
plt.clf()
plt.imshow(resid, **tim.ima)
plt.title('Residual - sky')
ps.savefig()
return dict(resid=resid,
sky=sky,
ps=ps,
tim=tim,
tr=tr,
mod=mod,
mask=mask,
orig_catalog=orig_catalog,
pixscale=pixscale)
def wise_cutouts(ra, dec, radius, ps, pixscale=2.75, tractor_base='.',
unwise_dir='unwise-coadds'):
'''
radius in arcsec.
pixscale: WISE pixel scale in arcsec/pixel;
make this smaller than 2.75 to oversample.
'''
npix = int(np.ceil(radius / pixscale))
print('Image size:', npix)
W = H = npix
pix = pixscale / 3600.
wcs = Tan(ra, dec, (W+1)/2., (H+1)/2., -pix, 0., 0., pix,float(W),float(H))
# Find DECaLS bricks overlapping
survey = LegacySurveyData()
B = bricks_touching_wcs(wcs, survey=survey)
print('Found', len(B), 'bricks overlapping')
TT = []
for b in B.brickname:
fn = os.path.join(tractor_base, 'tractor', b[:3],
'tractor-%s.fits' % b)
T = fits_table(fn)
print('Read', len(T), 'from', b)
primhdr = fitsio.read_header(fn)
TT.append(T)
T = merge_tables(TT)
print('Total of', len(T), 'sources')
T.cut(T.brick_primary)
print(len(T), 'primary')
margin = 20
ok,xx,yy = wcs.radec2pixelxy(T.ra, T.dec)
I = np.flatnonzero((xx > -margin) * (yy > -margin) *
(xx < W+margin) * (yy < H+margin))
T.cut(I)
print(len(T), 'within ROI')
# Pull out DECaLS coadds (image, model, resid).
dwcs = wcs.scale(2. * pixscale / 0.262)
dh,dw = dwcs.shape
print('DECaLS resampled shape:', dh,dw)
tags = ['image', 'model', 'resid']
coimgs = [np.zeros((dh,dw,3), np.uint8) for t in tags]
for b in B.brickname:
fn = os.path.join(tractor_base, 'coadd', b[:3], b,
'legacysurvey-%s-image-r.fits' % b)
bwcs = Tan(fn)
try:
Yo,Xo,Yi,Xi,nil = resample_with_wcs(dwcs, bwcs)
except ResampleError:
continue
if len(Yo) == 0:
continue
print('Resampling', len(Yo), 'pixels from', b)
xl,xh,yl,yh = Xi.min(), Xi.max(), Yi.min(), Yi.max()
print('python legacypipe/runbrick.py -b %s --zoom %i %i %i %i --outdir cluster --pixpsf --splinesky --pipe --no-early-coadds' %
(b, xl-5, xh+5, yl-5, yh+5) + ' -P \'pickles/cluster-%(brick)s-%%(stage)s.pickle\'')
for i,tag in enumerate(tags):
fn = os.path.join(tractor_base, 'coadd', b[:3], b,
'legacysurvey-%s-%s.jpg' % (b, tag))
img = plt.imread(fn)
img = np.flipud(img)
coimgs[i][Yo,Xo,:] = img[Yi,Xi,:]
tt = dict(image='Image', model='Model', resid='Resid')
for img,tag in zip(coimgs, tags):
plt.clf()
dimshow(img, ticks=False)
plt.title('DECaLS grz %s' % tt[tag])
ps.savefig()
# Find unWISE tiles overlapping
tiles = unwise_tiles_touching_wcs(wcs)
print('Cut to', len(tiles), 'unWISE tiles')
# Here we assume the targetwcs is axis-aligned and that the
# edge midpoints yield the RA,Dec limits (true for TAN).
r,d = wcs.pixelxy2radec(np.array([1, W, W/2, W/2]),
np.array([H/2, H/2, 1, H ]))
# the way the roiradec box is used, the min/max order doesn't matter
roiradec = [r[0], r[1], d[2], d[3]]
ra,dec = T.ra, T.dec
srcs = read_fits_catalog(T)
wbands = [1,2]
wanyband = 'w'
for band in wbands:
T.wise_flux[:, band-1] *= 10.**(primhdr['WISEAB%i' % band] / 2.5)
coimgs = [np.zeros((H,W), np.float32) for b in wbands]
comods = [np.zeros((H,W), np.float32) for b in wbands]
con = [np.zeros((H,W), np.uint8) for b in wbands]
for iband,band in enumerate(wbands):
print('Photometering WISE band', band)
wband = 'w%i' % band
for i,src in enumerate(srcs):
#print('Source', src, 'brightness', src.getBrightness(), 'params', src.getBrightness().getParams())
#src.getBrightness().setParams([T.wise_flux[i, band-1]])
src.setBrightness(NanoMaggies(**{wanyband: T.wise_flux[i, band-1]}))
# print('Set source brightness:', src.getBrightness())
# The tiles have some overlap, so for each source, keep the
# fit in the tile whose center is closest to the source.
for tile in tiles:
print('Reading tile', tile.coadd_id)
tim = get_unwise_tractor_image(unwise_dir, tile.coadd_id, band,
bandname=wanyband,
roiradecbox=roiradec)
if tim is None:
print('Actually, no overlap with tile', tile.coadd_id)
continue
print('Read image with shape', tim.shape)
# Select sources in play.
wisewcs = tim.wcs.wcs
H,W = tim.shape
ok,x,y = wisewcs.radec2pixelxy(ra, dec)
x = (x - 1.).astype(np.float32)
y = (y - 1.).astype(np.float32)
margin = 10.
I = np.flatnonzero((x >= -margin) * (x < W+margin) *
(y >= -margin) * (y < H+margin))
print(len(I), 'within the image + margin')
subcat = [srcs[i] for i in I]
tractor = Tractor([tim], subcat)
mod = tractor.getModelImage(0)
# plt.clf()
# dimshow(tim.getImage(), ticks=False)
# plt.title('WISE %s %s' % (tile.coadd_id, wband))
# ps.savefig()
# plt.clf()
# dimshow(mod, ticks=False)
# plt.title('WISE %s %s' % (tile.coadd_id, wband))
# ps.savefig()
try:
Yo,Xo,Yi,Xi,nil = resample_with_wcs(wcs, tim.wcs.wcs)
except ResampleError:
continue
if len(Yo) == 0:
continue
print('Resampling', len(Yo), 'pixels from WISE', tile.coadd_id,
band)
coimgs[iband][Yo,Xo] += tim.getImage()[Yi,Xi]
comods[iband][Yo,Xo] += mod[Yi,Xi]
con [iband][Yo,Xo] += 1
for img,mod,n in zip(coimgs, comods, con):
img /= np.maximum(n, 1)
mod /= np.maximum(n, 1)
for band,img,mod in zip(wbands, coimgs, comods):
lo,hi = np.percentile(img, [25,99])
plt.clf()
dimshow(img, vmin=lo, vmax=hi, ticks=False)
plt.title('WISE W%i Data' % band)
ps.savefig()
plt.clf()
dimshow(mod, vmin=lo, vmax=hi, ticks=False)
plt.title('WISE W%i Model' % band)
ps.savefig()
resid = img - mod
mx = np.abs(resid).max()
plt.clf()
dimshow(resid, vmin=-mx, vmax=mx, ticks=False)
plt.title('WISE W%i Resid' % band)
ps.savefig()
#kwa = dict(mn=-0.1, mx=2., arcsinh = 1.)
kwa = dict(mn=-0.1, mx=2., arcsinh=None)
rgb = _unwise_to_rgb(coimgs, **kwa)
plt.clf()
dimshow(rgb, ticks=False)
plt.title('WISE W1/W2 Data')
ps.savefig()
rgb = _unwise_to_rgb(comods, **kwa)
plt.clf()
dimshow(rgb, ticks=False)
plt.title('WISE W1/W2 Model')
ps.savefig()
kwa = dict(mn=-1, mx=1, arcsinh=None)
rgb = _unwise_to_rgb([img-mod for img,mod in zip(coimgs,comods)], **kwa)
plt.clf()
dimshow(rgb, ticks=False)
plt.title('WISE W1/W2 Resid')
ps.savefig()
