def galex_tractor_image(tile, band, galex_dir, radecbox, bandname):
from tractor import (NanoMaggies, Image, LinearPhotoCal,
ConstantFitsWcs, ConstantSky)
assert(band in ['n','f'])
#nicegbands = ['NUV', 'FUV']
#zps = dict(n=20.08, f=18.82)
#zp = zps[band]
imfn = os.path.join(galex_dir, tile.tilename.strip(),
'%s-%sd-intbgsub.fits.gz' % (tile.visitname.strip(), band))
gwcs = Tan(*[float(f) for f in
[tile.crval1, tile.crval2, tile.crpix1, tile.crpix2,
tile.cdelt1, 0., 0., tile.cdelt2, 3840., 3840.]])
(r0,r1,d0,d1) = radecbox
H,W = gwcs.shape
ok,xx,yy = gwcs.radec2pixelxy([r0,r0,r1,r1], [d0,d1,d1,d0])
#print('GALEX WCS pixel positions of RA,Dec box:', xx, yy)
if np.any(np.logical_not(ok)):
return None
x0 = np.clip(np.floor(xx-1).astype(int).min(), 0, W-1)
x1 = np.clip(np.ceil (xx-1).astype(int).max(), 0, W)
if x1-x0 <= 1:
return None
y0 = np.clip(np.floor(yy-1).astype(int).min(), 0, H-1)
y1 = np.clip(np.ceil (yy-1).astype(int).max(), 0, H)
if y1-y0 <= 1:
return None
debug('Reading GALEX subimage x0,y0', x0,y0, 'size', x1-x0, y1-y0)
gwcs = gwcs.get_subimage(x0, y0, x1 - x0, y1 - y0)
twcs = ConstantFitsWcs(gwcs)
roislice = (slice(y0, y1), slice(x0, x1))
fitsimg = fitsio.FITS(imfn)[0]
hdr = fitsimg.read_header()
img = fitsimg[roislice]
inverr = np.ones_like(img)
inverr[img == 0.] = 0.
zp = tile.get('%s_zpmag' % band)
photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp), band=bandname)
tsky = ConstantSky(0.)
name = 'GALEX ' + hdr['OBJECT'] + ' ' + band
psfimg = galex_psf(band, galex_dir)
tpsf = PixelizedPSF(psfimg)
tim = Image(data=img, inverr=inverr, psf=tpsf, wcs=twcs,
sky=tsky, photocal=photocal, name=name)
tim.roi = [x0,x1,y0,y1]
return tim
from legacypipe.runbrick import main
#from legacyanalysis.decals_sim import main as sim_main
from astrometry.util.fits import fits_table
if __name__ == '__main__':
travis = 'travis' in sys.argv
# demo RexGalaxy, with plots
if False:
from legacypipe.survey import RexGalaxy
from tractor import RaDecPos, NanoMaggies, PixPos
from tractor import ScalarParam
from tractor import Image, GaussianMixturePSF, LinearPhotoCal
from legacypipe.survey import LogRadius
rex = RexGalaxy(PixPos(1., 2.), NanoMaggies(r=3.), LogRadius(0.))
print('Rex:', rex)
print('Rex params:', rex.getParams())
print('Rex nparams:', rex.numberOfParams())
H, W = 100, 100
tim = Image(data=np.zeros((H, W), np.float32),
inverr=np.ones((H, W), np.float32),
psf=GaussianMixturePSF(1., 0., 0., 4., 4., 0.),
photocal=LinearPhotoCal(1., band='r'))
derivs = rex.getParamDerivatives(tim)
print('Derivs:', len(derivs))
print('Rex params:', rex.getParamNames())
import pylab as plt
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('rex')
def run_one_ccd(survey, catsurvey_north, catsurvey_south, resolve_dec, ccd,
opt, zoomslice, ps):
tlast = Time()
im = survey.get_image_object(ccd)
if opt.do_calib:
im.run_calibs(splinesky=True)
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
splinesky=True,
constant_invvar=opt.constant_invvar,
hybridPsf=opt.hybrid_psf,
normalizePsf=opt.normalize_psf,
old_calibs_ok=True)
print('Got tim:', tim, 'x0,y0', tim.x0, tim.y0)
tnow = Time()
print('Read image:', tnow - tlast)
tlast = tnow
# Apply outlier masks
if True:
# Outliers masks are computed within a survey (north/south for dr8), and are stored
# in a brick-oriented way, in the results directories.
north_ccd = (ccd.camera.strip() != 'decam')
catsurvey = catsurvey_north
if not north_ccd and catsurvey_south is not None:
catsurvey = catsurvey_south
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
for b in bricks:
from legacypipe.outliers import read_outlier_mask_file
print('Reading outlier mask for brick', b.brickname)
ok = read_outlier_mask_file(catsurvey, [tim],
b.brickname,
subimage=False,
output=False,
ps=ps)
if not ok:
print('WARNING: failed to read outliers mask file for brick',
b.brickname)
if opt.catalog:
T = fits_table(opt.catalog)
else:
chipwcs = tim.subwcs
T = get_catalog_in_wcs(chipwcs,
catsurvey_north,
catsurvey_south=catsurvey_south,
resolve_dec=resolve_dec)
if T is None:
print('No sources to photometer.')
return None
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
surveydir = survey.get_survey_dir()
del survey
if opt.move_gaia:
# Gaia stars: move RA,Dec to the epoch of this image.
I = np.flatnonzero(T.ref_epoch > 0)
if len(I):
from legacypipe.survey import radec_at_mjd
print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd())
ra, dec = radec_at_mjd(T.ra[I], T.dec[I],
T.ref_epoch[I].astype(float), T.pmra[I],
T.pmdec[I], T.parallax[I], tim.time.toMjd())
T.ra[I] = ra
T.dec[I] = dec
tnow = Time()
print('Read catalog:', tnow - tlast)
tlast = tnow
cat = read_fits_catalog(T, bands='r')
# Replace the brightness (which will be a NanoMaggies with g,r,z)
# with a NanoMaggies with this image's band only.
for src in cat:
src.brightness = NanoMaggies(**{tim.band: 1.})
tnow = Time()
print('Parse catalog:', tnow - tlast)
tlast = tnow
print('Forced photom...')
F = run_forced_phot(cat,
tim,
ceres=opt.ceres,
derivs=opt.derivs,
fixed_also=True,
agn=opt.agn,
do_forced=opt.forced,
do_apphot=opt.apphot,
get_model=opt.save_model,
ps=ps,
timing=True,
ceres_threads=opt.ceres_threads)
if opt.save_model:
# unpack results
F, model_img = F
F.release = T.release
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.camera = np.array([ccd.camera] * len(F))
F.expnum = np.array([im.expnum] * len(F)).astype(np.int64)
F.ccdname = np.array([im.ccdname] * len(F))
# "Denormalizing"
F.filter = np.array([tim.band] * len(F))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(F))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(F)).astype(np.float32)
F.psfsize = np.array([tim.psf_fwhm * tim.imobj.pixscale] * len(F)).astype(
np.float32)
F.ccd_cuts = np.array([ccd.ccd_cuts] * len(F))
F.airmass = np.array([ccd.airmass] * len(F))
### --> also add units to the dict below so the FITS headers have units
F.sky = np.array([tim.midsky / tim.zpscale / tim.imobj.pixscale**2] *
len(F)).astype(np.float32)
# in the same units as the depth maps -- flux inverse-variance.
F.psfdepth = np.array([(1. / (tim.sig1 / tim.psfnorm)**2)] *
len(F)).astype(np.float32)
F.galdepth = np.array([(1. / (tim.sig1 / tim.galnorm)**2)] *
len(F)).astype(np.float32)
# F.psfdepth = np.array([-2.5 * (np.log10(5. * tim.sig1 / tim.psfnorm) - 9)] * len(F)).astype(np.float32)
# F.galdepth = np.array([-2.5 * (np.log10(5. * tim.sig1 / tim.galnorm) - 9)] * len(F)).astype(np.float32)
# super units questions here
if opt.derivs:
cosdec = np.cos(np.deg2rad(T.dec))
F.dra = (F.flux_dra / F.flux) * 3600. / cosdec
F.ddec = (F.flux_ddec / F.flux) * 3600.
F.dra_ivar = F.flux_dra_ivar * (F.flux / 3600. * cosdec)**2
F.ddec_ivar = F.flux_ddec_ivar * (F.flux / 3600.)**2
F.delete_column('flux_dra')
F.delete_column('flux_ddec')
F.delete_column('flux_dra_ivar')
F.delete_column('flux_ddec_ivar')
F.flux = F.flux_fixed
F.flux_ivar = F.flux_fixed_ivar
F.delete_column('flux_fixed')
F.delete_column('flux_fixed_ivar')
for c in ['dra', 'ddec', 'dra_ivar', 'ddec_ivar', 'flux', 'flux_ivar']:
F.set(c, F.get(c).astype(np.float32))
F.ra = T.ra
F.dec = T.dec
ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
h, w = tim.shape
F.dqmask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h - 1),
np.clip(np.round(F.x).astype(int), 0, w - 1)]
program_name = sys.argv[0]
## FIXME -- from catalog?
release = 8002
version_hdr = get_version_header(program_name, surveydir, release)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn.strip())
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(
dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(
dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='PLVER', value=ccd.plver, comment='CP pipeline version'))
version_hdr.add_record(
dict(name='PLPROCID', value=ccd.plprocid, comment='CP pipeline id'))
version_hdr.add_record(
dict(name='PROCDATE', value=ccd.procdate, comment='CP image DATE'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
fitsio.write(opt.save_model, model_img, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
tnow = Time()
print('Forced phot:', tnow - tlast)
return F, version_hdr
def run_one_ccd(survey, catsurvey_north, catsurvey_south, resolve_dec, ccd,
opt, zoomslice, ps):
from functools import reduce
from legacypipe.bits import DQ_BITS
tlast = Time()
#print('Opt:', opt)
im = survey.get_image_object(ccd)
print('Run_one_ccd: checking cache', survey.cache_dir)
if survey.cache_dir is not None:
im.check_for_cached_files(survey)
if opt.do_calib:
im.run_calibs(splinesky=True)
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
constant_invvar=opt.constant_invvar,
hybridPsf=opt.hybrid_psf,
normalizePsf=opt.normalize_psf,
old_calibs_ok=True,
trim_edges=False)
print('Got tim:', tim) #, 'x0,y0', tim.x0, tim.y0)
chipwcs = tim.subwcs
H, W = tim.shape
tnow = Time()
print('Read image:', tnow - tlast)
tlast = tnow
if ccd.camera == 'decam':
# Halo subtraction
from legacypipe.halos import subtract_one
from legacypipe.reference import mask_radius_for_mag, read_gaia
ref_margin = mask_radius_for_mag(0.)
mpix = int(np.ceil(ref_margin * 3600. / chipwcs.pixel_scale()))
marginwcs = chipwcs.get_subimage(-mpix, -mpix, W + 2 * mpix,
H + 2 * mpix)
gaia = read_gaia(marginwcs, None)
keeprad = np.ceil(gaia.keep_radius * 3600. /
chipwcs.pixel_scale()).astype(int)
_, xx, yy = chipwcs.radec2pixelxy(gaia.ra, gaia.dec)
# cut to those touching the chip
gaia.cut((xx > -keeprad) * (xx < W + keeprad) * (yy > -keeprad) *
(yy < H + keeprad))
Igaia, = np.nonzero(gaia.isgaia * gaia.pointsource)
halostars = gaia[Igaia]
print('Got', len(gaia), 'Gaia stars,', len(halostars),
'for halo subtraction')
moffat = True
halos = subtract_one((tim, halostars, moffat))
tim.data -= halos
# The "north" and "south" directories often don't have
# 'survey-bricks" files of their own -- use the 'survey' one
# instead.
if catsurvey_south is not None:
try:
catsurvey_south.get_bricks_readonly()
except:
catsurvey_south.bricks = survey.get_bricks_readonly()
if catsurvey_north is not None:
try:
catsurvey_north.get_bricks_readonly()
except:
catsurvey_north.bricks = survey.get_bricks_readonly()
# Apply outlier masks
outlier_header = None
outlier_mask = None
posneg_mask = None
if opt.outlier_mask is not None:
posneg_mask = np.zeros(tim.shape, np.uint8)
# Outliers masks are computed within a survey (north/south for dr9), and are stored
# in a brick-oriented way, in the results directories.
north_ccd = (ccd.camera.strip() != 'decam')
catsurvey = catsurvey_north
if not north_ccd and catsurvey_south is not None:
catsurvey = catsurvey_south
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
for b in bricks:
print(
'Reading outlier mask for brick', b.brickname, ':',
catsurvey.find_file('outliers_mask',
brick=b.brickname,
output=False))
ok = read_outlier_mask_file(catsurvey, [tim],
b.brickname,
pos_neg_mask=posneg_mask,
subimage=False,
output=False,
ps=ps)
if not ok:
print('WARNING: failed to read outliers mask file for brick',
b.brickname)
if opt.outlier_mask is not None:
outlier_mask = np.zeros((ccd.height, ccd.width), np.uint8)
outlier_mask[tim.y0:tim.y0 + H, tim.x0:tim.x0 + W] = posneg_mask
del posneg_mask
# Grab original image headers (including WCS)
im = survey.get_image_object(ccd)
imhdr = im.read_image_header()
imhdr['CAMERA'] = ccd.camera
imhdr['EXPNUM'] = ccd.expnum
imhdr['CCDNAME'] = ccd.ccdname
imhdr['IMGFILE'] = ccd.image_filename.strip()
outlier_header = imhdr
if opt.catalog:
T = fits_table(opt.catalog)
else:
chipwcs = tim.subwcs
T = get_catalog_in_wcs(chipwcs,
survey,
catsurvey_north,
catsurvey_south=catsurvey_south,
resolve_dec=resolve_dec)
if T is None:
print('No sources to photometer.')
return None
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
surveydir = survey.get_survey_dir()
del survey
if opt.move_gaia:
# Gaia stars: move RA,Dec to the epoch of this image.
I = np.flatnonzero(T.ref_epoch > 0)
if len(I):
print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd())
ra, dec = radec_at_mjd(T.ra[I], T.dec[I],
T.ref_epoch[I].astype(float), T.pmra[I],
T.pmdec[I], T.parallax[I], tim.time.toMjd())
T.ra[I] = ra
T.dec[I] = dec
tnow = Time()
print('Read catalog:', tnow - tlast)
tlast = tnow
# Find SGA galaxies outside this chip and subtract them before we begin.
chipwcs = tim.subwcs
_, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
sga_out = (T.ref_cat == 'L3') * np.logical_not(
(xx >= 1) * (xx <= W) * (yy >= 1) * (yy <= H))
I = np.flatnonzero(sga_out)
if len(I):
print(len(I), 'SGA galaxies are outside the image. Subtracting...')
cat = read_fits_catalog(T[I], bands=[tim.band])
tr = Tractor([tim], cat)
mod = tr.getModelImage(0)
tim.data -= mod
I = np.flatnonzero(np.logical_not(sga_out))
T.cut(I)
cat = read_fits_catalog(T, bands='r')
# Replace the brightness (which will be a NanoMaggies with g,r,z)
# with a NanoMaggies with this image's band only.
for src in cat:
src.brightness = NanoMaggies(**{tim.band: 1.})
tnow = Time()
print('Parse catalog:', tnow - tlast)
tlast = tnow
print('Forced photom...')
F = run_forced_phot(cat,
tim,
ceres=opt.ceres,
derivs=opt.derivs,
fixed_also=True,
agn=opt.agn,
do_forced=opt.forced,
do_apphot=opt.apphot,
get_model=opt.save_model,
ps=ps,
timing=True,
ceres_threads=opt.ceres_threads)
if opt.save_model:
# unpack results
F, model_img = F
F.release = T.release
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.camera = np.array([ccd.camera] * len(F))
F.expnum = np.array([im.expnum] * len(F), dtype=np.int64)
F.ccdname = np.array([im.ccdname] * len(F))
# "Denormalizing"
F.filter = np.array([tim.band] * len(F))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(F))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(F), dtype=np.float32)
F.psfsize = np.array([tim.psf_fwhm * tim.imobj.pixscale] * len(F),
dtype=np.float32)
F.ccd_cuts = np.array([ccd.ccd_cuts] * len(F))
F.airmass = np.array([ccd.airmass] * len(F), dtype=np.float32)
### --> also add units to the dict below so the FITS headers have units
F.sky = np.array([tim.midsky / tim.zpscale / tim.imobj.pixscale**2] *
len(F),
dtype=np.float32)
# in the same units as the depth maps -- flux inverse-variance.
F.psfdepth = np.array([(1. / (tim.sig1 / tim.psfnorm)**2)] * len(F),
dtype=np.float32)
F.galdepth = np.array([(1. / (tim.sig1 / tim.galnorm)**2)] * len(F),
dtype=np.float32)
F.fwhm = np.array([tim.psf_fwhm] * len(F), dtype=np.float32)
F.skyrms = np.array([ccd.skyrms] * len(F), dtype=np.float32)
F.ccdzpt = np.array([ccd.ccdzpt] * len(F), dtype=np.float32)
F.ccdrarms = np.array([ccd.ccdrarms] * len(F), dtype=np.float32)
F.ccddecrms = np.array([ccd.ccddecrms] * len(F), dtype=np.float32)
F.ccdphrms = np.array([ccd.ccdphrms] * len(F), dtype=np.float32)
if opt.derivs:
cosdec = np.cos(np.deg2rad(T.dec))
with np.errstate(divide='ignore', invalid='ignore'):
F.dra = (F.flux_dra / F.flux) * 3600. / cosdec
F.ddec = (F.flux_ddec / F.flux) * 3600.
F.dra[F.flux == 0] = 0.
F.ddec[F.flux == 0] = 0.
F.dra_ivar = F.flux_dra_ivar * (F.flux / 3600. * cosdec)**2
F.ddec_ivar = F.flux_ddec_ivar * (F.flux / 3600.)**2
F.delete_column('flux_dra')
F.delete_column('flux_ddec')
F.delete_column('flux_dra_ivar')
F.delete_column('flux_ddec_ivar')
F.flux = F.flux_fixed
F.flux_ivar = F.flux_fixed_ivar
F.delete_column('flux_fixed')
F.delete_column('flux_fixed_ivar')
for c in ['dra', 'ddec', 'dra_ivar', 'ddec_ivar', 'flux', 'flux_ivar']:
F.set(c, F.get(c).astype(np.float32))
F.ra = T.ra
F.dec = T.dec
_, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
h, w = tim.shape
ix = np.round(F.x).astype(int)
iy = np.round(F.y).astype(int)
F.dqmask = tim.dq[np.clip(iy, 0, h - 1), np.clip(ix, 0, w - 1)]
# Set an OUT-OF-BOUNDS bit.
F.dqmask[reduce(np.logical_or,
[ix < 0, ix >= w, iy < 0, iy >= h])] |= DQ_BITS['edge2']
program_name = sys.argv[0]
## FIXME -- from catalog?
release = 9999
version_hdr = get_version_header(program_name, surveydir, release)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn.strip())
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(
dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(
dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='PLVER', value=ccd.plver, comment='CP pipeline version'))
version_hdr.add_record(
dict(name='PLPROCID', value=ccd.plprocid, comment='CP pipeline id'))
version_hdr.add_record(
dict(name='PROCDATE', value=ccd.procdate, comment='CP image DATE'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
fitsio.write(opt.save_model, model_img, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
tnow = Time()
print('Forced phot:', tnow - tlast)
return F, version_hdr, outlier_mask, outlier_header
def main():
import optparse
from astrometry.util.plotutils import PlotSequence
from astrometry.util.util import Tan
parser = optparse.OptionParser(usage='%prog [options] incat.fits out.fits')
parser.add_option('-r', '--ralo', dest='ralo', type=float,
help='Minimum RA')
parser.add_option('-R', '--rahi', dest='rahi', type=float,
help='Maximum RA')
parser.add_option('-d', '--declo', dest='declo', type=float,
help='Minimum Dec')
parser.add_option('-D', '--dechi', dest='dechi', type=float,
help='Maximum Dec')
parser.add_option('-b', '--band', dest='bands', action='append', type=int,
default=[], help='WISE band to photometer (default: 1,2)')
parser.add_option('-u', '--unwise', dest='unwise_dir',
default='unwise-coadds',
help='Directory containing unWISE coadds')
parser.add_option('--no-ceres', dest='ceres', action='store_false',
default=True,
help='Use scipy lsqr rather than Ceres Solver?')
parser.add_option('--ceres-block', '-B', dest='ceresblock', type=int,
default=8,
help='Ceres image block size (default: %default)')
parser.add_option('--plots', dest='plots',
default=False, action='store_true')
parser.add_option('--save-fits', dest='save_fits',
default=False, action='store_true')
# parser.add_option('--ellipses', action='store_true',
# help='Assume catalog shapes are ellipse descriptions (not r,ab,phi)')
# parser.add_option('--ra', help='Center RA')
# parser.add_option('--dec', help='Center Dec')
# parser.add_option('--width', help='Degrees width (in RA*cos(Dec))')
# parser.add_option('--height', help='Degrees height (Dec)')
opt, args = parser.parse_args()
if len(args) != 2:
parser.print_help()
sys.exit(-1)
if len(opt.bands) == 0:
opt.bands = [1, 2]
# Allow specifying bands like "123"
bb = []
for band in opt.bands:
for s in str(band):
bb.append(int(s))
opt.bands = bb
print('Bands', opt.bands)
ps = None
if opt.plots:
ps = PlotSequence('unwise')
infn, outfn = args
T = fits_table(infn)
print('Read', len(T), 'sources from', infn)
if opt.declo is not None:
T.cut(T.dec >= opt.declo)
if opt.dechi is not None:
T.cut(T.dec <= opt.dechi)
# Let's be a bit smart about RA wrap-around. Compute the 'center'
# of the RA points, use the cross product against that to define
# inequality (clockwise-of).
r = np.deg2rad(T.ra)
x = np.mean(np.cos(r))
y = np.mean(np.sin(r))
rr = np.hypot(x, y)
x /= rr
y /= rr
midra = np.rad2deg(np.arctan2(y, x))
midra += 360. * (midra < 0)
xx = np.cos(r)
yy = np.sin(r)
T.cross = x * yy - y * xx
minra = T.ra[np.argmin(T.cross)]
maxra = T.ra[np.argmax(T.cross)]
if opt.ralo is not None:
r = np.deg2rad(opt.ralo)
xx = np.cos(r)
yy = np.sin(r)
crosscut = x * yy - y * xx
T.cut(T.cross >= crosscut)
print('Cut to', len(T), 'with RA >', opt.ralo)
if opt.rahi is not None:
r = np.deg2rad(opt.rahi)
xx = np.cos(r)
yy = np.sin(r)
crosscut = x * yy - y * xx
T.cut(T.cross <= crosscut)
print('Cut to', len(T), 'with RA <', opt.rahi)
if opt.declo is None:
opt.declo = T.dec.min()
if opt.dechi is None:
opt.dechi = T.dec.max()
if opt.ralo is None:
opt.ralo = T.ra[np.argmin(T.cross)]
if opt.rahi is None:
opt.rahi = T.ra[np.argmax(T.cross)]
T.delete_column('cross')
print('RA range:', opt.ralo, opt.rahi)
print('Dec range:', opt.declo, opt.dechi)
x = np.mean([np.cos(np.deg2rad(r)) for r in (opt.ralo, opt.rahi)])
y = np.mean([np.sin(np.deg2rad(r)) for r in (opt.ralo, opt.rahi)])
midra = np.rad2deg(np.arctan2(y, x))
midra += 360. * (midra < 0)
middec = (opt.declo + opt.dechi) / 2.
print('RA,Dec center:', midra, middec)
pixscale = 2.75 / 3600.
H = (opt.dechi - opt.declo) / pixscale
dra = 2. * min(np.abs(midra - opt.ralo), np.abs(midra - opt.rahi))
W = dra * np.cos(np.deg2rad(middec)) / pixscale
margin = 5
W = int(W) + margin * 2
H = int(H) + margin * 2
print('W,H', W, H)
targetwcs = Tan(midra, middec, (W + 1) / 2., (H + 1) / 2.,
-pixscale, 0., 0., pixscale, float(W), float(H))
#print('Target WCS:', targetwcs)
ra0, dec0 = targetwcs.pixelxy2radec(0.5, 0.5)
ra1, dec1 = targetwcs.pixelxy2radec(W + 0.5, H + 0.5)
roiradecbox = [ra0, ra1, dec0, dec1]
#print('ROI RA,Dec box', roiradecbox)
tiles = unwise_tiles_touching_wcs(targetwcs)
print('Cut to', len(tiles), 'unWISE tiles')
disable_galaxy_cache()
cols = T.get_columns()
all_ptsrcs = not('type' in cols)
if not all_ptsrcs:
assert('shapeexp' in cols)
assert('shapedev' in cols)
assert('fracdev' in cols)
wanyband = 'w'
print('Creating Tractor catalog...')
cat = []
for i, t in enumerate(T):
pos = RaDecPos(t.ra, t.dec)
flux = NanoMaggies(**{wanyband: 1.})
if all_ptsrcs:
cat.append(PointSource(pos, flux))
continue
tt = t.type.strip()
if tt in ['PTSRC', 'STAR', 'S']:
cat.append(PointSource(pos, flux))
elif tt in ['EXP', 'E']:
shape = EllipseE(*t.shapeexp)
cat.append(ExpGalaxy(pos, flux, shape))
elif tt in ['DEV', 'D']:
shape = EllipseE(*t.shapedev)
cat.append(DevGalaxy(pos, flux, shape))
elif tt in ['COMP', 'C']:
eshape = EllipseE(*t.shapeexp)
dshape = EllipseE(*t.shapedev)
cat.append(FixedCompositeGalaxy(pos, flux, t.fracdev,
eshape, dshape))
else:
print('Did not understand row', i, 'of input catalog:')
t.about()
assert(False)
W = unwise_forcedphot(cat, tiles, roiradecbox=roiradecbox,
bands=opt.bands, unwise_dir=opt.unwise_dir,
use_ceres=opt.ceres, ceres_block=opt.ceresblock,
save_fits=opt.save_fits, ps=ps)
W.writeto(outfn)
def scan_dchisq(seeing, target_dchisq, ps, e1=0.):
pixscale = 0.262
psfsigma = seeing / pixscale / 2.35
print('PSF sigma:', psfsigma, 'pixels')
psf = GaussianMixturePSF(1., 0., 0., psfsigma**2, psfsigma**2, 0.)
sig1 = 0.01
psfnorm = 1./(2. * np.sqrt(np.pi) * psfsigma)
detsig1 = sig1 / psfnorm
sz = 50
cd = pixscale / 3600.
wcs = Tan(0., 0., float(sz/2), float(sz/2), -cd, 0., 0., cd,
float(sz), float(sz))
band = 'r'
tim = Image(data=np.zeros((sz,sz)), inverr=np.ones((sz,sz)) / sig1,
psf=psf,
wcs = ConstantFitsWcs(wcs),
photocal = LinearPhotoCal(1., band=band))
re_vals = np.logspace(-1., 0., 50)
all_runs = []
mods = []
for i,re in enumerate(re_vals):
true_src = ExpGalaxy(RaDecPos(0., 0.),
NanoMaggies(**{band: 1.}),
EllipseE(re, e1, 0.))
print('True source:', true_src)
tr = Tractor([tim], [true_src])
tr.freezeParams('images')
true_mod = tr.getModelImage(0)
dchisq_none = np.sum((true_mod * tim.inverr)**2)
scale = np.sqrt(target_dchisq / dchisq_none)
true_src.brightness.setParams([scale])
true_mod = tr.getModelImage(0)
dchisq_none = np.sum((true_mod * tim.inverr)**2)
mods.append(true_mod)
tim.data = true_mod
exp_src = true_src.copy()
psf_src = PointSource(true_src.pos.copy(), true_src.brightness.copy())
simp_src = SimpleGalaxy(true_src.pos.copy(), true_src.brightness.copy())
dchisqs = []
#for src in [psf_src, simp_src, exp_src]:
for src in [psf_src, simp_src]:
src.freezeParam('pos')
#print('Fitting source:', src)
#src.printThawedParams()
tr.catalog[0] = src
tr.optimize_loop()
#print('Fitted:', src)
mod = tr.getModelImage(0)
dchisqs.append(dchisq_none - np.sum(((true_mod - mod) * tim.inverr)**2))
#print('dchisq:', dchisqs[-1])
dchisqs.append(dchisq_none)
all_runs.append([re,] + dchisqs)
all_runs = np.array(all_runs)
re = all_runs[:,0]
dchi_psf = all_runs[:,1]
dchi_simp = all_runs[:,2]
dchi_exp = all_runs[:,3]
dchi_ps = np.maximum(dchi_psf, dchi_simp)
dchi_cut1 = dchi_ps + 3+9
dchi_cut2 = dchi_ps + dchi_psf * 0.02
dchi_cut3 = dchi_ps + dchi_psf * 0.008
plt.clf()
plt.plot(re, dchi_psf, 'k-', label='PSF')
plt.plot(re, dchi_simp, 'b-', label='SIMP')
plt.plot(re, dchi_exp, 'r-', label='EXP')
plt.plot(re, dchi_cut2, 'm--', alpha=0.5, lw=2, label='Cut: 2%')
plt.plot(re, dchi_cut3, 'm:', alpha=0.5, lw=2, label='Cut: 0.08%')
plt.plot(re, dchi_cut1, 'm-', alpha=0.5, lw=2, label='Cut: 12')
plt.xlabel('True r_e (arcsec)')
plt.ylabel('dchisq')
#plt.legend(loc='lower left')
plt.legend(loc='upper right')
tt = 'Seeing = %g arcsec, S/N ~ %i' % (seeing, int(np.round(np.sqrt(target_dchisq))))
if e1 != 0.:
tt += ', Ellipticity %g' % e1
plt.title(tt)
plt.ylim(0.90 * target_dchisq, 1.05 * target_dchisq)
# aspect = 1.2
# ax = plt.axis()
# dre = (ax[1]-ax[0]) / 20 / aspect
# dchi = (ax[3]-ax[2]) / 20
# I = np.linspace(0, len(re_vals)-1, 8).astype(int)
# for mod,re in [(mods[i], re_vals[i]) for i in I]:
# print('extent:', [re-dre, re+dre, ax[2], ax[2]+dchi])
# plt.imshow(mod, interpolation='nearest', origin='lower', aspect='auto',
# extent=[re-dre, re+dre, ax[2], ax[2]+dchi], cmap='gray')
# plt.axis(ax)
ps.savefig()
def run_sed_matched_filters(SEDs,
bands,
detmaps,
detivs,
omit_xy,
targetwcs,
nsigma=5,
saddle_fraction=0.1,
saddle_min=2.,
saturated_pix=None,
exclusion_radius=4.,
veto_map=None,
mp=None,
plots=False,
ps=None,
rgbimg=None):
'''
Runs a given set of SED-matched filters.
Parameters
----------
SEDs : list of (name, sed) tuples
The SEDs to run. The `sed` values are lists the same length
as `bands`.
bands : list of string
The band names of `detmaps` and `detivs`.
detmaps : numpy array, float
Detection maps for each of the listed `bands`.
detivs : numpy array, float
Inverse-variances of the `detmaps`.
omit_xy : None, or (xx,yy,rr) tuple
Existing sources to avoid: x, y, radius.
targetwcs : WCS object
WCS object to use to convert pixel values into RA,Decs for the
returned Tractor PointSource objects.
nsigma : float, optional
Detection threshold
saturated_pix : None or list of numpy arrays, booleans
Passed through to sed_matched_detection.
A map of pixels that are always considered "hot" when
determining whether a new source touches hot pixels of an
existing source.
exclusion_radius: int
How many pixels around an existing source to veto
plots : boolean, optional
Create plots?
ps : PlotSequence object
Create plots?
mp : multiproc object
Multiprocessing
Returns
-------
Tnew : fits_table
Table of new sources detected
newcat : list of PointSource objects
Newly detected objects, with positions and fluxes, as Tractor
PointSource objects.
hot : numpy array of bool
"Hot pixels" containing sources.
See also
--------
sed_matched_detection : run a single SED-matched filter.
'''
from astrometry.util.fits import fits_table
from tractor import PointSource, RaDecPos, NanoMaggies
if omit_xy is not None:
xx, yy, rr = omit_xy
n0 = len(xx)
else:
xx, yy, rr = [], [], []
n0 = 0
H, W = detmaps[0].shape
hot = np.zeros((H, W), bool)
peaksn = []
apsn = []
for sedname, sed in SEDs:
if plots:
pps = ps
else:
pps = None
sedhot, px, py, peakval, apval = sed_matched_detection(
sedname,
sed,
detmaps,
detivs,
bands,
xx,
yy,
rr,
nsigma=nsigma,
saddle_fraction=saddle_fraction,
saddle_min=saddle_min,
saturated_pix=saturated_pix,
veto_map=veto_map,
ps=pps,
rgbimg=rgbimg)
if sedhot is None:
continue
info('SED', sedname, ':', len(px), 'new peaks')
hot |= sedhot
# With an empty xx, np.append turns it into a double!
xx = np.append(xx, px).astype(int)
yy = np.append(yy, py).astype(int)
rr = np.append(rr, np.zeros_like(px) + exclusion_radius).astype(int)
peaksn.extend(peakval)
apsn.extend(apval)
# New peaks:
peakx = xx[n0:]
peaky = yy[n0:]
Tnew = None
newcat = []
if len(peakx):
# Add sources for the new peaks we found
pr, pd = targetwcs.pixelxy2radec(peakx + 1, peaky + 1)
info('Adding', len(pr), 'new sources')
# Also create FITS table for new sources
Tnew = fits_table()
Tnew.ra = pr
Tnew.dec = pd
Tnew.ibx = peakx
Tnew.iby = peaky
assert (len(peaksn) == len(Tnew))
assert (len(apsn) == len(Tnew))
Tnew.peaksn = np.array(peaksn)
Tnew.apsn = np.array(apsn)
for r, d, x, y in zip(pr, pd, peakx, peaky):
fluxes = dict([(band, detmap[y, x])
for band, detmap in zip(bands, detmaps)])
newcat.append(
PointSource(RaDecPos(r, d), NanoMaggies(order=bands,
**fluxes)))
return Tnew, newcat, hot
def unwise_coadds(onegal,
galaxy=None,
radius_mosaic=30,
radius_mask=None,
pixscale=2.75,
ref_pixscale=0.262,
output_dir=None,
unwise_dir=None,
verbose=False,
log=None,
centrals=True):
'''Generate custom unWISE cutouts.
radius_mosaic and radius_mask in arcsec
pixscale: WISE pixel scale in arcsec/pixel; make this smaller than 2.75
to oversample.
'''
import fitsio
import matplotlib.pyplot as plt
from astrometry.util.util import Tan
from astrometry.util.fits import fits_table
from astrometry.libkd.spherematch import match_radec
from astrometry.util.resample import resample_with_wcs, ResampleError
from wise.forcedphot import unwise_tiles_touching_wcs
from wise.unwise import get_unwise_tractor_image
from tractor import Tractor, Image, NanoMaggies
from legacypipe.survey import imsave_jpeg
from legacypipe.catalog import read_fits_catalog
if galaxy is None:
galaxy = 'galaxy'
if output_dir is None:
output_dir = '.'
if unwise_dir is None:
unwise_dir = os.environ.get('UNWISE_COADDS_DIR')
if radius_mask is None:
radius_mask = radius_mosaic
radius_search = 5.0 # [arcsec]
else:
radius_search = radius_mask
# Initialize the WCS object.
W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int') # [pixels]
targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0,
-pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W),
float(H))
# Read the custom Tractor catalog.
tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy))
if not os.path.isfile(tractorfile):
print('Missing Tractor catalog {}'.format(tractorfile),
flush=True,
file=log)
return 0
primhdr = fitsio.read_header(tractorfile)
cat = fits_table(tractorfile)
print('Read {} sources from {}'.format(len(cat), tractorfile),
flush=True,
file=log)
keep = np.ones(len(cat)).astype(bool)
if centrals:
# Find the large central galaxy and mask out (ignore) all the models
# which are within its elliptical mask.
# This algorithm will have to change for mosaics not centered on large
# galaxies, e.g., in galaxy groups.
m1, m2, d12 = match_radec(cat.ra,
cat.dec,
onegal['RA'],
onegal['DEC'],
radius_search / 3600.0,
nearest=False)
if len(m1) == 0:
print('No central galaxies found at the central coordinates!',
flush=True,
file=log)
else:
pixfactor = ref_pixscale / pixscale # shift the optical Tractor positions
for mm in m1:
morphtype = cat.type[mm].strip()
if morphtype == 'EXP' or morphtype == 'COMP':
e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[
mm], cat.shapeexp_r[mm] # [arcsec]
elif morphtype == 'DEV' or morphtype == 'COMP':
e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[
mm], cat.shapedev_r[mm] # [arcsec]
else:
r50 = None
if r50:
majoraxis = r50 * 5 / pixscale # [pixels]
ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2)
these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis,
ba * majoraxis,
np.radians(phi),
cat.bx * pixfactor,
cat.by * pixfactor)
if np.sum(these) > 0:
#keep[these] = False
pass
print('Hack!')
keep[mm] = False
#srcs = read_fits_catalog(cat)
#_srcs = np.array(srcs)[~keep].tolist()
#mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0)
#import matplotlib.pyplot as plt
##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png')
#plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png')
#pdb.set_trace()
srcs = read_fits_catalog(cat)
for src in srcs:
src.freezeAllBut('brightness')
#srcs_nocentral = np.array(srcs)[keep].tolist()
cat_nocentral = cat[keep]
## Find and remove all the objects within XX arcsec of the target
## coordinates.
#m1, m2, d12 = match_radec(T.ra, T.dec, onegal['RA'], onegal['DEC'], 5/3600.0, nearest=False)
#if len(d12) == 0:
# print('No matching galaxies found -- probably not what you wanted.')
# #raise ValueError
# nocentral = np.ones(len(T)).astype(bool)
#else:
# nocentral = ~np.isin(T.objid, T[m1].objid)
#T_nocentral = T[nocentral]
# Find and read the overlapping unWISE tiles. Assume the targetwcs is
# axis-aligned and that the edge midpoints yield the RA, Dec limits (true
# for TAN). Note: the way the roiradec box is used, the min/max order
# doesn't matter.
r, d = targetwcs.pixelxy2radec(np.array([1, W, W / 2, W / 2]),
np.array([H / 2, H / 2, 1, H]))
roiradec = [r[0], r[1], d[2], d[3]]
tiles = unwise_tiles_touching_wcs(targetwcs)
wbands = [1, 2, 3, 4]
wanyband = 'w'
vega_to_ab = dict(w1=2.699, w2=3.339, w3=5.174, w4=6.620)
# Convert the AB WISE fluxes in the Tractor catalog to Vega nanomaggies so
# they're consistent with the coadds, below.
for band in wbands:
f = cat.get('flux_w{}'.format(band))
e = cat.get('flux_ivar_w{}'.format(band))
print('Setting negative fluxes equal to zero!')
f[f < 0] = 0
#f[f/e < 3] = 0
f *= 10**(0.4 * vega_to_ab['w{}'.format(band)])
coimgs = [np.zeros((H, W), np.float32) for b in wbands]
comods = [np.zeros((H, W), np.float32) for b in wbands]
comods_nocentral = [np.zeros((H, W), np.float32) for b in wbands]
con = [np.zeros((H, W), np.uint8) for b in wbands]
for iband, band in enumerate(wbands):
for ii, src in enumerate(srcs):
src.setBrightness(
NanoMaggies(
**{wanyband: cat.get('flux_w{}'.format(band))[ii]}))
srcs_nocentral = np.array(srcs)[keep].tolist()
#srcs_nocentral = np.array(srcs)[nocentral].tolist()
# The tiles have some overlap, so for each source, keep the fit in the
# tile whose center is closest to the source.
for tile in tiles:
#print('Reading tile {}'.format(tile.coadd_id))
tim = get_unwise_tractor_image(unwise_dir,
tile.coadd_id,
band,
bandname=wanyband,
roiradecbox=roiradec)
if tim is None:
print('Actually, no overlap with tile {}'.format(
tile.coadd_id))
continue
print('Read image {} with shape {}'.format(tile.coadd_id,
tim.shape))
def _unwise_mod(tim, use_cat, use_srcs, margin=10):
# Select sources in play.
wisewcs = tim.wcs.wcs
timH, timW = tim.shape
ok, x, y = wisewcs.radec2pixelxy(use_cat.ra, use_cat.dec)
x = (x - 1.).astype(np.float32)
y = (y - 1.).astype(np.float32)
I = np.flatnonzero((x >= -margin) * (x < timW + margin) *
(y >= -margin) * (y < timH + margin))
#print('Found {} sources within the image + margin = {} pixels'.format(len(I), margin))
subcat = [use_srcs[i] for i in I]
tractor = Tractor([tim], subcat)
mod = tractor.getModelImage(0)
return mod
mod = _unwise_mod(tim, cat, srcs)
mod_nocentral = _unwise_mod(tim, cat_nocentral, srcs_nocentral)
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, tim.wcs.wcs)
except ResampleError:
continue
if len(Yo) == 0:
continue
# The models are already in AB nanomaggies, but the tiles / tims are
# in Vega nanomaggies, so convert them here.
coimgs[iband][Yo, Xo] += tim.getImage()[Yi, Xi]
comods[iband][Yo, Xo] += mod[Yi, Xi]
comods_nocentral[iband][Yo, Xo] += mod_nocentral[Yi, Xi]
con[iband][Yo, Xo] += 1
## Convert back to nanomaggies.
#vega2ab = vega_to_ab['w{}'.format(band)]
#coimgs[iband] *= 10**(-0.4 * vega2ab)
#comods[iband] *= 10**(-0.4 * vega2ab)
#comods_nocentral[iband] *= 10**(-0.4 * vega2ab)
for img, mod, mod_nocentral, n in zip(coimgs, comods, comods_nocentral,
con):
img /= np.maximum(n, 1)
mod /= np.maximum(n, 1)
mod_nocentral /= np.maximum(n, 1)
coresids = [img - mod for img, mod in list(zip(coimgs, comods))]
# Subtract the model image which excludes the central (comod_nocentral)
# from the data (coimg) to isolate the light of the central
# (coimg_central).
coimgs_central = [
img - mod for img, mod in list(zip(coimgs, comods_nocentral))
]
# Write out the final images with and without the central and converted into
# AB nanomaggies.
for coadd, imtype in zip((coimgs, comods, comods_nocentral),
('image', 'model', 'model-nocentral')):
for img, band in zip(coadd, wbands):
vega2ab = vega_to_ab['w{}'.format(band)]
fitsfile = os.path.join(
output_dir, '{}-{}-W{}.fits'.format(galaxy, imtype, band))
if verbose:
print('Writing {}'.format(fitsfile))
fitsio.write(fitsfile, img * 10**(-0.4 * vega2ab), clobber=True)
# Generate color WISE images.
kwa = dict(mn=-1, mx=100, arcsinh=0.5)
#kwa = dict(mn=-0.05, mx=1., arcsinh=0.5)
#kwa = dict(mn=-0.1, mx=2., arcsinh=None)
for imgs, imtype in zip(
(coimgs, comods, coresids, comods_nocentral, coimgs_central),
('image', 'model', 'resid', 'model-nocentral', 'image-central')):
rgb = _unwise_to_rgb(imgs[:2], **kwa) # W1, W2
jpgfile = os.path.join(output_dir,
'{}-{}-W1W2.jpg'.format(galaxy, imtype))
if verbose:
print('Writing {}'.format(jpgfile))
imsave_jpeg(jpgfile, rgb, origin='lower')
return 1
def get_reference_sources(survey,
targetwcs,
pixscale,
bands,
tycho_stars=True,
gaia_stars=True,
large_galaxies=True,
star_clusters=True):
from legacypipe.survey import GaiaSource
from legacypipe.survey import LegacyEllipseWithPriors
from tractor import NanoMaggies, RaDecPos
from tractor.galaxy import ExpGalaxy
from tractor.ellipses import EllipseESoft
H, W = targetwcs.shape
H, W = int(H), int(W)
# How big of a margin to search for bright stars and star clusters --
# this should be based on the maximum radius they are considered to
# affect.
ref_margin = 0.125
mpix = int(np.ceil(ref_margin * 3600. / pixscale))
marginwcs = targetwcs.get_subimage(-mpix, -mpix, W + 2 * mpix,
H + 2 * mpix)
refs = []
# Tycho-2 stars
if tycho_stars:
tycho = read_tycho2(survey, marginwcs)
if len(tycho):
refs.append(tycho)
# Add Gaia stars
gaia = None
if gaia_stars:
from astrometry.libkd.spherematch import match_radec
gaia = read_gaia(marginwcs)
if gaia is not None:
gaia.isbright = (gaia.phot_g_mean_mag < 13.)
gaia.ismedium = (gaia.phot_g_mean_mag < 16.)
gaia.donotfit = np.zeros(len(gaia), bool)
# Handle sources that appear in both Gaia and Tycho-2 by
# dropping the entry from Tycho-2.
if len(gaia) and len(tycho):
# Before matching, apply proper motions to bring them to
# the same epoch. We want to use the more-accurate Gaia
# proper motions, so rewind Gaia positions to the
# approximate epoch of Tycho-2: 1991.5.
cosdec = np.cos(np.deg2rad(gaia.dec))
gra = gaia.ra + (1991.5 - gaia.ref_epoch) * gaia.pmra / (
3600. * 1000.) / cosdec
gdec = gaia.dec + (1991.5 - gaia.ref_epoch) * gaia.pmdec / (3600. *
1000.)
I, J, _ = match_radec(tycho.ra,
tycho.dec,
gra,
gdec,
1. / 3600.,
nearest=True)
debug('Matched', len(I), 'Tycho-2 stars to Gaia stars.')
if len(I):
keep = np.ones(len(tycho), bool)
keep[I] = False
tycho.cut(keep)
gaia.isbright[J] = True
if gaia is not None and len(gaia) > 0:
refs.append(gaia)
# Read the catalog of star (open and globular) clusters and add them to the
# set of reference stars (with the isbright bit set).
if star_clusters:
clusters = read_star_clusters(marginwcs)
if clusters is not None:
debug('Found', len(clusters), 'star clusters nearby')
clusters.iscluster = np.ones(len(clusters), bool)
refs.append(clusters)
# Read large galaxies nearby.
if large_galaxies:
galaxies = read_large_galaxies(survey, targetwcs)
if galaxies is not None:
# Resolve possible Gaia-large-galaxy duplicates
if gaia and len(gaia):
I, J, _ = match_radec(galaxies.ra,
galaxies.dec,
gaia.ra,
gaia.dec,
2. / 3600.,
nearest=True)
print('Matched', len(I), 'large galaxies to Gaia stars.')
if len(I):
gaia.donotfit[J] = True
refs.append(galaxies)
refcat = None
if len(refs):
refs = merge_tables([r for r in refs if r is not None],
columns='fillzero')
if len(refs) == 0:
return None, None
refs.radius_pix = np.ceil(refs.radius * 3600. / pixscale).astype(int)
ok, xx, yy = targetwcs.radec2pixelxy(refs.ra, refs.dec)
# ibx = integer brick coords
refs.ibx = np.round(xx - 1.).astype(int)
refs.iby = np.round(yy - 1.).astype(int)
# cut ones whose position + radius are outside the brick bounds.
refs.cut((xx > -refs.radius_pix) * (xx < W + refs.radius_pix) *
(yy > -refs.radius_pix) * (yy < H + refs.radius_pix))
# mark ones that are actually inside the brick area.
refs.in_bounds = ((refs.ibx >= 0) * (refs.ibx < W) * (refs.iby >= 0) *
(refs.iby < H))
for col in [
'isbright', 'ismedium', 'islargegalaxy', 'iscluster', 'donotfit'
]:
if not col in refs.get_columns():
refs.set(col, np.zeros(len(refs), bool))
## Create Tractor sources from reference stars
refcat = []
for g in refs:
if g.donotfit or g.iscluster:
refcat.append(None)
elif g.islargegalaxy:
fluxes = dict([(band, NanoMaggies.magToNanomaggies(g.mag))
for band in bands])
assert (np.all(np.isfinite(list(fluxes.values()))))
rr = g.radius * 3600. / 0.5 # factor of two accounts for R(25)-->reff
pa = 180 - g.pa
if not np.isfinite(pa):
pa = 0.
logr, ee1, ee2 = EllipseESoft.rAbPhiToESoft(rr, g.ba, pa)
gal = ExpGalaxy(RaDecPos(g.ra, g.dec),
NanoMaggies(order=bands, **fluxes),
LegacyEllipseWithPriors(logr, ee1, ee2))
refcat.append(gal)
else:
# Gaia star -- which we want to create a source for, regardless of
# whether it is marked medium | bright (or neither).
refcat.append(GaiaSource.from_catalog(g, bands))
for src in refcat:
if src:
src.is_reference_source = True
return refs, refcat
def read_fits_catalog(T,
hdr=None,
invvars=False,
bands='grz',
allbands='ugrizY',
ellipseClass=None):
from tractor import NanoMaggies
'''
This is currently a weird hybrid of dynamic and hard-coded.
Return list of tractor Sources.
If invvars=True, return sources,invvars
where invvars is a list matching sources.getParams()
'''
if hdr is None:
hdr = T._header
rev_typemap = dict([(v, k) for k, v in fits_typemap.items()])
ivbandcols = []
ibands = np.array([allbands.index(b) for b in bands])
ivs = []
cat = []
for i, t in enumerate(T):
clazz = rev_typemap[t.type.strip()]
pos = RaDecPos(t.ra, t.dec)
assert (np.isfinite(t.ra))
assert (np.isfinite(t.dec))
shorttype = fits_short_typemap[clazz]
assert (np.all(np.isfinite(t.decam_flux[ibands])))
br = NanoMaggies(order=bands, **dict(zip(bands, t.decam_flux[ibands])))
params = [pos, br]
if invvars:
# ASSUME & hard-code that the position and brightness are
# the first params
ivs.extend([t.ra_ivar, t.dec_ivar] +
list(t.decam_flux_ivar[ibands]))
if issubclass(clazz, (DevGalaxy, ExpGalaxy)):
if ellipseClass is not None:
eclazz = ellipseClass
else:
# hard-code knowledge that third param is the ellipse
eclazz = hdr['TR_%s_T3' % shorttype]
# drop any double-quoted weirdness
eclazz = eclazz.replace('"', '')
# look up that string... to avoid eval()
eclazz = ellipse_types[eclazz]
if issubclass(clazz, DevGalaxy):
assert (np.all([np.isfinite(x) for x in t.shapedev]))
ell = eclazz(*t.shapedev)
else:
assert (np.all([np.isfinite(x) for x in t.shapeexp]))
ell = eclazz(*t.shapeexp)
params.append(ell)
if invvars:
if issubclass(clazz, DevGalaxy):
ivs.extend(t.shapedev_ivar)
else:
ivs.extend(t.shapeexp_ivar)
elif issubclass(clazz, FixedCompositeGalaxy):
# hard-code knowledge that params are fracDev, shapeE, shapeD
assert (np.isfinite(t.fracdev))
params.append(t.fracdev)
if ellipseClass is not None:
expeclazz = deveclazz = ellipseClass
else:
expeclazz = hdr['TR_%s_T4' % shorttype]
deveclazz = hdr['TR_%s_T5' % shorttype]
expeclazz = expeclazz.replace('"', '')
deveclazz = deveclazz.replace('"', '')
expeclazz = ellipse_types[expeclazz]
deveclazz = ellipse_types[deveclazz]
assert (np.all([np.isfinite(x) for x in t.shapedev]))
assert (np.all([np.isfinite(x) for x in t.shapeexp]))
ee = expeclazz(*t.shapeexp)
de = deveclazz(*t.shapedev)
params.append(ee)
params.append(de)
if invvars:
ivs.append(t.fracdev_ivar)
ivs.extend(t.shapeexp_ivar)
ivs.extend(t.shapedev_ivar)
elif issubclass(clazz, PointSource):
pass
else:
raise RuntimeError('Unknown class %s' % str(clazz))
src = clazz(*params)
cat.append(src)
if invvars:
ivs = np.array(ivs)
ivs[np.logical_not(np.isfinite(ivs))] = 0
return cat, ivs
return cat
def get_galaxy_sources(galaxies, bands):
from legacypipe.catalog import fits_reverse_typemap
from legacypipe.survey import (LegacySersicIndex, LegacyEllipseWithPriors,
LogRadius, RexGalaxy)
from tractor import NanoMaggies, RaDecPos, PointSource
from tractor.ellipses import EllipseE, EllipseESoft
from tractor.galaxy import DevGalaxy, ExpGalaxy
from tractor.sersic import SersicGalaxy
# Factor of HyperLEDA to set the galaxy max radius
radius_max_factor = 2.
srcs = [None for g in galaxies]
# If we have pre-burned galaxies, re-create the Tractor sources for them.
I, = np.nonzero(galaxies.preburned)
for ii, g in zip(I, galaxies[I]):
typ = fits_reverse_typemap[g.type.strip()]
pos = RaDecPos(g.ra, g.dec)
fluxes = dict([(band, g.get('flux_%s' % band)) for band in bands])
bright = NanoMaggies(order=bands, **fluxes)
shape = None
# put the Rex branch first, because Rex is a subclass of ExpGalaxy!
if issubclass(typ, RexGalaxy):
assert (np.isfinite(g.shape_r))
logre = np.log(g.shape_r)
shape = LogRadius(logre)
# set prior max at 2x SGA radius
shape.setMaxLogRadius(logre + np.log(radius_max_factor))
elif issubclass(typ, (DevGalaxy, ExpGalaxy, SersicGalaxy)):
assert (np.isfinite(g.shape_r))
assert (np.isfinite(g.shape_e1))
assert (np.isfinite(g.shape_e2))
shape = EllipseE(g.shape_r, g.shape_e1, g.shape_e2)
# switch to softened ellipse (better fitting behavior)
shape = EllipseESoft.fromEllipseE(shape)
# and then to our custom ellipse class
logre = shape.logre
shape = LegacyEllipseWithPriors(logre, shape.ee1, shape.ee2)
assert (np.all(np.isfinite(shape.getParams())))
# set prior max at 2x SGA radius
shape.setMaxLogRadius(logre + np.log(radius_max_factor))
if issubclass(typ, PointSource):
src = typ(pos, bright)
# this catches Rex too
elif issubclass(typ, (DevGalaxy, ExpGalaxy)):
src = typ(pos, bright, shape)
elif issubclass(typ, (SersicGalaxy)):
assert (np.isfinite(g.sersic))
sersic = LegacySersicIndex(g.sersic)
src = typ(pos, bright, shape, sersic)
else:
raise RuntimeError('Unknown preburned SGA source type "%s"' % typ)
debug('Created', src)
assert (np.isfinite(src.getLogPrior()))
srcs[ii] = src
# SGA parent catalog: 'preburned' is not set
# This also can happen in the preburned/ellipse catalog when fitting
# fails, or no-grz, etc.
I, = np.nonzero(np.logical_not(galaxies.preburned))
for ii, g in zip(I, galaxies[I]):
# Initialize each source with an exponential disk--
fluxes = dict([(band, NanoMaggies.magToNanomaggies(g.mag))
for band in bands])
assert (np.all(np.isfinite(list(fluxes.values()))))
rr = g.radius * 3600. / 2 # factor of two accounts for R(25)-->reff [arcsec]
assert (np.isfinite(rr))
assert (np.isfinite(g.ba))
assert (np.isfinite(g.pa))
ba = g.ba
if ba <= 0.0 or ba > 1.0:
# Make round!
ba = 1.0
logr, ee1, ee2 = EllipseESoft.rAbPhiToESoft(
rr, ba, 180 - g.pa) # note the 180 rotation
assert (np.isfinite(logr))
assert (np.isfinite(ee1))
assert (np.isfinite(ee2))
shape = LegacyEllipseWithPriors(logr, ee1, ee2)
shape.setMaxLogRadius(logr + np.log(radius_max_factor))
src = ExpGalaxy(RaDecPos(g.ra, g.dec),
NanoMaggies(order=bands, **fluxes), shape)
assert (np.isfinite(src.getLogPrior()))
src.needs_initial_flux = True
srcs[ii] = src
return srcs
def stage_galex_forced(
survey=None,
cat=None,
T=None,
targetwcs=None,
targetrd=None,
W=None, H=None,
pixscale=None,
brickname=None,
galex_dir=None,
brick=None,
version_header=None,
maskbits=None,
mp=None,
record_event=None,
ps=None,
plots=False,
**kwargs):
'''
After the model fits are finished, we can perform forced
photometry of the GALEX coadds.
'''
from legacypipe.runbrick import _add_stage_version
from legacypipe.bits import MASKBITS
#from legacypipe.galex import galex_phot, galex_tiles_touching_wcs
#from legacypipe.unwise import unwise_phot, collapse_unwise_bitmask, unwise_tiles_touching_wcs
from legacypipe.survey import wise_apertures_arcsec
from tractor import NanoMaggies
record_event and record_event('stage_galex_forced: starting')
_add_stage_version(version_header, 'GALEX', 'galex_forced')
galex_apertures_arcsec = wise_apertures_arcsec
if not plots:
ps = None
# Skip if $GALEX_DIR or --galex-dir not set.
if galex_dir is None:
info('GALEX_DIR not set -- skipping GALEX')
return None
tiles = galex_tiles_touching_wcs(targetwcs, galex_dir)
info('Cut to', len(tiles), 'GALEX tiles')
# the way the roiradec box is used, the min/max order doesn't matter
roiradec = [targetrd[0,0], targetrd[2,0], targetrd[0,1], targetrd[2,1]]
# Sources to photometer
do_phot = np.ones(len(cat), bool)
# Drop sources within the CLUSTER mask from forced photometry.
Icluster = None
if maskbits is not None:
incluster = (maskbits & MASKBITS['CLUSTER'] > 0)
if np.any(incluster):
print('Checking for sources inside CLUSTER mask')
ra = np.array([src.getPosition().ra for src in cat])
dec = np.array([src.getPosition().dec for src in cat])
ok,xx,yy = targetwcs.radec2pixelxy(ra, dec)
xx = np.round(xx - 1).astype(int)
yy = np.round(yy - 1).astype(int)
I = np.flatnonzero(ok * (xx >= 0)*(xx < W) * (yy >= 0)*(yy < H))
if len(I):
Icluster = I[incluster[yy[I], xx[I]]]
print('Found', len(Icluster), 'of', len(cat), 'sources inside CLUSTER mask')
do_phot[Icluster] = False
Nskipped = len(do_phot) - np.sum(do_phot)
gcat = []
for i in np.flatnonzero(do_phot):
src = cat[i]
src = src.copy()
src.setBrightness(NanoMaggies(galex=1.))
gcat.append(src)
# use pixelized PSF model
pixpsf = True
# Photometer the two bands in parallel
args = []
for band in ['n','f']:
btiles = tiles[tiles.get('has_%s' % band)]
if len(btiles) == 0:
continue
args.append((galex_dir, gcat, btiles, band, roiradec, pixpsf, ps))
# Run the forced photometry!
record_event and record_event('stage_galex_forced: photometry')
phots = mp.map(galex_phot, args)
record_event and record_event('stage_galex_forced: results')
# Unpack results...
GALEX = None
if len(phots):
# The "phot" results for the full-depth coadds are one table per
# band. Merge all those columns.
galex_models = []
for i,p in enumerate(phots[:len(args)]):
if p is None:
(_,_,tiles,band) = args[i][:4]
print('"None" result from GALEX forced phot:', tiles, band)
continue
galex_models.extend(p.models)
if GALEX is None:
GALEX = p.phot
else:
GALEX.add_columns_from(p.phot)
# Create the WCS into which we'll resample the tiles.
# Same center as "targetwcs" but bigger pixel scale.
gpixscale = 1.5
gra = np.array([src.getPosition().ra for src in cat])
gdec = np.array([src.getPosition().dec for src in cat])
rc,dc = targetwcs.radec_center()
ww = int(W * pixscale / gpixscale)
hh = int(H * pixscale / gpixscale)
gcoadds = GalexCoadd(rc, dc, ww, hh, gpixscale)
gcoadds.add(galex_models)
apphot = gcoadds.finish(survey, brickname, version_header,
apradec=(gra,gdec),
apertures=galex_apertures_arcsec/gpixscale)
api,apd,apr = apphot
for iband,band in enumerate(['n','f']):
niceband = band + 'uv'
GALEX.set('apflux_%s' % niceband, api[iband])
GALEX.set('apflux_resid_%s' % niceband, apr[iband])
d = apd[iband]
iv = np.zeros_like(d)
iv[d != 0.] = 1./(d[d != 0]**2)
GALEX.set('apflux_ivar_%s' % niceband, iv)
fluxcol = 'flux_'+niceband
if not fluxcol in GALEX.get_columns():
GALEX.set(fluxcol, np.zeros(len(GALEX), np.float32))
GALEX.set('flux_ivar_'+niceband, np.zeros(len(GALEX), np.float32))
if Nskipped > 0:
assert(len(GALEX) == len(wcat))
GALEX = _fill_skipped_values(GALEX, Nskipped, do_phot)
assert(len(GALEX) == len(cat))
assert(len(GALEX) == len(T))
debug('Returning: GALEX', GALEX)
if logger.isEnabledFor(logging.DEBUG):
GALEX.about()
return dict(GALEX=GALEX,
version_header=version_header,
galex_apertures_arcsec=galex_apertures_arcsec)
def run_one_ccd(survey, catsurvey, ccd, opt, zoomslice, ps):
tlast = Time()
im = survey.get_image_object(ccd)
if opt.do_calib:
#from legacypipe.survey import run_calibs
#kwa = dict(splinesky=True)
#run_calibs((im, kwa))
im.run_calibs(splinesky=True)
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True,
constant_invvar=opt.constant_invvar,
hybridPsf=opt.hybrid_psf,
normalizePsf=opt.normalize_psf,
old_calibs_ok=True)
print('Got tim:', tim)
tnow = Time()
print('Read image:', tnow-tlast)
tlast = tnow
if opt.catalog:
T = fits_table(opt.catalog)
else:
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = catsurvey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn, columns=[
'ra', 'dec', 'brick_primary', 'type', 'release',
'brickid', 'brickname', 'objid',
'fracdev', 'flux_r',
'shapedev_r', 'shapedev_e1', 'shapedev_e2',
'shapeexp_r', 'shapeexp_e1', 'shapeexp_e2',
'ref_epoch', 'pmra', 'pmdec', 'parallax'
])
ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W,H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) *
(yy >= -margin) * (yy <= (H+margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
for col in ['out_of_bounds', 'left_blob']:
if col in T.get_columns():
T.cut(T.get(col) == False)
print('Cut to', len(T), 'on', col)
if len(T):
TT.append(T)
if len(TT) == 0:
print('No sources to photometer.')
return None
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
print('Total of', len(T), 'catalog sources')
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I, = np.nonzero([t == 'COMP' and not np.isfinite(r) for t,r in zip(T.type, T.shapedev_r)])
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I, = np.nonzero([t == 'COMP' and not np.isfinite(r) for t,r in zip(T.type, T.shapeexp_r)])
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
surveydir = survey.get_survey_dir()
del survey
if opt.move_gaia:
# Gaia stars: move RA,Dec to the epoch of this image.
I = np.flatnonzero(T.ref_epoch > 0)
if len(I):
from legacypipe.survey import radec_at_mjd
print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd())
ra,dec = radec_at_mjd(T.ra[I], T.dec[I], T.ref_epoch[I].astype(float),
T.pmra[I], T.pmdec[I], T.parallax[I],
tim.time.toMjd())
T.ra [I] = ra
T.dec[I] = dec
kwargs = {}
cols = T.get_columns()
if 'flux_r' in cols and not 'decam_flux_r' in cols:
kwargs.update(fluxPrefix='')
tnow = Time()
print('Read catalog:', tnow-tlast)
tlast = tnow
cat = read_fits_catalog(T, bands='r', **kwargs)
# Replace the brightness (which will be a NanoMaggies with g,r,z)
# with a NanoMaggies with this image's band only.
for src in cat:
src.brightness = NanoMaggies(**{tim.band: 1.})
tnow = Time()
print('Parse catalog:', tnow-tlast)
tlast = tnow
print('Forced photom...')
F = run_forced_phot(cat, tim,
ceres=opt.ceres,
derivs=opt.derivs,
fixed_also=True,
agn=opt.agn,
do_forced=opt.forced,
do_apphot=opt.apphot,
get_model=opt.save_model,
ps=ps)
if opt.save_model:
# unpack results
F,model_img = F
F.release = T.release
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.camera = np.array([ccd.camera] * len(F))
F.expnum = np.array([im.expnum] * len(F)).astype(np.int32)
F.ccdname = np.array([im.ccdname] * len(F))
# "Denormalizing"
F.filter = np.array([tim.band] * len(F))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(F))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(F)).astype(np.float32)
F.psfsize = np.array([tim.psf_fwhm * tim.imobj.pixscale] * len(F)).astype(np.float32)
#### FIXME -- units?
### --> also add units to the dict below so the FITS headers have units
F.sky = np.array([tim.midsky / tim.zpscale / tim.imobj.pixscale**2] * len(F)).astype(np.float32)
# in the same units as the depth maps -- flux inverse-variance.
F.psfdepth = np.array([(1. / (tim.sig1 / tim.psfnorm)**2)] * len(F)).astype(np.float32)
F.galdepth = np.array([(1. / (tim.sig1 / tim.galnorm)**2)] * len(F)).astype(np.float32)
# F.psfdepth = np.array([-2.5 * (np.log10(5. * tim.sig1 / tim.psfnorm) - 9)] * len(F)).astype(np.float32)
# F.galdepth = np.array([-2.5 * (np.log10(5. * tim.sig1 / tim.galnorm) - 9)] * len(F)).astype(np.float32)
# super units questions here
if opt.derivs:
cosdec = np.cos(np.deg2rad(T.dec))
F.dra = (F.flux_dra / F.flux) * 3600. / cosdec
F.ddec = (F.flux_ddec / F.flux) * 3600.
F.dra_ivar = F.flux_dra_ivar * (F.flux / 3600. * cosdec)**2
F.ddec_ivar = F.flux_ddec_ivar * (F.flux / 3600.)**2
F.delete_column('flux_dra')
F.delete_column('flux_ddec')
F.delete_column('flux_dra_ivar')
F.delete_column('flux_ddec_ivar')
F.flux = F.flux_fixed
F.flux_ivar = F.flux_fixed_ivar
F.delete_column('flux_fixed')
F.delete_column('flux_fixed_ivar')
for c in ['dra', 'ddec', 'dra_ivar', 'ddec_ivar', 'flux', 'flux_ivar']:
F.set(c, F.get(c).astype(np.float32))
F.ra = T.ra
F.dec = T.dec
ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x-1).astype(np.float32)
F.y = (y-1).astype(np.float32)
## FIXME -- read outlier_masks?
h,w = tim.shape
F.mask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h-1),
np.clip(np.round(F.x).astype(int), 0, w-1)]
program_name = sys.argv[0]
## FIXME -- from catalog?
release = 0
version_hdr = get_version_header(program_name, surveydir, release)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn.strip())
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(dict(name='EXPOSURE',
value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname),
comment='Name of this image'))
version_hdr.add_record(dict(name='PLVER', value=ccd.plver, comment='CP pipeline version'))
version_hdr.add_record(dict(name='PROCDATE', value=ccd.procdate, comment='CP image DATE'))
keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV',
'INSTRUME']
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
fitsio.write(opt.save_model, model_img, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
tnow = Time()
print('Forced phot:', tnow-tlast)
return F,version_hdr
x = w // 2
if y is None:
y = h // 2
psfmod = tim.psf.getPointSourcePatch(x, y).patch
from tractor.galaxy import ExpGalaxy
from tractor.ellipses import EllipseE
from tractor.patch import ModelMask
band = tim.band
if x is None:
x = w // 2
if y is None:
y = h // 2
pos = tim.wcs.pixelToPosition(x, y)
gal = SimpleGalaxy(pos, NanoMaggies(**{band: 1.}))
S = 32
mm = ModelMask(int(x - S), int(y - S), 2 * S + 1, 2 * S + 1)
galmod = gal.getModelPatch(tim, modelMask=mm).patch
print('Galaxy model shape', galmod.shape)
mx = max(psfmod.max(), galmod.max())
mn = min(psfmod.min(), galmod.min())
ima = dict(interpolation='nearest', origin='lower', vmin=mn, vmax=mx)
print('Range', mn, mx)
plt.clf()
plt.subplot(1, 2, 1)
plt.imshow(psfmod, **ima)
plt.title('PSF model')
def one_brick(X):
(ibrick, brick) = X
bands = ['g', 'r', 'z']
print('Brick', brick.brickname)
wcs = wcs_for_brick(brick, W=94, H=94, pixscale=10.)
BH, BW = wcs.shape
targetrd = np.array([
wcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (BW, 1), (BW, BH), (1, BH), (1, 1)]
])
survey = LegacySurveyData()
C = survey.ccds_touching_wcs(wcs)
if C is None:
print('No CCDs touching brick')
return None
I = np.flatnonzero(C.ccd_cuts == 0)
if len(I) == 0:
print('No good CCDs touching brick')
return None
C.cut(I)
print(len(C), 'CCDs touching brick')
depths = {}
for band in bands:
d = np.zeros((BH, BW), np.float32)
depths[band] = d
npix = dict([(band, 0) for band in bands])
nexps = dict([(band, 0) for band in bands])
# survey.get_approx_wcs(ccd)
for ccd in C:
#im = survey.get_image_object(ccd)
awcs = survey.get_approx_wcs(ccd)
imh, imw = ccd.height, ccd.width
x0, y0 = 0, 0
x1 = x0 + imw
y1 = y0 + imh
imgpoly = [(1, 1), (1, imh), (imw, imh), (imw, 1)]
ok, tx, ty = awcs.radec2pixelxy(targetrd[:-1, 0], targetrd[:-1, 1])
tpoly = list(zip(tx, ty))
clip = clip_polygon(imgpoly, tpoly)
clip = np.array(clip)
if len(clip) == 0:
continue
x0, y0 = np.floor(clip.min(axis=0)).astype(int)
x1, y1 = np.ceil(clip.max(axis=0)).astype(int)
#slc = slice(y0,y1+1), slice(x0,x1+1)
awcs = awcs.get_subimage(x0, y0, x1 - x0, y1 - y0)
ah, aw = awcs.shape
#print('Image', ccd.expnum, ccd.ccdname, ccd.filter, 'overlap', x0,x1, y0,y1, '->', (1+x1-x0),'x',(1+y1-y0))
# Find bbox in brick space
r, d = awcs.pixelxy2radec([1, 1, aw, aw], [1, ah, ah, 1])
ok, bx, by = wcs.radec2pixelxy(r, d)
bx0 = np.clip(np.round(bx.min()).astype(int) - 1, 0, BW - 1)
bx1 = np.clip(np.round(bx.max()).astype(int) - 1, 0, BW - 1)
by0 = np.clip(np.round(by.min()).astype(int) - 1, 0, BH - 1)
by1 = np.clip(np.round(by.max()).astype(int) - 1, 0, BH - 1)
#print('Brick', bx0,bx1,by0,by1)
band = ccd.filter[0]
assert (band in bands)
ccdzpt = ccd.ccdzpt + 2.5 * np.log10(ccd.exptime)
psf_sigma = ccd.fwhm / 2.35
psfnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
orig_zpscale = zpscale = NanoMaggies.zeropointToScale(ccdzpt)
sig1 = ccd.sig1 / orig_zpscale
detsig1 = sig1 / psfnorm
# print('Image', ccd.expnum, ccd.ccdname, ccd.filter,
# 'PSF depth', -2.5 * (np.log10(5.*detsig1) - 9), 'exptime', ccd.exptime,
# 'sig1', ccd.sig1, 'zpt', ccd.ccdzpt, 'fwhm', ccd.fwhm,
# 'filename', ccd.image_filename.strip())
depths[band][by0:by1 + 1, bx0:bx1 + 1] += (1. / detsig1**2)
npix[band] += (y1 + 1 - y0) * (x1 + 1 - x0)
nexps[band] += 1
for band in bands:
det = np.median(depths[band])
# compute stats for 5-sigma detection
with np.errstate(divide='ignore'):
depth = 5. / np.sqrt(det)
# that's flux in nanomaggies -- convert to mag
depth = -2.5 * (np.log10(depth) - 9)
if not np.isfinite(depth):
depth = 0.
depths[band] = depth
#bricks.get('psfdepth_' + band)[ibrick] = depth
print(brick.brickname, 'median PSF depth', band, ':', depth, 'npix',
npix[band], 'nexp', nexps[band])
#'npix', bricks.get('npix_'+band)[ibrick],
#'nexp', bricks.get('nexp_'+band)[ibrick])
return (npix, nexps, depths)
'--no-wise', '--force-all', '--no-write', '--ceres',
'--survey-dir', surveydir,
'--outdir', 'out-testcase3-ceres',
'--no-depth-cut'])
sys.exit(0)
# demo RexGalaxy, with plots
if False:
from legacypipe.survey import RexGalaxy
from tractor import RaDecPos, NanoMaggies, PixPos
from tractor import ScalarParam
from tractor import Image, GaussianMixturePSF, LinearPhotoCal
from legacypipe.survey import LogRadius
rex = RexGalaxy(
PixPos(1., 2.),
NanoMaggies(r=3.),
LogRadius(0.))
print('Rex:', rex)
print('Rex params:', rex.getParams())
print('Rex nparams:', rex.numberOfParams())
H,W = 100,100
tim = Image(data=np.zeros((H,W), np.float32),
inverr=np.ones((H,W), np.float32),
psf=GaussianMixturePSF(1., 0., 0., 4., 4., 0.),
photocal=LinearPhotoCal(1., band='r'))
derivs = rex.getParamDerivatives(tim)
print('Derivs:', len(derivs))
print('Rex params:', rex.getParamNames())
import pylab as plt
from astrometry.util.plotutils import PlotSequence
E = C[I]
print(len(E), 'exposures')
E.index = np.arange(len(E))
E.passnum = np.zeros(len(E), np.uint8)
E.depthfraction = np.zeros(len(E), np.float32)
# Compute which pass number each exposure would be called.
zp0 = DecamImage.nominal_zeropoints()
# HACK -- this is copied from obsbot
kx = dict(g = 0.178, r = 0.094, z = 0.060,)
for band in bands:
B = E[E.filter == band]
B.cut(np.argsort(B.seeing))
Nsigma = 5.
sig = NanoMaggies.magToNanomaggies(target[band]) / Nsigma
targetiv = 1./sig**2
for exp in B:
thisdetiv = 1. / (exp.sig1 / exp.galnorm)**2
# Which pass number would this image be assigned?
trans = 10.**(-0.4 * (zp0[band] - exp.ccdzpt
- kx[band]*(exp.airmass - 1.)))
seeing_good = exp.seeing < 1.3
seeing_fair = exp.seeing < 2.0
trans_good = trans > 0.9
trans_fair = trans > 0.7
if seeing_good and trans_good:
E.passnum[exp.index] = 1
elif ((seeing_good and trans_fair) or (seeing_fair and trans_good)):
E.passnum[exp.index] = 2
def main():
# I read a DESI DR8 target catalog, cut to ELGs, then took a narrow
# r-mag slice around the peak of that distribution;
# desi/target/catalogs/dr8/0.31.1/targets/main/resolve/targets-dr8-hp-1,5,11,50,55,60,83,84,86,89,91,98,119,155,158,174,186,187,190.fits')
# Then took the median g and z mags
# And opened the coadd invvar mags for a random brick in that footprint
# (0701p000) to find the median per-pixel invvars.
r = 23.0
g = 23.4
z = 22.2
# Selecting EXPs, the peak of the shapeexp_r was ~ 0.75".
r_e = 0.75
# Image properties:
giv = 77000.
riv = 27000.
ziv = 8000.
# PSF sizes were roughly equal, 1.16, 1.17, 1.19"
# -> sigma = 1.9 DECam pixels
psf_sigma = 1.9
H, W = 1000, 1000
seed = 42
np.random.seed(seed)
ra, dec = 70., 1.
brick = BrickDuck(ra, dec, 'custom-0700p010')
wcs = wcs_for_brick(brick, W=W, H=H)
bands = 'grz'
tims = []
for band, iv in zip(bands, [giv, riv, ziv]):
img = np.zeros((H, W), np.float32)
tiv = np.zeros_like(img) + iv
s = psf_sigma**2
psf = GaussianMixturePSF(1., 0., 0., s, s, 0.)
twcs = ConstantFitsWcs(wcs)
sky = ConstantSky(0.)
photocal = LinearPhotoCal(1., band=band)
tai = TAITime(None, mjd=55700.)
tim = tractor.Image(data=img,
invvar=tiv,
psf=psf,
wcs=twcs,
sky=sky,
photocal=photocal,
name='fake %s' % band,
time=tai)
tim.skyver = ('1', '1')
tim.psfver = ('1', '1')
tim.plver = '1'
tim.x0 = tim.y0 = 0
tim.subwcs = wcs
tim.psfnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
tim.galnorm = tim.psfnorm
tim.propid = '2020A-000'
tim.band = band
tim.dq = None
tim.sig1 = 1. / np.sqrt(iv)
tim.psf_sigma = psf_sigma
tim.primhdr = fitsio.FITSHDR()
tims.append(tim)
# Simulated catalog
gflux = NanoMaggies.magToNanomaggies(g)
rflux = NanoMaggies.magToNanomaggies(r)
zflux = NanoMaggies.magToNanomaggies(z)
box = 50
CX, CY = np.meshgrid(np.arange(box // 2, W, box),
np.arange(box // 2, H, box))
ny, nx = CX.shape
BA, PHI = np.meshgrid(np.linspace(0.1, 1.0, nx), np.linspace(0., 180., ny))
cat = []
for cx, cy, ba, phi in zip(CX.ravel(), CY.ravel(), BA.ravel(),
PHI.ravel()):
#print('x,y %.1f,%.1f, ba %.2f, phi %.2f' % (cx, cy, ba, phi))
r, d = wcs.pixelxy2radec(cx + 1, cy + 1)
src = ExpGalaxy(RaDecPos(r, d),
NanoMaggies(order=bands, g=gflux, r=rflux, z=zflux),
EllipseE.fromRAbPhi(r_e, ba, phi))
cat.append(src)
from legacypipe.catalog import prepare_fits_catalog
TT = fits_table()
TT.bx = CX.ravel()
TT.by = CY.ravel()
TT.ba = BA.ravel()
TT.phi = PHI.ravel()
tcat = tractor.Catalog(*cat)
T2 = prepare_fits_catalog(tcat, None, TT, bands, save_invvars=False)
T2.writeto('sim-cat.fits')
tr = Tractor(tims, cat)
for band, tim in zip(bands, tims):
mod = tr.getModelImage(tim)
mod += np.random.standard_normal(
size=tim.shape) * 1. / tim.getInvError()
fitsio.write('sim-%s.fits' % band, mod, clobber=True)
tim.data = mod
ccds = fits_table()
ccds.filter = np.array([f for f in bands])
ccds.ccd_cuts = np.zeros(len(ccds), np.int16)
ccds.imgfn = np.array([tim.name for tim in tims])
ccds.propid = np.array(['2020A-000'] * len(ccds))
ccds.fwhm = np.zeros(len(ccds), np.float32) + psf_sigma * 2.35
ccds.mjd_obs = np.zeros(len(ccds))
ccds.camera = np.array(['fake'] * len(ccds))
survey = FakeLegacySurvey(ccds, tims)
import logging
verbose = False
if verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
run_brick(None,
survey,
radec=(ra, dec),
width=W,
height=H,
do_calibs=False,
gaia_stars=False,
large_galaxies=False,
tycho_stars=False,
forceall=True,
outliers=False) #, stages=['image_coadds'])
def rbmain():
travis = 'travis' in sys.argv
extra_args = [
'--old-calibs-ok',
#'--verbose',
]
if travis:
extra_args.extend(
['--no-wise-ceres', '--no-gaia', '--no-large-galaxies'])
if 'ceres' in sys.argv:
surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--ceres',
'--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres',
'--no-depth-cut'
])
sys.exit(0)
# demo RexGalaxy, with plots
if False:
from legacypipe.survey import RexGalaxy
from tractor import NanoMaggies, PixPos
from tractor import Image, GaussianMixturePSF, LinearPhotoCal
from legacypipe.survey import LogRadius
rex = RexGalaxy(PixPos(1., 2.), NanoMaggies(r=3.), LogRadius(0.))
print('Rex:', rex)
print('Rex params:', rex.getParams())
print('Rex nparams:', rex.numberOfParams())
H, W = 100, 100
tim = Image(data=np.zeros((H, W), np.float32),
inverr=np.ones((H, W), np.float32),
psf=GaussianMixturePSF(1., 0., 0., 4., 4., 0.),
photocal=LinearPhotoCal(1., band='r'))
derivs = rex.getParamDerivatives(tim)
print('Derivs:', len(derivs))
print('Rex params:', rex.getParamNames())
import pylab as plt
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('rex')
for d, nm in zip(derivs, rex.getParamNames()):
plt.clf()
plt.imshow(d.patch, interpolation='nearest', origin='lower')
plt.title('Derivative %s' % nm)
ps.savefig()
sys.exit(0)
# Test RexGalaxy
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6-rex'
main(args=[
'--brick',
'1102p240',
'--zoom',
'500',
'600',
'650',
'750',
'--force-all',
'--no-write',
'--no-wise',
#'--rex', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
] + extra_args)
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0] == 'PSF ')
cmd = (
'(cd %s && sha256sum -c %s)' %
(outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum')))
print(cmd)
rtn = os.system(cmd)
assert (rtn == 0)
# Test with a Tycho-2 star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6'
main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--survey-dir', surveydir,
'--outdir', outdir
] + extra_args)
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0] == 'PSF ')
# Test that we can run splinesky calib if required...
from legacypipe.decam import DecamImage
DecamImage.splinesky_boxsize = 128
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
outdir = 'out-testcase4'
fn = os.path.join(surveydir, 'calib', 'decam', 'splinesky', '00431',
'00431608', 'decam-00431608-N3.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
os.path.join(surveydir, 'images', 'unwise-tr'), '--blob-image',
'--no-hybrid-psf', '--survey-dir', surveydir, '--outdir', outdir
] + extra_args + ['-v'])
print('Checking for calib file', fn)
assert (os.path.exists(fn))
# Wrap-around, hybrid PSF
surveydir = os.path.join(os.path.dirname(__file__), 'testcase8')
outdir = 'out-testcase8'
main(args=[
'--brick',
'1209p050',
'--zoom',
'720',
'1095',
'3220',
'3500',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
] + extra_args)
# Test with a Tycho-2 star + another saturated star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase7')
outdir = 'out-testcase7'
# remove --no-gaia
my_extra_args = [a for a in extra_args if a != '--no-gaia']
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia-dr2')
os.environ['GAIA_CAT_VER'] = '2'
main(args=[
'--brick',
'1102p240',
'--zoom',
'250',
'350',
'1550',
'1650',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
] + my_extra_args)
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 4)
# Check skipping blobs outside the brick's unique area.
# (this now doesn't detect any sources at all, reasonably)
# surveydir = os.path.join(os.path.dirname(__file__), 'testcase5')
# outdir = 'out-testcase5'
#
# fn = os.path.join(outdir, 'tractor', '186', 'tractor-1867p255.fits')
# if os.path.exists(fn):
# os.unlink(fn)
#
# main(args=['--brick', '1867p255', '--zoom', '0', '150', '0', '150',
# '--force-all', '--no-write', '--coadd-bw',
# '--survey-dir', surveydir,
# '--early-coadds',
# '--outdir', outdir] + extra_args)
#
# assert(os.path.exists(fn))
# T = fits_table(fn)
# assert(len(T) == 1)
# Custom RA,Dec; blob ra,dec.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
outdir = 'out-testcase4b'
# Catalog written with one entry (--blobradec)
fn = os.path.join(outdir, 'tractor', 'cus',
'tractor-custom-186743p25461.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--radec', '186.743965', '25.461788', '--width', '250', '--height',
'250', '--force-all', '--no-write', '--no-wise', '--blobradec',
'186.740369', '25.453855', '--survey-dir', surveydir, '--outdir',
outdir
] + extra_args)
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 1)
surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
outdir = 'out-testcase3'
checkpoint_fn = os.path.join(outdir, 'checkpoint.pickle')
if os.path.exists(checkpoint_fn):
os.unlink(checkpoint_fn)
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
] + extra_args)
# Read catalog into Tractor sources to test read_fits_catalog
from legacypipe.catalog import read_fits_catalog
from legacypipe.survey import LegacySurveyData, GaiaSource
from tractor.galaxy import DevGalaxy
from tractor import PointSource
survey = LegacySurveyData(survey_dir=outdir)
fn = survey.find_file('tractor', brick='2447p120')
print('Checking', fn)
T = fits_table(fn)
cat = read_fits_catalog(T, fluxPrefix='')
print('Read catalog:', cat)
assert (len(cat) == 2)
src = cat[0]
assert (type(src) == DevGalaxy)
assert (np.abs(src.pos.ra - 244.77973) < 0.00001)
assert (np.abs(src.pos.dec - 12.07233) < 0.00001)
src = cat[1]
print('Source', src)
assert (type(src) in [PointSource, GaiaSource])
assert (np.abs(src.pos.ra - 244.77830) < 0.00001)
assert (np.abs(src.pos.dec - 12.07250) < 0.00001)
# DevGalaxy(pos=RaDecPos[244.77975494973529, 12.072348111713127], brightness=NanoMaggies: g=19.2, r=17.9, z=17.1, shape=re=2.09234, e1=-0.198453, e2=0.023652,
# PointSource(RaDecPos[244.77833280764278, 12.072521274981987], NanoMaggies: g=25, r=23, z=21.7)
# Check that we can run again, using that checkpoint file.
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
] + extra_args)
# Assert...... something?
# Test --checkpoint without --threads
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1'
] + extra_args)
# MzLS + BASS data
# surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass')
# main(args=['--brick', '3521p002', '--zoom', '2400', '2450', '1200', '1250',
# '--no-wise', '--force-all', '--no-write',
# '--survey-dir', surveydir2,
# '--outdir', 'out-mzlsbass'])
# From Kaylan's Bootes pre-DR4 run
# surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass3')
# main(args=['--brick', '2173p350', '--zoom', '100', '200', '100', '200',
# '--no-wise', '--force-all', '--no-write',
# '--survey-dir', surveydir2,
# '--outdir', 'out-mzlsbass3'] + extra_args)
# With plots!
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', 'out-testcase3', '--plots', '--nblobs', '1'
] + extra_args)
# Decals Image Simulations
# Uncomment WHEN galsim build for Travis
#os.environ["DECALS_SIM_DIR"]= os.path.join(os.path.dirname(__file__),'image_sims')
#brick= '2447p120'
#sim_main(args=['--brick', brick, '-n', '2', '-o', 'STAR', \
# '-ic', '1', '--rmag-range', '18', '26', '--threads', '1',\
# '--zoom', '1020', '1070', '2775', '2815'])
# Check if correct files written out
#rt_dir= os.path.join(os.getenv('DECALS_SIM_DIR'),brick,'star','001')
#assert( os.path.exists(os.path.join(rt_dir,'../','metacat-'+brick+'-star.fits')) )
#for fn in ['tractor-%s-star-01.fits' % brick,'simcat-%s-star-01.fits' % brick]:
# assert( os.path.exists(os.path.join(rt_dir,fn)) )
#for fn in ['image','model','resid','simscoadd']:
# assert( os.path.exists(os.path.join(rt_dir,'qa-'+brick+'-star-'+fn+'-01.jpg')) )
if not travis:
# With ceres
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--ceres',
'--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres'
] + extra_args)
def read_fits_catalog(T,
hdr=None,
invvars=False,
bands='grz',
ellipseClass=EllipseE,
sersicIndexClass=SersicIndex):
'''
Return list of tractor Sources.
If invvars=True, return sources,invvars
where invvars is a list matching sources.getParams()
If *ellipseClass* is set, assume that type for galaxy shapes; if None,
read the type from the header.
'''
from tractor import NanoMaggies, RaDecPos
if hdr is None:
hdr = T._header
rev_typemap = fits_reverse_typemap
ivs = []
cat = []
for t in T:
typestr = t.type.strip()
clazz = rev_typemap[typestr]
assert (np.isfinite(t.ra))
assert (np.isfinite(t.dec))
pos = RaDecPos(t.ra, t.dec)
fluxes = {}
for b in bands:
fluxes[b] = t.get('flux_' + b)
assert (np.all(np.isfinite(fluxes[b])))
br = NanoMaggies(order=bands, **fluxes)
params = [pos, br]
if invvars:
fluxivs = []
for b in bands:
fluxivs.append(t.get('flux_ivar_' + b))
ivs.extend([t.ra_ivar, t.dec_ivar] + fluxivs)
if issubclass(clazz, (DevGalaxy, ExpGalaxy, SersicGalaxy)):
assert (np.isfinite(t.shape_r))
assert (np.isfinite(t.shape_e1))
assert (np.isfinite(t.shape_e2))
ell = ellipseClass(t.shape_r, t.shape_e1, t.shape_e2)
params.append(ell)
if invvars:
ivs.extend([t.shape_r_ivar, t.shape_e1_ivar, t.shape_e2_ivar])
elif issubclass(clazz, PointSource):
pass
else:
raise RuntimeError('Unknown class %s' % str(clazz))
if issubclass(clazz, SersicGalaxy):
assert (np.isfinite(t.sersic))
si = sersicIndexClass(t.sersic)
params.append(si)
if invvars:
ivs.append(t.sersic_ivar)
src = clazz(*params)
cat.append(src)
if invvars:
ivs = np.array(ivs)
ivs[np.logical_not(np.isfinite(ivs))] = 0
return cat, ivs
return cat
all_runs = []
tim = Image(data=np.zeros((sz,sz)), inverr=np.ones((sz,sz)) / sig1,
psf=psf,
wcs = ConstantFitsWcs(wcs),
photocal = LinearPhotoCal(1., band=band))
for i,sn in enumerate(sn_vals):
for j,re in enumerate(re_vals):
## HACK -- this is the flux required for a PSF to be
## detected at target S/N... adjust for galaxy?
flux = sn * detsig1
# Create round EXP galaxy
#PixPos(sz/2, sz/2),
true_src = ExpGalaxy(RaDecPos(0., 0.),
NanoMaggies(**{band: flux}),
EllipseE(re, 0., 0.))
tr = Tractor([tim], [true_src])
tr.freezeParams('images')
true_mod = tr.getModelImage(0)
ima = dict(interpolation='nearest', origin='lower',
vmin=-2.*sig1, vmax=5.*sig1, cmap='hot')
this_dchisqs = []
flux_sns = []
for k in range(Nper):
noise = np.random.normal(scale=sig1, size=true_mod.shape)
def galex_coadds(onegal,
galaxy=None,
radius_mosaic=30,
radius_mask=None,
pixscale=1.5,
ref_pixscale=0.262,
output_dir=None,
galex_dir=None,
log=None,
centrals=True,
verbose=False):
'''Generate custom GALEX cutouts.
radius_mosaic and radius_mask in arcsec
pixscale: GALEX pixel scale in arcsec/pixel.
'''
import fitsio
import matplotlib.pyplot as plt
from astrometry.libkd.spherematch import match_radec
from astrometry.util.resample import resample_with_wcs, OverlapError
from tractor import (Tractor, NanoMaggies, Image, LinearPhotoCal,
NCircularGaussianPSF, ConstantFitsWcs, ConstantSky)
from legacypipe.survey import imsave_jpeg
from legacypipe.catalog import read_fits_catalog
if galaxy is None:
galaxy = 'galaxy'
if galex_dir is None:
galex_dir = os.environ.get('GALEX_DIR')
if output_dir is None:
output_dir = '.'
if radius_mask is None:
radius_mask = radius_mosaic
radius_search = 5.0 # [arcsec]
else:
radius_search = radius_mask
W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int') # [pixels]
targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0,
-pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W),
float(H))
# Read the custom Tractor catalog
tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy))
if not os.path.isfile(tractorfile):
print('Missing Tractor catalog {}'.format(tractorfile))
return 0
cat = fits_table(tractorfile)
print('Read {} sources from {}'.format(len(cat), tractorfile),
flush=True,
file=log)
keep = np.ones(len(cat)).astype(bool)
if centrals:
# Find the large central galaxy and mask out (ignore) all the models
# which are within its elliptical mask.
# This algorithm will have to change for mosaics not centered on large
# galaxies, e.g., in galaxy groups.
m1, m2, d12 = match_radec(cat.ra,
cat.dec,
onegal['RA'],
onegal['DEC'],
radius_search / 3600.0,
nearest=False)
if len(m1) == 0:
print('No central galaxies found at the central coordinates!',
flush=True,
file=log)
else:
pixfactor = ref_pixscale / pixscale # shift the optical Tractor positions
for mm in m1:
morphtype = cat.type[mm].strip()
if morphtype == 'EXP' or morphtype == 'COMP':
e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[
mm], cat.shapeexp_r[mm] # [arcsec]
elif morphtype == 'DEV' or morphtype == 'COMP':
e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[
mm], cat.shapedev_r[mm] # [arcsec]
else:
r50 = None
if r50:
majoraxis = r50 * 5 / pixscale # [pixels]
ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2)
these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis,
ba * majoraxis,
np.radians(phi),
cat.bx * pixfactor,
cat.by * pixfactor)
if np.sum(these) > 0:
#keep[these] = False
pass
print('Hack!')
keep[mm] = False
#srcs = read_fits_catalog(cat)
#_srcs = np.array(srcs)[~keep].tolist()
#mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0)
#import matplotlib.pyplot as plt
##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png')
#plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png')
#pdb.set_trace()
srcs = read_fits_catalog(cat)
for src in srcs:
src.freezeAllBut('brightness')
#srcs_nocentral = np.array(srcs)[keep].tolist()
# Find all overlapping GALEX tiles and then read the tims.
galex_tiles = _read_galex_tiles(targetwcs,
galex_dir,
log=log,
verbose=verbose)
gbands = ['n', 'f']
nicegbands = ['NUV', 'FUV']
zps = dict(n=20.08, f=18.82)
coimgs, comods, coresids, coimgs_central, comods_nocentral = [], [], [], [], []
for niceband, band in zip(nicegbands, gbands):
J = np.flatnonzero(galex_tiles.get('has_' + band))
print(len(J), 'GALEX tiles have coverage in band', band)
coimg = np.zeros((H, W), np.float32)
comod = np.zeros((H, W), np.float32)
cowt = np.zeros((H, W), np.float32)
comod_nocentral = np.zeros((H, W), np.float32)
for src in srcs:
src.setBrightness(NanoMaggies(**{band: 1}))
for j in J:
brick = galex_tiles[j]
fn = os.path.join(
galex_dir, brick.tilename.strip(),
'%s-%sd-intbgsub.fits.gz' % (brick.brickname, band))
#print(fn)
gwcs = Tan(*[
float(f) for f in [
brick.crval1, brick.crval2, brick.crpix1, brick.crpix2,
brick.cdelt1, 0., 0., brick.cdelt2, 3840., 3840.
]
])
img = fitsio.read(fn)
#print('Read', img.shape)
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, gwcs, [], 3)
except OverlapError:
continue
K = np.flatnonzero(img[Yi, Xi] != 0.)
if len(K) == 0:
continue
Yo, Xo, Yi, Xi = Yo[K], Xo[K], Yi[K], Xi[K]
wt = brick.get(band + 'exptime')
coimg[Yo, Xo] += wt * img[Yi, Xi]
cowt[Yo, Xo] += wt
x0, x1, y0, y1 = min(Xi), max(Xi), min(Yi), max(Yi)
subwcs = gwcs.get_subimage(x0, y0, x1 - x0 + 1, y1 - y0 + 1)
twcs = ConstantFitsWcs(subwcs)
timg = img[y0:y1 + 1, x0:x1 + 1]
tie = np.ones_like(timg) ## HACK!
#hdr = fitsio.read_header(fn)
#zp = hdr['']
zp = zps[band]
photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp),
band=band)
tsky = ConstantSky(0.0)
# HACK -- circular Gaussian PSF of fixed size...
# in arcsec
#fwhms = dict(NUV=6.0, FUV=6.0)
# -> sigma in pixels
#sig = fwhms[band] / 2.35 / twcs.pixel_scale()
sig = 6.0 / np.sqrt(8 * np.log(2)) / twcs.pixel_scale()
tpsf = NCircularGaussianPSF([sig], [1.])
tim = Image(data=timg,
inverr=tie,
psf=tpsf,
wcs=twcs,
sky=tsky,
photocal=photocal,
name='GALEX ' + band + brick.brickname)
## Build the model image with and without the central galaxy model.
tractor = Tractor([tim], srcs)
mod = tractor.getModelImage(0)
tractor.freezeParam('images')
tractor.optimize_forced_photometry(priors=False,
shared_params=False)
mod = tractor.getModelImage(0)
srcs_nocentral = np.array(srcs)[keep].tolist()
#srcs_nocentral = np.array(srcs)[nocentral].tolist()
tractor_nocentral = Tractor([tim], srcs_nocentral)
mod_nocentral = tractor_nocentral.getModelImage(0)
comod[Yo, Xo] += wt * mod[Yi - y0, Xi - x0]
comod_nocentral[Yo, Xo] += wt * mod_nocentral[Yi - y0, Xi - x0]
coimg /= np.maximum(cowt, 1e-18)
comod /= np.maximum(cowt, 1e-18)
comod_nocentral /= np.maximum(cowt, 1e-18)
coresid = coimg - comod
# Subtract the model image which excludes the central (comod_nocentral)
# from the data (coimg) to isolate the light of the central
# (coimg_central).
coimg_central = coimg - comod_nocentral
coimgs.append(coimg)
comods.append(comod)
coresids.append(coresid)
comods_nocentral.append(comod_nocentral)
coimgs_central.append(coimg_central)
# Write out the final images with and without the central, making sure
# to apply the zeropoint to go from counts/s to AB nanomaggies.
# https://asd.gsfc.nasa.gov/archive/galex/FAQ/counts_background.html
for thisimg, imtype in zip((coimg, comod, comod_nocentral),
('image', 'model', 'model-nocentral')):
fitsfile = os.path.join(
output_dir, '{}-{}-{}.fits'.format(galaxy, imtype, niceband))
if verbose:
print('Writing {}'.format(fitsfile))
fitsio.write(fitsfile,
thisimg * 10**(-0.4 * (zp - 22.5)),
clobber=True)
# Build a color mosaic (but note that the images here are in units of
# background-subtracted counts/s).
#_galex_rgb = _galex_rgb_moustakas
#_galex_rgb = _galex_rgb_dstn
_galex_rgb = _galex_rgb_official
for imgs, imtype in zip(
(coimgs, comods, coresids, comods_nocentral, coimgs_central),
('image', 'model', 'resid', 'model-nocentral', 'image-central')):
rgb = _galex_rgb(imgs)
jpgfile = os.path.join(output_dir,
'{}-{}-FUVNUV.jpg'.format(galaxy, imtype))
if verbose:
print('Writing {}'.format(jpgfile))
imsave_jpeg(jpgfile, rgb, origin='lower')
return 1
def unwise_forcedphot(cat, tiles, bands=[1, 2, 3, 4], roiradecbox=None,
unwise_dir='.',
use_ceres=True, ceres_block=8,
save_fits=False, ps=None,
psf_broadening=None):
'''
Given a list of tractor sources *cat*
and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id)
runs forced photometry, returning a FITS table the same length as *cat*.
'''
# Severely limit sizes of models
for src in cat:
if isinstance(src, PointSource):
src.fixedRadius = 10
else:
src.halfsize = 10
wantims = ((ps is not None) or save_fits)
wanyband = 'w'
fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix',
'pronexp']
Nsrcs = len(cat)
phot = fits_table()
phot.tile = np.array([' '] * Nsrcs)
ra = np.array([src.getPosition().ra for src in cat])
dec = np.array([src.getPosition().dec for src in cat])
for band in bands:
print('Photometering WISE band', band)
wband = 'w%i' % band
# The tiles have some overlap, so for each source, keep the
# fit in the tile whose center is closest to the source.
tiledists = np.empty(Nsrcs)
tiledists[:] = 1e100
flux_invvars = np.zeros(Nsrcs, np.float32)
fitstats = dict([(k, np.zeros(Nsrcs, np.float32)) for k in fskeys])
nexp = np.zeros(Nsrcs, np.int16)
mjd = np.zeros(Nsrcs, np.float64)
for tile in tiles:
print('Reading tile', tile.coadd_id)
tim = get_unwise_tractor_image(unwise_dir, tile.coadd_id, band,
bandname=wanyband, roiradecbox=roiradecbox)
if tim is None:
print('Actually, no overlap with tile', tile.coadd_id)
continue
if psf_broadening is not None:
# psf_broadening is a factor by which the PSF FWHMs
# should be scaled; the PSF is a little wider
# post-reactivation.
psf = tim.getPsf()
from tractor import GaussianMixturePSF
if isinstance(psf, GaussianMixturePSF):
#
print('Broadening PSF: from', psf)
p0 = psf.getParams()
#print('Params:', p0)
pnames = psf.getParamNames()
#print('Param names:', pnames)
p1 = [p * psf_broadening**2 if 'var' in name else p
for (p, name) in zip(p0, pnames)]
#print('Broadened:', p1)
psf.setParams(p1)
print('Broadened PSF:', psf)
else:
print(
'WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf))
print('Read image with shape', tim.shape)
# Select sources in play.
wcs = tim.wcs.wcs
H, W = tim.shape
ok, x, y = wcs.radec2pixelxy(ra, dec)
x = (x - 1.).astype(np.float32)
y = (y - 1.).astype(np.float32)
margin = 10.
I = np.flatnonzero((x >= -margin) * (x < W + margin) *
(y >= -margin) * (y < H + margin))
print(len(I), 'within the image + margin')
inbox = ((x[I] >= -0.5) * (x[I] < (W - 0.5)) *
(y[I] >= -0.5) * (y[I] < (H - 0.5)))
print(sum(inbox), 'strictly within the image')
# Compute L_inf distance to (full) tile center.
tilewcs = unwise_tile_wcs(tile.ra, tile.dec)
cx, cy = tilewcs.crpix
ok, tx, ty = tilewcs.radec2pixelxy(ra[I], dec[I])
td = np.maximum(np.abs(tx - cx), np.abs(ty - cy))
closest = (td < tiledists[I])
tiledists[I[closest]] = td[closest]
keep = inbox * closest
# Source indices (in the full "cat") to keep (the fit values for)
srci = I[keep]
if not len(srci):
print('No sources to be kept; skipping.')
continue
phot.tile[srci] = tile.coadd_id
nexp[srci] = tim.nuims[np.clip(np.round(y[srci]).astype(int), 0, H - 1),
np.clip(np.round(x[srci]).astype(int), 0, W - 1)]
# Source indices in the margins
margi = I[np.logical_not(keep)]
# sources in the box -- at the start of the subcat list.
subcat = [cat[i] for i in srci]
# include *copies* of sources in the margins
# (that way we automatically don't save the results)
subcat.extend([cat[i].copy() for i in margi])
assert(len(subcat) == len(I))
# FIXME -- set source radii, ...?
minsb = 0.
fitsky = False
# Look in image and set radius based on peak height??
tractor = Tractor([tim], subcat)
if use_ceres:
from tractor.ceres_optimizer import CeresOptimizer
tractor.optimizer = CeresOptimizer(BW=ceres_block,
BH=ceres_block)
tractor.freezeParamsRecursive('*')
tractor.thawPathsTo(wanyband)
kwa = dict(fitstat_extras=[('pronexp', [tim.nims])])
t0 = Time()
R = tractor.optimize_forced_photometry(
minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True,
variance=True, shared_params=False,
wantims=wantims, **kwa)
print('unWISE forced photometry took', Time() - t0)
if use_ceres:
term = R.ceres_status['termination']
print('Ceres termination status:', term)
# Running out of memory can cause failure to converge
# and term status = 2.
# Fail completely in this case.
if term != 0:
raise RuntimeError(
'Ceres terminated with status %i' % term)
if wantims:
ims0 = R.ims0
ims1 = R.ims1
IV, fs = R.IV, R.fitstats
if save_fits:
import fitsio
(dat, mod, ie, chi, roi) = ims1[0]
wcshdr = fitsio.FITSHDR()
tim.wcs.wcs.add_to_header(wcshdr)
tag = 'fit-%s-w%i' % (tile.coadd_id, band)
fitsio.write('%s-data.fits' %
tag, dat, clobber=True, header=wcshdr)
fitsio.write('%s-mod.fits' % tag, mod,
clobber=True, header=wcshdr)
fitsio.write('%s-chi.fits' % tag, chi,
clobber=True, header=wcshdr)
if ps:
tag = '%s W%i' % (tile.coadd_id, band)
(dat, mod, ie, chi, roi) = ims1[0]
sig1 = tim.sig1
plt.clf()
plt.imshow(dat, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
plt.colorbar()
plt.title('%s: data' % tag)
ps.savefig()
plt.clf()
plt.imshow(mod, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
plt.colorbar()
plt.title('%s: model' % tag)
ps.savefig()
plt.clf()
plt.imshow(chi, interpolation='nearest', origin='lower',
cmap='gray', vmin=-5, vmax=+5)
plt.colorbar()
plt.title('%s: chi' % tag)
ps.savefig()
# Save results for this tile.
# the "keep" sources are at the beginning of the "subcat" list
flux_invvars[srci] = IV[:len(srci)].astype(np.float32)
if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'):
mjd[srci] = (tim.mjdmin + tim.mjdmax) / 2.
if fs is None:
continue
for k in fskeys:
x = getattr(fs, k)
# fitstats are returned only for un-frozen sources
fitstats[k][srci] = np.array(x).astype(np.float32)[:len(srci)]
# Note, this is *outside* the loop over tiles.
# The fluxes are saved in the source objects, and will be set based on
# the 'tiledists' logic above.
nm = np.array([src.getBrightness().getBand(wanyband) for src in cat])
nm_ivar = flux_invvars
# Sources out of bounds, eg, never change from their default
# (1-sigma or whatever) initial fluxes. Zero them out instead.
nm[nm_ivar == 0] = 0.
phot.set(wband + '_nanomaggies', nm.astype(np.float32))
phot.set(wband + '_nanomaggies_ivar', nm_ivar)
dnm = np.zeros(len(nm_ivar), np.float32)
okiv = (nm_ivar > 0)
dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32)
okflux = (nm > 0)
mag = np.zeros(len(nm), np.float32)
mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux])
).astype(np.float32)
dmag = np.zeros(len(nm), np.float32)
ok = (okiv * okflux)
dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok])
).astype(np.float32)
mag[np.logical_not(okflux)] = np.nan
dmag[np.logical_not(ok)] = np.nan
phot.set(wband + '_mag', mag)
phot.set(wband + '_mag_err', dmag)
for k in fskeys:
phot.set(wband + '_' + k, fitstats[k])
phot.set(wband + '_nexp', nexp)
if not np.all(mjd == 0):
phot.set(wband + '_mjd', mjd)
return phot
comods = []
srcs = read_fits_catalog(T)
for src in srcs:
src.freezeAllBut('brightness')
for band in gbands:
J = np.flatnonzero(galex.get('has_' + band))
print(len(J), 'GALEX tiles have coverage in band', band)
coimg = np.zeros((H, W), np.float32)
comod = np.zeros((H, W), np.float32)
cowt = np.zeros((H, W), np.float32)
for src in srcs:
src.setBrightness(NanoMaggies(**{band: 1}))
for j in J:
brick = galex[j]
fn = os.path.join(
opt.galex_dir, brick.tilename.strip(),
'%s-%sd-intbgsub.fits.gz' % (brick.brickname, band))
print(fn)
gwcs = Tan(*[
float(f) for f in [
brick.crval1, brick.crval2, brick.crpix1, brick.crpix2,
brick.cdelt1, 0., 0., brick.cdelt2, 3840., 3840.
]
])
img = fitsio.read(fn)
def main():
# Where are the data?
datadir = os.path.join(os.path.dirname(__file__), 'data-decam')
name = 'decam-520206-S16'
imagefn = os.path.join(datadir, '%s-image-sub.fits' % name)
invvarfn = os.path.join(datadir, '%s-invvar-sub.fits' % name)
psfexfn = os.path.join(datadir, '%s-psfex.fits' % name)
catfn = os.path.join(datadir, 'tractor-1816p325-sub.fits')
# Read the image and inverse-variance maps.
image = fitsio.read(imagefn)
invvar = fitsio.read(invvarfn)
# The DECam inverse-variance maps are unfortunately corrupted
# by fpack, causing zeros to become negative. Fix those.
invvar[invvar < np.median(invvar) * 0.1] = 0.
H, W = image.shape
print('Subimage size:', image.shape)
# For the PSF model, we need to know what subimage region this is:
subimage_offset = (35, 1465)
# We also need the calibrated zeropoint.
zeropoint = 24.7787
# What filter was this image taken in? (z)
prim_header = fitsio.read_header(imagefn)
band = prim_header['FILTER'].strip()[0]
print('Band:', band)
# These DECam images were calibrated so that the zeropoints need
# an exposure-time factor, so add that in.
exptime = prim_header['EXPTIME']
zeropoint += 2.5 * np.log10(exptime)
# Read the PsfEx model file
psf = PixelizedPsfEx(psfexfn)
# Instantiate a constant pixelized PSF at the image center
# (of the subimage)
x0, y0 = subimage_offset
psf = psf.constantPsfAt(x0 + W / 2., y0 + H / 2.)
# Load the WCS model from the header
# We convert from the RA---TPV type to RA---SIP
header = fitsio.read_header(imagefn, ext=1)
wcsobj = wcs_pv2sip_hdr(header, stepsize=10)
# We'll just use a rough sky estimate...
skyval = np.median(image)
# Create the Tractor Image (tim).
tim = Image(data=image,
invvar=invvar,
psf=psf,
wcs=ConstantFitsWcs(wcsobj),
sky=ConstantSky(skyval),
photocal=LinearPhotoCal(
NanoMaggies.zeropointToScale(zeropoint), band=band))
# Read the official DECaLS DR3 catalog -- it has only two sources in this subimage.
catalog = fits_table(catfn)
print('Read', len(catalog), 'sources')
print('Source types:', catalog.type)
# Create Tractor sources corresponding to these two catalog
# entries.
# In DECaLS, the "SIMP" type is a round Exponential galaxy with a
# fixed 0.45" radius, but we'll treat it as a general Exp galaxy.
sources = []
for c in catalog:
# Create a "position" object given the catalog RA,Dec
position = RaDecPos(c.ra, c.dec)
# Create a "brightness" object; in the catalog, the fluxes are
# stored in a [ugrizY] array, so pull out the right index
band_index = 'ugrizY'.index(band)
flux = c.decam_flux[band_index]
brightness = NanoMaggies(**{band: flux})
# Depending on the source classification in the catalog, pull
# out different fields for the galaxy shape, and for the
# galaxy type. The DECaLS catalogs, conveniently, store
# galaxy shapes as (radius, e1, e2) ellipses.
if c.type.strip() == 'DEV':
shape = EllipseE(c.shapedev_r, c.shapedev_e1, c.shapedev_e2)
galclass = DevGalaxy
elif c.type.strip() == 'SIMP':
shape = EllipseE(c.shapeexp_r, c.shapeexp_e1, c.shapeexp_e2)
galclass = ExpGalaxy
else:
assert (False)
# Create the tractor galaxy object
source = galclass(position, brightness, shape)
print('Created', source)
sources.append(source)
# Create the Tractor object -- a list of tractor Images and a list of tractor sources.
tractor = Tractor([tim], sources)
# Render the initial model image.
print('Getting initial model...')
mod = tractor.getModelImage(0)
make_plot(tim, mod, 'Initial Scene', 'mod0.png')
# Instantiate a new source at the location of the unmodelled peak.
print('Adding new source...')
# Find the peak very naively...
ipeak = np.argmax((image - mod) * tim.inverr)
iy, ix = np.unravel_index(ipeak, tim.shape)
print('Residual peak at', ix, iy)
# Compute the RA,Dec location of the peak...
radec = tim.getWcs().pixelToPosition(ix, iy)
print('RA,Dec', radec)
# Try modelling it as a point source.
# We'll initialize the brightness arbitrarily to 1 nanomaggy (= mag 22.5)
brightness = NanoMaggies(**{band: 1.})
source = PointSource(radec, brightness)
# Add it to the catalog!
tractor.catalog.append(source)
# Render the new model image with this source added.
mod = tractor.getModelImage(0)
make_plot(tim, mod, 'New Source (Before Fit)', 'mod1.png')
print('Fitting new source...')
# Now we're going to fit for the properties of the new source we
# added.
# We don't want to fit for any of the image calibration properties:
tractor.freezeParam('images')
# And we don't (yet) want to fit the existing sources. The new
# source is index number 2, so freeze everything else in the catalog.
tractor.catalog.freezeAllBut(2)
print('Fitting parameters:')
tractor.printThawedParams()
# Do the actual optimization:
tractor.optimize_loop()
mod = tractor.getModelImage(0)
make_plot(tim, mod, 'New Source Fit', 'mod2.png')
print('Fitting sources simultaneously...')
# Now let's unfreeze all the sources and fit them simultaneously.
tractor.catalog.thawAllParams()
tractor.printThawedParams()
tractor.optimize_loop()
mod = tractor.getModelImage(0)
make_plot(tim, mod, 'Simultaneous Fit', 'mod3.png')
def wise_cutouts(ra,
dec,
radius,
ps,
pixscale=2.75,
survey_dir=None,
unwise_dir=None):
'''
radius in arcsec.
pixscale: WISE pixel scale in arcsec/pixel;
make this smaller than 2.75 to oversample.
'''
if unwise_dir is None:
unwise_dir = os.environ.get('UNWISE_COADDS_DIR')
npix = int(np.ceil(radius / pixscale))
print('Image size:', npix)
W = H = npix
pix = pixscale / 3600.
wcs = Tan(ra, dec, (W + 1) / 2., (H + 1) / 2., -pix, 0., 0., pix, float(W),
float(H))
# Find DECaLS bricks overlapping
survey = LegacySurveyData(survey_dir=survey_dir)
B = bricks_touching_wcs(wcs, survey=survey)
print('Found', len(B), 'bricks overlapping')
TT = []
for b in B.brickname:
fn = survey.find_file('tractor', brick=b)
T = fits_table(fn)
print('Read', len(T), 'from', b)
primhdr = fitsio.read_header(fn)
TT.append(T)
T = merge_tables(TT)
print('Total of', len(T), 'sources')
T.cut(T.brick_primary)
print(len(T), 'primary')
margin = 20
ok, xx, yy = wcs.radec2pixelxy(T.ra, T.dec)
I = np.flatnonzero((xx > -margin) * (yy > -margin) * (xx < W + margin) *
(yy < H + margin))
T.cut(I)
print(len(T), 'within ROI')
#return wcs,T
# Pull out DECaLS coadds (image, model, resid).
dwcs = wcs.scale(2. * pixscale / 0.262)
dh, dw = dwcs.shape
print('DECaLS resampled shape:', dh, dw)
tags = ['image', 'model', 'resid']
coimgs = [np.zeros((dh, dw, 3), np.uint8) for t in tags]
for b in B.brickname:
fn = survey.find_file('image', brick=b, band='r')
bwcs = Tan(fn, 1) # ext 1: .fz
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(dwcs, bwcs)
except ResampleError:
continue
if len(Yo) == 0:
continue
print('Resampling', len(Yo), 'pixels from', b)
xl, xh, yl, yh = Xi.min(), Xi.max(), Yi.min(), Yi.max()
#print('python legacypipe/runbrick.py -b %s --zoom %i %i %i %i --outdir cluster --pixpsf --splinesky --pipe --no-early-coadds' %
# (b, xl-5, xh+5, yl-5, yh+5) + ' -P \'pickles/cluster-%(brick)s-%%(stage)s.pickle\'')
for i, tag in enumerate(tags):
fn = survey.find_file(tag + '-jpeg', brick=b)
img = plt.imread(fn)
img = np.flipud(img)
coimgs[i][Yo, Xo, :] = img[Yi, Xi, :]
tt = dict(image='Image', model='Model', resid='Resid')
for img, tag in zip(coimgs, tags):
plt.clf()
dimshow(img, ticks=False)
plt.title('DECaLS grz %s' % tt[tag])
ps.savefig()
# Find unWISE tiles overlapping
tiles = unwise_tiles_touching_wcs(wcs)
print('Cut to', len(tiles), 'unWISE tiles')
# Here we assume the targetwcs is axis-aligned and that the
# edge midpoints yield the RA,Dec limits (true for TAN).
r, d = wcs.pixelxy2radec(np.array([1, W, W / 2, W / 2]),
np.array([H / 2, H / 2, 1, H]))
# the way the roiradec box is used, the min/max order doesn't matter
roiradec = [r[0], r[1], d[2], d[3]]
ra, dec = T.ra, T.dec
srcs = read_fits_catalog(T)
wbands = [1, 2, 3, 4]
wanyband = 'w'
for band in wbands:
f = T.get('flux_w%i' % band)
f *= 10.**(primhdr['WISEAB%i' % band] / 2.5)
coimgs = [np.zeros((H, W), np.float32) for b in wbands]
comods = [np.zeros((H, W), np.float32) for b in wbands]
con = [np.zeros((H, W), np.uint8) for b in wbands]
for iband, band in enumerate(wbands):
print('Photometering WISE band', band)
wband = 'w%i' % band
for i, src in enumerate(srcs):
#print('Source', src, 'brightness', src.getBrightness(), 'params', src.getBrightness().getParams())
#src.getBrightness().setParams([T.wise_flux[i, band-1]])
src.setBrightness(
NanoMaggies(**{wanyband: T.get('flux_w%i' % band)[i]}))
# print('Set source brightness:', src.getBrightness())
# The tiles have some overlap, so for each source, keep the
# fit in the tile whose center is closest to the source.
for tile in tiles:
print('Reading tile', tile.coadd_id)
tim = get_unwise_tractor_image(unwise_dir,
tile.coadd_id,
band,
bandname=wanyband,
roiradecbox=roiradec)
if tim is None:
print('Actually, no overlap with tile', tile.coadd_id)
continue
print('Read image with shape', tim.shape)
# Select sources in play.
wisewcs = tim.wcs.wcs
H, W = tim.shape
ok, x, y = wisewcs.radec2pixelxy(ra, dec)
x = (x - 1.).astype(np.float32)
y = (y - 1.).astype(np.float32)
margin = 10.
I = np.flatnonzero((x >= -margin) * (x < W + margin) *
(y >= -margin) * (y < H + margin))
print(len(I), 'within the image + margin')
subcat = [srcs[i] for i in I]
tractor = Tractor([tim], subcat)
mod = tractor.getModelImage(0)
# plt.clf()
# dimshow(tim.getImage(), ticks=False)
# plt.title('WISE %s %s' % (tile.coadd_id, wband))
# ps.savefig()
# plt.clf()
# dimshow(mod, ticks=False)
# plt.title('WISE %s %s' % (tile.coadd_id, wband))
# ps.savefig()
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(wcs, tim.wcs.wcs)
except ResampleError:
continue
if len(Yo) == 0:
continue
print('Resampling', len(Yo), 'pixels from WISE', tile.coadd_id,
band)
coimgs[iband][Yo, Xo] += tim.getImage()[Yi, Xi]
comods[iband][Yo, Xo] += mod[Yi, Xi]
con[iband][Yo, Xo] += 1
for img, mod, n in zip(coimgs, comods, con):
img /= np.maximum(n, 1)
mod /= np.maximum(n, 1)
for band, img, mod in zip(wbands, coimgs, comods):
lo, hi = np.percentile(img, [25, 99])
plt.clf()
dimshow(img, vmin=lo, vmax=hi, ticks=False)
plt.title('WISE W%i Data' % band)
ps.savefig()
plt.clf()
dimshow(mod, vmin=lo, vmax=hi, ticks=False)
plt.title('WISE W%i Model' % band)
ps.savefig()
resid = img - mod
mx = np.abs(resid).max()
plt.clf()
dimshow(resid, vmin=-mx, vmax=mx, ticks=False)
plt.title('WISE W%i Resid' % band)
ps.savefig()
#kwa = dict(mn=-0.1, mx=2., arcsinh = 1.)
kwa = dict(mn=-0.1, mx=2., arcsinh=None)
rgb = _unwise_to_rgb(coimgs[:2], **kwa)
plt.clf()
dimshow(rgb, ticks=False)
plt.title('WISE W1/W2 Data')
ps.savefig()
rgb = _unwise_to_rgb(comods[:2], **kwa)
plt.clf()
dimshow(rgb, ticks=False)
plt.title('WISE W1/W2 Model')
ps.savefig()
kwa = dict(mn=-1, mx=1, arcsinh=None)
rgb = _unwise_to_rgb(
[img - mod for img, mod in list(zip(coimgs, comods))[:2]], **kwa)
plt.clf()
dimshow(rgb, ticks=False)
plt.title('WISE W1/W2 Resid')
ps.savefig()
return wcs, T
def read_fits_catalog(T,
hdr=None,
invvars=False,
bands='grz',
allbands=None,
ellipseClass=EllipseE,
unpackShape=True,
fluxPrefix=''):
'''
This is currently a weird hybrid of dynamic and hard-coded.
Return list of tractor Sources.
If invvars=True, return sources,invvars
where invvars is a list matching sources.getParams()
If *ellipseClass* is set, assume that type for galaxy shapes; if None,
read the type from the header.
If *unpackShapes* is True and *ellipseClass* is EllipseE, read
catalog entries "shapeexp_r", "shapeexp_e1", "shapeexp_e2" rather than
"shapeExp", and similarly for "dev".
'''
from tractor import NanoMaggies
if hdr is None:
hdr = T._header
if allbands is None:
allbands = bands
rev_typemap = fits_reverse_typemap
if unpackShape and ellipseClass != EllipseE:
print('Not doing unpackShape because ellipseClass != EllipseE.')
unpackShape = False
if unpackShape:
T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T
T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T
ibands = np.array([allbands.index(b) for b in bands])
ivs = []
cat = []
for t in T:
clazz = rev_typemap[t.type.strip()]
pos = RaDecPos(t.ra, t.dec)
assert (np.isfinite(t.ra))
assert (np.isfinite(t.dec))
shorttype = fits_short_typemap[clazz]
if fluxPrefix + 'flux' in t.get_columns():
flux = np.atleast_1d(t.get(fluxPrefix + 'flux'))
assert (np.all(np.isfinite(flux[ibands])))
br = NanoMaggies(order=bands, **dict(zip(bands, flux[ibands])))
else:
fluxes = {}
for b in bands:
fluxes[b] = t.get(fluxPrefix + 'flux_' + b)
assert (np.all(np.isfinite(fluxes[b])))
br = NanoMaggies(order=bands, **fluxes)
params = [pos, br]
if invvars:
# ASSUME & hard-code that the position and brightness are
# the first params
if fluxPrefix + 'flux_ivar' in t.get_columns():
fluxiv = np.atleast_1d(t.get(fluxPrefix + 'flux_ivar'))
fluxivs = list(fluxiv[ibands])
else:
fluxivs = []
for b in bands:
fluxivs.append(t.get(fluxPrefix + 'flux_ivar_' + b))
ivs.extend([t.ra_ivar, t.dec_ivar] + fluxivs)
if issubclass(clazz, (DevGalaxy, ExpGalaxy)):
if ellipseClass is not None:
eclazz = ellipseClass
else:
# hard-code knowledge that third param is the ellipse
eclazz = hdr['TR_%s_T3' % shorttype]
# drop any double-quoted weirdness
eclazz = eclazz.replace('"', '')
# look up that string... to avoid eval()
eclazz = ellipse_types[eclazz]
if issubclass(clazz, DevGalaxy):
assert (np.all([np.isfinite(x) for x in t.shapedev]))
ell = eclazz(*t.shapedev)
else:
assert (np.all([np.isfinite(x) for x in t.shapeexp]))
ell = eclazz(*t.shapeexp)
params.append(ell)
if invvars:
if issubclass(clazz, DevGalaxy):
ivs.extend(t.shapedev_ivar)
else:
ivs.extend(t.shapeexp_ivar)
elif issubclass(clazz, FixedCompositeGalaxy):
# hard-code knowledge that params are fracDev, shapeE, shapeD
assert (np.isfinite(t.fracdev))
params.append(t.fracdev)
if ellipseClass is not None:
expeclazz = deveclazz = ellipseClass
else:
expeclazz = hdr['TR_%s_T4' % shorttype]
deveclazz = hdr['TR_%s_T5' % shorttype]
expeclazz = expeclazz.replace('"', '')
deveclazz = deveclazz.replace('"', '')
expeclazz = ellipse_types[expeclazz]
deveclazz = ellipse_types[deveclazz]
assert (np.all([np.isfinite(x) for x in t.shapedev]))
assert (np.all([np.isfinite(x) for x in t.shapeexp]))
ee = expeclazz(*t.shapeexp)
de = deveclazz(*t.shapedev)
params.append(ee)
params.append(de)
if invvars:
ivs.append(t.fracdev_ivar)
ivs.extend(t.shapeexp_ivar)
ivs.extend(t.shapedev_ivar)
elif issubclass(clazz, PointSource):
pass
else:
raise RuntimeError('Unknown class %s' % str(clazz))
src = clazz(*params)
cat.append(src)
if invvars:
ivs = np.array(ivs)
ivs[np.logical_not(np.isfinite(ivs))] = 0
return cat, ivs
return cat
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None,
use_ceres=True, ceres_block=8,
save_fits=False, get_models=False, ps=None,
psf_broadening=None,
pixelized_psf=False,
get_masks=None,
move_crpix=False,
modelsky_dir=None):
'''
Given a list of tractor sources *cat*
and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id)
runs forced photometry, returning a FITS table the same length as *cat*.
*get_masks*: the WCS to resample mask bits into.
'''
from tractor import NanoMaggies, PointSource, Tractor, ExpGalaxy, DevGalaxy, FixedCompositeGalaxy
if not pixelized_psf and psf_broadening is None:
# PSF broadening in post-reactivation data, by band.
# Newer version from Aaron's email to decam-chatter, 2018-06-14.
broadening = { 1: 1.0405, 2: 1.0346, 3: None, 4: None }
psf_broadening = broadening[band]
if False:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('wise-forced-w%i' % band)
plots = (ps is not None)
if plots:
import pylab as plt
wantims = (plots or save_fits or get_models)
wanyband = 'w'
if get_models:
models = {}
wband = 'w%i' % band
fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix',
'pronexp']
Nsrcs = len(cat)
phot = fits_table()
# Filled in based on unique tile overlap
phot.wise_coadd_id = np.array([' '] * Nsrcs)
phot.set(wband + '_psfdepth', np.zeros(len(phot), np.float32))
ra = np.array([src.getPosition().ra for src in cat])
dec = np.array([src.getPosition().dec for src in cat])
nexp = np.zeros(Nsrcs, np.int16)
mjd = np.zeros(Nsrcs, np.float64)
central_flux = np.zeros(Nsrcs, np.float32)
fitstats = {}
tims = []
if get_masks:
mh,mw = get_masks.shape
maskmap = np.zeros((mh,mw), np.uint32)
for tile in tiles:
print('Reading WISE tile', tile.coadd_id, 'band', band)
tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band,
bandname=wanyband, roiradecbox=roiradecbox)
if tim is None:
print('Actually, no overlap with tile', tile.coadd_id)
continue
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage(), interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data' % tag)
ps.savefig()
plt.clf()
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
range=(-5,10), bins=100)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.title(tag)
ps.savefig()
if move_crpix and band in [1, 2]:
realwcs = tim.wcs.wcs
x,y = realwcs.crpix
tile_crpix = tile.get('crpix_w%i' % band)
dx = tile_crpix[0] - 1024.5
dy = tile_crpix[1] - 1024.5
realwcs.set_crpix(x+dx, y+dy)
#print('CRPIX', x,y, 'shift by', dx,dy, 'to', realwcs.crpix)
if modelsky_dir and band in [1, 2]:
fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band))
if not os.path.exists(fn):
raise RuntimeError('WARNING: does not exist:', fn)
x0,x1,y0,y1 = tim.roi
bg = fitsio.FITS(fn)[2][y0:y1, x0:x1]
#print('Read background map:', bg.shape, bg.dtype, 'vs image', tim.shape)
if plots:
plt.clf()
plt.subplot(1,2,1)
plt.imshow(tim.getImage(), interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=5 * sig1)
plt.subplot(1,2,2)
plt.imshow(bg, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=5 * sig1)
tag = '%s W%i' % (tile.coadd_id, band)
plt.suptitle(tag)
ps.savefig()
plt.clf()
ha = dict(range=(-5,10), bins=100, histtype='step')
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
color='b', label='Original', **ha)
plt.hist(((tim.getImage()-bg) * tim.inverr)[tim.inverr > 0].ravel(),
color='g', label='Minus Background', **ha)
plt.axvline(0, color='k', alpha=0.5)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.legend()
plt.title(tag + ': background')
ps.savefig()
# Actually subtract the background!
tim.data -= bg
# Floor the per-pixel variances
if band in [1,2]:
# in Vega nanomaggies per pixel
floor_sigma = {1: 0.5, 2: 2.0}
with np.errstate(divide='ignore'):
new_ie = 1. / np.hypot(1./tim.inverr, floor_sigma[band])
new_ie[tim.inverr == 0] = 0.
if plots:
plt.clf()
plt.plot((1. / tim.inverr[tim.inverr>0]).ravel(), (1./new_ie[tim.inverr>0]).ravel(), 'b.')
plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band))
plt.xlabel('original')
plt.ylabel('floored')
ps.savefig()
tim.inverr = new_ie
# Read mask file?
if get_masks:
from astrometry.util.resample import resample_with_wcs, OverlapError
# unwise_dir can be a colon-separated list of paths
tilemask = None
for d in tile.unwise_dir.split(':'):
fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id,
'unwise-%s-msk.fits.gz' % tile.coadd_id)
if os.path.exists(fn):
print('Reading unWISE mask file', fn)
x0,x1,y0,y1 = tim.roi
tilemask = fitsio.FITS(fn)[0][y0:y1,x0:x1]
break
if tilemask is None:
print('unWISE mask file for tile', tile.coadd_id, 'does not exist')
else:
try:
tanwcs = tim.wcs.wcs
assert(tanwcs.shape == tilemask.shape)
Yo,Xo,Yi,Xi,_ = resample_with_wcs(get_masks, tanwcs, intType=np.int16)
# Only deal with mask pixels that are set.
I, = np.nonzero(tilemask[Yi,Xi] > 0)
# Trim to unique area for this tile
rr,dd = get_masks.pixelxy2radec(Yo[I]+1, Xo[I]+1)
good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
I = I[good]
maskmap[Yo[I],Xo[I]] = tilemask[Yi[I], Xi[I]]
except OverlapError:
# Shouldn't happen by this point
print('No overlap between WISE tile', tile.coadd_id, 'and brick')
# The tiles have some overlap, so zero out pixels outside the
# tile's unique area.
th,tw = tim.shape
xx,yy = np.meshgrid(np.arange(tw), np.arange(th))
rr,dd = tim.wcs.wcs.pixelxy2radec(xx+1, yy+1)
unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
#print(np.sum(unique), 'of', (th*tw), 'pixels in this tile are unique')
tim.inverr[unique == False] = 0.
del xx,yy,rr,dd,unique
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage() * (tim.inverr > 0),
interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data (unique)' % tag)
ps.savefig()
if pixelized_psf:
import unwise_psf
if (band == 1) or (band == 2):
# we only have updated PSFs for W1 and W2
psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id,
modelname='neo4_unwisecat')
else:
psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id)
if band == 4:
# oversample (the unwise_psf models are at native W4 5.5"/pix,
# while the unWISE coadds are made at 2.75"/pix.
ph,pw = psfimg.shape
subpsf = np.zeros((ph*2-1, pw*2-1), np.float32)
from astrometry.util.util import lanczos3_interpolate
xx,yy = np.meshgrid(np.arange(0., pw-0.51, 0.5, dtype=np.float32),
np.arange(0., ph-0.51, 0.5, dtype=np.float32))
xx = xx.ravel()
yy = yy.ravel()
ix = xx.astype(np.int32)
iy = yy.astype(np.int32)
dx = (xx - ix).astype(np.float32)
dy = (yy - iy).astype(np.float32)
psfimg = psfimg.astype(np.float32)
rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg])
if plots:
plt.clf()
plt.imshow(psfimg, interpolation='nearest', origin='lower')
plt.title('Original PSF model')
ps.savefig()
plt.clf()
plt.imshow(subpsf, interpolation='nearest', origin='lower')
plt.title('Subsampled PSF model')
ps.savefig()
psfimg = subpsf
del xx, yy, ix, iy, dx, dy
from tractor.psf import PixelizedPSF
psfimg /= psfimg.sum()
fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0}
psfimg *= fluxrescales[band]
tim.psf = PixelizedPSF(psfimg)
if psf_broadening is not None and not pixelized_psf:
# psf_broadening is a factor by which the PSF FWHMs
# should be scaled; the PSF is a little wider
# post-reactivation.
psf = tim.getPsf()
from tractor import GaussianMixturePSF
if isinstance(psf, GaussianMixturePSF):
#
print('Broadening PSF: from', psf)
p0 = psf.getParams()
pnames = psf.getParamNames()
p1 = [p * psf_broadening**2 if 'var' in name else p
for (p, name) in zip(p0, pnames)]
psf.setParams(p1)
print('Broadened PSF:', psf)
else:
print('WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf))
wcs = tim.wcs.wcs
ok,x,y = wcs.radec2pixelxy(ra, dec)
x = np.round(x - 1.).astype(int)
y = np.round(y - 1.).astype(int)
good = (x >= 0) * (x < tw) * (y >= 0) * (y < th)
# Which sources are in this brick's unique area?
usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
I, = np.nonzero(good * usrc)
nexp[I] = tim.nuims[y[I], x[I]]
if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'):
mjd[I] = (tim.mjdmin + tim.mjdmax) / 2.
phot.wise_coadd_id[I] = tile.coadd_id
central_flux[I] = tim.getImage()[y[I], x[I]]
del x,y,good,usrc
# PSF norm for depth
psf = tim.getPsf()
h,w = tim.shape
patch = psf.getPointSourcePatch(h//2, w//2).patch
psfnorm = np.sqrt(np.sum(patch**2))
# To handle zero-depth, we return 1/nanomaggies^2 units rather than mags.
psfdepth = 1. / (tim.sig1 / psfnorm)**2
phot.get(wband + '_psfdepth')[I] = psfdepth
tim.tile = tile
tims.append(tim)
if plots:
plt.clf()
mn,mx = 0.1, 20000
plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100,
range=(np.log10(mn), np.log10(mx)))
logt = np.arange(0, 5)
plt.xticks(logt, ['%i' % i for i in 10.**logt])
plt.title('Central fluxes (W%i)' % band)
plt.axvline(np.log10(20000), color='k')
plt.axvline(np.log10(1000), color='k')
ps.savefig()
# Eddie's non-secret recipe:
#- central pixel <= 1000: 19x19 pix box size
#- central pixel in 1000 - 20000: 59x59 box size
#- central pixel > 20000 or saturated: 149x149 box size
#- object near "bright star": 299x299 box size
nbig = nmedium = nsmall = 0
for src,cflux in zip(cat, central_flux):
if cflux > 20000:
R = 100
nbig += 1
elif cflux > 1000:
R = 30
nmedium += 1
else:
R = 15
nsmall += 1
if isinstance(src, PointSource):
src.fixedRadius = R
else:
### FIXME -- sizes for galaxies..... can we set PSF size separately?
galrad = 0
# RexGalaxy is a subclass of ExpGalaxy
if isinstance(src, (ExpGalaxy, DevGalaxy)):
galrad = src.shape.re
elif isinstance(src, FixedCompositeGalaxy):
galrad = max(src.shapeExp.re, src.shapeDev.re)
pixscale = 2.75
src.halfsize = int(np.hypot(R, galrad * 5 / pixscale))
#print('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small')
minsb = 0.
fitsky = False
tractor = Tractor(tims, cat)
if use_ceres:
from tractor.ceres_optimizer import CeresOptimizer
tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block)
tractor.freezeParamsRecursive('*')
tractor.thawPathsTo(wanyband)
kwa = dict(fitstat_extras=[('pronexp', [tim.nims for tim in tims])])
t0 = Time()
R = tractor.optimize_forced_photometry(
minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True,
variance=True, shared_params=False,
wantims=wantims, **kwa)
print('unWISE forced photometry took', Time() - t0)
if use_ceres:
term = R.ceres_status['termination']
# Running out of memory can cause failure to converge
# and term status = 2.
# Fail completely in this case.
if term != 0:
print('Ceres termination status:', term)
raise RuntimeError(
'Ceres terminated with status %i' % term)
if wantims:
ims1 = R.ims1
flux_invvars = R.IV
if R.fitstats is not None:
for k in fskeys:
x = getattr(R.fitstats, k)
fitstats[k] = np.array(x).astype(np.float32)
if save_fits:
for i,tim in enumerate(tims):
tile = tim.tile
(dat, mod, ie, chi, roi) = ims1[i]
wcshdr = fitsio.FITSHDR()
tim.wcs.wcs.add_to_header(wcshdr)
tag = 'fit-%s-w%i' % (tile.coadd_id, band)
fitsio.write('%s-data.fits' %
tag, dat, clobber=True, header=wcshdr)
fitsio.write('%s-mod.fits' % tag, mod,
clobber=True, header=wcshdr)
fitsio.write('%s-chi.fits' % tag, chi,
clobber=True, header=wcshdr)
if plots:
# Create models for just the brightest sources
bright_cat = [src for src in cat
if src.getBrightness().getBand(wanyband) > 1000]
print('Bright soures:', len(bright_cat))
btr = Tractor(tims, bright_cat)
for tim in tims:
mod = btr.getModelImage(tim)
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
sig1 = tim.sig1
plt.clf()
plt.imshow(mod, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: bright-star models' % tag)
ps.savefig()
if get_models:
for i,tim in enumerate(tims):
tile = tim.tile
(dat, mod, ie, chi, roi) = ims1[i]
models[(tile.coadd_id, band)] = (mod, dat, ie, tim.roi, tim.wcs.wcs)
if plots:
for i,tim in enumerate(tims):
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
(dat, mod, ie, chi, roi) = ims1[i]
sig1 = tim.sig1
plt.clf()
plt.imshow(dat, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: data' % tag)
ps.savefig()
plt.clf()
plt.imshow(mod, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: model' % tag)
ps.savefig()
plt.clf()
plt.imshow(chi, interpolation='nearest', origin='lower',
cmap='gray', vmin=-5, vmax=+5)
plt.colorbar()
plt.title('%s: chi' % tag)
ps.savefig()
nm = np.array([src.getBrightness().getBand(wanyband) for src in cat])
nm_ivar = flux_invvars
# Sources out of bounds, eg, never change from their default
# (1-sigma or whatever) initial fluxes. Zero them out instead.
nm[nm_ivar == 0] = 0.
phot.set(wband + '_nanomaggies', nm.astype(np.float32))
phot.set(wband + '_nanomaggies_ivar', nm_ivar.astype(np.float32))
dnm = np.zeros(len(nm_ivar), np.float32)
okiv = (nm_ivar > 0)
dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32)
okflux = (nm > 0)
mag = np.zeros(len(nm), np.float32)
mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux])
).astype(np.float32)
dmag = np.zeros(len(nm), np.float32)
ok = (okiv * okflux)
dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok])
).astype(np.float32)
mag[np.logical_not(okflux)] = np.nan
dmag[np.logical_not(ok)] = np.nan
phot.set(wband + '_mag', mag)
phot.set(wband + '_mag_err', dmag)
for k in fskeys:
phot.set(wband + '_' + k, fitstats[k])
phot.set(wband + '_nexp', nexp)
if not np.all(mjd == 0):
phot.set(wband + '_mjd', mjd)
rtn = wphotduck()
rtn.phot = phot
rtn.models = None
rtn.maskmap = None
if get_models:
rtn.models = models
if get_masks:
rtn.maskmap = maskmap
return rtn
hspace=0, wspace=0)
# transpose image
dimshow(tim.getImage().T, vmin=-2.*tim.sig1, vmax=10.*tim.sig1)
plt.title('DECaLS ' + tim.name)
ps.savefig()
ax = plt.axis()
plt.plot(stars.iy, stars.ix, 'r.')
plt.axis(ax)
plt.title('Pan-STARRS stars in DECaLS ' + tim.name)
ps.savefig()
iband = ps1cat.ps1band[tim.band]
stars.mag = stars.median[:,iband]
stars.flux = NanoMaggies.magToNanomaggies(stars.mag)
stars.cut(stars.flux > 1.)
print len(stars), 'brighter than 22.5'
stars.cut(np.argsort(stars.mag))
ima = dict(vmin=-2.*tim.sig1, vmax=10.*tim.sig1, ticks=False)
plt.clf()
#R,C = 10,14
#R,C = 9,13
R,C = 10,13
def main(survey=None, opt=None):
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
import pylab as plt
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.expnum)
filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
filename = opt.expnum
# Try parsing HDU number
try:
hdu = int(opt.ccdname)
ccdname = None
except:
hdu = -1
ccdname = opt.ccdname
if survey is None:
survey = LegacySurveyData()
catsurvey = survey
if opt.catalog_dir is not None:
catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir)
if filename is not None and hdu >= 0:
# FIXME -- try looking up in CCDs file?
# Read metadata from file
print('Warning: faking metadata from file contents')
T = exposure_metadata([filename], hdus=[hdu])
print('Metadata:')
T.about()
if not 'ccdzpt' in T.columns():
phdr = fitsio.read_header(filename)
T.ccdzpt = np.array([phdr['MAGZERO']])
print('WARNING: using header MAGZERO')
T.ccdraoff = np.array([0.])
T.ccddecoff = np.array([0.])
print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.')
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if hdu >= 0:
T.cut(T.image_hdu == hdu)
print(len(T), 'with HDU', hdu)
if filename is not None:
T.cut(np.array([f.strip() == filename for f in T.image_filename]))
print(len(T), 'with filename', filename)
if opt.camera is not None:
T.cut(T.camera == opt.camera)
print(len(T), 'with camera', opt.camera)
assert (len(T) == 1)
ccd = T[0]
im = survey.get_image_object(ccd)
if opt.do_calib:
#from legacypipe.survey import run_calibs
#kwa = dict(splinesky=True)
#run_calibs((im, kwa))
im.run_calibs(splinesky=True)
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
splinesky=True,
constant_invvar=opt.constant_invvar,
hybridPsf=opt.hybrid_psf,
normalizePsf=opt.normalize_psf)
print('Got tim:', tim)
print('Read image:', Time() - t0)
if opt.catfn in ['DR1', 'DR2', 'DR3', 'DR5', 'DR']:
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = catsurvey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) *
(yy >= -margin) * (yy <= (H + margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
for col in ['out_of_bounds', 'left_blob']:
if col in T.get_columns():
T.cut(T.get(col) == False)
print('Cut to', len(T), 'on', col)
if len(T):
TT.append(T)
if len(TT) == 0:
print('No sources to photometer.')
return 0
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
print('Total of', len(T), 'catalog sources')
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
surveydir = survey.get_survey_dir()
del survey
kwargs = {}
cols = T.get_columns()
if 'flux_r' in cols and not 'decam_flux_r' in cols:
kwargs.update(fluxPrefix='')
cat = read_fits_catalog(T, **kwargs)
# Replace the brightness (which will be a NanoMaggies with g,r,z)
# with a NanoMaggies with this image's band only.
for src in cat:
src.brightness = NanoMaggies(**{tim.band: 1.})
print('Read catalog:', Time() - t0)
print('Forced photom...')
F = run_forced_phot(cat,
tim,
ceres=opt.ceres,
derivs=opt.derivs,
fixed_also=True,
agn=opt.agn,
do_forced=opt.forced,
do_apphot=opt.apphot,
ps=ps)
t0 = Time()
F.release = T.release
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.camera = np.array([ccd.camera] * len(F))
F.expnum = np.array([im.expnum] * len(F)).astype(np.int32)
F.ccdname = np.array([im.ccdname] * len(F))
# "Denormalizing"
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32)
F.ra = T.ra
F.dec = T.dec
ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
h, w = tim.shape
F.mask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h - 1),
np.clip(np.round(F.x).astype(int), 0, w - 1)]
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, surveydir)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(
dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(
dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='EXPOSURE',
value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname),
comment='Name of this image'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2'
}
columns = F.get_columns()
for i, col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
print('Getting model image...')
tr = Tractor([tim], cat)
mod = tr.getModelImage(tim)
fitsio.write(opt.save_model, mod, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
print('Finished forced phot:', Time() - t0)
return 0
