def get_catalog_in_wcs(chipwcs,
catsurvey_north,
catsurvey_south=None,
resolve_dec=None,
margin=20):
TT = []
surveys = [(catsurvey_north, True)]
if catsurvey_south is not None:
surveys.append((catsurvey_south, False))
for catsurvey, north in surveys:
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
if resolve_dec is not None:
from astrometry.util.starutil_numpy import radectolb
bricks.gal_l, bricks.gal_b = radectolb(bricks.ra, bricks.dec)
for b in bricks:
# Skip bricks that are entirely on the wrong side of the resolve line (NGC only)
if resolve_dec is not None and b.gal_b > 0:
if north and b.dec2 <= resolve_dec:
continue
if not (north) and b.dec1 >= resolve_dec:
continue
# there is some overlap with this brick... read the catalog.
fn = catsurvey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn,
columns=[
'ra', 'dec', 'brick_primary', 'type', 'release',
'brickid', 'brickname', 'objid', 'fracdev',
'flux_r', 'shapedev_r', 'shapedev_e1',
'shapedev_e2', 'shapeexp_r', 'shapeexp_e1',
'shapeexp_e2', 'ref_epoch', 'pmra', 'pmdec',
'parallax'
])
if resolve_dec is not None and b.gal_b > 0:
if north:
T.cut(T.dec >= resolve_dec)
print('Cut to', len(T), 'north of the resolve line')
else:
T.cut(T.dec < resolve_dec)
print('Cut to', len(T), 'south of the resolve line')
ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
I, = np.nonzero((xx >= -margin) * (xx <= (W + margin)) *
(yy >= -margin) * (yy <= (H + margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
for col in ['out_of_bounds', 'left_blob']:
if col in T.get_columns():
T.cut(T.get(col) == False)
print('Cut to', len(T), 'on', col)
# drop DUP sources
I, = np.nonzero([t.strip() != 'DUP' for t in T.type])
T.cut(I)
print('Cut to', len(T), 'after removing DUP')
if len(T):
TT.append(T)
if len(TT) == 0:
return None
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
print('Total of', len(T), 'catalog sources')
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I, = np.nonzero([
t == 'COMP' and not np.isfinite(r)
for t, r in zip(T.type, T.shapedev_r)
])
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I, = np.nonzero([
t == 'COMP' and not np.isfinite(r)
for t, r in zip(T.type, T.shapeexp_r)
])
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
return T
def run_one_ccd(survey, catsurvey_north, catsurvey_south, resolve_dec, ccd,
opt, zoomslice, ps):
tlast = Time()
im = survey.get_image_object(ccd)
if opt.do_calib:
im.run_calibs(splinesky=True)
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
splinesky=True,
constant_invvar=opt.constant_invvar,
hybridPsf=opt.hybrid_psf,
normalizePsf=opt.normalize_psf,
old_calibs_ok=True)
print('Got tim:', tim, 'x0,y0', tim.x0, tim.y0)
tnow = Time()
print('Read image:', tnow - tlast)
tlast = tnow
# Apply outlier masks
if True:
# Outliers masks are computed within a survey (north/south for dr8), and are stored
# in a brick-oriented way, in the results directories.
north_ccd = (ccd.camera.strip() != 'decam')
catsurvey = catsurvey_north
if not north_ccd and catsurvey_south is not None:
catsurvey = catsurvey_south
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
for b in bricks:
from legacypipe.outliers import read_outlier_mask_file
print('Reading outlier mask for brick', b.brickname)
ok = read_outlier_mask_file(catsurvey, [tim],
b.brickname,
subimage=False,
output=False,
ps=ps)
if not ok:
print('WARNING: failed to read outliers mask file for brick',
b.brickname)
if opt.catalog:
T = fits_table(opt.catalog)
else:
chipwcs = tim.subwcs
T = get_catalog_in_wcs(chipwcs,
catsurvey_north,
catsurvey_south=catsurvey_south,
resolve_dec=resolve_dec)
if T is None:
print('No sources to photometer.')
return None
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
surveydir = survey.get_survey_dir()
del survey
if opt.move_gaia:
# Gaia stars: move RA,Dec to the epoch of this image.
I = np.flatnonzero(T.ref_epoch > 0)
if len(I):
from legacypipe.survey import radec_at_mjd
print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd())
ra, dec = radec_at_mjd(T.ra[I], T.dec[I],
T.ref_epoch[I].astype(float), T.pmra[I],
T.pmdec[I], T.parallax[I], tim.time.toMjd())
T.ra[I] = ra
T.dec[I] = dec
tnow = Time()
print('Read catalog:', tnow - tlast)
tlast = tnow
cat = read_fits_catalog(T, bands='r')
# Replace the brightness (which will be a NanoMaggies with g,r,z)
# with a NanoMaggies with this image's band only.
for src in cat:
src.brightness = NanoMaggies(**{tim.band: 1.})
tnow = Time()
print('Parse catalog:', tnow - tlast)
tlast = tnow
print('Forced photom...')
F = run_forced_phot(cat,
tim,
ceres=opt.ceres,
derivs=opt.derivs,
fixed_also=True,
agn=opt.agn,
do_forced=opt.forced,
do_apphot=opt.apphot,
get_model=opt.save_model,
ps=ps,
timing=True,
ceres_threads=opt.ceres_threads)
if opt.save_model:
# unpack results
F, model_img = F
F.release = T.release
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.camera = np.array([ccd.camera] * len(F))
F.expnum = np.array([im.expnum] * len(F)).astype(np.int64)
F.ccdname = np.array([im.ccdname] * len(F))
# "Denormalizing"
F.filter = np.array([tim.band] * len(F))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(F))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(F)).astype(np.float32)
F.psfsize = np.array([tim.psf_fwhm * tim.imobj.pixscale] * len(F)).astype(
np.float32)
F.ccd_cuts = np.array([ccd.ccd_cuts] * len(F))
F.airmass = np.array([ccd.airmass] * len(F))
### --> also add units to the dict below so the FITS headers have units
F.sky = np.array([tim.midsky / tim.zpscale / tim.imobj.pixscale**2] *
len(F)).astype(np.float32)
# in the same units as the depth maps -- flux inverse-variance.
F.psfdepth = np.array([(1. / (tim.sig1 / tim.psfnorm)**2)] *
len(F)).astype(np.float32)
F.galdepth = np.array([(1. / (tim.sig1 / tim.galnorm)**2)] *
len(F)).astype(np.float32)
# F.psfdepth = np.array([-2.5 * (np.log10(5. * tim.sig1 / tim.psfnorm) - 9)] * len(F)).astype(np.float32)
# F.galdepth = np.array([-2.5 * (np.log10(5. * tim.sig1 / tim.galnorm) - 9)] * len(F)).astype(np.float32)
# super units questions here
if opt.derivs:
cosdec = np.cos(np.deg2rad(T.dec))
F.dra = (F.flux_dra / F.flux) * 3600. / cosdec
F.ddec = (F.flux_ddec / F.flux) * 3600.
F.dra_ivar = F.flux_dra_ivar * (F.flux / 3600. * cosdec)**2
F.ddec_ivar = F.flux_ddec_ivar * (F.flux / 3600.)**2
F.delete_column('flux_dra')
F.delete_column('flux_ddec')
F.delete_column('flux_dra_ivar')
F.delete_column('flux_ddec_ivar')
F.flux = F.flux_fixed
F.flux_ivar = F.flux_fixed_ivar
F.delete_column('flux_fixed')
F.delete_column('flux_fixed_ivar')
for c in ['dra', 'ddec', 'dra_ivar', 'ddec_ivar', 'flux', 'flux_ivar']:
F.set(c, F.get(c).astype(np.float32))
F.ra = T.ra
F.dec = T.dec
ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
h, w = tim.shape
F.dqmask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h - 1),
np.clip(np.round(F.x).astype(int), 0, w - 1)]
program_name = sys.argv[0]
## FIXME -- from catalog?
release = 8002
version_hdr = get_version_header(program_name, surveydir, release)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn.strip())
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(
dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(
dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='PLVER', value=ccd.plver, comment='CP pipeline version'))
version_hdr.add_record(
dict(name='PLPROCID', value=ccd.plprocid, comment='CP pipeline id'))
version_hdr.add_record(
dict(name='PROCDATE', value=ccd.procdate, comment='CP image DATE'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
fitsio.write(opt.save_model, model_img, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
tnow = Time()
print('Forced phot:', tnow - tlast)
return F, version_hdr
def run_one_ccd(survey, catsurvey_north, catsurvey_south, resolve_dec, ccd,
opt, zoomslice, ps):
from functools import reduce
from legacypipe.bits import DQ_BITS
tlast = Time()
#print('Opt:', opt)
im = survey.get_image_object(ccd)
print('Run_one_ccd: checking cache', survey.cache_dir)
if survey.cache_dir is not None:
im.check_for_cached_files(survey)
if opt.do_calib:
im.run_calibs(splinesky=True)
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
constant_invvar=opt.constant_invvar,
hybridPsf=opt.hybrid_psf,
normalizePsf=opt.normalize_psf,
old_calibs_ok=True,
trim_edges=False)
print('Got tim:', tim) #, 'x0,y0', tim.x0, tim.y0)
chipwcs = tim.subwcs
H, W = tim.shape
tnow = Time()
print('Read image:', tnow - tlast)
tlast = tnow
if ccd.camera == 'decam':
# Halo subtraction
from legacypipe.halos import subtract_one
from legacypipe.reference import mask_radius_for_mag, read_gaia
ref_margin = mask_radius_for_mag(0.)
mpix = int(np.ceil(ref_margin * 3600. / chipwcs.pixel_scale()))
marginwcs = chipwcs.get_subimage(-mpix, -mpix, W + 2 * mpix,
H + 2 * mpix)
gaia = read_gaia(marginwcs, None)
keeprad = np.ceil(gaia.keep_radius * 3600. /
chipwcs.pixel_scale()).astype(int)
_, xx, yy = chipwcs.radec2pixelxy(gaia.ra, gaia.dec)
# cut to those touching the chip
gaia.cut((xx > -keeprad) * (xx < W + keeprad) * (yy > -keeprad) *
(yy < H + keeprad))
Igaia, = np.nonzero(gaia.isgaia * gaia.pointsource)
halostars = gaia[Igaia]
print('Got', len(gaia), 'Gaia stars,', len(halostars),
'for halo subtraction')
moffat = True
halos = subtract_one((tim, halostars, moffat))
tim.data -= halos
# The "north" and "south" directories often don't have
# 'survey-bricks" files of their own -- use the 'survey' one
# instead.
if catsurvey_south is not None:
try:
catsurvey_south.get_bricks_readonly()
except:
catsurvey_south.bricks = survey.get_bricks_readonly()
if catsurvey_north is not None:
try:
catsurvey_north.get_bricks_readonly()
except:
catsurvey_north.bricks = survey.get_bricks_readonly()
# Apply outlier masks
outlier_header = None
outlier_mask = None
posneg_mask = None
if opt.outlier_mask is not None:
posneg_mask = np.zeros(tim.shape, np.uint8)
# Outliers masks are computed within a survey (north/south for dr9), and are stored
# in a brick-oriented way, in the results directories.
north_ccd = (ccd.camera.strip() != 'decam')
catsurvey = catsurvey_north
if not north_ccd and catsurvey_south is not None:
catsurvey = catsurvey_south
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
for b in bricks:
print(
'Reading outlier mask for brick', b.brickname, ':',
catsurvey.find_file('outliers_mask',
brick=b.brickname,
output=False))
ok = read_outlier_mask_file(catsurvey, [tim],
b.brickname,
pos_neg_mask=posneg_mask,
subimage=False,
output=False,
ps=ps)
if not ok:
print('WARNING: failed to read outliers mask file for brick',
b.brickname)
if opt.outlier_mask is not None:
outlier_mask = np.zeros((ccd.height, ccd.width), np.uint8)
outlier_mask[tim.y0:tim.y0 + H, tim.x0:tim.x0 + W] = posneg_mask
del posneg_mask
# Grab original image headers (including WCS)
im = survey.get_image_object(ccd)
imhdr = im.read_image_header()
imhdr['CAMERA'] = ccd.camera
imhdr['EXPNUM'] = ccd.expnum
imhdr['CCDNAME'] = ccd.ccdname
imhdr['IMGFILE'] = ccd.image_filename.strip()
outlier_header = imhdr
if opt.catalog:
T = fits_table(opt.catalog)
else:
chipwcs = tim.subwcs
T = get_catalog_in_wcs(chipwcs,
survey,
catsurvey_north,
catsurvey_south=catsurvey_south,
resolve_dec=resolve_dec)
if T is None:
print('No sources to photometer.')
return None
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
surveydir = survey.get_survey_dir()
del survey
if opt.move_gaia:
# Gaia stars: move RA,Dec to the epoch of this image.
I = np.flatnonzero(T.ref_epoch > 0)
if len(I):
print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd())
ra, dec = radec_at_mjd(T.ra[I], T.dec[I],
T.ref_epoch[I].astype(float), T.pmra[I],
T.pmdec[I], T.parallax[I], tim.time.toMjd())
T.ra[I] = ra
T.dec[I] = dec
tnow = Time()
print('Read catalog:', tnow - tlast)
tlast = tnow
# Find SGA galaxies outside this chip and subtract them before we begin.
chipwcs = tim.subwcs
_, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
sga_out = (T.ref_cat == 'L3') * np.logical_not(
(xx >= 1) * (xx <= W) * (yy >= 1) * (yy <= H))
I = np.flatnonzero(sga_out)
if len(I):
print(len(I), 'SGA galaxies are outside the image. Subtracting...')
cat = read_fits_catalog(T[I], bands=[tim.band])
tr = Tractor([tim], cat)
mod = tr.getModelImage(0)
tim.data -= mod
I = np.flatnonzero(np.logical_not(sga_out))
T.cut(I)
cat = read_fits_catalog(T, bands='r')
# Replace the brightness (which will be a NanoMaggies with g,r,z)
# with a NanoMaggies with this image's band only.
for src in cat:
src.brightness = NanoMaggies(**{tim.band: 1.})
tnow = Time()
print('Parse catalog:', tnow - tlast)
tlast = tnow
print('Forced photom...')
F = run_forced_phot(cat,
tim,
ceres=opt.ceres,
derivs=opt.derivs,
fixed_also=True,
agn=opt.agn,
do_forced=opt.forced,
do_apphot=opt.apphot,
get_model=opt.save_model,
ps=ps,
timing=True,
ceres_threads=opt.ceres_threads)
if opt.save_model:
# unpack results
F, model_img = F
F.release = T.release
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.camera = np.array([ccd.camera] * len(F))
F.expnum = np.array([im.expnum] * len(F), dtype=np.int64)
F.ccdname = np.array([im.ccdname] * len(F))
# "Denormalizing"
F.filter = np.array([tim.band] * len(F))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(F))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(F), dtype=np.float32)
F.psfsize = np.array([tim.psf_fwhm * tim.imobj.pixscale] * len(F),
dtype=np.float32)
F.ccd_cuts = np.array([ccd.ccd_cuts] * len(F))
F.airmass = np.array([ccd.airmass] * len(F), dtype=np.float32)
### --> also add units to the dict below so the FITS headers have units
F.sky = np.array([tim.midsky / tim.zpscale / tim.imobj.pixscale**2] *
len(F),
dtype=np.float32)
# in the same units as the depth maps -- flux inverse-variance.
F.psfdepth = np.array([(1. / (tim.sig1 / tim.psfnorm)**2)] * len(F),
dtype=np.float32)
F.galdepth = np.array([(1. / (tim.sig1 / tim.galnorm)**2)] * len(F),
dtype=np.float32)
F.fwhm = np.array([tim.psf_fwhm] * len(F), dtype=np.float32)
F.skyrms = np.array([ccd.skyrms] * len(F), dtype=np.float32)
F.ccdzpt = np.array([ccd.ccdzpt] * len(F), dtype=np.float32)
F.ccdrarms = np.array([ccd.ccdrarms] * len(F), dtype=np.float32)
F.ccddecrms = np.array([ccd.ccddecrms] * len(F), dtype=np.float32)
F.ccdphrms = np.array([ccd.ccdphrms] * len(F), dtype=np.float32)
if opt.derivs:
cosdec = np.cos(np.deg2rad(T.dec))
with np.errstate(divide='ignore', invalid='ignore'):
F.dra = (F.flux_dra / F.flux) * 3600. / cosdec
F.ddec = (F.flux_ddec / F.flux) * 3600.
F.dra[F.flux == 0] = 0.
F.ddec[F.flux == 0] = 0.
F.dra_ivar = F.flux_dra_ivar * (F.flux / 3600. * cosdec)**2
F.ddec_ivar = F.flux_ddec_ivar * (F.flux / 3600.)**2
F.delete_column('flux_dra')
F.delete_column('flux_ddec')
F.delete_column('flux_dra_ivar')
F.delete_column('flux_ddec_ivar')
F.flux = F.flux_fixed
F.flux_ivar = F.flux_fixed_ivar
F.delete_column('flux_fixed')
F.delete_column('flux_fixed_ivar')
for c in ['dra', 'ddec', 'dra_ivar', 'ddec_ivar', 'flux', 'flux_ivar']:
F.set(c, F.get(c).astype(np.float32))
F.ra = T.ra
F.dec = T.dec
_, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
h, w = tim.shape
ix = np.round(F.x).astype(int)
iy = np.round(F.y).astype(int)
F.dqmask = tim.dq[np.clip(iy, 0, h - 1), np.clip(ix, 0, w - 1)]
# Set an OUT-OF-BOUNDS bit.
F.dqmask[reduce(np.logical_or,
[ix < 0, ix >= w, iy < 0, iy >= h])] |= DQ_BITS['edge2']
program_name = sys.argv[0]
## FIXME -- from catalog?
release = 9999
version_hdr = get_version_header(program_name, surveydir, release)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn.strip())
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(
dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(
dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='PLVER', value=ccd.plver, comment='CP pipeline version'))
version_hdr.add_record(
dict(name='PLPROCID', value=ccd.plprocid, comment='CP pipeline id'))
version_hdr.add_record(
dict(name='PROCDATE', value=ccd.procdate, comment='CP image DATE'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
fitsio.write(opt.save_model, model_img, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
tnow = Time()
print('Forced phot:', tnow - tlast)
return F, version_hdr, outlier_mask, outlier_header
def get_catalog_in_wcs(chipwcs,
survey,
catsurvey_north,
catsurvey_south=None,
resolve_dec=None,
margin=20):
TT = []
surveys = [(catsurvey_north, True)]
if catsurvey_south is not None:
surveys.append((catsurvey_south, False))
columns = [
'ra',
'dec',
'brick_primary',
'type',
'release',
'brickid',
'brickname',
'objid',
'flux_g',
'flux_r',
'flux_z',
'sersic',
'shape_r',
'shape_e1',
'shape_e2',
'ref_epoch',
'pmra',
'pmdec',
'parallax',
'ref_cat',
'ref_id',
]
for catsurvey, north in surveys:
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
if resolve_dec is not None:
from astrometry.util.starutil_numpy import radectolb
bricks.gal_l, bricks.gal_b = radectolb(bricks.ra, bricks.dec)
for b in bricks:
# Skip bricks that are entirely on the wrong side of the resolve line (NGC only)
if resolve_dec is not None:
# Northern survey, brick too far south (max dec is below the resolve line)
if north and b.dec2 <= resolve_dec:
continue
# Southern survey, brick too far north (min dec is above the resolve line), but only in the North Galactic Cap
if not (north) and b.dec1 >= resolve_dec and b.gal_b > 0:
continue
# there is some overlap with this brick... read the catalog.
fn = catsurvey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn, columns=columns)
if resolve_dec is not None:
if north:
T.cut(T.dec >= resolve_dec)
print('Cut to', len(T), 'north of the resolve line')
elif b.gal_b > 0:
# Northern galactic cap only: cut Southern survey
T.cut(T.dec < resolve_dec)
print('Cut to', len(T), 'south of the resolve line')
_, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
# Cut to sources that are inside the image+margin
T.cut((xx >= -margin) * (xx <= (W + margin)) * (yy >= -margin) *
(yy <= (H + margin)))
T.cut(T.brick_primary)
#print('Cut to', len(T), 'on brick_primary')
# drop DUP sources
I, = np.nonzero([t.strip() != 'DUP' for t in T.type])
T.cut(I)
#print('Cut to', len(T), 'after removing DUP')
if len(T):
TT.append(T)
if len(TT) == 0:
return None
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
SGA = find_missing_sga(T, chipwcs, survey, surveys, columns)
if SGA is not None:
## Add 'em in!
T = merge_tables([T, SGA], columns='fillzero')
print('Total of', len(T), 'catalog sources')
return T
def run_one_ccd(survey, catsurvey, ccd, opt, zoomslice, ps):
tlast = Time()
im = survey.get_image_object(ccd)
if opt.do_calib:
#from legacypipe.survey import run_calibs
#kwa = dict(splinesky=True)
#run_calibs((im, kwa))
im.run_calibs(splinesky=True)
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True,
constant_invvar=opt.constant_invvar,
hybridPsf=opt.hybrid_psf,
normalizePsf=opt.normalize_psf,
old_calibs_ok=True)
print('Got tim:', tim)
tnow = Time()
print('Read image:', tnow-tlast)
tlast = tnow
if opt.catalog:
T = fits_table(opt.catalog)
else:
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = catsurvey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn, columns=[
'ra', 'dec', 'brick_primary', 'type', 'release',
'brickid', 'brickname', 'objid',
'fracdev', 'flux_r',
'shapedev_r', 'shapedev_e1', 'shapedev_e2',
'shapeexp_r', 'shapeexp_e1', 'shapeexp_e2',
'ref_epoch', 'pmra', 'pmdec', 'parallax'
])
ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W,H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) *
(yy >= -margin) * (yy <= (H+margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
for col in ['out_of_bounds', 'left_blob']:
if col in T.get_columns():
T.cut(T.get(col) == False)
print('Cut to', len(T), 'on', col)
if len(T):
TT.append(T)
if len(TT) == 0:
print('No sources to photometer.')
return None
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
print('Total of', len(T), 'catalog sources')
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I, = np.nonzero([t == 'COMP' and not np.isfinite(r) for t,r in zip(T.type, T.shapedev_r)])
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I, = np.nonzero([t == 'COMP' and not np.isfinite(r) for t,r in zip(T.type, T.shapeexp_r)])
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
surveydir = survey.get_survey_dir()
del survey
if opt.move_gaia:
# Gaia stars: move RA,Dec to the epoch of this image.
I = np.flatnonzero(T.ref_epoch > 0)
if len(I):
from legacypipe.survey import radec_at_mjd
print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd())
ra,dec = radec_at_mjd(T.ra[I], T.dec[I], T.ref_epoch[I].astype(float),
T.pmra[I], T.pmdec[I], T.parallax[I],
tim.time.toMjd())
T.ra [I] = ra
T.dec[I] = dec
kwargs = {}
cols = T.get_columns()
if 'flux_r' in cols and not 'decam_flux_r' in cols:
kwargs.update(fluxPrefix='')
tnow = Time()
print('Read catalog:', tnow-tlast)
tlast = tnow
cat = read_fits_catalog(T, bands='r', **kwargs)
# Replace the brightness (which will be a NanoMaggies with g,r,z)
# with a NanoMaggies with this image's band only.
for src in cat:
src.brightness = NanoMaggies(**{tim.band: 1.})
tnow = Time()
print('Parse catalog:', tnow-tlast)
tlast = tnow
print('Forced photom...')
F = run_forced_phot(cat, tim,
ceres=opt.ceres,
derivs=opt.derivs,
fixed_also=True,
agn=opt.agn,
do_forced=opt.forced,
do_apphot=opt.apphot,
get_model=opt.save_model,
ps=ps)
if opt.save_model:
# unpack results
F,model_img = F
F.release = T.release
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.camera = np.array([ccd.camera] * len(F))
F.expnum = np.array([im.expnum] * len(F)).astype(np.int32)
F.ccdname = np.array([im.ccdname] * len(F))
# "Denormalizing"
F.filter = np.array([tim.band] * len(F))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(F))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(F)).astype(np.float32)
F.psfsize = np.array([tim.psf_fwhm * tim.imobj.pixscale] * len(F)).astype(np.float32)
#### FIXME -- units?
### --> also add units to the dict below so the FITS headers have units
F.sky = np.array([tim.midsky / tim.zpscale / tim.imobj.pixscale**2] * len(F)).astype(np.float32)
# in the same units as the depth maps -- flux inverse-variance.
F.psfdepth = np.array([(1. / (tim.sig1 / tim.psfnorm)**2)] * len(F)).astype(np.float32)
F.galdepth = np.array([(1. / (tim.sig1 / tim.galnorm)**2)] * len(F)).astype(np.float32)
# F.psfdepth = np.array([-2.5 * (np.log10(5. * tim.sig1 / tim.psfnorm) - 9)] * len(F)).astype(np.float32)
# F.galdepth = np.array([-2.5 * (np.log10(5. * tim.sig1 / tim.galnorm) - 9)] * len(F)).astype(np.float32)
# super units questions here
if opt.derivs:
cosdec = np.cos(np.deg2rad(T.dec))
F.dra = (F.flux_dra / F.flux) * 3600. / cosdec
F.ddec = (F.flux_ddec / F.flux) * 3600.
F.dra_ivar = F.flux_dra_ivar * (F.flux / 3600. * cosdec)**2
F.ddec_ivar = F.flux_ddec_ivar * (F.flux / 3600.)**2
F.delete_column('flux_dra')
F.delete_column('flux_ddec')
F.delete_column('flux_dra_ivar')
F.delete_column('flux_ddec_ivar')
F.flux = F.flux_fixed
F.flux_ivar = F.flux_fixed_ivar
F.delete_column('flux_fixed')
F.delete_column('flux_fixed_ivar')
for c in ['dra', 'ddec', 'dra_ivar', 'ddec_ivar', 'flux', 'flux_ivar']:
F.set(c, F.get(c).astype(np.float32))
F.ra = T.ra
F.dec = T.dec
ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x-1).astype(np.float32)
F.y = (y-1).astype(np.float32)
## FIXME -- read outlier_masks?
h,w = tim.shape
F.mask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h-1),
np.clip(np.round(F.x).astype(int), 0, w-1)]
program_name = sys.argv[0]
## FIXME -- from catalog?
release = 0
version_hdr = get_version_header(program_name, surveydir, release)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn.strip())
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(dict(name='EXPOSURE',
value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname),
comment='Name of this image'))
version_hdr.add_record(dict(name='PLVER', value=ccd.plver, comment='CP pipeline version'))
version_hdr.add_record(dict(name='PROCDATE', value=ccd.procdate, comment='CP image DATE'))
keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV',
'INSTRUME']
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
fitsio.write(opt.save_model, model_img, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
tnow = Time()
print('Forced phot:', tnow-tlast)
return F,version_hdr
def main(survey=None, opt=None):
'''Driver function for forced photometry of individual DECam images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0,x1,y0,y1) = opt.zoom
zoomslice = (slice(y0,y1), slice(x0,x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.filename)
opt.filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
# Try parsing HDU number
try:
opt.hdu = int(opt.hdu)
ccdname = None
except:
ccdname = opt.hdu
opt.hdu = -1
if survey is None:
survey = LegacySurveyData()
if opt.filename is not None and opt.hdu >= 0:
# Read metadata from file
T = exposure_metadata([opt.filename], hdus=[opt.hdu])
print('Metadata:')
T.about()
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if opt.hdu >= 0:
T.cut(T.image_hdu == opt.hdu)
print(len(T), 'with HDU', opt.hdu)
if opt.filename is not None:
T.cut(np.array([f.strip() == opt.filename for f in T.image_filename]))
print(len(T), 'with filename', opt.filename)
assert(len(T) == 1)
ccd = T[0]
im = survey.get_image_object(ccd)
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True,
constant_invvar=opt.constant_invvar)
print('Got tim:', tim)
print('Read image:', Time()-t0)
if opt.catfn in ['DR1', 'DR2', 'DR3']:
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=survey)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = survey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W,H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) *
(yy >= -margin) * (yy <= (H+margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print('Cut to', len(T), 'on out_of_bounds and left_blob')
if len(T):
TT.append(T)
if len(TT) == 0:
print('No sources to photometer.')
return 0
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
surveydir = survey.get_survey_dir()
del survey
cat = read_fits_catalog(T)
# print('Got cat:', cat)
print('Read catalog:', Time()-t0)
print('Forced photom...')
opti = None
forced_kwargs = {}
if opt.ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
opti = CeresOptimizer(BW=B, BH=B)
#forced_kwargs.update(verbose=True)
for src in cat:
# Limit sizes of huge models
from tractor.galaxy import ProfileGalaxy
if isinstance(src, ProfileGalaxy):
px,py = tim.wcs.positionToPixel(src.getPosition())
h = src._getUnitFluxPatchSize(tim, px, py, tim.modelMinval)
MAXHALF = 128
if h > MAXHALF:
print('halfsize', h,'for',src,'-> setting to',MAXHALF)
src.halfsize = MAXHALF
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
for src in cat:
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
disable_galaxy_cache()
F = fits_table()
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32)
ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x-1).astype(np.float32)
F.y = (y-1).astype(np.float32)
if opt.forced:
if opt.plots is None:
forced_kwargs.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True, fitstats=True,
shared_params=False, priors=False, **forced_kwargs)
if opt.plots:
(data,mod,ie,chi,roi) = R.ims1[0]
ima = tim.ima
imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5)
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
F.flux = np.array([src.getBrightness().getFlux(tim.band)
for src in cat]).astype(np.float32)
F.flux_ivar = R.IV.astype(np.float32)
F.fracflux = R.fitstats.profracflux.astype(np.float32)
F.rchi2 = R.fitstats.prochi2 .astype(np.float32)
print('Forced photom:', Time()-t0)
if opt.apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
print('Apertures:', apertures, 'pixels')
for rad in apertures:
aper = photutils.CircularAperture(apxy, rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = ap.astype(np.float32)
ap = 1./(np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux_ivar = ap.astype(np.float32)
print('Aperture photom:', Time()-t0)
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, surveydir)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(dict(name='EXPOSURE',
value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname),
comment='Name of this image'))
keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV',
'INSTRUME']
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {'exptime':'sec', 'flux':'nanomaggy', 'flux_ivar':'1/nanomaggy^2'}
columns = F.get_columns()
for i,col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
print('Getting model image...')
mod = tr.getModelImage(tim)
fitsio.write(opt.save_model, mod, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
print('Finished forced phot:', Time()-t0)
return 0
def main(survey=None, opt=None):
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
import pylab as plt
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.expnum)
filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
filename = opt.expnum
# Try parsing HDU number
try:
hdu = int(opt.ccdname)
ccdname = None
except:
hdu = -1
ccdname = opt.ccdname
if survey is None:
survey = LegacySurveyData()
catsurvey = survey
if opt.catalog_dir is not None:
catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir)
if filename is not None and hdu >= 0:
# FIXME -- try looking up in CCDs file?
# Read metadata from file
print('Warning: faking metadata from file contents')
T = exposure_metadata([filename], hdus=[hdu])
print('Metadata:')
T.about()
if not 'ccdzpt' in T.columns():
phdr = fitsio.read_header(filename)
T.ccdzpt = np.array([phdr['MAGZERO']])
print('WARNING: using header MAGZERO')
T.ccdraoff = np.array([0.])
T.ccddecoff = np.array([0.])
print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.')
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if hdu >= 0:
T.cut(T.image_hdu == hdu)
print(len(T), 'with HDU', hdu)
if filename is not None:
T.cut(np.array([f.strip() == filename for f in T.image_filename]))
print(len(T), 'with filename', filename)
if opt.camera is not None:
T.cut(T.camera == opt.camera)
print(len(T), 'with camera', opt.camera)
assert (len(T) == 1)
ccd = T[0]
im = survey.get_image_object(ccd)
if opt.do_calib:
#from legacypipe.survey import run_calibs
#kwa = dict(splinesky=True)
#run_calibs((im, kwa))
im.run_calibs(splinesky=True)
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
splinesky=True,
constant_invvar=opt.constant_invvar,
hybridPsf=opt.hybrid_psf,
normalizePsf=opt.normalize_psf)
print('Got tim:', tim)
print('Read image:', Time() - t0)
if opt.catfn in ['DR1', 'DR2', 'DR3', 'DR5', 'DR']:
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = catsurvey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) *
(yy >= -margin) * (yy <= (H + margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
for col in ['out_of_bounds', 'left_blob']:
if col in T.get_columns():
T.cut(T.get(col) == False)
print('Cut to', len(T), 'on', col)
if len(T):
TT.append(T)
if len(TT) == 0:
print('No sources to photometer.')
return 0
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
print('Total of', len(T), 'catalog sources')
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
surveydir = survey.get_survey_dir()
del survey
kwargs = {}
cols = T.get_columns()
if 'flux_r' in cols and not 'decam_flux_r' in cols:
kwargs.update(fluxPrefix='')
cat = read_fits_catalog(T, **kwargs)
# Replace the brightness (which will be a NanoMaggies with g,r,z)
# with a NanoMaggies with this image's band only.
for src in cat:
src.brightness = NanoMaggies(**{tim.band: 1.})
print('Read catalog:', Time() - t0)
print('Forced photom...')
F = run_forced_phot(cat,
tim,
ceres=opt.ceres,
derivs=opt.derivs,
fixed_also=True,
agn=opt.agn,
do_forced=opt.forced,
do_apphot=opt.apphot,
ps=ps)
t0 = Time()
F.release = T.release
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.camera = np.array([ccd.camera] * len(F))
F.expnum = np.array([im.expnum] * len(F)).astype(np.int32)
F.ccdname = np.array([im.ccdname] * len(F))
# "Denormalizing"
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32)
F.ra = T.ra
F.dec = T.dec
ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
h, w = tim.shape
F.mask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h - 1),
np.clip(np.round(F.x).astype(int), 0, w - 1)]
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, surveydir)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(
dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(
dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='EXPOSURE',
value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname),
comment='Name of this image'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2'
}
columns = F.get_columns()
for i, col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
print('Getting model image...')
tr = Tractor([tim], cat)
mod = tr.getModelImage(tim)
fitsio.write(opt.save_model, mod, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
print('Finished forced phot:', Time() - t0)
return 0