def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--name1", help="Name for first data set")
parser.add_argument("--name2", help="Name for second data set")
parser.add_argument("--plot-prefix", default="compare", help='Prefix for plot filenames; default "%default"')
parser.add_argument("--match", default=1.0, help="Astrometric cross-match distance in arcsec")
parser.add_argument("dir1", help="First directory to compare")
parser.add_argument("dir2", help="Second directory to compare")
opt = parser.parse_args()
ps = PlotSequence(opt.plot_prefix)
name1 = opt.name1
if name1 is None:
name1 = os.path.basename(opt.dir1)
if not len(name1):
name1 = os.path.basename(os.path.dirname(opt.dir1))
name2 = opt.name2
if name2 is None:
name2 = os.path.basename(opt.dir2)
if not len(name2):
name2 = os.path.basename(os.path.dirname(opt.dir2))
tt = "Comparing %s to %s" % (name1, name2)
# regex for tractor-*.fits catalog filename
catre = re.compile("tractor-.*.fits")
cat1, cat2 = [], []
for basedir, cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
for dirpath, dirnames, filenames in os.walk(basedir, followlinks=True):
for fn in filenames:
if not catre.match(fn):
print("Skipping", fn, "due to filename")
continue
fn = os.path.join(dirpath, fn)
t = fits_table(fn)
print(len(t), "from", fn)
cat.append(t)
cat1 = merge_tables(cat1, columns="fillzero")
cat2 = merge_tables(cat2, columns="fillzero")
print("Total of", len(cat1), "from", name1)
print("Total of", len(cat2), "from", name2)
cat1.cut(cat1.brick_primary)
cat2.cut(cat2.brick_primary)
print("Total of", len(cat1), "BRICK_PRIMARY from", name1)
print("Total of", len(cat2), "BRICK_PRIMARY from", name2)
cat1.cut((cat1.decam_anymask[:, 1] == 0) * (cat1.decam_anymask[:, 2] == 0) * (cat1.decam_anymask[:, 4] == 0))
cat2.cut((cat2.decam_anymask[:, 1] == 0) * (cat2.decam_anymask[:, 2] == 0) * (cat2.decam_anymask[:, 4] == 0))
print("Total of", len(cat1), "unmasked from", name1)
print("Total of", len(cat2), "unmasked from", name2)
I, J, d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match / 3600.0, nearest=True)
print(len(I), "matched")
matched1 = cat1[I]
matched2 = cat2[J]
def decals_dr3():
basedir = os.environ['LEGACY_SURVEY_DIR']
cam = 'decam'
image_basedir = os.path.join(basedir, 'images')
TT = []
#zpdir = '/project/projectdirs/cosmo/work/decam/cats/ZeroPoints'
for fn,dirnms in [
('/global/cscratch1/sd/desiproc/zeropoints/decals-zpt-dr3pr1233b.fits',
['CP20140810_?_v2',
'CP20141227', 'CP20150108', 'CP20150326',
'CP20150407', 'CP20151010', 'CP20151028', 'CP20151126',
'CP20151226', 'CP20160107', 'CP20160225',
'COSMOS', 'CPDES82',
'NonDECaLS/*',
]),
]:
T = normalize_zeropoints(fn, dirnms, image_basedir, cam)
TT.append(T)
T = merge_tables(TT)
outfn = 'zp.fits'
T.writeto(outfn)
print('Wrote', outfn)
nd = np.array(['NonDECaLS' in fn for fn in T.image_filename])
I = np.flatnonzero(nd)
T[I].writeto('survey-ccds-nondecals.fits')
I = np.flatnonzero(np.logical_not(nd))
T[I].writeto('survey-ccds-decals.fits')
for fn in ['survey-ccds-nondecals.fits', 'survey-ccds-decals.fits']:
os.system('gzip --best ' + fn)
def decals_dr3_plus():
basedir = os.environ['LEGACY_SURVEY_DIR']
cam = 'decam'
image_basedir = os.path.join(basedir, 'images')
TT = []
expnums = [547257, 535154, 535106]
for fn,dirnms in [
('/global/homes/a/arjundey/ZeroPoints/decals-zpt-20160407.fits',
['CP20160407',]),
('/global/homes/a/arjundey/ZeroPoints/decals-zpt-20160606.fits',
['CP20160606',]),
]:
T = fits_table(fn)
T.cut(np.nonzero([e not in expnums for e in T.expnum])[0])
normalize_zeropoints(fn, dirnms, image_basedir, cam, T=T)
TT.append(T)
dirnms = ['CP20160407','CP20160606']
for fn in [
'/global/cscratch1/sd/arjundey/ZeroPoints/zeropoint-c4d_160408_023844_oki_r_v1.fits',
'/global/cscratch1/sd/arjundey/ZeroPoints/zeropoint-c4d_160408_001343_oki_r_v1.fits',
'/global/cscratch1/sd/arjundey/ZeroPoints/zeropoint-c4d_160607_023641_oki_g_v1.fits',
]:
T = normalize_zeropoints(fn, dirnms, image_basedir, cam)
TT.append(T)
T = merge_tables(TT)
outfn = 'survey-ccds-dr3plus.fits'
T.writeto(outfn)
print('Wrote', outfn)
for fn in [outfn]:
os.system('gzip --best ' + fn)
def bok_dr4():
basedir = os.environ['LEGACY_SURVEY_DIR'] #'/scratch1/scratchdirs/desiproc/DRs/dr4-bootes/legacypipe-dir'
cam = '90prime'
image_basedir = os.path.join(basedir, 'images')
#/scratch1/scratchdirs/desiproc/DRs/cp-images/bootes/project/projectdirs/cosmo/staging/bok/BOK_CP/CP20160703
#/scratch2/scratchdirs/arjundey/ForKaylan/zeropoint-ksb_160704_052458_ooi_g_v1.fits
TT = []
for fn,dirnms in [
('arjuns-ccds-90prime.fits',
list(np.loadtxt('bok_cpdirs.txt',dtype=str))),
#(os.path.join(zpdir, 'r/zeropoint-BOK20150413_g.fits'),
# [os.path.join(zpdir, 'g')]),
]:
T = normalize_zeropoints(fn, dirnms, image_basedir, cam)
# fake up the exposure number
T.expnum = (T.mjd_obs * 100000.).astype(int)
# compute extension name
T.ccdname = np.array(['ccd%i' % n for n in T.ccdnum])
# compute FWHM from Seeing
pixscale = 0.45
T.fwhm = T.seeing / pixscale
T.expid = np.array(['%10i-%s' % (expnum,extname.strip())
for expnum,extname in zip(T.expnum, T.ccdname)])
TT.append(T)
T = merge_tables(TT)
outfn = 'survey-ccds-90prime.fits.gz'
T.writeto(outfn)
print('Wrote', outfn)
def mzls_dr4(v2=True):
if v2:
zpname='arjuns-ccds-mzls-v2thruMarch19.fits'
savefn='survey-ccds-mzls-v2thruMarch19.fits.gz'
else:
zpname='arjuns-ccds-mzls-v3.fits'
savefn='survey-ccds-mzls-v3.fits.gz'
basedir = os.environ['LEGACY_SURVEY_DIR']
cam = 'mosaic'
image_basedir = os.path.join(basedir, 'images')
TT = []
# mzls_cpdirs_...txt is list of all possible CP directories
for fn,dirnms in [
(zpname,
list(np.loadtxt('mzls_cpdirs_thruMarch19.txt',dtype=str))),
]:
T = fits_table(fn)
normalize_zeropoints(fn, dirnms, image_basedir, cam, T=T)
TT.append(T)
T = merge_tables(TT)
I = np.flatnonzero(T.fwhm == 0)
if len(I):
T.fwhm[I] = T.seeing[I] / 0.262
T.writeto(savefn)
print('Wrote', savefn)
def decals_dr2():
decals_dir = os.environ['DECALS_DIR']
cam = 'decam'
image_basedir = os.path.join(decals_dir, 'images')
TT = []
#zpdir = '/project/projectdirs/cosmo/work/decam/cats/ZeroPoints'
for fn,dirnms in [
('/scratch1/scratchdirs/desiproc/ZeroPoints/decals-zpt-all-2015oct30.fits',
['CP20140810_?_v2', 'CP20141227', 'CP20150108', 'CP20150326',
'CP20150407', 'NonDECaLS/*','COSMOS', 'CPDES82']),
# (os.path.join(zpdir, 'decals-zpt-20140810.fits'), ['CP20140810_?_v2']),
# (os.path.join(zpdir, 'decals-zpt-20141227.fits'), ['CP20141227']),
# (os.path.join(zpdir, 'decals-zpt-20150108.fits'), ['CP20150108']),
# (os.path.join(zpdir, 'decals-zpt-20150326.fits'), ['CP20150326']),
# (os.path.join(zpdir, 'decals-zpt-20150407.fits'), ['CP20150407']),
# (os.path.join(zpdir, 'decals-zpt-nondecals.fits'), ['NonDECaLS/*','COSMOS', 'CPDES82']),
]:
T = normalize_zeropoints(fn, dirnms, image_basedir, cam)
TT.append(T)
T = merge_tables(TT)
outfn = 'zp.fits'
T.writeto(outfn)
print('Wrote', outfn)
def stack_tables(fn_list, textfile=True, shuffle=None):
'''concatenates fits tables
Args:
shuffle: set to an integer to randomly reads up to the first "shuffle" cats only
'''
if shuffle:
assert (isinstance(shuffle, int))
if textfile:
fns = read_lines(fn_list)
else:
fns = fn_list
if len(fns) < 1: raise ValueError('Error: fns=', fns)
if shuffle:
print('shuffling %d' % shuffle)
seed = 7
np.random.seed(seed)
inds = np.arange(len(fns))
np.random.shuffle(inds)
fns = fns[inds]
cats = []
for i, fn in enumerate(fns):
print('reading %s %d/%d' % (fn, i + 1, len(fns)))
if shuffle and i >= shuffle:
print('shuffle_1000 turned ON, stopping read')
break
try:
tab = fits_table(fn)
cats.append(tab)
except IOError:
print('Fits file does not exist: %s' % fn)
return merge_tables(cats, columns='fillzero')
def mzls_to_20160315():
basedir = os.environ['LEGACY_SURVEY_DIR']
cam = 'mosaic'
image_basedir = os.path.join(basedir, 'images')
TT = []
for fn,dirnms in [
('/global/homes/a/arjundey/ZeroPoints/mzls-zpt-all.fits',
['CP20160202','CP20160203','CP20160204','CP20160205','CP20160206','CP20160208',
'CP20160209','CP20160210','CP20160211','CP20160212','CP20160213','CP20160214',
'CP20160215','CP20160216','CP20160217','CP20160219','CP20160224','CP20160225',
'CP20160226','CP20160227','CP20160228','CP20160229','CP20160301','CP20160302',
'CP20160303','CP20160304','CP20160305','CP20160306','CP20160308','CP20160309',
'CP20160310','CP20160311','CP20160312','CP20160313','CP20160314','CP20160315',
'CP20160316','CP20160317','CP20160318','CP20160319','CP20160320','CP20160325',
'CP20160326','CP20160327','CP20160328','CP20160330','CP20160331','CP20160401',
'CP20160402','CP20160403','CP20160404','CP20160408',]),
]:
T = fits_table(fn)
normalize_zeropoints(fn, dirnms, image_basedir, cam, T=T)
TT.append(T)
T = merge_tables(TT)
I = np.flatnonzero(T.fwhm == 0)
if len(I):
T.fwhm[I] = T.seeing[I] / 0.262
outfn = 'survey-ccds-mzls-to-20160315.fits'
T.writeto(outfn)
print('Wrote', outfn)
for fn in [outfn]:
os.system('gzip --best ' + fn)
def get_healpix_catalogs(self, healpixes):
from astrometry.util.fits import merge_tables
cats = []
for hp in healpixes:
cats.append(self.get_healpix_catalog(hp))
if len(cats) == 1:
return cats[0]
return merge_tables(cats)
def cat_sdss(req, ver):
import json
import numpy as np
from astrometry.util.starutil_numpy import degrees_between, radectoxyz, xyztoradec
from map.views import sdss_ccds_near
from astrometry.util.fits import fits_table, merge_tables
tag = 'sdss-cat'
ralo = float(req.GET['ralo'])
rahi = float(req.GET['rahi'])
declo = float(req.GET['declo'])
dechi = float(req.GET['dechi'])
ver = int(ver)
if not ver in catversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
rad = degrees_between(ralo, declo, rahi, dechi) / 2.
xyz1 = radectoxyz(ralo, declo)
xyz2 = radectoxyz(rahi, dechi)
xyz = (xyz1 + xyz2)
xyz /= np.sqrt(np.sum(xyz**2))
rc,dc = xyztoradec(xyz)
rad = rad + np.hypot(10.,14.)/2./60.
ccds = sdss_ccds_near(rc[0], dc[0], rad)
if ccds is None:
print('No SDSS CCDs nearby')
return HttpResponse(json.dumps(dict(rd=[])),
content_type='application/json')
print(len(ccds), 'SDSS CCDs')
T = []
for ccd in ccds:
# env/BOSS_PHOTOOBJ/301/2073/3/photoObj-002073-3-0088.fits
fn = os.path.join(settings.SDSS_BASEDIR, 'env', 'BOSS_PHOTOOBJ',
str(ccd.rerun), str(ccd.run), str(ccd.camcol),
'photoObj-%06i-%i-%04i.fits' % (ccd.run, ccd.camcol, ccd.field))
print('Reading', fn)
T.append(fits_table(fn, columns='ra dec objid mode objc_type objc_flags objc_flags nchild tai expflux devflux psfflux cmodelflux fracdev mjd'.split()))
T = merge_tables(T)
T.cut((T.dec >= declo) * (T.dec <= dechi))
# FIXME
T.cut((T.ra >= ralo) * (T.ra <= rahi))
# primary
T.cut(T.mode == 1)
types = ['P' if t == 6 else 'C' for t in T.objc_type]
fluxes = [p if t == 6 else c for t,p,c in zip(T.objc_type, T.psfflux, T.cmodelflux)]
return HttpResponse(json.dumps(dict(
rd=[(float(o.ra),float(o.dec)) for o in T],
sourcetype=types,
fluxes = [dict(u=float(f[0]), g=float(f[1]), r=float(f[2]),
i=float(f[3]), z=float(f[4])) for f in fluxes],
)),
content_type='application/json')
def get_healpix_rangesearch_catalogs(self, healpixes, rc, dc, rad):
cats = []
for hp in healpixes:
(kd, tab) = self.get_healpix_tree(hp)
I = tree_search_radec(kd, rc, dc, rad)
if len(I):
cats.append(tab[I])
if len(cats) == 1:
return cats[0] #.copy()
return merge_tables(cats)
def decals_dr3_extra():
# /global/homes/a/arjundey/ZeroPoints/decals-zpt-dr3-all.fits
T = fits_table('/global/cscratch1/sd/desiproc/zeropoints/decals-zpt-dr3-all.fits')
S1 = fits_table('survey-ccds-nondecals.fits.gz')
S2 = fits_table('survey-ccds-decals.fits.gz')
S = merge_tables([S1,S2])
gotchips = set(zip(S.expnum, S.ccdname))
got = np.array([(e,c) in gotchips for e,c in zip(T.expnum, T.ccdname)])
print('Found', sum(got), 'of', len(T), 'dr3-all CCDs in existing surveys tables of size', len(S))
I = np.flatnonzero(np.logical_not(got))
T.cut(I)
print(len(T), 'remaining')
#print('Directories:', np.unique([os.path.basename(os.path.dirname(fn.strip())) for fn in T.filename]))
print('Filenames:', np.unique(T.filename))
T.writeto('extras.fits')
basedir = os.environ['LEGACY_SURVEY_DIR']
cam = 'decam'
image_basedir = os.path.join(basedir, 'images')
TT = []
for fn,dirnms in [
('extras.fits',
['CP20140810_?_v2',
'CP20141227', 'CP20150108', 'CP20150326',
'CP20150407', 'CP20151010', 'CP20151028', 'CP20151126',
'CP20151226', 'CP20160107', 'CP20160225',
'COSMOS', 'CPDES82',
'NonDECaLS/*',
]),
]:
T = normalize_zeropoints(fn, dirnms, image_basedir, cam)
TT.append(T)
T = merge_tables(TT)
outfn = 'survey-ccds-extra.fits'
T.writeto(outfn)
print('Wrote', outfn)
def run(self, bricks_to_merge):
Tlist = []
for brick in bricks_to_merge:
fn = self.table_fn(brick)
if os.path.exists(fn):
tab = fits_table(self.table_fn(brick))
Tlist.append(tab)
else:
print('Skipping brick %s b/c randoms table doesnt exist' % fn)
T = merge_tables(Tlist, columns='fillzero')
T.writeto(self.savefn)
print('Wrote %s' % self.savefn)
def merge_splinesky(survey, expnum, C, skyoutfn, opt):
skies = []
imobjs = []
Cgood = []
for ccd in C:
im = survey.get_image_object(ccd)
fn = im.skyfn
if not os.path.exists(fn):
print('File not found:', fn)
if opt.all_found:
return 0
continue
imobjs.append(im)
Cgood.append(ccd)
for ccd, im in zip(Cgood, imobjs):
fn = im.skyfn
print('Reading', fn)
T = None
try:
T = fits_table(fn)
except KeyboardInterrupt as e:
raise e
except:
print('Failed to read file', fn, ':', sys.exc_info()[1])
if T is not None:
skies.append(T)
if len(skies) == 0:
return
T = fits_table()
T.gridw = np.array([t.gridvals[0].shape[1] for t in skies])
T.gridh = np.array([t.gridvals[0].shape[0] for t in skies])
padded = pad_arrays([t.gridvals[0] for t in skies])
T.gridvals = np.concatenate([[p] for p in padded])
padded = pad_arrays([t.xgrid[0] for t in skies])
T.xgrid = np.concatenate([[p] for p in padded])
padded = pad_arrays([t.ygrid[0] for t in skies])
T.ygrid = np.concatenate([[p] for p in padded])
cols = skies[0].columns()
for c in ['gridvals', 'xgrid', 'ygrid', 'gridw', 'gridh']:
cols.remove(c)
T.add_columns_from(merge_tables(skies, columns=cols))
fn = skyoutfn
trymakedirs(fn, dir=True)
tmpfn = os.path.join(os.path.dirname(fn), 'tmp-' + os.path.basename(fn))
T.writeto(tmpfn)
os.rename(tmpfn, fn)
print('Wrote', fn)
return 1
def cat_vcc(req, ver):
import json
tag = 'ngc'
ralo = float(req.GET['ralo'])
rahi = float(req.GET['rahi'])
declo = float(req.GET['declo'])
dechi = float(req.GET['dechi'])
ver = int(ver)
if not ver in catversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
from astrometry.util.fits import fits_table, merge_tables
import numpy as np
from decals import settings
TT = []
T = fits_table(os.path.join(settings.DATA_DIR, 'virgo-cluster-cat-2.fits'))
print(len(T), 'in VCC 2; ra', ralo, rahi, 'dec', declo, dechi)
T.cut((T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi))
print(len(T), 'in cut')
TT.append(T)
T = fits_table(os.path.join(settings.DATA_DIR, 'virgo-cluster-cat-3.fits'))
print(len(T), 'in VCC 3; ra', ralo, rahi, 'dec', declo, dechi)
T.cut((T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi))
print(len(T), 'in cut')
T.evcc_id = np.array(['-']*len(T))
T.rename('id', 'vcc_id')
TT.append(T)
T = merge_tables(TT)
rd = list((float(r),float(d)) for r,d in zip(T.ra, T.dec))
names = []
for t in T:
evcc = t.evcc_id.strip()
vcc = t.vcc_id.strip()
ngc = t.ngc.strip()
nms = []
if evcc != '-':
nms.append('EVCC ' + evcc)
if vcc != '-':
nms.append('VCC ' + vcc)
if ngc != '-':
nms.append('NGC ' + ngc)
names.append(' / '.join(nms))
return HttpResponse(json.dumps(dict(rd=rd, name=names)),
content_type='application/json')
def cat_vcc(req, ver):
import json
tag = 'ngc'
ralo = float(req.GET['ralo'])
rahi = float(req.GET['rahi'])
declo = float(req.GET['declo'])
dechi = float(req.GET['dechi'])
ver = int(ver)
if not ver in catversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
from astrometry.util.fits import fits_table, merge_tables
import numpy as np
from decals import settings
TT = []
T = fits_table(os.path.join(settings.DATA_DIR, 'virgo-cluster-cat-2.fits'))
print(len(T), 'in VCC 2; ra', ralo, rahi, 'dec', declo, dechi)
T.cut((T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi))
print(len(T), 'in cut')
TT.append(T)
T = fits_table(os.path.join(settings.DATA_DIR, 'virgo-cluster-cat-3.fits'))
print(len(T), 'in VCC 3; ra', ralo, rahi, 'dec', declo, dechi)
T.cut((T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi))
print(len(T), 'in cut')
T.evcc_id = np.array(['-'] * len(T))
T.rename('id', 'vcc_id')
TT.append(T)
T = merge_tables(TT)
rd = list((float(r), float(d)) for r, d in zip(T.ra, T.dec))
names = []
for t in T:
evcc = t.evcc_id.strip()
vcc = t.vcc_id.strip()
ngc = t.ngc.strip()
nms = []
if evcc != '-':
nms.append('EVCC ' + evcc)
if vcc != '-':
nms.append('VCC ' + vcc)
if ngc != '-':
nms.append('NGC ' + ngc)
names.append(' / '.join(nms))
return HttpResponse(json.dumps(dict(rd=rd, name=names)),
content_type='application/json')
def __init__(self, imagedir, nom, opt):
super(NgcBot, self).__init__(imagedir, backlog=False)
self.nom = nom
self.opt = opt
# Read catalogs
TT = []
import astrometry
catdir = os.path.join(os.path.dirname(astrometry.__file__), 'catalogs')
fn = os.path.join(catdir, 'ngc2000.fits')
print('Reading', fn)
T = fits_table(fn)
T.delete_column('ngcnum')
TT.append(T)
fn = os.path.join(catdir, 'ic2000.fits')
print('Reading', fn)
T = fits_table(fn)
T.delete_column('icnum')
TT.append(T)
self.cat = merge_tables(TT, columns='fillzero')
del TT
print('Total of', len(self.cat), 'NGC/IC objects')
omit = [
'OC', # open cluster
'Ast', # asterism
'***', # triple star
'D*', # double star
'*', # star
'-', # called nonexistent in RNGC
'PD' # plate defect
]
# '?', # uncertain type or may not exist
# '', # unidentified or type unknown
print(Counter(self.cat.classification))
self.cat.cut(
np.flatnonzero(
np.array(
[t.strip() not in omit for t in self.cat.classification])))
print('Cut to', len(self.cat), 'NGC/IC objects')
print('Remaining classifications:', np.unique(self.cat.classification))
self.spec = fits_table('specObj-dr12-trim-2.fits')
self.cached_flats = {}
self.read_flats = opt.flats
def __init__(self, ccds_used_wildcard=None):
'''plot ra,dec of ccds used (the footprint for this run)
Args:
ccds_used_wildcard: path to ccds file or path containing an asterisk
'''
if "*" in ccds_used_wildcard:
fns = glob(ccds_used_wildcard)
assert (len(fns) > 0)
ccds = [
fits_table(fn, columns=['ra', 'dec', 'expid']) for fn in fns
]
self.ccds = merge_tables(ccds)
del ccds
else:
self.ccds = fits_table(ccds_used_wildcard)
def decals_dr3_fix392400():
T = fits_table('survey-ccds-decals.fits.gz')
print('Started with', len(T), 'CCDs')
T.cut(T.expnum != 392400)
print('Removed bad CCDs:', len(T))
basedir = os.environ['LEGACY_SURVEY_DIR']
cam = 'decam'
image_basedir = os.path.join(basedir, 'images')
fn = '/global/cscratch1/sd/desiproc/zeropoints/zeropoint-c4d_141228_060426_ooi_g_v1.fits'
dirnms = ['CP20141227']
Tnew = normalize_zeropoints(fn, dirnms, image_basedir, cam)
print('Replacement CCDs:', len(Tnew))
T = merge_tables([T, Tnew])
print('Merged:', len(T))
T.writeto('new-survey-ccds-decals.fits.gz')
def vipers(self):
# Data
w1 = fits_table(os.path.join(self.truth_dir, 'vipers-w1.fits.gz'))
w4 = fits_table(os.path.join(self.truth_dir, 'vipers-w4.fits.gz'))
# Bricks
for data in [w1, w4]:
data.set('ra', data.get('alpha'))
data.set('dec', data.get('delta'))
bnames = {}
for data, key in zip([w1, w4], ['w1', 'w4']):
bnames[key]= self.bricks_in_region(rlo=data.get('ra').min(), rhi=data.get('ra').max(),\
dlo=data.get('dec').min(),dhi=data.get('dec').max())
bricks = np.array([])
for key in bnames.keys():
bricks = np.concatenate((bricks, bnames[key]))
# Tractor Catalogues --> file list
catlist = os.path.join(self.save_dir, 'vipers_dr3_bricks.txt')
if not os.path.exists(catlist):
fout = open(catlist, 'w')
for b in bricks:
fn = os.path.join(self.dr3_dir,
'tractor/%s/tractor-%s.fits' % (b[:3], b))
fout.write('%s\n' % fn)
fout.close()
print('Wrote %s' % catlist)
# Merge w1,w4 for matching
vipers = []
for fn in [os.path.join(self.truth_dir,'vipers-w1.fits.gz'),\
os.path.join(self.truth_dir,'vipers-w4.fits.gz')]:
vipers.append(fits_table(fn))
vipers = merge_tables(vipers, columns='fillzero')
vipers.set('ra', data.get('alpha'))
vipers.set('dec', data.get('delta'))
# Match
fits_funcs = CatalogueFuncs()
dr3 = fits_funcs.stack(
os.path.join(self.save_dir, 'vipers_dr3_bricks.txt'))
mat = Matcher()
imatch, imiss, d2d = mat.match_within(vipers, dr3) #,dist=1./3600)
vipers.cut(imatch['ref'])
dr3.cut(imatch['obs'])
# Save
vipers.writeto(
os.path.join(self.save_dir, 'vipersw1w4-dr3matched.fits'))
dr3.writeto(os.path.join(self.save_dir, 'dr3-vipersw1w4matched.fits'))
print('Wrote %s\nWrote %s' % (os.path.join(self.save_dir,'vipersw1w4-dr3matched.fits'),\
os.path.join(self.save_dir,'dr3-vipersw1w4matched.fits')))
def get_cat(self,ra,dec):
"""Read the healpixed PS1 catalogs given input ra,dec coordinates."""
from astrometry.util.fits import fits_table, merge_tables
from astrometry.util.util import radecdegtohealpix, healpix_xy_to_ring
ipring = np.empty(len(ra)).astype(int)
for iobj, (ra1, dec1) in enumerate(zip(ra,dec)):
hpxy = radecdegtohealpix(ra1,dec1,self.nside)
ipring[iobj] = healpix_xy_to_ring(hpxy,self.nside)
pix = np.unique(ipring)
cat = list()
for ipix in pix:
fname = os.path.join(self.ps1dir,'ps1-'+'{:05d}'.format(ipix)+'.fits')
print('Reading {}'.format(fname))
cat.append(fits_table(fname))
cat = merge_tables(cat)
return cat
def get_cat(self,ra,dec):
"""Read the healpixed PS1 catalogs given input ra,dec coordinates."""
from astrometry.util.fits import fits_table, merge_tables
from astrometry.util.util import radecdegtohealpix, healpix_xy_to_ring
ipring = np.empty(len(ra)).astype(int)
for iobj, (ra1, dec1) in enumerate(zip(ra,dec)):
hpxy = radecdegtohealpix(ra1,dec1,self.nside)
ipring[iobj] = healpix_xy_to_ring(hpxy,self.nside)
pix = np.unique(ipring)
cat = list()
for ipix in pix:
fname = os.path.join(self.ps1dir,'ps1-'+'{:05d}'.format(ipix)+'.fits')
print('Reading {}'.format(fname))
cat.append(fits_table(fname))
cat = merge_tables(cat)
return cat
def brick_catalog_for_radec_box(ralo, rahi, declo, dechi,
decals, catpattern, bricks=None):
'''
Merges multiple Tractor brick catalogs to cover an RA,Dec
bounding-box.
No cleverness with RA wrap-around; assumes ralo < rahi.
decals: Decals object
bricks: table of bricks, eg from Decals.get_bricks()
catpattern: filename pattern of catalog files to read,
eg "pipebrick-cats/tractor-phot-%06i.its"
'''
assert(ralo < rahi)
assert(declo < dechi)
if bricks is None:
bricks = decals.get_bricks_readonly()
I = decals.bricks_touching_radec_box(bricks, ralo, rahi, declo, dechi)
print(len(I), 'bricks touch RA,Dec box')
TT = []
hdr = None
for i in I:
brick = bricks[i]
fn = catpattern % brick.brickid
print('Catalog', fn)
if not os.path.exists(fn):
print('Warning: catalog does not exist:', fn)
continue
T = fits_table(fn, header=True)
if T is None or len(T) == 0:
print('Warning: empty catalog', fn)
continue
T.cut((T.ra >= ralo ) * (T.ra <= rahi) *
(T.dec >= declo) * (T.dec <= dechi))
TT.append(T)
if len(TT) == 0:
return None
T = merge_tables(TT)
# arbitrarily keep the first header
T._header = TT[0]._header
return T
def brick_catalog_for_radec_box(ralo, rahi, declo, dechi,
survey, catpattern, bricks=None):
'''
Merges multiple Tractor brick catalogs to cover an RA,Dec
bounding-box.
No cleverness with RA wrap-around; assumes ralo < rahi.
survey: LegacySurveyData object
bricks: table of bricks, eg from LegacySurveyData.get_bricks()
catpattern: filename pattern of catalog files to read,
eg "pipebrick-cats/tractor-phot-%06i.its"
'''
assert(ralo < rahi)
assert(declo < dechi)
if bricks is None:
bricks = survey.get_bricks_readonly()
I = survey.bricks_touching_radec_box(bricks, ralo, rahi, declo, dechi)
print(len(I), 'bricks touch RA,Dec box')
TT = []
hdr = None
for i in I:
brick = bricks[i]
fn = catpattern % brick.brickid
print('Catalog', fn)
if not os.path.exists(fn):
print('Warning: catalog does not exist:', fn)
continue
T = fits_table(fn, header=True)
if T is None or len(T) == 0:
print('Warning: empty catalog', fn)
continue
T.cut((T.ra >= ralo ) * (T.ra <= rahi) *
(T.dec >= declo) * (T.dec <= dechi))
TT.append(T)
if len(TT) == 0:
return None
T = merge_tables(TT)
# arbitrarily keep the first header
T._header = TT[0]._header
return T
def merge_psfex(survey, expnum, C, psfoutfn, opt):
psfex = []
imobjs = []
Cgood = []
fns = []
for ccd in C:
im = survey.get_image_object(ccd)
for fn in [im.psffn, im.old_single_psffn]:
if os.path.exists(fn):
break
if not os.path.exists(fn):
print('File not found:', fn)
if opt.all_found:
return 0
continue
imobjs.append(im)
Cgood.append(ccd)
fns.append(fn)
for fn, ccd, im in zip(fns, Cgood, imobjs):
print('Reading', fn)
T = fits_table(fn)
cols = T.get_columns()
if not 'plver' in cols:
from legacypipe.image import psfex_single_to_merged
T = psfex_single_to_merged(fn, ccd.expnum, ccd.ccdname)
for k in ['plver', 'procdate', 'plprocid']:
T.set(k, np.array([getattr(ccd, k)]))
psfex.append(T)
if len(psfex) == 0:
return
padded = pad_arrays([p.psf_mask[0] for p in psfex])
cols = psfex[0].columns()
cols.remove('psf_mask')
T = merge_tables(psfex, columns=cols)
T.psf_mask = np.concatenate([[p] for p in padded])
fn = psfoutfn
trymakedirs(fn, dir=True)
tmpfn = os.path.join(os.path.dirname(fn), 'tmp-' + os.path.basename(fn))
T.writeto(tmpfn)
os.rename(tmpfn, fn)
print('Wrote', fn)
return 1
def decals_run19():
basedir = os.environ['LEGACY_SURVEY_DIR']
cam = 'decam'
image_basedir = os.path.join(basedir, 'images')
TT = []
for fn,dirnms in [
('/global/homes/a/arjundey/ZeroPoints/decals-zpt-20160801_20161011.fits',
['CP20160801', 'CP20161011']),
]:
T = fits_table(fn)
normalize_zeropoints(fn, dirnms, image_basedir, cam, T=T)
TT.append(T)
T = merge_tables(TT)
outfn = 'survey-ccds-run19.fits'
T.writeto(outfn)
print('Wrote', outfn)
for fn in [outfn]:
os.system('gzip --best ' + fn)
def decals_dr3_dedup():
SN = fits_table('survey-ccds-nondecals.fits.gz')
SD = fits_table('survey-ccds-decals.fits.gz')
sne = np.unique(SN.expnum)
sde = np.unique(SD.expnum)
isec = set(sne).intersection(sde)
print(len(isec), 'exposures in common between "decals" and "nondecals"')
# These are two versions of CP reductions in CPDES82 and CPHETDEX.
I = np.flatnonzero(np.array([e in isec for e in SD.expnum]))
print(len(I), 'rows in "decals"')
I = np.flatnonzero(np.array([e not in isec for e in SD.expnum]))
SD.cut(I)
print(len(SD), 'rows remaining')
# Now, also move "decals" filenames containing "CPDES82" to "nondecals".
I = np.flatnonzero(np.array(['CPDES82' in fn for fn in SD.image_filename]))
keep = np.ones(len(SD), bool)
keep[I] = False
print('Merging:')
SN.about()
SD[I].about()
SN2 = merge_tables((SN, SD[I]), columns='fillzero')
SD2 = SD[keep]
print('Moved CPDES82: now', len(SN2), 'non-decals and', len(SD2), 'decals')
SN2.writeto('survey-ccds-nondecals.fits')
SD2.writeto('survey-ccds-decals.fits')
SE = fits_table('survey-ccds-extra.fits.gz')
SN = fits_table('survey-ccds-nondecals.fits')
SD = fits_table('survey-ccds-decals.fits')
sne = np.unique(SN.expnum)
sde = np.unique(SD.expnum)
see = np.unique(SE.expnum)
i1 = set(sne).intersection(sde)
i2 = set(sne).intersection(see)
i3 = set(sde).intersection(see)
print('Intersections:', len(i1), len(i2), len(i3))
def deep2(self):
# Data
deep2 = {}
for key in ['1', '2', '3', '4']:
deep2[key] = fits_table(
'/project/projectdirs/desi/target/analysis/truth/deep2-field%s.fits.gz'
% key)
# Bricks
bnames = {}
for key in deep2.keys():
bnames[key]= self.bricks_in_region(rlo=deep2[key].get('ra').min(), rhi=deep2[key].get('ra').max(),\
dlo=deep2[key].get('dec').min(),dhi=deep2[key].get('dec').max())
print('Field=%s, Num Bricks=%d, Bricks:' % (key, len(bnames[key])),
bnames[key])
bricks = np.array([])
for key in bnames.keys():
bricks = np.concatenate((bricks, bnames[key]))
# Tractor Catalogues --> file list
catlist = os.path.join(self.save_dir, 'deep2_dr3_bricks.txt')
if not os.path.exists(catlist):
fout = open(catlist, 'w')
for b in bricks:
fn = os.path.join(self.dr3_dir,
'tractor/%s/tractor-%s.fits' % (b[:3], b))
fout.write('%s\n' % fn)
fout.close()
print('Wrote %s' % catlist)
# Merge for matching
dp2 = [deep2['2'], deep2['3'], deep2['4']]
dp2 = merge_tables(dp2, columns='fillzero')
# Match
fits_funcs = CatalogueFuncs()
dr3 = fits_funcs.stack(
os.path.join(self.save_dir, 'deep2_dr3_bricks.txt'))
mat = Matcher()
imatch, imiss, d2d = mat.match_within(dp2, dr3) #,dist=1./3600)
dp2.cut(imatch['ref'])
dr3.cut(imatch['obs'])
fits_funcs.set_extra_data(dr3)
# Save
dp2.writeto(os.path.join(self.save_dir, 'deep2f234-dr3matched.fits'))
dr3.writeto(os.path.join(self.save_dir, 'dr3-deep2f234matched.fits'))
print('Wrote %s\nWrote %s' % (os.path.join(self.save_dir,'deep2f234-dr3matched.fits'),\
os.path.join(self.save_dir,'dr3-deep2f234matched.fits')))
def get_ccds(self):
'''
Returns the table of CCDs.
'''
from glob import glob
fns = self.find_file('ccds')
fns.sort()
fns = self.filter_ccds_files(fns)
TT = []
for fn in fns:
print('Reading CCDs from', fn)
# cols = (
# 'exptime filter propid crpix1 crpix2 crval1 crval2 ' +
# 'cd1_1 cd1_2 cd2_1 cd2_2 ccdname ccdzpt ccdraoff ccddecoff ' +
# 'ccdnmatch camera image_hdu image_filename width height ' +
# 'ra dec zpt expnum fwhm mjd_obs').split()
#T = fits_table(fn, columns=cols)
T = fits_table(fn)
print('Got', len(T), 'CCDs')
TT.append(T)
T = merge_tables(TT, columns='fillzero')
print('Total of', len(T), 'CCDs')
del TT
cols = T.columns()
# Make DR1 CCDs table somewhat compatible with DR2
if 'extname' in cols and not 'ccdname' in cols:
T.ccdname = T.extname
if not 'camera' in cols:
T.camera = np.array(['decam'] * len(T))
if 'cpimage' in cols and not 'image_filename' in cols:
T.image_filename = T.cpimage
if 'cpimage_hdu' in cols and not 'image_hdu' in cols:
T.image_hdu = T.cpimage_hdu
# Remove trailing spaces from 'ccdname' column
if 'ccdname' in T.columns():
# "N4 " -> "N4"
T.ccdname = np.array([s.strip() for s in T.ccdname])
# Remove trailing spaces from 'camera' column.
T.camera = np.array([c.strip() for c in T.camera])
return T
def read_chunks(ra, dec):
chunkdir = os.environ['PS1CHUNKS']
phi = np.radians(ra)
theta = np.radians(90 - dec)
pixels = healpy.pixelfunc.ang2pix(32, theta, phi)
pixels = np.unique(pixels)
cat = []
for pix in pixels:
fn = chunkdir + 'ps1-%05d.fits' % pix
try:
cat.append(fits_table(fn, ext=1))
except ValueError:
continue
except IOError:
continue
if not cat:
raise NoCalibrationStars
return merge_tables(cat)
def merge_bybrick(bricks, outdir='', prefix='', cleanup=False):
for brick in bricks:
outfn = get_brick_merged_fn(brickname=brick,
outdir=outdir,
prefix=prefix)
if cleanup:
if os.path.exists(outfn):
# Safe to remove the pieces that went into outfn
rm_fns = get_brick_sample_fns(brickname=brick,
outdir=outdir,
prefix=prefix)
if not rm_fns is None:
print('removing files like: %s' % rm_fns[0])
try:
for rm_fn in rm_fns:
os.remove(rm_fn)
except OSError:
pass
continue
elif os.path.exists(outfn):
# We are creating outfn, don't spend time deleting
continue
else:
print('outfn=%s' % outfn)
fns = get_brick_sample_fns(brickname=brick,
outdir=outdir,
prefix=prefix)
if fns is None:
# Wildcard found nothing see outdir/nofns_wildcard.txt
continue
cats = []
for i, fn in enumerate(fns):
print('reading %d/%d' % (i + 1, len(fns)))
try:
tab = fits_table(fn)
cats.append(tab)
except IOError:
print('Fits file does not exist: %s' % fn)
cat = merge_tables(cats, columns='fillzero')
cat.writeto(outfn)
print('Wrote %s' % outfn)
def gather_arjuns_zpts(cpimage_list='bootes-90prime-abspath.txt',
name='arjuns-ccds-90prime.fits'):
zpdir='/scratch2/scratchdirs/arjundey/ForKaylan'
# Get list of cpimages want zeropoints files for
cps=np.loadtxt(cpimage_list,dtype=str)
cats=[]
for i,cp in enumerate(cps):
print('reading %d/%d' % (i+1,len(cps)))
if '90prime' in name:
zpname= os.path.basename(cp).replace('ksb','zeropoint-ksb').replace('.fits.fz','.fits')
elif 'mzls' in name:
zpname= os.path.basename(cp).replace('k4m','zeropoint-k4m').replace('.fits.fz','.fits')
else:
raise ValueError('%s' % name)
zpname=os.path.join(zpdir,zpname)
try:
cats.append( fits_table(zpname) )
except IOError:
print('WARNING: Cannot read this file: %s' % zpname)
bigcat=merge_tables(cats, columns='fillzero')
bigcat.writeto(name)
def decals_run16():
basedir = os.environ['LEGACY_SURVEY_DIR']
cam = 'decam'
image_basedir = os.path.join(basedir, 'images')
TT = []
for fn,dirnms in [
('/global/homes/a/arjundey/ZeroPoints/decals-zpt-20160709_20.fits',
['CP20160709', 'CP20160720']),
]:
T = fits_table(fn)
#T.cut(np.nonzero([e not in expnums for e in T.expnum])[0])
normalize_zeropoints(fn, dirnms, image_basedir, cam, T=T)
TT.append(T)
T = merge_tables(TT)
outfn = 'survey-ccds-run16.fits'
T.writeto(outfn)
print('Wrote', outfn)
for fn in [outfn]:
os.system('gzip --best ' + fn)
def _get_decals_cat(wcs, tag='decals'):
from astrometry.util.fits import fits_table, merge_tables
from map.views import get_survey
basedir = settings.DATA_DIR
H, W = wcs.shape
X = wcs.pixelxy2radec([1, 1, 1, W / 2, W, W, W, W / 2],
[1, H / 2, H, H, H, H / 2, 1, 1])
r, d = X[-2:]
#catpat = os.path.join(basedir, 'cats', tag, '%(brickname).3s',
# 'tractor-%(brickname)s.fits')
survey = get_survey(tag)
B = survey.get_bricks_readonly()
I = survey.bricks_touching_radec_box(B, r.min(), r.max(), d.min(), d.max())
#print(len(I), 'bricks touching RA,Dec box', r.min(),r.max(), d.min(),d.max())
cat = []
hdr = None
for brickname in B.brickname[I]:
catfn = survey.find_file('tractor', brick=brickname)
if not os.path.exists(catfn):
print('Does not exist:', catfn)
continue
debug('Reading catalog', catfn)
T = fits_table(catfn)
T.cut(T.brick_primary)
print('File', catfn, 'cut to', len(T), 'primary')
ok, xx, yy = wcs.radec2pixelxy(T.ra, T.dec)
T.cut((xx > 0) * (yy > 0) * (xx < W) * (yy < H))
cat.append(T)
if hdr is None:
hdr = T.get_header()
if len(cat) == 0:
cat = None
else:
cat = merge_tables(cat, columns='fillzero')
return cat, hdr
def _get_decals_cat(wcs, tag='decals'):
from astrometry.util.fits import fits_table, merge_tables
from map.views import get_survey
H,W = wcs.shape
X = wcs.pixelxy2radec([1,1,1,W/2,W,W,W,W/2],
[1,H/2,H,H,H,H/2,1,1])
r,d = X[-2:]
#catpat = os.path.join(basedir, 'cats', tag, '%(brickname).3s',
# 'tractor-%(brickname)s.fits')
survey = get_survey(tag)
B = survey.get_bricks_readonly()
I = survey.bricks_touching_radec_box(B, r.min(), r.max(), d.min(), d.max())
#print(len(I), 'bricks touching RA,Dec box', r.min(),r.max(), d.min(),d.max())
cat = []
hdr = None
for brickname in B.brickname[I]:
catfn = survey.find_file('tractor', brick=brickname)
if not os.path.exists(catfn):
print('Does not exist:', catfn)
continue
debug('Reading catalog', catfn)
T = fits_table(catfn)
T.cut(T.brick_primary)
print('File', catfn, 'cut to', len(T), 'primary')
ok,xx,yy = wcs.radec2pixelxy(T.ra, T.dec)
T.cut((xx > 0) * (yy > 0) * (xx < W) * (yy < H))
cat.append(T)
if hdr is None:
hdr = T.get_header()
if len(cat) == 0:
cat = None
else:
cat = merge_tables(cat, columns='fillzero')
return cat,hdr
def get_DR5_ccds(bricknames):
path = '/global/cscratch1/sd/desiproc/DR5_out/'
T = []
for brick in bricknames:
bri = brick[:3]
ccd_fn = os.path.join(
path,
'coadd/%s/%s/legacysurvey-%s-ccds.fits' % (bri, brick, brick))
try:
t = fits_table(ccd_fn)
t.set('brickname', np.array([brick] * len(t)))
T.append(t)
#ccd_fns.append(os.path.join(path,
# 'coadd/%s/%s/legacysurvey-%s-ccds.fits' %
# (bri,brick,brickv))
except IOError:
print('not found: %s' % ccd_fn)
TT = merge_tables(T, columns='fillzero')
del T
savefn = 'brick_allccds.fits'
TT.writeto(savefn)
print('Wrote %s' % savefn)
def main_serial(doWhat=None, derived_dir=None, randoms_subset=False):
"""merges the rank tables that are stored in merge_tmp/"""
saveDir = dir_for_serial(derived_dir)
try:
os.makedirs(saveDir)
except OSError:
pass
if doWhat == 'randoms':
wild = "randoms_rank*.fits"
if randoms_subset:
outfn = os.path.join(saveDir, 'randoms_subset.fits')
else:
outfn = os.path.join(saveDir, 'randoms.fits')
elif doWhat == 'summary':
wild = "summary_rank*.fits"
outfn = os.path.join(saveDir, "summary.fits")
if os.path.exists(outfn):
print('Merged table already exists %s' % outfn)
return
search = os.path.join(dir_for_mpi(derived_dir), wild)
tab_fns = glob(search)
if len(tab_fns) == 0:
raise ValueError('found nothing with search: %s' % search)
tabs = []
for fn in tab_fns:
if randoms_subset:
T = fits_table_cols(fn)
else:
T = fits_table(fn)
tabs.append(T)
print('Merging %d tables' % len(tabs))
tab = merge_tables(tabs, columns='fillzero')
tab.writeto(outfn)
print('Wrote %s' % outfn)
print('has %d rows' % len(tab))
def get_annotated_ccds(self):
'''
Returns the annotated table of CCDs.
'''
from glob import glob
fns = self.find_file('annotated-ccds')
TT = []
for fn in fns:
print('Reading annotated CCDs from', fn)
T = fits_table(fn)
print('Got', len(T), 'CCDs')
TT.append(T)
T = merge_tables(TT, columns='fillzero')
print('Total of', len(T), 'CCDs')
del TT
# Remove trailing spaces from 'ccdname' column
if 'ccdname' in T.columns():
# "N4 " -> "N4"
T.ccdname = np.array([s.strip() for s in T.ccdname])
# Remove trailing spaces from 'camera' column.
T.camera = np.array([c.strip() for c in T.camera])
return T
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-o',
'--out',
dest='outfn',
help='Output filename',
default='TMP/nexp.fits')
parser.add_argument('--merge',
action='store_true',
help='Merge sub-tables')
parser.add_argument('--plot', action='store_true', help='Plot results')
parser.add_argument('files',
metavar='nexp-file.fits.gz',
nargs='+',
help='List of nexp files to process')
opt = parser.parse_args()
fns = opt.files
if opt.merge:
from astrometry.util.fits import merge_tables
TT = []
for fn in fns:
T = fits_table(fn)
print(fn, '->', len(T))
TT.append(T)
T = merge_tables(TT)
T.writeto(opt.outfn)
print('Wrote', opt.outfn)
if opt.plot:
T = fits_table(opt.files[0])
import pylab as plt
import matplotlib
ax = [360, 0, -21, 36]
def radec_plot():
plt.axis(ax)
plt.xlabel('RA (deg)')
plt.xticks(np.arange(0, 361, 45))
plt.ylabel('Dec (deg)')
gl = np.arange(361)
gb = np.zeros_like(gl)
from astrometry.util.starutil_numpy import lbtoradec
rr, dd = lbtoradec(gl, gb)
plt.plot(rr, dd, 'k-', alpha=0.5, lw=1)
rr, dd = lbtoradec(gl, gb + 10)
plt.plot(rr, dd, 'k-', alpha=0.25, lw=1)
rr, dd = lbtoradec(gl, gb - 10)
plt.plot(rr, dd, 'k-', alpha=0.25, lw=1)
plt.figure(figsize=(8, 5))
plt.subplots_adjust(left=0.1, right=0.98, top=0.93)
# Map of the tile centers we want to observe...
O = fits_table('obstatus/decam-tiles_obstatus.fits')
O.cut(O.in_desi == 1)
rr, dd = np.meshgrid(np.linspace(ax[1], ax[0], 700),
np.linspace(ax[2], ax[3], 200))
from astrometry.libkd.spherematch import match_radec
I, J, d = match_radec(O.ra, O.dec, rr.ravel(), dd.ravel(), 1.)
desimap = np.zeros(rr.shape, bool)
desimap.flat[J] = True
def desi_map():
# Show the DESI tile map in the background.
from astrometry.util.plotutils import antigray
plt.imshow(desimap,
origin='lower',
interpolation='nearest',
extent=[ax[1], ax[0], ax[2], ax[3]],
aspect='auto',
cmap=antigray,
vmax=8)
for band in 'grz':
plt.clf()
desi_map()
N = T.get('nexp_%s' % band)
I = np.flatnonzero(N > 0)
#cm = matplotlib.cm.get_cmap('jet', 6)
#cm = matplotlib.cm.get_cmap('winter', 5)
cm = matplotlib.cm.viridis
cm = matplotlib.cm.get_cmap(cm, 5)
plt.scatter(T.ra[I],
T.dec[I],
c=N[I],
s=2,
edgecolors='none',
vmin=0.5,
vmax=5.5,
cmap=cm)
radec_plot()
cax = colorbar_axes(plt.gca(), frac=0.06)
plt.colorbar(cax=cax, ticks=range(6))
#plt.colorbar(ticks=range(6))
plt.title('DECaLS DR3: Number of exposures in %s' % band)
plt.savefig('nexp-%s.png' % band)
plt.clf()
desi_map()
plt.scatter(T.ra,
T.dec,
c=T.get('nexp_%s' % band),
s=2,
edgecolors='none',
vmin=0,
vmax=2.)
radec_plot()
plt.colorbar()
plt.title('DECaLS DR3: PSF size, band %s' % band)
plt.savefig('psfsize-%s.png' % band)
return 0
for col in ['nobjs', 'npsf', 'nsimp', 'nexp', 'ndev', 'ncomp']:
plt.clf()
desi_map()
N = T.get(col)
mx = np.percentile(N, 99.5)
plt.scatter(T.ra,
T.dec,
c=N,
s=2,
edgecolors='none',
vmin=0,
vmax=mx)
radec_plot()
plt.colorbar()
plt.title('DECaLS DR3: Number of objects of type %s' % col[1:])
plt.savefig('nobjs-%s.png' % col[1:])
Ntot = T.nobjs
for col in ['npsf', 'nsimp', 'nexp', 'ndev', 'ncomp']:
plt.clf()
desi_map()
N = T.get(col) / Ntot.astype(np.float32)
mx = np.percentile(N, 99.5)
plt.scatter(T.ra,
T.dec,
c=N,
s=2,
edgecolors='none',
vmin=0,
vmax=mx)
radec_plot()
plt.colorbar()
plt.title('DECaLS DR3: Fraction of objects of type %s' % col[1:])
plt.savefig('fobjs-%s.png' % col[1:])
return 0
# fnpats = opt.files
# fns = []
# for pat in fnpats:
# pfns = glob(pat)
# fns.extend(pfns)
# print('Pattern', pat, '->', len(pfns), 'files')
#fns = glob('coadd/*/*/*-nexp*')
#fns = glob('coadd/000/*/*-nexp*')
#fns = glob('coadd/000/0001*/*-nexp*')
fns.sort()
print(len(fns), 'nexp files')
brickset = set()
bricklist = []
gn = []
rn = []
zn = []
gnhist = []
rnhist = []
znhist = []
nnhist = 6
gdepth = []
rdepth = []
zdepth = []
ibricks = []
nsrcs = []
npsf = []
nsimp = []
nexp = []
ndev = []
ncomp = []
gpsfsize = []
rpsfsize = []
zpsfsize = []
ebv = []
gtrans = []
rtrans = []
ztrans = []
bricks = fits_table('survey-bricks.fits.gz')
#sfd = SFDMap()
W = H = 3600
# H=3600
# xx,yy = np.meshgrid(np.arange(W), np.arange(H))
unique = np.ones((H, W), bool)
tlast = 0
for fn in fns:
print('File', fn)
words = fn.split('/')
dirprefix = '/'.join(words[:-4])
print('Directory prefix:', dirprefix)
words = words[-4:]
brick = words[2]
print('Brick', brick)
if not brick in brickset:
brickset.add(brick)
bricklist.append(brick)
gn.append(0)
rn.append(0)
zn.append(0)
gnhist.append([0 for i in range(nnhist)])
rnhist.append([0 for i in range(nnhist)])
znhist.append([0 for i in range(nnhist)])
index = -1
ibrick = np.nonzero(bricks.brickname == brick)[0][0]
ibricks.append(ibrick)
tfn = os.path.join(dirprefix, 'tractor', brick[:3],
'tractor-%s.fits' % brick)
print('Tractor filename', tfn)
T = fits_table(tfn,
columns=[
'brick_primary', 'type', 'decam_psfsize', 'ebv',
'decam_mw_transmission'
])
T.cut(T.brick_primary)
nsrcs.append(len(T))
types = Counter([t.strip() for t in T.type])
npsf.append(types['PSF'])
nsimp.append(types['SIMP'])
nexp.append(types['EXP'])
ndev.append(types['DEV'])
ncomp.append(types['COMP'])
print('N sources', nsrcs[-1])
gpsfsize.append(np.median(T.decam_psfsize[:, 1]))
rpsfsize.append(np.median(T.decam_psfsize[:, 2]))
zpsfsize.append(np.median(T.decam_psfsize[:, 4]))
ebv.append(np.median(T.ebv))
gtrans.append(np.median(T.decam_mw_transmission[:, 1]))
rtrans.append(np.median(T.decam_mw_transmission[:, 2]))
ztrans.append(np.median(T.decam_mw_transmission[:, 4]))
br = bricks[ibrick]
print('Computing unique brick pixels...')
#wcs = Tan(fn, 0)
#W,H = int(wcs.get_width()), int(wcs.get_height())
pixscale = 0.262 / 3600.
wcs = Tan(br.ra, br.dec, W / 2. + 0.5, H / 2. + 0.5, -pixscale, 0.,
0., pixscale, float(W), float(H))
import time
t0 = time.clock()
unique[:, :] = True
find_unique_pixels(wcs, W, H, unique, br.ra1, br.ra2, br.dec1,
br.dec2)
# for i in range(W/2):
# allin = True
# lo,hi = i, W-i-1
# # one slice per side
# side = slice(lo,hi+1)
# top = (lo, side)
# bot = (hi, side)
# left = (side, lo)
# right = (side, hi)
# for slc in [top, bot, left, right]:
# #print('xx,yy', xx[slc], yy[slc])
# rr,dd = wcs.pixelxy2radec(xx[slc]+1, yy[slc]+1)
# U = (rr >= br.ra1 ) * (rr < br.ra2 ) * (dd >= br.dec1) * (dd < br.dec2)
# #print('Pixel', i, ':', np.sum(U), 'of', len(U), 'pixels are unique')
# allin *= np.all(U)
# unique[slc] = U
# if allin:
# print('Scanned to pixel', i)
# break
t1 = time.clock()
U = np.flatnonzero(unique)
t2 = time.clock()
print(len(U), 'of', W * H, 'pixels are unique to this brick')
# #t3 = time.clock()
#rr,dd = wcs.pixelxy2radec(xx+1, yy+1)
# #t4 = time.clock()
# #u = (rr >= br.ra1 ) * (rr < br.ra2 ) * (dd >= br.dec1) * (dd < br.dec2)
# #t5 = time.clock()
# #U2 = np.flatnonzero(u)
#U2 = np.flatnonzero((rr >= br.ra1 ) * (rr < br.ra2 ) *
# (dd >= br.dec1) * (dd < br.dec2))
#assert(np.all(U == U2))
#assert(len(U) == len(U2))
# #t6 = time.clock()
# print(len(U2), 'of', W*H, 'pixels are unique to this brick')
#
#print(t0-tlast, 'other time')
#tlast = time.clock() #t2
#print('t1:', t1-t0, 't2', t2-t1)
# #print('t4:', t4-t3, 't5', t5-t4, 't6', t6-t5)
#
else:
index = bricklist.index(brick)
assert (index == len(bricklist) - 1)
index = bricklist.index(brick)
assert (index == len(bricklist) - 1)
filepart = words[-1]
filepart = filepart.replace('.fits.gz', '')
print('File:', filepart)
band = filepart[-1]
assert (band in 'grz')
nlist, nhist = dict(g=(gn, gnhist), r=(rn, rnhist),
z=(zn, znhist))[band]
upix = fitsio.read(fn).flat[U]
med = np.median(upix)
print('Band', band, ': Median', med)
nlist[index] = med
hist = nhist[index]
for i in range(nnhist):
if i < nnhist - 1:
hist[i] = np.sum(upix == i)
else:
hist[i] = np.sum(upix >= i)
assert (sum(hist) == len(upix))
print('Number of exposures histogram:', hist)
ibricks = np.array(ibricks)
print('Maximum number of sources:', max(nsrcs))
T = fits_table()
T.brickname = np.array(bricklist)
T.ra = bricks.ra[ibricks]
T.dec = bricks.dec[ibricks]
T.nexp_g = np.array(gn).astype(np.int16)
T.nexp_r = np.array(rn).astype(np.int16)
T.nexp_z = np.array(zn).astype(np.int16)
T.nexphist_g = np.array(gnhist).astype(np.int32)
T.nexphist_r = np.array(rnhist).astype(np.int32)
T.nexphist_z = np.array(znhist).astype(np.int32)
T.nobjs = np.array(nsrcs).astype(np.int16)
T.npsf = np.array(npsf).astype(np.int16)
T.nsimp = np.array(nsimp).astype(np.int16)
T.nexp = np.array(nexp).astype(np.int16)
T.ndev = np.array(ndev).astype(np.int16)
T.ncomp = np.array(ncomp).astype(np.int16)
T.psfsize_g = np.array(gpsfsize).astype(np.float32)
T.psfsize_r = np.array(rpsfsize).astype(np.float32)
T.psfsize_z = np.array(zpsfsize).astype(np.float32)
T.ebv = np.array(ebv).astype(np.float32)
T.trans_g = np.array(gtrans).astype(np.float32)
T.trans_r = np.array(rtrans).astype(np.float32)
T.trans_z = np.array(ztrans).astype(np.float32)
T.writeto(opt.outfn)
def main(survey=None, opt=None):
print(' '.join(sys.argv))
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = tlast = Time()
if opt.skip and os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing first arg as exposure number (otherwise, it's a filename)
try:
expnum = int(opt.expnum)
filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
filename = opt.expnum
# Try parsing HDU: "all" or HDU name or HDU number.
all_hdus = (opt.ccdname == 'all')
hdu = -1
ccdname = None
if not all_hdus:
try:
hdu = int(opt.ccdname)
except:
ccdname = opt.ccdname
if survey is None:
survey = LegacySurveyData(survey_dir=opt.survey_dir)
catsurvey_north = survey
catsurvey_south = None
if opt.catalog_dir_north is not None:
assert (opt.catalog_dir_south is not None)
assert (opt.catalog_resolve_dec_ngc is not None)
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)
if opt.catalog_dir is not None:
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)
if filename is not None and hdu >= 0:
# FIXME -- try looking up in CCDs file?
# Read metadata from file
print('Warning: faking metadata from file contents')
T = exposure_metadata([filename], hdus=[hdu])
print('Metadata:')
T.about()
if not 'ccdzpt' in T.columns():
phdr = fitsio.read_header(filename)
T.ccdzpt = np.array([phdr['MAGZERO']])
print('WARNING: using header MAGZERO')
T.ccdraoff = np.array([0.])
T.ccddecoff = np.array([0.])
print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.')
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and ccdname', ccdname)
if hdu >= 0:
T.cut(T.image_hdu == hdu)
print(len(T), 'with HDU', hdu)
if filename is not None:
T.cut(np.array([f.strip() == filename for f in T.image_filename]))
print(len(T), 'with filename', filename)
if opt.camera is not None:
T.cut(T.camera == opt.camera)
print(len(T), 'with camera', opt.camera)
if not all_hdus:
assert (len(T) == 1)
args = []
for ccd in T:
args.append((survey, catsurvey_north, catsurvey_south,
opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))
if opt.threads:
from astrometry.util.multiproc import multiproc
from astrometry.util.timingpool import TimingPool, TimingPoolMeas
pool = TimingPool(opt.threads)
poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
Time.add_measurement(poolmeas)
mp = multiproc(None, pool=pool)
tm = Time()
FF = mp.map(bounce_one_ccd, args)
print('Multi-processing forced-phot:', Time() - tm)
else:
FF = map(bounce_one_ccd, args)
FF = [F for F in FF if F is not None]
if len(FF) == 0:
print('No photometry results to write.')
return 0
# Keep only the first header
_, version_hdr = FF[0]
FF = [F for F, hdr in FF]
F = merge_tables(FF)
if all_hdus:
version_hdr.delete('CPHDU')
version_hdr.delete('CCDNAME')
units = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2',
'apflux': 'nanomaggy',
'apflux_ivar': '1/nanomaggy^2',
'psfdepth': '1/nanomaggy^2',
'galdepth': '1/nanomaggy^2',
'sky': 'nanomaggy/arcsec^2',
'psfsize': 'arcsec'
}
if opt.derivs:
units.update({
'dra': 'arcsec',
'ddec': 'arcsec',
'dra_ivar': '1/arcsec^2',
'ddec_ivar': '1/arcsec^2'
})
columns = F.get_columns()
order = [
'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'ccd_cuts',
'airmass', 'sky', 'psfdepth', 'galdepth', 'ra', 'dec', 'flux',
'flux_ivar', 'fracflux', 'rchisq', 'fracmasked', 'apflux',
'apflux_ivar', 'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar',
'ddec_ivar'
]
columns = [c for c in order if c in columns]
# Set units headers (must happen after column ordering is set!)
hdr = fitsio.FITSHDR()
for i, col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.outfn))
fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
F.writeto(tmpfn, header=hdr, append=True, columns=columns)
os.rename(tmpfn, opt.outfn)
print('Wrote', opt.outfn)
tnow = Time()
print('Total:', tnow - t0)
return 0
meas = measure_raw_mosaic3(fn, ext=extstring, n_fwhm=1)
#print('Measurement:', meas.keys())
M = fits_table()
for k in [
'airmass', 'extension', 'pixscale', 'nmatched', 'ra_ccd',
'dec_ccd', 'band', 'zp', 'rawsky', 'ndetected',
'skybright', 'dy', 'transparency', 'seeing', 'dx',
'exptime', 'zp_skysub', 'zp_med', 'zp_med_skysub', 'affine'
]:
M.set(k, np.array([meas[k]]))
M.filename = np.array([fn])
M.extname = np.array([extstring])
phdr = meas['primhdr']
M.expnum = np.array([phdr['EXPNUM']])
MM.append(M)
M = merge_tables(MM)
M.writeto(cofn)
C = fits_table(cofn)
from camera_mosaic import nominal_cal
nom = nominal_cal
C.extension = np.array([ext.strip() for ext in C.extension])
CDs = dict([(ext, nom.cdmatrix(ext)) for ext in np.unique(C.extension)])
C.cd = np.array([CDs[ext] for ext in C.extension])
C.dra = (C.cd[:, 0] * C.dx + C.cd[:, 1] * C.dy) * 3600.
C.ddec = (C.cd[:, 2] * C.dx + C.cd[:, 3] * C.dy) * 3600.
if not 'expnum' in C.columns():
def brick_cat_from_overlap(self,
surveyObj,
margin=10.,
purge_duplicates=True):
'''
Return a catalog of sources within bricks touching the overlap region
between the BASS and DECaLS CCDs. Does not cut sources to the overlap
region.
If purge_duplicates=True then duplicates (i.e. common sources from
overlapping bricks) are removed based on depth.
'''
brks = self.bricks_in_overlap(surveyObj, margin=margin)
if not len(brks):
return None
tractorDir = surveyObj.find_file('tractor').split('/')
tractorDir = '/'.join(tractorDir[0:-2])
tabs = []
# merge brick catalogs
for brk in brks:
path = os.path.join(tractorDir, brk.brickname[0:3],
'tractor-' + brk.brickname + '.fits')
tabs.append(fits_table(path))
if len(tabs) == 1: return tabs[0]
tab = merge_tables(tabs)
if not (purge_duplicates):
return tab
# find groups of connected sources within a small radius
# this is obviously not an exact method!
grps = cluster_radec(tab.ra, tab.dec, 0.3 / 3600.)
keep = np.ones(len(tab), dtype=bool)
for grp in grps:
brknms = tab.brickname[grp]
# TODO a small number of srcs in the same brick will be
# selected as duplicates!
if len(grp) > 2:
# if there are more than 2 sources in a group then call
# the closest 2 duplicates
ra = tab.ra[grp]
dec = tab.dec[grp]
m1, m2, d12 = match_radec(ra,
dec,
ra,
dec,
0.3 / 3600.,
notself=True,
nearest=True)
i = np.argmin(d12)
inxa = m1[i]
inxb = m2[i]
# chuck the one with larger ivar
try:
dpth = np.array([
tab.get('psfdepth_' + self.filter)[inxa],
tab.get('psfdepth_' + self.filter)[inxb]
])
except KeyError:
b = 'ugrizY'.find(self.filter)
dpth = np.array([
tab.get('decam_depth')[inxa, b],
tab.get('decam_depth')[inxb, b]
])
j = np.argmin(dpth)
keep[grp[np.array([inxa, inxb])[j]]] = False
else:
try:
dpth = tab.get('psfdepth_' + self.filter)[grp]
except KeyError:
b = 'ugrizY'.find(self.filter)
dpth = np.array([
tab.get('decam_depth')[grp[0], b],
tab.get('decam_depth')[grp[1], b]
])
j = np.argmin(dpth)
keep[grp[j]] = False
if np.any(~keep):
return tab[keep]
return tab
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--name1', help='Name for first data set')
parser.add_argument('--name2', help='Name for second data set')
parser.add_argument('--plot-prefix',
default='compare',
help='Prefix for plot filenames; default "%default"')
parser.add_argument('--match',
default=1.0,
help='Astrometric cross-match distance in arcsec')
parser.add_argument('dir1', help='First directory to compare')
parser.add_argument('dir2', help='Second directory to compare')
opt = parser.parse_args()
ps = PlotSequence(opt.plot_prefix)
name1 = opt.name1
if name1 is None:
name1 = os.path.basename(opt.dir1)
if not len(name1):
name1 = os.path.basename(os.path.dirname(opt.dir1))
name2 = opt.name2
if name2 is None:
name2 = os.path.basename(opt.dir2)
if not len(name2):
name2 = os.path.basename(os.path.dirname(opt.dir2))
tt = 'Comparing %s to %s' % (name1, name2)
# regex for tractor-*.fits catalog filename
catre = re.compile('tractor-.*.fits')
cat1, cat2 = [], []
for basedir, cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
for dirpath, dirnames, filenames in os.walk(basedir, followlinks=True):
for fn in filenames:
if not catre.match(fn):
print('Skipping', fn, 'due to filename')
continue
fn = os.path.join(dirpath, fn)
t = fits_table(fn)
print(len(t), 'from', fn)
cat.append(t)
cat1 = merge_tables(cat1, columns='fillzero')
cat2 = merge_tables(cat2, columns='fillzero')
print('Total of', len(cat1), 'from', name1)
print('Total of', len(cat2), 'from', name2)
cat1.cut(cat1.brick_primary)
cat2.cut(cat2.brick_primary)
print('Total of', len(cat1), 'BRICK_PRIMARY from', name1)
print('Total of', len(cat2), 'BRICK_PRIMARY from', name2)
cat1.cut((cat1.decam_anymask[:, 1] == 0) *
(cat1.decam_anymask[:, 2] == 0) * (cat1.decam_anymask[:, 4] == 0))
cat2.cut((cat2.decam_anymask[:, 1] == 0) *
(cat2.decam_anymask[:, 2] == 0) * (cat2.decam_anymask[:, 4] == 0))
print('Total of', len(cat1), 'unmasked from', name1)
print('Total of', len(cat2), 'unmasked from', name2)
I, J, d = match_radec(cat1.ra,
cat1.dec,
cat2.ra,
cat2.dec,
opt.match / 3600.,
nearest=True)
print(len(I), 'matched')
plt.clf()
plt.hist(d * 3600., 100)
plt.xlabel('Match distance (arcsec)')
plt.title(tt)
ps.savefig()
matched1 = cat1[I]
matched2 = cat2[J]
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
K = np.flatnonzero((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0))
print('Median mw_trans', band, 'is',
np.median(matched1.decam_mw_transmission[:, iband]))
plt.clf()
plt.errorbar(
matched1.decam_flux[K, iband],
matched2.decam_flux[K, iband],
fmt='.',
color=cc,
xerr=1. / np.sqrt(matched1.decam_flux_ivar[K, iband]),
yerr=1. / np.sqrt(matched2.decam_flux_ivar[K, iband]),
alpha=0.1,
)
plt.xlabel('%s flux: %s' % (name1, band))
plt.ylabel('%s flux: %s' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
plt.axis([-100, 1000, -100, 1000])
plt.title(tt)
ps.savefig()
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
good = ((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0))
K = np.flatnonzero(good)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
P = np.flatnonzero(good * psf1 * psf2)
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1. / iv1 + 1. / iv2)
plt.clf()
plt.plot(
mag1[K],
(matched2.decam_flux[K, iband] - matched1.decam_flux[K, iband]) /
std[K],
'.',
alpha=0.1,
color=cc)
plt.plot(
mag1[P],
(matched2.decam_flux[P, iband] - matched1.decam_flux[P, iband]) /
std[P],
'.',
alpha=0.1,
color='k')
plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' %
(name2, name1, band))
plt.xlabel('%s mag: %s' % (name1, band))
plt.axhline(0, color='k', alpha=0.5)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
plt.clf()
lp, lt = [], []
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
good = ((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0))
#good = True
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1. / iv1 + 1. / iv2)
#std = np.hypot(std, 0.01)
G = np.flatnonzero(good * psf1 * psf2 * np.isfinite(mag1) *
(mag1 >= 20) *
(mag1 < dict(g=24, r=23.5, z=22.5)[band]))
n, b, p = plt.hist(
(matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) /
std[G],
range=(-4, 4),
bins=50,
histtype='step',
color=cc,
normed=True)
sig = (matched2.decam_flux[G, iband] -
matched1.decam_flux[G, iband]) / std[G]
print('Raw mean and std of points:', np.mean(sig), np.std(sig))
med = np.median(sig)
rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.
print('Median and percentile-based sigma:', med, rsigma)
lp.append(p[0])
lt.append('%s: %.2f +- %.2f' % (band, med, rsigma))
bins = []
gaussint = []
for blo, bhi in zip(b, b[1:]):
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
#bins.extend([blo,bhi])
#gaussint.extend([c,c])
bins.append((blo + bhi) / 2.)
gaussint.append(c)
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.title(tt)
plt.xlabel('Flux difference / error (sigma)')
plt.axvline(0, color='k', alpha=0.1)
plt.ylim(0, 0.45)
plt.legend(lp, lt, loc='upper right')
ps.savefig()
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
plt.clf()
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
matched2.decam_flux[:, iband], matched2.decam_flux_ivar[:, iband])
meanmag = NanoMaggies.nanomaggiesToMag(
(matched1.decam_flux[:, iband] + matched2.decam_flux[:, iband]) /
2.)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
good = ((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0) * np.isfinite(mag1) *
np.isfinite(mag2))
K = np.flatnonzero(good)
P = np.flatnonzero(good * psf1 * psf2)
plt.errorbar(mag1[K],
mag2[K],
fmt='.',
color=cc,
xerr=magerr1[K],
yerr=magerr2[K],
alpha=0.1)
plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s (mag)' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
plt.axis([24, 16, 24, 16])
plt.title(tt)
ps.savefig()
plt.clf()
plt.errorbar(mag1[K],
mag2[K] - mag1[K],
fmt='.',
color=cc,
xerr=magerr1[K],
yerr=magerr2[K],
alpha=0.1)
plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -1, 1])
plt.title(tt)
ps.savefig()
magbins = np.arange(16, 24.001, 0.5)
plt.clf()
plt.plot(mag1[K],
(mag2[K] - mag1[K]) / np.hypot(magerr1[K], magerr2[K]),
'.',
color=cc,
alpha=0.1)
plt.plot(mag1[P],
(mag2[P] - mag1[P]) / np.hypot(magerr1[P], magerr2[P]),
'k.',
alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
plt.clf()
plt.plot(meanmag[P], y[P], 'k.', alpha=0.1)
midmag = []
vals = np.zeros((len(magbins) - 1, 5))
median_err1 = []
iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)
# FIXME -- should we do some stats after taking off the mean difference?
for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
midmag.append((mlo + mhi) / 2.)
median_err1.append(np.median(magerr1[I]))
if len(I) == 0:
continue
# median and +- 1 sigma quantiles
ybin = y[I]
vals[bini, 0] = np.percentile(ybin, 16)
vals[bini, 1] = np.median(ybin)
vals[bini, 2] = np.percentile(ybin, 84)
# +- 2 sigma quantiles
vals[bini, 3] = np.percentile(ybin, 2.3)
vals[bini, 4] = np.percentile(ybin, 97.7)
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss,
'vs expected for Gaussian;', len(ybin), 'points')
# if iqd > iqd_gauss:
# # What error adding in quadrature would you need to make the IQD match?
# err = median_err1[-1]
# target_err = err * (iqd / iqd_gauss)
# sys_err = np.sqrt(target_err**2 - err**2)
# print('--> add systematic error', sys_err)
# ~ Johan's cuts
mlo = 21.
mhi = dict(g=24., r=23.5, z=22.5)[band]
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
ybin = y[I]
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', mlo, mhi, 'band', band,
': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;',
len(ybin), 'points')
if iqd > iqd_gauss:
# What error adding in quadrature would you need to make
# the IQD match?
err = np.median(np.hypot(magerr1[I], magerr2[I]))
print('Median error (hypot):', err)
target_err = err * (iqd / iqd_gauss)
print('Target:', target_err)
sys_err = np.sqrt((target_err**2 - err**2) / 2.)
print('--> add systematic error', sys_err)
# check...
err_sys = np.hypot(np.hypot(magerr1, sys_err),
np.hypot(magerr2, sys_err))
ysys = (mag2 - mag1) / err_sys
ysys = ysys[I]
print('Resulting median error:', np.median(err_sys[I]))
iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian')
# Hmmm, this doesn't work... totally overshoots.
plt.errorbar(midmag,
vals[:, 1],
fmt='o',
color='b',
yerr=(vals[:, 1] - vals[:, 0], vals[:, 2] - vals[:, 1]),
capthick=3,
zorder=20)
plt.errorbar(midmag,
vals[:, 1],
fmt='o',
color='b',
yerr=(vals[:, 1] - vals[:, 3], vals[:, 4] - vals[:, 1]),
capthick=2,
zorder=20)
plt.axhline(1., color='b', alpha=0.2)
plt.axhline(-1., color='b', alpha=0.2)
plt.axhline(2., color='b', alpha=0.2)
plt.axhline(-2., color='b', alpha=0.2)
for mag, err, y in zip(midmag, median_err1, vals[:, 3]):
if not np.isfinite(err):
continue
if y < -6:
continue
plt.text(mag,
y - 0.1,
'%.3f' % err,
va='top',
ha='center',
color='k',
fontsize=10)
plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axvline(21, color='k', alpha=0.3)
plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3)
plt.axis([24.1, 16, -6, 6])
plt.title(tt)
ps.savefig()
#magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
if band == 'g':
magbins = [20, 24]
elif band == 'r':
magbins = [20, 23.5]
elif band == 'z':
magbins = [20, 22.5]
slo, shi = -5, 5
plt.clf()
ha = dict(bins=25, range=(slo, shi), histtype='step', normed=True)
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
midmag = []
nn = []
rgbs = []
lt, lp = [], []
for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
if len(I) == 0:
continue
ybin = y[I]
rgb = [0., 0., 0.]
rgb[0] = float(bini) / (len(magbins) - 1)
rgb[2] = 1. - rgb[0]
n, b, p = plt.hist(ybin, color=rgb, **ha)
lt.append('mag %g to %g' % (mlo, mhi))
lp.append(p[0])
midmag.append((mlo + mhi) / 2.)
nn.append(n)
rgbs.append(rgb)
bins = []
gaussint = []
for blo, bhi in zip(b, b[1:]):
#midbin.append((blo+bhi)/2.)
#gaussint.append(scipy.stats.norm.cdf(bhi) -
# scipy.stats.norm.cdf(blo))
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
bins.extend([blo, bhi])
gaussint.extend([c, c])
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo, shi)
ps.savefig()
bincenters = b[:-1] + (b[1] - b[0]) / 2.
plt.clf()
lp = []
for n, rgb, mlo, mhi in zip(nn, rgbs, magbins, magbins[1:]):
p = plt.plot(bincenters, n, '-', color=rgb)
lp.append(p[0])
plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo, shi)
ps.savefig()
def main():
# one_file('/global/cfs/cdirs/cosmo/staging/decam/CP/V4.8.2a/CP20190719/c4d_190720_102549_oow_r_v1.fits.fz')
# one_file('/global/cfs/cdirs/cosmo/staging/decam/CP/V4.8.2a/CP20190719/c4d_190720_100244_oow_g_v1.fits.fz')
# one_file('/global/cfs/cdirs/cosmo/staging/decam/CP/V4.8.2a/CP20190719/c4d_190720_100433_oow_g_v1.fits.fz')
# one_file('/global/cfs/cdirs/cosmo/staging/decam/CP/V4.8.2a/CP20190719/c4d_190720_102401_oow_g_v1.fits.fz')
# one_file('/global/cfs/cdirs/cosmo/staging/decam/CP/V4.8.2a/CP20190719/c4d_190720_100622_oow_r_v1.fits.fz')
# one_file('/global/cfs/cdirs/cosmo/staging/decam/CP/V4.8.2a/CP20190719/c4d_190720_100802_oow_r_v1.fits.fz')
# one_file('/global/cfs/cdirs/cosmo/staging/decam/CP/V4.8.2a/CP20190719/c4d_190720_102727_oow_r_v1.fits.fz')
# one_file('/global/cfs/cdirs/cosmo/staging/decam/CP/V4.8.2a/CP20190719/c4d_190720_102212_oow_g_v1.fits.fz')
# return
if True:
#dirs = glob(dirprefix + '90prime/CP/V2.3/CP*')
dirs = glob(dirprefix + 'mosaic/CP/V4.3/CP*')
#dirs = glob(dirprefix + 'mosaic/CP/V4.3/CP2017*')
#dirs = (glob(dirprefix + 'mosaic/CP/V4.3/CP2015*') +
# glob(dirprefix + 'mosaic/CP/V4.3/CP2016*'))
#dirs = glob(dirprefix + 'decam/CP/V4.8.2a/CP*')
dirs.sort()
dirs = list(reversed(dirs))
keepdirs = []
for dirnm in dirs:
outfn = 'oow-stats-' + '-'.join(dirnm.split('/')[-4:]) + '.fits'
if os.path.exists(outfn):
print('skipping', outfn)
continue
keepdirs.append(dirnm)
dirs = keepdirs
print('Directories to run:')
for dirnm in dirs:
print(' ', dirnm)
# import argparse
# parser = argparse.ArgumentParser()
# parser.add_argument('dirs', nargs='+',
# help='Directories to process')
# args = parser.parse_args()
#
# dirs = args.dirs
# print('Dirs:', dirs)
mp = multiproc(32)
#mp = multiproc(1)
for dirnm in dirs:
print('Dir', dirnm)
outfn = 'oow-stats-' + '-'.join(dirnm.split('/')[-4:]) + '.fits'
print('looking for', outfn)
if os.path.exists(outfn):
print('skipping', outfn)
continue
pat = os.path.join(dirnm, '*_oow_*.fits.fz')
print('Pattern', pat)
fns = glob(pat)
fns.sort()
print(len(fns), 'oow files')
TT = mp.map(one_file, fns)
T = merge_tables(TT)
T.writeto(outfn)
print('Wrote', outfn)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--name1', help='Name for first data set')
parser.add_argument('--name2', help='Name for second data set')
parser.add_argument('--plot-prefix', default='compare',
help='Prefix for plot filenames; default "%default"')
parser.add_argument('--match', default=1.0,
help='Astrometric cross-match distance in arcsec')
parser.add_argument('dir1', help='First directory to compare')
parser.add_argument('dir2', help='Second directory to compare')
opt = parser.parse_args()
ps = PlotSequence(opt.plot_prefix)
name1 = opt.name1
if name1 is None:
name1 = os.path.basename(opt.dir1)
if not len(name1):
name1 = os.path.basename(os.path.dirname(opt.dir1))
name2 = opt.name2
if name2 is None:
name2 = os.path.basename(opt.dir2)
if not len(name2):
name2 = os.path.basename(os.path.dirname(opt.dir2))
tt = 'Comparing %s to %s' % (name1, name2)
# regex for tractor-*.fits catalog filename
catre = re.compile('tractor-.*.fits')
cat1,cat2 = [],[]
for basedir,cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
for dirpath,dirnames,filenames in os.walk(basedir, followlinks=True):
for fn in filenames:
if not catre.match(fn):
print('Skipping', fn, 'due to filename')
continue
fn = os.path.join(dirpath, fn)
t = fits_table(fn)
print(len(t), 'from', fn)
cat.append(t)
cat1 = merge_tables(cat1, columns='fillzero')
cat2 = merge_tables(cat2, columns='fillzero')
print('Total of', len(cat1), 'from', name1)
print('Total of', len(cat2), 'from', name2)
cat1.cut(cat1.brick_primary)
cat2.cut(cat2.brick_primary)
print('Total of', len(cat1), 'BRICK_PRIMARY from', name1)
print('Total of', len(cat2), 'BRICK_PRIMARY from', name2)
cat1.cut((cat1.decam_anymask[:,1] == 0) *
(cat1.decam_anymask[:,2] == 0) *
(cat1.decam_anymask[:,4] == 0))
cat2.cut((cat2.decam_anymask[:,1] == 0) *
(cat2.decam_anymask[:,2] == 0) *
(cat2.decam_anymask[:,4] == 0))
print('Total of', len(cat1), 'unmasked from', name1)
print('Total of', len(cat2), 'unmasked from', name2)
I,J,d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match/3600.,
nearest=True)
print(len(I), 'matched')
plt.clf()
plt.hist(d * 3600., 100)
plt.xlabel('Match distance (arcsec)')
plt.title(tt)
ps.savefig()
matched1 = cat1[I]
matched2 = cat2[J]
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
K = np.flatnonzero((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0))
print('Median mw_trans', band, 'is',
np.median(matched1.decam_mw_transmission[:,iband]))
plt.clf()
plt.errorbar(matched1.decam_flux[K,iband],
matched2.decam_flux[K,iband],
fmt='.', color=cc,
xerr=1./np.sqrt(matched1.decam_flux_ivar[K,iband]),
yerr=1./np.sqrt(matched2.decam_flux_ivar[K,iband]),
alpha=0.1,
)
plt.xlabel('%s flux: %s' % (name1, band))
plt.ylabel('%s flux: %s' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.)
plt.axis([-100, 1000, -100, 1000])
plt.title(tt)
ps.savefig()
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
good = ((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0))
K = np.flatnonzero(good)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
P = np.flatnonzero(good * psf1 * psf2)
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1./iv1 + 1./iv2)
plt.clf()
plt.plot(mag1[K],
(matched2.decam_flux[K,iband] - matched1.decam_flux[K,iband]) / std[K],
'.', alpha=0.1, color=cc)
plt.plot(mag1[P],
(matched2.decam_flux[P,iband] - matched1.decam_flux[P,iband]) / std[P],
'.', alpha=0.1, color='k')
plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' % (name2, name1, band))
plt.xlabel('%s mag: %s' % (name1, band))
plt.axhline(0, color='k', alpha=0.5)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
plt.clf()
lp,lt = [],[]
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
good = ((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0))
#good = True
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1./iv1 + 1./iv2)
#std = np.hypot(std, 0.01)
G = np.flatnonzero(good * psf1 * psf2 *
np.isfinite(mag1) *
(mag1 >= 20) * (mag1 < dict(g=24, r=23.5, z=22.5)[band]))
n,b,p = plt.hist((matched2.decam_flux[G,iband] -
matched1.decam_flux[G,iband]) / std[G],
range=(-4, 4), bins=50, histtype='step', color=cc,
normed=True)
sig = (matched2.decam_flux[G,iband] -
matched1.decam_flux[G,iband]) / std[G]
print('Raw mean and std of points:', np.mean(sig), np.std(sig))
med = np.median(sig)
rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.
print('Median and percentile-based sigma:', med, rsigma)
lp.append(p[0])
lt.append('%s: %.2f +- %.2f' % (band, med, rsigma))
bins = []
gaussint = []
for blo,bhi in zip(b, b[1:]):
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
#bins.extend([blo,bhi])
#gaussint.extend([c,c])
bins.append((blo+bhi)/2.)
gaussint.append(c)
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.title(tt)
plt.xlabel('Flux difference / error (sigma)')
plt.axvline(0, color='k', alpha=0.1)
plt.ylim(0, 0.45)
plt.legend(lp, lt, loc='upper right')
ps.savefig()
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
plt.clf()
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
matched2.decam_flux[:,iband], matched2.decam_flux_ivar[:,iband])
meanmag = NanoMaggies.nanomaggiesToMag((
matched1.decam_flux[:,iband] + matched2.decam_flux[:,iband]) / 2.)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
good = ((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0) *
np.isfinite(mag1) * np.isfinite(mag2))
K = np.flatnonzero(good)
P = np.flatnonzero(good * psf1 * psf2)
plt.errorbar(mag1[K], mag2[K], fmt='.', color=cc,
xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s (mag)' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.)
plt.axis([24, 16, 24, 16])
plt.title(tt)
ps.savefig()
plt.clf()
plt.errorbar(mag1[K], mag2[K] - mag1[K], fmt='.', color=cc,
xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -1, 1])
plt.title(tt)
ps.savefig()
magbins = np.arange(16, 24.001, 0.5)
plt.clf()
plt.plot(mag1[K], (mag2[K]-mag1[K]) / np.hypot(magerr1[K], magerr2[K]),
'.', color=cc, alpha=0.1)
plt.plot(mag1[P], (mag2[P]-mag1[P]) / np.hypot(magerr1[P], magerr2[P]),
'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
plt.clf()
plt.plot(meanmag[P], y[P], 'k.', alpha=0.1)
midmag = []
vals = np.zeros((len(magbins)-1, 5))
median_err1 = []
iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)
# FIXME -- should we do some stats after taking off the mean difference?
for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
midmag.append((mlo+mhi)/2.)
median_err1.append(np.median(magerr1[I]))
if len(I) == 0:
continue
# median and +- 1 sigma quantiles
ybin = y[I]
vals[bini,0] = np.percentile(ybin, 16)
vals[bini,1] = np.median(ybin)
vals[bini,2] = np.percentile(ybin, 84)
# +- 2 sigma quantiles
vals[bini,3] = np.percentile(ybin, 2.3)
vals[bini,4] = np.percentile(ybin, 97.7)
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss,
'vs expected for Gaussian;', len(ybin), 'points')
# if iqd > iqd_gauss:
# # What error adding in quadrature would you need to make the IQD match?
# err = median_err1[-1]
# target_err = err * (iqd / iqd_gauss)
# sys_err = np.sqrt(target_err**2 - err**2)
# print('--> add systematic error', sys_err)
# ~ Johan's cuts
mlo = 21.
mhi = dict(g=24., r=23.5, z=22.5)[band]
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
ybin = y[I]
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', mlo, mhi, 'band', band, ': IQD is factor',
iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points')
if iqd > iqd_gauss:
# What error adding in quadrature would you need to make
# the IQD match?
err = np.median(np.hypot(magerr1[I], magerr2[I]))
print('Median error (hypot):', err)
target_err = err * (iqd / iqd_gauss)
print('Target:', target_err)
sys_err = np.sqrt((target_err**2 - err**2) / 2.)
print('--> add systematic error', sys_err)
# check...
err_sys = np.hypot(np.hypot(magerr1, sys_err),
np.hypot(magerr2, sys_err))
ysys = (mag2 - mag1) / err_sys
ysys = ysys[I]
print('Resulting median error:', np.median(err_sys[I]))
iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian')
# Hmmm, this doesn't work... totally overshoots.
plt.errorbar(midmag, vals[:,1], fmt='o', color='b',
yerr=(vals[:,1]-vals[:,0], vals[:,2]-vals[:,1]),
capthick=3, zorder=20)
plt.errorbar(midmag, vals[:,1], fmt='o', color='b',
yerr=(vals[:,1]-vals[:,3], vals[:,4]-vals[:,1]),
capthick=2, zorder=20)
plt.axhline( 1., color='b', alpha=0.2)
plt.axhline(-1., color='b', alpha=0.2)
plt.axhline( 2., color='b', alpha=0.2)
plt.axhline(-2., color='b', alpha=0.2)
for mag,err,y in zip(midmag, median_err1, vals[:,3]):
if not np.isfinite(err):
continue
if y < -6:
continue
plt.text(mag, y-0.1, '%.3f' % err, va='top', ha='center', color='k',
fontsize=10)
plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axvline(21, color='k', alpha=0.3)
plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3)
plt.axis([24.1, 16, -6, 6])
plt.title(tt)
ps.savefig()
#magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
if band == 'g':
magbins = [20, 24]
elif band == 'r':
magbins = [20, 23.5]
elif band == 'z':
magbins = [20, 22.5]
slo,shi = -5,5
plt.clf()
ha = dict(bins=25, range=(slo,shi), histtype='step', normed=True)
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
midmag = []
nn = []
rgbs = []
lt,lp = [],[]
for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
if len(I) == 0:
continue
ybin = y[I]
rgb = [0.,0.,0.]
rgb[0] = float(bini) / (len(magbins)-1)
rgb[2] = 1. - rgb[0]
n,b,p = plt.hist(ybin, color=rgb, **ha)
lt.append('mag %g to %g' % (mlo,mhi))
lp.append(p[0])
midmag.append((mlo+mhi)/2.)
nn.append(n)
rgbs.append(rgb)
bins = []
gaussint = []
for blo,bhi in zip(b, b[1:]):
#midbin.append((blo+bhi)/2.)
#gaussint.append(scipy.stats.norm.cdf(bhi) -
# scipy.stats.norm.cdf(blo))
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
bins.extend([blo,bhi])
gaussint.extend([c,c])
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo,shi)
ps.savefig()
bincenters = b[:-1] + (b[1]-b[0])/2.
plt.clf()
lp = []
for n,rgb,mlo,mhi in zip(nn, rgbs, magbins, magbins[1:]):
p = plt.plot(bincenters, n, '-', color=rgb)
lp.append(p[0])
plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo,shi)
ps.savefig()
def compare_mags(TT, name, ps):
for i,T in enumerate(TT):
T.set('exp', np.zeros(len(T), np.uint8)+i)
plt.clf()
ap = 5
for i,T in enumerate(TT):
cc = 'rgb'[i]
plt.plot(T.flux, T.apflux[:, ap] / T.flux,
'.', color=cc, alpha=0.5)
ff, frac = [],[]
mags = np.arange(14, 24)
for mlo,mhi in zip(mags, mags[1:]):
flo = NanoMaggies.magToNanomaggies(mhi)
fhi = NanoMaggies.magToNanomaggies(mlo)
I = np.flatnonzero((T.flux > flo) * (T.flux <= fhi))
ff.append(np.sqrt(flo * fhi))
frac.append(np.median(T.apflux[I,ap] / T.flux[I]))
plt.plot(ff, frac, 'o-', color=cc)
plt.xscale('symlog')
plt.xlim(1., 1e3)
plt.ylim(0.9, 1.1)
plt.xlabel('Forced-phot flux')
plt.ylabel('Aperture / Forced-phot flux')
plt.axhline(1, color='k', alpha=0.1)
plt.title('%s region: Aperture %i fluxes' % (name, ap))
ps.savefig()
T = merge_tables(TT)
T.bobjid = T.brickid.astype(int) * 10000 + T.objid
bomap = {}
for i,bo in enumerate(T.bobjid):
try:
bomap[bo].append(i)
except KeyError:
bomap[bo] = [i]
II = []
for bo,ii in bomap.items():
if len(ii) != 3:
continue
II.append(ii)
II = np.array(II)
print 'II', II.shape
exps = T.exp[II]
print 'exposures:', exps
assert(np.all(T.exp[II[:,0]] == 0))
assert(np.all(T.exp[II[:,1]] == 1))
assert(np.all(T.exp[II[:,2]] == 2))
fluxes = T.flux[II]
print 'fluxes', fluxes.shape
meanflux = np.mean(fluxes, axis=1)
print 'meanfluxes', meanflux.shape
plt.clf()
for i in range(3):
plt.plot(meanflux, fluxes[:,i] / meanflux, '.',
color='rgb'[i], alpha=0.5)
#plt.yscale('symlog')
plt.xscale('symlog')
plt.xlabel('Mean flux (nanomaggies)')
plt.ylabel('Forced-phot flux / Mean')
#plt.ylim(0, 2)
plt.ylim(0.9, 1.1)
plt.xlim(0, 1e3)
plt.axhline(1, color='k', alpha=0.1)
plt.title('%s region: Forced-phot fluxes' % name)
ps.savefig()
for ap in [4,5,6]:
apfluxes = T.apflux[:,ap][II,]
print 'ap fluxes', apfluxes.shape
plt.clf()
for i in range(3):
plt.plot(meanflux, apfluxes[:,i] / meanflux, '.',
color='rgb'[i], alpha=0.5)
plt.xscale('symlog')
plt.xlabel('Mean flux (nanomaggies)')
plt.ylabel('Aperture(%i) flux / Mean' % ap)
plt.ylim(0.9, 1.1)
plt.xlim(0, 1e3)
plt.axhline(1, color='k', alpha=0.1)
plt.title('%s region: Aperture %i fluxes' % (name, ap))
ps.savefig()
plt.clf()
for i in range(3):
plt.plot(fluxes[:,i], apfluxes[:,i] / fluxes[:,i], '.',
color='rgb'[i], alpha=0.5)
plt.xscale('symlog')
plt.xlim(0, 1e3)
plt.ylim(0.9, 1.1)
plt.xlabel('Forced-phot flux')
plt.ylabel('Aperture / Forced-phot flux')
plt.axhline(1, color='k', alpha=0.1)
plt.title('%s region: Aperture %i fluxes' % (name, ap))
ps.savefig()
# print('Reading WCS from', flt2)
# wcs2 = Sip(flt2, 0)
# wcs2.ensure_inverse_polynomials()
print('T1 X,Y ranges', T1.x.min(), T1.x.max(), T1.y.min(), T1.y.max())
print('T2 X,Y ranges', T2.x.min(), T2.x.max(), T2.y.min(), T2.y.max())
# ~ 1e-6, 0.0006
# ok,x,y = wcs2.radec2pixelxy(T2.ra, T2.dec)
# print('Scatter wcs x vs catalog x:', np.mean(x - T2.x), np.std(x - T2.x))
# print('Scatter wcs y vs catalog y:', np.mean(y - T2.y), np.std(y - T2.y))
ok, x, y = wcs1.radec2pixelxy(T2.ra, T2.dec)
print('Converted X,Y ranges:', x.min(), x.max(), y.min(), y.max())
T2.x = x
T2.y = y
TT = merge_tables([T1, T2])
TT.writeto(gstout)
print('Wrote', gstout)
hdr = pyfits.open(flt1)[0].header
hdr['IMAGEW'] = 4096
hdr['IMAGEH'] = 4096
pyfits.writeto(fltout, None, header=hdr, clobber=True)
print('Wrote', fltout)
#cmd = 'cp "%s" "%s"' % (flt1, fltout)
#print(cmd)
#os.system(cmd)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build-sample', action='store_true', help='Build the sample.')
parser.add_argument('--jpg-cutouts', action='store_true', help='Get jpg cutouts from the viewer.')
parser.add_argument('--ccd-cutouts', action='store_true', help='Get CCD cutouts of each galaxy.')
parser.add_argument('--runbrick', action='store_true', help='Run the pipeline.')
parser.add_argument('--build-webpage', action='store_true', help='(Re)build the web content.')
args = parser.parse_args()
# Top-level directory
key = 'LEGACY_SURVEY_LARGE_GALAXIES'
if key not in os.environ:
print('Required ${} environment variable not set'.format(key))
return 0
largedir = os.getenv(key)
samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')
# --------------------------------------------------
# Build the sample of large galaxies based on the available imaging.
if args.build_sample:
# Read the parent catalog.
cat = read_rc3()
# Create a simple WCS object for each object and find all the CCDs
# touching that WCS footprint.
survey = LegacySurveyData(version='dr2') # hack!
allccds = survey.get_ccds()
keep = np.concatenate((survey.apply_blacklist(allccds),
survey.photometric_ccds(allccds)))
allccds.cut(keep)
ccdlist = []
outcat = []
for gal in cat:
galwcs = _simplewcs(gal)
ccds1 = allccds[ccds_touching_wcs(galwcs, allccds)]
ccds1 = ccds1[_uniqccds(ccds1)]
if len(ccds1) > 0 and 'g' in ccds1.filter and 'r' in ccds1.filter and 'z' in ccds1.filter:
print('Found {} CCDs for {}, D(25)={:.4f}'.format(
len(ccds1), gal['GALAXY'], gal['RADIUS']))
ccdsfile = os.path.join(largedir, 'ccds', '{}-ccds.fits'.format(gal['GALAXY'].strip().lower()))
print(' Writing {}'.format(ccdsfile))
if os.path.isfile(ccdsfile):
os.remove(ccdsfile)
ccds1.writeto(ccdsfile)
ccdlist.append(ccds1)
if len(outcat) == 0:
outcat = gal
else:
outcat = vstack((outcat, gal))
#if gal['GALAXY'] == 'MCG5-19-36':
# pdb.set_trace()
# Write out the final catalog.
samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')
if os.path.isfile(samplefile):
os.remove(samplefile)
print('Writing {}'.format(samplefile))
outcat.write(samplefile)
print(outcat)
# Do we need to transfer any of the data to nyx?
_getfiles(merge_tables(ccdlist))
# --------------------------------------------------
# Get data, model, and residual cutouts from the legacysurvey viewer. Also
# get thumbnails that are lower resolution.
if args.jpg_cutouts:
thumbsize = 100
sample = fits.getdata(samplefile, 1)
for gal in sample:
size = np.ceil(10*gal['RADIUS']/PIXSCALE)
thumbpixscale = PIXSCALE*size/thumbsize
#imageurl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
# '&pixscale={:.3f}&size={:g}'.format(PIXSCALE, size)
#imagejpg = os.path.join(largedir, 'cutouts', gal['GALAXY'].strip().lower()+'-image.jpg')
#if os.path.isfile(imagejpg):
# os.remove(imagejpg)
#os.system('wget --continue -O {:s} "{:s}"' .format(imagejpg, imageurl))
thumburl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
'&pixscale={:.3f}&size={:g}'.format(thumbpixscale, thumbsize)
thumbjpg = os.path.join(largedir, 'cutouts', gal['GALAXY'].strip().lower()+'-image-thumb.jpg')
if os.path.isfile(thumbjpg):
os.remove(thumbjpg)
os.system('wget --continue -O {:s} "{:s}"' .format(thumbjpg, thumburl))
# --------------------------------------------------
# (Re)build the webpage.
if args.build_webpage:
# index.html
html = open(os.path.join(largedir, 'index.html'), 'w')
html.write('<html><body>\n')
html.write('<h1>Sample of Large Galaxies</h1>\n')
html.write('<table border="2" width="30%">\n')
html.write('<tbody>\n')
sample = fits.getdata(samplefile, 1)
for gal in sample:
# Add coordinates and sizes here.
galaxy = gal['GALAXY'].strip().lower()
html.write('<tr>\n')
html.write('<td><a href="html/{}.html">{}</a></td>\n'.format(galaxy, galaxy.upper()))
html.write('<td><a href="http://legacysurvey.org/viewer/?ra={:.6f}&dec={:.6f}" target="_blank"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'.format(gal['RA'], gal['DEC'], galaxy, galaxy.upper()))
# html.write('<td><a href="html/{}.html"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'.format(galaxy, galaxy, galaxy.upper()))
html.write('</tr>\n')
html.write('</tbody>\n')
html.write('</table>\n')
html.write('</body></html>\n')
html.close()
sys.exit(1)
# individual galaxy pages
for gal in sample[:3]:
galaxy = gal['GALAXY'].strip().lower()
html = open(os.path.join(largedir, 'html/{}.html'.format(galaxy)), 'w')
html.write('<html><body>\n')
html.write('<a href=../cutouts/{}.jpg><img src=../cutouts/{}-image.jpg alt={} /></a>\n'.format(galaxy, galaxy, galaxy, galaxy.upper()))
html.write('</body></html>\n')
html.close()
# --------------------------------------------------
# Get cutouts of all the CCDs for each galaxy.
if args.ccd_cutouts:
sample = fits.getdata(samplefile, 1)
for gal in sample[1:2]:
galaxy = gal['GALAXY'].strip().lower()
ccdsfile = os.path.join(largedir, 'ccds', '{}-ccds.fits'.format(galaxy))
ccds = fits.getdata(ccdsfile)
pdb.set_trace()
# --------------------------------------------------
# Run the pipeline.
if args.runbrick:
sample = fits.getdata(samplefile, 1)
for gal in sample[1:2]:
galaxy = gal['GALAXY'].strip().lower()
diam = 10*np.ceil(gal['RADIUS']/PIXSCALE).astype('int16') # [pixels]
# Note: zoom is relative to the center of an imaginary brick with
# dimensions (0, 3600, 0, 3600).
survey = LegacySurveyData(version='dr2', output_dir=largedir)
run_brick(None, survey, radec=(gal['RA'], gal['DEC']), blobxy=zip([diam/2], [diam/2]),
threads=1, zoom=(1800-diam/2, 1800+diam/2, 1800-diam/2, 1800+diam/2),
wise=False, forceAll=True, writePickles=False, do_calibs=False,
write_metrics=False, pixPsf=True, splinesky=True,
early_coadds=True, stages=['writecat'], ceres=False)
pdb.set_trace()
def main():
survey = LegacySurveyData()
ccds = survey.get_ccds()
print(len(ccds), 'CCDs')
expnums = np.unique(ccds.expnum)
print(len(expnums), 'unique exposures')
for expnum in expnums:
expnumstr = '%08i' % expnum
skyoutfn = os.path.join('splinesky', expnumstr[:5], 'decam-%s.fits' % expnumstr)
psfoutfn = os.path.join('psfex', expnumstr[:5], 'decam-%s.fits' % expnumstr)
if os.path.exists(skyoutfn) and os.path.exists(psfoutfn):
print('Exposure', expnum, 'is done already')
continue
C = ccds[ccds.expnum == expnum]
print(len(C), 'CCDs in expnum', expnum)
psfex = []
psfhdrvals = []
splinesky = []
skyhdrvals = []
for ccd in C:
im = survey.get_image_object(ccd)
fn = im.splineskyfn
if os.path.exists(fn):
T = fits_table(fn)
splinesky.append(T)
# print(fn)
# T.about()
hdr = fitsio.read_header(fn)
skyhdrvals.append([hdr[k] for k in [
'SKY', 'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname])
else:
print('File not found:', fn)
fn = im.psffn
if os.path.exists(fn):
T = fits_table(fn)
hdr = fitsio.read_header(fn, ext=1)
keys = ['LOADED', 'ACCEPTED', 'CHI2', 'POLNAXIS',
'POLNGRP', 'PSF_FWHM', 'PSF_SAMP', 'PSFNAXIS',
'PSFAXIS1', 'PSFAXIS2', 'PSFAXIS3',]
if hdr['POLNAXIS'] == 0:
# No polynomials. Fake it.
T.polgrp1 = np.array([0])
T.polgrp2 = np.array([0])
T.polname1 = np.array(['fake'])
T.polname2 = np.array(['fake'])
T.polzero1 = np.array([0])
T.polzero2 = np.array([0])
T.polscal1 = np.array([1])
T.polscal2 = np.array([1])
T.poldeg1 = np.array([0])
T.poldeg2 = np.array([0])
else:
keys.extend([
'POLGRP1', 'POLNAME1', 'POLZERO1', 'POLSCAL1',
'POLGRP2', 'POLNAME2', 'POLZERO2', 'POLSCAL2',
'POLDEG1'])
for k in keys:
T.set(k.lower(), np.array([hdr[k]]))
psfex.append(T)
#print(fn)
#T.about()
hdr = fitsio.read_header(fn)
psfhdrvals.append([hdr.get(k,'') for k in [
'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname])
else:
print('File not found:', fn)
if len(psfex):
padded = pad_arrays([p.psf_mask[0] for p in psfex])
cols = psfex[0].columns()
cols.remove('psf_mask')
T = merge_tables(psfex, columns=cols)
T.psf_mask = np.concatenate([[p] for p in padded])
T.legpipev = np.array([h[0] for h in psfhdrvals])
T.plver = np.array([h[1] for h in psfhdrvals])
T.expnum = np.array([h[2] for h in psfhdrvals])
T.ccdname = np.array([h[3] for h in psfhdrvals])
fn = psfoutfn
trymakedirs(fn, dir=True)
T.writeto(fn)
print('Wrote', fn)
if len(splinesky):
T = fits_table()
T.gridw = np.array([t.gridvals[0].shape[1] for t in splinesky])
T.gridh = np.array([t.gridvals[0].shape[0] for t in splinesky])
padded = pad_arrays([t.gridvals[0] for t in splinesky])
T.gridvals = np.concatenate([[p] for p in padded])
padded = pad_arrays([t.xgrid[0] for t in splinesky])
T.xgrid = np.concatenate([[p] for p in padded])
padded = pad_arrays([t.xgrid[0] for t in splinesky])
T.ygrid = np.concatenate([[p] for p in padded])
cols = splinesky[0].columns()
print('Columns:', cols)
for c in ['gridvals', 'xgrid', 'ygrid']:
cols.remove(c)
T.add_columns_from(merge_tables(splinesky, columns=cols))
T.skyclass = np.array([h[0] for h in skyhdrvals])
T.legpipev = np.array([h[1] for h in skyhdrvals])
T.plver = np.array([h[2] for h in skyhdrvals])
T.expnum = np.array([h[3] for h in skyhdrvals])
T.ccdname = np.array([h[4] for h in skyhdrvals])
fn = skyoutfn
trymakedirs(fn, dir=True)
T.writeto(fn)
print('Wrote', fn)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-o', '--out', dest='outfn', help='Output filename',
default='TMP/nexp.fits')
parser.add_argument('--merge', action='store_true', help='Merge sub-tables')
parser.add_argument('--plot', action='store_true', help='Plot results')
parser.add_argument('files', metavar='nexp-file.fits.gz', nargs='+',
help='List of nexp files to process')
opt = parser.parse_args()
fns = opt.files
if opt.merge:
from astrometry.util.fits import merge_tables
TT = []
for fn in fns:
T = fits_table(fn)
print(fn, '->', len(T))
TT.append(T)
T = merge_tables(TT)
T.writeto(opt.outfn)
print('Wrote', opt.outfn)
if opt.plot:
T = fits_table(opt.files[0])
import pylab as plt
import matplotlib
ax = [360, 0, -21, 36]
def radec_plot():
plt.axis(ax)
plt.xlabel('RA (deg)')
plt.xticks(np.arange(0, 361, 45))
plt.ylabel('Dec (deg)')
gl = np.arange(361)
gb = np.zeros_like(gl)
from astrometry.util.starutil_numpy import lbtoradec
rr,dd = lbtoradec(gl, gb)
plt.plot(rr, dd, 'k-', alpha=0.5, lw=1)
rr,dd = lbtoradec(gl, gb+10)
plt.plot(rr, dd, 'k-', alpha=0.25, lw=1)
rr,dd = lbtoradec(gl, gb-10)
plt.plot(rr, dd, 'k-', alpha=0.25, lw=1)
plt.figure(figsize=(8,5))
plt.subplots_adjust(left=0.1, right=0.98, top=0.93)
# Map of the tile centers we want to observe...
O = fits_table('obstatus/decam-tiles_obstatus.fits')
O.cut(O.in_desi == 1)
rr,dd = np.meshgrid(np.linspace(ax[1],ax[0], 700),
np.linspace(ax[2],ax[3], 200))
from astrometry.libkd.spherematch import match_radec
I,J,d = match_radec(O.ra, O.dec, rr.ravel(), dd.ravel(), 1.)
desimap = np.zeros(rr.shape, bool)
desimap.flat[J] = True
def desi_map():
# Show the DESI tile map in the background.
from astrometry.util.plotutils import antigray
plt.imshow(desimap, origin='lower', interpolation='nearest',
extent=[ax[1],ax[0],ax[2],ax[3]], aspect='auto',
cmap=antigray, vmax=8)
for band in 'grz':
plt.clf()
desi_map()
N = T.get('nexp_%s' % band)
I = np.flatnonzero(N > 0)
#cm = matplotlib.cm.get_cmap('jet', 6)
#cm = matplotlib.cm.get_cmap('winter', 5)
cm = matplotlib.cm.viridis
cm = matplotlib.cm.get_cmap(cm, 5)
plt.scatter(T.ra[I], T.dec[I], c=N[I], s=2,
edgecolors='none',
vmin=0.5, vmax=5.5, cmap=cm)
radec_plot()
cax = colorbar_axes(plt.gca(), frac=0.06)
plt.colorbar(cax=cax, ticks=range(6))
#plt.colorbar(ticks=range(6))
plt.title('DECaLS DR3: Number of exposures in %s' % band)
plt.savefig('nexp-%s.png' % band)
plt.clf()
desi_map()
plt.scatter(T.ra, T.dec, c=T.get('nexp_%s' % band), s=2,
edgecolors='none', vmin=0, vmax=2.)
radec_plot()
plt.colorbar()
plt.title('DECaLS DR3: PSF size, band %s' % band)
plt.savefig('psfsize-%s.png' % band)
return 0
for col in ['nobjs', 'npsf', 'nsimp', 'nexp', 'ndev', 'ncomp']:
plt.clf()
desi_map()
N = T.get(col)
mx = np.percentile(N, 99.5)
plt.scatter(T.ra, T.dec, c=N, s=2,
edgecolors='none', vmin=0, vmax=mx)
radec_plot()
plt.colorbar()
plt.title('DECaLS DR3: Number of objects of type %s' % col[1:])
plt.savefig('nobjs-%s.png' % col[1:])
Ntot = T.nobjs
for col in ['npsf', 'nsimp', 'nexp', 'ndev', 'ncomp']:
plt.clf()
desi_map()
N = T.get(col) / Ntot.astype(np.float32)
mx = np.percentile(N, 99.5)
plt.scatter(T.ra, T.dec, c=N, s=2,
edgecolors='none', vmin=0, vmax=mx)
radec_plot()
plt.colorbar()
plt.title('DECaLS DR3: Fraction of objects of type %s' % col[1:])
plt.savefig('fobjs-%s.png' % col[1:])
return 0
# fnpats = opt.files
# fns = []
# for pat in fnpats:
# pfns = glob(pat)
# fns.extend(pfns)
# print('Pattern', pat, '->', len(pfns), 'files')
#fns = glob('coadd/*/*/*-nexp*')
#fns = glob('coadd/000/*/*-nexp*')
#fns = glob('coadd/000/0001*/*-nexp*')
fns.sort()
print(len(fns), 'nexp files')
brickset = set()
bricklist = []
gn = []
rn = []
zn = []
gnhist = []
rnhist = []
znhist = []
nnhist = 6
gdepth = []
rdepth = []
zdepth = []
ibricks = []
nsrcs = []
npsf = []
nsimp = []
nexp = []
ndev = []
ncomp = []
gpsfsize = []
rpsfsize = []
zpsfsize = []
ebv = []
gtrans = []
rtrans = []
ztrans = []
bricks = fits_table('survey-bricks.fits.gz')
#sfd = SFDMap()
W = H = 3600
# H=3600
# xx,yy = np.meshgrid(np.arange(W), np.arange(H))
unique = np.ones((H,W), bool)
tlast = 0
for fn in fns:
print('File', fn)
words = fn.split('/')
dirprefix = '/'.join(words[:-4])
print('Directory prefix:', dirprefix)
words = words[-4:]
brick = words[2]
print('Brick', brick)
if not brick in brickset:
brickset.add(brick)
bricklist.append(brick)
gn.append(0)
rn.append(0)
zn.append(0)
gnhist.append([0 for i in range(nnhist)])
rnhist.append([0 for i in range(nnhist)])
znhist.append([0 for i in range(nnhist)])
index = -1
ibrick = np.nonzero(bricks.brickname == brick)[0][0]
ibricks.append(ibrick)
tfn = os.path.join(dirprefix, 'tractor', brick[:3], 'tractor-%s.fits'%brick)
print('Tractor filename', tfn)
T = fits_table(tfn, columns=['brick_primary', 'type', 'decam_psfsize',
'ebv', 'decam_mw_transmission'])
T.cut(T.brick_primary)
nsrcs.append(len(T))
types = Counter([t.strip() for t in T.type])
npsf.append(types['PSF'])
nsimp.append(types['SIMP'])
nexp.append(types['EXP'])
ndev.append(types['DEV'])
ncomp.append(types['COMP'])
print('N sources', nsrcs[-1])
gpsfsize.append(np.median(T.decam_psfsize[:,1]))
rpsfsize.append(np.median(T.decam_psfsize[:,2]))
zpsfsize.append(np.median(T.decam_psfsize[:,4]))
ebv.append(np.median(T.ebv))
gtrans.append(np.median(T.decam_mw_transmission[:,1]))
rtrans.append(np.median(T.decam_mw_transmission[:,2]))
ztrans.append(np.median(T.decam_mw_transmission[:,4]))
br = bricks[ibrick]
print('Computing unique brick pixels...')
#wcs = Tan(fn, 0)
#W,H = int(wcs.get_width()), int(wcs.get_height())
pixscale = 0.262/3600.
wcs = Tan(br.ra, br.dec, W/2.+0.5, H/2.+0.5,
-pixscale, 0., 0., pixscale,
float(W), float(H))
import time
t0 = time.clock()
unique[:,:] = True
find_unique_pixels(wcs, W, H, unique,
br.ra1, br.ra2, br.dec1, br.dec2)
# for i in range(W/2):
# allin = True
# lo,hi = i, W-i-1
# # one slice per side
# side = slice(lo,hi+1)
# top = (lo, side)
# bot = (hi, side)
# left = (side, lo)
# right = (side, hi)
# for slc in [top, bot, left, right]:
# #print('xx,yy', xx[slc], yy[slc])
# rr,dd = wcs.pixelxy2radec(xx[slc]+1, yy[slc]+1)
# U = (rr >= br.ra1 ) * (rr < br.ra2 ) * (dd >= br.dec1) * (dd < br.dec2)
# #print('Pixel', i, ':', np.sum(U), 'of', len(U), 'pixels are unique')
# allin *= np.all(U)
# unique[slc] = U
# if allin:
# print('Scanned to pixel', i)
# break
t1 = time.clock()
U = np.flatnonzero(unique)
t2 = time.clock()
print(len(U), 'of', W*H, 'pixels are unique to this brick')
# #t3 = time.clock()
#rr,dd = wcs.pixelxy2radec(xx+1, yy+1)
# #t4 = time.clock()
# #u = (rr >= br.ra1 ) * (rr < br.ra2 ) * (dd >= br.dec1) * (dd < br.dec2)
# #t5 = time.clock()
# #U2 = np.flatnonzero(u)
#U2 = np.flatnonzero((rr >= br.ra1 ) * (rr < br.ra2 ) *
# (dd >= br.dec1) * (dd < br.dec2))
#assert(np.all(U == U2))
#assert(len(U) == len(U2))
# #t6 = time.clock()
# print(len(U2), 'of', W*H, 'pixels are unique to this brick')
#
#print(t0-tlast, 'other time')
#tlast = time.clock() #t2
#print('t1:', t1-t0, 't2', t2-t1)
# #print('t4:', t4-t3, 't5', t5-t4, 't6', t6-t5)
#
else:
index = bricklist.index(brick)
assert(index == len(bricklist)-1)
index = bricklist.index(brick)
assert(index == len(bricklist)-1)
filepart = words[-1]
filepart = filepart.replace('.fits.gz', '')
print('File:', filepart)
band = filepart[-1]
assert(band in 'grz')
nlist,nhist = dict(g=(gn,gnhist), r=(rn,rnhist), z=(zn,znhist))[band]
upix = fitsio.read(fn).flat[U]
med = np.median(upix)
print('Band', band, ': Median', med)
nlist[index] = med
hist = nhist[index]
for i in range(nnhist):
if i < nnhist-1:
hist[i] = np.sum(upix == i)
else:
hist[i] = np.sum(upix >= i)
assert(sum(hist) == len(upix))
print('Number of exposures histogram:', hist)
ibricks = np.array(ibricks)
print('Maximum number of sources:', max(nsrcs))
T = fits_table()
T.brickname = np.array(bricklist)
T.ra = bricks.ra [ibricks]
T.dec = bricks.dec[ibricks]
T.nexp_g = np.array(gn).astype(np.int16)
T.nexp_r = np.array(rn).astype(np.int16)
T.nexp_z = np.array(zn).astype(np.int16)
T.nexphist_g = np.array(gnhist).astype(np.int32)
T.nexphist_r = np.array(rnhist).astype(np.int32)
T.nexphist_z = np.array(znhist).astype(np.int32)
T.nobjs = np.array(nsrcs).astype(np.int16)
T.npsf = np.array(npsf ).astype(np.int16)
T.nsimp = np.array(nsimp).astype(np.int16)
T.nexp = np.array(nexp ).astype(np.int16)
T.ndev = np.array(ndev ).astype(np.int16)
T.ncomp = np.array(ncomp).astype(np.int16)
T.psfsize_g = np.array(gpsfsize).astype(np.float32)
T.psfsize_r = np.array(rpsfsize).astype(np.float32)
T.psfsize_z = np.array(zpsfsize).astype(np.float32)
T.ebv = np.array(ebv).astype(np.float32)
T.trans_g = np.array(gtrans).astype(np.float32)
T.trans_r = np.array(rtrans).astype(np.float32)
T.trans_z = np.array(ztrans).astype(np.float32)
T.writeto(opt.outfn)
print('t depthhi', t.depthhi)
assert(np.all(t.depthlo == T.depthlo))
assert(np.all(t.depthhi == T.depthhi))
cols = t.get_columns()
t.brickname = np.array([brickname_from_filename(fn)] * len(t))
for band in 'grz':
col = 'counts_ptsrc_%s' % band
if not col in cols:
continue
C = T.get(col)
C += t.get(col)
col = 'counts_gal_%s' % band
C = T.get(col)
C += t.get(col)
TT.append(t)
T.delete_column('brickname')
T.writeto(summaryfn)
print('Wrote', summaryfn)
T = merge_tables(TT, columns='fillzero')
T.writeto(outfn)
print('Wrote', outfn)
def get_catalog_in_wcs(chipwcs,
catsurvey_north,
catsurvey_south=None,
resolve_dec=None,
margin=20):
TT = []
surveys = [(catsurvey_north, True)]
if catsurvey_south is not None:
surveys.append((catsurvey_south, False))
for catsurvey, north in surveys:
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
if resolve_dec is not None:
from astrometry.util.starutil_numpy import radectolb
bricks.gal_l, bricks.gal_b = radectolb(bricks.ra, bricks.dec)
for b in bricks:
# Skip bricks that are entirely on the wrong side of the resolve line (NGC only)
if resolve_dec is not None and b.gal_b > 0:
if north and b.dec2 <= resolve_dec:
continue
if not (north) and b.dec1 >= resolve_dec:
continue
# there is some overlap with this brick... read the catalog.
fn = catsurvey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn,
columns=[
'ra', 'dec', 'brick_primary', 'type', 'release',
'brickid', 'brickname', 'objid', 'fracdev',
'flux_r', 'shapedev_r', 'shapedev_e1',
'shapedev_e2', 'shapeexp_r', 'shapeexp_e1',
'shapeexp_e2', 'ref_epoch', 'pmra', 'pmdec',
'parallax'
])
if resolve_dec is not None and b.gal_b > 0:
if north:
T.cut(T.dec >= resolve_dec)
print('Cut to', len(T), 'north of the resolve line')
else:
T.cut(T.dec < resolve_dec)
print('Cut to', len(T), 'south of the resolve line')
ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
I, = np.nonzero((xx >= -margin) * (xx <= (W + margin)) *
(yy >= -margin) * (yy <= (H + margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
for col in ['out_of_bounds', 'left_blob']:
if col in T.get_columns():
T.cut(T.get(col) == False)
print('Cut to', len(T), 'on', col)
# drop DUP sources
I, = np.nonzero([t.strip() != 'DUP' for t in T.type])
T.cut(I)
print('Cut to', len(T), 'after removing DUP')
if len(T):
TT.append(T)
if len(TT) == 0:
return None
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
print('Total of', len(T), 'catalog sources')
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I, = np.nonzero([
t == 'COMP' and not np.isfinite(r)
for t, r in zip(T.type, T.shapedev_r)
])
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I, = np.nonzero([
t == 'COMP' and not np.isfinite(r)
for t, r in zip(T.type, T.shapeexp_r)
])
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
return T
# wcs2.ensure_inverse_polynomials()
print('T1 X,Y ranges', T1.x.min(), T1.x.max(), T1.y.min(), T1.y.max())
print('T2 X,Y ranges', T2.x.min(), T2.x.max(), T2.y.min(), T2.y.max())
# ~ 1e-6, 0.0006
# ok,x,y = wcs2.radec2pixelxy(T2.ra, T2.dec)
# print('Scatter wcs x vs catalog x:', np.mean(x - T2.x), np.std(x - T2.x))
# print('Scatter wcs y vs catalog y:', np.mean(y - T2.y), np.std(y - T2.y))
ok,x,y = wcs1.radec2pixelxy(T2.ra, T2.dec)
print('Converted X,Y ranges:', x.min(), x.max(), y.min(), y.max())
T2.x = x
T2.y = y
TT = merge_tables([T1,T2])
TT.writeto(gstout)
print('Wrote', gstout)
hdr = pyfits.open(flt1)[0].header
hdr['IMAGEW'] = 4096
hdr['IMAGEH'] = 4096
pyfits.writeto(fltout, None, header=hdr, clobber=True)
print('Wrote', fltout)
#cmd = 'cp "%s" "%s"' % (flt1, fltout)
#print(cmd)
#os.system(cmd)
dr3_fns= glob(os.path.join(NERSC_ROOT,'configs/dr3eBOSS/dr3',
"survey-ccds-*.fits.gz"))
assert(len(eboss_fns) > 0)
assert(len(dr3_fns) > 0)
dr3= stack_tables(dr3_fns,textfile=False)
eboss= stack_tables(eboss_fns,textfile=False)
dr3.set('pid', add_str_arrays([dr3.expnum.astype(str),
np.char.strip(dr3.image_filename)]))
eboss.set('pid', add_str_arrays([eboss.expnum.astype(str),
np.char.strip(eboss.image_filename)]))
dr3.cut( in_eboss(dr3))
eboss.cut( in_eboss(eboss))
T= merge_tables([dr3,eboss], columns='fillzero')
keep= rm_duplicates(T)
T.cut(keep)
name='survey-ccds-ebossDR3.fits'
T.writeto(name)
print('Wrote %s' % name)
a=set(dr3.pid).union(set(eboss.pid))
fn=[lin.split("decam/")[1] for lin in a]
name='eboss_image_list.txt'
with open(name,'w') as foo:
for f in fn:
foo.write('%s\n' % f)
print('Wrote %s' % name)
pixscale = 0.262
apertures = apertures_arcsec / pixscale
#ps = PlotSequence('uber')
if False:
C = fits_table('coadd/000/0001p000/decals-0001p000-ccds.fits')
for c in C:
T,hdr = apphot_ps1stars(c, ps, apertures, decals)
T.writeto('apphot-%08i-%s.fits' % (c.expnum, c.ccdname), header=hdr)
sys.exit(0)
C = decals.get_ccds_readonly()
exps = [ 346352, 347460, 347721 ]
for e in exps:
print
print
print 'Exposure', e
print
E = C[C.expnum == e]
TT = []
for i,c in enumerate(E):
print
print 'Exposure', e, 'chip', i, 'of', len(E)
print
T,hdr = apphot_ps1stars(c, ps, apertures, decals)
TT.append(T)
T = merge_tables(TT)
T.writeto('apphot-%08i.fits' % e, primheader=hdr)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build-sample',
action='store_true',
help='Build the sample.')
parser.add_argument('--jpg-cutouts',
action='store_true',
help='Get jpg cutouts from the viewer.')
parser.add_argument('--ccd-cutouts',
action='store_true',
help='Get CCD cutouts of each galaxy.')
parser.add_argument('--runbrick',
action='store_true',
help='Run the pipeline.')
parser.add_argument('--build-webpage',
action='store_true',
help='(Re)build the web content.')
args = parser.parse_args()
# Top-level directory
key = 'LEGACY_SURVEY_LARGE_GALAXIES'
if key not in os.environ:
print('Required ${} environment variable not set'.format(key))
return 0
largedir = os.getenv(key)
samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')
# --------------------------------------------------
# Build the sample of large galaxies based on the available imaging.
if args.build_sample:
# Read the parent catalog.
cat = read_rc3()
# Create a simple WCS object for each object and find all the CCDs
# touching that WCS footprint.
survey = LegacySurveyData(version='dr2') # hack!
allccds = survey.get_ccds()
keep = np.concatenate((survey.apply_blacklist(allccds),
survey.photometric_ccds(allccds)))
allccds.cut(keep)
ccdlist = []
outcat = []
for gal in cat:
galwcs = _simplewcs(gal)
ccds1 = allccds[ccds_touching_wcs(galwcs, allccds)]
ccds1 = ccds1[_uniqccds(ccds1)]
if len(
ccds1
) > 0 and 'g' in ccds1.filter and 'r' in ccds1.filter and 'z' in ccds1.filter:
print('Found {} CCDs for {}, D(25)={:.4f}'.format(
len(ccds1), gal['GALAXY'], gal['RADIUS']))
ccdsfile = os.path.join(
largedir, 'ccds',
'{}-ccds.fits'.format(gal['GALAXY'].strip().lower()))
print(' Writing {}'.format(ccdsfile))
if os.path.isfile(ccdsfile):
os.remove(ccdsfile)
ccds1.writeto(ccdsfile)
ccdlist.append(ccds1)
if len(outcat) == 0:
outcat = gal
else:
outcat = vstack((outcat, gal))
#if gal['GALAXY'] == 'MCG5-19-36':
# pdb.set_trace()
# Write out the final catalog.
samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')
if os.path.isfile(samplefile):
os.remove(samplefile)
print('Writing {}'.format(samplefile))
outcat.write(samplefile)
print(outcat)
# Do we need to transfer any of the data to nyx?
_getfiles(merge_tables(ccdlist))
# --------------------------------------------------
# Get data, model, and residual cutouts from the legacysurvey viewer. Also
# get thumbnails that are lower resolution.
if args.jpg_cutouts:
thumbsize = 100
sample = fits.getdata(samplefile, 1)
for gal in sample:
size = np.ceil(10 * gal['RADIUS'] / PIXSCALE)
thumbpixscale = PIXSCALE * size / thumbsize
#imageurl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
# '&pixscale={:.3f}&size={:g}'.format(PIXSCALE, size)
#imagejpg = os.path.join(largedir, 'cutouts', gal['GALAXY'].strip().lower()+'-image.jpg')
#if os.path.isfile(imagejpg):
# os.remove(imagejpg)
#os.system('wget --continue -O {:s} "{:s}"' .format(imagejpg, imageurl))
thumburl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
'&pixscale={:.3f}&size={:g}'.format(thumbpixscale, thumbsize)
thumbjpg = os.path.join(
largedir, 'cutouts',
gal['GALAXY'].strip().lower() + '-image-thumb.jpg')
if os.path.isfile(thumbjpg):
os.remove(thumbjpg)
os.system('wget --continue -O {:s} "{:s}"'.format(
thumbjpg, thumburl))
# --------------------------------------------------
# (Re)build the webpage.
if args.build_webpage:
# index.html
html = open(os.path.join(largedir, 'index.html'), 'w')
html.write('<html><body>\n')
html.write('<h1>Sample of Large Galaxies</h1>\n')
html.write('<table border="2" width="30%">\n')
html.write('<tbody>\n')
sample = fits.getdata(samplefile, 1)
for gal in sample:
# Add coordinates and sizes here.
galaxy = gal['GALAXY'].strip().lower()
html.write('<tr>\n')
html.write('<td><a href="html/{}.html">{}</a></td>\n'.format(
galaxy, galaxy.upper()))
html.write(
'<td><a href="http://legacysurvey.org/viewer/?ra={:.6f}&dec={:.6f}" target="_blank"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'
.format(gal['RA'], gal['DEC'], galaxy, galaxy.upper()))
# html.write('<td><a href="html/{}.html"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'.format(galaxy, galaxy, galaxy.upper()))
html.write('</tr>\n')
html.write('</tbody>\n')
html.write('</table>\n')
html.write('</body></html>\n')
html.close()
sys.exit(1)
# individual galaxy pages
for gal in sample[:3]:
galaxy = gal['GALAXY'].strip().lower()
html = open(os.path.join(largedir, 'html/{}.html'.format(galaxy)),
'w')
html.write('<html><body>\n')
html.write(
'<a href=../cutouts/{}.jpg><img src=../cutouts/{}-image.jpg alt={} /></a>\n'
.format(galaxy, galaxy, galaxy, galaxy.upper()))
html.write('</body></html>\n')
html.close()
# --------------------------------------------------
# Get cutouts of all the CCDs for each galaxy.
if args.ccd_cutouts:
sample = fits.getdata(samplefile, 1)
for gal in sample[1:2]:
galaxy = gal['GALAXY'].strip().lower()
ccdsfile = os.path.join(largedir, 'ccds',
'{}-ccds.fits'.format(galaxy))
ccds = fits.getdata(ccdsfile)
pdb.set_trace()
# --------------------------------------------------
# Run the pipeline.
if args.runbrick:
sample = fits.getdata(samplefile, 1)
for gal in sample[1:2]:
galaxy = gal['GALAXY'].strip().lower()
diam = 10 * np.ceil(gal['RADIUS'] / PIXSCALE).astype(
'int16') # [pixels]
# Note: zoom is relative to the center of an imaginary brick with
# dimensions (0, 3600, 0, 3600).
survey = LegacySurveyData(version='dr2', output_dir=largedir)
run_brick(None,
survey,
radec=(gal['RA'], gal['DEC']),
blobxy=zip([diam / 2], [diam / 2]),
threads=1,
zoom=(1800 - diam / 2, 1800 + diam / 2, 1800 - diam / 2,
1800 + diam / 2),
wise=False,
forceAll=True,
writePickles=False,
do_calibs=False,
write_metrics=False,
pixPsf=True,
splinesky=True,
early_coadds=True,
stages=['writecat'],
ceres=False)
pdb.set_trace()
if __name__ == '__main__':
ps = PlotSequence('morph')
from glob import glob
from astrometry.util.fits import merge_tables, fits_table
# glob doesn't understand {}
#fns = glob('dr3/tractor/011/tractor-011?p0{02,05,07,10}.fits')
fns = glob('dr3/tractor/011/tractor-011?p0*.fits')
#fns = glob('dr3/metrics/011/all-models-011?p0*.fits')
fns = [fn for fn in fns if fn[-7:-5] in ['02','05','07','10']]
assert(len(fns) == 16)
T = merge_tables([fits_table(fn) for fn in fns])
print(len(T), 'sources')
dchipsf = T.dchisq[:,0]
dchisimp = T.dchisq[:,1]
dchidev = T.dchisq[:,2]
dchiexp = T.dchisq[:,3]
model = np.array(['P' if p > s else 'S' for p,s in zip(dchipsf, dchisimp)])
dchi = dchipsf * (model == 'P') + dchisimp * (model == 'S')
# which sources have galaxy models computed
I = np.flatnonzero(dchiexp)
print(len(I), 'have EXP,DEV models')
galaxy_margin = 12.
continue
print 'Reading', fn
T = fits_table(fn)
ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W,H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) *
(yy >= -margin) * (yy <= (H+margin)))
T.cut(I)
print 'Cut to', len(T), 'sources within image + margin'
#print 'Brick_primary:', np.unique(T.brick_primary)
T.cut(T.brick_primary)
print 'Cut to', len(T), 'on brick_primary'
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print 'Cut to', len(T), 'on out_of_bounds and left_blob'
TT.append(T)
T = merge_tables(TT)
T._header = TT[0]._header
del TT
#T.writeto('cat.fits')
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print 'Converting', len(I), 'bogus COMP galaxies to EXP'
for i in I:
T.type[i] = 'EXP'
def main(decals=None, opt=None):
'''Driver function for forced photometry of individual DECam images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0,x1,y0,y1) = opt.zoom
zoomslice = (slice(y0,y1), slice(x0,x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.filename)
opt.filename = None
except:
# make this 'None' for decals.find_ccds()
expnum = None
# Try parsing HDU number
try:
opt.hdu = int(opt.hdu)
ccdname = None
except:
ccdname = opt.hdu
opt.hdu = -1
if decals is None:
decals = Decals()
if opt.filename is not None and opt.hdu >= 0:
# Read metadata from file
T = exposure_metadata([opt.filename], hdus=[opt.hdu])
print('Metadata:')
T.about()
else:
# Read metadata from decals-ccds.fits table
T = decals.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if opt.hdu >= 0:
T.cut(T.image_hdu == opt.hdu)
print(len(T), 'with HDU', opt.hdu)
if opt.filename is not None:
T.cut(np.array([f.strip() == opt.filename for f in T.image_filename]))
print(len(T), 'with filename', opt.filename)
assert(len(T) == 1)
im = decals.get_image_object(T[0])
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True)
print('Got tim:', tim)
if opt.catfn in ['DR1', 'DR2']:
if opt.catalog_path is None:
opt.catalog_path = opt.catfn.lower()
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, decals=decals)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3],
'tractor-%s.fits' % b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W,H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) *
(yy >= -margin) * (yy <= (H+margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print('Cut to', len(T), 'on out_of_bounds and left_blob')
TT.append(T)
T = merge_tables(TT)
T._header = TT[0]._header
del TT
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T
T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T
cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE)
# print('Got cat:', cat)
print('Forced photom...')
opti = None
if opt.ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
opti = CeresOptimizer(BW=B, BH=B)
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
for src in cat:
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
F = fits_table()
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T))
ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x-1).astype(np.float32)
F.y = (y-1).astype(np.float32)
if opt.apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
print('Apertures:', apertures, 'pixels')
for rad in apertures:
aper = photutils.CircularAperture(apxy, rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = ap
ap = 1./(np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux_ivar = ap
if opt.forced:
kwa = {}
if opt.plots is None:
kwa.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True, fitstats=True,
shared_params=False, **kwa)
if opt.plots:
(data,mod,ie,chi,roi) = R.ims1[0]
ima = tim.ima
imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5)
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
F.flux = np.array([src.getBrightness().getFlux(tim.band)
for src in cat]).astype(np.float32)
F.flux_ivar = R.IV.astype(np.float32)
F.fracflux = R.fitstats.profracflux.astype(np.float32)
F.rchi2 = R.fitstats.prochi2 .astype(np.float32)
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, decals.decals_dir)
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
fname = os.path.join(d2, d1, fname)
print('Trimmed filename to', fname)
#version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file'))
version_hdr.add_record(dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext'))
version_hdr.add_record(dict(name='CAMERA', value='DECam', comment='Dark Energy Camera'))
version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num'))
version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name'))
version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image'))
keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV',
'INSTRUME']
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {'mjd':'sec', 'exptime':'sec', 'flux':'nanomaggy',
'flux_ivar':'1/nanomaggy^2'}
columns = F.get_columns()
for i,col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
print('Finished forced phot:', Time()-t0)
return 0
