def depth(infile,nside=NSIDE,signal_to_noise=10.):
MAGS = bfields('MAG_PSF',BANDS)
MAGERRS = bfields('MAGERR_PSF',BANDS)
SPREADS = bfields('WAVG_SPREAD_MODEL',BANDS)
# From propagation of errors:
# mag = -2.5 * log10(flux)
# magerr = -2.5/ln(10) * fluxerr/flux
mag_snr = (2.5/np.log(10)) / (signal_to_noise)
logger.info(infile)
ret = dict()
for band,mag,magerr,spread in zip(BANDS,MAGS,MAGERRS,SPREADS):
data = fitsio.read(infile,columns=['RA','DEC',mag,magerr,spread])
h, edges = np.histogram(data[mag], bins=np.arange(17, 30, 0.1))
mag_bright_end = edges[np.argmax(h)] - 3.
cut = (np.fabs(data[spread]) < 0.002) & (data[mag] > mag_bright_end) & (data[mag] < 30.)
d = data[cut]
if len(d) < 2:
logger.warn("Insufficent objects in %s-band"%band)
ret[band] = [np.array([],dtype=int),np.array([])]
continue
pix = ang2pix(nside,d['RA'],d['DEC'])
match = ugali.utils.projector.match(d['RA'],d['DEC'],d['RA'],d['DEC'],nnearest=2)
delta_mag = d[mag][match[1]] - d[mag][match[0]]
delta_log_magerr = np.log10(d[magerr][match[1]]) - np.log10(d[magerr][match[0]])
old = np.seterr(divide='ignore',invalid='ignore')
ratio = delta_log_magerr / delta_mag
cut_nan_inf = np.isfinite(ratio) & (delta_mag > 0.5)
np.seterr(**old)
if cut_nan_inf.sum() < 2:
logger.warn("Insufficent objects in %s-band"%band)
ret[band] = [np.array([],dtype=int),np.array([])]
continue
kde = scipy.stats.gaussian_kde(ratio[cut_nan_inf])
values = np.linspace(0., 1., 1000)
kde_values = kde.evaluate(values)
slope = values[np.argmax(kde_values)]
maglims = d[mag] - ((np.log10(d[magerr]) - np.log10(mag_snr)) / slope)
hpx = np.unique(pix)
maglim = nd.median(maglims,labels=pix,index=hpx)
ret[band] = [hpx,maglim]
return ret
def check_nan(args):
""" Check columns for nan values """
kwargs = dict(select=lambda x: bool(np.any(np.isnan(x.view('>f4')))))
kwargs.update(columns=bfields(['WAVG_MAGRMS_AUTO', 'WAVG_MAGRMS_PSF'],
BANDS),
msg="Bad MAGRMS value in %(filename)s")
ret = check_columns(args, **kwargs)
kwargs.update(columns=bfields(['WAVG_SPREADRMS_MODEL'], BANDS),
msg="Bad SPREADRMS value in %(filename)s")
ret |= check_columns(args, **kwargs)
return ret
def quality_cuts(cat,key=None):
"""
Prune down the catalog by selecting on detection band, etc.
"""
nobjs = len(cat)
sel = np.zeros(nobjs,dtype=bool)
# Objects with detections in the minimum number of bands.
# Careful with this, the 'view' can be dangerous...
columns = bfields(['MAG_PSF'],BANDS)
dtype = cat.dtype[columns[0]]
mags = cat[columns].view(dtype).reshape((cat.size,-1))
sel |= (np.sum(mags < BADMAG,axis=1) >= MINBANDS)
"""
# Objects with r,i,z
mags = cat[bfields(['MAG_PSF','MAG_AUTO'],BANDS)].view(np.float).reshape((cat.size,-1))
sel = np.any(mags!=BADMAG,axis=1)
# Objects with g,r
mags = cat[bfields(['MAG_PSF','MAG_AUTO'],['g','r'])].view(np.float).reshape((cat.size,-1))
sel = np.all(mags!=BADMAG,axis=1)
# Objects with any of r,i,z
mags = cat[bfields(['MAG_PSF','MAG_AUTO'],['r','i','z'])].view(np.float).reshape((cat.size,-1))
sel = np.any(mags!=BADMAG,axis=1)
"""
if key is None:
return cat[sel]
else:
# Most objects accepted; key selection on the omitted IDs for speed
ksel = ~np.in1d(key[OBJECT_ID],cat[OBJECT_ID][~sel])
return cat[sel],key[ksel]
def rms_photometry(catfile,nside=64,band=None,plot=False):
""" Calculate photometric repeatability """
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
columns = ['RA','DEC']
spread,nepochs = bfields(['WAVG_SPREAD_MODEL','NEPOCHS'],band)
mag,magerr,magrms = bfields(['WAVG_MAG_PSF','WAVG_MAGERR_PSF','WAVG_MAGRMS_PSF'],band)
columns += [spread, nepochs, mag, magerr, magrms]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
msg = '%s (RA,DEC) = %.2f,%.2f'%(os.path.basename(catfile),ra,dec)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < r < 20 and 0.0 < (g-i) < 1.5
sel = (np.fabs(cat[spread]) < 0.002) & \
(cat[mag] > 16) & (cat[mag] < 18) &\
(cat[magrms] < 90) &\
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
pix = ang2pix(nside,cat['RA'],cat['DEC'])
upix = np.unique(pix)
stat = nd.median(cat[magrms],labels=pix,index=upix)
if False:
plt.figure()
plt.hist(cat[magrms],bins=50)
import pdb; pdb.set_trace()
return upix,stat
def check_match(args):
""" Check match fraction """
def select(x):
nobjs = float(len(x))
frac = (x > 0).sum() / nobjs
bad = (frac < 0.1) and (nobjs > 1e4)
if bad:
msg = 'Match fraction = %.2f;' % frac
print color(msg, 'yellow'),
return bad
msg = color("Poor match in %(filename)s", 'yellow')
kwargs = dict(columns=bfields('NEPOCHS', band), select=select, msg=msg)
return check_columns(args, **kwargs)
def check_keys(cat,key):
"""
Check the number of non-unique objects against the number of
measured magnitudes.
"""
nkey = (keys['UNIQUE_ID'] >0).sum()
columns = bfields(['MAG_PSF'],BANDS)
dtype = cat.dtype[columns[0]]
ncat = (cat[columns].view(dtype).reshape(len(cat),-1) < 90).sum()
if ncat != nkey:
msg = "Number of keys does not match catalog"
raise ValueError(msg)
return
def teff(infile,nside=NSIDE,mode='median'):
TEFFS = bfields('TEFF',BANDS)
logger.info(infile)
ret = dict()
for band,teff in zip(BANDS,TEFFS):
data = fitsio.read(infile,columns=['RA','DEC',teff])
pix = ang2pix(nside,data['RA'],data['DEC'])
hpx = np.unique(pix)
if mode.lower() is 'median':
teff_value = nd.median(data[teff],labels=pix,index=hpx)
elif mode.lower() is 'mean':
teff_value = nd.mean(data[teff],labels=pix,index=hpx)
else:
msg = 'Unrecognized mode: %s'%mode
raise Exception(msg)
ret[band] = [hpx,maglim]
return ret
def distance(args,plot=False):
nice = os.nice(0)
os.nice(10-nice)
pix,nside = args
catfile = glob.glob('cat/*_%05d.fits'%pix)[0]
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
print(catfile)
columns = [OBJECT_ID,'RA','DEC']
spread,mag,nepochs = bfields(['WAVG_SPREAD_MODEL','MAG_PSF','NEPOCHS'],band)
columns += [spread,mag,nepochs]
cat = load_infiles([catfile],columns)
sel = (np.fabs(cat[spread])<0.002)&(cat[mag]>16)&(cat[mag]<22)&(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
print("WARNING: No catalog objects passing selection")
return np.array([],dtype=int), np.array([])
ra,dec = cat['RA'],cat['DEC']
m = ugali.utils.projector.match(ra,dec,ra,dec,nnearest=2)
sep = m[-1]
hpx = ang2pix(nside,ra,dec)
peak = nd.median(sep,labels=hpx,index=np.unique(hpx))
if plot:
plt.figure()
draw_angsep(sep)
if isinstance(plot,basestring):
outfile = plot
plt.savefig(outfile,bbox_inches='tight')
return hpx,peak
def gaia_photometry(catfile,nside=64,band=None,plot=False,version='edr3'):
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
#columns = [OBJECT_ID,'RA','DEC']
columns = ['RA','DEC']
spread,nepochs = ['WAVG_SPREAD_MODEL_R','NEPOCHS_R']
mag_g,mag_r,mag_i,mag_z = bfields(['MAG_PSF'],['g','r','i','z'])
#mag_g,mag_r,mag_i,mag_z = bfields(['WAVG_MAG_PSF'],['g','r','i','z'])
columns += [spread, nepochs, mag_g, mag_r, mag_i, mag_z]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
radius = np.degrees(hp.max_pixrad(NSIDE))
msg = '%s (RA,DEC,RAD) = %.2f,%.2f,%.2f'%(os.path.basename(catfile),ra,dec,radius)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < r < 20 and 0.0 < (g-i) < 1.5
sel = (np.fabs(cat[spread])<0.002) & \
(cat[mag_g]<90) & (cat[mag_r]<90) & (cat[mag_i]<90) & (cat[mag_z]<90) & \
(cat[mag_r]>16) & (cat[mag_r]<20) & \
((cat[mag_g] - cat[mag_i]) > 0.0) & \
((cat[mag_g] - cat[mag_i]) < 1.5) & \
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
#msg = "Getting external catalog: %s"%catalog
ext = get_gaia_catalog(hpx,version=version)
m = match_query(cat['RA'],cat['DEC'],ext['RA'],ext['DEC'])
# Use a fairly wide matching radius (2 arcsec)
cut = 1.0
sel = m[-1]*3600. < cut
cat_match = cat[m[0][sel]]
ext_match = ext[m[1][sel]]
cat_G = gaia_transform(cat_match[mag_g],cat_match[mag_r],cat_match[mag_i],cat_match[mag_z],
version=version)
# Need to put Gaia flux onto the AB system
ext_G = -2.5 * np.log10(ext_match['PHOT_G_MEAN_FLUX']) + 25.7934
diff = cat_G - ext_G
pix = ang2pix(nside,cat_match['RA'],cat_match['DEC'])
upix = np.unique(pix)
stat = nd.median(diff,labels=pix,index=upix)
if False:
plt.figure()
plt.hist(cat_G - ext_G)
import pdb; pdb.set_trace()
return upix,stat
plt.savefig(pltdir+'angsep_%05d.png'%opts.pix,bbox_inches='tight')
plt.figure()
plt.hist(sep*3600.,log=True,**kwargs)
c = (b[:-1]+b[1:])/2.
peak = c[np.argmax(n)]; text = '%.1f (mas)'%(peak*1e3)
plot_peak(peak,text,lw=2,ls='--',c='r')
plt.xlabel("Separation (arcsec)")
plt.ylabel("Counts")
plt.savefig(pltdir+'log_angsep_%05d.png'%opts.pix,bbox_inches='tight')
# Spread Model
kwargs.update(dict(bins=np.linspace(-0.01,0.04,100)))
plt.figure()
for x in bfields(['SPREAD_MODEL','WAVG_SPREAD_MODEL'],['R','I']):
plt.hist(coadd[x],label=x,**kwargs)
plt.xlabel('SPREAD')
plt.ylabel('Counts')
plt.legend()
plt.savefig(pltdir+'spread_%05d.png'%opts.pix,bbox_inches='tight')
plt.figure()
plt.hist(coadd['SPREAD_MODEL_R']+3*coadd['SPREADERR_MODEL_R'],
label='SPREAD_R+3*ERR',**kwargs)
plt.hist(coadd['WAVG_SPREAD_MODEL_R']+3*coadd['WAVG_SPREADRMS_MODEL_R'],
label='WAVG_SPREAD_R+3*RMS',**kwargs)
plt.hist(coadd['WAVG_SPREAD_MODEL_R']+3*coadd['WAVG_SPREADERR_MODEL_R'],
label='WAVG_SPREAD_R+3*ERR',**kwargs)
plt.xlabel('SPREAD_R + 3*ERR')
def residuals(pixel,
nside=NSIDE,
plot=False,
y1a1dir='y1a1/v1/hpx/',
y2q1dir='cat/'):
y1a1file = os.path.join(y1a1dir, 'cat_hpx_%05d.fits' % pixel)
y2q1file = os.path.join(y2q1dir, 'cat_hpx_%05d.fits' % pixel)
print y1a1file, y2q1file
y1a1 = fitsio.read(y1a1file, columns=Y1A1_COLUMNS)
y2q1 = fitsio.read(y2q1file, columns=Y2Q1_COLUMNS)
y2q1 = y2q1[(y2q1['WAVG_SPREAD_MODEL_R'] < 0.002)]
#hpx = ang2pix(nside,y1a1['RA'],y1a1['DEC'])
#y1a1 = recfuncs.rec_append_fields('HPX',hpx,dtypes=int)
#
#hpx = ang2pix(nside,y2q1['RA'],y2q1['DEC'])
#y2q1 = recfuncs.rec_append_fields('HPX',hpx,dtypes=int)
#if plot:
#
# fig,axes = plt.subplots(1,2,figsize=(12,5))
kwargs = dict(histtype='step', lw=1.5, normed=True)
ret = odict()
for band in BANDS:
mag, magerr = bfields(['WAVG_MAG_PSF', 'WAVG_MAGERR_PSF'], band)
color = COLORS[band]
y1 = y1a1[(y1a1[mag] < 22) & (y1a1[mag] > 17)]
y2 = y2q1[(y2q1[mag] < 22) & (y2q1[mag] > 17)]
match = ugali.utils.projector.match(y1['RA'], y1['DEC'], y2['RA'],
y2['DEC'])
sepsec = 3600. * match[-1]
sel = (sepsec < 1.0)
idx1 = match[0][sel]
idx2 = match[1][sel]
y1_match = y1[idx1]
y2_match = y2[idx2]
res = (y2[mag][idx2] - y1[mag][idx1])
res_clip, lo, hi = scipy.stats.sigmaclip(res, 5, 5)
mu, sigma = norm.fit(res_clip)
median = np.median(res_clip)
ret[band] = (median, mu, sigma)
if plot:
bins = np.linspace(-0.1, 0.1, 100)
centers = (bins[1:] + bins[:-1]) / 2.
kwargs['bins'] = bins
axes[0].hist(res, color=color, **kwargs)
mu, sigma = norm.fit(res[(res > bins.min()) & (res < bins.max())])
label = r'$%s\ (\mu=%.2f,\sigma=%.2f)$' % (band, mu, sigma)
axes[0].plot(centers,
norm.pdf(centers, mu, sigma),
color=color,
label=label)
if plot:
plt.sca(axes[0])
plt.legend(fontsize=8)
plt.sca(axes[1])
plt.legend(fontsize=8)
return pixel, ret
import pylab as plt
import numpy as np
import scipy.stats
from scipy.stats import norm
import healpy
from utils import bfields
from const import BANDS, COLORS
from footprint import blank
import footprint
import plotting
import ugali.utils.projector
from ugali.utils.shell import mkdir
COLUMNS = ['RA', 'DEC'] + bfields(['WAVG_MAG_PSF', 'WAVG_MAGERR_PSF'], BANDS)
Y1A1_COLUMNS = COLUMNS
Y2Q1_COLUMNS = COLUMNS + ['WAVG_SPREAD_MODEL_R']
NSIDE = 16
def residuals(pixel,
nside=NSIDE,
plot=False,
y1a1dir='y1a1/v1/hpx/',
y2q1dir='cat/'):
y1a1file = os.path.join(y1a1dir, 'cat_hpx_%05d.fits' % pixel)
y2q1file = os.path.join(y2q1dir, 'cat_hpx_%05d.fits' % pixel)
print y1a1file, y2q1file
y1a1 = fitsio.read(y1a1file, columns=Y1A1_COLUMNS)
def create_columns(bands=BANDS):
COLUMNS = ['RA', 'DEC'] + bfields('WAVG_MAG_PSF', bands)
#COLUMNS = ['RA','DEC']+bfields('WAVG_MAG_PSF',BANDS)+bfields('NEPOCHS',BANDS)
return COLUMNS
def external_astrometry(catfile,catalog='GAIA',nside=64,band='r',plot=False):
#nice = os.nice(0)
#os.nice(10-nice)
if band=='all': band = 'r'
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
columns = [OBJECT_ID,'RA','DEC']
spread,mag,nepochs = bfields(['WAVG_SPREAD_MODEL','MAG_PSF','NEPOCHS'],band)
columns += [spread,mag,nepochs]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
radius = np.degrees(healpy.max_pixrad(NSIDE))
msg = '%s (RA,DEC,RAD) = %.2f,%.2f,%.2f'%(os.path.basename(catfile),ra,dec,radius)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < mag < 21
sel = (np.fabs(cat[spread])<0.002) & \
(cat[mag]>16) & \
(cat[mag]<21) & \
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
#print "Getting external catalog: %s"%catalog
if catalog in list(CATALOGS.keys()):
ext = get_vizier_catalog(ra,dec,radius,**CATALOGS[catalog])
else:
ext = get_local_catalog(ra,dec,radius,catalog)
m = match_query(cat['RA'],cat['DEC'],ext['_RAJ2000'],ext['_DEJ2000'])
# Use a fairly wide matching radius (2 arcsec)
cut = 2.0
sel = m[-1]*3600. < cut
sepdeg = m[-1][sel]
sepsec = m[-1][sel] * 3600.
sepmas = sepsec * 1000.
sep = sepmas
pix = ang2pix(nside,cat['RA'][sel],cat['DEC'][sel])
upix = np.unique(pix)
try:
peak = nd.median(sep,labels=pix,index=upix)
except ValueError:
import pdb; pdb.set_trace()
if plot:
plt.figure()
draw_angsep(sep,bins=np.linspace(0,cut*1000.,101))
if isinstance(plot,basestring):
outfile = plot
plt.savefig(outfile,bbox_inches='tight')
#return cat,ext,m
return upix,peak
def create_output_columns(bands):
""" Create an ordered dictionary of output column names. """
columns = odict(
[(i,('i8',-1)) for i in [OBJECT_ID]]
+ [(c,('f8',np.nan)) for c in COORDS]
+ [(i,('i8',-1)) for i in HEALPIX]
+ [(f,('i2',0)) for f in bfields(NEPOCHS,bands)]
+ [(f,('f4',BADMAG)) for f in bfields(MAGS,bands)]
+ [(f,('f4',-1)) for f in bfields(SPREAD[:1],bands)]
+ [(f,('f4',1)) for f in bfields(SPREAD[1:],bands)]
+ [(f,('f4',-1)) for f in bfields(CLASS,bands)]
+ [(f,('i2',99)) for f in bfields(FLAGS,bands)]
+ [(f,('f4',BADMAG)) for f in bfields(WAVGMAGS,bands)]
+ [(f,('f4',-1)) for f in bfields(WAVGSPREAD[:1],bands)]
+ [(f,('f4',1)) for f in bfields(WAVGSPREAD[1:],bands)]
#+ [(f,('f4',-1)) for f in bfields(WAVGCLASS,bands)] # Y2Q1
+ [(f,('i2',99)) for f in bfields(WAVGFLAGS,bands)]
+ [(f,('i4',-1)) for f in bfields(EXPNUM,bands)]
+ [(f,('f4',-1)) for f in bfields(TEFF,bands)]
+ [(f,('f4',-1)) for f in bfields(IMAGE,bands)]
+ [(f,('f4',-1)) for f in bfields(EXTINCTION,bands)]
)
return columns
def coadd_objects(data,bands=BANDS):
"""
Create a unique object catalog.
Parameters:
-----------
data : the pre-band catalogs
bands : the bands to combine
Returns:
-------
cat,keys : the coadd catalog and associated matching keys
"""
unique_ids = np.unique(data[OBJECT_ID])
nobjs = len(unique_ids)
output_columns = create_output_columns(bands)
dtype = [(k,v[0]) for k,v in output_columns.items()]
cat = np.recarray(nobjs,dtype=dtype)
for k,v in output_columns.items():
cat[k] = v[1]
cat[OBJECT_ID] = unique_ids
keys = copy.copy(data[IDX+EXPNUM])
keys = recfuncs.rec_append_fields(keys,UNIQUE_ID,-np.ones(len(keys)),dtypes='>i8')
# OBJECT_NUMBER has a different meaning (and type) in Y1A1 and Y2N.
# Standardize it here (wouldn't be necessary if done on download).
# ADW: Is this working properly?
if not keys.dtype['OBJECT_NUMBER'] is not np.dtype('>i8'):
keys['OBJECT_NUMBER'] = keys['OBJECT_NUMBER'].astype('>i8')
x = coadd_coords(data['RA'],data['DEC'],data[OBJECT_ID],index=unique_ids)
for i,f in enumerate(COORDS):
cat[f] = x[i]
x = coadd_healpix(cat[COORDS[0]],cat[COORDS[1]],NSIDES,nest=True)
for i,f in enumerate(HEALPIX):
cat[f] = x[i]
for b in bands:
logger.info("=== Creating %s-band catalog ==="%b)
# This is an annoying feature of fitsio...
band = '{0:<{1}}'.format(b,len(data[BAND][0]))
sel = np.where(data[BAND]==band)[0]
if len(sel) == 0:
logger.warning("No objects found in %s-band."%b)
continue
d = data[sel]
labels = d[OBJECT_ID]
index,inverse,counts = np.unique(labels,False,True,True)
# Match unique indices of this selection to the catalog indices
idx = np.searchsorted(cat[OBJECT_ID],index)
# Set the unique keys
i = best_values(sel,d['T_EFF']*d['EXPTIME'],labels,index)
keys[UNIQUE_ID][i] = keys[OBJECT_ID][i]
for f in NEPOCHS:
cat[bfields(f,b)][idx] = counts
x = best_values(d[BEST],d['T_EFF']*d['EXPTIME'],labels,index)
for i,f in enumerate(BEST):
cat[bfields(f,b)][idx] = x[i]
x = coadd_mag(d[MAGPSF[0]],d[MAGPSF[1]],labels,index,inverse,counts)
for i,f in enumerate(WAVGMAGPSF):
cat[bfields(f,b)][idx] = x[i]
x = coadd_mag(d[MAGAUTO[0]],d[MAGAUTO[1]],labels,index,inverse,counts)
for i,f in enumerate(WAVGMAGAUTO):
cat[bfields(f,b)][idx] = x[i]
x = coadd_spread(d[SPREAD[0]],d[SPREAD[1]],labels,index,inverse,counts)
for i,f in enumerate(WAVGSPREAD):
cat[bfields(f,b)][idx] = x[i]
try:
x = coadd_class(d['CLASS_STAR'],labels,index)
for i,f in enumerate(WAVGCLASS):
cat[bfields(f,b)][idx] = x[i]
except ValueError,msg:
logger.warning(msg)
for i,(f,w) in enumerate(zip(FLAGS,WAVGFLAGS)):
x = coadd_flags(d[f],labels,index)
cat[bfields(w,b)][idx] = x
def internal_astrometry(catfile,hpxfile,nside=128,band='r',plot=False):
""" Calculate internal relative astrometry.
Parameters
----------
catfile : merged catalog file
hpxfile : single epoch catalog file(s)
nside : nside for calculation
band : band to use
plot : plot output
Returns
-------
stats : output statistics
"""
nice = os.nice(0)
os.nice(10-nice)
if band=='all': band = 'r'
#print catfile,hpxfile,nside
#catfile = glob.glob('cat/*_%05d.fits'%pix)[0]
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
columns = [OBJECT_ID,'RA','DEC']
spread,mag,nepochs = bfields(['WAVG_SPREAD_MODEL','MAG_PSF','NEPOCHS'],band)
columns += [spread,mag,nepochs]
cat = load_infiles([catfile],columns)
# Select stars with 17 < mag < 21
sel = (np.fabs(cat[spread])<0.002) & \
(cat[mag]>17) & \
(cat[mag]<21) & \
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
print("WARNING: No objects passing selection in: %s"%catfile)
return np.array([],dtype=int), np.array([])
#hpxfiles = glob.glob('hpx/%s/*_%05d.fits'%(band,pix))
hpx = load_infiles(hpxfile, [OBJECT_ID, 'RA', 'DEC'])
hpx = hpx[np.in1d(hpx[OBJECT_ID],cat[OBJECT_ID])]
if len(hpx) == 0:
print("WARNING: No matched objects in: %s"%hpxfile)
return np.array([],dtype=int), np.array([])
#keyfile = 'key/key_hpx_%05d.fits'%pix
#key = load_infiles([keyfile],[OBJECT_ID,'FILENAME','OBJECT_NUMBER'])
#key = key[np.in1d(key[OBJECT_ID],cat[OBJECT_ID])]
#
#key_id = np.char.add(key['FILENAME'],key['OBJECT_NUMBER'].astype(str))
#hpx_id = np.char.add(hpx['FILENAME'],hpx['OBJECT_NUMBER'].astype(str))
#
#hpx = hpx[np.in1d(hpx_id,key_id)]
uid,inv,cts = np.unique(hpx[OBJECT_ID],False,True,True)
# Make sure that the order matches between coadd and single epoch.
if not np.all(uid == cat[OBJECT_ID]):
cat = cat[np.in1d(cat[OBJECT_ID],hpx[OBJECT_ID])]
if not np.all(uid == cat[OBJECT_ID]):
cat = cat[np.argsort(cat[OBJECT_ID])]
assert np.all(uid == cat[OBJECT_ID])
ra,dec = cat['RA'][inv],cat['DEC'][inv]
sepdeg = angsep(ra,dec,hpx['RA'],hpx['DEC'])
sepsec = sepdeg * 3600.
sepmas = sepsec * 1000.
sel = [sepsec > 1e-5]
sep = sepmas[sel]
pix = ang2pix(nside,ra[sel],dec[sel])
upix = np.unique(pix)
peak = nd.median(sep,labels=pix,index=upix)
if plot:
plt.figure()
draw_angsep(sep)
if isinstance(plot,basestring):
outfile = plot
plt.savefig(outfile,bbox_inches='tight')
return upix,peak
if section == 'catalog':
catdir = config['catdir']
if not dir_exists(catdir): continue
keydir = config['keydir']
if not dir_exists(keydir): continue
files = sorted(glob.glob(catdir + '/*.fits'))
nfiles = len(files)
print " Matching:"
CATCOUNT = 0
bad = False
for i, f in enumerate(files):
print_running(i + 1, nfiles, indent=4, step=1)
epochs = bfields('NEPOCHS', bands)
epoch = bfields('NEPOCHS', band)
coords = ['RA', 'DEC']
#spread = bfields('SPREAD_MODEL',bands)
columns = epochs + coords
data = fitsio.read(f, columns=columns)
CATCOUNT += len(data)
sel = np.isnan(data['RA']) | np.isnan(data['DEC'])
if np.any(sel):
msg = "RA,DEC NaN in %s" % f
print color(msg, 'red')
print 4 * ' ' + str(data[['RA', 'DEC']][sel])
bad = True