def read_desdr2(ra,dec,radius=1.5):
""" Read the reference catalog in a localized region.
Parameters
----------
ra : ra (deg)
dec : dec (deg)
radius : regional radius (deg)
Returns
-------
refcat : reference catalog columns
"""
nside = 32
pixels = ang2disc(nside, ra, dec, radius, inclusive=True)
dirname = '/data/des40.b/data/y6a1/gold/1.1/healpix'
basename = 'y6_gold_1_0_%05d.fits'
filenames = [os.path.join(dirname,basename%p) for p in pixels]
#filenames = [f for f in filenames if os.path.exists(f)]
columns = DESDR2_COLUMNS
refcat = utils.load_infiles(filenames,columns=columns)
refcat = refcat[refcat['WAVG_MAG_PSF_R'] < 21]
mapping = dict(zip(DESDR2_COLUMNS,REFCAT_COLUMNS))
new_names = [mapping[n] for n in refcat.dtype.names]
refcat.dtype.names = new_names
return refcat
def get_gaia_catalog(hpx,columns=['RA','DEC','PHOT_G_MEAN_FLUX'],version='dr2'):
if version == 'dr2':
dirname = '/data/des40.b/data/gaia/dr2/healpix/'
elif version == 'edr3':
dirname = '/data/des40.b/data/gaia/edr3/healpix/'
else:
raise Exception("Unrecognized Gaia version: %s"%version)
basename = 'GaiaSource_%05d.fits'
#print(dirname)
pixels = [hpx]
filenames = [os.path.join(dirname,basename%p) for p in pixels]
filenames = [f for f in filenames if os.path.exists(f)]
cat = load_infiles(filenames,columns=columns)
return cat
def rms_photometry(catfile,nside=64,band=None,plot=False):
""" Calculate photometric repeatability """
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
columns = ['RA','DEC']
spread,nepochs = bfields(['WAVG_SPREAD_MODEL','NEPOCHS'],band)
mag,magerr,magrms = bfields(['WAVG_MAG_PSF','WAVG_MAGERR_PSF','WAVG_MAGRMS_PSF'],band)
columns += [spread, nepochs, mag, magerr, magrms]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
msg = '%s (RA,DEC) = %.2f,%.2f'%(os.path.basename(catfile),ra,dec)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < r < 20 and 0.0 < (g-i) < 1.5
sel = (np.fabs(cat[spread]) < 0.002) & \
(cat[mag] > 16) & (cat[mag] < 18) &\
(cat[magrms] < 90) &\
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
pix = ang2pix(nside,cat['RA'],cat['DEC'])
upix = np.unique(pix)
stat = nd.median(cat[magrms],labels=pix,index=upix)
if False:
plt.figure()
plt.hist(cat[magrms],bins=50)
import pdb; pdb.set_trace()
return upix,stat
def distance(args,plot=False):
nice = os.nice(0)
os.nice(10-nice)
pix,nside = args
catfile = glob.glob('cat/*_%05d.fits'%pix)[0]
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
print(catfile)
columns = [OBJECT_ID,'RA','DEC']
spread,mag,nepochs = bfields(['WAVG_SPREAD_MODEL','MAG_PSF','NEPOCHS'],band)
columns += [spread,mag,nepochs]
cat = load_infiles([catfile],columns)
sel = (np.fabs(cat[spread])<0.002)&(cat[mag]>16)&(cat[mag]<22)&(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
print("WARNING: No catalog objects passing selection")
return np.array([],dtype=int), np.array([])
ra,dec = cat['RA'],cat['DEC']
m = ugali.utils.projector.match(ra,dec,ra,dec,nnearest=2)
sep = m[-1]
hpx = ang2pix(nside,ra,dec)
peak = nd.median(sep,labels=hpx,index=np.unique(hpx))
if plot:
plt.figure()
draw_angsep(sep)
if isinstance(plot,basestring):
outfile = plot
plt.savefig(outfile,bbox_inches='tight')
return hpx,peak
def read_refcat(ra,dec,radius=1.5):
""" Read the reference catalog in a localized region.
Parameters
----------
ra : ra (deg)
dec : dec (deg)
radius : regional radius (deg)
Returns
-------
refcat : reference catalog columns
"""
nside = 32
pixels = ang2disc(nside, ra, dec, radius, inclusive=True)
dirname = '/data/des40.b/data/atlas-refcat2/healpix'
basename = 'atlas-refcat2_%05d.fits'
filenames = [os.path.join(dirname,basename%p) for p in pixels]
columns = REFCAT_COLUMNS
refcat = utils.load_infiles(filenames,columns=columns)
return refcat
def gaia_photometry(catfile,nside=64,band=None,plot=False,version='edr3'):
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
#columns = [OBJECT_ID,'RA','DEC']
columns = ['RA','DEC']
spread,nepochs = ['WAVG_SPREAD_MODEL_R','NEPOCHS_R']
mag_g,mag_r,mag_i,mag_z = bfields(['MAG_PSF'],['g','r','i','z'])
#mag_g,mag_r,mag_i,mag_z = bfields(['WAVG_MAG_PSF'],['g','r','i','z'])
columns += [spread, nepochs, mag_g, mag_r, mag_i, mag_z]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
radius = np.degrees(hp.max_pixrad(NSIDE))
msg = '%s (RA,DEC,RAD) = %.2f,%.2f,%.2f'%(os.path.basename(catfile),ra,dec,radius)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < r < 20 and 0.0 < (g-i) < 1.5
sel = (np.fabs(cat[spread])<0.002) & \
(cat[mag_g]<90) & (cat[mag_r]<90) & (cat[mag_i]<90) & (cat[mag_z]<90) & \
(cat[mag_r]>16) & (cat[mag_r]<20) & \
((cat[mag_g] - cat[mag_i]) > 0.0) & \
((cat[mag_g] - cat[mag_i]) < 1.5) & \
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
#msg = "Getting external catalog: %s"%catalog
ext = get_gaia_catalog(hpx,version=version)
m = match_query(cat['RA'],cat['DEC'],ext['RA'],ext['DEC'])
# Use a fairly wide matching radius (2 arcsec)
cut = 1.0
sel = m[-1]*3600. < cut
cat_match = cat[m[0][sel]]
ext_match = ext[m[1][sel]]
cat_G = gaia_transform(cat_match[mag_g],cat_match[mag_r],cat_match[mag_i],cat_match[mag_z],
version=version)
# Need to put Gaia flux onto the AB system
ext_G = -2.5 * np.log10(ext_match['PHOT_G_MEAN_FLUX']) + 25.7934
diff = cat_G - ext_G
pix = ang2pix(nside,cat_match['RA'],cat_match['DEC'])
upix = np.unique(pix)
stat = nd.median(diff,labels=pix,index=upix)
if False:
plt.figure()
plt.hist(cat_G - ext_G)
import pdb; pdb.set_trace()
return upix,stat
def internal_astrometry(catfile,hpxfile,nside=128,band='r',plot=False):
""" Calculate internal relative astrometry.
Parameters
----------
catfile : merged catalog file
hpxfile : single epoch catalog file(s)
nside : nside for calculation
band : band to use
plot : plot output
Returns
-------
stats : output statistics
"""
nice = os.nice(0)
os.nice(10-nice)
if band=='all': band = 'r'
#print catfile,hpxfile,nside
#catfile = glob.glob('cat/*_%05d.fits'%pix)[0]
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
columns = [OBJECT_ID,'RA','DEC']
spread,mag,nepochs = bfields(['WAVG_SPREAD_MODEL','MAG_PSF','NEPOCHS'],band)
columns += [spread,mag,nepochs]
cat = load_infiles([catfile],columns)
# Select stars with 17 < mag < 21
sel = (np.fabs(cat[spread])<0.002) & \
(cat[mag]>17) & \
(cat[mag]<21) & \
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
print("WARNING: No objects passing selection in: %s"%catfile)
return np.array([],dtype=int), np.array([])
#hpxfiles = glob.glob('hpx/%s/*_%05d.fits'%(band,pix))
hpx = load_infiles(hpxfile, [OBJECT_ID, 'RA', 'DEC'])
hpx = hpx[np.in1d(hpx[OBJECT_ID],cat[OBJECT_ID])]
if len(hpx) == 0:
print("WARNING: No matched objects in: %s"%hpxfile)
return np.array([],dtype=int), np.array([])
#keyfile = 'key/key_hpx_%05d.fits'%pix
#key = load_infiles([keyfile],[OBJECT_ID,'FILENAME','OBJECT_NUMBER'])
#key = key[np.in1d(key[OBJECT_ID],cat[OBJECT_ID])]
#
#key_id = np.char.add(key['FILENAME'],key['OBJECT_NUMBER'].astype(str))
#hpx_id = np.char.add(hpx['FILENAME'],hpx['OBJECT_NUMBER'].astype(str))
#
#hpx = hpx[np.in1d(hpx_id,key_id)]
uid,inv,cts = np.unique(hpx[OBJECT_ID],False,True,True)
# Make sure that the order matches between coadd and single epoch.
if not np.all(uid == cat[OBJECT_ID]):
cat = cat[np.in1d(cat[OBJECT_ID],hpx[OBJECT_ID])]
if not np.all(uid == cat[OBJECT_ID]):
cat = cat[np.argsort(cat[OBJECT_ID])]
assert np.all(uid == cat[OBJECT_ID])
ra,dec = cat['RA'][inv],cat['DEC'][inv]
sepdeg = angsep(ra,dec,hpx['RA'],hpx['DEC'])
sepsec = sepdeg * 3600.
sepmas = sepsec * 1000.
sel = [sepsec > 1e-5]
sep = sepmas[sel]
pix = ang2pix(nside,ra[sel],dec[sel])
upix = np.unique(pix)
peak = nd.median(sep,labels=pix,index=upix)
if plot:
plt.figure()
draw_angsep(sep)
if isinstance(plot,basestring):
outfile = plot
plt.savefig(outfile,bbox_inches='tight')
return upix,peak
def external_astrometry(catfile,catalog='GAIA',nside=64,band='r',plot=False):
#nice = os.nice(0)
#os.nice(10-nice)
if band=='all': band = 'r'
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
columns = [OBJECT_ID,'RA','DEC']
spread,mag,nepochs = bfields(['WAVG_SPREAD_MODEL','MAG_PSF','NEPOCHS'],band)
columns += [spread,mag,nepochs]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
radius = np.degrees(healpy.max_pixrad(NSIDE))
msg = '%s (RA,DEC,RAD) = %.2f,%.2f,%.2f'%(os.path.basename(catfile),ra,dec,radius)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < mag < 21
sel = (np.fabs(cat[spread])<0.002) & \
(cat[mag]>16) & \
(cat[mag]<21) & \
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
#print "Getting external catalog: %s"%catalog
if catalog in list(CATALOGS.keys()):
ext = get_vizier_catalog(ra,dec,radius,**CATALOGS[catalog])
else:
ext = get_local_catalog(ra,dec,radius,catalog)
m = match_query(cat['RA'],cat['DEC'],ext['_RAJ2000'],ext['_DEJ2000'])
# Use a fairly wide matching radius (2 arcsec)
cut = 2.0
sel = m[-1]*3600. < cut
sepdeg = m[-1][sel]
sepsec = m[-1][sel] * 3600.
sepmas = sepsec * 1000.
sep = sepmas
pix = ang2pix(nside,cat['RA'][sel],cat['DEC'][sel])
upix = np.unique(pix)
try:
peak = nd.median(sep,labels=pix,index=upix)
except ValueError:
import pdb; pdb.set_trace()
if plot:
plt.figure()
draw_angsep(sep,bins=np.linspace(0,cut*1000.,101))
if isinstance(plot,basestring):
outfile = plot
plt.savefig(outfile,bbox_inches='tight')
#return cat,ext,m
return upix,peak
parser.add_argument('-f','--force',action='store_true')
parser.add_argument('-v','--verbose',action='store_true')
parser.add_argument('-b','--bands',default=None,action='append')
parser.add_argument('--min-bands',default=None,type=int)
opts = parser.parse_args()
if vars(opts).get('verbose'): logger.setLevel(logger.DEBUG)
if opts.bands: BANDS = opts.bands
if opts.min_bands: MINBANDS = opts.min_bands
if os.path.exists(opts.outfile) and not opts.force:
logger.warning("Found %s; skipping..."%opts.outfile)
sys.exit()
logger.info("Loading files: %s"%opts.infiles)
data = load_infiles(opts.infiles,INPUT_COLS)
logger.info("All objects: %i"%len(data))
good = good_objects(data)
logger.info("Good objects: %i"%len(good))
if len(good) == 0:
logger.warning("No good objects found; exiting...")
sys.exit()
cat,key = coadd_objects(good,bands=BANDS)
logger.info("Unique objects: %i"%len(cat))
catalog,keys = quality_cuts(cat,key)
check_keys(catalog,keys)
logger.info("Quality objects: %i"%len(catalog))