def pixelizeCatalog(infiles, config, force=False):
"""
Break catalog up into a set of healpix files.
"""
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
outdir = mkdir(config['catalog']['dirname'])
filenames = config.getFilenames()
for ii,infile in enumerate(infiles):
logger.info('(%i/%i) %s'%(ii+1, len(infiles), infile))
f = pyfits.open(infile)
data = f[1].data
header = f[1].header
logger.info("%i objects found"%len(data))
if not len(data): continue
glon,glat = cel2gal(data['RA'],data['DEC'])
catalog_pix = ang2pix(nside_catalog,glon,glat,coord='GAL')
pixel_pix = ang2pix(nside_pixel,glon,glat,coord='GAL')
names = [n.upper() for n in data.columns.names]
ra_idx = names.index('RA'); dec_idx = names.index('DEC')
idx = ra_idx if ra_idx > dec_idx else dec_idx
catalog_pix_name = 'PIX%i'%nside_catalog
pixel_pix_name = 'PIX%i'%nside_pixel
coldefs = pyfits.ColDefs(
[pyfits.Column(name='GLON',format='1D',array=glon),
pyfits.Column(name='GLAT',format='1D',array=glat),
pyfits.Column(name=catalog_pix_name,format='1J',array=catalog_pix),
pyfits.Column(name=pixel_pix_name ,format='1J',array=pixel_pix)]
)
hdu = pyfits.new_table(data.columns[:idx+1]+coldefs+data.columns[idx+1:])
table = hdu.data
for pix in numpy.unique(catalog_pix):
logger.debug("Processing pixel %s"%pix)
outfile = filenames.data['catalog'][pix]
if not os.path.exists(outfile):
logger.debug("Creating %s"%outfile)
names = [n.upper() for n in table.columns.names]
formats = table.columns.formats
columns = [pyfits.Column(n,f) for n,f in zip(names,formats)]
out = pyfits.HDUList([pyfits.PrimaryHDU(),pyfits.new_table(columns)])
out[1].header['NSIDE'] = nside_catalog
out[1].header['PIX'] = pix
out.writeto(outfile)
hdulist = pyfits.open(outfile,mode='update')
t1 = hdulist[1].data
# Could we speed up with sorting and indexing?
t2 = table[ table[catalog_pix_name] == pix ]
nrows1 = t1.shape[0]
nrows2 = t2.shape[0]
nrows = nrows1 + nrows2
out = pyfits.new_table(t1.columns, nrows=nrows)
for name in t1.columns.names:
out.data.field(name)[nrows1:]=t2.field(name)
hdulist[1] = out
logger.debug("Writing %s"%outfile)
hdulist.flush()
hdulist.close()
def pixelizeCatalog(infiles, config, force=False):
"""
Break catalog into chunks by healpix pixel.
Parameters:
-----------
infiles : List of input files
config : Configuration file
force : Overwrite existing files (depricated)
Returns:
--------
None
"""
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
outdir = mkdir(config['catalog']['dirname'])
filenames = config.getFilenames()
for i, filename in enumerate(infiles):
logger.info('(%i/%i) %s' % (i + 1, len(infiles), filename))
data = fitsio.read(filename)
logger.info("%i objects found" % len(data))
if not len(data): continue
glon, glat = cel2gal(data['RA'], data['DEC'])
cat_pix = ang2pix(nside_catalog, glon, glat)
pix_pix = ang2pix(nside_pixel, glon, glat)
cat_pix_name = 'PIX%i' % nside_catalog
pix_pix_name = 'PIX%i' % nside_pixel
data = mlab.rec_append_fields(
data,
names=['GLON', 'GLAT', cat_pix_name, pix_pix_name],
arrs=[glon, glat, cat_pix, pix_pix],
dtypes=['f4', 'f4', int, int])
for pix in np.unique(cat_pix):
logger.debug("Processing pixel %s" % pix)
arr = data[cat_pix == pix]
outfile = filenames.data['catalog'][pix]
if not os.path.exists(outfile):
logger.debug("Creating %s" % outfile)
out = fitsio.FITS(outfile, mode='rw')
out.write(arr)
hdr = ugali.utils.healpix.header_odict(nside=nside_catalog,
coord='G')
for key in ['PIXTYPE', 'ORDERING', 'NSIDE', 'COORDSYS']:
out[1].write_key(*list(hdr[key].values()))
out[1].write_key('PIX',
pix,
comment='HEALPIX pixel for this file')
else:
out = fitsio.FITS(outfile, mode='rw')
out[1].append(arr)
logger.debug("Writing %s" % outfile)
out.close()
def inFootprint(footprint,ra,dec):
"""
Check if set of ra,dec combinations are in footprint.
Careful, input files must be in celestial coordinates.
filename : Either healpix map or mangle polygon file
ra,dec : Celestial coordinates
Returns:
inside : boolean array of coordinates in footprint
"""
if footprint is None:
return np.ones(len(ra),dtype=bool)
try:
if isinstance(footprint,str) and os.path.exists(footprint):
filename = footprint
#footprint = hp.read_map(filename,verbose=False)
#footprint = fitsio.read(filename)['I'].ravel()
footprint = read_map(filename)
nside = hp.npix2nside(len(footprint))
pix = ang2pix(nside,ra,dec)
inside = (footprint[pix] > 0)
except IOError:
logger.warning("Failed to load healpix footprint; trying to use mangle...")
inside = inMangle(filename,ra,dec)
return inside
def inFootprint(footprint,ra,dec):
"""
Check if set of ra,dec combinations are in footprint.
Careful, input files must be in celestial coordinates.
filename : Either healpix map or mangle polygon file
ra,dec : Celestial coordinates
Returns:
inside : boolean array of coordinates in footprint
"""
if footprint is None:
return np.ones(len(ra),dtype=bool)
try:
if isinstance(footprint,str) and os.path.exists(footprint):
filename = footprint
#footprint = hp.read_map(filename,verbose=False)
#footprint = fitsio.read(filename)['I'].ravel()
footprint = read_map(filename)
nside = hp.npix2nside(len(footprint))
pix = ang2pix(nside,ra,dec)
inside = (footprint[pix] > 0)
except IOError:
logger.warning("Failed to load healpix footprint; trying to use mangle...")
inside = inMangle(filename,ra,dec)
return inside
def clip_catalog(self):
# ROI-specific catalog
logger.debug("Clipping full catalog...")
cut_observable = self.mask.restrictCatalogToObservableSpace(
self.catalog_full)
# All objects within disk ROI
logger.debug("Creating roi catalog...")
self.catalog_roi = self.catalog_full.applyCut(cut_observable)
self.catalog_roi.project(self.roi.projector)
self.catalog_roi.spatialBin(self.roi)
# All objects interior to the background annulus
logger.debug("Creating interior catalog...")
cut_interior = np.in1d(
ang2pix(self.config['coords']['nside_pixel'], self.catalog_roi.lon,
self.catalog_roi.lat), self.roi.pixels_interior)
#cut_interior = self.roi.inInterior(self.catalog_roi.lon,self.catalog_roi.lat)
self.catalog_interior = self.catalog_roi.applyCut(cut_interior)
self.catalog_interior.project(self.roi.projector)
self.catalog_interior.spatialBin(self.roi)
# Set the default catalog
#logger.info("Using interior ROI for likelihood calculation")
self.catalog = self.catalog_interior
def toy_background(self, mc_source_id=2, seed=None):
"""
Quick uniform background generation.
"""
logger.info("Running toy background simulation...")
size = 20000
nstar = np.random.poisson(size)
#np.random.seed(0)
logger.info("Simulating %i background stars..." % nstar)
### # Random points from roi pixels
### idx = np.random.randint(len(self.roi.pixels)-1,size=nstar)
### pix = self.roi.pixels[idx]
# Random points drawn from subpixels
logger.info("Generating uniform positions...")
idx = np.random.randint(0, len(self.subpix) - 1, size=nstar)
lon, lat = pix2ang(self.nside_subpixel, self.subpix[idx])
pix = ang2pix(self.nside_pixel, lon, lat)
lon, lat = pix2ang(self.nside_pixel, pix)
# Single color
#mag_1 = 19.05*np.ones(len(pix))
#mag_2 = 19.10*np.ones(len(pix))
# Uniform in color
logger.info("Generating uniform CMD...")
mag_1 = np.random.uniform(self.config['mag']['min'],
self.config['mag']['max'],
size=nstar)
color = np.random.uniform(self.config['color']['min'],
self.config['color']['max'],
size=nstar)
mag_2 = mag_1 - color
# There is probably a better way to do this step without creating the full HEALPix map
mask = -1. * numpy.ones(healpy.nside2npix(self.nside_pixel))
mask[self.roi.pixels] = self.mask.mask_1.mask_roi_sparse
mag_lim_1 = mask[pix]
mask = -1. * numpy.ones(healpy.nside2npix(self.nside_pixel))
mask[self.roi.pixels] = self.mask.mask_2.mask_roi_sparse
mag_lim_2 = mask[pix]
#mag_err_1 = 1.0*np.ones(len(pix))
#mag_err_2 = 1.0*np.ones(len(pix))
mag_err_1 = self.photo_err_1(mag_lim_1 - mag_1)
mag_err_2 = self.photo_err_2(mag_lim_2 - mag_2)
mc_source_id = mc_source_id * numpy.ones(len(mag_1))
select = (mag_lim_1 > mag_1) & (mag_lim_2 > mag_2)
hdu = ugali.observation.catalog.makeHDU(
self.config, mag_1[select], mag_err_1[select], mag_2[select],
mag_err_2[select], lon[select], lat[select], mc_source_id[select])
catalog = ugali.observation.catalog.Catalog(self.config, data=hdu.data)
return catalog
def write(self, outfile):
"""
Save the likelihood results as a sparse HEALPix map.
"""
data = odict()
data['PIXEL'] = self.roi.pixels_target
# Full data output (too large for survey)
if self.config['scan']['full_pdf']:
data['LOG_LIKELIHOOD'] = self.log_likelihood_sparse_array.T
data['RICHNESS'] = self.richness_sparse_array.T
data['RICHNESS_LOWER'] = self.richness_lower_sparse_array.T
data['RICHNESS_UPPER'] = self.richness_upper_sparse_array.T
data['RICHNESS_LIMIT'] = self.richness_upper_limit_sparse_array.T
#data['STELLAR_MASS']=self.stellar_mass_sparse_array.T
data[
'FRACTION_OBSERVABLE'] = self.fraction_observable_sparse_array.T
else:
data['LOG_LIKELIHOOD'] = self.log_likelihood_sparse_array.T
data['RICHNESS'] = self.richness_sparse_array.T
data[
'FRACTION_OBSERVABLE'] = self.fraction_observable_sparse_array.T
# Convert to 32bit float
for k in data.keys()[1:]:
data[k] = data[k].astype('f4', copy=False)
# Stellar mass can be calculated from STELLAR * RICHNESS
header = odict()
header['STELLAR'] = round(self.stellar_mass_conversion, 8)
header['LKDNSIDE'] = self.config['coords']['nside_likelihood']
header['LKDPIX'] = ang2pix(self.config['coords']['nside_likelihood'],
self.roi.lon, self.roi.lat)
header['NROI'] = self.roi.inROI(self.loglike.catalog_roi.lon,
self.loglike.catalog_roi.lat).sum()
header['NANNULUS'] = self.roi.inAnnulus(
self.loglike.catalog_roi.lon, self.loglike.catalog_roi.lat).sum()
header['NINSIDE'] = self.roi.inInterior(
self.loglike.catalog_roi.lon, self.loglike.catalog_roi.lat).sum()
header['NTARGET'] = self.roi.inTarget(
self.loglike.catalog_roi.lon, self.loglike.catalog_roi.lat).sum()
# Flatten if there is only a single distance modulus
# ADW: Is this really what we want to do?
if len(self.distance_modulus_array) == 1:
for key in data:
data[key] = data[key].flatten()
logger.info("Writing %s..." % outfile)
write_partial_map(outfile,
data,
nside=self.config['coords']['nside_pixel'],
header=header,
clobber=True)
fitsio.write(outfile,
dict(DISTANCE_MODULUS=self.distance_modulus_array.astype(
'f4', copy=False)),
extname='DISTANCE_MODULUS',
clobber=False)
def finalizeObjects(self, objects):
objs = numpy.recarray(len(objects),
dtype=[('NAME','S24'),
('TS','f4'),
('GLON','f4'),
('GLAT','f4'),
('RA','f4'),
('DEC','f4'),
('MODULUS','f4'),
('DISTANCE','f4'),
('RICHNESS','f4'),
('MASS','f4'),
('NANNULUS','i4'),
('NINTERIOR','i4'),
])
objs['TS'] = self.values[objects['IDX_MAX'],objects['ZIDX_MAX']]
lon,lat = objects['X_MAX'],objects['Y_MAX']
coordsys = self.config['coords']['coordsys']
if coordsys.lower() == 'gal':
print("GAL coordintes")
objs['GLON'],objs['GLAT'] = lon,lat
objs['RA'],objs['DEC'] = gal2cel(lon,lat)
else:
print("CEL coordintes")
objs['RA'],objs['DEC'] = lon,lat
objs['GLON'],objs['GLAT'] = cel2gal(lon,lat)
modulus = objects['Z_MAX']
objs['MODULUS'] = modulus
objs['DISTANCE'] = mod2dist(modulus)
nside = healpy.npix2nside(len(self.nannulus))
pix = ang2pix(nside,lon,lat)
richness = self.richness[objects['IDX_MAX'],objects['ZIDX_MAX']]
objs['RICHNESS'] = richness
objs['MASS'] = richness * self.stellar[pix]
objs['NANNULUS'] = self.nannulus[pix].astype(int)
objs['NINTERIOR'] = self.ninterior[pix].astype(int)
# Default name formatting
# http://cdsarc.u-strasbg.fr/ftp/pub/iau/
# http://cds.u-strasbg.fr/vizier/Dic/iau-spec.htx
fmt = "J%(hour)02i%(hmin)04.1f%(deg)+03i%(dmin)02i"
for obj,_ra,_dec in zip(objs,objs['RA'],objs['DEC']):
hms = dec2hms(_ra); dms = dec2dms(_dec)
params = dict(hour=hms[0],hmin=hms[1]+hms[2]/60.,
deg=dms[0],dmin=dms[1]+dms[2]/60.)
obj['NAME'] = fmt%params
out = recfuncs.merge_arrays([objs,objects],usemask=False,
asrecarray=True,flatten=True)
return out
def write(self, outfile):
"""
Save the likelihood results as a sparse HEALPix map.
"""
data = odict()
data['PIXEL']=self.roi.pixels_target
# Full data output (too large for survey)
if self.config['scan']['full_pdf']:
data['LOG_LIKELIHOOD']=self.log_likelihood_sparse_array.T
data['RICHNESS']=self.richness_sparse_array.T
data['RICHNESS_LOWER']=self.richness_lower_sparse_array.T
data['RICHNESS_UPPER']=self.richness_upper_sparse_array.T
data['RICHNESS_LIMIT']=self.richness_upper_limit_sparse_array.T
#data['STELLAR_MASS']=self.stellar_mass_sparse_array.T
data['FRACTION_OBSERVABLE']=self.fraction_observable_sparse_array.T
else:
data['LOG_LIKELIHOOD']=self.log_likelihood_sparse_array.T
data['RICHNESS']=self.richness_sparse_array.T
data['FRACTION_OBSERVABLE']=self.fraction_observable_sparse_array.T
# Convert to 32bit float
for k in list(data.keys())[1:]:
data[k] = data[k].astype('f4',copy=False)
# Stellar mass can be calculated from STELLAR * RICHNESS
header = odict()
header['STELLAR']=round(self.stellar_mass_conversion,8)
header['LKDNSIDE']=self.config['coords']['nside_likelihood']
header['LKDPIX']=ang2pix(self.config['coords']['nside_likelihood'],
self.roi.lon,self.roi.lat)
header['NROI']=self.roi.inROI(self.loglike.catalog_roi.lon,
self.loglike.catalog_roi.lat).sum()
header['NANNULUS']=self.roi.inAnnulus(self.loglike.catalog_roi.lon,
self.loglike.catalog_roi.lat).sum()
header['NINSIDE']=self.roi.inInterior(self.loglike.catalog_roi.lon,
self.loglike.catalog_roi.lat).sum()
header['NTARGET']=self.roi.inTarget(self.loglike.catalog_roi.lon,
self.loglike.catalog_roi.lat).sum()
# Flatten if there is only a single distance modulus
# ADW: Is this really what we want to do?
if len(self.distance_modulus_array) == 1:
for key in data:
data[key] = data[key].flatten()
logger.info("Writing %s..."%outfile)
write_partial_map(outfile,data,
nside=self.config['coords']['nside_pixel'],
header=header,
clobber=True
)
fitsio.write(outfile,
dict(DISTANCE_MODULUS=self.distance_modulus_array.astype('f4',copy=False)),
extname='DISTANCE_MODULUS',
clobber=False)
def depth(infile,nside=NSIDE,signal_to_noise=10.):
MAGS = bfields('MAG_PSF',BANDS)
MAGERRS = bfields('MAGERR_PSF',BANDS)
SPREADS = bfields('WAVG_SPREAD_MODEL',BANDS)
# From propagation of errors:
# mag = -2.5 * log10(flux)
# magerr = -2.5/ln(10) * fluxerr/flux
mag_snr = (2.5/np.log(10)) / (signal_to_noise)
logger.info(infile)
ret = dict()
for band,mag,magerr,spread in zip(BANDS,MAGS,MAGERRS,SPREADS):
data = fitsio.read(infile,columns=['RA','DEC',mag,magerr,spread])
h, edges = np.histogram(data[mag], bins=np.arange(17, 30, 0.1))
mag_bright_end = edges[np.argmax(h)] - 3.
cut = (np.fabs(data[spread]) < 0.002) & (data[mag] > mag_bright_end) & (data[mag] < 30.)
d = data[cut]
if len(d) < 2:
logger.warn("Insufficent objects in %s-band"%band)
ret[band] = [np.array([],dtype=int),np.array([])]
continue
pix = ang2pix(nside,d['RA'],d['DEC'])
match = ugali.utils.projector.match(d['RA'],d['DEC'],d['RA'],d['DEC'],nnearest=2)
delta_mag = d[mag][match[1]] - d[mag][match[0]]
delta_log_magerr = np.log10(d[magerr][match[1]]) - np.log10(d[magerr][match[0]])
old = np.seterr(divide='ignore',invalid='ignore')
ratio = delta_log_magerr / delta_mag
cut_nan_inf = np.isfinite(ratio) & (delta_mag > 0.5)
np.seterr(**old)
if cut_nan_inf.sum() < 2:
logger.warn("Insufficent objects in %s-band"%band)
ret[band] = [np.array([],dtype=int),np.array([])]
continue
kde = scipy.stats.gaussian_kde(ratio[cut_nan_inf])
values = np.linspace(0., 1., 1000)
kde_values = kde.evaluate(values)
slope = values[np.argmax(kde_values)]
maglims = d[mag] - ((np.log10(d[magerr]) - np.log10(mag_snr)) / slope)
hpx = np.unique(pix)
maglim = nd.median(maglims,labels=pix,index=hpx)
ret[band] = [hpx,maglim]
return ret
def toy_background(self,mc_source_id=2,seed=None):
"""
Quick uniform background generation.
"""
logger.info("Running toy background simulation...")
size = 20000
nstar = np.random.poisson(size)
#np.random.seed(0)
logger.info("Simulating %i background stars..."%nstar)
### # Random points from roi pixels
### idx = np.random.randint(len(self.roi.pixels)-1,size=nstar)
### pix = self.roi.pixels[idx]
# Random points drawn from subpixels
logger.info("Generating uniform positions...")
idx = np.random.randint(0,len(self.subpix)-1,size=nstar)
lon,lat = pix2ang(self.nside_subpixel,self.subpix[idx])
pix = ang2pix(self.nside_pixel, lon, lat)
lon,lat = pix2ang(self.nside_pixel,pix)
# Single color
#mag_1 = 19.05*np.ones(len(pix))
#mag_2 = 19.10*np.ones(len(pix))
# Uniform in color
logger.info("Generating uniform CMD...")
mag_1 = np.random.uniform(self.config['mag']['min'],self.config['mag']['max'],size=nstar)
color = np.random.uniform(self.config['color']['min'],self.config['color']['max'],size=nstar)
mag_2 = mag_1 - color
# There is probably a better way to do this step without creating the full HEALPix map
mask = -1. * numpy.ones(healpy.nside2npix(self.nside_pixel))
mask[self.roi.pixels] = self.mask.mask_1.mask_roi_sparse
mag_lim_1 = mask[pix]
mask = -1. * numpy.ones(healpy.nside2npix(self.nside_pixel))
mask[self.roi.pixels] = self.mask.mask_2.mask_roi_sparse
mag_lim_2 = mask[pix]
#mag_err_1 = 1.0*np.ones(len(pix))
#mag_err_2 = 1.0*np.ones(len(pix))
mag_err_1 = self.photo_err_1(mag_lim_1 - mag_1)
mag_err_2 = self.photo_err_2(mag_lim_2 - mag_2)
mc_source_id = mc_source_id * numpy.ones(len(mag_1))
select = (mag_lim_1>mag_1)&(mag_lim_2>mag_2)
hdu = ugali.observation.catalog.makeHDU(self.config,mag_1[select],mag_err_1[select],
mag_2[select],mag_err_2[select],
lon[select],lat[select],mc_source_id[select])
catalog = ugali.observation.catalog.Catalog(self.config, data=hdu.data)
return catalog
def finalizeObjects(self, objects):
objs = numpy.recarray(len(objects),
dtype=[('NAME','S24'),
('TS','f4'),
('GLON','f4'),
('GLAT','f4'),
('RA','f4'),
('DEC','f4'),
('MODULUS','f4'),
('DISTANCE','f4'),
('RICHNESS','f4'),
('MASS','f4'),
('NANNULUS','i4'),
('NINTERIOR','i4'),
])
objs['TS'] = self.values[objects['IDX_MAX'],objects['ZIDX_MAX']]
glon,glat = objects['X_MAX'],objects['Y_MAX']
objs['GLON'],objs['GLAT'] = glon,glat
ra,dec = gal2cel(glon,glat)
objs['RA'],objs['DEC'] = ra,dec
modulus = objects['Z_MAX']
objs['MODULUS'] = modulus
objs['DISTANCE'] = mod2dist(modulus)
#ninterior = ugali.utils.skymap.readSparseHealpixMap(self.roifile,'NINSIDE')
#nannulus = ugali.utils.skymap.readSparseHealpixMap(self.roifile,'NANNULUS')
#stellar = ugali.utils.skymap.readSparseHealpixMap(self.roifile,'STELLAR')
nside = healpy.npix2nside(len(self.nannulus))
pix = ang2pix(nside,glon,glat)
richness = self.richness[objects['IDX_MAX'],objects['ZIDX_MAX']]
objs['RICHNESS'] = richness
objs['MASS'] = richness * self.stellar[pix]
objs['NANNULUS'] = self.nannulus[pix].astype(int)
objs['NINTERIOR'] = self.ninterior[pix].astype(int)
# Default name formatting
# http://cdsarc.u-strasbg.fr/ftp/pub/iau/
# http://cds.u-strasbg.fr/vizier/Dic/iau-spec.htx
fmt = "J%(hour)02i%(hmin)04.1f%(deg)+03i%(dmin)02i"
for obj,_ra,_dec in zip(objs,ra,dec):
hms = dec2hms(_ra); dms = dec2dms(_dec)
params = dict(hour=hms[0],hmin=hms[1]+hms[2]/60.,
deg=dms[0],dmin=dms[1]+dms[2]/60.)
obj['NAME'] = fmt%params
out = recfuncs.merge_arrays([objs,objects],usemask=False,asrecarray=True,flatten=True)
# This is safer than viewing as FITS_rec
return pyfits.new_table(out).data
def stellarDensity(infile, nside=256, lon_field='RA', lat_field='DEC'):
area = hp.nside2pixarea(nside,degrees=True)
logger.debug("Reading %s"%infile)
data = fitsio.read(infile,columns=[lon_field,lat_field])
lon,lat = data[lon_field],data[lat_field]
pix = ang2pix(nside,lon,lat)
counts = collections.Counter(pix)
pixels, number = np.array(sorted(counts.items())).T
density = number/area
return pixels, density
def sky(self,lon=None,lat=None,size=1):
logger.info("Generating %i random points..."%size)
# Random longitue and latitude
lon,lat = ugali.utils.stats.sky(lon,lat,size=10*size)
# Random healpix coordinates inside footprint
nside_pixel = self.config['coords']['nside_pixel']
pixels = ang2pix(nside_pixel,lon,lat)
if np.unique(pixels).size > 1:
inside = ugali.utils.skymap.inFootprint(self.config,pixels,nside=nside_pixel)
else:
inside = np.ones(len(pixels),dtype=bool)
return lon[inside][:size],lat[inside][:size]
def stellarDensity(infile, nside=256, lon_field='RA', lat_field='DEC'):
area = hp.nside2pixarea(nside, degrees=True)
logger.debug("Reading %s" % infile)
data = fitsio.read(infile, columns=[lon_field, lat_field])
lon, lat = data[lon_field], data[lat_field]
pix = ang2pix(nside, lon, lat)
counts = collections.Counter(pix)
pixels, number = np.array(sorted(counts.items())).T
density = number / area
return pixels, density
def sky(self,lon=None,lat=None,size=1):
logger.info("Generating %i random points..."%size)
# Random longitue and latitude
lon,lat = ugali.utils.stats.sky(lon,lat,size=10*size)
# Random healpix coordinates inside footprint
nside_pixel = self.config['coords']['nside_pixel']
pixels = ang2pix(nside_pixel,lon,lat)
if np.unique(pixels).size > 1:
inside = ugali.utils.skymap.inFootprint(self.config,pixels,nside=nside_pixel)
else:
inside = np.ones(len(pixels),dtype=bool)
return lon[inside][:size],lat[inside][:size]
def stellarDensity(infile, nside=2**8):
area = healpy.nside2pixarea(nside, degrees=True)
logger.debug("Reading %s" % infile)
data = fitsio.read(infile, columns=['GLON', 'GLAT'])
glon, glat = data['GLON'], data['GLAT']
pix = ang2pix(nside, glon, glat)
counts = collections.Counter(pix)
pixels, number = numpy.array(sorted(counts.items())).T
density = number / area
return pixels, density
def finalizeObjects(self, objects):
objs = np.recarray(len(objects),
dtype=[
('NAME', 'S24'),
('TS', 'f4'),
('GLON', 'f4'),
('GLAT', 'f4'),
('RA', 'f4'),
('DEC', 'f4'),
('MODULUS', 'f4'),
('DISTANCE', 'f4'),
('RICHNESS', 'f4'),
('MASS', 'f4'),
('NANNULUS', 'i4'),
('NINTERIOR', 'i4'),
])
objs['TS'] = self.values[objects['IDX_MAX'], objects['ZIDX_MAX']]
lon, lat = objects['X_MAX'], objects['Y_MAX']
coordsys = self.config['coords']['coordsys']
if coordsys.lower() == 'gal':
print("GAL coordintes")
objs['GLON'], objs['GLAT'] = lon, lat
objs['RA'], objs['DEC'] = gal2cel(lon, lat)
else:
print("CEL coordintes")
objs['RA'], objs['DEC'] = lon, lat
objs['GLON'], objs['GLAT'] = cel2gal(lon, lat)
modulus = objects['Z_MAX']
objs['MODULUS'] = modulus
objs['DISTANCE'] = mod2dist(modulus)
nside = healpy.npix2nside(len(self.nannulus))
pix = ang2pix(nside, lon, lat)
richness = self.richness[objects['IDX_MAX'], objects['ZIDX_MAX']]
objs['RICHNESS'] = richness
objs['MASS'] = richness * self.stellar[pix]
objs['NANNULUS'] = self.nannulus[pix].astype(int)
objs['NINTERIOR'] = self.ninterior[pix].astype(int)
objs['NAME'] = ang2iau(objs['RA'], objs['DEC'], coord='cel')
out = recfuncs.merge_arrays([objs, objects],
usemask=False,
asrecarray=True,
flatten=True)
return out
def satellite(self,stellar_mass,distance_modulus,mc_source_id=1,seed=None,**kwargs):
"""
Create a simulated satellite. Returns a catalog object.
"""
if seed is not None: np.random.seed(seed)
isochrone = kwargs.pop('isochrone',self.isochrone)
kernel = kwargs.pop('kernel',self.kernel)
for k,v in kwargs.items():
if k in kernel.params.keys(): setattr(kernel,k,v)
mag_1, mag_2 = isochrone.simulate(stellar_mass, distance_modulus)
lon, lat = kernel.simulate(len(mag_1))
logger.info("Simulating %i satellite stars..."%len(mag_1))
pix = ang2pix(self.config['coords']['nside_pixel'], lon, lat)
# There is probably a better way to do this step without creating the full HEALPix map
mask = -1. * numpy.ones(healpy.nside2npix(self.config['coords']['nside_pixel']))
mask[self.roi.pixels] = self.mask.mask_1.mask_roi_sparse
mag_lim_1 = mask[pix]
mask = -1. * numpy.ones(healpy.nside2npix(self.config['coords']['nside_pixel']))
mask[self.roi.pixels] = self.mask.mask_2.mask_roi_sparse
mag_lim_2 = mask[pix]
mag_err_1 = self.photo_err_1(mag_lim_1 - mag_1)
mag_err_2 = self.photo_err_2(mag_lim_2 - mag_2)
# Randomize magnitudes by their errors
mag_obs_1 = mag_1+numpy.random.normal(size=len(mag_1))*mag_err_1
mag_obs_2 = mag_2+numpy.random.normal(size=len(mag_2))*mag_err_2
#mag_obs_1 = mag_1
#mag_obs_2 = mag_2
#select = numpy.logical_and(mag_obs_1 < mag_lim_1, mag_obs_2 < mag_lim_2)
select = (mag_lim_1>mag_obs_1)&(mag_lim_2>mag_obs_2)
# Make sure objects lie within the original cmd (should also be done later...)
#select &= (ugali.utils.binning.take2D(self.mask.solid_angle_cmd, mag_obs_1 - mag_obs_2, mag_obs_1,self.roi.bins_color, self.roi.bins_mag) > 0)
#return mag_1_obs[cut], mag_2_obs[cut], lon[cut], lat[cut]
logger.info("Clipping %i simulated satellite stars..."%(~select).sum())
mc_source_id = mc_source_id * numpy.ones(len(mag_1))
hdu = ugali.observation.catalog.makeHDU(self.config,mag_obs_1[select],mag_err_1[select],
mag_obs_2[select],mag_err_2[select],
lon[select],lat[select],mc_source_id[select])
catalog = ugali.observation.catalog.Catalog(self.config, data=hdu.data)
return catalog
def satellite(self,stellar_mass,distance_modulus,mc_source_id=1,seed=None,**kwargs):
"""
Create a simulated satellite. Returns a catalog object.
"""
if seed is not None: np.random.seed(seed)
isochrone = kwargs.pop('isochrone',self.isochrone)
kernel = kwargs.pop('kernel',self.kernel)
for k,v in kwargs.items():
if k in kernel.params.keys(): setattr(kernel,k,v)
mag_1, mag_2 = isochrone.simulate(stellar_mass, distance_modulus)
lon, lat = kernel.simulate(len(mag_1))
logger.info("Simulating %i satellite stars..."%len(mag_1))
pix = ang2pix(self.config['coords']['nside_pixel'], lon, lat)
# There is probably a better way to do this step without creating the full HEALPix map
mask = -1. * numpy.ones(healpy.nside2npix(self.config['coords']['nside_pixel']))
mask[self.roi.pixels] = self.mask.mask_1.mask_roi_sparse
mag_lim_1 = mask[pix]
mask = -1. * numpy.ones(healpy.nside2npix(self.config['coords']['nside_pixel']))
mask[self.roi.pixels] = self.mask.mask_2.mask_roi_sparse
mag_lim_2 = mask[pix]
mag_err_1 = self.photo_err_1(mag_lim_1 - mag_1)
mag_err_2 = self.photo_err_2(mag_lim_2 - mag_2)
# Randomize magnitudes by their errors
mag_obs_1 = mag_1+numpy.random.normal(size=len(mag_1))*mag_err_1
mag_obs_2 = mag_2+numpy.random.normal(size=len(mag_2))*mag_err_2
#mag_obs_1 = mag_1
#mag_obs_2 = mag_2
#select = numpy.logical_and(mag_obs_1 < mag_lim_1, mag_obs_2 < mag_lim_2)
select = (mag_lim_1>mag_obs_1)&(mag_lim_2>mag_obs_2)
# Make sure objects lie within the original cmd (should also be done later...)
#select &= (ugali.utils.binning.take2D(self.mask.solid_angle_cmd, mag_obs_1 - mag_obs_2, mag_obs_1,self.roi.bins_color, self.roi.bins_mag) > 0)
#return mag_1_obs[cut], mag_2_obs[cut], lon[cut], lat[cut]
logger.info("Clipping %i simulated satellite stars..."%(~select).sum())
mc_source_id = mc_source_id * numpy.ones(len(mag_1))
hdu = ugali.observation.catalog.makeHDU(self.config,mag_obs_1[select],mag_err_1[select],
mag_obs_2[select],mag_err_2[select],
lon[select],lat[select],mc_source_id[select])
catalog = ugali.observation.catalog.Catalog(self.config, data=hdu.data)
return catalog
def spatialBin(self, roi):
"""
Calculate indices of ROI pixels corresponding to object locations.
"""
if hasattr(self,'pixel_roi_index') and hasattr(self,'pixel'):
logger.warning('Catalog alread spatially binned')
return
# ADW: Not safe to set index = -1 (since it will access last entry);
# np.inf would be better...
self.pixel = ang2pix(self.config['coords']['nside_pixel'],self.lon,self.lat)
self.pixel_roi_index = roi.indexROI(self.lon,self.lat)
logger.info("Found %i objects outside ROI"%(self.pixel_roi_index < 0).sum())
def surveyPixel(lon, lat, nside_pix, nside_subpix = None):
"""
Return the set of HEALPix pixels that cover the given coordinates at resolution nside.
Optionally return the set of subpixels within those pixels at resolution nside_subpix
"""
pix = np.unique(ang2pix(nside_pix, lon, lat))
if nside_subpix is None:
return pix
else:
subpix_array = []
for ii in range(0, len(pix)):
subpix = subpixel(pix[ii], nside_pix, nside_subpix)
subpix_array.append(subpix)
return pix, np.array(subpix_array)
def surveyPixel(lon, lat, nside_pix, nside_subpix=None):
"""
Return the set of HEALPix pixels that cover the given coordinates at resolution nside.
Optionally return the set of subpixels within those pixels at resolution nside_subpix
"""
pix = numpy.unique(ang2pix(nside_pix, lon, lat))
if nside_subpix is None:
return pix
else:
subpix_array = []
for ii in range(0, len(pix)):
subpix = subpixel(pix[ii], nside_pix, nside_subpix)
subpix_array.append(subpix)
return pix, numpy.array(subpix_array)
def stellarDensity(infile, nside=2**8):
area = healpy.nside2pixarea(nside,degrees=True)
f = pyfits.open(infile)
data = f[1].data
logger.debug("Reading %s"%infile)
glon,glat = data['GLON'],data['GLAT']
pix = ang2pix(nside,glon,glat,coord='GAL')
counts = collections.Counter(pix)
pixels, number = numpy.array(sorted(counts.items())).T
density = number/area
f.close()
return pixels, density
def spatialBin(self, roi):
"""
Calculate indices of ROI pixels corresponding to object locations.
"""
if hasattr(self,'pixel_roi_index') and hasattr(self,'pixel'):
logger.warning('Catalog alread spatially binned')
return
# ADW: Not safe to set index = -1 (since it will access last entry);
# np.inf would be better...
self.pixel = ang2pix(self.config['coords']['nside_pixel'],self.lon,self.lat)
self.pixel_roi_index = roi.indexROI(self.lon,self.lat)
if numpy.any(self.pixel_roi_index < 0):
logger.warning("Objects found outside ROI")
def plot_skymap(infiles,nside=1024):
counts = np.zeros(healpy.nside2npix(nside),dtype=int)
skymap = healpy.UNSEEN * np.ones(healpy.nside2npix(nside))
for i,f in enumerate(infiles):
if i%10 == 0: print "(%i/%i)"%(i,len(infiles))
d = fitsio.read(f,columns=['RA','DEC','MAG_PSF_G'])
sel = (d['MAG_PSF_G'] < 23)
d = d[sel]
pix,cts = np.unique(ang2pix(nside,d['RA'],d['DEC']),return_counts=True)
counts[pix] += cts
idx = np.where(counts>0)
pixarea = healpy.nside2pixarea(nside,degrees=True)
skymap[idx] = np.log10(counts[idx]/pixarea)
im = healpy.mollview(skymap,title='Object Density (g < 23)',return_projected_map = True)
healpy.graticule()
return counts,im
def rms_photometry(catfile,nside=64,band=None,plot=False):
""" Calculate photometric repeatability """
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
columns = ['RA','DEC']
spread,nepochs = bfields(['WAVG_SPREAD_MODEL','NEPOCHS'],band)
mag,magerr,magrms = bfields(['WAVG_MAG_PSF','WAVG_MAGERR_PSF','WAVG_MAGRMS_PSF'],band)
columns += [spread, nepochs, mag, magerr, magrms]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
msg = '%s (RA,DEC) = %.2f,%.2f'%(os.path.basename(catfile),ra,dec)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < r < 20 and 0.0 < (g-i) < 1.5
sel = (np.fabs(cat[spread]) < 0.002) & \
(cat[mag] > 16) & (cat[mag] < 18) &\
(cat[magrms] < 90) &\
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
pix = ang2pix(nside,cat['RA'],cat['DEC'])
upix = np.unique(pix)
stat = nd.median(cat[magrms],labels=pix,index=upix)
if False:
plt.figure()
plt.hist(cat[magrms],bins=50)
import pdb; pdb.set_trace()
return upix,stat
def teff(infile,nside=NSIDE,mode='median'):
TEFFS = bfields('TEFF',BANDS)
logger.info(infile)
ret = dict()
for band,teff in zip(BANDS,TEFFS):
data = fitsio.read(infile,columns=['RA','DEC',teff])
pix = ang2pix(nside,data['RA'],data['DEC'])
hpx = np.unique(pix)
if mode.lower() is 'median':
teff_value = nd.median(data[teff],labels=pix,index=hpx)
elif mode.lower() is 'mean':
teff_value = nd.mean(data[teff],labels=pix,index=hpx)
else:
msg = 'Unrecognized mode: %s'%mode
raise Exception(msg)
ret[band] = [hpx,maglim]
return ret
def inFootprint(filename,ra,dec):
"""
Check if set of ra,dec combinations are in footprint.
Careful, input files must be in celestial coordinates.
filename : Either healpix map or mangle polygon file
ra,dec : Celestial coordinates
Returns:
inside : boolean array of coordinates in footprint
"""
try:
footprint = healpy.read_map(filename,verbose=False)
nside = healpy.npix2nside(len(footprint))
pix = ang2pix(nside,ra,dec)
inside = (footprint[pix] > 0)
except IOError:
logger.warning("Failed to load healpix footprint; using MANGLE...")
inside = inMangle(filename,ra,dec)
return inside
def distance(args,plot=False):
nice = os.nice(0)
os.nice(10-nice)
pix,nside = args
catfile = glob.glob('cat/*_%05d.fits'%pix)[0]
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
print(catfile)
columns = [OBJECT_ID,'RA','DEC']
spread,mag,nepochs = bfields(['WAVG_SPREAD_MODEL','MAG_PSF','NEPOCHS'],band)
columns += [spread,mag,nepochs]
cat = load_infiles([catfile],columns)
sel = (np.fabs(cat[spread])<0.002)&(cat[mag]>16)&(cat[mag]<22)&(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
print("WARNING: No catalog objects passing selection")
return np.array([],dtype=int), np.array([])
ra,dec = cat['RA'],cat['DEC']
m = ugali.utils.projector.match(ra,dec,ra,dec,nnearest=2)
sep = m[-1]
hpx = ang2pix(nside,ra,dec)
peak = nd.median(sep,labels=hpx,index=np.unique(hpx))
if plot:
plt.figure()
draw_angsep(sep)
if isinstance(plot,basestring):
outfile = plot
plt.savefig(outfile,bbox_inches='tight')
return hpx,peak
def write(self, outfile):
"""
Save the likelihood fitting results as a sparse HEALPix map.
"""
# Full data output (too large for survey)
if self.config['scan']['full_pdf']:
data_dict = {'LOG_LIKELIHOOD': self.log_likelihood_sparse_array.transpose(),
'RICHNESS': self.richness_sparse_array.transpose(),
'RICHNESS_LOWER': self.richness_lower_sparse_array.transpose(),
'RICHNESS_UPPER': self.richness_upper_sparse_array.transpose(),
'RICHNESS_LIMIT': self.richness_upper_limit_sparse_array.transpose(),
#'STELLAR_MASS': self.stellar_mass_sparse_array.transpose(),
'FRACTION_OBSERVABLE': self.fraction_observable_sparse_array.transpose()}
else:
data_dict = {'LOG_LIKELIHOOD': self.log_likelihood_sparse_array.transpose(),
'RICHNESS': self.richness_sparse_array.transpose(),
'FRACTION_OBSERVABLE': self.fraction_observable_sparse_array.transpose()}
# Stellar Mass can be calculated from STELLAR * RICHNESS
header_dict = {
'STELLAR' : round(self.stellar_mass_conversion,8),
'LKDNSIDE': self.config['coords']['nside_likelihood'],
'LKDPIX' : ang2pix(self.config['coords']['nside_likelihood'],self.roi.lon,self.roi.lat),
'NROI' : self.roi.inROI(self.loglike.catalog_roi.lon,self.loglike.catalog_roi.lat).sum(),
'NANNULUS': self.roi.inAnnulus(self.loglike.catalog_roi.lon,self.loglike.catalog_roi.lat).sum(),
'NINSIDE' : self.roi.inInterior(self.loglike.catalog_roi.lon,self.loglike.catalog_roi.lat).sum(),
'NTARGET' : self.roi.inTarget(self.loglike.catalog_roi.lon,self.loglike.catalog_roi.lat).sum(),
}
# In case there is only a single distance modulus
if len(self.distance_modulus_array) == 1:
for key in data_dict:
data_dict[key] = data_dict[key].flatten()
ugali.utils.skymap.writeSparseHealpixMap(self.roi.pixels_target,
data_dict,
self.config['coords']['nside_pixel'],
outfile,
distance_modulus_array=self.distance_modulus_array,
coordsys='NULL', ordering='NULL',
header_dict=header_dict)
def clip_catalog(self):
# ROI-specific catalog
logger.debug("Clipping full catalog...")
cut_observable = self.mask.restrictCatalogToObservableSpace(self.catalog_full)
# All objects within disk ROI
logger.debug("Creating roi catalog...")
self.catalog_roi = self.catalog_full.applyCut(cut_observable)
self.catalog_roi.project(self.roi.projector)
self.catalog_roi.spatialBin(self.roi)
# All objects interior to the background annulus
logger.debug("Creating interior catalog...")
cut_interior = numpy.in1d(ang2pix(self.config['coords']['nside_pixel'], self.catalog_roi.lon, self.catalog_roi.lat),
self.roi.pixels_interior)
#cut_interior = self.roi.inInterior(self.catalog_roi.lon,self.catalog_roi.lat)
self.catalog_interior = self.catalog_roi.applyCut(cut_interior)
self.catalog_interior.project(self.roi.projector)
self.catalog_interior.spatialBin(self.roi)
# Set the default catalog
#logger.info("Using interior ROI for likelihood calculation")
self.catalog = self.catalog_interior
def query(outfile, ra, dec, coord='equ', nside=128):
query = """--- Select stars near a given location
select CATALOG_ID, RA, DEC, HPX2048,
WAVG_MAG_PSF_G, WAVG_MAG_PSF_R, WAVG_SPREAD_MODEL_R
from Y2Q1_OBJECTS_V1
where %(position)s
and WAVG_MAG_PSF_G between 10 and 30
and WAVG_MAG_PSF_R between 10 and 30
and abs(WAVG_SPREAD_MODEL_R) < 0.002;
> %(outfile)s
"""
if coord.lower() == 'equ':
delta = 0.2
position = """RA between %(ra).2f - %(delta).2f and %(ra).2f + %(delta).2f
and DEC between %(dec).2f - %(delta).2f and %(dec).2f + %(delta).2f""" % dict(
ra=ra, dec=dec, delta=delta)
elif coord.lower() == 'hpx':
pixel = ang2pix(nside, ra, dec, nest=True)
factor = healpy.nside2npix(2048) / healpy.nside2npix(nside)
position = """HPX2048 > (%(pixel)i)*%(factor)i and HPX2048 < (%(pixel)i + 1)*%(factor)i""" % dict(
pixel=pixel, factor=factor)
return query % dict(position=position, outfile=outfile)
def __init__(self, config, lon, lat):
self.config = Config(config)
self.lon = lon
self.lat = lat
self.projector = ugali.utils.projector.Projector(self.lon, self.lat)
self.vec = vec = ang2vec(self.lon, self.lat)
self.pix = ang2pix(self.config['coords']['nside_likelihood'],self.lon,self.lat)
# Pixels from the entire ROI disk
pix = query_disc(self.config['coords']['nside_pixel'], vec,
self.config['coords']['roi_radius'])
self.pixels = PixelRegion(self.config['coords']['nside_pixel'],pix)
# Pixels in the interior region
pix = query_disc(self.config['coords']['nside_pixel'], vec,
self.config['coords']['roi_radius_interior'])
self.pixels_interior = PixelRegion(self.config['coords']['nside_pixel'],pix)
# Pixels in the outer annulus
pix = query_disc(self.config['coords']['nside_pixel'], vec,
self.config['coords']['roi_radius_annulus'])
pix = numpy.setdiff1d(self.pixels, pix)
self.pixels_annulus = PixelRegion(self.config['coords']['nside_pixel'],pix)
# Pixels within target healpix region
pix = ugali.utils.skymap.subpixel(self.pix,self.config['coords']['nside_likelihood'],
self.config['coords']['nside_pixel'])
self.pixels_target = PixelRegion(self.config['coords']['nside_pixel'],pix)
# Boolean arrays for selecting given pixels
# (Careful, this works because pixels are pre-sorted by query_disc before in1d)
self.pixel_interior_cut = numpy.in1d(self.pixels, self.pixels_interior)
# ADW: Updated for more general ROI shapes
#self.pixel_annulus_cut = ~self.pixel_interior_cut
self.pixel_annulus_cut = numpy.in1d(self.pixels, self.pixels_annulus)
# # These should be unnecessary now
# self.centers_lon, self.centers_lat = self.pixels.lon, self.pixels.lat
# self.centers_lon_interior,self.centers_lat_interior = self.pixels_interior.lon,self.pixels_interior.lat
# self.centers_lon_target, self.centers_lat_target = self.pixels_target.lon, self.pixels_target.lat
self.area_pixel = healpy.nside2pixarea(self.config.params['coords']['nside_pixel'],degrees=True) # deg^2
"""
self.centers_x = self._centers(self.bins_x)
self.centers_y = self._centers(self.bins_y)
self.delta_x = self.config.params['coords']['pixel_size']
self.delta_y = self.config.params['coords']['pixel_size']
# Should actually try to take projection effects into account for better accuracy
# MC integration perhaps?
# Throw points in a cone around full ROI and see what fraction fall in
self.area_pixel = self.config.params['coords']['pixel_size']**2
self.centers_lon, self.centers_lat = self.projector.imageToSphere(self.centers_x, self.centers_y)
"""
# ADW: These are really bin edges, should be careful and consistent
self.bins_mag = numpy.linspace(self.config.params['mag']['min'],
self.config.params['mag']['max'],
self.config.params['mag']['n_bins'] + 1)
self.bins_color = numpy.linspace(self.config.params['color']['min'],
self.config.params['color']['max'],
self.config.params['color']['n_bins'] + 1)
self.centers_mag = ugali.utils.binning.centers(self.bins_mag)
self.centers_color = ugali.utils.binning.centers(self.bins_color)
self.delta_mag = self.bins_mag[1] - self.bins_mag[0]
self.delta_color = self.bins_color[1] - self.bins_color[0]
# Axis labels
self.label_x = 'x (deg)'
self.label_y = 'y (deg)'
if self.config.params['catalog']['band_1_detection']:
self.label_mag = '%s (mag)'%(self.config.params['catalog']['mag_1_field'])
else:
self.label_mag = '%s (mag)'%(self.config.params['catalog']['mag_2_field'])
self.label_color = '%s - %s (mag)'%(self.config.params['catalog']['mag_1_field'],
self.config.params['catalog']['mag_2_field'])
def pdf(self,lon,lat):
pix = ang2pix(self.nside,lon,lat)
return self.norm * self._pdf(pix)
parser = argparse.ArgumentParser(description=description)
opts = parser.parse_args()
nside = 128
HPX = 'HPX%i' % nside
outdir = mkdir('release/calibration')
gcmfile = 'y1a1_gcm_v0.fits'
if not os.path.exists(gcmfile):
query = download.gcm_query()
download.download(gcmfile, query, section='dessci')
print "Loading %s..." % gcmfile
gcm = fitsio.read(gcmfile)
names = ['MAG_ZERO', HPX]
values = [gcm['ZEROPOINT'] + 25, ang2pix(nside, gcm['RA'], gcm['DEC'])]
gcm = recfn.append_fields(gcm,
names,
values,
usemask=False,
asrecarray=True)
gcm['BAND'] = np.char.strip(gcm['BAND'])
qslrfile = 'y2n_y1a1_qslr_v6.fits'
if not os.path.exists(qslrfile):
query = download.qslr_query()
download.download(qslrfile, query, section='dessci')
print "Loading %s..." % qslrfile
qslr = fitsio.read(qslrfile)
names = [HPX]
values = [ang2pix(nside, qslr['RA_MEAN'], qslr['DEC_MEAN'])]
def background(self, mc_source_id=2, seed=None):
"""
Create a simulation of the background stellar population.
Because some stars have been clipped to generate the CMD,
this function tends to slightly underestimate (~1%) the
background as compared to the true catalog.
The simulation of background object colors relies on the
data-derived CMD. As such, it is a binned random generator
and thus has some fundamental limitations.
- The expected number of counts per bin is drawn ra
There are a few limitations of this procedure:
- Colors are drawn from the CMD of the background annulus
- The number of stars per CMD bin is randomized according to the CMD
- The colors/mags are then uniformly distributed within the bin
- This leads to trouble with large bins when the cloud-in-cells
algorithm is applied to the simulated data
- The positions are chosen randomly over the spherical cap of the ROI
- Objects that are outside of the
WARNING: The cloud-in-cells method of generating
the CMD leads to some difficulties since it disperses
objects from high-density zones to low density zones.
- Magnitudes are not randomized according to their errors
"""
if seed is not None: np.random.seed(seed)
self._setup_cmd()
# Randomize the number of stars per bin according to Poisson distribution
nstar_per_bin = numpy.random.poisson(lam=self.bkg_lambda)
nstar = nstar_per_bin.sum()
logger.info("Simulating %i background stars..." % nstar)
if not self.config['simulate'].get('uniform'):
logger.info("Generating colors from background CMD.")
# Distribute the stars within each CMD bin
delta_color = self.bkg_centers_color[1] - self.bkg_centers_color[0]
delta_mag = self.bkg_centers_mag[1] - self.bkg_centers_mag[0]
# Distribute points within each color-mag bins
xx, yy = np.meshgrid(self.bkg_centers_color, self.bkg_centers_mag)
color = numpy.repeat(xx.flatten(), repeats=nstar_per_bin.flatten())
color += numpy.random.uniform(-delta_color / 2.,
delta_color / 2.,
size=nstar)
mag_1 = numpy.repeat(yy.flatten(), repeats=nstar_per_bin.flatten())
mag_1 += numpy.random.uniform(-delta_mag / 2.,
delta_mag / 2.,
size=nstar)
else:
# Uniform color-magnitude distribution
logger.info("Generating uniform CMD.")
mag_1 = np.random.uniform(self.config['mag']['min'],
self.config['mag']['max'],
size=nstar)
color = np.random.uniform(self.config['color']['min'],
self.config['color']['max'],
size=nstar)
mag_2 = mag_1 - color
# Random points drawn from healpix subpixels
logger.info("Generating uniform positions...")
idx = np.random.randint(0, len(self.subpix) - 1, size=nstar)
lon, lat = pix2ang(self.nside_subpixel, self.subpix[idx])
nside_pixel = self.nside_pixel
pix = ang2pix(nside_pixel, lon, lat)
# There is probably a better way to do this step without creating the full HEALPix map
mask = -1. * numpy.ones(healpy.nside2npix(nside_pixel))
mask[self.roi.pixels] = self.mask.mask_1.mask_roi_sparse
mag_lim_1 = mask[pix]
mask = -1. * numpy.ones(healpy.nside2npix(nside_pixel))
mask[self.roi.pixels] = self.mask.mask_2.mask_roi_sparse
mag_lim_2 = mask[pix]
mag_err_1 = self.photo_err_1(mag_lim_1 - mag_1)
mag_err_2 = self.photo_err_2(mag_lim_2 - mag_2)
mc_source_id = mc_source_id * numpy.ones(len(mag_1))
# ADW: Should magnitudes be randomized by the erros?
#mag_1 += (numpy.random.normal(size=len(mag_1)) * mag_err_1)
#mag_2 += (numpy.random.normal(size=len(mag_2)) * mag_err_2)
select = (mag_lim_1 > mag_1) & (mag_lim_2 > mag_2)
### # Make sure objects lie within the original cmd (should be done later...)
### select &= (ugali.utils.binning.take2D(self.mask.solid_angle_cmd, color, mag_1,
### self.roi.bins_color, self.roi.bins_mag) > 0)
logger.info("Clipping %i simulated background stars..." %
(~select).sum())
hdu = ugali.observation.catalog.makeHDU(
self.config, mag_1[select], mag_err_1[select], mag_2[select],
mag_err_2[select], lon[select], lat[select], mc_source_id[select])
catalog = ugali.observation.catalog.Catalog(self.config, data=hdu.data)
return catalog
def background(self,mc_source_id=2,seed=None):
"""
Create a simulation of the background stellar population.
Because some stars have been clipped to generate the CMD,
this function tends to slightly underestimate (~1%) the
background as compared to the true catalog.
The simulation of background object colors relies on the
data-derived CMD. As such, it is a binned random generator
and thus has some fundamental limitations.
- The expected number of counts per bin is drawn ra
There are a few limitations of this procedure:
- Colors are drawn from the CMD of the background annulus
- The number of stars per CMD bin is randomized according to the CMD
- The colors/mags are then uniformly distributed within the bin
- This leads to trouble with large bins when the cloud-in-cells
algorithm is applied to the simulated data
- The positions are chosen randomly over the spherical cap of the ROI
- Objects that are outside of the
WARNING: The cloud-in-cells method of generating
the CMD leads to some difficulties since it disperses
objects from high-density zones to low density zones.
- Magnitudes are not randomized according to their errors
"""
if seed is not None: np.random.seed(seed)
self._setup_cmd()
# Randomize the number of stars per bin according to Poisson distribution
nstar_per_bin = numpy.random.poisson(lam=self.bkg_lambda)
nstar = nstar_per_bin.sum()
logger.info("Simulating %i background stars..."%nstar)
if not self.config['simulate'].get('uniform'):
logger.info("Generating colors from background CMD.")
# Distribute the stars within each CMD bin
delta_color = self.bkg_centers_color[1]-self.bkg_centers_color[0]
delta_mag = self.bkg_centers_mag[1]-self.bkg_centers_mag[0]
# Distribute points within each color-mag bins
xx,yy = np.meshgrid(self.bkg_centers_color,self.bkg_centers_mag)
color = numpy.repeat(xx.flatten(),repeats=nstar_per_bin.flatten())
color += numpy.random.uniform(-delta_color/2.,delta_color/2.,size=nstar)
mag_1 = numpy.repeat(yy.flatten(),repeats=nstar_per_bin.flatten())
mag_1 += numpy.random.uniform(-delta_mag/2.,delta_mag/2.,size=nstar)
else:
# Uniform color-magnitude distribution
logger.info("Generating uniform CMD.")
mag_1 = np.random.uniform(self.config['mag']['min'],self.config['mag']['max'],size=nstar)
color = np.random.uniform(self.config['color']['min'],self.config['color']['max'],size=nstar)
mag_2 = mag_1 - color
# Random points drawn from healpix subpixels
logger.info("Generating uniform positions...")
idx = np.random.randint(0,len(self.subpix)-1,size=nstar)
lon,lat = pix2ang(self.nside_subpixel,self.subpix[idx])
nside_pixel = self.nside_pixel
pix = ang2pix(nside_pixel, lon, lat)
# There is probably a better way to do this step without creating the full HEALPix map
mask = -1. * numpy.ones(healpy.nside2npix(nside_pixel))
mask[self.roi.pixels] = self.mask.mask_1.mask_roi_sparse
mag_lim_1 = mask[pix]
mask = -1. * numpy.ones(healpy.nside2npix(nside_pixel))
mask[self.roi.pixels] = self.mask.mask_2.mask_roi_sparse
mag_lim_2 = mask[pix]
mag_err_1 = self.photo_err_1(mag_lim_1 - mag_1)
mag_err_2 = self.photo_err_2(mag_lim_2 - mag_2)
mc_source_id = mc_source_id * numpy.ones(len(mag_1))
# ADW: Should magnitudes be randomized by the erros?
#mag_1 += (numpy.random.normal(size=len(mag_1)) * mag_err_1)
#mag_2 += (numpy.random.normal(size=len(mag_2)) * mag_err_2)
select = (mag_lim_1>mag_1)&(mag_lim_2>mag_2)
### # Make sure objects lie within the original cmd (should be done later...)
### select &= (ugali.utils.binning.take2D(self.mask.solid_angle_cmd, color, mag_1,
### self.roi.bins_color, self.roi.bins_mag) > 0)
logger.info("Clipping %i simulated background stars..."%(~select).sum())
hdu = ugali.observation.catalog.makeHDU(self.config,mag_1[select],mag_err_1[select],
mag_2[select],mag_err_2[select],
lon[select],lat[select],mc_source_id[select])
catalog = ugali.observation.catalog.Catalog(self.config, data=hdu.data)
return catalog
def finalizeObjects(self, objects):
objs = numpy.recarray(len(objects),
dtype=[
('NAME', 'S24'),
('TS', 'f4'),
('GLON', 'f4'),
('GLAT', 'f4'),
('RA', 'f4'),
('DEC', 'f4'),
('MODULUS', 'f4'),
('DISTANCE', 'f4'),
('RICHNESS', 'f4'),
('MASS', 'f4'),
('NANNULUS', 'i4'),
('NINTERIOR', 'i4'),
])
objs['TS'] = self.values[objects['IDX_MAX'], objects['ZIDX_MAX']]
glon, glat = objects['X_MAX'], objects['Y_MAX']
objs['GLON'], objs['GLAT'] = glon, glat
ra, dec = gal2cel(glon, glat)
objs['RA'], objs['DEC'] = ra, dec
modulus = objects['Z_MAX']
objs['MODULUS'] = modulus
objs['DISTANCE'] = mod2dist(modulus)
#ninterior = ugali.utils.skymap.readSparseHealpixMap(self.roifile,'NINSIDE')
#nannulus = ugali.utils.skymap.readSparseHealpixMap(self.roifile,'NANNULUS')
#stellar = ugali.utils.skymap.readSparseHealpixMap(self.roifile,'STELLAR')
nside = healpy.npix2nside(len(self.nannulus))
pix = ang2pix(nside, glon, glat)
richness = self.richness[objects['IDX_MAX'], objects['ZIDX_MAX']]
objs['RICHNESS'] = richness
objs['MASS'] = richness * self.stellar[pix]
objs['NANNULUS'] = self.nannulus[pix].astype(int)
objs['NINTERIOR'] = self.ninterior[pix].astype(int)
# Default name formatting
# http://cdsarc.u-strasbg.fr/ftp/pub/iau/
# http://cds.u-strasbg.fr/vizier/Dic/iau-spec.htx
fmt = "J%(hour)02i%(hmin)04.1f%(deg)+03i%(dmin)02i"
for obj, _ra, _dec in zip(objs, ra, dec):
hms = dec2hms(_ra)
dms = dec2dms(_dec)
params = dict(hour=hms[0],
hmin=hms[1] + hms[2] / 60.,
deg=dms[0],
dmin=dms[1] + dms[2] / 60.)
obj['NAME'] = fmt % params
out = recfuncs.merge_arrays([objs, objects],
usemask=False,
asrecarray=True,
flatten=True)
# This is safer than viewing as FITS_rec
return pyfits.new_table(out).data
def pixel(self):
nside = self.config.params['coords']['nside_pixel']
pixel = ang2pix(nside,float(self.source.lon),float(self.source.lat))
return pixel
def gaia_photometry(catfile,nside=64,band=None,plot=False,version='edr3'):
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
#columns = [OBJECT_ID,'RA','DEC']
columns = ['RA','DEC']
spread,nepochs = ['WAVG_SPREAD_MODEL_R','NEPOCHS_R']
mag_g,mag_r,mag_i,mag_z = bfields(['MAG_PSF'],['g','r','i','z'])
#mag_g,mag_r,mag_i,mag_z = bfields(['WAVG_MAG_PSF'],['g','r','i','z'])
columns += [spread, nepochs, mag_g, mag_r, mag_i, mag_z]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
radius = np.degrees(hp.max_pixrad(NSIDE))
msg = '%s (RA,DEC,RAD) = %.2f,%.2f,%.2f'%(os.path.basename(catfile),ra,dec,radius)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < r < 20 and 0.0 < (g-i) < 1.5
sel = (np.fabs(cat[spread])<0.002) & \
(cat[mag_g]<90) & (cat[mag_r]<90) & (cat[mag_i]<90) & (cat[mag_z]<90) & \
(cat[mag_r]>16) & (cat[mag_r]<20) & \
((cat[mag_g] - cat[mag_i]) > 0.0) & \
((cat[mag_g] - cat[mag_i]) < 1.5) & \
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
#msg = "Getting external catalog: %s"%catalog
ext = get_gaia_catalog(hpx,version=version)
m = match_query(cat['RA'],cat['DEC'],ext['RA'],ext['DEC'])
# Use a fairly wide matching radius (2 arcsec)
cut = 1.0
sel = m[-1]*3600. < cut
cat_match = cat[m[0][sel]]
ext_match = ext[m[1][sel]]
cat_G = gaia_transform(cat_match[mag_g],cat_match[mag_r],cat_match[mag_i],cat_match[mag_z],
version=version)
# Need to put Gaia flux onto the AB system
ext_G = -2.5 * np.log10(ext_match['PHOT_G_MEAN_FLUX']) + 25.7934
diff = cat_G - ext_G
pix = ang2pix(nside,cat_match['RA'],cat_match['DEC'])
upix = np.unique(pix)
stat = nd.median(diff,labels=pix,index=upix)
if False:
plt.figure()
plt.hist(cat_G - ext_G)
import pdb; pdb.set_trace()
return upix,stat
def write(self, outfile):
"""
Save the likelihood fitting results as a sparse HEALPix map.
"""
# Full data output (too large for survey)
if self.config['scan']['full_pdf']:
data_dict = {
'LOG_LIKELIHOOD':
self.log_likelihood_sparse_array.transpose(),
'RICHNESS':
self.richness_sparse_array.transpose(),
'RICHNESS_LOWER':
self.richness_lower_sparse_array.transpose(),
'RICHNESS_UPPER':
self.richness_upper_sparse_array.transpose(),
'RICHNESS_LIMIT':
self.richness_upper_limit_sparse_array.transpose(),
#'STELLAR_MASS': self.stellar_mass_sparse_array.transpose(),
'FRACTION_OBSERVABLE':
self.fraction_observable_sparse_array.transpose()
}
else:
data_dict = {
'LOG_LIKELIHOOD':
self.log_likelihood_sparse_array.transpose(),
'RICHNESS':
self.richness_sparse_array.transpose(),
'FRACTION_OBSERVABLE':
self.fraction_observable_sparse_array.transpose()
}
# Stellar Mass can be calculated from STELLAR * RICHNESS
header_dict = {
'STELLAR':
round(self.stellar_mass_conversion, 8),
'LKDNSIDE':
self.config['coords']['nside_likelihood'],
'LKDPIX':
ang2pix(self.config['coords']['nside_likelihood'], self.roi.lon,
self.roi.lat),
'NROI':
self.roi.inROI(self.loglike.catalog_roi.lon,
self.loglike.catalog_roi.lat).sum(),
'NANNULUS':
self.roi.inAnnulus(self.loglike.catalog_roi.lon,
self.loglike.catalog_roi.lat).sum(),
'NINSIDE':
self.roi.inInterior(self.loglike.catalog_roi.lon,
self.loglike.catalog_roi.lat).sum(),
'NTARGET':
self.roi.inTarget(self.loglike.catalog_roi.lon,
self.loglike.catalog_roi.lat).sum(),
}
# In case there is only a single distance modulus
if len(self.distance_modulus_array) == 1:
for key in data_dict:
data_dict[key] = data_dict[key].flatten()
ugali.utils.skymap.writeSparseHealpixMap(
self.roi.pixels_target,
data_dict,
self.config['coords']['nside_pixel'],
outfile,
distance_modulus_array=self.distance_modulus_array,
coordsys='NULL',
ordering='NULL',
header_dict=header_dict)
def finalizeObjects(self, objects):
objs = numpy.recarray(len(objects),
dtype=[
('NAME', 'S24'),
('TS', 'f4'),
('GLON', 'f4'),
('GLAT', 'f4'),
('RA', 'f4'),
('DEC', 'f4'),
('MODULUS', 'f4'),
('DISTANCE', 'f4'),
('RICHNESS', 'f4'),
('MASS', 'f4'),
('NANNULUS', 'i4'),
('NINTERIOR', 'i4'),
])
objs['TS'] = self.values[objects['IDX_MAX'], objects['ZIDX_MAX']]
lon, lat = objects['X_MAX'], objects['Y_MAX']
coordsys = self.config['coords']['coordsys']
if coordsys.lower() == 'gal':
print("GAL coordintes")
objs['GLON'], objs['GLAT'] = lon, lat
objs['RA'], objs['DEC'] = gal2cel(lon, lat)
else:
print("CEL coordintes")
objs['RA'], objs['DEC'] = lon, lat
objs['GLON'], objs['GLAT'] = cel2gal(lon, lat)
modulus = objects['Z_MAX']
objs['MODULUS'] = modulus
objs['DISTANCE'] = mod2dist(modulus)
nside = healpy.npix2nside(len(self.nannulus))
pix = ang2pix(nside, lon, lat)
richness = self.richness[objects['IDX_MAX'], objects['ZIDX_MAX']]
objs['RICHNESS'] = richness
objs['MASS'] = richness * self.stellar[pix]
objs['NANNULUS'] = self.nannulus[pix].astype(int)
objs['NINTERIOR'] = self.ninterior[pix].astype(int)
# Default name formatting
# http://cdsarc.u-strasbg.fr/ftp/pub/iau/
# http://cds.u-strasbg.fr/vizier/Dic/iau-spec.htx
fmt = "J%(hour)02i%(hmin)04.1f%(deg)+03i%(dmin)02i"
for obj, _ra, _dec in zip(objs, objs['RA'], objs['DEC']):
hms = dec2hms(_ra)
dms = dec2dms(_dec)
params = dict(hour=hms[0],
hmin=hms[1] + hms[2] / 60.,
deg=dms[0],
dmin=dms[1] + dms[2] / 60.)
obj['NAME'] = fmt % params
out = recfuncs.merge_arrays([objs, objects],
usemask=False,
asrecarray=True,
flatten=True)
return out
def __init__(self, config, lon, lat):
self.config = Config(config)
self.lon = lon
self.lat = lat
self.projector = ugali.utils.projector.Projector(self.lon, self.lat)
self.vec = vec = ang2vec(self.lon, self.lat)
self.pix = ang2pix(self.config['coords']['nside_likelihood'], self.lon,
self.lat)
# Pixels from the entire ROI disk
pix = query_disc(self.config['coords']['nside_pixel'], vec,
self.config['coords']['roi_radius'])
self.pixels = PixelRegion(self.config['coords']['nside_pixel'], pix)
# Pixels in the interior region
pix = query_disc(self.config['coords']['nside_pixel'], vec,
self.config['coords']['roi_radius_interior'])
self.pixels_interior = PixelRegion(
self.config['coords']['nside_pixel'], pix)
# Pixels in the outer annulus
pix = query_disc(self.config['coords']['nside_pixel'], vec,
self.config['coords']['roi_radius_annulus'])
pix = np.setdiff1d(self.pixels, pix)
self.pixels_annulus = PixelRegion(self.config['coords']['nside_pixel'],
pix)
# Pixels within target healpix region
pix = ugali.utils.skymap.subpixel(
self.pix, self.config['coords']['nside_likelihood'],
self.config['coords']['nside_pixel'])
self.pixels_target = PixelRegion(self.config['coords']['nside_pixel'],
pix)
# Boolean arrays for selecting given pixels
# (Careful, this works because pixels are pre-sorted by query_disc before in1d)
self.pixel_interior_cut = np.in1d(self.pixels, self.pixels_interior)
self.pixel_annulus_cut = np.in1d(self.pixels, self.pixels_annulus)
# Some pixel properties
self.area_pixel = hp.nside2pixarea(
self.config.params['coords']['nside_pixel'], degrees=True) # deg^2
self.max_pixrad = np.degrees(
hp.max_pixrad(self.config['coords']['nside_pixel'])) # deg
# ADW: These are really bin edges, should be careful and consistent
# It would be cleaner to separate the CMD from ROI
self.bins_mag = np.linspace(self.config.params['mag']['min'],
self.config.params['mag']['max'],
self.config.params['mag']['n_bins'] + 1)
self.bins_color = np.linspace(
self.config.params['color']['min'],
self.config.params['color']['max'],
self.config.params['color']['n_bins'] + 1)
self.centers_mag = ugali.utils.binning.centers(self.bins_mag)
self.centers_color = ugali.utils.binning.centers(self.bins_color)
self.delta_mag = self.bins_mag[1] - self.bins_mag[0]
self.delta_color = self.bins_color[1] - self.bins_color[0]
def pdf(self, lon, lat):
pix = ang2pix(self.nside, lon, lat)
return self.norm * self._pdf(pix)
def calculate(self, infile, field=1, simple=False):
logger.info("Calculating magnitude limit from %s" % infile)
#manglefile = self.config['mangle']['infile_%i'%field]
#footfile = self.config['data']['footprint']
#try:
# footprint = fitsio.read(footfile)['I'].ravel()
#except:
# logger.warn("Couldn't open %s; will try again."%footfile)
# footprint = footfile
mag_column = self.config['catalog']['mag_%i_field' % field]
magerr_column = self.config['catalog']['mag_err_%i_field' % field]
# For simple maglims
release = self.config['data']['release'].lower()
band = self.config['catalog']['mag_%i_band' % field]
pixel_pix_name = 'PIX%i' % self.nside_pixel
# If the data already has a healpix pixel assignment then use it
# Otherwise recalculate...
try:
data = fitsio.read(infile, columns=[pixel_pix_name])
except ValueError as e:
logger.info(str(e))
columns = [
self.config['catalog']['lon_field'],
self.config['catalog']['lat_field']
]
data = fitsio.read(infile, columns=columns)[columns]
pix = ang2pix(self.nside_pixel, data[columns[0]], data[columns[1]])
data = recfuncs.rec_append_fields(data, pixel_pix_name, pix)
#mask_pixels = np.arange( hp.nside2npix(self.nside_mask), dtype='int')
mask_maglims = np.zeros(hp.nside2npix(self.nside_mask))
out_pixels = np.zeros(0, dtype='int')
out_maglims = np.zeros(0)
# Find the objects in each pixel
pixel_pix = data[pixel_pix_name]
mask_pix = ugali.utils.skymap.superpixel(pixel_pix, self.nside_pixel,
self.nside_mask)
count = Counter(mask_pix)
pixels = sorted(count.keys())
pix_digi = np.digitize(mask_pix, pixels).argsort()
idx = 0
min_num = 500
signal_to_noise = 10.
magerr_lim = 1 / signal_to_noise
for pix in pixels:
# Calculate the magnitude limit in each pixel
num = count[pix]
objs = data[pix_digi[idx:idx + num]]
idx += num
if simple:
# Set constant magnitude limits
logger.debug("Simple magnitude limit for %s" % infile)
mask_maglims[pix] = MAGLIMS[release][band]
elif num < min_num:
logger.info('Found <%i objects in pixel %i' % (min_num, pix))
mask_maglims[pix] = 0
else:
mag = objs[mag_column]
magerr = objs[magerr_column]
# Estimate the magnitude limit as suggested by:
# https://deswiki.cosmology.illinois.edu/confluence/display/DO/SVA1+Release+Document
# (https://desweb.cosmology.illinois.edu/confluence/display/Operations/SVA1+Doc)
maglim = np.median(mag[(magerr > 0.9 * magerr_lim)
& (magerr < 1.1 * magerr_lim)])
# Alternative method to estimate the magnitude limit by fitting median
#mag_min, mag_max = mag.min(),mag.max()
#mag_bins = np.arange(mag_min,mag_max,0.1) #0.1086?
#x,y = ugali.utils.binning.binnedMedian(mag,magerr,mag_bins)
#x,y = x[~np.isnan(y)],y[~np.isnan(y)]
#magerr_med = interp1d(x,y)
#mag0 = np.median(x)
#maglim = brentq(lambda a: magerr_med(a)-magerr_lim,x.min(),x.max(),disp=False)
# Median from just objects near magerr cut
mask_maglims[pix] = maglim
logger.debug("%i (n=%i): maglim=%g" %
(pix, num, mask_maglims[pix]))
subpix = ugali.utils.skymap.subpixel(pix, self.nside_mask,
self.nside_pixel)
maglims = np.zeros(len(subpix)) + mask_maglims[pix]
out_pixels = np.append(out_pixels, subpix)
out_maglims = np.append(out_maglims, maglims)
# Remove empty pixels
logger.info("Removing empty pixels")
idx = np.nonzero(out_maglims > 0)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
# Remove pixels outside the footprint
if self.footfile:
logger.info("Checking footprint against %s" % self.footfile)
lon, lat = pix2ang(self.nside_pixel, out_pixels)
if self.config['coords']['coordsys'] == 'gal':
ra, dec = gal2cel(lon, lat)
else:
ra, dec = lon, lat
footprint = inFootprint(self.footprint, ra, dec)
idx = np.nonzero(footprint)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
logger.info("MAGLIM = %.3f +/- %.3f" %
(np.mean(out_maglims), np.std(out_maglims)))
return out_pixels, out_maglims
def pixelizeCatalog(infiles, config, force=False):
"""
Break catalog into chunks by healpix pixel.
Parameters:
-----------
infiles : List of input files
config : Configuration file
force : Overwrite existing files (depricated)
Returns:
--------
None
"""
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
coordsys = config['coords']['coordsys'].upper()
outdir = mkdir(config['catalog']['dirname'])
filenames = config.getFilenames()
lon_field = config['catalog']['lon_field'].upper()
lat_field = config['catalog']['lat_field'].upper()
# ADW: It would probably be better (and more efficient) to do the
# pixelizing and the new column insertion separately.
for i,filename in enumerate(infiles):
logger.info('(%i/%i) %s'%(i+1, len(infiles), filename))
data = fitsio.read(filename)
logger.info("%i objects found"%len(data))
if not len(data): continue
columns = map(str.upper,data.dtype.names)
names,arrs = [],[]
if (lon_field in columns) and (lat_field in columns):
lon,lat = data[lon_field],data[lat_field]
elif coordsys == 'GAL':
msg = "Columns '%s' and '%s' not found."%(lon_field,lat_field)
msg += "\nConverting from RA,DEC"
logger.warning(msg)
lon,lat = cel2gal(data['RA'],data['DEC'])
names += [lon_field,lat_field]
arrs += [lon,lat]
elif coordsys == 'CEL':
msg = "Columns '%s' and '%s' not found."%(lon_field,lat_field)
msg += "\nConverting from GLON,GLAT"
lon,lat = gal2cel(data['GLON'],data['GLAT'])
names += [lon_field,lat_field]
arrs += [lon,lat]
cat_pix = ang2pix(nside_catalog,lon,lat)
pix_pix = ang2pix(nside_pixel,lon,lat)
cat_pix_name = 'PIX%i'%nside_catalog
pix_pix_name = 'PIX%i'%nside_pixel
try:
names += [cat_pix_name,pix_pix_name]
arrs += [cat_pix,pix_pix]
data=mlab.rec_append_fields(data,names=names,arrs=arrs)
except ValueError as e:
logger.warn(str(e)+'; not adding column.')
#data[cat_pix_name] = cat_pix
#data[pix_pix_name] = pix_pix
for pix in np.unique(cat_pix):
logger.debug("Processing pixel %s"%pix)
arr = data[cat_pix == pix]
outfile = filenames.data['catalog'][pix]
if not os.path.exists(outfile):
logger.debug("Creating %s"%outfile)
out=fitsio.FITS(outfile,mode='rw')
out.write(arr)
hdr=healpix.header_odict(nside=nside_catalog,
coord=coordsys[0])
for key in ['PIXTYPE','ORDERING','NSIDE','COORDSYS']:
out[1].write_key(*list(hdr[key].values()))
out[1].write_key('PIX',pix,comment='HEALPIX pixel for this file')
else:
out=fitsio.FITS(outfile,mode='rw')
out[1].append(arr)
logger.debug("Writing %s"%outfile)
out.close()
def calculate(self, infile, field=1, simple=False):
logger.info("Calculating magnitude limit from %s"%infile)
#manglefile = self.config['mangle']['infile_%i'%field]
#footfile = self.config['data']['footprint']
#try:
# footprint = fitsio.read(footfile)['I'].ravel()
#except:
# logger.warn("Couldn't open %s; will try again."%footfile)
# footprint = footfile
mag_column = self.config['catalog']['mag_%i_field'%field]
magerr_column = self.config['catalog']['mag_err_%i_field'%field]
# For simple maglims
release = self.config['data']['release'].lower()
band = self.config['catalog']['mag_%i_band'%field]
pixel_pix_name = 'PIX%i'%self.nside_pixel
# If the data already has a healpix pixel assignment then use it
# Otherwise recalculate...
try:
data = fitsio.read(infile,columns=[pixel_pix_name])
except ValueError as e:
logger.info(str(e))
columns=[self.config['catalog']['lon_field'],
self.config['catalog']['lat_field']]
data = fitsio.read(infile,columns=columns)[columns]
pix = ang2pix(self.nside_pixel,data[columns[0]],data[columns[1]])
data = recfuncs.rec_append_fields(data,pixel_pix_name,pix)
#mask_pixels = np.arange( hp.nside2npix(self.nside_mask), dtype='int')
mask_maglims = np.zeros(hp.nside2npix(self.nside_mask))
out_pixels = np.zeros(0,dtype='int')
out_maglims = np.zeros(0)
# Find the objects in each pixel
pixel_pix = data[pixel_pix_name]
mask_pix = ugali.utils.skymap.superpixel(pixel_pix,self.nside_pixel,self.nside_mask)
count = Counter(mask_pix)
pixels = sorted(count.keys())
pix_digi = np.digitize(mask_pix,pixels).argsort()
idx = 0
min_num = 500
signal_to_noise = 10.
magerr_lim = 1/signal_to_noise
for pix in pixels:
# Calculate the magnitude limit in each pixel
num = count[pix]
objs = data[pix_digi[idx:idx+num]]
idx += num
if simple:
# Set constant magnitude limits
logger.debug("Simple magnitude limit for %s"%infile)
mask_maglims[pix] = MAGLIMS[release][band]
elif num < min_num:
logger.info('Found <%i objects in pixel %i'%(min_num,pix))
mask_maglims[pix] = 0
else:
mag = objs[mag_column]
magerr = objs[magerr_column]
# Estimate the magnitude limit as suggested by:
# https://deswiki.cosmology.illinois.edu/confluence/display/DO/SVA1+Release+Document
# (https://desweb.cosmology.illinois.edu/confluence/display/Operations/SVA1+Doc)
maglim = np.median(mag[(magerr>0.9*magerr_lim)&(magerr<1.1*magerr_lim)])
# Alternative method to estimate the magnitude limit by fitting median
#mag_min, mag_max = mag.min(),mag.max()
#mag_bins = np.arange(mag_min,mag_max,0.1) #0.1086?
#x,y = ugali.utils.binning.binnedMedian(mag,magerr,mag_bins)
#x,y = x[~np.isnan(y)],y[~np.isnan(y)]
#magerr_med = interp1d(x,y)
#mag0 = np.median(x)
#maglim = brentq(lambda a: magerr_med(a)-magerr_lim,x.min(),x.max(),disp=False)
# Median from just objects near magerr cut
mask_maglims[pix] = maglim
logger.debug("%i (n=%i): maglim=%g"%(pix,num,mask_maglims[pix]))
subpix = ugali.utils.skymap.subpixel(pix, self.nside_mask, self.nside_pixel)
maglims = np.zeros(len(subpix)) + mask_maglims[pix]
out_pixels = np.append(out_pixels,subpix)
out_maglims = np.append(out_maglims,maglims)
# Remove empty pixels
logger.info("Removing empty pixels")
idx = np.nonzero(out_maglims > 0)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
# Remove pixels outside the footprint
if self.footfile:
logger.info("Checking footprint against %s"%self.footfile)
lon,lat = pix2ang(self.nside_pixel,out_pixels)
if self.config['coords']['coordsys'] == 'gal':
ra,dec = gal2cel(lon,lat)
else:
ra,dec = lon,lat
footprint = inFootprint(self.footprint,ra,dec)
idx = np.nonzero(footprint)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
logger.info("MAGLIM = %.3f +/- %.3f"%(np.mean(out_maglims),np.std(out_maglims)))
return out_pixels,out_maglims
