def toy_background(self, mc_source_id=2, seed=None):
"""
Quick uniform background generation.
"""
logger.info("Running toy background simulation...")
size = 20000
nstar = np.random.poisson(size)
#np.random.seed(0)
logger.info("Simulating %i background stars..." % nstar)
### # Random points from roi pixels
### idx = np.random.randint(len(self.roi.pixels)-1,size=nstar)
### pix = self.roi.pixels[idx]
# Random points drawn from subpixels
logger.info("Generating uniform positions...")
idx = np.random.randint(0, len(self.subpix) - 1, size=nstar)
lon, lat = pix2ang(self.nside_subpixel, self.subpix[idx])
pix = ang2pix(self.nside_pixel, lon, lat)
lon, lat = pix2ang(self.nside_pixel, pix)
# Single color
#mag_1 = 19.05*np.ones(len(pix))
#mag_2 = 19.10*np.ones(len(pix))
# Uniform in color
logger.info("Generating uniform CMD...")
mag_1 = np.random.uniform(self.config['mag']['min'],
self.config['mag']['max'],
size=nstar)
color = np.random.uniform(self.config['color']['min'],
self.config['color']['max'],
size=nstar)
mag_2 = mag_1 - color
# There is probably a better way to do this step without creating the full HEALPix map
mask = -1. * numpy.ones(healpy.nside2npix(self.nside_pixel))
mask[self.roi.pixels] = self.mask.mask_1.mask_roi_sparse
mag_lim_1 = mask[pix]
mask = -1. * numpy.ones(healpy.nside2npix(self.nside_pixel))
mask[self.roi.pixels] = self.mask.mask_2.mask_roi_sparse
mag_lim_2 = mask[pix]
#mag_err_1 = 1.0*np.ones(len(pix))
#mag_err_2 = 1.0*np.ones(len(pix))
mag_err_1 = self.photo_err_1(mag_lim_1 - mag_1)
mag_err_2 = self.photo_err_2(mag_lim_2 - mag_2)
mc_source_id = mc_source_id * numpy.ones(len(mag_1))
select = (mag_lim_1 > mag_1) & (mag_lim_2 > mag_2)
hdu = ugali.observation.catalog.makeHDU(
self.config, mag_1[select], mag_err_1[select], mag_2[select],
mag_err_2[select], lon[select], lat[select], mc_source_id[select])
catalog = ugali.observation.catalog.Catalog(self.config, data=hdu.data)
return catalog
def toy_background(self,mc_source_id=2,seed=None):
"""
Quick uniform background generation.
"""
logger.info("Running toy background simulation...")
size = 20000
nstar = np.random.poisson(size)
#np.random.seed(0)
logger.info("Simulating %i background stars..."%nstar)
### # Random points from roi pixels
### idx = np.random.randint(len(self.roi.pixels)-1,size=nstar)
### pix = self.roi.pixels[idx]
# Random points drawn from subpixels
logger.info("Generating uniform positions...")
idx = np.random.randint(0,len(self.subpix)-1,size=nstar)
lon,lat = pix2ang(self.nside_subpixel,self.subpix[idx])
pix = ang2pix(self.nside_pixel, lon, lat)
lon,lat = pix2ang(self.nside_pixel,pix)
# Single color
#mag_1 = 19.05*np.ones(len(pix))
#mag_2 = 19.10*np.ones(len(pix))
# Uniform in color
logger.info("Generating uniform CMD...")
mag_1 = np.random.uniform(self.config['mag']['min'],self.config['mag']['max'],size=nstar)
color = np.random.uniform(self.config['color']['min'],self.config['color']['max'],size=nstar)
mag_2 = mag_1 - color
# There is probably a better way to do this step without creating the full HEALPix map
mask = -1. * numpy.ones(healpy.nside2npix(self.nside_pixel))
mask[self.roi.pixels] = self.mask.mask_1.mask_roi_sparse
mag_lim_1 = mask[pix]
mask = -1. * numpy.ones(healpy.nside2npix(self.nside_pixel))
mask[self.roi.pixels] = self.mask.mask_2.mask_roi_sparse
mag_lim_2 = mask[pix]
#mag_err_1 = 1.0*np.ones(len(pix))
#mag_err_2 = 1.0*np.ones(len(pix))
mag_err_1 = self.photo_err_1(mag_lim_1 - mag_1)
mag_err_2 = self.photo_err_2(mag_lim_2 - mag_2)
mc_source_id = mc_source_id * numpy.ones(len(mag_1))
select = (mag_lim_1>mag_1)&(mag_lim_2>mag_2)
hdu = ugali.observation.catalog.makeHDU(self.config,mag_1[select],mag_err_1[select],
mag_2[select],mag_err_2[select],
lon[select],lat[select],mc_source_id[select])
catalog = ugali.observation.catalog.Catalog(self.config, data=hdu.data)
return catalog
def _parse_coords(self,opts):
""" Parse target coordinates in various ways...
"""
# The coordinates are mutually exclusive, so
# shouldn't have to worry about over-writing them.
if 'coords' in vars(opts): return
radius = vars(opts).get('radius',0)
gal = None
if vars(opts).get('gal') is not None:
gal = opts.gal
elif vars(opts).get('cel') is not None:
gal = cel2gal(*opts.cel)
elif vars(opts).get('hpx') is not None:
gal = pix2ang(*opts.hpx)
if gal is not None:
opts.coords = [(gal[0],gal[1],radius)]
opts.names = [vars(opts).get('name','')]
else:
opts.coords = None
opts.names = None
if vars(opts).get('targets') is not None:
opts.names,opts.coords = self.parse_targets(opts.targets)
if vars(opts).get('radius') is not None:
opts.coords['radius'] = vars(opts).get('radius')
def randomPositions(input, nside_pix, n=1):
"""
Generate n random positions within a full HEALPix mask of booleans, or a set of (lon, lat) coordinates.
input is either a
(1) full HEALPix mask of booleans, or
(2) a set of (lon, lat) coordinates for catalog objects that define the occupied pixels.
nside_pix is meant to be at coarser resolution than the input mask or catalog object positions
so that gaps from star holes, bleed trails, cosmic rays, etc. are filled in.
Return the longitude and latitude of the random positions (deg) and the total area (deg^2).
"""
input = numpy.array(input)
if len(input.shape) == 1:
if healpy.npix2nside(len(input)) < nside_pix:
logger.warning(
'Expected coarser resolution nside_pix in skymap.randomPositions'
)
subpix = numpy.nonzero(
input
)[0] # All the valid pixels in the mask at the NSIDE for the input mask
lon, lat = pix2ang(healpy.npix2nside(len(input)), subpix)
elif len(input.shape) == 2:
lon, lat = input[0], input[1] # All catalog object positions
else:
logger.warning(
'Unexpected input dimensions for skymap.randomPositions')
pix = surveyPixel(lon, lat, nside_pix)
# Area with which the random points are thrown
area = len(pix) * healpy.nside2pixarea(nside_pix, degrees=True)
# Create mask at the coarser resolution
mask = numpy.tile(False, healpy.nside2npix(nside_pix))
mask[pix] = True
# Estimate the number of points that need to be thrown based off
# coverage fraction of the HEALPix mask
coverage_fraction = float(numpy.sum(mask)) / len(mask)
n_throw = int(n / coverage_fraction)
lon, lat = [], []
count = 0
while len(lon) < n:
lon_throw = numpy.random.uniform(0., 360., n_throw)
lat_throw = numpy.degrees(
numpy.arcsin(numpy.random.uniform(-1., 1., n_throw)))
pix_throw = ugali.utils.healpix.angToPix(nside_pix, lon_throw,
lat_throw)
cut = mask[pix_throw].astype(bool)
lon = numpy.append(lon, lon_throw[cut])
lat = numpy.append(lat, lat_throw[cut])
count += 1
if count > 10:
raise RuntimeError('Too many loops...')
return lon[0:n], lat[0:n], area
def _parse_coords(self, opts):
""" Parse target coordinates in various ways...
"""
# The coordinates are mutually exclusive, so
# shouldn't have to worry about over-writing them.
if 'coords' in vars(opts): return
radius = vars(opts).get('radius', 0)
gal = None
if vars(opts).get('gal') is not None:
gal = opts.gal
elif vars(opts).get('cel') is not None:
gal = cel2gal(*opts.cel)
elif vars(opts).get('hpx') is not None:
gal = pix2ang(*opts.hpx)
if gal is not None:
opts.coords = [(gal[0], gal[1], radius)]
opts.names = [vars(opts).get('name', '')]
else:
opts.coords = None
opts.names = None
if vars(opts).get('targets') is not None:
opts.names, opts.coords = self.parse_targets(opts.targets)
if vars(opts).get('radius') is not None:
opts.coords['radius'] = vars(opts).get('radius')
def randomPositions(input, nside_pix, n=1):
"""
Generate n random positions within a full HEALPix mask of booleans, or a set of (lon, lat) coordinates.
nside_pix is meant to be at coarser resolution than the input mask or catalog object positions
so that gaps from star holes, bleed trails, cosmic rays, etc. are filled in.
Return the longitude and latitude of the random positions and the total area (deg^2).
Probably there is a faster algorithm, but limited much more by the simulation and fitting time
than by the time it takes to generate random positions within the mask.
"""
input = numpy.array(input)
if len(input.shape) == 1:
subpix = numpy.nonzero(
input
)[0] # All the valid pixels in the mask at the NSIDE for the input mask
lon, lat = pix2ang(healpy.npix2nside(len(input)), subpix)
elif len(input.shape) == 2:
lon, lat = input[0], input[1] # All catalog object positions
else:
logger.warning(
'Unexpected input dimensions for skymap.randomPositions')
pix = surveyPixel(lon, lat, nside_pix)
# Area with which the random points are thrown
area = len(pix) * healpy.nside2pixarea(nside_pix, degrees=True)
lon = []
lat = []
for ii in range(0, n):
# Choose an unmasked pixel at random, which is OK because HEALPix is an equal area scheme
pix_ii = pix[numpy.random.randint(0, len(pix))]
lon_ii, lat_ii = ugali.utils.projector.pixToAng(nside_pix, pix_ii)
projector = ugali.utils.projector.Projector(lon_ii, lat_ii)
inside = False
while not inside:
# Apply random offset
arcminToDegree = 1 / 60.
resolution = arcminToDegree * healpy.nside2resol(nside_pix,
arcmin=True)
x = 2. * (numpy.random.rand() -
0.5) * resolution # Using factor 2 to be conservative
y = 2. * (numpy.random.rand() - 0.5) * resolution
lon_candidate, lat_candidate = projector.imageToSphere(x, y)
# Make sure that the random position does indeed fall within the randomly selected pixel
if ugali.utils.projector.angToPix(nside_pix, lon_candidate,
lat_candidate) == pix_ii:
inside = True
lon.append(lon_candidate)
lat.append(lat_candidate)
return numpy.array(lon), numpy.array(lat), area
def randomPositions(input, nside_pix, n=1):
"""
Generate n random positions within a full HEALPix mask of booleans, or a set of (lon, lat) coordinates.
Parameters:
-----------
input : (1) full HEALPix mask of booleans, or (2) a set of (lon, lat) coordinates for catalog objects that define the occupied pixels.
nside_pix : nside_pix is meant to be at coarser resolution than the input mask or catalog object positions
so that gaps from star holes, bleed trails, cosmic rays, etc. are filled in.
Returns:
--------
lon,lat,area : Return the longitude and latitude of the random positions (deg) and the total area (deg^2).
"""
input = np.array(input)
if len(input.shape) == 1:
if hp.npix2nside(len(input)) < nside_pix:
logger.warning('Expected coarser resolution nside_pix in skymap.randomPositions')
subpix = np.nonzero(input)[0] # All the valid pixels in the mask at the NSIDE for the input mask
lon, lat = pix2ang(hp.npix2nside(len(input)), subpix)
elif len(input.shape) == 2:
lon, lat = input[0], input[1] # All catalog object positions
else:
logger.warning('Unexpected input dimensions for skymap.randomPositions')
pix = surveyPixel(lon, lat, nside_pix)
# Area with which the random points are thrown
area = len(pix) * hp.nside2pixarea(nside_pix, degrees=True)
# Create mask at the coarser resolution
mask = np.tile(False, hp.nside2npix(nside_pix))
mask[pix] = True
# Estimate the number of points that need to be thrown based off
# coverage fraction of the HEALPix mask
coverage_fraction = float(np.sum(mask)) / len(mask)
n_throw = int(n / coverage_fraction)
lon, lat = [], []
count = 0
while len(lon) < n:
lon_throw = np.random.uniform(0., 360., n_throw)
lat_throw = np.degrees(np.arcsin(np.random.uniform(-1., 1., n_throw)))
pix_throw = ugali.utils.healpix.angToPix(nside_pix, lon_throw, lat_throw)
cut = mask[pix_throw].astype(bool)
lon = np.append(lon, lon_throw[cut])
lat = np.append(lat, lat_throw[cut])
count += 1
if count > 10:
raise RuntimeError('Too many loops...')
return lon[0:n], lat[0:n], area
def __new__(cls, nside, pixels):
# Input array is an already formed ndarray instance
# We first cast to be our class type
obj = np.asarray(pixels).view(cls)
# add the new attribute to the created instance
obj._nside = nside
obj._pix = pixels
obj._lon,obj._lat = pix2ang(nside,pixels)
# Finally, we must return the newly created object:
return obj
def __new__(cls, nside, pixels):
# Input array is an already formed ndarray instance
# We first cast to be our class type
obj = np.asarray(pixels).view(cls)
# add the new attribute to the created instance
obj._nside = nside
obj._pix = pixels
obj._lon, obj._lat = pix2ang(nside, pixels)
# Finally, we must return the newly created object:
return obj
def parse_targets(filename):
"""
Load a text file with target coordinates. Returns
an array of target locations in Galactic coordinates.
File description:
[NAME] [LON] [LAT] [RADIUS] [COORD]
The values of LON and LAT will depend on COORD:
COORD = [GAL | CEL | HPX ],
LON = [GLON | RA | NSIDE]
LAT = [GLAT | DEC | PIX ]
"""
base,ext = os.path.splitext(filename)
if (ext=='.fits'):
import fitsio
data = fitsio.read(filename)
elif (ext=='.txt'):
from numpy.lib import NumpyVersion
if NumpyVersion(np.__version__) < '1.14.0':
data = np.genfromtxt(filename,names=True,dtype=None)
else:
data = np.genfromtxt(filename,names=True,dtype=None,encoding=None)
#data = np.genfromtxt(filename,unpack=True,usecols=list(range(5)),dtype=object,names=True)
elif (ext=='.yaml'):
import yaml
data = [(k,v['kernel']['lon']['value'],v['kernel']['lat']['value'],0.5,'CEL') for k,v in yaml.load(open(filename)).items()]
data = np.rec.fromrecords(data,names=['name','lon','lat','radius','coord'])
else:
msg = "Unrecognized file type: %s"%filename
raise IOError(msg)
data = np.atleast_1d(data)
data.dtype.names = list(map(str.lower,data.dtype.names))
# Deal with one-line input files
#if data.ndim == 1: data = np.array([data]).T
names = data['name']
out = data[['lon','lat','radius']].copy()
coord = np.char.lower(data['coord'])
gal = (coord=='gal')
cel = (coord=='cel')
hpx = (coord=='hpx')
if cel.any():
glon,glat = cel2gal(data['lon'][cel],data['lat'][cel])
out['lon'][cel] = glon
out['lat'][cel] = glat
if hpx.any():
glon,glat = pix2ang(data['lat'][hpx],data['lon'][hpx])
out['lon'][hpx] = glon
out['lat'][hpx] = glat
return names,out.view(np.ndarray)
def randomPositions(input, nside_pix, n=1):
"""
Generate n random positions within a full HEALPix mask of booleans, or a set of (lon, lat) coordinates.
nside_pix is meant to be at coarser resolution than the input mask or catalog object positions
so that gaps from star holes, bleed trails, cosmic rays, etc. are filled in.
Return the longitude and latitude of the random positions and the total area (deg^2).
Probably there is a faster algorithm, but limited much more by the simulation and fitting time
than by the time it takes to generate random positions within the mask.
"""
input = numpy.array(input)
if len(input.shape) == 1:
subpix = numpy.nonzero(input)[0] # All the valid pixels in the mask at the NSIDE for the input mask
lon, lat = pix2ang(healpy.npix2nside(len(input)), subpix)
elif len(input.shape) == 2:
lon, lat = input[0], input[1] # All catalog object positions
else:
logger.warning('Unexpected input dimensions for skymap.randomPositions')
pix = surveyPixel(lon, lat, nside_pix)
# Area with which the random points are thrown
area = len(pix) * healpy.nside2pixarea(nside_pix, degrees=True)
lon = []
lat = []
for ii in range(0, n):
# Choose an unmasked pixel at random, which is OK because HEALPix is an equal area scheme
pix_ii = pix[numpy.random.randint(0, len(pix))]
lon_ii, lat_ii = ugali.utils.projector.pixToAng(nside_pix, pix_ii)
projector = ugali.utils.projector.Projector(lon_ii, lat_ii)
inside = False
while not inside:
# Apply random offset
arcminToDegree = 1 / 60.
resolution = arcminToDegree * healpy.nside2resol(nside_pix, arcmin=True)
x = 2. * (numpy.random.rand() - 0.5) * resolution # Using factor 2 to be conservative
y = 2. * (numpy.random.rand() - 0.5) * resolution
lon_candidate, lat_candidate = projector.imageToSphere(x, y)
# Make sure that the random position does indeed fall within the randomly selected pixel
if ugali.utils.projector.angToPix(nside_pix, lon_candidate, lat_candidate) == pix_ii:
inside = True
lon.append(lon_candidate)
lat.append(lat_candidate)
return numpy.array(lon), numpy.array(lat), area
def parse_targets(filename):
"""
Load a text file with target coordinates. Returns
an array of target locations in Galactic coordinates.
File description:
[NAME] [LON] [LAT] [RADIUS] [COORD]
The values of LON and LAT will depend on COORD:
COORD = [GAL | CEL | HPX ],
LON = [GLON | RA | NSIDE]
LAT = [GLAT | DEC | PIX ]
"""
base, ext = os.path.splitext(filename)
if (ext == '.fits'):
import fitsio
data = fitsio.read(filename)
else:
from numpy.lib import NumpyVersion
if NumpyVersion(np.__version__) < '1.14.0':
data = np.genfromtxt(filename, names=True, dtype=None)
else:
data = np.genfromtxt(filename,
names=True,
dtype=None,
encoding=None)
#data = np.genfromtxt(filename,unpack=True,usecols=list(range(5)),dtype=object,names=True)
data = np.atleast_1d(data)
data.dtype.names = list(map(str.lower, data.dtype.names))
# Deal with one-line input files
#if data.ndim == 1: data = np.array([data]).T
names = data['name']
out = data[['lon', 'lat', 'radius']].copy()
coord = np.char.lower(data['coord'])
gal = (coord == 'gal')
cel = (coord == 'cel')
hpx = (coord == 'hpx')
if cel.any():
glon, glat = cel2gal(data['lon'][cel], data['lat'][cel])
out['lon'][cel] = glon
out['lat'][cel] = glat
if hpx.any():
glon, glat = pix2ang(data['lat'][hpx], data['lon'][hpx])
out['lon'][hpx] = glon
out['lat'][hpx] = glat
return names, out.view(np.ndarray)
def draw_maglim_pixel(skymap,**kwargs):
nside = healpy.npix2nside(len(skymap))
pix = np.where(skymap > 0)
if len(pix[0]) == 0:
logger.warn("No maglims found")
return
ra,dec = pix2ang(nside,pix)
ra_center,dec_center = np.median(ra),np.median(dec)
vmin,vmax = maglim_range(skymap)
kwargs.setdefault('rot',(ra_center, dec_center, 0.))
kwargs.setdefault('min',vmin)
kwargs.setdefault('max',vmax)
healpy.gnomview(skymap,**kwargs)
def calc_surface_intensity(self, factor=10):
"""Calculate the surface intensity for each pixel in the interior
region of the ROI. Pixels are adaptively subsampled around the
kernel centroid out to a radius of 'factor * max_pixrad'.
Parameters:
-----------
factor : the radius of the oversample region in units of max_pixrad
Returns:
--------
surface_intensity : the surface intensity at each pixel
"""
# First we calculate the surface intensity at native resolution
pixels = self.roi.pixels_interior
nside_in = self.config['coords']['nside_pixel']
surface_intensity = self.kernel.pdf(pixels.lon, pixels.lat)
# Then we recalculate the surface intensity around the kernel
# centroid at higher resolution
for i in np.arange(1, 5):
# Select pixels within the region of interest
nside_out = 2**i * nside_in
radius = factor * np.degrees(hp.max_pixrad(nside_out))
pix = ang2disc(nside_in,
self.kernel.lon,
self.kernel.lat,
radius,
inclusive=True)
# Select pix within the interior region of the ROI
idx = ugali.utils.healpix.index_pix_in_pixels(pix, pixels)
pix = pix[(idx >= 0)]
idx = idx[(idx >= 0)]
# Reset the surface intensity for the subsampled pixels
subpix = ugali.utils.healpix.ud_grade_ipix(pix, nside_in,
nside_out)
pix_lon, pix_lat = pix2ang(nside_out, subpix)
surface_intensity[idx] = np.mean(self.kernel.pdf(pix_lon, pix_lat),
axis=1)
return surface_intensity
def rms_photometry(catfile,nside=64,band=None,plot=False):
""" Calculate photometric repeatability """
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
columns = ['RA','DEC']
spread,nepochs = bfields(['WAVG_SPREAD_MODEL','NEPOCHS'],band)
mag,magerr,magrms = bfields(['WAVG_MAG_PSF','WAVG_MAGERR_PSF','WAVG_MAGRMS_PSF'],band)
columns += [spread, nepochs, mag, magerr, magrms]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
msg = '%s (RA,DEC) = %.2f,%.2f'%(os.path.basename(catfile),ra,dec)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < r < 20 and 0.0 < (g-i) < 1.5
sel = (np.fabs(cat[spread]) < 0.002) & \
(cat[mag] > 16) & (cat[mag] < 18) &\
(cat[magrms] < 90) &\
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
pix = ang2pix(nside,cat['RA'],cat['DEC'])
upix = np.unique(pix)
stat = nd.median(cat[magrms],labels=pix,index=upix)
if False:
plt.figure()
plt.hist(cat[magrms],bins=50)
import pdb; pdb.set_trace()
return upix,stat
def parse_targets(filename):
"""
Load a text file with target coordinates. Returns
an array of target locations in Galactic coordinates.
File description:
[NAME] [LON] [LAT] [RADIUS] [COORD]
The values of LON and LAT will depend on COORD:
COORD = [GAL | CEL | HPX ],
LON = [GLON | RA | NSIDE]
LAT = [GLAT | DEC | PIX ]
"""
base,ext = os.path.splitext(filename)
if ext == '.fits':
f = pyfits.open(filename)
data = f[1].data.view(np.recarray)
data = recfuncs.append_fields(data,'RADIUS',np.zeros(len(data)),usemask=False)
return data['NAME'],data[['GLON','GLAT','RADIUS']]
elif (ext=='.txt') or (ext=='.dat'):
data = np.loadtxt(filename,unpack=True,usecols=list(range(5)),dtype=object)
# Deal with one-line input files
if data.ndim == 1: data = np.array([data]).T
names = data[0]
out = data[1:4].astype(float)
lon,lat,radius = out
coord = np.array([s.lower() for s in data[4]])
gal = (coord=='gal')
cel = (coord=='cel')
hpx = (coord=='hpx')
if cel.any():
glon,glat = cel2gal(lon[cel],lat[cel])
out[0][cel] = glon
out[1][cel] = glat
if hpx.any():
glon,glat = pix2ang(lat[hpx],lon[hpx])
out[0][hpx] = glon
out[1][hpx] = glat
return names,out.T
def parse_targets(filename):
"""
Load a text file with target coordinates. Returns
an array of target locations in Galactic coordinates.
File description:
[NAME] [LON] [LAT] [RADIUS] [COORD]
The values of LON and LAT will depend on COORD:
COORD = [GAL | CEL | HPX ],
LON = [GLON | RA | NSIDE]
LAT = [GLAT | DEC | PIX ]
"""
base,ext = os.path.splitext(filename)
if ext == '.fits':
f = pyfits.open(filename)
data = f[1].data.view(np.recarray)
data = recfuncs.append_fields(data,'RADIUS',np.zeros(len(data)),usemask=False)
return data['NAME'],data[['GLON','GLAT','RADIUS']]
elif (ext=='.txt') or (ext=='.dat'):
data = np.loadtxt(filename,unpack=True,usecols=range(5),dtype=object)
# Deal with one-line input files
if data.ndim == 1: data = np.array([data]).T
names = data[0]
out = data[1:4].astype(float)
lon,lat,radius = out
coord = np.array([s.lower() for s in data[4]])
gal = (coord=='gal')
cel = (coord=='cel')
hpx = (coord=='hpx')
if cel.any():
glon,glat = cel2gal(lon[cel],lat[cel])
out[0][cel] = glon
out[1][cel] = glat
if hpx.any():
glon,glat = pix2ang(lat[hpx],lon[hpx])
out[0][hpx] = glon
out[1][hpx] = glat
return names,out.T
def coarseFootprint(input, nside_pix):
"""
Generate a coarse healpix mask of booleans from a finer healpix
mask or a set of (lon, lat) coordinates.
Parameters:
-----------
input : (1) full HEALPix mask of booleans, or (2) a set of (lon, lat) coordinates for catalog objects that define the occupied pixels.
nside_pix : nside_pix is meant to be at coarser (or equivalent) resolution than the input mask or catalog object positions
so that gaps from star holes, bleed trails, cosmic rays, etc. are filled in.
Returns:
--------
lon,lat,area : Return the longitude and latitude of the random positions (deg) and the total area (deg^2).
"""
input = np.array(input)
if len(input.shape) == 1:
if hp.npix2nside(len(input)) < nside_pix:
logger.warning(
'Expected coarser resolution nside_pix in skymap.randomPositions'
)
subpix = np.nonzero(
input
)[0] # All the valid pixels in the mask at the NSIDE for the input mask
lon, lat = pix2ang(hp.npix2nside(len(input)), subpix)
elif len(input.shape) == 2:
lon, lat = input[0], input[1] # All catalog object positions
else:
logger.warning(
'Unexpected input dimensions for skymap.randomPositions')
pix = surveyPixel(lon, lat, nside_pix)
# Area with which the random points are thrown
area = len(pix) * hp.nside2pixarea(nside_pix, degrees=True)
# Create mask at the coarser resolution
mask = np.tile(False, hp.nside2npix(nside_pix))
mask[pix] = True
return mask
def calc_surface_intensity(self, factor=10):
"""Calculate the surface intensity for each pixel in the interior
region of the ROI. Pixels are adaptively subsampled around the
kernel centroid out to a radius of 'factor * max_pixrad'.
Parameters:
-----------
factor : the radius of the oversample region in units of max_pixrad
Returns:
--------
surface_intensity : the surface intensity at each pixel
"""
# First we calculate the surface intensity at native resolution
pixels = self.roi.pixels_interior
nside_in = self.config['coords']['nside_pixel']
surface_intensity = self.kernel.pdf(pixels.lon,pixels.lat)
# Then we recalculate the surface intensity around the kernel
# centroid at higher resolution
for i in np.arange(1,5):
# Select pixels within the region of interest
nside_out = 2**i * nside_in
radius = factor*np.degrees(hp.max_pixrad(nside_out))
pix = ang2disc(nside_in,self.kernel.lon,self.kernel.lat,
radius,inclusive=True)
# Select pix within the interior region of the ROI
idx = ugali.utils.healpix.index_pix_in_pixels(pix,pixels)
pix = pix[(idx >= 0)]; idx = idx[(idx >= 0)]
# Reset the surface intensity for the subsampled pixels
subpix = ugali.utils.healpix.ud_grade_ipix(pix,nside_in,nside_out)
pix_lon,pix_lat = pix2ang(nside_out,subpix)
surface_intensity[idx]=np.mean(self.kernel.pdf(pix_lon,pix_lat),axis=1)
return surface_intensity
def allSkyCoordinates(nside):
"""
Generate a set of coordinates at the centers of pixels of resolutions nside across the full sky.
"""
lon,lat = pix2ang(nside, np.arange(0, hp.nside2npix(nside)))
return lon, lat
def findObjects(pixels, values, nside, zvalues, rev, good):
"""
Characterize labelled candidates in a multi-dimensional HEALPix map.
Parameters:
values : (Sparse) HEALPix array of data values
nside : HEALPix dimensionality
pixels : Pixel values associated to (sparse) HEALPix array
zvalues : Values of the z-dimension (usually distance modulus)
rev : Reverse indices for pixels in each "island"
good : Array containg labels for each "island"
Returns:
objs : numpy.recarray of object characteristics
"""
ngood = len(good)
objs = numpy.recarray((ngood,),
dtype=[('LABEL','i4'),
('NPIX','i4'),
('VAL_MAX','f4'),
('IDX_MAX','i4'),
('ZIDX_MAX','i4'),
('PIX_MAX','i4'),
('X_MAX','f4'),
('Y_MAX','f4'),
('Z_MAX','f4'),
('X_CENT','f4'),
('Y_CENT','f4'),
('Z_CENT','f4'),
('X_BARY','f4'),
('Y_BARY','f4'),
('Z_BARY','f4'),
('CUT','i2'),])
objs['CUT'][:] = 0
shape = values.shape
ncol = shape[1]
for i in range(0,ngood):
logger.debug("i=%i",i)
# This code could use some cleanup...
indices=rev[rev[good[i]]:rev[good[i]+1]]
npix = len(indices)
idx = indices // ncol # This is the spatial index
zidx = indices % ncol # This is the distance index
pix = pixels[idx] # This is the healpix pixel
xval,yval = pix2ang(nside, pix)
zval = zvalues[zidx]
objs[i]['LABEL'] = good[i]
objs[i]['NPIX'] = npix
logger.debug("LABEL=%i"%objs[i]['LABEL'])
logger.debug("NPIX=%i"%objs[i]['NPIX'])
island = values[idx,zidx]
idxmax = island.argmax()
xval_max,yval_max,zval_max = xval[idxmax],yval[idxmax],zval[idxmax]
objs[i]['VAL_MAX'] = island[idxmax]
objs[i]['IDX_MAX'] = idx[idxmax]
objs[i]['ZIDX_MAX'] = zidx[idxmax]
objs[i]['PIX_MAX'] = pix[idxmax]
objs[i]['X_MAX'] = xval_max
objs[i]['Y_MAX'] = yval_max
objs[i]['Z_MAX'] = zval_max
proj = Projector(xval_max,yval_max)
xpix,ypix = proj.sphereToImage(xval,yval)
# Projected centroid
x_cent,y_cent,zval_cent = numpy.average([xpix,ypix,zval],axis=1)
xval_cent, yval_cent = proj.imageToSphere(x_cent,y_cent)
objs[i]['X_CENT'] = xval_cent
objs[i]['Y_CENT'] = yval_cent
objs[i]['Z_CENT'] = zval_cent
# Projected barycenter
weights=[island,island,island]
x_bary,y_bary,zval_bary = numpy.average([xpix,ypix,zval],weights=weights,axis=1)
xval_bary,yval_bary = proj.imageToSphere(x_bary, y_bary)
objs[i]['X_BARY'] = xval_bary
objs[i]['Y_BARY'] = yval_bary
objs[i]['Z_BARY'] = zval_bary
return objs
def submit(self, pixels, queue=None, debug=False, configfile=None):
"""
Submit the likelihood job for the given pixel(s).
"""
# For backwards compatibility
batch = self.config['scan'].get('batch', self.config['batch'])
queue = batch.get('default', 'medium') if queue is None else queue
# Need to develop some way to take command line arguments...
self.batch = ugali.utils.batch.batchFactory(queue,
**batch.get(queue, {}))
self.batch.max_jobs = self.config['scan'].get('max_jobs', 200)
if np.isscalar(pixels): pixels = np.array([pixels])
outdir = mkdir(self.config['output']['likedir'])
logdir = mkdir(join(outdir, 'log'))
subdir = mkdir(join(outdir, 'sub'))
# Save the current configuation settings; avoid writing
# file multiple times if configfile passed as argument.
if configfile is None:
shutil.copy(self.config.filename, outdir)
configfile = join(outdir, os.path.basename(self.config.filename))
lon, lat = pix2ang(self.nside_likelihood, pixels)
commands = []
chunk = self.config['scan'].get('chunk', 25)
istart = 0
logger.info('=== Submit Likelihood ===')
for ii, pix in enumerate(pixels):
msg = ' (%i/%i) pixel=%i nside=%i; (lon, lat) = (%.2f, %.2f)'
msg = msg % (ii + 1, len(pixels), pix, self.nside_likelihood,
lon[ii], lat[ii])
logger.info(msg)
# Create outfile name
outfile = self.config.likefile % (
pix, self.config['coords']['coordsys'].lower())
outbase = os.path.basename(outfile)
jobname = batch.get('jobname', 'ugali')
# Submission command
sub = not os.path.exists(outfile)
cmd = self.command(outfile, configfile, pix)
commands.append([ii, cmd, lon[ii], lat[ii], sub])
if chunk == 0:
# No chunking
command = cmd
submit = sub
logfile = join(logdir, os.path.splitext(outbase)[0] + '.log')
elif (len(commands) % chunk == 0) or (ii + 1 == len(pixels)):
# End of chunk, create submission script
commands = np.array(commands, dtype=object)
istart, iend = commands[0][0], commands[-1][0]
subfile = join(subdir, 'submit_%08i_%08i.sh' % (istart, iend))
logfile = join(logdir, 'submit_%08i_%08i.log' % (istart, iend))
command = "sh %s" % subfile
submit = np.any(commands[:, -1])
if submit: self.write_script(subfile, commands)
else:
# Not end of chunk
continue
commands = []
# Actual job submission
if not submit:
logger.info(self.skip)
continue
else:
job = self.batch.submit(command, jobname, logfile)
logger.info(" " + job)
time.sleep(0.5)
def findObjects(pixels, values, nside, zvalues, rev, good):
"""
Characterize labelled candidates in a multi-dimensional HEALPix map.
Parameters:
values : (Sparse) HEALPix array of data values
nside : HEALPix dimensionality
pixels : Pixel values associated to (sparse) HEALPix array
zvalues : Values of the z-dimension (usually distance modulus)
rev : Reverse indices for pixels in each "island"
good : Array containg labels for each "island"
Returns:
objs : numpy.recarray of object characteristics
"""
ngood = len(good)
objs = numpy.recarray((ngood, ),
dtype=[
('LABEL', 'i4'),
('NPIX', 'i4'),
('VAL_MAX', 'f4'),
('IDX_MAX', 'i4'),
('ZIDX_MAX', 'i4'),
('PIX_MAX', 'i4'),
('X_MAX', 'f4'),
('Y_MAX', 'f4'),
('Z_MAX', 'f4'),
('X_CENT', 'f4'),
('Y_CENT', 'f4'),
('Z_CENT', 'f4'),
('X_BARY', 'f4'),
('Y_BARY', 'f4'),
('Z_BARY', 'f4'),
('CUT', 'i2'),
])
objs['CUT'][:] = 0
shape = values.shape
ncol = shape[1]
for i in range(0, ngood):
logger.debug("i=%i", i)
# This code could use some cleanup...
indices = rev[rev[good[i]]:rev[good[i] + 1]]
npix = len(indices)
idx = indices // ncol # This is the spatial index
zidx = indices % ncol # This is the distance index
pix = pixels[idx] # This is the healpix pixel
xval, yval = pix2ang(nside, pix)
zval = zvalues[zidx]
objs[i]['LABEL'] = good[i]
objs[i]['NPIX'] = npix
logger.debug("LABEL=%i" % objs[i]['LABEL'])
logger.debug("NPIX=%i" % objs[i]['NPIX'])
island = values[idx, zidx]
idxmax = island.argmax()
xval_max, yval_max, zval_max = xval[idxmax], yval[idxmax], zval[
idxmax]
objs[i]['VAL_MAX'] = island[idxmax]
objs[i]['IDX_MAX'] = idx[idxmax]
objs[i]['ZIDX_MAX'] = zidx[idxmax]
objs[i]['PIX_MAX'] = pix[idxmax]
objs[i]['X_MAX'] = xval_max
objs[i]['Y_MAX'] = yval_max
objs[i]['Z_MAX'] = zval_max
proj = Projector(xval_max, yval_max)
xpix, ypix = proj.sphereToImage(xval, yval)
# Projected centroid
x_cent, y_cent, zval_cent = numpy.average([xpix, ypix, zval],
axis=1)
xval_cent, yval_cent = proj.imageToSphere(x_cent, y_cent)
objs[i]['X_CENT'] = xval_cent
objs[i]['Y_CENT'] = yval_cent
objs[i]['Z_CENT'] = zval_cent
# Projected barycenter
weights = [island, island, island]
x_bary, y_bary, zval_bary = numpy.average([xpix, ypix, zval],
weights=weights,
axis=1)
xval_bary, yval_bary = proj.imageToSphere(x_bary, y_bary)
objs[i]['X_BARY'] = xval_bary
objs[i]['Y_BARY'] = yval_bary
objs[i]['Z_BARY'] = zval_bary
return objs
def background(self, mc_source_id=2, seed=None):
"""
Create a simulation of the background stellar population.
Because some stars have been clipped to generate the CMD,
this function tends to slightly underestimate (~1%) the
background as compared to the true catalog.
The simulation of background object colors relies on the
data-derived CMD. As such, it is a binned random generator
and thus has some fundamental limitations.
- The expected number of counts per bin is drawn ra
There are a few limitations of this procedure:
- Colors are drawn from the CMD of the background annulus
- The number of stars per CMD bin is randomized according to the CMD
- The colors/mags are then uniformly distributed within the bin
- This leads to trouble with large bins when the cloud-in-cells
algorithm is applied to the simulated data
- The positions are chosen randomly over the spherical cap of the ROI
- Objects that are outside of the
WARNING: The cloud-in-cells method of generating
the CMD leads to some difficulties since it disperses
objects from high-density zones to low density zones.
- Magnitudes are not randomized according to their errors
"""
if seed is not None: np.random.seed(seed)
self._setup_cmd()
# Randomize the number of stars per bin according to Poisson distribution
nstar_per_bin = numpy.random.poisson(lam=self.bkg_lambda)
nstar = nstar_per_bin.sum()
logger.info("Simulating %i background stars..." % nstar)
if not self.config['simulate'].get('uniform'):
logger.info("Generating colors from background CMD.")
# Distribute the stars within each CMD bin
delta_color = self.bkg_centers_color[1] - self.bkg_centers_color[0]
delta_mag = self.bkg_centers_mag[1] - self.bkg_centers_mag[0]
# Distribute points within each color-mag bins
xx, yy = np.meshgrid(self.bkg_centers_color, self.bkg_centers_mag)
color = numpy.repeat(xx.flatten(), repeats=nstar_per_bin.flatten())
color += numpy.random.uniform(-delta_color / 2.,
delta_color / 2.,
size=nstar)
mag_1 = numpy.repeat(yy.flatten(), repeats=nstar_per_bin.flatten())
mag_1 += numpy.random.uniform(-delta_mag / 2.,
delta_mag / 2.,
size=nstar)
else:
# Uniform color-magnitude distribution
logger.info("Generating uniform CMD.")
mag_1 = np.random.uniform(self.config['mag']['min'],
self.config['mag']['max'],
size=nstar)
color = np.random.uniform(self.config['color']['min'],
self.config['color']['max'],
size=nstar)
mag_2 = mag_1 - color
# Random points drawn from healpix subpixels
logger.info("Generating uniform positions...")
idx = np.random.randint(0, len(self.subpix) - 1, size=nstar)
lon, lat = pix2ang(self.nside_subpixel, self.subpix[idx])
nside_pixel = self.nside_pixel
pix = ang2pix(nside_pixel, lon, lat)
# There is probably a better way to do this step without creating the full HEALPix map
mask = -1. * numpy.ones(healpy.nside2npix(nside_pixel))
mask[self.roi.pixels] = self.mask.mask_1.mask_roi_sparse
mag_lim_1 = mask[pix]
mask = -1. * numpy.ones(healpy.nside2npix(nside_pixel))
mask[self.roi.pixels] = self.mask.mask_2.mask_roi_sparse
mag_lim_2 = mask[pix]
mag_err_1 = self.photo_err_1(mag_lim_1 - mag_1)
mag_err_2 = self.photo_err_2(mag_lim_2 - mag_2)
mc_source_id = mc_source_id * numpy.ones(len(mag_1))
# ADW: Should magnitudes be randomized by the erros?
#mag_1 += (numpy.random.normal(size=len(mag_1)) * mag_err_1)
#mag_2 += (numpy.random.normal(size=len(mag_2)) * mag_err_2)
select = (mag_lim_1 > mag_1) & (mag_lim_2 > mag_2)
### # Make sure objects lie within the original cmd (should be done later...)
### select &= (ugali.utils.binning.take2D(self.mask.solid_angle_cmd, color, mag_1,
### self.roi.bins_color, self.roi.bins_mag) > 0)
logger.info("Clipping %i simulated background stars..." %
(~select).sum())
hdu = ugali.observation.catalog.makeHDU(
self.config, mag_1[select], mag_err_1[select], mag_2[select],
mag_err_2[select], lon[select], lat[select], mc_source_id[select])
catalog = ugali.observation.catalog.Catalog(self.config, data=hdu.data)
return catalog
def calculate(self, infile, field=1, simple=False):
logger.info("Calculating magnitude limit from %s"%infile)
#manglefile = self.config['mangle']['infile_%i'%field]
#footfile = self.config['data']['footprint']
#try:
# footprint = fitsio.read(footfile)['I'].ravel()
#except:
# logger.warn("Couldn't open %s; will try again."%footfile)
# footprint = footfile
mag_column = self.config['catalog']['mag_%i_field'%field]
magerr_column = self.config['catalog']['mag_err_%i_field'%field]
# For simple maglims
release = self.config['data']['release'].lower()
band = self.config['catalog']['mag_%i_band'%field]
pixel_pix_name = 'PIX%i'%self.nside_pixel
data = fitsio.read(infile,columns=[pixel_pix_name])
#mask_pixels = np.arange( hp.nside2npix(self.nside_mask), dtype='int')
mask_maglims = np.zeros(hp.nside2npix(self.nside_mask))
out_pixels = np.zeros(0,dtype='int')
out_maglims = np.zeros(0)
# Find the objects in each pixel
pixel_pix = data[pixel_pix_name]
mask_pix = ugali.utils.skymap.superpixel(pixel_pix,self.nside_pixel,self.nside_mask)
count = Counter(mask_pix)
pixels = sorted(count.keys())
pix_digi = np.digitize(mask_pix,pixels).argsort()
idx = 0
min_num = 500
signal_to_noise = 10.
magerr_lim = 1/signal_to_noise
for pix in pixels:
# Calculate the magnitude limit in each pixel
num = count[pix]
objs = data[pix_digi[idx:idx+num]]
idx += num
if simple:
# Set constant magnitude limits
logger.debug("Simple magnitude limit for %s"%infile)
mask_maglims[pix] = MAGLIMS[release][band]
elif num < min_num:
logger.info('Found <%i objects in pixel %i'%(min_num,pix))
mask_maglims[pix] = 0
else:
mag = objs[mag_column]
magerr = objs[magerr_column]
# Estimate the magnitude limit as suggested by:
# https://deswiki.cosmology.illinois.edu/confluence/display/DO/SVA1+Release+Document
# (https://desweb.cosmology.illinois.edu/confluence/display/Operations/SVA1+Doc)
maglim = np.median(mag[(magerr>0.9*magerr_lim)&(magerr<1.1*magerr_lim)])
# Alternative method to estimate the magnitude limit by fitting median
#mag_min, mag_max = mag.min(),mag.max()
#mag_bins = np.arange(mag_min,mag_max,0.1) #0.1086?
#x,y = ugali.utils.binning.binnedMedian(mag,magerr,mag_bins)
#x,y = x[~np.isnan(y)],y[~np.isnan(y)]
#magerr_med = interp1d(x,y)
#mag0 = np.median(x)
#maglim = brentq(lambda a: magerr_med(a)-magerr_lim,x.min(),x.max(),disp=False)
# Median from just objects near magerr cut
mask_maglims[pix] = maglim
logger.debug("%i (n=%i): maglim=%g"%(pix,num,mask_maglims[pix]))
subpix = ugali.utils.skymap.subpixel(pix, self.nside_mask, self.nside_pixel)
maglims = np.zeros(len(subpix)) + mask_maglims[pix]
out_pixels = np.append(out_pixels,subpix)
out_maglims = np.append(out_maglims,maglims)
# Remove empty pixels
logger.info("Removing empty pixels")
idx = np.nonzero(out_maglims > 0)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
# Remove pixels outside the footprint
if self.footfile:
logger.info("Checking footprint against %s"%self.footfile)
glon,glat = pix2ang(self.nside_pixel,out_pixels)
ra,dec = gal2cel(glon,glat)
footprint = inFootprint(self.footprint,ra,dec)
idx = np.nonzero(footprint)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
logger.info("MAGLIM = %.3f +/- %.3f"%(np.mean(out_maglims),np.std(out_maglims)))
return out_pixels,out_maglims
def calculate(self, infile, field=1, simple=False):
logger.info("Calculating magnitude limit from %s"%infile)
#manglefile = self.config['mangle']['infile_%i'%field]
footfile = self.config['data']['footprint']
mag_column = self.config['catalog']['mag_%i_field'%field]
magerr_column = self.config['catalog']['mag_err_%i_field'%field]
# For simple maglims
release = self.config['data']['release'].lower()
band = self.config['catalog']['mag_%i_band'%field]
f = pyfits.open(infile)
header = f[1].header
data = f[1].data
#mask_pixels = numpy.arange( healpy.nside2npix(self.nside_mask), dtype='int')
mask_maglims = numpy.zeros( healpy.nside2npix(self.nside_mask) )
out_pixels = numpy.zeros(0,dtype='int')
out_maglims = numpy.zeros(0)
# Find the objects in each pixel
pixel_pix = data['PIX%i'%self.nside_pixel]
mask_pix = ugali.utils.skymap.superpixel(pixel_pix,self.nside_pixel,self.nside_mask)
count = Counter(mask_pix)
pixels = sorted(count.keys())
pix_digi = numpy.digitize(mask_pix,pixels).argsort()
idx = 0
min_num = 500
signal_to_noise = 10.
magerr_lim = 1/signal_to_noise
for pix in pixels:
# Calculate the magnitude limit in each pixel
num = count[pix]
objs = data[pix_digi[idx:idx+num]]
idx += num
if simple:
# Set constant magnitude limits
logger.debug("Simple magnitude limit for %s"%infile)
mask_maglims[pix] = MAGLIMS[release][band]
elif num < min_num:
logger.info('Found <%i objects in pixel %i'%(min_num,pix))
mask_maglims[pix] = 0
else:
mag = objs[mag_column]
magerr = objs[magerr_column]
# Estimate the magnitude limit as suggested by:
# https://deswiki.cosmology.illinois.edu/confluence/display/DO/SVA1+Release+Document
# (https://desweb.cosmology.illinois.edu/confluence/display/Operations/SVA1+Doc)
maglim = numpy.median(mag[(magerr>0.9*magerr_lim)&(magerr<1.1*magerr_lim)])
# Alternative method to estimate the magnitude limit by fitting median
#mag_min, mag_max = mag.min(),mag.max()
#mag_bins = numpy.arange(mag_min,mag_max,0.1) #0.1086?
#x,y = ugali.utils.binning.binnedMedian(mag,magerr,mag_bins)
#x,y = x[~numpy.isnan(y)],y[~numpy.isnan(y)]
#magerr_med = interp1d(x,y)
#mag0 = numpy.median(x)
#maglim = brentq(lambda a: magerr_med(a)-magerr_lim,x.min(),x.max(),disp=False)
# Median from just objects near magerr cut
mask_maglims[pix] = maglim
logger.debug("%i (n=%i): maglim=%g"%(pix,num,mask_maglims[pix]))
subpix = ugali.utils.skymap.subpixel(pix, self.nside_mask, self.nside_pixel)
maglims = numpy.zeros(len(subpix)) + mask_maglims[pix]
out_pixels = numpy.append(out_pixels,subpix)
out_maglims = numpy.append(out_maglims,maglims)
# Remove empty pixels
logger.info("Removing empty pixels")
idx = numpy.nonzero(out_maglims > 0)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
# Remove pixels outside the footprint
logger.info("Checking footprint against %s"%footfile)
glon,glat = pix2ang(self.nside_pixel,out_pixels)
ra,dec = gal2cel(glon,glat)
footprint = inFootprint(footfile,ra,dec)
idx = numpy.nonzero(footprint)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
logger.info("MAGLIM = %.3f +/- %.3f"%(numpy.mean(out_maglims),numpy.std(out_maglims)))
return out_pixels,out_maglims
def background(self,mc_source_id=2,seed=None):
"""
Create a simulation of the background stellar population.
Because some stars have been clipped to generate the CMD,
this function tends to slightly underestimate (~1%) the
background as compared to the true catalog.
The simulation of background object colors relies on the
data-derived CMD. As such, it is a binned random generator
and thus has some fundamental limitations.
- The expected number of counts per bin is drawn ra
There are a few limitations of this procedure:
- Colors are drawn from the CMD of the background annulus
- The number of stars per CMD bin is randomized according to the CMD
- The colors/mags are then uniformly distributed within the bin
- This leads to trouble with large bins when the cloud-in-cells
algorithm is applied to the simulated data
- The positions are chosen randomly over the spherical cap of the ROI
- Objects that are outside of the
WARNING: The cloud-in-cells method of generating
the CMD leads to some difficulties since it disperses
objects from high-density zones to low density zones.
- Magnitudes are not randomized according to their errors
"""
if seed is not None: np.random.seed(seed)
self._setup_cmd()
# Randomize the number of stars per bin according to Poisson distribution
nstar_per_bin = numpy.random.poisson(lam=self.bkg_lambda)
nstar = nstar_per_bin.sum()
logger.info("Simulating %i background stars..."%nstar)
if not self.config['simulate'].get('uniform'):
logger.info("Generating colors from background CMD.")
# Distribute the stars within each CMD bin
delta_color = self.bkg_centers_color[1]-self.bkg_centers_color[0]
delta_mag = self.bkg_centers_mag[1]-self.bkg_centers_mag[0]
# Distribute points within each color-mag bins
xx,yy = np.meshgrid(self.bkg_centers_color,self.bkg_centers_mag)
color = numpy.repeat(xx.flatten(),repeats=nstar_per_bin.flatten())
color += numpy.random.uniform(-delta_color/2.,delta_color/2.,size=nstar)
mag_1 = numpy.repeat(yy.flatten(),repeats=nstar_per_bin.flatten())
mag_1 += numpy.random.uniform(-delta_mag/2.,delta_mag/2.,size=nstar)
else:
# Uniform color-magnitude distribution
logger.info("Generating uniform CMD.")
mag_1 = np.random.uniform(self.config['mag']['min'],self.config['mag']['max'],size=nstar)
color = np.random.uniform(self.config['color']['min'],self.config['color']['max'],size=nstar)
mag_2 = mag_1 - color
# Random points drawn from healpix subpixels
logger.info("Generating uniform positions...")
idx = np.random.randint(0,len(self.subpix)-1,size=nstar)
lon,lat = pix2ang(self.nside_subpixel,self.subpix[idx])
nside_pixel = self.nside_pixel
pix = ang2pix(nside_pixel, lon, lat)
# There is probably a better way to do this step without creating the full HEALPix map
mask = -1. * numpy.ones(healpy.nside2npix(nside_pixel))
mask[self.roi.pixels] = self.mask.mask_1.mask_roi_sparse
mag_lim_1 = mask[pix]
mask = -1. * numpy.ones(healpy.nside2npix(nside_pixel))
mask[self.roi.pixels] = self.mask.mask_2.mask_roi_sparse
mag_lim_2 = mask[pix]
mag_err_1 = self.photo_err_1(mag_lim_1 - mag_1)
mag_err_2 = self.photo_err_2(mag_lim_2 - mag_2)
mc_source_id = mc_source_id * numpy.ones(len(mag_1))
# ADW: Should magnitudes be randomized by the erros?
#mag_1 += (numpy.random.normal(size=len(mag_1)) * mag_err_1)
#mag_2 += (numpy.random.normal(size=len(mag_2)) * mag_err_2)
select = (mag_lim_1>mag_1)&(mag_lim_2>mag_2)
### # Make sure objects lie within the original cmd (should be done later...)
### select &= (ugali.utils.binning.take2D(self.mask.solid_angle_cmd, color, mag_1,
### self.roi.bins_color, self.roi.bins_mag) > 0)
logger.info("Clipping %i simulated background stars..."%(~select).sum())
hdu = ugali.observation.catalog.makeHDU(self.config,mag_1[select],mag_err_1[select],
mag_2[select],mag_err_2[select],
lon[select],lat[select],mc_source_id[select])
catalog = ugali.observation.catalog.Catalog(self.config, data=hdu.data)
return catalog
def calculate(self, infile, field=1, simple=False):
logger.info("Calculating magnitude limit from %s"%infile)
#manglefile = self.config['mangle']['infile_%i'%field]
#footfile = self.config['data']['footprint']
#try:
# footprint = fitsio.read(footfile)['I'].ravel()
#except:
# logger.warn("Couldn't open %s; will try again."%footfile)
# footprint = footfile
mag_column = self.config['catalog']['mag_%i_field'%field]
magerr_column = self.config['catalog']['mag_err_%i_field'%field]
# For simple maglims
release = self.config['data']['release'].lower()
band = self.config['catalog']['mag_%i_band'%field]
pixel_pix_name = 'PIX%i'%self.nside_pixel
# If the data already has a healpix pixel assignment then use it
# Otherwise recalculate...
try:
data = fitsio.read(infile,columns=[pixel_pix_name])
except ValueError as e:
logger.info(str(e))
columns=[self.config['catalog']['lon_field'],
self.config['catalog']['lat_field']]
data = fitsio.read(infile,columns=columns)[columns]
pix = ang2pix(self.nside_pixel,data[columns[0]],data[columns[1]])
data = recfuncs.rec_append_fields(data,pixel_pix_name,pix)
#mask_pixels = np.arange( hp.nside2npix(self.nside_mask), dtype='int')
mask_maglims = np.zeros(hp.nside2npix(self.nside_mask))
out_pixels = np.zeros(0,dtype='int')
out_maglims = np.zeros(0)
# Find the objects in each pixel
pixel_pix = data[pixel_pix_name]
mask_pix = ugali.utils.skymap.superpixel(pixel_pix,self.nside_pixel,self.nside_mask)
count = Counter(mask_pix)
pixels = sorted(count.keys())
pix_digi = np.digitize(mask_pix,pixels).argsort()
idx = 0
min_num = 500
signal_to_noise = 10.
magerr_lim = 1/signal_to_noise
for pix in pixels:
# Calculate the magnitude limit in each pixel
num = count[pix]
objs = data[pix_digi[idx:idx+num]]
idx += num
if simple:
# Set constant magnitude limits
logger.debug("Simple magnitude limit for %s"%infile)
mask_maglims[pix] = MAGLIMS[release][band]
elif num < min_num:
logger.info('Found <%i objects in pixel %i'%(min_num,pix))
mask_maglims[pix] = 0
else:
mag = objs[mag_column]
magerr = objs[magerr_column]
# Estimate the magnitude limit as suggested by:
# https://deswiki.cosmology.illinois.edu/confluence/display/DO/SVA1+Release+Document
# (https://desweb.cosmology.illinois.edu/confluence/display/Operations/SVA1+Doc)
maglim = np.median(mag[(magerr>0.9*magerr_lim)&(magerr<1.1*magerr_lim)])
# Alternative method to estimate the magnitude limit by fitting median
#mag_min, mag_max = mag.min(),mag.max()
#mag_bins = np.arange(mag_min,mag_max,0.1) #0.1086?
#x,y = ugali.utils.binning.binnedMedian(mag,magerr,mag_bins)
#x,y = x[~np.isnan(y)],y[~np.isnan(y)]
#magerr_med = interp1d(x,y)
#mag0 = np.median(x)
#maglim = brentq(lambda a: magerr_med(a)-magerr_lim,x.min(),x.max(),disp=False)
# Median from just objects near magerr cut
mask_maglims[pix] = maglim
logger.debug("%i (n=%i): maglim=%g"%(pix,num,mask_maglims[pix]))
subpix = ugali.utils.skymap.subpixel(pix, self.nside_mask, self.nside_pixel)
maglims = np.zeros(len(subpix)) + mask_maglims[pix]
out_pixels = np.append(out_pixels,subpix)
out_maglims = np.append(out_maglims,maglims)
# Remove empty pixels
logger.info("Removing empty pixels")
idx = np.nonzero(out_maglims > 0)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
# Remove pixels outside the footprint
if self.footfile:
logger.info("Checking footprint against %s"%self.footfile)
lon,lat = pix2ang(self.nside_pixel,out_pixels)
if self.config['coords']['coordsys'] == 'gal':
ra,dec = gal2cel(lon,lat)
else:
ra,dec = lon,lat
footprint = inFootprint(self.footprint,ra,dec)
idx = np.nonzero(footprint)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
logger.info("MAGLIM = %.3f +/- %.3f"%(np.mean(out_maglims),np.std(out_maglims)))
return out_pixels,out_maglims
def gaia_photometry(catfile,nside=64,band=None,plot=False,version='edr3'):
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
#columns = [OBJECT_ID,'RA','DEC']
columns = ['RA','DEC']
spread,nepochs = ['WAVG_SPREAD_MODEL_R','NEPOCHS_R']
mag_g,mag_r,mag_i,mag_z = bfields(['MAG_PSF'],['g','r','i','z'])
#mag_g,mag_r,mag_i,mag_z = bfields(['WAVG_MAG_PSF'],['g','r','i','z'])
columns += [spread, nepochs, mag_g, mag_r, mag_i, mag_z]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
radius = np.degrees(hp.max_pixrad(NSIDE))
msg = '%s (RA,DEC,RAD) = %.2f,%.2f,%.2f'%(os.path.basename(catfile),ra,dec,radius)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < r < 20 and 0.0 < (g-i) < 1.5
sel = (np.fabs(cat[spread])<0.002) & \
(cat[mag_g]<90) & (cat[mag_r]<90) & (cat[mag_i]<90) & (cat[mag_z]<90) & \
(cat[mag_r]>16) & (cat[mag_r]<20) & \
((cat[mag_g] - cat[mag_i]) > 0.0) & \
((cat[mag_g] - cat[mag_i]) < 1.5) & \
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
#msg = "Getting external catalog: %s"%catalog
ext = get_gaia_catalog(hpx,version=version)
m = match_query(cat['RA'],cat['DEC'],ext['RA'],ext['DEC'])
# Use a fairly wide matching radius (2 arcsec)
cut = 1.0
sel = m[-1]*3600. < cut
cat_match = cat[m[0][sel]]
ext_match = ext[m[1][sel]]
cat_G = gaia_transform(cat_match[mag_g],cat_match[mag_r],cat_match[mag_i],cat_match[mag_z],
version=version)
# Need to put Gaia flux onto the AB system
ext_G = -2.5 * np.log10(ext_match['PHOT_G_MEAN_FLUX']) + 25.7934
diff = cat_G - ext_G
pix = ang2pix(nside,cat_match['RA'],cat_match['DEC'])
upix = np.unique(pix)
stat = nd.median(diff,labels=pix,index=upix)
if False:
plt.figure()
plt.hist(cat_G - ext_G)
import pdb; pdb.set_trace()
return upix,stat
def submit(self, pixels, queue=None, debug=False, configfile=None):
"""
Submit the likelihood job for the given pixel(s).
"""
queue = self.config['batch']['cluster'] if queue is None else queue
local = (queue == 'local')
# Need to develop some way to take command line arguments...
self.batch = ugali.utils.batch.batchFactory(queue,**self.config['batch']['opts'])
if numpy.isscalar(pixels): pixels = numpy.array([pixels])
outdir = mkdir(self.config['output']['likedir'])
logdir = mkdir(join(outdir,'log'))
subdir = mkdir(join(outdir,'sub'))
# Save the current configuation settings; avoid writing
# file multiple times if configfile passed as argument.
if configfile is None:
if local:
configfile = self.configfile
else:
configfile = '%s/config_queue.py'%(outdir)
self.config.write(configfile)
lon,lat = pix2ang(self.nside_likelihood,pixels)
commands = []
chunk = self.config['batch']['chunk']
istart = 0
logger.info('=== Submit Likelihood ===')
for ii,pix in enumerate(pixels):
logger.info(' (%i/%i) pixel=%i nside=%i; (lon, lat) = (%.2f, %.2f)'%(ii+1,len(pixels),pix, self.nside_likelihood,lon[ii],lat[ii]))
# Create outfile name
outfile = self.config.likefile%(pix,self.config['coords']['coordsys'].lower())
outbase = os.path.basename(outfile)
jobname = self.config['batch']['jobname']
# Submission command
sub = not os.path.exists(outfile)
cmd = self.command(outfile,configfile,pix)
commands.append([ii,cmd,lon[ii],lat[ii],sub])
if local or chunk == 0:
# Not chunking
command = cmd
submit = sub
logfile = join(logdir,os.path.splitext(outbase)[0]+'.log')
elif (len(commands)%chunk==0) or (ii+1 == len(pixels)):
# End of chunk, create submission script
commands = np.array(commands,dtype=object)
istart, iend = commands[0][0], commands[-1][0]
subfile = join(subdir,'submit_%08i_%08i.sh'%(istart,iend))
logfile = join(logdir,'submit_%08i_%08i.log'%(istart,iend))
command = "sh %s"%subfile
submit = np.any(commands[:,-1])
if submit: self.write_script(subfile,commands)
else:
# Not end of chunk
continue
commands=[]
# Actual job submission
if not submit:
logger.info(self.skip)
continue
else:
while True:
njobs = self.batch.njobs()
if njobs < self.config['batch']['max_jobs']:
break
else:
logger.info('%i jobs already in queue, waiting...'%(njobs))
time.sleep(5*chunk)
job = self.batch.submit(command,jobname,logfile)
logger.info(" "+job)
time.sleep(0.5)
def external_astrometry(catfile,catalog='GAIA',nside=64,band='r',plot=False):
#nice = os.nice(0)
#os.nice(10-nice)
if band=='all': band = 'r'
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
columns = [OBJECT_ID,'RA','DEC']
spread,mag,nepochs = bfields(['WAVG_SPREAD_MODEL','MAG_PSF','NEPOCHS'],band)
columns += [spread,mag,nepochs]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
radius = np.degrees(healpy.max_pixrad(NSIDE))
msg = '%s (RA,DEC,RAD) = %.2f,%.2f,%.2f'%(os.path.basename(catfile),ra,dec,radius)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < mag < 21
sel = (np.fabs(cat[spread])<0.002) & \
(cat[mag]>16) & \
(cat[mag]<21) & \
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
#print "Getting external catalog: %s"%catalog
if catalog in list(CATALOGS.keys()):
ext = get_vizier_catalog(ra,dec,radius,**CATALOGS[catalog])
else:
ext = get_local_catalog(ra,dec,radius,catalog)
m = match_query(cat['RA'],cat['DEC'],ext['_RAJ2000'],ext['_DEJ2000'])
# Use a fairly wide matching radius (2 arcsec)
cut = 2.0
sel = m[-1]*3600. < cut
sepdeg = m[-1][sel]
sepsec = m[-1][sel] * 3600.
sepmas = sepsec * 1000.
sep = sepmas
pix = ang2pix(nside,cat['RA'][sel],cat['DEC'][sel])
upix = np.unique(pix)
try:
peak = nd.median(sep,labels=pix,index=upix)
except ValueError:
import pdb; pdb.set_trace()
if plot:
plt.figure()
draw_angsep(sep,bins=np.linspace(0,cut*1000.,101))
if isinstance(plot,basestring):
outfile = plot
plt.savefig(outfile,bbox_inches='tight')
#return cat,ext,m
return upix,peak
def allSkyCoordinates(nside):
"""
Generate a set of coordinates at the centers of pixels of resolutions nside across the full sky.
"""
lon, lat = pix2ang(nside, np.arange(0, healpy.nside2npix(nside)))
return lon, lat
