def inFootprint(config, pixels, nside=None):
"""
Open each valid filename for the set of pixels and determine the set
of subpixels with valid data.
"""
config = Config(config)
nside_catalog = config['coords']['nside_catalog']
nside_likelihood = config['coords']['nside_likelihood']
nside_pixel = config['coords']['nside_pixel']
if numpy.isscalar(pixels): pixels = numpy.array([pixels])
if nside is None: nside = nside_likelihood
filenames = config.getFilenames()
catalog_pixels = filenames['pix'].compressed()
inside = numpy.zeros(len(pixels), dtype=bool)
if not nside_catalog:
catalog_pix = [0]
else:
catalog_pix = superpixel(pixels, nside, nside_catalog)
catalog_pix = numpy.intersect1d(catalog_pix, catalog_pixels)
for fnames in filenames[catalog_pix]:
logger.debug("Loading %s" % filenames['mask_1'])
subpix_1, val_1 = ugali.utils.skymap.readSparseHealpixMap(
fnames['mask_1'], 'MAGLIM', construct_map=False)
logger.debug("Loading %s" % fnames['mask_2'])
subpix_2, val_2 = ugali.utils.skymap.readSparseHealpixMap(
fnames['mask_2'], 'MAGLIM', construct_map=False)
subpix = numpy.intersect1d(subpix_1, subpix_2)
superpix = numpy.unique(superpixel(subpix, nside_pixel, nside))
inside |= numpy.in1d(pixels, superpix)
return inside
def inFootprint(config, pixels, nside=None):
"""
Open each valid filename for the set of pixels and determine the set
of subpixels with valid data.
"""
config = Config(config)
nside_catalog = config['coords']['nside_catalog']
nside_likelihood = config['coords']['nside_likelihood']
nside_pixel = config['coords']['nside_pixel']
if np.isscalar(pixels): pixels = np.array([pixels])
if nside is None: nside = nside_likelihood
filenames = config.getFilenames()
catalog_pixels = filenames['pix'].compressed()
inside = np.zeros(len(pixels), dtype=bool)
if not nside_catalog:
catalog_pix = [0]
else:
catalog_pix = superpixel(pixels,nside,nside_catalog)
catalog_pix = np.intersect1d(catalog_pix,catalog_pixels)
for fnames in filenames[catalog_pix]:
logger.debug("Loading %s"%filenames['mask_1'])
#subpix_1,val_1 = ugali.utils.skymap.readSparseHealpixMap(fnames['mask_1'],'MAGLIM',construct_map=False)
_nside,subpix_1,val_1 = ugali.utils.healpix.read_partial_map(fnames['mask_1'],'MAGLIM',fullsky=False)
logger.debug("Loading %s"%fnames['mask_2'])
#subpix_2,val_2 = ugali.utils.skymap.readSparseHealpixMap(fnames['mask_2'],'MAGLIM',construct_map=False)
_nside,subpix_2,val_2 = ugali.utils.healpix.read_partial_map(fnames['mask_2'],'MAGLIM',fullsky=False)
subpix = np.intersect1d(subpix_1,subpix_2)
superpix = np.unique(superpixel(subpix,nside_pixel,nside))
inside |= np.in1d(pixels, superpix)
return inside
def __init__(self, config, roi=None, data=None, filenames=None):
"""
Class to store information about detected objects. This class
augments the raw data array with several aliases and derived
quantities.
Parameters:
-----------
config : Configuration object
roi : Region of Interest to load catalog data for
data : Data array object
filenames : FITS filenames to read catalog from
Returns:
--------
catalog : The Catalog object
"""
self.config = Config(config)
if data is None:
self._parse(roi, filenames)
else:
self.data = data
self._defineVariables()
def inFootprint(config, pixels, nside=None):
"""
Open each valid filename for the set of pixels and determine the set
of subpixels with valid data.
Parameters
----------
config : config
Configuration (file or object)
pixels : array or int
List of pixels to create footprint for
nside : int, optional
Healpix nside
Returns
-------
inside : array
Boolean array of whether pixel is in footprint
"""
logger.info("Calculating survey footprint...")
config = Config(config)
nside_catalog = config['coords']['nside_catalog']
nside_likelihood = config['coords']['nside_likelihood']
nside_pixel = config['coords']['nside_pixel']
if np.isscalar(pixels): pixels = np.array([pixels])
if nside is None: nside = nside_likelihood
filenames = config.getFilenames()
catalog_pixels = filenames['pix'].compressed()
inside = np.zeros(len(pixels), dtype=bool)
if not nside_catalog:
catalog_pix = [0]
else:
catalog_pix = superpixel(pixels, nside, nside_catalog)
catalog_pix = np.intersect1d(catalog_pix, catalog_pixels)
fnames = filenames[catalog_pix]
# Load the first mask
logger.debug("Loading %s" % fnames['mask_1'])
_nside, subpix1, val1 = read_partial_map(fnames['mask_1'],
'MAGLIM',
fullsky=False,
multiproc=8)
# Load the second mask
logger.debug("Loading %s" % fnames['mask_2'])
_nside, subpix2, val2 = read_partial_map(fnames['mask_2'],
'MAGLIM',
fullsky=False,
multiproc=8)
# Run the subpixels
subpix = np.intersect1d(subpix1, subpix2)
superpix = np.unique(superpixel(subpix, nside_pixel, nside))
inside |= np.in1d(pixels, superpix)
return inside
def simple_split(config,dirname='split',force=False):
config = Config(config)
filenames = config.getFilenames()
healpix = filenames['pix'].compressed()
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
release = config['data']['release'].lower()
band_1 = config['catalog']['mag_1_band']
band_2 = config['catalog']['mag_2_band']
mangledir = config['mangle']['dirname']
mangle_file_1 = join(mangledir,config['mangle']['filename_1'])
logger.info("Reading %s..."%mangle_file_1)
mangle_1 = healpy.read_map(mangle_file_1)
mangle_file_2 = join(mangledir,config['mangle']['filename_2'])
logger.info("Reading %s..."%mangle_file_2)
mangle_2 = healpy.read_map(mangle_file_2)
basedir,basename = os.path.split(config['mask']['dirname'])
if basename == dirname:
msg = "Input and output directory are the same."
raise Exception(msg)
outdir = mkdir(os.path.join(basedir,dirname))
mask_1 = os.path.basename(config['mask']['basename_1'])
mask_2 = os.path.basename(config['mask']['basename_2'])
for band,mangle,base in [(band_1,mangle_1,mask_1),(band_2,mangle_2,mask_2)]:
maglim = MAGLIMS[release][band]
nside_mangle = healpy.npix2nside(len(mangle))
if nside_mangle != nside_pixel:
msg = "Mangle nside different from pixel nside"
logger.warning(msg)
#raise Exception(msg)
pixels = np.nonzero((mangle>0)&(mangle>maglim))[0]
print len(pixels)
superpix = superpixel(pixels,nside_mangle,nside_catalog)
print healpix
for hpx in healpix:
outfile = join(outdir,base)%hpx
if os.path.exists(outfile) and not force:
logger.warning("Found %s; skipping..."%outfile)
continue
pix = pixels[superpix == hpx]
print hpx, len(pix)
maglims = maglim*np.ones(len(pix))
data = dict(MAGLIM=maglims )
logger.info('Writing %s...'%outfile)
ugali.utils.skymap.writeSparseHealpixMap(pix,data,nside_pixel,outfile)
def createSource(config, section=None, **kwargs):
config = Config(config)
source = Source()
if config.get(section) is not None:
params = config.get(section).get('source')
else:
params = config.get('source')
if params is not None:
source.load(params)
source.set_params(**kwargs)
return source
def createSource(config, section=None, **kwargs):
config = Config(config)
source = Source()
if config.get(section) is not None:
params = config.get(section).get('source')
else:
params = config.get('source')
if params is not None:
source.load(params)
source.set_params(**kwargs)
return source
def __init__(self, config, roi=None, data=None, filenames=None):
"""
Class to store information about detected objects. This class
augments the raw data array with several aliases and derived
quantities.
Parameters:
-----------
config : Configuration object
roi : Region of Interest to load catalog data for
data : Data array object
filenames : FITS filenames to read catalog from
Returns:
--------
catalog : The Catalog object
"""
self.config = Config(config)
if data is None:
self._parse(roi,filenames)
else:
self.data = data
self._defineVariables()
def __init__(self, config, observation, source):
# Currently assuming that input mask is ROI-specific
self.config = Config(config)
self.roi = observation.roi
self.mask = observation.mask
self.catalog_full = observation.catalog
self.clip_catalog()
# The source model (includes kernel and isochrone)
self.source = source
# Effective bin size in color-magnitude space
self.delta_mag = self.config['likelihood']['delta_mag']
self.spatial_only = self.config['likelihood'].get(
'spatial_only', False)
self.color_only = self.config['likelihood'].get('color_only', False)
if self.spatial_only and self.color_only:
msg = "Both 'spatial_only' and 'color_only' set"
logger.error(msg)
raise ValueError(msg)
elif self.spatial_only:
logger.warning(
"Likelihood calculated from spatial information only!!!")
elif self.color_only:
logger.warning(
"Likelihood calculated from color information only!!!")
self.calc_background()
def parse_args(self):
self.opts = self.parser.parse_args()
if not self.opts.run:
self.opts.run = self.components
self.config = Config(self.opts.config)
self.batch = ugali.utils.batch.batchFactory(self.opts.queue)
def __init__(self, config, loglike): # What it should be...
"""
Object to efficiently search over a grid of ROI positions.
Parameters:
-----------
config : Configuration object or filename.
loglike : Log-likelihood object
Returns:
--------
grid : GridSearch instance
"""
self.config = Config(config)
self.loglike = loglike
self.source = self.loglike.source
self.roi = self.loglike.roi
self.mask = self.loglike.mask
logger.info(str(self.loglike))
self.stellar_mass_conversion = self.source.stellar_mass()
self.distance_modulus_array = np.asarray(
self.config['scan']['distance_modulus_array'])
self.extension_array = np.asarray(self.config['scan'].get(
'extension_array', [self.source.extension]))
def _setup_cmd(self, mode='cloud-in-cells'):
"""
The purpose here is to create a more finely binned
background CMD to sample from.
"""
# Only setup once...
if hasattr(self, 'bkg_lambda'): return
logger.info("Setup color...")
# In the limit theta->0: 2*pi*(1-cos(theta)) -> pi*theta**2
# (Remember to convert from sr to deg^2)
#solid_angle_roi = sr2deg(2*np.pi*(1-np.cos(np.radians(self.roi_radius))))
solid_angle_roi = self.roi.area_pixel * len(self.roi.pixels)
# Large CMD bins cause problems when simulating
config = Config(self.config)
config['color']['n_bins'] *= 5 #10
config['mag']['n_bins'] *= 1 #2
#config['mask']['minimum_solid_angle'] = 0
roi = ugali.analysis.loglike.createROI(config, self.roi.lon,
self.roi.lat)
mask = ugali.analysis.loglike.createMask(config, roi)
self.bkg_centers_color = roi.centers_color
self.bkg_centers_mag = roi.centers_mag
# Background CMD has units: [objs / deg^2 / mag^2]
cmd_background = mask.backgroundCMD(self.catalog, mode)
self.bkg_lambda = cmd_background * solid_angle_roi * roi.delta_color * roi.delta_mag
np.sum(self.bkg_lambda)
# Clean up
del config, roi, mask
def parse_args(self):
self.opts = self.parser.parse_args()
if not self.opts.run:
self.opts.run = self.defaults
self.config = Config(self.opts.config)
# Setup the batch system
#kwargs = self.config['batch'].get(self.opts.queue,dict())
self.batch = ugali.utils.batch.batch_factory(self.opts.queue)
def __init__(self, config, loglike, samples=None):
self.config = Config(config)
self.alpha = self.config['results'].get('alpha', 0.10)
self.nwalkers = self.config['mcmc'].get('nwalkers', 100)
self.nburn = self.config['results'].get('nburn', 10)
self.loglike = loglike
self.source = self.loglike.source
self.params = list(self.source.get_free_params().keys())
self.samples = samples
def __init__(self, config, roi):
self.config = Config(config)
self.roi = roi
filenames = self.config.getFilenames()
catalog_pixels = self.roi.getCatalogPixels()
self.mask_1 = MaskBand(filenames['mask_1'][catalog_pixels], self.roi)
self.mask_2 = MaskBand(filenames['mask_2'][catalog_pixels], self.roi)
self.minimum_solid_angle = self.config.params['mask'][
'minimum_solid_angle'] # deg^2
# FIXME: Need to parallelize CMD and MMD formulation
self._solidAngleCMD()
self._pruneCMD(self.minimum_solid_angle)
#self._solidAngleMMD()
#self._pruneMMD(self.minimum_solid_angle)
self._photometricErrors()
def __init__(self, config, roi=None, data=None):
"""
Class to store information about detected objects.
The raw data from the fits file is stored. lon and lat are derived quantities
based on chosen coordinate system.
INPUTS:
config: Config object
roi[None] : Region of Interest to load catalog data for
data[None]: pyfits table data (fitsrec) object.
"""
#self = config.merge(config_merge) # Maybe you would want to update parameters??
self.config = Config(config)
if data is None:
self._parse(roi)
else:
self.data = data
self._defineVariables()
def footprint(config, nside=None):
"""
UNTESTED.
Should return a boolean array representing the pixels in the footprint.
"""
config = Config(config)
if nside is None:
nside = config['coords']['nside_pixel']
elif nside < config['coords']['nside_catalog']:
raise Exception('Requested nside=%i is greater than catalog_nside' %
nside)
elif nside > config['coords']['nside_pixel']:
raise Exception('Requested nside=%i is less than pixel_nside' % nside)
pix = numpy.arange(healpy.nside2npix(nside), dtype=int)
return inFootprint(config, pix)
def __init__(self, config, roi, maglim_1=23, maglim_2=23):
self.config = Config(config)
self.roi = roi
self.mask_1 = SimpleMaskBand(maglim_1,self.roi)
self.mask_2 = SimpleMaskBand(maglim_2,self.roi)
self.minimum_solid_angle = self.config.params['mask']['minimum_solid_angle'] # deg^2
# FIXME: Need to parallelize CMD and MMD formulation
self._solidAngleCMD()
self._pruneCMD(self.minimum_solid_angle)
#self._solidAngleMMD()
#self._pruneMMD(self.minimum_solid_angle)
self._photometricErrors()
def __init__(self, config, loglike): # What it should be...
"""
Object to efficiently search over a grid of ROI positions.
"""
self.config = Config(config)
self.loglike = loglike
self.roi = self.loglike.roi
self.mask = self.loglike.mask
#logger.info("Creating log-likelihood...")
#self.loglike=LogLikelihood(config,roi,mask,catalog,isochrone,kernel)
#self.loglike=LogLikelihood(config,observation,source)
logger.info(str(self.loglike))
self.stellar_mass_conversion = self.loglike.source.stellar_mass()
self.distance_modulus_array = np.asarray(
self.config['scan']['distance_modulus_array'])
def __init__(self, config, loglike):
self.config = Config(config)
self.nsamples = self.config['mcmc'].get('nsamples',100)
self.nthreads = self.config['mcmc'].get('nthreads',16)
#self.nthreads = multiprocessing.cpu_count()
self.nchunk = self.config['mcmc'].get('nchunk',25)
self.nwalkers = self.config['mcmc'].get('nwalkers',50)
self.nburn = self.config['mcmc'].get('nburn',10)
self.loglike = loglike
self.source = self.loglike.source
self.params = list(self.source.get_free_params().keys())
self.samples = None
self.priors = odict(list(zip(self.params,
len(self.params)*[UniformPrior()])))
self.priors['extension'] = InversePrior()
self.pool = Pool()
def __init__(self, config, roi):
self.config = Config(config)
self.roi = roi
filenames = self.config.getFilenames()
catalog_pixels = self.roi.getCatalogPixels()
self.mask_1 = MaskBand(filenames['mask_1'][catalog_pixels],self.roi)
self.mask_2 = MaskBand(filenames['mask_2'][catalog_pixels],self.roi)
self._fracRoiSparse()
self.minimum_solid_angle = self.config.params['mask']['minimum_solid_angle'] # deg^2
# FIXME: Need to parallelize CMD and MMD formulation
self._solidAngleCMD()
self._pruneCMD(self.minimum_solid_angle)
#self._solidAngleMMD()
#self._pruneMMD(self.minimum_solid_angle)
self._photometricErrors()
def __init__(self, config, coords):
self.config = Config(config)
# Should only be one coordinate
if len(coords) != 1: raise Exception('Must specify one coordinate.')
self.lon, self.lat, radius = coords[0]
self._setup()
class Maglims(object):
""" Object for deriving magnitude limits from the catalog """
def __init__(self, config):
self.config = Config(config)
self._setup()
def _setup(self):
self.nside_catalog = self.config['coords']['nside_catalog']
self.nside_mask = self.config['coords']['nside_mask']
self.nside_pixel = self.config['coords']['nside_pixel']
self.filenames = self.config.getFilenames()
self.footfile = self.config['data']['footprint']
try:
self.footprint = fitsio.read(self.footfile)['I'].ravel()
except:
logger.warn("Couldn't open %s; will pass through."%self.footfile)
self.footprint = self.footfile
def run(self,field=None,simple=False,force=False):
"""
Loop through pixels containing catalog objects and calculate
the magnitude limit. This gets a bit convoluted due to all
the different pixel resolutions...
"""
if field is None: fields = [1,2]
else: fields = [field]
for filenames in self.filenames.compress(~self.filenames.mask['catalog']).data:
infile = filenames['catalog']
for f in fields:
outfile = filenames['mask_%i'%f]
if os.path.exists(outfile) and not force:
logger.info("Found %s; skipping..."%outfile)
continue
pixels,maglims=self.calculate(infile,f,simple)
logger.info("Creating %s"%outfile)
outdir = mkdir(os.path.dirname(outfile))
data = odict()
data['PIXEL']=pixels
data['MAGLIM']=maglims.astype('f4')
ugali.utils.healpix.write_partial_map(outfile,data,
self.nside_pixel)
def calculate(self, infile, field=1, simple=False):
logger.info("Calculating magnitude limit from %s"%infile)
#manglefile = self.config['mangle']['infile_%i'%field]
#footfile = self.config['data']['footprint']
#try:
# footprint = fitsio.read(footfile)['I'].ravel()
#except:
# logger.warn("Couldn't open %s; will try again."%footfile)
# footprint = footfile
mag_column = self.config['catalog']['mag_%i_field'%field]
magerr_column = self.config['catalog']['mag_err_%i_field'%field]
# For simple maglims
release = self.config['data']['release'].lower()
band = self.config['catalog']['mag_%i_band'%field]
pixel_pix_name = 'PIX%i'%self.nside_pixel
# If the data already has a healpix pixel assignment then use it
# Otherwise recalculate...
try:
data = fitsio.read(infile,columns=[pixel_pix_name])
except ValueError as e:
logger.info(str(e))
columns=[self.config['catalog']['lon_field'],
self.config['catalog']['lat_field']]
data = fitsio.read(infile,columns=columns)[columns]
pix = ang2pix(self.nside_pixel,data[columns[0]],data[columns[1]])
data = recfuncs.rec_append_fields(data,pixel_pix_name,pix)
#mask_pixels = np.arange( hp.nside2npix(self.nside_mask), dtype='int')
mask_maglims = np.zeros(hp.nside2npix(self.nside_mask))
out_pixels = np.zeros(0,dtype='int')
out_maglims = np.zeros(0)
# Find the objects in each pixel
pixel_pix = data[pixel_pix_name]
mask_pix = ugali.utils.skymap.superpixel(pixel_pix,self.nside_pixel,self.nside_mask)
count = Counter(mask_pix)
pixels = sorted(count.keys())
pix_digi = np.digitize(mask_pix,pixels).argsort()
idx = 0
min_num = 500
signal_to_noise = 10.
magerr_lim = 1/signal_to_noise
for pix in pixels:
# Calculate the magnitude limit in each pixel
num = count[pix]
objs = data[pix_digi[idx:idx+num]]
idx += num
if simple:
# Set constant magnitude limits
logger.debug("Simple magnitude limit for %s"%infile)
mask_maglims[pix] = MAGLIMS[release][band]
elif num < min_num:
logger.info('Found <%i objects in pixel %i'%(min_num,pix))
mask_maglims[pix] = 0
else:
mag = objs[mag_column]
magerr = objs[magerr_column]
# Estimate the magnitude limit as suggested by:
# https://deswiki.cosmology.illinois.edu/confluence/display/DO/SVA1+Release+Document
# (https://desweb.cosmology.illinois.edu/confluence/display/Operations/SVA1+Doc)
maglim = np.median(mag[(magerr>0.9*magerr_lim)&(magerr<1.1*magerr_lim)])
# Alternative method to estimate the magnitude limit by fitting median
#mag_min, mag_max = mag.min(),mag.max()
#mag_bins = np.arange(mag_min,mag_max,0.1) #0.1086?
#x,y = ugali.utils.binning.binnedMedian(mag,magerr,mag_bins)
#x,y = x[~np.isnan(y)],y[~np.isnan(y)]
#magerr_med = interp1d(x,y)
#mag0 = np.median(x)
#maglim = brentq(lambda a: magerr_med(a)-magerr_lim,x.min(),x.max(),disp=False)
# Median from just objects near magerr cut
mask_maglims[pix] = maglim
logger.debug("%i (n=%i): maglim=%g"%(pix,num,mask_maglims[pix]))
subpix = ugali.utils.skymap.subpixel(pix, self.nside_mask, self.nside_pixel)
maglims = np.zeros(len(subpix)) + mask_maglims[pix]
out_pixels = np.append(out_pixels,subpix)
out_maglims = np.append(out_maglims,maglims)
# Remove empty pixels
logger.info("Removing empty pixels")
idx = np.nonzero(out_maglims > 0)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
# Remove pixels outside the footprint
if self.footfile:
logger.info("Checking footprint against %s"%self.footfile)
lon,lat = pix2ang(self.nside_pixel,out_pixels)
if self.config['coords']['coordsys'] == 'gal':
ra,dec = gal2cel(lon,lat)
else:
ra,dec = lon,lat
footprint = inFootprint(self.footprint,ra,dec)
idx = np.nonzero(footprint)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
logger.info("MAGLIM = %.3f +/- %.3f"%(np.mean(out_maglims),np.std(out_maglims)))
return out_pixels,out_maglims
class Catalog:
def __init__(self, config, roi=None, data=None, filenames=None):
"""
Class to store information about detected objects. This class
augments the raw data array with several aliases and derived
quantities.
Parameters:
-----------
config : Configuration object
roi : Region of Interest to load catalog data for
data : Data array object
filenames : FITS filenames to read catalog from
Returns:
--------
catalog : The Catalog object
"""
self.config = Config(config)
if data is None:
self._parse(roi, filenames)
else:
self.data = data
self._defineVariables()
def __add__(self, other):
return mergeCatalogs([self, other])
def __len__(self):
return len(self.objid)
def applyCut(self, cut):
"""
Return a new catalog which is a subset of objects selected
using the input cut array.
NOTE: This should really be a selection.
"""
return Catalog(self.config, data=self.data[cut])
def bootstrap(self, mc_bit=0x10, seed=None):
"""
Return a random catalog by boostrapping the colors of the objects in the current catalog.
"""
if seed is not None: np.random.seed(seed)
data = copy.deepcopy(self.data)
idx = np.random.randint(0, len(data), len(data))
data[self.config['catalog']['mag_1_field']][:] = self.mag_1[idx]
data[self.config['catalog']
['mag_err_1_field']][:] = self.mag_err_1[idx]
data[self.config['catalog']['mag_2_field']][:] = self.mag_2[idx]
data[self.config['catalog']
['mag_err_2_field']][:] = self.mag_err_2[idx]
data[self.config['catalog']['mc_source_id_field']][:] |= mc_bit
return Catalog(self.config, data=data)
def project(self, projector=None):
"""
Project coordinates on sphere to image plane using Projector class.
"""
if projector is None:
try:
self.projector = ugali.utils.projector.Projector(
self.config['coords']['reference'][0],
self.config['coords']['reference'][1])
except KeyError:
logger.warning(
'Projection reference point is median (lon, lat) of catalog objects'
)
self.projector = ugali.utils.projector.Projector(
np.median(self.lon), np.median(self.lat))
else:
self.projector = projector
self.x, self.y = self.projector.sphereToImage(self.lon, self.lat)
def spatialBin(self, roi):
"""
Calculate indices of ROI pixels corresponding to object locations.
"""
if hasattr(self, 'pixel_roi_index') and hasattr(self, 'pixel'):
logger.warning('Catalog alread spatially binned')
return
# ADW: Not safe to set index = -1 (since it will access last entry);
# np.inf would be better...
self.pixel = ang2pix(self.config['coords']['nside_pixel'], self.lon,
self.lat)
self.pixel_roi_index = roi.indexROI(self.lon, self.lat)
if np.any(self.pixel_roi_index < 0):
logger.warning("Objects found outside ROI")
def write(self, outfile, clobber=True, **kwargs):
"""
Write the current object catalog to FITS file.
Parameters:
-----------
filename : the FITS file to write.
clobber : remove existing file
kwargs : passed to fitsio.write
Returns:
--------
None
"""
fitsio.write(outfile, self.data, clobber=True, **kwargs)
def _parse(self, roi=None, filenames=None):
"""
Parse catalog FITS files into recarray.
Parameters:
-----------
roi : The region of interest; if 'roi=None', read all catalog files
Returns:
--------
None
"""
if (roi is not None) and (filenames is not None):
msg = "Cannot take both roi and filenames"
raise Exception(msg)
if roi is not None:
pixels = roi.getCatalogPixels()
filenames = self.config.getFilenames()['catalog'][pixels]
elif filenames is None:
filenames = self.config.getFilenames()['catalog'].compressed()
else:
filenames = np.atleast_1d(filenames)
if len(filenames) == 0:
msg = "No catalog files found."
raise Exception(msg)
# Load the data
self.data = load_infiles(filenames)
# Apply a selection cut
self._applySelection()
# Cast data to recarray (historical reasons)
self.data = self.data.view(np.recarray)
def _applySelection(self, selection=None):
# ADW: This is a hack (eval is unsafe!)
if selection is None:
selection = self.config['catalog'].get('selection')
if not selection:
pass
elif 'self.data' not in selection:
msg = "Selection does not contain 'data'"
raise Exception(msg)
else:
logger.warning('Evaluating selection: \n"%s"' % selection)
sel = eval(selection)
self.data = self.data[sel]
def _defineVariables(self):
"""
Helper funtion to define pertinent variables from catalog data.
ADW (20170627): This has largely been replaced by properties.
"""
logger.info('Catalog contains %i objects' % (len(self.data)))
mc_source_id_field = self.config['catalog']['mc_source_id_field']
if mc_source_id_field is not None:
if mc_source_id_field not in self.data.dtype.names:
array = np.zeros(len(self.data), dtype=int)
self.data = mlab.rec_append_fields(self.data,
names=mc_source_id_field,
arrs=array)
logger.info('Found %i simulated objects' %
(np.sum(self.mc_source_id > 0)))
# Use properties to avoid duplicating the data
@property
def objid(self):
return self.data[self.config['catalog']['objid_field']]
@property
def lon(self):
return self.data[self.config['catalog']['lon_field']]
@property
def lat(self):
return self.data[self.config['catalog']['lat_field']]
@property
def mag_1(self):
return self.data[self.config['catalog']['mag_1_field']]
@property
def mag_err_1(self):
return self.data[self.config['catalog']['mag_err_1_field']]
@property
def mag_2(self):
return self.data[self.config['catalog']['mag_2_field']]
@property
def mag_err_2(self):
return self.data[self.config['catalog']['mag_err_2_field']]
@property
def mag(self):
if self.config['catalog']['band_1_detection']: return self.mag_1
else: return self.mag_2
@property
def mag_err(self):
if self.config['catalog']['band_1_detection']: return self.mag_err_1
else: return self.mag_err_2
@property
def color(self):
return self.mag_1 - self.mag_2
@property
def color_err(self):
return np.sqrt(self.mag_err_1**2 + self.mag_err_2**2)
@property
def mc_source_id(self):
return self.data.field(self.config['catalog']['mc_source_id_field'])
# This assumes Galactic coordinates
@property
def ra_dec(self):
return gal2cel(self.lon, self.lat)
@property
def ra(self):
return self.ra_dec[0]
@property
def dec(self):
return self.ra_dec[1]
@property
def glon_glat(self):
return self.lon, self.lat
@property
def glon(self):
return self.lon
@property
def glat(self):
return self.lat
def __init__(self, config, mergefile=None, roifile=None):
self.config = Config(config)
self._config()
if mergefile is not None: self.mergefile = mergefile
if roifile is not None: self.roifile = roifile
self._load()
def __init__(self, config):
self.config = Config(config)
self._setup()
def split(config,dirname='split',force=False):
""" Take a pre-existing maglim map and divide it into
chunks consistent with the catalog pixels. """
config = Config(config)
filenames = config.getFilenames()
#healpix = filenames['pix'].compressed()
# Check that things are ok
basedir,basename = os.path.split(config['mask']['dirname'])
#if basename == dirname:
# msg = "Input and output directory are the same."
# raise Exception(msg)
outdir = mkdir(os.path.join(basedir,dirname))
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
release = config['data']['release'].lower()
band1 = config['catalog']['mag_1_band']
band2 = config['catalog']['mag_2_band']
# Read the magnitude limits
maglimdir = config['maglim']['dirname']
maglimfile_1 = join(maglimdir,config['maglim']['filename_1'])
logger.info("Reading %s..."%maglimfile_1)
maglim1 = read_map(maglimfile_1)
maglimfile_2 = join(maglimdir,config['maglim']['filename_2'])
logger.info("Reading %s..."%maglimfile_2)
maglim2 = read_map(maglimfile_2)
# Read the footprint
footfile = config['data']['footprint']
logger.info("Reading %s..."%footfile)
footprint = read_map(footfile)
# Output mask names
mask1 = os.path.basename(config['mask']['basename_1'])
mask2 = os.path.basename(config['mask']['basename_2'])
for band,maglim,base in [(band1,maglim1,mask1),(band2,maglim2,mask2)]:
nside_maglim = hp.npix2nside(len(maglim))
if nside_maglim != nside_pixel:
msg = "Mask nside different from pixel nside"
logger.warning(msg)
#raise Exception(msg)
pixels = np.nonzero(maglim>0)[0]
superpix = superpixel(pixels,nside_maglim,nside_catalog)
healpix = np.unique(superpix)
for hpx in healpix:
outfile = join(outdir,base)%hpx
if os.path.exists(outfile) and not force:
logger.warning("Found %s; skipping..."%outfile)
continue
pix = pixels[superpix == hpx]
print(hpx, len(pix))
logger.info('Writing %s...'%outfile)
data = odict()
data['PIXEL']=pix
data['MAGLIM']=maglim[pix].astype('f4')
data['FRACDET']=footprint[pix].astype('f4')
ugali.utils.healpix.write_partial_map(outfile,data,nside_pixel)
def __init__(self, config, lon, lat):
self.config = Config(config)
self.lon = lon
self.lat = lat
self.projector = ugali.utils.projector.Projector(self.lon, self.lat)
self.vec = vec = ang2vec(self.lon, self.lat)
self.pix = ang2pix(self.config['coords']['nside_likelihood'],self.lon,self.lat)
# Pixels from the entire ROI disk
pix = query_disc(self.config['coords']['nside_pixel'], vec,
self.config['coords']['roi_radius'])
self.pixels = PixelRegion(self.config['coords']['nside_pixel'],pix)
# Pixels in the interior region
pix = query_disc(self.config['coords']['nside_pixel'], vec,
self.config['coords']['roi_radius_interior'])
self.pixels_interior = PixelRegion(self.config['coords']['nside_pixel'],pix)
# Pixels in the outer annulus
pix = query_disc(self.config['coords']['nside_pixel'], vec,
self.config['coords']['roi_radius_annulus'])
pix = numpy.setdiff1d(self.pixels, pix)
self.pixels_annulus = PixelRegion(self.config['coords']['nside_pixel'],pix)
# Pixels within target healpix region
pix = ugali.utils.skymap.subpixel(self.pix,self.config['coords']['nside_likelihood'],
self.config['coords']['nside_pixel'])
self.pixels_target = PixelRegion(self.config['coords']['nside_pixel'],pix)
# Boolean arrays for selecting given pixels
# (Careful, this works because pixels are pre-sorted by query_disc before in1d)
self.pixel_interior_cut = numpy.in1d(self.pixels, self.pixels_interior)
# ADW: Updated for more general ROI shapes
#self.pixel_annulus_cut = ~self.pixel_interior_cut
self.pixel_annulus_cut = numpy.in1d(self.pixels, self.pixels_annulus)
# # These should be unnecessary now
# self.centers_lon, self.centers_lat = self.pixels.lon, self.pixels.lat
# self.centers_lon_interior,self.centers_lat_interior = self.pixels_interior.lon,self.pixels_interior.lat
# self.centers_lon_target, self.centers_lat_target = self.pixels_target.lon, self.pixels_target.lat
self.area_pixel = healpy.nside2pixarea(self.config.params['coords']['nside_pixel'],degrees=True) # deg^2
"""
self.centers_x = self._centers(self.bins_x)
self.centers_y = self._centers(self.bins_y)
self.delta_x = self.config.params['coords']['pixel_size']
self.delta_y = self.config.params['coords']['pixel_size']
# Should actually try to take projection effects into account for better accuracy
# MC integration perhaps?
# Throw points in a cone around full ROI and see what fraction fall in
self.area_pixel = self.config.params['coords']['pixel_size']**2
self.centers_lon, self.centers_lat = self.projector.imageToSphere(self.centers_x, self.centers_y)
"""
# ADW: These are really bin edges, should be careful and consistent
self.bins_mag = numpy.linspace(self.config.params['mag']['min'],
self.config.params['mag']['max'],
self.config.params['mag']['n_bins'] + 1)
self.bins_color = numpy.linspace(self.config.params['color']['min'],
self.config.params['color']['max'],
self.config.params['color']['n_bins'] + 1)
self.centers_mag = ugali.utils.binning.centers(self.bins_mag)
self.centers_color = ugali.utils.binning.centers(self.bins_color)
self.delta_mag = self.bins_mag[1] - self.bins_mag[0]
self.delta_color = self.bins_color[1] - self.bins_color[0]
# Axis labels
self.label_x = 'x (deg)'
self.label_y = 'y (deg)'
if self.config.params['catalog']['band_1_detection']:
self.label_mag = '%s (mag)'%(self.config.params['catalog']['mag_1_field'])
else:
self.label_mag = '%s (mag)'%(self.config.params['catalog']['mag_2_field'])
self.label_color = '%s - %s (mag)'%(self.config.params['catalog']['mag_1_field'],
self.config.params['catalog']['mag_2_field'])
class Maglims(object):
def __init__(self, config):
self.config = Config(config)
self._setup()
def _setup(self):
self.nside_catalog = self.config['coords']['nside_catalog']
self.nside_mask = self.config['coords']['nside_mask']
self.nside_pixel = self.config['coords']['nside_pixel']
self.filenames = self.config.getFilenames()
def run(self,field=None,simple=False,force=False):
"""
Loop through pixels containing catalog objects and calculate
the magnitude limit. This gets a bit convoluted due to all
the different pixel resolutions...
"""
if field is None: fields = [1,2]
else: fields = [field]
for filenames in self.filenames.compress(~self.filenames.mask['catalog']).data:
infile = filenames['catalog']
for f in fields:
outfile = filenames['mask_%i'%f]
if os.path.exists(outfile) and not force:
logger.info("Found %s; skipping..."%outfile)
continue
pixels,maglims=self.calculate(infile,f,simple)
logger.info("Creating %s"%outfile)
outdir = mkdir(os.path.dirname(outfile))
data_dict = dict( MAGLIM=maglims )
ugali.utils.skymap.writeSparseHealpixMap(pixels,data_dict,self.nside_pixel,outfile)
def calculate(self, infile, field=1, simple=False):
logger.info("Calculating magnitude limit from %s"%infile)
#manglefile = self.config['mangle']['infile_%i'%field]
footfile = self.config['data']['footprint']
mag_column = self.config['catalog']['mag_%i_field'%field]
magerr_column = self.config['catalog']['mag_err_%i_field'%field]
# For simple maglims
release = self.config['data']['release'].lower()
band = self.config['catalog']['mag_%i_band'%field]
f = pyfits.open(infile)
header = f[1].header
data = f[1].data
#mask_pixels = numpy.arange( healpy.nside2npix(self.nside_mask), dtype='int')
mask_maglims = numpy.zeros( healpy.nside2npix(self.nside_mask) )
out_pixels = numpy.zeros(0,dtype='int')
out_maglims = numpy.zeros(0)
# Find the objects in each pixel
pixel_pix = data['PIX%i'%self.nside_pixel]
mask_pix = ugali.utils.skymap.superpixel(pixel_pix,self.nside_pixel,self.nside_mask)
count = Counter(mask_pix)
pixels = sorted(count.keys())
pix_digi = numpy.digitize(mask_pix,pixels).argsort()
idx = 0
min_num = 500
signal_to_noise = 10.
magerr_lim = 1/signal_to_noise
for pix in pixels:
# Calculate the magnitude limit in each pixel
num = count[pix]
objs = data[pix_digi[idx:idx+num]]
idx += num
if simple:
# Set constant magnitude limits
logger.debug("Simple magnitude limit for %s"%infile)
mask_maglims[pix] = MAGLIMS[release][band]
elif num < min_num:
logger.info('Found <%i objects in pixel %i'%(min_num,pix))
mask_maglims[pix] = 0
else:
mag = objs[mag_column]
magerr = objs[magerr_column]
# Estimate the magnitude limit as suggested by:
# https://deswiki.cosmology.illinois.edu/confluence/display/DO/SVA1+Release+Document
# (https://desweb.cosmology.illinois.edu/confluence/display/Operations/SVA1+Doc)
maglim = numpy.median(mag[(magerr>0.9*magerr_lim)&(magerr<1.1*magerr_lim)])
# Alternative method to estimate the magnitude limit by fitting median
#mag_min, mag_max = mag.min(),mag.max()
#mag_bins = numpy.arange(mag_min,mag_max,0.1) #0.1086?
#x,y = ugali.utils.binning.binnedMedian(mag,magerr,mag_bins)
#x,y = x[~numpy.isnan(y)],y[~numpy.isnan(y)]
#magerr_med = interp1d(x,y)
#mag0 = numpy.median(x)
#maglim = brentq(lambda a: magerr_med(a)-magerr_lim,x.min(),x.max(),disp=False)
# Median from just objects near magerr cut
mask_maglims[pix] = maglim
logger.debug("%i (n=%i): maglim=%g"%(pix,num,mask_maglims[pix]))
subpix = ugali.utils.skymap.subpixel(pix, self.nside_mask, self.nside_pixel)
maglims = numpy.zeros(len(subpix)) + mask_maglims[pix]
out_pixels = numpy.append(out_pixels,subpix)
out_maglims = numpy.append(out_maglims,maglims)
# Remove empty pixels
logger.info("Removing empty pixels")
idx = numpy.nonzero(out_maglims > 0)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
# Remove pixels outside the footprint
logger.info("Checking footprint against %s"%footfile)
glon,glat = pix2ang(self.nside_pixel,out_pixels)
ra,dec = gal2cel(glon,glat)
footprint = inFootprint(footfile,ra,dec)
idx = numpy.nonzero(footprint)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
logger.info("MAGLIM = %.3f +/- %.3f"%(numpy.mean(out_maglims),numpy.std(out_maglims)))
return out_pixels,out_maglims
class Catalog:
def __init__(self, config, roi=None, data=None, filenames=None):
"""
Class to store information about detected objects. This class
augments the raw data array with several aliases and derived
quantities.
Parameters:
-----------
config : Configuration object
roi : Region of Interest to load catalog data for
data : Data array object
filenames : FITS filenames to read catalog from
Returns:
--------
catalog : The Catalog object
"""
self.config = Config(config)
if data is None:
self._parse(roi,filenames)
else:
self.data = data
self._defineVariables()
def __add__(self, other):
return mergeCatalogs([self,other])
def __len__(self):
return len(self.objid)
def __eq__(self, other):
return bool((self.data == other.data).all())
def applyCut(self, cut):
"""
Return a new catalog which is a subset of objects selected
using the input cut array.
NOTE: This is really a *selection* (i.e., objects are retained if the value of 'cut' is True)
"""
return Catalog(self.config, data=self.data[cut])
def bootstrap(self, mc_bit=0x10, seed=None):
"""
Return a random catalog by boostrapping the colors of the objects in the current catalog.
"""
if seed is not None: np.random.seed(seed)
data = copy.deepcopy(self.data)
idx = np.random.randint(0,len(data),len(data))
data[self.config['catalog']['mag_1_field']][:] = self.mag_1[idx]
data[self.config['catalog']['mag_err_1_field']][:] = self.mag_err_1[idx]
data[self.config['catalog']['mag_2_field']][:] = self.mag_2[idx]
data[self.config['catalog']['mag_err_2_field']][:] = self.mag_err_2[idx]
data[self.config['catalog']['mc_source_id_field']][:] |= mc_bit
return Catalog(self.config, data=data)
def project(self, projector = None):
"""
Project coordinates on sphere to image plane using Projector class.
"""
msg = "'%s.project': ADW 2018-05-05"%self.__class__.__name__
DeprecationWarning(msg)
if projector is None:
try:
self.projector = ugali.utils.projector.Projector(self.config['coords']['reference'][0],
self.config['coords']['reference'][1])
except KeyError:
logger.warning('Projection reference point is median (lon, lat) of catalog objects')
self.projector = ugali.utils.projector.Projector(np.median(self.lon), np.median(self.lat))
else:
self.projector = projector
self.x, self.y = self.projector.sphereToImage(self.lon, self.lat)
def spatialBin(self, roi):
"""
Calculate indices of ROI pixels corresponding to object locations.
"""
if hasattr(self,'pixel_roi_index') and hasattr(self,'pixel'):
logger.warning('Catalog alread spatially binned')
return
# ADW: Not safe to set index = -1 (since it will access last entry);
# np.inf would be better...
self.pixel = ang2pix(self.config['coords']['nside_pixel'],self.lon,self.lat)
self.pixel_roi_index = roi.indexROI(self.lon,self.lat)
logger.info("Found %i objects outside ROI"%(self.pixel_roi_index < 0).sum())
def write(self, outfile, clobber=True, **kwargs):
"""
Write the current object catalog to FITS file.
Parameters:
-----------
filename : the FITS file to write.
clobber : remove existing file
kwargs : passed to fitsio.write
Returns:
--------
None
"""
fitsio.write(outfile,self.data,clobber=True,**kwargs)
def _parse(self, roi=None, filenames=None):
"""
Parse catalog FITS files into recarray.
Parameters:
-----------
roi : The region of interest; if 'roi=None', read all catalog files
Returns:
--------
None
"""
if (roi is not None) and (filenames is not None):
msg = "Cannot take both roi and filenames"
raise Exception(msg)
if roi is not None:
pixels = roi.getCatalogPixels()
filenames = self.config.getFilenames()['catalog'][pixels]
elif filenames is None:
filenames = self.config.getFilenames()['catalog'].compressed()
else:
filenames = np.atleast_1d(filenames)
if len(filenames) == 0:
msg = "No catalog files found."
raise Exception(msg)
# Load the data
self.data = load_infiles(filenames)
# Apply a selection cut
self._applySelection()
# Cast data to recarray (historical reasons)
self.data = self.data.view(np.recarray)
def _applySelection(self,selection=None):
# ADW: This is a hack (eval is unsafe!)
if selection is None:
selection = self.config['catalog'].get('selection')
if not selection:
return
elif 'self.data' not in selection:
msg = "Selection does not contain 'data'"
raise Exception(msg)
else:
logger.info('Evaluating selection: \n"%s"'%selection)
sel = eval(selection)
self.data = self.data[sel]
def _defineVariables(self):
"""
Helper funtion to define pertinent variables from catalog data.
ADW (20170627): This has largely been replaced by properties.
"""
logger.info('Catalog contains %i objects'%(len(self.data)))
mc_source_id_field = self.config['catalog']['mc_source_id_field']
if mc_source_id_field is not None:
if mc_source_id_field not in self.data.dtype.names:
array = np.zeros(len(self.data),dtype='>i8') # FITS byte-order convention
self.data = mlab.rec_append_fields(self.data,
names=mc_source_id_field,
arrs=array)
logger.info('Found %i simulated objects'%(np.sum(self.mc_source_id>0)))
# Use properties to avoid duplicating the data
@property
def objid(self): return self.data[self.config['catalog']['objid_field']]
@property
def lon(self): return self.data[self.config['catalog']['lon_field']]
@property
def lat(self): return self.data[self.config['catalog']['lat_field']]
@property
def coordsys(self):
return self.config['coords']['coordsys'].lower()
@property
def mag_1(self): return self.data[self.config['catalog']['mag_1_field']]
@property
def mag_err_1(self):
return self.data[self.config['catalog']['mag_err_1_field']]
@property
def mag_2(self):
return self.data[self.config['catalog']['mag_2_field']]
@property
def mag_err_2(self):
return self.data[self.config['catalog']['mag_err_2_field']]
@property
def mag(self):
if self.config['catalog']['band_1_detection']: return self.mag_1
else: return self.mag_2
@property
def mag_err(self):
if self.config['catalog']['band_1_detection']: return self.mag_err_1
else: return self.mag_err_2
@property
def color(self): return self.mag_1 - self.mag_2
@property
def color_err(self): return np.sqrt(self.mag_err_1**2 + self.mag_err_2**2)
@property
def mc_source_id(self):
return self.data[self.config['catalog']['mc_source_id_field']]
# This assumes Galactic coordinates
@property
def ra_dec(self):
if self.coordsys == 'cel': return self.lon, self.lat
else: return gal2cel(self.lon,self.lat)
@property
def ra(self): return self.ra_dec[0]
@property
def dec(self): return self.ra_dec[1]
@property
def glon_glat(self):
if self.coordsys == 'gal': return self.lon,self.lat
else: return cel2gal(self.lon,self.lat)
@property
def glon(self): return self.glon_glat[0]
@property
def glat(self): return self.glon_glat[1]
def __init__(self, config):
self.config = Config(config)
self._setup()
def createGridSearch(config, lon, lat):
config = Config(config)
obs = createObservation(config, lon, lat)
src = createSource(config, 'scan', lon=lon, lat=lat)
loglike = LogLikelihood(config, obs, src)
return GridSearch(config, loglike)
def __init__(self, config, catfile=None, popfile=None):
self.config = Config(config)
self.population = self.read_population(popfile)
self.catalog = self.read_catalog(catfile)
self.mlimit = -1
class Catalog:
def __init__(self, config, roi=None, data=None):
"""
Class to store information about detected objects.
The raw data from the fits file is stored. lon and lat are derived quantities
based on chosen coordinate system.
INPUTS:
config: Config object
roi[None] : Region of Interest to load catalog data for
data[None]: pyfits table data (fitsrec) object.
"""
#self = config.merge(config_merge) # Maybe you would want to update parameters??
self.config = Config(config)
if data is None:
self._parse(roi)
else:
self.data = data
self._defineVariables()
def __add__(self, other):
return mergeCatalogs([self,other])
def __len__(self):
return len(self.objid)
def applyCut(self, cut):
"""
Return a new catalog which is a subset of objects selected using the input cut array.
"""
return Catalog(self.config, data=self.data[cut])
def bootstrap(self, mc_bit=0x10, seed=None):
"""
Return a random catalog by boostrapping the colors of the objects in the current catalog.
"""
if seed is not None: numpy.random.seed(seed)
data = copy.deepcopy(self.data)
idx = numpy.random.randint(0,len(data),len(data))
data[self.config['catalog']['mag_1_field']][:] = self.mag_1[idx]
data[self.config['catalog']['mag_err_1_field']][:] = self.mag_err_1[idx]
data[self.config['catalog']['mag_2_field']][:] = self.mag_2[idx]
data[self.config['catalog']['mag_err_2_field']][:] = self.mag_err_2[idx]
data[self.config['catalog']['mc_source_id_field']][:] |= mc_bit
return Catalog(self.config, data=data)
def project(self, projector = None):
"""
Project coordinates on sphere to image plane using Projector class.
"""
if projector is None:
try:
self.projector = ugali.utils.projector.Projector(self.config['coords']['reference'][0],
self.config['coords']['reference'][1])
except KeyError:
logger.warning('Projection reference point is median (lon, lat) of catalog objects')
self.projector = ugali.utils.projector.Projector(numpy.median(self.lon), numpy.median(self.lat))
else:
self.projector = projector
self.x, self.y = self.projector.sphereToImage(self.lon, self.lat)
def spatialBin(self, roi):
"""
Calculate indices of ROI pixels corresponding to object locations.
"""
if hasattr(self,'pixel_roi_index') and hasattr(self,'pixel'):
logger.warning('Catalog alread spatially binned')
return
# ADW: Not safe to set index = -1 (since it will access last entry);
# np.inf would be better...
self.pixel = ang2pix(self.config['coords']['nside_pixel'],self.lon,self.lat)
self.pixel_roi_index = roi.indexROI(self.lon,self.lat)
if numpy.any(self.pixel_roi_index < 0):
logger.warning("Objects found outside ROI")
def write(self, outfile):
"""
Write the current object catalog to fits file.
"""
hdu = pyfits.BinTableHDU(self.data)
hdu.writeto(outfile, clobber=True)
def _parse(self, roi=None):
"""
Helper function to parse a catalog file and return a pyfits table.
CSV format not yet validated.
!!! Careful, reading a large catalog is memory intensive !!!
"""
filenames = self.config.getFilenames()
if len(filenames['catalog'].compressed()) == 0:
raise Exception("No catalog file found")
elif roi is not None:
pixels = roi.getCatalogPixels()
self.data = readCatalogData(filenames['catalog'][pixels])
elif len(filenames['catalog'].compressed()) == 1:
file_type = filenames[0].split('.')[-1].strip().lower()
if file_type == 'csv':
self.data = numpy.recfromcsv(filenames[0], delimiter = ',')
elif file_type in ['fit', 'fits']:
self.data = pyfits.open(filenames[0])[1].data
else:
logger.warning('Unrecognized catalog file extension %s'%(file_type))
else:
self.data = readCatalogData(filenames['catalog'].compressed())
# ADW: This is horrible and should never be done...
selection = self.config['catalog'].get('selection')
if not selection:
pass
elif 'self.data' not in selection:
msg = "Selection does not contain 'data'"
raise Exception(msg)
else:
logger.warning('Evaluating selection: \n"%s"'%selection)
sel = eval(selection)
self.data = self.data[sel]
#print 'Found %i objects'%(len(self.data))
def _defineVariables(self):
"""
Helper funtion to define pertinent variables from catalog data.
"""
self.objid = self.data.field(self.config['catalog']['objid_field'])
self.lon = self.data.field(self.config['catalog']['lon_field'])
self.lat = self.data.field(self.config['catalog']['lat_field'])
#if self.config['catalog']['coordsys'].lower() == 'cel' \
# and self.config['coords']['coordsys'].lower() == 'gal':
# logger.info('Converting catalog objects from CELESTIAL to GALACTIC cboordinates')
# self.lon, self.lat = ugali.utils.projector.celToGal(self.lon, self.lat)
#elif self.config['catalog']['coordsys'].lower() == 'gal' \
# and self.config['coords']['coordsys'].lower() == 'cel':
# logger.info('Converting catalog objects from GALACTIC to CELESTIAL coordinates')
# self.lon, self.lat = ugali.utils.projector.galToCel(self.lon, self.lat)
self.mag_1 = self.data.field(self.config['catalog']['mag_1_field'])
self.mag_err_1 = self.data.field(self.config['catalog']['mag_err_1_field'])
self.mag_2 = self.data.field(self.config['catalog']['mag_2_field'])
self.mag_err_2 = self.data.field(self.config['catalog']['mag_err_2_field'])
if self.config['catalog']['mc_source_id_field'] is not None:
if self.config['catalog']['mc_source_id_field'] in self.data.names:
self.mc_source_id = self.data.field(self.config['catalog']['mc_source_id_field'])
logger.info('Found %i MC source objects'%(numpy.sum(self.mc_source_id > 0)))
else:
#ADW: This is pretty kludgy, please fix... (FIXME)
columns_array = [pyfits.Column(name = self.config['catalog']['mc_source_id_field'],
format = 'I',
array = numpy.zeros(len(self.data)))]
hdu = pyfits.new_table(columns_array)
self.data = pyfits.new_table(pyfits.new_table(self.data.view(np.recarray)).columns + hdu.columns).data
self.mc_source_id = self.data.field(self.config['catalog']['mc_source_id_field'])
# should be @property
if self.config['catalog']['band_1_detection']:
self.mag = self.mag_1
self.mag_err = self.mag_err_1
else:
self.mag = self.mag_2
self.mag_err = self.mag_err_2
# should be @property
self.color = self.mag_1 - self.mag_2
self.color_err = numpy.sqrt(self.mag_err_1**2 + self.mag_err_2**2)
logger.info('Catalog contains %i objects'%(len(self.data)))
# This assumes Galactic coordinates
@property
def ra_dec(self): return gal2cel(self.lon,self.lat)
@property
def ra(self): return self.ra_dec[0]
@property
def dec(self): return self.ra_dec[1]
@property
def glon_glat(self): return self.lon,self.lat
@property
def glon(self): return self.lon
@property
def glat(self): return self.lat
class Maglims(object):
""" Object for deriving magnitude limits from the catalog """
def __init__(self, config):
self.config = Config(config)
self._setup()
def _setup(self):
self.nside_catalog = self.config['coords']['nside_catalog']
self.nside_mask = self.config['coords']['nside_mask']
self.nside_pixel = self.config['coords']['nside_pixel']
self.filenames = self.config.getFilenames()
self.footfile = self.config['data']['footprint']
try:
self.footprint = fitsio.read(self.footfile)['I'].ravel()
except:
logger.warn("Couldn't open %s; will pass through."%self.footfile)
self.footprint = self.footfile
def run(self,field=None,simple=False,force=False):
"""
Loop through pixels containing catalog objects and calculate
the magnitude limit. This gets a bit convoluted due to all
the different pixel resolutions...
"""
if field is None: fields = [1,2]
else: fields = [field]
for filenames in self.filenames.compress(~self.filenames.mask['catalog']).data:
infile = filenames['catalog']
for f in fields:
outfile = filenames['mask_%i'%f]
if os.path.exists(outfile) and not force:
logger.info("Found %s; skipping..."%outfile)
continue
pixels,maglims=self.calculate(infile,f,simple)
logger.info("Creating %s"%outfile)
outdir = mkdir(os.path.dirname(outfile))
data = odict()
data['PIXEL']=pixels
data['MAGLIM']=maglims.astype('f4')
ugali.utils.healpix.write_partial_map(outfile,data,
self.nside_pixel)
def calculate(self, infile, field=1, simple=False):
logger.info("Calculating magnitude limit from %s"%infile)
#manglefile = self.config['mangle']['infile_%i'%field]
#footfile = self.config['data']['footprint']
#try:
# footprint = fitsio.read(footfile)['I'].ravel()
#except:
# logger.warn("Couldn't open %s; will try again."%footfile)
# footprint = footfile
mag_column = self.config['catalog']['mag_%i_field'%field]
magerr_column = self.config['catalog']['mag_err_%i_field'%field]
# For simple maglims
release = self.config['data']['release'].lower()
band = self.config['catalog']['mag_%i_band'%field]
pixel_pix_name = 'PIX%i'%self.nside_pixel
data = fitsio.read(infile,columns=[pixel_pix_name])
#mask_pixels = np.arange( hp.nside2npix(self.nside_mask), dtype='int')
mask_maglims = np.zeros(hp.nside2npix(self.nside_mask))
out_pixels = np.zeros(0,dtype='int')
out_maglims = np.zeros(0)
# Find the objects in each pixel
pixel_pix = data[pixel_pix_name]
mask_pix = ugali.utils.skymap.superpixel(pixel_pix,self.nside_pixel,self.nside_mask)
count = Counter(mask_pix)
pixels = sorted(count.keys())
pix_digi = np.digitize(mask_pix,pixels).argsort()
idx = 0
min_num = 500
signal_to_noise = 10.
magerr_lim = 1/signal_to_noise
for pix in pixels:
# Calculate the magnitude limit in each pixel
num = count[pix]
objs = data[pix_digi[idx:idx+num]]
idx += num
if simple:
# Set constant magnitude limits
logger.debug("Simple magnitude limit for %s"%infile)
mask_maglims[pix] = MAGLIMS[release][band]
elif num < min_num:
logger.info('Found <%i objects in pixel %i'%(min_num,pix))
mask_maglims[pix] = 0
else:
mag = objs[mag_column]
magerr = objs[magerr_column]
# Estimate the magnitude limit as suggested by:
# https://deswiki.cosmology.illinois.edu/confluence/display/DO/SVA1+Release+Document
# (https://desweb.cosmology.illinois.edu/confluence/display/Operations/SVA1+Doc)
maglim = np.median(mag[(magerr>0.9*magerr_lim)&(magerr<1.1*magerr_lim)])
# Alternative method to estimate the magnitude limit by fitting median
#mag_min, mag_max = mag.min(),mag.max()
#mag_bins = np.arange(mag_min,mag_max,0.1) #0.1086?
#x,y = ugali.utils.binning.binnedMedian(mag,magerr,mag_bins)
#x,y = x[~np.isnan(y)],y[~np.isnan(y)]
#magerr_med = interp1d(x,y)
#mag0 = np.median(x)
#maglim = brentq(lambda a: magerr_med(a)-magerr_lim,x.min(),x.max(),disp=False)
# Median from just objects near magerr cut
mask_maglims[pix] = maglim
logger.debug("%i (n=%i): maglim=%g"%(pix,num,mask_maglims[pix]))
subpix = ugali.utils.skymap.subpixel(pix, self.nside_mask, self.nside_pixel)
maglims = np.zeros(len(subpix)) + mask_maglims[pix]
out_pixels = np.append(out_pixels,subpix)
out_maglims = np.append(out_maglims,maglims)
# Remove empty pixels
logger.info("Removing empty pixels")
idx = np.nonzero(out_maglims > 0)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
# Remove pixels outside the footprint
if self.footfile:
logger.info("Checking footprint against %s"%self.footfile)
glon,glat = pix2ang(self.nside_pixel,out_pixels)
ra,dec = gal2cel(glon,glat)
footprint = inFootprint(self.footprint,ra,dec)
idx = np.nonzero(footprint)[0]
out_pixels = out_pixels[idx]
out_maglims = out_maglims[idx]
logger.info("MAGLIM = %.3f +/- %.3f"%(np.mean(out_maglims),np.std(out_maglims)))
return out_pixels,out_maglims
class ROI(object):
def __init__(self, config, lon, lat):
self.config = Config(config)
self.lon = lon
self.lat = lat
self.projector = ugali.utils.projector.Projector(self.lon, self.lat)
self.vec = vec = ang2vec(self.lon, self.lat)
self.pix = ang2pix(self.config['coords']['nside_likelihood'],self.lon,self.lat)
# Pixels from the entire ROI disk
pix = query_disc(self.config['coords']['nside_pixel'], vec,
self.config['coords']['roi_radius'])
self.pixels = PixelRegion(self.config['coords']['nside_pixel'],pix)
# Pixels in the interior region
pix = query_disc(self.config['coords']['nside_pixel'], vec,
self.config['coords']['roi_radius_interior'])
self.pixels_interior = PixelRegion(self.config['coords']['nside_pixel'],pix)
# Pixels in the outer annulus
pix = query_disc(self.config['coords']['nside_pixel'], vec,
self.config['coords']['roi_radius_annulus'])
pix = numpy.setdiff1d(self.pixels, pix)
self.pixels_annulus = PixelRegion(self.config['coords']['nside_pixel'],pix)
# Pixels within target healpix region
pix = ugali.utils.skymap.subpixel(self.pix,self.config['coords']['nside_likelihood'],
self.config['coords']['nside_pixel'])
self.pixels_target = PixelRegion(self.config['coords']['nside_pixel'],pix)
# Boolean arrays for selecting given pixels
# (Careful, this works because pixels are pre-sorted by query_disc before in1d)
self.pixel_interior_cut = numpy.in1d(self.pixels, self.pixels_interior)
# ADW: Updated for more general ROI shapes
#self.pixel_annulus_cut = ~self.pixel_interior_cut
self.pixel_annulus_cut = numpy.in1d(self.pixels, self.pixels_annulus)
# # These should be unnecessary now
# self.centers_lon, self.centers_lat = self.pixels.lon, self.pixels.lat
# self.centers_lon_interior,self.centers_lat_interior = self.pixels_interior.lon,self.pixels_interior.lat
# self.centers_lon_target, self.centers_lat_target = self.pixels_target.lon, self.pixels_target.lat
self.area_pixel = healpy.nside2pixarea(self.config.params['coords']['nside_pixel'],degrees=True) # deg^2
"""
self.centers_x = self._centers(self.bins_x)
self.centers_y = self._centers(self.bins_y)
self.delta_x = self.config.params['coords']['pixel_size']
self.delta_y = self.config.params['coords']['pixel_size']
# Should actually try to take projection effects into account for better accuracy
# MC integration perhaps?
# Throw points in a cone around full ROI and see what fraction fall in
self.area_pixel = self.config.params['coords']['pixel_size']**2
self.centers_lon, self.centers_lat = self.projector.imageToSphere(self.centers_x, self.centers_y)
"""
# ADW: These are really bin edges, should be careful and consistent
self.bins_mag = numpy.linspace(self.config.params['mag']['min'],
self.config.params['mag']['max'],
self.config.params['mag']['n_bins'] + 1)
self.bins_color = numpy.linspace(self.config.params['color']['min'],
self.config.params['color']['max'],
self.config.params['color']['n_bins'] + 1)
self.centers_mag = ugali.utils.binning.centers(self.bins_mag)
self.centers_color = ugali.utils.binning.centers(self.bins_color)
self.delta_mag = self.bins_mag[1] - self.bins_mag[0]
self.delta_color = self.bins_color[1] - self.bins_color[0]
# Axis labels
self.label_x = 'x (deg)'
self.label_y = 'y (deg)'
if self.config.params['catalog']['band_1_detection']:
self.label_mag = '%s (mag)'%(self.config.params['catalog']['mag_1_field'])
else:
self.label_mag = '%s (mag)'%(self.config.params['catalog']['mag_2_field'])
self.label_color = '%s - %s (mag)'%(self.config.params['catalog']['mag_1_field'],
self.config.params['catalog']['mag_2_field'])
#self.precomputeAngsep()
def plot(self, value=None, pixel=None):
"""
Plot the ROI
"""
import ugali.utils.plotting
map_roi = numpy.array(healpy.UNSEEN \
* numpy.ones(healpy.nside2npix(self.config.params['coords']['nside_pixel'])))
if value is None:
#map_roi[self.pixels] = ugali.utils.projector.angsep(self.lon, self.lat, self.centers_lon, self.centers_lat)
map_roi[self.pixels] = 1
map_roi[self.pixels_annulus] = 0
map_roi[self.pixels_target] = 2
elif value is not None and pixel is None:
map_roi[self.pixels] = value
elif value is not None and pixel is not None:
map_roi[pixel] = value
else:
logger.error("Can't parse input")
ugali.utils.plotting.zoomedHealpixMap('Region of Interest',
map_roi,
self.lon, self.lat,
self.config.params['coords']['roi_radius'])
# ADW: Maybe these should be associated with the PixelRegion objects
def inPixels(self,lon,lat,pixels):
""" Function for testing if coordintes in set of ROI pixels. """
nside = self.config.params['coords']['nside_pixel']
return ugali.utils.healpix.in_pixels(lon,lat,pixels,nside)
def inROI(self,lon,lat):
return self.inPixels(lon,lat,self.pixels)
def inAnnulus(self,lon,lat):
return self.inPixels(lon,lat,self.pixels_annulus)
def inInterior(self,lon,lat):
return self.inPixels(lon,lat,self.pixels_interior)
def inTarget(self,lon,lat):
return self.inPixels(lon,lat,self.pixels_target)
def indexPixels(self,lon,lat,pixels):
nside = self.config.params['coords']['nside_pixel']
return ugali.utils.healpix.index_pixels(lon,lat,pixels,nside)
def indexROI(self,lon,lat):
return self.indexPixels(lon,lat,self.pixels)
def indexAnnulus(self,lon,lat):
return self.indexPixels(lon,lat,self.pixels_annulus)
def indexInterior(self,lon,lat):
return self.indexPixels(lon,lat,self.pixels_interior)
def indexTarget(self,lon,lat):
return self.indexPixels(lon,lat,self.pixels_target)
def getCatalogPixels(self):
"""
Return the catalog pixels spanned by this ROI.
"""
filenames = self.config.getFilenames()
nside_catalog = self.config.params['coords']['nside_catalog']
nside_pixel = self.config.params['coords']['nside_pixel']
# All possible catalog pixels spanned by the ROI
superpix = ugali.utils.skymap.superpixel(self.pixels,nside_pixel,nside_catalog)
superpix = numpy.unique(superpix)
# Only catalog pixels that exist in catalog files
pixels = numpy.intersect1d(superpix, filenames['pix'].compressed())
return pixels
class Mask(object):
"""
Contains maps of completeness depth in magnitudes for multiple observing bands, and associated products.
"""
def __init__(self, config, roi):
self.config = Config(config)
self.roi = roi
filenames = self.config.getFilenames()
catalog_pixels = self.roi.getCatalogPixels()
self.mask_1 = MaskBand(filenames['mask_1'][catalog_pixels], self.roi)
self.mask_2 = MaskBand(filenames['mask_2'][catalog_pixels], self.roi)
self._fracRoiSparse()
self.minimum_solid_angle = self.config.params['mask'][
'minimum_solid_angle'] # deg^2
# FIXME: Need to parallelize CMD and MMD formulation
self._solidAngleCMD()
self._pruneCMD(self.minimum_solid_angle)
#self._solidAngleMMD()
#self._pruneMMD(self.minimum_solid_angle)
self._photometricErrors()
@property
def mask_roi_unique(self):
"""
Assemble a set of unique magnitude tuples for the ROI
"""
# There is no good inherent way in numpy to do this...
# http://stackoverflow.com/q/16970982/
# Also possible and simple:
#return np.unique(zip(self.mask_1.mask_roi_sparse,self.mask_2.mask_roi_sparse))
A = np.vstack(
[self.mask_1.mask_roi_sparse, self.mask_2.mask_roi_sparse]).T
B = A[np.lexsort(A.T[::-1])]
return B[np.concatenate(([True], np.any(B[1:] != B[:-1], axis=1)))]
@property
def mask_roi_digi(self):
"""
Get the index of the unique magnitude tuple for each pixel in the ROI.
"""
# http://stackoverflow.com/q/24205045/#24206440
A = np.vstack(
[self.mask_1.mask_roi_sparse, self.mask_2.mask_roi_sparse]).T
B = self.mask_roi_unique
AA = np.ascontiguousarray(A)
BB = np.ascontiguousarray(B)
dt = np.dtype((np.void, AA.dtype.itemsize * AA.shape[1]))
a = AA.view(dt).ravel()
b = BB.view(dt).ravel()
idx = np.argsort(b)
indices = np.searchsorted(b[idx], a)
return idx[indices]
@property
def frac_annulus_sparse(self):
return self.frac_roi_sparse[self.roi.pixel_annulus_cut]
@property
def frac_interior_sparse(self):
return self.frac_roi_sparse[self.roi.pixel_interior_cut]
def _fracRoiSparse(self):
"""
Calculate an approximate pixel coverage fraction from the two masks.
We have no way to know a priori how much the coverage of the
two masks overlap in a give pixel. For example, masks that each
have frac = 0.5 could have a combined frac = [0.0 to 0.5].
The limits will be:
max: min(frac1,frac2)
min: max((frac1+frac2)-1, 0.0)
Sometimes we are lucky and our fracdet is actually already
calculated for the two masks combined, so that the max
condition is satisfied. That is what we will assume...
"""
self.frac_roi_sparse = np.min(
[self.mask_1.frac_roi_sparse, self.mask_2.frac_roi_sparse], axis=0)
return self.frac_roi_sparse
def _solidAngleMMD(self):
"""
Compute solid angle within the mask annulus (deg^2) as a
function of mag_1 and mag_2
"""
# Take upper corner of the magnitude bin
mag_2, mag_1 = np.meshgrid(self.roi.bins_mag[1:],
self.roi.bins_mag[1:])
# Havent tested since adding fracdet
unmasked_mag_1 = (self.mask_1.mask_annulus_sparse[:, np.newaxis] >
mag_1[:, np.newaxis])
unmasked_mag_2 = (self.mask_2.mask_annulus_sparse[:, np.newaxis] >
mag_2[:, np.newaxis])
n_unmasked_pixels = (unmasked_mag_1 * unmasked_mag_2 *
self.frac_annulus_sparse).sum(axis=1)
self.solid_angle_mmd = self.roi.area_pixel * n_unmasked_pixels
if self.solid_angle_mmd.sum() == 0:
msg = "Mask annulus contains no solid angle."
logger.error(msg)
raise Exception(msg)
def _pruneMMD(self, minimum_solid_angle):
"""
Remove regions of magnitude-magnitude space where the unmasked solid angle is
statistically insufficient to estimate the background.
INPUTS:
solid_angle[1]: minimum solid angle (deg^2)
"""
logger.info('Pruning mask based on minimum solid angle of %.2f deg^2' %
(minimum_solid_angle))
solid_angle_mmd = self.solid_angle_mmd * (self.solid_angle_mmd >
minimum_solid_angle)
if solid_angle_mmd.sum() == 0:
msg = "Pruned mask contains no solid angle."
logger.error(msg)
raise Exception(msg)
self.solid_angle_mmd = solid_angle_mmd
# Compute which magnitudes the clipping correspond to
index_mag_1, index_mag_2 = np.nonzero(self.solid_angle_mmd)
self.mag_1_clip = self.roi.bins_mag[1:][np.max(index_mag_1)]
self.mag_2_clip = self.roi.bins_mag[1:][np.max(index_mag_2)]
logger.info('Clipping mask 1 at %.2f mag' % (self.mag_1_clip))
logger.info('Clipping mask 2 at %.2f mag' % (self.mag_2_clip))
self.mask_1.mask_roi_sparse = np.clip(self.mask_1.mask_roi_sparse, 0.,
self.mag_1_clip)
self.mask_2.mask_roi_sparse = np.clip(self.mask_2.mask_roi_sparse, 0.,
self.mag_2_clip)
def _solidAngleCMD(self):
"""
Compute solid angle within the mask annulus (deg^2) as a
function of color and magnitude.
"""
self.solid_angle_cmd = np.zeros(
[len(self.roi.centers_mag),
len(self.roi.centers_color)])
for index_mag in np.arange(len(self.roi.centers_mag)):
for index_color in np.arange(len(self.roi.centers_color)):
# mag and color at bin center
mag = self.roi.centers_mag[index_mag]
color = self.roi.centers_color[index_color]
if self.config.params['catalog']['band_1_detection']:
# Evaluating at the center of the color-mag bin, be consistent!
#mag_1 = self.roi.centers_mag[index_mag]
#color = self.roi.centers_color[index_color]
#mag_2 = mag_1 - color
# Evaluating at corner of the color-mag bin, be consistent!
mag_1 = mag + (0.5 * self.roi.delta_mag)
mag_2 = mag - color + (0.5 * self.roi.delta_color)
else:
# Evaluating at the center of the color-mag bin, be consistent!
#mag_2 = self.roi.centers_mag[index_mag]
#color = self.roi.centers_color[index_color]
#mag_1 = mag_2 + color
# Evaluating at corner of the color-mag bin, be consistent!
mag_1 = mag + color + (0.5 * self.roi.delta_color)
mag_2 = mag + (0.5 * self.roi.delta_mag)
# ADW: Is there a problem here?
#self.solid_angle_cmd[index_mag, index_color] = self.roi.area_pixel * np.sum((self.mask_1.mask > mag_1) * (self.mask_2.mask > mag_2))
# ADW: I think we want to keep pixels that are >= mag
unmasked_mag_1 = (self.mask_1.mask_annulus_sparse >= mag_1)
unmasked_mag_2 = (self.mask_2.mask_annulus_sparse >= mag_2)
n_unmasked_pixels = np.sum(unmasked_mag_1 * unmasked_mag_2 *
self.frac_annulus_sparse)
#n_unmasked_pixels = np.sum((self.mask_1.mask_annulus_sparse > mag_1) \
# * (self.mask_2.mask_annulus_sparse > mag_2))
self.solid_angle_cmd[
index_mag,
index_color] = self.roi.area_pixel * n_unmasked_pixels
if self.solid_angle_cmd.sum() == 0:
msg = "Mask annulus contains no solid angle."
logger.error(msg)
raise Exception(msg)
return self.solid_angle_cmd
def _solidAngleCMD(self):
"""
Compute solid angle within the mask annulus (deg^2) as a
function of color and magnitude.
Returns:
--------
solid_angle_cmd : 2d array
"""
self.solid_angle_cmd = np.zeros(
[len(self.roi.centers_mag),
len(self.roi.centers_color)])
idx_mag, idx_color = np.where(self.solid_angle_cmd == 0)
mag = self.roi.centers_mag[idx_mag]
color = self.roi.centers_color[idx_color]
if self.config.params['catalog']['band_1_detection']:
# Evaluating at corner of the color-mag bin, be consistent!
mag_1 = mag + (0.5 * self.roi.delta_mag)
mag_2 = mag - color + (0.5 * self.roi.delta_color)
else:
# Evaluating at corner of the color-mag bin, be consistent!
mag_1 = mag + color + (0.5 * self.roi.delta_color)
mag_2 = mag + (0.5 * self.roi.delta_mag)
n_unmasked_pixels = np.zeros_like(mag)
for i in np.arange(len(mag_1)):
unmasked_mag_1 = (self.mask_1.mask_annulus_sparse >= mag_1[i])
unmasked_mag_2 = (self.mask_2.mask_annulus_sparse >= mag_2[i])
n_unmasked_pixels[i] = np.sum(unmasked_mag_1 * unmasked_mag_2 *
self.frac_annulus_sparse)
self.solid_angle_cmd[
idx_mag, idx_color] = self.roi.area_pixel * n_unmasked_pixels
if self.solid_angle_cmd.sum() == 0:
msg = "Mask annulus contains no solid angle."
logger.error(msg)
raise Exception(msg)
return self.solid_angle_cmd
def _pruneCMD(self, minimum_solid_angle):
"""
Remove regions of color-magnitude space where the unmasked solid angle is
statistically insufficient to estimate the background.
ADW: Why are we clipping at the bin center instead of edge?
INPUTS:
solid_angle[1]: minimum solid angle (deg^2)
"""
logger.info('Pruning mask based on minimum solid angle of %.2f deg^2' %
(minimum_solid_angle))
self.solid_angle_cmd *= self.solid_angle_cmd > minimum_solid_angle
if self.solid_angle_cmd.sum() == 0:
msg = "Pruned mask contains no solid angle."
logger.error(msg)
raise Exception(msg)
# Compute which magnitudes the clipping correspond to
index_mag, index_color = np.nonzero(self.solid_angle_cmd)
mag = self.roi.centers_mag[index_mag]
color = self.roi.centers_color[index_color]
if self.config.params['catalog']['band_1_detection']:
mag_1 = mag
mag_2 = mag_1 - color
self.mag_1_clip = np.max(mag_1) + (0.5 * self.roi.delta_mag)
self.mag_2_clip = np.max(mag_2) + (0.5 * self.roi.delta_color)
else:
mag_2 = mag
mag_1 = color + mag_2
self.mag_1_clip = np.max(mag_1) + (0.5 * self.roi.delta_color)
self.mag_2_clip = np.max(mag_2) + (0.5 * self.roi.delta_mag)
logger.info('Clipping mask 1 at %.2f mag' % (self.mag_1_clip))
logger.info('Clipping mask 2 at %.2f mag' % (self.mag_2_clip))
self.mask_1.mask_roi_sparse = np.clip(self.mask_1.mask_roi_sparse, 0.,
self.mag_1_clip)
self.mask_2.mask_roi_sparse = np.clip(self.mask_2.mask_roi_sparse, 0.,
self.mag_2_clip)
def completeness(self, delta, method='step'):
"""
Return the completeness as a function of magnitude.
ADW: Eventually want a completeness mask to set overall efficiency.
"""
delta = np.asarray(delta)
if method == 'step':
func = lambda delta: (delta > 0).astype(float)
elif method == 'erf':
# Trust the SDSS EDR???
# 95% completeness:
def func(delta):
# Efficiency at bright end (assumed to be 100%)
e = 1.0
# EDR says full width is ~0.5 mag
width = 0.2
# This should be the halfway point in the curve
return (e / 2.0) * (1 / np.sqrt(2 * width)) * (
np.sqrt(2 * width) - scipy.special.erf(-delta))
elif method == 'flemming':
# Functional form taken from Fleming et al. AJ 109, 1044 (1995)
# http://adsabs.harvard.edu/abs/1995AJ....109.1044F
# f = 1/2 [1 - alpha(V - Vlim)/sqrt(1 + alpha^2 (V - Vlim)^2)]
# CAREFUL: This definition is for Vlim = 50% completeness
def func(delta):
alpha = 2.0
return 0.5 * (
1 - (alpha * delta) / np.sqrt(1 + alpha**2 * delta**2))
else:
raise Exception('...')
return func(delta)
def _photometricErrors(self, catalog=None, n_per_bin=100):
"""
Realistic photometric errors estimated from catalog objects and mask.
Extend below the magnitude threshold with a flat extrapolation.
"""
if catalog is None:
# Simple proxy for photometric errors
release = self.config['data']['release']
band_1 = self.config['catalog'].get('mag_1_band')
if not band_1: band_1 = self.config['isochrone']['mag_1_field']
band_2 = self.config['catalog'].get('mag_2_band')
if not band_2: band_2 = self.config['isochrone']['mag_2_field']
DELMIN = 0.0
pars_1 = MAGERR_PARAMS[release][band_1]
def photo_err_1(delta):
p = pars_1
return np.clip(
np.exp(p[0] * delta + p[1]) + p[2], 0,
np.exp(p[0] * (DELMIN) + p[1]) + p[2])
pars_2 = MAGERR_PARAMS[release][band_2]
def photo_err_2(delta):
p = pars_2
return np.clip(
np.exp(p[0] * delta + p[1]) + p[2], 0,
np.exp(p[0] * (DELMIN) + p[1]) + p[2])
else:
catalog.spatialBin(self.roi)
if len(catalog.mag_1) < n_per_bin:
logger.warning(
"Catalog contains fewer objects than requested to calculate errors."
)
#n_per_bin = int(len(catalog.mag_1) / 3)
return self._photometricErrors(catalog=None)
# Band 1
mag_1_thresh = self.mask_1.mask_roi_sparse[
catalog.pixel_roi_index] - catalog.mag_1
sorting_indices = np.argsort(mag_1_thresh)
mag_1_thresh_sort = mag_1_thresh[sorting_indices]
mag_err_1_sort = catalog.mag_err_1[sorting_indices]
# ADW: Can't this be done with np.median(axis=?)
mag_1_thresh_medians = []
mag_err_1_medians = []
for i in range(0, int(len(mag_1_thresh) / float(n_per_bin))):
mag_1_thresh_medians.append(
np.median(mag_1_thresh_sort[n_per_bin * i:n_per_bin *
(i + 1)]))
mag_err_1_medians.append(
np.median(mag_err_1_sort[n_per_bin * i:n_per_bin *
(i + 1)]))
if mag_1_thresh_medians[0] > 0.:
mag_1_thresh_medians = np.insert(mag_1_thresh_medians, 0, -99.)
mag_err_1_medians = np.insert(mag_err_1_medians, 0,
mag_err_1_medians[0])
photo_err_1 = scipy.interpolate.interp1d(
mag_1_thresh_medians,
mag_err_1_medians,
bounds_error=False,
fill_value=mag_err_1_medians[-1])
# Band 2
mag_2_thresh = self.mask_2.mask_roi_sparse[
catalog.pixel_roi_index] - catalog.mag_2
sorting_indices = np.argsort(mag_2_thresh)
mag_2_thresh_sort = mag_2_thresh[sorting_indices]
mag_err_2_sort = catalog.mag_err_2[sorting_indices]
mag_2_thresh_medians = []
mag_err_2_medians = []
for i in range(0, int(len(mag_2_thresh) / float(n_per_bin))):
mag_2_thresh_medians.append(
np.median(mag_2_thresh_sort[n_per_bin * i:n_per_bin *
(i + 1)]))
mag_err_2_medians.append(
np.median(mag_err_2_sort[n_per_bin * i:n_per_bin *
(i + 1)]))
if mag_2_thresh_medians[0] > 0.:
mag_2_thresh_medians = np.insert(mag_2_thresh_medians, 0, -99.)
mag_err_2_medians = np.insert(mag_err_2_medians, 0,
mag_err_2_medians[0])
photo_err_2 = scipy.interpolate.interp1d(
mag_2_thresh_medians,
mag_err_2_medians,
bounds_error=False,
fill_value=mag_err_2_medians[-1])
self.photo_err_1 = photo_err_1
self.photo_err_2 = photo_err_2
return self.photo_err_1, self.photo_err_2
def plotSolidAngleCMD(self):
"""
Solid angle within the mask as a function of color and magnitude.
"""
msg = "'%s.plotSolidAngleCMD': ADW 2018-05-05" % self.__class__.__name__
DeprecationWarning(msg)
import ugali.utils.plotting
ugali.utils.plotting.twoDimensionalHistogram(
'mask',
'color',
'magnitude',
self.solid_angle_cmd,
self.roi.bins_color,
self.roi.bins_mag,
lim_x=[self.roi.bins_color[0], self.roi.bins_color[-1]],
lim_y=[self.roi.bins_mag[-1], self.roi.bins_mag[0]])
def plotSolidAngleMMD(self):
"""
Solid angle within the mask as a function of color and magnitude.
"""
msg = "'%s.plotSolidAngleMMD': ADW 2018-05-05" % self.__class__.__name__
DeprecationWarning(msg)
import ugali.utils.plotting
ugali.utils.plotting.twoDimensionalHistogram(
'mask',
'mag_2',
'mag_1',
self.solid_angle_mmd,
self.roi.bins_mag,
self.roi.bins_mag,
lim_x=[self.roi.bins_mag[0], self.roi.bins_mag[-1]],
lim_y=[self.roi.bins_mag[-1], self.roi.bins_mag[0]])
def backgroundMMD(self, catalog, method='cloud-in-cells', weights=None):
"""
Generate an empirical background model in magnitude-magnitude space.
INPUTS:
catalog: Catalog object
OUTPUTS:
background
"""
# Select objects in annulus
cut_annulus = self.roi.inAnnulus(catalog.lon, catalog.lat)
mag_1 = catalog.mag_1[cut_annulus]
mag_2 = catalog.mag_2[cut_annulus]
# Units are (deg^2)
solid_angle = ugali.utils.binning.take2D(self.solid_angle_mmd, mag_2,
mag_1, self.roi.bins_mag,
self.roi.bins_mag)
# Weight each object before binning
# Divide by solid angle and bin size in magnitudes to get number density
# [objs / deg^2 / mag^2]
if weights is None:
number_density = (solid_angle * self.roi.delta_mag**2)**(-1)
else:
number_density = weights * (solid_angle *
self.roi.delta_mag**2)**(-1)
method = str(method).lower()
if method == 'cloud-in-cells':
# Apply cloud-in-cells algorithm
mmd_background = ugali.utils.binning.cloudInCells(
mag_2,
mag_1, [self.roi.bins_mag, self.roi.bins_mag],
weights=number_density)[0]
elif method == 'bootstrap':
# Not implemented
raise ValueError("Bootstrap method not implemented")
mag_1 + (mag_1_err * np.random.normal(0, 1., len(mag_1)))
mag_2 + (mag_2_err * np.random.normal(0, 1., len(mag_2)))
elif method == 'histogram':
# Apply raw histogram
mmd_background = np.histogram2d(
mag_1,
mag_2,
bins=[self.roi.bins_mag, self.roi.bins_mag],
weights=number_density)[0]
elif method == 'kde':
# Gridded kernel density estimator
logger.warning("### KDE not implemented properly")
mmd_background = ugali.utils.binning.kernelDensity(
mag_2,
mag_1, [self.roi.bins_mag, self.roi.bins_mag],
weights=number_density)[0]
elif method == 'uniform':
logger.warning("### WARNING: Uniform CMD")
hist = np.histogram2d(mag_1,
mag_2,
bins=[self.roi.bins_mag, self.roi.bins_mag],
weights=number_density)[0]
mmd_background = np.mean(hist) * np.ones(hist.shape)
observable = (self.solid_angle_mmd > self.minimum_solid_angle)
mmd_background *= observable
return mmd_background
else:
raise ValueError("Unrecognized method: %s" % method)
## Account for the objects that spill out of the observable space
## But what about the objects that spill out to red colors??
#for index_color in range(0, len(self.roi.centers_color)):
# for index_mag in range(0, len(self.roi.centers_mag)):
# if self.solid_angle_cmd[index_mag][index_color] < self.minimum_solid_angle:
# cmd_background[index_mag - 1][index_color] += cmd_background[index_mag][index_color]
# cmd_background[index_mag][index_color] = 0.
# break
mmd_area = self.solid_angle_mmd * self.roi.delta_mag**2 # [deg^2 * mag^2]
# ADW: This accounts for leakage to faint magnitudes
# But what about the objects that spill out to red colors??
# Maximum obsevable magnitude index for each color (uses the fact that
# np.argmin returns first minimum (zero) instance found.
# NOTE: More complicated maps may have holes causing problems
observable = (self.solid_angle_mmd > self.minimum_solid_angle)
index_mag_1 = observable.argmin(axis=0) - 1
index_mag_2 = np.arange(len(self.roi.centers_mag))
# Add the cumulative leakage back into the last bin of the CMD
leakage = (mmd_background * ~observable).sum(axis=0)
### mmd_background[[index_mag_1,index_mag_2]] += leakage
# Zero out all non-observable bins
### mmd_background *= observable
# Avoid dividing by zero by setting empty bins to the value of the
# minimum filled bin of the CMD. This choice is arbitrary and
# could be replaced by a static minimum, some fraction of the
# CMD maximum, some median clipped minimum, etc. However, should
# be robust against outliers with very small values.
min_mmd_background = max(mmd_background[mmd_background > 0.].min(),
1e-4 * mmd_background.max())
mmd_background[observable] = mmd_background[observable].clip(
min_mmd_background)
### # ADW: This is a fudge factor introduced to renormalize the CMD
### # to the number of input stars in the annulus. While leakage
### # will still smooth the distribution, it shouldn't result in
### fudge_factor = len(mag) / float((cmd_background*cmd_area).sum())
### cmd_background *= fudge_factor
return mmd_background
def backgroundCMD(self, catalog, mode='cloud-in-cells', weights=None):
"""
Generate an empirical background model in color-magnitude space.
INPUTS:
catalog: Catalog object
OUTPUTS:
background
"""
# Select objects in annulus
cut_annulus = self.roi.inAnnulus(catalog.lon, catalog.lat)
color = catalog.color[cut_annulus]
mag = catalog.mag[cut_annulus]
# Units are (deg^2)
solid_angle = ugali.utils.binning.take2D(self.solid_angle_cmd, color,
mag, self.roi.bins_color,
self.roi.bins_mag)
# Weight each object before binning
# Divide by solid angle and bin size in magnitudes to get number density
# [objs / deg^2 / mag^2]
if weights is None:
number_density = (solid_angle * self.roi.delta_color *
self.roi.delta_mag)**(-1)
else:
number_density = weights * (solid_angle * self.roi.delta_color *
self.roi.delta_mag)**(-1)
mode = str(mode).lower()
if mode == 'cloud-in-cells':
# Apply cloud-in-cells algorithm
cmd_background = ugali.utils.binning.cloudInCells(
color,
mag, [self.roi.bins_color, self.roi.bins_mag],
weights=number_density)[0]
elif mode == 'bootstrap':
# Not implemented
raise ValueError("Bootstrap mode not implemented")
mag_1_array = catalog.mag_1
mag_2_array = catalog.mag_2
catalog.mag_1 + (catalog.mag_1_err *
np.random.normal(0, 1., len(catalog.mag_1)))
catalog.mag_2 + (catalog.mag_2_err *
np.random.normal(0, 1., len(catalog.mag_2)))
elif mode == 'histogram':
# Apply raw histogram
cmd_background = np.histogram2d(
mag,
color,
bins=[self.roi.bins_mag, self.roi.bins_color],
weights=number_density)[0]
elif mode == 'kde':
# Gridded kernel density estimator
logger.warning("### KDE not implemented properly")
cmd_background = ugali.utils.binning.kernelDensity(
color,
mag, [self.roi.bins_color, self.roi.bins_mag],
weights=number_density)[0]
elif mode == 'uniform':
logger.warning("### WARNING: Uniform CMD")
hist = np.histogram2d(
mag,
color,
bins=[self.roi.bins_mag, self.roi.bins_color],
weights=number_density)[0]
cmd_background = np.mean(hist) * np.ones(hist.shape)
observable = (self.solid_angle_cmd > self.minimum_solid_angle)
cmd_background *= observable
return cmd_background
else:
raise ValueError("Unrecognized mode: %s" % mode)
## Account for the objects that spill out of the observable space
## But what about the objects that spill out to red colors??
#for index_color in range(0, len(self.roi.centers_color)):
# for index_mag in range(0, len(self.roi.centers_mag)):
# if self.solid_angle_cmd[index_mag][index_color] < self.minimum_solid_angle:
# cmd_background[index_mag - 1][index_color] += cmd_background[index_mag][index_color]
# cmd_background[index_mag][index_color] = 0.
# break
cmd_area = self.solid_angle_cmd * self.roi.delta_color * self.roi.delta_mag # [deg^2 * mag^2]
# ADW: This accounts for leakage to faint magnitudes
# But what about the objects that spill out to red colors??
# Maximum obsevable magnitude index for each color (uses the fact that
# np.argmin returns first minimum (zero) instance found.
# NOTE: More complicated maps may have holes causing problems
observable = (self.solid_angle_cmd > self.minimum_solid_angle)
index_mag = observable.argmin(axis=0) - 1
index_color = np.arange(len(self.roi.centers_color))
# Add the cumulative leakage back into the last bin of the CMD
leakage = (cmd_background * ~observable).sum(axis=0)
cmd_background[[index_mag, index_color]] += leakage
# Zero out all non-observable bins
cmd_background *= observable
# Avoid dividing by zero by setting empty bins to the value of the
# minimum filled bin of the CMD. This choice is arbitrary and
# could be replaced by a static minimum, some fraction of the
# CMD maximum, some median clipped minimum, etc. However, should
# be robust against outliers with very small values.
min_cmd_background = max(cmd_background[cmd_background > 0.].min(),
1e-4 * cmd_background.max())
cmd_background[observable] = cmd_background[observable].clip(
min_cmd_background)
### # ADW: This is a fudge factor introduced to renormalize the CMD
### # to the number of input stars in the annulus. While leakage
### # will still smooth the distribution, it shouldn't result in
### fudge_factor = len(mag) / float((cmd_background*cmd_area).sum())
### cmd_background *= fudge_factor
return cmd_background
def restrictCatalogToObservableSpaceMMD(self, catalog):
"""
Retain only the catalog objects which fall within the observable (i.e., unmasked) space.
Parameters:
catalog: a Catalog object
Returns:
sel : boolean selection array where True means the object would be observable (i.e., unmasked).
ADW: Careful, this function is fragile! The selection here should
be the same as isochrone.observableFraction space. However,
for technical reasons it is faster to do the calculation with
broadcasting there.
"""
# ADW: This creates a slope in color-magnitude space near the magnitude limit
# i.e., if color=g-r then you can't have an object with g-r=1 and mag_r > mask_r-1
# Depending on which is the detection band, this slope will appear at blue
# or red colors. When it occurs at blue colors, it effects very few objects.
# However, when occuring for red objects it can cut many objects. It is
# unclear that this is being correctly accounted for in the likelihood
catalog.spatialBin(self.roi)
sel_roi = (catalog.pixel_roi_index >= 0
) # Objects outside ROI have pixel_roi_index of -1
sel_mag_1 = catalog.mag_1 < self.mask_1.mask_roi_sparse[
catalog.pixel_roi_index]
sel_mag_2 = catalog.mag_2 < self.mask_2.mask_roi_sparse[
catalog.pixel_roi_index]
# and are located in the region of mag-mag space where background can be estimated
sel_mmd = ugali.utils.binning.take2D(
self.solid_angle_mmd, catalog.mag_2, catalog.mag_1,
self.roi.bins_mag, self.roi.bins_mag) > 0.
sel = np.all([sel_roi, sel_mag_1, sel_mag_2, sel_mmd], axis=0)
return sel
def restrictCatalogToObservableSpaceCMD(self, catalog):
"""
Retain only the catalog objects which fall within the
observable (i.e., unmasked) space. NOTE: This returns a
*selection* (i.e., objects are retained if the value of the
output array is True).
Parameters:
catalog: a Catalog object
Returns:
sel : boolean selection array where True means the object would be observable (i.e., unmasked).
ADW: Careful, this function is fragile! The selection here should
be the same as isochrone.observableFraction space. However,
for technical reasons it is faster to do the calculation with
broadcasting there.
"""
# ADW: This creates a slope in color-magnitude space near the magnitude limit
# i.e., if color=g-r then you can't have an object with g-r=1 and mag_r > mask_r-1
# Depending on which is the detection band, this slope will appear at blue
# or red colors. When it occurs at blue colors, it effects very few objects.
# However, when occuring for red objects it can cut many objects. It is
# unclear that this is being correctly accounted for in the likelihood
### # Check that the objects fall in the color-magnitude space of the ROI
### # ADW: I think this is degenerate with the cut_cmd
### sel_mag = np.logical_and(catalog.mag > self.roi.bins_mag[0],
### catalog.mag < self.roi.bins_mag[-1])
### sel_color = np.logical_and(catalog.color > self.roi.bins_color[0],
### catalog.color < self.roi.bins_color[-1])
# and are observable in the ROI-specific mask for both bands
#if not hasattr(catalog, 'pixel_roi_index'): # TODO: An attempt to save computations, but not robust
# catalog.spatialBin(self.roi)
catalog.spatialBin(self.roi)
sel_roi = (catalog.pixel_roi_index >= 0
) # Objects outside ROI have pixel_roi_index of -1
sel_mag_1 = catalog.mag_1 < self.mask_1.mask_roi_sparse[
catalog.pixel_roi_index]
sel_mag_2 = catalog.mag_2 < self.mask_2.mask_roi_sparse[
catalog.pixel_roi_index]
# and are located in the region of color-magnitude space where background can be estimated
sel_cmd = ugali.utils.binning.take2D(
self.solid_angle_cmd, catalog.color, catalog.mag,
self.roi.bins_color, self.roi.bins_mag) > 0.
sel = np.all([sel_roi, sel_mag_1, sel_mag_2, sel_cmd], axis=0)
return sel
# FIXME: Need to parallelize CMD and MMD formulation
restrictCatalogToObservableSpace = restrictCatalogToObservableSpaceCMD
def __init__(self, config, seed=None):
self.config = Config(config)
class Mask(object):
"""
Contains maps of completeness depth in magnitudes for multiple observing bands, and associated products.
"""
def __init__(self, config, roi):
self.config = Config(config)
self.roi = roi
filenames = self.config.getFilenames()
catalog_pixels = self.roi.getCatalogPixels()
self.mask_1 = MaskBand(filenames['mask_1'][catalog_pixels],self.roi)
self.mask_2 = MaskBand(filenames['mask_2'][catalog_pixels],self.roi)
self._fracRoiSparse()
self.minimum_solid_angle = self.config.params['mask']['minimum_solid_angle'] # deg^2
# FIXME: Need to parallelize CMD and MMD formulation
self._solidAngleCMD()
self._pruneCMD(self.minimum_solid_angle)
#self._solidAngleMMD()
#self._pruneMMD(self.minimum_solid_angle)
self._photometricErrors()
@property
def mask_roi_unique(self):
"""
Assemble a set of unique magnitude tuples for the ROI
"""
# There is no good inherent way in numpy to do this...
# http://stackoverflow.com/q/16970982/
# Also possible and simple:
#return np.unique(zip(self.mask_1.mask_roi_sparse,self.mask_2.mask_roi_sparse))
A = np.vstack([self.mask_1.mask_roi_sparse,self.mask_2.mask_roi_sparse]).T
B = A[np.lexsort(A.T[::-1])]
return B[np.concatenate(([True],np.any(B[1:]!=B[:-1],axis=1)))]
@property
def mask_roi_digi(self):
"""
Get the index of the unique magnitude tuple for each pixel in the ROI.
"""
# http://stackoverflow.com/q/24205045/#24206440
A = np.vstack([self.mask_1.mask_roi_sparse,self.mask_2.mask_roi_sparse]).T
B = self.mask_roi_unique
AA = np.ascontiguousarray(A)
BB = np.ascontiguousarray(B)
dt = np.dtype((np.void, AA.dtype.itemsize * AA.shape[1]))
a = AA.view(dt).ravel()
b = BB.view(dt).ravel()
idx = np.argsort(b)
indices = np.searchsorted(b[idx],a)
return idx[indices]
@property
def frac_annulus_sparse(self):
return self.frac_roi_sparse[self.roi.pixel_annulus_cut]
@property
def frac_interior_sparse(self):
return self.frac_roi_sparse[self.roi.pixel_interior_cut]
def _fracRoiSparse(self):
"""
Calculate an approximate pixel coverage fraction from the two masks.
We have no way to know a priori how much the coverage of the
two masks overlap in a give pixel. For example, masks that each
have frac = 0.5 could have a combined frac = [0.0 to 0.5].
The limits will be:
max: min(frac1,frac2)
min: max((frac1+frac2)-1, 0.0)
Sometimes we are lucky and our fracdet is actually already
calculated for the two masks combined, so that the max
condition is satisfied. That is what we will assume...
"""
self.frac_roi_sparse = np.min([self.mask_1.frac_roi_sparse,self.mask_2.frac_roi_sparse],axis=0)
return self.frac_roi_sparse
def _solidAngleMMD(self):
"""
Compute solid angle within the mask annulus (deg^2) as a
function of mag_1 and mag_2
"""
# Take upper corner of the magnitude bin
mag_2,mag_1 = np.meshgrid(self.roi.bins_mag[1:],self.roi.bins_mag[1:])
# Havent tested since adding fracdet
unmasked_mag_1 = (self.mask_1.mask_annulus_sparse[:,np.newaxis]>mag_1[:,np.newaxis])
unmasked_mag_2 = (self.mask_2.mask_annulus_sparse[:,np.newaxis]>mag_2[:,np.newaxis])
n_unmasked_pixels = (unmasked_mag_1*unmasked_mag_2*self.frac_annulus_sparse).sum(axis=1)
self.solid_angle_mmd = self.roi.area_pixel * n_unmasked_pixels
if self.solid_angle_mmd.sum() == 0:
msg = "Mask annulus contains no solid angle."
logger.error(msg)
raise Exception(msg)
def _pruneMMD(self, minimum_solid_angle):
"""
Remove regions of magnitude-magnitude space where the unmasked solid angle is
statistically insufficient to estimate the background.
INPUTS:
solid_angle[1]: minimum solid angle (deg^2)
"""
logger.info('Pruning mask based on minimum solid angle of %.2f deg^2'%(minimum_solid_angle))
solid_angle_mmd = self.solid_angle_mmd*(self.solid_angle_mmd > minimum_solid_angle)
if solid_angle_mmd.sum() == 0:
msg = "Pruned mask contains no solid angle."
logger.error(msg)
raise Exception(msg)
self.solid_angle_mmd = solid_angle_mmd
# Compute which magnitudes the clipping correspond to
index_mag_1, index_mag_2 = np.nonzero(self.solid_angle_mmd)
self.mag_1_clip = self.roi.bins_mag[1:][np.max(index_mag_1)]
self.mag_2_clip = self.roi.bins_mag[1:][np.max(index_mag_2)]
logger.info('Clipping mask 1 at %.2f mag'%(self.mag_1_clip) )
logger.info('Clipping mask 2 at %.2f mag'%(self.mag_2_clip) )
self.mask_1.mask_roi_sparse = np.clip(self.mask_1.mask_roi_sparse, 0., self.mag_1_clip)
self.mask_2.mask_roi_sparse = np.clip(self.mask_2.mask_roi_sparse, 0., self.mag_2_clip)
def _solidAngleCMD(self):
"""
Compute solid angle within the mask annulus (deg^2) as a
function of color and magnitude.
"""
self.solid_angle_cmd = np.zeros([len(self.roi.centers_mag),
len(self.roi.centers_color)])
for index_mag in np.arange(len(self.roi.centers_mag)):
for index_color in np.arange(len(self.roi.centers_color)):
# mag and color at bin center
mag = self.roi.centers_mag[index_mag]
color = self.roi.centers_color[index_color]
if self.config.params['catalog']['band_1_detection']:
# Evaluating at the center of the color-mag bin, be consistent!
#mag_1 = self.roi.centers_mag[index_mag]
#color = self.roi.centers_color[index_color]
#mag_2 = mag_1 - color
# Evaluating at corner of the color-mag bin, be consistent!
mag_1 = mag + (0.5 * self.roi.delta_mag)
mag_2 = mag - color + (0.5 * self.roi.delta_color)
else:
# Evaluating at the center of the color-mag bin, be consistent!
#mag_2 = self.roi.centers_mag[index_mag]
#color = self.roi.centers_color[index_color]
#mag_1 = mag_2 + color
# Evaluating at corner of the color-mag bin, be consistent!
mag_1 = mag + color + (0.5 * self.roi.delta_color)
mag_2 = mag + (0.5 * self.roi.delta_mag)
# ADW: Is there a problem here?
#self.solid_angle_cmd[index_mag, index_color] = self.roi.area_pixel * np.sum((self.mask_1.mask > mag_1) * (self.mask_2.mask > mag_2))
# ADW: I think we want to keep pixels that are >= mag
unmasked_mag_1 = (self.mask_1.mask_annulus_sparse >= mag_1)
unmasked_mag_2 = (self.mask_2.mask_annulus_sparse >= mag_2)
n_unmasked_pixels = np.sum(unmasked_mag_1*unmasked_mag_2*self.frac_annulus_sparse)
#n_unmasked_pixels = np.sum((self.mask_1.mask_annulus_sparse > mag_1) \
# * (self.mask_2.mask_annulus_sparse > mag_2))
self.solid_angle_cmd[index_mag, index_color] = self.roi.area_pixel * n_unmasked_pixels
if self.solid_angle_cmd.sum() == 0:
msg = "Mask annulus contains no solid angle."
logger.error(msg)
raise Exception(msg)
return self.solid_angle_cmd
def _solidAngleCMD(self):
"""
Compute solid angle within the mask annulus (deg^2) as a
function of color and magnitude.
Returns:
--------
solid_angle_cmd : 2d array
"""
self.solid_angle_cmd = np.zeros([len(self.roi.centers_mag),
len(self.roi.centers_color)])
idx_mag,idx_color=np.where(self.solid_angle_cmd == 0)
mag = self.roi.centers_mag[idx_mag]
color = self.roi.centers_color[idx_color]
if self.config.params['catalog']['band_1_detection']:
# Evaluating at corner of the color-mag bin, be consistent!
mag_1 = mag + (0.5 * self.roi.delta_mag)
mag_2 = mag - color + (0.5 * self.roi.delta_color)
else:
# Evaluating at corner of the color-mag bin, be consistent!
mag_1 = mag + color + (0.5 * self.roi.delta_color)
mag_2 = mag + (0.5 * self.roi.delta_mag)
n_unmasked_pixels = np.zeros_like(mag)
for i in np.arange(len(mag_1)):
unmasked_mag_1 = (self.mask_1.mask_annulus_sparse >= mag_1[i])
unmasked_mag_2 = (self.mask_2.mask_annulus_sparse >= mag_2[i])
n_unmasked_pixels[i] = np.sum(unmasked_mag_1 * unmasked_mag_2 *
self.frac_annulus_sparse)
self.solid_angle_cmd[idx_mag, idx_color] = self.roi.area_pixel * n_unmasked_pixels
if self.solid_angle_cmd.sum() == 0:
msg = "Mask annulus contains no solid angle."
logger.error(msg)
raise Exception(msg)
return self.solid_angle_cmd
def _pruneCMD(self, minimum_solid_angle):
"""
Remove regions of color-magnitude space where the unmasked solid angle is
statistically insufficient to estimate the background.
ADW: Why are we clipping at the bin center instead of edge?
INPUTS:
solid_angle[1]: minimum solid angle (deg^2)
"""
logger.info('Pruning mask based on minimum solid angle of %.2f deg^2'%(minimum_solid_angle))
self.solid_angle_cmd *= self.solid_angle_cmd > minimum_solid_angle
if self.solid_angle_cmd.sum() == 0:
msg = "Pruned mask contains no solid angle."
logger.error(msg)
raise Exception(msg)
# Compute which magnitudes the clipping correspond to
index_mag, index_color = np.nonzero(self.solid_angle_cmd)
mag = self.roi.centers_mag[index_mag]
color = self.roi.centers_color[index_color]
if self.config.params['catalog']['band_1_detection']:
mag_1 = mag
mag_2 = mag_1 - color
self.mag_1_clip = np.max(mag_1) + (0.5 * self.roi.delta_mag)
self.mag_2_clip = np.max(mag_2) + (0.5 * self.roi.delta_color)
else:
mag_2 = mag
mag_1 = color + mag_2
self.mag_1_clip = np.max(mag_1) + (0.5 * self.roi.delta_color)
self.mag_2_clip = np.max(mag_2) + (0.5 * self.roi.delta_mag)
logger.info('Clipping mask 1 at %.2f mag'%(self.mag_1_clip) )
logger.info('Clipping mask 2 at %.2f mag'%(self.mag_2_clip) )
self.mask_1.mask_roi_sparse = np.clip(self.mask_1.mask_roi_sparse, 0., self.mag_1_clip)
self.mask_2.mask_roi_sparse = np.clip(self.mask_2.mask_roi_sparse, 0., self.mag_2_clip)
def completeness(self, delta, method='step'):
"""
Return the completeness as a function of magnitude.
ADW: Eventually want a completeness mask to set overall efficiency.
"""
delta = np.asarray(delta)
if method == 'step':
func = lambda delta: (delta > 0).astype(float)
elif method == 'erf':
# Trust the SDSS EDR???
# 95% completeness:
def func(delta):
# Efficiency at bright end (assumed to be 100%)
e = 1.0
# EDR says full width is ~0.5 mag
width = 0.2
# This should be the halfway point in the curve
return (e/2.0)*(1/np.sqrt(2*width))*(np.sqrt(2*width)-scipy.special.erf(-delta))
elif method == 'flemming':
# Functional form taken from Fleming et al. AJ 109, 1044 (1995)
# http://adsabs.harvard.edu/abs/1995AJ....109.1044F
# f = 1/2 [1 - alpha(V - Vlim)/sqrt(1 + alpha^2 (V - Vlim)^2)]
# CAREFUL: This definition is for Vlim = 50% completeness
def func(delta):
alpha = 2.0
return 0.5 * (1 - (alpha * delta)/np.sqrt(1+alpha**2 * delta**2))
else:
raise Exception('...')
return func(delta)
def _photometricErrors(self, catalog=None, n_per_bin=100):
"""
Realistic photometric errors estimated from catalog objects and mask.
Extend below the magnitude threshold with a flat extrapolation.
"""
if catalog is None:
# Simple proxy for photometric errors
release = self.config['data']['release']
band_1 = self.config['catalog'].get('mag_1_band')
if not band_1: band_1 = self.config['isochrone']['mag_1_field']
band_2 = self.config['catalog'].get('mag_2_band')
if not band_2: band_2 = self.config['isochrone']['mag_2_field']
DELMIN = 0.0
pars_1 = MAGERR_PARAMS[release][band_1]
def photo_err_1(delta):
p = pars_1
return np.clip(np.exp(p[0]*delta+p[1])+p[2], 0, np.exp(p[0]*(DELMIN)+p[1])+p[2])
pars_2 = MAGERR_PARAMS[release][band_2]
def photo_err_2(delta):
p = pars_2
return np.clip(np.exp(p[0]*delta+p[1])+p[2], 0, np.exp(p[0]*(DELMIN)+p[1])+p[2])
else:
catalog.spatialBin(self.roi)
if len(catalog.mag_1) < n_per_bin:
logger.warning("Catalog contains fewer objects than requested to calculate errors.")
#n_per_bin = int(len(catalog.mag_1) / 3)
return self._photometricErrors(catalog=None)
# Band 1
mag_1_thresh = self.mask_1.mask_roi_sparse[catalog.pixel_roi_index] - catalog.mag_1
sorting_indices = np.argsort(mag_1_thresh)
mag_1_thresh_sort = mag_1_thresh[sorting_indices]
mag_err_1_sort = catalog.mag_err_1[sorting_indices]
# ADW: Can't this be done with np.median(axis=?)
mag_1_thresh_medians = []
mag_err_1_medians = []
for i in range(0, int(len(mag_1_thresh) / float(n_per_bin))):
mag_1_thresh_medians.append(np.median(mag_1_thresh_sort[n_per_bin * i: n_per_bin * (i + 1)]))
mag_err_1_medians.append(np.median(mag_err_1_sort[n_per_bin * i: n_per_bin * (i + 1)]))
if mag_1_thresh_medians[0] > 0.:
mag_1_thresh_medians = np.insert(mag_1_thresh_medians, 0, -99.)
mag_err_1_medians = np.insert(mag_err_1_medians, 0, mag_err_1_medians[0])
photo_err_1 = scipy.interpolate.interp1d(mag_1_thresh_medians, mag_err_1_medians,
bounds_error=False, fill_value=mag_err_1_medians[-1])
# Band 2
mag_2_thresh = self.mask_2.mask_roi_sparse[catalog.pixel_roi_index] - catalog.mag_2
sorting_indices = np.argsort(mag_2_thresh)
mag_2_thresh_sort = mag_2_thresh[sorting_indices]
mag_err_2_sort = catalog.mag_err_2[sorting_indices]
mag_2_thresh_medians = []
mag_err_2_medians = []
for i in range(0, int(len(mag_2_thresh) / float(n_per_bin))):
mag_2_thresh_medians.append(np.median(mag_2_thresh_sort[n_per_bin * i: n_per_bin * (i + 1)]))
mag_err_2_medians.append(np.median(mag_err_2_sort[n_per_bin * i: n_per_bin * (i + 1)]))
if mag_2_thresh_medians[0] > 0.:
mag_2_thresh_medians = np.insert(mag_2_thresh_medians, 0, -99.)
mag_err_2_medians = np.insert(mag_err_2_medians, 0, mag_err_2_medians[0])
photo_err_2 = scipy.interpolate.interp1d(mag_2_thresh_medians, mag_err_2_medians,
bounds_error=False, fill_value=mag_err_2_medians[-1])
self.photo_err_1=photo_err_1
self.photo_err_2=photo_err_2
return self.photo_err_1, self.photo_err_2
def plotSolidAngleCMD(self):
"""
Solid angle within the mask as a function of color and magnitude.
"""
msg = "'%s.plotSolidAngleCMD': ADW 2018-05-05"%self.__class__.__name__
DeprecationWarning(msg)
import ugali.utils.plotting
ugali.utils.plotting.twoDimensionalHistogram('mask', 'color', 'magnitude',
self.solid_angle_cmd,
self.roi.bins_color,
self.roi.bins_mag,
lim_x = [self.roi.bins_color[0],
self.roi.bins_color[-1]],
lim_y = [self.roi.bins_mag[-1],
self.roi.bins_mag[0]])
def plotSolidAngleMMD(self):
"""
Solid angle within the mask as a function of color and magnitude.
"""
msg = "'%s.plotSolidAngleMMD': ADW 2018-05-05"%self.__class__.__name__
DeprecationWarning(msg)
import ugali.utils.plotting
ugali.utils.plotting.twoDimensionalHistogram('mask', 'mag_2', 'mag_1',
self.solid_angle_mmd,
self.roi.bins_mag,
self.roi.bins_mag,
lim_x = [self.roi.bins_mag[0],
self.roi.bins_mag[-1]],
lim_y = [self.roi.bins_mag[-1],
self.roi.bins_mag[0]])
def backgroundMMD(self, catalog, method='cloud-in-cells', weights=None):
"""
Generate an empirical background model in magnitude-magnitude space.
INPUTS:
catalog: Catalog object
OUTPUTS:
background
"""
# Select objects in annulus
cut_annulus = self.roi.inAnnulus(catalog.lon,catalog.lat)
mag_1 = catalog.mag_1[cut_annulus]
mag_2 = catalog.mag_2[cut_annulus]
# Units are (deg^2)
solid_angle = ugali.utils.binning.take2D(self.solid_angle_mmd, mag_2, mag_1,
self.roi.bins_mag, self.roi.bins_mag)
# Weight each object before binning
# Divide by solid angle and bin size in magnitudes to get number density
# [objs / deg^2 / mag^2]
if weights is None:
number_density = (solid_angle*self.roi.delta_mag**2)**(-1)
else:
number_density = weights*(solid_angle*self.roi.delta_mag**2)**(-1)
method = str(method).lower()
if method == 'cloud-in-cells':
# Apply cloud-in-cells algorithm
mmd_background = ugali.utils.binning.cloudInCells(mag_2,mag_1,
[self.roi.bins_mag,self.roi.bins_mag],
weights=number_density)[0]
elif method == 'bootstrap':
# Not implemented
raise ValueError("Bootstrap method not implemented")
mag_1 + (mag_1_err * np.random.normal(0, 1., len(mag_1)))
mag_2 + (mag_2_err * np.random.normal(0, 1., len(mag_2)))
elif method == 'histogram':
# Apply raw histogram
mmd_background = np.histogram2d(mag_1,mag_2,bins=[self.roi.bins_mag,self.roi.bins_mag],
weights=number_density)[0]
elif method == 'kde':
# Gridded kernel density estimator
logger.warning("### KDE not implemented properly")
mmd_background = ugali.utils.binning.kernelDensity(mag_2,mag_1,
[self.roi.bins_mag,self.roi.bins_mag],
weights=number_density)[0]
elif method == 'uniform':
logger.warning("### WARNING: Uniform CMD")
hist = np.histogram2d(mag_1,mag_2,bins=[self.roi.bins_mag,self.roi.bins_mag],
weights=number_density)[0]
mmd_background = np.mean(hist)*np.ones(hist.shape)
observable = (self.solid_angle_mmd > self.minimum_solid_angle)
mmd_background *= observable
return mmd_background
else:
raise ValueError("Unrecognized method: %s"%method)
## Account for the objects that spill out of the observable space
## But what about the objects that spill out to red colors??
#for index_color in range(0, len(self.roi.centers_color)):
# for index_mag in range(0, len(self.roi.centers_mag)):
# if self.solid_angle_cmd[index_mag][index_color] < self.minimum_solid_angle:
# cmd_background[index_mag - 1][index_color] += cmd_background[index_mag][index_color]
# cmd_background[index_mag][index_color] = 0.
# break
mmd_area = self.solid_angle_mmd*self.roi.delta_mag**2 # [deg^2 * mag^2]
# ADW: This accounts for leakage to faint magnitudes
# But what about the objects that spill out to red colors??
# Maximum obsevable magnitude index for each color (uses the fact that
# np.argmin returns first minimum (zero) instance found.
# NOTE: More complicated maps may have holes causing problems
observable = (self.solid_angle_mmd > self.minimum_solid_angle)
index_mag_1 = observable.argmin(axis=0) - 1
index_mag_2 = np.arange(len(self.roi.centers_mag))
# Add the cumulative leakage back into the last bin of the CMD
leakage = (mmd_background * ~observable).sum(axis=0)
### mmd_background[[index_mag_1,index_mag_2]] += leakage
# Zero out all non-observable bins
### mmd_background *= observable
# Avoid dividing by zero by setting empty bins to the value of the
# minimum filled bin of the CMD. This choice is arbitrary and
# could be replaced by a static minimum, some fraction of the
# CMD maximum, some median clipped minimum, etc. However, should
# be robust against outliers with very small values.
min_mmd_background = max(mmd_background[mmd_background > 0.].min(),
1e-4*mmd_background.max())
mmd_background[observable] = mmd_background[observable].clip(min_mmd_background)
### # ADW: This is a fudge factor introduced to renormalize the CMD
### # to the number of input stars in the annulus. While leakage
### # will still smooth the distribution, it shouldn't result in
### fudge_factor = len(mag) / float((cmd_background*cmd_area).sum())
### cmd_background *= fudge_factor
return mmd_background
def backgroundCMD(self, catalog, mode='cloud-in-cells', weights=None):
"""
Generate an empirical background model in color-magnitude space.
INPUTS:
catalog: Catalog object
OUTPUTS:
background
"""
# Select objects in annulus
cut_annulus = self.roi.inAnnulus(catalog.lon,catalog.lat)
color = catalog.color[cut_annulus]
mag = catalog.mag[cut_annulus]
# Units are (deg^2)
solid_angle = ugali.utils.binning.take2D(self.solid_angle_cmd, color, mag,
self.roi.bins_color, self.roi.bins_mag)
# Weight each object before binning
# Divide by solid angle and bin size in magnitudes to get number density
# [objs / deg^2 / mag^2]
if weights is None:
number_density = (solid_angle*self.roi.delta_color*self.roi.delta_mag)**(-1)
else:
number_density = weights*(solid_angle*self.roi.delta_color*self.roi.delta_mag)**(-1)
mode = str(mode).lower()
if mode == 'cloud-in-cells':
# Apply cloud-in-cells algorithm
cmd_background = ugali.utils.binning.cloudInCells(color,mag,
[self.roi.bins_color,self.roi.bins_mag],
weights=number_density)[0]
elif mode == 'bootstrap':
# Not implemented
raise ValueError("Bootstrap mode not implemented")
mag_1_array = catalog.mag_1
mag_2_array = catalog.mag_2
catalog.mag_1 + (catalog.mag_1_err * np.random.normal(0, 1., len(catalog.mag_1)))
catalog.mag_2 + (catalog.mag_2_err * np.random.normal(0, 1., len(catalog.mag_2)))
elif mode == 'histogram':
# Apply raw histogram
cmd_background = np.histogram2d(mag,color,bins=[self.roi.bins_mag,self.roi.bins_color],
weights=number_density)[0]
elif mode == 'kde':
# Gridded kernel density estimator
logger.warning("### KDE not implemented properly")
cmd_background = ugali.utils.binning.kernelDensity(color,mag,
[self.roi.bins_color,self.roi.bins_mag],
weights=number_density)[0]
elif mode == 'uniform':
logger.warning("### WARNING: Uniform CMD")
hist = np.histogram2d(mag,color,bins=[self.roi.bins_mag,self.roi.bins_color], weights=number_density)[0]
cmd_background = np.mean(hist)*np.ones(hist.shape)
observable = (self.solid_angle_cmd > self.minimum_solid_angle)
cmd_background *= observable
return cmd_background
else:
raise ValueError("Unrecognized mode: %s"%mode)
## Account for the objects that spill out of the observable space
## But what about the objects that spill out to red colors??
#for index_color in range(0, len(self.roi.centers_color)):
# for index_mag in range(0, len(self.roi.centers_mag)):
# if self.solid_angle_cmd[index_mag][index_color] < self.minimum_solid_angle:
# cmd_background[index_mag - 1][index_color] += cmd_background[index_mag][index_color]
# cmd_background[index_mag][index_color] = 0.
# break
cmd_area = self.solid_angle_cmd*self.roi.delta_color*self.roi.delta_mag # [deg^2 * mag^2]
# ADW: This accounts for leakage to faint magnitudes
# But what about the objects that spill out to red colors??
# Maximum obsevable magnitude index for each color (uses the fact that
# np.argmin returns first minimum (zero) instance found.
# NOTE: More complicated maps may have holes causing problems
observable = (self.solid_angle_cmd > self.minimum_solid_angle)
index_mag = observable.argmin(axis=0) - 1
index_color = np.arange(len(self.roi.centers_color))
# Add the cumulative leakage back into the last bin of the CMD
leakage = (cmd_background * ~observable).sum(axis=0)
cmd_background[[index_mag,index_color]] += leakage
# Zero out all non-observable bins
cmd_background *= observable
# Avoid dividing by zero by setting empty bins to the value of the
# minimum filled bin of the CMD. This choice is arbitrary and
# could be replaced by a static minimum, some fraction of the
# CMD maximum, some median clipped minimum, etc. However, should
# be robust against outliers with very small values.
min_cmd_background = max(cmd_background[cmd_background > 0.].min(),
1e-4*cmd_background.max())
cmd_background[observable] = cmd_background[observable].clip(min_cmd_background)
### # ADW: This is a fudge factor introduced to renormalize the CMD
### # to the number of input stars in the annulus. While leakage
### # will still smooth the distribution, it shouldn't result in
### fudge_factor = len(mag) / float((cmd_background*cmd_area).sum())
### cmd_background *= fudge_factor
return cmd_background
def restrictCatalogToObservableSpaceMMD(self, catalog):
"""
Retain only the catalog objects which fall within the observable (i.e., unmasked) space.
Parameters:
catalog: a Catalog object
Returns:
sel : boolean selection array where True means the object would be observable (i.e., unmasked).
ADW: Careful, this function is fragile! The selection here should
be the same as isochrone.observableFraction space. However,
for technical reasons it is faster to do the calculation with
broadcasting there.
"""
# ADW: This creates a slope in color-magnitude space near the magnitude limit
# i.e., if color=g-r then you can't have an object with g-r=1 and mag_r > mask_r-1
# Depending on which is the detection band, this slope will appear at blue
# or red colors. When it occurs at blue colors, it effects very few objects.
# However, when occuring for red objects it can cut many objects. It is
# unclear that this is being correctly accounted for in the likelihood
catalog.spatialBin(self.roi)
sel_roi = (catalog.pixel_roi_index >= 0) # Objects outside ROI have pixel_roi_index of -1
sel_mag_1 = catalog.mag_1 < self.mask_1.mask_roi_sparse[catalog.pixel_roi_index]
sel_mag_2 = catalog.mag_2 < self.mask_2.mask_roi_sparse[catalog.pixel_roi_index]
# and are located in the region of mag-mag space where background can be estimated
sel_mmd = ugali.utils.binning.take2D(self.solid_angle_mmd,
catalog.mag_2, catalog.mag_1,
self.roi.bins_mag, self.roi.bins_mag) > 0.
sel = np.all([sel_roi,sel_mag_1,sel_mag_2,sel_mmd], axis=0)
return sel
def restrictCatalogToObservableSpaceCMD(self, catalog):
"""
Retain only the catalog objects which fall within the
observable (i.e., unmasked) space. NOTE: This returns a
*selection* (i.e., objects are retained if the value of the
output array is True).
Parameters:
catalog: a Catalog object
Returns:
sel : boolean selection array where True means the object would be observable (i.e., unmasked).
ADW: Careful, this function is fragile! The selection here should
be the same as isochrone.observableFraction space. However,
for technical reasons it is faster to do the calculation with
broadcasting there.
"""
# ADW: This creates a slope in color-magnitude space near the magnitude limit
# i.e., if color=g-r then you can't have an object with g-r=1 and mag_r > mask_r-1
# Depending on which is the detection band, this slope will appear at blue
# or red colors. When it occurs at blue colors, it effects very few objects.
# However, when occuring for red objects it can cut many objects. It is
# unclear that this is being correctly accounted for in the likelihood
### # Check that the objects fall in the color-magnitude space of the ROI
### # ADW: I think this is degenerate with the cut_cmd
### sel_mag = np.logical_and(catalog.mag > self.roi.bins_mag[0],
### catalog.mag < self.roi.bins_mag[-1])
### sel_color = np.logical_and(catalog.color > self.roi.bins_color[0],
### catalog.color < self.roi.bins_color[-1])
# and are observable in the ROI-specific mask for both bands
#if not hasattr(catalog, 'pixel_roi_index'): # TODO: An attempt to save computations, but not robust
# catalog.spatialBin(self.roi)
catalog.spatialBin(self.roi)
sel_roi = (catalog.pixel_roi_index >= 0) # Objects outside ROI have pixel_roi_index of -1
sel_mag_1 = catalog.mag_1 < self.mask_1.mask_roi_sparse[catalog.pixel_roi_index]
sel_mag_2 = catalog.mag_2 < self.mask_2.mask_roi_sparse[catalog.pixel_roi_index]
# and are located in the region of color-magnitude space where background can be estimated
sel_cmd = ugali.utils.binning.take2D(self.solid_angle_cmd,
catalog.color, catalog.mag,
self.roi.bins_color, self.roi.bins_mag) > 0.
sel = np.all([sel_roi,sel_mag_1,sel_mag_2,sel_cmd], axis=0)
return sel
# FIXME: Need to parallelize CMD and MMD formulation
restrictCatalogToObservableSpace = restrictCatalogToObservableSpaceCMD
def satellite(isochrone, kernel, stellar_mass, distance_modulus, **kwargs):
"""
Wrapping the isochrone and kernel simulate functions.
"""
mag_1, mag_2 = isochrone.simulate(stellar_mass, distance_modulus)
lon, lat = kernel.simulate(len(mag_1))
return mag_1, mag_2, lon, lat
############################################################
if __name__ == "__main__":
import ugali.utils.parser
description = "Script for executing the likelihood scan."
parser = ugali.utils.parser.Parser(description=description)
parser.add_config()
parser.add_argument('outfile',
metavar='outfile.fits',
help='Output fits file.')
parser.add_debug()
parser.add_verbose()
parser.add_seed()
opts = parser.parse_args()
config = Config(opts.config)
generator = Generator(config, opts.seed)
sim, like1, like2 = generator.run(opts.outfile)
def __init__(self, config, seed=None):
self.config = Config(config)
self.seed = seed
if self.seed is not None: np.random.seed(self.seed)
def split(config,dirname='split',force=False):
""" Take a pre-existing maglim map and divide it into
chunks consistent with the catalog pixels. """
config = Config(config)
filenames = config.getFilenames()
#healpix = filenames['pix'].compressed()
# Check that things are ok
basedir,basename = os.path.split(config['mask']['dirname'])
#if basename == dirname:
# msg = "Input and output directory are the same."
# raise Exception(msg)
outdir = mkdir(os.path.join(basedir,dirname))
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
release = config['data']['release'].lower()
band1 = config['catalog']['mag_1_band']
band2 = config['catalog']['mag_2_band']
# Read the magnitude limits
maglimdir = config['maglim']['dirname']
maglimfile_1 = join(maglimdir,config['maglim']['filename_1'])
logger.info("Reading %s..."%maglimfile_1)
maglim1 = read_map(maglimfile_1)
maglimfile_2 = join(maglimdir,config['maglim']['filename_2'])
logger.info("Reading %s..."%maglimfile_2)
maglim2 = read_map(maglimfile_2)
# Read the footprint
footfile = config['data']['footprint']
logger.info("Reading %s..."%footfile)
footprint = read_map(footfile)
# Output mask names
mask1 = os.path.basename(config['mask']['basename_1'])
mask2 = os.path.basename(config['mask']['basename_2'])
for band,maglim,base in [(band1,maglim1,mask1),(band2,maglim2,mask2)]:
nside_maglim = hp.npix2nside(len(maglim))
if nside_maglim != nside_pixel:
msg = "Mask nside different from pixel nside"
logger.warning(msg)
#raise Exception(msg)
pixels = np.nonzero(maglim>0)[0]
superpix = superpixel(pixels,nside_maglim,nside_catalog)
healpix = np.unique(superpix)
for hpx in healpix:
outfile = join(outdir,base)%hpx
if os.path.exists(outfile) and not force:
logger.warning("Found %s; skipping..."%outfile)
continue
pix = pixels[superpix == hpx]
print(hpx, len(pix))
logger.info('Writing %s...'%outfile)
data = odict()
data['PIXEL']=pix
data['MAGLIM']=maglim[pix].astype('f4')
data['FRACDET']=footprint[pix].astype('f4')
ugali.utils.healpix.write_partial_map(outfile,data,nside_pixel)
