def inFootprint(self, pixels, nside=None):
"""
Open each valid filename for the set of pixels and determine the set
of subpixels with valid data.
"""
if np.isscalar(pixels): pixels = np.array([pixels])
if nside is None: nside = self.nside_likelihood
inside = np.zeros(len(pixels), dtype='bool')
if not self.nside_catalog:
catalog_pix = [0]
else:
catalog_pix = superpixel(pixels, nside, self.nside_catalog)
catalog_pix = np.intersect1d(catalog_pix, self.catalog_pixels)
for filenames in self.filenames[catalog_pix]:
# ADW: Need to replace with healpix functions...
#logger.debug("Loading %s"%filenames['mask_1'])
#subpix_1,val_1 = ugali.utils.skymap.readSparseHealpixMap(filenames['mask_1'],'MAGLIM',construct_map=False)
_n, subpix_1, val_1 = read_partial_map(filenames['mask_1'],
'MAGLIM',
fullsky=False)
#logger.debug("Loading %s"%filenames['mask_2'])
#subpix_2,val_2 = ugali.utils.skymap.readSparseHealpixMap(filenames['mask_2'],'MAGLIM',construct_map=False)
_n, subpix_2, val_2 = read_partial_map(filenames['mask_2'],
'MAGLIM',
fullsky=False)
subpix = np.intersect1d(subpix_1, subpix_2)
superpix = np.unique(superpixel(subpix, self.nside_pixel, nside))
inside |= np.in1d(pixels, superpix)
return inside
def inFootprint(config, pixels, nside=None):
"""
Open each valid filename for the set of pixels and determine the set
of subpixels with valid data.
"""
config = Config(config)
nside_catalog = config['coords']['nside_catalog']
nside_likelihood = config['coords']['nside_likelihood']
nside_pixel = config['coords']['nside_pixel']
if np.isscalar(pixels): pixels = np.array([pixels])
if nside is None: nside = nside_likelihood
filenames = config.getFilenames()
catalog_pixels = filenames['pix'].compressed()
inside = np.zeros(len(pixels), dtype=bool)
if not nside_catalog:
catalog_pix = [0]
else:
catalog_pix = superpixel(pixels,nside,nside_catalog)
catalog_pix = np.intersect1d(catalog_pix,catalog_pixels)
for fnames in filenames[catalog_pix]:
logger.debug("Loading %s"%filenames['mask_1'])
#subpix_1,val_1 = ugali.utils.skymap.readSparseHealpixMap(fnames['mask_1'],'MAGLIM',construct_map=False)
_nside,subpix_1,val_1 = ugali.utils.healpix.read_partial_map(fnames['mask_1'],'MAGLIM',fullsky=False)
logger.debug("Loading %s"%fnames['mask_2'])
#subpix_2,val_2 = ugali.utils.skymap.readSparseHealpixMap(fnames['mask_2'],'MAGLIM',construct_map=False)
_nside,subpix_2,val_2 = ugali.utils.healpix.read_partial_map(fnames['mask_2'],'MAGLIM',fullsky=False)
subpix = np.intersect1d(subpix_1,subpix_2)
superpix = np.unique(superpixel(subpix,nside_pixel,nside))
inside |= np.in1d(pixels, superpix)
return inside
def inFootprint(config, pixels, nside=None):
"""
Open each valid filename for the set of pixels and determine the set
of subpixels with valid data.
"""
config = Config(config)
nside_catalog = config['coords']['nside_catalog']
nside_likelihood = config['coords']['nside_likelihood']
nside_pixel = config['coords']['nside_pixel']
if numpy.isscalar(pixels): pixels = numpy.array([pixels])
if nside is None: nside = nside_likelihood
filenames = config.getFilenames()
catalog_pixels = filenames['pix'].compressed()
inside = numpy.zeros(len(pixels), dtype=bool)
if not nside_catalog:
catalog_pix = [0]
else:
catalog_pix = superpixel(pixels, nside, nside_catalog)
catalog_pix = numpy.intersect1d(catalog_pix, catalog_pixels)
for fnames in filenames[catalog_pix]:
logger.debug("Loading %s" % filenames['mask_1'])
subpix_1, val_1 = ugali.utils.skymap.readSparseHealpixMap(
fnames['mask_1'], 'MAGLIM', construct_map=False)
logger.debug("Loading %s" % fnames['mask_2'])
subpix_2, val_2 = ugali.utils.skymap.readSparseHealpixMap(
fnames['mask_2'], 'MAGLIM', construct_map=False)
subpix = numpy.intersect1d(subpix_1, subpix_2)
superpix = numpy.unique(superpixel(subpix, nside_pixel, nside))
inside |= numpy.in1d(pixels, superpix)
return inside
def inFootprint(config, pixels, nside=None):
"""
Open each valid filename for the set of pixels and determine the set
of subpixels with valid data.
Parameters
----------
config : config
Configuration (file or object)
pixels : array or int
List of pixels to create footprint for
nside : int, optional
Healpix nside
Returns
-------
inside : array
Boolean array of whether pixel is in footprint
"""
logger.info("Calculating survey footprint...")
config = Config(config)
nside_catalog = config['coords']['nside_catalog']
nside_likelihood = config['coords']['nside_likelihood']
nside_pixel = config['coords']['nside_pixel']
if np.isscalar(pixels): pixels = np.array([pixels])
if nside is None: nside = nside_likelihood
filenames = config.getFilenames()
catalog_pixels = filenames['pix'].compressed()
inside = np.zeros(len(pixels), dtype=bool)
if not nside_catalog:
catalog_pix = [0]
else:
catalog_pix = superpixel(pixels, nside, nside_catalog)
catalog_pix = np.intersect1d(catalog_pix, catalog_pixels)
fnames = filenames[catalog_pix]
# Load the first mask
logger.debug("Loading %s" % fnames['mask_1'])
_nside, subpix1, val1 = read_partial_map(fnames['mask_1'],
'MAGLIM',
fullsky=False,
multiproc=8)
# Load the second mask
logger.debug("Loading %s" % fnames['mask_2'])
_nside, subpix2, val2 = read_partial_map(fnames['mask_2'],
'MAGLIM',
fullsky=False,
multiproc=8)
# Run the subpixels
subpix = np.intersect1d(subpix1, subpix2)
superpix = np.unique(superpixel(subpix, nside_pixel, nside))
inside |= np.in1d(pixels, superpix)
return inside
def inFootprint(self, pixels, nside=None):
"""
Open each valid filename for the set of pixels and determine the set
of subpixels with valid data.
"""
if numpy.isscalar(pixels): pixels = numpy.array([pixels])
if nside is None: nside = self.nside_likelihood
inside = numpy.zeros( len(pixels), dtype='bool')
if not self.nside_catalog:
catalog_pix = [0]
else:
catalog_pix = superpixel(pixels,nside,self.nside_catalog)
catalog_pix = numpy.intersect1d(catalog_pix,self.catalog_pixels)
for filenames in self.filenames[catalog_pix]:
#logger.debug("Loading %s"%filenames['mask_1'])
subpix_1,val_1 = ugali.utils.skymap.readSparseHealpixMap(filenames['mask_1'],'MAGLIM',construct_map=False)
#logger.debug("Loading %s"%filenames['mask_2'])
subpix_2,val_2 = ugali.utils.skymap.readSparseHealpixMap(filenames['mask_2'],'MAGLIM',construct_map=False)
subpix = numpy.intersect1d(subpix_1,subpix_2)
superpix = numpy.unique(ugali.utils.skymap.superpixel(subpix,self.nside_pixel,nside))
inside |= numpy.in1d(pixels, superpix)
return inside
def simple_split(config,dirname='split',force=False):
config = Config(config)
filenames = config.getFilenames()
healpix = filenames['pix'].compressed()
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
release = config['data']['release'].lower()
band_1 = config['catalog']['mag_1_band']
band_2 = config['catalog']['mag_2_band']
mangledir = config['mangle']['dirname']
mangle_file_1 = join(mangledir,config['mangle']['filename_1'])
logger.info("Reading %s..."%mangle_file_1)
mangle_1 = healpy.read_map(mangle_file_1)
mangle_file_2 = join(mangledir,config['mangle']['filename_2'])
logger.info("Reading %s..."%mangle_file_2)
mangle_2 = healpy.read_map(mangle_file_2)
basedir,basename = os.path.split(config['mask']['dirname'])
if basename == dirname:
msg = "Input and output directory are the same."
raise Exception(msg)
outdir = mkdir(os.path.join(basedir,dirname))
mask_1 = os.path.basename(config['mask']['basename_1'])
mask_2 = os.path.basename(config['mask']['basename_2'])
for band,mangle,base in [(band_1,mangle_1,mask_1),(band_2,mangle_2,mask_2)]:
maglim = MAGLIMS[release][band]
nside_mangle = healpy.npix2nside(len(mangle))
if nside_mangle != nside_pixel:
msg = "Mangle nside different from pixel nside"
logger.warning(msg)
#raise Exception(msg)
pixels = np.nonzero((mangle>0)&(mangle>maglim))[0]
print len(pixels)
superpix = superpixel(pixels,nside_mangle,nside_catalog)
print healpix
for hpx in healpix:
outfile = join(outdir,base)%hpx
if os.path.exists(outfile) and not force:
logger.warning("Found %s; skipping..."%outfile)
continue
pix = pixels[superpix == hpx]
print hpx, len(pix)
maglims = maglim*np.ones(len(pix))
data = dict(MAGLIM=maglims )
logger.info('Writing %s...'%outfile)
ugali.utils.skymap.writeSparseHealpixMap(pix,data,nside_pixel,outfile)
def split(config,dirname='split',force=False):
""" Take a pre-existing maglim map and divide it into
chunks consistent with the catalog pixels. """
config = Config(config)
filenames = config.getFilenames()
#healpix = filenames['pix'].compressed()
# Check that things are ok
basedir,basename = os.path.split(config['mask']['dirname'])
#if basename == dirname:
# msg = "Input and output directory are the same."
# raise Exception(msg)
outdir = mkdir(os.path.join(basedir,dirname))
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
release = config['data']['release'].lower()
band1 = config['catalog']['mag_1_band']
band2 = config['catalog']['mag_2_band']
# Read the magnitude limits
maglimdir = config['maglim']['dirname']
maglimfile_1 = join(maglimdir,config['maglim']['filename_1'])
logger.info("Reading %s..."%maglimfile_1)
maglim1 = read_map(maglimfile_1)
maglimfile_2 = join(maglimdir,config['maglim']['filename_2'])
logger.info("Reading %s..."%maglimfile_2)
maglim2 = read_map(maglimfile_2)
# Read the footprint
footfile = config['data']['footprint']
logger.info("Reading %s..."%footfile)
footprint = read_map(footfile)
# Output mask names
mask1 = os.path.basename(config['mask']['basename_1'])
mask2 = os.path.basename(config['mask']['basename_2'])
for band,maglim,base in [(band1,maglim1,mask1),(band2,maglim2,mask2)]:
nside_maglim = hp.npix2nside(len(maglim))
if nside_maglim != nside_pixel:
msg = "Mask nside different from pixel nside"
logger.warning(msg)
#raise Exception(msg)
pixels = np.nonzero(maglim>0)[0]
superpix = superpixel(pixels,nside_maglim,nside_catalog)
healpix = np.unique(superpix)
for hpx in healpix:
outfile = join(outdir,base)%hpx
if os.path.exists(outfile) and not force:
logger.warning("Found %s; skipping..."%outfile)
continue
pix = pixels[superpix == hpx]
print(hpx, len(pix))
logger.info('Writing %s...'%outfile)
data = odict()
data['PIXEL']=pix
data['MAGLIM']=maglim[pix].astype('f4')
data['FRACDET']=footprint[pix].astype('f4')
ugali.utils.healpix.write_partial_map(outfile,data,nside_pixel)
def split(config,dirname='split',force=False):
""" Take a pre-existing maglim map and divide it into
chunks consistent with the catalog pixels. """
config = Config(config)
filenames = config.getFilenames()
#healpix = filenames['pix'].compressed()
# Check that things are ok
basedir,basename = os.path.split(config['mask']['dirname'])
#if basename == dirname:
# msg = "Input and output directory are the same."
# raise Exception(msg)
outdir = mkdir(os.path.join(basedir,dirname))
nside_catalog = config['coords']['nside_catalog']
nside_pixel = config['coords']['nside_pixel']
release = config['data']['release'].lower()
band1 = config['catalog']['mag_1_band']
band2 = config['catalog']['mag_2_band']
# Read the magnitude limits
maglimdir = config['maglim']['dirname']
maglimfile_1 = join(maglimdir,config['maglim']['filename_1'])
logger.info("Reading %s..."%maglimfile_1)
maglim1 = read_map(maglimfile_1)
maglimfile_2 = join(maglimdir,config['maglim']['filename_2'])
logger.info("Reading %s..."%maglimfile_2)
maglim2 = read_map(maglimfile_2)
# Read the footprint
footfile = config['data']['footprint']
logger.info("Reading %s..."%footfile)
footprint = read_map(footfile)
# Output mask names
mask1 = os.path.basename(config['mask']['basename_1'])
mask2 = os.path.basename(config['mask']['basename_2'])
for band,maglim,base in [(band1,maglim1,mask1),(band2,maglim2,mask2)]:
nside_maglim = hp.npix2nside(len(maglim))
if nside_maglim != nside_pixel:
msg = "Mask nside different from pixel nside"
logger.warning(msg)
#raise Exception(msg)
pixels = np.nonzero(maglim>0)[0]
superpix = superpixel(pixels,nside_maglim,nside_catalog)
healpix = np.unique(superpix)
for hpx in healpix:
outfile = join(outdir,base)%hpx
if os.path.exists(outfile) and not force:
logger.warning("Found %s; skipping..."%outfile)
continue
pix = pixels[superpix == hpx]
print(hpx, len(pix))
logger.info('Writing %s...'%outfile)
data = odict()
data['PIXEL']=pix
data['MAGLIM']=maglim[pix].astype('f4')
data['FRACDET']=footprint[pix].astype('f4')
ugali.utils.healpix.write_partial_map(outfile,data,nside_pixel)
def run(self):
if 'scan' in self.opts.run:
logger.info("Running 'scan'...")
farm = Farm(self.config, verbose=self.opts.verbose)
farm.submit_all(coords=self.opts.coords,
queue=self.opts.queue,
debug=self.opts.debug)
if 'merge' in self.opts.run:
logger.info("Running 'merge'...")
mergefile = self.config.mergefile
roifile = self.config.roifile
filenames = self.config.likefile.split('_%')[0] + '_*.fits'
infiles = np.array(sorted(glob.glob(filenames)))
if 'mergedir' in self.config['output']:
mkdir(self.config['output']['mergedir'])
pixels = np.char.rpartition(
np.char.rpartition(infiles, '_')[:, 0], '_')[:, -1]
pixels = pixels.astype(int)
superpixel = healpix.superpixel(
pixels, self.config['coords']['nside_likelihood'],
self.config['coords']['nside_merge'])
for pix in np.unique(superpixel):
outfile = mergefile % pix
if exists(outfile) and not self.opts.force:
logger.warn(" Found %s; skipping..." % outfile)
else:
healpix.merge_partial_maps(infiles[superpixel == pix],
outfile,
multiproc=8)
if exists(roifile) and not self.opts.force:
logger.warn(" Found %s; skipping..." % roifile)
else:
logger.info(" Merging likelihood headers...")
healpix.merge_likelihood_headers(infiles, roifile)
if 'tar' in self.opts.run:
logger.info("Running 'tar'...")
outdir = mkdir(self.config['output']['likedir'])
logdir = mkdir(join(outdir, 'log'))
scanfile = self.config.likefile.split('_%')[0] + '_[0-9]*.fits'
tarfile = join(self.config.likefile.split('_%')[0] + '_pixels.tar.gz')
jobname = 'tar'
logfile = os.path.join(logdir, 'scan_tar.log')
cmd = 'tar --remove-files -cvzf %s %s' % (tarfile, scanfile)
if exists(tarfile) and not self.opts.force:
logger.warn(" Found %s; skipping..." % tarfile)
else:
logger.info(" Tarring likelihood files...")
logger.info(cmd)
self.batch.submit(cmd, jobname, logfile)
if 'plot' in self.opts.run:
# WARNING: Loading the full 3D healpix map is memory intensive.
logger.info("Running 'plot'...")
# Should do this in environment variable
import matplotlib
matplotlib.use('Agg')
import pylab as plt
import ugali.utils.plotting as plotting
skymap = ugali.utils.skymap.readSparseHealpixMap(
self.config.mergefile, 'LOG_LIKELIHOOD')[1]
plotting.plotSkymap(skymap)
outdir = mkdir(self.config['output']['plotdir'])
basename = os.path.basename(
self.config.mergefile.replace('.fits', '.png'))
outfile = os.path.join(outdir, basename)
plt.savefig(outfile)
if 'check' in self.opts.run:
# Check the completion fraction
logger.info("Running 'check'...")
import fitsio
import numpy as np
import healpy as hp
from ugali.utils.skymap import inFootprint
# Load the ROI file
roi = fitsio.read(self.config.roifile)
done = roi['PIXEL']
# Get all target pixels
nside = self.config['coords']['nside_likelihood']
pixels = np.arange(hp.nside2npix(nside))
pixarea = hp.nside2pixarea(nside, degrees=True)
foot = pixels[inFootprint(self.config, pixels)]
# And find the pixels that haven't been processed
undone = ~np.in1d(foot, done)
hpxmap = np.zeros(len(pixels))
hpxmap[foot[undone]] = True
logger.info("Found %i incomplete pixels with an area of %.1f deg^2." %
(hpxmap.sum(), hpxmap.sum() * pixarea))
hp.write_map('check.fits.gz', hpxmap)