def get_map_range(hpxmap, pixel=None, nside=None, wrap_angle=180):
""" Calculate the longitude and latitude range for a map. """
check_hpxmap(hpxmap, pixel, nside)
if isinstance(hpxmap, np.ma.MaskedArray):
hpxmap = hpxmap.data
if pixel is None:
nside = hp.get_nside(hpxmap)
pixel = np.arange(len(hpxmap), dtype=int)
ipring, = np.where(np.isfinite(hpxmap) & (hpxmap != hp.UNSEEN))
theta, phi = hp.pix2ang(nside, pixel[ipring])
lon = np.mod(np.degrees(phi), 360)
lat = 90.0 - np.degrees(theta)
# Small offset to add to make sure we get the whole pixel
eps = np.degrees(hp.max_pixrad(nside))
# CHECK ME
hi, = np.where(lon > wrap_angle)
lon[hi] -= 360.0
lon_min = max(np.nanmin(lon) - eps, wrap_angle - 360)
lon_max = min(np.nanmax(lon) + eps, wrap_angle)
lat_min = max(np.nanmin(lat) - eps, -90)
lat_max = min(np.nanmax(lat) + eps, 90)
return (lon_min, lon_max), (lat_min, lat_max)
def __init__(self,
nside=256,
opsimdf=None,
healpixelizedOpSim=None,
preComputedMap=None,
leafsize=50,
fovRadius=1.75,
raCol='ditheredRA',
decCol='ditheredDec'):
"""
nside : int, power of 2, defaults to 256
nside parameter of healpix. determines the size of the tiles
so that there are 12 * nside **2 equally sized tiles covering
the sphere.
"""
self.nside = nside
self.maxradius = hp.max_pixrad(nside=self.nside)
self.npix = hp.nside2npix(nside)
self._tileArea = hp.nside2pixarea(nside)
self.hpOpSim = healpixelizedOpSim
self._preComputedMap = preComputedMap
self.opsimdf = opsimdf
if self.hpOpSim is None and self.preComputedMap is None and self.opsimdf is None:
raise ValueError('hpOpSim and preComputedMap and opsimdf cannot'
' both be None')
self._preComputedEngine = None
self.fovRadius = np.radians(fovRadius)
self.pointingTree = None
if self.opsimdf is not None:
self.pointingTree = PointingTree(self.opsimdf,
raCol=raCol,
decCol=decCol,
leafSize=leafsize,
indexCol='obsHistID')
def calc_signal_spatial(self):
# At the pixel level over the ROI
pix_lon, pix_lat = self.roi.pixels_interior.lon, self.roi.pixels_interior.lat
nside = self.config['coords']['nside_pixel']
#self.angsep_sparse = angsep(self.lon,self.lat,pix_lon,pix_lat)
#self.surface_intensity_sparse = self.kernel.surfaceIntensity(self.angsep_sparse)
if self.kernel.extension < 2 * np.degrees(healpy.max_pixrad(nside)):
#msg = "small kernel"
#msg += ("\n"+str(self.params))
#logger.warning(msg)
idx = self.roi.indexInterior(self.kernel.lon, self.kernel.lat)
self.surface_intensity_sparse = np.zeros(len(pix_lon))
self.surface_intensity_sparse[idx] = 1.0 / self.roi.area_pixel
else:
self.surface_intensity_sparse = self.kernel.pdf(pix_lon, pix_lat)
# On the object-by-object level
#self.angsep_object = angsep(self.lon,self.lat,self.catalog.lon,self.catalog.lat)
#self.surface_intensity_object = self.kernel.surfaceIntensity(self.angsep_object)
self.surface_intensity_object = self.kernel.pdf(
self.catalog.lon, self.catalog.lat)
# Spatial component of signal probability
#u_spatial = self.roi.area_pixel * self.surface_intensity_object
u_spatial = self.surface_intensity_object
return u_spatial
def test_attributes(self):
"""Test basic, default attributes are set on creation
"""
skymap = self.TEST_SKY_MAP
assert skymap.info is None
assert (not skymap.nest)
assert skymap.nside == self.NSIDE
assert skymap.npix == self.NPIX
assert_array_equal(skymap.pindex, numpy.arange(self.NPIX))
assert (not skymap.partial)
assert isinstance(skymap.explicit, dict)
assert len(skymap.explicit) == self.NPIX
assert skymap.resolution.value == healpy.nside2resol(
self.NSIDE, arcmin=True)
assert skymap.resolution.unit == units.arcmin
assert skymap.pixrad.value == healpy.max_pixrad(
self.NSIDE) * units.rad.to("arcmin")
assert skymap.pixrad.unit == units.arcmin
assert skymap.pixarea.value == healpy.nside2pixarea(
self.NSIDE, degrees=True)
assert skymap.pixarea.unit == units.deg ** 2
assert skymap.area.value == skymap.size * skymap.pixarea.value
assert skymap.area.unit == units.deg ** 2
assert isinstance(skymap.directions, numpy.ndarray)
assert_array_equal(skymap.directions.shape, (skymap.npix, 3))
def __call__(self):
#load module
mod = importlib.import_module(self.ctx.params.beam_profile_provider)
#delegate loading of profile
params = self.ctx.params
#setting up angular grid
beam_area = np.radians(params.beam_elevation) * np.radians(
params.beam_azimut) # [rad]
pixel_area = hp.nside2pixarea(params.beam_nside, degrees=False)
pixels = np.floor(np.sqrt(beam_area / pixel_area))
pixels = pixels if pixels % 2 == 1 else pixels + 1
pixel_size = hp.max_pixrad(params.beam_nside)
theta = (np.linspace(0, pixels, pixels * 2 + 1) -
pixels / 2) * (pixel_size)
phi = theta
beam_spec = BeamSpec(phi, theta, pixels**2)
frequencies = np.arange(params.beam_frequency_min,
params.beam_frequency_max,
params.beam_frequency_pixscale)
beam_profiles, beam_norms = mod.load_beam_profile(
beam_spec, frequencies, self.ctx.params)
self.ctx.beam_spec = beam_spec
self.ctx.frequencies = frequencies
self.ctx.beam_profiles = beam_profiles
self.ctx.beam_norms = beam_norms
def __call__(self):
#load module
mod = importlib.import_module(self.ctx.params.beam_profile_provider)
#delegate loading of profile
params = self.ctx.params
#setting up angular grid
beam_area = np.radians(params.beam_elevation) * np.radians(params.beam_azimut) # [rad]
pixel_area = hp.nside2pixarea(params.beam_nside, degrees=False)
pixels = np.floor(np.sqrt(beam_area / pixel_area))
pixels = pixels if pixels%2==1 else pixels+1
pixel_size = hp.max_pixrad(params.beam_nside)
theta = (np.linspace(0, pixels, pixels*2+1)-pixels/2)*(pixel_size)
phi = theta
beam_spec = BeamSpec(phi, theta, pixels**2)
frequencies = np.arange(params.beam_frequency_min, params.beam_frequency_max, params.beam_frequency_pixscale)
beam_profiles, beam_norms = mod.load_beam_profile(beam_spec, frequencies, self.ctx.params)
self.ctx.beam_spec = beam_spec
self.ctx.frequencies = frequencies
self.ctx.beam_profiles = beam_profiles
self.ctx.beam_norms = beam_norms
def get_pix_range(nside, pixel, nest=False, wrap_angle=180):
"""
Calculate the longitude and latitude range for a map.
Parameters
----------
nside : map nside
pixel : pixel index
wrap_angle : map wrapping angle (deg)
Returns
-------
[[lon_min,lon_max],[lat_min,lat_max]] : map range (deg)
"""
lon, lat = hp.pix2ang(nside, pixel, nest=nest, lonlat=True)
# Small offset to add to make sure we get the whole pixel
eps = np.degrees(hp.max_pixrad(nside))
# CHECK ME
# hi,=np.where(lon > wrap_angle)
# lon[hi] -= 360.0
lon_min = max(np.nanmin(lon) - eps, 0) # ,wrap_angle-360)
lon_max = min(np.nanmax(lon) + eps, 360) # ,wrap_angle)
lat_min = max(np.nanmin(lat) - eps, -90)
lat_max = min(np.nanmax(lat) + eps, 90)
return (lon_min, lon_max), (lat_min, lat_max)
def calc_signal_spatial(self):
# At the pixel level over the ROI
pix_lon,pix_lat = self.roi.pixels_interior.lon,self.roi.pixels_interior.lat
nside = self.config['coords']['nside_pixel']
#self.angsep_sparse = angsep(self.lon,self.lat,pix_lon,pix_lat)
#self.surface_intensity_sparse = self.kernel.surfaceIntensity(self.angsep_sparse)
if self.kernel.extension < 2*np.degrees(healpy.max_pixrad(nside)):
#msg = "small kernel"
#msg += ("\n"+str(self.params))
#logger.warning(msg)
idx = self.roi.indexInterior(self.kernel.lon,self.kernel.lat)
self.surface_intensity_sparse = np.zeros(len(pix_lon))
self.surface_intensity_sparse[idx] = 1.0/self.roi.area_pixel
else:
self.surface_intensity_sparse = self.kernel.pdf(pix_lon,pix_lat)
# On the object-by-object level
#self.angsep_object = angsep(self.lon,self.lat,self.catalog.lon,self.catalog.lat)
#self.surface_intensity_object = self.kernel.surfaceIntensity(self.angsep_object)
self.surface_intensity_object = self.kernel.pdf(self.catalog.lon,self.catalog.lat)
# Spatial component of signal probability
#u_spatial = self.roi.area_pixel * self.surface_intensity_object
u_spatial = self.surface_intensity_object
return u_spatial
def draw_pixel(hpxmap, **kwargs):
if isinstance(hpxmap, np.ma.MaskedArray):
pix = np.where(~hpxmap.mask)
else:
pix = np.where((np.isfinite(hpxmap)) & (hpxmap != hp.UNSEEN))
vmin, vmax = np.percentile(hpxmap[pix], [0.5, 99.5])
kwargs.setdefault('vmin', vmin)
kwargs.setdefault('vmax', vmax)
kwargs.setdefault('rasterized', True)
kwargs.setdefault('cmap', 'jet')
nside = hp.npix2nside(len(hpxmap))
pixrad = np.degrees(hp.max_pixrad(nside))
ra, dec = hp.pix2ang(nside, pix, lonlat=True)
xmin, xmax = ra.min() - pixrad, ra.max() + pixrad
ymin, ymax = dec.min() - pixrad, dec.max() + pixrad
delta = 0.01
xx, yy = np.meshgrid(np.arange(xmin, xmax, delta),
np.arange(ymin, ymax, delta))
pp = hp.ang2pix(nside, xx, yy, lonlat=True)
ax = plt.gca()
im = ax.pcolormesh(xx[::-1], yy, hpxmap[pp], **kwargs)
ax.set_xlabel('RA (deg)')
ax.set_ylabel('DEC (deg)')
ax.set_xlim(xmax, xmin)
ax.set_ylim(ymin, ymax)
ax.grid(ls=':', color='black', lw=0.5)
return im
def find_cone_coords(self):
""" """
cone_coords = []
cone_ids = []
scan_radius = np.degrees(hp.max_pixrad(self.cone_nside))
self.logger.info("Finding search pixels:")
for i in tqdm(range(hp.nside2npix(self.cone_nside))):
ra, dec = self.extract_ra_dec(self.cone_nside, i)
ra_rad = np.radians(ra)
dec_rad = np.radians(dec)
if np.logical_and(ra > self.ra_min - scan_radius,
ra < self.ra_max + scan_radius):
if np.logical_and(
dec > self.dec_min - scan_radius,
dec < self.dec_max + scan_radius,
):
cone_coords.append((ra_rad, dec_rad))
cone_ids.append(i)
cone_coords = np.array(cone_coords,
dtype=np.dtype([("ra", float), ("dec", float)]))
return cone_ids, cone_coords
def max_pixrad(testcase):
cs = []
for norder in range(16):
nside = 1 << norder
args = (nside, )
cs.append(dict(args=args, expected=healpy.max_pixrad(*args)))
testcase['max_pixrad'] = cs
def calculate_nside_resolution():
NSIDE = [2**i for i in range(11)]
print('given nside | number of pixels | resolution (pixel size in degree) | Maximum angular distance (degree) | pixel area (in square degrees)')
for nside in NSIDE:
npix = hp.nside2npix(nside)
resol = np.rad2deg(hp.nside2resol(nside))
maxrad = np.rad2deg(hp.max_pixrad(nside))
pixarea = hp.nside2pixarea(nside, degrees=True)
print('{0:^11} | {1:^16} | {2:^33.4f} | {3:^33.4f} | {4:^30.6f}'.format(nside, npix, resol, maxrad, pixarea))
def subpixel(superpix, nside_superpix, nside_subpix):
"""
Return the indices of sub-pixels (resolution nside_subpix) within the super-pixel with (resolution nside_superpix).
"""
if nside_superpix==nside_subpix: return superpix
vec = hp.pix2vec(nside_superpix, superpix)
radius = np.degrees(2. * hp.max_pixrad(nside_superpix))
subpix = query_disc(nside_subpix, vec, radius)
pix_for_subpix = superpixel(subpix,nside_subpix,nside_superpix)
# Might be able to speed up array indexing...
return subpix[pix_for_subpix == superpix]
def pixrad(self):
"""Angular size of a pixel in this `SkyMap`
Returns the maximum angular distance (arcminutes) between any pixel
center and its corners
This returns an instance of `~astropy.units.Quantity` with explicit
units, which can then be converted by the user as-needed.
"""
radius = healpy.max_pixrad(self.nside) * units.Unit("rad")
return radius.to("arcmin")
def setUp(self):
if not healpy:
self.skipTest("Missing healpy dependency.")
self.config = HealpixSkyMap.ConfigClass()
nside = 2**self.config.log2NSide
self._NumTracts = healpy.nside2npix(nside) # Number of tracts to expect
self._NeighborAngularSeparation = healpy.max_pixrad(
nside) * afwGeom.radians # Expected tract separation
self._SkyMapClass = HealpixSkyMap # Class of SkyMap to test
self._SkyMapName = "healpix" # Name of SkyMap class to test
self._numNeighbors = 1 # Number of neighbours
def subpixel(superpix, nside_superpix, nside_subpix):
"""
Return the indices of sub-pixels (resolution nside_subpix) within the super-pixel with (resolution nside_superpix).
"""
if nside_superpix == nside_subpix:
return superpix
vec = healpy.pix2vec(nside_superpix, superpix)
radius = np.degrees(2.0 * healpy.max_pixrad(nside_superpix))
subpix = query_disc(nside_subpix, vec, radius)
pix_for_subpix = superpixel(subpix, nside_subpix, nside_superpix)
# Might be able to speed up array indexing...
return subpix[pix_for_subpix == superpix]
def calculate_nside_resolution():
NSIDE = [2**i for i in range(11)]
print(
'given nside | number of pixels | resolution (pixel size in degree) | Maximum angular distance (degree) | pixel area (in square degrees)'
)
for nside in NSIDE:
npix = hp.nside2npix(nside)
resol = np.rad2deg(hp.nside2resol(nside))
maxrad = np.rad2deg(hp.max_pixrad(nside))
pixarea = hp.nside2pixarea(nside, degrees=True)
print(
'{0:^11} | {1:^16} | {2:^33.4f} | {3:^33.4f} | {4:^30.6f}'.format(
nside, npix, resol, maxrad, pixarea))
def subpixel(superpix, nside_superpix, nside_subpix):
"""
Return the indices of sub-pixels (resolution nside_subpix) within
the super-pixel with (resolution nside_superpix).
ADW: It would be better to convert to next and do this explicitly
"""
if nside_superpix == nside_subpix: return superpix
vec = hp.pix2vec(nside_superpix, superpix)
radius = np.degrees(2. * hp.max_pixrad(nside_superpix))
subpix = query_disc(nside_subpix, vec, radius)
pix_for_subpix = superpixel(subpix, nside_subpix, nside_superpix)
# Might be able to speed up array indexing...
return subpix[pix_for_subpix == superpix]
def setUp(self):
if not healpy:
self.skipTest("Missing healpy dependency.")
config = HealpixSkyMap.ConfigClass()
nside = 2**config.log2NSide
self.setAttributes(
SkyMapClass=HealpixSkyMap,
name="healpix",
config=config,
numTracts=healpy.nside2npix(nside),
numNeighbors=1,
neighborAngularSeparation=healpy.max_pixrad(nside) * geom.radians,
)
def setUp(self):
if not healpy:
self.skipTest("Missing healpy dependency.")
config = HealpixSkyMap.ConfigClass()
nside = 2**config.log2NSide
self.setAttributes(
SkyMapClass=HealpixSkyMap,
name="healpix",
config=config,
numTracts=healpy.nside2npix(nside),
numNeighbors=1,
neighborAngularSeparation=healpy.max_pixrad(nside) * geom.radians,
)
def _setup_subpix(self,nside=2**16):
"""
Subpixels for random position generation.
"""
# Only setup once...
if hasattr(self,'subpix'): return
# Simulate over full ROI
self.roi_radius = self.config['coords']['roi_radius']
# Setup background spatial stuff
logger.info("Setup subpixels...")
self.nside_pixel = self.config['coords']['nside_pixel']
self.nside_subpixel = self.nside_pixel * 2**4 # Could be config parameter
epsilon = np.degrees(healpy.max_pixrad(self.nside_pixel)) # Pad roi radius to cover edge healpix
subpix = ugali.utils.healpix.query_disc(self.nside_subpixel,self.roi.vec,self.roi_radius+epsilon)
superpix = ugali.utils.healpix.superpixel(subpix,self.nside_subpixel,self.nside_pixel)
self.subpix = subpix[np.in1d(superpix,self.roi.pixels)]
def _setup_subpix(self,nside=2**16):
"""
Subpixels for random position generation.
"""
# Only setup once...
if hasattr(self,'subpix'): return
# Simulate over full ROI
self.roi_radius = self.config['coords']['roi_radius']
# Setup background spatial stuff
logger.info("Setup subpixels...")
self.nside_pixel = self.config['coords']['nside_pixel']
self.nside_subpixel = self.nside_pixel * 2**4 # Could be config parameter
epsilon = np.degrees(healpy.max_pixrad(self.nside_pixel)) # Pad roi radius to cover edge healpix
subpix = ugali.utils.healpix.query_disc(self.nside_subpixel,self.roi.vec,self.roi_radius+epsilon)
superpix = ugali.utils.healpix.superpixel(subpix,self.nside_subpixel,self.nside_pixel)
self.subpix = subpix[np.in1d(superpix,self.roi.pixels)]
def calc_surface_intensity(self, factor=10):
"""Calculate the surface intensity for each pixel in the interior
region of the ROI. Pixels are adaptively subsampled around the
kernel centroid out to a radius of 'factor * max_pixrad'.
Parameters:
-----------
factor : the radius of the oversample region in units of max_pixrad
Returns:
--------
surface_intensity : the surface intensity at each pixel
"""
# First we calculate the surface intensity at native resolution
pixels = self.roi.pixels_interior
nside_in = self.config['coords']['nside_pixel']
surface_intensity = self.kernel.pdf(pixels.lon, pixels.lat)
# Then we recalculate the surface intensity around the kernel
# centroid at higher resolution
for i in np.arange(1, 5):
# Select pixels within the region of interest
nside_out = 2**i * nside_in
radius = factor * np.degrees(hp.max_pixrad(nside_out))
pix = ang2disc(nside_in,
self.kernel.lon,
self.kernel.lat,
radius,
inclusive=True)
# Select pix within the interior region of the ROI
idx = ugali.utils.healpix.index_pix_in_pixels(pix, pixels)
pix = pix[(idx >= 0)]
idx = idx[(idx >= 0)]
# Reset the surface intensity for the subsampled pixels
subpix = ugali.utils.healpix.ud_grade_ipix(pix, nside_in,
nside_out)
pix_lon, pix_lat = pix2ang(nside_out, subpix)
surface_intensity[idx] = np.mean(self.kernel.pdf(pix_lon, pix_lat),
axis=1)
return surface_intensity
def plot(self, observatory=None):
"""Plots the observed pointings.
Parameters
----------
observatory : str
Plot only the points for that observatory. Otherwise, plots all
the pointings.
"""
color_cycler = cycler.cycler(
bgcolor=['b', 'r', 'g', 'y', 'm', 'c', 'k'])
fig, ax = get_axes(projection='mollweide')
data = self.schedule[self.schedule['ra'] > 0.]
if observatory:
data = data[data['observatory'] == observatory]
for ii, sty in zip(range(len(self.targets)),
itertools.cycle(color_cycler)):
target = self.targets[ii]
name = target.name
nside = target._get_nside(ifu=self.ifu)
radius = healpy.max_pixrad(nside, degrees=True)
target_data = data[data['target'] == name]
plot_ellipse(ax,
target_data['ra'],
target_data['dec'],
width=radius,
origin=__MOLLWEIDE_ORIGIN__,
**sty)
if observatory is not None:
ax.set_title(f'Observatory: {observatory}')
return fig
def calc_signal_spatial(self):
"""
Calculate the spatial signal probability for each catalog object.
Parameters:
-----------
None
Returns:
--------
u_spatial : array of spatial probabilities per object
"""
# At the pixel level over the ROI
pix_lon, pix_lat = self.roi.pixels_interior.lon, self.roi.pixels_interior.lat
nside = self.config['coords']['nside_pixel']
#self.angsep_sparse = angsep(self.lon,self.lat,pix_lon,pix_lat)
#self.surface_intensity_sparse = self.kernel.surfaceIntensity(self.angsep_sparse)
# For small kernels, use the single pixel where they reside; otherwise, calculate over roi
if self.kernel.extension < 2 * np.degrees(healpy.max_pixrad(nside)):
#msg = "small kernel"
#msg += ("\n"+str(self.params))
#logger.warning(msg)
idx = self.roi.indexInterior(self.kernel.lon, self.kernel.lat)
self.surface_intensity_sparse = np.zeros(len(pix_lon))
self.surface_intensity_sparse[idx] = 1.0 / self.roi.area_pixel
else:
self.surface_intensity_sparse = self.kernel.pdf(pix_lon, pix_lat)
# On the object-by-object level
#self.angsep_object = angsep(self.lon,self.lat,self.catalog.lon,self.catalog.lat)
#self.surface_intensity_object = self.kernel.surfaceIntensity(self.angsep_object)
self.surface_intensity_object = self.kernel.pdf(
self.catalog.lon, self.catalog.lat)
# Spatial component of signal probability
#u_spatial = self.roi.area_pixel * self.surface_intensity_object
u_spatial = self.surface_intensity_object
return u_spatial
def calc_surface_intensity(self, factor=10):
"""Calculate the surface intensity for each pixel in the interior
region of the ROI. Pixels are adaptively subsampled around the
kernel centroid out to a radius of 'factor * max_pixrad'.
Parameters:
-----------
factor : the radius of the oversample region in units of max_pixrad
Returns:
--------
surface_intensity : the surface intensity at each pixel
"""
# First we calculate the surface intensity at native resolution
pixels = self.roi.pixels_interior
nside_in = self.config['coords']['nside_pixel']
surface_intensity = self.kernel.pdf(pixels.lon,pixels.lat)
# Then we recalculate the surface intensity around the kernel
# centroid at higher resolution
for i in np.arange(1,5):
# Select pixels within the region of interest
nside_out = 2**i * nside_in
radius = factor*np.degrees(hp.max_pixrad(nside_out))
pix = ang2disc(nside_in,self.kernel.lon,self.kernel.lat,
radius,inclusive=True)
# Select pix within the interior region of the ROI
idx = ugali.utils.healpix.index_pix_in_pixels(pix,pixels)
pix = pix[(idx >= 0)]; idx = idx[(idx >= 0)]
# Reset the surface intensity for the subsampled pixels
subpix = ugali.utils.healpix.ud_grade_ipix(pix,nside_in,nside_out)
pix_lon,pix_lat = pix2ang(nside_out,subpix)
surface_intensity[idx]=np.mean(self.kernel.pdf(pix_lon,pix_lat),axis=1)
return surface_intensity
def main(argv=None):
""" Main Function """
# Call initial parser from init_utils
parser = ap.ArgumentParser(
description="""Create HDF5 Astrometry file.""", add_help=True
)
parser.add_argument(
"-sdir",
"--shotdir",
help="""Directory for shot H5 files to ingest""",
type=str,
default="/data/05350/ecooper/hdr2.1/reduction/data",
)
parser.add_argument(
"-sl",
"--shotlist",
help="""Text file of DATE OBS list""",
type=str,
default="hdr2.1.shotlist",
)
parser.add_argument(
"-of",
"--outfilename",
type=str,
help="""Relative or absolute path for output HDF5
file.""",
default=None,
)
parser.add_argument("-survey", "--survey", type=str, default="hdr2.1")
args = parser.parse_args(argv)
args.log = setup_logging()
fileh = tb.open_file(
args.outfilename, mode="w", title=args.survey.upper() + " Fiber Index file "
)
shotlist = Table.read(
args.shotlist, format="ascii.no_header", names=["date", "obs"]
)
tableFibers = fileh.create_table(
fileh.root,
"FiberIndex",
VIRUSFiberIndex,
"Survey Fiber Coord Info",
expectedrows=140369264,
)
# set up HEALPIX options
Nside = 2 ** 15
hp.max_pixrad(Nside, degrees=True) * 3600 # in unit of arcsec
config = HDRconfig(survey=args.survey)
badshot = np.loadtxt(config.badshot, dtype=int)
for shotrow in shotlist:
datevshot = str(shotrow["date"]) + "v" + str(shotrow["obs"]).zfill(3)
shotid = int(str(shotrow["date"]) + str(shotrow["obs"]).zfill(3))
date = shotrow["date"]
try:
args.log.info("Ingesting %s" % datevshot)
file_obs = tb.open_file(op.join(args.shotdir, datevshot + ".h5"), "r")
tableFibers_i = file_obs.root.Data.FiberIndex
for row_i in tableFibers_i:
row_main = tableFibers.row
for col in tableFibers_i.colnames:
row_main[col] = row_i[col]
fiberid = row_i["fiber_id"]
try:
row_main["healpix"] = hp.ang2pix(
Nside, row_i["ra"], row_i["dec"], lonlat=True)
except:
row_main["healpix"] = 0
row_main["shotid"] = shotid
row_main["date"] = date
row_main["datevobs"] = datevshot
row_main["specid"] = fiberid[20:23]
row_main["ifuslot"] = fiberid[24:27]
row_main["ifuid"] = fiberid[28:31]
row_main["amp"] = fiberid[32:34]
row_main.append()
file_obs.close()
except:
if shotid in badshot:
pass
else:
args.log.error("could not ingest %s" % datevshot)
tableFibers.cols.healpix.create_csindex()
tableFibers.cols.ra.create_csindex()
tableFibers.flush()
fileh.close()
def plotTilePointings(self,
tileID,
raCol='ditheredRA',
decCol='ditheredDec',
radius=1.75,
paddingFactors=1,
query=None,
ax=None,
projection='cyl',
tile_centers=None,
corners=None,
drawPointings=True,
**kwargs):
"""
Plot the Healpix Tile and the maximal set of pointings overlapping with
it.
Parameters
----------
tileID : int, mandatory
Healpix tileID in the nested scheme
raCol : string, defaults to 'ditheredRA'
column name with ra that should be used
decCol : string, defaults to 'ditheredDec'
column name with dec that should be used
radius : float, defaults to 1.75, degrees
radius of the field of view
paddingFactors: float, defaults to 1.0
controls the size of the figure wrt angular dimensions of the tile.
paddingFactors=1 is designed to get all the centers of pointings
overlapping the tile inside the figure
query : string
query for the pandas dataframe to select only some of the observations
ax : instance of matplotlib.figure.axes
axes for figure. New figure created insitue if not provided
projections: string, defaults to `cyl`
string to specify the `Basemap.projection`
tile_centers : tuples of 2 floats, degrees, defaults to None
(ra of the center of the figure, dec of the center of the figure)
in degrees
if None, the center is set by using the center of the Healpixel of
tileID and `self.nside`
corners: tuple of floats, defaults to None
kwargs: passed to the plotting of the the field of veiw
drawPointings : Bool, defaults to True
if False, draws only Tiles
Returns
-------
fig, tile_centers, corners
..notes: Have not thought about wrapover
"""
# if axes object not provided setup figure
if ax is None:
fig, ax = plt.subplots()
# size of box
padding = np.degrees(hp.max_pixrad(self.nside)) + radius
# center of the figure
if tile_centers is None:
ra_tile = self.tileCenter(tileID)[0][0]
dec_tile = self.tileCenter(tileID)[1][0]
else:
ra_tile, dec_tile = tile_centers
# corner of the figure
if corners is not None:
llcrnrlat, llcrnrlon, urcrnrlat, urcrnrlon = corners
else:
llcrnrlat = dec_tile - padding * paddingFactors
urcrnrlat = dec_tile + padding * paddingFactors
llcrnrlon = ra_tile - padding * paddingFactors
urcrnrlon = ra_tile + padding * paddingFactors
# Instantiate basemap
m = Basemap(llcrnrlat=llcrnrlat,
llcrnrlon=llcrnrlon,
urcrnrlat=urcrnrlat,
urcrnrlon=urcrnrlon,
projection=projection,
lon_0=ra_tile,
lat_0=dec_tile,
ax=ax)
# Draw some parallels and meridians to get a spatial sense
parallels = np.linspace(llcrnrlat, urcrnrlat, 3)
meridians = np.linspace(llcrnrlon, urcrnrlon, 3)
m.drawparallels(parallels, labels=(1, 0, 0, 0))
m.drawmeridians(meridians, labels=(0, 1, 1, 1))
# obtain the boundaries of the Healpixels and create the polygon patch
lon, lat = healpix_boundaries(tileID,
nside=self.nside,
units='degrees',
convention='celestial',
step=10,
nest=True)
x, y = m(lon, lat)
xy = zip(x, y)
healpixels = Polygon(xy,
facecolor='w',
fill=False,
alpha=1.,
edgecolor='k',
lw=2)
if drawPointings:
# Obtain the centers of pointings
ra, dec = self.pointingCenters(tileID,
raCol=raCol,
decCol=decCol,
query=query)
for ra, dec in zip(ra, dec):
m.tissot(ra, dec, radius, 100, **kwargs)
# Draw patch
ax.add_patch(healpixels)
# really important for the case where ax is None
if ax is not None:
fig = ax.figure
return fig, (ra_tile, dec_tile), (llcrnrlat, llcrnrlon, urcrnrlat,
urcrnrlon)
def gaia_photometry(catfile,nside=64,band=None,plot=False,version='edr3'):
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
#columns = [OBJECT_ID,'RA','DEC']
columns = ['RA','DEC']
spread,nepochs = ['WAVG_SPREAD_MODEL_R','NEPOCHS_R']
mag_g,mag_r,mag_i,mag_z = bfields(['MAG_PSF'],['g','r','i','z'])
#mag_g,mag_r,mag_i,mag_z = bfields(['WAVG_MAG_PSF'],['g','r','i','z'])
columns += [spread, nepochs, mag_g, mag_r, mag_i, mag_z]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
radius = np.degrees(hp.max_pixrad(NSIDE))
msg = '%s (RA,DEC,RAD) = %.2f,%.2f,%.2f'%(os.path.basename(catfile),ra,dec,radius)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < r < 20 and 0.0 < (g-i) < 1.5
sel = (np.fabs(cat[spread])<0.002) & \
(cat[mag_g]<90) & (cat[mag_r]<90) & (cat[mag_i]<90) & (cat[mag_z]<90) & \
(cat[mag_r]>16) & (cat[mag_r]<20) & \
((cat[mag_g] - cat[mag_i]) > 0.0) & \
((cat[mag_g] - cat[mag_i]) < 1.5) & \
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
#msg = "Getting external catalog: %s"%catalog
ext = get_gaia_catalog(hpx,version=version)
m = match_query(cat['RA'],cat['DEC'],ext['RA'],ext['DEC'])
# Use a fairly wide matching radius (2 arcsec)
cut = 1.0
sel = m[-1]*3600. < cut
cat_match = cat[m[0][sel]]
ext_match = ext[m[1][sel]]
cat_G = gaia_transform(cat_match[mag_g],cat_match[mag_r],cat_match[mag_i],cat_match[mag_z],
version=version)
# Need to put Gaia flux onto the AB system
ext_G = -2.5 * np.log10(ext_match['PHOT_G_MEAN_FLUX']) + 25.7934
diff = cat_G - ext_G
pix = ang2pix(nside,cat_match['RA'],cat_match['DEC'])
upix = np.unique(pix)
stat = nd.median(diff,labels=pix,index=upix)
if False:
plt.figure()
plt.hist(cat_G - ext_G)
import pdb; pdb.set_trace()
return upix,stat
def external_astrometry(catfile,catalog='GAIA',nside=64,band='r',plot=False):
#nice = os.nice(0)
#os.nice(10-nice)
if band=='all': band = 'r'
if not os.path.exists(catfile):
msg = "Couldn't find %s"%catfile
raise Exception(msg)
columns = [OBJECT_ID,'RA','DEC']
spread,mag,nepochs = bfields(['WAVG_SPREAD_MODEL','MAG_PSF','NEPOCHS'],band)
columns += [spread,mag,nepochs]
# Hack to get pixel location
hpx = int(catfile.split('_')[-1].split('.')[0])
#hpx = ang2pix(NSIDE, cat['RA'], cat['DEC'])
ra,dec = pix2ang(NSIDE, hpx)
radius = np.degrees(healpy.max_pixrad(NSIDE))
msg = '%s (RA,DEC,RAD) = %.2f,%.2f,%.2f'%(os.path.basename(catfile),ra,dec,radius)
print(msg)
#print "Getting coadd catalog: DES"
cat = load_infiles([catfile],columns)
# Select stars with 16 < mag < 21
sel = (np.fabs(cat[spread])<0.002) & \
(cat[mag]>16) & \
(cat[mag]<21) & \
(cat[nepochs] > 1)
cat = cat[sel]
if len(cat) == 0:
msg = "WARNING: No objects passing selection in: %s"%catfile
print(msg)
return np.array([],dtype=int), np.array([])
#print "Getting external catalog: %s"%catalog
if catalog in list(CATALOGS.keys()):
ext = get_vizier_catalog(ra,dec,radius,**CATALOGS[catalog])
else:
ext = get_local_catalog(ra,dec,radius,catalog)
m = match_query(cat['RA'],cat['DEC'],ext['_RAJ2000'],ext['_DEJ2000'])
# Use a fairly wide matching radius (2 arcsec)
cut = 2.0
sel = m[-1]*3600. < cut
sepdeg = m[-1][sel]
sepsec = m[-1][sel] * 3600.
sepmas = sepsec * 1000.
sep = sepmas
pix = ang2pix(nside,cat['RA'][sel],cat['DEC'][sel])
upix = np.unique(pix)
try:
peak = nd.median(sep,labels=pix,index=upix)
except ValueError:
import pdb; pdb.set_trace()
if plot:
plt.figure()
draw_angsep(sep,bins=np.linspace(0,cut*1000.,101))
if isinstance(plot,basestring):
outfile = plot
plt.savefig(outfile,bbox_inches='tight')
#return cat,ext,m
return upix,peak
def main():
# parse command-line arguments
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', action='store_true',
help='more verbose output')
parser.add_argument('--order', type=int, default=5,
help='order of healpix hierachical tesselation')
parser.add_argument('--input', type=str, default=None,
help='input file name')
args = parser.parse_args()
nside = 2**args.order
npix = hp.nside2npix(nside)
m = np.arange(npix)
print 'HEALPix map info:'
print '\tn pixels:', npix
print '\tresolution:', hp.nside2resol(nside)
print '\tpixel area:', hp.nside2pixarea(nside)
print '\tmax distance from center:', hp.max_pixrad(nside)
print 'xi binning info:'
transerve_comoving_scale = 3820.43 # Mpc, at z = 2.1
maxang = 200/transerve_comoving_scale
print '\tmax ang scale:', maxang
print 'test stuff:'
test_theta = 1.58853
test_phi = 0.154
testvec = hp.ang2vec(test_theta, test_phi)
testmax = 0.03
print '\ttheta, phi:', test_theta, test_phi
print '\t3d vec:', testvec
print '\tpixel index:', hp.ang2pix(nside, test_theta, test_phi)
print '\tclosest neighbors:', hp.pixelfunc.get_all_neighbours(nside, test_theta, test_phi)
print '\tneighbors within max ang scale:', hp.query_disc(nside, testvec, testmax, inclusive=True)
# include_pix = hp.query_disc(nside, hp.ang2vec(5./6.*np.pi/2, 7./6.*np.pi), 2*maxang, inclusive=True)
# print '\tn pixels for test:', len(include_pix)
qso_pos = np.loadtxt(args.input)
print 'Shape of input data: ', qso_pos.shape
m = {}
qso_pos[:,0] = np.deg2rad(qso_pos[:,0])
qso_pos[:,1] = np.deg2rad(90-qso_pos[:,1])
class Quasar():
def __init__(self, theta, phi, z):
self.theta = theta
self.phi = phi
self.z = z
# fake this for now
self.wave = np.log10((1+z)*(1100+np.arange(10)))
quasars = []
for i, qso in enumerate(qso_pos):
ra, dec, z = qso
phi = ra
theta = dec
q = Quasar(theta, phi, z)
map_index = hp.ang2pix(nside, theta, phi)
# if map_index not in include_pix:
# continue
#q.map_index = map_index
if map_index in m:
m[map_index].append(i)
else:
m[map_index] = [i]
q.i = i
quasars.append(q)
print 'Number of pixels with at least one quasar: %d' % (len(m.keys()))
print 'Average number of quasars per pixel: %f' % (float(len(quasars))/len(m.keys()))
nearby_pixels_sum = 0
nearby_quasars_sum = 0
for q_index, q in enumerate(quasars):
nearby_pixels = hp.query_disc(nside, hp.ang2vec(q.theta, q.phi), maxang, inclusive=True)
nearby_pixels_sum += len(nearby_pixels)
for nearby_pixel in nearby_pixels:
if nearby_pixel in m:
nearby_quasars = m[nearby_pixel]
nearby_quasars_sum += len(nearby_quasars)
p_phi = qso_pos[nearby_quasars,0]
p_theta = qso_pos[nearby_quasars,1]
angdist = np.sin(q.theta)*np.sin(p_theta) + np.cos(q.theta)*np.cos(p_theta)*(np.cos(q.phi)*np.cos(p_phi) + np.sin(q.phi)*np.sin(p_phi))
for p_i, p_index in enumerate(nearby_quasars):
theta, phi, z = qso_pos[p_index]
p = Quasar(theta, phi, z)
loglamdist = abs(np.subtract.outer(q.wave, p.wave))
p_angdist = angdist[p_i]
#losdist = abs(np.log(q_wave/p.wave))
print 'Average number of pixels within 200 Mpc of a quasar: %f' % (float(nearby_pixels_sum)/len(quasars))
print 'Average number of quasars within nearby pixels of a quasar: %f' % (float(nearby_quasars_sum)/len(quasars))
mask = np.ones(npix).astype(np.bool)
mask[m.keys()] = False
mm = hp.ma(np.zeros(npix))
mm.mask = mask
mm[~mm.mask] = [len(m[i]) for i in m.keys()]
hp.mollview(mm.filled(), rot=(270,0,0), flip='geo')
plt.show()
def __init__(self, config, lon, lat):
self.config = Config(config)
self.lon = lon
self.lat = lat
self.projector = ugali.utils.projector.Projector(self.lon, self.lat)
self.vec = vec = ang2vec(self.lon, self.lat)
self.pix = ang2pix(self.config['coords']['nside_likelihood'], self.lon,
self.lat)
# Pixels from the entire ROI disk
pix = query_disc(self.config['coords']['nside_pixel'], vec,
self.config['coords']['roi_radius'])
self.pixels = PixelRegion(self.config['coords']['nside_pixel'], pix)
# Pixels in the interior region
pix = query_disc(self.config['coords']['nside_pixel'], vec,
self.config['coords']['roi_radius_interior'])
self.pixels_interior = PixelRegion(
self.config['coords']['nside_pixel'], pix)
# Pixels in the outer annulus
pix = query_disc(self.config['coords']['nside_pixel'], vec,
self.config['coords']['roi_radius_annulus'])
pix = np.setdiff1d(self.pixels, pix)
self.pixels_annulus = PixelRegion(self.config['coords']['nside_pixel'],
pix)
# Pixels within target healpix region
pix = ugali.utils.skymap.subpixel(
self.pix, self.config['coords']['nside_likelihood'],
self.config['coords']['nside_pixel'])
self.pixels_target = PixelRegion(self.config['coords']['nside_pixel'],
pix)
# Boolean arrays for selecting given pixels
# (Careful, this works because pixels are pre-sorted by query_disc before in1d)
self.pixel_interior_cut = np.in1d(self.pixels, self.pixels_interior)
self.pixel_annulus_cut = np.in1d(self.pixels, self.pixels_annulus)
# Some pixel properties
self.area_pixel = hp.nside2pixarea(
self.config.params['coords']['nside_pixel'], degrees=True) # deg^2
self.max_pixrad = np.degrees(
hp.max_pixrad(self.config['coords']['nside_pixel'])) # deg
# ADW: These are really bin edges, should be careful and consistent
# It would be cleaner to separate the CMD from ROI
self.bins_mag = np.linspace(self.config.params['mag']['min'],
self.config.params['mag']['max'],
self.config.params['mag']['n_bins'] + 1)
self.bins_color = np.linspace(
self.config.params['color']['min'],
self.config.params['color']['max'],
self.config.params['color']['n_bins'] + 1)
self.centers_mag = ugali.utils.binning.centers(self.bins_mag)
self.centers_color = ugali.utils.binning.centers(self.bins_color)
self.delta_mag = self.bins_mag[1] - self.bins_mag[0]
self.delta_color = self.bins_color[1] - self.bins_color[0]
def Hopkins_method(ts_map_array, ra, dec, fdr, psf_size, N, nside, hopkins_correction=True):
# Compute the number of correlated pixels, i.e., the number of pixels within a PSF distance
# NOTE: query_disc gets the *radius* of the disc as input, not the diameter, so we can directly use psf_size
n_correlated_pixels = len(hp.query_disc(nside, (0, 0, 1), np.deg2rad(psf_size), inclusive=True, fact=4))
print("Got a map with nside %i. Using a number of correlated pixels of %i" % (nside, n_correlated_pixels))
# This number should be close to psf_size (within ~1 deg)
approx_corr_rad = np.sqrt(n_correlated_pixels / np.pi) * np.rad2deg(hp.max_pixrad(nside))
print("Approx. correlation radius (should be within 1 deg of PSF size): %.2f deg" % approx_corr_rad)
# This notation follows Clements, Sarkar and Guo (2012):
# https://web.njit.edu/~wguo/Clements_Sarkar_Guo%202012.pdf
# Get the corresponding p-values
# We use here the result from Mattox et al. 1996, that TS is distributed as 0.5 chi2(1 dof)
p_values = 0.5 * scipy.stats.chi2(1).sf(ts_map_array) # type: np.ndarray
# Order the p-values from the smallest to the largest. We use argsort to keep track of the reordering
sort_idx = p_values.argsort()
# Find
# kBH = max
# {j: P(j) <= j alpha / N}
alpha_over_N = fdr / N
# Hopkins adjustement
if hopkins_correction:
print("Using Hopkins correction")
C_N = np.sum(1.0 / np.arange(1, n_correlated_pixels+1))
else:
C_N = 1
threshold = np.arange(1, N+1) * alpha_over_N / C_N
print("Minimum threshold: %.2g, maximum threshold: %.2g" % (threshold.min(), threshold.max()))
idx = (p_values[sort_idx] <= threshold)
# Find the largest p-value satisfying the condition
if np.sum(idx) == 0:
# No points above the threshold
return [], [], []
else:
# Detections
# Find out the last point above the threshold, all points before in the sorted p_values array
# are considered detection
rightmost_p_value_below_threshold_idx = idx.nonzero()[0].max()
points_to_keep_idx = sort_idx[:rightmost_p_value_below_threshold_idx+1]
return ra[points_to_keep_idx], dec[points_to_keep_idx], ts_map_array[points_to_keep_idx]
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
primtargs : :class:`~numpy.ndarray`
An array of primary targets.
scndout : :class`~numpy.ndarray`
Name of a sub-directory to which to write the information in
`desitarget.secondary.outdatamodel` with `TARGETID` and (the
highest) `PRIORITY_INIT` updated with matching primary info.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
pix : :class:`list`, optional, defaults to `None`
Limit secondary targets to (NESTED) HEALpixels that touch
pix at the supplied `nside`, as a speed-up.
nside : :class:`int`, optional, defaults to `None`
The (NESTED) HEALPixel nside to be used with `pixlist`.
Returns
-------
:class:`~numpy.ndarray`
The array of primary targets, with the `SCND_TARGET` bit
populated for matches to secondary targets
"""
# ADM add a SCND_TARGET column to the primary targets.
dt = primtargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(primtargs), dtype=dt)
for col in primtargs.dtype.names:
targs[col] = primtargs[col]
# ADM check if this is an SV or main survey file.
cols, mx, surv = main_cmx_or_sv(targs, scnd=True)
log.info('running on the {} survey...'.format(surv))
if surv != 'main':
scxdir = os.path.join(scxdir, surv)
# ADM read in non-OVERRIDE secondary targets.
scxtargs = read_files(scxdir, mx[3])
scxtargs = scxtargs[~scxtargs["OVERRIDE"]]
# ADM match primary targets to non-OVERRIDE secondary targets.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if nside is not None and pix is not None:
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(primtargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(primtargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(scndout, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
desi_mask = desimasks[0]
targs[desicols[0]][umtargs] |= desi_mask.SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# APC Secondary target bits only affect PRIORITY, NUMOBS and
# APC obsconditions for specific DESI_TARGET bits
# APC See https://github.com/desihub/desitarget/pull/530
# APC Only consider primary targets with secondary bits set
scnd_update = (targs[desicols[0]] & desi_mask['SCND_ANY']) != 0
if np.any(scnd_update):
# APC Allow changes to primaries if the DESI_TARGET bitmask has
# APC only the following bits set, in any combination.
log.info(
'Testing if secondary targets can update {} matched primaries'.
format(scnd_update.sum()))
update_from_scnd_bits = desi_mask['SCND_ANY'] | desi_mask[
'MWS_ANY'] | desi_mask['STD_BRIGHT'] | desi_mask[
'STD_FAINT'] | desi_mask['STD_WD']
scnd_update &= ((targs[desicols[0]] & ~update_from_scnd_bits) == 0)
log.info(
'Setting new priority, numobs and obsconditions from secondary for {} matched primaries'
.format(scnd_update.sum()))
# APC Primary and secondary obsconditions are or'd
scnd_obscon = set_obsconditions(targs[scnd_update], scnd=True)
targs['OBSCONDITIONS'][scnd_update] &= scnd_obscon
# APC bit of a hack here
# APC Check for _BRIGHT, _DARK split in column names
darkbright = 'NUMOBS_INIT_DARK' in targs.dtype.names
if darkbright:
ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"]
else:
ender, obscon = [""], [
"DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18"
]
# APC secondaries can increase priority and numobs
for edr, oc in zip(ender, obscon):
pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr
scnd_priority, scnd_numobs = initial_priority_numobs(
targs[scnd_update], obscon=oc, scnd=True)
targs[nc][scnd_update] = np.maximum(targs[nc][scnd_update],
scnd_numobs)
targs[pc][scnd_update] = np.maximum(targs[pc][scnd_update],
scnd_priority)
# ADM update the secondary targets with the primary information.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM the maximum priority will be used to break ties in the
# ADM unlikely event that a secondary matches two primaries.
hipri = np.maximum(targs["PRIORITY_INIT_DARK"],
targs["PRIORITY_INIT_BRIGHT"])
scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs]
# ADM write the secondary targets that have updated TARGETIDs.
ii = scxtargs["TARGETID"] != -1
nmatches = np.sum(ii)
log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format(
nmatches, scndout,
time() - start))
if nmatches > 0:
hdr = fitsio.FITSHDR()
hdr["SURVEY"] = surv
fitsio.write(scndout,
scxtargs[ii],
extname='SCND_TARG',
header=hdr,
clobber=True)
log.info('Done...t={:.1f}s'.format(time() - start))
return targs
def plot_overlap_with_observations(self,
fields=None,
pid=None,
first_det_window_days=None,
min_sep=0.01):
""" """
(
double_in_plane_pixels,
double_in_plane_probs,
single_in_plane_pixels,
single_in_plane_prob,
veto_pixels,
plane_pixels,
plane_probs,
times,
double_no_plane_prob,
double_no_plane_pixels,
single_no_plane_prob,
single_no_plane_pixels,
overlapping_fields,
) = self.calculate_overlap_with_observations(
fields=fields,
pid=pid,
first_det_window_days=first_det_window_days,
min_sep=min_sep,
)
fig = plt.figure()
plt.subplot(projection="aitoff")
self.overlap_fields = list(set(overlapping_fields))
self.overlap_prob = np.sum(double_in_plane_probs +
double_no_plane_prob) * 100.0
size = hp.max_pixrad(self.nside)**2 * 50.0
veto_pos = np.array([
hp.pixelfunc.pix2ang(self.nside, i, lonlat=True)
for i in veto_pixels
]).T
if len(veto_pos) > 0:
plt.scatter(
np.radians(self.wrap_around_180(veto_pos[0])),
np.radians(veto_pos[1]),
color="red",
s=size,
)
plane_pos = np.array([
hp.pixelfunc.pix2ang(self.nside, i, lonlat=True)
for i in plane_pixels
]).T
if len(plane_pos) > 0:
plt.scatter(
np.radians(self.wrap_around_180(plane_pos[0])),
np.radians(plane_pos[1]),
color="green",
s=size,
)
single_pos = np.array([
hp.pixelfunc.pix2ang(self.nside, i, lonlat=True)
for i in single_no_plane_pixels
]).T
if len(single_pos) > 0:
plt.scatter(
np.radians(self.wrap_around_180(single_pos[0])),
np.radians(single_pos[1]),
c=single_no_plane_prob,
vmin=0.0,
vmax=max(self.data[self.key]),
s=size,
cmap="gray",
)
plot_pos = np.array([
hp.pixelfunc.pix2ang(self.nside, i, lonlat=True)
for i in double_no_plane_pixels
]).T
if len(plot_pos) > 0:
plt.scatter(
np.radians(self.wrap_around_180(plot_pos[0])),
np.radians(plot_pos[1]),
c=double_no_plane_prob,
vmin=0.0,
vmax=max(self.data[self.key]),
s=size,
)
red_patch = mpatches.Patch(color="red", label="Not observed")
gray_patch = mpatches.Patch(color="gray", label="Observed once")
violet_patch = mpatches.Patch(color="green",
label="Observed Galactic Plane (|b|<10)")
plt.legend(handles=[red_patch, gray_patch, violet_patch])
message = (
"In total, {0:.2f} % of the contour was observed at least once.\n"
"This estimate includes {1:.2f} % of the contour "
"at a galactic latitude <10 deg.\n"
"In total, {2:.2f} % of the contour was observed at least twice. \n"
"In total, {3:.2f} % of the contour was observed at least twice, "
"and excluding low galactic latitudes.\n"
"These estimates account for chip gaps.".format(
100 *
(np.sum(double_in_plane_probs) + np.sum(single_in_plane_prob) +
np.sum(single_no_plane_prob) + np.sum(double_no_plane_prob)),
100 * np.sum(plane_probs),
100.0 *
(np.sum(double_in_plane_probs) + np.sum(double_no_plane_prob)),
100.0 * np.sum(double_no_plane_prob),
))
all_pix = single_in_plane_pixels + double_in_plane_pixels
n_pixels = len(single_in_plane_pixels + double_in_plane_pixels +
double_no_plane_pixels + single_no_plane_pixels)
n_double = len(double_no_plane_pixels + double_in_plane_pixels)
n_plane = len(plane_pixels)
self.healpix_area = (
hp.pixelfunc.nside2pixarea(self.nside, degrees=True) * n_pixels)
self.double_extragalactic_area = (
hp.pixelfunc.nside2pixarea(self.nside, degrees=True) * n_double)
plane_area = hp.pixelfunc.nside2pixarea(self.nside,
degrees=True) * n_plane
self.overlap_fields = overlapping_fields
# area = (2. * base_ztf_rad)**2 * float(len(overlapping_fields))
# n_fields = len(overlapping_fields)
self.logger.info(
f"{n_pixels} pixels were covered, covering approximately {self.healpix_area:.2g} sq deg."
)
self.logger.info(
f"{n_double} pixels were covered at least twice (b>10), covering approximately {self.double_extragalactic_area:.2g} sq deg."
)
self.logger.info(
f"{n_plane} pixels were covered at low galactic latitude, covering approximately {plane_area:.2g} sq deg."
)
return fig, message
def match_secondary(infile, scxtargs, sep=1., scxdir=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
infile : :class:`str`
The full path to a file containing primary targets.
scxtargs : :class:`~numpy.ndarray`
An array of secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets. If passed,
this is written to the output primary file header as `SCNDDIR`.
Returns
-------
:class:`~numpy.ndarray`
The array of secondary targets, with the `TARGETID` bit
updated with any matching primary targets from `infile`.
Notes
-----
- The primary target `infiles` are written back to their
original path with `.fits` changed to `-wscnd.fits` and the
`SCND_TARGET` bit populated for matching targets.
"""
# ADM just the file name for logging.
fn = os.path.basename(infile)
# ADM read in the primary targets.
log.info('Reading primary targets file {}...t={:.1f}s'.format(
infile,
time() - start))
intargs, hdr = fitsio.read(infile, extension="TARGETS", header=True)
# ADM fail if file's already been matched to secondary targets.
if "SCNDDIR" in hdr:
msg = "{} already matched to secondary targets".format(fn) \
+ " (did you mean to remove {}?)!!!".format(fn)
log.critical(msg)
raise ValueError(msg)
# ADM add the SCNDDIR to the primary targets file header.
hdr["SCNDDIR"] = scxdir
# ADM add a SCND_TARGET column to the primary targets.
dt = intargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(intargs), dtype=dt)
for col in intargs.dtype.names:
targs[col] = intargs[col]
# ADM match to all secondary targets for non-custom primary files.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if 'FILEHPX' in hdr:
nside, pix = hdr['FILENSID'], hdr['FILEHPX']
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(intargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(intargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(fn, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# ADM update the secondary targets with the primary TARGETID.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM form the output primary file name and write the file.
base, ext = os.path.splitext(infile)
outfile = "{}{}{}".format(base, '-wscnd', ext)
log.info('Writing updated primary targets to {}...t={:.1f}s'.format(
outfile,
time() - start))
fitsio.write(outfile, targs, extname='TARGETS', header=hdr, clobber=True)
log.info('Done for {}...t={:.1f}s'.format(fn, time() - start))
return scxtargs
def uniform_in_pixel(nside, ipix, size, nest=False):
'''Uniform distribution of points over healpix pixel.
Draws randomly distributed points from the healpix pixel `ipix` for a map
with a given `nside` parameter.
Parameters
----------
nside : int
Healpix map `nside` parameter.
ipix : int
Healpix map pixel index.
size : int
Number of points to draw.
nest : bool, optional
If True assume ``NESTED`` pixel ordering, otherwise ``RING`` pixel
ordering. Default is ``RING`` pixel ordering.
Returns
-------
coords : `~astropy.coordinates.SkyCoord`
Randomly distributed points over the healpix pixel.
Warnings
--------
This function requires the ``healpy`` package.
Examples
--------
See :ref:`User Documentation <skypy.position.uniform_in_pixel>`.
'''
from healpy import pix2ang, max_pixrad, nside2pixarea, ang2pix
# get the centre of the healpix pixel as a SkyCoord
centre_lon, centre_lat = pix2ang(nside, ipix, nest=nest, lonlat=True)
centre = SkyCoord(centre_lon, centre_lat, unit=units.deg)
# get the maximum radius of a healpix pixel in radian
r = max_pixrad(nside)
# use that radius as the aperture of a spherical area in steradian
area = TWO_PI * (1 - np.cos(r)) * units.sr
# oversampling factor = 1/(probability of the draw)
over = area.value / nside2pixarea(nside)
# the array of longitudes and latitudes of the sample
lon, lat = np.empty(0), np.empty(0)
# rejection sampling over irregularly shaped healpix pixels
miss = size
while miss > 0:
# get the coordinates in a circular aperture around centre
sample = uniform_around(centre, area, int(np.ceil(miss * over)))
# get longitude and latitude of the sample
sample_lon, sample_lat = sample.ra.deg, sample.dec.deg
# accept those positions that are inside the correct pixel
accept = ipix == ang2pix(nside,
sample_lon,
sample_lat,
nest=nest,
lonlat=True)
# store the new positions
lon = np.append(lon, np.extract(accept, sample_lon))
lat = np.append(lat, np.extract(accept, sample_lat))
miss = size - len(lon)
# construct the coordinates
return SkyCoord(lon[:size], lat[:size], unit=units.deg)
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
primtargs : :class:`~numpy.ndarray`
An array of primary targets.
scndout : :class`~numpy.ndarray`
Name of a sub-directory to which to write the information in
`desitarget.secondary.outdatamodel` with `TARGETID` and (the
highest) `PRIORITY_INIT` updated with matching primary info.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
pix : :class:`list`, optional, defaults to `None`
Limit secondary targets to (NESTED) HEALpixels that touch
pix at the supplied `nside`, as a speed-up.
nside : :class:`int`, optional, defaults to `None`
The (NESTED) HEALPixel nside to be used with `pixlist`.
Returns
-------
:class:`~numpy.ndarray`
The array of primary targets, with the `SCND_TARGET` bit
populated for matches to secondary targets
"""
# ADM add a SCND_TARGET column to the primary targets.
dt = primtargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(primtargs), dtype=dt)
for col in primtargs.dtype.names:
targs[col] = primtargs[col]
# ADM check if this is an SV or main survey file.
cols, mx, surv = main_cmx_or_sv(targs, scnd=True)
log.info('running on the {} survey...'.format(surv))
if surv != 'main':
scxdir = os.path.join(scxdir, surv)
# ADM read in non-OVERRIDE secondary targets.
scxtargs = read_files(scxdir, mx[3])
scxtargs = scxtargs[~scxtargs["OVERRIDE"]]
# ADM match primary targets to non-OVERRIDE secondary targets.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if nside is not None and pix is not None:
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(primtargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(primtargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(scndout, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# ADM update the secondary targets with the primary information.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM the maximum priority will be used to break ties in the
# ADM unlikely event that a secondary matches two primaries.
hipri = np.maximum(targs["PRIORITY_INIT_DARK"],
targs["PRIORITY_INIT_BRIGHT"])
scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs]
# ADM write the secondary targets that have updated TARGETIDs.
ii = scxtargs["TARGETID"] != -1
nmatches = np.sum(ii)
log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format(
nmatches, scndout,
time() - start))
if nmatches > 0:
hdr = fitsio.FITSHDR()
hdr["SURVEY"] = surv
fitsio.write(scndout,
scxtargs[ii],
extname='SCND_TARG',
header=hdr,
clobber=True)
log.info('Done...t={:.1f}s'.format(time() - start))
return targs
