def get_vert_bbox(llra, lldec, urra, urdec, nside, nest):
"""Get a list of the vertices for all Healpix pixels inside a given bounding
box, resolution and pixelization schema.
Args:
llra(float): Lower left RA for the bounding box.
lldec(float): Lower left DEC for the bounding box.
urra(float): Upper right RA for the bounding box.
urdec(float): Upper right DEC for the bounding box.
nside(int) : Healpix resolution given by nside.
nest(bool): Pixelization schema, True: Nested Schema, False: Ring Schema
Returns:
A list with the vertices for all pixels inside the bounding box.
"""
# Find central point for the bbox
mid_ra = (urra + llra) / 2.
mid_dec = (urdec + lldec) / 2.
# Convert to radians
phi = mid_ra / 180. * np.pi
th = (90. - mid_dec) / 180. * np.pi
pix = hp.ang2pix(nside, th, phi, nest) # Pixel at center
neig = hp.get_all_neighbours(nside, pix, None,
nest) # Get initial 8 neighbors
# Create a set
allp = set(neig)
checked_big = set()
checked_all = set()
k = 0
# Find all pixels inside bounding box by looking at all the neighbors'
# neighbors recursively using a set to speed up the process.
while True:
if k == 10000:
raise RuntimeError('Did not converge, increase k limit')
# Create a copy of the intial set of neighbors
temp = allp.copy()
for i in temp.copy().difference(checked_big):
ntemp = set(hp.get_all_neighbours(nside, i, None, nest))
for j in ntemp.difference(checked_all):
th2, phi2 = hp.pix2ang(nside, j, nest)
# Check if given neighbor is inside the bbox
if is_inside_bbox(phi2 * 180. / np.pi, 90 - th2 * 180. / np.pi,
llra, lldec, urra, urdec):
temp.add(j)
# Update set to keep track of checked pixels
checked_all.update(ntemp)
checked_big.add(i)
if temp == allp:
break
else:
allp.update(temp)
k += 1
all_verts = []
# Get vertices for pixels inside bounding box
for i in allp:
all_verts.append(get_vert(i, nside, nest))
# Return complete list
return all_verts
def get_all_neighbours(Nside, i, depth_neighbours=1):
pixel_list = hp.get_all_neighbours(Nside, i, nest=True)
pixel_tmp = pixel_list
depth_neighbours -= 1
while depth_neighbours != 0 :
pixel_tmp = hp.get_all_neighbours(Nside, pixel_tmp, nest=True)
pixel_tmp = np.reshape(pixel_tmp, pixel_tmp.size)
pixel_list = np.append(pixel_list, pixel_tmp)
depth_neighbours -= 1
return pixel_list
def get_neighbours(ipix, nside, order):
"""
Given a pixel index in the Healpix pixelization and the nside parameter, estimated
the indices of nearest neighbour pixels. The larger is `order`,
the farther neighbours are included.
"""
if order == 0:
return np.unique(hp.get_all_neighbours(theta=ipix, nside=nside))
else:
ipix = np.unique(hp.get_all_neighbours(theta=ipix, nside=nside))
return get_neighbours(ipix, nside, order - 1)
def neighbors_of_neighbors(nside, th, phi):
"""
Finds the pixel numbers of the 8 neighbors of the the point (th,phi),
then find the 8 neighbors of each of those points. The are the 64 pixel
indices of the "neighbors of neighbors" of the point (th,phi).
"""
neighbors = hp.get_all_neighbours(nside, th, phi=phi)
tn, pn = hp.pix2ang(nside, neighbors)
nn = hp.get_all_neighbours(nside, tn, phi=pn)
return nn.flatten()
def neighbors_of_neighbors(nside, th, phi):
"""
Finds the pixel numbers of the 8 neighbors of the the point (th,phi),
then find the 8 neighbors of each of those points. The are the 64 pixel
indices of the "neighbors of neighbors" of the point (th,phi).
"""
neighbors = hp.get_all_neighbours(nside, th, phi=phi)
tn, pn = hp.pix2ang(nside, neighbors)
nn = hp.get_all_neighbours(nside, tn, phi=pn)
return nn.flatten()
def find_survey_boundary(nside, ra, dec, min_neighbors=8):
'''
Find pixels that has fewer than specified number of neighbors (occupied
pixels).
Inputs
------
nside: NSIDE parameter in healpix;
ra, dec: coordinates of the healpix pixels;
Output
------
idx: indices of the pixels at survey boundary.
'''
# pixel indices in healpy
pix_id = hp.pixelfunc.ang2pix(nside, ra, dec, lonlat=True)
# count number of neighbors
neighbors = hp.get_all_neighbours(nside, ra, dec, lonlat=True).T
mask = np.in1d(neighbors, pix_id).reshape(neighbors.shape)
neighbors_count = np.sum(mask, axis=1)
idx = np.where(neighbors_count < min_neighbors)
return idx
def angular_distance_from_mask(mask):
"""
For each pixel of a Healpix map, return the smallest angular distance
to a set of masked pixels (of value True), in degrees.
Parameter
---------
maskok : boolean Healpix map
The Healpix mask that defines the set of masked pixels (of value True)
whose smallest angular distance to each pixel of a Healpix map of same
nside is computed.
"""
nside = hp.npix2nside(len(mask))
# get the list of pixels on the external border of the mask
ip = np.arange(12 * nside**2)[~mask]
neigh = hp.get_all_neighbours(nside, ip)
nn = np.unique(neigh.ravel())
if nn[0] == -1:
nn = nn[1:]
nn = nn[mask[nn]]
# get unit vectors for border and inner pixels
vecpix_inner = np.array(hp.pix2vec(nside, ip))
vecpix_outer = np.array(hp.pix2vec(nside, nn))
# get angles between the border pixels and inner pixels
cosang = np.dot(vecpix_inner.T, vecpix_outer)
mapang = np.zeros(12 * nside**2)
mapang[~mask] = np.degrees(np.arccos(np.max(cosang, axis=1)))
return mapang
def angular_distance_from_mask(mask):
"""
For each pixel of a Healpix map, return the smallest angular distance
to a set of masked pixels (of value True), in degrees.
Parameter
---------
maskok : boolean Healpix map
The Healpix mask that defines the set of masked pixels (of value True)
whose smallest angular distance to each pixel of a Healpix map of same
nside is computed.
"""
nside = hp.npix2nside(len(mask))
# get the list of pixels on the external border of the mask
ip = np.arange(12*nside**2)[~mask]
neigh = hp.get_all_neighbours(nside, ip)
nn = np.unique(neigh.ravel())
if nn[0] == -1:
nn = nn[1:]
nn = nn[mask[nn]]
# get unit vectors for border and inner pixels
vecpix_inner = np.array(hp.pix2vec(nside, ip))
vecpix_outer = np.array(hp.pix2vec(nside, nn))
# get angles between the border pixels and inner pixels
cosang = np.dot(vecpix_inner.T, vecpix_outer)
mapang = np.zeros(12*nside**2)
mapang[~mask] = np.degrees(np.arccos(np.max(cosang, axis=1)))
return mapang
def tracts_outline(df):
#list of tracts
tracts = df.select("tract").distinct().toPandas()
df_withpix = df.groupBy(["tract", "ipix"]).count().cache()
ipix = np.arange(hp.nside2npix(nside))
pixborder = []
print("NSIDE={}".format(nside))
for t in tracts['tract'].values:
print("treating tract={}".format(t))
#create a map just for this tract
df_map = df_withpix.filter(df_withpix.tract == int(t))
#create the healpix map
tract_p = df_map.toPandas()
#plt.hist(tract_p['count'].values)
tract_map = np.full(hp.nside2npix(nside), hp.UNSEEN)
tract_map[tract_p['ipix'].values] = 1
# for lit pixels compute the neighbours
ipix1 = tract_p['ipix'].values
theta, phi = hp.pix2ang(nside, ipix1)
neighbours = hp.get_all_neighbours(nside, theta, phi, 0).transpose()
# border if at least one neighbours is UNSEEN
mask = [(hp.UNSEEN in tract_map[neighbours[pixel]])
for pixel in range(len(ipix1))]
pixborder += list(ipix1[mask])
return pixborder
def group_neighbours(group):
neighbours = np.array([])
for i in group:
neighbours = np.append(
neighbours,
hp.get_all_neighbours(region_nsides, int(i), nest=True))
return neighbours
def contour_pix(map, vals, all_neighbours=True): """ given a healpix map (map) and a set of values, we find and return lists of pixels that constitute the boarder of those sets """ npix = len(map) nside = hp.npix2nside(npix) ### separate into pixel sets based in p_values pix = np.arange(npix) boarders = [] for val in vals: _pix = pix[map>=val] ### pull out included pixels truth = np.zeros((npix,), bool) truth[_pix] = True boarder = np.zeros((npix,),bool) ### defines which modes are in the boarder for ipix in _pix: if all_neighbours: boarder[ipix] = not truth[[n for n in hp.get_all_neighbours(nside, ipix) if n != -1]].all() else: boarder[ipix] = not truth[[n for n in hp.get_neighbours(nside, ipix)[0] if n != -1]].all() boarders.append( pix[boarder] ) return boarders
def find_rims(holes, nside):
hole_indices = np.unique(holes[1])
'''hole_sizes = np.load(holes_dir + 'nilc_pr1_builtmask_holes_ring_sizes.npy') #Hole index, hole size
hole_indices = hole_sizes[0,np.argsort(hole_sizes[1])[::-1]] #Descending order of size'''
nholes = len(hole_indices)
#nholes = 100 #10 largest holes
circpixs = [None] * nholes
rimpixs = [None] * nholes
rimindex = [None] * nholes
for hole in xrange(nholes):
print "Forming rim to hole", hole + 1, "/", nholes
#Extracting hole pixels
hole_index = hole_indices[hole]
circpixs[hole] = holes[0, holes[1] == hole_index]
pix_neighbours = hp.get_all_neighbours(nside,
circpixs[hole]) #Inc. "-1"
rimpixs[hole] = np.setdiff1d(
np.concatenate(pix_neighbours),
np.concatenate(
(holes[0],
np.array([-1])))) #Remove any existing hole pixels & "-1"
#lim = len(rimpixs[hole])
for iter in xrange(3): #Ensure a minimum rim thickness of 3 pixels
pix_neighbours = hp.get_all_neighbours(nside, rimpixs[hole])
new_rimpixs = np.setdiff1d(
np.concatenate(pix_neighbours),
np.concatenate((holes[0], np.array([-1]),
rimpixs[hole]))) # - holes & rim & "-1"
rimpixs[hole] = np.concatenate((rimpixs[hole], new_rimpixs))
lim = len(rimpixs[hole])
while lim < 80: #Iterate until have at least 80 border pixels (N_side = 2048)
pix_neighbours = hp.get_all_neighbours(nside, rimpixs[hole])
new_rimpixs = np.setdiff1d(
np.concatenate(pix_neighbours),
np.concatenate((holes[0], np.array([-1]),
rimpixs[hole]))) # - holes & rim & "-1"
rimpixs[hole] = np.concatenate((rimpixs[hole], new_rimpixs))
lim = len(rimpixs[hole])
rimindex[hole] = np.array([hole_index] * lim)
np.save(
'/Users/keir/Documents/s2let_ilc_planck/nilc_pr1_builtmask_rims_ring_enlarged3.npy',
np.vstack((np.concatenate(rimpixs), np.concatenate(rimindex))))
return np.concatenate(rimpixs), np.concatenate(rimindex)
def prep_indes(nside=256, fname=None):
if fname == None:
fname = os.getenv(
'CSCRATCH'
) + '/BGS/SV-ASSIGN/des/des.ply' ## '/global/cscratch1/sd/raichoor/desits/des.ply'
##
nside = np.int(nside)
npix = hp.nside2npix(nside)
# checking hp pixels
mng = pymangle.Mangle(fname)
theta, phi = hp.pix2ang(nside, np.arange(0, npix, 1), nest=False)
hpra, hpdec = 180. / np.pi * phi, 90. - 180. / np.pi * theta
allhp = mng.polyid(hpra, hpdec)
hpindes = (allhp != -1).astype(int)
# pixels with all neighbours in des.
hpindes_secure = np.array([
i for i in range(npix)
if hpindes[i] + hpindes[hp.get_all_neighbours(nside, i)].sum() == 9
])
# pixels with all neighbours outside des.
hpoutdes_secure = np.array([
i for i in range(npix)
if hpindes[i] + hpindes[hp.get_all_neighbours(nside, i)].sum() == 0
])
# hpind to be checked
tmp = np.ones(npix, dtype=bool)
tmp[hpindes_secure] = False
tmp[hpoutdes_secure] = False
hp_tbc = np.array(range(npix))[tmp]
np.savetxt(os.getenv('CSCRATCH') + '/BGS/SV-ASSIGN/des/hpindes_secure.txt',
hpindes_secure,
fmt='%d')
np.savetxt(os.getenv('CSCRATCH') + '/BGS/SV-ASSIGN/des/hp_tbc.txt',
hp_tbc,
fmt='%d')
def _get_neighbors(self, idx):
import healpy as hp
nside = self._get_nside(idx)
idx_nb = (hp.get_all_neighbours(nside, idx[0], nest=self.nest), )
idx_nb += tuple(
[t[None, ...] * np.ones_like(idx_nb[0]) for t in idx[1:]])
return idx_nb
def identify_islands(nside, ra, dec):
'''
Label the pixels by their island; (islands are pixels that are
connected to each other); lone pixels are given the index -1.
Note: might need to set a higher recursion limit for it to work,
e.g.: sys.setrecursionlimit(3000)
Inputs
------
nside: NSIDE parameter in healpix;
ra, dec: coordinates of the healpix pixels;
Output
------
labels: island index of each pixel.
'''
# Get max recursion depth
rc_limit = sys.getrecursionlimit()
############## find neighbors ###############
npix = hp.nside2npix(nside)
# pixel indices in healpy
pix_id = hp.pixelfunc.ang2pix(nside, ra, dec, lonlat=True)
# find neighbors
neighbors = hp.get_all_neighbours(nside, ra, dec, lonlat=True).T
mask = np.in1d(neighbors, pix_id).reshape(neighbors.shape)
# assign index number -1 to unoccupied neighboring pixels
neighbors[~mask] = -1
# Convert the neighbors array from healpix indices to generic
# numpy indices
pointer = -99 * np.ones(npix, dtype=int)
for index in range(len(ra)):
pointer[pix_id[index]] = index
neighbors[mask] = pointer[neighbors[mask]]
########## label connected boundary pixels ############
# Recursively assign island index;
# It's messy because of the maximum recursion depth
label_now = 0
labels = -1 * np.ones(len(ra), dtype=int)
while not np.all(labels != -1):
index = np.argmax(labels == -1)
labels[index] = label_now
while True:
recursion_flag = True
idx = np.where(labels == label_now)[0]
for index in idx:
recursion_flag = recursion_flag and _spread_label(
index, neighbors, labels, label_now, rc_limit)
if recursion_flag:
break
label_now += 1
return labels
def searcharound_9HEALPix(ra, dec, src_coll, hp_key, hp_order, hp_nest,
hp_resol, find_one):
"""
Returns sources in catalog contained in the 9 healpixels
around the target coordinate.
Parameters:
-----------
ra/dec: `float`
sky coordinates of the target direction. They have to be in
the same format as those used to find the healpix id.
src_coll: `pymongo.collection.Collection`
collection with the sources to be queried.
hp_key: `str`
name of the document key conatining the HEALPix index.
hp_order: `int`
order of the HEALPix grid used to group sources.
hp_nest: `bool`
weather or not the HEALPix grid has nested geometry.
find_one: `bool`
if True the collection is searched with the find_one method returning
just the first result of the query. if False, the method
find is used, returning all matching documents.
Returns:
--------
cptable: `astropy.table.Table`/None
astropy table of the catalog entry for the found counterpart. If
no counpterpart is found returns None.
"""
# find the index of the target pixel and its neighbours
nside = 2**hp_order
target_pix = int(ang2pix(nside, ra, dec, nest=hp_nest, lonlat=True))
neighbs = get_all_neighbours(nside, ra, dec, nest=hp_nest, lonlat=True)
# remove non-existing neigbours (in case of E/W/N/S) and add center pixel
pix_group = [int(pix_id)
for pix_id in neighbs if pix_id != -1] + [target_pix]
# query the database for sources in these pixels
if find_one:
qresults = [src_coll.find_one({hp_key: {"$in": pix_group}})]
else:
qresults = [o for o in src_coll.find({hp_key: {"$in": pix_group}})]
if len(qresults) == 0:
return None
else:
return Table(qresults)
def _fillPeriphery(self, filter):
"""
Fill in peripheral cells of shiftmap with average of nearest neighbors.
"""
all_neighbor_pix = numpy.unique(
healpy.get_all_neighbours(
self.nside, numpy.nonzero(self.zeropoint_shiftmap.data.field(filter) != healpy.UNSEEN)[0]
)
)
filled_pix = numpy.nonzero(self.zeropoint_shiftmap.data.field(filter) != healpy.UNSEEN)[0]
periphery_pix = numpy.setdiff1d(all_neighbor_pix, filled_pix)
shiftmap_filled = numpy.ma.masked_array(
self.zeropoint_shiftmap.data.field(filter), self.zeropoint_shiftmap.data.field(filter) == healpy.UNSEEN
)
self.zeropoint_shiftmap.data.field(filter)[periphery_pix] = numpy.array(
numpy.mean(shiftmap_filled[healpy.get_all_neighbours(self.nside, periphery_pix)], axis=0),
dtype=shiftmap_filled.dtype,
)
self.peripheral_area = healpy.nside2pixarea(self.nside, degrees=True) * len(periphery_pix)
def get_outline(mapin):
nside = _hp.get_nside(mapin)
outline = _np.array([_hp.UNSEEN]*_hp.nside2npix(nside))
pix = _np.where(mapin>0.0)[0]
for p in pix:
ptmp = _hp.get_all_neighbours(nside,p)
i = _np.where(mapin[ptmp] == 0.0)[0]
outline[ptmp[i]] = 1.0
return outline
def load_data(self, stars=True, galaxies=False):
"""
Load local catalogue data by healpix
Apply star and galaxy filters
"""
data_array = []
data_dir = "/home/js01093/dwarf/simple_adl/simple_adl/data_dir/delvemc_blue" # Temp for now
pix_nside_select = hp.ang2pix(nside=self.nside, theta=self.ra, phi=self.dec, nest=False, lonlat=True)
pix_nside_neighbors = np.concatenate([[pix_nside_select], hp.get_all_neighbours(self.nside, pix_nside_select)])
print('Center healpixel: {}'.format(pix_nside_select))
print('Healpixels: {}'.format(pix_nside_neighbors))
for pix_nside in pix_nside_neighbors:
inlist = glob.glob('{}/*_{:05d}.fits'.format(data_dir, pix_nside)) # is it searching them *all* and returning strongest peaks right now?
for infile in inlist:
if not os.path.exists(infile):
continue
data_array.append(fits.read(infile))
data_raw = np.concatenate(data_array) # JS: Extend data array, here we extend with neighbouring healpix data too?
# Return stars:
if stars == True:
star_filter = (self.star_filter(data_raw) == 1)
data = data_raw[star_filter]
# Return galaxies:
elif galaxies == True:
galaxy_filter = (self.galaxy_filter(data_raw) == 1)
data = data_raw[galaxy_filter]
# Return both?
elif stars == True and galaxies == True:
star_galaxy_filter = (self.star_filter(data_raw) == 1 or self.star_filter(data_raw) == 1)
data = data_raw[star_galaxy_filter]
self.data = data
return self.data
def fill_empty_pixels(sky_map,
max_iter,
fail_fill_value=0,
pol=True,
verbose=False):
"""
Fill pixels with NAN with a fail_fill_value.
"""
if np.sum(np.isnan(sky_map)) == 0:
if verbose:
prompt("There are no empty pixels")
return
nside = hp.get_nside(sky_map)
if pol:
dim = sky_map.shape[0]
for i in xrange(max_iter):
empty_pix = np.where(np.isnan(sky_map[0]))[0]
theta, phi = hp.pix2ang(nside, empty_pix)
neighbours = hp.get_all_neighbours(nside, theta, phi).T
for j in range(dim):
fill_values = np.nanmean(sky_map[j][neighbours], axis=-1)
sky_map[j][empty_pix] = fill_values
if np.sum(np.isnan(sky_map)) == 0:
break
else:
for i in xrange(max_iter):
empty_pix = np.where(np.isnan(sky_map))[0]
theta, phi = hp.pix2ang(nside, empty_pix)
neighbours = hp.get_all_neighbours(nside, theta, phi).T
fill_values = np.nanmean(sky_map[neighbours], axis=-1)
sky_map[empty_pix] = fill_values
if np.sum(np.isnan(sky_map)) == 0:
break
num_empty_pix = np.sum(np.isnan(sky_map))
if num_empty_pix:
prompt(
"{} empty pixels remaining after {} iterations. Filling empty pixels with {}\n"
.format(num_empty_pix, max_iter, fail_fill_value))
sky_map[np.isnan(sky_map)] = fail_fill_value
def check_neighbourhood(self, pixels_map, threshold, check_map,
pixel_index):
neighbours_indexes = hp.get_all_neighbours(hp.get_nside(pixels_map),
pixel_index)
hp.get_all_neighbours(hp.get_nside(pixels_map), neighbours_indexes[0])
# Loop over all neighbours
for i in range(len(neighbours_indexes)):
# Checking if its already checked
if check_map[neighbours_indexes[i]] == 0:
check_map[neighbours_indexes[i]] = 1
# Checking if the selected pixel is in the cluster
if pixels_map[neighbours_indexes[
i]] >= threshold and self.touched_mask == 0:
self.add_pixel(neighbours_indexes[i],
pixels_map[neighbours_indexes[i]])
self.check_neighbourhood(pixels_map, threshold, check_map,
neighbours_indexes[i])
elif check_map[neighbours_indexes[i]] == 2:
self.touched_mask = 1
def _fill_periphery(self):
'''
Fill in peripheral cells of shiftmap with average of nearest neighbors
'''
self.peripheral_area = np.zeros(self.nmag,dtype=np.float32)
for i in xrange(self.nmag):
all_neighbor_pix = np.unique(hp.get_all_neighbours(self.nside,np.nonzero(self.shiftmap[i,:] != hp.UNSEEN)[0]))
# remove any negatives...
gd,=np.where(all_neighbor_pix >= 0)
all_neighbor_pix = all_neighbor_pix[gd]
filled_pix, = np.nonzero(self.shiftmap[i,:] != hp.UNSEEN)
periphery_pix = np.setdiff1d(all_neighbor_pix, filled_pix)
shiftmap_filled = np.ma.masked_array(self.shiftmap[i,:], self.shiftmap[i,:] == hp.UNSEEN)
self.shiftmap[i,periphery_pix] = np.array(np.mean(shiftmap_filled[hp.get_all_neighbours(self.nside,periphery_pix)],axis=0),dtype=shiftmap_filled.dtype)
self.peripheral_area[i] = hp.nside2pixarea(self.nside, degrees=True) * periphery_pix.size
def get_nearby_healpix_list(self, obs, sideHP=32768, nestedHP=True):
''' See original cnd.py for more details '''
# If arcsecRadius < hp.nside2resol(32768, arcmin = True)*60 [ which is ~ 6.4 arc-sec ], ...
# ... then only need to search adjacent HP
listHP = list(hp.get_all_neighbours(sideHP, obs.Healpix,
nest=nestedHP))
listHP.append(obs.Healpix)
# If arcsecRadius > hp.nside2resol(32768, arcmin = True)*60, then need to search neighbors of neighbours ...
arcsecRadius = self.get_arcsecRadius(obs)
if arcsecRadius > hp.nside2resol(sideHP, arcmin=True) * 60:
for i in range(
int(arcsecRadius //
(hp.nside2resol(sideHP, arcmin=True) * 60))):
tmpHP = [h for h in listHP]
for h in tmpHP:
listHP.extend(
list(hp.get_all_neighbours(sideHP, h, nest=nestedHP)))
listHP = list(set(listHP))
return listHP
def get_edge(mask):
edge = []
for i in range(len(mask)):
if mask[i] == 1:
neighbours = hp.get_all_neighbours(hp.npix2nside(len(mask)), i)
edge_pixel = False
for neighbour in neighbours:
if neighbour != -1:
if mask[neighbour] == 0:
edge_pixel = True
if edge_pixel:
edge.append(i)
return np.array(edge)
def hpx_neighbours(ipix, vicinity, nside, ordering='Ring'):
neighbours = np.array([ipix])
remote = np.array([0])
nested = False
if ordering is 'Nest':
nested = True
if vicinity != 0:
for r in range(1, vicinity + 1):
if r == 1:
liste = hp.get_all_neighbours(nside, ipix, nest=nested)
liste = liste[np.where(liste >= 0)]
remote = np.concatenate([remote, np.tile(r, liste.size)])
neighbours = np.concatenate([neighbours, liste])
previouslist = np.copy(liste)
else:
liste = hp.get_all_neighbours(nside, previouslist, nest=nested)
liste = np.unique(liste)
liste = liste[np.where(liste >= 0)]
## Eliminate those already in list
previouslist = np.concatenate([neighbours, liste])
liste, inds = np.unique(previouslist, return_inverse=True)
indx = inds[0:len(neighbours)]
previouslist = np.delete(liste, indx)
neighbours = np.concatenate([liste[indx], previouslist])
remote = np.concatenate(
[remote, np.tile(r, previouslist.size)])
return (neighbours, remote)
def __into_boarders(nside, pix, nest=False):
"""
extracts the boarder from the list of pixels (pix)
"""
### establish an array representing the included pixels
npix = hp.nside2npix(nside)
truth = np.zeros((npix, ), bool)
truth[pix] = True
abstruth = np.zeros(
(npix, ),
bool) ### need completely separate object, not just a pointer
abstruth[pix] = True
pixnums = np.arange(npix) ### convenient array we establish once
boarders = []
while truth.any():
ipix = pixnums[truth][0] ### take the first pixel
truth[ipix] = False ### remove it from the global set
boarder = []
to_check = [ipix] ### add it to the list of things to check
while len(to_check): # there are pixels in this mode we have to check
ipix = to_check.pop() # take one pixel from those to be checked.
isinterior = True
for neighbour in hp.get_all_neighbours(
nside, ipix, nest=nest): # get neighbors as rtheta, rphi
if neighbour == -1: ### when neighbour == -1, there is no corresponding pixel in this direction
pass
else:
isinterior *= abstruth[
neighbour] ### check to see if the neighbor is in the set
### we don't care if it has already been visited.
# try to find pixel in skymap
if truth[
neighbour]: ### pixel in the set and has not been visited before
truth[
neighbour] = False ### remove neighbour from global set
to_check.append(
neighbour) ### add to list of things to check
else: ### pixel not in the set or has been visited before
pass
if not isinterior:
boarder.append(ipix)
boarders.append(boarder)
return boarders
def get_borders_from_ipix(part):
# Get the pixel ID from the iterator
ipix = [*part]
if len(ipix) == 0:
# Empty partition
yield []
else:
# Get the 8 neighbours of all unique pixels.
theta, phi = hp.pix2ang(nside, ipix)
neighbours = hp.get_all_neighbours(nside, theta, phi, 0).flatten()
# Yield only pixels at the borders
unseen = np.array([i for i in neighbours if i not in ipix])
yield unseen
def _fillPeriphery(self, filter):
'''
Fill in peripheral cells of shiftmap with average of nearest neighbors.
'''
all_neighbor_pix = numpy.unique(
healpy.get_all_neighbours(
self.nside,
numpy.nonzero(
self.zeropoint_shiftmap.data.field(filter) != healpy.UNSEEN
)[0]))
filled_pix = numpy.nonzero(
self.zeropoint_shiftmap.data.field(filter) != healpy.UNSEEN)[0]
periphery_pix = numpy.setdiff1d(all_neighbor_pix, filled_pix)
shiftmap_filled = numpy.ma.masked_array(
self.zeropoint_shiftmap.data.field(filter),
self.zeropoint_shiftmap.data.field(filter) == healpy.UNSEEN)
self.zeropoint_shiftmap.data.field(
filter)[periphery_pix] = numpy.array(numpy.mean(
shiftmap_filled[healpy.get_all_neighbours(
self.nside, periphery_pix)],
axis=0),
dtype=shiftmap_filled.dtype)
self.peripheral_area = healpy.nside2pixarea(
self.nside, degrees=True) * len(periphery_pix)
def fast_stack(ras,
decs,
inmap,
weights=None,
prob_weights=None,
nsides=2048,
iterations=500,
bootstrap=False):
if weights is None:
weights = np.ones(len(ras))
outerkappa = []
lons, lats = equatorial_to_galactic(ras, decs)
inmap = convergence_map.set_unseen_to_nan(inmap)
pix = hp.ang2pix(nsides, lons, lats, lonlat=True)
neighborpix = hp.get_all_neighbours(nsides, pix)
centerkappa = inmap[pix]
neighborkappa = np.nanmean(inmap[neighborpix], axis=0)
centerkappa = np.nanmean([centerkappa, neighborkappa], axis=0)
weights[np.isnan(centerkappa)] = 0
centerkappa[np.isnan(centerkappa)] = 0
if prob_weights is not None:
true_weights_for_sum = weights * np.array(prob_weights)
weightsum = np.sum(true_weights_for_sum)
else:
weightsum = np.sum(weights)
centerstack = np.sum(weights * centerkappa) / weightsum
if iterations > 0:
for x in range(iterations):
bootidx = np.random.choice(len(lons), len(lons))
outerkappa.append(
np.nanmean(inmap[hp.ang2pix(nsides,
lons[bootidx],
lats[bootidx],
lonlat=True)]))
if bootstrap:
return centerstack, outerkappa
else:
return centerstack, np.nanstd(outerkappa)
else:
return centerstack
def map_ang_from_edges(maskok):
print('Calculating Angle Mask Map')
nsmask = hp.npix2nside(len(maskok))
### Get the list of pixels on the external border of the mask
ip = np.arange(12*nsmask**2)
neigh = hp.get_all_neighbours(nsmask, ip[maskok])
nn = np.unique(np.sort(neigh.ravel()))
nn = nn[maskok[nn] == False]
### Get unit vectors for border and inner pixels
vecpixarray_inner = np.array(hp.pix2vec(nsmask,ip[maskok]))
vecpixarray_outer = np.array(hp.pix2vec(nsmask,nn))
### get angles between the border pixels and inner pixels
cosang = np.dot(np.transpose(vecpixarray_inner),vecpixarray_outer)
mapang = np.zeros(12*nsmask**2)
mapang[maskok] = np.degrees(np.arccos(np.max(cosang,axis=1)))
return mapang
def fixHI(hifile='/Volumes/TimeMachine/data/NHI_HPX.fits', nside_out=256):
'''
read HI column density
rotate it from G to C ("decrease" nside if needed)
fill some negative pixels
by assigning the mean of neighbors
'''
assert (nside_out < 1024), 'nside_out should be < 1024'
# H II map
hii = ft.FITS(hifile, lower=True)
Hii = hii[1].read()
Hiic = G_to_C(Hii['nhi'], res_out=nside_out)
Hineg = np.argwhere(Hiic<=0.0).flatten()
neighbors = hp.get_all_neighbours(nside_out, Hineg)
Hiic[Hineg] = np.mean(Hiic[neighbors], axis=0) # fill in negative pixels
return Hiic
def hpneighbouridx(level, idx):
"""
Neighbours of a Healpix pixel (in nested format)
:param level: Resolution level
:param idx: Index of the Healpix pixel
:return: Neighbouring nodes of the pixel
"""
nside = 2**level
th, ph = hp.pix2ang(nside, idx, nest=True)
ninds = hp.get_all_neighbours(nside, th, ph, nest=True)
nbrs = []
lev = int(np.log2(nside))
for item in ninds:
nbrs.append((lev, item))
return nbrs
def grow_mask_1pix(self): # grows the mask by one pixel
if self.nmaps == 1:
mask = self.mask.copy()
else:
mask = self.mask[0, :].copy()
if self.ordering == 'NEST':
nested = True
else:
nested = False
listpix = [i for i in range(self.npix) if not mask[i]]
for pixel in listpix:
ipix = hp.get_all_neighbours(self.nside, pixel, nest=nested)
n = np.count_nonzero(mask[ipix])
if n > 0:
if self.nmaps == 1:
self.mask[pixel] = True
else:
self.mask[:, pixel] = True
def get_outlines(dir_out, nside):
import os
outline = _np.array([_hp.UNSEEN]*_hp.nside2npix(nside))
for f in os.listdir(dir_out):
print 'Reading',f
m = _hp.read_map(dir_out+'/'+f)
pix = _np.where(m > 0.0)[0]
for p in pix:
ptmp = _hp.get_all_neighbours(nside, p)
i = _np.where(m[ptmp] == 0.0)[0]
outline[ptmp[i]] = 1.0
_hp.write_map('outline.fits',outline)
print 'Wrote outline.fits'
return
def load_local_data(self, ra, dec):
"""
Load data corresponding to the 8 nearest neighbors of the
centroid at (ra, dec) into memory.
"""
hpixel = ugali.utils.healpix.angToPix(self.nside, ra, dec)
hpixels = np.concatenate([[hpixel], hp.get_all_neighbours(self.nside, hpixel)])
data_array = []
for hpix in hpixels:
inlist = glob.glob('{}/*_{:05d}.fits'.format(self.dirname, hpix))
for infile in inlist:
if not os.path.exists(infile):
continue
data_array.append(fits.read(infile))
data = np.concatenate(data_array)
return data
def __init__(self, nside_out=256, path=f'{templ_dir}NHI_HPX.fits'):
self.nside_out = nside_out
nside_in = 1024
self.ordering = 'ring'
self.unit = 'Lenz et. al. HI'
if nside_out!= nside_in:warnings.warn('upgrading/downgrading HI column density')
nhi = ft.FITS(path, lower=True)
nhi = nhi[1].read()['nhi'] # only nhi column
nhi_c = G_to_C(nhi, res_in=nside_in, res_out=nside_out)
nhi_neg_hpix = np.argwhere(nhi_c <= 0.0).flatten()
neighbors = hp.get_all_neighbours(nside_out, nhi_neg_hpix)
nhi_c[nhi_neg_hpix] = np.mean(nhi_c[neighbors], axis=0)
self.map = np.log10(nhi_c)
def get_local_warm_spots(self, log10p_threshold=2, min_ang_dist=1):
r"""Extract local warm spots from a p-value skymap.
Parameters
----------
log10p_threshold: float, default=2
Threshold on log10(p-value), above local warm spots should be considered.
min_ang_dist: float, units: degree, default: 1
Minimal distance between two local warm spots.
Returns
-------
ndarry ("dec":float, "ra": float, "pVal": float)
List of local warm spots. Each warm spot is described by a tuple (dec, ra, p-value)
"""
log10p = self.log10p_map
# get npix and nside
npix = len(log10p)
nside = healpy.npix2nside(npix)
# mask large p-values and infs
mask = np.logical_and(log10p > log10p_threshold, np.isfinite(log10p))
warm_spots_idx = []
for pix in np.arange(npix)[mask]:
theta, phi = healpy.pix2ang(nside, pix)
# if no larger neighbour, we are at a spot
neighbours = healpy.get_all_neighbours(nside, theta, phi)
if not any(log10p[neighbours] > log10p[pix]):
warm_spots_idx.append(pix)
# get pVal and direction of spots and sort them
p_spots = log10p[warm_spots_idx]
theta_spots, phi_spots = healpy.pix2ang(nside, warm_spots_idx)
# fill into record-array
spots = LocalWarmSpotList()
spots.add(theta=theta_spots, phi=phi_spots, pVal=p_spots)
spots = SkylabAllSkyScan.apply_seperation(spots, min_ang_dist)
return spots
def __init__(self, nside):
"""
Parameters
----------
nside : integer
The Healpix map nside.
"""
npix = 12 * nside**2
ipix = np.arange(npix, dtype=np.int32)
neighbours = hp.get_all_neighbours(nside, ipix)
s = FSRMatrix((npix, npix), ncolmax=9, dtype_index=np.int32)
s.data.index[:, 0] = ipix
s.data.index[:, 1:] = neighbours.T
h2 = 4 * np.pi / npix
s.data.value[:, 0] = -20 / (6 * h2)
s.data.value[:, 1:] = np.array([1, 4, 1, 4, 1, 4, 1, 4]) / (6 * h2)
self.nside = nside
SparseOperator.__init__(self, s)
def find_pointings(self, reward_map, m5_map, single_visit_depth=22.):
"""
Paramters
---------
reward_map : np.array
A healpix map where unmasked pixels should be tesselated with pointings.
m5_map : np.array
A healpix map of the co-added 5-sigma limiting depth at each point for the
current status of the survey.
single_visit_depth : float
The rough depth of a single visit.
"""
# Check that target map is correct nside
# Find the peak in the target_map
# Maybe just use the peak pixel and it's 8 neighbors and go with that?
max_index = np.where(reward_map == np.max(reward_map))[0].max()
peak_pixels = hp.get_all_neighbours(max_index)
pass
#target_ra, target_dec =
def flood_fill(nside, ipix, m, nest=False):
"""Stack-based flood fill algorithm in HEALPix coordinates.
Based on <http://en.wikipedia.org/w/index.php?title=Flood_fill&oldid=566525693#Alternative_implementations>.
"""
# Initialize stack with starting pixel index.
stack = [ipix]
while stack:
# Pop last pixel off of the stack.
ipix = stack.pop()
# Is this pixel in need of filling?
if m[ipix]:
# Fill in this pixel.
m[ipix] = False
# Find the pixels neighbors.
neighbors = hp.get_all_neighbours(nside, ipix, nest=nest)
# All pixels have up to 8 neighbors. If a pixel has less than 8
# neighbors, then some entries of the array are set to -1. We
# have to skip those.
neighbors = neighbors[neighbors != -1]
# Push neighboring pixels onto the stack.
stack.extend(neighbors)
def MaskBorders(map,nest=True):
#
# Masks pixels in the border of the footprint
#
import healpy as hp
import copy
nside = hp.npix2nside(len(map))
masked_map = copy.copy(map)
for ipix,ival in enumerate(map):
if ival == 0:
continue
else:
neighb = hp.get_all_neighbours(nside,ipix,nest=nest)
for ineighb in neighb:
if ineighb == -1:
continue
if map[ineighb] == 0:
masked_map[ipix]=0
break
return masked_map
def forward_match(self, catalog2, maxsep):
"""matches self to external catalog in the "LEFT JOIN" sense. ie looks for closest object in catalog2 to each galaxy in self. catalog2 should be of same class as self and already be mapped to pixels with the same number of nsides"""
if self.NSIDE != catalog2.NSIDE:
raise MatchError("NSIDES dont agree!!")
matches = {}
maxsep = np.radians(maxsep/3600.0) #convert arcsec to radians
neighbor_pix = hp.get_all_neighbours(self.NSIDE, self.pix).T
for count, a in enumerate(self.pix):
if a in catalog2.pix_and_neigh:
pix_to_check = np.append(neighbor_pix[count][:],[a])
gallist = [catalog2.pix_info.get(b,[]) for b in pix_to_check]
chain = itertools.chain.from_iterable(gallist)
dec_array2 = np.array([catalog2.dec[tmp] for tmp in chain])
chain = itertools.chain.from_iterable(gallist)
ra_array2 = np.array([catalog2.ra[tmp] for tmp in chain])
chain = itertools.chain.from_iterable(gallist)
galcount2 = np.array([catalog2.galcount[tmp] for tmp in chain])
dis = self.calc_distance(self.ra[count], self.dec[count], ra_array2,dec_array2) #in radians
try:
sorted_dis = np.argsort(dis)[0]
best_gal = galcount2[sorted_dis]
dis = dis[sorted_dis]
dis = np.degrees(dis)*3600.0 #back to arcsec
except ValueError:
dis = np.inf
else:
dis = np.inf
if dis >maxsep:
best_gal = -999
dis = -999
matches[self.galcount[count]] = (best_gal, dis)
return matches
def getIntersection(self, aschema, p1, p2, nside=8, nest=True):
"""
Given an association schema, determine which files intersect
"""
idx = []
if aschema=='halogalaxy':
for p in p1:
pix = [p]
nbrs = hp.get_all_neighbours(nside, p, nest=nest)
pix.append(nbrs)
pix = np.array(pix)
#iterate through pixel lists of secondary file
#type. If any pixels in these files are
#neighbors of the primary pixel then we
#need to read this file
for i, ip in enumerate(p2):
fidx = np.in1d(ip, pix)
if fidx.any():
idx.append(i)
return np.array(set(idx))
self.zeropoint_shiftmap.data.field(filter), numpy.radians(90.0 - dec), numpy.radians(ra)
)
else:
zeropoint_shift = self.zeropoint_shiftmap.data.field(filter)[pix]
# Use median zeropoint shift for objects outside interpolated region
median_fitted_zeropoint_shift = numpy.median(zeropoint_shift[numpy.fabs(zeropoint_shift) <= 10.0])
zeropoint_shift[numpy.fabs(zeropoint_shift) > 10.0] = median_fitted_zeropoint_shift
return zeropoint_shift
############################################################
"""
# Fill in peripheral cells of shiftmap with average of nearest neighbors
neighbor = numpy.take(self.zeropoint_shiftmap.data.field(filter),
healpy.get_all_neighbours(self.nside, range(0, healpy.nside2npix(self.nside))))
neighbor_mask = numpy.ma.masked_array(neighbor, neighbor == healpy.UNSEEN)
mean_neighbor = neighbor_mask.mean(axis=0)
indices = numpy.nonzero(numpy.logical_and(mean_neighbor > 0,
self.zeropoint_shiftmap.data.field(filter) == healpy.UNSEEN))[0]
neighbor_shiftmap = self.zeropoint_shiftmap.data.field(filter)
neighbor_shiftmap[indices] = numpy.array(mean_neighbor[indices].compressed().tolist(), dtype=neighbor_shiftmap.dtype)
"""
# Plot
# if plot:
# ra_center, dec_center = numpy.median(ra), numpy.median(dec)
def get_random_cata_heal(ipath, ifile, mfile, nside, ofile="rand_points.fits", mode="grow", opath="./", do_plot=True):
"""Given an input catalog (ifile) and mask (mfile, in healpix format,
0-masked, 1-unmasked), this function generate
a random catalog. The input catalog should contains ra and dec in
equitorial coordinate system and the output will in galactic coordinate.
mode = grow -> grows the mask by one healpix pixel
mode = shrink -> shrink the mask by one pixel (not yet implimented)
mode = None -> Nothing will be done to the mask
"""
ofile = os.path.join(opath, ofile)
ifile = os.path.join(ipath, ifile)
# Read input RA and DEC
f = fits.open(ifile)[1].data
ra = f["ra"]
dec = f["dec"]
# Convert RA DEC to l and b
coords = co.ICRS(ra=ra, dec=dec, unit=(u.degree, u.degree))
ra = coords.galactic.l.degree
dec = coords.galactic.b.degree
# Generating the corresponding healpix pixels
healpixels = return_healpixel(ra, dec, nside=nside, nest=False)
uniquepixels = np.unique(healpixels)
# Generating angular mask from input file
mask = np.zeros(hp.nside2npix(nside)).astype(np.int)
mask[uniquepixels] = 1
if mode == "grow":
# Generating neighbour pixels. This will grow the mask
neigh_pixels = hp.get_all_neighbours(nside, uniquepixels).ravel()
neigh_pixels = np.unique(neigh_pixels)
mask[neigh_pixels] = 1
elif mode == "shrink":
pass
# mask = hp.smoothing(mask, 0.01)
# mask = np.where(mask > 0.1, 1, 0)
# The given mask
imask = hp.read_map(mfile)
imask = hp.ud_grade(imask, nside)
# Union of input catalog mask and input mask
mask = (imask * mask).astype("bool")
# Unique unmasked pixels
uniquepixels = np.arange(hp.nside2npix(nside))[mask]
# generating random points and their corresponding healpix pixels
rra = np.random.uniform(0, 360, 100000)
rdec = np.random.uniform(-90, 90, 100000)
healpixels_rand = return_healpixel(rra, rdec, nside=nside, nest=False)
# Random pixels found in unmasked pixels
i = np.in1d(healpixels_rand, uniquepixels)
rra = rra[i]
rdec = rdec[i]
write_fits_table(ofile, ["RA", "DEC"], [rra, rdec])
if do_plot:
hp.mollview(mask, coord="G", fig=1)
hp.mollview(mask, coord="G", fig=2)
pl.figure(3)
pl.scatter(rra, rdec, c="k", label="Random")
pl.scatter(ra, dec, c="r", s=0.2, edgecolor="r", label="Groups")
pl.show()
quality_flag = self.zeropoint_shiftmap.data.field('n_des')[pix]
quality_flag[quality_flag > 0] = 0
quality_flag[quality_flag < 0] = 2
if self.fill_periphery:
quality_flag[numpy.logical_and(quality_flag == 2, numpy.fabs(zeropoint_shift) <= 10.)] = 1
# Use median zeropoint shift for objects outside interpolated region
median_fitted_zeropoint_shift = numpy.median(zeropoint_shift[numpy.fabs(zeropoint_shift) <= 10.])
zeropoint_shift[numpy.fabs(zeropoint_shift) > 10.] = median_fitted_zeropoint_shift
return zeropoint_shift, quality_flag
############################################################
'''
# Fill in peripheral cells of shiftmap with average of nearest neighbors
neighbor = numpy.take(self.zeropoint_shiftmap.data.field(filter),
healpy.get_all_neighbours(self.nside, range(0, healpy.nside2npix(self.nside))))
neighbor_mask = numpy.ma.masked_array(neighbor, neighbor == healpy.UNSEEN)
mean_neighbor = neighbor_mask.mean(axis=0)
indices = numpy.nonzero(numpy.logical_and(mean_neighbor > 0,
self.zeropoint_shiftmap.data.field(filter) == healpy.UNSEEN))[0]
neighbor_shiftmap = self.zeropoint_shiftmap.data.field(filter)
neighbor_shiftmap[indices] = numpy.array(mean_neighbor[indices].compressed().tolist(), dtype=neighbor_shiftmap.dtype)
'''
# Plot
#if plot:
# ra_center, dec_center = numpy.median(ra), numpy.median(dec)
def boarder_to_lines(boarder, nside, verbose=False): """ takes a list of pixels (boarder) and converts it to a list of positions represnting a line tracing the boarder """ #============================================================================================== ### dot every point along the ring # theta, phi = hp.pix2ang(nside, boarder) # pos = [list(theta), list(phi)] # return [pos] #============================================================================================== ### walk around the rings npix = hp.nside2npix(nside) pix = np.arange(npix) boarder_truth = np.zeros((npix,),bool) ### original boarder boarder_truth[boarder] = True truth = np.zeros((npix,),bool) ### we change this to denote which pixels have not been visited truth[boarder] = True visit = np.zeros((npix,),bool) ### pixels we have visited ipix = pix[truth][0] ### pull out the first pixel truth[ipix] = False ### turn that pixel off visit[ipix] = True line = [ipix] ### start of this line lines = [] ### iterate over boarder while truth.any(): if verbose: print "%d remaining pixels"%np.sum(truth) print "ipix : %d"%ipix for n in hp.get_all_neighbours(nside, ipix): if n == -1: ### neighbour doesn't exist pass elif boarder_truth[n]: ### neighbour is in boarder if verbose: print "\t%d in boarder"%n if truth[n]: ### pixel has not been visited if verbose: print "\t\thas not been visited" line.append( n ) ipix = n truth[n] = False visit[n] = True break else: ### pixel has been visited. End line and start another if verbose: print "\t\thas been visited" line.append( n ) lines.append( line ) truth[n] = False visit[n] = True ### find a new starting point! for _ipix in pix[visit]: ### all pixels we've visited for _n in hp.get_all_neighbours(nside, _ipix): if truth[_n]: ### neighbours a pixel we haven't seen if verbose: print "\t\t\tnew spur at %d"%_n line = [_ipix, _n] truth[_n] = False visit[_n] = True ipix = _n break else: ### didn't find any new spurs starting at _ipix, continue continue break ### we did find a new spur! else: ### didn't find any new spurs from any pixel we have visited if verbose: print "\t\t\tno new spur found" if truth.any(): ### there are still pixels to be found ipix = pix[truth][0] if verbose: print "\t\t\tnew ring at %d"%ipix truth[ipix] = False visit[ipix] = True line = [ipix] break else: ### neighbour is not in boarder if verbose: print "\t%d not in boarder"%n else: ### no neighbours are in boarder_truth. How did we get to this pixel? raise StandardError, "no neighbours aroudn %d found in boarder? How did we get to this pixel?"%ipix ### just end the line and start another ### check that we've visited all pixels if (visit != boarder_truth).any(): raise StandardError, "visit != boarder_truth. Somehow we missed pixels?" ### close the remaining line for n in hp.get_all_neighbours(nside, ipix): if n == -1: pass elif boarder_truth[n]: line.append( n ) ### close the line break else: raise StandardError, "hanging contour line ending at %d. How did we get to this pixel?"%ipix lines.append( line ) ### transform pixel numbers into coords for plotting pos = [] for line in lines: theta, phi = hp.pix2ang(nside, line) pos.append( (list(theta), list(phi)) ) ### remove spurious lines? ### these may be caused by cutting a corner and then coming back and going the other way around. ### a characterisitc would be that there are 3 points in the line, and all 3 points are neighbours of one another return pos
def find_neighbors(nside, pix_idx, n_neighbors):
'''
Find the neighbors of each pixel.
Each pixel is defined by a HEALPix nside and pixel index (in nested
order).
Returns two arrays:
neighbor_idx (n_pix, n_neighbors) Index of each neighbor in
the nside and pix_idx
arrays.
neighbor_dist (n_pix, n_neighbors) Distance to each neighbor.
'''
# Determine (l, b) and which downsampled pixel
# each (nside, pix_idx) combo belongs to
nside_rough = np.min(nside)
if nside_rough != 1:
nside_rough /= 2
l = np.empty(pix_idx.size, dtype='f8')
b = np.empty(pix_idx.size, dtype='f8')
pix_idx_rough = np.empty(pix_idx.size, dtype='i8')
nside_unique = np.unique(nside)
for n in nside_unique:
idx = (nside == n)
factor = (n / nside_rough)**2
pix_idx_rough[idx] = pix_idx[idx] / factor
l[idx], b[idx] = hputils.pix2lb(n, pix_idx[idx], nest=True)
# For each downsampled pixel, determine nearest neighbors of all subpixels
neighbor_idx = -np.ones((pix_idx.size, n_neighbors), dtype='i8')
neighbor_dist = np.inf * np.ones((pix_idx.size, n_neighbors), dtype='f8')
for i_rough in np.unique(pix_idx_rough):
rough_neighbors = hp.get_all_neighbours(nside_rough, i_rough, nest=True)
idx_centers = np.argwhere(pix_idx_rough == i_rough)[:,0]
tmp = [np.argwhere(pix_idx_rough == i)[:,0] for i in rough_neighbors]
tmp.append(idx_centers)
idx_search = np.hstack(tmp)
dist = gc_dist(l[idx_centers], b[idx_centers],
l[idx_search], b[idx_search])
tmp = np.argsort(dist, axis=1)[:, 1:n_neighbors+1]
fill = idx_search[tmp]
neighbor_idx[idx_centers, :fill.shape[1]] = fill
fill = np.sort(dist, axis=1)[:, 1:n_neighbors+1]
neighbor_dist[idx_centers, :fill.shape[1]] = fill
return neighbor_idx, neighbor_dist
def genAngContours(pairs, neighbors, pix_dict, pfd, nfd, chunknum, chunks, real=None, rfd=None):
print len(neighbors)
#objids_removed = 0
#rem_angles = []
if ((real != None) and (rfd != None) and not cs['REAL_PAIRS']):
real = real[1:].astype(float)
real = real[np.argsort(real[:,rfd['angle']])]
angles = real[:,-1]
#angles = angles[np.argsort(angles)]
#real = real[np.argsort(angles)]
#print pairs[0]
#print pfd
#print min(angles)
#print max(angles)
lb = (chunknum * len(pairs) / chunks)
ub = ((chunknum+1) * len(pairs) / chunks)
pairs = pairs[1:]
neighbors = neighbors[1:]
pair_objid1 = pairs[:,pfd['objid1']]
pair_objid2 = pairs[:,pfd['objid2']]
neighbor_objid = neighbors[:,nfd['objid']]
pairs = pairs.astype(np.float)
neighbors = neighbors.astype(np.float)
print neighbors[:5,nfd['weights']]
pairs = pairs[lb:ub]
no_neighbors = 0
oob_pairs = 0
oob_neighbors = 0
x_bins = np.linspace(-1. * half_width, half_width, cs['xbins'])[1:-1]
y_bins = np.linspace(-1. * half_height, half_height, cs['ybins'])[1:-1]
grid = np.zeros(shape=(cs['xbins'], cs['ybins']))
t = time.time()
print 'Adding to contours...'
for p in range(len(pairs)):
print (p + lb)
curr = pairs[p]
near_pix = np.append(hp.get_all_neighbours(nside, int(curr[pfd['pix']]), nest=True), [curr[pfd['pix']]])
near_pix = near_pix[np.where(near_pix != -1)[0]]
candidate_inds = []
for j in near_pix:
#print j
try:
candidate_inds.append(pix_dict[int(j)])
except KeyError:
pass
candidate_inds = np.concatenate(candidate_inds)
#a = len(candidate_inds)
#print '###'
#print neighbor_objid[candidate_inds]
#print pair_objid1[p]
#print pair_objid2[p]
#t = neighbor_objid[candidate_inds]
#out_inds = np.where(
# (t == pair_objid1[p].astype('str'))
# | (t == pair_objid2[p].astype('str'))
# )[0]
#print out_inds
#candidate_inds = candidate_inds[np.where(
# (neighbor_objid[candidate_inds] != pair_objid1[p].astype('str'))
# & (neighbor_objid[candidate_inds] != pair_objid2[p].astype('str'))
# )[0]]
#a -= len(candidate_inds)
#objids_removed += a
#print len(candidate_inds)
candidates = neighbors[np.array(candidate_inds)]
#candidates=neighbors
#print curr[pfd['ra1']]
if cs['REAL_PAIRS']:
distances = cf.physicalDistance(
(curr[pfd['ra_mid']], curr[pfd['dec_mid']]),
zip(candidates[:,nfd['ra_gal']], candidates[:,nfd['dec_gal']]),
np.array([curr[pfd['z']]] * len(candidates)) if pair_adds else candidates[:,nfd['z']]
)
else:
curr_angle = curr[pfd['angle']]
#print curr_angle
match_ind = np.searchsorted(angles, curr_angle)
#print angles[match_ind]
real_z = real[match_ind, rfd['z']]
distances = cf.physicalDistance(
(curr[pfd['ra_mid']], curr[pfd['dec_mid']]),
zip(candidates[:,nfd['ra_gal']], candidates[:,nfd['dec_gal']]),
np.array([real_z] * len(candidates)) if pair_adds else candidates[:,nfd['z']]
)
#print real_z
#print curr_angle
#print real_z
pair_radius = cf.physicalDistance(
(curr[pfd['ra_mid']], curr[pfd['dec_mid']]),
[(curr[pfd['ra1']], curr[pfd['dec1']])],
[curr[pfd['z']] if cs['REAL_PAIRS'] else real_z]
)
#print pair_radius
print len(candidates)
inbounds = candidates[np.where(distances < cs['max_distance'])[0]]
if len(inbounds) == 0: continue
print len(inbounds)
#print len(inbounds)
oob_neighbors += len(candidates) - len(inbounds)
distances = distances[np.where(distances < cs['max_distance'])[0]]
#print distances[5:10]
unrotated_neighbors = np.ones((len(inbounds), 3))
unrotated_neighbors[:,1] = inbounds[:,nfd['ra_gal']]
unrotated_neighbors[:,2] = inbounds[:,nfd['dec_gal']]
unrotated_mid = (1., curr[pfd['ra_mid']], curr[pfd['dec_mid']])
unrotated_right = (1., curr[pfd['ra1']], curr[pfd['dec1']])
rotated_mid, rotated_right, rotated_neighbors = sg.rotate(
unrotated_mid,
unrotated_right,
unrotated_neighbors
)
#print cf.physicalDistance(
# (rotated_mid[1], rotated_mid[2]),
# zip(rotated_neighbors[:,1], rotated_neighbors[:,2]),
# np.array([curr[pfd['z']]] * len(rotated_neighbors)) if pair_adds else candidates[:,nfd['z']])[5:10]
if (pair_radius > 5 or pair_radius < 3):
oob_pairs +=1
print 'oob'
continue
print len(rotated_neighbors)
neighbor_angles = sg.calc_distance(
(rotated_mid[1], rotated_mid[2]),
zip(rotated_neighbors[:,1], rotated_neighbors[:,2])
)
neighbor_angles = np.array(neighbor_angles)
#rem_angles.append(neighbor_angles[out_inds])
#print np.shape(neighbor_angles)
#a = len(neighbor_angles)
#print neighbor_angles[:5]
#print np.where(neighbor_angles > .0000096)
#neighbor_angles = neighbor_angles[np.where(neighbor_angles > .0000096)[0]]
#print np.shape(neighbor_angles)
#objids_removed += (a - len(neighbor_angles))
#print (a - len(neighbor_angles))
#print '###'
xs = np.multiply(np.array(map(np.cos, rotated_neighbors[:,1])), neighbor_angles)
ys = np.multiply(np.array(map(np.sin, rotated_neighbors[:,1])), neighbor_angles)
xbs = map(lambda j: bs.bisect(x_bins, j), xs)
ybs = map(lambda j: bs.bisect(y_bins, j), ys)
def add_to_bins(ybin, xbin, weight):
grid[ybin, xbin] += weight
return
print inbounds[:5,nfd['weights']]
if cs['WEIGHTED']: weights = inbounds[:,nfd['weights']]
else: weights = np.array([1.] * len(inbounds))
map(add_to_bins, ybs, xbs, weights)
#print len(weights)
#print oob_neighbors
#rem_angles = np.concatenate(rem_angles)
#print rem_angles
print np.max(grid)
#print objids_removed
return grid
def set_pixel_list(self):
"""creates a list of all pixels and their neighbors for quicker matching"""
self.pix_and_neigh = set(set(list(self.pix)).union(set(list(hp.get_all_neighbours(self.NSIDE, self.pix).flatten()))))
return self.pix_and_neigh
# print hp.get_nside(test_map) # print hp.maptype(test_map) # print hp.get_map_size(test_map) # print len(test_map) # print test_map[0:10]*np.float(fact) # print test_map n = 16 rang = range(0,12*n**2) npix = hp.nside2npix(n) angs = hp.pix2ang(n,rang) phi = angs[0] theta = angs[1] nb = hp.get_all_neighbours(n, 3.1, 888.8, nest=True) # print npix # print type(angs) # print len(angs) # print nb print tmap print len(tmap) print type(tmap[0]) print tmap[0] print len(tmap[0]) print np.shape(tmap) print np.average(tmap)
def EqualAreaSamples(ra,dec,nside,diff_max=0.05,it_max=100,npix_ini=10,nest=True,verbose=True,optimize=True):
#
# Divide a ra,dec list into equal area samples. Elements == 0 are put in the same sample '-1'.
# Try to optimise the variance in number of entries per pixels normalised by the mean to be below diff_max
#
# diff_max : Normalised difference in number of pixels (max(abs(nentries - mean(nentries)))/mean(nentries)) below wich optimization is considered succesful
# it_max : Maximum number of iteration in optimisation process
# npix_ini : Initial number of pix to exchange at each step (must be small to obtain satisfactory samples)
#
# Note : Variance exclude isolated samples
#
# Return two lists:
# - pixel number (in healpy standard)
# - Sample to which the pixel belongs
# Both lists have len = len(map)
#
import healpy as hp
import numpy as np
from sklearn.neighbors import BallTree
# import random
theta,phi = RaDecToThetaPhi(ra,dec,degree=True)
samples = hp.ang2pix(nside,theta,phi,nest=nest)
'''If optimize == false return those samples'''
set_samples = list(set(samples))
if optimize == False:
set_samples_sort = np.sort(set_samples)
for i,isample in enumerate(set_samples_sort):
samples[samples==isample] = i
return samples
'''Get nb. entries per sample'''
nentries = [len(samples[samples==i]) for i in set_samples]
'''Get neighbours of samples'''
#neighb = np.transpose(hp.get_neighbours(nside,set_samples,nest=nest)[0])
neighb = np.transpose(hp.get_all_neighbours(nside,set_samples,nest=nest))
good_neighb = []
for i,ineighb in enumerate(neighb):
good_neighb_i = []
i_theta,i_phi = hp.pix2ang(nside,set_samples[i],nest=nest)
for isamp in ineighb:
if isamp in set_samples:
if set_samples.index(isamp) == i:
continue
isamp_theta,isamp_phi = hp.pix2ang(nside,isamp,nest=nest)
if (isamp_theta == i_theta) | (isamp_phi == i_phi):
continue
good_neighb_i.append(set_samples.index(isamp))
good_neighb.append(good_neighb_i)
neighb = good_neighb
'''Remove sample with no neighbours'''
set_samples_n = [x for x,y in zip(set_samples,neighb) if len(y)>0]
nentries_n = [x for x,y in zip(nentries,neighb) if len(y)>0]
neighb_n = [x for x,y in zip(neighb,neighb) if len(y)>0]
'''Get RA DEC of samples'''
samp_theta,samp_phi = hp.pix2ang(nside,set_samples_n,nest=nest)
samp_ra,samp_dec = ThetaPhiToRaDec(samp_theta,samp_phi,degree=False)
samp_radec = np.transpose([samp_ra,samp_dec])
'''Get starting mean and diff'''
mean = np.mean(nentries_n)
diff = np.max(np.abs(nentries_n-mean))/float(mean)
if verbose:
print 'Initial diff :',diff
'''Start optimization'''
n_it = 0
ind = np.arange(len(ra))
np.random.seed(1)
inf_loop = 0
change_samp = True
while float(diff) > diff_max:
if diff*mean > 2*npix_ini:
npix_ex = npix_ini
else:
npix_ex = int(diff*mean/2.)
if inf_loop >= 100:
npix_ex = int(npix_ex/2)
inf_loop = 0
'''Get random sample'''
if change_samp:
chage_samp = False
iprev = isamp
while isamp == iprev:
isamp = np.random.choice(np.arange(len(set_samples_n)))
'''Look for bigger neighbours'''
samp_neighbs = [set_samples_n[i] for i in neighb_n[isamp] if nentries_n[i]>nentries_n[isamp]+npix_ex]
if len(samp_neighbs)==0:
change_samp = True
inf_loop+=1
continue
'''Get objects in those neighbouring samples'''
good_ra = np.array([ra[i] for i in ind if samples[i] in samp_neighbs])
good_dec = np.array([dec[i] for i in ind if samples[i] in samp_neighbs])
good_ra*=np.pi/180.
good_dec*=np.pi/180.
good_ind = [i for i in ind if samples[i] in samp_neighbs]
'''Construct BallTree with pixels of biggest samples'''
radec = np.transpose([good_ra,good_dec])
tree = BallTree(radec, leaf_size = 1, metric='haversine')
exchange_radec = tree.query(samp_radec[isamp],k=npix_ex,return_distance=False)[0]
'''Exchange ra,dec from bigger samples to smaller sample'''
exchange_ind = [good_ind[i] for i in exchange_radec]
samples[exchange_ind] = set_samples_n[isamp]
nentries_n = [len(samples[samples==i]) for i in set_samples_n]
'''Recompute mean and diff'''
diff = np.max(np.abs(nentries_n-mean))/float(mean)
if verbose:
print 'nentries :',nentries_n,' mean :',mean,' n_it :',n_it
'''Increase iter'''
n_it+= 1
if n_it >= it_max:
if verbose:
print 'Aborting optimization after',n_it,'iterations'
break
'''Correction'''
if verbose:
print 'Final diff :',diff
print nentries_n,mean
set_samples_sort = np.sort(set_samples)
for i,isample in enumerate(set_samples_sort):
samples[samples==isample] = i
return samples
for hpx4index in hpx4indices:
print 'working on hpx pixel', hpx4index + 1, '/', len(hpx4indices)
hpx64index_bin = [bin(hpx4index)]
for t in range(int(np.log2(64) - np.log2(4))):
hpx64index_bin = [a + '00' for a in hpx64index_bin] + \
[a + '01' for a in hpx64index_bin] + \
[a + '10' for a in hpx64index_bin] + \
[a + '11' for a in hpx64index_bin]
hpx64index = [int(a, 2) for a in hpx64index_bin]
hpx64index_enlarged = np.unique(
reduce(
lambda a, b: a + b,
[
list(healpy.get_all_neighbours(64, a, nest=True))
for a in hpx64index
]
))
hpx64index_enlarged = np.unique(
reduce(
lambda a, b: a + b,
[
list(healpy.get_all_neighbours(64, a, nest=True))
for a in hpx64index_enlarged
]
))
# now compute the 2048s indices for this tile, note for targetnside==1024
# this is of course 1024 ;-)
uniform_sky = np.ones(npix)*100.#*u.K #completely arbitrary choice of noise level XXX UN-USED RIGHT NOW noise = np.zeros(npix) for i in range(npix): noise[i] = random.uniform(-100,100)#* u.K ####uniform sky tests and point source tests if opts.map is None: sky=uniform_sky elif opts.map is 'point': sky = np.zeros(npix) #define a point source theta0 = np.pi/2. phi0 = 0. pix0 = hp.ang2pix(nside,theta0,phi0) #make it slightly less point-like nbs = hp.get_all_neighbours(nside,theta0,phi=phi0) sky[pix0]=500#*u.K for nb in nbs: sky[nb]=500#*u.K sky = rotate_hmap(sky,[180,0]) else: sky = hp.read_map(opts.map) #promote sky to matrix for frequency axis sky = np.outer(np.ones(nfreq),sky)*pow(nu/150e6,-0.7) #decompose sky into alm's n_alm = len(m) alm = np.zeros((nfreq,n_alm),dtype='complex128') print 'Calculating sky a_lm values:' for i in range(nfreq): if not opts.noverb: print nu[i,0]/1e6,'MHz' alm[i,:] = hp.map2alm(sky[i,:],lmax=lmax,iter=3)
def jones2celestial_basis(jones, z0_cza=None):
if z0_cza is None:
z0_cza = np.radians(120.7215)
npix = jones.shape[0]
nside = hp.npix2nside(npix)
hpxidx = np.arange(npix)
cza, ra = hp.pix2ang(nside, hpxidx)
z0 = irf.r_hat_cart(z0_cza, 0.)
RotAxis = np.cross(z0, np.array([0,0,1.]))
RotAxis /= np.sqrt(np.dot(RotAxis,RotAxis))
RotAngle = np.arccos(np.dot(z0, [0,0,1.]))
R_z0 = irf.rotation_matrix(RotAxis, RotAngle)
R_jones = irf.rotate_jones(jones, R_z0, multiway=True) # beams are now pointed at -31 deg latitude
jones_out = np.zeros((npix, 2,2), dtype=np.complex128)
########
## This next bit is a routine to patch the topological hole by grabbing pixel
## data from a neighborhood of the corrupted pixels.
## It uses the crucial assumption that in the ra/cza basis the dipoles
## are orthogonal at zenith. This means that for the diagonal components,
## the zenith pixels should be a local maximum, while for the off-diagonal
## components the zenith pixels should be a local minimum (in absolute value).
## Using this assumption, we can cover the corrupted pixel(s) in the
## zenith neighborhood by the maximum pixel of the neighborhood
## for the diagonal, and the minimum of the neighborhood for the off-diagonal.
## As long as the function is relatively flat in this neighborhood, this should
## be a good fix
jones_b = transform_basis(nside, R_jones, z0_cza, R_z0)
cf = [np.real,np.imag]
u = [1.,1.j]
z0pix = hp.vec2pix(nside, z0[0],z0[1],z0[2])
if nside < 128:
z0_nhbrs = hp.get_all_neighbours(nside, z0_cza, phi=0.)
else:
z0_nhbrs = neighbors_of_neighbors(nside, z0_cza, phi=0.)
jones_c = np.zeros((npix,2,2,2), dtype=np.float64)
for k in range(2):
jones_c[:,:,:,k] = cf[k](jones_b)
for i in range(2):
for j in range(2):
for k in range(2):
z0_nbhd = jones_c[z0_nhbrs,i,j,k]
if i == j:
fill_val_pix = np.argmax(abs(z0_nbhd))
fill_val = z0_nbhd[fill_val_pix]
else:
fill_val_pix = np.argmin(abs(z0_nbhd))
fill_val = z0_nbhd[fill_val_pix]
jones_c[z0_nhbrs,i,j,k] = fill_val
jones_c[z0pix,i,j,k] = fill_val
jones_out = jones_c[:,:,:,0] + 1j*jones_c[:,:,:,1]
return jones_out
def plot_sky_projection_healpy_linear_zoom(Sliced_Halo_data,Output_Para):
rc('font',family='serif')
fdir = './Output/plots/HEALPixZoom/'
nside = Output_Para.nside
Sl_n = len(Sliced_Halo_data)
Z_max = max(Sliced_Halo_data[Sl_n-1].Z_red[:])
ques = True
while (ques):
k = Read_Integer_Input("We have %i redshift slice which one you want exctract the halo ? "%Sl_n)
k = int(k) - 1
while(k > (Sl_n-1) or k < 0 ):
print "Invalid Number."
k = Read_Integer_Input("Please choose an integer between 1 and %i : "%Sl_n)
k = int(k) - 1
pix = zeros(12*nside**2)
n = len(Sliced_Halo_data[k].RA[:])
if (n == 0):
print "There is no halo in this piece of redshift, please choos another one ..."
raw_input("Press enter to continue ... ")
break
for i in range(n):
j = hp.ang2pix(nside,DtoR*(90.0-Sliced_Halo_data[k].DEC[i]),DtoR*(Sliced_Halo_data[k].RA[i]))
neighbour = hp.get_all_neighbours(nside,j)
Fx = 10.0**Sliced_Halo_data[k].lgFx[i]
pix[j] += Fx/2.0
pix[neighbour[0]] += Fx/16.0
pix[neighbour[1]] += Fx/16.0
pix[neighbour[2]] += Fx/16.0
pix[neighbour[3]] += Fx/16.0
pix[neighbour[4]] += Fx/16.0
pix[neighbour[5]] += Fx/16.0
pix[neighbour[6]] += Fx/16.0
pix[neighbour[7]] += Fx/16.0
halo_n = Read_Integer_Input("We have %i halos in this slice which one you want to exctract ? "%n)
halo_n = halo_n - 1
while(halo_n > (n-1) or halo_n < 0 ):
print "Invalid Number."
halo_n = Read_Integer_Input("Please choose an integer between 1 and %i : "%n)
halo_n = halo_n - 1
# TEST
# for i in range(halo_n,halo_n+1):
# neighbour = hp.get_all_neighbours(nside,DtoR*(90.0-Sliced_Halo_data[k].DEC[i]),DtoR*(Sliced_Halo_data[k].RA[i]))
# j = hp.ang2pix(nside,DtoR*(90.0-Sliced_Halo_data[k].DEC[i]),DtoR*(Sliced_Halo_data[k].RA[i]))
# pix[j] += 2.0
# pix[neighbour[0]] += 1.0
# pix[neighbour[1]] += 1.0
# pix[neighbour[2]] += 1.0
# pix[neighbour[3]] += 1.0
# pix[neighbour[4]] += 1.0
# pix[neighbour[5]] += 1.0
# pix[neighbour[6]] += 1.0
# pix[neighbour[7]] += 1.0
# print j, neighbour
# neighbour = hp.get_all_neighbours(nside,j)
# print j, neighbour
clf()
hp.gnomview(pix, fig=1 , rot=(Sliced_Halo_data[k].RA[halo_n],Sliced_Halo_data[k].DEC[halo_n],0.0) , xsize=Output_Para.xsize , reso=Output_Para.resolution , degree=Output_Para.resol_degree)
hp.graticule()
show()
# clf()
# hp.mollview(pix, fig = 1)
# hp.graticule()
# show()
close()
save_ques = Read_YN_Input("Do you want to save its picture (please enter Y, y, N, or n)? ")
if save_ques :
rc('text',usetex=True)
clf()
hp.gnomview(pix, fig=1 , rot=(Sliced_Halo_data[k].RA[halo_n],Sliced_Halo_data[k].DEC[halo_n],0.0) , xsize=Output_Para.xsize , reso=Output_Para.resolution , degree=Output_Para.resol_degree)
hp.graticule()
fname = 'sky_projection_HEALPix_Linear_%i_%i_%i.pdf'%(nside,(halo_n+1),(k+1))
title(r'Redshift is between %0.2f and %0.2f'%(Sliced_Halo_data[k].z_min,Sliced_Halo_data[k].z_max),fontsize = 20)
print 'Saving plot', fname
# savefig(fdir+fname,bbox_inches='tight')
savefig(fdir+fname)
rc('text',usetex=False)
close()
ques = False
# ques = Read_YN_Input("Do you want to extract another halo (please enter Y, y, N, or n)? ")
return 0
def DESm2h(tolyfile, nside_out, nside_tiny, coord='GAL',weights=['maglims'], nside_large=16, nside_ini=128, project='OPS' ):
"""
Routine designed to convert DES mangle mask to healpix masks without having the full combined masks
The only combined mask that is required is the "tolygon polygons" file, that represent the footprint of the mask at the tile level
Inputs:
tolyfile : toylogon file
Parameters:
nside_out: Nside of the final Healpix mask that are produced (typically nside_out=2048 or 4096)
nside_tiny: Nside of the finest resolution (typically nside_tiny=16384 or 32768)
weights: list os strings that indicate what maps are to be produced. Typically weights=['maglims', 'bitmask', 'time', 'weight']
nside_large: very small nside (coarse resultion to minimize time. It is faster to run polyid on a big array of points than multiple polyid calls with smaller arrays ). Typically nside_large=8 or 16
nside_ini : smallest nside needed tobe sure to probe all the tolygon file. One tile is .5 sq deg. that requires probing the nside=128 pixels
coord: coordinante system. Should be only GAL or EQU
project: Either ACT or OPS. used to find the mangle files on deslogin
Outputs:
N+1 healpix maps at resolution Nside_out, in Nested format and in the desired coordinates, where N is the number of weights considered. The first map (ind =0), is the detection fraction, i.e. the fraction of tiny pixel that are not 0 inside a final pixel.
"""
if coord!='GAL' and coord!='EQU':
print "Please indidicate a correct coordinate system. Use either GAL or EQU"
return -1
nmaps=len(weights)
maps=nm.zeros((12*nside_map_final**2, nmaps+1))+hp.UNSEEN
####################################################################################################
### 1. get at least one healpix pixel center within each tile of the mask. One tile is .5 sq deg. that requires probing the nside=128 pixels
######################################################
## load ra, dec of healpix center for nside=nside_ini and galactic coordinates
print 'Loading healpix center coordinates for nside= %d'%nside_ini
if coord=='GAL':
ra,dec= mu.get_hpx_coord_GAL_inradec(nside_ini)
if coord=='EQU':
ra,dec= mu.get_hpx_coord_radec(nside_ini)
#Load tolygon files
print 'loading tolygons file'
mng=pym.Mangle(tolyfile)
print 'done'
##### get the weight of the nside=128 pixels against the tolygon mask. Tell whether pixels are in the mask
w=mng.weight(ra,dec)
ind=nm.where(w !=0)[0]
### ind contains the pixels that are in the mask. to make sure that we are indeed having all the pixels that have even the tiniest overlap with the mask, we take twice the neighbouring pixels
a=hp.get_all_neighbours(nside_ini,ind, nest=True)
a=a[a>0]
b=nm.concatenate([a, ind])
b=nm.unique(b)
c=hp.get_all_neighbours(nside_ini,b, nest=True)
c=c[c>0]
c=nm.concatenate([c,b])
d=nm.unique(c)
### d now contains all the mid-scale pixels that we are going to consider.
###########################################################################################
### 2. consider even larger pixels to minimize io time. it is faster to run polyid on a big array of points than multiple polyid calls with smaller arrays
##########################################################################################
dlarge=nm.int32(d/4**(nm.log(nside_ini/nside_large)/nm.log(2)))
dlarge=nm.unique(dlarge)
print "There are %d large nside=%d pixels in this mask'"%(dlarge.shape[0], nside_large)
# dlarge now contains the ids of the nside= nside_large pixels
############################################################################################
### 3. loading the table with the mangle_run, coadd_run ad coaddtile_id
############################################################################################
#### Need to add something to get the tiles.dat files
ff=open('/Users/benoitl/Documents/Post_doc/DES/mangle_healpix/dev/y1p1_s82_tiles.dat')
lines=ff.readlines()
N=len(lines)-1
dum1=[]
for i in range(1,N+1):
dum=lines[i].strip().split(',')
dum1.append(dum)
tile_tab=nm.array(dum1)
ff.close()
#### tile_tab contains all the info af the tiles in the mask: coaddtile_id, tilename, mangle_run, coadd_run. Those should be needed to retreive the individual tiles masks
npix_per_fat=nm.int(4**(nm.log(nside_tiny/nside_map_final)/nm.log(2))) ## number of tiny pixels per pixels of the final healpix map.
kk=0
for lpixid in dlarge: ##### loop on the large pixels
kk+=1
print '%d / %d big pixels'%(kk, dlarge.shape[0])
## get all the pixels of the final map for the large pixel lpixid
fat=mu.get_sons2(nm.array([lpixid]), nside_large,nside_map_final)
##these are the ipix of all the tiny hpx pixels within the fat
sons=mu.get_sons2(fat, nside_map_final, nside_tiny)
jmaps=nm.zeros((len(sons), nmaps+1))
####################################################################
### Need to transform back the coord of the sons into EQU to be able to match them agains the mangle mask
Npixel=sons.shape[0]
R=hp.Rotator(coord='gc')
theta,phi=hp.pix2ang(nside_tiny, sons, nest=True)
theta=nm.pi/2 -theta
if coord=='GAL':
ra, dec= R(phi*180/nm.pi, theta*180/nm.pi, lonlat=True)
ra=nm.mod(ra, 360)
if coord=='EQU':
ra=phi*180/nm.pi
dec=theta*180/nm.pi
ra=nm.mod(ra, 360)
ra,dec=mu.perturb_radec(nside_tiny, ra,dec)
#### Need to split the sons pixels according to which tile thy fall in.
p=mng.polyid(ra, dec)
a,b,c=nm.unique(p, return_index=True, return_inverse=True)
print 'There are %d tiles in this large pixel of the sky in which we will look one by one'%(a.shape[0])
#####For each tile within this large pixel lpixid interoggate the mangle masks at the position of the tiny pixels.
for i in range(a.shape[0]):
t1=a[i]
if t1!=-1:
ind_1= nm.where(tile_tab[:,0]=='%s'%t1)[0] ## get the line in tile_tab that correspond to that tile
mangle_run=tile_tab[ind_1,2][0]
coadd_run=tile_tab[ind_1,3][0]
# t1 is the tileid of the firsttile in which there are some pixles.
#ind1=where(c==i) are the indices of those pixels
indd=nm.where(c==i)
verbose=False
# loading the mangle mask for that first tile. The path of the files with have to be modified when ported to the deslogin
if verbose:
print 'loading file %s'%'/Users/benoitl/Documents/Post_doc/DES/DESdata/OPS/Final_masks/mangle/y1p1_s82_copy/%s/mangle/%s_holymolys_weight_i.pol'%(mangle_run, coadd_run)
nmg1=pym.Mangle('/Users/benoitl/Documents/Post_doc/DES/DESdata/OPS/Final_masks/mangle/y1p1_s82_copy/%s/mangle/%s_holymolys_weight_i.pol'%(mangle_run, coadd_run))
w1= nmg1.weight(ra[indd], dec[indd])
jmaps[indd,0]=nm.float64(w1)
# jweight[indd]=nm.float64(w1)
del(w1)
it=0
for wei in weights:
it+=1
### load maglim
nmg1.read_weights('/Users/benoitl/Documents/Post_doc/DES/DESdata/OPS/Final_masks/mangle/y1p1_s82_copy/%s/mangle/%s_molys_i.%s'%(mangle_run, coadd_run, wei))
w1= nmg1.weight(ra[indd], dec[indd])
jmaps[indd, it]=nm.float64(w1)
# jmaglim[indd]=nm.float64(w1)
del(w1)
###fin boucle sur tile
jjmaps=nm.reshape(jmaps, (Npixel/npix_per_fat, npix_per_fat,nmaps+1))
mm=nm.mean(jjmaps, axis=1, dtype=nm.float64)
mfrac=nm.sum(jjmaps[:,:,0]>0, axis=1)/(1.0*npix_per_fat)
maps[fat,1:]=mm[:,1:]
maps[fat,0]=mfrac
del( sons, fat, jjmaps,jmaps,mm, mfrac,)
del(p, a,b,c)
return maps
def contour(m, levels, nest=False, degrees=False, simplify=True):
import pkg_resources
try:
pkg_resources.require('healpy >= 1.9.0')
except:
raise RuntimeError('This function requires healpy >= 1.9.0.')
try:
import networkx as nx
except:
raise RuntimeError('This function requires the networkx package.')
# Determine HEALPix resolution
npix = len(m)
nside = hp.npix2nside(npix)
min_area = 0.4 * hp.nside2pixarea(nside)
# Compute faces, vertices, and neighbors.
# vertices is an N X 3 array of the distinct vertices of the HEALPix faces.
# faces is an npix X 4 array mapping HEALPix faces to their vertices.
# neighbors is an npix X 4 array mapping faces to their nearest neighbors.
faces = np.ascontiguousarray(
np.rollaxis(hp.boundaries(nside, np.arange(npix), nest=nest), 2, 1))
dtype = faces.dtype
faces = faces.view(np.dtype((np.void, dtype.itemsize * 3)))
vertices, faces = np.unique(faces.ravel(), return_inverse=True)
faces = faces.reshape(-1, 4)
vertices = vertices.view(dtype).reshape(-1, 3)
neighbors = hp.get_all_neighbours(nside, np.arange(npix), nest=nest)[::2].T
# Loop over the requested contours.
paths = []
for level in levels:
# Find credible region
indicator = (m >= level)
# Construct a graph of the eges of the contour.
graph = nx.Graph()
face_pairs = set()
for ipix1, ipix2 in enumerate(neighbors):
for ipix2 in ipix2:
# Determine if we have already considered this pair of faces.
new_face_pair = frozenset((ipix1, ipix2))
if new_face_pair in face_pairs: continue
face_pairs.add(new_face_pair)
# Determine if this pair of faces are on a boundary of the
# credible level.
if indicator[ipix1] == indicator[ipix2]: continue
# Add all common edges of this pair of faces.
i1 = np.concatenate((faces[ipix1], [faces[ipix1][0]]))
i2 = np.concatenate((faces[ipix2], [faces[ipix2][0]]))
edges1 = frozenset(frozenset(_) for _ in zip(i1[:-1], i1[1:]))
edges2 = frozenset(frozenset(_) for _ in zip(i2[:-1], i2[1:]))
for edge in edges1 & edges2:
graph.add_edge(*edge)
graph = nx.freeze(graph)
# Record a closed path for each cycle in the graph.
cycles = [
np.take(vertices, cycle, axis=0) for cycle in nx.cycle_basis(graph)]
# Simplify paths if requested
if simplify:
cycles = [_simplify(cycle, min_area) for cycle in cycles]
cycles = [cycle for cycle in cycles if len(cycle) > 2]
# Convert to lists
cycles = [
_vec2radec(cycle, degrees=degrees).tolist() for cycle in cycles]
# Add to output paths
paths.append([cycle + [cycle[0]] for cycle in cycles])
return paths
def maskingAlgorithmGeneralized(myBundles, plotHandler, dataLabel, nside= 128,
findValue= 'unmasked', relation= '=',
newValue= 'masked',
pixelRadius= 6, returnBorderIndices= False,
printInfo= True, plotIntermediatePlots= True,
skyMapColorMin= -0.12, skyMapColorMax= 0.12):
"""
Assign newValue to all pixels in a skymap within pixelRadius of pixels with value <, >, or = findValue.
Required arguments:
* myBundles - a dictionary for metricBundles.
* plotHandler - lsst.sims.maf.plots.plotHandler.PlotHandler object for skymaps and power spectra
* dataLabel - a string describing that date, i.e. 'numGal'
Pre-Set/optional arguments:
* nside - HEALpix resolution parameter. default value: 128
* findValue - if related to mask, must be either 'masked' or 'unmasked'. otherwise, must be a number. default: 'unmasked'
* relation - must be '>','=','<'. default: '='
* newValue - if related to mask, must be either 'masked' or 'unmasked'. otherwise, must be a number. default: 'masked'
* pixelRadius - number of pixels to consider around a given pixel. default: 6
* returnBorderIndices - set to True to return the array of indices of the pixels whose values/mask are changed. default: False
* printInfo - set to False if do not want to print intermediate info. default: True
* plotIntermediatePlots - set to False if do not want to plot intermediate plots. default: True
* skyMapColorMin - colorMin label value for skymap plotDict label. default: -0.12
* skyMapColorMax - colorMax label value for skymap plotDict label. default: 0.12
"""
# find pixels such that (pixelValue (relation) findValue) AND their neighbors dont have that (relation) findValue.
# then assign newValue to all these pixels.
# relation must be '>','=','<'
# data indices are the pixels numbers ..
# make sure that relation is compatible with findValue
if ((findValue == 'masked') | (findValue == 'unmasked')):
if (relation != '='):
print 'ERROR: must have relation== "=" if findValue is related to mask.'
print 'Setting: relation= "="'
relation= '='
print ''
# translate findValue into what has to be assigned
findValueToConsider= findValue
if (str(findValue)).__contains__('mask'):
if (findValue == 'masked'):
findValueToConsider= True
if (findValue == 'unmasked'):
findValueToConsider= False
# translate newValue into what has to be assigned
newValueToAssign= newValue
if (str(newValue)).__contains__('mask'):
if (newValue == 'masked'):
newValueToAssign= True
if (newValue == 'unmasked'):
newValueToAssign= False
borders= {}
for dither in myBundles:
totalBorderPixel= []
# find the appropriate array to look at.
if (str(findValue)).__contains__('mask'):
origArray= myBundles[dither].metricValues.mask.copy() # mask array
else:
origArray= myBundles[dither].metricValues.data.copy() # data array
for r in range(0,pixelRadius):
borderPixel= []
tempCopy= copy.deepcopy(myBundles)
# ignore the pixels whose neighbors formed the border in previous run
if (r != 0):
origArray[totalBorderPixel]= newValueToAssign
# find the pixels that satisfy the relation with findValue and whose neighbors dont
for i in range(0, len(origArray)):
neighborsPixels= hp.get_all_neighbours(nside,i) # i is the pixel number
for j in neighborsPixels:
condition= None
if (relation == '<'):
condition= ((origArray[i] < findValueToConsider) & (origArray[j] >= findValueToConsider))
if (relation == '='):
condition= ((origArray[i] == findValueToConsider) & (origArray[j] != findValueToConsider))
if (relation == '>'):
condition= ((origArray[i] > findValueToConsider) & (origArray[j] <= findValueToConsider))
if (condition == None):
print 'ERROR: invalid relation: ', relation
print 'Aborting.'
stop
if condition:
if (j != -1): # -1 entries correspond to inexistent neighbors
borderPixel.append(i)
borderPixel= np.unique(borderPixel)
totalBorderPixel.extend(borderPixel)
if printInfo:
print 'Border pixels from run', r+1, ':', len(borderPixel)
print 'Total pixels so far: ', len(totalBorderPixel)
print ''
# plot found pixels
if plotIntermediatePlots:
if (str(newValue)).__contains__('mask'):
tempCopy[dither].metricValues.mask[:]= newValueToAssign
tempCopy[dither].metricValues.mask[totalBorderPixel]= not(newValueToAssign)
tempCopy[dither].metricValues.data[totalBorderPixel]= -500
plotDict = {'xlabel': dataLabel, 'title':'%s: %s Round # %s' %(dither, dataLabel, r+1),
'logScale': False, 'labelsize': 9,'colorMin':-550, 'colorMax': 550}
else:
tempCopy[dither].metricValues.mask[:]= True
tempCopy[dither].metricValues.mask[totalBorderPixel]= False
tempCopy[dither].metricValues.data[totalBorderPixel]= newValueToAssign
plotDict = {'xlabel': dataLabel, 'title':'%s %s Round # %s' %(dither, dataLabel, r+1),
'logScale': False, 'labelsize': 9, 'maxl': 500}
tempCopy[dither].setPlotDict(plotDict)
tempCopy[dither].setPlotFuncs([plots.HealpixSkyMap(), plots.HealpixPowerSpectrum()])
tempCopy[dither].plot(plotHandler=plotHandler)
plt.show()
borders[dither]= totalBorderPixel # save the found pixels with the appropriate key
# change the original map/array.
for dither in myBundles:
totalBorderPixel= borders[dither]
if (str(newValue)).__contains__('mask'):
myBundles[dither].metricValues.mask[totalBorderPixel]= newValueToAssign
myBundles[dither].metricValues.data[totalBorderPixel]= 0.0
else:
myBundles[dither].metricValues.data[totalBorderPixel]= newValueToAssign
plotDict = {'xlabel':dataLabel, 'title':'%s: %s MaskedMap; pixelRadius: %s ' %(dither,dataLabel, pixelRadius),
'logScale': False, 'labelsize': 8,'colorMin': skyMapColorMin, 'colorMax': skyMapColorMax}
myBundles[dither].setPlotDict(plotDict)
myBundles[dither].setPlotFuncs([plots.HealpixSkyMap()])
myBundles[dither].plot(plotHandler=plotHandler)
plotDict = {'xlabel':dataLabel, 'title':'%s: %s MaskedMap; pixelRadius: %s ' %(dither,dataLabel, pixelRadius),
'logScale': False, 'labelsize': 12, 'maxl': 500}
myBundles[dither].setPlotDict(plotDict)
myBundles[dither].setPlotFuncs([plots.HealpixPowerSpectrum()])
myBundles[dither].plot(plotHandler=plotHandler)
plt.show()
if returnBorderIndices:
return [myBundles, borders]
else:
return myBundles
