def read_map2(filename,field=0,dtype=np.float64,nest=False,hdu=1,h=False,verbose=True,memmap=False):
hdr=fitsio.read_header(filename,ext=hdu)
fullsky = False
try:
if (hdr['OBJECT'].strip() == 'PARTIAL') :
# partial sky format
fullsky=False
else:
fullsky=True
except:
# if no OBJECT in header, assume full sky
fullsky=True
if fullsky:
m=hp.read_map(filename,field=field,dtype=dtype,nest=nest,hdu=hdu,h=h,verbose=verbose,memmap=memmap)
else:
# partial sky
st=fitsio.read(filename,ext=1)
nside=hdr['NSIDE']
m=np.zeros(12*nside*nside,dtype=dtype) + hp.UNSEEN
if ((hdr['ORDERING'].strip() == 'NESTED') and (not nest)) :
# change from nest to ring...
m[hp.nest2ring(nside,st['PIXEL'])] = st['SIGNAL']
elif ((hdr['ORDERING'].strip() == 'RING') and (nest)):
# change from ring to nest...
m[hp.ring2nest(nside,st['PIXEL'])] = st['SIGNAL']
else :
# straight up
m[st['PIXEL']] = st['SIGNAL']
return m
def _get_converted_data(self, scheme):
"""
internal routine to get the data converted to the requested
scheme
If the scheme would be unchanged, a reference to the data is returned
"""
import healpy
scheme_num = get_scheme_num(scheme)
if scheme_num == self.hpix.scheme_num:
return self.data
if scheme_num==NESTED:
ipring=numpy.arange(self.hpix.npix,dtype='i8')
ipnest=healpy.ring2nest(self.hpix.nside, ipring)
newdata=self.data.copy()
newdata[ipnest]=self.data
else:
ipnest=numpy.arange(self.hpix.npix,dtype='i8')
ipring=healpy.nest2ring(self.hpix.nside, ipnest)
newdata=self.data.copy()
newdata[ipring]=self.data
return newdata
def get_index_list(nside, nest, region):
""" Returns the list of pixels indices for all the pixels in a region
nside : HEALPix nside parameter
nest : True for 'NESTED', False = 'RING'
region : HEALPix region string
"""
tokens = parse_hpxregion(region)
if tokens[0] == 'DISK':
vec = coords_to_vec(float(tokens[1]), float(tokens[2]))
ilist = hp.query_disc(nside, vec[0], np.radians(float(tokens[3])),
inclusive=False, nest=nest)
elif tokens[0] == 'DISK_INC':
vec = coords_to_vec(float(tokens[1]), float(tokens[2]))
ilist = hp.query_disc(nside, vec[0], np.radians(float(tokens[3])),
inclusive=True, fact=int(tokens[4]),
nest=nest)
elif tokens[0] == 'HPX_PIXEL':
nside_pix = int(tokens[2])
if tokens[1] == 'NESTED':
ipix_ring = hp.nest2ring(nside_pix, int(tokens[3]))
elif tokens[1] == 'RING':
ipix_ring = int(tokens[3])
else:
raise Exception(
"Did not recognize ordering scheme %s" % tokens[1])
ilist = match_hpx_pixel(nside, nest, nside_pix, ipix_ring)
else:
raise Exception(
"HPX.get_index_list did not recognize region type %s" % tokens[0])
return ilist
def _main():
"""
This is the main routine.
"""
data = np.loadtxt("all_ts_data_0.1GeVplus.gz",skiprows=32,unpack=True)
print len(data[0])
nside=int(round(mt.sqrt(len(data[9])/12)))
tsarray=data[0]*0
RA=data[8]
decl=data[9]
# index=DeclRaToIndex(decl,RA,nside)
index=hp.nest2ring(nside,np.array(data[0]).astype(int))
tsarray[index]=data[22]
tsarray = hp.sphtfunc.smoothing(tsarray,sigma = 0.008)
# tsmap=0*tsarray
hp.mollview(np.arcsinh(tsarray)/mt.log(10.0),coord='C', title='TS Map', unit='prob',xsize = 2048)
# hp.graticule()
plt.savefig("ts.png")
plt.show()
def inpaint(m,num_degrades=1,nside_in=2048):
"""
Inpaints missing pixels by degrading the map (while ignoring missing pixels),
then setting the missing values correpsonding to their value in the degraded map.
"""
import healpy as H
nside_deg = nside_in/(2**num_degrades)
badpix = arange(12*nside_in**2)[m==H.UNSEEN]
badpix_deg = H.nest2ring(nside_deg,H.ring2nest(nside_in,badpix) >> 2*num_degrades)
m_deg = H.ud_grade(m,nside_deg)
m2=m.copy()
m2[badpix]=m_deg[badpix_deg]
return m2
def convert_pixel_to_map(pixel, valkey='SIGNAL', pixkey='PIXEL',
nested=False, nside=4096, return_seen=True):
ycoord = pixel[pixkey]
if nested:
# convert to ring
ycoord = hp.nest2ring(nside, ycoord)
m = np.zeros(hp.nside2npix(nside), dtype=pixel[pixkey].dtype) + hp.UNSEEN
m[ycoord] = pixel[valkey]
if return_seen:
indices, = np.where(m != hp.UNSEEN)
m = m[indices]
return indices, m
else:
return np.arange(m.size), m
def to_swapped(self):
import healpy as hp
hpx_out = self.geom.to_swapped()
map_out = self.__class__(hpx_out, meta=copy.deepcopy(self.meta))
idx = self.geom.get_idx(flat=True)
vals = self.get_by_idx(idx)
if self.geom.nside.size > 1:
nside = self.geom.nside[idx[1:]]
else:
nside = self.geom.nside
if self.geom.nest:
idx_new = tuple([hp.nest2ring(nside, idx[0])]) + idx[1:]
else:
idx_new = tuple([hp.ring2nest(nside, idx[0])]) + idx[1:]
map_out.set_by_idx(idx_new, vals)
return map_out
def to_swapped(self):
import healpy as hp
hpx_out = self.geom.to_swapped()
map_out = self._init_copy(geom=hpx_out, data=None)
idx = self.geom.get_idx(flat=True)
vals = self.get_by_idx(idx)
if self.geom.nside.size > 1:
nside = self.geom.nside[idx[1:]]
else:
nside = self.geom.nside
if self.geom.nest:
idx_new = tuple([hp.nest2ring(nside, idx[0])]) + idx[1:]
else:
idx_new = tuple([hp.ring2nest(nside, idx[0])]) + idx[1:]
map_out.set_by_idx(idx_new, vals)
return map_out
def query(self, sources, chunksize=1000, method='array'):
"""
Returns the selection function at the requested coordinates.
Args:
sources (:obj:`selectionfunctions.source.Source`): The coordinates, magnitude and colour to query.
Returns:
Selection function at the specified coordinates, as a fraction.
"""
# Convert coordinates to healpix indices
if method == 'array': nside = hp.npix2nside(self.x.shape[2])
else: nside = self.nside
hpxidx = coord2healpix(sources.coord, 'icrs', nside, nest=True)
# Extract Gaia G magnitude
mag = sources.photometry.measurement['gaia_g']
try:
color = sources.photometry.measurement['gaia_g_gaia_rp']
except KeyError:
color = np.zeros(len(mag))
# Evaluate selection function
if method == 'array':
selection_function = self._selection_function(mag, color, hpxidx)
elif method == 'gp':
# Load spherical basis
if not hasattr(self, 'basis'):
print(f'Spherical Basis: {self.spherical_basis_file}')
self._load_spherical_basis()
# Switch positions to ring ordering
pix = hp.nest2ring(self.nside, hpxidx)
# Don't evaluate for outside range
selection_function = np.zeros(len(pix))
subset = (mag > self.Mbins[0]) & (mag < self.Mbins[-1])
selection_function[subset] = self._selection_function_gp(
mag[subset], color[subset], pix[subset])
elif method == 'basis':
raise ValueError('basis method not implemented yet.')
return selection_function
def to_swapped(self):
import healpy as hp
hpx_out = self.geom.to_swapped()
map_out = self._init_copy(geom=hpx_out, data=None)
idx = self.geom.get_idx(flat=True)
vals = self.get_by_idx(idx)
if self.geom.nside.size > 1:
nside = self.geom.nside[idx[1:]]
else:
nside = self.geom.nside
if self.geom.nest:
idx_new = tuple([hp.nest2ring(nside, idx[0])]) + idx[1:]
else:
idx_new = tuple([hp.ring2nest(nside, idx[0])]) + idx[1:]
map_out.set_by_idx(idx_new, vals)
return map_out
def read_map(filename, HDU=0, field=0, nest=False):
"""Read Healpix map
all columns of the specified HDU are read into a compound numpy MASKED array
if nest is not None, the map is converted if need to NEST or RING ordering.
this function requires healpy"""
m, h = read(filename, HDU=HDU, return_header=True)
try:
m = m[field]
except exceptions.KeyError:
m = m.values()[field]
nside = healpy.npix2nside(m.size)
if not nest is None:
if h.get('ORDERING', False):
if h['ORDERING'] == 'NESTED' and not nest:
idx = healpy.ring2nest(nside,np.arange(m.size,dtype=np.int32))
m = m[idx]
elif h['ORDERING'] == 'RING' and nest:
idx = healpy.nest2ring(nside,np.arange(m.size,dtype=np.int32))
m = m[idx]
return healpy.ma(m)
def convert_pixel_to_map(pixel,
valkey='SIGNAL',
pixkey='PIXEL',
nested=False,
nside=4096,
return_seen=True):
ycoord = pixel[pixkey]
if nested:
# convert to ring
ycoord = hp.nest2ring(nside, ycoord)
m = np.zeros(hp.nside2npix(nside), dtype=pixel[pixkey].dtype) + hp.UNSEEN
m[ycoord] = pixel[valkey]
if return_seen:
indices, = np.where(m != hp.UNSEEN)
m = m[indices]
return indices, m
else:
return np.arange(m.size), m
def face_pix(nside, face=0, ordering='Ring'):
while face > 11:
face -= 12
if nside is 1:
subpix = np.array([face])
else:
tmpnside = 2
subpix = np.array([[3, 1], [2, 0]])
#subpix = np.array([[2,0],[3,1]])
## Enlarge if necessary
##----------------------
while tmpnside < nside:
tmpnside *= 2
pix = np.zeros((tmpnside, tmpnside), dtype=np.int64)
pix[0:tmpnside // 2,
0:tmpnside // 2] = subpix + 3 * (tmpnside**2) // 4
pix[tmpnside // 2:tmpnside, tmpnside // 2:tmpnside] = subpix
pix[tmpnside // 2:tmpnside,
0:tmpnside // 2] = subpix + 2 * (tmpnside**2) // 4
pix[0:tmpnside // 2,
tmpnside // 2:tmpnside] = subpix + (tmpnside**2) // 4
'''
pix[0:tmpnside/2,0:tmpnside/2] = subpix + 2*(tmpnside**2)/4
pix[tmpnside/2:tmpnside,tmpnside/2:tmpnside] = subpix + (tmpnside**2)/4
pix[tmpnside/2:tmpnside,0:tmpnside/2] = subpix + 3*(tmpnside**2)/4
pix[0:tmpnside/2,tmpnside/2:tmpnside] = subpix
'''
subpix = pix
subpix += face * (nside**2)
if ordering is 'Ring':
subpix = hp.nest2ring(nside, subpix)
return subpix
def get_index_list(nside, nest, region):
""" Returns the list of pixels indices for all the pixels in a region
nside : HEALPix nside parameter
nest : True for 'NESTED', False = 'RING'
region : HEALPix region string
"""
tokens = parse_hpxregion(region)
if tokens[0] == 'DISK':
vec = coords_to_vec(float(tokens[1]), float(tokens[2]))
ilist = hp.query_disc(nside,
vec[0],
np.radians(float(tokens[3])),
inclusive=False,
nest=nest)
elif tokens[0] == 'DISK_INC':
vec = coords_to_vec(float(tokens[1]), float(tokens[2]))
ilist = hp.query_disc(nside,
vec[0],
np.radians(float(tokens[3])),
inclusive=True,
fact=int(tokens[4]),
nest=nest)
elif tokens[0] == 'HPX_PIXEL':
nside_pix = int(tokens[2])
if tokens[1] == 'NESTED':
ipix_ring = hp.nest2ring(nside_pix, int(tokens[3]))
elif tokens[1] == 'RING':
ipix_ring = int(tokens[3])
else:
raise Exception("Did not recognize ordering scheme %s" %
tokens[1])
ilist = match_hpx_pixel(nside, nest, nside_pix, ipix_ring)
else:
raise Exception(
"HPX.get_index_list did not recognize region type %s" %
tokens[0])
return ilist
def to_swapped_scheme(self):
import healpy as hp
hpx_out = self.geom.to_swapped()
map_out = self.__class__(hpx_out)
idx = list(self.geom.get_idx())
vals = self.get_by_idx(idx)
msk = vals > 0
idx = [t[msk] for t in idx]
vals = vals[msk]
if self.geom.nside.size > 1:
nside = self.geom.nside[idx[1:]]
else:
nside = self.geom.nside
if self.geom.nest:
idx_new = tuple([hp.nest2ring(nside, idx[0])] + idx[1:])
else:
idx_new = tuple([hp.ring2nest(nside, idx[0])] + idx[1:])
map_out.set_by_pix(idx_new, vals)
return map_out
def query(self, sources, chunksize=1000):
"""
Returns the selection function at the requested coordinates.
Args:
coords (:obj:`astropy.coordinates.SkyCoord`): The coordinates to query.
Returns:
Selection function at the specified coordinates, as a fraction.
"""
# Convert coordinates to healpix indices
hpxidx = coord2healpix(sources.coord, 'icrs', self.nside, nest=True)
# Extract Gaia G magnitude
mag = sources.photometry.measurement['gaia_g']
try:
color = sources.photometry.measurement['gaia_bp_gaia_rp']
except KeyError:
color = np.zeros(len(mag))
# Switch positions to ring ordering
pix = hp.nest2ring(self.nside, hpxidx)
# Evaluate selection function
selection_function = self._selection_function(mag,
color,
pix,
chunksize=chunksize)
if self._bounds == True:
_outside_bounds = np.where((mag < self._g_min)
| (mag > self._g_max))
selection_function[_outside_bounds] = 0.0
return selection_function
def spread_pixels(self, Nside_low, Nside_high, ID, order='nest'):
"""
returns a list of pixel IDs in the Nside_high resolution
from a pixel ID in the Nside_low resolution.
"""
from math import log
import numpy as np
import healpy as hp
if order != 'nest' and order != 'ring':
raise KeyError('ERROR: check order in spread_pixels')
if Nside_low == Nside_high:
if isinstance(ID, list):
return ID
else:
return [ID]
if Nside_low > Nside_high:
raise KeyError('ERROR using spread_pixels')
Llow = int(log(Nside_low, 2))
Lhigh = int(log(Nside_high, 2))
if isinstance(ID, list) or isinstance(ID, np.ndarray):
pixids = []
if order == 'ring':
for ipix in ID:
j = hp.ring2nest(Nside_low, ipix)
pixids.append(j)
else:
pixids = ID
pix_IDs = []
for id in pixids:
b = bin(id)
DN = Lhigh - Llow
a = [bin(i)[2:].zfill(2 * DN) for i in range(4**DN)]
for i in a:
x = (b[2:].zfill(Llow) + i)
pix_IDs.append(int(x, 2))
elif isinstance(ID, int) or isinstance(ID, np.int64):
if order == 'ring':
pixids = hp.ring2nest(Nside_low, ID)
else:
pixids = ID
b = bin(pixids)
DN = Lhigh - Llow
a = [bin(i)[2:].zfill(2 * DN) for i in range(4**DN)]
pix_IDs = []
for i in a:
x = (b[2:].zfill(Llow) + i)
pix_IDs.append(int(x, 2))
else:
print('wtf')
pix_IDs = 0
if order == 'ring':
for i in range(len(pix_IDs)):
pix_IDs[i] = hp.nest2ring(Nside_high, pix_IDs[i])
return (pix_IDs)
def _unstage_data(self, comm, data, nsamp, nnz, nnz_full,
obs_period_ranges, detectors, pixels_dtype, nside,
weight_dtype):
""" Clear Madam buffers, restore pointing into TOAST caches
and cache the destriped signal.
"""
auto_timer = timing.auto_timer(type(self).__name__)
self._madam_timestamps = None
self._cache.destroy('timestamps')
if self._conserve_memory:
nodecomm = comm.Split_type(MPI.COMM_TYPE_SHARED, comm.rank)
nread = nodecomm.size
else:
nodecomm = MPI.COMM_SELF
nread = 1
for iread in range(nread):
nodecomm.Barrier()
if nodecomm.rank % nread != iread:
continue
if self._name_out is not None:
global_offset = 0
for obs, period_ranges in zip(data.obs, obs_period_ranges):
tod = obs['tod']
nlocal = tod.local_samples[1]
for idet, det in enumerate(detectors):
signal = np.ones(nlocal) * np.nan
offset = global_offset
for istart, istop in period_ranges:
nn = istop - istart
dslice = slice(idet * nsamp + offset,
idet * nsamp + offset + nn)
signal[istart:istop] = self._madam_signal[dslice]
offset += nn
cachename = "{}_{}".format(self._name_out, det)
tod.cache.put(cachename, signal, replace=True)
global_offset = offset
self._madam_signal = None
self._cache.destroy('signal')
if not self._purge_pixels:
# restore the pixels from the Madam buffers
global_offset = 0
for obs, period_ranges in zip(data.obs, obs_period_ranges):
tod = obs['tod']
nlocal = tod.local_samples[1]
for idet, det in enumerate(detectors):
pixels = -np.ones(nlocal, dtype=pixels_dtype)
offset = global_offset
for istart, istop in period_ranges:
nn = istop - istart
dslice = slice(idet * nsamp + offset,
idet * nsamp + offset + nn)
pixels[istart:istop] = self._madam_pixels[dslice]
offset += nn
npix = 12 * nside**2
good = np.logical_and(pixels >= 0, pixels < npix)
if not self._pixels_nested:
pixels[good] = hp.nest2ring(nside, pixels[good])
pixels[np.logical_not(good)] = -1
cachename = "{}_{}".format(self._pixels, det)
tod.cache.put(cachename, pixels, replace=True)
global_offset = offset
self._madam_pixels = None
self._cache.destroy('pixels')
if not self._purge_weights and nnz == nnz_full:
# restore the weights from the Madam buffers
global_offset = 0
for obs, period_ranges in zip(data.obs, obs_period_ranges):
tod = obs['tod']
nlocal = tod.local_samples[1]
for idet, det in enumerate(detectors):
weights = np.zeros([nlocal, nnz], dtype=weight_dtype)
offset = global_offset
for istart, istop in period_ranges:
nn = istop - istart
dwslice = slice((idet * nsamp + offset) * nnz,
(idet * nsamp + offset + nn) * nnz)
weights[istart:istop] = self._madam_pixweights[
dwslice].reshape([-1, nnz])
offset += nn
cachename = "{}_{}".format(self._weights, det)
tod.cache.put(cachename, weights, replace=True)
global_offset = offset
self._madam_pixweights = None
self._cache.destroy('pixweights')
del nodecomm
return
def read_sky_map(filename, nest=False, distances=False, moc=False, **kwargs):
"""
Read a LIGO/Virgo-type sky map and return a tuple of the HEALPix array
and a dictionary of metadata from the header.
Parameters
----------
filename: string
Path to the optionally gzip-compressed FITS file.
nest: bool, optional
If omitted or False, then detect the pixel ordering in the FITS file
and rearrange if necessary to RING indexing before returning.
If True, then detect the pixel ordering and rearrange if necessary to
NESTED indexing before returning.
If None, then preserve the ordering from the FITS file.
Regardless of the value of this option, the ordering used in the FITS
file is indicated as the value of the 'nest' key in the metadata
dictionary.
distances: bool, optional
If true, then read also read the additional HEALPix layers representing
the conditional mean and standard deviation of distance as a function
of sky location.
moc: bool, optional
If true, then preserve multi-order structure if present.
Example
-------
Test that we can read a legacy IDL-compatible file
(https://bugs.ligo.org/redmine/issues/5168):
>>> import tempfile
>>> with tempfile.NamedTemporaryFile(suffix='.fits') as f:
... nside = 512
... npix = hp.nside2npix(nside)
... ipix_nest = np.arange(npix)
... hp.write_map(f.name, ipix_nest, nest=True, column_names=['PROB'])
... m, meta = read_sky_map(f.name)
... np.testing.assert_array_equal(m, hp.ring2nest(nside, ipix_nest))
"""
m = Table.read(filename, format='fits', **kwargs)
# Remove some keys that we do not need
for key in (
'PIXTYPE', 'EXTNAME', 'NSIDE', 'FIRSTPIX', 'LASTPIX', 'INDXSCHM'):
m.meta.pop(key, None)
if m.meta.pop('COORDSYS', 'C') != 'C':
raise ValueError('LALInference only reads and writes sky maps in '
'equatorial coordinates.')
try:
value = m.meta.pop('ORDERING')
except KeyError:
pass
else:
if value == 'RING':
m.meta['nest'] = False
elif value == 'NESTED':
m.meta['nest'] = True
elif value == 'NUNIQ':
pass
else:
raise ValueError(
'ORDERING card in header has unknown value: {0}'.format(value))
for fits_key, rows in itertools.groupby(
FITS_META_MAPPING, lambda row: row[1]):
try:
value = m.meta.pop(fits_key)
except KeyError:
pass
else:
for row in rows:
key, _, _, _, from_fits = row
if from_fits is not None:
m.meta[key] = from_fits(value)
if 'UNIQ' not in m.colnames:
m = Table([col.ravel() for col in m.columns.values()], meta=m.meta)
if 'UNIQ' in m.colnames and not moc:
from ..bayestar.sky_map import rasterize
m = rasterize(m)
m.meta['nest'] = True
elif 'UNIQ' not in m.colnames and moc:
from ..bayestar.sky_map import derasterize
if not m.meta['nest']:
npix = len(m)
nside = hp.npix2nside(npix)
m = m[hp.nest2ring(nside, np.arange(npix))]
m = derasterize(m)
m.meta.pop('nest', None)
if 'UNIQ' not in m.colnames:
npix = len(m)
nside = hp.npix2nside(npix)
if nest is None:
pass
elif m.meta['nest'] and not nest:
m = m[hp.ring2nest(nside, np.arange(npix))]
elif not m.meta['nest'] and nest:
m = m[hp.nest2ring(nside, np.arange(npix))]
if moc:
return m
elif distances:
return tuple(
np.asarray(m[name]) for name in DEFAULT_NESTED_NAMES), m.meta
else:
return np.asarray(m[DEFAULT_NESTED_NAMES[0]]), m.meta
print('pix_nest =', pix_nest)
all_pix_ring = np.arange(192)
all_pix_nest = hp.ring2nest(Nside, all_pix_ring)
hp.mollview(all_pix_nest, title='Nest scheme', unit='Pix index')
pyplot.savefig('mollweide_view_nest.png')
#%%
"""
STEP 3:Nest to ring
"""
# Nside = 4
print('pix_nest =', pix_nest)
pix_ring = hp.nest2ring(Nside, pix_nest)
print('pix_ring =', pix_ring)
all_pix_ring = hp.nest2ring(Nside, all_pix_nest)
hp.mollview(all_pix_ring, title='Ring scheme', unit='Pix index')
pyplot.savefig('mollweide_view_ring.png')
#%%
"""
STEP 4: Reordering a map
"""
mapa_ring = hp.read_map('COM_CMB_IQU-smica_1024_R2.02_full.fits')
hp.mollview(mapa_ring, title='CMB map - Ring', unit='K')
pyplot.savefig('mollweide_view_cmb.png')
def test_lookup(self):
"""
Test lookup functionality
"""
np.random.seed(12345)
nside_coverage = 32
nside_map = 1024
full_map = np.zeros(hp.nside2npix(nside_map)) + hp.UNSEEN
full_map[0:200000] = np.random.random(size=200000)
sparse_map = healsparse.HealSparseMap(healpix_map=full_map,
nside_coverage=nside_coverage)
n_rand = 100000
ra = np.random.random(n_rand) * 360.0
dec = np.random.random(n_rand) * 180.0 - 90.0
theta = np.radians(90.0 - dec)
phi = np.radians(ra)
ipnest = hp.ang2pix(nside_map, theta, phi, nest=True)
test_values = full_map[ipnest]
# Test the pixel lookup
comp_values = sparse_map.get_values_pix(ipnest)
testing.assert_almost_equal(comp_values, test_values)
# Test pixel lookup (valid pixels)
# Note that this tests all the downstream functions
valid_mask = sparse_map.get_values_pix(ipnest, valid_mask=True)
testing.assert_equal(valid_mask, comp_values > hp.UNSEEN)
# Test pixel lookup (ring)
ipring = hp.nest2ring(nside_map, ipnest)
comp_values = sparse_map.get_values_pix(ipring, nest=False)
testing.assert_almost_equal(comp_values, test_values)
# Test pixel lookup (higher nside)
comp_values = sparse_map.get_values_pix(hp.ang2pix(4096,
ra,
dec,
lonlat=True,
nest=True),
nside=4096)
testing.assert_almost_equal(comp_values, test_values)
# Test pixel lookup (lower nside)
lowres_pix = hp.ang2pix(256, ra, dec, lonlat=True, nest=True)
self.assertRaises(ValueError,
sparse_map.get_values_pix,
lowres_pix,
nside=256)
# Test the theta/phi lookup
comp_values = sparse_map.get_values_pos(theta, phi, lonlat=False)
testing.assert_almost_equal(comp_values, test_values)
# Test the ra/dec lookup
comp_values = sparse_map.get_values_pos(ra, dec, lonlat=True)
testing.assert_almost_equal(comp_values, test_values)
# Test the list of valid pixels
valid_pixels = sparse_map.valid_pixels
testing.assert_equal(valid_pixels, np.where(full_map > hp.UNSEEN)[0])
# Test the position of valid pixels
ra_sp, dec_sp = sparse_map.valid_pixels_pos(lonlat=True)
_ra_sp, _dec_sp = hp.pix2ang(nside_map,
np.where(full_map > hp.UNSEEN)[0],
lonlat=True,
nest=True)
testing.assert_equal(ra_sp, _ra_sp)
testing.assert_equal(dec_sp, _dec_sp)
# Test position of valid pixels and valid pixels
valid_pixels, ra_sp, dec_sp = sparse_map.valid_pixels_pos(
lonlat=True, return_pixels=True)
_ra_sp, _dec_sp = hp.pix2ang(nside_map,
np.where(full_map > hp.UNSEEN)[0],
lonlat=True,
nest=True)
testing.assert_equal(ra_sp, _ra_sp)
testing.assert_equal(dec_sp, _dec_sp)
testing.assert_equal(valid_pixels, np.where(full_map > hp.UNSEEN)[0])
def convert_transmission_to_deltas(obj_path,
out_dir,
in_dir=None,
in_filenames=None,
lambda_min=3600.,
lambda_max=5500.,
lambda_min_rest_frame=1040.,
lambda_max_rest_frame=1200.,
delta_log_lambda=None,
delta_lambda=None,
lin_spaced=False,
max_num_spec=None,
nproc=None,
use_old_weights=False,
out_healpix_order='RING'):
"""Convert transmission files to picca delta files
Args:
obj_path: str
Path to the catalog of object to extract the transmission from
out_dir: str
Path to the directory where delta files will be written
in_dir: str or None - default: None
Path to the directory containing the transmission files directory.
If 'None', then in_files must be a non-empty array
in_filenames: array of str or None - default: None
List of the filenames for the transmission files. Ignored if in_dir
is not 'None'
lambda_min: float - default: 3600.
Minimum observed wavelength in Angstrom
lambda_max: float - default: 5500.
Maximum observed wavelength in Angstrom
lambda_min_rest_frame: float - default: 1040.
Minimum Rest Frame wavelength in Angstrom
lambda_max_rest_frame: float - default: 1200.
Maximum Rest Frame wavelength in Angstrom
delta_log_lambda: float - default: None
Variation of the logarithm of the wavelength between two pixels
delta_lambda: float - default: None
Variation of the wavelength between two pixels
lin_spaced: float - default: False
Whether to use linear spacing for the wavelength binning
max_num_spec: int or None - default: None
Maximum number of spectra to read. 'None' for no maximum
nproc: int or None - default: None
Number of cpus to use for I/O operations. If None, defaults to os.cpu_count().
use_old_weights: boolean - default: False
Whether to use the old weights based only on the bin size
out_healpix_order: string: 'RING' or 'NEST' - default: 'RING'
Healpix numbering scheme for output files.
"""
# read catalog of objects
hdul = fitsio.FITS(obj_path)
key_val = np.char.strip(
np.array([
hdul[1].read_header()[key] for key in hdul[1].read_header().keys()
]).astype(str))
if 'TARGETID' in key_val:
objs_thingid = hdul[1]['TARGETID'][:]
elif 'THING_ID' in key_val:
objs_thingid = hdul[1]['THING_ID'][:]
w = hdul[1]['Z'][:] > max(0., lambda_min / lambda_max_rest_frame - 1.)
w &= hdul[1]['Z'][:] < max(0., lambda_max / lambda_min_rest_frame - 1.)
objs_ra = hdul[1]['RA'][:][w].astype('float64') * np.pi / 180.
objs_dec = hdul[1]['DEC'][:][w].astype('float64') * np.pi / 180.
objs_thingid = objs_thingid[w]
hdul.close()
userprint('INFO: Found {} quasars'.format(objs_ra.size))
# Load list of transmission files
if ((in_dir is None and in_filenames is None)
or (in_dir is not None and in_filenames is not None)):
userprint(("ERROR: No transmisson input files or both 'in_dir' and "
"'in_filenames' given"))
sys.exit()
elif in_dir is not None:
files = sorted(glob.glob(in_dir + '/*/*/transmission*.fits*'))
files = np.sort(np.array(files))
hdul = fitsio.FITS(files[0])
in_nside = hdul['METADATA'].read_header()['HPXNSIDE']
nest = hdul['METADATA'].read_header()['HPXNEST']
hdul.close()
in_healpixs = healpy.ang2pix(in_nside,
np.pi / 2. - objs_dec,
objs_ra,
nest=nest)
if files[0].endswith('.gz'):
end_of_file = '.gz'
else:
end_of_file = ''
files = np.sort(
np.array([("{}/{}/{healpix}/transmission-{}-{healpix}"
".fits{}").format(in_dir,
int(healpix // 100),
in_nside,
end_of_file,
healpix=healpix)
for healpix in np.unique(in_healpixs)]))
else:
files = np.sort(np.array(in_filenames))
nest = None
userprint('INFO: Found {} files'.format(files.size))
# Check if we should compute linear or log spaced deltas
# Use the x_min/x_max/delta_x variables to stand in for either
# linear of log spaced parameters
if lin_spaced:
x_min = lambda_min
x_max = lambda_max
delta_x = delta_lambda if delta_lambda is not None else 3.
else:
x_min = np.log10(lambda_min)
x_max = np.log10(lambda_max)
delta_x = delta_log_lambda if delta_lambda is not None else 3.e-4
num_bins = int((x_max - x_min) / delta_x) + 1
# Read the transmission files in parallel
arguments = [(f, num_bins, objs_thingid, lambda_min, lambda_max,
lambda_min_rest_frame, lambda_max_rest_frame,
delta_log_lambda, delta_lambda, lin_spaced) for f in files]
pool = Pool(processes=nproc)
read_results = pool.starmap(read_transmission_file, arguments)
pool.close()
# Read and merge the results
stack_flux = np.zeros(num_bins)
stack_weight = np.zeros(num_bins)
deltas = {}
for res in read_results:
if res is not None:
healpix_deltas = res[0]
healpix_stack_flux = res[1]
healpix_stack_weight = res[2]
for key in healpix_deltas.keys():
if key not in deltas.keys():
deltas[key] = []
deltas[key] += healpix_deltas[key]
stack_flux += healpix_stack_flux
stack_weight += healpix_stack_weight
num_spec = np.sum([len(deltas[healpix]) for healpix in deltas])
if (max_num_spec is not None and num_spec >= max_num_spec):
break
userprint('\n')
# normalize stacked transmission
w = stack_weight > 0.
mean_flux = stack_flux
mean_flux[w] /= stack_weight[w]
# save results
out_filenames = {}
for healpix in sorted(deltas):
if nest is None:
if out_healpix_order is None:
out_healpix = healpix
else:
raise ValueError(
'Input HEALPix scheme not known, cannot'
'convert to scheme {}'.format(out_healpix_order))
else:
if nest:
if out_healpix_order.lower() == 'nest':
out_healpix = healpix
elif out_healpix_order.lower() == 'ring':
out_healpix = healpy.nest2ring(int(in_nside), int(healpix))
else:
raise ValueError('HEALPix scheme {} not recognised'.format(
out_healpix_order))
else:
if out_healpix_order.lower() == 'nest':
out_healpix = healpy.ring2nest(int(in_nside), int(healpix))
elif out_healpix_order.lower() == 'ring':
out_healpix = healpix
else:
raise ValueError('HEALPix scheme {} not recognised'.format(
out_healpix_order))
print('Input nested? {} // in_healpix={} // out_healpix={}'.format(
nest, healpix, out_healpix))
out_filenames[healpix] = out_dir + '/delta-{}'.format(
out_healpix) + '.fits.gz'
if use_old_weights:
flux_variance = None
else:
# Compute variance
stack_variance = np.zeros(len(mean_flux))
var_weights = np.zeros(len(mean_flux))
for hpix_deltas in deltas.values():
for delta in hpix_deltas:
lambda_array = delta.log_lambda
if lin_spaced:
lambda_array = 10**(lambda_array)
norm_lambda = (lambda_array - x_min) / delta_x + 0.5
bins = np.floor(np.around(norm_lambda, decimals=3)).astype(int)
stack_variance[bins] += (delta.delta - mean_flux[bins])**2
var_weights[bins] += np.ones(len(bins))
w = var_weights > 0.
flux_variance = stack_variance
flux_variance[w] /= var_weights[w]
arguments = [(deltas[hpix], mean_flux, flux_variance, hpix,
out_filenames[hpix], x_min, delta_x, lin_spaced)
for hpix in deltas.keys()]
pool = Pool(processes=nproc)
_ = pool.starmap(write_delta_from_transmission, arguments)
pool.close()
userprint("")
# Output the mean flux and other info
dir_name = os.path.basename(os.path.normpath(out_dir))
rebin_lambda = (x_min + np.arange(num_bins) * delta_x)
results = fitsio.FITS(out_dir + '/../{}-stats.fits.gz'.format(dir_name),
'rw',
clobber=True)
cols = [rebin_lambda, mean_flux, stack_weight, flux_variance, var_weights]
names = ['LAMBDA', 'MEANFLUX', 'WEIGHTS', 'VAR', 'VARWEIGHTS']
header = {}
header['L_MIN'] = lambda_min
header['L_MAX'] = lambda_max
header['LR_MIN'] = lambda_min_rest_frame
header['LR_MAX'] = lambda_max_rest_frame
header['DEL_LL'] = delta_log_lambda
header['DEL_L'] = delta_lambda
header['LINEAR'] = lin_spaced
results.write(cols, names=names, header=header, extname='STATS')
results.close()
userprint("")
def plot_sp_info(datadir, maptype, xlabel, insetlabel, threshold,
generate_maps, histmin, histmax):
footprint = fitsio.read(datadir +
'masks/y3a2_footprint_griz_1exp_v2.0.fits.gz',
ext=1)['I'].ravel()
tdataSP_foot = np.empty(hp.nside2npix(nside))
tdataSP_foot.fill(hp.UNSEEN)
PIX_TO_ARCSEC = 0.263
for i, (band, v) in enumerate(BAND_COLORS.items()):
if band == 'u':
continue
if band == 'Y' and (maptype == 'sof' or maptype == 'mof'):
continue
print('Processing band', band)
if maptype == 'maglim':
filename = 'sp_maps/y3a2_' + band + '_o.4096_t.32768_maglim_EQU.fits'
tdataSP = fitsio.read(datadir + filename)
elif maptype == 'sof':
filename = 'sp_maps/y3a2_gold_2_2_1_sof_nside4096_nest_' + band + '_depth.fits'
tdataSP = fitsio.read(datadir + filename)['I'].ravel()
elif maptype == 'auto':
filename = 'sp_maps/y3a2_gold_2_2_1_auto_nside4096_nest_' + band + '_depth.fits.gz'
tdataSP = fitsio.read(datadir + filename)['I'].ravel()
elif maptype == 'fwhm':
filename = 'sp_maps/y3a2_' + band + '_o.4096_t.32768_FWHM.WMEAN_EQU.fits.gz'
tdataSP = fitsio.read(datadir + filename)
elif maptype == 'skybrite':
filename = 'sp_maps/y3a2_' + band + '_o.4096_t.32768_SKYBRITE.WMEAN_EQU.fits.gz'
tdataSP = fitsio.read(datadir + filename)
elif maptype == 'airmass':
filename = 'sp_maps/y3a2_' + band + '_o.4096_t.32768_AIRMASS.WMEAN_EQU.fits.gz'
tdataSP = fitsio.read(datadir + filename)
elif maptype == 'exptime':
filename = 'sp_maps/y3a2_' + band + '_o.4096_t.32768_EXPTIME.SUM_EQU.fits.gz'
tdataSP = fitsio.read(datadir + filename)
elif maptype == 'skyvar':
filename = 'sp_maps/y3a2_' + band + '_o.4096_t.32768_SKYVAR.UNCERTAINTY_EQU.fits'
tdataSP = fitsio.read(datadir + filename)
elif maptype == 'sbcontrast':
filename = 'sp_maps/Y3A1_SURFACE_BRIGHTNESS-v1.fits'
tdataSP = fitsio.read(datadir + filename)
if band == 'g':
sel = (tdataSP['band'] == b'g') & (tdataSP['size'] == 10)
elif band == 'r':
sel = (tdataSP['band'] == b'r') & (tdataSP['size'] == 10)
elif band == 'i':
sel = (tdataSP['band'] == b'i') & (tdataSP['size'] == 10)
elif band == 'z':
sel = (tdataSP['band'] == b'z') & (tdataSP['size'] == 10)
else:
sel = (tdataSP['band'] == b'Y') & (tdataSP['size'] == 10)
tdataSP = tdataSP[sel]
else:
print('Map type', maptype, 'not found')
sys.exit(1)
if len(tdataSP) != hp.nside2npix(nside):
tdataSP_foot[tdataSP['PIXEL']] = tdataSP['SIGNAL']
tdataSP_foot = tdataSP_foot * footprint
else:
tdataSP_foot = tdataSP * footprint
tdataSP_foot[tdataSP_foot > threshold] = hp.UNSEEN
tdataSP_foot[tdataSP_foot < 0] = hp.UNSEEN
mask = (tdataSP_foot != hp.UNSEEN)
if generate_maps:
fig = plt.figure(figsize=(12., 4.))
gs = plt.GridSpec(1, 4, wspace=0.001)
fig.add_subplot(gs[:2])
smap = DESSkymap()
smap.draw_hpxmap(tdataSP['SIGNAL'],
hp.nest2ring(nside, tdataSP['PIXEL']), nside)
if maptype == 'skybrite':
smap.draw_inset_colorbar(fontsize=10,
format='%.1f',
label=insetlabel)
else:
smap.draw_inset_colorbar(fontsize=10, label=insetlabel)
smap.draw_des()
fig.add_subplot(gs[3])
upper, lower = np.percentile(tdataSP_foot[mask], [84.075, 15.825])
med = np.median(tdataSP_foot[mask])
print('Median', med, '+', upper - med, '-', med - lower)
if maptype == 'skyvar':
medmag = -2.5 * np.log10(med / PIX_TO_ARCSEC**2) + 30
print('Median (mag/arcsec**2)', medmag)
value = tdataSP_foot[mask]
plt.hist(value,
normed=True,
histtype='step',
bins=100,
linewidth=2,
color=v,
label=band,
range=[histmin, histmax])
plt.xlabel(xlabel)
plt.xticks(fontsize=12)
plt.ylabel('PDF')
plt.tight_layout()
plt.legend(loc='upper right')
if generate_maps:
#plt.savefig('/Users/nsevilla/y3gold-paper/figs/y3gold_'+maptype+'_'+band+'_map.png')
plt.savefig('/Users/nsevilla/y3gold-paper/figs/y3gold_' + maptype +
'_' + band + '_map.pdf')
if generate_maps == False:
#plt.savefig('/Users/nsevilla/y3gold-paper/figs/y3gold_'+maptype+'_hist.png')
plt.savefig('/Users/nsevilla/y3gold-paper/figs/y3gold_' + maptype +
'_hist.pdf')
def read_sky_map(filename, nest=False, distances=False, moc=False, **kwargs):
"""
Read a LIGO/Virgo-type sky map and return a tuple of the HEALPix array
and a dictionary of metadata from the header.
Parameters
----------
filename: string
Path to the optionally gzip-compressed FITS file.
nest: bool, optional
If omitted or False, then detect the pixel ordering in the FITS file
and rearrange if necessary to RING indexing before returning.
If True, then detect the pixel ordering and rearrange if necessary to
NESTED indexing before returning.
If None, then preserve the ordering from the FITS file.
Regardless of the value of this option, the ordering used in the FITS
file is indicated as the value of the 'nest' key in the metadata
dictionary.
distances: bool, optional
If true, then read also read the additional HEALPix layers representing
the conditional mean and standard deviation of distance as a function
of sky location.
moc: bool, optional
If true, then preserve multi-order structure if present.
Example
-------
Test that we can read a legacy IDL-compatible file
(https://bugs.ligo.org/redmine/issues/5168):
>>> import tempfile
>>> with tempfile.NamedTemporaryFile(suffix='.fits') as f:
... nside = 512
... npix = hp.nside2npix(nside)
... ipix_nest = np.arange(npix)
... hp.write_map(f.name, ipix_nest, nest=True, column_names=['PROB'])
... m, meta = read_sky_map(f.name)
... np.testing.assert_array_equal(m, hp.ring2nest(nside, ipix_nest))
"""
m = Table.read(filename, format='fits', **kwargs)
# Remove some keys that we do not need
for key in ('PIXTYPE', 'EXTNAME', 'NSIDE', 'FIRSTPIX', 'LASTPIX',
'INDXSCHM', 'MOCORDER'):
m.meta.pop(key, None)
if m.meta.pop('COORDSYS', 'C') != 'C':
raise ValueError('LALInference only reads and writes sky maps in '
'equatorial coordinates.')
try:
value = m.meta.pop('ORDERING')
except KeyError:
pass
else:
if value == 'RING':
m.meta['nest'] = False
elif value == 'NESTED':
m.meta['nest'] = True
elif value == 'NUNIQ':
pass
else:
raise ValueError(
'ORDERING card in header has unknown value: {0}'.format(value))
for fits_key, rows in itertools.groupby(FITS_META_MAPPING,
lambda row: row[1]):
try:
value = m.meta.pop(fits_key)
except KeyError:
pass
else:
for row in rows:
key, _, _, _, from_fits = row
if from_fits is not None:
m.meta[key] = from_fits(value)
if 'UNIQ' not in m.colnames:
m = Table([col.ravel() for col in m.columns.values()], meta=m.meta)
if 'UNIQ' in m.colnames and not moc:
from ..bayestar.sky_map import rasterize
m = rasterize(m)
m.meta['nest'] = True
elif 'UNIQ' not in m.colnames and moc:
from ..bayestar.sky_map import derasterize
if not m.meta['nest']:
npix = len(m)
nside = hp.npix2nside(npix)
m = m[hp.nest2ring(nside, np.arange(npix))]
m = derasterize(m)
m.meta.pop('nest', None)
if 'UNIQ' not in m.colnames:
npix = len(m)
nside = hp.npix2nside(npix)
if nest is None:
pass
elif m.meta['nest'] and not nest:
m = m[hp.ring2nest(nside, np.arange(npix))]
elif not m.meta['nest'] and nest:
m = m[hp.nest2ring(nside, np.arange(npix))]
if moc:
return m
elif distances:
return tuple(np.asarray(m[name])
for name in DEFAULT_NESTED_NAMES), m.meta
else:
return np.asarray(m[DEFAULT_NESTED_NAMES[0]]), m.meta
overwrite=True,
nest=True,
fits_IDL=False,
partial=True)
#Make plot of the healpix map
upix, uval = [], []
for jj, vv in enumerate(hp_aux):
if vv != hp.UNSEEN:
upix.append(jj)
uval.append(vv)
upix = np.array(upix)
uval = np.array(uval)
k = hp.nest2ring(4096, upix)
m = Skymap(projection='cass',
lon_0=67.,
lat_0=-60.,
celestial=False,
llcrnrlon=56.68,
urcrnrlon=77.46,
llcrnrlat=-64.2,
urcrnrlat=-55.40,
parallels=False,
meridians=False)
m.draw_hpxmap(uval, k, nside=4096, xsize=1000)
plt.savefig('healpix_imageLow.png')
else:
print('Exist\nReading\n')
read = np.loadtxt('PIXELS/PixelList.txt')
pixel = pd.DataFrame(read, columns=['HPIX_' + str(params['NSIDE'])])
strips = pixel['HPIX_' + str(params['NSIDE'])].astype(int)
len_strips = len(strips)
print("\n\n")
time0 = time()
for num, pix in enumerate(strips):
timei = time()
theta, phi = hp.pix2ang(params['NSIDE'], pix, lonlat=True, nest=False)
params['pixel'] = pix
if not ver.exist_or_not_file(params, restart):
print(ver.exist_or_not_file(params, restart))
tab = qr.query_function(params)
timef = strftime('%H:%M:%S', gmtime(time() - timei))
print("Program's time (hh:mm:ss): {}".format(timef))
print("Pixel {}".format(pix))
params['pixel'] = hp.nest2ring(params['NSIDE'], pix)
if len(tab) > 0: wr.write_fits(tab, params)
else: print("Not save. 0 galaxies")
print("\n\n")
print("End Programm: {0}".format(time0 - time()))
assert(x[i] == i)
assert(x[1501] == 0)
x.indexedsparse = True # Ring to indexed
assert(x.nside == 64)
assert(x.npix_allocated == 1500)
for i in range(1,1500):
assert(x[i] == i)
assert(x[1501] == 0)
k,v = x.nonzero_pixels()
assert(len(k) == len(v) == 1500)
assert(set(v) == set(a)) # Lazy test that doesn't care about order
import healpy as hp
p0 = hp.nest2ring(64, (hp.ring2nest(32, 0) * 4 + np.arange(4)).astype(int))
v0 = sum([x[int(i)] for i in p0])
x2 = x.rebin(2, norm=False)
assert(x2[0] == v0)
assert(np.sum(x2) == np.sum(v))
x.shift_ra = True
# check attributes
xc = x.clone()
for attr in attrs:
assert getattr(xc, attr) == getattr(x, attr), "Attribute {} mismatched on clone()".format(attr)
xc = x.clone(False)
for attr in attrs:
assert getattr(xc, attr) == getattr(x, attr), "Attribute {} mismatched on clone(False)".format(attr)
for ip in [0, 1]:
mp_sky = hp.smoothing((mpfg + mpc)[ip, :, inu],
fwhm=beam_max,
verbose=False)
mps_d[0, ip, :, inu] = mp_sky + mps_no1[ip, :, inu]
mps_d[1, ip, :, inu] = mp_sky + mps_no2[ip, :, inu]
mps_d_rw = np.array([mps_no1, mps_no2]) + (mpfg + mpc)[None, :, :, :]
else:
mps_d = np.array([mps_no1, mps_no2]) + (mpfg + mpc)[None, :, :, :]
if OUTPUT_LEVEL > 0:
hp.write_map(predir + 'cmb_true.fits', amc, overwrite=True)
#Domain decomposition
ipnest_sub = hp.ring2nest(NSIDE_SPEC_DEFAULT,
np.arange(hp.nside2npix(NSIDE_SPEC_DEFAULT)))
ipring = hp.nest2ring(NSIDE_DEFAULT, np.arange(hp.nside2npix(NSIDE_DEFAULT)))
ip_patches_good = []
for ip_sub_ring in np.arange(hp.nside2npix(NSIDE_SPEC_DEFAULT)):
ips_ring = ipring[ipnest_sub[ip_sub_ring] * NSIDE_RATIO**2 +
np.arange(NSIDE_RATIO**2)]
if np.sum(msk[ips_ring] > ZER0) > 0:
ip_patches_good.append(ips_ring[msk[ips_ring] > ZER0])
plotmap = -1. * np.ones(hp.nside2npix(NSIDE_DEFAULT))
for i, ips in enumerate(ip_patches_good):
plotmap[ips] = i
#Zero mask outside edges for safety
ids_bad = np.where(msk < ZER0)[0]
msk[ids_bad] = 0.
if OUTPUT_LEVEL > 0:
Z_unmatch = z_nbr_rm[~ind] Z_match = z_nbr_rm[ind] Zerr_match = zerr_nbr_rm[ind] memid_match = memid_rm[ind] z_spt_tem = z_spt_match[ind] SNR_spt_tem = xi_spt_match[ind] m500_spt_tem = m500_spt_match[ind] z_spt_unmatch = z_spt_match[~ind] zerr_spt_unmatch = zerr_spt_match[~ind] zerr_spt_tem = zerr_spt_match[ind] separation_match = separation[ind] ra_spt_mat = ra_spt_match[ind] dec_spt_mat = dec_spt_match[ind] pix_spt_match = hp_pix_spt_match[ind] pix_spt_unmatch = hp_pix_spt_match[~ind] pix_spt_match = hp.nest2ring(4096, pix_spt_match) pix_spt_unmatch = hp.nest2ring(4096, pix_spt_unmatch) print len(z_spt_tem), 'after ra/dec matching <3Mpc/h' print len(ind) - len(z_spt_tem), 'number of unmatched clusters' ''' lambda_match = lam_act[ind] Z_unmatch = Ztile_des[~ind] Z_match = Ztile_des[ind] rades_match = rades[ind] decdes_match = decdes[ind] Zerr_match = Zerrtile_des[ind] memid_match = memid_des[ind] z_act_tem = z_act_match4[ind] z_act_unmatch = z_act_match4[~ind] zerr_act_tem = zerr_act_match4[ind]
def get_position(Nsides, pixel_nest):
pix_r = hp.nest2ring(Nsides, pixel_nest)
theta, phi = hp.pixelfunc.pix2ang(Nsides, pix_r)
return (pix_r, theta, phi)
def __init__(self, config):
# record for posterity
self.depthfile = config.depthfile
depthinfo, hdr = fitsio.read(self.depthfile, ext=1, header=True, upper=True)
# convert into catalog for convenience...
depthinfo = Catalog(depthinfo)
#self.npix = depthinfo.hpix.size
nside_mask = hdr['NSIDE']
nest = hdr['NEST']
self.nsig = hdr['NSIG']
self.zp = hdr['ZP']
self.nband = hdr['NBAND']
self.w = hdr['W']
self.eff = hdr['EFF']
self.config_area = config.area
self.submask_hpix = config.hpix
self.submask_nside = config.nside
self.submask_border = config.border
self.galfile_nside = config.galfile_nside
if nest != 1:
hpix_ring = depthinfo.hpix
else:
hpix_ring = hp.nest2ring(nside_mask, depthinfo.hpix)
# if we have a sub-region of the sky, cut down mask to save memory
if self.submask_hpix > 0:
border = np.radians(self.submask_border) + hp.nside2resol(nside_mask)
theta, phi = hp.pix2ang(self.submask_nside, self.submask_hpix)
radius = np.sqrt(2) * (hp.nside2resol(self.submask_nside)/2. + border)
pixint = hp.query_disc(nside_mask, hp.ang2vec(theta, phi),
radius, inclusive=False)
suba, subb = esutil.numpy_util.match(pixint, hpix_ring)
hpix_ring = hpix_ring[subb]
duse = subb
else:
duse = np.arange(hpix_ring.size, dtype='i4')
self.nside = nside_mask
self.offset = np.min(hpix_ring) - 1
self.ntot = np.max(hpix_ring) - np.min(hpix_ring) + 3
self.npix = hpix_ring.size
self.fracgood = np.zeros(self.npix + 1, dtype='f4')
try:
self.fracgood_float = 1
self.fracgood[0:self.npix] = depthinfo[duse].fracgood
except AttributeError:
self.fracgood_float = 0
self.fracgood[0:self.npix] = 0
self.exptime = np.zeros(self.npix + 1, dtype='f4')
self.exptime[0:self.npix] = depthinfo[duse].exptime
self.limmag = np.zeros(self.npix + 1, dtype='f4')
self.limmag[0:self.npix] = depthinfo[duse].limmag
self.m50 = np.zeros(self.npix + 1, dtype='f4')
self.m50[0:self.npix] = depthinfo[duse].m50
# And the overflow bins
self.fracgood[self.npix] = hp.UNSEEN
self.exptime[self.npix] = hp.UNSEEN
self.limmag[self.npix] = hp.UNSEEN
self.m50[self.npix] = hp.UNSEEN
# The look-up table
# Set default to overflow bin
self.hpix_to_index = np.zeros(self.ntot, dtype='i4') + self.npix
self.hpix_to_index[hpix_ring - self.offset] = np.arange(self.npix)
def calculate_overlap_with_observations(self,
fields=None,
pid=None,
first_det_window_days=3.0,
min_sep=0.01):
if fields is None:
mns = self.get_multi_night_summary(first_det_window_days)
else:
class MNS:
def __init__(self, data):
self.data = pandas.DataFrame(
data, columns=["field", "ra", "dec", "datetime"])
data = []
for f in fields:
ra, dec = ztfquery_fields.get_field_centroid(f)[0]
for i in range(2):
t = Time(self.t_min.jd + 0.1 * i, format="jd").utc
t.format = "isot"
t = t.value
data.append([f, ra, dec, t])
mns = MNS(data)
# Skip all 64 simultaneous quadrant entries, we only need one per observation for qa log
# data = mns.data.copy().iloc[::64]
data = mns.data.copy()
ras = np.ones_like(data["field"]) * np.nan
decs = np.ones_like(data["field"]) * np.nan
# Actually load up ra/dec
veto_fields = []
for field in list(set(data["field"])):
mask = data["field"] == field
res = ztfquery_fields.get_field_centroid(field)
if len(res) > 0:
ras[mask] = res[0][0]
decs[mask] = res[0][1]
else:
veto_fields.append(field)
if len(veto_fields) > 0:
self.logger.info(
f"No RA/Dec found by ztfquery for fields {veto_fields}. "
f"These observation have to be ignored.")
data["ra"] = ras
data["dec"] = decs
mask = np.array([~np.isnan(x) for x in data["ra"]])
data = data[mask]
if pid is not None:
pid_mask = data["pid"] == str(pid)
data = data[pid_mask]
obs_times = np.array([
Time(
data["datetime"].iat[i].replace(" ", "T"),
format="isot",
scale="utc",
) for i in range(len(data))
])
if first_det_window_days is not None:
first_det_mask = [
x < Time(self.t_min.jd + first_det_window_days,
format="jd").utc for x in obs_times
]
data = data[first_det_mask]
obs_times = obs_times[first_det_mask]
self.logger.info(f"Most recent observation found is {obs_times[-1]}")
self.logger.info("Unpacking observations")
pix_map = dict()
pix_obs_times = dict()
infile = os.path.join("nuztf", "data",
f"ztf_fields_ipix_nside={self.nside}.pickle")
# Generate a lookup table for field healpix
# if none exists (because this is computationally costly)
if not os.path.isfile(infile):
self.generate_flatpix_file()
with open(infile, "rb") as f:
field_pix = pickle.load(f)
f.close()
for i, obs_time in enumerate(tqdm(obs_times)):
field = data["field"].iat[i]
flat_pix = field_pix[field]
t = obs_time.jd
for p in flat_pix:
if p not in pix_obs_times.keys():
pix_obs_times[p] = [t]
else:
pix_obs_times[p] += [t]
if p not in pix_map.keys():
pix_map[p] = [field]
else:
pix_map[p] += [field]
npix = hp.nside2npix(self.nside)
theta, phi = hp.pix2ang(self.nside, np.arange(npix), nest=False)
radecs = SkyCoord(ra=phi * u.rad, dec=(0.5 * np.pi - theta) * u.rad)
idx = np.where(np.abs(radecs.galactic.b.deg) <= 10.0)[0]
double_in_plane_pixels = []
double_in_plane_probs = []
single_in_plane_pixels = []
single_in_plane_prob = []
veto_pixels = []
plane_pixels = []
plane_probs = []
times = []
double_no_plane_prob = []
double_no_plane_pixels = []
single_no_plane_prob = []
single_no_plane_pixels = []
overlapping_fields = []
for i, p in enumerate(tqdm(hp.nest2ring(self.nside, self.pixel_nos))):
if p in pix_obs_times.keys():
if p in idx:
plane_pixels.append(p)
plane_probs.append(self.map_probs[i])
obs = pix_obs_times[p]
# check which healpix are observed twice
if max(obs) - min(obs) > min_sep:
# is it in galactic plane or not?
if p not in idx:
double_no_plane_prob.append(self.map_probs[i])
double_no_plane_pixels.append(p)
else:
double_in_plane_probs.append(self.map_probs[i])
double_in_plane_pixels.append(p)
else:
if p not in idx:
single_no_plane_pixels.append(p)
single_no_plane_prob.append(self.map_probs[i])
else:
single_in_plane_prob.append(self.map_probs[i])
single_in_plane_pixels.append(p)
overlapping_fields += pix_map[p]
times += list(obs)
else:
veto_pixels.append(p)
overlapping_fields = sorted(list(set(overlapping_fields)))
try:
self.first_obs = Time(min(times), format="jd")
self.first_obs.utc.format = "isot"
self.last_obs = Time(max(times), format="jd")
self.last_obs.utc.format = "isot"
except ValueError:
err = (
f"No observations of this field were found at any time between {self.t_min} and"
f"{obs_times[-1]}. Coverage overlap is 0%, but recent observations might be missing!"
)
self.logger.error(err)
raise ValueError(err)
self.logger.info(f"Observations started at {self.first_obs.jd}")
return (
double_in_plane_pixels,
double_in_plane_probs,
single_in_plane_pixels,
single_in_plane_prob,
veto_pixels,
plane_pixels,
plane_probs,
times,
double_no_plane_prob,
double_no_plane_pixels,
single_no_plane_prob,
single_no_plane_pixels,
overlapping_fields,
)
hpx2048index = [int(a, 2) for a in hpx2048index_bin]
hpx2048index_enlarged = [int(a, 2) for a in hpx2048index_enlarged_bin]
mask_enlarged = np.in1d(correctallhpxindices, hpx2048index_enlarged)
numspecs_enlarged = len(mask_enlarged[mask_enlarged])
if numspecs_enlarged < 1:
continue
print 'number of specs we want', len(hpx2048index)
print 'number of specs we will use for gridding', numspecs_enlarged
fileindices = np.unique(infodict['allfileindices'][mask_enlarged])
filenames = np.array(infodict['filenames'])[fileindices]
print 'number of files contributing', len(filenames)
targethpx = healpy.nest2ring(targetnside, hpx2048index)
theta, phi = healpy.pix2ang(targetnside, targethpx, nest=False)
header = pf.getheader(datapath + filenames[0], 1)
# bugfix:
header['CDELT3'] = 1288.2149691241
if targetres is not None:
print 'computing new gridding kernel, ',
print 'to match target resolution of', targetres * 60, 'arcmin'
if targetres < header['BMAJ']:
print 'error, target resolution smaller than original resolution'
sys.exit(1)
if targetres < header['BMAJ'] * np.sqrt(1 + 0.05 ** 2):
print 'warning, target resolution too small ',
print '(gridding kernel would be less than 5%% of original resolution'
kernelsizefwhm = np.sqrt(targetres ** 2 - header['BMAJ'] ** 2)
def tile_geometry(polygon, nside, ring=False, return_coords=False, inclusive=True):
"""Returns a list of pixels that tile a Shapely polygon.
While `healpy.query_polygon` provides an efficient method to tile a
polygon, that function assumes that the lines joining the vertices of the
polygon are the segments of the great circles that intersect the vertices.
This is relatively irrelevant for polygons that cover a small area on the
sky but for large polygons the result is significantly different from what
one would expect.
This function tiles a `Polygon <shapely:Polygon>` assuming an Euclidian
distance between the vertices. The polygon must have longitude
limits defined between -360 and 720 degrees.
If ``inclusive=True`` the function will return all pixels that overlap with
the region. The area of the pixel is approximated as a circle with the
same area as a pixel. This approximation is good for low latitudes but
becomes worse for higher latitudes, where the shape of the pixel cannot
be approximated by a circle. If ``inclusive=False`` only pixels whose
centre is inside the region will be returned.
Parameters
----------
polygon : shapely:Polygon or ~numpy.ndarray
The polygon to tile. If an array, it must be a collection of ``Nx2``
points defining the position of the vertices. The polygon must be
convex.
nside : int
The nside of the pixels used to tile the polygon.
ring : bool
By default the function returns the values in the nested pixel
ordering. If ``ring=True`` the equivalent ring pixels will be returned.
return_coords : bool
If `True`, returns an array with the longitude and latitude of the
pixels in degrees.
inclusive : bool
Whether to return pixels that only partially overlap with the region.
Returns
-------
tiling : `~numpy.ndarray`
An array with the list of pixels that tile the geometry or (if
``return_coords=True``) the longitude and latitude of the pixels.
"""
# nside = 2^k
assert numpy.log2(nside).is_integer(), 'nside is not a power of 2.'
k_end = int(numpy.log2(nside))
if not isinstance(polygon, shapely.geometry.Polygon):
polygon = shapely.geometry.Polygon(polygon.tolist())
# Create a prepared polygon. This allows only contained and intersect
# operations but that's all we need and it's more efficient.
prep_polygon = shapely.prepared.PreparedGeometry(polygon)
pixels = []
intersect = []
for kk in range(0, k_end + 1):
nside_k = 2**kk
# Approximates the pixel as a circle of radius r.
rr = numpy.sqrt(healpy.nside2pixarea(nside_k, degrees=True) / numpy.pi)
# If k=0 (first HealPix level) we test all the 12 pixels. Otherwise we
# take the pixels that overlapped in the previous level and test each
# one of their children.
if kk == 0 and len(intersect) == 0:
pix_to_test = list(range(0, 12))
else:
pix_to_test = nested_regrade(intersect, 2**(kk - 1), nside_k).flatten().tolist()
intersect = []
for pix in pix_to_test:
lon, lat = healpy.pix2ang(nside_k, pix, nest=True, lonlat=True)
# We offset the pixel to check so that if the polygon wraps around
# [0, 360] we still overlap with it.
for offset in [0, -360, 360]:
# Create a Point object with a radius dd centred at the
# position of the pixel +/- the offset.
point = shapely.geometry.Point(lon + offset, lat).buffer(rr)
# If a pixel is completely contained by the polygon, adds
# all the nested pixels at the nside resolution and we are done.
if prep_polygon.contains(point):
if nside_k < nside:
pixels += nested_regrade(pix, nside_k, nside).flatten().tolist()
else:
pixels.append(pix)
break
# If we are at the final nside level and the pixel intersects
# with the polygon, we include it.
if nside_k == nside and inclusive and prep_polygon.intersects(point):
pixels.append(pix)
break
# If we are not yet at the final nside level we want to be greedy.
# We create a pixel with twice the radius and check if it intersects.
# If it does we add it to the list of pixels to test in the next
# nside level. We do this to compensate for the fact that pixels
# at high latitudes are significantly non-circular and if we
# just use point we may be missing some pixels.
if nside_k < nside:
point_2r = shapely.geometry.Point(lon + offset, lat).buffer(2. * rr)
if prep_polygon.intersects(point_2r):
intersect.append(pix)
break
intersect = numpy.unique(intersect).tolist()
if len(pixels) == 0:
raise ValueError('the list of tiling pixels is empty.')
pixels = numpy.unique(pixels)
if return_coords:
lon, lat = healpy.pix2ang(nside, pixels, nest=True, lonlat=True)
return numpy.array([lon, lat]).T
if ring:
return healpy.nest2ring(nside, pixels)
return pixels
stars_color=opts.stars_color,
stars_alpha=opts.stars_alpha,
arms=arms,
arms_color=opts.arms_color,
arms_linewidth=opts.arms_linewidth,
arms_alpha=opts.arms_alpha,
)
for label in labels:
fits = maps[label]['fits']
if opts.verbose:
print "reading map from", fits
post, header = hp.read_map(fits, h=True, verbose=False)
npix = len(post)
if (dict(header)['ORDERING'] == 'NEST'): ### convert to RING ordering
post = hp.nest2ring(nside, post)
nside = hp.npix2nside(npix)
if opts.verbose:
print " nside=%d" % nside
fig, ax = ct.genCR_fig_ax(figind,
figwidth=opts.figwidth,
figheight=opts.figheight,
grid=opts.grid)
figind += 1
ct.heatmap(post,
ax,
xlim,
ylim,
color_map=opts.color_map,
maps_s2_noise[:,1:,:]*=2
maps_noise_weights=1./maps_s2_noise
npix_spec=hp.nside2npix(nside_spec)
ipix0=hp.ring2nest(par.nside_spec,hp.ang2pix(par.nside_spec,theta_patch*np.pi/180,phi_patch*np.pi/180))
print "pixel", ipix0
map_mean=np.zeros([par.n_pix,par.n_pol,par.n_comp])
map_sigma=np.zeros([par.n_pix,par.n_pol,par.n_comp])
map_xspec_mean=np.zeros([npix_spec,par.n_spec_vary])
map_xspec_sigma=np.zeros([npix_spec,par.n_spec_vary])
#for ipix in [ipix0] :
for ipix in np.arange(npix_spec) :
if ipix!=ipix0 :
continue
ipix_list=hp.nest2ring(par.nside,ipix*par.n_sub+np.arange(par.n_sub))
if plot_stuff :
map_show=np.zeros(par.n_pix); map_show[ipix_list]=1.0; hp.mollview(map_show); plt.show()
patch_cmb_true=map_cmb_true[ipix_list,:]
patch_obs=maps_obs[ipix_list,:,:]
patch_noise_weights=maps_noise_weights[ipix_list,:,:]
if plot_stuff :
for ipol in np.arange(par.n_pol) :
plt.title("True CMB %d"%ipol)
plt.imshow(np.reshape(patch_cmb_true[:,ipol],(par.n_side_sub,par.n_side_sub)),interpolation='none',origin='lower');
plt.show()
if plot_stuff==2 :
def worker(image, return_dict, procnum):
#Read image header and get coordinates of the image
print(i, 'working with image', image)
hdr = pf.open(image)
w = astrowcs.WCS(hdr[1].header)
corners_image = w.calc_footprint(center=False)
corners_pixels = w.calc_footprint(center=True)
nx = hdr[1].header['naxis1']
ny = hdr[1].header['naxis2']
coords1 = corners_image[0]
coords2 = corners_image[1]
coords3 = corners_image[2]
coords4 = corners_image[3]
#Read image data
image_data = pf.getdata(image)
#Define healsparse polygon within the image
ra = [coords1[0], coords4[0], coords3[0], coords2[0]]
dec = [coords1[1], coords4[1], coords3[1], coords2[1]]
raC = (np.max(ra) + np.min(ra)) / 2.
decC = (np.max(dec) + np.min(dec)) / 2.
nside = 2**17
poly = hs.Polygon(ra=ra, dec=dec, value=1)
smap = poly.get_map(nside=nside, dtype=np.int16)
a = smap.validPixels
b = hp.nest2ring(nside, a)
#a are pixels in NEST, b in RING
#Get center coordinates of each pixel
raF, decF = hp.pix2ang(nside, a, lonlat=True, nest=True)
#Get nx, ny in image from healsparse pixels
myhdr = hdr[1].header
wcs = wc.WCS(myhdr)
xn, yn = wcs.sky2image(raF, decF)
#Get associated weight values
values = []
for x, y in zip(xn, yn):
values.append(image_data[int(y - 1), int(x - 1)])
values = np.array(values)
#Define healsparse map
hsp_map_2 = hs.HealSparseMap.makeEmpty(512, nside, dtype=np.int16)
hsp_map_2.updateValues(a, values)
#Degrade to nside=4096
low_res_hsp = hsp_map_2.degrade(4096)
j = low_res_hsp.validPixels
test_values_2 = low_res_hsp.getValuePixel(j)
#Uncomment if you want in RING format
#k=hp.nest2ring(4096,j)
#hp_aux = np.zeros(hp.nside2npix(4096))+hp.UNSEEN
#hp_aux[j] = test_values_2
hdr.close()
return_dict[procnum] = np.array([j, test_values_2]).transpose()
def healpix2radec(ipix,nside):
ipix = healpy.nest2ring(nside,ipix)
(theta,phi) = healpy.pix2ang(nside,ipix)
return polar2radec(theta,phi)
def read_sky_map(filename, nest=False, distances=False, moc=False):
"""
Read a LIGO/Virgo-type sky map and return a tuple of the HEALPix array
and a dictionary of metadata from the header.
Parameters
----------
filename: string
Path to the optionally gzip-compressed FITS file.
nest: bool, optional
If omitted or False, then detect the pixel ordering in the FITS file
and rearrange if necessary to RING indexing before returning.
If True, then detect the pixel ordering and rearrange if necessary to
NESTED indexing before returning.
If None, then preserve the ordering from the FITS file.
Regardless of the value of this option, the ordering used in the FITS
file is indicated as the value of the 'nest' key in the metadata
dictionary.
distances: bool, optional
If true, then read also read the additional HEALPix layers representing
the conditional mean and standard deviation of distance as a function
of sky location.
moc: bool, optional
If true, then preserve multi-order structure if present.
"""
m = Table.read(filename, format='fits')
del m.meta['PIXTYPE']
if m.meta.pop('COORDSYS', 'C') != 'C':
raise ValueError('LALInference only reads and writes sky maps in equatorial coordinates.')
try:
value = m.meta.pop('ORDERING')
except KeyError:
pass
else:
if value == 'RING':
m.meta['nest'] = False
elif value == 'NESTED':
m.meta['nest'] = True
elif value == 'NUNIQ':
pass
else:
raise ValueError(
'ORDERING card in header has unknown value: {0}'.format(value))
for fits_key, rows in itertools.groupby(FITS_META_MAPPING, lambda row: row[1]):
try:
value = m.meta.pop(fits_key)
except KeyError:
pass
else:
for row in rows:
key, _, _, _, from_fits = row
if from_fits is not None:
m.meta[key] = from_fits(value)
if 'UNIQ' in m.colnames and not moc:
from ..bayestar.sky_map import rasterize
m = rasterize(m)
m.meta['nest'] = True
if 'UNIQ' not in m.colnames:
npix = len(m)
nside = hp.npix2nside(npix)
if nest is None:
pass
elif m.meta['nest'] and not nest:
m = m[hp.ring2nest(nside, np.arange(npix))]
elif not m.meta['nest'] and nest:
m = m[hp.nest2ring(nside, np.arange(npix))]
if not moc:
if distances:
return tuple(np.asarray(m[name]).ravel() for name in DEFAULT_NESTED_NAMES), m.meta
else:
return np.asarray(m[DEFAULT_NESTED_NAMES[0]]).ravel(), m.meta
return m
def hsp2hpx(hspmap):
""" Convert a healsparse map to a sparse healpix map """
nside = hspmap.nside_sparse
pixels = hp.nest2ring(nside, hspmap.valid_pixels)
hpxmap = hspmap.get_values_pix(pixels)
return hpxmap, pixels, nside
def read_map(filename, nest=False, hdu=None, h=False, verbose=True):
"""Read a healpix map from a fits file. Partial-sky files,
if properly identified, are expanded to full size and filled with UNSEEN.
Uses fitsio to mirror much (but not all) of the functionality of healpy.read_map
Parameters:
-----------
filename : str
the fits file name
nest : bool, optional
If True return the map in NEST ordering, otherwise in RING ordering;
use fits keyword ORDERING to decide whether conversion is needed or not
If None, no conversion is performed.
hdu : int, optional
the header number to look at (start at 0)
h : bool, optional
If True, return also the header. Default: False.
verbose : bool, optional
If True, print a number of diagnostic messages
Returns
-------
m [, header] : array, optionally with header appended
The map read from the file, and the header if *h* is True.
"""
data, hdr = fitsio.read(filename, header=True, ext=hdu)
nside = int(hdr.get('NSIDE'))
if verbose: print('NSIDE = {0:d}'.format(nside))
if not healpy.isnsideok(nside):
raise ValueError('Wrong nside parameter.')
sz = healpy.nside2npix(nside)
ordering = hdr.get('ORDERING', 'UNDEF').strip()
if verbose: print('ORDERING = {0:s} in fits file'.format(ordering))
schm = hdr.get('INDXSCHM', 'UNDEF').strip()
if verbose: print('INDXSCHM = {0:s}'.format(schm))
if schm == 'EXPLICIT':
partial = True
elif schm == 'IMPLICIT':
partial = False
# monkey patch on a field method
fields = data.dtype.names
# Could be done more efficiently (but complicated) by reordering first
if hdr['INDXSCHM'] == 'EXPLICIT':
m = healpy.UNSEEN * np.ones(sz, dtype=data[fields[1]].dtype)
m[data[fields[0]]] = data[fields[1]]
else:
m = data[fields[0]].ravel()
if (not healpy.isnpixok(m.size) or (sz > 0 and sz != m.size)) and verbose:
print('nside={0:d}, sz={1:d}, m.size={2:d}'.format(nside, sz, m.size))
raise ValueError('Wrong nside parameter.')
if nest is not None:
if nest and ordering.startswith('RING'):
idx = healpy.nest2ring(nside, np.arange(m.size, dtype=np.int32))
m = m[idx]
if verbose: print('Ordering converted to NEST')
elif (not nest) and ordering.startswith('NESTED'):
idx = healpy.ring2nest(nside, np.arange(m.size, dtype=np.int32))
m = m[idx]
if verbose: print('Ordering converted to RING')
if h: return m, hdr
else: return m
def _unstage_data(
self,
comm,
data,
nsamp,
nnz,
nnz_full,
obs_period_ranges,
detectors,
signal_type,
pixels_dtype,
nside,
weight_dtype,
):
""" Clear Madam buffers, restore pointing into TOAST caches
and cache the destriped signal.
"""
auto_timer = timing.auto_timer(type(self).__name__)
self._madam_timestamps = None
self._cache.destroy("timestamps")
if self._conserve_memory:
nodecomm = comm.Split_type(MPI.COMM_TYPE_SHARED, comm.rank)
nread = nodecomm.size
else:
nodecomm = MPI.COMM_SELF
nread = 1
for iread in range(nread):
nodecomm.Barrier()
if nodecomm.rank % nread != iread:
continue
if self._name_out is not None:
global_offset = 0
for obs, period_ranges in zip(data.obs, obs_period_ranges):
tod = obs["tod"]
nlocal = tod.local_samples[1]
for idet, det in enumerate(detectors):
signal = np.ones(nlocal, dtype=signal_type) * np.nan
offset = global_offset
for istart, istop in period_ranges:
nn = istop - istart
dslice = slice(
idet * nsamp + offset, idet * nsamp + offset + nn
)
signal[istart:istop] = self._madam_signal[dslice]
offset += nn
cachename = "{}_{}".format(self._name_out, det)
tod.cache.put(cachename, signal, replace=True)
global_offset = offset
self._madam_signal = None
self._cache.destroy("signal")
if not self._purge_pixels:
# restore the pixels from the Madam buffers
global_offset = 0
for obs, period_ranges in zip(data.obs, obs_period_ranges):
tod = obs["tod"]
nlocal = tod.local_samples[1]
for idet, det in enumerate(detectors):
pixels = -np.ones(nlocal, dtype=pixels_dtype)
offset = global_offset
for istart, istop in period_ranges:
nn = istop - istart
dslice = slice(
idet * nsamp + offset, idet * nsamp + offset + nn
)
pixels[istart:istop] = self._madam_pixels[dslice]
offset += nn
npix = 12 * nside ** 2
good = np.logical_and(pixels >= 0, pixels < npix)
if not self._pixels_nested:
pixels[good] = hp.nest2ring(nside, pixels[good])
pixels[np.logical_not(good)] = -1
cachename = "{}_{}".format(self._pixels, det)
tod.cache.put(cachename, pixels, replace=True)
global_offset = offset
self._madam_pixels = None
self._cache.destroy("pixels")
if not self._purge_weights and nnz == nnz_full:
# restore the weights from the Madam buffers
global_offset = 0
for obs, period_ranges in zip(data.obs, obs_period_ranges):
tod = obs["tod"]
nlocal = tod.local_samples[1]
for idet, det in enumerate(detectors):
weights = np.zeros([nlocal, nnz], dtype=weight_dtype)
offset = global_offset
for istart, istop in period_ranges:
nn = istop - istart
dwslice = slice(
(idet * nsamp + offset) * nnz,
(idet * nsamp + offset + nn) * nnz,
)
weights[istart:istop] = self._madam_pixweights[
dwslice
].reshape([-1, nnz])
offset += nn
cachename = "{}_{}".format(self._weights, det)
tod.cache.put(cachename, weights, replace=True)
global_offset = offset
self._madam_pixweights = None
self._cache.destroy("pixweights")
del nodecomm
return
