def set_maps(self, map1, map2, mjd1, mjd2):
"""
set the two cloud maps to use to generate cloud motion predictions
"""
# Put things in correct order if not already
if mjd2 < mjd1:
self.mjd1 = mjd2
self.mjd2 = mjd1
self.map2 = map1
self.map1 = map2
else:
self.mjd1 = mjd1
self.mjd2 = mjd2
self.map2 = map2
self.map1 = map1
self.nside = hp.npix2nside(map1.size)
if hp.npix2nside(map2.size) != self.nside:
raise ValueError('nside of map1 and map2 do not match.')
# What is the time between images
self.dt = self.mjd2-self.mjd1
# Diff the maps, project to a screen.
# hmm, should probably change this to be a diff on screen rather than on sphere?
diff = diff_hp(self.map1, self.map2)
cloudyArea, self.cloudmask = cloudyness(diff)
self._find_dx_dy()
def randomPositions(input, nside_pix, n=1):
"""
Generate n random positions within a full HEALPix mask of booleans, or a set of (lon, lat) coordinates.
Parameters:
-----------
input : (1) full HEALPix mask of booleans, or (2) a set of (lon, lat) coordinates for catalog objects that define the occupied pixels.
nside_pix : nside_pix is meant to be at coarser resolution than the input mask or catalog object positions
so that gaps from star holes, bleed trails, cosmic rays, etc. are filled in.
Returns:
--------
lon,lat,area : Return the longitude and latitude of the random positions (deg) and the total area (deg^2).
"""
input = np.array(input)
if len(input.shape) == 1:
if hp.npix2nside(len(input)) < nside_pix:
logger.warning('Expected coarser resolution nside_pix in skymap.randomPositions')
subpix = np.nonzero(input)[0] # All the valid pixels in the mask at the NSIDE for the input mask
lon, lat = pix2ang(hp.npix2nside(len(input)), subpix)
elif len(input.shape) == 2:
lon, lat = input[0], input[1] # All catalog object positions
else:
logger.warning('Unexpected input dimensions for skymap.randomPositions')
pix = surveyPixel(lon, lat, nside_pix)
# Area with which the random points are thrown
area = len(pix) * hp.nside2pixarea(nside_pix, degrees=True)
# Create mask at the coarser resolution
mask = np.tile(False, hp.nside2npix(nside_pix))
mask[pix] = True
# Estimate the number of points that need to be thrown based off
# coverage fraction of the HEALPix mask
coverage_fraction = float(np.sum(mask)) / len(mask)
n_throw = int(n / coverage_fraction)
lon, lat = [], []
count = 0
while len(lon) < n:
lon_throw = np.random.uniform(0., 360., n_throw)
lat_throw = np.degrees(np.arcsin(np.random.uniform(-1., 1., n_throw)))
pix_throw = ugali.utils.healpix.angToPix(nside_pix, lon_throw, lat_throw)
cut = mask[pix_throw].astype(bool)
lon = np.append(lon, lon_throw[cut])
lat = np.append(lat, lat_throw[cut])
count += 1
if count > 10:
raise RuntimeError('Too many loops...')
return lon[0:n], lat[0:n], area
def get_combination(self, survey_name):
'''Generates a healpix map for a given survey that is a
combination of multiple other surveys in the configuration
file.
Handler in .cfg file: "combination"
Parameters
----------
survey_name : string
Configuration file block specifying survey parameters
Returns
-------
hpx_map : array-like
The healpix map associated with the survey
Notes
-----
The configuration file must contain a 'components' entry which lists
the different configuration file entries, separated by a ',' that make
up this survey.
'''
components = self.config.get(survey_name, 'components').split(',')
components = [tmp.strip() for tmp in components]
hpx_maps, coord = self.load_survey(components[0])
nside_hpx = H.npix2nside(len(hpx_maps[0]))
for component in components[1:]:
tmp_maps, coord_tmp = self.load_survey(component)
if coord_tmp != coord:
raise ValueError('''All maps in combination must be in the
same coordinate system''')
for tmp_map in tmp_maps:
nside_tmp = H.npix2nside(len(tmp_map))
if nside_tmp > nside_hpx:
nside_hpx = nside_tmp
hpx_maps.append(tmp_map)
nmaps = len(hpx_maps)
for i in range(nmaps):
hpx_maps[i] = H.ud_grade(hpx_maps[i], nside_hpx)
return hpx_maps, coord
def sphtrans_complex_pol(hpmaps, lmax=None, centered=False, lside=None):
"""Spherical harmonic transform of the polarisation on the sky (can be
complex).
Accepts complex T, Q, U and optionally V like maps, and returns :math:`a^T_{lm}`,
:math:`a^E_{lm}`, :math:`a^B_{lm}` and :math:`a^V_{lm}`.
Parameters
----------
hpmaps : np.ndarray
A list of complex Healpix maps, assumed to be T, Q, U (and optionally V).
lmax : scalar, optional
The maximum l to calculate. If `None` (default), calculate up to 3*nside
- 1.
centered : boolean, optional
If False (default) similar to an FFT, alm[l,:lmax+1] contains m >= 0,
and the latter half alm[l,lmax+1:], contains m < 0. If True the first
half opf alm[l,:] contains m < 0, and the second half m > 0. m = 0 is
the central column.
Returns
-------
alm_T, alm_E, alm_B : np.ndarray
A 2d array of alms, packed as alm[l,m].
"""
if lmax is None:
lmax = 3*healpy.npix2nside(hpmaps[0].size) - 1
rlms = _make_full_alm(sphtrans_real_pol([hpmap.real for hpmap in hpmaps], lmax=lmax, lside=lside), centered=centered)
ilms = _make_full_alm(sphtrans_real_pol([hpmap.imag for hpmap in hpmaps], lmax=lmax, lside=lside), centered=centered)
alms = [rlm + 1.0J * ilm for rlm, ilm in zip(rlms, ilms)]
return alms
def RaDec2MapValue(map=None, nest=False, cat=None, ra=None, dec=None):
"""
Find the value of a map for each entry in the catalog.
Parameters
----------
map (structured array)
The HEALPix map
cat (None/structured array)
If not None, the structured data array (e.g. numpy recarray)
ra (float array/str)
If `cat` is None, an array of the RA values. Otherwise, the column name for the RA column in `cat`.
dec (float array/str)
if `cat` is None, an array of the DEC values. Otherwise, the column name for the DEC column in `cat`.
nest (bool)
Whether or not to use nested format.
Returns
-------
vals (array)
Array of the map values for each entry
"""
if type(map)==str:
nest = _NestFromHeaderHP(map, -1)
mask = _hp.read_map(map, nest=nest)
else:
mask = map
nside = _hp.npix2nside(mask.size)
pix = RaDec2Healpix(ra=ra, dec=dec, nside=nside, nest=nest, cat=cat)
return mask[pix]
def clmFromMap(h, lmax):
"""
Given a pixel map, and a maximum l-value, return the corresponding C_{lm}
values.
@param h: Sky power map
@param lmax: Up to which order we'll be expanding
return: clm values
Use real_sph_harm for the map
"""
npixels = len(h)
nside = hp.npix2nside(npixels)
pixels = hp.pix2ang(nside, np.arange(npixels), nest=False)
clm = np.zeros( (lmax+1)**2 )
ind = 0
for ll in range(lmax+1):
for mm in range(-ll, ll+1):
clm[ind] += np.sum(h * real_sph_harm(mm, ll, pixels[1], pixels[0]))
ind += 1
return clm * 4 * np.pi / npixels
def inFootprint(footprint,ra,dec):
"""
Check if set of ra,dec combinations are in footprint.
Careful, input files must be in celestial coordinates.
filename : Either healpix map or mangle polygon file
ra,dec : Celestial coordinates
Returns:
inside : boolean array of coordinates in footprint
"""
if footprint is None:
return np.ones(len(ra),dtype=bool)
try:
if isinstance(footprint,str) and os.path.exists(footprint):
filename = footprint
#footprint = hp.read_map(filename,verbose=False)
#footprint = fitsio.read(filename)['I'].ravel()
footprint = read_map(filename)
nside = hp.npix2nside(len(footprint))
pix = ang2pix(nside,ra,dec)
inside = (footprint[pix] > 0)
except IOError:
logger.warning("Failed to load healpix footprint; trying to use mangle...")
inside = inMangle(filename,ra,dec)
return inside
def orfFromMap_fast(psr_locs, usermap, response=None):
"""
Calculate the correlation basis matrices using the pixel-space
transormations
@param psr_locs: Location of the pulsars [phi, theta]
@param usermap: Provide a healpix map for GW power
Note: GW directions are in direction of GW propagation
"""
if response is None:
npsrs = len(psr_locs)
pphi = psr_locs[:,0]
ptheta = psr_locs[:,1]
# Create the pixels
nside = hp.npix2nside(len(usermap))
npixels = hp.nside2npix(nside)
pixels = hp.pix2ang(nside, np.arange(npixels), nest=False)
gwtheta = pixels[0]
gwphi = pixels[1]
# Create the signal response matrix
F_e = signalResponse_fast(ptheta, pphi, gwtheta, gwphi)
elif response is not None:
F_e = response
# Double the power (one for each polarization)
sh = np.array([usermap, usermap]).T.flatten()
# Create the cross-pulsar covariance
hdcov_F = np.dot(F_e * sh, F_e.T)
# The pulsar term is added (only diagonals: uncorrelated)
return hdcov_F + np.diag(np.diag(hdcov_F))
def contour_pix(map, vals, all_neighbours=True): """ given a healpix map (map) and a set of values, we find and return lists of pixels that constitute the boarder of those sets """ npix = len(map) nside = hp.npix2nside(npix) ### separate into pixel sets based in p_values pix = np.arange(npix) boarders = [] for val in vals: _pix = pix[map>=val] ### pull out included pixels truth = np.zeros((npix,), bool) truth[_pix] = True boarder = np.zeros((npix,),bool) ### defines which modes are in the boarder for ipix in _pix: if all_neighbours: boarder[ipix] = not truth[[n for n in hp.get_all_neighbours(nside, ipix) if n != -1]].all() else: boarder[ipix] = not truth[[n for n in hp.get_neighbours(nside, ipix)[0] if n != -1]].all() boarders.append( pix[boarder] ) return boarders
def taylor_interpol_iter(m, pos, order=3, verbose=False, lmax=None):
"""Given a healpix map m[npix], and a set of positions
pos[{theta,phi},...], evaluate the values at those positions
using harmonic Taylor interpolation to the given order (3 by
default). Successively yields values for each cumulative order
up to the specified one. If verbose is specified, it will print
progress information to stderr."""
nside = hp.npix2nside(m.size)
if lmax is None: lmax = 3*nside
# Find the healpix pixel centers closest to pos,
# and our deviation from these pixel centers.
ipos = hp.ang2pix(nside, pos[0], pos[1])
pos0 = np.array(hp.pix2ang(nside, ipos))
dpos = pos[:2]-pos0
# Take wrapping into account
bad = dpos[1]>np.pi
dpos[1,bad] = dpos[1,bad]-2*np.pi
bad = dpos[1]<-np.pi
dpos[1,bad] = dpos[1,bad]+2*np.pi
# Since healpix' dphi actually returns dphi/sintheta, we choose
# to expand in terms of dphi*sintheta instead.
dpos[1] *= np.sin(pos0[0])
del pos0
# We will now Taylor expand our healpix field to
# get approximations for the values at our chosen
# locations. The structure of this section is
# somewhat complicated by the fact that alm2map_der1 returns
# two different derivatives at the same time.
derivs = [[m]]
res = m[ipos]
yield res
for o in range(1,order+1):
# Compute our derivatives
derivs2 = [None for i in range(o+1)]
used = [False for i in range(o+1)]
# Loop through previous level in steps of two (except last)
if verbose: tprint("order %d" % o)
for i in range(o):
# Each alm2map_der1 provides two derivatives, so avoid
# doing double work.
if i < o-1 and i % 2 == 1:
continue
a = hp.map2alm(derivs[i], use_weights=True, lmax=lmax, iter=0)
derivs[i] = None
dtheta, dphi = hp.alm2map_der1(a, nside, lmax=lmax)[-2:]
derivs2[i:i+2] = [dtheta,dphi]
del a, dtheta, dphi
# Use these to compute the next level
for j in range(i,min(i+2,o+1)):
if used[j]: continue
N = comb(o,j)/factorial(o)
res += N * derivs2[j][ipos] * dpos[0]**(o-j) * dpos[1]**j
used[j] = True
# If we are at the last order, we don't need to waste memory
# storing the derivatives any more
if o == order: derivs2[j] = None
derivs = derivs2
yield res
def read_hpx_maps(fns):
'''Read in one or more healpix maps and add them together. Must input
an array of strings even if only inputting a single map.
Parameters
----------
fns : list of strings
The filenames for the healpix maps to read in.
Returns
-------
hpx_map: array-like
A healpix map that is the sum of the Healpix maps in the input files.
Notes
-----
The nside of the output map will be the nside of the file map in the list.
Every other map will be upgraded or downgraded that that nside value.
'''
hpx_map = H.read_map(fns[0], verbose=False)
nside = H.npix2nside(len(hpx_map))
for fn_tmp in fns[1:]:
tmp_map = H.read_map(fn_tmp, verbose=False)
hpx_map += H.ud_grade(tmp_map, nside)
return hpx_map
def grid_theta_phi_to_healpix(theta,phi,inbeam):
"""
inputs:
theta (angle down from pixel 0, deg)
phi (CW longitude angle, deg)
inbeam: input beam values matching the corresponding theta and phi coords
"""
print len(theta),len(phi),len(inbeam)
nside = hp.npix2nside(len(inbeam))
pixes = hp.ang2pix(nside,theta,phi)
beam = np.zeros(hp.nside2npix(nside))
rms = np.zeros(hp.nside2npix(nside))
counts = np.zeros(hp.nside2npix(nside))
for i,pix in enumerate(pixes):
beam[pix] += inbeam[i]
rms[pix] += inbeam[i]**2
counts[pix] += 1
beam[counts>0] /= counts[counts>0]
rms[counts>0] /= counts[counts>0]
rms -= beam**2
rms = np.sqrt(rms)
beam[counts==0] = hp.UNSEEN
counts[counts==0] = hp.UNSEEN
rms[counts==0] = hp.UNSEEN
return beam,rms,counts
def __init__(self,
acquisition_or_rec_map,
input_map,
coverage=None,
mode='all',
coverage_thr=0.0,
tol=1e-4,
maxiter=300,
pickable=True,
noise=True,
weighted_noise=False,
run_analysis=True,
calculate_coverage=False):
modes = 'all', 'each'
if mode not in modes:
raise ValueError("The only modes implemented are {0}.".format(
strenum(modes, 'and')))
self.pickable = pickable
if isinstance(acquisition_or_rec_map, QubicAcquisition):
acquisition = acquisition_or_rec_map
convolution = acquisition.get_convolution_peak_operator()
self._scene_band = acquisition.scene.nu / 1e9
self._scene_nside = acquisition.scene.nside
self._scene_kind = acquisition.scene.kind
if not pickable:
self.acquisition = acquisition
self._detectors_number = len(acquisition.instrument.detector)
if run_analysis:
time0 = time()
self._tod, self.input_map_convolved = map2tod(
acquisition, input_map, convolution=True)
time1 = time()
print time1 - time0, 'for map2tod'
if noise:
# 4.7 corresponds to a one year of data noise
self._tod += 4.7 * acquisition.get_noise()
if mode == 'all':
tod2map_func = tod2map_all
elif mode == 'each':
tod2map_func = tod2map_each
time0 = time()
self.reconstructed_map, self.coverage = tod2map_func(
acquisition, self._tod, tol=tol, maxiter=maxiter)
time1 = time()
print time1 - time0, 'for tod2map'
if calculate_coverage:
self.calc_coverage()
self.coverage = convolution(self.coverage)
self.coverage_thr = coverage_thr
else:
if coverage == None:
raise TypeError("Must provide the coverage map!")
self._coverage_thr = coverage_thr
self._scene_band = None
self._scene_nside = hp.npix2nside(len(coverage))
self._scene_kind = 'IQU' if acquisition_or_rec_map.shape[1] == 3 else 'I'
self._detectors_number = 1984 if self._scene_kind == 'IQU' else 992
self.input_map_convolved = input_map # here the input map must be already convolved!
self.reconstructed_map = acquisition_or_rec_map
self.coverage = coverage
def coord_x2y(map, x, y):
"""Rotate a map from galactic co-ordinates into celestial co-ordinates.
This will operate on a series of maps (provided the Healpix index is last).
Parameters
----------
map : np.ndarray[..., npix]
Healpix map.
x, y : {'C', 'G', 'E'}
Coordinate system to transform to/from. Celestial, 'C'; Galactic 'G'; or
Ecliptic 'E'.
Returns
-------
rotmap : np.ndarray
The rotated map.
"""
if x not in ['C', 'G', 'E'] or y not in ['C', 'G', 'E']:
raise Exception("Co-ordinate system invalid.")
npix = map.shape[-1]
angpos = ang_positions(healpy.npix2nside(npix))
theta, phi = healpy.Rotator(coord=[y, x])(angpos[:, 0], angpos[:, 1])
map_flat = map.reshape((-1, npix))
for i in range(map_flat.shape[0]):
map_flat[i] = healpy.get_interp_val(map_flat[i], theta, phi)
return map_flat.reshape(map.shape)
def _reconstruct_nested_breadthfirst(m, extra):
max_npix = len(m)
max_nside = hp.npix2nside(max_npix)
max_order = _nside2order(max_nside)
seen = np.zeros(max_npix, dtype=bool)
for order in range(max_order + 1):
nside = _order2nside(order)
npix = hp.nside2npix(nside)
skip = max_npix // npix
if skip > 1:
b = m.reshape(-1, skip)
a = b[:, 0].reshape(-1, 1)
b = b[:, 1:]
aseen = seen.reshape(-1, skip)
eq = ((a == b) | ((a != a) & (b != b))).all(1) & (~aseen).all(1)
else:
eq = ~seen
for ipix in np.flatnonzero(eq):
ipix0 = ipix * skip
ipix1 = (ipix + 1) * skip
seen[ipix0:ipix1] = True
if extra:
yield _HEALPixTreeVisitExtra(
nside, max_nside, ipix, ipix0, ipix1, m[ipix0])
else:
yield _HEALPixTreeVisit(nside, ipix)
def _interpolate_level(m):
"""Recursive multi-resolution interpolation. Modifies `m` in place."""
# Determine resolution.
npix = len(m)
if npix > 12:
# Determine which pixels comprise multi-pixel tiles.
ipix = np.flatnonzero(
(m[0::4] == m[1::4]) &
(m[0::4] == m[2::4]) &
(m[0::4] == m[3::4]))
if len(ipix):
ipix = (4 * ipix +
np.expand_dims(np.arange(4, dtype=np.intp), 1)).T.ravel()
nside = hp.npix2nside(npix)
# Downsample.
m_lores = hp.ud_grade(
m, nside // 2, order_in='NESTED', order_out='NESTED')
# Interpolate recursively.
_interpolate_level(m_lores)
# Record interpolated multi-pixel tiles.
m[ipix] = hp.get_interp_val(
m_lores, *hp.pix2ang(nside, ipix, nest=True), nest=True)
def sphtrans_sky(skymap, lmax=None):
"""Transform a 3d skymap, slice by slice.
Parameters
----------
skymap : np.ndarray[freq, healpix_index]
lmax : integer, optional
Maximum l to transform.
Returns
-------
alms : np.ndarray[frequencies, l, m]
"""
nfreq = skymap.shape[0]
if lmax is None:
lmax = 3*healpy.npix2nside(skymap.shape[1]) - 1
alm_freq = np.empty((nfreq, lmax+1, lmax + 1), dtype=np.complex128)
for i in range(nfreq):
alm_freq[i] = sphtrans_real(skymap[i].astype(np.float64), lmax)
return alm_freq
def smooth_variance_map(var_m, fwhm):
"""Smooth a variance map
Algorithm from 'Pixel errors in convolved maps'
J.P. Leahy, version 0.2
Parameters
----------
var_m : array
input variance map
fwhm : float (radians)
target fwhm
Returns
-------
smoothed_var_m : array
smoothed variance map
"""
# smooth map
fwhm_variance = fwhm / np.sqrt(2)
smoothed_var_m = hp.smoothing(var_m, fwhm=fwhm_variance, regression=False)
# normalization factor
pix_area = hp.nside2pixarea(hp.npix2nside(len(var_m)))
orig_beam_width = fwhm/np.sqrt(8*np.log(2))
A_vb = pix_area / (4. * np.pi * orig_beam_width**2)
smoothed_var_m *= A_vb
return smoothed_var_m
def sphtrans_real(hpmap, lmax=None, lside=None):
"""Spherical Harmonic transform of a real map.
Parameters
----------
hpmap : np.ndarray
A Healpix map.
lmax : scalar, optional
The maximum l to calculate. If `None` (default), calculate up
to 3*nside - 1.
Returns
-------
alm : np.ndarray
A 2d array of alms, packed as alm[l,m].
Notes
-----
This only includes m > 0. As this is the transform of a real field:
.. math:: a_{l -m} = (-1)^m a_{lm}^*
"""
if lmax == None:
lmax = 3*healpy.npix2nside(hpmap.size) - 1
if lside == None or lside < lmax:
lside = lmax
alm = np.zeros([lside+1, lside+1], dtype=np.complex128)
tlm = healpy.map2alm(np.ascontiguousarray(hpmap), lmax=lmax, use_weights=_weight, iter=_iter)
alm[np.triu_indices(lmax+1)] = tlm
return alm.T
def sphtrans_complex(hpmap, lmax = None, centered = False, lside = None):
"""Spherical harmonic transform of a complex function.
Parameters
----------
hpmap : np.ndarray
A complex Healpix map.
lmax : scalar, optional
The maximum l to calculate. If `None` (default), calculate up to 3*nside
- 1.
centered : boolean, optional
If False (default) similar to an FFT, alm[l,:lmax+1] contains m >= 0,
and the latter half alm[l,lmax+1:], contains m < 0. If True the first
half of alm[l,:] contains m < 0, and the second half m > 0. m = 0 is
the central column.
Returns
-------
alm : np.ndarray
A 2d array of alms, packed as alm[l,m].
"""
if lmax == None:
lmax = 3*healpy.npix2nside(hpmap.size) - 1
rlm = _make_full_alm(sphtrans_real(hpmap.real, lmax = lmax, lside=lside),
centered = centered)
ilm = _make_full_alm(sphtrans_real(hpmap.imag, lmax = lmax, lside=lside),
centered = centered)
alm = rlm + 1.0J * ilm
return alm
def pseudo_pdf(self, dec_in, ra_in):
"""
Return pixel probability for a given dec_in and ra_in. Note, uses healpy functions to identify correct pixel.
"""
th, ph = HealPixSampler.decra2thph(dec_in, ra_in)
res = healpy.npix2nside(len(self.skymap))
return self.skymap[healpy.ang2pix(res, th, ph)]
def get_hem_Cls(skymap, direction, LMAX=256, deg=90.):
"""
from the given healpix skymap, return Cls for two hemispheres defined by the
direction given, useful to study the possible scale dependence of power modulation
direction should be a unit vector
"""
# generate hemispherical mask
NPIX=len(skymap)
NSIDE=hp.npix2nside(NPIX)
maskp=np.array([0.]*NPIX)
disc=hp.query_disc(nside=NSIDE, vec=direction, radius=0.0174532925*deg)
maskp[disc]=1.
#skymap=hp.remove_monopole(skymap)
map1=hp.ma(skymap)
map1.mask=maskp
Clsp=hp.anafast(map1, lmax=LMAX)
if (deg<90.):
maskm=np.array([0.]*NPIX)
disc=hp.query_disc(nside=NSIDE, vec=-direction, radius=0.0174532925*deg)
maskm[disc]=1.
map1.mask=maskm
else:
map1.mask=np.logical_not(maskp)
Clsm=hp.anafast(map1, lmax=LMAX)
return [Clsp, Clsm]
def __expand_valid(self, min_p=1e-7):
#
# Determine what the 'quanta' of probabilty is
#
if self._massp == 1.0:
# This is to ensure we don't blow away everything because the map
# is very spread out
min_p = min(min_p, max(self.skymap))
else:
# NOTE: Only valid if CDF descending order is kept
min_p = self.pseudo_pdf(*self.valid_points_decra[-1])
self.valid_points_hist = []
ns = healpy.npix2nside(len(self.skymap))
# Renormalize first so that the vector histogram is properly normalized
self._renorm = 0
# Account for probability lost due to cut off
for i, v in enumerate(self.skymap >= min_p):
self._renorm += self.skymap[i] if v else 0
for pt in self.valid_points_decra:
th, ph = HealPixSampler.decra2thph(pt[0], pt[1])
pix = healpy.ang2pix(ns, th, ph)
if self.skymap[pix] < min_p:
continue
self.valid_points_hist.extend([pt]*int(round(self.pseudo_pdf(*pt)/min_p)))
self.valid_points_hist = numpy.array(self.valid_points_hist).T
def rotate_jones(j, rotmat, multiway=True):
"""
Rotates the scalar components of a complex-valued 2x2 matrix field, relative
to the Healpix coordinate frame.
"""
if multiway == True:
j = flatten_jones(j)
npix = len(j[:,0])
nside = hp.npix2nside(npix)
hpxidx = np.arange(npix)
c, a = hp.pix2ang(nside, hpxidx)
t, p = rotate_sphr_coords(rotmat, c, a)
intp = lambda m: hp.get_interp_val(m, t, p)
# This is the fastest by ~2%. 99.1ms vs 101ms for the loop. (at nside=2**6)
# ...totally worth it!
jones = (np.asarray(map(intp,j.T))).T
# 101ms
# jR = np.empty_like(j)
# for i in range(8):
# jR[:,i] = intp(j[:,i])
# 102ms SO SLOW WTF NUMPY?
#jR = np.apply_along_axis(intp, 0, j)
# ahem. apparently this is just synactic sugar
if multiway == True:
jones = inverse_flatten_jones(jones)
return jones
def __init__(self, scheme, array):
import healpy
array = array.ravel()
nside = healpy.npix2nside(array.size)
self.hpix = HealPix(scheme, nside)
self.data = numpy.array(array, ndmin=1, copy=False)
def find_highest_probability_pixel( sky_map ):
"""
finding the highest probability pixel RA[deg], DEC[deg].
"""
import healpy as hp
import numpy as np
# read probability skymap
hpx = hp.read_map( sky_map, verbose = False )
# number of pixels
npix = len( hpx )
# nside: resolution for the HEALPix map
nside = hp.npix2nside( npix )
# pixel position
ipix_max = np.argmax( hpx )
hpx[ ipix_max ]
# sky coordinate
theta, phi = hp.pix2ang( nside, ipix_max )
ra_max = np.rad2deg( phi )
dec_max = np.rad2deg( 0.5 * np.pi - theta )
print ' or insert the highest probability pixel located \n at RA =', str(( '% .5f' % ra_max))+'°', 'and Dec =',str(( '% .5f' % dec_max))+'°.'
def zoncaview(m):
"""
m is a healpix sky map, such as provided by WMAP or Planck.
"""
nside = hp.npix2nside(len(m))
vmin = -1e3; vmax = 1e3
# Set up some grids:
xsize = ysize = 1000
theta = np.linspace(np.pi, 0, ysize)
phi = np.linspace(-np.pi, np.pi, xsize)
longitude = np.radians(np.linspace(-180, 180, xsize))
latitude = np.radians(np.linspace(-90, 90, ysize))
# Project the map to a rectangular matrix xsize x ysize:
PHI, THETA = np.meshgrid(phi, theta)
grid_pix = hp.ang2pix(nside, THETA, PHI)
grid_map = m[grid_pix]
# Create a sphere:
r = 0.3
x = r*np.sin(THETA)*np.cos(PHI)
y = r*np.sin(THETA)*np.sin(PHI)
z = r*np.cos(THETA)
# The figure:
mlab.figure(1, bgcolor=(1, 1, 1), fgcolor=(0, 0, 0), size=(400, 300))
mlab.clf()
mlab.mesh(x, y, z, scalars=grid_map, colormap="jet", vmin=vmin, vmax=vmax)
mlab.draw()
return
def get_spectra(maps, maskmap, lmax, min_ell, delta_ell, wl=None, Mllmat=None, MllBinned=None, ellpq=None, MllBinnedInv=None): nside = hp.npix2nside(len(maps[0])) if wl == None: wl = hp.anafast(maskmap,regression=False) wl = wl[0:lmax+1] if Mllmat == None: Mllmat=Mll(lmax,wl) cutmaps = np.array([maps[0] * maskmap, maps[1] * maskmap, maps[2] * maskmap]) cls = hp.anafast(cutmaps,pol=True) allcls= np.array([cls[0][0:lmax+1], cls[1][0:lmax+1], cls[2][0:lmax+1], cls[3][0:lmax+1], cls[4][0:lmax+1], cls[5][0:lmax+1]]) if ellpq == None: all_ell = min_ell-1 + np.arange(1000)*delta_ell max_ell = np.max(all_ell[all_ell < lmax]) nbins = np.int((max_ell + 1 - min_ell) / delta_ell) ell_low = min_ell + np.arange(nbins)*delta_ell ell_hi = ell_low + delta_ell - 1 the_ell = np.arange(lmax+1) newl, p, q = make_pq(the_ell, ell_low, ell_hi) else: newl = ellpq[0] p = ellpq[1] q = ellpq[2] if MllBinned == None: MllBinned = MllBin(Mllmat, p, q) if MllBinnedInv == None: MllBinnedInv = np.linalg.inv(MllBinned) ClsBinned = BinCls(p,allcls) newcls = np.dot(MllBinnedInv, np.ravel(np.array(ClsBinned))) newcls = np.reshape(newcls, (6, p.shape[0])) return newcls, newl, Mllmat, MllBinned, MllBinnedInv, p, q, allcls
def make_counts_maps(ra,dec,z,z_tics,mask):
"""
Make counts maps from ra,dec and z data.
@param ra Right assention of the object in degrees
@param dec Declination of the object in degrees
@param z Redshift of the object
@param z_tics an array of points that define the z-slizes
@param mask The mask to be applied. This should be a vecor or zero or one
entries and it should have the same number of pxiels as the output map.
Returns counts map as a numpy array of dimensions [npix,len(z_tics)-1]
"""
print "\ncreating the counts map"
# sanity checks
if len(ra) != len(dec) : raise RuntimeError("ra and dec have different sizes")
if len(ra) != len(z) : raise RuntimeError("ra and z have different sizes")
if len(z_tics) <2 : raise RuntimeError("at least 2 tics required to define a bin")
nside=0
try:
nside = hp.npix2nside(np.shape(mask)[0])
except:
raise RuntimeError("length of mask not a healpix npix")
# define the maps
nmaps = len(z_tics)-1
npix = np.shape(mask)[0]
counts = np.zeros([npix,nmaps])
# go through each object and add them into the correct bins
for i in range(len(ra)):
# convert (ra,dec) to (theta,phi)
theta=-degr2rad*dec[i] + np.pi/2.
phi=degr2rad*ra[i]
# find the pixel id
try:
pixid=hp.ang2pix(nside,theta,phi)
except:
print "wrong theta and phi: theta= ",theta,"phi= ",phi,"ra= ",ra[i],"dec= ",dec[i]
#raise RuntimeError("wrong theta and phi")
# is the pixel in the mask?
if mask[pixid]>0 :
# find the bin to which the current object belong to
binid=np.searchsorted(z_tics,z[i])-1
# is the bind in the range?
if binid >=0 and binid < len(z_tics)-1:
try:
counts[pixid,binid] += 1
except:
print "index error binid= ",binid,"len(z_tics)= ",len(z_tics),"z[i]= ",z[i]
# return the maps
return counts
def randomPositionsMask(mask, nside_pix, n):
"""
Generate n random positions within a HEALPix mask of booleans.
KCB: Likely superceded by the randomPositions function, but more generic.
"""
npix = len(mask)
nside = hp.npix2nside(npix)
# Estimate the number of points that need to be thrown based off
# coverage fraction of the HEALPix mask
coverage_fraction = float(np.sum(mask)) / len(mask)
n_throw = int(n / coverage_fraction)
lon, lat = [], []
latch = True
count = 0
while len(lon) < n:
lon_throw = np.random.uniform(0., 360., n_throw)
lat_throw = np.degrees(np.arcsin(np.random.uniform(-1., 1., n_throw)))
pix = ugali.utils.healpix.angToPix(nside, lon_throw, lat_throw)
cut = mask[pix].astype(bool)
lon = np.append(lon, lon_throw[cut])
lat = np.append(lat, lat_throw[cut])
count += 1
if count > 10:
raise RuntimeError('Too many loops...')
return lon[0:n], lat[0:n]
ids = data['arr_1']
pixel_kappas = data['arr_2']
pixel_area = hp.nside2pixarea(nside_k2)
k2 = np.zeros(nside_k2**2 * 12)
k2[ids] = pixel_kappas
Cls2 = hp.sphtfunc.anafast(k2, k00) / pixel_area
ell2 = np.arange(Cls2.size)
##- Open Midpoint kappa
wkappa_mid = fits.open(
'kappa_midpoints/kappa-cut-lensed-truecorr-rt100-rp100-nside128.fits.gz'
)[1].data.wkappa
kappa_mid = fits.open(
'kappa_midpoints/kappa-cut-lensed-truecorr-rt100-rp100-nside128.fits.gz'
)[1].data.kappa
nside_mid = int(hp.npix2nside(kappa_mid.size))
##- Mask off areas outside DESI footprint
mask = wkappa_mid != 0
mask &= (kappa_mid>np.percentile(kappa_mid[mask], 0.5)) & \
(kappa_mid<np.percentile(kappa_mid[mask], 99.5))
kappa_mid[~mask] = hp.UNSEEN
##- Reset midpoint map resolution if necessary
if (nside_mid != nside):
kappa_mid = hp.ud_grade(kappa_mid, nside)
##- Get Midpoint Cls
clx_mid = hp.sphtfunc.anafast(k00, kappa_mid)
ell_mid = np.arange(clx_mid.size)
def main(filename, zmin=None, zmax=None):
#Read the parameter file
params = read_redm_param(filename)
if zmin != None and zmax != None:
params['zmin'] = zmin
params['zmax'] = zmax
print >> sys.stderr, "Parameter file read successful"
#Read in the data
print >> sys.stderr, "Reading clusters file..."
if os.path.isfile(params['cluster_file']) == False:
print >> sys.stderr, "ERROR: Cluster catalog file " + params[
'cluster_file'] + " not found"
sys.exit(1)
cat = pyfits.open(params['cluster_file'])
cat = cat[1].data
print >> sys.stderr, len(cat), " clusters read in"
print >> sys.stderr, np.max(cat['z'])
print >> sys.stderr, "Reading members file..."
if os.path.isfile(params['member_file']) == False:
print >> sys.stderr, "ERROR: Member catalog file " + params[
'member_file'] + " not found"
sys.exit(1)
mem = pyfits.open(params['member_file'])
mem = mem[1].data
print >> sys.stderr, len(mem), " galaxies read in"
#If asked to use zred instead of z_lambda, set up for that
if int(params['use_zred']) == 1:
[cat, mem] = set_zred.set_zred(cat, mem)
#If not asked to use lambda_errors in calculations, set them to zero
if int(params['use_lambda_err']) == 0:
cat['lambda_chisq_e'][:] = cat['lambda_chisq_e'][:] * 0.
#Also read in data to abundance match to, if requested
if int(params['ABM']) == 1:
print >> sys.stderr, "Reading in clusters to use for abundance matching..."
if os.path.isfile(params['abm_file']) == False:
print >> sys.stderr, "ERROR: ABM catalog file " + params[
'abm_file'] + " not found"
sys.exit(1)
abm = pyfits.open(params['abm_file'])
abm = abm[1].data
print >> sys.stderr, len(abm), " clusters for ABM"
if params['stellarmass']:
mag = mem['MSTAR_50']
else:
#Read in amag data if it exists, otherwise run kcorrect
if os.path.isfile(params['kcorr_file']) == False:
#Running kcorrect -- note this requires running IDL
#os.system('setenv CATFILE '+params['cluster_file'])
os.environ['CATFILE'] = params['cluster_file']
os.environ['MEMFILE'] = params['member_file']
os.environ['KCORRFILE'] = params['kcorr_file']
os.environ['USE_DES'] = params['use_des']
os.environ['NO_UBAND'] = params['no_uband']
os.environ['LRG'] = params['LRG']
os.environ['BANDSHIFT'] = params['bandshift']
os.environ['USE_ZRED'] = params['use_zred']
os.system(
'/afs/slac/g/ki/software/idl/idl70/bin/idl < /afs/slac.stanford.edu/u/ki/chto100/code/redmapper_clf/redmapper/rm_kcorr_wrapper.pro'
)
if os.path.isfile(params['kcorr_file']) == False:
print >> sys.stderr, "ERROR: Kcorrect failed, no file found"
sys.exit(1)
kcorr = pyfits.open(params['kcorr_file'])
kcorr = kcorr[0].data
print >> sys.stderr, len(kcorr), " galaxies kcorrected"
if len(kcorr) != len(mem):
print >> sys.stderr, "ERROR! Number of galaxies != number of kcorrect values"
sys.exit(1)
#Pick up the mags we want for all galaxies
if (int(params['obs_clf']) == 1) & (int(params['use_lum']) == 0):
mag = mem['imag']
else:
if int(params['use_REFMAG']) == 1:
if params['Analband'] == "r":
print >> sys.stderr, "using r band"
mag = mem['REFMAG'] + (kcorr[:, 1] -
mem['model_mag'][:, 3])
elif params['Analband'] == "i":
mag = mem['REFMAG'] + (kcorr[:, 2] -
mem['model_mag'][:, 3])
else:
print >> sys.stderr, "ERRPR: OOPS I don't know the band you specified (because I am too lazy), which is {0}".format(
params['Analband'])
else:
print(params['use_des'])
if len(mem['model_mag'][0]) == 4:
mag = mem['imag'] + (kcorr[:, 2] - mem['model_mag'][:, 2])
else:
mag = mem['imag'] + (kcorr[:, 2] - mem['model_mag'][:, 3])
#Pulling limits from the parameters structure
lm_min = np.array(params['lm_min'])
lm_max = np.array(params['lm_max'])
zmin = np.array(params['zmin'])
zmax = np.array(params['zmax'])
lm_min = lm_min.astype(float)
lm_max = lm_max.astype(float)
zmin = zmin.astype(float)
zmax = zmax.astype(float)
#If requested, switch to ABM lambda limits
if int(params['ABM']) == 1:
print >> sys.stderr, "Running abundance matching..."
lm_min = abm_limits(cat, abm, params['area'], params['abm_area'], zmin,
zmax, lm_min)
lm_max = abm_limits(cat, abm, params['area'], params['abm_area'], zmin,
zmax, lm_max)
print "abundance matching: lm_min", lm_min
print "abundance matching: lm_max", lm_max
else:
my_nz = len(zmin)
lm_min = np.repeat([lm_min], my_nz, axis=0)
lm_max = np.repeat([lm_max], my_nz, axis=0)
#Get central absolute mags
#Will only recalcuate index if it doesn't exist; otherwise, read in the index
#and get the magnitudes from the mag values already available
#Updated version -- for redmapper v5.10 and greater -- uses indices supplied
#in the catalog files for finding the necessary index
c_names = cat.columns.names
use_id_cent = False
#Check to see if the central IDs are available
for name in c_names:
if name == 'ID_CENT':
use_id_cent = True
break
'''
if use_id_cent:
cengalindex = np.zeros_like(cat['id_cent'])
cenmag = np.zeros_like(cengalindex).astype(float)
#Hash table taking galaxy ID to index
offset = np.min(mem['id'])
g_index = np.zeros(np.max(mem['id'])-offset+1)-1
g_index[mem['id']-offset] = np.array(range(len(mem)))
#modified by chto to handle that cat['id_cent']=0
for i in range(len(cengalindex[0])):
index = np.where(cat['id_cent'][:,i]-offset>=0)
cengalindex[index,i] = g_index[cat['id_cent'][index,i]-offset]
cenmag[index,i] = mag[cengalindex[index,i]]
if int(params['weight_cen'])==0:
cenmag = cenmag[0]
cengalindex = cengalindex[0]
del g_index
'''
#############
#Algorithm changed by Chto
#The reason is that is np.zeros of a large number will cause mem error in Python
#############
if use_id_cent:
cengalindex = np.zeros_like(cat['id_cent'])
cenmag = np.zeros_like(cengalindex).astype(float)
#Hash table taking galaxy ID to index
offset = np.min(mem['id'])
g_index = dok_matrix((np.max(mem['id']) - offset + 1, 1), dtype=np.int)
g_index[mem['id'] - offset] = np.array(range(len(mem)))[:, np.newaxis]
if len(cenmag.shape) == 1:
index = np.where(cat['id_cent'][:] - offset >= 0)[0]
cengalindex[index] = g_index[cat['id_cent'][index] -
offset].toarray().flatten()
cenmag[index] = mag[cengalindex[index].astype(int)]
cengalindex = cengalindex.astype(int)
cengalindex = cengalindex.reshape(-1, 1)
cenmag = cenmag.reshape(-1, 1)
else:
for i in range(len(cengalindex[0])):
index = np.where(cat['id_cent'][:, i] - offset >= 0)[0]
cengalindex[index, i] = g_index[cat['id_cent'][index, i] -
offset].toarray().flatten()
cenmag[index, i] = mag[cengalindex[index, i]]
if int(params['weight_cen']) == 0:
cenmag = cenmag[:, 0]
cengalindex = cengalindex[:, 0]
cengalindex = cengalindex.reshape(-1, 1)
cenmag = cenmag.reshape(-1, 1)
del g_index
gc.collect()
#########################
else:
#No central IDs found -- doing it the hard way
if os.path.isfile(params['cindex_file']) == False:
print >> sys.stderr, "Getting central magnitudes..."
[cenmag, cengalindex
] = get_central_mag(cat,
mem,
mag,
weight_cen=int(params['weight_cen']))
cengalindex = cengalindex.astype(long)
hdu = pyfits.PrimaryHDU(cengalindex)
hdu.writeto(params['cindex_file'])
else:
print >> sys.stderr, "Reading in central magnitudes..."
cengalindex = pyfits.open(params['cindex_file'])
cengalindex = cengalindex[0].data
cengalindex = cengalindex.astype(long)
cenmag = np.zeros_like(cengalindex).astype(float)
print len(cenmag)
for i in range(len(cenmag)):
cenmag[i] = mag[cengalindex[i]]
#If available, get the limiting magnitude information
#Currently set for the more generous cut
use_limmag = False
for name in c_names:
if name == 'LIM_LIMMAG':
use_limmag = True
limmag = cat['LIM_LIMMAG']
break
#Convert limiting magnitude to absolute if needed; based on central
#galaxy's k-correction
if use_limmag:
if params['dohaloCat']:
limmag = cat['LIM_LIMMAG_DERED']
else:
if int(params['get_limmag_5sigma']) == 1:
limmag = get_limmag_5sigma(limmag, cat['lim_exptime'])
if int(params['use_des']) == 1:
if params['Analband'] == "r":
limmag = limmag - (cat['MODEL_MAG'][:, 1] - cenmag[:, 0])
elif params['Analband'] == "i":
limmag = limmag - (cat['MODEL_MAG'][:, 2] - cenmag[:, 0])
else:
print >> sys.stderr, "ERRPR: OOPS I don't know the band you specified (because I am too lazy), which is {0}".format(
params['Analband'])
else:
limmag = limmag - (cat['imag'] - cenmag[:, 0])
else:
limmag = []
if params['stellarmass']:
limmag = []
#For the rest of the calculations, since these are predicated on the
#given lambda/z thresholds, remove all the clusters we don't care about.
#This should speed things up significantly/avoid memory issues.
print >> sys.stderr, "Ultimate max z cut is at: ", float(
params['zcut_max'])
clist = np.where(cat['z_lambda'] < float(params['zcut_max']))[0]
cmlist = np.where(mem['z'] < float(params['zcut_max']))[0]
cat = cat[clist]
cenmag = cenmag[clist]
cengalindex = cengalindex[clist]
if len(limmag) != 0:
limmag = limmag[clist]
#Note that we also must trim galaxies
mem = mem[cmlist]
if not params['stellarmass']:
kcorr = kcorr[cmlist]
idn_list = np.zeros(len(mag))
idn_list[cmlist] = np.array(range(len(cmlist)))
mag = mag[cmlist]
cengalindex = idn_list[cengalindex]
cengalindex = cengalindex.astype(long)
del cmlist, clist
print "cmlist, clist: ", gc.collect()
####
##Try to know what is the magnitude.
#WARNING: Cuts should be VERY generous, otherwise, may have issues with P(z) tails...
#Make the main output directory
os.system("mkdir -p " + params['outdir'])
#Now that we have our magnitudes, add central corrections if necessary
if int(params['use_dr8_cen_corr']) == 1:
if int(params['weight_cen']) == 1:
cenmag[:, 0] = cenmag[:, 0] + correct_dr8_cen([0.213, -0.08],
cat['z_lambda'])
cenmag[:, 1] = cenmag[:, 1] + correct_dr8_cen([0.104, -0.036],
cat['z_lambda'])
mag[cengalindex[:, 0]] = mag[cengalindex[:, 0]] + correct_dr8_cen(
[0.213, -0.08], mem['z'][cengalindex[:, 0]])
corrlist = np.where(cat['ncent_good'] >= 2)[0]
if len(corrlist) > 0:
mag[cengalindex[
corrlist,
1]] = mag[cengalindex[corrlist, 1]] + correct_dr8_cen(
[0.104, -0.036], mem['z'][cengalindex[corrlist, 1]])
else:
cenmag = cenmag + correct_dr8_cen([0.213, -0.08], cat['z_lambda'])
mag[cengalindex] = mag[cengalindex] + correct_dr8_cen(
[0.213, -0.08], cat['z_lambda'])
#Convert everything to log(L) if requested
if int(params['use_lum']) == 1:
#Value currently hard-coded to offset for z=0.3 bandshift in SDSS i-band
abs_solar = float(params['abs_solar'])
mag = np.log10(
mag_to_Lsolar(mag,
use_des=int(params['use_des']),
abs_solar=abs_solar))
cenmag = np.log10(
mag_to_Lsolar(cenmag,
use_des=int(params['use_des']),
abs_solar=abs_solar))
if use_limmag:
limmag = np.log10(
mag_to_Lsolar(limmag,
use_des=int(params['use_des']),
abs_solar=abs_solar))
np.save(params['outdir'] + "limmag.npy", limmag)
np.save(params['outdir'] + "cengalindex.npy", cengalindex)
print "I save the limiting magnitude.... HAAH"
print >> sys.stderr, "Finished converting mags to log(Lsolar)"
np.save(params['outdir'] + "cen_mag.npy", cenmag)
# limmag=[] #so hacky :( this is not my style
print "magnitude distribution: "
print np.histogram(mag, bins=np.array(range(35)) * 0.08 + 9.3)
np.save(params['outdir'] + "mag.npy", mag)
#Fix the normaliztion of the p(z) so that triangular integration works okay in pz_utils
##Modified by chto@@
dz = cat['pzbins'][:, 1] - cat['pzbins'][:, 0]
weight = np.sum(cat['pz'], axis=1) * dz
cat['pz'] = cat['pz'] / weight[:, None]
del dz, weight
print "dz, weight: ", gc.collect()
print >> sys.stderr, "Done renormalizing P(z)"
print >> sys.stderr, "Max lambda: ", np.max(cat['lambda_chisq'])
#Now produce the galaxy samples and a hash table that takes cluster ID to the first
#of its listed galaxies
#Create an array that gives p_cen for all member galaxies
pcen_all = np.zeros(len(mem))
if int(params['weight_cen']) == 0:
pcen_all[cengalindex] = 0 * cengalindex + 1.
else:
for i in range(len(cengalindex[0])):
clist = np.where(cengalindex[:, i] != -1)[0]
#print len(cengalindex),cengalindex[i][0],clist[0]
pcen_all[cengalindex[
clist,
i]] = pcen_all[cengalindex[clist, i]] + cat['p_cen'][clist, i]
np.save(params['outdir'] + "pcen_all.py", pcen_all)
#Reassign p with p_ext if requested
if int(params['use_p_ext']) > 0:
mem['p'][:] = pext_correct.pext_correct_full(cat, mem, pcen_all,
int(params['use_p_ext']),
int(params['ncolors']))
#Add a systematic probability offset if requested
if float(params['p_offset']) != 0:
mem['p'][:] = mem['p'][:] + float(params['p_offset'])
#Fix any objects with p>1 or p<0
plist = np.where(mem['p'] > 1.)[0]
if len(plist) > 0:
mem['p'][plist] = 0. * plist + 1.
plist = np.where(mem['p'] < 0.)[0]
if len(plist) > 0:
mem['p'][plist] = 0. * plist
#Now make the satellite lists
print >> sys.stderr, "PCEN: ", np.max(pcen_all), len(cat), len(
np.where(pcen_all > 0)[0]), len(
np.where(pcen_all > 0.9)[0]), np.sum(pcen_all)
#Make necessary bootstrap samples
#Includes redshifts taken from P(z)
#Only want to do this once
#Also allows covariance estimates between measuresments, which are
#Not currently implemented
#Number of bootstrap samples -- current hard-coded
nboot = dodotune.njack
bootlist = pz_utils.make_boot_samples_simple(nboot, cat)
[match_index,
gboot] = pz_utils.make_boot_samples_gal_full(bootlist,
cat['mem_match_id'],
mem['mem_match_id'],
mem['p'] * (1 - pcen_all))
print >> sys.stderr, "Done setting up bootstrap samples"
assert params['jackknife_error']
if float(params['jackknife_error']) != 0:
print "doing jackknife CLF"
njack = dodotune.njack
jacklist = pz_utils.make_jack_samples_simple(njack, cat)
match_index_jack = pz_utils.getjackgal(jacklist[0],
cat['mem_match_id'],
mem['mem_match_id'])
assert (mem['mem_match_id'][match_index_jack[cat['mem_match_id']]
[:, 0]] == cat['mem_match_id']).all()
# [match_index_jack, gjack] = pz_utils.make_jack_samples_gal_full(jacklist,cat['mem_match_id'],
# mem['mem_match_id'],mem['p']*(1-pcen_all))
print >> sys.stderr, "Done setting up jackknife samples"
#If requested, calculate n(z)
if int(params['do_nz']) == 1:
print >> sys.stderr, "Calculating n(z)..."
redm_nz_calc(cat,
params['area'],
params['outdir'],
bootlist,
descale=bool(int(params['nz_descale'])))
print >> sys.stderr, "Done calculating n(z)"
#If requested, calculate n(lambda)
if int(params['do_nlambda']) == 1:
print >> sys.stderr, "Calculating n(lambda)..."
redm_nlambda_err(cat['lambda_chisq'], cat['lambda_chisq_e'],
cat['z_lambda'], cat['pz'], cat['pzbins'], bootlist,
params['outdir'], zmin, zmax, params['area'])
print >> sys.stderr, "Done calculating n(lambda)"
redm_bigcount(cat['lambda_chisq'], cat['z_lambda'], zmin, zmax,
params['outdir'])
print >> sys.stderr, "Done calculating bonus listing of massive clusters"
print >> sys.stderr, "lm_min: ", lm_min, "lm_max: ", lm_max, "zmin: ", zmin, "zmax: ", zmax
#Calculate the CLF
if int(params['do_clf']) == 1:
#np.save("mag.npy",mag)
#np.save("cenmag.npy",cenmag)
#np.save("cat.npy",cat)
#np.save("mem.npy", mem)
print >> sys.stderr, "Calculating CLF..."
if params["troughNames"] is not None:
trough_data = np.array([
hp.read_map(params['troughNames'].format(i)) for i in range(5)
])
nside = hp.npix2nside(trough_data.shape[1])
def getTroughProb(RaDec):
ra, dec = RaDec
return trough_data[:, DeclRaToIndex(ra, dec, nside)]
positionArray = np.array([cat['RA'], cat['DEC']]).T
positionArray_mem = np.array([mem['RA'], mem['DEC']]).T
troughProb = np.array(map(getTroughProb, positionArray))
troughProb_mem = np.array(map(getTroughProb, positionArray_mem))
oldmemP = copy.deepcopy(mem['p'])
oldcatpcen = copy.deepcopy(cat['p_cen'])
oldcatpsat = copy.deepcopy(cat['p_sat'])
for i in range(5):
mem['p'] *= troughProb_mem[:, i]
cat['p_cen'] *= troughProb[:, i].reshape(-1, 1)
cat['p_sat'] *= troughProb[:, i].reshape(-1, 1)
outdir = params['outdir'] + "trough_{0}/".format(i)
os.system("mkdir -p " + outdir)
redm_clf.redm_clf(cat,
mem,
mag,
cenmag,
cengalindex,
lm_min,
lm_max,
zmin,
zmax,
pcen_all,
bootlist,
gboot,
match_index,
outdir,
weight_cen=int(params['weight_cen']),
obs_clf=int(params['obs_clf']),
use_lum=int(params['use_lum']),
limmag=limmag,
trough=True,
stellarMass=params['stellarmass'])
mem['p'] = oldmemP
cat['p_cen'] = oldcatpcen
cat['p_sat'] = oldcatpsat
else:
#print limmag
if float(params['jackknife_error']) != 0:
print "doing jackknife CLF"
redm_clf.redm_clf(cat,
mem,
mag,
cenmag,
cengalindex,
lm_min,
lm_max,
zmin,
zmax,
pcen_all,
jacklist,
gboot=None,
match_index=match_index_jack,
outdir=params['outdir'],
weight_cen=int(params['weight_cen']),
obs_clf=int(params['obs_clf']),
use_lum=int(params['use_lum']),
limmag=limmag,
stellarMass=params['stellarmass'],
jackknifeerr=int(params['jackknife_error']))
else:
print "doing bootstrap CLF"
redm_clf.redm_clf(cat,
mem,
mag,
cenmag,
cengalindex,
lm_min,
lm_max,
zmin,
zmax,
pcen_all,
bootlist,
gboot,
match_index,
params['outdir'],
weight_cen=int(params['weight_cen']),
obs_clf=int(params['obs_clf']),
use_lum=int(params['use_lum']),
limmag=limmag,
stellarMass=params['stellarmass'])
print >> sys.stderr, "Done calculating CLF"
#PLot CLF:
if int(params['plot_clf']) == 1:
redm_clf.plot_clf(lm_min, lm_max, zmin, zmax, indir=params['outdir'])
# assert False
#Calculate the radial profiles
if int(params['do_rpr']) == 1:
print >> sys.stderr, "Calculating radial profiles..."
#First, get necessary input parameter limits on magnitude
rpr_minlum = np.array(params['rpr_minlum'])
rpr_maxlum = np.array(params['rpr_maxlum'])
rpr_minlum = rpr_minlum.astype(float)
rpr_maxlum = rpr_maxlum.astype(float)
#Basic weirdness checking for luminosity limits
error_check = 0
elist = np.where(rpr_minlum > rpr_maxlum)[0]
if len(elist) > 0:
print >> sys.stderr, "ERROR: Require rpr_minlum < rpr_maxlum"
error_check = 1
if int(params['obs_clf']) == 1 or int(
params['use_lum']) == 1 and min(rpr_minlum) < 0:
print >> sys.stderr, "ERROR: Radial profiles are using app mags or solar lum,"
print >> sys.stderr, " but rpr_minlum/rpr_maxlum < 0"
error_check = 1
if int(params['obs_clf']) == 0 and int(
params['use_lum']) == 0 and max(rpr_maxlum) > 0:
print >> sys.stderr, "ERROR: Radial profiles are using abs mags,"
print >> sys.stderr, " but rpr_minlum/rpr_maxlum > 0"
error_check = 1
if error_check == 0:
redm_rpr.redm_rpr(cat, mem, mag, lm_min, lm_max, zmin, zmax,
rpr_minlum, rpr_maxlum, bootlist, gboot,
params['outdir'])
else:
print >> sys.stderr, "SKIPPING RADIAL PROFILES"
#Calculate magnitude gaps
if int(params['do_mgap']) == 1:
print >> sys.stderr, "Calculating magnitude gaps..."
redm_mgap.redm_mgap(cat,
mem,
cenmag,
cengalindex,
mag,
zmin,
zmax,
lm_min,
lm_max,
bootlist,
gboot,
params['outdir'],
use_lum=bool(int(params['use_lum'])),
use_obs=bool(int(params['obs_clf'])),
weight_cen=bool(int(params['weight_cen'])))
#calculate the probability that the brightest galaxy is not the central galaxy
if int(params['do_pbcg']) == 1:
print >> sys.stderr, "Calculating P(BCG!=central)..."
redm_pbcg.get_p_bcg_not_cen(cat,
cenmag,
cengalindex,
cat['p_cen'],
mem['mem_match_id'],
mag,
mem['p'],
zmin,
zmax,
params['outdir'],
use_lum=int(params['use_lum']),
weight_cen=int(params['weight_cen']))
#Calculate the distribution of the brightest satellite galaxy
if int(params['do_bsat']) == 1:
print >> sys.stderr, "Calculating brightest satellite clf..."
redm_bright_sat.get_brightest_satellite_all(
cat,
mem,
mag,
cengalindex,
lm_min,
lm_max,
zmin,
zmax,
bootlist,
gboot,
match_index,
params['outdir'],
weight_cen=int(params['weight_cen']),
use_lum=int(params['use_lum']),
obs_clf=int(params['obs_clf']))
print >> sys.stderr, "Calculating joint brightest sat-central distribution..."
count_arr = redm_bright_sat.get_bright_sat_cen_all(
cat,
mem,
mag,
cengalindex,
cenmag,
lm_min,
lm_max,
zmin,
zmax,
bootlist,
gboot,
match_index,
params['outdir'],
weight_cen=int(params['weight_cen']),
use_lum=int(params['use_lum']),
obs_clf=int(params['obs_clf']))
#output the params
np.save(params['outdir'] + "params.npy", params)
def next(self):
if self.iter == self.iter_num:
mpiutil.barrier()
super(SurveySim, self).next()
mock_n = self.mock_n
freq = self.params['freq']
delta_nu = self.params['delta_nu']
#dfreq = freq[1] - freq[0]
freq_n = freq.shape[0]
block_time = self.block_time[:self.iter + 1]
block_n = int(block_time[-1] / self.obs_int.to(u.s).value)
idx_st = int(np.sum(block_time[:-1]) / self.obs_int.to(u.s).value)
idx_ed = idx_st + block_n
t_list = self.SM.t_list[idx_st:idx_ed]
ra_list = self.SM.ra[idx_st:idx_ed, :]
dec_list = self.SM.dec[idx_st:idx_ed, :]
radec_shp = ra_list.shape
_sky = np.zeros(
(mock_n, block_n, freq_n, len(self.ants))) + self.params['T_rec']
if self.params['fg_syn_model'] is not None:
logger.info("add syn")
raise ValueError('fg mode not ready')
syn_model_nside = hp.npix2nside(self.syn_model.shape[0])
_idx_pix = hp.ang2pix(syn_model_nside,
ra_list.flat,
dec_list.flat,
lonlat=True)
_sky += self.syn_model[_idx_pix, :][None, ...] #/ 2.
if self.params['HI_model'] is not None:
logger.info("add HI")
#HI_model_nside = hp.npix2nside(self.HI_model.shape[0])
#_idx_pix = hp.ang2pix(HI_model_nside, ra_list, dec_list, lonlat=True)
#_HI = self.HI_model[_idx_pix, :]
HI_model_ra = self.HI_model.get_axis('ra')
HI_model_dec = self.HI_model.get_axis('dec')
_HI = np.rollaxis(self.HI_model, 1, 4)
if not self.params['multibeam']:
_ra_list = ra_list[:, 0]
_dec_list = dec_list[:, 0]
ra_idx = np.argmin(np.abs(_ra_list[:, None] -
HI_model_ra[None, :]),
axis=1)
dec_idx = np.argmin(np.abs(_dec_list[:, None] -
HI_model_dec[None, :]),
axis=1)
_sky = _sky + _HI[:, list(ra_idx),
list(dec_idx), :, None] #/ 2.
else:
for bb in range(radec_shp[1]):
_ra_list = ra_list[:, bb]
_dec_list = dec_list[:, bb]
ra_idx = np.argmin(np.abs(_ra_list[:, None] -
HI_model_ra[None, :]),
axis=1)
dec_idx = np.argmin(np.abs(_dec_list[:, None] -
HI_model_dec[None, :]),
axis=1)
_sky[...,
bb] = _sky[..., bb] + _HI[:,
list(ra_idx),
list(dec_idx), :] #/ 2.
rvis = np.empty([mock_n, block_n, freq_n, 2, len(self.ants)])
if self.params['fnoise']:
logger.info("add 1/f")
for i in range(len(self.ants)):
for j in range(mock_n):
rvis[j,...,0,i] = _sky[j, ..., i]\
* (self.FN.realisation(freq_n, block_n, delta_nu)+ 1.)
rvis[j,...,1,i] = _sky[j, ..., i]\
* (self.FN.realisation(freq_n, block_n, delta_nu)+ 1.)
else:
logger.info("no 1/f")
rvis = _sky[..., None, :]
WN = self._RMS * np.random.randn(mock_n, block_n, freq_n, 2,
len(self.ants))
logger.info(" %f K(%f K^2)" % (np.std(WN), np.var(WN)))
rvis = rvis + WN
del WN
gc.collect()
del _sky
gc.collect()
#shp = (mock_n, block_n, freq_n, 4, len(self.blorder))
#vis = np.empty(shp, dtype=np.complex)
#vis[:, :, :, 0, self.auto_idx] = rvis[:, :, :, 0, :] + 0. * 1j
#vis[:, :, :, 1, self.auto_idx] = rvis[:, :, :, 1, :] + 0. * 1j
for ii in range(mock_n):
#for ii in self.HI_mock_ids:
#shp = (block_n, freq_n, 4, len(self.blorder))
#vis = np.empty(shp, dtype=np.complex)
#vis[:, :, 0, self.auto_idx] = rvis[ii, :, :, 0, :] + 0. * 1j
#vis[:, :, 1, self.auto_idx] = rvis[ii, :, :, 1, :] + 0. * 1j
#output_prefix = '/sim_mock%03d/%s_%s_%s_%s'%(
# ii, self.params['prefix'], self.params['survey_mode'],
# self.params['HI_scenario'], self.params['HI_model_type'])
#self.write_to_file(rvis[ii, ...] + 0. * 1j, output_prefix=output_prefix)
#self.write_to_file(rvis[ii, ...], output_prefix=output_prefix)
self.write_to_file(rvis[ii, ...], mock=ii)
#del vis
#gc.collect()
del rvis
gc.collect()
self.iter += 1
