def read(fname, splits=None, type="auto", maxmaps=1000, **kwargs):
"""Read the maps, ivars, beam, gain and fequency from a mapdata file, and
return them as a bunch of maps:[map,...], ivars:[ivar,...], beam:[:],
gain:num, freq:num. All maps are read by default. Use the splits argument
to read only a subset, e.g. splits=[0] to read only the first map for
each of maps and ivars.
All unrecognized arguments are forwarded to enmap.read_fits and used when
reading the maps, allowing for subset reading etc."""
import numpy as np
from pixell import enmap
if type == "auto": type = infer_type(fname)
if type == "zip": work, flexget = zipfile.ZipFile(fname, "r"), zip_flexopen
elif type == "dir": work, flexget = fname, dir_flexopen
elif type == "mapinfo": work, flexget = fname, mapinfo_flexget
else: raise ValueError("Unrecognized type '%s'" % str(type))
data = bunch.Bunch(maps=[], ivars=[], beam=None, gain=None, freq=None)
with flexget(work, "info.txt") as f:
read_info(f, data)
with flexget(work, "beam.txt") as f:
# This supports theformats [l,b,...] and [b]. The beam is assumed to
# start at l=0 and have a step of 1
data.beam = read_beam(f)
if splits is None:
for i in range(0, maxmaps):
try:
with flexget(work, "map%d.fits" % (i + 1)) as f:
mapfile = enmap.read_fits(f, **kwargs)
with flexget(work, "ivar%d.fits" % (i + 1)) as f:
ivarfile = enmap.read_fits(f, **kwargs)
except FileNotFoundError:
break
data["maps"].append(mapfile)
data["ivars"].append(ivarfile)
else:
for i in range(splits):
with flexget(work, "map%d.fits" % (i + 1)) as f:
data["maps"].append(enmap.read_fits(f, **kwargs))
with flexget(work, "ivar%d.fits" % (i + 1)) as f:
data["ivars"].append(enmap.read_fits(f, **kwargs))
if type == "zip": work.close()
return data
def openMap_remote(fname):
'''Opens remotely the map.
Returns the enmap object.
'''
# transferDataTools.checkIfExistsAndCopyIfNot(fname)
fitsFullPath = os.path.join(envVars.remotePath1, fname)
print("Loading map remotely")
print "Loading fits file: %s..." % fitsFullPath
fitsMap = enmap.read_fits(fitsFullPath)
if len(fitsMap.shape) == 3:
fitsMap = fitsMap[0, :, :] # temp map
return fitsMap
def openMap_local(fname):
'''Checks if map exists and dowloads it.
Returns the enmap object.
'''
transferDataTools.checkIfExistsAndCopyIfNot(fname)
fitsFullPath = os.path.join(envVars.localDataPath, fname)
print("Loading map locally...")
print "Loading fits file: %s..." % fitsFullPath
fitsMap = enmap.read_fits(fitsFullPath) # temp map
if len(fitsMap.shape) == 3:
fitsMap = fitsMap[0, :, :]
return fitsMap
def read(self, filename, weight=None, dtype=None, wtype=None):
hdu = fits.open(filename)
self.fheader = hdu[0].header
self.data = hdu[0].data
if self.data.ndim == 3:
# Work-around, for abscal runs on enki planet maps.
from pixell import enmap # You might need pixell to load 3-d maps.
m = enmap.read_fits(filename)[0]
self.fheader = m.wcs.to_header()
self.fheader['NAXIS2'], self.fheader['NAXIS1'] = m.data.shape
self.data = np.array(m, dtype=dtype)
self._load_header()
return
if dtype is not None:
self.data = self.data.astype(dtype)
self._load_header()
if weight in [True, None]:
# Convert True/None to False, a valid weights filename, or an exception
_rw = None
if filename.endswith('.fits'):
_rw = filename[:-4] + 'weight.fits'
if not os.path.exists(_rw):
_rw = None
if _rw is None:
if weight is None:
# Fine, no weights.
weight = False
else:
raise RuntimeError(
"Failed to guess weights filename for %s" % filename)
else:
weight = _rw
if weight != False:
if not os.path.exists(weight):
raise RuntimeError("Weights file %s not found" % weight)
hdu = fits.open(weight)
self.weight = hdu[0].data
if wtype is not None:
self.weight = self.weight.astype(wtype)
else:
self.weight = None
p2d = enmap.enmap(np.load("%stilec_hybrid_covariance_%s_%s.npy" % (kroot,qid,qid)),wcs)
cents,p1d = binner.bin(p2d)
pl.add(cents,p1d,label='sim cov')
kcoadd = enmap.enmap(np.load("%skcoadd_%s.npy" % (droot,qid)),wcs)
p2d = np.real(kcoadd*kcoadd.conj())/w2
cents,p1d = binner.bin(p2d)
pl.add(cents,p1d,label='data power',ls='--')
p2d = enmap.enmap(np.load("%stilec_hybrid_covariance_%s_%s.npy" % (droot,qid,qid)),wcs)
cents,p1d = binner.bin(p2d)
pl.add(cents,p1d,label='data cov',ls='--')
p2d = enmap.read_fits("/scratch/r/rbond/msyriac/dump/smoothed_beamed_%s_%s.fits" % (qid,qid))
cents,p1d = binner.bin(p2d)
pl.add(cents,p1d,label='sim S cov smoothed',ls='--')
pl._ax.set_xlim(10,1000)
pl._ax.set_ylim(1e-3,10)
pl.vline(x=80)
pl.done("lfi_power_comp_%s.png" % qid)
cl_coupled = nmt.compute_coupled_cell(f1, f2)
# Uncoupling pseudo-Cls
# For one spin-0 field and one spin-2 field, NaMaster gives: n_cls=2, [C_TE,C_TB]
# For two spin-2 fields, NaMaster gives: n_cls=4, [C_E1E2,C_E1B2,C_E2B1,C_B1B2]
logger.info('Decoupling cls.')
cl_uncoupled = wsp.decouple_cell(cl_coupled)
cl_out = np.vstack((ells_uncoupled, cl_uncoupled))
elif config['mode'] == 'flat':
from pixell import enmap
spin1, spin2 = config['spins']
logger.info('Spins: spin1 = {}, spin2 = {}.'.format(spin1, spin2))
map1 = enmap.read_fits(config['path2map1'])
logger.info('Read map1 from {}.'.format(config['path2map1']))
map2 = enmap.read_fits(config['path2map2'])
logger.info('Read map2 from {}.'.format(config['path2map2']))
mask1 = enmap.read_fits(config['path2mask1'])
logger.info('Read mask1 from {}.'.format(config['path2mask1']))
mask2 = enmap.read_fits(config['path2mask2'])
logger.info('Read mask2 from {}.'.format(config['path2mask2']))
b = nmt.NmtBinFlat(config['l0_bins'], config['lf_bins'])
# The effective sampling rate for these bandpowers can be obtained calling:
ells_uncoupled = b.get_effective_ells()
if spin1 == 2 and spin2 == 0:
# Define flat sky spin-2 map
def read_data(self):
"""
Reads in the data needed for the generation of noise-only maps and
saves them as class attributes.
These are:
CMB temperature: noise only map (half mission half difference map)
galaxy overdensity: boolean mask of galaxy survey footprint
gamma: structured array with galaxy position and ellipticity catalog
:param:
:return data: dictionary with data needed to generate noise realisations for each
probe
"""
data = {}
for i, probe in enumerate(self.params['probes']):
logger.info('Reading noise data for probe = {}.'.format(probe))
logger.info('Noisemodel = {}.'.format(self.params['noisemodel']))
if self.params['noisemodel'] == 'data':
if probe == 'gamma':
# assert 'shearrot' in self.params, 'Requesting noise model from data but shearrot parameter not provided. Aborting.'
assert 'path2wcsheader' in self.params, 'Requesting noise model from data but path2wcsheader parameter not provided. Aborting.'
assert 'path2shearcat' in self.params, 'Requesting noise model from data but path2shearcat parameter not provided. Aborting.'
data[probe] = {}
hdulist = fits.open(self.params['path2shearcat'])
data[probe]['shearcat'] = hdulist[1].data
logger.info('Read {}.'.format(
self.params['path2shearcat']))
tempmap = enmap.read_fits(self.params['path2wcsheader'])
logger.info('Read {}.'.format(
self.params['path2wcsheader']))
if len(tempmap.shape) == 3:
wcsobj = tempmap[0].wcs
mapshape = tempmap[0].shape
else:
wcsobj = tempmap.wcs
mapshape = tempmap.shape
data[probe]['wcsobj'] = wcsobj
data[probe]['mapshape'] = mapshape
if self.params['posfromshearcat'] == 0:
assert 'path2shearmask' in self.params, 'Requesting randomized galaxy positions for gamma but path2shearmask not provided. Aborting.'
tempmap = enmap.read_fits(
self.params['path2shearmask'])
data[probe]['shearmask'] = tempmap
elif probe == 'deltag':
assert 'Ngal' in self.params, 'Requesting noise model from data but Ngal parameter not provided. Aborting.'
assert 'path2deltagmask' in self.params, 'Requesting noise model from data but path to galaxy mask not provided. Aborting.'
data[probe] = {}
tempmap = enmap.read_fits(self.params['path2deltagmask'])
if len(tempmap.shape) == 3:
tempmask = tempmap[0].astype('bool').astype('int')
else:
tempmask = tempmap.astype('bool').astype('int')
data[probe]['deltagmask'] = tempmask
logger.info('Read {}.'.format(
self.params['path2deltagmask']))
data[probe]['Ngal'] = self.params['Ngal']
else:
raise NotImplementedError(
'Probes other than deltag, gamma not implemented at the moment. Aborting.'
)
else:
assert 'path2noisecls' in self.params, 'Requesting theretical noise model but path2noisecls parameter not provided. Aborting.'
data[probe] = {}
data[probe]['noisecls'] = np.genfromtxt(
self.params['path2noisecls'][i], usecols={1})
logger.info('Read {}.'.format(self.params['path2noisecls']))
return data
for file_path in file_lists:
try:
assert (os.path.exists(file_path))
except AssertionError:
print("Missing File: %s" % file_path)
exit(1)
# build tile if needed
tile_setting = nemo_config['nemo']['tiles'].split('&')
ngrids = [1, 1]
# start automatic tile generation
noise_tiles = {'autoBorderDeg': 0.5}
tiles = []
if 'auto' in tile_setting:
ivar = enmap.read_fits(weight_file)
shape, wcs = ivar.shape, ivar.wcs
default_extent = np.array(nemo_config['nemo']['default_tile_extent']) * utils.degree
ngrids = maps.nrect_grid(ivar, default_extent)
nygrid, nxgrid = ngrids
# some simplification
if nxgrid * nygrid <= 8:
ngrids = [1, 1]
nxgrid = nygrid = 1
grid_pix = maps.rect_grid_pix(shape, ngrids)
valid_grid = maps.threshold_grids(ivar, grid_pix)
# put this in the setting!
threshold_factor = 1.3 if patch not in ['cmb', 'boss'] else 0.8
def getActpolNoiseSim(noiseSeed,
psa,
noisePsdDir,
freqs,
verbose=True,
useCovSqrt=True,
killFactor=30.,
fillValue=0.,
noiseDiagsOnly=False,
splitWanted=None):
#return array of T, Q, U
#to-do: these are currently using numpy.FFT and are slow; switch to FFTW if installed.
#Could also have an alternative version of this using enlib tools.
if useCovSqrt:
#in this case it was the CovSqrt's that were saved. This is based on Mat's code in orphics.
if verbose:
print(
'getActpolNoiseSim(): getting weight maps; assuming I for all')
iqu = 'I' #FOR NOW
# stackOfMaskMaps = [enmap.read_map(noisePsdDir + 'totalWeightMap' \
# + iqu + '_' + psa + '_' + freq + '_fromenlib.fits') \
# for freq in freqs ]
if splitWanted is None:
stackOfMaskMaps = [enmap.read_map(noisePsdDir + 'totalWeightMap'\
+ iqu + '_' + psa + '_' + freq + '_fromenlib.fits') \
for freq in freqs ]
else:
stackOfMaskMaps = [enmap.read_map(noisePsdDir + 'weightMap_split' + str(splitWanted) \
+ iqu + '_' + psa + '_' + freq + '_fromenlib.fits') \
for freq in freqs ]
thisWcs = stackOfMaskMaps[0].wcs
maskMaps = enmap.enmap(np.stack(stackOfMaskMaps), thisWcs)
#first one is for IXI, QxQ, UXU only
print("loading")
if False:
print('loading ' + noisePsdDir +
'/bigMatrixNoisePsdsCovSqrtDiags_' + psa + '.fits HACKING')
covsqrt = enmap.read_fits(noisePsdDir +
'/bigMatrixNoisePsdsCovSqrtDiags_' +
psa + '.fits')
if False:
print('loading ' + noisePsdDir + '/bigMatrixNoisePsdsCovSqrt_' +
psa + '.fits')
covsqrt = enmap.read_fits(noisePsdDir +
'/bigMatrixNoisePsdsCovSqrt_' + psa +
'.fits')
if noiseDiagsOnly:
print('loading ' + noisePsdDir +
'/noisePsds_flattened_covSqrtDiags_' + psa + '.fits')
covsqrt = enmap.read_fits(noisePsdDir +
'/noisePsds_flattened_covSqrtDiags_' +
psa + '.fits')
elif True:
print('loading ' + noisePsdDir + '/noisePsds_flattened_covSqrt_' +
psa + '.fits')
covsqrt = enmap.read_fits(noisePsdDir +
'/noisePsds_flattened_covSqrt_' + psa +
'.fits')
print("loading done")
if verbose:
print('getActpolNoiseSim(): running map_mul to make random phases')
#get the right normalization
covsqrt *= np.sqrt(
np.prod(covsqrt.shape[-2:]) /
enmap.area(covsqrt.shape[-2:], thisWcs))
np.random.seed(noiseSeed)
print("randmap")
rmap = enmap.rand_gauss_harm(
(covsqrt.shape[0], covsqrt.shape[-2:][0], covsqrt.shape[-2:][1]),
thisWcs)
print("randmap done")
print("map_mul")
kmap = enmap.map_mul(covsqrt, rmap)
print("map_mul done")
#old way:
# kmapReshape = kmap.reshape((4, kmap.shape[-2:][0], kmap.shape[-2:][1]))
# outMaps = enmap.ifft(kmapReshape).real
# kmap /= sqrt(mask)
if verbose:
print('getActpolNoiseSim(): inverse transforming')
print('you are transforming %d maps' % kmap.shape[0])
spin = np.repeat([0], kmap.shape[0])
print("fft")
outMaps = enmap.harm2map(kmap, iau=False, spin=spin)
print("fft done")
#now reshape to have shape [nfreqs, 3, Ny, Nx]
#The "order = 'F' (row vs. column ordering) is due to the ordering that is done
#in makeNoisePsds.py for the dichroic arrays,
#namely I90, Q90, U90, I150, Q150, U150.
outMaps = outMaps.reshape(len(freqs),
outMaps.shape[0] / len(freqs),
outMaps.shape[-2],
outMaps.shape[-1],
order='F')
for fi, freq in enumerate(freqs):
#Note each frequency has its own maskmap, so this loop is important
thisMaskMap = np.squeeze(maskMaps[fi])
outMaps[fi, :, :, :] /= np.sqrt(thisMaskMap)
#Loop over T,Q,U. Couldn't think of clever way to vectorize this part..
for z in range(outMaps.shape[-3]):
outMaps[fi, z][thisMaskMap < thisMaskMap[np.where(np.isfinite(thisMaskMap))].max() / killFactor] \
= fillValue
if verbose:
print('getActpolNoiseSim(): done ')
return outMaps
else:
raise ValueError('older ways of getting the noise maps are deprecated')
config = yaml.load(open(args.path2config))
logger.info('Read config from {}.'.format(args.path2config))
nrealiz = config['simparams']['nrealiz']
if config['mode'] == 'curved':
from MockSurveyCurved_parallel import MockSurveyParallel
import healpy as hp
mask = hp.read_map(config['path2mask'])
elif config['mode'] == 'flat':
from MockSurvey_parallel import MockSurveyParallel
from pixell import enmap
mask = enmap.read_fits(config['path2mask'])
else:
raise NotImplementedError()
# Here assuming for simplicity that masks are the same
masks = [mask, mask, mask, mask, mask, mask]
if 'l0_bins' in config['simparams']:
config['simparams']['l0_bins'] = np.array(
config['simparams']['l0_bins'])
if 'l1_bins' in config['simparams']:
config['simparams']['l1_bins'] = np.array(
config['simparams']['l1_bins'])
if 'spins' in config['simparams']:
config['simparams']['spins'] = np.array(config['simparams']['spins'])
import numpy as np
import os, sys
# directory setup
postfix = 'simple_test_output'
output_dir = os.path.join('./', postfix)
output_path = lambda x: os.path.join(output_dir, x)
resource_dir = pitas.config.get_resource_dir()
resource_path = lambda x: os.path.join(resource_dir, x)
pitas.pitas_io.create_dir(output_dir)
# miscs
lmax = 3000
bin_edges = pitas.util.get_default_bin_edges(lmax)
taper = enmap.read_fits(resource_path('test_taper.fits'))
# initialize pitas (if it is a first time, it will take few minutes to compute mode coupling)
overwrite = True
transfer = None # no transfer function here
mcm_identifier = "simple_test" # we can tag mode coupling matrix with a string. If PITAS finds the precomputed mcm with the same tag, it automatically reloads. Set overwrite=True if you don't want this.
pitas_lib = pitas.power.PITAS(mcm_identifier,
taper,
taper,
bin_edges,
lmax,
transfer,
overwrite=overwrite,
output_dir=None)
binner = pitas_lib.binner
lbin = pitas_lib.bin_center