def extend_iqu_map(source_dict=None, target_dict=None, map_dict=None):
if source_dict != None:
imap = algebra.make_vect(algebra.load(source_dict['imap']))
qmap = algebra.make_vect(algebra.load(source_dict['qmap']))
umap = algebra.make_vect(algebra.load(source_dict['umap']))
if source_dict.has_key('imap_weight'):
imap_weight = algebra.make_vect(algebra.load(source_dict['imap_weight']))
qmap_weight = algebra.make_vect(algebra.load(source_dict['qmap_weight']))
umap_weight = algebra.make_vect(algebra.load(source_dict['umap_weight']))
elif source_dict.has_key('imap_inv'):
imap_weight, info = find_weight_re_diagnal(source_dict['imap_inv'])
qmap_weight, info = find_weight_re_diagnal(source_dict['qmap_inv'])
umap_weight, info = find_weight_re_diagnal(source_dict['umap_inv'])
else:
print 'Warning: no weight'
imap_weight = algebra.ones_like(imap)
qmap_weight = algebra.ones_like(imap)
umap_weight = algebra.ones_like(imap)
elif map_dict != None:
imap = map_dict['imap']
qmap = map_dict['qmap']
umap = map_dict['umap']
if 'imap_weight' in map_dict.keys():
imap_weight = map_dict['imap_weight']
qmap_weight = map_dict['qmap_weight']
umap_weight = map_dict['umap_weight']
else:
print 'Warning: no weight'
imap_weight = algebra.ones_like(imap)
qmap_weight = algebra.ones_like(imap)
umap_weight = algebra.ones_like(imap)
else:
print "Error: Can not find I Q U maps"
exit()
iqu = algebra.info_array(imap.tolist() + qmap.tolist() + umap.tolist())
iqu = algebra.make_vect(iqu)
iqu.info = imap.info
iqu.copy_axis_info(imap)
iqu_weight = algebra.info_array(imap_weight.tolist() +
qmap_weight.tolist() +
umap_weight.tolist())
iqu_weight = algebra.make_vect(iqu_weight)
iqu_weight.info = imap_weight.info
iqu_weight.copy_axis_info(imap_weight)
if target_dict != None:
algebra.save(target_dict['map'], iqu)
algebra.save(target_dict['weight'], iqu_weight)
else:
map_dict = {}
map_dict['map'] = iqu
map_dict['weight'] = iqu_weight
return map_dict
def getmap(imap_fname, nmap_fname, mmap_fname=None, half=None):
"""
get the matrix of intensity map and noise map
"""
#in_root = params['input_root']
imap = algebra.load(imap_fname)
imap = algebra.make_vect(imap)
if half!=None:
imap = getmap_halfz(imap, half)
#print "--The neam value for imap is:",imap.flatten().mean(),"--"
#imap = imap - imap.flatten().mean()
if imap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
try:
nmap = algebra.load(nmap_fname)
nmap = algebra.make_vect(nmap)
if half!=None:
nmap = getmap_halfz(nmap, half)
bad = nmap<1.e-5*nmap.flatten().max()
nmap[bad] = 0.
non0 = nmap.nonzero()
#imap[non0] = imap[non0]/nmap[non0]
except IOError:
print 'NO Noise File :: Set Noise to One'
nmap = algebra.info_array(sp.ones(imap.shape))
nmap.axes = imap.axes
nmap = algebra.make_vect(nmap)
nmap.info = imap.info
if nmap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
if mmap_fname != None:
try:
mmap = algebra.load(mmap_fname)
mmap = algebra.make_vect(mmap)
if half!=None:
mmap = getmap_halfz(mmap, half)
except IOError:
print 'NO Mock File :: Make it!'
mmap = algebra.info_array(
2.*np.random.rand(imap.shape[0],imap.shape[1], imap.shape[2])-0.5)
mmap.axes = imap.axes
mmap = algebra.make_vect(mmap)
return imap, nmap, mmap
else:
return imap, nmap
def mktmp(rgn_i,rgn_j,rgn_k,srgn_i1,srgn_i2,srgn_j1,srgn_j2,srgn_k1,srgn_k2,outfilename):
"""Write to disk a file representing an empty matrix of given dimensions. Also write an identically
shaped array of booleans, which are true if the index points to the subregion.
rgn_i/j/k : the dimensions of the full region to be simulated
srgn_i/j/k : the dimensions of the deep integration subregion
outfilename: the name of the file to be created
"""
regiontype = np.zeros((rgn_i,rgn_j,rgn_k), bool)
array = np.zeros((rgn_i,rgn_j,rgn_k))
for i in range(0,rgn_i):
for j in range(0,rgn_j):
for k in range(0,rgn_k):
if (i>=(srgn_i1-1) and i<=(srgn_i2-1)):
if (j>=(srgn_j1-1) and j<=(srgn_j2-1)):
if (k>=(srgn_k1-1) and k<=(srgn_k2-1)):
regiontype[i,j,k]=True
else:
regiontype[i,j,k]=False
region=algebra.info_array(array)
regiontypename = 'bool' + outfilename
np.save(regiontypename, regiontype)
algebra.save(outfilename,region)
print "done"
template_map = algebra.make_vect(algebra.load(outfilename))
def process_map(self, imap_fname, nmap_fname, mock_fname=None):
params = self.params
out_root = params['output_root']
in_root = params['input_root']
imap = algebra.load(in_root + imap_fname)
imap = algebra.make_vect(imap)
print imap.flatten().mean()
imap = imap - imap.flatten().mean()
if imap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
try:
nmap = algebra.load(in_root + nmap_fname)
nmap = algebra.make_vect(nmap)
bad = nmap<1.e-5*nmap.flatten().max()
nmap[bad] = 0.
non0 = nmap.nonzero()
#imap[non0] = imap[non0]/nmap[non0]
except IOError:
print 'NO Noise File :: Set Noise to One'
nmap = algebra.info_array(sp.ones(imap.shape))
nmap.axes = imap.axes
nmap = algebra.make_vect(nmap)
nmap.info = imap.info
if nmap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
if mock_fname != None:
mmap = algebra.info_array(
2.*np.random.rand(imap.shape[0],imap.shape[1], imap.shape[2])-0.5)
mmap.axes = imap.axes
mmap = algebra.make_vect(mmap)
box, nbox, mbox = self.fill(imap, nmap, mmap)
pkrm_nfname = out_root + 'fftbox_' + mock_fname
algebra.save(pkrm_nfname, mbox)
else:
box, nbox = self.fill(imap, nmap)
pkrm_fname = out_root + 'fftbox_' + imap_fname
algebra.save(pkrm_fname, box)
pkrm_nfname = out_root + 'fftbox_' + nmap_fname
algebra.save(pkrm_nfname, nbox)
def process_map(self, imap_fname, nmap_fname, ii, mock_fname=None):
params = self.params
sigma = params['sigma']
mu = params['mu']
out_root = params['output_root']
in_root = params['input_root']
imap = algebra.load(in_root + imap_fname)
imap = algebra.make_vect(imap)
#print imap.flatten().mean()
imap = imap - imap.flatten().mean()
if imap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
print ' :: Set Noise to Gaussian'
np.random.seed()
nmap = algebra.info_array(
sigma*np.random.randn(imap.shape[0],imap.shape[1], imap.shape[2])+mu)
nmap.axes = imap.axes
nmap = algebra.make_vect(nmap)
nmap.info = imap.info
if nmap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
## add noise to map ##
imap = imap + nmap
non0 = nmap.nonzero()
nmap[non0] = (1./sigma)**2
#if mock_fname != None:
# mmap = algebra.info_array(
# 2.*np.random.randn(imap.shape[0],imap.shape[1], imap.shape[2])-0.5)
# mmap.axes = imap.axes
# mmap = algebra.make_vect(mmap)
# box, nbox, mbox = self.fill(imap, nmap, mmap)
# pkrm_nfname = out_root + 'fftbox_' + mock_fname
# algebra.save(pkrm_nfname, mbox)
#else:
# box, nbox = self.fill(imap, nmap)
hr = params['hr']
mid = params['mid']
last = params['last']
pol_str = params['polarizations'][0]
end = pol_str
if len(last)!=0:
end = end + last[ii]
end = end + '_' + str(ii)
imap_fname = hr[ii] + mid[0] + end + '.npy'
nmap_fname = hr[ii] + mid[1] + end + '.npy'
pkrm_fname = out_root + imap_fname
algebra.save(pkrm_fname, imap)
pkrm_nfname = out_root + nmap_fname
algebra.save(pkrm_nfname, nmap)
def fgrm(self, imap, fname):
freq = imap.get_axis('freq')/1.e6
imap = imap.swapaxes(0,2)
fg = algebra.info_array(sp.zeros(imap.shape))
fg.axes = ('ra','dec','freq')
fg = algebra.make_vect(fg)
if self.plot==True:
plt.figure(figsize=(8,12))
#plot the map without foreground removeing
plt.subplot(311)
#plt.title('Map without foreground remove')
plt.xlabel('Frequece (MHz)')
plt.ylabel('$\Delta$ T(Kelvin) With Foreground')
for i in range(0,imap.shape[0]):
#for j in range(1, 2):
for j in range(0, imap.shape[1]):
plt.plot(freq, imap[i][j])
for i in range(0,imap.shape[0]):
#for j in range(1, 2):
for j in range(0, imap.shape[1]):
y = imap[i][j]
y = np.asmatrix(y).T
X = np.ones(shape=(3,imap.shape[2]))
X[1] = np.log10(freq)
X[2] = np.square(np.log10(freq))
XT = np.asmatrix(X)
X = XT.T
a = ((XT*X).I*XT)*y
yy = np.asarray((X*a).T)
y = np.asarray((y-X*a).T)
imap[i][j] = y
fg[i][j] = yy
if self.plot==True:
plt.subplot(312)
#plt.title('Foreground')
plt.xlabel('Frequece (MHz)')
plt.ylabel('$\Delta$ T(Kelvin) Foreground')
for i in range(0,imap.shape[0]):
#for j in range(1, 2):
for j in range(0, imap.shape[1]):
plt.plot(freq, fg[i][j])
plt.subplot(313)
#plt.title('Map with foreground remove')
plt.xlabel('Frequece (MHz)')
plt.ylabel('$\Delta$ T(Kelvin) Without Foreground')
for i in range(0,imap.shape[0]):
#for j in range(1, 2):
for j in range(0, imap.shape[1]):
plt.plot(freq, imap[i][j])
plt.savefig(fname, format='png')
def execute(self, nprocesses=1):
params = self.params
# Make parent directory and write parameter file.
kiyopy.utils.mkparents(params['output_root'])
parse_ini.write_params(params, params['output_root']+'params.ini',prefix='pk_')
in_root = params['input_root']
out_root = params['output_root']
mid = params['mid']
all_out_fname_list = []
all_in_fname_list = []
#### Process ####
pol_str = params['polarizations'][0]
hr_str = params['hr'][0]
imap_fname = in_root + hr_str + mid + pol_str + '.npy'
imap = algebra.load(imap_fname)
imap = algebra.make_vect(imap)
if imap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
nmap_fname = in_root + hr_str + 'noise_inv_diag_' + pol_str + '.npy'
nmap = algebra.load(nmap_fname)
nmap = algebra.make_vect(nmap)
#noise normal
normal = (nmap**2).sum()
#Using map in different day
hr_str = params['hr'][1]
imap_fname2 = in_root + hr_str + mid + pol_str + '.npy'
imap2 = algebra.load(imap_fname2)
imap2 = algebra.make_vect(imap2)
if imap2.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
nmap_fname = in_root + hr_str + 'noise_inv_diag_' + pol_str + '.npy'
nmap2 = algebra.load(nmap_fname)
nmap2 = algebra.make_vect(nmap2)
#noise normal
normal2 = (nmap2**2).sum()
normal = sqrt(normal)*sqrt(normal2)
mapshape = np.array(imap.shape)
#print imap.shape
r = self.discrete(self.fq2r(imap.get_axis('freq')))
ra = self.discrete(imap.get_axis('ra'))*deg2rad
de = self.discrete(imap.get_axis('dec'))*deg2rad
ra0= ra[int(ra.shape[0]/2)]
ra = ra - ra0
dr = r.ptp()/r.shape[0]
dra= ra.ptp()/ra.shape[0]
dde= de.ptp()/de.shape[0]
disc_n = params['discrete']
#imap = imap.swapaxes(1,2) # change the ra and dec
#print imap.shape
mapinf = [dr, dra, dde, disc_n]
mapinf = np.array(mapinf)
#print mapinf
#print r
#print ra
#print de
box = algebra.info_array(sp.zeros(params['boxshape']))
box.axes = ('x','y','z')
box = algebra.make_vect(box)
boxshape = np.array(box.shape)
box2 = algebra.info_array(sp.zeros(params['boxshape']))
box2.axes = ('x','y','z')
box2 = algebra.make_vect(box2)
xrange0 = params['Xrange'][0]
yrange0 = params['Yrange'][0]
zrange0 = params['Zrange'][0]
boxunit = params['boxunit']
shapex = params['boxshape'][2]
shapera = ra.shape[0]
V = params['boxunit']**3
boxinf = [xrange0, yrange0, zrange0, boxunit]
boxinf = np.array(boxinf)
print "Filling the BOX"
MakePower.Filling(imap, imap2, box, box2, r, ra, de, boxinf, mapinf)
print "FFTing "
fftbox = fftn(box)
fftbox = fftbox.real**2 + fftbox.imag**2
fftbox2= fftn(box2)
fftbox2 = fftbox2.real**2 + fftbox2.imag**2
fftbox = fftbox2
#fftbox = fftbox.__pow__(0.5)*fftbox2.__pow__(0.5)
PK = np.zeros(40)
k = np.zeros(40)
PK2 = np.zeros(shape=(10, 10))
k2 = np.zeros(shape=(2, 10))
MakePower.Make(fftbox, PK, k, PK2, k2)
kunit = 2.*pi/(boxshape[0]*boxunit)
k = k*kunit
k2 = k2*kunit
PK = PK*V*params['boxshape'][0]**3/normal
PK2 = PK2*V*params['boxshape'][0]**3/normal
sp.save(out_root+'PK', PK)
sp.save(out_root+'PK2', PK2)
non0 = PK.nonzero()
if self.plot==True:
plt.figure(figsize=(8,8))
#print k
#print PK
plt.subplot('211')
plt.scatter(k.take(non0), PK.take(non0))
plt.loglog()
plt.ylim(ymin=1.e1)
plt.xlim(xmin=k.min())
plt.title('Power Spectrum')
plt.xlabel('$k$')
plt.ylabel('$P(k) (Kelvin^{2}(h^{-1}Mpc)^3)$')
PK = PK*V*params['boxshape'][0]**3/1.2e12*k*k*k/2./pi/pi
#print PK
plt.subplot('212')
plt.scatter(k.take(non0), PK.take(non0))
plt.loglog()
plt.ylim(ymin=1.e-9)
plt.xlim(xmin=k.min())
plt.xlabel('$k (h Mpc^{-1})$')
plt.ylabel('$\Delta^2 (Kelvin^{2})$')
#plt.show()
plt.savefig(out_root+'power.eps', format='eps')
PK2 = np.log10(PK2)
plt.figure(figsize=(6,6))
extent = (k2[0][0], k2[0][-1], k2[1][0], k2[1][-1])
plt.imshow(PK2, origin='lower', extent = extent, interpolation='nearest')
plt.xlabel('$k vertical (h Mpc^{-1})$')
plt.ylabel('$k parallel (h Mpc^{-1})$')
cb = plt.colorbar()
cb.set_label('$lg(P^{2D}_{k_pk_v}) (Kelvin^2(h^{-1}Mpc)^3)$')
plt.loglog()
plt.savefig(out_root+'power2.eps', format='eps')
#plt.show()
print 'Finished @_@ '
return PK
def fill(params, imap, nmap, mmap=None):
"""
Function that used to fill the fftbox with the intensity map
params : the params dict for each module
imap : the direction to the intensity maps
nmap : the direction to the noise maps
mmap : the direction to the mock maps
It will return the fft box and nbox.
box is for the intensity maps, while
nbox is for the noise intensity maps.
If the mmap!=None, it also return mbox
which for the mock maps
"""
#params = self.params
mapshape = np.array(imap.shape)
r = fq2r(imap.get_axis('freq'))
ra = imap.get_axis('ra')*deg2rad
de = imap.get_axis('dec')*deg2rad
ra0= ra[int(ra.shape[0]/2)]
ra = ra - ra0
dra= ra.ptp()/ra.shape[0]
dde= de.ptp()/de.shape[0]
#ra_far = 0
#de_far = 0
#if ra.min()*ra.max()>0:
# if fabs(ra.min())<fabs(ra.max()):
# ra_far = ra.min()
# else:
# ra_far = ra.max()
#if de.min()*de.max()>0:
# if fabs(de.min())<fabs(de.max()):
# de_far = de.min()
# else:
# de_far = de.max()
#point = []
#for i in range(3):
# point.append([xyzv(ra.min(), de.min(), r.min())[i],
# xyzv(ra.max(), de.min(), r.min())[i],
# xyzv(ra.min(), de.max(), r.min())[i],
# xyzv(ra.max(), de.max(), r.min())[i],
# xyzv(ra.min(), de.min(), r.max())[i],
# xyzv(ra.max(), de.min(), r.max())[i],
# xyzv(ra.min(), de.max(), r.max())[i],
# xyzv(ra.max(), de.max(), r.max())[i],
# xyzv(ra_far, de_far, r.max())[i],
# ])
# point[i].sort()
#print point
#print params['boxshape']
#print (point[0][-1]-point[0][0])/params['boxshape'][0]
#print (point[1][-1]-point[1][0])/params['boxshape'][1]
#print (point[2][-1]-point[2][0])/params['boxshape'][2]
#return 0
#print r.min(), r.max()
#print xyz(ra.min(), de.min(), r.min())
#print xyz(ra.max(), de.min(), r.min())
#print xyz(ra.min(), de.max(), r.min())
#print xyz(ra.max(), de.max(), r.min())
#print xyz(ra.min(), de.min(), r.max())
#print xyz(ra.max(), de.min(), r.max())
#print xyz(ra.min(), de.max(), r.max())
#print xyz(ra.max(), de.max(), r.max())
###return 0
mapinf = [ra.min(), dra, de.min(), dde]
mapinf = np.array(mapinf)
box = algebra.info_array(sp.zeros(params['boxshape']))
box.axes = ('x','y','z')
box = algebra.make_vect(box)
box_xrange = params['Xrange']
box_yrange = params['Yrange']
box_zrange = params['Zrange']
box_unit = params['boxunit']
box_disc = params['discrete']
print box_xrange, box_yrange, box_zrange, box_unit,
box_x = np.arange(box_xrange[0], box_xrange[1], box_unit/box_disc)
box_y = np.arange(box_yrange[0], box_yrange[1], box_unit/box_disc)
box_z = np.arange(box_zrange[0], box_zrange[1], box_unit/box_disc)
#print box_x.shape
#print box_y.shape
#print box_z.shape
boxshape = np.array(box.shape)*int(box_disc)
boxinf0 = [0, 0, 0]
boxinf0 = np.array(boxinf0)
boxinf1 = [boxshape[0], boxshape[1], boxshape[2]]
boxinf1 = np.array(boxinf1)
print "MapPrepare: Filling the FFT BOX"
MakePower.Filling(
imap, r, mapinf, box, boxinf0, boxinf1, box_x, box_y, box_z)
nbox = algebra.info_array(sp.zeros(params['boxshape']))
nbox.axes = ('x','y','z')
nbox = algebra.make_vect(nbox)
#nbox = algebra.info_array(sp.ones(params['boxshape']))
#nbox.axes = ('x','y','z')
#nbox = algebra.make_vect(nbox)
#print boxinf1
MakePower.Filling(
nmap, r, mapinf, nbox, boxinf0, boxinf1, box_x, box_y, box_z)
if mmap != None:
mbox = algebra.info_array(sp.zeros(params['boxshape']))
mbox.axes = ('x','y','z')
mbox = algebra.make_vect(mbox)
MakePower.Filling(
mmap, r, mapinf, mbox, boxinf0, boxinf1, box_x, box_y, box_z)
return box, nbox, mbox
else:
return box, nbox
def execute(self, nprocesses=1):
"""Worker funciton."""
params = self.params
# Make parent directory and write parameter file.
kiyopy.utils.mkparents(params['output_root'])
parse_ini.write_params(params,
params['output_root'] + 'params.ini',
prefix=prefix)
save_noise_diag = params['save_noise_diag']
in_root = params['input_root']
all_out_fname_list = []
all_in_fname_list = []
# Figure out what the band names are.
bands = params['bands']
if not bands:
map_files = glob.glob(in_root + 'dirty_map_' + pol_str + "_*.npy")
bands = []
root_len = len(in_root + 'dirty_map_')
for file_name in map_files:
bands.append(file_name[root_len:-4])
# Loop over files to process.
for pol_str in params['polarizations']:
for band in bands:
if band == -1:
band_str = ''
else:
band_str = "_" + repr(band)
dmap_fname = (in_root + 'dirty_map_' + pol_str + band_str +
'.npy')
all_in_fname_list.append(
kiyopy.utils.abbreviate_file_path(dmap_fname))
# Load the dirty map and the noise matrix.
dirty_map = algebra.load(dmap_fname)
dirty_map = algebra.make_vect(dirty_map)
if dirty_map.axes != ('freq', 'ra', 'dec'):
msg = ("Expeced dirty map to have axes ('freq',"
"'ra', 'dec'), but it has axes: " +
str(dirty_map.axes))
raise ce.DataError(msg)
shape = dirty_map.shape
# Initialize the clean map.
clean_map = algebra.info_array(sp.zeros(dirty_map.shape))
clean_map.info = dict(dirty_map.info)
clean_map = algebra.make_vect(clean_map)
# If needed, initialize a map for the noise diagonal.
if save_noise_diag:
noise_diag = algebra.zeros_like(clean_map)
if params["from_eig"]:
# Solving from eigen decomposition of the noise instead of
# the noise itself.
# Load in the decomposition.
evects_fname = (in_root + 'noise_evects_' + pol_str +
+band_str + '.npy')
if self.feedback > 1:
print "Using dirty map: " + dmap_fname
print "Using eigenvectors: " + evects_fname
evects = algebra.open_memmap(evects_fname, 'r')
evects = algebra.make_mat(evects)
evals_inv_fname = (in_root + 'noise_evalsinv_' + pol_str +
"_" + repr(band) + '.npy')
evals_inv = algebra.load(evals_inv_fname)
evals_inv = algebra.make_mat(evals_inv)
# Solve for the map.
if params["save_noise_diag"]:
clean_map, noise_diag = solve_from_eig(
evals_inv, evects, dirty_map, True, self.feedback)
else:
clean_map = solve_from_eig(evals_inv, evects,
dirty_map, False,
self.feedback)
# Delete the eigen vectors to recover memory.
del evects
else:
# Solving from the noise.
noise_fname = (in_root + 'noise_inv_' + pol_str +
band_str + '.npy')
if self.feedback > 1:
print "Using dirty map: " + dmap_fname
print "Using noise inverse: " + noise_fname
all_in_fname_list.append(
kiyopy.utils.abbreviate_file_path(noise_fname))
noise_inv = algebra.open_memmap(noise_fname, 'r')
noise_inv = algebra.make_mat(noise_inv)
# Two cases for the noise. If its the same shape as the map
# then the noise is diagonal. Otherwise, it should be
# block diagonal in frequency.
if noise_inv.ndim == 3:
if noise_inv.axes != ('freq', 'ra', 'dec'):
msg = ("Expeced noise matrix to have axes "
"('freq', 'ra', 'dec'), but it has: " +
str(noise_inv.axes))
raise ce.DataError(msg)
# Noise inverse can fit in memory, so copy it.
noise_inv_memory = sp.array(noise_inv, copy=True)
# Find the non-singular (covered) pixels.
max_information = noise_inv_memory.max()
good_data = noise_inv_memory < 1.0e-10 * max_information
# Make the clean map.
clean_map[good_data] = (dirty_map[good_data] /
noise_inv_memory[good_data])
if save_noise_diag:
noise_diag[good_data] = \
1/noise_inv_memory[good_data]
elif noise_inv.ndim == 5:
if noise_inv.axes != ('freq', 'ra', 'dec', 'ra',
'dec'):
msg = ("Expeced noise matrix to have axes "
"('freq', 'ra', 'dec', 'ra', 'dec'), "
"but it has: " + str(noise_inv.axes))
raise ce.DataError(msg)
# Arrange the dirty map as a vector.
dirty_map_vect = sp.array(dirty_map) # A view.
dirty_map_vect.shape = (shape[0], shape[1] * shape[2])
frequencies = dirty_map.get_axis('freq') / 1.0e6
# Allowcate memory only once.
noise_inv_freq = sp.empty(
(shape[1], shape[2], shape[1], shape[2]),
dtype=float)
if self.feedback > 1:
print "Inverting noise matrix."
# Block diagonal in frequency so loop over frequencies.
for ii in xrange(dirty_map.shape[0]):
if self.feedback > 1:
print "Frequency: ", "%5.1f" % (
frequencies[ii]),
if self.feedback > 2:
print ", start mmap read:",
sys.stdout.flush()
noise_inv_freq[...] = noise_inv[ii, ...]
if self.feedback > 2:
print "done, start eig:",
sys.stdout.flush()
noise_inv_freq.shape = (shape[1] * shape[2],
shape[1] * shape[2])
# Solve the map making equation by diagonalization.
noise_inv_diag, Rot = sp.linalg.eigh(
noise_inv_freq, overwrite_a=True)
if self.feedback > 2:
print "done",
map_rotated = sp.dot(Rot.T, dirty_map_vect[ii])
# Zero out infinite noise modes.
bad_modes = (noise_inv_diag <
1.0e-5 * noise_inv_diag.max())
if self.feedback > 1:
print ", discarded: ",
print "%4.1f" % (100.0 * sp.sum(bad_modes) /
bad_modes.size),
print "% of modes",
if self.feedback > 2:
print ", start rotations:",
sys.stdout.flush()
map_rotated[bad_modes] = 0.
noise_inv_diag[bad_modes] = 1.0
# Solve for the clean map and rotate back.
map_rotated /= noise_inv_diag
map = sp.dot(Rot, map_rotated)
if self.feedback > 2:
print "done",
sys.stdout.flush()
# Fill the clean array.
map.shape = (shape[1], shape[2])
clean_map[ii, ...] = map
if save_noise_diag:
# Using C = R Lambda R^T
# where Lambda = diag(1/noise_inv_diag).
temp_noise_diag = 1 / noise_inv_diag
temp_noise_diag[bad_modes] = 0
# Multiply R by the diagonal eigenvalue matrix.
# Broadcasting does equivalent of mult by diag
# matrix.
temp_mat = Rot * temp_noise_diag
# Multiply by R^T, but only calculate the
# diagonal elements.
for jj in range(shape[1] * shape[2]):
temp_noise_diag[jj] = sp.dot(
temp_mat[jj, :], Rot[jj, :])
temp_noise_diag.shape = (shape[1], shape[2])
noise_diag[ii, ...] = temp_noise_diag
# Return workspace memory to origional shape.
noise_inv_freq.shape = (shape[1], shape[2],
shape[1], shape[2])
if self.feedback > 1:
print ""
sys.stdout.flush()
elif noise_inv.ndim == 6:
if save_noise_diag:
# OLD WAY.
#clean_map, noise_diag, chol = solve(noise_inv,
# dirty_map, True, feedback=self.feedback)
# NEW WAY.
clean_map, noise_diag, noise_inv_diag, chol = \
solve(noise_fname, noise_inv, dirty_map,
True, feedback=self.feedback)
else:
# OLD WAY.
#clean_map, chol = solve(noise_inv, dirty_map,
# False, feedback=self.feedback)
# NEW WAY.
clean_map, noise_inv_diag, chol = \
solve(noise_fname, noise_inv, dirty_map,
False, feedback=self.feedback)
if params['save_cholesky']:
chol_fname = (params['output_root'] + 'chol_' +
pol_str + band_str + '.npy')
sp.save(chol_fname, chol)
if params['save_noise_inv_diag']:
noise_inv_diag_fname = (params['output_root'] +
'noise_inv_diag_' +
pol_str + band_str +
'.npy')
algebra.save(noise_inv_diag_fname, noise_inv_diag)
# Delete the cholesky to recover memory.
del chol
else:
raise ce.DataError("Noise matrix has bad shape.")
# In all cases delete the noise object to recover memeory.
del noise_inv
# Write the clean map to file.
out_fname = (params['output_root'] + 'clean_map_' + pol_str +
band_str + '.npy')
if self.feedback > 1:
print "Writing clean map to: " + out_fname
algebra.save(out_fname, clean_map)
all_out_fname_list.append(
kiyopy.utils.abbreviate_file_path(out_fname))
if save_noise_diag:
noise_diag_fname = (params['output_root'] + 'noise_diag_' +
pol_str + band_str + '.npy')
algebra.save(noise_diag_fname, noise_diag)
all_out_fname_list.append(
kiyopy.utils.abbreviate_file_path(noise_diag_fname))
# Check the clean map for faileur.
if not sp.alltrue(sp.isfinite(clean_map)):
n_bad = sp.sum(sp.logical_not(sp.isfinite(clean_map)))
msg = ("Non finite entries found in clean map. Solve"
" failed. %d out of %d entries bad" %
(n_bad, clean_map.size))
raise RuntimeError(msg)
def GetRadioFFTbox(self):
params = self.params
resultf = params["hr"][0]
if len(params["last"]) != 0:
resultf = resultf + params["last"][0]
resultf = resultf + "-" + params["hr"][1]
if len(params["last"]) != 0:
resultf = resultf + params["last"][1]
# Make parent directory and write parameter file.
kiyopy.utils.mkparents(params["output_root"])
parse_ini.write_params(params, params["output_root"] + "params.ini", prefix="pk_")
in_root = params["input_root"]
out_root = params["output_root"]
mid = params["mid"]
all_out_fname_list = []
all_in_fname_list = []
OmegaHI = params["OmegaHI"]
Omegam = params["Omegam"]
OmegaL = params["OmegaL"]
fkpp = params["FKPpk"]
FKPweight = params["FKPweight"]
#### Process ####
pol_str = params["polarizations"][0]
hr_str = params["hr"][0]
end = pol_str
if len(params["last"]) != 0:
end = end + params["last"][0]
imap_fname = in_root + hr_str + mid[0] + end + ".npy"
imap = algebra.load(imap_fname)
imap = algebra.make_vect(imap)
if imap.axes != ("freq", "ra", "dec"):
raise ce.DataError("AXES ERROR!")
box = np.load(imap_fname)
nmap_fname = in_root + hr_str + mid[1] + end + ".npy"
try:
nmap = algebra.load(nmap_fname)
nmap = algebra.make_vect(nmap)
bad = nmap < 1.0e-5 * nmap.flatten().max()
nmap[bad] = 0.0
non0 = nmap.nonzero()
# imap[non0] = imap[non0]/nmap[non0]
except IOError:
print "NO Noise File :: Set Noise to One"
nmap = algebra.info_array(sp.ones(imap.shape))
nmap.axes = imap.axes
nmap = algebra.make_vect(nmap)
if FKPweight:
for i in range(nmap.shape[0]):
# nmap[i] = nmap[i]/(1.+nmap[i]*fkpp)
nmap[i] = 1.0 / (1.0 + nmap[i] * fkpp)
# Using map in different day
hr_str = params["hr"][1]
end = pol_str
if len(params["last"]) != 0:
end = end + params["last"][1]
imap_fname = in_root + hr_str + mid[0] + end + ".npy"
imap2 = algebra.load(imap_fname)
imap2 = algebra.make_vect(imap2)
if imap2.axes != ("freq", "ra", "dec"):
raise ce.DataError("AXES ERROR!")
box2 = np.load(imap_fname)
nmap_fname = in_root + hr_str + mid[1] + end + ".npy"
try:
nmap2 = algebra.load(nmap_fname)
nmap2 = algebra.make_vect(nmap2)
bad = nmap2 < 1.0e-5 * nmap2.flatten().max()
nmap2[bad] = 0.0
non0 = nmap2.nonzero()
# imap2[non0] = imap2[non0]/nmap2[non0]
except IOError:
print "NO Noise File :: Set Noise to One"
nmap2 = algebra.info_array(sp.ones(imap2.shape))
nmap2.axes = imap2.axes
nmap2 = algebra.make_vect(nmap2)
if FKPweight:
for i in range(nmap.shape[0]):
# nmap2[i] = nmap2[i]/(1.+nmap2[i]*fkpp)
nmap2[i] = 1.0 / (1.0 + nmap2[i] * fkpp)
# print imap2.flatten().min()
# print dmap2.flatten().min()
# print nmap2.flatten().sum()
mapshape = np.array(imap.shape)
# print imap.shape
r = self.discrete(self.fq2r(imap.get_axis("freq")))
ra = self.discrete(imap.get_axis("ra")) * deg2rad
de = self.discrete(imap.get_axis("dec")) * deg2rad
ra0 = ra[int(ra.shape[0] / 2)]
ra = ra - ra0
dr = r.ptp() / r.shape[0]
dra = ra.ptp() / ra.shape[0]
dde = de.ptp() / de.shape[0]
disc_n = params["discrete"]
# print r.min(), r.max()
# print self.xyz(ra.min(), de.min(), r.min())
# print self.xyz(ra.max(), de.min(), r.min())
# print self.xyz(ra.min(), de.max(), r.min())
# print self.xyz(ra.max(), de.max(), r.min())
# print self.xyz(ra.min(), de.min(), r.max())
# print self.xyz(ra.max(), de.min(), r.max())
# print self.xyz(ra.min(), de.max(), r.max())
# print self.xyz(ra.max(), de.max(), r.max())
# return 0
mapinf = [dr, dra, dde, disc_n]
mapinf = np.array(mapinf)
# box = algebra.info_array(sp.zeros(params['boxshape']))
# box.axes = ('x','y','z')
# box = algebra.make_vect(box)
# boxshape = np.array(box.shape)
# box2 = algebra.info_array(sp.zeros(params['boxshape']))
# box2.axes = ('x','y','z')
# box2 = algebra.make_vect(box2)
xrange0 = params["Xrange"][0]
yrange0 = params["Yrange"][0]
zrange0 = params["Zrange"][0]
boxunit = params["boxunit"]
shapex = params["boxshape"][2]
shapera = ra.shape[0]
V = params["boxunit"] ** 3
boxinf = [xrange0, yrange0, zrange0, boxunit]
boxinf = np.array(boxinf)
print "PowerMaker: Filling the BOX"
# MakePower.Filling(imap, imap2, box, box2, r, ra, de, boxinf, mapinf)
# MakePower.Filling(dmap, dmap2, box, box2, r, ra, de, boxinf, mapinf)
# normalize
# nbox = algebra.info_array(sp.zeros(params['boxshape']))
# nbox.axes = ('x','y','z')
# nbox = algebra.make_vect(nbox)
# nbox2 = algebra.info_array(sp.zeros(params['boxshape']))
# nbox2.axes = ('x','y','z')
# nbox2 = algebra.make_vect(nbox2)
##non0 = nmap.nonzero()
##nmap[non0] = np.sqrt(1./nmap[non0])
##non0 = nmap2.nonzero()
##nmap2[non0] = np.sqrt(1./nmap2[non0])
# MakePower.Filling(nmap, nmap2, nbox, nbox2, r, ra, de, boxinf, mapinf)
nbox = algebra.info_array(sp.ones(params["boxshape"]))
nbox.axes = ("x", "y", "z")
nbox = algebra.make_vect(nbox)
nbox2 = algebra.info_array(sp.ones(params["boxshape"]))
nbox2.axes = ("x", "y", "z")
nbox2 = algebra.make_vect(nbox2)
if params["saveweight"]:
sp.save(out_root + "Weight_" + resultf, nbox)
sp.save(out_root + "Weight2_" + resultf, nbox2)
print "\t::Weight Saved "
normal = (nbox ** 2).flatten().sum()
normal2 = (nbox2 ** 2).flatten().sum()
normal = sqrt(normal) * sqrt(normal2)
# print normal
box = box * nbox
box2 = box2 * nbox2
print "PowerMaker: FFTing "
inputa = np.zeros(params["boxshape"], dtype=complex)
inputa.real = box
outputa = np.zeros(params["boxshape"], dtype=complex)
fft = FFTW.Plan(inputa, outputa, direction="forward", flags=["measure"])
FFTW.execute(fft)
# box = outputa.real**2 + outputa.imag**2
# inputb = np.zeros(params['boxshape'], dtype=complex)
# inputb.real = box2
# outputb = np.zeros(params['boxshape'], dtype=complex)
# fft = FFTW.Plan(inputb,outputb, direction='forward', flags=['measure'])
# FFTW.execute(fft)
fftbox = (outputa * (outputa.conjugate())).real
fftbox = fftbox * V / normal # /2./pi/pi/pi
# boxN = params['boxshape'][0]*params['boxshape'][1]*params['boxshape'][2]
# fftbox = 2*fftbox*V/boxN #/2./pi/pi/pi
return fftbox
def GetWindowFunctionData(self):
params = self.params
boxshape = params['boxshape']
boxunit = params['boxunit']
resultf = params['hr'][0]
if len(params['last']) != 0:
resultf = resultf + params['last'][0]
resultf = resultf + '-' + params['hr'][1]
if len(params['last']) != 0:
resultf = resultf + params['last'][1]
FKPweight = params['FKPweight']
in_root = params['input_root']
out_root = params['output_root']
mid = params['mid']
fkpp = params['FKPpk']
# Make parent directory and write parameter file.
kiyopy.utils.mkparents(params['output_root'])
parse_ini.write_params(params,
params['output_root']+'params.ini',prefix='wd_' )
all_out_fname_list = []
all_in_fname_list = []
#### Process ####
pol_str = params['polarizations'][0]
hr_str = params['hr'][0]
end = pol_str
if len(params['last']) != 0:
end = end + params['last'][0]
imap_fname = in_root + hr_str + mid[0] + end + '.npy'
imap = algebra.load(imap_fname)
imap = algebra.make_vect(imap)
if imap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
nmap_fname = in_root + hr_str + mid[1] + end + '.npy'
try:
nmap = algebra.load(nmap_fname)
nmap = algebra.make_vect(nmap)
bad = nmap<1.e-5*nmap.flatten().max()
nmap[bad] = 0.
non0 = nmap.nonzero()
#imap[non0] = imap[non0]/nmap[non0]
except IOError:
print 'NO Noise File :: Set Noise to One'
nmap = algebra.info_array(sp.ones(imap.shape))
nmap.axes = imap.axes
nmap = algebra.make_vect(nmap)
nmap.info = imap.info
if FKPweight:
for i in range(nmap.shape[0]):
nmap[i] = 1./(1.+nmap[i]*fkpp)
#nmap[i] = nmap[i]/(1.+nmap[i]*fkpp)
hr_str = params['hr'][1]
end = pol_str
if len(params['last']) != 0:
end = end + params['last'][1]
nmap_fname = in_root + hr_str + mid[1] + end + '.npy'
try:
nmap2 = algebra.load(nmap_fname)
nmap2 = algebra.make_vect(nmap2)
bad = nmap2<1.e-5*nmap2.flatten().max()
nmap2[bad] = 0.
non0 = nmap2.nonzero()
except IOError:
print 'NO Noise File :: Set Noise to One'
nmap2 = algebra.info_array(sp.ones(imap.shape))
nmap2.axes = imap.axes
nmap2 = algebra.make_vect(nmap2)
if FKPweight:
for i in range(nmap.shape[0]):
nmap2[i] = 1./(1.+nmap2[i]*fkpp)
#nmap2[i] = nmap2[i]/(1.+nmap2[i]*fkpp)
weight, weight2 = self.fill(nmap, nmap2)
#MakePower.Filling(nmap, weight, r, ra, de, boxinf, mapinf)
#MakePower.Filling(nmap2, weight2, r, ra, de, boxinf, mapinf)
#weight_fname = in_root + 'Weight_' + resultf + '.npy'
#weight = algebra.load(weight_fname)
#weight = algebra.make_vect(weight)
#weight_fname = in_root + 'Weight2_' + resultf + '.npy'
#weight2 = algebra.load(weight_fname)
#weight2 = algebra.make_vect(weight2)
normal = (weight**2).flatten().sum()
normal2 = (weight2**2).flatten().sum()
normal = sqrt(normal)*sqrt(normal2)
print "WindowFunctionMaker: FFTing "
inputa = np.zeros(params['boxshape'], dtype=complex)
outputa = np.zeros(params['boxshape'], dtype=complex)
fft = FFTW.Plan(inputa,outputa, direction='forward', flags=['measure'])
inputa.imag = 0.
inputa.real = weight
FFTW.execute(fft)
#inputa = np.zeros(params['boxshape'], dtype=complex)
outputb = np.zeros(params['boxshape'], dtype=complex)
fft = FFTW.Plan(inputa,outputb, direction='forward', flags=['measure'])
inputa.imag = 0.
inputa.real = weight2
FFTW.execute(fft)
V = params['boxunit']**3
fftbox = (outputa*(outputb.conjugate())).real
fftbox = fftbox*V/normal
WindowF = np.zeros(40)
k = np.zeros(40)
WindowF2 = np.zeros(shape=(10, 10))
k2 = np.zeros(shape=(2, 10))
MakePower.Make(fftbox, WindowF, k, WindowF2, k2)
kunit = 2.*pi/(params['boxunit'])
k = k*kunit
k2 = k2*kunit
#WindowF = WindowF#/V
#WindowF2 = WindowF2#/V
return WindowF, k
def execute(self, nprocesses=1) :
"""Worker funciton."""
params = self.params
# Make parent directory and write parameter file.
kiyopy.utils.mkparents(params['output_root'])
parse_ini.write_params(params, params['output_root'] + 'params.ini',
prefix='mm_')
save_noise_diag = params['save_noise_diag']
in_root = params['input_root']
all_out_fname_list = []
all_in_fname_list = []
# Loop over files to process.
for pol_str in params['polarizations']:
dmap_fname = in_root + 'dirty_map_' + pol_str + '.npy'
noise_fname = in_root + 'noise_inv_' + pol_str + '.npy'
all_in_fname_list.append(
kiyopy.utils.abbreviate_file_path(dmap_fname))
all_in_fname_list.append(
kiyopy.utils.abbreviate_file_path(noise_fname))
# Load the dirty map and the noise matrix.
dirty_map = algebra.load(dmap_fname)
dirty_map = algebra.make_vect(dirty_map)
if dirty_map.axes != ('freq', 'ra', 'dec') :
raise ce.DataError("Expeced dirty map to have axes "
"('freq', 'ra', 'dec'), but it has axes: "
+ str(dirty_map.axes))
shape = dirty_map.shape
noise_inv = algebra.open_memmap(noise_fname, 'r')
noise_inv = algebra.make_mat(noise_inv)
# Initialize the clean map.
clean_map = algebra.info_array(sp.zeros(dirty_map.shape))
clean_map.info = dict(dirty_map.info)
clean_map = algebra.make_vect(clean_map)
# If needed, initialize a map for the noise diagonal.
if save_noise_diag :
noise_diag = algebra.zeros_like(clean_map)
# Two cases for the noise. If its the same shape as the map then
# the noise is diagonal. Otherwise, it should be block diagonal in
# frequency.
if noise_inv.ndim == 3 :
if noise_inv.axes != ('freq', 'ra', 'dec') :
raise ce.DataError("Expeced noise matrix to have axes "
"('freq', 'ra', 'dec'), but it has: "
+ str(noise_inv.axes))
# Noise inverse can fit in memory, so copy it.
noise_inv_memory = sp.array(noise_inv, copy=True)
# Find the non-singular (covered) pixels.
max_information = noise_inv_memory.max()
good_data = noise_inv_memory < 1.0e-10*max_information
# Make the clean map.
clean_map[good_data] = (dirty_map[good_data]
/ noise_inv_memory[good_data])
if save_noise_diag :
noise_diag[good_data] = 1/noise_inv_memory[good_data]
elif noise_inv.ndim == 5 :
if noise_inv.axes != ('freq', 'ra', 'dec', 'ra', 'dec') :
raise ce.DataError("Expeced noise matrix to have axes "
"('freq', 'ra', 'dec', 'ra', 'dec'), "
"but it has: "
+ str(noise_inv.axes))
# Arrange the dirty map as a vector.
dirty_map_vect = sp.array(dirty_map) # A view.
dirty_map_vect.shape = (shape[0], shape[1]*shape[2])
frequencies = dirty_map.get_axis('freq')/1.0e6
# Allowcate memory only once.
noise_inv_freq = sp.empty((shape[1], shape[2], shape[1],
shape[2]), dtype=float)
if self.feedback > 1 :
print "Inverting noise matrix."
# Block diagonal in frequency so loop over frequencies.
for ii in xrange(dirty_map.shape[0]) :
if self.feedback > 1:
print "Frequency: ", "%5.1f"%(frequencies[ii]),
if self.feedback > 2:
print ", start mmap read:",
sys.stdout.flush()
noise_inv_freq[...] = noise_inv[ii, ...]
if self.feedback > 2:
print "done, start eig:",
sys.stdout.flush()
noise_inv_freq.shape = (shape[1]*shape[2],
shape[1]*shape[2])
# Solve the map making equation by diagonalization.
noise_inv_diag, Rot = sp.linalg.eigh(noise_inv_freq,
overwrite_a=True)
if self.feedback > 2:
print "done",
map_rotated = sp.dot(Rot.T, dirty_map_vect[ii])
# Zero out infinite noise modes.
bad_modes = noise_inv_diag < 1.0e-5*noise_inv_diag.max()
if self.feedback > 1:
print ", discarded: ",
print "%4.1f"%(100.0*sp.sum(bad_modes)/bad_modes.size),
print "% of modes",
if self.feedback > 2:
print ", start rotations:",
sys.stdout.flush()
map_rotated[bad_modes] = 0.
noise_inv_diag[bad_modes] = 1.0
# Solve for the clean map and rotate back.
map_rotated /= noise_inv_diag
map = sp.dot(Rot, map_rotated)
if self.feedback > 2:
print "done",
sys.stdout.flush()
# Fill the clean array.
map.shape = (shape[1], shape[2])
clean_map[ii, ...] = map
if save_noise_diag :
# Using C = R Lambda R^T
# where Lambda = diag(1/noise_inv_diag).
temp_noise_diag = 1/noise_inv_diag
temp_noise_diag[bad_modes] = 0
# Multiply R by the diagonal eigenvalue matrix.
# Broadcasting does equivalent of mult by diag matrix.
temp_mat = Rot*temp_noise_diag
# Multiply by R^T, but only calculate the diagonal
# elements.
for jj in range(shape[1]*shape[2]) :
temp_noise_diag[jj] = sp.dot(temp_mat[jj,:],
Rot[jj,:])
temp_noise_diag.shape = (shape[1], shape[2])
noise_diag[ii, ...] = temp_noise_diag
# Return workspace memory to origional shape.
noise_inv_freq.shape = (shape[1], shape[2],
shape[1], shape[2])
if self.feedback > 1:
print ""
sys.stdout.flush()
elif noise_inv.ndim == 6 :
raise NotImplementedError("Full noise matrix not yet "
"implemented. Best we can do is "
"block diagonal in frequency.")
else :
raise ce.DataError("Noise matrix has bad shape.")
# Write the clean map to file.
out_fname = params['output_root'] + 'clean_map_' + pol_str + '.npy'
algebra.save(out_fname, clean_map)
all_out_fname_list.append(
kiyopy.utils.abbreviate_file_path(out_fname))
if save_noise_diag :
noise_diag_fname = (params['output_root'] + 'noise_diag_'
+ pol_str + '.npy')
algebra.save(noise_diag_fname, noise_diag)
all_out_fname_list.append(
kiyopy.utils.abbreviate_file_path(noise_diag_fname))
# Finally update the history object.
history = hist.read(in_root + 'history.hist')
history.add('Read map and noise files:', all_in_fname_list)
history.add('Converted dirty map to clean map.', all_out_fname_list)
h_fname = params['output_root'] + "history.hist"
history.write(h_fname)
def execute(self, nprocesses=1):
params = self.params
# Make parent directory and write parameter file.
kiyopy.utils.mkparents(params['output_root'])
parse_ini.write_params(params, params['output_root']+'params.ini',prefix='jk_')
in_root = params['input_root']
out_root = params['output_root']
mid = params['mid']
all_out_fname_list = []
all_in_fname_list = []
n_processes = params['processes']
#### Process ####
pol_str = params['polarizations'][0]
#hr_str = params['hr'][0]
for hr_str, ii in zip(params['hr'], range(len(params['hr']))):
end = pol_str
if (len(params['last'])!=0):
end = pol_str + params['last'][ii]
print 'Making JK Map for:' + hr_str[:-1]
#imap_fname = in_root + hr_str + 'dirty_map_' + pol_str + '.npy'
imap_fname = in_root + hr_str + mid[0] + end + '.npy'
imap = algebra.load(imap_fname)
imap = algebra.make_vect(imap)
imap = imap - imap.flatten().mean()
if imap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
imap_fname = in_root + hr_str + mid[1] + end + '.npy'
try:
nmap = algebra.load(imap_fname)
nmap = algebra.make_vect(nmap)
if nmap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
except IOError:
print 'NO Noise File :: Set Noise to One'
nmap = algebra.info_array(sp.ones(imap.shape))
nmap.axes = imap.axes
nmap = algebra.make_vect(nmap)
nmap.info = imap.info
if nmap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
shape = np.array(imap.shape)
print shape
begin0=0
begin1=0
begin2=0
stop0 =0
stop1 =0
stop2 =0
r = self.fq2r(imap.get_axis('freq'))
dr = (r.max()-r.min())/params['jkn0']
ranger = np.searchsorted(r,np.arange(r[0], r[-1], dr))
num = 0
#print range(0, shape[1], shape[1]/params['jkn1'])
#print range(0, shape[2], shape[2]/params['jkn2'])
#return 0
#jkmap = algebra.info_array(sp.zeros(imap.shape))
#jkmap.info = dict(imap.info)
#jkmap = algebra.make_vect(jkmap)
#njkmap = algebra.info_array(sp.zeros(nmap.shape))
#njkmap.info = dict(nmap.info)
#njkmap = algebra.make_vect(njkmap)
list_abandon = []
for ii in range(params['jkn0']):
begin0 = ranger[ii]
if begin0==ranger[-1]:
stop0 = shape[0]
else:
stop0 = ranger[ii+1]
for begin1 in range(0, shape[1], shape[1]/params['jkn1']):
stop1 = begin1 + shape[1]/params['jkn1']
for begin2 in range(0, shape[2], shape[2]/params['jkn2']):
stop2 = begin2 + shape[2]/params['jkn2']
list_abandon.append(
[begin0, stop0, begin1, stop1, begin2, stop2])
num = num + 1
n_new = n_processes
n_map = len(list_abandon)
if n_new <=1:
for ii in range(n_map):
jkmap_fname = \
out_root+hr_str+'jk'+str(ii)+mid[0]+end+'.npy'
jknmap_fname = \
out_root+hr_str+'jk'+str(num)+mid[1]+end+'.npy'
self.process_map(
list_abandon[ii], imap, nmap, jkmap_fname, jknmap_fname)
elif n_new >32:
raise ValueError("Processes limit is 32")
else:
process_list = range(n_new)
for ii in xrange(n_new+n_map):
if ii >= n_new:
process_list[ii%n_new].join()
if process_list[ii%n_new].exitcode != 0:
raise RuntimeError("A thred faild with exit code"
+ str(process_list[ii%n_new].exitcode))
if ii < n_map:
jkmap_fname = \
hr_str+'jk'+str(ii)+mid[0]+end+'.npy'
jknmap_fname = \
hr_str+'jk'+str(ii)+mid[1]+end+'.npy'
process_list[ii%n_new] = mp.Process(
target=self.process_map, args=(list_abandon[ii],
imap, nmap, jkmap_fname, jknmap_fname))
process_list[ii%n_new].start()
def GetFFTbox(self):
params = self.params
in_root = params['input_root']
out_root = params['output_root']
filename = params['filename']
selection = params['selection']
OmegaHI = params['OmegaHI']
Omegam = params['Omegam']
OmegaL = params['OmegaL']
FKPweight = params['FKPweight']
imap_fname = in_root + filename + '.npy'
imap = algebra.load(imap_fname)
imap = algebra.make_vect(imap)
if imap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
nmap_fname = in_root + selection + '.npy'
try:
nmap = algebra.load(nmap_fname)
nmap = algebra.make_vect(nmap)
bad = nmap<1.e-5*nmap.flatten().max()
nmap[bad] = 0.
non0 = nmap.nonzero()
#imap[non0] = imap[non0]/nmap[non0]
except IOError:
print 'NO Noise File :: Set Noise to One'
nmap = algebra.info_array(sp.ones(imap.shape))
nmap.axes = imap.axes
nmap = algebra.make_vect(nmap)
if FKPweight:
for i in range(nmap.shape[0]):
nmap[i] = nmap[i]/(1.+nmap[i]*1.e4)
#Using map in different day
imap2 = imap.copy()
if imap2.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
nmap2 = nmap.copy()
mapshape = np.array(imap.shape)
r = self.discrete(self.fq2r(imap.get_axis('freq')))
ra = self.discrete(imap.get_axis('ra'))*deg2rad
de = self.discrete(imap.get_axis('dec'))*deg2rad
ra0= ra[int(ra.shape[0]/2)]
ra = ra - ra0
dr = r.ptp()/r.shape[0]
dra= ra.ptp()/ra.shape[0]
dde= de.ptp()/de.shape[0]
disc_n = params['discrete']
mapinf = [dr, dra, dde, disc_n]
mapinf = np.array(mapinf)
box = algebra.info_array(sp.zeros(params['boxshape']))
box.axes = ('x','y','z')
box = algebra.make_vect(box)
box2 = algebra.info_array(sp.zeros(params['boxshape']))
box2.axes = ('x','y','z')
box2 = algebra.make_vect(box2)
boxshape = np.array(box.shape)
xrange0 = params['Xrange'][0]
yrange0 = params['Yrange'][0]
zrange0 = params['Zrange'][0]
boxunit = params['boxunit']
shapex = params['boxshape'][2]
shapera = ra.shape[0]
V = params['boxunit']**3
boxinf = [xrange0, yrange0, zrange0, boxunit]
boxinf = np.array(boxinf)
MakePower.Filling(imap, imap2, box, box2, r, ra, de, boxinf, mapinf)
#MakePower.Filling(dmap, dmap2, box, box2, r, ra, de, boxinf, mapinf)
# normalize
nbox = algebra.info_array(sp.zeros(params['boxshape']))
nbox.axes = ('x','y','z')
nbox = algebra.make_vect(nbox)
nbox2 = algebra.info_array(sp.zeros(params['boxshape']))
nbox2.axes = ('x','y','z')
nbox2 = algebra.make_vect(nbox2)
#non0 = nmap.nonzero()
#nmap[non0] = np.sqrt(1./nmap[non0])
#non0 = nmap2.nonzero()
#nmap2[non0] = np.sqrt(1./nmap2[non0])
MakePower.Filling(nmap, nmap2, nbox, nbox2, r, ra, de, boxinf, mapinf)
#if params['saveweight']:
# sp.save(out_root+'Weight_'+resultf, nbox)
# sp.save(out_root+'Weight2_'+resultf, nbox2)
# print '\t::Weight Saved '
normal = (nbox**2).flatten().sum()
normal2 = (nbox2**2).flatten().sum()
normal = sqrt(normal)*sqrt(normal2)
box = box*nbox
box2 = box2*nbox2
inputa = np.zeros(params['boxshape'], dtype=complex)
inputa.real = box
outputa = np.zeros(params['boxshape'], dtype=complex)
fft = FFTW.Plan(inputa,outputa, direction='forward', flags=['measure'])
FFTW.execute(fft)
box = outputa.real**2 + outputa.imag**2
inputa = np.zeros(params['boxshape'], dtype=complex)
inputa.real = box2
outputa = np.zeros(params['boxshape'], dtype=complex)
fft = FFTW.Plan(inputa,outputa, direction='forward', flags=['measure'])
FFTW.execute(fft)
box2 = outputa.real**2 + outputa.imag**2
fftbox = box.__pow__(0.5)*box2.__pow__(0.5)
fftbox = fftbox*V*V/normal/V #/2./pi/pi/pi
return fftbox
def fill(self, imap, nmap, mmap=None):
params = self.params
mapshape = np.array(imap.shape)
r = self.fq2r(imap.get_axis('freq'))
ra = imap.get_axis('ra')*deg2rad
de = imap.get_axis('dec')*deg2rad
ra0= ra[int(ra.shape[0]/2)]
ra = ra - ra0
dra= ra.ptp()/ra.shape[0]
dde= de.ptp()/de.shape[0]
#print r.min(), r.max()
#print self.xyz(ra.min(), de.min(), r.min())
#print self.xyz(ra.max(), de.min(), r.min())
#print self.xyz(ra.min(), de.max(), r.min())
#print self.xyz(ra.max(), de.max(), r.min())
#print self.xyz(ra.min(), de.min(), r.max())
#print self.xyz(ra.max(), de.min(), r.max())
#print self.xyz(ra.min(), de.max(), r.max())
#print self.xyz(ra.max(), de.max(), r.max())
###return 0
mapinf = [ra.min(), dra, de.min(), dde]
mapinf = np.array(mapinf)
box = algebra.info_array(sp.zeros(params['boxshape']))
box.axes = ('x','y','z')
box = algebra.make_vect(box)
box_xrange = params['Xrange']
box_yrange = params['Yrange']
box_zrange = params['Zrange']
box_unit = params['boxunit']
box_disc = params['discrete']
box_x = np.arange(box_xrange[0], box_xrange[1], box_unit/box_disc)
box_y = np.arange(box_yrange[0], box_yrange[1], box_unit/box_disc)
box_z = np.arange(box_zrange[0], box_zrange[1], box_unit/box_disc)
#print box_x.shape
#print box_y.shape
#print box_z.shape
boxshape = np.array(box.shape)*box_disc
boxinf0 = [0, 0, 0]
boxinf0 = np.array(boxinf0)
boxinf1 = [boxshape[0], boxshape[1], boxshape[2]]
boxinf1 = np.array(boxinf1)
print "MapPrepare: Filling the FFT BOX"
MakePower.Filling(
imap, r, mapinf, box, boxinf0, boxinf1, box_x, box_y, box_z)
nbox = algebra.info_array(sp.zeros(params['boxshape']))
nbox.axes = ('x','y','z')
nbox = algebra.make_vect(nbox)
#nbox = algebra.info_array(sp.ones(params['boxshape']))
#nbox.axes = ('x','y','z')
#nbox = algebra.make_vect(nbox)
MakePower.Filling(
nmap, r, mapinf, nbox, boxinf0, boxinf1, box_x, box_y, box_z)
if mmap != None:
mbox = algebra.info_array(sp.zeros(params['boxshape']))
mbox.axes = ('x','y','z')
mbox = algebra.make_vect(mbox)
MakePower.Filling(
mmap, r, mapinf, mbox, boxinf0, boxinf1, box_x, box_y, box_z)
return box, nbox, mbox
else:
return box, nbox
def execute(self, nprocesses=1) :
"""Worker funciton."""
params = self.params
# Make parent directory and write parameter file.
kiyopy.utils.mkparents(params['output_root'])
parse_ini.write_params(params, params['output_root'] + 'params.ini',
prefix=prefix)
save_noise_diag = params['save_noise_diag']
in_root = params['input_root']
all_out_fname_list = []
all_in_fname_list = []
# Figure out what the band names are.
bands = params['bands']
if not bands:
map_files = glob.glob(in_root + 'dirty_map_' + pol_str + "_*.npy")
bands = []
root_len = len(in_root + 'dirty_map_')
for file_name in map_files:
bands.append(file_name[root_len:-4])
# Loop over files to process.
for pol_str in params['polarizations']:
for band in bands:
if band == -1:
band_str = ''
else:
band_str = "_" + repr(band)
dmap_fname = (in_root + 'dirty_map_' + pol_str +
band_str + '.npy')
all_in_fname_list.append(
kiyopy.utils.abbreviate_file_path(dmap_fname))
# Load the dirty map and the noise matrix.
dirty_map = algebra.load(dmap_fname)
dirty_map = algebra.make_vect(dirty_map)
if dirty_map.axes != ('freq', 'ra', 'dec') :
msg = ("Expeced dirty map to have axes ('freq',"
"'ra', 'dec'), but it has axes: "
+ str(dirty_map.axes))
raise ce.DataError(msg)
shape = dirty_map.shape
# Initialize the clean map.
clean_map = algebra.info_array(sp.zeros(dirty_map.shape))
clean_map.info = dict(dirty_map.info)
clean_map = algebra.make_vect(clean_map)
# If needed, initialize a map for the noise diagonal.
if save_noise_diag :
noise_diag = algebra.zeros_like(clean_map)
if params["from_eig"]:
# Solving from eigen decomposition of the noise instead of
# the noise itself.
# Load in the decomposition.
evects_fname = (in_root + 'noise_evects_' + pol_str +
+ band_str + '.npy')
if self.feedback > 1:
print "Using dirty map: " + dmap_fname
print "Using eigenvectors: " + evects_fname
evects = algebra.open_memmap(evects_fname, 'r')
evects = algebra.make_mat(evects)
evals_inv_fname = (in_root + 'noise_evalsinv_' + pol_str
+ "_" + repr(band) + '.npy')
evals_inv = algebra.load(evals_inv_fname)
evals_inv = algebra.make_mat(evals_inv)
# Solve for the map.
if params["save_noise_diag"]:
clean_map, noise_diag = solve_from_eig(evals_inv,
evects, dirty_map, True, self.feedback)
else:
clean_map = solve_from_eig(evals_inv,
evects, dirty_map, False, self.feedback)
# Delete the eigen vectors to recover memory.
del evects
else:
# Solving from the noise.
noise_fname = (in_root + 'noise_inv_' + pol_str +
band_str + '.npy')
if self.feedback > 1:
print "Using dirty map: " + dmap_fname
print "Using noise inverse: " + noise_fname
all_in_fname_list.append(
kiyopy.utils.abbreviate_file_path(noise_fname))
noise_inv = algebra.open_memmap(noise_fname, 'r')
noise_inv = algebra.make_mat(noise_inv)
# Two cases for the noise. If its the same shape as the map
# then the noise is diagonal. Otherwise, it should be
# block diagonal in frequency.
if noise_inv.ndim == 3 :
if noise_inv.axes != ('freq', 'ra', 'dec') :
msg = ("Expeced noise matrix to have axes "
"('freq', 'ra', 'dec'), but it has: "
+ str(noise_inv.axes))
raise ce.DataError(msg)
# Noise inverse can fit in memory, so copy it.
noise_inv_memory = sp.array(noise_inv, copy=True)
# Find the non-singular (covered) pixels.
max_information = noise_inv_memory.max()
good_data = noise_inv_memory < 1.0e-10*max_information
# Make the clean map.
clean_map[good_data] = (dirty_map[good_data]
/ noise_inv_memory[good_data])
if save_noise_diag :
noise_diag[good_data] = \
1/noise_inv_memory[good_data]
elif noise_inv.ndim == 5 :
if noise_inv.axes != ('freq', 'ra', 'dec', 'ra',
'dec'):
msg = ("Expeced noise matrix to have axes "
"('freq', 'ra', 'dec', 'ra', 'dec'), "
"but it has: " + str(noise_inv.axes))
raise ce.DataError(msg)
# Arrange the dirty map as a vector.
dirty_map_vect = sp.array(dirty_map) # A view.
dirty_map_vect.shape = (shape[0], shape[1]*shape[2])
frequencies = dirty_map.get_axis('freq')/1.0e6
# Allowcate memory only once.
noise_inv_freq = sp.empty((shape[1], shape[2],
shape[1], shape[2]), dtype=float)
if self.feedback > 1 :
print "Inverting noise matrix."
# Block diagonal in frequency so loop over frequencies.
for ii in xrange(dirty_map.shape[0]) :
if self.feedback > 1:
print "Frequency: ", "%5.1f"%(frequencies[ii]),
if self.feedback > 2:
print ", start mmap read:",
sys.stdout.flush()
noise_inv_freq[...] = noise_inv[ii, ...]
if self.feedback > 2:
print "done, start eig:",
sys.stdout.flush()
noise_inv_freq.shape = (shape[1]*shape[2],
shape[1]*shape[2])
# Solve the map making equation by diagonalization.
noise_inv_diag, Rot = sp.linalg.eigh(
noise_inv_freq, overwrite_a=True)
if self.feedback > 2:
print "done",
map_rotated = sp.dot(Rot.T, dirty_map_vect[ii])
# Zero out infinite noise modes.
bad_modes = (noise_inv_diag
< 1.0e-5 * noise_inv_diag.max())
if self.feedback > 1:
print ", discarded: ",
print "%4.1f" % (100.0 * sp.sum(bad_modes)
/ bad_modes.size),
print "% of modes",
if self.feedback > 2:
print ", start rotations:",
sys.stdout.flush()
map_rotated[bad_modes] = 0.
noise_inv_diag[bad_modes] = 1.0
# Solve for the clean map and rotate back.
map_rotated /= noise_inv_diag
map = sp.dot(Rot, map_rotated)
if self.feedback > 2:
print "done",
sys.stdout.flush()
# Fill the clean array.
map.shape = (shape[1], shape[2])
clean_map[ii, ...] = map
if save_noise_diag :
# Using C = R Lambda R^T
# where Lambda = diag(1/noise_inv_diag).
temp_noise_diag = 1/noise_inv_diag
temp_noise_diag[bad_modes] = 0
# Multiply R by the diagonal eigenvalue matrix.
# Broadcasting does equivalent of mult by diag
# matrix.
temp_mat = Rot*temp_noise_diag
# Multiply by R^T, but only calculate the
# diagonal elements.
for jj in range(shape[1]*shape[2]) :
temp_noise_diag[jj] = sp.dot(
temp_mat[jj,:], Rot[jj,:])
temp_noise_diag.shape = (shape[1], shape[2])
noise_diag[ii, ...] = temp_noise_diag
# Return workspace memory to origional shape.
noise_inv_freq.shape = (shape[1], shape[2],
shape[1], shape[2])
if self.feedback > 1:
print ""
sys.stdout.flush()
elif noise_inv.ndim == 6 :
if save_noise_diag:
# OLD WAY.
#clean_map, noise_diag, chol = solve(noise_inv,
# dirty_map, True, feedback=self.feedback)
# NEW WAY.
clean_map, noise_diag, noise_inv_diag, chol = \
solve(noise_fname, noise_inv, dirty_map,
True, feedback=self.feedback)
else:
# OLD WAY.
#clean_map, chol = solve(noise_inv, dirty_map,
# False, feedback=self.feedback)
# NEW WAY.
clean_map, noise_inv_diag, chol = \
solve(noise_fname, noise_inv, dirty_map,
False, feedback=self.feedback)
if params['save_cholesky']:
chol_fname = (params['output_root'] + 'chol_'
+ pol_str + band_str + '.npy')
sp.save(chol_fname, chol)
if params['save_noise_inv_diag']:
noise_inv_diag_fname = (params['output_root'] +
'noise_inv_diag_' + pol_str + band_str
+ '.npy')
algebra.save(noise_inv_diag_fname, noise_inv_diag)
# Delete the cholesky to recover memory.
del chol
else :
raise ce.DataError("Noise matrix has bad shape.")
# In all cases delete the noise object to recover memeory.
del noise_inv
# Write the clean map to file.
out_fname = (params['output_root'] + 'clean_map_'
+ pol_str + band_str + '.npy')
if self.feedback > 1:
print "Writing clean map to: " + out_fname
algebra.save(out_fname, clean_map)
all_out_fname_list.append(
kiyopy.utils.abbreviate_file_path(out_fname))
if save_noise_diag :
noise_diag_fname = (params['output_root'] + 'noise_diag_'
+ pol_str + band_str + '.npy')
algebra.save(noise_diag_fname, noise_diag)
all_out_fname_list.append(
kiyopy.utils.abbreviate_file_path(noise_diag_fname))
# Check the clean map for faileur.
if not sp.alltrue(sp.isfinite(clean_map)):
n_bad = sp.sum(sp.logical_not(sp.isfinite(clean_map)))
msg = ("Non finite entries found in clean map. Solve"
" failed. %d out of %d entries bad"
% (n_bad, clean_map.size))
raise RuntimeError(msg)
def execute(self, nprocesses=1):
params = self.params
resultf = params['hr'][0]
if len(params['last']) != 0:
resultf = resultf + params['last'][0]
resultf = resultf + '-' + params['hr'][1]
if len(params['last']) != 0:
resultf = resultf + params['last'][1]
# Make parent directory and write parameter file.
kiyopy.utils.mkparents(params['output_root'])
parse_ini.write_params(params, params['output_root']+'params.ini',prefix='pk_' )
in_root = params['input_root']
out_root = params['output_root']
mid = params['mid']
all_out_fname_list = []
all_in_fname_list = []
#### Process ####
pol_str = params['polarizations'][0]
hr_str = params['hr'][0]
end = pol_str
if len(params['last']) != 0:
end = end + params['last'][0]
imap_fname = in_root + hr_str + mid[0] + end + '.npy'
imap = algebra.load(imap_fname)
imap = algebra.make_vect(imap)
if imap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
nmap_fname = in_root + hr_str + mid[1] + end + '.npy'
try:
nmap = algebra.load(nmap_fname)
nmap = algebra.make_vect(nmap)
bad = nmap<1.e-5*nmap.flatten().max()
nmap[bad] = 0.
#non0 = nmap.nonzero()
#imap[non0] = imap[non0]/nmap[non0]
except IOError:
print 'NO Noise File :: Set Noise to One'
nmap = algebra.info_array(sp.ones(imap.shape))
nmap.axes = imap.axes
nmap = algebra.make_vect(nmap)
## Using map in different day
hr_str = params['hr'][1]
end = pol_str
if len(params['last']) != 0:
end = end + params['last'][1]
imap_fname = in_root + hr_str + mid[0] + end + '.npy'
imap2 = algebra.load(imap_fname)
imap2 = algebra.make_vect(imap2)
if imap2.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
nmap_fname = in_root + hr_str + mid[1] + end + '.npy'
try:
nmap2 = algebra.load(nmap_fname)
nmap2 = algebra.make_vect(nmap2)
bad = nmap2<1.e-5*nmap2.flatten().max()
nmap2[bad] = 0.
#non0 = nmap2.nonzero()
#imap2[non0] = imap2[non0]/nmap2[non0]
except IOError:
print 'NO Noise File :: Set Noise to One'
nmap2 = algebra.info_array(sp.ones(imap2.shape))
nmap2.axes = imap2.axes
nmap2 = algebra.make_vect(nmap2)
## FKP ##
fkp = algebra.info_array(sp.ones(imap.shape))
fkp.axes = imap.axes
fkp = algebra.make_vect(fkp)
fkp2 = algebra.info_array(sp.ones(imap.shape))
fkp2.axes = imap.axes
fkp2 = algebra.make_vect(fkp2)
OmegaHI = params['OmegaHI']
Omegam = params['Omegam']
OmegaL = params['OmegaL']
freq = imap.get_axis('freq')
z = 1.4e9/freq -1.
a3 = (1+z)**(-3)
Tb = 0.3e-3 * (OmegaHI/1.e-3) * ((Omegam + a3*OmegaL)/0.29)**(-0.5)\
* ((1.+z)/2.5)**0.5
for i in range(fkp.shape[0]):
fkp[i] = fkp[i]*Tb[i]/(1.+Tb[i]*1.e4)
fkp2[i] = fkp2[i]*Tb[i]/(1.+Tb[i]*1.e4)
#print imap2.flatten().min()
#print dmap2.flatten().min()
#print nmap2.flatten().sum()
mapshape = np.array(imap.shape)
#print imap.shape
r = self.discrete(self.fq2r(imap.get_axis('freq')))
ra = self.discrete(imap.get_axis('ra'))*deg2rad
de = self.discrete(imap.get_axis('dec'))*deg2rad
ra0= ra[int(ra.shape[0]/2)]
ra = ra - ra0
dr = r.ptp()/r.shape[0]
dra= ra.ptp()/ra.shape[0]
dde= de.ptp()/de.shape[0]
disc_n = params['discrete']
#print r.min(), r.max()
#print self.xyz(ra.min(), de.min(), r.min())
#print self.xyz(ra.max(), de.min(), r.min())
#print self.xyz(ra.min(), de.max(), r.min())
#print self.xyz(ra.max(), de.max(), r.min())
#print self.xyz(ra.min(), de.min(), r.max())
#print self.xyz(ra.max(), de.min(), r.max())
#print self.xyz(ra.min(), de.max(), r.max())
#print self.xyz(ra.max(), de.max(), r.max())
#return 0
mapinf = [dr, dra, dde, disc_n]
mapinf = np.array(mapinf)
box = algebra.info_array(sp.zeros(params['boxshape']))
box.axes = ('x','y','z')
box = algebra.make_vect(box)
boxshape = np.array(box.shape)
box2 = algebra.info_array(sp.zeros(params['boxshape']))
box2.axes = ('x','y','z')
box2 = algebra.make_vect(box2)
xrange0 = params['Xrange'][0]
yrange0 = params['Yrange'][0]
zrange0 = params['Zrange'][0]
boxunit = params['boxunit']
shapex = params['boxshape'][2]
shapera = ra.shape[0]
boxinf = [xrange0, yrange0, zrange0, boxunit]
boxinf = np.array(boxinf)
print "PowerMaker: Filling the BOX"
MakePower.Filling(imap, imap2, box, box2, r, ra, de, boxinf, mapinf)
#MakePower.Filling(dmap, dmap2, box, box2, r, ra, de, boxinf, mapinf)
# normalize
nbox = algebra.info_array(sp.zeros(params['boxshape']))
nbox.axes = ('x','y','z')
nbox = algebra.make_vect(nbox)
nbox2 = algebra.info_array(sp.zeros(params['boxshape']))
nbox2.axes = ('x','y','z')
nbox2 = algebra.make_vect(nbox2)
#non0 = nmap.nonzero()
#nmap[non0] = np.sqrt(1./nmap[non0])
#non0 = nmap2.nonzero()
#nmap2[non0] = np.sqrt(1./nmap2[non0])
MakePower.Filling(nmap, nmap2, nbox, nbox2, r, ra, de, boxinf, mapinf)
if params['saveweight']:
sp.save(out_root+'Weight_'+resultf, nbox)
sp.save(out_root+'Weight2_'+resultf, nbox2)
print '\t::Weight Saved'
## FKP box ##
fkpbox = algebra.info_array(sp.zeros(params['boxshape']))
fkpbox.axes = ('x','y','z')
fkpbox = algebra.make_vect(fkpbox)
fkpbox2 = algebra.info_array(sp.zeros(params['boxshape']))
fkpbox2.axes = ('x','y','z')
fkpbox2 = algebra.make_vect(fkpbox2)
MakePower.Filling(fkp, fkp2, fkpbox, fkpbox2, r, ra, de, boxinf, mapinf)
fkpnormal = (fkpbox**2).flatten().sum()
#nbox = np.ones(params['boxshape'])
#nbox2 = np.ones(params['boxshape'])
#bad = nbox<1.e-5*nbox.flatten().max()
#nbox[bad] = 0.
#non0 = nbox.nonzero()
#nbox[non0] = 1./nbox[non0]
normal = (nbox**2).flatten().sum()
#bad = nbox2<1.e-5*nbox2.flatten().max()
#nbox2[bad] = 0.
#non0 = nbox2.nonzero()
#nbox2[non0] = 1./nbox2[non0]
normal2 = (nbox2**2).flatten().sum()
normal = sqrt(normal)*sqrt(normal2)
#print normal
#box = box*nbox
#box2 = box2*nbox2
box = box*fkpbox
box2 = box2*fkpbox2
print "PowerMaker: FFTing "
inputa = np.zeros(params['boxshape'], dtype=complex)
inputa.real = box.copy()
outputa = np.zeros(params['boxshape'], dtype=complex)
fft = FFTW.Plan(inputa,outputa, direction='forward', flags=['measure'])
FFTW.execute(fft)
box = outputa.real**2 + outputa.imag**2
inputa = np.zeros(params['boxshape'], dtype=complex)
inputa.real = box2.copy()
outputa = np.zeros(params['boxshape'], dtype=complex)
fft = FFTW.Plan(inputa,outputa, direction='forward', flags=['measure'])
FFTW.execute(fft)
box2 = outputa.real**2 + outputa.imag**2
fftbox = box.__pow__(0.5)*box2.__pow__(0.5)
V = params['boxunit']**3
#fftbox = fftbox*V*V/normal #/2./pi/pi/pi
fftbox = fftbox*V*V/fkpnormal #/2./pi/pi/pi
#boxN = params['boxshape'][0]*params['boxshape'][1]*params['boxshape'][2]
#fftbox = fftbox*V*V/boxN #/2./pi/pi/pi
PK = np.zeros(40)
k = np.zeros(40)
PK2 = np.zeros(shape=(10, 10))
k2 = np.zeros(shape=(2, 10))
MakePower.Make(fftbox, PK, k, PK2, k2)
kunit = 2.*pi/(boxunit)
k = k*kunit
k2 = k2*kunit
PK = PK/V
PK2 = PK2/V
#PK = PK/(V*boxN)
#PK2 = PK2/V/boxN
non0 = PK.nonzero()
#print PK2
#print PK
#print k[non0]
#return 0
sp.save(out_root+'fkpPK_'+resultf, PK)
sp.save(out_root+'fkpPK2_'+resultf, PK2)
sp.save(out_root+'fkpk_'+resultf, k)
if self.plot==True:
print PK[non0]
plt.figure(figsize=(8,8))
#print k
#print PK
plt.subplot('211')
plt.scatter(k.take(non0), PK.take(non0))
plt.loglog()
plt.ylim(ymin=1.e-8)
plt.xlim(xmin=k.min(), xmax=k.max())
plt.title('Power Spectrum')
plt.xlabel('$k$')
plt.ylabel('$P(k) (Kelvin^{2}(h^{-1}Mpc)^3)$')
PK = PK*k*k*k/2./pi/pi
#print PK
plt.subplot('212')
plt.scatter(k.take(non0), PK.take(non0))
plt.loglog()
plt.ylim(ymin=1.e-16)
plt.xlim(xmin=k.min(), xmax=k.max())
plt.xlabel('$k (h Mpc^{-1})$')
plt.ylabel('$\Delta^2 (Kelvin^{2})$')
#plt.show()
plt.savefig(out_root+'fkppower_'+resultf+'.eps', format='eps')
PK2 = np.log10(PK2)
plt.figure(figsize=(6,6))
extent = (k2[0][0], k2[0][-1], k2[1][0], k2[1][-1])
plt.imshow(PK2, origin='lower', extent = extent, interpolation='nearest')
plt.xlabel('$k vertical (h Mpc^{-1})$')
plt.ylabel('$k parallel (h Mpc^{-1})$')
cb = plt.colorbar()
cb.set_label('$lg(P^{2D}_{k_pk_v}) (Kelvin^2(h^{-1}Mpc)^3)$')
plt.loglog()
plt.savefig(out_root+'fkppower2_'+resultf+'.eps', format='eps')
plt.show()
#print 'Finished @_@ '
return PK
