def load_pairs(self):
r"""load the set of map/noise pairs specified by keys handed to the
database. This sets up operations on the quadratic product
Q = map1^T noise_inv1 B noise_inv2 map2
"""
par = self.params
(self.pairlist, pairdict) = dp.cross_maps(par['map1'], par['map2'],
par['noise_inv1'],
par['noise_inv2'],
noise_inv_suffix=";noise_weight",
verbose=False,
tack_on=self.tack_on_input,
db_to_use=self.datapath_db)
for pairitem in self.pairlist:
pdict = pairdict[pairitem]
print "-" * 80
dp.print_dictionary(pdict, sys.stdout,
key_list=['map1', 'noise_inv1',
'map2', 'noise_inv2'])
map1 = algebra.make_vect(algebra.load(pdict['map1']))
map2 = algebra.make_vect(algebra.load(pdict['map2']))
if par['simfile'] is not None:
print "adding %s with multiplier %s" % (par['simfile'],
par['sim_multiplier'])
sim = algebra.make_vect(algebra.load(par['simfile']))
sim *= par['sim_multiplier']
print sim.shape, map1.shape
else:
sim = algebra.zeros_like(map1)
noise_inv1 = algebra.make_vect(algebra.load(pdict['noise_inv1']))
noise_inv2 = algebra.make_vect(algebra.load(pdict['noise_inv2']))
pair = map_pair.MapPair(map1 + sim, map2 + sim,
noise_inv1, noise_inv2,
self.freq_list)
pair.set_names(pdict['tag1'], pdict['tag2'])
pair.params = self.params
self.pairs[pairitem] = pair
if par['subtract_inputmap_from_sim'] or \
par['subtract_sim_from_inputmap']:
if par['subtract_inputmap_from_sim']:
pair_parallel_track = map_pair.MapPair(map1, map2,
noise_inv1, noise_inv2,
self.freq_list)
if par['subtract_sim_from_inputmap']:
pair_parallel_track = map_pair.MapPair(sim, sim,
noise_inv1, noise_inv2,
self.freq_list)
pair_parallel_track.set_names(pdict['tag1'], pdict['tag2'])
pair_parallel_track.params = self.params
self.pairs_parallel_track[pairitem] = pair_parallel_track
def generate_proc_sim(input_file, weightfile, output_file,
meansub=False, degrade=False):
r"""make the maps with various combinations of beam conv/meansub"""
print "%s -> %s (beam, etc.)" % (input_file, output_file)
simmap = algebra.make_vect(algebra.load(input_file))
if degrade:
print "performing common resolution convolution"
beam_data = sp.array([0.316148488246, 0.306805630985, 0.293729620792,
0.281176247549, 0.270856788455, 0.26745856078,
0.258910010848, 0.249188429031])
freq_data = sp.array([695, 725, 755, 785, 815, 845, 875, 905],
dtype=float)
freq_data *= 1.0e6
beam_diff = sp.sqrt(max(1.1 * beam_data) ** 2 - (beam_data) ** 2)
common_resolution = beam.GaussianBeam(beam_diff, freq_data)
# Convolve to a common resolution.
simmap = common_resolution.apply(simmap)
if meansub:
print "performing mean subtraction"
noise_inv = algebra.make_vect(algebra.load(weightfile))
means = sp.sum(sp.sum(noise_inv * simmap, -1), -1)
means /= sp.sum(sp.sum(noise_inv, -1), -1)
means.shape += (1, 1)
simmap -= means
# the weights will be zero in some places
simmap[noise_inv < 1.e-20] = 0.
# extra sanity?
simmap[np.isinf(simmap)] = 0.
simmap[np.isnan(simmap)] = 0.
print "saving to" + output_file
algebra.save(output_file, simmap)
def generate_windows(window="blackman"):
datapath_db = data_paths.DataPath()
# first generate a window for the full physical volume
filename = datapath_db.fetch('simideal_15hr_physical', intend_read=True,
pick='1')
print filename
pcube = algebra.make_vect(algebra.load(filename))
pwindow = algebra.make_vect(fftutil.window_nd(pcube.shape, name=window),
axis_names=('freq', 'ra', 'dec'))
pwindow.copy_axis_info(pcube)
print pwindow.shape
algebra.save("physical_window.npy", pwindow)
# now generate one for the observed region and project onto the physical
# volume.
filename = datapath_db.fetch('simideal_15hr_beam', intend_read=True,
pick='1')
print filename
ocube = algebra.make_vect(algebra.load(filename))
owindow = algebra.make_vect(fftutil.window_nd(ocube.shape, name=window),
axis_names=('freq', 'ra', 'dec'))
owindow.copy_axis_info(ocube)
print owindow.shape
print owindow.axes
algebra.save("observed_window.npy", owindow)
pwindow = physical_gridding.physical_grid(owindow, refinement=2)
print pwindow.shape
algebra.save("observed_window_physreg.npy", pwindow)
def add_sim_radio():
"""script: go through a list of simulations and add those to a selected map
"""
root_file = "/mnt/raid-project/gmrt/eswitzer/wiggleZ/"
radio_file = root_file + "modetest_combined_maps/combined_41-73_cleaned_clean_15.npy"
root_sim = "/mnt/raid-project/gmrt/calinliv/wiggleZ/simulations/test100/"
root_out = root_file + "simulations_plus_data/"
radio_data = algebra.make_vect(algebra.load(radio_file))
for simindex in range(1,101):
simname = root_sim + "simulated_signal_map_" + \
repr(simindex)+"_with_beam.npy"
filename = root_out + "simulated_signal_plusdata_map_" + \
repr(simindex)+"_with_beam.npy"
simoutname = root_out + "simulated_signal_map_" + \
repr(simindex)+"_with_beam.npy"
sim_data = algebra.make_vect(algebra.load(simname))
sim_data /= 1000.
outmap = copy.deepcopy(radio_data)
outmap += sim_data
algebra.save(filename, outmap)
algebra.save(simoutname, sim_data)
print filename
def plot_difference(filename1, filename2, title, sigmarange=6., sigmacut=None,
transverse=False, outputdir="./", multiplier=1000.,
logscale=False, fractional=False,
ignore=None, diff_filename="./difference.npy"):
"""make movies of the difference of two maps (assuming same dimensions)"""
map1 = algebra.make_vect(algebra.load(filename1))
map2 = algebra.make_vect(algebra.load(filename2))
if fractional:
difftitle = "fractional diff."
dmap = (map1 - map2) / map1 * 100.
else:
difftitle = "difference"
dmap = map1 - map2
algebra.save(diff_filename, dmap)
make_cube_movie(diff_filename,
difftitle, cube_frame_dir, sigmarange=6.,
sigmacut=sigmacut, outputdir=outputdir, ignore=ignore,
multiplier=multiplier, transverse=transverse,
logscale=False)
make_cube_movie(filename1,
title, cube_frame_dir, sigmarange=sigmarange,
sigmacut=sigmacut, outputdir=outputdir, ignore=ignore,
multiplier=multiplier, transverse=transverse,
logscale=logscale, filetag_suffix="_1")
make_cube_movie(filename2,
title, cube_frame_dir, sigmarange=sigmarange,
sigmacut=sigmacut, outputdir=outputdir, ignore=ignore,
multiplier=multiplier, transverse=transverse,
logscale=logscale, filetag_suffix="_2")
def calculate_xspec_file(cube1_file,
cube2_file,
bins,
weight1_file=None,
weight2_file=None,
truncate=False,
window="blackman",
return_3d=False,
unitless=True):
cube1 = algebra.make_vect(algebra.load(cube1_file))
cube2 = algebra.make_vect(algebra.load(cube2_file))
if weight1_file is None:
weight1 = algebra.ones_like(cube1)
else:
weight1 = algebra.make_vect(algebra.load(weight1_file))
if weight2_file is None:
weight2 = algebra.ones_like(cube2)
else:
weight2 = algebra.make_vect(algebra.load(weight2_file))
print cube1.shape, cube2.shape, weight1.shape, weight2.shape
return calculate_xspec(cube1,
cube2,
weight1,
weight2,
bins=bins,
window=window,
unitless=unitless,
truncate=truncate,
return_3d=return_3d)
def load_pairs(self):
r"""load the set of map/noise pairs specified by keys handed to the
database. This sets up operations on the quadratic product
Q = map1^T noise_inv1 B noise_inv2 map2
"""
par = self.params
(self.pairlist,
pairdict) = dp.cross_maps(par['map1'],
par['map2'],
par['noise_inv1'],
par['noise_inv2'],
noise_inv_suffix=";noise_weight",
verbose=False,
tack_on=self.tack_on_input,
db_to_use=self.datapath_db)
for pairitem in self.pairlist:
pdict = pairdict[pairitem]
print "-" * 80
dp.print_dictionary(
pdict,
sys.stdout,
key_list=['map1', 'noise_inv1', 'map2', 'noise_inv2'])
map1 = algebra.make_vect(algebra.load(pdict['map1']))
map2 = algebra.make_vect(algebra.load(pdict['map2']))
if par['simfile'] is not None:
print "adding %s with multiplier %s" % (par['simfile'],
par['sim_multiplier'])
sim = algebra.make_vect(algebra.load(par['simfile']))
sim *= par['sim_multiplier']
print sim.shape, map1.shape
else:
sim = algebra.zeros_like(map1)
noise_inv1 = algebra.make_vect(algebra.load(pdict['noise_inv1']))
noise_inv2 = algebra.make_vect(algebra.load(pdict['noise_inv2']))
pair = map_pair.MapPair(map1 + sim, map2 + sim, noise_inv1,
noise_inv2, self.freq_list)
pair.set_names(pdict['tag1'], pdict['tag2'])
pair.params = self.params
self.pairs[pairitem] = pair
if par['subtract_inputmap_from_sim'] or \
par['subtract_sim_from_inputmap']:
if par['subtract_inputmap_from_sim']:
pair_parallel_track = map_pair.MapPair(
map1, map2, noise_inv1, noise_inv2, self.freq_list)
if par['subtract_sim_from_inputmap']:
pair_parallel_track = map_pair.MapPair(
sim, sim, noise_inv1, noise_inv2, self.freq_list)
pair_parallel_track.set_names(pdict['tag1'], pdict['tag2'])
pair_parallel_track.params = self.params
self.pairs_parallel_track[pairitem] = pair_parallel_track
def realize_simulation(self):
"""do basic handling to call Richard's simulation code
this produces self.sim_map and self.sim_map_phys
"""
if self.scenario == "nostr":
print "running dd+vv and no streaming case"
simobj = corr21cm.Corr21cm.like_kiyo_map(self.template_map)
maps = simobj.get_kiyo_field_physical(refinement=self.refinement)
else:
if self.scenario == "str":
print "running dd+vv and streaming simulation"
simobj = corr21cm.Corr21cm.like_kiyo_map(
self.template_map, sigma_v=self.streaming_dispersion)
maps = simobj.get_kiyo_field_physical(
refinement=self.refinement)
if self.scenario == "ideal":
print "running dd-only and no mean simulation"
simobj = corr21cm.Corr21cm.like_kiyo_map(self.template_map)
maps = simobj.get_kiyo_field_physical(
refinement=self.refinement,
density_only=True,
no_mean=True,
no_evolution=True)
(gbtsim, gbtphys, physdim) = maps
# process the physical-space map
self.sim_map_phys = algebra.make_vect(gbtphys,
axis_names=('freq', 'ra', 'dec'))
pshp = self.sim_map_phys.shape
# define the axes of the physical map; several alternatives are commented
info = {}
info['axes'] = ('freq', 'ra', 'dec')
info['type'] = 'vect'
info['freq_delta'] = abs(physdim[0] - physdim[1]) / float(pshp[0] - 1)
info['freq_centre'] = physdim[0] + info['freq_delta'] * float(
pshp[0] // 2)
# 'freq_centre': abs(physdim[0] + physdim[1]) / 2.,
info['ra_delta'] = abs(physdim[2]) / float(pshp[1] - 1)
#info['ra_centre'] = info['ra_delta'] * float(pshp[1] // 2)
# 'ra_centre': abs(physdim[2]) / 2.,
info['ra_centre'] = 0.
info['dec_delta'] = abs(physdim[3]) / float(pshp[2] - 1)
#info['dec_centre'] = info['dec_delta'] * float(pshp[2] // 2)
# 'dec_centre': abs(physdim[3]) / 2.,
info['dec_centre'] = 0.
self.sim_map_phys.info = info
# process the map in observation coordinates
self.sim_map = algebra.make_vect(gbtsim,
axis_names=('freq', 'ra', 'dec'))
self.sim_map.copy_axis_info(self.template_map)
def find_avg_fsky(map_key, tack_on=None, refinement=2, pad=5, order=1):
"""Take all the pairs that enter the autopower, open their weight files and
find the fsky for each data treatment
In a pipeline
fsky = find_avg_fsky(self.params["map_key"],
tack_on=self.params["tack_on"],
refinement=self.params["refinement"],
pad=self.params["pad"],
order=self.params["order"])
"""
fsky = {}
datapath_db = dp.DataPath()
map_cases = datapath_db.fileset_cases(map_key, "pair;type;treatment")
# This code is essentially verbatim for the permutation in the real
# autopower
unique_pairs = dp.GBTauto_cross_pairs(map_cases['pair'],
map_cases['pair'],
cross_sym="_with_")
treatment_list = map_cases['treatment']
for treatment in treatment_list:
for item in unique_pairs:
dbkeydict = {}
mapset0 = (map_key, item[0], treatment)
mapset1 = (map_key, item[1], treatment)
dbkeydict['noiseinv1_key'] = "%s:%s;noise_inv;%s" % mapset0
dbkeydict['noiseinv2_key'] = "%s:%s;noise_inv;%s" % mapset1
files = dp.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db,
tack_on=tack_on)
print files['noiseinv1_key'], files['noiseinv2_key']
weight1 = algebra.make_vect(algebra.load(files['noiseinv1_key']))
weight2 = algebra.make_vect(algebra.load(files['noiseinv2_key']))
physweight1 = bh.repackage_kiyo(
pg.physical_grid(weight1,
refinement=refinement,
pad=pad,
order=order))
physweight2 = bh.repackage_kiyo(
pg.physical_grid(weight2,
refinement=refinement,
pad=pad,
order=order))
#fsky = np.sum(physweight1 * physweight2)**2
#fsky /= np.sum(physweight1**2 * physweight2**2)
#fsky /= float(physweight1.size)
fsky = np.sum(weight1 * weight2)**2
fsky /= np.sum(weight1**2 * weight2**2)
fsky /= float(weight1.size)
print "volume factor in noise weight: ", fsky
return fsky
def add_manual_mask(source_key, cut_freq_list=None,
signal_name='map', noise_inv_name='noise_inv',
weight_name='weight', divider_token=";"):
r"""
`source_key` is the file db key for the maps to combine
`signal_name` is the tag in the file db entry for the signal maps
`noise_inv_name` is the tag in the file db entry for the N^-1 weights
`weight_name` is the tag in the file db entry for the weights to write out
`divider_token` is the token that divides the map section name
from the data type e.g. "A_with_B;noise_inv"
"""
datapath_db = data_paths.DataPath()
source_fdb = datapath_db.fetch(source_key, silent=True)
source_fdict = source_fdb[1]
# accumulate all the files to combine
noise_inv_keys = {}
weight_keys = {}
signal_keys = {}
for filekey in source_fdb[0]:
if divider_token in filekey:
data_type = filekey.split(divider_token)[1]
map_section = filekey.split(divider_token)[0]
if data_type == signal_name:
signal_keys[map_section] = source_fdict[filekey]
if data_type == noise_inv_name:
noise_inv_keys[map_section] = source_fdict[filekey]
if data_type == weight_name:
weight_keys[map_section] = source_fdict[filekey]
for mapkey in signal_keys:
signal_file = signal_keys[mapkey]
noise_inv_file = noise_inv_keys[mapkey]
weight_file = weight_keys[mapkey]
print "loading pair: %s %s -> %s" % \
(signal_file, noise_inv_file, weight_file)
signal_map = algebra.make_vect(algebra.load(signal_file))
weightmap = algebra.make_vect(algebra.load(noise_inv_file))
# set the new weights to zero where the N^-1 is small
# or the signal map is inf or nan
weightmap[np.isnan(weightmap)] = 0.
weightmap[np.isinf(weightmap)] = 0.
weightmap[np.isnan(signal_map)] = 0.
weightmap[np.isinf(signal_map)] = 0.
weightmap[weightmap < 1.e-20] = 0.
if cut_freq_list is not None:
for cutindex in cut_freq_list:
weightmap[cutindex, :, :] = 0.
# could also determine the filename here, outside of the database
#outputdir = datapath_db.fetch_parent(source_key, return_path=True)
#weight_out = "%s/%s" % (outputdir, source_key)
algebra.compressed_array_summary(weightmap, "new weight map")
algebra.save(weight_file, weightmap)
def map_pair_cal(uncal_maplist, uncal_weightlist, calfactor_outlist,
dirtymap_inlist, dirtymap_outlist,
convolve=True, factorizable_noise=True,
sub_weighted_mean=True, freq_list=range(256)):
map1file = uncal_maplist.pop(0)
weight1file = uncal_weightlist.pop(0)
calfactor_outlist.pop(0)
dirtymap_out0 = dirtymap_outlist.pop(0)
dirtymap_in0 = dirtymap_inlist.pop(0)
# do nothing to the reference map
ref_dirtymap = algebra.make_vect(algebra.load(dirtymap_in0))
algebra.save(dirtymap_out0, ref_dirtymap)
# load maps into pairs
svdout = shelve.open("correlation_pairs.shelve")
for map2file, weight2file, calfactor_outfile, \
dirty_infile, dirty_outfile in zip(uncal_maplist, \
uncal_weightlist, calfactor_outlist,
dirtymap_inlist, dirtymap_outlist):
print map1file, weight1file, map2file, weight2file
pair = map_pair.MapPair(map1file, map2file,
weight1file, weight2file,
freq_list, avoid_db=True)
if factorizable_noise:
pair.make_noise_factorizable()
if sub_weighted_mean:
pair.subtract_weighted_mean()
if convolve:
pair.degrade_resolution()
(corr, counts) = pair.correlate()
svd_info = ce.get_freq_svd_modes(corr, len(freq_list))
svdout[map2file] = svd_info
# write out the left right and cal factors
leftmode = svd_info[1][0]
rightmode = svd_info[2][0]
calfactor = leftmode/rightmode
facout = open(calfactor_outfile, "w")
for outvals in zip(leftmode, rightmode, calfactor):
facout.write("%10.15g %10.15g %10.15g\n" % outvals)
facout.close()
newmap = algebra.make_vect(algebra.load(dirty_infile))
newmap[freq_list, :, :] *= calfactor[:,np.newaxis,np.newaxis]
algebra.save(dirty_outfile, newmap)
print dirty_outfile
svdout.close()
def __init__(self,
map1,
map2,
noise_inv1,
noise_inv2,
freq,
input_filenames=False,
conv_factor=1.1):
r"""
arguments: map1, map2, noise_inv1, noise_inv2, freq
conv_factor is the factor by which to multiply the largest beam
in the convolution to a common resolution
"""
if input_filenames:
self.map1 = algebra.make_vect(algebra.load(map1))
self.map2 = algebra.make_vect(algebra.load(map2))
if noise_inv1:
print "loading noise1 file: " + noise_inv1
self.noise_inv1 = algebra.make_vect(algebra.load(noise_inv1))
else:
print "WARNING: map1 has unity weight; no file given"
self.noise_inv1 = algebra.ones_like(self.map1)
if noise_inv2:
print "loading noise2 file: " + noise_inv2
self.noise_inv2 = algebra.make_vect(algebra.load(noise_inv2))
else:
print "WARNING: map2 has unity weight; no file given"
self.noise_inv2 = algebra.ones_like(self.map2)
else:
self.map1 = map1
self.map2 = map2
self.noise_inv1 = noise_inv1
self.noise_inv2 = noise_inv2
self.freq = freq
self.conv_factor = conv_factor
# maps in physical coordinates (derived)
self.phys_map1 = None
self.phys_map2 = None
self.phys_noise_inv1 = None
self.phys_noise_inv2 = None
# give infinite noise to masked bands
self.sanitize()
# Set attributes.
self.left_modes = 0
self.right_modes = 0
# For saving, to keep track of each mapname.
self.map1_name = ''
self.map2_name = ''
# Which section [A, B, C, D...] the maps is from.
self.map1_code = ''
self.map2_code = ''
def subtract_frequency_modes_slow(self, modes1, modes2=None):
r"""Subtract frequency mode from the map.
Parameters
---------
modes1: list of 1D arrays.
Arrays must be the same length as self.freq. Modes to subtract out
of the map one.
modes2: list of 1D arrays.
Modes to subtract out of map 2. If `None` set to `modes1`.
"""
if modes2 == None:
modes2 = modes1
map1 = self.map1
map2 = self.map2
freq = self.freq
# First map.
outmap_left = sp.empty((len(modes1), ) + map1.shape[1:])
outmap_left = algebra.make_vect(outmap_left,
axis_names=('freq', 'ra', 'dec'))
outmap_left.copy_axis_info(map1)
for ira in range(map1.shape[1]):
for jdec in range(map1.shape[2]):
# if sp.any(map1.data.mask[ira, jdec, freq]):
# continue
# else:
for mode_index, mode_vector in enumerate(modes1):
# v.shape = freq.shape
mode_vector = mode_vector.reshape(freq.shape)
# amp = sp.sum(mode_vector*map1.data[ira, jdec, freq])
amp = sp.dot(mode_vector, map1[freq, ira, jdec])
map1[freq, ira, jdec] -= amp * mode_vector
outmap_left[mode_index, ira, jdec] = amp
self.left_modes = outmap_left
# Second map.
outmap_right = sp.empty((len(modes2), ) + map2.shape[1:])
outmap_right = algebra.make_vect(outmap_right,
axis_names=('freq', 'ra', 'dec'))
outmap_right.copy_axis_info(map2)
for ira in range(map2.shape[1]):
for jdec in range(map2.shape[2]):
# if sp.any(map2.data.mask[ira, jdec, freq]):
# continue
# else:
for mode_index, mode_vector in enumerate(modes2):
# mode_vector.shape = freq.shape
mode_vector = mode_vector.reshape(freq.shape)
amp = sp.dot(mode_vector, map2[freq, ira, jdec])
map2[freq, ira, jdec] -= amp * mode_vector
outmap_right[mode_index, ira, jdec] = amp
self.right_modes = outmap_right
def process_map(self, imap_fname, nmap_fname, ii, mock_fname=None):
params = self.params
sigma = params['sigma']
mu = params['mu']
out_root = params['output_root']
in_root = params['input_root']
imap = algebra.load(in_root + imap_fname)
imap = algebra.make_vect(imap)
#print imap.flatten().mean()
imap = imap - imap.flatten().mean()
if imap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
print ' :: Set Noise to Gaussian'
np.random.seed()
nmap = algebra.info_array(
sigma*np.random.randn(imap.shape[0],imap.shape[1], imap.shape[2])+mu)
nmap.axes = imap.axes
nmap = algebra.make_vect(nmap)
nmap.info = imap.info
if nmap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
## add noise to map ##
imap = imap + nmap
non0 = nmap.nonzero()
nmap[non0] = (1./sigma)**2
#if mock_fname != None:
# mmap = algebra.info_array(
# 2.*np.random.randn(imap.shape[0],imap.shape[1], imap.shape[2])-0.5)
# mmap.axes = imap.axes
# mmap = algebra.make_vect(mmap)
# box, nbox, mbox = self.fill(imap, nmap, mmap)
# pkrm_nfname = out_root + 'fftbox_' + mock_fname
# algebra.save(pkrm_nfname, mbox)
#else:
# box, nbox = self.fill(imap, nmap)
hr = params['hr']
mid = params['mid']
last = params['last']
pol_str = params['polarizations'][0]
end = pol_str
if len(last)!=0:
end = end + last[ii]
end = end + '_' + str(ii)
imap_fname = hr[ii] + mid[0] + end + '.npy'
nmap_fname = hr[ii] + mid[1] + end + '.npy'
pkrm_fname = out_root + imap_fname
algebra.save(pkrm_fname, imap)
pkrm_nfname = out_root + nmap_fname
algebra.save(pkrm_nfname, nmap)
def realize_simulation(self):
"""do basic handling to call Richard's simulation code
this produces self.sim_map and self.sim_map_phys
"""
if self.scenario == "nostr":
print "running dd+vv and no streaming case"
simobj = corr21cm.Corr21cm.like_kiyo_map(self.template_map)
maps = simobj.get_kiyo_field_physical(refinement=self.refinement)
else:
if self.scenario == "str":
print "running dd+vv and streaming simulation"
simobj = corr21cm.Corr21cm.like_kiyo_map(self.template_map,
sigma_v=self.streaming_dispersion)
maps = simobj.get_kiyo_field_physical(refinement=self.refinement)
if self.scenario == "ideal":
print "running dd-only and no mean simulation"
simobj = corr21cm.Corr21cm.like_kiyo_map(self.template_map)
maps = simobj.get_kiyo_field_physical(
refinement=self.refinement,
density_only=True,
no_mean=True,
no_evolution=True)
(gbtsim, gbtphys, physdim) = maps
# process the physical-space map
self.sim_map_phys = algebra.make_vect(gbtphys, axis_names=('freq', 'ra', 'dec'))
pshp = self.sim_map_phys.shape
# define the axes of the physical map; several alternatives are commented
info = {}
info['axes'] = ('freq', 'ra', 'dec')
info['type'] = 'vect'
info['freq_delta'] = abs(physdim[0] - physdim[1]) / float(pshp[0] - 1)
info['freq_centre'] = physdim[0] + info['freq_delta'] * float(pshp[0] // 2)
# 'freq_centre': abs(physdim[0] + physdim[1]) / 2.,
info['ra_delta'] = abs(physdim[2]) / float(pshp[1] - 1)
#info['ra_centre'] = info['ra_delta'] * float(pshp[1] // 2)
# 'ra_centre': abs(physdim[2]) / 2.,
info['ra_centre'] = 0.
info['dec_delta'] = abs(physdim[3]) / float(pshp[2] - 1)
#info['dec_centre'] = info['dec_delta'] * float(pshp[2] // 2)
# 'dec_centre': abs(physdim[3]) / 2.,
info['dec_centre'] = 0.
self.sim_map_phys.info = info
# process the map in observation coordinates
self.sim_map = algebra.make_vect(gbtsim, axis_names=('freq', 'ra', 'dec'))
self.sim_map.copy_axis_info(self.template_map)
def test_scheme(template_file, sim_filename1, sim_filename2):
r"""look at some differences between maps"""
template_map = algebra.make_vect(algebra.load(template_file))
gbtsim1 = realize_simulation(template_map, scenario="streaming", seed=5489, refinement=1.0)
gbtsim2 = realize_simulation(template_map, seed=5489, refinement=1.0)
sim_map1 = algebra.make_vect(gbtsim1, axis_names=("freq", "ra", "dec"))
sim_map2 = algebra.make_vect(gbtsim2, axis_names=("freq", "ra", "dec"))
sim_map1.copy_axis_info(template_map)
sim_map2.copy_axis_info(template_map)
algebra.save(sim_filename1, sim_map1)
algebra.save(sim_filename2, sim_map2)
def divide_iqu_map(source_dict=None, target_dict=None, map_dict=None):
if source_dict != None:
iqu = algebra.make_vect(algebra.load(source_dict['map']))
iqu_weight = algebra.make_vect(algebra.load(source_dict['weight']))
elif map_dict != None:
iqu = algebra.make_vect(map_dict['map'])
iqu_weight = algebra.make_vect(map_dict['weight'])
else:
print "Error: Can not find iqu map"
nfreq = iqu.shape[0]/3
imap = algebra.make_vect(iqu[ 0*nfreq : 1*nfreq, ...])
qmap = algebra.make_vect(iqu[ 1*nfreq : 2*nfreq, ...])
umap = algebra.make_vect(iqu[ 2*nfreq : 3*nfreq, ...])
imap.info = iqu.info
qmap.info = iqu.info
umap.info = iqu.info
imap.copy_axis_info(iqu)
qmap.copy_axis_info(iqu)
umap.copy_axis_info(iqu)
imap_weight = algebra.make_vect(iqu_weight[ 0*nfreq : 1*nfreq, ...])
qmap_weight = algebra.make_vect(iqu_weight[ 1*nfreq : 2*nfreq, ...])
umap_weight = algebra.make_vect(iqu_weight[ 2*nfreq : 3*nfreq, ...])
imap_weight.info = iqu_weight.info
qmap_weight.info = iqu_weight.info
umap_weight.info = iqu_weight.info
imap_weight.copy_axis_info(iqu_weight)
qmap_weight.copy_axis_info(iqu_weight)
umap_weight.copy_axis_info(iqu_weight)
if target_dict != None:
algebra.save(target_dict['imap'], imap)
algebra.save(target_dict['qmap'], qmap)
algebra.save(target_dict['umap'], umap)
algebra.save(target_dict['imap_weight'], imap_weight)
algebra.save(target_dict['qmap_weight'], qmap_weight)
algebra.save(target_dict['umap_weight'], umap_weight)
else:
map_dict = {}
map_dict['imap'] = imap
map_dict['qmap'] = qmap
map_dict['umap'] = umap
map_dict['imap_weight'] = imap_weight
map_dict['qmap_weight'] = qmap_weight
map_dict['umap_weight'] = umap_weight
return map_dict
def getmap(imap_fname, nmap_fname, mmap_fname=None, half=None):
"""
get the matrix of intensity map and noise map
"""
#in_root = params['input_root']
imap = algebra.load(imap_fname)
imap = algebra.make_vect(imap)
if half!=None:
imap = getmap_halfz(imap, half)
#print "--The neam value for imap is:",imap.flatten().mean(),"--"
#imap = imap - imap.flatten().mean()
if imap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
try:
nmap = algebra.load(nmap_fname)
nmap = algebra.make_vect(nmap)
if half!=None:
nmap = getmap_halfz(nmap, half)
bad = nmap<1.e-5*nmap.flatten().max()
nmap[bad] = 0.
non0 = nmap.nonzero()
#imap[non0] = imap[non0]/nmap[non0]
except IOError:
print 'NO Noise File :: Set Noise to One'
nmap = algebra.info_array(sp.ones(imap.shape))
nmap.axes = imap.axes
nmap = algebra.make_vect(nmap)
nmap.info = imap.info
if nmap.axes != ('freq', 'ra', 'dec') :
raise ce.DataError('AXES ERROR!')
if mmap_fname != None:
try:
mmap = algebra.load(mmap_fname)
mmap = algebra.make_vect(mmap)
if half!=None:
mmap = getmap_halfz(mmap, half)
except IOError:
print 'NO Mock File :: Make it!'
mmap = algebra.info_array(
2.*np.random.rand(imap.shape[0],imap.shape[1], imap.shape[2])-0.5)
mmap.axes = imap.axes
mmap = algebra.make_vect(mmap)
return imap, nmap, mmap
else:
return imap, nmap
def __init__(self,
map1,
map2,
noise_inv1,
noise_inv2,
freq,
input_filenames=False):
r"""
arguments: map1, map2, noise_inv1, noise_inv2, freq
"""
if input_filenames:
self.map1 = algebra.make_vect(algebra.load(map1))
self.map2 = algebra.make_vect(algebra.load(map2))
self.noise_inv1 = algebra.make_vect(algebra.load(noise_inv1))
self.noise_inv2 = algebra.make_vect(algebra.load(noise_inv2))
else:
self.map1 = map1
self.map2 = map2
self.noise_inv1 = noise_inv1
self.noise_inv2 = noise_inv2
# older method that uses the database
# self.datapath_db = data_paths.DataPath()
# self.map1 = self.datapath_db.fetch_multi(map1)
# self.map2 = self.datapath_db.fetch_multi(map2)
# self.noise_inv1 = self.datapath_db.fetch_multi(noise_inv1)
# self.noise_inv2 = self.datapath_db.fetch_multi(noise_inv2)
self.freq = freq
# set the physical-dimension maps to None
self.phys_map1 = None
self.phys_map2 = None
self.phys_noise_inv1 = None
self.phys_noise_inv2 = None
# Give infinite noise to unconsidered frequencies
self.sanitize()
# Set attributes.
self.counts = 0
self.modes1 = 0
self.modes2 = 0
self.left_modes = 0
self.right_modes = 0
# For saving, to keep track of each mapname.
self.map1_name = ''
self.map2_name = ''
# Which section [A, B, C, D...] the maps is from.
self.map1_code = ''
self.map2_code = ''
def __init__(self, map1, map2, noise_inv1, noise_inv2, freq,
input_filenames=False, conv_factor=1.1):
r"""
arguments: map1, map2, noise_inv1, noise_inv2, freq
conv_factor is the factor by which to multiply the largest beam
in the convolution to a common resolution
"""
if input_filenames:
self.map1 = algebra.make_vect(algebra.load(map1))
self.map2 = algebra.make_vect(algebra.load(map2))
if noise_inv1:
print "loading noise1 file: " + noise_inv1
self.noise_inv1 = algebra.make_vect(algebra.load(noise_inv1))
else:
print "WARNING: map1 has unity weight; no file given"
self.noise_inv1 = algebra.ones_like(self.map1)
if noise_inv2:
print "loading noise2 file: " + noise_inv2
self.noise_inv2 = algebra.make_vect(algebra.load(noise_inv2))
else:
print "WARNING: map2 has unity weight; no file given"
self.noise_inv2 = algebra.ones_like(self.map2)
else:
self.map1 = map1
self.map2 = map2
self.noise_inv1 = noise_inv1
self.noise_inv2 = noise_inv2
self.freq = freq
self.conv_factor = conv_factor
# maps in physical coordinates (derived)
self.phys_map1 = None
self.phys_map2 = None
self.phys_noise_inv1 = None
self.phys_noise_inv2 = None
# give infinite noise to masked bands
self.sanitize()
# Set attributes.
self.left_modes = 0
self.right_modes = 0
# For saving, to keep track of each mapname.
self.map1_name = ''
self.map2_name = ''
# Which section [A, B, C, D...] the maps is from.
self.map1_code = ''
self.map2_code = ''
def load_pairs(self, regenerate=True):
r"""load the set of map/noise pairs specified by keys handed to the
database. This sets up operations on the quadratic product
Q = map1^T noise_inv1 B noise_inv2 map2
"""
par = self.params
(self.pairlist, pairdict) = dp.cross_maps(par['map1'],
par['map2'],
par['noise_inv1'],
par['noise_inv2'],
verbose=False)
for pairitem in self.pairlist:
pdict = pairdict[pairitem]
print "-" * 80
dp.print_dictionary(
pdict,
sys.stdout,
key_list=['map1', 'noise_inv1', 'map2', 'noise_inv2'])
map1 = algebra.make_vect(algebra.load(pdict['map1']))
map2 = algebra.make_vect(algebra.load(pdict['map2']))
sim = algebra.make_vect(algebra.load(par['simfile']))
if not par['no_weights']:
noise_inv1 = self.process_noise_inv(pdict['noise_inv1'],
regenerate=regenerate)
noise_inv2 = self.process_noise_inv(pdict['noise_inv2'],
regenerate=regenerate)
else:
noise_inv1 = algebra.ones_like(map1)
noise_inv2 = algebra.ones_like(map2)
pair = map_pair.MapPair(map1 + sim, map2 + sim, noise_inv1,
noise_inv2, self.freq_list)
pair.set_names(pdict['tag1'], pdict['tag2'])
pair.lags = self.lags
pair.params = self.params
self.pairs[pairitem] = pair
pair_nosim = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2,
self.freq_list)
pair_nosim.set_names(pdict['tag1'], pdict['tag2'])
pair_nosim.lags = self.lags
pair_nosim.params = self.params
self.pairs_nosim[pairitem] = pair_nosim
def test_scheme(template_file, sim_filename1, sim_filename2):
r"""look at some differences between maps"""
template_map = algebra.make_vect(algebra.load(template_file))
gbtsim1 = realize_simulation(template_map,
scenario='streaming',
seed=5489,
refinement=1.)
gbtsim2 = realize_simulation(template_map, seed=5489, refinement=1.)
sim_map1 = algebra.make_vect(gbtsim1, axis_names=('freq', 'ra', 'dec'))
sim_map2 = algebra.make_vect(gbtsim2, axis_names=('freq', 'ra', 'dec'))
sim_map1.copy_axis_info(template_map)
sim_map2.copy_axis_info(template_map)
algebra.save(sim_filename1, sim_map1)
algebra.save(sim_filename2, sim_map2)
def get_map(self):
map = sp.zeros((self.nf, self.nra, self.ndec))
map = al.make_vect(map, ('freq', 'ra', 'dec'))
map.set_axis_info('freq', 800e6, 1e6)
map.set_axis_info('ra', 21, self.map_size / self.nra)
map.set_axis_info('dec', 0, self.map_size / self.ndec)
return map
def setUp(self):
Reader = fitsGBT.Reader("./testdata/testfile_guppi_combined.fits",
feedback=0)
self.Blocks = Reader.read((), 1)
Data = self.Blocks[0]
Data.calc_freq()
params = {'dm_deweight_time_slope': True}
Maker = dirty_map.DirtyMapMaker(params, feedback=0)
n_chan = Data.dims[-1]
Maker.n_chan = n_chan
Maker.pols = (1, 2, 3, 4)
Maker.pol_ind = 0
Maker.band_centres = (Data.freq[Data.dims[-1] // 2], )
Maker.band_ind = 0
map = sp.zeros((Data.dims[-1], 32, 15))
map = al.make_vect(map, ('freq', 'ra', 'dec'))
map.set_axis_info('freq', Data.freq[Data.dims[-1] // 2],
Data.field['CRVAL1'])
map.set_axis_info('ra', 218, 0.075)
map.set_axis_info('dec', 2, 0.075)
Maker.map = map
self.Maker = Maker
# The variances of each channel.
self.norms = (sp.arange(1., 2., 0.25)[:, None] *
(sp.arange(1., 2., 1. / n_chan)[None, :]))
for Data in self.Blocks:
Data.data[...] = random.randn(*Data.data.shape)
Data.data *= sp.sqrt(self.norms[:, None, :])
Data.data += 50.
def fetch_multi(self, data_obj, db_token="db:", silent=False,
intend_read=True):
r"""Handle various sorts of file pointers/data
if `data_obj`
is an array, return a deep copy of it
is a string:
if it begins with "db:" -- string after db is a db key
otherwise assume it is a file and try to open it
"""
if isinstance(data_obj, str):
if data_obj[0:len(db_token)] == db_token:
db_key = data_obj[len(db_token):]
filename = self.fetch(db_key, intend_read=intend_read,
silent=silent)
else:
filename = data_obj
prefix = "non-db filename "
ft.path_properties(filename, intend_read=intend_read,
is_file=True,
prefix=prefix, silent=silent)
ret_data = algebra.make_vect(algebra.load(filename))
else:
ret_data = copy.deepcopy(data_obj)
return ret_data
def repackage_pickle_as_shelve(pklfile, shelvefile):
"""Take pickled output from Liviu's code and combine the data from various
sources into a common shelve file. [script, not production]
"""
print pklfile
f = open(pklfile, "r")
F = cPickle.load(f)
f.close()
# Setting axis info after pickling.
map_file = F.params["input_root"] + "sec_A_15hr_41-73_clean_map_I.npy"
exMap = algebra.make_vect(algebra.load(map_file))
for Pair in F.Pairs:
Pair.Map1.info = exMap.info
Pair.Map2.info = exMap.info
Pair.Noise_inv1.info = exMap.info
Pair.Noise_inv2.info = exMap.info
for corrindex in range(6):
shelvename = shelvefile + "_" + repr(corrindex) + ".shelve"
corr_shelve = shelve.open(shelvename)
print shelvename
corr_shelve["corr"] = F.Pairs[corrindex].corr
corr_shelve["counts"] = F.Pairs[corrindex].counts
corr_shelve["freq_axis"] = F.Pairs[corrindex].Map1.get_axis("freq")
corr_shelve["params"] = F.params
corr_shelve.close()
def mktmp(rgn_i,rgn_j,rgn_k,srgn_i1,srgn_i2,srgn_j1,srgn_j2,srgn_k1,srgn_k2,outfilename):
"""Write to disk a file representing an empty matrix of given dimensions. Also write an identically
shaped array of booleans, which are true if the index points to the subregion.
rgn_i/j/k : the dimensions of the full region to be simulated
srgn_i/j/k : the dimensions of the deep integration subregion
outfilename: the name of the file to be created
"""
regiontype = np.zeros((rgn_i,rgn_j,rgn_k), bool)
array = np.zeros((rgn_i,rgn_j,rgn_k))
for i in range(0,rgn_i):
for j in range(0,rgn_j):
for k in range(0,rgn_k):
if (i>=(srgn_i1-1) and i<=(srgn_i2-1)):
if (j>=(srgn_j1-1) and j<=(srgn_j2-1)):
if (k>=(srgn_k1-1) and k<=(srgn_k2-1)):
regiontype[i,j,k]=True
else:
regiontype[i,j,k]=False
region=algebra.info_array(array)
regiontypename = 'bool' + outfilename
np.save(regiontypename, regiontype)
algebra.save(outfilename,region)
print "done"
template_map = algebra.make_vect(algebra.load(outfilename))
def execute_assembledir(self):
# link the weights through to the simulation directory
for (weight_file_in, weight_file_out) in \
zip(self.input_weight_maps, self.output_weight_maps):
os.symlink(weight_file_in, weight_file_out)
os.symlink(weight_file_in + ".meta", weight_file_out + ".meta")
signalfile = self.output_root + self.output_signal
signalmap = algebra.make_vect(algebra.load(signalfile))
signalmap *= self.multiplier
# now load the signal simulation add thermal noise and save
for (thermal_file, mapfile) in \
zip(self.output_thermal, self.output_maps):
thermalmap = algebra.make_vect(algebra.load(thermal_file))
algebra.save(mapfile, signalmap + thermalmap)
def ReadMeta(data_path):
"""return freq ra dec"""
data = algebra.make_vect(algebra.load(data_path))
freq = data.get_axis("freq")
ra = data.get_axis("ra")
dec = data.get_axis("dec")
return freq, ra, dec
def process_noise_inv(self, filename, regenerate=True):
r"""buffer reading the noise inverse files for speed and also
save to a file in the intermediate output path.
If the cached file exists as an intermediate product, load it else
produce it.
"""
if filename not in self.noisefiledict:
basename = filename.split("/")[-1].split(".npy")[0]
filename_diag = "%s/%s_diag.npy" % (self.output_root, basename)
exists = os.access(filename_diag, os.F_OK)
if exists and not regenerate:
print "loading pre-diagonalized noise: " + filename_diag
self.noisefiledict[filename] = algebra.make_vect(algebra.load(filename_diag))
else:
print "loading noise: " + filename
# TODO: have this be smarter about reading various noise cov
# inputs
noise_inv = algebra.make_mat(algebra.open_memmap(filename, mode="r"))
self.noisefiledict[filename] = noise_inv.mat_diag()
# self.noisefiledict[filename] = algebra.make_vect(
# algebra.load(filename))
algebra.save(filename_diag, self.noisefiledict[filename])
return copy.deepcopy(self.noisefiledict[filename])
def deactivated_test_extreme_index(self):
"""Set of parameters know to have cased issues in the past with
numerical stability."""
nf = 40
nt = 150
n = nf * nt
dt = 0.26214
BW = 1. / dt / 2.
time_stream = sp.zeros((nf, nt))
time_stream = al.make_vect(time_stream, axis_names=("freq", "time"))
time = dt * (sp.arange(nt) + 50)
N = dirty_map.Noise(time_stream, time)
# Thermal.
thermal = sp.zeros(nf, dtype=float) + 0.0002 * BW * 2.
thermal[22] = dirty_map.T_infinity**2
N.add_thermal(thermal)
# Time mean and slope.
N.deweight_time_mean()
N.deweight_time_slope()
# Extreem index over_f bit.
mode = -sp.ones(nf, dtype=float) / sp.sqrt(nf - 1)
mode[22] = 0
# Parameters measured from one of the data sets. Known to screw things
# up.
#N.add_over_f_freq_mode(8.128e-7, -4.586, 1.0, 1.422e-7, mode, True)
N.add_over_f_freq_mode(0.001729, -0.777, 1.0, 1e-8, mode, True)
#N.orthogonalize_modes()
N.finalize()
# Check if the fast inverse works.
N_mat = N.get_mat()
N_mat.shape = (n, n)
N_inv = N.get_inverse()
N_inv.shape = (n, n)
def deactivated_test_extreme_index(self):
"""Set of parameters know to have cased issues in the past with
numerical stability."""
nf = 40
nt = 150
n = nf * nt
dt = 0.26214
BW = 1. / dt / 2.
time_stream = sp.zeros((nf, nt))
time_stream = al.make_vect(time_stream, axis_names=("freq", "time"))
time = dt * (sp.arange(nt) + 50)
N = dirty_map.Noise(time_stream, time)
# Thermal.
thermal = sp.zeros(nf, dtype=float) + 0.0002 * BW * 2.
thermal[22] = dirty_map.T_infinity**2
N.add_thermal(thermal)
# Time mean and slope.
N.deweight_time_mean()
N.deweight_time_slope()
# Extreem index over_f bit.
mode = -sp.ones(nf, dtype=float) / sp.sqrt(nf - 1)
mode[22] = 0
# Parameters measured from one of the data sets. Known to screw things
# up.
#N.add_over_f_freq_mode(8.128e-7, -4.586, 1.0, 1.422e-7, mode, True)
N.add_over_f_freq_mode(0.001729, -0.777, 1.0, 1e-8, mode, True)
#N.orthogonalize_modes()
N.finalize()
# Check if the fast inverse works.
N_mat = N.get_mat()
N_mat.shape = (n, n)
N_inv = N.get_inverse()
N_inv.shape = (n, n)
def __init__(self, parameter_file=None, params_dict=None, feedback=0):
self.params = params_dict
if parameter_file:
self.params = parse_ini.parse(parameter_file,
params_init,
prefix=prefix)
self.output_file = self.params['output_file']
self.delta_temp_file = self.params['delta_temp_file']
self.total_integration = self.params['total_integration']
self.weight_map = algebra.make_vect(
algebra.load(self.params['weight_file']))
self.max_stdev = self.params['max_stdev']
# set the random seed
if (self.params['seed'] < 0):
# The usual seed is not fine enough for parallel jobs
randsource = open("/dev/random", "rb")
self.seed = struct.unpack("I", randsource.read(4))[0]
#self.seed = abs(long(outfile_physical.__hash__()))
else:
self.seed = self.params['seed']
random.seed(self.seed)
def fetch_multi(self,
data_obj,
db_token="db:",
silent=False,
intend_read=False):
r"""Handle various sorts of file pointers/data
if `data_obj`
is an array, return a deep copy of it
is a string:
if it begins with "db:" -- string after db is a db key
otherwise assume it is a file and try to open it
"""
if isinstance(data_obj, str):
if data_obj[0:len(db_token)] == db_token:
db_key = data_obj[len(db_token):]
filename = self.fetch(db_key,
intend_read=intend_read,
silent=silent)
else:
filename = data_obj
prefix = "non-db filename "
ft.path_properties(filename,
intend_read=intend_read,
is_file=True,
prefix=prefix,
silent=silent)
ret_data = algebra.make_vect(algebra.load(filename))
else:
ret_data = copy.deepcopy(data_obj)
return ret_data
def setUp(self) :
# Read in just to fiugre out the band structure.
this_test_file = 'testdata/testfile_guppi_rotated.fits'
Reader = fitsGBT.Reader(this_test_file, feedback=0)
Blocks = Reader.read((0,),())
bands = ()
for Data in Blocks:
n_chan = Data.dims[3]
Data.calc_freq()
freq = Data.freq
delta = abs(sp.mean(sp.diff(freq)))
centre = freq[n_chan//2]
band = int(centre/1e6)
bands += (band,)
map = sp.zeros((n_chan, 15, 11))
map = algebra.make_vect(map, axis_names=('freq', 'ra', 'dec'))
map.set_axis_info('freq', centre, -delta)
map.set_axis_info('ra', 218, -0.2)
map.set_axis_info('dec', 2, 0.2)
algebra.save('./testout_clean_map_I_' + str(band) + '.npy', map)
self.params = {'sm_input_root' : 'testdata/',
'sm_file_middles' : ("testfile",),
'sm_input_end' : "_guppi_rotated.fits",
'sm_output_root' : "./testout_",
'sm_output_end' : "_sub.fits",
'sm_solve_for_gain' : True,
'sm_gain_output_end' : 'gain.pickle',
'sm_map_input_root' : './testout_',
'sm_map_type' : 'clean_map_',
'sm_map_polarizations' : ('I',),
'sm_map_bands' : bands
}
def setUp(self):
# Read in just to fiugre out the band structure.
this_test_file = 'testdata/testfile_guppi_rotated.fits'
Reader = fitsGBT.Reader(this_test_file, feedback=0)
Blocks = Reader.read((0, ), ())
bands = ()
for Data in Blocks:
n_chan = Data.dims[3]
Data.calc_freq()
freq = Data.freq
delta = abs(sp.mean(sp.diff(freq)))
centre = freq[n_chan // 2]
band = int(centre / 1e6)
bands += (band, )
map = sp.zeros((n_chan, 15, 11))
map = algebra.make_vect(map, axis_names=('freq', 'ra', 'dec'))
map.set_axis_info('freq', centre, -delta)
map.set_axis_info('ra', 218, -0.2)
map.set_axis_info('dec', 2, 0.2)
algebra.save('./testout_clean_map_I_' + str(band) + '.npy', map)
self.params = {
'sm_input_root': 'testdata/',
'sm_file_middles': ("testfile", ),
'sm_input_end': "_guppi_rotated.fits",
'sm_output_root': "./testout_",
'sm_output_end': "_sub.fits",
'sm_solve_for_gain': True,
'sm_gain_output_end': 'gain.pickle',
'sm_map_input_root': './testout_',
'sm_map_type': 'clean_map_',
'sm_map_polarizations': ('I', ),
'sm_map_bands': bands
}
def map_pair_cal(uncal_maplist, uncal_weightlist, calfactor_outlist,
dirtymap_inlist, dirtymap_outlist,
convolve=True, factorizable_noise=False,
sub_weighted_mean=True, freq_list=range(256)):
map1file = reference_clean
weight1file = reference_weight
#map1file = uncal_maplist.pop(0)
#weight1file = uncal_weightlist.pop(0)
#calfactor_outlist.pop(0)
#dirtymap_out0 = dirtymap_outlist.pop(0)
#dirtymap_in0 = dirtymap_inlist.pop(0)
# do nothing to the reference map
#ref_dirtymap = algebra.make_vect(algebra.load(dirtymap_in0))
#algebra.save(dirtymap_out0, ref_dirtymap)
# load maps into pairs
svdout = shelve.open("correlation_pairs_v2.shelve")
for map2file, weight2file, calfactor_outfile, \
dirty_infile, dirty_outfile in zip(uncal_maplist, \
uncal_weightlist, calfactor_outlist,
dirtymap_inlist, dirtymap_outlist):
print map1file, weight1file, map2file, weight2file
pair = map_pair.MapPair(map1file, map2file,
weight1file, weight2file,
freq_list, avoid_db=True)
if factorizable_noise:
pair.make_noise_factorizable()
if sub_weighted_mean:
pair.subtract_weighted_mean()
if convolve:
pair.degrade_resolution()
(corr, counts) = pair.correlate()
svd_info = ce.get_freq_svd_modes(corr, len(freq_list))
svdout[map2file] = svd_info
# write out the left right and cal factors
leftmode = svd_info[1][0]
rightmode = svd_info[2][0]
calfactor = leftmode/rightmode
facout = open(calfactor_outfile, "w")
for outvals in zip(leftmode, rightmode, calfactor):
facout.write("%10.15g %10.15g %10.15g\n" % outvals)
facout.close()
newmap = algebra.make_vect(algebra.load(dirty_infile))
newmap[freq_list, :, :] *= calfactor[:,np.newaxis,np.newaxis]
algebra.save(dirty_outfile, newmap)
print dirty_outfile
svdout.close()
def make_opt_sim(self):
r"""this produces self.sim_map_optsim"""
print "making sim of optically-selected galaxies"
selection_function = \
self.datapath_db.fetch_multi(self.params['selection_file'])
poisson_vect = np.vectorize(np.random.poisson)
print self.sim_map_delta
mean_num_gal = (self.sim_map_delta + 1.) * selection_function
print mean_num_gal
self.sim_map_optsim = poisson_vect(mean_num_gal)
self.sim_map_optsim = \
algebra.make_vect(self.sim_map_optsim.astype(float), axis_names=('freq', 'ra', 'dec'))
if self.params['optcatalog_file']:
optical_catalog = \
self.datapath_db.fetch_multi(self.params['optcatalog_file'])
# convert from delta to N
optical_catalog = (optical_catalog + 1.) * selection_function
print np.sum(optical_catalog), np.sum(self.sim_map_optsim)
self.sim_map_optsim = self.sim_map_optsim / selection_function - 1.
self.sim_map_optsim.copy_axis_info(self.sim_map_delta)
def repackage_pickle_as_shelve(pklfile, shelvefile):
"""Take pickled output from Liviu's code and combine the data from various
sources into a common shelve file. [script, not production]
"""
print pklfile
f = open(pklfile, "r")
F = cPickle.load(f)
f.close()
# Setting axis info after pickling.
map_file = F.params["input_root"] + "sec_A_15hr_41-73_clean_map_I.npy"
exMap = algebra.make_vect(algebra.load(map_file))
for Pair in F.Pairs:
Pair.Map1.info = exMap.info
Pair.Map2.info = exMap.info
Pair.Noise_inv1.info = exMap.info
Pair.Noise_inv2.info = exMap.info
for corrindex in range(6):
shelvename = shelvefile + "_" + repr(corrindex) + ".shelve"
corr_shelve = shelve.open(shelvename)
print shelvename
corr_shelve["corr"] = F.Pairs[corrindex].corr
corr_shelve["counts"] = F.Pairs[corrindex].counts
corr_shelve["freq_axis"] = F.Pairs[corrindex].Map1.get_axis('freq')
corr_shelve["params"] = F.params
corr_shelve.close()
def process_noise_inv(self, filename, regenerate=True):
r"""buffer reading the noise inverse files for speed and also
save to a file in the intermediate output path.
If the cached file exists as an intermediate product, load it else
produce it.
"""
if filename not in self.noisefiledict:
basename = filename.split("/")[-1].split(".npy")[0]
filename_diag = "%s/%s_diag.npy" % \
(self.output_root, basename)
exists = os.access(filename_diag, os.F_OK)
if exists and not regenerate:
print "loading pre-diagonalized noise: " + filename_diag
self.noisefiledict[filename] = algebra.make_vect(
algebra.load(filename_diag))
else:
print "loading noise: " + filename
# TODO: have this be smarter about reading various noise cov
# inputs
noise_inv = algebra.make_mat(
algebra.open_memmap(filename, mode='r'))
self.noisefiledict[filename] = noise_inv.mat_diag()
#self.noisefiledict[filename] = algebra.make_vect(
# algebra.load(filename))
algebra.save(filename_diag, self.noisefiledict[filename])
return copy.deepcopy(self.noisefiledict[filename])
def setUp(self):
# Make a positive definite noise matrix, clean map, and dirty_map.
self.nra = 10
self.ndec = 5
self.nf = 20
self.shape = (self.nf, self.nra, self.ndec)
self.size = self.nra * self.ndec * self.nf
# Clean map.
clean_map = sp.empty(self.shape, dtype=float)
clean_map = al.make_vect(clean_map, axis_names=('freq', 'ra', 'dec'))
clean_map[...] = sp.sin(sp.arange(self.nf))[:, None, None]
clean_map *= sp.cos(sp.arange(self.nra))[:, None]
clean_map *= sp.cos(sp.arange(self.ndec))
# Noise inverse matrix.
noise_inv = sp.empty(self.shape * 2, dtype=float)
noise_inv = al.make_mat(noise_inv,
axis_names=('freq', 'ra', 'dec') * 2,
row_axes=(0, 1, 2),
col_axes=(3, 4, 5))
rand_mat = rand.randn(*((self.size, ) * 2))
information_factor = 1.e6 # K**-2
rand_mat = sp.dot(rand_mat, rand_mat.transpose()) * information_factor
noise_inv.flat[...] = rand_mat.flat
# Dirty map.
dirty_map = al.partial_dot(noise_inv, clean_map)
# Store in self.
self.clean_map = clean_map
self.noise_inv = noise_inv
self.dirty_map = dirty_map
def setUp(self):
Reader = fitsGBT.Reader("./testdata/testfile_guppi_combined.fits",
feedback=0)
self.Blocks = Reader.read((), 1)
Data = self.Blocks[0]
Data.calc_freq()
params = {'dm_deweight_time_slope' : True}
Maker = dirty_map.DirtyMapMaker(params, feedback=0)
n_chan = Data.dims[-1]
Maker.n_chan = n_chan
Maker.pols = (1, 2, 3, 4)
Maker.pol_ind = 0
Maker.band_centres = (Data.freq[Data.dims[-1]//2],)
Maker.band_ind = 0
map = sp.zeros((Data.dims[-1], 32, 15))
map = al.make_vect(map, ('freq', 'ra', 'dec'))
map.set_axis_info('freq', Data.freq[Data.dims[-1]//2],
Data.field['CRVAL1'])
map.set_axis_info('ra', 218, 0.075)
map.set_axis_info('dec', 2, 0.075)
Maker.map = map
self.Maker = Maker
# The variances of each channel.
self.norms = (sp.arange(1., 2., 0.25)[:,None]
* (sp.arange(1., 2., 1./n_chan)[None,:]))
for Data in self.Blocks:
Data.data[...] = random.randn(*Data.data.shape)
Data.data *= sp.sqrt(self.norms[:,None,:])
Data.data += 50.
def setUp(self):
# Make a positive definite noise matrix, clean map, and dirty_map.
self.nra = 10
self.ndec = 5
self.nf = 20
self.shape = (self.nf, self.nra, self.ndec)
self.size = self.nra * self.ndec * self.nf
# Clean map.
clean_map = sp.empty(self.shape, dtype=float)
clean_map = al.make_vect(clean_map, axis_names=('freq', 'ra', 'dec'))
clean_map[...] = sp.sin(sp.arange(self.nf))[:,None,None]
clean_map *= sp.cos(sp.arange(self.nra))[:,None]
clean_map *= sp.cos(sp.arange(self.ndec))
# Noise inverse matrix.
noise_inv = sp.empty(self.shape * 2, dtype=float)
noise_inv = al.make_mat(noise_inv, axis_names=('freq', 'ra', 'dec')*2,
row_axes=(0, 1, 2), col_axes=(3, 4, 5))
rand_mat = rand.randn(*((self.size,) * 2))
information_factor = 1.e6 # K**-2
rand_mat = sp.dot(rand_mat, rand_mat.transpose()) * information_factor
noise_inv.flat[...] = rand_mat.flat
# Dirty map.
dirty_map = al.partial_dot(noise_inv, clean_map)
# Store in self.
self.clean_map = clean_map
self.noise_inv = noise_inv
self.dirty_map = dirty_map
def get_map(self):
map = sp.zeros((self.nf, self.nra, self.ndec))
map = al.make_vect(map, ('freq', 'ra', 'dec'))
map.set_axis_info('freq', 800e6, 1e6)
map.set_axis_info('ra', 21, self.map_size/self.nra)
map.set_axis_info('dec', 0, self.map_size/self.ndec)
return map
def make_opt_sim(self):
r"""this produces self.sim_map_optsim"""
print "making sim of optically-selected galaxies"
selection_function = \
self.datapath_db.fetch_multi(self.params['selection_file'])
poisson_vect = np.vectorize(np.random.poisson)
print self.sim_map_delta
mean_num_gal = (self.sim_map_delta + 1.) * selection_function
print mean_num_gal
self.sim_map_optsim = poisson_vect(mean_num_gal)
self.sim_map_optsim = \
algebra.make_vect(self.sim_map_optsim.astype(float), axis_names=('freq', 'ra', 'dec'))
if self.params['optcatalog_file']:
optical_catalog = \
self.datapath_db.fetch_multi(self.params['optcatalog_file'])
# convert from delta to N
optical_catalog = (optical_catalog + 1.) * selection_function
print np.sum(optical_catalog), np.sum(self.sim_map_optsim)
self.sim_map_optsim = self.sim_map_optsim / selection_function - 1.
self.sim_map_optsim.copy_axis_info(self.sim_map_delta)
def ReadMeta(data_path):
'''return freq ra dec'''
data = algebra.make_vect(algebra.load(data_path))
freq = data.get_axis('freq')
ra = data.get_axis('ra')
dec = data.get_axis('dec')
return freq,ra,dec
def extend_iqu_map(source_dict=None, target_dict=None, map_dict=None):
if source_dict != None:
imap = algebra.make_vect(algebra.load(source_dict['imap']))
qmap = algebra.make_vect(algebra.load(source_dict['qmap']))
umap = algebra.make_vect(algebra.load(source_dict['umap']))
if source_dict.has_key('imap_weight'):
imap_weight = algebra.make_vect(algebra.load(source_dict['imap_weight']))
qmap_weight = algebra.make_vect(algebra.load(source_dict['qmap_weight']))
umap_weight = algebra.make_vect(algebra.load(source_dict['umap_weight']))
elif source_dict.has_key('imap_inv'):
imap_weight, info = find_weight_re_diagnal(source_dict['imap_inv'])
qmap_weight, info = find_weight_re_diagnal(source_dict['qmap_inv'])
umap_weight, info = find_weight_re_diagnal(source_dict['umap_inv'])
else:
print 'Warning: no weight'
imap_weight = algebra.ones_like(imap)
qmap_weight = algebra.ones_like(imap)
umap_weight = algebra.ones_like(imap)
elif map_dict != None:
imap = map_dict['imap']
qmap = map_dict['qmap']
umap = map_dict['umap']
if 'imap_weight' in map_dict.keys():
imap_weight = map_dict['imap_weight']
qmap_weight = map_dict['qmap_weight']
umap_weight = map_dict['umap_weight']
else:
print 'Warning: no weight'
imap_weight = algebra.ones_like(imap)
qmap_weight = algebra.ones_like(imap)
umap_weight = algebra.ones_like(imap)
else:
print "Error: Can not find I Q U maps"
exit()
iqu = algebra.info_array(imap.tolist() + qmap.tolist() + umap.tolist())
iqu = algebra.make_vect(iqu)
iqu.info = imap.info
iqu.copy_axis_info(imap)
iqu_weight = algebra.info_array(imap_weight.tolist() +
qmap_weight.tolist() +
umap_weight.tolist())
iqu_weight = algebra.make_vect(iqu_weight)
iqu_weight.info = imap_weight.info
iqu_weight.copy_axis_info(imap_weight)
if target_dict != None:
algebra.save(target_dict['map'], iqu)
algebra.save(target_dict['weight'], iqu_weight)
else:
map_dict = {}
map_dict['map'] = iqu
map_dict['weight'] = iqu_weight
return map_dict
def produce_delta_map(self, optical_file, optical_selection_file):
map_optical = algebra.make_vect(algebra.load(optical_file))
map_nbar = algebra.make_vect(algebra.load(optical_selection_file))
old_settings = np.seterr(invalid="ignore", under="ignore")
map_delta = map_optical / map_nbar - 1.
np.seterr(**old_settings)
# TODO: also consider setting the nbar to zero outside of galaxies?
map_delta[np.isinf(map_delta)] = 0.
map_delta[np.isnan(map_delta)] = 0.
# if e.g. nbar is zero, then set the point as if there were no galaxies
# downstream, nbar=0 should coincide with zero weight anyway
#map_delta[np.isinf(map_delta)] = -1.
#map_delta[np.isnan(map_delta)] = -1.
return map_delta
def cross_power_est_highmem(arr1, arr2, weight1, weight2,
window="blackman", nonorm=False):
"""Calculate the cross-power spectrum of a two nD fields.
The arrays must be identical and have the same length (physically
and in pixel number) along each axis.
Same goal as above without the emphasis on saving memory.
This is the "tried and true" legacy function.
"""
if window:
window_function = fftutil.window_nd(arr1.shape, name=window)
weight1 *= window_function
weight2 *= window_function
warr1 = arr1 * weight1
warr2 = arr2 * weight2
ndim = arr1.ndim
fft_arr1 = np.fft.fftshift(np.fft.fftn(warr1))
fft_arr2 = np.fft.fftshift(np.fft.fftn(warr2))
xspec = fft_arr1 * fft_arr2.conj()
xspec = xspec.real
# correct for the weighting
product_weight = weight1 * weight2
xspec /= np.sum(product_weight)
# make the axes
k_axes = tuple(["k_" + axis_name for axis_name in arr1.axes])
xspec_arr = algebra.make_vect(xspec, axis_names=k_axes)
info = {'axes': k_axes, 'type': 'vect'}
width = np.zeros(ndim)
for axis_index in range(ndim):
n_axis = arr1.shape[axis_index]
axis_name = arr1.axes[axis_index]
axis_vector = arr1.get_axis(axis_name)
delta_axis = abs(axis_vector[1] - axis_vector[0])
width[axis_index] = delta_axis
k_axis = np.fft.fftshift(np.fft.fftfreq(n_axis, d=delta_axis))
k_axis *= 2. * math.pi
delta_k_axis = abs(k_axis[1] - k_axis[0])
k_name = k_axes[axis_index]
info[k_name + "_delta"] = delta_k_axis
info[k_name + "_centre"] = 0.
#print k_axis
#print k_name, n_axis, delta_axis
xspec_arr.info = info
#print xspec_arr.get_axis("k_dec")
if not nonorm:
xspec_arr *= width.prod()
return xspec_arr
def find_map_region(
min_ra,
max_ra,
min_dec,
max_dec,
target_sample=0.25,
multiplier=16,
search_start=0,
max_freq=700.391,
min_freq=899.609,
n_freq=256,
exact_freq=True,
):
r"""target 0.25 pixels/FWHM"""
# do multiples of 16 so that 256 freq * 16N is a multiple of 4096
ra_sampling = target_sample * 2.0 # initial value that will not fail
n_ra = search_start
while ra_sampling > target_sample:
n_ra += multiplier
ra_sampling = find_map_dimensions(
define_map_region(min_freq, max_freq, min_ra, max_ra, min_dec, max_dec, n_freq, n_ra, 32, exact_freq=True),
silent=True,
)
ra_sampling = ra_sampling[0]
dec_sampling = target_sample * 2.0
n_dec = search_start
while dec_sampling > target_sample:
n_dec += multiplier
dec_sampling = find_map_dimensions(
define_map_region(
min_freq, max_freq, min_ra, max_ra, min_dec, max_dec, n_freq, n_ra, n_dec, exact_freq=True
),
silent=True,
)
dec_sampling = dec_sampling[1]
ra_sample_ratio = target_sample / ra_sampling
dec_sample_ratio = target_sample / dec_sampling
print "n_ra=%d, samp=%g, ratio=%g" % (n_ra, ra_sampling, target_sample / ra_sampling)
print "n_dec=%d, samp=%g, ratio=%g" % (n_dec, dec_sampling, target_sample / dec_sampling)
print "original ra=(%g,%g), dec=(%g,%g)" % (min_ra, max_ra, min_dec, max_dec)
template_map = define_map_region(min_freq, max_freq, min_ra, max_ra, min_dec, max_dec, n_freq, n_ra, n_dec)
# now expand the map a bit so that pixels are exactly 0.25 deg
info = template_map.info
blank = sp.zeros((n_freq, n_ra, n_dec))
info["ra_delta"] *= ra_sample_ratio
info["dec_delta"] *= dec_sample_ratio
map_prod = algebra.make_vect(blank, axis_names=("freq", "ra", "dec"))
map_prod.info = info
return map_prod
def cross_power_est_highmem(arr1, arr2, weight1, weight2,
window="blackman", nonorm=False):
"""Calculate the cross-power spectrum of a two nD fields.
The arrays must be identical and have the same length (physically
and in pixel number) along each axis.
Same goal as above without the emphasis on saving memory.
This is the "tried and true" legacy function.
"""
if window:
window_function = fftutil.window_nd(arr1.shape, name=window)
weight1 *= window_function
weight2 *= window_function
warr1 = arr1 * weight1
warr2 = arr2 * weight2
ndim = arr1.ndim
fft_arr1 = np.fft.fftshift(np.fft.fftn(warr1))
fft_arr2 = np.fft.fftshift(np.fft.fftn(warr2))
xspec = fft_arr1 * fft_arr2.conj()
xspec = xspec.real
# correct for the weighting
product_weight = weight1 * weight2
xspec /= np.sum(product_weight)
# make the axes
k_axes = tuple(["k_" + axis_name for axis_name in arr1.axes])
xspec_arr = algebra.make_vect(xspec, axis_names=k_axes)
info = {'axes': k_axes, 'type': 'vect'}
width = np.zeros(ndim)
for axis_index in range(ndim):
n_axis = arr1.shape[axis_index]
axis_name = arr1.axes[axis_index]
axis_vector = arr1.get_axis(axis_name)
delta_axis = abs(axis_vector[1] - axis_vector[0])
width[axis_index] = delta_axis
k_axis = np.fft.fftshift(np.fft.fftfreq(n_axis, d=delta_axis))
k_axis *= 2. * math.pi
delta_k_axis = abs(k_axis[1] - k_axis[0])
k_name = k_axes[axis_index]
info[k_name + "_delta"] = delta_k_axis
info[k_name + "_centre"] = 0.
#print k_axis
#print k_name, n_axis, delta_axis
xspec_arr.info = info
#print xspec_arr.get_axis("k_dec")
if not nonorm:
xspec_arr *= width.prod()
return xspec_arr
def template_map_axes(filename):
"""Open a numpy array map and extract its axis/etc. information
"""
print "using the volume template file: " + filename
template_map = algebra.make_vect(algebra.load(filename))
freq_axis = template_map.get_axis('freq')
ra_axis = template_map.get_axis('ra')
dec_axis = template_map.get_axis('dec')
return (freq_axis, ra_axis, dec_axis, template_map.shape, template_map)
def add_sim_to_data(simkey, datakey, replace=False):
datapath_db = data_paths.DataPath()
mapA_file = datapath_db.fetch(datakey + ":A;clean_map", intend_read=True)
mapB_file = datapath_db.fetch(datakey + ":B;clean_map", intend_read=True)
mapC_file = datapath_db.fetch(datakey + ":C;clean_map", intend_read=True)
mapD_file = datapath_db.fetch(datakey + ":D;clean_map", intend_read=True)
simfile = datapath_db.fetch(simkey + ":1", intend_read=True)
simmap = algebra.make_vect(algebra.load(simfile))
mapset = [mapA_file, mapB_file, mapC_file, mapD_file]
for mapfile in mapset:
print mapfile, simfile
origmap = algebra.make_vect(algebra.load(mapfile))
if replace:
algebra.save(mapfile, simmap)
else:
algebra.save(mapfile, origmap + simmap)