def generate_proc_sim(input_file, weightfile, output_file,
meansub=False, degrade=False):
r"""make the maps with various combinations of beam conv/meansub"""
print "%s -> %s (beam, etc.)" % (input_file, output_file)
simmap = algebra.make_vect(algebra.load(input_file))
if degrade:
print "performing common resolution convolution"
beam_data = sp.array([0.316148488246, 0.306805630985, 0.293729620792,
0.281176247549, 0.270856788455, 0.26745856078,
0.258910010848, 0.249188429031])
freq_data = sp.array([695, 725, 755, 785, 815, 845, 875, 905],
dtype=float)
freq_data *= 1.0e6
beam_diff = sp.sqrt(max(1.1 * beam_data) ** 2 - (beam_data) ** 2)
common_resolution = beam.GaussianBeam(beam_diff, freq_data)
# Convolve to a common resolution.
simmap = common_resolution.apply(simmap)
if meansub:
print "performing mean subtraction"
noise_inv = algebra.make_vect(algebra.load(weightfile))
means = sp.sum(sp.sum(noise_inv * simmap, -1), -1)
means /= sp.sum(sp.sum(noise_inv, -1), -1)
means.shape += (1, 1)
simmap -= means
# the weights will be zero in some places
simmap[noise_inv < 1.e-20] = 0.
# extra sanity?
simmap[np.isinf(simmap)] = 0.
simmap[np.isnan(simmap)] = 0.
print "saving to" + output_file
algebra.save(output_file, simmap)
def degrade_to_common_res(self):
r"""this produces self.sim_map_degrade"""
print "degrading to common resolution"
# this depends on having simulations with the means subtracted
if self.sim_map_meansub is None:
self.subtract_mean()
beam_diff = np.sqrt(max(1.1 * self.beam_data)**2 - (self.beam_data)**2)
common_resolution = beam.GaussianBeam(beam_diff, self.freq_data)
# Convolve to a common resolution.
self.sim_map_degrade = common_resolution.apply(self.sim_map_meansub)
def degrade_resolution(self, mode="constant"):
r"""Convolves the maps down to the lowest resolution.
Also convolves the noise, making sure to deweight pixels near the edge
as well. Converts noise to factorizable form by averaging.
mode is the ndimage.convolve flag for behavior at the edge
"""
print "degrading the resolution to a common beam: ", self.conv_factor
noise1 = self.noise_inv1
noise2 = self.noise_inv2
# Get the beam data.
beam_data = sp.array([
0.316148488246, 0.306805630985, 0.293729620792, 0.281176247549,
0.270856788455, 0.26745856078, 0.258910010848, 0.249188429031
])
freq_data = sp.array([695, 725, 755, 785, 815, 845, 875, 905],
dtype=float)
freq_data *= 1.0e6
beam_diff = sp.sqrt(
max(self.conv_factor * beam_data)**2 - (beam_data)**2)
common_resolution = beam.GaussianBeam(beam_diff, freq_data)
# Convolve to a common resolution.
self.map2 = common_resolution.apply(self.map2)
self.map1 = common_resolution.apply(self.map1)
# This block of code needs to be split off into a function and applied
# twice (so we are sure to do the same thing to each).
noise1[noise1 < 1.e-30] = 1.e-30
noise1 = 1. / noise1
noise1 = common_resolution.apply(noise1, mode=mode, cval=1.e30)
noise1 = common_resolution.apply(noise1, mode=mode, cval=1.e30)
noise1 = 1. / noise1
noise1[noise1 < 1.e-20] = 0.
noise2[noise2 < 1.e-30] = 1.e-30
noise2 = 1 / noise2
noise2 = common_resolution.apply(noise2, mode=mode, cval=1.e30)
noise2 = common_resolution.apply(noise2, mode=mode, cval=1.e30)
noise2 = 1. / noise2
noise2[noise2 < 1.e-20] = 0.
self.noise_inv1 = algebra.as_alg_like(noise1, self.noise_inv1)
self.noise_inv2 = algebra.as_alg_like(noise2, self.noise_inv2)
def save_and_plot(array, template, filename):
array = algebra.make_vect(array, axis_names=('freq', 'ra', 'dec'))
array.copy_axis_info(template)
beam_data = sp.array([0.316148488246, 0.306805630985, 0.293729620792,
0.281176247549, 0.270856788455, 0.26745856078,
0.258910010848, 0.249188429031])
freq_data = sp.array([695, 725, 755, 785, 815, 845, 875, 905],
dtype=float)
freq_data *= 1.0e6
beamobj = beam.GaussianBeam(beam_data, freq_data)
array_beam = beamobj.apply(array)
algebra.save(filename, array_beam)
outputdir = "/cita/d/www/home/eswitzer/movies/"
pc.make_cube_movie(filename, "Temperature (mK)", pc.cube_frame_dir,
sigmarange=3., outputdir=outputdir, multiplier=1000.,
transverse=False, filetag_suffix="_trial")
def convolve_by_beam(self):
r"""this produces self.sim_map_withbeam"""
print "convolving simulation by beam"
beamobj = beam.GaussianBeam(self.beam_data, self.freq_data)
self.sim_map_withbeam = beamobj.apply(self.sim_map)
rf = cr.realisation(z1, z2, thetax, thetay, nf, nx, ny)[::-1,...]
psnw, kpar, kperp = ps_estimation.ps_azimuth(rf, window = False)
psww, kpar, kperp = ps_estimation.ps_azimuth(rf, window = True)
lag0 = np.cov(rf.reshape((256, 128*128)))
bc = bincov(lag0)
a = algebra.make_vect(rf, axis_names = ('freq', 'ra', 'dec'))
a.set_axis_info('freq', (f1+f2)/2.0, (f1-f2)/nf)
a.set_axis_info('ra', 0.0, thetax / nx)
a.set_axis_info('dec', 0.0, thetay / ny)
b = beam.GaussianBeam(width = [0.25, 0.25*f2/f1], freq = [f2, f1])
ab = b.apply(a)
def wigglez_correlation(init_filename):
"""Perform the cross-correlation between the WiggleZ dataset and GBT
"""
np.set_printoptions(threshold=np.nan)
print "starting wigglez correlation run with file: " + init_filename
params = parse_ini.parse(init_filename, params_default,
prefix=prefix, feedback=10)
optical_file1 = params['optical_root1'] + params['optical_data_file1']
optical_selection_file1 = params['optical_root1'] + \
params['optical_selection_file1']
optical_file2 = params['optical_root2'] + params['optical_data_file2']
optical_selection_file2 = params['optical_root2'] + \
params['optical_selection_file2']
map_opt1 = algebra.make_vect(algebra.load(optical_file1))
map_nbar1 = algebra.make_vect(algebra.load(optical_selection_file1))
map_opt2 = algebra.make_vect(algebra.load(optical_file2))
map_nbar2 = algebra.make_vect(algebra.load(optical_selection_file2))
algebra.compressed_array_summary(map_opt1, "opt map 1 as loaded")
algebra.compressed_array_summary(map_nbar1, "nbar map 1 as loaded")
algebra.compressed_array_summary(map_opt2, "opt map 2 as loaded")
algebra.compressed_array_summary(map_nbar2, "nbar map 2 as loaded")
if params['convolve']:
beam_data = sp.array([0.316148488246, 0.306805630985, 0.293729620792,
0.281176247549, 0.270856788455, 0.26745856078,
0.258910010848, 0.249188429031])
# also consider the beam model where everything is degraded to the
# lowest resolution; e.g. in a MapPair
beam_data_degrade = sp.zeros(8) + max(beam_data)
freq_data = sp.array([695, 725, 755, 785, 815, 845, 875, 905],
dtype=float)
freq_data *= 1.e6
psf = beam.GaussianBeam(beam_data_degrade, freq_data)
# Convolve the optical data to the lowest radio resolution
print "convolving the first map"
map_opt1 = psf.apply(map_opt1)
algebra.compressed_array_summary(map_opt1, "opt 1 after convolution")
print "convolving the second map"
map_opt2 = psf.apply(map_opt2)
algebra.compressed_array_summary(map_opt2, "opt 2 after convolution")
# how should the covariance be convolved?
# nbar is N^-1; convolve B N B^T?,
# or just convolve nbar? use?:
#map_nbar[map_nbar<1.e-30] = 1.e-30
#map_nbar = 1./map_nbar
#map_nbar = psf.apply(map_nbar, cval=1.e30)
#map_nbar = 1./map_nbar
#map_nbar[map_nbar<1.e-20] = 0
print "convolving the first covariance/selection"
map_nbar1 = psf.apply(map_nbar1)
algebra.compressed_array_summary(map_nbar1, "nbar 1 after convolution")
print "convolving the second covariance/selection"
map_nbar2 = psf.apply(map_nbar2)
algebra.compressed_array_summary(map_nbar2, "nbar 2 after convolution")
# convert to delta-overdensity
map_opt1 = map_opt1 / map_nbar1 - 1.
map_opt2 = map_opt2 / map_nbar2 - 1.
#algebra.compressed_array_summary(map_opt1, "opt 1 after conversion to delta")
#algebra.compressed_array_summary(map_opt2, "opt 2 after conversion to delta")
# set the NaNs and infs to zero in data and weights
nan_array = np.isnan(map_opt1)
map_opt1[nan_array] = 0.
map_nbar1[nan_array] = 0.
inf_array = np.isinf(map_opt1)
map_opt1[inf_array] = 0.
map_nbar1[inf_array] = 0.
nan_array = np.isnan(map_opt2)
map_opt2[nan_array] = 0.
map_nbar2[nan_array] = 0.
inf_array = np.isinf(map_opt2)
map_opt2[inf_array] = 0.
map_nbar2[inf_array] = 0.
freqlist = params['freq'] # full: range(map_radio.shape[0])
algebra.compressed_array_summary(map_opt1[freqlist, :, :],
"opt map 1 as entering the correlation function")
algebra.compressed_array_summary(map_nbar1[freqlist, :, :],
"nbar 1 as entering the correlation function")
algebra.compressed_array_summary(map_opt2[freqlist, :, :],
"opt map 2 as entering the correlation function")
algebra.compressed_array_summary(map_nbar2[freqlist, :, :],
"nbar 2 N^-1 as entering the correlation function")
cross_pair = fs.MapPair(map_opt1, map_opt2, map_nbar1, map_nbar2,
freqlist)
if params['subtract_mean']:
cross_pair.subtract_weighted_mean()
(corr, counts) = cross_pair.correlate(params['lags'],
speedup=params['speedup'])
corr_shelve = shelve.open(params['output_shelve_file'])
corr_shelve["corr"] = corr
corr_shelve["counts"] = counts
corr_shelve["freq_axis"] = map_opt1.get_axis('freq')
corr_shelve["params"] = params
corr_shelve.close()
def make_cube_movie(source_key, colorbar_title, frame_dir,
filetag_suffix="", saveslice=None, saveslice_file=None,
outputdir="./", sigmarange=6., ignore=None, multiplier=1.,
transverse=False, title=None, sigmacut=None,
logscale=False, physical=False, convolve=False, tag=None):
"""Make a stack of spatial slice maps and animate them
transverse plots along RA and freq and image plane is in Dec
First mask any points that exceed `sigmacut`, and then report the extent of
`sigmarange` away from the mean
"""
datapath_db = data_paths.DataPath()
cosmology = Cosmology()
littleh = (cosmology.H0 / 100.0)
beam_data = np.array([0.316148488246, 0.306805630985, 0.293729620792,
0.281176247549, 0.270856788455, 0.26745856078,
0.258910010848, 0.249188429031])
freq_data = np.array([695, 725, 755, 785, 815, 845, 875, 905],
dtype=float)
beam_fwhm = interp1d(freq_data, beam_data)
freq_data_Hz = freq_data * 1.0e6
if tag is None:
tag = '_'.join(source_key.split(";"))
tag = '-'.join(tag.split(":"))
# for a given path
#tag = ".".join(source_key.split(".")[:-1]) # extract root name
#tag = tag.split("/")[-1]
if title is None:
title = "%(tag)s (i = %(index)d, "
title += "freq = %(freq)3.1f MHz, z = %(redshift)3.3f, "
title += "Dc=%(distance)3.0f cMpc)"
print tag
fileprefix = frame_dir + tag
nlevels = 500
if transverse:
orientation = "_freqRA"
else:
orientation = "_RADec"
# prepare the data
#cube = algebra.make_vect(algebra.load(source_key)) * multiplier
cube = datapath_db.fetch_multi(source_key) * multiplier
if logscale:
cube[cube < 0] = np.min(np.abs(cube))
cube = np.log10(cube)
isnan = np.isnan(cube)
isinf = np.isinf(cube)
maskarray = ma.mask_or(isnan, isinf)
if ignore is not None:
maskarray = ma.mask_or(maskarray, (cube == ignore))
convolved = ""
if convolve:
convolved = "_convolved"
beamobj = beam.GaussianBeam(beam_data, freq_data_Hz)
cube = beamobj.apply(cube)
if sigmacut:
#np.set_printoptions(threshold=np.nan, precision=4)
deviation = np.abs((cube - np.mean(cube)) / np.std(cube))
extend_maskarray = (cube > (sigmacut * deviation))
maskarray = ma.mask_or(extend_maskarray, maskarray)
mcube = ma.masked_array(cube, mask=maskarray)
try:
whmaskarray = np.where(maskarray)[0]
mask_fraction = float(len(whmaskarray)) / float(cube.size)
except:
mask_fraction = 0.
print "fraction of map clipped: %f" % mask_fraction
(cube_mean, cube_std) = (mcube.mean(), mcube.std())
print "cube mean=%g std=%g" % (cube_mean, cube_std)
try:
len(sigmarange)
color_axis = np.linspace(sigmarange[0], sigmarange[1],
nlevels, endpoint=True)
except TypeError:
if (sigmarange > 0.):
color_axis = np.linspace(cube_mean - sigmarange * cube_std,
cube_mean + sigmarange * cube_std,
nlevels, endpoint=True)
else:
if saveslice is not None:
color_axis = np.linspace(ma.min(mcube[saveslice, :, :]),
ma.max(mcube[saveslice, :, :]),
nlevels, endpoint=True)
else:
color_axis = np.linspace(ma.min(mcube), ma.max(mcube),
nlevels, endpoint=True)
print "using range: [%g, %g]" % (np.min(color_axis), np.max(color_axis))
freq_axis = cube.get_axis('freq')
(ra_axis, dec_axis) = (cube.get_axis('ra'), cube.get_axis('dec'))
runlist = []
# TODO: make transverse work with gnuplot
if transverse:
for decind in range(cube.shape[2]):
fulltitle = tag + " (dec = %3.1f)" % (dec_axis[decind])
runlist.append((decind, cube[:, :, decind], freq_axis,
ra_axis, color_axis, ["Freq", "Ra"], 20.,
fulltitle, colorbar_title, fileprefix, 800.))
else:
for freqind in range(cube.shape[0]):
outfilename = fileprefix + str('.%03d' % freqind) + '.jpeg'
# if in angle, freq coordinates
if not physical:
freq_MHz = freq_axis[freqind] / 1.e6
z_here = cc.freq_21cm_MHz / freq_MHz - 1.
comoving_distance = cosmology.comoving_distance(z_here) / littleh
proper_distance = cosmology.proper_distance(z_here) / littleh
angular_scale = 20. / units.degree / proper_distance
title_info = {"index": freqind,
"redshift": z_here,
"distance": comoving_distance,
"freq": freq_MHz,
"tag": tag}
fulltitle = title % title_info
if (freq_MHz <= freq_data.min()) or \
(freq_MHz >= freq_data.max()):
fwhm = 0.
else:
fwhm = beam_fwhm(freq_MHz)
FWHM_circle = {"primitive": "circle",
"center_x": 0.9,
"center_y": 0.15,
"radius": fwhm / 2.,
"width": 5,
"color": "purple" }
region_scale = {"primitive": "rect",
"center_x": 0.9,
"center_y": 0.15,
"size_x": angular_scale,
"size_y": angular_scale,
"width": 3,
"color": "black" }
draw_objects = [FWHM_circle, region_scale]
runlist.append((outfilename, cube[freqind, :, :], ra_axis,
dec_axis, color_axis, ["RA", "Dec"], 1.,
fulltitle, colorbar_title, draw_objects))
else:
distance = freq_axis[freqind] / 1.e6
fulltitle = "%s (i = %d, Dc = %10.3f cMpc)" % \
(title, freqind, distance)
runlist.append((outfilename, cube[freqind, :, :], ra_axis,
dec_axis, color_axis,
["x (RA, cMpc/h)", "y (Dec, cMpc/h)"], 1.,
fulltitle, colorbar_title, draw_objects))
if saveslice is not None:
print "saving just slice %d" % saveslice
(outfilename, cube_slice, xaxis, yaxis, vaxis, xylabels, \
aspect, fulltitle, cbar_title, draw_objects) = runlist[saveslice]
plot_slice.gnuplot_2D(outfilename, cube_slice, xaxis, yaxis, vaxis, xylabels,
aspect, fulltitle, cbar_title,
eps_outfile=saveslice_file,
draw_objects=draw_objects)
else:
pool = multiprocessing.Pool(processes=(multiprocessing.cpu_count() - 4))
pool.map(gnuplot_single_slice, runlist)
#gnuplot_single_slice(runlist[0]) # for troubleshooting
argument = frame_dir + tag + '.%03d.jpeg'
outfile = outputdir + tag + filetag_suffix + orientation + convolved + '.mp4'
subprocess.check_call(('ffmpeg', '-vb', '2500000', '-r', '10',
'-y', '-i', argument, outfile))
for fileindex in range(len(runlist)):
os.remove(fileprefix + str('.%03d' % fileindex) + '.jpeg')