def calculate_correlation(self):
r"""Note that multiprocessing's map() is more elegant than Process,
but fails for handing in complex map_pair objects
"""
process_list = []
for pairitem in self.pairlist:
filename = self.output_root
filename_svd = filename + "SVD_pair_%s.pkl" % pairitem
map1 = copy.deepcopy(np.array(self.pairs[pairitem].map1))
map2 = copy.deepcopy(np.array(self.pairs[pairitem].map2))
weight1 = copy.deepcopy(np.array(self.pairs[pairitem].noise_inv1))
weight2 = copy.deepcopy(np.array(self.pairs[pairitem].noise_inv2))
freqs1 = copy.deepcopy(self.pairs[pairitem].freq1)
freqs2 = copy.deepcopy(self.pairs[pairitem].freq2)
if self.params['clip_weight_percent'] is not None:
print "Note: your are clipping the weight maps"
percentile = self.params['clip_weight_percent']
mask1 = self.define_weightmask(weight1, percentile=percentile)
mask2 = self.define_weightmask(weight2[:weight1.shape[0], ...],
percentile=percentile)
weight1 = self.saturate_weight(weight1, mask1)
weight2 = self.saturate_weight(weight2, mask1)
(freq_cov, counts) = find_modes.freq_covariance(map1, map2,
weight1, weight2,
freqs1, freqs2,
no_weight=self.params['weighted_SVD'])
n_modes = min(len(freqs1),len(freqs2))
svd_info = find_modes.get_freq_svd_modes(freq_cov, n_modes)
ft.save_pickle(svd_info, filename_svd)
def freq_covariance(self):
return find_modes.freq_covariance(self.map1, self.map2,
self.noise_inv1,
self.noise_inv2,
self.freq, self.freq)
def calculate_correlation(self):
r"""find the covariance in frequency space, take SVD"""
mode_inputdata_out = h5py.File(self.modeinput_filename, "w")
svd_data_out = h5py.File(self.svd_filename, "w")
# define the subdirectories to hold the SVD outputs
cov_out = svd_data_out.create_group("cov")
counts_out = svd_data_out.create_group("cov_counts")
svd_vals1_out = svd_data_out.create_group("svd_vals1")
svd_vals2_out = svd_data_out.create_group("svd_vals2")
svd_modes1_out = svd_data_out.create_group("svd_modes1")
svd_modes2_out = svd_data_out.create_group("svd_modes2")
svd_extmodes1_out = svd_data_out.create_group("svd_extmodes1")
svd_extmodes2_out = svd_data_out.create_group("svd_extmodes2")
# define the subdirectories to hold the input map/weights
mode_inputdata_map1 = mode_inputdata_out.create_group("map1")
mode_inputdata_map2 = mode_inputdata_out.create_group("map2")
mode_inputdata_weight1 = mode_inputdata_out.create_group("weight1")
mode_inputdata_weight2 = mode_inputdata_out.create_group("weight2")
nfreq = len(self.freq_list)
for pairitem in self.pairlist:
print self.indexlist_ext
# start with the base maps and list of good freq indices
map1 = copy.deepcopy(np.array(self.pairs[pairitem].map1))
map2 = copy.deepcopy(np.array(self.pairs[pairitem].map2))
weight1 = copy.deepcopy(np.array(self.pairs[pairitem].noise_inv1))
weight2 = copy.deepcopy(np.array(self.pairs[pairitem].noise_inv2))
freqs = copy.deepcopy(self.pairs[pairitem].freq)
# TODO: does this need to be restricted to the freq_list?
if self.params['clip_weight_percent'] is not None:
print "WARNING: you are clipping the weight maps"
mask1 = self.define_weightmask(weight1,
percentile=self.params['clip_weight_percent'])
mask2 = self.define_weightmask(weight2,
percentile=self.params['clip_weight_percent'])
# now append the extended datasets
for index in self.indexlist_ext:
nfreqind = map1.shape[0]
map1 = np.concatenate([map1,
np.array(self.pairs_ext[index][pairitem].map1)])
map2 = np.concatenate([map2,
np.array(self.pairs_ext[index][pairitem].map2)])
weight1 = np.concatenate([weight1,
np.array(self.pairs_ext[index][pairitem].noise_inv1)])
weight2 = np.concatenate([weight2,
np.array(self.pairs_ext[index][pairitem].noise_inv2)])
freq_arr = np.array(self.pairs_ext[index][pairitem].freq)
freqs = np.concatenate([freqs, freq_arr + nfreqind])
if self.params['clip_weight_percent'] is not None:
weight1 = self.saturate_weight(weight1, mask1)
weight2 = self.saturate_weight(weight2, mask2)
mode_inputdata_map1[pairitem] = map1
mode_inputdata_map2[pairitem] = map2
mode_inputdata_weight1[pairitem] = weight1
mode_inputdata_weight2[pairitem] = weight2
# note that if weighted_SVD, these weights have been applied
# earlier
(freq_cov, counts) = find_modes.freq_covariance(map1, map2,
weight1, weight2,
freqs, freqs,
no_weight=self.params['weighted_SVD'])
cov_out[pairitem] = freq_cov
counts_out[pairitem] = counts
# now find the SVD of this covariance
# note that the choice of 1x4 and 4x1 might be reversed, but this
# should only matter if the extended map (polarizations) are split
# into sections that have correlated noise
svd_info = find_modes.get_freq_svd_modes(freq_cov[0: nfreq, :], nfreq)
svd_vals1_out[pairitem] = svd_info[0]
svd_modes1_out[pairitem] = svd_info[1]
svd_extmodes1_out[pairitem] = svd_info[2]
svd_info = find_modes.get_freq_svd_modes(freq_cov[:, 0: nfreq], nfreq)
svd_vals2_out[pairitem] = svd_info[0]
svd_extmodes2_out[pairitem] = svd_info[1]
svd_modes2_out[pairitem] = svd_info[2]
mode_inputdata_out.close()
svd_data_out.close()
def calculate_correlation(self):
r"""find the covariance in frequency space, take SVD"""
mode_inputdata_out = h5py.File(self.modeinput_filename, "w")
svd_data_out = h5py.File(self.svd_filename, "w")
# define the subdirectories to hold the SVD outputs
cov_out = svd_data_out.create_group("cov")
counts_out = svd_data_out.create_group("cov_counts")
svd_vals1_out = svd_data_out.create_group("svd_vals1")
svd_vals2_out = svd_data_out.create_group("svd_vals2")
svd_modes1_out = svd_data_out.create_group("svd_modes1")
svd_modes2_out = svd_data_out.create_group("svd_modes2")
svd_extmodes1_out = svd_data_out.create_group("svd_extmodes1")
svd_extmodes2_out = svd_data_out.create_group("svd_extmodes2")
# define the subdirectories to hold the input map/weights
mode_inputdata_map1 = mode_inputdata_out.create_group("map1")
mode_inputdata_map2 = mode_inputdata_out.create_group("map2")
mode_inputdata_weight1 = mode_inputdata_out.create_group("weight1")
mode_inputdata_weight2 = mode_inputdata_out.create_group("weight2")
nfreq = len(self.freq_list)
for pairitem in self.pairlist:
print self.indexlist_ext
# start with the base maps and list of good freq indices
map1 = copy.deepcopy(np.array(self.pairs[pairitem].map1))
map2 = copy.deepcopy(np.array(self.pairs[pairitem].map2))
weight1 = copy.deepcopy(np.array(self.pairs[pairitem].noise_inv1))
weight2 = copy.deepcopy(np.array(self.pairs[pairitem].noise_inv2))
freqs = copy.deepcopy(self.pairs[pairitem].freq)
# TODO: does this need to be restricted to the freq_list?
if self.params['clip_weight_percent'] is not None:
print "WARNING: you are clipping the weight maps"
mask1 = self.define_weightmask(
weight1, percentile=self.params['clip_weight_percent'])
mask2 = self.define_weightmask(
weight2, percentile=self.params['clip_weight_percent'])
# now append the extended datasets
for index in self.indexlist_ext:
nfreqind = map1.shape[0]
map1 = np.concatenate(
[map1,
np.array(self.pairs_ext[index][pairitem].map1)])
map2 = np.concatenate(
[map2,
np.array(self.pairs_ext[index][pairitem].map2)])
weight1 = np.concatenate([
weight1,
np.array(self.pairs_ext[index][pairitem].noise_inv1)
])
weight2 = np.concatenate([
weight2,
np.array(self.pairs_ext[index][pairitem].noise_inv2)
])
freq_arr = np.array(self.pairs_ext[index][pairitem].freq)
freqs = np.concatenate([freqs, freq_arr + nfreqind])
if self.params['clip_weight_percent'] is not None:
weight1 = self.saturate_weight(weight1, mask1)
weight2 = self.saturate_weight(weight2, mask2)
mode_inputdata_map1[pairitem] = map1
mode_inputdata_map2[pairitem] = map2
mode_inputdata_weight1[pairitem] = weight1
mode_inputdata_weight2[pairitem] = weight2
# note that if weighted_SVD, these weights have been applied
# earlier
(freq_cov, counts) = find_modes.freq_covariance(
map1,
map2,
weight1,
weight2,
freqs,
freqs,
no_weight=self.params['weighted_SVD'])
cov_out[pairitem] = freq_cov
counts_out[pairitem] = counts
# now find the SVD of this covariance
# note that the choice of 1x4 and 4x1 might be reversed, but this
# should only matter if the extended map (polarizations) are split
# into sections that have correlated noise
svd_info = find_modes.get_freq_svd_modes(freq_cov[0:nfreq, :],
nfreq)
svd_vals1_out[pairitem] = svd_info[0]
svd_modes1_out[pairitem] = svd_info[1]
svd_extmodes1_out[pairitem] = svd_info[2]
svd_info = find_modes.get_freq_svd_modes(freq_cov[:, 0:nfreq],
nfreq)
svd_vals2_out[pairitem] = svd_info[0]
svd_extmodes2_out[pairitem] = svd_info[1]
svd_modes2_out[pairitem] = svd_info[2]
mode_inputdata_out.close()
svd_data_out.close()
def freq_covariance(self):
return find_modes.freq_covariance(self.map1, self.map2,
self.noise_inv1, self.noise_inv2,
self.freq, self.freq)