def load_DataFrame(file_name, skip=0, root_file_name=None):
"""
Loads a `pandas.DataFrame` from a text file
with column names in the first line, preceded by ``#``.
Can skip any number of first lines, and thin with some factor.
"""
with open(file_name, "r", encoding="utf-8-sig") as inp:
top_line = inp.readline().strip()
if not top_line.startswith('#'):
# try getdist format chains with .paramnames file
if root_file_name and os.path.exists(root_file_name +
'.paramnames'):
from getdist import ParamNames
from cobaya.conventions import OutPar, derived_par_name_separator
names = ParamNames(root_file_name + '.paramnames').list()
for i, name in enumerate(names):
if name.startswith(OutPar.chi2 +
'_') and not name.startswith(
OutPar.chi2 +
derived_par_name_separator):
names[i] = name.replace(
OutPar.chi2 + '_',
OutPar.chi2 + derived_par_name_separator)
cols = ['weight', 'minuslogpost'] + names
inp.seek(0)
else:
raise LoggedError(
log, "Input sample file does not have header: %s",
file_name)
else:
cols = [a.strip() for a in top_line.lstrip("#").split()]
if 0 < skip < 1:
# turn into #lines (need to know total line number)
n = sum(1 for _ in inp)
skip = int(round(skip * n)) + 1 # match getdist
inp.seek(0)
data = pd.read_csv(inp,
sep=" ",
header=None,
names=cols,
comment="#",
skipinitialspace=True,
skiprows=skip,
index_col=False)
return data
def readIni(self, ini):
self.map_names = ini.split('map_names', default=[])
self.has_map_names = len(self.map_names)
if self.has_map_names:
# e.g. have multiple frequencies for given field measurement
map_fields = ini.split('map_fields')
if len(map_fields) != len(self.map_names):
raise Exception('CMBLikes: number of map_fields does not match map_names')
self.map_fields = [self.typeIndex(f) for f in map_fields]
else:
self.map_names = self.field_names
self.map_fields = np.arange(len(self.map_names), dtype=int)
fields_use = ini.split('fields_use', [])
if len(fields_use):
index_use = [self.typeIndex(f) for f in fields_use]
use_theory_field = [i in index_use for i in range(self.tot_theory_fields)]
else:
if not self.has_map_names:
raise Exception('CMBlikes: must have fields_use or map_names')
use_theory_field = [True] * self.tot_theory_fields
maps_use = ini.split('maps_use', [])
if len(maps_use):
if np.any([not i for i in use_theory_field]):
print('CMBlikes WARNING: maps_use overrides fields_use')
self.use_map = np.zeros(len(self.map_names), dtype=bool)
for j, map_used in enumerate(maps_use):
if map_used in self.map_names:
self.use_map[self.map_names.index(map_used)] = True
else:
raise ValueError('CMBlikes: maps_use item not found - %s' % map_used)
else:
self.use_map = [use_theory_field[self.map_fields[i]] for i in range(len(self.map_names))]
# Bandpowers can depend on more fields than are actually used in likelihood
# e.g. for correcting leakage or other linear corrections
self.require_map = self.use_map[:]
if self.has_map_names:
if ini.hasKey('fields_required'):
raise Exception('CMBLikes: use maps_required not fields_required')
maps_use = ini.split('maps_required', [])
else:
maps_use = ini.split('fields_required', [])
if len(maps_use):
for j, map_used in enumerate(maps_use):
if map_used in self.map_names:
self.require_map[self.map_names.index(map_used)] = True
else:
raise ValueError('CMBlikes: required item not found %s' % map_used)
self.required_theory_field = [False for _ in self.field_names]
for i in range(len(self.map_names)):
if self.require_map[i]:
self.required_theory_field[self.map_fields[i]] = True
self.ncl_used = 0 # set later reading covmat
self.like_approx = ini.string('like_approx', 'gaussian')
self.nmaps = np.count_nonzero(self.use_map)
self.nmaps_required = np.count_nonzero(self.require_map)
self.required_order = np.zeros(self.nmaps_required, dtype=int)
self.map_required_index = -np.ones(len(self.map_names), dtype=int)
ix = 0
for i in range(len(self.map_names)):
if self.require_map[i]:
self.map_required_index[i] = ix
self.required_order[ix] = i
ix += 1
self.map_used_index = -np.ones(len(self.map_names), dtype=int)
ix = 0
self.used_map_order = []
for i, map_name in enumerate(self.map_names):
if self.use_map[i]:
self.map_used_index[i] = ix
self.used_map_order.append(map_name)
ix += 1
self.ncl = (self.nmaps * (self.nmaps + 1)) // 2
self.pcl_lmax = ini.int('cl_lmax')
self.pcl_lmin = ini.int('cl_lmin')
self.binned = ini.bool('binned', True)
if self.binned:
self.nbins = ini.int('nbins')
self.bin_min = ini.int('use_min', 1) - 1
self.bin_max = ini.int('use_max', self.nbins) - 1
self.nbins_used = self.bin_max - self.bin_min + 1 # needed by readBinWindows
self.bins = self.readBinWindows(ini, 'bin_window')
else:
if self.nmaps != self.nmaps_required:
raise Exception('CMBlikes: unbinned likelihood must have nmaps==nmaps_required')
self.nbins = self.pcl_lmax - self.pcl_lmin + 1
if self.like_approx != 'exact':
print('WARNING: Unbinned likelihoods untested in this version')
self.bin_min = ini.int('use_min', self.pcl_lmin)
self.bin_max = ini.int('use_max', self.pcl_lmax)
self.nbins_used = self.bin_max - self.bin_min + 1
self.full_bandpower_headers, self.full_bandpowers, self.bandpowers = \
self.ReadClArr(ini, 'cl_hat', return_full=True)
if self.like_approx == 'HL':
self.cl_fiducial = self.ReadClArr(ini, 'cl_fiducial')
else:
self.cl_fiducial = None
includes_noise = ini.bool('cl_hat_includes_noise', False)
self.cl_noise = None
if self.like_approx != 'gaussian' or includes_noise:
self.cl_noise = self.ReadClArr(ini, 'cl_noise')
if not includes_noise:
self.bandpowers += self.cl_noise
elif self.like_approx == 'gaussian':
self.bandpowers -= self.cl_noise
self.cl_lmax = np.zeros((self.tot_theory_fields, self.tot_theory_fields))
for i in range(self.tot_theory_fields):
if self.required_theory_field[i]: self.cl_lmax[i, i] = self.pcl_lmax
if self.required_theory_field[0] and self.required_theory_field[1]:
self.cl_lmax[1, 0] = self.pcl_lmax
if self.like_approx != 'gaussian':
cl_fiducial_includes_noise = ini.bool('cl_fiducial_includes_noise', False)
self.bandpower_matrix = np.zeros((self.nbins_used, self.nmaps, self.nmaps))
self.noise_matrix = self.bandpower_matrix.copy()
self.fiducial_sqrt_matrix = self.bandpower_matrix.copy()
if self.cl_fiducial is not None and not cl_fiducial_includes_noise:
self.cl_fiducial += self.cl_noise
for b in range(self.nbins_used):
self.elements_to_matrix(self.bandpowers[:, b], self.bandpower_matrix[b, :, :])
if self.cl_noise is not None:
self.elements_to_matrix(self.cl_noise[:, b], self.noise_matrix[b, :, :])
if self.cl_fiducial is not None:
self.elements_to_matrix(self.cl_fiducial[:, b], self.fiducial_sqrt_matrix[b, :, :])
self.fiducial_sqrt_matrix[b, :, :] = sqrtm(self.fiducial_sqrt_matrix[b, :, :])
if self.like_approx == 'exact':
self.fsky = ini.float('fullsky_exact_fksy')
else:
self.cov = self.ReadCovmat(ini)
self.covinv = np.linalg.inv(self.cov)
if 'linear_correction_fiducial_file' in ini.params:
self.fid_correction = self.ReadClArr(ini, 'linear_correction_fiducial')
self.linear_correction = self.readBinWindows(ini, 'linear_correction_bin_window')
else:
self.linear_correction = None
if ini.hasKey('nuisance_params'):
s = ini.relativeFileName('nuisance_params')
self.nuisance_params = ParamNames(s)
if ini.hasKey('calibration_param'):
raise Exception('calibration_param not allowed with nuisance_params')
if ini.hasKey('calibration_paramname'):
self.calibration_param = ini.string('calibration_paramname')
else:
self.calibration_param = None
elif ini.string('calibration_param', ''):
s = ini.relativeFileName('calibration_param')
if not '.paramnames' in s:
raise Exception('calibration_param must be paramnames file unless nuisance_params also specified')
self.nuisance_params = ParamNames(s)
self.calibration_param = self.nuisance_params.list()[0]
else:
self.calibration_param = None
if self.calibration_param is not None:
self.log_calibration_prior = ini.float('log_calibration_prior', -1)
self.aberration_coeff = ini.float('aberration_coeff', 0.0)
self.map_cls = self.init_map_cls(self.nmaps_required, self.required_order)
class DatasetLikelihood(object):
def __init__(self, fname, dataset_params={}, field_names=['T', 'E', 'B', 'P'], map_separator='x'):
self.field_names = field_names
self.tot_theory_fields = len(field_names)
self.map_separator = map_separator
# aberration will be corrected if aberration_coeff is non - zero
self.aberration_coeff = 0.0
self.log_calibration_prior = -1 # if >0 use log prior on calibration parameter
if '.dataset' in fname:
self.loadDataset(fname, dataset_params)
else:
raise Exception('DatasetLikelihood only supports .dataset files')
def typeIndex(self, field):
return self.field_names.index(field)
def PairStringToMapIndices(self, S):
if len(S) == 2:
if self.has_map_names:
raise Exception('CMBlikes: CL names must use MAP1xMAP2 names')
return self.map_names.index(S[0]), self.map_names.index(S[1])
else:
try:
i = S.index(self.map_separator)
except ValueError:
raise ValueError('CMBLikes: invalid spectrum name %s' % S)
return self.map_names.index(S[0:i]), self.map_names.index(S[i + 1:])
def PairStringToUsedMapIndices(self, used_index, S):
i1, i2 = self.PairStringToMapIndices(S)
i1 = used_index[i1]
i2 = used_index[i2]
if i2 > i1:
return i2, i1
else:
return i1, i2
def UseString_to_cols(self, L):
cl_i_j = self.UseString_to_Cl_i_j(L, self.map_used_index)
cols = -np.ones(cl_i_j.shape[1], dtype=int)
for i in range(cl_i_j.shape[1]):
i1, i2 = cl_i_j[:, i]
if i1 == -1 or i2 == -1: continue
ix = 0
for ii in range(self.nmaps):
for jj in range(ii + 1):
if ii == i1 and jj == i2:
cols[i] = ix
ix += 1
return cols
def UseString_to_Cl_i_j(self, S, used_index):
if not isinstance(S, (list, tuple)): S = S.split()
cl_i_j = np.zeros((2, len(S)), dtype=int)
for i, p in enumerate(S):
cl_i_j[:, i] = self.PairStringToUsedMapIndices(used_index, p)
return cl_i_j
def MapPair_to_Theory_i_j(self, order, pair):
i = self.map_fields[order[pair[0]]]
j = self.map_fields[order[pair[1]]]
if i <= j:
return i, j
else:
return j, i
def Cl_used_i_j_name(self, pair):
return self.Cl_i_j_name(self.used_map_order, pair)
def Cl_i_j_name(self, names, pair):
name1 = names[pair[0]]
name2 = names[pair[1]]
if self.has_map_names:
return name1 + self.map_separator + name2
else:
return name1 + name2
def GetColsFromOrder(self, order):
# Converts string Order = TT TE EE XY... or AAAxBBB AAAxCCC BBxCC
# into indices into array of power spectra (and -1 if not present)
cols = np.empty(self.ncl, dtype=int)
cols[:] = -1
names = order.strip().split()
ix = 0
for i in range(self.nmaps):
for j in range(i + 1):
name = self.Cl_used_i_j_name([i, j])
if not name in names and i != j:
name = self.Cl_used_i_j_name([j, i])
if name in names:
if cols[ix] != -1:
raise Exception('GetColsFromOrder: duplicate CL type')
cols[ix] = names.index(name)
ix += 1
return cols
def elements_to_matrix(self, X, M):
ix = 0
for i in range(self.nmaps):
M[i, 0:i] = X[ix:ix + i]
M[0:i, i] = X[ix:ix + i]
ix += i
M[i, i] = X[ix]
ix += 1
def matrix_to_elements(self, M, X):
ix = 0
for i in range(self.nmaps):
X[ix:ix + i + 1] = M[i, 0:i + 1]
ix += i + 1
def ReadClArr(self, ini, file_stem, return_full=False):
# read file of CL or bins (indexed by L)
filename = ini.relativeFileName(file_stem + '_file')
cl = np.zeros((self.ncl, self.nbins_used))
order = ini.string(file_stem + '_order', '')
if not order:
incols = lastTopComment(filename)
if not incols:
raise Exception('No column order given for ' + filename)
else:
incols = 'L ' + order
cols = self.GetColsFromOrder(incols)
data = np.loadtxt(filename)
Ls = data[:, 0].astype(int)
if self.binned: Ls -= 1
for i, L in enumerate(Ls):
if L >= self.bin_min and L <= self.bin_max:
for ix in range(self.ncl):
if cols[ix] != -1:
cl[ix, L - self.bin_min] = data[i, cols[ix]]
if L < self.bin_max:
raise Exception('CMBLikes_ReadClArr: C_l file does not go up to maximum used: %s' % self.bin_max)
if return_full:
return incols.split(), data, cl
else:
return cl
def readBinWindows(self, ini, file_stem):
bins = BinWindows(self.pcl_lmin, self.pcl_lmax, self.nbins_used)
in_cl = ini.split(file_stem + '_in_order')
out_cl = ini.split(file_stem + '_out_order', in_cl)
bins.cols_in = self.UseString_to_Cl_i_j(in_cl, self.map_required_index)
bins.cols_out = self.UseString_to_cols(out_cl)
norder = bins.cols_in.shape[1]
if norder != bins.cols_out.shape[0]:
raise Exception('_in_order and _out_order must have same number of entries')
bins.binning_matrix = np.zeros((norder, self.nbins_used, self.pcl_lmax - self.pcl_lmin + 1))
windows = ini.relativeFileName(file_stem + '_files')
for b in range(self.nbins_used):
window = np.loadtxt(windows % (b + 1 + self.bin_min))
Err = False
for i, L in enumerate(window[:, 0].astype(int)):
if self.pcl_lmin <= L <= self.pcl_lmax:
bins.binning_matrix[:, b, L - self.pcl_lmin] = window[i, 1:]
else:
Err = Err or any(window[i, 1:] != 0)
if Err: print('WARNING: %s %u outside pcl_lmin-cl_max range: %s' % (file_stem, b, windows % (b + 1)))
if ini.hasKey(file_stem + '_fix_cl_file'):
raise Exception('fix_cl_file not implemented yet')
return bins
def init_map_cls(self, nmaps, order):
if nmaps != len(order): raise ValueError('CMBLikes init_map_cls: size mismatch')
class CrossPowerSpectrum(object):
pass
cls = np.empty((nmaps, nmaps), dtype=object)
for i in range(nmaps):
for j in range(i + 1):
CL = CrossPowerSpectrum()
cls[i, j] = CL
CL.map_ij = [order[i], order[j]]
CL.theory_ij = self.MapPair_to_Theory_i_j(order, [i, j])
CL.CL = np.zeros(self.pcl_lmax - self.pcl_lmin + 1)
return cls
def loadDataset(self, froot, dataset_params):
if not '.dataset' in froot: froot += '.dataset'
ini = IniFile(froot)
ini.params.update(dataset_params)
self.readIni(ini)
def readIni(self, ini):
self.map_names = ini.split('map_names', default=[])
self.has_map_names = len(self.map_names)
if self.has_map_names:
# e.g. have multiple frequencies for given field measurement
map_fields = ini.split('map_fields')
if len(map_fields) != len(self.map_names):
raise Exception('CMBLikes: number of map_fields does not match map_names')
self.map_fields = [self.typeIndex(f) for f in map_fields]
else:
self.map_names = self.field_names
self.map_fields = np.arange(len(self.map_names), dtype=int)
fields_use = ini.split('fields_use', [])
if len(fields_use):
index_use = [self.typeIndex(f) for f in fields_use]
use_theory_field = [i in index_use for i in range(self.tot_theory_fields)]
else:
if not self.has_map_names:
raise Exception('CMBlikes: must have fields_use or map_names')
use_theory_field = [True] * self.tot_theory_fields
maps_use = ini.split('maps_use', [])
if len(maps_use):
if np.any([not i for i in use_theory_field]):
print('CMBlikes WARNING: maps_use overrides fields_use')
self.use_map = np.zeros(len(self.map_names), dtype=bool)
for j, map_used in enumerate(maps_use):
if map_used in self.map_names:
self.use_map[self.map_names.index(map_used)] = True
else:
raise ValueError('CMBlikes: maps_use item not found - %s' % map_used)
else:
self.use_map = [use_theory_field[self.map_fields[i]] for i in range(len(self.map_names))]
# Bandpowers can depend on more fields than are actually used in likelihood
# e.g. for correcting leakage or other linear corrections
self.require_map = self.use_map[:]
if self.has_map_names:
if ini.hasKey('fields_required'):
raise Exception('CMBLikes: use maps_required not fields_required')
maps_use = ini.split('maps_required', [])
else:
maps_use = ini.split('fields_required', [])
if len(maps_use):
for j, map_used in enumerate(maps_use):
if map_used in self.map_names:
self.require_map[self.map_names.index(map_used)] = True
else:
raise ValueError('CMBlikes: required item not found %s' % map_used)
self.required_theory_field = [False for _ in self.field_names]
for i in range(len(self.map_names)):
if self.require_map[i]:
self.required_theory_field[self.map_fields[i]] = True
self.ncl_used = 0 # set later reading covmat
self.like_approx = ini.string('like_approx', 'gaussian')
self.nmaps = np.count_nonzero(self.use_map)
self.nmaps_required = np.count_nonzero(self.require_map)
self.required_order = np.zeros(self.nmaps_required, dtype=int)
self.map_required_index = -np.ones(len(self.map_names), dtype=int)
ix = 0
for i in range(len(self.map_names)):
if self.require_map[i]:
self.map_required_index[i] = ix
self.required_order[ix] = i
ix += 1
self.map_used_index = -np.ones(len(self.map_names), dtype=int)
ix = 0
self.used_map_order = []
for i, map_name in enumerate(self.map_names):
if self.use_map[i]:
self.map_used_index[i] = ix
self.used_map_order.append(map_name)
ix += 1
self.ncl = (self.nmaps * (self.nmaps + 1)) // 2
self.pcl_lmax = ini.int('cl_lmax')
self.pcl_lmin = ini.int('cl_lmin')
self.binned = ini.bool('binned', True)
if self.binned:
self.nbins = ini.int('nbins')
self.bin_min = ini.int('use_min', 1) - 1
self.bin_max = ini.int('use_max', self.nbins) - 1
self.nbins_used = self.bin_max - self.bin_min + 1 # needed by readBinWindows
self.bins = self.readBinWindows(ini, 'bin_window')
else:
if self.nmaps != self.nmaps_required:
raise Exception('CMBlikes: unbinned likelihood must have nmaps==nmaps_required')
self.nbins = self.pcl_lmax - self.pcl_lmin + 1
if self.like_approx != 'exact':
print('WARNING: Unbinned likelihoods untested in this version')
self.bin_min = ini.int('use_min', self.pcl_lmin)
self.bin_max = ini.int('use_max', self.pcl_lmax)
self.nbins_used = self.bin_max - self.bin_min + 1
self.full_bandpower_headers, self.full_bandpowers, self.bandpowers = \
self.ReadClArr(ini, 'cl_hat', return_full=True)
if self.like_approx == 'HL':
self.cl_fiducial = self.ReadClArr(ini, 'cl_fiducial')
else:
self.cl_fiducial = None
includes_noise = ini.bool('cl_hat_includes_noise', False)
self.cl_noise = None
if self.like_approx != 'gaussian' or includes_noise:
self.cl_noise = self.ReadClArr(ini, 'cl_noise')
if not includes_noise:
self.bandpowers += self.cl_noise
elif self.like_approx == 'gaussian':
self.bandpowers -= self.cl_noise
self.cl_lmax = np.zeros((self.tot_theory_fields, self.tot_theory_fields))
for i in range(self.tot_theory_fields):
if self.required_theory_field[i]: self.cl_lmax[i, i] = self.pcl_lmax
if self.required_theory_field[0] and self.required_theory_field[1]:
self.cl_lmax[1, 0] = self.pcl_lmax
if self.like_approx != 'gaussian':
cl_fiducial_includes_noise = ini.bool('cl_fiducial_includes_noise', False)
self.bandpower_matrix = np.zeros((self.nbins_used, self.nmaps, self.nmaps))
self.noise_matrix = self.bandpower_matrix.copy()
self.fiducial_sqrt_matrix = self.bandpower_matrix.copy()
if self.cl_fiducial is not None and not cl_fiducial_includes_noise:
self.cl_fiducial += self.cl_noise
for b in range(self.nbins_used):
self.elements_to_matrix(self.bandpowers[:, b], self.bandpower_matrix[b, :, :])
if self.cl_noise is not None:
self.elements_to_matrix(self.cl_noise[:, b], self.noise_matrix[b, :, :])
if self.cl_fiducial is not None:
self.elements_to_matrix(self.cl_fiducial[:, b], self.fiducial_sqrt_matrix[b, :, :])
self.fiducial_sqrt_matrix[b, :, :] = sqrtm(self.fiducial_sqrt_matrix[b, :, :])
if self.like_approx == 'exact':
self.fsky = ini.float('fullsky_exact_fksy')
else:
self.cov = self.ReadCovmat(ini)
self.covinv = np.linalg.inv(self.cov)
if 'linear_correction_fiducial_file' in ini.params:
self.fid_correction = self.ReadClArr(ini, 'linear_correction_fiducial')
self.linear_correction = self.readBinWindows(ini, 'linear_correction_bin_window')
else:
self.linear_correction = None
if ini.hasKey('nuisance_params'):
s = ini.relativeFileName('nuisance_params')
self.nuisance_params = ParamNames(s)
if ini.hasKey('calibration_param'):
raise Exception('calibration_param not allowed with nuisance_params')
if ini.hasKey('calibration_paramname'):
self.calibration_param = ini.string('calibration_paramname')
else:
self.calibration_param = None
elif ini.string('calibration_param', ''):
s = ini.relativeFileName('calibration_param')
if not '.paramnames' in s:
raise Exception('calibration_param must be paramnames file unless nuisance_params also specified')
self.nuisance_params = ParamNames(s)
self.calibration_param = self.nuisance_params.list()[0]
else:
self.calibration_param = None
if self.calibration_param is not None:
self.log_calibration_prior = ini.float('log_calibration_prior', -1)
self.aberration_coeff = ini.float('aberration_coeff', 0.0)
self.map_cls = self.init_map_cls(self.nmaps_required, self.required_order)
def ReadCovmat(self, ini):
# read the covariance matrix, and the array of which CL are in the covariance,
# which then defines which set of bandpowers are used (subject to other restrictions)
covmat_cl = ini.string('covmat_cl', allowEmpty=False)
if ini.string('covmat_format', 'text') != 'text':
raise Exception('Only text oovmat supported in python so far')
self.full_cov = np.loadtxt(ini.relativeFileName('covmat_fiducial'))
covmat_scale = ini.float('covmat_scale', 1.0)
cl_in_index = self.UseString_to_cols(covmat_cl)
self.ncl_used = np.sum(cl_in_index >= 0)
self.cl_used_index = np.zeros(self.ncl_used, dtype=int)
cov_cl_used = np.zeros(self.ncl_used, dtype=int)
ix = 0
for i, index in enumerate(cl_in_index):
if index >= 0:
self.cl_used_index[ix] = index
cov_cl_used[ix] = i
ix += 1
if self.binned:
num_in = len(cl_in_index)
pcov = np.empty((self.nbins_used * self.ncl_used, self.nbins_used * self.ncl_used))
for binx in range(self.nbins_used):
for biny in range(self.nbins_used):
pcov[binx * self.ncl_used: (binx + 1) * self.ncl_used,
biny * self.ncl_used: (biny + 1) * self.ncl_used] = \
covmat_scale * self.full_cov[np.ix_((binx + self.bin_min) * num_in + cov_cl_used,
(biny + self.bin_min) * num_in + cov_cl_used)]
else:
raise Exception('unbinned covariance not implemented')
return pcov
def get_binned_theory(self, ClArray, data_params={}):
# Useful for plotting, not used for likelihood
self.get_theory_map_cls(ClArray, data_params)
return self.get_binned_map_cls(self.map_cls)
def get_full_bandpower_column(self, col_name):
"""
Get columns from the input bandpower file. Note used by likelihood but may be useful for plotting.
:param col_name: name of column, as in the top comment header of the file
:return: column values
"""
return self.full_bandpowers[:, self.full_bandpower_headers.index(col_name)]
def diag_sigma(self):
return np.sqrt(np.diag(self.full_cov))
def plot(self, column='PP', ClArray=None, ls=None, ax=None):
lbin = self.full_bandpowers[:, self.full_bandpower_headers.index('L_av')]
binned_phicl_err = self.diag_sigma()
ax = ax or plt.gca()
bandpowers = self.full_bandpowers[:, self.full_bandpower_headers.index('PP')]
if 'L_min' in self.full_bandpower_headers:
lmin = self.full_bandpowers[:, self.full_bandpower_headers.index('L_min')]
lmax = self.full_bandpowers[:, self.full_bandpower_headers.index('L_max')]
ax.errorbar(lbin, bandpowers, yerr=binned_phicl_err, xerr=[lbin - lmin, lmax - lbin], fmt='o')
else:
ax.errorbar(lbin, bandpowers, yerr=binned_phicl_err, fmt='o')
if ClArray is not None:
if isinstance(ClArray, ClsArray):
i, j = self.MapPair_to_Theory_i_j(range(len(self.map_names)), self.PairStringToMapIndices(column))
ClArray = ClArray.get([i, j])
if ls is None: ls = np.arange(len(ClArray))
ax.plot(ls, ClArray, color='k')
ax.set_xlim([2, ls[-1]])
def get_binned_map_cls(self, Cls, corrections=True):
band = self.bins.bin(Cls)
if self.linear_correction is not None and corrections:
band += self.linear_correction.bin(Cls) - self.fid_correction.T
return band
def get_theory_map_cls(self, ClArray, data_params={}):
for i in range(self.nmaps_required):
for j in range(i + 1):
CL = self.map_cls[i, j]
cls = ClArray.get(CL.theory_ij)
if cls is not None:
CL.CL[:] = cls[self.pcl_lmin:self.pcl_lmax + 1]
else:
CL.CL[:] = 0
self.adapt_theory_for_maps(self.map_cls, data_params)
def adapt_theory_for_maps(self, cls, data_params):
if self.aberration_coeff: self.add_aberration(cls)
self.add_foregrounds(cls, data_params)
if self.calibration_param is not None and self.calibration_param in data_params:
for i in range(self.nmaps_required):
for j in range(i + 1):
CL = cls[i, j]
if CL is not None:
if CL.theory_ij[0] <= 2 and CL.theory_ij[1] <= 2:
CL.CL /= data_params[self.calibration_param] ** 2
def add_foregrounds(self, cls, data_params):
pass
def add_aberration(self, cls):
# adapted from CosmoMC function by Christian Reichardt
ells = np.arange(self.pcl_lmin, self.pcl_lmax + 1)
cl_norm = ells * (ells + 1)
for i in range(self.nmaps_required):
for j in range(i + 1):
CL = cls[i, j]
if CL is not None:
if CL.theory_ij[0] <= 2 and CL.theory_ij[1] <= 2:
# first get Cl instead of Dl
cl_deriv = CL.CL / cl_norm
# second take derivative dCl/dl
cl_deriv[1:-1] = (cl_deriv[2:] - cl_deriv[:-2]) / 2
# handle endpoints approximately
cl_deriv[0] = cl_deriv[1]
cl_deriv[-1] = cl_deriv[-2]
# reapply to Dl's.
# note never took 2pi out, so not putting it back either
cl_deriv *= cl_norm
# also multiply by ell since really wanted ldCl/dl
cl_deriv *= ells
CL.CL += self.aberration_coeff * cl_deriv
def write_likelihood_data(self, filename, data_params={}):
cls = self.init_map_cls(self.nmaps_required, self.required_order)
self.add_foregrounds(cls, data_params)
with open(filename, 'w') as f:
cols = []
for i in range(self.nmaps_required):
for j in range(i + 1):
cols.append(self.Cl_i_j_name(self.map_names, cls[i, j].map_ij))
f.write('# L' + ("%17s " * len(cols)) % tuple(cols) + '\n')
for b in range(self.pcl_lmin, self.pcl_lmax + 1):
c = [b]
for i in range(self.nmaps_required):
for j in range(i + 1):
c.append(cls[i, j].CL[b - self.pcl_lmin])
f.write(("%I5 " + "%17.8e " * len(cols)) % tuple(c))
def transform(self, C, Chat, Cfhalf):
# HL transformation of the matrices
if C.shape[0] == 1:
rat = Chat[0, 0] / C[0, 0]
C[0, 0] = np.sign(rat - 1) * np.sqrt(2 * np.maximum(0, rat - np.log(rat) - 1)) * Cfhalf[0, 0] ** 2
return
diag, U = np.linalg.eigh(C)
rot = U.T.dot(Chat).dot(U)
roots = np.sqrt(diag)
for i, root in enumerate(roots):
rot[i, :] /= root
rot[:, i] /= root
U.dot(rot.dot(U.T), rot)
diag, rot = np.linalg.eigh(rot)
diag = np.sign(diag - 1) * np.sqrt(2 * np.maximum(0, diag - np.log(diag) - 1))
Cfhalf.dot(rot, U)
for i, d in enumerate(diag):
rot[:, i] = U[:, i] * d
rot.dot(U.T, C)
def exact_chi_sq(self, C, Chat, L):
if C.shape[0] == 1:
return (2 * L + 1) * self.fsky * (Chat[0, 0] / C[0, 0] - 1 - np.log(Chat[0, 0] / C[0, 0]))
else:
M = np.linalg.inv(C).dot(Chat)
return (2 * L + 1) * self.fsky * (np.trace(M) - self.nmaps - np.linalg.slogdet(M)[1])
def chi_squared(self, ClArray, data_params={}, return_binned_theory=False):
self.get_theory_map_cls(ClArray, data_params)
C = np.empty((self.nmaps, self.nmaps))
bigX = np.empty(self.nbins_used * self.ncl_used)
vecp = np.empty(self.ncl)
chisq = 0
if self.binned:
binned_theory = self.get_binned_map_cls(self.map_cls)
else:
Cs = np.zeros((self.nbins_used, self.nmaps, self.nmaps))
for i in range(self.nmaps):
for j in range(i + 1):
CL = self.map_cls[i, j]
if CL is not None:
Cs[:, i, j] = CL.CL[self.bin_min - self.pcl_lmin:self.bin_max - self.pcl_lmin + 1]
Cs[:, j, i] = CL.CL[self.bin_min - self.pcl_lmin:self.bin_max - self.pcl_lmin + 1]
for bin in range(self.nbins_used):
if self.binned:
self.elements_to_matrix(binned_theory[bin, :], C)
else:
C[:, :] = Cs[bin, :, :]
if self.cl_noise is not None:
C += self.noise_matrix[bin]
if self.like_approx == 'exact':
chisq += self.exact_chi_sq(C, self.bandpower_matrix[bin], self.bin_min + bin)
continue
elif self.like_approx == 'HL':
self.transform(C, self.bandpower_matrix[bin], self.fiducial_sqrt_matrix[bin])
elif self.like_approx == 'gaussian':
C -= self.bandpower_matrix[bin]
self.matrix_to_elements(C, vecp)
bigX[bin * self.ncl_used:(bin + 1) * self.ncl_used] = vecp[self.cl_used_index]
if self.like_approx != 'exact':
chisq = fast_chi_squared(self.covinv, bigX)
if self.log_calibration_prior > 0:
chisq += (np.log(data_params[self.calibration_param]) / self.log_calibration_prior) ** 2
if return_binned_theory and self.binned:
return chisq, binned_theory
else:
return chisq
def init_params(self, ini):
self.field_names = getattr(self, 'field_names', ['T', 'E', 'B', 'P'])
self.tot_theory_fields = len(self.field_names)
self.map_names = ini.split('map_names', default=[])
self.has_map_names = bool(self.map_names)
if self.has_map_names:
# e.g. have multiple frequencies for given field measurement
map_fields = ini.split('map_fields')
if len(map_fields) != len(self.map_names):
raise LoggedError(
self.log, 'number of map_fields does not match map_names')
self.map_fields = [self.typeIndex(f) for f in map_fields]
else:
self.map_names = self.field_names
self.map_fields = np.arange(len(self.map_names), dtype=int)
fields_use = ini.split('fields_use', [])
if len(fields_use):
index_use = [self.typeIndex(f) for f in fields_use]
use_theory_field = [
i in index_use for i in range(self.tot_theory_fields)
]
else:
if not self.has_map_names:
raise LoggedError(self.log,
'must have fields_use or map_names')
use_theory_field = [True] * self.tot_theory_fields
maps_use = ini.split('maps_use', [])
if len(maps_use):
if np.any(not i for i in use_theory_field):
self.log.warning('maps_use overrides fields_use')
self.use_map = np.zeros(len(self.map_names), dtype=bool)
for j, map_used in enumerate(maps_use):
if map_used in self.map_names:
self.use_map[self.map_names.index(map_used)] = True
else:
raise LoggedError(
self.log, 'maps_use item not found - %s' % map_used)
else:
self.use_map = [
use_theory_field[self.map_fields[i]]
for i in range(len(self.map_names))
]
# Bandpowers can depend on more fields than are actually used in likelihood
# e.g. for correcting leakage or other linear corrections
self.require_map = self.use_map[:]
if self.has_map_names:
if ini.hasKey('fields_required'):
raise LoggedError(self.log,
'use maps_required not fields_required')
maps_use = ini.split('maps_required', [])
else:
maps_use = ini.split('fields_required', [])
if len(maps_use):
for j, map_used in enumerate(maps_use):
if map_used in self.map_names:
self.require_map[self.map_names.index(map_used)] = True
else:
raise LoggedError(self.log,
'required item not found %s' % map_used)
self.required_theory_field = [False for _ in self.field_names]
for i in range(len(self.map_names)):
if self.require_map[i]:
self.required_theory_field[self.map_fields[i]] = True
self.ncl_used = 0 # set later reading covmat
self.like_approx = ini.string('like_approx', 'gaussian')
self.nmaps = np.count_nonzero(self.use_map)
self.nmaps_required = np.count_nonzero(self.require_map)
self.required_order = np.zeros(self.nmaps_required, dtype=int)
self.map_required_index = -np.ones(len(self.map_names), dtype=int)
ix = 0
for i in range(len(self.map_names)):
if self.require_map[i]:
self.map_required_index[i] = ix
self.required_order[ix] = i
ix += 1
self.map_used_index = -np.ones(len(self.map_names), dtype=int)
ix = 0
self.used_map_order = []
for i, map_name in enumerate(self.map_names):
if self.use_map[i]:
self.map_used_index[i] = ix
self.used_map_order.append(map_name)
ix += 1
self.ncl = (self.nmaps * (self.nmaps + 1)) // 2
self.pcl_lmax = ini.int('cl_lmax')
self.pcl_lmin = ini.int('cl_lmin')
self.binned = ini.bool('binned', True)
if self.binned:
self.nbins = ini.int('nbins')
self.bin_min = ini.int('use_min', 1) - 1
self.bin_max = ini.int('use_max', self.nbins) - 1
self.nbins_used = self.bin_max - self.bin_min + 1 # needed by read_bin_windows
self.bins = self.read_bin_windows(ini, 'bin_window')
else:
if self.nmaps != self.nmaps_required:
raise LoggedError(
self.log,
'unbinned likelihood must have nmaps==nmaps_required')
self.nbins = self.pcl_lmax - self.pcl_lmin + 1
if self.like_approx != 'exact':
self.log.warning(
'Unbinned likelihoods untested in this version')
self.bin_min = ini.int('use_min', self.pcl_lmin)
self.bin_max = ini.int('use_max', self.pcl_lmax)
self.nbins_used = self.bin_max - self.bin_min + 1
self.full_bandpower_headers, self.full_bandpowers, self.bandpowers = \
self.read_cl_array(ini, 'cl_hat', return_full=True)
if self.like_approx == 'HL':
self.cl_fiducial = self.read_cl_array(ini, 'cl_fiducial')
else:
self.cl_fiducial = None
includes_noise = ini.bool('cl_hat_includes_noise', False)
self.cl_noise = None
if self.like_approx != 'gaussian' or includes_noise:
self.cl_noise = self.read_cl_array(ini, 'cl_noise')
if not includes_noise:
self.bandpowers += self.cl_noise
elif self.like_approx == 'gaussian':
self.bandpowers -= self.cl_noise
self.cl_lmax = np.zeros(
(self.tot_theory_fields, self.tot_theory_fields))
for i in range(self.tot_theory_fields):
if self.required_theory_field[i]:
self.cl_lmax[i, i] = self.pcl_lmax
if self.required_theory_field[0] and self.required_theory_field[1]:
self.cl_lmax[1, 0] = self.pcl_lmax
if self.like_approx != 'gaussian':
cl_fiducial_includes_noise = ini.bool('cl_fiducial_includes_noise',
False)
else:
cl_fiducial_includes_noise = False
self.bandpower_matrix = np.zeros(
(self.nbins_used, self.nmaps, self.nmaps))
self.noise_matrix = self.bandpower_matrix.copy()
self.fiducial_sqrt_matrix = self.bandpower_matrix.copy()
if self.cl_fiducial is not None and not cl_fiducial_includes_noise:
self.cl_fiducial += self.cl_noise
for b in range(self.nbins_used):
self.elements_to_matrix(self.bandpowers[:, b],
self.bandpower_matrix[b, :, :])
if self.cl_noise is not None:
self.elements_to_matrix(self.cl_noise[:, b],
self.noise_matrix[b, :, :])
if self.cl_fiducial is not None:
self.elements_to_matrix(self.cl_fiducial[:, b],
self.fiducial_sqrt_matrix[b, :, :])
self.fiducial_sqrt_matrix[b, :, :] = (sqrtm(
self.fiducial_sqrt_matrix[b, :, :]))
if self.like_approx == 'exact':
self.fsky = ini.float('fullsky_exact_fksy')
else:
self.cov = self.ReadCovmat(ini)
self.covinv = np.linalg.inv(self.cov)
if 'linear_correction_fiducial_file' in ini.params:
self.fid_correction = self.read_cl_array(
ini, 'linear_correction_fiducial')
self.linear_correction = self.read_bin_windows(
ini, 'linear_correction_bin_window')
else:
self.linear_correction = None
if ini.hasKey('nuisance_params'):
s = ini.relativeFileName('nuisance_params')
self.nuisance_params = ParamNames(s)
if ini.hasKey('calibration_param'):
raise Exception(
'calibration_param not allowed with nuisance_params')
if ini.hasKey('calibration_paramname'):
self.calibration_param = ini.string('calibration_paramname')
else:
self.calibration_param = None
elif ini.string('calibration_param', ''):
s = ini.relativeFileName('calibration_param')
if '.paramnames' not in s:
raise LoggedError(
self.log,
'calibration_param must be paramnames file unless '
'nuisance_params also specified')
self.nuisance_params = ParamNames(s)
self.calibration_param = self.nuisance_params.list()[0]
else:
self.calibration_param = None
if ini.hasKey('log_calibration_prior'):
self.log.warning('log_calibration_prior in .dataset ignored, '
'set separately in .yaml file')
self.aberration_coeff = ini.float('aberration_coeff', 0.0)
self.map_cls = self.init_map_cls(self.nmaps_required,
self.required_order)
class _CMBlikes(_DataSetLikelihood):
# Data type for aggregated chi2 (case sensitive)
type = "CMB"
map_separator: str
lmin: Sequence[int]
lmax: Sequence[int]
lav: Sequence[int]
Ahat: np.ndarray # not used by likelihood
def get_requirements(self):
# State requisites to the theory code
requested_cls = [
self.field_names[i] + self.field_names[j]
for i, j in zip(*np.where(self.cl_lmax != 0))
]
# l_max has to take into account the window function of the lensing
# so we check the computed l_max ("l_max" option) is higher than the requested one
requested_l_max = int(np.max(self.cl_lmax))
if (getattr(self, "l_max", None) or np.inf) < requested_l_max:
raise LoggedError(
self.log, "You are setting a very low l_max. "
"The likelihood value will probably not be correct. "
"Make sure to make 'l_max'>=%d", requested_l_max)
self.l_max = max(requested_l_max, getattr(self, "l_max", 0) or 0)
return {"Cl": {cl: self.l_max for cl in requested_cls}}
def typeIndex(self, field):
return self.field_names.index(field)
def PairStringToMapIndices(self, S):
if len(S) == 2:
if self.has_map_names:
raise LoggedError(self.log,
'CL names must use MAP1xMAP2 names')
return self.map_names.index(S[0]), self.map_names.index(S[1])
else:
try:
i = S.index(self.map_separator)
except ValueError:
raise LoggedError(self.log, 'invalid spectrum name %s' % S)
return self.map_names.index(S[0:i]), self.map_names.index(S[i +
1:])
def PairStringToUsedMapIndices(self, used_index, S):
i1, i2 = self.PairStringToMapIndices(S)
i1 = used_index[i1]
i2 = used_index[i2]
if i2 > i1:
return i2, i1
else:
return i1, i2
def UseString_to_cols(self, L):
cl_i_j = self.UseString_to_Cl_i_j(L, self.map_used_index)
cols = -np.ones(cl_i_j.shape[1], dtype=int)
for i in range(cl_i_j.shape[1]):
i1, i2 = cl_i_j[:, i]
if i1 == -1 or i2 == -1:
continue
ix = 0
for ii in range(self.nmaps):
for jj in range(ii + 1):
if ii == i1 and jj == i2:
cols[i] = ix
ix += 1
return cols
def UseString_to_Cl_i_j(self, S, used_index):
if not isinstance(S, (list, tuple)):
S = S.split()
cl_i_j = np.zeros((2, len(S)), dtype=int)
for i, p in enumerate(S):
cl_i_j[:, i] = self.PairStringToUsedMapIndices(used_index, p)
return cl_i_j
def MapPair_to_Theory_i_j(self, order, pair):
i = self.map_fields[order[pair[0]]]
j = self.map_fields[order[pair[1]]]
if i <= j:
return i, j
else:
return j, i
def Cl_used_i_j_name(self, pair):
return self.Cl_i_j_name(self.used_map_order, pair)
def Cl_i_j_name(self, names, pair):
name1 = names[pair[0]]
name2 = names[pair[1]]
if self.has_map_names:
return name1 + self.map_separator + name2
else:
return name1 + name2
def get_cols_from_order(self, order):
# Converts string Order = TT TE EE XY... or AAAxBBB AAAxCCC BBxCC
# into indices into array of power spectra (and -1 if not present)
cols = np.empty(self.ncl, dtype=int)
cols[:] = -1
names = order.strip().split()
ix = 0
for i in range(self.nmaps):
for j in range(i + 1):
name = self.Cl_used_i_j_name([i, j])
if name not in names and i != j:
name = self.Cl_used_i_j_name([j, i])
if name in names:
if cols[ix] != -1:
raise LoggedError(
self.log, 'get_cols_from_order: duplicate CL type')
cols[ix] = names.index(name)
ix += 1
return cols
def elements_to_matrix(self, X, M):
ix = 0
for i in range(self.nmaps):
M[i, 0:i] = X[ix:ix + i]
M[0:i, i] = X[ix:ix + i]
ix += i
M[i, i] = X[ix]
ix += 1
def matrix_to_elements(self, M, X):
ix = 0
for i in range(self.nmaps):
X[ix:ix + i + 1] = M[i, 0:i + 1]
ix += i + 1
def read_cl_array(self, ini, file_stem, return_full=False):
# read file of CL or bins (indexed by L)
filename = ini.relativeFileName(file_stem + '_file')
cl = np.zeros((self.ncl, self.nbins_used))
order = ini.string(file_stem + '_order', '')
if not order:
incols = last_top_comment(filename)
if not incols:
raise LoggedError(self.log,
'No column order given for ' + filename)
else:
incols = 'L ' + order
cols = self.get_cols_from_order(incols)
data = np.loadtxt(filename)
Ls = data[:, 0].astype(int)
if self.binned:
Ls -= 1
for i, L in enumerate(Ls):
if self.bin_min <= L <= self.bin_max:
for ix in range(self.ncl):
if cols[ix] != -1:
cl[ix, L - self.bin_min] = data[i, cols[ix]]
if Ls[-1] < self.bin_max:
raise LoggedError(
self.log,
'CMBLikes_ReadClArr: C_l file does not go up to maximum used: '
'%s', self.bin_max)
if return_full:
return incols.split(), data, cl
else:
return cl
def read_bin_windows(self, ini, file_stem):
bins = BinWindows(self.pcl_lmin, self.pcl_lmax, self.nbins_used,
self.ncl)
in_cl = ini.split(file_stem + '_in_order')
out_cl = ini.split(file_stem + '_out_order', in_cl)
bins.cols_in = self.UseString_to_Cl_i_j(in_cl, self.map_required_index)
bins.cols_out = self.UseString_to_cols(out_cl)
norder = bins.cols_in.shape[1]
if norder != bins.cols_out.shape[0]:
raise LoggedError(
self.log,
'_in_order and _out_order must have same number of entries')
bins.binning_matrix = np.zeros(
(norder, self.nbins_used, self.pcl_lmax - self.pcl_lmin + 1))
windows = ini.relativeFileName(file_stem + '_files')
for b in range(self.nbins_used):
window = np.loadtxt(windows % (b + 1 + self.bin_min))
err = False
for i, L in enumerate(window[:, 0].astype(int)):
if self.pcl_lmin <= L <= self.pcl_lmax:
bins.binning_matrix[:, b, L - self.pcl_lmin] = window[i,
1:]
else:
err = err or any(window[i, 1:] != 0)
if err:
self.log.warning('%s %u outside pcl_lmin-cl_max range: %s' %
(file_stem, b, windows % (b + 1)))
if ini.hasKey(file_stem + '_fix_cl_file'):
raise LoggedError(self.log, 'fix_cl_file not implemented yet')
return bins
def init_map_cls(self, nmaps, order):
if nmaps != len(order):
raise LoggedError(self.log, 'init_map_cls: size mismatch')
class CrossPowerSpectrum:
pass
cls = np.empty((nmaps, nmaps), dtype=object)
for i in range(nmaps):
for j in range(i + 1):
CL = CrossPowerSpectrum()
cls[i, j] = CL
CL.map_ij = [order[i], order[j]]
CL.theory_ij = self.MapPair_to_Theory_i_j(order, [i, j])
CL.CL = np.zeros(self.pcl_lmax - self.pcl_lmin + 1)
return cls
def init_params(self, ini):
self.field_names = getattr(self, 'field_names', ['T', 'E', 'B', 'P'])
self.tot_theory_fields = len(self.field_names)
self.map_names = ini.split('map_names', default=[])
self.has_map_names = bool(self.map_names)
if self.has_map_names:
# e.g. have multiple frequencies for given field measurement
map_fields = ini.split('map_fields')
if len(map_fields) != len(self.map_names):
raise LoggedError(
self.log, 'number of map_fields does not match map_names')
self.map_fields = [self.typeIndex(f) for f in map_fields]
else:
self.map_names = self.field_names
self.map_fields = np.arange(len(self.map_names), dtype=int)
fields_use = ini.split('fields_use', [])
if len(fields_use):
index_use = [self.typeIndex(f) for f in fields_use]
use_theory_field = [
i in index_use for i in range(self.tot_theory_fields)
]
else:
if not self.has_map_names:
raise LoggedError(self.log,
'must have fields_use or map_names')
use_theory_field = [True] * self.tot_theory_fields
maps_use = ini.split('maps_use', [])
if len(maps_use):
if np.any(not i for i in use_theory_field):
self.log.warning('maps_use overrides fields_use')
self.use_map = np.zeros(len(self.map_names), dtype=bool)
for j, map_used in enumerate(maps_use):
if map_used in self.map_names:
self.use_map[self.map_names.index(map_used)] = True
else:
raise LoggedError(
self.log, 'maps_use item not found - %s' % map_used)
else:
self.use_map = [
use_theory_field[self.map_fields[i]]
for i in range(len(self.map_names))
]
# Bandpowers can depend on more fields than are actually used in likelihood
# e.g. for correcting leakage or other linear corrections
self.require_map = self.use_map[:]
if self.has_map_names:
if ini.hasKey('fields_required'):
raise LoggedError(self.log,
'use maps_required not fields_required')
maps_use = ini.split('maps_required', [])
else:
maps_use = ini.split('fields_required', [])
if len(maps_use):
for j, map_used in enumerate(maps_use):
if map_used in self.map_names:
self.require_map[self.map_names.index(map_used)] = True
else:
raise LoggedError(self.log,
'required item not found %s' % map_used)
self.required_theory_field = [False for _ in self.field_names]
for i in range(len(self.map_names)):
if self.require_map[i]:
self.required_theory_field[self.map_fields[i]] = True
self.ncl_used = 0 # set later reading covmat
self.like_approx = ini.string('like_approx', 'gaussian')
self.nmaps = np.count_nonzero(self.use_map)
self.nmaps_required = np.count_nonzero(self.require_map)
self.required_order = np.zeros(self.nmaps_required, dtype=int)
self.map_required_index = -np.ones(len(self.map_names), dtype=int)
ix = 0
for i in range(len(self.map_names)):
if self.require_map[i]:
self.map_required_index[i] = ix
self.required_order[ix] = i
ix += 1
self.map_used_index = -np.ones(len(self.map_names), dtype=int)
ix = 0
self.used_map_order = []
for i, map_name in enumerate(self.map_names):
if self.use_map[i]:
self.map_used_index[i] = ix
self.used_map_order.append(map_name)
ix += 1
self.ncl = (self.nmaps * (self.nmaps + 1)) // 2
self.pcl_lmax = ini.int('cl_lmax')
self.pcl_lmin = ini.int('cl_lmin')
self.binned = ini.bool('binned', True)
if self.binned:
self.nbins = ini.int('nbins')
self.bin_min = ini.int('use_min', 1) - 1
self.bin_max = ini.int('use_max', self.nbins) - 1
self.nbins_used = self.bin_max - self.bin_min + 1 # needed by read_bin_windows
self.bins = self.read_bin_windows(ini, 'bin_window')
else:
if self.nmaps != self.nmaps_required:
raise LoggedError(
self.log,
'unbinned likelihood must have nmaps==nmaps_required')
self.nbins = self.pcl_lmax - self.pcl_lmin + 1
if self.like_approx != 'exact':
self.log.warning(
'Unbinned likelihoods untested in this version')
self.bin_min = ini.int('use_min', self.pcl_lmin)
self.bin_max = ini.int('use_max', self.pcl_lmax)
self.nbins_used = self.bin_max - self.bin_min + 1
self.full_bandpower_headers, self.full_bandpowers, self.bandpowers = \
self.read_cl_array(ini, 'cl_hat', return_full=True)
if self.like_approx == 'HL':
self.cl_fiducial = self.read_cl_array(ini, 'cl_fiducial')
else:
self.cl_fiducial = None
includes_noise = ini.bool('cl_hat_includes_noise', False)
self.cl_noise = None
if self.like_approx != 'gaussian' or includes_noise:
self.cl_noise = self.read_cl_array(ini, 'cl_noise')
if not includes_noise:
self.bandpowers += self.cl_noise
elif self.like_approx == 'gaussian':
self.bandpowers -= self.cl_noise
self.cl_lmax = np.zeros(
(self.tot_theory_fields, self.tot_theory_fields))
for i in range(self.tot_theory_fields):
if self.required_theory_field[i]:
self.cl_lmax[i, i] = self.pcl_lmax
if self.required_theory_field[0] and self.required_theory_field[1]:
self.cl_lmax[1, 0] = self.pcl_lmax
if self.like_approx != 'gaussian':
cl_fiducial_includes_noise = ini.bool('cl_fiducial_includes_noise',
False)
else:
cl_fiducial_includes_noise = False
self.bandpower_matrix = np.zeros(
(self.nbins_used, self.nmaps, self.nmaps))
self.noise_matrix = self.bandpower_matrix.copy()
self.fiducial_sqrt_matrix = self.bandpower_matrix.copy()
if self.cl_fiducial is not None and not cl_fiducial_includes_noise:
self.cl_fiducial += self.cl_noise
for b in range(self.nbins_used):
self.elements_to_matrix(self.bandpowers[:, b],
self.bandpower_matrix[b, :, :])
if self.cl_noise is not None:
self.elements_to_matrix(self.cl_noise[:, b],
self.noise_matrix[b, :, :])
if self.cl_fiducial is not None:
self.elements_to_matrix(self.cl_fiducial[:, b],
self.fiducial_sqrt_matrix[b, :, :])
self.fiducial_sqrt_matrix[b, :, :] = (sqrtm(
self.fiducial_sqrt_matrix[b, :, :]))
if self.like_approx == 'exact':
self.fsky = ini.float('fullsky_exact_fksy')
else:
self.cov = self.ReadCovmat(ini)
self.covinv = np.linalg.inv(self.cov)
if 'linear_correction_fiducial_file' in ini.params:
self.fid_correction = self.read_cl_array(
ini, 'linear_correction_fiducial')
self.linear_correction = self.read_bin_windows(
ini, 'linear_correction_bin_window')
else:
self.linear_correction = None
if ini.hasKey('nuisance_params'):
s = ini.relativeFileName('nuisance_params')
self.nuisance_params = ParamNames(s)
if ini.hasKey('calibration_param'):
raise Exception(
'calibration_param not allowed with nuisance_params')
if ini.hasKey('calibration_paramname'):
self.calibration_param = ini.string('calibration_paramname')
else:
self.calibration_param = None
elif ini.string('calibration_param', ''):
s = ini.relativeFileName('calibration_param')
if '.paramnames' not in s:
raise LoggedError(
self.log,
'calibration_param must be paramnames file unless '
'nuisance_params also specified')
self.nuisance_params = ParamNames(s)
self.calibration_param = self.nuisance_params.list()[0]
else:
self.calibration_param = None
if ini.hasKey('log_calibration_prior'):
self.log.warning('log_calibration_prior in .dataset ignored, '
'set separately in .yaml file')
self.aberration_coeff = ini.float('aberration_coeff', 0.0)
self.map_cls = self.init_map_cls(self.nmaps_required,
self.required_order)
def ReadCovmat(self, ini):
"""Read the covariance matrix, and the array of which CL are in the covariance,
which then defines which set of bandpowers are used
(subject to other restrictions)."""
covmat_cl = ini.string('covmat_cl', allowEmpty=False)
self.full_cov = np.loadtxt(ini.relativeFileName('covmat_fiducial'))
covmat_scale = ini.float('covmat_scale', 1.0)
cl_in_index = self.UseString_to_cols(covmat_cl)
self.ncl_used = np.sum(cl_in_index >= 0)
self.cl_used_index = np.zeros(self.ncl_used, dtype=int)
cov_cl_used = np.zeros(self.ncl_used, dtype=int)
ix = 0
for i, index in enumerate(cl_in_index):
if index >= 0:
self.cl_used_index[ix] = index
cov_cl_used[ix] = i
ix += 1
if self.binned:
num_in = len(cl_in_index)
pcov = np.empty((self.nbins_used * self.ncl_used,
self.nbins_used * self.ncl_used))
for binx in range(self.nbins_used):
for biny in range(self.nbins_used):
pcov[binx * self.ncl_used:(binx + 1) * self.ncl_used,
biny * self.ncl_used:(biny + 1) *
self.ncl_used] = (covmat_scale * self.full_cov[np.ix_(
(binx + self.bin_min) * num_in + cov_cl_used,
(biny + self.bin_min) * num_in + cov_cl_used)])
else:
raise LoggedError(self.log,
'unbinned covariance not implemented yet')
return pcov
def writeData(self, froot):
np.savetxt(froot + '_cov.dat', self.cov)
# self.saveCl(froot + '_fid_cl.dat', self.fid_cl[:, 1:],
# cols=['TT', 'EE', 'TE', 'PP'])
with open(froot + '_bandpowers.dat', 'w', encoding="utf-8") as f:
f.write("#%4s %5s %5s %8s %12s %10s %7s\n" %
('bin', 'L_min', 'L_max', 'L_av', 'PP', 'Error', 'Ahat'))
for b in range(self.nbins):
f.write("%5u %5u %5u %8.2f %12.5e %10.3e %7.3f\n" %
(b + 1, self.lmin[b], self.lmax[b], self.lav[b],
self.bandpowers[b], np.sqrt(
self.cov[b, b]), self.Ahat[b]))
self.bins.write(froot, 'bin')
if self.linear_correction is not None:
self.linear_correction.write(froot, 'linear_correction_bin')
with open(froot + '_lensing_fiducial_correction',
'w',
encoding="utf-8") as f:
f.write("#%4s %12s \n" % ('bin', 'PP'))
for b in range(self.nbins):
f.write("%5u %12.5e\n" % (b + 1, self.fid_correction[b]))
def diag_sigma(self):
return np.sqrt(np.diag(self.full_cov))
def plot_lensing(self, column='PP', ells=None, units="muK2", ax=None):
if not np.count_nonzero(self.map_cls):
raise LoggedError(
self.log, "No Cl's have been computed yet. "
"Make sure you have evaluated the likelihood.")
try:
Cl_theo = self.provider.get_Cl(ell_factor=True, units=units)
Cl = Cl_theo.get(column.lower())
except KeyError:
raise LoggedError(self.log,
"'%s' spectrum has not been computed." % column)
import matplotlib.pyplot as plt
lbin = self.full_bandpowers[:,
self.full_bandpower_headers.index('L_av')]
binned_phicl_err = self.diag_sigma()
ax = ax or plt.gca()
bandpowers = self.full_bandpowers[:,
self.full_bandpower_headers.
index('PP')]
if 'L_min' in self.full_bandpower_headers:
lmin = self.full_bandpowers[:,
self.full_bandpower_headers.
index('L_min')]
lmax = self.full_bandpowers[:,
self.full_bandpower_headers.
index('L_max')]
ax.errorbar(lbin,
bandpowers,
yerr=binned_phicl_err,
xerr=[lbin - lmin, lmax - lbin],
fmt='o')
else:
ax.errorbar(lbin, bandpowers, yerr=binned_phicl_err, fmt='o')
if ells is not None:
Cl = Cl[ells]
else:
ells = Cl_theo["ell"]
ax.plot(ells, Cl, color='k')
ax.set_xlim([2, ells[-1]])
return ax
def get_binned_map_cls(self, Cls, corrections=True):
band = self.bins.bin(Cls)
if self.linear_correction is not None and corrections:
band += self.linear_correction.bin(Cls) - self.fid_correction.T
return band
def get_theory_map_cls(self, Cls, data_params=None):
for i in range(self.nmaps_required):
for j in range(i + 1):
CL = self.map_cls[i, j]
combination = "".join(
[self.field_names[k] for k in CL.theory_ij]).lower()
cls = Cls.get(combination)
if cls is not None:
CL.CL[:] = cls[self.pcl_lmin:self.pcl_lmax + 1]
else:
CL.CL[:] = 0
self.adapt_theory_for_maps(self.map_cls, data_params or {})
def adapt_theory_for_maps(self, cls, data_params):
if self.aberration_coeff:
self.add_aberration(cls)
self.add_foregrounds(cls, data_params)
if self.calibration_param is not None and self.calibration_param in data_params:
for i in range(self.nmaps_required):
for j in range(i + 1):
CL = cls[i, j]
if CL is not None:
if CL.theory_ij[0] <= 2 and CL.theory_ij[1] <= 2:
CL.CL /= data_params[self.calibration_param]**2
def add_foregrounds(self, cls, data_params):
pass
def add_aberration(self, cls):
# adapted from CosmoMC function by Christian Reichardt
ells = np.arange(self.pcl_lmin, self.pcl_lmax + 1)
cl_norm = ells * (ells + 1)
for i in range(self.nmaps_required):
for j in range(i + 1):
CL = cls[i, j]
if CL is not None:
if CL.theory_ij[0] <= 2 and CL.theory_ij[1] <= 2:
# first get Cl instead of Dl
cl_deriv = CL.CL / cl_norm
# second take derivative dCl/dl
cl_deriv[1:-1] = (cl_deriv[2:] - cl_deriv[:-2]) / 2
# handle endpoints approximately
cl_deriv[0] = cl_deriv[1]
cl_deriv[-1] = cl_deriv[-2]
# reapply to Dl's.
# note never took 2pi out, so not putting it back either
cl_deriv *= cl_norm
# also multiply by ell since really wanted ldCl/dl
cl_deriv *= ells
CL.CL += self.aberration_coeff * cl_deriv
def write_likelihood_data(self, filename, data_params=None):
cls = self.init_map_cls(self.nmaps_required, self.required_order)
self.add_foregrounds(cls, data_params or {})
with open(filename, 'w', encoding="utf-8") as f:
cols = []
for i in range(self.nmaps_required):
for j in range(i + 1):
cols.append(
self.Cl_i_j_name(self.map_names, cls[i, j].map_ij))
f.write('# L' + ("%17s " * len(cols)) % tuple(cols) + '\n')
for b in range(self.pcl_lmin, self.pcl_lmax + 1):
c = [b]
for i in range(self.nmaps_required):
for j in range(i + 1):
c.append(cls[i, j].CL[b - self.pcl_lmin])
f.write(("%I5 " + "%17.8e " * len(cols)) % tuple(c))
def transform(self, C, Chat, Cfhalf):
# HL transformation of the matrices
if C.shape[0] == 1:
rat = Chat[0, 0] / C[0, 0]
C[0, 0] = (np.sign(rat - 1) *
np.sqrt(2 * np.maximum(0, rat - np.log(rat) - 1)) *
Cfhalf[0, 0]**2)
return
diag, U = np.linalg.eigh(C)
rot = U.T.dot(Chat).dot(U)
roots = np.sqrt(diag)
for i, root in enumerate(roots):
rot[i, :] /= root
rot[:, i] /= root
U.dot(rot.dot(U.T), rot)
diag, rot = np.linalg.eigh(rot)
diag = np.sign(diag - 1) * np.sqrt(
2 * np.maximum(0, diag - np.log(diag) - 1))
Cfhalf.dot(rot, U)
for i, d in enumerate(diag):
rot[:, i] = U[:, i] * d
rot.dot(U.T, C)
def exact_chi_sq(self, C, Chat, L):
if C.shape[0] == 1:
return ((2 * L + 1) * self.fsky *
(Chat[0, 0] / C[0, 0] - 1 - np.log(Chat[0, 0] / C[0, 0])))
else:
M = np.linalg.inv(C).dot(Chat)
return ((2 * L + 1) * self.fsky *
(np.trace(M) - self.nmaps - np.linalg.slogdet(M)[1]))
def logp(self, **data_params):
cls = self.provider.get_Cl(ell_factor=True)
return self.log_likelihood(cls, **data_params)
def log_likelihood(self, dls, **data_params):
r"""
Get log likelihood from the dls (CMB C_l scaled by L(L+1)/2\pi)
:param dls: dictionary of d_l ('tt', etc)
:param data_params: likelihood nuisance parameters
:return: log likelihood
"""
self.get_theory_map_cls(dls, data_params)
C = np.empty((self.nmaps, self.nmaps))
big_x = np.empty(self.nbins_used * self.ncl_used)
vecp = np.empty(self.ncl)
chisq = 0
if self.binned:
binned_theory = self.get_binned_map_cls(self.map_cls)
else:
Cs = np.zeros((self.nbins_used, self.nmaps, self.nmaps))
for i in range(self.nmaps):
for j in range(i + 1):
CL = self.map_cls[i, j]
if CL is not None:
Cs[:, i, j] = CL.CL[self.bin_min -
self.pcl_lmin:self.bin_max -
self.pcl_lmin + 1]
Cs[:, j, i] = CL.CL[self.bin_min -
self.pcl_lmin:self.bin_max -
self.pcl_lmin + 1]
for b in range(self.nbins_used):
if self.binned:
self.elements_to_matrix(binned_theory[b, :], C)
else:
C[:, :] = Cs[b, :, :]
if self.cl_noise is not None:
C += self.noise_matrix[b]
if self.like_approx == 'exact':
chisq += self.exact_chi_sq(C, self.bandpower_matrix[b],
self.bin_min + b)
continue
elif self.like_approx == 'HL':
try:
self.transform(C, self.bandpower_matrix[b],
self.fiducial_sqrt_matrix[b])
except np.linalg.LinAlgError:
self.log.debug("Likelihood computation failed.")
return -np.inf
elif self.like_approx == 'gaussian':
C -= self.bandpower_matrix[b]
self.matrix_to_elements(C, vecp)
big_x[b * self.ncl_used:(b + 1) *
self.ncl_used] = vecp[self.cl_used_index]
if self.like_approx == 'exact':
return -0.5 * chisq
return -0.5 * self._fast_chi_squared(self.covinv, big_x)
class DatasetLikelihood(object):
def __init__(self,
fname,
dataset_params={},
field_names=['T', 'E', 'B', 'P'],
map_separator='x'):
self.field_names = field_names
self.tot_theory_fields = len(field_names)
self.map_separator = map_separator
# aberration will be corrected if aberration_coeff is non - zero
self.aberration_coeff = 0.0
self.log_calibration_prior = -1 # if >0 use log prior on calibration parameter
if '.dataset' in fname:
self.loadDataset(fname, dataset_params)
else:
raise Exception('DatasetLikelihood only supports .dataset files')
def typeIndex(self, field):
return self.field_names.index(field)
def PairStringToMapIndices(self, S):
if len(S) == 2:
if self.has_map_names:
raise Exception('CMBlikes: CL names must use MAP1xMAP2 names')
return self.map_names.index(S[0]), self.map_names.index(S[1])
else:
try:
i = S.index(self.map_separator)
except ValueError:
raise ValueError('CMBLikes: invalid spectrum name %s' % S)
return self.map_names.index(S[0:i]), self.map_names.index(S[i +
1:])
def PairStringToUsedMapIndices(self, used_index, S):
i1, i2 = self.PairStringToMapIndices(S)
i1 = used_index[i1]
i2 = used_index[i2]
if i2 > i1:
return i2, i1
else:
return i1, i2
def UseString_to_cols(self, L):
cl_i_j = self.UseString_to_Cl_i_j(L, self.map_used_index)
cols = -np.ones(cl_i_j.shape[1], dtype=int)
for i in range(cl_i_j.shape[1]):
i1, i2 = cl_i_j[:, i]
if i1 == -1 or i2 == -1: continue
ix = 0
for ii in range(self.nmaps):
for jj in range(ii + 1):
if ii == i1 and jj == i2:
cols[i] = ix
ix += 1
return cols
def UseString_to_Cl_i_j(self, S, used_index):
if not isinstance(S, (list, tuple)): S = S.split()
cl_i_j = np.zeros((2, len(S)), dtype=int)
for i, p in enumerate(S):
cl_i_j[:, i] = self.PairStringToUsedMapIndices(used_index, p)
return cl_i_j
def MapPair_to_Theory_i_j(self, order, pair):
i = self.map_fields[order[pair[0]]]
j = self.map_fields[order[pair[1]]]
if i <= j:
return i, j
else:
return j, i
def Cl_used_i_j_name(self, pair):
return self.Cl_i_j_name(self.used_map_order, pair)
def Cl_i_j_name(self, names, pair):
name1 = names[pair[0]]
name2 = names[pair[1]]
if self.has_map_names:
return name1 + self.map_separator + name2
else:
return name1 + name2
def GetColsFromOrder(self, order):
# Converts string Order = TT TE EE XY... or AAAxBBB AAAxCCC BBxCC
# into indices into array of power spectra (and -1 if not present)
cols = np.empty(self.ncl, dtype=int)
cols[:] = -1
names = order.strip().split()
ix = 0
for i in range(self.nmaps):
for j in range(i + 1):
name = self.Cl_used_i_j_name([i, j])
if not name in names and i <> j:
name = self.Cl_used_i_j_name([j, i])
if name in names:
if cols[ix] <> -1:
raise Exception('GetColsFromOrder: duplicate CL type')
cols[ix] = names.index(name)
ix += 1
return cols
def elements_to_matrix(self, X, M):
ix = 0
for i in range(self.nmaps):
M[i, 0:i] = X[ix:ix + i]
M[0:i, i] = X[ix:ix + i]
ix += i
M[i, i] = X[ix]
ix += 1
def matrix_to_elements(self, M, X):
ix = 0
for i in range(self.nmaps):
X[ix:ix + i + 1] = M[i, 0:i + 1]
ix += i + 1
def ReadClArr(self, ini, file_stem, return_full=False):
# read file of CL or bins (indexed by L)
filename = ini.relativeFileName(file_stem + '_file')
cl = np.zeros((self.ncl, self.nbins_used))
order = ini.string(file_stem + '_order', '')
if not order:
incols = lastTopComment(filename)
if not incols:
raise Exception('No column order given for ' + filename)
else:
incols = 'L ' + order
cols = self.GetColsFromOrder(incols)
data = np.loadtxt(filename)
Ls = data[:, 0].astype(int)
if self.binned: Ls -= 1
for i, L in enumerate(Ls):
if L >= self.bin_min and L <= self.bin_max:
for ix in range(self.ncl):
if cols[ix] <> -1:
cl[ix, L - self.bin_min] = data[i, cols[ix]]
if L < self.bin_max:
raise Exception(
'CMBLikes_ReadClArr: C_l file does not go up to maximum used: %s'
% self.bin_max)
if return_full:
return incols.split(), data, cl
else:
return cl
def readBinWindows(self, ini, file_stem):
bins = BinWindows(self.pcl_lmin, self.pcl_lmax, self.nbins_used)
in_cl = ini.split(file_stem + '_in_order')
out_cl = ini.split(file_stem + '_out_order', in_cl)
bins.cols_in = self.UseString_to_Cl_i_j(in_cl, self.map_required_index)
bins.cols_out = self.UseString_to_cols(out_cl)
norder = bins.cols_in.shape[1]
if norder != bins.cols_out.shape[0]:
raise Exception(
'_in_order and _out_order must have same number of entries')
bins.binning_matrix = np.zeros(
(norder, self.nbins_used, self.pcl_lmax - self.pcl_lmin + 1))
windows = ini.relativeFileName(file_stem + '_files')
for b in range(self.nbins_used):
window = np.loadtxt(windows % (b + 1 + self.bin_min))
Err = False
for i, L in enumerate(window[:, 0].astype(int)):
if self.pcl_lmin <= L <= self.pcl_lmax:
bins.binning_matrix[:, b, L - self.pcl_lmin] = window[i,
1:]
else:
Err = Err or any(window[i, 1:] != 0)
if Err:
print('WARNING: %s %u outside pcl_lmin-cl_max range: %s' %
(file_stem, b, windows % (b + 1)))
if ini.hasKey(file_stem + '_fix_cl_file'):
raise Exception('fix_cl_file not implemented yet')
return bins
def init_map_cls(self, nmaps, order):
if nmaps <> len(order):
raise ValueError('CMBLikes init_map_cls: size mismatch')
class CrossPowerSpectrum(object):
pass
cls = np.empty((nmaps, nmaps), dtype=object)
for i in range(nmaps):
for j in range(i + 1):
CL = CrossPowerSpectrum()
cls[i, j] = CL
CL.map_ij = [order[i], order[j]]
CL.theory_ij = self.MapPair_to_Theory_i_j(order, [i, j])
CL.CL = np.zeros(self.pcl_lmax - self.pcl_lmin + 1)
return cls
def loadDataset(self, froot, dataset_params):
if not '.dataset' in froot: froot += '.dataset'
ini = IniFile(froot)
ini.params.update(dataset_params)
self.readIni(ini)
def readIni(self, ini):
self.map_names = ini.split('map_names', default=[])
self.has_map_names = len(self.map_names)
if self.has_map_names:
# e.g. have multiple frequencies for given field measurement
map_fields = ini.split('map_fields')
if len(map_fields) <> len(self.map_names):
raise Exception(
'CMBLikes: number of map_fields does not match map_names')
self.map_fields = [self.typeIndex(f) for f in map_fields]
else:
self.map_names = self.field_names
self.map_fields = np.arange(len(self.map_names), dtype=int)
fields_use = ini.split('fields_use', [])
if len(fields_use):
index_use = [self.typeIndex(f) for f in fields_use]
use_theory_field = [
i in index_use for i in range(self.tot_theory_fields)
]
else:
if not self.has_map_names:
raise Exception('CMBlikes: must have fields_use or map_names')
use_theory_field = [True] * self.tot_theory_fields
maps_use = ini.split('maps_use', [])
if len(maps_use):
if np.any([not i for i in use_theory_field]):
print('CMBlikes WARNING: maps_use overrides fields_use')
self.use_map = np.zeros(len(self.map_names), dtype=bool)
for j, map_used in enumerate(maps_use):
if map_used in self.map_names:
self.use_map[self.map_names.index(map_used)] = True
else:
raise ValueError('CMBlikes: maps_use item not found - %s' %
map_used)
else:
self.use_map = [
use_theory_field[self.map_fields[i]]
for i in range(len(self.map_names))
]
# Bandpowers can depend on more fields than are actually used in likelihood
# e.g. for correcting leakage or other linear corrections
self.require_map = self.use_map[:]
if self.has_map_names:
if ini.hasKey('fields_required'):
raise Exception(
'CMBLikes: use maps_required not fields_required')
maps_use = ini.split('maps_required', [])
else:
maps_use = ini.split('fields_required', [])
if len(maps_use):
for j, map_used in enumerate(maps_use):
if map_used in self.map_names:
self.require_map[self.map_names.index(map_used)] = True
else:
raise ValueError('CMBlikes: required item not found %s' %
map_used)
self.required_theory_field = [False for _ in self.field_names]
for i in range(len(self.map_names)):
if self.require_map[i]:
self.required_theory_field[self.map_fields[i]] = True
self.ncl_used = 0 # set later reading covmat
self.like_approx = ini.string('like_approx', 'gaussian')
self.nmaps = np.count_nonzero(self.use_map)
self.nmaps_required = np.count_nonzero(self.require_map)
self.required_order = np.zeros(self.nmaps_required, dtype=int)
self.map_required_index = -np.ones(len(self.map_names), dtype=int)
ix = 0
for i in range(len(self.map_names)):
if self.require_map[i]:
self.map_required_index[i] = ix
self.required_order[ix] = i
ix += 1
self.map_used_index = -np.ones(len(self.map_names), dtype=int)
ix = 0
self.used_map_order = []
for i, map_name in enumerate(self.map_names):
if self.use_map[i]:
self.map_used_index[i] = ix
self.used_map_order.append(map_name)
ix += 1
self.ncl = (self.nmaps * (self.nmaps + 1)) // 2
self.pcl_lmax = ini.int('cl_lmax')
self.pcl_lmin = ini.int('cl_lmin')
self.binned = ini.bool('binned', True)
if self.binned:
self.nbins = ini.int('nbins')
self.bin_min = ini.int('use_min', 1) - 1
self.bin_max = ini.int('use_max', self.nbins) - 1
self.nbins_used = self.bin_max - self.bin_min + 1 # needed by readBinWindows
self.bins = self.readBinWindows(ini, 'bin_window')
else:
if self.nmaps <> self.nmaps_required:
raise Exception(
'CMBlikes: unbinned likelihood must have nmaps==nmaps_required'
)
self.nbins = self.pcl_lmax - self.pcl_lmin + 1
if self.like_approx != 'exact':
print('WARNING: Unbinned likelihoods untested in this version')
self.bin_min = ini.int('use_min', self.pcl_lmin)
self.bin_max = ini.int('use_max', self.pcl_lmax)
self.nbins_used = self.bin_max - self.bin_min + 1
self.full_bandpower_headers, self.full_bandpowers, self.bandpowers = \
self.ReadClArr(ini, 'cl_hat', return_full=True)
if self.like_approx == 'HL':
self.cl_fiducial = self.ReadClArr(ini, 'cl_fiducial')
else:
self.cl_fiducial = None
includes_noise = ini.bool('cl_hat_includes_noise', False)
self.cl_noise = None
if self.like_approx <> 'gaussian' or includes_noise:
self.cl_noise = self.ReadClArr(ini, 'cl_noise')
if not includes_noise:
self.bandpowers += self.cl_noise
elif self.like_approx == 'gaussian':
self.bandpowers -= self.cl_noise
self.cl_lmax = np.zeros(
(self.tot_theory_fields, self.tot_theory_fields))
for i in range(self.tot_theory_fields):
if self.required_theory_field[i]:
self.cl_lmax[i, i] = self.pcl_lmax
if self.required_theory_field[0] and self.required_theory_field[1]:
self.cl_lmax[1, 0] = self.pcl_lmax
if self.like_approx != 'gaussian':
cl_fiducial_includes_noise = ini.bool('cl_fiducial_includes_noise',
False)
self.bandpower_matrix = np.zeros(
(self.nbins_used, self.nmaps, self.nmaps))
self.noise_matrix = self.bandpower_matrix.copy()
self.fiducial_sqrt_matrix = self.bandpower_matrix.copy()
if self.cl_fiducial is not None and not cl_fiducial_includes_noise:
self.cl_fiducial += self.cl_noise
for b in range(self.nbins_used):
self.elements_to_matrix(self.bandpowers[:, b],
self.bandpower_matrix[b, :, :])
if self.cl_noise is not None:
self.elements_to_matrix(self.cl_noise[:, b],
self.noise_matrix[b, :, :])
if self.cl_fiducial is not None:
self.elements_to_matrix(self.cl_fiducial[:, b],
self.fiducial_sqrt_matrix[b, :, :])
self.fiducial_sqrt_matrix[b, :, :] = sqrtm(
self.fiducial_sqrt_matrix[b, :, :])
if self.like_approx == 'exact':
self.fsky = ini.float('fullsky_exact_fksy')
else:
self.cov = self.ReadCovmat(ini)
self.covinv = np.linalg.inv(self.cov)
if 'linear_correction_fiducial_file' in ini.params:
self.fid_correction = self.ReadClArr(ini,
'linear_correction_fiducial')
self.linear_correction = self.readBinWindows(
ini, 'linear_correction_bin_window')
else:
self.linear_correction = None
if ini.hasKey('nuisance_params'):
s = ini.relativeFileName('nuisance_params')
self.nuisance_params = ParamNames(s)
if ini.hasKey('calibration_param'):
self.calibration_param = ini.string('calibration_param')
if '.paramnames' in self.calibration_param:
raise Exception(
'calibration_param should be name of parameter in nuisance_params'
)
else:
self.calibration_param = None
elif ini.hasKey('calibration_param'):
s = ini.relativeFileName('calibration_param')
if not '.paramnames' in s:
raise Exception(
'calibration_param must be paramnames file unless nuisance_params also specified'
)
self.nuisance_params = ParamNames(s)
self.calibration_param = self.nuisance_params.list()[0]
else:
self.calibration_param = None
if self.calibration_param is not None:
self.log_calibration_prior = ini.float('log_calibration_prior', -1)
self.aberration_coeff = ini.float('aberration_coeff', 0.0)
self.map_cls = self.init_map_cls(self.nmaps_required,
self.required_order)
def ReadCovmat(self, ini):
# read the covariance matrix, and the array of which CL are in the covariance,
# which then defines which set of bandpowers are used (subject to other restrictions)
covmat_cl = ini.string('covmat_cl', allowEmpty=False)
self.full_cov = np.loadtxt(ini.relativeFileName('covmat_fiducial'))
covmat_scale = ini.float('covmat_scale', 1.0)
cl_in_index = self.UseString_to_cols(covmat_cl)
self.ncl_used = np.sum(cl_in_index >= 0)
self.cl_used_index = np.zeros(self.ncl_used, dtype=int)
cov_cl_used = np.zeros(self.ncl_used, dtype=int)
ix = 0
for i, index in enumerate(cl_in_index):
if index >= 0:
self.cl_used_index[ix] = index
cov_cl_used[ix] = i
ix += 1
if self.binned:
num_in = len(cl_in_index)
pcov = np.empty((self.nbins_used * self.ncl_used,
self.nbins_used * self.ncl_used))
for binx in range(self.nbins_used):
for biny in range(self.nbins_used):
pcov[binx * self.ncl_used: (binx + 1) * self.ncl_used,
biny * self.ncl_used: (biny + 1) * self.ncl_used] = \
covmat_scale * self.full_cov[np.ix_((binx + self.bin_min) * num_in + cov_cl_used,
(biny + self.bin_min) * num_in + cov_cl_used)]
else:
raise Exception('unbinned covariance not implemented yet')
return pcov
def writeData(self, froot):
np.savetxt(froot + '_cov.dat', self.cov)
# self.saveCl(froot + '_fid_cl.dat', self.fid_cl[:, 1:], cols=['TT', 'EE', 'TE', 'PP'])
with open(froot + '_bandpowers.dat', 'w') as f:
f.write("#%4s %5s %5s %8s %12s %10s %7s\n" %
('bin', 'L_min', 'L_max', 'L_av', 'PP', 'Error', 'Ahat'))
for b in range(self.nbins):
f.write("%5u %5u %5u %8.2f %12.5e %10.3e %7.3f\n" %
(b + 1, self.lmin[b], self.lmax[b], self.lav[b],
self.bandpowers[b], np.sqrt(
self.cov[b, b]), self.Ahat[b]))
self.bins.write(froot, 'bin')
if self.linear_correction is not None:
self.linear_correction.write(froot, 'linear_correction_bin')
with open(froot + '_lensing_fiducial_correction', 'w') as f:
f.write("#%4s %12s \n" % ('bin', 'PP'))
for b in range(self.nbins):
f.write("%5u %12.5e\n" % (b + 1, self.fid_correction[b]))
def get_binned_theory(self, ClArray, data_params={}):
# Useful for plotting, not used for likelihood
self.get_theory_map_cls(ClArray, data_params)
return self.get_binned_map_cls(self.map_cls)
def get_full_bandpower_column(self, col_name):
"""
Get columns from the input bandpower file. Note used by likelihood but may be useful for plotting.
:param col_name: name of column, as in the top comment header of the file
:return: column values
"""
return self.full_bandpowers[:,
self.full_bandpower_headers.index(col_name
)]
def diag_sigma(self):
return np.sqrt(np.diag(self.full_cov))
def plot(self, column='PP', ClArray=None, ls=None, ax=None):
lbin = self.full_bandpowers[:,
self.full_bandpower_headers.index('L_av')]
binned_phicl_err = self.diag_sigma()
ax = ax or plt.gca()
bandpowers = self.full_bandpowers[:,
self.full_bandpower_headers.
index('PP')]
if 'L_min' in self.full_bandpower_headers:
lmin = self.full_bandpowers[:,
self.full_bandpower_headers.
index('L_min')]
lmax = self.full_bandpowers[:,
self.full_bandpower_headers.
index('L_max')]
ax.errorbar(lbin,
bandpowers,
yerr=binned_phicl_err,
xerr=[lbin - lmin, lmax - lbin],
fmt='o')
else:
ax.errorbar(lbin, bandpowers, yerr=binned_phicl_err, fmt='o')
if ClArray is not None:
if isinstance(ClArray, ClsArray):
i, j = self.MapPair_to_Theory_i_j(
range(len(self.map_names)),
self.PairStringToMapIndices(column))
ClArray = ClArray.get([i, j])
if ls is None: ls = np.arange(len(ClArray))
ax.plot(ls, ClArray, color='k')
ax.set_xlim([2, ls[-1]])
def get_binned_map_cls(self, Cls, corrections=True):
band = self.bins.bin(Cls)
if self.linear_correction is not None and corrections:
band += self.linear_correction.bin(Cls) - self.fid_correction.T
return band
def get_theory_map_cls(self, ClArray, data_params={}):
for i in range(self.nmaps_required):
for j in range(i + 1):
CL = self.map_cls[i, j]
cls = ClArray.get(CL.theory_ij)
if cls is not None:
CL.CL[:] = cls[self.pcl_lmin:self.pcl_lmax + 1]
else:
CL.CL[:] = 0
self.adapt_theory_for_maps(self.map_cls, data_params)
def adapt_theory_for_maps(self, cls, data_params):
if self.aberration_coeff: self.add_aberration(cls)
self.add_foregrounds(cls, data_params)
if self.calibration_param is not None and self.calibration_param in data_params:
for i in range(self.nmaps_required):
for j in range(i + 1):
CL = cls[i, j]
if CL is not None:
if CL.theory_ij[0] <= 2 and CL.theory_ij[1] <= 2:
CL.CL /= data_params[self.calibration_param]**2
def add_foregrounds(self, cls, data_params):
pass
def add_aberration(self, cls):
# adapted from CosmoMC function by Christian Reichardt
ells = np.arange(self.pcl_lmin, self.pcl_lmax + 1)
cl_norm = ells * (ells + 1)
for i in range(self.nmaps_required):
for j in range(i + 1):
CL = cls[i, j]
if CL is not None:
if CL.theory_ij[0] <= 2 and CL.theory_ij[1] <= 2:
# first get Cl instead of Dl
cl_deriv = CL.CL / cl_norm
# second take derivative dCl/dl
cl_deriv[1:-1] = (cl_deriv[2:] - cl_deriv[:-2]) / 2
# handle endpoints approximately
cl_deriv[0] = cl_deriv[1]
cl_deriv[-1] = cl_deriv[-2]
# reapply to Dl's.
# note never took 2pi out, so not putting it back either
cl_deriv *= cl_norm
# also multiply by ell since really wanted ldCl/dl
cl_deriv *= ells
CL.CL += self.aberration_coeff * cl_deriv
def write_likelihood_data(self, filename, data_params={}):
cls = self.init_map_cls(self.nmaps_required, self.required_order)
self.add_foregrounds(cls, data_params)
with open(filename, 'w') as f:
cols = []
for i in range(self.nmaps_required):
for j in range(i + 1):
cols.append(
self.Cl_i_j_name(self.map_names, cls[i, j].map_ij))
f.write('# L' + ("%17s " * len(cols)) % tuple(cols) + '\n')
for b in range(self.pcl_lmin, self.pcl_lmax + 1):
c = [b]
for i in range(self.nmaps_required):
for j in range(i + 1):
c.append(cls[i, j].CL[b - self.pcl_lmin])
f.write(("%I5 " + "%17.8e " * len(cols)) % tuple(c))
def transform(self, C, Chat, Cfhalf):
# HL transformation of the matrices
if C.shape[0] == 1:
rat = Chat[0, 0] / C[0, 0]
C[0, 0] = np.sign(rat - 1) * np.sqrt(
2 * np.maximum(0, rat - np.log(rat) - 1)) * Cfhalf[0, 0]**2
return
diag, U = np.linalg.eigh(C)
rot = U.T.dot(Chat).dot(U)
roots = np.sqrt(diag)
for i, root in enumerate(roots):
rot[i, :] /= root
rot[:, i] /= root
U.dot(rot.dot(U.T), rot)
diag, rot = np.linalg.eigh(rot)
diag = np.sign(diag - 1) * np.sqrt(
2 * np.maximum(0, diag - np.log(diag) - 1))
Cfhalf.dot(rot, U)
for i, d in enumerate(diag):
rot[:, i] = U[:, i] * d
rot.dot(U.T, C)
def exact_chi_sq(self, C, Chat, L):
if C.shape[0] == 1:
return (2 * L + 1) * self.fsky * (Chat[0, 0] / C[0, 0] - 1 -
np.log(Chat[0, 0] / C[0, 0]))
else:
M = np.linalg.inv(C).dot(Chat)
return (2 * L + 1) * self.fsky * (np.trace(M) - self.nmaps -
np.linalg.slogdet(M)[1])
def chi_squared(self, ClArray, data_params={}):
self.get_theory_map_cls(ClArray, data_params)
C = np.empty((self.nmaps, self.nmaps))
bigX = np.empty(self.nbins_used * self.ncl_used)
vecp = np.empty(self.ncl)
chisq = 0
if self.binned:
binned_theory = self.get_binned_map_cls(self.map_cls)
else:
Cs = np.zeros((self.nbins_used, self.nmaps, self.nmaps))
for i in range(self.nmaps):
for j in range(i + 1):
CL = self.map_cls[i, j]
if CL is not None:
Cs[:, i, j] = CL.CL[self.bin_min -
self.pcl_lmin:self.bin_max -
self.pcl_lmin + 1]
Cs[:, j, i] = CL.CL[self.bin_min -
self.pcl_lmin:self.bin_max -
self.pcl_lmin + 1]
for bin in range(self.nbins_used):
if self.binned:
self.elements_to_matrix(binned_theory[bin, :], C)
else:
C[:, :] = Cs[bin, :, :]
if self.cl_noise is not None:
C += self.noise_matrix[bin]
if self.like_approx == 'exact':
chisq += self.exact_chi_sq(C, self.bandpower_matrix[bin],
self.bin_min + bin)
continue
elif self.like_approx == 'HL':
self.transform(C, self.bandpower_matrix[bin],
self.fiducial_sqrt_matrix[bin])
elif self.like_approx == 'gaussian':
C -= self.bandpower_matrix[bin]
self.matrix_to_elements(C, vecp)
bigX[bin * self.ncl_used:(bin + 1) *
self.ncl_used] = vecp[self.cl_used_index]
if self.like_approx != 'exact':
chisq = np.dot(bigX, np.dot(self.covinv, bigX))
if self.log_calibration_prior > 0:
chisq += (np.log(data_params[self.calibration_param]) /
self.log_calibration_prior)**2
return chisq
def __init__(self, clikfile, paramnames=None):
# paramnames will be used to do the mapping to the extra parameters
self.dffile = clikfile
self.name = os.path.splitext(os.path.basename(clikfile))[0]
if isinstance(paramnames, (list, tuple)):
self.parnames = paramnames
else:
if paramnames is None:
# match versions to the baseline .paramnames file
for rem in ['_', 'a_', 'b_', 'c_', 'd_']:
name = self.name.replace(rem, '_').replace('_bin1', '')
paramnames = os.path.join(
os.path.dirname(__file__),
'../../data/' + name + '.paramnames')
if os.path.exists(paramnames):
break
self.paramnamefile = paramnames
self.paramnames = ParamNames(paramnames)
self.parnames = self.paramnames.list()
self.clik = clik.clik(clikfile)
self._translate_parname(self.parnames)
self.fi = hpy.File(self.dffile)
# some metadata
self.hascl = self.fi["clik/lkl_0/has_cl"]
self.lmin = self.fi["clik/lkl_0/lmin"]
self.lmax = self.fi["clik/lkl_0/lmax"]
self.mt = self.fi["clik/lkl_0/m_channel_T"] * self.hascl[0]
self.me = self.fi["clik/lkl_0/m_channel_P"] * self.hascl[1]
self.mb = self.fi["clik/lkl_0/m_channel_P"] * self.hascl[2]
self.m = self.mt + self.me + self.mb
self.nb = self.fi["clik/lkl_0/nbins"] / self.hascl.sum()
self.rq_shape = (self.nb, self.m, self.m)
# binning details
self.blmin = self.fi["clik/lkl_0/bin_lmin"]
self.blmax = self.fi["clik/lkl_0/bin_lmax"]
self.b_ws = self.fi["clik/lkl_0/bin_ws"]
# the binning matrix is also simply obtained this way (but using it is slower than using the binning details, 'cause it's full of zeros)
self.bns = php.read_bins(self.fi["clik/lkl_0"])
# compute the binned ells
self.lm = np.dot(self.bns[:self.nb, :self.lmax - self.lmin + 1],
np.arange(self.lmin, self.lmax + 1))
# get the calibration part (and beam for plik 2015)
# cal and bal are functions that expect a vector of parameters whose name and ordering are given by cal.varpar and bal.varpar
# overal calibration is given by cal(pars)*vec(pars)*outer(acmb,acmb)[nm.newaxis,:,:]
self.cal = smh.calTP_from_smica(self.dffile)
self.bal = smh.beamTP_from_smica(self.dffile)
self.acmb = self.fi["clik/lkl_0/A_cmb"]
# get the binned Cl data array
self.rqh = self.fi["clik/lkl_0/Rq_hat"]
self.rqh.shape = self.rq_shape
# get the additive nuisance components
self.prms = smh.parametric_from_smica(self.dffile)
self.prms_name = [p.get_name() for p in self.prms]
# get the selection vector
self.oo, self.Jt = smh.ordering_from_smica(self.dffile)
# get the inverse covariance
self.siginv = self.fi["clik/lkl_0/criterion_gauss_mat"]
self.siginv.shape = (len(self.oo), len(self.oo))
ls = np.arange(self.lmax + 1)
self.llp1 = ls * (ls + 1) / (2 * np.pi)
self.llp1[0] = 1
self.indices = [(0, 0), (1, 1), (2, 2), (0, 1), (0, 2), (1, 2)]
self.spectra = ["tt", "ee", "bb", "te", "tb", "eb"]
class plik_likelihood(object):
def __init__(self, clikfile, paramnames=None):
# paramnames will be used to do the mapping to the extra parameters
self.dffile = clikfile
self.name = os.path.splitext(os.path.basename(clikfile))[0]
if isinstance(paramnames, (list, tuple)):
self.parnames = paramnames
else:
if paramnames is None:
# match versions to the baseline .paramnames file
for rem in ['_', 'a_', 'b_', 'c_', 'd_']:
name = self.name.replace(rem, '_').replace('_bin1', '')
paramnames = os.path.join(
os.path.dirname(__file__),
'../../data/' + name + '.paramnames')
if os.path.exists(paramnames):
break
self.paramnamefile = paramnames
self.paramnames = ParamNames(paramnames)
self.parnames = self.paramnames.list()
self.clik = clik.clik(clikfile)
self._translate_parname(self.parnames)
self.fi = hpy.File(self.dffile)
# some metadata
self.hascl = self.fi["clik/lkl_0/has_cl"]
self.lmin = self.fi["clik/lkl_0/lmin"]
self.lmax = self.fi["clik/lkl_0/lmax"]
self.mt = self.fi["clik/lkl_0/m_channel_T"] * self.hascl[0]
self.me = self.fi["clik/lkl_0/m_channel_P"] * self.hascl[1]
self.mb = self.fi["clik/lkl_0/m_channel_P"] * self.hascl[2]
self.m = self.mt + self.me + self.mb
self.nb = self.fi["clik/lkl_0/nbins"] / self.hascl.sum()
self.rq_shape = (self.nb, self.m, self.m)
# binning details
self.blmin = self.fi["clik/lkl_0/bin_lmin"]
self.blmax = self.fi["clik/lkl_0/bin_lmax"]
self.b_ws = self.fi["clik/lkl_0/bin_ws"]
# the binning matrix is also simply obtained this way (but using it is slower than using the binning details, 'cause it's full of zeros)
self.bns = php.read_bins(self.fi["clik/lkl_0"])
# compute the binned ells
self.lm = np.dot(self.bns[:self.nb, :self.lmax - self.lmin + 1],
np.arange(self.lmin, self.lmax + 1))
# get the calibration part (and beam for plik 2015)
# cal and bal are functions that expect a vector of parameters whose name and ordering are given by cal.varpar and bal.varpar
# overal calibration is given by cal(pars)*vec(pars)*outer(acmb,acmb)[nm.newaxis,:,:]
self.cal = smh.calTP_from_smica(self.dffile)
self.bal = smh.beamTP_from_smica(self.dffile)
self.acmb = self.fi["clik/lkl_0/A_cmb"]
# get the binned Cl data array
self.rqh = self.fi["clik/lkl_0/Rq_hat"]
self.rqh.shape = self.rq_shape
# get the additive nuisance components
self.prms = smh.parametric_from_smica(self.dffile)
self.prms_name = [p.get_name() for p in self.prms]
# get the selection vector
self.oo, self.Jt = smh.ordering_from_smica(self.dffile)
# get the inverse covariance
self.siginv = self.fi["clik/lkl_0/criterion_gauss_mat"]
self.siginv.shape = (len(self.oo), len(self.oo))
ls = np.arange(self.lmax + 1)
self.llp1 = ls * (ls + 1) / (2 * np.pi)
self.llp1[0] = 1
self.indices = [(0, 0), (1, 1), (2, 2), (0, 1), (0, 2), (1, 2)]
self.spectra = ["tt", "ee", "bb", "te", "tb", "eb"]
def _translate_parname(self, parnames):
self._translate = {}
for old, new in zip(self.clik.extra_parameter_names, parnames):
self._translate[old] = new
def get_nuisance_vector(self, nuisance_dict, clik_names=None):
# nuisance dict is a python dict whose keys are defined by the cosmomc paramnames
if clik_names is None:
clik_names = self.clik.extra_parameter_names
return np.array(
[nuisance_dict[self._translate[par]] for par in clik_names])
def get_cl_clik_ordering(self, cl_array):
mcls = []
lmax = self.lmax
for i, index in enumerate(self.indices[:4]):
if self.hascl[i]:
mcls += [cl_array.get(index)[:lmax + 1] / self.llp1]
for i in range(4, 6):
if self.hascl[i]:
lcls = np.zeros(lmax + 1)
mcls += [lcls]
return (mcls)
def get_clik_vector(self, cl_array, nuisance_dict):
mcls = self.get_cl_clik_ordering(cl_array)
extra = self.get_nuisance_vector(nuisance_dict)
return np.concatenate(mcls + [extra])
def get_unbinned_nuisance_rq(self, nuisance_dict):
# compute the nuisance part
oq = []
for p in self.prms:
# prepare the input vector for each component
pvec = self.get_nuisance_vector(nuisance_dict, p.varpar)
# compute the rq matrix for this component for those parameters
oq += [p(pvec)]
# correct shape for T only component when dealing with T+P case
if oq[-1].shape[1:] != self.rq_shape[1:]:
bet = np.zeros(
(oq[-1].shape[0], self.rq_shape[1], self.rq_shape[1]))
bet[:, :oq[-1].shape[1], :oq[-1].shape[1]] = oq[-1]
oq[-1] = bet
oq = np.array(oq)
return oq
def get_nuisance_rq(self, nuisance_dict):
# get the unbinned one
oq = self.get_unbinned_nuisance_rq(nuisance_dict)
oqb = np.zeros((len(oq), ) + self.rq_shape)
# bin it
nb = self.nb
m = self.m
mt = self.mt
me = self.me
mb = self.mb
blmin = self.blmin
blmax = self.blmax
b_ws = self.b_ws
lmin = self.lmin
lmax = self.lmax
for b in range(nb):
if oq.shape[0]:
oqb[:, b] = np.sum(
oq[:, blmin[b]:blmax[b] + 1] *
b_ws[np.newaxis, blmin[b]:blmax[b] + 1, np.newaxis,
np.newaxis], 1)
return oqb
def get_cmb_rq(self, cl_array):
# get the cls
mcls = self.get_cl_clik_ordering(cl_array)
cls = np.zeros((6, self.lmax + 1))
j = 0
for i in range(6):
if self.hascl[i]:
cls[i] = mcls[j]
j += 1
rq = np.zeros((self.nb, self.m, self.m))
nb = self.nb
m = self.m
mt = self.mt
me = self.me
mb = self.mb
blmin = self.blmin
blmax = self.blmax
b_ws = self.b_ws
lmin = self.lmin
lmax = self.lmax
# bin it (and order it in)
for b in range(nb):
if mt:
rq[b, :mt, :mt] += np.sum(
cls[0, lmin + blmin[b]:lmin + blmax[b] + 1] *
b_ws[blmin[b]:blmax[b] + 1])
if me:
rq[b, :mt, mt:mt + me] += np.sum(
cls[3, lmin + blmin[b]:lmin + blmax[b] + 1] *
b_ws[blmin[b]:blmax[b] + 1])
rq[b, mt:mt + me, :mt] += np.sum(
cls[3, lmin + blmin[b]:lmin + blmax[b] + 1] *
b_ws[blmin[b]:blmax[b] + 1])
if mb:
rq[b, :mt, mt + me:mb + mt + me] += np.sum(
cls[4, lmin + blmin[b]:lmin + blmax[b] + 1] *
b_ws[blmin[b]:blmax[b] + 1])
rq[b, mt + me:mb + mt + me, :mt] += np.sum(
cls[4, lmin + blmin[b]:lmin + blmax[b] + 1] *
b_ws[blmin[b]:blmax[b] + 1])
if me:
rq[b, mt:mt + me, mt:mt +
me] += np.sum(cls[1, lmin + blmin[b]:lmin + blmax[b] + 1] *
b_ws[blmin[b]:blmax[b] + 1])
if mb:
rq[b, mt:mt + me, mt + me:mb + mt + me] += np.sum(
cls[5, lmin + blmin[b]:lmin + blmax[b] + 1] *
b_ws[blmin[b]:blmax[b] + 1])
rq[b, mt + me:mb + mt + me, mt:mt + me] += np.sum(
cls[5, lmin + blmin[b]:lmin + blmax[b] + 1] *
b_ws[blmin[b]:blmax[b] + 1])
if mb:
rq[b, mt + me:mt + me + mb, mt + me:mb + mt +
me] += np.sum(cls[2, lmin + blmin[b]:lmin + blmax[b] + 1] *
b_ws[blmin[b]:blmax[b] + 1])
return rq
def get_calib(self, nuisance_dict):
# returns a matrix
cvec = self.get_nuisance_vector(nuisance_dict, self.cal.varpar)
g = self.cal(cvec)
cvec = self.get_nuisance_vector(nuisance_dict, self.bal.varpar)
bg = self.bal(cvec)
g *= bg
g *= np.outer(self.acmb, self.acmb)[np.newaxis, :, :]
return g
def get_model_vector(self, cl_array, nuisance_dict):
# get the calib
g = self.get_calib(nuisance_dict)
rq = self.get_cmb_rq(cl_array)
oqb = self.get_nuisance_rq(nuisance_dict)
rqt = g * (rq + np.sum(oqb, 0))
return rqt
def chi_squared(self, cl_array, nuisance_dict, clik=True):
if clik:
vec = self.get_clik_vector(cl_array, nuisance_dict)
return -2 * self.clik(vec)
else:
rqt = self.get_model_vector(cl_array, nuisance_dict)
delta_rq = rqt - self.rqh
delta = delta_rq.flat[self.oo]
return np.dot(delta, np.dot(self.siginv, delta))
def coadd_spectra(self, nuisance_dict):
good = self.Jt.sum(1) != 0
Jt = self.Jt[good]
g = self.get_calib(nuisance_dict)
Jt = Jt * g.flat[self.oo]
tm = np.concatenate([self.lm for h in self.hascl if h])
oqb = self.get_nuisance_rq(nuisance_dict)
Yo = (g * np.sum(oqb, 0)) - self.rqh
Yo = Yo.flat[self.oo]
Jt_siginv = np.zeros((Jt.shape))
# Jt is full of zero, it's quicker to compute the matrix multiplication this way
w0, w1 = np.where(Jt != 0)
for ii in range(len(w0)):
i = w0[ii]
j = w1[ii]
Jt_siginv[i] += Jt[i, j] * self.siginv[j]
Jt_siginv_Yo = np.dot(Jt_siginv, Yo)
nl = Jt.shape[0]
Jt_siginv_J = np.zeros((nl, nl))
for ii in range(len(w0)):
j = w0[ii]
i = w1[ii]
Jt_siginv_J[j] += Jt[j, i] * Jt_siginv[:, i]
try:
rVec = -scipy.linalg.solve(
Jt_siginv_J, Jt_siginv_Yo, assume_a='pos')
except:
rVec = -np.linalg.solve(Jt_siginv_J, Jt_siginv_Yo)
tVec = np.zeros(len(tm))
tVec[good] = rVec
tm.shape = (-1, len(self.lm))
tVec.shape = tm.shape
# beware returns inverse variance of the coadded
return tm, tVec, Jt_siginv_J
def coadded_TT(self, dict, want_cov=True):
ls, coadded, covinv = self.coadd_spectra(dict)
ls = ls[0, :]
TT = np.zeros(ls[-1] + 1)
fac = ls * (ls + 1) / (2 * np.pi)
TT[ls[0]:] = coadded[0, :] * fac
if want_cov:
cov = np.linalg.inv(covinv)[:len(ls), :len(ls)]
for i in range(cov.shape[0]):
cov[i, :] *= fac
cov[:, i] *= fac
return TT, cov
else:
return TT
def cosmomc_root_to_cobaya_info_dict(root: str, derived_to_input=()) -> InputDict:
"""
Given the root name of existing cosmomc chain files, tries to construct a Cobaya
input parameter dictionary with roughly equivalent settings. The output
dictionary can be used for importance sampling from CosmoMC chains in simple cases
using Cobaya's 'post'.
Parameters in the optional derived_to_input list are converted from being derived
parameters in CosmoMC to non-derived in Cobaya.
This is by no means guaranteed to produce valid or equivalent results, use at your
own risk with careful checking! Note that the parameter dictionary will not have
settings for CAMB, samplers etc, which is OK for importance sampling but you would
need to add them as necessary to reproduce results.
Parameter chains in CosmoMC format are available for Planck
from https://pla.esac.esa.int/pla/#home
"""
names = ParamNames(root + '.paramnames')
if os.path.exists(root + '.ranges'):
ranges = ParamBounds(root + '.ranges')
else:
ranges = None
d: ParamsDict = {}
info: InputDict = {'params': d}
for par, name in zip(names.names, names.list()):
if name.startswith('chi2_') and not name.startswith('chi2__'):
if name == 'chi2_prior':
continue
name = name.replace('chi2_', 'chi2__')
if name.startswith('minuslogprior') or name == 'chi2':
continue
param_dict: ParamDict = {'latex': par.label}
d[name] = param_dict
if par.renames:
param_dict['renames'] = par.renames
if par.isDerived:
if name not in derived_to_input:
param_dict['derived'] = True
else:
par.isDerived = False
if ranges and name in ranges.names:
if par.isDerived:
low_up = ranges.getLower(name), ranges.getUpper(name)
if any(r is not None for r in low_up):
param_dict['min'], param_dict['max'] = low_up
else:
param_dict["prior"] = [ranges.getLower(name), ranges.getUpper(name)]
if ranges:
d.update(ranges.fixedValueDict())
if names.numberOfName('As') == -1 and names.numberOfName('logA') != -1:
d['As'] = {'latex': r'A_\mathrm{s}', 'value': 'lambda logA: 1e-10*np.exp(logA)'}
if names.numberOfName('cosmomc_theta') == -1 and names.numberOfName('theta') != -1:
d['cosmomc_theta'] = {'latex': r'\theta_{\rm MC}',
'value': 'lambda theta: theta/100'}
# special case for CosmoMC (e.g. Planck) chains
if os.path.exists(root + '.inputparams'):
inputs = IniFile(root + '.inputparams')
for key, value in inputs.params.items():
if key.startswith('prior['):
if 'prior' not in info:
info['prior'] = {}
param = key[6:-1]
if param in d:
mean, std = (float(v.strip()) for v in value.split())
if not names.parWithName(param).isDerived:
info['prior'][param + '_prior'] = \
"lambda %s: stats.norm.logpdf(%s, loc=%g, scale=%g)" % (
param, param, mean, std)
if key.startswith('linear_combination['):
param = key.replace('linear_combination[', '')[:-1]
prior = inputs.params.get('prior[%s]' % param, None)
if prior:
weights = inputs.params.get('linear_combination_weights[%s]' % param,
None)
if not weights:
raise ValueError(
'linear_combination[%s] prior found but not weights' % param)
weights = [float(w.strip()) for w in weights.split()]
inputs = value.split()
if 'prior' not in info:
info['prior'] = {}
mean, std = (float(v.strip()) for v in prior.split())
linear = "".join(
"%+g*%s" % (_w, _p) for _w, _p in zip(weights, inputs))
info['prior']['SZ'] = \
"lambda %s: stats.norm.logpdf(%s, loc=%g, scale=%g)" % (
",".join(inputs), linear, mean, std)
if os.path.exists(root + '.likelihoods'):
info_like: LikesDict = {}
info['likelihood'] = info_like
with open(root + '.likelihoods', 'r') as f:
for line in f.readlines():
if line.strip():
like = line.split()[2]
info_like[like] = None
else:
print('You need to mention in the likelihood block with with "name: None"'
'for each likelihood in the input chain')
info['output'] = root
return info