class DatasetLikelihood(object): def __init__(self, fname, dataset_params={}, field_names=['T', 'E', 'B', 'P'], map_separator='x'): self.field_names = field_names self.tot_theory_fields = len(field_names) self.map_separator = map_separator # aberration will be corrected if aberration_coeff is non - zero self.aberration_coeff = 0.0 self.log_calibration_prior = -1 # if >0 use log prior on calibration parameter if '.dataset' in fname: self.loadDataset(fname, dataset_params) else: raise Exception('DatasetLikelihood only supports .dataset files') def typeIndex(self, field): return self.field_names.index(field) def PairStringToMapIndices(self, S): if len(S) == 2: if self.has_map_names: raise Exception('CMBlikes: CL names must use MAP1xMAP2 names') return self.map_names.index(S[0]), self.map_names.index(S[1]) else: try: i = S.index(self.map_separator) except ValueError: raise ValueError('CMBLikes: invalid spectrum name %s' % S) return self.map_names.index(S[0:i]), self.map_names.index(S[i + 1:]) def PairStringToUsedMapIndices(self, used_index, S): i1, i2 = self.PairStringToMapIndices(S) i1 = used_index[i1] i2 = used_index[i2] if i2 > i1: return i2, i1 else: return i1, i2 def UseString_to_cols(self, L): cl_i_j = self.UseString_to_Cl_i_j(L, self.map_used_index) cols = -np.ones(cl_i_j.shape[1], dtype=int) for i in range(cl_i_j.shape[1]): i1, i2 = cl_i_j[:, i] if i1 == -1 or i2 == -1: continue ix = 0 for ii in range(self.nmaps): for jj in range(ii + 1): if ii == i1 and jj == i2: cols[i] = ix ix += 1 return cols def UseString_to_Cl_i_j(self, S, used_index): if not isinstance(S, (list, tuple)): S = S.split() cl_i_j = np.zeros((2, len(S)), dtype=int) for i, p in enumerate(S): cl_i_j[:, i] = self.PairStringToUsedMapIndices(used_index, p) return cl_i_j def MapPair_to_Theory_i_j(self, order, pair): i = self.map_fields[order[pair[0]]] j = self.map_fields[order[pair[1]]] if i <= j: return i, j else: return j, i def Cl_used_i_j_name(self, pair): return self.Cl_i_j_name(self.used_map_order, pair) def Cl_i_j_name(self, names, pair): name1 = names[pair[0]] name2 = names[pair[1]] if self.has_map_names: return name1 + self.map_separator + name2 else: return name1 + name2 def GetColsFromOrder(self, order): # Converts string Order = TT TE EE XY... or AAAxBBB AAAxCCC BBxCC # into indices into array of power spectra (and -1 if not present) cols = np.empty(self.ncl, dtype=int) cols[:] = -1 names = order.strip().split() ix = 0 for i in range(self.nmaps): for j in range(i + 1): name = self.Cl_used_i_j_name([i, j]) if not name in names and i != j: name = self.Cl_used_i_j_name([j, i]) if name in names: if cols[ix] != -1: raise Exception('GetColsFromOrder: duplicate CL type') cols[ix] = names.index(name) ix += 1 return cols def elements_to_matrix(self, X, M): ix = 0 for i in range(self.nmaps): M[i, 0:i] = X[ix:ix + i] M[0:i, i] = X[ix:ix + i] ix += i M[i, i] = X[ix] ix += 1 def matrix_to_elements(self, M, X): ix = 0 for i in range(self.nmaps): X[ix:ix + i + 1] = M[i, 0:i + 1] ix += i + 1 def ReadClArr(self, ini, file_stem, return_full=False): # read file of CL or bins (indexed by L) filename = ini.relativeFileName(file_stem + '_file') cl = np.zeros((self.ncl, self.nbins_used)) order = ini.string(file_stem + '_order', '') if not order: incols = lastTopComment(filename) if not incols: raise Exception('No column order given for ' + filename) else: incols = 'L ' + order cols = self.GetColsFromOrder(incols) data = np.loadtxt(filename) Ls = data[:, 0].astype(int) if self.binned: Ls -= 1 for i, L in enumerate(Ls): if L >= self.bin_min and L <= self.bin_max: for ix in range(self.ncl): if cols[ix] != -1: cl[ix, L - self.bin_min] = data[i, cols[ix]] if L < self.bin_max: raise Exception('CMBLikes_ReadClArr: C_l file does not go up to maximum used: %s' % self.bin_max) if return_full: return incols.split(), data, cl else: return cl def readBinWindows(self, ini, file_stem): bins = BinWindows(self.pcl_lmin, self.pcl_lmax, self.nbins_used) in_cl = ini.split(file_stem + '_in_order') out_cl = ini.split(file_stem + '_out_order', in_cl) bins.cols_in = self.UseString_to_Cl_i_j(in_cl, self.map_required_index) bins.cols_out = self.UseString_to_cols(out_cl) norder = bins.cols_in.shape[1] if norder != bins.cols_out.shape[0]: raise Exception('_in_order and _out_order must have same number of entries') bins.binning_matrix = np.zeros((norder, self.nbins_used, self.pcl_lmax - self.pcl_lmin + 1)) windows = ini.relativeFileName(file_stem + '_files') for b in range(self.nbins_used): window = np.loadtxt(windows % (b + 1 + self.bin_min)) Err = False for i, L in enumerate(window[:, 0].astype(int)): if self.pcl_lmin <= L <= self.pcl_lmax: bins.binning_matrix[:, b, L - self.pcl_lmin] = window[i, 1:] else: Err = Err or any(window[i, 1:] != 0) if Err: print('WARNING: %s %u outside pcl_lmin-cl_max range: %s' % (file_stem, b, windows % (b + 1))) if ini.hasKey(file_stem + '_fix_cl_file'): raise Exception('fix_cl_file not implemented yet') return bins def init_map_cls(self, nmaps, order): if nmaps != len(order): raise ValueError('CMBLikes init_map_cls: size mismatch') class CrossPowerSpectrum(object): pass cls = np.empty((nmaps, nmaps), dtype=object) for i in range(nmaps): for j in range(i + 1): CL = CrossPowerSpectrum() cls[i, j] = CL CL.map_ij = [order[i], order[j]] CL.theory_ij = self.MapPair_to_Theory_i_j(order, [i, j]) CL.CL = np.zeros(self.pcl_lmax - self.pcl_lmin + 1) return cls def loadDataset(self, froot, dataset_params): if not '.dataset' in froot: froot += '.dataset' ini = IniFile(froot) ini.params.update(dataset_params) self.readIni(ini) def readIni(self, ini): self.map_names = ini.split('map_names', default=[]) self.has_map_names = len(self.map_names) if self.has_map_names: # e.g. have multiple frequencies for given field measurement map_fields = ini.split('map_fields') if len(map_fields) != len(self.map_names): raise Exception('CMBLikes: number of map_fields does not match map_names') self.map_fields = [self.typeIndex(f) for f in map_fields] else: self.map_names = self.field_names self.map_fields = np.arange(len(self.map_names), dtype=int) fields_use = ini.split('fields_use', []) if len(fields_use): index_use = [self.typeIndex(f) for f in fields_use] use_theory_field = [i in index_use for i in range(self.tot_theory_fields)] else: if not self.has_map_names: raise Exception('CMBlikes: must have fields_use or map_names') use_theory_field = [True] * self.tot_theory_fields maps_use = ini.split('maps_use', []) if len(maps_use): if np.any([not i for i in use_theory_field]): print('CMBlikes WARNING: maps_use overrides fields_use') self.use_map = np.zeros(len(self.map_names), dtype=bool) for j, map_used in enumerate(maps_use): if map_used in self.map_names: self.use_map[self.map_names.index(map_used)] = True else: raise ValueError('CMBlikes: maps_use item not found - %s' % map_used) else: self.use_map = [use_theory_field[self.map_fields[i]] for i in range(len(self.map_names))] # Bandpowers can depend on more fields than are actually used in likelihood # e.g. for correcting leakage or other linear corrections self.require_map = self.use_map[:] if self.has_map_names: if ini.hasKey('fields_required'): raise Exception('CMBLikes: use maps_required not fields_required') maps_use = ini.split('maps_required', []) else: maps_use = ini.split('fields_required', []) if len(maps_use): for j, map_used in enumerate(maps_use): if map_used in self.map_names: self.require_map[self.map_names.index(map_used)] = True else: raise ValueError('CMBlikes: required item not found %s' % map_used) self.required_theory_field = [False for _ in self.field_names] for i in range(len(self.map_names)): if self.require_map[i]: self.required_theory_field[self.map_fields[i]] = True self.ncl_used = 0 # set later reading covmat self.like_approx = ini.string('like_approx', 'gaussian') self.nmaps = np.count_nonzero(self.use_map) self.nmaps_required = np.count_nonzero(self.require_map) self.required_order = np.zeros(self.nmaps_required, dtype=int) self.map_required_index = -np.ones(len(self.map_names), dtype=int) ix = 0 for i in range(len(self.map_names)): if self.require_map[i]: self.map_required_index[i] = ix self.required_order[ix] = i ix += 1 self.map_used_index = -np.ones(len(self.map_names), dtype=int) ix = 0 self.used_map_order = [] for i, map_name in enumerate(self.map_names): if self.use_map[i]: self.map_used_index[i] = ix self.used_map_order.append(map_name) ix += 1 self.ncl = (self.nmaps * (self.nmaps + 1)) // 2 self.pcl_lmax = ini.int('cl_lmax') self.pcl_lmin = ini.int('cl_lmin') self.binned = ini.bool('binned', True) if self.binned: self.nbins = ini.int('nbins') self.bin_min = ini.int('use_min', 1) - 1 self.bin_max = ini.int('use_max', self.nbins) - 1 self.nbins_used = self.bin_max - self.bin_min + 1 # needed by readBinWindows self.bins = self.readBinWindows(ini, 'bin_window') else: if self.nmaps != self.nmaps_required: raise Exception('CMBlikes: unbinned likelihood must have nmaps==nmaps_required') self.nbins = self.pcl_lmax - self.pcl_lmin + 1 if self.like_approx != 'exact': print('WARNING: Unbinned likelihoods untested in this version') self.bin_min = ini.int('use_min', self.pcl_lmin) self.bin_max = ini.int('use_max', self.pcl_lmax) self.nbins_used = self.bin_max - self.bin_min + 1 self.full_bandpower_headers, self.full_bandpowers, self.bandpowers = \ self.ReadClArr(ini, 'cl_hat', return_full=True) if self.like_approx == 'HL': self.cl_fiducial = self.ReadClArr(ini, 'cl_fiducial') else: self.cl_fiducial = None includes_noise = ini.bool('cl_hat_includes_noise', False) self.cl_noise = None if self.like_approx != 'gaussian' or includes_noise: self.cl_noise = self.ReadClArr(ini, 'cl_noise') if not includes_noise: self.bandpowers += self.cl_noise elif self.like_approx == 'gaussian': self.bandpowers -= self.cl_noise self.cl_lmax = np.zeros((self.tot_theory_fields, self.tot_theory_fields)) for i in range(self.tot_theory_fields): if self.required_theory_field[i]: self.cl_lmax[i, i] = self.pcl_lmax if self.required_theory_field[0] and self.required_theory_field[1]: self.cl_lmax[1, 0] = self.pcl_lmax if self.like_approx != 'gaussian': cl_fiducial_includes_noise = ini.bool('cl_fiducial_includes_noise', False) self.bandpower_matrix = np.zeros((self.nbins_used, self.nmaps, self.nmaps)) self.noise_matrix = self.bandpower_matrix.copy() self.fiducial_sqrt_matrix = self.bandpower_matrix.copy() if self.cl_fiducial is not None and not cl_fiducial_includes_noise: self.cl_fiducial += self.cl_noise for b in range(self.nbins_used): self.elements_to_matrix(self.bandpowers[:, b], self.bandpower_matrix[b, :, :]) if self.cl_noise is not None: self.elements_to_matrix(self.cl_noise[:, b], self.noise_matrix[b, :, :]) if self.cl_fiducial is not None: self.elements_to_matrix(self.cl_fiducial[:, b], self.fiducial_sqrt_matrix[b, :, :]) self.fiducial_sqrt_matrix[b, :, :] = sqrtm(self.fiducial_sqrt_matrix[b, :, :]) if self.like_approx == 'exact': self.fsky = ini.float('fullsky_exact_fksy') else: self.cov = self.ReadCovmat(ini) self.covinv = np.linalg.inv(self.cov) if 'linear_correction_fiducial_file' in ini.params: self.fid_correction = self.ReadClArr(ini, 'linear_correction_fiducial') self.linear_correction = self.readBinWindows(ini, 'linear_correction_bin_window') else: self.linear_correction = None if ini.hasKey('nuisance_params'): s = ini.relativeFileName('nuisance_params') self.nuisance_params = ParamNames(s) if ini.hasKey('calibration_param'): raise Exception('calibration_param not allowed with nuisance_params') if ini.hasKey('calibration_paramname'): self.calibration_param = ini.string('calibration_paramname') else: self.calibration_param = None elif ini.string('calibration_param', ''): s = ini.relativeFileName('calibration_param') if not '.paramnames' in s: raise Exception('calibration_param must be paramnames file unless nuisance_params also specified') self.nuisance_params = ParamNames(s) self.calibration_param = self.nuisance_params.list()[0] else: self.calibration_param = None if self.calibration_param is not None: self.log_calibration_prior = ini.float('log_calibration_prior', -1) self.aberration_coeff = ini.float('aberration_coeff', 0.0) self.map_cls = self.init_map_cls(self.nmaps_required, self.required_order) def ReadCovmat(self, ini): # read the covariance matrix, and the array of which CL are in the covariance, # which then defines which set of bandpowers are used (subject to other restrictions) covmat_cl = ini.string('covmat_cl', allowEmpty=False) if ini.string('covmat_format', 'text') != 'text': raise Exception('Only text oovmat supported in python so far') self.full_cov = np.loadtxt(ini.relativeFileName('covmat_fiducial')) covmat_scale = ini.float('covmat_scale', 1.0) cl_in_index = self.UseString_to_cols(covmat_cl) self.ncl_used = np.sum(cl_in_index >= 0) self.cl_used_index = np.zeros(self.ncl_used, dtype=int) cov_cl_used = np.zeros(self.ncl_used, dtype=int) ix = 0 for i, index in enumerate(cl_in_index): if index >= 0: self.cl_used_index[ix] = index cov_cl_used[ix] = i ix += 1 if self.binned: num_in = len(cl_in_index) pcov = np.empty((self.nbins_used * self.ncl_used, self.nbins_used * self.ncl_used)) for binx in range(self.nbins_used): for biny in range(self.nbins_used): pcov[binx * self.ncl_used: (binx + 1) * self.ncl_used, biny * self.ncl_used: (biny + 1) * self.ncl_used] = \ covmat_scale * self.full_cov[np.ix_((binx + self.bin_min) * num_in + cov_cl_used, (biny + self.bin_min) * num_in + cov_cl_used)] else: raise Exception('unbinned covariance not implemented') return pcov def get_binned_theory(self, ClArray, data_params={}): # Useful for plotting, not used for likelihood self.get_theory_map_cls(ClArray, data_params) return self.get_binned_map_cls(self.map_cls) def get_full_bandpower_column(self, col_name): """ Get columns from the input bandpower file. Note used by likelihood but may be useful for plotting. :param col_name: name of column, as in the top comment header of the file :return: column values """ return self.full_bandpowers[:, self.full_bandpower_headers.index(col_name)] def diag_sigma(self): return np.sqrt(np.diag(self.full_cov)) def plot(self, column='PP', ClArray=None, ls=None, ax=None): lbin = self.full_bandpowers[:, self.full_bandpower_headers.index('L_av')] binned_phicl_err = self.diag_sigma() ax = ax or plt.gca() bandpowers = self.full_bandpowers[:, self.full_bandpower_headers.index('PP')] if 'L_min' in self.full_bandpower_headers: lmin = self.full_bandpowers[:, self.full_bandpower_headers.index('L_min')] lmax = self.full_bandpowers[:, self.full_bandpower_headers.index('L_max')] ax.errorbar(lbin, bandpowers, yerr=binned_phicl_err, xerr=[lbin - lmin, lmax - lbin], fmt='o') else: ax.errorbar(lbin, bandpowers, yerr=binned_phicl_err, fmt='o') if ClArray is not None: if isinstance(ClArray, ClsArray): i, j = self.MapPair_to_Theory_i_j(range(len(self.map_names)), self.PairStringToMapIndices(column)) ClArray = ClArray.get([i, j]) if ls is None: ls = np.arange(len(ClArray)) ax.plot(ls, ClArray, color='k') ax.set_xlim([2, ls[-1]]) def get_binned_map_cls(self, Cls, corrections=True): band = self.bins.bin(Cls) if self.linear_correction is not None and corrections: band += self.linear_correction.bin(Cls) - self.fid_correction.T return band def get_theory_map_cls(self, ClArray, data_params={}): for i in range(self.nmaps_required): for j in range(i + 1): CL = self.map_cls[i, j] cls = ClArray.get(CL.theory_ij) if cls is not None: CL.CL[:] = cls[self.pcl_lmin:self.pcl_lmax + 1] else: CL.CL[:] = 0 self.adapt_theory_for_maps(self.map_cls, data_params) def adapt_theory_for_maps(self, cls, data_params): if self.aberration_coeff: self.add_aberration(cls) self.add_foregrounds(cls, data_params) if self.calibration_param is not None and self.calibration_param in data_params: for i in range(self.nmaps_required): for j in range(i + 1): CL = cls[i, j] if CL is not None: if CL.theory_ij[0] <= 2 and CL.theory_ij[1] <= 2: CL.CL /= data_params[self.calibration_param] ** 2 def add_foregrounds(self, cls, data_params): pass def add_aberration(self, cls): # adapted from CosmoMC function by Christian Reichardt ells = np.arange(self.pcl_lmin, self.pcl_lmax + 1) cl_norm = ells * (ells + 1) for i in range(self.nmaps_required): for j in range(i + 1): CL = cls[i, j] if CL is not None: if CL.theory_ij[0] <= 2 and CL.theory_ij[1] <= 2: # first get Cl instead of Dl cl_deriv = CL.CL / cl_norm # second take derivative dCl/dl cl_deriv[1:-1] = (cl_deriv[2:] - cl_deriv[:-2]) / 2 # handle endpoints approximately cl_deriv[0] = cl_deriv[1] cl_deriv[-1] = cl_deriv[-2] # reapply to Dl's. # note never took 2pi out, so not putting it back either cl_deriv *= cl_norm # also multiply by ell since really wanted ldCl/dl cl_deriv *= ells CL.CL += self.aberration_coeff * cl_deriv def write_likelihood_data(self, filename, data_params={}): cls = self.init_map_cls(self.nmaps_required, self.required_order) self.add_foregrounds(cls, data_params) with open(filename, 'w') as f: cols = [] for i in range(self.nmaps_required): for j in range(i + 1): cols.append(self.Cl_i_j_name(self.map_names, cls[i, j].map_ij)) f.write('# L' + ("%17s " * len(cols)) % tuple(cols) + '\n') for b in range(self.pcl_lmin, self.pcl_lmax + 1): c = [b] for i in range(self.nmaps_required): for j in range(i + 1): c.append(cls[i, j].CL[b - self.pcl_lmin]) f.write(("%I5 " + "%17.8e " * len(cols)) % tuple(c)) def transform(self, C, Chat, Cfhalf): # HL transformation of the matrices if C.shape[0] == 1: rat = Chat[0, 0] / C[0, 0] C[0, 0] = np.sign(rat - 1) * np.sqrt(2 * np.maximum(0, rat - np.log(rat) - 1)) * Cfhalf[0, 0] ** 2 return diag, U = np.linalg.eigh(C) rot = U.T.dot(Chat).dot(U) roots = np.sqrt(diag) for i, root in enumerate(roots): rot[i, :] /= root rot[:, i] /= root U.dot(rot.dot(U.T), rot) diag, rot = np.linalg.eigh(rot) diag = np.sign(diag - 1) * np.sqrt(2 * np.maximum(0, diag - np.log(diag) - 1)) Cfhalf.dot(rot, U) for i, d in enumerate(diag): rot[:, i] = U[:, i] * d rot.dot(U.T, C) def exact_chi_sq(self, C, Chat, L): if C.shape[0] == 1: return (2 * L + 1) * self.fsky * (Chat[0, 0] / C[0, 0] - 1 - np.log(Chat[0, 0] / C[0, 0])) else: M = np.linalg.inv(C).dot(Chat) return (2 * L + 1) * self.fsky * (np.trace(M) - self.nmaps - np.linalg.slogdet(M)[1]) def chi_squared(self, ClArray, data_params={}, return_binned_theory=False): self.get_theory_map_cls(ClArray, data_params) C = np.empty((self.nmaps, self.nmaps)) bigX = np.empty(self.nbins_used * self.ncl_used) vecp = np.empty(self.ncl) chisq = 0 if self.binned: binned_theory = self.get_binned_map_cls(self.map_cls) else: Cs = np.zeros((self.nbins_used, self.nmaps, self.nmaps)) for i in range(self.nmaps): for j in range(i + 1): CL = self.map_cls[i, j] if CL is not None: Cs[:, i, j] = CL.CL[self.bin_min - self.pcl_lmin:self.bin_max - self.pcl_lmin + 1] Cs[:, j, i] = CL.CL[self.bin_min - self.pcl_lmin:self.bin_max - self.pcl_lmin + 1] for bin in range(self.nbins_used): if self.binned: self.elements_to_matrix(binned_theory[bin, :], C) else: C[:, :] = Cs[bin, :, :] if self.cl_noise is not None: C += self.noise_matrix[bin] if self.like_approx == 'exact': chisq += self.exact_chi_sq(C, self.bandpower_matrix[bin], self.bin_min + bin) continue elif self.like_approx == 'HL': self.transform(C, self.bandpower_matrix[bin], self.fiducial_sqrt_matrix[bin]) elif self.like_approx == 'gaussian': C -= self.bandpower_matrix[bin] self.matrix_to_elements(C, vecp) bigX[bin * self.ncl_used:(bin + 1) * self.ncl_used] = vecp[self.cl_used_index] if self.like_approx != 'exact': chisq = fast_chi_squared(self.covinv, bigX) if self.log_calibration_prior > 0: chisq += (np.log(data_params[self.calibration_param]) / self.log_calibration_prior) ** 2 if return_binned_theory and self.binned: return chisq, binned_theory else: return chisq
class _CMBlikes(_DataSetLikelihood): # Data type for aggregated chi2 (case sensitive) type = "CMB" map_separator: str lmin: Sequence[int] lmax: Sequence[int] lav: Sequence[int] Ahat: np.ndarray # not used by likelihood def get_requirements(self): # State requisites to the theory code requested_cls = [ self.field_names[i] + self.field_names[j] for i, j in zip(*np.where(self.cl_lmax != 0)) ] # l_max has to take into account the window function of the lensing # so we check the computed l_max ("l_max" option) is higher than the requested one requested_l_max = int(np.max(self.cl_lmax)) if (getattr(self, "l_max", None) or np.inf) < requested_l_max: raise LoggedError( self.log, "You are setting a very low l_max. " "The likelihood value will probably not be correct. " "Make sure to make 'l_max'>=%d", requested_l_max) self.l_max = max(requested_l_max, getattr(self, "l_max", 0) or 0) return {"Cl": {cl: self.l_max for cl in requested_cls}} def typeIndex(self, field): return self.field_names.index(field) def PairStringToMapIndices(self, S): if len(S) == 2: if self.has_map_names: raise LoggedError(self.log, 'CL names must use MAP1xMAP2 names') return self.map_names.index(S[0]), self.map_names.index(S[1]) else: try: i = S.index(self.map_separator) except ValueError: raise LoggedError(self.log, 'invalid spectrum name %s' % S) return self.map_names.index(S[0:i]), self.map_names.index(S[i + 1:]) def PairStringToUsedMapIndices(self, used_index, S): i1, i2 = self.PairStringToMapIndices(S) i1 = used_index[i1] i2 = used_index[i2] if i2 > i1: return i2, i1 else: return i1, i2 def UseString_to_cols(self, L): cl_i_j = self.UseString_to_Cl_i_j(L, self.map_used_index) cols = -np.ones(cl_i_j.shape[1], dtype=int) for i in range(cl_i_j.shape[1]): i1, i2 = cl_i_j[:, i] if i1 == -1 or i2 == -1: continue ix = 0 for ii in range(self.nmaps): for jj in range(ii + 1): if ii == i1 and jj == i2: cols[i] = ix ix += 1 return cols def UseString_to_Cl_i_j(self, S, used_index): if not isinstance(S, (list, tuple)): S = S.split() cl_i_j = np.zeros((2, len(S)), dtype=int) for i, p in enumerate(S): cl_i_j[:, i] = self.PairStringToUsedMapIndices(used_index, p) return cl_i_j def MapPair_to_Theory_i_j(self, order, pair): i = self.map_fields[order[pair[0]]] j = self.map_fields[order[pair[1]]] if i <= j: return i, j else: return j, i def Cl_used_i_j_name(self, pair): return self.Cl_i_j_name(self.used_map_order, pair) def Cl_i_j_name(self, names, pair): name1 = names[pair[0]] name2 = names[pair[1]] if self.has_map_names: return name1 + self.map_separator + name2 else: return name1 + name2 def get_cols_from_order(self, order): # Converts string Order = TT TE EE XY... or AAAxBBB AAAxCCC BBxCC # into indices into array of power spectra (and -1 if not present) cols = np.empty(self.ncl, dtype=int) cols[:] = -1 names = order.strip().split() ix = 0 for i in range(self.nmaps): for j in range(i + 1): name = self.Cl_used_i_j_name([i, j]) if name not in names and i != j: name = self.Cl_used_i_j_name([j, i]) if name in names: if cols[ix] != -1: raise LoggedError( self.log, 'get_cols_from_order: duplicate CL type') cols[ix] = names.index(name) ix += 1 return cols def elements_to_matrix(self, X, M): ix = 0 for i in range(self.nmaps): M[i, 0:i] = X[ix:ix + i] M[0:i, i] = X[ix:ix + i] ix += i M[i, i] = X[ix] ix += 1 def matrix_to_elements(self, M, X): ix = 0 for i in range(self.nmaps): X[ix:ix + i + 1] = M[i, 0:i + 1] ix += i + 1 def read_cl_array(self, ini, file_stem, return_full=False): # read file of CL or bins (indexed by L) filename = ini.relativeFileName(file_stem + '_file') cl = np.zeros((self.ncl, self.nbins_used)) order = ini.string(file_stem + '_order', '') if not order: incols = last_top_comment(filename) if not incols: raise LoggedError(self.log, 'No column order given for ' + filename) else: incols = 'L ' + order cols = self.get_cols_from_order(incols) data = np.loadtxt(filename) Ls = data[:, 0].astype(int) if self.binned: Ls -= 1 for i, L in enumerate(Ls): if self.bin_min <= L <= self.bin_max: for ix in range(self.ncl): if cols[ix] != -1: cl[ix, L - self.bin_min] = data[i, cols[ix]] if Ls[-1] < self.bin_max: raise LoggedError( self.log, 'CMBLikes_ReadClArr: C_l file does not go up to maximum used: ' '%s', self.bin_max) if return_full: return incols.split(), data, cl else: return cl def read_bin_windows(self, ini, file_stem): bins = BinWindows(self.pcl_lmin, self.pcl_lmax, self.nbins_used, self.ncl) in_cl = ini.split(file_stem + '_in_order') out_cl = ini.split(file_stem + '_out_order', in_cl) bins.cols_in = self.UseString_to_Cl_i_j(in_cl, self.map_required_index) bins.cols_out = self.UseString_to_cols(out_cl) norder = bins.cols_in.shape[1] if norder != bins.cols_out.shape[0]: raise LoggedError( self.log, '_in_order and _out_order must have same number of entries') bins.binning_matrix = np.zeros( (norder, self.nbins_used, self.pcl_lmax - self.pcl_lmin + 1)) windows = ini.relativeFileName(file_stem + '_files') for b in range(self.nbins_used): window = np.loadtxt(windows % (b + 1 + self.bin_min)) err = False for i, L in enumerate(window[:, 0].astype(int)): if self.pcl_lmin <= L <= self.pcl_lmax: bins.binning_matrix[:, b, L - self.pcl_lmin] = window[i, 1:] else: err = err or any(window[i, 1:] != 0) if err: self.log.warning('%s %u outside pcl_lmin-cl_max range: %s' % (file_stem, b, windows % (b + 1))) if ini.hasKey(file_stem + '_fix_cl_file'): raise LoggedError(self.log, 'fix_cl_file not implemented yet') return bins def init_map_cls(self, nmaps, order): if nmaps != len(order): raise LoggedError(self.log, 'init_map_cls: size mismatch') class CrossPowerSpectrum: pass cls = np.empty((nmaps, nmaps), dtype=object) for i in range(nmaps): for j in range(i + 1): CL = CrossPowerSpectrum() cls[i, j] = CL CL.map_ij = [order[i], order[j]] CL.theory_ij = self.MapPair_to_Theory_i_j(order, [i, j]) CL.CL = np.zeros(self.pcl_lmax - self.pcl_lmin + 1) return cls def init_params(self, ini): self.field_names = getattr(self, 'field_names', ['T', 'E', 'B', 'P']) self.tot_theory_fields = len(self.field_names) self.map_names = ini.split('map_names', default=[]) self.has_map_names = bool(self.map_names) if self.has_map_names: # e.g. have multiple frequencies for given field measurement map_fields = ini.split('map_fields') if len(map_fields) != len(self.map_names): raise LoggedError( self.log, 'number of map_fields does not match map_names') self.map_fields = [self.typeIndex(f) for f in map_fields] else: self.map_names = self.field_names self.map_fields = np.arange(len(self.map_names), dtype=int) fields_use = ini.split('fields_use', []) if len(fields_use): index_use = [self.typeIndex(f) for f in fields_use] use_theory_field = [ i in index_use for i in range(self.tot_theory_fields) ] else: if not self.has_map_names: raise LoggedError(self.log, 'must have fields_use or map_names') use_theory_field = [True] * self.tot_theory_fields maps_use = ini.split('maps_use', []) if len(maps_use): if np.any(not i for i in use_theory_field): self.log.warning('maps_use overrides fields_use') self.use_map = np.zeros(len(self.map_names), dtype=bool) for j, map_used in enumerate(maps_use): if map_used in self.map_names: self.use_map[self.map_names.index(map_used)] = True else: raise LoggedError( self.log, 'maps_use item not found - %s' % map_used) else: self.use_map = [ use_theory_field[self.map_fields[i]] for i in range(len(self.map_names)) ] # Bandpowers can depend on more fields than are actually used in likelihood # e.g. for correcting leakage or other linear corrections self.require_map = self.use_map[:] if self.has_map_names: if ini.hasKey('fields_required'): raise LoggedError(self.log, 'use maps_required not fields_required') maps_use = ini.split('maps_required', []) else: maps_use = ini.split('fields_required', []) if len(maps_use): for j, map_used in enumerate(maps_use): if map_used in self.map_names: self.require_map[self.map_names.index(map_used)] = True else: raise LoggedError(self.log, 'required item not found %s' % map_used) self.required_theory_field = [False for _ in self.field_names] for i in range(len(self.map_names)): if self.require_map[i]: self.required_theory_field[self.map_fields[i]] = True self.ncl_used = 0 # set later reading covmat self.like_approx = ini.string('like_approx', 'gaussian') self.nmaps = np.count_nonzero(self.use_map) self.nmaps_required = np.count_nonzero(self.require_map) self.required_order = np.zeros(self.nmaps_required, dtype=int) self.map_required_index = -np.ones(len(self.map_names), dtype=int) ix = 0 for i in range(len(self.map_names)): if self.require_map[i]: self.map_required_index[i] = ix self.required_order[ix] = i ix += 1 self.map_used_index = -np.ones(len(self.map_names), dtype=int) ix = 0 self.used_map_order = [] for i, map_name in enumerate(self.map_names): if self.use_map[i]: self.map_used_index[i] = ix self.used_map_order.append(map_name) ix += 1 self.ncl = (self.nmaps * (self.nmaps + 1)) // 2 self.pcl_lmax = ini.int('cl_lmax') self.pcl_lmin = ini.int('cl_lmin') self.binned = ini.bool('binned', True) if self.binned: self.nbins = ini.int('nbins') self.bin_min = ini.int('use_min', 1) - 1 self.bin_max = ini.int('use_max', self.nbins) - 1 self.nbins_used = self.bin_max - self.bin_min + 1 # needed by read_bin_windows self.bins = self.read_bin_windows(ini, 'bin_window') else: if self.nmaps != self.nmaps_required: raise LoggedError( self.log, 'unbinned likelihood must have nmaps==nmaps_required') self.nbins = self.pcl_lmax - self.pcl_lmin + 1 if self.like_approx != 'exact': self.log.warning( 'Unbinned likelihoods untested in this version') self.bin_min = ini.int('use_min', self.pcl_lmin) self.bin_max = ini.int('use_max', self.pcl_lmax) self.nbins_used = self.bin_max - self.bin_min + 1 self.full_bandpower_headers, self.full_bandpowers, self.bandpowers = \ self.read_cl_array(ini, 'cl_hat', return_full=True) if self.like_approx == 'HL': self.cl_fiducial = self.read_cl_array(ini, 'cl_fiducial') else: self.cl_fiducial = None includes_noise = ini.bool('cl_hat_includes_noise', False) self.cl_noise = None if self.like_approx != 'gaussian' or includes_noise: self.cl_noise = self.read_cl_array(ini, 'cl_noise') if not includes_noise: self.bandpowers += self.cl_noise elif self.like_approx == 'gaussian': self.bandpowers -= self.cl_noise self.cl_lmax = np.zeros( (self.tot_theory_fields, self.tot_theory_fields)) for i in range(self.tot_theory_fields): if self.required_theory_field[i]: self.cl_lmax[i, i] = self.pcl_lmax if self.required_theory_field[0] and self.required_theory_field[1]: self.cl_lmax[1, 0] = self.pcl_lmax if self.like_approx != 'gaussian': cl_fiducial_includes_noise = ini.bool('cl_fiducial_includes_noise', False) else: cl_fiducial_includes_noise = False self.bandpower_matrix = np.zeros( (self.nbins_used, self.nmaps, self.nmaps)) self.noise_matrix = self.bandpower_matrix.copy() self.fiducial_sqrt_matrix = self.bandpower_matrix.copy() if self.cl_fiducial is not None and not cl_fiducial_includes_noise: self.cl_fiducial += self.cl_noise for b in range(self.nbins_used): self.elements_to_matrix(self.bandpowers[:, b], self.bandpower_matrix[b, :, :]) if self.cl_noise is not None: self.elements_to_matrix(self.cl_noise[:, b], self.noise_matrix[b, :, :]) if self.cl_fiducial is not None: self.elements_to_matrix(self.cl_fiducial[:, b], self.fiducial_sqrt_matrix[b, :, :]) self.fiducial_sqrt_matrix[b, :, :] = (sqrtm( self.fiducial_sqrt_matrix[b, :, :])) if self.like_approx == 'exact': self.fsky = ini.float('fullsky_exact_fksy') else: self.cov = self.ReadCovmat(ini) self.covinv = np.linalg.inv(self.cov) if 'linear_correction_fiducial_file' in ini.params: self.fid_correction = self.read_cl_array( ini, 'linear_correction_fiducial') self.linear_correction = self.read_bin_windows( ini, 'linear_correction_bin_window') else: self.linear_correction = None if ini.hasKey('nuisance_params'): s = ini.relativeFileName('nuisance_params') self.nuisance_params = ParamNames(s) if ini.hasKey('calibration_param'): raise Exception( 'calibration_param not allowed with nuisance_params') if ini.hasKey('calibration_paramname'): self.calibration_param = ini.string('calibration_paramname') else: self.calibration_param = None elif ini.string('calibration_param', ''): s = ini.relativeFileName('calibration_param') if '.paramnames' not in s: raise LoggedError( self.log, 'calibration_param must be paramnames file unless ' 'nuisance_params also specified') self.nuisance_params = ParamNames(s) self.calibration_param = self.nuisance_params.list()[0] else: self.calibration_param = None if ini.hasKey('log_calibration_prior'): self.log.warning('log_calibration_prior in .dataset ignored, ' 'set separately in .yaml file') self.aberration_coeff = ini.float('aberration_coeff', 0.0) self.map_cls = self.init_map_cls(self.nmaps_required, self.required_order) def ReadCovmat(self, ini): """Read the covariance matrix, and the array of which CL are in the covariance, which then defines which set of bandpowers are used (subject to other restrictions).""" covmat_cl = ini.string('covmat_cl', allowEmpty=False) self.full_cov = np.loadtxt(ini.relativeFileName('covmat_fiducial')) covmat_scale = ini.float('covmat_scale', 1.0) cl_in_index = self.UseString_to_cols(covmat_cl) self.ncl_used = np.sum(cl_in_index >= 0) self.cl_used_index = np.zeros(self.ncl_used, dtype=int) cov_cl_used = np.zeros(self.ncl_used, dtype=int) ix = 0 for i, index in enumerate(cl_in_index): if index >= 0: self.cl_used_index[ix] = index cov_cl_used[ix] = i ix += 1 if self.binned: num_in = len(cl_in_index) pcov = np.empty((self.nbins_used * self.ncl_used, self.nbins_used * self.ncl_used)) for binx in range(self.nbins_used): for biny in range(self.nbins_used): pcov[binx * self.ncl_used:(binx + 1) * self.ncl_used, biny * self.ncl_used:(biny + 1) * self.ncl_used] = (covmat_scale * self.full_cov[np.ix_( (binx + self.bin_min) * num_in + cov_cl_used, (biny + self.bin_min) * num_in + cov_cl_used)]) else: raise LoggedError(self.log, 'unbinned covariance not implemented yet') return pcov def writeData(self, froot): np.savetxt(froot + '_cov.dat', self.cov) # self.saveCl(froot + '_fid_cl.dat', self.fid_cl[:, 1:], # cols=['TT', 'EE', 'TE', 'PP']) with open(froot + '_bandpowers.dat', 'w', encoding="utf-8") as f: f.write("#%4s %5s %5s %8s %12s %10s %7s\n" % ('bin', 'L_min', 'L_max', 'L_av', 'PP', 'Error', 'Ahat')) for b in range(self.nbins): f.write("%5u %5u %5u %8.2f %12.5e %10.3e %7.3f\n" % (b + 1, self.lmin[b], self.lmax[b], self.lav[b], self.bandpowers[b], np.sqrt( self.cov[b, b]), self.Ahat[b])) self.bins.write(froot, 'bin') if self.linear_correction is not None: self.linear_correction.write(froot, 'linear_correction_bin') with open(froot + '_lensing_fiducial_correction', 'w', encoding="utf-8") as f: f.write("#%4s %12s \n" % ('bin', 'PP')) for b in range(self.nbins): f.write("%5u %12.5e\n" % (b + 1, self.fid_correction[b])) def diag_sigma(self): return np.sqrt(np.diag(self.full_cov)) def plot_lensing(self, column='PP', ells=None, units="muK2", ax=None): if not np.count_nonzero(self.map_cls): raise LoggedError( self.log, "No Cl's have been computed yet. " "Make sure you have evaluated the likelihood.") try: Cl_theo = self.provider.get_Cl(ell_factor=True, units=units) Cl = Cl_theo.get(column.lower()) except KeyError: raise LoggedError(self.log, "'%s' spectrum has not been computed." % column) import matplotlib.pyplot as plt lbin = self.full_bandpowers[:, self.full_bandpower_headers.index('L_av')] binned_phicl_err = self.diag_sigma() ax = ax or plt.gca() bandpowers = self.full_bandpowers[:, self.full_bandpower_headers. index('PP')] if 'L_min' in self.full_bandpower_headers: lmin = self.full_bandpowers[:, self.full_bandpower_headers. index('L_min')] lmax = self.full_bandpowers[:, self.full_bandpower_headers. index('L_max')] ax.errorbar(lbin, bandpowers, yerr=binned_phicl_err, xerr=[lbin - lmin, lmax - lbin], fmt='o') else: ax.errorbar(lbin, bandpowers, yerr=binned_phicl_err, fmt='o') if ells is not None: Cl = Cl[ells] else: ells = Cl_theo["ell"] ax.plot(ells, Cl, color='k') ax.set_xlim([2, ells[-1]]) return ax def get_binned_map_cls(self, Cls, corrections=True): band = self.bins.bin(Cls) if self.linear_correction is not None and corrections: band += self.linear_correction.bin(Cls) - self.fid_correction.T return band def get_theory_map_cls(self, Cls, data_params=None): for i in range(self.nmaps_required): for j in range(i + 1): CL = self.map_cls[i, j] combination = "".join( [self.field_names[k] for k in CL.theory_ij]).lower() cls = Cls.get(combination) if cls is not None: CL.CL[:] = cls[self.pcl_lmin:self.pcl_lmax + 1] else: CL.CL[:] = 0 self.adapt_theory_for_maps(self.map_cls, data_params or {}) def adapt_theory_for_maps(self, cls, data_params): if self.aberration_coeff: self.add_aberration(cls) self.add_foregrounds(cls, data_params) if self.calibration_param is not None and self.calibration_param in data_params: for i in range(self.nmaps_required): for j in range(i + 1): CL = cls[i, j] if CL is not None: if CL.theory_ij[0] <= 2 and CL.theory_ij[1] <= 2: CL.CL /= data_params[self.calibration_param]**2 def add_foregrounds(self, cls, data_params): pass def add_aberration(self, cls): # adapted from CosmoMC function by Christian Reichardt ells = np.arange(self.pcl_lmin, self.pcl_lmax + 1) cl_norm = ells * (ells + 1) for i in range(self.nmaps_required): for j in range(i + 1): CL = cls[i, j] if CL is not None: if CL.theory_ij[0] <= 2 and CL.theory_ij[1] <= 2: # first get Cl instead of Dl cl_deriv = CL.CL / cl_norm # second take derivative dCl/dl cl_deriv[1:-1] = (cl_deriv[2:] - cl_deriv[:-2]) / 2 # handle endpoints approximately cl_deriv[0] = cl_deriv[1] cl_deriv[-1] = cl_deriv[-2] # reapply to Dl's. # note never took 2pi out, so not putting it back either cl_deriv *= cl_norm # also multiply by ell since really wanted ldCl/dl cl_deriv *= ells CL.CL += self.aberration_coeff * cl_deriv def write_likelihood_data(self, filename, data_params=None): cls = self.init_map_cls(self.nmaps_required, self.required_order) self.add_foregrounds(cls, data_params or {}) with open(filename, 'w', encoding="utf-8") as f: cols = [] for i in range(self.nmaps_required): for j in range(i + 1): cols.append( self.Cl_i_j_name(self.map_names, cls[i, j].map_ij)) f.write('# L' + ("%17s " * len(cols)) % tuple(cols) + '\n') for b in range(self.pcl_lmin, self.pcl_lmax + 1): c = [b] for i in range(self.nmaps_required): for j in range(i + 1): c.append(cls[i, j].CL[b - self.pcl_lmin]) f.write(("%I5 " + "%17.8e " * len(cols)) % tuple(c)) def transform(self, C, Chat, Cfhalf): # HL transformation of the matrices if C.shape[0] == 1: rat = Chat[0, 0] / C[0, 0] C[0, 0] = (np.sign(rat - 1) * np.sqrt(2 * np.maximum(0, rat - np.log(rat) - 1)) * Cfhalf[0, 0]**2) return diag, U = np.linalg.eigh(C) rot = U.T.dot(Chat).dot(U) roots = np.sqrt(diag) for i, root in enumerate(roots): rot[i, :] /= root rot[:, i] /= root U.dot(rot.dot(U.T), rot) diag, rot = np.linalg.eigh(rot) diag = np.sign(diag - 1) * np.sqrt( 2 * np.maximum(0, diag - np.log(diag) - 1)) Cfhalf.dot(rot, U) for i, d in enumerate(diag): rot[:, i] = U[:, i] * d rot.dot(U.T, C) def exact_chi_sq(self, C, Chat, L): if C.shape[0] == 1: return ((2 * L + 1) * self.fsky * (Chat[0, 0] / C[0, 0] - 1 - np.log(Chat[0, 0] / C[0, 0]))) else: M = np.linalg.inv(C).dot(Chat) return ((2 * L + 1) * self.fsky * (np.trace(M) - self.nmaps - np.linalg.slogdet(M)[1])) def logp(self, **data_params): cls = self.provider.get_Cl(ell_factor=True) return self.log_likelihood(cls, **data_params) def log_likelihood(self, dls, **data_params): r""" Get log likelihood from the dls (CMB C_l scaled by L(L+1)/2\pi) :param dls: dictionary of d_l ('tt', etc) :param data_params: likelihood nuisance parameters :return: log likelihood """ self.get_theory_map_cls(dls, data_params) C = np.empty((self.nmaps, self.nmaps)) big_x = np.empty(self.nbins_used * self.ncl_used) vecp = np.empty(self.ncl) chisq = 0 if self.binned: binned_theory = self.get_binned_map_cls(self.map_cls) else: Cs = np.zeros((self.nbins_used, self.nmaps, self.nmaps)) for i in range(self.nmaps): for j in range(i + 1): CL = self.map_cls[i, j] if CL is not None: Cs[:, i, j] = CL.CL[self.bin_min - self.pcl_lmin:self.bin_max - self.pcl_lmin + 1] Cs[:, j, i] = CL.CL[self.bin_min - self.pcl_lmin:self.bin_max - self.pcl_lmin + 1] for b in range(self.nbins_used): if self.binned: self.elements_to_matrix(binned_theory[b, :], C) else: C[:, :] = Cs[b, :, :] if self.cl_noise is not None: C += self.noise_matrix[b] if self.like_approx == 'exact': chisq += self.exact_chi_sq(C, self.bandpower_matrix[b], self.bin_min + b) continue elif self.like_approx == 'HL': try: self.transform(C, self.bandpower_matrix[b], self.fiducial_sqrt_matrix[b]) except np.linalg.LinAlgError: self.log.debug("Likelihood computation failed.") return -np.inf elif self.like_approx == 'gaussian': C -= self.bandpower_matrix[b] self.matrix_to_elements(C, vecp) big_x[b * self.ncl_used:(b + 1) * self.ncl_used] = vecp[self.cl_used_index] if self.like_approx == 'exact': return -0.5 * chisq return -0.5 * self._fast_chi_squared(self.covinv, big_x)
class plik_likelihood(object): def __init__(self, clikfile, paramnames=None): # paramnames will be used to do the mapping to the extra parameters self.dffile = clikfile self.name = os.path.splitext(os.path.basename(clikfile))[0] if isinstance(paramnames, (list, tuple)): self.parnames = paramnames else: if paramnames is None: # match versions to the baseline .paramnames file for rem in ['_', 'a_', 'b_', 'c_', 'd_']: name = self.name.replace(rem, '_').replace('_bin1', '') paramnames = os.path.join( os.path.dirname(__file__), '../../data/' + name + '.paramnames') if os.path.exists(paramnames): break self.paramnamefile = paramnames self.paramnames = ParamNames(paramnames) self.parnames = self.paramnames.list() self.clik = clik.clik(clikfile) self._translate_parname(self.parnames) self.fi = hpy.File(self.dffile) # some metadata self.hascl = self.fi["clik/lkl_0/has_cl"] self.lmin = self.fi["clik/lkl_0/lmin"] self.lmax = self.fi["clik/lkl_0/lmax"] self.mt = self.fi["clik/lkl_0/m_channel_T"] * self.hascl[0] self.me = self.fi["clik/lkl_0/m_channel_P"] * self.hascl[1] self.mb = self.fi["clik/lkl_0/m_channel_P"] * self.hascl[2] self.m = self.mt + self.me + self.mb self.nb = self.fi["clik/lkl_0/nbins"] / self.hascl.sum() self.rq_shape = (self.nb, self.m, self.m) # binning details self.blmin = self.fi["clik/lkl_0/bin_lmin"] self.blmax = self.fi["clik/lkl_0/bin_lmax"] self.b_ws = self.fi["clik/lkl_0/bin_ws"] # the binning matrix is also simply obtained this way (but using it is slower than using the binning details, 'cause it's full of zeros) self.bns = php.read_bins(self.fi["clik/lkl_0"]) # compute the binned ells self.lm = np.dot(self.bns[:self.nb, :self.lmax - self.lmin + 1], np.arange(self.lmin, self.lmax + 1)) # get the calibration part (and beam for plik 2015) # cal and bal are functions that expect a vector of parameters whose name and ordering are given by cal.varpar and bal.varpar # overal calibration is given by cal(pars)*vec(pars)*outer(acmb,acmb)[nm.newaxis,:,:] self.cal = smh.calTP_from_smica(self.dffile) self.bal = smh.beamTP_from_smica(self.dffile) self.acmb = self.fi["clik/lkl_0/A_cmb"] # get the binned Cl data array self.rqh = self.fi["clik/lkl_0/Rq_hat"] self.rqh.shape = self.rq_shape # get the additive nuisance components self.prms = smh.parametric_from_smica(self.dffile) self.prms_name = [p.get_name() for p in self.prms] # get the selection vector self.oo, self.Jt = smh.ordering_from_smica(self.dffile) # get the inverse covariance self.siginv = self.fi["clik/lkl_0/criterion_gauss_mat"] self.siginv.shape = (len(self.oo), len(self.oo)) ls = np.arange(self.lmax + 1) self.llp1 = ls * (ls + 1) / (2 * np.pi) self.llp1[0] = 1 self.indices = [(0, 0), (1, 1), (2, 2), (0, 1), (0, 2), (1, 2)] self.spectra = ["tt", "ee", "bb", "te", "tb", "eb"] def _translate_parname(self, parnames): self._translate = {} for old, new in zip(self.clik.extra_parameter_names, parnames): self._translate[old] = new def get_nuisance_vector(self, nuisance_dict, clik_names=None): # nuisance dict is a python dict whose keys are defined by the cosmomc paramnames if clik_names is None: clik_names = self.clik.extra_parameter_names return np.array( [nuisance_dict[self._translate[par]] for par in clik_names]) def get_cl_clik_ordering(self, cl_array): mcls = [] lmax = self.lmax for i, index in enumerate(self.indices[:4]): if self.hascl[i]: mcls += [cl_array.get(index)[:lmax + 1] / self.llp1] for i in range(4, 6): if self.hascl[i]: lcls = np.zeros(lmax + 1) mcls += [lcls] return (mcls) def get_clik_vector(self, cl_array, nuisance_dict): mcls = self.get_cl_clik_ordering(cl_array) extra = self.get_nuisance_vector(nuisance_dict) return np.concatenate(mcls + [extra]) def get_unbinned_nuisance_rq(self, nuisance_dict): # compute the nuisance part oq = [] for p in self.prms: # prepare the input vector for each component pvec = self.get_nuisance_vector(nuisance_dict, p.varpar) # compute the rq matrix for this component for those parameters oq += [p(pvec)] # correct shape for T only component when dealing with T+P case if oq[-1].shape[1:] != self.rq_shape[1:]: bet = np.zeros( (oq[-1].shape[0], self.rq_shape[1], self.rq_shape[1])) bet[:, :oq[-1].shape[1], :oq[-1].shape[1]] = oq[-1] oq[-1] = bet oq = np.array(oq) return oq def get_nuisance_rq(self, nuisance_dict): # get the unbinned one oq = self.get_unbinned_nuisance_rq(nuisance_dict) oqb = np.zeros((len(oq), ) + self.rq_shape) # bin it nb = self.nb m = self.m mt = self.mt me = self.me mb = self.mb blmin = self.blmin blmax = self.blmax b_ws = self.b_ws lmin = self.lmin lmax = self.lmax for b in range(nb): if oq.shape[0]: oqb[:, b] = np.sum( oq[:, blmin[b]:blmax[b] + 1] * b_ws[np.newaxis, blmin[b]:blmax[b] + 1, np.newaxis, np.newaxis], 1) return oqb def get_cmb_rq(self, cl_array): # get the cls mcls = self.get_cl_clik_ordering(cl_array) cls = np.zeros((6, self.lmax + 1)) j = 0 for i in range(6): if self.hascl[i]: cls[i] = mcls[j] j += 1 rq = np.zeros((self.nb, self.m, self.m)) nb = self.nb m = self.m mt = self.mt me = self.me mb = self.mb blmin = self.blmin blmax = self.blmax b_ws = self.b_ws lmin = self.lmin lmax = self.lmax # bin it (and order it in) for b in range(nb): if mt: rq[b, :mt, :mt] += np.sum( cls[0, lmin + blmin[b]:lmin + blmax[b] + 1] * b_ws[blmin[b]:blmax[b] + 1]) if me: rq[b, :mt, mt:mt + me] += np.sum( cls[3, lmin + blmin[b]:lmin + blmax[b] + 1] * b_ws[blmin[b]:blmax[b] + 1]) rq[b, mt:mt + me, :mt] += np.sum( cls[3, lmin + blmin[b]:lmin + blmax[b] + 1] * b_ws[blmin[b]:blmax[b] + 1]) if mb: rq[b, :mt, mt + me:mb + mt + me] += np.sum( cls[4, lmin + blmin[b]:lmin + blmax[b] + 1] * b_ws[blmin[b]:blmax[b] + 1]) rq[b, mt + me:mb + mt + me, :mt] += np.sum( cls[4, lmin + blmin[b]:lmin + blmax[b] + 1] * b_ws[blmin[b]:blmax[b] + 1]) if me: rq[b, mt:mt + me, mt:mt + me] += np.sum(cls[1, lmin + blmin[b]:lmin + blmax[b] + 1] * b_ws[blmin[b]:blmax[b] + 1]) if mb: rq[b, mt:mt + me, mt + me:mb + mt + me] += np.sum( cls[5, lmin + blmin[b]:lmin + blmax[b] + 1] * b_ws[blmin[b]:blmax[b] + 1]) rq[b, mt + me:mb + mt + me, mt:mt + me] += np.sum( cls[5, lmin + blmin[b]:lmin + blmax[b] + 1] * b_ws[blmin[b]:blmax[b] + 1]) if mb: rq[b, mt + me:mt + me + mb, mt + me:mb + mt + me] += np.sum(cls[2, lmin + blmin[b]:lmin + blmax[b] + 1] * b_ws[blmin[b]:blmax[b] + 1]) return rq def get_calib(self, nuisance_dict): # returns a matrix cvec = self.get_nuisance_vector(nuisance_dict, self.cal.varpar) g = self.cal(cvec) cvec = self.get_nuisance_vector(nuisance_dict, self.bal.varpar) bg = self.bal(cvec) g *= bg g *= np.outer(self.acmb, self.acmb)[np.newaxis, :, :] return g def get_model_vector(self, cl_array, nuisance_dict): # get the calib g = self.get_calib(nuisance_dict) rq = self.get_cmb_rq(cl_array) oqb = self.get_nuisance_rq(nuisance_dict) rqt = g * (rq + np.sum(oqb, 0)) return rqt def chi_squared(self, cl_array, nuisance_dict, clik=True): if clik: vec = self.get_clik_vector(cl_array, nuisance_dict) return -2 * self.clik(vec) else: rqt = self.get_model_vector(cl_array, nuisance_dict) delta_rq = rqt - self.rqh delta = delta_rq.flat[self.oo] return np.dot(delta, np.dot(self.siginv, delta)) def coadd_spectra(self, nuisance_dict): good = self.Jt.sum(1) != 0 Jt = self.Jt[good] g = self.get_calib(nuisance_dict) Jt = Jt * g.flat[self.oo] tm = np.concatenate([self.lm for h in self.hascl if h]) oqb = self.get_nuisance_rq(nuisance_dict) Yo = (g * np.sum(oqb, 0)) - self.rqh Yo = Yo.flat[self.oo] Jt_siginv = np.zeros((Jt.shape)) # Jt is full of zero, it's quicker to compute the matrix multiplication this way w0, w1 = np.where(Jt != 0) for ii in range(len(w0)): i = w0[ii] j = w1[ii] Jt_siginv[i] += Jt[i, j] * self.siginv[j] Jt_siginv_Yo = np.dot(Jt_siginv, Yo) nl = Jt.shape[0] Jt_siginv_J = np.zeros((nl, nl)) for ii in range(len(w0)): j = w0[ii] i = w1[ii] Jt_siginv_J[j] += Jt[j, i] * Jt_siginv[:, i] try: rVec = -scipy.linalg.solve( Jt_siginv_J, Jt_siginv_Yo, assume_a='pos') except: rVec = -np.linalg.solve(Jt_siginv_J, Jt_siginv_Yo) tVec = np.zeros(len(tm)) tVec[good] = rVec tm.shape = (-1, len(self.lm)) tVec.shape = tm.shape # beware returns inverse variance of the coadded return tm, tVec, Jt_siginv_J def coadded_TT(self, dict, want_cov=True): ls, coadded, covinv = self.coadd_spectra(dict) ls = ls[0, :] TT = np.zeros(ls[-1] + 1) fac = ls * (ls + 1) / (2 * np.pi) TT[ls[0]:] = coadded[0, :] * fac if want_cov: cov = np.linalg.inv(covinv)[:len(ls), :len(ls)] for i in range(cov.shape[0]): cov[i, :] *= fac cov[:, i] *= fac return TT, cov else: return TT
class DatasetLikelihood(object): def __init__(self, fname, dataset_params={}, field_names=['T', 'E', 'B', 'P'], map_separator='x'): self.field_names = field_names self.tot_theory_fields = len(field_names) self.map_separator = map_separator # aberration will be corrected if aberration_coeff is non - zero self.aberration_coeff = 0.0 self.log_calibration_prior = -1 # if >0 use log prior on calibration parameter if '.dataset' in fname: self.loadDataset(fname, dataset_params) else: raise Exception('DatasetLikelihood only supports .dataset files') def typeIndex(self, field): return self.field_names.index(field) def PairStringToMapIndices(self, S): if len(S) == 2: if self.has_map_names: raise Exception('CMBlikes: CL names must use MAP1xMAP2 names') return self.map_names.index(S[0]), self.map_names.index(S[1]) else: try: i = S.index(self.map_separator) except ValueError: raise ValueError('CMBLikes: invalid spectrum name %s' % S) return self.map_names.index(S[0:i]), self.map_names.index(S[i + 1:]) def PairStringToUsedMapIndices(self, used_index, S): i1, i2 = self.PairStringToMapIndices(S) i1 = used_index[i1] i2 = used_index[i2] if i2 > i1: return i2, i1 else: return i1, i2 def UseString_to_cols(self, L): cl_i_j = self.UseString_to_Cl_i_j(L, self.map_used_index) cols = -np.ones(cl_i_j.shape[1], dtype=int) for i in range(cl_i_j.shape[1]): i1, i2 = cl_i_j[:, i] if i1 == -1 or i2 == -1: continue ix = 0 for ii in range(self.nmaps): for jj in range(ii + 1): if ii == i1 and jj == i2: cols[i] = ix ix += 1 return cols def UseString_to_Cl_i_j(self, S, used_index): if not isinstance(S, (list, tuple)): S = S.split() cl_i_j = np.zeros((2, len(S)), dtype=int) for i, p in enumerate(S): cl_i_j[:, i] = self.PairStringToUsedMapIndices(used_index, p) return cl_i_j def MapPair_to_Theory_i_j(self, order, pair): i = self.map_fields[order[pair[0]]] j = self.map_fields[order[pair[1]]] if i <= j: return i, j else: return j, i def Cl_used_i_j_name(self, pair): return self.Cl_i_j_name(self.used_map_order, pair) def Cl_i_j_name(self, names, pair): name1 = names[pair[0]] name2 = names[pair[1]] if self.has_map_names: return name1 + self.map_separator + name2 else: return name1 + name2 def GetColsFromOrder(self, order): # Converts string Order = TT TE EE XY... or AAAxBBB AAAxCCC BBxCC # into indices into array of power spectra (and -1 if not present) cols = np.empty(self.ncl, dtype=int) cols[:] = -1 names = order.strip().split() ix = 0 for i in range(self.nmaps): for j in range(i + 1): name = self.Cl_used_i_j_name([i, j]) if not name in names and i <> j: name = self.Cl_used_i_j_name([j, i]) if name in names: if cols[ix] <> -1: raise Exception('GetColsFromOrder: duplicate CL type') cols[ix] = names.index(name) ix += 1 return cols def elements_to_matrix(self, X, M): ix = 0 for i in range(self.nmaps): M[i, 0:i] = X[ix:ix + i] M[0:i, i] = X[ix:ix + i] ix += i M[i, i] = X[ix] ix += 1 def matrix_to_elements(self, M, X): ix = 0 for i in range(self.nmaps): X[ix:ix + i + 1] = M[i, 0:i + 1] ix += i + 1 def ReadClArr(self, ini, file_stem, return_full=False): # read file of CL or bins (indexed by L) filename = ini.relativeFileName(file_stem + '_file') cl = np.zeros((self.ncl, self.nbins_used)) order = ini.string(file_stem + '_order', '') if not order: incols = lastTopComment(filename) if not incols: raise Exception('No column order given for ' + filename) else: incols = 'L ' + order cols = self.GetColsFromOrder(incols) data = np.loadtxt(filename) Ls = data[:, 0].astype(int) if self.binned: Ls -= 1 for i, L in enumerate(Ls): if L >= self.bin_min and L <= self.bin_max: for ix in range(self.ncl): if cols[ix] <> -1: cl[ix, L - self.bin_min] = data[i, cols[ix]] if L < self.bin_max: raise Exception( 'CMBLikes_ReadClArr: C_l file does not go up to maximum used: %s' % self.bin_max) if return_full: return incols.split(), data, cl else: return cl def readBinWindows(self, ini, file_stem): bins = BinWindows(self.pcl_lmin, self.pcl_lmax, self.nbins_used) in_cl = ini.split(file_stem + '_in_order') out_cl = ini.split(file_stem + '_out_order', in_cl) bins.cols_in = self.UseString_to_Cl_i_j(in_cl, self.map_required_index) bins.cols_out = self.UseString_to_cols(out_cl) norder = bins.cols_in.shape[1] if norder != bins.cols_out.shape[0]: raise Exception( '_in_order and _out_order must have same number of entries') bins.binning_matrix = np.zeros( (norder, self.nbins_used, self.pcl_lmax - self.pcl_lmin + 1)) windows = ini.relativeFileName(file_stem + '_files') for b in range(self.nbins_used): window = np.loadtxt(windows % (b + 1 + self.bin_min)) Err = False for i, L in enumerate(window[:, 0].astype(int)): if self.pcl_lmin <= L <= self.pcl_lmax: bins.binning_matrix[:, b, L - self.pcl_lmin] = window[i, 1:] else: Err = Err or any(window[i, 1:] != 0) if Err: print('WARNING: %s %u outside pcl_lmin-cl_max range: %s' % (file_stem, b, windows % (b + 1))) if ini.hasKey(file_stem + '_fix_cl_file'): raise Exception('fix_cl_file not implemented yet') return bins def init_map_cls(self, nmaps, order): if nmaps <> len(order): raise ValueError('CMBLikes init_map_cls: size mismatch') class CrossPowerSpectrum(object): pass cls = np.empty((nmaps, nmaps), dtype=object) for i in range(nmaps): for j in range(i + 1): CL = CrossPowerSpectrum() cls[i, j] = CL CL.map_ij = [order[i], order[j]] CL.theory_ij = self.MapPair_to_Theory_i_j(order, [i, j]) CL.CL = np.zeros(self.pcl_lmax - self.pcl_lmin + 1) return cls def loadDataset(self, froot, dataset_params): if not '.dataset' in froot: froot += '.dataset' ini = IniFile(froot) ini.params.update(dataset_params) self.readIni(ini) def readIni(self, ini): self.map_names = ini.split('map_names', default=[]) self.has_map_names = len(self.map_names) if self.has_map_names: # e.g. have multiple frequencies for given field measurement map_fields = ini.split('map_fields') if len(map_fields) <> len(self.map_names): raise Exception( 'CMBLikes: number of map_fields does not match map_names') self.map_fields = [self.typeIndex(f) for f in map_fields] else: self.map_names = self.field_names self.map_fields = np.arange(len(self.map_names), dtype=int) fields_use = ini.split('fields_use', []) if len(fields_use): index_use = [self.typeIndex(f) for f in fields_use] use_theory_field = [ i in index_use for i in range(self.tot_theory_fields) ] else: if not self.has_map_names: raise Exception('CMBlikes: must have fields_use or map_names') use_theory_field = [True] * self.tot_theory_fields maps_use = ini.split('maps_use', []) if len(maps_use): if np.any([not i for i in use_theory_field]): print('CMBlikes WARNING: maps_use overrides fields_use') self.use_map = np.zeros(len(self.map_names), dtype=bool) for j, map_used in enumerate(maps_use): if map_used in self.map_names: self.use_map[self.map_names.index(map_used)] = True else: raise ValueError('CMBlikes: maps_use item not found - %s' % map_used) else: self.use_map = [ use_theory_field[self.map_fields[i]] for i in range(len(self.map_names)) ] # Bandpowers can depend on more fields than are actually used in likelihood # e.g. for correcting leakage or other linear corrections self.require_map = self.use_map[:] if self.has_map_names: if ini.hasKey('fields_required'): raise Exception( 'CMBLikes: use maps_required not fields_required') maps_use = ini.split('maps_required', []) else: maps_use = ini.split('fields_required', []) if len(maps_use): for j, map_used in enumerate(maps_use): if map_used in self.map_names: self.require_map[self.map_names.index(map_used)] = True else: raise ValueError('CMBlikes: required item not found %s' % map_used) self.required_theory_field = [False for _ in self.field_names] for i in range(len(self.map_names)): if self.require_map[i]: self.required_theory_field[self.map_fields[i]] = True self.ncl_used = 0 # set later reading covmat self.like_approx = ini.string('like_approx', 'gaussian') self.nmaps = np.count_nonzero(self.use_map) self.nmaps_required = np.count_nonzero(self.require_map) self.required_order = np.zeros(self.nmaps_required, dtype=int) self.map_required_index = -np.ones(len(self.map_names), dtype=int) ix = 0 for i in range(len(self.map_names)): if self.require_map[i]: self.map_required_index[i] = ix self.required_order[ix] = i ix += 1 self.map_used_index = -np.ones(len(self.map_names), dtype=int) ix = 0 self.used_map_order = [] for i, map_name in enumerate(self.map_names): if self.use_map[i]: self.map_used_index[i] = ix self.used_map_order.append(map_name) ix += 1 self.ncl = (self.nmaps * (self.nmaps + 1)) // 2 self.pcl_lmax = ini.int('cl_lmax') self.pcl_lmin = ini.int('cl_lmin') self.binned = ini.bool('binned', True) if self.binned: self.nbins = ini.int('nbins') self.bin_min = ini.int('use_min', 1) - 1 self.bin_max = ini.int('use_max', self.nbins) - 1 self.nbins_used = self.bin_max - self.bin_min + 1 # needed by readBinWindows self.bins = self.readBinWindows(ini, 'bin_window') else: if self.nmaps <> self.nmaps_required: raise Exception( 'CMBlikes: unbinned likelihood must have nmaps==nmaps_required' ) self.nbins = self.pcl_lmax - self.pcl_lmin + 1 if self.like_approx != 'exact': print('WARNING: Unbinned likelihoods untested in this version') self.bin_min = ini.int('use_min', self.pcl_lmin) self.bin_max = ini.int('use_max', self.pcl_lmax) self.nbins_used = self.bin_max - self.bin_min + 1 self.full_bandpower_headers, self.full_bandpowers, self.bandpowers = \ self.ReadClArr(ini, 'cl_hat', return_full=True) if self.like_approx == 'HL': self.cl_fiducial = self.ReadClArr(ini, 'cl_fiducial') else: self.cl_fiducial = None includes_noise = ini.bool('cl_hat_includes_noise', False) self.cl_noise = None if self.like_approx <> 'gaussian' or includes_noise: self.cl_noise = self.ReadClArr(ini, 'cl_noise') if not includes_noise: self.bandpowers += self.cl_noise elif self.like_approx == 'gaussian': self.bandpowers -= self.cl_noise self.cl_lmax = np.zeros( (self.tot_theory_fields, self.tot_theory_fields)) for i in range(self.tot_theory_fields): if self.required_theory_field[i]: self.cl_lmax[i, i] = self.pcl_lmax if self.required_theory_field[0] and self.required_theory_field[1]: self.cl_lmax[1, 0] = self.pcl_lmax if self.like_approx != 'gaussian': cl_fiducial_includes_noise = ini.bool('cl_fiducial_includes_noise', False) self.bandpower_matrix = np.zeros( (self.nbins_used, self.nmaps, self.nmaps)) self.noise_matrix = self.bandpower_matrix.copy() self.fiducial_sqrt_matrix = self.bandpower_matrix.copy() if self.cl_fiducial is not None and not cl_fiducial_includes_noise: self.cl_fiducial += self.cl_noise for b in range(self.nbins_used): self.elements_to_matrix(self.bandpowers[:, b], self.bandpower_matrix[b, :, :]) if self.cl_noise is not None: self.elements_to_matrix(self.cl_noise[:, b], self.noise_matrix[b, :, :]) if self.cl_fiducial is not None: self.elements_to_matrix(self.cl_fiducial[:, b], self.fiducial_sqrt_matrix[b, :, :]) self.fiducial_sqrt_matrix[b, :, :] = sqrtm( self.fiducial_sqrt_matrix[b, :, :]) if self.like_approx == 'exact': self.fsky = ini.float('fullsky_exact_fksy') else: self.cov = self.ReadCovmat(ini) self.covinv = np.linalg.inv(self.cov) if 'linear_correction_fiducial_file' in ini.params: self.fid_correction = self.ReadClArr(ini, 'linear_correction_fiducial') self.linear_correction = self.readBinWindows( ini, 'linear_correction_bin_window') else: self.linear_correction = None if ini.hasKey('nuisance_params'): s = ini.relativeFileName('nuisance_params') self.nuisance_params = ParamNames(s) if ini.hasKey('calibration_param'): self.calibration_param = ini.string('calibration_param') if '.paramnames' in self.calibration_param: raise Exception( 'calibration_param should be name of parameter in nuisance_params' ) else: self.calibration_param = None elif ini.hasKey('calibration_param'): s = ini.relativeFileName('calibration_param') if not '.paramnames' in s: raise Exception( 'calibration_param must be paramnames file unless nuisance_params also specified' ) self.nuisance_params = ParamNames(s) self.calibration_param = self.nuisance_params.list()[0] else: self.calibration_param = None if self.calibration_param is not None: self.log_calibration_prior = ini.float('log_calibration_prior', -1) self.aberration_coeff = ini.float('aberration_coeff', 0.0) self.map_cls = self.init_map_cls(self.nmaps_required, self.required_order) def ReadCovmat(self, ini): # read the covariance matrix, and the array of which CL are in the covariance, # which then defines which set of bandpowers are used (subject to other restrictions) covmat_cl = ini.string('covmat_cl', allowEmpty=False) self.full_cov = np.loadtxt(ini.relativeFileName('covmat_fiducial')) covmat_scale = ini.float('covmat_scale', 1.0) cl_in_index = self.UseString_to_cols(covmat_cl) self.ncl_used = np.sum(cl_in_index >= 0) self.cl_used_index = np.zeros(self.ncl_used, dtype=int) cov_cl_used = np.zeros(self.ncl_used, dtype=int) ix = 0 for i, index in enumerate(cl_in_index): if index >= 0: self.cl_used_index[ix] = index cov_cl_used[ix] = i ix += 1 if self.binned: num_in = len(cl_in_index) pcov = np.empty((self.nbins_used * self.ncl_used, self.nbins_used * self.ncl_used)) for binx in range(self.nbins_used): for biny in range(self.nbins_used): pcov[binx * self.ncl_used: (binx + 1) * self.ncl_used, biny * self.ncl_used: (biny + 1) * self.ncl_used] = \ covmat_scale * self.full_cov[np.ix_((binx + self.bin_min) * num_in + cov_cl_used, (biny + self.bin_min) * num_in + cov_cl_used)] else: raise Exception('unbinned covariance not implemented yet') return pcov def writeData(self, froot): np.savetxt(froot + '_cov.dat', self.cov) # self.saveCl(froot + '_fid_cl.dat', self.fid_cl[:, 1:], cols=['TT', 'EE', 'TE', 'PP']) with open(froot + '_bandpowers.dat', 'w') as f: f.write("#%4s %5s %5s %8s %12s %10s %7s\n" % ('bin', 'L_min', 'L_max', 'L_av', 'PP', 'Error', 'Ahat')) for b in range(self.nbins): f.write("%5u %5u %5u %8.2f %12.5e %10.3e %7.3f\n" % (b + 1, self.lmin[b], self.lmax[b], self.lav[b], self.bandpowers[b], np.sqrt( self.cov[b, b]), self.Ahat[b])) self.bins.write(froot, 'bin') if self.linear_correction is not None: self.linear_correction.write(froot, 'linear_correction_bin') with open(froot + '_lensing_fiducial_correction', 'w') as f: f.write("#%4s %12s \n" % ('bin', 'PP')) for b in range(self.nbins): f.write("%5u %12.5e\n" % (b + 1, self.fid_correction[b])) def get_binned_theory(self, ClArray, data_params={}): # Useful for plotting, not used for likelihood self.get_theory_map_cls(ClArray, data_params) return self.get_binned_map_cls(self.map_cls) def get_full_bandpower_column(self, col_name): """ Get columns from the input bandpower file. Note used by likelihood but may be useful for plotting. :param col_name: name of column, as in the top comment header of the file :return: column values """ return self.full_bandpowers[:, self.full_bandpower_headers.index(col_name )] def diag_sigma(self): return np.sqrt(np.diag(self.full_cov)) def plot(self, column='PP', ClArray=None, ls=None, ax=None): lbin = self.full_bandpowers[:, self.full_bandpower_headers.index('L_av')] binned_phicl_err = self.diag_sigma() ax = ax or plt.gca() bandpowers = self.full_bandpowers[:, self.full_bandpower_headers. index('PP')] if 'L_min' in self.full_bandpower_headers: lmin = self.full_bandpowers[:, self.full_bandpower_headers. index('L_min')] lmax = self.full_bandpowers[:, self.full_bandpower_headers. index('L_max')] ax.errorbar(lbin, bandpowers, yerr=binned_phicl_err, xerr=[lbin - lmin, lmax - lbin], fmt='o') else: ax.errorbar(lbin, bandpowers, yerr=binned_phicl_err, fmt='o') if ClArray is not None: if isinstance(ClArray, ClsArray): i, j = self.MapPair_to_Theory_i_j( range(len(self.map_names)), self.PairStringToMapIndices(column)) ClArray = ClArray.get([i, j]) if ls is None: ls = np.arange(len(ClArray)) ax.plot(ls, ClArray, color='k') ax.set_xlim([2, ls[-1]]) def get_binned_map_cls(self, Cls, corrections=True): band = self.bins.bin(Cls) if self.linear_correction is not None and corrections: band += self.linear_correction.bin(Cls) - self.fid_correction.T return band def get_theory_map_cls(self, ClArray, data_params={}): for i in range(self.nmaps_required): for j in range(i + 1): CL = self.map_cls[i, j] cls = ClArray.get(CL.theory_ij) if cls is not None: CL.CL[:] = cls[self.pcl_lmin:self.pcl_lmax + 1] else: CL.CL[:] = 0 self.adapt_theory_for_maps(self.map_cls, data_params) def adapt_theory_for_maps(self, cls, data_params): if self.aberration_coeff: self.add_aberration(cls) self.add_foregrounds(cls, data_params) if self.calibration_param is not None and self.calibration_param in data_params: for i in range(self.nmaps_required): for j in range(i + 1): CL = cls[i, j] if CL is not None: if CL.theory_ij[0] <= 2 and CL.theory_ij[1] <= 2: CL.CL /= data_params[self.calibration_param]**2 def add_foregrounds(self, cls, data_params): pass def add_aberration(self, cls): # adapted from CosmoMC function by Christian Reichardt ells = np.arange(self.pcl_lmin, self.pcl_lmax + 1) cl_norm = ells * (ells + 1) for i in range(self.nmaps_required): for j in range(i + 1): CL = cls[i, j] if CL is not None: if CL.theory_ij[0] <= 2 and CL.theory_ij[1] <= 2: # first get Cl instead of Dl cl_deriv = CL.CL / cl_norm # second take derivative dCl/dl cl_deriv[1:-1] = (cl_deriv[2:] - cl_deriv[:-2]) / 2 # handle endpoints approximately cl_deriv[0] = cl_deriv[1] cl_deriv[-1] = cl_deriv[-2] # reapply to Dl's. # note never took 2pi out, so not putting it back either cl_deriv *= cl_norm # also multiply by ell since really wanted ldCl/dl cl_deriv *= ells CL.CL += self.aberration_coeff * cl_deriv def write_likelihood_data(self, filename, data_params={}): cls = self.init_map_cls(self.nmaps_required, self.required_order) self.add_foregrounds(cls, data_params) with open(filename, 'w') as f: cols = [] for i in range(self.nmaps_required): for j in range(i + 1): cols.append( self.Cl_i_j_name(self.map_names, cls[i, j].map_ij)) f.write('# L' + ("%17s " * len(cols)) % tuple(cols) + '\n') for b in range(self.pcl_lmin, self.pcl_lmax + 1): c = [b] for i in range(self.nmaps_required): for j in range(i + 1): c.append(cls[i, j].CL[b - self.pcl_lmin]) f.write(("%I5 " + "%17.8e " * len(cols)) % tuple(c)) def transform(self, C, Chat, Cfhalf): # HL transformation of the matrices if C.shape[0] == 1: rat = Chat[0, 0] / C[0, 0] C[0, 0] = np.sign(rat - 1) * np.sqrt( 2 * np.maximum(0, rat - np.log(rat) - 1)) * Cfhalf[0, 0]**2 return diag, U = np.linalg.eigh(C) rot = U.T.dot(Chat).dot(U) roots = np.sqrt(diag) for i, root in enumerate(roots): rot[i, :] /= root rot[:, i] /= root U.dot(rot.dot(U.T), rot) diag, rot = np.linalg.eigh(rot) diag = np.sign(diag - 1) * np.sqrt( 2 * np.maximum(0, diag - np.log(diag) - 1)) Cfhalf.dot(rot, U) for i, d in enumerate(diag): rot[:, i] = U[:, i] * d rot.dot(U.T, C) def exact_chi_sq(self, C, Chat, L): if C.shape[0] == 1: return (2 * L + 1) * self.fsky * (Chat[0, 0] / C[0, 0] - 1 - np.log(Chat[0, 0] / C[0, 0])) else: M = np.linalg.inv(C).dot(Chat) return (2 * L + 1) * self.fsky * (np.trace(M) - self.nmaps - np.linalg.slogdet(M)[1]) def chi_squared(self, ClArray, data_params={}): self.get_theory_map_cls(ClArray, data_params) C = np.empty((self.nmaps, self.nmaps)) bigX = np.empty(self.nbins_used * self.ncl_used) vecp = np.empty(self.ncl) chisq = 0 if self.binned: binned_theory = self.get_binned_map_cls(self.map_cls) else: Cs = np.zeros((self.nbins_used, self.nmaps, self.nmaps)) for i in range(self.nmaps): for j in range(i + 1): CL = self.map_cls[i, j] if CL is not None: Cs[:, i, j] = CL.CL[self.bin_min - self.pcl_lmin:self.bin_max - self.pcl_lmin + 1] Cs[:, j, i] = CL.CL[self.bin_min - self.pcl_lmin:self.bin_max - self.pcl_lmin + 1] for bin in range(self.nbins_used): if self.binned: self.elements_to_matrix(binned_theory[bin, :], C) else: C[:, :] = Cs[bin, :, :] if self.cl_noise is not None: C += self.noise_matrix[bin] if self.like_approx == 'exact': chisq += self.exact_chi_sq(C, self.bandpower_matrix[bin], self.bin_min + bin) continue elif self.like_approx == 'HL': self.transform(C, self.bandpower_matrix[bin], self.fiducial_sqrt_matrix[bin]) elif self.like_approx == 'gaussian': C -= self.bandpower_matrix[bin] self.matrix_to_elements(C, vecp) bigX[bin * self.ncl_used:(bin + 1) * self.ncl_used] = vecp[self.cl_used_index] if self.like_approx != 'exact': chisq = np.dot(bigX, np.dot(self.covinv, bigX)) if self.log_calibration_prior > 0: chisq += (np.log(data_params[self.calibration_param]) / self.log_calibration_prior)**2 return chisq
def cosmomc_root_to_cobaya_info_dict(root: str, derived_to_input=()) -> InputDict: """ Given the root name of existing cosmomc chain files, tries to construct a Cobaya input parameter dictionary with roughly equivalent settings. The output dictionary can be used for importance sampling from CosmoMC chains in simple cases using Cobaya's 'post'. Parameters in the optional derived_to_input list are converted from being derived parameters in CosmoMC to non-derived in Cobaya. This is by no means guaranteed to produce valid or equivalent results, use at your own risk with careful checking! Note that the parameter dictionary will not have settings for CAMB, samplers etc, which is OK for importance sampling but you would need to add them as necessary to reproduce results. Parameter chains in CosmoMC format are available for Planck from https://pla.esac.esa.int/pla/#home """ names = ParamNames(root + '.paramnames') if os.path.exists(root + '.ranges'): ranges = ParamBounds(root + '.ranges') else: ranges = None d: ParamsDict = {} info: InputDict = {'params': d} for par, name in zip(names.names, names.list()): if name.startswith('chi2_') and not name.startswith('chi2__'): if name == 'chi2_prior': continue name = name.replace('chi2_', 'chi2__') if name.startswith('minuslogprior') or name == 'chi2': continue param_dict: ParamDict = {'latex': par.label} d[name] = param_dict if par.renames: param_dict['renames'] = par.renames if par.isDerived: if name not in derived_to_input: param_dict['derived'] = True else: par.isDerived = False if ranges and name in ranges.names: if par.isDerived: low_up = ranges.getLower(name), ranges.getUpper(name) if any(r is not None for r in low_up): param_dict['min'], param_dict['max'] = low_up else: param_dict["prior"] = [ranges.getLower(name), ranges.getUpper(name)] if ranges: d.update(ranges.fixedValueDict()) if names.numberOfName('As') == -1 and names.numberOfName('logA') != -1: d['As'] = {'latex': r'A_\mathrm{s}', 'value': 'lambda logA: 1e-10*np.exp(logA)'} if names.numberOfName('cosmomc_theta') == -1 and names.numberOfName('theta') != -1: d['cosmomc_theta'] = {'latex': r'\theta_{\rm MC}', 'value': 'lambda theta: theta/100'} # special case for CosmoMC (e.g. Planck) chains if os.path.exists(root + '.inputparams'): inputs = IniFile(root + '.inputparams') for key, value in inputs.params.items(): if key.startswith('prior['): if 'prior' not in info: info['prior'] = {} param = key[6:-1] if param in d: mean, std = (float(v.strip()) for v in value.split()) if not names.parWithName(param).isDerived: info['prior'][param + '_prior'] = \ "lambda %s: stats.norm.logpdf(%s, loc=%g, scale=%g)" % ( param, param, mean, std) if key.startswith('linear_combination['): param = key.replace('linear_combination[', '')[:-1] prior = inputs.params.get('prior[%s]' % param, None) if prior: weights = inputs.params.get('linear_combination_weights[%s]' % param, None) if not weights: raise ValueError( 'linear_combination[%s] prior found but not weights' % param) weights = [float(w.strip()) for w in weights.split()] inputs = value.split() if 'prior' not in info: info['prior'] = {} mean, std = (float(v.strip()) for v in prior.split()) linear = "".join( "%+g*%s" % (_w, _p) for _w, _p in zip(weights, inputs)) info['prior']['SZ'] = \ "lambda %s: stats.norm.logpdf(%s, loc=%g, scale=%g)" % ( ",".join(inputs), linear, mean, std) if os.path.exists(root + '.likelihoods'): info_like: LikesDict = {} info['likelihood'] = info_like with open(root + '.likelihoods', 'r') as f: for line in f.readlines(): if line.strip(): like = line.split()[2] info_like[like] = None else: print('You need to mention in the likelihood block with with "name: None"' 'for each likelihood in the input chain') info['output'] = root return info