def load_dataset(self, filename, dataset_params): from getdist import IniFile ini = IniFile(filename) ini.params.update(dataset_params) self.indices = [] self.used_indices = [] self.used_items = [] self.fullcov = np.loadtxt(ini.relativeFileName('cov_file')) ntheta = ini.int('num_theta_bins') self.theta_bins = np.loadtxt(ini.relativeFileName('theta_bins_file')) self.iintrinsic_alignment_model = ini.string('intrinsic_alignment_model') self.data_types = ini.string('data_types').split() self.used_types = ini.list('used_data_types', self.data_types) with open(ini.relativeFileName('data_selection')) as f: header = f.readline() assert ('# type bin1 bin2 theta_min theta_max' == header.strip()) lines = f.readlines() ranges = {} for tp in self.data_types: ranges[tp] = np.empty((6, 6), dtype=object) for line in lines: items = line.split() if items[0] in self.used_types: bin1, bin2 = [int(x) - 1 for x in items[1:3]] ranges[items[0]][bin1][bin2] = [np.float64(x) for x in items[3:]] self.ranges = ranges self.nzbins = ini.int('num_z_bins') # for lensing sources self.nwbins = ini.int('num_gal_bins', 0) # for galaxies maxbin = max(self.nzbins, self.nwbins) cov_ix = 0 self.bin_pairs = [] self.data_arrays = [] self.thetas = [] for i, tp in enumerate(self.data_types): xi = np.loadtxt(ini.relativeFileName('measurements[%s]' % tp)) bin1 = xi[:, 0].astype(np.int) - 1 bin2 = xi[:, 1].astype(np.int) - 1 tbin = xi[:, 2].astype(np.int) - 1 corr = np.empty((maxbin, maxbin), dtype=np.object) corr[:, :] = None self.data_arrays.append(corr) self.bin_pairs.append([]) for f1, f2, ix, dat in zip(bin1, bin2, tbin, xi[:, 3]): self.indices.append((i, f1, f2, ix)) if not (f1, f2) in self.bin_pairs[i]: self.bin_pairs[i].append((f1, f2)) corr[f1, f2] = np.zeros(ntheta) corr[f1, f2][ix] = dat if ranges[tp][f1, f2] is not None: mn, mx = ranges[tp][f1, f2] if self.theta_bins[ix] > mn and self.theta_bins[ix] < mx: self.thetas.append(self.theta_bins[ix]) self.used_indices.append(cov_ix) self.used_items.append(self.indices[-1]) cov_ix += 1 nz_source = np.loadtxt(ini.relativeFileName('nz_file')) self.zmid = nz_source[:, 1] self.zbin_sp = [] for b in range(self.nzbins): self.zbin_sp += [UnivariateSpline(self.zmid, nz_source[:, b + 3], s=0)] nz_lens = np.loadtxt(ini.relativeFileName('nz_gal_file')) assert (np.array_equal(nz_lens[:, 1], self.zmid)) self.zbin_w_sp = [] for b in range(self.nwbins): self.zbin_w_sp += [UnivariateSpline(self.zmid, nz_lens[:, b + 3], s=0)] self.zmax = self.zmid[-1] self.kmax = ini.float('kmax', 15) # Actually computed, assumes extrapolated beyond that self._initialize()
def load_dataset(self, filename, dataset_params): from getdist import IniFile ini = IniFile(filename) ini.params.update(dataset_params) self.indices = [] self.used_indices = [] self.used_items = [] self.fullcov = np.loadtxt(ini.relativeFileName('cov_file')) ntheta = ini.int('num_theta_bins') self.theta_bins = np.loadtxt(ini.relativeFileName('theta_bins_file')) self.iintrinsic_alignment_model = ini.string( 'intrinsic_alignment_model') self.data_types = ini.string('data_types').split() self.used_types = ini.list('used_data_types', self.data_types) with open(ini.relativeFileName('data_selection')) as f: header = f.readline() assert ('# type bin1 bin2 theta_min theta_max' == header.strip()) lines = f.readlines() ranges = {} for tp in self.data_types: ranges[tp] = np.empty((6, 6), dtype=object) for line in lines: items = line.split() if items[0] in self.used_types: bin1, bin2 = [int(x) - 1 for x in items[1:3]] ranges[items[0]][bin1][bin2] = [ np.float64(x) for x in items[3:] ] self.ranges = ranges self.nzbins = ini.int('num_z_bins') # for lensing sources self.nwbins = ini.int('num_gal_bins', 0) # for galaxies maxbin = max(self.nzbins, self.nwbins) cov_ix = 0 self.bin_pairs = [] self.data_arrays = [] self.thetas = [] for i, tp in enumerate(self.data_types): xi = np.loadtxt(ini.relativeFileName('measurements[%s]' % tp)) bin1 = xi[:, 0].astype(np.int) - 1 bin2 = xi[:, 1].astype(np.int) - 1 tbin = xi[:, 2].astype(np.int) - 1 corr = np.empty((maxbin, maxbin), dtype=np.object) corr[:, :] = None self.data_arrays.append(corr) self.bin_pairs.append([]) for f1, f2, ix, dat in zip(bin1, bin2, tbin, xi[:, 3]): self.indices.append((i, f1, f2, ix)) if not (f1, f2) in self.bin_pairs[i]: self.bin_pairs[i].append((f1, f2)) corr[f1, f2] = np.zeros(ntheta) corr[f1, f2][ix] = dat if ranges[tp][f1, f2] is not None: mn, mx = ranges[tp][f1, f2] if self.theta_bins[ix] > mn and self.theta_bins[ix] < mx: self.thetas.append(self.theta_bins[ix]) self.used_indices.append(cov_ix) self.used_items.append(self.indices[-1]) cov_ix += 1 nz_source = np.loadtxt(ini.relativeFileName('nz_file')) self.zmid = nz_source[:, 1] self.zbin_sp = [] for b in range(self.nzbins): self.zbin_sp += [ UnivariateSpline(self.zmid, nz_source[:, b + 3], s=0) ] nz_lens = np.loadtxt(ini.relativeFileName('nz_gal_file')) assert (np.array_equal(nz_lens[:, 1], self.zmid)) self.zbin_w_sp = [] for b in range(self.nwbins): self.zbin_w_sp += [ UnivariateSpline(self.zmid, nz_lens[:, b + 3], s=0) ] self.zmax = self.zmid[-1] self.kmax = ini.float( 'kmax', 15) # Actually computed, assumes extrapolated beyond that self._initialize()
def __init__(self, dataset, dataset_params={}, silent=False): if not silent: print('loading: %s' % dataset) ini = IniFile(dataset) ini.params.update(dataset_params) spectra = np.loadtxt(ini.relativeFileName('cl_hat_file')) covmat_cl = ini.split('covmat_cl') use_cl = ini.split('use_cl', covmat_cl) if ini.hasKey('use_range'): used_ell = ini.params['use_range'] if isinstance(used_ell, dict): print('Using range %s' % used_ell) for key, value in used_ell.items(): used_ell[key] = range_to_ells(value) else: if not silent: print('Using range: %s' % used_ell) used_ell = range_to_ells(used_ell) else: used_ell = None data_vector = [] nX = 0 used_indices = [] with open(ini.relativeFileName('data_ranges', "r")) as f: lines = f.readlines() while not lines[-1].strip(): lines = lines[:-1] self.Nspec = len(lines) lmin = np.zeros(self.Nspec, dtype=np.int) lmax = np.zeros(self.Nspec, dtype=np.int) self.cl_names = [] self.ell_ranges = np.empty(self.Nspec, dtype=np.object) self.used_sizes = np.zeros(self.Nspec, dtype=np.int) for i, line in enumerate(lines): items = line.split() tp = items[0] self.cl_names.append(tp) lmin[i], lmax[i] = [int(x) for x in items[1:]] if lmax[i] and lmax[i] >= lmin[i]: n = lmax[i] - lmin[i] + 1 data_vector.append(spectra[lmin[i]:lmax[i] + 1, i]) if tp in use_cl: if used_ell is not None and (not isinstance( used_ell, dict) or tp in used_ell): if isinstance(used_ell, dict): ells = used_ell[tp] else: ells = used_ell self.ell_ranges[i] = np.array([ L for L in range(lmin[i], lmax[i] + 1) if L in ells ], dtype=np.int) used_indices.append(self.ell_ranges[i] + (nX - lmin[i])) else: used_indices.append(range(nX, nX + n)) self.ell_ranges[i] = range(lmin[i], lmax[i] + 1) self.used_sizes[i] = len(self.ell_ranges[i]) else: lmax[i] = -1 nX += n self.cl_used = np.array([name in use_cl for name in self.cl_names], dtype=bool) covfile = ini.relativeFileName('covmat_fiducial') with open(covfile, "rb") as f: cov = np.fromfile(f, dtype=[np.float32, np.float64]['64.bin' in covfile]) assert (nX**2 == cov.shape[0]) used_indices = np.concatenate(used_indices) self.data_vector = np.concatenate(data_vector)[used_indices] self.cov = cov.reshape(nX, nX)[np.ix_(used_indices, used_indices)].astype(np.float64) if not silent: for name, mn, mx in zip(self.cl_names, lmin, lmax): if name in use_cl: print(name, mn, mx) print('Number of data points: %s' % self.cov.shape[0]) self.lmax = lmax self.lmin = lmin max_l = np.max(self.lmax) self.ls = np.arange(max_l + 1) self.llp1 = self.ls * (self.ls + 1) if np.any(self.cl_used[:4]): pivot = 3000 self.sz_143 = self.read_normalized( ini.relativeFileName('sz143file'), pivot)[:max_l + 1] self.ksz = self.read_normalized(ini.relativeFileName('kszfile'), pivot)[:max_l + 1] self.tszxcib = self.read_normalized( ini.relativeFileName('tszxcibfile'), pivot)[:max_l + 1] self.cib_217 = self.read_normalized( ini.relativeFileName('cib217file'), pivot)[:max_l + 1] self.dust = np.vstack( (self.read_normalized( ini.relativeFileName('dust100file'))[:max_l + 1], self.read_normalized( ini.relativeFileName('dust143file'))[:max_l + 1], self.read_normalized( ini.relativeFileName('dust217file'))[:max_l + 1], self.read_normalized( ini.relativeFileName('dust143x217file'))[:max_l + 1])) self.lnrat = self.ls * 0 l_min = np.min(lmin[self.cl_used]) self.lnrat[l_min:] = np.log(self.ls[l_min:] / np.float64(pivot)) import hashlib cache_file = self.dataset_file.replace( '.dataset', '_covinv_%s.npy' % hashlib.md5(str(ini.params).encode('utf8')).hexdigest()) if use_cache and os.path.exists(cache_file): self.covinv = np.load(cache_file).astype(np.float64) else: self.covinv = np.linalg.inv(self.cov) if use_cache: np.save(cache_file, self.covinv.astype(np.float32))