def test_Sigma_dust_sync_betas_temp(self): NSIDE = 8 MODEL = 'd0s0' INSTRUMENT = 'LiteBIRD' SIGNAL_TO_NOISE = 10000 UNITS = 'uK_CMB' sky = get_sky(NSIDE, MODEL) instrument = get_instrument(INSTRUMENT) components = [ cm.Dust(150., temp=20., units=UNITS), cm.Synchrotron(150., units=UNITS) ] ref = [] for component in components: ref += component.defaults ref = np.array(ref) freq_maps = get_observation(instrument, sky, unit=UNITS) noise_maps = get_noise_realization(NSIDE, instrument, unit=UNITS) signal = freq_maps[:, 0, 0] # Same signal for all the pixels noise = noise_maps[:, 0] signal_ver = signal / np.dot(signal, signal)**0.5 noise_std = np.std([np.dot(n, signal_ver) for n in noise.T]) maps = signal_ver * noise_std * SIGNAL_TO_NOISE maps = maps[:, np.newaxis] + noise if not hasattr(instrument, 'depth_i'): instrument['depth_i'] = instrument.depth_p / np.sqrt(2) res = basic_comp_sep(components, instrument, maps, nside=NSIDE) diff = (res.x.T - ref) postS = np.mean(diff[..., None] * diff[..., None, :], axis=0) S = res.Sigma.T[0] aac(postS, S, rtol=1. / NSIDE)
def test_basic_comp_sep_T(self): res_T = basic_comp_sep(self.components, self.instrument, self.freq_maps[:, :1, :], tol=1e-12) aac(res_T.x, np.array(self.input), rtol=1e-4) aac(res_T.chi, 0, atol=0.05)
def test_PYMC_Model_unmocked_mcmc_sample(): pymc = inference_methods.PYMC_Model(1000, 2, 500, 'ch0') proteins, result = pymc.mcmc_sample(pd.DataFrame({ 'Protein ID': 'b d c a'.split(), 'ch0': [10, 250, 500, 1000], 'sum': [1000, 1000, 1000, 1000] }), bin_width=0.1) aae(proteins, 'b d c a'.split()) aae(result.sum(axis=1), [2000, 2000, 2000, 2000]) # 1000 samples * 2 chain bins = np.array(range(10)) / 10 aac(np.sum(result * bins[None, :] / 2000, axis=1), [0.38, 0.43, 0.45, 0.66], atol=0.05) proteins, result = pymc.mcmc_sample(pd.DataFrame({ 'Protein ID': ['b', 'd', 'c', 'a'] * 10, 'ch0': [10, 250, 500, 750] * 10, 'sum': [1000, 1000, 1000, 1000] * 10 }), bin_width=0.1) aae(proteins, 'b d c a'.split()) aae(result.sum(axis=1), [2000, 2000, 2000, 2000]) # 1000 samples * 2 chain aac(np.sum(result * bins[None, :] / 2000, axis=1), [0, 0.2, 0.45, 0.7], atol=0.05) # this is low because of binning
def test_Sigma_dust_sync_betas_temp(self): NSIDE = 8 MODEL = 'd0s0' INSTRUMENT = 'litebird' SIGNAL_TO_NOISE = 10000 UNITS = 'uK_CMB' sky = pysm.Sky(get_sky(NSIDE, MODEL)) instrument = pysm.Instrument( get_instrument(INSTRUMENT, NSIDE, units=UNITS)) components = [ cm.Dust(150., temp=20., units=UNITS), cm.Synchrotron(150., units=UNITS) ] ref = [] for component in components: ref += component.defaults ref = np.array(ref) with suppress_stdout(): freq_maps, noise_maps = instrument.observe(sky, write_outputs=False) signal = freq_maps[:, 0, 0] # Same signal for all the pixels noise = noise_maps[:, 0] signal_ver = signal / np.dot(signal, signal)**0.5 noise_std = np.std([np.dot(n, signal_ver) for n in noise.T]) maps = signal_ver * noise_std * SIGNAL_TO_NOISE maps = maps[:, np.newaxis] + noise res = basic_comp_sep(components, instrument, maps, nside=NSIDE) diff = (res.x.T - ref) postS = np.mean(diff[..., None] * diff[..., None, :], axis=0) S = res.Sigma.T[0] aac(postS, S, rtol=1. / NSIDE)
def test_TQU_weights(self): theta = hp.pix2ang(self.nside, np.arange(hp.nside2npix(self.nside)))[0] weights = 1.0 / (1.0 + np.exp(-20.0 * (theta - 0.25 * np.pi))) weights *= 1.0 / (1.0 + np.exp(20.0 * (theta - 0.65 * np.pi))) bins = np.arange(1000) * self.BINS_WIDTH with suppress_stdout(): res = harmonic_ilc(self.components, dict(Frequencies=self.freqs), self.d, lbins=bins, weights=weights, iter=10) lmax = res.cl_out.shape[-1] - 1 ell = np.arange(lmax + 1) ref = self.cl[:, :lmax + 1] norm_diff = (ref - res.cl_out[0, :4]) * np.sqrt((2 * ell + 1) / 2) norm_diff[:3] /= ref[:3] norm_diff = norm_diff[..., :int(2.5 * self.nside)] if False: # Debug plots import pylab as pl pl.plot(norm_diff[..., 2:].T) pl.show() pl.plot(res.cl_in[0].T) pl.plot(self.cl[:4].T, ls='--') pl.show() aac(norm_diff[..., 2:], np.zeros_like(norm_diff[..., 2:]), atol=5) # This is a weak test: # recovery is bad in polarization, mostly at small scales aac(res.s[0], self.s * weights, atol=self.TOL * self.s.max())
def test_P_dBdB_invN(self): invN = self.invN[0, 0] P_dBdB_analytic = P_dBdB(self.A, self.A_dB, self.A_dBdB, self.mm.comp_of_dB, invN) def get_P_displaced(i, j): def P_displaced(i_step, j_step): diff_params = [p for p in self.params] diff_params[i] = i_step * self.DX + diff_params[i] diff_params[j] = j_step * self.DX + diff_params[j] diff_A = self.mm.eval(self.nu, *diff_params) return P(diff_A, invN) return P_displaced for i in range(len(self.params)): for j in range(len(self.params)): Pdx = get_P_displaced(i, j) if i == j: P_dBdB_numerical = ( (-2 * Pdx(0, 0) + Pdx(+1, 0) + Pdx(-1, 0)) / self.DX**2) else: P_dBdB_numerical = ( (Pdx(1, 1) - Pdx(+1, -1) - Pdx(-1, 1) + Pdx(-1, -1)) / (4 * self.DX**2)) aac(P_dBdB_numerical, P_dBdB_analytic[i][j], rtol=3.0e-1)
def test_W_dBdB_invN(self): W_dBdB_analytic = W_dBdB(self.A, self.A_dB, self.A_dBdB, self.mm.comp_of_dB, self.invN) def get_W_displaced(i, j): def W_displaced(i_step, j_step): diff_params = [p for p in self.params] diff_params[i] = i_step * self.DX + diff_params[i] diff_params[j] = j_step * self.DX + diff_params[j] diff_A = self.mm.eval(self.nu, *diff_params) return W(diff_A, self.invN) return W_displaced for i in range(len(self.params)): for j in range(len(self.params)): Wdx = get_W_displaced(i, j) if i == j: W_dBdB_numerical = ( (-2 * Wdx(0, 0) + Wdx(+1, 0) + Wdx(-1, 0)) / self.DX**2) else: W_dBdB_numerical = ( (Wdx(1, 1) - Wdx(+1, -1) - Wdx(-1, 1) + Wdx(-1, -1)) / (4 * self.DX**2)) aac(W_dBdB_numerical, W_dBdB_analytic[i][j], rtol=2.5e-1)
def test_bandpass_integration(): frequency = np.array([[10., 20., 40.], [60., 90., 100.]]) transmittance = np.array([[1., 2., 4.], [1., 3., 1.]]) class Mock(Model): def eval(self, nu=None, par=None): if isinstance(nu, list): return fgf._bandpass_integration() return np.ones_like(nu) * np.array(par)[..., np.newaxis] mock = Mock() nus = list(zip(frequency, transmittance)) ress = mock(nus, 1) refs = [75., 80.] for res, ref in zip(refs, ress): aac(res, ref) ress = mock(nus, np.array([1., 2.])) refs = np.array([[75., 80.], [150., 160.]]) for res, ref in zip(refs, ress): aac(res, ref)
def test_QU_no_ids_no_patchy(self): with suppress_stdout(): res = ilc(self.components, dict(Frequencies=self.freqs), self.d[:, 1:]) aac(res.s[0], self.s[1:], atol=self.TOL) aac(res.freq_cov, self.exp_freq_cov, atol=self.TOL)
def test_QU_ids_patchy(self): patch_ids = np.arange(self.cov[1:].size).reshape(-1, 12) // 4 patch_ids = hp.ud_grade(patch_ids, self.NSIDE) with suppress_stdout(): res = ilc(self.components, dict(Frequencies=self.freqs), self.d_patchy[:, 1:], patch_ids) aac(res.s[0], self.s_patchy[1:], atol=self.TOL)
def test_TQU_ids_no_patchy(self): patch_ids = np.arange(self.cov.size).reshape(-1, 12) // 4 patch_ids = hp.ud_grade(patch_ids, self.NSIDE) with suppress_stdout(): res = ilc(self.components, dict(frequency=self.freqs), self.d, patch_ids) aac(res.s[0], self.s, atol=self.TOL)
def test_TQU_1_id_no_patchy(self): # No patch and one patch has to give the same result patch_ids = np.zeros(hp.nside2npix(self.NSIDE), dtype=int) with suppress_stdout(): res = ilc(self.components, dict(Frequencies=self.freqs), self.d) res_patch = ilc(self.components, dict(Frequencies=self.freqs), self.d, patch_ids) aac(res.s, res_patch.s) aac(res.freq_cov, res_patch.freq_cov[0])
def test(self): cm = NoCrossCorrelation(truncation_level=3.) imts = [PGA(), SA(0.3), SA(0.6), SA(1.0)] numpy.random.seed(42) eps = cm.get_inter_eps(imts, 2) # a 4x2 matrix aac(eps, numpy.array([[-0.318959, 1.640001], [0.616954, 0.249188], [-1.007084, -1.007184], [-1.560874, 1.103927]]), rtol=1e-5)
def test(self): cm = FullCrossCorrelation(truncation_level=3.) imts = [PGA(), SA(0.3), SA(0.6), SA(1.0)] numpy.random.seed(42) eps = cm.get_inter_eps(imts, 2) # a 4x2 matrix with same eps per IMT aac(eps, numpy.array([[-0.318959, 1.640001], [-0.318959, 1.640001], [-0.318959, 1.640001], [-0.318959, 1.640001]]), rtol=1e-5)
def test_W_dB(self): W_dB_analytic = W_dB(self.A, self.A_dB, self.mm.comp_of_dB) W_params = W(self.A) for i in range(len(self.params)): diff_params = [p for p in self.params] diff_params[i] = self.DX + diff_params[i] diff_A = self.mm.eval(self.nu, *diff_params) diff_W = W(diff_A) W_dB_numerical = (diff_W - W_params) / self.DX aac(W_dB_numerical, W_dB_analytic[i], rtol=1e-3)
def test_QU_ids_patchy_mask(self): mask_good = (np.arange(hp.nside2npix(self.NSIDE)) % 13).astype(bool) patch_ids = np.arange(self.cov[1:].size).reshape(-1, 12) // 4 patch_ids = hp.ud_grade(patch_ids, self.NSIDE) data = self.d_patchy[:, 1:].copy() data[..., ~mask_good] = hp.UNSEEN ref = self.s_patchy[1:].copy() ref[..., ~mask_good] = hp.UNSEEN with suppress_stdout(): res = ilc(self.components, dict(Frequencies=self.freqs), data, patch_ids) aac(res.s[0], ref, atol=self.TOL)
def test_dependence_on_nu0_RJ(self): NSIDE = 8 MODEL = 'c1s0' INSTRUMENT = 'litebird' sky = pysm.Sky(get_sky(NSIDE, MODEL)) instrument = pysm.Instrument( get_instrument(INSTRUMENT, NSIDE, units='uK_RJ')) components100 = [ cm.CMB(units='K_RJ'), cm.Synchrotron(100., units='K_RJ') ] components10 = [ cm.CMB(units='K_RJ'), cm.Synchrotron(10., units='K_RJ') ] with suppress_stdout(): freq_maps, _ = instrument.observe(sky, write_outputs=False) res100 = basic_comp_sep(components100, instrument, freq_maps) res10 = basic_comp_sep(components10, instrument, freq_maps) aac(res100.Sigma, res10.Sigma) aac(res100.x, res10.x) aac(res100.s[0], res10.s[0]) aac(res100.s[1], res10.s[1] * 10**res10.x[0])
def test(self, tag): # NOTE: this does not test the funciton when the nsides are actually # defferent. It just checks that, if they are all the same, the results # is equal to the basic_comp_sep result despite all the parameters were # fitted at once. _, sky_tag, comp_sep_tag = tag.split('___') data, s, x = _get_sky(sky_tag) components, instrument, nsidepar = comp_sep_tag.split('__') components = _get_component(components) for c in components: c.defaults = [1.1 * d for d in c.defaults] instrument = _get_instrument(instrument) nsidepar = _get_nside(nsidepar) assert len(nsidepar) == 1 nsidepar = nsidepar[0] # basic comp sep call res_multipatch = basic_comp_sep(components, instrument, data, nsidepar) # multi res comp sep call, with equal resolutions for each spectral index res_multires = multi_res_comp_sep(components, instrument, data, nsides=[nsidepar] * len(x)) aac(res_multipatch.s, s, rtol=2e-5) aac(res_multires.s, s, rtol=2e-5) aac(res_multipatch.s, res_multires.s, rtol=2e-5) for res_x, xx in zip(res_multires.x, x): aac(res_x, xx, rtol=2e-5)
def test(self, tag): _, sky_tag, comp_sep_tag = tag.split('___') data, s, x = _get_sky(sky_tag) components, instrument, nsidepar = comp_sep_tag.split('__') components = _get_component(components) for c in components: c.defaults = [1.1 * d for d in c.defaults] nside = hp.get_nside(data[0]) instrument = _get_instrument(instrument, nside) nsidepar = _get_nside(nsidepar) assert len(nsidepar) == 1 nsidepar = nsidepar[0] cov = self._get_cov(instrument, sky_tag.split('__')[0], nside) res = weighted_comp_sep(components, instrument, data, cov, nsidepar) if len(x): aac(res.x, x, rtol=1e-5) aac(res.s, s, rtol=1e-4) aac(res.chi[data == hp.UNSEEN], hp.UNSEEN, rtol=0) aac(res.chi[data != hp.UNSEEN], 0, atol=0.05)
def test_1_rlz(self): # test with one GMPE, 1 TRT, checking additivity inp = read_input( PARAM, gsim_logic_tree_file=os.path.join(CWD, 'data', 'lt01.xml')) [cmaker] = inp.cmakerdict.values() [src_group] = inp.groups ctxs = cmaker.from_srcs(src_group, inp.sitecol) assert len(ctxs) == 100 ctxs1 = ctxs[:50] ctxs2 = ctxs[50:] c1, s1 = cmaker.get_cs_contrib(ctxs1, imti, imls) c2, s2 = cmaker.get_cs_contrib(ctxs2, imti, imls) c, s = cmaker.get_cs_contrib(ctxs, imti, imls) aac((c1 + c2) / (s1 + s2), c / s)
def test(self): corma = self.cm._get_correlation_matrix(self.imts) aac( corma, numpy.array([[1., 0.71678166, 0.41330149, 0.23046633], [0.71678166, 1., 0.83261724, 0.68322083], [0.41330149, 0.83261724, 1., 0.88167281], [0.23046633, 0.68322083, 0.88167281, 1.]])) numpy.random.seed(42) eps = self.cm.get_inter_eps(self.imts, 2) # a 4x2 matrix aac(eps, numpy.array([[-0.60624845, -0.3066977], [-0.13441677, 0.62122982], [-0.667341, 0.28429873], [-0.3573499, 0.0350888]]), rtol=1e-5)
def test_no_stokes(self): NFREQ = 3 NSIDE = 2 np.random.seed(0) alms = [ hp.map2alm(np.random.normal(size=(12 * NSIDE**2))) for i in range(NFREQ) ] res = _empirical_harmonic_covariance(alms) ref = np.empty_like(res) for f1 in range(NFREQ): for f2 in range(NFREQ): ref[f1, f2] = hp.alm2cl(alms[f1], alms[f2]) aac(ref, res)
def test_1_rlz(self): # test with one GMPE, 1 TRT, checking additivity inp = read_input(PARAM, gsim_logic_tree_file=os.path.join( CWD, 'data', 'lt01.xml')) [cmaker] = inp.cmakerdict.values() [src_group] = inp.groups ctxs = cmaker.from_srcs(src_group, inp.sitecol) assert len(ctxs) == 100 ctxs1 = ctxs[:50] ctxs2 = ctxs[50:] dic1 = cmaker.get_cs_contrib(ctxs1, imti, imls)[0] dic2 = cmaker.get_cs_contrib(ctxs2, imti, imls)[0] dic = cmaker.get_cs_contrib(ctxs, imti, imls)[0] aac((dic1['_c'] + dic2['_c']) / (dic1['_s'] + dic2['_s']), dic['_c'] / dic['_s'])
def test_PYMC_Model_fit_histogram(pymc, mocker): mock_sample = mocker.patch.object(inference_methods.PYMC_Model, 'mcmc_sample', return_value=('a b c'.split(), np.array( [[0, 10, 20, 10, 0], [40, 0, 0, 0, 0], [0, 0, 0, 30, 10]]))) result = pymc.fit_histogram( pd.DataFrame({'Protein ID': 'a b c'.split()}), # doesn't matter bin_width=0.2) mock_sample.assert_called_with(mocker.ANY, 0.2) # because pandas df equality is infuriating! assert (result.index == 'a b c'.split()).all() assert result.index.name == 'Protein ID' aac(result.columns.values, [0, 0.2, 0.4, 0.6, 0.8]) assert (result.values == [[0, 10, 20, 10, 0], [40, 0, 0, 0, 0], [0, 0, 0, 30, 10]]).all()
def test_point(self): # point source with 3 ruptures, checking additivity inp = read_input( PARAM, source_model_file=os.path.join(CWD, 'data', 'point.xml'), gsim_logic_tree_file=os.path.join(CWD, 'data', 'lt01.xml')) [cmaker] = inp.cmakerdict.values() [src_group] = inp.groups ctxs = cmaker.from_srcs(src_group, inp.sitecol) aac([ctx.occurrence_rate for ctx in ctxs], [0.00018, 0.00018, 0.00054]) ctxs1 = ctxs[:2] ctxs2 = ctxs[2:] # check that the total spectra is a weighted mean of the two # rupture spectra; the weight is the same for all IMTs c1, s1 = cmaker.get_cs_contrib(ctxs1, imti, imls) c2, s2 = cmaker.get_cs_contrib(ctxs2, imti, imls) comp_spectra = (c1 + c2) / (s1 + s2) c, s = cmaker.get_cs_contrib(ctxs, imti, imls) aac(comp_spectra, c / s)
def test_2_rlzs(self): # test with two GMPEs, 1 TRT inp = read_input(PARAM) [cmaker] = inp.cmakerdict.values() [src_group] = inp.groups ctxs = cmaker.from_srcs(src_group, inp.sitecol) [c_], [s_] = cmaker.get_cs_contrib(ctxs, imti, imls) spectra = [c / s for c, s in zip(c_, s_)] # check the result expected = os.path.join(CWD, 'expected', 'spectra2.csv') if OVERWRITE_EXPECTED: rlzs = list(inp.gsim_lt) spectra_to_df(spectra, cmaker.imts, rlzs).to_csv( expected, index=False, line_terminator='\r\n', float_format='%.6f') df = pandas.read_csv(expected) for g, gsim in enumerate(cmaker.gsims): dfg = df[df.rlz_id == g] aac(dfg.cs_exp, np.exp(spectra[g][0]), atol=1e-6) aac(dfg.cs_std, np.sqrt(spectra[g][1]), atol=1e-6)
def test_6_rlzs(self): # test with 2x3 realizations and TRTA, TRTB # rlzs_by_g = 012, 345, 03, 14, 25 inp = read_input( PARAM, source_model_file=os.path.join(CWD, 'data', 'sm02.xml')) rlzs = list(inp.gsim_lt) R = len(rlzs) # compute the contributions by trt all_cs = [] for src_group in inp.groups: cmaker = inp.cmakerdict[src_group.trt] ctxs = cmaker.from_srcs(src_group, inp.sitecol) [c], [s] = cmaker.get_cs_contrib(ctxs, imti, imls) for cs in zip(c, s): all_cs.append(cs) # compose the contributions by rlz, 0+2, 0+3, 0+4, 1+2, 1+3, 1+4 rlzs_by_g = inp.gsim_lt.get_rlzs_by_g() nums = np.zeros((R, 2, len(cmaker.imts))) denums = np.zeros(R) for g, rlz_ids in enumerate(rlzs_by_g): c, s = all_cs[g] for r in rlz_ids: nums[r] += c denums[r] += s spectra = [nums[r] / denums[r] for r in range(R)] # check the results expected = os.path.join(CWD, 'expected', 'spectra6.csv') if OVERWRITE_EXPECTED: spectra_to_df(spectra, cmaker.imts, rlzs).to_csv( expected, index=False, line_terminator='\r\n', float_format='%.6f') df = pandas.read_csv(expected) for rlz in rlzs: r = rlz.ordinal df_rlz = df[df.rlz_id == r] aac(df_rlz.cs_exp, np.exp(spectra[r][0]), atol=1e-6) aac(df_rlz.cs_std, np.sqrt(spectra[r][1]), atol=1e-6)
def test_TQU(self): bins = np.arange(1000) * self.BINS_WIDTH with suppress_stdout(): res = harmonic_ilc(self.components, dict(Frequencies=self.freqs), self.d, lbins=bins, iter=10) lmax = res.cl_out.shape[-1] - 1 ell = np.arange(lmax + 1) ref = self.cl[:, :lmax + 1] norm_diff = (ref - res.cl_out[0, :4]) * np.sqrt((2 * ell + 1) / 2) norm_diff[:3] /= ref[:3] aac(norm_diff[..., 2:int(2.5 * self.nside)], np.zeros_like(norm_diff[..., 2:int(2.5 * self.nside)]), atol=5) # This is a very weak test: # recovery is bad at small scales at the poles, especially in Q and U aac(res.s[0], self.s, atol=3 * self.TOL * self.s.max())
def test(self, tag): _, sky_tag, comp_sep_tag = tag.split('___') data, s, x = _get_sky(sky_tag) components, instrument, nsidepar = comp_sep_tag.split('__') components = _get_component(components) for c in components: c.defaults = [1.1 * d for d in c.defaults] instrument = _get_instrument(instrument) nsidepar = _get_nside(nsidepar) # multi res comp sep call, with equal resolutions for each spectral index res_multires = multi_res_comp_sep(components, instrument, data, nsides=nsidepar) aac(res_multires.s, s, rtol=2e-5) for res_x, xx in zip(res_multires.x, x): aac(res_x, xx, rtol=2e-5)
def test_PYMC_Model_fit_quantiles(pymc, mocker): return_quants = np.zeros((3, 10000)) return_quants[0, 1000:1101] = 1 # 100 1's return_quants[1, 0] = 1 # first index return_quants[2, 9999] = 1 # last index mock_sample = mocker.patch.object(inference_methods.PYMC_Model, 'mcmc_sample', return_value=('a c b'.split(), return_quants)) result = pymc.fit_quantiles( pd.DataFrame({'Protein ID': 'a b c'.split()}), # doesn't matter [0.025, 0.5, 0.975]) assert mock_sample.call_args[1]['bin_width'] == 0.0001 assert (result.index == 'a c b'.split()).all() assert result.index.name == 'Protein ID' assert (result.columns.values == ['0.025', '0.5', '0.975']).all() print(result) aac(result.values, [ [0.1002, 0.1050, 0.1098], [0, 0, 0], [0.9999, 0.9999, 0.9999], ])