def test_zestimation(nspec: int):
filename = filenames[nspec]
if not os.path.exists(filename):
plate, mjd, fiber_id = re.findall(
r"spec-([0-9]+)-([0-9]+)-([0-9]+).fits",
filename,
)[0]
retrieve_raw_spec(int(plate), int(mjd), int(fiber_id))
params = ZParameters()
z_qso_samples = ZSamples(params)
wavelengths, flux, noise_variance, pixel_mask = read_spec(filename)
z_qso_gp = ZGPMAT(
params,
z_qso_samples,
learned_file=
"data/dr12q/processed/learned_zqso_only_model_outdata_normout_dr9q_minus_concordance_norm_1176-1256.mat",
)
tic = time.time()
z_qso_gp.inference_z_qso(wavelengths, flux, noise_variance, pixel_mask)
print("Z True : {:.3g}".format(z_qsos[nspec]))
toc = time.time()
print("spent {} mins; {} seconds".format((toc - tic) // 60,
(toc - tic) % 60))
return z_qso_gp.z_map, z_qsos[nspec]
def test_read_spec():
if not os.path.exists("spec-7340-56825-0576.fits"):
retrieve_raw_spec(7340, 56825, 576) # an arbitrary spectrum
wavelengths, flux, noise_variance, pixel_mask = read_spec(
"spec-7340-56825-0576.fits"
)
assert min(wavelengths) > 1216
assert len(flux) == len(noise_variance)
assert type(pixel_mask[0]) is np.bool_
def download_ho_2020_spectrum(num_quasars: int = 5):
"""
Download first N spectra from Ho-Bird-Garnett (2020) catalogue.
"""
assert num_quasars <= 100
# first 100 from catalogue
plates = np.array([
6173, 6177, 4354, 6498, 6177, 4216, 6182, 4296, 7134, 6877, 6177, 4277,
4415, 4216, 4216, 7167, 6177, 4354, 7144, 6177, 7147, 7144, 6511, 6511,
6151, 4216, 4535, 6182, 7034, 6177, 6151, 6498, 7147, 6182, 4354, 6177,
6177, 4354, 6879, 6151, 7144, 4354, 4277, 6879, 6498, 6182, 6879, 4535,
7167, 6879, 4535, 4216, 4216, 4415, 6182, 6511, 6207, 4216, 6177, 4296,
4277, 7034, 4277, 6152, 6172, 7033, 4216, 4277, 6498, 7033, 4415, 4535,
6877, 6170, 4296, 6498, 6513, 6177, 4535, 6151, 4216, 4296, 4296, 7147,
4535, 4296, 7167, 6172, 4535, 6172, 4216, 7147, 4296, 7167, 4216, 7147,
4296, 6177, 6879, 7034
])
mjds = np.array([
56238, 56268, 55810, 56565, 56268, 55477, 56190, 55499, 56566, 56544,
56268, 55506, 55831, 55477, 55477, 56604, 56268, 55810, 56564, 56268,
56574, 56564, 56540, 56540, 56265, 55477, 55860, 56190, 56564, 56268,
56265, 56565, 56574, 56190, 55810, 56268, 56268, 55810, 56539, 56265,
56564, 55810, 55506, 56539, 56565, 56190, 56539, 55860, 56604, 56539,
55860, 55477, 55477, 55831, 56190, 56540, 56239, 55477, 56268, 55499,
55506, 56564, 55506, 56164, 56269, 56565, 55477, 55506, 56565, 56565,
55831, 55860, 56544, 56240, 55499, 56565, 56543, 56268, 55860, 56265,
55477, 55499, 55499, 56574, 55860, 55499, 56604, 56269, 55860, 56269,
55477, 56574, 55499, 56604, 55477, 56574, 55499, 56268, 56539, 56564
])
fiber_ids = np.array([
528, 595, 646, 177, 608, 312, 652, 364, 594, 564, 648, 896, 554, 302,
292, 290, 384, 686, 752, 640, 860, 266, 92, 86, 88, 732, 680, 342, 358,
386, 936, 844, 171, 338, 702, 584, 393, 709, 439, 78, 221, 700, 872,
580, 838, 326, 436, 302, 259, 427, 361, 718, 276, 466, 642, 114, 134,
724, 360, 386, 862, 657, 106, 4, 643, 2, 290, 152, 157, 14, 580, 315,
440, 573, 390, 158, 892, 366, 316, 954, 280, 656, 630, 138, 734, 382,
796, 628, 304, 342, 756, 889, 398, 238, 248, 900, 392, 656, 405, 647
])
for plate, mjd, fiber_id in zip(plates[:num_quasars], mjds[:num_quasars],
fiber_ids[:num_quasars]):
filename = "spec-{}-{}-{}.fits".format(plate, mjd,
str(fiber_id).zfill(4))
print(filename)
if not os.path.exists(filename):
retrieve_raw_spec(plate, mjd, fiber_id) # the spectrum at paper
def test_prior():
# test 1
filename = "spec-5309-55929-0362.fits"
if not os.path.exists(filename):
retrieve_raw_spec(5309, 55929, 362) # the spectrum at paper
z_qso = 3.166
param = Parameters()
# prepare these files by running the MATLAB scripts until build_catalog.m
prior = PriorCatalog(
param,
"data/dr12q/processed/catalog.mat",
"data/dla_catalogs/dr9q_concordance/processed/los_catalog",
"data/dla_catalogs/dr9q_concordance/processed/dla_catalog",
)
dla_samples = DLASamplesMAT(param, prior,
"data/dr12q/processed/dla_samples_a03.mat")
wavelengths, flux, noise_variance, pixel_mask = read_spec(filename)
rest_wavelengths = param.emitted_wavelengths(wavelengths, z_qso)
# DLA GP Model
dla_gp = DLAGPMAT(
param,
prior,
dla_samples,
3000.0,
"data/dr12q/processed/learned_qso_model_lyseries_variance_kim_dr9q_minus_concordance.mat",
True,
)
dla_gp.set_data(rest_wavelengths,
flux,
noise_variance,
pixel_mask,
z_qso,
build_model=True)
log_priors = dla_gp.log_priors(z_qso, max_dlas=4)
catalog_log_priors = np.array(
[-2.53774598, -4.97413739, -7.40285925, -9.74851888])
assert np.all(np.abs(log_priors - catalog_log_priors) < 1e-4)
def prepare_subdla_model(plate: int = 5309,
mjd: int = 55929,
fiber_id: int = 362,
z_qso: float = 3.166) -> SubDLAGPMAT:
"""
Return a SubDLAGP instance from an input SDSS DR12 spectrum.
"""
filename = "spec-{}-{}-{}.fits".format(plate, mjd, str(fiber_id).zfill(4))
if not os.path.exists(filename):
retrieve_raw_spec(plate, mjd, fiber_id) # the spectrum at paper
param = Parameters()
# prepare these files by running the MATLAB scripts until build_catalog.m
prior = PriorCatalog(
param,
"data/dr12q/processed/catalog.mat",
"data/dla_catalogs/dr9q_concordance/processed/los_catalog",
"data/dla_catalogs/dr9q_concordance/processed/dla_catalog",
)
subdla_samples = SubDLASamplesMAT(
param, prior, "data/dr12q/processed/subdla_samples.mat")
wavelengths, flux, noise_variance, pixel_mask = read_spec(filename)
rest_wavelengths = param.emitted_wavelengths(wavelengths, z_qso)
# DLA GP Model
subdla_gp = SubDLAGPMAT(
param,
prior,
subdla_samples,
3000.0,
"data/dr12q/processed/learned_qso_model_lyseries_variance_kim_dr9q_minus_concordance.mat",
True,
)
subdla_gp.set_data(rest_wavelengths,
flux,
noise_variance,
pixel_mask,
z_qso,
build_model=True)
return subdla_gp
def test_log_likelihood_no_dla():
# test 1
filename = "spec-5309-55929-0362.fits"
if not os.path.exists(filename):
retrieve_raw_spec(5309, 55929, 362) # the spectrum at paper
z_qso = 3.166
param = Parameters()
# prepare these files by running the MATLAB scripts until build_catalog.m
prior = PriorCatalog(
param,
"data/dr12q/processed/catalog.mat",
"data/dla_catalogs/dr9q_concordance/processed/los_catalog",
"data/dla_catalogs/dr9q_concordance/processed/dla_catalog",
)
wavelengths, flux, noise_variance, pixel_mask = read_spec(filename)
rest_wavelengths = param.emitted_wavelengths(wavelengths, z_qso)
gp = NullGPMAT(
param,
prior,
"data/dr12q/processed/learned_qso_model_lyseries_variance_kim_dr9q_minus_concordance.mat",
)
gp.set_data(rest_wavelengths,
flux,
noise_variance,
pixel_mask,
z_qso,
build_model=True)
log_likelihood_no_dla = gp.log_model_evidence()
print(
"log p( D | z_QSO, no DLA ) : {:.5g}".format(log_likelihood_no_dla))
assert (np.abs(log_likelihood_no_dla - (-889.04809017)) < 1
) # there is some numerical difference
plt.figure(figsize=(16, 5))
plt.plot(gp.x, gp.y, label="observed flux")
plt.plot(gp.rest_wavelengths, gp.mu, label="null GP")
plt.plot(gp.x, gp.this_mu, label="interpolated null GP")
plt.xlabel("rest wavelengths")
plt.ylabel("normalised flux")
plt.legend()
plt.savefig("test1.pdf", format="pdf", dpi=300)
plt.clf()
plt.close()
# test 2
filename = "spec-3816-55272-0076.fits"
z_qso = 3.68457627
if not os.path.exists(filename):
retrieve_raw_spec(3816, 55272, 76) # the spectrum at paper
wavelengths, flux, noise_variance, pixel_mask = read_spec(filename)
rest_wavelengths = param.emitted_wavelengths(wavelengths, z_qso)
gp.set_data(rest_wavelengths,
flux,
noise_variance,
pixel_mask,
z_qso,
build_model=True)
log_likelihood_no_dla = gp.log_model_evidence()
print(
"log p( D | z_QSO, no DLA ) : {:.5g}".format(log_likelihood_no_dla))
assert np.abs(log_likelihood_no_dla - (-734.3727266)) < 1
plt.figure(figsize=(16, 5))
plt.plot(gp.x, gp.y, label="observed flux")
plt.plot(gp.rest_wavelengths, gp.mu, label="null GP")
plt.plot(gp.x, gp.this_mu, label="interpolated null GP")
plt.xlabel("rest wavelengths")
plt.ylabel("normalised flux")
plt.legend()
plt.savefig("test2.pdf", format="pdf", dpi=300)
plt.clf()
plt.close()
def test_dla_model_evidences(broadening: bool = True):
# test 1
filename = "spec-5309-55929-0362.fits"
if not os.path.exists(filename):
retrieve_raw_spec(5309, 55929, 362) # the spectrum at paper
z_qso = 3.166
param = Parameters()
# prepare these files by running the MATLAB scripts until build_catalog.m
prior = PriorCatalog(
param,
"data/dr12q/processed/catalog.mat",
"data/dla_catalogs/dr9q_concordance/processed/los_catalog",
"data/dla_catalogs/dr9q_concordance/processed/dla_catalog",
)
dla_samples = DLASamplesMAT(param, prior,
"data/dr12q/processed/dla_samples_a03.mat")
wavelengths, flux, noise_variance, pixel_mask = read_spec(filename)
rest_wavelengths = param.emitted_wavelengths(wavelengths, z_qso)
# DLA GP Model
dla_gp = DLAGPMAT(
param,
prior,
dla_samples,
3000.0,
"data/dr12q/processed/learned_qso_model_lyseries_variance_kim_dr9q_minus_concordance.mat",
broadening,
)
dla_gp.set_data(rest_wavelengths,
flux,
noise_variance,
pixel_mask,
z_qso,
build_model=True)
tic = time.time()
max_dlas = 4
log_likelihoods_dla = dla_gp.log_model_evidences(max_dlas)
toc = time.time()
# very time consuming: ~ 4 mins for a single spectrum without parallelized.
print("spent {} mins; {} seconds".format((toc - tic) // 60,
(toc - tic) % 60))
# log likelihood results from the catalog
catalog_log_likelihoods_dla = np.array(
[-688.91647288, -633.00070813, -634.08569242, -640.77120558])
for i in range(max_dlas):
print("log p( D | z_QSO, DLA{} ) : {:.5g}; MATLAB value: {:.5g}".
format(i + 1, log_likelihoods_dla[i],
catalog_log_likelihoods_dla[i]))
# the accuracy down to 2.5 in log scale, this needs to be investigated.
assert np.all(
np.abs(catalog_log_likelihoods_dla - log_likelihoods_dla) < 2.5)
def test_dla_model(broadening: bool = True):
# test 1
filename = "spec-5309-55929-0362.fits"
if not os.path.exists(filename):
retrieve_raw_spec(5309, 55929, 362) # the spectrum at paper
z_qso = 3.166
param = Parameters()
# prepare these files by running the MATLAB scripts until build_catalog.m
prior = PriorCatalog(
param,
"data/dr12q/processed/catalog.mat",
"data/dla_catalogs/dr9q_concordance/processed/los_catalog",
"data/dla_catalogs/dr9q_concordance/processed/dla_catalog",
)
dla_samples = DLASamplesMAT(param, prior,
"data/dr12q/processed/dla_samples_a03.mat")
wavelengths, flux, noise_variance, pixel_mask = read_spec(filename)
rest_wavelengths = param.emitted_wavelengths(wavelengths, z_qso)
# DLA GP Model
dla_gp = DLAGPMAT(
param,
prior,
dla_samples,
3000.0,
"data/dr12q/processed/learned_qso_model_lyseries_variance_kim_dr9q_minus_concordance.mat",
broadening,
)
dla_gp.set_data(rest_wavelengths,
flux,
noise_variance,
pixel_mask,
z_qso,
build_model=True)
# These are the MAPs from the paper
z_dlas = np.array([2.52182382, 3.03175723])
nhis = 10**np.array([20.63417494, 22.28420156])
sample_log_likelihood_dla = dla_gp.sample_log_likelihood_k_dlas(
z_dlas, nhis)
print("log p( D | z_QSO, zdlas, nhis ) : {:.5g}".format(
sample_log_likelihood_dla))
# Build a Null model
gp = NullGPMAT(
param,
prior,
"data/dr12q/processed/learned_qso_model_lyseries_variance_kim_dr9q_minus_concordance.mat",
)
gp.set_data(rest_wavelengths,
flux,
noise_variance,
pixel_mask,
z_qso,
build_model=True)
log_likelihood_no_dla = gp.log_model_evidence()
print(
"log p( D | z_QSO, no DLA ) : {:.5g}".format(log_likelihood_no_dla))
assert sample_log_likelihood_dla > log_likelihood_no_dla