def test_xwick(self):
"""
Test wick covariances for cross
"""
userprint("\n")
### Send
cmd = "picca_xwick.py"
cmd += " --in-dir " + self._masterFiles + "/test_delta/Delta_LYA/"
cmd += " --drq " + self._masterFiles + "/test_delta/cat.fits"
cmd += " --out " + self._branchFiles + "/Products/Correlations/xwick.fits.gz"
cmd += " --cf1d " + self._masterFiles + "/test_cor/cf1d.fits.gz"
cmd += " --rp-min -60.0"
cmd += " --rp-max +60.0"
cmd += " --rt-max +60.0"
cmd += " --np 30"
cmd += " --nt 15"
cmd += " --rej 0.99"
cmd += " --nproc 1"
print(repr(cmd))
picca.bin.picca_xwick.main(cmd.split()[1:])
### Test
if self._test:
path1 = self._masterFiles + "/test_cor/xwick.fits.gz"
path2 = self._branchFiles + "/Products/Correlations/xwick.fits.gz"
self.compare_fits(path1, path2, "picca_xwick.py")
return
def send_metal_dmat_cross(self):
userprint("\n")
### Send
cmd = " picca_metal_dmat.py"
cmd += " --in-dir " + self._branchFiles + "/Products/Delta_LYA/Delta/"
cmd += " --in-dir2 " + self._branchFiles + "/Products/Delta_LYA/Delta/"
cmd += " --out " + self._branchFiles + "/Products/Correlations/metal_dmat_cross.fits.gz"
cmd += " --abs-igm SiIII\(1207\)"
cmd += " --abs-igm2 SiIII\(1207\)"
cmd += " --rp-min -60.0"
cmd += " --rp-max +60.0"
cmd += " --rt-max +60.0"
cmd += " --np 30"
cmd += " --nt 15"
cmd += " --rej 0.99 "
cmd += " --nproc 1"
cmd += ' --remove-same-half-plate-close-pairs'
cmd += " --unfold-cf"
subprocess.call(cmd, shell=True)
### Test
if self._test:
path1 = self._masterFiles + "/metal_dmat_cross.fits.gz"
path2 = self._branchFiles + "/Products/Correlations/metal_dmat_cross.fits.gz"
self.compare_fits(path1, path2, "picca_metal_dmat.py")
return
def send_xwick(self):
userprint("\n")
### Send
cmd = " picca_xwick.py"
cmd += " --in-dir " + self._branchFiles + "/Products/Delta_LYA/Delta/"
cmd += " --drq " + self._branchFiles + "/Products/cat.fits"
cmd += " --out " + self._branchFiles + "/Products/Correlations/xwick.fits.gz"
cmd += " --cf1d " + self._branchFiles + "/Products/Correlations/cf1d.fits.gz"
cmd += " --rp-min -60.0"
cmd += " --rp-max +60.0"
cmd += " --rt-max +60.0"
cmd += " --np 30"
cmd += " --nt 15"
cmd += " --rej 0.99 "
cmd += " --nproc 1"
subprocess.call(cmd, shell=True)
### Test
if self._test:
path1 = self._masterFiles + "/xwick.fits.gz"
path2 = self._branchFiles + "/Products/Correlations/xwick.fits.gz"
self.compare_fits(path1, path2, "picca_xwick.py")
return
def send_delta_Pk1D(self):
userprint("\n")
### Path
path_to_etc = self.picca_base + '/etc/'
### Send
cmd = " picca_deltas.py"
cmd += " --in-dir " + self._masterFiles + "/test_Pk1D/Spectra_test/"
cmd += " --drq " + self._masterFiles + "/test_Pk1D/DRQ_test.fits"
cmd += " --out-dir " + self._branchFiles + "/Products/Delta_Pk1D/Delta/"
cmd += " --iter-out-prefix " + self._branchFiles + "/Products/Delta_Pk1D/Log/delta_attributes"
cmd += " --log " + self._branchFiles + "/Products/Delta_Pk1D/Log/input.log"
cmd += " --delta-format Pk1D --mode spec --order 0 --use-constant-weight"
cmd += " --rebin 1 --lambda-min 3650. --lambda-max 7200.0 --lambda-rest-min 1050.0 --lambda-rest-max 1180"
cmd += " --nproc 1"
cmd += " --best-obs"
cmd += " --mask-file " + path_to_etc + "/list_veto_line_Pk1D.txt"
subprocess.call(cmd, shell=True)
### Test
if self._test:
path1 = self._masterFiles + "/test_Pk1D/delta_attributes_Pk1D.fits.gz"
path2 = self._branchFiles + "/Products/Delta_Pk1D/Log/delta_attributes.fits.gz"
self.compare_fits(path1, path2, "picca_deltas.py")
path1 = self._masterFiles + "/test_Pk1D/delta-64_Pk1D.fits.gz"
path2 = self._branchFiles + "/Products/Delta_Pk1D/Delta/delta-64.fits.gz"
self.compare_fits(path1, path2, "picca_deltas.py")
path1 = self._masterFiles + "/test_Pk1D/delta-80_Pk1D.fits.gz"
path2 = self._branchFiles + "/Products/Delta_Pk1D/Delta/delta-80.fits.gz"
self.compare_fits(path1, path2, "picca_deltas.py")
return
def compare_values(val1, val2):
if not np.array_equal(val1, val2):
userprint("WARNING: {}: not exactly equal, using allclose".format(
nameRun))
allclose = np.allclose(val1, val2)
self.assertTrue(allclose, "{}".format(nameRun))
return
def cached_growth_factor_de(z, zref=None, Om=None, OL=None, **kwargs):
'''
Implements eq. 7.77 from S. Dodelson's Modern Cosmology book
'''
userprint('Calculating growth factor for Om = {}, OL = {}'.format(Om, OL))
def hubble(z, Om, OL):
return np.sqrt(Om * (1 + z)**3 + OL + (1 - Om - OL) * (1 + z)**2)
def dD1(a, Om, OL):
z = 1 / a - 1
return 1. / (a * hubble(z=z, Om=Om, OL=OL))**3
## Calculate D1 in 100 values of z between 0 and zmax, then interpolate
nbins = 100
zmax = 5.
z = zmax * np.arange(nbins, dtype=float) / (nbins - 1)
D1 = np.zeros(nbins, dtype=float)
pars = (Om, OL)
for i in range(nbins):
a = 1 / (1 + z[i])
D1[i] = 5 / 2. * Om * hubble(z[i], *pars) * quad(dD1, 0, a,
args=pars)[0]
D1 = interp1d(z, D1)
return D1
def test_metal_dmat_cross(self):
"""
Test metal cross distortion matrix
"""
importlib.reload(picca.cf)
userprint("\n")
### Send
cmd = "picca_metal_dmat.py"
cmd += " --in-dir " + self._masterFiles + "/test_delta/Delta_LYA/"
cmd += " --in-dir2 " + self._masterFiles + "/test_delta/Delta_LYA/"
cmd += " --out " + self._branchFiles + \
"/Products/Correlations/metal_dmat_cross.fits.gz"
cmd += r" --abs-igm SiIII(1207)"
cmd += r" --abs-igm2 SiIII(1207)"
cmd += " --rp-min -60.0"
cmd += " --rp-max +60.0"
cmd += " --rt-max +60.0"
cmd += " --np 30"
cmd += " --nt 15"
cmd += " --rej 0.99"
cmd += " --nproc 1"
cmd += ' --remove-same-half-plate-close-pairs'
cmd += " --unfold-cf"
print(repr(cmd))
picca.bin.picca_metal_dmat.main(cmd.split()[1:])
### Test
if self._test:
path1 = self._masterFiles + "/test_cor/metal_dmat_cross.fits.gz"
path2 = self._branchFiles + "/Products/Correlations/metal_dmat_cross.fits.gz"
self.compare_fits(path1, path2, "picca_metal_dmat.py")
return
def test_metal_xdmat(self):
"""
Test metal cross distortion matrix
"""
userprint("\n")
### Send
cmd = "picca_metal_xdmat.py"
cmd += " --in-dir " + self._masterFiles + "/test_delta/Delta_LYA/"
cmd += " --drq " + self._masterFiles + "/test_delta/cat.fits"
cmd += " --out " + self._branchFiles + "/Products/Correlations/metal_xdmat.fits.gz"
cmd += r" --abs-igm SiIII(1207)"
cmd += " --rp-min -60.0"
cmd += " --rp-max +60.0"
cmd += " --rt-max +60.0"
cmd += " --np 30"
cmd += " --nt 15"
cmd += " --rej 0.99"
cmd += " --nproc 1"
print(repr(cmd))
picca.bin.picca_metal_xdmat.main(cmd.split()[1:])
### Test
if self._test:
path1 = self._masterFiles + "/test_cor/metal_xdmat.fits.gz"
path2 = self._branchFiles + "/Products/Correlations/metal_xdmat.fits.gz"
self.compare_fits(path1, path2, "picca_metal_xdmat.py")
return
def test_cf_cross(self):
"""
Test export of cross correlation function
"""
importlib.reload(picca.cf)
userprint("\n")
### Send
cmd = "picca_cf.py"
cmd += " --in-dir " + self._masterFiles + "/test_delta/Delta_LYA/"
cmd += " --in-dir2 " + self._masterFiles + "/test_delta/Delta_LYA/"
cmd += " --out " + self._branchFiles + "/Products/Correlations/cf_cross.fits.gz"
cmd += " --rp-min -60.0"
cmd += " --rp-max +60.0"
cmd += " --rt-max +60.0"
cmd += " --np 30"
cmd += " --nt 15"
cmd += " --nproc 1"
cmd += ' --remove-same-half-plate-close-pairs'
cmd += " --unfold-cf"
print(repr(cmd))
picca.bin.picca_cf.main(cmd.split()[1:])
### Test
if self._test:
path1 = self._masterFiles + "/test_cor/cf_cross.fits.gz"
path2 = self._branchFiles + "/Products/Correlations/cf_cross.fits.gz"
self.compare_fits(path1, path2, "picca_cf.py")
return
def send_delta(self):
"""
Test the continuum fitting routines on randomly generated eBOSS mock data
"""
import picca.bin.picca_deltas as picca_deltas
userprint("\n")
### Send
cmd = "picca_deltas.py"
cmd += " --in-dir " + self._branchFiles + "/Products/Spectra/"
cmd += " --drq " + self._branchFiles + "/Products/cat.fits"
cmd += " --out-dir " + self._branchFiles + "/Products/Delta_LYA/Delta/"
cmd += " --iter-out-prefix " + self._branchFiles + \
"/Products/Delta_LYA/Log/delta_attributes"
cmd += " --log " + self._branchFiles + "/Products/Delta_LYA/Log/input.log"
cmd += " --nproc 1"
picca_deltas.main(cmd.split()[1:])
### Test
if self._test:
path1 = self._masterFiles + "/test_delta/delta_attributes.fits.gz"
path2 = self._branchFiles + "/Products/Delta_LYA/Log/delta_attributes.fits.gz"
self.compare_fits(path1, path2, "picca_deltas.py")
return
def test_dmat(self):
"""
Test distortion matrix
"""
importlib.reload(picca.cf)
userprint("\n")
### Send
cmd = "picca_dmat.py"
cmd += " --in-dir " + self._masterFiles + "/test_delta/Delta_LYA/"
cmd += " --out " + self._branchFiles + "/Products/Correlations/dmat.fits.gz"
cmd += " --rp-min +0.0"
cmd += " --rp-max +60.0"
cmd += " --rt-max +60.0"
cmd += " --np 15"
cmd += " --nt 15"
cmd += " --rej 0.99"
cmd += " --nproc 1"
cmd += ' --remove-same-half-plate-close-pairs'
print(repr(cmd))
picca.bin.picca_dmat.main(cmd.split()[1:])
### Test
if self._test:
path1 = self._masterFiles + "/test_cor/dmat.fits.gz"
path2 = self._branchFiles + "/Products/Correlations/dmat.fits.gz"
self.compare_fits(path1, path2, "picca_dmat.py")
return
def produce_cat(self, nObj, name="cat", thidoffset=0):
"""
produces a fake catalog for testing
"""
userprint("\n")
userprint("Create cat with number of object = ", nObj)
### Create random catalog
ra = 10. * np.random.random_sample(nObj)
dec = 10. * np.random.random_sample(nObj)
plate = np.random.randint(266, high=10001, size=nObj)
mjd = np.random.randint(51608, high=57521, size=nObj)
fiberid = np.random.randint(1, high=1001, size=nObj)
thid = np.arange(thidoffset + 1, thidoffset + nObj + 1)
z_qso = (3.6 - 2.0) * np.random.random_sample(nObj) + 2.0
### Save
out = fitsio.FITS(self._branchFiles + "/Products/" + name + ".fits",
'rw',
clobber=True)
cols = [ra, dec, thid, plate, mjd, fiberid, z_qso]
names = ['RA', 'DEC', 'THING_ID', 'PLATE', 'MJD', 'FIBERID', 'Z']
out.write(cols, names=names, extname='CAT')
out.close()
if self._test:
path1 = self._masterFiles + "/test_delta/" + name + ".fits"
path2 = self._branchFiles + "/Products/" + name + ".fits"
self.compare_fits(path1, path2, "produce cat")
return
def test_cor(self):
self.picca_base = resource_filename('picca',
'./').replace('py/picca/./', '')
self.send_requirements()
np.random.seed(42)
userprint("\n")
self._test = True
self._masterFiles = self.picca_base + '/py/picca/test/data/'
self.produce_folder()
self.produce_cat(nObj=1000)
self.produce_forests()
self.produce_cat(nObj=1000, name="random", thidoffset=1000)
self.produce_cat_minisv(nObj=1000)
self.produce_forests_minisv()
self.send_delta_Pk1D_minisv()
self.send_delta()
self.send_cf1d()
self.send_cf1d_cross()
self.send_cf_angl()
self.send_cf()
self.send_dmat()
self.send_metal_dmat()
self.send_wick()
self.send_export_cf()
self.send_cf_cross()
self.send_dmat_cross()
self.send_metal_dmat_cross()
self.send_export_cf_cross()
self.send_xcf_angl()
self.send_xcf()
self.send_xdmat()
self.send_metal_xdmat()
self.send_xwick()
self.send_export_xcf()
self.send_export_cross_covariance_cf_xcf()
self.send_co()
self.send_export_co()
self.send_fitter2()
self.send_delta_Pk1D()
self.send_Pk1D()
if self._test:
self.remove_folder()
return
def test_fitter2(self):
self._test = True
userprint("\n")
### copy ini files to branch
filestocopy = glob.glob(self._masterFiles + 'test_fitter2/*ini')
for f in filestocopy:
shutil.copy(f, self._branchFiles + '/Products/Correlations/Fit/')
### Set path in chi2.ini
path = self._branchFiles + '/Products/Correlations/Fit/chi2.ini'
value = self._branchFiles + '/Products/Correlations/Fit/config_cf.ini '
value += self._branchFiles + '/Products/Correlations/Fit/config_xcf.ini '
value += self._branchFiles + '/Products/Correlations/Fit/config_cf_cross.ini '
self.update_system_status_values(path, 'data sets', 'ini files', value)
value = 'PlanckDR12/PlanckDR12.fits'
self.update_system_status_values(path, 'fiducial', 'filename', value)
value = self._branchFiles + '/Products/Correlations/Fit/result_fitter2.h5'
self.update_system_status_values(path, 'output', 'filename', value)
### Set path in config_cf.ini
path = self._branchFiles + '/Products/Correlations/Fit/config_cf.ini'
value = self._masterFiles + '/test_cor/exported_cf.fits.gz'
self.update_system_status_values(path, 'data', 'filename', value)
value = self._masterFiles + '/test_cor/metal_dmat.fits.gz'
self.update_system_status_values(path, 'metals', 'filename', value)
### Set path in config_cf_cross.ini
path = self._branchFiles + '/Products/Correlations/Fit/config_cf_cross.ini'
value = self._masterFiles + '/test_cor/exported_cf_cross.fits.gz'
self.update_system_status_values(path, 'data', 'filename', value)
value = self._masterFiles + '/test_cor/metal_dmat_cross.fits.gz'
self.update_system_status_values(path, 'metals', 'filename', value)
### Set path in config_xcf.ini
path = self._branchFiles + '/Products/Correlations/Fit/config_xcf.ini'
value = self._masterFiles + '/test_cor/exported_xcf.fits.gz'
self.update_system_status_values(path, 'data', 'filename', value)
value = self._masterFiles + '/test_cor/metal_xdmat.fits.gz'
self.update_system_status_values(path, 'metals', 'filename', value)
### Send
picca_fitter2.main(self._branchFiles +
'/Products/Correlations/Fit/chi2.ini')
###These commented lines are to simplify accessing test outputs if needed
#if os.path.exists(self._masterFiles+'new/'):
# os.rmdir(self._masterFiles+'new/')
#shutil.copytree(self._branchFiles,self._masterFiles+'new/')
### Test
if self._test:
path1 = self._masterFiles + '/test_fitter2/result_fitter2.h5'
path2 = self._branchFiles + '/Products/Correlations/Fit/result_fitter2.h5'
self.compare_h5py(path1, path2, "picca_fitter2")
def remove_folder(self):
"""
Remove the produced folders
"""
userprint("\n")
shutil.rmtree(self._branchFiles, ignore_errors=True)
return
def compute_dm_dp(data, dic_init, freep, pars):
dm = {}
for p in freep:
userprint('Parameter {}'.format(p))
g = functools.partial(xi_mod_p, data=data, dic_init=dic_init, pname=p, pars=pars)
dm[p] = derivative(g,pars[p])
return dm
def compute_xi(healpixs):
"""Computes the correlation function for each of the healpixs.
Args:
healpixs: array of ints
List of healpix numbers
Returns:
The following variables:
weights: Total weights in the correlation function pixels
r_par: Parallel distance of the correlation function pixels
r_trans: Transverse distance of the correlation function pixels
z: Redshift of the correlation function pixels
num_pairs: Number of pairs in the correlation function pixels
"""
weights = np.zeros(num_bins_r_par * num_bins_r_trans)
r_par = np.zeros(num_bins_r_par * num_bins_r_trans)
r_trans = np.zeros(num_bins_r_par * num_bins_r_trans)
z = np.zeros(num_bins_r_par * num_bins_r_trans)
num_pairs = np.zeros(num_bins_r_par * num_bins_r_trans, dtype=np.int64)
for healpix in healpixs:
for obj1 in objs[healpix]:
userprint(("\rcomputing xi: "
"{}%").format(round(counter.value * 100. / num_data,
2)),
end="")
with lock:
counter.value += 1
if obj1.neighbours.size == 0:
continue
ang = obj1 ^ obj1.neighbours
z2 = np.array([obj2.z_qso for obj2 in obj1.neighbours])
r_comov2 = np.array([obj2.r_comov for obj2 in obj1.neighbours])
dist_m2 = np.array([obj2.dist_m for obj2 in obj1.neighbours])
weights2 = np.array([obj2.weights for obj2 in obj1.neighbours])
(rebin_weight, rebin_r_par, rebin_r_trans, rebin_z,
rebin_num_pairs) = compute_xi_forest_pairs(
obj1.z_qso, obj1.r_comov, obj1.dist_m, obj1.weights, z2,
r_comov2, dist_m2, weights2, ang)
weights[:len(rebin_weight)] += rebin_weight
r_par[:len(rebin_r_par)] += rebin_r_par
r_trans[:len(rebin_r_trans)] += rebin_r_trans
z[:len(rebin_z)] += rebin_z
num_pairs[:len(rebin_num_pairs)] += rebin_num_pairs
setattr(obj1, "neighbours", None)
w = weights > 0.
r_par[w] /= weights[w]
r_trans[w] /= weights[w]
z[w] /= weights[w]
return weights, r_par, r_trans, z, num_pairs
def compare_fits(self, path1, path2, nameRun=""):
userprint("\n")
m = fitsio.FITS(path1)
self.assertTrue(os.path.isfile(path2), "{}".format(nameRun))
b = fitsio.FITS(path2)
self.assertEqual(len(m), len(b), "{}".format(nameRun))
for i, _ in enumerate(m):
###
r_m = m[i].read_header().records()
ld_m = []
for el in r_m:
name = el['name']
if len(name) > 5 and name[:5] == "TTYPE":
ld_m += [el['value'].replace(" ", "")]
###
r_b = b[i].read_header().records()
ld_b = []
for el in r_b:
name = el['name']
if len(name) > 5 and name[:5] == "TTYPE":
ld_b += [el['value'].replace(" ", "")]
self.assertListEqual(ld_m, ld_b, "{}".format(nameRun))
for k in ld_m:
d_m = m[i][k][:]
d_b = b[i][k][:]
if d_m.dtype in ['<U23', 'S23'
]: ### for fitsio old version compatibility
d_m = np.char.strip(d_m)
if d_b.dtype in ['<U23', 'S23'
]: ### for fitsio old version compatibility
d_b = np.char.strip(d_b)
self.assertEqual(d_m.size, d_b.size,
"{}: Header key is {}".format(nameRun, k))
if not np.array_equal(d_m, d_b):
userprint(
"WARNING: {}: Header key is {}, arrays are not exactly equal, using allclose"
.format(nameRun, k))
diff = d_m - d_b
w = d_m != 0.
diff[w] = np.absolute(diff[w] / d_m[w])
allclose = np.allclose(d_m, d_b)
self.assertTrue(
allclose,
"{}: Header key is {}, maximum relative difference is {}"
.format(nameRun, k, diff.max()))
m.close()
b.close()
return
def send_export_cf(self):
userprint("\n")
### Send
cmd = " picca_export.py"
cmd += " --data " + self._branchFiles + "/Products/Correlations/cf.fits.gz"
cmd += " --dmat " + self._branchFiles + "/Products/Correlations/dmat.fits.gz"
cmd += " --out " + self._branchFiles + "/Products/Correlations/exported_cf.fits.gz"
returncode = subprocess.call(cmd, shell=True)
self.assertEqual(returncode, 0, "export_cf did not finish")
return
def produce_folder(self):
"""
Create the necessary folders
"""
userprint("\n")
lst_fold = ["/Products/", "/Products/Pk1D/"]
for fold in lst_fold:
if not os.path.isdir(self._branchFiles + fold):
os.mkdir(self._branchFiles + fold)
return
def setUpClass(cls):
"""
sets up directory structure in tmp
"""
cls._branchFiles = tempfile.mkdtemp() + "/"
cls.produce_folder(cls)
cls.picca_base = resource_filename('picca',
'./').replace('py/picca/./', '')
cls.send_requirements(cls.load_requirements(cls.picca_base))
np.random.seed(42)
cls._masterFiles = cls.picca_base + '/py/picca/test/data/'
cls._test=True
userprint("\n")
def from_image(file):
"""Initialize instance from an ascii file.
Args:
file: string
Name of the fits file containing the image data
Returns:
a list of Delta instances
"""
hdu = fitsio.FITS(file)
deltas_image = hdu[0].read().astype(float)
ivar_image = hdu[1].read().astype(float)
log_lambda_image = hdu[2].read().astype(float)
ra = hdu[3]["RA"][:].astype(np.float64) * np.pi / 180.
dec = hdu[3]["DEC"][:].astype(np.float64) * np.pi / 180.
z = hdu[3]["Z"][:].astype(np.float64)
plate = hdu[3]["PLATE"][:]
mjd = hdu[3]["MJD"][:]
fiberid = hdu[3]["FIBER"]
thingid = hdu[3]["THING_ID"][:]
nspec = hdu[0].read().shape[1]
deltas = []
for index in range(nspec):
if index % 100 == 0:
userprint("\rreading deltas {} of {}".format(index, nspec),
end="")
delta = deltas_image[:, index]
ivar = ivar_image[:, index]
w = ivar > 0
delta = delta[w]
aux_ivar = ivar[w]
log_lambda = log_lambda_image[w]
order = 1
exposures_diff = None
mean_snr = None
mean_reso = None
delta_log_lambda = None
mean_z = None
deltas.append(
Delta(thingid[index], ra[index], dec[index], z[index],
plate[index], mjd[index], fiberid[index], log_lambda,
aux_ivar, None, delta, order, ivar, exposures_diff,
mean_snr, mean_reso, mean_z, delta_log_lambda))
hdu.close()
return deltas
def send_fitter2(self):
userprint("\n")
### copy ini files to branch
cmd = 'cp '+self._masterFiles+'/*ini ' + \
self._branchFiles+'/Products/Correlations/Fit/'
subprocess.call(cmd, shell=True)
### Set path in chi2.ini
path = self._branchFiles + '/Products/Correlations/Fit/chi2.ini'
value = self._branchFiles + '/Products/Correlations/Fit/config_cf.ini '
value += self._branchFiles + '/Products/Correlations/Fit/config_xcf.ini '
value += self._branchFiles + '/Products/Correlations/Fit/config_cf_cross.ini '
update_system_status_values(path, 'data sets', 'ini files', value)
value = 'PlanckDR12/PlanckDR12.fits'
update_system_status_values(path, 'fiducial', 'filename', value)
value = self._branchFiles + '/Products/Correlations/Fit/result_fitter2.h5'
update_system_status_values(path, 'output', 'filename', value)
### Set path in config_cf.ini
path = self._branchFiles + '/Products/Correlations/Fit/config_cf.ini'
value = self._branchFiles + '/Products/Correlations/exported_cf.fits.gz'
update_system_status_values(path, 'data', 'filename', value)
value = self._branchFiles + '/Products/Correlations/metal_dmat.fits.gz'
update_system_status_values(path, 'metals', 'filename', value)
### Set path in config_cf_cross.ini
path = self._branchFiles + '/Products/Correlations/Fit/config_cf_cross.ini'
value = self._branchFiles + '/Products/Correlations/exported_cf_cross.fits.gz'
update_system_status_values(path, 'data', 'filename', value)
value = self._branchFiles + '/Products/Correlations/metal_dmat_cross.fits.gz'
update_system_status_values(path, 'metals', 'filename', value)
### Set path in config_xcf.ini
path = self._branchFiles + '/Products/Correlations/Fit/config_xcf.ini'
value = self._branchFiles + '/Products/Correlations/exported_xcf.fits.gz'
update_system_status_values(path, 'data', 'filename', value)
value = self._branchFiles + '/Products/Correlations/metal_xdmat.fits.gz'
update_system_status_values(path, 'metals', 'filename', value)
### Send
cmd = ' picca_fitter2.py '+self._branchFiles + \
'/Products/Correlations/Fit/chi2.ini'
subprocess.call(cmd, shell=True)
### Test
if self._test:
path1 = self._masterFiles + '/result_fitter2.h5'
path2 = self._branchFiles + '/Products/Correlations/Fit/result_fitter2.h5'
self.compare_h5py(path1, path2, "picca_fitter2")
def _minimize(self):
t0 = time.time()
par_names = [name for d in self.data for name in d.pars_init]
kwargs = {
name: val
for d in self.data for name, val in d.pars_init.items()
}
kwargs.update({
name: err
for d in self.data for name, err in d.par_error.items()
})
kwargs.update({
name: lim
for d in self.data for name, lim in d.par_limit.items()
})
kwargs.update({
name: fix
for d in self.data for name, fix in d.par_fixed.items()
})
## do an initial "fast" minimization fixing everything except the biases
kwargs_init = {}
for k, v in kwargs.items():
kwargs_init[k] = v
for name in par_names:
if name[:4] != "bias":
kwargs_init["fix_" + name] = True
mig_init = iminuit.Minuit(self,
forced_parameters=self.par_names,
errordef=1,
print_level=1,
**kwargs_init)
mig_init.migrad()
mig_init.print_param()
## now get the best fit values for the biases and start a full minimization
for name, value in mig_init.values.items():
kwargs[name] = value
mig = iminuit.Minuit(self,
forced_parameters=self.par_names,
errordef=1,
print_level=1,
**kwargs)
mig.migrad()
mig.print_param()
userprint("INFO: minimized in {}".format(time.time() - t0))
sys.stdout.flush()
return mig
def _format_unknown(self, unknown):
dic_unknown = {}
i = 0
while (i < len(unknown)):
try:
if (unknown[i][:2] == '--'):
dic_unknown[unknown[i][2:]] = float(unknown[i + 1])
i += 2
else:
userprint(
' picca/py/picca/fitter/parameters.py:: unknown entry = ',
unknown[i])
userprint(' Exit')
sys.exit(0)
i += 1
except:
userprint(
' picca/py/picca/fitter/parameters.py:: unknown entry = ',
unknown[i])
userprint(' Exit')
sys.exit(0)
i += 1
return dic_unknown
def test_export_cf(self):
"""
Test export of correlation function
"""
userprint("\n")
### Send
cmd = "picca_export.py"
cmd += " --data " + self._masterFiles + "/test_cor/cf.fits.gz"
cmd += " --dmat " + self._masterFiles + "/test_cor/dmat.fits.gz"
cmd += " --out " + self._branchFiles + "/Products/Correlations/exported_cf.fits.gz"
print(repr(cmd))
picca.bin.picca_export.main(cmd.split()[1:])
return
def rebin_diff_noise(delta_log_lambda, log_lambda, exposures_diff):
"""Rebin the semidifference between two customized coadded spectra to
construct the noise array
The rebinning is done by combining 3 of the original pixels into analysis
pixels.
Args:
delta_log_lambda: float
Variation of the logarithm of the wavelength between two pixels
log_lambda: array of floats
Array containing the logarithm of the wavelengths (in Angs)
exposures_diff: array of floats
Semidifference between two customized coadded spectra obtained from
weighted averages of the even-number exposures, for the first
spectrum, and of the odd-number exposures, for the second one
Returns:
The noise array
"""
rebin = 3
if exposures_diff.size < rebin:
userprint("Warning: exposures_diff.size too small for rebin")
return exposures_diff
rebin_delta_log_lambda = rebin * delta_log_lambda
# rebin not mixing pixels separated by masks
bins = np.floor((log_lambda - log_lambda.min()) / rebin_delta_log_lambda +
0.5).astype(int)
rebin_exposure_diff = np.bincount(bins.astype(int), weights=exposures_diff)
rebin_counts = np.bincount(bins.astype(int))
w = (rebin_counts > 0)
if len(rebin_counts) == 0:
userprint("Error: exposures_diff size = 0 ", exposures_diff)
rebin_exposure_diff = rebin_exposure_diff[w] / np.sqrt(rebin_counts[w])
# now merge the rebinned array into a noise array
noise = np.zeros(exposures_diff.size)
for index in range(len(exposures_diff) // len(rebin_exposure_diff) + 1):
length_max = min(len(exposures_diff),
(index + 1) * len(rebin_exposure_diff))
noise[index *
len(rebin_exposure_diff):length_max] = rebin_exposure_diff[:(
length_max - index * len(rebin_exposure_diff))]
# shuffle the array before the next iteration
np.random.shuffle(rebin_exposure_diff)
return noise
def send_export_co(self):
userprint("\n")
### Send
cmd = " picca_export_co.py"
cmd += " --DD-file " + self._branchFiles + "/Products/Correlations/co_DD.fits.gz"
cmd += " --RR-file " + self._branchFiles + "/Products/Correlations/Co_Random/co_RR.fits.gz"
cmd += " --DR-file " + self._branchFiles + "/Products/Correlations/Co_Random/co_DR.fits.gz"
cmd += " --RD-file " + self._branchFiles + "/Products/Correlations/Co_Random/co_RD.fits.gz"
cmd += " --out " + self._branchFiles + "/Products/Correlations/exported_co.fits.gz"
cmd += " --get-cov-from-poisson"
returncode = subprocess.call(cmd, shell=True)
self.assertEqual(returncode, 0, "picca_export_co did not finish")
return
def add_cross(self, dic_init):
self.pname.append("qso_metal_boost")
self.pinit.append(dic_init["qso_metal_boost"])
self.different_drp = dic_init['different_drp']
if (self.different_drp):
if not self.grid:
userprint("different drp and metal matrix not implemented")
sys.exit(1)
for name in self.met_names:
self.pname.append("drp_" + name)
self.pinit.append(dic_init["drp_" + name])
met_prefix = self.met_prefix
nmet = self.nmet
met_names = self.met_names
if self.grid:
to = np.loadtxt(met_prefix + '_QSO_' + met_names[0] + '.grid')
self.nd_cross = to[:, 0].size
### Get the grid of the metals
self.grid_qso_met = np.zeros([self.nd_cross, nmet, 3])
for i in range(nmet):
fmet = met_prefix + '_QSO_' + met_names[i] + '.grid'
userprint(' Reading cross correlation metal grid : ')
userprint(' ', fmet)
to = np.loadtxt(fmet)
idx = to[:, 0].astype(int)
self.grid_qso_met[idx, i, 0] = to[:, 1]
self.grid_qso_met[idx, i, 1] = to[:, 2]
self.grid_qso_met[idx, i, 2] = to[:, 3]
userprint()
else:
h = fitsio.FITS(self.met_prefix)
self.abs_igm_cross = [i.strip() for i in h[1]["ABS_IGM"][:]]
self.xdmat = {}
self.xrp = {}
self.xrt = {}
self.xzeff = {}
self.prev_pmet = {
'growth_rate': 0.,
'drp': 0.,
'qso_evol': [0., 0.]
}
self.prev_xi_qso_met = {}
for i in self.abs_igm_cross:
self.xdmat[i] = h[2]["DM_" + i][:]
self.xrp[i] = h[2]["RP_" + i][:]
self.xrt[i] = h[2]["RT_" + i][:]
self.xzeff[i] = h[2]["Z_" + i][:]
self.prev_pmet['beta_' + i] = 0.
self.prev_pmet['alpha_' + i] = 0.
self.prev_xi_qso_met[i] = np.zeros(self.xdmat[i].shape[0])
def send_Pk1D(self):
userprint("\n")
### Send
cmd = " picca_Pk1D.py"
cmd += " --in-dir " + self._masterFiles + "/test_Pk1D/delta_Pk1D/"
cmd += " --out-dir " + self._branchFiles + "/Products/Pk1D/"
subprocess.call(cmd, shell=True)
### Test
if self._test:
path1 = self._masterFiles + "/test_Pk1D/Pk1D.fits.gz"
path2 = self._branchFiles + "/Products/Pk1D/Pk1D-0.fits.gz"
self.compare_fits(path1, path2, "picca_Pk1D.py")
return
