def test_xwick(self): """ Test wick covariances for cross """ userprint("\n") ### Send cmd = "picca_xwick.py" cmd += " --in-dir " + self._masterFiles + "/test_delta/Delta_LYA/" cmd += " --drq " + self._masterFiles + "/test_delta/cat.fits" cmd += " --out " + self._branchFiles + "/Products/Correlations/xwick.fits.gz" cmd += " --cf1d " + self._masterFiles + "/test_cor/cf1d.fits.gz" cmd += " --rp-min -60.0" cmd += " --rp-max +60.0" cmd += " --rt-max +60.0" cmd += " --np 30" cmd += " --nt 15" cmd += " --rej 0.99" cmd += " --nproc 1" print(repr(cmd)) picca.bin.picca_xwick.main(cmd.split()[1:]) ### Test if self._test: path1 = self._masterFiles + "/test_cor/xwick.fits.gz" path2 = self._branchFiles + "/Products/Correlations/xwick.fits.gz" self.compare_fits(path1, path2, "picca_xwick.py") return
def send_metal_dmat_cross(self): userprint("\n") ### Send cmd = " picca_metal_dmat.py" cmd += " --in-dir " + self._branchFiles + "/Products/Delta_LYA/Delta/" cmd += " --in-dir2 " + self._branchFiles + "/Products/Delta_LYA/Delta/" cmd += " --out " + self._branchFiles + "/Products/Correlations/metal_dmat_cross.fits.gz" cmd += " --abs-igm SiIII\(1207\)" cmd += " --abs-igm2 SiIII\(1207\)" cmd += " --rp-min -60.0" cmd += " --rp-max +60.0" cmd += " --rt-max +60.0" cmd += " --np 30" cmd += " --nt 15" cmd += " --rej 0.99 " cmd += " --nproc 1" cmd += ' --remove-same-half-plate-close-pairs' cmd += " --unfold-cf" subprocess.call(cmd, shell=True) ### Test if self._test: path1 = self._masterFiles + "/metal_dmat_cross.fits.gz" path2 = self._branchFiles + "/Products/Correlations/metal_dmat_cross.fits.gz" self.compare_fits(path1, path2, "picca_metal_dmat.py") return
def send_xwick(self): userprint("\n") ### Send cmd = " picca_xwick.py" cmd += " --in-dir " + self._branchFiles + "/Products/Delta_LYA/Delta/" cmd += " --drq " + self._branchFiles + "/Products/cat.fits" cmd += " --out " + self._branchFiles + "/Products/Correlations/xwick.fits.gz" cmd += " --cf1d " + self._branchFiles + "/Products/Correlations/cf1d.fits.gz" cmd += " --rp-min -60.0" cmd += " --rp-max +60.0" cmd += " --rt-max +60.0" cmd += " --np 30" cmd += " --nt 15" cmd += " --rej 0.99 " cmd += " --nproc 1" subprocess.call(cmd, shell=True) ### Test if self._test: path1 = self._masterFiles + "/xwick.fits.gz" path2 = self._branchFiles + "/Products/Correlations/xwick.fits.gz" self.compare_fits(path1, path2, "picca_xwick.py") return
def send_delta_Pk1D(self): userprint("\n") ### Path path_to_etc = self.picca_base + '/etc/' ### Send cmd = " picca_deltas.py" cmd += " --in-dir " + self._masterFiles + "/test_Pk1D/Spectra_test/" cmd += " --drq " + self._masterFiles + "/test_Pk1D/DRQ_test.fits" cmd += " --out-dir " + self._branchFiles + "/Products/Delta_Pk1D/Delta/" cmd += " --iter-out-prefix " + self._branchFiles + "/Products/Delta_Pk1D/Log/delta_attributes" cmd += " --log " + self._branchFiles + "/Products/Delta_Pk1D/Log/input.log" cmd += " --delta-format Pk1D --mode spec --order 0 --use-constant-weight" cmd += " --rebin 1 --lambda-min 3650. --lambda-max 7200.0 --lambda-rest-min 1050.0 --lambda-rest-max 1180" cmd += " --nproc 1" cmd += " --best-obs" cmd += " --mask-file " + path_to_etc + "/list_veto_line_Pk1D.txt" subprocess.call(cmd, shell=True) ### Test if self._test: path1 = self._masterFiles + "/test_Pk1D/delta_attributes_Pk1D.fits.gz" path2 = self._branchFiles + "/Products/Delta_Pk1D/Log/delta_attributes.fits.gz" self.compare_fits(path1, path2, "picca_deltas.py") path1 = self._masterFiles + "/test_Pk1D/delta-64_Pk1D.fits.gz" path2 = self._branchFiles + "/Products/Delta_Pk1D/Delta/delta-64.fits.gz" self.compare_fits(path1, path2, "picca_deltas.py") path1 = self._masterFiles + "/test_Pk1D/delta-80_Pk1D.fits.gz" path2 = self._branchFiles + "/Products/Delta_Pk1D/Delta/delta-80.fits.gz" self.compare_fits(path1, path2, "picca_deltas.py") return
def compare_values(val1, val2): if not np.array_equal(val1, val2): userprint("WARNING: {}: not exactly equal, using allclose".format( nameRun)) allclose = np.allclose(val1, val2) self.assertTrue(allclose, "{}".format(nameRun)) return
def cached_growth_factor_de(z, zref=None, Om=None, OL=None, **kwargs): ''' Implements eq. 7.77 from S. Dodelson's Modern Cosmology book ''' userprint('Calculating growth factor for Om = {}, OL = {}'.format(Om, OL)) def hubble(z, Om, OL): return np.sqrt(Om * (1 + z)**3 + OL + (1 - Om - OL) * (1 + z)**2) def dD1(a, Om, OL): z = 1 / a - 1 return 1. / (a * hubble(z=z, Om=Om, OL=OL))**3 ## Calculate D1 in 100 values of z between 0 and zmax, then interpolate nbins = 100 zmax = 5. z = zmax * np.arange(nbins, dtype=float) / (nbins - 1) D1 = np.zeros(nbins, dtype=float) pars = (Om, OL) for i in range(nbins): a = 1 / (1 + z[i]) D1[i] = 5 / 2. * Om * hubble(z[i], *pars) * quad(dD1, 0, a, args=pars)[0] D1 = interp1d(z, D1) return D1
def test_metal_dmat_cross(self): """ Test metal cross distortion matrix """ importlib.reload(picca.cf) userprint("\n") ### Send cmd = "picca_metal_dmat.py" cmd += " --in-dir " + self._masterFiles + "/test_delta/Delta_LYA/" cmd += " --in-dir2 " + self._masterFiles + "/test_delta/Delta_LYA/" cmd += " --out " + self._branchFiles + \ "/Products/Correlations/metal_dmat_cross.fits.gz" cmd += r" --abs-igm SiIII(1207)" cmd += r" --abs-igm2 SiIII(1207)" cmd += " --rp-min -60.0" cmd += " --rp-max +60.0" cmd += " --rt-max +60.0" cmd += " --np 30" cmd += " --nt 15" cmd += " --rej 0.99" cmd += " --nproc 1" cmd += ' --remove-same-half-plate-close-pairs' cmd += " --unfold-cf" print(repr(cmd)) picca.bin.picca_metal_dmat.main(cmd.split()[1:]) ### Test if self._test: path1 = self._masterFiles + "/test_cor/metal_dmat_cross.fits.gz" path2 = self._branchFiles + "/Products/Correlations/metal_dmat_cross.fits.gz" self.compare_fits(path1, path2, "picca_metal_dmat.py") return
def test_metal_xdmat(self): """ Test metal cross distortion matrix """ userprint("\n") ### Send cmd = "picca_metal_xdmat.py" cmd += " --in-dir " + self._masterFiles + "/test_delta/Delta_LYA/" cmd += " --drq " + self._masterFiles + "/test_delta/cat.fits" cmd += " --out " + self._branchFiles + "/Products/Correlations/metal_xdmat.fits.gz" cmd += r" --abs-igm SiIII(1207)" cmd += " --rp-min -60.0" cmd += " --rp-max +60.0" cmd += " --rt-max +60.0" cmd += " --np 30" cmd += " --nt 15" cmd += " --rej 0.99" cmd += " --nproc 1" print(repr(cmd)) picca.bin.picca_metal_xdmat.main(cmd.split()[1:]) ### Test if self._test: path1 = self._masterFiles + "/test_cor/metal_xdmat.fits.gz" path2 = self._branchFiles + "/Products/Correlations/metal_xdmat.fits.gz" self.compare_fits(path1, path2, "picca_metal_xdmat.py") return
def test_cf_cross(self): """ Test export of cross correlation function """ importlib.reload(picca.cf) userprint("\n") ### Send cmd = "picca_cf.py" cmd += " --in-dir " + self._masterFiles + "/test_delta/Delta_LYA/" cmd += " --in-dir2 " + self._masterFiles + "/test_delta/Delta_LYA/" cmd += " --out " + self._branchFiles + "/Products/Correlations/cf_cross.fits.gz" cmd += " --rp-min -60.0" cmd += " --rp-max +60.0" cmd += " --rt-max +60.0" cmd += " --np 30" cmd += " --nt 15" cmd += " --nproc 1" cmd += ' --remove-same-half-plate-close-pairs' cmd += " --unfold-cf" print(repr(cmd)) picca.bin.picca_cf.main(cmd.split()[1:]) ### Test if self._test: path1 = self._masterFiles + "/test_cor/cf_cross.fits.gz" path2 = self._branchFiles + "/Products/Correlations/cf_cross.fits.gz" self.compare_fits(path1, path2, "picca_cf.py") return
def send_delta(self): """ Test the continuum fitting routines on randomly generated eBOSS mock data """ import picca.bin.picca_deltas as picca_deltas userprint("\n") ### Send cmd = "picca_deltas.py" cmd += " --in-dir " + self._branchFiles + "/Products/Spectra/" cmd += " --drq " + self._branchFiles + "/Products/cat.fits" cmd += " --out-dir " + self._branchFiles + "/Products/Delta_LYA/Delta/" cmd += " --iter-out-prefix " + self._branchFiles + \ "/Products/Delta_LYA/Log/delta_attributes" cmd += " --log " + self._branchFiles + "/Products/Delta_LYA/Log/input.log" cmd += " --nproc 1" picca_deltas.main(cmd.split()[1:]) ### Test if self._test: path1 = self._masterFiles + "/test_delta/delta_attributes.fits.gz" path2 = self._branchFiles + "/Products/Delta_LYA/Log/delta_attributes.fits.gz" self.compare_fits(path1, path2, "picca_deltas.py") return
def test_dmat(self): """ Test distortion matrix """ importlib.reload(picca.cf) userprint("\n") ### Send cmd = "picca_dmat.py" cmd += " --in-dir " + self._masterFiles + "/test_delta/Delta_LYA/" cmd += " --out " + self._branchFiles + "/Products/Correlations/dmat.fits.gz" cmd += " --rp-min +0.0" cmd += " --rp-max +60.0" cmd += " --rt-max +60.0" cmd += " --np 15" cmd += " --nt 15" cmd += " --rej 0.99" cmd += " --nproc 1" cmd += ' --remove-same-half-plate-close-pairs' print(repr(cmd)) picca.bin.picca_dmat.main(cmd.split()[1:]) ### Test if self._test: path1 = self._masterFiles + "/test_cor/dmat.fits.gz" path2 = self._branchFiles + "/Products/Correlations/dmat.fits.gz" self.compare_fits(path1, path2, "picca_dmat.py") return
def produce_cat(self, nObj, name="cat", thidoffset=0): """ produces a fake catalog for testing """ userprint("\n") userprint("Create cat with number of object = ", nObj) ### Create random catalog ra = 10. * np.random.random_sample(nObj) dec = 10. * np.random.random_sample(nObj) plate = np.random.randint(266, high=10001, size=nObj) mjd = np.random.randint(51608, high=57521, size=nObj) fiberid = np.random.randint(1, high=1001, size=nObj) thid = np.arange(thidoffset + 1, thidoffset + nObj + 1) z_qso = (3.6 - 2.0) * np.random.random_sample(nObj) + 2.0 ### Save out = fitsio.FITS(self._branchFiles + "/Products/" + name + ".fits", 'rw', clobber=True) cols = [ra, dec, thid, plate, mjd, fiberid, z_qso] names = ['RA', 'DEC', 'THING_ID', 'PLATE', 'MJD', 'FIBERID', 'Z'] out.write(cols, names=names, extname='CAT') out.close() if self._test: path1 = self._masterFiles + "/test_delta/" + name + ".fits" path2 = self._branchFiles + "/Products/" + name + ".fits" self.compare_fits(path1, path2, "produce cat") return
def test_cor(self): self.picca_base = resource_filename('picca', './').replace('py/picca/./', '') self.send_requirements() np.random.seed(42) userprint("\n") self._test = True self._masterFiles = self.picca_base + '/py/picca/test/data/' self.produce_folder() self.produce_cat(nObj=1000) self.produce_forests() self.produce_cat(nObj=1000, name="random", thidoffset=1000) self.produce_cat_minisv(nObj=1000) self.produce_forests_minisv() self.send_delta_Pk1D_minisv() self.send_delta() self.send_cf1d() self.send_cf1d_cross() self.send_cf_angl() self.send_cf() self.send_dmat() self.send_metal_dmat() self.send_wick() self.send_export_cf() self.send_cf_cross() self.send_dmat_cross() self.send_metal_dmat_cross() self.send_export_cf_cross() self.send_xcf_angl() self.send_xcf() self.send_xdmat() self.send_metal_xdmat() self.send_xwick() self.send_export_xcf() self.send_export_cross_covariance_cf_xcf() self.send_co() self.send_export_co() self.send_fitter2() self.send_delta_Pk1D() self.send_Pk1D() if self._test: self.remove_folder() return
def test_fitter2(self): self._test = True userprint("\n") ### copy ini files to branch filestocopy = glob.glob(self._masterFiles + 'test_fitter2/*ini') for f in filestocopy: shutil.copy(f, self._branchFiles + '/Products/Correlations/Fit/') ### Set path in chi2.ini path = self._branchFiles + '/Products/Correlations/Fit/chi2.ini' value = self._branchFiles + '/Products/Correlations/Fit/config_cf.ini ' value += self._branchFiles + '/Products/Correlations/Fit/config_xcf.ini ' value += self._branchFiles + '/Products/Correlations/Fit/config_cf_cross.ini ' self.update_system_status_values(path, 'data sets', 'ini files', value) value = 'PlanckDR12/PlanckDR12.fits' self.update_system_status_values(path, 'fiducial', 'filename', value) value = self._branchFiles + '/Products/Correlations/Fit/result_fitter2.h5' self.update_system_status_values(path, 'output', 'filename', value) ### Set path in config_cf.ini path = self._branchFiles + '/Products/Correlations/Fit/config_cf.ini' value = self._masterFiles + '/test_cor/exported_cf.fits.gz' self.update_system_status_values(path, 'data', 'filename', value) value = self._masterFiles + '/test_cor/metal_dmat.fits.gz' self.update_system_status_values(path, 'metals', 'filename', value) ### Set path in config_cf_cross.ini path = self._branchFiles + '/Products/Correlations/Fit/config_cf_cross.ini' value = self._masterFiles + '/test_cor/exported_cf_cross.fits.gz' self.update_system_status_values(path, 'data', 'filename', value) value = self._masterFiles + '/test_cor/metal_dmat_cross.fits.gz' self.update_system_status_values(path, 'metals', 'filename', value) ### Set path in config_xcf.ini path = self._branchFiles + '/Products/Correlations/Fit/config_xcf.ini' value = self._masterFiles + '/test_cor/exported_xcf.fits.gz' self.update_system_status_values(path, 'data', 'filename', value) value = self._masterFiles + '/test_cor/metal_xdmat.fits.gz' self.update_system_status_values(path, 'metals', 'filename', value) ### Send picca_fitter2.main(self._branchFiles + '/Products/Correlations/Fit/chi2.ini') ###These commented lines are to simplify accessing test outputs if needed #if os.path.exists(self._masterFiles+'new/'): # os.rmdir(self._masterFiles+'new/') #shutil.copytree(self._branchFiles,self._masterFiles+'new/') ### Test if self._test: path1 = self._masterFiles + '/test_fitter2/result_fitter2.h5' path2 = self._branchFiles + '/Products/Correlations/Fit/result_fitter2.h5' self.compare_h5py(path1, path2, "picca_fitter2")
def remove_folder(self): """ Remove the produced folders """ userprint("\n") shutil.rmtree(self._branchFiles, ignore_errors=True) return
def compute_dm_dp(data, dic_init, freep, pars): dm = {} for p in freep: userprint('Parameter {}'.format(p)) g = functools.partial(xi_mod_p, data=data, dic_init=dic_init, pname=p, pars=pars) dm[p] = derivative(g,pars[p]) return dm
def compute_xi(healpixs): """Computes the correlation function for each of the healpixs. Args: healpixs: array of ints List of healpix numbers Returns: The following variables: weights: Total weights in the correlation function pixels r_par: Parallel distance of the correlation function pixels r_trans: Transverse distance of the correlation function pixels z: Redshift of the correlation function pixels num_pairs: Number of pairs in the correlation function pixels """ weights = np.zeros(num_bins_r_par * num_bins_r_trans) r_par = np.zeros(num_bins_r_par * num_bins_r_trans) r_trans = np.zeros(num_bins_r_par * num_bins_r_trans) z = np.zeros(num_bins_r_par * num_bins_r_trans) num_pairs = np.zeros(num_bins_r_par * num_bins_r_trans, dtype=np.int64) for healpix in healpixs: for obj1 in objs[healpix]: userprint(("\rcomputing xi: " "{}%").format(round(counter.value * 100. / num_data, 2)), end="") with lock: counter.value += 1 if obj1.neighbours.size == 0: continue ang = obj1 ^ obj1.neighbours z2 = np.array([obj2.z_qso for obj2 in obj1.neighbours]) r_comov2 = np.array([obj2.r_comov for obj2 in obj1.neighbours]) dist_m2 = np.array([obj2.dist_m for obj2 in obj1.neighbours]) weights2 = np.array([obj2.weights for obj2 in obj1.neighbours]) (rebin_weight, rebin_r_par, rebin_r_trans, rebin_z, rebin_num_pairs) = compute_xi_forest_pairs( obj1.z_qso, obj1.r_comov, obj1.dist_m, obj1.weights, z2, r_comov2, dist_m2, weights2, ang) weights[:len(rebin_weight)] += rebin_weight r_par[:len(rebin_r_par)] += rebin_r_par r_trans[:len(rebin_r_trans)] += rebin_r_trans z[:len(rebin_z)] += rebin_z num_pairs[:len(rebin_num_pairs)] += rebin_num_pairs setattr(obj1, "neighbours", None) w = weights > 0. r_par[w] /= weights[w] r_trans[w] /= weights[w] z[w] /= weights[w] return weights, r_par, r_trans, z, num_pairs
def compare_fits(self, path1, path2, nameRun=""): userprint("\n") m = fitsio.FITS(path1) self.assertTrue(os.path.isfile(path2), "{}".format(nameRun)) b = fitsio.FITS(path2) self.assertEqual(len(m), len(b), "{}".format(nameRun)) for i, _ in enumerate(m): ### r_m = m[i].read_header().records() ld_m = [] for el in r_m: name = el['name'] if len(name) > 5 and name[:5] == "TTYPE": ld_m += [el['value'].replace(" ", "")] ### r_b = b[i].read_header().records() ld_b = [] for el in r_b: name = el['name'] if len(name) > 5 and name[:5] == "TTYPE": ld_b += [el['value'].replace(" ", "")] self.assertListEqual(ld_m, ld_b, "{}".format(nameRun)) for k in ld_m: d_m = m[i][k][:] d_b = b[i][k][:] if d_m.dtype in ['<U23', 'S23' ]: ### for fitsio old version compatibility d_m = np.char.strip(d_m) if d_b.dtype in ['<U23', 'S23' ]: ### for fitsio old version compatibility d_b = np.char.strip(d_b) self.assertEqual(d_m.size, d_b.size, "{}: Header key is {}".format(nameRun, k)) if not np.array_equal(d_m, d_b): userprint( "WARNING: {}: Header key is {}, arrays are not exactly equal, using allclose" .format(nameRun, k)) diff = d_m - d_b w = d_m != 0. diff[w] = np.absolute(diff[w] / d_m[w]) allclose = np.allclose(d_m, d_b) self.assertTrue( allclose, "{}: Header key is {}, maximum relative difference is {}" .format(nameRun, k, diff.max())) m.close() b.close() return
def send_export_cf(self): userprint("\n") ### Send cmd = " picca_export.py" cmd += " --data " + self._branchFiles + "/Products/Correlations/cf.fits.gz" cmd += " --dmat " + self._branchFiles + "/Products/Correlations/dmat.fits.gz" cmd += " --out " + self._branchFiles + "/Products/Correlations/exported_cf.fits.gz" returncode = subprocess.call(cmd, shell=True) self.assertEqual(returncode, 0, "export_cf did not finish") return
def produce_folder(self): """ Create the necessary folders """ userprint("\n") lst_fold = ["/Products/", "/Products/Pk1D/"] for fold in lst_fold: if not os.path.isdir(self._branchFiles + fold): os.mkdir(self._branchFiles + fold) return
def setUpClass(cls): """ sets up directory structure in tmp """ cls._branchFiles = tempfile.mkdtemp() + "/" cls.produce_folder(cls) cls.picca_base = resource_filename('picca', './').replace('py/picca/./', '') cls.send_requirements(cls.load_requirements(cls.picca_base)) np.random.seed(42) cls._masterFiles = cls.picca_base + '/py/picca/test/data/' cls._test=True userprint("\n")
def from_image(file): """Initialize instance from an ascii file. Args: file: string Name of the fits file containing the image data Returns: a list of Delta instances """ hdu = fitsio.FITS(file) deltas_image = hdu[0].read().astype(float) ivar_image = hdu[1].read().astype(float) log_lambda_image = hdu[2].read().astype(float) ra = hdu[3]["RA"][:].astype(np.float64) * np.pi / 180. dec = hdu[3]["DEC"][:].astype(np.float64) * np.pi / 180. z = hdu[3]["Z"][:].astype(np.float64) plate = hdu[3]["PLATE"][:] mjd = hdu[3]["MJD"][:] fiberid = hdu[3]["FIBER"] thingid = hdu[3]["THING_ID"][:] nspec = hdu[0].read().shape[1] deltas = [] for index in range(nspec): if index % 100 == 0: userprint("\rreading deltas {} of {}".format(index, nspec), end="") delta = deltas_image[:, index] ivar = ivar_image[:, index] w = ivar > 0 delta = delta[w] aux_ivar = ivar[w] log_lambda = log_lambda_image[w] order = 1 exposures_diff = None mean_snr = None mean_reso = None delta_log_lambda = None mean_z = None deltas.append( Delta(thingid[index], ra[index], dec[index], z[index], plate[index], mjd[index], fiberid[index], log_lambda, aux_ivar, None, delta, order, ivar, exposures_diff, mean_snr, mean_reso, mean_z, delta_log_lambda)) hdu.close() return deltas
def send_fitter2(self): userprint("\n") ### copy ini files to branch cmd = 'cp '+self._masterFiles+'/*ini ' + \ self._branchFiles+'/Products/Correlations/Fit/' subprocess.call(cmd, shell=True) ### Set path in chi2.ini path = self._branchFiles + '/Products/Correlations/Fit/chi2.ini' value = self._branchFiles + '/Products/Correlations/Fit/config_cf.ini ' value += self._branchFiles + '/Products/Correlations/Fit/config_xcf.ini ' value += self._branchFiles + '/Products/Correlations/Fit/config_cf_cross.ini ' update_system_status_values(path, 'data sets', 'ini files', value) value = 'PlanckDR12/PlanckDR12.fits' update_system_status_values(path, 'fiducial', 'filename', value) value = self._branchFiles + '/Products/Correlations/Fit/result_fitter2.h5' update_system_status_values(path, 'output', 'filename', value) ### Set path in config_cf.ini path = self._branchFiles + '/Products/Correlations/Fit/config_cf.ini' value = self._branchFiles + '/Products/Correlations/exported_cf.fits.gz' update_system_status_values(path, 'data', 'filename', value) value = self._branchFiles + '/Products/Correlations/metal_dmat.fits.gz' update_system_status_values(path, 'metals', 'filename', value) ### Set path in config_cf_cross.ini path = self._branchFiles + '/Products/Correlations/Fit/config_cf_cross.ini' value = self._branchFiles + '/Products/Correlations/exported_cf_cross.fits.gz' update_system_status_values(path, 'data', 'filename', value) value = self._branchFiles + '/Products/Correlations/metal_dmat_cross.fits.gz' update_system_status_values(path, 'metals', 'filename', value) ### Set path in config_xcf.ini path = self._branchFiles + '/Products/Correlations/Fit/config_xcf.ini' value = self._branchFiles + '/Products/Correlations/exported_xcf.fits.gz' update_system_status_values(path, 'data', 'filename', value) value = self._branchFiles + '/Products/Correlations/metal_xdmat.fits.gz' update_system_status_values(path, 'metals', 'filename', value) ### Send cmd = ' picca_fitter2.py '+self._branchFiles + \ '/Products/Correlations/Fit/chi2.ini' subprocess.call(cmd, shell=True) ### Test if self._test: path1 = self._masterFiles + '/result_fitter2.h5' path2 = self._branchFiles + '/Products/Correlations/Fit/result_fitter2.h5' self.compare_h5py(path1, path2, "picca_fitter2")
def _minimize(self): t0 = time.time() par_names = [name for d in self.data for name in d.pars_init] kwargs = { name: val for d in self.data for name, val in d.pars_init.items() } kwargs.update({ name: err for d in self.data for name, err in d.par_error.items() }) kwargs.update({ name: lim for d in self.data for name, lim in d.par_limit.items() }) kwargs.update({ name: fix for d in self.data for name, fix in d.par_fixed.items() }) ## do an initial "fast" minimization fixing everything except the biases kwargs_init = {} for k, v in kwargs.items(): kwargs_init[k] = v for name in par_names: if name[:4] != "bias": kwargs_init["fix_" + name] = True mig_init = iminuit.Minuit(self, forced_parameters=self.par_names, errordef=1, print_level=1, **kwargs_init) mig_init.migrad() mig_init.print_param() ## now get the best fit values for the biases and start a full minimization for name, value in mig_init.values.items(): kwargs[name] = value mig = iminuit.Minuit(self, forced_parameters=self.par_names, errordef=1, print_level=1, **kwargs) mig.migrad() mig.print_param() userprint("INFO: minimized in {}".format(time.time() - t0)) sys.stdout.flush() return mig
def _format_unknown(self, unknown): dic_unknown = {} i = 0 while (i < len(unknown)): try: if (unknown[i][:2] == '--'): dic_unknown[unknown[i][2:]] = float(unknown[i + 1]) i += 2 else: userprint( ' picca/py/picca/fitter/parameters.py:: unknown entry = ', unknown[i]) userprint(' Exit') sys.exit(0) i += 1 except: userprint( ' picca/py/picca/fitter/parameters.py:: unknown entry = ', unknown[i]) userprint(' Exit') sys.exit(0) i += 1 return dic_unknown
def test_export_cf(self): """ Test export of correlation function """ userprint("\n") ### Send cmd = "picca_export.py" cmd += " --data " + self._masterFiles + "/test_cor/cf.fits.gz" cmd += " --dmat " + self._masterFiles + "/test_cor/dmat.fits.gz" cmd += " --out " + self._branchFiles + "/Products/Correlations/exported_cf.fits.gz" print(repr(cmd)) picca.bin.picca_export.main(cmd.split()[1:]) return
def rebin_diff_noise(delta_log_lambda, log_lambda, exposures_diff): """Rebin the semidifference between two customized coadded spectra to construct the noise array The rebinning is done by combining 3 of the original pixels into analysis pixels. Args: delta_log_lambda: float Variation of the logarithm of the wavelength between two pixels log_lambda: array of floats Array containing the logarithm of the wavelengths (in Angs) exposures_diff: array of floats Semidifference between two customized coadded spectra obtained from weighted averages of the even-number exposures, for the first spectrum, and of the odd-number exposures, for the second one Returns: The noise array """ rebin = 3 if exposures_diff.size < rebin: userprint("Warning: exposures_diff.size too small for rebin") return exposures_diff rebin_delta_log_lambda = rebin * delta_log_lambda # rebin not mixing pixels separated by masks bins = np.floor((log_lambda - log_lambda.min()) / rebin_delta_log_lambda + 0.5).astype(int) rebin_exposure_diff = np.bincount(bins.astype(int), weights=exposures_diff) rebin_counts = np.bincount(bins.astype(int)) w = (rebin_counts > 0) if len(rebin_counts) == 0: userprint("Error: exposures_diff size = 0 ", exposures_diff) rebin_exposure_diff = rebin_exposure_diff[w] / np.sqrt(rebin_counts[w]) # now merge the rebinned array into a noise array noise = np.zeros(exposures_diff.size) for index in range(len(exposures_diff) // len(rebin_exposure_diff) + 1): length_max = min(len(exposures_diff), (index + 1) * len(rebin_exposure_diff)) noise[index * len(rebin_exposure_diff):length_max] = rebin_exposure_diff[:( length_max - index * len(rebin_exposure_diff))] # shuffle the array before the next iteration np.random.shuffle(rebin_exposure_diff) return noise
def send_export_co(self): userprint("\n") ### Send cmd = " picca_export_co.py" cmd += " --DD-file " + self._branchFiles + "/Products/Correlations/co_DD.fits.gz" cmd += " --RR-file " + self._branchFiles + "/Products/Correlations/Co_Random/co_RR.fits.gz" cmd += " --DR-file " + self._branchFiles + "/Products/Correlations/Co_Random/co_DR.fits.gz" cmd += " --RD-file " + self._branchFiles + "/Products/Correlations/Co_Random/co_RD.fits.gz" cmd += " --out " + self._branchFiles + "/Products/Correlations/exported_co.fits.gz" cmd += " --get-cov-from-poisson" returncode = subprocess.call(cmd, shell=True) self.assertEqual(returncode, 0, "picca_export_co did not finish") return
def add_cross(self, dic_init): self.pname.append("qso_metal_boost") self.pinit.append(dic_init["qso_metal_boost"]) self.different_drp = dic_init['different_drp'] if (self.different_drp): if not self.grid: userprint("different drp and metal matrix not implemented") sys.exit(1) for name in self.met_names: self.pname.append("drp_" + name) self.pinit.append(dic_init["drp_" + name]) met_prefix = self.met_prefix nmet = self.nmet met_names = self.met_names if self.grid: to = np.loadtxt(met_prefix + '_QSO_' + met_names[0] + '.grid') self.nd_cross = to[:, 0].size ### Get the grid of the metals self.grid_qso_met = np.zeros([self.nd_cross, nmet, 3]) for i in range(nmet): fmet = met_prefix + '_QSO_' + met_names[i] + '.grid' userprint(' Reading cross correlation metal grid : ') userprint(' ', fmet) to = np.loadtxt(fmet) idx = to[:, 0].astype(int) self.grid_qso_met[idx, i, 0] = to[:, 1] self.grid_qso_met[idx, i, 1] = to[:, 2] self.grid_qso_met[idx, i, 2] = to[:, 3] userprint() else: h = fitsio.FITS(self.met_prefix) self.abs_igm_cross = [i.strip() for i in h[1]["ABS_IGM"][:]] self.xdmat = {} self.xrp = {} self.xrt = {} self.xzeff = {} self.prev_pmet = { 'growth_rate': 0., 'drp': 0., 'qso_evol': [0., 0.] } self.prev_xi_qso_met = {} for i in self.abs_igm_cross: self.xdmat[i] = h[2]["DM_" + i][:] self.xrp[i] = h[2]["RP_" + i][:] self.xrt[i] = h[2]["RT_" + i][:] self.xzeff[i] = h[2]["Z_" + i][:] self.prev_pmet['beta_' + i] = 0. self.prev_pmet['alpha_' + i] = 0. self.prev_xi_qso_met[i] = np.zeros(self.xdmat[i].shape[0])
def send_Pk1D(self): userprint("\n") ### Send cmd = " picca_Pk1D.py" cmd += " --in-dir " + self._masterFiles + "/test_Pk1D/delta_Pk1D/" cmd += " --out-dir " + self._branchFiles + "/Products/Pk1D/" subprocess.call(cmd, shell=True) ### Test if self._test: path1 = self._masterFiles + "/test_Pk1D/Pk1D.fits.gz" path2 = self._branchFiles + "/Products/Pk1D/Pk1D-0.fits.gz" self.compare_fits(path1, path2, "picca_Pk1D.py") return