def main(A): """convolve the tau(mass) field, add in thermal broadening and redshift distortion """ sightlines = Sightlines(A) maker = SpectraMaker(A, sightlines) fgpa = FGPAmodel(A) Npixels = sightlines.Npixels.sum() spectaureal = numpy.memmap(A.SpectraOutputTauReal, mode='w+', dtype='f4', shape=Npixels) spectaured = numpy.memmap(A.SpectraOutputTauRed, mode='w+', dtype='f4', shape=Npixels) specdelta = numpy.memmap(A.SpectraOutputDelta, mode='w+', dtype='f4', shape=Npixels) def work(i): sl2 = slice(sightlines.PixelOffset[i], sightlines.PixelOffset[i] + sightlines.Npixels[i]) result = maker.convolve(i, Afunc=fgpa.Afunc, Bfunc=fgpa.Bfunc) spectaureal[sl2] = result.taureal spectaured[sl2] = result.taured specdelta[sl2] = result.delta sightlines.Z_RED[i] = result.Zqso chunkmap(work, range(len(sightlines)), 100) spectaureal.flush() spectaured.flush() specdelta.flush() sightlines.Z_RED.flush()
def main(config): DB = BootstrapDB(config) Ndof = len(DB.dummy.compress()) if mpicov.world.rank == 0: numpy.random.seed(9999) seeds = numpy.random.randint(low=0, high=99999999999, size=config.BigN) else: seeds = [] myseeds = mpicov.world.scatter(numpy.array_split(seeds, mpicov.world.size)) print 'This Task = ', mpicov.world.rank, 'Number of samples = ', \ len(myseeds), 'seed0 =', myseeds[0] myxi = sharedmem.empty((len(myseeds), Ndof), dtype='f8') def work(i): rng = numpy.random.RandomState(myseeds[i]) choice = rng.choice(len(DB), size=DB.Nchunks) sample = DB(choice) myxi[i][...] = sample.compress() print 'build samples' chunkmap(work, range(len(myxi)), 100) print 'done samples' print 'covariance matrix' cov = mpicov.cov(myxi, rowvar=0, ddof=0) print 'done covariance matrix' print numpy.nanmin(numpy.diag(cov)) if mpicov.world.rank == 0: numpy.savez(config.CovarianceMatrixOutput, cov=cov, BigN=config.BigN, dummy=DB.dummy, xi_cov=DB.dummy.copy().uncompress(myxi[0]), r=DB.r, mu=DB.mu)
def __init__(self, config): def getfilename(mock): dir = os.path.join(config.prefix, mock) paramfile = os.path.join(config.prefix, mock, 'paramfile') c = Config(paramfile, basedir=dir) return os.path.join(c.datadir, 'bootstrap.npz') if config.UseMocks is None: filenames = sorted(list(glob(os.path.join(config.prefix, '[0-9]*', '*', 'bootstrap.npz')))) else: filenames = [getfilename(mock) for mock in config.UseMocks] files = [ numpy.load(f) for f in filenames] print 'using', len(filenames), ' files', filenames self.r = files[0]['r'] self.mu = files[0]['mu'] # b/c they all have the same cosmology self.eigenmodes = EigenModes(numpy.load(config.EigenModesOutput)['eigenmodes']) self.DQDQ, self.RQDQ, self.RQRQ = sharedmem.empty( [3, len(files)] + list(files[0]['DQDQ'].shape)) self.DQDFsum1, self.RQDFsum1, self.DFDFsum1 = sharedmem.empty( [3, len(files)] + list(files[0]['DQDQ'].shape)) self.DQDFsum2, self.RQDFsum2, self.DFDFsum2 = sharedmem.empty( [3, len(files)] + list(files[0]['DQDQ'].shape)) self.ND, self.NR = sharedmem.empty([2, len(files)] + list(files[0]['Qchunksize'].shape)) def read(i): file = files[i] self.DQDQ[i] = file['DQDQ'] self.RQDQ[i] = file['RQDQ'] self.RQRQ[i] = file['RQRQ'] self.DQDFsum1[i] = file['DQDFsum1'][0] self.RQDFsum1[i] = file['RQDFsum1'][0] self.DFDFsum1[i] = file['DFDFsum1'][0] self.DQDFsum2[i] = file['DQDFsum2'] self.RQDFsum2[i] = file['RQDFsum2'] self.DFDFsum2[i] = file['DFDFsum2'] self.ND[i] = file['Qchunksize'] self.NR[i] = file['Rchunksize'] chunkmap(read, range(len(files)), 1) self.Nchunks = self.DQDQ[0].shape[0] # build the currelation function on the first sample # use it as a template self.dummy = self(0)
def getforest(A, Zmin, Zmax, RfLamMin, RfLamMax, combine=1): spectra = SpectraOutput(A) meanFred = MeanFractionMeasured(A, kind='red') meanFreal = MeanFractionMeasured(A, kind='real') combine = numpy.minimum(spectra.sightlines.Npixels.max(), combine) # will combine every this many pixels Npixels1 = spectra.sightlines.Npixels // combine Offset1 = numpy.concatenate([[0], numpy.cumsum(Npixels1)]) Npixels = Npixels1.sum() print Npixels1.min(), Npixels1.max() print spectra.sightlines.Npixels.min(), spectra.sightlines.Npixels.max() data = sharedmem.empty(Npixels, ('f4', 3)) DFred, DFreal, Delta = data.T pos = sharedmem.empty(Npixels, ('f4', 3)) x, y, z = pos.T mask = sharedmem.empty(Npixels, '?') id = sharedmem.empty(Npixels, 'i4') spectra.taured spectra.taureal def work(i): def combinepixels(value, method=numpy.mean): # reduce the number of pixels with 'method' return \ method(value[:Npixels1[i] * combine]\ .reshape([Npixels1[i]] + [combine]), axis=-1) sl = slice(Offset1[i], Npixels1[i] + Offset1[i]) a = spectra.a[i] Fred = numpy.exp(-spectra.taured[i]) / meanFred(a) - 1 Freal = numpy.exp(-spectra.taureal[i]) / meanFreal(a) - 1 DFred[sl] = combinepixels(Fred) DFreal[sl] = combinepixels(Freal) Delta[sl] = combinepixels(spectra.delta[i]) p = spectra.position(i) x[sl] = combinepixels(p[:, 0]) y[sl] = combinepixels(p[:, 1]) z[sl] = combinepixels(p[:, 2]) m = spectra.z[i] > Zmin m &= spectra.z[i] < Zmax m &= spectra.RfLam(i) > RfLamMin m &= spectra.RfLam(i) < RfLamMax mask[sl] = combinepixels(m, method=numpy.all) id[sl] = i chunkmap(work, range(len(spectra)), 100) return data[mask], pos[mask], id[mask]
def main(config): global cov DB = BootstrapDB(config) MASK = DB.dummy.imesh >= 0 MASK &= DB.dummy.rmesh <= config.rmax MASK &= DB.dummy.rmesh >= config.rmin print "dof in fitting", MASK.sum() # create a dummy to test the fitting p0 = [-0.2, 3.5, 1.5, 1.5] eigenmodes = DB.eigenmodes dummy = eigenmodes(p0) covfull = numpy.load(config.CovarianceMatrixOutput)["cov"] cov = covfull[MASK][:, MASK] print "inverting" INV = linalg.inv(covfull[MASK][:, MASK]) print "inverted" x, chi = fit1(dummy, eigenmodes, INV, MASK) print "x =", x print "p0 = bF, bQ, BF, BQ", p0 error = poles_err(dummy, covfull) fitted = sharedmem.empty((len(DB), len(p0))) chi = sharedmem.empty((len(DB))) samples, models = [], [] sharedmem.set_debug(True) def work(i): sample = DB(i) print "fitting", i fitted[i], chi[i] = fit1(sample, eigenmodes, INV, MASK) model = eigenmodes(fitted[i]) print zip(sample[0].monopole, model[0].monopole) return i, sample, model def reduce(rt): i, s, m = rt samples.append((i, s)) models.append((i, m)) chunkmap(work, range(len(DB)), 100, reduce=reduce) samples = [s for i, s in sorted(samples)] models = [s for i, s in sorted(models)] numpy.savez("fit.npz", samples=samples, models=models, fittedparameters=fitted, chi=chi, error=error)
def cost(Af, Bf): def work(i): sl = slice(sightlines.PixelOffset[i], sightlines.PixelOffset[i] + sightlines.Npixels[i]) if sightlines.Npixels[i] == 0: return taured = maker.convolve(i, Afunc=lambda x: Af, Bfunc=lambda x: Bf, returns=['taured']).taured F[sl] = numpy.exp(-taured) chunkmap(work, range(0, len(sightlines), 100), 100) F1 = F[~numpy.isnan(F)] xmeanF = F1.mean() xstdF = F1.std() v = (xmeanF/ meanF - 1) , (xstdF / stdF - 1) return v
def main(A): sightlines = Sightlines(A) spectra = SpectraOutput(A) loglam = sharedmem.empty(sightlines.Npixels.sum(), 'f4') chunksize = 100 #sharedmem.set_debug(True) def work(i): sl = slice(sightlines.PixelOffset[i], sightlines.PixelOffset[i] + sightlines.Npixels[i]) loglam[sl] = spectra.LogLam[i] chunkmap(work, range(len(sightlines)), chunksize=100) Nbins = 400 zBins = numpy.linspace(2.0, 4.0, Nbins + 1, endpoint=True) LogLamBins = numpy.log10(1216.0 * (1 + zBins )) z = 0.5 * (zBins[1:] + zBins[:-1]) ind = numpy.digitize(loglam, LogLamBins) N = numpy.bincount(ind, minlength=Nbins+2) N[N == 0] = 1.0 F = numpy.exp(-spectra.taured.data) K1 = numpy.bincount(ind, F, minlength=Nbins+2) / N K2 = numpy.bincount(ind, F ** 2, minlength=Nbins+2) / N meanF = K1 varF = K2 - K1 ** 2 meanF = meanF[1:-1] varF = varF[1:-1] F = numpy.exp(-spectra.taureal.data) K1 = numpy.bincount(ind, F, minlength=Nbins+2) / N K2 = numpy.bincount(ind, F ** 2, minlength=Nbins+2) / N meanFreal = K1 varFreal = K2 - K1 ** 2 meanFreal = meanFreal[1:-1] varFreal = varFreal[1:-1] print z print meanF print varF numpy.savez(A.MeasuredMeanFractionOutput, a=1/(1+z), abins=1/(1+zBins), xmeanF=meanF, xvarF=varF, xmeanFreal=meanFreal, xvarFreal=varFreal, )
def main(A): MatchUp = numpy.load('matchup-result.npy') spectra = SpectraOutput(A) print MatchUp.dtype def work(i): match = MatchUp[i] dirname = os.path.join( 'raw', '%04d' % match['PLATE']) try: os.makedirs(dirname) except OSError: pass print i, '/', len(MatchUp) filename = os.path.join( dirname, 'mockraw-%04d-%d-%04d.fits' % (match['PLATE'], match['MJD'], match['FIBER']) ) ind = match['Q_INDEX'] data = numpy.rec.fromarrays( [spectra.z[ind], numpy.exp(-spectra.taured[ind])], names=['Z', 'mockF']) header = dict( DEC=spectra.sightlines.DEC[ind] / numpy.pi * 180, RA=spectra.sightlines.RA[ind] / numpy.pi * 180, Z=spectra.sightlines.Z_RED[ind]) with fitsio.FITS(filename, mode='rw') as file: file.write_table(data, header=header) chunkmap(work, range(len(MatchUp)), chunksize=1000)