def main(A):
"""convolve the tau(mass) field,
add in thermal broadening and redshift distortion """
sightlines = Sightlines(A)
maker = SpectraMaker(A, sightlines)
fgpa = FGPAmodel(A)
Npixels = sightlines.Npixels.sum()
spectaureal = numpy.memmap(A.SpectraOutputTauReal, mode='w+',
dtype='f4', shape=Npixels)
spectaured = numpy.memmap(A.SpectraOutputTauRed, mode='w+',
dtype='f4', shape=Npixels)
specdelta = numpy.memmap(A.SpectraOutputDelta, mode='w+',
dtype='f4', shape=Npixels)
def work(i):
sl2 = slice(sightlines.PixelOffset[i],
sightlines.PixelOffset[i] + sightlines.Npixels[i])
result = maker.convolve(i, Afunc=fgpa.Afunc, Bfunc=fgpa.Bfunc)
spectaureal[sl2] = result.taureal
spectaured[sl2] = result.taured
specdelta[sl2] = result.delta
sightlines.Z_RED[i] = result.Zqso
chunkmap(work, range(len(sightlines)), 100)
spectaureal.flush()
spectaured.flush()
specdelta.flush()
sightlines.Z_RED.flush()
def main(config):
DB = BootstrapDB(config)
Ndof = len(DB.dummy.compress())
if mpicov.world.rank == 0:
numpy.random.seed(9999)
seeds = numpy.random.randint(low=0, high=99999999999, size=config.BigN)
else:
seeds = []
myseeds = mpicov.world.scatter(numpy.array_split(seeds, mpicov.world.size))
print 'This Task = ', mpicov.world.rank, 'Number of samples = ', \
len(myseeds), 'seed0 =', myseeds[0]
myxi = sharedmem.empty((len(myseeds), Ndof), dtype='f8')
def work(i):
rng = numpy.random.RandomState(myseeds[i])
choice = rng.choice(len(DB), size=DB.Nchunks)
sample = DB(choice)
myxi[i][...] = sample.compress()
print 'build samples'
chunkmap(work, range(len(myxi)), 100)
print 'done samples'
print 'covariance matrix'
cov = mpicov.cov(myxi, rowvar=0, ddof=0)
print 'done covariance matrix'
print numpy.nanmin(numpy.diag(cov))
if mpicov.world.rank == 0:
numpy.savez(config.CovarianceMatrixOutput, cov=cov,
BigN=config.BigN, dummy=DB.dummy,
xi_cov=DB.dummy.copy().uncompress(myxi[0]), r=DB.r, mu=DB.mu)
def __init__(self, config):
def getfilename(mock):
dir = os.path.join(config.prefix, mock)
paramfile = os.path.join(config.prefix, mock, 'paramfile')
c = Config(paramfile, basedir=dir)
return os.path.join(c.datadir, 'bootstrap.npz')
if config.UseMocks is None:
filenames = sorted(list(glob(os.path.join(config.prefix, '[0-9]*', '*',
'bootstrap.npz'))))
else:
filenames = [getfilename(mock) for mock in config.UseMocks]
files = [ numpy.load(f) for f in filenames]
print 'using', len(filenames), ' files', filenames
self.r = files[0]['r']
self.mu = files[0]['mu']
# b/c they all have the same cosmology
self.eigenmodes = EigenModes(numpy.load(config.EigenModesOutput)['eigenmodes'])
self.DQDQ, self.RQDQ, self.RQRQ = sharedmem.empty(
[3, len(files)] + list(files[0]['DQDQ'].shape))
self.DQDFsum1, self.RQDFsum1, self.DFDFsum1 = sharedmem.empty(
[3, len(files)] + list(files[0]['DQDQ'].shape))
self.DQDFsum2, self.RQDFsum2, self.DFDFsum2 = sharedmem.empty(
[3, len(files)] + list(files[0]['DQDQ'].shape))
self.ND, self.NR = sharedmem.empty([2, len(files)] +
list(files[0]['Qchunksize'].shape))
def read(i):
file = files[i]
self.DQDQ[i] = file['DQDQ']
self.RQDQ[i] = file['RQDQ']
self.RQRQ[i] = file['RQRQ']
self.DQDFsum1[i] = file['DQDFsum1'][0]
self.RQDFsum1[i] = file['RQDFsum1'][0]
self.DFDFsum1[i] = file['DFDFsum1'][0]
self.DQDFsum2[i] = file['DQDFsum2']
self.RQDFsum2[i] = file['RQDFsum2']
self.DFDFsum2[i] = file['DFDFsum2']
self.ND[i] = file['Qchunksize']
self.NR[i] = file['Rchunksize']
chunkmap(read, range(len(files)), 1)
self.Nchunks = self.DQDQ[0].shape[0]
# build the currelation function on the first sample
# use it as a template
self.dummy = self(0)
def getforest(A, Zmin, Zmax, RfLamMin, RfLamMax, combine=1):
spectra = SpectraOutput(A)
meanFred = MeanFractionMeasured(A, kind='red')
meanFreal = MeanFractionMeasured(A, kind='real')
combine = numpy.minimum(spectra.sightlines.Npixels.max(), combine)
# will combine every this many pixels
Npixels1 = spectra.sightlines.Npixels // combine
Offset1 = numpy.concatenate([[0], numpy.cumsum(Npixels1)])
Npixels = Npixels1.sum()
print Npixels1.min(), Npixels1.max()
print spectra.sightlines.Npixels.min(), spectra.sightlines.Npixels.max()
data = sharedmem.empty(Npixels, ('f4', 3))
DFred, DFreal, Delta = data.T
pos = sharedmem.empty(Npixels, ('f4', 3))
x, y, z = pos.T
mask = sharedmem.empty(Npixels, '?')
id = sharedmem.empty(Npixels, 'i4')
spectra.taured
spectra.taureal
def work(i):
def combinepixels(value, method=numpy.mean):
# reduce the number of pixels with 'method'
return \
method(value[:Npixels1[i] * combine]\
.reshape([Npixels1[i]] + [combine]),
axis=-1)
sl = slice(Offset1[i], Npixels1[i] + Offset1[i])
a = spectra.a[i]
Fred = numpy.exp(-spectra.taured[i]) / meanFred(a) - 1
Freal = numpy.exp(-spectra.taureal[i]) / meanFreal(a) - 1
DFred[sl] = combinepixels(Fred)
DFreal[sl] = combinepixels(Freal)
Delta[sl] = combinepixels(spectra.delta[i])
p = spectra.position(i)
x[sl] = combinepixels(p[:, 0])
y[sl] = combinepixels(p[:, 1])
z[sl] = combinepixels(p[:, 2])
m = spectra.z[i] > Zmin
m &= spectra.z[i] < Zmax
m &= spectra.RfLam(i) > RfLamMin
m &= spectra.RfLam(i) < RfLamMax
mask[sl] = combinepixels(m, method=numpy.all)
id[sl] = i
chunkmap(work, range(len(spectra)), 100)
return data[mask], pos[mask], id[mask]
def main(config):
global cov
DB = BootstrapDB(config)
MASK = DB.dummy.imesh >= 0
MASK &= DB.dummy.rmesh <= config.rmax
MASK &= DB.dummy.rmesh >= config.rmin
print "dof in fitting", MASK.sum()
# create a dummy to test the fitting
p0 = [-0.2, 3.5, 1.5, 1.5]
eigenmodes = DB.eigenmodes
dummy = eigenmodes(p0)
covfull = numpy.load(config.CovarianceMatrixOutput)["cov"]
cov = covfull[MASK][:, MASK]
print "inverting"
INV = linalg.inv(covfull[MASK][:, MASK])
print "inverted"
x, chi = fit1(dummy, eigenmodes, INV, MASK)
print "x =", x
print "p0 = bF, bQ, BF, BQ", p0
error = poles_err(dummy, covfull)
fitted = sharedmem.empty((len(DB), len(p0)))
chi = sharedmem.empty((len(DB)))
samples, models = [], []
sharedmem.set_debug(True)
def work(i):
sample = DB(i)
print "fitting", i
fitted[i], chi[i] = fit1(sample, eigenmodes, INV, MASK)
model = eigenmodes(fitted[i])
print zip(sample[0].monopole, model[0].monopole)
return i, sample, model
def reduce(rt):
i, s, m = rt
samples.append((i, s))
models.append((i, m))
chunkmap(work, range(len(DB)), 100, reduce=reduce)
samples = [s for i, s in sorted(samples)]
models = [s for i, s in sorted(models)]
numpy.savez("fit.npz", samples=samples, models=models, fittedparameters=fitted, chi=chi, error=error)
def cost(Af, Bf):
def work(i):
sl = slice(sightlines.PixelOffset[i],
sightlines.PixelOffset[i] + sightlines.Npixels[i])
if sightlines.Npixels[i] == 0: return
taured = maker.convolve(i,
Afunc=lambda x: Af,
Bfunc=lambda x: Bf, returns=['taured']).taured
F[sl] = numpy.exp(-taured)
chunkmap(work, range(0, len(sightlines), 100), 100)
F1 = F[~numpy.isnan(F)]
xmeanF = F1.mean()
xstdF = F1.std()
v = (xmeanF/ meanF - 1) , (xstdF / stdF - 1)
return v
def main(A):
sightlines = Sightlines(A)
spectra = SpectraOutput(A)
loglam = sharedmem.empty(sightlines.Npixels.sum(), 'f4')
chunksize = 100
#sharedmem.set_debug(True)
def work(i):
sl = slice(sightlines.PixelOffset[i],
sightlines.PixelOffset[i] + sightlines.Npixels[i])
loglam[sl] = spectra.LogLam[i]
chunkmap(work, range(len(sightlines)), chunksize=100)
Nbins = 400
zBins = numpy.linspace(2.0, 4.0, Nbins + 1, endpoint=True)
LogLamBins = numpy.log10(1216.0 * (1 + zBins ))
z = 0.5 * (zBins[1:] + zBins[:-1])
ind = numpy.digitize(loglam, LogLamBins)
N = numpy.bincount(ind, minlength=Nbins+2)
N[N == 0] = 1.0
F = numpy.exp(-spectra.taured.data)
K1 = numpy.bincount(ind, F, minlength=Nbins+2) / N
K2 = numpy.bincount(ind, F ** 2, minlength=Nbins+2) / N
meanF = K1
varF = K2 - K1 ** 2
meanF = meanF[1:-1]
varF = varF[1:-1]
F = numpy.exp(-spectra.taureal.data)
K1 = numpy.bincount(ind, F, minlength=Nbins+2) / N
K2 = numpy.bincount(ind, F ** 2, minlength=Nbins+2) / N
meanFreal = K1
varFreal = K2 - K1 ** 2
meanFreal = meanFreal[1:-1]
varFreal = varFreal[1:-1]
print z
print meanF
print varF
numpy.savez(A.MeasuredMeanFractionOutput,
a=1/(1+z), abins=1/(1+zBins),
xmeanF=meanF, xvarF=varF,
xmeanFreal=meanFreal,
xvarFreal=varFreal,
)
def main(A):
MatchUp = numpy.load('matchup-result.npy')
spectra = SpectraOutput(A)
print MatchUp.dtype
def work(i):
match = MatchUp[i]
dirname = os.path.join(
'raw',
'%04d' % match['PLATE'])
try:
os.makedirs(dirname)
except OSError:
pass
print i, '/', len(MatchUp)
filename = os.path.join(
dirname,
'mockraw-%04d-%d-%04d.fits' %
(match['PLATE'],
match['MJD'],
match['FIBER'])
)
ind = match['Q_INDEX']
data = numpy.rec.fromarrays(
[spectra.z[ind], numpy.exp(-spectra.taured[ind])],
names=['Z', 'mockF'])
header = dict(
DEC=spectra.sightlines.DEC[ind] / numpy.pi * 180,
RA=spectra.sightlines.RA[ind] / numpy.pi * 180,
Z=spectra.sightlines.Z_RED[ind])
with fitsio.FITS(filename, mode='rw') as file:
file.write_table(data, header=header)
chunkmap(work, range(len(MatchUp)), chunksize=1000)
