def driver(request):
from pyRSD.rsd import GalaxySpectrum
# add the PYRSD_DATA env var
os.environ['PYRSD_DATA'] = data_dir
# inititalize the model
config = {}
config['z'] = 0.55
config['cosmo_filename'] = 'runPB.ini'
config['kmin'] = 1e-3
config['kmax'] = 0.6
config['interpolate'] = True
m = GalaxySpectrum(**config)
# load the model
with cache_manager(m, "runPB_galaxy.npy") as model:
pass
# initialize the driver
path = os.path.join(data_dir, 'examples', 'params.dat')
driver = FittingDriver(path, init_model=False)
driver.model = model
# set fiducial
driver.set_fiducial()
return driver
def load_bestfit_model():
"""
Return a GalaxySpectrum model initialized with the mean of the best-fitting
theory for all 84 N-cutsky cubic boxes.
"""
# the model file
model = os.path.join(os.environ['RSDFIT'], 'data', 'models',
'model_nseries.npy')
# the directory of box 1
d = os.path.join(os.environ['RSDFIT_FITS'], 'cutsky', 'ChallengeMockN',
'fkp_1e4', 'box1')
d = os.path.join(d, 'poles',
'nlopt_gausscov_base_high_nbar_unscaled_kmax04')
# the bestfit values
fits = load_fits()
print("taking the mean of %d fits..." % len(fits))
driver = FittingDriver.from_directory(d, model_file=model)
theta = numpy.array(
[fits[name].mean() for name in driver.theory.free_names])
driver.theory.set_free_parameters(theta)
return driver.theory.model
def load_ezmock_driver(box_num,
version,
sample,
krange,
params,
z_weighted,
p=None):
"""
Load a set of ezmock fit results.
Returns a structued numpy array holding best-fit values for all free
parameters all mocks.
"""
from pyRSD.rsdfit.results import LBFGSResults
from pyRSD.rsdfit import FittingDriver
from pyRSD.rsdfit.parameters import ParameterSet
from collections import defaultdict
assert sample in ['N', 'S']
assert version in ['v1.8e-no', 'v1.8e-fph', 'v1.8e-reg']
d = os.path.join(os.environ['EBOSS_DIR'], 'fits', 'results', 'mocks',
'ezmock', version)
d = os.path.join(d, krange, params, '0.8-2.2')
assert os.path.isdir(d), "'%s' directory not found" % d
if p is None or p == 1.6:
p = [None, 1.6]
else:
p = [p]
matches = glob(os.path.join(d, f'QSO-{sample}-{box_num:04d}-*'))
match = None
for f in matches:
hashkeys = get_hashkeys(f, None)
if hashkeys['p'] in p and hashkeys['z-weighted'] == z_weighted:
match = f
if match is None:
raise ValueError((
f"no matches found: version={version}, sample={sample}, "
f"box={box_num}, krange={krange}, params={params}, z_weighted={z_weighted}, p={p}"
))
# load the driver
driver = FittingDriver.from_directory(match)
r = sorted(glob(os.path.join(match, '*.npz')),
key=os.path.getmtime,
reverse=True)
if len(r) == 0:
raise ValueError("warning: no npz results found in directory '%s'" %
os.path.normpath(match))
driver.results = r[0]
return driver
def load_data_results(version,
sample,
krange,
params,
zrange,
z_weighted,
ells=[0, 2],
p=None):
"""
Load a set of data fit results.
Returns a structued numpy array holding best-fit values for all free
parameters all mocks.
"""
from pyRSD.rsdfit import FittingDriver
assert sample in ['N', 'S']
assert version in ['v1.8', 'v1.9f']
d = os.path.join(os.environ['EBOSS_DIR'], 'fits', 'results', 'data',
version)
d = os.path.join(d, krange, params, zrange)
assert os.path.isdir(d), "'%s' directory not found" % d
if p is None or p == 1.6:
p = [None, 1.6]
else:
p = [p]
matches = glob(os.path.join(d, f'QSO-{sample}-*'))
match = None
stats = ['P%d' % ell for ell in ells]
for f in matches:
hashkeys = get_hashkeys(f, None)
if hashkeys['p'] in p and hashkeys[
'z-weighted'] == z_weighted and hashkeys['stats'] == stats:
match = f
if match is None:
raise ValueError(
(f"no matches found: version={version}, sample={sample}, "
f"krange={krange}, params={params}, zrange={zrange}, "
f"z_weighted={z_weighted}, p={p}"))
# load the driver
driver = FittingDriver.from_directory(match)
r = sorted(glob(os.path.join(match, '*.npz')),
key=os.path.getmtime,
reverse=True)
assert len(
r) > 0, "no npz results found in directory '%s'" % os.path.normpath(f)
driver.results = r[0]
return driver
def load_fits():
"""
Load a set of fit results.
Returns a structued numpy array holding best-fit values for all free
parameters all mocks.
"""
# find matches
pattern = os.path.join(os.environ['RSDFIT_FITS'], 'cutsky',
'ChallengeMockN', 'fkp_1e4', 'box*')
pattern = os.path.join(pattern, 'poles',
'nlopt_gausscov_base_high_nbar_unscaled_kmax04')
matches = glob(pattern)
assert len(matches) > 0
driver = None
data = defaultdict(list)
for f in matches:
r = sorted(glob(os.path.join(f, '*.npz')),
key=os.path.getmtime,
reverse=True)
assert len(
r
) > 0, "no npz results found in directory '%s'" % os.path.normpath(f)
th = ParameterSet.from_file(os.path.join(f, 'params.dat'),
tags='theory')
r = LBFGSResults.from_npz(r[0])
for param in r.free_names:
data[param].append(r[param])
th[param].value = r[param]
if driver is None:
driver = FittingDriver.from_directory(f, init_model=False)
# add fsigma8
if 'f' in r.free_names and 'sigma8_z' in r.free_names:
data['fsigma8'].append(r['f'] * r['sigma8_z'])
# the prior to add back
lnprior = sum(par.lnprior for par in th.free)
# add the reduced chi2
red_chi2 = (2 * (r.min_chi2 + lnprior)) / driver.dof
data['red_chi2'].append(red_chi2)
params = list(data.keys())
dtype = list(zip(params, ['f8'] * len(params)))
out = numpy.empty(len(matches), dtype=dtype)
for param in out.dtype.names:
out[param] = numpy.array(data[param])
return out
def driver(self):
"""
Return the ``FittingDriver``
"""
try:
return self._driver
except AttributeError:
d = self.fitting_dir
r = self.result
self._driver = FittingDriver.from_directory(d, results_file=r, model_file=rsd_model())
if model_kmax is not None:
self._driver.theory.model.kmax = model_kmax
return self._driver
def load_fits(sample, params):
"""
Load a set of ezmock fit results.
Returns a structued numpy array holding best-fit values for all free
parameters all mocks.
"""
# find matches
pattern = os.path.join(os.environ['EBOSS_FITS'], 'mocks', 'ezmock', 'v1.8e-fph', '0.0001-0.3', params)
assert os.path.isdir(pattern)
assert sample in 'NS'
pattern = os.path.join(pattern, '0.8-2.2', f'QSO-{sample}-*-P0+P2-mock-cov-*')
matches = glob(pattern)
assert len(matches) > 0
driver = None
data = defaultdict(list)
for f in matches:
r = sorted(glob(os.path.join(f, '*.npz')), key=os.path.getmtime, reverse=True)
assert len(r) > 0, "no npz results found in directory '%s'" %os.path.normpath(f)
th = ParameterSet.from_file(os.path.join(f, 'params.dat'), tags='theory')
r = LBFGSResults.from_npz(r[0])
for param in r.free_names:
data[param].append(r[param])
th[param].value = r[param]
if driver is None:
driver = FittingDriver.from_directory(f, init_model=False)
# add fsigma8
if 'f' in r.free_names and 'sigma8_z' in r.free_names:
data['fsigma8'].append(r['f'] * r['sigma8_z'])
# the prior to add back
lnprior = sum(par.lnprior for par in th.free)
# add the reduced chi2
red_chi2 = (2*(r.min_chi2 + lnprior)) / driver.dof
data['red_chi2'].append(red_chi2)
params = list(data.keys())
dtype = list(zip(params, ['f8']*len(params)))
out = numpy.empty(len(matches), dtype=dtype)
for param in out.dtype.names:
out[param] = numpy.array(data[param])
return out
def driver(request):
from pyRSD.rsd import QuasarSpectrum
# add the PYRSD_DATA env var
os.environ['PYRSD_DATA'] = data_dir
# inititalize the model
config = {}
config['z'] = 0.55
config['cosmo_filename'] = 'runPB.ini'
config['kmin'] = 1e-3
config['kmax'] = 0.6
m = QuasarSpectrum(**config)
# initialize the driver
path = os.path.join(data_dir, 'examples', 'params_qso.dat')
driver = FittingDriver(path, init_model=False)
driver.model = m
# set fiducial
driver.set_fiducial()
return driver
def load_bestfit_model(sample, params):
"""
Return a GalaxySpectrum model initialized with the mean of the best-fitting
theory.
"""
# the directory of box 1
d = os.path.join(os.environ['EBOSS_FITS'], 'mocks', 'ezmock', 'v1.8e-fph', '0.0001-0.3', params)
assert os.path.isdir(d)
assert sample in 'NS'
d = os.path.join(d, '0.8-2.2', f'QSO-{sample}-0001-P0+P2-mock-cov-*')
d = glob(d)[0]
# the bestfit values
fits = load_fits(sample, params)
print("taking the mean of %d fits..." %len(fits))
driver = FittingDriver.from_directory(d)
theta = numpy.array([fits[name].mean() for name in driver.theory.free_names])
driver.theory.set_free_parameters(theta)
return driver.theory.model
def plot_mcmc_bestfit():
"""
Load a mcmc bestfit and plot it
"""
from pyRSD.rsdfit import FittingDriver
import plotify as pfy
parser = argparse.ArgumentParser()
parser.formatter_class = argparse.RawTextHelpFormatter
# arguments
h = 'the name of directory holding the results'
parser.add_argument('results_dir', type=str, help=h)
h = 'the name of the results file to load'
parser.add_argument('results_file', type=str, help=h)
h = 'the name of the model to load'
parser.add_argument('model', type=str, help=h)
h = 'the output file name'
parser.add_argument('-o', '--output', type=str, help=h)
h = 'the number of burnin steps to set'
parser.add_argument('-b', '--burnin', type=int, help=h)
args = parser.parse_args()
# load the driver
kwargs = {'results_file':args.results_file, 'model_file':args.model}
driver = FittingDriver.from_directory(args.results_dir, **kwargs)
if args.burnin is not None:
driver.results.burnin = args.burnin
driver.set_fit_results()
driver.plot()
if args.output is not None:
pfy.savefig(args.output)
pfy.show()
def load_ezmock_results(version, sample, krange, params, z_weighted, p=None):
"""
Load a set of ezmock fit results.
Returns a structued numpy array holding best-fit values for all free
parameters all mocks.
"""
from pyRSD.rsdfit.results import LBFGSResults
from pyRSD.rsdfit import FittingDriver
from pyRSD.rsdfit.parameters import ParameterSet
from collections import defaultdict
assert sample in ['N', 'S']
assert version in ['v1.8e-no', 'v1.8e-fph', 'v1.8e-reg']
d = os.path.join(os.environ['EBOSS_DIR'], 'fits', 'results', 'mocks',
'ezmock', version)
d = os.path.join(d, krange, params, '0.8-2.2')
assert os.path.isdir(d), "'%s' directory not found" % d
if p is None or p == 1.6:
p = [None, 1.6]
else:
p = [p]
matches = glob(os.path.join(d, f'QSO-{sample}-0001-*'))
match = None
for f in matches:
hashkeys = get_hashkeys(f, None)
if hashkeys['p'] in p and hashkeys['z-weighted'] == z_weighted:
match = f
if match is None:
raise ValueError((
f"no matches found: version={version}, sample={sample}, "
f"krange={krange}, params={params}, z_weighted={z_weighted}, p={p}"
))
# load the driver
driver = FittingDriver.from_directory(match)
dirname, basename = os.path.split(match)
pattern = os.path.join(dirname, basename.replace('0001', '*'))
data = defaultdict(list)
matches = glob(pattern)
for f in matches:
r = sorted(glob(os.path.join(f, '*.npz')),
key=os.path.getmtime,
reverse=True)
if len(r) == 0:
raise ValueError(
"warning: no npz results found in directory '%s'" %
os.path.normpath(f))
th = ParameterSet.from_file(os.path.join(f, 'params.dat'),
tags='theory')
r = LBFGSResults.from_npz(r[0])
for param in r.free_names:
data[param].append(r[param])
th[param].value = r[param]
# add fsigma8
if 'f' in r.free_names and 'sigma8_z' in r.free_names:
data['fsigma8'].append(r['f'] * r['sigma8_z'])
# the prior to add back
lnprior = sum(par.lnprior for par in th.free)
# add the reduced chi2
red_chi2 = (2 * (r.min_chi2 + lnprior)) / driver.dof
data['red_chi2'].append(red_chi2)
params = list(data.keys())
dtype = list(zip(params, ['f8'] * len(params)))
out = numpy.empty(len(matches), dtype=dtype)
for param in out.dtype.names:
out[param] = numpy.array(data[param])
return out
def initialize_algorithm(self):
"""
Initialize the driver that runs the desired algorithm.
"""
# ``analysis`` mode
if self.mode == 'analyze':
from pyRSD.rsdfit import analysis
kws = {k: getattr(self, k) for k in self._config}
driver = analysis.AnalysisDriver(**kws)
# ``mcmc`` or ``nlopt`` mode
elif self.mode in ['mcmc', 'nlopt']:
params_file = os.path.join(self.folder, params_filename)
# initialize from an existing directory, with existing parameter file
if os.path.isdir(
self.folder
) and self.params == params_file and os.path.exists(params_file):
driver = FittingDriver.from_directory(self.folder,
model_file=self.model)
# initalize a new object from scratch
else:
init_model = self.model is None
driver = FittingDriver(self.params, init_model=init_model)
# initialize and save a model, if we need to
if not init_model:
driver.model = self.model
else:
if self.comm.rank == 0 and not self.no_save_model:
model_dir = driver.params.get('model_dir', self.folder)
driver.theory.model.to_npy(
os.path.join(model_dir, model_filename))
# only one rank needs to write out
if self.comm.rank == 0:
driver.to_file(os.path.join(self.folder, params_filename))
# have everyone wait
self.comm.barrier()
# store the command line arguments
driver.params.add('solver_type',
value=self.mode) # either nlopt or mcmc
driver.params.add('iterations', value=self.iterations)
if self.mode == 'mcmc':
driver.params.add('walkers', value=self.walkers)
# set max iterations for LBFGS
if self.mode == 'nlopt':
options = driver.params['lbfgs_options'].value
options['max_iter'] = self.iterations
# check if we need to find previous result
init_from = driver.params.get('init_from', None)
if init_from == 'result':
start_from = driver.params.get('start_from', None)
if start_from is None:
start_from = find_init_result(self.folder)
elif os.path.isdir(start_from):
start_from = find_init_result(start_from)
if not os.path.exists(start_from):
raise rsd_io.ConfigurationError(
"`start_from` parameter `%s` is not a valid path" %
start_from)
driver.params.add('start_from', value=start_from)
# ``restart`` mode
elif self.mode == 'restart':
# load the driver from param file, optionally reading model from file
driver = FittingDriver.from_directory(self.folder,
model_file=self.model)
# set driver values from command line
if self.burnin is not None:
driver.params.add('burnin', value=self.burnin)
self.algorithm = driver
