def load_bestfit_model(): """ Return a GalaxySpectrum model initialized with the mean of the best-fitting theory for all 84 N-cutsky cubic boxes. """ # the model file model = os.path.join(os.environ['RSDFIT'], 'data', 'models', 'model_nseries.npy') # the directory of box 1 d = os.path.join(os.environ['RSDFIT_FITS'], 'cutsky', 'ChallengeMockN', 'fkp_1e4', 'box1') d = os.path.join(d, 'poles', 'nlopt_gausscov_base_high_nbar_unscaled_kmax04') # the bestfit values fits = load_fits() print("taking the mean of %d fits..." % len(fits)) driver = FittingDriver.from_directory(d, model_file=model) theta = numpy.array( [fits[name].mean() for name in driver.theory.free_names]) driver.theory.set_free_parameters(theta) return driver.theory.model
def load_ezmock_driver(box_num, version, sample, krange, params, z_weighted, p=None): """ Load a set of ezmock fit results. Returns a structued numpy array holding best-fit values for all free parameters all mocks. """ from pyRSD.rsdfit.results import LBFGSResults from pyRSD.rsdfit import FittingDriver from pyRSD.rsdfit.parameters import ParameterSet from collections import defaultdict assert sample in ['N', 'S'] assert version in ['v1.8e-no', 'v1.8e-fph', 'v1.8e-reg'] d = os.path.join(os.environ['EBOSS_DIR'], 'fits', 'results', 'mocks', 'ezmock', version) d = os.path.join(d, krange, params, '0.8-2.2') assert os.path.isdir(d), "'%s' directory not found" % d if p is None or p == 1.6: p = [None, 1.6] else: p = [p] matches = glob(os.path.join(d, f'QSO-{sample}-{box_num:04d}-*')) match = None for f in matches: hashkeys = get_hashkeys(f, None) if hashkeys['p'] in p and hashkeys['z-weighted'] == z_weighted: match = f if match is None: raise ValueError(( f"no matches found: version={version}, sample={sample}, " f"box={box_num}, krange={krange}, params={params}, z_weighted={z_weighted}, p={p}" )) # load the driver driver = FittingDriver.from_directory(match) r = sorted(glob(os.path.join(match, '*.npz')), key=os.path.getmtime, reverse=True) if len(r) == 0: raise ValueError("warning: no npz results found in directory '%s'" % os.path.normpath(match)) driver.results = r[0] return driver
def load_data_results(version, sample, krange, params, zrange, z_weighted, ells=[0, 2], p=None): """ Load a set of data fit results. Returns a structued numpy array holding best-fit values for all free parameters all mocks. """ from pyRSD.rsdfit import FittingDriver assert sample in ['N', 'S'] assert version in ['v1.8', 'v1.9f'] d = os.path.join(os.environ['EBOSS_DIR'], 'fits', 'results', 'data', version) d = os.path.join(d, krange, params, zrange) assert os.path.isdir(d), "'%s' directory not found" % d if p is None or p == 1.6: p = [None, 1.6] else: p = [p] matches = glob(os.path.join(d, f'QSO-{sample}-*')) match = None stats = ['P%d' % ell for ell in ells] for f in matches: hashkeys = get_hashkeys(f, None) if hashkeys['p'] in p and hashkeys[ 'z-weighted'] == z_weighted and hashkeys['stats'] == stats: match = f if match is None: raise ValueError( (f"no matches found: version={version}, sample={sample}, " f"krange={krange}, params={params}, zrange={zrange}, " f"z_weighted={z_weighted}, p={p}")) # load the driver driver = FittingDriver.from_directory(match) r = sorted(glob(os.path.join(match, '*.npz')), key=os.path.getmtime, reverse=True) assert len( r) > 0, "no npz results found in directory '%s'" % os.path.normpath(f) driver.results = r[0] return driver
def load_fits(): """ Load a set of fit results. Returns a structued numpy array holding best-fit values for all free parameters all mocks. """ # find matches pattern = os.path.join(os.environ['RSDFIT_FITS'], 'cutsky', 'ChallengeMockN', 'fkp_1e4', 'box*') pattern = os.path.join(pattern, 'poles', 'nlopt_gausscov_base_high_nbar_unscaled_kmax04') matches = glob(pattern) assert len(matches) > 0 driver = None data = defaultdict(list) for f in matches: r = sorted(glob(os.path.join(f, '*.npz')), key=os.path.getmtime, reverse=True) assert len( r ) > 0, "no npz results found in directory '%s'" % os.path.normpath(f) th = ParameterSet.from_file(os.path.join(f, 'params.dat'), tags='theory') r = LBFGSResults.from_npz(r[0]) for param in r.free_names: data[param].append(r[param]) th[param].value = r[param] if driver is None: driver = FittingDriver.from_directory(f, init_model=False) # add fsigma8 if 'f' in r.free_names and 'sigma8_z' in r.free_names: data['fsigma8'].append(r['f'] * r['sigma8_z']) # the prior to add back lnprior = sum(par.lnprior for par in th.free) # add the reduced chi2 red_chi2 = (2 * (r.min_chi2 + lnprior)) / driver.dof data['red_chi2'].append(red_chi2) params = list(data.keys()) dtype = list(zip(params, ['f8'] * len(params))) out = numpy.empty(len(matches), dtype=dtype) for param in out.dtype.names: out[param] = numpy.array(data[param]) return out
def driver(self): """ Return the ``FittingDriver`` """ try: return self._driver except AttributeError: d = self.fitting_dir r = self.result self._driver = FittingDriver.from_directory(d, results_file=r, model_file=rsd_model()) if model_kmax is not None: self._driver.theory.model.kmax = model_kmax return self._driver
def load_fits(sample, params): """ Load a set of ezmock fit results. Returns a structued numpy array holding best-fit values for all free parameters all mocks. """ # find matches pattern = os.path.join(os.environ['EBOSS_FITS'], 'mocks', 'ezmock', 'v1.8e-fph', '0.0001-0.3', params) assert os.path.isdir(pattern) assert sample in 'NS' pattern = os.path.join(pattern, '0.8-2.2', f'QSO-{sample}-*-P0+P2-mock-cov-*') matches = glob(pattern) assert len(matches) > 0 driver = None data = defaultdict(list) for f in matches: r = sorted(glob(os.path.join(f, '*.npz')), key=os.path.getmtime, reverse=True) assert len(r) > 0, "no npz results found in directory '%s'" %os.path.normpath(f) th = ParameterSet.from_file(os.path.join(f, 'params.dat'), tags='theory') r = LBFGSResults.from_npz(r[0]) for param in r.free_names: data[param].append(r[param]) th[param].value = r[param] if driver is None: driver = FittingDriver.from_directory(f, init_model=False) # add fsigma8 if 'f' in r.free_names and 'sigma8_z' in r.free_names: data['fsigma8'].append(r['f'] * r['sigma8_z']) # the prior to add back lnprior = sum(par.lnprior for par in th.free) # add the reduced chi2 red_chi2 = (2*(r.min_chi2 + lnprior)) / driver.dof data['red_chi2'].append(red_chi2) params = list(data.keys()) dtype = list(zip(params, ['f8']*len(params))) out = numpy.empty(len(matches), dtype=dtype) for param in out.dtype.names: out[param] = numpy.array(data[param]) return out
def load_bestfit_model(sample, params): """ Return a GalaxySpectrum model initialized with the mean of the best-fitting theory. """ # the directory of box 1 d = os.path.join(os.environ['EBOSS_FITS'], 'mocks', 'ezmock', 'v1.8e-fph', '0.0001-0.3', params) assert os.path.isdir(d) assert sample in 'NS' d = os.path.join(d, '0.8-2.2', f'QSO-{sample}-0001-P0+P2-mock-cov-*') d = glob(d)[0] # the bestfit values fits = load_fits(sample, params) print("taking the mean of %d fits..." %len(fits)) driver = FittingDriver.from_directory(d) theta = numpy.array([fits[name].mean() for name in driver.theory.free_names]) driver.theory.set_free_parameters(theta) return driver.theory.model
def plot_mcmc_bestfit(): """ Load a mcmc bestfit and plot it """ from pyRSD.rsdfit import FittingDriver import plotify as pfy parser = argparse.ArgumentParser() parser.formatter_class = argparse.RawTextHelpFormatter # arguments h = 'the name of directory holding the results' parser.add_argument('results_dir', type=str, help=h) h = 'the name of the results file to load' parser.add_argument('results_file', type=str, help=h) h = 'the name of the model to load' parser.add_argument('model', type=str, help=h) h = 'the output file name' parser.add_argument('-o', '--output', type=str, help=h) h = 'the number of burnin steps to set' parser.add_argument('-b', '--burnin', type=int, help=h) args = parser.parse_args() # load the driver kwargs = {'results_file':args.results_file, 'model_file':args.model} driver = FittingDriver.from_directory(args.results_dir, **kwargs) if args.burnin is not None: driver.results.burnin = args.burnin driver.set_fit_results() driver.plot() if args.output is not None: pfy.savefig(args.output) pfy.show()
def load_ezmock_results(version, sample, krange, params, z_weighted, p=None): """ Load a set of ezmock fit results. Returns a structued numpy array holding best-fit values for all free parameters all mocks. """ from pyRSD.rsdfit.results import LBFGSResults from pyRSD.rsdfit import FittingDriver from pyRSD.rsdfit.parameters import ParameterSet from collections import defaultdict assert sample in ['N', 'S'] assert version in ['v1.8e-no', 'v1.8e-fph', 'v1.8e-reg'] d = os.path.join(os.environ['EBOSS_DIR'], 'fits', 'results', 'mocks', 'ezmock', version) d = os.path.join(d, krange, params, '0.8-2.2') assert os.path.isdir(d), "'%s' directory not found" % d if p is None or p == 1.6: p = [None, 1.6] else: p = [p] matches = glob(os.path.join(d, f'QSO-{sample}-0001-*')) match = None for f in matches: hashkeys = get_hashkeys(f, None) if hashkeys['p'] in p and hashkeys['z-weighted'] == z_weighted: match = f if match is None: raise ValueError(( f"no matches found: version={version}, sample={sample}, " f"krange={krange}, params={params}, z_weighted={z_weighted}, p={p}" )) # load the driver driver = FittingDriver.from_directory(match) dirname, basename = os.path.split(match) pattern = os.path.join(dirname, basename.replace('0001', '*')) data = defaultdict(list) matches = glob(pattern) for f in matches: r = sorted(glob(os.path.join(f, '*.npz')), key=os.path.getmtime, reverse=True) if len(r) == 0: raise ValueError( "warning: no npz results found in directory '%s'" % os.path.normpath(f)) th = ParameterSet.from_file(os.path.join(f, 'params.dat'), tags='theory') r = LBFGSResults.from_npz(r[0]) for param in r.free_names: data[param].append(r[param]) th[param].value = r[param] # add fsigma8 if 'f' in r.free_names and 'sigma8_z' in r.free_names: data['fsigma8'].append(r['f'] * r['sigma8_z']) # the prior to add back lnprior = sum(par.lnprior for par in th.free) # add the reduced chi2 red_chi2 = (2 * (r.min_chi2 + lnprior)) / driver.dof data['red_chi2'].append(red_chi2) params = list(data.keys()) dtype = list(zip(params, ['f8'] * len(params))) out = numpy.empty(len(matches), dtype=dtype) for param in out.dtype.names: out[param] = numpy.array(data[param]) return out
def initialize_algorithm(self): """ Initialize the driver that runs the desired algorithm. """ # ``analysis`` mode if self.mode == 'analyze': from pyRSD.rsdfit import analysis kws = {k: getattr(self, k) for k in self._config} driver = analysis.AnalysisDriver(**kws) # ``mcmc`` or ``nlopt`` mode elif self.mode in ['mcmc', 'nlopt']: params_file = os.path.join(self.folder, params_filename) # initialize from an existing directory, with existing parameter file if os.path.isdir( self.folder ) and self.params == params_file and os.path.exists(params_file): driver = FittingDriver.from_directory(self.folder, model_file=self.model) # initalize a new object from scratch else: init_model = self.model is None driver = FittingDriver(self.params, init_model=init_model) # initialize and save a model, if we need to if not init_model: driver.model = self.model else: if self.comm.rank == 0 and not self.no_save_model: model_dir = driver.params.get('model_dir', self.folder) driver.theory.model.to_npy( os.path.join(model_dir, model_filename)) # only one rank needs to write out if self.comm.rank == 0: driver.to_file(os.path.join(self.folder, params_filename)) # have everyone wait self.comm.barrier() # store the command line arguments driver.params.add('solver_type', value=self.mode) # either nlopt or mcmc driver.params.add('iterations', value=self.iterations) if self.mode == 'mcmc': driver.params.add('walkers', value=self.walkers) # set max iterations for LBFGS if self.mode == 'nlopt': options = driver.params['lbfgs_options'].value options['max_iter'] = self.iterations # check if we need to find previous result init_from = driver.params.get('init_from', None) if init_from == 'result': start_from = driver.params.get('start_from', None) if start_from is None: start_from = find_init_result(self.folder) elif os.path.isdir(start_from): start_from = find_init_result(start_from) if not os.path.exists(start_from): raise rsd_io.ConfigurationError( "`start_from` parameter `%s` is not a valid path" % start_from) driver.params.add('start_from', value=start_from) # ``restart`` mode elif self.mode == 'restart': # load the driver from param file, optionally reading model from file driver = FittingDriver.from_directory(self.folder, model_file=self.model) # set driver values from command line if self.burnin is not None: driver.params.add('burnin', value=self.burnin) self.algorithm = driver