def test_gaussian(self): print("test_gaussian") v = -6.2e-16 s = 1.0e-18 self.m.F1.value self.m.F1.prior = Prior(norm(loc=v, scale=s)) print(self.m.F1.prior_pdf(v)) assert np.isclose(self.m.F1.prior_pdf(v), 1.0 / (s * np.sqrt(2.0 * np.pi)))
def test_uniform_bounded(self): print("test_uniform_bounded") self.m.ECC.prior = Prior(UniformBoundedRV(0.0, 1.0)) assert self.m.ECC.prior_pdf() == 1.0 assert self.m.ECC.prior_pdf(logpdf=True) == 0.0 assert self.m.ECC.prior_pdf(-0.1) == 0.0 assert self.m.ECC.prior_pdf(-0.1, logpdf=True) == -np.inf assert self.m.ECC.prior_pdf(1.1) == 0.0 assert self.m.ECC.prior_pdf(1.1, logpdf=True) == -np.inf
def test_inclination_prior(self): self.m.SINI.prior = Prior(InclinationPrior()) v = 0.5 # Check the prior PDF matches hand computation assert np.isclose(self.m.SINI.prior_pdf(v), 0.5773502691896258) # Check that np.exp(logpdf) == pdf assert np.isclose(self.m.SINI.prior.pdf(v), np.exp(self.m.SINI.prior.logpdf(v))) v = 0.9 # Check the prior PDF matches hand computation assert np.isclose(self.m.SINI.prior_pdf(v), 2.0647416048350564) # Check that np.exp(logpdf) == pdf assert np.isclose(self.m.SINI.prior.pdf(v), np.exp(self.m.SINI.prior.logpdf(v)))
def set_priors_basic(ftr, priorerrfact=10.0): """Basic method to set priors on parameters in the model. This adds a gaussian prior on each parameter with width equal to the par file uncertainty * priorerrfact and then puts in some special cases """ fkeys, fvals, ferrs = ftr.fitkeys, ftr.fitvals, ftr.fiterrs for key, val, err in zip(fkeys[:-1], fvals[:-1], ferrs[:-1]): if key == 'SINI' or key == 'E' or key == 'ECC': getattr(ftr.model, key).prior = Prior(uniform(0.0, 1.0)) elif key == 'PX': getattr(ftr.model, key).prior = Prior(uniform(0.0, 10.0)) elif key.startswith('GLPH'): getattr(ftr.model, key).prior = Prior(uniform(-0.4, 1.0)) else: if err == 0 and not getattr(ftr.model, key).frozen: ftr.priors_set = False raise ValueError('Parameter %s does not have uncertainty in par file' % key) getattr(ftr.model, key).prior = Prior(norm(loc=float(val), scale=float(err*priorerrfact))) ftr.priors_set = True
def test_gaussian_bounded(self): print("test_gaussian_bounded") self.m.M2.prior = Prior( GaussianBoundedRV(loc=0.26, scale=0.10, lower_bound=0.0, upper_bound=0.6) ) assert self.m.M2.prior_pdf(-0.1) == 0.0 assert self.m.M2.prior_pdf(0.7) == 0.0 assert self.m.M2.prior_pdf(-0.1, logpdf=True) == -np.inf assert self.m.M2.prior_pdf(0.7, logpdf=True) == -np.inf assert np.isclose( self.m.M2.prior_pdf(0.26 + 0.1) / self.m.M2.prior_pdf(0.26), 0.60653065971263342, ) # Test that integral is 1.0, not safe since using _rv private var assert np.isclose(self.m.M2.prior._rv.cdf(0.6), 1.0)
def main(argv=None): parser = argparse.ArgumentParser( description= "PINT tool for MCMC optimization of timing models using event data.") parser.add_argument("eventfile", help="event file to use") parser.add_argument("parfile", help="par file to read model from") parser.add_argument("gaussianfile", help="gaussian file that defines template") parser.add_argument("--ft2", help="Path to FT2 file.", default=None) parser.add_argument( "--weightcol", help="name of weight column (or 'CALC' to have them computed", default=None, ) parser.add_argument("--nwalkers", help="Number of MCMC walkers (def 200)", type=int, default=200) parser.add_argument( "--burnin", help="Number of MCMC steps for burn in (def 100)", type=int, default=100, ) parser.add_argument( "--nsteps", help="Number of MCMC steps to compute (def 1000)", type=int, default=1000, ) parser.add_argument("--minMJD", help="Earliest MJD to use (def 54680)", type=float, default=54680.0) parser.add_argument("--maxMJD", help="Latest MJD to use (def 57250)", type=float, default=57250.0) parser.add_argument("--phs", help="Starting phase offset [0-1] (def is to measure)", type=float) parser.add_argument("--phserr", help="Error on starting phase", type=float, default=0.03) parser.add_argument( "--minWeight", help="Minimum weight to include (def 0.05)", type=float, default=0.05, ) parser.add_argument( "--wgtexp", help= "Raise computed weights to this power (or 0.0 to disable any rescaling of weights)", type=float, default=0.0, ) parser.add_argument( "--testWeights", help="Make plots to evalute weight cuts?", default=False, action="store_true", ) parser.add_argument( "--doOpt", help="Run initial scipy opt before MCMC?", default=False, action="store_true", ) parser.add_argument( "--initerrfact", help= "Multiply par file errors by this factor when initializing walker starting values", type=float, default=0.1, ) parser.add_argument( "--priorerrfact", help= "Multiple par file errors by this factor when setting gaussian prior widths", type=float, default=10.0, ) parser.add_argument( "--usepickle", help="Read events from pickle file, if available?", default=False, action="store_true", ) global nwalkers, nsteps, ftr args = parser.parse_args(argv) eventfile = args.eventfile parfile = args.parfile gaussianfile = args.gaussianfile weightcol = args.weightcol if args.ft2 is not None: # Instantiate Fermi observatory once so it gets added to the observatory registry get_satellite_observatory("Fermi", args.ft2) nwalkers = args.nwalkers burnin = args.burnin nsteps = args.nsteps if burnin >= nsteps: log.error("burnin must be < nsteps") sys.exit(1) nbins = 256 # For likelihood calculation based on gaussians file outprof_nbins = 256 # in the text file, for pygaussfit.py, for instance minMJD = args.minMJD maxMJD = args.maxMJD # Usually set by coverage of IERS file minWeight = args.minWeight do_opt_first = args.doOpt wgtexp = args.wgtexp # Read in initial model modelin = pint.models.get_model(parfile) # The custom_timing version below is to manually construct the TimingModel # class, which allows it to be pickled. This is needed for parallelizing # the emcee call over a number of threads. So far, it isn't quite working # so it is disabled. The code above constructs the TimingModel class # dynamically, as usual. # modelin = custom_timing(parfile) # Remove the dispersion delay as it is unnecessary # modelin.delay_funcs['L1'].remove(modelin.dispersion_delay) # Set the target coords for automatic weighting if necessary if "ELONG" in modelin.params: tc = SkyCoord( modelin.ELONG.quantity, modelin.ELAT.quantity, frame="barycentrictrueecliptic", ) else: tc = SkyCoord(modelin.RAJ.quantity, modelin.DECJ.quantity, frame="icrs") target = tc if weightcol == "CALC" else None # TODO: make this properly handle long double if not args.usepickle or (not (os.path.isfile(eventfile + ".pickle") or os.path.isfile(eventfile + ".pickle.gz"))): # Read event file and return list of TOA objects tl = fermi.load_Fermi_TOAs(eventfile, weightcolumn=weightcol, targetcoord=target, minweight=minWeight) # Limit the TOAs to ones in selected MJD range and above minWeight tl = [ tl[ii] for ii in range(len(tl)) if (tl[ii].mjd.value > minMJD and tl[ii].mjd.value < maxMJD and ( weightcol is None or tl[ii].flags["weight"] > minWeight)) ] log.info("There are %d events we will use" % len(tl)) # Now convert to TOAs object and compute TDBs and posvels ts = toa.TOAs(toalist=tl) ts.filename = eventfile ts.compute_TDBs() ts.compute_posvels(ephem="DE421", planets=False) ts.pickle() else: # read the events in as a pickle file picklefile = toa._check_pickle(eventfile) if not picklefile: picklefile = eventfile ts = toa.TOAs(picklefile) if weightcol is not None: if weightcol == "CALC": weights = np.asarray([x["weight"] for x in ts.table["flags"]]) log.info("Original weights have min / max weights %.3f / %.3f" % (weights.min(), weights.max())) # Rescale the weights, if requested (by having wgtexp != 0.0) if wgtexp != 0.0: weights **= wgtexp wmx, wmn = weights.max(), weights.min() # make the highest weight = 1, but keep min weight the same weights = wmn + ((weights - wmn) * (1.0 - wmn) / (wmx - wmn)) for ii, x in enumerate(ts.table["flags"]): x["weight"] = weights[ii] weights = np.asarray([x["weight"] for x in ts.table["flags"]]) log.info("There are %d events, with min / max weights %.3f / %.3f" % (len(weights), weights.min(), weights.max())) else: weights = None log.info("There are %d events, no weights are being used." % ts.ntoas) # Now load in the gaussian template and normalize it gtemplate = read_gaussfitfile(gaussianfile, nbins) gtemplate /= gtemplate.mean() # Set the priors on the parameters in the model, before # instantiating the emcee_fitter # Currently, this adds a gaussian prior on each parameter # with width equal to the par file uncertainty * priorerrfact, # and then puts in some special cases. # *** This should be replaced/supplemented with a way to specify # more general priors on parameters that need certain bounds phs = 0.0 if args.phs is None else args.phs fitkeys, fitvals, fiterrs = get_fit_keyvals(modelin, phs=phs, phserr=args.phserr) for key, v, e in zip(fitkeys[:-1], fitvals[:-1], fiterrs[:-1]): if key == "SINI" or key == "E" or key == "ECC": getattr(modelin, key).prior = Prior(uniform(0.0, 1.0)) elif key == "PX": getattr(modelin, key).prior = Prior(uniform(0.0, 10.0)) elif key.startswith("GLPH"): getattr(modelin, key).prior = Prior(uniform(-0.5, 1.0)) else: getattr(modelin, key).prior = Prior( norm(loc=float(v), scale=float(e * args.priorerrfact))) # Now define the requirements for emcee ftr = emcee_fitter(ts, modelin, gtemplate, weights, phs, args.phserr) # Use this if you want to see the effect of setting minWeight if args.testWeights: log.info("Checking H-test vs weights") ftr.prof_vs_weights(use_weights=True) ftr.prof_vs_weights(use_weights=False) sys.exit() # Now compute the photon phases and see if we see a pulse phss = ftr.get_event_phases() maxbin, like_start = marginalize_over_phase(phss, gtemplate, weights=ftr.weights, minimize=True, showplot=False) log.info("Starting pulse likelihood: %f" % like_start) if args.phs is None: fitvals[-1] = 1.0 - maxbin[0] / float(len(gtemplate)) if fitvals[-1] > 1.0: fitvals[-1] -= 1.0 if fitvals[-1] < 0.0: fitvals[-1] += 1.0 log.info("Starting pulse phase: %f" % fitvals[-1]) else: log.warning("Measured starting pulse phase is %f, but using %f" % (1.0 - maxbin / float(len(gtemplate)), args.phs)) fitvals[-1] = args.phs ftr.fitvals[-1] = fitvals[-1] ftr.phaseogram(plotfile=ftr.model.PSR.value + "_pre.png") plt.close() # ftr.phaseogram() # Write out the starting pulse profile vs, xs = np.histogram(ftr.get_event_phases(), outprof_nbins, range=[0, 1], weights=ftr.weights) f = open(ftr.model.PSR.value + "_prof_pre.txt", "w") for x, v in zip(xs, vs): f.write("%.5f %12.5f\n" % (x, v)) f.close() # Try normal optimization first to see how it goes if do_opt_first: result = op.minimize(ftr.minimize_func, np.zeros_like(ftr.fitvals)) newfitvals = np.asarray(result["x"]) * ftr.fiterrs + ftr.fitvals like_optmin = -result["fun"] log.info("Optimization likelihood: %f" % like_optmin) ftr.set_params(dict(zip(ftr.fitkeys, newfitvals))) ftr.phaseogram() else: like_optmin = -np.inf # Set up the initial conditions for the emcee walkers. Use the # scipy.optimize newfitvals instead if they are better ndim = ftr.n_fit_params if like_start > like_optmin: # Keep the starting deviations small... pos = [ ftr.fitvals + ftr.fiterrs * args.initerrfact * np.random.randn(ndim) for ii in range(nwalkers) ] # Set starting params for param in [ "GLPH_1", "GLEP_1", "SINI", "M2", "E", "ECC", "PX", "A1" ]: if param in ftr.fitkeys: idx = ftr.fitkeys.index(param) if param == "GLPH_1": svals = np.random.uniform(-0.5, 0.5, nwalkers) elif param == "GLEP_1": svals = np.random.uniform(minMJD + 100, maxMJD - 100, nwalkers) # svals = 55422.0 + np.random.randn(nwalkers) elif param == "SINI": svals = np.random.uniform(0.0, 1.0, nwalkers) elif param == "M2": svals = np.random.uniform(0.1, 0.6, nwalkers) elif param in ["E", "ECC", "PX", "A1"]: # Ensure all positive svals = np.fabs(ftr.fitvals[idx] + ftr.fiterrs[idx] * np.random.randn(nwalkers)) if param in ["E", "ECC"]: svals[svals > 1.0] = 1.0 - (svals[svals > 1.0] - 1.0) for ii in range(nwalkers): pos[ii][idx] = svals[ii] else: pos = [ newfitvals + ftr.fiterrs * args.initerrfact * np.random.randn(ndim) for i in range(nwalkers) ] # Set the 0th walker to have the initial pre-fit solution # This way, one walker should always be in a good position pos[0] = ftr.fitvals import emcee # Following are for parallel processing tests... if 0: def unwrapped_lnpost(theta, ftr=ftr): return ftr.lnposterior(theta) import pathos.multiprocessing as mp pool = mp.ProcessPool(nodes=8) sampler = emcee.EnsembleSampler(nwalkers, ndim, unwrapped_lnpost, pool=pool, args=[ftr]) else: sampler = emcee.EnsembleSampler(nwalkers, ndim, ftr.lnposterior) # The number is the number of points in the chain sampler.run_mcmc(pos, nsteps) def chains_to_dict(names, sampler): chains = [sampler.chain[:, :, ii].T for ii in range(len(names))] return dict(zip(names, chains)) def plot_chains(chain_dict, file=False): npts = len(chain_dict) fig, axes = plt.subplots(npts, 1, sharex=True, figsize=(8, 9)) for ii, name in enumerate(chain_dict.keys()): axes[ii].plot(chain_dict[name], color="k", alpha=0.3) axes[ii].set_ylabel(name) axes[npts - 1].set_xlabel("Step Number") fig.tight_layout() if file: fig.savefig(file) plt.close() else: plt.show() plt.close() chains = chains_to_dict(ftr.fitkeys, sampler) plot_chains(chains, file=ftr.model.PSR.value + "_chains.png") # Make the triangle plot. samples = sampler.chain[:, burnin:, :].reshape((-1, ndim)) try: import corner fig = corner.corner( samples, labels=ftr.fitkeys, bins=50, truths=ftr.maxpost_fitvals, plot_contours=True, ) fig.savefig(ftr.model.PSR.value + "_triangle.png") plt.close() except ImportError: pass # Make a phaseogram with the 50th percentile values # ftr.set_params(dict(zip(ftr.fitkeys, np.percentile(samples, 50, axis=0)))) # Make a phaseogram with the best MCMC result ftr.set_params(dict(zip(ftr.fitkeys[:-1], ftr.maxpost_fitvals[:-1]))) ftr.phaseogram(plotfile=ftr.model.PSR.value + "_post.png") plt.close() # Write out the output pulse profile vs, xs = np.histogram(ftr.get_event_phases(), outprof_nbins, range=[0, 1], weights=ftr.weights) f = open(ftr.model.PSR.value + "_prof_post.txt", "w") for x, v in zip(xs, vs): f.write("%.5f %12.5f\n" % (x, v)) f.close() # Write out the par file for the best MCMC parameter est f = open(ftr.model.PSR.value + "_post.par", "w") f.write(ftr.model.as_parfile()) f.close() # Print the best MCMC values and ranges ranges = map( lambda v: (v[1], v[2] - v[1], v[1] - v[0]), zip(*np.percentile(samples, [16, 50, 84], axis=0)), ) log.info("Post-MCMC values (50th percentile +/- (16th/84th percentile):") for name, vals in zip(ftr.fitkeys, ranges): log.info("%8s:" % name + "%25.15g (+ %12.5g / - %12.5g)" % vals) # Put the same stuff in a file f = open(ftr.model.PSR.value + "_results.txt", "w") f.write("Post-MCMC values (50th percentile +/- (16th/84th percentile):\n") for name, vals in zip(ftr.fitkeys, ranges): f.write("%8s:" % name + " %25.15g (+ %12.5g / - %12.5g)\n" % vals) f.write("\nMaximum likelihood par file:\n") f.write(ftr.model.as_parfile()) f.close() from six.moves import cPickle as pickle pickle.dump(samples, open(ftr.model.PSR.value + "_samples.pickle", "wb"))
# Now load in the gaussian template and normalize it gtemplate = pu.read_gaussfitfile(gaussianfile, nbins) gtemplate /= gtemplate.sum() # Set the priors on the parameters in the model, before # instantiating the emcee_fitter # Currently, this adds a gaussian prior on each parameter # with width equal to the par file uncertainty * priorerrfact, # and then puts in some special cases. # *** This should be replaced/supplemented with a way to specify # more general priors on parameters that need certain bounds fitkeys, fitvals, fiterrs = get_fit_keyvals(modelin) for key, v, e in zip(fitkeys, fitvals, fiterrs): if key == 'SINI' or key == 'E' or key == 'ECC': getattr(modelin, key).prior = Prior(UniformBoundedRV(0.0, 1.0)) elif key == 'PX': getattr(modelin, key).prior = Prior(UniformBoundedRV(0.0, 10.0)) elif key.startswith('GLPH'): getattr(modelin, key).prior = Prior(UniformBoundedRV(-0.5, 0.5)) else: getattr(modelin, key).prior = Prior( norm(loc=float(v), scale=float(e * args.priorerrfact))) # Now define the requirements for emcee ftr = emcee_fitter(ts, modelin, gtemplate, weights) # Use this if you want to see the effect of setting minWeight if args.testWeights: log.info("Checking H-test vs weights") ftr.prof_vs_weights(use_weights=True)