def main(mpi=False):
if not os.path.exists("data"):
raise IOError("This must be run from the top level of the project.")
np.random.seed(42)
pool = get_pool(mpi=mpi)
if not os.path.exists("output"):
os.mkdir("output")
# read the data
tbl = ascii.read("data/xue08_bhb.txt")
# set up the cache file
cache_file = os.path.join("output", "samples.npy")
cache_dtype = np.float64
cache_shape = (len(tbl), cache_nsteps*nwalkers, 3)
if not os.path.exists(cache_file):
# make sure file exists
cache = np.memmap(cache_file, dtype=cache_dtype, mode='w+', shape=cache_shape)
cache = np.memmap(cache_file, dtype=cache_dtype, mode='r+', shape=cache_shape)
# coordinate object and error in distance
c = coord.ICRS(ra=tbl['RAdeg'], dec=tbl['DEdeg'], distance=tbl['d'])
err_dist = u.Quantity(tbl['d']*0.1)
# heliocentric line-of-sight velocity
obs_vlos = u.Quantity(tbl['HRVel']).to(u.km/u.s).value
err_vlos = u.Quantity(tbl['e_HRVel']).to(u.km/u.s).value
sampler = None
nstars = len(c)
for i in range(nstars):
logger.info("Star {}".format(i))
if sampler is not None:
sampler.reset()
if np.any(cache[i] != 0.):
logger.debug("Skipping -- data already cached...")
continue
logger.debug("Running MCMC sampling...")
args = (c[i],obs_vlos[i],err_vlos[i])
sampler = emcee.EnsembleSampler(nwalkers=nwalkers, dim=3, args=args,
lnpostfn=ln_posterior, pool=pool)
p0 = np.zeros((nwalkers,3))
p0[:,0] = np.random.normal(0., 0.1, size=nwalkers) # mas/yr
p0[:,1] = np.random.normal(0., 0.1, size=nwalkers) # mas/yr
p0[:,2] = np.random.normal(obs_vlos[i], err_vlos[i]/10., size=nwalkers) # km/s
_ = sampler.run_mcmc(p0, nmcmc_steps)
cache[i] = np.vstack(sampler.chain[:,-16:])
cache.flush()
def main(mpi=False):
pool = get_pool(mpi=mpi)
# regular orbit, so variance should be small, but appears large due to aliasing...
tasks = []
for p in [1,2,3,4]:
for window_width in 2**np.arange(5,8+1,1):
for nsteps_per_period in 2**np.arange(8, 14+1, 2):
tasks.append((p,window_width,nsteps_per_period))
pool.map(worker, tasks)
pool.close()
sys.exit(0)
def main(path, mpi=False, threshold_diffusion=1E-6):
np.random.seed(42)
# get a pool object for multiprocessing / MPI
pool = get_pool(mpi=mpi)
if mpi:
logger.info("Using MPI")
allfreqs_filename = os.path.join(path, "allfreqs.dat")
w0_filename = os.path.join(path, 'w0.npy')
w0 = np.load(w0_filename)
norbits = len(w0)
logger.info("Number of orbits: {}".format(norbits))
if not os.path.exists(allfreqs_filename):
raise IOError("allfreqs file doesn't exist!")
d = read_allfreqs(allfreqs_filename, norbits)
# containers
nperiods = np.zeros(norbits)
freq_diff = np.zeros(norbits)
# loop orbits
loop = d[d['loop']]
loopf = loop['fRphiz']
nperiods[d['loop']] = loop['dt']*loop['nsteps'] / (2*np.pi/np.abs(loopf[:,0]).max(axis=-1))
freq_diff[d['loop']] = np.abs((loopf[:,1] - loopf[:,0]) / loopf[:,0]).max(axis=1) / nperiods[d['loop']] / 2.
# box orbits
box = d[~d['loop']]
boxf = box['fxyz']
nperiods[~d['loop']] = box['dt']*box['nsteps'] / (2*np.pi/np.abs(boxf[:,0]).max(axis=-1))
freq_diff[~d['loop']] = np.abs((boxf[:,1] - boxf[:,0]) / boxf[:,0]).max(axis=1) / nperiods[~d['loop']] / 2.
redo_ix = (np.logical_not(d['success']) | np.logical_not(np.isfinite(nperiods)) |
(freq_diff > threshold_diffusion) | np.logical_not(np.isfinite(freq_diff)))
not_done = np.where(redo_ix)[0]
tasks = [dict(index=i, w0_filename=w0_filename,
allfreqs_filename=allfreqs_filename,
potential=potential, dt=1., nsteps=400000) for i in not_done]
pool.map(worker, tasks)
pool.close()
def run(self, **kwargs):
args = self._parse_args()
for k,v in kwargs.items():
if hasattr(args, k):
# overwrite with kwarg value
setattr(args, k, v)
np.random.seed(args.seed)
if args.config_filename is None:
raise ValueError("You must define 'config_filename.'")
# Set logger level based on verbose flags
if args.verbose:
logger.setLevel(logging.DEBUG)
elif args.quiet:
logger.setLevel(logging.ERROR)
else:
logger.setLevel(logging.INFO)
# if MPI, use load balancing
if args.mpi:
kwargs = dict(loadbalance=True)
else:
kwargs = dict()
# get a pool object for multiprocessing / MPI
pool = get_pool(mpi=args.mpi, **kwargs)
if args.mpi:
logger.info("|----------- Using MPI -----------|")
else:
logger.info("|----------- Running in serial -----------|")
if args.index is None:
index = None
else:
try:
index = slice(*map(int, args.index.split(":")))
except:
try:
index = np.array(map(int,args.index.split(",")))
except:
index = None
# Instantiate the experiment class
with self.ExperimentClass.from_config(cache_path=args.path,
config_filename=args.config_filename,
overwrite=args.overwrite) as experiment:
experiment._ensure_cache_exists()
if index is None:
indices = np.arange(experiment.norbits, dtype=int)
else:
indices = np.arange(experiment.norbits, dtype=int)[index]
try:
pool.map(experiment, indices, callback=experiment.callback)
except:
pool.close()
logger.error("Unexpected error!")
raise
else:
pool.close()
dest="_continue",
default=False,
help="Continue the mcmc")
args = parser.parse_args()
# Set logger level based on verbose flags
if args.verbose:
logger.setLevel(logging.DEBUG)
elif args.quiet:
logger.setLevel(logging.ERROR)
else:
logger.setLevel(logging.INFO)
np.random.seed(args.seed)
pool = get_pool(mpi=args.mpi)
if args._continue:
continue_sampling(true_potential_name=args.true_potential_name,
fit_potential_name=args.fit_potential_name,
index=args.index,
pool=pool,
n_iterations=args.mcmc_steps,
frac_distance_err=args.frac_distance_err,
name=args.name)
sys.exit(0)
main(true_potential_name=args.true_potential_name,
fit_potential_name=args.fit_potential_name,
n_stars=args.n_stars,
n_burn=args.mcmc_burn,
def main(name, nstars, nburn, nwalk, mpi=False, save=False):
pool = get_pool(mpi=mpi)
np.random.seed(42)
# parameters
true_sat_mass = 2.5E5
nstars = 8
dt = -0.1
# read in the SCF simulation data
s = scf.SCFReader(os.path.join(scfpath, "simulations/runs/spherical/"))
tbl = s.last_snap(units=galactic)
total_time = tbl.meta['time']
nsteps = abs(int(total_time/dt))
stream_tbl = tbl[(tbl["tub"] != 0)]
prog_w = np.median(scf.tbl_to_w(tbl[(tbl["tub"] == 0)]), axis=0)
# pluck out a certain number of stars...
ixs = []
while len(ixs) < nstars:
ixs = np.unique(np.random.randint(len(stream_tbl), size=nstars))
data_w = scf.tbl_to_w(stream_tbl[ixs])
prog_E = true_potential.total_energy(prog_w[:3], prog_w[3:])
dE = (true_potential.total_energy(data_w[:,:3], data_w[:,3:]) - prog_E) / prog_E
betas = -2*(dE > 0).astype(int) + 1.
# test true potential
# vals = np.linspace(0.1, 3.5, 32)
# lls = []
# for val in vals:
# p = [val, true_params['v_c'], np.log(20.)]
# ll = ln_posterior_nfw(p, dt, nsteps, prog_w, data_w, betas, true_sat_mass)
# lls.append(ll)
# plt.figure()
# plt.plot(vals, lls)
# plt.show()
# return
# ----------------------------
# Emcee
if name == 'nfw':
# NFW
print("Firing up sampler for NFW")
ndim = 3
nwalkers = 128*ndim
sampler = kombine.Sampler(nwalkers=nwalkers, ndim=ndim, lnpostfn=ln_posterior_nfw,
args=(dt, nsteps, prog_w, data_w, betas, true_sat_mass),
pool=pool)
p0 = np.zeros((nwalkers,ndim))
p0[:,0] = np.random.normal(1., 0.05, size=nwalkers) # alpha
p0[:,1] = np.random.normal(0.2, 0.02, size=nwalkers) # v_c
p0[:,2] = np.random.normal(3., 0.1, size=nwalkers) # log_r_s
sampler = sample_dat_ish(sampler, p0, nburn=nburn, nwalk=nwalk)
pool.close()
if save:
np.save("/vega/astro/users/amp2217/projects/streams/output/michigan_hack/nfw_chain.npy", sampler.chain)
np.save("/vega/astro/users/amp2217/projects/streams/output/michigan_hack/nfw_lnprob.npy", sampler.lnprobability)
elif name == "hernq":
# Hernquist
print("Firing up sampler for Hernquist")
ndim = 3
nwalkers = 128*ndim
sampler = kombine.Sampler(nwalkers=nwalkers, ndim=ndim, lnpostfn=ln_posterior_hernq,
args=(dt, nsteps, prog_w, data_w, betas, true_sat_mass),
pool=pool)
p0 = np.zeros((nwalkers,ndim))
p0[:,0] = np.random.normal(1., 0.05, size=nwalkers) # alpha
p0[:,1] = np.random.normal(26., 0.15, size=nwalkers) # log_m
p0[:,2] = np.random.normal(20., 0.5, size=nwalkers) # c
sampler = sample_dat_ish(sampler, p0, nburn=nburn, nwalk=nwalk)
pool.close()
if save:
np.save("/vega/astro/users/amp2217/projects/streams/output/michigan_hack/hernq_chain.npy", sampler.chain)
np.save("/vega/astro/users/amp2217/projects/streams/output/michigan_hack/hernq_lnprob.npy", sampler.lnprobability)
elif name == "bfe":
# BFE
print("Firing up sampler for BFE")
ndim = 7
nwalkers = 128*ndim
sampler = kombine.Sampler(nwalkers=nwalkers, ndim=ndim, lnpostfn=ln_posterior_bfe,
args=(dt, nsteps, prog_w, data_w, betas, true_sat_mass),
pool=pool)
p0 = np.zeros((nwalkers,ndim))
p0[:,0] = np.random.normal(1., 0.05, size=nwalkers) # alpha
p0[:,1] = np.random.normal(26., 0.15, size=nwalkers) # log_m
p0[:,2] = np.random.normal(20., 0.5, size=nwalkers) # c
p0[:,3] = np.random.uniform(0.9, 1.1, size=nwalkers) # c1
p0[:,4] = np.random.uniform(0., 0.1, size=nwalkers) # c2
p0[:,5] = np.random.uniform(0., 0.05, size=nwalkers) # c3
p0[:,6] = np.random.uniform(0., 0.02, size=nwalkers) # c4
# p0[:,7] = np.random.uniform(-0., -0.02, size=nwalkers) # c5
# p0[:,8] = np.random.uniform(-0., -0.02, size=nwalkers) # c6
sampler = sample_dat_ish(sampler, p0, nburn=nburn, nwalk=nwalk)
pool.close()
if save:
np.save("/vega/astro/users/amp2217/projects/streams/output/michigan_hack/bfe_chain.npy", sampler.chain)
np.save("/vega/astro/users/amp2217/projects/streams/output/michigan_hack/bfe_lnprob.npy", sampler.lnprobability)
sys.exit(0)
def main(data_file, potential_name, mpi=False, n_walkers=None, n_iterations=None,
overwrite=False):
np.random.seed(42)
pool = get_pool(mpi=mpi)
# load data file, uncertainties, and potential
data_file = os.path.abspath(data_file)
with open(data_file, "rb") as f:
data,err,R = pickle.load(f) # R is rotation matrix from Galactic to stream coords
potential = potentials[potential_name]
freeze = dict()
# these estimated from the plots
freeze['phi2_sigma'] = np.radians(0.5)
freeze['d_sigma'] = 1.5
freeze['vr_sigma'] = (1*u.km/u.s).decompose(galactic).value
freeze['t_forw'] = 0.
if potential_name == 'spherical':
freeze['t_back'] = -600. # HACK: figured out at bottom of notebook
potential_freeze_params = ['r_h', 'q1', 'q2', 'q3', 'phi']
elif potential_name == 'triaxial':
freeze['t_back'] = -600 # HACK: figured out at bottom of notebook
potential_freeze_params = ['r_h', 'q1', 'phi']
for k in potential_freeze_params:
logger.debug("freezing potential:{}".format(k))
freeze['potential_{}'.format(k)] = potential.parameters[k].value
pot_guess = []
for k in potential.parameters.keys():
if k in potential_freeze_params: continue
logger.debug("varying potential:{}".format(k))
pot_guess += [potential.parameters[k].value]
idx = data['phi1'].argmin()
p0_guess = [data['phi2'].radian[idx],
data['distance'].decompose(galactic).value[idx],
data['mul'].decompose(galactic).value[idx],
data['mub'].decompose(galactic).value[idx],
data['vr'].decompose(galactic).value[idx]]
p0_guess = p0_guess + pot_guess
logger.debug("Initial guess: {}".format(p0_guess))
# first, optimize to get a good guess to initialize MCMC
args = (data, err, R, potential.__class__, 0.5, freeze)
logger.info("optimizing ln_posterior...")
res = so.minimize(lambda *args,**kwargs: -ln_posterior(*args, **kwargs),
x0=p0_guess, method='powell', args=args)
logger.info("finished optimizing")
logger.debug("optimization returned: {}".format(res))
if not res.success:
pool.close()
raise ValueError("Failed to optimize!")
# now, create initial conditions for MCMC walkers in a small ball around the
# optimized parameter vector
if n_walkers is None:
n_walkers = 8*len(p0_guess)
sampler = emcee.EnsembleSampler(nwalkers=n_walkers, dim=len(p0_guess), lnpostfn=ln_posterior,
args=args, pool=pool)
mcmc_p0 = emcee.utils.sample_ball(res.x, 1E-3*np.array(p0_guess), size=n_walkers)
if n_iterations is None:
n_iterations = 1024
logger.info("running mcmc sampler with {} walkers for {} steps".format(n_walkers, n_iterations))
_ = sampler.run_mcmc(mcmc_p0, N=n_iterations)
logger.info("finished sampling")
pool.close()
# same sampler to pickle file
data_file_basename = os.path.splitext(os.path.basename(data_file))[0]
sampler_path = os.path.join("results", "{}_sampler.pickle".format(data_file_basename))
if not os.path.exists("results"):
os.mkdir("results")
if os.path.exists(sampler_path) and overwrite:
os.remove(sampler_path)
sampler.lnprobfn = None
sampler.pool = None
logger.debug("saving emcee sampler to: {}".format(sampler_path))
with open(sampler_path, 'wb') as f:
pickle.dump(sampler, f)
sys.exit(0)
def main(name, nstars, nburn, nwalk, mpi=False, save=False):
pool = get_pool(mpi=mpi)
np.random.seed(42)
# parameters
true_sat_mass = 2.5E5
nstars = 8
dt = -0.1
# read in the SCF simulation data
s = scf.SCFReader(os.path.join(scfpath, "simulations/runs/spherical/"))
tbl = s.last_snap(units=galactic)
total_time = tbl.meta['time']
nsteps = abs(int(total_time / dt))
stream_tbl = tbl[(tbl["tub"] != 0)]
prog_w = np.median(scf.tbl_to_w(tbl[(tbl["tub"] == 0)]), axis=0)
# pluck out a certain number of stars...
ixs = []
while len(ixs) < nstars:
ixs = np.unique(np.random.randint(len(stream_tbl), size=nstars))
data_w = scf.tbl_to_w(stream_tbl[ixs])
prog_E = true_potential.total_energy(prog_w[:3], prog_w[3:])
dE = (true_potential.total_energy(data_w[:, :3], data_w[:, 3:]) -
prog_E) / prog_E
betas = -2 * (dE > 0).astype(int) + 1.
# test true potential
# vals = np.linspace(0.1, 3.5, 32)
# lls = []
# for val in vals:
# p = [val, true_params['v_c'], np.log(20.)]
# ll = ln_posterior_nfw(p, dt, nsteps, prog_w, data_w, betas, true_sat_mass)
# lls.append(ll)
# plt.figure()
# plt.plot(vals, lls)
# plt.show()
# return
# ----------------------------
# Emcee
if name == 'nfw':
# NFW
print("Firing up sampler for NFW")
ndim = 3
nwalkers = 128 * ndim
sampler = kombine.Sampler(nwalkers=nwalkers,
ndim=ndim,
lnpostfn=ln_posterior_nfw,
args=(dt, nsteps, prog_w, data_w, betas,
true_sat_mass),
pool=pool)
p0 = np.zeros((nwalkers, ndim))
p0[:, 0] = np.random.normal(1., 0.05, size=nwalkers) # alpha
p0[:, 1] = np.random.normal(0.2, 0.02, size=nwalkers) # v_c
p0[:, 2] = np.random.normal(3., 0.1, size=nwalkers) # log_r_s
sampler = sample_dat_ish(sampler, p0, nburn=nburn, nwalk=nwalk)
pool.close()
if save:
np.save(
"/vega/astro/users/amp2217/projects/streams/output/michigan_hack/nfw_chain.npy",
sampler.chain)
np.save(
"/vega/astro/users/amp2217/projects/streams/output/michigan_hack/nfw_lnprob.npy",
sampler.lnprobability)
elif name == "hernq":
# Hernquist
print("Firing up sampler for Hernquist")
ndim = 3
nwalkers = 128 * ndim
sampler = kombine.Sampler(nwalkers=nwalkers,
ndim=ndim,
lnpostfn=ln_posterior_hernq,
args=(dt, nsteps, prog_w, data_w, betas,
true_sat_mass),
pool=pool)
p0 = np.zeros((nwalkers, ndim))
p0[:, 0] = np.random.normal(1., 0.05, size=nwalkers) # alpha
p0[:, 1] = np.random.normal(26., 0.15, size=nwalkers) # log_m
p0[:, 2] = np.random.normal(20., 0.5, size=nwalkers) # c
sampler = sample_dat_ish(sampler, p0, nburn=nburn, nwalk=nwalk)
pool.close()
if save:
np.save(
"/vega/astro/users/amp2217/projects/streams/output/michigan_hack/hernq_chain.npy",
sampler.chain)
np.save(
"/vega/astro/users/amp2217/projects/streams/output/michigan_hack/hernq_lnprob.npy",
sampler.lnprobability)
elif name == "bfe":
# BFE
print("Firing up sampler for BFE")
ndim = 7
nwalkers = 128 * ndim
sampler = kombine.Sampler(nwalkers=nwalkers,
ndim=ndim,
lnpostfn=ln_posterior_bfe,
args=(dt, nsteps, prog_w, data_w, betas,
true_sat_mass),
pool=pool)
p0 = np.zeros((nwalkers, ndim))
p0[:, 0] = np.random.normal(1., 0.05, size=nwalkers) # alpha
p0[:, 1] = np.random.normal(26., 0.15, size=nwalkers) # log_m
p0[:, 2] = np.random.normal(20., 0.5, size=nwalkers) # c
p0[:, 3] = np.random.uniform(0.9, 1.1, size=nwalkers) # c1
p0[:, 4] = np.random.uniform(0., 0.1, size=nwalkers) # c2
p0[:, 5] = np.random.uniform(0., 0.05, size=nwalkers) # c3
p0[:, 6] = np.random.uniform(0., 0.02, size=nwalkers) # c4
# p0[:,7] = np.random.uniform(-0., -0.02, size=nwalkers) # c5
# p0[:,8] = np.random.uniform(-0., -0.02, size=nwalkers) # c6
sampler = sample_dat_ish(sampler, p0, nburn=nburn, nwalk=nwalk)
pool.close()
if save:
np.save(
"/vega/astro/users/amp2217/projects/streams/output/michigan_hack/bfe_chain.npy",
sampler.chain)
np.save(
"/vega/astro/users/amp2217/projects/streams/output/michigan_hack/bfe_lnprob.npy",
sampler.lnprobability)
sys.exit(0)
