def main(mpi=False): if not os.path.exists("data"): raise IOError("This must be run from the top level of the project.") np.random.seed(42) pool = get_pool(mpi=mpi) if not os.path.exists("output"): os.mkdir("output") # read the data tbl = ascii.read("data/xue08_bhb.txt") # set up the cache file cache_file = os.path.join("output", "samples.npy") cache_dtype = np.float64 cache_shape = (len(tbl), cache_nsteps*nwalkers, 3) if not os.path.exists(cache_file): # make sure file exists cache = np.memmap(cache_file, dtype=cache_dtype, mode='w+', shape=cache_shape) cache = np.memmap(cache_file, dtype=cache_dtype, mode='r+', shape=cache_shape) # coordinate object and error in distance c = coord.ICRS(ra=tbl['RAdeg'], dec=tbl['DEdeg'], distance=tbl['d']) err_dist = u.Quantity(tbl['d']*0.1) # heliocentric line-of-sight velocity obs_vlos = u.Quantity(tbl['HRVel']).to(u.km/u.s).value err_vlos = u.Quantity(tbl['e_HRVel']).to(u.km/u.s).value sampler = None nstars = len(c) for i in range(nstars): logger.info("Star {}".format(i)) if sampler is not None: sampler.reset() if np.any(cache[i] != 0.): logger.debug("Skipping -- data already cached...") continue logger.debug("Running MCMC sampling...") args = (c[i],obs_vlos[i],err_vlos[i]) sampler = emcee.EnsembleSampler(nwalkers=nwalkers, dim=3, args=args, lnpostfn=ln_posterior, pool=pool) p0 = np.zeros((nwalkers,3)) p0[:,0] = np.random.normal(0., 0.1, size=nwalkers) # mas/yr p0[:,1] = np.random.normal(0., 0.1, size=nwalkers) # mas/yr p0[:,2] = np.random.normal(obs_vlos[i], err_vlos[i]/10., size=nwalkers) # km/s _ = sampler.run_mcmc(p0, nmcmc_steps) cache[i] = np.vstack(sampler.chain[:,-16:]) cache.flush()
def main(mpi=False): pool = get_pool(mpi=mpi) # regular orbit, so variance should be small, but appears large due to aliasing... tasks = [] for p in [1,2,3,4]: for window_width in 2**np.arange(5,8+1,1): for nsteps_per_period in 2**np.arange(8, 14+1, 2): tasks.append((p,window_width,nsteps_per_period)) pool.map(worker, tasks) pool.close() sys.exit(0)
def main(path, mpi=False, threshold_diffusion=1E-6): np.random.seed(42) # get a pool object for multiprocessing / MPI pool = get_pool(mpi=mpi) if mpi: logger.info("Using MPI") allfreqs_filename = os.path.join(path, "allfreqs.dat") w0_filename = os.path.join(path, 'w0.npy') w0 = np.load(w0_filename) norbits = len(w0) logger.info("Number of orbits: {}".format(norbits)) if not os.path.exists(allfreqs_filename): raise IOError("allfreqs file doesn't exist!") d = read_allfreqs(allfreqs_filename, norbits) # containers nperiods = np.zeros(norbits) freq_diff = np.zeros(norbits) # loop orbits loop = d[d['loop']] loopf = loop['fRphiz'] nperiods[d['loop']] = loop['dt']*loop['nsteps'] / (2*np.pi/np.abs(loopf[:,0]).max(axis=-1)) freq_diff[d['loop']] = np.abs((loopf[:,1] - loopf[:,0]) / loopf[:,0]).max(axis=1) / nperiods[d['loop']] / 2. # box orbits box = d[~d['loop']] boxf = box['fxyz'] nperiods[~d['loop']] = box['dt']*box['nsteps'] / (2*np.pi/np.abs(boxf[:,0]).max(axis=-1)) freq_diff[~d['loop']] = np.abs((boxf[:,1] - boxf[:,0]) / boxf[:,0]).max(axis=1) / nperiods[~d['loop']] / 2. redo_ix = (np.logical_not(d['success']) | np.logical_not(np.isfinite(nperiods)) | (freq_diff > threshold_diffusion) | np.logical_not(np.isfinite(freq_diff))) not_done = np.where(redo_ix)[0] tasks = [dict(index=i, w0_filename=w0_filename, allfreqs_filename=allfreqs_filename, potential=potential, dt=1., nsteps=400000) for i in not_done] pool.map(worker, tasks) pool.close()
def run(self, **kwargs): args = self._parse_args() for k,v in kwargs.items(): if hasattr(args, k): # overwrite with kwarg value setattr(args, k, v) np.random.seed(args.seed) if args.config_filename is None: raise ValueError("You must define 'config_filename.'") # Set logger level based on verbose flags if args.verbose: logger.setLevel(logging.DEBUG) elif args.quiet: logger.setLevel(logging.ERROR) else: logger.setLevel(logging.INFO) # if MPI, use load balancing if args.mpi: kwargs = dict(loadbalance=True) else: kwargs = dict() # get a pool object for multiprocessing / MPI pool = get_pool(mpi=args.mpi, **kwargs) if args.mpi: logger.info("|----------- Using MPI -----------|") else: logger.info("|----------- Running in serial -----------|") if args.index is None: index = None else: try: index = slice(*map(int, args.index.split(":"))) except: try: index = np.array(map(int,args.index.split(","))) except: index = None # Instantiate the experiment class with self.ExperimentClass.from_config(cache_path=args.path, config_filename=args.config_filename, overwrite=args.overwrite) as experiment: experiment._ensure_cache_exists() if index is None: indices = np.arange(experiment.norbits, dtype=int) else: indices = np.arange(experiment.norbits, dtype=int)[index] try: pool.map(experiment, indices, callback=experiment.callback) except: pool.close() logger.error("Unexpected error!") raise else: pool.close()
dest="_continue", default=False, help="Continue the mcmc") args = parser.parse_args() # Set logger level based on verbose flags if args.verbose: logger.setLevel(logging.DEBUG) elif args.quiet: logger.setLevel(logging.ERROR) else: logger.setLevel(logging.INFO) np.random.seed(args.seed) pool = get_pool(mpi=args.mpi) if args._continue: continue_sampling(true_potential_name=args.true_potential_name, fit_potential_name=args.fit_potential_name, index=args.index, pool=pool, n_iterations=args.mcmc_steps, frac_distance_err=args.frac_distance_err, name=args.name) sys.exit(0) main(true_potential_name=args.true_potential_name, fit_potential_name=args.fit_potential_name, n_stars=args.n_stars, n_burn=args.mcmc_burn,
def main(name, nstars, nburn, nwalk, mpi=False, save=False): pool = get_pool(mpi=mpi) np.random.seed(42) # parameters true_sat_mass = 2.5E5 nstars = 8 dt = -0.1 # read in the SCF simulation data s = scf.SCFReader(os.path.join(scfpath, "simulations/runs/spherical/")) tbl = s.last_snap(units=galactic) total_time = tbl.meta['time'] nsteps = abs(int(total_time/dt)) stream_tbl = tbl[(tbl["tub"] != 0)] prog_w = np.median(scf.tbl_to_w(tbl[(tbl["tub"] == 0)]), axis=0) # pluck out a certain number of stars... ixs = [] while len(ixs) < nstars: ixs = np.unique(np.random.randint(len(stream_tbl), size=nstars)) data_w = scf.tbl_to_w(stream_tbl[ixs]) prog_E = true_potential.total_energy(prog_w[:3], prog_w[3:]) dE = (true_potential.total_energy(data_w[:,:3], data_w[:,3:]) - prog_E) / prog_E betas = -2*(dE > 0).astype(int) + 1. # test true potential # vals = np.linspace(0.1, 3.5, 32) # lls = [] # for val in vals: # p = [val, true_params['v_c'], np.log(20.)] # ll = ln_posterior_nfw(p, dt, nsteps, prog_w, data_w, betas, true_sat_mass) # lls.append(ll) # plt.figure() # plt.plot(vals, lls) # plt.show() # return # ---------------------------- # Emcee if name == 'nfw': # NFW print("Firing up sampler for NFW") ndim = 3 nwalkers = 128*ndim sampler = kombine.Sampler(nwalkers=nwalkers, ndim=ndim, lnpostfn=ln_posterior_nfw, args=(dt, nsteps, prog_w, data_w, betas, true_sat_mass), pool=pool) p0 = np.zeros((nwalkers,ndim)) p0[:,0] = np.random.normal(1., 0.05, size=nwalkers) # alpha p0[:,1] = np.random.normal(0.2, 0.02, size=nwalkers) # v_c p0[:,2] = np.random.normal(3., 0.1, size=nwalkers) # log_r_s sampler = sample_dat_ish(sampler, p0, nburn=nburn, nwalk=nwalk) pool.close() if save: np.save("/vega/astro/users/amp2217/projects/streams/output/michigan_hack/nfw_chain.npy", sampler.chain) np.save("/vega/astro/users/amp2217/projects/streams/output/michigan_hack/nfw_lnprob.npy", sampler.lnprobability) elif name == "hernq": # Hernquist print("Firing up sampler for Hernquist") ndim = 3 nwalkers = 128*ndim sampler = kombine.Sampler(nwalkers=nwalkers, ndim=ndim, lnpostfn=ln_posterior_hernq, args=(dt, nsteps, prog_w, data_w, betas, true_sat_mass), pool=pool) p0 = np.zeros((nwalkers,ndim)) p0[:,0] = np.random.normal(1., 0.05, size=nwalkers) # alpha p0[:,1] = np.random.normal(26., 0.15, size=nwalkers) # log_m p0[:,2] = np.random.normal(20., 0.5, size=nwalkers) # c sampler = sample_dat_ish(sampler, p0, nburn=nburn, nwalk=nwalk) pool.close() if save: np.save("/vega/astro/users/amp2217/projects/streams/output/michigan_hack/hernq_chain.npy", sampler.chain) np.save("/vega/astro/users/amp2217/projects/streams/output/michigan_hack/hernq_lnprob.npy", sampler.lnprobability) elif name == "bfe": # BFE print("Firing up sampler for BFE") ndim = 7 nwalkers = 128*ndim sampler = kombine.Sampler(nwalkers=nwalkers, ndim=ndim, lnpostfn=ln_posterior_bfe, args=(dt, nsteps, prog_w, data_w, betas, true_sat_mass), pool=pool) p0 = np.zeros((nwalkers,ndim)) p0[:,0] = np.random.normal(1., 0.05, size=nwalkers) # alpha p0[:,1] = np.random.normal(26., 0.15, size=nwalkers) # log_m p0[:,2] = np.random.normal(20., 0.5, size=nwalkers) # c p0[:,3] = np.random.uniform(0.9, 1.1, size=nwalkers) # c1 p0[:,4] = np.random.uniform(0., 0.1, size=nwalkers) # c2 p0[:,5] = np.random.uniform(0., 0.05, size=nwalkers) # c3 p0[:,6] = np.random.uniform(0., 0.02, size=nwalkers) # c4 # p0[:,7] = np.random.uniform(-0., -0.02, size=nwalkers) # c5 # p0[:,8] = np.random.uniform(-0., -0.02, size=nwalkers) # c6 sampler = sample_dat_ish(sampler, p0, nburn=nburn, nwalk=nwalk) pool.close() if save: np.save("/vega/astro/users/amp2217/projects/streams/output/michigan_hack/bfe_chain.npy", sampler.chain) np.save("/vega/astro/users/amp2217/projects/streams/output/michigan_hack/bfe_lnprob.npy", sampler.lnprobability) sys.exit(0)
def main(data_file, potential_name, mpi=False, n_walkers=None, n_iterations=None, overwrite=False): np.random.seed(42) pool = get_pool(mpi=mpi) # load data file, uncertainties, and potential data_file = os.path.abspath(data_file) with open(data_file, "rb") as f: data,err,R = pickle.load(f) # R is rotation matrix from Galactic to stream coords potential = potentials[potential_name] freeze = dict() # these estimated from the plots freeze['phi2_sigma'] = np.radians(0.5) freeze['d_sigma'] = 1.5 freeze['vr_sigma'] = (1*u.km/u.s).decompose(galactic).value freeze['t_forw'] = 0. if potential_name == 'spherical': freeze['t_back'] = -600. # HACK: figured out at bottom of notebook potential_freeze_params = ['r_h', 'q1', 'q2', 'q3', 'phi'] elif potential_name == 'triaxial': freeze['t_back'] = -600 # HACK: figured out at bottom of notebook potential_freeze_params = ['r_h', 'q1', 'phi'] for k in potential_freeze_params: logger.debug("freezing potential:{}".format(k)) freeze['potential_{}'.format(k)] = potential.parameters[k].value pot_guess = [] for k in potential.parameters.keys(): if k in potential_freeze_params: continue logger.debug("varying potential:{}".format(k)) pot_guess += [potential.parameters[k].value] idx = data['phi1'].argmin() p0_guess = [data['phi2'].radian[idx], data['distance'].decompose(galactic).value[idx], data['mul'].decompose(galactic).value[idx], data['mub'].decompose(galactic).value[idx], data['vr'].decompose(galactic).value[idx]] p0_guess = p0_guess + pot_guess logger.debug("Initial guess: {}".format(p0_guess)) # first, optimize to get a good guess to initialize MCMC args = (data, err, R, potential.__class__, 0.5, freeze) logger.info("optimizing ln_posterior...") res = so.minimize(lambda *args,**kwargs: -ln_posterior(*args, **kwargs), x0=p0_guess, method='powell', args=args) logger.info("finished optimizing") logger.debug("optimization returned: {}".format(res)) if not res.success: pool.close() raise ValueError("Failed to optimize!") # now, create initial conditions for MCMC walkers in a small ball around the # optimized parameter vector if n_walkers is None: n_walkers = 8*len(p0_guess) sampler = emcee.EnsembleSampler(nwalkers=n_walkers, dim=len(p0_guess), lnpostfn=ln_posterior, args=args, pool=pool) mcmc_p0 = emcee.utils.sample_ball(res.x, 1E-3*np.array(p0_guess), size=n_walkers) if n_iterations is None: n_iterations = 1024 logger.info("running mcmc sampler with {} walkers for {} steps".format(n_walkers, n_iterations)) _ = sampler.run_mcmc(mcmc_p0, N=n_iterations) logger.info("finished sampling") pool.close() # same sampler to pickle file data_file_basename = os.path.splitext(os.path.basename(data_file))[0] sampler_path = os.path.join("results", "{}_sampler.pickle".format(data_file_basename)) if not os.path.exists("results"): os.mkdir("results") if os.path.exists(sampler_path) and overwrite: os.remove(sampler_path) sampler.lnprobfn = None sampler.pool = None logger.debug("saving emcee sampler to: {}".format(sampler_path)) with open(sampler_path, 'wb') as f: pickle.dump(sampler, f) sys.exit(0)
def main(name, nstars, nburn, nwalk, mpi=False, save=False): pool = get_pool(mpi=mpi) np.random.seed(42) # parameters true_sat_mass = 2.5E5 nstars = 8 dt = -0.1 # read in the SCF simulation data s = scf.SCFReader(os.path.join(scfpath, "simulations/runs/spherical/")) tbl = s.last_snap(units=galactic) total_time = tbl.meta['time'] nsteps = abs(int(total_time / dt)) stream_tbl = tbl[(tbl["tub"] != 0)] prog_w = np.median(scf.tbl_to_w(tbl[(tbl["tub"] == 0)]), axis=0) # pluck out a certain number of stars... ixs = [] while len(ixs) < nstars: ixs = np.unique(np.random.randint(len(stream_tbl), size=nstars)) data_w = scf.tbl_to_w(stream_tbl[ixs]) prog_E = true_potential.total_energy(prog_w[:3], prog_w[3:]) dE = (true_potential.total_energy(data_w[:, :3], data_w[:, 3:]) - prog_E) / prog_E betas = -2 * (dE > 0).astype(int) + 1. # test true potential # vals = np.linspace(0.1, 3.5, 32) # lls = [] # for val in vals: # p = [val, true_params['v_c'], np.log(20.)] # ll = ln_posterior_nfw(p, dt, nsteps, prog_w, data_w, betas, true_sat_mass) # lls.append(ll) # plt.figure() # plt.plot(vals, lls) # plt.show() # return # ---------------------------- # Emcee if name == 'nfw': # NFW print("Firing up sampler for NFW") ndim = 3 nwalkers = 128 * ndim sampler = kombine.Sampler(nwalkers=nwalkers, ndim=ndim, lnpostfn=ln_posterior_nfw, args=(dt, nsteps, prog_w, data_w, betas, true_sat_mass), pool=pool) p0 = np.zeros((nwalkers, ndim)) p0[:, 0] = np.random.normal(1., 0.05, size=nwalkers) # alpha p0[:, 1] = np.random.normal(0.2, 0.02, size=nwalkers) # v_c p0[:, 2] = np.random.normal(3., 0.1, size=nwalkers) # log_r_s sampler = sample_dat_ish(sampler, p0, nburn=nburn, nwalk=nwalk) pool.close() if save: np.save( "/vega/astro/users/amp2217/projects/streams/output/michigan_hack/nfw_chain.npy", sampler.chain) np.save( "/vega/astro/users/amp2217/projects/streams/output/michigan_hack/nfw_lnprob.npy", sampler.lnprobability) elif name == "hernq": # Hernquist print("Firing up sampler for Hernquist") ndim = 3 nwalkers = 128 * ndim sampler = kombine.Sampler(nwalkers=nwalkers, ndim=ndim, lnpostfn=ln_posterior_hernq, args=(dt, nsteps, prog_w, data_w, betas, true_sat_mass), pool=pool) p0 = np.zeros((nwalkers, ndim)) p0[:, 0] = np.random.normal(1., 0.05, size=nwalkers) # alpha p0[:, 1] = np.random.normal(26., 0.15, size=nwalkers) # log_m p0[:, 2] = np.random.normal(20., 0.5, size=nwalkers) # c sampler = sample_dat_ish(sampler, p0, nburn=nburn, nwalk=nwalk) pool.close() if save: np.save( "/vega/astro/users/amp2217/projects/streams/output/michigan_hack/hernq_chain.npy", sampler.chain) np.save( "/vega/astro/users/amp2217/projects/streams/output/michigan_hack/hernq_lnprob.npy", sampler.lnprobability) elif name == "bfe": # BFE print("Firing up sampler for BFE") ndim = 7 nwalkers = 128 * ndim sampler = kombine.Sampler(nwalkers=nwalkers, ndim=ndim, lnpostfn=ln_posterior_bfe, args=(dt, nsteps, prog_w, data_w, betas, true_sat_mass), pool=pool) p0 = np.zeros((nwalkers, ndim)) p0[:, 0] = np.random.normal(1., 0.05, size=nwalkers) # alpha p0[:, 1] = np.random.normal(26., 0.15, size=nwalkers) # log_m p0[:, 2] = np.random.normal(20., 0.5, size=nwalkers) # c p0[:, 3] = np.random.uniform(0.9, 1.1, size=nwalkers) # c1 p0[:, 4] = np.random.uniform(0., 0.1, size=nwalkers) # c2 p0[:, 5] = np.random.uniform(0., 0.05, size=nwalkers) # c3 p0[:, 6] = np.random.uniform(0., 0.02, size=nwalkers) # c4 # p0[:,7] = np.random.uniform(-0., -0.02, size=nwalkers) # c5 # p0[:,8] = np.random.uniform(-0., -0.02, size=nwalkers) # c6 sampler = sample_dat_ish(sampler, p0, nburn=nburn, nwalk=nwalk) pool.close() if save: np.save( "/vega/astro/users/amp2217/projects/streams/output/michigan_hack/bfe_chain.npy", sampler.chain) np.save( "/vega/astro/users/amp2217/projects/streams/output/michigan_hack/bfe_lnprob.npy", sampler.lnprobability) sys.exit(0)