demonstrations = utils.rollout_from_usa(10, 100, u_sa_A, mdp_env_A) print("demonstration") print(demonstrations) #Now let's run Bayesian IRL on this demo in this mdp with a placeholder feature to see what happens. beta = 100.0 step_stdev = 0.1 burn = 100 skip = 5 num_samples = 2000 sample_norm = None birl = bayesian_irl.BayesianIRL(mdp_env_A, beta, step_stdev, debug=False, mcmc_norm=sample_norm) map_w, map_u, r_chain, u_chain = birl.sample_posterior(demonstrations, num_samples, True) import matplotlib.pyplot as plt for w in range(len(r_chain[0])): plt.plot(r_chain[:, w], label="feature {}".format(w)) plt.legend() plt.show() print("MAP") print("map_weights", map_w) map_r = np.dot(mdp_env_A.state_features, map_w)
print("demo", d, demo) traj_demonstrations.append(demo) for s_a in demo: demo_set.add(s_a) demonstrations = list(demo_set) print("demonstration") print(demonstrations) state_feature_list = [tuple(fs) for fs in mdp_env.state_features] # pg.get_policy_string_from_trajectory(traj_demonstrations[0], state_feature_list, mdp_env, filename="./figs/lava_corridor/demo_lava_world.png") #Now let's run Bayesian IRL on this demo in this mdp with a placeholder feature to see what happens. birl = bayesian_irl.BayesianIRL(mdp_env, beta, step_stdev, debug=False, mcmc_norm=mcmc_norm, likelihood=likelihood) # r_hyps = [#[0,-1], # [-1,0], # #[-0.1232111, -0.8767889], # #[-0.25691869, -0.74308131], # [-0.05609254, -0.94390746], # [-0.57066167, 0.42933833], # [-0.55, 0.45], # [-.1,-0.9] # ] # for i,r in enumerate(r_hyps): # print() # print("TRIAL ", i) # #generate random rewards
#demonstrations = [(1,0), (0,0)] # mdp_env = mdp_worlds.machine_teaching_toy_featurized() # demonstrations = [(2,3),(5,0),(4,0),(3,2)] mdp_env = mdp_worlds.lava_ambiguous_aaai18() u_sa = mdp.solve_mdp_lp(mdp_env) #generate demo from state 5 to terminal demonstrations = utils.rollout_from_usa(5, 10, u_sa, mdp_env) print(demonstrations) traj_demonstrations = [demonstrations] beta = 10.0 step_stdev = 0.1 birl = bayesian_irl.BayesianIRL(mdp_env, beta, step_stdev, debug=False) num_samples = 200 burn = 50 skip = 2 map_w, map_u, r_chain, u_chain = birl.sample_posterior( demonstrations, num_samples) print("map_weights", map_w) map_r = np.dot(mdp_env.state_features, map_w) utils.print_as_grid(map_r, mdp_env) #print("Map policy") #utils.print_policy_from_occupancies(map_u, mdp_env) # print("chain") # for r in r_chain: # print(r)
print(demonstrations) state_feature_list = [tuple(fs) for fs in train_mdp.state_features] #Now let's run Bayesian IRL on this demo in this mdp with a placeholder feature to see what happens. beta = 10.0 step_stdev = 0.05 num_samples = 1000 mcmc_norm = "inf" likelihood = "birl" birl = bayesian_irl.BayesianIRL(train_mdp, beta, step_stdev, debug=False, mcmc_norm=mcmc_norm, likelihood=likelihood, prior="non-pos") map_w, map_u, r_chain, u_chain = birl.sample_posterior(demonstrations, num_samples, True) print(train_mdp.feature_weights) burn = 200 skip = 10 r_chain_burned = r_chain[burn::skip] u_expert = utils.u_sa_from_demos(traj_demonstrations, train_mdp) expert_returns = np.sort(np.dot(r_chain_burned, u_expert))