def train(env, episode_count=1000): state_now = env.reset() agent = vg.ValueIterationAgent(env) ## values, _ = agent.value_iteration(error=0.01, deterministic=False) agent.load() # plot test! plot(env, agent) # generate demonstrations trajs = generate_demonstrations(env, agent, n_trajs=100, len_traj=100, rand_start=True) # feature selection feat_map = feature_basis(env) # run irl T = agent.get_transition_mat() T = np.swapaxes(T,1,2) rewards = maxent_irl(np.array(feat_map), T, gamma=0.95, trajs=trajs, lr=0.01, n_iters=20) print rewards # value iteration agent.save()
def main(): N_STATES = H * W N_ACTIONS = 5 # init the gridworld # rmap_gt is the ground truth for rewards rmap_gt = np.zeros([H, W]) rmap_gt[H - 1, W - 1] = R_MAX # rmap_gt[H-1, 0] = R_MAX gw = gridworld.GridWorld(rmap_gt, {}, 1 - ACT_RAND) rewards_gt = np.reshape(rmap_gt, H * W, order='F') P_a = gw.get_transition_mat() values_gt, policy_gt = value_iteration.value_iteration(P_a, rewards_gt, GAMMA, error=0.01, deterministic=True) # use identity matrix as feature feat_map = np.eye(N_STATES) print(feat_map.shape) # other two features. due to the linear nature, # the following two features might not work as well as the identity. # feat_map = feature_basis(gw) # feat_map = feature_coord(gw) np.random.seed(1) trajs = generate_demonstrations(gw, policy_gt, n_trajs=N_TRAJS, len_traj=L_TRAJ, rand_start=RAND_START) rewards = maxent_irl(feat_map, P_a, GAMMA, trajs, LEARNING_RATE, N_ITERS) values, _ = value_iteration.value_iteration(P_a, rewards, GAMMA, error=0.01, deterministic=True) # plots plt.figure(figsize=(20, 4)) plt.subplot(1, 4, 1) img_utils.heatmap2d(rmap_gt, 'Rewards Map - Ground Truth', block=False) plt.subplot(1, 4, 2) img_utils.heatmap2d(np.reshape(values_gt, (H, W), order='F'), 'Value Map - Ground Truth', block=False) plt.subplot(1, 4, 3) img_utils.heatmap2d(np.reshape(rewards, (H, W), order='F'), 'Reward Map - Recovered', block=False) plt.subplot(1, 4, 4) img_utils.heatmap2d(np.reshape(values, (H, W), order='F'), 'Value Map - Recovered', block=False) plt.show()
def main(): for seed in range(1): N_STATES = H * W # init the gridworld # rmap_gt is the ground truth for rewards rmap_gt = np.zeros([H, W]) #goal coordinates rmap_gt[H - 1, W - 1] = R_MAX gw = gridworld.GridWorld(rmap_gt, {}, 1 - ACT_RAND) rewards_gt = np.reshape(rmap_gt, H * W, order='F') P_a = gw.get_transition_mat() values_gt, policy_gt = value_iteration.value_iteration( P_a, rewards_gt, GAMMA, error=0.01, deterministic=True) # use identity matrix as feature feat_map = np.eye(N_STATES) # other two features. due to the linear nature, # the following two features might not work as well as the identity. # feat_map = feature_basis(gw) # feat_map = feature_coord(gw) np.random.seed(0) #trajs = generate_demonstrations(gw, policy_gt, n_trajs=N_TRAJS, len_traj=L_TRAJ, rand_start=RAND_START) trajs = mod.exp1_case2() rewards = maxent_irl(gw, feat_map, P_a, GAMMA, trajs, LEARNING_RATE, N_ITERS) #np.savetxt('results/rewards.txt', rewards) #values, policy = value_iteration.value_iteration(P_a, rewards, GAMMA, error=0.01, deterministic=True) # plots plt.figure(figsize=(20, 20)) img_utils.heatmap2d(np.reshape(rewards, (H, W), order='F'), 'Reward Map', block=False) plt.plot() #now = datetime.datetime.now() #figname = "results/rewards_{0:%m%d%H%M}".format(now) + ".png" figname = "results/rewards_seed{0}".format(seed) + ".png" plt.savefig(figname)
def test_irl_algorithms(gw, P_a, rmap_gt, policy_gt, trajs, feat_map): print('LP IRL training ..') rewards_lpirl = lp_irl(P_a, policy_gt, gamma=0.3, l1=10, R_max=R_MAX) print('Max Ent IRL training ..') rewards_maxent = maxent_irl(feat_map, P_a, GAMMA, trajs, LEARNING_RATE * 2, N_ITERS * 2) print('Deep Max Ent IRL training ..') rewards = deep_maxent_irl(feat_map, P_a, GAMMA, trajs, LEARNING_RATE, N_ITERS) values, _ = value_iteration.value_iteration(P_a, rewards, GAMMA, error=0.01, deterministic=True) # plots plt.figure(figsize=(20, 8)) plt.subplot(1, 4, 1) img_utils.heatmap2d(to_plot(rmap_gt), 'Rewards Map - Ground Truth', block=False, text=False) plt.subplot(1, 4, 2) img_utils.heatmap2d(to_plot(rewards_lpirl), 'Reward Map - LP', block=False, text=False) plt.subplot(1, 4, 3) img_utils.heatmap2d(to_plot(rewards_maxent), 'Reward Map - Maxent', block=False, text=False) plt.subplot(1, 4, 4) img_utils.heatmap2d(to_plot(rewards), 'Reward Map - Deep Maxent', block=False, text=False) plt.show()
def main(): N_STATES = H * W N_ACTIONS = 5 rmap_gt = np.zeros([H, W]) rmap_gt[H - 2, W - 2] = R_MAX rmap_gt[1, 1] = R_MAX # rmap_gt[H/2, W/2] = R_MAX gw = gridworld.GridWorld(rmap_gt, {}, 1 - ACT_RAND) rewards_gt = np.reshape(rmap_gt, H * W, order='F') P_a = gw.get_transition_mat() values_gt, policy_gt = value_iteration.value_iteration(P_a, rewards_gt, GAMMA, error=0.01, deterministic=True) rewards_gt = normalize(values_gt) gw = gridworld.GridWorld(np.reshape(rewards_gt, (H, W), order='F'), {}, 1 - ACT_RAND) P_a = gw.get_transition_mat() values_gt, policy_gt = value_iteration.value_iteration(P_a, rewards_gt, GAMMA, error=0.01, deterministic=True) # use identity matrix as feature feat_map = np.eye(N_STATES) trajs = generate_demonstrations(gw, policy_gt, n_trajs=N_TRAJS, len_traj=L_TRAJ, rand_start=RAND_START) print('LP IRL training ..') rewards_lpirl = lp_irl(P_a, policy_gt, gamma=0.3, l1=10, R_max=R_MAX) print('Max Ent IRL training ..') rewards_maxent = maxent_irl(feat_map, P_a, GAMMA, trajs, LEARNING_RATE * 2, N_ITERS * 2) print('Deep Max Ent IRL training ..') rewards_deep = deep_maxent_irl(feat_map, P_a, GAMMA, trajs, LEARNING_RATE, N_ITERS) print('Deep Siamese Max Ent IRL training ..') rewards = deep_siamese_maxent_irl(feat_map, P_a, GAMMA, trajs, LEARNING_RATE, N_ITERS) # plots plt.figure(figsize=(20, 5)) plt.subplot(1, 5, 1) img_utils.heatmap2d(np.reshape(rewards_gt, (H, W), order='F'), 'Rewards Map - Ground Truth', block=False) plt.subplot(1, 5, 2) img_utils.heatmap2d(np.reshape(rewards_lpirl, (H, W), order='F'), 'Reward Map - LP', block=False) plt.subplot(1, 5, 3) img_utils.heatmap2d(np.reshape(rewards_maxent, (H, W), order='F'), 'Reward Map - Maxent', block=False) plt.subplot(1, 5, 4) img_utils.heatmap2d(np.reshape(rewards_deep, (H, W), order='F'), 'Reward Map - Deep Maxent', block=False) plt.subplot(1, 5, 5) img_utils.heatmap2d(np.reshape(rewards, (H, W), order='F'), 'Reward Map - Deep Policy Maxent', block=False) plt.show()
def main(grid_size, discount, n_objects, n_colours, n_trajectories, epochs, learning_rate, structure): wind = 0.3 trajectory_length = 8 l1 = l2 = 0 ow = objectworld.Objectworld(grid_size, n_objects, n_colours, wind, discount) print(ow.objects.keys()) rewards_gt = np.array([ow.reward(s) for s in range(ow.n_states)]) policy_gt = find_policy(ow.n_states, ow.n_actions, ow.transition_probability, rewards_gt, ow.discount, stochastic=False) trajs = ow.generate_trajectories(N_TRAJS, L_TRAJ, lambda s: policy_gt[s]) feat_map = ow.feature_matrix(ow.objects, discrete=False) rewards_inv = np.array( [ow.inverse_reward(s_inv) for s_inv in range(ow.n_states)]) policy_inv = find_inverted_policy(ow.n_states, ow.n_actions, ow.transition_probability, rewards_inv, ow.discount, stochastic=False) trajs_inv = ow.generate_inverse_trajectories( N_TRAJS, L_TRAJ, lambda s_inv: policy_inv[s_inv]) feat_map_inv = ow.inv_feature_matrix(ow.inverted_objects, discrete=False) print('LP IRL training ..') rewards_lpirl = lp_irl(ow.transition_probability, policy_gt, gamma=0.3, l1=10, R_max=R_MAX) print('Max Ent IRL training ..') rewards_maxent = maxent_irl(feat_map, ow.transition_probability, GAMMA, trajs, LEARNING_RATE * 2, N_ITERS * 2) print('Deep Max Ent IRL training ..') rewards_deep = deep_maxent_irl(feat_map, ow.transition_probability, GAMMA, trajs, LEARNING_RATE, N_ITERS) print('Deep Siamese Max Ent IRL training ..') rewards = deep_siamese_maxent_irl(feat_map, feat_map_inv, ow.transition_probability, GAMMA, trajs, trajs_inv, LEARNING_RATE, N_ITERS) # plots plt.figure(figsize=(20, 5)) plt.subplot(1, 5, 1) img_utils.heatmap2d(np.reshape(rewards_gt, (H, W), order='F'), 'Rewards Map - Ground Truth', block=False) plt.subplot(1, 5, 2) img_utils.heatmap2d(np.reshape(rewards_lpirl, (H, W), order='F'), 'Reward Map - LP', block=False) plt.subplot(1, 5, 3) img_utils.heatmap2d(np.reshape(rewards_maxent, (H, W), order='F'), 'Reward Map - Maxent', block=False) plt.subplot(1, 5, 4) img_utils.heatmap2d(np.reshape(rewards_deep, (H, W), order='F'), 'Reward Map - Deep Maxent', block=False) plt.subplot(1, 5, 5) img_utils.heatmap2d(np.reshape(rewards, (H, W), order='F'), 'Reward Map - Deep Siamese Maxent', block=False) plt.show()
def main(): N_STATES = H * W N_ACTIONS = 4 rmap_gt = set_rewards() gw = gridworld.GridWorld(rmap_gt, {(H - 1, W - 1)}, 1 - ACT_RAND) rewards_gt = np.reshape(rmap_gt, H * W, order='F') P_a = gw.get_transition_mat() values_gt, policy_gt = value_iteration.value_iteration(P_a, rewards_gt, GAMMA, error=0.01, deterministic=True) path_gt = gw.display_path_grid(policy_gt) rmap_gt = gw.get_reward_mat() #temp plt.figure(figsize=(20, 4)) plt.subplot(1, 3, 1) img_utils.heatmap2d(rmap_gt, 'Rewards Map - Ground Truth', block=False) plt.subplot(1, 3, 2) img_utils.heatmap2d(np.reshape(values_gt, (H, W), order='F'), 'Value Map - Ground Truth', block=False) plt.subplot(1, 3, 3) img_utils.heatmap2d(np.reshape(path_gt, (H, W), order='F'), 'Path Map - Ground Truth', block=False) plt.show() sys.exit() # feat_map = np.eye(N_STATES) # feat_map = feature_basis(gw) # feat_map = feature_coord(gw) feat_map = feature_histogram(gw) np.random.seed(1) trajs = generate_demonstrations(gw, policy_gt, n_trajs=N_TRAJS, len_traj=L_TRAJ, rand_start=RAND_START) rewards = maxent_irl(feat_map, P_a, GAMMA, trajs, LEARNING_RATE, N_ITERS) values, policy = value_iteration.value_iteration(P_a, rewards, GAMMA, error=0.01, deterministic=True) path = gw.display_path_grid(policy) # plots plt.figure(figsize=(20, 4)) plt.subplot(2, 4, 1) img_utils.heatmap2d(rmap_gt, 'Rewards Map - Ground Truth', block=False) plt.subplot(2, 4, 2) img_utils.heatmap2d(np.reshape(values_gt, (H, W), order='F'), 'Value Map - Ground Truth', block=False) plt.subplot(2, 4, 3) img_utils.heatmap2d(np.reshape(rewards, (H, W), order='F'), 'Reward Map - Recovered', block=False) plt.subplot(2, 4, 4) img_utils.heatmap2d(np.reshape(values, (H, W), order='F'), 'Value Map - Recovered', block=False) plt.subplot(2, 4, 5) img_utils.heatmap2d(np.reshape(path_gt, (H, W), order='F'), 'Path Map - Ground Truth', block=False) plt.subplot(2, 4, 7) img_utils.heatmap2d(np.reshape(path, (H, W), order='F'), 'Path Map - Recovered', block=False) plt.show()
def main(): N_STATES = H * W N_ACTIONS = 5 start_coordinates = (pixel_locations[0]['location-lat'][0], pixel_locations[0]['location-long'][0]) end_coordinates = ( pixel_locations[0]['location-lat'][len(pixel_locations[0].index) - 1], pixel_locations[0]['location-long'][len(pixel_locations[0].index) - 1]) rmap_gt = np.zeros([W, H]) rmap_gt[int(start_coordinates[0]), int(start_coordinates[1])] = R_MAX rmap_gt[int(end_coordinates[0]), int(end_coordinates[1])] = R_MAX # rmap_gt[H/2, W/2] = R_MAX gw = gridworld.GridWorld(rmap_gt, {}, 1 - ACT_RAND) rewards_gt = np.reshape(rmap_gt, H * W, order='F') P_a = gw.get_transition_mat() values_gt, policy_gt = value_iteration.value_iteration(P_a, rewards_gt, GAMMA, error=0.01, deterministic=True) rewards_gt = normalize(values_gt) gw = gridworld.GridWorld(np.reshape(rewards_gt, (H, W), order='F'), {}, 1 - ACT_RAND) P_a = gw.get_transition_mat() values_gt, policy_gt = value_iteration.value_iteration(P_a, rewards_gt, GAMMA, error=0.01, deterministic=True) # use identity matrix as feature # feat_map = np.eye(N_STATES) coast_map = np.load('Feature Maps/small_maps/coast.npy') coast_map = np.reshape(coast_map, (600, 1)) forest_map = np.load('Feature Maps/small_maps/forest.npy') forest_map = np.reshape(coast_map, (600, 1)) land_map = np.load('Feature Maps/small_maps/land.npy') land_map = np.reshape(coast_map, (600, 1)) feat_map = np.hstack((coast_map, forest_map, land_map)) # populate trajectories trajs = [] terminal_state = end_coordinates for x in range(len(pixel_locations)): trajs.append([]) for i in range(len(pixel_locations[x]) - 1): loc = pixel_locations[x].iloc[i] next_loc = pixel_locations[x].iloc[i + 1] action = get_action(loc, next_loc) reward = rmap_gt[int(next_loc[0]), int(next_loc[1])] is_done = np.array_equal(next_loc, terminal_state) trajs[x].append( Step(cur_state=int(gw.pos2idx(loc)), action=action, next_state=int(gw.pos2idx(next_loc)), reward=reward, done=is_done)) print 'LP IRL training ..' rewards_lpirl = lp_irl(P_a, policy_gt, gamma=0.3, l1=100, R_max=R_MAX) print 'Max Ent IRL training ..' rewards_maxent = maxent_irl(feat_map, P_a, GAMMA, trajs, LEARNING_RATE, N_ITERS) # print 'Deep Max Ent IRL training ..' # rewards = deep_maxent_irl(feat_map, P_a, GAMMA, trajs, LEARNING_RATE, 10) # plots fig = plt.figure() plt.subplot(1, 2, 1) img_utils.heatmap2d(np.reshape(rewards_gt, (H, W), order='F'), 'Rewards Map - Ground Truth', block=False) fig.savefig('GroundTruth.png') plt.subplot(1, 1, 1) img_utils.heatmap2d(np.reshape(rewards_lpirl, (H, W), order='F'), 'Reward Map - LP', block=False) fig.savefig('LP.png') plt.subplot(1, 1, 1) img_utils.heatmap2d(np.reshape(rewards_maxent, (H, W), order='F'), 'Reward Map - Maxent', block=False) fig.savefig('MaxEnt.png')
def main(): N_STATES = H * W N_ACTIONS = 5 """while True: print "BAD_STATE入力" bad = raw_input('>> ') if bad == 'ok': break Bad_states.append(bad) """ #print Bad_states # init the gridworld # rmap_gt is the ground truth for rewards rmap_gt = np.zeros([H, W]) rmap_gt[H - 1, W - 1] = R_MAX # rmap_gt[H-1, 0] = R_MAX gw = gridworld.GridWorld(rmap_gt, {}, 1 - ACT_RAND) rewards_gt = np.reshape(rmap_gt, H * W, order='F') P_a = gw.get_transition_mat() values_gt, policy_gt = value_iteration.value_iteration(P_a, rewards_gt, GAMMA, error=0.01, deterministic=True) # use identity matrix as feature feat_map = np.eye(N_STATES) # other two features. due to the linear nature, # the following two features might not work as well as the identity. # feat_map = feature_basis(gw) # feat_map = feature_coord(gw) np.random.seed(1) trajs = generate_demonstrations(gw, policy_gt, n_trajs=N_TRAJS, len_traj=L_TRAJ, rand_start=RAND_START) rewards = maxent_irl(feat_map, P_a, GAMMA, trajs, LEARNING_RATE, N_ITERS) #new_rewards = reward_decrease(rewards, R_GAMMA, BAD_X, BAD_Y) np.savetxt('results/rewards.txt', rewards) #print rewards values, policy = value_iteration.value_iteration(P_a, rewards, GAMMA, error=0.01, deterministic=True) #print policy # plots plt.figure(figsize=(20, 20)) img_utils.heatmap2d(np.reshape(rewards, (H, W), order='F'), 'Reward Map', block=False) plt.plot() plt.figure(figsize=(20, 20)) img_utils.heatmap2d(np.reshape(values, (H, W), order='F'), 'Policy Map', block=False) plt.plot() plt.show()
C.set_terminal_idx() if rank_features: C.convert_to_rankings() # demonstrations canonical_user_demo = [canonical_demo] canonical_trajectories = get_trajectories(C.states, canonical_user_demo, C.transition) print("Training ...") # using abstract features abstract_features = np.array([C.get_features(state) for state in C.states]) norm_abstract_features = abstract_features / np.linalg.norm(abstract_features, axis=0) canonical_rewards_abstract, canonical_weights_abstract = maxent_irl( C, norm_abstract_features, canonical_trajectories, optim, init) print("Weights have been learned for the canonical task! Fingers X-ed.") print("Weights -", canonical_weights_abstract) # scale weights if scale_weights: canonical_weights_abstract /= max(canonical_weights_abstract) # ----------------------------------------- Testing: Predict complex -------------------------------------------- # sample_complex_demo = [1, 3, 5, 0, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6, 7] complex_survey_actions = [0, 4, 1, 5, 6, 7, 2, 3] action_counts = [1, 1, 4, 1, 4, 1, 4, 1] preferred_order = [ df[q][idx] for q in
def main(): N_STATES = H * W N_ACTIONS = 5 # init the gridworld # rmap_gt is the ground truth for rewards rmap_gt = np.zeros([H, W]) rmap_gt[H - 1, W - 1] = R_MAX # rmap_gt[H-1, 0] = R_MAX gw = gridworld.GridWorld(rmap_gt, {}, 1 - ACT_RAND) rewards_gt = np.reshape(rmap_gt, H * W, order='F') # P_a = gw.get_transition_mat( ) #this is the transitin probablities of the matrix 5 action what is the probability of moving from state s1 to s2 give the action #getting the transition probabilities in my case is just impossible ... values_gt, policy_gt = value_iteration.value_iteration( P_a, rewards_gt, GAMMA, error=0.01, deterministic=True ) #value iteration and policy acoding to the currrent rewards 0 # use identity matrix as feature feat_map = np.eye(N_STATES) #features as one hot encoding # other two features. due to the linear nature, # the following two features might not work as well as the identity. # feat_map = feature_basis(gw) # feat_map = feature_coord(gw) np.random.seed(1) trajs = generate_demonstrations( gw, policy_gt, n_trajs=N_TRAJS, len_traj=L_TRAJ, rand_start=RAND_START) #this is the trajectories rewards = maxent_irl( feat_map, P_a, GAMMA, trajs, LEARNING_RATE, N_ITERS ) #need to input the feature map , transition priobalibliteis og the world pdb.set_trace() values, _ = value_iteration.value_iteration(P_a, rewards, GAMMA, error=0.01, deterministic=True) # plots plt.figure(figsize=(20, 4)) plt.subplot(1, 4, 1) img_utils.heatmap2d(rmap_gt, 'Rewards Map - Ground Truth', block=False) plt.subplot(1, 4, 2) img_utils.heatmap2d(np.reshape(values_gt, (H, W), order='F'), 'Value Map - Ground Truth', block=False) plt.subplot(1, 4, 3) img_utils.heatmap2d(np.reshape(rewards, (H, W), order='F'), 'Reward Map - Recovered', block=False) plt.subplot(1, 4, 4) img_utils.heatmap2d(np.reshape(values, (H, W), order='F'), 'Value Map - Recovered', block=False) plt.show()
def main(): # named tuple to record demonstrations Step = namedtuple('Step','cur_state action next_state reward done') # argument parser for command line arguments parser = argparse.ArgumentParser(description=None) parser.add_argument('-wid', '--width', default=5, type=int, help='width of the gridworld') parser.add_argument('-hei', '--height', default=5, type=int, help='height of the gridworld') parser.add_argument('-lr', '--learning_rate', default=0.01, type=float, help='learning rate') parser.add_argument('-l', '--l_traj', default=20, type=int, help='length of expert trajectory') parser.add_argument('--no-rand_start', dest='rand_start', action='store_false', help='when sampling trajectories, fix start positions') parser.add_argument('--rand_start', dest='rand_start', action='store_true', help='when sampling trajectories, randomly pick start positions') parser.add_argument('--approx', dest='approx', action='store_true', help='flag to perform approximation of psa') parser.add_argument('-g', '--gamma', default=0.9, type=float, help='discount factor') parser.add_argument('-n', '--n_iters', default=20, type=int, help='number of iterations') parser.add_argument('-t', '--n_trajs', default=100, type=int, help='number of expert trajectories') parser.add_argument('-a', '--act_random', default=0.3, type=float, help='probability of acting randomly') # set default value for rand_start variable parser.set_defaults(rand_start=False) # parse and print arguments args = parser.parse_args() # arguments for environment and irl algorithm r_max = 1 gamma = args.gamma width = args.width height = args.height l_traj = args.l_traj approx = args.approx n_iters = args.n_iters n_trajs = args.n_trajs act_rand = args.act_random rand_start = args.rand_start learning_rate = args.learning_rate # variables for number of actions and states n_actions = 5 n_states = height * width # initialize the gridworld # rmap_gt is the ground truth for rewards rmap_gt = np.zeros([height, width]) rmap_gt[0, width-1] = r_max rmap_gt[height-1, 0] = r_max rmap_gt[height-1, width-1] = r_max # create grid world instance gw = gridworld.GridWorld(rmap_gt, {}, 1-act_rand) # get true rewards, state transition dynamics rewards_gt = np.reshape(rmap_gt, height*width, order='F') P_a_true = gw.get_transition_mat() trajs = generate_random(gw, n_actions, n_trajs=n_trajs, len_traj=l_traj, rand_start=rand_start) # get approximation of state transition dynamics P_a_approx = np.zeros((n_states, n_states, n_actions)) for traj in trajs: for t in range(len(traj)): P_a_approx[traj[t].cur_state, traj[t].next_state, traj[t].action] += 1 for s in range(n_states): for a in range(n_actions): if np.sum(P_a_approx[s,:,a]) != 0: P_a_approx[s,:,a] /= np.sum(P_a_approx[s,:,a]) if approx: P_a = P_a_approx else: P_a = P_a_true # get true value function and policy from reward map values_gt, policy_gt = value_iteration.value_iteration(P_a, rewards_gt, gamma, error=0.01, deterministic=True) # use identity matrix as feature feat_map = np.eye(n_states) # other two features. due to the linear nature, # the following two features might not work as well as the identity. # feat_map = feature_basis(gw) # feat_map = feature_coord(gw) trajs = generate_demonstrations(gw, policy_gt, n_trajs=n_trajs, len_traj=l_traj, rand_start=rand_start) # perform inverse reinforcement learning to get reward function rewards = maxent_irl(feat_map, P_a, gamma, trajs, learning_rate, n_iters) values, _ = value_iteration.value_iteration(P_a, rewards, gamma, error=0.01, deterministic=True) # plots plt.figure(figsize=(20,4)) plt.subplot(2, 2, 1) img_utils.heatmap2d(rmap_gt, 'Rewards Map - Ground Truth', block=False) plt.subplot(2, 2, 2) img_utils.heatmap2d(np.reshape(values_gt, (height,width), order='F'), 'Value Map - Ground Truth', block=False) plt.subplot(2, 2, 3) img_utils.heatmap2d(np.reshape(rewards, (height,width), order='F'), 'Reward Map - Recovered', block=False) plt.subplot(2, 2, 4) img_utils.heatmap2d(np.reshape(values, (height,width), order='F'), 'Value Map - Recovered', block=False) plt.show() # plots for state transition dynamics plt.figure(figsize=(10,4)) plt.subplot(2, 1, 1) img_utils.heatmap2d(np.reshape(P_a_true[10,:,2], (height,width), order='F'), 'True Dist', block=False) plt.subplot(2, 1, 2) img_utils.heatmap2d(np.reshape(P_a_approx[10,:,2], (height,width), order='F'), 'Approx Dist', block=False) plt.show()